From 5f57cb66393bb2dc4328acc3398b298fc6c541bf Mon Sep 17 00:00:00 2001 From: Arseny Sher Date: Thu, 16 Nov 2017 12:49:46 +0300 Subject: [PATCH 001/475] commit=True by default in PostgresNode.execute() and document it. Old default could lead to astonishing "where are my damned rows?" emotions. --- README.md | 2 +- testgres/testgres.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index fc285cf6..3e1034e9 100644 --- a/README.md +++ b/README.md @@ -98,7 +98,7 @@ Finally our temporary cluster is able to process queries. There are four ways to * `node.psql(database, query)` - runs query via `psql` command and returns tuple `(error code, stdout, stderr)` * `node.safe_psql(database, query)` - same as `psql()` except that it returns only `stdout`. If an error occures during the execution, an exception will be thrown. -* `node.execute(database, query)` - connects to postgresql server using `psycopg2` or `pg8000` library (depends on which is installed in your system) and returns two-dimensional array with data. +* `node.execute(database, query, username=None, commit=True)` - connects to postgresql server using `psycopg2` or `pg8000` library (depends on which is installed in your system) and returns two-dimensional array with data. * `node.connect(database='postgres')` - returns connection wrapper (`NodeConnection`) capable of running several queries within a single transaction. The last one is the most powerful: you can use `begin(isolation_level)`, `commit()` and `rollback()`: diff --git a/testgres/testgres.py b/testgres/testgres.py index 14c135bd..4a25c80e 100644 --- a/testgres/testgres.py +++ b/testgres/testgres.py @@ -962,7 +962,7 @@ def poll_query_until(self, raise TimeoutException('Query timeout') - def execute(self, dbname, query, username=None, commit=False): + def execute(self, dbname, query, username=None, commit=True): """ Execute a query and return all rows as list. From 49fab00ecd54d359feef9f782b78fdebf8aa06b4 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Thu, 16 Nov 2017 13:29:45 +0300 Subject: [PATCH 002/475] reformat code using yapf --- testgres/testgres.py | 1 - 1 file changed, 1 deletion(-) diff --git a/testgres/testgres.py b/testgres/testgres.py index 4a25c80e..33606838 100644 --- a/testgres/testgres.py +++ b/testgres/testgres.py @@ -44,7 +44,6 @@ from enum import Enum from distutils.version import LooseVersion - # Try to use psycopg2 by default. If psycopg2 isn't available then use # pg8000 which is slower but much more portable because uses only # pure-Python code From 18d6d7963b200ae46d408a7e7413cc37b35a14ed Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Thu, 16 Nov 2017 13:30:50 +0300 Subject: [PATCH 003/475] bump version to 1.3.4 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 632ea821..180efa56 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ setup( name='testgres', packages=['testgres'], - version='1.3.3', + version='1.3.4', description='Testing utility for PostgreSQL and its extensions', license='PostgreSQL', author='Ildar Musin', From 3cbb3a56393bb477440e288162351c8baf9829a0 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Thu, 16 Nov 2017 14:51:51 +0300 Subject: [PATCH 004/475] Update README.md --- README.md | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 3e1034e9..c7091981 100644 --- a/README.md +++ b/README.md @@ -94,12 +94,14 @@ Now we are ready to start: node.start() ``` -Finally our temporary cluster is able to process queries. There are four ways to run them: - -* `node.psql(database, query)` - runs query via `psql` command and returns tuple `(error code, stdout, stderr)` -* `node.safe_psql(database, query)` - same as `psql()` except that it returns only `stdout`. If an error occures during the execution, an exception will be thrown. -* `node.execute(database, query, username=None, commit=True)` - connects to postgresql server using `psycopg2` or `pg8000` library (depends on which is installed in your system) and returns two-dimensional array with data. -* `node.connect(database='postgres')` - returns connection wrapper (`NodeConnection`) capable of running several queries within a single transaction. +Finally, our temporary cluster is able to process queries. There are four ways to run them: + +| Command | Description | +|-------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------| +| `node.psql(database, query)` | Runs query via `psql` command and returns tuple `(error code, stdout, stderr)`. | +| `node.safe_psql(database, query)` | Same as `psql()` except that it returns only `stdout`. If an error occures during the execution, an exception will be thrown. | +| `node.execute(database, query, username=None, commit=True)` | Connects to PostgreSQL using `psycopg2` or `pg8000` (depends on which one is installed in your system) and returns two-dimensional array with data. | +| `node.connect(database='postgres')` | Returns connection wrapper (`NodeConnection`) capable of running several queries within a single transaction. | The last one is the most powerful: you can use `begin(isolation_level)`, `commit()` and `rollback()`: ```python From 4f00dbb09b639e528f270929367495db878cddda Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Fri, 17 Nov 2017 15:07:00 +0300 Subject: [PATCH 005/475] improved setup.py, new enum for transaction isolation levels --- setup.py | 16 ++++++++-- testgres/testgres.py | 73 ++++++++++++++++++++++++++++---------------- tests/test_simple.py | 25 +++++++++++++++ 3 files changed, 85 insertions(+), 29 deletions(-) diff --git a/setup.py b/setup.py index 180efa56..c9d8a02d 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,16 @@ -from distutils.core import setup +import sys + +try: + from setuptools import setup +except ImportError: + from distutils.core import setup + +# Basic dependencies +install_requires = ["pg8000", "six", "port-for"] + +# Add compatibility enum class +if sys.version_info < (3, 4): + install_requires.append("enum34") setup( name='testgres', @@ -11,4 +23,4 @@ url='https://github.com/postgrespro/testgres', keywords=['testing', 'postgresql'], classifiers=[], - install_requires=["pg8000", "six", "port-for", "enum34"]) + install_requires=install_requires) diff --git a/testgres/testgres.py b/testgres/testgres.py index 33606838..902a401d 100644 --- a/testgres/testgres.py +++ b/testgres/testgres.py @@ -171,19 +171,12 @@ def stopped(self): return self.stop_event.isSet() -def log_watch(node_name, pg_logname): +class IsolationLevel(Enum): """ - Starts thread for node that redirects - postgresql logs to python logging system + Transaction isolation level for NodeConnection """ - reader = TestgresLogger(node_name, open(pg_logname, 'r')) - reader.start() - - global util_threads - util_threads.append(reader) - - return reader + ReadUncommitted, ReadCommitted, RepeatableRead, Serializable = range(4) class NodeConnection(object): @@ -218,7 +211,7 @@ def __enter__(self): def __exit__(self, type, value, traceback): self.close() - def begin(self, isolation_level=0): + def begin(self, isolation_level=IsolationLevel.ReadCommitted): # yapf: disable levels = [ 'read uncommitted', @@ -227,40 +220,51 @@ def begin(self, isolation_level=0): 'serializable' ] - # Check if level is int [0..3] - if (isinstance(isolation_level, int) and - isolation_level in range(0, 4)): + # Check if level is an IsolationLevel + if (isinstance(isolation_level, IsolationLevel)): - # Replace index with isolation level type - isolation_level = levels[isolation_level] + # Get index of isolation level + level_idx = isolation_level.value + assert(level_idx in range(4)) - # Or it might be a string - elif (isinstance(isolation_level, six.text_type) and - isolation_level.lower() in levels): + # Replace isolation level with its name + isolation_level = levels[level_idx] - # Nothing to do here - pass - - # Something is wrong, emit exception else: - raise QueryException( - 'Invalid isolation level "{}"'.format(isolation_level)) + # Get name of isolation level + level_str = str(isolation_level).lower() + + # Validate level string + if level_str not in levels: + error = 'Invalid isolation level "{}"' + raise QueryException(error.format(level_str)) + + # Replace isolation level with its name + isolation_level = level_str - self.cursor.execute( - 'SET TRANSACTION ISOLATION LEVEL {}'.format(isolation_level)) + # Set isolation level + cmd = 'SET TRANSACTION ISOLATION LEVEL {}' + self.cursor.execute(cmd.format(isolation_level)) + + return self def commit(self): self.connection.commit() + return self + def rollback(self): self.connection.rollback() + return self + def execute(self, query, *args): self.cursor.execute(query, args) try: res = self.cursor.fetchall() + # pg8000 might return tuples if isinstance(res, tuple): res = [tuple(t) for t in res] @@ -1311,3 +1315,18 @@ def configure_testgres(**options): for key, option in options.items(): setattr(TestgresConfig, key, option) + + +def log_watch(node_name, pg_logname): + """ + Start thread for node that redirects + PostgreSQL logs to python logging system. + """ + + reader = TestgresLogger(node_name, open(pg_logname, 'r')) + reader.start() + + global util_threads + util_threads.append(reader) + + return reader diff --git a/tests/test_simple.py b/tests/test_simple.py index 93864174..ebfb06d3 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -23,6 +23,7 @@ from testgres import get_new_node, get_pg_config, configure_testgres from testgres import bound_ports from testgres import NodeStatus +from testgres import IsolationLevel class SimpleTest(unittest.TestCase): @@ -442,6 +443,30 @@ def test_configure(self): # return to the base state configure_testgres(cache_initdb=True, cache_pg_config=True) + def test_isolation_levels(self): + with get_new_node('node').init().start() as node: + with node.connect() as con: + # string levels + con.begin('Read Uncommitted').commit() + con.begin('Read Committed').commit() + con.begin('Repeatable Read').commit() + con.begin('Serializable').commit() + + # enum levels + con.begin(IsolationLevel.ReadUncommitted).commit() + con.begin(IsolationLevel.ReadCommitted).commit() + con.begin(IsolationLevel.RepeatableRead).commit() + con.begin(IsolationLevel.Serializable).commit() + + # check wrong level + got_exception = False + try: + con.begin('Garbage').commit() + except QueryException: + got_exception = True + + self.assertTrue(got_exception) + if __name__ == '__main__': unittest.main() From 650a7dfb23d4856a84a730c7939974c715880eea Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Thu, 23 Nov 2017 17:42:32 +0300 Subject: [PATCH 006/475] refactoring, fix tests, fix moron comments --- run_tests.sh | 20 +++++++------- setup.cfg | 2 +- testgres/testgres.py | 62 ++++++++++++++++++++++---------------------- tests/test_simple.py | 19 +++++++------- 4 files changed, 51 insertions(+), 52 deletions(-) diff --git a/run_tests.sh b/run_tests.sh index f5098de6..50d35ef5 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -5,26 +5,26 @@ set -eux if [ "$PYTHON" == "python2" ]; then - virtualenv="virtualenv --python=/usr/bin/python2" - pip=pip2 + VIRTUALENV="virtualenv --python=/usr/bin/python2" + PIP=pip2 else - virtualenv="virtualenv --python=/usr/bin/python3" - pip=pip3 + VIRTUALENV="virtualenv --python=/usr/bin/python3" + PIP=pip3 fi # prepare environment -cd .. -$virtualenv env +VENV_PATH=/tmp/testgres_venv +rm -rf $VENV_PATH +$VIRTUALENV $VENV_PATH export VIRTUAL_ENV_DISABLE_PROMPT=1 -source env/bin/activate -cd - +source $VENV_PATH/bin/activate # install utilities -$pip install coverage flake8 +$PIP install coverage flake8 # install testgres' dependencies export PYTHONPATH=$(pwd) -$pip install . +$PIP install . # test code quality flake8 . diff --git a/setup.cfg b/setup.cfg index 4e045d52..f8d1b92b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -3,4 +3,4 @@ description-file = README.md [flake8] ignore = E501, F401, F403, F841 -exclude = .git,__pycache__ +exclude = .git,__pycache__,env,venv diff --git a/testgres/testgres.py b/testgres/testgres.py index 902a401d..21cbaca3 100644 --- a/testgres/testgres.py +++ b/testgres/testgres.py @@ -363,7 +363,7 @@ def spawn_primary(self, name, destroy=True, use_logging=False): Create a primary node from a backup. Args: - name: name for a new node (str). + name: name for a new node. destroy: should we convert this backup into a node? use_logging: enable python logging. @@ -392,7 +392,7 @@ def spawn_replica(self, name, destroy=True, use_logging=False): Create a replica of the original node from a backup. Args: - name: name for a new node (str). + name: name for a new node. destroy: should we convert this backup into a node? use_logging: enable python logging. @@ -814,10 +814,10 @@ def psql(self, dbname, query=None, filename=None, username=None): Execute a query using psql. Args: - dbname: database name to connect to (str). - query: query to be executed (str). - filename: file with a query (str). - username: database user name (str). + dbname: database name to connect to. + query: query to be executed. + filename: file with a query. + username: database user name. Returns: A tuple of (code, stdout, stderr). @@ -857,9 +857,9 @@ def safe_psql(self, dbname, query, username=None): Execute a query using psql. Args: - dbname: database name to connect to (str). - query: query to be executed (str). - username: database user name (str). + dbname: database name to connect to. + query: query to be executed. + username: database user name. Returns: psql's output as str. @@ -875,8 +875,8 @@ def dump(self, dbname, filename=None): Dump database using pg_dump. Args: - dbname: database name to connect to (str). - filename: output file (str). + dbname: database name to connect to. + filename: output file. Returns: Path to file containing dump. @@ -900,8 +900,8 @@ def restore(self, dbname, filename, username=None): Restore database from pg_dump's file. Args: - dbname: database name to connect to (str). - filename: database dump taken by pg_dump (str). + dbname: database name to connect to. + filename: database dump taken by pg_dump. """ self.psql(dbname=dbname, filename=filename, username=username) @@ -919,9 +919,9 @@ def poll_query_until(self, Run a query once a second until it returs 'expected'. Args: - dbname: database name to connect to (str). - query: query to be executed (str). - username: database user name (str). + dbname: database name to connect to. + query: query to be executed. + username: database user name. max_attempts: how many times should we try? sleep_time: how long should we sleep after a failure? expected: what should be returned to break the cycle? @@ -970,9 +970,9 @@ def execute(self, dbname, query, username=None, commit=True): Execute a query and return all rows as list. Args: - dbname: database name to connect to (str). - query: query to be executed (str). - username: database user name (str). + dbname: database name to connect to. + query: query to be executed. + username: database user name. commit: should we commit this query? Returns: @@ -990,7 +990,7 @@ def backup(self, username=None, xlog_method=DEFAULT_XLOG_METHOD): Perform pg_basebackup. Args: - username: database user name (str). + username: database user name. xlog_method: a method for collecting the logs ('fetch' | 'stream'). Returns: @@ -1008,8 +1008,8 @@ def replicate(self, name, username=None, Create a replica of this node. Args: - name: replica's name (str). - username: database user name (str). + name: replica's name. + username: database user name. xlog_method: a method for collecting the logs ('fetch' | 'stream'). use_logging: enable python logging. """ @@ -1048,7 +1048,7 @@ def pgbench_init(self, dbname='postgres', scale=1, options=[]): Prepare database for pgbench (create tables etc). Args: - dbname: database name to connect to (str). + dbname: database name to connect to. scale: report this scale factor in output (int). options: additional options for pgbench (list). @@ -1071,7 +1071,7 @@ def pgbench(self, dbname='postgres', stdout=None, stderr=None, options=[]): Spawn a pgbench process. Args: - dbname: database name to connect to (str). + dbname: database name to connect to. stdout: stdout file to be used by Popen. stderr: stderr file to be used by Popen. options: additional options for pgbench (list). @@ -1091,8 +1091,8 @@ def connect(self, dbname='postgres', username=None): Connect to a database. Args: - dbname: database name to connect to (str). - username: database user name (str). + dbname: database name to connect to. + username: database user name. Returns: An instance of NodeConnection. @@ -1152,9 +1152,9 @@ def _execute_utility(util, args, logfile, write_to_pipe=True): Execute utility (pg_ctl, pg_dump etc) using get_bin_path(). Args: - util: utility to be executed (str). + util: utility to be executed. args: arguments for utility (list). - logfile: stores stdout and stderr (str). + logfile: path to file to store stdout and stderr. write_to_pipe: do we care about stdout? Returns: @@ -1174,7 +1174,7 @@ def _execute_utility(util, args, logfile, write_to_pipe=True): # get result out, _ = process.communicate() - out = '' if not out else out.decode('utf-8') + out = '' if not out else six.text_type(out) # write new log entry if possible try: @@ -1296,8 +1296,8 @@ def get_new_node(name, base_dir=None, use_logging=False): Create a new node (select port automatically). Args: - name: node's name (str). - base_dir: path to node's data directory (str). + name: node's name. + base_dir: path to node's data directory. use_logging: should we use custom logger? Returns: diff --git a/tests/test_simple.py b/tests/test_simple.py index ebfb06d3..2a793863 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -336,7 +336,6 @@ def test_poll_query_until(self): self.assertTrue(got_exception) def test_logging(self): - regex = re.compile('.+?LOG:.*') logfile = tempfile.NamedTemporaryFile('w', delete=True) log_conf = { @@ -362,17 +361,17 @@ def test_logging(self): logging.config.dictConfig(log_conf) - with get_new_node('master', use_logging=True) as node0, \ - get_new_node('slave1', use_logging=True) as node1, \ - get_new_node('slave2', use_logging=True) as node2: + with get_new_node('master', use_logging=True) as master: + master.init().start() - node0.init().start() - node1.init().start() - node2.init().start() + import time + time.sleep(0.5) + # check that master's port is found with open(logfile.name, 'r') as log: - for line in log: - self.assertTrue(regex.match(line)) + lines = log.readlines() + port = str(master.port) + self.assertTrue(any(port in s for s in lines)) def test_pgbench(self): with get_new_node('node') as node: @@ -403,7 +402,7 @@ def test_pg_ctl(self): node.init().start() status = node.pg_ctl(['status']) - self.assertTrue("server is running" in status) + self.assertTrue('PID' in status) def test_ports_management(self): # check that no ports have been bound yet From 61b75168cb86d9050d71f9f8385ec508320cc013 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Thu, 23 Nov 2017 20:52:01 +0300 Subject: [PATCH 007/475] fix support for utf-8 --- testgres/testgres.py | 58 +++++++++++++++++++++++++++++--------------- tests/test_simple.py | 9 +++++-- 2 files changed, 45 insertions(+), 22 deletions(-) diff --git a/testgres/testgres.py b/testgres/testgres.py index 21cbaca3..24dc2112 100644 --- a/testgres/testgres.py +++ b/testgres/testgres.py @@ -38,6 +38,7 @@ import tempfile import threading import time +import traceback import port_for @@ -349,7 +350,7 @@ def _prepare_dir(self, destroy): # Copy backup to new data dir shutil.copytree(data1, data2) except Exception as e: - raise BackupException(str(e)) + raise BackupException(_explain_exception(e)) else: base_dir = self.base_dir @@ -691,13 +692,13 @@ def print_node_file(node_file): return "### file not found ###\n" error_text = ( - "Cannot start node\n" - "{}\n" # pg_ctl log - "{}:\n----\n{}\n" # postgresql.log - "{}:\n----\n{}\n" # postgresql.conf - "{}:\n----\n{}\n" # pg_hba.conf - "{}:\n----\n{}\n" # recovery.conf - ).format(str(e), + u"Cannot start node\n" + u"{}\n" # pg_ctl log + u"{}:\n----\n{}\n" # postgresql.log + u"{}:\n----\n{}\n" # postgresql.conf + u"{}:\n----\n{}\n" # pg_hba.conf + u"{}:\n----\n{}\n" # recovery.conf + ).format(_explain_exception(e), log_filename, print_node_file(log_filename), conf_filename, print_node_file(conf_filename), hba_filename, print_node_file(hba_filename), @@ -867,7 +868,9 @@ def safe_psql(self, dbname, query, username=None): ret, out, err = self.psql(dbname, query, username=username) if ret: - raise QueryException(six.text_type(err)) + if err: + err = err.decode('utf-8') + raise QueryException(err) return out def dump(self, dbname, filename=None): @@ -1041,7 +1044,7 @@ def catchup(self): lsn = master.execute('postgres', poll_lsn)[0][0] self.poll_query_until('postgres', wait_lsn.format(lsn)) except Exception as e: - raise CatchUpException(str(e)) + raise CatchUpException(_explain_exception(e)) def pgbench_init(self, dbname='postgres', scale=1, options=[]): """ @@ -1103,6 +1106,15 @@ def connect(self, dbname='postgres', username=None): username=username) +def _explain_exception(e): + """ + Use this function instead of str(e). + """ + + lines = traceback.format_exception_only(type(e), e) + return ''.join(lines) + + def _cached_initdb(data_dir, initdb_logfile, initdb_params=[]): """ Perform initdb or use cached node files. @@ -1113,7 +1125,7 @@ def call_initdb(_data_dir): _params = [_data_dir, "-N"] + initdb_params _execute_utility("initdb", _params, initdb_logfile) except Exception as e: - raise InitNodeException(str(e)) + raise InitNodeException(_explain_exception(e)) # Call initdb if we have custom params if initdb_params or not TestgresConfig.cache_initdb: @@ -1144,7 +1156,7 @@ def rm_cached_data_dir(rm_dir): shutil.copytree(cached_data_dir, data_dir) except Exception as e: - raise InitNodeException(str(e)) + raise InitNodeException(_explain_exception(e)) def _execute_utility(util, args, logfile, write_to_pipe=True): @@ -1174,23 +1186,29 @@ def _execute_utility(util, args, logfile, write_to_pipe=True): # get result out, _ = process.communicate() - out = '' if not out else six.text_type(out) # write new log entry if possible try: with open(logfile, "a") as file_out: - # write util name + args - file_out.write(''.join(map(lambda x: str(x) + ' ', - [util] + args))) - file_out.write('\n') - file_out.write(out) + # write util name + file_out.write(util) + # write args + for arg in args: + file_out.write(arg) + with open(logfile, "ab") as file_out: + # write output + if out: + file_out.write(out) except IOError: pass + # decode output + out = '' if not out else out.decode('utf-8') + if process.returncode: error_text = ( - "{} failed\n" - "log:\n----\n{}\n" + u"{} failed\n" + u"log:\n----\n{}\n" ).format(util, out) raise ExecUtilException(error_text, process.returncode) diff --git a/tests/test_simple.py b/tests/test_simple.py index 2a793863..afa00f38 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -1,4 +1,5 @@ #!/usr/bin/env python +# coding: utf-8 import os import re @@ -364,14 +365,18 @@ def test_logging(self): with get_new_node('master', use_logging=True) as master: master.init().start() + # execute a dummy query a few times + for i in range(20): + master.execute('postgres', 'select 1') + + # let logging worker do the job import time time.sleep(0.5) # check that master's port is found with open(logfile.name, 'r') as log: lines = log.readlines() - port = str(master.port) - self.assertTrue(any(port in s for s in lines)) + self.assertTrue(any('select' in s for s in lines)) def test_pgbench(self): with get_new_node('node') as node: From 71d0af07f18e534c816b7e9da1554f1d3bd2fca8 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Fri, 24 Nov 2017 13:53:16 +0300 Subject: [PATCH 008/475] small fix --- testgres/testgres.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/testgres/testgres.py b/testgres/testgres.py index 24dc2112..01e184f5 100644 --- a/testgres/testgres.py +++ b/testgres/testgres.py @@ -868,8 +868,7 @@ def safe_psql(self, dbname, query, username=None): ret, out, err = self.psql(dbname, query, username=username) if ret: - if err: - err = err.decode('utf-8') + err = '' if not err else err.decode('utf-8') raise QueryException(err) return out From 5d5b471b2a87c7d8ab3f0a524c4af2ac028d0b56 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Fri, 24 Nov 2017 15:21:44 +0300 Subject: [PATCH 009/475] improve error handling and logging --- testgres/testgres.py | 50 ++++++++++++++++++++++---------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/testgres/testgres.py b/testgres/testgres.py index 01e184f5..73e44fec 100644 --- a/testgres/testgres.py +++ b/testgres/testgres.py @@ -440,15 +440,14 @@ def __init__(self, self.name = name self.host = '127.0.0.1' self.port = port or reserve_port() + self.base_dir = base_dir self.should_free_port = port is None - self.base_dir = base_dir or tempfile.mkdtemp() self.should_rm_dirs = base_dir is None self.use_logging = use_logging self.logger = None - # create directory if needed - if not os.path.exists(self.logs_dir): - os.makedirs(self.logs_dir) + # create directories if needed + self._prepare_dirs() def __enter__(self): return self @@ -486,6 +485,13 @@ def _create_recovery_conf(self, root_node): self.append_conf("recovery.conf", line) + def _prepare_dirs(self): + if not self.base_dir or not os.path.exists(self.base_dir): + self.base_dir = tempfile.mkdtemp() + + if not os.path.exists(self.logs_dir): + os.makedirs(self.logs_dir) + def init(self, allow_streaming=False, fsync=False, initdb_params=[]): """ Perform initdb for this node. @@ -499,11 +505,8 @@ def init(self, allow_streaming=False, fsync=False, initdb_params=[]): This instance of PostgresNode. """ - postgres_conf = os.path.join(self.data_dir, "postgresql.conf") - - # We don't have to reinit it if data directory exists - if os.path.isfile(postgres_conf): - raise InitNodeException('Node is already intialized') + # create directories if needed + self._prepare_dirs() # initialize this PostgreSQL node initdb_log = os.path.join(self.logs_dir, "initdb.log") @@ -1123,7 +1126,7 @@ def call_initdb(_data_dir): try: _params = [_data_dir, "-N"] + initdb_params _execute_utility("initdb", _params, initdb_logfile) - except Exception as e: + except ExecUtilException as e: raise InitNodeException(_explain_exception(e)) # Call initdb if we have custom params @@ -1143,17 +1146,16 @@ def rm_cached_data_dir(rm_dir): atexit.register(rm_cached_data_dir, TestgresConfig.cached_initdb_dir) - try: - # Fetch cached initdb dir - cached_data_dir = TestgresConfig.cached_initdb_dir + # Fetch cached initdb dir + cached_data_dir = TestgresConfig.cached_initdb_dir - # Initialize cached initdb - if not os.listdir(cached_data_dir): - call_initdb(cached_data_dir) + # Initialize cached initdb + if not os.listdir(cached_data_dir): + call_initdb(cached_data_dir) + try: # Copy cached initdb to current data dir shutil.copytree(cached_data_dir, data_dir) - except Exception as e: raise InitNodeException(_explain_exception(e)) @@ -1189,14 +1191,12 @@ def _execute_utility(util, args, logfile, write_to_pipe=True): # write new log entry if possible try: with open(logfile, "a") as file_out: - # write util name - file_out.write(util) - # write args - for arg in args: - file_out.write(arg) - with open(logfile, "ab") as file_out: - # write output - if out: + # write util name and args + file_out.write(' '.join([util] + args)) + file_out.write('\n') + if out: + with open(logfile, "ab") as file_out: + # write output file_out.write(out) except IOError: pass From 15ccedb59812dddf723dfd28103365f78d343e5d Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Fri, 24 Nov 2017 17:33:39 +0300 Subject: [PATCH 010/475] more tests --- tests/test_simple.py | 50 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/tests/test_simple.py b/tests/test_simple.py index afa00f38..0efca2f2 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -19,7 +19,8 @@ ExecUtilException, \ BackupException, \ QueryException, \ - CatchUpException + CatchUpException, \ + TimeoutException from testgres import get_new_node, get_pg_config, configure_testgres from testgres import bound_ports @@ -60,22 +61,58 @@ def test_restart(self): res = node.execute('postgres', 'select 2') self.assertEqual(res, [(2, )]) + def test_psql(self): + with get_new_node('test') as node: + node.init().start() + + # check default params + got_exception = False + try: + node.psql('postgres') + except QueryException as e: + got_exception = True + self.assertTrue(got_exception) + + # check returned values + res = node.psql('postgres', 'select 1') + self.assertEqual(res[0], 0) + self.assertEqual(res[1], b'1\n') + self.assertEqual(res[2], b'') + + # check returned values + res = node.safe_psql('postgres', 'select 1') + self.assertEqual(res, b'1\n') + + node.stop() + + # check psql on stopped node + got_exception = False + try: + node.safe_psql('postgres', 'select 1') + except QueryException as e: + got_exception = True + self.assertTrue(got_exception) + def test_status(self): + # check NodeStatus cast to bool condition_triggered = False if NodeStatus.Running: condition_triggered = True self.assertTrue(condition_triggered) + # check NodeStatus cast to bool condition_triggered = False if NodeStatus.Stopped: condition_triggered = True self.assertFalse(condition_triggered) + # check NodeStatus cast to bool condition_triggered = False if NodeStatus.Uninitialized: condition_triggered = True self.assertFalse(condition_triggered) + # check statuses after each operation with get_new_node('test') as node: self.assertEqual(node.get_pid(), 0) self.assertEqual(node.status(), NodeStatus.Uninitialized) @@ -336,6 +373,17 @@ def test_poll_query_until(self): got_exception = True self.assertTrue(got_exception) + # check timeout + got_exception = False + try: + node.poll_query_until(dbname='postgres', + query='select 1 > 2', + max_attempts=5, + sleep_time=0.2) + except TimeoutException as e: + got_exception = True + self.assertTrue(got_exception) + def test_logging(self): logfile = tempfile.NamedTemporaryFile('w', delete=True) From d7afaad5383c1fae01c2104c4a4eff1d0db9a970 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Fri, 24 Nov 2017 17:42:24 +0300 Subject: [PATCH 011/475] allow zero (infinite) 'max_attempts' in poll_query_until() --- testgres/testgres.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/testgres/testgres.py b/testgres/testgres.py index 73e44fec..ec6e8d14 100644 --- a/testgres/testgres.py +++ b/testgres/testgres.py @@ -922,20 +922,25 @@ def poll_query_until(self, raise_internal_error=True): """ Run a query once a second until it returs 'expected'. + Query should return single column. Args: dbname: database name to connect to. query: query to be executed. username: database user name. - max_attempts: how many times should we try? - sleep_time: how long should we sleep after a failure? + max_attempts: how many times should we try? 0 == infinite + sleep_time: how much should we sleep after a failure? expected: what should be returned to break the cycle? raise_programming_error: mute ProgrammingError? raise_internal_error: mute InternalError? """ + # sanity checks + assert(max_attempts >= 0) + assert(sleep_time > 0) + attempts = 0 - while attempts < max_attempts: + while max_attempts == 0 or attempts < max_attempts: try: res = self.execute(dbname=dbname, query=query, @@ -1010,7 +1015,7 @@ def replicate(self, name, username=None, xlog_method=DEFAULT_XLOG_METHOD, use_logging=False): """ - Create a replica of this node. + Create a binary replica of this node. Args: name: replica's name. From 07b42aeb244c0bb29d1d88b460b50bf446aa3a1a Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Fri, 24 Nov 2017 17:43:44 +0300 Subject: [PATCH 012/475] fix typos --- testgres/testgres.py | 2 +- tests/test_simple.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/testgres/testgres.py b/testgres/testgres.py index ec6e8d14..e678843e 100644 --- a/testgres/testgres.py +++ b/testgres/testgres.py @@ -852,7 +852,7 @@ def psql(self, dbname, query=None, filename=None, username=None): stdout=subprocess.PIPE, stderr=subprocess.PIPE) - # wait untill it finishes and get stdout and stderr + # wait until it finishes and get stdout and stderr out, err = process.communicate() return process.returncode, out, err diff --git a/tests/test_simple.py b/tests/test_simple.py index 0efca2f2..3131a0f9 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -470,7 +470,7 @@ def test_ports_management(self): port_2 = node.port self.assertEqual(port_1, port_2) - # check that port has been freed successfuly + # check that port has been freed successfully self.assertEqual(len(bound_ports), 0) def test_version_management(self): From 33c3b7bcdb4d328ce97a745d487b0b2b9984d5ee Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Fri, 24 Nov 2017 18:11:16 +0300 Subject: [PATCH 013/475] rewrite tests using assertRaises() --- tests/test_simple.py | 116 +++++++++++++------------------------------ 1 file changed, 35 insertions(+), 81 deletions(-) diff --git a/tests/test_simple.py b/tests/test_simple.py index 3131a0f9..26dd3f0b 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -31,26 +31,28 @@ class SimpleTest(unittest.TestCase): def test_double_init(self): with get_new_node('test') as node: - got_exception = False - - try: + # can't initialize node more than once + with self.assertRaises(InitNodeException): node.init() node.init() - except InitNodeException as e: - got_exception = True - self.assertTrue(got_exception) + def test_init_after_cleanup(self): + with get_new_node('test') as node: + node.init().start() + node.status() + node.safe_psql('postgres', 'select 1') + + node.cleanup() + + node.init().start() + node.status() + node.safe_psql('postgres', 'select 1') def test_uninitialized_start(self): with get_new_node('test') as node: - got_exception = False - - try: + # node is not initialized yet + with self.assertRaises(StartNodeException): node.start() - except StartNodeException as e: - got_exception = True - - self.assertTrue(got_exception) def test_restart(self): with get_new_node('test') as node: @@ -66,12 +68,8 @@ def test_psql(self): node.init().start() # check default params - got_exception = False - try: + with self.assertRaises(QueryException): node.psql('postgres') - except QueryException as e: - got_exception = True - self.assertTrue(got_exception) # check returned values res = node.psql('postgres', 'select 1') @@ -86,12 +84,8 @@ def test_psql(self): node.stop() # check psql on stopped node - got_exception = False - try: + with self.assertRaises(QueryException): node.safe_psql('postgres', 'select 1') - except QueryException as e: - got_exception = True - self.assertTrue(got_exception) def test_status(self): # check NodeStatus cast to bool @@ -181,28 +175,17 @@ def test_transactions(self): def test_control_data(self): with get_new_node('test') as node: - got_exception = False - try: + # node is not initialized yet + with self.assertRaises(ExecUtilException): node.get_control_data() - except ExecUtilException as e: - got_exception = True - self.assertTrue(got_exception) - got_exception = False - - try: - node.init() - data = node.get_control_data() - - # check returned dict - self.assertIsNotNone(data) - self.assertTrue(any('pg_control' in s for s in data.keys())) + node.init() + data = node.get_control_data() - except ExecUtilException as e: - print(e.message) - got_exception = True - self.assertFalse(got_exception) + # check returned dict + self.assertIsNotNone(data) + self.assertTrue(any('pg_control' in s for s in data.keys())) def test_backup_simple(self): with get_new_node('master') as master: @@ -238,19 +221,15 @@ def test_backup_exhaust(self): node.init(allow_streaming=True).start() with node.backup(xlog_method='fetch') as backup: + + # exhaust backup by creating new node with backup.spawn_primary('node1') as node1: pass - got_exception = False - try: + # now let's try to create one more node + with self.assertRaises(BackupException): with backup.spawn_primary('node2') as node2: pass - except BackupException as e: - got_exception = True - except Exception as e: - pass - - self.assertTrue(got_exception) def test_backup_and_replication(self): with get_new_node('node') as node, get_new_node('repl') as replica: @@ -296,12 +275,9 @@ def test_incorrect_catchup(self): with get_new_node('node') as node: node.init(allow_streaming=True).start() - got_exception = False - try: + # node has no master, can't catch up + with self.assertRaises(CatchUpException): node.catchup() - except CatchUpException as e: - got_exception = True - self.assertTrue(got_exception) def test_dump(self): with get_new_node('node1') as node1: @@ -348,41 +324,24 @@ def test_poll_query_until(self): self.assertTrue(end_time - start_time >= 5) # check 0 rows - got_exception = False - try: + with self.assertRaises(QueryException): node.poll_query_until( 'postgres', 'select * from pg_class where true = false') - except QueryException as e: - got_exception = True - self.assertTrue(got_exception) # check 0 columns - got_exception = False - try: + with self.assertRaises(QueryException): node.poll_query_until('postgres', 'select from pg_class limit 1') - except QueryException as e: - got_exception = True - self.assertTrue(got_exception) - # check None - got_exception = False - try: + with self.assertRaises(QueryException): node.poll_query_until('postgres', 'create table abc (val int)') - except QueryException as e: - got_exception = True - self.assertTrue(got_exception) # check timeout - got_exception = False - try: + with self.assertRaises(TimeoutException): node.poll_query_until(dbname='postgres', query='select 1 > 2', max_attempts=5, sleep_time=0.2) - except TimeoutException as e: - got_exception = True - self.assertTrue(got_exception) def test_logging(self): logfile = tempfile.NamedTemporaryFile('w', delete=True) @@ -511,13 +470,8 @@ def test_isolation_levels(self): con.begin(IsolationLevel.Serializable).commit() # check wrong level - got_exception = False - try: + with self.assertRaises(QueryException): con.begin('Garbage').commit() - except QueryException: - got_exception = True - - self.assertTrue(got_exception) if __name__ == '__main__': From b9b1d0159321212490edd2f72ff12411a15fdc63 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Fri, 24 Nov 2017 18:24:32 +0300 Subject: [PATCH 014/475] add test_custom_init() --- tests/test_simple.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test_simple.py b/tests/test_simple.py index 26dd3f0b..a3a7adab 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -29,6 +29,12 @@ class SimpleTest(unittest.TestCase): + def test_custom_init(self): + with get_new_node('test') as node: + # enable page checksums + node.init(initdb_params=['-k']).start() + node.safe_psql('postgres', 'select 1') + def test_double_init(self): with get_new_node('test') as node: # can't initialize node more than once From f4686a4827709f0501552d9e95f1b5de36696c43 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Fri, 24 Nov 2017 18:31:43 +0300 Subject: [PATCH 015/475] improve test_backup_simple(), set max_attempts=0 in poll_query_until() by default --- testgres/testgres.py | 9 ++++++--- tests/test_simple.py | 12 +++++++++++- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/testgres/testgres.py b/testgres/testgres.py index e678843e..9a672406 100644 --- a/testgres/testgres.py +++ b/testgres/testgres.py @@ -915,7 +915,7 @@ def poll_query_until(self, dbname, query, username=None, - max_attempts=60, + max_attempts=0, sleep_time=1, expected=True, raise_programming_error=True, @@ -1027,7 +1027,7 @@ def replicate(self, name, username=None, backup = self.backup(username=username, xlog_method=xlog_method) return backup.spawn_replica(name, use_logging=use_logging) - def catchup(self): + def catchup(self, username=None): """ Wait until async replica catches up with its master. """ @@ -1049,7 +1049,10 @@ def catchup(self): try: lsn = master.execute('postgres', poll_lsn)[0][0] - self.poll_query_until('postgres', wait_lsn.format(lsn)) + self.poll_query_until(dbname='postgres', + username=username, + query=wait_lsn.format(lsn), + max_attempts=0) # infinite except Exception as e: raise CatchUpException(_explain_exception(e)) diff --git a/tests/test_simple.py b/tests/test_simple.py index a3a7adab..d96575e7 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -195,8 +195,18 @@ def test_control_data(self): def test_backup_simple(self): with get_new_node('master') as master: - master.init(allow_streaming=True).start() + # enable streaming for backups + master.init(allow_streaming=True) + + # node must be running + with self.assertRaises(BackupException): + master.backup() + + # it's time to start node + master.start() + + # fill node with some data master.psql('postgres', 'create table test as select generate_series(1, 4) i') From a4fcaa2ceb443c7a4636fb5beeb9f10be0bd79c1 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Fri, 24 Nov 2017 18:53:55 +0300 Subject: [PATCH 016/475] bump version to 1.4.0 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index c9d8a02d..cbff2309 100644 --- a/setup.py +++ b/setup.py @@ -15,7 +15,7 @@ setup( name='testgres', packages=['testgres'], - version='1.3.4', + version='1.4.0', description='Testing utility for PostgreSQL and its extensions', license='PostgreSQL', author='Ildar Musin', From a0fa95ef68880ab03696b21b05b44698ce95512d Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Wed, 29 Nov 2017 16:20:45 +0300 Subject: [PATCH 017/475] take better care of pg_hba.conf --- testgres/testgres.py | 59 ++++++++++++++++++++++++++++++-------------- tests/test_simple.py | 2 +- 2 files changed, 42 insertions(+), 19 deletions(-) diff --git a/testgres/testgres.py b/testgres/testgres.py index 9a672406..01382d59 100644 --- a/testgres/testgres.py +++ b/testgres/testgres.py @@ -517,7 +517,10 @@ def init(self, allow_streaming=False, fsync=False, initdb_params=[]): return self - def default_conf(self, allow_streaming=False, fsync=False, log_statement='all'): + def default_conf(self, + allow_streaming=False, + fsync=False, + log_statement='all'): """ Apply default settings to this node. @@ -525,7 +528,7 @@ def default_conf(self, allow_streaming=False, fsync=False, log_statement='all'): allow_streaming: should this node add a hba entry for replication? fsync: should this node use fsync to keep data safe? log_statement: one of ('all', 'off', 'mod', 'ddl'), look at - postgresql docs for more information + PostgreSQL docs for more information Returns: This instance of PostgresNode. @@ -534,18 +537,33 @@ def default_conf(self, allow_streaming=False, fsync=False, log_statement='all'): postgres_conf = os.path.join(self.data_dir, "postgresql.conf") hba_conf = os.path.join(self.data_dir, "pg_hba.conf") - # add parameters to hba file - with open(hba_conf, "w") as conf: - conf.write("# TYPE\tDATABASE\tUSER\tADDRESS\t\tMETHOD\n" - "local\tall\t\tall\t\t\ttrust\n" - "host\tall\t\tall\t127.0.0.1/32\ttrust\n" - "host\tall\t\tall\t::1/128\t\ttrust\n" - # replication - "local\treplication\tall\t\t\ttrust\n" - "host\treplication\tall\t127.0.0.1/32\ttrust\n" - "host\treplication\tall\t::1/128\t\ttrust\n") - - # add parameters to config file + # filter lines in hba file + with open(hba_conf, "r+") as conf: + # get rid of comments and blank lines + lines = [ + s for s in conf.readlines() + if len(s.strip()) > 0 and not s.startswith('#') + ] + + # write filtered lines + conf.seek(0) + conf.truncate() + conf.writelines(lines) + + # replication-related settings + if allow_streaming: + new_lines = [ + "local\treplication\tall\t\t\ttrust\n", + "host\treplication\tall\t127.0.0.1/32\ttrust\n", + "host\treplication\tall\t::1/128\t\ttrust\n" + ] + + # write missing lines + for line in new_lines: + if line not in lines: + conf.write(line) + + # overwrite postgresql.conf file with open(postgres_conf, "w") as conf: if not fsync: conf.write("fsync = off\n") @@ -556,6 +574,7 @@ def default_conf(self, allow_streaming=False, fsync=False, log_statement='all'): self.host, self.port)) + # replication-related settings if allow_streaming: cur_ver = LooseVersion(get_pg_version()) min_ver = LooseVersion('9.6') @@ -563,10 +582,14 @@ def default_conf(self, allow_streaming=False, fsync=False, log_statement='all'): # select a proper wal_level for PostgreSQL wal_level = "hot_standby" if cur_ver < min_ver else "replica" - conf.write("max_wal_senders = 5\n" - "wal_keep_segments = 20\n" - "hot_standby = on\n" - "wal_level = {}\n".format(wal_level)) + max_wal_senders = 5 + wal_keep_segments = 20 + conf.write("hot_standby = on\n" + "max_wal_senders = {}\n" + "wal_keep_segments = {}\n" + "wal_level = {}\n".format(max_wal_senders, + wal_keep_segments, + wal_level)) return self diff --git a/tests/test_simple.py b/tests/test_simple.py index d96575e7..e88aac9a 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -248,7 +248,7 @@ def test_backup_exhaust(self): pass def test_backup_and_replication(self): - with get_new_node('node') as node, get_new_node('repl') as replica: + with get_new_node('node') as node: node.init(allow_streaming=True) node.start() node.psql('postgres', 'create table abc(a int, b int)') From bd4eb3c250e753e20c46d72b1d3668b8d05c6421 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Wed, 29 Nov 2017 18:23:05 +0300 Subject: [PATCH 018/475] fix authentication for replication in pg_hba.conf --- testgres/testgres.py | 15 ++++++++++++--- tests/test_simple.py | 15 +++++++++++++++ 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/testgres/testgres.py b/testgres/testgres.py index 01382d59..935e505e 100644 --- a/testgres/testgres.py +++ b/testgres/testgres.py @@ -552,10 +552,19 @@ def default_conf(self, # replication-related settings if allow_streaming: + # get auth method for host or local users + def get_auth_method(t): + return next((s.split()[-1] for s in lines + if s.startswith(t)), 'trust') + + # get auth methods + auth_local = get_auth_method('local') + auth_host = get_auth_method('host') + new_lines = [ - "local\treplication\tall\t\t\ttrust\n", - "host\treplication\tall\t127.0.0.1/32\ttrust\n", - "host\treplication\tall\t::1/128\t\ttrust\n" + "local\treplication\tall\t\t\t{}\n".format(auth_local), + "host\treplication\tall\t127.0.0.1/32\t{}\n".format(auth_host), + "host\treplication\tall\t::1/128\t\t{}\n".format(auth_host) ] # write missing lines diff --git a/tests/test_simple.py b/tests/test_simple.py index e88aac9a..4fa55204 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -35,6 +35,21 @@ def test_custom_init(self): node.init(initdb_params=['-k']).start() node.safe_psql('postgres', 'select 1') + with get_new_node('test') as node: + node.init(allow_streaming=True, + initdb_params=['--auth-local=reject', + '--auth-host=reject']) + + hba_file = os.path.join(node.data_dir, 'pg_hba.conf') + with open(hba_file, 'r') as conf: + lines = conf.readlines() + + # check number of lines + self.assertGreaterEqual(len(lines), 6) + + # there should be no trust entries at all + self.assertFalse(any('trust' in s for s in lines)) + def test_double_init(self): with get_new_node('test') as node: # can't initialize node more than once From 1444759567b5050bb308b859b9ff4b8238ca1f37 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Wed, 29 Nov 2017 18:44:26 +0300 Subject: [PATCH 019/475] bump version to 1.4.1 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index cbff2309..ba7fbf90 100644 --- a/setup.py +++ b/setup.py @@ -15,7 +15,7 @@ setup( name='testgres', packages=['testgres'], - version='1.4.0', + version='1.4.1', description='Testing utility for PostgreSQL and its extensions', license='PostgreSQL', author='Ildar Musin', From 72f6a7eed1980705a9f701df0e5b031d65e99d4d Mon Sep 17 00:00:00 2001 From: Arseny Sher Date: Wed, 6 Dec 2017 14:56:49 +0300 Subject: [PATCH 020/475] Allow to feed smth into stdin of psql and safe_psql. --- testgres/testgres.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/testgres/testgres.py b/testgres/testgres.py index 935e505e..b1e91cae 100644 --- a/testgres/testgres.py +++ b/testgres/testgres.py @@ -845,7 +845,7 @@ def cleanup(self, max_attempts=3): return self - def psql(self, dbname, query=None, filename=None, username=None): + def psql(self, dbname, query=None, filename=None, username=None, inp=None): """ Execute a query using psql. @@ -881,14 +881,15 @@ def psql(self, dbname, query=None, filename=None, username=None): # start psql process process = subprocess.Popen(psql_params, + stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) # wait until it finishes and get stdout and stderr - out, err = process.communicate() + out, err = process.communicate(input=inp) return process.returncode, out, err - def safe_psql(self, dbname, query, username=None): + def safe_psql(self, dbname, query, username=None, inp=None): """ Execute a query using psql. @@ -901,7 +902,7 @@ def safe_psql(self, dbname, query, username=None): psql's output as str. """ - ret, out, err = self.psql(dbname, query, username=username) + ret, out, err = self.psql(dbname, query, username=username, inp=inp) if ret: err = '' if not err else err.decode('utf-8') raise QueryException(err) From 43cb1d160b49c1bdea198feb817f456d369000be Mon Sep 17 00:00:00 2001 From: Arseny Sher Date: Wed, 6 Dec 2017 15:00:06 +0300 Subject: [PATCH 021/475] Enable node restart logging. --- testgres/testgres.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testgres/testgres.py b/testgres/testgres.py index b1e91cae..a349b218 100644 --- a/testgres/testgres.py +++ b/testgres/testgres.py @@ -776,7 +776,7 @@ def restart(self, params=[]): _params = ["restart", "-D", self.data_dir, "-w"] + params _execute_utility("pg_ctl", _params, self.utils_logname, - write_to_pipe=False) + write_to_pipe=True) return self From 59e8c8189fc4e82d7652b2533ac46ec94bc794a3 Mon Sep 17 00:00:00 2001 From: Arseny Sher Date: Fri, 8 Dec 2017 17:33:20 +0300 Subject: [PATCH 022/475] Renamed inp to input, added test for it, fixed restart logging. node.restart() logging was previously broken: -l option was not supplied, so its stdout/err wasn't redirected -- this is option of pg_ctl, not of postgres itself, and pg_ctl didn't reused it. That was also the reason of 'write_to_pipe' existence: if pg_ctl restart's stdout was fed to pipe, communicate() hanged, reading stdout of postgres infinitely. Now we ride on 'start' routine for restart and get the same behaviour, including logging. 'inp' param of safe_psql and psql was renamed to input for consistency (we don't use here standard input() function quite often, right?), test for it was added. --- testgres/testgres.py | 35 ++++++++++++++++------------------- tests/test_simple.py | 13 +++++++++++++ 2 files changed, 29 insertions(+), 19 deletions(-) diff --git a/testgres/testgres.py b/testgres/testgres.py index a349b218..568a300c 100644 --- a/testgres/testgres.py +++ b/testgres/testgres.py @@ -677,11 +677,12 @@ def get_control_data(self): return out_dict - def start(self, params=[]): + def start(self, restart=False, params=[]): """ - Start this node using pg_ctl. + (Re)start this node using pg_ctl. Args: + restart: restart or start? params: additional arguments for _execute_utility(). Returns: @@ -706,8 +707,9 @@ def start(self, params=[]): # choose recovery_filename recovery_filename = os.path.join(self.data_dir, "recovery.conf") + action = "restart" if restart else "start" _params = [ - "start", + action, "-D{}".format(self.data_dir), "-l{}".format(log_filename), "-w" @@ -727,7 +729,7 @@ def print_node_file(node_file): return "### file not found ###\n" error_text = ( - u"Cannot start node\n" + u"Cannot {} node\n".format(action) + u"{}\n" # pg_ctl log u"{}:\n----\n{}\n" # postgresql.log u"{}:\n----\n{}\n" # postgresql.conf @@ -773,10 +775,10 @@ def restart(self, params=[]): This instance of PostgresNode. """ - _params = ["restart", "-D", self.data_dir, "-w"] + params - _execute_utility("pg_ctl", _params, - self.utils_logname, - write_to_pipe=True) + if self.logger: + self.logger.stop() + + self.start(restart=True, params=params) return self @@ -845,7 +847,7 @@ def cleanup(self, max_attempts=3): return self - def psql(self, dbname, query=None, filename=None, username=None, inp=None): + def psql(self, dbname, query=None, filename=None, username=None, input=None): """ Execute a query using psql. @@ -886,10 +888,10 @@ def psql(self, dbname, query=None, filename=None, username=None, inp=None): stderr=subprocess.PIPE) # wait until it finishes and get stdout and stderr - out, err = process.communicate(input=inp) + out, err = process.communicate(input=input) return process.returncode, out, err - def safe_psql(self, dbname, query, username=None, inp=None): + def safe_psql(self, dbname, query, username=None, input=None): """ Execute a query using psql. @@ -902,7 +904,7 @@ def safe_psql(self, dbname, query, username=None, inp=None): psql's output as str. """ - ret, out, err = self.psql(dbname, query, username=username, inp=inp) + ret, out, err = self.psql(dbname, query, username=username, input=input) if ret: err = '' if not err else err.decode('utf-8') raise QueryException(err) @@ -1201,7 +1203,7 @@ def rm_cached_data_dir(rm_dir): raise InitNodeException(_explain_exception(e)) -def _execute_utility(util, args, logfile, write_to_pipe=True): +def _execute_utility(util, args, logfile): """ Execute utility (pg_ctl, pg_dump etc) using get_bin_path(). @@ -1209,7 +1211,6 @@ def _execute_utility(util, args, logfile, write_to_pipe=True): util: utility to be executed. args: arguments for utility (list). logfile: path to file to store stdout and stderr. - write_to_pipe: do we care about stdout? Returns: stdout of executed utility. @@ -1217,13 +1218,9 @@ def _execute_utility(util, args, logfile, write_to_pipe=True): # we can't use subprocess.DEVNULL on 2.7 with open(os.devnull, "w") as devnull: - - # choose file according to options - stdout_file = subprocess.PIPE if write_to_pipe else devnull - # run utility process = subprocess.Popen([get_bin_path(util)] + args, - stdout=stdout_file, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) # get result diff --git a/tests/test_simple.py b/tests/test_simple.py index 4fa55204..7434021f 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -102,6 +102,19 @@ def test_psql(self): res = node.safe_psql('postgres', 'select 1') self.assertEqual(res, b'1\n') + # check feeding input + node.safe_psql('postgres', 'create table horns (w int)') + node.safe_psql('postgres', 'copy horns from stdin (format csv)', + input= +b"""1 +2 +3 +\. +""") + sum = node.safe_psql('postgres', 'select sum(w) from horns') + self.assertEqual(sum, b'6\n') + node.safe_psql('postgres', 'drop table horns') + node.stop() # check psql on stopped node From f5732d15f0ed8e8d996d20193157f4b9aea68dde Mon Sep 17 00:00:00 2001 From: Arseny Sher Date: Fri, 8 Dec 2017 17:43:52 +0300 Subject: [PATCH 023/475] One-line input. --- tests/test_simple.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/tests/test_simple.py b/tests/test_simple.py index 7434021f..8220fea9 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -105,12 +105,7 @@ def test_psql(self): # check feeding input node.safe_psql('postgres', 'create table horns (w int)') node.safe_psql('postgres', 'copy horns from stdin (format csv)', - input= -b"""1 -2 -3 -\. -""") + input=b"1\n2\n3\n\.\n") sum = node.safe_psql('postgres', 'select sum(w) from horns') self.assertEqual(sum, b'6\n') node.safe_psql('postgres', 'drop table horns') From a329c426b78f45188e51b931c669f72adb1ed6b0 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Thu, 14 Dec 2017 17:17:08 +0300 Subject: [PATCH 024/475] refactoring, improved TestgresLogger, make some fields private --- testgres/testgres.py | 231 +++++++++++++++++++++++-------------------- 1 file changed, 123 insertions(+), 108 deletions(-) diff --git a/testgres/testgres.py b/testgres/testgres.py index 568a300c..de16e11b 100644 --- a/testgres/testgres.py +++ b/testgres/testgres.py @@ -59,12 +59,10 @@ # ports used by nodes bound_ports = set() -# threads for loggers -util_threads = [] - # rows returned by PG_CONFIG pg_config_data = {} +PG_LOG_FILE = "postgresql.log" UTILS_LOG_FILE = "utils.log" BACKUP_LOG_FILE = "backup.log" @@ -143,33 +141,36 @@ class TestgresLogger(threading.Thread): Helper class to implement reading from postgresql.log """ - def __init__(self, node_name, fd): - assert callable(fd.readline) - + def __init__(self, node_name, log_file_name): threading.Thread.__init__(self) - self.fd = fd - self.node_name = node_name - self.stop_event = threading.Event() - self.logger = logging.getLogger(node_name) - self.logger.setLevel(logging.INFO) + self._node_name = node_name + self._log_file_name = log_file_name + self._stop_event = threading.Event() + self._logger = logging.getLogger(node_name) + self._logger.setLevel(logging.INFO) def run(self): - while self.fd in select.select([self.fd], [], [], 0)[0]: - line = self.fd.readline() - if line: - extra = {'node': self.node_name} - self.logger.info(line.strip(), extra=extra) - elif self.stopped(): - break - else: - time.sleep(0.1) + # open log file for reading + with open(self._log_file_name, 'r') as fd: + # work until we're asked to stop + while not self._stop_event.is_set(): + # do we have new lines? + if fd in select.select([fd], [], [], 0)[0]: + for line in fd.readlines(): + extra = {'node': self._node_name} + self._logger.info(line, extra=extra) + else: + time.sleep(0.1) + + # don't forget to clear event + self._stop_event.clear() - def stop(self): - self.stop_event.set() + def stop(self, wait=True): + self._stop_event.set() - def stopped(self): - return self.stop_event.isSet() + if wait: + self.join() class IsolationLevel(Enum): @@ -381,7 +382,7 @@ def spawn_primary(self, name, destroy=True, use_logging=False): use_logging=use_logging) # New nodes should always remove dir tree - node.should_rm_dirs = True + node._should_rm_dirs = True node.append_conf("postgresql.conf", "\n") node.append_conf("postgresql.conf", "port = {}".format(node.port)) @@ -436,15 +437,18 @@ def __init__(self, master=None): global bound_ports + # public self.master = master self.name = name self.host = '127.0.0.1' self.port = port or reserve_port() self.base_dir = base_dir - self.should_free_port = port is None - self.should_rm_dirs = base_dir is None - self.use_logging = use_logging - self.logger = None + + # private + self._should_free_port = port is None + self._should_rm_dirs = base_dir is None + self._use_logging = use_logging + self._logger = None # create directories if needed self._prepare_dirs() @@ -470,9 +474,13 @@ def logs_dir(self): return os.path.join(self.base_dir, LOGS_DIR) @property - def utils_logname(self): + def utils_log_name(self): return os.path.join(self.logs_dir, UTILS_LOG_FILE) + @property + def pg_log_name(self): + return os.path.join(self.data_dir, PG_LOG_FILE) + @property def connstr(self): return "port={}".format(self.port) @@ -492,6 +500,53 @@ def _prepare_dirs(self): if not os.path.exists(self.logs_dir): os.makedirs(self.logs_dir) + def _maybe_start_logger(self): + if self._use_logging: + if not self._logger: + self._logger = TestgresLogger(self.name, self.pg_log_name) + self._logger.start() + + elif not self._logger.is_alive(): + self._logger.start() + + def _maybe_stop_logger(self): + if self._logger: + self._logger.stop() + + def _format_verbose_error(self): + # choose log_filename + log_filename = self.pg_log_name + + # choose conf_filename + conf_filename = os.path.join(self.data_dir, "postgresql.conf") + + # choose hba_filename + hba_filename = os.path.join(self.data_dir, "pg_hba.conf") + + # choose recovery_filename + recovery_filename = os.path.join(self.data_dir, "recovery.conf") + + def print_node_file(node_file): + if os.path.exists(node_file): + try: + with open(node_file, 'r') as f: + return f.read() + except Exception as e: + pass + return "### file not found ###\n" + + error_text = ( + u"{}:\n----\n{}\n" # log file, e.g. postgresql.log + u"{}:\n----\n{}\n" # postgresql.conf + u"{}:\n----\n{}\n" # pg_hba.conf + u"{}:\n----\n{}\n" # recovery.conf + ).format(log_filename, print_node_file(log_filename), + conf_filename, print_node_file(conf_filename), + hba_filename, print_node_file(hba_filename), + recovery_filename, print_node_file(recovery_filename)) + + return error_text + def init(self, allow_streaming=False, fsync=False, initdb_params=[]): """ Perform initdb for this node. @@ -630,7 +685,7 @@ def status(self): try: _params = ["status", "-D", self.data_dir] - _execute_utility("pg_ctl", _params, self.utils_logname) + _execute_utility("pg_ctl", _params, self.utils_log_name) return NodeStatus.Running except ExecUtilException as e: @@ -667,7 +722,7 @@ def get_control_data(self): else: _params = ["-D", self.data_dir] - data = _execute_utility("pg_controldata", _params, self.utils_logname) + data = _execute_utility("pg_controldata", _params, self.utils_log_name) out_dict = {} @@ -677,71 +732,34 @@ def get_control_data(self): return out_dict - def start(self, restart=False, params=[]): + def start(self, params=[]): """ - (Re)start this node using pg_ctl. + Start this node using pg_ctl. Args: - restart: restart or start? params: additional arguments for _execute_utility(). Returns: This instance of PostgresNode. """ - # choose log_filename - if self.use_logging: - tmpfile = tempfile.NamedTemporaryFile('w', dir=self.logs_dir, delete=False) - log_filename = tmpfile.name - - self.logger = log_watch(self.name, log_filename) - else: - log_filename = os.path.join(self.logs_dir, "postgresql.log") - - # choose conf_filename - conf_filename = os.path.join(self.data_dir, "postgresql.conf") - - # choose hba_filename - hba_filename = os.path.join(self.data_dir, "pg_hba.conf") - - # choose recovery_filename - recovery_filename = os.path.join(self.data_dir, "recovery.conf") - - action = "restart" if restart else "start" _params = [ - action, + "start", "-D{}".format(self.data_dir), - "-l{}".format(log_filename), + "-l{}".format(self.pg_log_name), "-w" ] + params try: - _execute_utility("pg_ctl", _params, self.utils_logname) - + _execute_utility("pg_ctl", _params, self.utils_log_name) except ExecUtilException as e: - def print_node_file(node_file): - if os.path.exists(node_file): - try: - with open(node_file, 'r') as f: - return f.read() - except Exception as e: - pass - return "### file not found ###\n" - - error_text = ( - u"Cannot {} node\n".format(action) + + msg = ( + u"Cannot start node\n" u"{}\n" # pg_ctl log - u"{}:\n----\n{}\n" # postgresql.log - u"{}:\n----\n{}\n" # postgresql.conf - u"{}:\n----\n{}\n" # pg_hba.conf - u"{}:\n----\n{}\n" # recovery.conf - ).format(_explain_exception(e), - log_filename, print_node_file(log_filename), - conf_filename, print_node_file(conf_filename), - hba_filename, print_node_file(hba_filename), - recovery_filename, print_node_file(recovery_filename)) - - raise StartNodeException(error_text) + ).format(self._format_verbose_error()) + raise StartNodeException(msg) + + self._maybe_start_logger() return self @@ -757,10 +775,9 @@ def stop(self, params=[]): """ _params = ["stop", "-D", self.data_dir, "-w"] + params - _execute_utility("pg_ctl", _params, self.utils_logname) + _execute_utility("pg_ctl", _params, self.utils_log_name) - if self.logger: - self.logger.stop() + self._maybe_stop_logger() return self @@ -775,10 +792,23 @@ def restart(self, params=[]): This instance of PostgresNode. """ - if self.logger: - self.logger.stop() + _params = [ + "restart", + "-D{}".format(self.data_dir), + "-l{}".format(self.pg_log_name), + "-w" + ] + params - self.start(restart=True, params=params) + try: + _execute_utility("pg_ctl", _params, self.utils_log_name) + except ExecUtilException as e: + msg = ( + u"Cannot restart node\n" + u"{}\n" # pg_ctl log + ).format(self._format_verbose_error()) + raise StartNodeException(msg) + + self._maybe_start_logger() return self @@ -791,7 +821,7 @@ def reload(self, params=[]): """ _params = ["reload", "-D", self.data_dir, "-w"] + params - _execute_utility("pg_ctl", _params, self.utils_logname) + _execute_utility("pg_ctl", _params, self.utils_log_name) def pg_ctl(self, params): """ @@ -802,14 +832,14 @@ def pg_ctl(self, params): """ _params = params + ["-D", self.data_dir, "-w"] - return _execute_utility("pg_ctl", _params, self.utils_logname) + return _execute_utility("pg_ctl", _params, self.utils_log_name) def free_port(self): """ Reclaim port owned by this node. """ - if self.should_free_port: + if self._should_free_port: release_port(self.port) def cleanup(self, max_attempts=3): @@ -835,7 +865,7 @@ def cleanup(self, max_attempts=3): attempts += 1 # remove directory tree if necessary - if self.should_rm_dirs: + if self._should_rm_dirs: # choose directory to be removed if TestgresConfig.node_cleanup_full: @@ -931,7 +961,7 @@ def dump(self, dbname, filename=None): dbname ] - _execute_utility("pg_dump", _params, self.utils_logname) + _execute_utility("pg_dump", _params, self.utils_log_name) return filename @@ -1110,7 +1140,7 @@ def pgbench_init(self, dbname='postgres', scale=1, options=[]): "-p{}".format(self.port) ] + options + [dbname] - _execute_utility("pgbench", _params, self.utils_logname) + _execute_utility("pgbench", _params, self.utils_log_name) return self @@ -1353,7 +1383,7 @@ def get_new_node(name, base_dir=None, use_logging=False): Args: name: node's name. base_dir: path to node's data directory. - use_logging: should we use custom logger? + use_logging: enable python logging. Returns: An instance of PostgresNode. @@ -1370,18 +1400,3 @@ def configure_testgres(**options): for key, option in options.items(): setattr(TestgresConfig, key, option) - - -def log_watch(node_name, pg_logname): - """ - Start thread for node that redirects - PostgreSQL logs to python logging system. - """ - - reader = TestgresLogger(node_name, open(pg_logname, 'r')) - reader.start() - - global util_threads - util_threads.append(reader) - - return reader From 7be2f8daaeed132a60b496374d00750508e180ab Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Thu, 14 Dec 2017 17:54:50 +0300 Subject: [PATCH 025/475] improve logging, make tests more reliable --- testgres/testgres.py | 6 ++++-- tests/test_simple.py | 10 ++++++---- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/testgres/testgres.py b/testgres/testgres.py index de16e11b..997ee562 100644 --- a/testgres/testgres.py +++ b/testgres/testgres.py @@ -158,8 +158,10 @@ def run(self): # do we have new lines? if fd in select.select([fd], [], [], 0)[0]: for line in fd.readlines(): - extra = {'node': self._node_name} - self._logger.info(line, extra=extra) + line = line.strip() + if line: + extra = {'node': self._node_name} + self._logger.info(line, extra=extra) else: time.sleep(0.1) diff --git a/tests/test_simple.py b/tests/test_simple.py index 8220fea9..8dd588af 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -7,6 +7,7 @@ import subprocess import tempfile import testgres +import time import unittest import logging.config @@ -408,21 +409,22 @@ def test_logging(self): logging.config.dictConfig(log_conf) - with get_new_node('master', use_logging=True) as master: + node_name = 'master' + with get_new_node(node_name, use_logging=True) as master: master.init().start() # execute a dummy query a few times for i in range(20): master.execute('postgres', 'select 1') + time.sleep(0.01) # let logging worker do the job - import time - time.sleep(0.5) + time.sleep(0.1) # check that master's port is found with open(logfile.name, 'r') as log: lines = log.readlines() - self.assertTrue(any('select' in s for s in lines)) + self.assertTrue(any(node_name in s for s in lines)) def test_pgbench(self): with get_new_node('node') as node: From 613c59ce0bebe93725c0740f759a270bfbee45e9 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Thu, 14 Dec 2017 18:31:46 +0300 Subject: [PATCH 026/475] more tests for coverage --- tests/test_simple.py | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/tests/test_simple.py b/tests/test_simple.py index 8dd588af..f3f5d454 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -53,10 +53,11 @@ def test_custom_init(self): def test_double_init(self): with get_new_node('test') as node: + node.init() + # can't initialize node more than once with self.assertRaises(InitNodeException): node.init() - node.init() def test_init_after_cleanup(self): with get_new_node('test') as node: @@ -70,6 +71,14 @@ def test_init_after_cleanup(self): node.status() node.safe_psql('postgres', 'select 1') + def test_double_start(self): + with get_new_node('test') as node: + node.init().start() + + # can't start node more than once + with self.assertRaises(StartNodeException): + node.start() + def test_uninitialized_start(self): with get_new_node('test') as node: # node is not initialized yet @@ -119,22 +128,13 @@ def test_psql(self): def test_status(self): # check NodeStatus cast to bool - condition_triggered = False - if NodeStatus.Running: - condition_triggered = True - self.assertTrue(condition_triggered) + self.assertTrue(NodeStatus.Running) # check NodeStatus cast to bool - condition_triggered = False - if NodeStatus.Stopped: - condition_triggered = True - self.assertFalse(condition_triggered) + self.assertFalse(NodeStatus.Stopped) # check NodeStatus cast to bool - condition_triggered = False - if NodeStatus.Uninitialized: - condition_triggered = True - self.assertFalse(condition_triggered) + self.assertFalse(NodeStatus.Uninitialized) # check statuses after each operation with get_new_node('test') as node: @@ -426,6 +426,10 @@ def test_logging(self): lines = log.readlines() self.assertTrue(any(node_name in s for s in lines)) + # test logger after restart + master.restart() + self.assertTrue(master._logger.is_alive()) + def test_pgbench(self): with get_new_node('node') as node: node.init().start().pgbench_init() From 5b15db9c3fc37a887312666aeda3486bf1ff7898 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Thu, 14 Dec 2017 20:51:30 +0300 Subject: [PATCH 027/475] reformat code using yapf --- tests/test_simple.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/tests/test_simple.py b/tests/test_simple.py index f3f5d454..5e3e7a9b 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -37,9 +37,9 @@ def test_custom_init(self): node.safe_psql('postgres', 'select 1') with get_new_node('test') as node: - node.init(allow_streaming=True, - initdb_params=['--auth-local=reject', - '--auth-host=reject']) + node.init( + allow_streaming=True, + initdb_params=['--auth-local=reject', '--auth-host=reject']) hba_file = os.path.join(node.data_dir, 'pg_hba.conf') with open(hba_file, 'r') as conf: @@ -114,8 +114,10 @@ def test_psql(self): # check feeding input node.safe_psql('postgres', 'create table horns (w int)') - node.safe_psql('postgres', 'copy horns from stdin (format csv)', - input=b"1\n2\n3\n\.\n") + node.safe_psql( + 'postgres', + 'copy horns from stdin (format csv)', + input=b"1\n2\n3\n\.\n") sum = node.safe_psql('postgres', 'select sum(w) from horns') self.assertEqual(sum, b'6\n') node.safe_psql('postgres', 'drop table horns') @@ -378,10 +380,11 @@ def test_poll_query_until(self): # check timeout with self.assertRaises(TimeoutException): - node.poll_query_until(dbname='postgres', - query='select 1 > 2', - max_attempts=5, - sleep_time=0.2) + node.poll_query_until( + dbname='postgres', + query='select 1 > 2', + max_attempts=5, + sleep_time=0.2) def test_logging(self): logfile = tempfile.NamedTemporaryFile('w', delete=True) From 9cc56398ffce7d1abe4b5db87040f6f2f0ac800c Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Thu, 14 Dec 2017 21:19:27 +0300 Subject: [PATCH 028/475] fixes, small cleanup --- .gitignore | 8 +++++--- testgres/testgres.py | 34 ++++++++++++++++++---------------- tests/test_simple.py | 4 +++- 3 files changed, 26 insertions(+), 20 deletions(-) diff --git a/.gitignore b/.gitignore index a1ce722d..0e2d40d1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,11 @@ *.pyc dist tags -*.egg-info +*.egg-info/ +*.egg Dockerfile .coverage -env -/build +env/ +venv/ +build/ coverage.xml diff --git a/testgres/testgres.py b/testgres/testgres.py index 997ee562..d6a253bb 100644 --- a/testgres/testgres.py +++ b/testgres/testgres.py @@ -30,8 +30,6 @@ import atexit import logging import os -import pwd -import select import shutil import six import subprocess @@ -156,6 +154,7 @@ def run(self): # work until we're asked to stop while not self._stop_event.is_set(): # do we have new lines? + import select # used only here if fd in select.select([fd], [], [], 0)[0]: for line in fd.readlines(): line = line.strip() @@ -307,9 +306,12 @@ def __init__(self, if base_dir and not os.path.exists(base_dir): os.makedirs(base_dir) + # public self.original_node = node self.base_dir = base_dir - self.available = True + + # private + self._available = True data_dir = os.path.join(self.base_dir, DATA_DIR) _params = [ @@ -337,17 +339,17 @@ def _prepare_dir(self, destroy): Path to data directory. """ - if not self.available: + if not self._available: raise BackupException('Backup is exhausted') # Do we want to use this backup several times? available = not destroy if available: - base_dir = tempfile.mkdtemp() + dest_base_dir = tempfile.mkdtemp() data1 = os.path.join(self.base_dir, DATA_DIR) - data2 = os.path.join(base_dir, DATA_DIR) + data2 = os.path.join(dest_base_dir, DATA_DIR) try: # Copy backup to new data dir @@ -355,12 +357,13 @@ def _prepare_dir(self, destroy): except Exception as e: raise BackupException(_explain_exception(e)) else: - base_dir = self.base_dir + dest_base_dir = self.base_dir - # Update value - self.available = available + # Is this backup exhausted? + self._available = available - return base_dir + # Return path to new node + return dest_base_dir def spawn_primary(self, name, destroy=True, use_logging=False): """ @@ -410,9 +413,9 @@ def spawn_replica(self, name, destroy=True, use_logging=False): return node def cleanup(self): - if self.available: + if self._available: shutil.rmtree(self.base_dir, ignore_errors=True) - self.available = False + self._available = False class NodeStatus(Enum): @@ -504,13 +507,11 @@ def _prepare_dirs(self): def _maybe_start_logger(self): if self._use_logging: - if not self._logger: + # spawn new logger if it doesn't exist or stopped + if not self._logger or not self._logger.is_alive(): self._logger = TestgresLogger(self.name, self.pg_log_name) self._logger.start() - elif not self._logger.is_alive(): - self._logger.start() - def _maybe_stop_logger(self): if self._logger: self._logger.stop() @@ -1290,6 +1291,7 @@ def default_username(): Return current user. """ + import pwd # used only here return pwd.getpwuid(os.getuid())[0] diff --git a/tests/test_simple.py b/tests/test_simple.py index 5e3e7a9b..386cfce8 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -429,7 +429,9 @@ def test_logging(self): lines = log.readlines() self.assertTrue(any(node_name in s for s in lines)) - # test logger after restart + # test logger after stop/start/restart + master.stop() + master.start() master.restart() self.assertTrue(master._logger.is_alive()) From 24aa4eab51b87ff17e0c62385ccf1e12565e1a05 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Thu, 14 Dec 2017 21:48:19 +0300 Subject: [PATCH 029/475] update README.md --- README.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index c7091981..8deaceaf 100644 --- a/README.md +++ b/README.md @@ -96,12 +96,12 @@ node.start() Finally, our temporary cluster is able to process queries. There are four ways to run them: -| Command | Description | -|-------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------| -| `node.psql(database, query)` | Runs query via `psql` command and returns tuple `(error code, stdout, stderr)`. | -| `node.safe_psql(database, query)` | Same as `psql()` except that it returns only `stdout`. If an error occures during the execution, an exception will be thrown. | -| `node.execute(database, query, username=None, commit=True)` | Connects to PostgreSQL using `psycopg2` or `pg8000` (depends on which one is installed in your system) and returns two-dimensional array with data. | -| `node.connect(database='postgres')` | Returns connection wrapper (`NodeConnection`) capable of running several queries within a single transaction. | +| Command | Description | +|----------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------| +| `node.psql(dbname, query)` | Runs query via `psql` command and returns tuple `(error code, stdout, stderr)`. | +| `node.safe_psql(dbname, query)` | Same as `psql()` except that it returns only `stdout`. If an error occures during the execution, an exception will be thrown. | +| `node.execute(dbname, query)` | Connects to PostgreSQL using `psycopg2` or `pg8000` (depends on which one is installed in your system) and returns two-dimensional array with data. | +| `node.connect(dbname, username)` | Returns connection wrapper (`NodeConnection`) capable of running several queries within a single transaction. | The last one is the most powerful: you can use `begin(isolation_level)`, `commit()` and `rollback()`: ```python @@ -127,6 +127,7 @@ with testgres.get_new_node('master') as master: master.init().start() with master.backup() as backup: replica = backup.spawn_replica('replica').start() + replica.catchup() # catch up with master print(replica.execute('postgres', 'select 1')) ``` From f59150b7f90d190d16719409a24717b5073f061f Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Fri, 15 Dec 2017 14:58:10 +0300 Subject: [PATCH 030/475] improve catchup() api, don't use /dev/null in _execute_utility(), don't create user's dir in NodeBackup --- .gitignore | 1 + testgres/testgres.py | 77 ++++++++++++++++++++++---------------------- 2 files changed, 39 insertions(+), 39 deletions(-) diff --git a/.gitignore b/.gitignore index 0e2d40d1..fbf7b5a7 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ env/ venv/ build/ coverage.xml +*.swp diff --git a/testgres/testgres.py b/testgres/testgres.py index d6a253bb..ce48a387 100644 --- a/testgres/testgres.py +++ b/testgres/testgres.py @@ -302,10 +302,6 @@ def __init__(self, username = username or default_username() base_dir = base_dir or tempfile.mkdtemp() - # Create directory if needed - if base_dir and not os.path.exists(base_dir): - os.makedirs(base_dir) - # public self.original_node = node self.base_dir = base_dir @@ -1095,7 +1091,7 @@ def replicate(self, name, username=None, backup = self.backup(username=username, xlog_method=xlog_method) return backup.spawn_replica(name, use_logging=use_logging) - def catchup(self, username=None): + def catchup(self, dbname='postgres', username=None): """ Wait until async replica catches up with its master. """ @@ -1116,8 +1112,13 @@ def catchup(self, username=None): raise CatchUpException("Master node is not specified") try: - lsn = master.execute('postgres', poll_lsn)[0][0] - self.poll_query_until(dbname='postgres', + # fetch latest LSN + lsn = master.execute(dbname=dbname, + username=username, + query=poll_lsn)[0][0] + + # wait until this LSN reaches replica + self.poll_query_until(dbname=dbname, username=username, query=wait_lsn.format(lsn), max_attempts=0) # infinite @@ -1249,41 +1250,39 @@ def _execute_utility(util, args, logfile): stdout of executed utility. """ - # we can't use subprocess.DEVNULL on 2.7 - with open(os.devnull, "w") as devnull: - # run utility - process = subprocess.Popen([get_bin_path(util)] + args, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) + # run utility + process = subprocess.Popen([get_bin_path(util)] + args, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) - # get result - out, _ = process.communicate() + # get result + out, _ = process.communicate() - # write new log entry if possible - try: - with open(logfile, "a") as file_out: - # write util name and args - file_out.write(' '.join([util] + args)) - file_out.write('\n') - if out: - with open(logfile, "ab") as file_out: - # write output - file_out.write(out) - except IOError: - pass - - # decode output - out = '' if not out else out.decode('utf-8') - - if process.returncode: - error_text = ( - u"{} failed\n" - u"log:\n----\n{}\n" - ).format(util, out) - - raise ExecUtilException(error_text, process.returncode) + # write new log entry if possible + try: + with open(logfile, "a") as file_out: + # write util name and args + file_out.write(' '.join([util] + args)) + file_out.write('\n') + if out: + with open(logfile, "ab") as file_out: + # write output + file_out.write(out) + except IOError: + pass + + # decode output + out = '' if not out else out.decode('utf-8') + + if process.returncode: + error_text = ( + u"{} failed\n" + u"log:\n----\n{}\n" + ).format(util, out) - return out + raise ExecUtilException(error_text, process.returncode) + + return out def default_username(): From 5f08f2efa44d57629776d0dd1383c8f84aef26f9 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Fri, 15 Dec 2017 15:16:09 +0300 Subject: [PATCH 031/475] reformat code, add 'commit' arg to poll_query_until() --- testgres/testgres.py | 5 +++-- tests/test_simple.py | 24 ++++++++++++++++++------ 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/testgres/testgres.py b/testgres/testgres.py index ce48a387..a7f886d8 100644 --- a/testgres/testgres.py +++ b/testgres/testgres.py @@ -154,7 +154,7 @@ def run(self): # work until we're asked to stop while not self._stop_event.is_set(): # do we have new lines? - import select # used only here + import select # used only here if fd in select.select([fd], [], [], 0)[0]: for line in fd.readlines(): line = line.strip() @@ -982,6 +982,7 @@ def poll_query_until(self, max_attempts=0, sleep_time=1, expected=True, + commit=True, raise_programming_error=True, raise_internal_error=True): """ @@ -1009,7 +1010,7 @@ def poll_query_until(self, res = self.execute(dbname=dbname, query=query, username=username, - commit=True) + commit=commit) if expected is None and res is None: return # done diff --git a/tests/test_simple.py b/tests/test_simple.py index 386cfce8..4c600c87 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -360,7 +360,8 @@ def test_poll_query_until(self): check_time = 'select extract(epoch from now()) - {} >= 5' start_time = node.execute('postgres', get_time)[0][0] - node.poll_query_until('postgres', check_time.format(start_time)) + node.poll_query_until( + dbname='postgres', query=check_time.format(start_time)) end_time = node.execute('postgres', get_time)[0][0] self.assertTrue(end_time - start_time >= 5) @@ -368,15 +369,26 @@ def test_poll_query_until(self): # check 0 rows with self.assertRaises(QueryException): node.poll_query_until( - 'postgres', 'select * from pg_class where true = false') + dbname='postgres', + query='select * from pg_class where true = false') # check 0 columns with self.assertRaises(QueryException): - node.poll_query_until('postgres', - 'select from pg_class limit 1') - # check None + node.poll_query_until( + dbname='postgres', query='select from pg_class limit 1') + # check None, fail with self.assertRaises(QueryException): - node.poll_query_until('postgres', 'create table abc (val int)') + node.poll_query_until( + dbname='postgres', query='create table abc (val int)') + + # check None, ok + node.poll_query_until( + dbname='postgres', query='create table def()', + expected=None) # returns nothing + + # check arbitrary expected value + node.poll_query_until( + dbname='postgres', query='select 1', expected=1) # check timeout with self.assertRaises(TimeoutException): From 8434cc377a8160ebb69545b9e463b5aedf3868d0 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Fri, 15 Dec 2017 15:43:43 +0300 Subject: [PATCH 032/475] fixes in poll_query_until() --- testgres/testgres.py | 3 ++- tests/test_simple.py | 18 ++++++++++++++---- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/testgres/testgres.py b/testgres/testgres.py index a7f886d8..c8a15b82 100644 --- a/testgres/testgres.py +++ b/testgres/testgres.py @@ -996,6 +996,7 @@ def poll_query_until(self, max_attempts: how many times should we try? 0 == infinite sleep_time: how much should we sleep after a failure? expected: what should be returned to break the cycle? + commit: should (possible) changes be committed? raise_programming_error: mute ProgrammingError? raise_internal_error: mute InternalError? """ @@ -1024,7 +1025,7 @@ def poll_query_until(self, if len(res[0]) == 0: raise QueryException('Query returned 0 columns') - if res[0][0]: + if res[0][0] == expected: return # done except pglib.ProgrammingError as e: diff --git a/tests/test_simple.py b/tests/test_simple.py index 4c600c87..1dbfd407 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -376,6 +376,7 @@ def test_poll_query_until(self): with self.assertRaises(QueryException): node.poll_query_until( dbname='postgres', query='select from pg_class limit 1') + # check None, fail with self.assertRaises(QueryException): node.poll_query_until( @@ -386,17 +387,26 @@ def test_poll_query_until(self): dbname='postgres', query='create table def()', expected=None) # returns nothing - # check arbitrary expected value + # check arbitrary expected value, fail + with self.assertRaises(TimeoutException): + node.poll_query_until( + dbname='postgres', + query='select 3', + expected=1, + max_attempts=3, + sleep_time=0.01) + + # check arbitrary expected value, ok node.poll_query_until( - dbname='postgres', query='select 1', expected=1) + dbname='postgres', query='select 2', expected=2) # check timeout with self.assertRaises(TimeoutException): node.poll_query_until( dbname='postgres', query='select 1 > 2', - max_attempts=5, - sleep_time=0.2) + max_attempts=3, + sleep_time=0.01) def test_logging(self): logfile = tempfile.NamedTemporaryFile('w', delete=True) From c256c0c3b84b63368280713520f6486da03bbd28 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Wed, 20 Dec 2017 17:20:26 +0300 Subject: [PATCH 033/475] huge refactoring, split code into several submodules --- MANIFEST | 10 +- testgres/__init__.py | 21 +- testgres/api.py | 45 ++ testgres/backup.py | 152 ++++++ testgres/config.py | 29 ++ testgres/connection.py | 125 +++++ testgres/consts.py | 11 + testgres/exceptions.py | 47 ++ testgres/logger.py | 45 ++ testgres/{testgres.py => node.py} | 778 ++++-------------------------- testgres/utils.py | 197 ++++++++ tests/test_simple.py | 24 +- 12 files changed, 803 insertions(+), 681 deletions(-) create mode 100644 testgres/api.py create mode 100644 testgres/backup.py create mode 100644 testgres/config.py create mode 100644 testgres/connection.py create mode 100644 testgres/consts.py create mode 100644 testgres/exceptions.py create mode 100644 testgres/logger.py rename testgres/{testgres.py => node.py} (52%) create mode 100644 testgres/utils.py diff --git a/MANIFEST b/MANIFEST index 23b879da..f1f8c166 100644 --- a/MANIFEST +++ b/MANIFEST @@ -2,4 +2,12 @@ setup.cfg setup.py testgres/__init__.py -testgres/testgres.py +testgres/api.py +testgres/backup.py +testgres/config.py +testgres/connection.py +testgres/consts.py +testgres/exceptions.py +testgres/logger.py +testgres/node.py +testgres/utils.py diff --git a/testgres/__init__.py b/testgres/__init__.py index 414e2ac5..29b017a9 100644 --- a/testgres/__init__.py +++ b/testgres/__init__.py @@ -1 +1,20 @@ -from .testgres import * +from .api import get_new_node +from .backup import NodeBackup +from .config import TestgresConfig, configure_testgres + +from .connection import \ + IsolationLevel, \ + NodeConnection, \ + InternalError, \ + ProgrammingError + +from .exceptions import * +from .node import NodeStatus, PostgresNode + +from .utils import \ + reserve_port, \ + release_port, \ + bound_ports, \ + get_bin_path, \ + get_pg_config, \ + get_pg_version diff --git a/testgres/api.py b/testgres/api.py new file mode 100644 index 00000000..04874e7c --- /dev/null +++ b/testgres/api.py @@ -0,0 +1,45 @@ +# coding: utf-8 +""" +Testing framework for PostgreSQL and its extensions + +This module was created under influence of Postgres TAP test feature +(PostgresNode.pm module). It can manage Postgres clusters: initialize, +edit configuration files, start/stop cluster, execute queries. The +typical flow may look like: + + with get_new_node('test') as node: + node.init() + node.start() + result = node.psql('postgres', 'SELECT 1') + print(result) + node.stop() + + Or: + + with get_new_node('node1') as node1: + node1.init().start() + with node1.backup() as backup: + with backup.spawn_primary('node2') as node2: + res = node2.start().execute('postgres', 'select 2') + print(res) + +Copyright (c) 2016, Postgres Professional +""" + +from .node import PostgresNode + + +def get_new_node(name, base_dir=None, use_logging=False): + """ + Create a new node (select port automatically). + + Args: + name: node's name. + base_dir: path to node's data directory. + use_logging: enable python logging. + + Returns: + An instance of PostgresNode. + """ + + return PostgresNode(name=name, base_dir=base_dir, use_logging=use_logging) diff --git a/testgres/backup.py b/testgres/backup.py new file mode 100644 index 00000000..49160d37 --- /dev/null +++ b/testgres/backup.py @@ -0,0 +1,152 @@ +# coding: utf-8 + +import os +import shutil +import tempfile + +from .consts import \ + DATA_DIR as _DATA_DIR, \ + BACKUP_LOG_FILE as _BACKUP_LOG_FILE, \ + DEFAULT_XLOG_METHOD as _DEFAULT_XLOG_METHOD + +from .exceptions import \ + BackupException + +from .utils import \ + default_username as _default_username, \ + execute_utility as _execute_utility, \ + explain_exception as _explain_exception + + +class NodeBackup(object): + """ + Smart object responsible for backups + """ + + @property + def log_file(self): + return os.path.join(self.base_dir, _BACKUP_LOG_FILE) + + def __init__(self, + node, + base_dir=None, + username=None, + xlog_method=_DEFAULT_XLOG_METHOD): + + if not node.status(): + raise BackupException('Node must be running') + + # Set default arguments + username = username or _default_username() + base_dir = base_dir or tempfile.mkdtemp() + + # public + self.original_node = node + self.base_dir = base_dir + + # private + self._available = True + + data_dir = os.path.join(self.base_dir, _DATA_DIR) + _params = [ + "-D{}".format(data_dir), "-p{}".format(node.port), + "-U{}".format(username), "-X{}".format(xlog_method) + ] + _execute_utility("pg_basebackup", _params, self.log_file) + + def __enter__(self): + return self + + def __exit__(self, type, value, traceback): + self.cleanup() + + def _prepare_dir(self, destroy): + """ + Provide a data directory for a copy of node. + + Args: + destroy: should we convert this backup into a node? + + Returns: + Path to data directory. + """ + + if not self._available: + raise BackupException('Backup is exhausted') + + # Do we want to use this backup several times? + available = not destroy + + if available: + dest_base_dir = tempfile.mkdtemp() + + data1 = os.path.join(self.base_dir, _DATA_DIR) + data2 = os.path.join(dest_base_dir, _DATA_DIR) + + try: + # Copy backup to new data dir + shutil.copytree(data1, data2) + except Exception as e: + raise BackupException(_explain_exception(e)) + else: + dest_base_dir = self.base_dir + + # Is this backup exhausted? + self._available = available + + # Return path to new node + return dest_base_dir + + def spawn_primary(self, name, destroy=True, use_logging=False): + """ + Create a primary node from a backup. + + Args: + name: name for a new node. + destroy: should we convert this backup into a node? + use_logging: enable python logging. + + Returns: + New instance of PostgresNode. + """ + + base_dir = self._prepare_dir(destroy) + + # Build a new PostgresNode + from .node import PostgresNode + node = PostgresNode( + name=name, + base_dir=base_dir, + master=self.original_node, + use_logging=use_logging) + + # New nodes should always remove dir tree + node._should_rm_dirs = True + + node.append_conf("postgresql.conf", "\n") + node.append_conf("postgresql.conf", "port = {}".format(node.port)) + + return node + + def spawn_replica(self, name, destroy=True, use_logging=False): + """ + Create a replica of the original node from a backup. + + Args: + name: name for a new node. + destroy: should we convert this backup into a node? + use_logging: enable python logging. + + Returns: + New instance of PostgresNode. + """ + + node = self.spawn_primary(name, destroy, use_logging=use_logging) + node._create_recovery_conf(self.original_node) + + return node + + def cleanup(self): + if self._available: + shutil.rmtree(self.base_dir, ignore_errors=True) + self._available = False diff --git a/testgres/config.py b/testgres/config.py new file mode 100644 index 00000000..b0026ec4 --- /dev/null +++ b/testgres/config.py @@ -0,0 +1,29 @@ +# coding: utf-8 + + +class TestgresConfig: + """ + Global config (override default settings). + """ + + # shall we cache pg_config results? + cache_pg_config = True + + # shall we use cached initdb instance? + cache_initdb = True + + # shall we create a temp dir for cached initdb? + cached_initdb_dir = None + + # shall we remove EVERYTHING (including logs)? + node_cleanup_full = True + + +def configure_testgres(**options): + """ + Configure testgres. + Look at TestgresConfig to check what can be changed. + """ + + for key, option in options.items(): + setattr(TestgresConfig, key, option) diff --git a/testgres/connection.py b/testgres/connection.py new file mode 100644 index 00000000..8577dd7f --- /dev/null +++ b/testgres/connection.py @@ -0,0 +1,125 @@ +# coding: utf-8 + +# we support both pg8000 and psycopg2 +try: + import psycopg2 as pglib +except ImportError: + try: + import pg8000 as pglib + except ImportError: + raise ImportError("You must have psycopg2 or pg8000 modules installed") + +from enum import Enum + +from .exceptions import QueryException +from .utils import default_username as _default_username + +# export these exceptions +InternalError = pglib.InternalError +ProgrammingError = pglib.ProgrammingError + + +class IsolationLevel(Enum): + """ + Transaction isolation level for NodeConnection + """ + + ReadUncommitted, ReadCommitted, RepeatableRead, Serializable = range(4) + + +class NodeConnection(object): + """ + Transaction wrapper returned by Node + """ + + def __init__(self, + parent_node, + dbname, + host="127.0.0.1", + username=None, + password=None): + + # Use default user if not specified + username = username or _default_username() + + self.parent_node = parent_node + + self.connection = pglib.connect( + database=dbname, + user=username, + port=parent_node.port, + host=host, + password=password) + + self.cursor = self.connection.cursor() + + def __enter__(self): + return self + + def __exit__(self, type, value, traceback): + self.close() + + def begin(self, isolation_level=IsolationLevel.ReadCommitted): + # yapf: disable + levels = [ + 'read uncommitted', + 'read committed', + 'repeatable read', + 'serializable' + ] + + # Check if level is an IsolationLevel + if (isinstance(isolation_level, IsolationLevel)): + + # Get index of isolation level + level_idx = isolation_level.value + assert(level_idx in range(4)) + + # Replace isolation level with its name + isolation_level = levels[level_idx] + + else: + # Get name of isolation level + level_str = str(isolation_level).lower() + + # Validate level string + if level_str not in levels: + error = 'Invalid isolation level "{}"' + raise QueryException(error.format(level_str)) + + # Replace isolation level with its name + isolation_level = level_str + + # Set isolation level + cmd = 'SET TRANSACTION ISOLATION LEVEL {}' + self.cursor.execute(cmd.format(isolation_level)) + + return self + + def commit(self): + self.connection.commit() + + return self + + def rollback(self): + self.connection.rollback() + + return self + + def execute(self, query, *args): + self.cursor.execute(query, args) + + try: + res = self.cursor.fetchall() + + # pg8000 might return tuples + if isinstance(res, tuple): + res = [tuple(t) for t in res] + + return res + except Exception: + return None + + def close(self): + self.cursor.close() + self.connection.close() diff --git a/testgres/consts.py b/testgres/consts.py new file mode 100644 index 00000000..226959ca --- /dev/null +++ b/testgres/consts.py @@ -0,0 +1,11 @@ +# names for dirs in base_dir +DATA_DIR = "data" +LOGS_DIR = "logs" + +# names for log files +PG_LOG_FILE = "postgresql.log" +UTILS_LOG_FILE = "utils.log" +BACKUP_LOG_FILE = "backup.log" + +# default argument value +DEFAULT_XLOG_METHOD = "fetch" diff --git a/testgres/exceptions.py b/testgres/exceptions.py new file mode 100644 index 00000000..a5578759 --- /dev/null +++ b/testgres/exceptions.py @@ -0,0 +1,47 @@ +# coding: utf-8 + + +class TestgresException(Exception): + """ + Base exception + """ + + pass + + +class ExecUtilException(TestgresException): + """ + Stores exit code + """ + + def __init__(self, message, exit_code=0): + super(ExecUtilException, self).__init__(message) + self.exit_code = exit_code + + +class ClusterTestgresException(TestgresException): + pass + + +class QueryException(TestgresException): + pass + + +class TimeoutException(TestgresException): + pass + + +class StartNodeException(TestgresException): + pass + + +class InitNodeException(TestgresException): + pass + + +class BackupException(TestgresException): + pass + + +class CatchUpException(TestgresException): + pass diff --git a/testgres/logger.py b/testgres/logger.py new file mode 100644 index 00000000..20e1b60a --- /dev/null +++ b/testgres/logger.py @@ -0,0 +1,45 @@ +# coding: utf-8 + +import logging +import select +import threading +import time + + +class TestgresLogger(threading.Thread): + """ + Helper class to implement reading from postgresql.log + """ + + def __init__(self, node_name, log_file_name): + threading.Thread.__init__(self) + + self._node_name = node_name + self._log_file_name = log_file_name + self._stop_event = threading.Event() + self._logger = logging.getLogger(node_name) + self._logger.setLevel(logging.INFO) + + def run(self): + # open log file for reading + with open(self._log_file_name, 'r') as fd: + # work until we're asked to stop + while not self._stop_event.is_set(): + # do we have new lines? + if fd in select.select([fd], [], [], 0)[0]: + for line in fd.readlines(): + line = line.strip() + if line: + extra = {'node': self._node_name} + self._logger.info(line, extra=extra) + else: + time.sleep(0.1) + + # don't forget to clear event + self._stop_event.clear() + + def stop(self, wait=True): + self._stop_event.set() + + if wait: + self.join() diff --git a/testgres/testgres.py b/testgres/node.py similarity index 52% rename from testgres/testgres.py rename to testgres/node.py index c8a15b82..d8024ff2 100644 --- a/testgres/testgres.py +++ b/testgres/node.py @@ -1,417 +1,46 @@ # coding: utf-8 -""" -testgres.py - Postgres testing utility - -This module was created under influence of Postgres TAP test feature -(PostgresNode.pm module). It can manage Postgres clusters: initialize, -edit configuration files, start/stop cluster, execute queries. The -typical flow may look like: - - with get_new_node('test') as node: - node.init() - node.start() - result = node.psql('postgres', 'SELECT 1') - print(result) - node.stop() - - Or: - - with get_new_node('node1') as node1: - node1.init().start() - with node1.backup() as backup: - with backup.spawn_primary('node2') as node2: - res = node2.start().execute('postgres', 'select 2') - print(res) - -Copyright (c) 2016, Postgres Professional -""" import atexit -import logging import os import shutil -import six import subprocess import tempfile -import threading import time -import traceback - -import port_for from enum import Enum -from distutils.version import LooseVersion - -# Try to use psycopg2 by default. If psycopg2 isn't available then use -# pg8000 which is slower but much more portable because uses only -# pure-Python code -try: - import psycopg2 as pglib -except ImportError: - try: - import pg8000 as pglib - except ImportError: - raise ImportError("You must have psycopg2 or pg8000 modules installed") - -# ports used by nodes -bound_ports = set() - -# rows returned by PG_CONFIG -pg_config_data = {} - -PG_LOG_FILE = "postgresql.log" -UTILS_LOG_FILE = "utils.log" -BACKUP_LOG_FILE = "backup.log" - -DATA_DIR = "data" -LOGS_DIR = "logs" - -DEFAULT_XLOG_METHOD = "fetch" - - -class TestgresConfig: - """ - Global config (override default settings) - """ - - # shall we cache pg_config results? - cache_pg_config = True - - # shall we use cached initdb instance? - cache_initdb = True - - # shall we create a temp dir for cached initdb? - cached_initdb_dir = None - - # shall we remove EVERYTHING (including logs)? - node_cleanup_full = True - - -class TestgresException(Exception): - """ - Base exception - """ - - pass - - -class ExecUtilException(TestgresException): - """ - Stores exit code - """ - - def __init__(self, message, exit_code=0): - super(ExecUtilException, self).__init__(message) - self.exit_code = exit_code - - -class ClusterTestgresException(TestgresException): - pass - - -class QueryException(TestgresException): - pass - - -class TimeoutException(TestgresException): - pass - - -class StartNodeException(TestgresException): - pass - - -class InitNodeException(TestgresException): - pass - - -class BackupException(TestgresException): - pass - - -class CatchUpException(TestgresException): - pass - - -class TestgresLogger(threading.Thread): - """ - Helper class to implement reading from postgresql.log - """ - - def __init__(self, node_name, log_file_name): - threading.Thread.__init__(self) - - self._node_name = node_name - self._log_file_name = log_file_name - self._stop_event = threading.Event() - self._logger = logging.getLogger(node_name) - self._logger.setLevel(logging.INFO) - - def run(self): - # open log file for reading - with open(self._log_file_name, 'r') as fd: - # work until we're asked to stop - while not self._stop_event.is_set(): - # do we have new lines? - import select # used only here - if fd in select.select([fd], [], [], 0)[0]: - for line in fd.readlines(): - line = line.strip() - if line: - extra = {'node': self._node_name} - self._logger.info(line, extra=extra) - else: - time.sleep(0.1) - - # don't forget to clear event - self._stop_event.clear() - - def stop(self, wait=True): - self._stop_event.set() - - if wait: - self.join() - - -class IsolationLevel(Enum): - """ - Transaction isolation level for NodeConnection - """ - - ReadUncommitted, ReadCommitted, RepeatableRead, Serializable = range(4) - - -class NodeConnection(object): - """ - Transaction wrapper returned by Node - """ - - def __init__(self, - parent_node, - dbname, - host="127.0.0.1", - username=None, - password=None): - - # Use default user if not specified - username = username or default_username() - - self.parent_node = parent_node - - self.connection = pglib.connect( - database=dbname, - user=username, - port=parent_node.port, - host=host, - password=password) - - self.cursor = self.connection.cursor() - - def __enter__(self): - return self - - def __exit__(self, type, value, traceback): - self.close() - - def begin(self, isolation_level=IsolationLevel.ReadCommitted): - # yapf: disable - levels = [ - 'read uncommitted', - 'read committed', - 'repeatable read', - 'serializable' - ] - - # Check if level is an IsolationLevel - if (isinstance(isolation_level, IsolationLevel)): - - # Get index of isolation level - level_idx = isolation_level.value - assert(level_idx in range(4)) - - # Replace isolation level with its name - isolation_level = levels[level_idx] - - else: - # Get name of isolation level - level_str = str(isolation_level).lower() - - # Validate level string - if level_str not in levels: - error = 'Invalid isolation level "{}"' - raise QueryException(error.format(level_str)) - - # Replace isolation level with its name - isolation_level = level_str - - # Set isolation level - cmd = 'SET TRANSACTION ISOLATION LEVEL {}' - self.cursor.execute(cmd.format(isolation_level)) - - return self - - def commit(self): - self.connection.commit() - - return self - - def rollback(self): - self.connection.rollback() - - return self - - def execute(self, query, *args): - self.cursor.execute(query, args) - - try: - res = self.cursor.fetchall() - - # pg8000 might return tuples - if isinstance(res, tuple): - res = [tuple(t) for t in res] - - return res - except Exception: - return None - - def close(self): - self.cursor.close() - self.connection.close() - - -class NodeBackup(object): - """ - Smart object responsible for backups - """ - - @property - def log_file(self): - return os.path.join(self.base_dir, BACKUP_LOG_FILE) - - def __init__(self, - node, - base_dir=None, - username=None, - xlog_method=DEFAULT_XLOG_METHOD): - - if not node.status(): - raise BackupException('Node must be running') - - # Set default arguments - username = username or default_username() - base_dir = base_dir or tempfile.mkdtemp() - - # public - self.original_node = node - self.base_dir = base_dir - - # private - self._available = True - - data_dir = os.path.join(self.base_dir, DATA_DIR) - _params = [ - "-D{}".format(data_dir), - "-p{}".format(node.port), - "-U{}".format(username), - "-X{}".format(xlog_method) - ] - _execute_utility("pg_basebackup", _params, self.log_file) - - def __enter__(self): - return self - - def __exit__(self, type, value, traceback): - self.cleanup() - def _prepare_dir(self, destroy): - """ - Provide a data directory for a copy of node. - - Args: - destroy: should we convert this backup into a node? - - Returns: - Path to data directory. - """ - - if not self._available: - raise BackupException('Backup is exhausted') - - # Do we want to use this backup several times? - available = not destroy - - if available: - dest_base_dir = tempfile.mkdtemp() - - data1 = os.path.join(self.base_dir, DATA_DIR) - data2 = os.path.join(dest_base_dir, DATA_DIR) - - try: - # Copy backup to new data dir - shutil.copytree(data1, data2) - except Exception as e: - raise BackupException(_explain_exception(e)) - else: - dest_base_dir = self.base_dir - - # Is this backup exhausted? - self._available = available +from .config import TestgresConfig - # Return path to new node - return dest_base_dir +from .connection import \ + NodeConnection, \ + InternalError, \ + ProgrammingError - def spawn_primary(self, name, destroy=True, use_logging=False): - """ - Create a primary node from a backup. - - Args: - name: name for a new node. - destroy: should we convert this backup into a node? - use_logging: enable python logging. - - Returns: - New instance of PostgresNode. - """ - - base_dir = self._prepare_dir(destroy) +from .consts import \ + DATA_DIR as _DATA_DIR, \ + LOGS_DIR as _LOGS_DIR, \ + PG_LOG_FILE as _PG_LOG_FILE, \ + UTILS_LOG_FILE as _UTILS_LOG_FILE, \ + DEFAULT_XLOG_METHOD as _DEFAULT_XLOG_METHOD - # Build a new PostgresNode - node = PostgresNode(name=name, - base_dir=base_dir, - master=self.original_node, - use_logging=use_logging) +from .exceptions import \ + CatchUpException, \ + ExecUtilException, \ + InitNodeException, \ + QueryException, \ + StartNodeException, \ + TimeoutException - # New nodes should always remove dir tree - node._should_rm_dirs = True +from .logger import TestgresLogger - node.append_conf("postgresql.conf", "\n") - node.append_conf("postgresql.conf", "port = {}".format(node.port)) - - return node - - def spawn_replica(self, name, destroy=True, use_logging=False): - """ - Create a replica of the original node from a backup. - - Args: - name: name for a new node. - destroy: should we convert this backup into a node? - use_logging: enable python logging. - - Returns: - New instance of PostgresNode. - """ - - node = self.spawn_primary(name, destroy, use_logging=use_logging) - node._create_recovery_conf(self.original_node) - - return node - - def cleanup(self): - if self._available: - shutil.rmtree(self.base_dir, ignore_errors=True) - self._available = False +from .utils import \ + get_bin_path, \ + get_pg_version, \ + pg_version_ge as _pg_version_ge, \ + reserve_port as _reserve_port, \ + release_port as _release_port, \ + execute_utility as _execute_utility, \ + explain_exception as _explain_exception class NodeStatus(Enum): @@ -442,7 +71,7 @@ def __init__(self, self.master = master self.name = name self.host = '127.0.0.1' - self.port = port or reserve_port() + self.port = port or _reserve_port() self.base_dir = base_dir # private @@ -468,29 +97,27 @@ def __exit__(self, type, value, traceback): @property def data_dir(self): - return os.path.join(self.base_dir, DATA_DIR) + return os.path.join(self.base_dir, _DATA_DIR) @property def logs_dir(self): - return os.path.join(self.base_dir, LOGS_DIR) + return os.path.join(self.base_dir, _LOGS_DIR) @property def utils_log_name(self): - return os.path.join(self.logs_dir, UTILS_LOG_FILE) + return os.path.join(self.logs_dir, _UTILS_LOG_FILE) @property def pg_log_name(self): - return os.path.join(self.data_dir, PG_LOG_FILE) + return os.path.join(self.data_dir, _PG_LOG_FILE) @property def connstr(self): return "port={}".format(self.port) def _create_recovery_conf(self, root_node): - line = ( - "primary_conninfo='{} application_name={}'\n" - "standby_mode=on\n" - ).format(root_node.connstr, self.name) + line = ("primary_conninfo='{} application_name={}'\n" + "standby_mode=on\n").format(root_node.connstr, self.name) self.append_conf("recovery.conf", line) @@ -535,14 +162,14 @@ def print_node_file(node_file): return "### file not found ###\n" error_text = ( - u"{}:\n----\n{}\n" # log file, e.g. postgresql.log - u"{}:\n----\n{}\n" # postgresql.conf - u"{}:\n----\n{}\n" # pg_hba.conf - u"{}:\n----\n{}\n" # recovery.conf - ).format(log_filename, print_node_file(log_filename), - conf_filename, print_node_file(conf_filename), - hba_filename, print_node_file(hba_filename), - recovery_filename, print_node_file(recovery_filename)) + u"{}:\n----\n{}\n" # log file, e.g. postgresql.log + u"{}:\n----\n{}\n" # postgresql.conf + u"{}:\n----\n{}\n" # pg_hba.conf + u"{}:\n----\n{}\n" # recovery.conf + ).format(log_filename, print_node_file(log_filename), conf_filename, + print_node_file(conf_filename), hba_filename, + print_node_file(hba_filename), recovery_filename, + print_node_file(recovery_filename)) return error_text @@ -617,7 +244,8 @@ def get_auth_method(t): new_lines = [ "local\treplication\tall\t\t\t{}\n".format(auth_local), - "host\treplication\tall\t127.0.0.1/32\t{}\n".format(auth_host), + "host\treplication\tall\t127.0.0.1/32\t{}\n".format( + auth_host), "host\treplication\tall\t::1/128\t\t{}\n".format(auth_host) ] @@ -633,26 +261,25 @@ def get_auth_method(t): conf.write("log_statement = {}\n" "listen_addresses = '{}'\n" - "port = {}\n".format(log_statement, - self.host, + "port = {}\n".format(log_statement, self.host, self.port)) # replication-related settings if allow_streaming: - cur_ver = LooseVersion(get_pg_version()) - min_ver = LooseVersion('9.6') # select a proper wal_level for PostgreSQL - wal_level = "hot_standby" if cur_ver < min_ver else "replica" + if _pg_version_ge('9.6'): + wal_level = "replica" + else: + wal_level = "hot_standby" max_wal_senders = 5 wal_keep_segments = 20 conf.write("hot_standby = on\n" "max_wal_senders = {}\n" "wal_keep_segments = {}\n" - "wal_level = {}\n".format(max_wal_senders, - wal_keep_segments, - wal_level)) + "wal_level = {}\n".format( + max_wal_senders, wal_keep_segments, wal_level)) return self @@ -713,13 +340,10 @@ def get_control_data(self): Return contents of pg_control file. """ - cur_ver = LooseVersion(get_pg_version()) - min_ver = LooseVersion('9.5') - - if cur_ver < min_ver: - _params = [self.data_dir] - else: + if _pg_version_ge('9.5'): _params = ["-D", self.data_dir] + else: + _params = [self.data_dir] data = _execute_utility("pg_controldata", _params, self.utils_log_name) @@ -743,10 +367,8 @@ def start(self, params=[]): """ _params = [ - "start", - "-D{}".format(self.data_dir), - "-l{}".format(self.pg_log_name), - "-w" + "start", "-D{}".format(self.data_dir), "-l{}".format( + self.pg_log_name), "-w" ] + params try: @@ -754,7 +376,7 @@ def start(self, params=[]): except ExecUtilException as e: msg = ( u"Cannot start node\n" - u"{}\n" # pg_ctl log + u"{}\n" # pg_ctl log ).format(self._format_verbose_error()) raise StartNodeException(msg) @@ -792,10 +414,8 @@ def restart(self, params=[]): """ _params = [ - "restart", - "-D{}".format(self.data_dir), - "-l{}".format(self.pg_log_name), - "-w" + "restart", "-D{}".format(self.data_dir), "-l{}".format( + self.pg_log_name), "-w" ] + params try: @@ -803,7 +423,7 @@ def restart(self, params=[]): except ExecUtilException as e: msg = ( u"Cannot restart node\n" - u"{}\n" # pg_ctl log + u"{}\n" # pg_ctl log ).format(self._format_verbose_error()) raise StartNodeException(msg) @@ -839,7 +459,7 @@ def free_port(self): """ if self._should_free_port: - release_port(self.port) + _release_port(self.port) def cleanup(self, max_attempts=3): """ @@ -855,11 +475,11 @@ def cleanup(self, max_attempts=3): while attempts < max_attempts: try: self.stop() - break # OK + break # OK except ExecUtilException as e: - pass # one more time + pass # one more time except Exception as e: - break # screw this + break # screw this attempts += 1 @@ -868,15 +488,16 @@ def cleanup(self, max_attempts=3): # choose directory to be removed if TestgresConfig.node_cleanup_full: - rm_dir = self.base_dir # everything + rm_dir = self.base_dir # everything else: - rm_dir = self.data_dir # just data, save logs + rm_dir = self.data_dir # just data, save logs shutil.rmtree(rm_dir, ignore_errors=True) return self - def psql(self, dbname, query=None, filename=None, username=None, input=None): + def psql(self, dbname, query=None, filename=None, username=None, + input=None): """ Execute a query using psql. @@ -892,10 +513,7 @@ def psql(self, dbname, query=None, filename=None, username=None, input=None): psql = get_bin_path("psql") psql_params = [ - psql, - "-XAtq", - "-h{}".format(self.host), - "-p{}".format(self.port), + psql, "-XAtq", "-h{}".format(self.host), "-p{}".format(self.port), dbname ] @@ -911,10 +529,11 @@ def psql(self, dbname, query=None, filename=None, username=None, input=None): psql_params.extend(("-U", username)) # start psql process - process = subprocess.Popen(psql_params, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + process = subprocess.Popen( + psql_params, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) # wait until it finishes and get stdout and stderr out, err = process.communicate(input=input) @@ -954,11 +573,7 @@ def dump(self, dbname, filename=None): f, filename = filename or tempfile.mkstemp() os.close(f) - _params = [ - "-p{}".format(self.port), - "-f{}".format(filename), - dbname - ] + _params = ["-p{}".format(self.port), "-f{}".format(filename), dbname] _execute_utility("pg_dump", _params, self.utils_log_name) @@ -1002,19 +617,20 @@ def poll_query_until(self, """ # sanity checks - assert(max_attempts >= 0) - assert(sleep_time > 0) + assert (max_attempts >= 0) + assert (sleep_time > 0) attempts = 0 while max_attempts == 0 or attempts < max_attempts: try: - res = self.execute(dbname=dbname, - query=query, - username=username, - commit=commit) + res = self.execute( + dbname=dbname, + query=query, + username=username, + commit=commit) if expected is None and res is None: - return # done + return # done if res is None: raise QueryException('Query returned None') @@ -1026,13 +642,13 @@ def poll_query_until(self, raise QueryException('Query returned 0 columns') if res[0][0] == expected: - return # done + return # done - except pglib.ProgrammingError as e: + except ProgrammingError as e: if raise_programming_error: raise e - except pglib.InternalError as e: + except InternalError as e: if raise_internal_error: raise e @@ -1061,7 +677,7 @@ def execute(self, dbname, query, username=None, commit=True): node_con.commit() return res - def backup(self, username=None, xlog_method=DEFAULT_XLOG_METHOD): + def backup(self, username=None, xlog_method=_DEFAULT_XLOG_METHOD): """ Perform pg_basebackup. @@ -1073,12 +689,13 @@ def backup(self, username=None, xlog_method=DEFAULT_XLOG_METHOD): A smart object of type NodeBackup. """ - return NodeBackup(node=self, - username=username, - xlog_method=xlog_method) + from .backup import NodeBackup + return NodeBackup(node=self, username=username, xlog_method=xlog_method) - def replicate(self, name, username=None, - xlog_method=DEFAULT_XLOG_METHOD, + def replicate(self, + name, + username=None, + xlog_method=_DEFAULT_XLOG_METHOD, use_logging=False): """ Create a binary replica of this node. @@ -1100,10 +717,7 @@ def catchup(self, dbname='postgres', username=None): master = self.master - cur_ver = LooseVersion(get_pg_version()) - min_ver = LooseVersion('10') - - if cur_ver >= min_ver: + if _pg_version_ge('10'): poll_lsn = "select pg_current_wal_lsn()::text" wait_lsn = "select pg_last_wal_replay_lsn() >= '{}'::pg_lsn" else: @@ -1115,15 +729,15 @@ def catchup(self, dbname='postgres', username=None): try: # fetch latest LSN - lsn = master.execute(dbname=dbname, - username=username, - query=poll_lsn)[0][0] + lsn = master.execute( + dbname=dbname, username=username, query=poll_lsn)[0][0] # wait until this LSN reaches replica - self.poll_query_until(dbname=dbname, - username=username, - query=wait_lsn.format(lsn), - max_attempts=0) # infinite + self.poll_query_until( + dbname=dbname, + username=username, + query=wait_lsn.format(lsn), + max_attempts=0) # infinite except Exception as e: raise CatchUpException(_explain_exception(e)) @@ -1140,11 +754,8 @@ def pgbench_init(self, dbname='postgres', scale=1, options=[]): This instance of PostgresNode. """ - _params = [ - "-i", - "-s{}".format(scale), - "-p{}".format(self.port) - ] + options + [dbname] + _params = ["-i", "-s{}".format(scale), "-p{}".format(self.port) + ] + options + [dbname] _execute_utility("pgbench", _params, self.utils_log_name) @@ -1182,18 +793,8 @@ def connect(self, dbname='postgres', username=None): An instance of NodeConnection. """ - return NodeConnection(parent_node=self, - dbname=dbname, - username=username) - - -def _explain_exception(e): - """ - Use this function instead of str(e). - """ - - lines = traceback.format_exception_only(type(e), e) - return ''.join(lines) + return NodeConnection( + parent_node=self, dbname=dbname, username=username) def _cached_initdb(data_dir, initdb_logfile, initdb_params=[]): @@ -1215,6 +816,7 @@ def call_initdb(_data_dir): else: # Set default temp dir for cached initdb if TestgresConfig.cached_initdb_dir is None: + def rm_cached_data_dir(rm_dir): shutil.rmtree(rm_dir, ignore_errors=True) @@ -1237,171 +839,3 @@ def rm_cached_data_dir(rm_dir): shutil.copytree(cached_data_dir, data_dir) except Exception as e: raise InitNodeException(_explain_exception(e)) - - -def _execute_utility(util, args, logfile): - """ - Execute utility (pg_ctl, pg_dump etc) using get_bin_path(). - - Args: - util: utility to be executed. - args: arguments for utility (list). - logfile: path to file to store stdout and stderr. - - Returns: - stdout of executed utility. - """ - - # run utility - process = subprocess.Popen([get_bin_path(util)] + args, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) - - # get result - out, _ = process.communicate() - - # write new log entry if possible - try: - with open(logfile, "a") as file_out: - # write util name and args - file_out.write(' '.join([util] + args)) - file_out.write('\n') - if out: - with open(logfile, "ab") as file_out: - # write output - file_out.write(out) - except IOError: - pass - - # decode output - out = '' if not out else out.decode('utf-8') - - if process.returncode: - error_text = ( - u"{} failed\n" - u"log:\n----\n{}\n" - ).format(util, out) - - raise ExecUtilException(error_text, process.returncode) - - return out - - -def default_username(): - """ - Return current user. - """ - - import pwd # used only here - return pwd.getpwuid(os.getuid())[0] - - -def get_bin_path(filename): - """ - Return full path to an executable using PG_BIN or PG_CONFIG. - """ - - pg_bin_path = os.environ.get("PG_BIN") - - if pg_bin_path: - return os.path.join(pg_bin_path, filename) - - pg_config = get_pg_config() - - if pg_config and "BINDIR" in pg_config: - return os.path.join(pg_config["BINDIR"], filename) - - return filename - - -def get_pg_version(): - """ - Return PostgreSQL version using PG_BIN or PG_CONFIG. - """ - - pg_bin_path = os.environ.get("PG_BIN") - - if pg_bin_path: - _params = ['--version'] - raw_ver = _execute_utility('psql', _params, os.devnull) - else: - raw_ver = get_pg_config()["VERSION"] - - # Cook version of PostgreSQL - version = raw_ver.strip().split(" ")[-1] \ - .partition('devel')[0] \ - .partition('beta')[0] \ - .partition('rc')[0] - - return version - - -def reserve_port(): - """ - Generate a new port and add it to 'bound_ports'. - """ - - port = port_for.select_random(exclude_ports=bound_ports) - bound_ports.add(port) - - return port - - -def release_port(port): - """ - Free port provided by reserve_port(). - """ - - bound_ports.remove(port) - - -def get_pg_config(): - """ - Return output of pg_config. - """ - - global pg_config_data - - if TestgresConfig.cache_pg_config and pg_config_data: - return pg_config_data - - data = {} - pg_config_cmd = os.environ.get("PG_CONFIG") or "pg_config" - out = six.StringIO(subprocess.check_output([pg_config_cmd], - universal_newlines=True)) - for line in out: - if line and "=" in line: - key, value = line.split("=", 1) - data[key.strip()] = value.strip() - - if TestgresConfig.cache_pg_config: - pg_config_data.clear() - pg_config_data.update(data) - - return data - - -def get_new_node(name, base_dir=None, use_logging=False): - """ - Create a new node (select port automatically). - - Args: - name: node's name. - base_dir: path to node's data directory. - use_logging: enable python logging. - - Returns: - An instance of PostgresNode. - """ - - return PostgresNode(name=name, base_dir=base_dir, use_logging=use_logging) - - -def configure_testgres(**options): - """ - Configure testgres. - Look at TestgresConfig to check what can be changed. - """ - - for key, option in options.items(): - setattr(TestgresConfig, key, option) diff --git a/testgres/utils.py b/testgres/utils.py new file mode 100644 index 00000000..2421c557 --- /dev/null +++ b/testgres/utils.py @@ -0,0 +1,197 @@ +# coding: utf-8 + +import os +import port_for +import six +import subprocess + +from distutils.version import LooseVersion + +from .config import TestgresConfig +from .exceptions import ExecUtilException + +# rows returned by PG_CONFIG +_pg_config_data = {} + +# ports used by nodes +bound_ports = set() + + +def reserve_port(): + """ + Generate a new port and add it to '_bound_ports'. + """ + + global bound_ports + port = port_for.select_random(exclude_ports=bound_ports) + bound_ports.add(port) + + return port + + +def release_port(port): + """ + Free port provided by reserve_port(). + """ + + global bound_ports + bound_ports.remove(port) + + +def default_username(): + """ + Return current user. + """ + + import pwd # used only here + return pwd.getpwuid(os.getuid())[0] + + +def explain_exception(e): + """ + Use this function instead of str(e). + """ + + import traceback # used only here + lines = traceback.format_exception_only(type(e), e) + return ''.join(lines) + + +def execute_utility(util, args, logfile): + """ + Execute utility (pg_ctl, pg_dump etc). + NOTE: 'util' is transformed using get_bin_path(). + + Args: + util: utility to be executed. + args: arguments for utility (list). + logfile: path to file to store stdout and stderr. + + Returns: + stdout of executed utility. + """ + + # run utility + process = subprocess.Popen( + [get_bin_path(util)] + args, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + + # get result + out, _ = process.communicate() + + # write new log entry if possible + try: + with open(logfile, "a") as file_out: + # write util name and args + file_out.write(' '.join([util] + args)) + file_out.write('\n') + if out: + with open(logfile, "ab") as file_out: + # write output + file_out.write(out) + except IOError: + pass + + # decode output + out = '' if not out else out.decode('utf-8') + + # format exception, if needed + error_code = process.returncode + if error_code: + error_text = ( + u"{} failed with exit code {}\n" + u"log:\n----\n{}\n" + ).format(util, error_code, out) + + raise ExecUtilException(error_text, error_code) + + return out + + +def get_bin_path(filename): + """ + Return absolute path to an executable using PG_BIN or PG_CONFIG. + This function does nothing if 'filename' is already absolute. + """ + + # check if it's already absolute + if os.path.isabs(filename): + return filename + + # try PG_BIN + pg_bin = os.environ.get("PG_BIN") + if pg_bin: + return os.path.join(pg_bin, filename) + + # try PG_CONFIG + pg_config = get_pg_config() + if pg_config and "BINDIR" in pg_config: + return os.path.join(pg_config["BINDIR"], filename) + + return filename + + +def get_pg_config(): + """ + Return output of pg_config. + NOTE: this fuction caches the result by default (see TestgresConfig). + """ + + global _pg_config_data + + # return cached data, if allowed to + if TestgresConfig.cache_pg_config and _pg_config_data: + return _pg_config_data + + # execute pg_config and get the output + pg_config_cmd = os.environ.get("PG_CONFIG") or "pg_config" + out = subprocess.check_output([pg_config_cmd]).decode('utf-8') + + data = {} + for line in out.splitlines(): + if line and "=" in line: + key, value = line.split("=", 1) + data[key.strip()] = value.strip() + + if TestgresConfig.cache_pg_config: + # cache data + _pg_config_data = data + else: + # clear for clarity + _pg_config_data.clear() + + return data + + +def get_pg_version(): + """ + Return PostgreSQL version using PG_BIN or PG_CONFIG. + """ + + pg_bin = os.environ.get("PG_BIN") + + if pg_bin: + # there might be no pg_config installed, try this first + raw_ver = execute_utility('psql', ['--version'], os.devnull) + else: + raw_ver = get_pg_config()['VERSION'] + + # cook version of PostgreSQL + version = raw_ver.strip().split(' ')[-1] \ + .partition('devel')[0] \ + .partition('beta')[0] \ + .partition('rc')[0] + + return version + + +def pg_version_ge(version): + """ + Check if PostgreSQL is 'version' or newer. + """ + + cur_ver = LooseVersion(get_pg_version()) + min_ver = LooseVersion(version) + + return cur_ver >= min_ver diff --git a/tests/test_simple.py b/tests/test_simple.py index 1dbfd407..37cf03b3 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -23,6 +23,7 @@ CatchUpException, \ TimeoutException +from testgres import TestgresConfig from testgres import get_new_node, get_pg_config, configure_testgres from testgres import bound_ports from testgres import NodeStatus @@ -513,18 +514,27 @@ def test_version_management(self): self.assertTrue(b > c) self.assertTrue(a > c) - def test_configure(self): + def test_config(self): # set global if it wasn't set pg_config = get_pg_config() configure_testgres(cache_initdb=True, cache_pg_config=True) - # check that is the same instance - self.assertEqual(id(get_pg_config()), id(testgres.pg_config_data)) - configure_testgres(cache_initdb=True, cache_pg_config=False) - self.assertNotEqual(id(get_pg_config()), id(testgres.pg_config_data)) + # check same instances + a = get_pg_config() + b = get_pg_config() + self.assertEqual(id(a), id(b)) - # return to the base state - configure_testgres(cache_initdb=True, cache_pg_config=True) + # modify setting + configure_testgres(cache_pg_config=False) + self.assertFalse(TestgresConfig.cache_pg_config) + + # check different instances + a = get_pg_config() + b = get_pg_config() + self.assertNotEqual(id(a), id(b)) + + # restore setting + configure_testgres(cache_pg_config=True) def test_isolation_levels(self): with get_new_node('node').init().start() as node: From 18fccb640491efc4a9312141df856c0530b6f4e2 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Wed, 20 Dec 2017 19:05:45 +0300 Subject: [PATCH 034/475] fix unicode one more time --- testgres/consts.py | 2 ++ testgres/node.py | 56 +++++++++++++++++++++++++--------------------- testgres/utils.py | 6 ++--- 3 files changed, 35 insertions(+), 29 deletions(-) diff --git a/testgres/consts.py b/testgres/consts.py index 226959ca..69b51778 100644 --- a/testgres/consts.py +++ b/testgres/consts.py @@ -1,3 +1,5 @@ +# coding: utf-8 + # names for dirs in base_dir DATA_DIR = "data" LOGS_DIR = "logs" diff --git a/testgres/node.py b/testgres/node.py index d8024ff2..8d751b73 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -1,6 +1,7 @@ # coding: utf-8 import atexit +import io import os import shutil import subprocess @@ -155,21 +156,22 @@ def _format_verbose_error(self): def print_node_file(node_file): if os.path.exists(node_file): try: - with open(node_file, 'r') as f: - return f.read() + with io.open(node_file, "r") as f: + return f.read().decode('utf-8') except Exception as e: pass return "### file not found ###\n" + # yapf: disable error_text = ( u"{}:\n----\n{}\n" # log file, e.g. postgresql.log u"{}:\n----\n{}\n" # postgresql.conf u"{}:\n----\n{}\n" # pg_hba.conf u"{}:\n----\n{}\n" # recovery.conf - ).format(log_filename, print_node_file(log_filename), conf_filename, - print_node_file(conf_filename), hba_filename, - print_node_file(hba_filename), recovery_filename, - print_node_file(recovery_filename)) + ).format(log_filename, print_node_file(log_filename), + conf_filename, print_node_file(conf_filename), + hba_filename, print_node_file(hba_filename), + recovery_filename, print_node_file(recovery_filename)) return error_text @@ -219,7 +221,7 @@ def default_conf(self, hba_conf = os.path.join(self.data_dir, "pg_hba.conf") # filter lines in hba file - with open(hba_conf, "r+") as conf: + with io.open(hba_conf, "r+") as conf: # get rid of comments and blank lines lines = [ s for s in conf.readlines() @@ -242,11 +244,11 @@ def get_auth_method(t): auth_local = get_auth_method('local') auth_host = get_auth_method('host') + # yapf: disable new_lines = [ - "local\treplication\tall\t\t\t{}\n".format(auth_local), - "host\treplication\tall\t127.0.0.1/32\t{}\n".format( - auth_host), - "host\treplication\tall\t::1/128\t\t{}\n".format(auth_host) + u"local\treplication\tall\t\t\t{}\n".format(auth_local), + u"host\treplication\tall\t127.0.0.1/32\t{}\n".format(auth_host), + u"host\treplication\tall\t::1/128\t\t{}\n".format(auth_host) ] # write missing lines @@ -255,14 +257,16 @@ def get_auth_method(t): conf.write(line) # overwrite postgresql.conf file - with open(postgres_conf, "w") as conf: + with io.open(postgres_conf, "w") as conf: if not fsync: - conf.write("fsync = off\n") + conf.write(u"fsync = off\n") - conf.write("log_statement = {}\n" - "listen_addresses = '{}'\n" - "port = {}\n".format(log_statement, self.host, - self.port)) + # yapf: disable + conf.write(u"log_statement = {}\n" + u"listen_addresses = '{}'\n" + u"port = {}\n".format(log_statement, + self.host, + self.port)) # replication-related settings if allow_streaming: @@ -273,13 +277,15 @@ def get_auth_method(t): else: wal_level = "hot_standby" + # yapf: disable max_wal_senders = 5 wal_keep_segments = 20 - conf.write("hot_standby = on\n" - "max_wal_senders = {}\n" - "wal_keep_segments = {}\n" - "wal_level = {}\n".format( - max_wal_senders, wal_keep_segments, wal_level)) + conf.write(u"hot_standby = on\n" + u"max_wal_senders = {}\n" + u"wal_keep_segments = {}\n" + u"wal_level = {}\n".format(max_wal_senders, + wal_keep_segments, + wal_level)) return self @@ -296,8 +302,8 @@ def append_conf(self, filename, string): """ config_name = os.path.join(self.data_dir, filename) - with open(config_name, "a") as conf: - conf.write(''.join([string, '\n'])) + with io.open(config_name, "a") as conf: + conf.write(u"".join([string, '\n'])) return self @@ -329,7 +335,7 @@ def get_pid(self): """ if self.status(): - with open(os.path.join(self.data_dir, 'postmaster.pid')) as f: + with io.open(os.path.join(self.data_dir, 'postmaster.pid')) as f: return int(f.readline()) # for clarity diff --git a/testgres/utils.py b/testgres/utils.py index 2421c557..10db83fd 100644 --- a/testgres/utils.py +++ b/testgres/utils.py @@ -99,10 +99,8 @@ def execute_utility(util, args, logfile): # format exception, if needed error_code = process.returncode if error_code: - error_text = ( - u"{} failed with exit code {}\n" - u"log:\n----\n{}\n" - ).format(util, error_code, out) + error_text = (u"{} failed with exit code {}\n" + u"log:\n----\n{}\n").format(util, error_code, out) raise ExecUtilException(error_text, error_code) From 33ec26047ac1e0e7676b2ee0fc43836225355701 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Wed, 20 Dec 2017 19:21:54 +0300 Subject: [PATCH 035/475] more tests for poll_query_until() --- tests/test_simple.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/test_simple.py b/tests/test_simple.py index 37cf03b3..59a16930 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -409,6 +409,21 @@ def test_poll_query_until(self): max_attempts=3, sleep_time=0.01) + # check ProgrammingError, fail + with self.assertRaises(testgres.ProgrammingError): + node.poll_query_until( + dbname='postgres', + query='dummy1') + + # check ProgrammingError, ok + with self.assertRaises(TimeoutException): + node.poll_query_until( + dbname='postgres', + query='dummy2', + max_attempts=3, + sleep_time=0.01, + raise_programming_error=False) + def test_logging(self): logfile = tempfile.NamedTemporaryFile('w', delete=True) From 8449b878e378805ed1b67815fb7f9beca09eb647 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Wed, 20 Dec 2017 22:44:43 +0300 Subject: [PATCH 036/475] make use of 'io', comments --- testgres/utils.py | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/testgres/utils.py b/testgres/utils.py index 10db83fd..8f149526 100644 --- a/testgres/utils.py +++ b/testgres/utils.py @@ -1,5 +1,6 @@ # coding: utf-8 +import io import os import port_for import six @@ -10,6 +11,7 @@ from .config import TestgresConfig from .exceptions import ExecUtilException + # rows returned by PG_CONFIG _pg_config_data = {} @@ -77,25 +79,26 @@ def execute_utility(util, args, logfile): stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - # get result + # get result and decode it out, _ = process.communicate() + out = '' if not out else out.decode('utf-8') # write new log entry if possible try: - with open(logfile, "a") as file_out: - # write util name and args - file_out.write(' '.join([util] + args)) - file_out.write('\n') - if out: - with open(logfile, "ab") as file_out: - # write output + with io.open(logfile, 'a') as file_out: + # write util's name and args + file_out.write(u' '.join([util] + args)) + + # write output + if out: + file_out.write(u'\n') file_out.write(out) + + # finally, a separator + file_out.write(u'\n') except IOError: pass - # decode output - out = '' if not out else out.decode('utf-8') - # format exception, if needed error_code = process.returncode if error_code: @@ -173,6 +176,7 @@ def get_pg_version(): # there might be no pg_config installed, try this first raw_ver = execute_utility('psql', ['--version'], os.devnull) else: + # ok, we have no other choice raw_ver = get_pg_config()['VERSION'] # cook version of PostgreSQL From 03edc2eb67e7c04a49f10e097b714bc53d587825 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Wed, 20 Dec 2017 22:46:24 +0300 Subject: [PATCH 037/475] remove unused module --- testgres/utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/testgres/utils.py b/testgres/utils.py index 8f149526..f3ed1725 100644 --- a/testgres/utils.py +++ b/testgres/utils.py @@ -3,7 +3,6 @@ import io import os import port_for -import six import subprocess from distutils.version import LooseVersion From 84f033ec58de0090a177c8d109da28102513168f Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Thu, 21 Dec 2017 13:47:56 +0300 Subject: [PATCH 038/475] fixes, per review by @ildus --- testgres/backup.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/testgres/backup.py b/testgres/backup.py index 49160d37..fef9b851 100644 --- a/testgres/backup.py +++ b/testgres/backup.py @@ -9,8 +9,7 @@ BACKUP_LOG_FILE as _BACKUP_LOG_FILE, \ DEFAULT_XLOG_METHOD as _DEFAULT_XLOG_METHOD -from .exceptions import \ - BackupException +from .exceptions import BackupException from .utils import \ default_username as _default_username, \ From f79efe0c524bb4733984359b61147a66062b4002 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Thu, 21 Dec 2017 15:24:49 +0300 Subject: [PATCH 039/475] attempt to fix codecov --- run_tests.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/run_tests.sh b/run_tests.sh index 50d35ef5..c6aa3094 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -35,5 +35,8 @@ coverage run tests/test_simple.py # show coverage coverage report +# attempt to fix codecov +set +eux + # send coverage stats to Codecov bash <(curl -s https://codecov.io/bash) From 8ff834915d77db5f8629ba7514c1c92744af15a1 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Fri, 22 Dec 2017 14:53:28 +0300 Subject: [PATCH 040/475] remove six from dependencies --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ba7fbf90..ecfe2a27 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ from distutils.core import setup # Basic dependencies -install_requires = ["pg8000", "six", "port-for"] +install_requires = ["pg8000", "port-for"] # Add compatibility enum class if sys.version_info < (3, 4): From 53c79f50f77091b597ca8fbc41eddb92ceccf02b Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Sat, 23 Dec 2017 02:46:07 +0300 Subject: [PATCH 041/475] refactoring, fix handling of PG_BIN & PG_CONFIG, fix test_reload() --- run_tests.sh | 29 +++++++- testgres/backup.py | 4 +- testgres/node.py | 167 +++++++++++++++++++++++++++++++------------ testgres/utils.py | 87 ++++++++++++---------- tests/test_simple.py | 19 +++-- 5 files changed, 213 insertions(+), 93 deletions(-) diff --git a/run_tests.sh b/run_tests.sh index c6aa3094..1e9d8ab5 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -12,6 +12,11 @@ else PIP=pip3 fi + +# check that pg_config exists +command -v pg_config + + # prepare environment VENV_PATH=/tmp/testgres_venv rm -rf $VENV_PATH @@ -29,12 +34,32 @@ $PIP install . # test code quality flake8 . -# run tests -coverage run tests/test_simple.py + +# remove existing coverage file +export COVERAGE_FILE=.coverage +rm -f $COVERAGE_FILE + + +# run tests (PATH) +time coverage run -a tests/test_simple.py + + +# run tests (PG_BIN) +export PG_BIN=$(dirname $(which pg_config)) +time coverage run -a tests/test_simple.py +unset PG_BIN + + +# run tests (PG_CONFIG) +export PG_CONFIG=$(which pg_config) +time coverage run -a tests/test_simple.py +unset PG_CONFIG + # show coverage coverage report + # attempt to fix codecov set +eux diff --git a/testgres/backup.py b/testgres/backup.py index fef9b851..d6db3a61 100644 --- a/testgres/backup.py +++ b/testgres/backup.py @@ -12,6 +12,7 @@ from .exceptions import BackupException from .utils import \ + get_bin_path, \ default_username as _default_username, \ execute_utility as _execute_utility, \ explain_exception as _explain_exception @@ -48,10 +49,11 @@ def __init__(self, data_dir = os.path.join(self.base_dir, _DATA_DIR) _params = [ + get_bin_path("pg_basebackup"), "-D{}".format(data_dir), "-p{}".format(node.port), "-U{}".format(username), "-X{}".format(xlog_method) ] - _execute_utility("pg_basebackup", _params, self.log_file) + _execute_utility(_params, self.log_file) def __enter__(self): return self diff --git a/testgres/node.py b/testgres/node.py index 8d751b73..dabae733 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -40,6 +40,7 @@ pg_version_ge as _pg_version_ge, \ reserve_port as _reserve_port, \ release_port as _release_port, \ + default_username as _default_username, \ execute_utility as _execute_utility, \ explain_exception as _explain_exception @@ -316,8 +317,13 @@ def status(self): """ try: - _params = ["status", "-D", self.data_dir] - _execute_utility("pg_ctl", _params, self.utils_log_name) + # yapf: disable + _params = [ + get_bin_path("pg_ctl"), + "-D", self.data_dir, + "status" + ] + _execute_utility(_params, self.utils_log_name) return NodeStatus.Running except ExecUtilException as e: @@ -346,12 +352,14 @@ def get_control_data(self): Return contents of pg_control file. """ - if _pg_version_ge('9.5'): - _params = ["-D", self.data_dir] - else: - _params = [self.data_dir] + # yapf: disable + _params = [ + get_bin_path("pg_controldata"), + "-D" if _pg_version_ge('9.5') else '', + self.data_dir + ] - data = _execute_utility("pg_controldata", _params, self.utils_log_name) + data = _execute_utility(_params, self.utils_log_name) out_dict = {} @@ -366,19 +374,23 @@ def start(self, params=[]): Start this node using pg_ctl. Args: - params: additional arguments for _execute_utility(). + params: additional arguments for pg_ctl. Returns: This instance of PostgresNode. """ + # yapf: disable _params = [ - "start", "-D{}".format(self.data_dir), "-l{}".format( - self.pg_log_name), "-w" + get_bin_path("pg_ctl"), + "-D", self.data_dir, + "-l", self.pg_log_name, + "-w", # wait + "start" ] + params try: - _execute_utility("pg_ctl", _params, self.utils_log_name) + _execute_utility(_params, self.utils_log_name) except ExecUtilException as e: msg = ( u"Cannot start node\n" @@ -395,14 +407,21 @@ def stop(self, params=[]): Stop this node using pg_ctl. Args: - params: additional arguments for _execute_utility(). + params: additional arguments for pg_ctl. Returns: This instance of PostgresNode. """ - _params = ["stop", "-D", self.data_dir, "-w"] + params - _execute_utility("pg_ctl", _params, self.utils_log_name) + # yapf: disable + _params = [ + get_bin_path("pg_ctl"), + "-D", self.data_dir, + "-w", # wait + "stop" + ] + params + + _execute_utility(_params, self.utils_log_name) self._maybe_stop_logger() @@ -413,19 +432,23 @@ def restart(self, params=[]): Restart this node using pg_ctl. Args: - params: additional arguments for _execute_utility(). + params: additional arguments for pg_ctl. Returns: This instance of PostgresNode. """ + # yapf: disable _params = [ - "restart", "-D{}".format(self.data_dir), "-l{}".format( - self.pg_log_name), "-w" + get_bin_path("pg_ctl"), + "-D", self.data_dir, + "-l", self.pg_log_name, + "-w", # wait + "restart" ] + params try: - _execute_utility("pg_ctl", _params, self.utils_log_name) + _execute_utility(_params, self.utils_log_name) except ExecUtilException as e: msg = ( u"Cannot restart node\n" @@ -441,23 +464,42 @@ def reload(self, params=[]): """ Reload config files using pg_ctl. + Args: + params: additional arguments for pg_ctl. + Returns: This instance of PostgresNode. """ - _params = ["reload", "-D", self.data_dir, "-w"] + params - _execute_utility("pg_ctl", _params, self.utils_log_name) + # yapf: disable + _params = [ + get_bin_path("pg_ctl"), + "-D", self.data_dir, + "-w", # wait + "reload" + ] + params + + _execute_utility(_params, self.utils_log_name) def pg_ctl(self, params): """ Invoke pg_ctl with params. + Args: + params: arguments for pg_ctl. + Returns: Stdout + stderr of pg_ctl. """ - _params = params + ["-D", self.data_dir, "-w"] - return _execute_utility("pg_ctl", _params, self.utils_log_name) + # yapf: disable + _params = [ + get_bin_path("pg_ctl"), + "-D", self.data_dir, + "-w" # wait + ] + params + + return _execute_utility(_params, self.utils_log_name) def free_port(self): """ @@ -471,6 +513,9 @@ def cleanup(self, max_attempts=3): """ Stop node if needed and remove its data directory. + Args: + max_attempts: how many times should we try to stop()? + Returns: This instance of PostgresNode. """ @@ -502,38 +547,52 @@ def cleanup(self, max_attempts=3): return self - def psql(self, dbname, query=None, filename=None, username=None, + def psql(self, + dbname, + query=None, + filename=None, + username=None, input=None): """ Execute a query using psql. Args: dbname: database name to connect to. - query: query to be executed. - filename: file with a query. username: database user name. + filename: file with a query. + query: query to be executed. + input: raw input to be passed. Returns: A tuple of (code, stdout, stderr). """ - psql = get_bin_path("psql") + # Set default username + username = username or _default_username() + + # yapf: disable psql_params = [ - psql, "-XAtq", "-h{}".format(self.host), "-p{}".format(self.port), + get_bin_path("psql"), + "-p", str(self.port), + "-h", self.host, + "-U", username, + "-X", # no .psqlrc + "-A", # unaligned output + "-t", # print rows only + "-q", # run quietly dbname ] + # select query source if query: psql_params.extend(("-c", query)) elif filename: psql_params.extend(("-f", filename)) + elif input: + pass else: raise QueryException('Query or filename must be provided') - # Specify user if needed - if username: - psql_params.extend(("-U", username)) - # start psql process process = subprocess.Popen( psql_params, @@ -551,8 +610,9 @@ def safe_psql(self, dbname, query, username=None, input=None): Args: dbname: database name to connect to. - query: query to be executed. username: database user name. + query: query to be executed. + input: raw input to be passed. Returns: psql's output as str. @@ -562,26 +622,38 @@ def safe_psql(self, dbname, query, username=None, input=None): if ret: err = '' if not err else err.decode('utf-8') raise QueryException(err) + return out - def dump(self, dbname, filename=None): + def dump(self, dbname, username=None, filename=None): """ - Dump database using pg_dump. + Dump database into a file using pg_dump. + NOTE: the file is not removed automatically. Args: dbname: database name to connect to. + username: database user name. filename: output file. Returns: - Path to file containing dump. + Path to a file containing dump. """ + # Set default arguments + username = username or _default_username() f, filename = filename or tempfile.mkstemp() os.close(f) - _params = ["-p{}".format(self.port), "-f{}".format(filename), dbname] + # yapf: disable + _params = [ + get_bin_path("pg_dump"), + "-p", str(self.port), + "-f", filename, + "-U", username, + "-d", dbname + ] - _execute_utility("pg_dump", _params, self.utils_log_name) + _execute_utility(_params, self.utils_log_name) return filename @@ -618,8 +690,8 @@ def poll_query_until(self, sleep_time: how much should we sleep after a failure? expected: what should be returned to break the cycle? commit: should (possible) changes be committed? - raise_programming_error: mute ProgrammingError? - raise_internal_error: mute InternalError? + raise_programming_error: enable ProgrammingError? + raise_internal_error: enable InternalError? """ # sanity checks @@ -760,10 +832,15 @@ def pgbench_init(self, dbname='postgres', scale=1, options=[]): This instance of PostgresNode. """ - _params = ["-i", "-s{}".format(scale), "-p{}".format(self.port) - ] + options + [dbname] + # yapf: disable + _params = [ + get_bin_path("pgbench"), + "-p", str(self.port), + "-s", str(scale), + "-i", # initialize + ] + options + [dbname] - _execute_utility("pgbench", _params, self.utils_log_name) + _execute_utility(_params, self.utils_log_name) return self @@ -782,7 +859,7 @@ def pgbench(self, dbname='postgres', stdout=None, stderr=None, options=[]): """ pgbench = get_bin_path("pgbench") - params = [pgbench, "-p", "%i" % self.port] + options + [dbname] + params = [pgbench, "-p", str(self.port)] + options + [dbname] proc = subprocess.Popen(params, stdout=stdout, stderr=stderr) return proc @@ -810,8 +887,8 @@ def _cached_initdb(data_dir, initdb_logfile, initdb_params=[]): def call_initdb(_data_dir): try: - _params = [_data_dir, "-N"] + initdb_params - _execute_utility("initdb", _params, initdb_logfile) + _params = [get_bin_path("initdb"), "-D", _data_dir, "-N"] + _execute_utility(_params + initdb_params, initdb_logfile) except ExecUtilException as e: raise InitNodeException(_explain_exception(e)) diff --git a/testgres/utils.py b/testgres/utils.py index f3ed1725..5ed60204 100644 --- a/testgres/utils.py +++ b/testgres/utils.py @@ -58,14 +58,12 @@ def explain_exception(e): return ''.join(lines) -def execute_utility(util, args, logfile): +def execute_utility(args, logfile): """ Execute utility (pg_ctl, pg_dump etc). - NOTE: 'util' is transformed using get_bin_path(). Args: - util: utility to be executed. - args: arguments for utility (list). + args: utility + arguments (list). logfile: path to file to store stdout and stderr. Returns: @@ -74,7 +72,7 @@ def execute_utility(util, args, logfile): # run utility process = subprocess.Popen( - [get_bin_path(util)] + args, + args, # util + params stdout=subprocess.PIPE, stderr=subprocess.STDOUT) @@ -86,7 +84,7 @@ def execute_utility(util, args, logfile): try: with io.open(logfile, 'a') as file_out: # write util's name and args - file_out.write(u' '.join([util] + args)) + file_out.write(u' '.join(args)) # write output if out: @@ -102,7 +100,7 @@ def execute_utility(util, args, logfile): error_code = process.returncode if error_code: error_text = (u"{} failed with exit code {}\n" - u"log:\n----\n{}\n").format(util, error_code, out) + u"log:\n----\n{}\n").format(args[0], error_code, out) raise ExecUtilException(error_text, error_code) @@ -119,64 +117,75 @@ def get_bin_path(filename): if os.path.isabs(filename): return filename + # try PG_CONFIG + pg_config = os.environ.get("PG_CONFIG") + if pg_config: + bindir = get_pg_config()["BINDIR"] + return os.path.join(bindir, filename) + # try PG_BIN pg_bin = os.environ.get("PG_BIN") if pg_bin: return os.path.join(pg_bin, filename) - # try PG_CONFIG - pg_config = get_pg_config() - if pg_config and "BINDIR" in pg_config: - return os.path.join(pg_config["BINDIR"], filename) - return filename def get_pg_config(): """ - Return output of pg_config. + Return output of pg_config (provided that it is installed). NOTE: this fuction caches the result by default (see TestgresConfig). """ global _pg_config_data + def cache_pg_config_data(cmd): + global _pg_config_data + + # execute pg_config and get the output + out = subprocess.check_output([cmd]).decode('utf-8') + + data = {} + for line in out.splitlines(): + if line and '=' in line: + key, _, value = line.partition('=') + data[key.strip()] = value.strip() + + _pg_config_data.clear() + + # cache data, if necessary + if TestgresConfig.cache_pg_config: + _pg_config_data = data + + return data + # return cached data, if allowed to if TestgresConfig.cache_pg_config and _pg_config_data: return _pg_config_data - # execute pg_config and get the output - pg_config_cmd = os.environ.get("PG_CONFIG") or "pg_config" - out = subprocess.check_output([pg_config_cmd]).decode('utf-8') - - data = {} - for line in out.splitlines(): - if line and "=" in line: - key, value = line.split("=", 1) - data[key.strip()] = value.strip() - - if TestgresConfig.cache_pg_config: - # cache data - _pg_config_data = data - else: - # clear for clarity - _pg_config_data.clear() + # try PG_CONFIG + pg_config = os.environ.get("PG_CONFIG") + if pg_config: + return cache_pg_config_data(pg_config) + + # try PG_BIN + pg_bin = os.environ.get("PG_BIN") + if pg_bin: + cmd = os.path.join(pg_bin, "pg_config") + return cache_pg_config_data(cmd) - return data + # try plain name + return cache_pg_config_data("pg_config") def get_pg_version(): """ - Return PostgreSQL version using PG_BIN or PG_CONFIG. + Return PostgreSQL version provided by postmaster. """ - pg_bin = os.environ.get("PG_BIN") - - if pg_bin: - # there might be no pg_config installed, try this first - raw_ver = execute_utility('psql', ['--version'], os.devnull) - else: - # ok, we have no other choice - raw_ver = get_pg_config()['VERSION'] + # get raw version (e.g. postgres (PostgreSQL) 9.5.7) + _params = [get_bin_path('postgres'), '--version'] + raw_ver = subprocess.check_output(_params).decode('utf-8') # cook version of PostgreSQL version = raw_ver.strip().split(' ')[-1] \ diff --git a/tests/test_simple.py b/tests/test_simple.py index 59a16930..6621eece 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -2,8 +2,6 @@ # coding: utf-8 import os -import re -import six import subprocess import tempfile import testgres @@ -351,7 +349,7 @@ def test_users(self): node.init().start() node.psql('postgres', 'create role test_user login') value = node.safe_psql('postgres', 'select 1', username='test_user') - self.assertEqual(value, six.b('1\n')) + self.assertEqual(value, b'1\n') def test_poll_query_until(self): with get_new_node('master') as node: @@ -491,11 +489,20 @@ def test_reload(self): with get_new_node('node') as node: node.init().start() - pid1 = node.get_pid() + cmd1 = "alter system set client_min_messages = DEBUG1" + cmd2 = "show client_min_messages" + + # change client_min_messages and save old value + cmm_old = node.execute(dbname='postgres', query=cmd2) + node.safe_psql(dbname='postgres', query=cmd1) + + # reload config node.reload() - pid2 = node.get_pid() - self.assertEqual(pid1, pid2) + # check new value + cmm_new = node.execute(dbname='postgres', query=cmd2) + self.assertEqual('debug1', cmm_new[0][0].lower()) + self.assertNotEqual(cmm_old, cmm_new) def test_pg_ctl(self): with get_new_node('node') as node: From 0e3a50e9b2793f06fb4a636523c88a8ad0f9ea30 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Sat, 23 Dec 2017 04:25:30 +0300 Subject: [PATCH 042/475] fix tests for 9.4 --- testgres/backup.py | 8 +++++-- testgres/node.py | 10 ++++----- testgres/utils.py | 3 +-- tests/test_simple.py | 53 ++++++++++++++++++++++++++++++++++---------- 4 files changed, 52 insertions(+), 22 deletions(-) diff --git a/testgres/backup.py b/testgres/backup.py index d6db3a61..06c42e3e 100644 --- a/testgres/backup.py +++ b/testgres/backup.py @@ -48,10 +48,14 @@ def __init__(self, self._available = True data_dir = os.path.join(self.base_dir, _DATA_DIR) + + # yapf: disable _params = [ get_bin_path("pg_basebackup"), - "-D{}".format(data_dir), "-p{}".format(node.port), - "-U{}".format(username), "-X{}".format(xlog_method) + "-p", str(node.port), + "-U", username, + "-D", data_dir, + "-X", xlog_method ] _execute_utility(_params, self.log_file) diff --git a/testgres/node.py b/testgres/node.py index dabae733..6e1d2e13 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -352,12 +352,10 @@ def get_control_data(self): Return contents of pg_control file. """ - # yapf: disable - _params = [ - get_bin_path("pg_controldata"), - "-D" if _pg_version_ge('9.5') else '', - self.data_dir - ] + # this one is tricky (blame PG 9.4) + _params = [get_bin_path("pg_controldata")] + _params += ["-D"] if _pg_version_ge('9.5') else [] + _params += [self.data_dir] data = _execute_utility(_params, self.utils_log_name) diff --git a/testgres/utils.py b/testgres/utils.py index 5ed60204..2672a97a 100644 --- a/testgres/utils.py +++ b/testgres/utils.py @@ -10,7 +10,6 @@ from .config import TestgresConfig from .exceptions import ExecUtilException - # rows returned by PG_CONFIG _pg_config_data = {} @@ -72,7 +71,7 @@ def execute_utility(args, logfile): # run utility process = subprocess.Popen( - args, # util + params + args, # util + params stdout=subprocess.PIPE, stderr=subprocess.STDOUT) diff --git a/tests/test_simple.py b/tests/test_simple.py index 6621eece..dba16e17 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -21,11 +21,34 @@ CatchUpException, \ TimeoutException -from testgres import TestgresConfig -from testgres import get_new_node, get_pg_config, configure_testgres +from testgres import \ + TestgresConfig, \ + configure_testgres + +from testgres import \ + NodeStatus, \ + IsolationLevel, \ + get_new_node + +from testgres import \ + get_bin_path, \ + get_pg_config + from testgres import bound_ports -from testgres import NodeStatus -from testgres import IsolationLevel + + +def util_is_executable(util): + exe_file = get_bin_path(util) + + # check if util exists + if os.path.exists(exe_file): + return True + + # check if util is in PATH + for path in os.environ["PATH"].split(os.pathsep): + exe_file = os.path.join(path, util) + if os.path.exists(exe_file): + return True class SimpleTest(unittest.TestCase): @@ -87,12 +110,19 @@ def test_uninitialized_start(self): def test_restart(self): with get_new_node('test') as node: node.init().start() + + # restart, ok res = node.execute('postgres', 'select 1') self.assertEqual(res, [(1, )]) node.restart() res = node.execute('postgres', 'select 2') self.assertEqual(res, [(2, )]) + # restart, fail + with self.assertRaises(StartNodeException): + node.append_conf('pg_hba.conf', 'DUMMY') + node.restart() + def test_psql(self): with get_new_node('test') as node: node.init().start() @@ -409,9 +439,7 @@ def test_poll_query_until(self): # check ProgrammingError, fail with self.assertRaises(testgres.ProgrammingError): - node.poll_query_until( - dbname='postgres', - query='dummy1') + node.poll_query_until(dbname='postgres', query='dummy1') # check ProgrammingError, ok with self.assertRaises(TimeoutException): @@ -471,6 +499,8 @@ def test_logging(self): master.restart() self.assertTrue(master._logger.is_alive()) + @unittest.skipUnless( + util_is_executable("pgbench"), "pgbench may be missing") def test_pgbench(self): with get_new_node('node') as node: node.init().start().pgbench_init() @@ -489,18 +519,17 @@ def test_reload(self): with get_new_node('node') as node: node.init().start() - cmd1 = "alter system set client_min_messages = DEBUG1" - cmd2 = "show client_min_messages" + cmd = "show client_min_messages" # change client_min_messages and save old value - cmm_old = node.execute(dbname='postgres', query=cmd2) - node.safe_psql(dbname='postgres', query=cmd1) + cmm_old = node.execute(dbname='postgres', query=cmd) + node.append_conf('postgresql.conf', 'client_min_messages = DEBUG1') # reload config node.reload() # check new value - cmm_new = node.execute(dbname='postgres', query=cmd2) + cmm_new = node.execute(dbname='postgres', query=cmd) self.assertEqual('debug1', cmm_new[0][0].lower()) self.assertNotEqual(cmm_old, cmm_new) From 17fb170dc4015135947e70f54579272062b5bc6b Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Tue, 26 Dec 2017 16:50:22 +0300 Subject: [PATCH 043/475] enforce some checks (flake8) --- setup.cfg | 4 ++-- testgres/node.py | 11 +++++------ tests/test_simple.py | 5 ++--- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/setup.cfg b/setup.cfg index f8d1b92b..ba6a57fc 100644 --- a/setup.cfg +++ b/setup.cfg @@ -2,5 +2,5 @@ description-file = README.md [flake8] -ignore = E501, F401, F403, F841 -exclude = .git,__pycache__,env,venv +ignore = E501 +exclude = .git,__pycache__,env,venv,testgres/__init__.py diff --git a/testgres/node.py b/testgres/node.py index 6e1d2e13..b219abc1 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -36,7 +36,6 @@ from .utils import \ get_bin_path, \ - get_pg_version, \ pg_version_ge as _pg_version_ge, \ reserve_port as _reserve_port, \ release_port as _release_port, \ @@ -159,7 +158,7 @@ def print_node_file(node_file): try: with io.open(node_file, "r") as f: return f.read().decode('utf-8') - except Exception as e: + except Exception: pass return "### file not found ###\n" @@ -389,7 +388,7 @@ def start(self, params=[]): try: _execute_utility(_params, self.utils_log_name) - except ExecUtilException as e: + except ExecUtilException: msg = ( u"Cannot start node\n" u"{}\n" # pg_ctl log @@ -447,7 +446,7 @@ def restart(self, params=[]): try: _execute_utility(_params, self.utils_log_name) - except ExecUtilException as e: + except ExecUtilException: msg = ( u"Cannot restart node\n" u"{}\n" # pg_ctl log @@ -525,9 +524,9 @@ def cleanup(self, max_attempts=3): try: self.stop() break # OK - except ExecUtilException as e: + except ExecUtilException: pass # one more time - except Exception as e: + except Exception: break # screw this attempts += 1 diff --git a/tests/test_simple.py b/tests/test_simple.py index dba16e17..90228632 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -294,12 +294,12 @@ def test_backup_exhaust(self): with node.backup(xlog_method='fetch') as backup: # exhaust backup by creating new node - with backup.spawn_primary('node1') as node1: + with backup.spawn_primary('node1') as node1: # noqa pass # now let's try to create one more node with self.assertRaises(BackupException): - with backup.spawn_primary('node2') as node2: + with backup.spawn_primary('node2') as node2: # noqa pass def test_backup_and_replication(self): @@ -567,7 +567,6 @@ def test_version_management(self): def test_config(self): # set global if it wasn't set - pg_config = get_pg_config() configure_testgres(cache_initdb=True, cache_pg_config=True) # check same instances From 6d50c5be2ac74ad0e3654d6a5941cdac16bb3884 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Tue, 26 Dec 2017 17:16:03 +0300 Subject: [PATCH 044/475] readme for tests --- tests/README.md | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 tests/README.md diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 00000000..9628e357 --- /dev/null +++ b/tests/README.md @@ -0,0 +1,29 @@ +### How do I run tests? + +#### Simple + +```bash +# Setup virtualenv +virtualenv venv +source venv/bin/activate + +# Install local version of testgres +pip install -U . + +# Set path to PostgreSQL +export PG_BIN=/path/to/pg/bin + +# Run tests +./tests/test_simple.py +``` + +#### All configurations + coverage + +```bash +# Set path to PostgreSQL and python version +export PATH=$PATH:/path/to/pg/bin +export PYTHON=3 (or 2) + +# Run tests +./run_tests.sh +``` From 95d8d7c1f5f0017fbb8351da247e9b7a4a15e78d Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Tue, 26 Dec 2017 17:36:12 +0300 Subject: [PATCH 045/475] small fixes in tests/README.md --- tests/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/README.md b/tests/README.md index 9628e357..0e941f11 100644 --- a/tests/README.md +++ b/tests/README.md @@ -22,7 +22,7 @@ export PG_BIN=/path/to/pg/bin ```bash # Set path to PostgreSQL and python version export PATH=$PATH:/path/to/pg/bin -export PYTHON=3 (or 2) +export PYTHON=3 # (or 2) # Run tests ./run_tests.sh From 8c6d72bb7d133fa0420a019181f4a5e088829bfb Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Tue, 26 Dec 2017 18:28:24 +0300 Subject: [PATCH 046/475] move cached_initdb() to a dedicated module --- MANIFEST | 1 + run_tests.sh | 2 +- testgres/cache.py | 60 +++++++++++++++++++++++++++++++++++++++++++++++ testgres/node.py | 48 ++----------------------------------- 4 files changed, 64 insertions(+), 47 deletions(-) create mode 100644 testgres/cache.py diff --git a/MANIFEST b/MANIFEST index f1f8c166..fc1d65b7 100644 --- a/MANIFEST +++ b/MANIFEST @@ -4,6 +4,7 @@ setup.py testgres/__init__.py testgres/api.py testgres/backup.py +testgres/cache.py testgres/config.py testgres/connection.py testgres/consts.py diff --git a/run_tests.sh b/run_tests.sh index 1e9d8ab5..b5f286da 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -13,7 +13,7 @@ else fi -# check that pg_config exists +echo check that pg_config exists command -v pg_config diff --git a/testgres/cache.py b/testgres/cache.py new file mode 100644 index 00000000..becfeb4d --- /dev/null +++ b/testgres/cache.py @@ -0,0 +1,60 @@ +# coding: utf-8 + +import atexit +import os +import shutil +import tempfile + +from .config import TestgresConfig + +from .exceptions import \ + InitNodeException, \ + ExecUtilException + +from .utils import \ + get_bin_path, \ + execute_utility as _execute_utility, \ + explain_exception as _explain_exception + + +def cached_initdb(data_dir, initdb_logfile, initdb_params=[]): + """ + Perform initdb or use cached node files. + """ + + def call_initdb(initdb_dir): + try: + _params = [get_bin_path("initdb"), "-D", initdb_dir, "-N"] + _execute_utility(_params + initdb_params, initdb_logfile) + except ExecUtilException as e: + raise InitNodeException(_explain_exception(e)) + + def rm_cached_data_dir(cached_data_dir): + shutil.rmtree(cached_data_dir, ignore_errors=True) + + # Call initdb if we have custom params or shouldn't cache it + if initdb_params or not TestgresConfig.cache_initdb: + call_initdb(data_dir) + else: + # Set default temp dir for cached initdb + if TestgresConfig.cached_initdb_dir is None: + + # Create default temp dir + TestgresConfig.cached_initdb_dir = tempfile.mkdtemp() + + # Schedule cleanup + atexit.register(rm_cached_data_dir, + TestgresConfig.cached_initdb_dir) + + # Fetch cached initdb dir + cached_data_dir = TestgresConfig.cached_initdb_dir + + # Initialize cached initdb + if not os.listdir(cached_data_dir): + call_initdb(cached_data_dir) + + try: + # Copy cached initdb to current data dir + shutil.copytree(cached_data_dir, data_dir) + except Exception as e: + raise InitNodeException(_explain_exception(e)) diff --git a/testgres/node.py b/testgres/node.py index b219abc1..9391d599 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -1,6 +1,5 @@ # coding: utf-8 -import atexit import io import os import shutil @@ -10,6 +9,8 @@ from enum import Enum +from .cache import cached_initdb as _cached_initdb + from .config import TestgresConfig from .connection import \ @@ -27,7 +28,6 @@ from .exceptions import \ CatchUpException, \ ExecUtilException, \ - InitNodeException, \ QueryException, \ StartNodeException, \ TimeoutException @@ -875,47 +875,3 @@ def connect(self, dbname='postgres', username=None): return NodeConnection( parent_node=self, dbname=dbname, username=username) - - -def _cached_initdb(data_dir, initdb_logfile, initdb_params=[]): - """ - Perform initdb or use cached node files. - """ - - def call_initdb(_data_dir): - try: - _params = [get_bin_path("initdb"), "-D", _data_dir, "-N"] - _execute_utility(_params + initdb_params, initdb_logfile) - except ExecUtilException as e: - raise InitNodeException(_explain_exception(e)) - - # Call initdb if we have custom params - if initdb_params or not TestgresConfig.cache_initdb: - call_initdb(data_dir) - # Else we can use cached dir - else: - # Set default temp dir for cached initdb - if TestgresConfig.cached_initdb_dir is None: - - def rm_cached_data_dir(rm_dir): - shutil.rmtree(rm_dir, ignore_errors=True) - - # Create default temp dir - TestgresConfig.cached_initdb_dir = tempfile.mkdtemp() - - # Schedule cleanup - atexit.register(rm_cached_data_dir, - TestgresConfig.cached_initdb_dir) - - # Fetch cached initdb dir - cached_data_dir = TestgresConfig.cached_initdb_dir - - # Initialize cached initdb - if not os.listdir(cached_data_dir): - call_initdb(cached_data_dir) - - try: - # Copy cached initdb to current data dir - shutil.copytree(cached_data_dir, data_dir) - except Exception as e: - raise InitNodeException(_explain_exception(e)) From 933c7ddb0eb0c261208116f532c6ede0e4675f3c Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Tue, 26 Dec 2017 19:20:59 +0300 Subject: [PATCH 047/475] remove pgbench_init(), add pgbench_run(), add self.host here and there --- README.md | 9 +++++++-- testgres/backup.py | 1 + testgres/node.py | 37 ++++++++++++++++++++----------------- tests/test_simple.py | 6 +++++- 4 files changed, 33 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 8deaceaf..73b4046c 100644 --- a/README.md +++ b/README.md @@ -139,9 +139,14 @@ with testgres.get_new_node('master') as master: ```python with testgres.get_new_node('master') as master: + # start new node master.init().start() - p = master.pg_bench_init(scale=10).pgbench(options=['-T', '60']) - p.wait() + + # initialize database for TPC-B + master.pgbench_run(options=['-i']) + + # run benchmark for 20 seconds and show results + print(master.pgbench_run(options=['-T', '20'])) ``` ## Authors diff --git a/testgres/backup.py b/testgres/backup.py index 06c42e3e..b5fd9696 100644 --- a/testgres/backup.py +++ b/testgres/backup.py @@ -53,6 +53,7 @@ def __init__(self, _params = [ get_bin_path("pg_basebackup"), "-p", str(node.port), + "-h", node.host, "-U", username, "-D", data_dir, "-X", xlog_method diff --git a/testgres/node.py b/testgres/node.py index 9391d599..ffd76bb5 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -645,6 +645,7 @@ def dump(self, dbname, username=None, filename=None): _params = [ get_bin_path("pg_dump"), "-p", str(self.port), + "-h", self.host, "-f", filename, "-U", username, "-d", dbname @@ -816,50 +817,52 @@ def catchup(self, dbname='postgres', username=None): except Exception as e: raise CatchUpException(_explain_exception(e)) - def pgbench_init(self, dbname='postgres', scale=1, options=[]): + def pgbench(self, dbname='postgres', stdout=None, stderr=None, options=[]): """ - Prepare database for pgbench (create tables etc). + Spawn a pgbench process. Args: dbname: database name to connect to. - scale: report this scale factor in output (int). + stdout: stdout file to be used by Popen. + stderr: stderr file to be used by Popen. options: additional options for pgbench (list). Returns: - This instance of PostgresNode. + Process created by subprocess.Popen. """ # yapf: disable _params = [ get_bin_path("pgbench"), "-p", str(self.port), - "-s", str(scale), - "-i", # initialize + "-h", self.host, ] + options + [dbname] - _execute_utility(_params, self.utils_log_name) + proc = subprocess.Popen(_params, stdout=stdout, stderr=stderr) - return self + return proc - def pgbench(self, dbname='postgres', stdout=None, stderr=None, options=[]): + def pgbench_run(self, dbname='postgres', options=[]): """ - Spawn a pgbench process. + Run pgbench with some options. + This event is logged (see self.utils_log_name). Args: dbname: database name to connect to. - stdout: stdout file to be used by Popen. - stderr: stderr file to be used by Popen. options: additional options for pgbench (list). Returns: - Process created by subprocess.Popen. + Stdout produced by pgbench. """ - pgbench = get_bin_path("pgbench") - params = [pgbench, "-p", str(self.port)] + options + [dbname] - proc = subprocess.Popen(params, stdout=stdout, stderr=stderr) + # yapf: disable + _params = [ + get_bin_path("pgbench"), + "-p", str(self.port), + "-h", self.host, + ] + options + [dbname] - return proc + return _execute_utility(_params, self.utils_log_name) def connect(self, dbname='postgres', username=None): """ diff --git a/tests/test_simple.py b/tests/test_simple.py index 90228632..0a4e8d20 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -503,8 +503,12 @@ def test_logging(self): util_is_executable("pgbench"), "pgbench may be missing") def test_pgbench(self): with get_new_node('node') as node: - node.init().start().pgbench_init() + node.init().start() + + # initialize pgbench + node.pgbench_run(options=['-i']) + # run TPC-B benchmark proc = node.pgbench( stdout=subprocess.PIPE, stderr=subprocess.STDOUT, From 882582cc4196569da27063bb25bbc554faa2b2f2 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Wed, 10 Jan 2018 14:53:31 +0300 Subject: [PATCH 048/475] small fixes --- Dockerfile.tmpl | 2 +- README.md | 50 ++++++++++++++++++++++++++++++++---------------- run_tests.sh | 15 ++++++--------- testgres/node.py | 6 ++++-- tests/README.md | 2 +- 5 files changed, 46 insertions(+), 29 deletions(-) diff --git a/Dockerfile.tmpl b/Dockerfile.tmpl index 8f1c267a..9d417a31 100644 --- a/Dockerfile.tmpl +++ b/Dockerfile.tmpl @@ -18,4 +18,4 @@ WORKDIR /pg/testgres RUN chown -R postgres:postgres /pg USER postgres -ENTRYPOINT PYTHON=${PYTHON} /run.sh +ENTRYPOINT PYTHON_VERSION=${PYTHON_VERSION} /run.sh diff --git a/README.md b/README.md index 73b4046c..05da6a86 100644 --- a/README.md +++ b/README.md @@ -39,20 +39,31 @@ python my_tests.py ### Logging -By default, `cleanup()` removes all temporary files (DB files, logs etc) that were created by testgres' API methods. If you'd like to keep logs, execute `configure_testgres(node_cleanup_full=False)` before running any tests. +By default, `cleanup()` removes all temporary files (DB files, logs etc) that were created by testgres' API methods. +If you'd like to keep logs, execute `configure_testgres(node_cleanup_full=False)` before running any tests. -> Note: context managers (aka `with`) call `cleanup()` automatically. +> Note: context managers (aka `with`) call `stop()` and `cleanup()` automatically. -Nodes support python logging system, so if you have configured logging -in your tests, you can use it to redirect postgres logs to yours. - -To do that, just use `use_logging` argument: +testgres supports [python logging](https://docs.python.org/3.6/library/logging.html), +which means that you can aggregate logs from several nodes into one file: ```python -node = testgres.get_new_node('master', use_logging=True) -``` +import io +import logging + +# write everything to /tmp/testgres.log +logfile = io.open('/tmp/testgres.log', 'w') +logger = logging.getLogger('testgres') +logger.FileHandler(logfile) -You can find working configuration example for logging in `tests/test_simple.py`. +# create two different nodes with logging +node1 = testgres.get_new_node('node1', use_logging=True).init().start() +node2 = testgres.get_new_node('node2', use_logging=True).init().start() + +# execute a few queries +node1.execute('postgres', 'select 1') +node2.execute('postgres', 'select 2') +``` ### Examples @@ -81,13 +92,16 @@ or with testgres.get_new_node('master', '/path/to/DB') as node: ``` -where `master` is a node's name, not a DB's name. Name matters if you're testing something like replication. Function `get_new_node()` only creates directory structure in specified directory (or somewhere in '/tmp' if we did not specify base directory) for cluster. After that, we have to initialize the PostgreSQL cluster: +where `master` is a node's name, not a DB's name. Name matters if you're testing something like replication. +Function `get_new_node()` only creates directory structure in specified directory (or somewhere in '/tmp' if +we did not specify base directory) for cluster. After that, we have to initialize the PostgreSQL cluster: ```python node.init() ``` -This function runs `initdb` command and adds some basic configuration to `postgresql.conf` and `pg_hba.conf` files. Function `init()` accepts optional parameter `allows_streaming` which configures cluster for streaming replication (default is `False`). +This function runs `initdb` command and adds some basic configuration to `postgresql.conf` and `pg_hba.conf` files. +Function `init()` accepts optional parameter `allows_streaming` which configures cluster for streaming replication (default is `False`). Now we are ready to start: ```python @@ -126,29 +140,33 @@ It's quite easy to create a backup and start a new replica: with testgres.get_new_node('master') as master: master.init().start() with master.backup() as backup: + # create and start a new replica replica = backup.spawn_replica('replica').start() - replica.catchup() # catch up with master + + # catch up with master node + replica.catchup() + + # execute a dummy query print(replica.execute('postgres', 'select 1')) ``` -> Note: you could take a look at [`pg_pathman`](https://github.com/postgrespro/pg_pathman) to get an idea of `testgres`' capabilities. - ### Benchmarks `testgres` also can help you to make benchmarks using `pgbench` from postgres installation: ```python with testgres.get_new_node('master') as master: - # start new node + # start a new node master.init().start() - # initialize database for TPC-B + # initialize default database for TPC-B master.pgbench_run(options=['-i']) # run benchmark for 20 seconds and show results print(master.pgbench_run(options=['-T', '20'])) ``` + ## Authors [Ildar Musin](https://github.com/zilder) Postgres Professional Ltd., Russia diff --git a/run_tests.sh b/run_tests.sh index b5f286da..93851b3c 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -4,19 +4,16 @@ set -eux -if [ "$PYTHON" == "python2" ]; then - VIRTUALENV="virtualenv --python=/usr/bin/python2" - PIP=pip2 -else - VIRTUALENV="virtualenv --python=/usr/bin/python3" - PIP=pip3 -fi +# choose python version +echo python version is $PYTHON_VERSION +VIRTUALENV="virtualenv --python=/usr/bin/python$PYTHON_VERSION" +PIP="pip$PYTHON_VERSION" -echo check that pg_config exists +# fail early +echo check that pg_config is in PATH command -v pg_config - # prepare environment VENV_PATH=/tmp/testgres_venv rm -rf $VENV_PATH diff --git a/testgres/node.py b/testgres/node.py index ffd76bb5..8967e1d0 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -876,5 +876,7 @@ def connect(self, dbname='postgres', username=None): An instance of NodeConnection. """ - return NodeConnection( - parent_node=self, dbname=dbname, username=username) + return NodeConnection(parent_node=self, + host=self.host, + dbname=dbname, + username=username) diff --git a/tests/README.md b/tests/README.md index 0e941f11..5a5ecd07 100644 --- a/tests/README.md +++ b/tests/README.md @@ -22,7 +22,7 @@ export PG_BIN=/path/to/pg/bin ```bash # Set path to PostgreSQL and python version export PATH=$PATH:/path/to/pg/bin -export PYTHON=3 # (or 2) +export PYTHON_VERSION=3 # or 2 # Run tests ./run_tests.sh From 9648aa2e2539c032373fc9f2ca57615820dad690 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Wed, 10 Jan 2018 16:45:33 +0300 Subject: [PATCH 049/475] fix docs --- README.md | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/README.md b/README.md index 05da6a86..2ca23a72 100644 --- a/README.md +++ b/README.md @@ -48,13 +48,10 @@ testgres supports [python logging](https://docs.python.org/3.6/library/logging.h which means that you can aggregate logs from several nodes into one file: ```python -import io import logging # write everything to /tmp/testgres.log -logfile = io.open('/tmp/testgres.log', 'w') -logger = logging.getLogger('testgres') -logger.FileHandler(logfile) +logging.basicConfig(filename='/tmp/testgres.log') # create two different nodes with logging node1 = testgres.get_new_node('node1', use_logging=True).init().start() @@ -71,8 +68,6 @@ node2.execute('postgres', 'select 2') Here is an example of what you can do with `testgres`: ```python -import testgres - with testgres.get_new_node('test') as node: node.init() # run initdb node.start() # start PostgreSQL From 2b5d7eaad0e45b3493a5976f4d45f13d806b59c7 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Wed, 10 Jan 2018 19:44:55 +0300 Subject: [PATCH 050/475] add unix_sockets=True to init() and default_conf() --- testgres/backup.py | 13 +++++++++++- testgres/node.py | 50 +++++++++++++++++++++++++++++++------------- tests/test_simple.py | 11 ++++++++++ 3 files changed, 59 insertions(+), 15 deletions(-) diff --git a/testgres/backup.py b/testgres/backup.py index b5fd9696..e086d912 100644 --- a/testgres/backup.py +++ b/testgres/backup.py @@ -32,6 +32,15 @@ def __init__(self, base_dir=None, username=None, xlog_method=_DEFAULT_XLOG_METHOD): + """ + Create a new backup. + + Args: + node: PostgresNode we're going to backup. + base_dir: where should we store it? + username: database user name. + xlog_method: none | fetch | stream (see docs) + """ if not node.status(): raise BackupException('Node must be running') @@ -43,6 +52,7 @@ def __init__(self, # public self.original_node = node self.base_dir = base_dir + self.username = username # private self._available = True @@ -148,7 +158,8 @@ def spawn_replica(self, name, destroy=True, use_logging=False): """ node = self.spawn_primary(name, destroy, use_logging=use_logging) - node._create_recovery_conf(self.original_node) + node._create_recovery_conf(username=self.username, + master=self.original_node) return node diff --git a/testgres/node.py b/testgres/node.py index 8967e1d0..494c251b 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -112,13 +112,20 @@ def utils_log_name(self): def pg_log_name(self): return os.path.join(self.data_dir, _PG_LOG_FILE) - @property - def connstr(self): - return "port={}".format(self.port) + def _create_recovery_conf(self, username, master): + # yapf: disable + conninfo = ( + u"user={} " + u"port={} " + u"host={} " + u"application_name={}" + ).format(username, master.port, master.host, master.name) - def _create_recovery_conf(self, root_node): - line = ("primary_conninfo='{} application_name={}'\n" - "standby_mode=on\n").format(root_node.connstr, self.name) + # yapf: disable + line = ( + "primary_conninfo='{}'\n" + "standby_mode=on\n" + ).format(conninfo) self.append_conf("recovery.conf", line) @@ -175,13 +182,18 @@ def print_node_file(node_file): return error_text - def init(self, allow_streaming=False, fsync=False, initdb_params=[]): + def init(self, + fsync=False, + unix_sockets=True, + allow_streaming=False, + initdb_params=[]): """ Perform initdb for this node. Args: - allow_streaming: should this node add a hba entry for replication? fsync: should this node use fsync to keep data safe? + unix_sockets: should we enable UNIX sockets? + allow_streaming: should this node add a hba entry for replication? initdb_params: parameters for initdb (list). Returns: @@ -196,22 +208,25 @@ def init(self, allow_streaming=False, fsync=False, initdb_params=[]): _cached_initdb(self.data_dir, initdb_log, initdb_params) # initialize default config files - self.default_conf(allow_streaming=allow_streaming, fsync=fsync) + self.default_conf(fsync=fsync, + unix_sockets=unix_sockets, + allow_streaming=allow_streaming) return self def default_conf(self, - allow_streaming=False, fsync=False, + unix_sockets=True, + allow_streaming=False, log_statement='all'): """ Apply default settings to this node. Args: - allow_streaming: should this node add a hba entry for replication? fsync: should this node use fsync to keep data safe? - log_statement: one of ('all', 'off', 'mod', 'ddl'), look at - PostgreSQL docs for more information + unix_sockets: should we enable UNIX sockets? + allow_streaming: should this node add a hba entry for replication? + log_statement: one of ('all', 'off', 'mod', 'ddl'). Returns: This instance of PostgresNode. @@ -258,6 +273,9 @@ def get_auth_method(t): # overwrite postgresql.conf file with io.open(postgres_conf, "w") as conf: + # remove old lines + conf.truncate() + if not fsync: conf.write(u"fsync = off\n") @@ -287,6 +305,10 @@ def get_auth_method(t): wal_keep_segments, wal_level)) + # disable UNIX sockets if asked to + if not unix_sockets: + conf.write(u"unix_socket_directories = ''\n") + return self def append_conf(self, filename, string): @@ -784,7 +806,7 @@ def replicate(self, """ backup = self.backup(username=username, xlog_method=xlog_method) - return backup.spawn_replica(name, use_logging=use_logging) + return backup.spawn_replica(name=name, use_logging=use_logging) def catchup(self, dbname='postgres', username=None): """ diff --git a/tests/test_simple.py b/tests/test_simple.py index 0a4e8d20..b6ca3ae5 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -590,6 +590,17 @@ def test_config(self): # restore setting configure_testgres(cache_pg_config=True) + def test_unix_sockets(self): + with get_new_node('node').init(unix_sockets=False) as node: + node.start() + + node.execute('postgres', 'select 1') + node.safe_psql('postgres', 'select 1') + + r = node.replicate('r').start() + r.execute('postgres', 'select 1') + r.safe_psql('postgres', 'select 1') + def test_isolation_levels(self): with get_new_node('node').init().start() as node: with node.connect() as con: From 21ff16870c719487ec853c3dd8597d9e06b630bc Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Thu, 11 Jan 2018 14:25:37 +0300 Subject: [PATCH 051/475] improved exception handling --- setup.py | 2 +- testgres/backup.py | 7 +++--- testgres/cache.py | 9 +++---- testgres/node.py | 60 +++++++++++++++++++--------------------------- testgres/utils.py | 10 -------- 5 files changed, 34 insertions(+), 54 deletions(-) diff --git a/setup.py b/setup.py index ecfe2a27..190fb11a 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ from distutils.core import setup # Basic dependencies -install_requires = ["pg8000", "port-for"] +install_requires = ["pg8000", "port-for", "six"] # Add compatibility enum class if sys.version_info < (3, 4): diff --git a/testgres/backup.py b/testgres/backup.py index e086d912..2a597555 100644 --- a/testgres/backup.py +++ b/testgres/backup.py @@ -4,6 +4,8 @@ import shutil import tempfile +from six import raise_from + from .consts import \ DATA_DIR as _DATA_DIR, \ BACKUP_LOG_FILE as _BACKUP_LOG_FILE, \ @@ -14,8 +16,7 @@ from .utils import \ get_bin_path, \ default_username as _default_username, \ - execute_utility as _execute_utility, \ - explain_exception as _explain_exception + execute_utility as _execute_utility class NodeBackup(object): @@ -103,7 +104,7 @@ def _prepare_dir(self, destroy): # Copy backup to new data dir shutil.copytree(data1, data2) except Exception as e: - raise BackupException(_explain_exception(e)) + raise_from(BackupException('Failed to copy files'), e) else: dest_base_dir = self.base_dir diff --git a/testgres/cache.py b/testgres/cache.py index becfeb4d..2c7e64b3 100644 --- a/testgres/cache.py +++ b/testgres/cache.py @@ -5,6 +5,8 @@ import shutil import tempfile +from six import raise_from + from .config import TestgresConfig from .exceptions import \ @@ -13,8 +15,7 @@ from .utils import \ get_bin_path, \ - execute_utility as _execute_utility, \ - explain_exception as _explain_exception + execute_utility as _execute_utility def cached_initdb(data_dir, initdb_logfile, initdb_params=[]): @@ -27,7 +28,7 @@ def call_initdb(initdb_dir): _params = [get_bin_path("initdb"), "-D", initdb_dir, "-N"] _execute_utility(_params + initdb_params, initdb_logfile) except ExecUtilException as e: - raise InitNodeException(_explain_exception(e)) + raise_from(InitNodeException("Failed to run initdb"), e) def rm_cached_data_dir(cached_data_dir): shutil.rmtree(cached_data_dir, ignore_errors=True) @@ -57,4 +58,4 @@ def rm_cached_data_dir(cached_data_dir): # Copy cached initdb to current data dir shutil.copytree(cached_data_dir, data_dir) except Exception as e: - raise InitNodeException(_explain_exception(e)) + raise_from(InitNodeException("Failed to copy files"), e) diff --git a/testgres/node.py b/testgres/node.py index 494c251b..33cd228a 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -8,6 +8,7 @@ import time from enum import Enum +from six import raise_from from .cache import cached_initdb as _cached_initdb @@ -40,8 +41,7 @@ reserve_port as _reserve_port, \ release_port as _release_port, \ default_username as _default_username, \ - execute_utility as _execute_utility, \ - explain_exception as _explain_exception + execute_utility as _execute_utility class NodeStatus(Enum): @@ -110,7 +110,7 @@ def utils_log_name(self): @property def pg_log_name(self): - return os.path.join(self.data_dir, _PG_LOG_FILE) + return os.path.join(self.logs_dir, _PG_LOG_FILE) def _create_recovery_conf(self, username, master): # yapf: disable @@ -148,37 +148,25 @@ def _maybe_stop_logger(self): self._logger.stop() def _format_verbose_error(self): - # choose log_filename - log_filename = self.pg_log_name - - # choose conf_filename - conf_filename = os.path.join(self.data_dir, "postgresql.conf") - - # choose hba_filename - hba_filename = os.path.join(self.data_dir, "pg_hba.conf") + # list of important files + files = [ + os.path.join(self.data_dir, "postgresql.conf"), + os.path.join(self.data_dir, "recovery.conf"), + os.path.join(self.data_dir, "pg_hba.conf"), + self.pg_log_name # main log file + ] - # choose recovery_filename - recovery_filename = os.path.join(self.data_dir, "recovery.conf") + error_text = "" - def print_node_file(node_file): - if os.path.exists(node_file): - try: - with io.open(node_file, "r") as f: - return f.read().decode('utf-8') - except Exception: - pass - return "### file not found ###\n" + for f in files: + # skip missing files + if not os.path.exists(f): + continue - # yapf: disable - error_text = ( - u"{}:\n----\n{}\n" # log file, e.g. postgresql.log - u"{}:\n----\n{}\n" # postgresql.conf - u"{}:\n----\n{}\n" # pg_hba.conf - u"{}:\n----\n{}\n" # recovery.conf - ).format(log_filename, print_node_file(log_filename), - conf_filename, print_node_file(conf_filename), - hba_filename, print_node_file(hba_filename), - recovery_filename, print_node_file(recovery_filename)) + # append contents + with io.open(f, "r") as _f: + lines = _f.read() + error_text += u"{}:\n----\n{}\n".format(f, lines) return error_text @@ -410,12 +398,12 @@ def start(self, params=[]): try: _execute_utility(_params, self.utils_log_name) - except ExecUtilException: + except ExecUtilException as e: msg = ( u"Cannot start node\n" u"{}\n" # pg_ctl log ).format(self._format_verbose_error()) - raise StartNodeException(msg) + raise_from(StartNodeException(msg), e) self._maybe_start_logger() @@ -468,12 +456,12 @@ def restart(self, params=[]): try: _execute_utility(_params, self.utils_log_name) - except ExecUtilException: + except ExecUtilException as e: msg = ( u"Cannot restart node\n" u"{}\n" # pg_ctl log ).format(self._format_verbose_error()) - raise StartNodeException(msg) + raise_from(StartNodeException(msg), e) self._maybe_start_logger() @@ -837,7 +825,7 @@ def catchup(self, dbname='postgres', username=None): query=wait_lsn.format(lsn), max_attempts=0) # infinite except Exception as e: - raise CatchUpException(_explain_exception(e)) + raise_from(CatchUpException('Failed to catch up'), e) def pgbench(self, dbname='postgres', stdout=None, stderr=None, options=[]): """ diff --git a/testgres/utils.py b/testgres/utils.py index 2672a97a..cd280221 100644 --- a/testgres/utils.py +++ b/testgres/utils.py @@ -47,16 +47,6 @@ def default_username(): return pwd.getpwuid(os.getuid())[0] -def explain_exception(e): - """ - Use this function instead of str(e). - """ - - import traceback # used only here - lines = traceback.format_exception_only(type(e), e) - return ''.join(lines) - - def execute_utility(args, logfile): """ Execute utility (pg_ctl, pg_dump etc). From 2ecd3a8360c7147535eba7b7b6d404561897808b Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Thu, 11 Jan 2018 15:03:03 +0300 Subject: [PATCH 052/475] fix error in tests, enable streaming by default --- setup.py | 4 ++++ testgres/node.py | 38 ++++++++++++++++++++++---------------- tests/test_simple.py | 7 ++++--- 3 files changed, 30 insertions(+), 19 deletions(-) diff --git a/setup.py b/setup.py index 190fb11a..525c9816 100644 --- a/setup.py +++ b/setup.py @@ -12,6 +12,10 @@ if sys.version_info < (3, 4): install_requires.append("enum34") +# Add compatibility ipaddress module +if sys.version_info < (3, 3): + install_requires.append("ipaddress") + setup( name='testgres', packages=['testgres'], diff --git a/testgres/node.py b/testgres/node.py index 33cd228a..1638f132 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -115,11 +115,18 @@ def pg_log_name(self): def _create_recovery_conf(self, username, master): # yapf: disable conninfo = ( - u"user={} " + u"application_name={} " u"port={} " - u"host={} " - u"application_name={}" - ).format(username, master.port, master.host, master.name) + u"user={} " + ).format(master.name, master.port, username) + + # host is tricky + try: + import ipaddress + ipaddress.ip_address(master.host) + conninfo += u"hostaddr={}".format(master.host) + except ValueError: + conninfo += u"host={}".format(master.host) # yapf: disable line = ( @@ -147,7 +154,7 @@ def _maybe_stop_logger(self): if self._logger: self._logger.stop() - def _format_verbose_error(self): + def _format_verbose_error(self, message=None): # list of important files files = [ os.path.join(self.data_dir, "postgresql.conf"), @@ -158,6 +165,11 @@ def _format_verbose_error(self): error_text = "" + # append message if asked to + if message: + error_text += message + error_text += '\n' * 2 + for f in files: # skip missing files if not os.path.exists(f): @@ -205,7 +217,7 @@ def init(self, def default_conf(self, fsync=False, unix_sockets=True, - allow_streaming=False, + allow_streaming=True, log_statement='all'): """ Apply default settings to this node. @@ -284,8 +296,8 @@ def get_auth_method(t): wal_level = "hot_standby" # yapf: disable - max_wal_senders = 5 - wal_keep_segments = 20 + max_wal_senders = 10 # default in PG 10 + wal_keep_segments = 20 # for convenience conf.write(u"hot_standby = on\n" u"max_wal_senders = {}\n" u"wal_keep_segments = {}\n" @@ -399,10 +411,7 @@ def start(self, params=[]): try: _execute_utility(_params, self.utils_log_name) except ExecUtilException as e: - msg = ( - u"Cannot start node\n" - u"{}\n" # pg_ctl log - ).format(self._format_verbose_error()) + msg = self._format_verbose_error('Cannot start node') raise_from(StartNodeException(msg), e) self._maybe_start_logger() @@ -457,10 +466,7 @@ def restart(self, params=[]): try: _execute_utility(_params, self.utils_log_name) except ExecUtilException as e: - msg = ( - u"Cannot restart node\n" - u"{}\n" # pg_ctl log - ).format(self._format_verbose_error()) + msg = self._format_verbose_error('Cannot restart node') raise_from(StartNodeException(msg), e) self._maybe_start_logger() diff --git a/tests/test_simple.py b/tests/test_simple.py index b6ca3ae5..79acb294 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -294,12 +294,12 @@ def test_backup_exhaust(self): with node.backup(xlog_method='fetch') as backup: # exhaust backup by creating new node - with backup.spawn_primary('node1') as node1: # noqa + with backup.spawn_primary('node1') as node1: # noqa pass # now let's try to create one more node with self.assertRaises(BackupException): - with backup.spawn_primary('node2') as node2: # noqa + with backup.spawn_primary('node2') as node2: # noqa pass def test_backup_and_replication(self): @@ -591,7 +591,8 @@ def test_config(self): configure_testgres(cache_pg_config=True) def test_unix_sockets(self): - with get_new_node('node').init(unix_sockets=False) as node: + with get_new_node('node') as node: + node.init(unix_sockets=False, allow_streaming=True) node.start() node.execute('postgres', 'select 1') From d243e494c46effebfeeb6cde52f314985cce9dac Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Thu, 11 Jan 2018 20:21:21 +0300 Subject: [PATCH 053/475] generate application name if asked to --- README.md | 6 +++--- testgres/api.py | 10 +++++----- testgres/backup.py | 28 ++++++++++++++++------------ testgres/node.py | 40 ++++++++++++++++++++++++++++------------ testgres/utils.py | 9 +++++++++ tests/test_simple.py | 15 +++++++++++++++ 6 files changed, 76 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index 2ca23a72..aad1a3ea 100644 --- a/README.md +++ b/README.md @@ -54,8 +54,8 @@ import logging logging.basicConfig(filename='/tmp/testgres.log') # create two different nodes with logging -node1 = testgres.get_new_node('node1', use_logging=True).init().start() -node2 = testgres.get_new_node('node2', use_logging=True).init().start() +node1 = testgres.get_new_node(use_logging=True).init().start() +node2 = testgres.get_new_node(use_logging=True).init().start() # execute a few queries node1.execute('postgres', 'select 1') @@ -87,7 +87,7 @@ or with testgres.get_new_node('master', '/path/to/DB') as node: ``` -where `master` is a node's name, not a DB's name. Name matters if you're testing something like replication. +where `master` is a node's application name. Name matters if you're testing something like replication. Function `get_new_node()` only creates directory structure in specified directory (or somewhere in '/tmp' if we did not specify base directory) for cluster. After that, we have to initialize the PostgreSQL cluster: diff --git a/testgres/api.py b/testgres/api.py index 04874e7c..4b607423 100644 --- a/testgres/api.py +++ b/testgres/api.py @@ -7,7 +7,7 @@ edit configuration files, start/stop cluster, execute queries. The typical flow may look like: - with get_new_node('test') as node: + with get_new_node() as node: node.init() node.start() result = node.psql('postgres', 'SELECT 1') @@ -16,10 +16,10 @@ Or: - with get_new_node('node1') as node1: + with get_new_node() as node1: node1.init().start() with node1.backup() as backup: - with backup.spawn_primary('node2') as node2: + with backup.spawn_primary() as node2: res = node2.start().execute('postgres', 'select 2') print(res) @@ -29,12 +29,12 @@ from .node import PostgresNode -def get_new_node(name, base_dir=None, use_logging=False): +def get_new_node(name=None, base_dir=None, use_logging=False): """ Create a new node (select port automatically). Args: - name: node's name. + name: node's application name. base_dir: path to node's data directory. use_logging: enable python logging. diff --git a/testgres/backup.py b/testgres/backup.py index 2a597555..da561cd4 100644 --- a/testgres/backup.py +++ b/testgres/backup.py @@ -114,12 +114,12 @@ def _prepare_dir(self, destroy): # Return path to new node return dest_base_dir - def spawn_primary(self, name, destroy=True, use_logging=False): + def spawn_primary(self, name=None, destroy=True, use_logging=False): """ Create a primary node from a backup. Args: - name: name for a new node. + name: primary's application name. destroy: should we convert this backup into a node? use_logging: enable python logging. @@ -127,15 +127,14 @@ def spawn_primary(self, name, destroy=True, use_logging=False): New instance of PostgresNode. """ + # Prepare a data directory for this node base_dir = self._prepare_dir(destroy) # Build a new PostgresNode from .node import PostgresNode - node = PostgresNode( - name=name, - base_dir=base_dir, - master=self.original_node, - use_logging=use_logging) + node = PostgresNode(name=name, + base_dir=base_dir, + use_logging=use_logging) # New nodes should always remove dir tree node._should_rm_dirs = True @@ -145,12 +144,12 @@ def spawn_primary(self, name, destroy=True, use_logging=False): return node - def spawn_replica(self, name, destroy=True, use_logging=False): + def spawn_replica(self, name=None, destroy=True, use_logging=False): """ Create a replica of the original node from a backup. Args: - name: name for a new node. + name: replica's application name. destroy: should we convert this backup into a node? use_logging: enable python logging. @@ -158,9 +157,14 @@ def spawn_replica(self, name, destroy=True, use_logging=False): New instance of PostgresNode. """ - node = self.spawn_primary(name, destroy, use_logging=use_logging) - node._create_recovery_conf(username=self.username, - master=self.original_node) + # Build a new PostgresNode + node = self.spawn_primary(name=name, + destroy=destroy, + use_logging=use_logging) + + # Assign it a master and a recovery file (private magic) + node._assign_master(self.original_node) + node._create_recovery_conf(username=self.username) return node diff --git a/testgres/node.py b/testgres/node.py index 1638f132..723566ac 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -41,6 +41,7 @@ reserve_port as _reserve_port, \ release_port as _release_port, \ default_username as _default_username, \ + generate_app_name as _generate_app_name, \ execute_utility as _execute_utility @@ -60,18 +61,23 @@ def __bool__(self): class PostgresNode(object): - def __init__(self, - name, - port=None, - base_dir=None, - use_logging=False, - master=None): + def __init__(self, name=None, port=None, base_dir=None, use_logging=False): + """ + Create a new node manually. + + Args: + name: node's application name. + port: port to accept connections. + base_dir: path to node's data directory. + use_logging: enable python logging. + """ + global bound_ports # public - self.master = master - self.name = name + self.master = None self.host = '127.0.0.1' + self.name = name or _generate_app_name() self.port = port or _reserve_port() self.base_dir = base_dir @@ -112,7 +118,14 @@ def utils_log_name(self): def pg_log_name(self): return os.path.join(self.logs_dir, _PG_LOG_FILE) - def _create_recovery_conf(self, username, master): + def _assign_master(self, master): + # now this node has a master + self.master = master + + def _create_recovery_conf(self, username): + # fetch master of this node + master = self.master + # yapf: disable conninfo = ( u"application_name={} " @@ -785,7 +798,7 @@ def backup(self, username=None, xlog_method=_DEFAULT_XLOG_METHOD): return NodeBackup(node=self, username=username, xlog_method=xlog_method) def replicate(self, - name, + name=None, username=None, xlog_method=_DEFAULT_XLOG_METHOD, use_logging=False): @@ -793,14 +806,17 @@ def replicate(self, Create a binary replica of this node. Args: - name: replica's name. + name: replica's application name. username: database user name. xlog_method: a method for collecting the logs ('fetch' | 'stream'). use_logging: enable python logging. """ + # transform backup into a replica backup = self.backup(username=username, xlog_method=xlog_method) - return backup.spawn_replica(name=name, use_logging=use_logging) + return backup.spawn_replica(name=name, + destroy=True, + use_logging=use_logging) def catchup(self, dbname='postgres', username=None): """ diff --git a/testgres/utils.py b/testgres/utils.py index cd280221..941e6e4c 100644 --- a/testgres/utils.py +++ b/testgres/utils.py @@ -47,6 +47,15 @@ def default_username(): return pwd.getpwuid(os.getuid())[0] +def generate_app_name(): + """ + Generate a new application name for node. + """ + + import uuid + return ''.join(['testgres-', str(uuid.uuid4())]) + + def execute_utility(args, logfile): """ Execute utility (pg_ctl, pg_dump etc). diff --git a/tests/test_simple.py b/tests/test_simple.py index 79acb294..89b8ede8 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -602,6 +602,21 @@ def test_unix_sockets(self): r.execute('postgres', 'select 1') r.safe_psql('postgres', 'select 1') + def test_auto_name(self): + with get_new_node().init(allow_streaming=True).start() as m: + with m.replicate().start() as r: + + # check that nodes are running + self.assertTrue(m.status()) + self.assertTrue(r.status()) + + # check their names + self.assertIsNotNone(m.name) + self.assertIsNotNone(r.name) + self.assertNotEqual(m.name, r.name) + self.assertTrue('testgres' in m.name) + self.assertTrue('testgres' in r.name) + def test_isolation_levels(self): with get_new_node('node').init().start() as node: with node.connect() as con: From 0756cab552002f72955f35b8c1e9545d703e44f4 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Fri, 12 Jan 2018 17:11:52 +0300 Subject: [PATCH 054/475] show N last log lines in exceptions (impl file_tail()) --- testgres/config.py | 17 +++++++++-------- testgres/node.py | 25 ++++++++++++++++--------- testgres/utils.py | 27 +++++++++++++++++++++++++++ 3 files changed, 52 insertions(+), 17 deletions(-) diff --git a/testgres/config.py b/testgres/config.py index b0026ec4..4be623c8 100644 --- a/testgres/config.py +++ b/testgres/config.py @@ -4,19 +4,20 @@ class TestgresConfig: """ Global config (override default settings). - """ - # shall we cache pg_config results? - cache_pg_config = True + Attributes: + cache_initdb: shall we use cached initdb instance? + cache_pg_config: shall we cache pg_config results? + cached_initdb_dir: shall we create a temp dir for cached initdb? + node_cleanup_full: shall we remove EVERYTHING (including logs)? + error_log_lines: N of log lines to be included into exception (0=inf). + """ - # shall we use cached initdb instance? cache_initdb = True - - # shall we create a temp dir for cached initdb? + cache_pg_config = True cached_initdb_dir = None - - # shall we remove EVERYTHING (including logs)? node_cleanup_full = True + error_log_lines = 20 def configure_testgres(**options): diff --git a/testgres/node.py b/testgres/node.py index 723566ac..b2a8cf20 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -37,6 +37,7 @@ from .utils import \ get_bin_path, \ + file_tail as _file_tail, \ pg_version_ge as _pg_version_ge, \ reserve_port as _reserve_port, \ release_port as _release_port, \ @@ -168,12 +169,12 @@ def _maybe_stop_logger(self): self._logger.stop() def _format_verbose_error(self, message=None): - # list of important files + # list of important files + N of last lines files = [ - os.path.join(self.data_dir, "postgresql.conf"), - os.path.join(self.data_dir, "recovery.conf"), - os.path.join(self.data_dir, "pg_hba.conf"), - self.pg_log_name # main log file + (os.path.join(self.data_dir, "postgresql.conf"), 0), + (os.path.join(self.data_dir, "recovery.conf"), 0), + (os.path.join(self.data_dir, "pg_hba.conf"), 0), + (self.pg_log_name, TestgresConfig.error_log_lines) ] error_text = "" @@ -183,14 +184,20 @@ def _format_verbose_error(self, message=None): error_text += message error_text += '\n' * 2 - for f in files: + for f, num_lines in files: # skip missing files if not os.path.exists(f): continue - # append contents - with io.open(f, "r") as _f: - lines = _f.read() + with io.open(f, "rb") as _f: + if num_lines > 0: + # take last N lines of file + lines = b''.join(_file_tail(_f, num_lines)).decode('utf-8') + else: + # read whole file + lines = _f.read().decode('utf-8') + + # append contents error_text += u"{}:\n----\n{}\n".format(f, lines) return error_text diff --git a/testgres/utils.py b/testgres/utils.py index 941e6e4c..82c8f2f3 100644 --- a/testgres/utils.py +++ b/testgres/utils.py @@ -1,5 +1,7 @@ # coding: utf-8 +from __future__ import division + import io import os import port_for @@ -203,3 +205,28 @@ def pg_version_ge(version): min_ver = LooseVersion(version) return cur_ver >= min_ver + + +def file_tail(f, num_lines): + """ + Get last N lines of a file. + """ + + assert(num_lines > 0) + + bufsize = 8192 + buffers = 1 + + end_pos = f.seek(0, os.SEEK_END) + + while True: + offset = max(0, end_pos - bufsize * buffers) + pos = f.seek(offset, os.SEEK_SET) + + lines = f.readlines() + cur_lines = len(lines) + + if cur_lines > num_lines or pos == 0: + return lines[-num_lines:] + + buffers = int(buffers * max(2, num_lines / max(cur_lines, 1))) From a4b37cae1ed93b78e8cfa3862fb8611c066a0abd Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Fri, 12 Jan 2018 18:38:09 +0300 Subject: [PATCH 055/475] more tests --- tests/test_simple.py | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/tests/test_simple.py b/tests/test_simple.py index 89b8ede8..285f7941 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -598,9 +598,9 @@ def test_unix_sockets(self): node.execute('postgres', 'select 1') node.safe_psql('postgres', 'select 1') - r = node.replicate('r').start() - r.execute('postgres', 'select 1') - r.safe_psql('postgres', 'select 1') + with node.replicate('r').start() as r: + r.execute('postgres', 'select 1') + r.safe_psql('postgres', 'select 1') def test_auto_name(self): with get_new_node().init(allow_streaming=True).start() as m: @@ -617,6 +617,29 @@ def test_auto_name(self): self.assertTrue('testgres' in m.name) self.assertTrue('testgres' in r.name) + def test_file_tail(self): + from testgres.utils import file_tail + + s1 = "the quick brown fox jumped over that lazy dog\n" + s2 = "abc\n" + s3 = "def\n" + + with tempfile.NamedTemporaryFile(mode='r+', delete=True) as f: + for i in range(1, 5000): + f.write(s1) + f.write(s2) + f.write(s3) + + f.seek(0) + lines = file_tail(f, 3) + self.assertEqual(lines[0], s1) + self.assertEqual(lines[1], s2) + self.assertEqual(lines[2], s3) + + f.seek(0) + lines = file_tail(f, 1) + self.assertEqual(lines[0], s3) + def test_isolation_levels(self): with get_new_node('node').init().start() as node: with node.connect() as con: From 35fd3a077d370e2d76fe5808ff6cc8a9abe9f825 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Sat, 13 Jan 2018 17:16:39 +0300 Subject: [PATCH 056/475] fixes for python 2 --- testgres/utils.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/testgres/utils.py b/testgres/utils.py index 82c8f2f3..c46b65bd 100644 --- a/testgres/utils.py +++ b/testgres/utils.py @@ -217,11 +217,13 @@ def file_tail(f, num_lines): bufsize = 8192 buffers = 1 - end_pos = f.seek(0, os.SEEK_END) + f.seek(0, os.SEEK_END) + end_pos = f.tell() while True: offset = max(0, end_pos - bufsize * buffers) - pos = f.seek(offset, os.SEEK_SET) + f.seek(offset, os.SEEK_SET) + pos = f.tell() lines = f.readlines() cur_lines = len(lines) From 4eb329f4743922f61c7a95b2c45d0ad2d1416914 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Thu, 18 Jan 2018 15:42:39 +0300 Subject: [PATCH 057/475] fix docstrings --- .travis.yml | 2 +- mk_dockerfile.sh | 2 ++ testgres/api.py | 31 +++++++++++++++++++------------ 3 files changed, 22 insertions(+), 13 deletions(-) create mode 100755 mk_dockerfile.sh diff --git a/.travis.yml b/.travis.yml index dfd39b62..3fd628cb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,7 @@ services: - docker install: - - sed -e 's/${PYTHON_VERSION}/'${PYTHON_VERSION}/g -e 's/${PG_VERSION}/'${PG_VERSION}/g Dockerfile.tmpl > Dockerfile + - ./mk_dockerfile.sh - docker-compose build script: diff --git a/mk_dockerfile.sh b/mk_dockerfile.sh new file mode 100755 index 00000000..d2aa3a8a --- /dev/null +++ b/mk_dockerfile.sh @@ -0,0 +1,2 @@ +set -eu +sed -e 's/${PYTHON_VERSION}/'${PYTHON_VERSION}/g -e 's/${PG_VERSION}/'${PG_VERSION}/g Dockerfile.tmpl > Dockerfile diff --git a/testgres/api.py b/testgres/api.py index 4b607423..685708e1 100644 --- a/testgres/api.py +++ b/testgres/api.py @@ -7,21 +7,28 @@ edit configuration files, start/stop cluster, execute queries. The typical flow may look like: - with get_new_node() as node: - node.init() - node.start() - result = node.psql('postgres', 'SELECT 1') - print(result) - node.stop() +>>> with get_new_node() as node: +... node.init().start() +... result = node.safe_psql('postgres', 'select 1') +... print(result.decode('utf-8').strip()) +... node.stop() + +1 + Or: - with get_new_node() as node1: - node1.init().start() - with node1.backup() as backup: - with backup.spawn_primary() as node2: - res = node2.start().execute('postgres', 'select 2') - print(res) +>>> with get_new_node() as master: +... master.init().start() +... with master.backup() as backup: +... with backup.spawn_replica() as replica: +... replica = replica.start() +... master.execute('postgres', 'create table test (val int4)') +... master.execute('postgres', 'insert into test values (0), (1), (2)') +... replica.catchup() # wait until changes are visible +... print(replica.execute('postgres', 'select count(*) from test')) + +[(3,)] Copyright (c) 2016, Postgres Professional """ From cd2ce892c982dec64cdbc9bd43b130a38c57753c Mon Sep 17 00:00:00 2001 From: Grigory Smolkin Date: Sat, 27 Jan 2018 15:40:27 +0300 Subject: [PATCH 058/475] fix base_dir ignore if provided base_dir do not exists --- testgres/node.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/testgres/node.py b/testgres/node.py index b2a8cf20..93766d2f 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -151,9 +151,12 @@ def _create_recovery_conf(self, username): self.append_conf("recovery.conf", line) def _prepare_dirs(self): - if not self.base_dir or not os.path.exists(self.base_dir): + if not self.base_dir: self.base_dir = tempfile.mkdtemp() + if not os.path.exists(self.base_dir): + os.makedirs(self.base_dir) + if not os.path.exists(self.logs_dir): os.makedirs(self.logs_dir) From 1b1d45623e5b80f1fa740514f53e6711f7fbd95c Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Wed, 31 Jan 2018 21:49:44 +0300 Subject: [PATCH 059/475] bring pgbench_init() back, make pgbench_run() much more flexible --- README.md | 8 +++--- testgres/node.py | 64 +++++++++++++++++++++++++++++++++++++++++--- tests/test_simple.py | 8 +++--- 3 files changed, 68 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index aad1a3ea..acec819c 100644 --- a/README.md +++ b/README.md @@ -154,11 +154,9 @@ with testgres.get_new_node('master') as master: # start a new node master.init().start() - # initialize default database for TPC-B - master.pgbench_run(options=['-i']) - - # run benchmark for 20 seconds and show results - print(master.pgbench_run(options=['-T', '20'])) + # initialize default DB and run bench for 10 seconds + res = master.pgbench_init(scale=2).pgbench_run(time=10) + print(res) ``` diff --git a/testgres/node.py b/testgres/node.py index b2a8cf20..f1714777 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -3,6 +3,7 @@ import io import os import shutil +import six import subprocess import tempfile import time @@ -856,12 +857,18 @@ def catchup(self, dbname='postgres', username=None): except Exception as e: raise_from(CatchUpException('Failed to catch up'), e) - def pgbench(self, dbname='postgres', stdout=None, stderr=None, options=[]): + def pgbench(self, + dbname='postgres', + username=None, + stdout=None, + stderr=None, + options=[]): """ Spawn a pgbench process. Args: dbname: database name to connect to. + username: database user name. stdout: stdout file to be used by Popen. stderr: stderr file to be used by Popen. options: additional options for pgbench (list). @@ -870,36 +877,85 @@ def pgbench(self, dbname='postgres', stdout=None, stderr=None, options=[]): Process created by subprocess.Popen. """ + # Set default arguments + username = username or _default_username() + # yapf: disable _params = [ get_bin_path("pgbench"), "-p", str(self.port), "-h", self.host, - ] + options + [dbname] + "-U", username, + ] + options + + # should be the last one + _params.append(dbname) proc = subprocess.Popen(_params, stdout=stdout, stderr=stderr) return proc - def pgbench_run(self, dbname='postgres', options=[]): + def pgbench_init(self, **kwargs): + """ + Small wrapper for pgbench_run(). + Sets initialize=True. + + Returns: + This instance of PostgresNode. + """ + + self.pgbench_run(initialize=True, **kwargs) + + return self + + def pgbench_run(self, + dbname='postgres', + username=None, + options=[], + **kwargs): """ Run pgbench with some options. This event is logged (see self.utils_log_name). Args: dbname: database name to connect to. + username: database user name. options: additional options for pgbench (list). + **kwargs: named options for pgbench. + Examples: + pgbench_run(initialize=True, scale=2) + pgbench_run(time=10) + Run pgbench --help to learn more. + Returns: Stdout produced by pgbench. """ + # Set default arguments + username = username or _default_username() + # yapf: disable _params = [ get_bin_path("pgbench"), "-p", str(self.port), "-h", self.host, - ] + options + [dbname] + "-U", username, + ] + options + + for key, value in six.iteritems(kwargs): + # rename keys for pgbench + key = key.replace('_', '-') + + # append option + if not isinstance(value, bool): + _params.append('--{}={}'.format(key, value)) + else: + assert (value is True) # just in case + _params.append('--{}'.format(key)) + + # should be the last one + _params.append(dbname) return _execute_utility(_params, self.utils_log_name) diff --git a/tests/test_simple.py b/tests/test_simple.py index 285f7941..fe52ad5f 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -505,14 +505,16 @@ def test_pgbench(self): with get_new_node('node') as node: node.init().start() - # initialize pgbench - node.pgbench_run(options=['-i']) + # initialize pgbench DB and run benchmarks + node.pgbench_init(scale=2, + foreign_keys=True, + options=['-q']).pgbench_run(time=2) # run TPC-B benchmark proc = node.pgbench( stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - options=['-T5']) + options=['-T3']) out, _ = proc.communicate() out = out.decode('utf-8') From 370c9341b2d85b23544e04f511b073514f0828b0 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Thu, 1 Feb 2018 14:56:35 +0300 Subject: [PATCH 060/475] PostgresNode.master should be a property --- testgres/node.py | 26 +++++++++++++------------- testgres/utils.py | 2 +- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index 22ae4216..38715d77 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -74,20 +74,18 @@ def __init__(self, name=None, port=None, base_dir=None, use_logging=False): use_logging: enable python logging. """ - global bound_ports - # public - self.master = None self.host = '127.0.0.1' self.name = name or _generate_app_name() self.port = port or _reserve_port() self.base_dir = base_dir # private - self._should_free_port = port is None self._should_rm_dirs = base_dir is None + self._should_free_port = port is None self._use_logging = use_logging self._logger = None + self._master = None # create directories if needed self._prepare_dirs() @@ -96,14 +94,16 @@ def __enter__(self): return self def __exit__(self, type, value, traceback): - global bound_ports - # stop node if necessary self.cleanup() # free port if necessary self.free_port() + @property + def master(self): + return self._master + @property def data_dir(self): return os.path.join(self.base_dir, _DATA_DIR) @@ -122,11 +122,12 @@ def pg_log_name(self): def _assign_master(self, master): # now this node has a master - self.master = master + self._master = master def _create_recovery_conf(self, username): # fetch master of this node master = self.master + assert (master is not None) # yapf: disable conninfo = ( @@ -834,7 +835,8 @@ def catchup(self, dbname='postgres', username=None): Wait until async replica catches up with its master. """ - master = self.master + if not self.master: + raise CatchUpException("Node doesn't have a master") if _pg_version_ge('10'): poll_lsn = "select pg_current_wal_lsn()::text" @@ -843,13 +845,11 @@ def catchup(self, dbname='postgres', username=None): poll_lsn = "select pg_current_xlog_location()::text" wait_lsn = "select pg_last_xlog_replay_location() >= '{}'::pg_lsn" - if not master: - raise CatchUpException("Master node is not specified") - try: # fetch latest LSN - lsn = master.execute( - dbname=dbname, username=username, query=poll_lsn)[0][0] + lsn = self.master.execute(dbname=dbname, + username=username, + query=poll_lsn)[0][0] # wait until this LSN reaches replica self.poll_query_until( diff --git a/testgres/utils.py b/testgres/utils.py index c46b65bd..7baa77fa 100644 --- a/testgres/utils.py +++ b/testgres/utils.py @@ -212,7 +212,7 @@ def file_tail(f, num_lines): Get last N lines of a file. """ - assert(num_lines > 0) + assert (num_lines > 0) bufsize = 8192 buffers = 1 From 5e3c0f3327ceee5e9418d86637018403857d98e5 Mon Sep 17 00:00:00 2001 From: Ivan Kartyshov Date: Sun, 4 Feb 2018 19:50:47 +0300 Subject: [PATCH 061/475] adding key -F to pg_dump --- setup.py | 0 testgres/node.py | 12 ++-- tests/test_pgdump.py | 140 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 148 insertions(+), 4 deletions(-) mode change 100644 => 100755 setup.py create mode 100755 tests/test_pgdump.py diff --git a/setup.py b/setup.py old mode 100644 new mode 100755 diff --git a/testgres/node.py b/testgres/node.py index b2a8cf20..18cf3be1 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -583,8 +583,8 @@ def cleanup(self, max_attempts=3): def psql(self, dbname, query=None, - filename=None, username=None, + filename=None, input=None): """ Execute a query using psql. @@ -658,7 +658,7 @@ def safe_psql(self, dbname, query, username=None, input=None): return out - def dump(self, dbname, username=None, filename=None): + def dump(self, dbname, username=None, filename=None, formate=None): """ Dump database into a file using pg_dump. NOTE: the file is not removed automatically. @@ -667,6 +667,7 @@ def dump(self, dbname, username=None, filename=None): dbname: database name to connect to. username: database user name. filename: output file. + formate: format argument p/c/d/t Returns: Path to a file containing dump. @@ -674,6 +675,7 @@ def dump(self, dbname, username=None, filename=None): # Set default arguments username = username or _default_username() + formate = formate or "p" f, filename = filename or tempfile.mkstemp() os.close(f) @@ -684,7 +686,8 @@ def dump(self, dbname, username=None, filename=None): "-h", self.host, "-f", filename, "-U", username, - "-d", dbname + "-d", dbname, + "-F", formate ] _execute_utility(_params, self.utils_log_name) @@ -697,10 +700,11 @@ def restore(self, dbname, filename, username=None): Args: dbname: database name to connect to. + username: database user name. filename: database dump taken by pg_dump. """ - self.psql(dbname=dbname, filename=filename, username=username) + self.psql(dbname=dbname, username=username, filename=filename) def poll_query_until(self, dbname, diff --git a/tests/test_pgdump.py b/tests/test_pgdump.py new file mode 100755 index 00000000..9ebf2b81 --- /dev/null +++ b/tests/test_pgdump.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python +# coding: utf-8 + +import os +import subprocess +import tempfile +import testgres +import time +import unittest + +import pdb + +import logging.config + +from distutils.version import LooseVersion + +from testgres import \ + InitNodeException, \ + StartNodeException, \ + ExecUtilException, \ + BackupException, \ + QueryException, \ + CatchUpException, \ + TimeoutException + +from testgres import \ + TestgresConfig, \ + configure_testgres + +from testgres import \ + NodeStatus, \ + IsolationLevel, \ + get_new_node + +from testgres import \ + get_bin_path, \ + get_pg_config + +from testgres import bound_ports + +# FIRST test from past +# with testgres.get_new_node('test') as node: +# node.init() # run initdb +# node.start() # start PostgreSQL +# #print(node.execute('postgres', 'select 1')) +# #print(node.psql('postgres', 'select 1')) +# print(node.connect('postgres', 'vis')) +# with node.connect() as con: +# con.begin('serializable') +# print(con.execute('select %s', 1)) +# con.rollback() +# node.stop() # stop PostgreSQL + + +# test replication behavoiur on HOT_STANDBY_FEEDBACK +# def set_trace(con, command="pg_debug"): +# pid = con.execute("select pg_backend_pid()")[0][0] +# p = subprocess.Popen([command], stdin=subprocess.PIPE) +# p.communicate(str(pid).encode()) + +# with get_new_node() as master: +# master.init().start() +# with master.backup() as backup: +# with backup.spawn_replica() as replica: +# replica = replica.start() +# master.execute('postgres', 'create table test (val int4)') +# master.execute('postgres', 'insert into test values (0), (1), (2)') +# replica.catchup() # wait until changes are visible +# with replica.connect() as con1: +# set_trace(con1) +# import pdb; pdb.set_trace() # Важно,если последний идет pdb,то pass +# pass + + # print(replica.execute('postgres', 'select count(*) from test')) + #print(replica.execute('postgres', ':gdb')) + + + +# SECOND test dump new keys +with get_new_node('node1') as node1: + node1.init().start() + + with node1.connect('postgres') as con: + con.begin() + con.execute('create table test (val int)') + con.execute('insert into test values (1), (2)') + con.commit() + + # take a new dump plain format + dump = node1.dump('postgres') + # self.assertTrue(os.path.isfile(dump)) + with get_new_node('node2') as node2: + node2.init().start().restore('postgres', dump) + res = node2.execute('postgres','select * from test order by val asc') + # self.assertListEqual(res, [(1, ), (2, )]) + # finally, remove dump + os.remove(dump) + + # take a new dump custom format + dump = node1.dump('postgres') + with get_new_node('node2') as node2: + node2.init().start().restore('postgres', dump) + res = node2.execute('postgres','select * from test order by val asc') + os.remove(dump) + + # take a new dump directory format + dump = node1.dump('postgres') + with get_new_node('node2') as node2: + node2.init().start().restore('postgres', dump) + res = node2.execute('postgres','select * from test order by val asc') + os.remove(dump) + + # take a new dump tar format + dump = node1.dump('postgres') + with get_new_node('node2') as node2: + node2.init().start().restore('postgres', dump) + res = node2.execute('postgres','select * from test order by val asc') + os.remove(dump) + + # take a new dump tar format + dump = node1.dump('postgres') + with get_new_node('node2') as node2: + node2.init().start().restore('postgres', dump) + res = node2.execute('postgres','select * from test order by val asc') + os.remove(dump) + +# 1) make dump to new place +# pg_dump mydb > db.sql +# 2) make dump to non default formate +# 2.5) Чтобы сформировать выгрузку в формате plain: +# pg_dump -Fp mydb > db.dump +# 2.1) Чтобы сформировать выгрузку в формате custom: +# pg_dump -Fc mydb > db.dump +# 2.2) Чтобы сформировать выгрузку в формате directory: +# pg_dump -Fd mydb -f dumpdir +# 2.3) ? Чтобы сформировать выгрузку в формате directory в 5 параллельных потоков: +# pg_dump -Fd mydb -j 5 -f dumpdir +# 2.4) Чтобы сформировать выгрузку в формате tar: +# pg_dump -Ft mydb > db.dump + From 2a64ba5df14aa047f8c68cbc3e418c56f1fa586d Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Sun, 4 Feb 2018 23:56:22 +0300 Subject: [PATCH 062/475] various improvements, change args of query performing functions * changed args of psql(), safe_psql(), execute(), poll_query_until() * introduced positional_args_hack() decorator for API functions * replaced inline config names with constants in consts.py * introduced default_dbname(), changed default_username() --- .gitignore | 15 ++-- testgres/backup.py | 5 +- testgres/connection.py | 38 +++++++--- testgres/consts.py | 4 ++ testgres/logger.py | 2 +- testgres/node.py | 154 ++++++++++++++++++++++++++++------------- testgres/utils.py | 90 ++++++++++++++++++++++-- tests/test_simple.py | 91 +++++++++++------------- 8 files changed, 279 insertions(+), 120 deletions(-) diff --git a/.gitignore b/.gitignore index fbf7b5a7..aebd9e97 100644 --- a/.gitignore +++ b/.gitignore @@ -1,12 +1,17 @@ *.pyc -dist -tags -*.egg-info/ *.egg -Dockerfile -.coverage +*.egg-info/ +dist + env/ venv/ build/ + +.coverage coverage.xml + +Dockerfile + +*~ *.swp +tags diff --git a/testgres/backup.py b/testgres/backup.py index da561cd4..fe64a137 100644 --- a/testgres/backup.py +++ b/testgres/backup.py @@ -8,6 +8,7 @@ from .consts import \ DATA_DIR as _DATA_DIR, \ + PG_CONF_FILE as _PG_CONF_FILE, \ BACKUP_LOG_FILE as _BACKUP_LOG_FILE, \ DEFAULT_XLOG_METHOD as _DEFAULT_XLOG_METHOD @@ -139,8 +140,8 @@ def spawn_primary(self, name=None, destroy=True, use_logging=False): # New nodes should always remove dir tree node._should_rm_dirs = True - node.append_conf("postgresql.conf", "\n") - node.append_conf("postgresql.conf", "port = {}".format(node.port)) + node.append_conf(_PG_CONF_FILE, "\n") + node.append_conf(_PG_CONF_FILE, "port = {}".format(node.port)) return node diff --git a/testgres/connection.py b/testgres/connection.py index 8577dd7f..da4f92e0 100644 --- a/testgres/connection.py +++ b/testgres/connection.py @@ -12,7 +12,11 @@ from enum import Enum from .exceptions import QueryException -from .utils import default_username as _default_username + +from .utils import \ + default_dbname as _default_dbname, \ + default_username as _default_username + # export these exceptions InternalError = pglib.InternalError @@ -33,25 +37,37 @@ class NodeConnection(object): """ def __init__(self, - parent_node, - dbname, - host="127.0.0.1", + node, + dbname=None, username=None, password=None): # Use default user if not specified + dbname = dbname or _default_dbname() username = username or _default_username() - self.parent_node = parent_node + self._node = node - self.connection = pglib.connect( + self._connection = pglib.connect( database=dbname, user=username, - port=parent_node.port, - host=host, - password=password) + password=password, + host=node.host, + port=node.port) + + self._cursor = self.connection.cursor() + + @property + def node(self): + return self._node + + @property + def connection(self): + return self._connection - self.cursor = self.connection.cursor() + @property + def cursor(self): + return self._cursor def __enter__(self): return self @@ -73,7 +89,7 @@ def begin(self, isolation_level=IsolationLevel.ReadCommitted): # Get index of isolation level level_idx = isolation_level.value - assert(level_idx in range(4)) + assert level_idx in range(4) # Replace isolation level with its name isolation_level = levels[level_idx] diff --git a/testgres/consts.py b/testgres/consts.py index 69b51778..e42a4fa7 100644 --- a/testgres/consts.py +++ b/testgres/consts.py @@ -4,6 +4,10 @@ DATA_DIR = "data" LOGS_DIR = "logs" +RECOVERY_CONF_FILE = "recovery.conf" +PG_CONF_FILE = "postgresql.conf" +HBA_CONF_FILE = "pg_hba.conf" + # names for log files PG_LOG_FILE = "postgresql.log" UTILS_LOG_FILE = "utils.log" diff --git a/testgres/logger.py b/testgres/logger.py index 20e1b60a..ee700f95 100644 --- a/testgres/logger.py +++ b/testgres/logger.py @@ -8,7 +8,7 @@ class TestgresLogger(threading.Thread): """ - Helper class to implement reading from postgresql.log + Helper class to implement reading from log files. """ def __init__(self, node_name, log_file_name): diff --git a/testgres/node.py b/testgres/node.py index 38715d77..89284e7e 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -23,6 +23,9 @@ from .consts import \ DATA_DIR as _DATA_DIR, \ LOGS_DIR as _LOGS_DIR, \ + PG_CONF_FILE as _PG_CONF_FILE, \ + HBA_CONF_FILE as _HBA_CONF_FILE, \ + RECOVERY_CONF_FILE as _RECOVERY_CONF_FILE, \ PG_LOG_FILE as _PG_LOG_FILE, \ UTILS_LOG_FILE as _UTILS_LOG_FILE, \ DEFAULT_XLOG_METHOD as _DEFAULT_XLOG_METHOD @@ -42,9 +45,12 @@ pg_version_ge as _pg_version_ge, \ reserve_port as _reserve_port, \ release_port as _release_port, \ + default_dbname as _default_dbname, \ default_username as _default_username, \ generate_app_name as _generate_app_name, \ - execute_utility as _execute_utility + execute_utility as _execute_utility, \ + method_decorator, \ + positional_args_hack class NodeStatus(Enum): @@ -63,7 +69,11 @@ def __bool__(self): class PostgresNode(object): - def __init__(self, name=None, port=None, base_dir=None, use_logging=False): + def __init__(self, + name=None, + port=None, + base_dir=None, + use_logging=False): """ Create a new node manually. @@ -121,13 +131,17 @@ def pg_log_name(self): return os.path.join(self.logs_dir, _PG_LOG_FILE) def _assign_master(self, master): + """NOTE: this is a private method!""" + # now this node has a master self._master = master def _create_recovery_conf(self, username): + """NOTE: this is a private method!""" + # fetch master of this node master = self.master - assert (master is not None) + assert master is not None # yapf: disable conninfo = ( @@ -150,9 +164,11 @@ def _create_recovery_conf(self, username): "standby_mode=on\n" ).format(conninfo) - self.append_conf("recovery.conf", line) + self.append_conf(_RECOVERY_CONF_FILE, line) def _prepare_dirs(self): + """NOTE: this is a private method!""" + if not self.base_dir: self.base_dir = tempfile.mkdtemp() @@ -163,6 +179,8 @@ def _prepare_dirs(self): os.makedirs(self.logs_dir) def _maybe_start_logger(self): + """NOTE: this is a private method!""" + if self._use_logging: # spawn new logger if it doesn't exist or stopped if not self._logger or not self._logger.is_alive(): @@ -170,15 +188,19 @@ def _maybe_start_logger(self): self._logger.start() def _maybe_stop_logger(self): + """NOTE: this is a private method!""" + if self._logger: self._logger.stop() def _format_verbose_error(self, message=None): + """NOTE: this is a private method!""" + # list of important files + N of last lines files = [ - (os.path.join(self.data_dir, "postgresql.conf"), 0), - (os.path.join(self.data_dir, "recovery.conf"), 0), - (os.path.join(self.data_dir, "pg_hba.conf"), 0), + (os.path.join(self.data_dir, _PG_CONF_FILE), 0), + (os.path.join(self.data_dir, _HBA_CONF_FILE), 0), + (os.path.join(self.data_dir, _RECOVERY_CONF_FILE), 0), (self.pg_log_name, TestgresConfig.error_log_lines) ] @@ -257,8 +279,8 @@ def default_conf(self, This instance of PostgresNode. """ - postgres_conf = os.path.join(self.data_dir, "postgresql.conf") - hba_conf = os.path.join(self.data_dir, "pg_hba.conf") + postgres_conf = os.path.join(self.data_dir, _PG_CONF_FILE) + hba_conf = os.path.join(self.data_dir, _HBA_CONF_FILE) # filter lines in hba file with io.open(hba_conf, "r+") as conf: @@ -296,7 +318,7 @@ def get_auth_method(t): if line not in lines: conf.write(line) - # overwrite postgresql.conf file + # overwrite config file with io.open(postgres_conf, "w") as conf: # remove old lines conf.truncate() @@ -585,27 +607,30 @@ def cleanup(self, max_attempts=3): return self + @method_decorator(positional_args_hack(['query'], ['dbname', 'query'])) def psql(self, - dbname, + *, query=None, filename=None, + dbname=None, username=None, input=None): """ Execute a query using psql. Args: + query: query to be executed. + filename: file with a query. dbname: database name to connect to. username: database user name. - filename: file with a query. - query: query to be executed. input: raw input to be passed. Returns: A tuple of (code, stdout, stderr). """ - # Set default username + # Set default arguments + dbname = dbname or _default_dbname() username = username or _default_username() # yapf: disable @@ -626,8 +651,6 @@ def psql(self, psql_params.extend(("-c", query)) elif filename: psql_params.extend(("-f", filename)) - elif input: - pass else: raise QueryException('Query or filename must be provided') @@ -642,45 +665,58 @@ def psql(self, out, err = process.communicate(input=input) return process.returncode, out, err - def safe_psql(self, dbname, query, username=None, input=None): + @method_decorator(positional_args_hack(['dbname', 'query'])) + def safe_psql(self, + query, + *, + dbname=None, + username=None, + input=None): """ Execute a query using psql. Args: + query: query to be executed. dbname: database name to connect to. username: database user name. - query: query to be executed. input: raw input to be passed. Returns: psql's output as str. """ - ret, out, err = self.psql(dbname, query, username=username, input=input) + ret, out, err = self.psql(query=query, + dbname=dbname, + username=username, + input=input) if ret: - err = '' if not err else err.decode('utf-8') - raise QueryException(err) + raise QueryException((err or b'').decode('utf-8')) return out - def dump(self, dbname, username=None, filename=None): + def dump(self, filename=None, dbname=None, username=None): """ Dump database into a file using pg_dump. NOTE: the file is not removed automatically. Args: + filename: database dump taken by pg_dump. dbname: database name to connect to. username: database user name. - filename: output file. Returns: Path to a file containing dump. """ + def tmpfile(): + fd, fname = tempfile.mkstemp() + os.close(fd) + return fname + # Set default arguments + dbname = dbname or _default_dbname() username = username or _default_username() - f, filename = filename or tempfile.mkstemp() - os.close(f) + filename = filename or tmpfile() # yapf: disable _params = [ @@ -696,20 +732,23 @@ def dump(self, dbname, username=None, filename=None): return filename - def restore(self, dbname, filename, username=None): + def restore(self, filename, dbname=None, username=None): """ Restore database from pg_dump's file. Args: - dbname: database name to connect to. filename: database dump taken by pg_dump. + dbname: database name to connect to. + username: database user name. """ - self.psql(dbname=dbname, filename=filename, username=username) + self.psql(filename=filename, dbname=dbname, username=username) + @method_decorator(positional_args_hack(['dbname', 'query'])) def poll_query_until(self, - dbname, query, + *, + dbname=None, username=None, max_attempts=0, sleep_time=1, @@ -734,8 +773,8 @@ def poll_query_until(self, """ # sanity checks - assert (max_attempts >= 0) - assert (sleep_time > 0) + assert max_attempts >= 0 + assert sleep_time > 0 attempts = 0 while max_attempts == 0 or attempts < max_attempts: @@ -774,24 +813,37 @@ def poll_query_until(self, raise TimeoutException('Query timeout') - def execute(self, dbname, query, username=None, commit=True): + @method_decorator(positional_args_hack(['dbname', 'query'])) + def execute(self, + query, + *, + dbname=None, + username=None, + password=None, + commit=True): """ Execute a query and return all rows as list. Args: - dbname: database name to connect to. query: query to be executed. + dbname: database name to connect to. username: database user name. + password: user's password. commit: should we commit this query? Returns: A list of tuples representing rows. """ - with self.connect(dbname, username) as node_con: + with self.connect(dbname=dbname, + username=username, + password=password) as node_con: + res = node_con.execute(query) + if commit: node_con.commit() + return res def backup(self, username=None, xlog_method=_DEFAULT_XLOG_METHOD): @@ -807,7 +859,9 @@ def backup(self, username=None, xlog_method=_DEFAULT_XLOG_METHOD): """ from .backup import NodeBackup - return NodeBackup(node=self, username=username, xlog_method=xlog_method) + return NodeBackup(node=self, + username=username, + xlog_method=xlog_method) def replicate(self, name=None, @@ -824,13 +878,14 @@ def replicate(self, use_logging: enable python logging. """ - # transform backup into a replica backup = self.backup(username=username, xlog_method=xlog_method) + + # transform backup into a replica return backup.spawn_replica(name=name, destroy=True, use_logging=use_logging) - def catchup(self, dbname='postgres', username=None): + def catchup(self, dbname=None, username=None): """ Wait until async replica catches up with its master. """ @@ -847,21 +902,21 @@ def catchup(self, dbname='postgres', username=None): try: # fetch latest LSN - lsn = self.master.execute(dbname=dbname, - username=username, - query=poll_lsn)[0][0] + lsn = self.master.execute(query=poll_lsn, + dbname=dbname, + username=username)[0][0] # wait until this LSN reaches replica self.poll_query_until( + query=wait_lsn.format(lsn), dbname=dbname, username=username, - query=wait_lsn.format(lsn), max_attempts=0) # infinite except Exception as e: raise_from(CatchUpException('Failed to catch up'), e) def pgbench(self, - dbname='postgres', + dbname=None, username=None, stdout=None, stderr=None, @@ -881,6 +936,7 @@ def pgbench(self, """ # Set default arguments + dbname = dbname or _default_dbname() username = username or _default_username() # yapf: disable @@ -912,7 +968,7 @@ def pgbench_init(self, **kwargs): return self def pgbench_run(self, - dbname='postgres', + dbname=None, username=None, options=[], **kwargs): @@ -936,6 +992,7 @@ def pgbench_run(self, """ # Set default arguments + dbname = dbname or _default_dbname() username = username or _default_username() # yapf: disable @@ -954,7 +1011,7 @@ def pgbench_run(self, if not isinstance(value, bool): _params.append('--{}={}'.format(key, value)) else: - assert (value is True) # just in case + assert value is True # just in case _params.append('--{}'.format(key)) # should be the last one @@ -962,19 +1019,20 @@ def pgbench_run(self, return _execute_utility(_params, self.utils_log_name) - def connect(self, dbname='postgres', username=None): + def connect(self, dbname=None, username=None, password=None): """ Connect to a database. Args: dbname: database name to connect to. username: database user name. + password: user's password. Returns: An instance of NodeConnection. """ - return NodeConnection(parent_node=self, - host=self.host, + return NodeConnection(node=self, dbname=dbname, - username=username) + username=username, + password=password) diff --git a/testgres/utils.py b/testgres/utils.py index 7baa77fa..b45104d5 100644 --- a/testgres/utils.py +++ b/testgres/utils.py @@ -2,9 +2,11 @@ from __future__ import division +import functools import io import os import port_for +import six import subprocess from distutils.version import LooseVersion @@ -40,13 +42,21 @@ def release_port(port): bound_ports.remove(port) +def default_dbname(): + """ + Return default DB name. + """ + + return 'postgres' + + def default_username(): """ - Return current user. + Return default username (current user). """ - import pwd # used only here - return pwd.getpwuid(os.getuid())[0] + import getpass + return getpass.getuser() def generate_app_name(): @@ -212,7 +222,7 @@ def file_tail(f, num_lines): Get last N lines of a file. """ - assert (num_lines > 0) + assert num_lines > 0 bufsize = 8192 buffers = 1 @@ -232,3 +242,75 @@ def file_tail(f, num_lines): return lines[-num_lines:] buffers = int(buffers * max(2, num_lines / max(cur_lines, 1))) + + +def positional_args_hack(*special_cases): + """ + Convert positional args described by + 'special_cases' into named args. + + Example: + @positional_args_hack([['dbname', 'query']]) + def some_api_func(...) + + This is useful for compatibility. + """ + + cases = dict() + + for case in special_cases: + k = len(case) + assert k not in six.iterkeys(cases), 'len must be unique' + cases[k] = case + + def decorator(function): + @functools.wraps(function) + def wrapper(*args, **kwargs): + k = len(args) + + if k in six.iterkeys(cases): + case = cases[k] + + for i in range(0, k): + arg_name = case[i] + arg_val = args[i] + + # transform into named + kwargs[arg_name] = arg_val + + # get rid of them + args = [] + + return function(*args, **kwargs) + + return wrapper + + return decorator + + +def method_decorator(decorator): + """ + Convert a function decorator into a method decorator. + """ + + def _dec(func): + def _wrapper(self, *args, **kwargs): + @decorator + def bound_func(*args2, **kwargs2): + return func.__get__(self, type(self))(*args2, **kwargs2) + + # 'bound_func' is a closure and can see 'self' + return bound_func(*args, **kwargs) + + # preserve docs + functools.update_wrapper(_wrapper, func) + + return _wrapper + + # preserve docs + functools.update_wrapper(_dec, decorator) + + # change name for easier debugging + _dec.__name__ = 'method_decorator({})'.format(decorator.__name__) + + return _dec diff --git a/tests/test_simple.py b/tests/test_simple.py index fe52ad5f..a24decd5 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -124,32 +124,32 @@ def test_restart(self): node.restart() def test_psql(self): - with get_new_node('test') as node: + with get_new_node() as node: node.init().start() - # check default params - with self.assertRaises(QueryException): - node.psql('postgres') + # check returned values (1 arg) + res = node.psql('select 1') + self.assertEqual(res, (0, b'1\n', b'')) - # check returned values - res = node.psql('postgres', 'select 1') - self.assertEqual(res[0], 0) - self.assertEqual(res[1], b'1\n') - self.assertEqual(res[2], b'') + # check returned values (2 args) + res = node.psql('postgres', 'select 2') + self.assertEqual(res, (0, b'2\n', b'')) - # check returned values - res = node.safe_psql('postgres', 'select 1') - self.assertEqual(res, b'1\n') + # check returned values (1 arg) + res = node.safe_psql('select 3') + self.assertEqual(res, b'3\n') + + # check returned values (2 args) + res = node.safe_psql('postgres', 'select 4') + self.assertEqual(res, b'4\n') # check feeding input - node.safe_psql('postgres', 'create table horns (w int)') + node.safe_psql('create table horns (w int)') node.safe_psql( - 'postgres', 'copy horns from stdin (format csv)', input=b"1\n2\n3\n\.\n") - sum = node.safe_psql('postgres', 'select sum(w) from horns') - self.assertEqual(sum, b'6\n') - node.safe_psql('postgres', 'drop table horns') + _sum = node.safe_psql('select sum(w) from horns') + self.assertEqual(_sum, b'6\n') node.stop() @@ -351,24 +351,23 @@ def test_incorrect_catchup(self): node.catchup() def test_dump(self): - with get_new_node('node1') as node1: - node1.init().start() + query_create = 'create table test as select generate_series(1, 2) as val' + query_select = 'select * from test order by val asc' - with node1.connect('postgres') as con: - con.begin() - con.execute('create table test (val int)') - con.execute('insert into test values (1), (2)') - con.commit() + with get_new_node().init().start() as node1: + + node1.execute('postgres', query_create) # take a new dump - dump = node1.dump('postgres') + dump = node1.dump() self.assertTrue(os.path.isfile(dump)) - with get_new_node('node2') as node2: - node2.init().start().restore('postgres', dump) + with get_new_node().init().start() as node2: + + # restore dump + node2.restore(filename=dump) - res = node2.execute('postgres', - 'select * from test order by val asc') + res = node2.execute('postgres', query_select) self.assertListEqual(res, [(1, ), (2, )]) # finally, remove dump @@ -378,7 +377,7 @@ def test_users(self): with get_new_node('master') as node: node.init().start() node.psql('postgres', 'create role test_user login') - value = node.safe_psql('postgres', 'select 1', username='test_user') + value = node.safe_psql('select 1', username='test_user') self.assertEqual(value, b'1\n') def test_poll_query_until(self): @@ -388,68 +387,62 @@ def test_poll_query_until(self): get_time = 'select extract(epoch from now())' check_time = 'select extract(epoch from now()) - {} >= 5' - start_time = node.execute('postgres', get_time)[0][0] - node.poll_query_until( - dbname='postgres', query=check_time.format(start_time)) - end_time = node.execute('postgres', get_time)[0][0] + start_time = node.execute(get_time)[0][0] + node.poll_query_until(query=check_time.format(start_time)) + end_time = node.execute(get_time)[0][0] self.assertTrue(end_time - start_time >= 5) # check 0 rows with self.assertRaises(QueryException): node.poll_query_until( - dbname='postgres', query='select * from pg_class where true = false') # check 0 columns with self.assertRaises(QueryException): - node.poll_query_until( - dbname='postgres', query='select from pg_class limit 1') + node.poll_query_until(query='select from pg_class limit 1') # check None, fail with self.assertRaises(QueryException): - node.poll_query_until( - dbname='postgres', query='create table abc (val int)') + node.poll_query_until(query='create table abc (val int)') # check None, ok - node.poll_query_until( - dbname='postgres', query='create table def()', - expected=None) # returns nothing + node.poll_query_until(query='create table def()', + expected=None) # returns nothing # check arbitrary expected value, fail with self.assertRaises(TimeoutException): node.poll_query_until( - dbname='postgres', query='select 3', expected=1, max_attempts=3, sleep_time=0.01) # check arbitrary expected value, ok - node.poll_query_until( - dbname='postgres', query='select 2', expected=2) + node.poll_query_until(query='select 2', expected=2) # check timeout with self.assertRaises(TimeoutException): node.poll_query_until( - dbname='postgres', query='select 1 > 2', max_attempts=3, sleep_time=0.01) # check ProgrammingError, fail with self.assertRaises(testgres.ProgrammingError): - node.poll_query_until(dbname='postgres', query='dummy1') + node.poll_query_until(query='dummy1') # check ProgrammingError, ok with self.assertRaises(TimeoutException): node.poll_query_until( - dbname='postgres', query='dummy2', max_attempts=3, sleep_time=0.01, raise_programming_error=False) + # check 1 arg, ok + node.poll_query_until('select true') + def test_logging(self): logfile = tempfile.NamedTemporaryFile('w', delete=True) @@ -482,7 +475,7 @@ def test_logging(self): # execute a dummy query a few times for i in range(20): - master.execute('postgres', 'select 1') + master.execute('select 1') time.sleep(0.01) # let logging worker do the job From ac75f15c322f6ae30c1771f43a032fce130ad376 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Mon, 5 Feb 2018 00:45:18 +0300 Subject: [PATCH 063/475] reformat code, simplify tests (needless args) --- testgres/connection.py | 7 +- testgres/node.py | 10 +- testgres/utils.py | 2 +- tests/test_simple.py | 288 +++++++++++++++++++---------------------- 4 files changed, 135 insertions(+), 172 deletions(-) diff --git a/testgres/connection.py b/testgres/connection.py index da4f92e0..ef232e33 100644 --- a/testgres/connection.py +++ b/testgres/connection.py @@ -17,7 +17,6 @@ default_dbname as _default_dbname, \ default_username as _default_username - # export these exceptions InternalError = pglib.InternalError ProgrammingError = pglib.ProgrammingError @@ -36,11 +35,7 @@ class NodeConnection(object): Transaction wrapper returned by Node """ - def __init__(self, - node, - dbname=None, - username=None, - password=None): + def __init__(self, node, dbname=None, username=None, password=None): # Use default user if not specified dbname = dbname or _default_dbname() diff --git a/testgres/node.py b/testgres/node.py index 89284e7e..34988366 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -69,11 +69,7 @@ def __bool__(self): class PostgresNode(object): - def __init__(self, - name=None, - port=None, - base_dir=None, - use_logging=False): + def __init__(self, name=None, port=None, base_dir=None, use_logging=False): """ Create a new node manually. @@ -609,7 +605,6 @@ def cleanup(self, max_attempts=3): @method_decorator(positional_args_hack(['query'], ['dbname', 'query'])) def psql(self, - *, query=None, filename=None, dbname=None, @@ -668,7 +663,6 @@ def psql(self, @method_decorator(positional_args_hack(['dbname', 'query'])) def safe_psql(self, query, - *, dbname=None, username=None, input=None): @@ -747,7 +741,6 @@ def restore(self, filename, dbname=None, username=None): @method_decorator(positional_args_hack(['dbname', 'query'])) def poll_query_until(self, query, - *, dbname=None, username=None, max_attempts=0, @@ -816,7 +809,6 @@ def poll_query_until(self, @method_decorator(positional_args_hack(['dbname', 'query'])) def execute(self, query, - *, dbname=None, username=None, password=None, diff --git a/testgres/utils.py b/testgres/utils.py index b45104d5..170e7580 100644 --- a/testgres/utils.py +++ b/testgres/utils.py @@ -250,7 +250,7 @@ def positional_args_hack(*special_cases): 'special_cases' into named args. Example: - @positional_args_hack([['dbname', 'query']]) + @positional_args_hack(['abc'], ['def', 'abc']) def some_api_func(...) This is useful for compatibility. diff --git a/tests/test_simple.py b/tests/test_simple.py index a24decd5..dc6cde9d 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -53,12 +53,12 @@ def util_is_executable(util): class SimpleTest(unittest.TestCase): def test_custom_init(self): - with get_new_node('test') as node: + with get_new_node() as node: # enable page checksums node.init(initdb_params=['-k']).start() - node.safe_psql('postgres', 'select 1') + node.safe_psql('select 1') - with get_new_node('test') as node: + with get_new_node() as node: node.init( allow_streaming=True, initdb_params=['--auth-local=reject', '--auth-host=reject']) @@ -74,27 +74,26 @@ def test_custom_init(self): self.assertFalse(any('trust' in s for s in lines)) def test_double_init(self): - with get_new_node('test') as node: - node.init() + with get_new_node().init() as node: # can't initialize node more than once with self.assertRaises(InitNodeException): node.init() def test_init_after_cleanup(self): - with get_new_node('test') as node: + with get_new_node() as node: node.init().start() node.status() - node.safe_psql('postgres', 'select 1') + node.safe_psql('select 1') node.cleanup() node.init().start() node.status() - node.safe_psql('postgres', 'select 1') + node.safe_psql('select 1') def test_double_start(self): - with get_new_node('test') as node: + with get_new_node() as node: node.init().start() # can't start node more than once @@ -102,20 +101,20 @@ def test_double_start(self): node.start() def test_uninitialized_start(self): - with get_new_node('test') as node: + with get_new_node() as node: # node is not initialized yet with self.assertRaises(StartNodeException): node.start() def test_restart(self): - with get_new_node('test') as node: + with get_new_node() as node: node.init().start() # restart, ok - res = node.execute('postgres', 'select 1') + res = node.execute('select 1') self.assertEqual(res, [(1, )]) node.restart() - res = node.execute('postgres', 'select 2') + res = node.execute('select 2') self.assertEqual(res, [(2, )]) # restart, fail @@ -123,39 +122,28 @@ def test_restart(self): node.append_conf('pg_hba.conf', 'DUMMY') node.restart() - def test_psql(self): + def test_reload(self): with get_new_node() as node: node.init().start() - # check returned values (1 arg) - res = node.psql('select 1') - self.assertEqual(res, (0, b'1\n', b'')) - - # check returned values (2 args) - res = node.psql('postgres', 'select 2') - self.assertEqual(res, (0, b'2\n', b'')) - - # check returned values (1 arg) - res = node.safe_psql('select 3') - self.assertEqual(res, b'3\n') + # change client_min_messages and save old value + cmm_old = node.execute('show client_min_messages') + node.append_conf('postgresql.conf', 'client_min_messages = DEBUG1') - # check returned values (2 args) - res = node.safe_psql('postgres', 'select 4') - self.assertEqual(res, b'4\n') + # reload config + node.reload() - # check feeding input - node.safe_psql('create table horns (w int)') - node.safe_psql( - 'copy horns from stdin (format csv)', - input=b"1\n2\n3\n\.\n") - _sum = node.safe_psql('select sum(w) from horns') - self.assertEqual(_sum, b'6\n') + # check new value + cmm_new = node.execute('show client_min_messages') + self.assertEqual('debug1', cmm_new[0][0].lower()) + self.assertNotEqual(cmm_old, cmm_new) - node.stop() + def test_pg_ctl(self): + with get_new_node() as node: + node.init().start() - # check psql on stopped node - with self.assertRaises(QueryException): - node.safe_psql('postgres', 'select 1') + status = node.pg_ctl(['status']) + self.assertTrue('PID' in status) def test_status(self): # check NodeStatus cast to bool @@ -168,7 +156,7 @@ def test_status(self): self.assertFalse(NodeStatus.Uninitialized) # check statuses after each operation - with get_new_node('test') as node: + with get_new_node() as node: self.assertEqual(node.get_pid(), 0) self.assertEqual(node.status(), NodeStatus.Uninitialized) @@ -192,28 +180,50 @@ def test_status(self): self.assertEqual(node.get_pid(), 0) self.assertEqual(node.status(), NodeStatus.Uninitialized) - def test_simple_queries(self): - with get_new_node('test') as node: - node.init().start() + def test_psql(self): + with get_new_node().init().start() as node: - res = node.psql('postgres', 'select 1') + # check returned values (1 arg) + res = node.psql('select 1') self.assertEqual(res, (0, b'1\n', b'')) - res = node.safe_psql('postgres', 'select 1') - self.assertEqual(res, b'1\n') + # check returned values (2 args) + res = node.psql('postgres', 'select 2') + self.assertEqual(res, (0, b'2\n', b'')) - res = node.execute('postgres', 'select 1') - self.assertListEqual(res, [(1, )]) + # check returned values (named) + res = node.psql(query='select 3', dbname='postgres') + self.assertEqual(res, (0, b'3\n', b'')) - with node.connect('postgres') as con: - res = con.execute('select 1') - self.assertListEqual(res, [(1, )]) + # check returned values (1 arg) + res = node.safe_psql('select 4') + self.assertEqual(res, b'4\n') + + # check returned values (2 args) + res = node.safe_psql('postgres', 'select 5') + self.assertEqual(res, b'5\n') + + # check returned values (named) + res = node.safe_psql(query='select 6', dbname='postgres') + self.assertEqual(res, b'6\n') + + # check feeding input + node.safe_psql('create table horns (w int)') + node.safe_psql( + 'copy horns from stdin (format csv)', input=b"1\n2\n3\n\.\n") + _sum = node.safe_psql('select sum(w) from horns') + self.assertEqual(_sum, b'6\n') + + node.stop() + + # check psql on stopped node + with self.assertRaises(QueryException): + node.safe_psql('select 1') def test_transactions(self): - with get_new_node('test') as node: - node.init().start() + with get_new_node().init().start() as node: - with node.connect('postgres') as con: + with node.connect() as con: con.begin() con.execute('create table test(val int)') con.execute('insert into test values (1)') @@ -235,7 +245,7 @@ def test_transactions(self): con.commit() def test_control_data(self): - with get_new_node('test') as node: + with get_new_node() as node: # node is not initialized yet with self.assertRaises(ExecUtilException): @@ -249,7 +259,7 @@ def test_control_data(self): self.assertTrue(any('pg_control' in s for s in data.keys())) def test_backup_simple(self): - with get_new_node('master') as master: + with get_new_node() as master: # enable streaming for backups master.init(allow_streaming=True) @@ -262,18 +272,16 @@ def test_backup_simple(self): master.start() # fill node with some data - master.psql('postgres', - 'create table test as select generate_series(1, 4) i') + master.psql('create table test as select generate_series(1, 4) i') with master.backup(xlog_method='stream') as backup: - with backup.spawn_primary('slave') as slave: + with backup.spawn_primary() as slave: slave.start() - res = slave.execute('postgres', - 'select * from test order by i asc') + res = slave.execute('select * from test order by i asc') self.assertListEqual(res, [(1, ), (2, ), (3, ), (4, )]) def test_backup_multiple(self): - with get_new_node('node') as node: + with get_new_node() as node: node.init(allow_streaming=True).start() with node.backup(xlog_method='fetch') as backup1, \ @@ -288,62 +296,60 @@ def test_backup_multiple(self): self.assertNotEqual(node1.base_dir, node2.base_dir) def test_backup_exhaust(self): - with get_new_node('node') as node: + with get_new_node() as node: node.init(allow_streaming=True).start() with node.backup(xlog_method='fetch') as backup: # exhaust backup by creating new node - with backup.spawn_primary('node1') as node1: # noqa + with backup.spawn_primary() as node1: # noqa pass # now let's try to create one more node with self.assertRaises(BackupException): - with backup.spawn_primary('node2') as node2: # noqa + with backup.spawn_primary() as node2: # noqa pass def test_backup_and_replication(self): - with get_new_node('node') as node: - node.init(allow_streaming=True) - node.start() - node.psql('postgres', 'create table abc(a int, b int)') - node.psql('postgres', 'insert into abc values (1, 2)') + with get_new_node() as node: + node.init(allow_streaming=True).start() + + node.psql('create table abc(a int, b int)') + node.psql('insert into abc values (1, 2)') backup = node.backup() - with backup.spawn_replica('replica') as replica: - replica.start() - res = replica.execute('postgres', 'select * from abc') + with backup.spawn_replica().start() as replica: + res = replica.execute('select * from abc') self.assertListEqual(res, [(1, 2)]) # Insert into master node - node.psql('postgres', 'insert into abc values (3, 4)') + node.psql('insert into abc values (3, 4)') # Wait until data syncronizes replica.catchup() # Check that this record was exported to replica - res = replica.execute('postgres', 'select * from abc') + res = replica.execute('select * from abc') self.assertListEqual(res, [(1, 2), (3, 4)]) def test_replicate(self): - with get_new_node('node') as node: + with get_new_node() as node: node.init(allow_streaming=True).start() - with node.replicate(name='replica') as replica: - res = replica.start().execute('postgres', 'select 1') + with node.replicate().start() as replica: + res = replica.execute('select 1') self.assertListEqual(res, [(1, )]) - node.execute( - 'postgres', 'create table test (val int)', commit=True) + node.execute('create table test (val int)', commit=True) replica.catchup() - res = node.execute('postgres', 'select * from test') + res = node.execute('select * from test') self.assertListEqual(res, []) def test_incorrect_catchup(self): - with get_new_node('node') as node: + with get_new_node() as node: node.init(allow_streaming=True).start() # node has no master, can't catch up @@ -356,7 +362,7 @@ def test_dump(self): with get_new_node().init().start() as node1: - node1.execute('postgres', query_create) + node1.execute(query_create) # take a new dump dump = node1.dump() @@ -367,21 +373,20 @@ def test_dump(self): # restore dump node2.restore(filename=dump) - res = node2.execute('postgres', query_select) + res = node2.execute(query_select) self.assertListEqual(res, [(1, ), (2, )]) # finally, remove dump os.remove(dump) def test_users(self): - with get_new_node('master') as node: - node.init().start() - node.psql('postgres', 'create role test_user login') + with get_new_node().init().start() as node: + node.psql('create role test_user login') value = node.safe_psql('select 1', username='test_user') self.assertEqual(value, b'1\n') def test_poll_query_until(self): - with get_new_node('master') as node: + with get_new_node() as node: node.init().start() get_time = 'select extract(epoch from now())' @@ -407,8 +412,8 @@ def test_poll_query_until(self): node.poll_query_until(query='create table abc (val int)') # check None, ok - node.poll_query_until(query='create table def()', - expected=None) # returns nothing + node.poll_query_until( + query='create table def()', expected=None) # returns nothing # check arbitrary expected value, fail with self.assertRaises(TimeoutException): @@ -424,9 +429,7 @@ def test_poll_query_until(self): # check timeout with self.assertRaises(TimeoutException): node.poll_query_until( - query='select 1 > 2', - max_attempts=3, - sleep_time=0.01) + query='select 1 > 2', max_attempts=3, sleep_time=0.01) # check ProgrammingError, fail with self.assertRaises(testgres.ProgrammingError): @@ -470,7 +473,7 @@ def test_logging(self): logging.config.dictConfig(log_conf) node_name = 'master' - with get_new_node(node_name, use_logging=True) as master: + with get_new_node(name=node_name, use_logging=True) as master: master.init().start() # execute a dummy query a few times @@ -495,13 +498,11 @@ def test_logging(self): @unittest.skipUnless( util_is_executable("pgbench"), "pgbench may be missing") def test_pgbench(self): - with get_new_node('node') as node: - node.init().start() + with get_new_node().init().start() as node: # initialize pgbench DB and run benchmarks - node.pgbench_init(scale=2, - foreign_keys=True, - options=['-q']).pgbench_run(time=2) + node.pgbench_init( + scale=2, foreign_keys=True, options=['-q']).pgbench_run(time=2) # run TPC-B benchmark proc = node.pgbench( @@ -514,56 +515,6 @@ def test_pgbench(self): self.assertTrue('tps' in out) - def test_reload(self): - with get_new_node('node') as node: - node.init().start() - - cmd = "show client_min_messages" - - # change client_min_messages and save old value - cmm_old = node.execute(dbname='postgres', query=cmd) - node.append_conf('postgresql.conf', 'client_min_messages = DEBUG1') - - # reload config - node.reload() - - # check new value - cmm_new = node.execute(dbname='postgres', query=cmd) - self.assertEqual('debug1', cmm_new[0][0].lower()) - self.assertNotEqual(cmm_old, cmm_new) - - def test_pg_ctl(self): - with get_new_node('node') as node: - node.init().start() - - status = node.pg_ctl(['status']) - self.assertTrue('PID' in status) - - def test_ports_management(self): - # check that no ports have been bound yet - self.assertEqual(len(bound_ports), 0) - - with get_new_node('node') as node: - # check that we've just bound a port - self.assertEqual(len(bound_ports), 1) - - # check that bound_ports contains our port - port_1 = list(bound_ports)[0] - port_2 = node.port - self.assertEqual(port_1, port_2) - - # check that port has been freed successfully - self.assertEqual(len(bound_ports), 0) - - def test_version_management(self): - a = LooseVersion('10.0') - b = LooseVersion('10') - c = LooseVersion('9.6.5') - - self.assertTrue(a > b) - self.assertTrue(b > c) - self.assertTrue(a > c) - def test_config(self): # set global if it wasn't set configure_testgres(cache_initdb=True, cache_pg_config=True) @@ -586,16 +537,16 @@ def test_config(self): configure_testgres(cache_pg_config=True) def test_unix_sockets(self): - with get_new_node('node') as node: + with get_new_node() as node: node.init(unix_sockets=False, allow_streaming=True) node.start() - node.execute('postgres', 'select 1') - node.safe_psql('postgres', 'select 1') + node.execute('select 1') + node.safe_psql('select 1') - with node.replicate('r').start() as r: - r.execute('postgres', 'select 1') - r.safe_psql('postgres', 'select 1') + with node.replicate().start() as r: + r.execute('select 1') + r.safe_psql('select 1') def test_auto_name(self): with get_new_node().init(allow_streaming=True).start() as m: @@ -636,7 +587,7 @@ def test_file_tail(self): self.assertEqual(lines[0], s3) def test_isolation_levels(self): - with get_new_node('node').init().start() as node: + with get_new_node().init().start() as node: with node.connect() as con: # string levels con.begin('Read Uncommitted').commit() @@ -654,6 +605,31 @@ def test_isolation_levels(self): with self.assertRaises(QueryException): con.begin('Garbage').commit() + def test_ports_management(self): + # check that no ports have been bound yet + self.assertEqual(len(bound_ports), 0) + + with get_new_node() as node: + # check that we've just bound a port + self.assertEqual(len(bound_ports), 1) + + # check that bound_ports contains our port + port_1 = list(bound_ports)[0] + port_2 = node.port + self.assertEqual(port_1, port_2) + + # check that port has been freed successfully + self.assertEqual(len(bound_ports), 0) + + def test_version_management(self): + a = LooseVersion('10.0') + b = LooseVersion('10') + c = LooseVersion('9.6.5') + + self.assertTrue(a > b) + self.assertTrue(b > c) + self.assertTrue(a > c) + if __name__ == '__main__': unittest.main() From 8430468d8e5365bc1280110f82f009819635f785 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Mon, 5 Feb 2018 01:07:56 +0300 Subject: [PATCH 064/475] remove all kinds of mangling --- testgres/backup.py | 30 +++++------ testgres/cache.py | 4 +- testgres/connection.py | 11 +++-- testgres/node.py | 110 ++++++++++++++++++++--------------------- 4 files changed, 78 insertions(+), 77 deletions(-) diff --git a/testgres/backup.py b/testgres/backup.py index fe64a137..a4484309 100644 --- a/testgres/backup.py +++ b/testgres/backup.py @@ -7,17 +7,17 @@ from six import raise_from from .consts import \ - DATA_DIR as _DATA_DIR, \ - PG_CONF_FILE as _PG_CONF_FILE, \ - BACKUP_LOG_FILE as _BACKUP_LOG_FILE, \ - DEFAULT_XLOG_METHOD as _DEFAULT_XLOG_METHOD + DATA_DIR, \ + PG_CONF_FILE, \ + BACKUP_LOG_FILE, \ + DEFAULT_XLOG_METHOD from .exceptions import BackupException from .utils import \ get_bin_path, \ - default_username as _default_username, \ - execute_utility as _execute_utility + default_username, \ + execute_utility class NodeBackup(object): @@ -27,13 +27,13 @@ class NodeBackup(object): @property def log_file(self): - return os.path.join(self.base_dir, _BACKUP_LOG_FILE) + return os.path.join(self.base_dir, BACKUP_LOG_FILE) def __init__(self, node, base_dir=None, username=None, - xlog_method=_DEFAULT_XLOG_METHOD): + xlog_method=DEFAULT_XLOG_METHOD): """ Create a new backup. @@ -48,7 +48,7 @@ def __init__(self, raise BackupException('Node must be running') # Set default arguments - username = username or _default_username() + username = username or default_username() base_dir = base_dir or tempfile.mkdtemp() # public @@ -59,7 +59,7 @@ def __init__(self, # private self._available = True - data_dir = os.path.join(self.base_dir, _DATA_DIR) + data_dir = os.path.join(self.base_dir, DATA_DIR) # yapf: disable _params = [ @@ -70,7 +70,7 @@ def __init__(self, "-D", data_dir, "-X", xlog_method ] - _execute_utility(_params, self.log_file) + execute_utility(_params, self.log_file) def __enter__(self): return self @@ -98,8 +98,8 @@ def _prepare_dir(self, destroy): if available: dest_base_dir = tempfile.mkdtemp() - data1 = os.path.join(self.base_dir, _DATA_DIR) - data2 = os.path.join(dest_base_dir, _DATA_DIR) + data1 = os.path.join(self.base_dir, DATA_DIR) + data2 = os.path.join(dest_base_dir, DATA_DIR) try: # Copy backup to new data dir @@ -140,8 +140,8 @@ def spawn_primary(self, name=None, destroy=True, use_logging=False): # New nodes should always remove dir tree node._should_rm_dirs = True - node.append_conf(_PG_CONF_FILE, "\n") - node.append_conf(_PG_CONF_FILE, "port = {}".format(node.port)) + node.append_conf(PG_CONF_FILE, "\n") + node.append_conf(PG_CONF_FILE, "port = {}".format(node.port)) return node diff --git a/testgres/cache.py b/testgres/cache.py index 2c7e64b3..2fad48f8 100644 --- a/testgres/cache.py +++ b/testgres/cache.py @@ -15,7 +15,7 @@ from .utils import \ get_bin_path, \ - execute_utility as _execute_utility + execute_utility def cached_initdb(data_dir, initdb_logfile, initdb_params=[]): @@ -26,7 +26,7 @@ def cached_initdb(data_dir, initdb_logfile, initdb_params=[]): def call_initdb(initdb_dir): try: _params = [get_bin_path("initdb"), "-D", initdb_dir, "-N"] - _execute_utility(_params + initdb_params, initdb_logfile) + execute_utility(_params + initdb_params, initdb_logfile) except ExecUtilException as e: raise_from(InitNodeException("Failed to run initdb"), e) diff --git a/testgres/connection.py b/testgres/connection.py index ef232e33..e15cca48 100644 --- a/testgres/connection.py +++ b/testgres/connection.py @@ -14,8 +14,9 @@ from .exceptions import QueryException from .utils import \ - default_dbname as _default_dbname, \ - default_username as _default_username + default_dbname, \ + default_username + # export these exceptions InternalError = pglib.InternalError @@ -37,9 +38,9 @@ class NodeConnection(object): def __init__(self, node, dbname=None, username=None, password=None): - # Use default user if not specified - dbname = dbname or _default_dbname() - username = username or _default_username() + # Set default arguments + dbname = dbname or default_dbname() + username = username or default_username() self._node = node diff --git a/testgres/node.py b/testgres/node.py index 34988366..1f7948b3 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -11,7 +11,7 @@ from enum import Enum from six import raise_from -from .cache import cached_initdb as _cached_initdb +from .cache import cached_initdb from .config import TestgresConfig @@ -21,14 +21,14 @@ ProgrammingError from .consts import \ - DATA_DIR as _DATA_DIR, \ - LOGS_DIR as _LOGS_DIR, \ - PG_CONF_FILE as _PG_CONF_FILE, \ - HBA_CONF_FILE as _HBA_CONF_FILE, \ - RECOVERY_CONF_FILE as _RECOVERY_CONF_FILE, \ - PG_LOG_FILE as _PG_LOG_FILE, \ - UTILS_LOG_FILE as _UTILS_LOG_FILE, \ - DEFAULT_XLOG_METHOD as _DEFAULT_XLOG_METHOD + DATA_DIR, \ + LOGS_DIR, \ + PG_CONF_FILE, \ + HBA_CONF_FILE, \ + RECOVERY_CONF_FILE, \ + PG_LOG_FILE, \ + UTILS_LOG_FILE, \ + DEFAULT_XLOG_METHOD from .exceptions import \ CatchUpException, \ @@ -41,14 +41,14 @@ from .utils import \ get_bin_path, \ - file_tail as _file_tail, \ - pg_version_ge as _pg_version_ge, \ - reserve_port as _reserve_port, \ - release_port as _release_port, \ - default_dbname as _default_dbname, \ - default_username as _default_username, \ - generate_app_name as _generate_app_name, \ - execute_utility as _execute_utility, \ + file_tail, \ + pg_version_ge, \ + reserve_port, \ + release_port, \ + default_dbname, \ + default_username, \ + generate_app_name, \ + execute_utility, \ method_decorator, \ positional_args_hack @@ -82,8 +82,8 @@ def __init__(self, name=None, port=None, base_dir=None, use_logging=False): # public self.host = '127.0.0.1' - self.name = name or _generate_app_name() - self.port = port or _reserve_port() + self.name = name or generate_app_name() + self.port = port or reserve_port() self.base_dir = base_dir # private @@ -112,19 +112,19 @@ def master(self): @property def data_dir(self): - return os.path.join(self.base_dir, _DATA_DIR) + return os.path.join(self.base_dir, DATA_DIR) @property def logs_dir(self): - return os.path.join(self.base_dir, _LOGS_DIR) + return os.path.join(self.base_dir, LOGS_DIR) @property def utils_log_name(self): - return os.path.join(self.logs_dir, _UTILS_LOG_FILE) + return os.path.join(self.logs_dir, UTILS_LOG_FILE) @property def pg_log_name(self): - return os.path.join(self.logs_dir, _PG_LOG_FILE) + return os.path.join(self.logs_dir, PG_LOG_FILE) def _assign_master(self, master): """NOTE: this is a private method!""" @@ -160,7 +160,7 @@ def _create_recovery_conf(self, username): "standby_mode=on\n" ).format(conninfo) - self.append_conf(_RECOVERY_CONF_FILE, line) + self.append_conf(RECOVERY_CONF_FILE, line) def _prepare_dirs(self): """NOTE: this is a private method!""" @@ -194,9 +194,9 @@ def _format_verbose_error(self, message=None): # list of important files + N of last lines files = [ - (os.path.join(self.data_dir, _PG_CONF_FILE), 0), - (os.path.join(self.data_dir, _HBA_CONF_FILE), 0), - (os.path.join(self.data_dir, _RECOVERY_CONF_FILE), 0), + (os.path.join(self.data_dir, PG_CONF_FILE), 0), + (os.path.join(self.data_dir, HBA_CONF_FILE), 0), + (os.path.join(self.data_dir, RECOVERY_CONF_FILE), 0), (self.pg_log_name, TestgresConfig.error_log_lines) ] @@ -215,7 +215,7 @@ def _format_verbose_error(self, message=None): with io.open(f, "rb") as _f: if num_lines > 0: # take last N lines of file - lines = b''.join(_file_tail(_f, num_lines)).decode('utf-8') + lines = b''.join(file_tail(_f, num_lines)).decode('utf-8') else: # read whole file lines = _f.read().decode('utf-8') @@ -248,7 +248,7 @@ def init(self, # initialize this PostgreSQL node initdb_log = os.path.join(self.logs_dir, "initdb.log") - _cached_initdb(self.data_dir, initdb_log, initdb_params) + cached_initdb(self.data_dir, initdb_log, initdb_params) # initialize default config files self.default_conf(fsync=fsync, @@ -275,8 +275,8 @@ def default_conf(self, This instance of PostgresNode. """ - postgres_conf = os.path.join(self.data_dir, _PG_CONF_FILE) - hba_conf = os.path.join(self.data_dir, _HBA_CONF_FILE) + postgres_conf = os.path.join(self.data_dir, PG_CONF_FILE) + hba_conf = os.path.join(self.data_dir, HBA_CONF_FILE) # filter lines in hba file with io.open(hba_conf, "r+") as conf: @@ -333,7 +333,7 @@ def get_auth_method(t): if allow_streaming: # select a proper wal_level for PostgreSQL - if _pg_version_ge('9.6'): + if pg_version_ge('9.6'): wal_level = "replica" else: wal_level = "hot_standby" @@ -387,7 +387,7 @@ def status(self): "-D", self.data_dir, "status" ] - _execute_utility(_params, self.utils_log_name) + execute_utility(_params, self.utils_log_name) return NodeStatus.Running except ExecUtilException as e: @@ -418,10 +418,10 @@ def get_control_data(self): # this one is tricky (blame PG 9.4) _params = [get_bin_path("pg_controldata")] - _params += ["-D"] if _pg_version_ge('9.5') else [] + _params += ["-D"] if pg_version_ge('9.5') else [] _params += [self.data_dir] - data = _execute_utility(_params, self.utils_log_name) + data = execute_utility(_params, self.utils_log_name) out_dict = {} @@ -452,7 +452,7 @@ def start(self, params=[]): ] + params try: - _execute_utility(_params, self.utils_log_name) + execute_utility(_params, self.utils_log_name) except ExecUtilException as e: msg = self._format_verbose_error('Cannot start node') raise_from(StartNodeException(msg), e) @@ -480,7 +480,7 @@ def stop(self, params=[]): "stop" ] + params - _execute_utility(_params, self.utils_log_name) + execute_utility(_params, self.utils_log_name) self._maybe_stop_logger() @@ -507,7 +507,7 @@ def restart(self, params=[]): ] + params try: - _execute_utility(_params, self.utils_log_name) + execute_utility(_params, self.utils_log_name) except ExecUtilException as e: msg = self._format_verbose_error('Cannot restart node') raise_from(StartNodeException(msg), e) @@ -535,7 +535,7 @@ def reload(self, params=[]): "reload" ] + params - _execute_utility(_params, self.utils_log_name) + execute_utility(_params, self.utils_log_name) def pg_ctl(self, params): """ @@ -555,7 +555,7 @@ def pg_ctl(self, params): "-w" # wait ] + params - return _execute_utility(_params, self.utils_log_name) + return execute_utility(_params, self.utils_log_name) def free_port(self): """ @@ -563,7 +563,7 @@ def free_port(self): """ if self._should_free_port: - _release_port(self.port) + release_port(self.port) def cleanup(self, max_attempts=3): """ @@ -625,8 +625,8 @@ def psql(self, """ # Set default arguments - dbname = dbname or _default_dbname() - username = username or _default_username() + dbname = dbname or default_dbname() + username = username or default_username() # yapf: disable psql_params = [ @@ -708,8 +708,8 @@ def tmpfile(): return fname # Set default arguments - dbname = dbname or _default_dbname() - username = username or _default_username() + dbname = dbname or default_dbname() + username = username or default_username() filename = filename or tmpfile() # yapf: disable @@ -722,7 +722,7 @@ def tmpfile(): "-d", dbname ] - _execute_utility(_params, self.utils_log_name) + execute_utility(_params, self.utils_log_name) return filename @@ -754,8 +754,8 @@ def poll_query_until(self, Query should return single column. Args: - dbname: database name to connect to. query: query to be executed. + dbname: database name to connect to. username: database user name. max_attempts: how many times should we try? 0 == infinite sleep_time: how much should we sleep after a failure? @@ -838,7 +838,7 @@ def execute(self, return res - def backup(self, username=None, xlog_method=_DEFAULT_XLOG_METHOD): + def backup(self, username=None, xlog_method=DEFAULT_XLOG_METHOD): """ Perform pg_basebackup. @@ -858,7 +858,7 @@ def backup(self, username=None, xlog_method=_DEFAULT_XLOG_METHOD): def replicate(self, name=None, username=None, - xlog_method=_DEFAULT_XLOG_METHOD, + xlog_method=DEFAULT_XLOG_METHOD, use_logging=False): """ Create a binary replica of this node. @@ -885,7 +885,7 @@ def catchup(self, dbname=None, username=None): if not self.master: raise CatchUpException("Node doesn't have a master") - if _pg_version_ge('10'): + if pg_version_ge('10'): poll_lsn = "select pg_current_wal_lsn()::text" wait_lsn = "select pg_last_wal_replay_lsn() >= '{}'::pg_lsn" else: @@ -928,8 +928,8 @@ def pgbench(self, """ # Set default arguments - dbname = dbname or _default_dbname() - username = username or _default_username() + dbname = dbname or default_dbname() + username = username or default_username() # yapf: disable _params = [ @@ -984,8 +984,8 @@ def pgbench_run(self, """ # Set default arguments - dbname = dbname or _default_dbname() - username = username or _default_username() + dbname = dbname or default_dbname() + username = username or default_username() # yapf: disable _params = [ @@ -1009,7 +1009,7 @@ def pgbench_run(self, # should be the last one _params.append(dbname) - return _execute_utility(_params, self.utils_log_name) + return execute_utility(_params, self.utils_log_name) def connect(self, dbname=None, username=None, password=None): """ From 7bcb4f33741c013e141ae84397e52a2feeb23262 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Mon, 5 Feb 2018 01:22:12 +0300 Subject: [PATCH 065/475] minor fixes --- testgres/node.py | 4 ++-- tests/test_simple.py | 6 +++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index 1f7948b3..e636a6a2 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -584,9 +584,9 @@ def cleanup(self, max_attempts=3): self.stop() break # OK except ExecUtilException: - pass # one more time + pass # one more time except Exception: - break # screw this + print('cannot stop node {}'.format(self.name)) attempts += 1 diff --git a/tests/test_simple.py b/tests/test_simple.py index dc6cde9d..cdf83440 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -214,9 +214,13 @@ def test_psql(self): _sum = node.safe_psql('select sum(w) from horns') self.assertEqual(_sum, b'6\n') + # check psql's default args, fails + with self.assertRaises(QueryException): + node.psql() + node.stop() - # check psql on stopped node + # check psql on stopped node, fails with self.assertRaises(QueryException): node.safe_psql('select 1') From 24221e673fc0bab0c6d34bc24d511d9e49007e28 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Mon, 5 Feb 2018 12:28:00 +0300 Subject: [PATCH 066/475] small fix for logger --- testgres/logger.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/testgres/logger.py b/testgres/logger.py index ee700f95..820998a6 100644 --- a/testgres/logger.py +++ b/testgres/logger.py @@ -25,15 +25,20 @@ def run(self): with open(self._log_file_name, 'r') as fd: # work until we're asked to stop while not self._stop_event.is_set(): + sleep_time = 0.1 + new_lines = False + # do we have new lines? if fd in select.select([fd], [], [], 0)[0]: for line in fd.readlines(): line = line.strip() if line: + new_lines = True extra = {'node': self._node_name} self._logger.info(line, extra=extra) - else: - time.sleep(0.1) + + if not new_lines: + time.sleep(sleep_time) # don't forget to clear event self._stop_event.clear() From 9140b98010df89da1e27df539392d7226d870bd1 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Tue, 6 Feb 2018 10:37:47 +0300 Subject: [PATCH 067/475] fix several incosistencies spotted by @Valeria1235 --- testgres/connection.py | 1 - testgres/node.py | 6 +++--- testgres/utils.py | 27 ++++++++++++--------------- tests/test_simple.py | 29 +++++++++++++++++++++-------- 4 files changed, 36 insertions(+), 27 deletions(-) diff --git a/testgres/connection.py b/testgres/connection.py index e15cca48..e86ff06d 100644 --- a/testgres/connection.py +++ b/testgres/connection.py @@ -17,7 +17,6 @@ default_dbname, \ default_username - # export these exceptions InternalError = pglib.InternalError ProgrammingError = pglib.ProgrammingError diff --git a/testgres/node.py b/testgres/node.py index e636a6a2..424d7eba 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -228,7 +228,7 @@ def _format_verbose_error(self, message=None): def init(self, fsync=False, unix_sockets=True, - allow_streaming=False, + allow_streaming=True, initdb_params=[]): """ Perform initdb for this node. @@ -401,7 +401,7 @@ def status(self): def get_pid(self): """ - Return postmaster's pid if node is running, else 0. + Return postmaster's PID if node is running, else 0. """ if self.status(): @@ -603,7 +603,7 @@ def cleanup(self, max_attempts=3): return self - @method_decorator(positional_args_hack(['query'], ['dbname', 'query'])) + @method_decorator(positional_args_hack(['dbname', 'query'])) def psql(self, query=None, filename=None, diff --git a/testgres/utils.py b/testgres/utils.py index 170e7580..ada4c461 100644 --- a/testgres/utils.py +++ b/testgres/utils.py @@ -23,10 +23,9 @@ def reserve_port(): """ - Generate a new port and add it to '_bound_ports'. + Generate a new port and add it to 'bound_ports'. """ - global bound_ports port = port_for.select_random(exclude_ports=bound_ports) bound_ports.add(port) @@ -38,7 +37,6 @@ def release_port(port): Free port provided by reserve_port(). """ - global bound_ports bound_ports.remove(port) @@ -65,7 +63,7 @@ def generate_app_name(): """ import uuid - return ''.join(['testgres-', str(uuid.uuid4())]) + return 'testgres-{}'.format(str(uuid.uuid4())) def execute_utility(args, logfile): @@ -147,11 +145,7 @@ def get_pg_config(): NOTE: this fuction caches the result by default (see TestgresConfig). """ - global _pg_config_data - def cache_pg_config_data(cmd): - global _pg_config_data - # execute pg_config and get the output out = subprocess.check_output([cmd]).decode('utf-8') @@ -161,16 +155,19 @@ def cache_pg_config_data(cmd): key, _, value = line.partition('=') data[key.strip()] = value.strip() - _pg_config_data.clear() - - # cache data, if necessary - if TestgresConfig.cache_pg_config: - _pg_config_data = data + # cache data + global _pg_config_data + _pg_config_data = data return data - # return cached data, if allowed to - if TestgresConfig.cache_pg_config and _pg_config_data: + # drop cache if asked to + if not TestgresConfig.cache_pg_config: + global _pg_config_data + _pg_config_data = {} + + # return cached data + if _pg_config_data: return _pg_config_data # try PG_CONFIG diff --git a/tests/test_simple.py b/tests/test_simple.py index cdf83440..79e0c913 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -38,16 +38,20 @@ def util_is_executable(util): - exe_file = get_bin_path(util) - - # check if util exists - if os.path.exists(exe_file): + def good_properties(f): + return ( + os.path.exists(f) and + os.path.isfile(f) and + os.access(f, os.X_OK) + ) + + # try to resolve it + if good_properties(get_bin_path(util)): return True # check if util is in PATH for path in os.environ["PATH"].split(os.pathsep): - exe_file = os.path.join(path, util) - if os.path.exists(exe_file): + if good_properties(os.path.join(path, util)): return True @@ -519,19 +523,28 @@ def test_pgbench(self): self.assertTrue('tps' in out) - def test_config(self): + def test_pg_config(self): # set global if it wasn't set - configure_testgres(cache_initdb=True, cache_pg_config=True) + configure_testgres(cache_pg_config=True) # check same instances a = get_pg_config() b = get_pg_config() self.assertEqual(id(a), id(b)) + # save right before config change + c1 = get_pg_config() + # modify setting configure_testgres(cache_pg_config=False) self.assertFalse(TestgresConfig.cache_pg_config) + # save right after config change + c2 = get_pg_config() + + # check different instances after config change + self.assertNotEqual(id(c1), id(c2)) + # check different instances a = get_pg_config() b = get_pg_config() From 58785525c8870025030efab4f7956e85cbe8c661 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Tue, 6 Feb 2018 13:33:09 +0300 Subject: [PATCH 068/475] bump version to 1.5.0 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 525c9816..66e6ac53 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,7 @@ setup( name='testgres', packages=['testgres'], - version='1.4.1', + version='1.5.0', description='Testing utility for PostgreSQL and its extensions', license='PostgreSQL', author='Ildar Musin', From cf03ee25ee0638c5755dc21df409fafe9326697b Mon Sep 17 00:00:00 2001 From: Valeria112 Date: Wed, 7 Feb 2018 00:51:50 +0300 Subject: [PATCH 069/475] simplified enums NodeStatus and IsolationLevel --- testgres/connection.py | 35 ++++++++++------------------------- testgres/node.py | 6 +++--- 2 files changed, 13 insertions(+), 28 deletions(-) diff --git a/testgres/connection.py b/testgres/connection.py index e86ff06d..a20b651a 100644 --- a/testgres/connection.py +++ b/testgres/connection.py @@ -27,7 +27,10 @@ class IsolationLevel(Enum): Transaction isolation level for NodeConnection """ - ReadUncommitted, ReadCommitted, RepeatableRead, Serializable = range(4) + ReadUncommitted = 'read uncommitted' + ReadCommitted = 'read committed' + RepeatableRead = 'repeatable read' + Serializable = 'serializable' class NodeConnection(object): @@ -71,39 +74,21 @@ def __exit__(self, type, value, traceback): self.close() def begin(self, isolation_level=IsolationLevel.ReadCommitted): - # yapf: disable - levels = [ - 'read uncommitted', - 'read committed', - 'repeatable read', - 'serializable' - ] - - # Check if level is an IsolationLevel - if (isinstance(isolation_level, IsolationLevel)): - - # Get index of isolation level - level_idx = isolation_level.value - assert level_idx in range(4) - - # Replace isolation level with its name - isolation_level = levels[level_idx] - - else: + # Check if level isn't an IsolationLevel + if not isinstance(isolation_level, IsolationLevel): # Get name of isolation level level_str = str(isolation_level).lower() # Validate level string - if level_str not in levels: + try: + isolation_level = IsolationLevel(level_str) + except ValueError: error = 'Invalid isolation level "{}"' raise QueryException(error.format(level_str)) - # Replace isolation level with its name - isolation_level = level_str - # Set isolation level cmd = 'SET TRANSACTION ISOLATION LEVEL {}' - self.cursor.execute(cmd.format(isolation_level)) + self.cursor.execute(cmd.format(isolation_level.value)) return self diff --git a/testgres/node.py b/testgres/node.py index 424d7eba..f7c8b7db 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -8,7 +8,7 @@ import tempfile import time -from enum import Enum +from enum import IntEnum from six import raise_from from .cache import cached_initdb @@ -53,7 +53,7 @@ positional_args_hack -class NodeStatus(Enum): +class NodeStatus(IntEnum): """ Status of a PostgresNode """ @@ -62,7 +62,7 @@ class NodeStatus(Enum): # for Python 3.x def __bool__(self): - return self.value == NodeStatus.Running.value + return self == NodeStatus.Running # for Python 2.x __nonzero__ = __bool__ From 0d53798684bfa20a77c4921c0c4f55c141df0573 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Wed, 7 Feb 2018 15:20:11 +0300 Subject: [PATCH 070/475] reworked logic in PostgresNode.__exit__() according to review by @gsmol --- testgres/config.py | 18 +++++++-- testgres/node.py | 91 ++++++++++++++++++++++++++-------------------- testgres/utils.py | 6 +++ 3 files changed, 71 insertions(+), 44 deletions(-) diff --git a/testgres/config.py b/testgres/config.py index 4be623c8..e2263d95 100644 --- a/testgres/config.py +++ b/testgres/config.py @@ -7,18 +7,28 @@ class TestgresConfig: Attributes: cache_initdb: shall we use cached initdb instance? - cache_pg_config: shall we cache pg_config results? cached_initdb_dir: shall we create a temp dir for cached initdb? - node_cleanup_full: shall we remove EVERYTHING (including logs)? + + cache_pg_config: shall we cache pg_config results? + error_log_lines: N of log lines to be included into exception (0=inf). + + node_cleanup_full: shall we remove EVERYTHING (including logs)? + node_cleanup_on_good_exit: remove base_dir on nominal __exit__(). + node_cleanup_on_bad_exit: remove base_dir on __exit__() via exception. """ cache_initdb = True - cache_pg_config = True cached_initdb_dir = None - node_cleanup_full = True + + cache_pg_config = True + error_log_lines = 20 + node_cleanup_full = True + node_cleanup_on_good_exit = True + node_cleanup_on_bad_exit = False + def configure_testgres(**options): """ diff --git a/testgres/node.py b/testgres/node.py index 424d7eba..0e8cfb76 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -40,6 +40,7 @@ from .logger import TestgresLogger from .utils import \ + eprint, \ get_bin_path, \ file_tail, \ pg_version_ge, \ @@ -80,14 +81,18 @@ def __init__(self, name=None, port=None, base_dir=None, use_logging=False): use_logging: enable python logging. """ - # public + # basic self.host = '127.0.0.1' self.name = name or generate_app_name() self.port = port or reserve_port() self.base_dir = base_dir + # defaults for __exit__() + self.cleanup_on_good_exit = TestgresConfig.node_cleanup_on_good_exit + self.cleanup_on_bad_exit = TestgresConfig.node_cleanup_on_bad_exit + self.shutdown_max_attempts = 3 + # private - self._should_rm_dirs = base_dir is None self._should_free_port = port is None self._use_logging = use_logging self._logger = None @@ -100,12 +105,23 @@ def __enter__(self): return self def __exit__(self, type, value, traceback): - # stop node if necessary - self.cleanup() - - # free port if necessary self.free_port() + got_exception = value is not None + c1 = self.cleanup_on_good_exit and not got_exception + c2 = self.cleanup_on_bad_exit and got_exception + + attempts = self.shutdown_max_attempts + + if c1 or c2: + self.cleanup(attempts) + else: + self._try_shutdown(attempts) + + @property + def pid(self): + return self.get_pid() + @property def master(self): return self._master @@ -126,6 +142,22 @@ def utils_log_name(self): def pg_log_name(self): return os.path.join(self.logs_dir, PG_LOG_FILE) + def _try_shutdown(self, max_attempts): + attempts = 0 + + # try stopping server N times + while attempts < max_attempts: + try: + self.stop() + break # OK + except ExecUtilException: + pass # one more time + except Exception: + # TODO: probably kill stray instance + eprint('cannot stop node {}'.format(self.name)) + + attempts += 1 + def _assign_master(self, master): """NOTE: this is a private method!""" @@ -163,8 +195,6 @@ def _create_recovery_conf(self, username): self.append_conf(RECOVERY_CONF_FILE, line) def _prepare_dirs(self): - """NOTE: this is a private method!""" - if not self.base_dir: self.base_dir = tempfile.mkdtemp() @@ -175,23 +205,17 @@ def _prepare_dirs(self): os.makedirs(self.logs_dir) def _maybe_start_logger(self): - """NOTE: this is a private method!""" - if self._use_logging: - # spawn new logger if it doesn't exist or stopped + # spawn new logger if it doesn't exist or is stopped if not self._logger or not self._logger.is_alive(): self._logger = TestgresLogger(self.name, self.pg_log_name) self._logger.start() def _maybe_stop_logger(self): - """NOTE: this is a private method!""" - if self._logger: self._logger.stop() def _format_verbose_error(self, message=None): - """NOTE: this is a private method!""" - # list of important files + N of last lines files = [ (os.path.join(self.data_dir, PG_CONF_FILE), 0), @@ -560,6 +584,7 @@ def pg_ctl(self, params): def free_port(self): """ Reclaim port owned by this node. + NOTE: does not free auto selected ports. """ if self._should_free_port: @@ -567,7 +592,8 @@ def free_port(self): def cleanup(self, max_attempts=3): """ - Stop node if needed and remove its data directory. + Stop node if needed and remove its data/logs directory. + NOTE: take a look at TestgresConfig.node_cleanup_full. Args: max_attempts: how many times should we try to stop()? @@ -576,30 +602,15 @@ def cleanup(self, max_attempts=3): This instance of PostgresNode. """ - attempts = 0 + self._try_shutdown(max_attempts) - # try stopping server - while attempts < max_attempts: - try: - self.stop() - break # OK - except ExecUtilException: - pass # one more time - except Exception: - print('cannot stop node {}'.format(self.name)) - - attempts += 1 - - # remove directory tree if necessary - if self._should_rm_dirs: - - # choose directory to be removed - if TestgresConfig.node_cleanup_full: - rm_dir = self.base_dir # everything - else: - rm_dir = self.data_dir # just data, save logs + # choose directory to be removed + if TestgresConfig.node_cleanup_full: + rm_dir = self.base_dir # everything + else: + rm_dir = self.data_dir # just data, save logs - shutil.rmtree(rm_dir, ignore_errors=True) + shutil.rmtree(rm_dir, ignore_errors=True) return self @@ -750,8 +761,8 @@ def poll_query_until(self, raise_programming_error=True, raise_internal_error=True): """ - Run a query once a second until it returs 'expected'. - Query should return single column. + Run a query once per second until it returns 'expected'. + Query should return a single value (1 row, 1 column). Args: query: query to be executed. diff --git a/testgres/utils.py b/testgres/utils.py index ada4c461..d4216222 100644 --- a/testgres/utils.py +++ b/testgres/utils.py @@ -1,6 +1,7 @@ # coding: utf-8 from __future__ import division +from __future__ import print_function import functools import io @@ -8,6 +9,7 @@ import port_for import six import subprocess +import sys from distutils.version import LooseVersion @@ -311,3 +313,7 @@ def bound_func(*args2, **kwargs2): _dec.__name__ = 'method_decorator({})'.format(decorator.__name__) return _dec + + +def eprint(*args, **kwargs): + print(*args, file=sys.stderr, **kwargs) From 3e91753df95c593d1a021546e8126df4c9c18ce4 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Wed, 7 Feb 2018 16:03:01 +0300 Subject: [PATCH 071/475] improve tests for coverage --- tests/test_simple.py | 85 +++++++++++++++++--------------------------- 1 file changed, 32 insertions(+), 53 deletions(-) diff --git a/tests/test_simple.py b/tests/test_simple.py index 79e0c913..57a7461a 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -2,6 +2,7 @@ # coding: utf-8 import os +import shutil import subprocess import tempfile import testgres @@ -60,7 +61,6 @@ def test_custom_init(self): with get_new_node() as node: # enable page checksums node.init(initdb_params=['-k']).start() - node.safe_psql('select 1') with get_new_node() as node: node.init( @@ -79,27 +79,36 @@ def test_custom_init(self): def test_double_init(self): with get_new_node().init() as node: - # can't initialize node more than once with self.assertRaises(InitNodeException): node.init() def test_init_after_cleanup(self): with get_new_node() as node: - node.init().start() - node.status() - node.safe_psql('select 1') - + node.init().start().execute('select 1') node.cleanup() + node.init().start().execute('select 1') - node.init().start() - node.status() - node.safe_psql('select 1') + def test_node_exit(self): + base_dir = None - def test_double_start(self): - with get_new_node() as node: - node.init().start() + with self.assertRaises(QueryException): + with get_new_node().init() as node: + base_dir = node.base_dir + node.safe_psql('select 1') + + # we should save the DB for "debugging" + self.assertTrue(os.path.exists(base_dir)) + shutil.rmtree(base_dir, ignore_errors=True) + + with get_new_node().init() as node: + base_dir = node.base_dir + + # should have been removed by default + self.assertFalse(os.path.exists(base_dir)) + def test_double_start(self): + with get_new_node().init().start() as node: # can't start node more than once with self.assertRaises(StartNodeException): node.start() @@ -150,38 +159,33 @@ def test_pg_ctl(self): self.assertTrue('PID' in status) def test_status(self): - # check NodeStatus cast to bool self.assertTrue(NodeStatus.Running) - - # check NodeStatus cast to bool self.assertFalse(NodeStatus.Stopped) - - # check NodeStatus cast to bool self.assertFalse(NodeStatus.Uninitialized) # check statuses after each operation with get_new_node() as node: - self.assertEqual(node.get_pid(), 0) + self.assertEqual(node.pid, 0) self.assertEqual(node.status(), NodeStatus.Uninitialized) node.init() - self.assertEqual(node.get_pid(), 0) + self.assertEqual(node.pid, 0) self.assertEqual(node.status(), NodeStatus.Stopped) node.start() - self.assertTrue(node.get_pid() > 0) + self.assertNotEqual(node.pid, 0) self.assertEqual(node.status(), NodeStatus.Running) node.stop() - self.assertEqual(node.get_pid(), 0) + self.assertEqual(node.pid, 0) self.assertEqual(node.status(), NodeStatus.Stopped) node.cleanup() - self.assertEqual(node.get_pid(), 0) + self.assertEqual(node.pid, 0) self.assertEqual(node.status(), NodeStatus.Uninitialized) def test_psql(self): @@ -283,8 +287,7 @@ def test_backup_simple(self): master.psql('create table test as select generate_series(1, 4) i') with master.backup(xlog_method='stream') as backup: - with backup.spawn_primary() as slave: - slave.start() + with backup.spawn_primary().start() as slave: res = slave.execute('select * from test order by i asc') self.assertListEqual(res, [(1, ), (2, ), (3, ), (4, )]) @@ -310,36 +313,12 @@ def test_backup_exhaust(self): with node.backup(xlog_method='fetch') as backup: # exhaust backup by creating new node - with backup.spawn_primary() as node1: # noqa + with backup.spawn_primary(): pass # now let's try to create one more node with self.assertRaises(BackupException): - with backup.spawn_primary() as node2: # noqa - pass - - def test_backup_and_replication(self): - with get_new_node() as node: - node.init(allow_streaming=True).start() - - node.psql('create table abc(a int, b int)') - node.psql('insert into abc values (1, 2)') - - backup = node.backup() - - with backup.spawn_replica().start() as replica: - res = replica.execute('select * from abc') - self.assertListEqual(res, [(1, 2)]) - - # Insert into master node - node.psql('insert into abc values (3, 4)') - - # Wait until data syncronizes - replica.catchup() - - # Check that this record was exported to replica - res = replica.execute('select * from abc') - self.assertListEqual(res, [(1, 2), (3, 4)]) + backup.spawn_primary() def test_replicate(self): with get_new_node() as node: @@ -574,8 +553,6 @@ def test_auto_name(self): self.assertTrue(r.status()) # check their names - self.assertIsNotNone(m.name) - self.assertIsNotNone(r.name) self.assertNotEqual(m.name, r.name) self.assertTrue('testgres' in m.name) self.assertTrue('testgres' in r.name) @@ -588,7 +565,9 @@ def test_file_tail(self): s3 = "def\n" with tempfile.NamedTemporaryFile(mode='r+', delete=True) as f: - for i in range(1, 5000): + sz = 0 + while sz < 3 * 8192: + sz += len(s1) f.write(s1) f.write(s2) f.write(s3) From d3c15285d952e550f0203736fa06cab9801b1214 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Thu, 8 Feb 2018 15:07:26 +0300 Subject: [PATCH 072/475] cleanup in MANIFEST.in & setup.py --- .gitignore | 5 +++-- MANIFEST | 14 -------------- MANIFEST.in | 8 ++++++++ setup.py | 9 +++++---- 4 files changed, 16 insertions(+), 20 deletions(-) delete mode 100644 MANIFEST create mode 100644 MANIFEST.in diff --git a/.gitignore b/.gitignore index aebd9e97..4cf6750a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,11 +1,12 @@ *.pyc *.egg *.egg-info/ -dist +.eggs/ +dist/ +build/ env/ venv/ -build/ .coverage coverage.xml diff --git a/MANIFEST b/MANIFEST deleted file mode 100644 index fc1d65b7..00000000 --- a/MANIFEST +++ /dev/null @@ -1,14 +0,0 @@ -# file GENERATED by distutils, do NOT edit -setup.cfg -setup.py -testgres/__init__.py -testgres/api.py -testgres/backup.py -testgres/cache.py -testgres/config.py -testgres/connection.py -testgres/consts.py -testgres/exceptions.py -testgres/logger.py -testgres/node.py -testgres/utils.py diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 00000000..8adcedf3 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,8 @@ +include LICENSE +include README.md +include setup.cfg + +recursive-include testgres *.py +recursive-include tests *.py + +global-exclude *.pyc diff --git a/setup.py b/setup.py index 66e6ac53..2a7e3dc8 100644 --- a/setup.py +++ b/setup.py @@ -17,14 +17,15 @@ install_requires.append("ipaddress") setup( + version='1.5.0', name='testgres', packages=['testgres'], - version='1.5.0', description='Testing utility for PostgreSQL and its extensions', + url='https://github.com/postgrespro/testgres', license='PostgreSQL', author='Ildar Musin', author_email='zildermann@gmail.com', - url='https://github.com/postgrespro/testgres', - keywords=['testing', 'postgresql'], + keywords=['test', 'testing', 'postgresql'], + install_requires=install_requires, classifiers=[], - install_requires=install_requires) +) From 5a2a920023c8d988d3d1237457a40360314d96fb Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Thu, 8 Feb 2018 16:01:31 +0300 Subject: [PATCH 073/475] simplify README.md --- README.md | 103 +++++++++++++++++++++--------------------------------- 1 file changed, 40 insertions(+), 63 deletions(-) diff --git a/README.md b/README.md index acec819c..634be17d 100644 --- a/README.md +++ b/README.md @@ -37,80 +37,34 @@ python my_tests.py ``` -### Logging - -By default, `cleanup()` removes all temporary files (DB files, logs etc) that were created by testgres' API methods. -If you'd like to keep logs, execute `configure_testgres(node_cleanup_full=False)` before running any tests. - -> Note: context managers (aka `with`) call `stop()` and `cleanup()` automatically. - -testgres supports [python logging](https://docs.python.org/3.6/library/logging.html), -which means that you can aggregate logs from several nodes into one file: - -```python -import logging - -# write everything to /tmp/testgres.log -logging.basicConfig(filename='/tmp/testgres.log') - -# create two different nodes with logging -node1 = testgres.get_new_node(use_logging=True).init().start() -node2 = testgres.get_new_node(use_logging=True).init().start() - -# execute a few queries -node1.execute('postgres', 'select 1') -node2.execute('postgres', 'select 2') -``` - - ### Examples Here is an example of what you can do with `testgres`: ```python -with testgres.get_new_node('test') as node: - node.init() # run initdb - node.start() # start PostgreSQL - print(node.execute('postgres', 'select 1')) - node.stop() # stop PostgreSQL -``` - -Let's walk through the code. First, you create a new node using: - -```python -with testgres.get_new_node('master') as node: -``` - -or - -```python -with testgres.get_new_node('master', '/path/to/DB') as node: -``` +# create a node with random name, port, etc +with testgres.get_new_node() as node: -where `master` is a node's application name. Name matters if you're testing something like replication. -Function `get_new_node()` only creates directory structure in specified directory (or somewhere in '/tmp' if -we did not specify base directory) for cluster. After that, we have to initialize the PostgreSQL cluster: + # run inidb + node.init() -```python -node.init() -``` + # start PostgreSQL + node.start() -This function runs `initdb` command and adds some basic configuration to `postgresql.conf` and `pg_hba.conf` files. -Function `init()` accepts optional parameter `allows_streaming` which configures cluster for streaming replication (default is `False`). -Now we are ready to start: + # execute a query in a default DB + print(node.execute('select 1')) -```python -node.start() +# ... node stops and its files are about to be removed ``` -Finally, our temporary cluster is able to process queries. There are four ways to run them: +There are four API methods for runnig queries: | Command | Description | |----------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------| -| `node.psql(dbname, query)` | Runs query via `psql` command and returns tuple `(error code, stdout, stderr)`. | -| `node.safe_psql(dbname, query)` | Same as `psql()` except that it returns only `stdout`. If an error occures during the execution, an exception will be thrown. | -| `node.execute(dbname, query)` | Connects to PostgreSQL using `psycopg2` or `pg8000` (depends on which one is installed in your system) and returns two-dimensional array with data. | -| `node.connect(dbname, username)` | Returns connection wrapper (`NodeConnection`) capable of running several queries within a single transaction. | +| `node.psql(query, ...)` | Runs query via `psql` command and returns tuple `(error code, stdout, stderr)`. | +| `node.safe_psql(query, ...)` | Same as `psql()` except that it returns only `stdout`. If an error occures during the execution, an exception will be thrown. | +| `node.execute(query, ...)` | Connects to PostgreSQL using `psycopg2` or `pg8000` (depends on which one is installed in your system) and returns two-dimensional array with data. | +| `node.connect(dbname, ...)` | Returns connection wrapper (`NodeConnection`) capable of running several queries within a single transaction. | The last one is the most powerful: you can use `begin(isolation_level)`, `commit()` and `rollback()`: ```python @@ -120,10 +74,30 @@ with node.connect() as con: con.rollback() ``` -To stop the server, run: + +### Logging + +By default, `cleanup()` removes all temporary files (DB files, logs etc) that were created by testgres' API methods. +If you'd like to keep logs, execute `configure_testgres(node_cleanup_full=False)` before running any tests. + +> Note: context managers (aka `with`) call `stop()` and `cleanup()` automatically. + +testgres supports [python logging](https://docs.python.org/3.6/library/logging.html), +which means that you can aggregate logs from several nodes into one file: ```python -node.stop() +import logging + +# write everything to /tmp/testgres.log +logging.basicConfig(filename='/tmp/testgres.log') + +# create two different nodes with logging +node1 = testgres.get_new_node(use_logging=True).init().start() +node2 = testgres.get_new_node(use_logging=True).init().start() + +# execute a few queries +node1.execute('select 1') +node2.execute('select 2') ``` @@ -134,7 +108,10 @@ It's quite easy to create a backup and start a new replica: ```python with testgres.get_new_node('master') as master: master.init().start() + + # create a backup with master.backup() as backup: + # create and start a new replica replica = backup.spawn_replica('replica').start() @@ -147,7 +124,7 @@ with testgres.get_new_node('master') as master: ### Benchmarks -`testgres` also can help you to make benchmarks using `pgbench` from postgres installation: +`testgres` is also capable of running benchmarks using `pgbench`: ```python with testgres.get_new_node('master') as master: From 8b40272753968724d36a06bacdfdf900e438553e Mon Sep 17 00:00:00 2001 From: Ildus Kurbangaliev Date: Thu, 8 Feb 2018 18:19:02 +0300 Subject: [PATCH 074/475] Add documentation about custom configuration --- README.md | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 634be17d..0757ef46 100644 --- a/README.md +++ b/README.md @@ -136,10 +136,31 @@ with testgres.get_new_node('master') as master: print(res) ``` +### Custom configuration + +It's often useful to extend default configuration provided by `testgres`. + +`testgres` have `default_conf` function that helps to control some basic +options. The `append_conf` function can be used to add custom +lines to configuration lines: + +```python +ext_conf = "shared_preload_libraries = 'postgres_fdw'\n" + +with testgres.get_new_node('master') as master: + master.default_conf(fsync=True, + unix_sockets=False, + allow_streaming=True, + log_statement='all') + master.append_conf('postgresql.conf', ext_conf) +``` + +Note that `default_conf` is called by `init` function and the latter overwrites +the configuration file. That means `init` should be called before `append_conf`. ## Authors -[Ildar Musin](https://github.com/zilder) Postgres Professional Ltd., Russia -[Dmitry Ivanov](https://github.com/funbringer) Postgres Professional Ltd., Russia -[Ildus Kurbangaliev](https://github.com/ildus) Postgres Professional Ltd., Russia +[Ildar Musin](https://github.com/zilder) Postgres Professional Ltd., Russia +[Dmitry Ivanov](https://github.com/funbringer) Postgres Professional Ltd., Russia +[Ildus Kurbangaliev](https://github.com/ildus) Postgres Professional Ltd., Russia [Yury Zhuravlev](https://github.com/stalkerg) From 88ab4b22f0b62ad23334c17c4d3a60fe5b79801c Mon Sep 17 00:00:00 2001 From: Ildus Kurbangaliev Date: Thu, 8 Feb 2018 18:20:39 +0300 Subject: [PATCH 075/475] Update README.md --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 0757ef46..4da77a66 100644 --- a/README.md +++ b/README.md @@ -149,9 +149,9 @@ ext_conf = "shared_preload_libraries = 'postgres_fdw'\n" with testgres.get_new_node('master') as master: master.default_conf(fsync=True, - unix_sockets=False, - allow_streaming=True, - log_statement='all') + unix_sockets=False, + allow_streaming=True, + log_statement='all') master.append_conf('postgresql.conf', ext_conf) ``` From f023166bb885542d3bfd83e9a5bbed92fd8b6e4b Mon Sep 17 00:00:00 2001 From: Ildus Kurbangaliev Date: Fri, 9 Feb 2018 12:34:16 +0300 Subject: [PATCH 076/475] Update README.md --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 4da77a66..7fe5f257 100644 --- a/README.md +++ b/README.md @@ -160,7 +160,7 @@ the configuration file. That means `init` should be called before `append_conf`. ## Authors -[Ildar Musin](https://github.com/zilder) Postgres Professional Ltd., Russia -[Dmitry Ivanov](https://github.com/funbringer) Postgres Professional Ltd., Russia -[Ildus Kurbangaliev](https://github.com/ildus) Postgres Professional Ltd., Russia -[Yury Zhuravlev](https://github.com/stalkerg) +[Ildar Musin](https://github.com/zilder) Postgres Professional Ltd., Russia +[Dmitry Ivanov](https://github.com/funbringer) Postgres Professional Ltd., Russia +[Ildus Kurbangaliev](https://github.com/ildus) Postgres Professional Ltd., Russia +[Yury Zhuravlev](https://github.com/stalkerg) From 80d4e0081426d7b43936a73074bf57fb8c2e71df Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Sun, 11 Feb 2018 23:19:52 +0300 Subject: [PATCH 077/475] minor fixes in README.md --- README.md | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 7fe5f257..135a66d4 100644 --- a/README.md +++ b/README.md @@ -136,27 +136,34 @@ with testgres.get_new_node('master') as master: print(res) ``` + ### Custom configuration It's often useful to extend default configuration provided by `testgres`. -`testgres` have `default_conf` function that helps to control some basic -options. The `append_conf` function can be used to add custom +`testgres` has `default_conf()` function that helps control some basic +options. The `append_conf()` function can be used to add custom lines to configuration lines: ```python -ext_conf = "shared_preload_libraries = 'postgres_fdw'\n" +ext_conf = "shared_preload_libraries = 'postgres_fdw'" -with testgres.get_new_node('master') as master: +# initialize a new node +with testgres.get_new_node().init() as master: + + # ... do something ... + + # reset main config file master.default_conf(fsync=True, - unix_sockets=False, - allow_streaming=True, - log_statement='all') + allow_streaming=True) + + # add a new config line master.append_conf('postgresql.conf', ext_conf) ``` -Note that `default_conf` is called by `init` function and the latter overwrites -the configuration file. That means `init` should be called before `append_conf`. +Note that `default_conf()` is called by `init()` function; both of them overwrite +the configuration file, which means that they should be called before `append_conf()`. + ## Authors From 1975e127dd02ffdada4dc8e03e92ca57e2682e8b Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Wed, 14 Feb 2018 22:32:10 +0300 Subject: [PATCH 078/475] make exceptions more useful --- testgres/consts.py | 1 + testgres/exceptions.py | 84 +++++++++++++++++++++++++++++++++--------- testgres/node.py | 46 +++++++++++------------ testgres/utils.py | 26 +++++++------ 4 files changed, 104 insertions(+), 53 deletions(-) diff --git a/testgres/consts.py b/testgres/consts.py index e42a4fa7..fea3dc92 100644 --- a/testgres/consts.py +++ b/testgres/consts.py @@ -5,6 +5,7 @@ LOGS_DIR = "logs" RECOVERY_CONF_FILE = "recovery.conf" +PG_AUTO_CONF_FILE = "postgresql.auto.conf" PG_CONF_FILE = "postgresql.conf" HBA_CONF_FILE = "pg_hba.conf" diff --git a/testgres/exceptions.py b/testgres/exceptions.py index a5578759..cd10e8dc 100644 --- a/testgres/exceptions.py +++ b/testgres/exceptions.py @@ -1,47 +1,95 @@ # coding: utf-8 +import six -class TestgresException(Exception): - """ - Base exception - """ - pass +@six.python_2_unicode_compatible +class TestgresException(Exception): + def __str__(self): + s = super(TestgresException, self).__str__() + return six.text_type(s) class ExecUtilException(TestgresException): - """ - Stores exit code - """ - - def __init__(self, message, exit_code=0): + def __init__(self, + message=None, + command=None, + exit_code=0, + out=None): super(ExecUtilException, self).__init__(message) + + self.message = message + self.command = command self.exit_code = exit_code + self.out = out + def __str__(self): + msg = [] -class ClusterTestgresException(TestgresException): - pass + if self.message: + msg.append(self.message) + + if self.command: + msg.append('Command: {}'.format(self.command)) + + if self.exit_code: + msg.append('Exit code: {}'.format(self.exit_code)) + + if self.out: + msg.append('----\n{}'.format(self.out)) + + return six.text_type('\n').join(msg) class QueryException(TestgresException): + def __init__(self, message=None, query=None): + super(QueryException, self).__init__(message) + + self.message = message + self.query = query + + def __str__(self): + msg = [] + + if self.message: + msg.append(self.message) + + if self.query: + msg.append('Query: {}'.format(self.query)) + + return six.text_type('\n').join(msg) + + +class TimeoutException(QueryException): pass -class TimeoutException(TestgresException): +class CatchUpException(QueryException): pass class StartNodeException(TestgresException): - pass + def __init__(self, message=None, files=None): + super(StartNodeException, self).__init__(message) + self.message = message + self.files = files -class InitNodeException(TestgresException): - pass + def __str__(self): + msg = [] + if self.message: + msg.append(self.message) -class BackupException(TestgresException): + for key, value in six.iteritems(self.files or {}): + msg.append('{}\n----\n{}\n'.format(key, value)) + + return six.text_type('\n').join(msg) + + +class InitNodeException(TestgresException): pass -class CatchUpException(TestgresException): +class BackupException(TestgresException): pass diff --git a/testgres/node.py b/testgres/node.py index 0e8cfb76..fe4744ca 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -24,6 +24,7 @@ DATA_DIR, \ LOGS_DIR, \ PG_CONF_FILE, \ + PG_AUTO_CONF_FILE, \ HBA_CONF_FILE, \ RECOVERY_CONF_FILE, \ PG_LOG_FILE, \ @@ -215,22 +216,18 @@ def _maybe_stop_logger(self): if self._logger: self._logger.stop() - def _format_verbose_error(self, message=None): - # list of important files + N of last lines + def _collect_special_files(self): + result = {} + + # list of important files + last N lines files = [ (os.path.join(self.data_dir, PG_CONF_FILE), 0), - (os.path.join(self.data_dir, HBA_CONF_FILE), 0), + (os.path.join(self.data_dir, PG_AUTO_CONF_FILE), 0), (os.path.join(self.data_dir, RECOVERY_CONF_FILE), 0), + (os.path.join(self.data_dir, HBA_CONF_FILE), 0), (self.pg_log_name, TestgresConfig.error_log_lines) ] - error_text = "" - - # append message if asked to - if message: - error_text += message - error_text += '\n' * 2 - for f, num_lines in files: # skip missing files if not os.path.exists(f): @@ -244,10 +241,10 @@ def _format_verbose_error(self, message=None): # read whole file lines = _f.read().decode('utf-8') - # append contents - error_text += u"{}:\n----\n{}\n".format(f, lines) + # fill dict + result[f] = lines - return error_text + return result def init(self, fsync=False, @@ -429,7 +426,8 @@ def get_pid(self): """ if self.status(): - with io.open(os.path.join(self.data_dir, 'postmaster.pid')) as f: + pid_file = os.path.join(self.data_dir, 'postmaster.pid') + with io.open(pid_file) as f: return int(f.readline()) # for clarity @@ -478,8 +476,9 @@ def start(self, params=[]): try: execute_utility(_params, self.utils_log_name) except ExecUtilException as e: - msg = self._format_verbose_error('Cannot start node') - raise_from(StartNodeException(msg), e) + msg = 'Cannot start node' + files = self._collect_special_files() + raise_from(StartNodeException(msg, files), e) self._maybe_start_logger() @@ -533,8 +532,9 @@ def restart(self, params=[]): try: execute_utility(_params, self.utils_log_name) except ExecUtilException as e: - msg = self._format_verbose_error('Cannot restart node') - raise_from(StartNodeException(msg), e) + msg = 'Cannot restart node' + files = self._collect_special_files() + raise_from(StartNodeException(msg, files), e) self._maybe_start_logger() @@ -695,7 +695,7 @@ def safe_psql(self, username=username, input=input) if ret: - raise QueryException((err or b'').decode('utf-8')) + raise QueryException((err or b'').decode('utf-8'), query) return out @@ -793,13 +793,13 @@ def poll_query_until(self, return # done if res is None: - raise QueryException('Query returned None') + raise QueryException('Query returned None', query) if len(res) == 0: - raise QueryException('Query returned 0 rows') + raise QueryException('Query returned 0 rows', query) if len(res[0]) == 0: - raise QueryException('Query returned 0 columns') + raise QueryException('Query returned 0 columns', query) if res[0][0] == expected: return # done @@ -916,7 +916,7 @@ def catchup(self, dbname=None, username=None): username=username, max_attempts=0) # infinite except Exception as e: - raise_from(CatchUpException('Failed to catch up'), e) + raise_from(CatchUpException("Failed to catch up", poll_lsn), e) def pgbench(self, dbname=None, diff --git a/testgres/utils.py b/testgres/utils.py index d4216222..46ac32a9 100644 --- a/testgres/utils.py +++ b/testgres/utils.py @@ -90,29 +90,31 @@ def execute_utility(args, logfile): out, _ = process.communicate() out = '' if not out else out.decode('utf-8') + # format command + command = u' '.join(args) + # write new log entry if possible try: with io.open(logfile, 'a') as file_out: - # write util's name and args - file_out.write(u' '.join(args)) + file_out.write(command) - # write output if out: + # comment-out lines + lines = ('# ' + l for l in out.splitlines()) file_out.write(u'\n') - file_out.write(out) + file_out.writelines(lines) - # finally, a separator file_out.write(u'\n') except IOError: pass - # format exception, if needed - error_code = process.returncode - if error_code: - error_text = (u"{} failed with exit code {}\n" - u"log:\n----\n{}\n").format(args[0], error_code, out) - - raise ExecUtilException(error_text, error_code) + exit_code = process.returncode + if exit_code: + message = 'Failed to execute utility' + raise ExecUtilException(message=message, + command=command, + exit_code=exit_code, + out=out) return out From a731c20364eb910e70e184f4b25d5d2ef05c83e4 Mon Sep 17 00:00:00 2001 From: Valeria112 Date: Wed, 14 Feb 2018 23:38:09 +0300 Subject: [PATCH 079/475] enum for xlog_method in NodeBackup --- testgres/backup.py | 25 +++++++++++++++++++++---- testgres/consts.py | 3 --- testgres/node.py | 12 +++++++----- 3 files changed, 28 insertions(+), 12 deletions(-) diff --git a/testgres/backup.py b/testgres/backup.py index a4484309..46cd435e 100644 --- a/testgres/backup.py +++ b/testgres/backup.py @@ -5,12 +5,12 @@ import tempfile from six import raise_from +from enum import Enum from .consts import \ DATA_DIR, \ PG_CONF_FILE, \ - BACKUP_LOG_FILE, \ - DEFAULT_XLOG_METHOD + BACKUP_LOG_FILE from .exceptions import BackupException @@ -20,6 +20,16 @@ execute_utility +class XLogMethod(Enum): + """ + Available WAL methods for NodeBackup + """ + + none = 'none' + fetch = 'fetch' + stream = 'stream' + + class NodeBackup(object): """ Smart object responsible for backups @@ -33,7 +43,7 @@ def __init__(self, node, base_dir=None, username=None, - xlog_method=DEFAULT_XLOG_METHOD): + xlog_method=XLogMethod.fetch): """ Create a new backup. @@ -47,6 +57,13 @@ def __init__(self, if not node.status(): raise BackupException('Node must be running') + # Check arguments + if not isinstance(xlog_method, XLogMethod): + try: + xlog_method = XLogMethod(xlog_method) + except ValueError: + raise BackupException('Invalid xlog_method "{}"'.format(xlog_method)) + # Set default arguments username = username or default_username() base_dir = base_dir or tempfile.mkdtemp() @@ -68,7 +85,7 @@ def __init__(self, "-h", node.host, "-U", username, "-D", data_dir, - "-X", xlog_method + "-X", xlog_method.value ] execute_utility(_params, self.log_file) diff --git a/testgres/consts.py b/testgres/consts.py index e42a4fa7..36b42722 100644 --- a/testgres/consts.py +++ b/testgres/consts.py @@ -12,6 +12,3 @@ PG_LOG_FILE = "postgresql.log" UTILS_LOG_FILE = "utils.log" BACKUP_LOG_FILE = "backup.log" - -# default argument value -DEFAULT_XLOG_METHOD = "fetch" diff --git a/testgres/node.py b/testgres/node.py index f7c8b7db..048927eb 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -27,8 +27,7 @@ HBA_CONF_FILE, \ RECOVERY_CONF_FILE, \ PG_LOG_FILE, \ - UTILS_LOG_FILE, \ - DEFAULT_XLOG_METHOD + UTILS_LOG_FILE from .exceptions import \ CatchUpException, \ @@ -52,6 +51,10 @@ method_decorator, \ positional_args_hack +from .backup import \ + XLogMethod, \ + NodeBackup + class NodeStatus(IntEnum): """ @@ -838,7 +841,7 @@ def execute(self, return res - def backup(self, username=None, xlog_method=DEFAULT_XLOG_METHOD): + def backup(self, username=None, xlog_method=XLogMethod.fetch): """ Perform pg_basebackup. @@ -850,7 +853,6 @@ def backup(self, username=None, xlog_method=DEFAULT_XLOG_METHOD): A smart object of type NodeBackup. """ - from .backup import NodeBackup return NodeBackup(node=self, username=username, xlog_method=xlog_method) @@ -858,7 +860,7 @@ def backup(self, username=None, xlog_method=DEFAULT_XLOG_METHOD): def replicate(self, name=None, username=None, - xlog_method=DEFAULT_XLOG_METHOD, + xlog_method=XLogMethod.fetch, use_logging=False): """ Create a binary replica of this node. From 1f4e0d3e0229f1363fdaac5609dc740b324d3d09 Mon Sep 17 00:00:00 2001 From: Ildus Kurbangaliev Date: Thu, 15 Feb 2018 15:15:19 +0300 Subject: [PATCH 080/475] Make "use_logging" parameter global --- README.md | 15 +++++++++++---- testgres/api.py | 5 ++--- testgres/backup.py | 12 ++++-------- testgres/config.py | 2 ++ testgres/node.py | 14 ++++---------- tests/test_simple.py | 5 ++++- 6 files changed, 27 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 135a66d4..919bbdd3 100644 --- a/README.md +++ b/README.md @@ -82,7 +82,7 @@ If you'd like to keep logs, execute `configure_testgres(node_cleanup_full=False) > Note: context managers (aka `with`) call `stop()` and `cleanup()` automatically. -testgres supports [python logging](https://docs.python.org/3.6/library/logging.html), +`testgres` supports [python logging](https://docs.python.org/3.6/library/logging.html), which means that you can aggregate logs from several nodes into one file: ```python @@ -91,13 +91,20 @@ import logging # write everything to /tmp/testgres.log logging.basicConfig(filename='/tmp/testgres.log') -# create two different nodes with logging -node1 = testgres.get_new_node(use_logging=True).init().start() -node2 = testgres.get_new_node(use_logging=True).init().start() +# enable logging, and create two different nodes +testgres.configure_testgres(enable_python_logging=True) +node1 = testgres.get_new_node().init().start() +node2 = testgres.get_new_node().init().start() # execute a few queries node1.execute('select 1') node2.execute('select 2') + +# disable logging +testgres.configure_testgres(enable_python_logging=False) + +Look at `tests/test_simple.py` file for a complete example of the logging +configuration. ``` diff --git a/testgres/api.py b/testgres/api.py index 685708e1..fd4f1bf2 100644 --- a/testgres/api.py +++ b/testgres/api.py @@ -36,17 +36,16 @@ from .node import PostgresNode -def get_new_node(name=None, base_dir=None, use_logging=False): +def get_new_node(name=None, base_dir=None): """ Create a new node (select port automatically). Args: name: node's application name. base_dir: path to node's data directory. - use_logging: enable python logging. Returns: An instance of PostgresNode. """ - return PostgresNode(name=name, base_dir=base_dir, use_logging=use_logging) + return PostgresNode(name=name, base_dir=base_dir) diff --git a/testgres/backup.py b/testgres/backup.py index a4484309..687f3f0c 100644 --- a/testgres/backup.py +++ b/testgres/backup.py @@ -115,14 +115,13 @@ def _prepare_dir(self, destroy): # Return path to new node return dest_base_dir - def spawn_primary(self, name=None, destroy=True, use_logging=False): + def spawn_primary(self, name=None, destroy=True): """ Create a primary node from a backup. Args: name: primary's application name. destroy: should we convert this backup into a node? - use_logging: enable python logging. Returns: New instance of PostgresNode. @@ -134,8 +133,7 @@ def spawn_primary(self, name=None, destroy=True, use_logging=False): # Build a new PostgresNode from .node import PostgresNode node = PostgresNode(name=name, - base_dir=base_dir, - use_logging=use_logging) + base_dir=base_dir) # New nodes should always remove dir tree node._should_rm_dirs = True @@ -145,14 +143,13 @@ def spawn_primary(self, name=None, destroy=True, use_logging=False): return node - def spawn_replica(self, name=None, destroy=True, use_logging=False): + def spawn_replica(self, name=None, destroy=True): """ Create a replica of the original node from a backup. Args: name: replica's application name. destroy: should we convert this backup into a node? - use_logging: enable python logging. Returns: New instance of PostgresNode. @@ -160,8 +157,7 @@ def spawn_replica(self, name=None, destroy=True, use_logging=False): # Build a new PostgresNode node = self.spawn_primary(name=name, - destroy=destroy, - use_logging=use_logging) + destroy=destroy) # Assign it a master and a recovery file (private magic) node._assign_master(self.original_node) diff --git a/testgres/config.py b/testgres/config.py index e2263d95..52c5ee58 100644 --- a/testgres/config.py +++ b/testgres/config.py @@ -11,6 +11,7 @@ class TestgresConfig: cache_pg_config: shall we cache pg_config results? + use_python_logging: use python logging configuration for all nodes. error_log_lines: N of log lines to be included into exception (0=inf). node_cleanup_full: shall we remove EVERYTHING (including logs)? @@ -23,6 +24,7 @@ class TestgresConfig: cache_pg_config = True + use_python_logging = False error_log_lines = 20 node_cleanup_full = True diff --git a/testgres/node.py b/testgres/node.py index 0e8cfb76..4d18a563 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -70,7 +70,7 @@ def __bool__(self): class PostgresNode(object): - def __init__(self, name=None, port=None, base_dir=None, use_logging=False): + def __init__(self, name=None, port=None, base_dir=None): """ Create a new node manually. @@ -78,7 +78,6 @@ def __init__(self, name=None, port=None, base_dir=None, use_logging=False): name: node's application name. port: port to accept connections. base_dir: path to node's data directory. - use_logging: enable python logging. """ # basic @@ -94,7 +93,6 @@ def __init__(self, name=None, port=None, base_dir=None, use_logging=False): # private self._should_free_port = port is None - self._use_logging = use_logging self._logger = None self._master = None @@ -205,7 +203,7 @@ def _prepare_dirs(self): os.makedirs(self.logs_dir) def _maybe_start_logger(self): - if self._use_logging: + if TestgresConfig.use_python_logging: # spawn new logger if it doesn't exist or is stopped if not self._logger or not self._logger.is_alive(): self._logger = TestgresLogger(self.name, self.pg_log_name) @@ -869,8 +867,7 @@ def backup(self, username=None, xlog_method=DEFAULT_XLOG_METHOD): def replicate(self, name=None, username=None, - xlog_method=DEFAULT_XLOG_METHOD, - use_logging=False): + xlog_method=DEFAULT_XLOG_METHOD): """ Create a binary replica of this node. @@ -878,15 +875,12 @@ def replicate(self, name: replica's application name. username: database user name. xlog_method: a method for collecting the logs ('fetch' | 'stream'). - use_logging: enable python logging. """ backup = self.backup(username=username, xlog_method=xlog_method) # transform backup into a replica - return backup.spawn_replica(name=name, - destroy=True, - use_logging=use_logging) + return backup.spawn_replica(name=name, destroy=True) def catchup(self, dbname=None, username=None): """ diff --git a/tests/test_simple.py b/tests/test_simple.py index 57a7461a..85861684 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -458,9 +458,10 @@ def test_logging(self): } logging.config.dictConfig(log_conf) + configure_testgres(use_python_logging=True) node_name = 'master' - with get_new_node(name=node_name, use_logging=True) as master: + with get_new_node(name=node_name) as master: master.init().start() # execute a dummy query a few times @@ -482,6 +483,8 @@ def test_logging(self): master.restart() self.assertTrue(master._logger.is_alive()) + configure_testgres(use_python_logging=False) + @unittest.skipUnless( util_is_executable("pgbench"), "pgbench may be missing") def test_pgbench(self): From cfa8cabd3b7d33c39ffbd8419c37fe590118e90f Mon Sep 17 00:00:00 2001 From: Ildus Kurbangaliev Date: Thu, 15 Feb 2018 15:57:33 +0300 Subject: [PATCH 081/475] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 919bbdd3..c649d29f 100644 --- a/README.md +++ b/README.md @@ -102,10 +102,10 @@ node2.execute('select 2') # disable logging testgres.configure_testgres(enable_python_logging=False) +``` Look at `tests/test_simple.py` file for a complete example of the logging configuration. -``` ### Backup & replication From 04600844d6c457090309fde7129b1cbf315f49fb Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Thu, 15 Feb 2018 15:58:42 +0300 Subject: [PATCH 082/475] specify versions in setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 2a7e3dc8..f307b6fa 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ from distutils.core import setup # Basic dependencies -install_requires = ["pg8000", "port-for", "six"] +install_requires = ["pg8000", "port-for>=0.4", "six>=1.9.0"] # Add compatibility enum class if sys.version_info < (3, 4): From 7536809cb101ac810c18847dc40f6043b46d4a0e Mon Sep 17 00:00:00 2001 From: Ildus Kurbangaliev Date: Thu, 15 Feb 2018 16:42:03 +0300 Subject: [PATCH 083/475] Fix application_name of replica --- testgres/node.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testgres/node.py b/testgres/node.py index 4d18a563..d8a1cbe0 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -174,7 +174,7 @@ def _create_recovery_conf(self, username): u"application_name={} " u"port={} " u"user={} " - ).format(master.name, master.port, username) + ).format(self.name, master.port, username) # host is tricky try: From b4cf4e6804f9c3662a7528926c3506ea9819a686 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Thu, 15 Feb 2018 15:52:47 +0300 Subject: [PATCH 084/475] small improvements in execute_utility() --- testgres/exceptions.py | 20 ++++++++++---------- testgres/node.py | 6 +++--- testgres/utils.py | 4 ++-- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/testgres/exceptions.py b/testgres/exceptions.py index cd10e8dc..69fe43db 100644 --- a/testgres/exceptions.py +++ b/testgres/exceptions.py @@ -3,13 +3,11 @@ import six -@six.python_2_unicode_compatible class TestgresException(Exception): - def __str__(self): - s = super(TestgresException, self).__str__() - return six.text_type(s) + pass +@six.python_2_unicode_compatible class ExecUtilException(TestgresException): def __init__(self, message=None, @@ -30,17 +28,18 @@ def __str__(self): msg.append(self.message) if self.command: - msg.append('Command: {}'.format(self.command)) + msg.append(u'Command: {}'.format(self.command)) if self.exit_code: - msg.append('Exit code: {}'.format(self.exit_code)) + msg.append(u'Exit code: {}'.format(self.exit_code)) if self.out: - msg.append('----\n{}'.format(self.out)) + msg.append(u'----\n{}'.format(self.out)) return six.text_type('\n').join(msg) +@six.python_2_unicode_compatible class QueryException(TestgresException): def __init__(self, message=None, query=None): super(QueryException, self).__init__(message) @@ -55,7 +54,7 @@ def __str__(self): msg.append(self.message) if self.query: - msg.append('Query: {}'.format(self.query)) + msg.append(u'Query: {}'.format(self.query)) return six.text_type('\n').join(msg) @@ -68,6 +67,7 @@ class CatchUpException(QueryException): pass +@six.python_2_unicode_compatible class StartNodeException(TestgresException): def __init__(self, message=None, files=None): super(StartNodeException, self).__init__(message) @@ -81,8 +81,8 @@ def __str__(self): if self.message: msg.append(self.message) - for key, value in six.iteritems(self.files or {}): - msg.append('{}\n----\n{}\n'.format(key, value)) + for f, lines in self.files or []: + msg.append(u'{}\n----\n{}\n'.format(f, lines)) return six.text_type('\n').join(msg) diff --git a/testgres/node.py b/testgres/node.py index fe4744ca..1d0b2ab0 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -217,7 +217,7 @@ def _maybe_stop_logger(self): self._logger.stop() def _collect_special_files(self): - result = {} + result = [] # list of important files + last N lines files = [ @@ -241,8 +241,8 @@ def _collect_special_files(self): # read whole file lines = _f.read().decode('utf-8') - # fill dict - result[f] = lines + # fill list + result.append((f, lines)) return result diff --git a/testgres/utils.py b/testgres/utils.py index 46ac32a9..b9152aba 100644 --- a/testgres/utils.py +++ b/testgres/utils.py @@ -100,7 +100,7 @@ def execute_utility(args, logfile): if out: # comment-out lines - lines = ('# ' + l for l in out.splitlines()) + lines = ('# ' + l for l in out.splitlines(True)) file_out.write(u'\n') file_out.writelines(lines) @@ -110,7 +110,7 @@ def execute_utility(args, logfile): exit_code = process.returncode if exit_code: - message = 'Failed to execute utility' + message = 'Utility exited with non-zero code' raise ExecUtilException(message=message, command=command, exit_code=exit_code, From 5a4aa2df3bca5556cc4dee69159459a149492e1b Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Thu, 15 Feb 2018 17:35:50 +0300 Subject: [PATCH 085/475] more tests for exceptions --- tests/test_simple.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/test_simple.py b/tests/test_simple.py index 85861684..edbfdf04 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -620,6 +620,11 @@ def test_ports_management(self): # check that port has been freed successfully self.assertEqual(len(bound_ports), 0) + def test_exceptions(self): + str(StartNodeException('msg', [('file', 'lines')])) + str(ExecUtilException('msg', 'cmd', 1, 'out')) + str(QueryException('msg', 'query')) + def test_version_management(self): a = LooseVersion('10.0') b = LooseVersion('10') From 740b8d8defb221d010a5a8e0a0bcceb32284e4ee Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Fri, 16 Feb 2018 04:21:29 +0300 Subject: [PATCH 086/475] small fixes in README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index c649d29f..3839981e 100644 --- a/README.md +++ b/README.md @@ -161,11 +161,11 @@ with testgres.get_new_node().init() as master: # ... do something ... # reset main config file - master.default_conf(fsync=True, + master.default_conf(fsync=True, allow_streaming=True) # add a new config line - master.append_conf('postgresql.conf', ext_conf) + master.append_conf('postgresql.conf', ext_conf) ``` Note that `default_conf()` is called by `init()` function; both of them overwrite From 72976a62d6a0dfacbdbd408027ea27b341305288 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Tue, 20 Feb 2018 22:56:31 +0300 Subject: [PATCH 087/475] initial implementation of config stack --- testgres/__init__.py | 8 ++- testgres/cache.py | 35 +++------- testgres/config.py | 149 +++++++++++++++++++++++++++++++++++++---- testgres/consts.py | 1 + testgres/exceptions.py | 6 +- testgres/node.py | 21 +++--- testgres/utils.py | 37 +++++----- tests/test_simple.py | 70 +++++++++---------- 8 files changed, 217 insertions(+), 110 deletions(-) diff --git a/testgres/__init__.py b/testgres/__init__.py index 29b017a9..7d5a7125 100644 --- a/testgres/__init__.py +++ b/testgres/__init__.py @@ -1,6 +1,12 @@ from .api import get_new_node from .backup import NodeBackup -from .config import TestgresConfig, configure_testgres + +from .config import \ + TestgresConfig, \ + configure_testgres, \ + scoped_config, \ + push_config, \ + pop_config from .connection import \ IsolationLevel, \ diff --git a/testgres/cache.py b/testgres/cache.py index 2fad48f8..d0eaac9c 100644 --- a/testgres/cache.py +++ b/testgres/cache.py @@ -1,13 +1,11 @@ # coding: utf-8 -import atexit import os import shutil -import tempfile from six import raise_from -from .config import TestgresConfig +from .config import testgres_config from .exceptions import \ InitNodeException, \ @@ -18,44 +16,31 @@ execute_utility -def cached_initdb(data_dir, initdb_logfile, initdb_params=[]): +def cached_initdb(data_dir, logfile=None, params=None): """ Perform initdb or use cached node files. """ - def call_initdb(initdb_dir): + def call_initdb(initdb_dir, log=None): try: _params = [get_bin_path("initdb"), "-D", initdb_dir, "-N"] - execute_utility(_params + initdb_params, initdb_logfile) + execute_utility(_params + (params or []), log) except ExecUtilException as e: raise_from(InitNodeException("Failed to run initdb"), e) - def rm_cached_data_dir(cached_data_dir): - shutil.rmtree(cached_data_dir, ignore_errors=True) - - # Call initdb if we have custom params or shouldn't cache it - if initdb_params or not TestgresConfig.cache_initdb: - call_initdb(data_dir) + if params or not testgres_config.cache_initdb: + call_initdb(data_dir, logfile) else: - # Set default temp dir for cached initdb - if TestgresConfig.cached_initdb_dir is None: - - # Create default temp dir - TestgresConfig.cached_initdb_dir = tempfile.mkdtemp() - - # Schedule cleanup - atexit.register(rm_cached_data_dir, - TestgresConfig.cached_initdb_dir) - # Fetch cached initdb dir - cached_data_dir = TestgresConfig.cached_initdb_dir + cached_data_dir = testgres_config.cached_initdb_dir # Initialize cached initdb - if not os.listdir(cached_data_dir): + if not os.path.exists(cached_data_dir) or \ + not os.listdir(cached_data_dir): call_initdb(cached_data_dir) try: # Copy cached initdb to current data dir shutil.copytree(cached_data_dir, data_dir) except Exception as e: - raise_from(InitNodeException("Failed to copy files"), e) + raise_from(InitNodeException("Failed to spawn a node"), e) diff --git a/testgres/config.py b/testgres/config.py index 52c5ee58..782f315c 100644 --- a/testgres/config.py +++ b/testgres/config.py @@ -1,26 +1,35 @@ # coding: utf-8 +import atexit +import copy +import shutil +import tempfile -class TestgresConfig: +from contextlib import contextmanager + + +class GlobalConfig(object): """ Global config (override default settings). Attributes: - cache_initdb: shall we use cached initdb instance? - cached_initdb_dir: shall we create a temp dir for cached initdb? + cache_initdb: shall we use cached initdb instance? + cached_initdb_dir: shall we create a temp dir for cached initdb? - cache_pg_config: shall we cache pg_config results? + cache_pg_config: shall we cache pg_config results? - use_python_logging: use python logging configuration for all nodes. - error_log_lines: N of log lines to be included into exception (0=inf). + use_python_logging: use python logging configuration for all nodes. + error_log_lines: N of log lines to be shown in exception (0=inf). - node_cleanup_full: shall we remove EVERYTHING (including logs)? + node_cleanup_full: shall we remove EVERYTHING (including logs)? node_cleanup_on_good_exit: remove base_dir on nominal __exit__(). node_cleanup_on_bad_exit: remove base_dir on __exit__() via exception. + + NOTE: attributes must not be callable or begin with __. """ cache_initdb = True - cached_initdb_dir = None + _cached_initdb_dir = None cache_pg_config = True @@ -31,12 +40,128 @@ class TestgresConfig: node_cleanup_on_good_exit = True node_cleanup_on_bad_exit = False + @property + def cached_initdb_dir(self): + return self._cached_initdb_dir + + @cached_initdb_dir.setter + def cached_initdb_dir(self, value): + self._cached_initdb_dir = value + + if value: + cached_initdb_dirs.add(value) + + def __init__(self, **options): + self.update(options) + + def __setitem__(self, key, value): + setattr(self, key, value) + + def __getitem__(self, key): + return getattr(self, key) + + def __setattr__(self, name, value): + if name not in self.keys(): + raise TypeError('Unknown option {}'.format(name)) + + super(GlobalConfig, self).__setattr__(name, value) + + def keys(self): + keys = [] + + for key in dir(GlobalConfig): + if not key.startswith('__') and not callable(self[key]): + keys.append(key) + + return keys + + def items(self): + return ((key, self[key]) for key in self.keys()) + + def update(self, config): + for key, value in config.items(): + self[key] = value + + return self + + def copy(self): + return copy.copy(self) + + +# cached dirs to be removed +cached_initdb_dirs = set() + +# default config object +testgres_config = GlobalConfig() + +# NOTE: for compatibility +TestgresConfig = testgres_config + +# stack of GlobalConfigs +config_stack = [testgres_config] + + +def rm_cached_initdb_dirs(): + for d in cached_initdb_dirs: + shutil.rmtree(d, ignore_errors=True) + + +def push_config(**options): + """ + Permanently set custom GlobalConfig options + and put previous settings on top of stack. + """ + + # push current config to stack + config_stack.append(testgres_config.copy()) + + return testgres_config.update(options) + + +def pop_config(): + """ + Set previous GlobalConfig options from stack. + """ + + if len(config_stack) <= 1: + raise IndexError('Reached initial config') + + # restore popped config + return testgres_config.update(config_stack.pop()) + + +@contextmanager +def scoped_config(**options): + """ + Temporarily set custom GlobalConfig options for this context. + + Example: + >>> with scoped_config(cache_initdb=False): + ... with get_new_node().init().start() as node: + ... print(node.execute('select 1')) + """ + + # set a new config with options + config = push_config(**options) + try: + # return it + yield config + finally: + # restore previous config + pop_config() + def configure_testgres(**options): """ - Configure testgres. - Look at TestgresConfig to check what can be changed. + Adjust current global options. + Look at GlobalConfig to learn what can be set. """ - for key, option in options.items(): - setattr(TestgresConfig, key, option) + testgres_config.update(options) + + +# NOTE: to be executed at exit() +atexit.register(rm_cached_initdb_dirs) + +# NOTE: assign initial cached dir for initdb +testgres_config.cached_initdb_dir = tempfile.mkdtemp() diff --git a/testgres/consts.py b/testgres/consts.py index fea3dc92..e8dd8c99 100644 --- a/testgres/consts.py +++ b/testgres/consts.py @@ -4,6 +4,7 @@ DATA_DIR = "data" LOGS_DIR = "logs" +# names for config files RECOVERY_CONF_FILE = "recovery.conf" PG_AUTO_CONF_FILE = "postgresql.auto.conf" PG_CONF_FILE = "postgresql.conf" diff --git a/testgres/exceptions.py b/testgres/exceptions.py index 69fe43db..6832c788 100644 --- a/testgres/exceptions.py +++ b/testgres/exceptions.py @@ -9,11 +9,7 @@ class TestgresException(Exception): @six.python_2_unicode_compatible class ExecUtilException(TestgresException): - def __init__(self, - message=None, - command=None, - exit_code=0, - out=None): + def __init__(self, message=None, command=None, exit_code=0, out=None): super(ExecUtilException, self).__init__(message) self.message = message diff --git a/testgres/node.py b/testgres/node.py index d39eff75..2afab665 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -13,7 +13,7 @@ from .cache import cached_initdb -from .config import TestgresConfig +from .config import testgres_config from .connection import \ NodeConnection, \ @@ -88,8 +88,8 @@ def __init__(self, name=None, port=None, base_dir=None): self.base_dir = base_dir # defaults for __exit__() - self.cleanup_on_good_exit = TestgresConfig.node_cleanup_on_good_exit - self.cleanup_on_bad_exit = TestgresConfig.node_cleanup_on_bad_exit + self.cleanup_on_good_exit = testgres_config.node_cleanup_on_good_exit + self.cleanup_on_bad_exit = testgres_config.node_cleanup_on_bad_exit self.shutdown_max_attempts = 3 # private @@ -150,7 +150,7 @@ def _try_shutdown(self, max_attempts): self.stop() break # OK except ExecUtilException: - pass # one more time + pass # one more time except Exception: # TODO: probably kill stray instance eprint('cannot stop node {}'.format(self.name)) @@ -204,7 +204,7 @@ def _prepare_dirs(self): os.makedirs(self.logs_dir) def _maybe_start_logger(self): - if TestgresConfig.use_python_logging: + if testgres_config.use_python_logging: # spawn new logger if it doesn't exist or is stopped if not self._logger or not self._logger.is_alive(): self._logger = TestgresLogger(self.name, self.pg_log_name) @@ -223,7 +223,7 @@ def _collect_special_files(self): (os.path.join(self.data_dir, PG_AUTO_CONF_FILE), 0), (os.path.join(self.data_dir, RECOVERY_CONF_FILE), 0), (os.path.join(self.data_dir, HBA_CONF_FILE), 0), - (self.pg_log_name, TestgresConfig.error_log_lines) + (self.pg_log_name, testgres_config.error_log_lines) ] for f, num_lines in files: @@ -248,7 +248,7 @@ def init(self, fsync=False, unix_sockets=True, allow_streaming=True, - initdb_params=[]): + initdb_params=None): """ Perform initdb for this node. @@ -266,8 +266,9 @@ def init(self, self._prepare_dirs() # initialize this PostgreSQL node - initdb_log = os.path.join(self.logs_dir, "initdb.log") - cached_initdb(self.data_dir, initdb_log, initdb_params) + cached_initdb(data_dir=self.data_dir, + logfile=self.utils_log_name, + params=initdb_params) # initialize default config files self.default_conf(fsync=fsync, @@ -603,7 +604,7 @@ def cleanup(self, max_attempts=3): self._try_shutdown(max_attempts) # choose directory to be removed - if TestgresConfig.node_cleanup_full: + if testgres_config.node_cleanup_full: rm_dir = self.base_dir # everything else: rm_dir = self.data_dir # just data, save logs diff --git a/testgres/utils.py b/testgres/utils.py index b9152aba..01befbc5 100644 --- a/testgres/utils.py +++ b/testgres/utils.py @@ -13,7 +13,7 @@ from distutils.version import LooseVersion -from .config import TestgresConfig +from .config import testgres_config from .exceptions import ExecUtilException # rows returned by PG_CONFIG @@ -68,7 +68,7 @@ def generate_app_name(): return 'testgres-{}'.format(str(uuid.uuid4())) -def execute_utility(args, logfile): +def execute_utility(args, logfile=None): """ Execute utility (pg_ctl, pg_dump etc). @@ -94,27 +94,26 @@ def execute_utility(args, logfile): command = u' '.join(args) # write new log entry if possible - try: - with io.open(logfile, 'a') as file_out: - file_out.write(command) + if logfile: + try: + with io.open(logfile, 'a') as file_out: + file_out.write(command) - if out: - # comment-out lines - lines = ('# ' + l for l in out.splitlines(True)) - file_out.write(u'\n') - file_out.writelines(lines) + if out: + # comment-out lines + lines = ('# ' + l for l in out.splitlines(True)) + file_out.write(u'\n') + file_out.writelines(lines) - file_out.write(u'\n') - except IOError: - pass + file_out.write(u'\n') + except IOError: + pass exit_code = process.returncode if exit_code: message = 'Utility exited with non-zero code' - raise ExecUtilException(message=message, - command=command, - exit_code=exit_code, - out=out) + raise ExecUtilException( + message=message, command=command, exit_code=exit_code, out=out) return out @@ -146,7 +145,7 @@ def get_bin_path(filename): def get_pg_config(): """ Return output of pg_config (provided that it is installed). - NOTE: this fuction caches the result by default (see TestgresConfig). + NOTE: this fuction caches the result by default (see GlobalConfig). """ def cache_pg_config_data(cmd): @@ -166,7 +165,7 @@ def cache_pg_config_data(cmd): return data # drop cache if asked to - if not TestgresConfig.cache_pg_config: + if not testgres_config.cache_pg_config: global _pg_config_data _pg_config_data = {} diff --git a/tests/test_simple.py b/tests/test_simple.py index edbfdf04..98593a7e 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -23,8 +23,7 @@ TimeoutException from testgres import \ - TestgresConfig, \ - configure_testgres + scoped_config from testgres import \ NodeStatus, \ @@ -458,32 +457,31 @@ def test_logging(self): } logging.config.dictConfig(log_conf) - configure_testgres(use_python_logging=True) - node_name = 'master' - with get_new_node(name=node_name) as master: - master.init().start() + with scoped_config(use_python_logging=True): + node_name = 'master' - # execute a dummy query a few times - for i in range(20): - master.execute('select 1') - time.sleep(0.01) + with get_new_node(name=node_name) as master: + master.init().start() - # let logging worker do the job - time.sleep(0.1) + # execute a dummy query a few times + for i in range(20): + master.execute('select 1') + time.sleep(0.01) - # check that master's port is found - with open(logfile.name, 'r') as log: - lines = log.readlines() - self.assertTrue(any(node_name in s for s in lines)) + # let logging worker do the job + time.sleep(0.1) - # test logger after stop/start/restart - master.stop() - master.start() - master.restart() - self.assertTrue(master._logger.is_alive()) + # check that master's port is found + with open(logfile.name, 'r') as log: + lines = log.readlines() + self.assertTrue(any(node_name in s for s in lines)) - configure_testgres(use_python_logging=False) + # test logger after stop/start/restart + master.stop() + master.start() + master.restart() + self.assertTrue(master._logger.is_alive()) @unittest.skipUnless( util_is_executable("pgbench"), "pgbench may be missing") @@ -506,9 +504,6 @@ def test_pgbench(self): self.assertTrue('tps' in out) def test_pg_config(self): - # set global if it wasn't set - configure_testgres(cache_pg_config=True) - # check same instances a = get_pg_config() b = get_pg_config() @@ -517,23 +512,22 @@ def test_pg_config(self): # save right before config change c1 = get_pg_config() - # modify setting - configure_testgres(cache_pg_config=False) - self.assertFalse(TestgresConfig.cache_pg_config) + # modify setting for this scope + with scoped_config(cache_pg_config=False) as config: - # save right after config change - c2 = get_pg_config() + # sanity check for value + self.assertFalse(config.cache_pg_config) - # check different instances after config change - self.assertNotEqual(id(c1), id(c2)) + # save right after config change + c2 = get_pg_config() - # check different instances - a = get_pg_config() - b = get_pg_config() - self.assertNotEqual(id(a), id(b)) + # check different instances after config change + self.assertNotEqual(id(c1), id(c2)) - # restore setting - configure_testgres(cache_pg_config=True) + # check different instances + a = get_pg_config() + b = get_pg_config() + self.assertNotEqual(id(a), id(b)) def test_unix_sockets(self): with get_new_node() as node: From dd8c5000fae3757248785a275a818a994f042673 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Tue, 20 Feb 2018 23:51:06 +0300 Subject: [PATCH 088/475] more tests for config stack --- tests/test_simple.py | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/tests/test_simple.py b/tests/test_simple.py index 98593a7e..60f437c9 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -23,7 +23,10 @@ TimeoutException from testgres import \ - scoped_config + TestgresConfig, \ + configure_testgres, \ + scoped_config, \ + pop_config from testgres import \ NodeStatus, \ @@ -529,6 +532,29 @@ def test_pg_config(self): b = get_pg_config() self.assertNotEqual(id(a), id(b)) + def test_config_stack(self): + # no such option + with self.assertRaises(TypeError): + configure_testgres(dummy=True) + + # we have only 1 config in stack + with self.assertRaises(IndexError): + pop_config() + + d0 = TestgresConfig.cached_initdb_dir + d1 = 'dummy_abc' + d2 = 'dummy_def' + + with scoped_config(cached_initdb_dir=d1) as c1: + self.assertEqual(c1.cached_initdb_dir, d1) + + with scoped_config(cached_initdb_dir=d2) as c2: + self.assertEqual(c2.cached_initdb_dir, d2) + + self.assertEqual(c1.cached_initdb_dir, d1) + + self.assertEqual(TestgresConfig.cached_initdb_dir, d0) + def test_unix_sockets(self): with get_new_node() as node: node.init(unix_sockets=False, allow_streaming=True) From 3378f28064c772ab015a0ae0a918a5873ab17c0b Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Wed, 21 Feb 2018 12:44:16 +0300 Subject: [PATCH 089/475] small fixes, improve tests --- testgres/config.py | 9 +++++---- tests/test_simple.py | 9 +++++++++ 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/testgres/config.py b/testgres/config.py index 782f315c..d192b6d5 100644 --- a/testgres/config.py +++ b/testgres/config.py @@ -98,7 +98,7 @@ def copy(self): TestgresConfig = testgres_config # stack of GlobalConfigs -config_stack = [testgres_config] +config_stack = [] def rm_cached_initdb_dirs(): @@ -123,7 +123,7 @@ def pop_config(): Set previous GlobalConfig options from stack. """ - if len(config_stack) <= 1: + if len(config_stack) == 0: raise IndexError('Reached initial config') # restore popped config @@ -141,9 +141,10 @@ def scoped_config(**options): ... print(node.execute('select 1')) """ - # set a new config with options - config = push_config(**options) try: + # set a new config with options + config = push_config(**options) + # return it yield config finally: diff --git a/tests/test_simple.py b/tests/test_simple.py index 60f437c9..6714ee32 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -549,7 +549,16 @@ def test_config_stack(self): self.assertEqual(c1.cached_initdb_dir, d1) with scoped_config(cached_initdb_dir=d2) as c2: + + stack_size = len(testgres.config.config_stack) + + # try to break a stack + with self.assertRaises(TypeError): + with scoped_config(dummy=True): + pass + self.assertEqual(c2.cached_initdb_dir, d2) + self.assertEqual(len(testgres.config.config_stack), stack_size) self.assertEqual(c1.cached_initdb_dir, d1) From e10520a939003a7a2dac36df389eedc8176a9c65 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Wed, 21 Feb 2018 16:23:33 +0300 Subject: [PATCH 090/475] Generate unique system ids in cached_initdb() --- testgres/cache.py | 19 ++++++++++++++++++- testgres/config.py | 2 ++ testgres/consts.py | 3 +++ testgres/utils.py | 21 +++++++++++++++++++++ tests/test_simple.py | 34 +++++++++++++++++++++++++++------- 5 files changed, 71 insertions(+), 8 deletions(-) diff --git a/testgres/cache.py b/testgres/cache.py index d0eaac9c..7b738411 100644 --- a/testgres/cache.py +++ b/testgres/cache.py @@ -1,5 +1,6 @@ # coding: utf-8 +import io import os import shutil @@ -7,13 +8,16 @@ from .config import testgres_config +from .consts import XLOG_CONTROL_FILE + from .exceptions import \ InitNodeException, \ ExecUtilException from .utils import \ get_bin_path, \ - execute_utility + execute_utility, \ + generate_system_id def cached_initdb(data_dir, logfile=None, params=None): @@ -42,5 +46,18 @@ def call_initdb(initdb_dir, log=None): try: # Copy cached initdb to current data dir shutil.copytree(cached_data_dir, data_dir) + + # Assign this node a unique system id if asked to + if testgres_config.cached_initdb_unique: + # XXX: write new unique system id to control file + # Some users might rely upon unique system ids, but + # our initdb caching mechanism breaks this contract. + pg_control = os.path.join(data_dir, XLOG_CONTROL_FILE) + with io.open(pg_control, "r+b") as f: + f.write(generate_system_id()) # overwrite id + + # XXX: build new WAL segment with our system id + _params = [get_bin_path("pg_resetwal"), "-D", data_dir, "-f"] + execute_utility(_params, logfile) except Exception as e: raise_from(InitNodeException("Failed to spawn a node"), e) diff --git a/testgres/config.py b/testgres/config.py index d192b6d5..4baf180f 100644 --- a/testgres/config.py +++ b/testgres/config.py @@ -15,6 +15,7 @@ class GlobalConfig(object): Attributes: cache_initdb: shall we use cached initdb instance? cached_initdb_dir: shall we create a temp dir for cached initdb? + cached_initdb_unique: shall we assign new node a unique system id? cache_pg_config: shall we cache pg_config results? @@ -30,6 +31,7 @@ class GlobalConfig(object): cache_initdb = True _cached_initdb_dir = None + cached_initdb_unique = False cache_pg_config = True diff --git a/testgres/consts.py b/testgres/consts.py index e8dd8c99..9f83b2d7 100644 --- a/testgres/consts.py +++ b/testgres/consts.py @@ -4,6 +4,9 @@ DATA_DIR = "data" LOGS_DIR = "logs" +# path to control file +XLOG_CONTROL_FILE = "global/pg_control" + # names for config files RECOVERY_CONF_FILE = "recovery.conf" PG_AUTO_CONF_FILE = "postgresql.auto.conf" diff --git a/testgres/utils.py b/testgres/utils.py index 01befbc5..2ffaf0a0 100644 --- a/testgres/utils.py +++ b/testgres/utils.py @@ -68,6 +68,27 @@ def generate_app_name(): return 'testgres-{}'.format(str(uuid.uuid4())) +def generate_system_id(): + """ + Generate a new 64-bit unique system identifier for node. + """ + + import datetime + import struct + + date = datetime.datetime.now() + secs = int(date.timestamp()) + usecs = date.microsecond + + system_id = 0 + system_id |= (secs << 32) + system_id |= (usecs << 12) + system_id |= (os.getpid() & 0xFFF) + + # pack ULL in native byte order + return struct.pack('=Q', system_id) + + def execute_utility(args, logfile=None): """ Execute utility (pg_ctl, pg_dump etc). diff --git a/tests/test_simple.py b/tests/test_simple.py index 6714ee32..c38e6808 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -38,15 +38,15 @@ get_pg_config from testgres import bound_ports +from testgres.utils import pg_version_ge def util_is_executable(util): def good_properties(f): - return ( - os.path.exists(f) and - os.path.isfile(f) and - os.access(f, os.X_OK) - ) + # yapf: disable + return (os.path.exists(f) and + os.path.isfile(f) and + os.access(f, os.X_OK)) # try to resolve it if good_properties(get_bin_path(util)): @@ -91,6 +91,27 @@ def test_init_after_cleanup(self): node.cleanup() node.init().start().execute('select 1') + @unittest.skipUnless(pg_version_ge('9.6'), 'query works on 9.6+') + def test_init_unique_system_id(self): + with scoped_config( + cache_initdb=True, cached_initdb_unique=True) as config: + + self.assertTrue(config.cache_initdb) + self.assertTrue(config.cached_initdb_unique) + + # spawn two nodes; ids must be different + with get_new_node().init().start() as node1, \ + get_new_node().init().start() as node2: + + # this function exists in PostgreSQL 9.6+ + query = 'select system_identifier from pg_control_system()' + + id1 = node1.execute(query)[0] + id2 = node2.execute(query)[0] + + # ids must increase + self.assertGreater(id2, id1) + def test_node_exit(self): base_dir = None @@ -486,8 +507,7 @@ def test_logging(self): master.restart() self.assertTrue(master._logger.is_alive()) - @unittest.skipUnless( - util_is_executable("pgbench"), "pgbench may be missing") + @unittest.skipUnless(util_is_executable('pgbench'), 'might be missing') def test_pgbench(self): with get_new_node().init().start() as node: From c696004909987b1743f341d77b8fe062abd35b9b Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Wed, 21 Feb 2018 17:52:21 +0300 Subject: [PATCH 091/475] fixes in code & tests --- testgres/cache.py | 5 +++++ testgres/utils.py | 8 +++++--- tests/test_simple.py | 5 +++-- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/testgres/cache.py b/testgres/cache.py index 7b738411..1c0d98ae 100644 --- a/testgres/cache.py +++ b/testgres/cache.py @@ -59,5 +59,10 @@ def call_initdb(initdb_dir, log=None): # XXX: build new WAL segment with our system id _params = [get_bin_path("pg_resetwal"), "-D", data_dir, "-f"] execute_utility(_params, logfile) + + except ExecUtilException as e: + msg = "Failed to reset WAL for system id" + raise_from(InitNodeException(msg), e) + except Exception as e: raise_from(InitNodeException("Failed to spawn a node"), e) diff --git a/testgres/utils.py b/testgres/utils.py index 2ffaf0a0..1a3d4f21 100644 --- a/testgres/utils.py +++ b/testgres/utils.py @@ -76,9 +76,11 @@ def generate_system_id(): import datetime import struct - date = datetime.datetime.now() - secs = int(date.timestamp()) - usecs = date.microsecond + date1 = datetime.datetime.utcfromtimestamp(0) + date2 = datetime.datetime.utcnow() + + secs = int((date2 - date1).total_seconds()) + usecs = date2.microsecond system_id = 0 system_id |= (secs << 32) diff --git a/tests/test_simple.py b/tests/test_simple.py index c38e6808..21bb9616 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -41,7 +41,7 @@ from testgres.utils import pg_version_ge -def util_is_executable(util): +def util_exists(util): def good_properties(f): # yapf: disable return (os.path.exists(f) and @@ -91,6 +91,7 @@ def test_init_after_cleanup(self): node.cleanup() node.init().start().execute('select 1') + @unittest.skipUnless(util_exists('pg_resetwal'), 'might be missing') @unittest.skipUnless(pg_version_ge('9.6'), 'query works on 9.6+') def test_init_unique_system_id(self): with scoped_config( @@ -507,7 +508,7 @@ def test_logging(self): master.restart() self.assertTrue(master._logger.is_alive()) - @unittest.skipUnless(util_is_executable('pgbench'), 'might be missing') + @unittest.skipUnless(util_exists('pgbench'), 'might be missing') def test_pgbench(self): with get_new_node().init().start() as node: From 83d122f116b28ffe62e62e7af36fdc463fd47f2e Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Thu, 22 Feb 2018 12:11:38 +0300 Subject: [PATCH 092/475] add a comment in generate_system_id() --- testgres/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/testgres/utils.py b/testgres/utils.py index 1a3d4f21..3f073e08 100644 --- a/testgres/utils.py +++ b/testgres/utils.py @@ -82,6 +82,7 @@ def generate_system_id(): secs = int((date2 - date1).total_seconds()) usecs = date2.microsecond + # see pg_resetwal.c : GuessControlValues() system_id = 0 system_id |= (secs << 32) system_id |= (usecs << 12) From c2f01569462e1493f339e016af3fbda9d65ab527 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Thu, 22 Feb 2018 12:28:52 +0300 Subject: [PATCH 093/475] make test_init_unique_system_id() more durable --- tests/test_simple.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/tests/test_simple.py b/tests/test_simple.py index 21bb9616..b907b5ec 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -94,8 +94,16 @@ def test_init_after_cleanup(self): @unittest.skipUnless(util_exists('pg_resetwal'), 'might be missing') @unittest.skipUnless(pg_version_ge('9.6'), 'query works on 9.6+') def test_init_unique_system_id(self): - with scoped_config( - cache_initdb=True, cached_initdb_unique=True) as config: + # this function exists in PostgreSQL 9.6+ + query = 'select system_identifier from pg_control_system()' + + with scoped_config(cache_initdb=False): + with get_new_node().init().start() as node0: + id0 = node0.execute(query)[0] + + # yapf: disable + with scoped_config(cache_initdb=True, + cached_initdb_unique=True) as config: self.assertTrue(config.cache_initdb) self.assertTrue(config.cached_initdb_unique) @@ -104,13 +112,11 @@ def test_init_unique_system_id(self): with get_new_node().init().start() as node1, \ get_new_node().init().start() as node2: - # this function exists in PostgreSQL 9.6+ - query = 'select system_identifier from pg_control_system()' - id1 = node1.execute(query)[0] id2 = node2.execute(query)[0] # ids must increase + self.assertGreater(id1, id0) self.assertGreater(id2, id1) def test_node_exit(self): From 04598307c8820c5da4b906ee543cabcafd0e185b Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Thu, 22 Feb 2018 15:54:50 +0300 Subject: [PATCH 094/475] add some prefixes for mkdtemp() --- testgres/backup.py | 6 ++++-- testgres/config.py | 8 ++++---- testgres/consts.py | 5 +++++ testgres/node.py | 10 +++++++--- 4 files changed, 20 insertions(+), 9 deletions(-) diff --git a/testgres/backup.py b/testgres/backup.py index 687f3f0c..9ad7e37a 100644 --- a/testgres/backup.py +++ b/testgres/backup.py @@ -8,6 +8,8 @@ from .consts import \ DATA_DIR, \ + TMP_NODE, \ + TMP_BACKUP, \ PG_CONF_FILE, \ BACKUP_LOG_FILE, \ DEFAULT_XLOG_METHOD @@ -49,7 +51,7 @@ def __init__(self, # Set default arguments username = username or default_username() - base_dir = base_dir or tempfile.mkdtemp() + base_dir = base_dir or tempfile.mkdtemp(prefix=TMP_BACKUP) # public self.original_node = node @@ -96,7 +98,7 @@ def _prepare_dir(self, destroy): available = not destroy if available: - dest_base_dir = tempfile.mkdtemp() + dest_base_dir = tempfile.mkdtemp(prefix=TMP_NODE) data1 = os.path.join(self.base_dir, DATA_DIR) data2 = os.path.join(dest_base_dir, DATA_DIR) diff --git a/testgres/config.py b/testgres/config.py index d192b6d5..c1f94d3b 100644 --- a/testgres/config.py +++ b/testgres/config.py @@ -7,6 +7,8 @@ from contextlib import contextmanager +from .consts import TMP_CACHE + class GlobalConfig(object): """ @@ -101,6 +103,7 @@ def copy(self): config_stack = [] +@atexit.register def rm_cached_initdb_dirs(): for d in cached_initdb_dirs: shutil.rmtree(d, ignore_errors=True) @@ -161,8 +164,5 @@ def configure_testgres(**options): testgres_config.update(options) -# NOTE: to be executed at exit() -atexit.register(rm_cached_initdb_dirs) - # NOTE: assign initial cached dir for initdb -testgres_config.cached_initdb_dir = tempfile.mkdtemp() +testgres_config.cached_initdb_dir = tempfile.mkdtemp(prefix=TMP_CACHE) diff --git a/testgres/consts.py b/testgres/consts.py index e8dd8c99..d20cf087 100644 --- a/testgres/consts.py +++ b/testgres/consts.py @@ -4,6 +4,11 @@ DATA_DIR = "data" LOGS_DIR = "logs" +# prefixes for temp dirs +TMP_NODE = 'tgsn_' +TMP_CACHE = 'tgsc_' +TMP_BACKUP = 'tgsb_' + # names for config files RECOVERY_CONF_FILE = "recovery.conf" PG_AUTO_CONF_FILE = "postgresql.auto.conf" diff --git a/testgres/node.py b/testgres/node.py index 2afab665..7d872515 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -23,6 +23,7 @@ from .consts import \ DATA_DIR, \ LOGS_DIR, \ + TMP_NODE, \ PG_CONF_FILE, \ PG_AUTO_CONF_FILE, \ HBA_CONF_FILE, \ @@ -106,7 +107,9 @@ def __enter__(self): def __exit__(self, type, value, traceback): self.free_port() - got_exception = value is not None + # NOTE: ctrl+C does not count! + got_exception = type is not None and type != KeyboardInterrupt + c1 = self.cleanup_on_good_exit and not got_exception c2 = self.cleanup_on_bad_exit and got_exception @@ -152,8 +155,9 @@ def _try_shutdown(self, max_attempts): except ExecUtilException: pass # one more time except Exception: - # TODO: probably kill stray instance + # TODO: probably should kill stray instance eprint('cannot stop node {}'.format(self.name)) + break attempts += 1 @@ -195,7 +199,7 @@ def _create_recovery_conf(self, username): def _prepare_dirs(self): if not self.base_dir: - self.base_dir = tempfile.mkdtemp() + self.base_dir = tempfile.mkdtemp(prefix=TMP_NODE) if not os.path.exists(self.base_dir): os.makedirs(self.base_dir) From 5bd8d48ebf3b69e0a4773c9139702727594aa58b Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Thu, 22 Feb 2018 17:38:36 +0300 Subject: [PATCH 095/475] add prefix for files produced by pg_dump --- testgres/consts.py | 1 + testgres/node.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/testgres/consts.py b/testgres/consts.py index d20cf087..30a21d9f 100644 --- a/testgres/consts.py +++ b/testgres/consts.py @@ -6,6 +6,7 @@ # prefixes for temp dirs TMP_NODE = 'tgsn_' +TMP_DUMP = 'tgsd_' TMP_CACHE = 'tgsc_' TMP_BACKUP = 'tgsb_' diff --git a/testgres/node.py b/testgres/node.py index 7d872515..809231ba 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -24,6 +24,7 @@ DATA_DIR, \ LOGS_DIR, \ TMP_NODE, \ + TMP_DUMP, \ PG_CONF_FILE, \ PG_AUTO_CONF_FILE, \ HBA_CONF_FILE, \ @@ -717,7 +718,7 @@ def dump(self, filename=None, dbname=None, username=None): """ def tmpfile(): - fd, fname = tempfile.mkstemp() + fd, fname = tempfile.mkstemp(prefix=TMP_DUMP) os.close(fd) return fname From 7153fd3a657a9ca181c40862307a40d984e4956e Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Tue, 27 Feb 2018 16:55:58 +0300 Subject: [PATCH 096/475] add 'temp_dir' setting to GlobalConfig --- testgres/backup.py | 11 ++++++++--- testgres/config.py | 10 ++++++++-- testgres/node.py | 7 ++++--- 3 files changed, 20 insertions(+), 8 deletions(-) diff --git a/testgres/backup.py b/testgres/backup.py index 9ad7e37a..fc9a1731 100644 --- a/testgres/backup.py +++ b/testgres/backup.py @@ -2,9 +2,11 @@ import os import shutil -import tempfile from six import raise_from +from tempfile import mkdtemp + +from .config import testgres_config from .consts import \ DATA_DIR, \ @@ -49,9 +51,11 @@ def __init__(self, if not node.status(): raise BackupException('Node must be running') + # yapf: disable # Set default arguments username = username or default_username() - base_dir = base_dir or tempfile.mkdtemp(prefix=TMP_BACKUP) + base_dir = base_dir or mkdtemp(prefix=TMP_BACKUP, + dir=testgres_config.temp_dir) # public self.original_node = node @@ -98,7 +102,8 @@ def _prepare_dir(self, destroy): available = not destroy if available: - dest_base_dir = tempfile.mkdtemp(prefix=TMP_NODE) + dest_base_dir = mkdtemp(prefix=TMP_NODE, + dir=testgres_config.temp_dir) data1 = os.path.join(self.base_dir, DATA_DIR) data2 = os.path.join(dest_base_dir, DATA_DIR) diff --git a/testgres/config.py b/testgres/config.py index 1b417a3e..a4fc1381 100644 --- a/testgres/config.py +++ b/testgres/config.py @@ -3,9 +3,9 @@ import atexit import copy import shutil -import tempfile from contextlib import contextmanager +from tempfile import mkdtemp from .consts import TMP_CACHE @@ -21,6 +21,8 @@ class GlobalConfig(object): cache_pg_config: shall we cache pg_config results? + temp_dir: base temp dir for nodes with default 'base_dir'. + use_python_logging: use python logging configuration for all nodes. error_log_lines: N of log lines to be shown in exception (0=inf). @@ -37,6 +39,8 @@ class GlobalConfig(object): cache_pg_config = True + temp_dir = None + use_python_logging = False error_log_lines = 20 @@ -166,5 +170,7 @@ def configure_testgres(**options): testgres_config.update(options) +# yapf: disable # NOTE: assign initial cached dir for initdb -testgres_config.cached_initdb_dir = tempfile.mkdtemp(prefix=TMP_CACHE) +testgres_config.cached_initdb_dir = mkdtemp(prefix=TMP_CACHE, + dir=testgres_config.temp_dir) diff --git a/testgres/node.py b/testgres/node.py index 809231ba..453a932f 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -5,11 +5,11 @@ import shutil import six import subprocess -import tempfile import time from enum import Enum from six import raise_from +from tempfile import mkstemp, mkdtemp from .cache import cached_initdb @@ -200,7 +200,8 @@ def _create_recovery_conf(self, username): def _prepare_dirs(self): if not self.base_dir: - self.base_dir = tempfile.mkdtemp(prefix=TMP_NODE) + self.base_dir = mkdtemp(prefix=TMP_NODE, + dir=testgres_config.temp_dir) if not os.path.exists(self.base_dir): os.makedirs(self.base_dir) @@ -718,7 +719,7 @@ def dump(self, filename=None, dbname=None, username=None): """ def tmpfile(): - fd, fname = tempfile.mkstemp(prefix=TMP_DUMP) + fd, fname = mkstemp(prefix=TMP_DUMP, dir=testgres_config.temp_dir) os.close(fd) return fname From f800bfcadfc8e05e12223ee4c560f08fb6d4a675 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Tue, 27 Feb 2018 18:01:03 +0300 Subject: [PATCH 097/475] move some code to dedicated modules --- testgres/backup.py | 3 +- testgres/cache.py | 5 +- testgres/connection.py | 7 ++- testgres/decorators.py | 74 ++++++++++++++++++++++++ testgres/defaults.py | 50 +++++++++++++++++ testgres/node.py | 16 ++++-- testgres/utils.py | 125 +---------------------------------------- 7 files changed, 144 insertions(+), 136 deletions(-) create mode 100644 testgres/decorators.py create mode 100644 testgres/defaults.py diff --git a/testgres/backup.py b/testgres/backup.py index fc9a1731..8cff97de 100644 --- a/testgres/backup.py +++ b/testgres/backup.py @@ -16,11 +16,12 @@ BACKUP_LOG_FILE, \ DEFAULT_XLOG_METHOD +from .defaults import default_username + from .exceptions import BackupException from .utils import \ get_bin_path, \ - default_username, \ execute_utility diff --git a/testgres/cache.py b/testgres/cache.py index 1c0d98ae..f784e9a1 100644 --- a/testgres/cache.py +++ b/testgres/cache.py @@ -10,14 +10,15 @@ from .consts import XLOG_CONTROL_FILE +from .defaults import generate_system_id + from .exceptions import \ InitNodeException, \ ExecUtilException from .utils import \ get_bin_path, \ - execute_utility, \ - generate_system_id + execute_utility def cached_initdb(data_dir, logfile=None, params=None): diff --git a/testgres/connection.py b/testgres/connection.py index e86ff06d..0f7a92af 100644 --- a/testgres/connection.py +++ b/testgres/connection.py @@ -9,13 +9,14 @@ except ImportError: raise ImportError("You must have psycopg2 or pg8000 modules installed") +from .defaults import \ + default_dbname, \ + default_username + from enum import Enum from .exceptions import QueryException -from .utils import \ - default_dbname, \ - default_username # export these exceptions InternalError = pglib.InternalError diff --git a/testgres/decorators.py b/testgres/decorators.py new file mode 100644 index 00000000..2a57b83d --- /dev/null +++ b/testgres/decorators.py @@ -0,0 +1,74 @@ +import six +import functools + + +def positional_args_hack(*special_cases): + """ + Convert positional args described by + 'special_cases' into named args. + + Example: + @positional_args_hack(['abc'], ['def', 'abc']) + def some_api_func(...) + + This is useful for compatibility. + """ + + cases = dict() + + for case in special_cases: + k = len(case) + assert k not in six.iterkeys(cases), 'len must be unique' + cases[k] = case + + def decorator(function): + @functools.wraps(function) + def wrapper(*args, **kwargs): + k = len(args) + + if k in six.iterkeys(cases): + case = cases[k] + + for i in range(0, k): + arg_name = case[i] + arg_val = args[i] + + # transform into named + kwargs[arg_name] = arg_val + + # get rid of them + args = [] + + return function(*args, **kwargs) + + return wrapper + + return decorator + + +def method_decorator(decorator): + """ + Convert a function decorator into a method decorator. + """ + + def _dec(func): + def _wrapper(self, *args, **kwargs): + @decorator + def bound_func(*args2, **kwargs2): + return func.__get__(self, type(self))(*args2, **kwargs2) + + # 'bound_func' is a closure and can see 'self' + return bound_func(*args, **kwargs) + + # preserve docs + functools.update_wrapper(_wrapper, func) + + return _wrapper + + # preserve docs + functools.update_wrapper(_dec, decorator) + + # change name for easier debugging + _dec.__name__ = 'method_decorator({})'.format(decorator.__name__) + + return _dec diff --git a/testgres/defaults.py b/testgres/defaults.py new file mode 100644 index 00000000..8d5b892e --- /dev/null +++ b/testgres/defaults.py @@ -0,0 +1,50 @@ +import datetime +import getpass +import os +import struct +import uuid + + +def default_dbname(): + """ + Return default DB name. + """ + + return 'postgres' + + +def default_username(): + """ + Return default username (current user). + """ + + return getpass.getuser() + + +def generate_app_name(): + """ + Generate a new application name for node. + """ + + return 'testgres-{}'.format(str(uuid.uuid4())) + + +def generate_system_id(): + """ + Generate a new 64-bit unique system identifier for node. + """ + + date1 = datetime.datetime.utcfromtimestamp(0) + date2 = datetime.datetime.utcnow() + + secs = int((date2 - date1).total_seconds()) + usecs = date2.microsecond + + # see pg_resetwal.c : GuessControlValues() + system_id = 0 + system_id |= (secs << 32) + system_id |= (usecs << 12) + system_id |= (os.getpid() & 0xFFF) + + # pack ULL in native byte order + return struct.pack('=Q', system_id) diff --git a/testgres/node.py b/testgres/node.py index 453a932f..ca64e918 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -33,6 +33,15 @@ UTILS_LOG_FILE, \ DEFAULT_XLOG_METHOD +from .decorators import \ + method_decorator, \ + positional_args_hack + +from .defaults import \ + default_dbname, \ + default_username, \ + generate_app_name + from .exceptions import \ CatchUpException, \ ExecUtilException, \ @@ -49,12 +58,7 @@ pg_version_ge, \ reserve_port, \ release_port, \ - default_dbname, \ - default_username, \ - generate_app_name, \ - execute_utility, \ - method_decorator, \ - positional_args_hack + execute_utility class NodeStatus(Enum): diff --git a/testgres/utils.py b/testgres/utils.py index 3f073e08..3bedd51d 100644 --- a/testgres/utils.py +++ b/testgres/utils.py @@ -3,11 +3,9 @@ from __future__ import division from __future__ import print_function -import functools import io import os import port_for -import six import subprocess import sys @@ -16,6 +14,7 @@ from .config import testgres_config from .exceptions import ExecUtilException + # rows returned by PG_CONFIG _pg_config_data = {} @@ -42,56 +41,6 @@ def release_port(port): bound_ports.remove(port) -def default_dbname(): - """ - Return default DB name. - """ - - return 'postgres' - - -def default_username(): - """ - Return default username (current user). - """ - - import getpass - return getpass.getuser() - - -def generate_app_name(): - """ - Generate a new application name for node. - """ - - import uuid - return 'testgres-{}'.format(str(uuid.uuid4())) - - -def generate_system_id(): - """ - Generate a new 64-bit unique system identifier for node. - """ - - import datetime - import struct - - date1 = datetime.datetime.utcfromtimestamp(0) - date2 = datetime.datetime.utcnow() - - secs = int((date2 - date1).total_seconds()) - usecs = date2.microsecond - - # see pg_resetwal.c : GuessControlValues() - system_id = 0 - system_id |= (secs << 32) - system_id |= (usecs << 12) - system_id |= (os.getpid() & 0xFFF) - - # pack ULL in native byte order - return struct.pack('=Q', system_id) - - def execute_utility(args, logfile=None): """ Execute utility (pg_ctl, pg_dump etc). @@ -268,77 +217,5 @@ def file_tail(f, num_lines): buffers = int(buffers * max(2, num_lines / max(cur_lines, 1))) -def positional_args_hack(*special_cases): - """ - Convert positional args described by - 'special_cases' into named args. - - Example: - @positional_args_hack(['abc'], ['def', 'abc']) - def some_api_func(...) - - This is useful for compatibility. - """ - - cases = dict() - - for case in special_cases: - k = len(case) - assert k not in six.iterkeys(cases), 'len must be unique' - cases[k] = case - - def decorator(function): - @functools.wraps(function) - def wrapper(*args, **kwargs): - k = len(args) - - if k in six.iterkeys(cases): - case = cases[k] - - for i in range(0, k): - arg_name = case[i] - arg_val = args[i] - - # transform into named - kwargs[arg_name] = arg_val - - # get rid of them - args = [] - - return function(*args, **kwargs) - - return wrapper - - return decorator - - -def method_decorator(decorator): - """ - Convert a function decorator into a method decorator. - """ - - def _dec(func): - def _wrapper(self, *args, **kwargs): - @decorator - def bound_func(*args2, **kwargs2): - return func.__get__(self, type(self))(*args2, **kwargs2) - - # 'bound_func' is a closure and can see 'self' - return bound_func(*args, **kwargs) - - # preserve docs - functools.update_wrapper(_wrapper, func) - - return _wrapper - - # preserve docs - functools.update_wrapper(_dec, decorator) - - # change name for easier debugging - _dec.__name__ = 'method_decorator({})'.format(decorator.__name__) - - return _dec - - def eprint(*args, **kwargs): print(*args, file=sys.stderr, **kwargs) From 8e5b09d66ca3fd5bad0c380aea5086e44f71f59c Mon Sep 17 00:00:00 2001 From: Ildus Kurbangaliev Date: Wed, 28 Feb 2018 14:17:16 +0300 Subject: [PATCH 098/475] Use kwargs for several functions to move through all possible arguments --- testgres/node.py | 29 ++++++++++------------------- 1 file changed, 10 insertions(+), 19 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index ca64e918..ce5070d8 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -112,7 +112,7 @@ def __enter__(self): def __exit__(self, type, value, traceback): self.free_port() - # NOTE: ctrl+C does not count! + # NOTE: Ctrl+C does not count! got_exception = type is not None and type != KeyboardInterrupt c1 = self.cleanup_on_good_exit and not got_exception @@ -681,16 +681,13 @@ def psql(self, return process.returncode, out, err @method_decorator(positional_args_hack(['dbname', 'query'])) - def safe_psql(self, - query, - dbname=None, - username=None, - input=None): + def safe_psql(self, query=None, **kwargs): """ Execute a query using psql. Args: query: query to be executed. + filename: file with a query. dbname: database name to connect to. username: database user name. input: raw input to be passed. @@ -699,10 +696,7 @@ def safe_psql(self, psql's output as str. """ - ret, out, err = self.psql(query=query, - dbname=dbname, - username=username, - input=input) + ret, out, err = self.psql(query=query, **kwargs) if ret: raise QueryException((err or b'').decode('utf-8'), query) @@ -858,27 +852,23 @@ def execute(self, return res - def backup(self, username=None, xlog_method=DEFAULT_XLOG_METHOD): + def backup(self, **kwargs): """ Perform pg_basebackup. Args: username: database user name. xlog_method: a method for collecting the logs ('fetch' | 'stream'). + base_dir: the base directory for data files and logs Returns: A smart object of type NodeBackup. """ from .backup import NodeBackup - return NodeBackup(node=self, - username=username, - xlog_method=xlog_method) + return NodeBackup(node=self, **kwargs) - def replicate(self, - name=None, - username=None, - xlog_method=DEFAULT_XLOG_METHOD): + def replicate(self, name=None, **kwargs): """ Create a binary replica of this node. @@ -886,9 +876,10 @@ def replicate(self, name: replica's application name. username: database user name. xlog_method: a method for collecting the logs ('fetch' | 'stream'). + base_dir: the base directory for data files and logs """ - backup = self.backup(username=username, xlog_method=xlog_method) + backup = self.backup(**kwargs) # transform backup into a replica return backup.spawn_replica(name=name, destroy=True) From 2693dac1ca486c6335c7b447f7372574fd3c291e Mon Sep 17 00:00:00 2001 From: Ildus Kurbangaliev Date: Wed, 28 Feb 2018 15:04:19 +0300 Subject: [PATCH 099/475] Use kwargs for PostgresNode.init --- testgres/node.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index ce5070d8..61c2fdcc 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -254,19 +254,15 @@ def _collect_special_files(self): return result - def init(self, - fsync=False, - unix_sockets=True, - allow_streaming=True, - initdb_params=None): + def init(self, initdb_params=None, **kwargs): """ Perform initdb for this node. Args: + initdb_params: parameters for initdb (list). fsync: should this node use fsync to keep data safe? unix_sockets: should we enable UNIX sockets? allow_streaming: should this node add a hba entry for replication? - initdb_params: parameters for initdb (list). Returns: This instance of PostgresNode. @@ -281,9 +277,7 @@ def init(self, params=initdb_params) # initialize default config files - self.default_conf(fsync=fsync, - unix_sockets=unix_sockets, - allow_streaming=allow_streaming) + self.default_conf(**kwargs) return self From 9b06dc2578beeeaec958ef3dd5fd58ea9f7f3767 Mon Sep 17 00:00:00 2001 From: Ildus Kurbangaliev Date: Wed, 28 Feb 2018 15:06:38 +0300 Subject: [PATCH 100/475] Remove unused constant from imports import node.py --- testgres/node.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index 61c2fdcc..ba848c2c 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -30,8 +30,7 @@ HBA_CONF_FILE, \ RECOVERY_CONF_FILE, \ PG_LOG_FILE, \ - UTILS_LOG_FILE, \ - DEFAULT_XLOG_METHOD + UTILS_LOG_FILE from .decorators import \ method_decorator, \ From 04e74abc6c1744b3be20b06976789cbb86f206f9 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Wed, 28 Feb 2018 15:10:38 +0300 Subject: [PATCH 101/475] allow single arg in append_conf() --- testgres/node.py | 13 +++++++------ tests/test_simple.py | 2 +- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index ca64e918..2a1d401c 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -384,21 +384,22 @@ def get_auth_method(t): return self - def append_conf(self, filename, string): + @method_decorator(positional_args_hack(['filename', 'line'])) + def append_conf(self, line, filename=PG_CONF_FILE): """ - Append line to a config file (i.e. postgresql.conf). + Append line to a config file. Args: - filename: name of the config file. - string: string to be appended to config. + line: string to be appended to config. + filename: config file (postgresql.conf by default). Returns: This instance of PostgresNode. """ config_name = os.path.join(self.data_dir, filename) - with io.open(config_name, "a") as conf: - conf.write(u"".join([string, '\n'])) + with io.open(config_name, 'a') as conf: + conf.write(u''.join([line, '\n'])) return self diff --git a/tests/test_simple.py b/tests/test_simple.py index b907b5ec..ee05dea1 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -171,7 +171,7 @@ def test_reload(self): # change client_min_messages and save old value cmm_old = node.execute('show client_min_messages') - node.append_conf('postgresql.conf', 'client_min_messages = DEBUG1') + node.append_conf('client_min_messages = DEBUG1') # reload config node.reload() From e1b5a510428806bf3a844f668ecd020b901d654b Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Thu, 1 Mar 2018 18:33:15 +0300 Subject: [PATCH 102/475] minor improvements --- testgres/utils.py | 2 +- tests/test_simple.py | 31 ++++++++++++++++++------------- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/testgres/utils.py b/testgres/utils.py index 3bedd51d..2ef7fa49 100644 --- a/testgres/utils.py +++ b/testgres/utils.py @@ -38,7 +38,7 @@ def release_port(port): Free port provided by reserve_port(). """ - bound_ports.remove(port) + bound_ports.discard(port) def execute_utility(args, logfile=None): diff --git a/tests/test_simple.py b/tests/test_simple.py index ee05dea1..70749c32 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -11,6 +11,7 @@ import logging.config +from contextlib import contextmanager from distutils.version import LooseVersion from testgres import \ @@ -58,6 +59,15 @@ def good_properties(f): return True +@contextmanager +def removing(f): + try: + yield f + finally: + if os.path.isfile(f): + os.remove(f) + + class SimpleTest(unittest.TestCase): def test_custom_init(self): with get_new_node() as node: @@ -382,19 +392,14 @@ def test_dump(self): node1.execute(query_create) # take a new dump - dump = node1.dump() - self.assertTrue(os.path.isfile(dump)) - - with get_new_node().init().start() as node2: - - # restore dump - node2.restore(filename=dump) - - res = node2.execute(query_select) - self.assertListEqual(res, [(1, ), (2, )]) - - # finally, remove dump - os.remove(dump) + with removing(node1.dump()) as dump: + with get_new_node().init().start() as node2: + # restore dump + self.assertTrue(os.path.isfile(dump)) + node2.restore(filename=dump) + + res = node2.execute(query_select) + self.assertListEqual(res, [(1, ), (2, )]) def test_users(self): with get_new_node().init().start() as node: From be1e3c599785cc4f955213629fa2c1d9db9638b1 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Thu, 1 Mar 2018 18:47:05 +0300 Subject: [PATCH 103/475] small refactoring --- testgres/backup.py | 20 +++++++------------- testgres/cache.py | 4 ++-- testgres/config.py | 21 +++++++++++++-------- testgres/node.py | 9 ++++----- tests/test_simple.py | 4 ++-- 5 files changed, 28 insertions(+), 30 deletions(-) diff --git a/testgres/backup.py b/testgres/backup.py index 8cff97de..b87b7759 100644 --- a/testgres/backup.py +++ b/testgres/backup.py @@ -1,13 +1,11 @@ # coding: utf-8 import os -import shutil +from shutil import rmtree, copytree from six import raise_from from tempfile import mkdtemp -from .config import testgres_config - from .consts import \ DATA_DIR, \ TMP_NODE, \ @@ -55,8 +53,7 @@ def __init__(self, # yapf: disable # Set default arguments username = username or default_username() - base_dir = base_dir or mkdtemp(prefix=TMP_BACKUP, - dir=testgres_config.temp_dir) + base_dir = base_dir or mkdtemp(prefix=TMP_BACKUP) # public self.original_node = node @@ -103,15 +100,14 @@ def _prepare_dir(self, destroy): available = not destroy if available: - dest_base_dir = mkdtemp(prefix=TMP_NODE, - dir=testgres_config.temp_dir) + dest_base_dir = mkdtemp(prefix=TMP_NODE) data1 = os.path.join(self.base_dir, DATA_DIR) data2 = os.path.join(dest_base_dir, DATA_DIR) try: # Copy backup to new data dir - shutil.copytree(data1, data2) + copytree(data1, data2) except Exception as e: raise_from(BackupException('Failed to copy files'), e) else: @@ -140,8 +136,7 @@ def spawn_primary(self, name=None, destroy=True): # Build a new PostgresNode from .node import PostgresNode - node = PostgresNode(name=name, - base_dir=base_dir) + node = PostgresNode(name=name, base_dir=base_dir) # New nodes should always remove dir tree node._should_rm_dirs = True @@ -164,8 +159,7 @@ def spawn_replica(self, name=None, destroy=True): """ # Build a new PostgresNode - node = self.spawn_primary(name=name, - destroy=destroy) + node = self.spawn_primary(name=name, destroy=destroy) # Assign it a master and a recovery file (private magic) node._assign_master(self.original_node) @@ -175,5 +169,5 @@ def spawn_replica(self, name=None, destroy=True): def cleanup(self): if self._available: - shutil.rmtree(self.base_dir, ignore_errors=True) + rmtree(self.base_dir, ignore_errors=True) self._available = False diff --git a/testgres/cache.py b/testgres/cache.py index f784e9a1..36d6e768 100644 --- a/testgres/cache.py +++ b/testgres/cache.py @@ -2,8 +2,8 @@ import io import os -import shutil +from shutil import copytree from six import raise_from from .config import testgres_config @@ -46,7 +46,7 @@ def call_initdb(initdb_dir, log=None): try: # Copy cached initdb to current data dir - shutil.copytree(cached_data_dir, data_dir) + copytree(cached_data_dir, data_dir) # Assign this node a unique system id if asked to if testgres_config.cached_initdb_unique: diff --git a/testgres/config.py b/testgres/config.py index a4fc1381..976263f7 100644 --- a/testgres/config.py +++ b/testgres/config.py @@ -2,9 +2,10 @@ import atexit import copy -import shutil +import tempfile from contextlib import contextmanager +from shutil import rmtree from tempfile import mkdtemp from .consts import TMP_CACHE @@ -39,8 +40,6 @@ class GlobalConfig(object): cache_pg_config = True - temp_dir = None - use_python_logging = False error_log_lines = 20 @@ -59,6 +58,14 @@ def cached_initdb_dir(self, value): if value: cached_initdb_dirs.add(value) + @property + def temp_dir(self): + return tempfile.tempdir + + @temp_dir.setter + def temp_dir(self, value): + tempfile.tempdir = value + def __init__(self, **options): self.update(options) @@ -110,9 +117,9 @@ def copy(self): @atexit.register -def rm_cached_initdb_dirs(): +def _rm_cached_initdb_dirs(): for d in cached_initdb_dirs: - shutil.rmtree(d, ignore_errors=True) + rmtree(d, ignore_errors=True) def push_config(**options): @@ -170,7 +177,5 @@ def configure_testgres(**options): testgres_config.update(options) -# yapf: disable # NOTE: assign initial cached dir for initdb -testgres_config.cached_initdb_dir = mkdtemp(prefix=TMP_CACHE, - dir=testgres_config.temp_dir) +testgres_config.cached_initdb_dir = mkdtemp(prefix=TMP_CACHE) diff --git a/testgres/node.py b/testgres/node.py index 853eb625..10c11ebc 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -2,12 +2,12 @@ import io import os -import shutil import six import subprocess import time from enum import Enum +from shutil import rmtree from six import raise_from from tempfile import mkstemp, mkdtemp @@ -203,8 +203,7 @@ def _create_recovery_conf(self, username): def _prepare_dirs(self): if not self.base_dir: - self.base_dir = mkdtemp(prefix=TMP_NODE, - dir=testgres_config.temp_dir) + self.base_dir = mkdtemp(prefix=TMP_NODE) if not os.path.exists(self.base_dir): os.makedirs(self.base_dir) @@ -613,7 +612,7 @@ def cleanup(self, max_attempts=3): else: rm_dir = self.data_dir # just data, save logs - shutil.rmtree(rm_dir, ignore_errors=True) + rmtree(rm_dir, ignore_errors=True) return self @@ -711,7 +710,7 @@ def dump(self, filename=None, dbname=None, username=None): """ def tmpfile(): - fd, fname = mkstemp(prefix=TMP_DUMP, dir=testgres_config.temp_dir) + fd, fname = mkstemp(prefix=TMP_DUMP) os.close(fd) return fname diff --git a/tests/test_simple.py b/tests/test_simple.py index 70749c32..555fdc73 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -2,7 +2,6 @@ # coding: utf-8 import os -import shutil import subprocess import tempfile import testgres @@ -13,6 +12,7 @@ from contextlib import contextmanager from distutils.version import LooseVersion +from shutil import rmtree from testgres import \ InitNodeException, \ @@ -139,7 +139,7 @@ def test_node_exit(self): # we should save the DB for "debugging" self.assertTrue(os.path.exists(base_dir)) - shutil.rmtree(base_dir, ignore_errors=True) + rmtree(base_dir, ignore_errors=True) with get_new_node().init() as node: base_dir = node.base_dir From cf9313b125a9cdf44c63ab7edc581ae179350644 Mon Sep 17 00:00:00 2001 From: Valeria112 Date: Fri, 2 Mar 2018 00:17:43 +0300 Subject: [PATCH 104/475] reorganize enums --- testgres/__init__.py | 4 ++-- testgres/backup.py | 17 ++++------------- testgres/connection.py | 13 +------------ testgres/consts.py | 5 +++++ testgres/enums.py | 37 +++++++++++++++++++++++++++++++++++++ testgres/node.py | 29 +++++++---------------------- 6 files changed, 56 insertions(+), 49 deletions(-) create mode 100644 testgres/enums.py diff --git a/testgres/__init__.py b/testgres/__init__.py index 29b017a9..3fb7ae4e 100644 --- a/testgres/__init__.py +++ b/testgres/__init__.py @@ -3,13 +3,13 @@ from .config import TestgresConfig, configure_testgres from .connection import \ - IsolationLevel, \ NodeConnection, \ InternalError, \ ProgrammingError from .exceptions import * -from .node import NodeStatus, PostgresNode +from .enums import * +from .node import PostgresNode from .utils import \ reserve_port, \ diff --git a/testgres/backup.py b/testgres/backup.py index 46cd435e..5aaf123f 100644 --- a/testgres/backup.py +++ b/testgres/backup.py @@ -5,12 +5,13 @@ import tempfile from six import raise_from -from enum import Enum +from .enums import XLogMethod from .consts import \ DATA_DIR, \ PG_CONF_FILE, \ - BACKUP_LOG_FILE + BACKUP_LOG_FILE, \ + DEFAULT_XLOG_METHOD from .exceptions import BackupException @@ -20,16 +21,6 @@ execute_utility -class XLogMethod(Enum): - """ - Available WAL methods for NodeBackup - """ - - none = 'none' - fetch = 'fetch' - stream = 'stream' - - class NodeBackup(object): """ Smart object responsible for backups @@ -43,7 +34,7 @@ def __init__(self, node, base_dir=None, username=None, - xlog_method=XLogMethod.fetch): + xlog_method=DEFAULT_XLOG_METHOD): """ Create a new backup. diff --git a/testgres/connection.py b/testgres/connection.py index a20b651a..650a3573 100644 --- a/testgres/connection.py +++ b/testgres/connection.py @@ -9,7 +9,7 @@ except ImportError: raise ImportError("You must have psycopg2 or pg8000 modules installed") -from enum import Enum +from .enums import IsolationLevel from .exceptions import QueryException @@ -22,17 +22,6 @@ ProgrammingError = pglib.ProgrammingError -class IsolationLevel(Enum): - """ - Transaction isolation level for NodeConnection - """ - - ReadUncommitted = 'read uncommitted' - ReadCommitted = 'read committed' - RepeatableRead = 'repeatable read' - Serializable = 'serializable' - - class NodeConnection(object): """ Transaction wrapper returned by Node diff --git a/testgres/consts.py b/testgres/consts.py index 36b42722..e64997f8 100644 --- a/testgres/consts.py +++ b/testgres/consts.py @@ -1,5 +1,7 @@ # coding: utf-8 +from .enums import XLogMethod + # names for dirs in base_dir DATA_DIR = "data" LOGS_DIR = "logs" @@ -12,3 +14,6 @@ PG_LOG_FILE = "postgresql.log" UTILS_LOG_FILE = "utils.log" BACKUP_LOG_FILE = "backup.log" + +# default argument value +DEFAULT_XLOG_METHOD = XLogMethod.fetch diff --git a/testgres/enums.py b/testgres/enums.py new file mode 100644 index 00000000..abc849be --- /dev/null +++ b/testgres/enums.py @@ -0,0 +1,37 @@ +from enum import Enum, IntEnum + + +class XLogMethod(Enum): + """ + Available WAL methods for NodeBackup + """ + + none = 'none' + fetch = 'fetch' + stream = 'stream' + + +class IsolationLevel(Enum): + """ + Transaction isolation level for NodeConnection + """ + + ReadUncommitted = 'read uncommitted' + ReadCommitted = 'read committed' + RepeatableRead = 'repeatable read' + Serializable = 'serializable' + + +class NodeStatus(IntEnum): + """ + Status of a PostgresNode + """ + + Running, Stopped, Uninitialized = range(3) + + # for Python 3.x + def __bool__(self): + return self == NodeStatus.Running + + # for Python 2.x + __nonzero__ = __bool__ diff --git a/testgres/node.py b/testgres/node.py index 048927eb..36b3ff3d 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -8,9 +8,10 @@ import tempfile import time -from enum import IntEnum from six import raise_from +from .enums import NodeStatus + from .cache import cached_initdb from .config import TestgresConfig @@ -27,7 +28,8 @@ HBA_CONF_FILE, \ RECOVERY_CONF_FILE, \ PG_LOG_FILE, \ - UTILS_LOG_FILE + UTILS_LOG_FILE, \ + DEFAULT_XLOG_METHOD from .exceptions import \ CatchUpException, \ @@ -51,24 +53,7 @@ method_decorator, \ positional_args_hack -from .backup import \ - XLogMethod, \ - NodeBackup - - -class NodeStatus(IntEnum): - """ - Status of a PostgresNode - """ - - Running, Stopped, Uninitialized = range(3) - - # for Python 3.x - def __bool__(self): - return self == NodeStatus.Running - - # for Python 2.x - __nonzero__ = __bool__ +from .backup import NodeBackup class PostgresNode(object): @@ -841,7 +826,7 @@ def execute(self, return res - def backup(self, username=None, xlog_method=XLogMethod.fetch): + def backup(self, username=None, xlog_method=DEFAULT_XLOG_METHOD): """ Perform pg_basebackup. @@ -860,7 +845,7 @@ def backup(self, username=None, xlog_method=XLogMethod.fetch): def replicate(self, name=None, username=None, - xlog_method=XLogMethod.fetch, + xlog_method=DEFAULT_XLOG_METHOD, use_logging=False): """ Create a binary replica of this node. From ec7605ec6bd4f3f3ed6e4467c7cc85423b6beed6 Mon Sep 17 00:00:00 2001 From: Valeria112 Date: Fri, 2 Mar 2018 01:34:33 +0300 Subject: [PATCH 105/475] add backup testcase --- tests/test_simple.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/test_simple.py b/tests/test_simple.py index 555fdc73..ab2a5804 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -360,6 +360,13 @@ def test_backup_exhaust(self): with self.assertRaises(BackupException): backup.spawn_primary() + def test_backup_wrong_xlog_method(self): + with get_new_node() as node: + node.init(allow_streaming=True).start() + + with self.assertRaises(BackupException, msg='Invalid xlog_method "wrong"'): + node.backup(xlog_method='wrong') + def test_replicate(self): with get_new_node() as node: node.init(allow_streaming=True).start() From bc285516a4f56cc5b4434f37b890c23e25ec9965 Mon Sep 17 00:00:00 2001 From: Valeria112 Date: Sat, 3 Mar 2018 22:03:29 +0300 Subject: [PATCH 106/475] README.md checkout --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index a2726bc1..3839981e 100644 --- a/README.md +++ b/README.md @@ -159,7 +159,7 @@ ext_conf = "shared_preload_libraries = 'postgres_fdw'" with testgres.get_new_node().init() as master: # ... do something ... - + # reset main config file master.default_conf(fsync=True, allow_streaming=True) @@ -174,7 +174,7 @@ the configuration file, which means that they should be called before `append_co ## Authors -[Ildar Musin](https://github.com/zilder) Postgres Professional Ltd., Russia -[Dmitry Ivanov](https://github.com/funbringer) Postgres Professional Ltd., Russia -[Ildus Kurbangaliev](https://github.com/ildus) Postgres Professional Ltd., Russia -[Yury Zhuravlev](https://github.com/stalkerg) +[Ildar Musin](https://github.com/zilder) Postgres Professional Ltd., Russia +[Dmitry Ivanov](https://github.com/funbringer) Postgres Professional Ltd., Russia +[Ildus Kurbangaliev](https://github.com/ildus) Postgres Professional Ltd., Russia +[Yury Zhuravlev](https://github.com/stalkerg) From d9473c646d3c7a9d67595ebdcef333b7917fe784 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Mon, 5 Mar 2018 12:41:26 +0300 Subject: [PATCH 107/475] get rid of DEFAULT_XLOG_METHOD (because kwargs) --- testgres/backup.py | 5 ++--- testgres/consts.py | 5 ----- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/testgres/backup.py b/testgres/backup.py index fe220536..1f853721 100644 --- a/testgres/backup.py +++ b/testgres/backup.py @@ -13,8 +13,7 @@ TMP_NODE, \ TMP_BACKUP, \ PG_CONF_FILE, \ - BACKUP_LOG_FILE, \ - DEFAULT_XLOG_METHOD + BACKUP_LOG_FILE from .defaults import default_username @@ -38,7 +37,7 @@ def __init__(self, node, base_dir=None, username=None, - xlog_method=DEFAULT_XLOG_METHOD): + xlog_method=XLogMethod.fetch): """ Create a new backup. diff --git a/testgres/consts.py b/testgres/consts.py index d995b243..1e719703 100644 --- a/testgres/consts.py +++ b/testgres/consts.py @@ -1,7 +1,5 @@ # coding: utf-8 -from .enums import XLogMethod - # names for dirs in base_dir DATA_DIR = "data" LOGS_DIR = "logs" @@ -25,6 +23,3 @@ PG_LOG_FILE = "postgresql.log" UTILS_LOG_FILE = "utils.log" BACKUP_LOG_FILE = "backup.log" - -# default argument value -DEFAULT_XLOG_METHOD = XLogMethod.fetch From 4241471d0738992f8c7e0e33493cbb2e764a0b85 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Mon, 5 Mar 2018 13:06:12 +0300 Subject: [PATCH 108/475] reformat code --- testgres/backup.py | 3 ++- testgres/connection.py | 1 - testgres/utils.py | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/testgres/backup.py b/testgres/backup.py index 1f853721..64f021ed 100644 --- a/testgres/backup.py +++ b/testgres/backup.py @@ -56,7 +56,8 @@ def __init__(self, try: xlog_method = XLogMethod(xlog_method) except ValueError: - raise BackupException('Invalid xlog_method "{}"'.format(xlog_method)) + raise BackupException( + 'Invalid xlog_method "{}"'.format(xlog_method)) # Set default arguments username = username or default_username() diff --git a/testgres/connection.py b/testgres/connection.py index dbbfa0f7..272d6db7 100644 --- a/testgres/connection.py +++ b/testgres/connection.py @@ -17,7 +17,6 @@ from .exceptions import QueryException - # export these exceptions InternalError = pglib.InternalError ProgrammingError = pglib.ProgrammingError diff --git a/testgres/utils.py b/testgres/utils.py index 2ef7fa49..a4108b23 100644 --- a/testgres/utils.py +++ b/testgres/utils.py @@ -14,7 +14,6 @@ from .config import testgres_config from .exceptions import ExecUtilException - # rows returned by PG_CONFIG _pg_config_data = {} From c75424d48458cb81a5e3ed832dc5197f843dd166 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Mon, 5 Mar 2018 15:56:45 +0300 Subject: [PATCH 109/475] replace function _prepare_dirs() with smart properties --- testgres/api.py | 18 +++------ testgres/consts.py | 1 + testgres/node.py | 96 +++++++++++++++++++++++++--------------------- 3 files changed, 58 insertions(+), 57 deletions(-) diff --git a/testgres/api.py b/testgres/api.py index fd4f1bf2..a697c1fc 100644 --- a/testgres/api.py +++ b/testgres/api.py @@ -33,19 +33,11 @@ Copyright (c) 2016, Postgres Professional """ -from .node import PostgresNode - +from functools import wraps -def get_new_node(name=None, base_dir=None): - """ - Create a new node (select port automatically). - - Args: - name: node's application name. - base_dir: path to node's data directory. +from .node import PostgresNode - Returns: - An instance of PostgresNode. - """ - return PostgresNode(name=name, base_dir=base_dir) +@wraps(PostgresNode.__init__) +def get_new_node(**kwargs): + return PostgresNode(**kwargs) diff --git a/testgres/consts.py b/testgres/consts.py index 1e719703..15400311 100644 --- a/testgres/consts.py +++ b/testgres/consts.py @@ -17,6 +17,7 @@ RECOVERY_CONF_FILE = "recovery.conf" PG_AUTO_CONF_FILE = "postgresql.auto.conf" PG_CONF_FILE = "postgresql.conf" +PG_PID_FILE = 'postmaster.pid' HBA_CONF_FILE = "pg_hba.conf" # names for log files diff --git a/testgres/node.py b/testgres/node.py index b4d6c8a9..5022f89f 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -31,7 +31,8 @@ HBA_CONF_FILE, \ RECOVERY_CONF_FILE, \ PG_LOG_FILE, \ - UTILS_LOG_FILE + UTILS_LOG_FILE, \ + PG_PID_FILE from .decorators import \ method_decorator, \ @@ -66,7 +67,7 @@ class PostgresNode(object): def __init__(self, name=None, port=None, base_dir=None): """ - Create a new node manually. + Create a new node. Args: name: node's application name. @@ -74,24 +75,25 @@ def __init__(self, name=None, port=None, base_dir=None): base_dir: path to node's data directory. """ + # private + self._should_free_port = port is None + self._base_dir = base_dir + self._logger = None + self._master = None + # basic self.host = '127.0.0.1' self.name = name or generate_app_name() self.port = port or reserve_port() - self.base_dir = base_dir # defaults for __exit__() self.cleanup_on_good_exit = testgres_config.node_cleanup_on_good_exit self.cleanup_on_bad_exit = testgres_config.node_cleanup_on_bad_exit self.shutdown_max_attempts = 3 - # private - self._should_free_port = port is None - self._logger = None - self._master = None - - # create directories if needed - self._prepare_dirs() + # NOTE: for compatibility + self.utils_log_name = self.utils_log_file + self.pg_log_name = self.pg_log_file def __enter__(self): return self @@ -121,19 +123,37 @@ def master(self): return self._master @property - def data_dir(self): - return os.path.join(self.base_dir, DATA_DIR) + def base_dir(self): + if not self._base_dir: + self._base_dir = mkdtemp(prefix=TMP_NODE) + + # NOTE :it's safe to create a new dir + if not os.path.exists(self._base_dir): + os.makedirs(self._base_dir) + + return self._base_dir @property def logs_dir(self): - return os.path.join(self.base_dir, LOGS_DIR) + path = os.path.join(self.base_dir, LOGS_DIR) + + # NOTE: it's safe to create a new dir + if not os.path.exists(path): + os.makedirs(path) + + return path + + @property + def data_dir(self): + # NOTE: we can't run initdb without user's args + return os.path.join(self.base_dir, DATA_DIR) @property - def utils_log_name(self): + def utils_log_file(self): return os.path.join(self.logs_dir, UTILS_LOG_FILE) @property - def pg_log_name(self): + def pg_log_file(self): return os.path.join(self.logs_dir, PG_LOG_FILE) def _try_shutdown(self, max_attempts): @@ -189,21 +209,11 @@ def _create_recovery_conf(self, username): self.append_conf(RECOVERY_CONF_FILE, line) - def _prepare_dirs(self): - if not self.base_dir: - self.base_dir = mkdtemp(prefix=TMP_NODE) - - if not os.path.exists(self.base_dir): - os.makedirs(self.base_dir) - - if not os.path.exists(self.logs_dir): - os.makedirs(self.logs_dir) - def _maybe_start_logger(self): if testgres_config.use_python_logging: # spawn new logger if it doesn't exist or is stopped if not self._logger or not self._logger.is_alive(): - self._logger = TestgresLogger(self.name, self.pg_log_name) + self._logger = TestgresLogger(self.name, self.pg_log_file) self._logger.start() def _maybe_stop_logger(self): @@ -219,7 +229,7 @@ def _collect_special_files(self): (os.path.join(self.data_dir, PG_AUTO_CONF_FILE), 0), (os.path.join(self.data_dir, RECOVERY_CONF_FILE), 0), (os.path.join(self.data_dir, HBA_CONF_FILE), 0), - (self.pg_log_name, testgres_config.error_log_lines) + (self.pg_log_file, testgres_config.error_log_lines) ] for f, num_lines in files: @@ -254,12 +264,9 @@ def init(self, initdb_params=None, **kwargs): This instance of PostgresNode. """ - # create directories if needed - self._prepare_dirs() - # initialize this PostgreSQL node cached_initdb(data_dir=self.data_dir, - logfile=self.utils_log_name, + logfile=self.utils_log_file, params=initdb_params) # initialize default config files @@ -398,7 +405,7 @@ def status(self): "-D", self.data_dir, "status" ] - execute_utility(_params, self.utils_log_name) + execute_utility(_params, self.utils_log_file) return NodeStatus.Running except ExecUtilException as e: @@ -416,7 +423,7 @@ def get_pid(self): """ if self.status(): - pid_file = os.path.join(self.data_dir, 'postmaster.pid') + pid_file = os.path.join(self.data_dir, PG_PID_FILE) with io.open(pid_file) as f: return int(f.readline()) @@ -433,7 +440,7 @@ def get_control_data(self): _params += ["-D"] if pg_version_ge('9.5') else [] _params += [self.data_dir] - data = execute_utility(_params, self.utils_log_name) + data = execute_utility(_params, self.utils_log_file) out_dict = {} @@ -458,13 +465,13 @@ def start(self, params=[]): _params = [ get_bin_path("pg_ctl"), "-D", self.data_dir, - "-l", self.pg_log_name, + "-l", self.pg_log_file, "-w", # wait "start" ] + params try: - execute_utility(_params, self.utils_log_name) + execute_utility(_params, self.utils_log_file) except ExecUtilException as e: msg = 'Cannot start node' files = self._collect_special_files() @@ -493,7 +500,7 @@ def stop(self, params=[]): "stop" ] + params - execute_utility(_params, self.utils_log_name) + execute_utility(_params, self.utils_log_file) self._maybe_stop_logger() @@ -514,13 +521,13 @@ def restart(self, params=[]): _params = [ get_bin_path("pg_ctl"), "-D", self.data_dir, - "-l", self.pg_log_name, + "-l", self.pg_log_file, "-w", # wait "restart" ] + params try: - execute_utility(_params, self.utils_log_name) + execute_utility(_params, self.utils_log_file) except ExecUtilException as e: msg = 'Cannot restart node' files = self._collect_special_files() @@ -549,7 +556,7 @@ def reload(self, params=[]): "reload" ] + params - execute_utility(_params, self.utils_log_name) + execute_utility(_params, self.utils_log_file) def pg_ctl(self, params): """ @@ -569,7 +576,7 @@ def pg_ctl(self, params): "-w" # wait ] + params - return execute_utility(_params, self.utils_log_name) + return execute_utility(_params, self.utils_log_file) def free_port(self): """ @@ -578,6 +585,7 @@ def free_port(self): """ if self._should_free_port: + self._should_free_port = False release_port(self.port) def cleanup(self, max_attempts=3): @@ -717,7 +725,7 @@ def tmpfile(): "-d", dbname ] - execute_utility(_params, self.utils_log_name) + execute_utility(_params, self.utils_log_file) return filename @@ -953,7 +961,7 @@ def pgbench_run(self, **kwargs): """ Run pgbench with some options. - This event is logged (see self.utils_log_name). + This event is logged (see self.utils_log_file). Args: dbname: database name to connect to. @@ -996,7 +1004,7 @@ def pgbench_run(self, # should be the last one _params.append(dbname) - return execute_utility(_params, self.utils_log_name) + return execute_utility(_params, self.utils_log_file) def connect(self, dbname=None, username=None, password=None): """ From ec94f99c71f8e12e68127b8d4ac71e23b8183edf Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Mon, 5 Mar 2018 16:10:19 +0300 Subject: [PATCH 110/475] fix typo --- testgres/node.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testgres/node.py b/testgres/node.py index 5022f89f..02cc0f52 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -127,7 +127,7 @@ def base_dir(self): if not self._base_dir: self._base_dir = mkdtemp(prefix=TMP_NODE) - # NOTE :it's safe to create a new dir + # NOTE: it's safe to create a new dir if not os.path.exists(self._base_dir): os.makedirs(self._base_dir) From 5bc608e2cd6f0662e4489b56ffe1e322221626a2 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Tue, 6 Mar 2018 22:28:00 +0300 Subject: [PATCH 111/475] compatibility fixes --- testgres/api.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/testgres/api.py b/testgres/api.py index a697c1fc..6bc276d5 100644 --- a/testgres/api.py +++ b/testgres/api.py @@ -39,5 +39,6 @@ @wraps(PostgresNode.__init__) -def get_new_node(**kwargs): - return PostgresNode(**kwargs) +def get_new_node(name=None, base_dir=None, **kwargs): + # NOTE: leave explicit 'name' and 'base_dir' for compatibility + return PostgresNode(name=name, base_dir=base_dir, **kwargs) From e20ed5e87ce6716d200a7932c0a967db386f85a7 Mon Sep 17 00:00:00 2001 From: Ildus Kurbangaliev Date: Wed, 7 Mar 2018 16:06:08 +0300 Subject: [PATCH 112/475] Add functions that return PIDs of various processes --- testgres/connection.py | 7 ++++ testgres/node.py | 85 ++++++++++++++++++++++++++++++++++++++++-- tests/test_simple.py | 57 +++++++++++++++++++++++++--- 3 files changed, 140 insertions(+), 9 deletions(-) diff --git a/testgres/connection.py b/testgres/connection.py index 272d6db7..4985fb8c 100644 --- a/testgres/connection.py +++ b/testgres/connection.py @@ -34,6 +34,7 @@ def __init__(self, node, dbname=None, username=None, password=None): username = username or default_username() self._node = node + self._backend_pid = None self._connection = pglib.connect( database=dbname, @@ -52,6 +53,12 @@ def node(self): def connection(self): return self._connection + @property + def backend_pid(self): + if self._backend_pid is None: + self._backend_pid = self.execute("select pg_backend_pid();")[0][0] + return self._backend_pid + @property def cursor(self): return self._cursor diff --git a/testgres/node.py b/testgres/node.py index 02cc0f52..d9b04f39 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -5,6 +5,12 @@ import six import subprocess import time +import warnings + +try: + import psutil +except ImportError: + psutil = None from shutil import rmtree from six import raise_from @@ -48,7 +54,8 @@ ExecUtilException, \ QueryException, \ StartNodeException, \ - TimeoutException + TimeoutException, \ + TestgresException from .logger import TestgresLogger @@ -116,7 +123,11 @@ def __exit__(self, type, value, traceback): @property def pid(self): - return self.get_pid() + return self.get_main_pid() + + @property + def auxiliary_pids(self): + return self.get_auxiliary_pids() @property def master(self): @@ -417,7 +428,7 @@ def status(self): elif e.exit_code == 4: return NodeStatus.Uninitialized - def get_pid(self): + def get_main_pid(self): """ Return postmaster's PID if node is running, else 0. """ @@ -428,7 +439,73 @@ def get_pid(self): return int(f.readline()) # for clarity - return 0 + return None + + def get_child_processes(self): + ''' Returns child processes for this node ''' + + if psutil is None: + warnings.warn("psutil module is not installed") + return None + + try: + postmaster = psutil.Process(self.pid) + except psutil.NoSuchProcess: + return None + + return postmaster.children(recursive=True) + + def get_auxiliary_pids(self): + ''' Returns dict with pids of auxiliary processes ''' + + children = self.get_child_processes() + if children is None: + return None + + result = {} + for child in children: + line = child.cmdline()[0] + if line.startswith('postgres: checkpointer'): + result['checkpointer'] = child.pid + elif line.startswith('postgres: background writer'): + result['bgwriter'] = child.pid + elif line.startswith('postgres: walwriter'): + result['walwriter'] = child.pid + elif line.startswith('postgres: autovacuum launcher'): + result['autovacuum_launcher'] = child.pid + elif line.startswith('postgres: stats collector'): + result['stats'] = child.pid + elif line.startswith('postgres: logical replication launcher'): + result['logical_replication_launcher'] = child.pid + elif line.startswith('postgres: walreceiver'): + result['walreceiver'] = child.pid + elif line.startswith('postgres: walsender'): + result.setdefault('walsenders', []) + result['walsenders'].append(child.pid) + elif line.startswith('postgres: startup'): + result['startup'] = child.pid + + return result + + def get_walsender_pid(self): + ''' Returns pid of according walsender for replica ''' + + if not self._master: + raise TestgresException("This node is not a replica") + + children = self._master.get_child_processes() + if children is None: + return None + + sql = 'select application_name, client_port from pg_stat_replication' + for name, client_port in self._master.execute(sql): + if name == self.name: + for child in children: + line = child.cmdline()[0] + if line.startswith('postgres: walsender') and str(client_port) in line: + return child.pid + + return None def get_control_data(self): """ diff --git a/tests/test_simple.py b/tests/test_simple.py index ab2a5804..89fac492 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -205,27 +205,27 @@ def test_status(self): # check statuses after each operation with get_new_node() as node: - self.assertEqual(node.pid, 0) + self.assertIsNone(node.pid) self.assertEqual(node.status(), NodeStatus.Uninitialized) node.init() - self.assertEqual(node.pid, 0) + self.assertIsNone(node.pid) self.assertEqual(node.status(), NodeStatus.Stopped) node.start() - self.assertNotEqual(node.pid, 0) + self.assertIsNotNone(node.pid) self.assertEqual(node.status(), NodeStatus.Running) node.stop() - self.assertEqual(node.pid, 0) + self.assertIsNone(node.pid) self.assertEqual(node.status(), NodeStatus.Stopped) node.cleanup() - self.assertEqual(node.pid, 0) + self.assertIsNone(node.pid) self.assertEqual(node.status(), NodeStatus.Uninitialized) def test_psql(self): @@ -702,6 +702,53 @@ def test_version_management(self): self.assertTrue(b > c) self.assertTrue(a > c) + def test_pids(self): + try: + import psutil + except ImportError: + psutil = None + + master_processes = ( + 'checkpointer', + 'bgwriter', + 'walwriter', + 'autovacuum_launcher', + 'stats', + 'logical_replication_launcher', + 'walsenders', + ) + repl_processes = ( + 'startup', + 'checkpointer', + 'bgwriter', + 'stats', + 'walreceiver', + ) + + with get_new_node('master') as master: + master.init().start() + + self.assertIsNotNone(master.pid) + with master.connect() as con: + self.assertTrue(con.backend_pid > 0) + + with master.backup() as backup: + with backup.spawn_replica('repl', True) as repl: + repl.start() + if psutil is None: + self.assertIsNone(master.auxiliary_pids) + self.assertIsNone(repl.auxiliary_pids) + else: + master_pids = master.auxiliary_pids + for name in master_processes: + self.assertTrue(name in master_pids) + self.assertTrue(len(master_pids['walsenders']) == 1) + + repl_pids = repl.auxiliary_pids + for name in repl_processes: + self.assertTrue(name in repl_pids) + self.assertTrue(repl.get_walsender_pid() == master_pids['walsenders'][0]) + if __name__ == '__main__': unittest.main() From 61b5d446c336ab0cb0707c1c455e223e4342663f Mon Sep 17 00:00:00 2001 From: Ildus Kurbangaliev Date: Wed, 7 Mar 2018 17:06:17 +0300 Subject: [PATCH 113/475] Add psutil installation in travis tests --- run_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run_tests.sh b/run_tests.sh index 93851b3c..25fb504b 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -22,7 +22,7 @@ export VIRTUAL_ENV_DISABLE_PROMPT=1 source $VENV_PATH/bin/activate # install utilities -$PIP install coverage flake8 +$PIP install coverage flake8 psutil # install testgres' dependencies export PYTHONPATH=$(pwd) From a0ebde3b4847f9e3a8181a36fc7d68d35aeb4e0c Mon Sep 17 00:00:00 2001 From: Ildus Kurbangaliev Date: Wed, 7 Mar 2018 17:27:29 +0300 Subject: [PATCH 114/475] Return 0 when no pid, do the same for backend pid, and add packages to Dockerfile template --- Dockerfile.tmpl | 4 ++-- testgres/connection.py | 4 ++-- testgres/node.py | 2 +- tests/test_simple.py | 10 +++++----- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Dockerfile.tmpl b/Dockerfile.tmpl index 9d417a31..de767cf4 100644 --- a/Dockerfile.tmpl +++ b/Dockerfile.tmpl @@ -2,10 +2,10 @@ FROM postgres:${PG_VERSION}-alpine ENV PYTHON=python${PYTHON_VERSION} RUN if [ "${PYTHON_VERSION}" = "2" ] ; then \ - apk add --no-cache curl python2 py-virtualenv py-pip; \ + apk add --no-cache curl python2 python2-dev gcc py-virtualenv py-pip; \ fi RUN if [ "${PYTHON_VERSION}" = "3" ] ; then \ - apk add --no-cache curl python3 py-virtualenv; \ + apk add --no-cache curl python3 python3-dev gcc py-virtualenv; \ fi ENV LANG=C.UTF-8 diff --git a/testgres/connection.py b/testgres/connection.py index 4985fb8c..5bf45b1b 100644 --- a/testgres/connection.py +++ b/testgres/connection.py @@ -34,7 +34,7 @@ def __init__(self, node, dbname=None, username=None, password=None): username = username or default_username() self._node = node - self._backend_pid = None + self._backend_pid = 0 self._connection = pglib.connect( database=dbname, @@ -55,7 +55,7 @@ def connection(self): @property def backend_pid(self): - if self._backend_pid is None: + if not self._backend_pid: self._backend_pid = self.execute("select pg_backend_pid();")[0][0] return self._backend_pid diff --git a/testgres/node.py b/testgres/node.py index d9b04f39..846bb284 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -439,7 +439,7 @@ def get_main_pid(self): return int(f.readline()) # for clarity - return None + return 0 def get_child_processes(self): ''' Returns child processes for this node ''' diff --git a/tests/test_simple.py b/tests/test_simple.py index 89fac492..d1803c3b 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -205,27 +205,27 @@ def test_status(self): # check statuses after each operation with get_new_node() as node: - self.assertIsNone(node.pid) + self.assertEqual(node.pid, 0) self.assertEqual(node.status(), NodeStatus.Uninitialized) node.init() - self.assertIsNone(node.pid) + self.assertEqual(node.pid, 0) self.assertEqual(node.status(), NodeStatus.Stopped) node.start() - self.assertIsNotNone(node.pid) + self.assertNotEqual(node.pid, 0) self.assertEqual(node.status(), NodeStatus.Running) node.stop() - self.assertIsNone(node.pid) + self.assertEqual(node.pid, 0) self.assertEqual(node.status(), NodeStatus.Stopped) node.cleanup() - self.assertIsNone(node.pid) + self.assertEqual(node.pid, 0) self.assertEqual(node.status(), NodeStatus.Uninitialized) def test_psql(self): From 23790b24ae033e4a2920b80a5d1906ffa7b68dd6 Mon Sep 17 00:00:00 2001 From: Ildus Kurbangaliev Date: Wed, 7 Mar 2018 17:35:07 +0300 Subject: [PATCH 115/475] Add build-base package to Dockerfile --- Dockerfile.tmpl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile.tmpl b/Dockerfile.tmpl index de767cf4..647f3704 100644 --- a/Dockerfile.tmpl +++ b/Dockerfile.tmpl @@ -2,10 +2,10 @@ FROM postgres:${PG_VERSION}-alpine ENV PYTHON=python${PYTHON_VERSION} RUN if [ "${PYTHON_VERSION}" = "2" ] ; then \ - apk add --no-cache curl python2 python2-dev gcc py-virtualenv py-pip; \ + apk add --no-cache curl python2 python2-dev build-base py-virtualenv py-pip; \ fi RUN if [ "${PYTHON_VERSION}" = "3" ] ; then \ - apk add --no-cache curl python3 python3-dev gcc py-virtualenv; \ + apk add --no-cache curl python3 python3-dev build-base py-virtualenv; \ fi ENV LANG=C.UTF-8 From 3f489724322499443e3bbc8d1277b3f56bbd4e3f Mon Sep 17 00:00:00 2001 From: Ildus Kurbangaliev Date: Wed, 7 Mar 2018 18:00:37 +0300 Subject: [PATCH 116/475] Use enum for process types --- Dockerfile.tmpl | 4 ++-- testgres/connection.py | 10 +++++----- testgres/enums.py | 15 +++++++++++++++ testgres/node.py | 28 ++++++++-------------------- tests/test_simple.py | 41 ++++++++++++++++++++++------------------- 5 files changed, 52 insertions(+), 46 deletions(-) diff --git a/Dockerfile.tmpl b/Dockerfile.tmpl index 647f3704..005b5d90 100644 --- a/Dockerfile.tmpl +++ b/Dockerfile.tmpl @@ -2,10 +2,10 @@ FROM postgres:${PG_VERSION}-alpine ENV PYTHON=python${PYTHON_VERSION} RUN if [ "${PYTHON_VERSION}" = "2" ] ; then \ - apk add --no-cache curl python2 python2-dev build-base py-virtualenv py-pip; \ + apk add --no-cache curl python2 python2-dev build-base musl-dev py-virtualenv py-pip; \ fi RUN if [ "${PYTHON_VERSION}" = "3" ] ; then \ - apk add --no-cache curl python3 python3-dev build-base py-virtualenv; \ + apk add --no-cache curl python3 python3-dev build-base musl-dev py-virtualenv; \ fi ENV LANG=C.UTF-8 diff --git a/testgres/connection.py b/testgres/connection.py index 5bf45b1b..eb8505d0 100644 --- a/testgres/connection.py +++ b/testgres/connection.py @@ -34,7 +34,7 @@ def __init__(self, node, dbname=None, username=None, password=None): username = username or default_username() self._node = node - self._backend_pid = 0 + self._pid = 0 self._connection = pglib.connect( database=dbname, @@ -54,10 +54,10 @@ def connection(self): return self._connection @property - def backend_pid(self): - if not self._backend_pid: - self._backend_pid = self.execute("select pg_backend_pid();")[0][0] - return self._backend_pid + def pid(self): + if not self._pid: + self._pid = self.execute("select pg_backend_pid();")[0][0] + return self._pid @property def cursor(self): diff --git a/testgres/enums.py b/testgres/enums.py index abc849be..d1db8541 100644 --- a/testgres/enums.py +++ b/testgres/enums.py @@ -35,3 +35,18 @@ def __bool__(self): # for Python 2.x __nonzero__ = __bool__ + + +class ProcessType(Enum): + """ + Types of postgres processes + """ + Checkpointer = 'postgres: checkpointer' + BackgroundWriter = 'postgres: background writer' + WalWriter = 'postgres: walwriter' + AutovacuumLauncher = 'postgres: autovacuum launcher' + StatsCollector = 'postgres: stats collector' + LogicalReplicationLauncher = 'postgres: logical replication launcher' + WalReceiver = 'postgres: walreceiver' + WalSender = 'postgres: walsender' + Startup = 'postgres: startup' diff --git a/testgres/node.py b/testgres/node.py index 846bb284..b228ff4e 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -16,7 +16,7 @@ from six import raise_from from tempfile import mkstemp, mkdtemp -from .enums import NodeStatus +from .enums import NodeStatus, ProcessType from .cache import cached_initdb @@ -465,25 +465,13 @@ def get_auxiliary_pids(self): result = {} for child in children: line = child.cmdline()[0] - if line.startswith('postgres: checkpointer'): - result['checkpointer'] = child.pid - elif line.startswith('postgres: background writer'): - result['bgwriter'] = child.pid - elif line.startswith('postgres: walwriter'): - result['walwriter'] = child.pid - elif line.startswith('postgres: autovacuum launcher'): - result['autovacuum_launcher'] = child.pid - elif line.startswith('postgres: stats collector'): - result['stats'] = child.pid - elif line.startswith('postgres: logical replication launcher'): - result['logical_replication_launcher'] = child.pid - elif line.startswith('postgres: walreceiver'): - result['walreceiver'] = child.pid - elif line.startswith('postgres: walsender'): - result.setdefault('walsenders', []) - result['walsenders'].append(child.pid) - elif line.startswith('postgres: startup'): - result['startup'] = child.pid + for ptype in ProcessType: + if ptype == ProcessType.WalSender \ + and line.startswith(ptype.value): + result.setdefault(ptype, []) + result[ptype].append(child.pid) + elif line.startswith(ptype.value): + result[ptype] = child.pid return result diff --git a/tests/test_simple.py b/tests/test_simple.py index d1803c3b..33bd80ac 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -40,6 +40,7 @@ from testgres import bound_ports from testgres.utils import pg_version_ge +from testgres.enums import ProcessType def util_exists(util): @@ -709,20 +710,20 @@ def test_pids(self): psutil = None master_processes = ( - 'checkpointer', - 'bgwriter', - 'walwriter', - 'autovacuum_launcher', - 'stats', - 'logical_replication_launcher', - 'walsenders', + ProcessType.Checkpointer, + ProcessType.BackgroundWriter, + ProcessType.WalWriter, + ProcessType.AutovacuumLauncher, + ProcessType.StatsCollector, + ProcessType.LogicalReplicationLauncher, + ProcessType.WalSender, ) repl_processes = ( - 'startup', - 'checkpointer', - 'bgwriter', - 'stats', - 'walreceiver', + ProcessType.Startup, + ProcessType.Checkpointer, + ProcessType.BackgroundWriter, + ProcessType.StatsCollector, + ProcessType.WalReceiver, ) with get_new_node('master') as master: @@ -730,7 +731,7 @@ def test_pids(self): self.assertIsNotNone(master.pid) with master.connect() as con: - self.assertTrue(con.backend_pid > 0) + self.assertTrue(con.pid > 0) with master.backup() as backup: with backup.spawn_replica('repl', True) as repl: @@ -740,14 +741,16 @@ def test_pids(self): self.assertIsNone(repl.auxiliary_pids) else: master_pids = master.auxiliary_pids - for name in master_processes: - self.assertTrue(name in master_pids) - self.assertTrue(len(master_pids['walsenders']) == 1) + for ptype in master_processes: + self.assertIn(ptype, master_pids) + self.assertTrue(len(master_pids[ProcessType.WalSender]) == 1) repl_pids = repl.auxiliary_pids - for name in repl_processes: - self.assertTrue(name in repl_pids) - self.assertTrue(repl.get_walsender_pid() == master_pids['walsenders'][0]) + for ptype in repl_processes: + self.assertIn(ptype, repl_pids) + + sender_pid = master_pids[ProcessType.WalSender][0] + self.assertTrue(repl.get_walsender_pid() == sender_pid) if __name__ == '__main__': From 3a24554a2bcf669153b769f2b445bd81a43f92d6 Mon Sep 17 00:00:00 2001 From: Ildus Kurbangaliev Date: Wed, 7 Mar 2018 18:03:45 +0300 Subject: [PATCH 117/475] Add linux-headers to Dockerfile --- Dockerfile.tmpl | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Dockerfile.tmpl b/Dockerfile.tmpl index 005b5d90..dc5878b6 100644 --- a/Dockerfile.tmpl +++ b/Dockerfile.tmpl @@ -2,10 +2,12 @@ FROM postgres:${PG_VERSION}-alpine ENV PYTHON=python${PYTHON_VERSION} RUN if [ "${PYTHON_VERSION}" = "2" ] ; then \ - apk add --no-cache curl python2 python2-dev build-base musl-dev py-virtualenv py-pip; \ + apk add --no-cache curl python2 python2-dev build-base musl-dev \ + linux-headers py-virtualenv py-pip; \ fi RUN if [ "${PYTHON_VERSION}" = "3" ] ; then \ - apk add --no-cache curl python3 python3-dev build-base musl-dev py-virtualenv; \ + apk add --no-cache curl python3 python3-dev build-base musl-dev \ + linux-headers py-virtualenv; \ fi ENV LANG=C.UTF-8 From 1bd538f51f86b916f020bdc296bc52142872da5c Mon Sep 17 00:00:00 2001 From: Ildus Kurbangaliev Date: Wed, 7 Mar 2018 18:25:07 +0300 Subject: [PATCH 118/475] Add older names support for pids --- testgres/node.py | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/testgres/node.py b/testgres/node.py index b228ff4e..769fc564 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -458,6 +458,21 @@ def get_child_processes(self): def get_auxiliary_pids(self): ''' Returns dict with pids of auxiliary processes ''' + alternative_names = { + ProcessType.LogicalReplicationLauncher: [ + 'postgres: bgworker: logical replication launcher' + ], + ProcessType.BackgroundWriter: [ + 'postgres: writer', + ], + ProcessType.WalWriter: [ + 'postgres: wal writer', + ], + ProcessType.WalReceiver: [ + 'postgres: wal receiver', + ], + } + children = self.get_child_processes() if children is None: return None @@ -467,11 +482,20 @@ def get_auxiliary_pids(self): line = child.cmdline()[0] for ptype in ProcessType: if ptype == ProcessType.WalSender \ - and line.startswith(ptype.value): + and (line.startswith(ptype.value) or + line.startswith('postgres: wal sender')): result.setdefault(ptype, []) result[ptype].append(child.pid) + break elif line.startswith(ptype.value): result[ptype] = child.pid + break + elif ptype in alternative_names: + names = alternative_names[ptype] + for name in names: + if line.startswith(name): + result[ptype] = child.pid + break return result From 152faf313687855a8487e0dd14fa7376be60efa0 Mon Sep 17 00:00:00 2001 From: Ildus Kurbangaliev Date: Mon, 12 Mar 2018 13:13:03 +0300 Subject: [PATCH 119/475] Add instance names for 9.6 and 10 --- testgres/node.py | 8 +++++--- tests/test_simple.py | 8 +++++--- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index 769fc564..0bf318ee 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -479,7 +479,7 @@ def get_auxiliary_pids(self): result = {} for child in children: - line = child.cmdline()[0] + line = ' '.join(child.cmdline()) for ptype in ProcessType: if ptype == ProcessType.WalSender \ and (line.startswith(ptype.value) or @@ -513,8 +513,10 @@ def get_walsender_pid(self): for name, client_port in self._master.execute(sql): if name == self.name: for child in children: - line = child.cmdline()[0] - if line.startswith('postgres: walsender') and str(client_port) in line: + line = ' '.join(child.cmdline()) + if (line.startswith(ProcessType.WalSender.value) or + line.startswith('postgres: wal sender')) and \ + str(client_port) in line: return child.pid return None diff --git a/tests/test_simple.py b/tests/test_simple.py index 33bd80ac..52a1fab0 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -709,15 +709,17 @@ def test_pids(self): except ImportError: psutil = None - master_processes = ( + master_processes = [ ProcessType.Checkpointer, ProcessType.BackgroundWriter, ProcessType.WalWriter, ProcessType.AutovacuumLauncher, ProcessType.StatsCollector, - ProcessType.LogicalReplicationLauncher, ProcessType.WalSender, - ) + ] + if pg_version_ge('10'): + master_processes.append(ProcessType.LogicalReplicationLauncher) + repl_processes = ( ProcessType.Startup, ProcessType.Checkpointer, From 8f1cdd244a94d902d538e2e781e8962cfe8bb6c8 Mon Sep 17 00:00:00 2001 From: Ildus Kurbangaliev Date: Mon, 12 Mar 2018 13:31:11 +0300 Subject: [PATCH 120/475] Test when psutil is not installed --- run_tests.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/run_tests.sh b/run_tests.sh index 25fb504b..9e17fed8 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -47,7 +47,8 @@ time coverage run -a tests/test_simple.py unset PG_BIN -# run tests (PG_CONFIG) +# run tests (PG_CONFIG), also without psutil +$PIP uninstall psutil export PG_CONFIG=$(which pg_config) time coverage run -a tests/test_simple.py unset PG_CONFIG From f6d1e5b5f8025205569811b35b811d375a0e4549 Mon Sep 17 00:00:00 2001 From: Ildus Kurbangaliev Date: Mon, 12 Mar 2018 13:42:31 +0300 Subject: [PATCH 121/475] Fix pytest uninstallation in tests --- run_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run_tests.sh b/run_tests.sh index 9e17fed8..53bfdc99 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -48,7 +48,7 @@ unset PG_BIN # run tests (PG_CONFIG), also without psutil -$PIP uninstall psutil +$PIP uninstall -y psutil export PG_CONFIG=$(which pg_config) time coverage run -a tests/test_simple.py unset PG_CONFIG From 390ed7cd0deba0f3a695a92937b2a6d06536f1f4 Mon Sep 17 00:00:00 2001 From: Ildus Kurbangaliev Date: Mon, 12 Mar 2018 13:59:16 +0300 Subject: [PATCH 122/475] Raise exception for functions that require psutil when psutil is not installed --- testgres/node.py | 3 +-- tests/test_simple.py | 9 ++++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index 0bf318ee..d8588844 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -445,8 +445,7 @@ def get_child_processes(self): ''' Returns child processes for this node ''' if psutil is None: - warnings.warn("psutil module is not installed") - return None + raise TestgresException("psutil module is not installed") try: postmaster = psutil.Process(self.pid) diff --git a/tests/test_simple.py b/tests/test_simple.py index 52a1fab0..e445a216 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -21,7 +21,8 @@ BackupException, \ QueryException, \ CatchUpException, \ - TimeoutException + TimeoutException, \ + TestgresException from testgres import \ TestgresConfig, \ @@ -739,8 +740,10 @@ def test_pids(self): with backup.spawn_replica('repl', True) as repl: repl.start() if psutil is None: - self.assertIsNone(master.auxiliary_pids) - self.assertIsNone(repl.auxiliary_pids) + with self.assertRaises(TestgresException): + master.auxiliary_pids + with self.assertRaises(TestgresException): + self.assertIsNone(repl.auxiliary_pids) else: master_pids = master.auxiliary_pids for ptype in master_processes: From 91a56e835c264d653cd3d22570926b2c7082a1a1 Mon Sep 17 00:00:00 2001 From: Ildus Kurbangaliev Date: Mon, 12 Mar 2018 14:01:58 +0300 Subject: [PATCH 123/475] Remove warnings module import for node.py --- testgres/node.py | 1 - 1 file changed, 1 deletion(-) diff --git a/testgres/node.py b/testgres/node.py index d8588844..0702ea68 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -5,7 +5,6 @@ import six import subprocess import time -import warnings try: import psutil From 0dda2ffe56278d67f11988623d0e9eb83daca73a Mon Sep 17 00:00:00 2001 From: Ildar Musin Date: Tue, 13 Mar 2018 14:50:06 +0300 Subject: [PATCH 124/475] Replication slots --- testgres/backup.py | 2 +- testgres/node.py | 26 +++++++++++++++++++++++--- tests/test_simple.py | 12 ++++++++++++ 3 files changed, 36 insertions(+), 4 deletions(-) diff --git a/testgres/backup.py b/testgres/backup.py index 64f021ed..1226d135 100644 --- a/testgres/backup.py +++ b/testgres/backup.py @@ -154,7 +154,7 @@ def spawn_primary(self, name=None, destroy=True): return node - def spawn_replica(self, name=None, destroy=True): + def spawn_replica(self, name=None, destroy=True, slot_name=None): """ Create a replica of the original node from a backup. diff --git a/testgres/node.py b/testgres/node.py index 02cc0f52..8974d493 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -179,7 +179,7 @@ def _assign_master(self, master): # now this node has a master self._master = master - def _create_recovery_conf(self, username): + def _create_recovery_conf(self, username, slot_name=None): """NOTE: this is a private method!""" # fetch master of this node @@ -207,6 +207,9 @@ def _create_recovery_conf(self, username): "standby_mode=on\n" ).format(conninfo) + if slot_name: + line += "primary_slot_name={}".format() + self.append_conf(RECOVERY_CONF_FILE, line) def _maybe_start_logger(self): @@ -856,7 +859,22 @@ def backup(self, **kwargs): return NodeBackup(node=self, **kwargs) - def replicate(self, name=None, **kwargs): + def create_replication_slot(self, slot_name, dbname=None, username=None): + """ + Create a physical replication slot. + + Args: + slot_name: slot name + dbname: database name + username: database user name + """ + query = "select pg_create_physical_replication_slot('{}')".format(slot_name) + + self.execute(query=query, + dbname=dbname or default_dbname(), + username=username or default_username()) + + def replicate(self, name=None, slot_name=None, **kwargs): """ Create a binary replica of this node. @@ -870,7 +888,9 @@ def replicate(self, name=None, **kwargs): backup = self.backup(**kwargs) # transform backup into a replica - return backup.spawn_replica(name=name, destroy=True) + return backup.spawn_replica(name=name, + destroy=True, + slot_name=slot_name) def catchup(self, dbname=None, username=None): """ diff --git a/tests/test_simple.py b/tests/test_simple.py index ab2a5804..fef0ead8 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -382,6 +382,18 @@ def test_replicate(self): res = node.execute('select * from test') self.assertListEqual(res, []) + def test_replication_slots(self): + query_create = 'create table test as select generate_series(1, 2) as val' + + with get_new_node() as node: + node.init(allow_streaming=True).start() + node.create_replication_slot('slot1') + node.execute(query_create) + + with node.replicate(slot_name='slot1').start() as replica: + res = replica.execute('select * from test') + self.assertListEqual(res, [(1, ), (2, )]) + def test_incorrect_catchup(self): with get_new_node() as node: node.init(allow_streaming=True).start() From 3e5dbec70ec20a4756ceb301f6e965e0a2e04e8f Mon Sep 17 00:00:00 2001 From: Ildar Musin Date: Tue, 13 Mar 2018 16:37:03 +0300 Subject: [PATCH 125/475] Set default max_replication_slots number --- testgres/consts.py | 3 +++ testgres/node.py | 18 +++++++++++------- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/testgres/consts.py b/testgres/consts.py index 15400311..5ca5b747 100644 --- a/testgres/consts.py +++ b/testgres/consts.py @@ -24,3 +24,6 @@ PG_LOG_FILE = "postgresql.log" UTILS_LOG_FILE = "utils.log" BACKUP_LOG_FILE = "backup.log" + +# default replication slots number +REPLICATION_SLOTS = 10 diff --git a/testgres/node.py b/testgres/node.py index 8974d493..11687365 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -32,7 +32,8 @@ RECOVERY_CONF_FILE, \ PG_LOG_FILE, \ UTILS_LOG_FILE, \ - PG_PID_FILE + PG_PID_FILE, \ + REPLICATION_SLOTS from .decorators import \ method_decorator, \ @@ -208,7 +209,7 @@ def _create_recovery_conf(self, username, slot_name=None): ).format(conninfo) if slot_name: - line += "primary_slot_name={}".format() + line += "primary_slot_name={}\n".format() self.append_conf(RECOVERY_CONF_FILE, line) @@ -343,11 +344,14 @@ def get_auth_method(t): conf.write(u"fsync = off\n") # yapf: disable - conf.write(u"log_statement = {}\n" - u"listen_addresses = '{}'\n" - u"port = {}\n".format(log_statement, - self.host, - self.port)) + conf.write( + u"log_statement = {}\n" + u"listen_addresses = '{}'\n" + u"port = {}\n" + u"max_replication_slots = {}\n".format(log_statement, + self.host, + self.port, + REPLICATION_SLOTS)) # replication-related settings if allow_streaming: From bc6c3027820017613f49a4d5522f8f5861fc298d Mon Sep 17 00:00:00 2001 From: Ildar Musin Date: Tue, 13 Mar 2018 17:01:57 +0300 Subject: [PATCH 126/475] Fix: primary_slot_name never actually was set --- testgres/backup.py | 2 +- testgres/node.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/testgres/backup.py b/testgres/backup.py index 1226d135..6e9aedb0 100644 --- a/testgres/backup.py +++ b/testgres/backup.py @@ -171,7 +171,7 @@ def spawn_replica(self, name=None, destroy=True, slot_name=None): # Assign it a master and a recovery file (private magic) node._assign_master(self.original_node) - node._create_recovery_conf(username=self.username) + node._create_recovery_conf(username=self.username, slot_name=slot_name) return node diff --git a/testgres/node.py b/testgres/node.py index 11687365..d4d01ee0 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -209,7 +209,7 @@ def _create_recovery_conf(self, username, slot_name=None): ).format(conninfo) if slot_name: - line += "primary_slot_name={}\n".format() + line += "primary_slot_name={}\n".format(slot_name) self.append_conf(RECOVERY_CONF_FILE, line) @@ -872,7 +872,9 @@ def create_replication_slot(self, slot_name, dbname=None, username=None): dbname: database name username: database user name """ - query = "select pg_create_physical_replication_slot('{}')".format(slot_name) + query = ( + "select pg_create_physical_replication_slot('{}')" + ).format(slot_name) self.execute(query=query, dbname=dbname or default_dbname(), From 8c96ba02dab8f1a7a0bc0a318bf06112b84b8af6 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Tue, 13 Mar 2018 20:03:21 +0300 Subject: [PATCH 127/475] small fixes and refactoring after a chat with @ildus --- run_tests.sh | 3 +- testgres/connection.py | 5 +- testgres/enums.py | 57 +++++++++--- testgres/node.py | 205 +++++++++++++++++++++-------------------- tests/test_simple.py | 89 +++++++++--------- 5 files changed, 198 insertions(+), 161 deletions(-) diff --git a/run_tests.sh b/run_tests.sh index 53bfdc99..25fb504b 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -47,8 +47,7 @@ time coverage run -a tests/test_simple.py unset PG_BIN -# run tests (PG_CONFIG), also without psutil -$PIP uninstall -y psutil +# run tests (PG_CONFIG) export PG_CONFIG=$(which pg_config) time coverage run -a tests/test_simple.py unset PG_CONFIG diff --git a/testgres/connection.py b/testgres/connection.py index eb8505d0..8c086379 100644 --- a/testgres/connection.py +++ b/testgres/connection.py @@ -34,7 +34,6 @@ def __init__(self, node, dbname=None, username=None, password=None): username = username or default_username() self._node = node - self._pid = 0 self._connection = pglib.connect( database=dbname, @@ -55,9 +54,7 @@ def connection(self): @property def pid(self): - if not self._pid: - self._pid = self.execute("select pg_backend_pid();")[0][0] - return self._pid + return self.execute("select pg_backend_pid()")[0][0] @property def cursor(self): diff --git a/testgres/enums.py b/testgres/enums.py index d1db8541..b7d5fb71 100644 --- a/testgres/enums.py +++ b/testgres/enums.py @@ -1,4 +1,5 @@ from enum import Enum, IntEnum +from six import iteritems class XLogMethod(Enum): @@ -39,14 +40,48 @@ def __bool__(self): class ProcessType(Enum): """ - Types of postgres processes - """ - Checkpointer = 'postgres: checkpointer' - BackgroundWriter = 'postgres: background writer' - WalWriter = 'postgres: walwriter' - AutovacuumLauncher = 'postgres: autovacuum launcher' - StatsCollector = 'postgres: stats collector' - LogicalReplicationLauncher = 'postgres: logical replication launcher' - WalReceiver = 'postgres: walreceiver' - WalSender = 'postgres: walsender' - Startup = 'postgres: startup' + Types of processes + """ + + AutovacuumLauncher = 'autovacuum launcher' + BackgroundWriter = 'background writer' + Checkpointer = 'checkpointer' + LogicalReplicationLauncher = 'logical replication launcher' + Startup = 'startup' + StatsCollector = 'stats collector' + WalReceiver = 'wal receiver' + WalSender = 'wal sender' + WalWriter = 'wal writer' + + # special value + Unknown = 'unknown' + + @staticmethod + def from_process(process): + # legacy names for older releases of PG + alternative_names = { + ProcessType.LogicalReplicationLauncher: [ + 'logical replication worker' + ], + ProcessType.BackgroundWriter: [ + 'writer' + ], + } + + # we deliberately cut special words and spaces + cmdline = ''.join(process.cmdline()) \ + .replace('postgres:', '', 1) \ + .replace('bgworker:', '', 1) \ + .replace(' ', '') + + for ptype in ProcessType: + if cmdline.startswith(ptype.value.replace(' ', '')): + return ptype + + for ptype, names in iteritems(alternative_names): + for name in names: + if cmdline.startswith(name.replace(' ', '')): + return ptype + + # default + return ProcessType.Unknown diff --git a/testgres/node.py b/testgres/node.py index 0702ea68..b671110d 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -2,17 +2,11 @@ import io import os -import six import subprocess import time -try: - import psutil -except ImportError: - psutil = None - from shutil import rmtree -from six import raise_from +from six import raise_from, iteritems from tempfile import mkstemp, mkdtemp from .enums import NodeStatus, ProcessType @@ -70,6 +64,28 @@ from .backup import NodeBackup +class ProcessProxy(object): + """ + Wrapper for psutil.Process + + Attributes: + process: wrapped psutill.Process object + ptype: instance of ProcessType + """ + + def __init__(self, process): + self.process = process + self.ptype = ProcessType.from_process(process) + + def __getattr__(self, name): + return getattr(self.process, name) + + def __str__(self): + pid = self.process.pid + cmdline = ' '.join(self.process.cmdline()).strip() + return '{} [{}]'.format(cmdline, pid) + + class PostgresNode(object): def __init__(self, name=None, port=None, base_dir=None): """ @@ -122,11 +138,88 @@ def __exit__(self, type, value, traceback): @property def pid(self): - return self.get_main_pid() + """ + Return postmaster's PID if node is running, else 0. + """ + + if self.status(): + pid_file = os.path.join(self.data_dir, PG_PID_FILE) + with io.open(pid_file) as f: + return int(f.readline()) + + # for clarity + return 0 @property def auxiliary_pids(self): - return self.get_auxiliary_pids() + """ + Returns a dict of { ProcessType : PID }. + """ + + result = {} + + for process in self.auxiliary_processes: + if process.ptype not in result: + result[process.ptype] = [] + + result[process.ptype].append(process.pid) + + return result + + @property + def auxiliary_processes(self): + """ + Returns a list of auxiliary processes. + Each process is represented by ProcessProxy object. + """ + + def is_aux(process): + return process.ptype != ProcessType.Unknown + + return list(filter(is_aux, self.child_processes)) + + @property + def child_processes(self): + """ + Returns a list of all child processes. + Each process is represented by ProcessProxy object. + """ + + try: + import psutil + except ImportError: + raise TestgresException("psutil module is not installed") + + # get a list of postmaster's children + children = psutil.Process(self.pid).children() + + return [ProcessProxy(p) for p in children] + + @property + def source_walsender(self): + """ + Returns master's walsender feeding this replica. + """ + + sql = """ + select pid + from pg_catalog.pg_stat_replication + where application_name = $1 + """ + + if not self.master: + raise TestgresException("Node doesn't have a master") + + # master should be on the same host + assert self.master.host == self.host + + with self.master.connect() as con: + for row in con.execute(sql, self.name): + for child in self.master.auxiliary_processes: + if child.pid == int(row[0]): + return child + + raise QueryException("Master doesn't send WAL to {}", self.name) @property def master(self): @@ -427,98 +520,6 @@ def status(self): elif e.exit_code == 4: return NodeStatus.Uninitialized - def get_main_pid(self): - """ - Return postmaster's PID if node is running, else 0. - """ - - if self.status(): - pid_file = os.path.join(self.data_dir, PG_PID_FILE) - with io.open(pid_file) as f: - return int(f.readline()) - - # for clarity - return 0 - - def get_child_processes(self): - ''' Returns child processes for this node ''' - - if psutil is None: - raise TestgresException("psutil module is not installed") - - try: - postmaster = psutil.Process(self.pid) - except psutil.NoSuchProcess: - return None - - return postmaster.children(recursive=True) - - def get_auxiliary_pids(self): - ''' Returns dict with pids of auxiliary processes ''' - - alternative_names = { - ProcessType.LogicalReplicationLauncher: [ - 'postgres: bgworker: logical replication launcher' - ], - ProcessType.BackgroundWriter: [ - 'postgres: writer', - ], - ProcessType.WalWriter: [ - 'postgres: wal writer', - ], - ProcessType.WalReceiver: [ - 'postgres: wal receiver', - ], - } - - children = self.get_child_processes() - if children is None: - return None - - result = {} - for child in children: - line = ' '.join(child.cmdline()) - for ptype in ProcessType: - if ptype == ProcessType.WalSender \ - and (line.startswith(ptype.value) or - line.startswith('postgres: wal sender')): - result.setdefault(ptype, []) - result[ptype].append(child.pid) - break - elif line.startswith(ptype.value): - result[ptype] = child.pid - break - elif ptype in alternative_names: - names = alternative_names[ptype] - for name in names: - if line.startswith(name): - result[ptype] = child.pid - break - - return result - - def get_walsender_pid(self): - ''' Returns pid of according walsender for replica ''' - - if not self._master: - raise TestgresException("This node is not a replica") - - children = self._master.get_child_processes() - if children is None: - return None - - sql = 'select application_name, client_port from pg_stat_replication' - for name, client_port in self._master.execute(sql): - if name == self.name: - for child in children: - line = ' '.join(child.cmdline()) - if (line.startswith(ProcessType.WalSender.value) or - line.startswith('postgres: wal sender')) and \ - str(client_port) in line: - return child.pid - - return None - def get_control_data(self): """ Return contents of pg_control file. @@ -1079,7 +1080,7 @@ def pgbench_run(self, "-U", username, ] + options - for key, value in six.iteritems(kwargs): + for key, value in iteritems(kwargs): # rename keys for pgbench key = key.replace('_', '-') diff --git a/tests/test_simple.py b/tests/test_simple.py index e445a216..0fcbb51e 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -61,6 +61,14 @@ def good_properties(f): return True +def module_exists(module): + try: + __import__(module) + return True + except ImportError: + return False + + @contextmanager def removing(f): try: @@ -704,58 +712,55 @@ def test_version_management(self): self.assertTrue(b > c) self.assertTrue(a > c) - def test_pids(self): - try: - import psutil - except ImportError: - psutil = None - + @unittest.skipUnless(module_exists('psutil'), 'might be missing') + def test_child_pids(self): master_processes = [ - ProcessType.Checkpointer, - ProcessType.BackgroundWriter, - ProcessType.WalWriter, ProcessType.AutovacuumLauncher, + ProcessType.BackgroundWriter, + ProcessType.Checkpointer, ProcessType.StatsCollector, ProcessType.WalSender, + ProcessType.WalWriter, ] + if pg_version_ge('10'): master_processes.append(ProcessType.LogicalReplicationLauncher) - repl_processes = ( + repl_processes = [ ProcessType.Startup, - ProcessType.Checkpointer, - ProcessType.BackgroundWriter, - ProcessType.StatsCollector, ProcessType.WalReceiver, - ) - - with get_new_node('master') as master: - master.init().start() - - self.assertIsNotNone(master.pid) - with master.connect() as con: - self.assertTrue(con.pid > 0) - - with master.backup() as backup: - with backup.spawn_replica('repl', True) as repl: - repl.start() - if psutil is None: - with self.assertRaises(TestgresException): - master.auxiliary_pids - with self.assertRaises(TestgresException): - self.assertIsNone(repl.auxiliary_pids) - else: - master_pids = master.auxiliary_pids - for ptype in master_processes: - self.assertIn(ptype, master_pids) - self.assertTrue(len(master_pids[ProcessType.WalSender]) == 1) - - repl_pids = repl.auxiliary_pids - for ptype in repl_processes: - self.assertIn(ptype, repl_pids) - - sender_pid = master_pids[ProcessType.WalSender][0] - self.assertTrue(repl.get_walsender_pid() == sender_pid) + ] + + with get_new_node().init().start() as master: + + # master node doesn't have a source walsender! + with self.assertRaises(TestgresException): + master.source_walsender + + with master.replicate().start() as replica: + + # test __str__ method + str(master.child_processes[0]) + + master_pids = master.auxiliary_pids + for ptype in master_processes: + self.assertIn(ptype, master_pids) + + replica_pids = replica.auxiliary_pids + for ptype in repl_processes: + self.assertIn(ptype, replica_pids) + + # there should be exactly 1 source walsender for replica + self.assertEqual(len(master_pids[ProcessType.WalSender]), 1) + pid1 = master_pids[ProcessType.WalSender][0] + pid2 = replica.source_walsender.pid + self.assertEqual(pid1, pid2) + + replica.stop() + + # there should be no walsender after we've stopped replica + with self.assertRaises(QueryException): + replica.source_walsender if __name__ == '__main__': From e46f239e9332e309f227c6d86211f9d2df475693 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Wed, 14 Mar 2018 13:33:22 +0300 Subject: [PATCH 128/475] small changes for yapf --- testgres/enums.py | 1 + 1 file changed, 1 insertion(+) diff --git a/testgres/enums.py b/testgres/enums.py index b7d5fb71..f60cf835 100644 --- a/testgres/enums.py +++ b/testgres/enums.py @@ -58,6 +58,7 @@ class ProcessType(Enum): @staticmethod def from_process(process): + # yapf: disable # legacy names for older releases of PG alternative_names = { ProcessType.LogicalReplicationLauncher: [ From e27f01b2087fa1bcb7cdea09ee199c0ef36510b4 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Wed, 14 Mar 2018 14:03:46 +0300 Subject: [PATCH 129/475] minor fixes to ProcessProxy --- testgres/node.py | 11 +++++------ tests/test_simple.py | 5 ++--- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index b671110d..54b6168a 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -80,10 +80,8 @@ def __init__(self, process): def __getattr__(self, name): return getattr(self.process, name) - def __str__(self): - pid = self.process.pid - cmdline = ' '.join(self.process.cmdline()).strip() - return '{} [{}]'.format(cmdline, pid) + def __repr__(self): + return '{} : {}'.format(str(self.ptype), repr(self.process)) class PostgresNode(object): @@ -219,7 +217,8 @@ def source_walsender(self): if child.pid == int(row[0]): return child - raise QueryException("Master doesn't send WAL to {}", self.name) + msg = "Master doesn't send WAL to {}".format(self.name) + raise TestgresException(msg) @property def master(self): @@ -968,7 +967,7 @@ def catchup(self, dbname=None, username=None): """ if not self.master: - raise CatchUpException("Node doesn't have a master") + raise TestgresException("Node doesn't have a master") if pg_version_ge('10'): poll_lsn = "select pg_current_wal_lsn()::text" diff --git a/tests/test_simple.py b/tests/test_simple.py index 0fcbb51e..0efafcc5 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -20,7 +20,6 @@ ExecUtilException, \ BackupException, \ QueryException, \ - CatchUpException, \ TimeoutException, \ TestgresException @@ -397,7 +396,7 @@ def test_incorrect_catchup(self): node.init(allow_streaming=True).start() # node has no master, can't catch up - with self.assertRaises(CatchUpException): + with self.assertRaises(TestgresException): node.catchup() def test_dump(self): @@ -759,7 +758,7 @@ def test_child_pids(self): replica.stop() # there should be no walsender after we've stopped replica - with self.assertRaises(QueryException): + with self.assertRaises(TestgresException): replica.source_walsender From 8045bbe8591fde7d17e47446a78591b98ffbef41 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Wed, 14 Mar 2018 15:01:45 +0300 Subject: [PATCH 130/475] add missing test --- tests/test_simple.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_simple.py b/tests/test_simple.py index 0efafcc5..fafea63c 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -736,6 +736,9 @@ def test_child_pids(self): with self.assertRaises(TestgresException): master.source_walsender + with master.connect() as con: + self.assertGreater(con.pid, 0) + with master.replicate().start() as replica: # test __str__ method From 1dc3c929d72599ec0bb7ab79eded6220af2412fe Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Thu, 15 Mar 2018 18:05:15 +0300 Subject: [PATCH 131/475] bump version to 1.6.0 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index f307b6fa..93a23547 100644 --- a/setup.py +++ b/setup.py @@ -17,7 +17,7 @@ install_requires.append("ipaddress") setup( - version='1.5.0', + version='1.6.0', name='testgres', packages=['testgres'], description='Testing utility for PostgreSQL and its extensions', From b417418053b26503b8437cc0b70f18650927f410 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Thu, 15 Mar 2018 18:06:44 +0300 Subject: [PATCH 132/475] make psutil a dependency --- setup.py | 2 +- testgres/node.py | 6 +----- tests/test_simple.py | 9 --------- 3 files changed, 2 insertions(+), 15 deletions(-) diff --git a/setup.py b/setup.py index 93a23547..4972e15c 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ from distutils.core import setup # Basic dependencies -install_requires = ["pg8000", "port-for>=0.4", "six>=1.9.0"] +install_requires = ["pg8000", "port-for>=0.4", "six>=1.9.0", "psutil"] # Add compatibility enum class if sys.version_info < (3, 4): diff --git a/testgres/node.py b/testgres/node.py index 54b6168a..993837bf 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -2,6 +2,7 @@ import io import os +import psutil import subprocess import time @@ -183,11 +184,6 @@ def child_processes(self): Each process is represented by ProcessProxy object. """ - try: - import psutil - except ImportError: - raise TestgresException("psutil module is not installed") - # get a list of postmaster's children children = psutil.Process(self.pid).children() diff --git a/tests/test_simple.py b/tests/test_simple.py index fafea63c..d1cf26a1 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -60,14 +60,6 @@ def good_properties(f): return True -def module_exists(module): - try: - __import__(module) - return True - except ImportError: - return False - - @contextmanager def removing(f): try: @@ -711,7 +703,6 @@ def test_version_management(self): self.assertTrue(b > c) self.assertTrue(a > c) - @unittest.skipUnless(module_exists('psutil'), 'might be missing') def test_child_pids(self): master_processes = [ ProcessType.AutovacuumLauncher, From ca5b5465a07e45f91f6258bfcc73a4dd06ae9e8c Mon Sep 17 00:00:00 2001 From: Ildar Musin Date: Fri, 16 Mar 2018 18:47:25 +0300 Subject: [PATCH 133/475] Logical replication --- testgres/node.py | 58 +++++++++++++++- testgres/pubsub.py | 161 +++++++++++++++++++++++++++++++++++++++++++ tests/test_simple.py | 66 ++++++++++++++++++ 3 files changed, 282 insertions(+), 3 deletions(-) create mode 100644 testgres/pubsub.py diff --git a/testgres/node.py b/testgres/node.py index 02cc0f52..c187e469 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -48,10 +48,13 @@ ExecUtilException, \ QueryException, \ StartNodeException, \ - TimeoutException + TimeoutException, \ + InitNodeException from .logger import TestgresLogger +from .pubsub import Publication, Subscription + from .utils import \ eprint, \ get_bin_path, \ @@ -278,6 +281,7 @@ def default_conf(self, fsync=False, unix_sockets=True, allow_streaming=True, + allow_logical=False, log_statement='all'): """ Apply default settings to this node. @@ -286,6 +290,7 @@ def default_conf(self, fsync: should this node use fsync to keep data safe? unix_sockets: should we enable UNIX sockets? allow_streaming: should this node add a hba entry for replication? + allow_logical: can this node be used as a logical replication publisher? log_statement: one of ('all', 'off', 'mod', 'ddl'). Returns: @@ -365,6 +370,12 @@ def get_auth_method(t): wal_keep_segments, wal_level)) + if allow_logical: + if not pg_version_ge('10'): + raise InitNodeException("Logical replication is only " + "available for Postgres 10 and newer") + conf.write(u"wal_level = logical\n") + # disable UNIX sockets if asked to if not unix_sockets: conf.write(u"unix_socket_directories = ''\n") @@ -751,7 +762,8 @@ def poll_query_until(self, expected=True, commit=True, raise_programming_error=True, - raise_internal_error=True): + raise_internal_error=True, + zero_rows_is_ok=False): """ Run a query once per second until it returns 'expected'. Query should return a single value (1 row, 1 column). @@ -788,7 +800,12 @@ def poll_query_until(self, raise QueryException('Query returned None', query) if len(res) == 0: - raise QueryException('Query returned 0 rows', query) + if zero_rows_is_ok: + time.sleep(sleep_time) + attempts += 1 + continue + else: + raise QueryException('Query returned 0 rows', query) if len(res[0]) == 0: raise QueryException('Query returned 0 columns', query) @@ -902,6 +919,41 @@ def catchup(self, dbname=None, username=None): except Exception as e: raise_from(CatchUpException("Failed to catch up", poll_lsn), e) + def publish(self, + pubname, + tables=None, + dbname=None, + username=None): + """ + Create publication for logical replication + + Args: + pubname: publication name + tables: tables names list + dbname: database name where objects or interest are located + username: replication username + """ + return Publication(pubname, self, tables, dbname, username) + + def subscribe(self, + publication, + subname, + dbname=None, + username=None, + **kwargs): + """ + Create subscription for logical replication + + Args: + subname: subscription name + publication: publication object obtained from publish() + + """ + return Subscription(subname, self, publication, + dbname=dbname, + username=username, + **kwargs) + def pgbench(self, dbname=None, username=None, diff --git a/testgres/pubsub.py b/testgres/pubsub.py new file mode 100644 index 00000000..35d8f67a --- /dev/null +++ b/testgres/pubsub.py @@ -0,0 +1,161 @@ +# coding: utf-8 + +from six import raise_from + +from .defaults import default_dbname, default_username +from .exceptions import CatchUpException +from .utils import pg_version_ge + + +class Publication(object): + def __init__(self, pubname, node, tables=None, dbname=None, username=None): + """ + Constructor + + Args: + pubname: publication name + node: publisher's node + tables: tables list or None for all tables + dbname: database name used to connect and perform subscription + username: username used to connect to the database + """ + self.name = pubname + self.node = node + self.dbname = dbname or default_dbname() + self.username = username or default_username() + + # create publication in database + t = 'table ' + ', '.join(tables) if tables else 'all tables' + query = "create publication {} for {}" + node.safe_psql(query.format(pubname, t), + dbname=dbname, + username=username) + + def close(self, dbname=None, username=None): + """ + Drop publication + """ + self.node.safe_psql("drop publication {}".format(self.name), + dbname=dbname, username=username) + + def add_tables(self, tables, dbname=None, username=None): + """ + Add tables + + Args: + tables: a list of tables to add to the publication + """ + if not tables: + raise ValueError("Tables list is empty") + + query = "alter publication {} add table {}" + self.node.safe_psql(query.format(self.name, ', '.join(tables)), + dbname=dbname or self.dbname, + username=username or self.username) + + +class Subscription(object): + def __init__(self, + subname, + node, + publication, + dbname=None, + username=None, + **kwargs): + """ + Constructor + + Args: + subname: subscription name + node: subscriber's node + publication: Publication object we are subscribing to + dbname: database name used to connect and perform subscription + username: username used to connect to the database + **kwargs: subscription parameters (see CREATE SUBSCRIPTION + in PostgreSQL documentation for more information) + """ + self.name = subname + self.node = node + self.pub = publication + + # connection info + conninfo = ( + u"dbname={} user={} host={} port={}" + ).format(self.pub.dbname, + self.pub.username, + self.pub.node.host, + self.pub.node.port) + + query = ( + "create subscription {} connection '{}' publication {}" + ).format(subname, conninfo, self.pub.name) + + # additional parameters + if kwargs: + params = ','.join('{}={}'.format(k, v) for k, v in kwargs.iteritems()) + query += " with ({})".format(params) + + node.safe_psql(query, dbname=dbname, username=username) + + def disable(self, dbname=None, username=None): + """ + Disables the running subscription. + """ + query = "alter subscription {} disable" + self.node.safe_psql(query.format(self.name), + dbname=None, + username=None) + + def enable(self, dbname=None, username=None): + """ + Enables the previously disabled subscription. + """ + query = "alter subscription {} enable" + self.node.safe_psql(query.format(self.name), + dbname=None, + username=None) + + def refresh(self, copy_data=True, dbname=None, username=None): + """ + Disables the running subscription. + """ + query = "alter subscription {} refresh publication with (copy_data={})" + self.node.safe_psql(query.format(self.name, copy_data), + dbname=dbname, + username=username) + + def close(self, dbname=None, username=None): + """ + Drops subscription + """ + self.node.safe_psql("drop subscription {}".format(self.name), + dbname=dbname, username=username) + + def catchup(self, username=None): + """ + Wait until subscription catches up with publication. + + Args: + username: remote node's user name + """ + if pg_version_ge('10'): + query = ( + "select pg_current_wal_lsn() - replay_lsn = 0 " + "from pg_stat_replication where application_name = '{}'" + ).format(self.name) + else: + query = ( + "select pg_current_xlog_location() - replay_location = 0 " + "from pg_stat_replication where application_name = '{}'" + ).format(self.name) + + try: + # wait until this LSN reaches subscriber + self.pub.node.poll_query_until( + query=query, + dbname=self.pub.dbname, + username=username or self.pub.username, + max_attempts=60, + zero_rows_is_ok=True) # statistics may have not updated yet + except Exception as e: + raise_from(CatchUpException("Failed to catch up", query), e) diff --git a/tests/test_simple.py b/tests/test_simple.py index ab2a5804..37b10e6d 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -382,6 +382,72 @@ def test_replicate(self): res = node.execute('select * from test') self.assertListEqual(res, []) + def test_logical_replication(self): + with get_new_node() as node1, get_new_node() as node2: + node1.init(allow_logical=True) + node1.start() + node2.init().start() + + create_table = 'create table test (a int, b int)' + node1.safe_psql(create_table) + node2.safe_psql(create_table) + + # create publication / create subscription + pub = node1.publish('mypub') + sub = node2.subscribe(pub, 'mysub') + + node1.safe_psql('insert into test values (1, 1), (2, 2)') + + # wait until changes apply on subscriber and check them + sub.catchup() + res = node2.execute('select * from test') + self.assertListEqual(res, [(1, 1), (2, 2)]) + + # disable and put some new data + sub.disable() + node1.safe_psql('insert into test values (3, 3)') + + # enable and ensure that data successfully transfered + sub.enable() + sub.catchup() + res = node2.execute('select * from test') + self.assertListEqual(res, [(1, 1), (2, 2), (3, 3)]) + + # Add new tables. Since we added "all tables" to publication + # (default behaviour of publish() method) we don't need + # to explicitely perform pub.add_table() + create_table = 'create table test2 (c char)' + node1.safe_psql(create_table) + node2.safe_psql(create_table) + sub.refresh() + + # put new data + node1.safe_psql('insert into test2 values (\'a\'), (\'b\')') + sub.catchup() + res = node2.execute('select * from test2') + self.assertListEqual(res, [('a',), ('b',)]) + + # drop subscription + sub.close() + pub.close() + + # create new publication and subscription for specific table + # (ommitting copying data as it's already done) + pub = node1.publish('newpub', tables=['test']) + sub = node2.subscribe(pub, 'newsub', copy_data=False) + + node1.safe_psql('insert into test values (4, 4)') + sub.catchup() + res = node2.execute('select * from test') + self.assertListEqual(res, [(1, 1), (2, 2), (3, 3), (4, 4)]) + + # explicitely add table + pub.add_tables(['test2']) + node1.safe_psql('insert into test2 values (\'c\')') + sub.catchup() + res = node2.execute('select * from test2') + self.assertListEqual(res, [('a',), ('b',)]) + def test_incorrect_catchup(self): with get_new_node() as node: node.init(allow_streaming=True).start() From bb01c7d787f960691539c44ceee73f9d326f2b40 Mon Sep 17 00:00:00 2001 From: Ildar Musin Date: Fri, 16 Mar 2018 19:05:38 +0300 Subject: [PATCH 134/475] Skip logical replication test on PostgreSQL versions below 10 --- tests/test_simple.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_simple.py b/tests/test_simple.py index 37b10e6d..3489fcf5 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -382,6 +382,7 @@ def test_replicate(self): res = node.execute('select * from test') self.assertListEqual(res, []) + @unittest.skipUnless(pg_version_ge('10'), 'requires 10+') def test_logical_replication(self): with get_new_node() as node1, get_new_node() as node2: node1.init(allow_logical=True) From 782484b971ed5ab94e03dd0ba4bc3224a0636d5d Mon Sep 17 00:00:00 2001 From: Ildar Musin Date: Fri, 16 Mar 2018 20:28:24 +0300 Subject: [PATCH 135/475] Fix logical replication for python3 --- testgres/pubsub.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testgres/pubsub.py b/testgres/pubsub.py index 35d8f67a..dde2e694 100644 --- a/testgres/pubsub.py +++ b/testgres/pubsub.py @@ -1,6 +1,6 @@ # coding: utf-8 -from six import raise_from +from six import raise_from, iteritems from .defaults import default_dbname, default_username from .exceptions import CatchUpException @@ -92,7 +92,7 @@ def __init__(self, # additional parameters if kwargs: - params = ','.join('{}={}'.format(k, v) for k, v in kwargs.iteritems()) + params = ','.join('{}={}'.format(k, v) for k, v in iteritems(kwargs)) query += " with ({})".format(params) node.safe_psql(query, dbname=dbname, username=username) From e3492bd8d3501137be43af9146845dd4dee8a0e5 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Mon, 19 Mar 2018 13:58:31 +0300 Subject: [PATCH 136/475] execute only a small subset of tests for PG_BIN and PG_CONFIG --- .dockerignore | 4 ++++ run_tests.sh | 14 ++++++++------ tests/README.md | 2 +- tests/test_simple.py | 21 +++++++++++++++++++-- 4 files changed, 32 insertions(+), 9 deletions(-) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..ff03e5de --- /dev/null +++ b/.dockerignore @@ -0,0 +1,4 @@ +dist +env +venv +*.egg-info diff --git a/run_tests.sh b/run_tests.sh index 25fb504b..52d5b034 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -42,15 +42,17 @@ time coverage run -a tests/test_simple.py # run tests (PG_BIN) -export PG_BIN=$(dirname $(which pg_config)) -time coverage run -a tests/test_simple.py -unset PG_BIN +time \ + PG_BIN=$(dirname $(which pg_config)) \ + ALT_CONFIG=1 \ + coverage run -a tests/test_simple.py # run tests (PG_CONFIG) -export PG_CONFIG=$(which pg_config) -time coverage run -a tests/test_simple.py -unset PG_CONFIG +time \ + PG_CONFIG=$(which pg_config) \ + ALT_CONFIG=1 \ + coverage run -a tests/test_simple.py # show coverage diff --git a/tests/README.md b/tests/README.md index 5a5ecd07..a6d50992 100644 --- a/tests/README.md +++ b/tests/README.md @@ -21,7 +21,7 @@ export PG_BIN=/path/to/pg/bin ```bash # Set path to PostgreSQL and python version -export PATH=$PATH:/path/to/pg/bin +export PATH=/path/to/pg/bin:$PATH export PYTHON_VERSION=3 # or 2 # Run tests diff --git a/tests/test_simple.py b/tests/test_simple.py index d1cf26a1..48f1130d 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -69,7 +69,7 @@ def removing(f): os.remove(f) -class SimpleTest(unittest.TestCase): +class TestgresTests(unittest.TestCase): def test_custom_init(self): with get_new_node() as node: # enable page checksums @@ -757,4 +757,21 @@ def test_child_pids(self): if __name__ == '__main__': - unittest.main() + if os.environ.get('ALT_CONFIG'): + suite = unittest.TestSuite() + + # Small subset of tests for alternative configs (PG_BIN or PG_CONFIG) + suite.addTest(TestgresTests('test_pg_config')) + suite.addTest(TestgresTests('test_pg_ctl')) + suite.addTest(TestgresTests('test_psql')) + suite.addTest(TestgresTests('test_replicate')) + + print('Running tests for alternative config:') + for t in suite: + print(t) + print() + + runner = unittest.TextTestRunner() + runner.run(suite) + else: + unittest.main() From e2e5a8d3d2018a6f1329f2567f9877ae801f73da Mon Sep 17 00:00:00 2001 From: Ildar Musin Date: Tue, 20 Mar 2018 14:33:04 +0300 Subject: [PATCH 137/475] Make replication slot creation implicit; minor refactoring --- testgres/node.py | 50 ++++++++++++++++++++++---------------------- tests/test_simple.py | 1 - 2 files changed, 25 insertions(+), 26 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index d4d01ee0..15a6f0b4 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -254,6 +254,21 @@ def _collect_special_files(self): return result + def _create_replication_slot(self, slot_name, dbname=None, username=None): + """ + Create a physical replication slot. + + Args: + slot_name: slot name + dbname: database name + username: database user name + """ + query = ( + "select pg_create_physical_replication_slot('{}')" + ).format(slot_name) + + self.execute(query=query, dbname=dbname, username=username) + def init(self, initdb_params=None, **kwargs): """ Perform initdb for this node. @@ -344,14 +359,11 @@ def get_auth_method(t): conf.write(u"fsync = off\n") # yapf: disable - conf.write( - u"log_statement = {}\n" - u"listen_addresses = '{}'\n" - u"port = {}\n" - u"max_replication_slots = {}\n".format(log_statement, - self.host, - self.port, - REPLICATION_SLOTS)) + conf.write(u"log_statement = {}\n" + u"listen_addresses = '{}'\n" + u"port = {}\n".format(log_statement, + self.host, + self.port)) # replication-related settings if allow_streaming: @@ -367,8 +379,10 @@ def get_auth_method(t): wal_keep_segments = 20 # for convenience conf.write(u"hot_standby = on\n" u"max_wal_senders = {}\n" + u"max_replication_slots = {}\n" u"wal_keep_segments = {}\n" u"wal_level = {}\n".format(max_wal_senders, + REPLICATION_SLOTS, wal_keep_segments, wal_level)) @@ -863,23 +877,6 @@ def backup(self, **kwargs): return NodeBackup(node=self, **kwargs) - def create_replication_slot(self, slot_name, dbname=None, username=None): - """ - Create a physical replication slot. - - Args: - slot_name: slot name - dbname: database name - username: database user name - """ - query = ( - "select pg_create_physical_replication_slot('{}')" - ).format(slot_name) - - self.execute(query=query, - dbname=dbname or default_dbname(), - username=username or default_username()) - def replicate(self, name=None, slot_name=None, **kwargs): """ Create a binary replica of this node. @@ -891,6 +888,9 @@ def replicate(self, name=None, slot_name=None, **kwargs): base_dir: the base directory for data files and logs """ + if slot_name: + self._create_replication_slot(slot_name, **kwargs) + backup = self.backup(**kwargs) # transform backup into a replica diff --git a/tests/test_simple.py b/tests/test_simple.py index fef0ead8..66b08a02 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -387,7 +387,6 @@ def test_replication_slots(self): with get_new_node() as node: node.init(allow_streaming=True).start() - node.create_replication_slot('slot1') node.execute(query_create) with node.replicate(slot_name='slot1').start() as replica: From 8bac32ebfe9f90eca744baf549ec56b9dd36c91f Mon Sep 17 00:00:00 2001 From: Ildar Musin Date: Tue, 20 Mar 2018 17:42:24 +0300 Subject: [PATCH 138/475] Sphinx configuration --- .gitignore | 2 + docs/Makefile | 20 +++++ docs/source/conf.py | 161 +++++++++++++++++++++++++++++++++++++++ docs/source/index.rst | 95 +++++++++++++++++++++++ docs/source/modules.rst | 7 ++ docs/source/testgres.rst | 118 ++++++++++++++++++++++++++++ 6 files changed, 403 insertions(+) create mode 100644 docs/Makefile create mode 100644 docs/source/conf.py create mode 100644 docs/source/index.rst create mode 100644 docs/source/modules.rst create mode 100644 docs/source/testgres.rst diff --git a/.gitignore b/.gitignore index 4cf6750a..c46fa861 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,8 @@ .eggs/ dist/ build/ +docs/build/ +docs/source/_* env/ venv/ diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 00000000..f33f6be0 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +SPHINXPROJ = testgres +SOURCEDIR = source +BUILDDIR = build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) \ No newline at end of file diff --git a/docs/source/conf.py b/docs/source/conf.py new file mode 100644 index 00000000..4e1608cf --- /dev/null +++ b/docs/source/conf.py @@ -0,0 +1,161 @@ +# -*- coding: utf-8 -*- +# +# Configuration file for the Sphinx documentation builder. +# +# This file does only contain a selection of the most common options. For a +# full list see the documentation: +# http://www.sphinx-doc.org/en/stable/config + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import os +import sys +sys.path.insert(0, os.path.abspath('../..')) + + +# -- Project information ----------------------------------------------------- + +project = u'testgres' +copyright = u'2016-2018, Postgres Professional' +author = u'Postgres Professional' + +# The short X.Y version +version = u'' +# The full version, including alpha/beta/rc tags +release = u'1.5' + + +# -- General configuration --------------------------------------------------- + +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +#extensions = [ +# 'sphinx.ext.autodoc', +#] + +extensions = ['sphinx.ext.autodoc', 'sphinxcontrib.napoleon'] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The master toctree document. +master_doc = 'index' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path . +exclude_patterns = [] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'alabaster' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# Custom sidebar templates, must be a dictionary that maps document names +# to template names. +# +# The default sidebars (for documents that don't match any pattern) are +# defined by theme itself. Builtin themes are using these templates by +# default: ``['localtoc.html', 'relations.html', 'sourcelink.html', +# 'searchbox.html']``. +# +# html_sidebars = {} + + +# -- Options for HTMLHelp output --------------------------------------------- + +# Output file base name for HTML help builder. +htmlhelp_basename = 'testgresdoc' + + +# -- Options for LaTeX output ------------------------------------------------ + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'testgres.tex', u'testgres Documentation', + u'Postgres Professional', 'manual'), +] + + +# -- Options for manual page output ------------------------------------------ + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, 'testgres', u'testgres Documentation', + [author], 1) +] + + +# -- Options for Texinfo output ---------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'testgres', u'testgres Documentation', + author, 'testgres', 'One line description of project.', + 'Miscellaneous'), +] + + +# -- Extension configuration ------------------------------------------------- diff --git a/docs/source/index.rst b/docs/source/index.rst new file mode 100644 index 00000000..9159e2f3 --- /dev/null +++ b/docs/source/index.rst @@ -0,0 +1,95 @@ + +Testgres documentation +====================== + +Testgres is a PostgreSQL testing utility + +Installation +============ + +To install testgres, run: + +.. code-block:: bash + + pip install testgres + +We encourage you to use ``virtualenv`` for your testing environment. + +Usage +===== + +Environment +----------- + +Note: by default testgres runs ``initdb``, ``pg_ctl``, ``psql`` provided by ``PATH``. + +There are several ways to specify a custom postgres installation: + +- export ``PG_CONFIG`` environment variable pointing to the ``pg_config`` executable; +- export ``PG_BIN`` environment variable pointing to the directory with executable files. + +Example: + +.. code-block:: bash + + export PG_BIN=$HOME/pg_10/bin + python my_tests.py + +Examples +-------- + +Here is an example of what you can do with ``testgres``: + +.. code-block:: python + + # create a node with random name, port, etc + with testgres.get_new_node() as node: + + # run inidb + node.init() + + # start PostgreSQL + node.start() + + # execute a query in a default DB + print(node.execute('select 1')) + + # ... node stops and its files are about to be removed + +Backup & replication +-------------------- + +It's quite easy to create a backup and start a new replica: + +.. code-block:: python + + with testgres.get_new_node('master') as master: + master.init().start() + + # create a backup + with master.backup() as backup: + + # create and start a new replica + replica = backup.spawn_replica('replica').start() + + # catch up with master node + replica.catchup() + + # execute a dummy query + print(replica.execute('postgres', 'select 1')) + +Modules +======= + +.. toctree:: + :maxdepth: 2 + + testgres + + +.. Indices and tables +.. ================== + +.. * :ref:`genindex` +.. * :ref:`modindex` +.. * :ref:`search` diff --git a/docs/source/modules.rst b/docs/source/modules.rst new file mode 100644 index 00000000..9aeb4d64 --- /dev/null +++ b/docs/source/modules.rst @@ -0,0 +1,7 @@ +testgres +======== + +.. toctree:: + :maxdepth: 4 + + testgres diff --git a/docs/source/testgres.rst b/docs/source/testgres.rst new file mode 100644 index 00000000..346f6530 --- /dev/null +++ b/docs/source/testgres.rst @@ -0,0 +1,118 @@ +testgres package +================ + +Submodules +---------- + +testgres.api module +------------------- + +.. automodule:: testgres.api + :members: + :undoc-members: + :show-inheritance: + +testgres.backup module +---------------------- + +.. automodule:: testgres.backup + :members: + :undoc-members: + :show-inheritance: + +testgres.cache module +--------------------- + +.. automodule:: testgres.cache + :members: + :undoc-members: + :show-inheritance: + +testgres.config module +---------------------- + +.. automodule:: testgres.config + :members: + :undoc-members: + :show-inheritance: + +testgres.connection module +-------------------------- + +.. automodule:: testgres.connection + :members: + :undoc-members: + :show-inheritance: + +testgres.consts module +---------------------- + +.. automodule:: testgres.consts + :members: + :undoc-members: + :show-inheritance: + +testgres.decorators module +-------------------------- + +.. automodule:: testgres.decorators + :members: + :undoc-members: + :show-inheritance: + +testgres.defaults module +------------------------ + +.. automodule:: testgres.defaults + :members: + :undoc-members: + :show-inheritance: + +testgres.enums module +--------------------- + +.. automodule:: testgres.enums + :members: + :undoc-members: + :show-inheritance: + +testgres.exceptions module +-------------------------- + +.. automodule:: testgres.exceptions + :members: + :undoc-members: + :show-inheritance: + +testgres.logger module +---------------------- + +.. automodule:: testgres.logger + :members: + :undoc-members: + :show-inheritance: + +testgres.node module +-------------------- + +.. automodule:: testgres.node + :members: + :undoc-members: + :show-inheritance: + +testgres.utils module +--------------------- + +.. automodule:: testgres.utils + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: testgres + :members: + :undoc-members: + :show-inheritance: From aca77bffacde3b3294684cbe6a0f678466a7832a Mon Sep 17 00:00:00 2001 From: Ildar Musin Date: Wed, 21 Mar 2018 14:13:38 +0300 Subject: [PATCH 139/475] Some documentation improvements and refactoring --- docs/README.md | 15 +++++++++++ docs/source/index.rst | 2 +- docs/source/modules.rst | 7 ----- docs/source/testgres.rst | 55 +++++++++++++++++++--------------------- 4 files changed, 42 insertions(+), 37 deletions(-) create mode 100644 docs/README.md delete mode 100644 docs/source/modules.rst diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 00000000..1e520e83 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,15 @@ +# Building the documentation + +Make sure you have `Sphinx` and `sphinxcontrib-napoleon` packages installed: + +``` +pip install Sphinx sphinxcontrib-napoleon +``` + +Then just run + +``` +make html +``` + +Documentation will be built in `build/html` directory. Other output formats are also available; run `make` without arguments to see the options. \ No newline at end of file diff --git a/docs/source/index.rst b/docs/source/index.rst index 9159e2f3..566d9a50 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -2,7 +2,7 @@ Testgres documentation ====================== -Testgres is a PostgreSQL testing utility +Testgres is a PostgreSQL testing framework. Installation ============ diff --git a/docs/source/modules.rst b/docs/source/modules.rst deleted file mode 100644 index 9aeb4d64..00000000 --- a/docs/source/modules.rst +++ /dev/null @@ -1,7 +0,0 @@ -testgres -======== - -.. toctree:: - :maxdepth: 4 - - testgres diff --git a/docs/source/testgres.rst b/docs/source/testgres.rst index 346f6530..63eadc80 100644 --- a/docs/source/testgres.rst +++ b/docs/source/testgres.rst @@ -1,107 +1,104 @@ testgres package ================ -Submodules ----------- - -testgres.api module -------------------- +testgres.api +------------ .. automodule:: testgres.api :members: :undoc-members: :show-inheritance: -testgres.backup module ----------------------- +testgres.backup +--------------- .. automodule:: testgres.backup :members: :undoc-members: :show-inheritance: -testgres.cache module ---------------------- +testgres.cache +-------------- .. automodule:: testgres.cache :members: :undoc-members: :show-inheritance: -testgres.config module ----------------------- +testgres.config +--------------- .. automodule:: testgres.config :members: :undoc-members: :show-inheritance: -testgres.connection module --------------------------- +testgres.connection +------------------- .. automodule:: testgres.connection :members: :undoc-members: :show-inheritance: -testgres.consts module ----------------------- +testgres.consts +--------------- .. automodule:: testgres.consts :members: :undoc-members: :show-inheritance: -testgres.decorators module --------------------------- +testgres.decorators +------------------- .. automodule:: testgres.decorators :members: :undoc-members: :show-inheritance: -testgres.defaults module ------------------------- +testgres.defaults +----------------- .. automodule:: testgres.defaults :members: :undoc-members: :show-inheritance: -testgres.enums module ---------------------- +testgres.enums +-------------- .. automodule:: testgres.enums :members: :undoc-members: :show-inheritance: -testgres.exceptions module --------------------------- +testgres.exceptions +------------------- .. automodule:: testgres.exceptions :members: :undoc-members: :show-inheritance: -testgres.logger module ----------------------- +testgres.logger +--------------- .. automodule:: testgres.logger :members: :undoc-members: :show-inheritance: -testgres.node module --------------------- +testgres.node +------------- .. automodule:: testgres.node :members: :undoc-members: :show-inheritance: -testgres.utils module ---------------------- +testgres.utils +-------------- .. automodule:: testgres.utils :members: From 8c199a40c67deb88a2cf46e245f12fd734e3852b Mon Sep 17 00:00:00 2001 From: Ildar Musin Date: Wed, 21 Mar 2018 14:27:05 +0300 Subject: [PATCH 140/475] Remove redundant comments from Sphinx configuration script --- docs/source/conf.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 4e1608cf..06600d78 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -38,10 +38,6 @@ # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -#extensions = [ -# 'sphinx.ext.autodoc', -#] - extensions = ['sphinx.ext.autodoc', 'sphinxcontrib.napoleon'] # Add any paths that contain templates here, relative to this directory. From 37120a2c7032c47b35a51a3638ad8f00a791a96a Mon Sep 17 00:00:00 2001 From: Ildar Musin Date: Wed, 21 Mar 2018 15:06:49 +0300 Subject: [PATCH 141/475] After yapf been applied --- docs/source/conf.py | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 06600d78..af013ec4 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -16,7 +16,6 @@ import sys sys.path.insert(0, os.path.abspath('../..')) - # -- Project information ----------------------------------------------------- project = u'testgres' @@ -28,7 +27,6 @@ # The full version, including alpha/beta/rc tags release = u'1.5' - # -- General configuration --------------------------------------------------- # If your documentation needs a minimal Sphinx version, state it here. @@ -67,7 +65,6 @@ # The name of the Pygments (syntax highlighting) style to use. pygments_style = 'sphinx' - # -- Options for HTML output ------------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for @@ -96,13 +93,11 @@ # # html_sidebars = {} - # -- Options for HTMLHelp output --------------------------------------------- # Output file base name for HTML help builder. htmlhelp_basename = 'testgresdoc' - # -- Options for LaTeX output ------------------------------------------------ latex_elements = { @@ -131,16 +126,11 @@ u'Postgres Professional', 'manual'), ] - # -- Options for manual page output ------------------------------------------ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). -man_pages = [ - (master_doc, 'testgres', u'testgres Documentation', - [author], 1) -] - +man_pages = [(master_doc, 'testgres', u'testgres Documentation', [author], 1)] # -- Options for Texinfo output ---------------------------------------------- @@ -148,10 +138,8 @@ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - (master_doc, 'testgres', u'testgres Documentation', - author, 'testgres', 'One line description of project.', - 'Miscellaneous'), + (master_doc, 'testgres', u'testgres Documentation', author, 'testgres', + 'One line description of project.', 'Miscellaneous'), ] - # -- Extension configuration ------------------------------------------------- From d4f48c87db9418f776cc332ccba514f99af974ab Mon Sep 17 00:00:00 2001 From: Ildar Musin Date: Wed, 21 Mar 2018 15:33:08 +0300 Subject: [PATCH 142/475] Run documentation build along with the tests --- .gitignore | 1 - run_tests.sh | 6 +++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index c46fa861..038d1952 100644 --- a/.gitignore +++ b/.gitignore @@ -5,7 +5,6 @@ dist/ build/ docs/build/ -docs/source/_* env/ venv/ diff --git a/run_tests.sh b/run_tests.sh index 52d5b034..5d986f85 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -22,7 +22,7 @@ export VIRTUAL_ENV_DISABLE_PROMPT=1 source $VENV_PATH/bin/activate # install utilities -$PIP install coverage flake8 psutil +$PIP install coverage flake8 psutil Sphinx sphinxcontrib-napoleon # install testgres' dependencies export PYTHONPATH=$(pwd) @@ -58,6 +58,10 @@ time \ # show coverage coverage report +# build documentation +cd docs +make html +cd .. # attempt to fix codecov set +eux From 98c96dd7f2a64039fbe5efb7b49f465e0fc0827e Mon Sep 17 00:00:00 2001 From: Ildar Musin Date: Wed, 21 Mar 2018 15:43:43 +0300 Subject: [PATCH 143/475] Remove redundunt _static directory from Sphinx configuration script --- docs/source/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index af013ec4..7ba79351 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -81,7 +81,7 @@ # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +html_static_path = [] # Custom sidebar templates, must be a dictionary that maps document names # to template names. From 59dbe426d75883cd09d3be842419826b317204c4 Mon Sep 17 00:00:00 2001 From: Ildar Musin Date: Wed, 21 Mar 2018 16:49:15 +0300 Subject: [PATCH 144/475] When creating new physical replication slot check that it doesn't already exist --- testgres/node.py | 10 +++++++++- tests/test_simple.py | 7 ++++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index 15a6f0b4..72a0707e 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -49,7 +49,8 @@ ExecUtilException, \ QueryException, \ StartNodeException, \ - TimeoutException + TimeoutException, \ + TestgresException from .logger import TestgresLogger @@ -263,6 +264,13 @@ def _create_replication_slot(self, slot_name, dbname=None, username=None): dbname: database name username: database user name """ + rs = self.execute("select exists (select * from pg_replication_slots " + "where slot_name = '{}')".format(slot_name), + dbname=dbname, username=username) + + if rs[0][0]: + raise TestgresException("Slot '{}' already exists".format(slot_name)) + query = ( "select pg_create_physical_replication_slot('{}')" ).format(slot_name) diff --git a/tests/test_simple.py b/tests/test_simple.py index 66b08a02..8f2c57d2 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -21,7 +21,8 @@ BackupException, \ QueryException, \ CatchUpException, \ - TimeoutException + TimeoutException, \ + TestgresException from testgres import \ TestgresConfig, \ @@ -393,6 +394,10 @@ def test_replication_slots(self): res = replica.execute('select * from test') self.assertListEqual(res, [(1, ), (2, )]) + # cannot create new slot with the same name + with self.assertRaises(TestgresException): + node._create_replication_slot('slot1') + def test_incorrect_catchup(self): with get_new_node() as node: node.init(allow_streaming=True).start() From 773b42abe49bca821fde363d16fd5586f725e4f8 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Wed, 21 Mar 2018 16:54:22 +0300 Subject: [PATCH 145/475] add pre-commit hook executing yapf --- hooks/install.sh | 3 +++ hooks/pre-commit | 13 +++++++++++++ 2 files changed, 16 insertions(+) create mode 100755 hooks/install.sh create mode 100755 hooks/pre-commit diff --git a/hooks/install.sh b/hooks/install.sh new file mode 100755 index 00000000..d1ea0366 --- /dev/null +++ b/hooks/install.sh @@ -0,0 +1,3 @@ +#!/bin/bash +DIR=$(dirname $0) +ln -s -f ../../hooks/pre-commit "$DIR/../.git/hooks/" diff --git a/hooks/pre-commit b/hooks/pre-commit new file mode 100755 index 00000000..3dfc5794 --- /dev/null +++ b/hooks/pre-commit @@ -0,0 +1,13 @@ +#!/bin/bash + +# capture the changed files that have been staged +changed_files=$(git diff --staged --name-only) + +for file in ${changed_files} +do + if [[ "${file##*.}" == "py" ]]; then + echo "Yapfing ${file}" + yapf ${file} -i + git add ${file} + fi +done From 7251c955d0fb0cbf084eeb0a8d1a0fcb1e97d916 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Wed, 21 Mar 2018 17:23:30 +0300 Subject: [PATCH 146/475] add README.md for git hooks --- hooks/README.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 hooks/README.md diff --git a/hooks/README.md b/hooks/README.md new file mode 100644 index 00000000..607a38ea --- /dev/null +++ b/hooks/README.md @@ -0,0 +1,7 @@ +### What's this? + +This is a set of git hooks to be executed on special events, e.g. before you commit your changes. To install them, just execute the `install.sh` script, and you're good to go! + +### What do they do? + +Currently there's only one hook (`pre-commit`) which formats changed python files with `yapf`. From 57eaea1178b2d95f9952f38f09d0a2002becde72 Mon Sep 17 00:00:00 2001 From: Ildar Musin Date: Wed, 21 Mar 2018 18:18:17 +0300 Subject: [PATCH 147/475] Added PostgresNode.__repr__() method and fix doctests --- testgres/api.py | 12 ++++++------ testgres/config.py | 2 ++ testgres/node.py | 4 ++++ 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/testgres/api.py b/testgres/api.py index 6bc276d5..7f9166e5 100644 --- a/testgres/api.py +++ b/testgres/api.py @@ -7,27 +7,27 @@ edit configuration files, start/stop cluster, execute queries. The typical flow may look like: ->>> with get_new_node() as node: +>>> with get_new_node('test') as node: ... node.init().start() ... result = node.safe_psql('postgres', 'select 1') ... print(result.decode('utf-8').strip()) ... node.stop() - +PostgresNode('test', port=..., base_dir=...) 1 - +PostgresNode('test', port=..., base_dir=...) Or: ->>> with get_new_node() as master: +>>> with get_new_node('master') as master: ... master.init().start() ... with master.backup() as backup: -... with backup.spawn_replica() as replica: +... with backup.spawn_replica('replica') as replica: ... replica = replica.start() ... master.execute('postgres', 'create table test (val int4)') ... master.execute('postgres', 'insert into test values (0), (1), (2)') ... replica.catchup() # wait until changes are visible ... print(replica.execute('postgres', 'select count(*) from test')) - +PostgresNode('master', port=..., base_dir=...) [(3,)] Copyright (c) 2016, Postgres Professional diff --git a/testgres/config.py b/testgres/config.py index 976263f7..e08baf1e 100644 --- a/testgres/config.py +++ b/testgres/config.py @@ -152,9 +152,11 @@ def scoped_config(**options): Temporarily set custom GlobalConfig options for this context. Example: + >>> from .api import get_new_node >>> with scoped_config(cache_initdb=False): ... with get_new_node().init().start() as node: ... print(node.execute('select 1')) + [(1,)] """ try: diff --git a/testgres/node.py b/testgres/node.py index 993837bf..fa24ab6d 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -135,6 +135,10 @@ def __exit__(self, type, value, traceback): else: self._try_shutdown(attempts) + def __repr__(self): + return "PostgresNode('{}', port={}, base_dir={})".format( + self.name, self.port, self.base_dir) + @property def pid(self): """ From 71e487eeec98a37bbaffd18ba718bc6e6afe5285 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Thu, 22 Mar 2018 13:28:55 +0300 Subject: [PATCH 148/475] inline _create_replication_slot() to _create_recovery_conf() --- testgres/backup.py | 7 +++--- testgres/node.py | 54 ++++++++++++++++++++------------------------ tests/test_simple.py | 10 +++----- 3 files changed, 31 insertions(+), 40 deletions(-) diff --git a/testgres/backup.py b/testgres/backup.py index 6e9aedb0..798ba4b4 100644 --- a/testgres/backup.py +++ b/testgres/backup.py @@ -154,12 +154,13 @@ def spawn_primary(self, name=None, destroy=True): return node - def spawn_replica(self, name=None, destroy=True, slot_name=None): + def spawn_replica(self, name=None, destroy=True, slot=None): """ Create a replica of the original node from a backup. Args: name: replica's application name. + slot: create a replication slot with the specified name. destroy: should we convert this backup into a node? Returns: @@ -171,11 +172,11 @@ def spawn_replica(self, name=None, destroy=True, slot_name=None): # Assign it a master and a recovery file (private magic) node._assign_master(self.original_node) - node._create_recovery_conf(username=self.username, slot_name=slot_name) + node._create_recovery_conf(username=self.username, slot=slot) return node def cleanup(self): if self._available: - rmtree(self.base_dir, ignore_errors=True) self._available = False + rmtree(self.base_dir, ignore_errors=True) diff --git a/testgres/node.py b/testgres/node.py index cc81ca82..c764e80b 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -278,7 +278,7 @@ def _assign_master(self, master): # now this node has a master self._master = master - def _create_recovery_conf(self, username, slot_name=None): + def _create_recovery_conf(self, username, slot=None): """NOTE: this is a private method!""" # fetch master of this node @@ -306,8 +306,26 @@ def _create_recovery_conf(self, username, slot_name=None): "standby_mode=on\n" ).format(conninfo) - if slot_name: - line += "primary_slot_name={}\n".format(slot_name) + if slot: + # Connect to master for some additional actions + with master.connect(username=username) as con: + # check if slot already exists + res = con.execute(""" + select exists ( + select from pg_catalog.pg_replication_slots + where slot_name = $1 + ) + """, slot) + + if res[0][0]: + raise TestgresException("Slot '{}' already exists".format(slot)) + + # TODO: we should drop this slot after replica's cleanup() + con.execute(""" + select pg_catalog.pg_create_physical_replication_slot($1) + """, slot) + + line += "primary_slot_name={}\n".format(slot) self.append_conf(RECOVERY_CONF_FILE, line) @@ -352,28 +370,6 @@ def _collect_special_files(self): return result - def _create_replication_slot(self, slot_name, dbname=None, username=None): - """ - Create a physical replication slot. - - Args: - slot_name: slot name - dbname: database name - username: database user name - """ - rs = self.execute("select exists (select * from pg_replication_slots " - "where slot_name = '{}')".format(slot_name), - dbname=dbname, username=username) - - if rs[0][0]: - raise TestgresException("Slot '{}' already exists".format(slot_name)) - - query = ( - "select pg_create_physical_replication_slot('{}')" - ).format(slot_name) - - self.execute(query=query, dbname=dbname, username=username) - def init(self, initdb_params=None, **kwargs): """ Perform initdb for this node. @@ -969,26 +965,24 @@ def backup(self, **kwargs): return NodeBackup(node=self, **kwargs) - def replicate(self, name=None, slot_name=None, **kwargs): + def replicate(self, name=None, slot=None, **kwargs): """ Create a binary replica of this node. Args: name: replica's application name. + slot: create a replication slot with the specified name. username: database user name. xlog_method: a method for collecting the logs ('fetch' | 'stream'). base_dir: the base directory for data files and logs """ - if slot_name: - self._create_replication_slot(slot_name, **kwargs) - backup = self.backup(**kwargs) # transform backup into a replica return backup.spawn_replica(name=name, destroy=True, - slot_name=slot_name) + slot=slot) def catchup(self, dbname=None, username=None): """ diff --git a/tests/test_simple.py b/tests/test_simple.py index fc3b4dd2..7453d65f 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -384,19 +384,15 @@ def test_replicate(self): self.assertListEqual(res, []) def test_replication_slots(self): - query_create = 'create table test as select generate_series(1, 2) as val' - with get_new_node() as node: node.init(allow_streaming=True).start() - node.execute(query_create) - with node.replicate(slot_name='slot1').start() as replica: - res = replica.execute('select * from test') - self.assertListEqual(res, [(1, ), (2, )]) + with node.replicate(slot='slot1').start() as replica: + replica.execute('select 1') # cannot create new slot with the same name with self.assertRaises(TestgresException): - node._create_replication_slot('slot1') + node.replicate(slot='slot1') def test_incorrect_catchup(self): with get_new_node() as node: From 008236d0cb7ce3c24c2c55c86aa9bcc480e7e873 Mon Sep 17 00:00:00 2001 From: Ildar Musin Date: Thu, 22 Mar 2018 13:37:02 +0300 Subject: [PATCH 149/475] Added test for PostgresNode.__repr__() method; removed node name from examples --- testgres/api.py | 12 ++++++------ testgres/node.py | 2 +- tests/test_simple.py | 6 ++++++ 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/testgres/api.py b/testgres/api.py index 7f9166e5..472a422a 100644 --- a/testgres/api.py +++ b/testgres/api.py @@ -7,27 +7,27 @@ edit configuration files, start/stop cluster, execute queries. The typical flow may look like: ->>> with get_new_node('test') as node: +>>> with get_new_node() as node: ... node.init().start() ... result = node.safe_psql('postgres', 'select 1') ... print(result.decode('utf-8').strip()) ... node.stop() -PostgresNode('test', port=..., base_dir=...) +PostgresNode(name='...', port=..., base_dir='...') 1 -PostgresNode('test', port=..., base_dir=...) +PostgresNode(name='...', port=..., base_dir='...') Or: ->>> with get_new_node('master') as master: +>>> with get_new_node() as master: ... master.init().start() ... with master.backup() as backup: -... with backup.spawn_replica('replica') as replica: +... with backup.spawn_replica() as replica: ... replica = replica.start() ... master.execute('postgres', 'create table test (val int4)') ... master.execute('postgres', 'insert into test values (0), (1), (2)') ... replica.catchup() # wait until changes are visible ... print(replica.execute('postgres', 'select count(*) from test')) -PostgresNode('master', port=..., base_dir=...) +PostgresNode(name='...', port=..., base_dir='...') [(3,)] Copyright (c) 2016, Postgres Professional diff --git a/testgres/node.py b/testgres/node.py index fa24ab6d..4971f1be 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -136,7 +136,7 @@ def __exit__(self, type, value, traceback): self._try_shutdown(attempts) def __repr__(self): - return "PostgresNode('{}', port={}, base_dir={})".format( + return "PostgresNode(name='{}', port={}, base_dir='{}')".format( self.name, self.port, self.base_dir) @property diff --git a/tests/test_simple.py b/tests/test_simple.py index 48f1130d..748d1df4 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -2,6 +2,7 @@ # coding: utf-8 import os +import re import subprocess import tempfile import testgres @@ -70,6 +71,11 @@ def removing(f): class TestgresTests(unittest.TestCase): + def test_node_repr(self): + with get_new_node() as node: + pattern = 'PostgresNode\(name=\'.+\', port=.+, base_dir=\'.+\'\)' + self.assertIsNotNone(re.match(pattern, str(node))) + def test_custom_init(self): with get_new_node() as node: # enable page checksums From 06ce6e24deb5bd1daf588c36560dc0b9fa0a3ea4 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Thu, 22 Mar 2018 14:02:16 +0300 Subject: [PATCH 150/475] fix ProcessProxy's __repr__() according to guidelines --- testgres/node.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index c764e80b..aecf3348 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -75,15 +75,17 @@ class ProcessProxy(object): ptype: instance of ProcessType """ - def __init__(self, process): + def __init__(self, process, ptype=None): self.process = process - self.ptype = ProcessType.from_process(process) + self.ptype = ptype or ProcessType.from_process(process) def __getattr__(self, name): return getattr(self.process, name) def __repr__(self): - return '{} : {}'.format(str(self.ptype), repr(self.process)) + return '{}(ptype={}, process={})'.format(self.__class__.__name__, + str(self.ptype), + repr(self.process)) class PostgresNode(object): @@ -980,9 +982,7 @@ def replicate(self, name=None, slot=None, **kwargs): backup = self.backup(**kwargs) # transform backup into a replica - return backup.spawn_replica(name=name, - destroy=True, - slot=slot) + return backup.spawn_replica(name=name, destroy=True, slot=slot) def catchup(self, dbname=None, username=None): """ From 749de80e9de4f61dbc4ee6825cb36e64287bbb4d Mon Sep 17 00:00:00 2001 From: Ildar Musin Date: Thu, 22 Mar 2018 14:05:24 +0300 Subject: [PATCH 151/475] Use __class__.__name__ instead of 'PostgresNode' literal --- testgres/node.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index 4971f1be..3f0d52a2 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -136,8 +136,8 @@ def __exit__(self, type, value, traceback): self._try_shutdown(attempts) def __repr__(self): - return "PostgresNode(name='{}', port={}, base_dir='{}')".format( - self.name, self.port, self.base_dir) + return "{}(name='{}', port={}, base_dir='{}')".format( + self.__class__.__name__, self.name, self.port, self.base_dir) @property def pid(self): From 04f0dd1601fb2af29664a6a36264b2512b23ea27 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Thu, 22 Mar 2018 14:10:57 +0300 Subject: [PATCH 152/475] introduce clean_on_error() for safer node handling --- testgres/backup.py | 31 +++++++++++++++++-------------- testgres/node.py | 8 ++++---- testgres/utils.py | 20 ++++++++++++++++++++ 3 files changed, 41 insertions(+), 18 deletions(-) diff --git a/testgres/backup.py b/testgres/backup.py index 798ba4b4..4ae5d5ba 100644 --- a/testgres/backup.py +++ b/testgres/backup.py @@ -21,7 +21,8 @@ from .utils import \ get_bin_path, \ - execute_utility + execute_utility, \ + clean_on_error class NodeBackup(object): @@ -56,8 +57,8 @@ def __init__(self, try: xlog_method = XLogMethod(xlog_method) except ValueError: - raise BackupException( - 'Invalid xlog_method "{}"'.format(xlog_method)) + msg = 'Invalid xlog_method "{}"'.format(xlog_method) + raise BackupException(msg) # Set default arguments username = username or default_username() @@ -144,15 +145,16 @@ def spawn_primary(self, name=None, destroy=True): # Build a new PostgresNode from .node import PostgresNode - node = PostgresNode(name=name, base_dir=base_dir) + with clean_on_error(PostgresNode(name=name, + base_dir=base_dir)) as node: - # New nodes should always remove dir tree - node._should_rm_dirs = True + # New nodes should always remove dir tree + node._should_rm_dirs = True - node.append_conf(PG_CONF_FILE, "\n") - node.append_conf(PG_CONF_FILE, "port = {}".format(node.port)) + node.append_conf(PG_CONF_FILE, "\n") + node.append_conf(PG_CONF_FILE, "port = {}".format(node.port)) - return node + return node def spawn_replica(self, name=None, destroy=True, slot=None): """ @@ -168,13 +170,14 @@ def spawn_replica(self, name=None, destroy=True, slot=None): """ # Build a new PostgresNode - node = self.spawn_primary(name=name, destroy=destroy) + with clean_on_error(self.spawn_primary(name=name, + destroy=destroy)) as node: - # Assign it a master and a recovery file (private magic) - node._assign_master(self.original_node) - node._create_recovery_conf(username=self.username, slot=slot) + # Assign it a master and a recovery file (private magic) + node._assign_master(self.original_node) + node._create_recovery_conf(username=self.username, slot=slot) - return node + return node def cleanup(self): if self._available: diff --git a/testgres/node.py b/testgres/node.py index aecf3348..37037bde 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -61,7 +61,8 @@ pg_version_ge, \ reserve_port, \ release_port, \ - execute_utility + execute_utility, \ + clean_on_error from .backup import NodeBackup @@ -979,10 +980,9 @@ def replicate(self, name=None, slot=None, **kwargs): base_dir: the base directory for data files and logs """ - backup = self.backup(**kwargs) - # transform backup into a replica - return backup.spawn_replica(name=name, destroy=True, slot=slot) + with clean_on_error(self.backup(**kwargs)) as backup: + return backup.spawn_replica(name=name, destroy=True, slot=slot) def catchup(self, dbname=None, username=None): """ diff --git a/testgres/utils.py b/testgres/utils.py index a4108b23..6704571b 100644 --- a/testgres/utils.py +++ b/testgres/utils.py @@ -9,6 +9,7 @@ import subprocess import sys +from contextlib import contextmanager from distutils.version import LooseVersion from .config import testgres_config @@ -217,4 +218,23 @@ def file_tail(f, num_lines): def eprint(*args, **kwargs): + """ + Print stuff to stderr. + """ + print(*args, file=sys.stderr, **kwargs) + + +@contextmanager +def clean_on_error(node): + """ + Context manager to wrap PostgresNode and such. + Calls cleanup() method when underlying code raises an exception. + """ + + try: + yield node + except Exception: + # TODO: should we wrap this in try-block? + node.cleanup() + raise From 5a81db19c18ea07856d358dc0825fe3ec72d8f4f Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Thu, 22 Mar 2018 14:14:54 +0300 Subject: [PATCH 153/475] all catalog objects must be schema-qualified --- testgres/connection.py | 2 +- testgres/node.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/testgres/connection.py b/testgres/connection.py index 8c086379..6447f685 100644 --- a/testgres/connection.py +++ b/testgres/connection.py @@ -54,7 +54,7 @@ def connection(self): @property def pid(self): - return self.execute("select pg_backend_pid()")[0][0] + return self.execute("select pg_catalog.pg_backend_pid()")[0][0] @property def cursor(self): diff --git a/testgres/node.py b/testgres/node.py index 37037bde..5b3768b2 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -993,11 +993,11 @@ def catchup(self, dbname=None, username=None): raise TestgresException("Node doesn't have a master") if pg_version_ge('10'): - poll_lsn = "select pg_current_wal_lsn()::text" - wait_lsn = "select pg_last_wal_replay_lsn() >= '{}'::pg_lsn" + poll_lsn = "select pg_catalog.pg_current_wal_lsn()::text" + wait_lsn = "select pg_catalog.pg_last_wal_replay_lsn() >= '{}'::pg_lsn" else: - poll_lsn = "select pg_current_xlog_location()::text" - wait_lsn = "select pg_last_xlog_replay_location() >= '{}'::pg_lsn" + poll_lsn = "select pg_catalog.pg_current_xlog_location()::text" + wait_lsn = "select pg_catalog.pg_last_xlog_replay_location() >= '{}'::pg_lsn" try: # fetch latest LSN From b1cba73cf385896e7307998f230ec9b4a43f8aa4 Mon Sep 17 00:00:00 2001 From: Ildar Musin Date: Thu, 22 Mar 2018 15:25:53 +0300 Subject: [PATCH 154/475] Some minor refactoring of logical replication --- testgres/node.py | 23 +++++++++-------------- testgres/pubsub.py | 17 +++++------------ tests/test_simple.py | 9 ++++----- 3 files changed, 18 insertions(+), 31 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index c53393c2..4288e863 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -847,8 +847,7 @@ def poll_query_until(self, expected=True, commit=True, raise_programming_error=True, - raise_internal_error=True, - zero_rows_is_ok=False): + raise_internal_error=True): """ Run a query once per second until it returns 'expected'. Query should return a single value (1 row, 1 column). @@ -884,18 +883,14 @@ def poll_query_until(self, if res is None: raise QueryException('Query returned None', query) - if len(res) == 0: - if zero_rows_is_ok: - time.sleep(sleep_time) - attempts += 1 - continue - else: - raise QueryException('Query returned 0 rows', query) - - if len(res[0]) == 0: - raise QueryException('Query returned 0 columns', query) - - if res[0][0] == expected: + # result set is not empty + if len(res): + if len(res[0]) == 0: + raise QueryException('Query returned 0 columns', query) + if res[0][0] == expected: + return # done + # empty result set is considered as None + elif expected is None: return # done except ProgrammingError as e: diff --git a/testgres/pubsub.py b/testgres/pubsub.py index dde2e694..3717e495 100644 --- a/testgres/pubsub.py +++ b/testgres/pubsub.py @@ -138,16 +138,10 @@ def catchup(self, username=None): Args: username: remote node's user name """ - if pg_version_ge('10'): - query = ( - "select pg_current_wal_lsn() - replay_lsn = 0 " - "from pg_stat_replication where application_name = '{}'" - ).format(self.name) - else: - query = ( - "select pg_current_xlog_location() - replay_location = 0 " - "from pg_stat_replication where application_name = '{}'" - ).format(self.name) + query = ( + "select pg_current_wal_lsn() - replay_lsn = 0 " + "from pg_stat_replication where application_name = '{}'" + ).format(self.name) try: # wait until this LSN reaches subscriber @@ -155,7 +149,6 @@ def catchup(self, username=None): query=query, dbname=self.pub.dbname, username=username or self.pub.username, - max_attempts=60, - zero_rows_is_ok=True) # statistics may have not updated yet + max_attempts=60) except Exception as e: raise_from(CatchUpException("Failed to catch up", query), e) diff --git a/tests/test_simple.py b/tests/test_simple.py index 62c8668b..1b962b1b 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -495,11 +495,6 @@ def test_poll_query_until(self): self.assertTrue(end_time - start_time >= 5) - # check 0 rows - with self.assertRaises(QueryException): - node.poll_query_until( - query='select * from pg_class where true = false') - # check 0 columns with self.assertRaises(QueryException): node.poll_query_until(query='select from pg_class limit 1') @@ -512,6 +507,10 @@ def test_poll_query_until(self): node.poll_query_until( query='create table def()', expected=None) # returns nothing + # check 0 rows equivalent to expected=None + node.poll_query_until( + query='select * from pg_class where true = false', expected=None) + # check arbitrary expected value, fail with self.assertRaises(TimeoutException): node.poll_query_until( From 954879aaf155d54bf50e15ef5bf3ef9d6d23e539 Mon Sep 17 00:00:00 2001 From: Ildar Musin Date: Thu, 22 Mar 2018 16:19:09 +0300 Subject: [PATCH 155/475] Added options_string() func --- testgres/node.py | 19 ++++++++++--------- testgres/pubsub.py | 21 ++++++++++----------- testgres/utils.py | 5 +++++ 3 files changed, 25 insertions(+), 20 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index 4288e863..d838bf79 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -63,7 +63,8 @@ pg_version_ge, \ reserve_port, \ release_port, \ - execute_utility + execute_utility, \ + options_string from .backup import NodeBackup @@ -288,25 +289,25 @@ def _create_recovery_conf(self, username): assert master is not None # yapf: disable - conninfo = ( - u"application_name={} " - u"port={} " - u"user={} " - ).format(self.name, master.port, username) + conninfo = { + "application_name": self.name, + "port": master.port, + "user": username + } # host is tricky try: import ipaddress ipaddress.ip_address(master.host) - conninfo += u"hostaddr={}".format(master.host) + conninfo["hostaddr"] = master.host except ValueError: - conninfo += u"host={}".format(master.host) + conninfo["host"] = master.host # yapf: disable line = ( "primary_conninfo='{}'\n" "standby_mode=on\n" - ).format(conninfo) + ).format(options_string(**conninfo)) self.append_conf(RECOVERY_CONF_FILE, line) diff --git a/testgres/pubsub.py b/testgres/pubsub.py index 3717e495..caef3727 100644 --- a/testgres/pubsub.py +++ b/testgres/pubsub.py @@ -1,10 +1,10 @@ # coding: utf-8 -from six import raise_from, iteritems +from six import raise_from from .defaults import default_dbname, default_username from .exceptions import CatchUpException -from .utils import pg_version_ge +from .utils import options_string class Publication(object): @@ -79,21 +79,20 @@ def __init__(self, self.pub = publication # connection info - conninfo = ( - u"dbname={} user={} host={} port={}" - ).format(self.pub.dbname, - self.pub.username, - self.pub.node.host, - self.pub.node.port) + conninfo = { + "dbname": self.pub.dbname, + "user": self.pub.username, + "host": self.pub.node.host, + "port": self.pub.node.port + } query = ( "create subscription {} connection '{}' publication {}" - ).format(subname, conninfo, self.pub.name) + ).format(subname, options_string(**conninfo), self.pub.name) # additional parameters if kwargs: - params = ','.join('{}={}'.format(k, v) for k, v in iteritems(kwargs)) - query += " with ({})".format(params) + query += " with ({})".format(options_string(**kwargs)) node.safe_psql(query, dbname=dbname, username=username) diff --git a/testgres/utils.py b/testgres/utils.py index a4108b23..2e4b5548 100644 --- a/testgres/utils.py +++ b/testgres/utils.py @@ -10,6 +10,7 @@ import sys from distutils.version import LooseVersion +from six import iteritems from .config import testgres_config from .exceptions import ExecUtilException @@ -218,3 +219,7 @@ def file_tail(f, num_lines): def eprint(*args, **kwargs): print(*args, file=sys.stderr, **kwargs) + + +def options_string(separator=u" ", **kwargs): + return separator.join(u"{}={}".format(k, v) for k, v in iteritems(kwargs)) From f4e0bd0e1c8e0f84909ceb5d5c0de0d5938ba77a Mon Sep 17 00:00:00 2001 From: Ildar Musin Date: Thu, 22 Mar 2018 17:45:21 +0300 Subject: [PATCH 156/475] Minor refactoring --- testgres/node.py | 13 ++++++------- testgres/pubsub.py | 20 ++++++++++---------- 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index 3556ede4..71e360af 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -1032,7 +1032,7 @@ def catchup(self, dbname=None, username=None): raise_from(CatchUpException("Failed to catch up", poll_lsn), e) def publish(self, - pubname, + name, tables=None, dbname=None, username=None): @@ -1045,11 +1045,12 @@ def publish(self, dbname: database name where objects or interest are located username: replication username """ - return Publication(pubname, self, tables, dbname, username) + return Publication(name=name, node=self, tables=tables, dbname=dbname, + username=username) def subscribe(self, publication, - subname, + name, dbname=None, username=None, **kwargs): @@ -1061,10 +1062,8 @@ def subscribe(self, publication: publication object obtained from publish() """ - return Subscription(subname, self, publication, - dbname=dbname, - username=username, - **kwargs) + return Subscription(name=name, node=self, publication=publication, + dbname=dbname, username=username, **kwargs) def pgbench(self, dbname=None, diff --git a/testgres/pubsub.py b/testgres/pubsub.py index 4b3db9cb..4cc77322 100644 --- a/testgres/pubsub.py +++ b/testgres/pubsub.py @@ -8,26 +8,26 @@ class Publication(object): - def __init__(self, pubname, node, tables=None, dbname=None, username=None): + def __init__(self, name, node, tables=None, dbname=None, username=None): """ Constructor Args: - pubname: publication name + name: publication name node: publisher's node tables: tables list or None for all tables dbname: database name used to connect and perform subscription username: username used to connect to the database """ - self.name = pubname + self.name = name self.node = node self.dbname = dbname or default_dbname() self.username = username or default_username() # create publication in database - t = 'table ' + ', '.join(tables) if tables else 'all tables' + t = "table " + ", ".join(tables) if tables else "all tables" query = "create publication {} for {}" - node.safe_psql(query.format(pubname, t), + node.safe_psql(query.format(name, t), dbname=dbname, username=username) @@ -49,14 +49,14 @@ def add_tables(self, tables, dbname=None, username=None): raise ValueError("Tables list is empty") query = "alter publication {} add table {}" - self.node.safe_psql(query.format(self.name, ', '.join(tables)), + self.node.safe_psql(query.format(self.name, ", ".join(tables)), dbname=dbname or self.dbname, username=username or self.username) class Subscription(object): def __init__(self, - subname, + name, node, publication, dbname=None, @@ -66,7 +66,7 @@ def __init__(self, Constructor Args: - subname: subscription name + name: subscription name node: subscriber's node publication: Publication object we are subscribing to dbname: database name used to connect and perform subscription @@ -74,7 +74,7 @@ def __init__(self, **kwargs: subscription parameters (see CREATE SUBSCRIPTION in PostgreSQL documentation for more information) """ - self.name = subname + self.name = name self.node = node self.pub = publication @@ -88,7 +88,7 @@ def __init__(self, query = ( "create subscription {} connection '{}' publication {}" - ).format(subname, options_string(**conninfo), self.pub.name) + ).format(name, options_string(**conninfo), self.pub.name) # additional parameters if kwargs: From f48623be51dc3c22ab8cca09f764ff5b9a0587c4 Mon Sep 17 00:00:00 2001 From: Ildar Musin Date: Thu, 22 Mar 2018 18:17:05 +0300 Subject: [PATCH 157/475] Add failing logical replication test for 9.6 --- tests/test_simple.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test_simple.py b/tests/test_simple.py index e8ac15e7..2de51df3 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -456,6 +456,12 @@ def test_logical_replication(self): res = node2.execute('select * from test2') self.assertListEqual(res, [('a',), ('b',)]) + @unittest.skipIf(pg_version_ge('10'), 'requires <10') + def test_logical_replication_fail(self): + with get_new_node() as node: + with self.assertRaises(InitNodeException): + node.init(allow_logical=True) + def test_replication_slots(self): with get_new_node() as node: node.init(allow_streaming=True).start() From 138c6ccffccfe5bdf4381a24583463e2416c38c2 Mon Sep 17 00:00:00 2001 From: Ildar Musin Date: Thu, 22 Mar 2018 18:44:27 +0300 Subject: [PATCH 158/475] Added test for Publication.add_tables() --- testgres/pubsub.py | 46 ++++++++++++++++++++++---------------------- tests/test_simple.py | 4 +++- 2 files changed, 26 insertions(+), 24 deletions(-) diff --git a/testgres/pubsub.py b/testgres/pubsub.py index 4cc77322..f0bedcb8 100644 --- a/testgres/pubsub.py +++ b/testgres/pubsub.py @@ -27,16 +27,16 @@ def __init__(self, name, node, tables=None, dbname=None, username=None): # create publication in database t = "table " + ", ".join(tables) if tables else "all tables" query = "create publication {} for {}" - node.safe_psql(query.format(name, t), - dbname=dbname, - username=username) + node.safe_psql(query.format(name, t), dbname=dbname, username=username) def drop(self, dbname=None, username=None): """ Drop publication """ - self.node.safe_psql("drop publication {}".format(self.name), - dbname=dbname, username=username) + self.node.safe_psql( + "drop publication {}".format(self.name), + dbname=dbname, + username=username) def add_tables(self, tables, dbname=None, username=None): """ @@ -49,9 +49,10 @@ def add_tables(self, tables, dbname=None, username=None): raise ValueError("Tables list is empty") query = "alter publication {} add table {}" - self.node.safe_psql(query.format(self.name, ", ".join(tables)), - dbname=dbname or self.dbname, - username=username or self.username) + self.node.safe_psql( + query.format(self.name, ", ".join(tables)), + dbname=dbname or self.dbname, + username=username or self.username) class Subscription(object): @@ -87,8 +88,8 @@ def __init__(self, } query = ( - "create subscription {} connection '{}' publication {}" - ).format(name, options_string(**conninfo), self.pub.name) + "create subscription {} connection '{}' publication {}").format( + name, options_string(**conninfo), self.pub.name) # additional parameters if kwargs: @@ -101,34 +102,33 @@ def disable(self, dbname=None, username=None): Disables the running subscription. """ query = "alter subscription {} disable" - self.node.safe_psql(query.format(self.name), - dbname=None, - username=None) + self.node.safe_psql(query.format(self.name), dbname=None, username=None) def enable(self, dbname=None, username=None): """ Enables the previously disabled subscription. """ query = "alter subscription {} enable" - self.node.safe_psql(query.format(self.name), - dbname=None, - username=None) + self.node.safe_psql(query.format(self.name), dbname=None, username=None) def refresh(self, copy_data=True, dbname=None, username=None): """ Disables the running subscription. """ query = "alter subscription {} refresh publication with (copy_data={})" - self.node.safe_psql(query.format(self.name, copy_data), - dbname=dbname, - username=username) + self.node.safe_psql( + query.format(self.name, copy_data), + dbname=dbname, + username=username) def drop(self, dbname=None, username=None): """ Drops subscription """ - self.node.safe_psql("drop subscription {}".format(self.name), - dbname=dbname, username=username) + self.node.safe_psql( + "drop subscription {}".format(self.name), + dbname=dbname, + username=username) def catchup(self, username=None): """ @@ -139,8 +139,8 @@ def catchup(self, username=None): """ query = ( "select pg_current_wal_lsn() - replay_lsn = 0 " - "from pg_stat_replication where application_name = '{}'" - ).format(self.name) + "from pg_stat_replication where application_name = '{}'").format( + self.name) try: # wait until this LSN reaches subscriber diff --git a/tests/test_simple.py b/tests/test_simple.py index 2de51df3..b5735a6f 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -423,7 +423,7 @@ def test_logical_replication(self): # Add new tables. Since we added "all tables" to publication # (default behaviour of publish() method) we don't need - # to explicitely perform pub.add_table() + # to explicitely perform pub.add_tables() create_table = 'create table test2 (c char)' node1.safe_psql(create_table) node2.safe_psql(create_table) @@ -450,6 +450,8 @@ def test_logical_replication(self): self.assertListEqual(res, [(1, 1), (2, 2), (3, 3), (4, 4)]) # explicitely add table + with self.assertRaises(ValueError): + pub.add_tables([]) # fail pub.add_tables(['test2']) node1.safe_psql('insert into test2 values (\'c\')') sub.catchup() From 6c3809303fc209b248f3a238f56530afcb0c4f51 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Fri, 23 Mar 2018 14:41:13 +0300 Subject: [PATCH 159/475] improve docs of config stack --- testgres/backup.py | 5 +++++ testgres/config.py | 38 +++++++++++++++++++++++++++++--------- 2 files changed, 34 insertions(+), 9 deletions(-) diff --git a/testgres/backup.py b/testgres/backup.py index 4ae5d5ba..c1056197 100644 --- a/testgres/backup.py +++ b/testgres/backup.py @@ -180,6 +180,11 @@ def spawn_replica(self, name=None, destroy=True, slot=None): return node def cleanup(self): + """ + Remove all files that belong to this backup. + No-op if it's been converted to a PostgresNode (destroy=True). + """ + if self._available: self._available = False rmtree(self.base_dir, ignore_errors=True) diff --git a/testgres/config.py b/testgres/config.py index e08baf1e..757e792c 100644 --- a/testgres/config.py +++ b/testgres/config.py @@ -13,19 +13,19 @@ class GlobalConfig(object): """ - Global config (override default settings). + Global configuration object which allows user to override default settings. Attributes: cache_initdb: shall we use cached initdb instance? - cached_initdb_dir: shall we create a temp dir for cached initdb? - cached_initdb_unique: shall we assign new node a unique system id? + cached_initdb_dir: path to a temp directory for cached initdb. + cached_initdb_unique: shall we give new node a unique system id? cache_pg_config: shall we cache pg_config results? - temp_dir: base temp dir for nodes with default 'base_dir'. + temp_dir: path to temp dir containing nodes with default 'base_dir'. - use_python_logging: use python logging configuration for all nodes. - error_log_lines: N of log lines to be shown in exception (0=inf). + use_python_logging: enable python logging subsystem (see logger.py). + error_log_lines: N of log lines to be shown in exceptions (0=inf). node_cleanup_full: shall we remove EVERYTHING (including logs)? node_cleanup_on_good_exit: remove base_dir on nominal __exit__(). @@ -82,6 +82,10 @@ def __setattr__(self, name, value): super(GlobalConfig, self).__setattr__(name, value) def keys(self): + """ + Return a list of all available settings. + """ + keys = [] for key in dir(GlobalConfig): @@ -91,15 +95,29 @@ def keys(self): return keys def items(self): + """ + Return setting-value pairs. + """ + return ((key, self[key]) for key in self.keys()) def update(self, config): + """ + Extract setting-value pairs from 'config' and + assign those values to corresponding settings + of this GlobalConfig object. + """ + for key, value in config.items(): self[key] = value return self def copy(self): + """ + Return a copy of this object. + """ + return copy.copy(self) @@ -124,8 +142,8 @@ def _rm_cached_initdb_dirs(): def push_config(**options): """ - Permanently set custom GlobalConfig options - and put previous settings on top of stack. + Permanently set custom GlobalConfig options and + put previous settings on top of the config stack. """ # push current config to stack @@ -150,10 +168,12 @@ def pop_config(): def scoped_config(**options): """ Temporarily set custom GlobalConfig options for this context. + Previous options are pushed to the config stack. Example: >>> from .api import get_new_node >>> with scoped_config(cache_initdb=False): + ... # create a new node with fresh initdb ... with get_new_node().init().start() as node: ... print(node.execute('select 1')) [(1,)] @@ -173,7 +193,7 @@ def scoped_config(**options): def configure_testgres(**options): """ Adjust current global options. - Look at GlobalConfig to learn what can be set. + Look at the GlobalConfig to learn about existing settings. """ testgres_config.update(options) From 019812cea2b75ea8d2e0a903c88bfe7bcdeed9c8 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Fri, 23 Mar 2018 14:42:46 +0300 Subject: [PATCH 160/475] fix yapf pattern usage in backup.py --- testgres/backup.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/testgres/backup.py b/testgres/backup.py index c1056197..2a9a982b 100644 --- a/testgres/backup.py +++ b/testgres/backup.py @@ -74,7 +74,6 @@ def __init__(self, data_dir = os.path.join(self.base_dir, DATA_DIR) - # yapf: disable _params = [ get_bin_path("pg_basebackup"), "-p", str(node.port), @@ -82,7 +81,7 @@ def __init__(self, "-U", username, "-D", data_dir, "-X", xlog_method.value - ] + ] # yapf: disable execute_utility(_params, self.log_file) def __enter__(self): @@ -145,8 +144,7 @@ def spawn_primary(self, name=None, destroy=True): # Build a new PostgresNode from .node import PostgresNode - with clean_on_error(PostgresNode(name=name, - base_dir=base_dir)) as node: + with clean_on_error(PostgresNode(name=name, base_dir=base_dir)) as node: # New nodes should always remove dir tree node._should_rm_dirs = True From 0dc92b1802f7fe2814f019e1ecce2e65f8800897 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Fri, 23 Mar 2018 14:44:41 +0300 Subject: [PATCH 161/475] fix yapf pattern usage in enums.py --- testgres/enums.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/testgres/enums.py b/testgres/enums.py index f60cf835..3d11427f 100644 --- a/testgres/enums.py +++ b/testgres/enums.py @@ -58,7 +58,6 @@ class ProcessType(Enum): @staticmethod def from_process(process): - # yapf: disable # legacy names for older releases of PG alternative_names = { ProcessType.LogicalReplicationLauncher: [ @@ -67,7 +66,7 @@ def from_process(process): ProcessType.BackgroundWriter: [ 'writer' ], - } + } # yapf: disable # we deliberately cut special words and spaces cmdline = ''.join(process.cmdline()) \ From e00a16d4d14ef9062e499d7682de93a61db06513 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Fri, 23 Mar 2018 14:51:36 +0300 Subject: [PATCH 162/475] fix yapf pattern usage in test_simple.py --- tests/test_simple.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tests/test_simple.py b/tests/test_simple.py index dfbe88b6..f639e92a 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -46,10 +46,9 @@ def util_exists(util): def good_properties(f): - # yapf: disable return (os.path.exists(f) and os.path.isfile(f) and - os.access(f, os.X_OK)) + os.access(f, os.X_OK)) # yapf: disable # try to resolve it if good_properties(get_bin_path(util)): @@ -118,9 +117,8 @@ def test_init_unique_system_id(self): with get_new_node().init().start() as node0: id0 = node0.execute(query)[0] - # yapf: disable - with scoped_config(cache_initdb=True, - cached_initdb_unique=True) as config: + with scoped_config( + cache_initdb=True, cached_initdb_unique=True) as config: self.assertTrue(config.cache_initdb) self.assertTrue(config.cached_initdb_unique) @@ -371,7 +369,8 @@ def test_backup_wrong_xlog_method(self): with get_new_node() as node: node.init(allow_streaming=True).start() - with self.assertRaises(BackupException, msg='Invalid xlog_method "wrong"'): + with self.assertRaises( + BackupException, msg='Invalid xlog_method "wrong"'): node.backup(xlog_method='wrong') def test_replicate(self): From 0051f4493558bca75dddef743d9ad5e106fcdf7e Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Fri, 23 Mar 2018 15:06:01 +0300 Subject: [PATCH 163/475] fix yapf pattern usage in node.py --- testgres/consts.py | 6 ++-- testgres/node.py | 87 +++++++++++++++++++--------------------------- 2 files changed, 40 insertions(+), 53 deletions(-) diff --git a/testgres/consts.py b/testgres/consts.py index 5ca5b747..123a034c 100644 --- a/testgres/consts.py +++ b/testgres/consts.py @@ -25,5 +25,7 @@ UTILS_LOG_FILE = "utils.log" BACKUP_LOG_FILE = "backup.log" -# default replication slots number -REPLICATION_SLOTS = 10 +# defaults for node settings +MAX_REPLICATION_SLOTS = 10 +MAX_WAL_SENDERS = 10 +WAL_KEEP_SEGMENTS = 20 diff --git a/testgres/node.py b/testgres/node.py index 62990520..0787c997 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -32,8 +32,12 @@ RECOVERY_CONF_FILE, \ PG_LOG_FILE, \ UTILS_LOG_FILE, \ - PG_PID_FILE, \ - REPLICATION_SLOTS + PG_PID_FILE + +from .consts import \ + MAX_WAL_SENDERS, \ + MAX_REPLICATION_SLOTS, \ + WAL_KEEP_SEGMENTS from .decorators import \ method_decorator, \ @@ -292,12 +296,11 @@ def _create_recovery_conf(self, username, slot=None): master = self.master assert master is not None - # yapf: disable conninfo = ( u"application_name={} " u"port={} " u"user={} " - ).format(self.name, master.port, username) + ).format(self.name, master.port, username) # yapf: disable # host is tricky try: @@ -307,11 +310,10 @@ def _create_recovery_conf(self, username, slot=None): except ValueError: conninfo += u"host={}".format(master.host) - # yapf: disable line = ( "primary_conninfo='{}'\n" "standby_mode=on\n" - ).format(conninfo) + ).format(conninfo) # yapf: disable if slot: # Connect to master for some additional actions @@ -325,7 +327,8 @@ def _create_recovery_conf(self, username, slot=None): """, slot) if res[0][0]: - raise TestgresException("Slot '{}' already exists".format(slot)) + raise TestgresException( + "Slot '{}' already exists".format(slot)) # TODO: we should drop this slot after replica's cleanup() con.execute(""" @@ -357,7 +360,7 @@ def _collect_special_files(self): (os.path.join(self.data_dir, RECOVERY_CONF_FILE), 0), (os.path.join(self.data_dir, HBA_CONF_FILE), 0), (self.pg_log_file, testgres_config.error_log_lines) - ] + ] # yapf: disable for f, num_lines in files: # skip missing files @@ -392,9 +395,10 @@ def init(self, initdb_params=None, **kwargs): """ # initialize this PostgreSQL node - cached_initdb(data_dir=self.data_dir, - logfile=self.utils_log_file, - params=initdb_params) + cached_initdb( + data_dir=self.data_dir, + logfile=self.utils_log_file, + params=initdb_params) # initialize default config files self.default_conf(**kwargs) @@ -446,12 +450,11 @@ def get_auth_method(t): auth_local = get_auth_method('local') auth_host = get_auth_method('host') - # yapf: disable new_lines = [ u"local\treplication\tall\t\t\t{}\n".format(auth_local), u"host\treplication\tall\t127.0.0.1/32\t{}\n".format(auth_host), u"host\treplication\tall\t::1/128\t\t{}\n".format(auth_host) - ] + ] # yapf: disable # write missing lines for line in new_lines: @@ -466,12 +469,11 @@ def get_auth_method(t): if not fsync: conf.write(u"fsync = off\n") - # yapf: disable conf.write(u"log_statement = {}\n" u"listen_addresses = '{}'\n" u"port = {}\n".format(log_statement, self.host, - self.port)) + self.port)) # yapf: disable # replication-related settings if allow_streaming: @@ -482,17 +484,14 @@ def get_auth_method(t): else: wal_level = "hot_standby" - # yapf: disable - max_wal_senders = 10 # default in PG 10 - wal_keep_segments = 20 # for convenience conf.write(u"hot_standby = on\n" u"max_wal_senders = {}\n" u"max_replication_slots = {}\n" u"wal_keep_segments = {}\n" - u"wal_level = {}\n".format(max_wal_senders, - REPLICATION_SLOTS, - wal_keep_segments, - wal_level)) + u"wal_level = {}\n".format(MAX_WAL_SENDERS, + MAX_REPLICATION_SLOTS, + WAL_KEEP_SEGMENTS, + wal_level)) # yapf: disable # disable UNIX sockets if asked to if not unix_sockets: @@ -528,12 +527,11 @@ def status(self): """ try: - # yapf: disable _params = [ get_bin_path("pg_ctl"), "-D", self.data_dir, "status" - ] + ] # yapf: disable execute_utility(_params, self.utils_log_file) return NodeStatus.Running @@ -577,14 +575,13 @@ def start(self, params=[]): This instance of PostgresNode. """ - # yapf: disable _params = [ get_bin_path("pg_ctl"), "-D", self.data_dir, "-l", self.pg_log_file, "-w", # wait "start" - ] + params + ] + params # yapf: disable try: execute_utility(_params, self.utils_log_file) @@ -608,13 +605,12 @@ def stop(self, params=[]): This instance of PostgresNode. """ - # yapf: disable _params = [ get_bin_path("pg_ctl"), "-D", self.data_dir, "-w", # wait "stop" - ] + params + ] + params # yapf: disable execute_utility(_params, self.utils_log_file) @@ -633,14 +629,13 @@ def restart(self, params=[]): This instance of PostgresNode. """ - # yapf: disable _params = [ get_bin_path("pg_ctl"), "-D", self.data_dir, "-l", self.pg_log_file, "-w", # wait "restart" - ] + params + ] + params # yapf: disable try: execute_utility(_params, self.utils_log_file) @@ -664,13 +659,12 @@ def reload(self, params=[]): This instance of PostgresNode. """ - # yapf: disable _params = [ get_bin_path("pg_ctl"), "-D", self.data_dir, "-w", # wait "reload" - ] + params + ] + params # yapf: disable execute_utility(_params, self.utils_log_file) @@ -685,12 +679,11 @@ def pg_ctl(self, params): Stdout + stderr of pg_ctl. """ - # yapf: disable _params = [ get_bin_path("pg_ctl"), "-D", self.data_dir, "-w" # wait - ] + params + ] + params # yapf: disable return execute_utility(_params, self.utils_log_file) @@ -753,7 +746,6 @@ def psql(self, dbname = dbname or default_dbname() username = username or default_username() - # yapf: disable psql_params = [ get_bin_path("psql"), "-p", str(self.port), @@ -764,7 +756,7 @@ def psql(self, "-t", # print rows only "-q", # run quietly dbname - ] + ] # yapf: disable # select query source if query: @@ -831,7 +823,6 @@ def tmpfile(): username = username or default_username() filename = filename or tmpfile() - # yapf: disable _params = [ get_bin_path("pg_dump"), "-p", str(self.port), @@ -839,7 +830,7 @@ def tmpfile(): "-f", filename, "-U", username, "-d", dbname - ] + ] # yapf: disable execute_utility(_params, self.utils_log_file) @@ -948,7 +939,7 @@ def execute(self, with self.connect(dbname=dbname, username=username, - password=password) as node_con: + password=password) as node_con: # yapf: disable res = node_con.execute(query) @@ -1007,7 +998,7 @@ def catchup(self, dbname=None, username=None): # fetch latest LSN lsn = self.master.execute(query=poll_lsn, dbname=dbname, - username=username)[0][0] + username=username)[0][0] # yapf: disable # wait until this LSN reaches replica self.poll_query_until( @@ -1042,13 +1033,12 @@ def pgbench(self, dbname = dbname or default_dbname() username = username or default_username() - # yapf: disable _params = [ get_bin_path("pgbench"), "-p", str(self.port), "-h", self.host, "-U", username, - ] + options + ] + options # yapf: disable # should be the last one _params.append(dbname) @@ -1070,11 +1060,7 @@ def pgbench_init(self, **kwargs): return self - def pgbench_run(self, - dbname=None, - username=None, - options=[], - **kwargs): + def pgbench_run(self, dbname=None, username=None, options=[], **kwargs): """ Run pgbench with some options. This event is logged (see self.utils_log_file). @@ -1098,13 +1084,12 @@ def pgbench_run(self, dbname = dbname or default_dbname() username = username or default_username() - # yapf: disable _params = [ get_bin_path("pgbench"), "-p", str(self.port), "-h", self.host, "-U", username, - ] + options + ] + options # yapf: disable for key, value in iteritems(kwargs): # rename keys for pgbench @@ -1114,7 +1099,7 @@ def pgbench_run(self, if not isinstance(value, bool): _params.append('--{}={}'.format(key, value)) else: - assert value is True # just in case + assert value is True # just in case _params.append('--{}'.format(key)) # should be the last one @@ -1138,4 +1123,4 @@ def connect(self, dbname=None, username=None, password=None): return NodeConnection(node=self, dbname=dbname, username=username, - password=password) + password=password) # yapf: disable From dcf1494232435439b06986ce194c6f12c6c5c254 Mon Sep 17 00:00:00 2001 From: Ildar Musin Date: Fri, 23 Mar 2018 19:56:50 +0300 Subject: [PATCH 164/475] Use Sphinx notation for class references --- LICENSE | 2 +- docs/source/testgres.rst | 65 ---------------------------------------- testgres/backup.py | 6 ++-- testgres/enums.py | 4 +-- testgres/node.py | 26 ++++++++-------- 5 files changed, 19 insertions(+), 84 deletions(-) diff --git a/LICENSE b/LICENSE index f57a1836..86b7be77 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ testgres is released under the PostgreSQL License, a liberal Open Source license, similar to the BSD or MIT licenses. -Copyright (c) 2016-2017, Postgres Professional +Copyright (c) 2016-2018, Postgres Professional Permission to use, copy, modify, and distribute this software and its documentation for any purpose, without fee, and without a written agreement is hereby granted, provided that the above copyright notice and this paragraph and the following two paragraphs appear in all copies. diff --git a/docs/source/testgres.rst b/docs/source/testgres.rst index 63eadc80..f65f09c2 100644 --- a/docs/source/testgres.rst +++ b/docs/source/testgres.rst @@ -17,22 +17,6 @@ testgres.backup :undoc-members: :show-inheritance: -testgres.cache --------------- - -.. automodule:: testgres.cache - :members: - :undoc-members: - :show-inheritance: - -testgres.config ---------------- - -.. automodule:: testgres.config - :members: - :undoc-members: - :show-inheritance: - testgres.connection ------------------- @@ -41,30 +25,6 @@ testgres.connection :undoc-members: :show-inheritance: -testgres.consts ---------------- - -.. automodule:: testgres.consts - :members: - :undoc-members: - :show-inheritance: - -testgres.decorators -------------------- - -.. automodule:: testgres.decorators - :members: - :undoc-members: - :show-inheritance: - -testgres.defaults ------------------ - -.. automodule:: testgres.defaults - :members: - :undoc-members: - :show-inheritance: - testgres.enums -------------- @@ -81,14 +41,6 @@ testgres.exceptions :undoc-members: :show-inheritance: -testgres.logger ---------------- - -.. automodule:: testgres.logger - :members: - :undoc-members: - :show-inheritance: - testgres.node ------------- @@ -96,20 +48,3 @@ testgres.node :members: :undoc-members: :show-inheritance: - -testgres.utils --------------- - -.. automodule:: testgres.utils - :members: - :undoc-members: - :show-inheritance: - - -Module contents ---------------- - -.. automodule:: testgres - :members: - :undoc-members: - :show-inheritance: diff --git a/testgres/backup.py b/testgres/backup.py index 4ae5d5ba..9974d210 100644 --- a/testgres/backup.py +++ b/testgres/backup.py @@ -43,7 +43,7 @@ def __init__(self, Create a new backup. Args: - node: PostgresNode we're going to backup. + node: :class:`.PostgresNode` we're going to backup. base_dir: where should we store it? username: database user name. xlog_method: none | fetch | stream (see docs) @@ -137,7 +137,7 @@ def spawn_primary(self, name=None, destroy=True): destroy: should we convert this backup into a node? Returns: - New instance of PostgresNode. + New instance of :class:`.PostgresNode`. """ # Prepare a data directory for this node @@ -166,7 +166,7 @@ def spawn_replica(self, name=None, destroy=True, slot=None): destroy: should we convert this backup into a node? Returns: - New instance of PostgresNode. + New instance of :class:`.PostgresNode`. """ # Build a new PostgresNode diff --git a/testgres/enums.py b/testgres/enums.py index f60cf835..c145079f 100644 --- a/testgres/enums.py +++ b/testgres/enums.py @@ -4,7 +4,7 @@ class XLogMethod(Enum): """ - Available WAL methods for NodeBackup + Available WAL methods for :class:`.NodeBackup` """ none = 'none' @@ -14,7 +14,7 @@ class XLogMethod(Enum): class IsolationLevel(Enum): """ - Transaction isolation level for NodeConnection + Transaction isolation level for :class:`.NodeConnection` """ ReadUncommitted = 'read uncommitted' diff --git a/testgres/node.py b/testgres/node.py index 62990520..825cd716 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -177,7 +177,7 @@ def auxiliary_pids(self): def auxiliary_processes(self): """ Returns a list of auxiliary processes. - Each process is represented by ProcessProxy object. + Each process is represented by :class:`.ProcessProxy` object. """ def is_aux(process): @@ -189,7 +189,7 @@ def is_aux(process): def child_processes(self): """ Returns a list of all child processes. - Each process is represented by ProcessProxy object. + Each process is represented by :class:`.ProcessProxy` object. """ # get a list of postmaster's children @@ -388,7 +388,7 @@ def init(self, initdb_params=None, **kwargs): allow_streaming: should this node add a hba entry for replication? Returns: - This instance of PostgresNode. + This instance of :class:`.PostgresNode` """ # initialize this PostgreSQL node @@ -416,7 +416,7 @@ def default_conf(self, log_statement: one of ('all', 'off', 'mod', 'ddl'). Returns: - This instance of PostgresNode. + This instance of :class:`.PostgresNode`. """ postgres_conf = os.path.join(self.data_dir, PG_CONF_FILE) @@ -510,7 +510,7 @@ def append_conf(self, line, filename=PG_CONF_FILE): filename: config file (postgresql.conf by default). Returns: - This instance of PostgresNode. + This instance of :class:`.PostgresNode`. """ config_name = os.path.join(self.data_dir, filename) @@ -524,7 +524,7 @@ def status(self): Check this node's status. Returns: - An instance of NodeStatus. + An instance of :class:`.NodeStatus`. """ try: @@ -574,7 +574,7 @@ def start(self, params=[]): params: additional arguments for pg_ctl. Returns: - This instance of PostgresNode. + This instance of :class:`.PostgresNode`. """ # yapf: disable @@ -605,7 +605,7 @@ def stop(self, params=[]): params: additional arguments for pg_ctl. Returns: - This instance of PostgresNode. + This instance of :class:`.PostgresNode`. """ # yapf: disable @@ -630,7 +630,7 @@ def restart(self, params=[]): params: additional arguments for pg_ctl. Returns: - This instance of PostgresNode. + This instance of :class:`.PostgresNode`. """ # yapf: disable @@ -661,7 +661,7 @@ def reload(self, params=[]): params: additional arguments for pg_ctl. Returns: - This instance of PostgresNode. + This instance of :class:`.PostgresNode`. """ # yapf: disable @@ -713,7 +713,7 @@ def cleanup(self, max_attempts=3): max_attempts: how many times should we try to stop()? Returns: - This instance of PostgresNode. + This instance of :class:`.PostgresNode`. """ self._try_shutdown(max_attempts) @@ -1063,7 +1063,7 @@ def pgbench_init(self, **kwargs): Sets initialize=True. Returns: - This instance of PostgresNode. + This instance of :class:`.PostgresNode`. """ self.pgbench_run(initialize=True, **kwargs) @@ -1132,7 +1132,7 @@ def connect(self, dbname=None, username=None, password=None): password: user's password. Returns: - An instance of NodeConnection. + An instance of :class:`.NodeConnection`. """ return NodeConnection(node=self, From f7bd228ff26ea22e147abd8907885c1437894eca Mon Sep 17 00:00:00 2001 From: Ildar Musin Date: Mon, 26 Mar 2018 13:46:13 +0300 Subject: [PATCH 165/475] Docs for PostgresNode.__init__() method --- docs/source/testgres.rst | 11 +++++++---- testgres/api.py | 10 ++++------ testgres/node.py | 2 +- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/docs/source/testgres.rst b/docs/source/testgres.rst index f65f09c2..1da84ab3 100644 --- a/docs/source/testgres.rst +++ b/docs/source/testgres.rst @@ -44,7 +44,10 @@ testgres.exceptions testgres.node ------------- -.. automodule:: testgres.node - :members: - :undoc-members: - :show-inheritance: +.. autoclass:: testgres.node.PostgresNode + :members: + + .. automethod:: __init__ + +.. autoclass:: testgres.node.ProcessProxy + :members: \ No newline at end of file diff --git a/testgres/api.py b/testgres/api.py index 472a422a..e90cf7bd 100644 --- a/testgres/api.py +++ b/testgres/api.py @@ -29,16 +29,14 @@ ... print(replica.execute('postgres', 'select count(*) from test')) PostgresNode(name='...', port=..., base_dir='...') [(3,)] - -Copyright (c) 2016, Postgres Professional """ - -from functools import wraps - from .node import PostgresNode -@wraps(PostgresNode.__init__) def get_new_node(name=None, base_dir=None, **kwargs): + """ + Simply a wrapper around :class:`.PostgresNode` constructor. + See :meth:`.PostgresNode.__init__` for details. + """ # NOTE: leave explicit 'name' and 'base_dir' for compatibility return PostgresNode(name=name, base_dir=base_dir, **kwargs) diff --git a/testgres/node.py b/testgres/node.py index 825cd716..d2ee3446 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -92,7 +92,7 @@ def __repr__(self): class PostgresNode(object): def __init__(self, name=None, port=None, base_dir=None): """ - Create a new node. + PostgresNode constructor. Args: name: node's application name. From 7594a7e22e4f0381ad51d4643c87fcd60fabc031 Mon Sep 17 00:00:00 2001 From: Ildar Musin Date: Mon, 26 Mar 2018 15:28:30 +0300 Subject: [PATCH 166/475] Make GlobalConfig more Sphinx-friendly --- docs/source/testgres.rst | 9 +++++++++ testgres/config.py | 37 ++++++++++++++++++------------------- 2 files changed, 27 insertions(+), 19 deletions(-) diff --git a/docs/source/testgres.rst b/docs/source/testgres.rst index 1da84ab3..fd9c2d4d 100644 --- a/docs/source/testgres.rst +++ b/docs/source/testgres.rst @@ -17,6 +17,15 @@ testgres.backup :undoc-members: :show-inheritance: +testgres.config +--------------- + +.. automodule:: testgres.config + :members: + :undoc-members: + :show-inheritance: + :member-order: groupwise + testgres.connection ------------------- diff --git a/testgres/config.py b/testgres/config.py index 757e792c..d1f853c0 100644 --- a/testgres/config.py +++ b/testgres/config.py @@ -14,41 +14,39 @@ class GlobalConfig(object): """ Global configuration object which allows user to override default settings. - - Attributes: - cache_initdb: shall we use cached initdb instance? - cached_initdb_dir: path to a temp directory for cached initdb. - cached_initdb_unique: shall we give new node a unique system id? - - cache_pg_config: shall we cache pg_config results? - - temp_dir: path to temp dir containing nodes with default 'base_dir'. - - use_python_logging: enable python logging subsystem (see logger.py). - error_log_lines: N of log lines to be shown in exceptions (0=inf). - - node_cleanup_full: shall we remove EVERYTHING (including logs)? - node_cleanup_on_good_exit: remove base_dir on nominal __exit__(). - node_cleanup_on_bad_exit: remove base_dir on __exit__() via exception. - - NOTE: attributes must not be callable or begin with __. """ + # NOTE: attributes must not be callable or begin with __. cache_initdb = True - _cached_initdb_dir = None + """ shall we use cached initdb instance? """ + cached_initdb_unique = False + """ shall we give new node a unique system id? """ cache_pg_config = True + """ shall we cache pg_config results? """ use_python_logging = False + """ enable python logging subsystem (see logger.py). """ + error_log_lines = 20 + """ N of log lines to be shown in exceptions (0=inf). """ node_cleanup_full = True + """ shall we remove EVERYTHING (including logs)? """ + node_cleanup_on_good_exit = True + """ remove base_dir on nominal __exit__(). """ + node_cleanup_on_bad_exit = False + """ remove base_dir on __exit__() via exception. """ + + _cached_initdb_dir = None + """ underlying class attribute for cached_initdb_dir property """ @property def cached_initdb_dir(self): + """ path to a temp directory for cached initdb. """ return self._cached_initdb_dir @cached_initdb_dir.setter @@ -60,6 +58,7 @@ def cached_initdb_dir(self, value): @property def temp_dir(self): + """ path to temp dir containing nodes with default 'base_dir'. """ return tempfile.tempdir @temp_dir.setter From 869920c89bf8d0483ca5416813e9cfe3b6c7ea4e Mon Sep 17 00:00:00 2001 From: Ildar Musin Date: Mon, 26 Mar 2018 16:37:59 +0300 Subject: [PATCH 167/475] Replace $1 notation in queries to %s as psycopg2 expects it --- testgres/node.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index bdfcf1a7..1ca19fc5 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -210,7 +210,7 @@ def source_walsender(self): sql = """ select pid from pg_catalog.pg_stat_replication - where application_name = $1 + where application_name = %s """ if not self.master: @@ -322,7 +322,7 @@ def _create_recovery_conf(self, username, slot=None): res = con.execute(""" select exists ( select from pg_catalog.pg_replication_slots - where slot_name = $1 + where slot_name = %s ) """, slot) @@ -332,7 +332,7 @@ def _create_recovery_conf(self, username, slot=None): # TODO: we should drop this slot after replica's cleanup() con.execute(""" - select pg_catalog.pg_create_physical_replication_slot($1) + select pg_catalog.pg_create_physical_replication_slot(%s) """, slot) line += "primary_slot_name={}\n".format(slot) From bc1002f7ed82610900dcbdcd8404e8a6d842f01a Mon Sep 17 00:00:00 2001 From: Ildar Musin Date: Tue, 27 Mar 2018 15:52:13 +0300 Subject: [PATCH 168/475] Additional subscription catchup test --- tests/test_simple.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/test_simple.py b/tests/test_simple.py index 3ec69b32..bf80b888 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -457,6 +457,29 @@ def test_logical_replication(self): res = node2.execute('select * from test2') self.assertListEqual(res, [('a',), ('b',)]) + @unittest.skipUnless(pg_version_ge('10'), 'requires 10+') + def test_logical_catchup(self): + """ Runs catchup for 100 times to be sure that it is consistent """ + with get_new_node() as node1, get_new_node() as node2: + node1.init(allow_logical=True) + node1.start() + node2.init().start() + + create_table = 'create table test (key int primary key, val int); ' + node1.safe_psql(create_table) + node1.safe_psql('alter table test replica identity default') + node2.safe_psql(create_table) + + # create publication / create subscription + sub = node2.subscribe(node1.publish('mypub'), 'mysub') + + for i in range(0, 100): + node1.execute('insert into test values ({0}, {0})'.format(i)) + sub.catchup() + res = node2.execute('select * from test') + self.assertListEqual(res, [(i, i,)]) + node1.execute('delete from test') + @unittest.skipIf(pg_version_ge('10'), 'requires <10') def test_logical_replication_fail(self): with get_new_node() as node: From 08ed6ef9719737daa7a797d741afbfe3c172d477 Mon Sep 17 00:00:00 2001 From: Ildar Musin Date: Tue, 27 Mar 2018 16:34:08 +0300 Subject: [PATCH 169/475] Some refactoring of logical replication API and formatting --- testgres/node.py | 34 ++++++++++++++++------------------ testgres/pubsub.py | 11 ++++++----- 2 files changed, 22 insertions(+), 23 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index 8994f605..d50b4c06 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -501,8 +501,9 @@ def get_auth_method(t): if allow_logical: if not pg_version_ge('10'): - raise InitNodeException("Logical replication is only " - "available for Postgres 10 and newer") + raise InitNodeException( + "Logical replication is only available for Postgres 10 " + "and newer") conf.write(u"wal_level = logical\n") # disable UNIX sockets if asked to @@ -1022,11 +1023,7 @@ def catchup(self, dbname=None, username=None): except Exception as e: raise_from(CatchUpException("Failed to catch up", poll_lsn), e) - def publish(self, - name, - tables=None, - dbname=None, - username=None): + def publish(self, name, **kwargs): """ Create publication for logical replication @@ -1036,25 +1033,26 @@ def publish(self, dbname: database name where objects or interest are located username: replication username """ - return Publication(name=name, node=self, tables=tables, dbname=dbname, - username=username) + return Publication(name=name, node=self, **kwargs) - def subscribe(self, - publication, - name, - dbname=None, - username=None, - **kwargs): + def subscribe(self, publication, name, dbname=None, username=None, + **params): """ Create subscription for logical replication Args: - subname: subscription name + name: subscription name publication: publication object obtained from publish() - + dbname: database name + username: replication username + params: subscription parameters (see documentation on `CREATE SUBSCRIPTION + `_ + for details) """ + # yapf: disable return Subscription(name=name, node=self, publication=publication, - dbname=dbname, username=username, **kwargs) + dbname=dbname, username=username, **params) + # yapf: enable def pgbench(self, dbname=None, diff --git a/testgres/pubsub.py b/testgres/pubsub.py index e19fc4a1..b22b0068 100644 --- a/testgres/pubsub.py +++ b/testgres/pubsub.py @@ -106,7 +106,7 @@ def __init__(self, publication, dbname=None, username=None, - **kwargs): + **params): """ Constructor. Use :meth:`.PostgresNode.subscribe()` instead of direct constructing subscription objects. @@ -118,8 +118,9 @@ def __init__(self, (see :meth:`.PostgresNode.publish()`) dbname: database name used to connect and perform subscription username: username used to connect to the database - **kwargs: subscription parameters (see ``CREATE SUBSCRIPTION`` - in PostgreSQL documentation for more information) + params: subscription parameters (see documentation on `CREATE SUBSCRIPTION + `_ + for details) """ self.name = name self.node = node @@ -138,8 +139,8 @@ def __init__(self, name, options_string(**conninfo), self.pub.name) # additional parameters - if kwargs: - query += " with ({})".format(options_string(**kwargs)) + if params: + query += " with ({})".format(options_string(**params)) node.safe_psql(query, dbname=dbname, username=username) From 34fdee25038b2dedd989912f28bb2f6fcf8cba7f Mon Sep 17 00:00:00 2001 From: Ildar Musin Date: Mon, 2 Apr 2018 14:03:07 +0300 Subject: [PATCH 170/475] PostgresNode.reload() now returns self --- testgres/node.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/testgres/node.py b/testgres/node.py index 1ca19fc5..d68f3d34 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -668,6 +668,8 @@ def reload(self, params=[]): execute_utility(_params, self.utils_log_file) + return self + def pg_ctl(self, params): """ Invoke pg_ctl with params. From c6e61b952acdb0488ccac4b5535e02b5821f33cb Mon Sep 17 00:00:00 2001 From: Ildar Musin Date: Mon, 2 Apr 2018 14:22:02 +0300 Subject: [PATCH 171/475] Add PostgresNode.promote() method which promotes standby node to master --- testgres/node.py | 22 ++++++++++++++++++++++ tests/test_simple.py | 14 ++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/testgres/node.py b/testgres/node.py index d68f3d34..b3e2d56b 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -670,6 +670,28 @@ def reload(self, params=[]): return self + def promote(self): + """ + Promote standby instance to master using pg_ctl. + + Returns: + This instance of :class:`.PostgresNode`. + """ + + _params = [ + get_bin_path("pg_ctl"), + "-D", self.data_dir, + "-w", # wait + "promote" + ] # yapf: disable + + execute_utility(_params, self.utils_log_file) + + # Node becomes master itself + self._master = None + + return self + def pg_ctl(self, params): """ Invoke pg_ctl with params. diff --git a/tests/test_simple.py b/tests/test_simple.py index f639e92a..cb6b30c3 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -407,6 +407,20 @@ def test_incorrect_catchup(self): with self.assertRaises(TestgresException): node.catchup() + def test_promotion(self): + with get_new_node() as master: + master.init().start() + master.safe_psql('create table abc(id serial)') + + with master.replicate().start() as replica: + master.stop() + replica.promote() + + # make standby becomes writable master + replica.safe_psql('insert into abc values (1)') + res = replica.safe_psql('select * from abc') + self.assertEqual(res, b'1\n') + def test_dump(self): query_create = 'create table test as select generate_series(1, 2) as val' query_select = 'select * from test order by val asc' From 06d10056909249fa5ea1df09fe70cd7e8b4d5e56 Mon Sep 17 00:00:00 2001 From: Ildar Musin Date: Mon, 2 Apr 2018 14:57:32 +0300 Subject: [PATCH 172/475] Add synchronous promotion support on pg of version < 10 --- testgres/node.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index b3e2d56b..6e3797a4 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -662,7 +662,6 @@ def reload(self, params=[]): _params = [ get_bin_path("pg_ctl"), "-D", self.data_dir, - "-w", # wait "reload" ] + params # yapf: disable @@ -670,9 +669,12 @@ def reload(self, params=[]): return self - def promote(self): + def promote(self, dbname=None, username=None): """ - Promote standby instance to master using pg_ctl. + Promote standby instance to master using pg_ctl. For PostgreSQL versions + below 10 some additional actions required to ensure that instance + became writable and hence `dbname` and `username` parameters may be + needed. Returns: This instance of :class:`.PostgresNode`. @@ -687,7 +689,19 @@ def promote(self): execute_utility(_params, self.utils_log_file) - # Node becomes master itself + # for versions below 10 `promote` is asynchronous so we need to wait + # until it actually becomes writable + if not pg_version_ge("10"): + check_query = "SHOW transaction_read_only" + + self.poll_query_until( + query=check_query, + expected="on", + dbname=dbname, + username=username, + max_attempts=0) # infinite + + # node becomes master itself self._master = None return self From 404acfbd2df2b88cf3b895558d03e2aa8613a7dd Mon Sep 17 00:00:00 2001 From: Ildus Kurbangaliev Date: Tue, 10 Apr 2018 12:21:33 +0300 Subject: [PATCH 173/475] Add wait option to start and stop functions --- testgres/node.py | 10 ++++++---- tests/test_simple.py | 5 +++++ 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index d68f3d34..71d8952f 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -564,12 +564,13 @@ def get_control_data(self): return out_dict - def start(self, params=[]): + def start(self, params=[], wait=True): """ Start this node using pg_ctl. Args: params: additional arguments for pg_ctl. + wait: wait until operation completes Returns: This instance of :class:`.PostgresNode`. @@ -579,7 +580,7 @@ def start(self, params=[]): get_bin_path("pg_ctl"), "-D", self.data_dir, "-l", self.pg_log_file, - "-w", # wait + "-w" if wait else '-W', # --wait or --no-wait "start" ] + params # yapf: disable @@ -594,12 +595,13 @@ def start(self, params=[]): return self - def stop(self, params=[]): + def stop(self, params=[], wait=True): """ Stop this node using pg_ctl. Args: params: additional arguments for pg_ctl. + wait: wait until operation completes Returns: This instance of :class:`.PostgresNode`. @@ -608,7 +610,7 @@ def stop(self, params=[]): _params = [ get_bin_path("pg_ctl"), "-D", self.data_dir, - "-w", # wait + "-w" if wait else '-W', # --wait or --no-wait "stop" ] + params # yapf: disable diff --git a/tests/test_simple.py b/tests/test_simple.py index f639e92a..bbb1b9bc 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -373,6 +373,11 @@ def test_backup_wrong_xlog_method(self): BackupException, msg='Invalid xlog_method "wrong"'): node.backup(xlog_method='wrong') + def test_pg_ctl_wait_option(self): + with get_new_node() as node: + node.init().start(wait=False) + node.stop(wait=False) + def test_replicate(self): with get_new_node() as node: node.init(allow_streaming=True).start() From 9652bb970c7a456bc4600cf59b9244a2ed0f6665 Mon Sep 17 00:00:00 2001 From: Ildus Kurbangaliev Date: Tue, 10 Apr 2018 12:36:57 +0300 Subject: [PATCH 174/475] Fix test for wait option --- testgres/node.py | 4 ++-- tests/test_simple.py | 10 +++++++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index 71d8952f..f93f8787 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -570,7 +570,7 @@ def start(self, params=[], wait=True): Args: params: additional arguments for pg_ctl. - wait: wait until operation completes + wait: wait until operation completes. Returns: This instance of :class:`.PostgresNode`. @@ -601,7 +601,7 @@ def stop(self, params=[], wait=True): Args: params: additional arguments for pg_ctl. - wait: wait until operation completes + wait: wait until operation completes. Returns: This instance of :class:`.PostgresNode`. diff --git a/tests/test_simple.py b/tests/test_simple.py index bbb1b9bc..b8670aed 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -376,7 +376,15 @@ def test_backup_wrong_xlog_method(self): def test_pg_ctl_wait_option(self): with get_new_node() as node: node.init().start(wait=False) - node.stop(wait=False) + while True: + try: + node.stop(wait=False) + except ExecUtilException: + # it's ok to break here since node could be not + # started yet + continue + else: + break def test_replicate(self): with get_new_node() as node: From 2dc0659889ccf714ef4092d0e849be1566f14f60 Mon Sep 17 00:00:00 2001 From: Ildus Kurbangaliev Date: Tue, 10 Apr 2018 13:23:51 +0300 Subject: [PATCH 175/475] Refactor test for test option --- tests/test_simple.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/test_simple.py b/tests/test_simple.py index b8670aed..9818e6a3 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -379,12 +379,11 @@ def test_pg_ctl_wait_option(self): while True: try: node.stop(wait=False) - except ExecUtilException: - # it's ok to break here since node could be not - # started yet - continue - else: break + except ExecUtilException: + # it's ok to get this exception here since node + # could be not started yet + pass def test_replicate(self): with get_new_node() as node: From 6596a9f6830d52f451684c8e719548067e509f7f Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Thu, 19 Apr 2018 17:36:38 +0300 Subject: [PATCH 176/475] add long_description to setup.py, bump version to 1.7.0 --- setup.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 4972e15c..3690a5ed 100644 --- a/setup.py +++ b/setup.py @@ -16,12 +16,18 @@ if sys.version_info < (3, 3): install_requires.append("ipaddress") +# Get contents of README file +with open('README.md', 'r') as f: + readme = f.read() + setup( - version='1.6.0', + version='1.7.0', name='testgres', packages=['testgres'], description='Testing utility for PostgreSQL and its extensions', url='https://github.com/postgrespro/testgres', + long_description=readme, + long_description_content_type='text/markdown', license='PostgreSQL', author='Ildar Musin', author_email='zildermann@gmail.com', From c40d93997429936dee9ed2230b9dcdbd3409bb6e Mon Sep 17 00:00:00 2001 From: Ivan Kartyshov Date: Sat, 28 Apr 2018 16:51:52 +0300 Subject: [PATCH 177/475] Tests fix and cleanup code --- testgres/consts.py | 3 ++ testgres/node.py | 21 +++--------- tests/test_simple.py | 81 +++++++------------------------------------- 3 files changed, 20 insertions(+), 85 deletions(-) diff --git a/testgres/consts.py b/testgres/consts.py index 0acdefa5..44650c6c 100644 --- a/testgres/consts.py +++ b/testgres/consts.py @@ -13,6 +13,9 @@ UTILS_LOG_FILE = "utils.log" BACKUP_LOG_FILE = "backup.log" +# name of dump/restore formates +DUMP_DIRECTORY = "directory" + # default argument values DEFAULT_XLOG_METHOD = "fetch" DEFAULT_DUMP_FORMAT = "plain" diff --git a/testgres/node.py b/testgres/node.py index 756a3fb6..cc11847e 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -29,7 +29,8 @@ PG_LOG_FILE, \ UTILS_LOG_FILE, \ DEFAULT_XLOG_METHOD, \ - DEFAULT_DUMP_FORMAT + DEFAULT_DUMP_FORMAT, \ + DUMP_DIRECTORY from .exceptions import \ CatchUpException, \ @@ -705,7 +706,7 @@ def dump(self, filename=None, dbname=None, username=None, format=DEFAULT_DUMP_FO """ def tmpfile(): - if format == 'directory': + if format == DUMP_DIRECTORY: fname = tempfile.mkdtemp() else: fd, fname = tempfile.mkstemp() @@ -743,17 +744,6 @@ def restore(self, filename, dbname=None, username=None): username: database user name. """ - # self.psql(filename=filename, dbname=dbname, username=username) - # yapf: disable - # _params = [ - # get_bin_path("pg_restore"), - # "-p", str(self.port), - # "-h", self.host, - # filename, - # "-U", username, - # "-d", dbname - # ] - # Set default arguments dbname = dbname or default_dbname() username = username or default_username() @@ -762,13 +752,10 @@ def restore(self, filename, dbname=None, username=None): get_bin_path("pg_restore"), "-p", str(self.port), "-h", self.host, - # filename, - # "-c", - # "-U", username, + "-U", username, "-d", dbname, filename ] - print(_params) execute_utility(_params, self.utils_log_name) diff --git a/tests/test_simple.py b/tests/test_simple.py index 2f16ae07..bbfc2167 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -11,8 +11,6 @@ import logging.config import shutil -import pdb - from distutils.version import LooseVersion from testgres import \ @@ -375,96 +373,43 @@ def test_dump(self): dump = node1.dump() self.assertTrue(os.path.isfile(dump)) with get_new_node().init().start() as node2: - # restore dump - node2.restore(filename=dump) + node2.psql(filename=dump, dbname=None, username=None) res = node2.execute(query_select) self.assertListEqual(res, [(1, ), (2, )]) - # finally, remove dump os.remove(dump) - def test_dump_plain(self): - query_create = 'create table test as select generate_series(1, 2) as val' - query_select = 'select * from test order by val asc' - - with get_new_node().init().start() as node1: - - node1.execute(query_create) - - # take a new dump plain format dump = node1.dump(format='plain') self.assertTrue(os.path.isfile(dump)) - with get_new_node().init().start() as node2: - # node2.restore(filename=dump) - node2.psql(filename=dump, dbname=None, username=None) #add - res = node2.execute(query_select) + with get_new_node().init().start() as node3: + node3.psql(filename=dump, dbname=None, username=None) + res = node3.execute(query_select) self.assertListEqual(res, [(1, ), (2, )]) os.remove(dump) - # with get_new_node().init().start() as node2: - # for f in ['plain', 'custom', 'directory', 'tar']: - # # dump - # dump = node1.dump(format=f) - # # create database - # # connect - # # restore - # node2.restore(filename=f) - # # check - # res = node2.execute(query_select) - # self.assertListEqual(res, [(1, ), (2, )]) - # # drop database - # os.remove(f) - - def test_dump_custom(self): - query_create = 'create table test as select generate_series(1, 2) as val' - query_select = 'select * from test order by val asc' - - with get_new_node().init().start() as node1: - - node1.execute(query_create) - # take a new dump custom format dump = node1.dump(format='custom') self.assertTrue(os.path.isfile(dump)) - with get_new_node().init().start() as node2: - node2.restore(filename=dump) - res = node2.execute(query_select) - print(res) + with get_new_node().init().start() as node4: + node4.restore(filename=dump) + res = node4.execute(query_select) self.assertListEqual(res, [(1, ), (2, )]) os.remove(dump) - def test_dump_directory(self): - query_create = 'create table test as select generate_series(1, 2) as val' - query_select = 'select * from test order by val asc' - - with get_new_node().init().start() as node1: - - node1.execute(query_create) - # take a new dump directory format dump = node1.dump(format='directory') self.assertTrue(os.path.isdir(dump)) - with get_new_node().init().start() as node2: - node2.restore(filename=dump) - res = node2.execute(query_select) + with get_new_node().init().start() as node5: + node5.restore(filename=dump) + res = node5.execute(query_select) self.assertListEqual(res, [(1, ), (2, )]) shutil.rmtree(dump, ignore_errors=True) - def test_dump_tar(self): - query_create = 'create table test as select generate_series(1, 2) as val' - query_select = 'select * from test order by val asc' - - with get_new_node().init().start() as node1: - - node1.execute(query_create) - # take a new dump tar format dump = node1.dump(format='tar') self.assertTrue(os.path.isfile(dump)) - with get_new_node().init().start() as node2: - node2.restore(filename=dump) - print("Restore finished") - pdb.set_trace() - res = node2.execute(query_select) + with get_new_node().init().start() as node6: + node6.restore(filename=dump) + res = node6.execute(query_select) self.assertListEqual(res, [(1, ), (2, )]) os.remove(dump) From 458e2b0891d608b249a3d8e92e932c5dddc0627b Mon Sep 17 00:00:00 2001 From: Ivan Kartyshov Date: Sat, 28 Apr 2018 18:22:57 +0300 Subject: [PATCH 178/475] Fixes and flake8 cleanups --- testgres/consts.py | 4 ---- testgres/enums.py | 11 +++++++++++ testgres/node.py | 29 +++++++++++++++-------------- tests/test_simple.py | 1 - 4 files changed, 26 insertions(+), 19 deletions(-) diff --git a/testgres/consts.py b/testgres/consts.py index bad852b8..123a034c 100644 --- a/testgres/consts.py +++ b/testgres/consts.py @@ -25,10 +25,6 @@ UTILS_LOG_FILE = "utils.log" BACKUP_LOG_FILE = "backup.log" -# name of dump/restore formates -DUMP_DIRECTORY = "directory" -DEFAULT_DUMP_FORMAT = "plain" - # defaults for node settings MAX_REPLICATION_SLOTS = 10 MAX_WAL_SENDERS = 10 diff --git a/testgres/enums.py b/testgres/enums.py index 1f6869de..fb68f2bb 100644 --- a/testgres/enums.py +++ b/testgres/enums.py @@ -85,3 +85,14 @@ def from_process(process): # default return ProcessType.Unknown + + +class DumpFormat(Enum): + """ + Available dump formats + """ + + Plain = 'plain' + Custom = 'custom' + Directory = 'directory' + Tar = 'tar' diff --git a/testgres/node.py b/testgres/node.py index 24e66cfc..a2595c55 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -10,7 +10,10 @@ from six import raise_from, iteritems from tempfile import mkstemp, mkdtemp -from .enums import NodeStatus, ProcessType +from .enums import \ + NodeStatus, \ + ProcessType, \ + DumpFormat from .cache import cached_initdb @@ -32,8 +35,6 @@ RECOVERY_CONF_FILE, \ PG_LOG_FILE, \ UTILS_LOG_FILE, \ - DEFAULT_DUMP_FORMAT, \ - DUMP_DIRECTORY, \ PG_PID_FILE from .consts import \ @@ -805,23 +806,27 @@ def safe_psql(self, query=None, **kwargs): return out - def dump(self, filename=None, dbname=None, username=None, format=DEFAULT_DUMP_FORMAT): + def dump(self, + filename=None, + dbname=None, + username=None, + format=DumpFormat.Plain): """ Dump database into a file using pg_dump. NOTE: the file is not removed automatically. Args: + filename: database dump taken by pg_dump. dbname: database name to connect to. username: database user name. - filename: output file. - format: format argument plain/custom/directory/tar + format: format argument plain/custom/directory/tar. Returns: Path to a file containing dump. """ def tmpfile(): - if format == DUMP_DIRECTORY: + if format == DumpFormat.Directory: fname = mkdtemp(prefix=TMP_DUMP) else: fd, fname = mkstemp(prefix=TMP_DUMP) @@ -852,7 +857,7 @@ def restore(self, filename, dbname=None, username=None): Restore database from pg_dump's file. Args: - filename: input file. + filename: database dump taken by pg_dump in custom/directory/tar formats. dbname: database name to connect to. username: database user name. """ @@ -862,17 +867,13 @@ def restore(self, filename, dbname=None, username=None): username = username or default_username() _params = [ - get_bin_path("pg_restore"), - "-p", str(self.port), - "-h", self.host, - "-U", username, - "-d", dbname, + get_bin_path("pg_restore"), "-p", + str(self.port), "-h", self.host, "-U", username, "-d", dbname, filename ] execute_utility(_params, self.utils_log_name) - @method_decorator(positional_args_hack(['dbname', 'query'])) def poll_query_until(self, query, diff --git a/tests/test_simple.py b/tests/test_simple.py index 8b4b7c60..5caf3628 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -10,7 +10,6 @@ import unittest import logging.config -import shutil from contextlib import contextmanager from distutils.version import LooseVersion From b7cdcb760ce95e7519101d1e5d5a2e85fa33ade6 Mon Sep 17 00:00:00 2001 From: Ivan Kartyshov Date: Sun, 29 Apr 2018 00:28:40 +0300 Subject: [PATCH 179/475] Fix tests using enum --- testgres/node.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index a2595c55..3e42f87c 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -810,7 +810,7 @@ def dump(self, filename=None, dbname=None, username=None, - format=DumpFormat.Plain): + format=DumpFormat.Plain.value): """ Dump database into a file using pg_dump. NOTE: the file is not removed automatically. @@ -826,7 +826,7 @@ def dump(self, """ def tmpfile(): - if format == DumpFormat.Directory: + if format == DumpFormat.Directory.value: fname = mkdtemp(prefix=TMP_DUMP) else: fd, fname = mkstemp(prefix=TMP_DUMP) From a4d1ab931139e57b8072fdc0cc033e5348766df5 Mon Sep 17 00:00:00 2001 From: Ivan Kartyshov Date: Sun, 29 Apr 2018 13:55:10 +0300 Subject: [PATCH 180/475] changes for test coverage --- tests/test_simple.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/test_simple.py b/tests/test_simple.py index 5caf3628..a89f4991 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -428,13 +428,13 @@ def test_dump(self): node1.execute(query_create) # take a new dump plain format - dump = node1.dump() - self.assertTrue(os.path.isfile(dump)) - with get_new_node().init().start() as node2: - node2.psql(filename=dump, dbname=None, username=None) - res = node2.execute(query_select) - self.assertListEqual(res, [(1, ), (2, )]) - os.remove(dump) + with removing(node1.dump()) as dump: + with get_new_node().init().start() as node2: + # restore dump + self.assertTrue(os.path.isfile(dump)) + node2.psql(filename=dump, dbname=None, username=None) + res = node2.execute(query_select) + self.assertListEqual(res, [(1, ), (2, )]) dump = node1.dump(format='plain') self.assertTrue(os.path.isfile(dump)) From a656ac1fc7b653f304c909b40d676c1c4a3b0db9 Mon Sep 17 00:00:00 2001 From: Ivan Kartyshov Date: Tue, 8 May 2018 13:43:44 +0300 Subject: [PATCH 181/475] pg_dump enum using cleanup --- testgres/node.py | 21 +++++++++++++++++---- tests/test_simple.py | 4 ++-- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index 3e42f87c..5bcab2f7 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -810,7 +810,7 @@ def dump(self, filename=None, dbname=None, username=None, - format=DumpFormat.Plain.value): + format=DumpFormat.Plain): """ Dump database into a file using pg_dump. NOTE: the file is not removed automatically. @@ -825,8 +825,17 @@ def dump(self, Path to a file containing dump. """ + # Check arguments + if not isinstance(format, DumpFormat): + try: + format = DumpFormat(format) + except ValueError: + msg = 'Invalid format "{}"'.format(format) + raise BackupException(msg) + + # Generate tmpfile or tmpdir def tmpfile(): - if format == DumpFormat.Directory.value: + if format == DumpFormat.Directory: fname = mkdtemp(prefix=TMP_DUMP) else: fd, fname = mkstemp(prefix=TMP_DUMP) @@ -845,7 +854,7 @@ def tmpfile(): "-f", filename, "-U", username, "-d", dbname, - "-F", format + "-F", format.value ] # yapf: disable execute_utility(_params, self.utils_log_file) @@ -872,7 +881,11 @@ def restore(self, filename, dbname=None, username=None): filename ] - execute_utility(_params, self.utils_log_name) + # try pg_restore if dump is binary formate, and psql if not + try: + execute_utility(_params, self.utils_log_name) + except ExecUtilException as e: + self.psql(filename=filename, dbname=dbname, username=username) @method_decorator(positional_args_hack(['dbname', 'query'])) def poll_query_until(self, diff --git a/tests/test_simple.py b/tests/test_simple.py index a89f4991..2b772ad9 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -432,14 +432,14 @@ def test_dump(self): with get_new_node().init().start() as node2: # restore dump self.assertTrue(os.path.isfile(dump)) - node2.psql(filename=dump, dbname=None, username=None) + node2.restore(filename=dump) res = node2.execute(query_select) self.assertListEqual(res, [(1, ), (2, )]) dump = node1.dump(format='plain') self.assertTrue(os.path.isfile(dump)) with get_new_node().init().start() as node3: - node3.psql(filename=dump, dbname=None, username=None) + node3.restore(filename=dump) res = node3.execute(query_select) self.assertListEqual(res, [(1, ), (2, )]) os.remove(dump) From 66cc9a6d5da206ec19784b957b2b2f7e8f35bc7b Mon Sep 17 00:00:00 2001 From: Ivan Kartyshov Date: Tue, 8 May 2018 14:07:22 +0300 Subject: [PATCH 182/475] Cleanup --- testgres/node.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index 5bcab2f7..574f4743 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -57,7 +57,8 @@ QueryException, \ StartNodeException, \ TimeoutException, \ - TestgresException + TestgresException, \ + BackupException from .logger import TestgresLogger @@ -884,7 +885,7 @@ def restore(self, filename, dbname=None, username=None): # try pg_restore if dump is binary formate, and psql if not try: execute_utility(_params, self.utils_log_name) - except ExecUtilException as e: + except ExecUtilException: self.psql(filename=filename, dbname=dbname, username=username) @method_decorator(positional_args_hack(['dbname', 'query'])) From e9d2a40162f9a79eccfc999f7a43360e7c9df625 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Tue, 8 May 2018 16:13:09 +0300 Subject: [PATCH 183/475] improve hooks/pre-commit, fix warnings found by flake8 --- hooks/pre-commit | 16 +++++++++++++--- tests/test_simple.py | 8 ++++---- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/hooks/pre-commit b/hooks/pre-commit index 3dfc5794..52531d14 100755 --- a/hooks/pre-commit +++ b/hooks/pre-commit @@ -1,13 +1,23 @@ #!/bin/bash +set -e + # capture the changed files that have been staged changed_files=$(git diff --staged --name-only) for file in ${changed_files} do if [[ "${file##*.}" == "py" ]]; then - echo "Yapfing ${file}" - yapf ${file} -i - git add ${file} + if command -v yapf > /dev/null; then + echo "Run yapf on ${file}" + yapf ${file} -i + git add ${file} + fi + + if command -v flake8 > /dev/null; then + echo "Run flake8 on ${file}" + flake8 ${file} + fi fi done + diff --git a/tests/test_simple.py b/tests/test_simple.py index 9818e6a3..afa142ac 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -46,8 +46,8 @@ def util_exists(util): def good_properties(f): - return (os.path.exists(f) and - os.path.isfile(f) and + return (os.path.exists(f) and # noqa: W504 + os.path.isfile(f) and # noqa: W504 os.access(f, os.X_OK)) # yapf: disable # try to resolve it @@ -72,7 +72,7 @@ def removing(f): class TestgresTests(unittest.TestCase): def test_node_repr(self): with get_new_node() as node: - pattern = 'PostgresNode\(name=\'.+\', port=.+, base_dir=\'.+\'\)' + pattern = r"PostgresNode\(name='.+', port=.+, base_dir='.+'\)" self.assertIsNotNone(re.match(pattern, str(node))) def test_custom_init(self): @@ -263,7 +263,7 @@ def test_psql(self): # check feeding input node.safe_psql('create table horns (w int)') node.safe_psql( - 'copy horns from stdin (format csv)', input=b"1\n2\n3\n\.\n") + 'copy horns from stdin (format csv)', input=b"1\n2\n3\n\\.\n") _sum = node.safe_psql('select sum(w) from horns') self.assertEqual(_sum, b'6\n') From 70aa1daa55e4f791ddce0949297ff9ca0263e8ba Mon Sep 17 00:00:00 2001 From: Ivan Kartyshov Date: Tue, 8 May 2018 16:55:21 +0300 Subject: [PATCH 184/475] Test_dump refactoring --- tests/test_simple.py | 47 ++++++++++++-------------------------------- 1 file changed, 13 insertions(+), 34 deletions(-) diff --git a/tests/test_simple.py b/tests/test_simple.py index 899e45f0..42e77a54 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -67,6 +67,8 @@ def removing(f): finally: if os.path.isfile(f): os.remove(f) + elif os.path.isdir(f): + rmtree(f, ignore_errors=True) class TestgresTests(unittest.TestCase): @@ -436,40 +438,17 @@ def test_dump(self): res = node2.execute(query_select) self.assertListEqual(res, [(1, ), (2, )]) - dump = node1.dump(format='plain') - self.assertTrue(os.path.isfile(dump)) - with get_new_node().init().start() as node3: - node3.restore(filename=dump) - res = node3.execute(query_select) - self.assertListEqual(res, [(1, ), (2, )]) - os.remove(dump) - - # take a new dump custom format - dump = node1.dump(format='custom') - self.assertTrue(os.path.isfile(dump)) - with get_new_node().init().start() as node4: - node4.restore(filename=dump) - res = node4.execute(query_select) - self.assertListEqual(res, [(1, ), (2, )]) - os.remove(dump) - - # take a new dump directory format - dump = node1.dump(format='directory') - self.assertTrue(os.path.isdir(dump)) - with get_new_node().init().start() as node5: - node5.restore(filename=dump) - res = node5.execute(query_select) - self.assertListEqual(res, [(1, ), (2, )]) - rmtree(dump, ignore_errors=True) - - # take a new dump tar format - dump = node1.dump(format='tar') - self.assertTrue(os.path.isfile(dump)) - with get_new_node().init().start() as node6: - node6.restore(filename=dump) - res = node6.execute(query_select) - self.assertListEqual(res, [(1, ), (2, )]) - os.remove(dump) + for format in ['plain', 'custom', 'directory', 'tar']: + with removing(node1.dump(format=format)) as dump: + with get_new_node().init().start() as node3: + if format == 'directory': + self.assertTrue(os.path.isdir(dump)) + else: + self.assertTrue(os.path.isfile(dump)) + # restore dump + node3.restore(filename=dump) + res = node3.execute(query_select) + self.assertListEqual(res, [(1, ), (2, )]) def test_users(self): with get_new_node().init().start() as node: From edcf4c66f8c3106e4b752a9cd7f87369f88fb267 Mon Sep 17 00:00:00 2001 From: Ivan Kartyshov Date: Tue, 8 May 2018 17:00:16 +0300 Subject: [PATCH 185/475] Cleanup --- testgres/node.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index 574f4743..d8ce1f03 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -877,10 +877,13 @@ def restore(self, filename, dbname=None, username=None): username = username or default_username() _params = [ - get_bin_path("pg_restore"), "-p", - str(self.port), "-h", self.host, "-U", username, "-d", dbname, + get_bin_path("pg_restore"), + "-p", str(self.port), + "-h", self.host, + "-U", username, + "-d", dbname, filename - ] + ] # yapf: disable # try pg_restore if dump is binary formate, and psql if not try: From bb88d6e0506a687ebc7ce94ed672e8b12da7da3c Mon Sep 17 00:00:00 2001 From: Ivan Kartyshov Date: Tue, 8 May 2018 17:09:40 +0300 Subject: [PATCH 186/475] Cleanup test_dump --- tests/test_simple.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/tests/test_simple.py b/tests/test_simple.py index 42e77a54..33defb12 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -428,16 +428,6 @@ def test_dump(self): with get_new_node().init().start() as node1: node1.execute(query_create) - - # take a new dump plain format - with removing(node1.dump()) as dump: - with get_new_node().init().start() as node2: - # restore dump - self.assertTrue(os.path.isfile(dump)) - node2.restore(filename=dump) - res = node2.execute(query_select) - self.assertListEqual(res, [(1, ), (2, )]) - for format in ['plain', 'custom', 'directory', 'tar']: with removing(node1.dump(format=format)) as dump: with get_new_node().init().start() as node3: From 4b279efd54b23838b9db3b89bb433b6378c069e9 Mon Sep 17 00:00:00 2001 From: Ildar Musin Date: Thu, 31 May 2018 16:49:57 +0300 Subject: [PATCH 187/475] minor refactoring --- testgres/connection.py | 8 ++++++- testgres/consts.py | 3 +++ testgres/node.py | 18 +++++++++----- testgres/pubsub.py | 54 ++++++++++++++++++++++-------------------- tests/test_simple.py | 17 ++++++++----- 5 files changed, 61 insertions(+), 39 deletions(-) diff --git a/testgres/connection.py b/testgres/connection.py index 6447f685..3943a4e2 100644 --- a/testgres/connection.py +++ b/testgres/connection.py @@ -27,7 +27,12 @@ class NodeConnection(object): Transaction wrapper returned by Node """ - def __init__(self, node, dbname=None, username=None, password=None): + def __init__(self, + node, + dbname=None, + username=None, + password=None, + autocommit=False): # Set default arguments dbname = dbname or default_dbname() @@ -42,6 +47,7 @@ def __init__(self, node, dbname=None, username=None, password=None): host=node.host, port=node.port) + self._connection.autocommit = autocommit self._cursor = self.connection.cursor() @property diff --git a/testgres/consts.py b/testgres/consts.py index 123a034c..f7f01d9d 100644 --- a/testgres/consts.py +++ b/testgres/consts.py @@ -29,3 +29,6 @@ MAX_REPLICATION_SLOTS = 10 MAX_WAL_SENDERS = 10 WAL_KEEP_SEGMENTS = 20 + +# logical replication settings +LOGICAL_REPL_MAX_CATCHUP_ATTEMPTS = 60 diff --git a/testgres/node.py b/testgres/node.py index d50b4c06..dcae9adb 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -953,13 +953,11 @@ def execute(self, with self.connect(dbname=dbname, username=username, - password=password) as node_con: # yapf: disable + password=password, + autocommit=commit) as node_con: # yapf: disable res = node_con.execute(query) - if commit: - node_con.commit() - return res def backup(self, **kwargs): @@ -1152,7 +1150,11 @@ def pgbench_run(self, dbname=None, username=None, options=[], **kwargs): return execute_utility(_params, self.utils_log_file) - def connect(self, dbname=None, username=None, password=None): + def connect(self, + dbname=None, + username=None, + password=None, + autocommit=False): """ Connect to a database. @@ -1160,6 +1162,9 @@ def connect(self, dbname=None, username=None, password=None): dbname: database name to connect to. username: database user name. password: user's password. + autocommit: commit each statement automatically. Also it should be + set to `True` for statements requiring to be run outside + a transaction? such as `VACUUM` or `CREATE DATABASE`. Returns: An instance of :class:`.NodeConnection`. @@ -1168,4 +1173,5 @@ def connect(self, dbname=None, username=None, password=None): return NodeConnection(node=self, dbname=dbname, username=username, - password=password) # yapf: disable + password=password, + autocommit=autocommit) # yapf: disable diff --git a/testgres/pubsub.py b/testgres/pubsub.py index b22b0068..e257569c 100644 --- a/testgres/pubsub.py +++ b/testgres/pubsub.py @@ -12,7 +12,7 @@ After that :meth:`~.PostgresNode.publish()` and :meth:`~.PostgresNode.subscribe()` methods may be used to setup replication. Example: ->>> from .api import get_new_node +>>> from testgres import get_new_node >>> with get_new_node() as nodeA, get_new_node() as nodeB: ... nodeA.init(allow_logical=True).start() ... nodeB.init().start() @@ -44,6 +44,7 @@ from six import raise_from +from .consts import LOGICAL_REPL_MAX_CATCHUP_ATTEMPTS from .defaults import default_dbname, default_username from .exceptions import CatchUpException from .utils import options_string @@ -56,11 +57,11 @@ def __init__(self, name, node, tables=None, dbname=None, username=None): constructing publication objects. Args: - name: publication name - node: publisher's node - tables: tables list or None for all tables - dbname: database name used to connect and perform subscription - username: username used to connect to the database + name: publication name. + node: publisher's node. + tables: tables list or None for all tables. + dbname: database name used to connect and perform subscription. + username: username used to connect to the database. """ self.name = name self.node = node @@ -70,7 +71,7 @@ def __init__(self, name, node, tables=None, dbname=None, username=None): # create publication in database t = "table " + ", ".join(tables) if tables else "all tables" query = "create publication {} for {}" - node.safe_psql(query.format(name, t), dbname=dbname, username=username) + node.execute(query.format(name, t), dbname=dbname, username=username) def drop(self, dbname=None, username=None): """ @@ -87,13 +88,13 @@ def add_tables(self, tables, dbname=None, username=None): created with empty tables list. Args: - tables: a list of tables to be added to the publication + tables: a list of tables to be added to the publication. """ if not tables: raise ValueError("Tables list is empty") query = "alter publication {} add table {}" - self.node.safe_psql( + self.node.execute( query.format(self.name, ", ".join(tables)), dbname=dbname or self.dbname, username=username or self.username) @@ -112,15 +113,15 @@ def __init__(self, constructing subscription objects. Args: - name: subscription name - node: subscriber's node + name: subscription name. + node: subscriber's node. publication: :class:`.Publication` object we are subscribing to - (see :meth:`.PostgresNode.publish()`) - dbname: database name used to connect and perform subscription - username: username used to connect to the database + (see :meth:`.PostgresNode.publish()`). + dbname: database name used to connect and perform subscription. + username: username used to connect to the database. params: subscription parameters (see documentation on `CREATE SUBSCRIPTION `_ - for details) + for details). """ self.name = name self.node = node @@ -142,28 +143,29 @@ def __init__(self, if params: query += " with ({})".format(options_string(**params)) - node.safe_psql(query, dbname=dbname, username=username) + # Note: cannot run 'create subscription' query in transaction mode + node.execute(query, dbname=dbname, username=username) def disable(self, dbname=None, username=None): """ Disables the running subscription. """ query = "alter subscription {} disable" - self.node.safe_psql(query.format(self.name), dbname=None, username=None) + self.node.execute(query.format(self.name), dbname=None, username=None) def enable(self, dbname=None, username=None): """ Enables the previously disabled subscription. """ query = "alter subscription {} enable" - self.node.safe_psql(query.format(self.name), dbname=None, username=None) + self.node.execute(query.format(self.name), dbname=None, username=None) def refresh(self, copy_data=True, dbname=None, username=None): """ Disables the running subscription. """ query = "alter subscription {} refresh publication with (copy_data={})" - self.node.safe_psql( + self.node.execute( query.format(self.name, copy_data), dbname=dbname, username=username) @@ -172,7 +174,7 @@ def drop(self, dbname=None, username=None): """ Drops subscription """ - self.node.safe_psql( + self.node.execute( "drop subscription {}".format(self.name), dbname=dbname, username=username) @@ -182,12 +184,12 @@ def catchup(self, username=None): Wait until subscription catches up with publication. Args: - username: remote node's user name + username: remote node's user name. """ - query = ( - "select pg_current_wal_lsn() - replay_lsn = 0 " - "from pg_stat_replication where application_name = '{}'").format( - self.name) + query = """ + select pg_current_wal_lsn() - replay_lsn = 0 + from pg_catalog.pg_stat_replication where application_name = '{}' + """.format(self.name) try: # wait until this LSN reaches subscriber @@ -195,6 +197,6 @@ def catchup(self, username=None): query=query, dbname=self.pub.dbname, username=username or self.pub.username, - max_attempts=60) + max_attempts=LOGICAL_REPL_MAX_CATCHUP_ATTEMPTS) except Exception as e: raise_from(CatchUpException("Failed to catch up", query), e) diff --git a/tests/test_simple.py b/tests/test_simple.py index bf80b888..c1173267 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -432,7 +432,7 @@ def test_logical_replication(self): node1.safe_psql('insert into test2 values (\'a\'), (\'b\')') sub.catchup() res = node2.execute('select * from test2') - self.assertListEqual(res, [('a',), ('b',)]) + self.assertListEqual(res, [('a', ), ('b', )]) # drop subscription sub.drop() @@ -450,12 +450,12 @@ def test_logical_replication(self): # explicitely add table with self.assertRaises(ValueError): - pub.add_tables([]) # fail + pub.add_tables([]) # fail pub.add_tables(['test2']) node1.safe_psql('insert into test2 values (\'c\')') sub.catchup() res = node2.execute('select * from test2') - self.assertListEqual(res, [('a',), ('b',)]) + self.assertListEqual(res, [('a', ), ('b', )]) @unittest.skipUnless(pg_version_ge('10'), 'requires 10+') def test_logical_catchup(self): @@ -477,7 +477,10 @@ def test_logical_catchup(self): node1.execute('insert into test values ({0}, {0})'.format(i)) sub.catchup() res = node2.execute('select * from test') - self.assertListEqual(res, [(i, i,)]) + self.assertListEqual(res, [( + i, + i, + )]) node1.execute('delete from test') @unittest.skipIf(pg_version_ge('10'), 'requires <10') @@ -544,7 +547,8 @@ def test_poll_query_until(self): # check 0 columns with self.assertRaises(QueryException): - node.poll_query_until(query='select from pg_class limit 1') + node.poll_query_until( + query='select from pg_catalog.pg_class limit 1') # check None, fail with self.assertRaises(QueryException): @@ -556,7 +560,8 @@ def test_poll_query_until(self): # check 0 rows equivalent to expected=None node.poll_query_until( - query='select * from pg_class where true = false', expected=None) + query='select * from pg_catalog.pg_class where true = false', + expected=None) # check arbitrary expected value, fail with self.assertRaises(TimeoutException): From 50e02ffe6100e0916fb239fbb55c59684c72fe8a Mon Sep 17 00:00:00 2001 From: Ildar Musin Date: Fri, 1 Jun 2018 12:45:07 +0300 Subject: [PATCH 188/475] change safe_psql() call to execute() in pubsub.py --- testgres/api.py | 2 +- testgres/pubsub.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/testgres/api.py b/testgres/api.py index 775d5ef8..e90cf7bd 100644 --- a/testgres/api.py +++ b/testgres/api.py @@ -28,7 +28,7 @@ ... replica.catchup() # wait until changes are visible ... print(replica.execute('postgres', 'select count(*) from test')) PostgresNode(name='...', port=..., base_dir='...') -[(3L,)] +[(3,)] """ from .node import PostgresNode diff --git a/testgres/pubsub.py b/testgres/pubsub.py index f8e5dcfe..bb153913 100644 --- a/testgres/pubsub.py +++ b/testgres/pubsub.py @@ -77,7 +77,7 @@ def drop(self, dbname=None, username=None): """ Drop publication """ - self.node.safe_psql( + self.node.execute( "drop publication {}".format(self.name), dbname=dbname, username=username) From 72d03735daceef76c56bd99aeb1eca3411e6230a Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Fri, 1 Jun 2018 16:21:32 +0300 Subject: [PATCH 189/475] replace pg_version_ge() with explicit per-node version --- testgres/node.py | 12 +++++++----- testgres/utils.py | 14 +++----------- tests/test_simple.py | 23 +++++++++++++++-------- 3 files changed, 25 insertions(+), 24 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index 54930a88..9978dcf6 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -66,10 +66,11 @@ from .pubsub import Publication, Subscription from .utils import \ + PgVer, \ eprint, \ get_bin_path, \ + get_pg_version, \ file_tail, \ - pg_version_ge, \ reserve_port, \ release_port, \ execute_utility, \ @@ -113,6 +114,7 @@ def __init__(self, name=None, port=None, base_dir=None): """ # private + self._pg_version = PgVer(get_pg_version()) self._should_free_port = port is None self._base_dir = base_dir self._logger = None @@ -489,7 +491,7 @@ def get_auth_method(t): if allow_streaming: # select a proper wal_level for PostgreSQL - if pg_version_ge('9.6'): + if self._pg_version >= '9.6': wal_level = "replica" else: wal_level = "hot_standby" @@ -504,7 +506,7 @@ def get_auth_method(t): wal_level)) # yapf: disable if allow_logical: - if not pg_version_ge('10'): + if self._pg_version < '10': raise InitNodeException( "Logical replication is only available for Postgres 10 " "and newer") @@ -568,7 +570,7 @@ def get_control_data(self): # this one is tricky (blame PG 9.4) _params = [get_bin_path("pg_controldata")] - _params += ["-D"] if pg_version_ge('9.5') else [] + _params += ["-D"] if self._pg_version >= '9.5' else [] _params += [self.data_dir] data = execute_utility(_params, self.utils_log_file) @@ -1042,7 +1044,7 @@ def catchup(self, dbname=None, username=None): if not self.master: raise TestgresException("Node doesn't have a master") - if pg_version_ge('10'): + if self._pg_version >= '10': poll_lsn = "select pg_catalog.pg_current_wal_lsn()::text" wait_lsn = "select pg_catalog.pg_last_wal_replay_lsn() >= '{}'::pg_lsn" else: diff --git a/testgres/utils.py b/testgres/utils.py index d8eeef53..4382f31a 100644 --- a/testgres/utils.py +++ b/testgres/utils.py @@ -22,6 +22,9 @@ # ports used by nodes bound_ports = set() +# re-export version type +PgVer = LooseVersion + def reserve_port(): """ @@ -180,17 +183,6 @@ def get_pg_version(): return version -def pg_version_ge(version): - """ - Check if PostgreSQL is 'version' or newer. - """ - - cur_ver = LooseVersion(get_pg_version()) - min_ver = LooseVersion(version) - - return cur_ver >= min_ver - - def file_tail(f, num_lines): """ Get last N lines of a file. diff --git a/tests/test_simple.py b/tests/test_simple.py index 1529c8e0..230cff47 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -12,7 +12,6 @@ import logging.config from contextlib import contextmanager -from distutils.version import LooseVersion from shutil import rmtree from testgres import \ @@ -32,16 +31,24 @@ from testgres import \ NodeStatus, \ + ProcessType, \ IsolationLevel, \ get_new_node from testgres import \ get_bin_path, \ - get_pg_config + get_pg_config, \ + get_pg_version +# NOTE: those are ugly imports from testgres import bound_ports -from testgres.utils import pg_version_ge -from testgres.enums import ProcessType +from testgres.utils import PgVer + + +def pg_version_ge(version): + cur_ver = PgVer(get_pg_version()) + min_ver = PgVer(version) + return cur_ver >= min_ver def util_exists(util): @@ -110,7 +117,7 @@ def test_init_after_cleanup(self): node.init().start().execute('select 1') @unittest.skipUnless(util_exists('pg_resetwal'), 'might be missing') - @unittest.skipUnless(pg_version_ge('9.6'), 'query works on 9.6+') + @unittest.skipUnless(pg_version_ge('9.6'), 'requires 9.6+') def test_init_unique_system_id(self): # this function exists in PostgreSQL 9.6+ query = 'select system_identifier from pg_control_system()' @@ -828,9 +835,9 @@ def test_exceptions(self): str(QueryException('msg', 'query')) def test_version_management(self): - a = LooseVersion('10.0') - b = LooseVersion('10') - c = LooseVersion('9.6.5') + a = PgVer('10.0') + b = PgVer('10') + c = PgVer('9.6.5') self.assertTrue(a > b) self.assertTrue(b > c) From 5ea4bd8d0f611762a3f545a8ec17b1e2422796ab Mon Sep 17 00:00:00 2001 From: Ildar Musin Date: Mon, 2 Apr 2018 14:22:02 +0300 Subject: [PATCH 190/475] Add PostgresNode.promote() method which promotes standby node to master --- testgres/node.py | 22 ++++++++++++++++++++++ tests/test_simple.py | 14 ++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/testgres/node.py b/testgres/node.py index 9978dcf6..99a32e6f 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -691,6 +691,28 @@ def reload(self, params=[]): return self + def promote(self): + """ + Promote standby instance to master using pg_ctl. + + Returns: + This instance of :class:`.PostgresNode`. + """ + + _params = [ + get_bin_path("pg_ctl"), + "-D", self.data_dir, + "-w", # wait + "promote" + ] # yapf: disable + + execute_utility(_params, self.utils_log_file) + + # Node becomes master itself + self._master = None + + return self + def pg_ctl(self, params): """ Invoke pg_ctl with params. diff --git a/tests/test_simple.py b/tests/test_simple.py index 230cff47..91348f34 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -529,6 +529,20 @@ def test_incorrect_catchup(self): with self.assertRaises(TestgresException): node.catchup() + def test_promotion(self): + with get_new_node() as master: + master.init().start() + master.safe_psql('create table abc(id serial)') + + with master.replicate().start() as replica: + master.stop() + replica.promote() + + # make standby becomes writable master + replica.safe_psql('insert into abc values (1)') + res = replica.safe_psql('select * from abc') + self.assertEqual(res, b'1\n') + def test_dump(self): query_create = 'create table test as select generate_series(1, 2) as val' query_select = 'select * from test order by val asc' From 7c48faebc13e4a8ac5c363f60c1da8ca67b82928 Mon Sep 17 00:00:00 2001 From: Ildar Musin Date: Mon, 2 Apr 2018 14:57:32 +0300 Subject: [PATCH 191/475] Add synchronous promotion support on pg of version < 10 --- testgres/node.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index 99a32e6f..ada2f950 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -683,7 +683,6 @@ def reload(self, params=[]): _params = [ get_bin_path("pg_ctl"), "-D", self.data_dir, - "-w", # wait "reload" ] + params # yapf: disable @@ -691,9 +690,12 @@ def reload(self, params=[]): return self - def promote(self): + def promote(self, dbname=None, username=None): """ - Promote standby instance to master using pg_ctl. + Promote standby instance to master using pg_ctl. For PostgreSQL versions + below 10 some additional actions required to ensure that instance + became writable and hence `dbname` and `username` parameters may be + needed. Returns: This instance of :class:`.PostgresNode`. @@ -708,7 +710,19 @@ def promote(self): execute_utility(_params, self.utils_log_file) - # Node becomes master itself + # for versions below 10 `promote` is asynchronous so we need to wait + # until it actually becomes writable + if not pg_version_ge("10"): + check_query = "SHOW transaction_read_only" + + self.poll_query_until( + query=check_query, + expected="on", + dbname=dbname, + username=username, + max_attempts=0) # infinite + + # node becomes master itself self._master = None return self From 838a80053a42715020b5dbccc5927de78b1156ee Mon Sep 17 00:00:00 2001 From: Ildar Musin Date: Mon, 4 Jun 2018 14:56:45 +0300 Subject: [PATCH 192/475] fix promote() method for Postgres versions below 10 --- testgres/node.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testgres/node.py b/testgres/node.py index ada2f950..97ce3a78 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -717,7 +717,7 @@ def promote(self, dbname=None, username=None): self.poll_query_until( query=check_query, - expected="on", + expected="off", dbname=dbname, username=username, max_attempts=0) # infinite From f5fe963b0966e33d7e4da3fac35a93f14c3872f5 Mon Sep 17 00:00:00 2001 From: Ildar Musin Date: Mon, 4 Jun 2018 19:11:21 +0300 Subject: [PATCH 193/475] Use pg_is_in_recovery() instead of relying on transaction_read_only GUC variable in promote() func --- testgres/node.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index 64419483..b597dd69 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -713,11 +713,11 @@ def promote(self, dbname=None, username=None): # for versions below 10 `promote` is asynchronous so we need to wait # until it actually becomes writable if self._pg_version < '10': - check_query = "SHOW transaction_read_only" + check_query = "SELECT pg_is_in_recovery()" self.poll_query_until( query=check_query, - expected="off", + expected=False, dbname=dbname, username=username, max_attempts=0) # infinite From 5650e8a6b761f51206ca685880a425102e117688 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Tue, 5 Jun 2018 12:47:50 +0300 Subject: [PATCH 194/475] add **kwargs to append_conf() --- testgres/backup.py | 5 +- testgres/consts.py | 4 +- testgres/node.py | 106 +++++++++++++++++++++++++------------------ tests/test_simple.py | 2 +- 4 files changed, 69 insertions(+), 48 deletions(-) diff --git a/testgres/backup.py b/testgres/backup.py index ea3a2ef2..d603d7c9 100644 --- a/testgres/backup.py +++ b/testgres/backup.py @@ -149,8 +149,9 @@ def spawn_primary(self, name=None, destroy=True): # New nodes should always remove dir tree node._should_rm_dirs = True - node.append_conf(PG_CONF_FILE, "\n") - node.append_conf(PG_CONF_FILE, "port = {}".format(node.port)) + # Set a new port + node.append_conf(filename=PG_CONF_FILE, line='\n') + node.append_conf(filename=PG_CONF_FILE, port=node.port) return node diff --git a/testgres/consts.py b/testgres/consts.py index f7f01d9d..dcde873a 100644 --- a/testgres/consts.py +++ b/testgres/consts.py @@ -26,9 +26,11 @@ BACKUP_LOG_FILE = "backup.log" # defaults for node settings +MAX_LOGICAL_REPLICATION_WORKERS = 5 MAX_REPLICATION_SLOTS = 10 -MAX_WAL_SENDERS = 10 +MAX_WORKER_PROCESSES = 10 WAL_KEEP_SEGMENTS = 20 +MAX_WAL_SENDERS = 10 # logical replication settings LOGICAL_REPL_MAX_CATCHUP_ATTEMPTS = 60 diff --git a/testgres/node.py b/testgres/node.py index 9978dcf6..aab0ed03 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -7,7 +7,7 @@ import time from shutil import rmtree -from six import raise_from, iteritems +from six import raise_from, iteritems, text_type from tempfile import mkstemp, mkdtemp from .enums import \ @@ -38,8 +38,10 @@ PG_PID_FILE from .consts import \ - MAX_WAL_SENDERS, \ + MAX_LOGICAL_REPLICATION_WORKERS, \ MAX_REPLICATION_SLOTS, \ + MAX_WORKER_PROCESSES, \ + MAX_WAL_SENDERS, \ WAL_KEEP_SEGMENTS from .decorators import \ @@ -329,25 +331,27 @@ def _create_recovery_conf(self, username, slot=None): # Connect to master for some additional actions with master.connect(username=username) as con: # check if slot already exists - res = con.execute(""" + res = con.execute( + """ select exists ( select from pg_catalog.pg_replication_slots where slot_name = %s ) - """, slot) + """, slot) if res[0][0]: raise TestgresException( "Slot '{}' already exists".format(slot)) # TODO: we should drop this slot after replica's cleanup() - con.execute(""" + con.execute( + """ select pg_catalog.pg_create_physical_replication_slot(%s) - """, slot) + """, slot) line += "primary_slot_name={}\n".format(slot) - self.append_conf(RECOVERY_CONF_FILE, line) + self.append_conf(filename=RECOVERY_CONF_FILE, line=line) def _maybe_start_logger(self): if testgres_config.use_python_logging: @@ -475,65 +479,79 @@ def get_auth_method(t): # overwrite config file with io.open(postgres_conf, "w") as conf: - # remove old lines conf.truncate() - if not fsync: - conf.write(u"fsync = off\n") + self.append_conf(fsync=fsync, + max_worker_processes=MAX_WORKER_PROCESSES, + log_statement=log_statement, + listen_addresses=self.host, + port=self.port) # yapf:disable - conf.write(u"log_statement = {}\n" - u"listen_addresses = '{}'\n" - u"port = {}\n".format(log_statement, - self.host, - self.port)) # yapf: disable + # common replication settings + if allow_streaming or allow_logical: + self.append_conf(max_replication_slots=MAX_REPLICATION_SLOTS, + max_wal_senders=MAX_WAL_SENDERS) # yapf: disable - # replication-related settings - if allow_streaming: + # binary replication + if allow_streaming: + # select a proper wal_level for PostgreSQL + wal_level = 'replica' if self._pg_version >= '9.6' else 'hot_standby' - # select a proper wal_level for PostgreSQL - if self._pg_version >= '9.6': - wal_level = "replica" - else: - wal_level = "hot_standby" - - conf.write(u"hot_standby = on\n" - u"max_wal_senders = {}\n" - u"max_replication_slots = {}\n" - u"wal_keep_segments = {}\n" - u"wal_level = {}\n".format(MAX_WAL_SENDERS, - MAX_REPLICATION_SLOTS, - WAL_KEEP_SEGMENTS, - wal_level)) # yapf: disable - - if allow_logical: - if self._pg_version < '10': - raise InitNodeException( - "Logical replication is only available for Postgres 10 " - "and newer") - conf.write(u"wal_level = logical\n") - - # disable UNIX sockets if asked to - if not unix_sockets: - conf.write(u"unix_socket_directories = ''\n") + self.append_conf(hot_standby=True, + wal_keep_segments=WAL_KEEP_SEGMENTS, + wal_level=wal_level) # yapf: disable + + # logical replication + if allow_logical: + if self._pg_version < '10': + raise InitNodeException("Logical replication is only " + "available on PostgreSQL 10 and newer") + + self.append_conf( + max_logical_replication_workers=MAX_LOGICAL_REPLICATION_WORKERS, + wal_level='logical') + + # disable UNIX sockets if asked to + if not unix_sockets: + self.append_conf(unix_socket_directories='') return self @method_decorator(positional_args_hack(['filename', 'line'])) - def append_conf(self, line, filename=PG_CONF_FILE): + def append_conf(self, line='', filename=PG_CONF_FILE, **kwargs): """ Append line to a config file. Args: line: string to be appended to config. filename: config file (postgresql.conf by default). + **kwargs: named config options. Returns: This instance of :class:`.PostgresNode`. + + Examples: + append_conf(fsync=False) + append_conf('log_connections = yes') + append_conf('postgresql.conf', 'synchronous_commit = off') """ + lines = [line] + + for option, value in iteritems(kwargs): + if isinstance(value, bool): + value = 'on' if value else 'off' + elif not str(value).replace('.', '', 1).isdigit(): + value = "'{}'".format(value) + + # format a new config line + lines.append('{} = {}'.format(option, value)) + config_name = os.path.join(self.data_dir, filename) with io.open(config_name, 'a') as conf: - conf.write(u''.join([line, '\n'])) + for line in lines: + conf.write(text_type(line)) + conf.write(text_type('\n')) return self diff --git a/tests/test_simple.py b/tests/test_simple.py index 230cff47..068619dc 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -195,7 +195,7 @@ def test_reload(self): # change client_min_messages and save old value cmm_old = node.execute('show client_min_messages') - node.append_conf('client_min_messages = DEBUG1') + node.append_conf(client_min_messages='DEBUG1') # reload config node.reload() From e461f5953d603a9790dc1dfecb4a365837c8ece9 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Tue, 5 Jun 2018 14:33:18 +0300 Subject: [PATCH 195/475] +1 example to append_conf() --- testgres/node.py | 1 + 1 file changed, 1 insertion(+) diff --git a/testgres/node.py b/testgres/node.py index aab0ed03..b4520e3a 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -533,6 +533,7 @@ def append_conf(self, line='', filename=PG_CONF_FILE, **kwargs): Examples: append_conf(fsync=False) append_conf('log_connections = yes') + append_conf(random_page_cost=1.5, fsync=True, ...) append_conf('postgresql.conf', 'synchronous_commit = off') """ From 80a3e556c046cc0aa835838cc8556479464b06ed Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Fri, 15 Jun 2018 18:26:48 +0300 Subject: [PATCH 196/475] examples for PostgresNode.poll_query_until() --- testgres/node.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/testgres/node.py b/testgres/node.py index 101b6000..0b5e55cf 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -986,6 +986,11 @@ def poll_query_until(self, commit: should (possible) changes be committed? raise_programming_error: enable ProgrammingError? raise_internal_error: enable InternalError? + + Examples: + poll_query_until('select true') + poll_query_until('postgres', "select now() > '01.01.2018'") + poll_query_until('select false', expected=True, max_attempts=4) """ # sanity checks From 7707844b7bd0259ebfd3c4c635ee323afd5384c6 Mon Sep 17 00:00:00 2001 From: Ildus Kurbangaliev Date: Mon, 25 Jun 2018 16:39:13 +0300 Subject: [PATCH 197/475] Add version property to PostgresNode: --- testgres/node.py | 10 ++++++++++ tests/test_simple.py | 6 ++++++ 2 files changed, 16 insertions(+) diff --git a/testgres/node.py b/testgres/node.py index 0b5e55cf..b171d965 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -278,6 +278,16 @@ def utils_log_file(self): def pg_log_file(self): return os.path.join(self.logs_dir, PG_LOG_FILE) + @property + def version(self): + """ + Return PostgreSQL version for this node. + + Returns: + Instance of :class:`distutils.version.LooseVersion`. + """ + return self._pg_version + def _try_shutdown(self, max_attempts): attempts = 0 diff --git a/tests/test_simple.py b/tests/test_simple.py index d395f25b..5689cdf0 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -857,6 +857,12 @@ def test_version_management(self): self.assertTrue(b > c) self.assertTrue(a > c) + version = get_pg_version() + with get_new_node() as node: + self.assertTrue(isinstance(version, str)) + self.assertTrue(isinstance(node.version, PgVer)) + self.assertTrue(node.version == version) + def test_child_pids(self): master_processes = [ ProcessType.AutovacuumLauncher, From 8b7d0d1abc4f3ff3fc92ff6be9cbaea202b91e90 Mon Sep 17 00:00:00 2001 From: Ildus Kurbangaliev Date: Mon, 25 Jun 2018 18:27:59 +0300 Subject: [PATCH 198/475] Fix tests for python 2 --- tests/test_simple.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_simple.py b/tests/test_simple.py index 5689cdf0..343eedd0 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -7,6 +7,7 @@ import tempfile import testgres import time +import six import unittest import logging.config @@ -859,7 +860,7 @@ def test_version_management(self): version = get_pg_version() with get_new_node() as node: - self.assertTrue(isinstance(version, str)) + self.assertTrue(isinstance(version, six.string_types)) self.assertTrue(isinstance(node.version, PgVer)) self.assertTrue(node.version == version) From c226759558e51b83a1fb2e1fb4cbc4f2084ce45c Mon Sep 17 00:00:00 2001 From: Ildus Kurbangaliev Date: Mon, 25 Jun 2018 18:57:22 +0300 Subject: [PATCH 199/475] Try to find pg_config from system PATH --- testgres/utils.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/testgres/utils.py b/testgres/utils.py index 4382f31a..6c6f0e9b 100644 --- a/testgres/utils.py +++ b/testgres/utils.py @@ -11,6 +11,7 @@ from contextlib import contextmanager from distutils.version import LooseVersion +from distutils.spawn import find_executable from six import iteritems from .config import testgres_config @@ -116,10 +117,15 @@ def get_bin_path(filename): if pg_bin: return os.path.join(pg_bin, filename) + pg_config_path = find_executable('pg_config') + if pg_config_path: + bindir = get_pg_config(pg_config_path)["BINDIR"] + return os.path.join(bindir, filename) + return filename -def get_pg_config(): +def get_pg_config(pg_config_path=None): """ Return output of pg_config (provided that it is installed). NOTE: this fuction caches the result by default (see GlobalConfig). @@ -150,8 +156,8 @@ def cache_pg_config_data(cmd): if _pg_config_data: return _pg_config_data - # try PG_CONFIG - pg_config = os.environ.get("PG_CONFIG") + # try specified pg_config path or PG_CONFIG + pg_config = pg_config_path or os.environ.get("PG_CONFIG") if pg_config: return cache_pg_config_data(pg_config) From 95b5c5debebe8864574d298addd671a1e03d588e Mon Sep 17 00:00:00 2001 From: Ildus Kurbangaliev Date: Tue, 26 Jun 2018 14:37:47 +0300 Subject: [PATCH 200/475] Fix tests for python 2 --- tests/test_simple.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_simple.py b/tests/test_simple.py index 343eedd0..2e557309 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -862,7 +862,7 @@ def test_version_management(self): with get_new_node() as node: self.assertTrue(isinstance(version, six.string_types)) self.assertTrue(isinstance(node.version, PgVer)) - self.assertTrue(node.version == version) + self.assertEqual(node.version, str(version)) def test_child_pids(self): master_processes = [ From e78ddd9b167cb924e32d4c5239083c85e601c56c Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Wed, 11 Jul 2018 12:42:13 +0300 Subject: [PATCH 201/475] add 'suppress' to PostgresNode.poll_query_until() to mute exceptions --- testgres/__init__.py | 4 +++- testgres/connection.py | 4 +++- testgres/node.py | 42 +++++++++++++++++------------------------- tests/test_simple.py | 2 +- 4 files changed, 24 insertions(+), 28 deletions(-) diff --git a/testgres/__init__.py b/testgres/__init__.py index c907b708..d2322f9a 100644 --- a/testgres/__init__.py +++ b/testgres/__init__.py @@ -10,8 +10,10 @@ from .connection import \ NodeConnection, \ + DatabaseError, \ InternalError, \ - ProgrammingError + ProgrammingError, \ + OperationalError from .exceptions import * from .enums import * diff --git a/testgres/connection.py b/testgres/connection.py index 3943a4e2..ab846a14 100644 --- a/testgres/connection.py +++ b/testgres/connection.py @@ -17,9 +17,11 @@ from .exceptions import QueryException -# export these exceptions +# export some exceptions +DatabaseError = pglib.DatabaseError InternalError = pglib.InternalError ProgrammingError = pglib.ProgrammingError +OperationalError = pglib.OperationalError class NodeConnection(object): diff --git a/testgres/node.py b/testgres/node.py index 0b5e55cf..519a30f6 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -19,10 +19,7 @@ from .config import testgres_config -from .connection import \ - NodeConnection, \ - InternalError, \ - ProgrammingError +from .connection import NodeConnection from .consts import \ DATA_DIR, \ @@ -531,10 +528,10 @@ def append_conf(self, line='', filename=PG_CONF_FILE, **kwargs): This instance of :class:`.PostgresNode`. Examples: - append_conf(fsync=False) - append_conf('log_connections = yes') - append_conf(random_page_cost=1.5, fsync=True, ...) - append_conf('postgresql.conf', 'synchronous_commit = off') + >>> append_conf(fsync=False) + >>> append_conf('log_connections = yes') + >>> append_conf(random_page_cost=1.5, fsync=True, ...) + >>> append_conf('postgresql.conf', 'synchronous_commit = off') """ lines = [line] @@ -970,8 +967,7 @@ def poll_query_until(self, sleep_time=1, expected=True, commit=True, - raise_programming_error=True, - raise_internal_error=True): + suppress=None): """ Run a query once per second until it returns 'expected'. Query should return a single value (1 row, 1 column). @@ -984,13 +980,13 @@ def poll_query_until(self, sleep_time: how much should we sleep after a failure? expected: what should be returned to break the cycle? commit: should (possible) changes be committed? - raise_programming_error: enable ProgrammingError? - raise_internal_error: enable InternalError? + suppress: a collection of exceptions to be suppressed. Examples: - poll_query_until('select true') - poll_query_until('postgres', "select now() > '01.01.2018'") - poll_query_until('select false', expected=True, max_attempts=4) + >>> poll_query_until('select true') + >>> poll_query_until('postgres', "select now() > '01.01.2018'") + >>> poll_query_until('select false', expected=True, max_attempts=4) + >>> poll_query_until('select 1', suppress={testgres.OperationalError}) """ # sanity checks @@ -1022,13 +1018,8 @@ def poll_query_until(self, elif expected is None: return # done - except ProgrammingError as e: - if raise_programming_error: - raise e - - except InternalError as e: - if raise_internal_error: - raise e + except tuple(suppress or []): + pass # we're suppressing them time.sleep(sleep_time) attempts += 1 @@ -1219,13 +1210,14 @@ def pgbench_run(self, dbname=None, username=None, options=[], **kwargs): options: additional options for pgbench (list). **kwargs: named options for pgbench. - Examples: - pgbench_run(initialize=True, scale=2) - pgbench_run(time=10) Run pgbench --help to learn more. Returns: Stdout produced by pgbench. + + Examples: + >>> pgbench_run(initialize=True, scale=2) + >>> pgbench_run(time=10) """ # Set default arguments diff --git a/tests/test_simple.py b/tests/test_simple.py index d395f25b..d741442a 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -625,7 +625,7 @@ def test_poll_query_until(self): query='dummy2', max_attempts=3, sleep_time=0.01, - raise_programming_error=False) + suppress={testgres.ProgrammingError}) # check 1 arg, ok node.poll_query_until('select true') From 7a40484869321ae4b65aeab88ccc1a1caafd97a9 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Wed, 11 Jul 2018 14:29:44 +0300 Subject: [PATCH 202/475] bump version to 1.8.0 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 3690a5ed..de6a427e 100755 --- a/setup.py +++ b/setup.py @@ -21,7 +21,7 @@ readme = f.read() setup( - version='1.7.0', + version='1.8.0', name='testgres', packages=['testgres'], description='Testing utility for PostgreSQL and its extensions', From bf339922b6a12e76adbdef69539aecb0137ed1ed Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Wed, 11 Jul 2018 15:58:09 +0300 Subject: [PATCH 203/475] Update README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 3839981e..caa94f1e 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,8 @@ [![codecov](https://codecov.io/gh/postgrespro/testgres/branch/master/graph/badge.svg)](https://codecov.io/gh/postgrespro/testgres) [![PyPI version](https://badge.fury.io/py/testgres.svg)](https://badge.fury.io/py/testgres) +[Documentation](https://postgrespro.github.io/testgres/) + # testgres PostgreSQL testing utility. Both Python 2.7 and 3.3+ are supported. From 39c6d797704b7d0864c32265b31619affae0f9e8 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Thu, 12 Jul 2018 14:52:52 +0300 Subject: [PATCH 204/475] make NodeBackup aware of inheritors of PostgresNode --- testgres/backup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testgres/backup.py b/testgres/backup.py index d603d7c9..d9865c1e 100644 --- a/testgres/backup.py +++ b/testgres/backup.py @@ -143,8 +143,8 @@ def spawn_primary(self, name=None, destroy=True): base_dir = self._prepare_dir(destroy) # Build a new PostgresNode - from .node import PostgresNode - with clean_on_error(PostgresNode(name=name, base_dir=base_dir)) as node: + NodeClass = self.original_node.__class__ + with clean_on_error(NodeClass(name=name, base_dir=base_dir)) as node: # New nodes should always remove dir tree node._should_rm_dirs = True From 6e303fe7f7fd2ce41f0e5c16a0eed44c569f0de2 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Thu, 12 Jul 2018 18:27:57 +0300 Subject: [PATCH 205/475] bump version to 1.8.1 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index de6a427e..3342da52 100755 --- a/setup.py +++ b/setup.py @@ -21,7 +21,7 @@ readme = f.read() setup( - version='1.8.0', + version='1.8.1', name='testgres', packages=['testgres'], description='Testing utility for PostgreSQL and its extensions', From 60d45724092fc4ff468bf9faa88477456ae28bf6 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Wed, 18 Jul 2018 13:26:33 +0300 Subject: [PATCH 206/475] safe_psql() should always raise error on wrong input (issue #52) --- testgres/node.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/testgres/node.py b/testgres/node.py index 77538896..e202fab4 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -812,7 +812,8 @@ def psql(self, filename=None, dbname=None, username=None, - input=None): + input=None, + **kwargs): """ Execute a query using psql. @@ -822,9 +823,18 @@ def psql(self, dbname: database name to connect to. username: database user name. input: raw input to be passed. + **kwargs: variables to be set before execution. Returns: A tuple of (code, stdout, stderr). + + Examples: + >>> psql('select 1') + (0, b'1\n', b'') + >>> psql('postgres', 'select 2') + (0, b'2\n', b'') + >>> psql(query='select 3', ON_ERROR_STOP=1) + (0, b'3\n', b'') """ # Set default arguments @@ -843,6 +853,10 @@ def psql(self, dbname ] # yapf: disable + # set variables before execution + for key, value in iteritems(kwargs): + psql_params.extend(["--set", '{}={}'.format(key, value)]) + # select query source if query: psql_params.extend(("-c", query)) @@ -874,10 +888,15 @@ def safe_psql(self, query=None, **kwargs): username: database user name. input: raw input to be passed. + **kwargs are passed to psql(). + Returns: psql's output as str. """ + # force this setting + kwargs['ON_ERROR_STOP'] = 1 + ret, out, err = self.psql(query=query, **kwargs) if ret: raise QueryException((err or b'').decode('utf-8'), query) From b9648ee4a2eee7d7213720306ddfa988e08f97e3 Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Wed, 18 Jul 2018 13:37:36 +0300 Subject: [PATCH 207/475] style fixes suggested by @ildus --- testgres/node.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index e202fab4..6d67d2cf 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -813,7 +813,7 @@ def psql(self, dbname=None, username=None, input=None, - **kwargs): + **variables): """ Execute a query using psql. @@ -823,7 +823,7 @@ def psql(self, dbname: database name to connect to. username: database user name. input: raw input to be passed. - **kwargs: variables to be set before execution. + **variables: vars to be set before execution. Returns: A tuple of (code, stdout, stderr). @@ -854,7 +854,7 @@ def psql(self, ] # yapf: disable # set variables before execution - for key, value in iteritems(kwargs): + for key, value in iteritems(variables): psql_params.extend(["--set", '{}={}'.format(key, value)]) # select query source From 25cf6881f55fbd27a7d1df9956086b60ecf8ee6e Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Wed, 18 Jul 2018 16:20:09 +0300 Subject: [PATCH 208/475] bump version to 1.8.2 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 3342da52..f422c982 100755 --- a/setup.py +++ b/setup.py @@ -21,7 +21,7 @@ readme = f.read() setup( - version='1.8.1', + version='1.8.2', name='testgres', packages=['testgres'], description='Testing utility for PostgreSQL and its extensions', From 19bbb72b1fe4fb83a90e46dad1a0b6b09c24749c Mon Sep 17 00:00:00 2001 From: Dmitry Ivanov Date: Thu, 19 Jul 2018 16:56:17 +0300 Subject: [PATCH 209/475] fix docs --- testgres/node.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index 6d67d2cf..0ae1d5ce 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -830,11 +830,8 @@ def psql(self, Examples: >>> psql('select 1') - (0, b'1\n', b'') >>> psql('postgres', 'select 2') - (0, b'2\n', b'') >>> psql(query='select 3', ON_ERROR_STOP=1) - (0, b'3\n', b'') """ # Set default arguments From cdc933ff4af3f956e43e5ed84974889cc5163127 Mon Sep 17 00:00:00 2001 From: Ildus Kurbangaliev Date: Fri, 23 Nov 2018 13:27:38 +0300 Subject: [PATCH 210/475] Skip cleanup of custom directory was provided --- testgres/node.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index 0ae1d5ce..86e6ddbf 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -115,10 +115,12 @@ def __init__(self, name=None, port=None, base_dir=None): # private self._pg_version = PgVer(get_pg_version()) self._should_free_port = port is None - self._base_dir = base_dir self._logger = None self._master = None + self._custom_base_dir = base_dir + self._base_dir = base_dir + # basic self.host = '127.0.0.1' self.name = name or generate_app_name() @@ -796,11 +798,13 @@ def cleanup(self, max_attempts=3): self._try_shutdown(max_attempts) - # choose directory to be removed - if testgres_config.node_cleanup_full: - rm_dir = self.base_dir # everything - else: - rm_dir = self.data_dir # just data, save logs + # only remove if base directory was temporary + if not self._custom_base_dir: + # choose directory to be removed + if testgres_config.node_cleanup_full: + rm_dir = self.base_dir # everything + else: + rm_dir = self.data_dir # just data, save logs rmtree(rm_dir, ignore_errors=True) From 2ef33a80d61d2dc3c78a7faa671ad4aa524e6cc4 Mon Sep 17 00:00:00 2001 From: Ildus Kurbangaliev Date: Fri, 23 Nov 2018 13:32:20 +0300 Subject: [PATCH 211/475] Fix tests --- testgres/node.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testgres/node.py b/testgres/node.py index 86e6ddbf..feb925f7 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -806,7 +806,7 @@ def cleanup(self, max_attempts=3): else: rm_dir = self.data_dir # just data, save logs - rmtree(rm_dir, ignore_errors=True) + rmtree(rm_dir, ignore_errors=True) return self From 95d37e9960b0b37d13282c3e1ad37a684d681026 Mon Sep 17 00:00:00 2001 From: Ildar Musin Date: Thu, 24 Jan 2019 18:04:09 +0100 Subject: [PATCH 212/475] Fix incorrect parameter name for enabling python logging in README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index caa94f1e..f30d0929 100644 --- a/README.md +++ b/README.md @@ -94,7 +94,7 @@ import logging logging.basicConfig(filename='/tmp/testgres.log') # enable logging, and create two different nodes -testgres.configure_testgres(enable_python_logging=True) +testgres.configure_testgres(use_python_logging=True) node1 = testgres.get_new_node().init().start() node2 = testgres.get_new_node().init().start() @@ -103,7 +103,7 @@ node1.execute('select 1') node2.execute('select 2') # disable logging -testgres.configure_testgres(enable_python_logging=False) +testgres.configure_testgres(use_python_logging=False) ``` Look at `tests/test_simple.py` file for a complete example of the logging From 4c3e87776c6305fa8ced9d62a7d8522774512aa0 Mon Sep 17 00:00:00 2001 From: Ildar Musin Date: Thu, 8 Aug 2019 16:56:15 +0200 Subject: [PATCH 213/475] Synchronous standbys (#46) Add set_synchronous_standbys() method --- docs/source/testgres.rst | 8 +++++++ testgres/__init__.py | 4 ++++ testgres/node.py | 44 +++++++++++++++++++++++++++++++++++- testgres/standby.py | 49 ++++++++++++++++++++++++++++++++++++++++ tests/test_simple.py | 44 ++++++++++++++++++++++++++++++++++++ 5 files changed, 148 insertions(+), 1 deletion(-) create mode 100644 testgres/standby.py diff --git a/docs/source/testgres.rst b/docs/source/testgres.rst index 80c86e84..c3c43f2a 100644 --- a/docs/source/testgres.rst +++ b/docs/source/testgres.rst @@ -61,6 +61,14 @@ testgres.node .. autoclass:: testgres.node.ProcessProxy :members: +testgres.standby +---------------- + +.. automodule:: testgres.standby + :members: + :undoc-members: + :show-inheritance: + testgres.pubsub --------------- diff --git a/testgres/__init__.py b/testgres/__init__.py index d2322f9a..f8d895a5 100644 --- a/testgres/__init__.py +++ b/testgres/__init__.py @@ -26,3 +26,7 @@ get_bin_path, \ get_pg_config, \ get_pg_version + +from .standby import \ + First, \ + Any diff --git a/testgres/node.py b/testgres/node.py index feb925f7..399136ad 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -6,6 +6,7 @@ import subprocess import time +from collections import Iterable from shutil import rmtree from six import raise_from, iteritems, text_type from tempfile import mkstemp, mkdtemp @@ -64,6 +65,8 @@ from .pubsub import Publication, Subscription +from .standby import First + from .utils import \ PgVer, \ eprint, \ @@ -699,7 +702,7 @@ def restart(self, params=[]): def reload(self, params=[]): """ - Reload config files using pg_ctl. + Asynchronously reload config files using pg_ctl. Args: params: additional arguments for pg_ctl. @@ -1117,6 +1120,45 @@ def replicate(self, name=None, slot=None, **kwargs): with clean_on_error(self.backup(**kwargs)) as backup: return backup.spawn_replica(name=name, destroy=True, slot=slot) + def set_synchronous_standbys(self, standbys): + """ + Set standby synchronization options. This corresponds to + `synchronous_standby_names `_ + option. Note that :meth:`~.PostgresNode.reload` or + :meth:`~.PostgresNode.restart` is needed for changes to take place. + + Args: + standbys: either :class:`.First` or :class:`.Any` object specifying + sychronization parameters or just a plain list of + :class:`.PostgresNode`s replicas which would be equivalent + to passing ``First(1, )``. For PostgreSQL 9.5 and below + it is only possible to specify a plain list of standbys as + `FIRST` and `ANY` keywords aren't supported. + + Example:: + + from testgres import get_new_node, First + + master = get_new_node().init().start() + with master.replicate().start() as standby: + master.append_conf("synchronous_commit = remote_apply") + master.set_synchronous_standbys(First(1, [standby])) + master.restart() + + """ + if self._pg_version >= '9.6': + if isinstance(standbys, Iterable): + standbys = First(1, standbys) + else: + if isinstance(standbys, Iterable): + standbys = u", ".join( + u"\"{}\"".format(r.name) for r in standbys) + else: + raise TestgresException("Feature isn't supported in " + "Postgres 9.5 and below") + + self.append_conf("synchronous_standby_names = '{}'".format(standbys)) + def catchup(self, dbname=None, username=None): """ Wait until async replica catches up with its master. diff --git a/testgres/standby.py b/testgres/standby.py new file mode 100644 index 00000000..e7ce408d --- /dev/null +++ b/testgres/standby.py @@ -0,0 +1,49 @@ +# coding: utf-8 + +import six + + +@six.python_2_unicode_compatible +class First: + """ + Specifies a priority-based synchronous replication and makes transaction + commits wait until their WAL records are replicated to ``num_sync`` + synchronous standbys chosen based on their priorities. + + Args: + sync_num (int): the number of standbys that transaction need to wait + for replies from + standbys (:obj:`list` of :class:`.PostgresNode`): the list of standby + nodes + """ + + def __init__(self, sync_num, standbys): + self.sync_num = sync_num + self.standbys = standbys + + def __str__(self): + return u"{} ({})".format(self.sync_num, u", ".join( + u"\"{}\"".format(r.name) for r in self.standbys)) + + +@six.python_2_unicode_compatible +class Any: + """ + Specifies a quorum-based synchronous replication and makes transaction + commits wait until their WAL records are replicated to at least ``num_sync`` + listed standbys. Only available for Postgres 10 and newer. + + Args: + sync_num (int): the number of standbys that transaction need to wait + for replies from + standbys (:obj:`list` of :class:`.PostgresNode`): the list of standby + nodes + """ + + def __init__(self, sync_num, standbys): + self.sync_num = sync_num + self.standbys = standbys + + def __str__(self): + return u"ANY {} ({})".format(self.sync_num, u", ".join( + u"\"{}\"".format(r.name) for r in self.standbys)) diff --git a/tests/test_simple.py b/tests/test_simple.py index 4853686d..0ef7306f 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -41,6 +41,10 @@ get_pg_config, \ get_pg_version +from testgres import \ + First, \ + Any + # NOTE: those are ugly imports from testgres import bound_ports from testgres.utils import PgVer @@ -410,6 +414,46 @@ def test_replicate(self): res = node.execute('select * from test') self.assertListEqual(res, []) + def test_synchronous_replication(self): + with get_new_node() as master: + old_version = not pg_version_ge('9.6') + + master.init(allow_streaming=True).start() + + if not old_version: + master.append_conf('synchronous_commit = remote_apply') + + # create standby + with master.replicate() as standby1, master.replicate() as standby2: + standby1.start() + standby2.start() + + # check formatting + self.assertEqual( + '1 ("{}", "{}")'.format(standby1.name, standby2.name), + str(First(1, (standby1, standby2)))) # yapf: disable + self.assertEqual( + 'ANY 1 ("{}", "{}")'.format(standby1.name, standby2.name), + str(Any(1, (standby1, standby2)))) # yapf: disable + + # set synchronous_standby_names + master.set_synchronous_standbys([standby1, standby2]) + master.restart() + + # the following part of the test is only applicable to newer + # versions of PostgresQL + if not old_version: + master.safe_psql('create table abc(a int)') + + # Create a large transaction that will take some time to apply + # on standby to check that it applies synchronously + # (If set synchronous_commit to 'on' or other lower level then + # standby most likely won't catchup so fast and test will fail) + master.safe_psql( + 'insert into abc select generate_series(1, 1000000)') + res = standby1.safe_psql('select count(*) from abc') + self.assertEqual(res, b'1000000\n') + @unittest.skipUnless(pg_version_ge('10'), 'requires 10+') def test_logical_replication(self): with get_new_node() as node1, get_new_node() as node2: From 568603c82cf2dadf406b66630f46f375e457ccc7 Mon Sep 17 00:00:00 2001 From: Ildar Musin Date: Fri, 9 Aug 2019 11:41:40 +0200 Subject: [PATCH 214/475] Remove emails from readme --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index f30d0929..f020273c 100644 --- a/README.md +++ b/README.md @@ -176,7 +176,7 @@ the configuration file, which means that they should be called before `append_co ## Authors -[Ildar Musin](https://github.com/zilder) Postgres Professional Ltd., Russia -[Dmitry Ivanov](https://github.com/funbringer) Postgres Professional Ltd., Russia -[Ildus Kurbangaliev](https://github.com/ildus) Postgres Professional Ltd., Russia -[Yury Zhuravlev](https://github.com/stalkerg) +[Ildar Musin](https://github.com/zilder) +[Dmitry Ivanov](https://github.com/funbringer) +[Ildus Kurbangaliev](https://github.com/ildus) +[Yury Zhuravlev](https://github.com/stalkerg) From ccdf25ec733f5b3234f4da3deb53f7078853e3cb Mon Sep 17 00:00:00 2001 From: Arseny Sher Date: Mon, 18 Nov 2019 21:11:46 +0300 Subject: [PATCH 215/475] Adapt to PG 12 where recovery.conf is obsolete. --- testgres/node.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index 399136ad..01b3ad56 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -336,8 +336,15 @@ def _create_recovery_conf(self, username, slot=None): line = ( "primary_conninfo='{}'\n" - "standby_mode=on\n" ).format(options_string(**conninfo)) # yapf: disable + # Since 12 recovery.conf had disappeared + if self.version >= '12': + signal_name = os.path.join(self.data_dir, "standby.signal") + # cross-python touch(). It is vulnerable to races, but who cares? + with open(signal_name, 'a'): + os.utime(signal_name, None) + else: + line += "standby_mode=on\n" if slot: # Connect to master for some additional actions @@ -363,7 +370,10 @@ def _create_recovery_conf(self, username, slot=None): line += "primary_slot_name={}\n".format(slot) - self.append_conf(filename=RECOVERY_CONF_FILE, line=line) + if self.version >= '12': + self.append_conf(line=line) + else: + self.append_conf(filename=RECOVERY_CONF_FILE, line=line) def _maybe_start_logger(self): if testgres_config.use_python_logging: From 7c10b4fa05d0fd0ee3a414a264acc60dabc9cea9 Mon Sep 17 00:00:00 2001 From: Arseny Sher Date: Tue, 19 Nov 2019 11:17:20 +0300 Subject: [PATCH 216/475] Add 11 and 12 to travis. --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 3fd628cb..0aa9e9fa 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,6 +22,8 @@ env: - PYTHON_VERSION=2 PG_VERSION=9.6 - PYTHON_VERSION=2 PG_VERSION=9.5 - PYTHON_VERSION=2 PG_VERSION=9.4 + - PYTHON_VERSION=3 PG_VERSION=12 + - PYTHON_VERSION=3 PG_VERSION=11 - PYTHON_VERSION=3 PG_VERSION=10 - PYTHON_VERSION=3 PG_VERSION=9.6 - PYTHON_VERSION=3 PG_VERSION=9.5 From eee6665dfa23a3a9cd8baabbc0799762d5d3c94a Mon Sep 17 00:00:00 2001 From: Arseny Sher Date: Tue, 19 Nov 2019 11:17:34 +0300 Subject: [PATCH 217/475] Bump version 1.8.3. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index f422c982..05ee76cd 100755 --- a/setup.py +++ b/setup.py @@ -21,7 +21,7 @@ readme = f.read() setup( - version='1.8.2', + version='1.8.3', name='testgres', packages=['testgres'], description='Testing utility for PostgreSQL and its extensions', From 87a0491f3b1ab5b2bf59f3c5d76b01985b8daba5 Mon Sep 17 00:00:00 2001 From: Arseny Sher Date: Tue, 19 Nov 2019 15:17:13 +0300 Subject: [PATCH 218/475] Try to fix caughtup in LR tests. --- testgres/pubsub.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/testgres/pubsub.py b/testgres/pubsub.py index bb153913..7e874746 100644 --- a/testgres/pubsub.py +++ b/testgres/pubsub.py @@ -186,12 +186,19 @@ def catchup(self, username=None): Args: username: remote node's user name. """ - query = """ - select pg_current_wal_lsn() - replay_lsn = 0 - from pg_catalog.pg_stat_replication where application_name = '{}' - """.format(self.name) try: + pub_lsn = pub.node.execute(query="select pg_current_wal_lsn()::text from pg_catalog.pg_stat_replication", + dbname=None, + username=None) # yapf: disable + # create dummy xact + pub.node.execute(query="select txid_current()", dbname=None, username=None) + query = """ + select {} - replay_lsn <= 0 + from pg_catalog.pg_stat_replication where application_name = '{}' + """.format(pub_lsn, self.name) + + # wait until this LSN reaches subscriber self.pub.node.poll_query_until( query=query, From ef89ed580067df89f43c2350444c6a2c318659ec Mon Sep 17 00:00:00 2001 From: Arseny Sher Date: Tue, 19 Nov 2019 16:03:57 +0300 Subject: [PATCH 219/475] Fix that blind fix. --- testgres/pubsub.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/testgres/pubsub.py b/testgres/pubsub.py index 7e874746..e103fdbe 100644 --- a/testgres/pubsub.py +++ b/testgres/pubsub.py @@ -186,19 +186,17 @@ def catchup(self, username=None): Args: username: remote node's user name. """ - try: - pub_lsn = pub.node.execute(query="select pg_current_wal_lsn()::text from pg_catalog.pg_stat_replication", - dbname=None, - username=None) # yapf: disable + pub_lsn = self.pub.node.execute(query="select pg_current_wal_lsn()", + dbname=None, + username=None)[0][0] # yapf: disable # create dummy xact - pub.node.execute(query="select txid_current()", dbname=None, username=None) + self.pub.node.execute(query="select txid_current()", dbname=None, username=None) query = """ - select {} - replay_lsn <= 0 + select '{}'::pg_lsn - replay_lsn <= 0 from pg_catalog.pg_stat_replication where application_name = '{}' """.format(pub_lsn, self.name) - # wait until this LSN reaches subscriber self.pub.node.poll_query_until( query=query, From 76ea566dbd593de6e52436e107b7c3f9863d9a61 Mon Sep 17 00:00:00 2001 From: Arseny Sher Date: Tue, 19 Nov 2019 18:06:10 +0300 Subject: [PATCH 220/475] Well, that was even trickier. --- testgres/pubsub.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/testgres/pubsub.py b/testgres/pubsub.py index e103fdbe..07b83543 100644 --- a/testgres/pubsub.py +++ b/testgres/pubsub.py @@ -190,7 +190,7 @@ def catchup(self, username=None): pub_lsn = self.pub.node.execute(query="select pg_current_wal_lsn()", dbname=None, username=None)[0][0] # yapf: disable - # create dummy xact + # create dummy xact, as LR replicates only on commit. self.pub.node.execute(query="select txid_current()", dbname=None, username=None) query = """ select '{}'::pg_lsn - replay_lsn <= 0 @@ -203,5 +203,17 @@ def catchup(self, username=None): dbname=self.pub.dbname, username=username or self.pub.username, max_attempts=LOGICAL_REPL_MAX_CATCHUP_ATTEMPTS) + + # Now, wait until there are no tablesync workers: probably + # replay_lsn above was sent with changes of new tables just skipped; + # they will be eaten by tablesync workers. + query = """ + select count(*) = 0 from pg_subscription_rel where srsubstate != 'r' + """ + self.node.poll_query_until( + query=query, + dbname=self.pub.dbname, + username=username or self.pub.username, + max_attempts=LOGICAL_REPL_MAX_CATCHUP_ATTEMPTS) except Exception as e: raise_from(CatchUpException("Failed to catch up", query), e) From 63463a1e3dcc974c3acb47f4ce1db1e0f37cd408 Mon Sep 17 00:00:00 2001 From: anastasia Date: Mon, 19 Oct 2020 17:20:44 +0300 Subject: [PATCH 221/475] Add PostgreSQL 13 support --- .travis.yml | 1 + testgres/consts.py | 1 + testgres/node.py | 11 ++++++++--- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0aa9e9fa..0e7a3701 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,6 +22,7 @@ env: - PYTHON_VERSION=2 PG_VERSION=9.6 - PYTHON_VERSION=2 PG_VERSION=9.5 - PYTHON_VERSION=2 PG_VERSION=9.4 + - PYTHON_VERSION=3 PG_VERSION=13 - PYTHON_VERSION=3 PG_VERSION=12 - PYTHON_VERSION=3 PG_VERSION=11 - PYTHON_VERSION=3 PG_VERSION=10 diff --git a/testgres/consts.py b/testgres/consts.py index dcde873a..98c84af6 100644 --- a/testgres/consts.py +++ b/testgres/consts.py @@ -30,6 +30,7 @@ MAX_REPLICATION_SLOTS = 10 MAX_WORKER_PROCESSES = 10 WAL_KEEP_SEGMENTS = 20 +WAL_KEEP_SIZE = 320 MAX_WAL_SENDERS = 10 # logical replication settings diff --git a/testgres/node.py b/testgres/node.py index 01b3ad56..81866e89 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -519,9 +519,14 @@ def get_auth_method(t): # select a proper wal_level for PostgreSQL wal_level = 'replica' if self._pg_version >= '9.6' else 'hot_standby' - self.append_conf(hot_standby=True, - wal_keep_segments=WAL_KEEP_SEGMENTS, - wal_level=wal_level) # yapf: disable + if self._pg_version < '13': + self.append_conf(hot_standby=True, + wal_keep_segments=WAL_KEEP_SEGMENTS, + wal_level=wal_level) # yapf: disable + else: + self.append_conf(hot_standby=True, + wal_keep_size=WAL_KEEP_SIZE, + wal_level=wal_level) # yapf: disable # logical replication if allow_logical: From 6e14fbb2d6e1b8ba679451a2c6893c01ccbc6e98 Mon Sep 17 00:00:00 2001 From: anastasia Date: Mon, 19 Oct 2020 17:31:19 +0300 Subject: [PATCH 222/475] Add PostgreSQL 13 support. Fix --- testgres/node.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/testgres/node.py b/testgres/node.py index 81866e89..6bd17620 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -40,7 +40,8 @@ MAX_REPLICATION_SLOTS, \ MAX_WORKER_PROCESSES, \ MAX_WAL_SENDERS, \ - WAL_KEEP_SEGMENTS + WAL_KEEP_SEGMENTS, \ + WAL_KEEP_SIZE from .decorators import \ method_decorator, \ From d65ab69bd55771497e3d3a2e2b2f975c86e17e4b Mon Sep 17 00:00:00 2001 From: anastasia Date: Mon, 19 Oct 2020 17:53:48 +0300 Subject: [PATCH 223/475] fix code style to pass flake8 test --- testgres/node.py | 4 ++-- testgres/utils.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index 6bd17620..5835541d 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -526,8 +526,8 @@ def get_auth_method(t): wal_level=wal_level) # yapf: disable else: self.append_conf(hot_standby=True, - wal_keep_size=WAL_KEEP_SIZE, - wal_level=wal_level) # yapf: disable + wal_keep_size=WAL_KEEP_SIZE, + wal_level=wal_level) # yapf: disable # logical replication if allow_logical: diff --git a/testgres/utils.py b/testgres/utils.py index 6c6f0e9b..47228e60 100644 --- a/testgres/utils.py +++ b/testgres/utils.py @@ -79,7 +79,7 @@ def execute_utility(args, logfile=None): if out: # comment-out lines - lines = ('# ' + l for l in out.splitlines(True)) + lines = ('# ' + line for line in out.splitlines(True)) file_out.write(u'\n') file_out.writelines(lines) From 7ee0c2016727bc0abdfd26f587086f76656e9b7d Mon Sep 17 00:00:00 2001 From: anastasia Date: Tue, 20 Oct 2020 12:08:58 +0300 Subject: [PATCH 224/475] Fix synchronous standbys test --- tests/test_simple.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_simple.py b/tests/test_simple.py index 0ef7306f..c27fe927 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -437,7 +437,7 @@ def test_synchronous_replication(self): str(Any(1, (standby1, standby2)))) # yapf: disable # set synchronous_standby_names - master.set_synchronous_standbys([standby1, standby2]) + master.set_synchronous_standbys(First(2, [standby1, standby2])) master.restart() # the following part of the test is only applicable to newer From 987e327ee003415b2b6b58b0c8b6bcb99db0a8cb Mon Sep 17 00:00:00 2001 From: anastasia Date: Tue, 20 Oct 2020 12:27:59 +0300 Subject: [PATCH 225/475] Skip test test_synchronous_replication for versions < 9.6 --- tests/test_simple.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_simple.py b/tests/test_simple.py index c27fe927..219d2f19 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -414,6 +414,7 @@ def test_replicate(self): res = node.execute('select * from test') self.assertListEqual(res, []) + @unittest.skipUnless(pg_version_ge('9.6'), 'requires 9.6+') def test_synchronous_replication(self): with get_new_node() as master: old_version = not pg_version_ge('9.6') From a3f6dd320529882417e04f960933f7ded191b0e5 Mon Sep 17 00:00:00 2001 From: anastasia Date: Tue, 20 Oct 2020 14:25:34 +0300 Subject: [PATCH 226/475] Revert "Skip cleanup of custom directory was provided" Per perort in [issue #57]: cleanup backward compatibility broken in 1.8.3. If this feature is needed, feel free to bring it back, but maintain compatibility. This reverts commit cdc933ff4af3f956e43e5ed84974889cc5163127. --- testgres/node.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index 5835541d..83e94705 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -119,12 +119,10 @@ def __init__(self, name=None, port=None, base_dir=None): # private self._pg_version = PgVer(get_pg_version()) self._should_free_port = port is None + self._base_dir = base_dir self._logger = None self._master = None - self._custom_base_dir = base_dir - self._base_dir = base_dir - # basic self.host = '127.0.0.1' self.name = name or generate_app_name() @@ -817,13 +815,11 @@ def cleanup(self, max_attempts=3): self._try_shutdown(max_attempts) - # only remove if base directory was temporary - if not self._custom_base_dir: - # choose directory to be removed - if testgres_config.node_cleanup_full: - rm_dir = self.base_dir # everything - else: - rm_dir = self.data_dir # just data, save logs + # choose directory to be removed + if testgres_config.node_cleanup_full: + rm_dir = self.base_dir # everything + else: + rm_dir = self.data_dir # just data, save logs rmtree(rm_dir, ignore_errors=True) From 2041d5e209f3780cdf7dffa02b0864f81398722a Mon Sep 17 00:00:00 2001 From: anastasia Date: Tue, 20 Oct 2020 14:54:13 +0300 Subject: [PATCH 227/475] Revert "Fix tests" Follows previous revert of "Skip cleanup of custom directory was provided" This reverts commit 2ef33a80d61d2dc3c78a7faa671ad4aa524e6cc4. --- testgres/node.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testgres/node.py b/testgres/node.py index 83e94705..fff06593 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -821,7 +821,7 @@ def cleanup(self, max_attempts=3): else: rm_dir = self.data_dir # just data, save logs - rmtree(rm_dir, ignore_errors=True) + rmtree(rm_dir, ignore_errors=True) return self From 1825ebdfa5daa5b5c928edb6b0cd7efe78c18fc3 Mon Sep 17 00:00:00 2001 From: Arseny Sher Date: Wed, 21 Oct 2020 14:11:19 +0300 Subject: [PATCH 228/475] Bump version 1.8.4. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 05ee76cd..d497067c 100755 --- a/setup.py +++ b/setup.py @@ -21,7 +21,7 @@ readme = f.read() setup( - version='1.8.3', + version='1.8.4', name='testgres', packages=['testgres'], description='Testing utility for PostgreSQL and its extensions', From 65f67b32be514fdc989654c7e5c09f5e25d8d40e Mon Sep 17 00:00:00 2001 From: Anisimov-ds Date: Tue, 15 Feb 2022 17:16:09 +0700 Subject: [PATCH 229/475] Fix some bugs with pipe on Windows --- testgres/node.py | 6 ++++-- testgres/utils.py | 34 +++++++++++++++++++++++++++------- 2 files changed, 31 insertions(+), 9 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index fff06593..8e64da01 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -865,8 +865,7 @@ def psql(self, "-X", # no .psqlrc "-A", # unaligned output "-t", # print rows only - "-q", # run quietly - dbname + "-q" # run quietly ] # yapf: disable # set variables before execution @@ -881,6 +880,9 @@ def psql(self, else: raise QueryException('Query or filename must be provided') + # should be the last one + psql_params.append(dbname) + # start psql process process = subprocess.Popen( psql_params, diff --git a/testgres/utils.py b/testgres/utils.py index 47228e60..a33f49b6 100644 --- a/testgres/utils.py +++ b/testgres/utils.py @@ -8,6 +8,7 @@ import port_for import subprocess import sys +import tempfile from contextlib import contextmanager from distutils.version import LooseVersion @@ -59,13 +60,32 @@ def execute_utility(args, logfile=None): """ # run utility - process = subprocess.Popen( - args, # util + params - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) - - # get result and decode it - out, _ = process.communicate() + if os.name == 'nt': + # using output to a temporary file in Windows + buf = tempfile.NamedTemporaryFile() + + process = subprocess.Popen( + args, # util + params + stdout=buf, + stderr=subprocess.STDOUT + ) + process.communicate() + + # get result + buf.file.flush() + buf.file.seek(0) + out = buf.file.read() + buf.close() + else: + process = subprocess.Popen( + args, # util + params + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + + # get result + out, _ = process.communicate() + + # decode result out = '' if not out else out.decode('utf-8') # format command From fe91e3f52816f0d69b292e2cbbaeeed21777e7d3 Mon Sep 17 00:00:00 2001 From: Shenghao Yang Date: Fri, 1 Apr 2022 18:16:26 +0800 Subject: [PATCH 230/475] node: fix deprecated Iterable import in py3.10 Import from collections.abc instead if import from collections fails. --- testgres/node.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/testgres/node.py b/testgres/node.py index 8e64da01..d1bb1d97 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -6,7 +6,11 @@ import subprocess import time -from collections import Iterable +try: + from collections import Iterable +except ImportError: + from collections.abc import Iterable + from shutil import rmtree from six import raise_from, iteritems, text_type from tempfile import mkstemp, mkdtemp From 562a361b33f8a26914b0fc48f0acfe537758d7bc Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" Date: Mon, 11 Apr 2022 17:11:12 +0300 Subject: [PATCH 231/475] Add 14 to travis --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 0e7a3701..80cb5260 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,6 +22,7 @@ env: - PYTHON_VERSION=2 PG_VERSION=9.6 - PYTHON_VERSION=2 PG_VERSION=9.5 - PYTHON_VERSION=2 PG_VERSION=9.4 + - PYTHON_VERSION=3 PG_VERSION=14 - PYTHON_VERSION=3 PG_VERSION=13 - PYTHON_VERSION=3 PG_VERSION=12 - PYTHON_VERSION=3 PG_VERSION=11 From 0654c210f8281fd9d79d0841dcc95dfa7106feb4 Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" Date: Mon, 11 Apr 2022 17:32:45 +0300 Subject: [PATCH 232/475] update travis config --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 80cb5260..04a9b37f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,6 @@ -sudo: required +os: linux + +dist: bionic language: python From 662f9eb3d9337c4581007746bd82d83576223206 Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" Date: Mon, 11 Apr 2022 17:36:59 +0300 Subject: [PATCH 233/475] make yapf3 and flake8 happy (somebody don't use pre-commit git hook) --- testgres/backup.py | 1 - testgres/cache.py | 1 - testgres/config.py | 1 - testgres/connection.py | 12 ++++----- testgres/decorators.py | 1 - testgres/logger.py | 1 - testgres/node.py | 61 ++++++++++++++++++++---------------------- testgres/pubsub.py | 32 +++++++++++----------- testgres/standby.py | 12 ++++----- testgres/utils.py | 10 +++---- tests/test_simple.py | 50 +++++++++++++++++----------------- 11 files changed, 84 insertions(+), 98 deletions(-) diff --git a/testgres/backup.py b/testgres/backup.py index d9865c1e..a725a1df 100644 --- a/testgres/backup.py +++ b/testgres/backup.py @@ -29,7 +29,6 @@ class NodeBackup(object): """ Smart object responsible for backups """ - @property def log_file(self): return os.path.join(self.base_dir, BACKUP_LOG_FILE) diff --git a/testgres/cache.py b/testgres/cache.py index 36d6e768..c3cd9971 100644 --- a/testgres/cache.py +++ b/testgres/cache.py @@ -25,7 +25,6 @@ def cached_initdb(data_dir, logfile=None, params=None): """ Perform initdb or use cached node files. """ - def call_initdb(initdb_dir, log=None): try: _params = [get_bin_path("initdb"), "-D", initdb_dir, "-N"] diff --git a/testgres/config.py b/testgres/config.py index d1f853c0..cfcdadc2 100644 --- a/testgres/config.py +++ b/testgres/config.py @@ -43,7 +43,6 @@ class GlobalConfig(object): _cached_initdb_dir = None """ underlying class attribute for cached_initdb_dir property """ - @property def cached_initdb_dir(self): """ path to a temp directory for cached initdb. """ diff --git a/testgres/connection.py b/testgres/connection.py index ab846a14..ee2a2128 100644 --- a/testgres/connection.py +++ b/testgres/connection.py @@ -28,7 +28,6 @@ class NodeConnection(object): """ Transaction wrapper returned by Node """ - def __init__(self, node, dbname=None, @@ -42,12 +41,11 @@ def __init__(self, self._node = node - self._connection = pglib.connect( - database=dbname, - user=username, - password=password, - host=node.host, - port=node.port) + self._connection = pglib.connect(database=dbname, + user=username, + password=password, + host=node.host, + port=node.port) self._connection.autocommit = autocommit self._cursor = self.connection.cursor() diff --git a/testgres/decorators.py b/testgres/decorators.py index 2a57b83d..7f383ae7 100644 --- a/testgres/decorators.py +++ b/testgres/decorators.py @@ -50,7 +50,6 @@ def method_decorator(decorator): """ Convert a function decorator into a method decorator. """ - def _dec(func): def _wrapper(self, *args, **kwargs): @decorator diff --git a/testgres/logger.py b/testgres/logger.py index 820998a6..b4648f44 100644 --- a/testgres/logger.py +++ b/testgres/logger.py @@ -10,7 +10,6 @@ class TestgresLogger(threading.Thread): """ Helper class to implement reading from log files. """ - def __init__(self, node_name, log_file_name): threading.Thread.__init__(self) diff --git a/testgres/node.py b/testgres/node.py index 8e64da01..646a5b2f 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -91,7 +91,6 @@ class ProcessProxy(object): process: wrapped psutill.Process object ptype: instance of ProcessType """ - def __init__(self, process, ptype=None): self.process = process self.ptype = ptype or ProcessType.from_process(process) @@ -196,7 +195,6 @@ def auxiliary_processes(self): Returns a list of auxiliary processes. Each process is represented by :class:`.ProcessProxy` object. """ - def is_aux(process): return process.ptype != ProcessType.Unknown @@ -430,10 +428,9 @@ def init(self, initdb_params=None, **kwargs): """ # initialize this PostgreSQL node - cached_initdb( - data_dir=self.data_dir, - logfile=self.utils_log_file, - params=initdb_params) + cached_initdb(data_dir=self.data_dir, + logfile=self.utils_log_file, + params=initdb_params) # initialize default config files self.default_conf(**kwargs) @@ -480,8 +477,8 @@ def default_conf(self, if allow_streaming: # get auth method for host or local users def get_auth_method(t): - return next((s.split()[-1] for s in lines - if s.startswith(t)), 'trust') + return next((s.split()[-1] + for s in lines if s.startswith(t)), 'trust') # get auth methods auth_local = get_auth_method('local') @@ -760,12 +757,11 @@ def promote(self, dbname=None, username=None): if self._pg_version < '10': check_query = "SELECT pg_is_in_recovery()" - self.poll_query_until( - query=check_query, - expected=False, - dbname=dbname, - username=username, - max_attempts=0) # infinite + self.poll_query_until(query=check_query, + expected=False, + dbname=dbname, + username=username, + max_attempts=0) # infinite # node becomes master itself self._master = None @@ -884,11 +880,10 @@ def psql(self, psql_params.append(dbname) # start psql process - process = subprocess.Popen( - psql_params, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + process = subprocess.Popen(psql_params, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) # wait until it finishes and get stdout and stderr out, err = process.communicate(input=input) @@ -1043,11 +1038,10 @@ def poll_query_until(self, attempts = 0 while max_attempts == 0 or attempts < max_attempts: try: - res = self.execute( - dbname=dbname, - query=query, - username=username, - commit=commit) + res = self.execute(dbname=dbname, + query=query, + username=username, + commit=commit) if expected is None and res is None: return # done @@ -1165,8 +1159,8 @@ def set_synchronous_standbys(self, standbys): standbys = First(1, standbys) else: if isinstance(standbys, Iterable): - standbys = u", ".join( - u"\"{}\"".format(r.name) for r in standbys) + standbys = u", ".join(u"\"{}\"".format(r.name) + for r in standbys) else: raise TestgresException("Feature isn't supported in " "Postgres 9.5 and below") @@ -1195,11 +1189,10 @@ def catchup(self, dbname=None, username=None): username=username)[0][0] # yapf: disable # wait until this LSN reaches replica - self.poll_query_until( - query=wait_lsn.format(lsn), - dbname=dbname, - username=username, - max_attempts=0) # infinite + self.poll_query_until(query=wait_lsn.format(lsn), + dbname=dbname, + username=username, + max_attempts=0) # infinite except Exception as e: raise_from(CatchUpException("Failed to catch up", poll_lsn), e) @@ -1215,7 +1208,11 @@ def publish(self, name, **kwargs): """ return Publication(name=name, node=self, **kwargs) - def subscribe(self, publication, name, dbname=None, username=None, + def subscribe(self, + publication, + name, + dbname=None, + username=None, **params): """ Create subscription for logical replication diff --git a/testgres/pubsub.py b/testgres/pubsub.py index 07b83543..da85caac 100644 --- a/testgres/pubsub.py +++ b/testgres/pubsub.py @@ -77,10 +77,9 @@ def drop(self, dbname=None, username=None): """ Drop publication """ - self.node.execute( - "drop publication {}".format(self.name), - dbname=dbname, - username=username) + self.node.execute("drop publication {}".format(self.name), + dbname=dbname, + username=username) def add_tables(self, tables, dbname=None, username=None): """ @@ -94,10 +93,9 @@ def add_tables(self, tables, dbname=None, username=None): raise ValueError("Tables list is empty") query = "alter publication {} add table {}" - self.node.execute( - query.format(self.name, ", ".join(tables)), - dbname=dbname or self.dbname, - username=username or self.username) + self.node.execute(query.format(self.name, ", ".join(tables)), + dbname=dbname or self.dbname, + username=username or self.username) class Subscription(object): @@ -165,19 +163,17 @@ def refresh(self, copy_data=True, dbname=None, username=None): Disables the running subscription. """ query = "alter subscription {} refresh publication with (copy_data={})" - self.node.execute( - query.format(self.name, copy_data), - dbname=dbname, - username=username) + self.node.execute(query.format(self.name, copy_data), + dbname=dbname, + username=username) def drop(self, dbname=None, username=None): """ Drops subscription """ - self.node.execute( - "drop subscription {}".format(self.name), - dbname=dbname, - username=username) + self.node.execute("drop subscription {}".format(self.name), + dbname=dbname, + username=username) def catchup(self, username=None): """ @@ -191,7 +187,9 @@ def catchup(self, username=None): dbname=None, username=None)[0][0] # yapf: disable # create dummy xact, as LR replicates only on commit. - self.pub.node.execute(query="select txid_current()", dbname=None, username=None) + self.pub.node.execute(query="select txid_current()", + dbname=None, + username=None) query = """ select '{}'::pg_lsn - replay_lsn <= 0 from pg_catalog.pg_stat_replication where application_name = '{}' diff --git a/testgres/standby.py b/testgres/standby.py index e7ce408d..859f874e 100644 --- a/testgres/standby.py +++ b/testgres/standby.py @@ -16,14 +16,14 @@ class First: standbys (:obj:`list` of :class:`.PostgresNode`): the list of standby nodes """ - def __init__(self, sync_num, standbys): self.sync_num = sync_num self.standbys = standbys def __str__(self): - return u"{} ({})".format(self.sync_num, u", ".join( - u"\"{}\"".format(r.name) for r in self.standbys)) + return u"{} ({})".format( + self.sync_num, + u", ".join(u"\"{}\"".format(r.name) for r in self.standbys)) @six.python_2_unicode_compatible @@ -39,11 +39,11 @@ class Any: standbys (:obj:`list` of :class:`.PostgresNode`): the list of standby nodes """ - def __init__(self, sync_num, standbys): self.sync_num = sync_num self.standbys = standbys def __str__(self): - return u"ANY {} ({})".format(self.sync_num, u", ".join( - u"\"{}\"".format(r.name) for r in self.standbys)) + return u"ANY {} ({})".format( + self.sync_num, + u", ".join(u"\"{}\"".format(r.name) for r in self.standbys)) diff --git a/testgres/utils.py b/testgres/utils.py index a33f49b6..84b17a6d 100644 --- a/testgres/utils.py +++ b/testgres/utils.py @@ -67,8 +67,7 @@ def execute_utility(args, logfile=None): process = subprocess.Popen( args, # util + params stdout=buf, - stderr=subprocess.STDOUT - ) + stderr=subprocess.STDOUT) process.communicate() # get result @@ -110,8 +109,10 @@ def execute_utility(args, logfile=None): exit_code = process.returncode if exit_code: message = 'Utility exited with non-zero code' - raise ExecUtilException( - message=message, command=command, exit_code=exit_code, out=out) + raise ExecUtilException(message=message, + command=command, + exit_code=exit_code, + out=out) return out @@ -150,7 +151,6 @@ def get_pg_config(pg_config_path=None): Return output of pg_config (provided that it is installed). NOTE: this fuction caches the result by default (see GlobalConfig). """ - def cache_pg_config_data(cmd): # execute pg_config and get the output out = subprocess.check_output([cmd]).decode('utf-8') diff --git a/tests/test_simple.py b/tests/test_simple.py index 219d2f19..00a50f5b 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -131,8 +131,8 @@ def test_init_unique_system_id(self): with get_new_node().init().start() as node0: id0 = node0.execute(query)[0] - with scoped_config( - cache_initdb=True, cached_initdb_unique=True) as config: + with scoped_config(cache_initdb=True, + cached_initdb_unique=True) as config: self.assertTrue(config.cache_initdb) self.assertTrue(config.cached_initdb_unique) @@ -276,8 +276,8 @@ def test_psql(self): # check feeding input node.safe_psql('create table horns (w int)') - node.safe_psql( - 'copy horns from stdin (format csv)', input=b"1\n2\n3\n\\.\n") + node.safe_psql('copy horns from stdin (format csv)', + input=b"1\n2\n3\n\\.\n") _sum = node.safe_psql('select sum(w) from horns') self.assertEqual(_sum, b'6\n') @@ -383,8 +383,8 @@ def test_backup_wrong_xlog_method(self): with get_new_node() as node: node.init(allow_streaming=True).start() - with self.assertRaises( - BackupException, msg='Invalid xlog_method "wrong"'): + with self.assertRaises(BackupException, + msg='Invalid xlog_method "wrong"'): node.backup(xlog_method='wrong') def test_pg_ctl_wait_option(self): @@ -637,8 +637,8 @@ def test_poll_query_until(self): node.poll_query_until(query='create table abc (val int)') # check None, ok - node.poll_query_until( - query='create table def()', expected=None) # returns nothing + node.poll_query_until(query='create table def()', + expected=None) # returns nothing # check 0 rows equivalent to expected=None node.poll_query_until( @@ -647,19 +647,19 @@ def test_poll_query_until(self): # check arbitrary expected value, fail with self.assertRaises(TimeoutException): - node.poll_query_until( - query='select 3', - expected=1, - max_attempts=3, - sleep_time=0.01) + node.poll_query_until(query='select 3', + expected=1, + max_attempts=3, + sleep_time=0.01) # check arbitrary expected value, ok node.poll_query_until(query='select 2', expected=2) # check timeout with self.assertRaises(TimeoutException): - node.poll_query_until( - query='select 1 > 2', max_attempts=3, sleep_time=0.01) + node.poll_query_until(query='select 1 > 2', + max_attempts=3, + sleep_time=0.01) # check ProgrammingError, fail with self.assertRaises(testgres.ProgrammingError): @@ -667,11 +667,10 @@ def test_poll_query_until(self): # check ProgrammingError, ok with self.assertRaises(TimeoutException): - node.poll_query_until( - query='dummy2', - max_attempts=3, - sleep_time=0.01, - suppress={testgres.ProgrammingError}) + node.poll_query_until(query='dummy2', + max_attempts=3, + sleep_time=0.01, + suppress={testgres.ProgrammingError}) # check 1 arg, ok node.poll_query_until('select true') @@ -732,14 +731,13 @@ def test_pgbench(self): with get_new_node().init().start() as node: # initialize pgbench DB and run benchmarks - node.pgbench_init( - scale=2, foreign_keys=True, options=['-q']).pgbench_run(time=2) + node.pgbench_init(scale=2, foreign_keys=True, + options=['-q']).pgbench_run(time=2) # run TPC-B benchmark - proc = node.pgbench( - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - options=['-T3']) + proc = node.pgbench(stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=['-T3']) out, _ = proc.communicate() out = out.decode('utf-8') From 0e6e6f0ba9d3d47fc5f2311a0d9007974976606d Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" Date: Mon, 11 Apr 2022 17:46:08 +0300 Subject: [PATCH 234/475] Commented out running travis tests for legacy versions (to reduce the load on dockerhub and make the "toomanyrequests: You have reached your pull rate limit." error less likely) --- .travis.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 04a9b37f..c06cab3d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,15 +20,15 @@ notifications: on_failure: always env: - - PYTHON_VERSION=2 PG_VERSION=10 - - PYTHON_VERSION=2 PG_VERSION=9.6 - - PYTHON_VERSION=2 PG_VERSION=9.5 - - PYTHON_VERSION=2 PG_VERSION=9.4 - PYTHON_VERSION=3 PG_VERSION=14 - PYTHON_VERSION=3 PG_VERSION=13 - PYTHON_VERSION=3 PG_VERSION=12 - PYTHON_VERSION=3 PG_VERSION=11 - PYTHON_VERSION=3 PG_VERSION=10 - - PYTHON_VERSION=3 PG_VERSION=9.6 - - PYTHON_VERSION=3 PG_VERSION=9.5 - - PYTHON_VERSION=3 PG_VERSION=9.4 +# - PYTHON_VERSION=3 PG_VERSION=9.6 +# - PYTHON_VERSION=3 PG_VERSION=9.5 +# - PYTHON_VERSION=3 PG_VERSION=9.4 +# - PYTHON_VERSION=2 PG_VERSION=10 +# - PYTHON_VERSION=2 PG_VERSION=9.6 +# - PYTHON_VERSION=2 PG_VERSION=9.5 +# - PYTHON_VERSION=2 PG_VERSION=9.4 From 73135272fd84108e9403eb9dada9ae2024efb773 Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" Date: Mon, 11 Apr 2022 19:52:09 +0300 Subject: [PATCH 235/475] python-3.10: moved more generic imports up --- testgres/node.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index ae41fe1f..cabbbd1a 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -7,9 +7,9 @@ import time try: - from collections import Iterable -except ImportError: from collections.abc import Iterable +except ImportError: + from collections import Iterable from shutil import rmtree from six import raise_from, iteritems, text_type From 402404900ff43a32f71fc7f2302001a0c37e986e Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" Date: Tue, 24 May 2022 17:21:44 +0300 Subject: [PATCH 236/475] Bump version 1.8.5. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d497067c..a5dc600e 100755 --- a/setup.py +++ b/setup.py @@ -21,7 +21,7 @@ readme = f.read() setup( - version='1.8.4', + version='1.8.5', name='testgres', packages=['testgres'], description='Testing utility for PostgreSQL and its extensions', From b84647eb13a44668fa89b1badb4ffb5c23cfdacb Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" Date: Mon, 3 Oct 2022 16:15:37 +0300 Subject: [PATCH 237/475] [PBCKP-289] fix F401 and F403 flake8 errors --- testgres/__init__.py | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/testgres/__init__.py b/testgres/__init__.py index f8d895a5..9d5e37cf 100644 --- a/testgres/__init__.py +++ b/testgres/__init__.py @@ -15,8 +15,23 @@ ProgrammingError, \ OperationalError -from .exceptions import * -from .enums import * +from .exceptions import \ + TestgresException, \ + ExecUtilException, \ + QueryException, \ + TimeoutException, \ + CatchUpException, \ + StartNodeException, \ + InitNodeException, \ + BackupException + +from .enums import \ + XLogMethod, \ + IsolationLevel, \ + NodeStatus, \ + ProcessType, \ + DumpFormat + from .node import PostgresNode from .utils import \ @@ -30,3 +45,15 @@ from .standby import \ First, \ Any + +__all__ = [ + "get_new_node", + "NodeBackup", + "TestgresConfig", "configure_testgres", "scoped_config", "push_config", "pop_config", + "NodeConnection", "DatabaseError", "InternalError", "ProgrammingError", "OperationalError", + "TestgresException", "ExecUtilException", "QueryException", "TimeoutException", "CatchUpException", "StartNodeException", "InitNodeException", "BackupException", + "XLogMethod", "IsolationLevel", "NodeStatus", "ProcessType", "DumpFormat", + "PostgresNode", + "reserve_port", "release_port", "bound_ports", "get_bin_path", "get_pg_config", "get_pg_version", + "First", "Any", +] From f457a29987ac86cbf5beb687618791d12e0cc51b Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" Date: Mon, 3 Oct 2022 17:13:33 +0300 Subject: [PATCH 238/475] [PBCKP-289] Fix sphinxcontrib.napoleon import error Error was: "Could not import extension sphinxcontrib.napoleon (exception: cannot import name 'Callable' from 'collections' (/usr/lib/python3.10/collections/__init__.py))" --- docs/README.md | 6 +++--- docs/source/conf.py | 4 ++-- run_tests.sh | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/README.md b/docs/README.md index 1e520e83..354ce2a2 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,9 +1,9 @@ # Building the documentation -Make sure you have `Sphinx` and `sphinxcontrib-napoleon` packages installed: +Make sure you have `Sphinx` package installed: ``` -pip install Sphinx sphinxcontrib-napoleon +pip install Sphinx ``` Then just run @@ -12,4 +12,4 @@ Then just run make html ``` -Documentation will be built in `build/html` directory. Other output formats are also available; run `make` without arguments to see the options. \ No newline at end of file +Documentation will be built in `build/html` directory. Other output formats are also available; run `make` without arguments to see the options. diff --git a/docs/source/conf.py b/docs/source/conf.py index 7ba79351..60709bd5 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -19,7 +19,7 @@ # -- Project information ----------------------------------------------------- project = u'testgres' -copyright = u'2016-2018, Postgres Professional' +copyright = u'2016-2022, Postgres Professional' author = u'Postgres Professional' # The short X.Y version @@ -36,7 +36,7 @@ # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = ['sphinx.ext.autodoc', 'sphinxcontrib.napoleon'] +extensions = ['sphinx.ext.autodoc', 'sphinx.ext.napoleon'] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] diff --git a/run_tests.sh b/run_tests.sh index 5d986f85..f71a055b 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# Copyright (c) 2017, Postgres Professional +# Copyright (c) 2017-2022 Postgres Professional set -eux @@ -22,7 +22,7 @@ export VIRTUAL_ENV_DISABLE_PROMPT=1 source $VENV_PATH/bin/activate # install utilities -$PIP install coverage flake8 psutil Sphinx sphinxcontrib-napoleon +$PIP install coverage flake8 psutil Sphinx # install testgres' dependencies export PYTHONPATH=$(pwd) From 52debdacbdf4a02bfbd96ae1364e8d580fdff68b Mon Sep 17 00:00:00 2001 From: "Mikhail A. Kulagin" Date: Mon, 3 Oct 2022 23:05:19 +0300 Subject: [PATCH 239/475] [PBCKP-289] fix distutils deprecation warnings Warning was: "DeprecationWarning: distutils Version classes are deprecated. Use packaging.version instead." --- testgres/node.py | 18 +++++++++--------- testgres/utils.py | 4 ++-- tests/test_simple.py | 10 ++++++++-- 3 files changed, 19 insertions(+), 13 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index cabbbd1a..378e6803 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -339,7 +339,7 @@ def _create_recovery_conf(self, username, slot=None): "primary_conninfo='{}'\n" ).format(options_string(**conninfo)) # yapf: disable # Since 12 recovery.conf had disappeared - if self.version >= '12': + if self.version >= PgVer('12'): signal_name = os.path.join(self.data_dir, "standby.signal") # cross-python touch(). It is vulnerable to races, but who cares? with open(signal_name, 'a'): @@ -371,7 +371,7 @@ def _create_recovery_conf(self, username, slot=None): line += "primary_slot_name={}\n".format(slot) - if self.version >= '12': + if self.version >= PgVer('12'): self.append_conf(line=line) else: self.append_conf(filename=RECOVERY_CONF_FILE, line=line) @@ -517,9 +517,9 @@ def get_auth_method(t): # binary replication if allow_streaming: # select a proper wal_level for PostgreSQL - wal_level = 'replica' if self._pg_version >= '9.6' else 'hot_standby' + wal_level = 'replica' if self._pg_version >= PgVer('9.6') else 'hot_standby' - if self._pg_version < '13': + if self._pg_version < PgVer('13'): self.append_conf(hot_standby=True, wal_keep_segments=WAL_KEEP_SEGMENTS, wal_level=wal_level) # yapf: disable @@ -530,7 +530,7 @@ def get_auth_method(t): # logical replication if allow_logical: - if self._pg_version < '10': + if self._pg_version < PgVer('10'): raise InitNodeException("Logical replication is only " "available on PostgreSQL 10 and newer") @@ -616,7 +616,7 @@ def get_control_data(self): # this one is tricky (blame PG 9.4) _params = [get_bin_path("pg_controldata")] - _params += ["-D"] if self._pg_version >= '9.5' else [] + _params += ["-D"] if self._pg_version >= PgVer('9.5') else [] _params += [self.data_dir] data = execute_utility(_params, self.utils_log_file) @@ -758,7 +758,7 @@ def promote(self, dbname=None, username=None): # for versions below 10 `promote` is asynchronous so we need to wait # until it actually becomes writable - if self._pg_version < '10': + if self._pg_version < PgVer('10'): check_query = "SELECT pg_is_in_recovery()" self.poll_query_until(query=check_query, @@ -1158,7 +1158,7 @@ def set_synchronous_standbys(self, standbys): master.restart() """ - if self._pg_version >= '9.6': + if self._pg_version >= PgVer('9.6'): if isinstance(standbys, Iterable): standbys = First(1, standbys) else: @@ -1179,7 +1179,7 @@ def catchup(self, dbname=None, username=None): if not self.master: raise TestgresException("Node doesn't have a master") - if self._pg_version >= '10': + if self._pg_version >= PgVer('10'): poll_lsn = "select pg_catalog.pg_current_wal_lsn()::text" wait_lsn = "select pg_catalog.pg_last_wal_replay_lsn() >= '{}'::pg_lsn" else: diff --git a/testgres/utils.py b/testgres/utils.py index 84b17a6d..4d99c69d 100644 --- a/testgres/utils.py +++ b/testgres/utils.py @@ -11,7 +11,7 @@ import tempfile from contextlib import contextmanager -from distutils.version import LooseVersion +from packaging.version import Version from distutils.spawn import find_executable from six import iteritems @@ -25,7 +25,7 @@ bound_ports = set() # re-export version type -PgVer = LooseVersion +PgVer = Version def reserve_port(): diff --git a/tests/test_simple.py b/tests/test_simple.py index 00a50f5b..9ddf5baa 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -896,16 +896,22 @@ def test_version_management(self): a = PgVer('10.0') b = PgVer('10') c = PgVer('9.6.5') + d = PgVer('15.0') + e = PgVer('15rc1') + f = PgVer('15beta4') - self.assertTrue(a > b) + self.assertTrue(a == b) self.assertTrue(b > c) self.assertTrue(a > c) + self.assertTrue(d > e) + self.assertTrue(e > f) + self.assertTrue(d > f) version = get_pg_version() with get_new_node() as node: self.assertTrue(isinstance(version, six.string_types)) self.assertTrue(isinstance(node.version, PgVer)) - self.assertEqual(node.version, str(version)) + self.assertEqual(node.version, PgVer(version)) def test_child_pids(self): master_processes = [ From aa31edfd17e985da9395cbd2e338a8b31e04c2db Mon Sep 17 00:00:00 2001 From: Sokolov Yura Date: Thu, 24 Nov 2022 15:40:42 +0300 Subject: [PATCH 240/475] README.md: fix travis urls Fixes #68 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f020273c..6b26ba96 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -[![Build Status](https://travis-ci.org/postgrespro/testgres.svg?branch=master)](https://travis-ci.org/postgrespro/testgres) +[![Build Status](https://travis-ci.com/postgrespro/testgres.svg?branch=master)](https://app.travis-ci.com/github/postgrespro/testgres/branches) [![codecov](https://codecov.io/gh/postgrespro/testgres/branch/master/graph/badge.svg)](https://codecov.io/gh/postgrespro/testgres) [![PyPI version](https://badge.fury.io/py/testgres.svg)](https://badge.fury.io/py/testgres) From 6dd87598c04dca942d556af023dd3c50b418f05e Mon Sep 17 00:00:00 2001 From: "v.shepard" Date: Thu, 8 Dec 2022 01:50:17 +0100 Subject: [PATCH 241/475] PBCKP-237 add s3 tests to jenkins --- __init__.py | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 __init__.py diff --git a/__init__.py b/__init__.py new file mode 100644 index 00000000..72005bdd --- /dev/null +++ b/__init__.py @@ -0,0 +1,11 @@ +from testgres.testgres import PostgresNode, QueryException, ProcessType, StartNodeException, reserve_port, release_port, \ + bound_ports, get_bin_path, get_pg_config, get_pg_version, First, Any + +__all__ = [ + "PostgresNode", + "QueryException", + "ProcessType", + "StartNodeException", + "reserve_port", "release_port", "bound_ports", "get_bin_path", "get_pg_config", "get_pg_version", + "First", "Any", +] From ceb1ee9553b7fb734cdb8241efc701d3b5671700 Mon Sep 17 00:00:00 2001 From: "v.shepard" Date: Fri, 9 Dec 2022 10:14:04 +0100 Subject: [PATCH 242/475] PBCKP-237 remove __init__.py --- __init__.py | 11 ----------- 1 file changed, 11 deletions(-) delete mode 100644 __init__.py diff --git a/__init__.py b/__init__.py deleted file mode 100644 index 72005bdd..00000000 --- a/__init__.py +++ /dev/null @@ -1,11 +0,0 @@ -from testgres.testgres import PostgresNode, QueryException, ProcessType, StartNodeException, reserve_port, release_port, \ - bound_ports, get_bin_path, get_pg_config, get_pg_version, First, Any - -__all__ = [ - "PostgresNode", - "QueryException", - "ProcessType", - "StartNodeException", - "reserve_port", "release_port", "bound_ports", "get_bin_path", "get_pg_config", "get_pg_version", - "First", "Any", -] From 0e8b1c8f59bd1b57d31b7d31ba82c4efa6149d0a Mon Sep 17 00:00:00 2001 From: Sofia Kopikova Date: Thu, 23 Mar 2023 13:33:46 +0300 Subject: [PATCH 243/475] Catch FileNotFoundError exception When we want to find some Postgres process in testgres psutils lib first gets list of all system processes (children() function in testgres's auxiliary_processes). Next testgres asks for process name (cmdline() function), so psutils goes to /proc/pid/ in Linux. In case any of processes listed in previous step, we get FileNotFoundError --- testgres/enums.py | 13 +++++++++---- tests/test_simple.py | 14 ++++++++++++++ 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/testgres/enums.py b/testgres/enums.py index fb68f2bb..d07d8068 100644 --- a/testgres/enums.py +++ b/testgres/enums.py @@ -1,5 +1,6 @@ from enum import Enum, IntEnum from six import iteritems +from psutil import NoSuchProcess class XLogMethod(Enum): @@ -68,11 +69,15 @@ def from_process(process): ], } # yapf: disable + try: + cmdline = ''.join(process.cmdline()) + except (FileNotFoundError, ProcessLookupError, NoSuchProcess): + return ProcessType.Unknown + # we deliberately cut special words and spaces - cmdline = ''.join(process.cmdline()) \ - .replace('postgres:', '', 1) \ - .replace('bgworker:', '', 1) \ - .replace(' ', '') + cmdline = cmdline.replace('postgres:', '', 1) \ + .replace('bgworker:', '', 1) \ + .replace(' ', '') for ptype in ProcessType: if cmdline.startswith(ptype.value.replace(' ', '')): diff --git a/tests/test_simple.py b/tests/test_simple.py index 9ddf5baa..d79fa79a 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -9,6 +9,7 @@ import time import six import unittest +import psutil import logging.config @@ -48,6 +49,7 @@ # NOTE: those are ugly imports from testgres import bound_ports from testgres.utils import PgVer +from testgres.node import ProcessProxy def pg_version_ge(version): @@ -965,6 +967,18 @@ def test_child_pids(self): with self.assertRaises(TestgresException): replica.source_walsender + def test_child_process_dies(self): + # test for FileNotFound exception during child_processes() function + with subprocess.Popen(["sleep", "60"]) as process: + self.assertEqual(process.poll(), None) + # collect list of processes currently running + children = psutil.Process(os.getpid()).children() + # kill a process, so received children dictionary becomes invalid + process.kill() + process.wait() + # try to handle children list -- missing processes will have ptype "ProcessType.Unknown" + [ProcessProxy(p) for p in children] + if __name__ == '__main__': if os.environ.get('ALT_CONFIG'): From 5d5084ff8354e7d3b628be65f6b844d56d9119ca Mon Sep 17 00:00:00 2001 From: "v.shepard" Date: Mon, 3 Apr 2023 22:24:58 +0200 Subject: [PATCH 244/475] Update version script --- publish_package.sh | 31 +++++++++++++++++++++++++++++++ setup.py | 2 +- 2 files changed, 32 insertions(+), 1 deletion(-) create mode 100755 publish_package.sh diff --git a/publish_package.sh b/publish_package.sh new file mode 100755 index 00000000..f9ef4eb5 --- /dev/null +++ b/publish_package.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +# Copyright (c) 2017-2022 Postgres Professional + +set -eux + + +# choose python version +echo python version is $PYTHON_VERSION +VIRTUALENV="virtualenv --python=/usr/bin/python$PYTHON_VERSION" +PIP="pip$PYTHON_VERSION" + + +# prepare environment +VENV_PATH=/tmp/testgres_venv +rm -rf $VENV_PATH +$VIRTUALENV $VENV_PATH +export VIRTUAL_ENV_DISABLE_PROMPT=1 +source $VENV_PATH/bin/activate + +# install utilities +$PIP install setuptools twine + +# create distribution of the package +alias python3='python' +python setup.py sdist bdist_wheel + +# upload dist +twine upload dist/* + +set +eux diff --git a/setup.py b/setup.py index a5dc600e..5c6f4a07 100755 --- a/setup.py +++ b/setup.py @@ -21,7 +21,7 @@ readme = f.read() setup( - version='1.8.5', + version='1.8.6', name='testgres', packages=['testgres'], description='Testing utility for PostgreSQL and its extensions', From aa013814d57a69f105a2f628259e5b7e7379acff Mon Sep 17 00:00:00 2001 From: "v.shepard" Date: Mon, 24 Apr 2023 12:05:34 +0200 Subject: [PATCH 245/475] PBCKP-601 add publish packages script --- publish_package.sh | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/publish_package.sh b/publish_package.sh index f9ef4eb5..8cfa23e2 100755 --- a/publish_package.sh +++ b/publish_package.sh @@ -4,26 +4,18 @@ set -eux - -# choose python version -echo python version is $PYTHON_VERSION -VIRTUALENV="virtualenv --python=/usr/bin/python$PYTHON_VERSION" -PIP="pip$PYTHON_VERSION" - - # prepare environment -VENV_PATH=/tmp/testgres_venv -rm -rf $VENV_PATH -$VIRTUALENV $VENV_PATH +venv_path=.venv +rm -rf "$venv_path" +virtualenv "$venv_path" export VIRTUAL_ENV_DISABLE_PROMPT=1 -source $VENV_PATH/bin/activate +. "$venv_path"/bin/activate # install utilities -$PIP install setuptools twine +pip3 install setuptools twine # create distribution of the package -alias python3='python' -python setup.py sdist bdist_wheel +python3 setup.py sdist bdist_wheel # upload dist twine upload dist/* From b3777b2c4bedd4435befb924a2e7b96bd5f00c76 Mon Sep 17 00:00:00 2001 From: "v.shepard" Date: Wed, 3 May 2023 12:00:20 +0200 Subject: [PATCH 246/475] PBCKP-377 optionaly keep test logs --- testgres/node.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index 378e6803..e9d8ffd9 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -801,13 +801,15 @@ def free_port(self): self._should_free_port = False release_port(self.port) - def cleanup(self, max_attempts=3): + def cleanup(self, max_attempts=3, full_clean=testgres_config.node_cleanup_full): """ Stop node if needed and remove its data/logs directory. NOTE: take a look at TestgresConfig.node_cleanup_full. Args: max_attempts: how many times should we try to stop()? + full_clean: True - clean all node data + False - clean only data dir, save logs Returns: This instance of :class:`.PostgresNode`. @@ -816,7 +818,7 @@ def cleanup(self, max_attempts=3): self._try_shutdown(max_attempts) # choose directory to be removed - if testgres_config.node_cleanup_full: + if full_clean: rm_dir = self.base_dir # everything else: rm_dir = self.data_dir # just data, save logs From 7a5a56d62a863ffaf887ab790b04cafa7b6958a0 Mon Sep 17 00:00:00 2001 From: Victoria Shepard <5807469+demonolock@users.noreply.github.com> Date: Wed, 3 May 2023 16:03:02 +0200 Subject: [PATCH 247/475] Pbckp 137 vs move pg node (#76) --- testgres/__init__.py | 4 +- testgres/node.py | 305 +++++++++++++++++++++++++++++++++++++++++-- tests/test_simple.py | 4 +- 3 files changed, 301 insertions(+), 12 deletions(-) diff --git a/testgres/__init__.py b/testgres/__init__.py index 9d5e37cf..1b33ba3b 100644 --- a/testgres/__init__.py +++ b/testgres/__init__.py @@ -32,7 +32,7 @@ ProcessType, \ DumpFormat -from .node import PostgresNode +from .node import PostgresNode, NodeApp from .utils import \ reserve_port, \ @@ -53,7 +53,7 @@ "NodeConnection", "DatabaseError", "InternalError", "ProgrammingError", "OperationalError", "TestgresException", "ExecUtilException", "QueryException", "TimeoutException", "CatchUpException", "StartNodeException", "InitNodeException", "BackupException", "XLogMethod", "IsolationLevel", "NodeStatus", "ProcessType", "DumpFormat", - "PostgresNode", + "PostgresNode", "NodeApp", "reserve_port", "release_port", "bound_ports", "get_bin_path", "get_pg_config", "get_pg_version", "First", "Any", ] diff --git a/testgres/node.py b/testgres/node.py index e9d8ffd9..de6a5032 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -2,15 +2,31 @@ import io import os +import random +import shutil +import signal +import threading +from queue import Queue + import psutil import subprocess import time + try: from collections.abc import Iterable except ImportError: from collections import Iterable +# we support both pg8000 and psycopg2 +try: + import psycopg2 as pglib +except ImportError: + try: + import pg8000 as pglib + except ImportError: + raise ImportError("You must have psycopg2 or pg8000 modules installed") + from shutil import rmtree from six import raise_from, iteritems, text_type from tempfile import mkstemp, mkdtemp @@ -86,6 +102,10 @@ from .backup import NodeBackup +InternalError = pglib.InternalError +ProgrammingError = pglib.ProgrammingError +OperationalError = pglib.OperationalError + class ProcessProxy(object): """ @@ -95,6 +115,7 @@ class ProcessProxy(object): process: wrapped psutill.Process object ptype: instance of ProcessType """ + def __init__(self, process, ptype=None): self.process = process self.ptype = ptype or ProcessType.from_process(process) @@ -140,6 +161,9 @@ def __init__(self, name=None, port=None, base_dir=None): self.utils_log_name = self.utils_log_file self.pg_log_name = self.pg_log_file + # Node state + self.is_started = False + def __enter__(self): return self @@ -629,9 +653,39 @@ def get_control_data(self): return out_dict + def slow_start(self, replica=False, dbname='template1', username=default_username()): + """ + Starts the PostgreSQL instance and then polls the instance + until it reaches the expected state (primary or replica). The state is checked + using the pg_is_in_recovery() function. + + Args: + dbname: + username: + replica: If True, waits for the instance to be in recovery (i.e., replica mode). + If False, waits for the instance to be in primary mode. Default is False. + """ + self.start() + + if replica: + query = 'SELECT pg_is_in_recovery()' + else: + query = 'SELECT not pg_is_in_recovery()' + # Call poll_query_until until the expected value is returned + self.poll_query_until(query=query, + dbname=dbname, + username=username, + suppress={InternalError, + QueryException, + ProgrammingError, + OperationalError}) + def start(self, params=[], wait=True): """ - Start this node using pg_ctl. + Starts the PostgreSQL node using pg_ctl if node has not been started. + By default, it waits for the operation to complete before returning. + Optionally, it can return immediately without waiting for the start operation + to complete by setting the `wait` parameter to False. Args: params: additional arguments for pg_ctl. @@ -640,6 +694,8 @@ def start(self, params=[], wait=True): Returns: This instance of :class:`.PostgresNode`. """ + if self.is_started: + return self _params = [ get_bin_path("pg_ctl"), @@ -657,20 +713,22 @@ def start(self, params=[], wait=True): raise_from(StartNodeException(msg, files), e) self._maybe_start_logger() - + self.is_started = True return self def stop(self, params=[], wait=True): """ - Stop this node using pg_ctl. + Stops the PostgreSQL node using pg_ctl if the node has been started. Args: - params: additional arguments for pg_ctl. - wait: wait until operation completes. + params: A list of additional arguments for pg_ctl. Defaults to None. + wait: If True, waits until the operation is complete. Defaults to True. Returns: This instance of :class:`.PostgresNode`. """ + if not self.is_started: + return self _params = [ get_bin_path("pg_ctl"), @@ -682,9 +740,25 @@ def stop(self, params=[], wait=True): execute_utility(_params, self.utils_log_file) self._maybe_stop_logger() - + self.is_started = False return self + def kill(self, someone=None): + """ + Kills the PostgreSQL node or a specified auxiliary process if the node is running. + + Args: + someone: A key to the auxiliary process in the auxiliary_pids dictionary. + If None, the main PostgreSQL node process will be killed. Defaults to None. + """ + if self.is_started: + sig = signal.SIGKILL if os.name != 'nt' else signal.SIGBREAK + if someone is None: + os.kill(self.pid, sig) + else: + os.kill(self.auxiliary_pids[someone][0], sig) + self.is_started = False + def restart(self, params=[]): """ Restart this node using pg_ctl. @@ -896,7 +970,7 @@ def psql(self, return process.returncode, out, err @method_decorator(positional_args_hack(['dbname', 'query'])) - def safe_psql(self, query=None, **kwargs): + def safe_psql(self, query=None, expect_error=False, **kwargs): """ Execute a query using psql. @@ -906,6 +980,8 @@ def safe_psql(self, query=None, **kwargs): dbname: database name to connect to. username: database user name. input: raw input to be passed. + expect_error: if True - fail if we didn't get ret + if False - fail if we got ret **kwargs are passed to psql(). @@ -918,7 +994,12 @@ def safe_psql(self, query=None, **kwargs): ret, out, err = self.psql(query=query, **kwargs) if ret: - raise QueryException((err or b'').decode('utf-8'), query) + if expect_error: + out = (err or b'').decode('utf-8') + else: + raise QueryException((err or b'').decode('utf-8'), query) + elif expect_error: + assert False, f"Exception was expected, but query finished successfully: `{query}` " return out @@ -1361,3 +1442,211 @@ def connect(self, username=username, password=password, autocommit=autocommit) # yapf: disable + + def table_checksum(self, table, dbname="postgres"): + con = self.connect(dbname=dbname) + + curname = "cur_" + str(random.randint(0, 2 ** 48)) + + con.execute(""" + DECLARE %s NO SCROLL CURSOR FOR + SELECT t::text FROM %s as t + """ % (curname, table)) + + que = Queue(maxsize=50) + sum = 0 + + rows = con.execute("FETCH FORWARD 2000 FROM %s" % curname) + if not rows: + return 0 + que.put(rows) + + th = None + if len(rows) == 2000: + def querier(): + try: + while True: + rows = con.execute("FETCH FORWARD 2000 FROM %s" % curname) + if not rows: + break + que.put(rows) + except Exception as e: + que.put(e) + else: + que.put(None) + + th = threading.Thread(target=querier) + th.start() + else: + que.put(None) + + while True: + rows = que.get() + if rows is None: + break + if isinstance(rows, Exception): + raise rows + # hash uses SipHash since Python3.4, therefore it is good enough + for row in rows: + sum += hash(row[0]) + + if th is not None: + th.join() + + con.execute("CLOSE %s; ROLLBACK;" % curname) + + con.close() + return sum + + def pgbench_table_checksums(self, dbname="postgres", + pgbench_tables=('pgbench_branches', + 'pgbench_tellers', + 'pgbench_accounts', + 'pgbench_history') + ): + return {(table, self.table_checksum(table, dbname)) + for table in pgbench_tables} + + def set_auto_conf(self, options, config='postgresql.auto.conf', rm_options={}): + """ + Update or remove configuration options in the specified configuration file, + updates the options specified in the options dictionary, removes any options + specified in the rm_options set, and writes the updated configuration back to + the file. + + Args: + options (dict): A dictionary containing the options to update or add, + with the option names as keys and their values as values. + config (str, optional): The name of the configuration file to update. + Defaults to 'postgresql.auto.conf'. + rm_options (set, optional): A set containing the names of the options to remove. + Defaults to an empty set. + """ + # parse postgresql.auto.conf + path = os.path.join(self.data_dir, config) + + with open(path, 'r') as f: + raw_content = f.read() + + current_options = {} + current_directives = [] + for line in raw_content.splitlines(): + + # ignore comments + if line.startswith('#'): + continue + + if line == '': + continue + + if line.startswith('include'): + current_directives.append(line) + continue + + name, var = line.partition('=')[::2] + name = name.strip() + var = var.strip() + var = var.strip('"') + var = var.strip("'") + + # remove options specified in rm_options list + if name in rm_options: + continue + + current_options[name] = var + + for option in options: + current_options[option] = options[option] + + auto_conf = '' + for option in current_options: + auto_conf += "{0} = '{1}'\n".format( + option, current_options[option]) + + for directive in current_directives: + auto_conf += directive + "\n" + + with open(path, 'wt') as f: + f.write(auto_conf) + + +class NodeApp: + + def __init__(self, test_path, nodes_to_cleanup): + self.test_path = test_path + self.nodes_to_cleanup = nodes_to_cleanup + + def make_empty( + self, + base_dir=None): + real_base_dir = os.path.join(self.test_path, base_dir) + shutil.rmtree(real_base_dir, ignore_errors=True) + os.makedirs(real_base_dir) + + node = PostgresNode(base_dir=real_base_dir) + node.should_rm_dirs = True + self.nodes_to_cleanup.append(node) + + return node + + def make_simple( + self, + base_dir=None, + set_replication=False, + ptrack_enable=False, + initdb_params=[], + pg_options={}): + + node = self.make_empty(base_dir) + node.init( + initdb_params=initdb_params, allow_streaming=set_replication) + + # set major version + with open(os.path.join(node.data_dir, 'PG_VERSION')) as f: + node.major_version_str = str(f.read().rstrip()) + node.major_version = float(node.major_version_str) + + # Sane default parameters + options = {} + options['max_connections'] = 100 + options['shared_buffers'] = '10MB' + options['fsync'] = 'off' + + options['wal_level'] = 'logical' + options['hot_standby'] = 'off' + + options['log_line_prefix'] = '%t [%p]: [%l-1] ' + options['log_statement'] = 'none' + options['log_duration'] = 'on' + options['log_min_duration_statement'] = 0 + options['log_connections'] = 'on' + options['log_disconnections'] = 'on' + options['restart_after_crash'] = 'off' + options['autovacuum'] = 'off' + + # Allow replication in pg_hba.conf + if set_replication: + options['max_wal_senders'] = 10 + + if ptrack_enable: + options['ptrack.map_size'] = '1' + options['shared_preload_libraries'] = 'ptrack' + + if node.major_version >= 13: + options['wal_keep_size'] = '200MB' + else: + options['wal_keep_segments'] = '12' + + # set default values + node.set_auto_conf(options) + + # Apply given parameters + node.set_auto_conf(pg_options) + + # kludge for testgres + # https://github.com/postgrespro/testgres/issues/54 + # for PG >= 13 remove 'wal_keep_segments' parameter + if node.major_version >= 13: + node.set_auto_conf({}, 'postgresql.conf', ['wal_keep_segments']) + + return node diff --git a/tests/test_simple.py b/tests/test_simple.py index d79fa79a..94420b04 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -171,8 +171,8 @@ def test_node_exit(self): def test_double_start(self): with get_new_node().init().start() as node: # can't start node more than once - with self.assertRaises(StartNodeException): - node.start() + node.start() + self.assertTrue(node.is_started) def test_uninitialized_start(self): with get_new_node() as node: From 003d334a103b392084893e7b77ee0c8a86fcaa76 Mon Sep 17 00:00:00 2001 From: Alexey Savchkov Date: Fri, 5 May 2023 11:25:55 +0700 Subject: [PATCH 248/475] Revert "PBCKP-377 optionaly keep test logs" This reverts commit b3777b2c4bedd4435befb924a2e7b96bd5f00c76. --- testgres/node.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index de6a5032..e6ac44b0 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -875,15 +875,13 @@ def free_port(self): self._should_free_port = False release_port(self.port) - def cleanup(self, max_attempts=3, full_clean=testgres_config.node_cleanup_full): + def cleanup(self, max_attempts=3): """ Stop node if needed and remove its data/logs directory. NOTE: take a look at TestgresConfig.node_cleanup_full. Args: max_attempts: how many times should we try to stop()? - full_clean: True - clean all node data - False - clean only data dir, save logs Returns: This instance of :class:`.PostgresNode`. @@ -892,7 +890,7 @@ def cleanup(self, max_attempts=3, full_clean=testgres_config.node_cleanup_full): self._try_shutdown(max_attempts) # choose directory to be removed - if full_clean: + if testgres_config.node_cleanup_full: rm_dir = self.base_dir # everything else: rm_dir = self.data_dir # just data, save logs From 01ce2293b375cee1357182146e21d793725bca42 Mon Sep 17 00:00:00 2001 From: komii Date: Mon, 8 May 2023 16:35:10 +0700 Subject: [PATCH 249/475] Adds packaging package to requirements because of using it on PBCKP-289 --- setup.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 5c6f4a07..5d36f5d0 100755 --- a/setup.py +++ b/setup.py @@ -6,7 +6,13 @@ from distutils.core import setup # Basic dependencies -install_requires = ["pg8000", "port-for>=0.4", "six>=1.9.0", "psutil"] +install_requires = [ + "pg8000", + "port-for>=0.4", + "six>=1.9.0", + "psutil", + "packaging", +] # Add compatibility enum class if sys.version_info < (3, 4): From 7894e5e44a05307980f03fcc7290d6f76f2e4e6e Mon Sep 17 00:00:00 2001 From: Alexey Savchkov Date: Wed, 10 May 2023 18:29:29 +0700 Subject: [PATCH 250/475] Update copyright years --- LICENSE | 2 +- docs/source/conf.py | 2 +- publish_package.sh | 3 +-- run_tests.sh | 2 +- 4 files changed, 4 insertions(+), 5 deletions(-) diff --git a/LICENSE b/LICENSE index 86b7be77..7e9cc712 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ testgres is released under the PostgreSQL License, a liberal Open Source license, similar to the BSD or MIT licenses. -Copyright (c) 2016-2018, Postgres Professional +Copyright (c) 2016-2023, Postgres Professional Permission to use, copy, modify, and distribute this software and its documentation for any purpose, without fee, and without a written agreement is hereby granted, provided that the above copyright notice and this paragraph and the following two paragraphs appear in all copies. diff --git a/docs/source/conf.py b/docs/source/conf.py index 60709bd5..688a850f 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -19,7 +19,7 @@ # -- Project information ----------------------------------------------------- project = u'testgres' -copyright = u'2016-2022, Postgres Professional' +copyright = u'2016-2023, Postgres Professional' author = u'Postgres Professional' # The short X.Y version diff --git a/publish_package.sh b/publish_package.sh index 8cfa23e2..9cc56e94 100755 --- a/publish_package.sh +++ b/publish_package.sh @@ -1,7 +1,5 @@ #!/usr/bin/env bash -# Copyright (c) 2017-2022 Postgres Professional - set -eux # prepare environment @@ -21,3 +19,4 @@ python3 setup.py sdist bdist_wheel twine upload dist/* set +eux + diff --git a/run_tests.sh b/run_tests.sh index f71a055b..73c459be 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# Copyright (c) 2017-2022 Postgres Professional +# Copyright (c) 2017-2023 Postgres Professional set -eux From 8979f83d0075d14403533ec0cb12e740dfc52157 Mon Sep 17 00:00:00 2001 From: Alexey Savchkov Date: Wed, 10 May 2023 18:40:58 +0700 Subject: [PATCH 251/475] Up version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 5d36f5d0..2e8c4ed0 100755 --- a/setup.py +++ b/setup.py @@ -27,7 +27,7 @@ readme = f.read() setup( - version='1.8.6', + version='1.8.7', name='testgres', packages=['testgres'], description='Testing utility for PostgreSQL and its extensions', From 3cc9d67b704b095376239bdfe396951ce51d9dc4 Mon Sep 17 00:00:00 2001 From: Victoria Shepard <5807469+demonolock@users.noreply.github.com> Date: Mon, 29 May 2023 12:23:38 +0200 Subject: [PATCH 252/475] Replace deprecated distutils find_executable on shutil which (#82) Co-authored-by: v.shepard --- testgres/utils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/testgres/utils.py b/testgres/utils.py index 4d99c69d..d8006d08 100644 --- a/testgres/utils.py +++ b/testgres/utils.py @@ -12,7 +12,10 @@ from contextlib import contextmanager from packaging.version import Version -from distutils.spawn import find_executable +try: + from shutil import which as find_executable +except ImportError: + from distutils.spawn import find_executable from six import iteritems from .config import testgres_config From 94f9b3565f9354d543a35af6c4d406b41a1cae22 Mon Sep 17 00:00:00 2001 From: Victoria Shepard <5807469+demonolock@users.noreply.github.com> Date: Tue, 6 Jun 2023 13:46:56 +0200 Subject: [PATCH 253/475] PBCKP-634 test shorter make_node (#83) --- setup.py | 2 +- testgres/node.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 2e8c4ed0..1332e8ab 100755 --- a/setup.py +++ b/setup.py @@ -27,7 +27,7 @@ readme = f.read() setup( - version='1.8.7', + version='1.8.8', name='testgres', packages=['testgres'], description='Testing utility for PostgreSQL and its extensions', diff --git a/testgres/node.py b/testgres/node.py index e6ac44b0..659a62f8 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -1593,8 +1593,10 @@ def make_simple( set_replication=False, ptrack_enable=False, initdb_params=[], - pg_options={}): - + pg_options={}, + checksum=True): + if checksum and '--data-checksums' not in initdb_params: + initdb_params.append('--data-checksums') node = self.make_empty(base_dir) node.init( initdb_params=initdb_params, allow_streaming=set_replication) From 592b60218930416281d26b39c39ec5f9a6d4fe01 Mon Sep 17 00:00:00 2001 From: Victoria Shepard <5807469+demonolock@users.noreply.github.com> Date: Fri, 21 Jul 2023 10:24:50 +0200 Subject: [PATCH 254/475] Support special version (#85) Co-authored-by: v.shepard --- testgres/utils.py | 12 ++++++++++-- tests/test_simple.py | 8 ++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/testgres/utils.py b/testgres/utils.py index d8006d08..9760908d 100644 --- a/testgres/utils.py +++ b/testgres/utils.py @@ -11,7 +11,8 @@ import tempfile from contextlib import contextmanager -from packaging.version import Version +from packaging.version import Version, InvalidVersion +import re try: from shutil import which as find_executable except ImportError: @@ -27,8 +28,15 @@ # ports used by nodes bound_ports = set() + # re-export version type -PgVer = Version +class PgVer(Version): + def __init__(self, version: str) -> None: + try: + super().__init__(version) + except InvalidVersion: + version = re.sub(r"[a-zA-Z].*", "", version) + super().__init__(version) def reserve_port(): diff --git a/tests/test_simple.py b/tests/test_simple.py index 94420b04..45c28a21 100755 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -901,6 +901,10 @@ def test_version_management(self): d = PgVer('15.0') e = PgVer('15rc1') f = PgVer('15beta4') + h = PgVer('15.3biha') + i = PgVer('15.3') + g = PgVer('15.3.1bihabeta1') + k = PgVer('15.3.1') self.assertTrue(a == b) self.assertTrue(b > c) @@ -908,6 +912,10 @@ def test_version_management(self): self.assertTrue(d > e) self.assertTrue(e > f) self.assertTrue(d > f) + self.assertTrue(h > f) + self.assertTrue(h == i) + self.assertTrue(g == k) + self.assertTrue(g > h) version = get_pg_version() with get_new_node() as node: From 09e9f01f60a8b265f38a5a78de5475a1162a515b Mon Sep 17 00:00:00 2001 From: Victoria Shepard <5807469+demonolock@users.noreply.github.com> Date: Mon, 24 Jul 2023 11:30:53 +0200 Subject: [PATCH 255/475] Up version 1.8.9 (#86) --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 1332e8ab..6d0c2256 100755 --- a/setup.py +++ b/setup.py @@ -27,7 +27,7 @@ readme = f.read() setup( - version='1.8.8', + version='1.8.9', name='testgres', packages=['testgres'], description='Testing utility for PostgreSQL and its extensions', From 6cb3a8080e1d61ff080b774f0cc5923dbb4b394b Mon Sep 17 00:00:00 2001 From: Victoria Shepard <5807469+demonolock@users.noreply.github.com> Date: Fri, 4 Aug 2023 15:51:03 +0200 Subject: [PATCH 256/475] Add work with remote host (#78) --- README.md | 27 + setup.py | 7 +- testgres/__init__.py | 8 +- testgres/api.py | 12 + testgres/backup.py | 18 +- testgres/cache.py | 21 +- testgres/config.py | 17 +- testgres/connection.py | 15 +- testgres/defaults.py | 9 +- testgres/node.py | 282 +++++---- testgres/operations/__init__.py | 0 testgres/operations/local_ops.py | 269 ++++++++ testgres/operations/os_ops.py | 101 +++ testgres/operations/remote_ops.py | 448 ++++++++++++++ testgres/pubsub.py | 2 +- testgres/utils.py | 106 ++-- tests/README.md | 29 + tests/test_remote.py | 198 ++++++ tests/test_simple_remote.py | 996 ++++++++++++++++++++++++++++++ 19 files changed, 2321 insertions(+), 244 deletions(-) create mode 100644 testgres/operations/__init__.py create mode 100644 testgres/operations/local_ops.py create mode 100644 testgres/operations/os_ops.py create mode 100644 testgres/operations/remote_ops.py create mode 100755 tests/test_remote.py create mode 100755 tests/test_simple_remote.py diff --git a/README.md b/README.md index 6b26ba96..29b974dc 100644 --- a/README.md +++ b/README.md @@ -173,6 +173,33 @@ with testgres.get_new_node().init() as master: Note that `default_conf()` is called by `init()` function; both of them overwrite the configuration file, which means that they should be called before `append_conf()`. +### Remote mode +Testgres supports the creation of PostgreSQL nodes on a remote host. This is useful when you want to run distributed tests involving multiple nodes spread across different machines. + +To use this feature, you need to use the RemoteOperations class. +Here is an example of how you might set this up: + +```python +from testgres import ConnectionParams, RemoteOperations, TestgresConfig, get_remote_node + +# Set up connection params +conn_params = ConnectionParams( + host='your_host', # replace with your host + username='user_name', # replace with your username + ssh_key='path_to_ssh_key' # replace with your SSH key path +) +os_ops = RemoteOperations(conn_params) + +# Add remote testgres config before test +TestgresConfig.set_os_ops(os_ops=os_ops) + +# Proceed with your test +def test_basic_query(self): + with get_remote_node(conn_params=conn_params) as node: + node.init().start() + res = node.execute('SELECT 1') + self.assertEqual(res, [(1,)]) +``` ## Authors diff --git a/setup.py b/setup.py index 6d0c2256..8cb0f70a 100755 --- a/setup.py +++ b/setup.py @@ -12,6 +12,9 @@ "six>=1.9.0", "psutil", "packaging", + "paramiko", + "fabric", + "sshtunnel" ] # Add compatibility enum class @@ -27,9 +30,9 @@ readme = f.read() setup( - version='1.8.9', + version='1.9.0', name='testgres', - packages=['testgres'], + packages=['testgres', 'testgres.operations'], description='Testing utility for PostgreSQL and its extensions', url='https://github.com/postgrespro/testgres', long_description=readme, diff --git a/testgres/__init__.py b/testgres/__init__.py index 1b33ba3b..b63c7df1 100644 --- a/testgres/__init__.py +++ b/testgres/__init__.py @@ -1,4 +1,4 @@ -from .api import get_new_node +from .api import get_new_node, get_remote_node from .backup import NodeBackup from .config import \ @@ -46,8 +46,13 @@ First, \ Any +from .operations.os_ops import OsOperations, ConnectionParams +from .operations.local_ops import LocalOperations +from .operations.remote_ops import RemoteOperations + __all__ = [ "get_new_node", + "get_remote_node", "NodeBackup", "TestgresConfig", "configure_testgres", "scoped_config", "push_config", "pop_config", "NodeConnection", "DatabaseError", "InternalError", "ProgrammingError", "OperationalError", @@ -56,4 +61,5 @@ "PostgresNode", "NodeApp", "reserve_port", "release_port", "bound_ports", "get_bin_path", "get_pg_config", "get_pg_version", "First", "Any", + "OsOperations", "LocalOperations", "RemoteOperations", "ConnectionParams" ] diff --git a/testgres/api.py b/testgres/api.py index e90cf7bd..e4b1cdd5 100644 --- a/testgres/api.py +++ b/testgres/api.py @@ -40,3 +40,15 @@ def get_new_node(name=None, base_dir=None, **kwargs): """ # NOTE: leave explicit 'name' and 'base_dir' for compatibility return PostgresNode(name=name, base_dir=base_dir, **kwargs) + + +def get_remote_node(name=None, conn_params=None): + """ + Simply a wrapper around :class:`.PostgresNode` constructor for remote node. + See :meth:`.PostgresNode.__init__` for details. + For remote connection you can add the next parameter: + conn_params = ConnectionParams(host='127.0.0.1', + ssh_key=None, + username=default_username()) + """ + return get_new_node(name=name, conn_params=conn_params) diff --git a/testgres/backup.py b/testgres/backup.py index a725a1df..a89e214d 100644 --- a/testgres/backup.py +++ b/testgres/backup.py @@ -2,9 +2,7 @@ import os -from shutil import rmtree, copytree from six import raise_from -from tempfile import mkdtemp from .enums import XLogMethod @@ -15,8 +13,6 @@ PG_CONF_FILE, \ BACKUP_LOG_FILE -from .defaults import default_username - from .exceptions import BackupException from .utils import \ @@ -47,7 +43,7 @@ def __init__(self, username: database user name. xlog_method: none | fetch | stream (see docs) """ - + self.os_ops = node.os_ops if not node.status(): raise BackupException('Node must be running') @@ -60,8 +56,8 @@ def __init__(self, raise BackupException(msg) # Set default arguments - username = username or default_username() - base_dir = base_dir or mkdtemp(prefix=TMP_BACKUP) + username = username or self.os_ops.get_user() + base_dir = base_dir or self.os_ops.mkdtemp(prefix=TMP_BACKUP) # public self.original_node = node @@ -107,14 +103,14 @@ def _prepare_dir(self, destroy): available = not destroy if available: - dest_base_dir = mkdtemp(prefix=TMP_NODE) + dest_base_dir = self.os_ops.mkdtemp(prefix=TMP_NODE) data1 = os.path.join(self.base_dir, DATA_DIR) data2 = os.path.join(dest_base_dir, DATA_DIR) try: # Copy backup to new data dir - copytree(data1, data2) + self.os_ops.copytree(data1, data2) except Exception as e: raise_from(BackupException('Failed to copy files'), e) else: @@ -143,7 +139,7 @@ def spawn_primary(self, name=None, destroy=True): # Build a new PostgresNode NodeClass = self.original_node.__class__ - with clean_on_error(NodeClass(name=name, base_dir=base_dir)) as node: + with clean_on_error(NodeClass(name=name, base_dir=base_dir, conn_params=self.original_node.os_ops.conn_params)) as node: # New nodes should always remove dir tree node._should_rm_dirs = True @@ -185,4 +181,4 @@ def cleanup(self): if self._available: self._available = False - rmtree(self.base_dir, ignore_errors=True) + self.os_ops.rmdirs(self.base_dir, ignore_errors=True) diff --git a/testgres/cache.py b/testgres/cache.py index c3cd9971..bf8658c9 100644 --- a/testgres/cache.py +++ b/testgres/cache.py @@ -1,9 +1,7 @@ # coding: utf-8 -import io import os -from shutil import copytree from six import raise_from from .config import testgres_config @@ -20,12 +18,16 @@ get_bin_path, \ execute_utility +from .operations.local_ops import LocalOperations +from .operations.os_ops import OsOperations -def cached_initdb(data_dir, logfile=None, params=None): + +def cached_initdb(data_dir, logfile=None, params=None, os_ops: OsOperations = LocalOperations()): """ Perform initdb or use cached node files. """ - def call_initdb(initdb_dir, log=None): + + def call_initdb(initdb_dir, log=logfile): try: _params = [get_bin_path("initdb"), "-D", initdb_dir, "-N"] execute_utility(_params + (params or []), log) @@ -39,13 +41,14 @@ def call_initdb(initdb_dir, log=None): cached_data_dir = testgres_config.cached_initdb_dir # Initialize cached initdb - if not os.path.exists(cached_data_dir) or \ - not os.listdir(cached_data_dir): + + if not os_ops.path_exists(cached_data_dir) or \ + not os_ops.listdir(cached_data_dir): call_initdb(cached_data_dir) try: # Copy cached initdb to current data dir - copytree(cached_data_dir, data_dir) + os_ops.copytree(cached_data_dir, data_dir) # Assign this node a unique system id if asked to if testgres_config.cached_initdb_unique: @@ -53,8 +56,8 @@ def call_initdb(initdb_dir, log=None): # Some users might rely upon unique system ids, but # our initdb caching mechanism breaks this contract. pg_control = os.path.join(data_dir, XLOG_CONTROL_FILE) - with io.open(pg_control, "r+b") as f: - f.write(generate_system_id()) # overwrite id + system_id = generate_system_id() + os_ops.write(pg_control, system_id, truncate=True, binary=True, read_and_write=True) # XXX: build new WAL segment with our system id _params = [get_bin_path("pg_resetwal"), "-D", data_dir, "-f"] diff --git a/testgres/config.py b/testgres/config.py index cfcdadc2..b6c43926 100644 --- a/testgres/config.py +++ b/testgres/config.py @@ -5,10 +5,10 @@ import tempfile from contextlib import contextmanager -from shutil import rmtree -from tempfile import mkdtemp from .consts import TMP_CACHE +from .operations.os_ops import OsOperations +from .operations.local_ops import LocalOperations class GlobalConfig(object): @@ -43,6 +43,9 @@ class GlobalConfig(object): _cached_initdb_dir = None """ underlying class attribute for cached_initdb_dir property """ + + os_ops = LocalOperations() + """ OsOperation object that allows work on remote host """ @property def cached_initdb_dir(self): """ path to a temp directory for cached initdb. """ @@ -54,6 +57,7 @@ def cached_initdb_dir(self, value): if value: cached_initdb_dirs.add(value) + return testgres_config.cached_initdb_dir @property def temp_dir(self): @@ -118,6 +122,11 @@ def copy(self): return copy.copy(self) + @staticmethod + def set_os_ops(os_ops: OsOperations): + testgres_config.os_ops = os_ops + testgres_config.cached_initdb_dir = os_ops.mkdtemp(prefix=TMP_CACHE) + # cached dirs to be removed cached_initdb_dirs = set() @@ -135,7 +144,7 @@ def copy(self): @atexit.register def _rm_cached_initdb_dirs(): for d in cached_initdb_dirs: - rmtree(d, ignore_errors=True) + testgres_config.os_ops.rmdirs(d, ignore_errors=True) def push_config(**options): @@ -198,4 +207,4 @@ def configure_testgres(**options): # NOTE: assign initial cached dir for initdb -testgres_config.cached_initdb_dir = mkdtemp(prefix=TMP_CACHE) +testgres_config.cached_initdb_dir = testgres_config.os_ops.mkdtemp(prefix=TMP_CACHE) diff --git a/testgres/connection.py b/testgres/connection.py index ee2a2128..aeb040ce 100644 --- a/testgres/connection.py +++ b/testgres/connection.py @@ -41,11 +41,11 @@ def __init__(self, self._node = node - self._connection = pglib.connect(database=dbname, - user=username, - password=password, - host=node.host, - port=node.port) + self._connection = node.os_ops.db_connect(dbname=dbname, + user=username, + password=password, + host=node.host, + port=node.port) self._connection.autocommit = autocommit self._cursor = self.connection.cursor() @@ -103,16 +103,15 @@ def rollback(self): def execute(self, query, *args): self.cursor.execute(query, args) - try: res = self.cursor.fetchall() - # pg8000 might return tuples if isinstance(res, tuple): res = [tuple(t) for t in res] return res - except Exception: + except Exception as e: + print("Error executing query: {}".format(e)) return None def close(self): diff --git a/testgres/defaults.py b/testgres/defaults.py index 8d5b892e..d77361d7 100644 --- a/testgres/defaults.py +++ b/testgres/defaults.py @@ -1,9 +1,9 @@ import datetime -import getpass -import os import struct import uuid +from .config import testgres_config as tconf + def default_dbname(): """ @@ -17,8 +17,7 @@ def default_username(): """ Return default username (current user). """ - - return getpass.getuser() + return tconf.os_ops.get_user() def generate_app_name(): @@ -44,7 +43,7 @@ def generate_system_id(): system_id = 0 system_id |= (secs << 32) system_id |= (usecs << 12) - system_id |= (os.getpid() & 0xFFF) + system_id |= (tconf.os_ops.get_pid() & 0xFFF) # pack ULL in native byte order return struct.pack('=Q', system_id) diff --git a/testgres/node.py b/testgres/node.py index 659a62f8..6483514b 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -1,18 +1,14 @@ # coding: utf-8 -import io import os import random -import shutil import signal +import subprocess import threading from queue import Queue -import psutil -import subprocess import time - try: from collections.abc import Iterable except ImportError: @@ -27,9 +23,7 @@ except ImportError: raise ImportError("You must have psycopg2 or pg8000 modules installed") -from shutil import rmtree from six import raise_from, iteritems, text_type -from tempfile import mkstemp, mkdtemp from .enums import \ NodeStatus, \ @@ -93,7 +87,6 @@ eprint, \ get_bin_path, \ get_pg_version, \ - file_tail, \ reserve_port, \ release_port, \ execute_utility, \ @@ -102,6 +95,10 @@ from .backup import NodeBackup +from .operations.os_ops import ConnectionParams +from .operations.local_ops import LocalOperations +from .operations.remote_ops import RemoteOperations + InternalError = pglib.InternalError ProgrammingError = pglib.ProgrammingError OperationalError = pglib.OperationalError @@ -130,7 +127,7 @@ def __repr__(self): class PostgresNode(object): - def __init__(self, name=None, port=None, base_dir=None): + def __init__(self, name=None, port=None, base_dir=None, conn_params: ConnectionParams = ConnectionParams()): """ PostgresNode constructor. @@ -148,10 +145,19 @@ def __init__(self, name=None, port=None, base_dir=None): self._master = None # basic - self.host = '127.0.0.1' self.name = name or generate_app_name() + if testgres_config.os_ops: + self.os_ops = testgres_config.os_ops + elif conn_params.ssh_key: + self.os_ops = RemoteOperations(conn_params) + else: + self.os_ops = LocalOperations(conn_params) + self.port = port or reserve_port() + self.host = self.os_ops.host + self.ssh_key = self.os_ops.ssh_key + # defaults for __exit__() self.cleanup_on_good_exit = testgres_config.node_cleanup_on_good_exit self.cleanup_on_bad_exit = testgres_config.node_cleanup_on_bad_exit @@ -195,8 +201,9 @@ def pid(self): if self.status(): pid_file = os.path.join(self.data_dir, PG_PID_FILE) - with io.open(pid_file) as f: - return int(f.readline()) + lines = self.os_ops.readlines(pid_file) + pid = int(lines[0]) if lines else None + return pid # for clarity return 0 @@ -236,7 +243,7 @@ def child_processes(self): """ # get a list of postmaster's children - children = psutil.Process(self.pid).children() + children = self.os_ops.get_process_children(self.pid) return [ProcessProxy(p) for p in children] @@ -274,11 +281,11 @@ def master(self): @property def base_dir(self): if not self._base_dir: - self._base_dir = mkdtemp(prefix=TMP_NODE) + self._base_dir = self.os_ops.mkdtemp(prefix=TMP_NODE) # NOTE: it's safe to create a new dir - if not os.path.exists(self._base_dir): - os.makedirs(self._base_dir) + if not self.os_ops.path_exists(self._base_dir): + self.os_ops.makedirs(self._base_dir) return self._base_dir @@ -287,8 +294,8 @@ def logs_dir(self): path = os.path.join(self.base_dir, LOGS_DIR) # NOTE: it's safe to create a new dir - if not os.path.exists(path): - os.makedirs(path) + if not self.os_ops.path_exists(path): + self.os_ops.makedirs(path) return path @@ -365,9 +372,7 @@ def _create_recovery_conf(self, username, slot=None): # Since 12 recovery.conf had disappeared if self.version >= PgVer('12'): signal_name = os.path.join(self.data_dir, "standby.signal") - # cross-python touch(). It is vulnerable to races, but who cares? - with open(signal_name, 'a'): - os.utime(signal_name, None) + self.os_ops.touch(signal_name) else: line += "standby_mode=on\n" @@ -425,19 +430,14 @@ def _collect_special_files(self): for f, num_lines in files: # skip missing files - if not os.path.exists(f): + if not self.os_ops.path_exists(f): continue - with io.open(f, "rb") as _f: - if num_lines > 0: - # take last N lines of file - lines = b''.join(file_tail(_f, num_lines)).decode('utf-8') - else: - # read whole file - lines = _f.read().decode('utf-8') + file_lines = self.os_ops.readlines(f, num_lines, binary=True, encoding=None) + lines = b''.join(file_lines) - # fill list - result.append((f, lines)) + # fill list + result.append((f, lines)) return result @@ -456,9 +456,11 @@ def init(self, initdb_params=None, **kwargs): """ # initialize this PostgreSQL node - cached_initdb(data_dir=self.data_dir, - logfile=self.utils_log_file, - params=initdb_params) + cached_initdb( + data_dir=self.data_dir, + logfile=self.utils_log_file, + os_ops=self.os_ops, + params=initdb_params) # initialize default config files self.default_conf(**kwargs) @@ -489,43 +491,41 @@ def default_conf(self, hba_conf = os.path.join(self.data_dir, HBA_CONF_FILE) # filter lines in hba file - with io.open(hba_conf, "r+") as conf: - # get rid of comments and blank lines - lines = [ - s for s in conf.readlines() - if len(s.strip()) > 0 and not s.startswith('#') - ] - - # write filtered lines - conf.seek(0) - conf.truncate() - conf.writelines(lines) - - # replication-related settings - if allow_streaming: - # get auth method for host or local users - def get_auth_method(t): - return next((s.split()[-1] - for s in lines if s.startswith(t)), 'trust') - - # get auth methods - auth_local = get_auth_method('local') - auth_host = get_auth_method('host') - - new_lines = [ - u"local\treplication\tall\t\t\t{}\n".format(auth_local), - u"host\treplication\tall\t127.0.0.1/32\t{}\n".format(auth_host), - u"host\treplication\tall\t::1/128\t\t{}\n".format(auth_host) - ] # yapf: disable - - # write missing lines - for line in new_lines: - if line not in lines: - conf.write(line) + # get rid of comments and blank lines + hba_conf_file = self.os_ops.readlines(hba_conf) + lines = [ + s for s in hba_conf_file + if len(s.strip()) > 0 and not s.startswith('#') + ] + + # write filtered lines + self.os_ops.write(hba_conf, lines, truncate=True) + + # replication-related settings + if allow_streaming: + # get auth method for host or local users + def get_auth_method(t): + return next((s.split()[-1] + for s in lines if s.startswith(t)), 'trust') + + # get auth methods + auth_local = get_auth_method('local') + auth_host = get_auth_method('host') + subnet_base = ".".join(self.os_ops.host.split('.')[:-1] + ['0']) + + new_lines = [ + u"local\treplication\tall\t\t\t{}\n".format(auth_local), + u"host\treplication\tall\t127.0.0.1/32\t{}\n".format(auth_host), + u"host\treplication\tall\t::1/128\t\t{}\n".format(auth_host), + u"host\treplication\tall\t{}/24\t\t{}\n".format(subnet_base, auth_host), + u"host\tall\tall\t{}/24\t\t{}\n".format(subnet_base, auth_host) + ] # yapf: disable + + # write missing lines + self.os_ops.write(hba_conf, new_lines) # overwrite config file - with io.open(postgres_conf, "w") as conf: - conf.truncate() + self.os_ops.write(postgres_conf, '', truncate=True) self.append_conf(fsync=fsync, max_worker_processes=MAX_WORKER_PROCESSES, @@ -595,15 +595,17 @@ def append_conf(self, line='', filename=PG_CONF_FILE, **kwargs): value = 'on' if value else 'off' elif not str(value).replace('.', '', 1).isdigit(): value = "'{}'".format(value) - - # format a new config line - lines.append('{} = {}'.format(option, value)) + if value == '*': + lines.append("{} = '*'".format(option)) + else: + # format a new config line + lines.append('{} = {}'.format(option, value)) config_name = os.path.join(self.data_dir, filename) - with io.open(config_name, 'a') as conf: - for line in lines: - conf.write(text_type(line)) - conf.write(text_type('\n')) + conf_text = '' + for line in lines: + conf_text += text_type(line) + '\n' + self.os_ops.write(config_name, conf_text) return self @@ -621,7 +623,11 @@ def status(self): "-D", self.data_dir, "status" ] # yapf: disable - execute_utility(_params, self.utils_log_file) + status_code, out, err = execute_utility(_params, self.utils_log_file, verbose=True) + if 'does not exist' in err: + return NodeStatus.Uninitialized + elif 'no server running' in out: + return NodeStatus.Stopped return NodeStatus.Running except ExecUtilException as e: @@ -653,7 +659,7 @@ def get_control_data(self): return out_dict - def slow_start(self, replica=False, dbname='template1', username=default_username()): + def slow_start(self, replica=False, dbname='template1', username=default_username(), max_attempts=0): """ Starts the PostgreSQL instance and then polls the instance until it reaches the expected state (primary or replica). The state is checked @@ -664,6 +670,7 @@ def slow_start(self, replica=False, dbname='template1', username=default_usernam username: replica: If True, waits for the instance to be in recovery (i.e., replica mode). If False, waits for the instance to be in primary mode. Default is False. + max_attempts: """ self.start() @@ -678,7 +685,8 @@ def slow_start(self, replica=False, dbname='template1', username=default_usernam suppress={InternalError, QueryException, ProgrammingError, - OperationalError}) + OperationalError}, + max_attempts=max_attempts) def start(self, params=[], wait=True): """ @@ -706,12 +714,13 @@ def start(self, params=[], wait=True): ] + params # yapf: disable try: - execute_utility(_params, self.utils_log_file) - except ExecUtilException as e: + exit_status, out, error = execute_utility(_params, self.utils_log_file, verbose=True) + if 'does not exist' in error: + raise Exception + except Exception as e: msg = 'Cannot start node' files = self._collect_special_files() raise_from(StartNodeException(msg, files), e) - self._maybe_start_logger() self.is_started = True return self @@ -779,7 +788,9 @@ def restart(self, params=[]): ] + params # yapf: disable try: - execute_utility(_params, self.utils_log_file) + error_code, out, error = execute_utility(_params, self.utils_log_file, verbose=True) + if 'could not start server' in error: + raise ExecUtilException except ExecUtilException as e: msg = 'Cannot restart node' files = self._collect_special_files() @@ -895,7 +906,7 @@ def cleanup(self, max_attempts=3): else: rm_dir = self.data_dir # just data, save logs - rmtree(rm_dir, ignore_errors=True) + self.os_ops.rmdirs(rm_dir, ignore_errors=True) return self @@ -948,7 +959,10 @@ def psql(self, # select query source if query: - psql_params.extend(("-c", query)) + if self.os_ops.remote: + psql_params.extend(("-c", '"{}"'.format(query))) + else: + psql_params.extend(("-c", query)) elif filename: psql_params.extend(("-f", filename)) else: @@ -956,16 +970,20 @@ def psql(self, # should be the last one psql_params.append(dbname) + if not self.os_ops.remote: + # start psql process + process = subprocess.Popen(psql_params, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + + # wait until it finishes and get stdout and stderr + out, err = process.communicate(input=input) + return process.returncode, out, err + else: + status_code, out, err = self.os_ops.exec_command(psql_params, verbose=True, input=input) - # start psql process - process = subprocess.Popen(psql_params, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - - # wait until it finishes and get stdout and stderr - out, err = process.communicate(input=input) - return process.returncode, out, err + return status_code, out, err @method_decorator(positional_args_hack(['dbname', 'query'])) def safe_psql(self, query=None, expect_error=False, **kwargs): @@ -989,15 +1007,19 @@ def safe_psql(self, query=None, expect_error=False, **kwargs): # force this setting kwargs['ON_ERROR_STOP'] = 1 - - ret, out, err = self.psql(query=query, **kwargs) + try: + ret, out, err = self.psql(query=query, **kwargs) + except ExecUtilException as e: + ret = e.exit_code + out = e.out + err = e.message if ret: if expect_error: out = (err or b'').decode('utf-8') else: raise QueryException((err or b'').decode('utf-8'), query) elif expect_error: - assert False, f"Exception was expected, but query finished successfully: `{query}` " + assert False, "Exception was expected, but query finished successfully: `{}` ".format(query) return out @@ -1031,10 +1053,9 @@ def dump(self, # Generate tmpfile or tmpdir def tmpfile(): if format == DumpFormat.Directory: - fname = mkdtemp(prefix=TMP_DUMP) + fname = self.os_ops.mkdtemp(prefix=TMP_DUMP) else: - fd, fname = mkstemp(prefix=TMP_DUMP) - os.close(fd) + fname = self.os_ops.mkstemp(prefix=TMP_DUMP) return fname # Set default arguments @@ -1119,9 +1140,9 @@ def poll_query_until(self, # sanity checks assert max_attempts >= 0 assert sleep_time > 0 - attempts = 0 while max_attempts == 0 or attempts < max_attempts: + print(f"Pooling {attempts}") try: res = self.execute(dbname=dbname, query=query, @@ -1350,7 +1371,7 @@ def pgbench(self, # should be the last one _params.append(dbname) - proc = subprocess.Popen(_params, stdout=stdout, stderr=stderr) + proc = self.os_ops.exec_command(_params, stdout=stdout, stderr=stderr, wait_exit=True, proc=True) return proc @@ -1523,18 +1544,16 @@ def set_auto_conf(self, options, config='postgresql.auto.conf', rm_options={}): # parse postgresql.auto.conf path = os.path.join(self.data_dir, config) - with open(path, 'r') as f: - raw_content = f.read() - + lines = self.os_ops.readlines(path) current_options = {} current_directives = [] - for line in raw_content.splitlines(): + for line in lines: # ignore comments if line.startswith('#'): continue - if line == '': + if line.strip() == '': continue if line.startswith('include'): @@ -1564,22 +1583,22 @@ def set_auto_conf(self, options, config='postgresql.auto.conf', rm_options={}): for directive in current_directives: auto_conf += directive + "\n" - with open(path, 'wt') as f: - f.write(auto_conf) + self.os_ops.write(path, auto_conf, truncate=True) class NodeApp: - def __init__(self, test_path, nodes_to_cleanup): + def __init__(self, test_path, nodes_to_cleanup, os_ops=LocalOperations()): self.test_path = test_path self.nodes_to_cleanup = nodes_to_cleanup + self.os_ops = os_ops def make_empty( self, base_dir=None): real_base_dir = os.path.join(self.test_path, base_dir) - shutil.rmtree(real_base_dir, ignore_errors=True) - os.makedirs(real_base_dir) + self.os_ops.rmdirs(real_base_dir, ignore_errors=True) + self.os_ops.makedirs(real_base_dir) node = PostgresNode(base_dir=real_base_dir) node.should_rm_dirs = True @@ -1602,27 +1621,24 @@ def make_simple( initdb_params=initdb_params, allow_streaming=set_replication) # set major version - with open(os.path.join(node.data_dir, 'PG_VERSION')) as f: - node.major_version_str = str(f.read().rstrip()) - node.major_version = float(node.major_version_str) - - # Sane default parameters - options = {} - options['max_connections'] = 100 - options['shared_buffers'] = '10MB' - options['fsync'] = 'off' - - options['wal_level'] = 'logical' - options['hot_standby'] = 'off' - - options['log_line_prefix'] = '%t [%p]: [%l-1] ' - options['log_statement'] = 'none' - options['log_duration'] = 'on' - options['log_min_duration_statement'] = 0 - options['log_connections'] = 'on' - options['log_disconnections'] = 'on' - options['restart_after_crash'] = 'off' - options['autovacuum'] = 'off' + pg_version_file = self.os_ops.read(os.path.join(node.data_dir, 'PG_VERSION')) + node.major_version_str = str(pg_version_file.rstrip()) + node.major_version = float(node.major_version_str) + + # Set default parameters + options = {'max_connections': 100, + 'shared_buffers': '10MB', + 'fsync': 'off', + 'wal_level': 'logical', + 'hot_standby': 'off', + 'log_line_prefix': '%t [%p]: [%l-1] ', + 'log_statement': 'none', + 'log_duration': 'on', + 'log_min_duration_statement': 0, + 'log_connections': 'on', + 'log_disconnections': 'on', + 'restart_after_crash': 'off', + 'autovacuum': 'off'} # Allow replication in pg_hba.conf if set_replication: diff --git a/testgres/operations/__init__.py b/testgres/operations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/testgres/operations/local_ops.py b/testgres/operations/local_ops.py new file mode 100644 index 00000000..89071282 --- /dev/null +++ b/testgres/operations/local_ops.py @@ -0,0 +1,269 @@ +import getpass +import os +import shutil +import stat +import subprocess +import tempfile + +import psutil + +from ..exceptions import ExecUtilException +from .os_ops import ConnectionParams, OsOperations +from .os_ops import pglib + +try: + from shutil import which as find_executable + from shutil import rmtree +except ImportError: + from distutils.spawn import find_executable + from distutils import rmtree + + +CMD_TIMEOUT_SEC = 60 +error_markers = [b'error', b'Permission denied', b'fatal'] + + +class LocalOperations(OsOperations): + def __init__(self, conn_params=None): + if conn_params is None: + conn_params = ConnectionParams() + super(LocalOperations, self).__init__(conn_params.username) + self.conn_params = conn_params + self.host = conn_params.host + self.ssh_key = None + self.remote = False + self.username = conn_params.username or self.get_user() + + # Command execution + def exec_command(self, cmd, wait_exit=False, verbose=False, + expect_error=False, encoding=None, shell=False, text=False, + input=None, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, proc=None): + """ + Execute a command in a subprocess. + + Args: + - cmd: The command to execute. + - wait_exit: Whether to wait for the subprocess to exit before returning. + - verbose: Whether to return verbose output. + - expect_error: Whether to raise an error if the subprocess exits with an error status. + - encoding: The encoding to use for decoding the subprocess output. + - shell: Whether to use shell when executing the subprocess. + - text: Whether to return str instead of bytes for the subprocess output. + - input: The input to pass to the subprocess. + - stdout: The stdout to use for the subprocess. + - stderr: The stderr to use for the subprocess. + - proc: The process to use for subprocess creation. + :return: The output of the subprocess. + """ + if os.name == 'nt': + with tempfile.NamedTemporaryFile() as buf: + process = subprocess.Popen(cmd, stdout=buf, stderr=subprocess.STDOUT) + process.communicate() + buf.seek(0) + result = buf.read().decode(encoding) + return result + else: + process = subprocess.Popen( + cmd, + shell=shell, + stdout=stdout, + stderr=stderr, + ) + if proc: + return process + result, error = process.communicate(input) + exit_status = process.returncode + + error_found = exit_status != 0 or any(marker in error for marker in error_markers) + + if encoding: + result = result.decode(encoding) + error = error.decode(encoding) + + if expect_error: + raise Exception(result, error) + + if exit_status != 0 or error_found: + if exit_status == 0: + exit_status = 1 + raise ExecUtilException(message='Utility exited with non-zero code. Error `{}`'.format(error), + command=cmd, + exit_code=exit_status, + out=result) + if verbose: + return exit_status, result, error + else: + return result + + # Environment setup + def environ(self, var_name): + return os.environ.get(var_name) + + def find_executable(self, executable): + return find_executable(executable) + + def is_executable(self, file): + # Check if the file is executable + return os.stat(file).st_mode & stat.S_IXUSR + + def set_env(self, var_name, var_val): + # Check if the directory is already in PATH + os.environ[var_name] = var_val + + # Get environment variables + def get_user(self): + return getpass.getuser() + + def get_name(self): + return os.name + + # Work with dirs + def makedirs(self, path, remove_existing=False): + if remove_existing: + shutil.rmtree(path, ignore_errors=True) + try: + os.makedirs(path) + except FileExistsError: + pass + + def rmdirs(self, path, ignore_errors=True): + return rmtree(path, ignore_errors=ignore_errors) + + def listdir(self, path): + return os.listdir(path) + + def path_exists(self, path): + return os.path.exists(path) + + @property + def pathsep(self): + os_name = self.get_name() + if os_name == "posix": + pathsep = ":" + elif os_name == "nt": + pathsep = ";" + else: + raise Exception("Unsupported operating system: {}".format(os_name)) + return pathsep + + def mkdtemp(self, prefix=None): + return tempfile.mkdtemp(prefix='{}'.format(prefix)) + + def mkstemp(self, prefix=None): + fd, filename = tempfile.mkstemp(prefix=prefix) + os.close(fd) # Close the file descriptor immediately after creating the file + return filename + + def copytree(self, src, dst): + return shutil.copytree(src, dst) + + # Work with files + def write(self, filename, data, truncate=False, binary=False, read_and_write=False): + """ + Write data to a file locally + Args: + filename: The file path where the data will be written. + data: The data to be written to the file. + truncate: If True, the file will be truncated before writing ('w' or 'wb' option); + if False (default), data will be appended ('a' or 'ab' option). + binary: If True, the data will be written in binary mode ('wb' or 'ab' option); + if False (default), the data will be written in text mode ('w' or 'a' option). + read_and_write: If True, the file will be opened with read and write permissions ('r+' option); + if False (default), only write permission will be used ('w', 'a', 'wb', or 'ab' option) + """ + # If it is a bytes str or list + if isinstance(data, bytes) or isinstance(data, list) and all(isinstance(item, bytes) for item in data): + binary = True + mode = "wb" if binary else "w" + if not truncate: + mode = "ab" if binary else "a" + if read_and_write: + mode = "r+b" if binary else "r+" + + with open(filename, mode) as file: + if isinstance(data, list): + file.writelines(data) + else: + file.write(data) + + def touch(self, filename): + """ + Create a new file or update the access and modification times of an existing file. + Args: + filename (str): The name of the file to touch. + + This method behaves as the 'touch' command in Unix. It's equivalent to calling 'touch filename' in the shell. + """ + # cross-python touch(). It is vulnerable to races, but who cares? + with open(filename, "a"): + os.utime(filename, None) + + def read(self, filename, encoding=None): + with open(filename, "r", encoding=encoding) as file: + return file.read() + + def readlines(self, filename, num_lines=0, binary=False, encoding=None): + """ + Read lines from a local file. + If num_lines is greater than 0, only the last num_lines lines will be read. + """ + assert num_lines >= 0 + mode = 'rb' if binary else 'r' + if num_lines == 0: + with open(filename, mode, encoding=encoding) as file: # open in binary mode + return file.readlines() + + else: + bufsize = 8192 + buffers = 1 + + with open(filename, mode, encoding=encoding) as file: # open in binary mode + file.seek(0, os.SEEK_END) + end_pos = file.tell() + + while True: + offset = max(0, end_pos - bufsize * buffers) + file.seek(offset, os.SEEK_SET) + pos = file.tell() + lines = file.readlines() + cur_lines = len(lines) + + if cur_lines >= num_lines or pos == 0: + return lines[-num_lines:] # get last num_lines from lines + + buffers = int( + buffers * max(2, int(num_lines / max(cur_lines, 1))) + ) # Adjust buffer size + + def isfile(self, remote_file): + return os.path.isfile(remote_file) + + def isdir(self, dirname): + return os.path.isdir(dirname) + + def remove_file(self, filename): + return os.remove(filename) + + # Processes control + def kill(self, pid, signal): + # Kill the process + cmd = "kill -{} {}".format(signal, pid) + return self.exec_command(cmd) + + def get_pid(self): + # Get current process id + return os.getpid() + + def get_process_children(self, pid): + return psutil.Process(pid).children() + + # Database control + def db_connect(self, dbname, user, password=None, host="localhost", port=5432): + conn = pglib.connect( + host=host, + port=port, + database=dbname, + user=user, + password=password, + ) + return conn diff --git a/testgres/operations/os_ops.py b/testgres/operations/os_ops.py new file mode 100644 index 00000000..9261cacf --- /dev/null +++ b/testgres/operations/os_ops.py @@ -0,0 +1,101 @@ +try: + import psycopg2 as pglib # noqa: F401 +except ImportError: + try: + import pg8000 as pglib # noqa: F401 + except ImportError: + raise ImportError("You must have psycopg2 or pg8000 modules installed") + + +class ConnectionParams: + def __init__(self, host='127.0.0.1', ssh_key=None, username=None): + self.host = host + self.ssh_key = ssh_key + self.username = username + + +class OsOperations: + def __init__(self, username=None): + self.ssh_key = None + self.username = username + + # Command execution + def exec_command(self, cmd, **kwargs): + raise NotImplementedError() + + # Environment setup + def environ(self, var_name): + raise NotImplementedError() + + def find_executable(self, executable): + raise NotImplementedError() + + def is_executable(self, file): + # Check if the file is executable + raise NotImplementedError() + + def set_env(self, var_name, var_val): + # Check if the directory is already in PATH + raise NotImplementedError() + + # Get environment variables + def get_user(self): + raise NotImplementedError() + + def get_name(self): + raise NotImplementedError() + + # Work with dirs + def makedirs(self, path, remove_existing=False): + raise NotImplementedError() + + def rmdirs(self, path, ignore_errors=True): + raise NotImplementedError() + + def listdir(self, path): + raise NotImplementedError() + + def path_exists(self, path): + raise NotImplementedError() + + @property + def pathsep(self): + raise NotImplementedError() + + def mkdtemp(self, prefix=None): + raise NotImplementedError() + + def copytree(self, src, dst): + raise NotImplementedError() + + # Work with files + def write(self, filename, data, truncate=False, binary=False, read_and_write=False): + raise NotImplementedError() + + def touch(self, filename): + raise NotImplementedError() + + def read(self, filename): + raise NotImplementedError() + + def readlines(self, filename): + raise NotImplementedError() + + def isfile(self, remote_file): + raise NotImplementedError() + + # Processes control + def kill(self, pid, signal): + # Kill the process + raise NotImplementedError() + + def get_pid(self): + # Get current process id + raise NotImplementedError() + + def get_process_children(self, pid): + raise NotImplementedError() + + # Database control + def db_connect(self, dbname, user, password=None, host="localhost", port=5432): + raise NotImplementedError() diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py new file mode 100644 index 00000000..6815c7f1 --- /dev/null +++ b/testgres/operations/remote_ops.py @@ -0,0 +1,448 @@ +import os +import tempfile +import time +from typing import Optional + +import sshtunnel + +import paramiko +from paramiko import SSHClient + +from ..exceptions import ExecUtilException + +from .os_ops import OsOperations, ConnectionParams +from .os_ops import pglib + +sshtunnel.SSH_TIMEOUT = 5.0 +sshtunnel.TUNNEL_TIMEOUT = 5.0 + + +error_markers = [b'error', b'Permission denied', b'fatal', b'No such file or directory'] + + +class PsUtilProcessProxy: + def __init__(self, ssh, pid): + self.ssh = ssh + self.pid = pid + + def kill(self): + command = "kill {}".format(self.pid) + self.ssh.exec_command(command) + + def cmdline(self): + command = "ps -p {} -o cmd --no-headers".format(self.pid) + stdin, stdout, stderr = self.ssh.exec_command(command) + cmdline = stdout.read().decode('utf-8').strip() + return cmdline.split() + + +class RemoteOperations(OsOperations): + def __init__(self, conn_params: ConnectionParams): + super().__init__(conn_params.username) + self.conn_params = conn_params + self.host = conn_params.host + self.ssh_key = conn_params.ssh_key + self.ssh = self.ssh_connect() + self.remote = True + self.username = conn_params.username or self.get_user() + self.tunnel = None + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.close_tunnel() + if getattr(self, 'ssh', None): + self.ssh.close() + + def __del__(self): + if getattr(self, 'ssh', None): + self.ssh.close() + + def close_tunnel(self): + if getattr(self, 'tunnel', None): + self.tunnel.stop(force=True) + start_time = time.time() + while self.tunnel.is_active: + if time.time() - start_time > sshtunnel.TUNNEL_TIMEOUT: + break + time.sleep(0.5) + + def ssh_connect(self) -> Optional[SSHClient]: + key = self._read_ssh_key() + ssh = paramiko.SSHClient() + ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + ssh.connect(self.host, username=self.username, pkey=key) + return ssh + + def _read_ssh_key(self): + try: + with open(self.ssh_key, "r") as f: + key_data = f.read() + if "BEGIN OPENSSH PRIVATE KEY" in key_data: + key = paramiko.Ed25519Key.from_private_key_file(self.ssh_key) + else: + key = paramiko.RSAKey.from_private_key_file(self.ssh_key) + return key + except FileNotFoundError: + raise ExecUtilException(message="No such file or directory: '{}'".format(self.ssh_key)) + except Exception as e: + ExecUtilException(message="An error occurred while reading the ssh key: {}".format(e)) + + def exec_command(self, cmd: str, wait_exit=False, verbose=False, expect_error=False, + encoding=None, shell=True, text=False, input=None, stdin=None, stdout=None, + stderr=None, proc=None): + """ + Execute a command in the SSH session. + Args: + - cmd (str): The command to be executed. + """ + if self.ssh is None or not self.ssh.get_transport() or not self.ssh.get_transport().is_active(): + self.ssh = self.ssh_connect() + + if isinstance(cmd, list): + cmd = ' '.join(item.decode('utf-8') if isinstance(item, bytes) else item for item in cmd) + if input: + stdin, stdout, stderr = self.ssh.exec_command(cmd) + stdin.write(input) + stdin.flush() + else: + stdin, stdout, stderr = self.ssh.exec_command(cmd) + exit_status = 0 + if wait_exit: + exit_status = stdout.channel.recv_exit_status() + + if encoding: + result = stdout.read().decode(encoding) + error = stderr.read().decode(encoding) + else: + result = stdout.read() + error = stderr.read() + + if expect_error: + raise Exception(result, error) + + if encoding: + error_found = exit_status != 0 or any( + marker.decode(encoding) in error for marker in error_markers) + else: + error_found = exit_status != 0 or any( + marker in error for marker in error_markers) + + if error_found: + if exit_status == 0: + exit_status = 1 + if encoding: + message = "Utility exited with non-zero code. Error: {}".format(error.decode(encoding)) + else: + message = b"Utility exited with non-zero code. Error: " + error + raise ExecUtilException(message=message, + command=cmd, + exit_code=exit_status, + out=result) + + if verbose: + return exit_status, result, error + else: + return result + + # Environment setup + def environ(self, var_name: str) -> str: + """ + Get the value of an environment variable. + Args: + - var_name (str): The name of the environment variable. + """ + cmd = "echo ${}".format(var_name) + return self.exec_command(cmd, encoding='utf-8').strip() + + def find_executable(self, executable): + search_paths = self.environ("PATH") + if not search_paths: + return None + + search_paths = search_paths.split(self.pathsep) + for path in search_paths: + remote_file = os.path.join(path, executable) + if self.isfile(remote_file): + return remote_file + + return None + + def is_executable(self, file): + # Check if the file is executable + is_exec = self.exec_command("test -x {} && echo OK".format(file)) + return is_exec == b"OK\n" + + def set_env(self, var_name: str, var_val: str): + """ + Set the value of an environment variable. + Args: + - var_name (str): The name of the environment variable. + - var_val (str): The value to be set for the environment variable. + """ + return self.exec_command("export {}={}".format(var_name, var_val)) + + # Get environment variables + def get_user(self): + return self.exec_command("echo $USER", encoding='utf-8').strip() + + def get_name(self): + cmd = 'python3 -c "import os; print(os.name)"' + return self.exec_command(cmd, encoding='utf-8').strip() + + # Work with dirs + def makedirs(self, path, remove_existing=False): + """ + Create a directory in the remote server. + Args: + - path (str): The path to the directory to be created. + - remove_existing (bool): If True, the existing directory at the path will be removed. + """ + if remove_existing: + cmd = "rm -rf {} && mkdir -p {}".format(path, path) + else: + cmd = "mkdir -p {}".format(path) + try: + exit_status, result, error = self.exec_command(cmd, verbose=True) + except ExecUtilException as e: + raise Exception("Couldn't create dir {} because of error {}".format(path, e.message)) + if exit_status != 0: + raise Exception("Couldn't create dir {} because of error {}".format(path, error)) + return result + + def rmdirs(self, path, verbose=False, ignore_errors=True): + """ + Remove a directory in the remote server. + Args: + - path (str): The path to the directory to be removed. + - verbose (bool): If True, return exit status, result, and error. + - ignore_errors (bool): If True, do not raise error if directory does not exist. + """ + cmd = "rm -rf {}".format(path) + exit_status, result, error = self.exec_command(cmd, verbose=True) + if verbose: + return exit_status, result, error + else: + return result + + def listdir(self, path): + """ + List all files and directories in a directory. + Args: + path (str): The path to the directory. + """ + result = self.exec_command("ls {}".format(path)) + return result.splitlines() + + def path_exists(self, path): + result = self.exec_command("test -e {}; echo $?".format(path), encoding='utf-8') + return int(result.strip()) == 0 + + @property + def pathsep(self): + os_name = self.get_name() + if os_name == "posix": + pathsep = ":" + elif os_name == "nt": + pathsep = ";" + else: + raise Exception("Unsupported operating system: {}".format(os_name)) + return pathsep + + def mkdtemp(self, prefix=None): + """ + Creates a temporary directory in the remote server. + Args: + - prefix (str): The prefix of the temporary directory name. + """ + if prefix: + temp_dir = self.exec_command("mktemp -d {}XXXXX".format(prefix), encoding='utf-8') + else: + temp_dir = self.exec_command("mktemp -d", encoding='utf-8') + + if temp_dir: + if not os.path.isabs(temp_dir): + temp_dir = os.path.join('/home', self.username, temp_dir.strip()) + return temp_dir + else: + raise ExecUtilException("Could not create temporary directory.") + + def mkstemp(self, prefix=None): + if prefix: + temp_dir = self.exec_command("mktemp {}XXXXX".format(prefix), encoding='utf-8') + else: + temp_dir = self.exec_command("mktemp", encoding='utf-8') + + if temp_dir: + if not os.path.isabs(temp_dir): + temp_dir = os.path.join('/home', self.username, temp_dir.strip()) + return temp_dir + else: + raise ExecUtilException("Could not create temporary directory.") + + def copytree(self, src, dst): + if not os.path.isabs(dst): + dst = os.path.join('~', dst) + if self.isdir(dst): + raise FileExistsError("Directory {} already exists.".format(dst)) + return self.exec_command("cp -r {} {}".format(src, dst)) + + # Work with files + def write(self, filename, data, truncate=False, binary=False, read_and_write=False, encoding='utf-8'): + """ + Write data to a file on a remote host + + Args: + - filename (str): The file path where the data will be written. + - data (bytes or str): The data to be written to the file. + - truncate (bool): If True, the file will be truncated before writing ('w' or 'wb' option); + if False (default), data will be appended ('a' or 'ab' option). + - binary (bool): If True, the data will be written in binary mode ('wb' or 'ab' option); + if False (default), the data will be written in text mode ('w' or 'a' option). + - read_and_write (bool): If True, the file will be opened with read and write permissions ('r+' option); + if False (default), only write permission will be used ('w', 'a', 'wb', or 'ab' option). + """ + mode = "wb" if binary else "w" + if not truncate: + mode = "ab" if binary else "a" + if read_and_write: + mode = "r+b" if binary else "r+" + + with tempfile.NamedTemporaryFile(mode=mode, delete=False) as tmp_file: + if not truncate: + with self.ssh_connect() as ssh: + sftp = ssh.open_sftp() + try: + sftp.get(filename, tmp_file.name) + tmp_file.seek(0, os.SEEK_END) + except FileNotFoundError: + pass # File does not exist yet, we'll create it + sftp.close() + if isinstance(data, bytes) and not binary: + data = data.decode(encoding) + elif isinstance(data, str) and binary: + data = data.encode(encoding) + if isinstance(data, list): + # ensure each line ends with a newline + data = [(s if isinstance(s, str) else s.decode('utf-8')).rstrip('\n') + '\n' for s in data] + tmp_file.writelines(data) + else: + tmp_file.write(data) + tmp_file.flush() + + with self.ssh_connect() as ssh: + sftp = ssh.open_sftp() + remote_directory = os.path.dirname(filename) + try: + sftp.stat(remote_directory) + except IOError: + sftp.mkdir(remote_directory) + sftp.put(tmp_file.name, filename) + sftp.close() + + os.remove(tmp_file.name) + + def touch(self, filename): + """ + Create a new file or update the access and modification times of an existing file on the remote server. + + Args: + filename (str): The name of the file to touch. + + This method behaves as the 'touch' command in Unix. It's equivalent to calling 'touch filename' in the shell. + """ + self.exec_command("touch {}".format(filename)) + + def read(self, filename, binary=False, encoding=None): + cmd = "cat {}".format(filename) + result = self.exec_command(cmd, encoding=encoding) + + if not binary and result: + result = result.decode(encoding or 'utf-8') + + return result + + def readlines(self, filename, num_lines=0, binary=False, encoding=None): + if num_lines > 0: + cmd = "tail -n {} {}".format(num_lines, filename) + else: + cmd = "cat {}".format(filename) + + result = self.exec_command(cmd, encoding=encoding) + + if not binary and result: + lines = result.decode(encoding or 'utf-8').splitlines() + else: + lines = result.splitlines() + + return lines + + def isfile(self, remote_file): + stdout = self.exec_command("test -f {}; echo $?".format(remote_file)) + result = int(stdout.strip()) + return result == 0 + + def isdir(self, dirname): + cmd = "if [ -d {} ]; then echo True; else echo False; fi".format(dirname) + response = self.exec_command(cmd) + return response.strip() == b"True" + + def remove_file(self, filename): + cmd = "rm {}".format(filename) + return self.exec_command(cmd) + + # Processes control + def kill(self, pid, signal): + # Kill the process + cmd = "kill -{} {}".format(signal, pid) + return self.exec_command(cmd) + + def get_pid(self): + # Get current process id + return int(self.exec_command("echo $$", encoding='utf-8')) + + def get_process_children(self, pid): + command = "pgrep -P {}".format(pid) + stdin, stdout, stderr = self.ssh.exec_command(command) + children = stdout.readlines() + return [PsUtilProcessProxy(self.ssh, int(child_pid.strip())) for child_pid in children] + + # Database control + def db_connect(self, dbname, user, password=None, host="127.0.0.1", port=5432, ssh_key=None): + """ + Connects to a PostgreSQL database on the remote system. + Args: + - dbname (str): The name of the database to connect to. + - user (str): The username for the database connection. + - password (str, optional): The password for the database connection. Defaults to None. + - host (str, optional): The IP address of the remote system. Defaults to "localhost". + - port (int, optional): The port number of the PostgreSQL service. Defaults to 5432. + + This function establishes a connection to a PostgreSQL database on the remote system using the specified + parameters. It returns a connection object that can be used to interact with the database. + """ + self.close_tunnel() + self.tunnel = sshtunnel.open_tunnel( + (host, 22), # Remote server IP and SSH port + ssh_username=user or self.username, + ssh_pkey=ssh_key or self.ssh_key, + remote_bind_address=(host, port), # PostgreSQL server IP and PostgreSQL port + local_bind_address=('localhost', port) # Local machine IP and available port + ) + + self.tunnel.start() + + try: + conn = pglib.connect( + host=host, # change to 'localhost' because we're connecting through a local ssh tunnel + port=self.tunnel.local_bind_port, # use the local bind port set up by the tunnel + database=dbname, + user=user or self.username, + password=password + ) + + return conn + except Exception as e: + self.tunnel.stop() + raise ExecUtilException("Could not create db tunnel. {}".format(e)) diff --git a/testgres/pubsub.py b/testgres/pubsub.py index da85caac..1be673bb 100644 --- a/testgres/pubsub.py +++ b/testgres/pubsub.py @@ -214,4 +214,4 @@ def catchup(self, username=None): username=username or self.pub.username, max_attempts=LOGICAL_REPL_MAX_CATCHUP_ATTEMPTS) except Exception as e: - raise_from(CatchUpException("Failed to catch up", query), e) + raise_from(CatchUpException("Failed to catch up"), e) diff --git a/testgres/utils.py b/testgres/utils.py index 9760908d..5e12eba9 100644 --- a/testgres/utils.py +++ b/testgres/utils.py @@ -3,24 +3,18 @@ from __future__ import division from __future__ import print_function -import io import os import port_for -import subprocess import sys -import tempfile from contextlib import contextmanager from packaging.version import Version, InvalidVersion import re -try: - from shutil import which as find_executable -except ImportError: - from distutils.spawn import find_executable + from six import iteritems -from .config import testgres_config from .exceptions import ExecUtilException +from .config import testgres_config as tconf # rows returned by PG_CONFIG _pg_config_data = {} @@ -58,7 +52,7 @@ def release_port(port): bound_ports.discard(port) -def execute_utility(args, logfile=None): +def execute_utility(args, logfile=None, verbose=False): """ Execute utility (pg_ctl, pg_dump etc). @@ -69,63 +63,28 @@ def execute_utility(args, logfile=None): Returns: stdout of executed utility. """ - - # run utility - if os.name == 'nt': - # using output to a temporary file in Windows - buf = tempfile.NamedTemporaryFile() - - process = subprocess.Popen( - args, # util + params - stdout=buf, - stderr=subprocess.STDOUT) - process.communicate() - - # get result - buf.file.flush() - buf.file.seek(0) - out = buf.file.read() - buf.close() - else: - process = subprocess.Popen( - args, # util + params - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) - - # get result - out, _ = process.communicate() - + exit_status, out, error = tconf.os_ops.exec_command(args, verbose=True) # decode result - out = '' if not out else out.decode('utf-8') - - # format command - command = u' '.join(args) + out = '' if not out else out + if isinstance(out, bytes): + out = out.decode('utf-8') + if isinstance(error, bytes): + error = error.decode('utf-8') # write new log entry if possible if logfile: try: - with io.open(logfile, 'a') as file_out: - file_out.write(command) - - if out: - # comment-out lines - lines = ('# ' + line for line in out.splitlines(True)) - file_out.write(u'\n') - file_out.writelines(lines) - - file_out.write(u'\n') + tconf.os_ops.write(filename=logfile, data=args, truncate=True) + if out: + # comment-out lines + lines = [u'\n'] + ['# ' + line for line in out.splitlines()] + [u'\n'] + tconf.os_ops.write(filename=logfile, data=lines) except IOError: - pass - - exit_code = process.returncode - if exit_code: - message = 'Utility exited with non-zero code' - raise ExecUtilException(message=message, - command=command, - exit_code=exit_code, - out=out) - - return out + raise ExecUtilException("Problem with writing to logfile `{}` during run command `{}`".format(logfile, args)) + if verbose: + return exit_status, out, error + else: + return out def get_bin_path(filename): @@ -133,23 +92,25 @@ def get_bin_path(filename): Return absolute path to an executable using PG_BIN or PG_CONFIG. This function does nothing if 'filename' is already absolute. """ - # check if it's already absolute if os.path.isabs(filename): return filename + if tconf.os_ops.remote: + pg_config = os.environ.get("PG_CONFIG_REMOTE") or os.environ.get("PG_CONFIG") + else: + # try PG_CONFIG - get from local machine + pg_config = os.environ.get("PG_CONFIG") - # try PG_CONFIG - pg_config = os.environ.get("PG_CONFIG") if pg_config: bindir = get_pg_config()["BINDIR"] return os.path.join(bindir, filename) # try PG_BIN - pg_bin = os.environ.get("PG_BIN") + pg_bin = tconf.os_ops.environ("PG_BIN") if pg_bin: return os.path.join(pg_bin, filename) - pg_config_path = find_executable('pg_config') + pg_config_path = tconf.os_ops.find_executable('pg_config') if pg_config_path: bindir = get_pg_config(pg_config_path)["BINDIR"] return os.path.join(bindir, filename) @@ -160,11 +121,12 @@ def get_bin_path(filename): def get_pg_config(pg_config_path=None): """ Return output of pg_config (provided that it is installed). - NOTE: this fuction caches the result by default (see GlobalConfig). + NOTE: this function caches the result by default (see GlobalConfig). """ + def cache_pg_config_data(cmd): # execute pg_config and get the output - out = subprocess.check_output([cmd]).decode('utf-8') + out = tconf.os_ops.exec_command(cmd, encoding='utf-8') data = {} for line in out.splitlines(): @@ -179,7 +141,7 @@ def cache_pg_config_data(cmd): return data # drop cache if asked to - if not testgres_config.cache_pg_config: + if not tconf.cache_pg_config: global _pg_config_data _pg_config_data = {} @@ -188,7 +150,11 @@ def cache_pg_config_data(cmd): return _pg_config_data # try specified pg_config path or PG_CONFIG - pg_config = pg_config_path or os.environ.get("PG_CONFIG") + if tconf.os_ops.remote: + pg_config = pg_config_path or os.environ.get("PG_CONFIG_REMOTE") or os.environ.get("PG_CONFIG") + else: + # try PG_CONFIG - get from local machine + pg_config = pg_config_path or os.environ.get("PG_CONFIG") if pg_config: return cache_pg_config_data(pg_config) @@ -209,7 +175,7 @@ def get_pg_version(): # get raw version (e.g. postgres (PostgreSQL) 9.5.7) _params = [get_bin_path('postgres'), '--version'] - raw_ver = subprocess.check_output(_params).decode('utf-8') + raw_ver = tconf.os_ops.exec_command(_params, encoding='utf-8') # cook version of PostgreSQL version = raw_ver.strip().split(' ')[-1] \ diff --git a/tests/README.md b/tests/README.md index a6d50992..d89efc7e 100644 --- a/tests/README.md +++ b/tests/README.md @@ -27,3 +27,32 @@ export PYTHON_VERSION=3 # or 2 # Run tests ./run_tests.sh ``` + + +#### Remote host tests + +1. Start remote host or docker container +2. Make sure that you run ssh +```commandline +sudo apt-get install openssh-server +sudo systemctl start sshd +``` +3. You need to connect to the remote host at least once to add it to the known hosts file +4. Generate ssh keys +5. Set up params for tests + + +```commandline +conn_params = ConnectionParams( + host='remote_host', + username='username', + ssh_key=/path/to/your/ssh/key' +) +os_ops = RemoteOperations(conn_params) +``` +If you have different path to `PG_CONFIG` on your local and remote host you can set up `PG_CONFIG_REMOTE`, this value will be +using during work with remote host. + +`test_remote` - Tests for RemoteOperations class. + +`test_simple_remote` - Tests that create node and check it. The same as `test_simple`, but for remote node. \ No newline at end of file diff --git a/tests/test_remote.py b/tests/test_remote.py new file mode 100755 index 00000000..3794349c --- /dev/null +++ b/tests/test_remote.py @@ -0,0 +1,198 @@ +import os + +import pytest + +from testgres import ExecUtilException +from testgres import RemoteOperations +from testgres import ConnectionParams + + +class TestRemoteOperations: + + @pytest.fixture(scope="function", autouse=True) + def setup(self): + conn_params = ConnectionParams(host=os.getenv('RDBMS_TESTPOOL1_HOST') or '172.18.0.3', + username='dev', + ssh_key=os.getenv( + 'RDBMS_TESTPOOL_SSHKEY') or '../../container_files/postgres/ssh/id_ed25519') + self.operations = RemoteOperations(conn_params) + + yield + self.operations.__del__() + + def test_exec_command_success(self): + """ + Test exec_command for successful command execution. + """ + cmd = "python3 --version" + response = self.operations.exec_command(cmd, wait_exit=True) + + assert b'Python 3.' in response + + def test_exec_command_failure(self): + """ + Test exec_command for command execution failure. + """ + cmd = "nonexistent_command" + try: + exit_status, result, error = self.operations.exec_command(cmd, verbose=True, wait_exit=True) + except ExecUtilException as e: + error = e.message + assert error == b'Utility exited with non-zero code. Error: bash: line 1: nonexistent_command: command not found\n' + + def test_is_executable_true(self): + """ + Test is_executable for an existing executable. + """ + cmd = "postgres" + response = self.operations.is_executable(cmd) + + assert response is True + + def test_is_executable_false(self): + """ + Test is_executable for a non-executable. + """ + cmd = "python" + response = self.operations.is_executable(cmd) + + assert response is False + + def test_makedirs_and_rmdirs_success(self): + """ + Test makedirs and rmdirs for successful directory creation and removal. + """ + cmd = "pwd" + pwd = self.operations.exec_command(cmd, wait_exit=True, encoding='utf-8').strip() + + path = "{}/test_dir".format(pwd) + + # Test makedirs + self.operations.makedirs(path) + assert self.operations.path_exists(path) + + # Test rmdirs + self.operations.rmdirs(path) + assert not self.operations.path_exists(path) + + def test_makedirs_and_rmdirs_failure(self): + """ + Test makedirs and rmdirs for directory creation and removal failure. + """ + # Try to create a directory in a read-only location + path = "/root/test_dir" + + # Test makedirs + with pytest.raises(Exception): + self.operations.makedirs(path) + + # Test rmdirs + try: + exit_status, result, error = self.operations.rmdirs(path, verbose=True) + except ExecUtilException as e: + error = e.message + assert error == b"Utility exited with non-zero code. Error: rm: cannot remove '/root/test_dir': Permission denied\n" + + def test_listdir(self): + """ + Test listdir for listing directory contents. + """ + path = "/etc" + files = self.operations.listdir(path) + + assert isinstance(files, list) + + def test_path_exists_true(self): + """ + Test path_exists for an existing path. + """ + path = "/etc" + response = self.operations.path_exists(path) + + assert response is True + + def test_path_exists_false(self): + """ + Test path_exists for a non-existing path. + """ + path = "/nonexistent_path" + response = self.operations.path_exists(path) + + assert response is False + + def test_write_text_file(self): + """ + Test write for writing data to a text file. + """ + filename = "/tmp/test_file.txt" + data = "Hello, world!" + + self.operations.write(filename, data, truncate=True) + self.operations.write(filename, data) + + response = self.operations.read(filename) + + assert response == data + data + + def test_write_binary_file(self): + """ + Test write for writing data to a binary file. + """ + filename = "/tmp/test_file.bin" + data = b"\x00\x01\x02\x03" + + self.operations.write(filename, data, binary=True, truncate=True) + + response = self.operations.read(filename, binary=True) + + assert response == data + + def test_read_text_file(self): + """ + Test read for reading data from a text file. + """ + filename = "/etc/hosts" + + response = self.operations.read(filename) + + assert isinstance(response, str) + + def test_read_binary_file(self): + """ + Test read for reading data from a binary file. + """ + filename = "/usr/bin/python3" + + response = self.operations.read(filename, binary=True) + + assert isinstance(response, bytes) + + def test_touch(self): + """ + Test touch for creating a new file or updating access and modification times of an existing file. + """ + filename = "/tmp/test_file.txt" + + self.operations.touch(filename) + + assert self.operations.isfile(filename) + + def test_isfile_true(self): + """ + Test isfile for an existing file. + """ + filename = "/etc/hosts" + + response = self.operations.isfile(filename) + + assert response is True + + def test_isfile_false(self): + """ + Test isfile for a non-existing file. + """ + filename = "/nonexistent_file.txt" + + response = self.operations.isfile(filename) + + assert response is False diff --git a/tests/test_simple_remote.py b/tests/test_simple_remote.py new file mode 100755 index 00000000..e8386383 --- /dev/null +++ b/tests/test_simple_remote.py @@ -0,0 +1,996 @@ +#!/usr/bin/env python +# coding: utf-8 + +import os +import re +import subprocess +import tempfile + +import testgres +import time +import six +import unittest +import psutil + +import logging.config + +from contextlib import contextmanager + +from testgres.exceptions import \ + InitNodeException, \ + StartNodeException, \ + ExecUtilException, \ + BackupException, \ + QueryException, \ + TimeoutException, \ + TestgresException + +from testgres.config import \ + TestgresConfig, \ + configure_testgres, \ + scoped_config, \ + pop_config, testgres_config + +from testgres import \ + NodeStatus, \ + ProcessType, \ + IsolationLevel, \ + get_remote_node, \ + RemoteOperations + +from testgres import \ + get_bin_path, \ + get_pg_config, \ + get_pg_version + +from testgres import \ + First, \ + Any + +# NOTE: those are ugly imports +from testgres import bound_ports +from testgres.utils import PgVer +from testgres.node import ProcessProxy, ConnectionParams + +conn_params = ConnectionParams(host=os.getenv('RDBMS_TESTPOOL1_HOST') or '172.18.0.3', + username='dev', + ssh_key=os.getenv( + 'RDBMS_TESTPOOL_SSHKEY') or '../../container_files/postgres/ssh/id_ed25519') +os_ops = RemoteOperations(conn_params) +testgres_config.set_os_ops(os_ops=os_ops) + + +def pg_version_ge(version): + cur_ver = PgVer(get_pg_version()) + min_ver = PgVer(version) + return cur_ver >= min_ver + + +def util_exists(util): + def good_properties(f): + return (os_ops.path_exists(f) and # noqa: W504 + os_ops.isfile(f) and # noqa: W504 + os_ops.is_executable(f)) # yapf: disable + + # try to resolve it + if good_properties(get_bin_path(util)): + return True + + # check if util is in PATH + for path in os_ops.environ("PATH").split(os_ops.pathsep): + if good_properties(os.path.join(path, util)): + return True + + +@contextmanager +def removing(f): + try: + yield f + finally: + if os_ops.isfile(f): + os_ops.remove_file(f) + + elif os_ops.isdir(f): + os_ops.rmdirs(f, ignore_errors=True) + + +class TestgresRemoteTests(unittest.TestCase): + + def test_node_repr(self): + with get_remote_node(conn_params=conn_params) as node: + pattern = r"PostgresNode\(name='.+', port=.+, base_dir='.+'\)" + self.assertIsNotNone(re.match(pattern, str(node))) + + def test_custom_init(self): + with get_remote_node(conn_params=conn_params) as node: + # enable page checksums + node.init(initdb_params=['-k']).start() + + with get_remote_node(conn_params=conn_params) as node: + node.init( + allow_streaming=True, + initdb_params=['--auth-local=reject', '--auth-host=reject']) + + hba_file = os.path.join(node.data_dir, 'pg_hba.conf') + lines = os_ops.readlines(hba_file) + + # check number of lines + self.assertGreaterEqual(len(lines), 6) + + # there should be no trust entries at all + self.assertFalse(any('trust' in s for s in lines)) + + def test_double_init(self): + with get_remote_node(conn_params=conn_params).init() as node: + # can't initialize node more than once + with self.assertRaises(InitNodeException): + node.init() + + def test_init_after_cleanup(self): + with get_remote_node(conn_params=conn_params) as node: + node.init().start().execute('select 1') + node.cleanup() + node.init().start().execute('select 1') + + @unittest.skipUnless(util_exists('pg_resetwal'), 'might be missing') + @unittest.skipUnless(pg_version_ge('9.6'), 'requires 9.6+') + def test_init_unique_system_id(self): + # FAIL + # this function exists in PostgreSQL 9.6+ + query = 'select system_identifier from pg_control_system()' + + with scoped_config(cache_initdb=False): + with get_remote_node(conn_params=conn_params).init().start() as node0: + id0 = node0.execute(query)[0] + + with scoped_config(cache_initdb=True, + cached_initdb_unique=True) as config: + self.assertTrue(config.cache_initdb) + self.assertTrue(config.cached_initdb_unique) + + # spawn two nodes; ids must be different + with get_remote_node(conn_params=conn_params).init().start() as node1, \ + get_remote_node(conn_params=conn_params).init().start() as node2: + id1 = node1.execute(query)[0] + id2 = node2.execute(query)[0] + + # ids must increase + self.assertGreater(id1, id0) + self.assertGreater(id2, id1) + + def test_node_exit(self): + with self.assertRaises(QueryException): + with get_remote_node(conn_params=conn_params).init() as node: + base_dir = node.base_dir + node.safe_psql('select 1') + + # we should save the DB for "debugging" + self.assertTrue(os_ops.path_exists(base_dir)) + os_ops.rmdirs(base_dir, ignore_errors=True) + + with get_remote_node(conn_params=conn_params).init() as node: + base_dir = node.base_dir + + # should have been removed by default + self.assertFalse(os_ops.path_exists(base_dir)) + + def test_double_start(self): + with get_remote_node(conn_params=conn_params).init().start() as node: + # can't start node more than once + node.start() + self.assertTrue(node.is_started) + + def test_uninitialized_start(self): + with get_remote_node(conn_params=conn_params) as node: + # node is not initialized yet + with self.assertRaises(StartNodeException): + node.start() + + def test_restart(self): + with get_remote_node(conn_params=conn_params) as node: + node.init().start() + + # restart, ok + res = node.execute('select 1') + self.assertEqual(res, [(1,)]) + node.restart() + res = node.execute('select 2') + self.assertEqual(res, [(2,)]) + + # restart, fail + with self.assertRaises(StartNodeException): + node.append_conf('pg_hba.conf', 'DUMMY') + node.restart() + + def test_reload(self): + with get_remote_node(conn_params=conn_params) as node: + node.init().start() + + # change client_min_messages and save old value + cmm_old = node.execute('show client_min_messages') + node.append_conf(client_min_messages='DEBUG1') + + # reload config + node.reload() + + # check new value + cmm_new = node.execute('show client_min_messages') + self.assertEqual('debug1', cmm_new[0][0].lower()) + self.assertNotEqual(cmm_old, cmm_new) + + def test_pg_ctl(self): + with get_remote_node(conn_params=conn_params) as node: + node.init().start() + + status = node.pg_ctl(['status']) + self.assertTrue('PID' in status) + + def test_status(self): + self.assertTrue(NodeStatus.Running) + self.assertFalse(NodeStatus.Stopped) + self.assertFalse(NodeStatus.Uninitialized) + + # check statuses after each operation + with get_remote_node(conn_params=conn_params) as node: + self.assertEqual(node.pid, 0) + self.assertEqual(node.status(), NodeStatus.Uninitialized) + + node.init() + + self.assertEqual(node.pid, 0) + self.assertEqual(node.status(), NodeStatus.Stopped) + + node.start() + + self.assertNotEqual(node.pid, 0) + self.assertEqual(node.status(), NodeStatus.Running) + + node.stop() + + self.assertEqual(node.pid, 0) + self.assertEqual(node.status(), NodeStatus.Stopped) + + node.cleanup() + + self.assertEqual(node.pid, 0) + self.assertEqual(node.status(), NodeStatus.Uninitialized) + + def test_psql(self): + with get_remote_node(conn_params=conn_params).init().start() as node: + # check returned values (1 arg) + res = node.psql('select 1') + self.assertEqual(res, (0, b'1\n', b'')) + + # check returned values (2 args) + res = node.psql('postgres', 'select 2') + self.assertEqual(res, (0, b'2\n', b'')) + + # check returned values (named) + res = node.psql(query='select 3', dbname='postgres') + self.assertEqual(res, (0, b'3\n', b'')) + + # check returned values (1 arg) + res = node.safe_psql('select 4') + self.assertEqual(res, b'4\n') + + # check returned values (2 args) + res = node.safe_psql('postgres', 'select 5') + self.assertEqual(res, b'5\n') + + # check returned values (named) + res = node.safe_psql(query='select 6', dbname='postgres') + self.assertEqual(res, b'6\n') + + # check feeding input + node.safe_psql('create table horns (w int)') + node.safe_psql('copy horns from stdin (format csv)', + input=b"1\n2\n3\n\\.\n") + _sum = node.safe_psql('select sum(w) from horns') + self.assertEqual(_sum, b'6\n') + + # check psql's default args, fails + with self.assertRaises(QueryException): + node.psql() + + node.stop() + + # check psql on stopped node, fails + with self.assertRaises(QueryException): + node.safe_psql('select 1') + + def test_transactions(self): + with get_remote_node(conn_params=conn_params).init().start() as node: + with node.connect() as con: + con.begin() + con.execute('create table test(val int)') + con.execute('insert into test values (1)') + con.commit() + + con.begin() + con.execute('insert into test values (2)') + res = con.execute('select * from test order by val asc') + self.assertListEqual(res, [(1,), (2,)]) + con.rollback() + + con.begin() + res = con.execute('select * from test') + self.assertListEqual(res, [(1,)]) + con.rollback() + + con.begin() + con.execute('drop table test') + con.commit() + + def test_control_data(self): + with get_remote_node(conn_params=conn_params) as node: + # node is not initialized yet + with self.assertRaises(ExecUtilException): + node.get_control_data() + + node.init() + data = node.get_control_data() + + # check returned dict + self.assertIsNotNone(data) + self.assertTrue(any('pg_control' in s for s in data.keys())) + + def test_backup_simple(self): + with get_remote_node(conn_params=conn_params) as master: + # enable streaming for backups + master.init(allow_streaming=True) + + # node must be running + with self.assertRaises(BackupException): + master.backup() + + # it's time to start node + master.start() + + # fill node with some data + master.psql('create table test as select generate_series(1, 4) i') + + with master.backup(xlog_method='stream') as backup: + with backup.spawn_primary().start() as slave: + res = slave.execute('select * from test order by i asc') + self.assertListEqual(res, [(1,), (2,), (3,), (4,)]) + + def test_backup_multiple(self): + with get_remote_node(conn_params=conn_params) as node: + node.init(allow_streaming=True).start() + + with node.backup(xlog_method='fetch') as backup1, \ + node.backup(xlog_method='fetch') as backup2: + self.assertNotEqual(backup1.base_dir, backup2.base_dir) + + with node.backup(xlog_method='fetch') as backup: + with backup.spawn_primary('node1', destroy=False) as node1, \ + backup.spawn_primary('node2', destroy=False) as node2: + self.assertNotEqual(node1.base_dir, node2.base_dir) + + def test_backup_exhaust(self): + with get_remote_node(conn_params=conn_params) as node: + node.init(allow_streaming=True).start() + + with node.backup(xlog_method='fetch') as backup: + # exhaust backup by creating new node + with backup.spawn_primary(): + pass + + # now let's try to create one more node + with self.assertRaises(BackupException): + backup.spawn_primary() + + def test_backup_wrong_xlog_method(self): + with get_remote_node(conn_params=conn_params) as node: + node.init(allow_streaming=True).start() + + with self.assertRaises(BackupException, + msg='Invalid xlog_method "wrong"'): + node.backup(xlog_method='wrong') + + def test_pg_ctl_wait_option(self): + with get_remote_node(conn_params=conn_params) as node: + node.init().start(wait=False) + while True: + try: + node.stop(wait=False) + break + except ExecUtilException: + # it's ok to get this exception here since node + # could be not started yet + pass + + def test_replicate(self): + with get_remote_node(conn_params=conn_params) as node: + node.init(allow_streaming=True).start() + + with node.replicate().start() as replica: + res = replica.execute('select 1') + self.assertListEqual(res, [(1,)]) + + node.execute('create table test (val int)', commit=True) + + replica.catchup() + + res = node.execute('select * from test') + self.assertListEqual(res, []) + + @unittest.skipUnless(pg_version_ge('9.6'), 'requires 9.6+') + def test_synchronous_replication(self): + with get_remote_node(conn_params=conn_params) as master: + old_version = not pg_version_ge('9.6') + + master.init(allow_streaming=True).start() + + if not old_version: + master.append_conf('synchronous_commit = remote_apply') + + # create standby + with master.replicate() as standby1, master.replicate() as standby2: + standby1.start() + standby2.start() + + # check formatting + self.assertEqual( + '1 ("{}", "{}")'.format(standby1.name, standby2.name), + str(First(1, (standby1, standby2)))) # yapf: disable + self.assertEqual( + 'ANY 1 ("{}", "{}")'.format(standby1.name, standby2.name), + str(Any(1, (standby1, standby2)))) # yapf: disable + + # set synchronous_standby_names + master.set_synchronous_standbys(First(2, [standby1, standby2])) + master.restart() + + # the following part of the test is only applicable to newer + # versions of PostgresQL + if not old_version: + master.safe_psql('create table abc(a int)') + + # Create a large transaction that will take some time to apply + # on standby to check that it applies synchronously + # (If set synchronous_commit to 'on' or other lower level then + # standby most likely won't catchup so fast and test will fail) + master.safe_psql( + 'insert into abc select generate_series(1, 1000000)') + res = standby1.safe_psql('select count(*) from abc') + self.assertEqual(res, b'1000000\n') + + @unittest.skipUnless(pg_version_ge('10'), 'requires 10+') + def test_logical_replication(self): + with get_remote_node(conn_params=conn_params) as node1, get_remote_node(conn_params=conn_params) as node2: + node1.init(allow_logical=True) + node1.start() + node2.init().start() + + create_table = 'create table test (a int, b int)' + node1.safe_psql(create_table) + node2.safe_psql(create_table) + + # create publication / create subscription + pub = node1.publish('mypub') + sub = node2.subscribe(pub, 'mysub') + + node1.safe_psql('insert into test values (1, 1), (2, 2)') + + # wait until changes apply on subscriber and check them + sub.catchup() + res = node2.execute('select * from test') + self.assertListEqual(res, [(1, 1), (2, 2)]) + + # disable and put some new data + sub.disable() + node1.safe_psql('insert into test values (3, 3)') + + # enable and ensure that data successfully transfered + sub.enable() + sub.catchup() + res = node2.execute('select * from test') + self.assertListEqual(res, [(1, 1), (2, 2), (3, 3)]) + + # Add new tables. Since we added "all tables" to publication + # (default behaviour of publish() method) we don't need + # to explicitely perform pub.add_tables() + create_table = 'create table test2 (c char)' + node1.safe_psql(create_table) + node2.safe_psql(create_table) + sub.refresh() + + # put new data + node1.safe_psql('insert into test2 values (\'a\'), (\'b\')') + sub.catchup() + res = node2.execute('select * from test2') + self.assertListEqual(res, [('a',), ('b',)]) + + # drop subscription + sub.drop() + pub.drop() + + # create new publication and subscription for specific table + # (ommitting copying data as it's already done) + pub = node1.publish('newpub', tables=['test']) + sub = node2.subscribe(pub, 'newsub', copy_data=False) + + node1.safe_psql('insert into test values (4, 4)') + sub.catchup() + res = node2.execute('select * from test') + self.assertListEqual(res, [(1, 1), (2, 2), (3, 3), (4, 4)]) + + # explicitely add table + with self.assertRaises(ValueError): + pub.add_tables([]) # fail + pub.add_tables(['test2']) + node1.safe_psql('insert into test2 values (\'c\')') + sub.catchup() + res = node2.execute('select * from test2') + self.assertListEqual(res, [('a',), ('b',)]) + + @unittest.skipUnless(pg_version_ge('10'), 'requires 10+') + def test_logical_catchup(self): + """ Runs catchup for 100 times to be sure that it is consistent """ + with get_remote_node(conn_params=conn_params) as node1, get_remote_node(conn_params=conn_params) as node2: + node1.init(allow_logical=True) + node1.start() + node2.init().start() + + create_table = 'create table test (key int primary key, val int); ' + node1.safe_psql(create_table) + node1.safe_psql('alter table test replica identity default') + node2.safe_psql(create_table) + + # create publication / create subscription + sub = node2.subscribe(node1.publish('mypub'), 'mysub') + + for i in range(0, 100): + node1.execute('insert into test values ({0}, {0})'.format(i)) + sub.catchup() + res = node2.execute('select * from test') + self.assertListEqual(res, [( + i, + i, + )]) + node1.execute('delete from test') + + @unittest.skipIf(pg_version_ge('10'), 'requires <10') + def test_logical_replication_fail(self): + with get_remote_node(conn_params=conn_params) as node: + with self.assertRaises(InitNodeException): + node.init(allow_logical=True) + + def test_replication_slots(self): + with get_remote_node(conn_params=conn_params) as node: + node.init(allow_streaming=True).start() + + with node.replicate(slot='slot1').start() as replica: + replica.execute('select 1') + + # cannot create new slot with the same name + with self.assertRaises(TestgresException): + node.replicate(slot='slot1') + + def test_incorrect_catchup(self): + with get_remote_node(conn_params=conn_params) as node: + node.init(allow_streaming=True).start() + + # node has no master, can't catch up + with self.assertRaises(TestgresException): + node.catchup() + + def test_promotion(self): + with get_remote_node(conn_params=conn_params) as master: + master.init().start() + master.safe_psql('create table abc(id serial)') + + with master.replicate().start() as replica: + master.stop() + replica.promote() + + # make standby becomes writable master + replica.safe_psql('insert into abc values (1)') + res = replica.safe_psql('select * from abc') + self.assertEqual(res, b'1\n') + + def test_dump(self): + query_create = 'create table test as select generate_series(1, 2) as val' + query_select = 'select * from test order by val asc' + + with get_remote_node(conn_params=conn_params).init().start() as node1: + + node1.execute(query_create) + for format in ['plain', 'custom', 'directory', 'tar']: + with removing(node1.dump(format=format)) as dump: + with get_remote_node(conn_params=conn_params).init().start() as node3: + if format == 'directory': + self.assertTrue(os_ops.isdir(dump)) + else: + self.assertTrue(os_ops.isfile(dump)) + # restore dump + node3.restore(filename=dump) + res = node3.execute(query_select) + self.assertListEqual(res, [(1,), (2,)]) + + def test_users(self): + with get_remote_node(conn_params=conn_params).init().start() as node: + node.psql('create role test_user login') + value = node.safe_psql('select 1', username='test_user') + self.assertEqual(b'1\n', value) + + def test_poll_query_until(self): + with get_remote_node(conn_params=conn_params) as node: + node.init().start() + + get_time = 'select extract(epoch from now())' + check_time = 'select extract(epoch from now()) - {} >= 5' + + start_time = node.execute(get_time)[0][0] + node.poll_query_until(query=check_time.format(start_time)) + end_time = node.execute(get_time)[0][0] + + self.assertTrue(end_time - start_time >= 5) + + # check 0 columns + with self.assertRaises(QueryException): + node.poll_query_until( + query='select from pg_catalog.pg_class limit 1') + + # check None, fail + with self.assertRaises(QueryException): + node.poll_query_until(query='create table abc (val int)') + + # check None, ok + node.poll_query_until(query='create table def()', + expected=None) # returns nothing + + # check 0 rows equivalent to expected=None + node.poll_query_until( + query='select * from pg_catalog.pg_class where true = false', + expected=None) + + # check arbitrary expected value, fail + with self.assertRaises(TimeoutException): + node.poll_query_until(query='select 3', + expected=1, + max_attempts=3, + sleep_time=0.01) + + # check arbitrary expected value, ok + node.poll_query_until(query='select 2', expected=2) + + # check timeout + with self.assertRaises(TimeoutException): + node.poll_query_until(query='select 1 > 2', + max_attempts=3, + sleep_time=0.01) + + # check ProgrammingError, fail + with self.assertRaises(testgres.ProgrammingError): + node.poll_query_until(query='dummy1') + + # check ProgrammingError, ok + with self.assertRaises(TimeoutException): + node.poll_query_until(query='dummy2', + max_attempts=3, + sleep_time=0.01, + suppress={testgres.ProgrammingError}) + + # check 1 arg, ok + node.poll_query_until('select true') + + def test_logging(self): + # FAIL + logfile = tempfile.NamedTemporaryFile('w', delete=True) + + log_conf = { + 'version': 1, + 'handlers': { + 'file': { + 'class': 'logging.FileHandler', + 'filename': logfile.name, + 'formatter': 'base_format', + 'level': logging.DEBUG, + }, + }, + 'formatters': { + 'base_format': { + 'format': '%(node)-5s: %(message)s', + }, + }, + 'root': { + 'handlers': ('file',), + 'level': 'DEBUG', + }, + } + + logging.config.dictConfig(log_conf) + + with scoped_config(use_python_logging=True): + node_name = 'master' + + with get_remote_node(name=node_name) as master: + master.init().start() + + # execute a dummy query a few times + for i in range(20): + master.execute('select 1') + time.sleep(0.01) + + # let logging worker do the job + time.sleep(0.1) + + # check that master's port is found + with open(logfile.name, 'r') as log: + lines = log.readlines() + self.assertTrue(any(node_name in s for s in lines)) + + # test logger after stop/start/restart + master.stop() + master.start() + master.restart() + self.assertTrue(master._logger.is_alive()) + + @unittest.skipUnless(util_exists('pgbench'), 'might be missing') + def test_pgbench(self): + with get_remote_node(conn_params=conn_params).init().start() as node: + # initialize pgbench DB and run benchmarks + node.pgbench_init(scale=2, foreign_keys=True, + options=['-q']).pgbench_run(time=2) + + # run TPC-B benchmark + out = node.pgbench(stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=['-T3']) + self.assertTrue(b'tps = ' in out) + + def test_pg_config(self): + # check same instances + a = get_pg_config() + b = get_pg_config() + self.assertEqual(id(a), id(b)) + + # save right before config change + c1 = get_pg_config() + # modify setting for this scope + with scoped_config(cache_pg_config=False) as config: + # sanity check for value + self.assertFalse(config.cache_pg_config) + + # save right after config change + c2 = get_pg_config() + + # check different instances after config change + self.assertNotEqual(id(c1), id(c2)) + + # check different instances + a = get_pg_config() + b = get_pg_config() + self.assertNotEqual(id(a), id(b)) + + def test_config_stack(self): + # no such option + with self.assertRaises(TypeError): + configure_testgres(dummy=True) + + # we have only 1 config in stack + with self.assertRaises(IndexError): + pop_config() + + d0 = TestgresConfig.cached_initdb_dir + d1 = 'dummy_abc' + d2 = 'dummy_def' + + with scoped_config(cached_initdb_dir=d1) as c1: + self.assertEqual(c1.cached_initdb_dir, d1) + + with scoped_config(cached_initdb_dir=d2) as c2: + stack_size = len(testgres.config.config_stack) + + # try to break a stack + with self.assertRaises(TypeError): + with scoped_config(dummy=True): + pass + + self.assertEqual(c2.cached_initdb_dir, d2) + self.assertEqual(len(testgres.config.config_stack), stack_size) + + self.assertEqual(c1.cached_initdb_dir, d1) + + self.assertEqual(TestgresConfig.cached_initdb_dir, d0) + + def test_unix_sockets(self): + with get_remote_node(conn_params=conn_params) as node: + node.init(unix_sockets=False, allow_streaming=True) + node.start() + + res_exec = node.execute('select 1') + res_psql = node.safe_psql('select 1') + self.assertEqual(res_exec, [(1,)]) + self.assertEqual(res_psql, b'1\n') + + with node.replicate().start() as r: + res_exec = r.execute('select 1') + res_psql = r.safe_psql('select 1') + self.assertEqual(res_exec, [(1,)]) + self.assertEqual(res_psql, b'1\n') + + def test_auto_name(self): + with get_remote_node(conn_params=conn_params).init(allow_streaming=True).start() as m: + with m.replicate().start() as r: + # check that nodes are running + self.assertTrue(m.status()) + self.assertTrue(r.status()) + + # check their names + self.assertNotEqual(m.name, r.name) + self.assertTrue('testgres' in m.name) + self.assertTrue('testgres' in r.name) + + def test_file_tail(self): + from testgres.utils import file_tail + + s1 = "the quick brown fox jumped over that lazy dog\n" + s2 = "abc\n" + s3 = "def\n" + + with tempfile.NamedTemporaryFile(mode='r+', delete=True) as f: + sz = 0 + while sz < 3 * 8192: + sz += len(s1) + f.write(s1) + f.write(s2) + f.write(s3) + + f.seek(0) + lines = file_tail(f, 3) + self.assertEqual(lines[0], s1) + self.assertEqual(lines[1], s2) + self.assertEqual(lines[2], s3) + + f.seek(0) + lines = file_tail(f, 1) + self.assertEqual(lines[0], s3) + + def test_isolation_levels(self): + with get_remote_node(conn_params=conn_params).init().start() as node: + with node.connect() as con: + # string levels + con.begin('Read Uncommitted').commit() + con.begin('Read Committed').commit() + con.begin('Repeatable Read').commit() + con.begin('Serializable').commit() + + # enum levels + con.begin(IsolationLevel.ReadUncommitted).commit() + con.begin(IsolationLevel.ReadCommitted).commit() + con.begin(IsolationLevel.RepeatableRead).commit() + con.begin(IsolationLevel.Serializable).commit() + + # check wrong level + with self.assertRaises(QueryException): + con.begin('Garbage').commit() + + def test_ports_management(self): + # check that no ports have been bound yet + self.assertEqual(len(bound_ports), 0) + + with get_remote_node(conn_params=conn_params) as node: + # check that we've just bound a port + self.assertEqual(len(bound_ports), 1) + + # check that bound_ports contains our port + port_1 = list(bound_ports)[0] + port_2 = node.port + self.assertEqual(port_1, port_2) + + # check that port has been freed successfully + self.assertEqual(len(bound_ports), 0) + + def test_exceptions(self): + str(StartNodeException('msg', [('file', 'lines')])) + str(ExecUtilException('msg', 'cmd', 1, 'out')) + str(QueryException('msg', 'query')) + + def test_version_management(self): + a = PgVer('10.0') + b = PgVer('10') + c = PgVer('9.6.5') + d = PgVer('15.0') + e = PgVer('15rc1') + f = PgVer('15beta4') + + self.assertTrue(a == b) + self.assertTrue(b > c) + self.assertTrue(a > c) + self.assertTrue(d > e) + self.assertTrue(e > f) + self.assertTrue(d > f) + + version = get_pg_version() + with get_remote_node(conn_params=conn_params) as node: + self.assertTrue(isinstance(version, six.string_types)) + self.assertTrue(isinstance(node.version, PgVer)) + self.assertEqual(node.version, PgVer(version)) + + def test_child_pids(self): + master_processes = [ + ProcessType.AutovacuumLauncher, + ProcessType.BackgroundWriter, + ProcessType.Checkpointer, + ProcessType.StatsCollector, + ProcessType.WalSender, + ProcessType.WalWriter, + ] + + if pg_version_ge('10'): + master_processes.append(ProcessType.LogicalReplicationLauncher) + + repl_processes = [ + ProcessType.Startup, + ProcessType.WalReceiver, + ] + + with get_remote_node(conn_params=conn_params).init().start() as master: + + # master node doesn't have a source walsender! + with self.assertRaises(TestgresException): + master.source_walsender + + with master.connect() as con: + self.assertGreater(con.pid, 0) + + with master.replicate().start() as replica: + + # test __str__ method + str(master.child_processes[0]) + + master_pids = master.auxiliary_pids + for ptype in master_processes: + self.assertIn(ptype, master_pids) + + replica_pids = replica.auxiliary_pids + for ptype in repl_processes: + self.assertIn(ptype, replica_pids) + + # there should be exactly 1 source walsender for replica + self.assertEqual(len(master_pids[ProcessType.WalSender]), 1) + pid1 = master_pids[ProcessType.WalSender][0] + pid2 = replica.source_walsender.pid + self.assertEqual(pid1, pid2) + + replica.stop() + + # there should be no walsender after we've stopped replica + with self.assertRaises(TestgresException): + replica.source_walsender + + def test_child_process_dies(self): + # test for FileNotFound exception during child_processes() function + with subprocess.Popen(["sleep", "60"]) as process: + self.assertEqual(process.poll(), None) + # collect list of processes currently running + children = psutil.Process(os.getpid()).children() + # kill a process, so received children dictionary becomes invalid + process.kill() + process.wait() + # try to handle children list -- missing processes will have ptype "ProcessType.Unknown" + [ProcessProxy(p) for p in children] + + +if __name__ == '__main__': + if os_ops.environ('ALT_CONFIG'): + suite = unittest.TestSuite() + + # Small subset of tests for alternative configs (PG_BIN or PG_CONFIG) + suite.addTest(TestgresRemoteTests('test_pg_config')) + suite.addTest(TestgresRemoteTests('test_pg_ctl')) + suite.addTest(TestgresRemoteTests('test_psql')) + suite.addTest(TestgresRemoteTests('test_replicate')) + + print('Running tests for alternative config:') + for t in suite: + print(t) + print() + + runner = unittest.TextTestRunner() + runner.run(suite) + else: + unittest.main() From ad40c0f18711df689712ef917e04fede48e3baa2 Mon Sep 17 00:00:00 2001 From: Victoria Shepard <5807469+demonolock@users.noreply.github.com> Date: Tue, 17 Oct 2023 12:45:32 +0200 Subject: [PATCH 257/475] Remove using paramiko (#89) * Remove using paramiko * Up version 1.9.1 --- README.md | 2 +- setup.py | 3 +- testgres/__init__.py | 4 +- testgres/cache.py | 4 +- testgres/operations/local_ops.py | 12 +- testgres/operations/remote_ops.py | 210 +++++++++++++----------------- testgres/utils.py | 6 +- tests/test_remote.py | 3 - tests/test_simple_remote.py | 1 - 9 files changed, 108 insertions(+), 137 deletions(-) diff --git a/README.md b/README.md index 29b974dc..a2a0ec7e 100644 --- a/README.md +++ b/README.md @@ -176,7 +176,7 @@ the configuration file, which means that they should be called before `append_co ### Remote mode Testgres supports the creation of PostgreSQL nodes on a remote host. This is useful when you want to run distributed tests involving multiple nodes spread across different machines. -To use this feature, you need to use the RemoteOperations class. +To use this feature, you need to use the RemoteOperations class. This feature is only supported with Linux. Here is an example of how you might set this up: ```python diff --git a/setup.py b/setup.py index 8cb0f70a..074de8a1 100755 --- a/setup.py +++ b/setup.py @@ -12,7 +12,6 @@ "six>=1.9.0", "psutil", "packaging", - "paramiko", "fabric", "sshtunnel" ] @@ -30,7 +29,7 @@ readme = f.read() setup( - version='1.9.0', + version='1.9.1', name='testgres', packages=['testgres', 'testgres.operations'], description='Testing utility for PostgreSQL and its extensions', diff --git a/testgres/__init__.py b/testgres/__init__.py index b63c7df1..383daf2d 100644 --- a/testgres/__init__.py +++ b/testgres/__init__.py @@ -46,6 +46,8 @@ First, \ Any +from .config import testgres_config + from .operations.os_ops import OsOperations, ConnectionParams from .operations.local_ops import LocalOperations from .operations.remote_ops import RemoteOperations @@ -53,7 +55,7 @@ __all__ = [ "get_new_node", "get_remote_node", - "NodeBackup", + "NodeBackup", "testgres_config", "TestgresConfig", "configure_testgres", "scoped_config", "push_config", "pop_config", "NodeConnection", "DatabaseError", "InternalError", "ProgrammingError", "OperationalError", "TestgresException", "ExecUtilException", "QueryException", "TimeoutException", "CatchUpException", "StartNodeException", "InitNodeException", "BackupException", diff --git a/testgres/cache.py b/testgres/cache.py index bf8658c9..21198e83 100644 --- a/testgres/cache.py +++ b/testgres/cache.py @@ -57,7 +57,9 @@ def call_initdb(initdb_dir, log=logfile): # our initdb caching mechanism breaks this contract. pg_control = os.path.join(data_dir, XLOG_CONTROL_FILE) system_id = generate_system_id() - os_ops.write(pg_control, system_id, truncate=True, binary=True, read_and_write=True) + cur_pg_control = os_ops.read(pg_control, binary=True) + new_pg_control = system_id + cur_pg_control[len(system_id):] + os_ops.write(pg_control, new_pg_control, truncate=True, binary=True, read_and_write=True) # XXX: build new WAL segment with our system id _params = [get_bin_path("pg_resetwal"), "-D", data_dir, "-f"] diff --git a/testgres/operations/local_ops.py b/testgres/operations/local_ops.py index 89071282..318ae675 100644 --- a/testgres/operations/local_ops.py +++ b/testgres/operations/local_ops.py @@ -198,9 +198,15 @@ def touch(self, filename): with open(filename, "a"): os.utime(filename, None) - def read(self, filename, encoding=None): - with open(filename, "r", encoding=encoding) as file: - return file.read() + def read(self, filename, encoding=None, binary=False): + mode = "rb" if binary else "r" + with open(filename, mode) as file: + content = file.read() + if binary: + return content + if isinstance(content, bytes): + return content.decode(encoding or 'utf-8') + return content def readlines(self, filename, num_lines=0, binary=False, encoding=None): """ diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index 6815c7f1..5d9bfe7e 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -1,13 +1,12 @@ +import locale +import logging import os +import subprocess import tempfile import time -from typing import Optional import sshtunnel -import paramiko -from paramiko import SSHClient - from ..exceptions import ExecUtilException from .os_ops import OsOperations, ConnectionParams @@ -16,6 +15,9 @@ sshtunnel.SSH_TIMEOUT = 5.0 sshtunnel.TUNNEL_TIMEOUT = 5.0 +ConsoleEncoding = locale.getdefaultlocale()[1] +if not ConsoleEncoding: + ConsoleEncoding = 'UTF-8' error_markers = [b'error', b'Permission denied', b'fatal', b'No such file or directory'] @@ -31,33 +33,29 @@ def kill(self): def cmdline(self): command = "ps -p {} -o cmd --no-headers".format(self.pid) - stdin, stdout, stderr = self.ssh.exec_command(command) - cmdline = stdout.read().decode('utf-8').strip() + stdin, stdout, stderr = self.ssh.exec_command(command, verbose=True, encoding=ConsoleEncoding) + cmdline = stdout.strip() return cmdline.split() class RemoteOperations(OsOperations): def __init__(self, conn_params: ConnectionParams): + if os.name != "posix": + raise EnvironmentError("Remote operations are supported only on Linux!") + super().__init__(conn_params.username) self.conn_params = conn_params self.host = conn_params.host self.ssh_key = conn_params.ssh_key - self.ssh = self.ssh_connect() self.remote = True self.username = conn_params.username or self.get_user() - self.tunnel = None + self.add_known_host(self.host) def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): self.close_tunnel() - if getattr(self, 'ssh', None): - self.ssh.close() - - def __del__(self): - if getattr(self, 'ssh', None): - self.ssh.close() def close_tunnel(self): if getattr(self, 'tunnel', None): @@ -68,26 +66,17 @@ def close_tunnel(self): break time.sleep(0.5) - def ssh_connect(self) -> Optional[SSHClient]: - key = self._read_ssh_key() - ssh = paramiko.SSHClient() - ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - ssh.connect(self.host, username=self.username, pkey=key) - return ssh - - def _read_ssh_key(self): + def add_known_host(self, host): + cmd = 'ssh-keyscan -H %s >> /home/%s/.ssh/known_hosts' % (host, os.getlogin()) try: - with open(self.ssh_key, "r") as f: - key_data = f.read() - if "BEGIN OPENSSH PRIVATE KEY" in key_data: - key = paramiko.Ed25519Key.from_private_key_file(self.ssh_key) - else: - key = paramiko.RSAKey.from_private_key_file(self.ssh_key) - return key - except FileNotFoundError: - raise ExecUtilException(message="No such file or directory: '{}'".format(self.ssh_key)) - except Exception as e: - ExecUtilException(message="An error occurred while reading the ssh key: {}".format(e)) + subprocess.check_call( + cmd, + shell=True, + ) + logging.info("Successfully added %s to known_hosts." % host) + except subprocess.CalledProcessError as e: + raise ExecUtilException(message="Failed to add %s to known_hosts. Error: %s" % (host, str(e)), command=cmd, + exit_code=e.returncode, out=e.stderr) def exec_command(self, cmd: str, wait_exit=False, verbose=False, expect_error=False, encoding=None, shell=True, text=False, input=None, stdin=None, stdout=None, @@ -97,49 +86,34 @@ def exec_command(self, cmd: str, wait_exit=False, verbose=False, expect_error=Fa Args: - cmd (str): The command to be executed. """ - if self.ssh is None or not self.ssh.get_transport() or not self.ssh.get_transport().is_active(): - self.ssh = self.ssh_connect() - - if isinstance(cmd, list): - cmd = ' '.join(item.decode('utf-8') if isinstance(item, bytes) else item for item in cmd) - if input: - stdin, stdout, stderr = self.ssh.exec_command(cmd) - stdin.write(input) - stdin.flush() - else: - stdin, stdout, stderr = self.ssh.exec_command(cmd) - exit_status = 0 - if wait_exit: - exit_status = stdout.channel.recv_exit_status() + if isinstance(cmd, str): + ssh_cmd = ['ssh', f"{self.username}@{self.host}", '-i', self.ssh_key, cmd] + elif isinstance(cmd, list): + ssh_cmd = ['ssh', f"{self.username}@{self.host}", '-i', self.ssh_key] + cmd + process = subprocess.Popen(ssh_cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + result, error = process.communicate(input) + exit_status = process.returncode if encoding: - result = stdout.read().decode(encoding) - error = stderr.read().decode(encoding) - else: - result = stdout.read() - error = stderr.read() + result = result.decode(encoding) + error = error.decode(encoding) if expect_error: raise Exception(result, error) - if encoding: - error_found = exit_status != 0 or any( - marker.decode(encoding) in error for marker in error_markers) + if not error: + error_found = 0 else: error_found = exit_status != 0 or any( - marker in error for marker in error_markers) + marker in error for marker in [b'error', b'Permission denied', b'fatal', b'No such file or directory']) if error_found: - if exit_status == 0: - exit_status = 1 - if encoding: - message = "Utility exited with non-zero code. Error: {}".format(error.decode(encoding)) - else: + if isinstance(error, bytes): message = b"Utility exited with non-zero code. Error: " + error - raise ExecUtilException(message=message, - command=cmd, - exit_code=exit_status, - out=result) + else: + message = f"Utility exited with non-zero code. Error: {error}" + raise ExecUtilException(message=message, command=cmd, exit_code=exit_status, out=result) if verbose: return exit_status, result, error @@ -154,7 +128,7 @@ def environ(self, var_name: str) -> str: - var_name (str): The name of the environment variable. """ cmd = "echo ${}".format(var_name) - return self.exec_command(cmd, encoding='utf-8').strip() + return self.exec_command(cmd, encoding=ConsoleEncoding).strip() def find_executable(self, executable): search_paths = self.environ("PATH") @@ -185,11 +159,11 @@ def set_env(self, var_name: str, var_val: str): # Get environment variables def get_user(self): - return self.exec_command("echo $USER", encoding='utf-8').strip() + return self.exec_command("echo $USER", encoding=ConsoleEncoding).strip() def get_name(self): cmd = 'python3 -c "import os; print(os.name)"' - return self.exec_command(cmd, encoding='utf-8').strip() + return self.exec_command(cmd, encoding=ConsoleEncoding).strip() # Work with dirs def makedirs(self, path, remove_existing=False): @@ -236,7 +210,7 @@ def listdir(self, path): return result.splitlines() def path_exists(self, path): - result = self.exec_command("test -e {}; echo $?".format(path), encoding='utf-8') + result = self.exec_command("test -e {}; echo $?".format(path), encoding=ConsoleEncoding) return int(result.strip()) == 0 @property @@ -257,22 +231,25 @@ def mkdtemp(self, prefix=None): - prefix (str): The prefix of the temporary directory name. """ if prefix: - temp_dir = self.exec_command("mktemp -d {}XXXXX".format(prefix), encoding='utf-8') + command = ["ssh", "-i", self.ssh_key, f"{self.username}@{self.host}", f"mktemp -d {prefix}XXXXX"] else: - temp_dir = self.exec_command("mktemp -d", encoding='utf-8') + command = ["ssh", "-i", self.ssh_key, f"{self.username}@{self.host}", "mktemp -d"] - if temp_dir: + result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + + if result.returncode == 0: + temp_dir = result.stdout.strip() if not os.path.isabs(temp_dir): - temp_dir = os.path.join('/home', self.username, temp_dir.strip()) + temp_dir = os.path.join('/home', self.username, temp_dir) return temp_dir else: - raise ExecUtilException("Could not create temporary directory.") + raise ExecUtilException(f"Could not create temporary directory. Error: {result.stderr}") def mkstemp(self, prefix=None): if prefix: - temp_dir = self.exec_command("mktemp {}XXXXX".format(prefix), encoding='utf-8') + temp_dir = self.exec_command("mktemp {}XXXXX".format(prefix), encoding=ConsoleEncoding) else: - temp_dir = self.exec_command("mktemp", encoding='utf-8') + temp_dir = self.exec_command("mktemp", encoding=ConsoleEncoding) if temp_dir: if not os.path.isabs(temp_dir): @@ -289,20 +266,7 @@ def copytree(self, src, dst): return self.exec_command("cp -r {} {}".format(src, dst)) # Work with files - def write(self, filename, data, truncate=False, binary=False, read_and_write=False, encoding='utf-8'): - """ - Write data to a file on a remote host - - Args: - - filename (str): The file path where the data will be written. - - data (bytes or str): The data to be written to the file. - - truncate (bool): If True, the file will be truncated before writing ('w' or 'wb' option); - if False (default), data will be appended ('a' or 'ab' option). - - binary (bool): If True, the data will be written in binary mode ('wb' or 'ab' option); - if False (default), the data will be written in text mode ('w' or 'a' option). - - read_and_write (bool): If True, the file will be opened with read and write permissions ('r+' option); - if False (default), only write permission will be used ('w', 'a', 'wb', or 'ab' option). - """ + def write(self, filename, data, truncate=False, binary=False, read_and_write=False, encoding=ConsoleEncoding): mode = "wb" if binary else "w" if not truncate: mode = "ab" if binary else "a" @@ -311,35 +275,29 @@ def write(self, filename, data, truncate=False, binary=False, read_and_write=Fal with tempfile.NamedTemporaryFile(mode=mode, delete=False) as tmp_file: if not truncate: - with self.ssh_connect() as ssh: - sftp = ssh.open_sftp() - try: - sftp.get(filename, tmp_file.name) - tmp_file.seek(0, os.SEEK_END) - except FileNotFoundError: - pass # File does not exist yet, we'll create it - sftp.close() + scp_cmd = ['scp', '-i', self.ssh_key, f"{self.username}@{self.host}:{filename}", tmp_file.name] + subprocess.run(scp_cmd, check=False) # The file might not exist yet + tmp_file.seek(0, os.SEEK_END) + if isinstance(data, bytes) and not binary: data = data.decode(encoding) elif isinstance(data, str) and binary: data = data.encode(encoding) + if isinstance(data, list): - # ensure each line ends with a newline - data = [(s if isinstance(s, str) else s.decode('utf-8')).rstrip('\n') + '\n' for s in data] + data = [(s if isinstance(s, str) else s.decode(ConsoleEncoding)).rstrip('\n') + '\n' for s in data] tmp_file.writelines(data) else: tmp_file.write(data) + tmp_file.flush() - with self.ssh_connect() as ssh: - sftp = ssh.open_sftp() - remote_directory = os.path.dirname(filename) - try: - sftp.stat(remote_directory) - except IOError: - sftp.mkdir(remote_directory) - sftp.put(tmp_file.name, filename) - sftp.close() + scp_cmd = ['scp', '-i', self.ssh_key, tmp_file.name, f"{self.username}@{self.host}:{filename}"] + subprocess.run(scp_cmd, check=True) + + remote_directory = os.path.dirname(filename) + mkdir_cmd = ['ssh', '-i', self.ssh_key, f"{self.username}@{self.host}", f"mkdir -p {remote_directory}"] + subprocess.run(mkdir_cmd, check=True) os.remove(tmp_file.name) @@ -359,7 +317,7 @@ def read(self, filename, binary=False, encoding=None): result = self.exec_command(cmd, encoding=encoding) if not binary and result: - result = result.decode(encoding or 'utf-8') + result = result.decode(encoding or ConsoleEncoding) return result @@ -372,7 +330,7 @@ def readlines(self, filename, num_lines=0, binary=False, encoding=None): result = self.exec_command(cmd, encoding=encoding) if not binary and result: - lines = result.decode(encoding or 'utf-8').splitlines() + lines = result.decode(encoding or ConsoleEncoding).splitlines() else: lines = result.splitlines() @@ -400,13 +358,18 @@ def kill(self, pid, signal): def get_pid(self): # Get current process id - return int(self.exec_command("echo $$", encoding='utf-8')) + return int(self.exec_command("echo $$", encoding=ConsoleEncoding)) def get_process_children(self, pid): - command = "pgrep -P {}".format(pid) - stdin, stdout, stderr = self.ssh.exec_command(command) - children = stdout.readlines() - return [PsUtilProcessProxy(self.ssh, int(child_pid.strip())) for child_pid in children] + command = ["ssh", "-i", self.ssh_key, f"{self.username}@{self.host}", f"pgrep -P {pid}"] + + result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + + if result.returncode == 0: + children = result.stdout.strip().splitlines() + return [PsUtilProcessProxy(self, int(child_pid.strip())) for child_pid in children] + else: + raise ExecUtilException(f"Error in getting process children. Error: {result.stderr}") # Database control def db_connect(self, dbname, user, password=None, host="127.0.0.1", port=5432, ssh_key=None): @@ -424,18 +387,19 @@ def db_connect(self, dbname, user, password=None, host="127.0.0.1", port=5432, s """ self.close_tunnel() self.tunnel = sshtunnel.open_tunnel( - (host, 22), # Remote server IP and SSH port - ssh_username=user or self.username, - ssh_pkey=ssh_key or self.ssh_key, - remote_bind_address=(host, port), # PostgreSQL server IP and PostgreSQL port - local_bind_address=('localhost', port) # Local machine IP and available port + (self.host, 22), # Remote server IP and SSH port + ssh_username=self.username, + ssh_pkey=self.ssh_key, + remote_bind_address=(self.host, port), # PostgreSQL server IP and PostgreSQL port + local_bind_address=('localhost', 0) + # Local machine IP and available port (0 means it will pick any available port) ) - self.tunnel.start() try: + # Use localhost and self.tunnel.local_bind_port to connect conn = pglib.connect( - host=host, # change to 'localhost' because we're connecting through a local ssh tunnel + host='localhost', # Connect to localhost port=self.tunnel.local_bind_port, # use the local bind port set up by the tunnel database=dbname, user=user or self.username, diff --git a/testgres/utils.py b/testgres/utils.py index 5e12eba9..b7df70d1 100644 --- a/testgres/utils.py +++ b/testgres/utils.py @@ -118,11 +118,13 @@ def get_bin_path(filename): return filename -def get_pg_config(pg_config_path=None): +def get_pg_config(pg_config_path=None, os_ops=None): """ Return output of pg_config (provided that it is installed). NOTE: this function caches the result by default (see GlobalConfig). """ + if os_ops: + tconf.os_ops = os_ops def cache_pg_config_data(cmd): # execute pg_config and get the output @@ -146,7 +148,7 @@ def cache_pg_config_data(cmd): _pg_config_data = {} # return cached data - if _pg_config_data: + if not pg_config_path and _pg_config_data: return _pg_config_data # try specified pg_config path or PG_CONFIG diff --git a/tests/test_remote.py b/tests/test_remote.py index 3794349c..2e0f0676 100755 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -17,9 +17,6 @@ def setup(self): 'RDBMS_TESTPOOL_SSHKEY') or '../../container_files/postgres/ssh/id_ed25519') self.operations = RemoteOperations(conn_params) - yield - self.operations.__del__() - def test_exec_command_success(self): """ Test exec_command for successful command execution. diff --git a/tests/test_simple_remote.py b/tests/test_simple_remote.py index e8386383..44e77fbd 100755 --- a/tests/test_simple_remote.py +++ b/tests/test_simple_remote.py @@ -135,7 +135,6 @@ def test_init_after_cleanup(self): @unittest.skipUnless(util_exists('pg_resetwal'), 'might be missing') @unittest.skipUnless(pg_version_ge('9.6'), 'requires 9.6+') def test_init_unique_system_id(self): - # FAIL # this function exists in PostgreSQL 9.6+ query = 'select system_identifier from pg_control_system()' From 46eb92acd92711d48579c53fdc6950231fb04016 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=92=D0=B0=D1=88=C3=90Viktoria=20Shepard?= Date: Tue, 17 Oct 2023 17:18:22 +0200 Subject: [PATCH 258/475] Remove fabric --- setup.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 074de8a1..1012df62 100755 --- a/setup.py +++ b/setup.py @@ -12,7 +12,6 @@ "six>=1.9.0", "psutil", "packaging", - "fabric", "sshtunnel" ] @@ -29,7 +28,7 @@ readme = f.read() setup( - version='1.9.1', + version='1.9.2', name='testgres', packages=['testgres', 'testgres.operations'], description='Testing utility for PostgreSQL and its extensions', From 263ff9c26bb59efb11b032af270206ebf0f02596 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=92=D0=B0=D1=88=C3=90Viktoria=20Shepard?= Date: Tue, 17 Oct 2023 21:36:25 +0200 Subject: [PATCH 259/475] Remove sshtunnel --- setup.py | 3 +- testgres/exceptions.py | 11 +++- testgres/node.py | 2 +- testgres/operations/local_ops.py | 13 +++-- testgres/operations/remote_ops.py | 91 +++++++++++++++---------------- tests/test_simple_remote.py | 7 ++- 6 files changed, 68 insertions(+), 59 deletions(-) diff --git a/setup.py b/setup.py index 1012df62..16d4c300 100755 --- a/setup.py +++ b/setup.py @@ -11,8 +11,7 @@ "port-for>=0.4", "six>=1.9.0", "psutil", - "packaging", - "sshtunnel" + "packaging" ] # Add compatibility enum class diff --git a/testgres/exceptions.py b/testgres/exceptions.py index 6832c788..ee329031 100644 --- a/testgres/exceptions.py +++ b/testgres/exceptions.py @@ -32,7 +32,16 @@ def __str__(self): if self.out: msg.append(u'----\n{}'.format(self.out)) - return six.text_type('\n').join(msg) + return self.convert_and_join(msg) + + @staticmethod + def convert_and_join(msg_list): + # Convert each byte element in the list to str + str_list = [six.text_type(item, 'utf-8') if isinstance(item, bytes) else six.text_type(item) for item in + msg_list] + + # Join the list into a single string with the specified delimiter + return six.text_type('\n').join(str_list) @six.python_2_unicode_compatible diff --git a/testgres/node.py b/testgres/node.py index 6483514b..84c25327 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -1371,7 +1371,7 @@ def pgbench(self, # should be the last one _params.append(dbname) - proc = self.os_ops.exec_command(_params, stdout=stdout, stderr=stderr, wait_exit=True, proc=True) + proc = self.os_ops.exec_command(_params, stdout=stdout, stderr=stderr, wait_exit=True, get_process=True) return proc diff --git a/testgres/operations/local_ops.py b/testgres/operations/local_ops.py index 318ae675..a692750e 100644 --- a/testgres/operations/local_ops.py +++ b/testgres/operations/local_ops.py @@ -18,7 +18,6 @@ from distutils.spawn import find_executable from distutils import rmtree - CMD_TIMEOUT_SEC = 60 error_markers = [b'error', b'Permission denied', b'fatal'] @@ -37,7 +36,8 @@ def __init__(self, conn_params=None): # Command execution def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False, encoding=None, shell=False, text=False, - input=None, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, proc=None): + input=None, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + get_process=None, timeout=None): """ Execute a command in a subprocess. @@ -69,9 +69,14 @@ def exec_command(self, cmd, wait_exit=False, verbose=False, stdout=stdout, stderr=stderr, ) - if proc: + if get_process: return process - result, error = process.communicate(input) + + try: + result, error = process.communicate(input, timeout=timeout) + except subprocess.TimeoutExpired: + process.kill() + raise ExecUtilException("Command timed out after {} seconds.".format(timeout)) exit_status = process.returncode error_found = exit_status != 0 or any(marker in error for marker in error_markers) diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index 5d9bfe7e..421c0a6d 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -3,17 +3,19 @@ import os import subprocess import tempfile -import time -import sshtunnel +# we support both pg8000 and psycopg2 +try: + import psycopg2 as pglib +except ImportError: + try: + import pg8000 as pglib + except ImportError: + raise ImportError("You must have psycopg2 or pg8000 modules installed") from ..exceptions import ExecUtilException from .os_ops import OsOperations, ConnectionParams -from .os_ops import pglib - -sshtunnel.SSH_TIMEOUT = 5.0 -sshtunnel.TUNNEL_TIMEOUT = 5.0 ConsoleEncoding = locale.getdefaultlocale()[1] if not ConsoleEncoding: @@ -50,21 +52,28 @@ def __init__(self, conn_params: ConnectionParams): self.remote = True self.username = conn_params.username or self.get_user() self.add_known_host(self.host) + self.tunnel_process = None def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): - self.close_tunnel() + self.close_ssh_tunnel() - def close_tunnel(self): - if getattr(self, 'tunnel', None): - self.tunnel.stop(force=True) - start_time = time.time() - while self.tunnel.is_active: - if time.time() - start_time > sshtunnel.TUNNEL_TIMEOUT: - break - time.sleep(0.5) + def establish_ssh_tunnel(self, local_port, remote_port): + """ + Establish an SSH tunnel from a local port to a remote PostgreSQL port. + """ + ssh_cmd = ['-N', '-L', f"{local_port}:localhost:{remote_port}"] + self.tunnel_process = self.exec_command(ssh_cmd, get_process=True, timeout=300) + + def close_ssh_tunnel(self): + if hasattr(self, 'tunnel_process'): + self.tunnel_process.terminate() + self.tunnel_process.wait() + del self.tunnel_process + else: + print("No active tunnel to close.") def add_known_host(self, host): cmd = 'ssh-keyscan -H %s >> /home/%s/.ssh/known_hosts' % (host, os.getlogin()) @@ -78,21 +87,29 @@ def add_known_host(self, host): raise ExecUtilException(message="Failed to add %s to known_hosts. Error: %s" % (host, str(e)), command=cmd, exit_code=e.returncode, out=e.stderr) - def exec_command(self, cmd: str, wait_exit=False, verbose=False, expect_error=False, + def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False, encoding=None, shell=True, text=False, input=None, stdin=None, stdout=None, - stderr=None, proc=None): + stderr=None, get_process=None, timeout=None): """ Execute a command in the SSH session. Args: - cmd (str): The command to be executed. """ + ssh_cmd = [] if isinstance(cmd, str): ssh_cmd = ['ssh', f"{self.username}@{self.host}", '-i', self.ssh_key, cmd] elif isinstance(cmd, list): ssh_cmd = ['ssh', f"{self.username}@{self.host}", '-i', self.ssh_key] + cmd process = subprocess.Popen(ssh_cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if get_process: + return process + + try: + result, error = process.communicate(input, timeout=timeout) + except subprocess.TimeoutExpired: + process.kill() + raise ExecUtilException("Command timed out after {} seconds.".format(timeout)) - result, error = process.communicate(input) exit_status = process.returncode if encoding: @@ -372,41 +389,19 @@ def get_process_children(self, pid): raise ExecUtilException(f"Error in getting process children. Error: {result.stderr}") # Database control - def db_connect(self, dbname, user, password=None, host="127.0.0.1", port=5432, ssh_key=None): + def db_connect(self, dbname, user, password=None, host="localhost", port=5432): """ - Connects to a PostgreSQL database on the remote system. - Args: - - dbname (str): The name of the database to connect to. - - user (str): The username for the database connection. - - password (str, optional): The password for the database connection. Defaults to None. - - host (str, optional): The IP address of the remote system. Defaults to "localhost". - - port (int, optional): The port number of the PostgreSQL service. Defaults to 5432. - - This function establishes a connection to a PostgreSQL database on the remote system using the specified - parameters. It returns a connection object that can be used to interact with the database. + Established SSH tunnel and Connects to a PostgreSQL """ - self.close_tunnel() - self.tunnel = sshtunnel.open_tunnel( - (self.host, 22), # Remote server IP and SSH port - ssh_username=self.username, - ssh_pkey=self.ssh_key, - remote_bind_address=(self.host, port), # PostgreSQL server IP and PostgreSQL port - local_bind_address=('localhost', 0) - # Local machine IP and available port (0 means it will pick any available port) - ) - self.tunnel.start() - + self.establish_ssh_tunnel(local_port=port, remote_port=5432) try: - # Use localhost and self.tunnel.local_bind_port to connect conn = pglib.connect( - host='localhost', # Connect to localhost - port=self.tunnel.local_bind_port, # use the local bind port set up by the tunnel + host=host, + port=port, database=dbname, - user=user or self.username, - password=password + user=user, + password=password, ) - return conn except Exception as e: - self.tunnel.stop() - raise ExecUtilException("Could not create db tunnel. {}".format(e)) + raise Exception(f"Could not connect to the database. Error: {e}") diff --git a/tests/test_simple_remote.py b/tests/test_simple_remote.py index 44e77fbd..1042f3c4 100755 --- a/tests/test_simple_remote.py +++ b/tests/test_simple_remote.py @@ -735,9 +735,10 @@ def test_pgbench(self): options=['-q']).pgbench_run(time=2) # run TPC-B benchmark - out = node.pgbench(stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - options=['-T3']) + proc = node.pgbench(stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=['-T3']) + out = proc.communicate()[0] self.assertTrue(b'tps = ' in out) def test_pg_config(self): From fd11545624381d1d868ea76c01b439a1754360f4 Mon Sep 17 00:00:00 2001 From: Viktoriia Shepard Date: Fri, 27 Oct 2023 23:00:54 +0200 Subject: [PATCH 260/475] Change setting user in start_node --- testgres/node.py | 4 +++- testgres/operations/local_ops.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index 84c25327..52e6d2ee 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -659,7 +659,7 @@ def get_control_data(self): return out_dict - def slow_start(self, replica=False, dbname='template1', username=default_username(), max_attempts=0): + def slow_start(self, replica=False, dbname='template1', username=None, max_attempts=0): """ Starts the PostgreSQL instance and then polls the instance until it reaches the expected state (primary or replica). The state is checked @@ -672,6 +672,8 @@ def slow_start(self, replica=False, dbname='template1', username=default_usernam If False, waits for the instance to be in primary mode. Default is False. max_attempts: """ + if not username: + username = default_username() self.start() if replica: diff --git a/testgres/operations/local_ops.py b/testgres/operations/local_ops.py index a692750e..36b14058 100644 --- a/testgres/operations/local_ops.py +++ b/testgres/operations/local_ops.py @@ -117,7 +117,7 @@ def set_env(self, var_name, var_val): # Get environment variables def get_user(self): - return getpass.getuser() + return self.username or getpass.getuser() def get_name(self): return os.name From eafa7f0da736a7dcf9362845db769905efbef306 Mon Sep 17 00:00:00 2001 From: vshepard Date: Thu, 9 Nov 2023 21:11:45 +0100 Subject: [PATCH 261/475] problem with get_pg_version in Homebrew Fix issue 87 --- testgres/utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/testgres/utils.py b/testgres/utils.py index b7df70d1..db75fadc 100644 --- a/testgres/utils.py +++ b/testgres/utils.py @@ -179,6 +179,9 @@ def get_pg_version(): _params = [get_bin_path('postgres'), '--version'] raw_ver = tconf.os_ops.exec_command(_params, encoding='utf-8') + # Remove "(Homebrew)" if present + raw_ver = raw_ver.replace('(Homebrew)', '').strip() + # cook version of PostgreSQL version = raw_ver.strip().split(' ')[-1] \ .partition('devel')[0] \ From 4200b8096f198a36d8bf109d1b8da679b66322ad Mon Sep 17 00:00:00 2001 From: vshepard Date: Wed, 22 Nov 2023 09:34:13 +0100 Subject: [PATCH 262/475] RemoteOperations add_known_host macos fix --- testgres/operations/remote_ops.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index 421c0a6d..0a545834 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -3,6 +3,7 @@ import os import subprocess import tempfile +import platform # we support both pg8000 and psycopg2 try: @@ -42,7 +43,8 @@ def cmdline(self): class RemoteOperations(OsOperations): def __init__(self, conn_params: ConnectionParams): - if os.name != "posix": + + if not platform.system().lower() == "linux": raise EnvironmentError("Remote operations are supported only on Linux!") super().__init__(conn_params.username) @@ -76,16 +78,14 @@ def close_ssh_tunnel(self): print("No active tunnel to close.") def add_known_host(self, host): - cmd = 'ssh-keyscan -H %s >> /home/%s/.ssh/known_hosts' % (host, os.getlogin()) + known_hosts_path = os.path.expanduser("~/.ssh/known_hosts") + cmd = 'ssh-keyscan -H %s >> %s' % (host, known_hosts_path) + try: - subprocess.check_call( - cmd, - shell=True, - ) + subprocess.check_call(cmd, shell=True) logging.info("Successfully added %s to known_hosts." % host) except subprocess.CalledProcessError as e: - raise ExecUtilException(message="Failed to add %s to known_hosts. Error: %s" % (host, str(e)), command=cmd, - exit_code=e.returncode, out=e.stderr) + raise Exception("Failed to add %s to known_hosts. Error: %s" % (host, str(e))) def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False, encoding=None, shell=True, text=False, input=None, stdin=None, stdout=None, From 1a2f6dad098a1963d6ef82c707cc26f4c5fadc55 Mon Sep 17 00:00:00 2001 From: Victoria Shepard <5807469+demonolock@users.noreply.github.com> Date: Tue, 19 Dec 2023 17:05:14 +0100 Subject: [PATCH 263/475] Fix initdb error on Windows (#99) --- setup.py | 8 +- testgres/__init__.py | 4 +- testgres/helpers/__init__.py | 0 testgres/helpers/port_manager.py | 40 +++++++++ testgres/node.py | 8 +- testgres/operations/local_ops.py | 133 +++++++++++++++++------------- testgres/operations/os_ops.py | 8 +- testgres/operations/remote_ops.py | 55 ++++++------ testgres/utils.py | 10 ++- tests/test_remote.py | 9 +- tests/test_simple.py | 47 ++++++++--- tests/test_simple_remote.py | 7 +- 12 files changed, 207 insertions(+), 122 deletions(-) create mode 100644 testgres/helpers/__init__.py create mode 100644 testgres/helpers/port_manager.py mode change 100755 => 100644 tests/test_simple.py diff --git a/setup.py b/setup.py index 16d4c300..e0287659 100755 --- a/setup.py +++ b/setup.py @@ -27,16 +27,16 @@ readme = f.read() setup( - version='1.9.2', + version='1.9.3', name='testgres', - packages=['testgres', 'testgres.operations'], + packages=['testgres', 'testgres.operations', 'testgres.helpers'], description='Testing utility for PostgreSQL and its extensions', url='https://github.com/postgrespro/testgres', long_description=readme, long_description_content_type='text/markdown', license='PostgreSQL', - author='Ildar Musin', - author_email='zildermann@gmail.com', + author='Postgres Professional', + author_email='testgres@postgrespro.ru', keywords=['test', 'testing', 'postgresql'], install_requires=install_requires, classifiers=[], diff --git a/testgres/__init__.py b/testgres/__init__.py index 383daf2d..8d0e38c6 100644 --- a/testgres/__init__.py +++ b/testgres/__init__.py @@ -52,6 +52,8 @@ from .operations.local_ops import LocalOperations from .operations.remote_ops import RemoteOperations +from .helpers.port_manager import PortManager + __all__ = [ "get_new_node", "get_remote_node", @@ -62,6 +64,6 @@ "XLogMethod", "IsolationLevel", "NodeStatus", "ProcessType", "DumpFormat", "PostgresNode", "NodeApp", "reserve_port", "release_port", "bound_ports", "get_bin_path", "get_pg_config", "get_pg_version", - "First", "Any", + "First", "Any", "PortManager", "OsOperations", "LocalOperations", "RemoteOperations", "ConnectionParams" ] diff --git a/testgres/helpers/__init__.py b/testgres/helpers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/testgres/helpers/port_manager.py b/testgres/helpers/port_manager.py new file mode 100644 index 00000000..6afdf8a9 --- /dev/null +++ b/testgres/helpers/port_manager.py @@ -0,0 +1,40 @@ +import socket +import random +from typing import Set, Iterable, Optional + + +class PortForException(Exception): + pass + + +class PortManager: + def __init__(self, ports_range=(1024, 65535)): + self.ports_range = ports_range + + @staticmethod + def is_port_free(port: int) -> bool: + """Check if a port is free to use.""" + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + try: + s.bind(("", port)) + return True + except OSError: + return False + + def find_free_port(self, ports: Optional[Set[int]] = None, exclude_ports: Optional[Iterable[int]] = None) -> int: + """Return a random unused port number.""" + if ports is None: + ports = set(range(1024, 65535)) + + if exclude_ports is None: + exclude_ports = set() + + ports.difference_update(set(exclude_ports)) + + sampled_ports = random.sample(tuple(ports), min(len(ports), 100)) + + for port in sampled_ports: + if self.is_port_free(port): + return port + + raise PortForException("Can't select a port") diff --git a/testgres/node.py b/testgres/node.py index 52e6d2ee..20cf4264 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -623,8 +623,8 @@ def status(self): "-D", self.data_dir, "status" ] # yapf: disable - status_code, out, err = execute_utility(_params, self.utils_log_file, verbose=True) - if 'does not exist' in err: + status_code, out, error = execute_utility(_params, self.utils_log_file, verbose=True) + if error and 'does not exist' in error: return NodeStatus.Uninitialized elif 'no server running' in out: return NodeStatus.Stopped @@ -717,7 +717,7 @@ def start(self, params=[], wait=True): try: exit_status, out, error = execute_utility(_params, self.utils_log_file, verbose=True) - if 'does not exist' in error: + if error and 'does not exist' in error: raise Exception except Exception as e: msg = 'Cannot start node' @@ -791,7 +791,7 @@ def restart(self, params=[]): try: error_code, out, error = execute_utility(_params, self.utils_log_file, verbose=True) - if 'could not start server' in error: + if error and 'could not start server' in error: raise ExecUtilException except ExecUtilException as e: msg = 'Cannot restart node' diff --git a/testgres/operations/local_ops.py b/testgres/operations/local_ops.py index 36b14058..93ebf012 100644 --- a/testgres/operations/local_ops.py +++ b/testgres/operations/local_ops.py @@ -8,8 +8,7 @@ import psutil from ..exceptions import ExecUtilException -from .os_ops import ConnectionParams, OsOperations -from .os_ops import pglib +from .os_ops import ConnectionParams, OsOperations, pglib, get_default_encoding try: from shutil import which as find_executable @@ -22,6 +21,14 @@ error_markers = [b'error', b'Permission denied', b'fatal'] +def has_errors(output): + if output: + if isinstance(output, str): + output = output.encode(get_default_encoding()) + return any(marker in output for marker in error_markers) + return False + + class LocalOperations(OsOperations): def __init__(self, conn_params=None): if conn_params is None: @@ -33,72 +40,80 @@ def __init__(self, conn_params=None): self.remote = False self.username = conn_params.username or self.get_user() - # Command execution - def exec_command(self, cmd, wait_exit=False, verbose=False, - expect_error=False, encoding=None, shell=False, text=False, - input=None, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - get_process=None, timeout=None): - """ - Execute a command in a subprocess. - - Args: - - cmd: The command to execute. - - wait_exit: Whether to wait for the subprocess to exit before returning. - - verbose: Whether to return verbose output. - - expect_error: Whether to raise an error if the subprocess exits with an error status. - - encoding: The encoding to use for decoding the subprocess output. - - shell: Whether to use shell when executing the subprocess. - - text: Whether to return str instead of bytes for the subprocess output. - - input: The input to pass to the subprocess. - - stdout: The stdout to use for the subprocess. - - stderr: The stderr to use for the subprocess. - - proc: The process to use for subprocess creation. - :return: The output of the subprocess. - """ - if os.name == 'nt': - with tempfile.NamedTemporaryFile() as buf: - process = subprocess.Popen(cmd, stdout=buf, stderr=subprocess.STDOUT) - process.communicate() - buf.seek(0) - result = buf.read().decode(encoding) - return result - else: + @staticmethod + def _raise_exec_exception(message, command, exit_code, output): + """Raise an ExecUtilException.""" + raise ExecUtilException(message=message.format(output), + command=command, + exit_code=exit_code, + out=output) + + @staticmethod + def _process_output(encoding, temp_file_path): + """Process the output of a command from a temporary file.""" + with open(temp_file_path, 'rb') as temp_file: + output = temp_file.read() + if encoding: + output = output.decode(encoding) + return output, None # In Windows stderr writing in stdout + + def _run_command(self, cmd, shell, input, stdin, stdout, stderr, get_process, timeout, encoding): + """Execute a command and return the process and its output.""" + if os.name == 'nt' and stdout is None: # Windows + with tempfile.NamedTemporaryFile(mode='w+b', delete=False) as temp_file: + stdout = temp_file + stderr = subprocess.STDOUT + process = subprocess.Popen( + cmd, + shell=shell, + stdin=stdin or subprocess.PIPE if input is not None else None, + stdout=stdout, + stderr=stderr, + ) + if get_process: + return process, None, None + temp_file_path = temp_file.name + + # Wait process finished + process.wait() + + output, error = self._process_output(encoding, temp_file_path) + return process, output, error + else: # Other OS process = subprocess.Popen( cmd, shell=shell, - stdout=stdout, - stderr=stderr, + stdin=stdin or subprocess.PIPE if input is not None else None, + stdout=stdout or subprocess.PIPE, + stderr=stderr or subprocess.PIPE, ) if get_process: - return process - + return process, None, None try: - result, error = process.communicate(input, timeout=timeout) + output, error = process.communicate(input=input.encode(encoding) if input else None, timeout=timeout) + if encoding: + output = output.decode(encoding) + error = error.decode(encoding) + return process, output, error except subprocess.TimeoutExpired: process.kill() raise ExecUtilException("Command timed out after {} seconds.".format(timeout)) - exit_status = process.returncode - - error_found = exit_status != 0 or any(marker in error for marker in error_markers) - if encoding: - result = result.decode(encoding) - error = error.decode(encoding) - - if expect_error: - raise Exception(result, error) - - if exit_status != 0 or error_found: - if exit_status == 0: - exit_status = 1 - raise ExecUtilException(message='Utility exited with non-zero code. Error `{}`'.format(error), - command=cmd, - exit_code=exit_status, - out=result) - if verbose: - return exit_status, result, error - else: - return result + def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False, encoding=None, shell=False, + text=False, input=None, stdin=None, stdout=None, stderr=None, get_process=False, timeout=None): + """ + Execute a command in a subprocess and handle the output based on the provided parameters. + """ + process, output, error = self._run_command(cmd, shell, input, stdin, stdout, stderr, get_process, timeout, encoding) + if get_process: + return process + if process.returncode != 0 or (has_errors(error) and not expect_error): + self._raise_exec_exception('Utility exited with non-zero code. Error `{}`', cmd, process.returncode, error) + + if verbose: + return process.returncode, output, error + else: + return output # Environment setup def environ(self, var_name): @@ -210,7 +225,7 @@ def read(self, filename, encoding=None, binary=False): if binary: return content if isinstance(content, bytes): - return content.decode(encoding or 'utf-8') + return content.decode(encoding or get_default_encoding()) return content def readlines(self, filename, num_lines=0, binary=False, encoding=None): diff --git a/testgres/operations/os_ops.py b/testgres/operations/os_ops.py index 9261cacf..dd6613cf 100644 --- a/testgres/operations/os_ops.py +++ b/testgres/operations/os_ops.py @@ -1,3 +1,5 @@ +import locale + try: import psycopg2 as pglib # noqa: F401 except ImportError: @@ -14,6 +16,10 @@ def __init__(self, host='127.0.0.1', ssh_key=None, username=None): self.username = username +def get_default_encoding(): + return locale.getdefaultlocale()[1] or 'UTF-8' + + class OsOperations: def __init__(self, username=None): self.ssh_key = None @@ -75,7 +81,7 @@ def write(self, filename, data, truncate=False, binary=False, read_and_write=Fal def touch(self, filename): raise NotImplementedError() - def read(self, filename): + def read(self, filename, encoding, binary): raise NotImplementedError() def readlines(self, filename): diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index 0a545834..01251e1c 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -1,4 +1,3 @@ -import locale import logging import os import subprocess @@ -15,12 +14,7 @@ raise ImportError("You must have psycopg2 or pg8000 modules installed") from ..exceptions import ExecUtilException - -from .os_ops import OsOperations, ConnectionParams - -ConsoleEncoding = locale.getdefaultlocale()[1] -if not ConsoleEncoding: - ConsoleEncoding = 'UTF-8' +from .os_ops import OsOperations, ConnectionParams, get_default_encoding error_markers = [b'error', b'Permission denied', b'fatal', b'No such file or directory'] @@ -36,7 +30,7 @@ def kill(self): def cmdline(self): command = "ps -p {} -o cmd --no-headers".format(self.pid) - stdin, stdout, stderr = self.ssh.exec_command(command, verbose=True, encoding=ConsoleEncoding) + stdin, stdout, stderr = self.ssh.exec_command(command, verbose=True, encoding=get_default_encoding()) cmdline = stdout.strip() return cmdline.split() @@ -51,6 +45,10 @@ def __init__(self, conn_params: ConnectionParams): self.conn_params = conn_params self.host = conn_params.host self.ssh_key = conn_params.ssh_key + if self.ssh_key: + self.ssh_cmd = ["-i", self.ssh_key] + else: + self.ssh_cmd = [] self.remote = True self.username = conn_params.username or self.get_user() self.add_known_host(self.host) @@ -97,9 +95,9 @@ def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False, """ ssh_cmd = [] if isinstance(cmd, str): - ssh_cmd = ['ssh', f"{self.username}@{self.host}", '-i', self.ssh_key, cmd] + ssh_cmd = ['ssh', f"{self.username}@{self.host}"] + self.ssh_cmd + [cmd] elif isinstance(cmd, list): - ssh_cmd = ['ssh', f"{self.username}@{self.host}", '-i', self.ssh_key] + cmd + ssh_cmd = ['ssh', f"{self.username}@{self.host}"] + self.ssh_cmd + cmd process = subprocess.Popen(ssh_cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) if get_process: return process @@ -145,7 +143,7 @@ def environ(self, var_name: str) -> str: - var_name (str): The name of the environment variable. """ cmd = "echo ${}".format(var_name) - return self.exec_command(cmd, encoding=ConsoleEncoding).strip() + return self.exec_command(cmd, encoding=get_default_encoding()).strip() def find_executable(self, executable): search_paths = self.environ("PATH") @@ -176,11 +174,11 @@ def set_env(self, var_name: str, var_val: str): # Get environment variables def get_user(self): - return self.exec_command("echo $USER", encoding=ConsoleEncoding).strip() + return self.exec_command("echo $USER", encoding=get_default_encoding()).strip() def get_name(self): cmd = 'python3 -c "import os; print(os.name)"' - return self.exec_command(cmd, encoding=ConsoleEncoding).strip() + return self.exec_command(cmd, encoding=get_default_encoding()).strip() # Work with dirs def makedirs(self, path, remove_existing=False): @@ -227,7 +225,7 @@ def listdir(self, path): return result.splitlines() def path_exists(self, path): - result = self.exec_command("test -e {}; echo $?".format(path), encoding=ConsoleEncoding) + result = self.exec_command("test -e {}; echo $?".format(path), encoding=get_default_encoding()) return int(result.strip()) == 0 @property @@ -248,9 +246,9 @@ def mkdtemp(self, prefix=None): - prefix (str): The prefix of the temporary directory name. """ if prefix: - command = ["ssh", "-i", self.ssh_key, f"{self.username}@{self.host}", f"mktemp -d {prefix}XXXXX"] + command = ["ssh"] + self.ssh_cmd + [f"{self.username}@{self.host}", f"mktemp -d {prefix}XXXXX"] else: - command = ["ssh", "-i", self.ssh_key, f"{self.username}@{self.host}", "mktemp -d"] + command = ["ssh"] + self.ssh_cmd + [f"{self.username}@{self.host}", "mktemp -d"] result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) @@ -264,9 +262,9 @@ def mkdtemp(self, prefix=None): def mkstemp(self, prefix=None): if prefix: - temp_dir = self.exec_command("mktemp {}XXXXX".format(prefix), encoding=ConsoleEncoding) + temp_dir = self.exec_command("mktemp {}XXXXX".format(prefix), encoding=get_default_encoding()) else: - temp_dir = self.exec_command("mktemp", encoding=ConsoleEncoding) + temp_dir = self.exec_command("mktemp", encoding=get_default_encoding()) if temp_dir: if not os.path.isabs(temp_dir): @@ -283,7 +281,9 @@ def copytree(self, src, dst): return self.exec_command("cp -r {} {}".format(src, dst)) # Work with files - def write(self, filename, data, truncate=False, binary=False, read_and_write=False, encoding=ConsoleEncoding): + def write(self, filename, data, truncate=False, binary=False, read_and_write=False, encoding=None): + if not encoding: + encoding = get_default_encoding() mode = "wb" if binary else "w" if not truncate: mode = "ab" if binary else "a" @@ -292,7 +292,7 @@ def write(self, filename, data, truncate=False, binary=False, read_and_write=Fal with tempfile.NamedTemporaryFile(mode=mode, delete=False) as tmp_file: if not truncate: - scp_cmd = ['scp', '-i', self.ssh_key, f"{self.username}@{self.host}:{filename}", tmp_file.name] + scp_cmd = ['scp'] + self.ssh_cmd + [f"{self.username}@{self.host}:{filename}", tmp_file.name] subprocess.run(scp_cmd, check=False) # The file might not exist yet tmp_file.seek(0, os.SEEK_END) @@ -302,18 +302,17 @@ def write(self, filename, data, truncate=False, binary=False, read_and_write=Fal data = data.encode(encoding) if isinstance(data, list): - data = [(s if isinstance(s, str) else s.decode(ConsoleEncoding)).rstrip('\n') + '\n' for s in data] + data = [(s if isinstance(s, str) else s.decode(get_default_encoding())).rstrip('\n') + '\n' for s in data] tmp_file.writelines(data) else: tmp_file.write(data) tmp_file.flush() - - scp_cmd = ['scp', '-i', self.ssh_key, tmp_file.name, f"{self.username}@{self.host}:{filename}"] + scp_cmd = ['scp'] + self.ssh_cmd + [tmp_file.name, f"{self.username}@{self.host}:{filename}"] subprocess.run(scp_cmd, check=True) remote_directory = os.path.dirname(filename) - mkdir_cmd = ['ssh', '-i', self.ssh_key, f"{self.username}@{self.host}", f"mkdir -p {remote_directory}"] + mkdir_cmd = ['ssh'] + self.ssh_cmd + [f"{self.username}@{self.host}", f"mkdir -p {remote_directory}"] subprocess.run(mkdir_cmd, check=True) os.remove(tmp_file.name) @@ -334,7 +333,7 @@ def read(self, filename, binary=False, encoding=None): result = self.exec_command(cmd, encoding=encoding) if not binary and result: - result = result.decode(encoding or ConsoleEncoding) + result = result.decode(encoding or get_default_encoding()) return result @@ -347,7 +346,7 @@ def readlines(self, filename, num_lines=0, binary=False, encoding=None): result = self.exec_command(cmd, encoding=encoding) if not binary and result: - lines = result.decode(encoding or ConsoleEncoding).splitlines() + lines = result.decode(encoding or get_default_encoding()).splitlines() else: lines = result.splitlines() @@ -375,10 +374,10 @@ def kill(self, pid, signal): def get_pid(self): # Get current process id - return int(self.exec_command("echo $$", encoding=ConsoleEncoding)) + return int(self.exec_command("echo $$", encoding=get_default_encoding())) def get_process_children(self, pid): - command = ["ssh", "-i", self.ssh_key, f"{self.username}@{self.host}", f"pgrep -P {pid}"] + command = ["ssh"] + self.ssh_cmd + [f"{self.username}@{self.host}", f"pgrep -P {pid}"] result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) diff --git a/testgres/utils.py b/testgres/utils.py index db75fadc..b21fc2c8 100644 --- a/testgres/utils.py +++ b/testgres/utils.py @@ -4,7 +4,7 @@ from __future__ import print_function import os -import port_for + import sys from contextlib import contextmanager @@ -13,6 +13,7 @@ from six import iteritems +from .helpers.port_manager import PortManager from .exceptions import ExecUtilException from .config import testgres_config as tconf @@ -37,8 +38,8 @@ def reserve_port(): """ Generate a new port and add it to 'bound_ports'. """ - - port = port_for.select_random(exclude_ports=bound_ports) + port_mng = PortManager() + port = port_mng.find_free_port(exclude_ports=bound_ports) bound_ports.add(port) return port @@ -80,7 +81,8 @@ def execute_utility(args, logfile=None, verbose=False): lines = [u'\n'] + ['# ' + line for line in out.splitlines()] + [u'\n'] tconf.os_ops.write(filename=logfile, data=lines) except IOError: - raise ExecUtilException("Problem with writing to logfile `{}` during run command `{}`".format(logfile, args)) + raise ExecUtilException( + "Problem with writing to logfile `{}` during run command `{}`".format(logfile, args)) if verbose: return exit_status, out, error else: diff --git a/tests/test_remote.py b/tests/test_remote.py index 2e0f0676..e0e4a555 100755 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -11,10 +11,9 @@ class TestRemoteOperations: @pytest.fixture(scope="function", autouse=True) def setup(self): - conn_params = ConnectionParams(host=os.getenv('RDBMS_TESTPOOL1_HOST') or '172.18.0.3', - username='dev', - ssh_key=os.getenv( - 'RDBMS_TESTPOOL_SSHKEY') or '../../container_files/postgres/ssh/id_ed25519') + conn_params = ConnectionParams(host=os.getenv('RDBMS_TESTPOOL1_HOST') or '127.0.0.1', + username=os.getenv('USER'), + ssh_key=os.getenv('RDBMS_TESTPOOL_SSHKEY')) self.operations = RemoteOperations(conn_params) def test_exec_command_success(self): @@ -41,7 +40,7 @@ def test_is_executable_true(self): """ Test is_executable for an existing executable. """ - cmd = "postgres" + cmd = os.getenv('PG_CONFIG') response = self.operations.is_executable(cmd) assert response is True diff --git a/tests/test_simple.py b/tests/test_simple.py old mode 100755 new mode 100644 index 45c28a21..9d31d4d9 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -74,6 +74,24 @@ def good_properties(f): return True +def rm_carriage_returns(out): + """ + In Windows we have additional '\r' symbols in output. + Let's get rid of them. + """ + if os.name == 'nt': + if isinstance(out, (int, float, complex)): + return out + elif isinstance(out, tuple): + return tuple(rm_carriage_returns(item) for item in out) + elif isinstance(out, bytes): + return out.replace(b'\r', b'') + else: + return out.replace('\r', '') + else: + return out + + @contextmanager def removing(f): try: @@ -123,7 +141,7 @@ def test_init_after_cleanup(self): node.cleanup() node.init().start().execute('select 1') - @unittest.skipUnless(util_exists('pg_resetwal'), 'might be missing') + @unittest.skipUnless(util_exists('pg_resetwal.exe' if os.name == 'nt' else 'pg_resetwal'), 'pgbench might be missing') @unittest.skipUnless(pg_version_ge('9.6'), 'requires 9.6+') def test_init_unique_system_id(self): # this function exists in PostgreSQL 9.6+ @@ -254,34 +272,34 @@ def test_psql(self): # check returned values (1 arg) res = node.psql('select 1') - self.assertEqual(res, (0, b'1\n', b'')) + self.assertEqual(rm_carriage_returns(res), (0, b'1\n', b'')) # check returned values (2 args) res = node.psql('postgres', 'select 2') - self.assertEqual(res, (0, b'2\n', b'')) + self.assertEqual(rm_carriage_returns(res), (0, b'2\n', b'')) # check returned values (named) res = node.psql(query='select 3', dbname='postgres') - self.assertEqual(res, (0, b'3\n', b'')) + self.assertEqual(rm_carriage_returns(res), (0, b'3\n', b'')) # check returned values (1 arg) res = node.safe_psql('select 4') - self.assertEqual(res, b'4\n') + self.assertEqual(rm_carriage_returns(res), b'4\n') # check returned values (2 args) res = node.safe_psql('postgres', 'select 5') - self.assertEqual(res, b'5\n') + self.assertEqual(rm_carriage_returns(res), b'5\n') # check returned values (named) res = node.safe_psql(query='select 6', dbname='postgres') - self.assertEqual(res, b'6\n') + self.assertEqual(rm_carriage_returns(res), b'6\n') # check feeding input node.safe_psql('create table horns (w int)') node.safe_psql('copy horns from stdin (format csv)', input=b"1\n2\n3\n\\.\n") _sum = node.safe_psql('select sum(w) from horns') - self.assertEqual(_sum, b'6\n') + self.assertEqual(rm_carriage_returns(_sum), b'6\n') # check psql's default args, fails with self.assertRaises(QueryException): @@ -455,7 +473,7 @@ def test_synchronous_replication(self): master.safe_psql( 'insert into abc select generate_series(1, 1000000)') res = standby1.safe_psql('select count(*) from abc') - self.assertEqual(res, b'1000000\n') + self.assertEqual(rm_carriage_returns(res), b'1000000\n') @unittest.skipUnless(pg_version_ge('10'), 'requires 10+') def test_logical_replication(self): @@ -589,7 +607,7 @@ def test_promotion(self): # make standby becomes writable master replica.safe_psql('insert into abc values (1)') res = replica.safe_psql('select * from abc') - self.assertEqual(res, b'1\n') + self.assertEqual(rm_carriage_returns(res), b'1\n') def test_dump(self): query_create = 'create table test as select generate_series(1, 2) as val' @@ -614,6 +632,7 @@ def test_users(self): with get_new_node().init().start() as node: node.psql('create role test_user login') value = node.safe_psql('select 1', username='test_user') + value = rm_carriage_returns(value) self.assertEqual(value, b'1\n') def test_poll_query_until(self): @@ -728,7 +747,7 @@ def test_logging(self): master.restart() self.assertTrue(master._logger.is_alive()) - @unittest.skipUnless(util_exists('pgbench'), 'might be missing') + @unittest.skipUnless(util_exists('pgbench.exe' if os.name == 'nt' else 'pgbench'), 'pgbench might be missing') def test_pgbench(self): with get_new_node().init().start() as node: @@ -744,6 +763,8 @@ def test_pgbench(self): out, _ = proc.communicate() out = out.decode('utf-8') + proc.stdout.close() + self.assertTrue('tps' in out) def test_pg_config(self): @@ -977,7 +998,9 @@ def test_child_pids(self): def test_child_process_dies(self): # test for FileNotFound exception during child_processes() function - with subprocess.Popen(["sleep", "60"]) as process: + cmd = ["timeout", "60"] if os.name == 'nt' else ["sleep", "60"] + + with subprocess.Popen(cmd, shell=True) as process: # shell=True might be needed on Windows self.assertEqual(process.poll(), None) # collect list of processes currently running children = psutil.Process(os.getpid()).children() diff --git a/tests/test_simple_remote.py b/tests/test_simple_remote.py index 1042f3c4..d51820ba 100755 --- a/tests/test_simple_remote.py +++ b/tests/test_simple_remote.py @@ -52,10 +52,9 @@ from testgres.utils import PgVer from testgres.node import ProcessProxy, ConnectionParams -conn_params = ConnectionParams(host=os.getenv('RDBMS_TESTPOOL1_HOST') or '172.18.0.3', - username='dev', - ssh_key=os.getenv( - 'RDBMS_TESTPOOL_SSHKEY') or '../../container_files/postgres/ssh/id_ed25519') +conn_params = ConnectionParams(host=os.getenv('RDBMS_TESTPOOL1_HOST') or '127.0.0.1', + username=os.getenv('USER'), + ssh_key=os.getenv('RDBMS_TESTPOOL_SSHKEY')) os_ops = RemoteOperations(conn_params) testgres_config.set_os_ops(os_ops=os_ops) From 79a8dc5e17f631dd8dc74b1bfa97c910c4c2c7b1 Mon Sep 17 00:00:00 2001 From: homper Date: Fri, 22 Dec 2023 16:23:13 +0100 Subject: [PATCH 264/475] Remove unnecessary output (#88) --- testgres/connection.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/testgres/connection.py b/testgres/connection.py index aeb040ce..882498a9 100644 --- a/testgres/connection.py +++ b/testgres/connection.py @@ -110,8 +110,7 @@ def execute(self, query, *args): res = [tuple(t) for t in res] return res - except Exception as e: - print("Error executing query: {}".format(e)) + except Exception: return None def close(self): From 5218b113fda81ee93e202d4a9ead0891f4ea475d Mon Sep 17 00:00:00 2001 From: Victoria Shepard <5807469+demonolock@users.noreply.github.com> Date: Tue, 2 Jan 2024 17:54:44 +0100 Subject: [PATCH 265/475] Add function pg_update (#97) --- docker-compose.yml | 4 +- testgres/cache.py | 7 +-- testgres/node.py | 84 +++++++++++++++++++++++++------- testgres/operations/local_ops.py | 2 +- testgres/utils.py | 5 +- tests/test_simple.py | 13 +++++ 6 files changed, 90 insertions(+), 25 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 471ab779..86edf9a4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,2 +1,4 @@ -tests: +version: '3.8' +services: + tests: build: . diff --git a/testgres/cache.py b/testgres/cache.py index 21198e83..f17b54b5 100644 --- a/testgres/cache.py +++ b/testgres/cache.py @@ -22,19 +22,20 @@ from .operations.os_ops import OsOperations -def cached_initdb(data_dir, logfile=None, params=None, os_ops: OsOperations = LocalOperations()): +def cached_initdb(data_dir, logfile=None, params=None, os_ops: OsOperations = LocalOperations(), bin_path=None, cached=True): """ Perform initdb or use cached node files. """ def call_initdb(initdb_dir, log=logfile): try: - _params = [get_bin_path("initdb"), "-D", initdb_dir, "-N"] + initdb_path = os.path.join(bin_path, 'initdb') if bin_path else get_bin_path("initdb") + _params = [initdb_path, "-D", initdb_dir, "-N"] execute_utility(_params + (params or []), log) except ExecUtilException as e: raise_from(InitNodeException("Failed to run initdb"), e) - if params or not testgres_config.cache_initdb: + if params or not testgres_config.cache_initdb or not cached: call_initdb(data_dir, logfile) else: # Fetch cached initdb dir diff --git a/testgres/node.py b/testgres/node.py index 20cf4264..0f1dcf98 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -127,7 +127,7 @@ def __repr__(self): class PostgresNode(object): - def __init__(self, name=None, port=None, base_dir=None, conn_params: ConnectionParams = ConnectionParams()): + def __init__(self, name=None, port=None, base_dir=None, conn_params: ConnectionParams = ConnectionParams(), bin_dir=None, prefix=None): """ PostgresNode constructor. @@ -135,12 +135,15 @@ def __init__(self, name=None, port=None, base_dir=None, conn_params: ConnectionP name: node's application name. port: port to accept connections. base_dir: path to node's data directory. + bin_dir: path to node's binary directory. """ # private - self._pg_version = PgVer(get_pg_version()) + self._pg_version = PgVer(get_pg_version(bin_dir)) self._should_free_port = port is None self._base_dir = base_dir + self._bin_dir = bin_dir + self._prefix = prefix self._logger = None self._master = None @@ -281,7 +284,7 @@ def master(self): @property def base_dir(self): if not self._base_dir: - self._base_dir = self.os_ops.mkdtemp(prefix=TMP_NODE) + self._base_dir = self.os_ops.mkdtemp(prefix=self._prefix or TMP_NODE) # NOTE: it's safe to create a new dir if not self.os_ops.path_exists(self._base_dir): @@ -289,6 +292,12 @@ def base_dir(self): return self._base_dir + @property + def bin_dir(self): + if not self._bin_dir: + self._bin_dir = os.path.dirname(get_bin_path("pg_config")) + return self._bin_dir + @property def logs_dir(self): path = os.path.join(self.base_dir, LOGS_DIR) @@ -441,7 +450,7 @@ def _collect_special_files(self): return result - def init(self, initdb_params=None, **kwargs): + def init(self, initdb_params=None, cached=True, **kwargs): """ Perform initdb for this node. @@ -460,7 +469,9 @@ def init(self, initdb_params=None, **kwargs): data_dir=self.data_dir, logfile=self.utils_log_file, os_ops=self.os_ops, - params=initdb_params) + params=initdb_params, + bin_path=self.bin_dir, + cached=False) # initialize default config files self.default_conf(**kwargs) @@ -619,7 +630,7 @@ def status(self): try: _params = [ - get_bin_path("pg_ctl"), + self._get_bin_path('pg_ctl'), "-D", self.data_dir, "status" ] # yapf: disable @@ -645,7 +656,7 @@ def get_control_data(self): """ # this one is tricky (blame PG 9.4) - _params = [get_bin_path("pg_controldata")] + _params = [self._get_bin_path("pg_controldata")] _params += ["-D"] if self._pg_version >= PgVer('9.5') else [] _params += [self.data_dir] @@ -708,7 +719,7 @@ def start(self, params=[], wait=True): return self _params = [ - get_bin_path("pg_ctl"), + self._get_bin_path("pg_ctl"), "-D", self.data_dir, "-l", self.pg_log_file, "-w" if wait else '-W', # --wait or --no-wait @@ -742,7 +753,7 @@ def stop(self, params=[], wait=True): return self _params = [ - get_bin_path("pg_ctl"), + self._get_bin_path("pg_ctl"), "-D", self.data_dir, "-w" if wait else '-W', # --wait or --no-wait "stop" @@ -782,7 +793,7 @@ def restart(self, params=[]): """ _params = [ - get_bin_path("pg_ctl"), + self._get_bin_path("pg_ctl"), "-D", self.data_dir, "-l", self.pg_log_file, "-w", # wait @@ -814,7 +825,7 @@ def reload(self, params=[]): """ _params = [ - get_bin_path("pg_ctl"), + self._get_bin_path("pg_ctl"), "-D", self.data_dir, "reload" ] + params # yapf: disable @@ -835,7 +846,7 @@ def promote(self, dbname=None, username=None): """ _params = [ - get_bin_path("pg_ctl"), + self._get_bin_path("pg_ctl"), "-D", self.data_dir, "-w", # wait "promote" @@ -871,7 +882,7 @@ def pg_ctl(self, params): """ _params = [ - get_bin_path("pg_ctl"), + self._get_bin_path("pg_ctl"), "-D", self.data_dir, "-w" # wait ] + params # yapf: disable @@ -945,7 +956,7 @@ def psql(self, username = username or default_username() psql_params = [ - get_bin_path("psql"), + self._get_bin_path("psql"), "-p", str(self.port), "-h", self.host, "-U", username, @@ -1066,7 +1077,7 @@ def tmpfile(): filename = filename or tmpfile() _params = [ - get_bin_path("pg_dump"), + self._get_bin_path("pg_dump"), "-p", str(self.port), "-h", self.host, "-f", filename, @@ -1094,7 +1105,7 @@ def restore(self, filename, dbname=None, username=None): username = username or default_username() _params = [ - get_bin_path("pg_restore"), + self._get_bin_path("pg_restore"), "-p", str(self.port), "-h", self.host, "-U", username, @@ -1364,7 +1375,7 @@ def pgbench(self, username = username or default_username() _params = [ - get_bin_path("pgbench"), + self._get_bin_path("pgbench"), "-p", str(self.port), "-h", self.host, "-U", username, @@ -1416,7 +1427,7 @@ def pgbench_run(self, dbname=None, username=None, options=[], **kwargs): username = username or default_username() _params = [ - get_bin_path("pgbench"), + self._get_bin_path("pgbench"), "-p", str(self.port), "-h", self.host, "-U", username, @@ -1587,6 +1598,43 @@ def set_auto_conf(self, options, config='postgresql.auto.conf', rm_options={}): self.os_ops.write(path, auto_conf, truncate=True) + def upgrade_from(self, old_node): + """ + Upgrade this node from an old node using pg_upgrade. + + Args: + old_node: An instance of PostgresNode representing the old node. + """ + if not os.path.exists(old_node.data_dir): + raise Exception("Old node must be initialized") + + if not os.path.exists(self.data_dir): + self.init() + + pg_upgrade_binary = self._get_bin_path("pg_upgrade") + + if not os.path.exists(pg_upgrade_binary): + raise Exception("pg_upgrade does not exist in the new node's binary path") + + upgrade_command = [ + pg_upgrade_binary, + "--old-bindir", old_node.bin_dir, + "--new-bindir", self.bin_dir, + "--old-datadir", old_node.data_dir, + "--new-datadir", self.data_dir, + "--old-port", str(old_node.port), + "--new-port", str(self.port), + ] + + return self.os_ops.exec_command(upgrade_command) + + def _get_bin_path(self, filename): + if self.bin_dir: + bin_path = os.path.join(self.bin_dir, filename) + else: + bin_path = get_bin_path(filename) + return bin_path + class NodeApp: diff --git a/testgres/operations/local_ops.py b/testgres/operations/local_ops.py index 93ebf012..ef360d3b 100644 --- a/testgres/operations/local_ops.py +++ b/testgres/operations/local_ops.py @@ -44,7 +44,7 @@ def __init__(self, conn_params=None): def _raise_exec_exception(message, command, exit_code, output): """Raise an ExecUtilException.""" raise ExecUtilException(message=message.format(output), - command=command, + command=' '.join(command) if isinstance(command, list) else command, exit_code=exit_code, out=output) diff --git a/testgres/utils.py b/testgres/utils.py index b21fc2c8..d84bb2b5 100644 --- a/testgres/utils.py +++ b/testgres/utils.py @@ -172,13 +172,14 @@ def cache_pg_config_data(cmd): return cache_pg_config_data("pg_config") -def get_pg_version(): +def get_pg_version(bin_dir=None): """ Return PostgreSQL version provided by postmaster. """ # get raw version (e.g. postgres (PostgreSQL) 9.5.7) - _params = [get_bin_path('postgres'), '--version'] + postgres_path = os.path.join(bin_dir, 'postgres') if bin_dir else get_bin_path('postgres') + _params = [postgres_path, '--version'] raw_ver = tconf.os_ops.exec_command(_params, encoding='utf-8') # Remove "(Homebrew)" if present diff --git a/tests/test_simple.py b/tests/test_simple.py index 9d31d4d9..a013f478 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -1010,6 +1010,19 @@ def test_child_process_dies(self): # try to handle children list -- missing processes will have ptype "ProcessType.Unknown" [ProcessProxy(p) for p in children] + def test_upgrade_node(self): + old_bin_dir = os.path.dirname(get_bin_path("pg_config")) + new_bin_dir = os.path.dirname(get_bin_path("pg_config")) + node_old = get_new_node(prefix='node_old', bin_dir=old_bin_dir) + node_old.init() + node_old.start() + node_old.stop() + node_new = get_new_node(prefix='node_new', bin_dir=new_bin_dir) + node_new.init(cached=False) + res = node_new.upgrade_from(old_node=node_old) + node_new.start() + self.assertTrue(b'Upgrade Complete' in res) + if __name__ == '__main__': if os.environ.get('ALT_CONFIG'): From a65da28b1717efeea4c01fd6df197c056a6c1846 Mon Sep 17 00:00:00 2001 From: Victoria Shepard <5807469+demonolock@users.noreply.github.com> Date: Thu, 18 Jan 2024 13:56:48 +0100 Subject: [PATCH 266/475] Add pg_probackup plugin (#92) --- setup.py | 2 +- testgres/plugins/__init__.py | 9 + testgres/plugins/pg_probackup2/README.md | 65 ++ testgres/plugins/pg_probackup2/__init__.py | 0 .../pg_probackup2/pg_probackup2/__init__.py | 0 .../pg_probackup2/pg_probackup2/app.py | 762 ++++++++++++++++++ .../pg_probackup2/pg_probackup2/gdb.py | 346 ++++++++ .../pg_probackup2/init_helpers.py | 207 +++++ .../pg_probackup2/storage/__init__.py | 0 .../pg_probackup2/storage/fs_backup.py | 101 +++ .../pg_probackup2/storage/s3_backup.py | 134 +++ .../pg_probackup2/tests/basic_test.py | 79 ++ testgres/plugins/pg_probackup2/setup.py | 18 + 13 files changed, 1722 insertions(+), 1 deletion(-) create mode 100644 testgres/plugins/__init__.py create mode 100644 testgres/plugins/pg_probackup2/README.md create mode 100644 testgres/plugins/pg_probackup2/__init__.py create mode 100644 testgres/plugins/pg_probackup2/pg_probackup2/__init__.py create mode 100644 testgres/plugins/pg_probackup2/pg_probackup2/app.py create mode 100644 testgres/plugins/pg_probackup2/pg_probackup2/gdb.py create mode 100644 testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py create mode 100644 testgres/plugins/pg_probackup2/pg_probackup2/storage/__init__.py create mode 100644 testgres/plugins/pg_probackup2/pg_probackup2/storage/fs_backup.py create mode 100644 testgres/plugins/pg_probackup2/pg_probackup2/storage/s3_backup.py create mode 100644 testgres/plugins/pg_probackup2/pg_probackup2/tests/basic_test.py create mode 100644 testgres/plugins/pg_probackup2/setup.py diff --git a/setup.py b/setup.py index e0287659..b006c8bf 100755 --- a/setup.py +++ b/setup.py @@ -27,7 +27,7 @@ readme = f.read() setup( - version='1.9.3', + version='1.10.0', name='testgres', packages=['testgres', 'testgres.operations', 'testgres.helpers'], description='Testing utility for PostgreSQL and its extensions', diff --git a/testgres/plugins/__init__.py b/testgres/plugins/__init__.py new file mode 100644 index 00000000..e60331f0 --- /dev/null +++ b/testgres/plugins/__init__.py @@ -0,0 +1,9 @@ +from pg_probackup2.gdb import GDBobj +from pg_probackup2.app import ProbackupApp, ProbackupException +from pg_probackup2.init_helpers import init_params +from pg_probackup2.storage.fs_backup import FSTestBackupDir +from pg_probackup2.storage.s3_backup import S3TestBackupDir + +__all__ = [ + "ProbackupApp", "ProbackupException", "init_params", "FSTestBackupDir", "S3TestBackupDir", "GDBobj" +] diff --git a/testgres/plugins/pg_probackup2/README.md b/testgres/plugins/pg_probackup2/README.md new file mode 100644 index 00000000..b62bf24b --- /dev/null +++ b/testgres/plugins/pg_probackup2/README.md @@ -0,0 +1,65 @@ +# testgres - pg_probackup2 + +Ccontrol and testing utility for [pg_probackup2](https://github.com/postgrespro/pg_probackup). Python 3.5+ is supported. + + +## Installation + +To install `testgres`, run: + +``` +pip install testgres-pg_probackup +``` + +We encourage you to use `virtualenv` for your testing environment. +The package requires testgres~=1.9.3. + +## Usage + +### Environment + +> Note: by default testgres runs `initdb`, `pg_ctl`, `psql` provided by `PATH`. + +There are several ways to specify a custom postgres installation: + +* export `PG_CONFIG` environment variable pointing to the `pg_config` executable; +* export `PG_BIN` environment variable pointing to the directory with executable files. + +Example: + +```bash +export PG_BIN=$HOME/pg/bin +python my_tests.py +``` + + +### Examples + +Here is an example of what you can do with `testgres-pg_probackup2`: + +```python +# You can see full script here plugins/pg_probackup2/pg_probackup2/tests/basic_test.py +def test_full_backup(self): + # Setting up a simple test node + node = self.pg_node.make_simple('node', pg_options={"fsync": "off", "synchronous_commit": "off"}) + + # Initialize and configure Probackup + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) + + # Start the node and initialize pgbench + node.slow_start() + node.pgbench_init(scale=100, no_vacuum=True) + + # Perform backup and validation + backup_id = self.pb.backup_node('node', node) + out = self.pb.validate('node', backup_id) + + # Check if the backup is valid + self.assertIn(f"INFO: Backup {backup_id} is valid", out) +``` + +## Authors + +[Postgres Professional](https://postgrespro.ru/about) diff --git a/testgres/plugins/pg_probackup2/__init__.py b/testgres/plugins/pg_probackup2/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/__init__.py b/testgres/plugins/pg_probackup2/pg_probackup2/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/app.py b/testgres/plugins/pg_probackup2/pg_probackup2/app.py new file mode 100644 index 00000000..2c31de51 --- /dev/null +++ b/testgres/plugins/pg_probackup2/pg_probackup2/app.py @@ -0,0 +1,762 @@ +import contextlib +import importlib +import json +import os +import re +import subprocess +import sys +import threading +import time +import unittest + +import testgres + +from .storage.fs_backup import TestBackupDir, FSTestBackupDir +from .gdb import GDBobj +from .init_helpers import init_params + +warning = """ +Wrong splint in show_pb +Original Header:f +{header} +Original Body: +{body} +Splitted Header +{header_split} +Splitted Body +{body_split} +""" + + +class ProbackupException(Exception): + def __init__(self, message, cmd): + self.message = message + self.cmd = cmd + + def __str__(self): + return '\n ERROR: {0}\n CMD: {1}'.format(repr(self.message), self.cmd) + + +class ProbackupApp: + + def __init__(self, test_class: unittest.TestCase, + pg_node, pb_log_path, test_env, auto_compress_alg, backup_dir): + self.test_class = test_class + self.pg_node = pg_node + self.pb_log_path = pb_log_path + self.test_env = test_env + self.auto_compress_alg = auto_compress_alg + self.backup_dir = backup_dir + self.probackup_path = init_params.probackup_path + self.probackup_old_path = init_params.probackup_old_path + self.remote = init_params.remote + self.verbose = init_params.verbose + self.archive_compress = init_params.archive_compress + self.test_class.output = None + + def run(self, command, gdb=False, old_binary=False, return_id=True, env=None, + skip_log_directory=False, expect_error=False, use_backup_dir=True): + """ + Run pg_probackup + backup_dir: target directory for making backup + command: commandline options + expect_error: option for ignoring errors and getting error message as a result of running the function + gdb: when True it returns GDBObj(), when tuple('suspend', port) it runs probackup + in suspended gdb mode with attachable gdb port, for local debugging + """ + if isinstance(use_backup_dir, TestBackupDir): + command = [command[0], *use_backup_dir.pb_args, *command[1:]] + elif use_backup_dir: + command = [command[0], *self.backup_dir.pb_args, *command[1:]] + + if not self.probackup_old_path and old_binary: + print('PGPROBACKUPBIN_OLD is not set') + exit(1) + + if old_binary: + binary_path = self.probackup_old_path + else: + binary_path = self.probackup_path + + if not env: + env = self.test_env + + strcommand = ' '.join(str(p) for p in command) + if '--log-level-file' in strcommand and \ + '--log-directory' not in strcommand and \ + not skip_log_directory: + command += ['--log-directory=' + self.pb_log_path] + strcommand += ' ' + command[-1] + + if 'pglz' in strcommand and \ + ' -j' not in strcommand and '--thread' not in strcommand: + command += ['-j', '1'] + strcommand += ' -j 1' + + self.test_class.cmd = binary_path + ' ' + strcommand + if self.verbose: + print(self.test_class.cmd) + + cmdline = [binary_path, *command] + if gdb is True: + # general test flow for using GDBObj + return GDBobj(cmdline, self.test_class) + + try: + result = None + if type(gdb) is tuple and gdb[0] == 'suspend': + # special test flow for manually debug probackup + gdb_port = gdb[1] + cmdline = ['gdbserver'] + ['localhost:' + str(gdb_port)] + cmdline + print("pg_probackup gdb suspended, waiting gdb connection on localhost:{0}".format(gdb_port)) + + self.test_class.output = subprocess.check_output( + cmdline, + stderr=subprocess.STDOUT, + env=env + ).decode('utf-8', errors='replace') + if command[0] == 'backup' and return_id: + # return backup ID + for line in self.test_class.output.splitlines(): + if 'INFO: Backup' and 'completed' in line: + result = line.split()[2] + else: + result = self.test_class.output + if expect_error is True: + assert False, f"Exception was expected, but run finished successful with result: `{result}`\n" \ + f"CMD: {self.test_class.cmd}" + elif expect_error: + assert False, f"Exception was expected {expect_error}, but run finished successful with result: `{result}`\n" \ + f"CMD: {self.test_class.cmd}" + return result + except subprocess.CalledProcessError as e: + self.test_class.output = e.output.decode('utf-8').replace("\r", "") + if expect_error: + return self.test_class.output + else: + raise ProbackupException(self.test_class.output, self.test_class.cmd) + + def init(self, options=None, old_binary=False, skip_log_directory=False, expect_error=False, use_backup_dir=True): + if options is None: + options = [] + return self.run(['init'] + options, + old_binary=old_binary, + skip_log_directory=skip_log_directory, + expect_error=expect_error, + use_backup_dir=use_backup_dir + ) + + def add_instance(self, instance, node, old_binary=False, options=None, expect_error=False): + if options is None: + options = [] + cmd = [ + 'add-instance', + '--instance={0}'.format(instance), + '-D', node.data_dir + ] + + # don`t forget to kill old_binary after remote ssh release + if self.remote and not old_binary: + options = options + [ + '--remote-proto=ssh', + '--remote-host=localhost'] + + return self.run(cmd + options, old_binary=old_binary, expect_error=expect_error) + + def set_config(self, instance, old_binary=False, options=None, expect_error=False): + if options is None: + options = [] + cmd = [ + 'set-config', + '--instance={0}'.format(instance), + ] + + return self.run(cmd + options, old_binary=old_binary, expect_error=expect_error) + + def set_backup(self, instance, backup_id=False, + old_binary=False, options=None, expect_error=False): + if options is None: + options = [] + cmd = [ + 'set-backup', + ] + + if instance: + cmd = cmd + ['--instance={0}'.format(instance)] + + if backup_id: + cmd = cmd + ['-i', backup_id] + + return self.run(cmd + options, old_binary=old_binary, expect_error=expect_error) + + def del_instance(self, instance, old_binary=False, expect_error=False): + + return self.run([ + 'del-instance', + '--instance={0}'.format(instance), + ], + old_binary=old_binary, + expect_error=expect_error + ) + + def backup_node( + self, instance, node, data_dir=False, + backup_type='full', datname=False, options=None, + gdb=False, + old_binary=False, return_id=True, no_remote=False, + env=None, + expect_error=False, + sync=False + ): + if options is None: + options = [] + if not node and not data_dir: + print('You must provide ether node or data_dir for backup') + exit(1) + + if not datname: + datname = 'postgres' + + cmd_list = [ + 'backup', + '--instance={0}'.format(instance), + # "-D", pgdata, + '-p', '%i' % node.port, + '-d', datname + ] + + if data_dir: + cmd_list += ['-D', self._node_dir(data_dir)] + + # don`t forget to kill old_binary after remote ssh release + if self.remote and not old_binary and not no_remote: + options = options + [ + '--remote-proto=ssh', + '--remote-host=localhost'] + + if self.auto_compress_alg and '--compress' in options and \ + self.archive_compress and self.archive_compress != 'zlib': + options = [o if o != '--compress' else f'--compress-algorithm={self.archive_compress}' + for o in options] + + if backup_type: + cmd_list += ['-b', backup_type] + + if not (old_binary or sync): + cmd_list += ['--no-sync'] + + return self.run(cmd_list + options, gdb, old_binary, return_id, env=env, + expect_error=expect_error) + + def backup_replica_node(self, instance, node, data_dir=False, *, + master, backup_type='full', datname=False, + options=None, env=None): + """ + Try to reliably run backup on replica by switching wal at master + at the moment pg_probackup is waiting for archived wal segment + """ + if options is None: + options = [] + assert '--stream' not in options or backup_type == 'page', \ + "backup_replica_node should be used with one of archive-mode or " \ + "page-stream mode" + + options = options.copy() + if not any('--log-level-file' in x for x in options): + options.append('--log-level-file=INFO') + + gdb = self.backup_node( + instance, node, data_dir, + backup_type=backup_type, + datname=datname, + options=options, + env=env, + gdb=True) + gdb.set_breakpoint('wait_wal_lsn') + # we need to break on wait_wal_lsn in pg_stop_backup + gdb.run_until_break() + if backup_type == 'page': + self.switch_wal_segment(master) + if '--stream' not in options: + gdb.continue_execution_until_break() + self.switch_wal_segment(master) + gdb.continue_execution_until_exit() + + output = self.read_pb_log() + self.unlink_pg_log() + parsed_output = re.compile(r'Backup \S+ completed').search(output) + assert parsed_output, f"Expected: `Backup 'backup_id' completed`, but found `{output}`" + backup_id = parsed_output[0].split(' ')[1] + return (backup_id, output) + + def checkdb_node( + self, use_backup_dir=False, instance=False, data_dir=False, + options=None, gdb=False, old_binary=False, + skip_log_directory=False, + expect_error=False + ): + if options is None: + options = [] + cmd_list = ["checkdb"] + + if instance: + cmd_list += ["--instance={0}".format(instance)] + + if data_dir: + cmd_list += ["-D", self._node_dir(data_dir)] + + return self.run(cmd_list + options, gdb, old_binary, + skip_log_directory=skip_log_directory, expect_error=expect_error, + use_backup_dir=use_backup_dir) + + def merge_backup( + self, instance, backup_id, + gdb=False, old_binary=False, options=None, expect_error=False): + if options is None: + options = [] + cmd_list = [ + 'merge', + '--instance={0}'.format(instance), + '-i', backup_id + ] + + return self.run(cmd_list + options, gdb, old_binary, expect_error=expect_error) + + def restore_node( + self, instance, node=None, restore_dir=None, + backup_id=None, old_binary=False, options=None, + gdb=False, + expect_error=False, + sync=False + ): + if options is None: + options = [] + if node: + if isinstance(node, str): + data_dir = node + else: + data_dir = node.data_dir + elif restore_dir: + data_dir = self._node_dir(restore_dir) + else: + raise ValueError("You must provide ether node or base_dir for backup") + + cmd_list = [ + 'restore', + '-D', data_dir, + '--instance={0}'.format(instance) + ] + + # don`t forget to kill old_binary after remote ssh release + if self.remote and not old_binary: + options = options + [ + '--remote-proto=ssh', + '--remote-host=localhost'] + + if backup_id: + cmd_list += ['-i', backup_id] + + if not (old_binary or sync): + cmd_list += ['--no-sync'] + + return self.run(cmd_list + options, gdb=gdb, old_binary=old_binary, expect_error=expect_error) + + def catchup_node( + self, + backup_mode, source_pgdata, destination_node, + options=None, + remote_host='localhost', + expect_error=False, + gdb=False + ): + + if options is None: + options = [] + cmd_list = [ + 'catchup', + '--backup-mode={0}'.format(backup_mode), + '--source-pgdata={0}'.format(source_pgdata), + '--destination-pgdata={0}'.format(destination_node.data_dir) + ] + if self.remote: + cmd_list += ['--remote-proto=ssh', '--remote-host=%s' % remote_host] + if self.verbose: + cmd_list += [ + '--log-level-file=VERBOSE', + '--log-directory={0}'.format(destination_node.logs_dir) + ] + + return self.run(cmd_list + options, gdb=gdb, expect_error=expect_error, use_backup_dir=False) + + def show( + self, instance=None, backup_id=None, + options=None, as_text=False, as_json=True, old_binary=False, + env=None, + expect_error=False, + gdb=False + ): + + if options is None: + options = [] + backup_list = [] + specific_record = {} + cmd_list = [ + 'show', + ] + if instance: + cmd_list += ['--instance={0}'.format(instance)] + + if backup_id: + cmd_list += ['-i', backup_id] + + # AHTUNG, WARNING will break json parsing + if as_json: + cmd_list += ['--format=json', '--log-level-console=error'] + + if as_text: + # You should print it when calling as_text=true + return self.run(cmd_list + options, old_binary=old_binary, env=env, + expect_error=expect_error, gdb=gdb) + + # get show result as list of lines + if as_json: + text_json = str(self.run(cmd_list + options, old_binary=old_binary, env=env, + expect_error=expect_error, gdb=gdb)) + try: + if expect_error: + return text_json + data = json.loads(text_json) + except ValueError: + assert False, f"Couldn't parse {text_json} as json. " \ + f"Check that you don't have additional messages inside the log or use 'as_text=True'" + + for instance_data in data: + # find specific instance if requested + if instance and instance_data['instance'] != instance: + continue + + for backup in reversed(instance_data['backups']): + # find specific backup if requested + if backup_id: + if backup['id'] == backup_id: + return backup + else: + backup_list.append(backup) + + if backup_id is not None: + assert False, "Failed to find backup with ID: {0}".format(backup_id) + + return backup_list + else: + show_splitted = self.run(cmd_list + options, old_binary=old_binary, env=env, + expect_error=expect_error).splitlines() + if instance is not None and backup_id is None: + # cut header(ID, Mode, etc) from show as single string + header = show_splitted[1:2][0] + # cut backup records from show as single list + # with string for every backup record + body = show_splitted[3:] + # inverse list so oldest record come first + body = body[::-1] + # split string in list with string for every header element + header_split = re.split(' +', header) + # Remove empty items + for i in header_split: + if i == '': + header_split.remove(i) + continue + header_split = [ + header_element.rstrip() for header_element in header_split + ] + for backup_record in body: + backup_record = backup_record.rstrip() + # split list with str for every backup record element + backup_record_split = re.split(' +', backup_record) + # Remove empty items + for i in backup_record_split: + if i == '': + backup_record_split.remove(i) + if len(header_split) != len(backup_record_split): + print(warning.format( + header=header, body=body, + header_split=header_split, + body_split=backup_record_split) + ) + exit(1) + new_dict = dict(zip(header_split, backup_record_split)) + backup_list.append(new_dict) + return backup_list + else: + # cut out empty lines and lines started with # + # and other garbage then reconstruct it as dictionary + # print show_splitted + sanitized_show = [item for item in show_splitted if item] + sanitized_show = [ + item for item in sanitized_show if not item.startswith('#') + ] + # print sanitized_show + for line in sanitized_show: + name, var = line.partition(' = ')[::2] + var = var.strip('"') + var = var.strip("'") + specific_record[name.strip()] = var + + if not specific_record: + assert False, "Failed to find backup with ID: {0}".format(backup_id) + + return specific_record + + def show_archive( + self, instance=None, options=None, + as_text=False, as_json=True, old_binary=False, + tli=0, + expect_error=False + ): + if options is None: + options = [] + cmd_list = [ + 'show', + '--archive', + ] + if instance: + cmd_list += ['--instance={0}'.format(instance)] + + # AHTUNG, WARNING will break json parsing + if as_json: + cmd_list += ['--format=json', '--log-level-console=error'] + + if as_text: + # You should print it when calling as_text=true + return self.run(cmd_list + options, old_binary=old_binary, expect_error=expect_error) + + if as_json: + if as_text: + data = self.run(cmd_list + options, old_binary=old_binary, expect_error=expect_error) + else: + data = json.loads(self.run(cmd_list + options, old_binary=old_binary, expect_error=expect_error)) + + if instance: + instance_timelines = None + for instance_name in data: + if instance_name['instance'] == instance: + instance_timelines = instance_name['timelines'] + break + + if tli > 0: + for timeline in instance_timelines: + if timeline['tli'] == tli: + return timeline + + return {} + + if instance_timelines: + return instance_timelines + + return data + else: + show_splitted = self.run(cmd_list + options, old_binary=old_binary, + expect_error=expect_error).splitlines() + print(show_splitted) + exit(1) + + def validate( + self, instance=None, backup_id=None, + options=None, old_binary=False, gdb=False, expect_error=False + ): + if options is None: + options = [] + cmd_list = [ + 'validate', + ] + if instance: + cmd_list += ['--instance={0}'.format(instance)] + if backup_id: + cmd_list += ['-i', backup_id] + + return self.run(cmd_list + options, old_binary=old_binary, gdb=gdb, + expect_error=expect_error) + + def delete( + self, instance, backup_id=None, + options=None, old_binary=False, gdb=False, expect_error=False): + if options is None: + options = [] + cmd_list = [ + 'delete', + ] + + cmd_list += ['--instance={0}'.format(instance)] + if backup_id: + cmd_list += ['-i', backup_id] + + return self.run(cmd_list + options, old_binary=old_binary, gdb=gdb, + expect_error=expect_error) + + def delete_expired( + self, instance, options=None, old_binary=False, expect_error=False): + if options is None: + options = [] + cmd_list = [ + 'delete', + '--instance={0}'.format(instance) + ] + return self.run(cmd_list + options, old_binary=old_binary, expect_error=expect_error) + + def show_config(self, instance, old_binary=False, expect_error=False, gdb=False): + out_dict = {} + cmd_list = [ + 'show-config', + '--instance={0}'.format(instance) + ] + + res = self.run(cmd_list, old_binary=old_binary, expect_error=expect_error, gdb=gdb).splitlines() + for line in res: + if not line.startswith('#'): + name, var = line.partition(' = ')[::2] + out_dict[name] = var + return out_dict + + def run_binary(self, command, asynchronous=False, env=None): + + if not env: + env = self.test_env + + if self.verbose: + print([' '.join(map(str, command))]) + try: + if asynchronous: + return subprocess.Popen( + command, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=env + ) + else: + self.test_class.output = subprocess.check_output( + command, + stderr=subprocess.STDOUT, + env=env + ).decode('utf-8') + return self.test_class.output + except subprocess.CalledProcessError as e: + raise ProbackupException(e.output.decode('utf-8'), command) + + def _node_dir(self, base_dir): + return os.path.join(self.pg_node.test_path, base_dir) + + def set_archiving( + self, instance, node, replica=False, + overwrite=False, compress=True, old_binary=False, + log_level=False, archive_timeout=False, + custom_archive_command=None): + + # parse postgresql.auto.conf + options = {} + if replica: + options['archive_mode'] = 'always' + options['hot_standby'] = 'on' + else: + options['archive_mode'] = 'on' + + if custom_archive_command is None: + archive_command = " ".join([f'"{init_params.probackup_path}"', + 'archive-push', *self.backup_dir.pb_args]) + if os.name == "nt": + archive_command = archive_command.replace("\\", "\\\\") + archive_command += f' --instance={instance}' + + # don`t forget to kill old_binary after remote ssh release + if init_params.remote and not old_binary: + archive_command += ' --remote-proto=ssh --remote-host=localhost' + + if init_params.archive_compress and compress: + archive_command += ' --compress-algorithm=' + init_params.archive_compress + + if overwrite: + archive_command += ' --overwrite' + + archive_command += ' --log-level-console=VERBOSE' + archive_command += ' -j 5' + archive_command += ' --batch-size 10' + archive_command += ' --no-sync' + + if archive_timeout: + archive_command += f' --archive-timeout={archive_timeout}' + + if os.name == 'posix': + archive_command += ' --wal-file-path=%p --wal-file-name=%f' + + elif os.name == 'nt': + archive_command += ' --wal-file-path="%p" --wal-file-name="%f"' + + if log_level: + archive_command += f' --log-level-console={log_level}' + else: # custom_archive_command is not None + archive_command = custom_archive_command + options['archive_command'] = archive_command + + node.set_auto_conf(options) + + def switch_wal_segment(self, node, sleep_seconds=1, and_tx=False): + """ + Execute pg_switch_wal() in given node + + Args: + node: an instance of PostgresNode or NodeConnection class + """ + if isinstance(node, testgres.PostgresNode): + with node.connect('postgres') as con: + if and_tx: + con.execute('select txid_current()') + lsn = con.execute('select pg_switch_wal()')[0][0] + else: + lsn = node.execute('select pg_switch_wal()')[0][0] + + if sleep_seconds > 0: + time.sleep(sleep_seconds) + return lsn + + @contextlib.contextmanager + def switch_wal_after(self, node, seconds, and_tx=True): + tm = threading.Timer(seconds, self.switch_wal_segment, [node, 0, and_tx]) + tm.start() + try: + yield + finally: + tm.cancel() + tm.join() + + def read_pb_log(self): + with open(os.path.join(self.pb_log_path, 'pg_probackup.log')) as fl: + return fl.read() + + def unlink_pg_log(self): + os.unlink(os.path.join(self.pb_log_path, 'pg_probackup.log')) + + def load_backup_class(fs_type): + fs_type = os.environ.get('PROBACKUP_FS_TYPE') + implementation = f"{__package__}.fs_backup.FSTestBackupDir" + if fs_type: + implementation = fs_type + + print("Using ", implementation) + module_name, class_name = implementation.rsplit(sep='.', maxsplit=1) + + module = importlib.import_module(module_name) + + return getattr(module, class_name) + + +# Local or S3 backup +fs_backup_class = FSTestBackupDir +if os.environ.get('PG_PROBACKUP_S3_TEST', os.environ.get('PROBACKUP_S3_TYPE_FULL_TEST')): + root = os.path.realpath(os.path.join(os.path.dirname(__file__), '../..')) + if root not in sys.path: + sys.path.append(root) + from pg_probackup2.storage.s3_backup import S3TestBackupDir + + fs_backup_class = S3TestBackupDir + + def build_backup_dir(self, backup='backup'): + return fs_backup_class(rel_path=self.rel_path, backup=backup) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/gdb.py b/testgres/plugins/pg_probackup2/pg_probackup2/gdb.py new file mode 100644 index 00000000..0b61da65 --- /dev/null +++ b/testgres/plugins/pg_probackup2/pg_probackup2/gdb.py @@ -0,0 +1,346 @@ +import functools +import os +import re +import subprocess +import sys +import unittest +from time import sleep + + +class GdbException(Exception): + def __init__(self, message="False"): + self.message = message + + def __str__(self): + return '\n ERROR: {0}\n'.format(repr(self.message)) + + +class GDBobj: + _gdb_enabled = False + _gdb_ok = False + _gdb_ptrace_ok = False + + def __init__(self, cmd, env, attach=False): + self.verbose = env.verbose + self.output = '' + self._did_quit = False + self.has_breakpoint = False + + # Check gdb flag is set up + if not hasattr(env, "_gdb_decorated") or not env._gdb_decorated: + raise GdbException("Test should be decorated with @needs_gdb") + if not self._gdb_enabled: + raise GdbException("No `PGPROBACKUP_GDB=on` is set.") + if not self._gdb_ok: + if not self._gdb_ptrace_ok: + raise GdbException("set /proc/sys/kernel/yama/ptrace_scope to 0" + " to run GDB tests") + raise GdbException("No gdb usage possible.") + + # Check gdb presense + try: + gdb_version, _ = subprocess.Popen( + ['gdb', '--version'], + stdout=subprocess.PIPE + ).communicate() + except OSError: + raise GdbException("Couldn't find gdb on the path") + + self.base_cmd = [ + 'gdb', + '--interpreter', + 'mi2', + ] + + if attach: + self.cmd = self.base_cmd + ['--pid'] + cmd + else: + self.cmd = self.base_cmd + ['--args'] + cmd + + # Get version + gdb_version_number = re.search( + br"^GNU gdb [^\d]*(\d+)\.(\d)", + gdb_version) + self.major_version = int(gdb_version_number.group(1)) + self.minor_version = int(gdb_version_number.group(2)) + + if self.verbose: + print([' '.join(map(str, self.cmd))]) + + self.proc = subprocess.Popen( + self.cmd, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + bufsize=0, + text=True, + errors='replace', + ) + self.gdb_pid = self.proc.pid + + while True: + line = self.get_line() + + if 'No such process' in line: + raise GdbException(line) + + if not line.startswith('(gdb)'): + pass + else: + break + + def __del__(self): + if not self._did_quit and hasattr(self, "proc"): + try: + self.quit() + except subprocess.TimeoutExpired: + self.kill() + + def get_line(self): + line = self.proc.stdout.readline() + self.output += line + return line + + def kill(self): + self._did_quit = True + self.proc.kill() + self.proc.wait(3) + self.proc.stdin.close() + self.proc.stdout.close() + + def set_breakpoint(self, location): + + result = self._execute('break ' + location) + self.has_breakpoint = True + for line in result: + if line.startswith('~"Breakpoint'): + return + + elif line.startswith('=breakpoint-created'): + return + + elif line.startswith('^error'): # or line.startswith('(gdb)'): + break + + elif line.startswith('&"break'): + pass + + elif line.startswith('&"Function'): + raise GdbException(line) + + elif line.startswith('&"No line'): + raise GdbException(line) + + elif line.startswith('~"Make breakpoint pending on future shared'): + raise GdbException(line) + + raise GdbException( + 'Failed to set breakpoint.\n Output:\n {0}'.format(result) + ) + + def remove_all_breakpoints(self): + if not self.has_breakpoint: + return + + result = self._execute('delete') + self.has_breakpoint = False + for line in result: + + if line.startswith('^done'): + return + + raise GdbException( + 'Failed to remove breakpoints.\n Output:\n {0}'.format(result) + ) + + def run_until_break(self): + result = self._execute('run', False) + for line in result: + if line.startswith('*stopped,reason="breakpoint-hit"'): + return + raise GdbException( + 'Failed to run until breakpoint.\n' + ) + + def continue_execution_until_running(self): + result = self._execute('continue') + + for line in result: + if line.startswith('*running') or line.startswith('^running'): + return + if line.startswith('*stopped,reason="breakpoint-hit"'): + continue + if line.startswith('*stopped,reason="exited-normally"'): + continue + + raise GdbException( + 'Failed to continue execution until running.\n' + ) + + def signal(self, sig): + if 'KILL' in sig: + self.remove_all_breakpoints() + self._execute(f'signal {sig}') + + def continue_execution_until_exit(self): + self.remove_all_breakpoints() + result = self._execute('continue', False) + + for line in result: + if line.startswith('*running'): + continue + if line.startswith('*stopped,reason="breakpoint-hit"'): + continue + if line.startswith('*stopped,reason="exited') or line == '*stopped\n': + self.quit() + return + + raise GdbException( + 'Failed to continue execution until exit.\n' + ) + + def continue_execution_until_error(self): + self.remove_all_breakpoints() + result = self._execute('continue', False) + + for line in result: + if line.startswith('^error'): + return + if line.startswith('*stopped,reason="exited'): + return + if line.startswith( + '*stopped,reason="signal-received",signal-name="SIGABRT"'): + return + + raise GdbException( + 'Failed to continue execution until error.\n') + + def continue_execution_until_break(self, ignore_count=0): + if ignore_count > 0: + result = self._execute( + 'continue ' + str(ignore_count), + False + ) + else: + result = self._execute('continue', False) + + for line in result: + if line.startswith('*stopped,reason="breakpoint-hit"'): + return + if line.startswith('*stopped,reason="exited-normally"'): + break + + raise GdbException( + 'Failed to continue execution until break.\n') + + def show_backtrace(self): + return self._execute("backtrace", running=False) + + def stopped_in_breakpoint(self): + while True: + line = self.get_line() + if self.verbose: + print(line) + if line.startswith('*stopped,reason="breakpoint-hit"'): + return True + + def detach(self): + if not self._did_quit: + self._execute('detach') + + def quit(self): + if not self._did_quit: + self._did_quit = True + self.proc.terminate() + self.proc.wait(3) + self.proc.stdin.close() + self.proc.stdout.close() + + # use for breakpoint, run, continue + def _execute(self, cmd, running=True): + output = [] + self.proc.stdin.flush() + self.proc.stdin.write(cmd + '\n') + self.proc.stdin.flush() + sleep(1) + + # look for command we just send + while True: + line = self.get_line() + if self.verbose: + print(repr(line)) + + if cmd not in line: + continue + else: + break + + while True: + line = self.get_line() + output += [line] + if self.verbose: + print(repr(line)) + if line.startswith('^done') or line.startswith('*stopped'): + break + if line.startswith('^error'): + break + if running and (line.startswith('*running') or line.startswith('^running')): + # if running and line.startswith('*running'): + break + return output + + +def _set_gdb(self): + test_env = os.environ.copy() + self._gdb_enabled = test_env.get('PGPROBACKUP_GDB') == 'ON' + self._gdb_ok = self._gdb_enabled + if not self._gdb_enabled or sys.platform != 'linux': + return + try: + with open('/proc/sys/kernel/yama/ptrace_scope') as f: + ptrace = f.read() + except FileNotFoundError: + self._gdb_ptrace_ok = True + return + self._gdb_ptrace_ok = int(ptrace) == 0 + self._gdb_ok = self._gdb_ok and self._gdb_ptrace_ok + + +def _check_gdb_flag_or_skip_test(): + if not GDBobj._gdb_enabled: + return ("skip", + "Specify PGPROBACKUP_GDB and build without " + "optimizations for run this test" + ) + if GDBobj._gdb_ok: + return None + if not GDBobj._gdb_ptrace_ok: + return ("fail", "set /proc/sys/kernel/yama/ptrace_scope to 0" + " to run GDB tests") + else: + return ("fail", "use of gdb is not possible") + + +def needs_gdb(func): + check = _check_gdb_flag_or_skip_test() + if not check: + @functools.wraps(func) + def ok_wrapped(self): + self._gdb_decorated = True + func(self) + + return ok_wrapped + reason = check[1] + if check[0] == "skip": + return unittest.skip(reason)(func) + elif check[0] == "fail": + @functools.wraps(func) + def fail_wrapper(self): + self.fail(reason) + + return fail_wrapper + else: + raise "Wrong action {0}".format(check) + + +_set_gdb(GDBobj) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py new file mode 100644 index 00000000..7af21eb6 --- /dev/null +++ b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py @@ -0,0 +1,207 @@ +from functools import reduce +import getpass +import os +import re +import shutil +import subprocess +import sys +import testgres + +try: + import lz4.frame # noqa: F401 + + HAVE_LZ4 = True +except ImportError as e: + HAVE_LZ4 = False + LZ4_error = e + +try: + import zstd # noqa: F401 + + HAVE_ZSTD = True +except ImportError as e: + HAVE_ZSTD = False + ZSTD_error = e + +delete_logs = os.getenv('KEEP_LOGS') not in ['1', 'y', 'Y'] + +try: + testgres.configure_testgres( + cache_initdb=False, + cached_initdb_dir=False, + node_cleanup_full=delete_logs) +except Exception as e: + print("Can't configure testgres: {0}".format(e)) + + +class Init(object): + def __init__(self): + if '-v' in sys.argv or '--verbose' in sys.argv: + self.verbose = True + else: + self.verbose = False + + self._pg_config = testgres.get_pg_config() + self.is_enterprise = self._pg_config.get('PGPRO_EDITION', None) == 'enterprise' + self.is_shardman = self._pg_config.get('PGPRO_EDITION', None) == 'shardman' + self.is_pgpro = 'PGPRO_EDITION' in self._pg_config + self.is_nls_enabled = 'enable-nls' in self._pg_config['CONFIGURE'] + self.is_lz4_enabled = '-llz4' in self._pg_config['LIBS'] + version = self._pg_config['VERSION'].rstrip('develalphabetapre') + parts = [*version.split(' ')[1].split('.'), '0', '0'][:3] + parts[0] = re.match(r'\d+', parts[0]).group() + self.pg_config_version = reduce(lambda v, x: v * 100 + int(x), parts, 0) + + test_env = os.environ.copy() + envs_list = [ + 'LANGUAGE', + 'LC_ALL', + 'PGCONNECT_TIMEOUT', + 'PGDATA', + 'PGDATABASE', + 'PGHOSTADDR', + 'PGREQUIRESSL', + 'PGSERVICE', + 'PGSSLMODE', + 'PGUSER', + 'PGPORT', + 'PGHOST' + ] + + for e in envs_list: + test_env.pop(e, None) + + test_env['LC_MESSAGES'] = 'C' + test_env['LC_TIME'] = 'C' + self._test_env = test_env + + # Get the directory from which the script was executed + self.source_path = os.getcwd() + tmp_path = test_env.get('PGPROBACKUP_TMP_DIR') + if tmp_path and os.path.isabs(tmp_path): + self.tmp_path = tmp_path + else: + self.tmp_path = os.path.abspath( + os.path.join(self.source_path, tmp_path or os.path.join('tests', 'tmp_dirs')) + ) + + os.makedirs(self.tmp_path, exist_ok=True) + + self.username = getpass.getuser() + + self.probackup_path = None + if 'PGPROBACKUPBIN' in test_env: + if shutil.which(test_env["PGPROBACKUPBIN"]): + self.probackup_path = test_env["PGPROBACKUPBIN"] + else: + if self.verbose: + print('PGPROBACKUPBIN is not an executable file') + + if not self.probackup_path: + probackup_path_tmp = os.path.join( + testgres.get_pg_config()['BINDIR'], 'pg_probackup') + + if os.path.isfile(probackup_path_tmp): + if not os.access(probackup_path_tmp, os.X_OK): + print('{0} is not an executable file'.format( + probackup_path_tmp)) + else: + self.probackup_path = probackup_path_tmp + + if not self.probackup_path: + probackup_path_tmp = self.source_path + + if os.path.isfile(probackup_path_tmp): + if not os.access(probackup_path_tmp, os.X_OK): + print('{0} is not an executable file'.format( + probackup_path_tmp)) + else: + self.probackup_path = probackup_path_tmp + + if not self.probackup_path: + print('pg_probackup binary is not found') + exit(1) + + if os.name == 'posix': + self.EXTERNAL_DIRECTORY_DELIMITER = ':' + os.environ['PATH'] = os.path.dirname( + self.probackup_path) + ':' + os.environ['PATH'] + + elif os.name == 'nt': + self.EXTERNAL_DIRECTORY_DELIMITER = ';' + os.environ['PATH'] = os.path.dirname( + self.probackup_path) + ';' + os.environ['PATH'] + + self.probackup_old_path = None + if 'PGPROBACKUPBIN_OLD' in test_env: + if (os.path.isfile(test_env['PGPROBACKUPBIN_OLD']) and os.access(test_env['PGPROBACKUPBIN_OLD'], os.X_OK)): + self.probackup_old_path = test_env['PGPROBACKUPBIN_OLD'] + else: + if self.verbose: + print('PGPROBACKUPBIN_OLD is not an executable file') + + self.probackup_version = None + self.old_probackup_version = None + + probackup_version_output = subprocess.check_output( + [self.probackup_path, "--version"], + stderr=subprocess.STDOUT, + ).decode('utf-8') + self.probackup_version = re.search(r"\d+\.\d+\.\d+", + probackup_version_output + ).group(0) + compressions = re.search(r"\(compressions: ([^)]*)\)", + probackup_version_output).group(1) + self.probackup_compressions = {s.strip() for s in compressions.split(',')} + + if self.probackup_old_path: + old_probackup_version_output = subprocess.check_output( + [self.probackup_old_path, "--version"], + stderr=subprocess.STDOUT, + ).decode('utf-8') + self.old_probackup_version = re.search(r"\d+\.\d+\.\d+", + old_probackup_version_output + ).group(0) + + self.remote = test_env.get('PGPROBACKUP_SSH_REMOTE', None) == 'ON' + self.ptrack = test_env.get('PG_PROBACKUP_PTRACK', None) == 'ON' and self.pg_config_version >= 110000 + + self.paranoia = test_env.get('PG_PROBACKUP_PARANOIA', None) == 'ON' + env_compress = test_env.get('ARCHIVE_COMPRESSION', None) + if env_compress: + env_compress = env_compress.lower() + if env_compress in ('on', 'zlib'): + self.compress_suffix = '.gz' + self.archive_compress = 'zlib' + elif env_compress == 'lz4': + if not HAVE_LZ4: + raise LZ4_error + if 'lz4' not in self.probackup_compressions: + raise Exception("pg_probackup is not compiled with lz4 support") + self.compress_suffix = '.lz4' + self.archive_compress = 'lz4' + elif env_compress == 'zstd': + if not HAVE_ZSTD: + raise ZSTD_error + if 'zstd' not in self.probackup_compressions: + raise Exception("pg_probackup is not compiled with zstd support") + self.compress_suffix = '.zst' + self.archive_compress = 'zstd' + else: + self.compress_suffix = '' + self.archive_compress = False + + cfs_compress = test_env.get('PG_PROBACKUP_CFS_COMPRESS', None) + if cfs_compress: + self.cfs_compress = cfs_compress.lower() + else: + self.cfs_compress = self.archive_compress + + os.environ["PGAPPNAME"] = "pg_probackup" + self.delete_logs = delete_logs + + def test_env(self): + return self._test_env.copy() + + +init_params = Init() diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/storage/__init__.py b/testgres/plugins/pg_probackup2/pg_probackup2/storage/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/storage/fs_backup.py b/testgres/plugins/pg_probackup2/pg_probackup2/storage/fs_backup.py new file mode 100644 index 00000000..d076432a --- /dev/null +++ b/testgres/plugins/pg_probackup2/pg_probackup2/storage/fs_backup.py @@ -0,0 +1,101 @@ +""" +Utilities for accessing pg_probackup backup data on file system. +""" +import os +import shutil + +from ..init_helpers import init_params + + +class TestBackupDir: + + def list_instance_backups(self, instance): + raise NotImplementedError() + + def list_files(self, sub_dir, recursive=False): + raise NotImplementedError() + + def list_dirs(self, sub_dir): + raise NotImplementedError() + + def read_file(self, sub_path, *, text=True): + raise NotImplementedError() + + def write_file(self, sub_path, data, *, text=True): + raise NotImplementedError() + + def cleanup(self): + raise NotImplementedError() + + def remove_file(self, sub_path): + raise NotImplementedError() + + def remove_dir(self, sub_path): + raise NotImplementedError() + + def exists(self, sub_path): + raise NotImplementedError() + + +class FSTestBackupDir(TestBackupDir): + is_file_based = True + + """ Backup directory. Usually created by running pg_probackup init -B """ + + def __init__(self, *, rel_path, backup): + self.path = os.path.join(init_params.tmp_path, rel_path, backup) + self.pb_args = ('-B', self.path) + + def list_instance_backups(self, instance): + full_path = os.path.join(self.path, 'backups', instance) + return sorted((x for x in os.listdir(full_path) + if os.path.isfile(os.path.join(full_path, x, 'backup.control')))) + + def list_files(self, sub_dir, recursive=False): + full_path = os.path.join(self.path, sub_dir) + if not recursive: + return [f for f in os.listdir(full_path) + if os.path.isfile(os.path.join(full_path, f))] + files = [] + for rootdir, dirs, files_in_dir in os.walk(full_path): + rootdir = rootdir[len(self.path) + 1:] + files.extend(os.path.join(rootdir, file) for file in files_in_dir) + return files + + def list_dirs(self, sub_dir): + full_path = os.path.join(self.path, sub_dir) + return [f for f in os.listdir(full_path) + if os.path.isdir(os.path.join(full_path, f))] + + def read_file(self, sub_path, *, text=True): + full_path = os.path.join(self.path, sub_path) + with open(full_path, 'r' if text else 'rb') as fin: + return fin.read() + + def write_file(self, sub_path, data, *, text=True): + full_path = os.path.join(self.path, sub_path) + with open(full_path, 'w' if text else 'wb') as fout: + fout.write(data) + + def cleanup(self): + shutil.rmtree(self.path, ignore_errors=True) + + def remove_file(self, sub_path): + os.remove(os.path.join(self.path, sub_path)) + + def remove_dir(self, sub_path): + full_path = os.path.join(self.path, sub_path) + shutil.rmtree(full_path, ignore_errors=True) + + def exists(self, sub_path): + full_path = os.path.join(self.path, sub_path) + return os.path.exists(full_path) + + def __str__(self): + return self.path + + def __repr__(self): + return "FSTestBackupDir" + str(self.path) + + def __fspath__(self): + return self.path diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/storage/s3_backup.py b/testgres/plugins/pg_probackup2/pg_probackup2/storage/s3_backup.py new file mode 100644 index 00000000..c6b764fb --- /dev/null +++ b/testgres/plugins/pg_probackup2/pg_probackup2/storage/s3_backup.py @@ -0,0 +1,134 @@ +import os +import io +import sys + +import minio +from minio import Minio +from minio.deleteobjects import DeleteObject +import urllib3 +from .fs_backup import TestBackupDir + +root = os.path.realpath(os.path.join(os.path.dirname(__file__), '../..')) +if root not in sys.path: + sys.path.append(root) + +# Should fail if either of env vars does not exist +host = os.environ['PG_PROBACKUP_S3_HOST'] +port = os.environ['PG_PROBACKUP_S3_PORT'] +access = os.environ['PG_PROBACKUP_S3_ACCESS_KEY'] +secret = os.environ['PG_PROBACKUP_S3_SECRET_ACCESS_KEY'] +bucket = os.environ['PG_PROBACKUP_S3_BUCKET_NAME'] +path_suffix = os.environ.get("PG_PROBACKUP_TEST_TMP_SUFFIX") +https = os.environ.get("PG_PROBACKUP_S3_HTTPS") + +s3_type = os.environ.get('PG_PROBACKUP_S3_TEST', os.environ.get('PROBACKUP_S3_TYPE_FULL_TEST')) +tmp_path = os.environ.get('PGPROBACKUP_TMP_DIR', default='') + +status_forcelist = [413, # RequestBodyTooLarge + 429, # TooManyRequests + 500, # InternalError + 503, # ServerBusy + ] + + +class S3TestBackupDir(TestBackupDir): + is_file_based = False + + def __init__(self, *, rel_path, backup): + path = "pg_probackup" + if path_suffix: + path += "_" + path_suffix + if tmp_path == '' or os.path.isabs(tmp_path): + self.path = f"{path}{tmp_path}/{rel_path}/{backup}" + else: + self.path = f"{path}/{tmp_path}/{rel_path}/{backup}" + + secure: bool = False + if https in ['ON', 'HTTPS']: + secure = True + self.conn = Minio(host + ":" + port, secure=secure, access_key=access, + secret_key=secret, http_client=urllib3.PoolManager(retries=urllib3.Retry(total=5, + backoff_factor=1, + status_forcelist=status_forcelist))) + if not self.conn.bucket_exists(bucket): + raise Exception(f"Test bucket {bucket} does not exist.") + self.pb_args = ('-B', '/' + self.path, f'--s3={s3_type}') + return + + def list_instance_backups(self, instance): + full_path = os.path.join(self.path, 'backups', instance) + candidates = self.conn.list_objects(bucket, prefix=full_path, recursive=True) + return [os.path.basename(os.path.dirname(x.object_name)) + for x in candidates if x.object_name.endswith('backup.control')] + + def list_files(self, sub_dir, recursive=False): + full_path = os.path.join(self.path, sub_dir) + # Need '/' in the end to find inside the folder + full_path_dir = full_path if full_path[-1] == '/' else full_path + '/' + object_list = self.conn.list_objects(bucket, prefix=full_path_dir, recursive=recursive) + return [obj.object_name.replace(full_path_dir, '', 1) + for obj in object_list + if not obj.is_dir] + + def list_dirs(self, sub_dir): + full_path = os.path.join(self.path, sub_dir) + # Need '/' in the end to find inside the folder + full_path_dir = full_path if full_path[-1] == '/' else full_path + '/' + object_list = self.conn.list_objects(bucket, prefix=full_path_dir, recursive=False) + return [obj.object_name.replace(full_path_dir, '', 1).rstrip('\\/') + for obj in object_list + if obj.is_dir] + + def read_file(self, sub_path, *, text=True): + full_path = os.path.join(self.path, sub_path) + bytes = self.conn.get_object(bucket, full_path).read() + if not text: + return bytes + return bytes.decode('utf-8') + + def write_file(self, sub_path, data, *, text=True): + full_path = os.path.join(self.path, sub_path) + if text: + data = data.encode('utf-8') + self.conn.put_object(bucket, full_path, io.BytesIO(data), length=len(data)) + + def cleanup(self): + self.remove_dir('') + + def remove_file(self, sub_path): + full_path = os.path.join(self.path, sub_path) + self.conn.remove_object(bucket, full_path) + + def remove_dir(self, sub_path): + if sub_path: + full_path = os.path.join(self.path, sub_path) + else: + full_path = self.path + objs = self.conn.list_objects(bucket, prefix=full_path, recursive=True, + include_version=True) + delobjs = (DeleteObject(o.object_name, o.version_id) for o in objs) + errs = list(self.conn.remove_objects(bucket, delobjs)) + if errs: + strerrs = "; ".join(str(err) for err in errs) + raise Exception("There were errors: {0}".format(strerrs)) + + def exists(self, sub_path): + full_path = os.path.join(self.path, sub_path) + try: + self.conn.stat_object(bucket, full_path) + return True + except minio.error.S3Error as s3err: + if s3err.code == 'NoSuchKey': + return False + raise s3err + except Exception as err: + raise err + + def __str__(self): + return '/' + self.path + + def __repr__(self): + return "S3TestBackupDir" + str(self.path) + + def __fspath__(self): + return self.path diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/tests/basic_test.py b/testgres/plugins/pg_probackup2/pg_probackup2/tests/basic_test.py new file mode 100644 index 00000000..f5a82d38 --- /dev/null +++ b/testgres/plugins/pg_probackup2/pg_probackup2/tests/basic_test.py @@ -0,0 +1,79 @@ +import os +import shutil +import unittest +import testgres +from pg_probackup2.app import ProbackupApp +from pg_probackup2.init_helpers import Init, init_params +from pg_probackup2.app import build_backup_dir + + +class TestUtils: + @staticmethod + def get_module_and_function_name(test_id): + try: + module_name = test_id.split('.')[-2] + fname = test_id.split('.')[-1] + except IndexError: + print(f"Couldn't get module name and function name from test_id: `{test_id}`") + module_name, fname = test_id.split('(')[1].split('.')[1], test_id.split('(')[0] + return module_name, fname + + +class ProbackupTest(unittest.TestCase): + def setUp(self): + self.setup_test_environment() + self.setup_test_paths() + self.setup_backup_dir() + self.setup_probackup() + + def setup_test_environment(self): + self.output = None + self.cmd = None + self.nodes_to_cleanup = [] + self.module_name, self.fname = TestUtils.get_module_and_function_name(self.id()) + self.test_env = Init().test_env() + + def setup_test_paths(self): + self.rel_path = os.path.join(self.module_name, self.fname) + self.test_path = os.path.join(init_params.tmp_path, self.rel_path) + os.makedirs(self.test_path) + self.pb_log_path = os.path.join(self.test_path, "pb_log") + + def setup_backup_dir(self): + self.backup_dir = build_backup_dir(self, 'backup') + self.backup_dir.cleanup() + + def setup_probackup(self): + self.pg_node = testgres.NodeApp(self.test_path, self.nodes_to_cleanup) + self.pb = ProbackupApp(self, self.pg_node, self.pb_log_path, self.test_env, + auto_compress_alg='zlib', backup_dir=self.backup_dir) + + def tearDown(self): + if os.path.exists(self.test_path): + shutil.rmtree(self.test_path) + + +class BasicTest(ProbackupTest): + def test_full_backup(self): + # Setting up a simple test node + node = self.pg_node.make_simple('node', pg_options={"fsync": "off", "synchronous_commit": "off"}) + + # Initialize and configure Probackup + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) + + # Start the node and initialize pgbench + node.slow_start() + node.pgbench_init(scale=100, no_vacuum=True) + + # Perform backup and validation + backup_id = self.pb.backup_node('node', node) + out = self.pb.validate('node', backup_id) + + # Check if the backup is valid + self.assertIn(f"INFO: Backup {backup_id} is valid", out) + + +if __name__ == "__main__": + unittest.main() diff --git a/testgres/plugins/pg_probackup2/setup.py b/testgres/plugins/pg_probackup2/setup.py new file mode 100644 index 00000000..371eb078 --- /dev/null +++ b/testgres/plugins/pg_probackup2/setup.py @@ -0,0 +1,18 @@ +try: + from setuptools import setup +except ImportError: + from distutils.core import setup + +setup( + version='0.0.1', + name='testgres_pg_probackup2', + packages=['pg_probackup2', 'pg_probackup2.storage'], + description='Plugin for testgres that manages pg_probackup2', + url='https://github.com/postgrespro/testgres', + long_description_content_type='text/markdown', + license='PostgreSQL', + author='Postgres Professional', + author_email='testgres@postgrespro.ru', + keywords=['pg_probackup', 'testing', 'testgres'], + install_requires=['testgres>=1.9.2'] +) From 0edc937b08c7b562c8b16626d445b0b2a5d78f1a Mon Sep 17 00:00:00 2001 From: Victoria Shepard <5807469+demonolock@users.noreply.github.com> Date: Thu, 18 Jan 2024 15:44:47 +0100 Subject: [PATCH 267/475] Fix get_pg_version for linux mint (#101) issue #100 --- testgres/utils.py | 15 +++++++++------ tests/test_simple.py | 12 +++++++++++- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/testgres/utils.py b/testgres/utils.py index d84bb2b5..745a2555 100644 --- a/testgres/utils.py +++ b/testgres/utils.py @@ -177,20 +177,23 @@ def get_pg_version(bin_dir=None): Return PostgreSQL version provided by postmaster. """ - # get raw version (e.g. postgres (PostgreSQL) 9.5.7) + # Get raw version (e.g., postgres (PostgreSQL) 9.5.7) postgres_path = os.path.join(bin_dir, 'postgres') if bin_dir else get_bin_path('postgres') _params = [postgres_path, '--version'] raw_ver = tconf.os_ops.exec_command(_params, encoding='utf-8') - # Remove "(Homebrew)" if present - raw_ver = raw_ver.replace('(Homebrew)', '').strip() + return parse_pg_version(raw_ver) - # cook version of PostgreSQL - version = raw_ver.strip().split(' ')[-1] \ + +def parse_pg_version(version_out): + # Generalize removal of system-specific suffixes (anything in parentheses) + raw_ver = re.sub(r'\([^)]*\)', '', version_out).strip() + + # Cook version of PostgreSQL + version = raw_ver.split(' ')[-1] \ .partition('devel')[0] \ .partition('beta')[0] \ .partition('rc')[0] - return version diff --git a/tests/test_simple.py b/tests/test_simple.py index a013f478..8cb0d94e 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -48,7 +48,7 @@ # NOTE: those are ugly imports from testgres import bound_ports -from testgres.utils import PgVer +from testgres.utils import PgVer, parse_pg_version from testgres.node import ProcessProxy @@ -1023,6 +1023,16 @@ def test_upgrade_node(self): node_new.start() self.assertTrue(b'Upgrade Complete' in res) + def test_parse_pg_version(self): + # Linux Mint + assert parse_pg_version("postgres (PostgreSQL) 15.5 (Ubuntu 15.5-1.pgdg22.04+1)") == "15.5" + # Linux Ubuntu + assert parse_pg_version("postgres (PostgreSQL) 12.17") == "12.17" + # Windows + assert parse_pg_version("postgres (PostgreSQL) 11.4") == "11.4" + # Macos + assert parse_pg_version("postgres (PostgreSQL) 14.9 (Homebrew)") == "14.9" + if __name__ == '__main__': if os.environ.get('ALT_CONFIG'): From 992f0c8415d92625ffa6d8a1b2186c79791ed91e Mon Sep 17 00:00:00 2001 From: vshepard Date: Tue, 30 Jan 2024 07:23:05 +0100 Subject: [PATCH 268/475] Move file control for tests --- .../pg_probackup2/pg_probackup2/app.py | 14 +- .../pg_probackup2/storage/s3_backup.py | 134 ------------------ 2 files changed, 3 insertions(+), 145 deletions(-) delete mode 100644 testgres/plugins/pg_probackup2/pg_probackup2/storage/s3_backup.py diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/app.py b/testgres/plugins/pg_probackup2/pg_probackup2/app.py index 2c31de51..6b176488 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/app.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/app.py @@ -37,6 +37,9 @@ def __str__(self): return '\n ERROR: {0}\n CMD: {1}'.format(repr(self.message), self.cmd) +# Local or S3 backup +fs_backup_class = FSTestBackupDir + class ProbackupApp: def __init__(self, test_class: unittest.TestCase, @@ -747,16 +750,5 @@ def load_backup_class(fs_type): return getattr(module, class_name) - -# Local or S3 backup -fs_backup_class = FSTestBackupDir -if os.environ.get('PG_PROBACKUP_S3_TEST', os.environ.get('PROBACKUP_S3_TYPE_FULL_TEST')): - root = os.path.realpath(os.path.join(os.path.dirname(__file__), '../..')) - if root not in sys.path: - sys.path.append(root) - from pg_probackup2.storage.s3_backup import S3TestBackupDir - - fs_backup_class = S3TestBackupDir - def build_backup_dir(self, backup='backup'): return fs_backup_class(rel_path=self.rel_path, backup=backup) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/storage/s3_backup.py b/testgres/plugins/pg_probackup2/pg_probackup2/storage/s3_backup.py deleted file mode 100644 index c6b764fb..00000000 --- a/testgres/plugins/pg_probackup2/pg_probackup2/storage/s3_backup.py +++ /dev/null @@ -1,134 +0,0 @@ -import os -import io -import sys - -import minio -from minio import Minio -from minio.deleteobjects import DeleteObject -import urllib3 -from .fs_backup import TestBackupDir - -root = os.path.realpath(os.path.join(os.path.dirname(__file__), '../..')) -if root not in sys.path: - sys.path.append(root) - -# Should fail if either of env vars does not exist -host = os.environ['PG_PROBACKUP_S3_HOST'] -port = os.environ['PG_PROBACKUP_S3_PORT'] -access = os.environ['PG_PROBACKUP_S3_ACCESS_KEY'] -secret = os.environ['PG_PROBACKUP_S3_SECRET_ACCESS_KEY'] -bucket = os.environ['PG_PROBACKUP_S3_BUCKET_NAME'] -path_suffix = os.environ.get("PG_PROBACKUP_TEST_TMP_SUFFIX") -https = os.environ.get("PG_PROBACKUP_S3_HTTPS") - -s3_type = os.environ.get('PG_PROBACKUP_S3_TEST', os.environ.get('PROBACKUP_S3_TYPE_FULL_TEST')) -tmp_path = os.environ.get('PGPROBACKUP_TMP_DIR', default='') - -status_forcelist = [413, # RequestBodyTooLarge - 429, # TooManyRequests - 500, # InternalError - 503, # ServerBusy - ] - - -class S3TestBackupDir(TestBackupDir): - is_file_based = False - - def __init__(self, *, rel_path, backup): - path = "pg_probackup" - if path_suffix: - path += "_" + path_suffix - if tmp_path == '' or os.path.isabs(tmp_path): - self.path = f"{path}{tmp_path}/{rel_path}/{backup}" - else: - self.path = f"{path}/{tmp_path}/{rel_path}/{backup}" - - secure: bool = False - if https in ['ON', 'HTTPS']: - secure = True - self.conn = Minio(host + ":" + port, secure=secure, access_key=access, - secret_key=secret, http_client=urllib3.PoolManager(retries=urllib3.Retry(total=5, - backoff_factor=1, - status_forcelist=status_forcelist))) - if not self.conn.bucket_exists(bucket): - raise Exception(f"Test bucket {bucket} does not exist.") - self.pb_args = ('-B', '/' + self.path, f'--s3={s3_type}') - return - - def list_instance_backups(self, instance): - full_path = os.path.join(self.path, 'backups', instance) - candidates = self.conn.list_objects(bucket, prefix=full_path, recursive=True) - return [os.path.basename(os.path.dirname(x.object_name)) - for x in candidates if x.object_name.endswith('backup.control')] - - def list_files(self, sub_dir, recursive=False): - full_path = os.path.join(self.path, sub_dir) - # Need '/' in the end to find inside the folder - full_path_dir = full_path if full_path[-1] == '/' else full_path + '/' - object_list = self.conn.list_objects(bucket, prefix=full_path_dir, recursive=recursive) - return [obj.object_name.replace(full_path_dir, '', 1) - for obj in object_list - if not obj.is_dir] - - def list_dirs(self, sub_dir): - full_path = os.path.join(self.path, sub_dir) - # Need '/' in the end to find inside the folder - full_path_dir = full_path if full_path[-1] == '/' else full_path + '/' - object_list = self.conn.list_objects(bucket, prefix=full_path_dir, recursive=False) - return [obj.object_name.replace(full_path_dir, '', 1).rstrip('\\/') - for obj in object_list - if obj.is_dir] - - def read_file(self, sub_path, *, text=True): - full_path = os.path.join(self.path, sub_path) - bytes = self.conn.get_object(bucket, full_path).read() - if not text: - return bytes - return bytes.decode('utf-8') - - def write_file(self, sub_path, data, *, text=True): - full_path = os.path.join(self.path, sub_path) - if text: - data = data.encode('utf-8') - self.conn.put_object(bucket, full_path, io.BytesIO(data), length=len(data)) - - def cleanup(self): - self.remove_dir('') - - def remove_file(self, sub_path): - full_path = os.path.join(self.path, sub_path) - self.conn.remove_object(bucket, full_path) - - def remove_dir(self, sub_path): - if sub_path: - full_path = os.path.join(self.path, sub_path) - else: - full_path = self.path - objs = self.conn.list_objects(bucket, prefix=full_path, recursive=True, - include_version=True) - delobjs = (DeleteObject(o.object_name, o.version_id) for o in objs) - errs = list(self.conn.remove_objects(bucket, delobjs)) - if errs: - strerrs = "; ".join(str(err) for err in errs) - raise Exception("There were errors: {0}".format(strerrs)) - - def exists(self, sub_path): - full_path = os.path.join(self.path, sub_path) - try: - self.conn.stat_object(bucket, full_path) - return True - except minio.error.S3Error as s3err: - if s3err.code == 'NoSuchKey': - return False - raise s3err - except Exception as err: - raise err - - def __str__(self): - return '/' + self.path - - def __repr__(self): - return "S3TestBackupDir" + str(self.path) - - def __fspath__(self): - return self.path From 979671de764f2832d502a1e93405aff165ce78dc Mon Sep 17 00:00:00 2001 From: Victoria Shepard <5807469+demonolock@users.noreply.github.com> Date: Tue, 30 Jan 2024 08:31:04 +0100 Subject: [PATCH 269/475] Move file control for tests (#106) --- testgres/plugins/__init__.py | 3 +-- testgres/plugins/pg_probackup2/pg_probackup2/app.py | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/testgres/plugins/__init__.py b/testgres/plugins/__init__.py index e60331f0..8c19a23b 100644 --- a/testgres/plugins/__init__.py +++ b/testgres/plugins/__init__.py @@ -2,8 +2,7 @@ from pg_probackup2.app import ProbackupApp, ProbackupException from pg_probackup2.init_helpers import init_params from pg_probackup2.storage.fs_backup import FSTestBackupDir -from pg_probackup2.storage.s3_backup import S3TestBackupDir __all__ = [ - "ProbackupApp", "ProbackupException", "init_params", "FSTestBackupDir", "S3TestBackupDir", "GDBobj" + "ProbackupApp", "ProbackupException", "init_params", "FSTestBackupDir", "GDBobj" ] diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/app.py b/testgres/plugins/pg_probackup2/pg_probackup2/app.py index 6b176488..a8050f94 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/app.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/app.py @@ -4,7 +4,6 @@ import os import re import subprocess -import sys import threading import time import unittest @@ -37,9 +36,10 @@ def __str__(self): return '\n ERROR: {0}\n CMD: {1}'.format(repr(self.message), self.cmd) -# Local or S3 backup +# Local backup control fs_backup_class = FSTestBackupDir + class ProbackupApp: def __init__(self, test_class: unittest.TestCase, From a078792ccb11854bb0959d84ee5a93a0e956a33b Mon Sep 17 00:00:00 2001 From: z-kasymalieva <149158086+z-kasymalieva@users.noreply.github.com> Date: Tue, 30 Jan 2024 16:34:08 +0300 Subject: [PATCH 270/475] Fix del_instance function to receive options from tests (#105) --- .../plugins/pg_probackup2/pg_probackup2/app.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/app.py b/testgres/plugins/pg_probackup2/pg_probackup2/app.py index a8050f94..40524864 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/app.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/app.py @@ -192,15 +192,13 @@ def set_backup(self, instance, backup_id=False, return self.run(cmd + options, old_binary=old_binary, expect_error=expect_error) - def del_instance(self, instance, old_binary=False, expect_error=False): - - return self.run([ - 'del-instance', - '--instance={0}'.format(instance), - ], - old_binary=old_binary, - expect_error=expect_error - ) + def del_instance(self, instance, options=None, old_binary=False, expect_error=False): + if options is None: + options = [] + cmd = ['del-instance', '--instance={0}'.format(instance)] + options + return self.run(cmd, + old_binary=old_binary, + expect_error=expect_error) def backup_node( self, instance, node, data_dir=False, From 40aa655985debd8ccc3d6d9e13899a31f89834cc Mon Sep 17 00:00:00 2001 From: Oleg Gurev Date: Tue, 30 Jan 2024 16:39:28 +0300 Subject: [PATCH 271/475] set default test locale to en (#107) --- testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py index 7af21eb6..23777e92 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py @@ -52,6 +52,7 @@ def __init__(self): parts[0] = re.match(r'\d+', parts[0]).group() self.pg_config_version = reduce(lambda v, x: v * 100 + int(x), parts, 0) + os.environ['LANGUAGE'] = 'en' # set default locale language to en. All messages will use this locale test_env = os.environ.copy() envs_list = [ 'LANGUAGE', From 70d2f276a6ceda42b7d66b6937e2857d5332f4c6 Mon Sep 17 00:00:00 2001 From: z-kasymalieva <149158086+z-kasymalieva@users.noreply.github.com> Date: Fri, 2 Feb 2024 13:34:13 +0300 Subject: [PATCH 272/475] fix Error executing query (#110) --- testgres/connection.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/testgres/connection.py b/testgres/connection.py index 882498a9..49b74844 100644 --- a/testgres/connection.py +++ b/testgres/connection.py @@ -104,13 +104,13 @@ def rollback(self): def execute(self, query, *args): self.cursor.execute(query, args) try: - res = self.cursor.fetchall() # pg8000 might return tuples - if isinstance(res, tuple): - res = [tuple(t) for t in res] - + res = [tuple(t) for t in self.cursor.fetchall()] return res - except Exception: + except ProgrammingError: + return None + except Exception as e: + print("Error executing query: {}\n {}".format(repr(e), query)) return None def close(self): From 6424451bdc967a090ad4fb1ca155f59418a5269d Mon Sep 17 00:00:00 2001 From: "z.kasymalieva" Date: Fri, 2 Feb 2024 17:27:01 +0300 Subject: [PATCH 273/475] [create_archive_push] archive_push_command in testgres_pg_probackup added --- .../plugins/pg_probackup2/pg_probackup2/app.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/app.py b/testgres/plugins/pg_probackup2/pg_probackup2/app.py index 40524864..2c9953e6 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/app.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/app.py @@ -748,5 +748,19 @@ def load_backup_class(fs_type): return getattr(module, class_name) + def archive_push(self, instance, node, wal_file_name, wal_file_path=None, options=None, expect_error=False): + if options is None: + options = [] + cmd = [ + 'archive-push', + '--instance={0}'.format(instance), + '--wal-file-name={0}'.format(wal_file_name), + ] + if wal_file_path is None: + cmd = cmd + ['--wal-file-path={0}'.format(os.path.join(node.data_dir, 'pg_wal'))] + else: + cmd = cmd + ['--wal-file-path={0}'.format(wal_file_path)] + return self.run(cmd + options, expect_error=expect_error) + def build_backup_dir(self, backup='backup'): return fs_backup_class(rel_path=self.rel_path, backup=backup) From bd37658ef3f615a3ab78231361583f08178fc1c6 Mon Sep 17 00:00:00 2001 From: Victoria Shepard <5807469+demonolock@users.noreply.github.com> Date: Mon, 12 Feb 2024 15:50:48 +0100 Subject: [PATCH 274/475] Abs backup path (#108) --- .../plugins/pg_probackup2/pg_probackup2/storage/fs_backup.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/storage/fs_backup.py b/testgres/plugins/pg_probackup2/pg_probackup2/storage/fs_backup.py index d076432a..6c9d1463 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/storage/fs_backup.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/storage/fs_backup.py @@ -43,7 +43,10 @@ class FSTestBackupDir(TestBackupDir): """ Backup directory. Usually created by running pg_probackup init -B """ def __init__(self, *, rel_path, backup): - self.path = os.path.join(init_params.tmp_path, rel_path, backup) + backup_prefix = os.environ.get('PG_PROBACKUP_TEST_BACKUP_DIR_PREFIX') + if backup_prefix and not os.path.isabs(backup_prefix): + raise Exception(f"PG_PROBACKUP_TEST_BACKUP_DIR_PREFIX must be an absolute path, current value: {backup_prefix}") + self.path = os.path.join(backup_prefix or init_params.tmp_path, rel_path, backup) self.pb_args = ('-B', self.path) def list_instance_backups(self, instance): From 62238ac3db4920f9c1b0caffee22e34cbe6e2aeb Mon Sep 17 00:00:00 2001 From: asavchkov Date: Tue, 13 Feb 2024 15:54:06 +0700 Subject: [PATCH 275/475] Describe the test backup directory prefix --- testgres/plugins/pg_probackup2/README.md | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/testgres/plugins/pg_probackup2/README.md b/testgres/plugins/pg_probackup2/README.md index b62bf24b..c45f2e3a 100644 --- a/testgres/plugins/pg_probackup2/README.md +++ b/testgres/plugins/pg_probackup2/README.md @@ -16,22 +16,14 @@ The package requires testgres~=1.9.3. ## Usage -### Environment +### Environment variables -> Note: by default testgres runs `initdb`, `pg_ctl`, `psql` provided by `PATH`. - -There are several ways to specify a custom postgres installation: - -* export `PG_CONFIG` environment variable pointing to the `pg_config` executable; -* export `PG_BIN` environment variable pointing to the directory with executable files. - -Example: - -```bash -export PG_BIN=$HOME/pg/bin -python my_tests.py -``` +| Variable | Required | Default value | Description | +| - | - | - | - | +| PGPROBACKUP_TMP_DIR | No | tmp_dirs | The root of the temporary directory hierarchy where tests store data and logs. Relative paths start from the `tests` directory. | +| PG_PROBACKUP_TEST_BACKUP_DIR_PREFIX | No | Temporary test hierarchy | Prefix of the test backup directories. Must be an absolute path. Use this variable to store test backups in a location other than the temporary test hierarchy. | +See [Testgres](https://github.com/postgrespro/testgres/tree/master#environment) on how to configure a custom Postgres installation using `PG_CONFIG` and `PG_BIN` environment variables. ### Examples From d98cee279af487f7bcf588ddae73af63052e4958 Mon Sep 17 00:00:00 2001 From: asavchkov Date: Wed, 14 Feb 2024 19:34:27 +0700 Subject: [PATCH 276/475] Update the Readme of the pg_probackup2 package --- testgres/plugins/pg_probackup2/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testgres/plugins/pg_probackup2/README.md b/testgres/plugins/pg_probackup2/README.md index c45f2e3a..5139ab0f 100644 --- a/testgres/plugins/pg_probackup2/README.md +++ b/testgres/plugins/pg_probackup2/README.md @@ -20,7 +20,7 @@ The package requires testgres~=1.9.3. | Variable | Required | Default value | Description | | - | - | - | - | -| PGPROBACKUP_TMP_DIR | No | tmp_dirs | The root of the temporary directory hierarchy where tests store data and logs. Relative paths start from the `tests` directory. | +| PGPROBACKUP_TMP_DIR | No | tests/tmp_dirs | The root of the temporary directory hierarchy where tests store data and logs. Relative paths start from the current working directory. | | PG_PROBACKUP_TEST_BACKUP_DIR_PREFIX | No | Temporary test hierarchy | Prefix of the test backup directories. Must be an absolute path. Use this variable to store test backups in a location other than the temporary test hierarchy. | See [Testgres](https://github.com/postgrespro/testgres/tree/master#environment) on how to configure a custom Postgres installation using `PG_CONFIG` and `PG_BIN` environment variables. From 29d41fba1fd5a235afe9d3d50b98bd6ae906dd99 Mon Sep 17 00:00:00 2001 From: vshepard Date: Thu, 15 Feb 2024 08:04:43 +0100 Subject: [PATCH 277/475] Add s3 env vars --- testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py index 23777e92..f81386aa 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py @@ -201,6 +201,10 @@ def __init__(self): os.environ["PGAPPNAME"] = "pg_probackup" self.delete_logs = delete_logs + # s3 params + self.s3_config_file = test_env.get('PG_PROBACKUP_S3_CONFIG_FILE') + self.s3_type = test_env.get('PG_PROBACKUP_S3_TEST') + def test_env(self): return self._test_env.copy() From 6b15b7b5660850da51528c91aafb6b3101429094 Mon Sep 17 00:00:00 2001 From: z-kasymalieva <149158086+z-kasymalieva@users.noreply.github.com> Date: Tue, 27 Feb 2024 08:23:00 +0300 Subject: [PATCH 278/475] [create_archive_get] Function testgres_pg_probackup2 archive-get added (#112) --- testgres/plugins/pg_probackup2/pg_probackup2/app.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/app.py b/testgres/plugins/pg_probackup2/pg_probackup2/app.py index 2c9953e6..07825673 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/app.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/app.py @@ -762,5 +762,16 @@ def archive_push(self, instance, node, wal_file_name, wal_file_path=None, option cmd = cmd + ['--wal-file-path={0}'.format(wal_file_path)] return self.run(cmd + options, expect_error=expect_error) + def archive_get(self, instance, wal_file_name, wal_file_path, options=None, expect_error=False): + if options is None: + options = [] + cmd = [ + 'archive-get', + '--instance={0}'.format(instance), + '--wal-file-name={0}'.format(wal_file_name), + '--wal-file-path={0}'.format(wal_file_path), + ] + return self.run(cmd + options, expect_error=expect_error) + def build_backup_dir(self, backup='backup'): return fs_backup_class(rel_path=self.rel_path, backup=backup) From 2baaf7c5a05207d4f26fa378b532cbbdd3b1772c Mon Sep 17 00:00:00 2001 From: asavchkov <79832668+asavchkov@users.noreply.github.com> Date: Thu, 7 Mar 2024 04:13:12 +0700 Subject: [PATCH 279/475] Run the archive command through exec (#113) * Run the archive command through exec * Invoke exec only on Linux --- testgres/plugins/pg_probackup2/pg_probackup2/app.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/app.py b/testgres/plugins/pg_probackup2/pg_probackup2/app.py index 07825673..94dcd997 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/app.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/app.py @@ -663,7 +663,11 @@ def set_archiving( if custom_archive_command is None: archive_command = " ".join([f'"{init_params.probackup_path}"', 'archive-push', *self.backup_dir.pb_args]) - if os.name == "nt": + if os.name == 'posix': + # Dash produces a core dump when it gets a SIGQUIT from its + # child process so replace the shell with pg_probackup + archive_command = 'exec ' + archive_command + elif os.name == "nt": archive_command = archive_command.replace("\\", "\\\\") archive_command += f' --instance={instance}' From e4a6a471da8b215b1b07a94a5aa0841b632c34aa Mon Sep 17 00:00:00 2001 From: Victoria Shepard <5807469+demonolock@users.noreply.github.com> Date: Thu, 7 Mar 2024 00:04:45 +0100 Subject: [PATCH 280/475] Add print commands from run (#114) Co-authored-by: vshepard --- testgres/plugins/pg_probackup2/pg_probackup2/app.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/app.py b/testgres/plugins/pg_probackup2/pg_probackup2/app.py index 94dcd997..4e6e91ff 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/app.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/app.py @@ -1,6 +1,7 @@ import contextlib import importlib import json +import logging import os import re import subprocess @@ -101,6 +102,7 @@ def run(self, command, gdb=False, old_binary=False, return_id=True, env=None, print(self.test_class.cmd) cmdline = [binary_path, *command] + logging.info(' '.join(cmdline)) if gdb is True: # general test flow for using GDBObj return GDBobj(cmdline, self.test_class) From 8436008a5459d530bb9686212517d592ffcd75e8 Mon Sep 17 00:00:00 2001 From: Victoria Shepard <5807469+demonolock@users.noreply.github.com> Date: Mon, 11 Mar 2024 10:58:11 +0100 Subject: [PATCH 281/475] Remove init param (#115) --- .../pg_probackup2/init_helpers.py | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py index f81386aa..f392d1b9 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py @@ -148,21 +148,24 @@ def __init__(self): [self.probackup_path, "--version"], stderr=subprocess.STDOUT, ).decode('utf-8') - self.probackup_version = re.search(r"\d+\.\d+\.\d+", - probackup_version_output - ).group(0) - compressions = re.search(r"\(compressions: ([^)]*)\)", - probackup_version_output).group(1) - self.probackup_compressions = {s.strip() for s in compressions.split(',')} + match = re.search(r"\d+\.\d+\.\d+", + probackup_version_output) + self.probackup_version = match.group(0) if match else None + match = re.search(r"\(compressions: ([^)]*)\)", probackup_version_output) + compressions = match.group(1) if match else None + if compressions: + self.probackup_compressions = {s.strip() for s in compressions.split(',')} + else: + self.probackup_compressions = [] if self.probackup_old_path: old_probackup_version_output = subprocess.check_output( [self.probackup_old_path, "--version"], stderr=subprocess.STDOUT, ).decode('utf-8') - self.old_probackup_version = re.search(r"\d+\.\d+\.\d+", - old_probackup_version_output - ).group(0) + match = re.search(r"\d+\.\d+\.\d+", + old_probackup_version_output) + self.old_probackup_version = match.group(0) if match else None self.remote = test_env.get('PGPROBACKUP_SSH_REMOTE', None) == 'ON' self.ptrack = test_env.get('PG_PROBACKUP_PTRACK', None) == 'ON' and self.pg_config_version >= 110000 @@ -202,7 +205,6 @@ def __init__(self): self.delete_logs = delete_logs # s3 params - self.s3_config_file = test_env.get('PG_PROBACKUP_S3_CONFIG_FILE') self.s3_type = test_env.get('PG_PROBACKUP_S3_TEST') def test_env(self): From 19f66abfb1d8f0303d12138ed65f9a8a00d07bda Mon Sep 17 00:00:00 2001 From: Victoria Shepard <5807469+demonolock@users.noreply.github.com> Date: Tue, 12 Mar 2024 11:22:43 +0100 Subject: [PATCH 282/475] Remove logging in run (#116) --- testgres/plugins/pg_probackup2/pg_probackup2/app.py | 2 -- testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py | 3 --- 2 files changed, 5 deletions(-) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/app.py b/testgres/plugins/pg_probackup2/pg_probackup2/app.py index 4e6e91ff..94dcd997 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/app.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/app.py @@ -1,7 +1,6 @@ import contextlib import importlib import json -import logging import os import re import subprocess @@ -102,7 +101,6 @@ def run(self, command, gdb=False, old_binary=False, return_id=True, env=None, print(self.test_class.cmd) cmdline = [binary_path, *command] - logging.info(' '.join(cmdline)) if gdb is True: # general test flow for using GDBObj return GDBobj(cmdline, self.test_class) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py index f392d1b9..e3dd9e4f 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py @@ -204,9 +204,6 @@ def __init__(self): os.environ["PGAPPNAME"] = "pg_probackup" self.delete_logs = delete_logs - # s3 params - self.s3_type = test_env.get('PG_PROBACKUP_S3_TEST') - def test_env(self): return self._test_env.copy() From 356cd5246f596fc26e7b076ea74223a5ff491025 Mon Sep 17 00:00:00 2001 From: Victoria Shepard <5807469+demonolock@users.noreply.github.com> Date: Thu, 14 Mar 2024 09:21:07 +0100 Subject: [PATCH 283/475] Up version testgres_pg_probackup2 --- testgres/plugins/pg_probackup2/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testgres/plugins/pg_probackup2/setup.py b/testgres/plugins/pg_probackup2/setup.py index 371eb078..381d7ae2 100644 --- a/testgres/plugins/pg_probackup2/setup.py +++ b/testgres/plugins/pg_probackup2/setup.py @@ -4,7 +4,7 @@ from distutils.core import setup setup( - version='0.0.1', + version='0.0.2', name='testgres_pg_probackup2', packages=['pg_probackup2', 'pg_probackup2.storage'], description='Plugin for testgres that manages pg_probackup2', From 4c70b80b84da5d0c950b645b28aee85593374239 Mon Sep 17 00:00:00 2001 From: egarbuz <165897130+egarbuz@users.noreply.github.com> Date: Thu, 4 Apr 2024 12:02:39 +0300 Subject: [PATCH 284/475] Adding port and database name into params list for add-instance command (#118) --- testgres/plugins/pg_probackup2/pg_probackup2/app.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/app.py b/testgres/plugins/pg_probackup2/pg_probackup2/app.py index 94dcd997..1ab71109 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/app.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/app.py @@ -149,13 +149,19 @@ def init(self, options=None, old_binary=False, skip_log_directory=False, expect_ use_backup_dir=use_backup_dir ) - def add_instance(self, instance, node, old_binary=False, options=None, expect_error=False): + def add_instance(self, instance, node, old_binary=False, options=None, expect_error=False, datname=False): if options is None: options = [] + + if not datname: + datname = 'postgres' + cmd = [ 'add-instance', '--instance={0}'.format(instance), - '-D', node.data_dir + '-D', node.data_dir, + '--pgport', '%i' % node.port, + '--pgdatabase', datname ] # don`t forget to kill old_binary after remote ssh release From 3dc640fb7d6df7e5bbdd1970376773f3d9b7bd09 Mon Sep 17 00:00:00 2001 From: "e.garbuz" Date: Thu, 4 Apr 2024 14:11:09 +0300 Subject: [PATCH 285/475] Value pg_probackup major_version added to the class Init --- testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py index e3dd9e4f..b6bf90f4 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py @@ -204,6 +204,8 @@ def __init__(self): os.environ["PGAPPNAME"] = "pg_probackup" self.delete_logs = delete_logs + self.major_version = int(self.probackup_version.split('.')[0]) + def test_env(self): return self._test_env.copy() From fba3bd36a6f86c6be42f87f3fb10e873b02dbec3 Mon Sep 17 00:00:00 2001 From: Sofia Kopikova Date: Thu, 4 Apr 2024 19:16:02 +0300 Subject: [PATCH 286/475] Fix error 'Is another postmaster already running on port XXX' Sometimes when we abnormally shutdown node its port stays busy. So I added retry attempts to start() function in case we encounter such error. Test for this case was added (test_the_same_port). --- testgres/node.py | 32 ++++++++++++++++++++++++-------- tests/test_simple.py | 8 ++++++++ 2 files changed, 32 insertions(+), 8 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index 0f1dcf98..c9b34d7b 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -726,14 +726,30 @@ def start(self, params=[], wait=True): "start" ] + params # yapf: disable - try: - exit_status, out, error = execute_utility(_params, self.utils_log_file, verbose=True) - if error and 'does not exist' in error: - raise Exception - except Exception as e: - msg = 'Cannot start node' - files = self._collect_special_files() - raise_from(StartNodeException(msg, files), e) + startup_retries = 5 + while True: + try: + exit_status, out, error = execute_utility(_params, self.utils_log_file, verbose=True) + if error and 'does not exist' in error: + raise Exception + except Exception as e: + files = self._collect_special_files() + if any(len(file) > 1 and 'Is another postmaster already ' + 'running on port' in file[1].decode() for + file in files): + print("Detected an issue with connecting to port {0}. " + "Trying another port after a 5-second sleep...".format(self.port)) + self.port = reserve_port() + options = {} + options['port'] = str(self.port) + self.set_auto_conf(options) + startup_retries -= 1 + time.sleep(5) + continue + + msg = 'Cannot start node' + raise_from(StartNodeException(msg, files), e) + break self._maybe_start_logger() self.is_started = True return self diff --git a/tests/test_simple.py b/tests/test_simple.py index 8cb0d94e..ba23c3ed 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -1033,6 +1033,14 @@ def test_parse_pg_version(self): # Macos assert parse_pg_version("postgres (PostgreSQL) 14.9 (Homebrew)") == "14.9" + def test_the_same_port(self): + with get_new_node() as node: + node.init().start() + + with get_new_node() as node2: + node2.port = node.port + node2.init().start() + if __name__ == '__main__': if os.environ.get('ALT_CONFIG'): From 2a0c37b9e139cd0a2fc40eedc4edff1fa96dbf66 Mon Sep 17 00:00:00 2001 From: "e.garbuz" Date: Mon, 8 Apr 2024 17:57:55 +0300 Subject: [PATCH 287/475] Add check on digit for probackup_version --- testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py index b6bf90f4..109a411e 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py @@ -204,7 +204,9 @@ def __init__(self): os.environ["PGAPPNAME"] = "pg_probackup" self.delete_logs = delete_logs - self.major_version = int(self.probackup_version.split('.')[0]) + self.major_version = 0 + if self.probackup_version.split('.')[0].isdigit: + self.major_version = int(self.probackup_version.split('.')[0]) def test_env(self): return self._test_env.copy() From 5db4ae393e4b4f69b5721b27e9c80fc3dcd5c044 Mon Sep 17 00:00:00 2001 From: "e.garbuz" Date: Tue, 9 Apr 2024 13:01:07 +0300 Subject: [PATCH 288/475] Break when pg_probackup version is incorrect --- testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py index 109a411e..318877af 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py @@ -204,9 +204,11 @@ def __init__(self): os.environ["PGAPPNAME"] = "pg_probackup" self.delete_logs = delete_logs - self.major_version = 0 if self.probackup_version.split('.')[0].isdigit: self.major_version = int(self.probackup_version.split('.')[0]) + else: + print('Pg_probackup version is not correct!') + sys.exit(1) def test_env(self): return self._test_env.copy() From 3a90ecd41293aa08c2db662ce73ecad0f34b8636 Mon Sep 17 00:00:00 2001 From: "e.garbuz" Date: Tue, 9 Apr 2024 13:36:42 +0300 Subject: [PATCH 289/475] Adding print incorrect version --- testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py index 318877af..f8c9f3b3 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py @@ -207,7 +207,7 @@ def __init__(self): if self.probackup_version.split('.')[0].isdigit: self.major_version = int(self.probackup_version.split('.')[0]) else: - print('Pg_probackup version is not correct!') + print('Pg_probackup version \"{}\" is not correct!'.format(self.probackup_version)) sys.exit(1) def test_env(self): From ea05a4afc54a9ed65769921b9221afec378a6df1 Mon Sep 17 00:00:00 2001 From: "e.garbuz" Date: Wed, 10 Apr 2024 09:45:17 +0300 Subject: [PATCH 290/475] Correction of remarks --- testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py index f8c9f3b3..9231ac04 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py @@ -204,10 +204,10 @@ def __init__(self): os.environ["PGAPPNAME"] = "pg_probackup" self.delete_logs = delete_logs - if self.probackup_version.split('.')[0].isdigit: + if self.probackup_version.split('.')[0].isdigit(): self.major_version = int(self.probackup_version.split('.')[0]) else: - print('Pg_probackup version \"{}\" is not correct!'.format(self.probackup_version)) + print('Pg_probackup version \"{}\" is not correct! Expected that the major pg_probackup version should be a number.'.format(self.probackup_version)) sys.exit(1) def test_env(self): From 5b8a8369cbc0517c4aecacfa35b2955d8e9a4a0d Mon Sep 17 00:00:00 2001 From: "e.garbuz" Date: Wed, 10 Apr 2024 14:23:31 +0300 Subject: [PATCH 291/475] Changed error message --- testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py index 9231ac04..73731a6e 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py @@ -207,7 +207,7 @@ def __init__(self): if self.probackup_version.split('.')[0].isdigit(): self.major_version = int(self.probackup_version.split('.')[0]) else: - print('Pg_probackup version \"{}\" is not correct! Expected that the major pg_probackup version should be a number.'.format(self.probackup_version)) + print('Can\'t process pg_probackup version \"{}\": the major version is expected to be a number'.format(self.probackup_version)) sys.exit(1) def test_env(self): From eb0476209523b4809a4a7e20e4f2cd9968dd5cb9 Mon Sep 17 00:00:00 2001 From: Victoria Shepard <5807469+demonolock@users.noreply.github.com> Date: Tue, 7 May 2024 15:53:49 +0300 Subject: [PATCH 292/475] Add options to pg_basebackup (#121) --- testgres/backup.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/testgres/backup.py b/testgres/backup.py index a89e214d..cecb0f7b 100644 --- a/testgres/backup.py +++ b/testgres/backup.py @@ -33,7 +33,8 @@ def __init__(self, node, base_dir=None, username=None, - xlog_method=XLogMethod.fetch): + xlog_method=XLogMethod.fetch, + options=None): """ Create a new backup. @@ -43,6 +44,8 @@ def __init__(self, username: database user name. xlog_method: none | fetch | stream (see docs) """ + if not options: + options = [] self.os_ops = node.os_ops if not node.status(): raise BackupException('Node must be running') @@ -77,6 +80,7 @@ def __init__(self, "-D", data_dir, "-X", xlog_method.value ] # yapf: disable + _params += options execute_utility(_params, self.log_file) def __enter__(self): From 0cf30a7dc0078645052a7fa9afddb5f6999df5d8 Mon Sep 17 00:00:00 2001 From: Victoria Shepard <5807469+demonolock@users.noreply.github.com> Date: Tue, 7 May 2024 20:07:32 +0300 Subject: [PATCH 293/475] Fix port bind (#117) * Fix port bind * Up testgres version - 1.10.1 --- setup.py | 2 +- testgres/operations/remote_ops.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index b006c8bf..412e8823 100755 --- a/setup.py +++ b/setup.py @@ -27,7 +27,7 @@ readme = f.read() setup( - version='1.10.0', + version='1.10.1', name='testgres', packages=['testgres', 'testgres.operations', 'testgres.helpers'], description='Testing utility for PostgreSQL and its extensions', diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index 01251e1c..f182768b 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -4,6 +4,8 @@ import tempfile import platform +from ..utils import reserve_port + # we support both pg8000 and psycopg2 try: import psycopg2 as pglib @@ -392,7 +394,7 @@ def db_connect(self, dbname, user, password=None, host="localhost", port=5432): """ Established SSH tunnel and Connects to a PostgreSQL """ - self.establish_ssh_tunnel(local_port=port, remote_port=5432) + self.establish_ssh_tunnel(local_port=reserve_port(), remote_port=5432) try: conn = pglib.connect( host=host, From 8a25cb375138442c149f93051579a04704bbe38a Mon Sep 17 00:00:00 2001 From: Victoria Shepard <5807469+demonolock@users.noreply.github.com> Date: Tue, 21 May 2024 01:57:29 +0300 Subject: [PATCH 294/475] Add pgbench_with_wait function (#122) --- testgres/node.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/testgres/node.py b/testgres/node.py index c9b34d7b..d1784cb9 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -1371,7 +1371,7 @@ def pgbench(self, username=None, stdout=None, stderr=None, - options=[]): + options=None): """ Spawn a pgbench process. @@ -1385,6 +1385,8 @@ def pgbench(self, Returns: Process created by subprocess.Popen. """ + if options is None: + options = [] # Set default arguments dbname = dbname or default_dbname() @@ -1404,6 +1406,29 @@ def pgbench(self, return proc + def pgbench_with_wait(self, + dbname=None, + username=None, + stdout=None, + stderr=None, + options=None): + """ + Do pgbench command and wait. + + Args: + dbname: database name to connect to. + username: database user name. + stdout: stdout file to be used by Popen. + stderr: stderr file to be used by Popen. + options: additional options for pgbench (list). + """ + if options is None: + options = [] + + with self.pgbench(dbname, username, stdout, stderr, options) as pgbench: + pgbench.wait() + return + def pgbench_init(self, **kwargs): """ Small wrapper for pgbench_run(). From e375302a114cd4df3ceed54d6526f250c44c08e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabr=C3=ADzio=20de=20Royes=20Mello?= Date: Fri, 24 May 2024 17:36:14 -0300 Subject: [PATCH 295/475] Add options to pg_upgrade (#125) --- testgres/node.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/testgres/node.py b/testgres/node.py index d1784cb9..e5e8fd5f 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -1639,7 +1639,7 @@ def set_auto_conf(self, options, config='postgresql.auto.conf', rm_options={}): self.os_ops.write(path, auto_conf, truncate=True) - def upgrade_from(self, old_node): + def upgrade_from(self, old_node, options=None): """ Upgrade this node from an old node using pg_upgrade. @@ -1652,6 +1652,9 @@ def upgrade_from(self, old_node): if not os.path.exists(self.data_dir): self.init() + if not options: + options = [] + pg_upgrade_binary = self._get_bin_path("pg_upgrade") if not os.path.exists(pg_upgrade_binary): @@ -1666,6 +1669,7 @@ def upgrade_from(self, old_node): "--old-port", str(old_node.port), "--new-port", str(self.port), ] + upgrade_command += options return self.os_ops.exec_command(upgrade_command) From 63db2752501f9ac58a05accc686aec3cd2825c20 Mon Sep 17 00:00:00 2001 From: Vyacheslav Makarov <50846161+MakSl@users.noreply.github.com> Date: Tue, 28 May 2024 18:07:38 +0300 Subject: [PATCH 296/475] Added command execution time measurement (#123) This will be useful to keep track of possible performance degradation when code changes. --- testgres/plugins/pg_probackup2/pg_probackup2/app.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/app.py b/testgres/plugins/pg_probackup2/pg_probackup2/app.py index 1ab71109..3d2e0101 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/app.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/app.py @@ -56,6 +56,7 @@ def __init__(self, test_class: unittest.TestCase, self.verbose = init_params.verbose self.archive_compress = init_params.archive_compress self.test_class.output = None + self.execution_time = None def run(self, command, gdb=False, old_binary=False, return_id=True, env=None, skip_log_directory=False, expect_error=False, use_backup_dir=True): @@ -113,11 +114,15 @@ def run(self, command, gdb=False, old_binary=False, return_id=True, env=None, cmdline = ['gdbserver'] + ['localhost:' + str(gdb_port)] + cmdline print("pg_probackup gdb suspended, waiting gdb connection on localhost:{0}".format(gdb_port)) + start_time = time.time() self.test_class.output = subprocess.check_output( cmdline, stderr=subprocess.STDOUT, env=env ).decode('utf-8', errors='replace') + end_time = time.time() + self.execution_time = end_time - start_time + if command[0] == 'backup' and return_id: # return backup ID for line in self.test_class.output.splitlines(): From c1cfc26b4b74c36fd6cccda2ee00fdce60dabc92 Mon Sep 17 00:00:00 2001 From: Victoria Shepard <5807469+demonolock@users.noreply.github.com> Date: Thu, 30 May 2024 11:28:33 +0200 Subject: [PATCH 297/475] Add parsing backup_id in run app.py (#126) --- .../plugins/pg_probackup2/pg_probackup2/app.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/app.py b/testgres/plugins/pg_probackup2/pg_probackup2/app.py index 3d2e0101..1a4ca9e7 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/app.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/app.py @@ -124,10 +124,7 @@ def run(self, command, gdb=False, old_binary=False, return_id=True, env=None, self.execution_time = end_time - start_time if command[0] == 'backup' and return_id: - # return backup ID - for line in self.test_class.output.splitlines(): - if 'INFO: Backup' and 'completed' in line: - result = line.split()[2] + result = self.get_backup_id() else: result = self.test_class.output if expect_error is True: @@ -144,6 +141,19 @@ def run(self, command, gdb=False, old_binary=False, return_id=True, env=None, else: raise ProbackupException(self.test_class.output, self.test_class.cmd) + def get_backup_id(self): + if init_params.major_version > 2: + pattern = re.compile(r"Backup (.*) completed successfully.") + for line in self.test_class.output.splitlines(): + match = pattern.search(line) + if match: + return match.group(1) + else: + for line in self.test_class.output.splitlines(): + if 'INFO: Backup' and 'completed' in line: + return line.split()[2] + return None + def init(self, options=None, old_binary=False, skip_log_directory=False, expect_error=False, use_backup_dir=True): if options is None: options = [] From b4268f8bdaaf67e0f7677c0f801210a0085bb594 Mon Sep 17 00:00:00 2001 From: Victoria Shepard <5807469+demonolock@users.noreply.github.com> Date: Wed, 5 Jun 2024 09:34:35 +0200 Subject: [PATCH 298/475] Revert "Fix port bind (#117)" (#128) This reverts commit 0cf30a7dc0078645052a7fa9afddb5f6999df5d8. --- setup.py | 2 +- testgres/operations/remote_ops.py | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index 412e8823..b006c8bf 100755 --- a/setup.py +++ b/setup.py @@ -27,7 +27,7 @@ readme = f.read() setup( - version='1.10.1', + version='1.10.0', name='testgres', packages=['testgres', 'testgres.operations', 'testgres.helpers'], description='Testing utility for PostgreSQL and its extensions', diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index f182768b..01251e1c 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -4,8 +4,6 @@ import tempfile import platform -from ..utils import reserve_port - # we support both pg8000 and psycopg2 try: import psycopg2 as pglib @@ -394,7 +392,7 @@ def db_connect(self, dbname, user, password=None, host="localhost", port=5432): """ Established SSH tunnel and Connects to a PostgreSQL """ - self.establish_ssh_tunnel(local_port=reserve_port(), remote_port=5432) + self.establish_ssh_tunnel(local_port=port, remote_port=5432) try: conn = pglib.connect( host=host, From 0728583c9b6a9df660cfd209220d8031893d136c Mon Sep 17 00:00:00 2001 From: asavchkov Date: Mon, 24 Jun 2024 17:01:31 +0700 Subject: [PATCH 299/475] Up version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b006c8bf..412e8823 100755 --- a/setup.py +++ b/setup.py @@ -27,7 +27,7 @@ readme = f.read() setup( - version='1.10.0', + version='1.10.1', name='testgres', packages=['testgres', 'testgres.operations', 'testgres.helpers'], description='Testing utility for PostgreSQL and its extensions', From 529b4df245ff73b9db757854174f29bb208a0c4c Mon Sep 17 00:00:00 2001 From: asavchkov Date: Mon, 24 Jun 2024 17:16:50 +0700 Subject: [PATCH 300/475] Up pg_probackup2 version --- testgres/plugins/pg_probackup2/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testgres/plugins/pg_probackup2/setup.py b/testgres/plugins/pg_probackup2/setup.py index 381d7ae2..2ff5a503 100644 --- a/testgres/plugins/pg_probackup2/setup.py +++ b/testgres/plugins/pg_probackup2/setup.py @@ -4,7 +4,7 @@ from distutils.core import setup setup( - version='0.0.2', + version='0.0.3', name='testgres_pg_probackup2', packages=['pg_probackup2', 'pg_probackup2.storage'], description='Plugin for testgres that manages pg_probackup2', From 7847380ed604ddbcc4abf57115777c7064dfd716 Mon Sep 17 00:00:00 2001 From: asavchkov <79832668+asavchkov@users.noreply.github.com> Date: Wed, 26 Jun 2024 04:04:40 +0700 Subject: [PATCH 301/475] Add an SSH port parameter (#131) --- testgres/operations/os_ops.py | 3 ++- testgres/operations/remote_ops.py | 26 +++++++++++++++----------- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/testgres/operations/os_ops.py b/testgres/operations/os_ops.py index dd6613cf..236a08c6 100644 --- a/testgres/operations/os_ops.py +++ b/testgres/operations/os_ops.py @@ -10,8 +10,9 @@ class ConnectionParams: - def __init__(self, host='127.0.0.1', ssh_key=None, username=None): + def __init__(self, host='127.0.0.1', port=None, ssh_key=None, username=None): self.host = host + self.port = port self.ssh_key = ssh_key self.username = username diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index 01251e1c..697b4258 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -44,11 +44,13 @@ def __init__(self, conn_params: ConnectionParams): super().__init__(conn_params.username) self.conn_params = conn_params self.host = conn_params.host + self.port = conn_params.port self.ssh_key = conn_params.ssh_key + self.ssh_args = [] if self.ssh_key: - self.ssh_cmd = ["-i", self.ssh_key] - else: - self.ssh_cmd = [] + self.ssh_args += ["-i", self.ssh_key] + if self.port: + self.ssh_args += ["-p", self.port] self.remote = True self.username = conn_params.username or self.get_user() self.add_known_host(self.host) @@ -95,9 +97,9 @@ def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False, """ ssh_cmd = [] if isinstance(cmd, str): - ssh_cmd = ['ssh', f"{self.username}@{self.host}"] + self.ssh_cmd + [cmd] + ssh_cmd = ['ssh', f"{self.username}@{self.host}"] + self.ssh_args + [cmd] elif isinstance(cmd, list): - ssh_cmd = ['ssh', f"{self.username}@{self.host}"] + self.ssh_cmd + cmd + ssh_cmd = ['ssh', f"{self.username}@{self.host}"] + self.ssh_args + cmd process = subprocess.Popen(ssh_cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) if get_process: return process @@ -246,9 +248,9 @@ def mkdtemp(self, prefix=None): - prefix (str): The prefix of the temporary directory name. """ if prefix: - command = ["ssh"] + self.ssh_cmd + [f"{self.username}@{self.host}", f"mktemp -d {prefix}XXXXX"] + command = ["ssh"] + self.ssh_args + [f"{self.username}@{self.host}", f"mktemp -d {prefix}XXXXX"] else: - command = ["ssh"] + self.ssh_cmd + [f"{self.username}@{self.host}", "mktemp -d"] + command = ["ssh"] + self.ssh_args + [f"{self.username}@{self.host}", "mktemp -d"] result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) @@ -291,8 +293,10 @@ def write(self, filename, data, truncate=False, binary=False, read_and_write=Fal mode = "r+b" if binary else "r+" with tempfile.NamedTemporaryFile(mode=mode, delete=False) as tmp_file: + # For scp the port is specified by a "-P" option + scp_args = ['-P' if x == '-p' else x for x in self.ssh_args] if not truncate: - scp_cmd = ['scp'] + self.ssh_cmd + [f"{self.username}@{self.host}:{filename}", tmp_file.name] + scp_cmd = ['scp'] + scp_args + [f"{self.username}@{self.host}:{filename}", tmp_file.name] subprocess.run(scp_cmd, check=False) # The file might not exist yet tmp_file.seek(0, os.SEEK_END) @@ -308,11 +312,11 @@ def write(self, filename, data, truncate=False, binary=False, read_and_write=Fal tmp_file.write(data) tmp_file.flush() - scp_cmd = ['scp'] + self.ssh_cmd + [tmp_file.name, f"{self.username}@{self.host}:{filename}"] + scp_cmd = ['scp'] + scp_args + [tmp_file.name, f"{self.username}@{self.host}:{filename}"] subprocess.run(scp_cmd, check=True) remote_directory = os.path.dirname(filename) - mkdir_cmd = ['ssh'] + self.ssh_cmd + [f"{self.username}@{self.host}", f"mkdir -p {remote_directory}"] + mkdir_cmd = ['ssh'] + self.ssh_args + [f"{self.username}@{self.host}", f"mkdir -p {remote_directory}"] subprocess.run(mkdir_cmd, check=True) os.remove(tmp_file.name) @@ -377,7 +381,7 @@ def get_pid(self): return int(self.exec_command("echo $$", encoding=get_default_encoding())) def get_process_children(self, pid): - command = ["ssh"] + self.ssh_cmd + [f"{self.username}@{self.host}", f"pgrep -P {pid}"] + command = ["ssh"] + self.ssh_args + [f"{self.username}@{self.host}", f"pgrep -P {pid}"] result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) From 2bf16a5a5b5ec64b93e5ebbd0891ef8edbe25f0f Mon Sep 17 00:00:00 2001 From: asavchkov <79832668+asavchkov@users.noreply.github.com> Date: Thu, 27 Jun 2024 18:18:33 +0700 Subject: [PATCH 302/475] Pass DB port to NodeApp (#132) --- testgres/node.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index e5e8fd5f..f7109b0c 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -127,7 +127,7 @@ def __repr__(self): class PostgresNode(object): - def __init__(self, name=None, port=None, base_dir=None, conn_params: ConnectionParams = ConnectionParams(), bin_dir=None, prefix=None): + def __init__(self, name=None, base_dir=None, port=None, conn_params: ConnectionParams = ConnectionParams(), bin_dir=None, prefix=None): """ PostgresNode constructor. @@ -156,9 +156,9 @@ def __init__(self, name=None, port=None, base_dir=None, conn_params: ConnectionP else: self.os_ops = LocalOperations(conn_params) + self.host = self.os_ops.host self.port = port or reserve_port() - self.host = self.os_ops.host self.ssh_key = self.os_ops.ssh_key # defaults for __exit__() @@ -1690,12 +1690,13 @@ def __init__(self, test_path, nodes_to_cleanup, os_ops=LocalOperations()): def make_empty( self, - base_dir=None): + base_dir=None, + port=None): real_base_dir = os.path.join(self.test_path, base_dir) self.os_ops.rmdirs(real_base_dir, ignore_errors=True) self.os_ops.makedirs(real_base_dir) - node = PostgresNode(base_dir=real_base_dir) + node = PostgresNode(base_dir=real_base_dir, port=port) node.should_rm_dirs = True self.nodes_to_cleanup.append(node) @@ -1704,6 +1705,7 @@ def make_empty( def make_simple( self, base_dir=None, + port=None, set_replication=False, ptrack_enable=False, initdb_params=[], @@ -1711,7 +1713,7 @@ def make_simple( checksum=True): if checksum and '--data-checksums' not in initdb_params: initdb_params.append('--data-checksums') - node = self.make_empty(base_dir) + node = self.make_empty(base_dir, port) node.init( initdb_params=initdb_params, allow_streaming=set_replication) From 4543f80fbc75bac80e2d229f8fe8ba8d2c65f551 Mon Sep 17 00:00:00 2001 From: asavchkov <79832668+asavchkov@users.noreply.github.com> Date: Fri, 28 Jun 2024 18:50:19 +0700 Subject: [PATCH 303/475] Make use of the default SSH user (#133) Make use of the default SSH user --- testgres/node.py | 26 +++++++------------------- testgres/operations/local_ops.py | 6 +----- testgres/operations/os_ops.py | 3 +-- testgres/operations/remote_ops.py | 28 +++++++++++++--------------- 4 files changed, 22 insertions(+), 41 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index f7109b0c..13d13294 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -63,7 +63,6 @@ from .defaults import \ default_dbname, \ - default_username, \ generate_app_name from .exceptions import \ @@ -683,8 +682,6 @@ def slow_start(self, replica=False, dbname='template1', username=None, max_attem If False, waits for the instance to be in primary mode. Default is False. max_attempts: """ - if not username: - username = default_username() self.start() if replica: @@ -694,7 +691,7 @@ def slow_start(self, replica=False, dbname='template1', username=None, max_attem # Call poll_query_until until the expected value is returned self.poll_query_until(query=query, dbname=dbname, - username=username, + username=username or self.os_ops.username, suppress={InternalError, QueryException, ProgrammingError, @@ -967,15 +964,13 @@ def psql(self, >>> psql(query='select 3', ON_ERROR_STOP=1) """ - # Set default arguments dbname = dbname or default_dbname() - username = username or default_username() psql_params = [ self._get_bin_path("psql"), "-p", str(self.port), "-h", self.host, - "-U", username, + "-U", username or self.os_ops.username, "-X", # no .psqlrc "-A", # unaligned output "-t", # print rows only @@ -1087,9 +1082,6 @@ def tmpfile(): fname = self.os_ops.mkstemp(prefix=TMP_DUMP) return fname - # Set default arguments - dbname = dbname or default_dbname() - username = username or default_username() filename = filename or tmpfile() _params = [ @@ -1097,8 +1089,8 @@ def tmpfile(): "-p", str(self.port), "-h", self.host, "-f", filename, - "-U", username, - "-d", dbname, + "-U", username or self.os_ops.username, + "-d", dbname or default_dbname(), "-F", format.value ] # yapf: disable @@ -1118,7 +1110,7 @@ def restore(self, filename, dbname=None, username=None): # Set default arguments dbname = dbname or default_dbname() - username = username or default_username() + username = username or self.os_ops.username _params = [ self._get_bin_path("pg_restore"), @@ -1388,15 +1380,13 @@ def pgbench(self, if options is None: options = [] - # Set default arguments dbname = dbname or default_dbname() - username = username or default_username() _params = [ self._get_bin_path("pgbench"), "-p", str(self.port), "-h", self.host, - "-U", username, + "-U", username or self.os_ops.username ] + options # yapf: disable # should be the last one @@ -1463,15 +1453,13 @@ def pgbench_run(self, dbname=None, username=None, options=[], **kwargs): >>> pgbench_run(time=10) """ - # Set default arguments dbname = dbname or default_dbname() - username = username or default_username() _params = [ self._get_bin_path("pgbench"), "-p", str(self.port), "-h", self.host, - "-U", username, + "-U", username or self.os_ops.username ] + options # yapf: disable for key, value in iteritems(kwargs): diff --git a/testgres/operations/local_ops.py b/testgres/operations/local_ops.py index ef360d3b..313d7060 100644 --- a/testgres/operations/local_ops.py +++ b/testgres/operations/local_ops.py @@ -38,7 +38,7 @@ def __init__(self, conn_params=None): self.host = conn_params.host self.ssh_key = None self.remote = False - self.username = conn_params.username or self.get_user() + self.username = conn_params.username or getpass.getuser() @staticmethod def _raise_exec_exception(message, command, exit_code, output): @@ -130,10 +130,6 @@ def set_env(self, var_name, var_val): # Check if the directory is already in PATH os.environ[var_name] = var_val - # Get environment variables - def get_user(self): - return self.username or getpass.getuser() - def get_name(self): return os.name diff --git a/testgres/operations/os_ops.py b/testgres/operations/os_ops.py index 236a08c6..0b5efff9 100644 --- a/testgres/operations/os_ops.py +++ b/testgres/operations/os_ops.py @@ -45,9 +45,8 @@ def set_env(self, var_name, var_val): # Check if the directory is already in PATH raise NotImplementedError() - # Get environment variables def get_user(self): - raise NotImplementedError() + return self.username def get_name(self): raise NotImplementedError() diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index 697b4258..83965336 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -1,8 +1,9 @@ -import logging +import getpass import os +import logging +import platform import subprocess import tempfile -import platform # we support both pg8000 and psycopg2 try: @@ -52,7 +53,8 @@ def __init__(self, conn_params: ConnectionParams): if self.port: self.ssh_args += ["-p", self.port] self.remote = True - self.username = conn_params.username or self.get_user() + self.username = conn_params.username or getpass.getuser() + self.ssh_dest = f"{self.username}@{self.host}" if conn_params.username else self.host self.add_known_host(self.host) self.tunnel_process = None @@ -97,9 +99,9 @@ def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False, """ ssh_cmd = [] if isinstance(cmd, str): - ssh_cmd = ['ssh', f"{self.username}@{self.host}"] + self.ssh_args + [cmd] + ssh_cmd = ['ssh', self.ssh_dest] + self.ssh_args + [cmd] elif isinstance(cmd, list): - ssh_cmd = ['ssh', f"{self.username}@{self.host}"] + self.ssh_args + cmd + ssh_cmd = ['ssh', self.ssh_dest] + self.ssh_args + cmd process = subprocess.Popen(ssh_cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) if get_process: return process @@ -174,10 +176,6 @@ def set_env(self, var_name: str, var_val: str): """ return self.exec_command("export {}={}".format(var_name, var_val)) - # Get environment variables - def get_user(self): - return self.exec_command("echo $USER", encoding=get_default_encoding()).strip() - def get_name(self): cmd = 'python3 -c "import os; print(os.name)"' return self.exec_command(cmd, encoding=get_default_encoding()).strip() @@ -248,9 +246,9 @@ def mkdtemp(self, prefix=None): - prefix (str): The prefix of the temporary directory name. """ if prefix: - command = ["ssh"] + self.ssh_args + [f"{self.username}@{self.host}", f"mktemp -d {prefix}XXXXX"] + command = ["ssh"] + self.ssh_args + [self.ssh_dest, f"mktemp -d {prefix}XXXXX"] else: - command = ["ssh"] + self.ssh_args + [f"{self.username}@{self.host}", "mktemp -d"] + command = ["ssh"] + self.ssh_args + [self.ssh_dest, "mktemp -d"] result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) @@ -296,7 +294,7 @@ def write(self, filename, data, truncate=False, binary=False, read_and_write=Fal # For scp the port is specified by a "-P" option scp_args = ['-P' if x == '-p' else x for x in self.ssh_args] if not truncate: - scp_cmd = ['scp'] + scp_args + [f"{self.username}@{self.host}:{filename}", tmp_file.name] + scp_cmd = ['scp'] + scp_args + [f"{self.ssh_dest}:{filename}", tmp_file.name] subprocess.run(scp_cmd, check=False) # The file might not exist yet tmp_file.seek(0, os.SEEK_END) @@ -312,11 +310,11 @@ def write(self, filename, data, truncate=False, binary=False, read_and_write=Fal tmp_file.write(data) tmp_file.flush() - scp_cmd = ['scp'] + scp_args + [tmp_file.name, f"{self.username}@{self.host}:{filename}"] + scp_cmd = ['scp'] + scp_args + [tmp_file.name, f"{self.ssh_dest}:{filename}"] subprocess.run(scp_cmd, check=True) remote_directory = os.path.dirname(filename) - mkdir_cmd = ['ssh'] + self.ssh_args + [f"{self.username}@{self.host}", f"mkdir -p {remote_directory}"] + mkdir_cmd = ['ssh'] + self.ssh_args + [self.ssh_dest, f"mkdir -p {remote_directory}"] subprocess.run(mkdir_cmd, check=True) os.remove(tmp_file.name) @@ -381,7 +379,7 @@ def get_pid(self): return int(self.exec_command("echo $$", encoding=get_default_encoding())) def get_process_children(self, pid): - command = ["ssh"] + self.ssh_args + [f"{self.username}@{self.host}", f"pgrep -P {pid}"] + command = ["ssh"] + self.ssh_args + [self.ssh_dest, f"pgrep -P {pid}"] result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) From a128b12cb18ff11c4aef65491dce091e849ef471 Mon Sep 17 00:00:00 2001 From: asavchkov <79832668+asavchkov@users.noreply.github.com> Date: Mon, 1 Jul 2024 15:47:40 +0700 Subject: [PATCH 304/475] Remove SSH tunnel (#136) --- testgres/node.py | 16 ++++-- testgres/operations/os_ops.py | 11 +++- testgres/operations/remote_ops.py | 55 +++---------------- .../pg_probackup2/pg_probackup2/app.py | 9 ++- 4 files changed, 37 insertions(+), 54 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index 13d13294..479ea4ec 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -528,7 +528,9 @@ def get_auth_method(t): u"host\treplication\tall\t127.0.0.1/32\t{}\n".format(auth_host), u"host\treplication\tall\t::1/128\t\t{}\n".format(auth_host), u"host\treplication\tall\t{}/24\t\t{}\n".format(subnet_base, auth_host), - u"host\tall\tall\t{}/24\t\t{}\n".format(subnet_base, auth_host) + u"host\tall\tall\t{}/24\t\t{}\n".format(subnet_base, auth_host), + u"host\tall\tall\tall\t{}\n".format(auth_host), + u"host\treplication\tall\tall\t{}\n".format(auth_host) ] # yapf: disable # write missing lines @@ -1671,9 +1673,15 @@ def _get_bin_path(self, filename): class NodeApp: - def __init__(self, test_path, nodes_to_cleanup, os_ops=LocalOperations()): - self.test_path = test_path - self.nodes_to_cleanup = nodes_to_cleanup + def __init__(self, test_path=None, nodes_to_cleanup=None, os_ops=LocalOperations()): + if test_path: + if os.path.isabs(test_path): + self.test_path = test_path + else: + self.test_path = os.path.join(os_ops.cwd(), test_path) + else: + self.test_path = os_ops.cwd() + self.nodes_to_cleanup = nodes_to_cleanup if nodes_to_cleanup else [] self.os_ops = os_ops def make_empty( diff --git a/testgres/operations/os_ops.py b/testgres/operations/os_ops.py index 0b5efff9..76284049 100644 --- a/testgres/operations/os_ops.py +++ b/testgres/operations/os_ops.py @@ -1,4 +1,6 @@ +import getpass import locale +import sys try: import psycopg2 as pglib # noqa: F401 @@ -24,7 +26,7 @@ def get_default_encoding(): class OsOperations: def __init__(self, username=None): self.ssh_key = None - self.username = username + self.username = username or getpass.getuser() # Command execution def exec_command(self, cmd, **kwargs): @@ -34,6 +36,13 @@ def exec_command(self, cmd, **kwargs): def environ(self, var_name): raise NotImplementedError() + def cwd(self): + if sys.platform == 'linux': + cmd = 'pwd' + elif sys.platform == 'win32': + cmd = 'cd' + return self.exec_command(cmd).decode().rstrip() + def find_executable(self, executable): raise NotImplementedError() diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index 83965336..fa031075 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -1,6 +1,5 @@ import getpass import os -import logging import platform import subprocess import tempfile @@ -55,40 +54,10 @@ def __init__(self, conn_params: ConnectionParams): self.remote = True self.username = conn_params.username or getpass.getuser() self.ssh_dest = f"{self.username}@{self.host}" if conn_params.username else self.host - self.add_known_host(self.host) - self.tunnel_process = None def __enter__(self): return self - def __exit__(self, exc_type, exc_val, exc_tb): - self.close_ssh_tunnel() - - def establish_ssh_tunnel(self, local_port, remote_port): - """ - Establish an SSH tunnel from a local port to a remote PostgreSQL port. - """ - ssh_cmd = ['-N', '-L', f"{local_port}:localhost:{remote_port}"] - self.tunnel_process = self.exec_command(ssh_cmd, get_process=True, timeout=300) - - def close_ssh_tunnel(self): - if hasattr(self, 'tunnel_process'): - self.tunnel_process.terminate() - self.tunnel_process.wait() - del self.tunnel_process - else: - print("No active tunnel to close.") - - def add_known_host(self, host): - known_hosts_path = os.path.expanduser("~/.ssh/known_hosts") - cmd = 'ssh-keyscan -H %s >> %s' % (host, known_hosts_path) - - try: - subprocess.check_call(cmd, shell=True) - logging.info("Successfully added %s to known_hosts." % host) - except subprocess.CalledProcessError as e: - raise Exception("Failed to add %s to known_hosts. Error: %s" % (host, str(e))) - def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False, encoding=None, shell=True, text=False, input=None, stdin=None, stdout=None, stderr=None, get_process=None, timeout=None): @@ -293,6 +262,7 @@ def write(self, filename, data, truncate=False, binary=False, read_and_write=Fal with tempfile.NamedTemporaryFile(mode=mode, delete=False) as tmp_file: # For scp the port is specified by a "-P" option scp_args = ['-P' if x == '-p' else x for x in self.ssh_args] + if not truncate: scp_cmd = ['scp'] + scp_args + [f"{self.ssh_dest}:{filename}", tmp_file.name] subprocess.run(scp_cmd, check=False) # The file might not exist yet @@ -391,18 +361,11 @@ def get_process_children(self, pid): # Database control def db_connect(self, dbname, user, password=None, host="localhost", port=5432): - """ - Established SSH tunnel and Connects to a PostgreSQL - """ - self.establish_ssh_tunnel(local_port=port, remote_port=5432) - try: - conn = pglib.connect( - host=host, - port=port, - database=dbname, - user=user, - password=password, - ) - return conn - except Exception as e: - raise Exception(f"Could not connect to the database. Error: {e}") + conn = pglib.connect( + host=host, + port=port, + database=dbname, + user=user, + password=password, + ) + return conn diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/app.py b/testgres/plugins/pg_probackup2/pg_probackup2/app.py index 1a4ca9e7..ffad24d3 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/app.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/app.py @@ -43,14 +43,14 @@ def __str__(self): class ProbackupApp: def __init__(self, test_class: unittest.TestCase, - pg_node, pb_log_path, test_env, auto_compress_alg, backup_dir): + pg_node, pb_log_path, test_env, auto_compress_alg, backup_dir, probackup_path=None): self.test_class = test_class self.pg_node = pg_node self.pb_log_path = pb_log_path self.test_env = test_env self.auto_compress_alg = auto_compress_alg self.backup_dir = backup_dir - self.probackup_path = init_params.probackup_path + self.probackup_path = probackup_path or init_params.probackup_path self.probackup_old_path = init_params.probackup_old_path self.remote = init_params.remote self.verbose = init_params.verbose @@ -388,6 +388,7 @@ def catchup_node( backup_mode, source_pgdata, destination_node, options=None, remote_host='localhost', + remote_port=None, expect_error=False, gdb=False ): @@ -401,7 +402,9 @@ def catchup_node( '--destination-pgdata={0}'.format(destination_node.data_dir) ] if self.remote: - cmd_list += ['--remote-proto=ssh', '--remote-host=%s' % remote_host] + cmd_list += ['--remote-proto=ssh', f'--remote-host={remote_host}'] + if remote_port: + cmd_list.append(f'--remote-port={remote_port}') if self.verbose: cmd_list += [ '--log-level-file=VERBOSE', From 1d68e91a71f98f440fa429578fa1c289652ba10a Mon Sep 17 00:00:00 2001 From: Victoria Shepard <5807469+demonolock@users.noreply.github.com> Date: Mon, 1 Jul 2024 12:35:39 +0200 Subject: [PATCH 305/475] Fix node cleanup (#135) * Add expect_error to pg_upgrade * Fix node cleanup - rmdirs * Add cleanup parameter - clean full dir --- testgres/node.py | 13 +++++------ testgres/operations/local_ops.py | 37 ++++++++++++++++++++++++++------ 2 files changed, 38 insertions(+), 12 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index 479ea4ec..2ad6ce54 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -914,13 +914,14 @@ def free_port(self): self._should_free_port = False release_port(self.port) - def cleanup(self, max_attempts=3): + def cleanup(self, max_attempts=3, full=False): """ Stop node if needed and remove its data/logs directory. NOTE: take a look at TestgresConfig.node_cleanup_full. Args: max_attempts: how many times should we try to stop()? + full: clean full base dir Returns: This instance of :class:`.PostgresNode`. @@ -929,12 +930,12 @@ def cleanup(self, max_attempts=3): self._try_shutdown(max_attempts) # choose directory to be removed - if testgres_config.node_cleanup_full: + if testgres_config.node_cleanup_full or full: rm_dir = self.base_dir # everything else: rm_dir = self.data_dir # just data, save logs - self.os_ops.rmdirs(rm_dir, ignore_errors=True) + self.os_ops.rmdirs(rm_dir, ignore_errors=False) return self @@ -1629,7 +1630,7 @@ def set_auto_conf(self, options, config='postgresql.auto.conf', rm_options={}): self.os_ops.write(path, auto_conf, truncate=True) - def upgrade_from(self, old_node, options=None): + def upgrade_from(self, old_node, options=None, expect_error=False): """ Upgrade this node from an old node using pg_upgrade. @@ -1657,11 +1658,11 @@ def upgrade_from(self, old_node, options=None): "--old-datadir", old_node.data_dir, "--new-datadir", self.data_dir, "--old-port", str(old_node.port), - "--new-port", str(self.port), + "--new-port", str(self.port) ] upgrade_command += options - return self.os_ops.exec_command(upgrade_command) + return self.os_ops.exec_command(upgrade_command, expect_error=expect_error) def _get_bin_path(self, filename): if self.bin_dir: diff --git a/testgres/operations/local_ops.py b/testgres/operations/local_ops.py index 313d7060..b05a11e2 100644 --- a/testgres/operations/local_ops.py +++ b/testgres/operations/local_ops.py @@ -4,6 +4,7 @@ import stat import subprocess import tempfile +import time import psutil @@ -19,13 +20,18 @@ CMD_TIMEOUT_SEC = 60 error_markers = [b'error', b'Permission denied', b'fatal'] +err_out_markers = [b'Failure'] -def has_errors(output): +def has_errors(output=None, error=None): if output: if isinstance(output, str): output = output.encode(get_default_encoding()) - return any(marker in output for marker in error_markers) + return any(marker in output for marker in err_out_markers) + if error: + if isinstance(error, str): + error = error.encode(get_default_encoding()) + return any(marker in error for marker in error_markers) return False @@ -107,8 +113,8 @@ def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False, process, output, error = self._run_command(cmd, shell, input, stdin, stdout, stderr, get_process, timeout, encoding) if get_process: return process - if process.returncode != 0 or (has_errors(error) and not expect_error): - self._raise_exec_exception('Utility exited with non-zero code. Error `{}`', cmd, process.returncode, error) + if (process.returncode != 0 or has_errors(output=output, error=error)) and not expect_error: + self._raise_exec_exception('Utility exited with non-zero code. Error `{}`', cmd, process.returncode, error or output) if verbose: return process.returncode, output, error @@ -142,8 +148,27 @@ def makedirs(self, path, remove_existing=False): except FileExistsError: pass - def rmdirs(self, path, ignore_errors=True): - return rmtree(path, ignore_errors=ignore_errors) + def rmdirs(self, path, ignore_errors=True, retries=3, delay=1): + """ + Removes a directory and its contents, retrying on failure. + + :param path: Path to the directory. + :param ignore_errors: If True, ignore errors. + :param retries: Number of attempts to remove the directory. + :param delay: Delay between attempts in seconds. + """ + for attempt in range(retries): + try: + rmtree(path, ignore_errors=ignore_errors) + if not os.path.exists(path): + return True + except FileNotFoundError: + return True + except Exception as e: + print(f"Error: Failed to remove directory {path} on attempt {attempt + 1}: {e}") + time.sleep(delay) + print(f"Error: Failed to remove directory {path} after {retries} attempts.") + return False def listdir(self, path): return os.listdir(path) From 8b6b813f2245859dcd7572b458910cb9ad213654 Mon Sep 17 00:00:00 2001 From: Victoria Shepard <5807469+demonolock@users.noreply.github.com> Date: Thu, 4 Jul 2024 02:42:03 +0200 Subject: [PATCH 306/475] Remove polling (#137) * Change print on logging * Normalize error --------- Co-authored-by: vshepard --- testgres/config.py | 6 ++++++ testgres/connection.py | 3 ++- testgres/node.py | 11 +++++------ testgres/operations/local_ops.py | 5 +++-- testgres/operations/remote_ops.py | 10 +++++++++- .../plugins/pg_probackup2/pg_probackup2/app.py | 14 +++++++------- .../pg_probackup2/pg_probackup2/init_helpers.py | 11 ++++++----- .../pg_probackup2/tests/basic_test.py | 3 ++- testgres/utils.py | 1 - 9 files changed, 40 insertions(+), 24 deletions(-) diff --git a/testgres/config.py b/testgres/config.py index b6c43926..63719f1d 100644 --- a/testgres/config.py +++ b/testgres/config.py @@ -2,6 +2,8 @@ import atexit import copy +import logging +import os import tempfile from contextlib import contextmanager @@ -10,6 +12,10 @@ from .operations.os_ops import OsOperations from .operations.local_ops import LocalOperations +log_level = os.getenv('LOGGING_LEVEL', 'WARNING').upper() +log_format = os.getenv('LOGGING_FORMAT', '%(asctime)s - %(levelname)s - %(message)s').upper() +logging.basicConfig(level=log_level, format=log_format) + class GlobalConfig(object): """ diff --git a/testgres/connection.py b/testgres/connection.py index 49b74844..ccedd135 100644 --- a/testgres/connection.py +++ b/testgres/connection.py @@ -1,4 +1,5 @@ # coding: utf-8 +import logging # we support both pg8000 and psycopg2 try: @@ -110,7 +111,7 @@ def execute(self, query, *args): except ProgrammingError: return None except Exception as e: - print("Error executing query: {}\n {}".format(repr(e), query)) + logging.error("Error executing query: {}\n {}".format(repr(e), query)) return None def close(self): diff --git a/testgres/node.py b/testgres/node.py index 2ad6ce54..2ea49529 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -1,5 +1,5 @@ # coding: utf-8 - +import logging import os import random import signal @@ -736,11 +736,10 @@ def start(self, params=[], wait=True): if any(len(file) > 1 and 'Is another postmaster already ' 'running on port' in file[1].decode() for file in files): - print("Detected an issue with connecting to port {0}. " - "Trying another port after a 5-second sleep...".format(self.port)) + logging.warning("Detected an issue with connecting to port {0}. " + "Trying another port after a 5-second sleep...".format(self.port)) self.port = reserve_port() - options = {} - options['port'] = str(self.port) + options = {'port': str(self.port)} self.set_auto_conf(options) startup_retries -= 1 time.sleep(5) @@ -1166,7 +1165,6 @@ def poll_query_until(self, assert sleep_time > 0 attempts = 0 while max_attempts == 0 or attempts < max_attempts: - print(f"Pooling {attempts}") try: res = self.execute(dbname=dbname, query=query, @@ -1190,6 +1188,7 @@ def poll_query_until(self, return # done except tuple(suppress or []): + logging.info(f"Trying execute, attempt {attempts + 1}.\nQuery: {query}") pass # we're suppressing them time.sleep(sleep_time) diff --git a/testgres/operations/local_ops.py b/testgres/operations/local_ops.py index b05a11e2..b518a6cb 100644 --- a/testgres/operations/local_ops.py +++ b/testgres/operations/local_ops.py @@ -1,4 +1,5 @@ import getpass +import logging import os import shutil import stat @@ -165,9 +166,9 @@ def rmdirs(self, path, ignore_errors=True, retries=3, delay=1): except FileNotFoundError: return True except Exception as e: - print(f"Error: Failed to remove directory {path} on attempt {attempt + 1}: {e}") + logging.error(f"Error: Failed to remove directory {path} on attempt {attempt + 1}: {e}") time.sleep(delay) - print(f"Error: Failed to remove directory {path} after {retries} attempts.") + logging.error(f"Error: Failed to remove directory {path} after {retries} attempts.") return False def listdir(self, path): diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index fa031075..f85490ef 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -93,8 +93,10 @@ def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False, if not error: error_found = 0 else: + error = normalize_error(error) error_found = exit_status != 0 or any( - marker in error for marker in [b'error', b'Permission denied', b'fatal', b'No such file or directory']) + marker in error for marker in ['error', 'Permission denied', 'fatal', 'No such file or directory'] + ) if error_found: if isinstance(error, bytes): @@ -369,3 +371,9 @@ def db_connect(self, dbname, user, password=None, host="localhost", port=5432): password=password, ) return conn + + +def normalize_error(error): + if isinstance(error, bytes): + return error.decode() + return error diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/app.py b/testgres/plugins/pg_probackup2/pg_probackup2/app.py index ffad24d3..620dc563 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/app.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/app.py @@ -1,6 +1,7 @@ import contextlib import importlib import json +import logging import os import re import subprocess @@ -74,7 +75,7 @@ def run(self, command, gdb=False, old_binary=False, return_id=True, env=None, command = [command[0], *self.backup_dir.pb_args, *command[1:]] if not self.probackup_old_path and old_binary: - print('PGPROBACKUPBIN_OLD is not set') + logging.error('PGPROBACKUPBIN_OLD is not set') exit(1) if old_binary: @@ -107,12 +108,11 @@ def run(self, command, gdb=False, old_binary=False, return_id=True, env=None, return GDBobj(cmdline, self.test_class) try: - result = None if type(gdb) is tuple and gdb[0] == 'suspend': # special test flow for manually debug probackup gdb_port = gdb[1] cmdline = ['gdbserver'] + ['localhost:' + str(gdb_port)] + cmdline - print("pg_probackup gdb suspended, waiting gdb connection on localhost:{0}".format(gdb_port)) + logging.warning("pg_probackup gdb suspended, waiting gdb connection on localhost:{0}".format(gdb_port)) start_time = time.time() self.test_class.output = subprocess.check_output( @@ -233,7 +233,7 @@ def backup_node( if options is None: options = [] if not node and not data_dir: - print('You must provide ether node or data_dir for backup') + logging.error('You must provide ether node or data_dir for backup') exit(1) if not datname: @@ -502,7 +502,7 @@ def show( if i == '': backup_record_split.remove(i) if len(header_split) != len(backup_record_split): - print(warning.format( + logging.error(warning.format( header=header, body=body, header_split=header_split, body_split=backup_record_split) @@ -581,7 +581,7 @@ def show_archive( else: show_splitted = self.run(cmd_list + options, old_binary=old_binary, expect_error=expect_error).splitlines() - print(show_splitted) + logging.error(show_splitted) exit(1) def validate( @@ -769,7 +769,7 @@ def load_backup_class(fs_type): if fs_type: implementation = fs_type - print("Using ", implementation) + logging.info("Using ", implementation) module_name, class_name = implementation.rsplit(sep='.', maxsplit=1) module = importlib.import_module(module_name) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py index 73731a6e..2d19e980 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py @@ -1,3 +1,4 @@ +import logging from functools import reduce import getpass import os @@ -31,7 +32,7 @@ cached_initdb_dir=False, node_cleanup_full=delete_logs) except Exception as e: - print("Can't configure testgres: {0}".format(e)) + logging.warning("Can't configure testgres: {0}".format(e)) class Init(object): @@ -104,7 +105,7 @@ def __init__(self): if os.path.isfile(probackup_path_tmp): if not os.access(probackup_path_tmp, os.X_OK): - print('{0} is not an executable file'.format( + logging.warning('{0} is not an executable file'.format( probackup_path_tmp)) else: self.probackup_path = probackup_path_tmp @@ -114,13 +115,13 @@ def __init__(self): if os.path.isfile(probackup_path_tmp): if not os.access(probackup_path_tmp, os.X_OK): - print('{0} is not an executable file'.format( + logging.warning('{0} is not an executable file'.format( probackup_path_tmp)) else: self.probackup_path = probackup_path_tmp if not self.probackup_path: - print('pg_probackup binary is not found') + logging.error('pg_probackup binary is not found') exit(1) if os.name == 'posix': @@ -207,7 +208,7 @@ def __init__(self): if self.probackup_version.split('.')[0].isdigit(): self.major_version = int(self.probackup_version.split('.')[0]) else: - print('Can\'t process pg_probackup version \"{}\": the major version is expected to be a number'.format(self.probackup_version)) + logging.error('Can\'t process pg_probackup version \"{}\": the major version is expected to be a number'.format(self.probackup_version)) sys.exit(1) def test_env(self): diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/tests/basic_test.py b/testgres/plugins/pg_probackup2/pg_probackup2/tests/basic_test.py index f5a82d38..b63531ec 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/tests/basic_test.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/tests/basic_test.py @@ -1,3 +1,4 @@ +import logging import os import shutil import unittest @@ -14,7 +15,7 @@ def get_module_and_function_name(test_id): module_name = test_id.split('.')[-2] fname = test_id.split('.')[-1] except IndexError: - print(f"Couldn't get module name and function name from test_id: `{test_id}`") + logging.warning(f"Couldn't get module name and function name from test_id: `{test_id}`") module_name, fname = test_id.split('(')[1].split('.')[1], test_id.split('(')[0] return module_name, fname diff --git a/testgres/utils.py b/testgres/utils.py index 745a2555..a4ee7877 100644 --- a/testgres/utils.py +++ b/testgres/utils.py @@ -228,7 +228,6 @@ def eprint(*args, **kwargs): """ Print stuff to stderr. """ - print(*args, file=sys.stderr, **kwargs) From 5c7cf1808c0657f729da2e4e6a0867513905adfa Mon Sep 17 00:00:00 2001 From: egarbuz <165897130+egarbuz@users.noreply.github.com> Date: Mon, 15 Jul 2024 06:32:52 +0300 Subject: [PATCH 307/475] Use fog pg_probackup3 log level TRACE insted of VERBOSE and don't use -j and --batch-size keys (#138) --- testgres/plugins/pg_probackup2/pg_probackup2/app.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/app.py b/testgres/plugins/pg_probackup2/pg_probackup2/app.py index 620dc563..e656b66d 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/app.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/app.py @@ -705,9 +705,13 @@ def set_archiving( if overwrite: archive_command += ' --overwrite' - archive_command += ' --log-level-console=VERBOSE' - archive_command += ' -j 5' - archive_command += ' --batch-size 10' + if init_params.major_version > 2: + archive_command += ' --log-level-console=trace' + else: + archive_command += ' --log-level-console=VERBOSE' + archive_command += ' -j 5' + archive_command += ' --batch-size 10' + archive_command += ' --no-sync' if archive_timeout: From e1ed1589220a52d108644f30888e066142916ffc Mon Sep 17 00:00:00 2001 From: Victoria Shepard <5807469+demonolock@users.noreply.github.com> Date: Tue, 13 Aug 2024 17:40:22 +0200 Subject: [PATCH 308/475] Fix logger (#140) Co-authored-by: vshepard --- testgres/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testgres/config.py b/testgres/config.py index 63719f1d..67d467d3 100644 --- a/testgres/config.py +++ b/testgres/config.py @@ -13,7 +13,7 @@ from .operations.local_ops import LocalOperations log_level = os.getenv('LOGGING_LEVEL', 'WARNING').upper() -log_format = os.getenv('LOGGING_FORMAT', '%(asctime)s - %(levelname)s - %(message)s').upper() +log_format = os.getenv('LOGGING_FORMAT', '%(asctime)s - %(levelname)s - %(message)s') logging.basicConfig(level=log_level, format=log_format) From 8c193b2c203cda2480331c6c7c52d235ccad70d2 Mon Sep 17 00:00:00 2001 From: Victoria Shepard <5807469+demonolock@users.noreply.github.com> Date: Wed, 14 Aug 2024 22:49:33 +0200 Subject: [PATCH 309/475] Add force node stopping using SIGKILL in case of unsuccessful pg_ctl stop (#139) --- testgres/node.py | 23 +++++++++++++++++++++-- testgres/operations/local_ops.py | 4 ++-- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index 2ea49529..8b30476f 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -330,8 +330,9 @@ def version(self): """ return self._pg_version - def _try_shutdown(self, max_attempts): + def _try_shutdown(self, max_attempts, with_force=False): attempts = 0 + node_pid = self.pid # try stopping server N times while attempts < max_attempts: @@ -341,12 +342,30 @@ def _try_shutdown(self, max_attempts): except ExecUtilException: pass # one more time except Exception: - # TODO: probably should kill stray instance eprint('cannot stop node {}'.format(self.name)) break attempts += 1 + # If force stopping is enabled and PID is valid + if with_force and node_pid != 0: + # If we couldn't stop the node + p_status_output = self.os_ops.exec_command(cmd=f'ps -p {node_pid}', shell=True).decode('utf-8') + if self.status() != NodeStatus.Stopped and p_status_output and str(node_pid) in p_status_output: + try: + eprint(f'Force stopping node {self.name} with PID {node_pid}') + self.os_ops.kill(node_pid, signal.SIGKILL, expect_error=False) + except Exception: + # The node has already stopped + pass + + # Check that node stopped + p_status_output = self.os_ops.exec_command(f'ps -p {node_pid}', shell=True, expect_error=True).decode('utf-8') + if p_status_output and str(node_pid) in p_status_output: + eprint(f'Failed to stop node {self.name}.') + else: + eprint(f'Node {self.name} has been stopped successfully.') + def _assign_master(self, master): """NOTE: this is a private method!""" diff --git a/testgres/operations/local_ops.py b/testgres/operations/local_ops.py index b518a6cb..3d9e490e 100644 --- a/testgres/operations/local_ops.py +++ b/testgres/operations/local_ops.py @@ -293,10 +293,10 @@ def remove_file(self, filename): return os.remove(filename) # Processes control - def kill(self, pid, signal): + def kill(self, pid, signal, expect_error=False): # Kill the process cmd = "kill -{} {}".format(signal, pid) - return self.exec_command(cmd) + return self.exec_command(cmd, expect_error=expect_error) def get_pid(self): # Get current process id From 1d1d3f08e0349067e26d56d84f853b1941f481dc Mon Sep 17 00:00:00 2001 From: dura0ok Date: Fri, 16 Aug 2024 13:59:08 +0300 Subject: [PATCH 310/475] fix locale warning (#141) DeprecationWarning: 'locale.getdefaultlocale' is deprecated and slated for removal in Python 3.15. Use setlocale(), getencoding() and getlocale() instead. rewrite using getlocale --- testgres/operations/os_ops.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/testgres/operations/os_ops.py b/testgres/operations/os_ops.py index 76284049..34242040 100644 --- a/testgres/operations/os_ops.py +++ b/testgres/operations/os_ops.py @@ -20,7 +20,9 @@ def __init__(self, host='127.0.0.1', port=None, ssh_key=None, username=None): def get_default_encoding(): - return locale.getdefaultlocale()[1] or 'UTF-8' + if not hasattr(locale, 'getencoding'): + locale.getencoding = locale.getpreferredencoding + return locale.getencoding() or 'UTF-8' class OsOperations: From 98b028635f5c30d63c94ec07cfdd567fba36aa4c Mon Sep 17 00:00:00 2001 From: Victoria Shepard <5807469+demonolock@users.noreply.github.com> Date: Tue, 27 Aug 2024 09:41:41 +0200 Subject: [PATCH 311/475] Ignore error teardown (#142) --- testgres/node.py | 4 ++-- testgres/operations/local_ops.py | 5 +++-- testgres/operations/remote_ops.py | 4 ++-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index 8b30476f..1404f9cc 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -359,8 +359,8 @@ def _try_shutdown(self, max_attempts, with_force=False): # The node has already stopped pass - # Check that node stopped - p_status_output = self.os_ops.exec_command(f'ps -p {node_pid}', shell=True, expect_error=True).decode('utf-8') + # Check that node stopped - print only column pid without headers + p_status_output = self.os_ops.exec_command(f'ps -o pid= -p {node_pid}', shell=True, ignore_errors=True).decode('utf-8') if p_status_output and str(node_pid) in p_status_output: eprint(f'Failed to stop node {self.name}.') else: diff --git a/testgres/operations/local_ops.py b/testgres/operations/local_ops.py index 3d9e490e..a0a9926d 100644 --- a/testgres/operations/local_ops.py +++ b/testgres/operations/local_ops.py @@ -107,14 +107,15 @@ def _run_command(self, cmd, shell, input, stdin, stdout, stderr, get_process, ti raise ExecUtilException("Command timed out after {} seconds.".format(timeout)) def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False, encoding=None, shell=False, - text=False, input=None, stdin=None, stdout=None, stderr=None, get_process=False, timeout=None): + text=False, input=None, stdin=None, stdout=None, stderr=None, get_process=False, timeout=None, + ignore_errors=False): """ Execute a command in a subprocess and handle the output based on the provided parameters. """ process, output, error = self._run_command(cmd, shell, input, stdin, stdout, stderr, get_process, timeout, encoding) if get_process: return process - if (process.returncode != 0 or has_errors(output=output, error=error)) and not expect_error: + if not ignore_errors and ((process.returncode != 0 or has_errors(output=output, error=error)) and not expect_error): self._raise_exec_exception('Utility exited with non-zero code. Error `{}`', cmd, process.returncode, error or output) if verbose: diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index f85490ef..20095051 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -60,7 +60,7 @@ def __enter__(self): def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False, encoding=None, shell=True, text=False, input=None, stdin=None, stdout=None, - stderr=None, get_process=None, timeout=None): + stderr=None, get_process=None, timeout=None, ignore_errors=False): """ Execute a command in the SSH session. Args: @@ -98,7 +98,7 @@ def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False, marker in error for marker in ['error', 'Permission denied', 'fatal', 'No such file or directory'] ) - if error_found: + if not ignore_errors and error_found: if isinstance(error, bytes): message = b"Utility exited with non-zero code. Error: " + error else: From 78603f85d4332a7ea35877f5f7cb1967156150bd Mon Sep 17 00:00:00 2001 From: Victoria Shepard <5807469+demonolock@users.noreply.github.com> Date: Tue, 17 Sep 2024 19:35:43 +0200 Subject: [PATCH 312/475] Fix node.py: try_shutdown and cleanup (#144) --- testgres/node.py | 59 ++++++++++++++++++++++++------------------------ 1 file changed, 30 insertions(+), 29 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index 1404f9cc..5d95857f 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -334,37 +334,38 @@ def _try_shutdown(self, max_attempts, with_force=False): attempts = 0 node_pid = self.pid - # try stopping server N times - while attempts < max_attempts: - try: - self.stop() - break # OK - except ExecUtilException: - pass # one more time - except Exception: - eprint('cannot stop node {}'.format(self.name)) - break - - attempts += 1 - - # If force stopping is enabled and PID is valid - if with_force and node_pid != 0: - # If we couldn't stop the node - p_status_output = self.os_ops.exec_command(cmd=f'ps -p {node_pid}', shell=True).decode('utf-8') - if self.status() != NodeStatus.Stopped and p_status_output and str(node_pid) in p_status_output: + if node_pid > 0: + # try stopping server N times + while attempts < max_attempts: try: - eprint(f'Force stopping node {self.name} with PID {node_pid}') - self.os_ops.kill(node_pid, signal.SIGKILL, expect_error=False) + self.stop() + break # OK + except ExecUtilException: + pass # one more time except Exception: - # The node has already stopped - pass - - # Check that node stopped - print only column pid without headers - p_status_output = self.os_ops.exec_command(f'ps -o pid= -p {node_pid}', shell=True, ignore_errors=True).decode('utf-8') - if p_status_output and str(node_pid) in p_status_output: - eprint(f'Failed to stop node {self.name}.') - else: - eprint(f'Node {self.name} has been stopped successfully.') + eprint('cannot stop node {}'.format(self.name)) + break + + attempts += 1 + + # If force stopping is enabled and PID is valid + if with_force and node_pid != 0: + # If we couldn't stop the node + p_status_output = self.os_ops.exec_command(cmd=f'ps -p {node_pid}', shell=True).decode('utf-8') + if self.status() != NodeStatus.Stopped and p_status_output and str(node_pid) in p_status_output: + try: + eprint(f'Force stopping node {self.name} with PID {node_pid}') + self.os_ops.kill(node_pid, signal.SIGKILL, expect_error=False) + except Exception: + # The node has already stopped + pass + + # Check that node stopped - print only column pid without headers + p_status_output = self.os_ops.exec_command(f'ps -o pid= -p {node_pid}', shell=True, ignore_errors=True).decode('utf-8') + if p_status_output and str(node_pid) in p_status_output: + eprint(f'Failed to stop node {self.name}.') + else: + eprint(f'Node {self.name} has been stopped successfully.') def _assign_master(self, master): """NOTE: this is a private method!""" From 177724b0265c5977bfd6d190aab7c16f6972b3a0 Mon Sep 17 00:00:00 2001 From: Victoria Shepard <5807469+demonolock@users.noreply.github.com> Date: Fri, 20 Sep 2024 22:36:47 +0200 Subject: [PATCH 313/475] Fix node.py: try_shutdown and cleanup (#145) --- testgres/node.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testgres/node.py b/testgres/node.py index 5d95857f..b9bf9896 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -351,7 +351,7 @@ def _try_shutdown(self, max_attempts, with_force=False): # If force stopping is enabled and PID is valid if with_force and node_pid != 0: # If we couldn't stop the node - p_status_output = self.os_ops.exec_command(cmd=f'ps -p {node_pid}', shell=True).decode('utf-8') + p_status_output = self.os_ops.exec_command(cmd=f'ps -o pid= -p {node_pid}', shell=True, ignore_errors=True).decode('utf-8') if self.status() != NodeStatus.Stopped and p_status_output and str(node_pid) in p_status_output: try: eprint(f'Force stopping node {self.name} with PID {node_pid}') From 6db4520f3b2825ec71166a9ffa6cbe0c95ffefef Mon Sep 17 00:00:00 2001 From: Victoria Shepard <5807469+demonolock@users.noreply.github.com> Date: Sat, 26 Oct 2024 02:16:34 +0300 Subject: [PATCH 314/475] Fix test test_child_pids for PostgreSql 15 and higher (#146) --- tests/test_simple.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_simple.py b/tests/test_simple.py index ba23c3ed..43394718 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -957,6 +957,9 @@ def test_child_pids(self): if pg_version_ge('10'): master_processes.append(ProcessType.LogicalReplicationLauncher) + if pg_version_ge('14'): + master_processes.remove(ProcessType.StatsCollector) + repl_processes = [ ProcessType.Startup, ProcessType.WalReceiver, From 8a5c252ebc5dadd871bb92eec10959a8733167b4 Mon Sep 17 00:00:00 2001 From: Ekaterina Sokolova Date: Wed, 23 Oct 2024 14:57:06 +0300 Subject: [PATCH 315/475] Update configuration for Travis CI. --- .travis.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.travis.yml b/.travis.yml index c06cab3d..6f63a67b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,6 +20,9 @@ notifications: on_failure: always env: + - PYTHON_VERSION=3 PG_VERSION=17 + - PYTHON_VERSION=3 PG_VERSION=16 + - PYTHON_VERSION=3 PG_VERSION=15 - PYTHON_VERSION=3 PG_VERSION=14 - PYTHON_VERSION=3 PG_VERSION=13 - PYTHON_VERSION=3 PG_VERSION=12 @@ -32,3 +35,8 @@ env: # - PYTHON_VERSION=2 PG_VERSION=9.6 # - PYTHON_VERSION=2 PG_VERSION=9.5 # - PYTHON_VERSION=2 PG_VERSION=9.4 + +matrix: + allow_failures: + - env: PYTHON_VERSION=3 PG_VERSION=11 + - env: PYTHON_VERSION=3 PG_VERSION=10 From 569923a004165aa40530444b2154ad98e3f57cc7 Mon Sep 17 00:00:00 2001 From: asavchkov Date: Mon, 28 Oct 2024 17:10:17 +0700 Subject: [PATCH 316/475] Configure the unix socket directory --- testgres/node.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/testgres/node.py b/testgres/node.py index b9bf9896..cb1b8385 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -1751,7 +1751,8 @@ def make_simple( 'log_connections': 'on', 'log_disconnections': 'on', 'restart_after_crash': 'off', - 'autovacuum': 'off'} + 'autovacuum': 'off', + 'unix_socket_directories': '/tmp'} # Allow replication in pg_hba.conf if set_replication: From c0dca6bf8c9f8fa41ad076542ce9f4a0e68daa2d Mon Sep 17 00:00:00 2001 From: dura0ok Date: Mon, 18 Nov 2024 16:32:33 +0700 Subject: [PATCH 317/475] Add PG_PROBACKUP_WAL_TREE_ENABLED env param (#143) --- testgres/plugins/pg_probackup2/pg_probackup2/app.py | 4 ++++ testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py | 1 + 2 files changed, 5 insertions(+) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/app.py b/testgres/plugins/pg_probackup2/pg_probackup2/app.py index e656b66d..899a8092 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/app.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/app.py @@ -54,6 +54,7 @@ def __init__(self, test_class: unittest.TestCase, self.probackup_path = probackup_path or init_params.probackup_path self.probackup_old_path = init_params.probackup_old_path self.remote = init_params.remote + self.wal_tree_enabled = init_params.wal_tree_enabled self.verbose = init_params.verbose self.archive_compress = init_params.archive_compress self.test_class.output = None @@ -185,6 +186,9 @@ def add_instance(self, instance, node, old_binary=False, options=None, expect_er '--remote-proto=ssh', '--remote-host=localhost'] + if self.wal_tree_enabled: + options = options + ['--wal-tree'] + return self.run(cmd + options, old_binary=old_binary, expect_error=expect_error) def set_config(self, instance, old_binary=False, options=None, expect_error=False): diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py index 2d19e980..b7174a7c 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py @@ -170,6 +170,7 @@ def __init__(self): self.remote = test_env.get('PGPROBACKUP_SSH_REMOTE', None) == 'ON' self.ptrack = test_env.get('PG_PROBACKUP_PTRACK', None) == 'ON' and self.pg_config_version >= 110000 + self.wal_tree_enabled = test_env.get('PG_PROBACKUP_WAL_TREE_ENABLED', None) == 'ON' self.paranoia = test_env.get('PG_PROBACKUP_PARANOIA', None) == 'ON' env_compress = test_env.get('ARCHIVE_COMPRESSION', None) From a5cfdb65f56882a11cc38ddd408420467beac6e7 Mon Sep 17 00:00:00 2001 From: vshepard Date: Mon, 18 Nov 2024 11:57:51 +0100 Subject: [PATCH 318/475] Add bin_dir to make_simple --- testgres/node.py | 10 ++++++---- tests/test_simple.py | 19 ++++++++++++++++++- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index cb1b8385..c8c8c087 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -1707,12 +1707,13 @@ def __init__(self, test_path=None, nodes_to_cleanup=None, os_ops=LocalOperations def make_empty( self, base_dir=None, - port=None): + port=None, + bin_dir=None): real_base_dir = os.path.join(self.test_path, base_dir) self.os_ops.rmdirs(real_base_dir, ignore_errors=True) self.os_ops.makedirs(real_base_dir) - node = PostgresNode(base_dir=real_base_dir, port=port) + node = PostgresNode(base_dir=real_base_dir, port=port, bin_dir=bin_dir) node.should_rm_dirs = True self.nodes_to_cleanup.append(node) @@ -1726,10 +1727,11 @@ def make_simple( ptrack_enable=False, initdb_params=[], pg_options={}, - checksum=True): + checksum=True, + bin_dir=None): if checksum and '--data-checksums' not in initdb_params: initdb_params.append('--data-checksums') - node = self.make_empty(base_dir, port) + node = self.make_empty(base_dir, port, bin_dir=bin_dir) node.init( initdb_params=initdb_params, allow_streaming=set_replication) diff --git a/tests/test_simple.py b/tests/test_simple.py index 43394718..8f85a23b 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -23,7 +23,7 @@ BackupException, \ QueryException, \ TimeoutException, \ - TestgresException + TestgresException, NodeApp from testgres import \ TestgresConfig, \ @@ -1044,6 +1044,23 @@ def test_the_same_port(self): node2.port = node.port node2.init().start() + def test_make_simple_with_bin_dir(self): + with get_new_node() as node: + node.init().start() + bin_dir = node.bin_dir + + app = NodeApp() + correct_bin_dir = app.make_simple(base_dir=node.base_dir, bin_dir=bin_dir) + correct_bin_dir.slow_start() + correct_bin_dir.safe_psql("SELECT 1;") + + try: + wrong_bin_dir = app.make_empty(base_dir=node.base_dir, bin_dir="wrong/path") + wrong_bin_dir.slow_start() + raise RuntimeError("Error was expected.") # We should not reach this + except FileNotFoundError: + pass # Expected error + if __name__ == '__main__': if os.environ.get('ALT_CONFIG'): From 1cd6cd72ae761ec0b9f34ef8477c8b925de36965 Mon Sep 17 00:00:00 2001 From: Victoria Shepard <5807469+demonolock@users.noreply.github.com> Date: Mon, 18 Nov 2024 16:22:27 +0100 Subject: [PATCH 319/475] Fix style in test_simple.py (#150) --- tests/test_simple.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_simple.py b/tests/test_simple.py index 8f85a23b..62de1df5 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -1044,7 +1044,7 @@ def test_the_same_port(self): node2.port = node.port node2.init().start() - def test_make_simple_with_bin_dir(self): + def test_simple_with_bin_dir(self): with get_new_node() as node: node.init().start() bin_dir = node.bin_dir @@ -1059,7 +1059,7 @@ def test_make_simple_with_bin_dir(self): wrong_bin_dir.slow_start() raise RuntimeError("Error was expected.") # We should not reach this except FileNotFoundError: - pass # Expected error + pass # Expected error if __name__ == '__main__': From 5ae340c9b95430ac9065d58bb1d09882fddf7c30 Mon Sep 17 00:00:00 2001 From: Kian-Meng Ang Date: Wed, 20 Nov 2024 19:55:47 +0800 Subject: [PATCH 320/475] Fix typos (#152) Found via `codespell -L splitted` --- README.md | 4 ++-- testgres/node.py | 4 ++-- testgres/plugins/pg_probackup2/pg_probackup2/gdb.py | 2 +- tests/test_simple.py | 8 ++++---- tests/test_simple_remote.py | 8 ++++---- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index a2a0ec7e..f0071a90 100644 --- a/README.md +++ b/README.md @@ -59,12 +59,12 @@ with testgres.get_new_node() as node: # ... node stops and its files are about to be removed ``` -There are four API methods for runnig queries: +There are four API methods for running queries: | Command | Description | |----------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------| | `node.psql(query, ...)` | Runs query via `psql` command and returns tuple `(error code, stdout, stderr)`. | -| `node.safe_psql(query, ...)` | Same as `psql()` except that it returns only `stdout`. If an error occures during the execution, an exception will be thrown. | +| `node.safe_psql(query, ...)` | Same as `psql()` except that it returns only `stdout`. If an error occurs during the execution, an exception will be thrown. | | `node.execute(query, ...)` | Connects to PostgreSQL using `psycopg2` or `pg8000` (depends on which one is installed in your system) and returns two-dimensional array with data. | | `node.connect(dbname, ...)` | Returns connection wrapper (`NodeConnection`) capable of running several queries within a single transaction. | diff --git a/testgres/node.py b/testgres/node.py index c8c8c087..4ae30908 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -1143,7 +1143,7 @@ def restore(self, filename, dbname=None, username=None): filename ] # yapf: disable - # try pg_restore if dump is binary formate, and psql if not + # try pg_restore if dump is binary format, and psql if not try: execute_utility(_params, self.utils_log_name) except ExecUtilException: @@ -1286,7 +1286,7 @@ def set_synchronous_standbys(self, standbys): Args: standbys: either :class:`.First` or :class:`.Any` object specifying - sychronization parameters or just a plain list of + synchronization parameters or just a plain list of :class:`.PostgresNode`s replicas which would be equivalent to passing ``First(1, )``. For PostgreSQL 9.5 and below it is only possible to specify a plain list of standbys as diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/gdb.py b/testgres/plugins/pg_probackup2/pg_probackup2/gdb.py index 0b61da65..ceb1f6a9 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/gdb.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/gdb.py @@ -37,7 +37,7 @@ def __init__(self, cmd, env, attach=False): " to run GDB tests") raise GdbException("No gdb usage possible.") - # Check gdb presense + # Check gdb presence try: gdb_version, _ = subprocess.Popen( ['gdb', '--version'], diff --git a/tests/test_simple.py b/tests/test_simple.py index 62de1df5..41203a65 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -501,7 +501,7 @@ def test_logical_replication(self): sub.disable() node1.safe_psql('insert into test values (3, 3)') - # enable and ensure that data successfully transfered + # enable and ensure that data successfully transferred sub.enable() sub.catchup() res = node2.execute('select * from test') @@ -509,7 +509,7 @@ def test_logical_replication(self): # Add new tables. Since we added "all tables" to publication # (default behaviour of publish() method) we don't need - # to explicitely perform pub.add_tables() + # to explicitly perform pub.add_tables() create_table = 'create table test2 (c char)' node1.safe_psql(create_table) node2.safe_psql(create_table) @@ -526,7 +526,7 @@ def test_logical_replication(self): pub.drop() # create new publication and subscription for specific table - # (ommitting copying data as it's already done) + # (omitting copying data as it's already done) pub = node1.publish('newpub', tables=['test']) sub = node2.subscribe(pub, 'newsub', copy_data=False) @@ -535,7 +535,7 @@ def test_logical_replication(self): res = node2.execute('select * from test') self.assertListEqual(res, [(1, 1), (2, 2), (3, 3), (4, 4)]) - # explicitely add table + # explicitly add table with self.assertRaises(ValueError): pub.add_tables([]) # fail pub.add_tables(['test2']) diff --git a/tests/test_simple_remote.py b/tests/test_simple_remote.py index d51820ba..79bdb74c 100755 --- a/tests/test_simple_remote.py +++ b/tests/test_simple_remote.py @@ -480,7 +480,7 @@ def test_logical_replication(self): sub.disable() node1.safe_psql('insert into test values (3, 3)') - # enable and ensure that data successfully transfered + # enable and ensure that data successfully transferred sub.enable() sub.catchup() res = node2.execute('select * from test') @@ -488,7 +488,7 @@ def test_logical_replication(self): # Add new tables. Since we added "all tables" to publication # (default behaviour of publish() method) we don't need - # to explicitely perform pub.add_tables() + # to explicitly perform pub.add_tables() create_table = 'create table test2 (c char)' node1.safe_psql(create_table) node2.safe_psql(create_table) @@ -505,7 +505,7 @@ def test_logical_replication(self): pub.drop() # create new publication and subscription for specific table - # (ommitting copying data as it's already done) + # (omitting copying data as it's already done) pub = node1.publish('newpub', tables=['test']) sub = node2.subscribe(pub, 'newsub', copy_data=False) @@ -514,7 +514,7 @@ def test_logical_replication(self): res = node2.execute('select * from test') self.assertListEqual(res, [(1, 1), (2, 2), (3, 3), (4, 4)]) - # explicitely add table + # explicitly add table with self.assertRaises(ValueError): pub.add_tables([]) # fail pub.add_tables(['test2']) From 644eea8cd6c2e85536a1d18a5a8b6e5d0b949bce Mon Sep 17 00:00:00 2001 From: Vyacheslav Makarov <50846161+MakSl@users.noreply.github.com> Date: Wed, 20 Nov 2024 14:56:33 +0300 Subject: [PATCH 321/475] Refinement of the run function in app.py (#148) --- testgres/plugins/pg_probackup2/pg_probackup2/app.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/app.py b/testgres/plugins/pg_probackup2/pg_probackup2/app.py index 899a8092..57492814 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/app.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/app.py @@ -74,6 +74,8 @@ def run(self, command, gdb=False, old_binary=False, return_id=True, env=None, command = [command[0], *use_backup_dir.pb_args, *command[1:]] elif use_backup_dir: command = [command[0], *self.backup_dir.pb_args, *command[1:]] + else: + command = [command[0], *self.backup_dir.pb_args[2:], *command[1:]] if not self.probackup_old_path and old_binary: logging.error('PGPROBACKUPBIN_OLD is not set') From b5855e417cdba94766158b0d1fe23c90b717a967 Mon Sep 17 00:00:00 2001 From: MetalDream666 <61190185+MetalDream666@users.noreply.github.com> Date: Fri, 22 Nov 2024 12:30:39 +0300 Subject: [PATCH 322/475] Fix broken pipe from gdb after killing process (#151) --- testgres/plugins/pg_probackup2/pg_probackup2/gdb.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/gdb.py b/testgres/plugins/pg_probackup2/pg_probackup2/gdb.py index ceb1f6a9..2424c04d 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/gdb.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/gdb.py @@ -108,6 +108,9 @@ def kill(self): self.proc.stdin.close() self.proc.stdout.close() + def terminate_subprocess(self): + self._execute('kill') + def set_breakpoint(self, location): result = self._execute('break ' + location) From 655a386117c995618720f4eb0ec7cbc990221343 Mon Sep 17 00:00:00 2001 From: asavchkov Date: Fri, 29 Nov 2024 19:00:07 +0700 Subject: [PATCH 323/475] Up version --- setup.py | 2 +- testgres/plugins/pg_probackup2/setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 412e8823..a41094d6 100755 --- a/setup.py +++ b/setup.py @@ -27,7 +27,7 @@ readme = f.read() setup( - version='1.10.1', + version='1.10.3', name='testgres', packages=['testgres', 'testgres.operations', 'testgres.helpers'], description='Testing utility for PostgreSQL and its extensions', diff --git a/testgres/plugins/pg_probackup2/setup.py b/testgres/plugins/pg_probackup2/setup.py index 2ff5a503..ade2d85d 100644 --- a/testgres/plugins/pg_probackup2/setup.py +++ b/testgres/plugins/pg_probackup2/setup.py @@ -4,7 +4,7 @@ from distutils.core import setup setup( - version='0.0.3', + version='0.0.4', name='testgres_pg_probackup2', packages=['pg_probackup2', 'pg_probackup2.storage'], description='Plugin for testgres that manages pg_probackup2', From 2c26debe967920e0f2a74d430fe6410a8a9a0513 Mon Sep 17 00:00:00 2001 From: vshepard Date: Tue, 3 Dec 2024 01:30:03 +0100 Subject: [PATCH 324/475] Fix set_auto_conf with single quotes --- testgres/node.py | 13 +++++++++---- tests/test_simple.py | 27 +++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 4 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index 4ae30908..be5019a4 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -1627,17 +1627,22 @@ def set_auto_conf(self, options, config='postgresql.auto.conf', rm_options={}): name, var = line.partition('=')[::2] name = name.strip() var = var.strip() - var = var.strip('"') - var = var.strip("'") - # remove options specified in rm_options list + # Handle quoted values and remove escaping + if var.startswith("'") and var.endswith("'"): + var = var[1:-1].replace("''", "'") + + # Remove options specified in rm_options list if name in rm_options: continue current_options[name] = var for option in options: - current_options[option] = options[option] + value = options[option] + if isinstance(value, str): + value = value.replace("'", "\\'") + current_options[option] = value auto_conf = '' for option in current_options: diff --git a/tests/test_simple.py b/tests/test_simple.py index 41203a65..a11d7932 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -1061,6 +1061,33 @@ def test_simple_with_bin_dir(self): except FileNotFoundError: pass # Expected error + def test_set_auto_conf(self): + with get_new_node() as node: + node.init().start() + + options = { + "archive_command": "cp '%p' \"/mnt/server/archivedir/%f\"", + 'restore_command': 'cp "/mnt/server/archivedir/%f" \'%p\'', + } + + node.set_auto_conf(options) + node.stop() + node.slow_start() + + auto_conf_path = f"{node.data_dir}/postgresql.auto.conf" + with open(auto_conf_path, "r") as f: + content = f.read() + self.assertIn( + "archive_command = 'cp \\'%p\\' \"/mnt/server/archivedir/%f\"", + content, + "archive_command stored wrong" + ) + self.assertIn( + "restore_command = 'cp \"/mnt/server/archivedir/%f\" \\'%p\\''", + content, + "restore_command stored wrong" + ) + if __name__ == '__main__': if os.environ.get('ALT_CONFIG'): From a4092af44fae5a1a92e9c9fbfec449fd99c8e520 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 3 Dec 2024 16:50:47 +0300 Subject: [PATCH 325/475] Node.set_auto_conf is improved - we do not touch existing values - escaping of '\n', '\r', '\t', '\b' and '\\' is added - translation of bool into 'on|off' is added test_set_auto_conf is updated. --- testgres/node.py | 39 +++++++++++++++++++++++++++--------- tests/test_simple.py | 47 +++++++++++++++++++++++++++++++------------- 2 files changed, 63 insertions(+), 23 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index be5019a4..7469b0d6 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -1626,11 +1626,6 @@ def set_auto_conf(self, options, config='postgresql.auto.conf', rm_options={}): name, var = line.partition('=')[::2] name = name.strip() - var = var.strip() - - # Handle quoted values and remove escaping - if var.startswith("'") and var.endswith("'"): - var = var[1:-1].replace("''", "'") # Remove options specified in rm_options list if name in rm_options: @@ -1640,14 +1635,18 @@ def set_auto_conf(self, options, config='postgresql.auto.conf', rm_options={}): for option in options: value = options[option] - if isinstance(value, str): - value = value.replace("'", "\\'") + valueType = type(value) + + if valueType == str: + value = __class__._escape_config_value(value) + elif valueType == bool: + value = "on" if value else "off" + current_options[option] = value auto_conf = '' for option in current_options: - auto_conf += "{0} = '{1}'\n".format( - option, current_options[option]) + auto_conf += option + " = " + str(current_options[option]) + "\n" for directive in current_directives: auto_conf += directive + "\n" @@ -1695,6 +1694,28 @@ def _get_bin_path(self, filename): bin_path = get_bin_path(filename) return bin_path + def _escape_config_value(value): + result = "'" + + for ch in value: + if (ch == "'"): + result += "\\'" + elif (ch == "\n"): + result += "\\n" + elif (ch == "\r"): + result += "\\r" + elif (ch == "\t"): + result += "\\t" + elif (ch == "\b"): + result += "\\b" + elif (ch == "\\"): + result += "\\\\" + else: + result += ch + + result += "'" + return result + class NodeApp: diff --git a/tests/test_simple.py b/tests/test_simple.py index a11d7932..ffefda6c 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -1062,13 +1062,35 @@ def test_simple_with_bin_dir(self): pass # Expected error def test_set_auto_conf(self): + # elements contain [property id, value, storage value] + testData = [ + ["archive_command", + "cp '%p' \"/mnt/server/archivedir/%f\"", + "'cp \\'%p\\' \"/mnt/server/archivedir/%f\""], + ["restore_command", + 'cp "/mnt/server/archivedir/%f" \'%p\'', + "'cp \"/mnt/server/archivedir/%f\" \\'%p\\''"], + ["log_line_prefix", + "'\n\r\t\b\\\"", + "'\\\'\\n\\r\\t\\b\\\\\""], + ["log_connections", + True, + "on"], + ["log_disconnections", + False, + "off"], + ["autovacuum_max_workers", + 3, + "3"] + ] + with get_new_node() as node: node.init().start() - options = { - "archive_command": "cp '%p' \"/mnt/server/archivedir/%f\"", - 'restore_command': 'cp "/mnt/server/archivedir/%f" \'%p\'', - } + options = {} + + for x in testData: + options[x[0]] = x[1] node.set_auto_conf(options) node.stop() @@ -1077,16 +1099,13 @@ def test_set_auto_conf(self): auto_conf_path = f"{node.data_dir}/postgresql.auto.conf" with open(auto_conf_path, "r") as f: content = f.read() - self.assertIn( - "archive_command = 'cp \\'%p\\' \"/mnt/server/archivedir/%f\"", - content, - "archive_command stored wrong" - ) - self.assertIn( - "restore_command = 'cp \"/mnt/server/archivedir/%f\" \\'%p\\''", - content, - "restore_command stored wrong" - ) + + for x in testData: + self.assertIn( + x[0] + " = " + x[2], + content, + x[0] + " stored wrong" + ) if __name__ == '__main__': From ee7dc91f2f56d80ca975f091c4dd60a004770177 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 3 Dec 2024 19:01:12 +0300 Subject: [PATCH 326/475] Asserts in PostgresNode.set_auto_conf are added Let's control a correct usage of this function. Plus one assert was added in _escape_config_value. --- testgres/node.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/testgres/node.py b/testgres/node.py index 7469b0d6..74a18cdf 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -1634,6 +1634,10 @@ def set_auto_conf(self, options, config='postgresql.auto.conf', rm_options={}): current_options[name] = var for option in options: + assert type(option) == str + assert option != "" + assert option.strip() == option + value = options[option] valueType = type(value) @@ -1695,6 +1699,8 @@ def _get_bin_path(self, filename): return bin_path def _escape_config_value(value): + assert type(value) == str + result = "'" for ch in value: From 9dd8dfec44ef56f732761d31c94cf98764a820a8 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 3 Dec 2024 21:11:00 +0300 Subject: [PATCH 327/475] Flake8 E721 is suppressed --- testgres/node.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index 74a18cdf..d9fb0604 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -1634,7 +1634,7 @@ def set_auto_conf(self, options, config='postgresql.auto.conf', rm_options={}): current_options[name] = var for option in options: - assert type(option) == str + assert type(option) == str # noqa: E721 assert option != "" assert option.strip() == option @@ -1699,7 +1699,7 @@ def _get_bin_path(self, filename): return bin_path def _escape_config_value(value): - assert type(value) == str + assert type(value) == str # noqa: E721 result = "'" From 1335e517e6d8bc16f2d628446c02a03078c5957e Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Wed, 4 Dec 2024 10:00:58 +0300 Subject: [PATCH 328/475] PostgresNode::_escape_config_value is updated (code style) --- testgres/node.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index d9fb0604..1706de11 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -1704,17 +1704,17 @@ def _escape_config_value(value): result = "'" for ch in value: - if (ch == "'"): + if ch == "'": result += "\\'" - elif (ch == "\n"): + elif ch == "\n": result += "\\n" - elif (ch == "\r"): + elif ch == "\r": result += "\\r" - elif (ch == "\t"): + elif ch == "\t": result += "\\t" - elif (ch == "\b"): + elif ch == "\b": result += "\\b" - elif (ch == "\\"): + elif ch == "\\": result += "\\\\" else: result += ch From 6acbeb6517b6b4aac4d23f0a137821ddb6b7928f Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Fri, 6 Dec 2024 12:32:55 +0300 Subject: [PATCH 329/475] NodeApp::make_simple is refactored (tempfile.gettempdir) - [BUG FIX] Windows does not have "/tmp" directory. Let's use tempfile.gettempdir() - Aggregation of standard and custom options to avoid two calls of node.set_auto_conf --- testgres/node.py | 69 +++++++++++++++++++++++++++++++++++------------- 1 file changed, 51 insertions(+), 18 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index 1706de11..0e5bb866 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -5,6 +5,7 @@ import signal import subprocess import threading +import tempfile from queue import Queue import time @@ -1761,6 +1762,8 @@ def make_simple( pg_options={}, checksum=True, bin_dir=None): + assert type(pg_options) == dict # noqa: E721 + if checksum and '--data-checksums' not in initdb_params: initdb_params.append('--data-checksums') node = self.make_empty(base_dir, port, bin_dir=bin_dir) @@ -1773,20 +1776,22 @@ def make_simple( node.major_version = float(node.major_version_str) # Set default parameters - options = {'max_connections': 100, - 'shared_buffers': '10MB', - 'fsync': 'off', - 'wal_level': 'logical', - 'hot_standby': 'off', - 'log_line_prefix': '%t [%p]: [%l-1] ', - 'log_statement': 'none', - 'log_duration': 'on', - 'log_min_duration_statement': 0, - 'log_connections': 'on', - 'log_disconnections': 'on', - 'restart_after_crash': 'off', - 'autovacuum': 'off', - 'unix_socket_directories': '/tmp'} + options = { + 'max_connections': 100, + 'shared_buffers': '10MB', + 'fsync': 'off', + 'wal_level': 'logical', + 'hot_standby': 'off', + 'log_line_prefix': '%t [%p]: [%l-1] ', + 'log_statement': 'none', + 'log_duration': 'on', + 'log_min_duration_statement': 0, + 'log_connections': 'on', + 'log_disconnections': 'on', + 'restart_after_crash': 'off', + 'autovacuum': 'off', + # 'unix_socket_directories': '/tmp', + } # Allow replication in pg_hba.conf if set_replication: @@ -1801,11 +1806,16 @@ def make_simple( else: options['wal_keep_segments'] = '12' - # set default values - node.set_auto_conf(options) - # Apply given parameters - node.set_auto_conf(pg_options) + for x in pg_options: + options[x] = pg_options[x] + + # Define delayed propertyes + if not ("unix_socket_directories" in options.keys()): + options["unix_socket_directories"] = __class__._gettempdir() + + # Set config values + node.set_auto_conf(options) # kludge for testgres # https://github.com/postgrespro/testgres/issues/54 @@ -1814,3 +1824,26 @@ def make_simple( node.set_auto_conf({}, 'postgresql.conf', ['wal_keep_segments']) return node + + def _gettempdir(): + v = tempfile.gettempdir() + + # + # Paranoid checks + # + if type(v) != str: # noqa: E721 + __class__._raise_bugcheck("tempfile.gettempdir returned a value with type {0}.".format(type(v).__name__)) + + if v == "": + __class__._raise_bugcheck("tempfile.gettempdir returned an empty string.") + + if not os.path.exists(v): + __class__._raise_bugcheck("tempfile.gettempdir returned a not exist path [{0}].".format(v)) + + # OK + return v + + def _raise_bugcheck(msg): + assert type(msg) == str # noqa: E721 + assert msg != "" + raise Exception("[BUG CHECK] " + msg) From 9d6f61472817c47477561911c7361d713a1fb1ca Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Fri, 6 Dec 2024 15:22:16 +0300 Subject: [PATCH 330/475] NodeApp::make_simple is updated [comment] --- testgres/node.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testgres/node.py b/testgres/node.py index 0e5bb866..62509790 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -1790,7 +1790,7 @@ def make_simple( 'log_disconnections': 'on', 'restart_after_crash': 'off', 'autovacuum': 'off', - # 'unix_socket_directories': '/tmp', + # unix_socket_directories will be defined later } # Allow replication in pg_hba.conf From 1d44628c65ad9266f303ecbb21080ed790ff01d0 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Fri, 6 Dec 2024 15:47:20 +0300 Subject: [PATCH 331/475] NodeApp::make_simple uses iteritems(pg_options) --- testgres/node.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index 62509790..48a100a9 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -1807,8 +1807,8 @@ def make_simple( options['wal_keep_segments'] = '12' # Apply given parameters - for x in pg_options: - options[x] = pg_options[x] + for option_name, option_value in iteritems(pg_options): + options[option_name] = option_value # Define delayed propertyes if not ("unix_socket_directories" in options.keys()): From d07104f8885f0d0c9332a608a32fc915017ff891 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Fri, 6 Dec 2024 17:43:41 +0300 Subject: [PATCH 332/475] LocalOperations::_run_command is refactored LocalOperations::_run_command delegates its work into two new methods: - _run_command__nt - _run_command__generic --- testgres/operations/local_ops.py | 76 ++++++++++++++++++-------------- 1 file changed, 42 insertions(+), 34 deletions(-) diff --git a/testgres/operations/local_ops.py b/testgres/operations/local_ops.py index a0a9926d..5b7972ae 100644 --- a/testgres/operations/local_ops.py +++ b/testgres/operations/local_ops.py @@ -64,47 +64,55 @@ def _process_output(encoding, temp_file_path): output = output.decode(encoding) return output, None # In Windows stderr writing in stdout - def _run_command(self, cmd, shell, input, stdin, stdout, stderr, get_process, timeout, encoding): - """Execute a command and return the process and its output.""" - if os.name == 'nt' and stdout is None: # Windows - with tempfile.NamedTemporaryFile(mode='w+b', delete=False) as temp_file: - stdout = temp_file - stderr = subprocess.STDOUT - process = subprocess.Popen( - cmd, - shell=shell, - stdin=stdin or subprocess.PIPE if input is not None else None, - stdout=stdout, - stderr=stderr, - ) - if get_process: - return process, None, None - temp_file_path = temp_file.name - - # Wait process finished - process.wait() - - output, error = self._process_output(encoding, temp_file_path) - return process, output, error - else: # Other OS + def _run_command__nt(self, cmd, shell, input, stdin, stdout, stderr, get_process, timeout, encoding): + with tempfile.NamedTemporaryFile(mode='w+b', delete=False) as temp_file: + stdout = temp_file + stderr = subprocess.STDOUT process = subprocess.Popen( cmd, shell=shell, stdin=stdin or subprocess.PIPE if input is not None else None, - stdout=stdout or subprocess.PIPE, - stderr=stderr or subprocess.PIPE, + stdout=stdout, + stderr=stderr, ) if get_process: return process, None, None - try: - output, error = process.communicate(input=input.encode(encoding) if input else None, timeout=timeout) - if encoding: - output = output.decode(encoding) - error = error.decode(encoding) - return process, output, error - except subprocess.TimeoutExpired: - process.kill() - raise ExecUtilException("Command timed out after {} seconds.".format(timeout)) + temp_file_path = temp_file.name + + # Wait process finished + process.wait() + + output, error = self._process_output(encoding, temp_file_path) + return process, output, error + + def _run_command__generic(self, cmd, shell, input, stdin, stdout, stderr, get_process, timeout, encoding): + process = subprocess.Popen( + cmd, + shell=shell, + stdin=stdin or subprocess.PIPE if input is not None else None, + stdout=stdout or subprocess.PIPE, + stderr=stderr or subprocess.PIPE, + ) + if get_process: + return process, None, None + try: + output, error = process.communicate(input=input.encode(encoding) if input else None, timeout=timeout) + if encoding: + output = output.decode(encoding) + error = error.decode(encoding) + return process, output, error + except subprocess.TimeoutExpired: + process.kill() + raise ExecUtilException("Command timed out after {} seconds.".format(timeout)) + + def _run_command(self, cmd, shell, input, stdin, stdout, stderr, get_process, timeout, encoding): + """Execute a command and return the process and its output.""" + if os.name == 'nt' and stdout is None: # Windows + method = __class__._run_command__nt + else: # Other OS + method = __class__._run_command__generic + + return method(self, cmd, shell, input, stdin, stdout, stderr, get_process, timeout, encoding) def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False, encoding=None, shell=False, text=False, input=None, stdin=None, stdout=None, stderr=None, get_process=False, timeout=None, From cf1d227d7fee878247a26f7f1b8636464d52c576 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Sat, 7 Dec 2024 21:40:41 +0300 Subject: [PATCH 333/475] Local and remote test code formatting has been synchronized test_simple.py and test_simple_remote.py have been compared and synchronized. --- tests/test_simple.py | 7 ------- tests/test_simple_remote.py | 2 +- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/tests/test_simple.py b/tests/test_simple.py index ffefda6c..51cdc896 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -153,7 +153,6 @@ def test_init_unique_system_id(self): with scoped_config(cache_initdb=True, cached_initdb_unique=True) as config: - self.assertTrue(config.cache_initdb) self.assertTrue(config.cached_initdb_unique) @@ -376,13 +375,11 @@ def test_backup_multiple(self): with node.backup(xlog_method='fetch') as backup1, \ node.backup(xlog_method='fetch') as backup2: - self.assertNotEqual(backup1.base_dir, backup2.base_dir) with node.backup(xlog_method='fetch') as backup: with backup.spawn_primary('node1', destroy=False) as node1, \ backup.spawn_primary('node2', destroy=False) as node2: - self.assertNotEqual(node1.base_dir, node2.base_dir) def test_backup_exhaust(self): @@ -390,7 +387,6 @@ def test_backup_exhaust(self): node.init(allow_streaming=True).start() with node.backup(xlog_method='fetch') as backup: - # exhaust backup by creating new node with backup.spawn_primary(): pass @@ -778,7 +774,6 @@ def test_pg_config(self): # modify setting for this scope with scoped_config(cache_pg_config=False) as config: - # sanity check for value self.assertFalse(config.cache_pg_config) @@ -810,7 +805,6 @@ def test_config_stack(self): self.assertEqual(c1.cached_initdb_dir, d1) with scoped_config(cached_initdb_dir=d2) as c2: - stack_size = len(testgres.config.config_stack) # try to break a stack @@ -840,7 +834,6 @@ def test_unix_sockets(self): def test_auto_name(self): with get_new_node().init(allow_streaming=True).start() as m: with m.replicate().start() as r: - # check that nodes are running self.assertTrue(m.status()) self.assertTrue(r.status()) diff --git a/tests/test_simple_remote.py b/tests/test_simple_remote.py index 79bdb74c..936c31f2 100755 --- a/tests/test_simple_remote.py +++ b/tests/test_simple_remote.py @@ -94,7 +94,6 @@ def removing(f): class TestgresRemoteTests(unittest.TestCase): - def test_node_repr(self): with get_remote_node(conn_params=conn_params) as node: pattern = r"PostgresNode\(name='.+', port=.+, base_dir='.+'\)" @@ -748,6 +747,7 @@ def test_pg_config(self): # save right before config change c1 = get_pg_config() + # modify setting for this scope with scoped_config(cache_pg_config=False) as config: # sanity check for value From 22c649dcaa8129bc240d49b24f2e363832e70d9a Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Sat, 7 Dec 2024 23:14:00 +0300 Subject: [PATCH 334/475] test_local.py and test_exec_command_failure__expect_error are added test_local.py contains set of test for LocalOperations - test_exec_command_success - test_exec_command_failure - test_exec_command_failure__expect_error Changes in TestRemoteOperations: - test_exec_command_failure exptects an exception - new test test_exec_command_failure__expect_error was added TestRemoteOperations::test_exec_command_failure__expect_error will fail because RemoteOperations::exec_command does not handle the 'expect_error' parameter correctly. --- tests/test_local.py | 46 ++++++++++++++++++++++++++++++++++++++++++++ tests/test_remote.py | 23 ++++++++++++++++++---- 2 files changed, 65 insertions(+), 4 deletions(-) create mode 100644 tests/test_local.py diff --git a/tests/test_local.py b/tests/test_local.py new file mode 100644 index 00000000..1caba74b --- /dev/null +++ b/tests/test_local.py @@ -0,0 +1,46 @@ +import pytest + +from testgres import ExecUtilException +from testgres import LocalOperations + + +class TestLocalOperations: + + @pytest.fixture(scope="function", autouse=True) + def setup(self): + self.operations = LocalOperations() + + def test_exec_command_success(self): + """ + Test exec_command for successful command execution. + """ + cmd = "python3 --version" + response = self.operations.exec_command(cmd, wait_exit=True, shell=True) + + assert b'Python 3.' in response + + def test_exec_command_failure(self): + """ + Test exec_command for command execution failure. + """ + cmd = "nonexistent_command" + while True: + try: + self.operations.exec_command(cmd, wait_exit=True, shell=True) + except ExecUtilException as e: + error = e.message + break + raise Exception("We wait an exception!") + assert error == "Utility exited with non-zero code. Error `b'/bin/sh: 1: nonexistent_command: not found\\n'`" + + def test_exec_command_failure__expect_error(self): + """ + Test exec_command for command execution failure. + """ + cmd = "nonexistent_command" + + exit_status, result, error = self.operations.exec_command(cmd, verbose=True, wait_exit=True, shell=True, expect_error=True) + + assert error == b'/bin/sh: 1: nonexistent_command: not found\n' + assert exit_status == 127 + assert result == b'' diff --git a/tests/test_remote.py b/tests/test_remote.py index e0e4a555..565163f7 100755 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -30,12 +30,27 @@ def test_exec_command_failure(self): Test exec_command for command execution failure. """ cmd = "nonexistent_command" - try: - exit_status, result, error = self.operations.exec_command(cmd, verbose=True, wait_exit=True) - except ExecUtilException as e: - error = e.message + while True: + try: + self.operations.exec_command(cmd, verbose=True, wait_exit=True) + except ExecUtilException as e: + error = e.message + break + raise Exception("We wait an exception!") assert error == b'Utility exited with non-zero code. Error: bash: line 1: nonexistent_command: command not found\n' + def test_exec_command_failure__expect_error(self): + """ + Test exec_command for command execution failure. + """ + cmd = "nonexistent_command" + + exit_status, result, error = self.operations.exec_command(cmd, verbose=True, wait_exit=True, shell=True, expect_error=True) + + assert error == b'bash: line 1: nonexistent_command: command not found\n' + assert exit_status == 127 + assert result == b'' + def test_is_executable_true(self): """ Test is_executable for an existing executable. From 45cfbf738e1f5d7f46f8ab053f4a6a481abf53e7 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Sat, 7 Dec 2024 23:26:04 +0300 Subject: [PATCH 335/475] RemoteOperations::exec_command must not raise an exception when 'expect_error' is True (#159) This commit fixes an issue #159. --- testgres/operations/remote_ops.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index 20095051..88394eb7 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -83,26 +83,26 @@ def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False, exit_status = process.returncode - if encoding: - result = result.decode(encoding) - error = error.decode(encoding) - - if expect_error: - raise Exception(result, error) + assert type(result) == bytes # noqa: E721 + assert type(error) == bytes # noqa: E721 if not error: - error_found = 0 + error_found = False else: - error = normalize_error(error) error_found = exit_status != 0 or any( - marker in error for marker in ['error', 'Permission denied', 'fatal', 'No such file or directory'] + marker in error for marker in [b'error', b'Permission denied', b'fatal', b'No such file or directory'] ) - if not ignore_errors and error_found: - if isinstance(error, bytes): - message = b"Utility exited with non-zero code. Error: " + error - else: - message = f"Utility exited with non-zero code. Error: {error}" + assert type(error_found) == bool # noqa: E721 + + if encoding: + result = result.decode(encoding) + error = error.decode(encoding) + + if not ignore_errors and error_found and not expect_error: + error = normalize_error(error) + assert type(error) == str # noqa: E721 + message = "Utility exited with non-zero code. Error: " + error raise ExecUtilException(message=message, command=cmd, exit_code=exit_status, out=result) if verbose: From 1c64337682288201f4c163ad4f12afdca82e33cb Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Sun, 8 Dec 2024 00:12:41 +0300 Subject: [PATCH 336/475] TestLocalOperations tests skip Windows --- tests/test_local.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/test_local.py b/tests/test_local.py index 1caba74b..3493810f 100644 --- a/tests/test_local.py +++ b/tests/test_local.py @@ -1,4 +1,5 @@ import pytest +import platform from testgres import ExecUtilException from testgres import LocalOperations @@ -10,10 +11,16 @@ class TestLocalOperations: def setup(self): self.operations = LocalOperations() + def skip_if_windows(): + if platform.system().lower() == "windows": + pytest.skip("This test does not support Windows.") + def test_exec_command_success(self): """ Test exec_command for successful command execution. """ + __class__.skip_if_windows() + cmd = "python3 --version" response = self.operations.exec_command(cmd, wait_exit=True, shell=True) @@ -23,6 +30,8 @@ def test_exec_command_failure(self): """ Test exec_command for command execution failure. """ + __class__.skip_if_windows() + cmd = "nonexistent_command" while True: try: @@ -37,6 +46,8 @@ def test_exec_command_failure__expect_error(self): """ Test exec_command for command execution failure. """ + __class__.skip_if_windows() + cmd = "nonexistent_command" exit_status, result, error = self.operations.exec_command(cmd, verbose=True, wait_exit=True, shell=True, expect_error=True) From cb87d0a7b1c87a10c85787591c3d9ba4e6687e76 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Sun, 8 Dec 2024 15:26:58 +0300 Subject: [PATCH 337/475] tests.helpers.RunConditions is added RunConditions contains the code to check the execution condition of tests. It is used in TestLocalOperations. --- tests/__init__.py | 0 tests/helpers/__init__.py | 0 tests/helpers/run_conditions.py | 11 +++++++++++ tests/test_local.py | 13 +++++-------- 4 files changed, 16 insertions(+), 8 deletions(-) create mode 100644 tests/__init__.py create mode 100644 tests/helpers/__init__.py create mode 100644 tests/helpers/run_conditions.py diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/helpers/__init__.py b/tests/helpers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/helpers/run_conditions.py b/tests/helpers/run_conditions.py new file mode 100644 index 00000000..8d57f753 --- /dev/null +++ b/tests/helpers/run_conditions.py @@ -0,0 +1,11 @@ +import pytest +import platform + + +class RunConditions: + # It is not a test kit! + __test__ = False + + def skip_if_windows(): + if platform.system().lower() == "windows": + pytest.skip("This test does not support Windows.") diff --git a/tests/test_local.py b/tests/test_local.py index 3493810f..da26468b 100644 --- a/tests/test_local.py +++ b/tests/test_local.py @@ -1,9 +1,10 @@ import pytest -import platform from testgres import ExecUtilException from testgres import LocalOperations +from .helpers.run_conditions import RunConditions + class TestLocalOperations: @@ -11,15 +12,11 @@ class TestLocalOperations: def setup(self): self.operations = LocalOperations() - def skip_if_windows(): - if platform.system().lower() == "windows": - pytest.skip("This test does not support Windows.") - def test_exec_command_success(self): """ Test exec_command for successful command execution. """ - __class__.skip_if_windows() + RunConditions.skip_if_windows() cmd = "python3 --version" response = self.operations.exec_command(cmd, wait_exit=True, shell=True) @@ -30,7 +27,7 @@ def test_exec_command_failure(self): """ Test exec_command for command execution failure. """ - __class__.skip_if_windows() + RunConditions.skip_if_windows() cmd = "nonexistent_command" while True: @@ -46,7 +43,7 @@ def test_exec_command_failure__expect_error(self): """ Test exec_command for command execution failure. """ - __class__.skip_if_windows() + RunConditions.skip_if_windows() cmd = "nonexistent_command" From f9ddd043aceb3ac86f6b63cd7fba00575e0d44b1 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Sun, 8 Dec 2024 22:06:29 +0300 Subject: [PATCH 338/475] Proposal to fix #154 (v2) - The one way to generate ExecUtilException - RaiseError.UtilityExitedWithNonZeroCode - RaiseError is added - ExecUtilException::error is added (it contains the error data) - ExecUtilException::__str__ is updated - PostgresNode::psql and PostgresNode::safe_psql are updated - TestLocalOperations::test_exec_command_failure is updated - TestRemoteOperations::test_exec_command_failure is updated - TestRemoteOperations::test_makedirs_and_rmdirs_failure is updated --- testgres/exceptions.py | 11 +++++-- testgres/helpers/raise_error.py | 45 ++++++++++++++++++++++++++++ testgres/node.py | 50 ++++++++++++++++--------------- testgres/operations/local_ops.py | 20 +++++++------ testgres/operations/remote_ops.py | 14 ++++++--- tests/test_local.py | 2 +- tests/test_remote.py | 15 ++++++---- 7 files changed, 110 insertions(+), 47 deletions(-) create mode 100644 testgres/helpers/raise_error.py diff --git a/testgres/exceptions.py b/testgres/exceptions.py index ee329031..ff4381f4 100644 --- a/testgres/exceptions.py +++ b/testgres/exceptions.py @@ -9,13 +9,14 @@ class TestgresException(Exception): @six.python_2_unicode_compatible class ExecUtilException(TestgresException): - def __init__(self, message=None, command=None, exit_code=0, out=None): + def __init__(self, message=None, command=None, exit_code=0, out=None, error=None): super(ExecUtilException, self).__init__(message) self.message = message self.command = command self.exit_code = exit_code self.out = out + self.error = error def __str__(self): msg = [] @@ -24,13 +25,17 @@ def __str__(self): msg.append(self.message) if self.command: - msg.append(u'Command: {}'.format(self.command)) + command_s = ' '.join(self.command) if isinstance(self.command, list) else self.command, + msg.append(u'Command: {}'.format(command_s)) if self.exit_code: msg.append(u'Exit code: {}'.format(self.exit_code)) + if self.error: + msg.append(u'---- Error:\n{}'.format(self.error)) + if self.out: - msg.append(u'----\n{}'.format(self.out)) + msg.append(u'---- Out:\n{}'.format(self.out)) return self.convert_and_join(msg) diff --git a/testgres/helpers/raise_error.py b/testgres/helpers/raise_error.py new file mode 100644 index 00000000..c67833dd --- /dev/null +++ b/testgres/helpers/raise_error.py @@ -0,0 +1,45 @@ +from ..exceptions import ExecUtilException + + +class RaiseError: + def UtilityExitedWithNonZeroCode(cmd, exit_code, msg_arg, error, out): + assert type(exit_code) == int # noqa: E721 + + msg_arg_s = __class__._TranslateDataIntoString(msg_arg).strip() + assert type(msg_arg_s) == str # noqa: E721 + + if msg_arg_s == "": + msg_arg_s = "#no_error_message" + + message = "Utility exited with non-zero code. Error: `" + msg_arg_s + "`" + raise ExecUtilException( + message=message, + command=cmd, + exit_code=exit_code, + out=out, + error=error) + + def _TranslateDataIntoString(data): + if type(data) == bytes: # noqa: E721 + return __class__._TranslateDataIntoString__FromBinary(data) + + return str(data) + + def _TranslateDataIntoString__FromBinary(data): + assert type(data) == bytes # noqa: E721 + + try: + return data.decode('utf-8') + except UnicodeDecodeError: + pass + + return "#cannot_decode_text" + + def _BinaryIsASCII(data): + assert type(data) == bytes # noqa: E721 + + for b in data: + if not (b >= 0 and b <= 127): + return False + + return True diff --git a/testgres/node.py b/testgres/node.py index 48a100a9..8300d493 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -3,7 +3,6 @@ import os import random import signal -import subprocess import threading import tempfile from queue import Queue @@ -987,6 +986,25 @@ def psql(self, >>> psql(query='select 3', ON_ERROR_STOP=1) """ + return self._psql( + ignore_errors=True, + query=query, + filename=filename, + dbname=dbname, + username=username, + input=input, + **variables + ) + + def _psql( + self, + ignore_errors, + query=None, + filename=None, + dbname=None, + username=None, + input=None, + **variables): dbname = dbname or default_dbname() psql_params = [ @@ -1017,20 +1035,8 @@ def psql(self, # should be the last one psql_params.append(dbname) - if not self.os_ops.remote: - # start psql process - process = subprocess.Popen(psql_params, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - - # wait until it finishes and get stdout and stderr - out, err = process.communicate(input=input) - return process.returncode, out, err - else: - status_code, out, err = self.os_ops.exec_command(psql_params, verbose=True, input=input) - return status_code, out, err + return self.os_ops.exec_command(psql_params, verbose=True, input=input, ignore_errors=ignore_errors) @method_decorator(positional_args_hack(['dbname', 'query'])) def safe_psql(self, query=None, expect_error=False, **kwargs): @@ -1051,21 +1057,17 @@ def safe_psql(self, query=None, expect_error=False, **kwargs): Returns: psql's output as str. """ + assert type(kwargs) == dict # noqa: E721 + assert not ("ignore_errors" in kwargs.keys()) # force this setting kwargs['ON_ERROR_STOP'] = 1 try: - ret, out, err = self.psql(query=query, **kwargs) + ret, out, err = self._psql(ignore_errors=False, query=query, **kwargs) except ExecUtilException as e: - ret = e.exit_code - out = e.out - err = e.message - if ret: - if expect_error: - out = (err or b'').decode('utf-8') - else: - raise QueryException((err or b'').decode('utf-8'), query) - elif expect_error: + raise QueryException(e.message, query) + + if expect_error: assert False, "Exception was expected, but query finished successfully: `{}` ".format(query) return out diff --git a/testgres/operations/local_ops.py b/testgres/operations/local_ops.py index 5b7972ae..14c408c9 100644 --- a/testgres/operations/local_ops.py +++ b/testgres/operations/local_ops.py @@ -11,6 +11,7 @@ from ..exceptions import ExecUtilException from .os_ops import ConnectionParams, OsOperations, pglib, get_default_encoding +from ..helpers.raise_error import RaiseError try: from shutil import which as find_executable @@ -47,14 +48,6 @@ def __init__(self, conn_params=None): self.remote = False self.username = conn_params.username or getpass.getuser() - @staticmethod - def _raise_exec_exception(message, command, exit_code, output): - """Raise an ExecUtilException.""" - raise ExecUtilException(message=message.format(output), - command=' '.join(command) if isinstance(command, list) else command, - exit_code=exit_code, - out=output) - @staticmethod def _process_output(encoding, temp_file_path): """Process the output of a command from a temporary file.""" @@ -120,11 +113,20 @@ def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False, """ Execute a command in a subprocess and handle the output based on the provided parameters. """ + assert type(expect_error) == bool # noqa: E721 + assert type(ignore_errors) == bool # noqa: E721 + process, output, error = self._run_command(cmd, shell, input, stdin, stdout, stderr, get_process, timeout, encoding) if get_process: return process if not ignore_errors and ((process.returncode != 0 or has_errors(output=output, error=error)) and not expect_error): - self._raise_exec_exception('Utility exited with non-zero code. Error `{}`', cmd, process.returncode, error or output) + RaiseError.UtilityExitedWithNonZeroCode( + cmd=cmd, + exit_code=process.returncode, + msg_arg=error or output, + error=error, + out=output + ) if verbose: return process.returncode, output, error diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index 88394eb7..4340ec11 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -14,6 +14,7 @@ raise ImportError("You must have psycopg2 or pg8000 modules installed") from ..exceptions import ExecUtilException +from ..helpers.raise_error import RaiseError from .os_ops import OsOperations, ConnectionParams, get_default_encoding error_markers = [b'error', b'Permission denied', b'fatal', b'No such file or directory'] @@ -66,6 +67,9 @@ def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False, Args: - cmd (str): The command to be executed. """ + assert type(expect_error) == bool # noqa: E721 + assert type(ignore_errors) == bool # noqa: E721 + ssh_cmd = [] if isinstance(cmd, str): ssh_cmd = ['ssh', self.ssh_dest] + self.ssh_args + [cmd] @@ -100,10 +104,12 @@ def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False, error = error.decode(encoding) if not ignore_errors and error_found and not expect_error: - error = normalize_error(error) - assert type(error) == str # noqa: E721 - message = "Utility exited with non-zero code. Error: " + error - raise ExecUtilException(message=message, command=cmd, exit_code=exit_status, out=result) + RaiseError.UtilityExitedWithNonZeroCode( + cmd=cmd, + exit_code=exit_status, + msg_arg=error, + error=error, + out=result) if verbose: return exit_status, result, error diff --git a/tests/test_local.py b/tests/test_local.py index da26468b..cb96a3bc 100644 --- a/tests/test_local.py +++ b/tests/test_local.py @@ -37,7 +37,7 @@ def test_exec_command_failure(self): error = e.message break raise Exception("We wait an exception!") - assert error == "Utility exited with non-zero code. Error `b'/bin/sh: 1: nonexistent_command: not found\\n'`" + assert error == "Utility exited with non-zero code. Error: `/bin/sh: 1: nonexistent_command: not found`" def test_exec_command_failure__expect_error(self): """ diff --git a/tests/test_remote.py b/tests/test_remote.py index 565163f7..c1a91bc6 100755 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -37,7 +37,7 @@ def test_exec_command_failure(self): error = e.message break raise Exception("We wait an exception!") - assert error == b'Utility exited with non-zero code. Error: bash: line 1: nonexistent_command: command not found\n' + assert error == 'Utility exited with non-zero code. Error: `bash: line 1: nonexistent_command: command not found`' def test_exec_command_failure__expect_error(self): """ @@ -98,11 +98,14 @@ def test_makedirs_and_rmdirs_failure(self): self.operations.makedirs(path) # Test rmdirs - try: - exit_status, result, error = self.operations.rmdirs(path, verbose=True) - except ExecUtilException as e: - error = e.message - assert error == b"Utility exited with non-zero code. Error: rm: cannot remove '/root/test_dir': Permission denied\n" + while True: + try: + self.operations.rmdirs(path, verbose=True) + except ExecUtilException as e: + error = e.message + break + raise Exception("We wait an exception!") + assert error == "Utility exited with non-zero code. Error: `rm: cannot remove '/root/test_dir': Permission denied`" def test_listdir(self): """ From 2bb38dc45b69c4d5d4c93a65fea4b64b81511287 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Mon, 9 Dec 2024 11:30:51 +0300 Subject: [PATCH 339/475] [BUG FIX] PostgresNode::safe_psql did not respect "expect_error" parameter --- testgres/node.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/testgres/node.py b/testgres/node.py index 8300d493..11f73af2 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -1059,13 +1059,22 @@ def safe_psql(self, query=None, expect_error=False, **kwargs): """ assert type(kwargs) == dict # noqa: E721 assert not ("ignore_errors" in kwargs.keys()) + assert not ("expect_error" in kwargs.keys()) # force this setting kwargs['ON_ERROR_STOP'] = 1 try: ret, out, err = self._psql(ignore_errors=False, query=query, **kwargs) except ExecUtilException as e: - raise QueryException(e.message, query) + if not expect_error: + raise QueryException(e.message, query) + + if type(e.error) == bytes: # noqa: E721 + return e.error.decode("utf-8") # throw + + # [2024-12-09] This situation is not expected + assert False + return e.error if expect_error: assert False, "Exception was expected, but query finished successfully: `{}` ".format(query) From 45b8dc024dc39994bed029ebb1d363686ae71cbf Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Mon, 9 Dec 2024 15:07:21 +0300 Subject: [PATCH 340/475] [BUG FIX] A problem in psql/safe_psql and 'input' data was fixed [local_op] Both LocalOperations::exec_command and RemoteOperations::exec_command were updated. --- testgres/node.py | 12 ++++++++++++ testgres/operations/helpers.py | 15 +++++++++++++++ testgres/operations/local_ops.py | 9 ++++++++- testgres/operations/remote_ops.py | 7 ++++++- 4 files changed, 41 insertions(+), 2 deletions(-) create mode 100644 testgres/operations/helpers.py diff --git a/testgres/node.py b/testgres/node.py index 11f73af2..68e9b0eb 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -1005,6 +1005,18 @@ def _psql( username=None, input=None, **variables): + assert type(variables) == dict # noqa: E721 + + # + # We do not support encoding. It may be added later. Ok? + # + if input is None: + pass + elif type(input) == bytes: # noqa: E721 + pass + else: + raise Exception("Input data must be None or bytes.") + dbname = dbname or default_dbname() psql_params = [ diff --git a/testgres/operations/helpers.py b/testgres/operations/helpers.py new file mode 100644 index 00000000..d714d336 --- /dev/null +++ b/testgres/operations/helpers.py @@ -0,0 +1,15 @@ +class Helpers: + def PrepareProcessInput(input, encoding): + if not input: + return None + + if type(input) == str: # noqa: E721 + if encoding is None: + return input.encode() + + assert type(encoding) == str # noqa: E721 + return input.encode(encoding) + + # It is expected! + assert type(input) == bytes # noqa: E721 + return input diff --git a/testgres/operations/local_ops.py b/testgres/operations/local_ops.py index 14c408c9..833d1fd2 100644 --- a/testgres/operations/local_ops.py +++ b/testgres/operations/local_ops.py @@ -11,6 +11,7 @@ from ..exceptions import ExecUtilException from .os_ops import ConnectionParams, OsOperations, pglib, get_default_encoding +from .helpers import Helpers from ..helpers.raise_error import RaiseError try: @@ -58,6 +59,8 @@ def _process_output(encoding, temp_file_path): return output, None # In Windows stderr writing in stdout def _run_command__nt(self, cmd, shell, input, stdin, stdout, stderr, get_process, timeout, encoding): + # TODO: why don't we use the data from input? + with tempfile.NamedTemporaryFile(mode='w+b', delete=False) as temp_file: stdout = temp_file stderr = subprocess.STDOUT @@ -79,6 +82,10 @@ def _run_command__nt(self, cmd, shell, input, stdin, stdout, stderr, get_process return process, output, error def _run_command__generic(self, cmd, shell, input, stdin, stdout, stderr, get_process, timeout, encoding): + input_prepared = None + if not get_process: + input_prepared = Helpers.PrepareProcessInput(input, encoding) # throw + process = subprocess.Popen( cmd, shell=shell, @@ -89,7 +96,7 @@ def _run_command__generic(self, cmd, shell, input, stdin, stdout, stderr, get_pr if get_process: return process, None, None try: - output, error = process.communicate(input=input.encode(encoding) if input else None, timeout=timeout) + output, error = process.communicate(input=input_prepared, timeout=timeout) if encoding: output = output.decode(encoding) error = error.decode(encoding) diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index 4340ec11..cbaeb62b 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -16,6 +16,7 @@ from ..exceptions import ExecUtilException from ..helpers.raise_error import RaiseError from .os_ops import OsOperations, ConnectionParams, get_default_encoding +from .helpers import Helpers error_markers = [b'error', b'Permission denied', b'fatal', b'No such file or directory'] @@ -70,6 +71,10 @@ def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False, assert type(expect_error) == bool # noqa: E721 assert type(ignore_errors) == bool # noqa: E721 + input_prepared = None + if not get_process: + input_prepared = Helpers.PrepareProcessInput(input, encoding) # throw + ssh_cmd = [] if isinstance(cmd, str): ssh_cmd = ['ssh', self.ssh_dest] + self.ssh_args + [cmd] @@ -80,7 +85,7 @@ def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False, return process try: - result, error = process.communicate(input, timeout=timeout) + result, error = process.communicate(input=input_prepared, timeout=timeout) except subprocess.TimeoutExpired: process.kill() raise ExecUtilException("Command timed out after {} seconds.".format(timeout)) From f848a63fd3dce4a50d1a994f2b5ee1ce9e15a7b6 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Mon, 9 Dec 2024 17:50:50 +0300 Subject: [PATCH 341/475] PostgresNode::safe_psql raises InvalidOperationException If expect_error is True and no error is detected, safe_psql raises InvalidOperationException exception. Tests (local, remote) are added. --- testgres/__init__.py | 3 ++- testgres/exceptions.py | 3 +++ testgres/node.py | 5 +++-- tests/test_simple.py | 21 ++++++++++++++++++++- tests/test_simple_remote.py | 20 +++++++++++++++++++- 5 files changed, 47 insertions(+), 5 deletions(-) diff --git a/testgres/__init__.py b/testgres/__init__.py index 8d0e38c6..69d2ab4a 100644 --- a/testgres/__init__.py +++ b/testgres/__init__.py @@ -23,7 +23,8 @@ CatchUpException, \ StartNodeException, \ InitNodeException, \ - BackupException + BackupException, \ + InvalidOperationException from .enums import \ XLogMethod, \ diff --git a/testgres/exceptions.py b/testgres/exceptions.py index ff4381f4..b4d9f76b 100644 --- a/testgres/exceptions.py +++ b/testgres/exceptions.py @@ -103,3 +103,6 @@ class InitNodeException(TestgresException): class BackupException(TestgresException): pass + +class InvalidOperationException(TestgresException): + pass diff --git a/testgres/node.py b/testgres/node.py index 68e9b0eb..ae52f21b 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -73,7 +73,8 @@ TimeoutException, \ InitNodeException, \ TestgresException, \ - BackupException + BackupException, \ + InvalidOperationException from .logger import TestgresLogger @@ -1089,7 +1090,7 @@ def safe_psql(self, query=None, expect_error=False, **kwargs): return e.error if expect_error: - assert False, "Exception was expected, but query finished successfully: `{}` ".format(query) + raise InvalidOperationException("Exception was expected, but query finished successfully: `{}`.".format(query)) return out diff --git a/tests/test_simple.py b/tests/test_simple.py index 51cdc896..c4200c6f 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -23,7 +23,9 @@ BackupException, \ QueryException, \ TimeoutException, \ - TestgresException, NodeApp + TestgresException, \ + InvalidOperationException, \ + NodeApp from testgres import \ TestgresConfig, \ @@ -310,6 +312,23 @@ def test_psql(self): with self.assertRaises(QueryException): node.safe_psql('select 1') + def test_safe_psql__expect_error(self): + with get_new_node().init().start() as node: + err = node.safe_psql('select_or_not_select 1', expect_error=True) + self.assertTrue(type(err) == str) # noqa: E721 + self.assertIn('select_or_not_select', err) + self.assertIn('ERROR: syntax error at or near "select_or_not_select"', err) + + # --------- + with self.assertRaises(InvalidOperationException) as ctx: + node.safe_psql("select 1;", expect_error=True) + + self.assertEqual(str(ctx.exception), "Exception was expected, but query finished successfully: `select 1;`.") + + # --------- + res = node.safe_psql("select 1;", expect_error=False) + self.assertEqual(res, b'1\n') + def test_transactions(self): with get_new_node().init().start() as node: diff --git a/tests/test_simple_remote.py b/tests/test_simple_remote.py index 936c31f2..26ac7c61 100755 --- a/tests/test_simple_remote.py +++ b/tests/test_simple_remote.py @@ -23,7 +23,8 @@ BackupException, \ QueryException, \ TimeoutException, \ - TestgresException + TestgresException, \ + InvalidOperationException from testgres.config import \ TestgresConfig, \ @@ -295,6 +296,23 @@ def test_psql(self): with self.assertRaises(QueryException): node.safe_psql('select 1') + def test_safe_psql__expect_error(self): + with get_remote_node(conn_params=conn_params).init().start() as node: + err = node.safe_psql('select_or_not_select 1', expect_error=True) + self.assertTrue(type(err) == str) # noqa: E721 + self.assertIn('select_or_not_select', err) + self.assertIn('ERROR: syntax error at or near "select_or_not_select"', err) + + # --------- + with self.assertRaises(InvalidOperationException) as ctx: + node.safe_psql("select 1;", expect_error=True) + + self.assertEqual(str(ctx.exception), "Exception was expected, but query finished successfully: `select 1;`.") + + # --------- + res = node.safe_psql("select 1;", expect_error=False) + self.assertEqual(res, b'1\n') + def test_transactions(self): with get_remote_node(conn_params=conn_params).init().start() as node: with node.connect() as con: From db0744e7a575ebd5d7263c2a9fe2a193f7716c48 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Mon, 9 Dec 2024 18:02:12 +0300 Subject: [PATCH 342/475] A problem with InvalidOperationException and flake8 is fixed --- testgres/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testgres/__init__.py b/testgres/__init__.py index 69d2ab4a..665548d6 100644 --- a/testgres/__init__.py +++ b/testgres/__init__.py @@ -61,7 +61,7 @@ "NodeBackup", "testgres_config", "TestgresConfig", "configure_testgres", "scoped_config", "push_config", "pop_config", "NodeConnection", "DatabaseError", "InternalError", "ProgrammingError", "OperationalError", - "TestgresException", "ExecUtilException", "QueryException", "TimeoutException", "CatchUpException", "StartNodeException", "InitNodeException", "BackupException", + "TestgresException", "ExecUtilException", "QueryException", "TimeoutException", "CatchUpException", "StartNodeException", "InitNodeException", "BackupException", "InvalidOperationException", "XLogMethod", "IsolationLevel", "NodeStatus", "ProcessType", "DumpFormat", "PostgresNode", "NodeApp", "reserve_port", "release_port", "bound_ports", "get_bin_path", "get_pg_config", "get_pg_version", From 5bb1510bdb8a48d84bd2b5ea0c4a20f6951a8edf Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Mon, 9 Dec 2024 18:27:29 +0300 Subject: [PATCH 343/475] A code style is fixed [flake8] --- testgres/exceptions.py | 1 + 1 file changed, 1 insertion(+) diff --git a/testgres/exceptions.py b/testgres/exceptions.py index b4d9f76b..d61d4691 100644 --- a/testgres/exceptions.py +++ b/testgres/exceptions.py @@ -104,5 +104,6 @@ class InitNodeException(TestgresException): class BackupException(TestgresException): pass + class InvalidOperationException(TestgresException): pass From 31c7bce1e21e796d996873f3c9ae270e52992f88 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Mon, 9 Dec 2024 19:56:32 +0300 Subject: [PATCH 344/475] [BUG FIX] Wrappers for psql use subprocess.PIPE for stdout and stderr It fixes a problem with Windows. --- testgres/node.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/testgres/node.py b/testgres/node.py index ae52f21b..1037aca2 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -5,6 +5,7 @@ import signal import threading import tempfile +import subprocess from queue import Queue import time @@ -1049,7 +1050,13 @@ def _psql( # should be the last one psql_params.append(dbname) - return self.os_ops.exec_command(psql_params, verbose=True, input=input, ignore_errors=ignore_errors) + return self.os_ops.exec_command( + psql_params, + verbose=True, + input=input, + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + ignore_errors=ignore_errors) @method_decorator(positional_args_hack(['dbname', 'query'])) def safe_psql(self, query=None, expect_error=False, **kwargs): From b013801e5aff06d1daa13e805d6080906f953868 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Mon, 9 Dec 2024 20:23:56 +0300 Subject: [PATCH 345/475] [BUG FIX] TestgresTests::test_safe_psql__expect_error uses rm_carriage_returns It fixes a problem with this test on Windows. --- tests/test_simple.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_simple.py b/tests/test_simple.py index c4200c6f..fade468c 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -327,7 +327,7 @@ def test_safe_psql__expect_error(self): # --------- res = node.safe_psql("select 1;", expect_error=False) - self.assertEqual(res, b'1\n') + self.assertEqual(rm_carriage_returns(res), b'1\n') def test_transactions(self): with get_new_node().init().start() as node: From c49ee4cf5979b83dbcecb085a2b8473f32474271 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 10 Dec 2024 11:22:28 +0300 Subject: [PATCH 346/475] node.py is updated [formatting] The previous order of imports is restored for minimization number of changes. --- testgres/node.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testgres/node.py b/testgres/node.py index 1037aca2..32b1e244 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -3,9 +3,9 @@ import os import random import signal +import subprocess import threading import tempfile -import subprocess from queue import Queue import time From 6a0e71495740caa32cbd54cd4b2a3819e13de539 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 10 Dec 2024 11:47:48 +0300 Subject: [PATCH 347/475] Formatting --- testgres/node.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testgres/node.py b/testgres/node.py index 32b1e244..b5cbab27 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -74,7 +74,7 @@ TimeoutException, \ InitNodeException, \ TestgresException, \ - BackupException, \ + BackupException, \ InvalidOperationException from .logger import TestgresLogger From 3cc19d2afdd390699ec234f83496cece996f2fe0 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 10 Dec 2024 14:55:46 +0300 Subject: [PATCH 348/475] raise_error.py is moved into testgres/operations from testgres/helpers Let's store our things on one place. We use RaiseError only in testgres/operations structures currently. --- testgres/operations/local_ops.py | 2 +- testgres/{helpers => operations}/raise_error.py | 0 testgres/operations/remote_ops.py | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename testgres/{helpers => operations}/raise_error.py (100%) diff --git a/testgres/operations/local_ops.py b/testgres/operations/local_ops.py index 833d1fd2..d6daaa3b 100644 --- a/testgres/operations/local_ops.py +++ b/testgres/operations/local_ops.py @@ -11,8 +11,8 @@ from ..exceptions import ExecUtilException from .os_ops import ConnectionParams, OsOperations, pglib, get_default_encoding +from .raise_error import RaiseError from .helpers import Helpers -from ..helpers.raise_error import RaiseError try: from shutil import which as find_executable diff --git a/testgres/helpers/raise_error.py b/testgres/operations/raise_error.py similarity index 100% rename from testgres/helpers/raise_error.py rename to testgres/operations/raise_error.py diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index cbaeb62b..c48f867b 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -14,8 +14,8 @@ raise ImportError("You must have psycopg2 or pg8000 modules installed") from ..exceptions import ExecUtilException -from ..helpers.raise_error import RaiseError from .os_ops import OsOperations, ConnectionParams, get_default_encoding +from .raise_error import RaiseError from .helpers import Helpers error_markers = [b'error', b'Permission denied', b'fatal', b'No such file or directory'] From 7b70e9e7a0b22fb999e308dab8bccc46e8e22a65 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 10 Dec 2024 16:21:39 +0300 Subject: [PATCH 349/475] Helpers.GetDefaultEncoding is added This function is equal to os_ops.get_default_encoding and is used in: - Helpers.PrepareProcessInput - RaiseError._TranslateDataIntoString__FromBinary --- testgres/operations/helpers.py | 39 +++++++++++++++++++++++++++++- testgres/operations/raise_error.py | 3 ++- 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/testgres/operations/helpers.py b/testgres/operations/helpers.py index d714d336..b50f0baa 100644 --- a/testgres/operations/helpers.py +++ b/testgres/operations/helpers.py @@ -1,11 +1,48 @@ +import locale + + class Helpers: + def _make_get_default_encoding_func(): + # locale.getencoding is added in Python 3.11 + if hasattr(locale, 'getencoding'): + return locale.getencoding + + # It must exist + return locale.getpreferredencoding + + # Prepared pointer on function to get a name of system codepage + _get_default_encoding_func = _make_get_default_encoding_func() + + def GetDefaultEncoding(): + # + # Original idea/source was: + # + # def os_ops.get_default_encoding(): + # if not hasattr(locale, 'getencoding'): + # locale.getencoding = locale.getpreferredencoding + # return locale.getencoding() or 'UTF-8' + # + + assert __class__._get_default_encoding_func is not None + + r = __class__._get_default_encoding_func() + + if r: + assert r is not None + assert type(r) == str # noqa: E721 + assert r != "" + return r + + # Is it an unexpected situation? + return 'UTF-8' + def PrepareProcessInput(input, encoding): if not input: return None if type(input) == str: # noqa: E721 if encoding is None: - return input.encode() + return input.encode(__class__.GetDefaultEncoding()) assert type(encoding) == str # noqa: E721 return input.encode(encoding) diff --git a/testgres/operations/raise_error.py b/testgres/operations/raise_error.py index c67833dd..0e760e74 100644 --- a/testgres/operations/raise_error.py +++ b/testgres/operations/raise_error.py @@ -1,4 +1,5 @@ from ..exceptions import ExecUtilException +from .helpers import Helpers class RaiseError: @@ -29,7 +30,7 @@ def _TranslateDataIntoString__FromBinary(data): assert type(data) == bytes # noqa: E721 try: - return data.decode('utf-8') + return data.decode(Helpers.GetDefaultEncoding()) except UnicodeDecodeError: pass From cd0b5f8671d61214afb2985adb83cd3efb9b852a Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 10 Dec 2024 17:01:57 +0300 Subject: [PATCH 350/475] Code normalization - New debug checks - Normalization --- testgres/operations/local_ops.py | 15 +++++++++++---- testgres/operations/remote_ops.py | 3 +++ 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/testgres/operations/local_ops.py b/testgres/operations/local_ops.py index d6daaa3b..3e8ab8ca 100644 --- a/testgres/operations/local_ops.py +++ b/testgres/operations/local_ops.py @@ -86,6 +86,8 @@ def _run_command__generic(self, cmd, shell, input, stdin, stdout, stderr, get_pr if not get_process: input_prepared = Helpers.PrepareProcessInput(input, encoding) # throw + assert input_prepared is None or (type(input_prepared) == bytes) # noqa: E721 + process = subprocess.Popen( cmd, shell=shell, @@ -93,18 +95,23 @@ def _run_command__generic(self, cmd, shell, input, stdin, stdout, stderr, get_pr stdout=stdout or subprocess.PIPE, stderr=stderr or subprocess.PIPE, ) + assert not (process is None) if get_process: return process, None, None try: output, error = process.communicate(input=input_prepared, timeout=timeout) - if encoding: - output = output.decode(encoding) - error = error.decode(encoding) - return process, output, error except subprocess.TimeoutExpired: process.kill() raise ExecUtilException("Command timed out after {} seconds.".format(timeout)) + assert type(output) == bytes # noqa: E721 + assert type(error) == bytes # noqa: E721 + + if encoding: + output = output.decode(encoding) + error = error.decode(encoding) + return process, output, error + def _run_command(self, cmd, shell, input, stdin, stdout, stderr, get_process, timeout, encoding): """Execute a command and return the process and its output.""" if os.name == 'nt' and stdout is None: # Windows diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index c48f867b..00c50d93 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -75,12 +75,15 @@ def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False, if not get_process: input_prepared = Helpers.PrepareProcessInput(input, encoding) # throw + assert input_prepared is None or (type(input_prepared) == bytes) # noqa: E721 + ssh_cmd = [] if isinstance(cmd, str): ssh_cmd = ['ssh', self.ssh_dest] + self.ssh_args + [cmd] elif isinstance(cmd, list): ssh_cmd = ['ssh', self.ssh_dest] + self.ssh_args + cmd process = subprocess.Popen(ssh_cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + assert not (process is None) if get_process: return process From 1b7bba4479a98c9d207e0e89b16e8a8269e826a8 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 10 Dec 2024 20:37:56 +0300 Subject: [PATCH 351/475] Fix for TestgresRemoteTests::test_child_pids [thanks to Victoria Shepard] This change was extracted from #149. --- tests/test_simple_remote.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_simple_remote.py b/tests/test_simple_remote.py index 26ac7c61..c8dd2964 100755 --- a/tests/test_simple_remote.py +++ b/tests/test_simple_remote.py @@ -940,6 +940,9 @@ def test_child_pids(self): if pg_version_ge('10'): master_processes.append(ProcessType.LogicalReplicationLauncher) + if pg_version_ge('14'): + master_processes.remove(ProcessType.StatsCollector) + repl_processes = [ ProcessType.Startup, ProcessType.WalReceiver, From 0b1b3de1d7ca370e6b4bcdfec0030e4776065714 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 10 Dec 2024 21:42:25 +0300 Subject: [PATCH 352/475] Formatting It is part of PR #149. --- testgres/node.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index b5cbab27..0faf904b 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -739,13 +739,11 @@ def start(self, params=[], wait=True): if self.is_started: return self - _params = [ - self._get_bin_path("pg_ctl"), - "-D", self.data_dir, - "-l", self.pg_log_file, - "-w" if wait else '-W', # --wait or --no-wait - "start" - ] + params # yapf: disable + _params = [self._get_bin_path("pg_ctl"), + "-D", self.data_dir, + "-l", self.pg_log_file, + "-w" if wait else '-W', # --wait or --no-wait + "start"] + params # yapf: disable startup_retries = 5 while True: From e4c2e07b3ff0c4aa16188a60bb53f108e2d5c0ca Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Thu, 12 Dec 2024 14:22:11 +0300 Subject: [PATCH 353/475] reserve_port and release_port are "pointers" to functions for a port numbers management. This need to replace port numbers management in unit tests. --- testgres/utils.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/testgres/utils.py b/testgres/utils.py index a4ee7877..4bd232b1 100644 --- a/testgres/utils.py +++ b/testgres/utils.py @@ -34,7 +34,7 @@ def __init__(self, version: str) -> None: super().__init__(version) -def reserve_port(): +def internal__reserve_port(): """ Generate a new port and add it to 'bound_ports'. """ @@ -45,7 +45,7 @@ def reserve_port(): return port -def release_port(port): +def internal__release_port(port): """ Free port provided by reserve_port(). """ @@ -53,6 +53,10 @@ def release_port(port): bound_ports.discard(port) +reserve_port = internal__reserve_port +release_port = internal__release_port + + def execute_utility(args, logfile=None, verbose=False): """ Execute utility (pg_ctl, pg_dump etc). From 85d2aa3917f8210dce06531b563ec34c42e6b544 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Thu, 12 Dec 2024 14:38:39 +0300 Subject: [PATCH 354/475] TestgresTests::test_the_same_port is corrected --- tests/test_simple.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/tests/test_simple.py b/tests/test_simple.py index fade468c..6b04f8bd 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -1051,10 +1051,18 @@ def test_parse_pg_version(self): def test_the_same_port(self): with get_new_node() as node: node.init().start() + self.assertTrue(node._should_free_port) + self.assertEqual(type(node.port), int) - with get_new_node() as node2: - node2.port = node.port - node2.init().start() + with get_new_node(port=node.port) as node2: + self.assertEqual(type(node2.port), int) + self.assertEqual(node2.port, node.port) + self.assertFalse(node2._should_free_port) + + with self.assertRaises(StartNodeException) as ctx: + node2.init().start() + + self.assertIn("Cannot start node", str(ctx.exception)) def test_simple_with_bin_dir(self): with get_new_node() as node: From 88371d1a610c62591d3b3ec2d5e88b7a2437ffb9 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Thu, 12 Dec 2024 14:46:23 +0300 Subject: [PATCH 355/475] OsOperations::read_binary(self, filename, start_pos) is added It is a specialized function to read binary data from files. --- testgres/operations/local_ops.py | 11 ++++++++ testgres/operations/os_ops.py | 6 +++++ testgres/operations/remote_ops.py | 19 +++++++++++++ tests/test_local.py | 45 +++++++++++++++++++++++++++++++ tests/test_remote.py | 43 +++++++++++++++++++++++++++++ 5 files changed, 124 insertions(+) diff --git a/testgres/operations/local_ops.py b/testgres/operations/local_ops.py index 3e8ab8ca..65dc0965 100644 --- a/testgres/operations/local_ops.py +++ b/testgres/operations/local_ops.py @@ -308,6 +308,17 @@ def readlines(self, filename, num_lines=0, binary=False, encoding=None): buffers * max(2, int(num_lines / max(cur_lines, 1))) ) # Adjust buffer size + def read_binary(self, filename, start_pos): + assert type(filename) == str # noqa: E721 + assert type(start_pos) == int # noqa: E721 + assert start_pos >= 0 + + with open(filename, 'rb') as file: # open in a binary mode + file.seek(start_pos, os.SEEK_SET) + r = file.read() + assert type(r) == bytes # noqa: E721 + return r + def isfile(self, remote_file): return os.path.isfile(remote_file) diff --git a/testgres/operations/os_ops.py b/testgres/operations/os_ops.py index 34242040..82d44a4e 100644 --- a/testgres/operations/os_ops.py +++ b/testgres/operations/os_ops.py @@ -98,6 +98,12 @@ def read(self, filename, encoding, binary): def readlines(self, filename): raise NotImplementedError() + def read_binary(self, filename, start_pos): + assert type(filename) == str # noqa: E721 + assert type(start_pos) == int # noqa: E721 + assert start_pos >= 0 + raise NotImplementedError() + def isfile(self, remote_file): raise NotImplementedError() diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index 00c50d93..9d72731d 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -340,6 +340,16 @@ def readlines(self, filename, num_lines=0, binary=False, encoding=None): return lines + def read_binary(self, filename, start_pos): + assert type(filename) == str # noqa: E721 + assert type(start_pos) == int # noqa: E721 + assert start_pos >= 0 + + cmd = "tail -c +{} {}".format(start_pos + 1, __class__._escape_path(filename)) + r = self.exec_command(cmd) + assert type(r) == bytes # noqa: E721 + return r + def isfile(self, remote_file): stdout = self.exec_command("test -f {}; echo $?".format(remote_file)) result = int(stdout.strip()) @@ -386,6 +396,15 @@ def db_connect(self, dbname, user, password=None, host="localhost", port=5432): ) return conn + def _escape_path(path): + assert type(path) == str # noqa: E721 + assert path != "" # Ok? + + r = "'" + r += path + r += "'" + return r + def normalize_error(error): if isinstance(error, bytes): diff --git a/tests/test_local.py b/tests/test_local.py index cb96a3bc..812b4030 100644 --- a/tests/test_local.py +++ b/tests/test_local.py @@ -1,4 +1,7 @@ +import os + import pytest +import re from testgres import ExecUtilException from testgres import LocalOperations @@ -52,3 +55,45 @@ def test_exec_command_failure__expect_error(self): assert error == b'/bin/sh: 1: nonexistent_command: not found\n' assert exit_status == 127 assert result == b'' + + def test_read_binary__spec(self): + """ + Test LocalOperations::read_binary. + """ + filename = __file__ # current file + + with open(filename, 'rb') as file: # open in a binary mode + response0 = file.read() + + assert type(response0) == bytes # noqa: E721 + + response1 = self.operations.read_binary(filename, 0) + assert type(response1) == bytes # noqa: E721 + assert response1 == response0 + + response2 = self.operations.read_binary(filename, 1) + assert type(response2) == bytes # noqa: E721 + assert len(response2) < len(response1) + assert len(response2) + 1 == len(response1) + assert response2 == response1[1:] + + response3 = self.operations.read_binary(filename, len(response1)) + assert type(response3) == bytes # noqa: E721 + assert len(response3) == 0 + + response4 = self.operations.read_binary(filename, len(response2)) + assert type(response4) == bytes # noqa: E721 + assert len(response4) == 1 + assert response4[0] == response1[len(response1) - 1] + + response5 = self.operations.read_binary(filename, len(response1) + 1) + assert type(response5) == bytes # noqa: E721 + assert len(response5) == 0 + + def test_read_binary__spec__unk_file(self): + """ + Test LocalOperations::read_binary with unknown file. + """ + + with pytest.raises(FileNotFoundError, match=re.escape("[Errno 2] No such file or directory: '/dummy'")): + self.operations.read_binary("/dummy", 0) diff --git a/tests/test_remote.py b/tests/test_remote.py index c1a91bc6..c775f72d 100755 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -1,6 +1,7 @@ import os import pytest +import re from testgres import ExecUtilException from testgres import RemoteOperations @@ -181,6 +182,48 @@ def test_read_binary_file(self): assert isinstance(response, bytes) + def test_read_binary__spec(self): + """ + Test RemoteOperations::read_binary. + """ + filename = __file__ # currnt file + + with open(filename, 'rb') as file: # open in a binary mode + response0 = file.read() + + assert type(response0) == bytes # noqa: E721 + + response1 = self.operations.read_binary(filename, 0) + assert type(response1) == bytes # noqa: E721 + assert response1 == response0 + + response2 = self.operations.read_binary(filename, 1) + assert type(response2) == bytes # noqa: E721 + assert len(response2) < len(response1) + assert len(response2) + 1 == len(response1) + assert response2 == response1[1:] + + response3 = self.operations.read_binary(filename, len(response1)) + assert type(response3) == bytes # noqa: E721 + assert len(response3) == 0 + + response4 = self.operations.read_binary(filename, len(response2)) + assert type(response4) == bytes # noqa: E721 + assert len(response4) == 1 + assert response4[0] == response1[len(response1) - 1] + + response5 = self.operations.read_binary(filename, len(response1) + 1) + assert type(response5) == bytes # noqa: E721 + assert len(response5) == 0 + + def test_read_binary__spec__unk_file(self): + """ + Test RemoteOperations::read_binary with unknown file. + """ + + with pytest.raises(ExecUtilException, match=re.escape("tail: cannot open '/dummy' for reading: No such file or directory")): + self.operations.read_binary("/dummy", 0) + def test_touch(self): """ Test touch for creating a new file or updating access and modification times of an existing file. From 4fe189445b0351692b57fc908d366e4dd82cb9e6 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Thu, 12 Dec 2024 15:16:48 +0300 Subject: [PATCH 356/475] OsOperations::get_file_size(self, filename) is added It is a function to get a size of file. --- testgres/operations/local_ops.py | 5 +++ testgres/operations/os_ops.py | 3 ++ testgres/operations/remote_ops.py | 64 +++++++++++++++++++++++++++++++ tests/test_local.py | 21 ++++++++++ tests/test_remote.py | 21 ++++++++++ 5 files changed, 114 insertions(+) diff --git a/testgres/operations/local_ops.py b/testgres/operations/local_ops.py index 65dc0965..82d1711d 100644 --- a/testgres/operations/local_ops.py +++ b/testgres/operations/local_ops.py @@ -325,6 +325,11 @@ def isfile(self, remote_file): def isdir(self, dirname): return os.path.isdir(dirname) + def get_file_size(self, filename): + assert filename is not None + assert type(filename) == str # noqa: E721 + return os.path.getsize(filename) + def remove_file(self, filename): return os.remove(filename) diff --git a/testgres/operations/os_ops.py b/testgres/operations/os_ops.py index 82d44a4e..2ab41246 100644 --- a/testgres/operations/os_ops.py +++ b/testgres/operations/os_ops.py @@ -107,6 +107,9 @@ def read_binary(self, filename, start_pos): def isfile(self, remote_file): raise NotImplementedError() + def get_file_size(self, filename): + raise NotImplementedError() + # Processes control def kill(self, pid, signal): # Kill the process diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index 9d72731d..9f88140c 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -360,6 +360,70 @@ def isdir(self, dirname): response = self.exec_command(cmd) return response.strip() == b"True" + def get_file_size(self, filename): + C_ERR_SRC = "RemoteOpertions::get_file_size" + + assert filename is not None + assert type(filename) == str # noqa: E721 + cmd = "du -b " + __class__._escape_path(filename) + + s = self.exec_command(cmd, encoding=get_default_encoding()) + assert type(s) == str # noqa: E721 + + if len(s) == 0: + raise Exception( + "[BUG CHECK] Can't get size of file [{2}]. Remote operation returned an empty string. Check point [{0}][{1}].".format( + C_ERR_SRC, + "#001", + filename + ) + ) + + i = 0 + + while i < len(s) and s[i].isdigit(): + assert s[i] >= '0' + assert s[i] <= '9' + i += 1 + + if i == 0: + raise Exception( + "[BUG CHECK] Can't get size of file [{2}]. Remote operation returned a bad formatted string. Check point [{0}][{1}].".format( + C_ERR_SRC, + "#002", + filename + ) + ) + + if i == len(s): + raise Exception( + "[BUG CHECK] Can't get size of file [{2}]. Remote operation returned a bad formatted string. Check point [{0}][{1}].".format( + C_ERR_SRC, + "#003", + filename + ) + ) + + if not s[i].isspace(): + raise Exception( + "[BUG CHECK] Can't get size of file [{2}]. Remote operation returned a bad formatted string. Check point [{0}][{1}].".format( + C_ERR_SRC, + "#004", + filename + ) + ) + + r = 0 + + for i2 in range(0, i): + ch = s[i2] + assert ch >= '0' + assert ch <= '9' + # Here is needed to check overflow or that it is a human-valid result? + r = (r * 10) + ord(ch) - ord('0') + + return r + def remove_file(self, filename): cmd = "rm {}".format(filename) return self.exec_command(cmd) diff --git a/tests/test_local.py b/tests/test_local.py index 812b4030..a8a0bde0 100644 --- a/tests/test_local.py +++ b/tests/test_local.py @@ -97,3 +97,24 @@ def test_read_binary__spec__unk_file(self): with pytest.raises(FileNotFoundError, match=re.escape("[Errno 2] No such file or directory: '/dummy'")): self.operations.read_binary("/dummy", 0) + + def test_get_file_size(self): + """ + Test LocalOperations::get_file_size. + """ + filename = __file__ # current file + + sz0 = os.path.getsize(filename) + assert type(sz0) == int # noqa: E721 + + sz1 = self.operations.get_file_size(filename) + assert type(sz1) == int # noqa: E721 + assert sz1 == sz0 + + def test_get_file_size__unk_file(self): + """ + Test LocalOperations::get_file_size. + """ + + with pytest.raises(FileNotFoundError, match=re.escape("[Errno 2] No such file or directory: '/dummy'")): + self.operations.get_file_size("/dummy") diff --git a/tests/test_remote.py b/tests/test_remote.py index c775f72d..be1a56bb 100755 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -224,6 +224,27 @@ def test_read_binary__spec__unk_file(self): with pytest.raises(ExecUtilException, match=re.escape("tail: cannot open '/dummy' for reading: No such file or directory")): self.operations.read_binary("/dummy", 0) + def test_get_file_size(self): + """ + Test LocalOperations::get_file_size. + """ + filename = __file__ # current file + + sz0 = os.path.getsize(filename) + assert type(sz0) == int # noqa: E721 + + sz1 = self.operations.get_file_size(filename) + assert type(sz1) == int # noqa: E721 + assert sz1 == sz0 + + def test_get_file_size__unk_file(self): + """ + Test LocalOperations::get_file_size. + """ + + with pytest.raises(ExecUtilException, match=re.escape("du: cannot access '/dummy': No such file or directory")): + self.operations.get_file_size("/dummy") + def test_touch(self): """ Test touch for creating a new file or updating access and modification times of an existing file. From 28ac4252e5761394b74bb782e29b986f1ffd31d9 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Thu, 12 Dec 2024 15:28:39 +0300 Subject: [PATCH 357/475] Port numbers management is improved (#164) - We don't release a port number that was defined by client - We only check log files to detect port number conflicts - We use slightly smarter log file checking A test is added. --- testgres/node.py | 89 +++++++++++++++++++++++++------- tests/test_simple.py | 117 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 189 insertions(+), 17 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index 0faf904b..7f5aa648 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -83,13 +83,13 @@ from .standby import First +from . import utils + from .utils import \ PgVer, \ eprint, \ get_bin_path, \ get_pg_version, \ - reserve_port, \ - release_port, \ execute_utility, \ options_string, \ clean_on_error @@ -158,7 +158,7 @@ def __init__(self, name=None, base_dir=None, port=None, conn_params: ConnectionP self.os_ops = LocalOperations(conn_params) self.host = self.os_ops.host - self.port = port or reserve_port() + self.port = port or utils.reserve_port() self.ssh_key = self.os_ops.ssh_key @@ -471,6 +471,28 @@ def _collect_special_files(self): return result + def _collect_log_files(self): + # dictionary of log files + size in bytes + + files = [ + self.pg_log_file + ] # yapf: disable + + result = {} + + for f in files: + # skip missing files + if not self.os_ops.path_exists(f): + continue + + file_size = self.os_ops.get_file_size(f) + assert type(file_size) == int # noqa: E721 + assert file_size >= 0 + + result[f] = file_size + + return result + def init(self, initdb_params=None, cached=True, **kwargs): """ Perform initdb for this node. @@ -722,6 +744,22 @@ def slow_start(self, replica=False, dbname='template1', username=None, max_attem OperationalError}, max_attempts=max_attempts) + def _detect_port_conflict(self, log_files0, log_files1): + assert type(log_files0) == dict # noqa: E721 + assert type(log_files1) == dict # noqa: E721 + + for file in log_files1.keys(): + read_pos = 0 + + if file in log_files0.keys(): + read_pos = log_files0[file] # the previous size + + file_content = self.os_ops.read_binary(file, read_pos) + file_content_s = file_content.decode() + if 'Is another postmaster already running on port' in file_content_s: + return True + return False + def start(self, params=[], wait=True): """ Starts the PostgreSQL node using pg_ctl if node has not been started. @@ -745,27 +783,42 @@ def start(self, params=[], wait=True): "-w" if wait else '-W', # --wait or --no-wait "start"] + params # yapf: disable - startup_retries = 5 + log_files0 = self._collect_log_files() + assert type(log_files0) == dict # noqa: E721 + + nAttempt = 0 + timeout = 1 while True: + nAttempt += 1 try: exit_status, out, error = execute_utility(_params, self.utils_log_file, verbose=True) if error and 'does not exist' in error: raise Exception except Exception as e: - files = self._collect_special_files() - if any(len(file) > 1 and 'Is another postmaster already ' - 'running on port' in file[1].decode() for - file in files): - logging.warning("Detected an issue with connecting to port {0}. " - "Trying another port after a 5-second sleep...".format(self.port)) - self.port = reserve_port() - options = {'port': str(self.port)} - self.set_auto_conf(options) - startup_retries -= 1 - time.sleep(5) - continue + if self._should_free_port and nAttempt < 5: + log_files1 = self._collect_log_files() + if self._detect_port_conflict(log_files0, log_files1): + log_files0 = log_files1 + logging.warning( + "Detected an issue with connecting to port {0}. " + "Trying another port after a {1}-second sleep...".format(self.port, timeout) + ) + time.sleep(timeout) + timeout = min(2 * timeout, 5) + cur_port = self.port + new_port = utils.reserve_port() # throw + try: + options = {'port': str(new_port)} + self.set_auto_conf(options) + except: # noqa: E722 + utils.release_port(new_port) + raise + self.port = new_port + utils.release_port(cur_port) + continue msg = 'Cannot start node' + files = self._collect_special_files() raise_from(StartNodeException(msg, files), e) break self._maybe_start_logger() @@ -930,8 +983,10 @@ def free_port(self): """ if self._should_free_port: + port = self.port self._should_free_port = False - release_port(self.port) + self.port = None + utils.release_port(port) def cleanup(self, max_attempts=3, full=False): """ diff --git a/tests/test_simple.py b/tests/test_simple.py index 6b04f8bd..0a09135c 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -1064,6 +1064,123 @@ def test_the_same_port(self): self.assertIn("Cannot start node", str(ctx.exception)) + class tagPortManagerProxy: + sm_prev_testgres_reserve_port = None + sm_prev_testgres_release_port = None + + sm_DummyPortNumber = None + sm_DummyPortMaxUsage = None + + sm_DummyPortCurrentUsage = None + sm_DummyPortTotalUsage = None + + def __init__(self, dummyPortNumber, dummyPortMaxUsage): + assert type(dummyPortNumber) == int # noqa: E721 + assert type(dummyPortMaxUsage) == int # noqa: E721 + assert dummyPortNumber >= 0 + assert dummyPortMaxUsage >= 0 + + assert __class__.sm_prev_testgres_reserve_port is None + assert __class__.sm_prev_testgres_release_port is None + assert testgres.utils.reserve_port == testgres.utils.internal__reserve_port + assert testgres.utils.release_port == testgres.utils.internal__release_port + + __class__.sm_prev_testgres_reserve_port = testgres.utils.reserve_port + __class__.sm_prev_testgres_release_port = testgres.utils.release_port + + testgres.utils.reserve_port = __class__._proxy__reserve_port + testgres.utils.release_port = __class__._proxy__release_port + + assert testgres.utils.reserve_port == __class__._proxy__reserve_port + assert testgres.utils.release_port == __class__._proxy__release_port + + __class__.sm_DummyPortNumber = dummyPortNumber + __class__.sm_DummyPortMaxUsage = dummyPortMaxUsage + + __class__.sm_DummyPortCurrentUsage = 0 + __class__.sm_DummyPortTotalUsage = 0 + + def __enter__(self): + return self + + def __exit__(self, type, value, traceback): + assert __class__.sm_DummyPortCurrentUsage == 0 + + assert __class__.sm_prev_testgres_reserve_port is not None + assert __class__.sm_prev_testgres_release_port is not None + + assert testgres.utils.reserve_port == __class__._proxy__reserve_port + assert testgres.utils.release_port == __class__._proxy__release_port + + testgres.utils.reserve_port = __class__.sm_prev_testgres_reserve_port + testgres.utils.release_port = __class__.sm_prev_testgres_release_port + + __class__.sm_prev_testgres_reserve_port = None + __class__.sm_prev_testgres_release_port = None + + def _proxy__reserve_port(): + assert type(__class__.sm_DummyPortMaxUsage) == int # noqa: E721 + assert type(__class__.sm_DummyPortTotalUsage) == int # noqa: E721 + assert type(__class__.sm_DummyPortCurrentUsage) == int # noqa: E721 + assert __class__.sm_DummyPortTotalUsage >= 0 + assert __class__.sm_DummyPortCurrentUsage >= 0 + + assert __class__.sm_DummyPortTotalUsage <= __class__.sm_DummyPortMaxUsage + assert __class__.sm_DummyPortCurrentUsage <= __class__.sm_DummyPortTotalUsage + + assert __class__.sm_prev_testgres_reserve_port is not None + + if __class__.sm_DummyPortTotalUsage == __class__.sm_DummyPortMaxUsage: + return __class__.sm_prev_testgres_reserve_port() + + __class__.sm_DummyPortTotalUsage += 1 + __class__.sm_DummyPortCurrentUsage += 1 + return __class__.sm_DummyPortNumber + + def _proxy__release_port(dummyPortNumber): + assert type(dummyPortNumber) == int # noqa: E721 + + assert type(__class__.sm_DummyPortMaxUsage) == int # noqa: E721 + assert type(__class__.sm_DummyPortTotalUsage) == int # noqa: E721 + assert type(__class__.sm_DummyPortCurrentUsage) == int # noqa: E721 + assert __class__.sm_DummyPortTotalUsage >= 0 + assert __class__.sm_DummyPortCurrentUsage >= 0 + + assert __class__.sm_DummyPortTotalUsage <= __class__.sm_DummyPortMaxUsage + assert __class__.sm_DummyPortCurrentUsage <= __class__.sm_DummyPortTotalUsage + + assert __class__.sm_prev_testgres_release_port is not None + + if __class__.sm_DummyPortCurrentUsage > 0 and dummyPortNumber == __class__.sm_DummyPortNumber: + assert __class__.sm_DummyPortTotalUsage > 0 + __class__.sm_DummyPortCurrentUsage -= 1 + return + + return __class__.sm_prev_testgres_release_port(dummyPortNumber) + + def test_port_rereserve_during_node_start(self): + C_COUNT_OF_BAD_PORT_USAGE = 3 + + with get_new_node() as node1: + node1.init().start() + self.assertTrue(node1._should_free_port) + self.assertEqual(type(node1.port), int) # noqa: E721 + node1.safe_psql("SELECT 1;") + + with __class__.tagPortManagerProxy(node1.port, C_COUNT_OF_BAD_PORT_USAGE): + assert __class__.tagPortManagerProxy.sm_DummyPortNumber == node1.port + with get_new_node() as node2: + self.assertTrue(node2._should_free_port) + self.assertEqual(node2.port, node1.port) + + node2.init().start() + + self.assertNotEqual(node2.port, node1.port) + self.assertEqual(__class__.tagPortManagerProxy.sm_DummyPortCurrentUsage, 0) + self.assertEqual(__class__.tagPortManagerProxy.sm_DummyPortTotalUsage, C_COUNT_OF_BAD_PORT_USAGE) + + node2.safe_psql("SELECT 1;") + def test_simple_with_bin_dir(self): with get_new_node() as node: node.init().start() From 663612cb870fbeade43db3a0c6f771127fb650a2 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Sat, 14 Dec 2024 11:07:35 +0300 Subject: [PATCH 358/475] PostgresNode._C_MAX_START_ATEMPTS=5 is added (+ 1 new test) Also - TestgresTests.test_the_same_port is updated - TestgresTests.test_port_rereserve_during_node_start is updated - TestgresTests.test_port_conflict is added --- testgres/node.py | 14 +++++++++-- tests/test_simple.py | 58 ++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 68 insertions(+), 4 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index 7f5aa648..554c226d 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -128,6 +128,9 @@ def __repr__(self): class PostgresNode(object): + # a max number of node start attempts + _C_MAX_START_ATEMPTS = 5 + def __init__(self, name=None, base_dir=None, port=None, conn_params: ConnectionParams = ConnectionParams(), bin_dir=None, prefix=None): """ PostgresNode constructor. @@ -774,6 +777,9 @@ def start(self, params=[], wait=True): Returns: This instance of :class:`.PostgresNode`. """ + + assert __class__._C_MAX_START_ATEMPTS > 1 + if self.is_started: return self @@ -789,13 +795,17 @@ def start(self, params=[], wait=True): nAttempt = 0 timeout = 1 while True: + assert nAttempt >= 0 + assert nAttempt < __class__._C_MAX_START_ATEMPTS nAttempt += 1 try: exit_status, out, error = execute_utility(_params, self.utils_log_file, verbose=True) if error and 'does not exist' in error: raise Exception except Exception as e: - if self._should_free_port and nAttempt < 5: + assert nAttempt > 0 + assert nAttempt <= __class__._C_MAX_START_ATEMPTS + if self._should_free_port and nAttempt < __class__._C_MAX_START_ATEMPTS: log_files1 = self._collect_log_files() if self._detect_port_conflict(log_files0, log_files1): log_files0 = log_files1 @@ -806,7 +816,7 @@ def start(self, params=[], wait=True): time.sleep(timeout) timeout = min(2 * timeout, 5) cur_port = self.port - new_port = utils.reserve_port() # throw + new_port = utils.reserve_port() # can raise try: options = {'port': str(new_port)} self.set_auto_conf(options) diff --git a/tests/test_simple.py b/tests/test_simple.py index 0a09135c..93968466 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -1053,6 +1053,8 @@ def test_the_same_port(self): node.init().start() self.assertTrue(node._should_free_port) self.assertEqual(type(node.port), int) + node_port_copy = node.port + self.assertEqual(node.safe_psql("SELECT 1;"), b'1\n') with get_new_node(port=node.port) as node2: self.assertEqual(type(node2.port), int) @@ -1064,6 +1066,11 @@ def test_the_same_port(self): self.assertIn("Cannot start node", str(ctx.exception)) + # node is still working + self.assertEqual(node.port, node_port_copy) + self.assertTrue(node._should_free_port) + self.assertEqual(node.safe_psql("SELECT 3;"), b'3\n') + class tagPortManagerProxy: sm_prev_testgres_reserve_port = None sm_prev_testgres_release_port = None @@ -1159,13 +1166,16 @@ def _proxy__release_port(dummyPortNumber): return __class__.sm_prev_testgres_release_port(dummyPortNumber) def test_port_rereserve_during_node_start(self): + assert testgres.PostgresNode._C_MAX_START_ATEMPTS == 5 + C_COUNT_OF_BAD_PORT_USAGE = 3 with get_new_node() as node1: node1.init().start() self.assertTrue(node1._should_free_port) self.assertEqual(type(node1.port), int) # noqa: E721 - node1.safe_psql("SELECT 1;") + node1_port_copy = node1.port + self.assertEqual(node1.safe_psql("SELECT 1;"), b'1\n') with __class__.tagPortManagerProxy(node1.port, C_COUNT_OF_BAD_PORT_USAGE): assert __class__.tagPortManagerProxy.sm_DummyPortNumber == node1.port @@ -1176,10 +1186,54 @@ def test_port_rereserve_during_node_start(self): node2.init().start() self.assertNotEqual(node2.port, node1.port) + self.assertTrue(node2._should_free_port) self.assertEqual(__class__.tagPortManagerProxy.sm_DummyPortCurrentUsage, 0) self.assertEqual(__class__.tagPortManagerProxy.sm_DummyPortTotalUsage, C_COUNT_OF_BAD_PORT_USAGE) + self.assertTrue(node2.is_started) + + self.assertEqual(node2.safe_psql("SELECT 2;"), b'2\n') + + # node1 is still working + self.assertEqual(node1.port, node1_port_copy) + self.assertTrue(node1._should_free_port) + self.assertEqual(node1.safe_psql("SELECT 3;"), b'3\n') + + def test_port_conflict(self): + assert testgres.PostgresNode._C_MAX_START_ATEMPTS > 1 + + C_COUNT_OF_BAD_PORT_USAGE = testgres.PostgresNode._C_MAX_START_ATEMPTS + + with get_new_node() as node1: + node1.init().start() + self.assertTrue(node1._should_free_port) + self.assertEqual(type(node1.port), int) # noqa: E721 + node1_port_copy = node1.port + self.assertEqual(node1.safe_psql("SELECT 1;"), b'1\n') + + with __class__.tagPortManagerProxy(node1.port, C_COUNT_OF_BAD_PORT_USAGE): + assert __class__.tagPortManagerProxy.sm_DummyPortNumber == node1.port + with get_new_node() as node2: + self.assertTrue(node2._should_free_port) + self.assertEqual(node2.port, node1.port) + + with self.assertRaises(StartNodeException) as ctx: + node2.init().start() + + self.assertIn("Cannot start node", str(ctx.exception)) + + self.assertEqual(node2.port, node1.port) + self.assertTrue(node2._should_free_port) + self.assertEqual(__class__.tagPortManagerProxy.sm_DummyPortCurrentUsage, 1) + self.assertEqual(__class__.tagPortManagerProxy.sm_DummyPortTotalUsage, C_COUNT_OF_BAD_PORT_USAGE) + self.assertFalse(node2.is_started) + + # node2 must release our dummyPort (node1.port) + self.assertEqual(__class__.tagPortManagerProxy.sm_DummyPortCurrentUsage, 0) - node2.safe_psql("SELECT 1;") + # node1 is still working + self.assertEqual(node1.port, node1_port_copy) + self.assertTrue(node1._should_free_port) + self.assertEqual(node1.safe_psql("SELECT 3;"), b'3\n') def test_simple_with_bin_dir(self): with get_new_node() as node: From 2f3fc40d39562ffa62e77863df8316becd353dee Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Wed, 18 Dec 2024 21:54:32 +0300 Subject: [PATCH 359/475] OsOperations.isdir is added It a correction of base interface - OsOperations. It seems we forgot to add a declaration of the "abstract" method "isdir". --- testgres/operations/os_ops.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/testgres/operations/os_ops.py b/testgres/operations/os_ops.py index 2ab41246..d644509a 100644 --- a/testgres/operations/os_ops.py +++ b/testgres/operations/os_ops.py @@ -107,6 +107,9 @@ def read_binary(self, filename, start_pos): def isfile(self, remote_file): raise NotImplementedError() + def isdir(self, dirname): + raise NotImplementedError() + def get_file_size(self, filename): raise NotImplementedError() From 9f527435b28b3c9655d66ef7a2b5a120d767786d Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Wed, 18 Dec 2024 21:57:37 +0300 Subject: [PATCH 360/475] Tests for LocalOperations methods isdir and isfile are added --- tests/test_local.py | 64 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/tests/test_local.py b/tests/test_local.py index a8a0bde0..e223b090 100644 --- a/tests/test_local.py +++ b/tests/test_local.py @@ -118,3 +118,67 @@ def test_get_file_size__unk_file(self): with pytest.raises(FileNotFoundError, match=re.escape("[Errno 2] No such file or directory: '/dummy'")): self.operations.get_file_size("/dummy") + + def test_isfile_true(self): + """ + Test isfile for an existing file. + """ + filename = __file__ + + response = self.operations.isfile(filename) + + assert response is True + + def test_isfile_false__not_exist(self): + """ + Test isfile for a non-existing file. + """ + filename = os.path.join(os.path.dirname(__file__), "nonexistent_file.txt") + + response = self.operations.isfile(filename) + + assert response is False + + def test_isfile_false__directory(self): + """ + Test isfile for a firectory. + """ + name = os.path.dirname(__file__) + + assert self.operations.isdir(name) + + response = self.operations.isfile(name) + + assert response is False + + def test_isdir_true(self): + """ + Test isdir for an existing directory. + """ + name = os.path.dirname(__file__) + + response = self.operations.isdir(name) + + assert response is True + + def test_isdir_false__not_exist(self): + """ + Test isdir for a non-existing directory. + """ + name = os.path.join(os.path.dirname(__file__), "it_is_nonexistent_directory") + + response = self.operations.isdir(name) + + assert response is False + + def test_isdir_false__file(self): + """ + Test isdir for a file. + """ + name = __file__ + + assert self.operations.isfile(name) + + response = self.operations.isdir(name) + + assert response is False From 38ce127ec63189d4a35d1f73232435538329ef93 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Wed, 18 Dec 2024 21:58:30 +0300 Subject: [PATCH 361/475] Tests for RemoteOperations methods isdir and isfile are updated --- tests/test_remote.py | 50 +++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 47 insertions(+), 3 deletions(-) diff --git a/tests/test_remote.py b/tests/test_remote.py index be1a56bb..67d66549 100755 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -259,18 +259,62 @@ def test_isfile_true(self): """ Test isfile for an existing file. """ - filename = "/etc/hosts" + filename = __file__ response = self.operations.isfile(filename) assert response is True - def test_isfile_false(self): + def test_isfile_false__not_exist(self): """ Test isfile for a non-existing file. """ - filename = "/nonexistent_file.txt" + filename = os.path.join(os.path.dirname(__file__), "nonexistent_file.txt") response = self.operations.isfile(filename) assert response is False + + def test_isfile_false__directory(self): + """ + Test isfile for a firectory. + """ + name = os.path.dirname(__file__) + + assert self.operations.isdir(name) + + response = self.operations.isfile(name) + + assert response is False + + def test_isdir_true(self): + """ + Test isdir for an existing directory. + """ + name = os.path.dirname(__file__) + + response = self.operations.isdir(name) + + assert response is True + + def test_isdir_false__not_exist(self): + """ + Test isdir for a non-existing directory. + """ + name = os.path.join(os.path.dirname(__file__), "it_is_nonexistent_directory") + + response = self.operations.isdir(name) + + assert response is False + + def test_isdir_false__file(self): + """ + Test isdir for a file. + """ + name = __file__ + + assert self.operations.isfile(name) + + response = self.operations.isdir(name) + + assert response is False From 93d1122d0f59575a5b7f7b316f18d2824ecacc62 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 24 Dec 2024 11:51:52 +0300 Subject: [PATCH 362/475] Formatting --- testgres/node.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/testgres/node.py b/testgres/node.py index 554c226d..dff47cf6 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -131,7 +131,8 @@ class PostgresNode(object): # a max number of node start attempts _C_MAX_START_ATEMPTS = 5 - def __init__(self, name=None, base_dir=None, port=None, conn_params: ConnectionParams = ConnectionParams(), bin_dir=None, prefix=None): + def __init__(self, name=None, base_dir=None, port=None, conn_params: ConnectionParams = ConnectionParams(), + bin_dir=None, prefix=None): """ PostgresNode constructor. From 00fd9257158cd9b0aa7450b66762ad9c2c31f08b Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 24 Dec 2024 14:14:49 +0300 Subject: [PATCH 363/475] Node.start is refactored [Victoria Shepard' ideas are used, #149] - Save an orignal text of 'does not exist' error - When we reach maximum retry attempt of restarts - We log an error message - We raise exception "Cannot start node after multiple attempts" - A new port number is not tranlating into string - Reorganization TestgresTests.test_port_conflict is updated. --- testgres/node.py | 100 +++++++++++++++++++++++++++---------------- tests/test_simple.py | 2 +- 2 files changed, 63 insertions(+), 39 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index dff47cf6..7121339f 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -790,48 +790,72 @@ def start(self, params=[], wait=True): "-w" if wait else '-W', # --wait or --no-wait "start"] + params # yapf: disable - log_files0 = self._collect_log_files() - assert type(log_files0) == dict # noqa: E721 + def LOCAL__start_node(): + _, _, error = execute_utility(_params, self.utils_log_file, verbose=True) + assert type(error) == str # noqa: E721 + if error and 'does not exist' in error: + raise Exception(error) - nAttempt = 0 - timeout = 1 - while True: - assert nAttempt >= 0 - assert nAttempt < __class__._C_MAX_START_ATEMPTS - nAttempt += 1 + def LOCAL__raise_cannot_start_node(from_exception, msg): + assert isinstance(from_exception, Exception) + assert type(msg) == str # noqa: E721 + files = self._collect_special_files() + raise_from(StartNodeException(msg, files), from_exception) + + def LOCAL__raise_cannot_start_node__std(from_exception): + assert isinstance(from_exception, Exception) + LOCAL__raise_cannot_start_node(from_exception, 'Cannot start node') + + if not self._should_free_port: try: - exit_status, out, error = execute_utility(_params, self.utils_log_file, verbose=True) - if error and 'does not exist' in error: - raise Exception + LOCAL__start_node() except Exception as e: - assert nAttempt > 0 - assert nAttempt <= __class__._C_MAX_START_ATEMPTS - if self._should_free_port and nAttempt < __class__._C_MAX_START_ATEMPTS: + LOCAL__raise_cannot_start_node__std(e) + else: + assert self._should_free_port + assert __class__._C_MAX_START_ATEMPTS > 1 + + log_files0 = self._collect_log_files() + assert type(log_files0) == dict # noqa: E721 + + nAttempt = 0 + timeout = 1 + while True: + assert nAttempt >= 0 + assert nAttempt < __class__._C_MAX_START_ATEMPTS + nAttempt += 1 + try: + LOCAL__start_node() + except Exception as e: + assert nAttempt > 0 + assert nAttempt <= __class__._C_MAX_START_ATEMPTS + if nAttempt == __class__._C_MAX_START_ATEMPTS: + logging.error("Reached maximum retry attempts. Unable to start node.") + LOCAL__raise_cannot_start_node(e, "Cannot start node after multiple attempts") + log_files1 = self._collect_log_files() - if self._detect_port_conflict(log_files0, log_files1): - log_files0 = log_files1 - logging.warning( - "Detected an issue with connecting to port {0}. " - "Trying another port after a {1}-second sleep...".format(self.port, timeout) - ) - time.sleep(timeout) - timeout = min(2 * timeout, 5) - cur_port = self.port - new_port = utils.reserve_port() # can raise - try: - options = {'port': str(new_port)} - self.set_auto_conf(options) - except: # noqa: E722 - utils.release_port(new_port) - raise - self.port = new_port - utils.release_port(cur_port) - continue - - msg = 'Cannot start node' - files = self._collect_special_files() - raise_from(StartNodeException(msg, files), e) - break + if not self._detect_port_conflict(log_files0, log_files1): + LOCAL__raise_cannot_start_node__std(e) + + log_files0 = log_files1 + logging.warning( + "Detected a conflict with using the port {0}. " + "Trying another port after a {1}-second sleep...".format(self.port, timeout) + ) + time.sleep(timeout) + timeout = min(2 * timeout, 5) + cur_port = self.port + new_port = utils.reserve_port() # can raise + try: + options = {'port': str(new_port)} + self.set_auto_conf(options) + except: # noqa: E722 + utils.release_port(new_port) + raise + self.port = new_port + utils.release_port(cur_port) + continue + break self._maybe_start_logger() self.is_started = True return self diff --git a/tests/test_simple.py b/tests/test_simple.py index 93968466..9cf29c64 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -1219,7 +1219,7 @@ def test_port_conflict(self): with self.assertRaises(StartNodeException) as ctx: node2.init().start() - self.assertIn("Cannot start node", str(ctx.exception)) + self.assertIn("Cannot start node after multiple attempts", str(ctx.exception)) self.assertEqual(node2.port, node1.port) self.assertTrue(node2._should_free_port) From 2fa4426095acda3caadbdd47f6b029d699c0a0aa Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 24 Dec 2024 17:19:04 +0300 Subject: [PATCH 364/475] Formatting --- testgres/operations/remote_ops.py | 1 - 1 file changed, 1 deletion(-) diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index 9f88140c..3aa2d8c4 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -39,7 +39,6 @@ def cmdline(self): class RemoteOperations(OsOperations): def __init__(self, conn_params: ConnectionParams): - if not platform.system().lower() == "linux": raise EnvironmentError("Remote operations are supported only on Linux!") From a33860d0cb625516535cf5814cac2e89562a5f70 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 24 Dec 2024 20:47:20 +0300 Subject: [PATCH 365/475] PSQL passes a database name through the explicit '-d ' parameter --- testgres/node.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index 7121339f..f7d6e839 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -1107,13 +1107,12 @@ def _psql( else: raise Exception("Input data must be None or bytes.") - dbname = dbname or default_dbname() - psql_params = [ self._get_bin_path("psql"), "-p", str(self.port), "-h", self.host, "-U", username or self.os_ops.username, + "-d", dbname or default_dbname(), "-X", # no .psqlrc "-A", # unaligned output "-t", # print rows only @@ -1135,9 +1134,6 @@ def _psql( else: raise QueryException('Query or filename must be provided') - # should be the last one - psql_params.append(dbname) - return self.os_ops.exec_command( psql_params, verbose=True, From 3ca3ff7eef2687c717c3cbcedc04f37b98ec7ed9 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 24 Dec 2024 22:43:39 +0300 Subject: [PATCH 366/475] Ssh command line in RemoteOperations::execute is corrected We use subprocess.list2cmdline(cmd) to pack a user command line. It allows PostgresNode::_psql to build PSQL command line without a "special case" for remote-host. Also RemoteOperations::execute raises exception if 'cmd' parameter has an unknown type. --- testgres/node.py | 5 +---- testgres/operations/remote_ops.py | 5 ++++- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index f7d6e839..3d023399 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -1125,10 +1125,7 @@ def _psql( # select query source if query: - if self.os_ops.remote: - psql_params.extend(("-c", '"{}"'.format(query))) - else: - psql_params.extend(("-c", query)) + psql_params.extend(("-c", query)) elif filename: psql_params.extend(("-f", filename)) else: diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index 3aa2d8c4..fb5dd4b2 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -80,7 +80,10 @@ def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False, if isinstance(cmd, str): ssh_cmd = ['ssh', self.ssh_dest] + self.ssh_args + [cmd] elif isinstance(cmd, list): - ssh_cmd = ['ssh', self.ssh_dest] + self.ssh_args + cmd + ssh_cmd = ['ssh', self.ssh_dest] + self.ssh_args + [subprocess.list2cmdline(cmd)] + else: + raise ValueError("Invalid 'cmd' argument type - {0}".format(type(cmd).__name__)) + process = subprocess.Popen(ssh_cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) assert not (process is None) if get_process: From 7c0f1846205c89acc96139fca50b24b76dad4a28 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Wed, 25 Dec 2024 12:33:38 +0300 Subject: [PATCH 367/475] RemoteOperations is updated (read_binary, get_file_size) get_file_size and get_file_size use a list for command list arguments. It allows to use standard way to escape a filename. Our bicycle "_escape_path" is deleted. --- testgres/operations/remote_ops.py | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index fb5dd4b2..128a2a21 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -347,7 +347,7 @@ def read_binary(self, filename, start_pos): assert type(start_pos) == int # noqa: E721 assert start_pos >= 0 - cmd = "tail -c +{} {}".format(start_pos + 1, __class__._escape_path(filename)) + cmd = ["tail", "-c", "+{}".format(start_pos + 1), filename] r = self.exec_command(cmd) assert type(r) == bytes # noqa: E721 return r @@ -367,7 +367,7 @@ def get_file_size(self, filename): assert filename is not None assert type(filename) == str # noqa: E721 - cmd = "du -b " + __class__._escape_path(filename) + cmd = ["du", "-b", filename] s = self.exec_command(cmd, encoding=get_default_encoding()) assert type(s) == str # noqa: E721 @@ -462,15 +462,6 @@ def db_connect(self, dbname, user, password=None, host="localhost", port=5432): ) return conn - def _escape_path(path): - assert type(path) == str # noqa: E721 - assert path != "" # Ok? - - r = "'" - r += path - r += "'" - return r - def normalize_error(error): if isinstance(error, bytes): From 5b263f3c6231a98de7a72c1bf9445989e00a7f62 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Wed, 25 Dec 2024 13:43:25 +0300 Subject: [PATCH 368/475] Comments are corrected --- tests/test_remote.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_remote.py b/tests/test_remote.py index 67d66549..780ad46e 100755 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -226,7 +226,7 @@ def test_read_binary__spec__unk_file(self): def test_get_file_size(self): """ - Test LocalOperations::get_file_size. + Test RemoteOperations::get_file_size. """ filename = __file__ # current file @@ -239,7 +239,7 @@ def test_get_file_size(self): def test_get_file_size__unk_file(self): """ - Test LocalOperations::get_file_size. + Test RemoteOperations::get_file_size. """ with pytest.raises(ExecUtilException, match=re.escape("du: cannot access '/dummy': No such file or directory")): From 4e23e030cfb8b2531cae5abfd4f471809e7f7371 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Wed, 25 Dec 2024 13:50:05 +0300 Subject: [PATCH 369/475] OsOps::read methods were corrected Now they always read a file as binary. When 'binary' parameter is False we will use 'encoding' parameter to decode bytes into string. Binary read does not allow an usage of 'encoding' parameter (InvalidOperationException is raised). New tests are added. --- testgres/operations/local_ops.py | 36 ++++++++++++++--- testgres/operations/remote_ops.py | 35 +++++++++++++--- tests/test_local.py | 66 ++++++++++++++++++++++++++++++- tests/test_remote.py | 64 ++++++++++++++++++++++++++++++ 4 files changed, 189 insertions(+), 12 deletions(-) diff --git a/testgres/operations/local_ops.py b/testgres/operations/local_ops.py index 82d1711d..d6013ab5 100644 --- a/testgres/operations/local_ops.py +++ b/testgres/operations/local_ops.py @@ -10,6 +10,7 @@ import psutil from ..exceptions import ExecUtilException +from ..exceptions import InvalidOperationException from .os_ops import ConnectionParams, OsOperations, pglib, get_default_encoding from .raise_error import RaiseError from .helpers import Helpers @@ -266,13 +267,36 @@ def touch(self, filename): os.utime(filename, None) def read(self, filename, encoding=None, binary=False): - mode = "rb" if binary else "r" - with open(filename, mode) as file: + assert type(filename) == str # noqa: E721 + assert encoding is None or type(encoding) == str # noqa: E721 + assert type(binary) == bool # noqa: E721 + + if binary: + if encoding is not None: + raise InvalidOperationException("Enconding is not allowed for read binary operation") + + return self._read__binary(filename) + + # python behavior + assert None or "abc" == "abc" + assert "" or "abc" == "abc" + + return self._read__text_with_encoding(filename, encoding or get_default_encoding()) + + def _read__text_with_encoding(self, filename, encoding): + assert type(filename) == str # noqa: E721 + assert type(encoding) == str # noqa: E721 + content = self._read__binary(filename) + assert type(content) == bytes # noqa: E721 + content_s = content.decode(encoding) + assert type(content_s) == str # noqa: E721 + return content_s + + def _read__binary(self, filename): + assert type(filename) == str # noqa: E721 + with open(filename, 'rb') as file: # open in a binary mode content = file.read() - if binary: - return content - if isinstance(content, bytes): - return content.decode(encoding or get_default_encoding()) + assert type(content) == bytes # noqa: E721 return content def readlines(self, filename, num_lines=0, binary=False, encoding=None): diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index 128a2a21..abcb8fe1 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -14,6 +14,7 @@ raise ImportError("You must have psycopg2 or pg8000 modules installed") from ..exceptions import ExecUtilException +from ..exceptions import InvalidOperationException from .os_ops import OsOperations, ConnectionParams, get_default_encoding from .raise_error import RaiseError from .helpers import Helpers @@ -319,13 +320,37 @@ def touch(self, filename): self.exec_command("touch {}".format(filename)) def read(self, filename, binary=False, encoding=None): - cmd = "cat {}".format(filename) - result = self.exec_command(cmd, encoding=encoding) + assert type(filename) == str # noqa: E721 + assert encoding is None or type(encoding) == str # noqa: E721 + assert type(binary) == bool # noqa: E721 - if not binary and result: - result = result.decode(encoding or get_default_encoding()) + if binary: + if encoding is not None: + raise InvalidOperationException("Enconding is not allowed for read binary operation") - return result + return self._read__binary(filename) + + # python behavior + assert None or "abc" == "abc" + assert "" or "abc" == "abc" + + return self._read__text_with_encoding(filename, encoding or get_default_encoding()) + + def _read__text_with_encoding(self, filename, encoding): + assert type(filename) == str # noqa: E721 + assert type(encoding) == str # noqa: E721 + content = self._read__binary(filename) + assert type(content) == bytes # noqa: E721 + content_s = content.decode(encoding) + assert type(content_s) == str # noqa: E721 + return content_s + + def _read__binary(self, filename): + assert type(filename) == str # noqa: E721 + cmd = ["cat", filename] + content = self.exec_command(cmd) + assert type(content) == bytes # noqa: E721 + return content def readlines(self, filename, num_lines=0, binary=False, encoding=None): if num_lines > 0: diff --git a/tests/test_local.py b/tests/test_local.py index e223b090..47a63994 100644 --- a/tests/test_local.py +++ b/tests/test_local.py @@ -4,6 +4,7 @@ import re from testgres import ExecUtilException +from testgres import InvalidOperationException from testgres import LocalOperations from .helpers.run_conditions import RunConditions @@ -56,6 +57,67 @@ def test_exec_command_failure__expect_error(self): assert exit_status == 127 assert result == b'' + def test_read__text(self): + """ + Test LocalOperations::read for text data. + """ + filename = __file__ # current file + + with open(filename, 'r') as file: # open in a text mode + response0 = file.read() + + assert type(response0) == str # noqa: E721 + + response1 = self.operations.read(filename) + assert type(response1) == str # noqa: E721 + assert response1 == response0 + + response2 = self.operations.read(filename, encoding=None, binary=False) + assert type(response2) == str # noqa: E721 + assert response2 == response0 + + response3 = self.operations.read(filename, encoding="") + assert type(response3) == str # noqa: E721 + assert response3 == response0 + + response4 = self.operations.read(filename, encoding="UTF-8") + assert type(response4) == str # noqa: E721 + assert response4 == response0 + + def test_read__binary(self): + """ + Test LocalOperations::read for binary data. + """ + filename = __file__ # current file + + with open(filename, 'rb') as file: # open in a binary mode + response0 = file.read() + + assert type(response0) == bytes # noqa: E721 + + response1 = self.operations.read(filename, binary=True) + assert type(response1) == bytes # noqa: E721 + assert response1 == response0 + + def test_read__binary_and_encoding(self): + """ + Test LocalOperations::read for binary data and encoding. + """ + filename = __file__ # current file + + with pytest.raises( + InvalidOperationException, + match=re.escape("Enconding is not allowed for read binary operation")): + self.operations.read(filename, encoding="", binary=True) + + def test_read__unknown_file(self): + """ + Test LocalOperations::read with unknown file. + """ + + with pytest.raises(FileNotFoundError, match=re.escape("[Errno 2] No such file or directory: '/dummy'")): + self.operations.read("/dummy") + def test_read_binary__spec(self): """ Test LocalOperations::read_binary. @@ -95,7 +157,9 @@ def test_read_binary__spec__unk_file(self): Test LocalOperations::read_binary with unknown file. """ - with pytest.raises(FileNotFoundError, match=re.escape("[Errno 2] No such file or directory: '/dummy'")): + with pytest.raises( + FileNotFoundError, + match=re.escape("[Errno 2] No such file or directory: '/dummy'")): self.operations.read_binary("/dummy", 0) def test_get_file_size(self): diff --git a/tests/test_remote.py b/tests/test_remote.py index 780ad46e..7421ca3a 100755 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -4,6 +4,7 @@ import re from testgres import ExecUtilException +from testgres import InvalidOperationException from testgres import RemoteOperations from testgres import ConnectionParams @@ -182,6 +183,69 @@ def test_read_binary_file(self): assert isinstance(response, bytes) + def test_read__text(self): + """ + Test RemoteOperations::read for text data. + """ + filename = __file__ # current file + + with open(filename, 'r') as file: # open in a text mode + response0 = file.read() + + assert type(response0) == str # noqa: E721 + + response1 = self.operations.read(filename) + assert type(response1) == str # noqa: E721 + assert response1 == response0 + + response2 = self.operations.read(filename, encoding=None, binary=False) + assert type(response2) == str # noqa: E721 + assert response2 == response0 + + response3 = self.operations.read(filename, encoding="") + assert type(response3) == str # noqa: E721 + assert response3 == response0 + + response4 = self.operations.read(filename, encoding="UTF-8") + assert type(response4) == str # noqa: E721 + assert response4 == response0 + + def test_read__binary(self): + """ + Test RemoteOperations::read for binary data. + """ + filename = __file__ # current file + + with open(filename, 'rb') as file: # open in a binary mode + response0 = file.read() + + assert type(response0) == bytes # noqa: E721 + + response1 = self.operations.read(filename, binary=True) + assert type(response1) == bytes # noqa: E721 + assert response1 == response0 + + def test_read__binary_and_encoding(self): + """ + Test RemoteOperations::read for binary data and encoding. + """ + filename = __file__ # current file + + with pytest.raises( + InvalidOperationException, + match=re.escape("Enconding is not allowed for read binary operation")): + self.operations.read(filename, encoding="", binary=True) + + def test_read__unknown_file(self): + """ + Test RemoteOperations::read with unknown file. + """ + + with pytest.raises( + ExecUtilException, + match=re.escape("cat: /dummy: No such file or directory")): + self.operations.read("/dummy") + def test_read_binary__spec(self): """ Test RemoteOperations::read_binary. From 28c91c2e2670f74bae5beaa18a6f3a30f0f0dfa6 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Wed, 25 Dec 2024 15:11:07 +0300 Subject: [PATCH 370/475] [BUG FIX] xxxOperations::_read__text_with_encoding opens a file as text LocalOps uses "open(filename, mode='r', encoding=encoding)" RemoteOps uses "io.TextIOWrapper(io.BytesIO(binaryData), encoding=encoding)" It solves a problem on Windows. --- testgres/operations/local_ops.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/testgres/operations/local_ops.py b/testgres/operations/local_ops.py index d6013ab5..e1c3b9fd 100644 --- a/testgres/operations/local_ops.py +++ b/testgres/operations/local_ops.py @@ -286,11 +286,10 @@ def read(self, filename, encoding=None, binary=False): def _read__text_with_encoding(self, filename, encoding): assert type(filename) == str # noqa: E721 assert type(encoding) == str # noqa: E721 - content = self._read__binary(filename) - assert type(content) == bytes # noqa: E721 - content_s = content.decode(encoding) - assert type(content_s) == str # noqa: E721 - return content_s + with open(filename, mode='r', encoding=encoding) as file: # open in a text mode + content = file.read() + assert type(content) == str # noqa: E721 + return content def _read__binary(self, filename): assert type(filename) == str # noqa: E721 From 2679646fd6f12038887db3d123fc6bdc8d7fb6b8 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Wed, 25 Dec 2024 16:15:00 +0300 Subject: [PATCH 371/475] [BUG FIX] Part for RemoteOps changes... --- testgres/operations/remote_ops.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index abcb8fe1..bb00cfaf 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -3,6 +3,7 @@ import platform import subprocess import tempfile +import io # we support both pg8000 and psycopg2 try: @@ -341,7 +342,9 @@ def _read__text_with_encoding(self, filename, encoding): assert type(encoding) == str # noqa: E721 content = self._read__binary(filename) assert type(content) == bytes # noqa: E721 - content_s = content.decode(encoding) + buf0 = io.BytesIO(content) + buf1 = io.TextIOWrapper(buf0, encoding=encoding) + content_s = buf1.read() assert type(content_s) == str # noqa: E721 return content_s From 6c514bfe0308e912556105db0029541449af44b3 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Wed, 25 Dec 2024 16:15:57 +0300 Subject: [PATCH 372/475] Code normalization --- testgres/operations/local_ops.py | 4 ++-- testgres/operations/remote_ops.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/testgres/operations/local_ops.py b/testgres/operations/local_ops.py index e1c3b9fd..c88c16ca 100644 --- a/testgres/operations/local_ops.py +++ b/testgres/operations/local_ops.py @@ -278,8 +278,8 @@ def read(self, filename, encoding=None, binary=False): return self._read__binary(filename) # python behavior - assert None or "abc" == "abc" - assert "" or "abc" == "abc" + assert (None or "abc") == "abc" + assert ("" or "abc") == "abc" return self._read__text_with_encoding(filename, encoding or get_default_encoding()) diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index bb00cfaf..c0307195 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -332,8 +332,8 @@ def read(self, filename, binary=False, encoding=None): return self._read__binary(filename) # python behavior - assert None or "abc" == "abc" - assert "" or "abc" == "abc" + assert (None or "abc") == "abc" + assert ("" or "abc") == "abc" return self._read__text_with_encoding(filename, encoding or get_default_encoding()) From 0139bbd858d1fe4bf6566ff6c3e633c32b592063 Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Wed, 25 Dec 2024 16:57:42 +0300 Subject: [PATCH 373/475] [windows] TestgresTests is updated - test_the_same_port - test_port_rereserve_during_node_start - test_port_conflict - use rm_carriage_returns --- tests/test_simple.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/test_simple.py b/tests/test_simple.py index 9cf29c64..8148d05d 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -1054,7 +1054,7 @@ def test_the_same_port(self): self.assertTrue(node._should_free_port) self.assertEqual(type(node.port), int) node_port_copy = node.port - self.assertEqual(node.safe_psql("SELECT 1;"), b'1\n') + self.assertEqual(rm_carriage_returns(node.safe_psql("SELECT 1;")), b'1\n') with get_new_node(port=node.port) as node2: self.assertEqual(type(node2.port), int) @@ -1069,7 +1069,7 @@ def test_the_same_port(self): # node is still working self.assertEqual(node.port, node_port_copy) self.assertTrue(node._should_free_port) - self.assertEqual(node.safe_psql("SELECT 3;"), b'3\n') + self.assertEqual(rm_carriage_returns(node.safe_psql("SELECT 3;")), b'3\n') class tagPortManagerProxy: sm_prev_testgres_reserve_port = None @@ -1175,7 +1175,7 @@ def test_port_rereserve_during_node_start(self): self.assertTrue(node1._should_free_port) self.assertEqual(type(node1.port), int) # noqa: E721 node1_port_copy = node1.port - self.assertEqual(node1.safe_psql("SELECT 1;"), b'1\n') + self.assertEqual(rm_carriage_returns(node1.safe_psql("SELECT 1;")), b'1\n') with __class__.tagPortManagerProxy(node1.port, C_COUNT_OF_BAD_PORT_USAGE): assert __class__.tagPortManagerProxy.sm_DummyPortNumber == node1.port @@ -1191,12 +1191,12 @@ def test_port_rereserve_during_node_start(self): self.assertEqual(__class__.tagPortManagerProxy.sm_DummyPortTotalUsage, C_COUNT_OF_BAD_PORT_USAGE) self.assertTrue(node2.is_started) - self.assertEqual(node2.safe_psql("SELECT 2;"), b'2\n') + self.assertEqual(rm_carriage_returns(node2.safe_psql("SELECT 2;")), b'2\n') # node1 is still working self.assertEqual(node1.port, node1_port_copy) self.assertTrue(node1._should_free_port) - self.assertEqual(node1.safe_psql("SELECT 3;"), b'3\n') + self.assertEqual(rm_carriage_returns(node1.safe_psql("SELECT 3;")), b'3\n') def test_port_conflict(self): assert testgres.PostgresNode._C_MAX_START_ATEMPTS > 1 @@ -1208,7 +1208,7 @@ def test_port_conflict(self): self.assertTrue(node1._should_free_port) self.assertEqual(type(node1.port), int) # noqa: E721 node1_port_copy = node1.port - self.assertEqual(node1.safe_psql("SELECT 1;"), b'1\n') + self.assertEqual(rm_carriage_returns(node1.safe_psql("SELECT 1;")), b'1\n') with __class__.tagPortManagerProxy(node1.port, C_COUNT_OF_BAD_PORT_USAGE): assert __class__.tagPortManagerProxy.sm_DummyPortNumber == node1.port @@ -1233,7 +1233,7 @@ def test_port_conflict(self): # node1 is still working self.assertEqual(node1.port, node1_port_copy) self.assertTrue(node1._should_free_port) - self.assertEqual(node1.safe_psql("SELECT 3;"), b'3\n') + self.assertEqual(rm_carriage_returns(node1.safe_psql("SELECT 3;")), b'3\n') def test_simple_with_bin_dir(self): with get_new_node() as node: From 660ab62d96cda35d19ac69db556a986e4a241297 Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Wed, 25 Dec 2024 16:59:44 +0300 Subject: [PATCH 374/475] [windows] PostgresNode.start (LOCAL__start_node) is corrected [BUG FIX] execute_utility may return None in error. --- testgres/node.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testgres/node.py b/testgres/node.py index 3d023399..baf532de 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -792,7 +792,7 @@ def start(self, params=[], wait=True): def LOCAL__start_node(): _, _, error = execute_utility(_params, self.utils_log_file, verbose=True) - assert type(error) == str # noqa: E721 + assert error is None or type(error) == str # noqa: E721 if error and 'does not exist' in error: raise Exception(error) From 6465b4183f63d82edbbb24ab9a41fa36da551185 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Wed, 25 Dec 2024 17:34:35 +0300 Subject: [PATCH 375/475] A comment is added --- testgres/node.py | 1 + 1 file changed, 1 insertion(+) diff --git a/testgres/node.py b/testgres/node.py index baf532de..e203bb7d 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -791,6 +791,7 @@ def start(self, params=[], wait=True): "start"] + params # yapf: disable def LOCAL__start_node(): + # 'error' will be None on Windows _, _, error = execute_utility(_params, self.utils_log_file, verbose=True) assert error is None or type(error) == str # noqa: E721 if error and 'does not exist' in error: From 2c1dd97fac0053e0b61ab917838e5f88ecaead6b Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Wed, 25 Dec 2024 18:11:09 +0300 Subject: [PATCH 376/475] OsOps::read_binary is updated (offset) - the parameter 'start_pos' is renamed with 'offset' - we raise a runtime-error when 'offset' is negative Tests are added. --- testgres/operations/local_ops.py | 10 ++++++---- testgres/operations/os_ops.py | 6 +++--- testgres/operations/remote_ops.py | 10 ++++++---- tests/test_local.py | 10 ++++++++++ tests/test_remote.py | 10 ++++++++++ 5 files changed, 35 insertions(+), 11 deletions(-) diff --git a/testgres/operations/local_ops.py b/testgres/operations/local_ops.py index c88c16ca..8bdb22cd 100644 --- a/testgres/operations/local_ops.py +++ b/testgres/operations/local_ops.py @@ -331,13 +331,15 @@ def readlines(self, filename, num_lines=0, binary=False, encoding=None): buffers * max(2, int(num_lines / max(cur_lines, 1))) ) # Adjust buffer size - def read_binary(self, filename, start_pos): + def read_binary(self, filename, offset): assert type(filename) == str # noqa: E721 - assert type(start_pos) == int # noqa: E721 - assert start_pos >= 0 + assert type(offset) == int # noqa: E721 + + if offset < 0: + raise ValueError("Negative 'offset' is not supported.") with open(filename, 'rb') as file: # open in a binary mode - file.seek(start_pos, os.SEEK_SET) + file.seek(offset, os.SEEK_SET) r = file.read() assert type(r) == bytes # noqa: E721 return r diff --git a/testgres/operations/os_ops.py b/testgres/operations/os_ops.py index d644509a..35525b3c 100644 --- a/testgres/operations/os_ops.py +++ b/testgres/operations/os_ops.py @@ -98,10 +98,10 @@ def read(self, filename, encoding, binary): def readlines(self, filename): raise NotImplementedError() - def read_binary(self, filename, start_pos): + def read_binary(self, filename, offset): assert type(filename) == str # noqa: E721 - assert type(start_pos) == int # noqa: E721 - assert start_pos >= 0 + assert type(offset) == int # noqa: E721 + assert offset >= 0 raise NotImplementedError() def isfile(self, remote_file): diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index c0307195..2f34ecec 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -370,12 +370,14 @@ def readlines(self, filename, num_lines=0, binary=False, encoding=None): return lines - def read_binary(self, filename, start_pos): + def read_binary(self, filename, offset): assert type(filename) == str # noqa: E721 - assert type(start_pos) == int # noqa: E721 - assert start_pos >= 0 + assert type(offset) == int # noqa: E721 - cmd = ["tail", "-c", "+{}".format(start_pos + 1), filename] + if offset < 0: + raise ValueError("Negative 'offset' is not supported.") + + cmd = ["tail", "-c", "+{}".format(offset + 1), filename] r = self.exec_command(cmd) assert type(r) == bytes # noqa: E721 return r diff --git a/tests/test_local.py b/tests/test_local.py index 47a63994..d7adce17 100644 --- a/tests/test_local.py +++ b/tests/test_local.py @@ -162,6 +162,16 @@ def test_read_binary__spec__unk_file(self): match=re.escape("[Errno 2] No such file or directory: '/dummy'")): self.operations.read_binary("/dummy", 0) + def test_read_binary__spec__negative_offset(self): + """ + Test LocalOperations::read_binary with negative offset. + """ + + with pytest.raises( + ValueError, + match=re.escape("Negative 'offset' is not supported.")): + self.operations.read_binary(__file__, -1) + def test_get_file_size(self): """ Test LocalOperations::get_file_size. diff --git a/tests/test_remote.py b/tests/test_remote.py index 7421ca3a..7071a9d9 100755 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -288,6 +288,16 @@ def test_read_binary__spec__unk_file(self): with pytest.raises(ExecUtilException, match=re.escape("tail: cannot open '/dummy' for reading: No such file or directory")): self.operations.read_binary("/dummy", 0) + def test_read_binary__spec__negative_offset(self): + """ + Test RemoteOperations::read_binary with negative offset. + """ + + with pytest.raises( + ValueError, + match=re.escape("Negative 'offset' is not supported.")): + self.operations.read_binary(__file__, -1) + def test_get_file_size(self): """ Test RemoteOperations::get_file_size. From c6e6f1095c072b1828056a54e73054d2d9788f50 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Fri, 27 Dec 2024 15:49:36 +0300 Subject: [PATCH 377/475] PostgresNode::start is refactored We do not translate a new node port into string when we pack it in new option dictionary. --- testgres/node.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testgres/node.py b/testgres/node.py index e203bb7d..6f466ec9 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -848,7 +848,7 @@ def LOCAL__raise_cannot_start_node__std(from_exception): cur_port = self.port new_port = utils.reserve_port() # can raise try: - options = {'port': str(new_port)} + options = {'port': new_port} self.set_auto_conf(options) except: # noqa: E722 utils.release_port(new_port) From 9e2928ea4ef6a17e5cbb1401164194d631b44f1a Mon Sep 17 00:00:00 2001 From: "e.garbuz" Date: Mon, 20 Jan 2025 05:48:22 +0300 Subject: [PATCH 378/475] Add check backup-source for testing pg_probackup3 --- testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py index b7174a7c..078fdbab 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py @@ -172,6 +172,10 @@ def __init__(self): self.ptrack = test_env.get('PG_PROBACKUP_PTRACK', None) == 'ON' and self.pg_config_version >= 110000 self.wal_tree_enabled = test_env.get('PG_PROBACKUP_WAL_TREE_ENABLED', None) == 'ON' + self.bckp_source = test_env.get('PG_PROBACKUP_SOURCE', 'pro').lower() + if self.bckp_source not in ('base', 'direct', 'pro'): + raise Exception("Wrong PG_PROBACKUP_SOURCE value. Available options: base|direct|pro") + self.paranoia = test_env.get('PG_PROBACKUP_PARANOIA', None) == 'ON' env_compress = test_env.get('ARCHIVE_COMPRESSION', None) if env_compress: From ab7de699e9c90be4a83170e412547cb6849a366e Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Fri, 31 Jan 2025 11:09:17 +0300 Subject: [PATCH 379/475] PortManager::find_free_port is updated We will use exclude_ports only when it is not none. Creation of empty exclude_ports is removed. --- testgres/helpers/port_manager.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/testgres/helpers/port_manager.py b/testgres/helpers/port_manager.py index 6afdf8a9..a7e2a85f 100644 --- a/testgres/helpers/port_manager.py +++ b/testgres/helpers/port_manager.py @@ -26,10 +26,8 @@ def find_free_port(self, ports: Optional[Set[int]] = None, exclude_ports: Option if ports is None: ports = set(range(1024, 65535)) - if exclude_ports is None: - exclude_ports = set() - - ports.difference_update(set(exclude_ports)) + if exclude_ports is not None: + ports.difference_update(set(exclude_ports)) sampled_ports = random.sample(tuple(ports), min(len(ports), 100)) From 27c40d28645c677c5c05f2bd9409b3dc21b8c26d Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Fri, 31 Jan 2025 11:25:46 +0300 Subject: [PATCH 380/475] PortManager::find_free_port is updated Asserts are added: - ports must be the "set" - exclude_ports must be iterable Do not convert exclude_ports into "set" [optimization?] --- testgres/helpers/port_manager.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/testgres/helpers/port_manager.py b/testgres/helpers/port_manager.py index a7e2a85f..f59df259 100644 --- a/testgres/helpers/port_manager.py +++ b/testgres/helpers/port_manager.py @@ -26,8 +26,11 @@ def find_free_port(self, ports: Optional[Set[int]] = None, exclude_ports: Option if ports is None: ports = set(range(1024, 65535)) + assert type(ports) == set + if exclude_ports is not None: - ports.difference_update(set(exclude_ports)) + assert isinstance(exclude_ports, Iterable) + ports.difference_update(exclude_ports) sampled_ports = random.sample(tuple(ports), min(len(ports), 100)) From bc893d8b1d36f2073a3629c102c615e21240245a Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Fri, 31 Jan 2025 11:33:01 +0300 Subject: [PATCH 381/475] noqa: E721 --- testgres/helpers/port_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testgres/helpers/port_manager.py b/testgres/helpers/port_manager.py index f59df259..cfc5c096 100644 --- a/testgres/helpers/port_manager.py +++ b/testgres/helpers/port_manager.py @@ -26,7 +26,7 @@ def find_free_port(self, ports: Optional[Set[int]] = None, exclude_ports: Option if ports is None: ports = set(range(1024, 65535)) - assert type(ports) == set + assert type(ports) == set # noqa: E721 if exclude_ports is not None: assert isinstance(exclude_ports, Iterable) From 67beb95ce02bd824a0f0cc75fd52ebe426de666e Mon Sep 17 00:00:00 2001 From: asavchkov Date: Tue, 11 Feb 2025 18:52:41 +0700 Subject: [PATCH 382/475] Up version --- setup.py | 2 +- testgres/plugins/pg_probackup2/setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index a41094d6..16586b88 100755 --- a/setup.py +++ b/setup.py @@ -27,7 +27,7 @@ readme = f.read() setup( - version='1.10.3', + version='1.10.4', name='testgres', packages=['testgres', 'testgres.operations', 'testgres.helpers'], description='Testing utility for PostgreSQL and its extensions', diff --git a/testgres/plugins/pg_probackup2/setup.py b/testgres/plugins/pg_probackup2/setup.py index ade2d85d..619b8d39 100644 --- a/testgres/plugins/pg_probackup2/setup.py +++ b/testgres/plugins/pg_probackup2/setup.py @@ -4,7 +4,7 @@ from distutils.core import setup setup( - version='0.0.4', + version='0.0.5', name='testgres_pg_probackup2', packages=['pg_probackup2', 'pg_probackup2.storage'], description='Plugin for testgres that manages pg_probackup2', From 6fe28a5cea332d2e8f8fbc9374a9f84e4c36e287 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Mon, 17 Feb 2025 13:51:20 +0300 Subject: [PATCH 383/475] [BUG FIX] A problem with socket directory is fixed On non-Windows platform Postgres always looks for socket files in "/tmp" directory. --- testgres/node.py | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/testgres/node.py b/testgres/node.py index 6f466ec9..b85a62f2 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -6,6 +6,7 @@ import subprocess import threading import tempfile +import platform from queue import Queue import time @@ -1925,7 +1926,7 @@ def make_simple( # Define delayed propertyes if not ("unix_socket_directories" in options.keys()): - options["unix_socket_directories"] = __class__._gettempdir() + options["unix_socket_directories"] = __class__._gettempdir_for_socket() # Set config values node.set_auto_conf(options) @@ -1938,6 +1939,33 @@ def make_simple( return node + def _gettempdir_for_socket(): + platform_system_name = platform.system().lower() + + if platform_system_name == "windows": + return __class__._gettempdir() + + # + # [2025-02-17] Hot fix. + # + # Let's use hard coded path as Postgres likes. + # + # pg_config_manual.h: + # + # #ifndef WIN32 + # #define DEFAULT_PGSOCKET_DIR "/tmp" + # #else + # #define DEFAULT_PGSOCKET_DIR "" + # #endif + # + # On the altlinux-10 tempfile.gettempdir() may return + # the path to "private" temp directiry - "/temp/.private//" + # + # But Postgres want to find a socket file in "/tmp" (see above). + # + + return "/tmp" + def _gettempdir(): v = tempfile.gettempdir() From 89c8625252be857d8589141921aa185ac9db2d67 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Mon, 17 Feb 2025 14:02:24 +0300 Subject: [PATCH 384/475] A problem with tests.test_simple.TestgresTests.test_simple_with_bin_dir is fixed testgres generates the exception testgres.exceptions.ExecUtilException, but test traps the exception FileNotFoundError. Error message is: Utility exited with non-zero code. Error: `bash: line 1: wrong/path/postgres: No such file or directory` Command: ('wrong/path/postgres --version',) --- tests/test_simple.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/tests/test_simple.py b/tests/test_simple.py index 8148d05d..4e6fb573 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -1244,13 +1244,19 @@ def test_simple_with_bin_dir(self): correct_bin_dir = app.make_simple(base_dir=node.base_dir, bin_dir=bin_dir) correct_bin_dir.slow_start() correct_bin_dir.safe_psql("SELECT 1;") + correct_bin_dir.stop() + + while True: + try: + app.make_simple(base_dir=node.base_dir, bin_dir="wrong/path") + except FileNotFoundError: + break # Expected error + except ExecUtilException: + break # Expected error - try: - wrong_bin_dir = app.make_empty(base_dir=node.base_dir, bin_dir="wrong/path") - wrong_bin_dir.slow_start() raise RuntimeError("Error was expected.") # We should not reach this - except FileNotFoundError: - pass # Expected error + + return def test_set_auto_conf(self): # elements contain [property id, value, storage value] From 737e0b4fee2249247934b147fe60feb844b3da41 Mon Sep 17 00:00:00 2001 From: asavchkov Date: Tue, 18 Feb 2025 19:23:05 +0700 Subject: [PATCH 385/475] Up version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 16586b88..3f2474dd 100755 --- a/setup.py +++ b/setup.py @@ -27,7 +27,7 @@ readme = f.read() setup( - version='1.10.4', + version='1.10.5', name='testgres', packages=['testgres', 'testgres.operations', 'testgres.helpers'], description='Testing utility for PostgreSQL and its extensions', From 4493d69d80b803d1c48ffd49f2e6d33a35e2f734 Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Wed, 19 Feb 2025 11:10:21 +0300 Subject: [PATCH 386/475] OsOperations::cwd() is corrected (#182) * OsOperations::cwd() is corrected This patch fixes the following problems: - It does not work on Windows - It always returns LOCAL path --- testgres/operations/local_ops.py | 3 +++ testgres/operations/os_ops.py | 7 +------ testgres/operations/remote_ops.py | 4 ++++ tests/test_local.py | 17 +++++++++++++++++ tests/test_remote.py | 10 ++++++++++ 5 files changed, 35 insertions(+), 6 deletions(-) diff --git a/testgres/operations/local_ops.py b/testgres/operations/local_ops.py index 8bdb22cd..fc3e3954 100644 --- a/testgres/operations/local_ops.py +++ b/testgres/operations/local_ops.py @@ -152,6 +152,9 @@ def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False, def environ(self, var_name): return os.environ.get(var_name) + def cwd(self): + return os.getcwd() + def find_executable(self, executable): return find_executable(executable) diff --git a/testgres/operations/os_ops.py b/testgres/operations/os_ops.py index 35525b3c..00880863 100644 --- a/testgres/operations/os_ops.py +++ b/testgres/operations/os_ops.py @@ -1,6 +1,5 @@ import getpass import locale -import sys try: import psycopg2 as pglib # noqa: F401 @@ -39,11 +38,7 @@ def environ(self, var_name): raise NotImplementedError() def cwd(self): - if sys.platform == 'linux': - cmd = 'pwd' - elif sys.platform == 'win32': - cmd = 'cd' - return self.exec_command(cmd).decode().rstrip() + raise NotImplementedError() def find_executable(self, executable): raise NotImplementedError() diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index 2f34ecec..3ebc2e60 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -138,6 +138,10 @@ def environ(self, var_name: str) -> str: cmd = "echo ${}".format(var_name) return self.exec_command(cmd, encoding=get_default_encoding()).strip() + def cwd(self): + cmd = 'pwd' + return self.exec_command(cmd, encoding=get_default_encoding()).rstrip() + def find_executable(self, executable): search_paths = self.environ("PATH") if not search_paths: diff --git a/tests/test_local.py b/tests/test_local.py index d7adce17..568a4bc5 100644 --- a/tests/test_local.py +++ b/tests/test_local.py @@ -256,3 +256,20 @@ def test_isdir_false__file(self): response = self.operations.isdir(name) assert response is False + + def test_cwd(self): + """ + Test cwd. + """ + v = self.operations.cwd() + + assert v is not None + assert type(v) == str # noqa: E721 + + expectedValue = os.getcwd() + assert expectedValue is not None + assert type(expectedValue) == str # noqa: E721 + assert expectedValue != "" # research + + # Comp result + assert v == expectedValue diff --git a/tests/test_remote.py b/tests/test_remote.py index 7071a9d9..30c5d348 100755 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -392,3 +392,13 @@ def test_isdir_false__file(self): response = self.operations.isdir(name) assert response is False + + def test_cwd(self): + """ + Test cwd. + """ + v = self.operations.cwd() + + assert v is not None + assert type(v) == str # noqa: E721 + assert v != "" From 22c476347d4e8bfae832e641e00428ed4b6d14a2 Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Wed, 19 Feb 2025 15:26:56 +0300 Subject: [PATCH 387/475] OsOps::write method is corrected (#183) LocalOperations - [BUG FIX] (read_write=true and truncate=false) writes to begging of a file - Preparation of data is added (verification/encoding/decoding/eol) RemoteOperations - Preparation of data is corrected (verification/encoding/decoding/eol) - Temp file is always opened with "w+"/"w+b" modes. Tests are added. --- testgres/operations/local_ops.py | 53 ++++++++++++++++++------ testgres/operations/remote_ops.py | 35 ++++++++++------ tests/test_local.py | 69 +++++++++++++++++++++++++++++++ tests/test_remote.py | 69 +++++++++++++++++++++++++++++++ 4 files changed, 201 insertions(+), 25 deletions(-) diff --git a/testgres/operations/local_ops.py b/testgres/operations/local_ops.py index fc3e3954..5c79bb7e 100644 --- a/testgres/operations/local_ops.py +++ b/testgres/operations/local_ops.py @@ -235,27 +235,54 @@ def write(self, filename, data, truncate=False, binary=False, read_and_write=Fal Args: filename: The file path where the data will be written. data: The data to be written to the file. - truncate: If True, the file will be truncated before writing ('w' or 'wb' option); - if False (default), data will be appended ('a' or 'ab' option). - binary: If True, the data will be written in binary mode ('wb' or 'ab' option); - if False (default), the data will be written in text mode ('w' or 'a' option). - read_and_write: If True, the file will be opened with read and write permissions ('r+' option); - if False (default), only write permission will be used ('w', 'a', 'wb', or 'ab' option) + truncate: If True, the file will be truncated before writing ('w' option); + if False (default), data will be appended ('a' option). + binary: If True, the data will be written in binary mode ('b' option); + if False (default), the data will be written in text mode. + read_and_write: If True, the file will be opened with read and write permissions ('+' option); + if False (default), only write permission will be used. """ - # If it is a bytes str or list if isinstance(data, bytes) or isinstance(data, list) and all(isinstance(item, bytes) for item in data): binary = True - mode = "wb" if binary else "w" - if not truncate: - mode = "ab" if binary else "a" + + mode = "w" if truncate else "a" + if read_and_write: - mode = "r+b" if binary else "r+" + mode += "+" + + # If it is a bytes str or list + if binary: + mode += "b" + + assert type(mode) == str # noqa: E721 + assert mode != "" with open(filename, mode) as file: if isinstance(data, list): - file.writelines(data) + data2 = [__class__._prepare_line_to_write(s, binary) for s in data] + file.writelines(data2) else: - file.write(data) + data2 = __class__._prepare_data_to_write(data, binary) + file.write(data2) + + def _prepare_line_to_write(data, binary): + data = __class__._prepare_data_to_write(data, binary) + + if binary: + assert type(data) == bytes # noqa: E721 + return data.rstrip(b'\n') + b'\n' + + assert type(data) == str # noqa: E721 + return data.rstrip('\n') + '\n' + + def _prepare_data_to_write(data, binary): + if isinstance(data, bytes): + return data if binary else data.decode() + + if isinstance(data, str): + return data if not binary else data.encode() + + raise InvalidOperationException("Unknown type of data type [{0}].".format(type(data).__name__)) def touch(self, filename): """ diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index 3ebc2e60..f690e063 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -278,10 +278,6 @@ def write(self, filename, data, truncate=False, binary=False, read_and_write=Fal if not encoding: encoding = get_default_encoding() mode = "wb" if binary else "w" - if not truncate: - mode = "ab" if binary else "a" - if read_and_write: - mode = "r+b" if binary else "r+" with tempfile.NamedTemporaryFile(mode=mode, delete=False) as tmp_file: # For scp the port is specified by a "-P" option @@ -292,16 +288,12 @@ def write(self, filename, data, truncate=False, binary=False, read_and_write=Fal subprocess.run(scp_cmd, check=False) # The file might not exist yet tmp_file.seek(0, os.SEEK_END) - if isinstance(data, bytes) and not binary: - data = data.decode(encoding) - elif isinstance(data, str) and binary: - data = data.encode(encoding) - if isinstance(data, list): - data = [(s if isinstance(s, str) else s.decode(get_default_encoding())).rstrip('\n') + '\n' for s in data] - tmp_file.writelines(data) + data2 = [__class__._prepare_line_to_write(s, binary, encoding) for s in data] + tmp_file.writelines(data2) else: - tmp_file.write(data) + data2 = __class__._prepare_data_to_write(data, binary, encoding) + tmp_file.write(data2) tmp_file.flush() scp_cmd = ['scp'] + scp_args + [tmp_file.name, f"{self.ssh_dest}:{filename}"] @@ -313,6 +305,25 @@ def write(self, filename, data, truncate=False, binary=False, read_and_write=Fal os.remove(tmp_file.name) + def _prepare_line_to_write(data, binary, encoding): + data = __class__._prepare_data_to_write(data, binary, encoding) + + if binary: + assert type(data) == bytes # noqa: E721 + return data.rstrip(b'\n') + b'\n' + + assert type(data) == str # noqa: E721 + return data.rstrip('\n') + '\n' + + def _prepare_data_to_write(data, binary, encoding): + if isinstance(data, bytes): + return data if binary else data.decode(encoding) + + if isinstance(data, str): + return data if not binary else data.encode(encoding) + + raise InvalidOperationException("Unknown type of data type [{0}].".format(type(data).__name__)) + def touch(self, filename): """ Create a new file or update the access and modification times of an existing file on the remote server. diff --git a/tests/test_local.py b/tests/test_local.py index 568a4bc5..4051bfb5 100644 --- a/tests/test_local.py +++ b/tests/test_local.py @@ -2,6 +2,7 @@ import pytest import re +import tempfile from testgres import ExecUtilException from testgres import InvalidOperationException @@ -273,3 +274,71 @@ def test_cwd(self): # Comp result assert v == expectedValue + + class tagWriteData001: + def __init__(self, sign, source, cp_rw, cp_truncate, cp_binary, cp_data, result): + self.sign = sign + self.source = source + self.call_param__rw = cp_rw + self.call_param__truncate = cp_truncate + self.call_param__binary = cp_binary + self.call_param__data = cp_data + self.result = result + + sm_write_data001 = [ + tagWriteData001("A001", "1234567890", False, False, False, "ABC", "1234567890ABC"), + tagWriteData001("A002", b"1234567890", False, False, True, b"ABC", b"1234567890ABC"), + + tagWriteData001("B001", "1234567890", False, True, False, "ABC", "ABC"), + tagWriteData001("B002", "1234567890", False, True, False, "ABC1234567890", "ABC1234567890"), + tagWriteData001("B003", b"1234567890", False, True, True, b"ABC", b"ABC"), + tagWriteData001("B004", b"1234567890", False, True, True, b"ABC1234567890", b"ABC1234567890"), + + tagWriteData001("C001", "1234567890", True, False, False, "ABC", "1234567890ABC"), + tagWriteData001("C002", b"1234567890", True, False, True, b"ABC", b"1234567890ABC"), + + tagWriteData001("D001", "1234567890", True, True, False, "ABC", "ABC"), + tagWriteData001("D002", "1234567890", True, True, False, "ABC1234567890", "ABC1234567890"), + tagWriteData001("D003", b"1234567890", True, True, True, b"ABC", b"ABC"), + tagWriteData001("D004", b"1234567890", True, True, True, b"ABC1234567890", b"ABC1234567890"), + + tagWriteData001("E001", "\0001234567890\000", False, False, False, "\000ABC\000", "\0001234567890\000\000ABC\000"), + tagWriteData001("E002", b"\0001234567890\000", False, False, True, b"\000ABC\000", b"\0001234567890\000\000ABC\000"), + + tagWriteData001("F001", "a\nb\n", False, False, False, ["c", "d"], "a\nb\nc\nd\n"), + tagWriteData001("F002", b"a\nb\n", False, False, True, [b"c", b"d"], b"a\nb\nc\nd\n"), + + tagWriteData001("G001", "a\nb\n", False, False, False, ["c\n\n", "d\n"], "a\nb\nc\nd\n"), + tagWriteData001("G002", b"a\nb\n", False, False, True, [b"c\n\n", b"d\n"], b"a\nb\nc\nd\n"), + ] + + @pytest.fixture( + params=sm_write_data001, + ids=[x.sign for x in sm_write_data001], + ) + def write_data001(self, request): + assert isinstance(request, pytest.FixtureRequest) + assert type(request.param) == __class__.tagWriteData001 # noqa: E721 + return request.param + + def test_write(self, write_data001): + assert type(write_data001) == __class__.tagWriteData001 # noqa: E721 + + mode = "w+b" if write_data001.call_param__binary else "w+" + + with tempfile.NamedTemporaryFile(mode=mode, delete=True) as tmp_file: + tmp_file.write(write_data001.source) + tmp_file.flush() + + self.operations.write( + tmp_file.name, + write_data001.call_param__data, + read_and_write=write_data001.call_param__rw, + truncate=write_data001.call_param__truncate, + binary=write_data001.call_param__binary) + + tmp_file.seek(0) + + s = tmp_file.read() + + assert s == write_data001.result diff --git a/tests/test_remote.py b/tests/test_remote.py index 30c5d348..3e6b79dd 100755 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -2,6 +2,7 @@ import pytest import re +import tempfile from testgres import ExecUtilException from testgres import InvalidOperationException @@ -402,3 +403,71 @@ def test_cwd(self): assert v is not None assert type(v) == str # noqa: E721 assert v != "" + + class tagWriteData001: + def __init__(self, sign, source, cp_rw, cp_truncate, cp_binary, cp_data, result): + self.sign = sign + self.source = source + self.call_param__rw = cp_rw + self.call_param__truncate = cp_truncate + self.call_param__binary = cp_binary + self.call_param__data = cp_data + self.result = result + + sm_write_data001 = [ + tagWriteData001("A001", "1234567890", False, False, False, "ABC", "1234567890ABC"), + tagWriteData001("A002", b"1234567890", False, False, True, b"ABC", b"1234567890ABC"), + + tagWriteData001("B001", "1234567890", False, True, False, "ABC", "ABC"), + tagWriteData001("B002", "1234567890", False, True, False, "ABC1234567890", "ABC1234567890"), + tagWriteData001("B003", b"1234567890", False, True, True, b"ABC", b"ABC"), + tagWriteData001("B004", b"1234567890", False, True, True, b"ABC1234567890", b"ABC1234567890"), + + tagWriteData001("C001", "1234567890", True, False, False, "ABC", "1234567890ABC"), + tagWriteData001("C002", b"1234567890", True, False, True, b"ABC", b"1234567890ABC"), + + tagWriteData001("D001", "1234567890", True, True, False, "ABC", "ABC"), + tagWriteData001("D002", "1234567890", True, True, False, "ABC1234567890", "ABC1234567890"), + tagWriteData001("D003", b"1234567890", True, True, True, b"ABC", b"ABC"), + tagWriteData001("D004", b"1234567890", True, True, True, b"ABC1234567890", b"ABC1234567890"), + + tagWriteData001("E001", "\0001234567890\000", False, False, False, "\000ABC\000", "\0001234567890\000\000ABC\000"), + tagWriteData001("E002", b"\0001234567890\000", False, False, True, b"\000ABC\000", b"\0001234567890\000\000ABC\000"), + + tagWriteData001("F001", "a\nb\n", False, False, False, ["c", "d"], "a\nb\nc\nd\n"), + tagWriteData001("F002", b"a\nb\n", False, False, True, [b"c", b"d"], b"a\nb\nc\nd\n"), + + tagWriteData001("G001", "a\nb\n", False, False, False, ["c\n\n", "d\n"], "a\nb\nc\nd\n"), + tagWriteData001("G002", b"a\nb\n", False, False, True, [b"c\n\n", b"d\n"], b"a\nb\nc\nd\n"), + ] + + @pytest.fixture( + params=sm_write_data001, + ids=[x.sign for x in sm_write_data001], + ) + def write_data001(self, request): + assert isinstance(request, pytest.FixtureRequest) + assert type(request.param) == __class__.tagWriteData001 # noqa: E721 + return request.param + + def test_write(self, write_data001): + assert type(write_data001) == __class__.tagWriteData001 # noqa: E721 + + mode = "w+b" if write_data001.call_param__binary else "w+" + + with tempfile.NamedTemporaryFile(mode=mode, delete=True) as tmp_file: + tmp_file.write(write_data001.source) + tmp_file.flush() + + self.operations.write( + tmp_file.name, + write_data001.call_param__data, + read_and_write=write_data001.call_param__rw, + truncate=write_data001.call_param__truncate, + binary=write_data001.call_param__binary) + + tmp_file.seek(0) + + s = tmp_file.read() + + assert s == write_data001.result From 7fd2f07af75391503fb1c79a1bc9ec1f763923af Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Thu, 20 Feb 2025 14:56:02 +0300 Subject: [PATCH 388/475] RemoteOperations::exec_command updated (#185) - Exact enumeration of supported 'cmd' types - Refactoring --- testgres/operations/remote_ops.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index f690e063..a24fce50 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -78,14 +78,17 @@ def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False, assert input_prepared is None or (type(input_prepared) == bytes) # noqa: E721 - ssh_cmd = [] - if isinstance(cmd, str): - ssh_cmd = ['ssh', self.ssh_dest] + self.ssh_args + [cmd] - elif isinstance(cmd, list): - ssh_cmd = ['ssh', self.ssh_dest] + self.ssh_args + [subprocess.list2cmdline(cmd)] + if type(cmd) == str: # noqa: E721 + cmd_s = cmd + elif type(cmd) == list: # noqa: E721 + cmd_s = subprocess.list2cmdline(cmd) else: raise ValueError("Invalid 'cmd' argument type - {0}".format(type(cmd).__name__)) + assert type(cmd_s) == str # noqa: E721 + + ssh_cmd = ['ssh', self.ssh_dest] + self.ssh_args + [cmd_s] + process = subprocess.Popen(ssh_cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) assert not (process is None) if get_process: From e44aa9813daf04e8170f57c277923100aa04eadc Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Fri, 21 Feb 2025 20:04:08 +0300 Subject: [PATCH 389/475] RemoteOperations::exec_command explicitly transfers LANG, LANGUAGE and LC_* envvars to the server side (#187) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * RemoteOperations::exec_command updated - Exact enumeration of supported 'cmd' types - Refactoring * RemoteOperations::exec_command explicitly transfers LANG, LANGUAGE and LC_* envvars to the server side It should help resolve a problem with replacing a LANG variable by ssh-server. History. On our internal tests we got a problem on the Debian 11 and PostgresPro STD-13. One test returned the error from initdb: initdb: error: collations with different collate and ctype values ("en_US.UTF-8" and "C.UTF-8" accordingly) are not supported by ICU - TestRunner set variable LANG="C" - Python set variable LC_CTYPE="C.UTF-8" - Test call inidb through command "ssh test@localhost inidb -D ...." - SSH-server replaces LANG with value "en_US.UTF-8" (from etc/default/locale) - initdb calculate collate through this value of LANG variable and get en_US.UTF-8 So we have that: - ctype is C.UTF-8 - collate is en_US.UTF-8 ICU on the Debuan-11 (uconv v2.1 ICU 67.1) does not suppot this combination and inidb rturns the error. This patch generates a new command line for ssh: ssh test@localhost "LANG=\"...\";LC_xxx=\"...\";" It resolves this problem with initdb and should help resolve other problems with execution of command through SSH. Amen. * New tests in TestgresRemoteTests are added New tests: - test_init__LANG_С - test_init__unk_LANG_and_LC_CTYPE * TestgresRemoteTests.test_init__unk_LANG_and_LC_CTYPE is updated Let's test bad data with '\' and '"' symbols. * Static methods are marked with @staticmethod [thanks to Victoria Shepard] The following methods of RemoteOperations were corrected: - _make_exec_env_list - _does_put_envvar_into_exec_cmd - _quote_envvar * TestRemoteOperations::_quote_envvar is updated (typification) --- testgres/operations/remote_ops.py | 46 +++++++++++++++++- tests/test_simple_remote.py | 79 +++++++++++++++++++++++++++++++ 2 files changed, 124 insertions(+), 1 deletion(-) diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index a24fce50..af4c59f9 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -87,7 +87,12 @@ def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False, assert type(cmd_s) == str # noqa: E721 - ssh_cmd = ['ssh', self.ssh_dest] + self.ssh_args + [cmd_s] + cmd_items = __class__._make_exec_env_list() + cmd_items.append(cmd_s) + + env_cmd_s = ';'.join(cmd_items) + + ssh_cmd = ['ssh', self.ssh_dest] + self.ssh_args + [env_cmd_s] process = subprocess.Popen(ssh_cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) assert not (process is None) @@ -510,6 +515,45 @@ def db_connect(self, dbname, user, password=None, host="localhost", port=5432): ) return conn + @staticmethod + def _make_exec_env_list() -> list[str]: + result = list[str]() + for envvar in os.environ.items(): + if not __class__._does_put_envvar_into_exec_cmd(envvar[0]): + continue + qvalue = __class__._quote_envvar(envvar[1]) + assert type(qvalue) == str # noqa: E721 + result.append(envvar[0] + "=" + qvalue) + continue + + return result + + sm_envs_for_exec_cmd = ["LANG", "LANGUAGE"] + + @staticmethod + def _does_put_envvar_into_exec_cmd(name: str) -> bool: + assert type(name) == str # noqa: E721 + name = name.upper() + if name.startswith("LC_"): + return True + if name in __class__.sm_envs_for_exec_cmd: + return True + return False + + @staticmethod + def _quote_envvar(value: str) -> str: + assert type(value) == str # noqa: E721 + result = "\"" + for ch in value: + if ch == "\"": + result += "\\\"" + elif ch == "\\": + result += "\\\\" + else: + result += ch + result += "\"" + return result + def normalize_error(error): if isinstance(error, bytes): diff --git a/tests/test_simple_remote.py b/tests/test_simple_remote.py index c8dd2964..2b581ac9 100755 --- a/tests/test_simple_remote.py +++ b/tests/test_simple_remote.py @@ -119,6 +119,79 @@ def test_custom_init(self): # there should be no trust entries at all self.assertFalse(any('trust' in s for s in lines)) + def test_init__LANG_С(self): + # PBCKP-1744 + prev_LANG = os.environ.get("LANG") + + try: + os.environ["LANG"] = "C" + + with get_remote_node(conn_params=conn_params) as node: + node.init().start() + finally: + __class__.helper__restore_envvar("LANG", prev_LANG) + + def test_init__unk_LANG_and_LC_CTYPE(self): + # PBCKP-1744 + prev_LANG = os.environ.get("LANG") + prev_LANGUAGE = os.environ.get("LANGUAGE") + prev_LC_CTYPE = os.environ.get("LC_CTYPE") + prev_LC_COLLATE = os.environ.get("LC_COLLATE") + + try: + # TODO: Pass unkData through test parameter. + unkDatas = [ + ("UNKNOWN_LANG", "UNKNOWN_CTYPE"), + ("\"UNKNOWN_LANG\"", "\"UNKNOWN_CTYPE\""), + ("\\UNKNOWN_LANG\\", "\\UNKNOWN_CTYPE\\"), + ("\"UNKNOWN_LANG", "UNKNOWN_CTYPE\""), + ("\\UNKNOWN_LANG", "UNKNOWN_CTYPE\\"), + ("\\", "\\"), + ("\"", "\""), + ] + + for unkData in unkDatas: + logging.info("----------------------") + logging.info("Unk LANG is [{0}]".format(unkData[0])) + logging.info("Unk LC_CTYPE is [{0}]".format(unkData[1])) + + os.environ["LANG"] = unkData[0] + os.environ.pop("LANGUAGE", None) + os.environ["LC_CTYPE"] = unkData[1] + os.environ.pop("LC_COLLATE", None) + + assert os.environ.get("LANG") == unkData[0] + assert not ("LANGUAGE" in os.environ.keys()) + assert os.environ.get("LC_CTYPE") == unkData[1] + assert not ("LC_COLLATE" in os.environ.keys()) + + while True: + try: + with get_remote_node(conn_params=conn_params): + pass + except testgres.exceptions.ExecUtilException as e: + # + # Example of an error message: + # + # warning: setlocale: LC_CTYPE: cannot change locale (UNKNOWN_CTYPE): No such file or directory + # postgres (PostgreSQL) 14.12 + # + errMsg = str(e) + + logging.info("Error message is: {0}".format(errMsg)) + + assert "LC_CTYPE" in errMsg + assert unkData[1] in errMsg + assert "warning: setlocale: LC_CTYPE: cannot change locale (" + unkData[1] + "): No such file or directory" in errMsg + assert "postgres" in errMsg + break + raise Exception("We expected an error!") + finally: + __class__.helper__restore_envvar("LANG", prev_LANG) + __class__.helper__restore_envvar("LANGUAGE", prev_LANGUAGE) + __class__.helper__restore_envvar("LC_CTYPE", prev_LC_CTYPE) + __class__.helper__restore_envvar("LC_COLLATE", prev_LC_COLLATE) + def test_double_init(self): with get_remote_node(conn_params=conn_params).init() as node: # can't initialize node more than once @@ -994,6 +1067,12 @@ def test_child_process_dies(self): # try to handle children list -- missing processes will have ptype "ProcessType.Unknown" [ProcessProxy(p) for p in children] + def helper__restore_envvar(name, prev_value): + if prev_value is None: + os.environ.pop(name, None) + else: + os.environ[name] = prev_value + if __name__ == '__main__': if os_ops.environ('ALT_CONFIG'): From 44b99f08084d706a3cc9a67fa255579def9cb201 Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Fri, 21 Feb 2025 22:11:00 +0300 Subject: [PATCH 390/475] TestRemoteOperations::test_makedirs_and_rmdirs_success is updated (#188) The new checks are added. --- tests/test_remote.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_remote.py b/tests/test_remote.py index 3e6b79dd..4330b92f 100755 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -83,10 +83,12 @@ def test_makedirs_and_rmdirs_success(self): # Test makedirs self.operations.makedirs(path) + assert os.path.exists(path) assert self.operations.path_exists(path) # Test rmdirs self.operations.rmdirs(path) + assert not os.path.exists(path) assert not self.operations.path_exists(path) def test_makedirs_and_rmdirs_failure(self): From 40eaf7de413eb4b4eb94ed32111837f8aaa7507b Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Sun, 23 Feb 2025 19:34:57 +0300 Subject: [PATCH 391/475] Usage of @staticmethod (#189) * Usage of @staticmethod All the known static methods are marked with @staticmethod decorators. * Fix for "ERROR: Unexpected indentation. [docutils]" This commit should fix an unstable error in CI: /pg/testgres/testgres/api.py:docstring of testgres.api.get_remote_node:5: ERROR: Unexpected indentation. [docutils] --- testgres/api.py | 4 +--- testgres/node.py | 3 +++ testgres/operations/helpers.py | 3 +++ testgres/operations/local_ops.py | 2 ++ testgres/operations/raise_error.py | 4 ++++ testgres/operations/remote_ops.py | 2 ++ tests/test_simple.py | 2 ++ 7 files changed, 17 insertions(+), 3 deletions(-) diff --git a/testgres/api.py b/testgres/api.py index e4b1cdd5..6a96ee84 100644 --- a/testgres/api.py +++ b/testgres/api.py @@ -47,8 +47,6 @@ def get_remote_node(name=None, conn_params=None): Simply a wrapper around :class:`.PostgresNode` constructor for remote node. See :meth:`.PostgresNode.__init__` for details. For remote connection you can add the next parameter: - conn_params = ConnectionParams(host='127.0.0.1', - ssh_key=None, - username=default_username()) + conn_params = ConnectionParams(host='127.0.0.1', ssh_key=None, username=default_username()) """ return get_new_node(name=name, conn_params=conn_params) diff --git a/testgres/node.py b/testgres/node.py index b85a62f2..8a712753 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -1939,6 +1939,7 @@ def make_simple( return node + @staticmethod def _gettempdir_for_socket(): platform_system_name = platform.system().lower() @@ -1966,6 +1967,7 @@ def _gettempdir_for_socket(): return "/tmp" + @staticmethod def _gettempdir(): v = tempfile.gettempdir() @@ -1984,6 +1986,7 @@ def _gettempdir(): # OK return v + @staticmethod def _raise_bugcheck(msg): assert type(msg) == str # noqa: E721 assert msg != "" diff --git a/testgres/operations/helpers.py b/testgres/operations/helpers.py index b50f0baa..03e97edc 100644 --- a/testgres/operations/helpers.py +++ b/testgres/operations/helpers.py @@ -2,6 +2,7 @@ class Helpers: + @staticmethod def _make_get_default_encoding_func(): # locale.getencoding is added in Python 3.11 if hasattr(locale, 'getencoding'): @@ -13,6 +14,7 @@ def _make_get_default_encoding_func(): # Prepared pointer on function to get a name of system codepage _get_default_encoding_func = _make_get_default_encoding_func() + @staticmethod def GetDefaultEncoding(): # # Original idea/source was: @@ -36,6 +38,7 @@ def GetDefaultEncoding(): # Is it an unexpected situation? return 'UTF-8' + @staticmethod def PrepareProcessInput(input, encoding): if not input: return None diff --git a/testgres/operations/local_ops.py b/testgres/operations/local_ops.py index 5c79bb7e..91070fe7 100644 --- a/testgres/operations/local_ops.py +++ b/testgres/operations/local_ops.py @@ -265,6 +265,7 @@ def write(self, filename, data, truncate=False, binary=False, read_and_write=Fal data2 = __class__._prepare_data_to_write(data, binary) file.write(data2) + @staticmethod def _prepare_line_to_write(data, binary): data = __class__._prepare_data_to_write(data, binary) @@ -275,6 +276,7 @@ def _prepare_line_to_write(data, binary): assert type(data) == str # noqa: E721 return data.rstrip('\n') + '\n' + @staticmethod def _prepare_data_to_write(data, binary): if isinstance(data, bytes): return data if binary else data.decode() diff --git a/testgres/operations/raise_error.py b/testgres/operations/raise_error.py index 0e760e74..6031b238 100644 --- a/testgres/operations/raise_error.py +++ b/testgres/operations/raise_error.py @@ -3,6 +3,7 @@ class RaiseError: + @staticmethod def UtilityExitedWithNonZeroCode(cmd, exit_code, msg_arg, error, out): assert type(exit_code) == int # noqa: E721 @@ -20,12 +21,14 @@ def UtilityExitedWithNonZeroCode(cmd, exit_code, msg_arg, error, out): out=out, error=error) + @staticmethod def _TranslateDataIntoString(data): if type(data) == bytes: # noqa: E721 return __class__._TranslateDataIntoString__FromBinary(data) return str(data) + @staticmethod def _TranslateDataIntoString__FromBinary(data): assert type(data) == bytes # noqa: E721 @@ -36,6 +39,7 @@ def _TranslateDataIntoString__FromBinary(data): return "#cannot_decode_text" + @staticmethod def _BinaryIsASCII(data): assert type(data) == bytes # noqa: E721 diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index af4c59f9..51f5b2e8 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -313,6 +313,7 @@ def write(self, filename, data, truncate=False, binary=False, read_and_write=Fal os.remove(tmp_file.name) + @staticmethod def _prepare_line_to_write(data, binary, encoding): data = __class__._prepare_data_to_write(data, binary, encoding) @@ -323,6 +324,7 @@ def _prepare_line_to_write(data, binary, encoding): assert type(data) == str # noqa: E721 return data.rstrip('\n') + '\n' + @staticmethod def _prepare_data_to_write(data, binary, encoding): if isinstance(data, bytes): return data if binary else data.decode(encoding) diff --git a/tests/test_simple.py b/tests/test_simple.py index 4e6fb573..a751f0a3 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -1125,6 +1125,7 @@ def __exit__(self, type, value, traceback): __class__.sm_prev_testgres_reserve_port = None __class__.sm_prev_testgres_release_port = None + @staticmethod def _proxy__reserve_port(): assert type(__class__.sm_DummyPortMaxUsage) == int # noqa: E721 assert type(__class__.sm_DummyPortTotalUsage) == int # noqa: E721 @@ -1144,6 +1145,7 @@ def _proxy__reserve_port(): __class__.sm_DummyPortCurrentUsage += 1 return __class__.sm_DummyPortNumber + @staticmethod def _proxy__release_port(dummyPortNumber): assert type(dummyPortNumber) == int # noqa: E721 From 8824946283a305c0ee2b997aeb513efda8a073e7 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Sun, 23 Feb 2025 23:18:57 +0300 Subject: [PATCH 392/475] TestgresRemoteTests::helper__restore_envvar is marked with @staticmethod --- tests/test_simple_remote.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_simple_remote.py b/tests/test_simple_remote.py index 2b581ac9..9d12e618 100755 --- a/tests/test_simple_remote.py +++ b/tests/test_simple_remote.py @@ -1067,6 +1067,7 @@ def test_child_process_dies(self): # try to handle children list -- missing processes will have ptype "ProcessType.Unknown" [ProcessProxy(p) for p in children] + @staticmethod def helper__restore_envvar(name, prev_value): if prev_value is None: os.environ.pop(name, None) From fe03c24d35b8713f01df4d015859d1d099dfcd58 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Mon, 24 Feb 2025 13:49:43 +0300 Subject: [PATCH 393/475] TestgresRemoteTests.test_init__unk_LANG_and_LC_CTYPE is corrected Vanilla PG18 returns "...PostgreSQL 18devel" --- tests/test_simple_remote.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_simple_remote.py b/tests/test_simple_remote.py index 9d12e618..8b44623a 100755 --- a/tests/test_simple_remote.py +++ b/tests/test_simple_remote.py @@ -183,7 +183,7 @@ def test_init__unk_LANG_and_LC_CTYPE(self): assert "LC_CTYPE" in errMsg assert unkData[1] in errMsg assert "warning: setlocale: LC_CTYPE: cannot change locale (" + unkData[1] + "): No such file or directory" in errMsg - assert "postgres" in errMsg + assert ("postgres" in errMsg) or ("PostgreSQL" in errMsg) break raise Exception("We expected an error!") finally: From 50fc4c5fecc9aa722dbc660762a9d120f12e92fa Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Mon, 24 Feb 2025 14:12:50 +0300 Subject: [PATCH 394/475] [BUG FIX] PostgresNode::start is corrected Incorrect code to build a warning message. --- testgres/node.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index 8a712753..512650c1 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -841,8 +841,7 @@ def LOCAL__raise_cannot_start_node__std(from_exception): log_files0 = log_files1 logging.warning( - "Detected a conflict with using the port {0}. " - "Trying another port after a {1}-second sleep...".format(self.port, timeout) + "Detected a conflict with using the port {0}. Trying another port after a {1}-second sleep...".format(self.port, timeout) ) time.sleep(timeout) timeout = min(2 * timeout, 5) From 2b34236e9f8677cd20b75b37840841dbe0968b09 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Mon, 24 Feb 2025 15:03:23 +0300 Subject: [PATCH 395/475] execute_utility2, get_bin_path2, get_pg_config2 are added This the functions with explicit os_ops argument. testgres/utils.py - [add] def execute_utility2(os_ops: OsOperations, args, logfile=None, verbose=False) - [add] def get_bin_path2(os_ops: OsOperations, filename) - [add] def get_pg_config2(os_ops: OsOperations, pg_config_path): ATTENTION get_pg_config does not change tconf.os_ops now testgres/cache.py - cached_initdb - [add] make_utility_path - it is used for pg_resetwal, too. --- testgres/backup.py | 13 +++++++++---- testgres/cache.py | 24 ++++++++++++++++++------ testgres/node.py | 30 +++++++++++++++--------------- testgres/utils.py | 46 +++++++++++++++++++++++++++++++++++----------- 4 files changed, 77 insertions(+), 36 deletions(-) diff --git a/testgres/backup.py b/testgres/backup.py index cecb0f7b..619c0270 100644 --- a/testgres/backup.py +++ b/testgres/backup.py @@ -15,9 +15,11 @@ from .exceptions import BackupException +from .operations.os_ops import OsOperations + from .utils import \ - get_bin_path, \ - execute_utility, \ + get_bin_path2, \ + execute_utility2, \ clean_on_error @@ -44,6 +46,9 @@ def __init__(self, username: database user name. xlog_method: none | fetch | stream (see docs) """ + assert node.os_ops is not None + assert isinstance(node.os_ops, OsOperations) + if not options: options = [] self.os_ops = node.os_ops @@ -73,7 +78,7 @@ def __init__(self, data_dir = os.path.join(self.base_dir, DATA_DIR) _params = [ - get_bin_path("pg_basebackup"), + get_bin_path2(self.os_ops, "pg_basebackup"), "-p", str(node.port), "-h", node.host, "-U", username, @@ -81,7 +86,7 @@ def __init__(self, "-X", xlog_method.value ] # yapf: disable _params += options - execute_utility(_params, self.log_file) + execute_utility2(self.os_ops, _params, self.log_file) def __enter__(self): return self diff --git a/testgres/cache.py b/testgres/cache.py index f17b54b5..61d44868 100644 --- a/testgres/cache.py +++ b/testgres/cache.py @@ -15,8 +15,8 @@ ExecUtilException from .utils import \ - get_bin_path, \ - execute_utility + get_bin_path2, \ + execute_utility2 from .operations.local_ops import LocalOperations from .operations.os_ops import OsOperations @@ -27,11 +27,23 @@ def cached_initdb(data_dir, logfile=None, params=None, os_ops: OsOperations = Lo Perform initdb or use cached node files. """ + assert os_ops is not None + assert isinstance(os_ops, OsOperations) + + def make_utility_path(name): + assert name is not None + assert type(name) == str + + if bin_path: + return os.path.join(bin_path, name) + + return get_bin_path2(os_ops, name) + def call_initdb(initdb_dir, log=logfile): try: - initdb_path = os.path.join(bin_path, 'initdb') if bin_path else get_bin_path("initdb") + initdb_path = make_utility_path("initdb") _params = [initdb_path, "-D", initdb_dir, "-N"] - execute_utility(_params + (params or []), log) + execute_utility2(os_ops, _params + (params or []), log) except ExecUtilException as e: raise_from(InitNodeException("Failed to run initdb"), e) @@ -63,8 +75,8 @@ def call_initdb(initdb_dir, log=logfile): os_ops.write(pg_control, new_pg_control, truncate=True, binary=True, read_and_write=True) # XXX: build new WAL segment with our system id - _params = [get_bin_path("pg_resetwal"), "-D", data_dir, "-f"] - execute_utility(_params, logfile) + _params = [make_utility_path("pg_resetwal"), "-D", data_dir, "-f"] + execute_utility2(os_ops, _params, logfile) except ExecUtilException as e: msg = "Failed to reset WAL for system id" diff --git a/testgres/node.py b/testgres/node.py index 512650c1..56899b90 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -89,9 +89,9 @@ from .utils import \ PgVer, \ eprint, \ - get_bin_path, \ + get_bin_path2, \ get_pg_version, \ - execute_utility, \ + execute_utility2, \ options_string, \ clean_on_error @@ -301,7 +301,7 @@ def base_dir(self): @property def bin_dir(self): if not self._bin_dir: - self._bin_dir = os.path.dirname(get_bin_path("pg_config")) + self._bin_dir = os.path.dirname(get_bin_path2(self.os_ops, "pg_config")) return self._bin_dir @property @@ -684,7 +684,7 @@ def status(self): "-D", self.data_dir, "status" ] # yapf: disable - status_code, out, error = execute_utility(_params, self.utils_log_file, verbose=True) + status_code, out, error = execute_utility2(self.os_ops, _params, self.utils_log_file, verbose=True) if error and 'does not exist' in error: return NodeStatus.Uninitialized elif 'no server running' in out: @@ -710,7 +710,7 @@ def get_control_data(self): _params += ["-D"] if self._pg_version >= PgVer('9.5') else [] _params += [self.data_dir] - data = execute_utility(_params, self.utils_log_file) + data = execute_utility2(self.os_ops, _params, self.utils_log_file) out_dict = {} @@ -793,7 +793,7 @@ def start(self, params=[], wait=True): def LOCAL__start_node(): # 'error' will be None on Windows - _, _, error = execute_utility(_params, self.utils_log_file, verbose=True) + _, _, error = execute_utility2(self.os_ops, _params, self.utils_log_file, verbose=True) assert error is None or type(error) == str # noqa: E721 if error and 'does not exist' in error: raise Exception(error) @@ -882,7 +882,7 @@ def stop(self, params=[], wait=True): "stop" ] + params # yapf: disable - execute_utility(_params, self.utils_log_file) + execute_utility2(self.os_ops, _params, self.utils_log_file) self._maybe_stop_logger() self.is_started = False @@ -924,7 +924,7 @@ def restart(self, params=[]): ] + params # yapf: disable try: - error_code, out, error = execute_utility(_params, self.utils_log_file, verbose=True) + error_code, out, error = execute_utility2(self.os_ops, _params, self.utils_log_file, verbose=True) if error and 'could not start server' in error: raise ExecUtilException except ExecUtilException as e: @@ -953,7 +953,7 @@ def reload(self, params=[]): "reload" ] + params # yapf: disable - execute_utility(_params, self.utils_log_file) + execute_utility2(self.os_ops, _params, self.utils_log_file) return self @@ -975,7 +975,7 @@ def promote(self, dbname=None, username=None): "promote" ] # yapf: disable - execute_utility(_params, self.utils_log_file) + execute_utility2(self.os_ops, _params, self.utils_log_file) # for versions below 10 `promote` is asynchronous so we need to wait # until it actually becomes writable @@ -1010,7 +1010,7 @@ def pg_ctl(self, params): "-w" # wait ] + params # yapf: disable - return execute_utility(_params, self.utils_log_file) + return execute_utility2(self.os_ops, _params, self.utils_log_file) def free_port(self): """ @@ -1230,7 +1230,7 @@ def tmpfile(): "-F", format.value ] # yapf: disable - execute_utility(_params, self.utils_log_file) + execute_utility2(self.os_ops, _params, self.utils_log_file) return filename @@ -1259,7 +1259,7 @@ def restore(self, filename, dbname=None, username=None): # try pg_restore if dump is binary format, and psql if not try: - execute_utility(_params, self.utils_log_name) + execute_utility2(self.os_ops, _params, self.utils_log_name) except ExecUtilException: self.psql(filename=filename, dbname=dbname, username=username) @@ -1612,7 +1612,7 @@ def pgbench_run(self, dbname=None, username=None, options=[], **kwargs): # should be the last one _params.append(dbname) - return execute_utility(_params, self.utils_log_file) + return execute_utility2(self.os_ops, _params, self.utils_log_file) def connect(self, dbname=None, @@ -1809,7 +1809,7 @@ def _get_bin_path(self, filename): if self.bin_dir: bin_path = os.path.join(self.bin_dir, filename) else: - bin_path = get_bin_path(filename) + bin_path = get_bin_path2(self.os_ops, filename) return bin_path def _escape_config_value(value): diff --git a/testgres/utils.py b/testgres/utils.py index 4bd232b1..9645fc3b 100644 --- a/testgres/utils.py +++ b/testgres/utils.py @@ -16,6 +16,8 @@ from .helpers.port_manager import PortManager from .exceptions import ExecUtilException from .config import testgres_config as tconf +from .operations.os_ops import OsOperations +from .operations.remote_ops import RemoteOperations # rows returned by PG_CONFIG _pg_config_data = {} @@ -68,7 +70,14 @@ def execute_utility(args, logfile=None, verbose=False): Returns: stdout of executed utility. """ - exit_status, out, error = tconf.os_ops.exec_command(args, verbose=True) + return execute_utility2(tconf.os_ops, args, logfile, verbose) + + +def execute_utility2(os_ops: OsOperations, args, logfile=None, verbose=False): + assert os_ops is not None + assert isinstance(os_ops, OsOperations) + + exit_status, out, error = os_ops.exec_command(args, verbose=True) # decode result out = '' if not out else out if isinstance(out, bytes): @@ -79,11 +88,11 @@ def execute_utility(args, logfile=None, verbose=False): # write new log entry if possible if logfile: try: - tconf.os_ops.write(filename=logfile, data=args, truncate=True) + os_ops.write(filename=logfile, data=args, truncate=True) if out: # comment-out lines lines = [u'\n'] + ['# ' + line for line in out.splitlines()] + [u'\n'] - tconf.os_ops.write(filename=logfile, data=lines) + os_ops.write(filename=logfile, data=lines) except IOError: raise ExecUtilException( "Problem with writing to logfile `{}` during run command `{}`".format(logfile, args)) @@ -98,25 +107,32 @@ def get_bin_path(filename): Return absolute path to an executable using PG_BIN or PG_CONFIG. This function does nothing if 'filename' is already absolute. """ + return get_bin_path2(tconf.os_ops, filename) + + +def get_bin_path2(os_ops: OsOperations, filename): + assert os_ops is not None + assert isinstance(os_ops, OsOperations) + # check if it's already absolute if os.path.isabs(filename): return filename - if tconf.os_ops.remote: + if isinstance(os_ops, RemoteOperations): pg_config = os.environ.get("PG_CONFIG_REMOTE") or os.environ.get("PG_CONFIG") else: # try PG_CONFIG - get from local machine pg_config = os.environ.get("PG_CONFIG") if pg_config: - bindir = get_pg_config()["BINDIR"] + bindir = get_pg_config(pg_config, os_ops)["BINDIR"] return os.path.join(bindir, filename) # try PG_BIN - pg_bin = tconf.os_ops.environ("PG_BIN") + pg_bin = os_ops.environ("PG_BIN") if pg_bin: return os.path.join(pg_bin, filename) - pg_config_path = tconf.os_ops.find_executable('pg_config') + pg_config_path = os_ops.find_executable('pg_config') if pg_config_path: bindir = get_pg_config(pg_config_path)["BINDIR"] return os.path.join(bindir, filename) @@ -129,12 +145,20 @@ def get_pg_config(pg_config_path=None, os_ops=None): Return output of pg_config (provided that it is installed). NOTE: this function caches the result by default (see GlobalConfig). """ - if os_ops: - tconf.os_ops = os_ops + + if os_ops is None: + os_ops = tconf.os_ops + + return get_pg_config2(os_ops, pg_config_path) + + +def get_pg_config2(os_ops: OsOperations, pg_config_path): + assert os_ops is not None + assert isinstance(os_ops, OsOperations) def cache_pg_config_data(cmd): # execute pg_config and get the output - out = tconf.os_ops.exec_command(cmd, encoding='utf-8') + out = os_ops.exec_command(cmd, encoding='utf-8') data = {} for line in out.splitlines(): @@ -158,7 +182,7 @@ def cache_pg_config_data(cmd): return _pg_config_data # try specified pg_config path or PG_CONFIG - if tconf.os_ops.remote: + if isinstance(os_ops, RemoteOperations): pg_config = pg_config_path or os.environ.get("PG_CONFIG_REMOTE") or os.environ.get("PG_CONFIG") else: # try PG_CONFIG - get from local machine From ed3ef60be4c09b135d86737755c2d4cbb7c5657f Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Mon, 24 Feb 2025 16:38:42 +0300 Subject: [PATCH 396/475] Code style (flake8) --- testgres/cache.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testgres/cache.py b/testgres/cache.py index 61d44868..3ac63326 100644 --- a/testgres/cache.py +++ b/testgres/cache.py @@ -32,11 +32,11 @@ def cached_initdb(data_dir, logfile=None, params=None, os_ops: OsOperations = Lo def make_utility_path(name): assert name is not None - assert type(name) == str + assert type(name) == str # noqa: E721 if bin_path: return os.path.join(bin_path, name) - + return get_bin_path2(os_ops, name) def call_initdb(initdb_dir, log=logfile): From 0eeb705151b23b7dabb0b677ff9371279389a6cf Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Wed, 26 Feb 2025 11:17:29 +0300 Subject: [PATCH 397/475] Tests are based on pytest (#192) * Using pytest [pytest.raises] * Using pytest [pytest.skip] * Using pytest [assertIsNotNone] * Using pytest [assertFalse] * Using pytest [assertTrue] * Using pytest [assertEqual] * Using pytest [assertNotEqual] * Using pytest [assertGreaterEqual] * Using pytest [assertGreater] * Using pytest [assertIn] * Using pytest [assertListEqual] * unittest is not used * Code style (flake8) * Execution signature is removed * run_tests.sh installs pytest * run_tests.sh is updated run tests through pytest explicitly * Total refactoring of tests - TestgresRemoteTests does not use global variables and code - Explicit work with ..testgres folder * Code style (flake8) * Root __init__.py is added It is required for tests. * Code style (flake8) * pytest.ini is added * TestgresTests::test_ports_management is corrected Let's send warning about a garbage in the container "bound_ports" and continue working. * coding: utf-8 * Cleanup * CI runs all the tests of testgres. * Add install ssh (cherry picked from commit fec1e7ac9d6e2bfb43a01f0e370336ba5ed8e971) * Revert "Add install ssh" This reverts commit 537a9acb9dfb26d82251d2d68796a55989be8317. * Revert "CI runs all the tests of testgres." This reverts commit 2d2532c77e8d7521552c0f3511c119e90d55573e. * Test of probackup plugin is restored It works now (was runned with a fresh probackup2 and vanilla 18devel). * The test suite of a probackup plugin is based on pytest * Probackup plugin is updated Probackup plugin tests - They are skipped if PGPROBACKUPBIN is not defined Global variable init_params is None when PGPROBACKUPBIN is not defined or version is not processed * CI test use 4 cores * testgres.plugins.probackup2.Init was restored [thanks to Yuri Sokolov] * pytest.ini is updated [testpaths] Enumeration of all the known folders with tests. * test_child_pids (local, remote) is updated Multiple attempts and logging are added. * test_child_process_dies is updated Multiple attempts are added. --------- Co-authored-by: vshepard --- __init__.py | 0 pytest.ini | 9 + run_tests.sh | 12 +- testgres/plugins/__init__.py | 8 +- .../pg_probackup2/init_helpers.py | 13 +- .../pg_probackup2/tests/__init__.py | 0 .../pg_probackup2/tests/basic_test.py | 80 --- .../pg_probackup2/tests/test_basic.py | 95 +++ tests/helpers/run_conditions.py | 1 + tests/test_local.py | 7 +- tests/test_remote.py | 9 +- tests/test_simple.py | 607 +++++++++-------- tests/test_simple_remote.py | 615 ++++++++++-------- 13 files changed, 839 insertions(+), 617 deletions(-) create mode 100644 __init__.py create mode 100644 pytest.ini create mode 100644 testgres/plugins/pg_probackup2/pg_probackup2/tests/__init__.py delete mode 100644 testgres/plugins/pg_probackup2/pg_probackup2/tests/basic_test.py create mode 100644 testgres/plugins/pg_probackup2/pg_probackup2/tests/test_basic.py diff --git a/__init__.py b/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 00000000..c94eabc2 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,9 @@ +[pytest] +testpaths = ["./tests", "./testgres/plugins/pg_probackup2/pg_probackup2/tests"] +addopts = --strict-markers +markers = +#log_file = logs/pytest.log +log_file_level = NOTSET +log_file_format = %(levelname)8s [%(asctime)s] %(message)s +log_file_date_format=%Y-%m-%d %H:%M:%S + diff --git a/run_tests.sh b/run_tests.sh index 73c459be..e9d58b54 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -22,11 +22,11 @@ export VIRTUAL_ENV_DISABLE_PROMPT=1 source $VENV_PATH/bin/activate # install utilities -$PIP install coverage flake8 psutil Sphinx +$PIP install coverage flake8 psutil Sphinx pytest pytest-xdist psycopg2 six psutil # install testgres' dependencies export PYTHONPATH=$(pwd) -$PIP install . +# $PIP install . # test code quality flake8 . @@ -38,21 +38,19 @@ rm -f $COVERAGE_FILE # run tests (PATH) -time coverage run -a tests/test_simple.py +time coverage run -a -m pytest -l -v -n 4 -k "TestgresTests" # run tests (PG_BIN) time \ PG_BIN=$(dirname $(which pg_config)) \ - ALT_CONFIG=1 \ - coverage run -a tests/test_simple.py + coverage run -a -m pytest -l -v -n 4 -k "TestgresTests" # run tests (PG_CONFIG) time \ PG_CONFIG=$(which pg_config) \ - ALT_CONFIG=1 \ - coverage run -a tests/test_simple.py + coverage run -a -m pytest -l -v -n 4 -k "TestgresTests" # show coverage diff --git a/testgres/plugins/__init__.py b/testgres/plugins/__init__.py index 8c19a23b..824eadc6 100644 --- a/testgres/plugins/__init__.py +++ b/testgres/plugins/__init__.py @@ -1,7 +1,7 @@ -from pg_probackup2.gdb import GDBobj -from pg_probackup2.app import ProbackupApp, ProbackupException -from pg_probackup2.init_helpers import init_params -from pg_probackup2.storage.fs_backup import FSTestBackupDir +from .pg_probackup2.pg_probackup2.gdb import GDBobj +from .pg_probackup2.pg_probackup2.app import ProbackupApp, ProbackupException +from .pg_probackup2.pg_probackup2.init_helpers import init_params +from .pg_probackup2.pg_probackup2.storage.fs_backup import FSTestBackupDir __all__ = [ "ProbackupApp", "ProbackupException", "init_params", "FSTestBackupDir", "GDBobj" diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py index 078fdbab..c4570a39 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py @@ -121,8 +121,7 @@ def __init__(self): self.probackup_path = probackup_path_tmp if not self.probackup_path: - logging.error('pg_probackup binary is not found') - exit(1) + raise Exception('pg_probackup binary is not found') if os.name == 'posix': self.EXTERNAL_DIRECTORY_DELIMITER = ':' @@ -213,11 +212,15 @@ def __init__(self): if self.probackup_version.split('.')[0].isdigit(): self.major_version = int(self.probackup_version.split('.')[0]) else: - logging.error('Can\'t process pg_probackup version \"{}\": the major version is expected to be a number'.format(self.probackup_version)) - sys.exit(1) + raise Exception('Can\'t process pg_probackup version \"{}\": the major version is expected to be a number'.format(self.probackup_version)) def test_env(self): return self._test_env.copy() -init_params = Init() +try: + init_params = Init() +except Exception as e: + logging.error(str(e)) + logging.warning("testgres.plugins.probackup2.init_params is set to None.") + init_params = None diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/tests/__init__.py b/testgres/plugins/pg_probackup2/pg_probackup2/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/tests/basic_test.py b/testgres/plugins/pg_probackup2/pg_probackup2/tests/basic_test.py deleted file mode 100644 index b63531ec..00000000 --- a/testgres/plugins/pg_probackup2/pg_probackup2/tests/basic_test.py +++ /dev/null @@ -1,80 +0,0 @@ -import logging -import os -import shutil -import unittest -import testgres -from pg_probackup2.app import ProbackupApp -from pg_probackup2.init_helpers import Init, init_params -from pg_probackup2.app import build_backup_dir - - -class TestUtils: - @staticmethod - def get_module_and_function_name(test_id): - try: - module_name = test_id.split('.')[-2] - fname = test_id.split('.')[-1] - except IndexError: - logging.warning(f"Couldn't get module name and function name from test_id: `{test_id}`") - module_name, fname = test_id.split('(')[1].split('.')[1], test_id.split('(')[0] - return module_name, fname - - -class ProbackupTest(unittest.TestCase): - def setUp(self): - self.setup_test_environment() - self.setup_test_paths() - self.setup_backup_dir() - self.setup_probackup() - - def setup_test_environment(self): - self.output = None - self.cmd = None - self.nodes_to_cleanup = [] - self.module_name, self.fname = TestUtils.get_module_and_function_name(self.id()) - self.test_env = Init().test_env() - - def setup_test_paths(self): - self.rel_path = os.path.join(self.module_name, self.fname) - self.test_path = os.path.join(init_params.tmp_path, self.rel_path) - os.makedirs(self.test_path) - self.pb_log_path = os.path.join(self.test_path, "pb_log") - - def setup_backup_dir(self): - self.backup_dir = build_backup_dir(self, 'backup') - self.backup_dir.cleanup() - - def setup_probackup(self): - self.pg_node = testgres.NodeApp(self.test_path, self.nodes_to_cleanup) - self.pb = ProbackupApp(self, self.pg_node, self.pb_log_path, self.test_env, - auto_compress_alg='zlib', backup_dir=self.backup_dir) - - def tearDown(self): - if os.path.exists(self.test_path): - shutil.rmtree(self.test_path) - - -class BasicTest(ProbackupTest): - def test_full_backup(self): - # Setting up a simple test node - node = self.pg_node.make_simple('node', pg_options={"fsync": "off", "synchronous_commit": "off"}) - - # Initialize and configure Probackup - self.pb.init() - self.pb.add_instance('node', node) - self.pb.set_archiving('node', node) - - # Start the node and initialize pgbench - node.slow_start() - node.pgbench_init(scale=100, no_vacuum=True) - - # Perform backup and validation - backup_id = self.pb.backup_node('node', node) - out = self.pb.validate('node', backup_id) - - # Check if the backup is valid - self.assertIn(f"INFO: Backup {backup_id} is valid", out) - - -if __name__ == "__main__": - unittest.main() diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/tests/test_basic.py b/testgres/plugins/pg_probackup2/pg_probackup2/tests/test_basic.py new file mode 100644 index 00000000..ba788623 --- /dev/null +++ b/testgres/plugins/pg_probackup2/pg_probackup2/tests/test_basic.py @@ -0,0 +1,95 @@ +from __future__ import annotations + +import os +import shutil +import pytest + +from ...... import testgres +from ...pg_probackup2.app import ProbackupApp +from ...pg_probackup2.init_helpers import Init, init_params +from ..storage.fs_backup import FSTestBackupDir + + +class ProbackupTest: + pg_node: testgres.PostgresNode + + @staticmethod + def probackup_is_available() -> bool: + p = os.environ.get("PGPROBACKUPBIN") + + if p is None: + return False + + if not os.path.exists(p): + return False + + return True + + @pytest.fixture(autouse=True, scope="function") + def implicit_fixture(self, request: pytest.FixtureRequest): + assert isinstance(request, pytest.FixtureRequest) + self.helper__setUp(request) + yield + self.helper__tearDown() + + def helper__setUp(self, request: pytest.FixtureRequest): + assert isinstance(request, pytest.FixtureRequest) + + self.helper__setup_test_environment(request) + self.helper__setup_test_paths() + self.helper__setup_backup_dir() + self.helper__setup_probackup() + + def helper__setup_test_environment(self, request: pytest.FixtureRequest): + assert isinstance(request, pytest.FixtureRequest) + + self.output = None + self.cmd = None + self.nodes_to_cleanup = [] + self.module_name, self.fname = request.node.cls.__name__, request.node.name + self.test_env = Init().test_env() + + def helper__setup_test_paths(self): + self.rel_path = os.path.join(self.module_name, self.fname) + self.test_path = os.path.join(init_params.tmp_path, self.rel_path) + os.makedirs(self.test_path, exist_ok=True) + self.pb_log_path = os.path.join(self.test_path, "pb_log") + + def helper__setup_backup_dir(self): + self.backup_dir = self.helper__build_backup_dir('backup') + self.backup_dir.cleanup() + + def helper__setup_probackup(self): + self.pg_node = testgres.NodeApp(self.test_path, self.nodes_to_cleanup) + self.pb = ProbackupApp(self, self.pg_node, self.pb_log_path, self.test_env, + auto_compress_alg='zlib', backup_dir=self.backup_dir) + + def helper__tearDown(self): + if os.path.exists(self.test_path): + shutil.rmtree(self.test_path) + + def helper__build_backup_dir(self, backup='backup'): + return FSTestBackupDir(rel_path=self.rel_path, backup=backup) + + +@pytest.mark.skipif(not ProbackupTest.probackup_is_available(), reason="Check that PGPROBACKUPBIN is defined and is valid.") +class TestBasic(ProbackupTest): + def test_full_backup(self): + # Setting up a simple test node + node = self.pg_node.make_simple('node', pg_options={"fsync": "off", "synchronous_commit": "off"}) + + # Initialize and configure Probackup + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) + + # Start the node and initialize pgbench + node.slow_start() + node.pgbench_init(scale=100, no_vacuum=True) + + # Perform backup and validation + backup_id = self.pb.backup_node('node', node) + out = self.pb.validate('node', backup_id) + + # Check if the backup is valid + assert f"INFO: Backup {backup_id} is valid" in out diff --git a/tests/helpers/run_conditions.py b/tests/helpers/run_conditions.py index 8d57f753..11357c30 100644 --- a/tests/helpers/run_conditions.py +++ b/tests/helpers/run_conditions.py @@ -1,3 +1,4 @@ +# coding: utf-8 import pytest import platform diff --git a/tests/test_local.py b/tests/test_local.py index 4051bfb5..60a96c18 100644 --- a/tests/test_local.py +++ b/tests/test_local.py @@ -1,12 +1,13 @@ +# coding: utf-8 import os import pytest import re import tempfile -from testgres import ExecUtilException -from testgres import InvalidOperationException -from testgres import LocalOperations +from ..testgres import ExecUtilException +from ..testgres import InvalidOperationException +from ..testgres import LocalOperations from .helpers.run_conditions import RunConditions diff --git a/tests/test_remote.py b/tests/test_remote.py index 4330b92f..8b167e9f 100755 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -1,13 +1,14 @@ +# coding: utf-8 import os import pytest import re import tempfile -from testgres import ExecUtilException -from testgres import InvalidOperationException -from testgres import RemoteOperations -from testgres import ConnectionParams +from ..testgres import ExecUtilException +from ..testgres import InvalidOperationException +from ..testgres import RemoteOperations +from ..testgres import ConnectionParams class TestRemoteOperations: diff --git a/tests/test_simple.py b/tests/test_simple.py index a751f0a3..6c433cd4 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -1,22 +1,22 @@ -#!/usr/bin/env python # coding: utf-8 - import os import re import subprocess import tempfile -import testgres import time import six -import unittest +import pytest import psutil +import platform import logging.config from contextlib import contextmanager from shutil import rmtree -from testgres import \ +from .. import testgres + +from ..testgres import \ InitNodeException, \ StartNodeException, \ ExecUtilException, \ @@ -27,31 +27,32 @@ InvalidOperationException, \ NodeApp -from testgres import \ +from ..testgres import \ TestgresConfig, \ configure_testgres, \ scoped_config, \ pop_config -from testgres import \ +from ..testgres import \ NodeStatus, \ ProcessType, \ IsolationLevel, \ get_new_node -from testgres import \ +from ..testgres import \ get_bin_path, \ get_pg_config, \ get_pg_version -from testgres import \ +from ..testgres import \ First, \ Any # NOTE: those are ugly imports -from testgres import bound_ports -from testgres.utils import PgVer, parse_pg_version -from testgres.node import ProcessProxy +from ..testgres import bound_ports +from ..testgres.utils import PgVer, parse_pg_version +from ..testgres.utils import file_tail +from ..testgres.node import ProcessProxy def pg_version_ge(version): @@ -105,11 +106,11 @@ def removing(f): rmtree(f, ignore_errors=True) -class TestgresTests(unittest.TestCase): +class TestgresTests: def test_node_repr(self): with get_new_node() as node: pattern = r"PostgresNode\(name='.+', port=.+, base_dir='.+'\)" - self.assertIsNotNone(re.match(pattern, str(node))) + assert re.match(pattern, str(node)) is not None def test_custom_init(self): with get_new_node() as node: @@ -126,15 +127,15 @@ def test_custom_init(self): lines = conf.readlines() # check number of lines - self.assertGreaterEqual(len(lines), 6) + assert (len(lines) >= 6) # there should be no trust entries at all - self.assertFalse(any('trust' in s for s in lines)) + assert not (any('trust' in s for s in lines)) def test_double_init(self): with get_new_node().init() as node: # can't initialize node more than once - with self.assertRaises(InitNodeException): + with pytest.raises(expected_exception=InitNodeException): node.init() def test_init_after_cleanup(self): @@ -143,10 +144,11 @@ def test_init_after_cleanup(self): node.cleanup() node.init().start().execute('select 1') - @unittest.skipUnless(util_exists('pg_resetwal.exe' if os.name == 'nt' else 'pg_resetwal'), 'pgbench might be missing') - @unittest.skipUnless(pg_version_ge('9.6'), 'requires 9.6+') def test_init_unique_system_id(self): # this function exists in PostgreSQL 9.6+ + __class__.helper__skip_test_if_util_not_exist("pg_resetwal") + __class__.helper__skip_test_if_pg_version_is_not_ge("9.6") + query = 'select system_identifier from pg_control_system()' with scoped_config(cache_initdb=False): @@ -155,8 +157,8 @@ def test_init_unique_system_id(self): with scoped_config(cache_initdb=True, cached_initdb_unique=True) as config: - self.assertTrue(config.cache_initdb) - self.assertTrue(config.cached_initdb_unique) + assert (config.cache_initdb) + assert (config.cached_initdb_unique) # spawn two nodes; ids must be different with get_new_node().init().start() as node1, \ @@ -166,37 +168,37 @@ def test_init_unique_system_id(self): id2 = node2.execute(query)[0] # ids must increase - self.assertGreater(id1, id0) - self.assertGreater(id2, id1) + assert (id1 > id0) + assert (id2 > id1) def test_node_exit(self): base_dir = None - with self.assertRaises(QueryException): + with pytest.raises(expected_exception=QueryException): with get_new_node().init() as node: base_dir = node.base_dir node.safe_psql('select 1') # we should save the DB for "debugging" - self.assertTrue(os.path.exists(base_dir)) + assert (os.path.exists(base_dir)) rmtree(base_dir, ignore_errors=True) with get_new_node().init() as node: base_dir = node.base_dir # should have been removed by default - self.assertFalse(os.path.exists(base_dir)) + assert not (os.path.exists(base_dir)) def test_double_start(self): with get_new_node().init().start() as node: # can't start node more than once node.start() - self.assertTrue(node.is_started) + assert (node.is_started) def test_uninitialized_start(self): with get_new_node() as node: # node is not initialized yet - with self.assertRaises(StartNodeException): + with pytest.raises(expected_exception=StartNodeException): node.start() def test_restart(self): @@ -205,13 +207,13 @@ def test_restart(self): # restart, ok res = node.execute('select 1') - self.assertEqual(res, [(1, )]) + assert (res == [(1, )]) node.restart() res = node.execute('select 2') - self.assertEqual(res, [(2, )]) + assert (res == [(2, )]) # restart, fail - with self.assertRaises(StartNodeException): + with pytest.raises(expected_exception=StartNodeException): node.append_conf('pg_hba.conf', 'DUMMY') node.restart() @@ -228,106 +230,107 @@ def test_reload(self): # check new value cmm_new = node.execute('show client_min_messages') - self.assertEqual('debug1', cmm_new[0][0].lower()) - self.assertNotEqual(cmm_old, cmm_new) + assert ('debug1' == cmm_new[0][0].lower()) + assert (cmm_old != cmm_new) def test_pg_ctl(self): with get_new_node() as node: node.init().start() status = node.pg_ctl(['status']) - self.assertTrue('PID' in status) + assert ('PID' in status) def test_status(self): - self.assertTrue(NodeStatus.Running) - self.assertFalse(NodeStatus.Stopped) - self.assertFalse(NodeStatus.Uninitialized) + assert (NodeStatus.Running) + assert not (NodeStatus.Stopped) + assert not (NodeStatus.Uninitialized) # check statuses after each operation with get_new_node() as node: - self.assertEqual(node.pid, 0) - self.assertEqual(node.status(), NodeStatus.Uninitialized) + assert (node.pid == 0) + assert (node.status() == NodeStatus.Uninitialized) node.init() - self.assertEqual(node.pid, 0) - self.assertEqual(node.status(), NodeStatus.Stopped) + assert (node.pid == 0) + assert (node.status() == NodeStatus.Stopped) node.start() - self.assertNotEqual(node.pid, 0) - self.assertEqual(node.status(), NodeStatus.Running) + assert (node.pid != 0) + assert (node.status() == NodeStatus.Running) node.stop() - self.assertEqual(node.pid, 0) - self.assertEqual(node.status(), NodeStatus.Stopped) + assert (node.pid == 0) + assert (node.status() == NodeStatus.Stopped) node.cleanup() - self.assertEqual(node.pid, 0) - self.assertEqual(node.status(), NodeStatus.Uninitialized) + assert (node.pid == 0) + assert (node.status() == NodeStatus.Uninitialized) def test_psql(self): with get_new_node().init().start() as node: # check returned values (1 arg) res = node.psql('select 1') - self.assertEqual(rm_carriage_returns(res), (0, b'1\n', b'')) + assert (rm_carriage_returns(res) == (0, b'1\n', b'')) # check returned values (2 args) res = node.psql('postgres', 'select 2') - self.assertEqual(rm_carriage_returns(res), (0, b'2\n', b'')) + assert (rm_carriage_returns(res) == (0, b'2\n', b'')) # check returned values (named) res = node.psql(query='select 3', dbname='postgres') - self.assertEqual(rm_carriage_returns(res), (0, b'3\n', b'')) + assert (rm_carriage_returns(res) == (0, b'3\n', b'')) # check returned values (1 arg) res = node.safe_psql('select 4') - self.assertEqual(rm_carriage_returns(res), b'4\n') + assert (rm_carriage_returns(res) == b'4\n') # check returned values (2 args) res = node.safe_psql('postgres', 'select 5') - self.assertEqual(rm_carriage_returns(res), b'5\n') + assert (rm_carriage_returns(res) == b'5\n') # check returned values (named) res = node.safe_psql(query='select 6', dbname='postgres') - self.assertEqual(rm_carriage_returns(res), b'6\n') + assert (rm_carriage_returns(res) == b'6\n') # check feeding input node.safe_psql('create table horns (w int)') node.safe_psql('copy horns from stdin (format csv)', input=b"1\n2\n3\n\\.\n") _sum = node.safe_psql('select sum(w) from horns') - self.assertEqual(rm_carriage_returns(_sum), b'6\n') + assert (rm_carriage_returns(_sum) == b'6\n') # check psql's default args, fails - with self.assertRaises(QueryException): + with pytest.raises(expected_exception=QueryException): node.psql() node.stop() # check psql on stopped node, fails - with self.assertRaises(QueryException): + with pytest.raises(expected_exception=QueryException): node.safe_psql('select 1') def test_safe_psql__expect_error(self): with get_new_node().init().start() as node: err = node.safe_psql('select_or_not_select 1', expect_error=True) - self.assertTrue(type(err) == str) # noqa: E721 - self.assertIn('select_or_not_select', err) - self.assertIn('ERROR: syntax error at or near "select_or_not_select"', err) + assert (type(err) == str) # noqa: E721 + assert ('select_or_not_select' in err) + assert ('ERROR: syntax error at or near "select_or_not_select"' in err) # --------- - with self.assertRaises(InvalidOperationException) as ctx: + with pytest.raises( + expected_exception=InvalidOperationException, + match="^" + re.escape("Exception was expected, but query finished successfully: `select 1;`.") + "$" + ): node.safe_psql("select 1;", expect_error=True) - self.assertEqual(str(ctx.exception), "Exception was expected, but query finished successfully: `select 1;`.") - # --------- res = node.safe_psql("select 1;", expect_error=False) - self.assertEqual(rm_carriage_returns(res), b'1\n') + assert (rm_carriage_returns(res) == b'1\n') def test_transactions(self): with get_new_node().init().start() as node: @@ -341,12 +344,12 @@ def test_transactions(self): con.begin() con.execute('insert into test values (2)') res = con.execute('select * from test order by val asc') - self.assertListEqual(res, [(1, ), (2, )]) + assert (res == [(1, ), (2, )]) con.rollback() con.begin() res = con.execute('select * from test') - self.assertListEqual(res, [(1, )]) + assert (res == [(1, )]) con.rollback() con.begin() @@ -357,15 +360,15 @@ def test_control_data(self): with get_new_node() as node: # node is not initialized yet - with self.assertRaises(ExecUtilException): + with pytest.raises(expected_exception=ExecUtilException): node.get_control_data() node.init() data = node.get_control_data() # check returned dict - self.assertIsNotNone(data) - self.assertTrue(any('pg_control' in s for s in data.keys())) + assert data is not None + assert (any('pg_control' in s for s in data.keys())) def test_backup_simple(self): with get_new_node() as master: @@ -374,7 +377,7 @@ def test_backup_simple(self): master.init(allow_streaming=True) # node must be running - with self.assertRaises(BackupException): + with pytest.raises(expected_exception=BackupException): master.backup() # it's time to start node @@ -386,7 +389,7 @@ def test_backup_simple(self): with master.backup(xlog_method='stream') as backup: with backup.spawn_primary().start() as slave: res = slave.execute('select * from test order by i asc') - self.assertListEqual(res, [(1, ), (2, ), (3, ), (4, )]) + assert (res == [(1, ), (2, ), (3, ), (4, )]) def test_backup_multiple(self): with get_new_node() as node: @@ -394,12 +397,12 @@ def test_backup_multiple(self): with node.backup(xlog_method='fetch') as backup1, \ node.backup(xlog_method='fetch') as backup2: - self.assertNotEqual(backup1.base_dir, backup2.base_dir) + assert (backup1.base_dir != backup2.base_dir) with node.backup(xlog_method='fetch') as backup: with backup.spawn_primary('node1', destroy=False) as node1, \ backup.spawn_primary('node2', destroy=False) as node2: - self.assertNotEqual(node1.base_dir, node2.base_dir) + assert (node1.base_dir != node2.base_dir) def test_backup_exhaust(self): with get_new_node() as node: @@ -411,15 +414,17 @@ def test_backup_exhaust(self): pass # now let's try to create one more node - with self.assertRaises(BackupException): + with pytest.raises(expected_exception=BackupException): backup.spawn_primary() def test_backup_wrong_xlog_method(self): with get_new_node() as node: node.init(allow_streaming=True).start() - with self.assertRaises(BackupException, - msg='Invalid xlog_method "wrong"'): + with pytest.raises( + expected_exception=BackupException, + match="^" + re.escape('Invalid xlog_method "wrong"') + "$" + ): node.backup(xlog_method='wrong') def test_pg_ctl_wait_option(self): @@ -440,17 +445,18 @@ def test_replicate(self): with node.replicate().start() as replica: res = replica.execute('select 1') - self.assertListEqual(res, [(1, )]) + assert (res == [(1, )]) node.execute('create table test (val int)', commit=True) replica.catchup() res = node.execute('select * from test') - self.assertListEqual(res, []) + assert (res == []) - @unittest.skipUnless(pg_version_ge('9.6'), 'requires 9.6+') def test_synchronous_replication(self): + __class__.helper__skip_test_if_pg_version_is_not_ge("9.6") + with get_new_node() as master: old_version = not pg_version_ge('9.6') @@ -465,12 +471,12 @@ def test_synchronous_replication(self): standby2.start() # check formatting - self.assertEqual( - '1 ("{}", "{}")'.format(standby1.name, standby2.name), - str(First(1, (standby1, standby2)))) # yapf: disable - self.assertEqual( - 'ANY 1 ("{}", "{}")'.format(standby1.name, standby2.name), - str(Any(1, (standby1, standby2)))) # yapf: disable + assert ( + '1 ("{}", "{}")'.format(standby1.name, standby2.name) == str(First(1, (standby1, standby2))) + ) # yapf: disable + assert ( + 'ANY 1 ("{}", "{}")'.format(standby1.name, standby2.name) == str(Any(1, (standby1, standby2))) + ) # yapf: disable # set synchronous_standby_names master.set_synchronous_standbys(First(2, [standby1, standby2])) @@ -488,10 +494,11 @@ def test_synchronous_replication(self): master.safe_psql( 'insert into abc select generate_series(1, 1000000)') res = standby1.safe_psql('select count(*) from abc') - self.assertEqual(rm_carriage_returns(res), b'1000000\n') + assert (rm_carriage_returns(res) == b'1000000\n') - @unittest.skipUnless(pg_version_ge('10'), 'requires 10+') def test_logical_replication(self): + __class__.helper__skip_test_if_pg_version_is_not_ge("10") + with get_new_node() as node1, get_new_node() as node2: node1.init(allow_logical=True) node1.start() @@ -510,7 +517,7 @@ def test_logical_replication(self): # wait until changes apply on subscriber and check them sub.catchup() res = node2.execute('select * from test') - self.assertListEqual(res, [(1, 1), (2, 2)]) + assert (res == [(1, 1), (2, 2)]) # disable and put some new data sub.disable() @@ -520,7 +527,7 @@ def test_logical_replication(self): sub.enable() sub.catchup() res = node2.execute('select * from test') - self.assertListEqual(res, [(1, 1), (2, 2), (3, 3)]) + assert (res == [(1, 1), (2, 2), (3, 3)]) # Add new tables. Since we added "all tables" to publication # (default behaviour of publish() method) we don't need @@ -534,7 +541,7 @@ def test_logical_replication(self): node1.safe_psql('insert into test2 values (\'a\'), (\'b\')') sub.catchup() res = node2.execute('select * from test2') - self.assertListEqual(res, [('a', ), ('b', )]) + assert (res == [('a', ), ('b', )]) # drop subscription sub.drop() @@ -548,20 +555,21 @@ def test_logical_replication(self): node1.safe_psql('insert into test values (4, 4)') sub.catchup() res = node2.execute('select * from test') - self.assertListEqual(res, [(1, 1), (2, 2), (3, 3), (4, 4)]) + assert (res == [(1, 1), (2, 2), (3, 3), (4, 4)]) # explicitly add table - with self.assertRaises(ValueError): + with pytest.raises(expected_exception=ValueError): pub.add_tables([]) # fail pub.add_tables(['test2']) node1.safe_psql('insert into test2 values (\'c\')') sub.catchup() res = node2.execute('select * from test2') - self.assertListEqual(res, [('a', ), ('b', )]) + assert (res == [('a', ), ('b', )]) - @unittest.skipUnless(pg_version_ge('10'), 'requires 10+') def test_logical_catchup(self): """ Runs catchup for 100 times to be sure that it is consistent """ + __class__.helper__skip_test_if_pg_version_is_not_ge("10") + with get_new_node() as node1, get_new_node() as node2: node1.init(allow_logical=True) node1.start() @@ -579,16 +587,14 @@ def test_logical_catchup(self): node1.execute('insert into test values ({0}, {0})'.format(i)) sub.catchup() res = node2.execute('select * from test') - self.assertListEqual(res, [( - i, - i, - )]) + assert (res == [(i, i, )]) node1.execute('delete from test') - @unittest.skipIf(pg_version_ge('10'), 'requires <10') def test_logical_replication_fail(self): + __class__.helper__skip_test_if_pg_version_is_ge("10") + with get_new_node() as node: - with self.assertRaises(InitNodeException): + with pytest.raises(expected_exception=InitNodeException): node.init(allow_logical=True) def test_replication_slots(self): @@ -599,7 +605,7 @@ def test_replication_slots(self): replica.execute('select 1') # cannot create new slot with the same name - with self.assertRaises(TestgresException): + with pytest.raises(expected_exception=TestgresException): node.replicate(slot='slot1') def test_incorrect_catchup(self): @@ -607,7 +613,7 @@ def test_incorrect_catchup(self): node.init(allow_streaming=True).start() # node has no master, can't catch up - with self.assertRaises(TestgresException): + with pytest.raises(expected_exception=TestgresException): node.catchup() def test_promotion(self): @@ -622,7 +628,7 @@ def test_promotion(self): # make standby becomes writable master replica.safe_psql('insert into abc values (1)') res = replica.safe_psql('select * from abc') - self.assertEqual(rm_carriage_returns(res), b'1\n') + assert (rm_carriage_returns(res) == b'1\n') def test_dump(self): query_create = 'create table test as select generate_series(1, 2) as val' @@ -635,20 +641,20 @@ def test_dump(self): with removing(node1.dump(format=format)) as dump: with get_new_node().init().start() as node3: if format == 'directory': - self.assertTrue(os.path.isdir(dump)) + assert (os.path.isdir(dump)) else: - self.assertTrue(os.path.isfile(dump)) + assert (os.path.isfile(dump)) # restore dump node3.restore(filename=dump) res = node3.execute(query_select) - self.assertListEqual(res, [(1, ), (2, )]) + assert (res == [(1, ), (2, )]) def test_users(self): with get_new_node().init().start() as node: node.psql('create role test_user login') value = node.safe_psql('select 1', username='test_user') value = rm_carriage_returns(value) - self.assertEqual(value, b'1\n') + assert (value == b'1\n') def test_poll_query_until(self): with get_new_node() as node: @@ -661,15 +667,15 @@ def test_poll_query_until(self): node.poll_query_until(query=check_time.format(start_time)) end_time = node.execute(get_time)[0][0] - self.assertTrue(end_time - start_time >= 5) + assert (end_time - start_time >= 5) # check 0 columns - with self.assertRaises(QueryException): + with pytest.raises(expected_exception=QueryException): node.poll_query_until( query='select from pg_catalog.pg_class limit 1') # check None, fail - with self.assertRaises(QueryException): + with pytest.raises(expected_exception=QueryException): node.poll_query_until(query='create table abc (val int)') # check None, ok @@ -682,7 +688,7 @@ def test_poll_query_until(self): expected=None) # check arbitrary expected value, fail - with self.assertRaises(TimeoutException): + with pytest.raises(expected_exception=TimeoutException): node.poll_query_until(query='select 3', expected=1, max_attempts=3, @@ -692,17 +698,17 @@ def test_poll_query_until(self): node.poll_query_until(query='select 2', expected=2) # check timeout - with self.assertRaises(TimeoutException): + with pytest.raises(expected_exception=TimeoutException): node.poll_query_until(query='select 1 > 2', max_attempts=3, sleep_time=0.01) # check ProgrammingError, fail - with self.assertRaises(testgres.ProgrammingError): + with pytest.raises(expected_exception=testgres.ProgrammingError): node.poll_query_until(query='dummy1') # check ProgrammingError, ok - with self.assertRaises(TimeoutException): + with pytest.raises(expected_exception=(TimeoutException)): node.poll_query_until(query='dummy2', max_attempts=3, sleep_time=0.01, @@ -754,16 +760,17 @@ def test_logging(self): # check that master's port is found with open(logfile.name, 'r') as log: lines = log.readlines() - self.assertTrue(any(node_name in s for s in lines)) + assert (any(node_name in s for s in lines)) # test logger after stop/start/restart master.stop() master.start() master.restart() - self.assertTrue(master._logger.is_alive()) + assert (master._logger.is_alive()) - @unittest.skipUnless(util_exists('pgbench.exe' if os.name == 'nt' else 'pgbench'), 'pgbench might be missing') def test_pgbench(self): + __class__.helper__skip_test_if_util_not_exist("pgbench") + with get_new_node().init().start() as node: # initialize pgbench DB and run benchmarks @@ -780,13 +787,13 @@ def test_pgbench(self): proc.stdout.close() - self.assertTrue('tps' in out) + assert ('tps' in out) def test_pg_config(self): # check same instances a = get_pg_config() b = get_pg_config() - self.assertEqual(id(a), id(b)) + assert (id(a) == id(b)) # save right before config change c1 = get_pg_config() @@ -794,26 +801,26 @@ def test_pg_config(self): # modify setting for this scope with scoped_config(cache_pg_config=False) as config: # sanity check for value - self.assertFalse(config.cache_pg_config) + assert not (config.cache_pg_config) # save right after config change c2 = get_pg_config() # check different instances after config change - self.assertNotEqual(id(c1), id(c2)) + assert (id(c1) != id(c2)) # check different instances a = get_pg_config() b = get_pg_config() - self.assertNotEqual(id(a), id(b)) + assert (id(a) != id(b)) def test_config_stack(self): # no such option - with self.assertRaises(TypeError): + with pytest.raises(expected_exception=TypeError): configure_testgres(dummy=True) # we have only 1 config in stack - with self.assertRaises(IndexError): + with pytest.raises(expected_exception=IndexError): pop_config() d0 = TestgresConfig.cached_initdb_dir @@ -821,22 +828,22 @@ def test_config_stack(self): d2 = 'dummy_def' with scoped_config(cached_initdb_dir=d1) as c1: - self.assertEqual(c1.cached_initdb_dir, d1) + assert (c1.cached_initdb_dir == d1) with scoped_config(cached_initdb_dir=d2) as c2: stack_size = len(testgres.config.config_stack) # try to break a stack - with self.assertRaises(TypeError): + with pytest.raises(expected_exception=TypeError): with scoped_config(dummy=True): pass - self.assertEqual(c2.cached_initdb_dir, d2) - self.assertEqual(len(testgres.config.config_stack), stack_size) + assert (c2.cached_initdb_dir == d2) + assert (len(testgres.config.config_stack) == stack_size) - self.assertEqual(c1.cached_initdb_dir, d1) + assert (c1.cached_initdb_dir == d1) - self.assertEqual(TestgresConfig.cached_initdb_dir, d0) + assert (TestgresConfig.cached_initdb_dir == d0) def test_unix_sockets(self): with get_new_node() as node: @@ -854,17 +861,15 @@ def test_auto_name(self): with get_new_node().init(allow_streaming=True).start() as m: with m.replicate().start() as r: # check that nodes are running - self.assertTrue(m.status()) - self.assertTrue(r.status()) + assert (m.status()) + assert (r.status()) # check their names - self.assertNotEqual(m.name, r.name) - self.assertTrue('testgres' in m.name) - self.assertTrue('testgres' in r.name) + assert (m.name != r.name) + assert ('testgres' in m.name) + assert ('testgres' in r.name) def test_file_tail(self): - from testgres.utils import file_tail - s1 = "the quick brown fox jumped over that lazy dog\n" s2 = "abc\n" s3 = "def\n" @@ -879,13 +884,13 @@ def test_file_tail(self): f.seek(0) lines = file_tail(f, 3) - self.assertEqual(lines[0], s1) - self.assertEqual(lines[1], s2) - self.assertEqual(lines[2], s3) + assert (lines[0] == s1) + assert (lines[1] == s2) + assert (lines[2] == s3) f.seek(0) lines = file_tail(f, 1) - self.assertEqual(lines[0], s3) + assert (lines[0] == s3) def test_isolation_levels(self): with get_new_node().init().start() as node: @@ -903,24 +908,42 @@ def test_isolation_levels(self): con.begin(IsolationLevel.Serializable).commit() # check wrong level - with self.assertRaises(QueryException): + with pytest.raises(expected_exception=QueryException): con.begin('Garbage').commit() def test_ports_management(self): - # check that no ports have been bound yet - self.assertEqual(len(bound_ports), 0) + assert bound_ports is not None + assert type(bound_ports) == set # noqa: E721 + + if len(bound_ports) != 0: + logging.warning("bound_ports is not empty: {0}".format(bound_ports)) + + stage0__bound_ports = bound_ports.copy() with get_new_node() as node: - # check that we've just bound a port - self.assertEqual(len(bound_ports), 1) + assert bound_ports is not None + assert type(bound_ports) == set # noqa: E721 + + assert node.port is not None + assert type(node.port) == int # noqa: E721 + + logging.info("node port is {0}".format(node.port)) - # check that bound_ports contains our port - port_1 = list(bound_ports)[0] - port_2 = node.port - self.assertEqual(port_1, port_2) + assert node.port in bound_ports + assert node.port not in stage0__bound_ports + + assert stage0__bound_ports <= bound_ports + assert len(stage0__bound_ports) + 1 == len(bound_ports) + + stage1__bound_ports = stage0__bound_ports.copy() + stage1__bound_ports.add(node.port) + + assert stage1__bound_ports == bound_ports # check that port has been freed successfully - self.assertEqual(len(bound_ports), 0) + assert bound_ports is not None + assert type(bound_ports) == set # noqa: E721 + assert bound_ports == stage0__bound_ports def test_exceptions(self): str(StartNodeException('msg', [('file', 'lines')])) @@ -939,22 +962,22 @@ def test_version_management(self): g = PgVer('15.3.1bihabeta1') k = PgVer('15.3.1') - self.assertTrue(a == b) - self.assertTrue(b > c) - self.assertTrue(a > c) - self.assertTrue(d > e) - self.assertTrue(e > f) - self.assertTrue(d > f) - self.assertTrue(h > f) - self.assertTrue(h == i) - self.assertTrue(g == k) - self.assertTrue(g > h) + assert (a == b) + assert (b > c) + assert (a > c) + assert (d > e) + assert (e > f) + assert (d > f) + assert (h > f) + assert (h == i) + assert (g == k) + assert (g > h) version = get_pg_version() with get_new_node() as node: - self.assertTrue(isinstance(version, six.string_types)) - self.assertTrue(isinstance(node.version, PgVer)) - self.assertEqual(node.version, PgVer(version)) + assert (isinstance(version, six.string_types)) + assert (isinstance(node.version, PgVer)) + assert (node.version == PgVer(version)) def test_child_pids(self): master_processes = [ @@ -977,53 +1000,128 @@ def test_child_pids(self): ProcessType.WalReceiver, ] + def LOCAL__test_auxiliary_pids( + node: testgres.PostgresNode, + expectedTypes: list[ProcessType] + ) -> list[ProcessType]: + # returns list of the absence processes + assert node is not None + assert type(node) == testgres.PostgresNode # noqa: E721 + assert expectedTypes is not None + assert type(expectedTypes) == list # noqa: E721 + + pids = node.auxiliary_pids + assert pids is not None # noqa: E721 + assert type(pids) == dict # noqa: E721 + + result = list[ProcessType]() + for ptype in expectedTypes: + if not (ptype in pids): + result.append(ptype) + return result + + def LOCAL__check_auxiliary_pids__multiple_attempts( + node: testgres.PostgresNode, + expectedTypes: list[ProcessType]): + assert node is not None + assert type(node) == testgres.PostgresNode # noqa: E721 + assert expectedTypes is not None + assert type(expectedTypes) == list # noqa: E721 + + nAttempt = 0 + + while nAttempt < 5: + nAttempt += 1 + + logging.info("Test pids of [{0}] node. Attempt #{1}.".format( + node.name, + nAttempt + )) + + if nAttempt > 1: + time.sleep(1) + + absenceList = LOCAL__test_auxiliary_pids(node, expectedTypes) + assert absenceList is not None + assert type(absenceList) == list # noqa: E721 + if len(absenceList) == 0: + logging.info("Bingo!") + return + + logging.info("These processes are not found: {0}.".format(absenceList)) + continue + + raise Exception("Node {0} does not have the following processes: {1}.".format( + node.name, + absenceList + )) + with get_new_node().init().start() as master: # master node doesn't have a source walsender! - with self.assertRaises(TestgresException): + with pytest.raises(expected_exception=TestgresException): master.source_walsender with master.connect() as con: - self.assertGreater(con.pid, 0) + assert (con.pid > 0) with master.replicate().start() as replica: # test __str__ method str(master.child_processes[0]) - master_pids = master.auxiliary_pids - for ptype in master_processes: - self.assertIn(ptype, master_pids) + LOCAL__check_auxiliary_pids__multiple_attempts( + master, + master_processes) + + LOCAL__check_auxiliary_pids__multiple_attempts( + replica, + repl_processes) - replica_pids = replica.auxiliary_pids - for ptype in repl_processes: - self.assertIn(ptype, replica_pids) + master_pids = master.auxiliary_pids # there should be exactly 1 source walsender for replica - self.assertEqual(len(master_pids[ProcessType.WalSender]), 1) + assert (len(master_pids[ProcessType.WalSender]) == 1) pid1 = master_pids[ProcessType.WalSender][0] pid2 = replica.source_walsender.pid - self.assertEqual(pid1, pid2) + assert (pid1 == pid2) replica.stop() # there should be no walsender after we've stopped replica - with self.assertRaises(TestgresException): + with pytest.raises(expected_exception=TestgresException): replica.source_walsender def test_child_process_dies(self): # test for FileNotFound exception during child_processes() function cmd = ["timeout", "60"] if os.name == 'nt' else ["sleep", "60"] - with subprocess.Popen(cmd, shell=True) as process: # shell=True might be needed on Windows - self.assertEqual(process.poll(), None) - # collect list of processes currently running - children = psutil.Process(os.getpid()).children() - # kill a process, so received children dictionary becomes invalid - process.kill() - process.wait() - # try to handle children list -- missing processes will have ptype "ProcessType.Unknown" - [ProcessProxy(p) for p in children] + nAttempt = 0 + + while True: + if nAttempt == 5: + raise Exception("Max attempt number is exceed.") + + nAttempt += 1 + + logging.info("Attempt #{0}".format(nAttempt)) + + with subprocess.Popen(cmd, shell=True) as process: # shell=True might be needed on Windows + r = process.poll() + + if r is not None: + logging.warning("process.pool() returns an unexpected result: {0}.".format(r)) + continue + + assert r is None + # collect list of processes currently running + children = psutil.Process(os.getpid()).children() + # kill a process, so received children dictionary becomes invalid + process.kill() + process.wait() + # try to handle children list -- missing processes will have ptype "ProcessType.Unknown" + [ProcessProxy(p) for p in children] + break def test_upgrade_node(self): old_bin_dir = os.path.dirname(get_bin_path("pg_config")) @@ -1036,7 +1134,7 @@ def test_upgrade_node(self): node_new.init(cached=False) res = node_new.upgrade_from(old_node=node_old) node_new.start() - self.assertTrue(b'Upgrade Complete' in res) + assert (b'Upgrade Complete' in res) def test_parse_pg_version(self): # Linux Mint @@ -1051,25 +1149,26 @@ def test_parse_pg_version(self): def test_the_same_port(self): with get_new_node() as node: node.init().start() - self.assertTrue(node._should_free_port) - self.assertEqual(type(node.port), int) + assert (node._should_free_port) + assert (type(node.port) == int) # noqa: E721 node_port_copy = node.port - self.assertEqual(rm_carriage_returns(node.safe_psql("SELECT 1;")), b'1\n') + assert (rm_carriage_returns(node.safe_psql("SELECT 1;")) == b'1\n') with get_new_node(port=node.port) as node2: - self.assertEqual(type(node2.port), int) - self.assertEqual(node2.port, node.port) - self.assertFalse(node2._should_free_port) - - with self.assertRaises(StartNodeException) as ctx: + assert (type(node2.port) == int) # noqa: E721 + assert (node2.port == node.port) + assert not (node2._should_free_port) + + with pytest.raises( + expected_exception=StartNodeException, + match=re.escape("Cannot start node") + ): node2.init().start() - self.assertIn("Cannot start node", str(ctx.exception)) - # node is still working - self.assertEqual(node.port, node_port_copy) - self.assertTrue(node._should_free_port) - self.assertEqual(rm_carriage_returns(node.safe_psql("SELECT 3;")), b'3\n') + assert (node.port == node_port_copy) + assert (node._should_free_port) + assert (rm_carriage_returns(node.safe_psql("SELECT 3;")) == b'3\n') class tagPortManagerProxy: sm_prev_testgres_reserve_port = None @@ -1174,31 +1273,31 @@ def test_port_rereserve_during_node_start(self): with get_new_node() as node1: node1.init().start() - self.assertTrue(node1._should_free_port) - self.assertEqual(type(node1.port), int) # noqa: E721 + assert (node1._should_free_port) + assert (type(node1.port) == int) # noqa: E721 node1_port_copy = node1.port - self.assertEqual(rm_carriage_returns(node1.safe_psql("SELECT 1;")), b'1\n') + assert (rm_carriage_returns(node1.safe_psql("SELECT 1;")) == b'1\n') with __class__.tagPortManagerProxy(node1.port, C_COUNT_OF_BAD_PORT_USAGE): assert __class__.tagPortManagerProxy.sm_DummyPortNumber == node1.port with get_new_node() as node2: - self.assertTrue(node2._should_free_port) - self.assertEqual(node2.port, node1.port) + assert (node2._should_free_port) + assert (node2.port == node1.port) node2.init().start() - self.assertNotEqual(node2.port, node1.port) - self.assertTrue(node2._should_free_port) - self.assertEqual(__class__.tagPortManagerProxy.sm_DummyPortCurrentUsage, 0) - self.assertEqual(__class__.tagPortManagerProxy.sm_DummyPortTotalUsage, C_COUNT_OF_BAD_PORT_USAGE) - self.assertTrue(node2.is_started) + assert (node2.port != node1.port) + assert (node2._should_free_port) + assert (__class__.tagPortManagerProxy.sm_DummyPortCurrentUsage == 0) + assert (__class__.tagPortManagerProxy.sm_DummyPortTotalUsage == C_COUNT_OF_BAD_PORT_USAGE) + assert (node2.is_started) - self.assertEqual(rm_carriage_returns(node2.safe_psql("SELECT 2;")), b'2\n') + assert (rm_carriage_returns(node2.safe_psql("SELECT 2;")) == b'2\n') # node1 is still working - self.assertEqual(node1.port, node1_port_copy) - self.assertTrue(node1._should_free_port) - self.assertEqual(rm_carriage_returns(node1.safe_psql("SELECT 3;")), b'3\n') + assert (node1.port == node1_port_copy) + assert (node1._should_free_port) + assert (rm_carriage_returns(node1.safe_psql("SELECT 3;")) == b'3\n') def test_port_conflict(self): assert testgres.PostgresNode._C_MAX_START_ATEMPTS > 1 @@ -1207,35 +1306,36 @@ def test_port_conflict(self): with get_new_node() as node1: node1.init().start() - self.assertTrue(node1._should_free_port) - self.assertEqual(type(node1.port), int) # noqa: E721 + assert (node1._should_free_port) + assert (type(node1.port) == int) # noqa: E721 node1_port_copy = node1.port - self.assertEqual(rm_carriage_returns(node1.safe_psql("SELECT 1;")), b'1\n') + assert (rm_carriage_returns(node1.safe_psql("SELECT 1;")) == b'1\n') with __class__.tagPortManagerProxy(node1.port, C_COUNT_OF_BAD_PORT_USAGE): assert __class__.tagPortManagerProxy.sm_DummyPortNumber == node1.port with get_new_node() as node2: - self.assertTrue(node2._should_free_port) - self.assertEqual(node2.port, node1.port) + assert (node2._should_free_port) + assert (node2.port == node1.port) - with self.assertRaises(StartNodeException) as ctx: + with pytest.raises( + expected_exception=StartNodeException, + match=re.escape("Cannot start node after multiple attempts") + ): node2.init().start() - self.assertIn("Cannot start node after multiple attempts", str(ctx.exception)) - - self.assertEqual(node2.port, node1.port) - self.assertTrue(node2._should_free_port) - self.assertEqual(__class__.tagPortManagerProxy.sm_DummyPortCurrentUsage, 1) - self.assertEqual(__class__.tagPortManagerProxy.sm_DummyPortTotalUsage, C_COUNT_OF_BAD_PORT_USAGE) - self.assertFalse(node2.is_started) + assert (node2.port == node1.port) + assert (node2._should_free_port) + assert (__class__.tagPortManagerProxy.sm_DummyPortCurrentUsage == 1) + assert (__class__.tagPortManagerProxy.sm_DummyPortTotalUsage == C_COUNT_OF_BAD_PORT_USAGE) + assert not (node2.is_started) # node2 must release our dummyPort (node1.port) - self.assertEqual(__class__.tagPortManagerProxy.sm_DummyPortCurrentUsage, 0) + assert (__class__.tagPortManagerProxy.sm_DummyPortCurrentUsage == 0) # node1 is still working - self.assertEqual(node1.port, node1_port_copy) - self.assertTrue(node1._should_free_port) - self.assertEqual(rm_carriage_returns(node1.safe_psql("SELECT 3;")), b'3\n') + assert (node1.port == node1_port_copy) + assert (node1._should_free_port) + assert (rm_carriage_returns(node1.safe_psql("SELECT 3;")) == b'3\n') def test_simple_with_bin_dir(self): with get_new_node() as node: @@ -1300,29 +1400,28 @@ def test_set_auto_conf(self): content = f.read() for x in testData: - self.assertIn( - x[0] + " = " + x[2], - content, - x[0] + " stored wrong" - ) - - -if __name__ == '__main__': - if os.environ.get('ALT_CONFIG'): - suite = unittest.TestSuite() - - # Small subset of tests for alternative configs (PG_BIN or PG_CONFIG) - suite.addTest(TestgresTests('test_pg_config')) - suite.addTest(TestgresTests('test_pg_ctl')) - suite.addTest(TestgresTests('test_psql')) - suite.addTest(TestgresTests('test_replicate')) - - print('Running tests for alternative config:') - for t in suite: - print(t) - print() - - runner = unittest.TextTestRunner() - runner.run(suite) - else: - unittest.main() + assert x[0] + " = " + x[2] in content + + @staticmethod + def helper__skip_test_if_util_not_exist(name: str): + assert type(name) == str # noqa: E721 + + if platform.system().lower() == "windows": + name2 = name + ".exe" + else: + name2 = name + + if not util_exists(name2): + pytest.skip('might be missing') + + @staticmethod + def helper__skip_test_if_pg_version_is_not_ge(version: str): + assert type(version) == str # noqa: E721 + if not pg_version_ge(version): + pytest.skip('requires {0}+'.format(version)) + + @staticmethod + def helper__skip_test_if_pg_version_is_ge(version: str): + assert type(version) == str # noqa: E721 + if pg_version_ge(version): + pytest.skip('requires <{0}'.format(version)) diff --git a/tests/test_simple_remote.py b/tests/test_simple_remote.py index 8b44623a..e7cc5e5c 100755 --- a/tests/test_simple_remote.py +++ b/tests/test_simple_remote.py @@ -1,22 +1,21 @@ -#!/usr/bin/env python # coding: utf-8 - import os import re import subprocess import tempfile -import testgres import time import six -import unittest +import pytest import psutil import logging.config from contextlib import contextmanager -from testgres.exceptions import \ +from .. import testgres + +from ..testgres.exceptions import \ InitNodeException, \ StartNodeException, \ ExecUtilException, \ @@ -26,38 +25,33 @@ TestgresException, \ InvalidOperationException -from testgres.config import \ +from ..testgres.config import \ TestgresConfig, \ configure_testgres, \ scoped_config, \ pop_config, testgres_config -from testgres import \ +from ..testgres import \ NodeStatus, \ ProcessType, \ IsolationLevel, \ get_remote_node, \ RemoteOperations -from testgres import \ +from ..testgres import \ get_bin_path, \ get_pg_config, \ get_pg_version -from testgres import \ +from ..testgres import \ First, \ Any # NOTE: those are ugly imports -from testgres import bound_ports -from testgres.utils import PgVer -from testgres.node import ProcessProxy, ConnectionParams - -conn_params = ConnectionParams(host=os.getenv('RDBMS_TESTPOOL1_HOST') or '127.0.0.1', - username=os.getenv('USER'), - ssh_key=os.getenv('RDBMS_TESTPOOL_SSHKEY')) -os_ops = RemoteOperations(conn_params) -testgres_config.set_os_ops(os_ops=os_ops) +from ..testgres import bound_ports +from ..testgres.utils import PgVer +from ..testgres.utils import file_tail +from ..testgres.node import ProcessProxy, ConnectionParams def pg_version_ge(version): @@ -68,16 +62,16 @@ def pg_version_ge(version): def util_exists(util): def good_properties(f): - return (os_ops.path_exists(f) and # noqa: W504 - os_ops.isfile(f) and # noqa: W504 - os_ops.is_executable(f)) # yapf: disable + return (testgres_config.os_ops.path_exists(f) and # noqa: W504 + testgres_config.os_ops.isfile(f) and # noqa: W504 + testgres_config.os_ops.is_executable(f)) # yapf: disable # try to resolve it if good_properties(get_bin_path(util)): return True # check if util is in PATH - for path in os_ops.environ("PATH").split(os_ops.pathsep): + for path in testgres_config.os_ops.environ("PATH").split(testgres_config.os_ops.pathsep): if good_properties(os.path.join(path, util)): return True @@ -87,37 +81,56 @@ def removing(f): try: yield f finally: - if os_ops.isfile(f): - os_ops.remove_file(f) + if testgres_config.os_ops.isfile(f): + testgres_config.os_ops.remove_file(f) + + elif testgres_config.os_ops.isdir(f): + testgres_config.os_ops.rmdirs(f, ignore_errors=True) - elif os_ops.isdir(f): - os_ops.rmdirs(f, ignore_errors=True) +class TestgresRemoteTests: + sm_conn_params = ConnectionParams( + host=os.getenv('RDBMS_TESTPOOL1_HOST') or '127.0.0.1', + username=os.getenv('USER'), + ssh_key=os.getenv('RDBMS_TESTPOOL_SSHKEY')) + + sm_os_ops = RemoteOperations(sm_conn_params) + + @pytest.fixture(autouse=True, scope="class") + def implicit_fixture(self): + prev_ops = testgres_config.os_ops + assert prev_ops is not None + assert __class__.sm_os_ops is not None + testgres_config.set_os_ops(os_ops=__class__.sm_os_ops) + assert testgres_config.os_ops is __class__.sm_os_ops + yield + assert testgres_config.os_ops is __class__.sm_os_ops + testgres_config.set_os_ops(os_ops=prev_ops) + assert testgres_config.os_ops is prev_ops -class TestgresRemoteTests(unittest.TestCase): def test_node_repr(self): - with get_remote_node(conn_params=conn_params) as node: + with __class__.helper__get_node() as node: pattern = r"PostgresNode\(name='.+', port=.+, base_dir='.+'\)" - self.assertIsNotNone(re.match(pattern, str(node))) + assert re.match(pattern, str(node)) is not None def test_custom_init(self): - with get_remote_node(conn_params=conn_params) as node: + with __class__.helper__get_node() as node: # enable page checksums node.init(initdb_params=['-k']).start() - with get_remote_node(conn_params=conn_params) as node: + with __class__.helper__get_node() as node: node.init( allow_streaming=True, initdb_params=['--auth-local=reject', '--auth-host=reject']) hba_file = os.path.join(node.data_dir, 'pg_hba.conf') - lines = os_ops.readlines(hba_file) + lines = node.os_ops.readlines(hba_file) # check number of lines - self.assertGreaterEqual(len(lines), 6) + assert (len(lines) >= 6) # there should be no trust entries at all - self.assertFalse(any('trust' in s for s in lines)) + assert not (any('trust' in s for s in lines)) def test_init__LANG_С(self): # PBCKP-1744 @@ -126,7 +139,7 @@ def test_init__LANG_С(self): try: os.environ["LANG"] = "C" - with get_remote_node(conn_params=conn_params) as node: + with __class__.helper__get_node() as node: node.init().start() finally: __class__.helper__restore_envvar("LANG", prev_LANG) @@ -167,9 +180,9 @@ def test_init__unk_LANG_and_LC_CTYPE(self): while True: try: - with get_remote_node(conn_params=conn_params): + with __class__.helper__get_node(): pass - except testgres.exceptions.ExecUtilException as e: + except ExecUtilException as e: # # Example of an error message: # @@ -193,88 +206,89 @@ def test_init__unk_LANG_and_LC_CTYPE(self): __class__.helper__restore_envvar("LC_COLLATE", prev_LC_COLLATE) def test_double_init(self): - with get_remote_node(conn_params=conn_params).init() as node: + with __class__.helper__get_node().init() as node: # can't initialize node more than once - with self.assertRaises(InitNodeException): + with pytest.raises(expected_exception=InitNodeException): node.init() def test_init_after_cleanup(self): - with get_remote_node(conn_params=conn_params) as node: + with __class__.helper__get_node() as node: node.init().start().execute('select 1') node.cleanup() node.init().start().execute('select 1') - @unittest.skipUnless(util_exists('pg_resetwal'), 'might be missing') - @unittest.skipUnless(pg_version_ge('9.6'), 'requires 9.6+') def test_init_unique_system_id(self): # this function exists in PostgreSQL 9.6+ + __class__.helper__skip_test_if_util_not_exist("pg_resetwal") + __class__.helper__skip_test_if_pg_version_is_not_ge('9.6') + query = 'select system_identifier from pg_control_system()' with scoped_config(cache_initdb=False): - with get_remote_node(conn_params=conn_params).init().start() as node0: + with __class__.helper__get_node().init().start() as node0: id0 = node0.execute(query)[0] with scoped_config(cache_initdb=True, cached_initdb_unique=True) as config: - self.assertTrue(config.cache_initdb) - self.assertTrue(config.cached_initdb_unique) + assert (config.cache_initdb) + assert (config.cached_initdb_unique) # spawn two nodes; ids must be different - with get_remote_node(conn_params=conn_params).init().start() as node1, \ - get_remote_node(conn_params=conn_params).init().start() as node2: + with __class__.helper__get_node().init().start() as node1, \ + __class__.helper__get_node().init().start() as node2: id1 = node1.execute(query)[0] id2 = node2.execute(query)[0] # ids must increase - self.assertGreater(id1, id0) - self.assertGreater(id2, id1) + assert (id1 > id0) + assert (id2 > id1) def test_node_exit(self): - with self.assertRaises(QueryException): - with get_remote_node(conn_params=conn_params).init() as node: + with pytest.raises(expected_exception=QueryException): + with __class__.helper__get_node().init() as node: base_dir = node.base_dir node.safe_psql('select 1') # we should save the DB for "debugging" - self.assertTrue(os_ops.path_exists(base_dir)) - os_ops.rmdirs(base_dir, ignore_errors=True) + assert (__class__.sm_os_ops.path_exists(base_dir)) + __class__.sm_os_ops.rmdirs(base_dir, ignore_errors=True) - with get_remote_node(conn_params=conn_params).init() as node: + with __class__.helper__get_node().init() as node: base_dir = node.base_dir # should have been removed by default - self.assertFalse(os_ops.path_exists(base_dir)) + assert not (__class__.sm_os_ops.path_exists(base_dir)) def test_double_start(self): - with get_remote_node(conn_params=conn_params).init().start() as node: + with __class__.helper__get_node().init().start() as node: # can't start node more than once node.start() - self.assertTrue(node.is_started) + assert (node.is_started) def test_uninitialized_start(self): - with get_remote_node(conn_params=conn_params) as node: + with __class__.helper__get_node() as node: # node is not initialized yet - with self.assertRaises(StartNodeException): + with pytest.raises(expected_exception=StartNodeException): node.start() def test_restart(self): - with get_remote_node(conn_params=conn_params) as node: + with __class__.helper__get_node() as node: node.init().start() # restart, ok res = node.execute('select 1') - self.assertEqual(res, [(1,)]) + assert (res == [(1,)]) node.restart() res = node.execute('select 2') - self.assertEqual(res, [(2,)]) + assert (res == [(2,)]) # restart, fail - with self.assertRaises(StartNodeException): + with pytest.raises(expected_exception=StartNodeException): node.append_conf('pg_hba.conf', 'DUMMY') node.restart() def test_reload(self): - with get_remote_node(conn_params=conn_params) as node: + with __class__.helper__get_node() as node: node.init().start() # change client_min_messages and save old value @@ -286,108 +300,109 @@ def test_reload(self): # check new value cmm_new = node.execute('show client_min_messages') - self.assertEqual('debug1', cmm_new[0][0].lower()) - self.assertNotEqual(cmm_old, cmm_new) + assert ('debug1' == cmm_new[0][0].lower()) + assert (cmm_old != cmm_new) def test_pg_ctl(self): - with get_remote_node(conn_params=conn_params) as node: + with __class__.helper__get_node() as node: node.init().start() status = node.pg_ctl(['status']) - self.assertTrue('PID' in status) + assert ('PID' in status) def test_status(self): - self.assertTrue(NodeStatus.Running) - self.assertFalse(NodeStatus.Stopped) - self.assertFalse(NodeStatus.Uninitialized) + assert (NodeStatus.Running) + assert not (NodeStatus.Stopped) + assert not (NodeStatus.Uninitialized) # check statuses after each operation - with get_remote_node(conn_params=conn_params) as node: - self.assertEqual(node.pid, 0) - self.assertEqual(node.status(), NodeStatus.Uninitialized) + with __class__.helper__get_node() as node: + assert (node.pid == 0) + assert (node.status() == NodeStatus.Uninitialized) node.init() - self.assertEqual(node.pid, 0) - self.assertEqual(node.status(), NodeStatus.Stopped) + assert (node.pid == 0) + assert (node.status() == NodeStatus.Stopped) node.start() - self.assertNotEqual(node.pid, 0) - self.assertEqual(node.status(), NodeStatus.Running) + assert (node.pid != 0) + assert (node.status() == NodeStatus.Running) node.stop() - self.assertEqual(node.pid, 0) - self.assertEqual(node.status(), NodeStatus.Stopped) + assert (node.pid == 0) + assert (node.status() == NodeStatus.Stopped) node.cleanup() - self.assertEqual(node.pid, 0) - self.assertEqual(node.status(), NodeStatus.Uninitialized) + assert (node.pid == 0) + assert (node.status() == NodeStatus.Uninitialized) def test_psql(self): - with get_remote_node(conn_params=conn_params).init().start() as node: + with __class__.helper__get_node().init().start() as node: # check returned values (1 arg) res = node.psql('select 1') - self.assertEqual(res, (0, b'1\n', b'')) + assert (res == (0, b'1\n', b'')) # check returned values (2 args) res = node.psql('postgres', 'select 2') - self.assertEqual(res, (0, b'2\n', b'')) + assert (res == (0, b'2\n', b'')) # check returned values (named) res = node.psql(query='select 3', dbname='postgres') - self.assertEqual(res, (0, b'3\n', b'')) + assert (res == (0, b'3\n', b'')) # check returned values (1 arg) res = node.safe_psql('select 4') - self.assertEqual(res, b'4\n') + assert (res == b'4\n') # check returned values (2 args) res = node.safe_psql('postgres', 'select 5') - self.assertEqual(res, b'5\n') + assert (res == b'5\n') # check returned values (named) res = node.safe_psql(query='select 6', dbname='postgres') - self.assertEqual(res, b'6\n') + assert (res == b'6\n') # check feeding input node.safe_psql('create table horns (w int)') node.safe_psql('copy horns from stdin (format csv)', input=b"1\n2\n3\n\\.\n") _sum = node.safe_psql('select sum(w) from horns') - self.assertEqual(_sum, b'6\n') + assert (_sum == b'6\n') # check psql's default args, fails - with self.assertRaises(QueryException): + with pytest.raises(expected_exception=QueryException): node.psql() node.stop() # check psql on stopped node, fails - with self.assertRaises(QueryException): + with pytest.raises(expected_exception=QueryException): node.safe_psql('select 1') def test_safe_psql__expect_error(self): - with get_remote_node(conn_params=conn_params).init().start() as node: + with __class__.helper__get_node().init().start() as node: err = node.safe_psql('select_or_not_select 1', expect_error=True) - self.assertTrue(type(err) == str) # noqa: E721 - self.assertIn('select_or_not_select', err) - self.assertIn('ERROR: syntax error at or near "select_or_not_select"', err) + assert (type(err) == str) # noqa: E721 + assert ('select_or_not_select' in err) + assert ('ERROR: syntax error at or near "select_or_not_select"' in err) # --------- - with self.assertRaises(InvalidOperationException) as ctx: + with pytest.raises( + expected_exception=InvalidOperationException, + match="^" + re.escape("Exception was expected, but query finished successfully: `select 1;`.") + "$" + ): node.safe_psql("select 1;", expect_error=True) - self.assertEqual(str(ctx.exception), "Exception was expected, but query finished successfully: `select 1;`.") - # --------- res = node.safe_psql("select 1;", expect_error=False) - self.assertEqual(res, b'1\n') + assert (res == b'1\n') def test_transactions(self): - with get_remote_node(conn_params=conn_params).init().start() as node: + with __class__.helper__get_node().init().start() as node: with node.connect() as con: con.begin() con.execute('create table test(val int)') @@ -397,12 +412,12 @@ def test_transactions(self): con.begin() con.execute('insert into test values (2)') res = con.execute('select * from test order by val asc') - self.assertListEqual(res, [(1,), (2,)]) + assert (res == [(1,), (2,)]) con.rollback() con.begin() res = con.execute('select * from test') - self.assertListEqual(res, [(1,)]) + assert (res == [(1,)]) con.rollback() con.begin() @@ -410,25 +425,25 @@ def test_transactions(self): con.commit() def test_control_data(self): - with get_remote_node(conn_params=conn_params) as node: + with __class__.helper__get_node() as node: # node is not initialized yet - with self.assertRaises(ExecUtilException): + with pytest.raises(expected_exception=ExecUtilException): node.get_control_data() node.init() data = node.get_control_data() # check returned dict - self.assertIsNotNone(data) - self.assertTrue(any('pg_control' in s for s in data.keys())) + assert data is not None + assert (any('pg_control' in s for s in data.keys())) def test_backup_simple(self): - with get_remote_node(conn_params=conn_params) as master: + with __class__.helper__get_node() as master: # enable streaming for backups master.init(allow_streaming=True) # node must be running - with self.assertRaises(BackupException): + with pytest.raises(expected_exception=BackupException): master.backup() # it's time to start node @@ -440,23 +455,23 @@ def test_backup_simple(self): with master.backup(xlog_method='stream') as backup: with backup.spawn_primary().start() as slave: res = slave.execute('select * from test order by i asc') - self.assertListEqual(res, [(1,), (2,), (3,), (4,)]) + assert (res == [(1,), (2,), (3,), (4,)]) def test_backup_multiple(self): - with get_remote_node(conn_params=conn_params) as node: + with __class__.helper__get_node() as node: node.init(allow_streaming=True).start() with node.backup(xlog_method='fetch') as backup1, \ node.backup(xlog_method='fetch') as backup2: - self.assertNotEqual(backup1.base_dir, backup2.base_dir) + assert (backup1.base_dir != backup2.base_dir) with node.backup(xlog_method='fetch') as backup: with backup.spawn_primary('node1', destroy=False) as node1, \ backup.spawn_primary('node2', destroy=False) as node2: - self.assertNotEqual(node1.base_dir, node2.base_dir) + assert (node1.base_dir != node2.base_dir) def test_backup_exhaust(self): - with get_remote_node(conn_params=conn_params) as node: + with __class__.helper__get_node() as node: node.init(allow_streaming=True).start() with node.backup(xlog_method='fetch') as backup: @@ -465,19 +480,21 @@ def test_backup_exhaust(self): pass # now let's try to create one more node - with self.assertRaises(BackupException): + with pytest.raises(expected_exception=BackupException): backup.spawn_primary() def test_backup_wrong_xlog_method(self): - with get_remote_node(conn_params=conn_params) as node: + with __class__.helper__get_node() as node: node.init(allow_streaming=True).start() - with self.assertRaises(BackupException, - msg='Invalid xlog_method "wrong"'): + with pytest.raises( + expected_exception=BackupException, + match="^" + re.escape('Invalid xlog_method "wrong"') + "$" + ): node.backup(xlog_method='wrong') def test_pg_ctl_wait_option(self): - with get_remote_node(conn_params=conn_params) as node: + with __class__.helper__get_node() as node: node.init().start(wait=False) while True: try: @@ -489,23 +506,24 @@ def test_pg_ctl_wait_option(self): pass def test_replicate(self): - with get_remote_node(conn_params=conn_params) as node: + with __class__.helper__get_node() as node: node.init(allow_streaming=True).start() with node.replicate().start() as replica: res = replica.execute('select 1') - self.assertListEqual(res, [(1,)]) + assert (res == [(1,)]) node.execute('create table test (val int)', commit=True) replica.catchup() res = node.execute('select * from test') - self.assertListEqual(res, []) + assert (res == []) - @unittest.skipUnless(pg_version_ge('9.6'), 'requires 9.6+') def test_synchronous_replication(self): - with get_remote_node(conn_params=conn_params) as master: + __class__.helper__skip_test_if_pg_version_is_not_ge("9.6") + + with __class__.helper__get_node() as master: old_version = not pg_version_ge('9.6') master.init(allow_streaming=True).start() @@ -519,12 +537,12 @@ def test_synchronous_replication(self): standby2.start() # check formatting - self.assertEqual( - '1 ("{}", "{}")'.format(standby1.name, standby2.name), - str(First(1, (standby1, standby2)))) # yapf: disable - self.assertEqual( - 'ANY 1 ("{}", "{}")'.format(standby1.name, standby2.name), - str(Any(1, (standby1, standby2)))) # yapf: disable + assert ( + '1 ("{}", "{}")'.format(standby1.name, standby2.name) == str(First(1, (standby1, standby2))) + ) # yapf: disable + assert ( + 'ANY 1 ("{}", "{}")'.format(standby1.name, standby2.name) == str(Any(1, (standby1, standby2))) + ) # yapf: disable # set synchronous_standby_names master.set_synchronous_standbys(First(2, [standby1, standby2])) @@ -542,11 +560,12 @@ def test_synchronous_replication(self): master.safe_psql( 'insert into abc select generate_series(1, 1000000)') res = standby1.safe_psql('select count(*) from abc') - self.assertEqual(res, b'1000000\n') + assert (res == b'1000000\n') - @unittest.skipUnless(pg_version_ge('10'), 'requires 10+') def test_logical_replication(self): - with get_remote_node(conn_params=conn_params) as node1, get_remote_node(conn_params=conn_params) as node2: + __class__.helper__skip_test_if_pg_version_is_not_ge("10") + + with __class__.helper__get_node() as node1, __class__.helper__get_node() as node2: node1.init(allow_logical=True) node1.start() node2.init().start() @@ -564,7 +583,7 @@ def test_logical_replication(self): # wait until changes apply on subscriber and check them sub.catchup() res = node2.execute('select * from test') - self.assertListEqual(res, [(1, 1), (2, 2)]) + assert (res == [(1, 1), (2, 2)]) # disable and put some new data sub.disable() @@ -574,7 +593,7 @@ def test_logical_replication(self): sub.enable() sub.catchup() res = node2.execute('select * from test') - self.assertListEqual(res, [(1, 1), (2, 2), (3, 3)]) + assert (res == [(1, 1), (2, 2), (3, 3)]) # Add new tables. Since we added "all tables" to publication # (default behaviour of publish() method) we don't need @@ -588,7 +607,7 @@ def test_logical_replication(self): node1.safe_psql('insert into test2 values (\'a\'), (\'b\')') sub.catchup() res = node2.execute('select * from test2') - self.assertListEqual(res, [('a',), ('b',)]) + assert (res == [('a',), ('b',)]) # drop subscription sub.drop() @@ -602,21 +621,22 @@ def test_logical_replication(self): node1.safe_psql('insert into test values (4, 4)') sub.catchup() res = node2.execute('select * from test') - self.assertListEqual(res, [(1, 1), (2, 2), (3, 3), (4, 4)]) + assert (res == [(1, 1), (2, 2), (3, 3), (4, 4)]) # explicitly add table - with self.assertRaises(ValueError): + with pytest.raises(expected_exception=ValueError): pub.add_tables([]) # fail pub.add_tables(['test2']) node1.safe_psql('insert into test2 values (\'c\')') sub.catchup() res = node2.execute('select * from test2') - self.assertListEqual(res, [('a',), ('b',)]) + assert (res == [('a',), ('b',)]) - @unittest.skipUnless(pg_version_ge('10'), 'requires 10+') def test_logical_catchup(self): """ Runs catchup for 100 times to be sure that it is consistent """ - with get_remote_node(conn_params=conn_params) as node1, get_remote_node(conn_params=conn_params) as node2: + __class__.helper__skip_test_if_pg_version_is_not_ge("10") + + with __class__.helper__get_node() as node1, __class__.helper__get_node() as node2: node1.init(allow_logical=True) node1.start() node2.init().start() @@ -633,39 +653,37 @@ def test_logical_catchup(self): node1.execute('insert into test values ({0}, {0})'.format(i)) sub.catchup() res = node2.execute('select * from test') - self.assertListEqual(res, [( - i, - i, - )]) + assert (res == [(i, i, )]) node1.execute('delete from test') - @unittest.skipIf(pg_version_ge('10'), 'requires <10') def test_logical_replication_fail(self): - with get_remote_node(conn_params=conn_params) as node: - with self.assertRaises(InitNodeException): + __class__.helper__skip_test_if_pg_version_is_ge("10") + + with __class__.helper__get_node() as node: + with pytest.raises(expected_exception=InitNodeException): node.init(allow_logical=True) def test_replication_slots(self): - with get_remote_node(conn_params=conn_params) as node: + with __class__.helper__get_node() as node: node.init(allow_streaming=True).start() with node.replicate(slot='slot1').start() as replica: replica.execute('select 1') # cannot create new slot with the same name - with self.assertRaises(TestgresException): + with pytest.raises(expected_exception=TestgresException): node.replicate(slot='slot1') def test_incorrect_catchup(self): - with get_remote_node(conn_params=conn_params) as node: + with __class__.helper__get_node() as node: node.init(allow_streaming=True).start() # node has no master, can't catch up - with self.assertRaises(TestgresException): + with pytest.raises(expected_exception=TestgresException): node.catchup() def test_promotion(self): - with get_remote_node(conn_params=conn_params) as master: + with __class__.helper__get_node() as master: master.init().start() master.safe_psql('create table abc(id serial)') @@ -676,35 +694,35 @@ def test_promotion(self): # make standby becomes writable master replica.safe_psql('insert into abc values (1)') res = replica.safe_psql('select * from abc') - self.assertEqual(res, b'1\n') + assert (res == b'1\n') def test_dump(self): query_create = 'create table test as select generate_series(1, 2) as val' query_select = 'select * from test order by val asc' - with get_remote_node(conn_params=conn_params).init().start() as node1: + with __class__.helper__get_node().init().start() as node1: node1.execute(query_create) for format in ['plain', 'custom', 'directory', 'tar']: with removing(node1.dump(format=format)) as dump: - with get_remote_node(conn_params=conn_params).init().start() as node3: + with __class__.helper__get_node().init().start() as node3: if format == 'directory': - self.assertTrue(os_ops.isdir(dump)) + assert (node1.os_ops.isdir(dump)) else: - self.assertTrue(os_ops.isfile(dump)) + assert (node1.os_ops.isfile(dump)) # restore dump node3.restore(filename=dump) res = node3.execute(query_select) - self.assertListEqual(res, [(1,), (2,)]) + assert (res == [(1,), (2,)]) def test_users(self): - with get_remote_node(conn_params=conn_params).init().start() as node: + with __class__.helper__get_node().init().start() as node: node.psql('create role test_user login') value = node.safe_psql('select 1', username='test_user') - self.assertEqual(b'1\n', value) + assert (b'1\n' == value) def test_poll_query_until(self): - with get_remote_node(conn_params=conn_params) as node: + with __class__.helper__get_node() as node: node.init().start() get_time = 'select extract(epoch from now())' @@ -714,15 +732,15 @@ def test_poll_query_until(self): node.poll_query_until(query=check_time.format(start_time)) end_time = node.execute(get_time)[0][0] - self.assertTrue(end_time - start_time >= 5) + assert (end_time - start_time >= 5) # check 0 columns - with self.assertRaises(QueryException): + with pytest.raises(expected_exception=QueryException): node.poll_query_until( query='select from pg_catalog.pg_class limit 1') # check None, fail - with self.assertRaises(QueryException): + with pytest.raises(expected_exception=QueryException): node.poll_query_until(query='create table abc (val int)') # check None, ok @@ -735,7 +753,7 @@ def test_poll_query_until(self): expected=None) # check arbitrary expected value, fail - with self.assertRaises(TimeoutException): + with pytest.raises(expected_exception=TimeoutException): node.poll_query_until(query='select 3', expected=1, max_attempts=3, @@ -745,17 +763,17 @@ def test_poll_query_until(self): node.poll_query_until(query='select 2', expected=2) # check timeout - with self.assertRaises(TimeoutException): + with pytest.raises(expected_exception=TimeoutException): node.poll_query_until(query='select 1 > 2', max_attempts=3, sleep_time=0.01) # check ProgrammingError, fail - with self.assertRaises(testgres.ProgrammingError): + with pytest.raises(expected_exception=testgres.ProgrammingError): node.poll_query_until(query='dummy1') # check ProgrammingError, ok - with self.assertRaises(TimeoutException): + with pytest.raises(expected_exception=TimeoutException): node.poll_query_until(query='dummy2', max_attempts=3, sleep_time=0.01, @@ -808,17 +826,18 @@ def test_logging(self): # check that master's port is found with open(logfile.name, 'r') as log: lines = log.readlines() - self.assertTrue(any(node_name in s for s in lines)) + assert (any(node_name in s for s in lines)) # test logger after stop/start/restart master.stop() master.start() master.restart() - self.assertTrue(master._logger.is_alive()) + assert (master._logger.is_alive()) - @unittest.skipUnless(util_exists('pgbench'), 'might be missing') def test_pgbench(self): - with get_remote_node(conn_params=conn_params).init().start() as node: + __class__.helper__skip_test_if_util_not_exist("pgbench") + + with __class__.helper__get_node().init().start() as node: # initialize pgbench DB and run benchmarks node.pgbench_init(scale=2, foreign_keys=True, options=['-q']).pgbench_run(time=2) @@ -828,13 +847,13 @@ def test_pgbench(self): stderr=subprocess.STDOUT, options=['-T3']) out = proc.communicate()[0] - self.assertTrue(b'tps = ' in out) + assert (b'tps = ' in out) def test_pg_config(self): # check same instances a = get_pg_config() b = get_pg_config() - self.assertEqual(id(a), id(b)) + assert (id(a) == id(b)) # save right before config change c1 = get_pg_config() @@ -842,26 +861,26 @@ def test_pg_config(self): # modify setting for this scope with scoped_config(cache_pg_config=False) as config: # sanity check for value - self.assertFalse(config.cache_pg_config) + assert not (config.cache_pg_config) # save right after config change c2 = get_pg_config() # check different instances after config change - self.assertNotEqual(id(c1), id(c2)) + assert (id(c1) != id(c2)) # check different instances a = get_pg_config() b = get_pg_config() - self.assertNotEqual(id(a), id(b)) + assert (id(a) != id(b)) def test_config_stack(self): # no such option - with self.assertRaises(TypeError): + with pytest.raises(expected_exception=TypeError): configure_testgres(dummy=True) # we have only 1 config in stack - with self.assertRaises(IndexError): + with pytest.raises(expected_exception=IndexError): pop_config() d0 = TestgresConfig.cached_initdb_dir @@ -869,54 +888,52 @@ def test_config_stack(self): d2 = 'dummy_def' with scoped_config(cached_initdb_dir=d1) as c1: - self.assertEqual(c1.cached_initdb_dir, d1) + assert (c1.cached_initdb_dir == d1) with scoped_config(cached_initdb_dir=d2) as c2: stack_size = len(testgres.config.config_stack) # try to break a stack - with self.assertRaises(TypeError): + with pytest.raises(expected_exception=TypeError): with scoped_config(dummy=True): pass - self.assertEqual(c2.cached_initdb_dir, d2) - self.assertEqual(len(testgres.config.config_stack), stack_size) + assert (c2.cached_initdb_dir == d2) + assert (len(testgres.config.config_stack) == stack_size) - self.assertEqual(c1.cached_initdb_dir, d1) + assert (c1.cached_initdb_dir == d1) - self.assertEqual(TestgresConfig.cached_initdb_dir, d0) + assert (TestgresConfig.cached_initdb_dir == d0) def test_unix_sockets(self): - with get_remote_node(conn_params=conn_params) as node: + with __class__.helper__get_node() as node: node.init(unix_sockets=False, allow_streaming=True) node.start() res_exec = node.execute('select 1') res_psql = node.safe_psql('select 1') - self.assertEqual(res_exec, [(1,)]) - self.assertEqual(res_psql, b'1\n') + assert (res_exec == [(1,)]) + assert (res_psql == b'1\n') with node.replicate().start() as r: res_exec = r.execute('select 1') res_psql = r.safe_psql('select 1') - self.assertEqual(res_exec, [(1,)]) - self.assertEqual(res_psql, b'1\n') + assert (res_exec == [(1,)]) + assert (res_psql == b'1\n') def test_auto_name(self): - with get_remote_node(conn_params=conn_params).init(allow_streaming=True).start() as m: + with __class__.helper__get_node().init(allow_streaming=True).start() as m: with m.replicate().start() as r: # check that nodes are running - self.assertTrue(m.status()) - self.assertTrue(r.status()) + assert (m.status()) + assert (r.status()) # check their names - self.assertNotEqual(m.name, r.name) - self.assertTrue('testgres' in m.name) - self.assertTrue('testgres' in r.name) + assert (m.name != r.name) + assert ('testgres' in m.name) + assert ('testgres' in r.name) def test_file_tail(self): - from testgres.utils import file_tail - s1 = "the quick brown fox jumped over that lazy dog\n" s2 = "abc\n" s3 = "def\n" @@ -931,16 +948,16 @@ def test_file_tail(self): f.seek(0) lines = file_tail(f, 3) - self.assertEqual(lines[0], s1) - self.assertEqual(lines[1], s2) - self.assertEqual(lines[2], s3) + assert (lines[0] == s1) + assert (lines[1] == s2) + assert (lines[2] == s3) f.seek(0) lines = file_tail(f, 1) - self.assertEqual(lines[0], s3) + assert (lines[0] == s3) def test_isolation_levels(self): - with get_remote_node(conn_params=conn_params).init().start() as node: + with __class__.helper__get_node().init().start() as node: with node.connect() as con: # string levels con.begin('Read Uncommitted').commit() @@ -955,24 +972,24 @@ def test_isolation_levels(self): con.begin(IsolationLevel.Serializable).commit() # check wrong level - with self.assertRaises(QueryException): + with pytest.raises(expected_exception=QueryException): con.begin('Garbage').commit() def test_ports_management(self): # check that no ports have been bound yet - self.assertEqual(len(bound_ports), 0) + assert (len(bound_ports) == 0) - with get_remote_node(conn_params=conn_params) as node: + with __class__.helper__get_node() as node: # check that we've just bound a port - self.assertEqual(len(bound_ports), 1) + assert (len(bound_ports) == 1) # check that bound_ports contains our port port_1 = list(bound_ports)[0] port_2 = node.port - self.assertEqual(port_1, port_2) + assert (port_1 == port_2) # check that port has been freed successfully - self.assertEqual(len(bound_ports), 0) + assert (len(bound_ports) == 0) def test_exceptions(self): str(StartNodeException('msg', [('file', 'lines')])) @@ -987,18 +1004,18 @@ def test_version_management(self): e = PgVer('15rc1') f = PgVer('15beta4') - self.assertTrue(a == b) - self.assertTrue(b > c) - self.assertTrue(a > c) - self.assertTrue(d > e) - self.assertTrue(e > f) - self.assertTrue(d > f) + assert (a == b) + assert (b > c) + assert (a > c) + assert (d > e) + assert (e > f) + assert (d > f) version = get_pg_version() - with get_remote_node(conn_params=conn_params) as node: - self.assertTrue(isinstance(version, six.string_types)) - self.assertTrue(isinstance(node.version, PgVer)) - self.assertEqual(node.version, PgVer(version)) + with __class__.helper__get_node() as node: + assert (isinstance(version, six.string_types)) + assert (isinstance(node.version, PgVer)) + assert (node.version == PgVer(version)) def test_child_pids(self): master_processes = [ @@ -1021,51 +1038,132 @@ def test_child_pids(self): ProcessType.WalReceiver, ] - with get_remote_node(conn_params=conn_params).init().start() as master: + def LOCAL__test_auxiliary_pids( + node: testgres.PostgresNode, + expectedTypes: list[ProcessType] + ) -> list[ProcessType]: + # returns list of the absence processes + assert node is not None + assert type(node) == testgres.PostgresNode # noqa: E721 + assert expectedTypes is not None + assert type(expectedTypes) == list # noqa: E721 + + pids = node.auxiliary_pids + assert pids is not None # noqa: E721 + assert type(pids) == dict # noqa: E721 + + result = list[ProcessType]() + for ptype in expectedTypes: + if not (ptype in pids): + result.append(ptype) + return result + + def LOCAL__check_auxiliary_pids__multiple_attempts( + node: testgres.PostgresNode, + expectedTypes: list[ProcessType]): + assert node is not None + assert type(node) == testgres.PostgresNode # noqa: E721 + assert expectedTypes is not None + assert type(expectedTypes) == list # noqa: E721 + + nAttempt = 0 + + while nAttempt < 5: + nAttempt += 1 + + logging.info("Test pids of [{0}] node. Attempt #{1}.".format( + node.name, + nAttempt + )) + + if nAttempt > 1: + time.sleep(1) + + absenceList = LOCAL__test_auxiliary_pids(node, expectedTypes) + assert absenceList is not None + assert type(absenceList) == list # noqa: E721 + if len(absenceList) == 0: + logging.info("Bingo!") + return + + logging.info("These processes are not found: {0}.".format(absenceList)) + continue + + raise Exception("Node {0} does not have the following processes: {1}.".format( + node.name, + absenceList + )) + + with __class__.helper__get_node().init().start() as master: # master node doesn't have a source walsender! - with self.assertRaises(TestgresException): + with pytest.raises(expected_exception=TestgresException): master.source_walsender with master.connect() as con: - self.assertGreater(con.pid, 0) + assert (con.pid > 0) with master.replicate().start() as replica: # test __str__ method str(master.child_processes[0]) - master_pids = master.auxiliary_pids - for ptype in master_processes: - self.assertIn(ptype, master_pids) + LOCAL__check_auxiliary_pids__multiple_attempts( + master, + master_processes) - replica_pids = replica.auxiliary_pids - for ptype in repl_processes: - self.assertIn(ptype, replica_pids) + LOCAL__check_auxiliary_pids__multiple_attempts( + replica, + repl_processes) + + master_pids = master.auxiliary_pids # there should be exactly 1 source walsender for replica - self.assertEqual(len(master_pids[ProcessType.WalSender]), 1) + assert (len(master_pids[ProcessType.WalSender]) == 1) pid1 = master_pids[ProcessType.WalSender][0] pid2 = replica.source_walsender.pid - self.assertEqual(pid1, pid2) + assert (pid1 == pid2) replica.stop() # there should be no walsender after we've stopped replica - with self.assertRaises(TestgresException): + with pytest.raises(expected_exception=TestgresException): replica.source_walsender + # TODO: Why does not this test work with remote host? def test_child_process_dies(self): - # test for FileNotFound exception during child_processes() function - with subprocess.Popen(["sleep", "60"]) as process: - self.assertEqual(process.poll(), None) - # collect list of processes currently running - children = psutil.Process(os.getpid()).children() - # kill a process, so received children dictionary becomes invalid - process.kill() - process.wait() - # try to handle children list -- missing processes will have ptype "ProcessType.Unknown" - [ProcessProxy(p) for p in children] + nAttempt = 0 + + while True: + if nAttempt == 5: + raise Exception("Max attempt number is exceed.") + + nAttempt += 1 + + logging.info("Attempt #{0}".format(nAttempt)) + + # test for FileNotFound exception during child_processes() function + with subprocess.Popen(["sleep", "60"]) as process: + r = process.poll() + + if r is not None: + logging.warning("process.pool() returns an unexpected result: {0}.".format(r)) + continue + + assert r is None + # collect list of processes currently running + children = psutil.Process(os.getpid()).children() + # kill a process, so received children dictionary becomes invalid + process.kill() + process.wait() + # try to handle children list -- missing processes will have ptype "ProcessType.Unknown" + [ProcessProxy(p) for p in children] + break + + @staticmethod + def helper__get_node(): + assert __class__.sm_conn_params is not None + return get_remote_node(conn_params=__class__.sm_conn_params) @staticmethod def helper__restore_envvar(name, prev_value): @@ -1074,23 +1172,20 @@ def helper__restore_envvar(name, prev_value): else: os.environ[name] = prev_value + @staticmethod + def helper__skip_test_if_util_not_exist(name: str): + assert type(name) == str # noqa: E721 + if not util_exists(name): + pytest.skip('might be missing') -if __name__ == '__main__': - if os_ops.environ('ALT_CONFIG'): - suite = unittest.TestSuite() - - # Small subset of tests for alternative configs (PG_BIN or PG_CONFIG) - suite.addTest(TestgresRemoteTests('test_pg_config')) - suite.addTest(TestgresRemoteTests('test_pg_ctl')) - suite.addTest(TestgresRemoteTests('test_psql')) - suite.addTest(TestgresRemoteTests('test_replicate')) - - print('Running tests for alternative config:') - for t in suite: - print(t) - print() + @staticmethod + def helper__skip_test_if_pg_version_is_not_ge(version: str): + assert type(version) == str # noqa: E721 + if not pg_version_ge(version): + pytest.skip('requires {0}+'.format(version)) - runner = unittest.TextTestRunner() - runner.run(suite) - else: - unittest.main() + @staticmethod + def helper__skip_test_if_pg_version_is_ge(version: str): + assert type(version) == str # noqa: E721 + if pg_version_ge(version): + pytest.skip('requires <{0}'.format(version)) From 9911ed0c928b602c17a3b0845b24910bf291925a Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Thu, 27 Feb 2025 13:48:05 +0300 Subject: [PATCH 398/475] [run_tests.sh] A right way for obtaining of BINDIR and PG_CONFIG is used (#196) A problem was detected in container with Ubuntu 24.04 tests works with "/usr/bin/pg_config" but real pg_config is "/usr/lib/postgresql/17/bin/pg_config" To resovle this problem we will call "pg_config --bindir" and use it result for BINDIR and PG_CONFIG. --- run_tests.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/run_tests.sh b/run_tests.sh index e9d58b54..5cbbac60 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -43,13 +43,13 @@ time coverage run -a -m pytest -l -v -n 4 -k "TestgresTests" # run tests (PG_BIN) time \ - PG_BIN=$(dirname $(which pg_config)) \ + PG_BIN=$(pg_config --bindir) \ coverage run -a -m pytest -l -v -n 4 -k "TestgresTests" # run tests (PG_CONFIG) time \ - PG_CONFIG=$(which pg_config) \ + PG_CONFIG=$(pg_config --bindir)/pg_config \ coverage run -a -m pytest -l -v -n 4 -k "TestgresTests" From 6d67da2170becb944fc768ff4938f88b37e0f2a4 Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Thu, 27 Feb 2025 14:30:43 +0300 Subject: [PATCH 399/475] Update README.md Build status is corrected A correct URL to get a starus image is https://api.travis-ci.com/postgrespro/testgres.svg?branch=master --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f0071a90..a3b854f8 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -[![Build Status](https://travis-ci.com/postgrespro/testgres.svg?branch=master)](https://app.travis-ci.com/github/postgrespro/testgres/branches) +[![Build Status](https://api.travis-ci.com/postgrespro/testgres.svg?branch=master)](https://travis-ci.com/github/postgrespro/testgres) [![codecov](https://codecov.io/gh/postgrespro/testgres/branch/master/graph/badge.svg)](https://codecov.io/gh/postgrespro/testgres) [![PyPI version](https://badge.fury.io/py/testgres.svg)](https://badge.fury.io/py/testgres) From 3cc162715e3bc3c49b944e03038665dbd4445221 Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Fri, 28 Feb 2025 12:04:05 +0300 Subject: [PATCH 400/475] TestgresRemoteTests::test_ports_management is corrected (#198) It is synchronized with TestgresTests::test_ports_management. --- tests/test_simple_remote.py | 36 +++++++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/tests/test_simple_remote.py b/tests/test_simple_remote.py index e7cc5e5c..d4a28a2b 100755 --- a/tests/test_simple_remote.py +++ b/tests/test_simple_remote.py @@ -976,20 +976,38 @@ def test_isolation_levels(self): con.begin('Garbage').commit() def test_ports_management(self): - # check that no ports have been bound yet - assert (len(bound_ports) == 0) + assert bound_ports is not None + assert type(bound_ports) == set # noqa: E721 + + if len(bound_ports) != 0: + logging.warning("bound_ports is not empty: {0}".format(bound_ports)) + + stage0__bound_ports = bound_ports.copy() with __class__.helper__get_node() as node: - # check that we've just bound a port - assert (len(bound_ports) == 1) + assert bound_ports is not None + assert type(bound_ports) == set # noqa: E721 + + assert node.port is not None + assert type(node.port) == int # noqa: E721 + + logging.info("node port is {0}".format(node.port)) + + assert node.port in bound_ports + assert node.port not in stage0__bound_ports + + assert stage0__bound_ports <= bound_ports + assert len(stage0__bound_ports) + 1 == len(bound_ports) + + stage1__bound_ports = stage0__bound_ports.copy() + stage1__bound_ports.add(node.port) - # check that bound_ports contains our port - port_1 = list(bound_ports)[0] - port_2 = node.port - assert (port_1 == port_2) + assert stage1__bound_ports == bound_ports # check that port has been freed successfully - assert (len(bound_ports) == 0) + assert bound_ports is not None + assert type(bound_ports) == set # noqa: E721 + assert bound_ports == stage0__bound_ports def test_exceptions(self): str(StartNodeException('msg', [('file', 'lines')])) From de432edafd63e3f054cefaefcbe69b4b3ae90e4f Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Fri, 28 Feb 2025 19:43:38 +0300 Subject: [PATCH 401/475] execute_utility2 is updated (ignore_errors) (#201) - New parameters "ignore_errors" is added. Default value is False. - Asserts are added. --- testgres/utils.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/testgres/utils.py b/testgres/utils.py index 9645fc3b..76d42b02 100644 --- a/testgres/utils.py +++ b/testgres/utils.py @@ -73,11 +73,13 @@ def execute_utility(args, logfile=None, verbose=False): return execute_utility2(tconf.os_ops, args, logfile, verbose) -def execute_utility2(os_ops: OsOperations, args, logfile=None, verbose=False): +def execute_utility2(os_ops: OsOperations, args, logfile=None, verbose=False, ignore_errors=False): assert os_ops is not None assert isinstance(os_ops, OsOperations) + assert type(verbose) == bool # noqa: E721 + assert type(ignore_errors) == bool # noqa: E721 - exit_status, out, error = os_ops.exec_command(args, verbose=True) + exit_status, out, error = os_ops.exec_command(args, verbose=True, ignore_errors=ignore_errors) # decode result out = '' if not out else out if isinstance(out, bytes): From 22826e001bd32314da0bef24101a916f011a0fdc Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Sat, 1 Mar 2025 09:57:49 +0300 Subject: [PATCH 402/475] PostgresNode::pid is improved (#199) * PostgresNode::pid is improved - We do multiple attempts to read pid file. - We process a case when we see that node is stopped between test and read. - We process a case when pid-file is empty. * PostgresNode::pid is updated Assert is added. * execute_utility2 is updated (ignore_errors) - New parameters "ignore_errors" is added. Default value is False. - Asserts are added. * PostgresNode::_try_shutdown is rewrited (normalization) * PostgresNode::pid uses the data from "pg_ctl status" output. * PostgresNode::_try_shutdown is correct (return None) This method returns nothing (None). --- testgres/consts.py | 4 + testgres/node.py | 247 +++++++++++++++++++++++++++++++++++++-------- 2 files changed, 211 insertions(+), 40 deletions(-) diff --git a/testgres/consts.py b/testgres/consts.py index 98c84af6..89c49ab7 100644 --- a/testgres/consts.py +++ b/testgres/consts.py @@ -35,3 +35,7 @@ # logical replication settings LOGICAL_REPL_MAX_CATCHUP_ATTEMPTS = 60 + +PG_CTL__STATUS__OK = 0 +PG_CTL__STATUS__NODE_IS_STOPPED = 3 +PG_CTL__STATUS__BAD_DATADIR = 4 diff --git a/testgres/node.py b/testgres/node.py index 56899b90..859fe742 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -49,7 +49,9 @@ RECOVERY_CONF_FILE, \ PG_LOG_FILE, \ UTILS_LOG_FILE, \ - PG_PID_FILE + PG_CTL__STATUS__OK, \ + PG_CTL__STATUS__NODE_IS_STOPPED, \ + PG_CTL__STATUS__BAD_DATADIR \ from .consts import \ MAX_LOGICAL_REPLICATION_WORKERS, \ @@ -208,14 +210,136 @@ def pid(self): Return postmaster's PID if node is running, else 0. """ - if self.status(): - pid_file = os.path.join(self.data_dir, PG_PID_FILE) - lines = self.os_ops.readlines(pid_file) - pid = int(lines[0]) if lines else None - return pid + self__data_dir = self.data_dir - # for clarity - return 0 + _params = [ + self._get_bin_path('pg_ctl'), + "-D", self__data_dir, + "status" + ] # yapf: disable + + status_code, out, error = execute_utility2( + self.os_ops, + _params, + self.utils_log_file, + verbose=True, + ignore_errors=True) + + assert type(status_code) == int # noqa: E721 + assert type(out) == str # noqa: E721 + assert type(error) == str # noqa: E721 + + # ----------------- + if status_code == PG_CTL__STATUS__NODE_IS_STOPPED: + return 0 + + # ----------------- + if status_code == PG_CTL__STATUS__BAD_DATADIR: + return 0 + + # ----------------- + if status_code != PG_CTL__STATUS__OK: + errMsg = "Getting of a node status [data_dir is {0}] failed.".format(self__data_dir) + + raise ExecUtilException( + message=errMsg, + command=_params, + exit_code=status_code, + out=out, + error=error, + ) + + # ----------------- + assert status_code == PG_CTL__STATUS__OK + + if out == "": + __class__._throw_error__pg_ctl_returns_an_empty_string( + _params + ) + + C_PID_PREFIX = "(PID: " + + i = out.find(C_PID_PREFIX) + + if i == -1: + __class__._throw_error__pg_ctl_returns_an_unexpected_string( + out, + _params + ) + + assert i > 0 + assert i < len(out) + assert len(C_PID_PREFIX) <= len(out) + assert i <= len(out) - len(C_PID_PREFIX) + + i += len(C_PID_PREFIX) + start_pid_s = i + + while True: + if i == len(out): + __class__._throw_error__pg_ctl_returns_an_unexpected_string( + out, + _params + ) + + ch = out[i] + + if ch == ")": + break + + if ch.isdigit(): + i += 1 + continue + + __class__._throw_error__pg_ctl_returns_an_unexpected_string( + out, + _params + ) + assert False + + if i == start_pid_s: + __class__._throw_error__pg_ctl_returns_an_unexpected_string( + out, + _params + ) + + # TODO: Let's verify a length of pid string. + + pid = int(out[start_pid_s:i]) + + if pid == 0: + __class__._throw_error__pg_ctl_returns_a_zero_pid( + out, + _params + ) + + assert pid != 0 + return pid + + @staticmethod + def _throw_error__pg_ctl_returns_an_empty_string(_params): + errLines = [] + errLines.append("Utility pg_ctl returns empty string.") + errLines.append("Command line is {0}".format(_params)) + raise RuntimeError("\n".join(errLines)) + + @staticmethod + def _throw_error__pg_ctl_returns_an_unexpected_string(out, _params): + errLines = [] + errLines.append("Utility pg_ctl returns an unexpected string:") + errLines.append(out) + errLines.append("------------") + errLines.append("Command line is {0}".format(_params)) + raise RuntimeError("\n".join(errLines)) + + @staticmethod + def _throw_error__pg_ctl_returns_a_zero_pid(out, _params): + errLines = [] + errLines.append("Utility pg_ctl returns a zero pid. Output string is:") + errLines.append(out) + errLines.append("------------") + errLines.append("Command line is {0}".format(_params)) + raise RuntimeError("\n".join(errLines)) @property def auxiliary_pids(self): @@ -338,41 +462,84 @@ def version(self): return self._pg_version def _try_shutdown(self, max_attempts, with_force=False): + assert type(max_attempts) == int # noqa: E721 + assert type(with_force) == bool # noqa: E721 + assert max_attempts > 0 + attempts = 0 + + # try stopping server N times + while attempts < max_attempts: + attempts += 1 + try: + self.stop() + except ExecUtilException: + continue # one more time + except Exception: + eprint('cannot stop node {}'.format(self.name)) + break + + return # OK + + # If force stopping is enabled and PID is valid + if not with_force: + return + node_pid = self.pid + assert node_pid is not None + assert type(node_pid) == int # noqa: E721 - if node_pid > 0: - # try stopping server N times - while attempts < max_attempts: - try: - self.stop() - break # OK - except ExecUtilException: - pass # one more time - except Exception: - eprint('cannot stop node {}'.format(self.name)) - break - - attempts += 1 - - # If force stopping is enabled and PID is valid - if with_force and node_pid != 0: - # If we couldn't stop the node - p_status_output = self.os_ops.exec_command(cmd=f'ps -o pid= -p {node_pid}', shell=True, ignore_errors=True).decode('utf-8') - if self.status() != NodeStatus.Stopped and p_status_output and str(node_pid) in p_status_output: - try: - eprint(f'Force stopping node {self.name} with PID {node_pid}') - self.os_ops.kill(node_pid, signal.SIGKILL, expect_error=False) - except Exception: - # The node has already stopped - pass - - # Check that node stopped - print only column pid without headers - p_status_output = self.os_ops.exec_command(f'ps -o pid= -p {node_pid}', shell=True, ignore_errors=True).decode('utf-8') - if p_status_output and str(node_pid) in p_status_output: - eprint(f'Failed to stop node {self.name}.') - else: - eprint(f'Node {self.name} has been stopped successfully.') + if node_pid == 0: + return + + # TODO: [2025-02-28] It is really the old ugly code. We have to rewrite it! + + ps_command = ['ps', '-o', 'pid=', '-p', str(node_pid)] + + ps_output = self.os_ops.exec_command(cmd=ps_command, shell=True, ignore_errors=True).decode('utf-8') + assert type(ps_output) == str # noqa: E721 + + if ps_output == "": + return + + if ps_output != str(node_pid): + __class__._throw_bugcheck__unexpected_result_of_ps( + ps_output, + ps_command) + + try: + eprint('Force stopping node {0} with PID {1}'.format(self.name, node_pid)) + self.os_ops.kill(node_pid, signal.SIGKILL, expect_error=False) + except Exception: + # The node has already stopped + pass + + # Check that node stopped - print only column pid without headers + ps_output = self.os_ops.exec_command(cmd=ps_command, shell=True, ignore_errors=True).decode('utf-8') + assert type(ps_output) == str # noqa: E721 + + if ps_output == "": + eprint('Node {0} has been stopped successfully.'.format(self.name)) + return + + if ps_output == str(node_pid): + eprint('Failed to stop node {0}.'.format(self.name)) + return + + __class__._throw_bugcheck__unexpected_result_of_ps( + ps_output, + ps_command) + + @staticmethod + def _throw_bugcheck__unexpected_result_of_ps(result, cmd): + assert type(result) == str # noqa: E721 + assert type(cmd) == list # noqa: E721 + errLines = [] + errLines.append("[BUG CHECK] Unexpected result of command ps:") + errLines.append(result) + errLines.append("-----") + errLines.append("Command line is {0}".format(cmd)) + raise RuntimeError("\n".join(errLines)) def _assign_master(self, master): """NOTE: this is a private method!""" From b0f90d94d81a6470e2ef4f904e79ff1114d69bed Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Sat, 1 Mar 2025 11:13:27 +0300 Subject: [PATCH 403/475] [RemoteOperations] A call of mktemp is fixed (#202) When we define a template we have to use "-t" option. It forces mktemp to return a path instead name. The following methods of RemoteOperations are fixed: - mkdtemp - mkstemp --- testgres/operations/remote_ops.py | 46 +++++++++++++++++++------------ 1 file changed, 28 insertions(+), 18 deletions(-) diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index 51f5b2e8..2a4e5c78 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -247,32 +247,42 @@ def mkdtemp(self, prefix=None): - prefix (str): The prefix of the temporary directory name. """ if prefix: - command = ["ssh"] + self.ssh_args + [self.ssh_dest, f"mktemp -d {prefix}XXXXX"] + command = ["mktemp", "-d", "-t", prefix + "XXXXX"] else: - command = ["ssh"] + self.ssh_args + [self.ssh_dest, "mktemp -d"] + command = ["mktemp", "-d"] - result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + exit_status, result, error = self.exec_command(command, verbose=True, encoding=get_default_encoding(), ignore_errors=True) - if result.returncode == 0: - temp_dir = result.stdout.strip() - if not os.path.isabs(temp_dir): - temp_dir = os.path.join('/home', self.username, temp_dir) - return temp_dir - else: - raise ExecUtilException(f"Could not create temporary directory. Error: {result.stderr}") + assert type(result) == str # noqa: E721 + assert type(error) == str # noqa: E721 + + if exit_status != 0: + raise ExecUtilException("Could not create temporary directory. Error code: {0}. Error message: {1}".format(exit_status, error)) + + temp_dir = result.strip() + return temp_dir def mkstemp(self, prefix=None): + """ + Creates a temporary file in the remote server. + Args: + - prefix (str): The prefix of the temporary directory name. + """ if prefix: - temp_dir = self.exec_command("mktemp {}XXXXX".format(prefix), encoding=get_default_encoding()) + command = ["mktemp", "-t", prefix + "XXXXX"] else: - temp_dir = self.exec_command("mktemp", encoding=get_default_encoding()) + command = ["mktemp"] - if temp_dir: - if not os.path.isabs(temp_dir): - temp_dir = os.path.join('/home', self.username, temp_dir.strip()) - return temp_dir - else: - raise ExecUtilException("Could not create temporary directory.") + exit_status, result, error = self.exec_command(command, verbose=True, encoding=get_default_encoding(), ignore_errors=True) + + assert type(result) == str # noqa: E721 + assert type(error) == str # noqa: E721 + + if exit_status != 0: + raise ExecUtilException("Could not create temporary file. Error code: {0}. Error message: {1}".format(exit_status, error)) + + temp_file = result.strip() + return temp_file def copytree(self, src, dst): if not os.path.isabs(dst): From 71772122b6485a71c6831dbe11690d970c809f38 Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Sat, 1 Mar 2025 23:39:42 +0300 Subject: [PATCH 404/475] TestRemoteOperations::test_is_executable_true is corrected (#204) Let's test a real pg_config. --- tests/test_remote.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/test_remote.py b/tests/test_remote.py index 8b167e9f..e457de07 100755 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -8,7 +8,9 @@ from ..testgres import ExecUtilException from ..testgres import InvalidOperationException from ..testgres import RemoteOperations +from ..testgres import LocalOperations from ..testgres import ConnectionParams +from ..testgres import utils as testgres_utils class TestRemoteOperations: @@ -59,7 +61,11 @@ def test_is_executable_true(self): """ Test is_executable for an existing executable. """ - cmd = os.getenv('PG_CONFIG') + local_ops = LocalOperations() + cmd = testgres_utils.get_bin_path2(local_ops, "pg_config") + cmd = local_ops.exec_command([cmd, "--bindir"], encoding="utf-8") + cmd = cmd.rstrip() + cmd = os.path.join(cmd, "pg_config") response = self.operations.is_executable(cmd) assert response is True From 0ffd5f0c4c5ff682ffaec8ee7a6145d55e904aec Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Sat, 1 Mar 2025 23:40:29 +0300 Subject: [PATCH 405/475] Total refactoring of os_ops::execute_command (#203) * Total refactoring of os_ops::execute_command Main - We check only an exit code to detect an error. - If someone utility returns a result through an exit code, a caller side should set ignore_errors=true and process this case itself. - If expect_error is true and no errors occurred, we raise an InvalidOperationException. * The old behaviour of RaiseError.UtilityExitedWithNonZeroCode is restored Let's rollback the new code to avoid problems with probackup2' tests. --- testgres/operations/local_ops.py | 34 +++++----- testgres/operations/raise_error.py | 31 +++++---- testgres/operations/remote_ops.py | 101 +++++++++++++++++++---------- testgres/utils.py | 13 ++-- tests/test_local.py | 5 +- tests/test_remote.py | 10 +-- tests/test_simple_remote.py | 43 ++++++------ 7 files changed, 141 insertions(+), 96 deletions(-) diff --git a/testgres/operations/local_ops.py b/testgres/operations/local_ops.py index 91070fe7..51003174 100644 --- a/testgres/operations/local_ops.py +++ b/testgres/operations/local_ops.py @@ -23,20 +23,6 @@ from distutils import rmtree CMD_TIMEOUT_SEC = 60 -error_markers = [b'error', b'Permission denied', b'fatal'] -err_out_markers = [b'Failure'] - - -def has_errors(output=None, error=None): - if output: - if isinstance(output, str): - output = output.encode(get_default_encoding()) - return any(marker in output for marker in err_out_markers) - if error: - if isinstance(error, str): - error = error.encode(get_default_encoding()) - return any(marker in error for marker in error_markers) - return False class LocalOperations(OsOperations): @@ -134,19 +120,29 @@ def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False, process, output, error = self._run_command(cmd, shell, input, stdin, stdout, stderr, get_process, timeout, encoding) if get_process: return process - if not ignore_errors and ((process.returncode != 0 or has_errors(output=output, error=error)) and not expect_error): + + if expect_error: + if process.returncode == 0: + raise InvalidOperationException("We expected an execution error.") + elif ignore_errors: + pass + elif process.returncode == 0: + pass + else: + assert not expect_error + assert not ignore_errors + assert process.returncode != 0 RaiseError.UtilityExitedWithNonZeroCode( cmd=cmd, exit_code=process.returncode, msg_arg=error or output, error=error, - out=output - ) + out=output) if verbose: return process.returncode, output, error - else: - return output + + return output # Environment setup def environ(self, var_name): diff --git a/testgres/operations/raise_error.py b/testgres/operations/raise_error.py index 6031b238..0d14be5a 100644 --- a/testgres/operations/raise_error.py +++ b/testgres/operations/raise_error.py @@ -7,13 +7,27 @@ class RaiseError: def UtilityExitedWithNonZeroCode(cmd, exit_code, msg_arg, error, out): assert type(exit_code) == int # noqa: E721 - msg_arg_s = __class__._TranslateDataIntoString(msg_arg).strip() + msg_arg_s = __class__._TranslateDataIntoString(msg_arg) assert type(msg_arg_s) == str # noqa: E721 + msg_arg_s = msg_arg_s.strip() if msg_arg_s == "": msg_arg_s = "#no_error_message" - message = "Utility exited with non-zero code. Error: `" + msg_arg_s + "`" + message = "Utility exited with non-zero code (" + str(exit_code) + "). Error: `" + msg_arg_s + "`" + raise ExecUtilException( + message=message, + command=cmd, + exit_code=exit_code, + out=out, + error=error) + + @staticmethod + def CommandExecutionError(cmd, exit_code, message, error, out): + assert type(exit_code) == int # noqa: E721 + assert type(message) == str # noqa: E721 + assert message != "" + raise ExecUtilException( message=message, command=cmd, @@ -23,6 +37,9 @@ def UtilityExitedWithNonZeroCode(cmd, exit_code, msg_arg, error, out): @staticmethod def _TranslateDataIntoString(data): + if data is None: + return "" + if type(data) == bytes: # noqa: E721 return __class__._TranslateDataIntoString__FromBinary(data) @@ -38,13 +55,3 @@ def _TranslateDataIntoString__FromBinary(data): pass return "#cannot_decode_text" - - @staticmethod - def _BinaryIsASCII(data): - assert type(data) == bytes # noqa: E721 - - for b in data: - if not (b >= 0 and b <= 127): - return False - - return True diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index 2a4e5c78..dc392bee 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -100,41 +100,40 @@ def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False, return process try: - result, error = process.communicate(input=input_prepared, timeout=timeout) + output, error = process.communicate(input=input_prepared, timeout=timeout) except subprocess.TimeoutExpired: process.kill() raise ExecUtilException("Command timed out after {} seconds.".format(timeout)) - exit_status = process.returncode - - assert type(result) == bytes # noqa: E721 + assert type(output) == bytes # noqa: E721 assert type(error) == bytes # noqa: E721 - if not error: - error_found = False - else: - error_found = exit_status != 0 or any( - marker in error for marker in [b'error', b'Permission denied', b'fatal', b'No such file or directory'] - ) - - assert type(error_found) == bool # noqa: E721 - if encoding: - result = result.decode(encoding) + output = output.decode(encoding) error = error.decode(encoding) - if not ignore_errors and error_found and not expect_error: + if expect_error: + if process.returncode == 0: + raise InvalidOperationException("We expected an execution error.") + elif ignore_errors: + pass + elif process.returncode == 0: + pass + else: + assert not expect_error + assert not ignore_errors + assert process.returncode != 0 RaiseError.UtilityExitedWithNonZeroCode( cmd=cmd, - exit_code=exit_status, + exit_code=process.returncode, msg_arg=error, error=error, - out=result) + out=output) if verbose: - return exit_status, result, error - else: - return result + return process.returncode, output, error + + return output # Environment setup def environ(self, var_name: str) -> str: @@ -165,8 +164,30 @@ def find_executable(self, executable): def is_executable(self, file): # Check if the file is executable - is_exec = self.exec_command("test -x {} && echo OK".format(file)) - return is_exec == b"OK\n" + command = ["test", "-x", file] + + exit_status, output, error = self.exec_command(cmd=command, encoding=get_default_encoding(), ignore_errors=True, verbose=True) + + assert type(output) == str # noqa: E721 + assert type(error) == str # noqa: E721 + + if exit_status == 0: + return True + + if exit_status == 1: + return False + + errMsg = "Test operation returns an unknown result code: {0}. File name is [{1}].".format( + exit_status, + file) + + RaiseError.CommandExecutionError( + cmd=command, + exit_code=exit_status, + msg_arg=errMsg, + error=error, + out=output + ) def set_env(self, var_name: str, var_val: str): """ @@ -251,15 +272,21 @@ def mkdtemp(self, prefix=None): else: command = ["mktemp", "-d"] - exit_status, result, error = self.exec_command(command, verbose=True, encoding=get_default_encoding(), ignore_errors=True) + exec_exitcode, exec_output, exec_error = self.exec_command(command, verbose=True, encoding=get_default_encoding(), ignore_errors=True) - assert type(result) == str # noqa: E721 - assert type(error) == str # noqa: E721 + assert type(exec_exitcode) == int # noqa: E721 + assert type(exec_output) == str # noqa: E721 + assert type(exec_error) == str # noqa: E721 - if exit_status != 0: - raise ExecUtilException("Could not create temporary directory. Error code: {0}. Error message: {1}".format(exit_status, error)) + if exec_exitcode != 0: + RaiseError.CommandExecutionError( + cmd=command, + exit_code=exec_exitcode, + message="Could not create temporary directory.", + error=exec_error, + out=exec_output) - temp_dir = result.strip() + temp_dir = exec_output.strip() return temp_dir def mkstemp(self, prefix=None): @@ -273,15 +300,21 @@ def mkstemp(self, prefix=None): else: command = ["mktemp"] - exit_status, result, error = self.exec_command(command, verbose=True, encoding=get_default_encoding(), ignore_errors=True) + exec_exitcode, exec_output, exec_error = self.exec_command(command, verbose=True, encoding=get_default_encoding(), ignore_errors=True) - assert type(result) == str # noqa: E721 - assert type(error) == str # noqa: E721 + assert type(exec_exitcode) == int # noqa: E721 + assert type(exec_output) == str # noqa: E721 + assert type(exec_error) == str # noqa: E721 - if exit_status != 0: - raise ExecUtilException("Could not create temporary file. Error code: {0}. Error message: {1}".format(exit_status, error)) + if exec_exitcode != 0: + RaiseError.CommandExecutionError( + cmd=command, + exit_code=exec_exitcode, + message="Could not create temporary file.", + error=exec_error, + out=exec_output) - temp_file = result.strip() + temp_file = exec_output.strip() return temp_file def copytree(self, src, dst): diff --git a/testgres/utils.py b/testgres/utils.py index 76d42b02..093eaff6 100644 --- a/testgres/utils.py +++ b/testgres/utils.py @@ -18,6 +18,7 @@ from .config import testgres_config as tconf from .operations.os_ops import OsOperations from .operations.remote_ops import RemoteOperations +from .operations.helpers import Helpers as OsHelpers # rows returned by PG_CONFIG _pg_config_data = {} @@ -79,13 +80,13 @@ def execute_utility2(os_ops: OsOperations, args, logfile=None, verbose=False, ig assert type(verbose) == bool # noqa: E721 assert type(ignore_errors) == bool # noqa: E721 - exit_status, out, error = os_ops.exec_command(args, verbose=True, ignore_errors=ignore_errors) - # decode result + exit_status, out, error = os_ops.exec_command( + args, + verbose=True, + ignore_errors=ignore_errors, + encoding=OsHelpers.GetDefaultEncoding()) + out = '' if not out else out - if isinstance(out, bytes): - out = out.decode('utf-8') - if isinstance(error, bytes): - error = error.decode('utf-8') # write new log entry if possible if logfile: diff --git a/tests/test_local.py b/tests/test_local.py index 60a96c18..ee5e19a0 100644 --- a/tests/test_local.py +++ b/tests/test_local.py @@ -40,10 +40,11 @@ def test_exec_command_failure(self): try: self.operations.exec_command(cmd, wait_exit=True, shell=True) except ExecUtilException as e: - error = e.message + assert e.message == "Utility exited with non-zero code (127). Error: `/bin/sh: 1: nonexistent_command: not found`" + assert type(e.error) == bytes # noqa: E721 + assert e.error.strip() == b"/bin/sh: 1: nonexistent_command: not found" break raise Exception("We wait an exception!") - assert error == "Utility exited with non-zero code. Error: `/bin/sh: 1: nonexistent_command: not found`" def test_exec_command_failure__expect_error(self): """ diff --git a/tests/test_remote.py b/tests/test_remote.py index e457de07..6114e29e 100755 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -40,10 +40,11 @@ def test_exec_command_failure(self): try: self.operations.exec_command(cmd, verbose=True, wait_exit=True) except ExecUtilException as e: - error = e.message + assert e.message == "Utility exited with non-zero code (127). Error: `bash: line 1: nonexistent_command: command not found`" + assert type(e.error) == bytes # noqa: E721 + assert e.error.strip() == b"bash: line 1: nonexistent_command: command not found" break raise Exception("We wait an exception!") - assert error == 'Utility exited with non-zero code. Error: `bash: line 1: nonexistent_command: command not found`' def test_exec_command_failure__expect_error(self): """ @@ -114,10 +115,11 @@ def test_makedirs_and_rmdirs_failure(self): try: self.operations.rmdirs(path, verbose=True) except ExecUtilException as e: - error = e.message + assert e.message == "Utility exited with non-zero code (1). Error: `rm: cannot remove '/root/test_dir': Permission denied`" + assert type(e.error) == bytes # noqa: E721 + assert e.error.strip() == b"rm: cannot remove '/root/test_dir': Permission denied" break raise Exception("We wait an exception!") - assert error == "Utility exited with non-zero code. Error: `rm: cannot remove '/root/test_dir': Permission denied`" def test_listdir(self): """ diff --git a/tests/test_simple_remote.py b/tests/test_simple_remote.py index d4a28a2b..74b10635 100755 --- a/tests/test_simple_remote.py +++ b/tests/test_simple_remote.py @@ -178,27 +178,32 @@ def test_init__unk_LANG_and_LC_CTYPE(self): assert os.environ.get("LC_CTYPE") == unkData[1] assert not ("LC_COLLATE" in os.environ.keys()) - while True: + assert os.getenv('LANG') == unkData[0] + assert os.getenv('LANGUAGE') is None + assert os.getenv('LC_CTYPE') == unkData[1] + assert os.getenv('LC_COLLATE') is None + + exc: ExecUtilException = None + with __class__.helper__get_node() as node: try: - with __class__.helper__get_node(): - pass - except ExecUtilException as e: - # - # Example of an error message: - # - # warning: setlocale: LC_CTYPE: cannot change locale (UNKNOWN_CTYPE): No such file or directory - # postgres (PostgreSQL) 14.12 - # - errMsg = str(e) - - logging.info("Error message is: {0}".format(errMsg)) - - assert "LC_CTYPE" in errMsg - assert unkData[1] in errMsg - assert "warning: setlocale: LC_CTYPE: cannot change locale (" + unkData[1] + "): No such file or directory" in errMsg - assert ("postgres" in errMsg) or ("PostgreSQL" in errMsg) - break + node.init() # IT RAISES! + except InitNodeException as e: + exc = e.__cause__ + assert exc is not None + assert isinstance(exc, ExecUtilException) + + if exc is None: raise Exception("We expected an error!") + + assert isinstance(exc, ExecUtilException) + + errMsg = str(exc) + logging.info("Error message is {0}: {1}".format(type(exc).__name__, errMsg)) + + assert "warning: setlocale: LC_CTYPE: cannot change locale (" + unkData[1] + ")" in errMsg + assert "initdb: error: invalid locale settings; check LANG and LC_* environment variables" in errMsg + continue + finally: __class__.helper__restore_envvar("LANG", prev_LANG) __class__.helper__restore_envvar("LANGUAGE", prev_LANGUAGE) From 7abca7f5e36ea20c6d8dd5dcc29b34ca5c9090c1 Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Sun, 2 Mar 2025 17:33:10 +0300 Subject: [PATCH 406/475] xxx::test_logging is corrected (local, remote) (#205) - these tests configure logging wrong and create the conflicts with root logger - these tests (local and remote) conflict with each other --- tests/test_simple.py | 142 +++++++++++++++++++++------------- tests/test_simple_remote.py | 147 +++++++++++++++++++++++------------- 2 files changed, 184 insertions(+), 105 deletions(-) diff --git a/tests/test_simple.py b/tests/test_simple.py index 6c433cd4..37c3db44 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -8,8 +8,8 @@ import pytest import psutil import platform - -import logging.config +import logging +import uuid from contextlib import contextmanager from shutil import rmtree @@ -718,55 +718,95 @@ def test_poll_query_until(self): node.poll_query_until('select true') def test_logging(self): - logfile = tempfile.NamedTemporaryFile('w', delete=True) - - log_conf = { - 'version': 1, - 'handlers': { - 'file': { - 'class': 'logging.FileHandler', - 'filename': logfile.name, - 'formatter': 'base_format', - 'level': logging.DEBUG, - }, - }, - 'formatters': { - 'base_format': { - 'format': '%(node)-5s: %(message)s', - }, - }, - 'root': { - 'handlers': ('file', ), - 'level': 'DEBUG', - }, - } - - logging.config.dictConfig(log_conf) - - with scoped_config(use_python_logging=True): - node_name = 'master' - - with get_new_node(name=node_name) as master: - master.init().start() - - # execute a dummy query a few times - for i in range(20): - master.execute('select 1') - time.sleep(0.01) - - # let logging worker do the job - time.sleep(0.1) - - # check that master's port is found - with open(logfile.name, 'r') as log: - lines = log.readlines() - assert (any(node_name in s for s in lines)) - - # test logger after stop/start/restart - master.stop() - master.start() - master.restart() - assert (master._logger.is_alive()) + C_MAX_ATTEMPTS = 50 + # This name is used for testgres logging, too. + C_NODE_NAME = "testgres_tests." + __class__.__name__ + "test_logging-master-" + uuid.uuid4().hex + + logging.info("Node name is [{0}]".format(C_NODE_NAME)) + + with tempfile.NamedTemporaryFile('w', delete=True) as logfile: + formatter = logging.Formatter(fmt="%(node)-5s: %(message)s") + handler = logging.FileHandler(filename=logfile.name) + handler.formatter = formatter + logger = logging.getLogger(C_NODE_NAME) + assert logger is not None + assert len(logger.handlers) == 0 + + try: + # It disables to log on the root level + logger.propagate = False + logger.addHandler(handler) + + with scoped_config(use_python_logging=True): + with get_new_node(name=C_NODE_NAME) as master: + logging.info("Master node is initilizing") + master.init() + + logging.info("Master node is starting") + master.start() + + logging.info("Dummy query is executed a few times") + for _ in range(20): + master.execute('select 1') + time.sleep(0.01) + + # let logging worker do the job + time.sleep(0.1) + + logging.info("Master node log file is checking") + nAttempt = 0 + + while True: + assert nAttempt <= C_MAX_ATTEMPTS + if nAttempt == C_MAX_ATTEMPTS: + raise Exception("Test failed!") + + # let logging worker do the job + time.sleep(0.1) + + nAttempt += 1 + + logging.info("Attempt {0}".format(nAttempt)) + + # check that master's port is found + with open(logfile.name, 'r') as log: + lines = log.readlines() + + assert lines is not None + assert type(lines) == list # noqa: E721 + + def LOCAL__test_lines(): + for s in lines: + if any(C_NODE_NAME in s for s in lines): + logging.info("OK. We found the node_name in a line \"{0}\"".format(s)) + return True + return False + + if LOCAL__test_lines(): + break + + logging.info("Master node log file does not have an expected information.") + continue + + # test logger after stop/start/restart + logging.info("Master node is stopping...") + master.stop() + logging.info("Master node is staring again...") + master.start() + logging.info("Master node is restaring...") + master.restart() + assert (master._logger.is_alive()) + finally: + # It is a hack code to logging cleanup + logging._acquireLock() + assert logging.Logger.manager is not None + assert C_NODE_NAME in logging.Logger.manager.loggerDict.keys() + logging.Logger.manager.loggerDict.pop(C_NODE_NAME, None) + assert not (C_NODE_NAME in logging.Logger.manager.loggerDict.keys()) + assert not (handler in logging._handlers.values()) + logging._releaseLock() + # GO HOME! + return def test_pgbench(self): __class__.helper__skip_test_if_util_not_exist("pgbench") diff --git a/tests/test_simple_remote.py b/tests/test_simple_remote.py index 74b10635..a62085ce 100755 --- a/tests/test_simple_remote.py +++ b/tests/test_simple_remote.py @@ -8,8 +8,8 @@ import six import pytest import psutil - -import logging.config +import logging +import uuid from contextlib import contextmanager @@ -788,56 +788,95 @@ def test_poll_query_until(self): node.poll_query_until('select true') def test_logging(self): - # FAIL - logfile = tempfile.NamedTemporaryFile('w', delete=True) - - log_conf = { - 'version': 1, - 'handlers': { - 'file': { - 'class': 'logging.FileHandler', - 'filename': logfile.name, - 'formatter': 'base_format', - 'level': logging.DEBUG, - }, - }, - 'formatters': { - 'base_format': { - 'format': '%(node)-5s: %(message)s', - }, - }, - 'root': { - 'handlers': ('file',), - 'level': 'DEBUG', - }, - } - - logging.config.dictConfig(log_conf) - - with scoped_config(use_python_logging=True): - node_name = 'master' - - with get_remote_node(name=node_name) as master: - master.init().start() - - # execute a dummy query a few times - for i in range(20): - master.execute('select 1') - time.sleep(0.01) - - # let logging worker do the job - time.sleep(0.1) - - # check that master's port is found - with open(logfile.name, 'r') as log: - lines = log.readlines() - assert (any(node_name in s for s in lines)) - - # test logger after stop/start/restart - master.stop() - master.start() - master.restart() - assert (master._logger.is_alive()) + C_MAX_ATTEMPTS = 50 + # This name is used for testgres logging, too. + C_NODE_NAME = "testgres_tests." + __class__.__name__ + "test_logging-master-" + uuid.uuid4().hex + + logging.info("Node name is [{0}]".format(C_NODE_NAME)) + + with tempfile.NamedTemporaryFile('w', delete=True) as logfile: + formatter = logging.Formatter(fmt="%(node)-5s: %(message)s") + handler = logging.FileHandler(filename=logfile.name) + handler.formatter = formatter + logger = logging.getLogger(C_NODE_NAME) + assert logger is not None + assert len(logger.handlers) == 0 + + try: + # It disables to log on the root level + logger.propagate = False + logger.addHandler(handler) + + with scoped_config(use_python_logging=True): + with __class__.helper__get_node(name=C_NODE_NAME) as master: + logging.info("Master node is initilizing") + master.init() + + logging.info("Master node is starting") + master.start() + + logging.info("Dummy query is executed a few times") + for _ in range(20): + master.execute('select 1') + time.sleep(0.01) + + # let logging worker do the job + time.sleep(0.1) + + logging.info("Master node log file is checking") + nAttempt = 0 + + while True: + assert nAttempt <= C_MAX_ATTEMPTS + if nAttempt == C_MAX_ATTEMPTS: + raise Exception("Test failed!") + + # let logging worker do the job + time.sleep(0.1) + + nAttempt += 1 + + logging.info("Attempt {0}".format(nAttempt)) + + # check that master's port is found + with open(logfile.name, 'r') as log: + lines = log.readlines() + + assert lines is not None + assert type(lines) == list # noqa: E721 + + def LOCAL__test_lines(): + for s in lines: + if any(C_NODE_NAME in s for s in lines): + logging.info("OK. We found the node_name in a line \"{0}\"".format(s)) + return True + return False + + if LOCAL__test_lines(): + break + + logging.info("Master node log file does not have an expected information.") + continue + + # test logger after stop/start/restart + logging.info("Master node is stopping...") + master.stop() + logging.info("Master node is staring again...") + master.start() + logging.info("Master node is restaring...") + master.restart() + assert (master._logger.is_alive()) + finally: + # It is a hack code to logging cleanup + logging._acquireLock() + assert logging.Logger.manager is not None + assert C_NODE_NAME in logging.Logger.manager.loggerDict.keys() + logging.Logger.manager.loggerDict.pop(C_NODE_NAME, None) + assert not (C_NODE_NAME in logging.Logger.manager.loggerDict.keys()) + assert not (handler in logging._handlers.values()) + logging._releaseLock() + # GO HOME! + return def test_pgbench(self): __class__.helper__skip_test_if_util_not_exist("pgbench") @@ -1184,9 +1223,9 @@ def test_child_process_dies(self): break @staticmethod - def helper__get_node(): + def helper__get_node(name=None): assert __class__.sm_conn_params is not None - return get_remote_node(conn_params=__class__.sm_conn_params) + return get_remote_node(name=name, conn_params=__class__.sm_conn_params) @staticmethod def helper__restore_envvar(name, prev_value): From e1a5bb451186c871df56f79f78af78dd63d3381e Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Mon, 3 Mar 2025 12:37:26 +0300 Subject: [PATCH 407/475] Updating of CI-tests (#197) * Test dockerfile for ubuntu 24.04 is added * Cleanup (Dockerfile--ubuntu-24_04) * Cleanup. Including of 'postgres' in 'sudo' group is not required. * [run_tests.sh] A right way for obtaining of BINDIR and PG_CONFIG is used A problem was detected in container with Ubuntu 24.04 tests works with "/usr/bin/pg_config" but real pg_config is "/usr/lib/postgresql/17/bin/pg_config" To resovle this problem we will call "pg_config --bindir" and use it result for BINDIR and PG_CONFIG. * Dockerfile--ubuntu-24_04 is updated Let's use /pg/testgres/run_tests.sh directly. * Dockerfile--ubuntu-24_04 is updated (cleanup) curl is installing twice. * Dockerfile--ubuntu-24_04 is updated (cleanup) [del] musl-dev [del] mc * Dockerfile--ubuntu-24_04 is formatted * CI-test on Ubuntu 24.04 is added. * Dockerfile--std.tmpl is updated (refactoring) /pg/testgres/run_tests.sh is used directly. * Dockerfile--ubuntu-24_04.tmpl is updated * PostgresNode::pid is improved - We do multiple attempts to read pid file. - We process a case when we see that node is stopped between test and read. - We process a case when pid-file is empty. * PostgresNode::pid is updated Assert is added. * execute_utility2 is updated (ignore_errors) - New parameters "ignore_errors" is added. Default value is False. - Asserts are added. * PostgresNode::_try_shutdown is rewrited (normalization) * PostgresNode::pid uses the data from "pg_ctl status" output. * PostgresNode::_try_shutdown is correct (return None) This method returns nothing (None). * [RemoteOperations] A call of mktemp is fixed When we define a template we have to use "-t" option. It forces mktemp to return a path instead name. The following methods of RemoteOperations are fixed: - mkdtemp - mkstemp * Total refactoring of os_ops::execute_command Main - We check only an exit code to detect an error. - If someone utility returns a result through an exit code, a caller side should set ignore_errors=true and process this case itself. - If expect_error is true and no errors occurred, we raise an InvalidOperationException. * Dockerfile--ubuntu-24_04.tmpl is updated The folder "home/postgres" is not required now. * The old behaviour of RaiseError.UtilityExitedWithNonZeroCode is restored Let's rollback the new code to avoid problems with probackup2' tests. * TestRemoteOperations::test_is_executable_true is corrected Let's test a real pg_config. * xxx::test_logging is corrected (local, remote) - these tests configure logging wrong and create the conflicts with root logger - these tests (local and remote) conflict with each other * TEST_FILTER is added * CI on Ubuntu 24.04 runs all the tests --- .travis.yml | 28 ++++------ Dockerfile.tmpl => Dockerfile--std.tmpl | 6 +-- Dockerfile--ubuntu-24_04.tmpl | 69 +++++++++++++++++++++++++ mk_dockerfile.sh | 2 +- run_tests.sh | 7 +-- 5 files changed, 86 insertions(+), 26 deletions(-) rename Dockerfile.tmpl => Dockerfile--std.tmpl (80%) create mode 100644 Dockerfile--ubuntu-24_04.tmpl diff --git a/.travis.yml b/.travis.yml index 6f63a67b..4110835a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,23 +20,17 @@ notifications: on_failure: always env: - - PYTHON_VERSION=3 PG_VERSION=17 - - PYTHON_VERSION=3 PG_VERSION=16 - - PYTHON_VERSION=3 PG_VERSION=15 - - PYTHON_VERSION=3 PG_VERSION=14 - - PYTHON_VERSION=3 PG_VERSION=13 - - PYTHON_VERSION=3 PG_VERSION=12 - - PYTHON_VERSION=3 PG_VERSION=11 - - PYTHON_VERSION=3 PG_VERSION=10 -# - PYTHON_VERSION=3 PG_VERSION=9.6 -# - PYTHON_VERSION=3 PG_VERSION=9.5 -# - PYTHON_VERSION=3 PG_VERSION=9.4 -# - PYTHON_VERSION=2 PG_VERSION=10 -# - PYTHON_VERSION=2 PG_VERSION=9.6 -# - PYTHON_VERSION=2 PG_VERSION=9.5 -# - PYTHON_VERSION=2 PG_VERSION=9.4 + - TEST_PLATFORM=std PYTHON_VERSION=3 PG_VERSION=17 + - TEST_PLATFORM=std PYTHON_VERSION=3 PG_VERSION=16 + - TEST_PLATFORM=std PYTHON_VERSION=3 PG_VERSION=15 + - TEST_PLATFORM=std PYTHON_VERSION=3 PG_VERSION=14 + - TEST_PLATFORM=std PYTHON_VERSION=3 PG_VERSION=13 + - TEST_PLATFORM=std PYTHON_VERSION=3 PG_VERSION=12 + - TEST_PLATFORM=std PYTHON_VERSION=3 PG_VERSION=11 + - TEST_PLATFORM=std PYTHON_VERSION=3 PG_VERSION=10 + - TEST_PLATFORM=ubuntu-24_04 PYTHON_VERSION=3 PG_VERSION=17 matrix: allow_failures: - - env: PYTHON_VERSION=3 PG_VERSION=11 - - env: PYTHON_VERSION=3 PG_VERSION=10 + - env: TEST_PLATFORM=std PYTHON_VERSION=3 PG_VERSION=11 + - env: TEST_PLATFORM=std PYTHON_VERSION=3 PG_VERSION=10 diff --git a/Dockerfile.tmpl b/Dockerfile--std.tmpl similarity index 80% rename from Dockerfile.tmpl rename to Dockerfile--std.tmpl index dc5878b6..d844c9a3 100644 --- a/Dockerfile.tmpl +++ b/Dockerfile--std.tmpl @@ -11,13 +11,9 @@ RUN if [ "${PYTHON_VERSION}" = "3" ] ; then \ fi ENV LANG=C.UTF-8 -RUN mkdir -p /pg -COPY run_tests.sh /run.sh -RUN chmod 755 /run.sh - ADD . /pg/testgres WORKDIR /pg/testgres RUN chown -R postgres:postgres /pg USER postgres -ENTRYPOINT PYTHON_VERSION=${PYTHON_VERSION} /run.sh +ENTRYPOINT PYTHON_VERSION=${PYTHON_VERSION} bash run_tests.sh diff --git a/Dockerfile--ubuntu-24_04.tmpl b/Dockerfile--ubuntu-24_04.tmpl new file mode 100644 index 00000000..99be5343 --- /dev/null +++ b/Dockerfile--ubuntu-24_04.tmpl @@ -0,0 +1,69 @@ +FROM ubuntu:24.04 + +RUN apt update +RUN apt install -y sudo curl ca-certificates postgresql-common + +RUN bash /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh -y + +RUN install -d /usr/share/postgresql-common/pgdg +RUN curl -o /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc --fail https://www.postgresql.org/media/keys/ACCC4CF8.asc + +# It does not work +# RUN sh -c 'echo "deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' + +RUN apt update +RUN apt install -y postgresql-${PG_VERSION} + +RUN apt install -y python3 python3-dev python3-virtualenv +# RUN apt install -y mc + +# It is required for psycopg2 +RUN apt install -y libpq-dev +RUN apt install -y openssh-server + +# [2025-02-26] It adds the user 'postgres' in the group 'sudo' +# [2025-02-27] It is not required. +# RUN adduser postgres sudo + +ADD . /pg/testgres +WORKDIR /pg/testgres +RUN chown -R postgres /pg + +EXPOSE 22 + +RUN ssh-keygen -A + +# It enables execution of "sudo service ssh start" without password +RUN sh -c "echo postgres ALL=NOPASSWD:/usr/sbin/service ssh start" >> /etc/sudoers + +USER postgres + +ENV LANG=C.UTF-8 + +#ENTRYPOINT PYTHON_VERSION=3.12 /run.sh +ENTRYPOINT sh -c " \ +#set -eux; \ +echo HELLO FROM ENTRYPOINT; \ +echo HOME DIR IS [`realpath ~/`]; \ +echo POINT 1; \ +chmod go-w /var/lib/postgresql; \ +echo POINT 1.5; \ +mkdir -p ~/.ssh; \ +echo POINT 2; \ +service ssh enable; \ +echo POINT 3; \ +sudo service ssh start; \ +echo POINT 4; \ +ssh-keyscan -H localhost >> ~/.ssh/known_hosts; \ +echo POINT 5; \ +ssh-keyscan -H 127.0.0.1 >> ~/.ssh/known_hosts; \ +echo POINT 6; \ +ssh-keygen -t rsa -f ~/.ssh/id_rsa -q -N ''; \ +echo ----; \ +cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys; \ +echo ----; \ +chmod 600 ~/.ssh/authorized_keys; \ +echo ----; \ +ls -la ~/.ssh/; \ +echo ----; \ +TEST_FILTER="" PYTHON_VERSION=${PYTHON_VERSION} bash run_tests.sh;" diff --git a/mk_dockerfile.sh b/mk_dockerfile.sh index d2aa3a8a..8f7876a3 100755 --- a/mk_dockerfile.sh +++ b/mk_dockerfile.sh @@ -1,2 +1,2 @@ set -eu -sed -e 's/${PYTHON_VERSION}/'${PYTHON_VERSION}/g -e 's/${PG_VERSION}/'${PG_VERSION}/g Dockerfile.tmpl > Dockerfile +sed -e 's/${PYTHON_VERSION}/'${PYTHON_VERSION}/g -e 's/${PG_VERSION}/'${PG_VERSION}/g Dockerfile--${TEST_PLATFORM}.tmpl > Dockerfile diff --git a/run_tests.sh b/run_tests.sh index 5cbbac60..021f9d9f 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -4,6 +4,7 @@ set -eux +if [ -z ${TEST_FILTER+x} ]; then export TEST_FILTER="TestgresTests"; fi # choose python version echo python version is $PYTHON_VERSION @@ -38,19 +39,19 @@ rm -f $COVERAGE_FILE # run tests (PATH) -time coverage run -a -m pytest -l -v -n 4 -k "TestgresTests" +time coverage run -a -m pytest -l -v -n 4 -k "${TEST_FILTER}" # run tests (PG_BIN) time \ PG_BIN=$(pg_config --bindir) \ - coverage run -a -m pytest -l -v -n 4 -k "TestgresTests" + coverage run -a -m pytest -l -v -n 4 -k "${TEST_FILTER}" # run tests (PG_CONFIG) time \ PG_CONFIG=$(pg_config --bindir)/pg_config \ - coverage run -a -m pytest -l -v -n 4 -k "TestgresTests" + coverage run -a -m pytest -l -v -n 4 -k "${TEST_FILTER}" # show coverage From 3a1d08b3156b56609d02568d34771a6a9a535aa8 Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Tue, 4 Mar 2025 07:06:08 +0300 Subject: [PATCH 408/475] RemoteOperations::path_exists is updated (#206) - command is passed through list - we process all the result codes of test --- testgres/operations/remote_ops.py | 26 ++++++++++++++++++++++++-- tests/test_remote.py | 26 ++++++++++++++++---------- 2 files changed, 40 insertions(+), 12 deletions(-) diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index dc392bee..60d5265c 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -247,8 +247,30 @@ def listdir(self, path): return result.splitlines() def path_exists(self, path): - result = self.exec_command("test -e {}; echo $?".format(path), encoding=get_default_encoding()) - return int(result.strip()) == 0 + command = ["test", "-e", path] + + exit_status, output, error = self.exec_command(cmd=command, encoding=get_default_encoding(), ignore_errors=True, verbose=True) + + assert type(output) == str # noqa: E721 + assert type(error) == str # noqa: E721 + + if exit_status == 0: + return True + + if exit_status == 1: + return False + + errMsg = "Test operation returns an unknown result code: {0}. Path is [{1}].".format( + exit_status, + path) + + RaiseError.CommandExecutionError( + cmd=command, + exit_code=exit_status, + msg_arg=errMsg, + error=error, + out=output + ) @property def pathsep(self): diff --git a/tests/test_remote.py b/tests/test_remote.py index 6114e29e..85e65c24 100755 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -130,23 +130,29 @@ def test_listdir(self): assert isinstance(files, list) - def test_path_exists_true(self): + def test_path_exists_true__directory(self): """ - Test path_exists for an existing path. + Test path_exists for an existing directory. """ - path = "/etc" - response = self.operations.path_exists(path) + assert self.operations.path_exists("/etc") is True - assert response is True + def test_path_exists_true__file(self): + """ + Test path_exists for an existing file. + """ + assert self.operations.path_exists(__file__) is True - def test_path_exists_false(self): + def test_path_exists_false__directory(self): """ - Test path_exists for a non-existing path. + Test path_exists for a non-existing directory. """ - path = "/nonexistent_path" - response = self.operations.path_exists(path) + assert self.operations.path_exists("/nonexistent_path") is False - assert response is False + def test_path_exists_false__file(self): + """ + Test path_exists for a non-existing file. + """ + assert self.operations.path_exists("/etc/nonexistent_path.txt") is False def test_write_text_file(self): """ From ddcaea03a1d69cca22a172aeedbef79ba82f5d40 Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Tue, 4 Mar 2025 10:56:30 +0300 Subject: [PATCH 409/475] os_ops::rmdirs (local, remote) was refactored (#207) * os_ops::rmdirs (local, remote) was refactored LocalOperations::rmdirs - parameter 'retries' was remaned with 'attempts' - if ignore_errors we raise an error RemoteOperations::rmdirs - parameter 'verbose' was removed - method returns bool - we prevent to delete a file * [TestRemoteOperations] New tests for rmdirs are added. * test_pg_ctl_wait_option (local, remote) is corrected --- testgres/operations/local_ops.py | 41 ++++++++++---- testgres/operations/remote_ops.py | 42 +++++++++++--- tests/test_remote.py | 92 +++++++++++++++++++++++++++---- tests/test_simple.py | 65 ++++++++++++++++++---- tests/test_simple_remote.py | 65 ++++++++++++++++++---- 5 files changed, 255 insertions(+), 50 deletions(-) diff --git a/testgres/operations/local_ops.py b/testgres/operations/local_ops.py index 51003174..0fa7d0ad 100644 --- a/testgres/operations/local_ops.py +++ b/testgres/operations/local_ops.py @@ -174,7 +174,8 @@ def makedirs(self, path, remove_existing=False): except FileExistsError: pass - def rmdirs(self, path, ignore_errors=True, retries=3, delay=1): + # [2025-02-03] Old name of parameter attempts is "retries". + def rmdirs(self, path, ignore_errors=True, attempts=3, delay=1): """ Removes a directory and its contents, retrying on failure. @@ -183,18 +184,38 @@ def rmdirs(self, path, ignore_errors=True, retries=3, delay=1): :param retries: Number of attempts to remove the directory. :param delay: Delay between attempts in seconds. """ - for attempt in range(retries): + assert type(path) == str # noqa: E721 + assert type(ignore_errors) == bool # noqa: E721 + assert type(attempts) == int # noqa: E721 + assert type(delay) == int or type(delay) == float # noqa: E721 + assert attempts > 0 + assert delay >= 0 + + attempt = 0 + while True: + assert attempt < attempts + attempt += 1 try: - rmtree(path, ignore_errors=ignore_errors) - if not os.path.exists(path): - return True + rmtree(path) except FileNotFoundError: - return True + pass except Exception as e: - logging.error(f"Error: Failed to remove directory {path} on attempt {attempt + 1}: {e}") - time.sleep(delay) - logging.error(f"Error: Failed to remove directory {path} after {retries} attempts.") - return False + if attempt < attempt: + errMsg = "Failed to remove directory {0} on attempt {1} ({2}): {3}".format( + path, attempt, type(e).__name__, e + ) + logging.warning(errMsg) + time.sleep(delay) + continue + + assert attempt == attempts + if not ignore_errors: + raise + + return False + + # OK! + return True def listdir(self, path): return os.listdir(path) diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index 60d5265c..767df567 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -4,6 +4,7 @@ import subprocess import tempfile import io +import logging # we support both pg8000 and psycopg2 try: @@ -222,20 +223,45 @@ def makedirs(self, path, remove_existing=False): raise Exception("Couldn't create dir {} because of error {}".format(path, error)) return result - def rmdirs(self, path, verbose=False, ignore_errors=True): + def rmdirs(self, path, ignore_errors=True): """ Remove a directory in the remote server. Args: - path (str): The path to the directory to be removed. - - verbose (bool): If True, return exit status, result, and error. - ignore_errors (bool): If True, do not raise error if directory does not exist. """ - cmd = "rm -rf {}".format(path) - exit_status, result, error = self.exec_command(cmd, verbose=True) - if verbose: - return exit_status, result, error - else: - return result + assert type(path) == str # noqa: E721 + assert type(ignore_errors) == bool # noqa: E721 + + # ENOENT = 2 - No such file or directory + # ENOTDIR = 20 - Not a directory + + cmd1 = [ + "if", "[", "-d", path, "]", ";", + "then", "rm", "-rf", path, ";", + "elif", "[", "-e", path, "]", ";", + "then", "{", "echo", "cannot remove '" + path + "': it is not a directory", ">&2", ";", "exit", "20", ";", "}", ";", + "else", "{", "echo", "directory '" + path + "' does not exist", ">&2", ";", "exit", "2", ";", "}", ";", + "fi" + ] + + cmd2 = ["sh", "-c", subprocess.list2cmdline(cmd1)] + + try: + self.exec_command(cmd2, encoding=Helpers.GetDefaultEncoding()) + except ExecUtilException as e: + if e.exit_code == 2: # No such file or directory + return True + + if not ignore_errors: + raise + + errMsg = "Failed to remove directory {0} ({1}): {2}".format( + path, type(e).__name__, e + ) + logging.warning(errMsg) + return False + return True def listdir(self, path): """ diff --git a/tests/test_remote.py b/tests/test_remote.py index 85e65c24..b1c4e58c 100755 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -99,9 +99,9 @@ def test_makedirs_and_rmdirs_success(self): assert not os.path.exists(path) assert not self.operations.path_exists(path) - def test_makedirs_and_rmdirs_failure(self): + def test_makedirs_failure(self): """ - Test makedirs and rmdirs for directory creation and removal failure. + Test makedirs for failure. """ # Try to create a directory in a read-only location path = "/root/test_dir" @@ -110,16 +110,84 @@ def test_makedirs_and_rmdirs_failure(self): with pytest.raises(Exception): self.operations.makedirs(path) - # Test rmdirs - while True: - try: - self.operations.rmdirs(path, verbose=True) - except ExecUtilException as e: - assert e.message == "Utility exited with non-zero code (1). Error: `rm: cannot remove '/root/test_dir': Permission denied`" - assert type(e.error) == bytes # noqa: E721 - assert e.error.strip() == b"rm: cannot remove '/root/test_dir': Permission denied" - break - raise Exception("We wait an exception!") + def test_rmdirs(self): + path = self.operations.mkdtemp() + assert os.path.exists(path) + + assert self.operations.rmdirs(path, ignore_errors=False) is True + assert not os.path.exists(path) + + def test_rmdirs__01_with_subfolder(self): + # folder with subfolder + path = self.operations.mkdtemp() + assert os.path.exists(path) + + dir1 = os.path.join(path, "dir1") + assert not os.path.exists(dir1) + + self.operations.makedirs(dir1) + assert os.path.exists(dir1) + + assert self.operations.rmdirs(path, ignore_errors=False) is True + assert not os.path.exists(path) + assert not os.path.exists(dir1) + + def test_rmdirs__02_with_file(self): + # folder with file + path = self.operations.mkdtemp() + assert os.path.exists(path) + + file1 = os.path.join(path, "file1.txt") + assert not os.path.exists(file1) + + self.operations.touch(file1) + assert os.path.exists(file1) + + assert self.operations.rmdirs(path, ignore_errors=False) is True + assert not os.path.exists(path) + assert not os.path.exists(file1) + + def test_rmdirs__03_with_subfolder_and_file(self): + # folder with subfolder and file + path = self.operations.mkdtemp() + assert os.path.exists(path) + + dir1 = os.path.join(path, "dir1") + assert not os.path.exists(dir1) + + self.operations.makedirs(dir1) + assert os.path.exists(dir1) + + file1 = os.path.join(dir1, "file1.txt") + assert not os.path.exists(file1) + + self.operations.touch(file1) + assert os.path.exists(file1) + + assert self.operations.rmdirs(path, ignore_errors=False) is True + assert not os.path.exists(path) + assert not os.path.exists(dir1) + assert not os.path.exists(file1) + + def test_rmdirs__try_to_delete_nonexist_path(self): + path = "/root/test_dir" + + assert self.operations.rmdirs(path, ignore_errors=False) is True + + def test_rmdirs__try_to_delete_file(self): + path = self.operations.mkstemp() + assert os.path.exists(path) + + with pytest.raises(ExecUtilException) as x: + self.operations.rmdirs(path, ignore_errors=False) + + assert os.path.exists(path) + assert type(x.value) == ExecUtilException # noqa: E721 + assert x.value.message == "Utility exited with non-zero code (20). Error: `cannot remove '" + path + "': it is not a directory`" + assert type(x.value.error) == str # noqa: E721 + assert x.value.error.strip() == "cannot remove '" + path + "': it is not a directory" + assert type(x.value.exit_code) == int # noqa: E721 + assert x.value.exit_code == 20 def test_listdir(self): """ diff --git a/tests/test_simple.py b/tests/test_simple.py index 37c3db44..d9844fed 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -428,16 +428,61 @@ def test_backup_wrong_xlog_method(self): node.backup(xlog_method='wrong') def test_pg_ctl_wait_option(self): - with get_new_node() as node: - node.init().start(wait=False) - while True: - try: - node.stop(wait=False) - break - except ExecUtilException: - # it's ok to get this exception here since node - # could be not started yet - pass + C_MAX_ATTEMPTS = 50 + + node = get_new_node() + assert node.status() == testgres.NodeStatus.Uninitialized + node.init() + assert node.status() == testgres.NodeStatus.Stopped + node.start(wait=False) + nAttempt = 0 + while True: + if nAttempt == C_MAX_ATTEMPTS: + raise Exception("Could not stop node.") + + nAttempt += 1 + + if nAttempt > 1: + logging.info("Wait 1 second.") + time.sleep(1) + logging.info("") + + logging.info("Try to stop node. Attempt #{0}.".format(nAttempt)) + + try: + node.stop(wait=False) + break + except ExecUtilException as e: + # it's ok to get this exception here since node + # could be not started yet + logging.info("Node is not stopped. Exception ({0}): {1}".format(type(e).__name__, e)) + continue + + logging.info("OK. Stop command was executed. Let's wait while our node will stop really.") + nAttempt = 0 + while True: + if nAttempt == C_MAX_ATTEMPTS: + raise Exception("Could not stop node.") + + nAttempt += 1 + if nAttempt > 1: + logging.info("Wait 1 second.") + time.sleep(1) + logging.info("") + + logging.info("Attempt #{0}.".format(nAttempt)) + s1 = node.status() + + if s1 == testgres.NodeStatus.Running: + continue + + if s1 == testgres.NodeStatus.Stopped: + break + + raise Exception("Unexpected node status: {0}.".format(s1)) + + logging.info("OK. Node is stopped.") + node.cleanup() def test_replicate(self): with get_new_node() as node: diff --git a/tests/test_simple_remote.py b/tests/test_simple_remote.py index a62085ce..42527dbc 100755 --- a/tests/test_simple_remote.py +++ b/tests/test_simple_remote.py @@ -499,16 +499,61 @@ def test_backup_wrong_xlog_method(self): node.backup(xlog_method='wrong') def test_pg_ctl_wait_option(self): - with __class__.helper__get_node() as node: - node.init().start(wait=False) - while True: - try: - node.stop(wait=False) - break - except ExecUtilException: - # it's ok to get this exception here since node - # could be not started yet - pass + C_MAX_ATTEMPTS = 50 + + node = __class__.helper__get_node() + assert node.status() == testgres.NodeStatus.Uninitialized + node.init() + assert node.status() == testgres.NodeStatus.Stopped + node.start(wait=False) + nAttempt = 0 + while True: + if nAttempt == C_MAX_ATTEMPTS: + raise Exception("Could not stop node.") + + nAttempt += 1 + + if nAttempt > 1: + logging.info("Wait 1 second.") + time.sleep(1) + logging.info("") + + logging.info("Try to stop node. Attempt #{0}.".format(nAttempt)) + + try: + node.stop(wait=False) + break + except ExecUtilException as e: + # it's ok to get this exception here since node + # could be not started yet + logging.info("Node is not stopped. Exception ({0}): {1}".format(type(e).__name__, e)) + continue + + logging.info("OK. Stop command was executed. Let's wait while our node will stop really.") + nAttempt = 0 + while True: + if nAttempt == C_MAX_ATTEMPTS: + raise Exception("Could not stop node.") + + nAttempt += 1 + if nAttempt > 1: + logging.info("Wait 1 second.") + time.sleep(1) + logging.info("") + + logging.info("Attempt #{0}.".format(nAttempt)) + s1 = node.status() + + if s1 == testgres.NodeStatus.Running: + continue + + if s1 == testgres.NodeStatus.Stopped: + break + + raise Exception("Unexpected node status: {0}.".format(s1)) + + logging.info("OK. Node is stopped.") + node.cleanup() def test_replicate(self): with __class__.helper__get_node() as node: From e47cded2a784bfbc64c9a70aceedfe26f9a3b802 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 11 Mar 2025 10:27:59 +0300 Subject: [PATCH 410/475] [remote_ops] A problem with mktemp on Alpine Linux is fixed Five 'X' in template is not enough - Alpine returns "mktemp: : Invalid argument" error. Six 'X' is OK. --- testgres/operations/remote_ops.py | 4 ++-- tests/test_local.py | 17 +++++++++++++++++ tests/test_remote.py | 17 +++++++++++++++++ 3 files changed, 36 insertions(+), 2 deletions(-) diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index 767df567..11d9cd37 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -316,7 +316,7 @@ def mkdtemp(self, prefix=None): - prefix (str): The prefix of the temporary directory name. """ if prefix: - command = ["mktemp", "-d", "-t", prefix + "XXXXX"] + command = ["mktemp", "-d", "-t", prefix + "XXXXXX"] else: command = ["mktemp", "-d"] @@ -344,7 +344,7 @@ def mkstemp(self, prefix=None): - prefix (str): The prefix of the temporary directory name. """ if prefix: - command = ["mktemp", "-t", prefix + "XXXXX"] + command = ["mktemp", "-t", prefix + "XXXXXX"] else: command = ["mktemp"] diff --git a/tests/test_local.py b/tests/test_local.py index ee5e19a0..826c3f51 100644 --- a/tests/test_local.py +++ b/tests/test_local.py @@ -4,6 +4,7 @@ import pytest import re import tempfile +import logging from ..testgres import ExecUtilException from ..testgres import InvalidOperationException @@ -18,6 +19,22 @@ class TestLocalOperations: def setup(self): self.operations = LocalOperations() + def test_mkdtemp__default(self): + path = self.operations.mkdtemp() + logging.info("Path is [{0}].".format(path)) + assert os.path.exists(path) + os.rmdir(path) + assert not os.path.exists(path) + + def test_mkdtemp__custom(self): + C_TEMPLATE = "abcdef" + path = self.operations.mkdtemp(C_TEMPLATE) + logging.info("Path is [{0}].".format(path)) + assert os.path.exists(path) + assert C_TEMPLATE in os.path.basename(path) + os.rmdir(path) + assert not os.path.exists(path) + def test_exec_command_success(self): """ Test exec_command for successful command execution. diff --git a/tests/test_remote.py b/tests/test_remote.py index b1c4e58c..17c76c2c 100755 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -4,6 +4,7 @@ import pytest import re import tempfile +import logging from ..testgres import ExecUtilException from ..testgres import InvalidOperationException @@ -110,6 +111,22 @@ def test_makedirs_failure(self): with pytest.raises(Exception): self.operations.makedirs(path) + def test_mkdtemp__default(self): + path = self.operations.mkdtemp() + logging.info("Path is [{0}].".format(path)) + assert os.path.exists(path) + os.rmdir(path) + assert not os.path.exists(path) + + def test_mkdtemp__custom(self): + C_TEMPLATE = "abcdef" + path = self.operations.mkdtemp(C_TEMPLATE) + logging.info("Path is [{0}].".format(path)) + assert os.path.exists(path) + assert C_TEMPLATE in os.path.basename(path) + os.rmdir(path) + assert not os.path.exists(path) + def test_rmdirs(self): path = self.operations.mkdtemp() assert os.path.exists(path) From cf6f4cc6f6111a07302e4f6644baccbf58bb158a Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 11 Mar 2025 11:38:34 +0300 Subject: [PATCH 411/475] [remote] Tests are updated to support Alpine Linux --- tests/test_local.py | 16 +++++++++++++--- tests/test_remote.py | 36 ++++++++++++++++++++++++++++-------- tests/test_simple_remote.py | 10 +++++++++- 3 files changed, 50 insertions(+), 12 deletions(-) diff --git a/tests/test_local.py b/tests/test_local.py index 826c3f51..68e7db33 100644 --- a/tests/test_local.py +++ b/tests/test_local.py @@ -57,9 +57,17 @@ def test_exec_command_failure(self): try: self.operations.exec_command(cmd, wait_exit=True, shell=True) except ExecUtilException as e: - assert e.message == "Utility exited with non-zero code (127). Error: `/bin/sh: 1: nonexistent_command: not found`" + assert type(e.exit_code) == int # noqa: E721 + assert e.exit_code == 127 + + assert type(e.message) == str # noqa: E721 assert type(e.error) == bytes # noqa: E721 - assert e.error.strip() == b"/bin/sh: 1: nonexistent_command: not found" + + assert e.message.startswith("Utility exited with non-zero code (127). Error:") + assert "nonexistent_command" in e.message + assert "not found" in e.message + assert b"nonexistent_command" in e.error + assert b"not found" in e.error break raise Exception("We wait an exception!") @@ -73,9 +81,11 @@ def test_exec_command_failure__expect_error(self): exit_status, result, error = self.operations.exec_command(cmd, verbose=True, wait_exit=True, shell=True, expect_error=True) - assert error == b'/bin/sh: 1: nonexistent_command: not found\n' assert exit_status == 127 assert result == b'' + assert type(error) == bytes # noqa: E721 + assert b"nonexistent_command" in error + assert b"not found" in error def test_read__text(self): """ diff --git a/tests/test_remote.py b/tests/test_remote.py index 17c76c2c..1f771c62 100755 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -41,9 +41,17 @@ def test_exec_command_failure(self): try: self.operations.exec_command(cmd, verbose=True, wait_exit=True) except ExecUtilException as e: - assert e.message == "Utility exited with non-zero code (127). Error: `bash: line 1: nonexistent_command: command not found`" + assert type(e.exit_code) == int # noqa: E721 + assert e.exit_code == 127 + + assert type(e.message) == str # noqa: E721 assert type(e.error) == bytes # noqa: E721 - assert e.error.strip() == b"bash: line 1: nonexistent_command: command not found" + + assert e.message.startswith("Utility exited with non-zero code (127). Error:") + assert "nonexistent_command" in e.message + assert "not found" in e.message + assert b"nonexistent_command" in e.error + assert b"not found" in e.error break raise Exception("We wait an exception!") @@ -55,9 +63,11 @@ def test_exec_command_failure__expect_error(self): exit_status, result, error = self.operations.exec_command(cmd, verbose=True, wait_exit=True, shell=True, expect_error=True) - assert error == b'bash: line 1: nonexistent_command: command not found\n' assert exit_status == 127 assert result == b'' + assert type(error) == bytes # noqa: E721 + assert b"nonexistent_command" in error + assert b"not found" in error def test_is_executable_true(self): """ @@ -344,11 +354,13 @@ def test_read__unknown_file(self): Test RemoteOperations::read with unknown file. """ - with pytest.raises( - ExecUtilException, - match=re.escape("cat: /dummy: No such file or directory")): + with pytest.raises(ExecUtilException) as x: self.operations.read("/dummy") + assert "Utility exited with non-zero code (1)." in str(x.value) + assert "No such file or directory" in str(x.value) + assert "/dummy" in str(x.value) + def test_read_binary__spec(self): """ Test RemoteOperations::read_binary. @@ -388,9 +400,13 @@ def test_read_binary__spec__unk_file(self): Test RemoteOperations::read_binary with unknown file. """ - with pytest.raises(ExecUtilException, match=re.escape("tail: cannot open '/dummy' for reading: No such file or directory")): + with pytest.raises(ExecUtilException) as x: self.operations.read_binary("/dummy", 0) + assert "Utility exited with non-zero code (1)." in str(x.value) + assert "No such file or directory" in str(x.value) + assert "/dummy" in str(x.value) + def test_read_binary__spec__negative_offset(self): """ Test RemoteOperations::read_binary with negative offset. @@ -419,9 +435,13 @@ def test_get_file_size__unk_file(self): Test RemoteOperations::get_file_size. """ - with pytest.raises(ExecUtilException, match=re.escape("du: cannot access '/dummy': No such file or directory")): + with pytest.raises(ExecUtilException) as x: self.operations.get_file_size("/dummy") + assert "Utility exited with non-zero code (1)." in str(x.value) + assert "No such file or directory" in str(x.value) + assert "/dummy" in str(x.value) + def test_touch(self): """ Test touch for creating a new file or updating access and modification times of an existing file. diff --git a/tests/test_simple_remote.py b/tests/test_simple_remote.py index 42527dbc..cdad161c 100755 --- a/tests/test_simple_remote.py +++ b/tests/test_simple_remote.py @@ -163,6 +163,8 @@ def test_init__unk_LANG_and_LC_CTYPE(self): ("\"", "\""), ] + errorIsDetected = False + for unkData in unkDatas: logging.info("----------------------") logging.info("Unk LANG is [{0}]".format(unkData[0])) @@ -193,7 +195,10 @@ def test_init__unk_LANG_and_LC_CTYPE(self): assert isinstance(exc, ExecUtilException) if exc is None: - raise Exception("We expected an error!") + logging.warning("We expected an error!") + continue + + errorIsDetected = True assert isinstance(exc, ExecUtilException) @@ -204,6 +209,9 @@ def test_init__unk_LANG_and_LC_CTYPE(self): assert "initdb: error: invalid locale settings; check LANG and LC_* environment variables" in errMsg continue + if not errorIsDetected: + pytest.xfail("All the bad data are processed without errors!") + finally: __class__.helper__restore_envvar("LANG", prev_LANG) __class__.helper__restore_envvar("LANGUAGE", prev_LANGUAGE) From 44f280bc7aa51cce5374fb19c1c234b379e4fbf0 Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Tue, 11 Mar 2025 18:19:12 +0300 Subject: [PATCH 412/475] [CI] A run of all tests (local, remote) on Alpine Linux [PY3, PG17] is added (#210) * Dockerfile to run all tests on Alpine Linux * A run of all the tests on Alpine Linux [PY3, PG17] is added * [CI] The run of [STD, PY3, PG17] is removed It was replaced with [ALPINE, PY3, PG17]. * Test platform "alpine" was renamed with "std.all" --- .travis.yml | 2 +- Dockerfile--std.all.tmpl | 60 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 1 deletion(-) create mode 100644 Dockerfile--std.all.tmpl diff --git a/.travis.yml b/.travis.yml index 4110835a..7fb34808 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,7 +20,6 @@ notifications: on_failure: always env: - - TEST_PLATFORM=std PYTHON_VERSION=3 PG_VERSION=17 - TEST_PLATFORM=std PYTHON_VERSION=3 PG_VERSION=16 - TEST_PLATFORM=std PYTHON_VERSION=3 PG_VERSION=15 - TEST_PLATFORM=std PYTHON_VERSION=3 PG_VERSION=14 @@ -28,6 +27,7 @@ env: - TEST_PLATFORM=std PYTHON_VERSION=3 PG_VERSION=12 - TEST_PLATFORM=std PYTHON_VERSION=3 PG_VERSION=11 - TEST_PLATFORM=std PYTHON_VERSION=3 PG_VERSION=10 + - TEST_PLATFORM=std.all PYTHON_VERSION=3 PG_VERSION=17 - TEST_PLATFORM=ubuntu-24_04 PYTHON_VERSION=3 PG_VERSION=17 matrix: diff --git a/Dockerfile--std.all.tmpl b/Dockerfile--std.all.tmpl new file mode 100644 index 00000000..dfd9ab20 --- /dev/null +++ b/Dockerfile--std.all.tmpl @@ -0,0 +1,60 @@ +FROM postgres:${PG_VERSION}-alpine + +ENV PYTHON=python${PYTHON_VERSION} +RUN if [ "${PYTHON_VERSION}" = "2" ] ; then \ + apk add --no-cache curl python2 python2-dev build-base musl-dev \ + linux-headers py-virtualenv py-pip; \ + fi +RUN if [ "${PYTHON_VERSION}" = "3" ] ; then \ + apk add --no-cache curl python3 python3-dev build-base musl-dev \ + linux-headers py-virtualenv; \ + fi + +#RUN apk add --no-cache mc + +# Full version of "ps" command +RUN apk add --no-cache procps + +RUN apk add --no-cache openssh +RUN apk add --no-cache sudo + +ENV LANG=C.UTF-8 + +RUN addgroup -S sudo +RUN adduser postgres sudo + +EXPOSE 22 +RUN ssh-keygen -A + +ADD . /pg/testgres +WORKDIR /pg/testgres +RUN chown -R postgres:postgres /pg + +# It allows to use sudo without password +RUN sh -c "echo \"postgres ALL=(ALL:ALL) NOPASSWD:ALL\"">>/etc/sudoers + +# THIS CMD IS NEEDED TO CONNECT THROUGH SSH WITHOUT PASSWORD +RUN sh -c "echo "postgres:*" | chpasswd -e" + +USER postgres + +# THIS CMD IS NEEDED TO CONNECT THROUGH SSH WITHOUT PASSWORD +RUN chmod 700 ~/ + +RUN mkdir -p ~/.ssh +#RUN chmod 700 ~/.ssh + +#ENTRYPOINT PYTHON_VERSION=${PYTHON_VERSION} bash run_tests.sh + +ENTRYPOINT sh -c " \ +set -eux; \ +echo HELLO FROM ENTRYPOINT; \ +echo HOME DIR IS [`realpath ~/`]; \ +ssh-keygen -t rsa -f ~/.ssh/id_rsa -q -N ''; \ +cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys; \ +chmod 600 ~/.ssh/authorized_keys; \ +ls -la ~/.ssh/; \ +sudo /usr/sbin/sshd; \ +ssh-keyscan -H localhost >> ~/.ssh/known_hosts; \ +ssh-keyscan -H 127.0.0.1 >> ~/.ssh/known_hosts; \ +TEST_FILTER=\"\" PYTHON_VERSION=${PYTHON_VERSION} bash run_tests.sh;" From 600572857f100c37218d29ec5c4d6fa8c6d9d6c6 Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Tue, 11 Mar 2025 21:00:07 +0300 Subject: [PATCH 413/475] PsUtilProcessProxy is updated (refactoring) (#212) --- testgres/operations/remote_ops.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index 11d9cd37..d6917c82 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -26,17 +26,25 @@ class PsUtilProcessProxy: def __init__(self, ssh, pid): + assert isinstance(ssh, RemoteOperations) + assert type(pid) == int # noqa: E721 self.ssh = ssh self.pid = pid def kill(self): - command = "kill {}".format(self.pid) - self.ssh.exec_command(command) + assert isinstance(self.ssh, RemoteOperations) + assert type(self.pid) == int # noqa: E721 + command = ["kill", str(self.pid)] + self.ssh.exec_command(command, encoding=get_default_encoding()) def cmdline(self): - command = "ps -p {} -o cmd --no-headers".format(self.pid) - stdin, stdout, stderr = self.ssh.exec_command(command, verbose=True, encoding=get_default_encoding()) - cmdline = stdout.strip() + assert isinstance(self.ssh, RemoteOperations) + assert type(self.pid) == int # noqa: E721 + command = ["ps", "-p", str(self.pid), "-o", "cmd", "--no-headers"] + output = self.ssh.exec_command(command, encoding=get_default_encoding()) + assert type(output) == str # noqa: E721 + cmdline = output.strip() + # TODO: This code work wrong if command line contains quoted values. Yes? return cmdline.split() From 1a4655cb9011c3a01cd0e7d56606ba36fff40e43 Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Tue, 11 Mar 2025 21:46:10 +0300 Subject: [PATCH 414/475] Dockerfile for Ubuntu 2024.04 is updated (#211) * Dockerfile for Ubuntu 2024.04 updated - Using "chmod 700 ~/" instead "chmod go-w /var/lib/postgresql" - "~/.ssh" folder is prepared at image level - Entrypoint startup code is cleaned from trace messages * Dockerfile--ubuntu-24_04.tmpl is updated (double quotes) --- Dockerfile--ubuntu-24_04.tmpl | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/Dockerfile--ubuntu-24_04.tmpl b/Dockerfile--ubuntu-24_04.tmpl index 99be5343..fd1136d8 100644 --- a/Dockerfile--ubuntu-24_04.tmpl +++ b/Dockerfile--ubuntu-24_04.tmpl @@ -37,33 +37,22 @@ RUN ssh-keygen -A RUN sh -c "echo postgres ALL=NOPASSWD:/usr/sbin/service ssh start" >> /etc/sudoers USER postgres - ENV LANG=C.UTF-8 +RUN chmod 700 ~/ +RUN mkdir -p ~/.ssh + #ENTRYPOINT PYTHON_VERSION=3.12 /run.sh ENTRYPOINT sh -c " \ #set -eux; \ echo HELLO FROM ENTRYPOINT; \ echo HOME DIR IS [`realpath ~/`]; \ -echo POINT 1; \ -chmod go-w /var/lib/postgresql; \ -echo POINT 1.5; \ -mkdir -p ~/.ssh; \ -echo POINT 2; \ service ssh enable; \ -echo POINT 3; \ sudo service ssh start; \ -echo POINT 4; \ ssh-keyscan -H localhost >> ~/.ssh/known_hosts; \ -echo POINT 5; \ ssh-keyscan -H 127.0.0.1 >> ~/.ssh/known_hosts; \ -echo POINT 6; \ ssh-keygen -t rsa -f ~/.ssh/id_rsa -q -N ''; \ -echo ----; \ cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys; \ -echo ----; \ chmod 600 ~/.ssh/authorized_keys; \ -echo ----; \ ls -la ~/.ssh/; \ -echo ----; \ -TEST_FILTER="" PYTHON_VERSION=${PYTHON_VERSION} bash run_tests.sh;" +TEST_FILTER=\"\" PYTHON_VERSION=${PYTHON_VERSION} bash run_tests.sh;" From 438e84508c7c0b3d9f05336ffdd1df44ba23c36b Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Wed, 12 Mar 2025 08:48:16 +0300 Subject: [PATCH 415/475] test_pg_ctl_wait_option is updated (#213) When node is not stopped, we read and output a content of node log file to provide an additional information about this problem. It should help find a reason of unexpected problem with this test in CI. --- tests/test_simple.py | 13 +++++++++++++ tests/test_simple_remote.py | 13 +++++++++++++ 2 files changed, 26 insertions(+) diff --git a/tests/test_simple.py b/tests/test_simple.py index d9844fed..e13cf095 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -438,6 +438,19 @@ def test_pg_ctl_wait_option(self): nAttempt = 0 while True: if nAttempt == C_MAX_ATTEMPTS: + # + # [2025-03-11] + # We have an unexpected problem with this test in CI + # Let's get an additional information about this test failure. + # + logging.error("Node was not stopped.") + if not node.os_ops.path_exists(node.pg_log_file): + logging.warning("Node log does not exist.") + else: + logging.info("Let's read node log file [{0}]".format(node.pg_log_file)) + logFileData = node.os_ops.read(node.pg_log_file, binary=False) + logging.info("Node log file content:\n{0}".format(logFileData)) + raise Exception("Could not stop node.") nAttempt += 1 diff --git a/tests/test_simple_remote.py b/tests/test_simple_remote.py index cdad161c..d484f1e3 100755 --- a/tests/test_simple_remote.py +++ b/tests/test_simple_remote.py @@ -517,6 +517,19 @@ def test_pg_ctl_wait_option(self): nAttempt = 0 while True: if nAttempt == C_MAX_ATTEMPTS: + # + # [2025-03-11] + # We have an unexpected problem with this test in CI + # Let's get an additional information about this test failure. + # + logging.error("Node was not stopped.") + if not node.os_ops.path_exists(node.pg_log_file): + logging.warning("Node log does not exist.") + else: + logging.info("Let's read node log file [{0}]".format(node.pg_log_file)) + logFileData = node.os_ops.read(node.pg_log_file, binary=False) + logging.info("Node log file content:\n{0}".format(logFileData)) + raise Exception("Could not stop node.") nAttempt += 1 From f2c000c28e4f250c5dc319be557217d8433306cd Mon Sep 17 00:00:00 2001 From: Victoria Shepard <5807469+demonolock@users.noreply.github.com> Date: Wed, 12 Mar 2025 06:52:03 +0100 Subject: [PATCH 416/475] Fix auto conf test (#214) * Fix test_set_auto_conf for Postgresql 10, 11 * Remove allow failures --------- Co-authored-by: vshepard --- .travis.yml | 5 ----- tests/test_simple.py | 7 ++++--- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7fb34808..997945b5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,8 +29,3 @@ env: - TEST_PLATFORM=std PYTHON_VERSION=3 PG_VERSION=10 - TEST_PLATFORM=std.all PYTHON_VERSION=3 PG_VERSION=17 - TEST_PLATFORM=ubuntu-24_04 PYTHON_VERSION=3 PG_VERSION=17 - -matrix: - allow_failures: - - env: TEST_PLATFORM=std PYTHON_VERSION=3 PG_VERSION=11 - - env: TEST_PLATFORM=std PYTHON_VERSION=3 PG_VERSION=10 diff --git a/tests/test_simple.py b/tests/test_simple.py index e13cf095..e886a39c 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -1464,9 +1464,6 @@ def test_set_auto_conf(self): ["archive_command", "cp '%p' \"/mnt/server/archivedir/%f\"", "'cp \\'%p\\' \"/mnt/server/archivedir/%f\""], - ["restore_command", - 'cp "/mnt/server/archivedir/%f" \'%p\'', - "'cp \"/mnt/server/archivedir/%f\" \\'%p\\''"], ["log_line_prefix", "'\n\r\t\b\\\"", "'\\\'\\n\\r\\t\\b\\\\\""], @@ -1480,6 +1477,10 @@ def test_set_auto_conf(self): 3, "3"] ] + if pg_version_ge('12'): + testData.append(["restore_command", + 'cp "/mnt/server/archivedir/%f" \'%p\'', + "'cp \"/mnt/server/archivedir/%f\" \\'%p\\''"]) with get_new_node() as node: node.init().start() From e3eb6ae1a8ed63a6df516e59e406b674f4e5ea96 Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Wed, 12 Mar 2025 21:49:38 +0300 Subject: [PATCH 417/475] Refactoring of dockerfiles (#215) * Refactoring of dockerfiles Let's to try using a pure dockerfile' logic. It is the fist step. - We use docker instead docker-composer - We upgrade and use only "std" plaform. Other plaforms will be upgraded later * It is better to run a docker with "-t" options. This allows the colors in output data. * A problem with PYTHON_VERSION is fixed * Dockerfile--std.all.tmpl is updated * Dockerfile--ubuntu-24_04.tmpl * Dockerfile--std.tmpl is updated (formatting) * docker-compose.yml and mk_dockerfile.sh are deleted * [CI] Platform "std.all" was renamed with "std_all" Let's avoid using a "point" symbol in name of file. This symbol may create a problem in the future if we decide to use configuration docker files without extensions. * [CI] Platform name has the one format Dockerfiles--[-].tmpl - "std_all" -> "std-all" - "ubuntu-24_04" -> "ubuntu_24_04" * Dockerfile--ubuntu_24_04.tmpl is updated (minimization) --- .travis.yml | 9 ++-- ...--std.all.tmpl => Dockerfile--std-all.tmpl | 31 ++++++++------ Dockerfile--std.tmpl | 30 +++++++++----- ...4_04.tmpl => Dockerfile--ubuntu_24_04.tmpl | 41 ++++++++++++------- docker-compose.yml | 4 -- mk_dockerfile.sh | 2 - 6 files changed, 69 insertions(+), 48 deletions(-) rename Dockerfile--std.all.tmpl => Dockerfile--std-all.tmpl (60%) rename Dockerfile--ubuntu-24_04.tmpl => Dockerfile--ubuntu_24_04.tmpl (71%) delete mode 100644 docker-compose.yml delete mode 100755 mk_dockerfile.sh diff --git a/.travis.yml b/.travis.yml index 997945b5..3a889845 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,11 +8,10 @@ services: - docker install: - - ./mk_dockerfile.sh - - docker-compose build + - docker build --build-arg PG_VERSION="${PG_VERSION}" --build-arg PYTHON_VERSION="${PYTHON_VERSION}" -t tests -f Dockerfile--${TEST_PLATFORM}.tmpl . script: - - docker-compose run $(bash <(curl -s https://codecov.io/env)) tests + - docker run $(bash <(curl -s https://codecov.io/env)) -t tests notifications: email: @@ -27,5 +26,5 @@ env: - TEST_PLATFORM=std PYTHON_VERSION=3 PG_VERSION=12 - TEST_PLATFORM=std PYTHON_VERSION=3 PG_VERSION=11 - TEST_PLATFORM=std PYTHON_VERSION=3 PG_VERSION=10 - - TEST_PLATFORM=std.all PYTHON_VERSION=3 PG_VERSION=17 - - TEST_PLATFORM=ubuntu-24_04 PYTHON_VERSION=3 PG_VERSION=17 + - TEST_PLATFORM=std-all PYTHON_VERSION=3 PG_VERSION=17 + - TEST_PLATFORM=ubuntu_24_04 PYTHON_VERSION=3 PG_VERSION=17 diff --git a/Dockerfile--std.all.tmpl b/Dockerfile--std-all.tmpl similarity index 60% rename from Dockerfile--std.all.tmpl rename to Dockerfile--std-all.tmpl index dfd9ab20..c41c5a06 100644 --- a/Dockerfile--std.all.tmpl +++ b/Dockerfile--std-all.tmpl @@ -1,14 +1,21 @@ -FROM postgres:${PG_VERSION}-alpine - -ENV PYTHON=python${PYTHON_VERSION} -RUN if [ "${PYTHON_VERSION}" = "2" ] ; then \ - apk add --no-cache curl python2 python2-dev build-base musl-dev \ - linux-headers py-virtualenv py-pip; \ - fi -RUN if [ "${PYTHON_VERSION}" = "3" ] ; then \ - apk add --no-cache curl python3 python3-dev build-base musl-dev \ - linux-headers py-virtualenv; \ - fi +ARG PG_VERSION +ARG PYTHON_VERSION + +# --------------------------------------------- base1 +FROM postgres:${PG_VERSION}-alpine as base1 + +# --------------------------------------------- base2_with_python-2 +FROM base1 as base2_with_python-2 +RUN apk add --no-cache curl python2 python2-dev build-base musl-dev linux-headers py-virtualenv py-pip +ENV PYTHON_VERSION=2 + +# --------------------------------------------- base2_with_python-3 +FROM base1 as base2_with_python-3 +RUN apk add --no-cache curl python3 python3-dev build-base musl-dev linux-headers py-virtualenv +ENV PYTHON_VERSION=3 + +# --------------------------------------------- final +FROM base2_with_python-${PYTHON_VERSION} as final #RUN apk add --no-cache mc @@ -57,4 +64,4 @@ ls -la ~/.ssh/; \ sudo /usr/sbin/sshd; \ ssh-keyscan -H localhost >> ~/.ssh/known_hosts; \ ssh-keyscan -H 127.0.0.1 >> ~/.ssh/known_hosts; \ -TEST_FILTER=\"\" PYTHON_VERSION=${PYTHON_VERSION} bash run_tests.sh;" +TEST_FILTER=\"\" bash run_tests.sh;" diff --git a/Dockerfile--std.tmpl b/Dockerfile--std.tmpl index d844c9a3..91886ede 100644 --- a/Dockerfile--std.tmpl +++ b/Dockerfile--std.tmpl @@ -1,14 +1,22 @@ -FROM postgres:${PG_VERSION}-alpine +ARG PG_VERSION +ARG PYTHON_VERSION + +# --------------------------------------------- base1 +FROM postgres:${PG_VERSION}-alpine as base1 + +# --------------------------------------------- base2_with_python-2 +FROM base1 as base2_with_python-2 +RUN apk add --no-cache curl python2 python2-dev build-base musl-dev linux-headers py-virtualenv py-pip +ENV PYTHON_VERSION=2 + +# --------------------------------------------- base2_with_python-3 +FROM base1 as base2_with_python-3 +RUN apk add --no-cache curl python3 python3-dev build-base musl-dev linux-headers py-virtualenv +ENV PYTHON_VERSION=3 + +# --------------------------------------------- final +FROM base2_with_python-${PYTHON_VERSION} as final -ENV PYTHON=python${PYTHON_VERSION} -RUN if [ "${PYTHON_VERSION}" = "2" ] ; then \ - apk add --no-cache curl python2 python2-dev build-base musl-dev \ - linux-headers py-virtualenv py-pip; \ - fi -RUN if [ "${PYTHON_VERSION}" = "3" ] ; then \ - apk add --no-cache curl python3 python3-dev build-base musl-dev \ - linux-headers py-virtualenv; \ - fi ENV LANG=C.UTF-8 ADD . /pg/testgres @@ -16,4 +24,4 @@ WORKDIR /pg/testgres RUN chown -R postgres:postgres /pg USER postgres -ENTRYPOINT PYTHON_VERSION=${PYTHON_VERSION} bash run_tests.sh +ENTRYPOINT bash run_tests.sh diff --git a/Dockerfile--ubuntu-24_04.tmpl b/Dockerfile--ubuntu_24_04.tmpl similarity index 71% rename from Dockerfile--ubuntu-24_04.tmpl rename to Dockerfile--ubuntu_24_04.tmpl index fd1136d8..c1ddeab6 100644 --- a/Dockerfile--ubuntu-24_04.tmpl +++ b/Dockerfile--ubuntu_24_04.tmpl @@ -1,7 +1,17 @@ -FROM ubuntu:24.04 +ARG PG_VERSION +ARG PYTHON_VERSION + +# --------------------------------------------- base1 +FROM ubuntu:24.04 as base1 +ARG PG_VERSION + +RUN apt update +RUN apt install -y sudo curl ca-certificates +RUN apt update +RUN apt install -y openssh-server RUN apt update -RUN apt install -y sudo curl ca-certificates postgresql-common +RUN apt install -y postgresql-common RUN bash /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh -y @@ -14,21 +24,12 @@ RUN curl -o /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc --fail http RUN apt update RUN apt install -y postgresql-${PG_VERSION} -RUN apt install -y python3 python3-dev python3-virtualenv # RUN apt install -y mc -# It is required for psycopg2 -RUN apt install -y libpq-dev -RUN apt install -y openssh-server - # [2025-02-26] It adds the user 'postgres' in the group 'sudo' # [2025-02-27] It is not required. # RUN adduser postgres sudo -ADD . /pg/testgres -WORKDIR /pg/testgres -RUN chown -R postgres /pg - EXPOSE 22 RUN ssh-keygen -A @@ -36,13 +37,25 @@ RUN ssh-keygen -A # It enables execution of "sudo service ssh start" without password RUN sh -c "echo postgres ALL=NOPASSWD:/usr/sbin/service ssh start" >> /etc/sudoers -USER postgres +# --------------------------------------------- base2_with_python-3 +FROM base1 as base2_with_python-3 +RUN apt install -y python3 python3-dev python3-virtualenv libpq-dev +ENV PYTHON_VERSION=3 + +# --------------------------------------------- final +FROM base2_with_python-${PYTHON_VERSION} as final + +ADD . /pg/testgres +WORKDIR /pg/testgres +RUN chown -R postgres /pg + ENV LANG=C.UTF-8 +USER postgres + RUN chmod 700 ~/ RUN mkdir -p ~/.ssh -#ENTRYPOINT PYTHON_VERSION=3.12 /run.sh ENTRYPOINT sh -c " \ #set -eux; \ echo HELLO FROM ENTRYPOINT; \ @@ -55,4 +68,4 @@ ssh-keygen -t rsa -f ~/.ssh/id_rsa -q -N ''; \ cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys; \ chmod 600 ~/.ssh/authorized_keys; \ ls -la ~/.ssh/; \ -TEST_FILTER=\"\" PYTHON_VERSION=${PYTHON_VERSION} bash run_tests.sh;" +TEST_FILTER=\"\" bash ./run_tests.sh;" diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index 86edf9a4..00000000 --- a/docker-compose.yml +++ /dev/null @@ -1,4 +0,0 @@ -version: '3.8' -services: - tests: - build: . diff --git a/mk_dockerfile.sh b/mk_dockerfile.sh deleted file mode 100755 index 8f7876a3..00000000 --- a/mk_dockerfile.sh +++ /dev/null @@ -1,2 +0,0 @@ -set -eu -sed -e 's/${PYTHON_VERSION}/'${PYTHON_VERSION}/g -e 's/${PG_VERSION}/'${PG_VERSION}/g Dockerfile--${TEST_PLATFORM}.tmpl > Dockerfile From 230e5620d68ffef92f31fe67b4ed30881950847d Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Fri, 14 Mar 2025 00:10:11 +0300 Subject: [PATCH 418/475] RemoteOperations::listdir is corrected (#217) - It returns list[str] - New asserts are added Tests for listdir are updated. --- testgres/operations/remote_ops.py | 8 ++++++-- tests/test_local.py | 11 +++++++++++ tests/test_remote.py | 4 +++- 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index d6917c82..60161e3c 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -277,8 +277,12 @@ def listdir(self, path): Args: path (str): The path to the directory. """ - result = self.exec_command("ls {}".format(path)) - return result.splitlines() + command = ["ls", path] + output = self.exec_command(cmd=command, encoding=get_default_encoding()) + assert type(output) == str # noqa: E721 + result = output.splitlines() + assert type(result) == list # noqa: E721 + return result def path_exists(self, path): command = ["test", "-e", path] diff --git a/tests/test_local.py b/tests/test_local.py index 68e7db33..3ae93f76 100644 --- a/tests/test_local.py +++ b/tests/test_local.py @@ -87,6 +87,17 @@ def test_exec_command_failure__expect_error(self): assert b"nonexistent_command" in error assert b"not found" in error + def test_listdir(self): + """ + Test listdir for listing directory contents. + """ + path = "/etc" + files = self.operations.listdir(path) + assert isinstance(files, list) + for f in files: + assert f is not None + assert type(f) == str # noqa: E721 + def test_read__text(self): """ Test LocalOperations::read for text data. diff --git a/tests/test_remote.py b/tests/test_remote.py index 1f771c62..2c37e2c1 100755 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -222,8 +222,10 @@ def test_listdir(self): """ path = "/etc" files = self.operations.listdir(path) - assert isinstance(files, list) + for f in files: + assert f is not None + assert type(f) == str # noqa: E721 def test_path_exists_true__directory(self): """ From cc4361c6fce77df6380aed8e023241b2cdde915f Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Fri, 14 Mar 2025 18:44:28 +0300 Subject: [PATCH 419/475] get_process_children is updated --- testgres/operations/local_ops.py | 1 + testgres/operations/remote_ops.py | 7 ++++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/testgres/operations/local_ops.py b/testgres/operations/local_ops.py index 0fa7d0ad..6ae1cf2b 100644 --- a/testgres/operations/local_ops.py +++ b/testgres/operations/local_ops.py @@ -418,6 +418,7 @@ def get_pid(self): return os.getpid() def get_process_children(self, pid): + assert type(pid) == int # noqa: E721 return psutil.Process(pid).children() # Database control diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index 60161e3c..625a184b 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -599,15 +599,16 @@ def get_pid(self): return int(self.exec_command("echo $$", encoding=get_default_encoding())) def get_process_children(self, pid): - command = ["ssh"] + self.ssh_args + [self.ssh_dest, f"pgrep -P {pid}"] + assert type(pid) == int # noqa: E721 + command = ["ssh"] + self.ssh_args + [self.ssh_dest, "pgrep", "-P", str(pid)] result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) if result.returncode == 0: children = result.stdout.strip().splitlines() return [PsUtilProcessProxy(self, int(child_pid.strip())) for child_pid in children] - else: - raise ExecUtilException(f"Error in getting process children. Error: {result.stderr}") + + raise ExecUtilException(f"Error in getting process children. Error: {result.stderr}") # Database control def db_connect(self, dbname, user, password=None, host="localhost", port=5432): From bab1d8ec8671261787689dec99a4a3ff10499fd7 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Fri, 14 Mar 2025 18:49:58 +0300 Subject: [PATCH 420/475] os_ops::readlines is updated (revision) --- testgres/operations/local_ops.py | 16 +++++++++++++++- testgres/operations/remote_ops.py | 27 +++++++++++++++++++++++---- 2 files changed, 38 insertions(+), 5 deletions(-) diff --git a/testgres/operations/local_ops.py b/testgres/operations/local_ops.py index 6ae1cf2b..93a64787 100644 --- a/testgres/operations/local_ops.py +++ b/testgres/operations/local_ops.py @@ -352,12 +352,26 @@ def readlines(self, filename, num_lines=0, binary=False, encoding=None): Read lines from a local file. If num_lines is greater than 0, only the last num_lines lines will be read. """ + assert type(num_lines) == int # noqa: E721 + assert type(filename) == str # noqa: E721 + assert type(binary) == bool # noqa: E721 + assert encoding is None or type(encoding) == str # noqa: E721 assert num_lines >= 0 + + if binary: + assert encoding is None + pass + elif encoding is None: + encoding = get_default_encoding() + assert type(encoding) == str # noqa: E721 + else: + assert type(encoding) == str # noqa: E721 + pass + mode = 'rb' if binary else 'r' if num_lines == 0: with open(filename, mode, encoding=encoding) as file: # open in binary mode return file.readlines() - else: bufsize = 8192 buffers = 1 diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index 625a184b..e1ad6dac 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -484,18 +484,37 @@ def _read__binary(self, filename): return content def readlines(self, filename, num_lines=0, binary=False, encoding=None): + assert type(num_lines) == int # noqa: E721 + assert type(filename) == str # noqa: E721 + assert type(binary) == bool # noqa: E721 + assert encoding is None or type(encoding) == str # noqa: E721 + if num_lines > 0: - cmd = "tail -n {} {}".format(num_lines, filename) + cmd = ["tail", "-n", str(num_lines), filename] + else: + cmd = ["cat", filename] + + if binary: + assert encoding is None + pass + elif encoding is None: + encoding = get_default_encoding() + assert type(encoding) == str # noqa: E721 else: - cmd = "cat {}".format(filename) + assert type(encoding) == str # noqa: E721 + pass result = self.exec_command(cmd, encoding=encoding) + assert result is not None - if not binary and result: - lines = result.decode(encoding or get_default_encoding()).splitlines() + if binary: + assert type(result) == bytes # noqa: E721 + lines = result.splitlines() else: + assert type(result) == str # noqa: E721 lines = result.splitlines() + assert type(lines) == list # noqa: E721 return lines def read_binary(self, filename, offset): From c07f1127f110bf7dac28ef6f28cad363d404d551 Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Sun, 16 Mar 2025 14:07:04 +0300 Subject: [PATCH 421/475] Initialization of Helpers._get_default_encoding_func is corrected [py3.9] (#221) Python 3.9 does not undestand the following code: _get_default_encoding_func = _make_get_default_encoding_func() ERROR - TypeError: 'staticmethod' object is not callable https://app.travis-ci.com/github/postgrespro/testgres/jobs/631402370 The code: _get_default_encoding_func = _make_get_default_encoding_func.__func__() is processed without problems. --- testgres/operations/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testgres/operations/helpers.py b/testgres/operations/helpers.py index 03e97edc..ebbf0f73 100644 --- a/testgres/operations/helpers.py +++ b/testgres/operations/helpers.py @@ -12,7 +12,7 @@ def _make_get_default_encoding_func(): return locale.getpreferredencoding # Prepared pointer on function to get a name of system codepage - _get_default_encoding_func = _make_get_default_encoding_func() + _get_default_encoding_func = _make_get_default_encoding_func.__func__() @staticmethod def GetDefaultEncoding(): From ea114966771982f0a2fabaaeedd8c89bf8ec1feb Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Sun, 16 Mar 2025 18:04:31 +0300 Subject: [PATCH 422/475] testgres.utils.get_pg_version2 is added This a version of get_pg_version that requires and uses an explicit os_ops object. --- testgres/utils.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/testgres/utils.py b/testgres/utils.py index 093eaff6..a988effe 100644 --- a/testgres/utils.py +++ b/testgres/utils.py @@ -203,19 +203,29 @@ def cache_pg_config_data(cmd): return cache_pg_config_data("pg_config") -def get_pg_version(bin_dir=None): +def get_pg_version2(os_ops: OsOperations, bin_dir=None): """ Return PostgreSQL version provided by postmaster. """ + assert os_ops is not None + assert isinstance(os_ops, OsOperations) # Get raw version (e.g., postgres (PostgreSQL) 9.5.7) - postgres_path = os.path.join(bin_dir, 'postgres') if bin_dir else get_bin_path('postgres') + postgres_path = os.path.join(bin_dir, 'postgres') if bin_dir else get_bin_path2(os_ops, 'postgres') _params = [postgres_path, '--version'] - raw_ver = tconf.os_ops.exec_command(_params, encoding='utf-8') + raw_ver = os_ops.exec_command(_params, encoding='utf-8') return parse_pg_version(raw_ver) +def get_pg_version(bin_dir=None): + """ + Return PostgreSQL version provided by postmaster. + """ + + return get_pg_version2(tconf.os_ops, bin_dir) + + def parse_pg_version(version_out): # Generalize removal of system-specific suffixes (anything in parentheses) raw_ver = re.sub(r'\([^)]*\)', '', version_out).strip() From ac0a2bbd373b5d5852bfb3b3ff0ff6caec2c6633 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Mon, 17 Mar 2025 16:10:55 +0300 Subject: [PATCH 423/475] PostgresNode is updated [os_ops and clone_with_new_name_and_base_dir] 1) Constructor of PostgresNode can get an explicit os_ops object 2) PostgresNode::os_ops property is added 3) New method PostgresNode::clone_with_new_name_and_base_dir is added It is used to right clone an object in NodeBackup::spawn_primary --- testgres/backup.py | 15 +++++++++++-- testgres/node.py | 54 +++++++++++++++++++++++++++++++++++++++------- 2 files changed, 59 insertions(+), 10 deletions(-) diff --git a/testgres/backup.py b/testgres/backup.py index 619c0270..388697b7 100644 --- a/testgres/backup.py +++ b/testgres/backup.py @@ -147,8 +147,19 @@ def spawn_primary(self, name=None, destroy=True): base_dir = self._prepare_dir(destroy) # Build a new PostgresNode - NodeClass = self.original_node.__class__ - with clean_on_error(NodeClass(name=name, base_dir=base_dir, conn_params=self.original_node.os_ops.conn_params)) as node: + assert self.original_node is not None + + if (hasattr(self.original_node, "clone_with_new_name_and_base_dir")): + node = self.original_node.clone_with_new_name_and_base_dir(name=name, base_dir=base_dir) + else: + # For backward compatibility + NodeClass = self.original_node.__class__ + node = NodeClass(name=name, base_dir=base_dir, conn_params=self.original_node.os_ops.conn_params) + + assert node is not None + assert type(node) == self.original_node.__class__ # noqa: E721 + + with clean_on_error(node) as node: # New nodes should always remove dir tree node._should_rm_dirs = True diff --git a/testgres/node.py b/testgres/node.py index 859fe742..6d2417c4 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -100,6 +100,7 @@ from .backup import NodeBackup from .operations.os_ops import ConnectionParams +from .operations.os_ops import OsOperations from .operations.local_ops import LocalOperations from .operations.remote_ops import RemoteOperations @@ -135,7 +136,7 @@ class PostgresNode(object): _C_MAX_START_ATEMPTS = 5 def __init__(self, name=None, base_dir=None, port=None, conn_params: ConnectionParams = ConnectionParams(), - bin_dir=None, prefix=None): + bin_dir=None, prefix=None, os_ops=None): """ PostgresNode constructor. @@ -157,17 +158,20 @@ def __init__(self, name=None, base_dir=None, port=None, conn_params: ConnectionP # basic self.name = name or generate_app_name() - if testgres_config.os_ops: - self.os_ops = testgres_config.os_ops - elif conn_params.ssh_key: - self.os_ops = RemoteOperations(conn_params) + if os_ops is None: + os_ops = __class__._get_os_ops(conn_params) else: - self.os_ops = LocalOperations(conn_params) + assert conn_params is None + pass - self.host = self.os_ops.host + assert os_ops is not None + assert isinstance(os_ops, OsOperations) + self._os_ops = os_ops + + self.host = os_ops.host self.port = port or utils.reserve_port() - self.ssh_key = self.os_ops.ssh_key + self.ssh_key = os_ops.ssh_key # defaults for __exit__() self.cleanup_on_good_exit = testgres_config.node_cleanup_on_good_exit @@ -204,6 +208,40 @@ def __repr__(self): return "{}(name='{}', port={}, base_dir='{}')".format( self.__class__.__name__, self.name, self.port, self.base_dir) + @staticmethod + def _get_os_ops(conn_params: ConnectionParams) -> OsOperations: + if testgres_config.os_ops: + return testgres_config.os_ops + + assert type(conn_params) == ConnectionParams # noqa: E721 + + if conn_params.ssh_key: + return RemoteOperations(conn_params) + + return LocalOperations(conn_params) + + def clone_with_new_name_and_base_dir(self, name: str, base_dir: str): + assert name is None or type(name) == str # noqa: E721 + assert base_dir is None or type(base_dir) == str # noqa: E721 + + assert __class__ == PostgresNode + + node = PostgresNode( + name=name, + base_dir=base_dir, + conn_params=None, + bin_dir=self._bin_dir, + prefix=self._prefix, + os_ops=self._os_ops) + + return node + + @property + def os_ops(self) -> OsOperations: + assert self._os_ops is not None + assert isinstance(self._os_ops, OsOperations) + return self._os_ops + @property def pid(self): """ From dcb7f24592b157297606c29985621a28ebeebeaa Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Mon, 17 Mar 2025 16:34:30 +0300 Subject: [PATCH 424/475] TestTestgresCommon - generic tests for local and remote modes. This commit replaces #216 --- run_tests.sh | 4 +- tests/test_simple.py | 979 ---------------------------- tests/test_simple_remote.py | 993 +---------------------------- tests/test_testgres_common.py | 1131 +++++++++++++++++++++++++++++++++ 4 files changed, 1142 insertions(+), 1965 deletions(-) create mode 100644 tests/test_testgres_common.py diff --git a/run_tests.sh b/run_tests.sh index 021f9d9f..0fecde60 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -4,7 +4,9 @@ set -eux -if [ -z ${TEST_FILTER+x} ]; then export TEST_FILTER="TestgresTests"; fi +if [ -z ${TEST_FILTER+x} ]; \ +then export TEST_FILTER="TestgresTests or (TestTestgresCommon and (not remote_ops))"; \ +fi # choose python version echo python version is $PYTHON_VERSION diff --git a/tests/test_simple.py b/tests/test_simple.py index e886a39c..f648e558 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -2,29 +2,16 @@ import os import re import subprocess -import tempfile -import time -import six import pytest import psutil import platform import logging -import uuid - -from contextlib import contextmanager -from shutil import rmtree from .. import testgres from ..testgres import \ - InitNodeException, \ StartNodeException, \ ExecUtilException, \ - BackupException, \ - QueryException, \ - TimeoutException, \ - TestgresException, \ - InvalidOperationException, \ NodeApp from ..testgres import \ @@ -34,9 +21,6 @@ pop_config from ..testgres import \ - NodeStatus, \ - ProcessType, \ - IsolationLevel, \ get_new_node from ..testgres import \ @@ -44,14 +28,9 @@ get_pg_config, \ get_pg_version -from ..testgres import \ - First, \ - Any - # NOTE: those are ugly imports from ..testgres import bound_ports from ..testgres.utils import PgVer, parse_pg_version -from ..testgres.utils import file_tail from ..testgres.node import ProcessProxy @@ -95,17 +74,6 @@ def rm_carriage_returns(out): return out -@contextmanager -def removing(f): - try: - yield f - finally: - if os.path.isfile(f): - os.remove(f) - elif os.path.isdir(f): - rmtree(f, ignore_errors=True) - - class TestgresTests: def test_node_repr(self): with get_new_node() as node: @@ -132,740 +100,6 @@ def test_custom_init(self): # there should be no trust entries at all assert not (any('trust' in s for s in lines)) - def test_double_init(self): - with get_new_node().init() as node: - # can't initialize node more than once - with pytest.raises(expected_exception=InitNodeException): - node.init() - - def test_init_after_cleanup(self): - with get_new_node() as node: - node.init().start().execute('select 1') - node.cleanup() - node.init().start().execute('select 1') - - def test_init_unique_system_id(self): - # this function exists in PostgreSQL 9.6+ - __class__.helper__skip_test_if_util_not_exist("pg_resetwal") - __class__.helper__skip_test_if_pg_version_is_not_ge("9.6") - - query = 'select system_identifier from pg_control_system()' - - with scoped_config(cache_initdb=False): - with get_new_node().init().start() as node0: - id0 = node0.execute(query)[0] - - with scoped_config(cache_initdb=True, - cached_initdb_unique=True) as config: - assert (config.cache_initdb) - assert (config.cached_initdb_unique) - - # spawn two nodes; ids must be different - with get_new_node().init().start() as node1, \ - get_new_node().init().start() as node2: - - id1 = node1.execute(query)[0] - id2 = node2.execute(query)[0] - - # ids must increase - assert (id1 > id0) - assert (id2 > id1) - - def test_node_exit(self): - base_dir = None - - with pytest.raises(expected_exception=QueryException): - with get_new_node().init() as node: - base_dir = node.base_dir - node.safe_psql('select 1') - - # we should save the DB for "debugging" - assert (os.path.exists(base_dir)) - rmtree(base_dir, ignore_errors=True) - - with get_new_node().init() as node: - base_dir = node.base_dir - - # should have been removed by default - assert not (os.path.exists(base_dir)) - - def test_double_start(self): - with get_new_node().init().start() as node: - # can't start node more than once - node.start() - assert (node.is_started) - - def test_uninitialized_start(self): - with get_new_node() as node: - # node is not initialized yet - with pytest.raises(expected_exception=StartNodeException): - node.start() - - def test_restart(self): - with get_new_node() as node: - node.init().start() - - # restart, ok - res = node.execute('select 1') - assert (res == [(1, )]) - node.restart() - res = node.execute('select 2') - assert (res == [(2, )]) - - # restart, fail - with pytest.raises(expected_exception=StartNodeException): - node.append_conf('pg_hba.conf', 'DUMMY') - node.restart() - - def test_reload(self): - with get_new_node() as node: - node.init().start() - - # change client_min_messages and save old value - cmm_old = node.execute('show client_min_messages') - node.append_conf(client_min_messages='DEBUG1') - - # reload config - node.reload() - - # check new value - cmm_new = node.execute('show client_min_messages') - assert ('debug1' == cmm_new[0][0].lower()) - assert (cmm_old != cmm_new) - - def test_pg_ctl(self): - with get_new_node() as node: - node.init().start() - - status = node.pg_ctl(['status']) - assert ('PID' in status) - - def test_status(self): - assert (NodeStatus.Running) - assert not (NodeStatus.Stopped) - assert not (NodeStatus.Uninitialized) - - # check statuses after each operation - with get_new_node() as node: - assert (node.pid == 0) - assert (node.status() == NodeStatus.Uninitialized) - - node.init() - - assert (node.pid == 0) - assert (node.status() == NodeStatus.Stopped) - - node.start() - - assert (node.pid != 0) - assert (node.status() == NodeStatus.Running) - - node.stop() - - assert (node.pid == 0) - assert (node.status() == NodeStatus.Stopped) - - node.cleanup() - - assert (node.pid == 0) - assert (node.status() == NodeStatus.Uninitialized) - - def test_psql(self): - with get_new_node().init().start() as node: - - # check returned values (1 arg) - res = node.psql('select 1') - assert (rm_carriage_returns(res) == (0, b'1\n', b'')) - - # check returned values (2 args) - res = node.psql('postgres', 'select 2') - assert (rm_carriage_returns(res) == (0, b'2\n', b'')) - - # check returned values (named) - res = node.psql(query='select 3', dbname='postgres') - assert (rm_carriage_returns(res) == (0, b'3\n', b'')) - - # check returned values (1 arg) - res = node.safe_psql('select 4') - assert (rm_carriage_returns(res) == b'4\n') - - # check returned values (2 args) - res = node.safe_psql('postgres', 'select 5') - assert (rm_carriage_returns(res) == b'5\n') - - # check returned values (named) - res = node.safe_psql(query='select 6', dbname='postgres') - assert (rm_carriage_returns(res) == b'6\n') - - # check feeding input - node.safe_psql('create table horns (w int)') - node.safe_psql('copy horns from stdin (format csv)', - input=b"1\n2\n3\n\\.\n") - _sum = node.safe_psql('select sum(w) from horns') - assert (rm_carriage_returns(_sum) == b'6\n') - - # check psql's default args, fails - with pytest.raises(expected_exception=QueryException): - node.psql() - - node.stop() - - # check psql on stopped node, fails - with pytest.raises(expected_exception=QueryException): - node.safe_psql('select 1') - - def test_safe_psql__expect_error(self): - with get_new_node().init().start() as node: - err = node.safe_psql('select_or_not_select 1', expect_error=True) - assert (type(err) == str) # noqa: E721 - assert ('select_or_not_select' in err) - assert ('ERROR: syntax error at or near "select_or_not_select"' in err) - - # --------- - with pytest.raises( - expected_exception=InvalidOperationException, - match="^" + re.escape("Exception was expected, but query finished successfully: `select 1;`.") + "$" - ): - node.safe_psql("select 1;", expect_error=True) - - # --------- - res = node.safe_psql("select 1;", expect_error=False) - assert (rm_carriage_returns(res) == b'1\n') - - def test_transactions(self): - with get_new_node().init().start() as node: - - with node.connect() as con: - con.begin() - con.execute('create table test(val int)') - con.execute('insert into test values (1)') - con.commit() - - con.begin() - con.execute('insert into test values (2)') - res = con.execute('select * from test order by val asc') - assert (res == [(1, ), (2, )]) - con.rollback() - - con.begin() - res = con.execute('select * from test') - assert (res == [(1, )]) - con.rollback() - - con.begin() - con.execute('drop table test') - con.commit() - - def test_control_data(self): - with get_new_node() as node: - - # node is not initialized yet - with pytest.raises(expected_exception=ExecUtilException): - node.get_control_data() - - node.init() - data = node.get_control_data() - - # check returned dict - assert data is not None - assert (any('pg_control' in s for s in data.keys())) - - def test_backup_simple(self): - with get_new_node() as master: - - # enable streaming for backups - master.init(allow_streaming=True) - - # node must be running - with pytest.raises(expected_exception=BackupException): - master.backup() - - # it's time to start node - master.start() - - # fill node with some data - master.psql('create table test as select generate_series(1, 4) i') - - with master.backup(xlog_method='stream') as backup: - with backup.spawn_primary().start() as slave: - res = slave.execute('select * from test order by i asc') - assert (res == [(1, ), (2, ), (3, ), (4, )]) - - def test_backup_multiple(self): - with get_new_node() as node: - node.init(allow_streaming=True).start() - - with node.backup(xlog_method='fetch') as backup1, \ - node.backup(xlog_method='fetch') as backup2: - assert (backup1.base_dir != backup2.base_dir) - - with node.backup(xlog_method='fetch') as backup: - with backup.spawn_primary('node1', destroy=False) as node1, \ - backup.spawn_primary('node2', destroy=False) as node2: - assert (node1.base_dir != node2.base_dir) - - def test_backup_exhaust(self): - with get_new_node() as node: - node.init(allow_streaming=True).start() - - with node.backup(xlog_method='fetch') as backup: - # exhaust backup by creating new node - with backup.spawn_primary(): - pass - - # now let's try to create one more node - with pytest.raises(expected_exception=BackupException): - backup.spawn_primary() - - def test_backup_wrong_xlog_method(self): - with get_new_node() as node: - node.init(allow_streaming=True).start() - - with pytest.raises( - expected_exception=BackupException, - match="^" + re.escape('Invalid xlog_method "wrong"') + "$" - ): - node.backup(xlog_method='wrong') - - def test_pg_ctl_wait_option(self): - C_MAX_ATTEMPTS = 50 - - node = get_new_node() - assert node.status() == testgres.NodeStatus.Uninitialized - node.init() - assert node.status() == testgres.NodeStatus.Stopped - node.start(wait=False) - nAttempt = 0 - while True: - if nAttempt == C_MAX_ATTEMPTS: - # - # [2025-03-11] - # We have an unexpected problem with this test in CI - # Let's get an additional information about this test failure. - # - logging.error("Node was not stopped.") - if not node.os_ops.path_exists(node.pg_log_file): - logging.warning("Node log does not exist.") - else: - logging.info("Let's read node log file [{0}]".format(node.pg_log_file)) - logFileData = node.os_ops.read(node.pg_log_file, binary=False) - logging.info("Node log file content:\n{0}".format(logFileData)) - - raise Exception("Could not stop node.") - - nAttempt += 1 - - if nAttempt > 1: - logging.info("Wait 1 second.") - time.sleep(1) - logging.info("") - - logging.info("Try to stop node. Attempt #{0}.".format(nAttempt)) - - try: - node.stop(wait=False) - break - except ExecUtilException as e: - # it's ok to get this exception here since node - # could be not started yet - logging.info("Node is not stopped. Exception ({0}): {1}".format(type(e).__name__, e)) - continue - - logging.info("OK. Stop command was executed. Let's wait while our node will stop really.") - nAttempt = 0 - while True: - if nAttempt == C_MAX_ATTEMPTS: - raise Exception("Could not stop node.") - - nAttempt += 1 - if nAttempt > 1: - logging.info("Wait 1 second.") - time.sleep(1) - logging.info("") - - logging.info("Attempt #{0}.".format(nAttempt)) - s1 = node.status() - - if s1 == testgres.NodeStatus.Running: - continue - - if s1 == testgres.NodeStatus.Stopped: - break - - raise Exception("Unexpected node status: {0}.".format(s1)) - - logging.info("OK. Node is stopped.") - node.cleanup() - - def test_replicate(self): - with get_new_node() as node: - node.init(allow_streaming=True).start() - - with node.replicate().start() as replica: - res = replica.execute('select 1') - assert (res == [(1, )]) - - node.execute('create table test (val int)', commit=True) - - replica.catchup() - - res = node.execute('select * from test') - assert (res == []) - - def test_synchronous_replication(self): - __class__.helper__skip_test_if_pg_version_is_not_ge("9.6") - - with get_new_node() as master: - old_version = not pg_version_ge('9.6') - - master.init(allow_streaming=True).start() - - if not old_version: - master.append_conf('synchronous_commit = remote_apply') - - # create standby - with master.replicate() as standby1, master.replicate() as standby2: - standby1.start() - standby2.start() - - # check formatting - assert ( - '1 ("{}", "{}")'.format(standby1.name, standby2.name) == str(First(1, (standby1, standby2))) - ) # yapf: disable - assert ( - 'ANY 1 ("{}", "{}")'.format(standby1.name, standby2.name) == str(Any(1, (standby1, standby2))) - ) # yapf: disable - - # set synchronous_standby_names - master.set_synchronous_standbys(First(2, [standby1, standby2])) - master.restart() - - # the following part of the test is only applicable to newer - # versions of PostgresQL - if not old_version: - master.safe_psql('create table abc(a int)') - - # Create a large transaction that will take some time to apply - # on standby to check that it applies synchronously - # (If set synchronous_commit to 'on' or other lower level then - # standby most likely won't catchup so fast and test will fail) - master.safe_psql( - 'insert into abc select generate_series(1, 1000000)') - res = standby1.safe_psql('select count(*) from abc') - assert (rm_carriage_returns(res) == b'1000000\n') - - def test_logical_replication(self): - __class__.helper__skip_test_if_pg_version_is_not_ge("10") - - with get_new_node() as node1, get_new_node() as node2: - node1.init(allow_logical=True) - node1.start() - node2.init().start() - - create_table = 'create table test (a int, b int)' - node1.safe_psql(create_table) - node2.safe_psql(create_table) - - # create publication / create subscription - pub = node1.publish('mypub') - sub = node2.subscribe(pub, 'mysub') - - node1.safe_psql('insert into test values (1, 1), (2, 2)') - - # wait until changes apply on subscriber and check them - sub.catchup() - res = node2.execute('select * from test') - assert (res == [(1, 1), (2, 2)]) - - # disable and put some new data - sub.disable() - node1.safe_psql('insert into test values (3, 3)') - - # enable and ensure that data successfully transferred - sub.enable() - sub.catchup() - res = node2.execute('select * from test') - assert (res == [(1, 1), (2, 2), (3, 3)]) - - # Add new tables. Since we added "all tables" to publication - # (default behaviour of publish() method) we don't need - # to explicitly perform pub.add_tables() - create_table = 'create table test2 (c char)' - node1.safe_psql(create_table) - node2.safe_psql(create_table) - sub.refresh() - - # put new data - node1.safe_psql('insert into test2 values (\'a\'), (\'b\')') - sub.catchup() - res = node2.execute('select * from test2') - assert (res == [('a', ), ('b', )]) - - # drop subscription - sub.drop() - pub.drop() - - # create new publication and subscription for specific table - # (omitting copying data as it's already done) - pub = node1.publish('newpub', tables=['test']) - sub = node2.subscribe(pub, 'newsub', copy_data=False) - - node1.safe_psql('insert into test values (4, 4)') - sub.catchup() - res = node2.execute('select * from test') - assert (res == [(1, 1), (2, 2), (3, 3), (4, 4)]) - - # explicitly add table - with pytest.raises(expected_exception=ValueError): - pub.add_tables([]) # fail - pub.add_tables(['test2']) - node1.safe_psql('insert into test2 values (\'c\')') - sub.catchup() - res = node2.execute('select * from test2') - assert (res == [('a', ), ('b', )]) - - def test_logical_catchup(self): - """ Runs catchup for 100 times to be sure that it is consistent """ - __class__.helper__skip_test_if_pg_version_is_not_ge("10") - - with get_new_node() as node1, get_new_node() as node2: - node1.init(allow_logical=True) - node1.start() - node2.init().start() - - create_table = 'create table test (key int primary key, val int); ' - node1.safe_psql(create_table) - node1.safe_psql('alter table test replica identity default') - node2.safe_psql(create_table) - - # create publication / create subscription - sub = node2.subscribe(node1.publish('mypub'), 'mysub') - - for i in range(0, 100): - node1.execute('insert into test values ({0}, {0})'.format(i)) - sub.catchup() - res = node2.execute('select * from test') - assert (res == [(i, i, )]) - node1.execute('delete from test') - - def test_logical_replication_fail(self): - __class__.helper__skip_test_if_pg_version_is_ge("10") - - with get_new_node() as node: - with pytest.raises(expected_exception=InitNodeException): - node.init(allow_logical=True) - - def test_replication_slots(self): - with get_new_node() as node: - node.init(allow_streaming=True).start() - - with node.replicate(slot='slot1').start() as replica: - replica.execute('select 1') - - # cannot create new slot with the same name - with pytest.raises(expected_exception=TestgresException): - node.replicate(slot='slot1') - - def test_incorrect_catchup(self): - with get_new_node() as node: - node.init(allow_streaming=True).start() - - # node has no master, can't catch up - with pytest.raises(expected_exception=TestgresException): - node.catchup() - - def test_promotion(self): - with get_new_node() as master: - master.init().start() - master.safe_psql('create table abc(id serial)') - - with master.replicate().start() as replica: - master.stop() - replica.promote() - - # make standby becomes writable master - replica.safe_psql('insert into abc values (1)') - res = replica.safe_psql('select * from abc') - assert (rm_carriage_returns(res) == b'1\n') - - def test_dump(self): - query_create = 'create table test as select generate_series(1, 2) as val' - query_select = 'select * from test order by val asc' - - with get_new_node().init().start() as node1: - - node1.execute(query_create) - for format in ['plain', 'custom', 'directory', 'tar']: - with removing(node1.dump(format=format)) as dump: - with get_new_node().init().start() as node3: - if format == 'directory': - assert (os.path.isdir(dump)) - else: - assert (os.path.isfile(dump)) - # restore dump - node3.restore(filename=dump) - res = node3.execute(query_select) - assert (res == [(1, ), (2, )]) - - def test_users(self): - with get_new_node().init().start() as node: - node.psql('create role test_user login') - value = node.safe_psql('select 1', username='test_user') - value = rm_carriage_returns(value) - assert (value == b'1\n') - - def test_poll_query_until(self): - with get_new_node() as node: - node.init().start() - - get_time = 'select extract(epoch from now())' - check_time = 'select extract(epoch from now()) - {} >= 5' - - start_time = node.execute(get_time)[0][0] - node.poll_query_until(query=check_time.format(start_time)) - end_time = node.execute(get_time)[0][0] - - assert (end_time - start_time >= 5) - - # check 0 columns - with pytest.raises(expected_exception=QueryException): - node.poll_query_until( - query='select from pg_catalog.pg_class limit 1') - - # check None, fail - with pytest.raises(expected_exception=QueryException): - node.poll_query_until(query='create table abc (val int)') - - # check None, ok - node.poll_query_until(query='create table def()', - expected=None) # returns nothing - - # check 0 rows equivalent to expected=None - node.poll_query_until( - query='select * from pg_catalog.pg_class where true = false', - expected=None) - - # check arbitrary expected value, fail - with pytest.raises(expected_exception=TimeoutException): - node.poll_query_until(query='select 3', - expected=1, - max_attempts=3, - sleep_time=0.01) - - # check arbitrary expected value, ok - node.poll_query_until(query='select 2', expected=2) - - # check timeout - with pytest.raises(expected_exception=TimeoutException): - node.poll_query_until(query='select 1 > 2', - max_attempts=3, - sleep_time=0.01) - - # check ProgrammingError, fail - with pytest.raises(expected_exception=testgres.ProgrammingError): - node.poll_query_until(query='dummy1') - - # check ProgrammingError, ok - with pytest.raises(expected_exception=(TimeoutException)): - node.poll_query_until(query='dummy2', - max_attempts=3, - sleep_time=0.01, - suppress={testgres.ProgrammingError}) - - # check 1 arg, ok - node.poll_query_until('select true') - - def test_logging(self): - C_MAX_ATTEMPTS = 50 - # This name is used for testgres logging, too. - C_NODE_NAME = "testgres_tests." + __class__.__name__ + "test_logging-master-" + uuid.uuid4().hex - - logging.info("Node name is [{0}]".format(C_NODE_NAME)) - - with tempfile.NamedTemporaryFile('w', delete=True) as logfile: - formatter = logging.Formatter(fmt="%(node)-5s: %(message)s") - handler = logging.FileHandler(filename=logfile.name) - handler.formatter = formatter - logger = logging.getLogger(C_NODE_NAME) - assert logger is not None - assert len(logger.handlers) == 0 - - try: - # It disables to log on the root level - logger.propagate = False - logger.addHandler(handler) - - with scoped_config(use_python_logging=True): - with get_new_node(name=C_NODE_NAME) as master: - logging.info("Master node is initilizing") - master.init() - - logging.info("Master node is starting") - master.start() - - logging.info("Dummy query is executed a few times") - for _ in range(20): - master.execute('select 1') - time.sleep(0.01) - - # let logging worker do the job - time.sleep(0.1) - - logging.info("Master node log file is checking") - nAttempt = 0 - - while True: - assert nAttempt <= C_MAX_ATTEMPTS - if nAttempt == C_MAX_ATTEMPTS: - raise Exception("Test failed!") - - # let logging worker do the job - time.sleep(0.1) - - nAttempt += 1 - - logging.info("Attempt {0}".format(nAttempt)) - - # check that master's port is found - with open(logfile.name, 'r') as log: - lines = log.readlines() - - assert lines is not None - assert type(lines) == list # noqa: E721 - - def LOCAL__test_lines(): - for s in lines: - if any(C_NODE_NAME in s for s in lines): - logging.info("OK. We found the node_name in a line \"{0}\"".format(s)) - return True - return False - - if LOCAL__test_lines(): - break - - logging.info("Master node log file does not have an expected information.") - continue - - # test logger after stop/start/restart - logging.info("Master node is stopping...") - master.stop() - logging.info("Master node is staring again...") - master.start() - logging.info("Master node is restaring...") - master.restart() - assert (master._logger.is_alive()) - finally: - # It is a hack code to logging cleanup - logging._acquireLock() - assert logging.Logger.manager is not None - assert C_NODE_NAME in logging.Logger.manager.loggerDict.keys() - logging.Logger.manager.loggerDict.pop(C_NODE_NAME, None) - assert not (C_NODE_NAME in logging.Logger.manager.loggerDict.keys()) - assert not (handler in logging._handlers.values()) - logging._releaseLock() - # GO HOME! - return - def test_pgbench(self): __class__.helper__skip_test_if_util_not_exist("pgbench") @@ -955,60 +189,6 @@ def test_unix_sockets(self): r.execute('select 1') r.safe_psql('select 1') - def test_auto_name(self): - with get_new_node().init(allow_streaming=True).start() as m: - with m.replicate().start() as r: - # check that nodes are running - assert (m.status()) - assert (r.status()) - - # check their names - assert (m.name != r.name) - assert ('testgres' in m.name) - assert ('testgres' in r.name) - - def test_file_tail(self): - s1 = "the quick brown fox jumped over that lazy dog\n" - s2 = "abc\n" - s3 = "def\n" - - with tempfile.NamedTemporaryFile(mode='r+', delete=True) as f: - sz = 0 - while sz < 3 * 8192: - sz += len(s1) - f.write(s1) - f.write(s2) - f.write(s3) - - f.seek(0) - lines = file_tail(f, 3) - assert (lines[0] == s1) - assert (lines[1] == s2) - assert (lines[2] == s3) - - f.seek(0) - lines = file_tail(f, 1) - assert (lines[0] == s3) - - def test_isolation_levels(self): - with get_new_node().init().start() as node: - with node.connect() as con: - # string levels - con.begin('Read Uncommitted').commit() - con.begin('Read Committed').commit() - con.begin('Repeatable Read').commit() - con.begin('Serializable').commit() - - # enum levels - con.begin(IsolationLevel.ReadUncommitted).commit() - con.begin(IsolationLevel.ReadCommitted).commit() - con.begin(IsolationLevel.RepeatableRead).commit() - con.begin(IsolationLevel.Serializable).commit() - - # check wrong level - with pytest.raises(expected_exception=QueryException): - con.begin('Garbage').commit() - def test_ports_management(self): assert bound_ports is not None assert type(bound_ports) == set # noqa: E721 @@ -1043,153 +223,6 @@ def test_ports_management(self): assert type(bound_ports) == set # noqa: E721 assert bound_ports == stage0__bound_ports - def test_exceptions(self): - str(StartNodeException('msg', [('file', 'lines')])) - str(ExecUtilException('msg', 'cmd', 1, 'out')) - str(QueryException('msg', 'query')) - - def test_version_management(self): - a = PgVer('10.0') - b = PgVer('10') - c = PgVer('9.6.5') - d = PgVer('15.0') - e = PgVer('15rc1') - f = PgVer('15beta4') - h = PgVer('15.3biha') - i = PgVer('15.3') - g = PgVer('15.3.1bihabeta1') - k = PgVer('15.3.1') - - assert (a == b) - assert (b > c) - assert (a > c) - assert (d > e) - assert (e > f) - assert (d > f) - assert (h > f) - assert (h == i) - assert (g == k) - assert (g > h) - - version = get_pg_version() - with get_new_node() as node: - assert (isinstance(version, six.string_types)) - assert (isinstance(node.version, PgVer)) - assert (node.version == PgVer(version)) - - def test_child_pids(self): - master_processes = [ - ProcessType.AutovacuumLauncher, - ProcessType.BackgroundWriter, - ProcessType.Checkpointer, - ProcessType.StatsCollector, - ProcessType.WalSender, - ProcessType.WalWriter, - ] - - if pg_version_ge('10'): - master_processes.append(ProcessType.LogicalReplicationLauncher) - - if pg_version_ge('14'): - master_processes.remove(ProcessType.StatsCollector) - - repl_processes = [ - ProcessType.Startup, - ProcessType.WalReceiver, - ] - - def LOCAL__test_auxiliary_pids( - node: testgres.PostgresNode, - expectedTypes: list[ProcessType] - ) -> list[ProcessType]: - # returns list of the absence processes - assert node is not None - assert type(node) == testgres.PostgresNode # noqa: E721 - assert expectedTypes is not None - assert type(expectedTypes) == list # noqa: E721 - - pids = node.auxiliary_pids - assert pids is not None # noqa: E721 - assert type(pids) == dict # noqa: E721 - - result = list[ProcessType]() - for ptype in expectedTypes: - if not (ptype in pids): - result.append(ptype) - return result - - def LOCAL__check_auxiliary_pids__multiple_attempts( - node: testgres.PostgresNode, - expectedTypes: list[ProcessType]): - assert node is not None - assert type(node) == testgres.PostgresNode # noqa: E721 - assert expectedTypes is not None - assert type(expectedTypes) == list # noqa: E721 - - nAttempt = 0 - - while nAttempt < 5: - nAttempt += 1 - - logging.info("Test pids of [{0}] node. Attempt #{1}.".format( - node.name, - nAttempt - )) - - if nAttempt > 1: - time.sleep(1) - - absenceList = LOCAL__test_auxiliary_pids(node, expectedTypes) - assert absenceList is not None - assert type(absenceList) == list # noqa: E721 - if len(absenceList) == 0: - logging.info("Bingo!") - return - - logging.info("These processes are not found: {0}.".format(absenceList)) - continue - - raise Exception("Node {0} does not have the following processes: {1}.".format( - node.name, - absenceList - )) - - with get_new_node().init().start() as master: - - # master node doesn't have a source walsender! - with pytest.raises(expected_exception=TestgresException): - master.source_walsender - - with master.connect() as con: - assert (con.pid > 0) - - with master.replicate().start() as replica: - - # test __str__ method - str(master.child_processes[0]) - - LOCAL__check_auxiliary_pids__multiple_attempts( - master, - master_processes) - - LOCAL__check_auxiliary_pids__multiple_attempts( - replica, - repl_processes) - - master_pids = master.auxiliary_pids - - # there should be exactly 1 source walsender for replica - assert (len(master_pids[ProcessType.WalSender]) == 1) - pid1 = master_pids[ProcessType.WalSender][0] - pid2 = replica.source_walsender.pid - assert (pid1 == pid2) - - replica.stop() - - # there should be no walsender after we've stopped replica - with pytest.raises(expected_exception=TestgresException): - replica.source_walsender - def test_child_process_dies(self): # test for FileNotFound exception during child_processes() function cmd = ["timeout", "60"] if os.name == 'nt' else ["sleep", "60"] @@ -1512,15 +545,3 @@ def helper__skip_test_if_util_not_exist(name: str): if not util_exists(name2): pytest.skip('might be missing') - - @staticmethod - def helper__skip_test_if_pg_version_is_not_ge(version: str): - assert type(version) == str # noqa: E721 - if not pg_version_ge(version): - pytest.skip('requires {0}+'.format(version)) - - @staticmethod - def helper__skip_test_if_pg_version_is_ge(version: str): - assert type(version) == str # noqa: E721 - if pg_version_ge(version): - pytest.skip('requires <{0}'.format(version)) diff --git a/tests/test_simple_remote.py b/tests/test_simple_remote.py index d484f1e3..c16fe53f 100755 --- a/tests/test_simple_remote.py +++ b/tests/test_simple_remote.py @@ -2,28 +2,19 @@ import os import re import subprocess -import tempfile -import time -import six import pytest import psutil import logging -import uuid -from contextlib import contextmanager +from .helpers.os_ops_descrs import OsOpsDescrs +from .helpers.os_ops_descrs import OsOperations from .. import testgres from ..testgres.exceptions import \ InitNodeException, \ - StartNodeException, \ - ExecUtilException, \ - BackupException, \ - QueryException, \ - TimeoutException, \ - TestgresException, \ - InvalidOperationException + ExecUtilException from ..testgres.config import \ TestgresConfig, \ @@ -31,33 +22,13 @@ scoped_config, \ pop_config, testgres_config -from ..testgres import \ - NodeStatus, \ - ProcessType, \ - IsolationLevel, \ - get_remote_node, \ - RemoteOperations - from ..testgres import \ get_bin_path, \ - get_pg_config, \ - get_pg_version - -from ..testgres import \ - First, \ - Any + get_pg_config # NOTE: those are ugly imports from ..testgres import bound_ports -from ..testgres.utils import PgVer -from ..testgres.utils import file_tail -from ..testgres.node import ProcessProxy, ConnectionParams - - -def pg_version_ge(version): - cur_ver = PgVer(get_pg_version()) - min_ver = PgVer(version) - return cur_ver >= min_ver +from ..testgres.node import ProcessProxy def util_exists(util): @@ -76,25 +47,8 @@ def good_properties(f): return True -@contextmanager -def removing(f): - try: - yield f - finally: - if testgres_config.os_ops.isfile(f): - testgres_config.os_ops.remove_file(f) - - elif testgres_config.os_ops.isdir(f): - testgres_config.os_ops.rmdirs(f, ignore_errors=True) - - class TestgresRemoteTests: - sm_conn_params = ConnectionParams( - host=os.getenv('RDBMS_TESTPOOL1_HOST') or '127.0.0.1', - username=os.getenv('USER'), - ssh_key=os.getenv('RDBMS_TESTPOOL_SSHKEY')) - - sm_os_ops = RemoteOperations(sm_conn_params) + sm_os_ops = OsOpsDescrs.sm_remote_os_ops @pytest.fixture(autouse=True, scope="class") def implicit_fixture(self): @@ -218,732 +172,6 @@ def test_init__unk_LANG_and_LC_CTYPE(self): __class__.helper__restore_envvar("LC_CTYPE", prev_LC_CTYPE) __class__.helper__restore_envvar("LC_COLLATE", prev_LC_COLLATE) - def test_double_init(self): - with __class__.helper__get_node().init() as node: - # can't initialize node more than once - with pytest.raises(expected_exception=InitNodeException): - node.init() - - def test_init_after_cleanup(self): - with __class__.helper__get_node() as node: - node.init().start().execute('select 1') - node.cleanup() - node.init().start().execute('select 1') - - def test_init_unique_system_id(self): - # this function exists in PostgreSQL 9.6+ - __class__.helper__skip_test_if_util_not_exist("pg_resetwal") - __class__.helper__skip_test_if_pg_version_is_not_ge('9.6') - - query = 'select system_identifier from pg_control_system()' - - with scoped_config(cache_initdb=False): - with __class__.helper__get_node().init().start() as node0: - id0 = node0.execute(query)[0] - - with scoped_config(cache_initdb=True, - cached_initdb_unique=True) as config: - assert (config.cache_initdb) - assert (config.cached_initdb_unique) - - # spawn two nodes; ids must be different - with __class__.helper__get_node().init().start() as node1, \ - __class__.helper__get_node().init().start() as node2: - id1 = node1.execute(query)[0] - id2 = node2.execute(query)[0] - - # ids must increase - assert (id1 > id0) - assert (id2 > id1) - - def test_node_exit(self): - with pytest.raises(expected_exception=QueryException): - with __class__.helper__get_node().init() as node: - base_dir = node.base_dir - node.safe_psql('select 1') - - # we should save the DB for "debugging" - assert (__class__.sm_os_ops.path_exists(base_dir)) - __class__.sm_os_ops.rmdirs(base_dir, ignore_errors=True) - - with __class__.helper__get_node().init() as node: - base_dir = node.base_dir - - # should have been removed by default - assert not (__class__.sm_os_ops.path_exists(base_dir)) - - def test_double_start(self): - with __class__.helper__get_node().init().start() as node: - # can't start node more than once - node.start() - assert (node.is_started) - - def test_uninitialized_start(self): - with __class__.helper__get_node() as node: - # node is not initialized yet - with pytest.raises(expected_exception=StartNodeException): - node.start() - - def test_restart(self): - with __class__.helper__get_node() as node: - node.init().start() - - # restart, ok - res = node.execute('select 1') - assert (res == [(1,)]) - node.restart() - res = node.execute('select 2') - assert (res == [(2,)]) - - # restart, fail - with pytest.raises(expected_exception=StartNodeException): - node.append_conf('pg_hba.conf', 'DUMMY') - node.restart() - - def test_reload(self): - with __class__.helper__get_node() as node: - node.init().start() - - # change client_min_messages and save old value - cmm_old = node.execute('show client_min_messages') - node.append_conf(client_min_messages='DEBUG1') - - # reload config - node.reload() - - # check new value - cmm_new = node.execute('show client_min_messages') - assert ('debug1' == cmm_new[0][0].lower()) - assert (cmm_old != cmm_new) - - def test_pg_ctl(self): - with __class__.helper__get_node() as node: - node.init().start() - - status = node.pg_ctl(['status']) - assert ('PID' in status) - - def test_status(self): - assert (NodeStatus.Running) - assert not (NodeStatus.Stopped) - assert not (NodeStatus.Uninitialized) - - # check statuses after each operation - with __class__.helper__get_node() as node: - assert (node.pid == 0) - assert (node.status() == NodeStatus.Uninitialized) - - node.init() - - assert (node.pid == 0) - assert (node.status() == NodeStatus.Stopped) - - node.start() - - assert (node.pid != 0) - assert (node.status() == NodeStatus.Running) - - node.stop() - - assert (node.pid == 0) - assert (node.status() == NodeStatus.Stopped) - - node.cleanup() - - assert (node.pid == 0) - assert (node.status() == NodeStatus.Uninitialized) - - def test_psql(self): - with __class__.helper__get_node().init().start() as node: - # check returned values (1 arg) - res = node.psql('select 1') - assert (res == (0, b'1\n', b'')) - - # check returned values (2 args) - res = node.psql('postgres', 'select 2') - assert (res == (0, b'2\n', b'')) - - # check returned values (named) - res = node.psql(query='select 3', dbname='postgres') - assert (res == (0, b'3\n', b'')) - - # check returned values (1 arg) - res = node.safe_psql('select 4') - assert (res == b'4\n') - - # check returned values (2 args) - res = node.safe_psql('postgres', 'select 5') - assert (res == b'5\n') - - # check returned values (named) - res = node.safe_psql(query='select 6', dbname='postgres') - assert (res == b'6\n') - - # check feeding input - node.safe_psql('create table horns (w int)') - node.safe_psql('copy horns from stdin (format csv)', - input=b"1\n2\n3\n\\.\n") - _sum = node.safe_psql('select sum(w) from horns') - assert (_sum == b'6\n') - - # check psql's default args, fails - with pytest.raises(expected_exception=QueryException): - node.psql() - - node.stop() - - # check psql on stopped node, fails - with pytest.raises(expected_exception=QueryException): - node.safe_psql('select 1') - - def test_safe_psql__expect_error(self): - with __class__.helper__get_node().init().start() as node: - err = node.safe_psql('select_or_not_select 1', expect_error=True) - assert (type(err) == str) # noqa: E721 - assert ('select_or_not_select' in err) - assert ('ERROR: syntax error at or near "select_or_not_select"' in err) - - # --------- - with pytest.raises( - expected_exception=InvalidOperationException, - match="^" + re.escape("Exception was expected, but query finished successfully: `select 1;`.") + "$" - ): - node.safe_psql("select 1;", expect_error=True) - - # --------- - res = node.safe_psql("select 1;", expect_error=False) - assert (res == b'1\n') - - def test_transactions(self): - with __class__.helper__get_node().init().start() as node: - with node.connect() as con: - con.begin() - con.execute('create table test(val int)') - con.execute('insert into test values (1)') - con.commit() - - con.begin() - con.execute('insert into test values (2)') - res = con.execute('select * from test order by val asc') - assert (res == [(1,), (2,)]) - con.rollback() - - con.begin() - res = con.execute('select * from test') - assert (res == [(1,)]) - con.rollback() - - con.begin() - con.execute('drop table test') - con.commit() - - def test_control_data(self): - with __class__.helper__get_node() as node: - # node is not initialized yet - with pytest.raises(expected_exception=ExecUtilException): - node.get_control_data() - - node.init() - data = node.get_control_data() - - # check returned dict - assert data is not None - assert (any('pg_control' in s for s in data.keys())) - - def test_backup_simple(self): - with __class__.helper__get_node() as master: - # enable streaming for backups - master.init(allow_streaming=True) - - # node must be running - with pytest.raises(expected_exception=BackupException): - master.backup() - - # it's time to start node - master.start() - - # fill node with some data - master.psql('create table test as select generate_series(1, 4) i') - - with master.backup(xlog_method='stream') as backup: - with backup.spawn_primary().start() as slave: - res = slave.execute('select * from test order by i asc') - assert (res == [(1,), (2,), (3,), (4,)]) - - def test_backup_multiple(self): - with __class__.helper__get_node() as node: - node.init(allow_streaming=True).start() - - with node.backup(xlog_method='fetch') as backup1, \ - node.backup(xlog_method='fetch') as backup2: - assert (backup1.base_dir != backup2.base_dir) - - with node.backup(xlog_method='fetch') as backup: - with backup.spawn_primary('node1', destroy=False) as node1, \ - backup.spawn_primary('node2', destroy=False) as node2: - assert (node1.base_dir != node2.base_dir) - - def test_backup_exhaust(self): - with __class__.helper__get_node() as node: - node.init(allow_streaming=True).start() - - with node.backup(xlog_method='fetch') as backup: - # exhaust backup by creating new node - with backup.spawn_primary(): - pass - - # now let's try to create one more node - with pytest.raises(expected_exception=BackupException): - backup.spawn_primary() - - def test_backup_wrong_xlog_method(self): - with __class__.helper__get_node() as node: - node.init(allow_streaming=True).start() - - with pytest.raises( - expected_exception=BackupException, - match="^" + re.escape('Invalid xlog_method "wrong"') + "$" - ): - node.backup(xlog_method='wrong') - - def test_pg_ctl_wait_option(self): - C_MAX_ATTEMPTS = 50 - - node = __class__.helper__get_node() - assert node.status() == testgres.NodeStatus.Uninitialized - node.init() - assert node.status() == testgres.NodeStatus.Stopped - node.start(wait=False) - nAttempt = 0 - while True: - if nAttempt == C_MAX_ATTEMPTS: - # - # [2025-03-11] - # We have an unexpected problem with this test in CI - # Let's get an additional information about this test failure. - # - logging.error("Node was not stopped.") - if not node.os_ops.path_exists(node.pg_log_file): - logging.warning("Node log does not exist.") - else: - logging.info("Let's read node log file [{0}]".format(node.pg_log_file)) - logFileData = node.os_ops.read(node.pg_log_file, binary=False) - logging.info("Node log file content:\n{0}".format(logFileData)) - - raise Exception("Could not stop node.") - - nAttempt += 1 - - if nAttempt > 1: - logging.info("Wait 1 second.") - time.sleep(1) - logging.info("") - - logging.info("Try to stop node. Attempt #{0}.".format(nAttempt)) - - try: - node.stop(wait=False) - break - except ExecUtilException as e: - # it's ok to get this exception here since node - # could be not started yet - logging.info("Node is not stopped. Exception ({0}): {1}".format(type(e).__name__, e)) - continue - - logging.info("OK. Stop command was executed. Let's wait while our node will stop really.") - nAttempt = 0 - while True: - if nAttempt == C_MAX_ATTEMPTS: - raise Exception("Could not stop node.") - - nAttempt += 1 - if nAttempt > 1: - logging.info("Wait 1 second.") - time.sleep(1) - logging.info("") - - logging.info("Attempt #{0}.".format(nAttempt)) - s1 = node.status() - - if s1 == testgres.NodeStatus.Running: - continue - - if s1 == testgres.NodeStatus.Stopped: - break - - raise Exception("Unexpected node status: {0}.".format(s1)) - - logging.info("OK. Node is stopped.") - node.cleanup() - - def test_replicate(self): - with __class__.helper__get_node() as node: - node.init(allow_streaming=True).start() - - with node.replicate().start() as replica: - res = replica.execute('select 1') - assert (res == [(1,)]) - - node.execute('create table test (val int)', commit=True) - - replica.catchup() - - res = node.execute('select * from test') - assert (res == []) - - def test_synchronous_replication(self): - __class__.helper__skip_test_if_pg_version_is_not_ge("9.6") - - with __class__.helper__get_node() as master: - old_version = not pg_version_ge('9.6') - - master.init(allow_streaming=True).start() - - if not old_version: - master.append_conf('synchronous_commit = remote_apply') - - # create standby - with master.replicate() as standby1, master.replicate() as standby2: - standby1.start() - standby2.start() - - # check formatting - assert ( - '1 ("{}", "{}")'.format(standby1.name, standby2.name) == str(First(1, (standby1, standby2))) - ) # yapf: disable - assert ( - 'ANY 1 ("{}", "{}")'.format(standby1.name, standby2.name) == str(Any(1, (standby1, standby2))) - ) # yapf: disable - - # set synchronous_standby_names - master.set_synchronous_standbys(First(2, [standby1, standby2])) - master.restart() - - # the following part of the test is only applicable to newer - # versions of PostgresQL - if not old_version: - master.safe_psql('create table abc(a int)') - - # Create a large transaction that will take some time to apply - # on standby to check that it applies synchronously - # (If set synchronous_commit to 'on' or other lower level then - # standby most likely won't catchup so fast and test will fail) - master.safe_psql( - 'insert into abc select generate_series(1, 1000000)') - res = standby1.safe_psql('select count(*) from abc') - assert (res == b'1000000\n') - - def test_logical_replication(self): - __class__.helper__skip_test_if_pg_version_is_not_ge("10") - - with __class__.helper__get_node() as node1, __class__.helper__get_node() as node2: - node1.init(allow_logical=True) - node1.start() - node2.init().start() - - create_table = 'create table test (a int, b int)' - node1.safe_psql(create_table) - node2.safe_psql(create_table) - - # create publication / create subscription - pub = node1.publish('mypub') - sub = node2.subscribe(pub, 'mysub') - - node1.safe_psql('insert into test values (1, 1), (2, 2)') - - # wait until changes apply on subscriber and check them - sub.catchup() - res = node2.execute('select * from test') - assert (res == [(1, 1), (2, 2)]) - - # disable and put some new data - sub.disable() - node1.safe_psql('insert into test values (3, 3)') - - # enable and ensure that data successfully transferred - sub.enable() - sub.catchup() - res = node2.execute('select * from test') - assert (res == [(1, 1), (2, 2), (3, 3)]) - - # Add new tables. Since we added "all tables" to publication - # (default behaviour of publish() method) we don't need - # to explicitly perform pub.add_tables() - create_table = 'create table test2 (c char)' - node1.safe_psql(create_table) - node2.safe_psql(create_table) - sub.refresh() - - # put new data - node1.safe_psql('insert into test2 values (\'a\'), (\'b\')') - sub.catchup() - res = node2.execute('select * from test2') - assert (res == [('a',), ('b',)]) - - # drop subscription - sub.drop() - pub.drop() - - # create new publication and subscription for specific table - # (omitting copying data as it's already done) - pub = node1.publish('newpub', tables=['test']) - sub = node2.subscribe(pub, 'newsub', copy_data=False) - - node1.safe_psql('insert into test values (4, 4)') - sub.catchup() - res = node2.execute('select * from test') - assert (res == [(1, 1), (2, 2), (3, 3), (4, 4)]) - - # explicitly add table - with pytest.raises(expected_exception=ValueError): - pub.add_tables([]) # fail - pub.add_tables(['test2']) - node1.safe_psql('insert into test2 values (\'c\')') - sub.catchup() - res = node2.execute('select * from test2') - assert (res == [('a',), ('b',)]) - - def test_logical_catchup(self): - """ Runs catchup for 100 times to be sure that it is consistent """ - __class__.helper__skip_test_if_pg_version_is_not_ge("10") - - with __class__.helper__get_node() as node1, __class__.helper__get_node() as node2: - node1.init(allow_logical=True) - node1.start() - node2.init().start() - - create_table = 'create table test (key int primary key, val int); ' - node1.safe_psql(create_table) - node1.safe_psql('alter table test replica identity default') - node2.safe_psql(create_table) - - # create publication / create subscription - sub = node2.subscribe(node1.publish('mypub'), 'mysub') - - for i in range(0, 100): - node1.execute('insert into test values ({0}, {0})'.format(i)) - sub.catchup() - res = node2.execute('select * from test') - assert (res == [(i, i, )]) - node1.execute('delete from test') - - def test_logical_replication_fail(self): - __class__.helper__skip_test_if_pg_version_is_ge("10") - - with __class__.helper__get_node() as node: - with pytest.raises(expected_exception=InitNodeException): - node.init(allow_logical=True) - - def test_replication_slots(self): - with __class__.helper__get_node() as node: - node.init(allow_streaming=True).start() - - with node.replicate(slot='slot1').start() as replica: - replica.execute('select 1') - - # cannot create new slot with the same name - with pytest.raises(expected_exception=TestgresException): - node.replicate(slot='slot1') - - def test_incorrect_catchup(self): - with __class__.helper__get_node() as node: - node.init(allow_streaming=True).start() - - # node has no master, can't catch up - with pytest.raises(expected_exception=TestgresException): - node.catchup() - - def test_promotion(self): - with __class__.helper__get_node() as master: - master.init().start() - master.safe_psql('create table abc(id serial)') - - with master.replicate().start() as replica: - master.stop() - replica.promote() - - # make standby becomes writable master - replica.safe_psql('insert into abc values (1)') - res = replica.safe_psql('select * from abc') - assert (res == b'1\n') - - def test_dump(self): - query_create = 'create table test as select generate_series(1, 2) as val' - query_select = 'select * from test order by val asc' - - with __class__.helper__get_node().init().start() as node1: - - node1.execute(query_create) - for format in ['plain', 'custom', 'directory', 'tar']: - with removing(node1.dump(format=format)) as dump: - with __class__.helper__get_node().init().start() as node3: - if format == 'directory': - assert (node1.os_ops.isdir(dump)) - else: - assert (node1.os_ops.isfile(dump)) - # restore dump - node3.restore(filename=dump) - res = node3.execute(query_select) - assert (res == [(1,), (2,)]) - - def test_users(self): - with __class__.helper__get_node().init().start() as node: - node.psql('create role test_user login') - value = node.safe_psql('select 1', username='test_user') - assert (b'1\n' == value) - - def test_poll_query_until(self): - with __class__.helper__get_node() as node: - node.init().start() - - get_time = 'select extract(epoch from now())' - check_time = 'select extract(epoch from now()) - {} >= 5' - - start_time = node.execute(get_time)[0][0] - node.poll_query_until(query=check_time.format(start_time)) - end_time = node.execute(get_time)[0][0] - - assert (end_time - start_time >= 5) - - # check 0 columns - with pytest.raises(expected_exception=QueryException): - node.poll_query_until( - query='select from pg_catalog.pg_class limit 1') - - # check None, fail - with pytest.raises(expected_exception=QueryException): - node.poll_query_until(query='create table abc (val int)') - - # check None, ok - node.poll_query_until(query='create table def()', - expected=None) # returns nothing - - # check 0 rows equivalent to expected=None - node.poll_query_until( - query='select * from pg_catalog.pg_class where true = false', - expected=None) - - # check arbitrary expected value, fail - with pytest.raises(expected_exception=TimeoutException): - node.poll_query_until(query='select 3', - expected=1, - max_attempts=3, - sleep_time=0.01) - - # check arbitrary expected value, ok - node.poll_query_until(query='select 2', expected=2) - - # check timeout - with pytest.raises(expected_exception=TimeoutException): - node.poll_query_until(query='select 1 > 2', - max_attempts=3, - sleep_time=0.01) - - # check ProgrammingError, fail - with pytest.raises(expected_exception=testgres.ProgrammingError): - node.poll_query_until(query='dummy1') - - # check ProgrammingError, ok - with pytest.raises(expected_exception=TimeoutException): - node.poll_query_until(query='dummy2', - max_attempts=3, - sleep_time=0.01, - suppress={testgres.ProgrammingError}) - - # check 1 arg, ok - node.poll_query_until('select true') - - def test_logging(self): - C_MAX_ATTEMPTS = 50 - # This name is used for testgres logging, too. - C_NODE_NAME = "testgres_tests." + __class__.__name__ + "test_logging-master-" + uuid.uuid4().hex - - logging.info("Node name is [{0}]".format(C_NODE_NAME)) - - with tempfile.NamedTemporaryFile('w', delete=True) as logfile: - formatter = logging.Formatter(fmt="%(node)-5s: %(message)s") - handler = logging.FileHandler(filename=logfile.name) - handler.formatter = formatter - logger = logging.getLogger(C_NODE_NAME) - assert logger is not None - assert len(logger.handlers) == 0 - - try: - # It disables to log on the root level - logger.propagate = False - logger.addHandler(handler) - - with scoped_config(use_python_logging=True): - with __class__.helper__get_node(name=C_NODE_NAME) as master: - logging.info("Master node is initilizing") - master.init() - - logging.info("Master node is starting") - master.start() - - logging.info("Dummy query is executed a few times") - for _ in range(20): - master.execute('select 1') - time.sleep(0.01) - - # let logging worker do the job - time.sleep(0.1) - - logging.info("Master node log file is checking") - nAttempt = 0 - - while True: - assert nAttempt <= C_MAX_ATTEMPTS - if nAttempt == C_MAX_ATTEMPTS: - raise Exception("Test failed!") - - # let logging worker do the job - time.sleep(0.1) - - nAttempt += 1 - - logging.info("Attempt {0}".format(nAttempt)) - - # check that master's port is found - with open(logfile.name, 'r') as log: - lines = log.readlines() - - assert lines is not None - assert type(lines) == list # noqa: E721 - - def LOCAL__test_lines(): - for s in lines: - if any(C_NODE_NAME in s for s in lines): - logging.info("OK. We found the node_name in a line \"{0}\"".format(s)) - return True - return False - - if LOCAL__test_lines(): - break - - logging.info("Master node log file does not have an expected information.") - continue - - # test logger after stop/start/restart - logging.info("Master node is stopping...") - master.stop() - logging.info("Master node is staring again...") - master.start() - logging.info("Master node is restaring...") - master.restart() - assert (master._logger.is_alive()) - finally: - # It is a hack code to logging cleanup - logging._acquireLock() - assert logging.Logger.manager is not None - assert C_NODE_NAME in logging.Logger.manager.loggerDict.keys() - logging.Logger.manager.loggerDict.pop(C_NODE_NAME, None) - assert not (C_NODE_NAME in logging.Logger.manager.loggerDict.keys()) - assert not (handler in logging._handlers.values()) - logging._releaseLock() - # GO HOME! - return - def test_pgbench(self): __class__.helper__skip_test_if_util_not_exist("pgbench") @@ -1031,60 +259,6 @@ def test_unix_sockets(self): assert (res_exec == [(1,)]) assert (res_psql == b'1\n') - def test_auto_name(self): - with __class__.helper__get_node().init(allow_streaming=True).start() as m: - with m.replicate().start() as r: - # check that nodes are running - assert (m.status()) - assert (r.status()) - - # check their names - assert (m.name != r.name) - assert ('testgres' in m.name) - assert ('testgres' in r.name) - - def test_file_tail(self): - s1 = "the quick brown fox jumped over that lazy dog\n" - s2 = "abc\n" - s3 = "def\n" - - with tempfile.NamedTemporaryFile(mode='r+', delete=True) as f: - sz = 0 - while sz < 3 * 8192: - sz += len(s1) - f.write(s1) - f.write(s2) - f.write(s3) - - f.seek(0) - lines = file_tail(f, 3) - assert (lines[0] == s1) - assert (lines[1] == s2) - assert (lines[2] == s3) - - f.seek(0) - lines = file_tail(f, 1) - assert (lines[0] == s3) - - def test_isolation_levels(self): - with __class__.helper__get_node().init().start() as node: - with node.connect() as con: - # string levels - con.begin('Read Uncommitted').commit() - con.begin('Read Committed').commit() - con.begin('Repeatable Read').commit() - con.begin('Serializable').commit() - - # enum levels - con.begin(IsolationLevel.ReadUncommitted).commit() - con.begin(IsolationLevel.ReadCommitted).commit() - con.begin(IsolationLevel.RepeatableRead).commit() - con.begin(IsolationLevel.Serializable).commit() - - # check wrong level - with pytest.raises(expected_exception=QueryException): - con.begin('Garbage').commit() - def test_ports_management(self): assert bound_ports is not None assert type(bound_ports) == set # noqa: E721 @@ -1119,145 +293,6 @@ def test_ports_management(self): assert type(bound_ports) == set # noqa: E721 assert bound_ports == stage0__bound_ports - def test_exceptions(self): - str(StartNodeException('msg', [('file', 'lines')])) - str(ExecUtilException('msg', 'cmd', 1, 'out')) - str(QueryException('msg', 'query')) - - def test_version_management(self): - a = PgVer('10.0') - b = PgVer('10') - c = PgVer('9.6.5') - d = PgVer('15.0') - e = PgVer('15rc1') - f = PgVer('15beta4') - - assert (a == b) - assert (b > c) - assert (a > c) - assert (d > e) - assert (e > f) - assert (d > f) - - version = get_pg_version() - with __class__.helper__get_node() as node: - assert (isinstance(version, six.string_types)) - assert (isinstance(node.version, PgVer)) - assert (node.version == PgVer(version)) - - def test_child_pids(self): - master_processes = [ - ProcessType.AutovacuumLauncher, - ProcessType.BackgroundWriter, - ProcessType.Checkpointer, - ProcessType.StatsCollector, - ProcessType.WalSender, - ProcessType.WalWriter, - ] - - if pg_version_ge('10'): - master_processes.append(ProcessType.LogicalReplicationLauncher) - - if pg_version_ge('14'): - master_processes.remove(ProcessType.StatsCollector) - - repl_processes = [ - ProcessType.Startup, - ProcessType.WalReceiver, - ] - - def LOCAL__test_auxiliary_pids( - node: testgres.PostgresNode, - expectedTypes: list[ProcessType] - ) -> list[ProcessType]: - # returns list of the absence processes - assert node is not None - assert type(node) == testgres.PostgresNode # noqa: E721 - assert expectedTypes is not None - assert type(expectedTypes) == list # noqa: E721 - - pids = node.auxiliary_pids - assert pids is not None # noqa: E721 - assert type(pids) == dict # noqa: E721 - - result = list[ProcessType]() - for ptype in expectedTypes: - if not (ptype in pids): - result.append(ptype) - return result - - def LOCAL__check_auxiliary_pids__multiple_attempts( - node: testgres.PostgresNode, - expectedTypes: list[ProcessType]): - assert node is not None - assert type(node) == testgres.PostgresNode # noqa: E721 - assert expectedTypes is not None - assert type(expectedTypes) == list # noqa: E721 - - nAttempt = 0 - - while nAttempt < 5: - nAttempt += 1 - - logging.info("Test pids of [{0}] node. Attempt #{1}.".format( - node.name, - nAttempt - )) - - if nAttempt > 1: - time.sleep(1) - - absenceList = LOCAL__test_auxiliary_pids(node, expectedTypes) - assert absenceList is not None - assert type(absenceList) == list # noqa: E721 - if len(absenceList) == 0: - logging.info("Bingo!") - return - - logging.info("These processes are not found: {0}.".format(absenceList)) - continue - - raise Exception("Node {0} does not have the following processes: {1}.".format( - node.name, - absenceList - )) - - with __class__.helper__get_node().init().start() as master: - - # master node doesn't have a source walsender! - with pytest.raises(expected_exception=TestgresException): - master.source_walsender - - with master.connect() as con: - assert (con.pid > 0) - - with master.replicate().start() as replica: - - # test __str__ method - str(master.child_processes[0]) - - LOCAL__check_auxiliary_pids__multiple_attempts( - master, - master_processes) - - LOCAL__check_auxiliary_pids__multiple_attempts( - replica, - repl_processes) - - master_pids = master.auxiliary_pids - - # there should be exactly 1 source walsender for replica - assert (len(master_pids[ProcessType.WalSender]) == 1) - pid1 = master_pids[ProcessType.WalSender][0] - pid2 = replica.source_walsender.pid - assert (pid1 == pid2) - - replica.stop() - - # there should be no walsender after we've stopped replica - with pytest.raises(expected_exception=TestgresException): - replica.source_walsender - # TODO: Why does not this test work with remote host? def test_child_process_dies(self): nAttempt = 0 @@ -1290,8 +325,8 @@ def test_child_process_dies(self): @staticmethod def helper__get_node(name=None): - assert __class__.sm_conn_params is not None - return get_remote_node(name=name, conn_params=__class__.sm_conn_params) + assert isinstance(__class__.sm_os_ops, OsOperations) + return testgres.PostgresNode(name, conn_params=None, os_ops=__class__.sm_os_ops) @staticmethod def helper__restore_envvar(name, prev_value): @@ -1305,15 +340,3 @@ def helper__skip_test_if_util_not_exist(name: str): assert type(name) == str # noqa: E721 if not util_exists(name): pytest.skip('might be missing') - - @staticmethod - def helper__skip_test_if_pg_version_is_not_ge(version: str): - assert type(version) == str # noqa: E721 - if not pg_version_ge(version): - pytest.skip('requires {0}+'.format(version)) - - @staticmethod - def helper__skip_test_if_pg_version_is_ge(version: str): - assert type(version) == str # noqa: E721 - if pg_version_ge(version): - pytest.skip('requires <{0}'.format(version)) diff --git a/tests/test_testgres_common.py b/tests/test_testgres_common.py new file mode 100644 index 00000000..49740b61 --- /dev/null +++ b/tests/test_testgres_common.py @@ -0,0 +1,1131 @@ +from .helpers.os_ops_descrs import OsOpsDescr +from .helpers.os_ops_descrs import OsOpsDescrs +from .helpers.os_ops_descrs import OsOperations + +from ..testgres.node import PgVer +from ..testgres.node import PostgresNode +from ..testgres.utils import get_pg_version2 +from ..testgres.utils import file_tail +from ..testgres.utils import get_bin_path2 +from ..testgres import ProcessType +from ..testgres import NodeStatus +from ..testgres import IsolationLevel +from ..testgres import TestgresException +from ..testgres import InitNodeException +from ..testgres import StartNodeException +from ..testgres import QueryException +from ..testgres import ExecUtilException +from ..testgres import TimeoutException +from ..testgres import InvalidOperationException +from ..testgres import BackupException +from ..testgres import ProgrammingError +from ..testgres import scoped_config +from ..testgres import First, Any + +from contextlib import contextmanager + +import pytest +import six +import logging +import time +import tempfile +import uuid +import os +import re + + +@contextmanager +def removing(os_ops: OsOperations, f): + assert isinstance(os_ops, OsOperations) + + try: + yield f + finally: + if os_ops.isfile(f): + os_ops.remove_file(f) + + elif os_ops.isdir(f): + os_ops.rmdirs(f, ignore_errors=True) + + +class TestTestgresCommon: + sm_os_ops_descrs: list[OsOpsDescr] = [ + OsOpsDescrs.sm_local_os_ops_descr, + OsOpsDescrs.sm_remote_os_ops_descr + ] + + @pytest.fixture( + params=[descr.os_ops for descr in sm_os_ops_descrs], + ids=[descr.sign for descr in sm_os_ops_descrs] + ) + def os_ops(self, request: pytest.FixtureRequest) -> OsOperations: + assert isinstance(request, pytest.FixtureRequest) + assert isinstance(request.param, OsOperations) + return request.param + + def test_version_management(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + + a = PgVer('10.0') + b = PgVer('10') + c = PgVer('9.6.5') + d = PgVer('15.0') + e = PgVer('15rc1') + f = PgVer('15beta4') + h = PgVer('15.3biha') + i = PgVer('15.3') + g = PgVer('15.3.1bihabeta1') + k = PgVer('15.3.1') + + assert (a == b) + assert (b > c) + assert (a > c) + assert (d > e) + assert (e > f) + assert (d > f) + assert (h > f) + assert (h == i) + assert (g == k) + assert (g > h) + + version = get_pg_version2(os_ops) + + with __class__.helper__get_node(os_ops) as node: + assert (isinstance(version, six.string_types)) + assert (isinstance(node.version, PgVer)) + assert (node.version == PgVer(version)) + + def test_double_init(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + + with __class__.helper__get_node(os_ops).init() as node: + # can't initialize node more than once + with pytest.raises(expected_exception=InitNodeException): + node.init() + + def test_init_after_cleanup(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + + with __class__.helper__get_node(os_ops) as node: + node.init().start().execute('select 1') + node.cleanup() + node.init().start().execute('select 1') + + def test_init_unique_system_id(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + + # this function exists in PostgreSQL 9.6+ + current_version = get_pg_version2(os_ops) + + __class__.helper__skip_test_if_util_not_exist(os_ops, "pg_resetwal") + __class__.helper__skip_test_if_pg_version_is_not_ge(current_version, '9.6') + + query = 'select system_identifier from pg_control_system()' + + with scoped_config(cache_initdb=False): + with __class__.helper__get_node(os_ops).init().start() as node0: + id0 = node0.execute(query)[0] + + with scoped_config(cache_initdb=True, + cached_initdb_unique=True) as config: + assert (config.cache_initdb) + assert (config.cached_initdb_unique) + + # spawn two nodes; ids must be different + with __class__.helper__get_node(os_ops).init().start() as node1, \ + __class__.helper__get_node(os_ops).init().start() as node2: + id1 = node1.execute(query)[0] + id2 = node2.execute(query)[0] + + # ids must increase + assert (id1 > id0) + assert (id2 > id1) + + def test_node_exit(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + + with pytest.raises(expected_exception=QueryException): + with __class__.helper__get_node(os_ops).init() as node: + base_dir = node.base_dir + node.safe_psql('select 1') + + # we should save the DB for "debugging" + assert (os_ops.path_exists(base_dir)) + os_ops.rmdirs(base_dir, ignore_errors=True) + + with __class__.helper__get_node(os_ops).init() as node: + base_dir = node.base_dir + + # should have been removed by default + assert not (os_ops.path_exists(base_dir)) + + def test_double_start(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + + with __class__.helper__get_node(os_ops).init().start() as node: + # can't start node more than once + node.start() + assert (node.is_started) + + def test_uninitialized_start(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + + with __class__.helper__get_node(os_ops) as node: + # node is not initialized yet + with pytest.raises(expected_exception=StartNodeException): + node.start() + + def test_restart(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + + with __class__.helper__get_node(os_ops) as node: + node.init().start() + + # restart, ok + res = node.execute('select 1') + assert (res == [(1,)]) + node.restart() + res = node.execute('select 2') + assert (res == [(2,)]) + + # restart, fail + with pytest.raises(expected_exception=StartNodeException): + node.append_conf('pg_hba.conf', 'DUMMY') + node.restart() + + def test_reload(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + + with __class__.helper__get_node(os_ops) as node: + node.init().start() + + # change client_min_messages and save old value + cmm_old = node.execute('show client_min_messages') + node.append_conf(client_min_messages='DEBUG1') + + # reload config + node.reload() + + # check new value + cmm_new = node.execute('show client_min_messages') + assert ('debug1' == cmm_new[0][0].lower()) + assert (cmm_old != cmm_new) + + def test_pg_ctl(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + + with __class__.helper__get_node(os_ops) as node: + node.init().start() + + status = node.pg_ctl(['status']) + assert ('PID' in status) + + def test_status(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + + assert (NodeStatus.Running) + assert not (NodeStatus.Stopped) + assert not (NodeStatus.Uninitialized) + + # check statuses after each operation + with __class__.helper__get_node(os_ops) as node: + assert (node.pid == 0) + assert (node.status() == NodeStatus.Uninitialized) + + node.init() + + assert (node.pid == 0) + assert (node.status() == NodeStatus.Stopped) + + node.start() + + assert (node.pid != 0) + assert (node.status() == NodeStatus.Running) + + node.stop() + + assert (node.pid == 0) + assert (node.status() == NodeStatus.Stopped) + + node.cleanup() + + assert (node.pid == 0) + assert (node.status() == NodeStatus.Uninitialized) + + def test_child_pids(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + + master_processes = [ + ProcessType.AutovacuumLauncher, + ProcessType.BackgroundWriter, + ProcessType.Checkpointer, + ProcessType.StatsCollector, + ProcessType.WalSender, + ProcessType.WalWriter, + ] + + postgresVersion = get_pg_version2(os_ops) + + if __class__.helper__pg_version_ge(postgresVersion, '10'): + master_processes.append(ProcessType.LogicalReplicationLauncher) + + if __class__.helper__pg_version_ge(postgresVersion, '14'): + master_processes.remove(ProcessType.StatsCollector) + + repl_processes = [ + ProcessType.Startup, + ProcessType.WalReceiver, + ] + + def LOCAL__test_auxiliary_pids( + node: PostgresNode, + expectedTypes: list[ProcessType] + ) -> list[ProcessType]: + # returns list of the absence processes + assert node is not None + assert type(node) == PostgresNode # noqa: E721 + assert expectedTypes is not None + assert type(expectedTypes) == list # noqa: E721 + + pids = node.auxiliary_pids + assert pids is not None # noqa: E721 + assert type(pids) == dict # noqa: E721 + + result = list[ProcessType]() + for ptype in expectedTypes: + if not (ptype in pids): + result.append(ptype) + return result + + def LOCAL__check_auxiliary_pids__multiple_attempts( + node: PostgresNode, + expectedTypes: list[ProcessType]): + assert node is not None + assert type(node) == PostgresNode # noqa: E721 + assert expectedTypes is not None + assert type(expectedTypes) == list # noqa: E721 + + nAttempt = 0 + + while nAttempt < 5: + nAttempt += 1 + + logging.info("Test pids of [{0}] node. Attempt #{1}.".format( + node.name, + nAttempt + )) + + if nAttempt > 1: + time.sleep(1) + + absenceList = LOCAL__test_auxiliary_pids(node, expectedTypes) + assert absenceList is not None + assert type(absenceList) == list # noqa: E721 + if len(absenceList) == 0: + logging.info("Bingo!") + return + + logging.info("These processes are not found: {0}.".format(absenceList)) + continue + + raise Exception("Node {0} does not have the following processes: {1}.".format( + node.name, + absenceList + )) + + with __class__.helper__get_node(os_ops).init().start() as master: + + # master node doesn't have a source walsender! + with pytest.raises(expected_exception=TestgresException): + master.source_walsender + + with master.connect() as con: + assert (con.pid > 0) + + with master.replicate().start() as replica: + + # test __str__ method + str(master.child_processes[0]) + + LOCAL__check_auxiliary_pids__multiple_attempts( + master, + master_processes) + + LOCAL__check_auxiliary_pids__multiple_attempts( + replica, + repl_processes) + + master_pids = master.auxiliary_pids + + # there should be exactly 1 source walsender for replica + assert (len(master_pids[ProcessType.WalSender]) == 1) + pid1 = master_pids[ProcessType.WalSender][0] + pid2 = replica.source_walsender.pid + assert (pid1 == pid2) + + replica.stop() + + # there should be no walsender after we've stopped replica + with pytest.raises(expected_exception=TestgresException): + replica.source_walsender + + def test_exceptions(self): + str(StartNodeException('msg', [('file', 'lines')])) + str(ExecUtilException('msg', 'cmd', 1, 'out')) + str(QueryException('msg', 'query')) + + def test_auto_name(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + + with __class__.helper__get_node(os_ops).init(allow_streaming=True).start() as m: + with m.replicate().start() as r: + # check that nodes are running + assert (m.status()) + assert (r.status()) + + # check their names + assert (m.name != r.name) + assert ('testgres' in m.name) + assert ('testgres' in r.name) + + def test_file_tail(self): + s1 = "the quick brown fox jumped over that lazy dog\n" + s2 = "abc\n" + s3 = "def\n" + + with tempfile.NamedTemporaryFile(mode='r+', delete=True) as f: + sz = 0 + while sz < 3 * 8192: + sz += len(s1) + f.write(s1) + f.write(s2) + f.write(s3) + + f.seek(0) + lines = file_tail(f, 3) + assert (lines[0] == s1) + assert (lines[1] == s2) + assert (lines[2] == s3) + + f.seek(0) + lines = file_tail(f, 1) + assert (lines[0] == s3) + + def test_isolation_levels(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + with __class__.helper__get_node(os_ops).init().start() as node: + with node.connect() as con: + # string levels + con.begin('Read Uncommitted').commit() + con.begin('Read Committed').commit() + con.begin('Repeatable Read').commit() + con.begin('Serializable').commit() + + # enum levels + con.begin(IsolationLevel.ReadUncommitted).commit() + con.begin(IsolationLevel.ReadCommitted).commit() + con.begin(IsolationLevel.RepeatableRead).commit() + con.begin(IsolationLevel.Serializable).commit() + + # check wrong level + with pytest.raises(expected_exception=QueryException): + con.begin('Garbage').commit() + + def test_users(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + with __class__.helper__get_node(os_ops).init().start() as node: + node.psql('create role test_user login') + value = node.safe_psql('select 1', username='test_user') + value = __class__.helper__rm_carriage_returns(value) + assert (value == b'1\n') + + def test_poll_query_until(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + with __class__.helper__get_node(os_ops) as node: + node.init().start() + + get_time = 'select extract(epoch from now())' + check_time = 'select extract(epoch from now()) - {} >= 5' + + start_time = node.execute(get_time)[0][0] + node.poll_query_until(query=check_time.format(start_time)) + end_time = node.execute(get_time)[0][0] + + assert (end_time - start_time >= 5) + + # check 0 columns + with pytest.raises(expected_exception=QueryException): + node.poll_query_until( + query='select from pg_catalog.pg_class limit 1') + + # check None, fail + with pytest.raises(expected_exception=QueryException): + node.poll_query_until(query='create table abc (val int)') + + # check None, ok + node.poll_query_until(query='create table def()', + expected=None) # returns nothing + + # check 0 rows equivalent to expected=None + node.poll_query_until( + query='select * from pg_catalog.pg_class where true = false', + expected=None) + + # check arbitrary expected value, fail + with pytest.raises(expected_exception=TimeoutException): + node.poll_query_until(query='select 3', + expected=1, + max_attempts=3, + sleep_time=0.01) + + # check arbitrary expected value, ok + node.poll_query_until(query='select 2', expected=2) + + # check timeout + with pytest.raises(expected_exception=TimeoutException): + node.poll_query_until(query='select 1 > 2', + max_attempts=3, + sleep_time=0.01) + + # check ProgrammingError, fail + with pytest.raises(expected_exception=ProgrammingError): + node.poll_query_until(query='dummy1') + + # check ProgrammingError, ok + with pytest.raises(expected_exception=(TimeoutException)): + node.poll_query_until(query='dummy2', + max_attempts=3, + sleep_time=0.01, + suppress={ProgrammingError}) + + # check 1 arg, ok + node.poll_query_until('select true') + + def test_logging(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + C_MAX_ATTEMPTS = 50 + # This name is used for testgres logging, too. + C_NODE_NAME = "testgres_tests." + __class__.__name__ + "test_logging-master-" + uuid.uuid4().hex + + logging.info("Node name is [{0}]".format(C_NODE_NAME)) + + with tempfile.NamedTemporaryFile('w', delete=True) as logfile: + formatter = logging.Formatter(fmt="%(node)-5s: %(message)s") + handler = logging.FileHandler(filename=logfile.name) + handler.formatter = formatter + logger = logging.getLogger(C_NODE_NAME) + assert logger is not None + assert len(logger.handlers) == 0 + + try: + # It disables to log on the root level + logger.propagate = False + logger.addHandler(handler) + + with scoped_config(use_python_logging=True): + with __class__.helper__get_node(os_ops, name=C_NODE_NAME) as master: + logging.info("Master node is initilizing") + master.init() + + logging.info("Master node is starting") + master.start() + + logging.info("Dummy query is executed a few times") + for _ in range(20): + master.execute('select 1') + time.sleep(0.01) + + # let logging worker do the job + time.sleep(0.1) + + logging.info("Master node log file is checking") + nAttempt = 0 + + while True: + assert nAttempt <= C_MAX_ATTEMPTS + if nAttempt == C_MAX_ATTEMPTS: + raise Exception("Test failed!") + + # let logging worker do the job + time.sleep(0.1) + + nAttempt += 1 + + logging.info("Attempt {0}".format(nAttempt)) + + # check that master's port is found + with open(logfile.name, 'r') as log: + lines = log.readlines() + + assert lines is not None + assert type(lines) == list # noqa: E721 + + def LOCAL__test_lines(): + for s in lines: + if any(C_NODE_NAME in s for s in lines): + logging.info("OK. We found the node_name in a line \"{0}\"".format(s)) + return True + return False + + if LOCAL__test_lines(): + break + + logging.info("Master node log file does not have an expected information.") + continue + + # test logger after stop/start/restart + logging.info("Master node is stopping...") + master.stop() + logging.info("Master node is staring again...") + master.start() + logging.info("Master node is restaring...") + master.restart() + assert (master._logger.is_alive()) + finally: + # It is a hack code to logging cleanup + logging._acquireLock() + assert logging.Logger.manager is not None + assert C_NODE_NAME in logging.Logger.manager.loggerDict.keys() + logging.Logger.manager.loggerDict.pop(C_NODE_NAME, None) + assert not (C_NODE_NAME in logging.Logger.manager.loggerDict.keys()) + assert not (handler in logging._handlers.values()) + logging._releaseLock() + # GO HOME! + return + + def test_psql(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + with __class__.helper__get_node(os_ops).init().start() as node: + + # check returned values (1 arg) + res = node.psql('select 1') + assert (__class__.helper__rm_carriage_returns(res) == (0, b'1\n', b'')) + + # check returned values (2 args) + res = node.psql('postgres', 'select 2') + assert (__class__.helper__rm_carriage_returns(res) == (0, b'2\n', b'')) + + # check returned values (named) + res = node.psql(query='select 3', dbname='postgres') + assert (__class__.helper__rm_carriage_returns(res) == (0, b'3\n', b'')) + + # check returned values (1 arg) + res = node.safe_psql('select 4') + assert (__class__.helper__rm_carriage_returns(res) == b'4\n') + + # check returned values (2 args) + res = node.safe_psql('postgres', 'select 5') + assert (__class__.helper__rm_carriage_returns(res) == b'5\n') + + # check returned values (named) + res = node.safe_psql(query='select 6', dbname='postgres') + assert (__class__.helper__rm_carriage_returns(res) == b'6\n') + + # check feeding input + node.safe_psql('create table horns (w int)') + node.safe_psql('copy horns from stdin (format csv)', + input=b"1\n2\n3\n\\.\n") + _sum = node.safe_psql('select sum(w) from horns') + assert (__class__.helper__rm_carriage_returns(_sum) == b'6\n') + + # check psql's default args, fails + with pytest.raises(expected_exception=QueryException): + node.psql() + + node.stop() + + # check psql on stopped node, fails + with pytest.raises(expected_exception=QueryException): + node.safe_psql('select 1') + + def test_safe_psql__expect_error(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + with __class__.helper__get_node(os_ops).init().start() as node: + err = node.safe_psql('select_or_not_select 1', expect_error=True) + assert (type(err) == str) # noqa: E721 + assert ('select_or_not_select' in err) + assert ('ERROR: syntax error at or near "select_or_not_select"' in err) + + # --------- + with pytest.raises( + expected_exception=InvalidOperationException, + match="^" + re.escape("Exception was expected, but query finished successfully: `select 1;`.") + "$" + ): + node.safe_psql("select 1;", expect_error=True) + + # --------- + res = node.safe_psql("select 1;", expect_error=False) + assert (__class__.helper__rm_carriage_returns(res) == b'1\n') + + def test_transactions(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + with __class__.helper__get_node(os_ops).init().start() as node: + + with node.connect() as con: + con.begin() + con.execute('create table test(val int)') + con.execute('insert into test values (1)') + con.commit() + + con.begin() + con.execute('insert into test values (2)') + res = con.execute('select * from test order by val asc') + assert (res == [(1, ), (2, )]) + con.rollback() + + con.begin() + res = con.execute('select * from test') + assert (res == [(1, )]) + con.rollback() + + con.begin() + con.execute('drop table test') + con.commit() + + def test_control_data(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + with __class__.helper__get_node(os_ops) as node: + + # node is not initialized yet + with pytest.raises(expected_exception=ExecUtilException): + node.get_control_data() + + node.init() + data = node.get_control_data() + + # check returned dict + assert data is not None + assert (any('pg_control' in s for s in data.keys())) + + def test_backup_simple(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + with __class__.helper__get_node(os_ops) as master: + + # enable streaming for backups + master.init(allow_streaming=True) + + # node must be running + with pytest.raises(expected_exception=BackupException): + master.backup() + + # it's time to start node + master.start() + + # fill node with some data + master.psql('create table test as select generate_series(1, 4) i') + + with master.backup(xlog_method='stream') as backup: + with backup.spawn_primary().start() as slave: + res = slave.execute('select * from test order by i asc') + assert (res == [(1, ), (2, ), (3, ), (4, )]) + + def test_backup_multiple(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + with __class__.helper__get_node(os_ops) as node: + node.init(allow_streaming=True).start() + + with node.backup(xlog_method='fetch') as backup1, \ + node.backup(xlog_method='fetch') as backup2: + assert (backup1.base_dir != backup2.base_dir) + + with node.backup(xlog_method='fetch') as backup: + with backup.spawn_primary('node1', destroy=False) as node1, \ + backup.spawn_primary('node2', destroy=False) as node2: + assert (node1.base_dir != node2.base_dir) + + def test_backup_exhaust(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + with __class__.helper__get_node(os_ops) as node: + node.init(allow_streaming=True).start() + + with node.backup(xlog_method='fetch') as backup: + # exhaust backup by creating new node + with backup.spawn_primary(): + pass + + # now let's try to create one more node + with pytest.raises(expected_exception=BackupException): + backup.spawn_primary() + + def test_backup_wrong_xlog_method(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + with __class__.helper__get_node(os_ops) as node: + node.init(allow_streaming=True).start() + + with pytest.raises( + expected_exception=BackupException, + match="^" + re.escape('Invalid xlog_method "wrong"') + "$" + ): + node.backup(xlog_method='wrong') + + def test_pg_ctl_wait_option(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + C_MAX_ATTEMPTS = 50 + + node = __class__.helper__get_node(os_ops) + assert node.status() == NodeStatus.Uninitialized + node.init() + assert node.status() == NodeStatus.Stopped + node.start(wait=False) + nAttempt = 0 + while True: + if nAttempt == C_MAX_ATTEMPTS: + # + # [2025-03-11] + # We have an unexpected problem with this test in CI + # Let's get an additional information about this test failure. + # + logging.error("Node was not stopped.") + if not node.os_ops.path_exists(node.pg_log_file): + logging.warning("Node log does not exist.") + else: + logging.info("Let's read node log file [{0}]".format(node.pg_log_file)) + logFileData = node.os_ops.read(node.pg_log_file, binary=False) + logging.info("Node log file content:\n{0}".format(logFileData)) + + raise Exception("Could not stop node.") + + nAttempt += 1 + + if nAttempt > 1: + logging.info("Wait 1 second.") + time.sleep(1) + logging.info("") + + logging.info("Try to stop node. Attempt #{0}.".format(nAttempt)) + + try: + node.stop(wait=False) + break + except ExecUtilException as e: + # it's ok to get this exception here since node + # could be not started yet + logging.info("Node is not stopped. Exception ({0}): {1}".format(type(e).__name__, e)) + continue + + logging.info("OK. Stop command was executed. Let's wait while our node will stop really.") + nAttempt = 0 + while True: + if nAttempt == C_MAX_ATTEMPTS: + raise Exception("Could not stop node.") + + nAttempt += 1 + if nAttempt > 1: + logging.info("Wait 1 second.") + time.sleep(1) + logging.info("") + + logging.info("Attempt #{0}.".format(nAttempt)) + s1 = node.status() + + if s1 == NodeStatus.Running: + continue + + if s1 == NodeStatus.Stopped: + break + + raise Exception("Unexpected node status: {0}.".format(s1)) + + logging.info("OK. Node is stopped.") + node.cleanup() + + def test_replicate(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + with __class__.helper__get_node(os_ops) as node: + node.init(allow_streaming=True).start() + + with node.replicate().start() as replica: + res = replica.execute('select 1') + assert (res == [(1, )]) + + node.execute('create table test (val int)', commit=True) + + replica.catchup() + + res = node.execute('select * from test') + assert (res == []) + + def test_synchronous_replication(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + + current_version = get_pg_version2(os_ops) + + __class__.helper__skip_test_if_pg_version_is_not_ge(current_version, "9.6") + + with __class__.helper__get_node(os_ops) as master: + old_version = not __class__.helper__pg_version_ge(current_version, '9.6') + + master.init(allow_streaming=True).start() + + if not old_version: + master.append_conf('synchronous_commit = remote_apply') + + # create standby + with master.replicate() as standby1, master.replicate() as standby2: + standby1.start() + standby2.start() + + # check formatting + assert ( + '1 ("{}", "{}")'.format(standby1.name, standby2.name) == str(First(1, (standby1, standby2))) + ) # yapf: disable + assert ( + 'ANY 1 ("{}", "{}")'.format(standby1.name, standby2.name) == str(Any(1, (standby1, standby2))) + ) # yapf: disable + + # set synchronous_standby_names + master.set_synchronous_standbys(First(2, [standby1, standby2])) + master.restart() + + # the following part of the test is only applicable to newer + # versions of PostgresQL + if not old_version: + master.safe_psql('create table abc(a int)') + + # Create a large transaction that will take some time to apply + # on standby to check that it applies synchronously + # (If set synchronous_commit to 'on' or other lower level then + # standby most likely won't catchup so fast and test will fail) + master.safe_psql( + 'insert into abc select generate_series(1, 1000000)') + res = standby1.safe_psql('select count(*) from abc') + assert (__class__.helper__rm_carriage_returns(res) == b'1000000\n') + + def test_logical_replication(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + + current_version = get_pg_version2(os_ops) + + __class__.helper__skip_test_if_pg_version_is_not_ge(current_version, "10") + + with __class__.helper__get_node(os_ops) as node1, __class__.helper__get_node(os_ops) as node2: + node1.init(allow_logical=True) + node1.start() + node2.init().start() + + create_table = 'create table test (a int, b int)' + node1.safe_psql(create_table) + node2.safe_psql(create_table) + + # create publication / create subscription + pub = node1.publish('mypub') + sub = node2.subscribe(pub, 'mysub') + + node1.safe_psql('insert into test values (1, 1), (2, 2)') + + # wait until changes apply on subscriber and check them + sub.catchup() + res = node2.execute('select * from test') + assert (res == [(1, 1), (2, 2)]) + + # disable and put some new data + sub.disable() + node1.safe_psql('insert into test values (3, 3)') + + # enable and ensure that data successfully transferred + sub.enable() + sub.catchup() + res = node2.execute('select * from test') + assert (res == [(1, 1), (2, 2), (3, 3)]) + + # Add new tables. Since we added "all tables" to publication + # (default behaviour of publish() method) we don't need + # to explicitly perform pub.add_tables() + create_table = 'create table test2 (c char)' + node1.safe_psql(create_table) + node2.safe_psql(create_table) + sub.refresh() + + # put new data + node1.safe_psql('insert into test2 values (\'a\'), (\'b\')') + sub.catchup() + res = node2.execute('select * from test2') + assert (res == [('a', ), ('b', )]) + + # drop subscription + sub.drop() + pub.drop() + + # create new publication and subscription for specific table + # (omitting copying data as it's already done) + pub = node1.publish('newpub', tables=['test']) + sub = node2.subscribe(pub, 'newsub', copy_data=False) + + node1.safe_psql('insert into test values (4, 4)') + sub.catchup() + res = node2.execute('select * from test') + assert (res == [(1, 1), (2, 2), (3, 3), (4, 4)]) + + # explicitly add table + with pytest.raises(expected_exception=ValueError): + pub.add_tables([]) # fail + pub.add_tables(['test2']) + node1.safe_psql('insert into test2 values (\'c\')') + sub.catchup() + res = node2.execute('select * from test2') + assert (res == [('a', ), ('b', )]) + + def test_logical_catchup(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + """ Runs catchup for 100 times to be sure that it is consistent """ + + current_version = get_pg_version2(os_ops) + + __class__.helper__skip_test_if_pg_version_is_not_ge(current_version, "10") + + with __class__.helper__get_node(os_ops) as node1, __class__.helper__get_node(os_ops) as node2: + node1.init(allow_logical=True) + node1.start() + node2.init().start() + + create_table = 'create table test (key int primary key, val int); ' + node1.safe_psql(create_table) + node1.safe_psql('alter table test replica identity default') + node2.safe_psql(create_table) + + # create publication / create subscription + sub = node2.subscribe(node1.publish('mypub'), 'mysub') + + for i in range(0, 100): + node1.execute('insert into test values ({0}, {0})'.format(i)) + sub.catchup() + res = node2.execute('select * from test') + assert (res == [(i, i, )]) + node1.execute('delete from test') + + def test_logical_replication_fail(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + + current_version = get_pg_version2(os_ops) + + __class__.helper__skip_test_if_pg_version_is_ge(current_version, "10") + + with __class__.helper__get_node(os_ops) as node: + with pytest.raises(expected_exception=InitNodeException): + node.init(allow_logical=True) + + def test_replication_slots(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + with __class__.helper__get_node(os_ops) as node: + node.init(allow_streaming=True).start() + + with node.replicate(slot='slot1').start() as replica: + replica.execute('select 1') + + # cannot create new slot with the same name + with pytest.raises(expected_exception=TestgresException): + node.replicate(slot='slot1') + + def test_incorrect_catchup(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + with __class__.helper__get_node(os_ops) as node: + node.init(allow_streaming=True).start() + + # node has no master, can't catch up + with pytest.raises(expected_exception=TestgresException): + node.catchup() + + def test_promotion(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + with __class__.helper__get_node(os_ops) as master: + master.init().start() + master.safe_psql('create table abc(id serial)') + + with master.replicate().start() as replica: + master.stop() + replica.promote() + + # make standby becomes writable master + replica.safe_psql('insert into abc values (1)') + res = replica.safe_psql('select * from abc') + assert (__class__.helper__rm_carriage_returns(res) == b'1\n') + + def test_dump(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + query_create = 'create table test as select generate_series(1, 2) as val' + query_select = 'select * from test order by val asc' + + with __class__.helper__get_node(os_ops).init().start() as node1: + + node1.execute(query_create) + for format in ['plain', 'custom', 'directory', 'tar']: + with removing(os_ops, node1.dump(format=format)) as dump: + with __class__.helper__get_node(os_ops).init().start() as node3: + if format == 'directory': + assert (os.path.isdir(dump)) + else: + assert (os.path.isfile(dump)) + # restore dump + node3.restore(filename=dump) + res = node3.execute(query_select) + assert (res == [(1, ), (2, )]) + + @staticmethod + def helper__get_node(os_ops: OsOperations, name=None): + assert isinstance(os_ops, OsOperations) + return PostgresNode(name, conn_params=None, os_ops=os_ops) + + @staticmethod + def helper__skip_test_if_pg_version_is_not_ge(ver1: str, ver2: str): + assert type(ver1) == str # noqa: E721 + assert type(ver2) == str # noqa: E721 + if not __class__.helper__pg_version_ge(ver1, ver2): + pytest.skip('requires {0}+'.format(ver2)) + + @staticmethod + def helper__skip_test_if_pg_version_is_ge(ver1: str, ver2: str): + assert type(ver1) == str # noqa: E721 + assert type(ver2) == str # noqa: E721 + if __class__.helper__pg_version_ge(ver1, ver2): + pytest.skip('requires <{0}'.format(ver2)) + + @staticmethod + def helper__pg_version_ge(ver1: str, ver2: str) -> bool: + assert type(ver1) == str # noqa: E721 + assert type(ver2) == str # noqa: E721 + v1 = PgVer(ver1) + v2 = PgVer(ver2) + return v1 >= v2 + + @staticmethod + def helper__rm_carriage_returns(out): + """ + In Windows we have additional '\r' symbols in output. + Let's get rid of them. + """ + if isinstance(out, (int, float, complex)): + return out + + if isinstance(out, tuple): + return tuple(__class__.helper__rm_carriage_returns(item) for item in out) + + if isinstance(out, bytes): + return out.replace(b'\r', b'') + + assert type(out) == str # noqa: E721 + return out.replace('\r', '') + + @staticmethod + def helper__skip_test_if_util_not_exist(os_ops: OsOperations, name: str): + assert isinstance(os_ops, OsOperations) + assert type(name) == str # noqa: E721 + if not __class__.helper__util_exists(os_ops, name): + pytest.skip('might be missing') + + @staticmethod + def helper__util_exists(os_ops: OsOperations, util): + assert isinstance(os_ops, OsOperations) + + def good_properties(f): + return (os_ops.path_exists(f) and # noqa: W504 + os_ops.isfile(f) and # noqa: W504 + os_ops.is_executable(f)) # yapf: disable + + # try to resolve it + if good_properties(get_bin_path2(os_ops, util)): + return True + + # check if util is in PATH + for path in os_ops.environ("PATH").split(os.pathsep): + if good_properties(os.path.join(path, util)): + return True From 0b2c629318ea98333b6824fcade369efa349df34 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Mon, 17 Mar 2025 17:07:41 +0300 Subject: [PATCH 425/475] os_ops_descrs.py is added --- tests/helpers/os_ops_descrs.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 tests/helpers/os_ops_descrs.py diff --git a/tests/helpers/os_ops_descrs.py b/tests/helpers/os_ops_descrs.py new file mode 100644 index 00000000..02297adb --- /dev/null +++ b/tests/helpers/os_ops_descrs.py @@ -0,0 +1,32 @@ +from ...testgres.operations.os_ops import OsOperations +from ...testgres.operations.os_ops import ConnectionParams +from ...testgres.operations.local_ops import LocalOperations +from ...testgres.operations.remote_ops import RemoteOperations + +import os + + +class OsOpsDescr: + os_ops: OsOperations + sign: str + + def __init__(self, os_ops: OsOperations, sign: str): + assert isinstance(os_ops, OsOperations) + assert type(sign) == str # noqa: E721 + self.os_ops = os_ops + self.sign = sign + + +class OsOpsDescrs: + sm_remote_conn_params = ConnectionParams( + host=os.getenv('RDBMS_TESTPOOL1_HOST') or '127.0.0.1', + username=os.getenv('USER'), + ssh_key=os.getenv('RDBMS_TESTPOOL_SSHKEY')) + + sm_remote_os_ops = RemoteOperations(sm_remote_conn_params) + + sm_remote_os_ops_descr = OsOpsDescr(sm_remote_os_ops, "remote_ops") + + sm_local_os_ops = LocalOperations() + + sm_local_os_ops_descr = OsOpsDescr(sm_local_os_ops, "local_ops") From b597bf893633dd5e6701c925f8fd70b1f7fda128 Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Tue, 18 Mar 2025 19:25:11 +0300 Subject: [PATCH 426/475] Warnings with pytest are fixed (#223) 1) [pytest.ini] testpaths has another format. It is a spaces separated list. pytest warning: PytestConfigWarning: No files were found in testpaths; consider removing or adjusting your testpaths configuration. Searching recursively from the current directory instead. 2) pytest tries to find the test function in TestgresException class. Let's rename it to avoid this problem. pytest warning: PytestCollectionWarning: cannot collect test class 'TestgresException' because it has a __init__ constructor (from: tests/test_simple.py) class TestgresException(Exception): Of course, we can add __test__=False in TestgresException but it is not a good solution. --- pytest.ini | 2 +- tests/test_testgres_common.py | 14 +++++++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/pytest.ini b/pytest.ini index c94eabc2..9f5fa375 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,5 +1,5 @@ [pytest] -testpaths = ["./tests", "./testgres/plugins/pg_probackup2/pg_probackup2/tests"] +testpaths = tests testgres/plugins/pg_probackup2/pg_probackup2/tests addopts = --strict-markers markers = #log_file = logs/pytest.log diff --git a/tests/test_testgres_common.py b/tests/test_testgres_common.py index 49740b61..f42964c8 100644 --- a/tests/test_testgres_common.py +++ b/tests/test_testgres_common.py @@ -10,7 +10,11 @@ from ..testgres import ProcessType from ..testgres import NodeStatus from ..testgres import IsolationLevel -from ..testgres import TestgresException + +# New name prevents to collect test-functions in TestgresException and fixes +# the problem with pytest warning. +from ..testgres import TestgresException as testgres_TestgresException + from ..testgres import InitNodeException from ..testgres import StartNodeException from ..testgres import QueryException @@ -336,7 +340,7 @@ def LOCAL__check_auxiliary_pids__multiple_attempts( with __class__.helper__get_node(os_ops).init().start() as master: # master node doesn't have a source walsender! - with pytest.raises(expected_exception=TestgresException): + with pytest.raises(expected_exception=testgres_TestgresException): master.source_walsender with master.connect() as con: @@ -366,7 +370,7 @@ def LOCAL__check_auxiliary_pids__multiple_attempts( replica.stop() # there should be no walsender after we've stopped replica - with pytest.raises(expected_exception=TestgresException): + with pytest.raises(expected_exception=testgres_TestgresException): replica.source_walsender def test_exceptions(self): @@ -1013,7 +1017,7 @@ def test_replication_slots(self, os_ops: OsOperations): replica.execute('select 1') # cannot create new slot with the same name - with pytest.raises(expected_exception=TestgresException): + with pytest.raises(expected_exception=testgres_TestgresException): node.replicate(slot='slot1') def test_incorrect_catchup(self, os_ops: OsOperations): @@ -1022,7 +1026,7 @@ def test_incorrect_catchup(self, os_ops: OsOperations): node.init(allow_streaming=True).start() # node has no master, can't catch up - with pytest.raises(expected_exception=TestgresException): + with pytest.raises(expected_exception=testgres_TestgresException): node.catchup() def test_promotion(self, os_ops: OsOperations): From 87dbecb3a1c190099b43fc1510c92a6baa35fea3 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 18 Mar 2025 19:27:44 +0300 Subject: [PATCH 427/475] OsOperations::remove_file is added It seems to me we forgot to add it. --- testgres/operations/os_ops.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/testgres/operations/os_ops.py b/testgres/operations/os_ops.py index 00880863..f20a7a30 100644 --- a/testgres/operations/os_ops.py +++ b/testgres/operations/os_ops.py @@ -108,6 +108,10 @@ def isdir(self, dirname): def get_file_size(self, filename): raise NotImplementedError() + def remove_file(self, filename): + assert type(filename) == str # noqa: E721 + raise NotImplementedError() + # Processes control def kill(self, pid, signal): # Kill the process From 25c6a2fe3d0557f0a60d719c74e2f044cf22f6ac Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Tue, 18 Mar 2025 21:34:25 +0300 Subject: [PATCH 428/475] Log files (#224) Let's start writing test events into files to provide an access to the tests execution information. --- .gitignore | 1 + tests/conftest.py | 510 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 511 insertions(+) create mode 100644 tests/conftest.py diff --git a/.gitignore b/.gitignore index 038d1952..238181b5 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ dist/ build/ docs/build/ +logs/ env/ venv/ diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 00000000..e37c3c77 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,510 @@ +# ///////////////////////////////////////////////////////////////////////////// +# PyTest Configuration + +import _pytest.outcomes + +import pluggy +import pytest +import _pytest +import os +import logging +import pathlib +import math +import datetime + +# ///////////////////////////////////////////////////////////////////////////// + +C_ROOT_DIR__RELATIVE = ".." + +# ///////////////////////////////////////////////////////////////////////////// +# TestConfigPropNames + + +class TestConfigPropNames: + TEST_CFG__LOG_DIR = "TEST_CFG__LOG_DIR" + + +# ///////////////////////////////////////////////////////////////////////////// +# TestStartupData__Helper + + +class TestStartupData__Helper: + sm_StartTS = datetime.datetime.now() + + # -------------------------------------------------------------------- + def GetStartTS() -> datetime.datetime: + assert type(__class__.sm_StartTS) == datetime.datetime # noqa: E721 + return __class__.sm_StartTS + + # -------------------------------------------------------------------- + def CalcRootDir() -> str: + r = os.path.abspath(__file__) + r = os.path.dirname(r) + r = os.path.join(r, C_ROOT_DIR__RELATIVE) + r = os.path.abspath(r) + return r + + # -------------------------------------------------------------------- + def CalcCurrentTestWorkerSignature() -> str: + currentPID = os.getpid() + assert type(currentPID) + + startTS = __class__.sm_StartTS + assert type(startTS) + + result = "pytest-{0:04d}{1:02d}{2:02d}_{3:02d}{4:02d}{5:02d}".format( + startTS.year, + startTS.month, + startTS.day, + startTS.hour, + startTS.minute, + startTS.second, + ) + + gwid = os.environ.get("PYTEST_XDIST_WORKER") + + if gwid is not None: + result += "--xdist_" + str(gwid) + + result += "--" + "pid" + str(currentPID) + return result + + +# ///////////////////////////////////////////////////////////////////////////// +# TestStartupData + + +class TestStartupData: + sm_RootDir: str = TestStartupData__Helper.CalcRootDir() + sm_CurrentTestWorkerSignature: str = ( + TestStartupData__Helper.CalcCurrentTestWorkerSignature() + ) + + # -------------------------------------------------------------------- + def GetRootDir() -> str: + assert type(__class__.sm_RootDir) == str # noqa: E721 + return __class__.sm_RootDir + + # -------------------------------------------------------------------- + def GetCurrentTestWorkerSignature() -> str: + assert type(__class__.sm_CurrentTestWorkerSignature) == str # noqa: E721 + return __class__.sm_CurrentTestWorkerSignature + + +# /////////////////////////////////////////////////////////////////////////////# ///////////////////////////////////////////////////////////////////////////// +# Fixtures + + +# ///////////////////////////////////////////////////////////////////////////// +# TEST_PROCESS_STATS + + +class TEST_PROCESS_STATS: + cTotalTests: int = 0 + cNotExecutedTests: int = 0 + cExecutedTests: int = 0 + cPassedTests: int = 0 + cFailedTests: int = 0 + cXFailedTests: int = 0 + cSkippedTests: int = 0 + cNotXFailedTests: int = 0 + cUnexpectedTests: int = 0 + + FailedTests = list[str]() + XFailedTests = list[str]() + NotXFailedTests = list[str]() + + # -------------------------------------------------------------------- + def incrementTotalTestCount() -> None: + __class__.cTotalTests += 1 + + # -------------------------------------------------------------------- + def incrementNotExecutedTestCount() -> None: + __class__.cNotExecutedTests += 1 + + # -------------------------------------------------------------------- + def incrementExecutedTestCount() -> int: + __class__.cExecutedTests += 1 + return __class__.cExecutedTests + + # -------------------------------------------------------------------- + def incrementPassedTestCount() -> None: + __class__.cPassedTests += 1 + + # -------------------------------------------------------------------- + def incrementFailedTestCount(testID: str) -> None: + assert type(testID) == str # noqa: E721 + assert type(__class__.FailedTests) == list # noqa: E721 + + __class__.FailedTests.append(testID) # raise? + __class__.cFailedTests += 1 + + # -------------------------------------------------------------------- + def incrementXFailedTestCount(testID: str) -> None: + assert type(testID) == str # noqa: E721 + assert type(__class__.XFailedTests) == list # noqa: E721 + + __class__.XFailedTests.append(testID) # raise? + __class__.cXFailedTests += 1 + + # -------------------------------------------------------------------- + def incrementSkippedTestCount() -> None: + __class__.cSkippedTests += 1 + + # -------------------------------------------------------------------- + def incrementNotXFailedTests(testID: str) -> None: + assert type(testID) == str # noqa: E721 + assert type(__class__.NotXFailedTests) == list # noqa: E721 + + __class__.NotXFailedTests.append(testID) # raise? + __class__.cNotXFailedTests += 1 + + # -------------------------------------------------------------------- + def incrementUnexpectedTests() -> None: + __class__.cUnexpectedTests += 1 + + +# ///////////////////////////////////////////////////////////////////////////// + + +def timedelta_to_human_text(delta: datetime.timedelta) -> str: + assert isinstance(delta, datetime.timedelta) + + C_SECONDS_IN_MINUTE = 60 + C_SECONDS_IN_HOUR = 60 * C_SECONDS_IN_MINUTE + + v = delta.seconds + + cHours = int(v / C_SECONDS_IN_HOUR) + v = v - cHours * C_SECONDS_IN_HOUR + cMinutes = int(v / C_SECONDS_IN_MINUTE) + cSeconds = v - cMinutes * C_SECONDS_IN_MINUTE + + result = "" if delta.days == 0 else "{0} day(s) ".format(delta.days) + + result = result + "{:02d}:{:02d}:{:02d}.{:06d}".format( + cHours, cMinutes, cSeconds, delta.microseconds + ) + + return result + + +# ///////////////////////////////////////////////////////////////////////////// + + +def helper__makereport__setup( + item: pytest.Function, call: pytest.CallInfo, outcome: pluggy.Result +): + assert item is not None + assert call is not None + assert outcome is not None + assert type(item) == pytest.Function # noqa: E721 + assert type(call) == pytest.CallInfo # noqa: E721 + assert type(outcome) == pluggy.Result # noqa: E721 + + # logging.info("pytest_runtest_makereport - setup") + + TEST_PROCESS_STATS.incrementTotalTestCount() + + rep: pytest.TestReport = outcome.get_result() + assert rep is not None + assert type(rep) == pytest.TestReport # noqa: E721 + + if rep.outcome == "skipped": + TEST_PROCESS_STATS.incrementNotExecutedTestCount() + return + + assert rep.outcome == "passed" + + testNumber = TEST_PROCESS_STATS.incrementExecutedTestCount() + + testID = "" + + if item.cls is not None: + testID = item.cls.__module__ + "." + item.cls.__name__ + "::" + + testID = testID + item.name + + if testNumber > 1: + logging.info("") + + logging.info("******************************************************") + logging.info("* START TEST {0}".format(testID)) + logging.info("*") + logging.info("* Path : {0}".format(item.path)) + logging.info("* Number: {0}".format(testNumber)) + logging.info("*") + + +# ------------------------------------------------------------------------ +def helper__makereport__call( + item: pytest.Function, call: pytest.CallInfo, outcome: pluggy.Result +): + assert item is not None + assert call is not None + assert outcome is not None + assert type(item) == pytest.Function # noqa: E721 + assert type(call) == pytest.CallInfo # noqa: E721 + assert type(outcome) == pluggy.Result # noqa: E721 + + # logging.info("pytest_runtest_makereport - call") + + rep = outcome.get_result() + assert rep is not None + assert type(rep) == pytest.TestReport # noqa: E721 + + # -------- + testID = "" + + if item.cls is not None: + testID = item.cls.__module__ + "." + item.cls.__name__ + "::" + + testID = testID + item.name + + # -------- + assert call.start <= call.stop + + startDT = datetime.datetime.fromtimestamp(call.start) + assert type(startDT) == datetime.datetime # noqa: E721 + stopDT = datetime.datetime.fromtimestamp(call.stop) + assert type(stopDT) == datetime.datetime # noqa: E721 + + testDurration = stopDT - startDT + assert type(testDurration) == datetime.timedelta # noqa: E721 + + # -------- + exitStatus = None + if rep.outcome == "skipped": + assert call.excinfo is not None # research + assert call.excinfo.value is not None # research + + if type(call.excinfo.value) == _pytest.outcomes.Skipped: # noqa: E721 + assert not hasattr(rep, "wasxfail") + + TEST_PROCESS_STATS.incrementSkippedTestCount() + + exitStatus = "SKIPPED" + reasonText = str(call.excinfo.value) + reasonMsg = "SKIP REASON: {0}" + + elif type(call.excinfo.value) == _pytest.outcomes.XFailed: # noqa: E721 + TEST_PROCESS_STATS.incrementXFailedTestCount(testID) + + exitStatus = "XFAILED" + reasonText = str(call.excinfo.value) + reasonMsg = "XFAIL REASON: {0}" + else: + exitStatus = "XFAILED" + assert hasattr(rep, "wasxfail") + assert rep.wasxfail is not None + assert type(rep.wasxfail) == str # noqa: E721 + + TEST_PROCESS_STATS.incrementXFailedTestCount(testID) + + reasonText = rep.wasxfail + reasonMsg = "XFAIL REASON: {0}" + + logging.error(call.excinfo.value) + + if reasonText != "": + logging.info("*") + logging.info("* " + reasonMsg.format(reasonText)) + + elif rep.outcome == "failed": + assert call.excinfo is not None + assert call.excinfo.value is not None + + TEST_PROCESS_STATS.incrementFailedTestCount(testID) + + logging.error(call.excinfo.value) + exitStatus = "FAILED" + elif rep.outcome == "passed": + assert call.excinfo is None + + if hasattr(rep, "wasxfail"): + assert type(rep.wasxfail) == str # noqa: E721 + + TEST_PROCESS_STATS.incrementNotXFailedTests(testID) + + warnMsg = "Test is marked as xfail" + + if rep.wasxfail != "": + warnMsg += " [" + rep.wasxfail + "]" + + logging.warning(warnMsg) + exitStatus = "NOT XFAILED" + else: + assert not hasattr(rep, "wasxfail") + + TEST_PROCESS_STATS.incrementPassedTestCount() + exitStatus = "PASSED" + else: + TEST_PROCESS_STATS.incrementUnexpectedTests() + exitStatus = "UNEXPECTED [{0}]".format(rep.outcome) + assert False + + # -------- + logging.info("*") + logging.info("* DURATION : {0}".format(timedelta_to_human_text(testDurration))) + logging.info("*") + logging.info("* EXIT STATUS : {0}".format(exitStatus)) + logging.info("*") + logging.info("* STOP TEST {0}".format(testID)) + + +# ///////////////////////////////////////////////////////////////////////////// + + +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_makereport(item: pytest.Function, call: pytest.CallInfo): + assert item is not None + assert call is not None + assert type(item) == pytest.Function # noqa: E721 + assert type(call) == pytest.CallInfo # noqa: E721 + + # logging.info("[pytest_runtest_makereport][#001][{0}][{1}]".format(item.name, call.when)) + + outcome: pluggy.Result = yield + assert outcome is not None + assert type(outcome) == pluggy.Result # noqa: E721 + + # logging.info("[pytest_runtest_makereport][#002][{0}][{1}]".format(item.name, call.when)) + + rep: pytest.TestReport = outcome.get_result() + assert rep is not None + assert type(rep) == pytest.TestReport # noqa: E721 + + if call.when == "collect": + return + + if call.when == "setup": + helper__makereport__setup(item, call, outcome) + return + + if call.when == "call": + helper__makereport__call(item, call, outcome) + return + + if call.when == "teardown": + return + + assert False + + +# ///////////////////////////////////////////////////////////////////////////// + + +def helper__calc_W(n: int) -> int: + assert n > 0 + + x = int(math.log10(n)) + assert type(x) == int # noqa: E721 + assert x >= 0 + x += 1 + return x + + +# ------------------------------------------------------------------------ +def helper__print_test_list(tests: list[str]) -> None: + assert type(tests) == list # noqa: E721 + + assert helper__calc_W(9) == 1 + assert helper__calc_W(10) == 2 + assert helper__calc_W(11) == 2 + assert helper__calc_W(99) == 2 + assert helper__calc_W(100) == 3 + assert helper__calc_W(101) == 3 + assert helper__calc_W(999) == 3 + assert helper__calc_W(1000) == 4 + assert helper__calc_W(1001) == 4 + + W = helper__calc_W(len(tests)) + + templateLine = "{0:0" + str(W) + "d}. {1}" + + nTest = 0 + + while nTest < len(tests): + testID = tests[nTest] + assert type(testID) == str # noqa: E721 + nTest += 1 + logging.info(templateLine.format(nTest, testID)) + + +# ///////////////////////////////////////////////////////////////////////////// + + +@pytest.fixture(autouse=True, scope="session") +def run_after_tests(request: pytest.FixtureRequest): + assert isinstance(request, pytest.FixtureRequest) + + yield + + logging.info("--------------------------- [FAILED TESTS]") + logging.info("") + + assert len(TEST_PROCESS_STATS.FailedTests) == TEST_PROCESS_STATS.cFailedTests + + if len(TEST_PROCESS_STATS.FailedTests) > 0: + helper__print_test_list(TEST_PROCESS_STATS.FailedTests) + logging.info("") + + logging.info("--------------------------- [XFAILED TESTS]") + logging.info("") + + assert len(TEST_PROCESS_STATS.XFailedTests) == TEST_PROCESS_STATS.cXFailedTests + + if len(TEST_PROCESS_STATS.XFailedTests) > 0: + helper__print_test_list(TEST_PROCESS_STATS.XFailedTests) + logging.info("") + + logging.info("--------------------------- [NOT XFAILED TESTS]") + logging.info("") + + assert ( + len(TEST_PROCESS_STATS.NotXFailedTests) == TEST_PROCESS_STATS.cNotXFailedTests + ) + + if len(TEST_PROCESS_STATS.NotXFailedTests) > 0: + helper__print_test_list(TEST_PROCESS_STATS.NotXFailedTests) + logging.info("") + + logging.info("--------------------------- [SUMMARY STATISTICS]") + logging.info("") + logging.info("[TESTS]") + logging.info(" TOTAL : {0}".format(TEST_PROCESS_STATS.cTotalTests)) + logging.info(" EXECUTED : {0}".format(TEST_PROCESS_STATS.cExecutedTests)) + logging.info(" NOT EXECUTED: {0}".format(TEST_PROCESS_STATS.cNotExecutedTests)) + logging.info("") + logging.info(" PASSED : {0}".format(TEST_PROCESS_STATS.cPassedTests)) + logging.info(" FAILED : {0}".format(TEST_PROCESS_STATS.cFailedTests)) + logging.info(" XFAILED : {0}".format(TEST_PROCESS_STATS.cXFailedTests)) + logging.info(" NOT XFAILED : {0}".format(TEST_PROCESS_STATS.cNotXFailedTests)) + logging.info(" SKIPPED : {0}".format(TEST_PROCESS_STATS.cSkippedTests)) + logging.info(" UNEXPECTED : {0}".format(TEST_PROCESS_STATS.cUnexpectedTests)) + logging.info("") + + +# ///////////////////////////////////////////////////////////////////////////// + + +@pytest.hookimpl(trylast=True) +def pytest_configure(config: pytest.Config) -> None: + assert isinstance(config, pytest.Config) + + log_name = TestStartupData.GetCurrentTestWorkerSignature() + log_name += ".log" + + if TestConfigPropNames.TEST_CFG__LOG_DIR in os.environ: + log_path_v = os.environ[TestConfigPropNames.TEST_CFG__LOG_DIR] + log_path = pathlib.Path(log_path_v) + else: + log_path = config.rootpath.joinpath("logs") + + log_path.mkdir(exist_ok=True) + + logging_plugin = config.pluginmanager.get_plugin("logging-plugin") + logging_plugin.set_log_path(str(log_path / log_name)) + + +# ///////////////////////////////////////////////////////////////////////////// From 81a5eb43b7398cd3c436ffe32b21e20c4c9712b4 Mon Sep 17 00:00:00 2001 From: dura0ok Date: Wed, 19 Mar 2025 23:08:05 +0700 Subject: [PATCH 429/475] add FUSE support to plugin pg_probackup2 (#184) --- .../pg_probackup2/pg_probackup2/app.py | 43 ++++++++++++++++--- 1 file changed, 37 insertions(+), 6 deletions(-) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/app.py b/testgres/plugins/pg_probackup2/pg_probackup2/app.py index 57492814..d47cf51f 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/app.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/app.py @@ -45,6 +45,7 @@ class ProbackupApp: def __init__(self, test_class: unittest.TestCase, pg_node, pb_log_path, test_env, auto_compress_alg, backup_dir, probackup_path=None): + self.process = None self.test_class = test_class self.pg_node = pg_node self.pb_log_path = pb_log_path @@ -60,8 +61,35 @@ def __init__(self, test_class: unittest.TestCase, self.test_class.output = None self.execution_time = None + def form_daemon_process(self, cmdline, env): + def stream_output(stream: subprocess.PIPE) -> None: + try: + for line in iter(stream.readline, ''): + print(line) + self.test_class.output += line + finally: + stream.close() + + self.process = subprocess.Popen( + cmdline, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=env + ) + logging.info(f"Process started in background with PID: {self.process.pid}") + + if self.process.stdout and self.process.stderr: + stdout_thread = threading.Thread(target=stream_output, args=(self.process.stdout,), daemon=True) + stderr_thread = threading.Thread(target=stream_output, args=(self.process.stderr,), daemon=True) + + stdout_thread.start() + stderr_thread.start() + + return self.process.pid + def run(self, command, gdb=False, old_binary=False, return_id=True, env=None, - skip_log_directory=False, expect_error=False, use_backup_dir=True): + skip_log_directory=False, expect_error=False, use_backup_dir=True, daemonize=False): """ Run pg_probackup backup_dir: target directory for making backup @@ -118,11 +146,14 @@ def run(self, command, gdb=False, old_binary=False, return_id=True, env=None, logging.warning("pg_probackup gdb suspended, waiting gdb connection on localhost:{0}".format(gdb_port)) start_time = time.time() - self.test_class.output = subprocess.check_output( - cmdline, - stderr=subprocess.STDOUT, - env=env - ).decode('utf-8', errors='replace') + if daemonize: + return self.form_daemon_process(cmdline, env) + else: + self.test_class.output = subprocess.check_output( + cmdline, + stderr=subprocess.STDOUT, + env=env + ).decode('utf-8', errors='replace') end_time = time.time() self.execution_time = end_time - start_time From ddfaff401f0f55b5a51bb0490d12dda1631acbe3 Mon Sep 17 00:00:00 2001 From: asavchkov Date: Thu, 20 Mar 2025 10:13:51 +0700 Subject: [PATCH 430/475] pg-probackup2 version 0.0.6 --- testgres/plugins/pg_probackup2/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testgres/plugins/pg_probackup2/setup.py b/testgres/plugins/pg_probackup2/setup.py index 619b8d39..8bcfe7b4 100644 --- a/testgres/plugins/pg_probackup2/setup.py +++ b/testgres/plugins/pg_probackup2/setup.py @@ -4,7 +4,7 @@ from distutils.core import setup setup( - version='0.0.5', + version='0.0.6', name='testgres_pg_probackup2', packages=['pg_probackup2', 'pg_probackup2.storage'], description='Plugin for testgres that manages pg_probackup2', From f0bf7a8994b5dd39e5c2f841381d5e4be5e22297 Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Fri, 21 Mar 2025 14:15:24 +0300 Subject: [PATCH 431/475] CI files are updated (#225) * [CI] work with 'time' is corrected AltLinux 10 does not support the sequential "time coverage run ...". Because this OS does not has a builtin command 'time' in bash. https://forum.altlinux.org/index.php?topic=48342.0 We will install 'time' manually and use another command " time coverage run ..." that works without problems but it requires to install 'time' on Ubuntu 2024.04, too. AlpineLinux processes a new command line without any problems. * [CI] An initization of python virtualenv is simplified Let's avoid creating useless environment variables. --- Dockerfile--ubuntu_24_04.tmpl | 1 + run_tests.sh | 27 +++++++++------------------ 2 files changed, 10 insertions(+), 18 deletions(-) diff --git a/Dockerfile--ubuntu_24_04.tmpl b/Dockerfile--ubuntu_24_04.tmpl index c1ddeab6..3bdc6640 100644 --- a/Dockerfile--ubuntu_24_04.tmpl +++ b/Dockerfile--ubuntu_24_04.tmpl @@ -9,6 +9,7 @@ RUN apt update RUN apt install -y sudo curl ca-certificates RUN apt update RUN apt install -y openssh-server +RUN apt install -y time RUN apt update RUN apt install -y postgresql-common diff --git a/run_tests.sh b/run_tests.sh index 0fecde60..a40a97cf 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -8,24 +8,17 @@ if [ -z ${TEST_FILTER+x} ]; \ then export TEST_FILTER="TestgresTests or (TestTestgresCommon and (not remote_ops))"; \ fi -# choose python version -echo python version is $PYTHON_VERSION -VIRTUALENV="virtualenv --python=/usr/bin/python$PYTHON_VERSION" -PIP="pip$PYTHON_VERSION" - # fail early echo check that pg_config is in PATH command -v pg_config -# prepare environment -VENV_PATH=/tmp/testgres_venv +# prepare python environment +VENV_PATH="/tmp/testgres_venv" rm -rf $VENV_PATH -$VIRTUALENV $VENV_PATH +virtualenv --python="/usr/bin/python${PYTHON_VERSION}" "${VENV_PATH}" export VIRTUAL_ENV_DISABLE_PROMPT=1 -source $VENV_PATH/bin/activate - -# install utilities -$PIP install coverage flake8 psutil Sphinx pytest pytest-xdist psycopg2 six psutil +source "${VENV_PATH}/bin/activate" +pip install coverage flake8 psutil Sphinx pytest pytest-xdist psycopg2 six psutil # install testgres' dependencies export PYTHONPATH=$(pwd) @@ -45,15 +38,13 @@ time coverage run -a -m pytest -l -v -n 4 -k "${TEST_FILTER}" # run tests (PG_BIN) -time \ - PG_BIN=$(pg_config --bindir) \ - coverage run -a -m pytest -l -v -n 4 -k "${TEST_FILTER}" +PG_BIN=$(pg_config --bindir) \ +time coverage run -a -m pytest -l -v -n 4 -k "${TEST_FILTER}" # run tests (PG_CONFIG) -time \ - PG_CONFIG=$(pg_config --bindir)/pg_config \ - coverage run -a -m pytest -l -v -n 4 -k "${TEST_FILTER}" +PG_CONFIG=$(pg_config --bindir)/pg_config \ +time coverage run -a -m pytest -l -v -n 4 -k "${TEST_FILTER}" # show coverage From 76fa94cfa6d2645abbb59a027074bd97b9f389ee Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Mon, 24 Mar 2025 13:04:41 +0300 Subject: [PATCH 432/475] [CI] Run tests on AltLinux 10 and 11 (#219) This patch adds an automated tests of testgres on AltLinux 10 and 11. We will execute only "local" tests because AltLinux has an unexpected problem with SSH connection - it is created too slowly. --- .travis.yml | 2 + Dockerfile--altlinux_10.tmpl | 118 +++++++++++++++++++++++++++++++++++ Dockerfile--altlinux_11.tmpl | 118 +++++++++++++++++++++++++++++++++++ 3 files changed, 238 insertions(+) create mode 100644 Dockerfile--altlinux_10.tmpl create mode 100644 Dockerfile--altlinux_11.tmpl diff --git a/.travis.yml b/.travis.yml index 3a889845..7557a2ce 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,3 +28,5 @@ env: - TEST_PLATFORM=std PYTHON_VERSION=3 PG_VERSION=10 - TEST_PLATFORM=std-all PYTHON_VERSION=3 PG_VERSION=17 - TEST_PLATFORM=ubuntu_24_04 PYTHON_VERSION=3 PG_VERSION=17 + - TEST_PLATFORM=altlinux_10 PYTHON_VERSION=3 PG_VERSION=17 + - TEST_PLATFORM=altlinux_11 PYTHON_VERSION=3 PG_VERSION=17 diff --git a/Dockerfile--altlinux_10.tmpl b/Dockerfile--altlinux_10.tmpl new file mode 100644 index 00000000..e60e9320 --- /dev/null +++ b/Dockerfile--altlinux_10.tmpl @@ -0,0 +1,118 @@ +ARG PG_VERSION +ARG PYTHON_VERSION + +# --------------------------------------------- base1 +FROM alt:p10 as base1 +ARG PG_VERSION + +RUN apt-get update +RUN apt-get install -y sudo curl ca-certificates +RUN apt-get update +RUN apt-get install -y openssh-server openssh-clients +RUN apt-get install -y time + +# RUN apt-get install -y mc + +RUN apt-get install -y libsqlite3-devel + +EXPOSE 22 + +RUN ssh-keygen -A + +# --------------------------------------------- postgres +FROM base1 as base1_with_dev_tools + +RUN apt-get update + +RUN apt-get install -y git +RUN apt-get install -y gcc +RUN apt-get install -y make + +RUN apt-get install -y meson +RUN apt-get install -y flex +RUN apt-get install -y bison + +RUN apt-get install -y pkg-config +RUN apt-get install -y libssl-devel +RUN apt-get install -y libicu-devel +RUN apt-get install -y libzstd-devel +RUN apt-get install -y zlib-devel +RUN apt-get install -y liblz4-devel +RUN apt-get install -y libzstd-devel +RUN apt-get install -y libxml2-devel + +# --------------------------------------------- postgres +FROM base1_with_dev_tools as base1_with_pg-17 + +RUN git clone https://github.com/postgres/postgres.git -b REL_17_STABLE /pg/postgres/source + +WORKDIR /pg/postgres/source + +RUN ./configure --prefix=/pg/postgres/install --with-zlib --with-openssl --without-readline --with-lz4 --with-zstd --with-libxml +RUN make -j 4 install +RUN make -j 4 -C contrib install + +# SETUP PG_CONFIG +# When pg_config symlink in /usr/local/bin it returns a real (right) result of --bindir +RUN ln -s /pg/postgres/install/bin/pg_config -t /usr/local/bin + +# SETUP PG CLIENT LIBRARY +# libpq.so.5 is enough +RUN ln -s /pg/postgres/install/lib/libpq.so.5.17 /usr/lib64/libpq.so.5 + +# --------------------------------------------- base2_with_python-3 +FROM base1_with_pg-${PG_VERSION} as base2_with_python-3 +RUN apt-get install -y python3 +RUN apt-get install -y python3-dev +RUN apt-get install -y python3-module-virtualenv +RUN apt-get install -y python3-modules-sqlite3 + +# AltLinux does not have "generic" virtualenv utility. Let's create it. +RUN if [[ -f "/usr/bin/virtualenv" ]] ; then \ + echo AAA; \ + elif [[ -f "/usr/bin/virtualenv3" ]] ; then \ + ln -s /usr/bin/virtualenv3 /usr/bin/virtualenv; \ + else \ + echo "/usr/bin/virtualenv is not created!"; \ + exit 1; \ + fi + +ENV PYTHON_VERSION=3 + +# --------------------------------------------- final +FROM base2_with_python-${PYTHON_VERSION} as final + +RUN adduser test -G wheel + +# It enables execution of "sudo service ssh start" without password +RUN sh -c "echo \"WHEEL_USERS ALL=(ALL:ALL) NOPASSWD: ALL\"" >> /etc/sudoers + +ADD . /pg/testgres +WORKDIR /pg/testgres +RUN chown -R test /pg/testgres + +ENV LANG=C.UTF-8 + +USER test + +RUN chmod 700 ~/ +RUN mkdir -p ~/.ssh + +# +# Altlinux 10 and 11 too slowly create a new SSH connection (x6). +# +# So, we exclude the "remote" tests until this problem has been resolved. +# + +ENTRYPOINT sh -c " \ +set -eux; \ +echo HELLO FROM ENTRYPOINT; \ +echo HOME DIR IS [`realpath ~/`]; \ +sudo /usr/sbin/sshd; \ +ssh-keyscan -H localhost >> ~/.ssh/known_hosts; \ +ssh-keyscan -H 127.0.0.1 >> ~/.ssh/known_hosts; \ +ssh-keygen -t rsa -f ~/.ssh/id_rsa -q -N ''; \ +cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys; \ +chmod 600 ~/.ssh/authorized_keys; \ +ls -la ~/.ssh/; \ +TEST_FILTER=\"TestgresTests or (TestTestgresCommon and (not remote_ops))\" bash ./run_tests.sh;" diff --git a/Dockerfile--altlinux_11.tmpl b/Dockerfile--altlinux_11.tmpl new file mode 100644 index 00000000..4b591632 --- /dev/null +++ b/Dockerfile--altlinux_11.tmpl @@ -0,0 +1,118 @@ +ARG PG_VERSION +ARG PYTHON_VERSION + +# --------------------------------------------- base1 +FROM alt:p11 as base1 +ARG PG_VERSION + +RUN apt-get update +RUN apt-get install -y sudo curl ca-certificates +RUN apt-get update +RUN apt-get install -y openssh-server openssh-clients +RUN apt-get install -y time + +# RUN apt-get install -y mc + +RUN apt-get install -y libsqlite3-devel + +EXPOSE 22 + +RUN ssh-keygen -A + +# --------------------------------------------- postgres +FROM base1 as base1_with_dev_tools + +RUN apt-get update + +RUN apt-get install -y git +RUN apt-get install -y gcc +RUN apt-get install -y make + +RUN apt-get install -y meson +RUN apt-get install -y flex +RUN apt-get install -y bison + +RUN apt-get install -y pkg-config +RUN apt-get install -y libssl-devel +RUN apt-get install -y libicu-devel +RUN apt-get install -y libzstd-devel +RUN apt-get install -y zlib-devel +RUN apt-get install -y liblz4-devel +RUN apt-get install -y libzstd-devel +RUN apt-get install -y libxml2-devel + +# --------------------------------------------- postgres +FROM base1_with_dev_tools as base1_with_pg-17 + +RUN git clone https://github.com/postgres/postgres.git -b REL_17_STABLE /pg/postgres/source + +WORKDIR /pg/postgres/source + +RUN ./configure --prefix=/pg/postgres/install --with-zlib --with-openssl --without-readline --with-lz4 --with-zstd --with-libxml +RUN make -j 4 install +RUN make -j 4 -C contrib install + +# SETUP PG_CONFIG +# When pg_config symlink in /usr/local/bin it returns a real (right) result of --bindir +RUN ln -s /pg/postgres/install/bin/pg_config -t /usr/local/bin + +# SETUP PG CLIENT LIBRARY +# libpq.so.5 is enough +RUN ln -s /pg/postgres/install/lib/libpq.so.5.17 /usr/lib64/libpq.so.5 + +# --------------------------------------------- base2_with_python-3 +FROM base1_with_pg-${PG_VERSION} as base2_with_python-3 +RUN apt-get install -y python3 +RUN apt-get install -y python3-dev +RUN apt-get install -y python3-module-virtualenv +RUN apt-get install -y python3-modules-sqlite3 + +# AltLinux does not have "generic" virtualenv utility. Let's create it. +RUN if [[ -f "/usr/bin/virtualenv" ]] ; then \ + echo AAA; \ + elif [[ -f "/usr/bin/virtualenv3" ]] ; then \ + ln -s /usr/bin/virtualenv3 /usr/bin/virtualenv; \ + else \ + echo "/usr/bin/virtualenv is not created!"; \ + exit 1; \ + fi + +ENV PYTHON_VERSION=3 + +# --------------------------------------------- final +FROM base2_with_python-${PYTHON_VERSION} as final + +RUN adduser test -G wheel + +# It enables execution of "sudo service ssh start" without password +RUN sh -c "echo \"WHEEL_USERS ALL=(ALL:ALL) NOPASSWD: ALL\"" >> /etc/sudoers + +ADD . /pg/testgres +WORKDIR /pg/testgres +RUN chown -R test /pg/testgres + +ENV LANG=C.UTF-8 + +USER test + +RUN chmod 700 ~/ +RUN mkdir -p ~/.ssh + +# +# Altlinux 10 and 11 too slowly create a new SSH connection (x6). +# +# So, we exclude the "remote" tests until this problem has been resolved. +# + +ENTRYPOINT sh -c " \ +set -eux; \ +echo HELLO FROM ENTRYPOINT; \ +echo HOME DIR IS [`realpath ~/`]; \ +sudo /usr/sbin/sshd; \ +ssh-keyscan -H localhost >> ~/.ssh/known_hosts; \ +ssh-keyscan -H 127.0.0.1 >> ~/.ssh/known_hosts; \ +ssh-keygen -t rsa -f ~/.ssh/id_rsa -q -N ''; \ +cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys; \ +chmod 600 ~/.ssh/authorized_keys; \ +ls -la ~/.ssh/; \ +TEST_FILTER=\"TestgresTests or (TestTestgresCommon and (not remote_ops))\" bash ./run_tests.sh;" From bc18e5b9362d9634c468c616e40282cf7e30e5da Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Fri, 28 Mar 2025 15:55:58 +0300 Subject: [PATCH 433/475] conftest.py is updated [refactoring] (#226) New code can process a failure in fixtures and builds a list of these cases (achtung tests). --- tests/conftest.py | 146 +++++++++++++++++++++++++++++++--------------- 1 file changed, 98 insertions(+), 48 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index e37c3c77..0f65838e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,17 +1,18 @@ # ///////////////////////////////////////////////////////////////////////////// # PyTest Configuration -import _pytest.outcomes - import pluggy import pytest -import _pytest import os import logging import pathlib import math import datetime +import _pytest.outcomes +import _pytest.unittest +import _pytest.logging + # ///////////////////////////////////////////////////////////////////////////// C_ROOT_DIR__RELATIVE = ".." @@ -91,10 +92,6 @@ def GetCurrentTestWorkerSignature() -> str: return __class__.sm_CurrentTestWorkerSignature -# /////////////////////////////////////////////////////////////////////////////# ///////////////////////////////////////////////////////////////////////////// -# Fixtures - - # ///////////////////////////////////////////////////////////////////////////// # TEST_PROCESS_STATS @@ -109,10 +106,12 @@ class TEST_PROCESS_STATS: cSkippedTests: int = 0 cNotXFailedTests: int = 0 cUnexpectedTests: int = 0 + cAchtungTests: int = 0 FailedTests = list[str]() XFailedTests = list[str]() NotXFailedTests = list[str]() + AchtungTests = list[str]() # -------------------------------------------------------------------- def incrementTotalTestCount() -> None: @@ -163,6 +162,14 @@ def incrementNotXFailedTests(testID: str) -> None: def incrementUnexpectedTests() -> None: __class__.cUnexpectedTests += 1 + # -------------------------------------------------------------------- + def incrementAchtungTestCount(testID: str) -> None: + assert type(testID) == str # noqa: E721 + assert type(__class__.AchtungTests) == list # noqa: E721 + + __class__.AchtungTests.append(testID) # raise? + __class__.cAchtungTests += 1 + # ///////////////////////////////////////////////////////////////////////////// @@ -198,10 +205,13 @@ def helper__makereport__setup( assert item is not None assert call is not None assert outcome is not None - assert type(item) == pytest.Function # noqa: E721 + # it may be pytest.Function or _pytest.unittest.TestCaseFunction + assert isinstance(item, pytest.Function) assert type(call) == pytest.CallInfo # noqa: E721 assert type(outcome) == pluggy.Result # noqa: E721 + C_LINE1 = "******************************************************" + # logging.info("pytest_runtest_makereport - setup") TEST_PROCESS_STATS.incrementTotalTestCount() @@ -214,10 +224,6 @@ def helper__makereport__setup( TEST_PROCESS_STATS.incrementNotExecutedTestCount() return - assert rep.outcome == "passed" - - testNumber = TEST_PROCESS_STATS.incrementExecutedTestCount() - testID = "" if item.cls is not None: @@ -225,15 +231,35 @@ def helper__makereport__setup( testID = testID + item.name - if testNumber > 1: - logging.info("") + if rep.outcome == "passed": + testNumber = TEST_PROCESS_STATS.incrementExecutedTestCount() + + logging.info(C_LINE1) + logging.info("* START TEST {0}".format(testID)) + logging.info("*") + logging.info("* Path : {0}".format(item.path)) + logging.info("* Number: {0}".format(testNumber)) + logging.info("*") + return + + assert rep.outcome != "passed" + + TEST_PROCESS_STATS.incrementAchtungTestCount(testID) - logging.info("******************************************************") - logging.info("* START TEST {0}".format(testID)) + logging.info(C_LINE1) + logging.info("* ACHTUNG TEST {0}".format(testID)) logging.info("*") logging.info("* Path : {0}".format(item.path)) - logging.info("* Number: {0}".format(testNumber)) + logging.info("* Outcome is [{0}]".format(rep.outcome)) + + if rep.outcome == "failed": + assert call.excinfo is not None + assert call.excinfo.value is not None + logging.info("*") + logging.error(call.excinfo.value) + logging.info("*") + return # ------------------------------------------------------------------------ @@ -243,12 +269,11 @@ def helper__makereport__call( assert item is not None assert call is not None assert outcome is not None - assert type(item) == pytest.Function # noqa: E721 + # it may be pytest.Function or _pytest.unittest.TestCaseFunction + assert isinstance(item, pytest.Function) assert type(call) == pytest.CallInfo # noqa: E721 assert type(outcome) == pluggy.Result # noqa: E721 - # logging.info("pytest_runtest_makereport - call") - rep = outcome.get_result() assert rep is not None assert type(rep) == pytest.TestReport # noqa: E721 @@ -341,7 +366,8 @@ def helper__makereport__call( else: TEST_PROCESS_STATS.incrementUnexpectedTests() exitStatus = "UNEXPECTED [{0}]".format(rep.outcome) - assert False + # [2025-03-28] It may create a useless problem in new environment. + # assert False # -------- logging.info("*") @@ -350,6 +376,7 @@ def helper__makereport__call( logging.info("* EXIT STATUS : {0}".format(exitStatus)) logging.info("*") logging.info("* STOP TEST {0}".format(testID)) + logging.info("*") # ///////////////////////////////////////////////////////////////////////////// @@ -359,17 +386,14 @@ def helper__makereport__call( def pytest_runtest_makereport(item: pytest.Function, call: pytest.CallInfo): assert item is not None assert call is not None - assert type(item) == pytest.Function # noqa: E721 + # it may be pytest.Function or _pytest.unittest.TestCaseFunction + assert isinstance(item, pytest.Function) assert type(call) == pytest.CallInfo # noqa: E721 - # logging.info("[pytest_runtest_makereport][#001][{0}][{1}]".format(item.name, call.when)) - outcome: pluggy.Result = yield assert outcome is not None assert type(outcome) == pluggy.Result # noqa: E721 - # logging.info("[pytest_runtest_makereport][#002][{0}][{1}]".format(item.name, call.when)) - rep: pytest.TestReport = outcome.get_result() assert rep is not None assert type(rep) == pytest.TestReport # noqa: E721 @@ -440,41 +464,61 @@ def run_after_tests(request: pytest.FixtureRequest): yield - logging.info("--------------------------- [FAILED TESTS]") - logging.info("") - - assert len(TEST_PROCESS_STATS.FailedTests) == TEST_PROCESS_STATS.cFailedTests - - if len(TEST_PROCESS_STATS.FailedTests) > 0: - helper__print_test_list(TEST_PROCESS_STATS.FailedTests) - logging.info("") + C_LINE1 = "---------------------------" - logging.info("--------------------------- [XFAILED TESTS]") - logging.info("") + def LOCAL__print_line1_with_header(header: str): + assert type(C_LINE1) == str # noqa: E721 + assert type(header) == str # noqa: E721 + assert header != "" + logging.info(C_LINE1 + " [" + header + "]") - assert len(TEST_PROCESS_STATS.XFailedTests) == TEST_PROCESS_STATS.cXFailedTests + def LOCAL__print_test_list(header: str, test_count: int, test_list: list[str]): + assert type(header) == str # noqa: E721 + assert type(test_count) == int # noqa: E721 + assert type(test_list) == list # noqa: E721 + assert header != "" + assert test_count >= 0 + assert len(test_list) == test_count - if len(TEST_PROCESS_STATS.XFailedTests) > 0: - helper__print_test_list(TEST_PROCESS_STATS.XFailedTests) + LOCAL__print_line1_with_header(header) logging.info("") + if len(test_list) > 0: + helper__print_test_list(test_list) + logging.info("") + + # fmt: off + LOCAL__print_test_list( + "ACHTUNG TESTS", + TEST_PROCESS_STATS.cAchtungTests, + TEST_PROCESS_STATS.AchtungTests, + ) - logging.info("--------------------------- [NOT XFAILED TESTS]") - logging.info("") + LOCAL__print_test_list( + "FAILED TESTS", + TEST_PROCESS_STATS.cFailedTests, + TEST_PROCESS_STATS.FailedTests + ) - assert ( - len(TEST_PROCESS_STATS.NotXFailedTests) == TEST_PROCESS_STATS.cNotXFailedTests + LOCAL__print_test_list( + "XFAILED TESTS", + TEST_PROCESS_STATS.cXFailedTests, + TEST_PROCESS_STATS.XFailedTests, ) - if len(TEST_PROCESS_STATS.NotXFailedTests) > 0: - helper__print_test_list(TEST_PROCESS_STATS.NotXFailedTests) - logging.info("") + LOCAL__print_test_list( + "NOT XFAILED TESTS", + TEST_PROCESS_STATS.cNotXFailedTests, + TEST_PROCESS_STATS.NotXFailedTests, + ) + # fmt: on - logging.info("--------------------------- [SUMMARY STATISTICS]") + LOCAL__print_line1_with_header("SUMMARY STATISTICS") logging.info("") logging.info("[TESTS]") logging.info(" TOTAL : {0}".format(TEST_PROCESS_STATS.cTotalTests)) logging.info(" EXECUTED : {0}".format(TEST_PROCESS_STATS.cExecutedTests)) logging.info(" NOT EXECUTED: {0}".format(TEST_PROCESS_STATS.cNotExecutedTests)) + logging.info(" ACHTUNG : {0}".format(TEST_PROCESS_STATS.cAchtungTests)) logging.info("") logging.info(" PASSED : {0}".format(TEST_PROCESS_STATS.cPassedTests)) logging.info(" FAILED : {0}".format(TEST_PROCESS_STATS.cFailedTests)) @@ -503,7 +547,13 @@ def pytest_configure(config: pytest.Config) -> None: log_path.mkdir(exist_ok=True) - logging_plugin = config.pluginmanager.get_plugin("logging-plugin") + logging_plugin: _pytest.logging.LoggingPlugin = config.pluginmanager.get_plugin( + "logging-plugin" + ) + + assert logging_plugin is not None + assert isinstance(logging_plugin, _pytest.logging.LoggingPlugin) + logging_plugin.set_log_path(str(log_path / log_name)) From a5d6df452371206da124309ef7cddaf44d023299 Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Fri, 28 Mar 2025 23:36:42 +0300 Subject: [PATCH 434/475] [BUG FIX] PostgresNode must use get_pg_version2 (#227) --- testgres/node.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index 6d2417c4..2b5fc6d1 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -92,7 +92,7 @@ PgVer, \ eprint, \ get_bin_path2, \ - get_pg_version, \ + get_pg_version2, \ execute_utility2, \ options_string, \ clean_on_error @@ -148,16 +148,6 @@ def __init__(self, name=None, base_dir=None, port=None, conn_params: ConnectionP """ # private - self._pg_version = PgVer(get_pg_version(bin_dir)) - self._should_free_port = port is None - self._base_dir = base_dir - self._bin_dir = bin_dir - self._prefix = prefix - self._logger = None - self._master = None - - # basic - self.name = name or generate_app_name() if os_ops is None: os_ops = __class__._get_os_ops(conn_params) else: @@ -168,6 +158,17 @@ def __init__(self, name=None, base_dir=None, port=None, conn_params: ConnectionP assert isinstance(os_ops, OsOperations) self._os_ops = os_ops + self._pg_version = PgVer(get_pg_version2(os_ops, bin_dir)) + self._should_free_port = port is None + self._base_dir = base_dir + self._bin_dir = bin_dir + self._prefix = prefix + self._logger = None + self._master = None + + # basic + self.name = name or generate_app_name() + self.host = os_ops.host self.port = port or utils.reserve_port() From 56ae1a8ceb31a34cbf382d7884411389e293d83c Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Fri, 28 Mar 2025 23:38:48 +0300 Subject: [PATCH 435/475] get_pg_version2 is updated --- testgres/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testgres/utils.py b/testgres/utils.py index a988effe..62e95ff6 100644 --- a/testgres/utils.py +++ b/testgres/utils.py @@ -212,8 +212,8 @@ def get_pg_version2(os_ops: OsOperations, bin_dir=None): # Get raw version (e.g., postgres (PostgreSQL) 9.5.7) postgres_path = os.path.join(bin_dir, 'postgres') if bin_dir else get_bin_path2(os_ops, 'postgres') - _params = [postgres_path, '--version'] - raw_ver = os_ops.exec_command(_params, encoding='utf-8') + cmd = [postgres_path, '--version'] + raw_ver = os_ops.exec_command(cmd, encoding='utf-8') return parse_pg_version(raw_ver) From ee441caa91ebd96f0a1ac8244da36269a0d70d98 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Fri, 28 Mar 2025 23:39:40 +0300 Subject: [PATCH 436/475] get_pg_config2 is updated --- testgres/utils.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/testgres/utils.py b/testgres/utils.py index 62e95ff6..92383571 100644 --- a/testgres/utils.py +++ b/testgres/utils.py @@ -185,11 +185,15 @@ def cache_pg_config_data(cmd): return _pg_config_data # try specified pg_config path or PG_CONFIG + if pg_config_path: + return cache_pg_config_data(pg_config_path) + if isinstance(os_ops, RemoteOperations): - pg_config = pg_config_path or os.environ.get("PG_CONFIG_REMOTE") or os.environ.get("PG_CONFIG") + pg_config = os.environ.get("PG_CONFIG_REMOTE") or os.environ.get("PG_CONFIG") else: # try PG_CONFIG - get from local machine - pg_config = pg_config_path or os.environ.get("PG_CONFIG") + pg_config = os.environ.get("PG_CONFIG") + if pg_config: return cache_pg_config_data(pg_config) From 939ca6dfd29d6c6680ffd4f6441fbcae94da66f4 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Fri, 28 Mar 2025 23:40:33 +0300 Subject: [PATCH 437/475] TestTestgresCommon::test_get_pg_config2 is added --- tests/test_testgres_common.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/test_testgres_common.py b/tests/test_testgres_common.py index f42964c8..2440b8f0 100644 --- a/tests/test_testgres_common.py +++ b/tests/test_testgres_common.py @@ -5,6 +5,7 @@ from ..testgres.node import PgVer from ..testgres.node import PostgresNode from ..testgres.utils import get_pg_version2 +from ..testgres.utils import get_pg_config2 from ..testgres.utils import file_tail from ..testgres.utils import get_bin_path2 from ..testgres import ProcessType @@ -1064,6 +1065,31 @@ def test_dump(self, os_ops: OsOperations): res = node3.execute(query_select) assert (res == [(1, ), (2, )]) + def test_get_pg_config2(self, os_ops: OsOperations): + # check same instances + a = get_pg_config2(os_ops, None) + b = get_pg_config2(os_ops, None) + assert (id(a) == id(b)) + + # save right before config change + c1 = get_pg_config2(os_ops, None) + + # modify setting for this scope + with scoped_config(cache_pg_config=False) as config: + # sanity check for value + assert not (config.cache_pg_config) + + # save right after config change + c2 = get_pg_config2(os_ops, None) + + # check different instances after config change + assert (id(c1) != id(c2)) + + # check different instances + a = get_pg_config2(os_ops, None) + b = get_pg_config2(os_ops, None) + assert (id(a) != id(b)) + @staticmethod def helper__get_node(os_ops: OsOperations, name=None): assert isinstance(os_ops, OsOperations) From ca545891c7961b11b481f7befdc6e918289f454a Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Mon, 31 Mar 2025 13:45:52 +0300 Subject: [PATCH 438/475] pytest_runtest_makereport is updated (cleanup) (#228) Local 'rep' is not used. --- tests/conftest.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 0f65838e..e27eaeb3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -394,10 +394,6 @@ def pytest_runtest_makereport(item: pytest.Function, call: pytest.CallInfo): assert outcome is not None assert type(outcome) == pluggy.Result # noqa: E721 - rep: pytest.TestReport = outcome.get_result() - assert rep is not None - assert type(rep) == pytest.TestReport # noqa: E721 - if call.when == "collect": return From 712de460eb9a49998b88f32f14c096d77705dfd4 Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Mon, 31 Mar 2025 14:42:48 +0300 Subject: [PATCH 439/475] helper__build_test_id is added (conftest refactoring) (#229) --- tests/conftest.py | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index e27eaeb3..c6306454 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -199,6 +199,22 @@ def timedelta_to_human_text(delta: datetime.timedelta) -> str: # ///////////////////////////////////////////////////////////////////////////// +def helper__build_test_id(item: pytest.Function) -> str: + assert item is not None + assert isinstance(item, pytest.Function) + + testID = "" + + if item.cls is not None: + testID = item.cls.__module__ + "." + item.cls.__name__ + "::" + + testID = testID + item.name + + return testID + +# ///////////////////////////////////////////////////////////////////////////// + + def helper__makereport__setup( item: pytest.Function, call: pytest.CallInfo, outcome: pluggy.Result ): @@ -224,12 +240,7 @@ def helper__makereport__setup( TEST_PROCESS_STATS.incrementNotExecutedTestCount() return - testID = "" - - if item.cls is not None: - testID = item.cls.__module__ + "." + item.cls.__name__ + "::" - - testID = testID + item.name + testID = helper__build_test_id(item) if rep.outcome == "passed": testNumber = TEST_PROCESS_STATS.incrementExecutedTestCount() @@ -279,12 +290,7 @@ def helper__makereport__call( assert type(rep) == pytest.TestReport # noqa: E721 # -------- - testID = "" - - if item.cls is not None: - testID = item.cls.__module__ + "." + item.cls.__name__ + "::" - - testID = testID + item.name + testID = helper__build_test_id(item) # -------- assert call.start <= call.stop From cf19df91f33e44a7a130e04bca8b7d8cc1af155c Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Mon, 31 Mar 2025 14:50:58 +0300 Subject: [PATCH 440/475] pytest_runtest_makereport is updated (refactoring+documentation) --- tests/conftest.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index c6306454..ae528536 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -390,6 +390,12 @@ def helper__makereport__call( @pytest.hookimpl(hookwrapper=True) def pytest_runtest_makereport(item: pytest.Function, call: pytest.CallInfo): + # + # https://docs.pytest.org/en/7.1.x/how-to/writing_hook_functions.html#hookwrapper-executing-around-other-hooks + # + # Note that hook wrappers don’t return results themselves, + # they merely perform tracing or other side effects around the actual hook implementations. + # assert item is not None assert call is not None # it may be pytest.Function or _pytest.unittest.TestCaseFunction @@ -400,6 +406,8 @@ def pytest_runtest_makereport(item: pytest.Function, call: pytest.CallInfo): assert outcome is not None assert type(outcome) == pluggy.Result # noqa: E721 + assert type(call.when) == str + if call.when == "collect": return @@ -414,7 +422,9 @@ def pytest_runtest_makereport(item: pytest.Function, call: pytest.CallInfo): if call.when == "teardown": return - assert False + errMsg = "[pytest_runtest_makereport] unknown 'call.when' value: [{0}].".format(call.when) + + raise RuntimeError(errMsg) # ///////////////////////////////////////////////////////////////////////////// From d9db881901a202e76651c0940efd91e372fd56c8 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Mon, 31 Mar 2025 17:31:38 +0300 Subject: [PATCH 441/475] [FIX] Formatting [flake8] --- tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index ae528536..196dbf39 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -406,7 +406,7 @@ def pytest_runtest_makereport(item: pytest.Function, call: pytest.CallInfo): assert outcome is not None assert type(outcome) == pluggy.Result # noqa: E721 - assert type(call.when) == str + assert type(call.when) == str # noqa: E721 if call.when == "collect": return From 4502b86c3e0ffa4178411951074878cacbd42ca2 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Mon, 31 Mar 2025 17:34:59 +0300 Subject: [PATCH 442/475] helper__makereport__call is updated [revision] --- tests/conftest.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 196dbf39..ee03d1c3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -316,14 +316,14 @@ def helper__makereport__call( exitStatus = "SKIPPED" reasonText = str(call.excinfo.value) - reasonMsg = "SKIP REASON: {0}" + reasonMsgTempl = "SKIP REASON: {0}" elif type(call.excinfo.value) == _pytest.outcomes.XFailed: # noqa: E721 TEST_PROCESS_STATS.incrementXFailedTestCount(testID) exitStatus = "XFAILED" reasonText = str(call.excinfo.value) - reasonMsg = "XFAIL REASON: {0}" + reasonMsgTempl = "XFAIL REASON: {0}" else: exitStatus = "XFAILED" assert hasattr(rep, "wasxfail") @@ -333,13 +333,16 @@ def helper__makereport__call( TEST_PROCESS_STATS.incrementXFailedTestCount(testID) reasonText = rep.wasxfail - reasonMsg = "XFAIL REASON: {0}" + reasonMsgTempl = "XFAIL REASON: {0}" logging.error(call.excinfo.value) + assert type(reasonText) == str # noqa: E721 + if reasonText != "": + assert type(reasonMsgTempl) == str # noqa: E721 logging.info("*") - logging.info("* " + reasonMsg.format(reasonText)) + logging.info("* " + reasonMsgTempl.format(reasonText)) elif rep.outcome == "failed": assert call.excinfo is not None From 2090fbce9f6095da51364e03ab21763171ed650b Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Mon, 31 Mar 2025 23:02:12 +0300 Subject: [PATCH 443/475] conftest is updated [formatting+comments] --- tests/conftest.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index ee03d1c3..9e8ea368 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -399,6 +399,8 @@ def pytest_runtest_makereport(item: pytest.Function, call: pytest.CallInfo): # Note that hook wrappers don’t return results themselves, # they merely perform tracing or other side effects around the actual hook implementations. # + # https://docs.pytest.org/en/7.1.x/reference/reference.html#test-running-runtest-hooks + # assert item is not None assert call is not None # it may be pytest.Function or _pytest.unittest.TestCaseFunction @@ -425,7 +427,9 @@ def pytest_runtest_makereport(item: pytest.Function, call: pytest.CallInfo): if call.when == "teardown": return - errMsg = "[pytest_runtest_makereport] unknown 'call.when' value: [{0}].".format(call.when) + errMsg = "[pytest_runtest_makereport] unknown 'call.when' value: [{0}].".format( + call.when + ) raise RuntimeError(errMsg) From b91714142fb77d1af7450bfa58abf2cecf529c4d Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 1 Apr 2025 09:01:54 +0300 Subject: [PATCH 444/475] PostgresNode::start is updated [no logging.error] It does not uses logging.error when it can't reallocate port number. It throws exception only. Why? Our new test infrastructure will process logging.error and will increment an error counter. As result - some tests start failing. --- testgres/node.py | 3 +-- tests/test_simple.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index 2b5fc6d1..c8ae4204 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -1038,8 +1038,7 @@ def LOCAL__raise_cannot_start_node__std(from_exception): assert nAttempt > 0 assert nAttempt <= __class__._C_MAX_START_ATEMPTS if nAttempt == __class__._C_MAX_START_ATEMPTS: - logging.error("Reached maximum retry attempts. Unable to start node.") - LOCAL__raise_cannot_start_node(e, "Cannot start node after multiple attempts") + LOCAL__raise_cannot_start_node(e, "Cannot start node after multiple attempts.") log_files1 = self._collect_log_files() if not self._detect_port_conflict(log_files0, log_files1): diff --git a/tests/test_simple.py b/tests/test_simple.py index f648e558..6ca52cb0 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -450,7 +450,7 @@ def test_port_conflict(self): with pytest.raises( expected_exception=StartNodeException, - match=re.escape("Cannot start node after multiple attempts") + match=re.escape("Cannot start node after multiple attempts.") ): node2.init().start() From a9137dfeabfd0ac14b177811fef5c73296803734 Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Tue, 1 Apr 2025 16:15:30 +0300 Subject: [PATCH 445/475] [conftest] Advanced processing of logging (#230) * [conftest] Advanced processing of logging This patch does the following things: - it processes the calls of logging.error as test errors - it prints the number of errors/warnings for each test - it prints the total stats of errors/warnings/duration --- tests/conftest.py | 462 +++++++++++++++++++++++++++++++--- tests/test_conftest.py--devel | 80 ++++++ 2 files changed, 507 insertions(+), 35 deletions(-) create mode 100644 tests/test_conftest.py--devel diff --git a/tests/conftest.py b/tests/conftest.py index 9e8ea368..ff3b3cb4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,6 +8,7 @@ import pathlib import math import datetime +import typing import _pytest.outcomes import _pytest.unittest @@ -105,71 +106,169 @@ class TEST_PROCESS_STATS: cXFailedTests: int = 0 cSkippedTests: int = 0 cNotXFailedTests: int = 0 + cWarningTests: int = 0 cUnexpectedTests: int = 0 cAchtungTests: int = 0 - FailedTests = list[str]() - XFailedTests = list[str]() + FailedTests = list[str, int]() + XFailedTests = list[str, int]() NotXFailedTests = list[str]() + WarningTests = list[str, int]() AchtungTests = list[str]() + cTotalDuration: datetime.timedelta = datetime.timedelta() + + cTotalErrors: int = 0 + cTotalWarnings: int = 0 + # -------------------------------------------------------------------- def incrementTotalTestCount() -> None: + assert type(__class__.cTotalTests) == int # noqa: E721 + assert __class__.cTotalTests >= 0 + __class__.cTotalTests += 1 + assert __class__.cTotalTests > 0 + # -------------------------------------------------------------------- def incrementNotExecutedTestCount() -> None: + assert type(__class__.cNotExecutedTests) == int # noqa: E721 + assert __class__.cNotExecutedTests >= 0 + __class__.cNotExecutedTests += 1 + assert __class__.cNotExecutedTests > 0 + # -------------------------------------------------------------------- def incrementExecutedTestCount() -> int: + assert type(__class__.cExecutedTests) == int # noqa: E721 + assert __class__.cExecutedTests >= 0 + __class__.cExecutedTests += 1 + + assert __class__.cExecutedTests > 0 return __class__.cExecutedTests # -------------------------------------------------------------------- def incrementPassedTestCount() -> None: + assert type(__class__.cPassedTests) == int # noqa: E721 + assert __class__.cPassedTests >= 0 + __class__.cPassedTests += 1 + assert __class__.cPassedTests > 0 + # -------------------------------------------------------------------- - def incrementFailedTestCount(testID: str) -> None: + def incrementFailedTestCount(testID: str, errCount: int) -> None: assert type(testID) == str # noqa: E721 + assert type(errCount) == int # noqa: E721 + assert errCount > 0 assert type(__class__.FailedTests) == list # noqa: E721 + assert type(__class__.cFailedTests) == int # noqa: E721 + assert __class__.cFailedTests >= 0 - __class__.FailedTests.append(testID) # raise? + __class__.FailedTests.append((testID, errCount)) # raise? __class__.cFailedTests += 1 + assert len(__class__.FailedTests) > 0 + assert __class__.cFailedTests > 0 + assert len(__class__.FailedTests) == __class__.cFailedTests + + # -------- + assert type(__class__.cTotalErrors) == int # noqa: E721 + assert __class__.cTotalErrors >= 0 + + __class__.cTotalErrors += errCount + + assert __class__.cTotalErrors > 0 + # -------------------------------------------------------------------- - def incrementXFailedTestCount(testID: str) -> None: + def incrementXFailedTestCount(testID: str, errCount: int) -> None: assert type(testID) == str # noqa: E721 + assert type(errCount) == int # noqa: E721 + assert errCount >= 0 assert type(__class__.XFailedTests) == list # noqa: E721 + assert type(__class__.cXFailedTests) == int # noqa: E721 + assert __class__.cXFailedTests >= 0 - __class__.XFailedTests.append(testID) # raise? + __class__.XFailedTests.append((testID, errCount)) # raise? __class__.cXFailedTests += 1 + assert len(__class__.XFailedTests) > 0 + assert __class__.cXFailedTests > 0 + assert len(__class__.XFailedTests) == __class__.cXFailedTests + # -------------------------------------------------------------------- def incrementSkippedTestCount() -> None: + assert type(__class__.cSkippedTests) == int # noqa: E721 + assert __class__.cSkippedTests >= 0 + __class__.cSkippedTests += 1 + assert __class__.cSkippedTests > 0 + # -------------------------------------------------------------------- def incrementNotXFailedTests(testID: str) -> None: assert type(testID) == str # noqa: E721 assert type(__class__.NotXFailedTests) == list # noqa: E721 + assert type(__class__.cNotXFailedTests) == int # noqa: E721 + assert __class__.cNotXFailedTests >= 0 __class__.NotXFailedTests.append(testID) # raise? __class__.cNotXFailedTests += 1 + assert len(__class__.NotXFailedTests) > 0 + assert __class__.cNotXFailedTests > 0 + assert len(__class__.NotXFailedTests) == __class__.cNotXFailedTests + + # -------------------------------------------------------------------- + def incrementWarningTestCount(testID: str, warningCount: int) -> None: + assert type(testID) == str # noqa: E721 + assert type(warningCount) == int # noqa: E721 + assert testID != "" + assert warningCount > 0 + assert type(__class__.WarningTests) == list # noqa: E721 + assert type(__class__.cWarningTests) == int # noqa: E721 + assert __class__.cWarningTests >= 0 + + __class__.WarningTests.append((testID, warningCount)) # raise? + __class__.cWarningTests += 1 + + assert len(__class__.WarningTests) > 0 + assert __class__.cWarningTests > 0 + assert len(__class__.WarningTests) == __class__.cWarningTests + + # -------- + assert type(__class__.cTotalWarnings) == int # noqa: E721 + assert __class__.cTotalWarnings >= 0 + + __class__.cTotalWarnings += warningCount + + assert __class__.cTotalWarnings > 0 + # -------------------------------------------------------------------- def incrementUnexpectedTests() -> None: + assert type(__class__.cUnexpectedTests) == int # noqa: E721 + assert __class__.cUnexpectedTests >= 0 + __class__.cUnexpectedTests += 1 + assert __class__.cUnexpectedTests > 0 + # -------------------------------------------------------------------- def incrementAchtungTestCount(testID: str) -> None: assert type(testID) == str # noqa: E721 assert type(__class__.AchtungTests) == list # noqa: E721 + assert type(__class__.cAchtungTests) == int # noqa: E721 + assert __class__.cAchtungTests >= 0 __class__.AchtungTests.append(testID) # raise? __class__.cAchtungTests += 1 + assert len(__class__.AchtungTests) > 0 + assert __class__.cAchtungTests > 0 + assert len(__class__.AchtungTests) == __class__.cAchtungTests + # ///////////////////////////////////////////////////////////////////////////// @@ -212,6 +311,12 @@ def helper__build_test_id(item: pytest.Function) -> str: return testID + +# ///////////////////////////////////////////////////////////////////////////// + +g_error_msg_count_key = pytest.StashKey[int]() +g_warning_msg_count_key = pytest.StashKey[int]() + # ///////////////////////////////////////////////////////////////////////////// @@ -285,6 +390,16 @@ def helper__makereport__call( assert type(call) == pytest.CallInfo # noqa: E721 assert type(outcome) == pluggy.Result # noqa: E721 + # -------- + item_error_msg_count = item.stash.get(g_error_msg_count_key, 0) + assert type(item_error_msg_count) == int # noqa: E721 + assert item_error_msg_count >= 0 + + item_warning_msg_count = item.stash.get(g_warning_msg_count_key, 0) + assert type(item_warning_msg_count) == int # noqa: E721 + assert item_warning_msg_count >= 0 + + # -------- rep = outcome.get_result() assert rep is not None assert type(rep) == pytest.TestReport # noqa: E721 @@ -312,30 +427,35 @@ def helper__makereport__call( if type(call.excinfo.value) == _pytest.outcomes.Skipped: # noqa: E721 assert not hasattr(rep, "wasxfail") - TEST_PROCESS_STATS.incrementSkippedTestCount() - exitStatus = "SKIPPED" reasonText = str(call.excinfo.value) reasonMsgTempl = "SKIP REASON: {0}" - elif type(call.excinfo.value) == _pytest.outcomes.XFailed: # noqa: E721 - TEST_PROCESS_STATS.incrementXFailedTestCount(testID) + TEST_PROCESS_STATS.incrementSkippedTestCount() + elif type(call.excinfo.value) == _pytest.outcomes.XFailed: # noqa: E721 exitStatus = "XFAILED" reasonText = str(call.excinfo.value) reasonMsgTempl = "XFAIL REASON: {0}" + + TEST_PROCESS_STATS.incrementXFailedTestCount(testID, item_error_msg_count) + else: exitStatus = "XFAILED" assert hasattr(rep, "wasxfail") assert rep.wasxfail is not None assert type(rep.wasxfail) == str # noqa: E721 - TEST_PROCESS_STATS.incrementXFailedTestCount(testID) - reasonText = rep.wasxfail reasonMsgTempl = "XFAIL REASON: {0}" - logging.error(call.excinfo.value) + if type(call.excinfo.value) == SIGNAL_EXCEPTION: # noqa: E721 + pass + else: + logging.error(call.excinfo.value) + item_error_msg_count += 1 + + TEST_PROCESS_STATS.incrementXFailedTestCount(testID, item_error_msg_count) assert type(reasonText) == str # noqa: E721 @@ -348,9 +468,16 @@ def helper__makereport__call( assert call.excinfo is not None assert call.excinfo.value is not None - TEST_PROCESS_STATS.incrementFailedTestCount(testID) + if type(call.excinfo.value) == SIGNAL_EXCEPTION: # noqa: E721 + assert item_error_msg_count > 0 + pass + else: + logging.error(call.excinfo.value) + item_error_msg_count += 1 + + assert item_error_msg_count > 0 + TEST_PROCESS_STATS.incrementFailedTestCount(testID, item_error_msg_count) - logging.error(call.excinfo.value) exitStatus = "FAILED" elif rep.outcome == "passed": assert call.excinfo is None @@ -360,12 +487,12 @@ def helper__makereport__call( TEST_PROCESS_STATS.incrementNotXFailedTests(testID) - warnMsg = "Test is marked as xfail" + warnMsg = "NOTE: Test is marked as xfail" if rep.wasxfail != "": warnMsg += " [" + rep.wasxfail + "]" - logging.warning(warnMsg) + logging.info(warnMsg) exitStatus = "NOT XFAILED" else: assert not hasattr(rep, "wasxfail") @@ -378,11 +505,25 @@ def helper__makereport__call( # [2025-03-28] It may create a useless problem in new environment. # assert False + # -------- + if item_warning_msg_count > 0: + TEST_PROCESS_STATS.incrementWarningTestCount(testID, item_warning_msg_count) + + # -------- + assert type(TEST_PROCESS_STATS.cTotalDuration) == datetime.timedelta # noqa: E721 + assert type(testDurration) == datetime.timedelta # noqa: E721 + + TEST_PROCESS_STATS.cTotalDuration += testDurration + + assert testDurration <= TEST_PROCESS_STATS.cTotalDuration + # -------- logging.info("*") - logging.info("* DURATION : {0}".format(timedelta_to_human_text(testDurration))) + logging.info("* DURATION : {0}".format(timedelta_to_human_text(testDurration))) logging.info("*") - logging.info("* EXIT STATUS : {0}".format(exitStatus)) + logging.info("* EXIT STATUS : {0}".format(exitStatus)) + logging.info("* ERROR COUNT : {0}".format(item_error_msg_count)) + logging.info("* WARNING COUNT: {0}".format(item_warning_msg_count)) logging.info("*") logging.info("* STOP TEST {0}".format(testID)) logging.info("*") @@ -437,6 +578,186 @@ def pytest_runtest_makereport(item: pytest.Function, call: pytest.CallInfo): # ///////////////////////////////////////////////////////////////////////////// +class LogErrorWrapper2: + _old_method: any + _counter: typing.Optional[int] + + # -------------------------------------------------------------------- + def __init__(self): + self._old_method = None + self._counter = None + + # -------------------------------------------------------------------- + def __enter__(self): + assert self._old_method is None + assert self._counter is None + + self._old_method = logging.error + self._counter = 0 + + logging.error = self + return self + + # -------------------------------------------------------------------- + def __exit__(self, exc_type, exc_val, exc_tb): + assert self._old_method is not None + assert self._counter is not None + + assert logging.error is self + + logging.error = self._old_method + + self._old_method = None + self._counter = None + return False + + # -------------------------------------------------------------------- + def __call__(self, *args, **kwargs): + assert self._old_method is not None + assert self._counter is not None + + assert type(self._counter) == int # noqa: E721 + assert self._counter >= 0 + + r = self._old_method(*args, **kwargs) + + self._counter += 1 + assert self._counter > 0 + + return r + + +# ///////////////////////////////////////////////////////////////////////////// + + +class LogWarningWrapper2: + _old_method: any + _counter: typing.Optional[int] + + # -------------------------------------------------------------------- + def __init__(self): + self._old_method = None + self._counter = None + + # -------------------------------------------------------------------- + def __enter__(self): + assert self._old_method is None + assert self._counter is None + + self._old_method = logging.warning + self._counter = 0 + + logging.warning = self + return self + + # -------------------------------------------------------------------- + def __exit__(self, exc_type, exc_val, exc_tb): + assert self._old_method is not None + assert self._counter is not None + + assert logging.warning is self + + logging.warning = self._old_method + + self._old_method = None + self._counter = None + return False + + # -------------------------------------------------------------------- + def __call__(self, *args, **kwargs): + assert self._old_method is not None + assert self._counter is not None + + assert type(self._counter) == int # noqa: E721 + assert self._counter >= 0 + + r = self._old_method(*args, **kwargs) + + self._counter += 1 + assert self._counter > 0 + + return r + + +# ///////////////////////////////////////////////////////////////////////////// + + +class SIGNAL_EXCEPTION(Exception): + def __init__(self): + pass + + +# ///////////////////////////////////////////////////////////////////////////// + + +@pytest.hookimpl(hookwrapper=True) +def pytest_pyfunc_call(pyfuncitem: pytest.Function): + assert pyfuncitem is not None + assert isinstance(pyfuncitem, pytest.Function) + + debug__log_error_method = logging.error + assert debug__log_error_method is not None + + debug__log_warning_method = logging.warning + assert debug__log_warning_method is not None + + pyfuncitem.stash[g_error_msg_count_key] = 0 + pyfuncitem.stash[g_warning_msg_count_key] = 0 + + try: + with LogErrorWrapper2() as logErrorWrapper, LogWarningWrapper2() as logWarningWrapper: + assert type(logErrorWrapper) == LogErrorWrapper2 # noqa: E721 + assert logErrorWrapper._old_method is not None + assert type(logErrorWrapper._counter) == int # noqa: E721 + assert logErrorWrapper._counter == 0 + assert logging.error is logErrorWrapper + + assert type(logWarningWrapper) == LogWarningWrapper2 # noqa: E721 + assert logWarningWrapper._old_method is not None + assert type(logWarningWrapper._counter) == int # noqa: E721 + assert logWarningWrapper._counter == 0 + assert logging.warning is logWarningWrapper + + r: pluggy.Result = yield + + assert r is not None + assert type(r) == pluggy.Result # noqa: E721 + + assert logErrorWrapper._old_method is not None + assert type(logErrorWrapper._counter) == int # noqa: E721 + assert logErrorWrapper._counter >= 0 + assert logging.error is logErrorWrapper + + assert logWarningWrapper._old_method is not None + assert type(logWarningWrapper._counter) == int # noqa: E721 + assert logWarningWrapper._counter >= 0 + assert logging.warning is logWarningWrapper + + assert g_error_msg_count_key in pyfuncitem.stash + assert g_warning_msg_count_key in pyfuncitem.stash + + assert pyfuncitem.stash[g_error_msg_count_key] == 0 + assert pyfuncitem.stash[g_warning_msg_count_key] == 0 + + pyfuncitem.stash[g_error_msg_count_key] = logErrorWrapper._counter + pyfuncitem.stash[g_warning_msg_count_key] = logWarningWrapper._counter + + if r.exception is not None: + pass + elif logErrorWrapper._counter == 0: + pass + else: + assert logErrorWrapper._counter > 0 + r.force_exception(SIGNAL_EXCEPTION()) + finally: + assert logging.error is debug__log_error_method + assert logging.warning is debug__log_warning_method + pass + + +# ///////////////////////////////////////////////////////////////////////////// + + def helper__calc_W(n: int) -> int: assert n > 0 @@ -467,11 +788,42 @@ def helper__print_test_list(tests: list[str]) -> None: nTest = 0 - while nTest < len(tests): - testID = tests[nTest] - assert type(testID) == str # noqa: E721 + for t in tests: + assert type(t) == str # noqa: E721 + assert t != "" nTest += 1 - logging.info(templateLine.format(nTest, testID)) + logging.info(templateLine.format(nTest, t)) + + +# ------------------------------------------------------------------------ +def helper__print_test_list2(tests: list[str, int]) -> None: + assert type(tests) == list # noqa: E721 + + assert helper__calc_W(9) == 1 + assert helper__calc_W(10) == 2 + assert helper__calc_W(11) == 2 + assert helper__calc_W(99) == 2 + assert helper__calc_W(100) == 3 + assert helper__calc_W(101) == 3 + assert helper__calc_W(999) == 3 + assert helper__calc_W(1000) == 4 + assert helper__calc_W(1001) == 4 + + W = helper__calc_W(len(tests)) + + templateLine = "{0:0" + str(W) + "d}. {1} ({2})" + + nTest = 0 + + for t in tests: + assert type(t) == tuple # noqa: E721 + assert len(t) == 2 + assert type(t[0]) == str # noqa: E721 + assert type(t[1]) == int # noqa: E721 + assert t[0] != "" + assert t[1] >= 0 + nTest += 1 + logging.info(templateLine.format(nTest, t[0], t[1])) # ///////////////////////////////////////////////////////////////////////////// @@ -505,6 +857,22 @@ def LOCAL__print_test_list(header: str, test_count: int, test_list: list[str]): helper__print_test_list(test_list) logging.info("") + def LOCAL__print_test_list2( + header: str, test_count: int, test_list: list[str, int] + ): + assert type(header) == str # noqa: E721 + assert type(test_count) == int # noqa: E721 + assert type(test_list) == list # noqa: E721 + assert header != "" + assert test_count >= 0 + assert len(test_list) == test_count + + LOCAL__print_line1_with_header(header) + logging.info("") + if len(test_list) > 0: + helper__print_test_list2(test_list) + logging.info("") + # fmt: off LOCAL__print_test_list( "ACHTUNG TESTS", @@ -512,13 +880,13 @@ def LOCAL__print_test_list(header: str, test_count: int, test_list: list[str]): TEST_PROCESS_STATS.AchtungTests, ) - LOCAL__print_test_list( + LOCAL__print_test_list2( "FAILED TESTS", TEST_PROCESS_STATS.cFailedTests, TEST_PROCESS_STATS.FailedTests ) - LOCAL__print_test_list( + LOCAL__print_test_list2( "XFAILED TESTS", TEST_PROCESS_STATS.cXFailedTests, TEST_PROCESS_STATS.XFailedTests, @@ -529,22 +897,46 @@ def LOCAL__print_test_list(header: str, test_count: int, test_list: list[str]): TEST_PROCESS_STATS.cNotXFailedTests, TEST_PROCESS_STATS.NotXFailedTests, ) + + LOCAL__print_test_list2( + "WARNING TESTS", + TEST_PROCESS_STATS.cWarningTests, + TEST_PROCESS_STATS.WarningTests, + ) # fmt: on LOCAL__print_line1_with_header("SUMMARY STATISTICS") logging.info("") logging.info("[TESTS]") - logging.info(" TOTAL : {0}".format(TEST_PROCESS_STATS.cTotalTests)) - logging.info(" EXECUTED : {0}".format(TEST_PROCESS_STATS.cExecutedTests)) - logging.info(" NOT EXECUTED: {0}".format(TEST_PROCESS_STATS.cNotExecutedTests)) - logging.info(" ACHTUNG : {0}".format(TEST_PROCESS_STATS.cAchtungTests)) + logging.info(" TOTAL : {0}".format(TEST_PROCESS_STATS.cTotalTests)) + logging.info(" EXECUTED : {0}".format(TEST_PROCESS_STATS.cExecutedTests)) + logging.info(" NOT EXECUTED : {0}".format(TEST_PROCESS_STATS.cNotExecutedTests)) + logging.info(" ACHTUNG : {0}".format(TEST_PROCESS_STATS.cAchtungTests)) + logging.info("") + logging.info(" PASSED : {0}".format(TEST_PROCESS_STATS.cPassedTests)) + logging.info(" FAILED : {0}".format(TEST_PROCESS_STATS.cFailedTests)) + logging.info(" XFAILED : {0}".format(TEST_PROCESS_STATS.cXFailedTests)) + logging.info(" NOT XFAILED : {0}".format(TEST_PROCESS_STATS.cNotXFailedTests)) + logging.info(" SKIPPED : {0}".format(TEST_PROCESS_STATS.cSkippedTests)) + logging.info(" WITH WARNINGS: {0}".format(TEST_PROCESS_STATS.cWarningTests)) + logging.info(" UNEXPECTED : {0}".format(TEST_PROCESS_STATS.cUnexpectedTests)) + logging.info("") + + assert type(TEST_PROCESS_STATS.cTotalDuration) == datetime.timedelta # noqa: E721 + + LOCAL__print_line1_with_header("TIME") + logging.info("") + logging.info( + " TOTAL DURATION: {0}".format( + timedelta_to_human_text(TEST_PROCESS_STATS.cTotalDuration) + ) + ) + logging.info("") + + LOCAL__print_line1_with_header("TOTAL INFORMATION") logging.info("") - logging.info(" PASSED : {0}".format(TEST_PROCESS_STATS.cPassedTests)) - logging.info(" FAILED : {0}".format(TEST_PROCESS_STATS.cFailedTests)) - logging.info(" XFAILED : {0}".format(TEST_PROCESS_STATS.cXFailedTests)) - logging.info(" NOT XFAILED : {0}".format(TEST_PROCESS_STATS.cNotXFailedTests)) - logging.info(" SKIPPED : {0}".format(TEST_PROCESS_STATS.cSkippedTests)) - logging.info(" UNEXPECTED : {0}".format(TEST_PROCESS_STATS.cUnexpectedTests)) + logging.info(" TOTAL ERROR COUNT : {0}".format(TEST_PROCESS_STATS.cTotalErrors)) + logging.info(" TOTAL WARNING COUNT: {0}".format(TEST_PROCESS_STATS.cTotalWarnings)) logging.info("") diff --git a/tests/test_conftest.py--devel b/tests/test_conftest.py--devel new file mode 100644 index 00000000..67c1dafe --- /dev/null +++ b/tests/test_conftest.py--devel @@ -0,0 +1,80 @@ +import pytest +import logging + + +class TestConfest: + def test_failed(self): + raise Exception("TEST EXCEPTION!") + + def test_ok(self): + pass + + @pytest.mark.skip() + def test_mark_skip__no_reason(self): + pass + + @pytest.mark.xfail() + def test_mark_xfail__no_reason(self): + raise Exception("XFAIL EXCEPTION") + + @pytest.mark.xfail() + def test_mark_xfail__no_reason___no_error(self): + pass + + @pytest.mark.skip(reason="reason") + def test_mark_skip__with_reason(self): + pass + + @pytest.mark.xfail(reason="reason") + def test_mark_xfail__with_reason(self): + raise Exception("XFAIL EXCEPTION") + + @pytest.mark.xfail(reason="reason") + def test_mark_xfail__with_reason___no_error(self): + pass + + def test_exc_skip__no_reason(self): + pytest.skip() + + def test_exc_xfail__no_reason(self): + pytest.xfail() + + def test_exc_skip__with_reason(self): + pytest.skip(reason="SKIP REASON") + + def test_exc_xfail__with_reason(self): + pytest.xfail(reason="XFAIL EXCEPTION") + + def test_log_error(self): + logging.error("IT IS A LOG ERROR!") + + def test_log_error_and_exc(self): + logging.error("IT IS A LOG ERROR!") + + raise Exception("TEST EXCEPTION!") + + def test_log_error_and_warning(self): + logging.error("IT IS A LOG ERROR!") + logging.warning("IT IS A LOG WARNING!") + logging.error("IT IS THE SECOND LOG ERROR!") + logging.warning("IT IS THE SECOND LOG WARNING!") + + @pytest.mark.xfail() + def test_log_error_and_xfail_mark_without_reason(self): + logging.error("IT IS A LOG ERROR!") + + @pytest.mark.xfail(reason="It is a reason message") + def test_log_error_and_xfail_mark_with_reason(self): + logging.error("IT IS A LOG ERROR!") + + @pytest.mark.xfail() + def test_two_log_error_and_xfail_mark_without_reason(self): + logging.error("IT IS THE FIRST LOG ERROR!") + logging.info("----------") + logging.error("IT IS THE SECOND LOG ERROR!") + + @pytest.mark.xfail(reason="It is a reason message") + def test_two_log_error_and_xfail_mark_with_reason(self): + logging.error("IT IS THE FIRST LOG ERROR!") + logging.info("----------") + logging.error("IT IS THE SECOND LOG ERROR!") From 5a19e0b4d677bee7264c4ce3d7a323fc9f1e5559 Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Wed, 2 Apr 2025 14:33:51 +0300 Subject: [PATCH 446/475] TestOsOpsCommon is added [generic os_ops tests] (#231) Plus: - [BUG FIX] LocalOperations::is_executable is corrected - [FIX] OsOperations::mkstemp is added --- testgres/operations/local_ops.py | 3 +- testgres/operations/os_ops.py | 3 + tests/test_local.py | 327 ---------------- tests/test_os_ops_common.py | 650 +++++++++++++++++++++++++++++++ tests/test_remote.py | 523 ------------------------- 5 files changed, 655 insertions(+), 851 deletions(-) create mode 100644 tests/test_os_ops_common.py diff --git a/testgres/operations/local_ops.py b/testgres/operations/local_ops.py index 93a64787..35e94210 100644 --- a/testgres/operations/local_ops.py +++ b/testgres/operations/local_ops.py @@ -156,7 +156,8 @@ def find_executable(self, executable): def is_executable(self, file): # Check if the file is executable - return os.stat(file).st_mode & stat.S_IXUSR + assert stat.S_IXUSR != 0 + return (os.stat(file).st_mode & stat.S_IXUSR) == stat.S_IXUSR def set_env(self, var_name, var_val): # Check if the directory is already in PATH diff --git a/testgres/operations/os_ops.py b/testgres/operations/os_ops.py index f20a7a30..3c606871 100644 --- a/testgres/operations/os_ops.py +++ b/testgres/operations/os_ops.py @@ -77,6 +77,9 @@ def pathsep(self): def mkdtemp(self, prefix=None): raise NotImplementedError() + def mkstemp(self, prefix=None): + raise NotImplementedError() + def copytree(self, src, dst): raise NotImplementedError() diff --git a/tests/test_local.py b/tests/test_local.py index 3ae93f76..7b5e488d 100644 --- a/tests/test_local.py +++ b/tests/test_local.py @@ -3,15 +3,9 @@ import pytest import re -import tempfile -import logging -from ..testgres import ExecUtilException -from ..testgres import InvalidOperationException from ..testgres import LocalOperations -from .helpers.run_conditions import RunConditions - class TestLocalOperations: @@ -19,138 +13,6 @@ class TestLocalOperations: def setup(self): self.operations = LocalOperations() - def test_mkdtemp__default(self): - path = self.operations.mkdtemp() - logging.info("Path is [{0}].".format(path)) - assert os.path.exists(path) - os.rmdir(path) - assert not os.path.exists(path) - - def test_mkdtemp__custom(self): - C_TEMPLATE = "abcdef" - path = self.operations.mkdtemp(C_TEMPLATE) - logging.info("Path is [{0}].".format(path)) - assert os.path.exists(path) - assert C_TEMPLATE in os.path.basename(path) - os.rmdir(path) - assert not os.path.exists(path) - - def test_exec_command_success(self): - """ - Test exec_command for successful command execution. - """ - RunConditions.skip_if_windows() - - cmd = "python3 --version" - response = self.operations.exec_command(cmd, wait_exit=True, shell=True) - - assert b'Python 3.' in response - - def test_exec_command_failure(self): - """ - Test exec_command for command execution failure. - """ - RunConditions.skip_if_windows() - - cmd = "nonexistent_command" - while True: - try: - self.operations.exec_command(cmd, wait_exit=True, shell=True) - except ExecUtilException as e: - assert type(e.exit_code) == int # noqa: E721 - assert e.exit_code == 127 - - assert type(e.message) == str # noqa: E721 - assert type(e.error) == bytes # noqa: E721 - - assert e.message.startswith("Utility exited with non-zero code (127). Error:") - assert "nonexistent_command" in e.message - assert "not found" in e.message - assert b"nonexistent_command" in e.error - assert b"not found" in e.error - break - raise Exception("We wait an exception!") - - def test_exec_command_failure__expect_error(self): - """ - Test exec_command for command execution failure. - """ - RunConditions.skip_if_windows() - - cmd = "nonexistent_command" - - exit_status, result, error = self.operations.exec_command(cmd, verbose=True, wait_exit=True, shell=True, expect_error=True) - - assert exit_status == 127 - assert result == b'' - assert type(error) == bytes # noqa: E721 - assert b"nonexistent_command" in error - assert b"not found" in error - - def test_listdir(self): - """ - Test listdir for listing directory contents. - """ - path = "/etc" - files = self.operations.listdir(path) - assert isinstance(files, list) - for f in files: - assert f is not None - assert type(f) == str # noqa: E721 - - def test_read__text(self): - """ - Test LocalOperations::read for text data. - """ - filename = __file__ # current file - - with open(filename, 'r') as file: # open in a text mode - response0 = file.read() - - assert type(response0) == str # noqa: E721 - - response1 = self.operations.read(filename) - assert type(response1) == str # noqa: E721 - assert response1 == response0 - - response2 = self.operations.read(filename, encoding=None, binary=False) - assert type(response2) == str # noqa: E721 - assert response2 == response0 - - response3 = self.operations.read(filename, encoding="") - assert type(response3) == str # noqa: E721 - assert response3 == response0 - - response4 = self.operations.read(filename, encoding="UTF-8") - assert type(response4) == str # noqa: E721 - assert response4 == response0 - - def test_read__binary(self): - """ - Test LocalOperations::read for binary data. - """ - filename = __file__ # current file - - with open(filename, 'rb') as file: # open in a binary mode - response0 = file.read() - - assert type(response0) == bytes # noqa: E721 - - response1 = self.operations.read(filename, binary=True) - assert type(response1) == bytes # noqa: E721 - assert response1 == response0 - - def test_read__binary_and_encoding(self): - """ - Test LocalOperations::read for binary data and encoding. - """ - filename = __file__ # current file - - with pytest.raises( - InvalidOperationException, - match=re.escape("Enconding is not allowed for read binary operation")): - self.operations.read(filename, encoding="", binary=True) - def test_read__unknown_file(self): """ Test LocalOperations::read with unknown file. @@ -159,40 +21,6 @@ def test_read__unknown_file(self): with pytest.raises(FileNotFoundError, match=re.escape("[Errno 2] No such file or directory: '/dummy'")): self.operations.read("/dummy") - def test_read_binary__spec(self): - """ - Test LocalOperations::read_binary. - """ - filename = __file__ # current file - - with open(filename, 'rb') as file: # open in a binary mode - response0 = file.read() - - assert type(response0) == bytes # noqa: E721 - - response1 = self.operations.read_binary(filename, 0) - assert type(response1) == bytes # noqa: E721 - assert response1 == response0 - - response2 = self.operations.read_binary(filename, 1) - assert type(response2) == bytes # noqa: E721 - assert len(response2) < len(response1) - assert len(response2) + 1 == len(response1) - assert response2 == response1[1:] - - response3 = self.operations.read_binary(filename, len(response1)) - assert type(response3) == bytes # noqa: E721 - assert len(response3) == 0 - - response4 = self.operations.read_binary(filename, len(response2)) - assert type(response4) == bytes # noqa: E721 - assert len(response4) == 1 - assert response4[0] == response1[len(response1) - 1] - - response5 = self.operations.read_binary(filename, len(response1) + 1) - assert type(response5) == bytes # noqa: E721 - assert len(response5) == 0 - def test_read_binary__spec__unk_file(self): """ Test LocalOperations::read_binary with unknown file. @@ -203,29 +31,6 @@ def test_read_binary__spec__unk_file(self): match=re.escape("[Errno 2] No such file or directory: '/dummy'")): self.operations.read_binary("/dummy", 0) - def test_read_binary__spec__negative_offset(self): - """ - Test LocalOperations::read_binary with negative offset. - """ - - with pytest.raises( - ValueError, - match=re.escape("Negative 'offset' is not supported.")): - self.operations.read_binary(__file__, -1) - - def test_get_file_size(self): - """ - Test LocalOperations::get_file_size. - """ - filename = __file__ # current file - - sz0 = os.path.getsize(filename) - assert type(sz0) == int # noqa: E721 - - sz1 = self.operations.get_file_size(filename) - assert type(sz1) == int # noqa: E721 - assert sz1 == sz0 - def test_get_file_size__unk_file(self): """ Test LocalOperations::get_file_size. @@ -234,70 +39,6 @@ def test_get_file_size__unk_file(self): with pytest.raises(FileNotFoundError, match=re.escape("[Errno 2] No such file or directory: '/dummy'")): self.operations.get_file_size("/dummy") - def test_isfile_true(self): - """ - Test isfile for an existing file. - """ - filename = __file__ - - response = self.operations.isfile(filename) - - assert response is True - - def test_isfile_false__not_exist(self): - """ - Test isfile for a non-existing file. - """ - filename = os.path.join(os.path.dirname(__file__), "nonexistent_file.txt") - - response = self.operations.isfile(filename) - - assert response is False - - def test_isfile_false__directory(self): - """ - Test isfile for a firectory. - """ - name = os.path.dirname(__file__) - - assert self.operations.isdir(name) - - response = self.operations.isfile(name) - - assert response is False - - def test_isdir_true(self): - """ - Test isdir for an existing directory. - """ - name = os.path.dirname(__file__) - - response = self.operations.isdir(name) - - assert response is True - - def test_isdir_false__not_exist(self): - """ - Test isdir for a non-existing directory. - """ - name = os.path.join(os.path.dirname(__file__), "it_is_nonexistent_directory") - - response = self.operations.isdir(name) - - assert response is False - - def test_isdir_false__file(self): - """ - Test isdir for a file. - """ - name = __file__ - - assert self.operations.isfile(name) - - response = self.operations.isdir(name) - - assert response is False - def test_cwd(self): """ Test cwd. @@ -314,71 +55,3 @@ def test_cwd(self): # Comp result assert v == expectedValue - - class tagWriteData001: - def __init__(self, sign, source, cp_rw, cp_truncate, cp_binary, cp_data, result): - self.sign = sign - self.source = source - self.call_param__rw = cp_rw - self.call_param__truncate = cp_truncate - self.call_param__binary = cp_binary - self.call_param__data = cp_data - self.result = result - - sm_write_data001 = [ - tagWriteData001("A001", "1234567890", False, False, False, "ABC", "1234567890ABC"), - tagWriteData001("A002", b"1234567890", False, False, True, b"ABC", b"1234567890ABC"), - - tagWriteData001("B001", "1234567890", False, True, False, "ABC", "ABC"), - tagWriteData001("B002", "1234567890", False, True, False, "ABC1234567890", "ABC1234567890"), - tagWriteData001("B003", b"1234567890", False, True, True, b"ABC", b"ABC"), - tagWriteData001("B004", b"1234567890", False, True, True, b"ABC1234567890", b"ABC1234567890"), - - tagWriteData001("C001", "1234567890", True, False, False, "ABC", "1234567890ABC"), - tagWriteData001("C002", b"1234567890", True, False, True, b"ABC", b"1234567890ABC"), - - tagWriteData001("D001", "1234567890", True, True, False, "ABC", "ABC"), - tagWriteData001("D002", "1234567890", True, True, False, "ABC1234567890", "ABC1234567890"), - tagWriteData001("D003", b"1234567890", True, True, True, b"ABC", b"ABC"), - tagWriteData001("D004", b"1234567890", True, True, True, b"ABC1234567890", b"ABC1234567890"), - - tagWriteData001("E001", "\0001234567890\000", False, False, False, "\000ABC\000", "\0001234567890\000\000ABC\000"), - tagWriteData001("E002", b"\0001234567890\000", False, False, True, b"\000ABC\000", b"\0001234567890\000\000ABC\000"), - - tagWriteData001("F001", "a\nb\n", False, False, False, ["c", "d"], "a\nb\nc\nd\n"), - tagWriteData001("F002", b"a\nb\n", False, False, True, [b"c", b"d"], b"a\nb\nc\nd\n"), - - tagWriteData001("G001", "a\nb\n", False, False, False, ["c\n\n", "d\n"], "a\nb\nc\nd\n"), - tagWriteData001("G002", b"a\nb\n", False, False, True, [b"c\n\n", b"d\n"], b"a\nb\nc\nd\n"), - ] - - @pytest.fixture( - params=sm_write_data001, - ids=[x.sign for x in sm_write_data001], - ) - def write_data001(self, request): - assert isinstance(request, pytest.FixtureRequest) - assert type(request.param) == __class__.tagWriteData001 # noqa: E721 - return request.param - - def test_write(self, write_data001): - assert type(write_data001) == __class__.tagWriteData001 # noqa: E721 - - mode = "w+b" if write_data001.call_param__binary else "w+" - - with tempfile.NamedTemporaryFile(mode=mode, delete=True) as tmp_file: - tmp_file.write(write_data001.source) - tmp_file.flush() - - self.operations.write( - tmp_file.name, - write_data001.call_param__data, - read_and_write=write_data001.call_param__rw, - truncate=write_data001.call_param__truncate, - binary=write_data001.call_param__binary) - - tmp_file.seek(0) - - s = tmp_file.read() - - assert s == write_data001.result diff --git a/tests/test_os_ops_common.py b/tests/test_os_ops_common.py new file mode 100644 index 00000000..c3944c3b --- /dev/null +++ b/tests/test_os_ops_common.py @@ -0,0 +1,650 @@ +# coding: utf-8 +from .helpers.os_ops_descrs import OsOpsDescr +from .helpers.os_ops_descrs import OsOpsDescrs +from .helpers.os_ops_descrs import OsOperations +from .helpers.run_conditions import RunConditions + +import os + +import pytest +import re +import tempfile +import logging + +from ..testgres import InvalidOperationException +from ..testgres import ExecUtilException + + +class TestOsOpsCommon: + sm_os_ops_descrs: list[OsOpsDescr] = [ + OsOpsDescrs.sm_local_os_ops_descr, + OsOpsDescrs.sm_remote_os_ops_descr + ] + + @pytest.fixture( + params=[descr.os_ops for descr in sm_os_ops_descrs], + ids=[descr.sign for descr in sm_os_ops_descrs] + ) + def os_ops(self, request: pytest.FixtureRequest) -> OsOperations: + assert isinstance(request, pytest.FixtureRequest) + assert isinstance(request.param, OsOperations) + return request.param + + def test_exec_command_success(self, os_ops: OsOperations): + """ + Test exec_command for successful command execution. + """ + assert isinstance(os_ops, OsOperations) + + RunConditions.skip_if_windows() + + cmd = ["sh", "-c", "python3 --version"] + + response = os_ops.exec_command(cmd) + + assert b'Python 3.' in response + + def test_exec_command_failure(self, os_ops: OsOperations): + """ + Test exec_command for command execution failure. + """ + assert isinstance(os_ops, OsOperations) + + RunConditions.skip_if_windows() + + cmd = ["sh", "-c", "nonexistent_command"] + + while True: + try: + os_ops.exec_command(cmd) + except ExecUtilException as e: + assert type(e.exit_code) == int # noqa: E721 + assert e.exit_code == 127 + + assert type(e.message) == str # noqa: E721 + assert type(e.error) == bytes # noqa: E721 + + assert e.message.startswith("Utility exited with non-zero code (127). Error:") + assert "nonexistent_command" in e.message + assert "not found" in e.message + assert b"nonexistent_command" in e.error + assert b"not found" in e.error + break + raise Exception("We wait an exception!") + + def test_exec_command_failure__expect_error(self, os_ops: OsOperations): + """ + Test exec_command for command execution failure. + """ + assert isinstance(os_ops, OsOperations) + + RunConditions.skip_if_windows() + + cmd = ["sh", "-c", "nonexistent_command"] + + exit_status, result, error = os_ops.exec_command(cmd, verbose=True, expect_error=True) + + assert exit_status == 127 + assert result == b'' + assert type(error) == bytes # noqa: E721 + assert b"nonexistent_command" in error + assert b"not found" in error + + def test_is_executable_true(self, os_ops: OsOperations): + """ + Test is_executable for an existing executable. + """ + assert isinstance(os_ops, OsOperations) + + RunConditions.skip_if_windows() + + response = os_ops.is_executable("/bin/sh") + + assert response is True + + def test_is_executable_false(self, os_ops: OsOperations): + """ + Test is_executable for a non-executable. + """ + assert isinstance(os_ops, OsOperations) + + response = os_ops.is_executable(__file__) + + assert response is False + + def test_makedirs_and_rmdirs_success(self, os_ops: OsOperations): + """ + Test makedirs and rmdirs for successful directory creation and removal. + """ + assert isinstance(os_ops, OsOperations) + + RunConditions.skip_if_windows() + + cmd = "pwd" + pwd = os_ops.exec_command(cmd, wait_exit=True, encoding='utf-8').strip() + + path = "{}/test_dir".format(pwd) + + # Test makedirs + os_ops.makedirs(path) + assert os.path.exists(path) + assert os_ops.path_exists(path) + + # Test rmdirs + os_ops.rmdirs(path) + assert not os.path.exists(path) + assert not os_ops.path_exists(path) + + def test_makedirs_failure(self, os_ops: OsOperations): + """ + Test makedirs for failure. + """ + # Try to create a directory in a read-only location + assert isinstance(os_ops, OsOperations) + + RunConditions.skip_if_windows() + + path = "/root/test_dir" + + # Test makedirs + with pytest.raises(Exception): + os_ops.makedirs(path) + + def test_listdir(self, os_ops: OsOperations): + """ + Test listdir for listing directory contents. + """ + assert isinstance(os_ops, OsOperations) + + RunConditions.skip_if_windows() + + path = "/etc" + files = os_ops.listdir(path) + assert isinstance(files, list) + for f in files: + assert f is not None + assert type(f) == str # noqa: E721 + + def test_path_exists_true__directory(self, os_ops: OsOperations): + """ + Test path_exists for an existing directory. + """ + assert isinstance(os_ops, OsOperations) + + RunConditions.skip_if_windows() + + assert os_ops.path_exists("/etc") is True + + def test_path_exists_true__file(self, os_ops: OsOperations): + """ + Test path_exists for an existing file. + """ + assert isinstance(os_ops, OsOperations) + + RunConditions.skip_if_windows() + + assert os_ops.path_exists(__file__) is True + + def test_path_exists_false__directory(self, os_ops: OsOperations): + """ + Test path_exists for a non-existing directory. + """ + assert isinstance(os_ops, OsOperations) + + RunConditions.skip_if_windows() + + assert os_ops.path_exists("/nonexistent_path") is False + + def test_path_exists_false__file(self, os_ops: OsOperations): + """ + Test path_exists for a non-existing file. + """ + assert isinstance(os_ops, OsOperations) + + RunConditions.skip_if_windows() + + assert os_ops.path_exists("/etc/nonexistent_path.txt") is False + + def test_mkdtemp__default(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + + path = os_ops.mkdtemp() + logging.info("Path is [{0}].".format(path)) + assert os.path.exists(path) + os.rmdir(path) + assert not os.path.exists(path) + + def test_mkdtemp__custom(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + + C_TEMPLATE = "abcdef" + path = os_ops.mkdtemp(C_TEMPLATE) + logging.info("Path is [{0}].".format(path)) + assert os.path.exists(path) + assert C_TEMPLATE in os.path.basename(path) + os.rmdir(path) + assert not os.path.exists(path) + + def test_rmdirs(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + + path = os_ops.mkdtemp() + assert os.path.exists(path) + + assert os_ops.rmdirs(path, ignore_errors=False) is True + assert not os.path.exists(path) + + def test_rmdirs__01_with_subfolder(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + + # folder with subfolder + path = os_ops.mkdtemp() + assert os.path.exists(path) + + dir1 = os.path.join(path, "dir1") + assert not os.path.exists(dir1) + + os_ops.makedirs(dir1) + assert os.path.exists(dir1) + + assert os_ops.rmdirs(path, ignore_errors=False) is True + assert not os.path.exists(path) + assert not os.path.exists(dir1) + + def test_rmdirs__02_with_file(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + + # folder with file + path = os_ops.mkdtemp() + assert os.path.exists(path) + + file1 = os.path.join(path, "file1.txt") + assert not os.path.exists(file1) + + os_ops.touch(file1) + assert os.path.exists(file1) + + assert os_ops.rmdirs(path, ignore_errors=False) is True + assert not os.path.exists(path) + assert not os.path.exists(file1) + + def test_rmdirs__03_with_subfolder_and_file(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + + # folder with subfolder and file + path = os_ops.mkdtemp() + assert os.path.exists(path) + + dir1 = os.path.join(path, "dir1") + assert not os.path.exists(dir1) + + os_ops.makedirs(dir1) + assert os.path.exists(dir1) + + file1 = os.path.join(dir1, "file1.txt") + assert not os.path.exists(file1) + + os_ops.touch(file1) + assert os.path.exists(file1) + + assert os_ops.rmdirs(path, ignore_errors=False) is True + assert not os.path.exists(path) + assert not os.path.exists(dir1) + assert not os.path.exists(file1) + + def test_write_text_file(self, os_ops: OsOperations): + """ + Test write for writing data to a text file. + """ + assert isinstance(os_ops, OsOperations) + + RunConditions.skip_if_windows() + + filename = os_ops.mkstemp() + data = "Hello, world!" + + os_ops.write(filename, data, truncate=True) + os_ops.write(filename, data) + + response = os_ops.read(filename) + + assert response == data + data + + os_ops.remove_file(filename) + + def test_write_binary_file(self, os_ops: OsOperations): + """ + Test write for writing data to a binary file. + """ + assert isinstance(os_ops, OsOperations) + + RunConditions.skip_if_windows() + + filename = "/tmp/test_file.bin" + data = b"\x00\x01\x02\x03" + + os_ops.write(filename, data, binary=True, truncate=True) + + response = os_ops.read(filename, binary=True) + + assert response == data + + def test_read_text_file(self, os_ops: OsOperations): + """ + Test read for reading data from a text file. + """ + assert isinstance(os_ops, OsOperations) + + RunConditions.skip_if_windows() + + filename = "/etc/hosts" + + response = os_ops.read(filename) + + assert isinstance(response, str) + + def test_read_binary_file(self, os_ops: OsOperations): + """ + Test read for reading data from a binary file. + """ + assert isinstance(os_ops, OsOperations) + + RunConditions.skip_if_windows() + + filename = "/usr/bin/python3" + + response = os_ops.read(filename, binary=True) + + assert isinstance(response, bytes) + + def test_read__text(self, os_ops: OsOperations): + """ + Test OsOperations::read for text data. + """ + assert isinstance(os_ops, OsOperations) + + filename = __file__ # current file + + with open(filename, 'r') as file: # open in a text mode + response0 = file.read() + + assert type(response0) == str # noqa: E721 + + response1 = os_ops.read(filename) + assert type(response1) == str # noqa: E721 + assert response1 == response0 + + response2 = os_ops.read(filename, encoding=None, binary=False) + assert type(response2) == str # noqa: E721 + assert response2 == response0 + + response3 = os_ops.read(filename, encoding="") + assert type(response3) == str # noqa: E721 + assert response3 == response0 + + response4 = os_ops.read(filename, encoding="UTF-8") + assert type(response4) == str # noqa: E721 + assert response4 == response0 + + def test_read__binary(self, os_ops: OsOperations): + """ + Test OsOperations::read for binary data. + """ + filename = __file__ # current file + + with open(filename, 'rb') as file: # open in a binary mode + response0 = file.read() + + assert type(response0) == bytes # noqa: E721 + + response1 = os_ops.read(filename, binary=True) + assert type(response1) == bytes # noqa: E721 + assert response1 == response0 + + def test_read__binary_and_encoding(self, os_ops: OsOperations): + """ + Test OsOperations::read for binary data and encoding. + """ + assert isinstance(os_ops, OsOperations) + + filename = __file__ # current file + + with pytest.raises( + InvalidOperationException, + match=re.escape("Enconding is not allowed for read binary operation")): + os_ops.read(filename, encoding="", binary=True) + + def test_read_binary__spec(self, os_ops: OsOperations): + """ + Test OsOperations::read_binary. + """ + assert isinstance(os_ops, OsOperations) + + filename = __file__ # currnt file + + with open(filename, 'rb') as file: # open in a binary mode + response0 = file.read() + + assert type(response0) == bytes # noqa: E721 + + response1 = os_ops.read_binary(filename, 0) + assert type(response1) == bytes # noqa: E721 + assert response1 == response0 + + response2 = os_ops.read_binary(filename, 1) + assert type(response2) == bytes # noqa: E721 + assert len(response2) < len(response1) + assert len(response2) + 1 == len(response1) + assert response2 == response1[1:] + + response3 = os_ops.read_binary(filename, len(response1)) + assert type(response3) == bytes # noqa: E721 + assert len(response3) == 0 + + response4 = os_ops.read_binary(filename, len(response2)) + assert type(response4) == bytes # noqa: E721 + assert len(response4) == 1 + assert response4[0] == response1[len(response1) - 1] + + response5 = os_ops.read_binary(filename, len(response1) + 1) + assert type(response5) == bytes # noqa: E721 + assert len(response5) == 0 + + def test_read_binary__spec__negative_offset(self, os_ops: OsOperations): + """ + Test OsOperations::read_binary with negative offset. + """ + assert isinstance(os_ops, OsOperations) + + with pytest.raises( + ValueError, + match=re.escape("Negative 'offset' is not supported.")): + os_ops.read_binary(__file__, -1) + + def test_get_file_size(self, os_ops: OsOperations): + """ + Test OsOperations::get_file_size. + """ + assert isinstance(os_ops, OsOperations) + + filename = __file__ # current file + + sz0 = os.path.getsize(filename) + assert type(sz0) == int # noqa: E721 + + sz1 = os_ops.get_file_size(filename) + assert type(sz1) == int # noqa: E721 + assert sz1 == sz0 + + def test_isfile_true(self, os_ops: OsOperations): + """ + Test isfile for an existing file. + """ + assert isinstance(os_ops, OsOperations) + + filename = __file__ + + response = os_ops.isfile(filename) + + assert response is True + + def test_isfile_false__not_exist(self, os_ops: OsOperations): + """ + Test isfile for a non-existing file. + """ + assert isinstance(os_ops, OsOperations) + + filename = os.path.join(os.path.dirname(__file__), "nonexistent_file.txt") + + response = os_ops.isfile(filename) + + assert response is False + + def test_isfile_false__directory(self, os_ops: OsOperations): + """ + Test isfile for a firectory. + """ + assert isinstance(os_ops, OsOperations) + + name = os.path.dirname(__file__) + + assert os_ops.isdir(name) + + response = os_ops.isfile(name) + + assert response is False + + def test_isdir_true(self, os_ops: OsOperations): + """ + Test isdir for an existing directory. + """ + assert isinstance(os_ops, OsOperations) + + name = os.path.dirname(__file__) + + response = os_ops.isdir(name) + + assert response is True + + def test_isdir_false__not_exist(self, os_ops: OsOperations): + """ + Test isdir for a non-existing directory. + """ + assert isinstance(os_ops, OsOperations) + + name = os.path.join(os.path.dirname(__file__), "it_is_nonexistent_directory") + + response = os_ops.isdir(name) + + assert response is False + + def test_isdir_false__file(self, os_ops: OsOperations): + """ + Test isdir for a file. + """ + assert isinstance(os_ops, OsOperations) + + name = __file__ + + assert os_ops.isfile(name) + + response = os_ops.isdir(name) + + assert response is False + + def test_cwd(self, os_ops: OsOperations): + """ + Test cwd. + """ + assert isinstance(os_ops, OsOperations) + + v = os_ops.cwd() + + assert v is not None + assert type(v) == str # noqa: E721 + assert v != "" + + class tagWriteData001: + def __init__(self, sign, source, cp_rw, cp_truncate, cp_binary, cp_data, result): + self.sign = sign + self.source = source + self.call_param__rw = cp_rw + self.call_param__truncate = cp_truncate + self.call_param__binary = cp_binary + self.call_param__data = cp_data + self.result = result + + sm_write_data001 = [ + tagWriteData001("A001", "1234567890", False, False, False, "ABC", "1234567890ABC"), + tagWriteData001("A002", b"1234567890", False, False, True, b"ABC", b"1234567890ABC"), + + tagWriteData001("B001", "1234567890", False, True, False, "ABC", "ABC"), + tagWriteData001("B002", "1234567890", False, True, False, "ABC1234567890", "ABC1234567890"), + tagWriteData001("B003", b"1234567890", False, True, True, b"ABC", b"ABC"), + tagWriteData001("B004", b"1234567890", False, True, True, b"ABC1234567890", b"ABC1234567890"), + + tagWriteData001("C001", "1234567890", True, False, False, "ABC", "1234567890ABC"), + tagWriteData001("C002", b"1234567890", True, False, True, b"ABC", b"1234567890ABC"), + + tagWriteData001("D001", "1234567890", True, True, False, "ABC", "ABC"), + tagWriteData001("D002", "1234567890", True, True, False, "ABC1234567890", "ABC1234567890"), + tagWriteData001("D003", b"1234567890", True, True, True, b"ABC", b"ABC"), + tagWriteData001("D004", b"1234567890", True, True, True, b"ABC1234567890", b"ABC1234567890"), + + tagWriteData001("E001", "\0001234567890\000", False, False, False, "\000ABC\000", "\0001234567890\000\000ABC\000"), + tagWriteData001("E002", b"\0001234567890\000", False, False, True, b"\000ABC\000", b"\0001234567890\000\000ABC\000"), + + tagWriteData001("F001", "a\nb\n", False, False, False, ["c", "d"], "a\nb\nc\nd\n"), + tagWriteData001("F002", b"a\nb\n", False, False, True, [b"c", b"d"], b"a\nb\nc\nd\n"), + + tagWriteData001("G001", "a\nb\n", False, False, False, ["c\n\n", "d\n"], "a\nb\nc\nd\n"), + tagWriteData001("G002", b"a\nb\n", False, False, True, [b"c\n\n", b"d\n"], b"a\nb\nc\nd\n"), + ] + + @pytest.fixture( + params=sm_write_data001, + ids=[x.sign for x in sm_write_data001], + ) + def write_data001(self, request): + assert isinstance(request, pytest.FixtureRequest) + assert type(request.param) == __class__.tagWriteData001 # noqa: E721 + return request.param + + def test_write(self, write_data001: tagWriteData001, os_ops: OsOperations): + assert type(write_data001) == __class__.tagWriteData001 # noqa: E721 + assert isinstance(os_ops, OsOperations) + + mode = "w+b" if write_data001.call_param__binary else "w+" + + with tempfile.NamedTemporaryFile(mode=mode, delete=True) as tmp_file: + tmp_file.write(write_data001.source) + tmp_file.flush() + + os_ops.write( + tmp_file.name, + write_data001.call_param__data, + read_and_write=write_data001.call_param__rw, + truncate=write_data001.call_param__truncate, + binary=write_data001.call_param__binary) + + tmp_file.seek(0) + + s = tmp_file.read() + + assert s == write_data001.result + + def test_touch(self, os_ops: OsOperations): + """ + Test touch for creating a new file or updating access and modification times of an existing file. + """ + assert isinstance(os_ops, OsOperations) + + filename = os_ops.mkstemp() + + # TODO: this test does not check the result of 'touch' command! + + os_ops.touch(filename) + + assert os_ops.isfile(filename) + + os_ops.remove_file(filename) diff --git a/tests/test_remote.py b/tests/test_remote.py index 2c37e2c1..565b2d20 100755 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -2,16 +2,10 @@ import os import pytest -import re -import tempfile -import logging from ..testgres import ExecUtilException -from ..testgres import InvalidOperationException from ..testgres import RemoteOperations -from ..testgres import LocalOperations from ..testgres import ConnectionParams -from ..testgres import utils as testgres_utils class TestRemoteOperations: @@ -23,179 +17,6 @@ def setup(self): ssh_key=os.getenv('RDBMS_TESTPOOL_SSHKEY')) self.operations = RemoteOperations(conn_params) - def test_exec_command_success(self): - """ - Test exec_command for successful command execution. - """ - cmd = "python3 --version" - response = self.operations.exec_command(cmd, wait_exit=True) - - assert b'Python 3.' in response - - def test_exec_command_failure(self): - """ - Test exec_command for command execution failure. - """ - cmd = "nonexistent_command" - while True: - try: - self.operations.exec_command(cmd, verbose=True, wait_exit=True) - except ExecUtilException as e: - assert type(e.exit_code) == int # noqa: E721 - assert e.exit_code == 127 - - assert type(e.message) == str # noqa: E721 - assert type(e.error) == bytes # noqa: E721 - - assert e.message.startswith("Utility exited with non-zero code (127). Error:") - assert "nonexistent_command" in e.message - assert "not found" in e.message - assert b"nonexistent_command" in e.error - assert b"not found" in e.error - break - raise Exception("We wait an exception!") - - def test_exec_command_failure__expect_error(self): - """ - Test exec_command for command execution failure. - """ - cmd = "nonexistent_command" - - exit_status, result, error = self.operations.exec_command(cmd, verbose=True, wait_exit=True, shell=True, expect_error=True) - - assert exit_status == 127 - assert result == b'' - assert type(error) == bytes # noqa: E721 - assert b"nonexistent_command" in error - assert b"not found" in error - - def test_is_executable_true(self): - """ - Test is_executable for an existing executable. - """ - local_ops = LocalOperations() - cmd = testgres_utils.get_bin_path2(local_ops, "pg_config") - cmd = local_ops.exec_command([cmd, "--bindir"], encoding="utf-8") - cmd = cmd.rstrip() - cmd = os.path.join(cmd, "pg_config") - response = self.operations.is_executable(cmd) - - assert response is True - - def test_is_executable_false(self): - """ - Test is_executable for a non-executable. - """ - cmd = "python" - response = self.operations.is_executable(cmd) - - assert response is False - - def test_makedirs_and_rmdirs_success(self): - """ - Test makedirs and rmdirs for successful directory creation and removal. - """ - cmd = "pwd" - pwd = self.operations.exec_command(cmd, wait_exit=True, encoding='utf-8').strip() - - path = "{}/test_dir".format(pwd) - - # Test makedirs - self.operations.makedirs(path) - assert os.path.exists(path) - assert self.operations.path_exists(path) - - # Test rmdirs - self.operations.rmdirs(path) - assert not os.path.exists(path) - assert not self.operations.path_exists(path) - - def test_makedirs_failure(self): - """ - Test makedirs for failure. - """ - # Try to create a directory in a read-only location - path = "/root/test_dir" - - # Test makedirs - with pytest.raises(Exception): - self.operations.makedirs(path) - - def test_mkdtemp__default(self): - path = self.operations.mkdtemp() - logging.info("Path is [{0}].".format(path)) - assert os.path.exists(path) - os.rmdir(path) - assert not os.path.exists(path) - - def test_mkdtemp__custom(self): - C_TEMPLATE = "abcdef" - path = self.operations.mkdtemp(C_TEMPLATE) - logging.info("Path is [{0}].".format(path)) - assert os.path.exists(path) - assert C_TEMPLATE in os.path.basename(path) - os.rmdir(path) - assert not os.path.exists(path) - - def test_rmdirs(self): - path = self.operations.mkdtemp() - assert os.path.exists(path) - - assert self.operations.rmdirs(path, ignore_errors=False) is True - assert not os.path.exists(path) - - def test_rmdirs__01_with_subfolder(self): - # folder with subfolder - path = self.operations.mkdtemp() - assert os.path.exists(path) - - dir1 = os.path.join(path, "dir1") - assert not os.path.exists(dir1) - - self.operations.makedirs(dir1) - assert os.path.exists(dir1) - - assert self.operations.rmdirs(path, ignore_errors=False) is True - assert not os.path.exists(path) - assert not os.path.exists(dir1) - - def test_rmdirs__02_with_file(self): - # folder with file - path = self.operations.mkdtemp() - assert os.path.exists(path) - - file1 = os.path.join(path, "file1.txt") - assert not os.path.exists(file1) - - self.operations.touch(file1) - assert os.path.exists(file1) - - assert self.operations.rmdirs(path, ignore_errors=False) is True - assert not os.path.exists(path) - assert not os.path.exists(file1) - - def test_rmdirs__03_with_subfolder_and_file(self): - # folder with subfolder and file - path = self.operations.mkdtemp() - assert os.path.exists(path) - - dir1 = os.path.join(path, "dir1") - assert not os.path.exists(dir1) - - self.operations.makedirs(dir1) - assert os.path.exists(dir1) - - file1 = os.path.join(dir1, "file1.txt") - assert not os.path.exists(file1) - - self.operations.touch(file1) - assert os.path.exists(file1) - - assert self.operations.rmdirs(path, ignore_errors=False) is True - assert not os.path.exists(path) - assert not os.path.exists(dir1) - assert not os.path.exists(file1) - def test_rmdirs__try_to_delete_nonexist_path(self): path = "/root/test_dir" @@ -216,141 +37,6 @@ def test_rmdirs__try_to_delete_file(self): assert type(x.value.exit_code) == int # noqa: E721 assert x.value.exit_code == 20 - def test_listdir(self): - """ - Test listdir for listing directory contents. - """ - path = "/etc" - files = self.operations.listdir(path) - assert isinstance(files, list) - for f in files: - assert f is not None - assert type(f) == str # noqa: E721 - - def test_path_exists_true__directory(self): - """ - Test path_exists for an existing directory. - """ - assert self.operations.path_exists("/etc") is True - - def test_path_exists_true__file(self): - """ - Test path_exists for an existing file. - """ - assert self.operations.path_exists(__file__) is True - - def test_path_exists_false__directory(self): - """ - Test path_exists for a non-existing directory. - """ - assert self.operations.path_exists("/nonexistent_path") is False - - def test_path_exists_false__file(self): - """ - Test path_exists for a non-existing file. - """ - assert self.operations.path_exists("/etc/nonexistent_path.txt") is False - - def test_write_text_file(self): - """ - Test write for writing data to a text file. - """ - filename = "/tmp/test_file.txt" - data = "Hello, world!" - - self.operations.write(filename, data, truncate=True) - self.operations.write(filename, data) - - response = self.operations.read(filename) - - assert response == data + data - - def test_write_binary_file(self): - """ - Test write for writing data to a binary file. - """ - filename = "/tmp/test_file.bin" - data = b"\x00\x01\x02\x03" - - self.operations.write(filename, data, binary=True, truncate=True) - - response = self.operations.read(filename, binary=True) - - assert response == data - - def test_read_text_file(self): - """ - Test read for reading data from a text file. - """ - filename = "/etc/hosts" - - response = self.operations.read(filename) - - assert isinstance(response, str) - - def test_read_binary_file(self): - """ - Test read for reading data from a binary file. - """ - filename = "/usr/bin/python3" - - response = self.operations.read(filename, binary=True) - - assert isinstance(response, bytes) - - def test_read__text(self): - """ - Test RemoteOperations::read for text data. - """ - filename = __file__ # current file - - with open(filename, 'r') as file: # open in a text mode - response0 = file.read() - - assert type(response0) == str # noqa: E721 - - response1 = self.operations.read(filename) - assert type(response1) == str # noqa: E721 - assert response1 == response0 - - response2 = self.operations.read(filename, encoding=None, binary=False) - assert type(response2) == str # noqa: E721 - assert response2 == response0 - - response3 = self.operations.read(filename, encoding="") - assert type(response3) == str # noqa: E721 - assert response3 == response0 - - response4 = self.operations.read(filename, encoding="UTF-8") - assert type(response4) == str # noqa: E721 - assert response4 == response0 - - def test_read__binary(self): - """ - Test RemoteOperations::read for binary data. - """ - filename = __file__ # current file - - with open(filename, 'rb') as file: # open in a binary mode - response0 = file.read() - - assert type(response0) == bytes # noqa: E721 - - response1 = self.operations.read(filename, binary=True) - assert type(response1) == bytes # noqa: E721 - assert response1 == response0 - - def test_read__binary_and_encoding(self): - """ - Test RemoteOperations::read for binary data and encoding. - """ - filename = __file__ # current file - - with pytest.raises( - InvalidOperationException, - match=re.escape("Enconding is not allowed for read binary operation")): - self.operations.read(filename, encoding="", binary=True) - def test_read__unknown_file(self): """ Test RemoteOperations::read with unknown file. @@ -363,40 +49,6 @@ def test_read__unknown_file(self): assert "No such file or directory" in str(x.value) assert "/dummy" in str(x.value) - def test_read_binary__spec(self): - """ - Test RemoteOperations::read_binary. - """ - filename = __file__ # currnt file - - with open(filename, 'rb') as file: # open in a binary mode - response0 = file.read() - - assert type(response0) == bytes # noqa: E721 - - response1 = self.operations.read_binary(filename, 0) - assert type(response1) == bytes # noqa: E721 - assert response1 == response0 - - response2 = self.operations.read_binary(filename, 1) - assert type(response2) == bytes # noqa: E721 - assert len(response2) < len(response1) - assert len(response2) + 1 == len(response1) - assert response2 == response1[1:] - - response3 = self.operations.read_binary(filename, len(response1)) - assert type(response3) == bytes # noqa: E721 - assert len(response3) == 0 - - response4 = self.operations.read_binary(filename, len(response2)) - assert type(response4) == bytes # noqa: E721 - assert len(response4) == 1 - assert response4[0] == response1[len(response1) - 1] - - response5 = self.operations.read_binary(filename, len(response1) + 1) - assert type(response5) == bytes # noqa: E721 - assert len(response5) == 0 - def test_read_binary__spec__unk_file(self): """ Test RemoteOperations::read_binary with unknown file. @@ -409,29 +61,6 @@ def test_read_binary__spec__unk_file(self): assert "No such file or directory" in str(x.value) assert "/dummy" in str(x.value) - def test_read_binary__spec__negative_offset(self): - """ - Test RemoteOperations::read_binary with negative offset. - """ - - with pytest.raises( - ValueError, - match=re.escape("Negative 'offset' is not supported.")): - self.operations.read_binary(__file__, -1) - - def test_get_file_size(self): - """ - Test RemoteOperations::get_file_size. - """ - filename = __file__ # current file - - sz0 = os.path.getsize(filename) - assert type(sz0) == int # noqa: E721 - - sz1 = self.operations.get_file_size(filename) - assert type(sz1) == int # noqa: E721 - assert sz1 == sz0 - def test_get_file_size__unk_file(self): """ Test RemoteOperations::get_file_size. @@ -443,155 +72,3 @@ def test_get_file_size__unk_file(self): assert "Utility exited with non-zero code (1)." in str(x.value) assert "No such file or directory" in str(x.value) assert "/dummy" in str(x.value) - - def test_touch(self): - """ - Test touch for creating a new file or updating access and modification times of an existing file. - """ - filename = "/tmp/test_file.txt" - - self.operations.touch(filename) - - assert self.operations.isfile(filename) - - def test_isfile_true(self): - """ - Test isfile for an existing file. - """ - filename = __file__ - - response = self.operations.isfile(filename) - - assert response is True - - def test_isfile_false__not_exist(self): - """ - Test isfile for a non-existing file. - """ - filename = os.path.join(os.path.dirname(__file__), "nonexistent_file.txt") - - response = self.operations.isfile(filename) - - assert response is False - - def test_isfile_false__directory(self): - """ - Test isfile for a firectory. - """ - name = os.path.dirname(__file__) - - assert self.operations.isdir(name) - - response = self.operations.isfile(name) - - assert response is False - - def test_isdir_true(self): - """ - Test isdir for an existing directory. - """ - name = os.path.dirname(__file__) - - response = self.operations.isdir(name) - - assert response is True - - def test_isdir_false__not_exist(self): - """ - Test isdir for a non-existing directory. - """ - name = os.path.join(os.path.dirname(__file__), "it_is_nonexistent_directory") - - response = self.operations.isdir(name) - - assert response is False - - def test_isdir_false__file(self): - """ - Test isdir for a file. - """ - name = __file__ - - assert self.operations.isfile(name) - - response = self.operations.isdir(name) - - assert response is False - - def test_cwd(self): - """ - Test cwd. - """ - v = self.operations.cwd() - - assert v is not None - assert type(v) == str # noqa: E721 - assert v != "" - - class tagWriteData001: - def __init__(self, sign, source, cp_rw, cp_truncate, cp_binary, cp_data, result): - self.sign = sign - self.source = source - self.call_param__rw = cp_rw - self.call_param__truncate = cp_truncate - self.call_param__binary = cp_binary - self.call_param__data = cp_data - self.result = result - - sm_write_data001 = [ - tagWriteData001("A001", "1234567890", False, False, False, "ABC", "1234567890ABC"), - tagWriteData001("A002", b"1234567890", False, False, True, b"ABC", b"1234567890ABC"), - - tagWriteData001("B001", "1234567890", False, True, False, "ABC", "ABC"), - tagWriteData001("B002", "1234567890", False, True, False, "ABC1234567890", "ABC1234567890"), - tagWriteData001("B003", b"1234567890", False, True, True, b"ABC", b"ABC"), - tagWriteData001("B004", b"1234567890", False, True, True, b"ABC1234567890", b"ABC1234567890"), - - tagWriteData001("C001", "1234567890", True, False, False, "ABC", "1234567890ABC"), - tagWriteData001("C002", b"1234567890", True, False, True, b"ABC", b"1234567890ABC"), - - tagWriteData001("D001", "1234567890", True, True, False, "ABC", "ABC"), - tagWriteData001("D002", "1234567890", True, True, False, "ABC1234567890", "ABC1234567890"), - tagWriteData001("D003", b"1234567890", True, True, True, b"ABC", b"ABC"), - tagWriteData001("D004", b"1234567890", True, True, True, b"ABC1234567890", b"ABC1234567890"), - - tagWriteData001("E001", "\0001234567890\000", False, False, False, "\000ABC\000", "\0001234567890\000\000ABC\000"), - tagWriteData001("E002", b"\0001234567890\000", False, False, True, b"\000ABC\000", b"\0001234567890\000\000ABC\000"), - - tagWriteData001("F001", "a\nb\n", False, False, False, ["c", "d"], "a\nb\nc\nd\n"), - tagWriteData001("F002", b"a\nb\n", False, False, True, [b"c", b"d"], b"a\nb\nc\nd\n"), - - tagWriteData001("G001", "a\nb\n", False, False, False, ["c\n\n", "d\n"], "a\nb\nc\nd\n"), - tagWriteData001("G002", b"a\nb\n", False, False, True, [b"c\n\n", b"d\n"], b"a\nb\nc\nd\n"), - ] - - @pytest.fixture( - params=sm_write_data001, - ids=[x.sign for x in sm_write_data001], - ) - def write_data001(self, request): - assert isinstance(request, pytest.FixtureRequest) - assert type(request.param) == __class__.tagWriteData001 # noqa: E721 - return request.param - - def test_write(self, write_data001): - assert type(write_data001) == __class__.tagWriteData001 # noqa: E721 - - mode = "w+b" if write_data001.call_param__binary else "w+" - - with tempfile.NamedTemporaryFile(mode=mode, delete=True) as tmp_file: - tmp_file.write(write_data001.source) - tmp_file.flush() - - self.operations.write( - tmp_file.name, - write_data001.call_param__data, - read_and_write=write_data001.call_param__rw, - truncate=write_data001.call_param__truncate, - binary=write_data001.call_param__binary) - - tmp_file.seek(0) - - s = tmp_file.read() - - assert s == write_data001.result From 1e4a1e37aa8fba5f5254a5a8fd00b2028cfaf4de Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Wed, 2 Apr 2025 17:58:33 +0300 Subject: [PATCH 447/475] Refactoring of tests (#232) * TestLocalOperations is refactored * TestRemoteOperations is refactored --- tests/test_local.py | 31 ++++++++++++++++------------- tests/test_remote.py | 47 ++++++++++++++++++++++++-------------------- 2 files changed, 43 insertions(+), 35 deletions(-) diff --git a/tests/test_local.py b/tests/test_local.py index 7b5e488d..e82df989 100644 --- a/tests/test_local.py +++ b/tests/test_local.py @@ -1,27 +1,27 @@ # coding: utf-8 +from .helpers.os_ops_descrs import OsOpsDescrs +from .helpers.os_ops_descrs import OsOperations + import os import pytest import re -from ..testgres import LocalOperations - class TestLocalOperations: + @pytest.fixture + def os_ops(self): + return OsOpsDescrs.sm_local_os_ops - @pytest.fixture(scope="function", autouse=True) - def setup(self): - self.operations = LocalOperations() - - def test_read__unknown_file(self): + def test_read__unknown_file(self, os_ops: OsOperations): """ Test LocalOperations::read with unknown file. """ with pytest.raises(FileNotFoundError, match=re.escape("[Errno 2] No such file or directory: '/dummy'")): - self.operations.read("/dummy") + os_ops.read("/dummy") - def test_read_binary__spec__unk_file(self): + def test_read_binary__spec__unk_file(self, os_ops: OsOperations): """ Test LocalOperations::read_binary with unknown file. """ @@ -29,21 +29,24 @@ def test_read_binary__spec__unk_file(self): with pytest.raises( FileNotFoundError, match=re.escape("[Errno 2] No such file or directory: '/dummy'")): - self.operations.read_binary("/dummy", 0) + os_ops.read_binary("/dummy", 0) - def test_get_file_size__unk_file(self): + def test_get_file_size__unk_file(self, os_ops: OsOperations): """ Test LocalOperations::get_file_size. """ + assert isinstance(os_ops, OsOperations) with pytest.raises(FileNotFoundError, match=re.escape("[Errno 2] No such file or directory: '/dummy'")): - self.operations.get_file_size("/dummy") + os_ops.get_file_size("/dummy") - def test_cwd(self): + def test_cwd(self, os_ops: OsOperations): """ Test cwd. """ - v = self.operations.cwd() + assert isinstance(os_ops, OsOperations) + + v = os_ops.cwd() assert v is not None assert type(v) == str # noqa: E721 diff --git a/tests/test_remote.py b/tests/test_remote.py index 565b2d20..a37e258e 100755 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -1,33 +1,35 @@ # coding: utf-8 -import os -import pytest +from .helpers.os_ops_descrs import OsOpsDescrs +from .helpers.os_ops_descrs import OsOperations from ..testgres import ExecUtilException -from ..testgres import RemoteOperations -from ..testgres import ConnectionParams + +import os +import pytest class TestRemoteOperations: + @pytest.fixture + def os_ops(self): + return OsOpsDescrs.sm_remote_os_ops - @pytest.fixture(scope="function", autouse=True) - def setup(self): - conn_params = ConnectionParams(host=os.getenv('RDBMS_TESTPOOL1_HOST') or '127.0.0.1', - username=os.getenv('USER'), - ssh_key=os.getenv('RDBMS_TESTPOOL_SSHKEY')) - self.operations = RemoteOperations(conn_params) + def test_rmdirs__try_to_delete_nonexist_path(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) - def test_rmdirs__try_to_delete_nonexist_path(self): path = "/root/test_dir" - assert self.operations.rmdirs(path, ignore_errors=False) is True + assert os_ops.rmdirs(path, ignore_errors=False) is True + + def test_rmdirs__try_to_delete_file(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) - def test_rmdirs__try_to_delete_file(self): - path = self.operations.mkstemp() + path = os_ops.mkstemp() + assert type(path) == str # noqa: E721 assert os.path.exists(path) with pytest.raises(ExecUtilException) as x: - self.operations.rmdirs(path, ignore_errors=False) + os_ops.rmdirs(path, ignore_errors=False) assert os.path.exists(path) assert type(x.value) == ExecUtilException # noqa: E721 @@ -37,37 +39,40 @@ def test_rmdirs__try_to_delete_file(self): assert type(x.value.exit_code) == int # noqa: E721 assert x.value.exit_code == 20 - def test_read__unknown_file(self): + def test_read__unknown_file(self, os_ops: OsOperations): """ Test RemoteOperations::read with unknown file. """ + assert isinstance(os_ops, OsOperations) with pytest.raises(ExecUtilException) as x: - self.operations.read("/dummy") + os_ops.read("/dummy") assert "Utility exited with non-zero code (1)." in str(x.value) assert "No such file or directory" in str(x.value) assert "/dummy" in str(x.value) - def test_read_binary__spec__unk_file(self): + def test_read_binary__spec__unk_file(self, os_ops: OsOperations): """ Test RemoteOperations::read_binary with unknown file. """ + assert isinstance(os_ops, OsOperations) with pytest.raises(ExecUtilException) as x: - self.operations.read_binary("/dummy", 0) + os_ops.read_binary("/dummy", 0) assert "Utility exited with non-zero code (1)." in str(x.value) assert "No such file or directory" in str(x.value) assert "/dummy" in str(x.value) - def test_get_file_size__unk_file(self): + def test_get_file_size__unk_file(self, os_ops: OsOperations): """ Test RemoteOperations::get_file_size. """ + assert isinstance(os_ops, OsOperations) with pytest.raises(ExecUtilException) as x: - self.operations.get_file_size("/dummy") + os_ops.get_file_size("/dummy") assert "Utility exited with non-zero code (1)." in str(x.value) assert "No such file or directory" in str(x.value) From 46598b476888606c7b197f263f4d828253cf9d1d Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Wed, 2 Apr 2025 20:48:19 +0300 Subject: [PATCH 448/475] Test files and classes were renamed (#233) CI for AltLinux (10, 11) tests all the "local" cases. --- Dockerfile--altlinux_10.tmpl | 2 +- Dockerfile--altlinux_11.tmpl | 2 +- run_tests.sh | 2 +- tests/{test_local.py => test_os_ops_local.py} | 2 +- tests/{test_remote.py => test_os_ops_remote.py} | 2 +- tests/{test_simple.py => test_testgres_local.py} | 2 +- tests/{test_simple_remote.py => test_testgres_remote.py} | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) rename tests/{test_local.py => test_os_ops_local.py} (98%) rename tests/{test_remote.py => test_os_ops_remote.py} (98%) rename tests/{test_simple.py => test_testgres_local.py} (99%) rename tests/{test_simple_remote.py => test_testgres_remote.py} (99%) diff --git a/Dockerfile--altlinux_10.tmpl b/Dockerfile--altlinux_10.tmpl index e60e9320..a75e35a0 100644 --- a/Dockerfile--altlinux_10.tmpl +++ b/Dockerfile--altlinux_10.tmpl @@ -115,4 +115,4 @@ ssh-keygen -t rsa -f ~/.ssh/id_rsa -q -N ''; \ cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys; \ chmod 600 ~/.ssh/authorized_keys; \ ls -la ~/.ssh/; \ -TEST_FILTER=\"TestgresTests or (TestTestgresCommon and (not remote_ops))\" bash ./run_tests.sh;" +TEST_FILTER=\"TestTestgresLocal or TestOsOpsLocal or local_ops\" bash ./run_tests.sh;" diff --git a/Dockerfile--altlinux_11.tmpl b/Dockerfile--altlinux_11.tmpl index 4b591632..5b43da20 100644 --- a/Dockerfile--altlinux_11.tmpl +++ b/Dockerfile--altlinux_11.tmpl @@ -115,4 +115,4 @@ ssh-keygen -t rsa -f ~/.ssh/id_rsa -q -N ''; \ cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys; \ chmod 600 ~/.ssh/authorized_keys; \ ls -la ~/.ssh/; \ -TEST_FILTER=\"TestgresTests or (TestTestgresCommon and (not remote_ops))\" bash ./run_tests.sh;" +TEST_FILTER=\"TestTestgresLocal or TestOsOpsLocal or local_ops\" bash ./run_tests.sh;" diff --git a/run_tests.sh b/run_tests.sh index a40a97cf..8202aff5 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -5,7 +5,7 @@ set -eux if [ -z ${TEST_FILTER+x} ]; \ -then export TEST_FILTER="TestgresTests or (TestTestgresCommon and (not remote_ops))"; \ +then export TEST_FILTER="TestTestgresLocal or (TestTestgresCommon and (not remote_ops))"; \ fi # fail early diff --git a/tests/test_local.py b/tests/test_os_ops_local.py similarity index 98% rename from tests/test_local.py rename to tests/test_os_ops_local.py index e82df989..2e3c30b7 100644 --- a/tests/test_local.py +++ b/tests/test_os_ops_local.py @@ -8,7 +8,7 @@ import re -class TestLocalOperations: +class TestOsOpsLocal: @pytest.fixture def os_ops(self): return OsOpsDescrs.sm_local_os_ops diff --git a/tests/test_remote.py b/tests/test_os_ops_remote.py similarity index 98% rename from tests/test_remote.py rename to tests/test_os_ops_remote.py index a37e258e..58b09242 100755 --- a/tests/test_remote.py +++ b/tests/test_os_ops_remote.py @@ -9,7 +9,7 @@ import pytest -class TestRemoteOperations: +class TestOsOpsRemote: @pytest.fixture def os_ops(self): return OsOpsDescrs.sm_remote_os_ops diff --git a/tests/test_simple.py b/tests/test_testgres_local.py similarity index 99% rename from tests/test_simple.py rename to tests/test_testgres_local.py index 6ca52cb0..01f975a0 100644 --- a/tests/test_simple.py +++ b/tests/test_testgres_local.py @@ -74,7 +74,7 @@ def rm_carriage_returns(out): return out -class TestgresTests: +class TestTestgresLocal: def test_node_repr(self): with get_new_node() as node: pattern = r"PostgresNode\(name='.+', port=.+, base_dir='.+'\)" diff --git a/tests/test_simple_remote.py b/tests/test_testgres_remote.py similarity index 99% rename from tests/test_simple_remote.py rename to tests/test_testgres_remote.py index c16fe53f..2142e5ba 100755 --- a/tests/test_simple_remote.py +++ b/tests/test_testgres_remote.py @@ -47,7 +47,7 @@ def good_properties(f): return True -class TestgresRemoteTests: +class TestTestgresRemote: sm_os_ops = OsOpsDescrs.sm_remote_os_ops @pytest.fixture(autouse=True, scope="class") From ecd5427b7db2181a8b36b01797576d837e05eb46 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Wed, 2 Apr 2025 21:55:28 +0300 Subject: [PATCH 449/475] PostgresNode::source_walsender is updated [revision] --- testgres/node.py | 4 +++- tests/test_testgres_common.py | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/testgres/node.py b/testgres/node.py index c8ae4204..4ab98ea1 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -431,9 +431,11 @@ def source_walsender(self): where application_name = %s """ - if not self.master: + if self.master is None: raise TestgresException("Node doesn't have a master") + assert type(self.master) == PostgresNode + # master should be on the same host assert self.master.host == self.host diff --git a/tests/test_testgres_common.py b/tests/test_testgres_common.py index 2440b8f0..7cfb203e 100644 --- a/tests/test_testgres_common.py +++ b/tests/test_testgres_common.py @@ -348,6 +348,7 @@ def LOCAL__check_auxiliary_pids__multiple_attempts( assert (con.pid > 0) with master.replicate().start() as replica: + assert type(replica) == PostgresNode # test __str__ method str(master.child_processes[0]) From afee4a55a04893240fe674db7b134b646b005531 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Wed, 2 Apr 2025 22:03:55 +0300 Subject: [PATCH 450/475] [FIX] formatting (flake8) --- testgres/node.py | 2 +- tests/test_testgres_common.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index 4ab98ea1..1f8fca6e 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -434,7 +434,7 @@ def source_walsender(self): if self.master is None: raise TestgresException("Node doesn't have a master") - assert type(self.master) == PostgresNode + assert type(self.master) == PostgresNode # noqa: E721 # master should be on the same host assert self.master.host == self.host diff --git a/tests/test_testgres_common.py b/tests/test_testgres_common.py index 7cfb203e..4e23c4af 100644 --- a/tests/test_testgres_common.py +++ b/tests/test_testgres_common.py @@ -348,7 +348,7 @@ def LOCAL__check_auxiliary_pids__multiple_attempts( assert (con.pid > 0) with master.replicate().start() as replica: - assert type(replica) == PostgresNode + assert type(replica) == PostgresNode # noqa: E721 # test __str__ method str(master.child_processes[0]) From 6e7d315ab45a3065d1440819a369f7781c54a87e Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Sat, 5 Apr 2025 20:23:55 +0300 Subject: [PATCH 451/475] [FIX] The call of RaiseError.CommandExecutionError is corrected [message arg] --- testgres/operations/remote_ops.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index e1ad6dac..a3ecf637 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -193,7 +193,7 @@ def is_executable(self, file): RaiseError.CommandExecutionError( cmd=command, exit_code=exit_status, - msg_arg=errMsg, + message=errMsg, error=error, out=output ) @@ -305,7 +305,7 @@ def path_exists(self, path): RaiseError.CommandExecutionError( cmd=command, exit_code=exit_status, - msg_arg=errMsg, + message=errMsg, error=error, out=output ) From 14bc733db60712c99df20faea3efe659f9d9dafb Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Sun, 6 Apr 2025 19:14:21 +0300 Subject: [PATCH 452/475] PortManager (#234) Main - Old PortManager was deleted - PostresNode uses new "abstract" interface PortManager to reserve/release port number. - PostgresNode.free_port always resets port number - OsOperations::is_free_port is added - PostgresNode.start raises exception (InvalidOperationException) if node is None Refactoring - PostgresNode::port is RO-property now. It throws if port is None - PostgresNode::host is RO-property now - PostgresNode::ssh_key is RO-property now - PostgresNode::name is RO-property now --- Dockerfile--altlinux_10.tmpl | 2 +- Dockerfile--altlinux_11.tmpl | 2 +- Dockerfile--ubuntu_24_04.tmpl | 1 + run_tests.sh | 2 +- setup.py | 2 +- testgres/__init__.py | 6 +- testgres/exceptions.py | 4 + testgres/helpers/__init__.py | 0 testgres/helpers/port_manager.py | 41 --- testgres/node.py | 167 +++++++-- testgres/operations/local_ops.py | 11 + testgres/operations/os_ops.py | 4 + testgres/operations/remote_ops.py | 48 +++ testgres/port_manager.py | 102 +++++ testgres/utils.py | 32 +- tests/helpers/global_data.py | 78 ++++ tests/helpers/os_ops_descrs.py | 32 -- tests/test_os_ops_common.py | 105 +++++- tests/test_os_ops_local.py | 4 +- tests/test_os_ops_remote.py | 4 +- tests/test_testgres_common.py | 592 ++++++++++++++++++++++-------- tests/test_testgres_local.py | 57 --- tests/test_testgres_remote.py | 128 +------ 23 files changed, 993 insertions(+), 431 deletions(-) delete mode 100644 testgres/helpers/__init__.py delete mode 100644 testgres/helpers/port_manager.py create mode 100644 testgres/port_manager.py create mode 100644 tests/helpers/global_data.py delete mode 100644 tests/helpers/os_ops_descrs.py diff --git a/Dockerfile--altlinux_10.tmpl b/Dockerfile--altlinux_10.tmpl index a75e35a0..d78b05f5 100644 --- a/Dockerfile--altlinux_10.tmpl +++ b/Dockerfile--altlinux_10.tmpl @@ -115,4 +115,4 @@ ssh-keygen -t rsa -f ~/.ssh/id_rsa -q -N ''; \ cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys; \ chmod 600 ~/.ssh/authorized_keys; \ ls -la ~/.ssh/; \ -TEST_FILTER=\"TestTestgresLocal or TestOsOpsLocal or local_ops\" bash ./run_tests.sh;" +TEST_FILTER=\"TestTestgresLocal or TestOsOpsLocal or local\" bash ./run_tests.sh;" diff --git a/Dockerfile--altlinux_11.tmpl b/Dockerfile--altlinux_11.tmpl index 5b43da20..5c88585d 100644 --- a/Dockerfile--altlinux_11.tmpl +++ b/Dockerfile--altlinux_11.tmpl @@ -115,4 +115,4 @@ ssh-keygen -t rsa -f ~/.ssh/id_rsa -q -N ''; \ cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys; \ chmod 600 ~/.ssh/authorized_keys; \ ls -la ~/.ssh/; \ -TEST_FILTER=\"TestTestgresLocal or TestOsOpsLocal or local_ops\" bash ./run_tests.sh;" +TEST_FILTER=\"TestTestgresLocal or TestOsOpsLocal or local\" bash ./run_tests.sh;" diff --git a/Dockerfile--ubuntu_24_04.tmpl b/Dockerfile--ubuntu_24_04.tmpl index 3bdc6640..7a559776 100644 --- a/Dockerfile--ubuntu_24_04.tmpl +++ b/Dockerfile--ubuntu_24_04.tmpl @@ -10,6 +10,7 @@ RUN apt install -y sudo curl ca-certificates RUN apt update RUN apt install -y openssh-server RUN apt install -y time +RUN apt install -y netcat-traditional RUN apt update RUN apt install -y postgresql-common diff --git a/run_tests.sh b/run_tests.sh index 8202aff5..65c17dbf 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -5,7 +5,7 @@ set -eux if [ -z ${TEST_FILTER+x} ]; \ -then export TEST_FILTER="TestTestgresLocal or (TestTestgresCommon and (not remote_ops))"; \ +then export TEST_FILTER="TestTestgresLocal or (TestTestgresCommon and (not remote))"; \ fi # fail early diff --git a/setup.py b/setup.py index 3f2474dd..b47a1d8a 100755 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ setup( version='1.10.5', name='testgres', - packages=['testgres', 'testgres.operations', 'testgres.helpers'], + packages=['testgres', 'testgres.operations'], description='Testing utility for PostgreSQL and its extensions', url='https://github.com/postgrespro/testgres', long_description=readme, diff --git a/testgres/__init__.py b/testgres/__init__.py index 665548d6..339ae62e 100644 --- a/testgres/__init__.py +++ b/testgres/__init__.py @@ -34,6 +34,7 @@ DumpFormat from .node import PostgresNode, NodeApp +from .node import PortManager from .utils import \ reserve_port, \ @@ -53,8 +54,6 @@ from .operations.local_ops import LocalOperations from .operations.remote_ops import RemoteOperations -from .helpers.port_manager import PortManager - __all__ = [ "get_new_node", "get_remote_node", @@ -64,7 +63,8 @@ "TestgresException", "ExecUtilException", "QueryException", "TimeoutException", "CatchUpException", "StartNodeException", "InitNodeException", "BackupException", "InvalidOperationException", "XLogMethod", "IsolationLevel", "NodeStatus", "ProcessType", "DumpFormat", "PostgresNode", "NodeApp", + "PortManager", "reserve_port", "release_port", "bound_ports", "get_bin_path", "get_pg_config", "get_pg_version", - "First", "Any", "PortManager", + "First", "Any", "OsOperations", "LocalOperations", "RemoteOperations", "ConnectionParams" ] diff --git a/testgres/exceptions.py b/testgres/exceptions.py index d61d4691..20c1a8cf 100644 --- a/testgres/exceptions.py +++ b/testgres/exceptions.py @@ -7,6 +7,10 @@ class TestgresException(Exception): pass +class PortForException(TestgresException): + pass + + @six.python_2_unicode_compatible class ExecUtilException(TestgresException): def __init__(self, message=None, command=None, exit_code=0, out=None, error=None): diff --git a/testgres/helpers/__init__.py b/testgres/helpers/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/testgres/helpers/port_manager.py b/testgres/helpers/port_manager.py deleted file mode 100644 index cfc5c096..00000000 --- a/testgres/helpers/port_manager.py +++ /dev/null @@ -1,41 +0,0 @@ -import socket -import random -from typing import Set, Iterable, Optional - - -class PortForException(Exception): - pass - - -class PortManager: - def __init__(self, ports_range=(1024, 65535)): - self.ports_range = ports_range - - @staticmethod - def is_port_free(port: int) -> bool: - """Check if a port is free to use.""" - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: - try: - s.bind(("", port)) - return True - except OSError: - return False - - def find_free_port(self, ports: Optional[Set[int]] = None, exclude_ports: Optional[Iterable[int]] = None) -> int: - """Return a random unused port number.""" - if ports is None: - ports = set(range(1024, 65535)) - - assert type(ports) == set # noqa: E721 - - if exclude_ports is not None: - assert isinstance(exclude_ports, Iterable) - ports.difference_update(exclude_ports) - - sampled_ports = random.sample(tuple(ports), min(len(ports), 100)) - - for port in sampled_ports: - if self.is_port_free(port): - return port - - raise PortForException("Can't select a port") diff --git a/testgres/node.py b/testgres/node.py index 1f8fca6e..5039fc43 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -1,4 +1,6 @@ # coding: utf-8 +from __future__ import annotations + import logging import os import random @@ -10,6 +12,7 @@ from queue import Queue import time +import typing try: from collections.abc import Iterable @@ -80,14 +83,16 @@ BackupException, \ InvalidOperationException +from .port_manager import PortManager +from .port_manager import PortManager__ThisHost +from .port_manager import PortManager__Generic + from .logger import TestgresLogger from .pubsub import Publication, Subscription from .standby import First -from . import utils - from .utils import \ PgVer, \ eprint, \ @@ -126,17 +131,31 @@ def __getattr__(self, name): return getattr(self.process, name) def __repr__(self): - return '{}(ptype={}, process={})'.format(self.__class__.__name__, - str(self.ptype), - repr(self.process)) + return '{}(ptype={}, process={})'.format( + self.__class__.__name__, + str(self.ptype), + repr(self.process)) class PostgresNode(object): # a max number of node start attempts _C_MAX_START_ATEMPTS = 5 - def __init__(self, name=None, base_dir=None, port=None, conn_params: ConnectionParams = ConnectionParams(), - bin_dir=None, prefix=None, os_ops=None): + _name: typing.Optional[str] + _port: typing.Optional[int] + _should_free_port: bool + _os_ops: OsOperations + _port_manager: PortManager + + def __init__(self, + name=None, + base_dir=None, + port: typing.Optional[int] = None, + conn_params: ConnectionParams = ConnectionParams(), + bin_dir=None, + prefix=None, + os_ops: typing.Optional[OsOperations] = None, + port_manager: typing.Optional[PortManager] = None): """ PostgresNode constructor. @@ -145,21 +164,26 @@ def __init__(self, name=None, base_dir=None, port=None, conn_params: ConnectionP port: port to accept connections. base_dir: path to node's data directory. bin_dir: path to node's binary directory. + os_ops: None or correct OS operation object. + port_manager: None or correct port manager object. """ + assert port is None or type(port) == int # noqa: E721 + assert os_ops is None or isinstance(os_ops, OsOperations) + assert port_manager is None or isinstance(port_manager, PortManager) # private if os_ops is None: - os_ops = __class__._get_os_ops(conn_params) + self._os_ops = __class__._get_os_ops(conn_params) else: assert conn_params is None + assert isinstance(os_ops, OsOperations) + self._os_ops = os_ops pass - assert os_ops is not None - assert isinstance(os_ops, OsOperations) - self._os_ops = os_ops + assert self._os_ops is not None + assert isinstance(self._os_ops, OsOperations) - self._pg_version = PgVer(get_pg_version2(os_ops, bin_dir)) - self._should_free_port = port is None + self._pg_version = PgVer(get_pg_version2(self._os_ops, bin_dir)) self._base_dir = base_dir self._bin_dir = bin_dir self._prefix = prefix @@ -167,12 +191,29 @@ def __init__(self, name=None, base_dir=None, port=None, conn_params: ConnectionP self._master = None # basic - self.name = name or generate_app_name() + self._name = name or generate_app_name() + + if port is not None: + assert type(port) == int # noqa: E721 + assert port_manager is None + self._port = port + self._should_free_port = False + self._port_manager = None + else: + if port_manager is not None: + assert isinstance(port_manager, PortManager) + self._port_manager = port_manager + else: + self._port_manager = __class__._get_port_manager(self._os_ops) - self.host = os_ops.host - self.port = port or utils.reserve_port() + assert self._port_manager is not None + assert isinstance(self._port_manager, PortManager) - self.ssh_key = os_ops.ssh_key + self._port = self._port_manager.reserve_port() # raises + assert type(self._port) == int # noqa: E721 + self._should_free_port = True + + assert type(self._port) == int # noqa: E721 # defaults for __exit__() self.cleanup_on_good_exit = testgres_config.node_cleanup_on_good_exit @@ -207,7 +248,11 @@ def __exit__(self, type, value, traceback): def __repr__(self): return "{}(name='{}', port={}, base_dir='{}')".format( - self.__class__.__name__, self.name, self.port, self.base_dir) + self.__class__.__name__, + self.name, + str(self._port) if self._port is not None else "None", + self.base_dir + ) @staticmethod def _get_os_ops(conn_params: ConnectionParams) -> OsOperations: @@ -221,19 +266,39 @@ def _get_os_ops(conn_params: ConnectionParams) -> OsOperations: return LocalOperations(conn_params) + @staticmethod + def _get_port_manager(os_ops: OsOperations) -> PortManager: + assert os_ops is not None + assert isinstance(os_ops, OsOperations) + + if isinstance(os_ops, LocalOperations): + return PortManager__ThisHost() + + # TODO: Throw the exception "Please define a port manager." ? + return PortManager__Generic(os_ops) + def clone_with_new_name_and_base_dir(self, name: str, base_dir: str): assert name is None or type(name) == str # noqa: E721 assert base_dir is None or type(base_dir) == str # noqa: E721 assert __class__ == PostgresNode + if self._port_manager is None: + raise InvalidOperationException("PostgresNode without PortManager can't be cloned.") + + assert self._port_manager is not None + assert isinstance(self._port_manager, PortManager) + assert self._os_ops is not None + assert isinstance(self._os_ops, OsOperations) + node = PostgresNode( name=name, base_dir=base_dir, conn_params=None, bin_dir=self._bin_dir, prefix=self._prefix, - os_ops=self._os_ops) + os_ops=self._os_ops, + port_manager=self._port_manager) return node @@ -243,6 +308,33 @@ def os_ops(self) -> OsOperations: assert isinstance(self._os_ops, OsOperations) return self._os_ops + @property + def name(self) -> str: + if self._name is None: + raise InvalidOperationException("PostgresNode name is not defined.") + assert type(self._name) == str # noqa: E721 + return self._name + + @property + def host(self) -> str: + assert self._os_ops is not None + assert isinstance(self._os_ops, OsOperations) + return self._os_ops.host + + @property + def port(self) -> int: + if self._port is None: + raise InvalidOperationException("PostgresNode port is not defined.") + + assert type(self._port) == int # noqa: E721 + return self._port + + @property + def ssh_key(self) -> typing.Optional[str]: + assert self._os_ops is not None + assert isinstance(self._os_ops, OsOperations) + return self._os_ops.ssh_key + @property def pid(self): """ @@ -993,6 +1085,11 @@ def start(self, params=[], wait=True): if self.is_started: return self + if self._port is None: + raise InvalidOperationException("Can't start PostgresNode. Port is not defined.") + + assert type(self._port) == int # noqa: E721 + _params = [self._get_bin_path("pg_ctl"), "-D", self.data_dir, "-l", self.pg_log_file, @@ -1023,6 +1120,8 @@ def LOCAL__raise_cannot_start_node__std(from_exception): LOCAL__raise_cannot_start_node__std(e) else: assert self._should_free_port + assert self._port_manager is not None + assert isinstance(self._port_manager, PortManager) assert __class__._C_MAX_START_ATEMPTS > 1 log_files0 = self._collect_log_files() @@ -1048,20 +1147,20 @@ def LOCAL__raise_cannot_start_node__std(from_exception): log_files0 = log_files1 logging.warning( - "Detected a conflict with using the port {0}. Trying another port after a {1}-second sleep...".format(self.port, timeout) + "Detected a conflict with using the port {0}. Trying another port after a {1}-second sleep...".format(self._port, timeout) ) time.sleep(timeout) timeout = min(2 * timeout, 5) - cur_port = self.port - new_port = utils.reserve_port() # can raise + cur_port = self._port + new_port = self._port_manager.reserve_port() # can raise try: options = {'port': new_port} self.set_auto_conf(options) except: # noqa: E722 - utils.release_port(new_port) + self._port_manager.release_port(new_port) raise - self.port = new_port - utils.release_port(cur_port) + self._port = new_port + self._port_manager.release_port(cur_port) continue break self._maybe_start_logger() @@ -1222,14 +1321,22 @@ def pg_ctl(self, params): def free_port(self): """ Reclaim port owned by this node. - NOTE: does not free auto selected ports. + NOTE: this method does not release manually defined port but reset it. """ + assert type(self._should_free_port) == bool # noqa: E721 + + if not self._should_free_port: + self._port = None + else: + assert type(self._port) == int # noqa: E721 + + assert self._port_manager is not None + assert isinstance(self._port_manager, PortManager) - if self._should_free_port: - port = self.port + port = self._port self._should_free_port = False - self.port = None - utils.release_port(port) + self._port = None + self._port_manager.release_port(port) def cleanup(self, max_attempts=3, full=False): """ diff --git a/testgres/operations/local_ops.py b/testgres/operations/local_ops.py index 35e94210..39c81405 100644 --- a/testgres/operations/local_ops.py +++ b/testgres/operations/local_ops.py @@ -6,6 +6,7 @@ import subprocess import tempfile import time +import socket import psutil @@ -436,6 +437,16 @@ def get_process_children(self, pid): assert type(pid) == int # noqa: E721 return psutil.Process(pid).children() + def is_port_free(self, number: int) -> bool: + assert type(number) == int # noqa: E721 + + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + try: + s.bind(("", number)) + return True + except OSError: + return False + # Database control def db_connect(self, dbname, user, password=None, host="localhost", port=5432): conn = pglib.connect( diff --git a/testgres/operations/os_ops.py b/testgres/operations/os_ops.py index 3c606871..489a7cb2 100644 --- a/testgres/operations/os_ops.py +++ b/testgres/operations/os_ops.py @@ -127,6 +127,10 @@ def get_pid(self): def get_process_children(self, pid): raise NotImplementedError() + def is_port_free(self, number: int): + assert type(number) == int # noqa: E721 + raise NotImplementedError() + # Database control def db_connect(self, dbname, user, password=None, host="localhost", port=5432): raise NotImplementedError() diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index a3ecf637..ee747e52 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -629,6 +629,54 @@ def get_process_children(self, pid): raise ExecUtilException(f"Error in getting process children. Error: {result.stderr}") + def is_port_free(self, number: int) -> bool: + assert type(number) == int # noqa: E721 + + cmd = ["nc", "-w", "5", "-z", "-v", "localhost", str(number)] + + exit_status, output, error = self.exec_command(cmd=cmd, encoding=get_default_encoding(), ignore_errors=True, verbose=True) + + assert type(output) == str # noqa: E721 + assert type(error) == str # noqa: E721 + + if exit_status == 0: + return __class__._is_port_free__process_0(error) + + if exit_status == 1: + return __class__._is_port_free__process_1(error) + + errMsg = "nc returns an unknown result code: {0}".format(exit_status) + + RaiseError.CommandExecutionError( + cmd=cmd, + exit_code=exit_status, + message=errMsg, + error=error, + out=output + ) + + @staticmethod + def _is_port_free__process_0(error: str) -> bool: + assert type(error) == str # noqa: E721 + # + # Example of error text: + # "Connection to localhost (127.0.0.1) 1024 port [tcp/*] succeeded!\n" + # + # May be here is needed to check error message? + # + return False + + @staticmethod + def _is_port_free__process_1(error: str) -> bool: + assert type(error) == str # noqa: E721 + # + # Example of error text: + # "nc: connect to localhost (127.0.0.1) port 1024 (tcp) failed: Connection refused\n" + # + # May be here is needed to check error message? + # + return True + # Database control def db_connect(self, dbname, user, password=None, host="localhost", port=5432): conn = pglib.connect( diff --git a/testgres/port_manager.py b/testgres/port_manager.py new file mode 100644 index 00000000..164661e7 --- /dev/null +++ b/testgres/port_manager.py @@ -0,0 +1,102 @@ +from .operations.os_ops import OsOperations + +from .exceptions import PortForException + +from . import utils + +import threading +import random + + +class PortManager: + def __init__(self): + super().__init__() + + def reserve_port(self) -> int: + raise NotImplementedError("PortManager::reserve_port is not implemented.") + + def release_port(self, number: int) -> None: + assert type(number) == int # noqa: E721 + raise NotImplementedError("PortManager::release_port is not implemented.") + + +class PortManager__ThisHost(PortManager): + sm_single_instance: PortManager = None + sm_single_instance_guard = threading.Lock() + + def __init__(self): + pass + + def __new__(cls) -> PortManager: + assert __class__ == PortManager__ThisHost + assert __class__.sm_single_instance_guard is not None + + if __class__.sm_single_instance is None: + with __class__.sm_single_instance_guard: + __class__.sm_single_instance = super().__new__(cls) + assert __class__.sm_single_instance + assert type(__class__.sm_single_instance) == __class__ # noqa: E721 + return __class__.sm_single_instance + + def reserve_port(self) -> int: + return utils.reserve_port() + + def release_port(self, number: int) -> None: + assert type(number) == int # noqa: E721 + return utils.release_port(number) + + +class PortManager__Generic(PortManager): + _os_ops: OsOperations + _guard: object + # TODO: is there better to use bitmap fot _available_ports? + _available_ports: set[int] + _reserved_ports: set[int] + + def __init__(self, os_ops: OsOperations): + assert os_ops is not None + assert isinstance(os_ops, OsOperations) + self._os_ops = os_ops + self._guard = threading.Lock() + self._available_ports = set[int](range(1024, 65535)) + self._reserved_ports = set[int]() + + def reserve_port(self) -> int: + assert self._guard is not None + assert type(self._available_ports) == set # noqa: E721t + assert type(self._reserved_ports) == set # noqa: E721 + + with self._guard: + t = tuple(self._available_ports) + assert len(t) == len(self._available_ports) + sampled_ports = random.sample(t, min(len(t), 100)) + t = None + + for port in sampled_ports: + assert not (port in self._reserved_ports) + assert port in self._available_ports + + if not self._os_ops.is_port_free(port): + continue + + self._reserved_ports.add(port) + self._available_ports.discard(port) + assert port in self._reserved_ports + assert not (port in self._available_ports) + return port + + raise PortForException("Can't select a port.") + + def release_port(self, number: int) -> None: + assert type(number) == int # noqa: E721 + + assert self._guard is not None + assert type(self._reserved_ports) == set # noqa: E721 + + with self._guard: + assert number in self._reserved_ports + assert not (number in self._available_ports) + self._available_ports.add(number) + self._reserved_ports.discard(number) + assert not (number in self._reserved_ports) + assert number in self._available_ports diff --git a/testgres/utils.py b/testgres/utils.py index 92383571..10ae81b6 100644 --- a/testgres/utils.py +++ b/testgres/utils.py @@ -6,6 +6,8 @@ import os import sys +import socket +import random from contextlib import contextmanager from packaging.version import Version, InvalidVersion @@ -13,7 +15,7 @@ from six import iteritems -from .helpers.port_manager import PortManager +from .exceptions import PortForException from .exceptions import ExecUtilException from .config import testgres_config as tconf from .operations.os_ops import OsOperations @@ -41,11 +43,28 @@ def internal__reserve_port(): """ Generate a new port and add it to 'bound_ports'. """ - port_mng = PortManager() - port = port_mng.find_free_port(exclude_ports=bound_ports) - bound_ports.add(port) + def LOCAL__is_port_free(port: int) -> bool: + """Check if a port is free to use.""" + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + try: + s.bind(("", port)) + return True + except OSError: + return False - return port + ports = set(range(1024, 65535)) + assert type(ports) == set # noqa: E721 + assert type(bound_ports) == set # noqa: E721 + ports.difference_update(bound_ports) + + sampled_ports = random.sample(tuple(ports), min(len(ports), 100)) + + for port in sampled_ports: + if LOCAL__is_port_free(port): + bound_ports.add(port) + return port + + raise PortForException("Can't select a port") def internal__release_port(port): @@ -53,6 +72,9 @@ def internal__release_port(port): Free port provided by reserve_port(). """ + assert type(port) == int # noqa: E721 + assert port in bound_ports + bound_ports.discard(port) diff --git a/tests/helpers/global_data.py b/tests/helpers/global_data.py new file mode 100644 index 00000000..c21d7dd8 --- /dev/null +++ b/tests/helpers/global_data.py @@ -0,0 +1,78 @@ +from ...testgres.operations.os_ops import OsOperations +from ...testgres.operations.os_ops import ConnectionParams +from ...testgres.operations.local_ops import LocalOperations +from ...testgres.operations.remote_ops import RemoteOperations + +from ...testgres.node import PortManager +from ...testgres.node import PortManager__ThisHost +from ...testgres.node import PortManager__Generic + +import os + + +class OsOpsDescr: + sign: str + os_ops: OsOperations + + def __init__(self, sign: str, os_ops: OsOperations): + assert type(sign) == str # noqa: E721 + assert isinstance(os_ops, OsOperations) + self.sign = sign + self.os_ops = os_ops + + +class OsOpsDescrs: + sm_remote_conn_params = ConnectionParams( + host=os.getenv('RDBMS_TESTPOOL1_HOST') or '127.0.0.1', + username=os.getenv('USER'), + ssh_key=os.getenv('RDBMS_TESTPOOL_SSHKEY')) + + sm_remote_os_ops = RemoteOperations(sm_remote_conn_params) + + sm_remote_os_ops_descr = OsOpsDescr("remote_ops", sm_remote_os_ops) + + sm_local_os_ops = LocalOperations() + + sm_local_os_ops_descr = OsOpsDescr("local_ops", sm_local_os_ops) + + +class PortManagers: + sm_remote_port_manager = PortManager__Generic(OsOpsDescrs.sm_remote_os_ops) + + sm_local_port_manager = PortManager__ThisHost() + + sm_local2_port_manager = PortManager__Generic(OsOpsDescrs.sm_local_os_ops) + + +class PostgresNodeService: + sign: str + os_ops: OsOperations + port_manager: PortManager + + def __init__(self, sign: str, os_ops: OsOperations, port_manager: PortManager): + assert type(sign) == str # noqa: E721 + assert isinstance(os_ops, OsOperations) + assert isinstance(port_manager, PortManager) + self.sign = sign + self.os_ops = os_ops + self.port_manager = port_manager + + +class PostgresNodeServices: + sm_remote = PostgresNodeService( + "remote", + OsOpsDescrs.sm_remote_os_ops, + PortManagers.sm_remote_port_manager + ) + + sm_local = PostgresNodeService( + "local", + OsOpsDescrs.sm_local_os_ops, + PortManagers.sm_local_port_manager + ) + + sm_local2 = PostgresNodeService( + "local2", + OsOpsDescrs.sm_local_os_ops, + PortManagers.sm_local2_port_manager + ) diff --git a/tests/helpers/os_ops_descrs.py b/tests/helpers/os_ops_descrs.py deleted file mode 100644 index 02297adb..00000000 --- a/tests/helpers/os_ops_descrs.py +++ /dev/null @@ -1,32 +0,0 @@ -from ...testgres.operations.os_ops import OsOperations -from ...testgres.operations.os_ops import ConnectionParams -from ...testgres.operations.local_ops import LocalOperations -from ...testgres.operations.remote_ops import RemoteOperations - -import os - - -class OsOpsDescr: - os_ops: OsOperations - sign: str - - def __init__(self, os_ops: OsOperations, sign: str): - assert isinstance(os_ops, OsOperations) - assert type(sign) == str # noqa: E721 - self.os_ops = os_ops - self.sign = sign - - -class OsOpsDescrs: - sm_remote_conn_params = ConnectionParams( - host=os.getenv('RDBMS_TESTPOOL1_HOST') or '127.0.0.1', - username=os.getenv('USER'), - ssh_key=os.getenv('RDBMS_TESTPOOL_SSHKEY')) - - sm_remote_os_ops = RemoteOperations(sm_remote_conn_params) - - sm_remote_os_ops_descr = OsOpsDescr(sm_remote_os_ops, "remote_ops") - - sm_local_os_ops = LocalOperations() - - sm_local_os_ops_descr = OsOpsDescr(sm_local_os_ops, "local_ops") diff --git a/tests/test_os_ops_common.py b/tests/test_os_ops_common.py index c3944c3b..7d183775 100644 --- a/tests/test_os_ops_common.py +++ b/tests/test_os_ops_common.py @@ -1,7 +1,7 @@ # coding: utf-8 -from .helpers.os_ops_descrs import OsOpsDescr -from .helpers.os_ops_descrs import OsOpsDescrs -from .helpers.os_ops_descrs import OsOperations +from .helpers.global_data import OsOpsDescr +from .helpers.global_data import OsOpsDescrs +from .helpers.global_data import OsOperations from .helpers.run_conditions import RunConditions import os @@ -10,6 +10,8 @@ import re import tempfile import logging +import socket +import threading from ..testgres import InvalidOperationException from ..testgres import ExecUtilException @@ -648,3 +650,100 @@ def test_touch(self, os_ops: OsOperations): assert os_ops.isfile(filename) os_ops.remove_file(filename) + + def test_is_port_free__true(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + + C_LIMIT = 128 + + ports = set(range(1024, 65535)) + assert type(ports) == set # noqa: E721 + + ok_count = 0 + no_count = 0 + + for port in ports: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + try: + s.bind(("", port)) + except OSError: + continue + + r = os_ops.is_port_free(port) + + if r: + ok_count += 1 + logging.info("OK. Port {} is free.".format(port)) + else: + no_count += 1 + logging.warning("NO. Port {} is not free.".format(port)) + + if ok_count == C_LIMIT: + return + + if no_count == C_LIMIT: + raise RuntimeError("To many false positive test attempts.") + + if ok_count == 0: + raise RuntimeError("No one free port was found.") + + def test_is_port_free__false(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + + C_LIMIT = 10 + + ports = set(range(1024, 65535)) + assert type(ports) == set # noqa: E721 + + def LOCAL_server(s: socket.socket): + assert s is not None + assert type(s) == socket.socket # noqa: E721 + + try: + while True: + r = s.accept() + + if r is None: + break + except Exception as e: + assert e is not None + pass + + ok_count = 0 + no_count = 0 + + for port in ports: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + try: + s.bind(("", port)) + except OSError: + continue + + th = threading.Thread(target=LOCAL_server, args=[s]) + + s.listen(10) + + assert type(th) == threading.Thread # noqa: E721 + th.start() + + try: + r = os_ops.is_port_free(port) + finally: + s.shutdown(2) + th.join() + + if not r: + ok_count += 1 + logging.info("OK. Port {} is not free.".format(port)) + else: + no_count += 1 + logging.warning("NO. Port {} does not accept connection.".format(port)) + + if ok_count == C_LIMIT: + return + + if no_count == C_LIMIT: + raise RuntimeError("To many false positive test attempts.") + + if ok_count == 0: + raise RuntimeError("No one free port was found.") diff --git a/tests/test_os_ops_local.py b/tests/test_os_ops_local.py index 2e3c30b7..f60c3fc9 100644 --- a/tests/test_os_ops_local.py +++ b/tests/test_os_ops_local.py @@ -1,6 +1,6 @@ # coding: utf-8 -from .helpers.os_ops_descrs import OsOpsDescrs -from .helpers.os_ops_descrs import OsOperations +from .helpers.global_data import OsOpsDescrs +from .helpers.global_data import OsOperations import os diff --git a/tests/test_os_ops_remote.py b/tests/test_os_ops_remote.py index 58b09242..338e49f3 100755 --- a/tests/test_os_ops_remote.py +++ b/tests/test_os_ops_remote.py @@ -1,7 +1,7 @@ # coding: utf-8 -from .helpers.os_ops_descrs import OsOpsDescrs -from .helpers.os_ops_descrs import OsOperations +from .helpers.global_data import OsOpsDescrs +from .helpers.global_data import OsOperations from ..testgres import ExecUtilException diff --git a/tests/test_testgres_common.py b/tests/test_testgres_common.py index 4e23c4af..b286a1c6 100644 --- a/tests/test_testgres_common.py +++ b/tests/test_testgres_common.py @@ -1,6 +1,7 @@ -from .helpers.os_ops_descrs import OsOpsDescr -from .helpers.os_ops_descrs import OsOpsDescrs -from .helpers.os_ops_descrs import OsOperations +from .helpers.global_data import PostgresNodeService +from .helpers.global_data import PostgresNodeServices +from .helpers.global_data import OsOperations +from .helpers.global_data import PortManager from ..testgres.node import PgVer from ..testgres.node import PostgresNode @@ -37,6 +38,8 @@ import uuid import os import re +import subprocess +import typing @contextmanager @@ -54,22 +57,25 @@ def removing(os_ops: OsOperations, f): class TestTestgresCommon: - sm_os_ops_descrs: list[OsOpsDescr] = [ - OsOpsDescrs.sm_local_os_ops_descr, - OsOpsDescrs.sm_remote_os_ops_descr + sm_node_svcs: list[PostgresNodeService] = [ + PostgresNodeServices.sm_local, + PostgresNodeServices.sm_local2, + PostgresNodeServices.sm_remote, ] @pytest.fixture( - params=[descr.os_ops for descr in sm_os_ops_descrs], - ids=[descr.sign for descr in sm_os_ops_descrs] + params=sm_node_svcs, + ids=[descr.sign for descr in sm_node_svcs] ) - def os_ops(self, request: pytest.FixtureRequest) -> OsOperations: + def node_svc(self, request: pytest.FixtureRequest) -> PostgresNodeService: assert isinstance(request, pytest.FixtureRequest) - assert isinstance(request.param, OsOperations) + assert isinstance(request.param, PostgresNodeService) + assert isinstance(request.param.os_ops, OsOperations) + assert isinstance(request.param.port_manager, PortManager) return request.param - def test_version_management(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) + def test_version_management(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) a = PgVer('10.0') b = PgVer('10') @@ -93,42 +99,42 @@ def test_version_management(self, os_ops: OsOperations): assert (g == k) assert (g > h) - version = get_pg_version2(os_ops) + version = get_pg_version2(node_svc.os_ops) - with __class__.helper__get_node(os_ops) as node: + with __class__.helper__get_node(node_svc) as node: assert (isinstance(version, six.string_types)) assert (isinstance(node.version, PgVer)) assert (node.version == PgVer(version)) - def test_double_init(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) + def test_double_init(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) - with __class__.helper__get_node(os_ops).init() as node: + with __class__.helper__get_node(node_svc).init() as node: # can't initialize node more than once with pytest.raises(expected_exception=InitNodeException): node.init() - def test_init_after_cleanup(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) + def test_init_after_cleanup(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) - with __class__.helper__get_node(os_ops) as node: + with __class__.helper__get_node(node_svc) as node: node.init().start().execute('select 1') node.cleanup() node.init().start().execute('select 1') - def test_init_unique_system_id(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) + def test_init_unique_system_id(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) # this function exists in PostgreSQL 9.6+ - current_version = get_pg_version2(os_ops) + current_version = get_pg_version2(node_svc.os_ops) - __class__.helper__skip_test_if_util_not_exist(os_ops, "pg_resetwal") + __class__.helper__skip_test_if_util_not_exist(node_svc.os_ops, "pg_resetwal") __class__.helper__skip_test_if_pg_version_is_not_ge(current_version, '9.6') query = 'select system_identifier from pg_control_system()' with scoped_config(cache_initdb=False): - with __class__.helper__get_node(os_ops).init().start() as node0: + with __class__.helper__get_node(node_svc).init().start() as node0: id0 = node0.execute(query)[0] with scoped_config(cache_initdb=True, @@ -137,8 +143,8 @@ def test_init_unique_system_id(self, os_ops: OsOperations): assert (config.cached_initdb_unique) # spawn two nodes; ids must be different - with __class__.helper__get_node(os_ops).init().start() as node1, \ - __class__.helper__get_node(os_ops).init().start() as node2: + with __class__.helper__get_node(node_svc).init().start() as node1, \ + __class__.helper__get_node(node_svc).init().start() as node2: id1 = node1.execute(query)[0] id2 = node2.execute(query)[0] @@ -146,44 +152,44 @@ def test_init_unique_system_id(self, os_ops: OsOperations): assert (id1 > id0) assert (id2 > id1) - def test_node_exit(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) + def test_node_exit(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) with pytest.raises(expected_exception=QueryException): - with __class__.helper__get_node(os_ops).init() as node: + with __class__.helper__get_node(node_svc).init() as node: base_dir = node.base_dir node.safe_psql('select 1') # we should save the DB for "debugging" - assert (os_ops.path_exists(base_dir)) - os_ops.rmdirs(base_dir, ignore_errors=True) + assert (node_svc.os_ops.path_exists(base_dir)) + node_svc.os_ops.rmdirs(base_dir, ignore_errors=True) - with __class__.helper__get_node(os_ops).init() as node: + with __class__.helper__get_node(node_svc).init() as node: base_dir = node.base_dir # should have been removed by default - assert not (os_ops.path_exists(base_dir)) + assert not (node_svc.os_ops.path_exists(base_dir)) - def test_double_start(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) + def test_double_start(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) - with __class__.helper__get_node(os_ops).init().start() as node: + with __class__.helper__get_node(node_svc).init().start() as node: # can't start node more than once node.start() assert (node.is_started) - def test_uninitialized_start(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) + def test_uninitialized_start(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) - with __class__.helper__get_node(os_ops) as node: + with __class__.helper__get_node(node_svc) as node: # node is not initialized yet with pytest.raises(expected_exception=StartNodeException): node.start() - def test_restart(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) + def test_restart(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) - with __class__.helper__get_node(os_ops) as node: + with __class__.helper__get_node(node_svc) as node: node.init().start() # restart, ok @@ -198,10 +204,10 @@ def test_restart(self, os_ops: OsOperations): node.append_conf('pg_hba.conf', 'DUMMY') node.restart() - def test_reload(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) + def test_reload(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) - with __class__.helper__get_node(os_ops) as node: + with __class__.helper__get_node(node_svc) as node: node.init().start() # change client_min_messages and save old value @@ -216,24 +222,24 @@ def test_reload(self, os_ops: OsOperations): assert ('debug1' == cmm_new[0][0].lower()) assert (cmm_old != cmm_new) - def test_pg_ctl(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) + def test_pg_ctl(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) - with __class__.helper__get_node(os_ops) as node: + with __class__.helper__get_node(node_svc) as node: node.init().start() status = node.pg_ctl(['status']) assert ('PID' in status) - def test_status(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) + def test_status(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) assert (NodeStatus.Running) assert not (NodeStatus.Stopped) assert not (NodeStatus.Uninitialized) # check statuses after each operation - with __class__.helper__get_node(os_ops) as node: + with __class__.helper__get_node(node_svc) as node: assert (node.pid == 0) assert (node.status() == NodeStatus.Uninitialized) @@ -257,8 +263,8 @@ def test_status(self, os_ops: OsOperations): assert (node.pid == 0) assert (node.status() == NodeStatus.Uninitialized) - def test_child_pids(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) + def test_child_pids(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) master_processes = [ ProcessType.AutovacuumLauncher, @@ -269,7 +275,7 @@ def test_child_pids(self, os_ops: OsOperations): ProcessType.WalWriter, ] - postgresVersion = get_pg_version2(os_ops) + postgresVersion = get_pg_version2(node_svc.os_ops) if __class__.helper__pg_version_ge(postgresVersion, '10'): master_processes.append(ProcessType.LogicalReplicationLauncher) @@ -338,7 +344,7 @@ def LOCAL__check_auxiliary_pids__multiple_attempts( absenceList )) - with __class__.helper__get_node(os_ops).init().start() as master: + with __class__.helper__get_node(node_svc).init().start() as master: # master node doesn't have a source walsender! with pytest.raises(expected_exception=testgres_TestgresException): @@ -380,10 +386,10 @@ def test_exceptions(self): str(ExecUtilException('msg', 'cmd', 1, 'out')) str(QueryException('msg', 'query')) - def test_auto_name(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) + def test_auto_name(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) - with __class__.helper__get_node(os_ops).init(allow_streaming=True).start() as m: + with __class__.helper__get_node(node_svc).init(allow_streaming=True).start() as m: with m.replicate().start() as r: # check that nodes are running assert (m.status()) @@ -417,9 +423,9 @@ def test_file_tail(self): lines = file_tail(f, 1) assert (lines[0] == s3) - def test_isolation_levels(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) - with __class__.helper__get_node(os_ops).init().start() as node: + def test_isolation_levels(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc).init().start() as node: with node.connect() as con: # string levels con.begin('Read Uncommitted').commit() @@ -437,17 +443,17 @@ def test_isolation_levels(self, os_ops: OsOperations): with pytest.raises(expected_exception=QueryException): con.begin('Garbage').commit() - def test_users(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) - with __class__.helper__get_node(os_ops).init().start() as node: + def test_users(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc).init().start() as node: node.psql('create role test_user login') value = node.safe_psql('select 1', username='test_user') value = __class__.helper__rm_carriage_returns(value) assert (value == b'1\n') - def test_poll_query_until(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) - with __class__.helper__get_node(os_ops) as node: + def test_poll_query_until(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc) as node: node.init().start() get_time = 'select extract(epoch from now())' @@ -507,8 +513,8 @@ def test_poll_query_until(self, os_ops: OsOperations): # check 1 arg, ok node.poll_query_until('select true') - def test_logging(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) + def test_logging(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) C_MAX_ATTEMPTS = 50 # This name is used for testgres logging, too. C_NODE_NAME = "testgres_tests." + __class__.__name__ + "test_logging-master-" + uuid.uuid4().hex @@ -529,7 +535,7 @@ def test_logging(self, os_ops: OsOperations): logger.addHandler(handler) with scoped_config(use_python_logging=True): - with __class__.helper__get_node(os_ops, name=C_NODE_NAME) as master: + with __class__.helper__get_node(node_svc, name=C_NODE_NAME) as master: logging.info("Master node is initilizing") master.init() @@ -599,9 +605,9 @@ def LOCAL__test_lines(): # GO HOME! return - def test_psql(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) - with __class__.helper__get_node(os_ops).init().start() as node: + def test_psql(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc).init().start() as node: # check returned values (1 arg) res = node.psql('select 1') @@ -636,17 +642,20 @@ def test_psql(self, os_ops: OsOperations): # check psql's default args, fails with pytest.raises(expected_exception=QueryException): - node.psql() + r = node.psql() # raises! + logging.error("node.psql returns [{}]".format(r)) node.stop() # check psql on stopped node, fails with pytest.raises(expected_exception=QueryException): - node.safe_psql('select 1') + # [2025-04-03] This call does not raise exception! I do not know why. + r = node.safe_psql('select 1') # raises! + logging.error("node.safe_psql returns [{}]".format(r)) - def test_safe_psql__expect_error(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) - with __class__.helper__get_node(os_ops).init().start() as node: + def test_safe_psql__expect_error(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc).init().start() as node: err = node.safe_psql('select_or_not_select 1', expect_error=True) assert (type(err) == str) # noqa: E721 assert ('select_or_not_select' in err) @@ -663,9 +672,9 @@ def test_safe_psql__expect_error(self, os_ops: OsOperations): res = node.safe_psql("select 1;", expect_error=False) assert (__class__.helper__rm_carriage_returns(res) == b'1\n') - def test_transactions(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) - with __class__.helper__get_node(os_ops).init().start() as node: + def test_transactions(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc).init().start() as node: with node.connect() as con: con.begin() @@ -688,9 +697,9 @@ def test_transactions(self, os_ops: OsOperations): con.execute('drop table test') con.commit() - def test_control_data(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) - with __class__.helper__get_node(os_ops) as node: + def test_control_data(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc) as node: # node is not initialized yet with pytest.raises(expected_exception=ExecUtilException): @@ -703,9 +712,9 @@ def test_control_data(self, os_ops: OsOperations): assert data is not None assert (any('pg_control' in s for s in data.keys())) - def test_backup_simple(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) - with __class__.helper__get_node(os_ops) as master: + def test_backup_simple(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc) as master: # enable streaming for backups master.init(allow_streaming=True) @@ -725,9 +734,9 @@ def test_backup_simple(self, os_ops: OsOperations): res = slave.execute('select * from test order by i asc') assert (res == [(1, ), (2, ), (3, ), (4, )]) - def test_backup_multiple(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) - with __class__.helper__get_node(os_ops) as node: + def test_backup_multiple(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc) as node: node.init(allow_streaming=True).start() with node.backup(xlog_method='fetch') as backup1, \ @@ -739,9 +748,9 @@ def test_backup_multiple(self, os_ops: OsOperations): backup.spawn_primary('node2', destroy=False) as node2: assert (node1.base_dir != node2.base_dir) - def test_backup_exhaust(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) - with __class__.helper__get_node(os_ops) as node: + def test_backup_exhaust(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc) as node: node.init(allow_streaming=True).start() with node.backup(xlog_method='fetch') as backup: @@ -753,9 +762,9 @@ def test_backup_exhaust(self, os_ops: OsOperations): with pytest.raises(expected_exception=BackupException): backup.spawn_primary() - def test_backup_wrong_xlog_method(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) - with __class__.helper__get_node(os_ops) as node: + def test_backup_wrong_xlog_method(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc) as node: node.init(allow_streaming=True).start() with pytest.raises( @@ -764,11 +773,11 @@ def test_backup_wrong_xlog_method(self, os_ops: OsOperations): ): node.backup(xlog_method='wrong') - def test_pg_ctl_wait_option(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) + def test_pg_ctl_wait_option(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) C_MAX_ATTEMPTS = 50 - node = __class__.helper__get_node(os_ops) + node = __class__.helper__get_node(node_svc) assert node.status() == NodeStatus.Uninitialized node.init() assert node.status() == NodeStatus.Stopped @@ -835,9 +844,9 @@ def test_pg_ctl_wait_option(self, os_ops: OsOperations): logging.info("OK. Node is stopped.") node.cleanup() - def test_replicate(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) - with __class__.helper__get_node(os_ops) as node: + def test_replicate(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc) as node: node.init(allow_streaming=True).start() with node.replicate().start() as replica: @@ -851,14 +860,14 @@ def test_replicate(self, os_ops: OsOperations): res = node.execute('select * from test') assert (res == []) - def test_synchronous_replication(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) + def test_synchronous_replication(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) - current_version = get_pg_version2(os_ops) + current_version = get_pg_version2(node_svc.os_ops) __class__.helper__skip_test_if_pg_version_is_not_ge(current_version, "9.6") - with __class__.helper__get_node(os_ops) as master: + with __class__.helper__get_node(node_svc) as master: old_version = not __class__.helper__pg_version_ge(current_version, '9.6') master.init(allow_streaming=True).start() @@ -897,14 +906,14 @@ def test_synchronous_replication(self, os_ops: OsOperations): res = standby1.safe_psql('select count(*) from abc') assert (__class__.helper__rm_carriage_returns(res) == b'1000000\n') - def test_logical_replication(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) + def test_logical_replication(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) - current_version = get_pg_version2(os_ops) + current_version = get_pg_version2(node_svc.os_ops) __class__.helper__skip_test_if_pg_version_is_not_ge(current_version, "10") - with __class__.helper__get_node(os_ops) as node1, __class__.helper__get_node(os_ops) as node2: + with __class__.helper__get_node(node_svc) as node1, __class__.helper__get_node(node_svc) as node2: node1.init(allow_logical=True) node1.start() node2.init().start() @@ -971,15 +980,15 @@ def test_logical_replication(self, os_ops: OsOperations): res = node2.execute('select * from test2') assert (res == [('a', ), ('b', )]) - def test_logical_catchup(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) + def test_logical_catchup(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) """ Runs catchup for 100 times to be sure that it is consistent """ - current_version = get_pg_version2(os_ops) + current_version = get_pg_version2(node_svc.os_ops) __class__.helper__skip_test_if_pg_version_is_not_ge(current_version, "10") - with __class__.helper__get_node(os_ops) as node1, __class__.helper__get_node(os_ops) as node2: + with __class__.helper__get_node(node_svc) as node1, __class__.helper__get_node(node_svc) as node2: node1.init(allow_logical=True) node1.start() node2.init().start() @@ -999,20 +1008,20 @@ def test_logical_catchup(self, os_ops: OsOperations): assert (res == [(i, i, )]) node1.execute('delete from test') - def test_logical_replication_fail(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) + def test_logical_replication_fail(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) - current_version = get_pg_version2(os_ops) + current_version = get_pg_version2(node_svc.os_ops) __class__.helper__skip_test_if_pg_version_is_ge(current_version, "10") - with __class__.helper__get_node(os_ops) as node: + with __class__.helper__get_node(node_svc) as node: with pytest.raises(expected_exception=InitNodeException): node.init(allow_logical=True) - def test_replication_slots(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) - with __class__.helper__get_node(os_ops) as node: + def test_replication_slots(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc) as node: node.init(allow_streaming=True).start() with node.replicate(slot='slot1').start() as replica: @@ -1022,18 +1031,18 @@ def test_replication_slots(self, os_ops: OsOperations): with pytest.raises(expected_exception=testgres_TestgresException): node.replicate(slot='slot1') - def test_incorrect_catchup(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) - with __class__.helper__get_node(os_ops) as node: + def test_incorrect_catchup(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc) as node: node.init(allow_streaming=True).start() # node has no master, can't catch up with pytest.raises(expected_exception=testgres_TestgresException): node.catchup() - def test_promotion(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) - with __class__.helper__get_node(os_ops) as master: + def test_promotion(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc) as master: master.init().start() master.safe_psql('create table abc(id serial)') @@ -1046,17 +1055,17 @@ def test_promotion(self, os_ops: OsOperations): res = replica.safe_psql('select * from abc') assert (__class__.helper__rm_carriage_returns(res) == b'1\n') - def test_dump(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) + def test_dump(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) query_create = 'create table test as select generate_series(1, 2) as val' query_select = 'select * from test order by val asc' - with __class__.helper__get_node(os_ops).init().start() as node1: + with __class__.helper__get_node(node_svc).init().start() as node1: node1.execute(query_create) for format in ['plain', 'custom', 'directory', 'tar']: - with removing(os_ops, node1.dump(format=format)) as dump: - with __class__.helper__get_node(os_ops).init().start() as node3: + with removing(node_svc.os_ops, node1.dump(format=format)) as dump: + with __class__.helper__get_node(node_svc).init().start() as node3: if format == 'directory': assert (os.path.isdir(dump)) else: @@ -1066,14 +1075,16 @@ def test_dump(self, os_ops: OsOperations): res = node3.execute(query_select) assert (res == [(1, ), (2, )]) - def test_get_pg_config2(self, os_ops: OsOperations): + def test_get_pg_config2(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + # check same instances - a = get_pg_config2(os_ops, None) - b = get_pg_config2(os_ops, None) + a = get_pg_config2(node_svc.os_ops, None) + b = get_pg_config2(node_svc.os_ops, None) assert (id(a) == id(b)) # save right before config change - c1 = get_pg_config2(os_ops, None) + c1 = get_pg_config2(node_svc.os_ops, None) # modify setting for this scope with scoped_config(cache_pg_config=False) as config: @@ -1081,20 +1092,315 @@ def test_get_pg_config2(self, os_ops: OsOperations): assert not (config.cache_pg_config) # save right after config change - c2 = get_pg_config2(os_ops, None) + c2 = get_pg_config2(node_svc.os_ops, None) # check different instances after config change assert (id(c1) != id(c2)) # check different instances - a = get_pg_config2(os_ops, None) - b = get_pg_config2(os_ops, None) + a = get_pg_config2(node_svc.os_ops, None) + b = get_pg_config2(node_svc.os_ops, None) assert (id(a) != id(b)) + def test_pgbench(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + + __class__.helper__skip_test_if_util_not_exist(node_svc.os_ops, "pgbench") + + with __class__.helper__get_node(node_svc).init().start() as node: + # initialize pgbench DB and run benchmarks + node.pgbench_init( + scale=2, + foreign_keys=True, + options=['-q'] + ).pgbench_run(time=2) + + # run TPC-B benchmark + proc = node.pgbench(stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=['-T3']) + out = proc.communicate()[0] + assert (b'tps = ' in out) + + def test_unix_sockets(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + + with __class__.helper__get_node(node_svc) as node: + node.init(unix_sockets=False, allow_streaming=True) + node.start() + + res_exec = node.execute('select 1') + assert (res_exec == [(1,)]) + res_psql = node.safe_psql('select 1') + assert (res_psql == b'1\n') + + with node.replicate() as r: + assert type(r) == PostgresNode # noqa: E721 + r.start() + res_exec = r.execute('select 1') + assert (res_exec == [(1,)]) + res_psql = r.safe_psql('select 1') + assert (res_psql == b'1\n') + + def test_the_same_port(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + + with __class__.helper__get_node(node_svc) as node: + node.init().start() + assert (node._should_free_port) + assert (type(node.port) == int) # noqa: E721 + node_port_copy = node.port + r = node.safe_psql("SELECT 1;") + assert (__class__.helper__rm_carriage_returns(r) == b'1\n') + + with __class__.helper__get_node(node_svc, port=node.port) as node2: + assert (type(node2.port) == int) # noqa: E721 + assert (node2.port == node.port) + assert not (node2._should_free_port) + + with pytest.raises( + expected_exception=StartNodeException, + match=re.escape("Cannot start node") + ): + node2.init().start() + + # node is still working + assert (node.port == node_port_copy) + assert (node._should_free_port) + r = node.safe_psql("SELECT 3;") + assert (__class__.helper__rm_carriage_returns(r) == b'3\n') + + class tagPortManagerProxy(PortManager): + m_PrevPortManager: PortManager + + m_DummyPortNumber: int + m_DummyPortMaxUsage: int + + m_DummyPortCurrentUsage: int + m_DummyPortTotalUsage: int + + def __init__(self, prevPortManager: PortManager, dummyPortNumber: int, dummyPortMaxUsage: int): + assert isinstance(prevPortManager, PortManager) + assert type(dummyPortNumber) == int # noqa: E721 + assert type(dummyPortMaxUsage) == int # noqa: E721 + assert dummyPortNumber >= 0 + assert dummyPortMaxUsage >= 0 + + super().__init__() + + self.m_PrevPortManager = prevPortManager + + self.m_DummyPortNumber = dummyPortNumber + self.m_DummyPortMaxUsage = dummyPortMaxUsage + + self.m_DummyPortCurrentUsage = 0 + self.m_DummyPortTotalUsage = 0 + + def __enter__(self): + return self + + def __exit__(self, type, value, traceback): + assert self.m_DummyPortCurrentUsage == 0 + + assert self.m_PrevPortManager is not None + + def reserve_port(self) -> int: + assert type(self.m_DummyPortMaxUsage) == int # noqa: E721 + assert type(self.m_DummyPortTotalUsage) == int # noqa: E721 + assert type(self.m_DummyPortCurrentUsage) == int # noqa: E721 + assert self.m_DummyPortTotalUsage >= 0 + assert self.m_DummyPortCurrentUsage >= 0 + + assert self.m_DummyPortTotalUsage <= self.m_DummyPortMaxUsage + assert self.m_DummyPortCurrentUsage <= self.m_DummyPortTotalUsage + + assert self.m_PrevPortManager is not None + assert isinstance(self.m_PrevPortManager, PortManager) + + if self.m_DummyPortTotalUsage == self.m_DummyPortMaxUsage: + return self.m_PrevPortManager.reserve_port() + + self.m_DummyPortTotalUsage += 1 + self.m_DummyPortCurrentUsage += 1 + return self.m_DummyPortNumber + + def release_port(self, dummyPortNumber: int): + assert type(dummyPortNumber) == int # noqa: E721 + + assert type(self.m_DummyPortMaxUsage) == int # noqa: E721 + assert type(self.m_DummyPortTotalUsage) == int # noqa: E721 + assert type(self.m_DummyPortCurrentUsage) == int # noqa: E721 + assert self.m_DummyPortTotalUsage >= 0 + assert self.m_DummyPortCurrentUsage >= 0 + + assert self.m_DummyPortTotalUsage <= self.m_DummyPortMaxUsage + assert self.m_DummyPortCurrentUsage <= self.m_DummyPortTotalUsage + + assert self.m_PrevPortManager is not None + assert isinstance(self.m_PrevPortManager, PortManager) + + if self.m_DummyPortCurrentUsage > 0 and dummyPortNumber == self.m_DummyPortNumber: + assert self.m_DummyPortTotalUsage > 0 + self.m_DummyPortCurrentUsage -= 1 + return + + return self.m_PrevPortManager.release_port(dummyPortNumber) + + def test_port_rereserve_during_node_start(self, node_svc: PostgresNodeService): + assert type(node_svc) == PostgresNodeService # noqa: E721 + assert PostgresNode._C_MAX_START_ATEMPTS == 5 + + C_COUNT_OF_BAD_PORT_USAGE = 3 + + with __class__.helper__get_node(node_svc) as node1: + node1.init().start() + assert node1._should_free_port + assert type(node1.port) == int # noqa: E721 + node1_port_copy = node1.port + assert __class__.helper__rm_carriage_returns(node1.safe_psql("SELECT 1;")) == b'1\n' + + with __class__.tagPortManagerProxy(node_svc.port_manager, node1.port, C_COUNT_OF_BAD_PORT_USAGE) as proxy: + assert proxy.m_DummyPortNumber == node1.port + with __class__.helper__get_node(node_svc, port_manager=proxy) as node2: + assert node2._should_free_port + assert node2.port == node1.port + + node2.init().start() + + assert node2.port != node1.port + assert node2._should_free_port + assert proxy.m_DummyPortCurrentUsage == 0 + assert proxy.m_DummyPortTotalUsage == C_COUNT_OF_BAD_PORT_USAGE + assert node2.is_started + r = node2.safe_psql("SELECT 2;") + assert __class__.helper__rm_carriage_returns(r) == b'2\n' + + # node1 is still working + assert node1.port == node1_port_copy + assert node1._should_free_port + r = node1.safe_psql("SELECT 3;") + assert __class__.helper__rm_carriage_returns(r) == b'3\n' + + def test_port_conflict(self, node_svc: PostgresNodeService): + assert type(node_svc) == PostgresNodeService # noqa: E721 + assert PostgresNode._C_MAX_START_ATEMPTS > 1 + + C_COUNT_OF_BAD_PORT_USAGE = PostgresNode._C_MAX_START_ATEMPTS + + with __class__.helper__get_node(node_svc) as node1: + node1.init().start() + assert node1._should_free_port + assert type(node1.port) == int # noqa: E721 + node1_port_copy = node1.port + assert __class__.helper__rm_carriage_returns(node1.safe_psql("SELECT 1;")) == b'1\n' + + with __class__.tagPortManagerProxy(node_svc.port_manager, node1.port, C_COUNT_OF_BAD_PORT_USAGE) as proxy: + assert proxy.m_DummyPortNumber == node1.port + with __class__.helper__get_node(node_svc, port_manager=proxy) as node2: + assert node2._should_free_port + assert node2.port == node1.port + + with pytest.raises( + expected_exception=StartNodeException, + match=re.escape("Cannot start node after multiple attempts.") + ): + node2.init().start() + + assert node2.port == node1.port + assert node2._should_free_port + assert proxy.m_DummyPortCurrentUsage == 1 + assert proxy.m_DummyPortTotalUsage == C_COUNT_OF_BAD_PORT_USAGE + assert not node2.is_started + + # node2 must release our dummyPort (node1.port) + assert (proxy.m_DummyPortCurrentUsage == 0) + + # node1 is still working + assert node1.port == node1_port_copy + assert node1._should_free_port + r = node1.safe_psql("SELECT 3;") + assert __class__.helper__rm_carriage_returns(r) == b'3\n' + + def test_try_to_get_port_after_free_manual_port(self, node_svc: PostgresNodeService): + assert type(node_svc) == PostgresNodeService # noqa: E721 + + assert node_svc.port_manager is not None + assert isinstance(node_svc.port_manager, PortManager) + + with __class__.helper__get_node(node_svc) as node1: + assert node1 is not None + assert type(node1) == PostgresNode # noqa: E721 + assert node1.port is not None + assert type(node1.port) == int # noqa: E721 + with __class__.helper__get_node(node_svc, port=node1.port, port_manager=None) as node2: + assert node2 is not None + assert type(node1) == PostgresNode # noqa: E721 + assert node2 is not node1 + assert node2.port is not None + assert type(node2.port) == int # noqa: E721 + assert node2.port == node1.port + + logging.info("Release node2 port") + node2.free_port() + + logging.info("try to get node2.port...") + with pytest.raises( + InvalidOperationException, + match="^" + re.escape("PostgresNode port is not defined.") + "$" + ): + p = node2.port + assert p is None + + def test_try_to_start_node_after_free_manual_port(self, node_svc: PostgresNodeService): + assert type(node_svc) == PostgresNodeService # noqa: E721 + + assert node_svc.port_manager is not None + assert isinstance(node_svc.port_manager, PortManager) + + with __class__.helper__get_node(node_svc) as node1: + assert node1 is not None + assert type(node1) == PostgresNode # noqa: E721 + assert node1.port is not None + assert type(node1.port) == int # noqa: E721 + with __class__.helper__get_node(node_svc, port=node1.port, port_manager=None) as node2: + assert node2 is not None + assert type(node1) == PostgresNode # noqa: E721 + assert node2 is not node1 + assert node2.port is not None + assert type(node2.port) == int # noqa: E721 + assert node2.port == node1.port + + logging.info("Release node2 port") + node2.free_port() + + logging.info("node2 is trying to start...") + with pytest.raises( + InvalidOperationException, + match="^" + re.escape("Can't start PostgresNode. Port is not defined.") + "$" + ): + node2.start() + @staticmethod - def helper__get_node(os_ops: OsOperations, name=None): - assert isinstance(os_ops, OsOperations) - return PostgresNode(name, conn_params=None, os_ops=os_ops) + def helper__get_node( + node_svc: PostgresNodeService, + name: typing.Optional[str] = None, + port: typing.Optional[int] = None, + port_manager: typing.Optional[PortManager] = None + ) -> PostgresNode: + assert isinstance(node_svc, PostgresNodeService) + assert isinstance(node_svc.os_ops, OsOperations) + assert isinstance(node_svc.port_manager, PortManager) + + if port_manager is None: + port_manager = node_svc.port_manager + + return PostgresNode( + name, + port=port, + conn_params=None, + os_ops=node_svc.os_ops, + port_manager=port_manager if port is None else None + ) @staticmethod def helper__skip_test_if_pg_version_is_not_ge(ver1: str, ver2: str): diff --git a/tests/test_testgres_local.py b/tests/test_testgres_local.py index 01f975a0..bef80d0f 100644 --- a/tests/test_testgres_local.py +++ b/tests/test_testgres_local.py @@ -100,27 +100,6 @@ def test_custom_init(self): # there should be no trust entries at all assert not (any('trust' in s for s in lines)) - def test_pgbench(self): - __class__.helper__skip_test_if_util_not_exist("pgbench") - - with get_new_node().init().start() as node: - - # initialize pgbench DB and run benchmarks - node.pgbench_init(scale=2, foreign_keys=True, - options=['-q']).pgbench_run(time=2) - - # run TPC-B benchmark - proc = node.pgbench(stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - options=['-T3']) - - out, _ = proc.communicate() - out = out.decode('utf-8') - - proc.stdout.close() - - assert ('tps' in out) - def test_pg_config(self): # check same instances a = get_pg_config() @@ -177,18 +156,6 @@ def test_config_stack(self): assert (TestgresConfig.cached_initdb_dir == d0) - def test_unix_sockets(self): - with get_new_node() as node: - node.init(unix_sockets=False, allow_streaming=True) - node.start() - - node.execute('select 1') - node.safe_psql('select 1') - - with node.replicate().start() as r: - r.execute('select 1') - r.safe_psql('select 1') - def test_ports_management(self): assert bound_ports is not None assert type(bound_ports) == set # noqa: E721 @@ -277,30 +244,6 @@ def test_parse_pg_version(self): # Macos assert parse_pg_version("postgres (PostgreSQL) 14.9 (Homebrew)") == "14.9" - def test_the_same_port(self): - with get_new_node() as node: - node.init().start() - assert (node._should_free_port) - assert (type(node.port) == int) # noqa: E721 - node_port_copy = node.port - assert (rm_carriage_returns(node.safe_psql("SELECT 1;")) == b'1\n') - - with get_new_node(port=node.port) as node2: - assert (type(node2.port) == int) # noqa: E721 - assert (node2.port == node.port) - assert not (node2._should_free_port) - - with pytest.raises( - expected_exception=StartNodeException, - match=re.escape("Cannot start node") - ): - node2.init().start() - - # node is still working - assert (node.port == node_port_copy) - assert (node._should_free_port) - assert (rm_carriage_returns(node.safe_psql("SELECT 3;")) == b'3\n') - class tagPortManagerProxy: sm_prev_testgres_reserve_port = None sm_prev_testgres_release_port = None diff --git a/tests/test_testgres_remote.py b/tests/test_testgres_remote.py index 2142e5ba..ef4bd0c8 100755 --- a/tests/test_testgres_remote.py +++ b/tests/test_testgres_remote.py @@ -1,14 +1,12 @@ # coding: utf-8 import os import re -import subprocess import pytest -import psutil import logging -from .helpers.os_ops_descrs import OsOpsDescrs -from .helpers.os_ops_descrs import OsOperations +from .helpers.global_data import PostgresNodeService +from .helpers.global_data import PostgresNodeServices from .. import testgres @@ -27,8 +25,6 @@ get_pg_config # NOTE: those are ugly imports -from ..testgres import bound_ports -from ..testgres.node import ProcessProxy def util_exists(util): @@ -48,17 +44,17 @@ def good_properties(f): class TestTestgresRemote: - sm_os_ops = OsOpsDescrs.sm_remote_os_ops - @pytest.fixture(autouse=True, scope="class") def implicit_fixture(self): + cur_os_ops = PostgresNodeServices.sm_remote.os_ops + assert cur_os_ops is not None + prev_ops = testgres_config.os_ops assert prev_ops is not None - assert __class__.sm_os_ops is not None - testgres_config.set_os_ops(os_ops=__class__.sm_os_ops) - assert testgres_config.os_ops is __class__.sm_os_ops + testgres_config.set_os_ops(os_ops=cur_os_ops) + assert testgres_config.os_ops is cur_os_ops yield - assert testgres_config.os_ops is __class__.sm_os_ops + assert testgres_config.os_ops is cur_os_ops testgres_config.set_os_ops(os_ops=prev_ops) assert testgres_config.os_ops is prev_ops @@ -172,21 +168,6 @@ def test_init__unk_LANG_and_LC_CTYPE(self): __class__.helper__restore_envvar("LC_CTYPE", prev_LC_CTYPE) __class__.helper__restore_envvar("LC_COLLATE", prev_LC_COLLATE) - def test_pgbench(self): - __class__.helper__skip_test_if_util_not_exist("pgbench") - - with __class__.helper__get_node().init().start() as node: - # initialize pgbench DB and run benchmarks - node.pgbench_init(scale=2, foreign_keys=True, - options=['-q']).pgbench_run(time=2) - - # run TPC-B benchmark - proc = node.pgbench(stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - options=['-T3']) - out = proc.communicate()[0] - assert (b'tps = ' in out) - def test_pg_config(self): # check same instances a = get_pg_config() @@ -243,90 +224,19 @@ def test_config_stack(self): assert (TestgresConfig.cached_initdb_dir == d0) - def test_unix_sockets(self): - with __class__.helper__get_node() as node: - node.init(unix_sockets=False, allow_streaming=True) - node.start() - - res_exec = node.execute('select 1') - res_psql = node.safe_psql('select 1') - assert (res_exec == [(1,)]) - assert (res_psql == b'1\n') - - with node.replicate().start() as r: - res_exec = r.execute('select 1') - res_psql = r.safe_psql('select 1') - assert (res_exec == [(1,)]) - assert (res_psql == b'1\n') - - def test_ports_management(self): - assert bound_ports is not None - assert type(bound_ports) == set # noqa: E721 - - if len(bound_ports) != 0: - logging.warning("bound_ports is not empty: {0}".format(bound_ports)) - - stage0__bound_ports = bound_ports.copy() - - with __class__.helper__get_node() as node: - assert bound_ports is not None - assert type(bound_ports) == set # noqa: E721 - - assert node.port is not None - assert type(node.port) == int # noqa: E721 - - logging.info("node port is {0}".format(node.port)) - - assert node.port in bound_ports - assert node.port not in stage0__bound_ports - - assert stage0__bound_ports <= bound_ports - assert len(stage0__bound_ports) + 1 == len(bound_ports) - - stage1__bound_ports = stage0__bound_ports.copy() - stage1__bound_ports.add(node.port) - - assert stage1__bound_ports == bound_ports - - # check that port has been freed successfully - assert bound_ports is not None - assert type(bound_ports) == set # noqa: E721 - assert bound_ports == stage0__bound_ports - - # TODO: Why does not this test work with remote host? - def test_child_process_dies(self): - nAttempt = 0 - - while True: - if nAttempt == 5: - raise Exception("Max attempt number is exceed.") - - nAttempt += 1 - - logging.info("Attempt #{0}".format(nAttempt)) - - # test for FileNotFound exception during child_processes() function - with subprocess.Popen(["sleep", "60"]) as process: - r = process.poll() - - if r is not None: - logging.warning("process.pool() returns an unexpected result: {0}.".format(r)) - continue - - assert r is None - # collect list of processes currently running - children = psutil.Process(os.getpid()).children() - # kill a process, so received children dictionary becomes invalid - process.kill() - process.wait() - # try to handle children list -- missing processes will have ptype "ProcessType.Unknown" - [ProcessProxy(p) for p in children] - break - @staticmethod def helper__get_node(name=None): - assert isinstance(__class__.sm_os_ops, OsOperations) - return testgres.PostgresNode(name, conn_params=None, os_ops=__class__.sm_os_ops) + svc = PostgresNodeServices.sm_remote + + assert isinstance(svc, PostgresNodeService) + assert isinstance(svc.os_ops, testgres.OsOperations) + assert isinstance(svc.port_manager, testgres.PortManager) + + return testgres.PostgresNode( + name, + conn_params=None, + os_ops=svc.os_ops, + port_manager=svc.port_manager) @staticmethod def helper__restore_envvar(name, prev_value): From 307ef5fc523d2156d711b1493ee8d7d98f24ba82 Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Mon, 7 Apr 2025 07:31:53 +0300 Subject: [PATCH 453/475] Test refactoring (#236) * TestUtils is added * TestTestgresCommon::test_node_repr is added * test_get_pg_config2 moved to TestUtils * TestTestgresCommon.test_custom_init is added * TestConfig is added --- tests/test_config.py | 41 +++++++++++++++ tests/test_testgres_common.py | 54 ++++++++++---------- tests/test_testgres_local.py | 96 ++++------------------------------- tests/test_testgres_remote.py | 73 +++----------------------- tests/test_utils.py | 62 ++++++++++++++++++++++ 5 files changed, 145 insertions(+), 181 deletions(-) create mode 100644 tests/test_config.py create mode 100644 tests/test_utils.py diff --git a/tests/test_config.py b/tests/test_config.py new file mode 100644 index 00000000..05702e9a --- /dev/null +++ b/tests/test_config.py @@ -0,0 +1,41 @@ +from ..testgres import TestgresConfig +from ..testgres import configure_testgres +from ..testgres import scoped_config +from ..testgres import pop_config + +from .. import testgres + +import pytest + + +class TestConfig: + def test_config_stack(self): + # no such option + with pytest.raises(expected_exception=TypeError): + configure_testgres(dummy=True) + + # we have only 1 config in stack + with pytest.raises(expected_exception=IndexError): + pop_config() + + d0 = TestgresConfig.cached_initdb_dir + d1 = 'dummy_abc' + d2 = 'dummy_def' + + with scoped_config(cached_initdb_dir=d1) as c1: + assert (c1.cached_initdb_dir == d1) + + with scoped_config(cached_initdb_dir=d2) as c2: + stack_size = len(testgres.config.config_stack) + + # try to break a stack + with pytest.raises(expected_exception=TypeError): + with scoped_config(dummy=True): + pass + + assert (c2.cached_initdb_dir == d2) + assert (len(testgres.config.config_stack) == stack_size) + + assert (c1.cached_initdb_dir == d1) + + assert (TestgresConfig.cached_initdb_dir == d0) diff --git a/tests/test_testgres_common.py b/tests/test_testgres_common.py index b286a1c6..c384dfb2 100644 --- a/tests/test_testgres_common.py +++ b/tests/test_testgres_common.py @@ -6,7 +6,6 @@ from ..testgres.node import PgVer from ..testgres.node import PostgresNode from ..testgres.utils import get_pg_version2 -from ..testgres.utils import get_pg_config2 from ..testgres.utils import file_tail from ..testgres.utils import get_bin_path2 from ..testgres import ProcessType @@ -106,6 +105,32 @@ def test_version_management(self, node_svc: PostgresNodeService): assert (isinstance(node.version, PgVer)) assert (node.version == PgVer(version)) + def test_node_repr(self, node_svc: PostgresNodeService): + with __class__.helper__get_node(node_svc).init() as node: + pattern = r"PostgresNode\(name='.+', port=.+, base_dir='.+'\)" + assert re.match(pattern, str(node)) is not None + + def test_custom_init(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + + with __class__.helper__get_node(node_svc) as node: + # enable page checksums + node.init(initdb_params=['-k']).start() + + with __class__.helper__get_node(node_svc) as node: + node.init( + allow_streaming=True, + initdb_params=['--auth-local=reject', '--auth-host=reject']) + + hba_file = os.path.join(node.data_dir, 'pg_hba.conf') + lines = node.os_ops.readlines(hba_file) + + # check number of lines + assert (len(lines) >= 6) + + # there should be no trust entries at all + assert not (any('trust' in s for s in lines)) + def test_double_init(self, node_svc: PostgresNodeService): assert isinstance(node_svc, PostgresNodeService) @@ -1075,33 +1100,6 @@ def test_dump(self, node_svc: PostgresNodeService): res = node3.execute(query_select) assert (res == [(1, ), (2, )]) - def test_get_pg_config2(self, node_svc: PostgresNodeService): - assert isinstance(node_svc, PostgresNodeService) - - # check same instances - a = get_pg_config2(node_svc.os_ops, None) - b = get_pg_config2(node_svc.os_ops, None) - assert (id(a) == id(b)) - - # save right before config change - c1 = get_pg_config2(node_svc.os_ops, None) - - # modify setting for this scope - with scoped_config(cache_pg_config=False) as config: - # sanity check for value - assert not (config.cache_pg_config) - - # save right after config change - c2 = get_pg_config2(node_svc.os_ops, None) - - # check different instances after config change - assert (id(c1) != id(c2)) - - # check different instances - a = get_pg_config2(node_svc.os_ops, None) - b = get_pg_config2(node_svc.os_ops, None) - assert (id(a) != id(b)) - def test_pgbench(self, node_svc: PostgresNodeService): assert isinstance(node_svc, PostgresNodeService) diff --git a/tests/test_testgres_local.py b/tests/test_testgres_local.py index bef80d0f..9dbd455b 100644 --- a/tests/test_testgres_local.py +++ b/tests/test_testgres_local.py @@ -9,28 +9,18 @@ from .. import testgres -from ..testgres import \ - StartNodeException, \ - ExecUtilException, \ - NodeApp - -from ..testgres import \ - TestgresConfig, \ - configure_testgres, \ - scoped_config, \ - pop_config - -from ..testgres import \ - get_new_node - -from ..testgres import \ - get_bin_path, \ - get_pg_config, \ - get_pg_version +from ..testgres import StartNodeException +from ..testgres import ExecUtilException +from ..testgres import NodeApp +from ..testgres import scoped_config +from ..testgres import get_new_node +from ..testgres import get_bin_path +from ..testgres import get_pg_config +from ..testgres import get_pg_version # NOTE: those are ugly imports -from ..testgres import bound_ports -from ..testgres.utils import PgVer, parse_pg_version +from ..testgres.utils import bound_ports +from ..testgres.utils import PgVer from ..testgres.node import ProcessProxy @@ -75,31 +65,6 @@ def rm_carriage_returns(out): class TestTestgresLocal: - def test_node_repr(self): - with get_new_node() as node: - pattern = r"PostgresNode\(name='.+', port=.+, base_dir='.+'\)" - assert re.match(pattern, str(node)) is not None - - def test_custom_init(self): - with get_new_node() as node: - # enable page checksums - node.init(initdb_params=['-k']).start() - - with get_new_node() as node: - node.init( - allow_streaming=True, - initdb_params=['--auth-local=reject', '--auth-host=reject']) - - hba_file = os.path.join(node.data_dir, 'pg_hba.conf') - with open(hba_file, 'r') as conf: - lines = conf.readlines() - - # check number of lines - assert (len(lines) >= 6) - - # there should be no trust entries at all - assert not (any('trust' in s for s in lines)) - def test_pg_config(self): # check same instances a = get_pg_config() @@ -125,37 +90,6 @@ def test_pg_config(self): b = get_pg_config() assert (id(a) != id(b)) - def test_config_stack(self): - # no such option - with pytest.raises(expected_exception=TypeError): - configure_testgres(dummy=True) - - # we have only 1 config in stack - with pytest.raises(expected_exception=IndexError): - pop_config() - - d0 = TestgresConfig.cached_initdb_dir - d1 = 'dummy_abc' - d2 = 'dummy_def' - - with scoped_config(cached_initdb_dir=d1) as c1: - assert (c1.cached_initdb_dir == d1) - - with scoped_config(cached_initdb_dir=d2) as c2: - stack_size = len(testgres.config.config_stack) - - # try to break a stack - with pytest.raises(expected_exception=TypeError): - with scoped_config(dummy=True): - pass - - assert (c2.cached_initdb_dir == d2) - assert (len(testgres.config.config_stack) == stack_size) - - assert (c1.cached_initdb_dir == d1) - - assert (TestgresConfig.cached_initdb_dir == d0) - def test_ports_management(self): assert bound_ports is not None assert type(bound_ports) == set # noqa: E721 @@ -234,16 +168,6 @@ def test_upgrade_node(self): node_new.start() assert (b'Upgrade Complete' in res) - def test_parse_pg_version(self): - # Linux Mint - assert parse_pg_version("postgres (PostgreSQL) 15.5 (Ubuntu 15.5-1.pgdg22.04+1)") == "15.5" - # Linux Ubuntu - assert parse_pg_version("postgres (PostgreSQL) 12.17") == "12.17" - # Windows - assert parse_pg_version("postgres (PostgreSQL) 11.4") == "11.4" - # Macos - assert parse_pg_version("postgres (PostgreSQL) 14.9 (Homebrew)") == "14.9" - class tagPortManagerProxy: sm_prev_testgres_reserve_port = None sm_prev_testgres_release_port = None diff --git a/tests/test_testgres_remote.py b/tests/test_testgres_remote.py index ef4bd0c8..e38099b7 100755 --- a/tests/test_testgres_remote.py +++ b/tests/test_testgres_remote.py @@ -1,6 +1,5 @@ # coding: utf-8 import os -import re import pytest import logging @@ -10,19 +9,14 @@ from .. import testgres -from ..testgres.exceptions import \ - InitNodeException, \ - ExecUtilException +from ..testgres.exceptions import InitNodeException +from ..testgres.exceptions import ExecUtilException -from ..testgres.config import \ - TestgresConfig, \ - configure_testgres, \ - scoped_config, \ - pop_config, testgres_config +from ..testgres.config import scoped_config +from ..testgres.config import testgres_config -from ..testgres import \ - get_bin_path, \ - get_pg_config +from ..testgres import get_bin_path +from ..testgres import get_pg_config # NOTE: those are ugly imports @@ -58,30 +52,6 @@ def implicit_fixture(self): testgres_config.set_os_ops(os_ops=prev_ops) assert testgres_config.os_ops is prev_ops - def test_node_repr(self): - with __class__.helper__get_node() as node: - pattern = r"PostgresNode\(name='.+', port=.+, base_dir='.+'\)" - assert re.match(pattern, str(node)) is not None - - def test_custom_init(self): - with __class__.helper__get_node() as node: - # enable page checksums - node.init(initdb_params=['-k']).start() - - with __class__.helper__get_node() as node: - node.init( - allow_streaming=True, - initdb_params=['--auth-local=reject', '--auth-host=reject']) - - hba_file = os.path.join(node.data_dir, 'pg_hba.conf') - lines = node.os_ops.readlines(hba_file) - - # check number of lines - assert (len(lines) >= 6) - - # there should be no trust entries at all - assert not (any('trust' in s for s in lines)) - def test_init__LANG_С(self): # PBCKP-1744 prev_LANG = os.environ.get("LANG") @@ -193,37 +163,6 @@ def test_pg_config(self): b = get_pg_config() assert (id(a) != id(b)) - def test_config_stack(self): - # no such option - with pytest.raises(expected_exception=TypeError): - configure_testgres(dummy=True) - - # we have only 1 config in stack - with pytest.raises(expected_exception=IndexError): - pop_config() - - d0 = TestgresConfig.cached_initdb_dir - d1 = 'dummy_abc' - d2 = 'dummy_def' - - with scoped_config(cached_initdb_dir=d1) as c1: - assert (c1.cached_initdb_dir == d1) - - with scoped_config(cached_initdb_dir=d2) as c2: - stack_size = len(testgres.config.config_stack) - - # try to break a stack - with pytest.raises(expected_exception=TypeError): - with scoped_config(dummy=True): - pass - - assert (c2.cached_initdb_dir == d2) - assert (len(testgres.config.config_stack) == stack_size) - - assert (c1.cached_initdb_dir == d1) - - assert (TestgresConfig.cached_initdb_dir == d0) - @staticmethod def helper__get_node(name=None): svc = PostgresNodeServices.sm_remote diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 00000000..d4a4c9ad --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,62 @@ +from .helpers.global_data import OsOpsDescr +from .helpers.global_data import OsOpsDescrs +from .helpers.global_data import OsOperations + +from ..testgres.utils import parse_pg_version +from ..testgres.utils import get_pg_config2 +from ..testgres import scoped_config + +import pytest + + +class TestUtils: + sm_os_ops_descrs: list[OsOpsDescr] = [ + OsOpsDescrs.sm_local_os_ops_descr, + OsOpsDescrs.sm_remote_os_ops_descr + ] + + @pytest.fixture( + params=[descr.os_ops for descr in sm_os_ops_descrs], + ids=[descr.sign for descr in sm_os_ops_descrs] + ) + def os_ops(self, request: pytest.FixtureRequest) -> OsOperations: + assert isinstance(request, pytest.FixtureRequest) + assert isinstance(request.param, OsOperations) + return request.param + + def test_parse_pg_version(self): + # Linux Mint + assert parse_pg_version("postgres (PostgreSQL) 15.5 (Ubuntu 15.5-1.pgdg22.04+1)") == "15.5" + # Linux Ubuntu + assert parse_pg_version("postgres (PostgreSQL) 12.17") == "12.17" + # Windows + assert parse_pg_version("postgres (PostgreSQL) 11.4") == "11.4" + # Macos + assert parse_pg_version("postgres (PostgreSQL) 14.9 (Homebrew)") == "14.9" + + def test_get_pg_config2(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + + # check same instances + a = get_pg_config2(os_ops, None) + b = get_pg_config2(os_ops, None) + assert (id(a) == id(b)) + + # save right before config change + c1 = get_pg_config2(os_ops, None) + + # modify setting for this scope + with scoped_config(cache_pg_config=False) as config: + # sanity check for value + assert not (config.cache_pg_config) + + # save right after config change + c2 = get_pg_config2(os_ops, None) + + # check different instances after config change + assert (id(c1) != id(c2)) + + # check different instances + a = get_pg_config2(os_ops, None) + b = get_pg_config2(os_ops, None) + assert (id(a) != id(b)) From da2c493473fb124612a8e2f1baa74ac6b6ff980e Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Mon, 7 Apr 2025 12:38:03 +0300 Subject: [PATCH 454/475] OsOperation::db_connect is removed (#237) * OsOperation::db_connect is removed OsOperation does not work with databases. It provides an only OS functional. TODO: CI must explicitly test pg8000 and psycopg2. --- testgres/connection.py | 12 +++++++----- testgres/operations/local_ops.py | 13 +------------ testgres/operations/os_ops.py | 12 ------------ testgres/operations/remote_ops.py | 20 -------------------- 4 files changed, 8 insertions(+), 49 deletions(-) diff --git a/testgres/connection.py b/testgres/connection.py index ccedd135..b8dc49a9 100644 --- a/testgres/connection.py +++ b/testgres/connection.py @@ -42,11 +42,13 @@ def __init__(self, self._node = node - self._connection = node.os_ops.db_connect(dbname=dbname, - user=username, - password=password, - host=node.host, - port=node.port) + self._connection = pglib.connect( + database=dbname, + user=username, + password=password, + host=node.host, + port=node.port + ) self._connection.autocommit = autocommit self._cursor = self.connection.cursor() diff --git a/testgres/operations/local_ops.py b/testgres/operations/local_ops.py index 39c81405..9785d462 100644 --- a/testgres/operations/local_ops.py +++ b/testgres/operations/local_ops.py @@ -12,7 +12,7 @@ from ..exceptions import ExecUtilException from ..exceptions import InvalidOperationException -from .os_ops import ConnectionParams, OsOperations, pglib, get_default_encoding +from .os_ops import ConnectionParams, OsOperations, get_default_encoding from .raise_error import RaiseError from .helpers import Helpers @@ -446,14 +446,3 @@ def is_port_free(self, number: int) -> bool: return True except OSError: return False - - # Database control - def db_connect(self, dbname, user, password=None, host="localhost", port=5432): - conn = pglib.connect( - host=host, - port=port, - database=dbname, - user=user, - password=password, - ) - return conn diff --git a/testgres/operations/os_ops.py b/testgres/operations/os_ops.py index 489a7cb2..d25e76bc 100644 --- a/testgres/operations/os_ops.py +++ b/testgres/operations/os_ops.py @@ -1,14 +1,6 @@ import getpass import locale -try: - import psycopg2 as pglib # noqa: F401 -except ImportError: - try: - import pg8000 as pglib # noqa: F401 - except ImportError: - raise ImportError("You must have psycopg2 or pg8000 modules installed") - class ConnectionParams: def __init__(self, host='127.0.0.1', port=None, ssh_key=None, username=None): @@ -130,7 +122,3 @@ def get_process_children(self, pid): def is_port_free(self, number: int): assert type(number) == int # noqa: E721 raise NotImplementedError() - - # Database control - def db_connect(self, dbname, user, password=None, host="localhost", port=5432): - raise NotImplementedError() diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index ee747e52..25d02f38 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -6,15 +6,6 @@ import io import logging -# we support both pg8000 and psycopg2 -try: - import psycopg2 as pglib -except ImportError: - try: - import pg8000 as pglib - except ImportError: - raise ImportError("You must have psycopg2 or pg8000 modules installed") - from ..exceptions import ExecUtilException from ..exceptions import InvalidOperationException from .os_ops import OsOperations, ConnectionParams, get_default_encoding @@ -677,17 +668,6 @@ def _is_port_free__process_1(error: str) -> bool: # return True - # Database control - def db_connect(self, dbname, user, password=None, host="localhost", port=5432): - conn = pglib.connect( - host=host, - port=port, - database=dbname, - user=user, - password=password, - ) - return conn - @staticmethod def _make_exec_env_list() -> list[str]: result = list[str]() From 24014744bd031e34c2e2490127c40613edec01d5 Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Thu, 10 Apr 2025 09:07:05 +0300 Subject: [PATCH 455/475] A support of Python v3.8 (#238) * A support of Python 3.8 [typing] Python 3.8 does not support tuple[...], list[...], set[...] and so on. We will use the analogues from typing package: typing.Tuple[...], typing.List[...] and typing.Set[...]. * [CI] Jobs for testing with python 3.8.0, 3.8.latest, 3.9.latest, 3.10.latest and 3.11.latest [std2-all][alpine] are added --- .travis.yml | 5 ++ Dockerfile--std2-all.tmpl | 96 +++++++++++++++++++++++++++++++ run_tests2.sh | 68 ++++++++++++++++++++++ testgres/operations/remote_ops.py | 5 +- testgres/port_manager.py | 9 +-- tests/conftest.py | 22 ++++--- tests/test_os_ops_common.py | 3 +- tests/test_testgres_common.py | 10 ++-- tests/test_utils.py | 3 +- 9 files changed, 199 insertions(+), 22 deletions(-) create mode 100644 Dockerfile--std2-all.tmpl create mode 100755 run_tests2.sh diff --git a/.travis.yml b/.travis.yml index 7557a2ce..55b7afa9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,6 +19,11 @@ notifications: on_failure: always env: + - TEST_PLATFORM=std2-all PYTHON_VERSION=3.8.0 PG_VERSION=17 + - TEST_PLATFORM=std2-all PYTHON_VERSION=3.8 PG_VERSION=17 + - TEST_PLATFORM=std2-all PYTHON_VERSION=3.9 PG_VERSION=17 + - TEST_PLATFORM=std2-all PYTHON_VERSION=3.10 PG_VERSION=17 + - TEST_PLATFORM=std2-all PYTHON_VERSION=3.11 PG_VERSION=17 - TEST_PLATFORM=std PYTHON_VERSION=3 PG_VERSION=16 - TEST_PLATFORM=std PYTHON_VERSION=3 PG_VERSION=15 - TEST_PLATFORM=std PYTHON_VERSION=3 PG_VERSION=14 diff --git a/Dockerfile--std2-all.tmpl b/Dockerfile--std2-all.tmpl new file mode 100644 index 00000000..10d8280c --- /dev/null +++ b/Dockerfile--std2-all.tmpl @@ -0,0 +1,96 @@ +ARG PG_VERSION +ARG PYTHON_VERSION + +# --------------------------------------------- base1 +FROM postgres:${PG_VERSION}-alpine as base1 + +# --------------------------------------------- base2_with_python-3 +FROM base1 as base2_with_python-3 +RUN apk add --no-cache curl python3 python3-dev build-base musl-dev linux-headers + +# For pyenv +RUN apk add patch +RUN apk add git +RUN apk add xz-dev +RUN apk add zip +RUN apk add zlib-dev +RUN apk add libffi-dev +RUN apk add readline-dev +RUN apk add openssl openssl-dev +RUN apk add sqlite-dev +RUN apk add bzip2-dev + +# --------------------------------------------- base3_with_python-3.8.0 +FROM base2_with_python-3 as base3_with_python-3.8.0 +ENV PYTHON_VERSION=3.8.0 + +# --------------------------------------------- base3_with_python-3.8 +FROM base2_with_python-3 as base3_with_python-3.8 +ENV PYTHON_VERSION=3.8 + +# --------------------------------------------- base3_with_python-3.9 +FROM base2_with_python-3 as base3_with_python-3.9 +ENV PYTHON_VERSION=3.9 + +# --------------------------------------------- base3_with_python-3.10 +FROM base2_with_python-3 as base3_with_python-3.10 +ENV PYTHON_VERSION=3.10 + +# --------------------------------------------- base3_with_python-3.11 +FROM base2_with_python-3 as base3_with_python-3.11 +ENV PYTHON_VERSION=3.11 + +# --------------------------------------------- final +FROM base3_with_python-${PYTHON_VERSION} as final + +#RUN apk add --no-cache mc + +# Full version of "ps" command +RUN apk add --no-cache procps + +RUN apk add --no-cache openssh +RUN apk add --no-cache sudo + +ENV LANG=C.UTF-8 + +RUN addgroup -S sudo +RUN adduser postgres sudo + +EXPOSE 22 +RUN ssh-keygen -A + +ADD . /pg/testgres +WORKDIR /pg/testgres +RUN chown -R postgres:postgres /pg + +# It allows to use sudo without password +RUN sh -c "echo \"postgres ALL=(ALL:ALL) NOPASSWD:ALL\"">>/etc/sudoers + +# THIS CMD IS NEEDED TO CONNECT THROUGH SSH WITHOUT PASSWORD +RUN sh -c "echo "postgres:*" | chpasswd -e" + +USER postgres + +RUN curl https://raw.githubusercontent.com/pyenv/pyenv-installer/master/bin/pyenv-installer | bash + +RUN ~/.pyenv/bin/pyenv install ${PYTHON_VERSION} + +# THIS CMD IS NEEDED TO CONNECT THROUGH SSH WITHOUT PASSWORD +RUN chmod 700 ~/ + +RUN mkdir -p ~/.ssh +#RUN chmod 700 ~/.ssh + +ENTRYPOINT sh -c " \ +set -eux; \ +echo HELLO FROM ENTRYPOINT; \ +echo HOME DIR IS [`realpath ~/`]; \ +ssh-keygen -t rsa -f ~/.ssh/id_rsa -q -N ''; \ +cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys; \ +chmod 600 ~/.ssh/authorized_keys; \ +ls -la ~/.ssh/; \ +sudo /usr/sbin/sshd; \ +ssh-keyscan -H localhost >> ~/.ssh/known_hosts; \ +ssh-keyscan -H 127.0.0.1 >> ~/.ssh/known_hosts; \ +export PATH=\"~/.pyenv/bin:$PATH\"; \ +TEST_FILTER=\"\" bash run_tests2.sh;" diff --git a/run_tests2.sh b/run_tests2.sh new file mode 100755 index 00000000..173b19dc --- /dev/null +++ b/run_tests2.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash + +# Copyright (c) 2017-2025 Postgres Professional + +set -eux + +eval "$(pyenv init -)" +eval "$(pyenv virtualenv-init -)" + +pyenv virtualenv --force ${PYTHON_VERSION} cur +pyenv activate cur + +if [ -z ${TEST_FILTER+x} ]; \ +then export TEST_FILTER="TestTestgresLocal or (TestTestgresCommon and (not remote))"; \ +fi + +# fail early +echo check that pg_config is in PATH +command -v pg_config + +# prepare python environment +VENV_PATH="/tmp/testgres_venv" +rm -rf $VENV_PATH +python -m venv "${VENV_PATH}" +export VIRTUAL_ENV_DISABLE_PROMPT=1 +source "${VENV_PATH}/bin/activate" +pip install coverage flake8 psutil Sphinx pytest pytest-xdist psycopg2 six psutil + +# install testgres' dependencies +export PYTHONPATH=$(pwd) +# $PIP install . + +# test code quality +flake8 . + + +# remove existing coverage file +export COVERAGE_FILE=.coverage +rm -f $COVERAGE_FILE + + +# run tests (PATH) +time coverage run -a -m pytest -l -v -n 4 -k "${TEST_FILTER}" + + +# run tests (PG_BIN) +PG_BIN=$(pg_config --bindir) \ +time coverage run -a -m pytest -l -v -n 4 -k "${TEST_FILTER}" + + +# run tests (PG_CONFIG) +PG_CONFIG=$(pg_config --bindir)/pg_config \ +time coverage run -a -m pytest -l -v -n 4 -k "${TEST_FILTER}" + + +# show coverage +coverage report + +# build documentation +cd docs +make html +cd .. + +# attempt to fix codecov +set +eux + +# send coverage stats to Codecov +bash <(curl -s https://codecov.io/bash) diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index 25d02f38..33b61ac2 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -5,6 +5,7 @@ import tempfile import io import logging +import typing from ..exceptions import ExecUtilException from ..exceptions import InvalidOperationException @@ -669,8 +670,8 @@ def _is_port_free__process_1(error: str) -> bool: return True @staticmethod - def _make_exec_env_list() -> list[str]: - result = list[str]() + def _make_exec_env_list() -> typing.List[str]: + result: typing.List[str] = list() for envvar in os.environ.items(): if not __class__._does_put_envvar_into_exec_cmd(envvar[0]): continue diff --git a/testgres/port_manager.py b/testgres/port_manager.py index 164661e7..e2530470 100644 --- a/testgres/port_manager.py +++ b/testgres/port_manager.py @@ -6,6 +6,7 @@ import threading import random +import typing class PortManager: @@ -50,16 +51,16 @@ class PortManager__Generic(PortManager): _os_ops: OsOperations _guard: object # TODO: is there better to use bitmap fot _available_ports? - _available_ports: set[int] - _reserved_ports: set[int] + _available_ports: typing.Set[int] + _reserved_ports: typing.Set[int] def __init__(self, os_ops: OsOperations): assert os_ops is not None assert isinstance(os_ops, OsOperations) self._os_ops = os_ops self._guard = threading.Lock() - self._available_ports = set[int](range(1024, 65535)) - self._reserved_ports = set[int]() + self._available_ports: typing.Set[int] = set(range(1024, 65535)) + self._reserved_ports: typing.Set[int] = set() def reserve_port(self) -> int: assert self._guard is not None diff --git a/tests/conftest.py b/tests/conftest.py index ff3b3cb4..6f2f9e41 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -26,6 +26,10 @@ class TestConfigPropNames: TEST_CFG__LOG_DIR = "TEST_CFG__LOG_DIR" +# ///////////////////////////////////////////////////////////////////////////// + +T_TUPLE__str_int = typing.Tuple[str, int] + # ///////////////////////////////////////////////////////////////////////////// # TestStartupData__Helper @@ -110,11 +114,11 @@ class TEST_PROCESS_STATS: cUnexpectedTests: int = 0 cAchtungTests: int = 0 - FailedTests = list[str, int]() - XFailedTests = list[str, int]() - NotXFailedTests = list[str]() - WarningTests = list[str, int]() - AchtungTests = list[str]() + FailedTests: typing.List[T_TUPLE__str_int] = list() + XFailedTests: typing.List[T_TUPLE__str_int] = list() + NotXFailedTests: typing.List[str] = list() + WarningTests: typing.List[T_TUPLE__str_int] = list() + AchtungTests: typing.List[str] = list() cTotalDuration: datetime.timedelta = datetime.timedelta() @@ -769,7 +773,7 @@ def helper__calc_W(n: int) -> int: # ------------------------------------------------------------------------ -def helper__print_test_list(tests: list[str]) -> None: +def helper__print_test_list(tests: typing.List[str]) -> None: assert type(tests) == list # noqa: E721 assert helper__calc_W(9) == 1 @@ -796,7 +800,7 @@ def helper__print_test_list(tests: list[str]) -> None: # ------------------------------------------------------------------------ -def helper__print_test_list2(tests: list[str, int]) -> None: +def helper__print_test_list2(tests: typing.List[T_TUPLE__str_int]) -> None: assert type(tests) == list # noqa: E721 assert helper__calc_W(9) == 1 @@ -843,7 +847,7 @@ def LOCAL__print_line1_with_header(header: str): assert header != "" logging.info(C_LINE1 + " [" + header + "]") - def LOCAL__print_test_list(header: str, test_count: int, test_list: list[str]): + def LOCAL__print_test_list(header: str, test_count: int, test_list: typing.List[str]): assert type(header) == str # noqa: E721 assert type(test_count) == int # noqa: E721 assert type(test_list) == list # noqa: E721 @@ -858,7 +862,7 @@ def LOCAL__print_test_list(header: str, test_count: int, test_list: list[str]): logging.info("") def LOCAL__print_test_list2( - header: str, test_count: int, test_list: list[str, int] + header: str, test_count: int, test_list: typing.List[T_TUPLE__str_int] ): assert type(header) == str # noqa: E721 assert type(test_count) == int # noqa: E721 diff --git a/tests/test_os_ops_common.py b/tests/test_os_ops_common.py index 7d183775..ecfff5b2 100644 --- a/tests/test_os_ops_common.py +++ b/tests/test_os_ops_common.py @@ -12,13 +12,14 @@ import logging import socket import threading +import typing from ..testgres import InvalidOperationException from ..testgres import ExecUtilException class TestOsOpsCommon: - sm_os_ops_descrs: list[OsOpsDescr] = [ + sm_os_ops_descrs: typing.List[OsOpsDescr] = [ OsOpsDescrs.sm_local_os_ops_descr, OsOpsDescrs.sm_remote_os_ops_descr ] diff --git a/tests/test_testgres_common.py b/tests/test_testgres_common.py index c384dfb2..e1252de2 100644 --- a/tests/test_testgres_common.py +++ b/tests/test_testgres_common.py @@ -56,7 +56,7 @@ def removing(os_ops: OsOperations, f): class TestTestgresCommon: - sm_node_svcs: list[PostgresNodeService] = [ + sm_node_svcs: typing.List[PostgresNodeService] = [ PostgresNodeServices.sm_local, PostgresNodeServices.sm_local2, PostgresNodeServices.sm_remote, @@ -315,8 +315,8 @@ def test_child_pids(self, node_svc: PostgresNodeService): def LOCAL__test_auxiliary_pids( node: PostgresNode, - expectedTypes: list[ProcessType] - ) -> list[ProcessType]: + expectedTypes: typing.List[ProcessType] + ) -> typing.List[ProcessType]: # returns list of the absence processes assert node is not None assert type(node) == PostgresNode # noqa: E721 @@ -327,7 +327,7 @@ def LOCAL__test_auxiliary_pids( assert pids is not None # noqa: E721 assert type(pids) == dict # noqa: E721 - result = list[ProcessType]() + result: typing.List[ProcessType] = list() for ptype in expectedTypes: if not (ptype in pids): result.append(ptype) @@ -335,7 +335,7 @@ def LOCAL__test_auxiliary_pids( def LOCAL__check_auxiliary_pids__multiple_attempts( node: PostgresNode, - expectedTypes: list[ProcessType]): + expectedTypes: typing.List[ProcessType]): assert node is not None assert type(node) == PostgresNode # noqa: E721 assert expectedTypes is not None diff --git a/tests/test_utils.py b/tests/test_utils.py index d4a4c9ad..c05bd2fe 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -7,10 +7,11 @@ from ..testgres import scoped_config import pytest +import typing class TestUtils: - sm_os_ops_descrs: list[OsOpsDescr] = [ + sm_os_ops_descrs: typing.List[OsOpsDescr] = [ OsOpsDescrs.sm_local_os_ops_descr, OsOpsDescrs.sm_remote_os_ops_descr ] From ac782bb3354c596171c9d0498494fbb4b828463b Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Tue, 15 Apr 2025 14:50:35 +0300 Subject: [PATCH 456/475] [New] OsOps::execute_command supports a transfer of environment variables (exec_env) (#239) * [New] OsOps::execute_command supports a transfer of environment variables (exec_env) New feature allows to pass environment variables to an executed program. If variable in exec_env has None value, then this variable will be unset. PostgresNode::start and PostgresNode::slow_start supports exec_env. --- testgres/node.py | 12 +++--- testgres/operations/local_ops.py | 66 ++++++++++++++++++++++++++++--- testgres/operations/remote_ops.py | 44 ++++++++++++++++----- testgres/utils.py | 13 +++++- tests/test_os_ops_common.py | 64 ++++++++++++++++++++++++++++++ 5 files changed, 177 insertions(+), 22 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index 5039fc43..3a294044 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -1020,7 +1020,7 @@ def get_control_data(self): return out_dict - def slow_start(self, replica=False, dbname='template1', username=None, max_attempts=0): + def slow_start(self, replica=False, dbname='template1', username=None, max_attempts=0, exec_env=None): """ Starts the PostgreSQL instance and then polls the instance until it reaches the expected state (primary or replica). The state is checked @@ -1033,7 +1033,9 @@ def slow_start(self, replica=False, dbname='template1', username=None, max_attem If False, waits for the instance to be in primary mode. Default is False. max_attempts: """ - self.start() + assert exec_env is None or type(exec_env) == dict # noqa: E721 + + self.start(exec_env=exec_env) if replica: query = 'SELECT pg_is_in_recovery()' @@ -1065,7 +1067,7 @@ def _detect_port_conflict(self, log_files0, log_files1): return True return False - def start(self, params=[], wait=True): + def start(self, params=[], wait=True, exec_env=None): """ Starts the PostgreSQL node using pg_ctl if node has not been started. By default, it waits for the operation to complete before returning. @@ -1079,7 +1081,7 @@ def start(self, params=[], wait=True): Returns: This instance of :class:`.PostgresNode`. """ - + assert exec_env is None or type(exec_env) == dict # noqa: E721 assert __class__._C_MAX_START_ATEMPTS > 1 if self.is_started: @@ -1098,7 +1100,7 @@ def start(self, params=[], wait=True): def LOCAL__start_node(): # 'error' will be None on Windows - _, _, error = execute_utility2(self.os_ops, _params, self.utils_log_file, verbose=True) + _, _, error = execute_utility2(self.os_ops, _params, self.utils_log_file, verbose=True, exec_env=exec_env) assert error is None or type(error) == str # noqa: E721 if error and 'does not exist' in error: raise Exception(error) diff --git a/testgres/operations/local_ops.py b/testgres/operations/local_ops.py index 9785d462..74323bb8 100644 --- a/testgres/operations/local_ops.py +++ b/testgres/operations/local_ops.py @@ -9,6 +9,7 @@ import socket import psutil +import typing from ..exceptions import ExecUtilException from ..exceptions import InvalidOperationException @@ -46,9 +47,34 @@ def _process_output(encoding, temp_file_path): output = output.decode(encoding) return output, None # In Windows stderr writing in stdout - def _run_command__nt(self, cmd, shell, input, stdin, stdout, stderr, get_process, timeout, encoding): + def _run_command__nt(self, cmd, shell, input, stdin, stdout, stderr, get_process, timeout, encoding, exec_env=None): + assert exec_env is None or type(exec_env) == dict # noqa: E721 + # TODO: why don't we use the data from input? + extParams: typing.Dict[str, str] = dict() + + if exec_env is None: + pass + elif len(exec_env) == 0: + pass + else: + env = os.environ.copy() + assert type(env) == dict # noqa: E721 + for v in exec_env.items(): + assert type(v) == tuple # noqa: E721 + assert len(v) == 2 + assert type(v[0]) == str # noqa: E721 + assert v[0] != "" + + if v[1] is None: + env.pop(v[0], None) + else: + assert type(v[1]) == str # noqa: E721 + env[v[0]] = v[1] + + extParams["env"] = env + with tempfile.NamedTemporaryFile(mode='w+b', delete=False) as temp_file: stdout = temp_file stderr = subprocess.STDOUT @@ -58,6 +84,7 @@ def _run_command__nt(self, cmd, shell, input, stdin, stdout, stderr, get_process stdin=stdin or subprocess.PIPE if input is not None else None, stdout=stdout, stderr=stderr, + **extParams, ) if get_process: return process, None, None @@ -69,19 +96,45 @@ def _run_command__nt(self, cmd, shell, input, stdin, stdout, stderr, get_process output, error = self._process_output(encoding, temp_file_path) return process, output, error - def _run_command__generic(self, cmd, shell, input, stdin, stdout, stderr, get_process, timeout, encoding): + def _run_command__generic(self, cmd, shell, input, stdin, stdout, stderr, get_process, timeout, encoding, exec_env=None): + assert exec_env is None or type(exec_env) == dict # noqa: E721 + input_prepared = None if not get_process: input_prepared = Helpers.PrepareProcessInput(input, encoding) # throw assert input_prepared is None or (type(input_prepared) == bytes) # noqa: E721 + extParams: typing.Dict[str, str] = dict() + + if exec_env is None: + pass + elif len(exec_env) == 0: + pass + else: + env = os.environ.copy() + assert type(env) == dict # noqa: E721 + for v in exec_env.items(): + assert type(v) == tuple # noqa: E721 + assert len(v) == 2 + assert type(v[0]) == str # noqa: E721 + assert v[0] != "" + + if v[1] is None: + env.pop(v[0], None) + else: + assert type(v[1]) == str # noqa: E721 + env[v[0]] = v[1] + + extParams["env"] = env + process = subprocess.Popen( cmd, shell=shell, stdin=stdin or subprocess.PIPE if input is not None else None, stdout=stdout or subprocess.PIPE, stderr=stderr or subprocess.PIPE, + **extParams ) assert not (process is None) if get_process: @@ -100,25 +153,26 @@ def _run_command__generic(self, cmd, shell, input, stdin, stdout, stderr, get_pr error = error.decode(encoding) return process, output, error - def _run_command(self, cmd, shell, input, stdin, stdout, stderr, get_process, timeout, encoding): + def _run_command(self, cmd, shell, input, stdin, stdout, stderr, get_process, timeout, encoding, exec_env=None): """Execute a command and return the process and its output.""" if os.name == 'nt' and stdout is None: # Windows method = __class__._run_command__nt else: # Other OS method = __class__._run_command__generic - return method(self, cmd, shell, input, stdin, stdout, stderr, get_process, timeout, encoding) + return method(self, cmd, shell, input, stdin, stdout, stderr, get_process, timeout, encoding, exec_env=exec_env) def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False, encoding=None, shell=False, text=False, input=None, stdin=None, stdout=None, stderr=None, get_process=False, timeout=None, - ignore_errors=False): + ignore_errors=False, exec_env=None): """ Execute a command in a subprocess and handle the output based on the provided parameters. """ assert type(expect_error) == bool # noqa: E721 assert type(ignore_errors) == bool # noqa: E721 + assert exec_env is None or type(exec_env) == dict # noqa: E721 - process, output, error = self._run_command(cmd, shell, input, stdin, stdout, stderr, get_process, timeout, encoding) + process, output, error = self._run_command(cmd, shell, input, stdin, stdout, stderr, get_process, timeout, encoding, exec_env=exec_env) if get_process: return process diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index 33b61ac2..e722a2cb 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -64,7 +64,8 @@ def __enter__(self): def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False, encoding=None, shell=True, text=False, input=None, stdin=None, stdout=None, - stderr=None, get_process=None, timeout=None, ignore_errors=False): + stderr=None, get_process=None, timeout=None, ignore_errors=False, + exec_env=None): """ Execute a command in the SSH session. Args: @@ -72,6 +73,7 @@ def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False, """ assert type(expect_error) == bool # noqa: E721 assert type(ignore_errors) == bool # noqa: E721 + assert exec_env is None or type(exec_env) == dict # noqa: E721 input_prepared = None if not get_process: @@ -88,7 +90,7 @@ def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False, assert type(cmd_s) == str # noqa: E721 - cmd_items = __class__._make_exec_env_list() + cmd_items = __class__._make_exec_env_list(exec_env=exec_env) cmd_items.append(cmd_s) env_cmd_s = ';'.join(cmd_items) @@ -670,14 +672,38 @@ def _is_port_free__process_1(error: str) -> bool: return True @staticmethod - def _make_exec_env_list() -> typing.List[str]: - result: typing.List[str] = list() + def _make_exec_env_list(exec_env: typing.Dict) -> typing.List[str]: + env: typing.Dict[str, str] = dict() + + # ---------------------------------- SYSTEM ENV for envvar in os.environ.items(): - if not __class__._does_put_envvar_into_exec_cmd(envvar[0]): - continue - qvalue = __class__._quote_envvar(envvar[1]) - assert type(qvalue) == str # noqa: E721 - result.append(envvar[0] + "=" + qvalue) + if __class__._does_put_envvar_into_exec_cmd(envvar[0]): + env[envvar[0]] = envvar[1] + + # ---------------------------------- EXEC (LOCAL) ENV + if exec_env is None: + pass + else: + for envvar in exec_env.items(): + assert type(envvar) == tuple # noqa: E721 + assert len(envvar) == 2 + assert type(envvar[0]) == str # noqa: E721 + env[envvar[0]] = envvar[1] + + # ---------------------------------- FINAL BUILD + result: typing.List[str] = list() + for envvar in env.items(): + assert type(envvar) == tuple # noqa: E721 + assert len(envvar) == 2 + assert type(envvar[0]) == str # noqa: E721 + + if envvar[1] is None: + result.append("unset " + envvar[0]) + else: + assert type(envvar[1]) == str # noqa: E721 + qvalue = __class__._quote_envvar(envvar[1]) + assert type(qvalue) == str # noqa: E721 + result.append(envvar[0] + "=" + qvalue) continue return result diff --git a/testgres/utils.py b/testgres/utils.py index 10ae81b6..2ff6f2a0 100644 --- a/testgres/utils.py +++ b/testgres/utils.py @@ -96,17 +96,26 @@ def execute_utility(args, logfile=None, verbose=False): return execute_utility2(tconf.os_ops, args, logfile, verbose) -def execute_utility2(os_ops: OsOperations, args, logfile=None, verbose=False, ignore_errors=False): +def execute_utility2( + os_ops: OsOperations, + args, + logfile=None, + verbose=False, + ignore_errors=False, + exec_env=None, +): assert os_ops is not None assert isinstance(os_ops, OsOperations) assert type(verbose) == bool # noqa: E721 assert type(ignore_errors) == bool # noqa: E721 + assert exec_env is None or type(exec_env) == dict # noqa: E721 exit_status, out, error = os_ops.exec_command( args, verbose=True, ignore_errors=ignore_errors, - encoding=OsHelpers.GetDefaultEncoding()) + encoding=OsHelpers.GetDefaultEncoding(), + exec_env=exec_env) out = '' if not out else out diff --git a/tests/test_os_ops_common.py b/tests/test_os_ops_common.py index ecfff5b2..17c3151c 100644 --- a/tests/test_os_ops_common.py +++ b/tests/test_os_ops_common.py @@ -93,6 +93,70 @@ def test_exec_command_failure__expect_error(self, os_ops: OsOperations): assert b"nonexistent_command" in error assert b"not found" in error + def test_exec_command_with_exec_env(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + + RunConditions.skip_if_windows() + + C_ENV_NAME = "TESTGRES_TEST__EXEC_ENV_20250414" + + cmd = ["sh", "-c", "echo ${}".format(C_ENV_NAME)] + + exec_env = {C_ENV_NAME: "Hello!"} + + response = os_ops.exec_command(cmd, exec_env=exec_env) + assert response is not None + assert type(response) == bytes # noqa: E721 + assert response == b'Hello!\n' + + response = os_ops.exec_command(cmd) + assert response is not None + assert type(response) == bytes # noqa: E721 + assert response == b'\n' + + def test_exec_command__test_unset(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + + RunConditions.skip_if_windows() + + C_ENV_NAME = "LANG" + + cmd = ["sh", "-c", "echo ${}".format(C_ENV_NAME)] + + response1 = os_ops.exec_command(cmd) + assert response1 is not None + assert type(response1) == bytes # noqa: E721 + + if response1 == b'\n': + logging.warning("Environment variable {} is not defined.".format(C_ENV_NAME)) + return + + exec_env = {C_ENV_NAME: None} + response2 = os_ops.exec_command(cmd, exec_env=exec_env) + assert response2 is not None + assert type(response2) == bytes # noqa: E721 + assert response2 == b'\n' + + response3 = os_ops.exec_command(cmd) + assert response3 is not None + assert type(response3) == bytes # noqa: E721 + assert response3 == response1 + + def test_exec_command__test_unset_dummy_var(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + + RunConditions.skip_if_windows() + + C_ENV_NAME = "TESTGRES_TEST__DUMMY_VAR_20250414" + + cmd = ["sh", "-c", "echo ${}".format(C_ENV_NAME)] + + exec_env = {C_ENV_NAME: None} + response2 = os_ops.exec_command(cmd, exec_env=exec_env) + assert response2 is not None + assert type(response2) == bytes # noqa: E721 + assert response2 == b'\n' + def test_is_executable_true(self, os_ops: OsOperations): """ Test is_executable for an existing executable. From daa2b7b146dc5ed41c34f7d75ec0b8dcb8db00bd Mon Sep 17 00:00:00 2001 From: Victoria Shepard <5807469+demonolock@users.noreply.github.com> Date: Tue, 15 Apr 2025 20:32:33 +0200 Subject: [PATCH 457/475] Add maintain command (#175) --- .../plugins/pg_probackup2/pg_probackup2/app.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/app.py b/testgres/plugins/pg_probackup2/pg_probackup2/app.py index d47cf51f..5166e9b8 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/app.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/app.py @@ -842,5 +842,22 @@ def archive_get(self, instance, wal_file_name, wal_file_path, options=None, expe ] return self.run(cmd + options, expect_error=expect_error) + def maintain( + self, instance=None, backup_id=None, + options=None, old_binary=False, gdb=False, expect_error=False + ): + if options is None: + options = [] + cmd_list = [ + 'maintain', + ] + if instance: + cmd_list += ['--instance={0}'.format(instance)] + if backup_id: + cmd_list += ['-i', backup_id] + + return self.run(cmd_list + options, old_binary=old_binary, gdb=gdb, + expect_error=expect_error) + def build_backup_dir(self, backup='backup'): return fs_backup_class(rel_path=self.rel_path, backup=backup) From 330fd9abaf081a47ecd8f88767f18cb0bf3dcd2c Mon Sep 17 00:00:00 2001 From: Alexey Savchkov Date: Mon, 21 Apr 2025 20:09:27 +0700 Subject: [PATCH 458/475] Up versions --- setup.py | 2 +- testgres/plugins/pg_probackup2/setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index b47a1d8a..2c44b18f 100755 --- a/setup.py +++ b/setup.py @@ -27,7 +27,7 @@ readme = f.read() setup( - version='1.10.5', + version='1.11.0', name='testgres', packages=['testgres', 'testgres.operations'], description='Testing utility for PostgreSQL and its extensions', diff --git a/testgres/plugins/pg_probackup2/setup.py b/testgres/plugins/pg_probackup2/setup.py index 8bcfe7b4..7a3212e4 100644 --- a/testgres/plugins/pg_probackup2/setup.py +++ b/testgres/plugins/pg_probackup2/setup.py @@ -4,7 +4,7 @@ from distutils.core import setup setup( - version='0.0.6', + version='0.1.0', name='testgres_pg_probackup2', packages=['pg_probackup2', 'pg_probackup2.storage'], description='Plugin for testgres that manages pg_probackup2', From 6e5e4f5a9eb7f7a02df7056dcded7e0b68a6d1da Mon Sep 17 00:00:00 2001 From: Victoria Shepard <5807469+demonolock@users.noreply.github.com> Date: Thu, 24 Apr 2025 14:00:30 +0000 Subject: [PATCH 459/475] remove __init__.py from the root --- __init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 __init__.py diff --git a/__init__.py b/__init__.py deleted file mode 100644 index e69de29b..00000000 From 1a662f138f9e03b750b91455113ea3b58c0c986e Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Fri, 25 Apr 2025 15:33:13 +0300 Subject: [PATCH 460/475] [FIX] Tests include testgres by right way through import (#241) When we do not have root __init__.py tests must to import testgres through "import testgres" not through "from import testgres" --- .../pg_probackup2/tests/test_basic.py | 2 +- tests/helpers/global_data.py | 16 ++++---- tests/test_config.py | 10 ++--- tests/test_os_ops_common.py | 4 +- tests/test_os_ops_remote.py | 2 +- tests/test_testgres_common.py | 40 +++++++++---------- tests/test_testgres_local.py | 24 +++++------ tests/test_testgres_remote.py | 14 +++---- tests/test_utils.py | 6 +-- 9 files changed, 59 insertions(+), 59 deletions(-) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/tests/test_basic.py b/testgres/plugins/pg_probackup2/pg_probackup2/tests/test_basic.py index ba788623..f22a62bf 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/tests/test_basic.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/tests/test_basic.py @@ -4,7 +4,7 @@ import shutil import pytest -from ...... import testgres +import testgres from ...pg_probackup2.app import ProbackupApp from ...pg_probackup2.init_helpers import Init, init_params from ..storage.fs_backup import FSTestBackupDir diff --git a/tests/helpers/global_data.py b/tests/helpers/global_data.py index c21d7dd8..51bf4485 100644 --- a/tests/helpers/global_data.py +++ b/tests/helpers/global_data.py @@ -1,11 +1,11 @@ -from ...testgres.operations.os_ops import OsOperations -from ...testgres.operations.os_ops import ConnectionParams -from ...testgres.operations.local_ops import LocalOperations -from ...testgres.operations.remote_ops import RemoteOperations - -from ...testgres.node import PortManager -from ...testgres.node import PortManager__ThisHost -from ...testgres.node import PortManager__Generic +from testgres.operations.os_ops import OsOperations +from testgres.operations.os_ops import ConnectionParams +from testgres.operations.local_ops import LocalOperations +from testgres.operations.remote_ops import RemoteOperations + +from testgres.node import PortManager +from testgres.node import PortManager__ThisHost +from testgres.node import PortManager__Generic import os diff --git a/tests/test_config.py b/tests/test_config.py index 05702e9a..a80a11f1 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,9 +1,9 @@ -from ..testgres import TestgresConfig -from ..testgres import configure_testgres -from ..testgres import scoped_config -from ..testgres import pop_config +from testgres import TestgresConfig +from testgres import configure_testgres +from testgres import scoped_config +from testgres import pop_config -from .. import testgres +import testgres import pytest diff --git a/tests/test_os_ops_common.py b/tests/test_os_ops_common.py index 17c3151c..9c4d8857 100644 --- a/tests/test_os_ops_common.py +++ b/tests/test_os_ops_common.py @@ -14,8 +14,8 @@ import threading import typing -from ..testgres import InvalidOperationException -from ..testgres import ExecUtilException +from testgres import InvalidOperationException +from testgres import ExecUtilException class TestOsOpsCommon: diff --git a/tests/test_os_ops_remote.py b/tests/test_os_ops_remote.py index 338e49f3..65830218 100755 --- a/tests/test_os_ops_remote.py +++ b/tests/test_os_ops_remote.py @@ -3,7 +3,7 @@ from .helpers.global_data import OsOpsDescrs from .helpers.global_data import OsOperations -from ..testgres import ExecUtilException +from testgres import ExecUtilException import os import pytest diff --git a/tests/test_testgres_common.py b/tests/test_testgres_common.py index e1252de2..f4e5996f 100644 --- a/tests/test_testgres_common.py +++ b/tests/test_testgres_common.py @@ -3,29 +3,29 @@ from .helpers.global_data import OsOperations from .helpers.global_data import PortManager -from ..testgres.node import PgVer -from ..testgres.node import PostgresNode -from ..testgres.utils import get_pg_version2 -from ..testgres.utils import file_tail -from ..testgres.utils import get_bin_path2 -from ..testgres import ProcessType -from ..testgres import NodeStatus -from ..testgres import IsolationLevel +from testgres.node import PgVer +from testgres.node import PostgresNode +from testgres.utils import get_pg_version2 +from testgres.utils import file_tail +from testgres.utils import get_bin_path2 +from testgres import ProcessType +from testgres import NodeStatus +from testgres import IsolationLevel # New name prevents to collect test-functions in TestgresException and fixes # the problem with pytest warning. -from ..testgres import TestgresException as testgres_TestgresException - -from ..testgres import InitNodeException -from ..testgres import StartNodeException -from ..testgres import QueryException -from ..testgres import ExecUtilException -from ..testgres import TimeoutException -from ..testgres import InvalidOperationException -from ..testgres import BackupException -from ..testgres import ProgrammingError -from ..testgres import scoped_config -from ..testgres import First, Any +from testgres import TestgresException as testgres_TestgresException + +from testgres import InitNodeException +from testgres import StartNodeException +from testgres import QueryException +from testgres import ExecUtilException +from testgres import TimeoutException +from testgres import InvalidOperationException +from testgres import BackupException +from testgres import ProgrammingError +from testgres import scoped_config +from testgres import First, Any from contextlib import contextmanager diff --git a/tests/test_testgres_local.py b/tests/test_testgres_local.py index 9dbd455b..1dd98fe3 100644 --- a/tests/test_testgres_local.py +++ b/tests/test_testgres_local.py @@ -7,21 +7,21 @@ import platform import logging -from .. import testgres +import testgres -from ..testgres import StartNodeException -from ..testgres import ExecUtilException -from ..testgres import NodeApp -from ..testgres import scoped_config -from ..testgres import get_new_node -from ..testgres import get_bin_path -from ..testgres import get_pg_config -from ..testgres import get_pg_version +from testgres import StartNodeException +from testgres import ExecUtilException +from testgres import NodeApp +from testgres import scoped_config +from testgres import get_new_node +from testgres import get_bin_path +from testgres import get_pg_config +from testgres import get_pg_version # NOTE: those are ugly imports -from ..testgres.utils import bound_ports -from ..testgres.utils import PgVer -from ..testgres.node import ProcessProxy +from testgres.utils import bound_ports +from testgres.utils import PgVer +from testgres.node import ProcessProxy def pg_version_ge(version): diff --git a/tests/test_testgres_remote.py b/tests/test_testgres_remote.py index e38099b7..87cc0269 100755 --- a/tests/test_testgres_remote.py +++ b/tests/test_testgres_remote.py @@ -7,16 +7,16 @@ from .helpers.global_data import PostgresNodeService from .helpers.global_data import PostgresNodeServices -from .. import testgres +import testgres -from ..testgres.exceptions import InitNodeException -from ..testgres.exceptions import ExecUtilException +from testgres.exceptions import InitNodeException +from testgres.exceptions import ExecUtilException -from ..testgres.config import scoped_config -from ..testgres.config import testgres_config +from testgres.config import scoped_config +from testgres.config import testgres_config -from ..testgres import get_bin_path -from ..testgres import get_pg_config +from testgres import get_bin_path +from testgres import get_pg_config # NOTE: those are ugly imports diff --git a/tests/test_utils.py b/tests/test_utils.py index c05bd2fe..39e9dda0 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -2,9 +2,9 @@ from .helpers.global_data import OsOpsDescrs from .helpers.global_data import OsOperations -from ..testgres.utils import parse_pg_version -from ..testgres.utils import get_pg_config2 -from ..testgres import scoped_config +from testgres.utils import parse_pg_version +from testgres.utils import get_pg_config2 +from testgres import scoped_config import pytest import typing From 94d75725bb4d3bfb6667804e075e9b4d46e062d6 Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Mon, 28 Apr 2025 08:37:16 +0300 Subject: [PATCH 461/475] [#240] Using of node.psql with other host and port (#242) * [FIX] Tests include testgres by right way through import When we do not have root __init__.py tests must to import testgres through "import testgres" not through "from import testgres" * [#240] Using of node.psql with other host and port This patch adds the support of using other host and port in the following methods: - PostgresNode.psql (explicit new args: host and port) - PostgresNode.safe_psql (indirectly through **kwargs) It allows to run psql utility from one PostgreSQL instance to work with another one. If explicit host and port are not defined (are None), PostgresNode will use own ones. This patch closes #240. --- testgres/node.py | 29 +++++++++++- tests/test_testgres_common.py | 83 +++++++++++++++++++++++++++++++++++ 2 files changed, 110 insertions(+), 2 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index 3a294044..41504e89 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -1372,6 +1372,8 @@ def psql(self, dbname=None, username=None, input=None, + host: typing.Optional[str] = None, + port: typing.Optional[int] = None, **variables): """ Execute a query using psql. @@ -1382,6 +1384,8 @@ def psql(self, dbname: database name to connect to. username: database user name. input: raw input to be passed. + host: an explicit host of server. + port: an explicit port of server. **variables: vars to be set before execution. Returns: @@ -1393,6 +1397,10 @@ def psql(self, >>> psql(query='select 3', ON_ERROR_STOP=1) """ + assert host is None or type(host) == str # noqa: E721 + assert port is None or type(port) == int # noqa: E721 + assert type(variables) == dict # noqa: E721 + return self._psql( ignore_errors=True, query=query, @@ -1400,6 +1408,8 @@ def psql(self, dbname=dbname, username=username, input=input, + host=host, + port=port, **variables ) @@ -1411,7 +1421,11 @@ def _psql( dbname=None, username=None, input=None, + host: typing.Optional[str] = None, + port: typing.Optional[int] = None, **variables): + assert host is None or type(host) == str # noqa: E721 + assert port is None or type(port) == int # noqa: E721 assert type(variables) == dict # noqa: E721 # @@ -1424,10 +1438,21 @@ def _psql( else: raise Exception("Input data must be None or bytes.") + if host is None: + host = self.host + + if port is None: + port = self.port + + assert host is not None + assert port is not None + assert type(host) == str # noqa: E721 + assert type(port) == int # noqa: E721 + psql_params = [ self._get_bin_path("psql"), - "-p", str(self.port), - "-h", self.host, + "-p", str(port), + "-h", host, "-U", username or self.os_ops.username, "-d", dbname or default_dbname(), "-X", # no .psqlrc diff --git a/tests/test_testgres_common.py b/tests/test_testgres_common.py index f4e5996f..21fa00df 100644 --- a/tests/test_testgres_common.py +++ b/tests/test_testgres_common.py @@ -678,6 +678,89 @@ def test_psql(self, node_svc: PostgresNodeService): r = node.safe_psql('select 1') # raises! logging.error("node.safe_psql returns [{}]".format(r)) + def test_psql__another_port(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc).init() as node1: + with __class__.helper__get_node(node_svc).init() as node2: + node1.start() + node2.start() + assert node1.port != node2.port + assert node1.host == node2.host + + node1.stop() + + logging.info("test table in node2 is creating ...") + node2.safe_psql( + dbname="postgres", + query="create table test (id integer);" + ) + + logging.info("try to find test table through node1.psql ...") + res = node1.psql( + dbname="postgres", + query="select count(*) from pg_class where relname='test'", + host=node2.host, + port=node2.port, + ) + assert (__class__.helper__rm_carriage_returns(res) == (0, b'1\n', b'')) + + def test_psql__another_bad_host(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc).init() as node: + logging.info("try to execute node1.psql ...") + res = node.psql( + dbname="postgres", + query="select count(*) from pg_class where relname='test'", + host="DUMMY_HOST_NAME", + port=node.port, + ) + + res2 = __class__.helper__rm_carriage_returns(res) + + assert res2[0] != 0 + assert b"DUMMY_HOST_NAME" in res[2] + + def test_safe_psql__another_port(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc).init() as node1: + with __class__.helper__get_node(node_svc).init() as node2: + node1.start() + node2.start() + assert node1.port != node2.port + assert node1.host == node2.host + + node1.stop() + + logging.info("test table in node2 is creating ...") + node2.safe_psql( + dbname="postgres", + query="create table test (id integer);" + ) + + logging.info("try to find test table through node1.psql ...") + res = node1.safe_psql( + dbname="postgres", + query="select count(*) from pg_class where relname='test'", + host=node2.host, + port=node2.port, + ) + assert (__class__.helper__rm_carriage_returns(res) == b'1\n') + + def test_safe_psql__another_bad_host(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc).init() as node: + logging.info("try to execute node1.psql ...") + + with pytest.raises(expected_exception=Exception) as x: + node.safe_psql( + dbname="postgres", + query="select count(*) from pg_class where relname='test'", + host="DUMMY_HOST_NAME", + port=node.port, + ) + + assert "DUMMY_HOST_NAME" in str(x.value) + def test_safe_psql__expect_error(self, node_svc: PostgresNodeService): assert isinstance(node_svc, PostgresNodeService) with __class__.helper__get_node(node_svc).init().start() as node: From 2f550d8787130551e7c72d07070bf3dcc11dd586 Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Thu, 1 May 2025 23:21:48 +0300 Subject: [PATCH 462/475] Testgres tests create log dir in exact place (#243) When we do not define TEST_CFG__LOG_DIR it is expected the testgres tests will create a log directory in a root of testgres project folder. We used config.rootpath for detect this folder in pytest_configure function. It was occurred that config.rootpath can point to another (unexpected) place. So we will use exact code to calculate testgres project folder (see TestStartupData.GetRootLogDir) to avid this problem. --- tests/conftest.py | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 6f2f9e41..9e74879b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -50,6 +50,17 @@ def CalcRootDir() -> str: r = os.path.abspath(r) return r + # -------------------------------------------------------------------- + def CalcRootLogDir() -> str: + if TestConfigPropNames.TEST_CFG__LOG_DIR in os.environ: + resultPath = os.environ[TestConfigPropNames.TEST_CFG__LOG_DIR] + else: + rootDir = __class__.CalcRootDir() + resultPath = os.path.join(rootDir, "logs") + + assert type(resultPath) == str # noqa: E721 + return resultPath + # -------------------------------------------------------------------- def CalcCurrentTestWorkerSignature() -> str: currentPID = os.getpid() @@ -86,11 +97,18 @@ class TestStartupData: TestStartupData__Helper.CalcCurrentTestWorkerSignature() ) + sm_RootLogDir: str = TestStartupData__Helper.CalcRootLogDir() + # -------------------------------------------------------------------- def GetRootDir() -> str: assert type(__class__.sm_RootDir) == str # noqa: E721 return __class__.sm_RootDir + # -------------------------------------------------------------------- + def GetRootLogDir() -> str: + assert type(__class__.sm_RootLogDir) == str # noqa: E721 + return __class__.sm_RootLogDir + # -------------------------------------------------------------------- def GetCurrentTestWorkerSignature() -> str: assert type(__class__.sm_CurrentTestWorkerSignature) == str # noqa: E721 @@ -954,13 +972,9 @@ def pytest_configure(config: pytest.Config) -> None: log_name = TestStartupData.GetCurrentTestWorkerSignature() log_name += ".log" - if TestConfigPropNames.TEST_CFG__LOG_DIR in os.environ: - log_path_v = os.environ[TestConfigPropNames.TEST_CFG__LOG_DIR] - log_path = pathlib.Path(log_path_v) - else: - log_path = config.rootpath.joinpath("logs") + log_dir = TestStartupData.GetRootLogDir() - log_path.mkdir(exist_ok=True) + pathlib.Path(log_dir).mkdir(exist_ok=True) logging_plugin: _pytest.logging.LoggingPlugin = config.pluginmanager.get_plugin( "logging-plugin" @@ -969,7 +983,7 @@ def pytest_configure(config: pytest.Config) -> None: assert logging_plugin is not None assert isinstance(logging_plugin, _pytest.logging.LoggingPlugin) - logging_plugin.set_log_path(str(log_path / log_name)) + logging_plugin.set_log_path(os.path.join(log_dir, log_name)) # ///////////////////////////////////////////////////////////////////////////// From 5f8f5dd2e5684a340f640141f01b3edd5ebca5a9 Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Sun, 4 May 2025 06:10:29 +0300 Subject: [PATCH 463/475] [test] TestTestgresLocal.test_upgrade_node is corrected (#246) Let's "release" all our test nodes correctly. --- tests/test_testgres_local.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/test_testgres_local.py b/tests/test_testgres_local.py index 1dd98fe3..53c9e0f6 100644 --- a/tests/test_testgres_local.py +++ b/tests/test_testgres_local.py @@ -158,15 +158,15 @@ def test_child_process_dies(self): def test_upgrade_node(self): old_bin_dir = os.path.dirname(get_bin_path("pg_config")) new_bin_dir = os.path.dirname(get_bin_path("pg_config")) - node_old = get_new_node(prefix='node_old', bin_dir=old_bin_dir) - node_old.init() - node_old.start() - node_old.stop() - node_new = get_new_node(prefix='node_new', bin_dir=new_bin_dir) - node_new.init(cached=False) - res = node_new.upgrade_from(old_node=node_old) - node_new.start() - assert (b'Upgrade Complete' in res) + with get_new_node(prefix='node_old', bin_dir=old_bin_dir) as node_old: + node_old.init() + node_old.start() + node_old.stop() + with get_new_node(prefix='node_new', bin_dir=new_bin_dir) as node_new: + node_new.init(cached=False) + res = node_new.upgrade_from(old_node=node_old) + node_new.start() + assert (b'Upgrade Complete' in res) class tagPortManagerProxy: sm_prev_testgres_reserve_port = None From 0b331e6839002da9b51fb3ca6ca7db228373dff0 Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Sun, 4 May 2025 16:01:58 +0300 Subject: [PATCH 464/475] Releasing of reserved port in tests (#248) * [test] TestTestgresLocal.test_pg_ctl_wait_option is corrected Let's "release" all our test nodes correctly. * [test] TestTestgresLocal.test_simple_with_bin_dir is corrected Let's "release" all our test nodes correctly. --- tests/test_testgres_common.py | 15 ++++++++++++--- tests/test_testgres_local.py | 8 ++++---- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/tests/test_testgres_common.py b/tests/test_testgres_common.py index 21fa00df..b71dc5ad 100644 --- a/tests/test_testgres_common.py +++ b/tests/test_testgres_common.py @@ -883,10 +883,20 @@ def test_backup_wrong_xlog_method(self, node_svc: PostgresNodeService): def test_pg_ctl_wait_option(self, node_svc: PostgresNodeService): assert isinstance(node_svc, PostgresNodeService) - C_MAX_ATTEMPTS = 50 + with __class__.helper__get_node(node_svc) as node: + self.impl__test_pg_ctl_wait_option(node_svc, node) - node = __class__.helper__get_node(node_svc) + def impl__test_pg_ctl_wait_option( + self, + node_svc: PostgresNodeService, + node: PostgresNode + ) -> None: + assert isinstance(node_svc, PostgresNodeService) + assert isinstance(node, PostgresNode) assert node.status() == NodeStatus.Uninitialized + + C_MAX_ATTEMPTS = 50 + node.init() assert node.status() == NodeStatus.Stopped node.start(wait=False) @@ -950,7 +960,6 @@ def test_pg_ctl_wait_option(self, node_svc: PostgresNodeService): raise Exception("Unexpected node status: {0}.".format(s1)) logging.info("OK. Node is stopped.") - node.cleanup() def test_replicate(self, node_svc: PostgresNodeService): assert isinstance(node_svc, PostgresNodeService) diff --git a/tests/test_testgres_local.py b/tests/test_testgres_local.py index 53c9e0f6..63e5f37e 100644 --- a/tests/test_testgres_local.py +++ b/tests/test_testgres_local.py @@ -341,10 +341,10 @@ def test_simple_with_bin_dir(self): bin_dir = node.bin_dir app = NodeApp() - correct_bin_dir = app.make_simple(base_dir=node.base_dir, bin_dir=bin_dir) - correct_bin_dir.slow_start() - correct_bin_dir.safe_psql("SELECT 1;") - correct_bin_dir.stop() + with app.make_simple(base_dir=node.base_dir, bin_dir=bin_dir) as correct_bin_dir: + correct_bin_dir.slow_start() + correct_bin_dir.safe_psql("SELECT 1;") + correct_bin_dir.stop() while True: try: From c3b25b22f6004d592e1f53d0366486d367fb3b48 Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Sun, 4 May 2025 22:39:18 +0300 Subject: [PATCH 465/475] [#249] Fix of port number leak in NodeBackup::spawn_replica (#250) This patch has the following changes: 1) It adds a new argument release_resources to PostgresNode::cleanup method. Default value is False. 2) It fixes a port number leak in NodeBackup::spawn_replica through explicit call of PostgresNode::cleanup(release_resources=True). Closes #249. --- testgres/backup.py | 11 ++++++++--- testgres/node.py | 12 +++++++++--- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/testgres/backup.py b/testgres/backup.py index 388697b7..857c46d4 100644 --- a/testgres/backup.py +++ b/testgres/backup.py @@ -184,14 +184,19 @@ def spawn_replica(self, name=None, destroy=True, slot=None): """ # Build a new PostgresNode - with clean_on_error(self.spawn_primary(name=name, - destroy=destroy)) as node: + node = self.spawn_primary(name=name, destroy=destroy) + assert node is not None + try: # Assign it a master and a recovery file (private magic) node._assign_master(self.original_node) node._create_recovery_conf(username=self.username, slot=slot) + except: # noqa: E722 + # TODO: Pass 'final=True' ? + node.cleanup(release_resources=True) + raise - return node + return node def cleanup(self): """ diff --git a/testgres/node.py b/testgres/node.py index 41504e89..defc0b40 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -231,8 +231,6 @@ def __enter__(self): return self def __exit__(self, type, value, traceback): - self.free_port() - # NOTE: Ctrl+C does not count! got_exception = type is not None and type != KeyboardInterrupt @@ -246,6 +244,8 @@ def __exit__(self, type, value, traceback): else: self._try_shutdown(attempts) + self._release_resources() + def __repr__(self): return "{}(name='{}', port={}, base_dir='{}')".format( self.__class__.__name__, @@ -663,6 +663,9 @@ def _try_shutdown(self, max_attempts, with_force=False): ps_output, ps_command) + def _release_resources(self): + self.free_port() + @staticmethod def _throw_bugcheck__unexpected_result_of_ps(result, cmd): assert type(result) == str # noqa: E721 @@ -1340,7 +1343,7 @@ def free_port(self): self._port = None self._port_manager.release_port(port) - def cleanup(self, max_attempts=3, full=False): + def cleanup(self, max_attempts=3, full=False, release_resources=False): """ Stop node if needed and remove its data/logs directory. NOTE: take a look at TestgresConfig.node_cleanup_full. @@ -1363,6 +1366,9 @@ def cleanup(self, max_attempts=3, full=False): self.os_ops.rmdirs(rm_dir, ignore_errors=False) + if release_resources: + self._release_resources() + return self @method_decorator(positional_args_hack(['dbname', 'query'])) From a683c65ae222e1980aa141f8d215c9fc3eac9383 Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Mon, 5 May 2025 15:12:45 +0300 Subject: [PATCH 466/475] [Refactoring] Default port manager functions now use PortManager__Generic and LocalOperations (#251) * [Refactoring] Default port manager functions now use PortManager__Generic and LocalOperations This patch deletes a duplication of port manager code. Now utils.reserve_port and utils.release_port works through _old_port_manager - it is a global instance of PortManager__Generic that uses a global instance of LocalOperations. This commit is a part of work for #247. * [BUG FIX] PortManager__ThisHost::__new__ had MT-problem After MT-lock we must to check __class__.sm_single_instance again. Refactoring - PortManager__ThisHost::__new__ is replaced with an explicit PortManager__ThisHost::get_single_instance() - PortManager__ThisHost::__init__ is deleted --- setup.py | 2 +- testgres/impl/port_manager__generic.py | 64 ++++++++++++++++ testgres/impl/port_manager__this_host.py | 33 +++++++++ testgres/node.py | 6 +- testgres/port_manager.py | 93 ------------------------ testgres/utils.py | 42 ++++------- tests/helpers/global_data.py | 2 +- 7 files changed, 115 insertions(+), 127 deletions(-) create mode 100755 testgres/impl/port_manager__generic.py create mode 100755 testgres/impl/port_manager__this_host.py diff --git a/setup.py b/setup.py index 2c44b18f..0b209181 100755 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ setup( version='1.11.0', name='testgres', - packages=['testgres', 'testgres.operations'], + packages=['testgres', 'testgres.operations', 'testgres.impl'], description='Testing utility for PostgreSQL and its extensions', url='https://github.com/postgrespro/testgres', long_description=readme, diff --git a/testgres/impl/port_manager__generic.py b/testgres/impl/port_manager__generic.py new file mode 100755 index 00000000..a51af2bd --- /dev/null +++ b/testgres/impl/port_manager__generic.py @@ -0,0 +1,64 @@ +from ..operations.os_ops import OsOperations + +from ..port_manager import PortManager +from ..exceptions import PortForException + +import threading +import random +import typing + + +class PortManager__Generic(PortManager): + _os_ops: OsOperations + _guard: object + # TODO: is there better to use bitmap fot _available_ports? + _available_ports: typing.Set[int] + _reserved_ports: typing.Set[int] + + def __init__(self, os_ops: OsOperations): + assert os_ops is not None + assert isinstance(os_ops, OsOperations) + self._os_ops = os_ops + self._guard = threading.Lock() + self._available_ports: typing.Set[int] = set(range(1024, 65535)) + self._reserved_ports: typing.Set[int] = set() + + def reserve_port(self) -> int: + assert self._guard is not None + assert type(self._available_ports) == set # noqa: E721t + assert type(self._reserved_ports) == set # noqa: E721 + + with self._guard: + t = tuple(self._available_ports) + assert len(t) == len(self._available_ports) + sampled_ports = random.sample(t, min(len(t), 100)) + t = None + + for port in sampled_ports: + assert not (port in self._reserved_ports) + assert port in self._available_ports + + if not self._os_ops.is_port_free(port): + continue + + self._reserved_ports.add(port) + self._available_ports.discard(port) + assert port in self._reserved_ports + assert not (port in self._available_ports) + return port + + raise PortForException("Can't select a port.") + + def release_port(self, number: int) -> None: + assert type(number) == int # noqa: E721 + + assert self._guard is not None + assert type(self._reserved_ports) == set # noqa: E721 + + with self._guard: + assert number in self._reserved_ports + assert not (number in self._available_ports) + self._available_ports.add(number) + self._reserved_ports.discard(number) + assert not (number in self._reserved_ports) + assert number in self._available_ports diff --git a/testgres/impl/port_manager__this_host.py b/testgres/impl/port_manager__this_host.py new file mode 100755 index 00000000..0d56f356 --- /dev/null +++ b/testgres/impl/port_manager__this_host.py @@ -0,0 +1,33 @@ +from ..port_manager import PortManager + +from .. import utils + +import threading + + +class PortManager__ThisHost(PortManager): + sm_single_instance: PortManager = None + sm_single_instance_guard = threading.Lock() + + @staticmethod + def get_single_instance() -> PortManager: + assert __class__ == PortManager__ThisHost + assert __class__.sm_single_instance_guard is not None + + if __class__.sm_single_instance is not None: + assert type(__class__.sm_single_instance) == __class__ # noqa: E721 + return __class__.sm_single_instance + + with __class__.sm_single_instance_guard: + if __class__.sm_single_instance is None: + __class__.sm_single_instance = __class__() + assert __class__.sm_single_instance is not None + assert type(__class__.sm_single_instance) == __class__ # noqa: E721 + return __class__.sm_single_instance + + def reserve_port(self) -> int: + return utils.reserve_port() + + def release_port(self, number: int) -> None: + assert type(number) == int # noqa: E721 + return utils.release_port(number) diff --git a/testgres/node.py b/testgres/node.py index defc0b40..dd1a45d3 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -84,8 +84,8 @@ InvalidOperationException from .port_manager import PortManager -from .port_manager import PortManager__ThisHost -from .port_manager import PortManager__Generic +from .impl.port_manager__this_host import PortManager__ThisHost +from .impl.port_manager__generic import PortManager__Generic from .logger import TestgresLogger @@ -272,7 +272,7 @@ def _get_port_manager(os_ops: OsOperations) -> PortManager: assert isinstance(os_ops, OsOperations) if isinstance(os_ops, LocalOperations): - return PortManager__ThisHost() + return PortManager__ThisHost.get_single_instance() # TODO: Throw the exception "Please define a port manager." ? return PortManager__Generic(os_ops) diff --git a/testgres/port_manager.py b/testgres/port_manager.py index e2530470..1ae696c8 100644 --- a/testgres/port_manager.py +++ b/testgres/port_manager.py @@ -1,14 +1,3 @@ -from .operations.os_ops import OsOperations - -from .exceptions import PortForException - -from . import utils - -import threading -import random -import typing - - class PortManager: def __init__(self): super().__init__() @@ -19,85 +8,3 @@ def reserve_port(self) -> int: def release_port(self, number: int) -> None: assert type(number) == int # noqa: E721 raise NotImplementedError("PortManager::release_port is not implemented.") - - -class PortManager__ThisHost(PortManager): - sm_single_instance: PortManager = None - sm_single_instance_guard = threading.Lock() - - def __init__(self): - pass - - def __new__(cls) -> PortManager: - assert __class__ == PortManager__ThisHost - assert __class__.sm_single_instance_guard is not None - - if __class__.sm_single_instance is None: - with __class__.sm_single_instance_guard: - __class__.sm_single_instance = super().__new__(cls) - assert __class__.sm_single_instance - assert type(__class__.sm_single_instance) == __class__ # noqa: E721 - return __class__.sm_single_instance - - def reserve_port(self) -> int: - return utils.reserve_port() - - def release_port(self, number: int) -> None: - assert type(number) == int # noqa: E721 - return utils.release_port(number) - - -class PortManager__Generic(PortManager): - _os_ops: OsOperations - _guard: object - # TODO: is there better to use bitmap fot _available_ports? - _available_ports: typing.Set[int] - _reserved_ports: typing.Set[int] - - def __init__(self, os_ops: OsOperations): - assert os_ops is not None - assert isinstance(os_ops, OsOperations) - self._os_ops = os_ops - self._guard = threading.Lock() - self._available_ports: typing.Set[int] = set(range(1024, 65535)) - self._reserved_ports: typing.Set[int] = set() - - def reserve_port(self) -> int: - assert self._guard is not None - assert type(self._available_ports) == set # noqa: E721t - assert type(self._reserved_ports) == set # noqa: E721 - - with self._guard: - t = tuple(self._available_ports) - assert len(t) == len(self._available_ports) - sampled_ports = random.sample(t, min(len(t), 100)) - t = None - - for port in sampled_ports: - assert not (port in self._reserved_ports) - assert port in self._available_ports - - if not self._os_ops.is_port_free(port): - continue - - self._reserved_ports.add(port) - self._available_ports.discard(port) - assert port in self._reserved_ports - assert not (port in self._available_ports) - return port - - raise PortForException("Can't select a port.") - - def release_port(self, number: int) -> None: - assert type(number) == int # noqa: E721 - - assert self._guard is not None - assert type(self._reserved_ports) == set # noqa: E721 - - with self._guard: - assert number in self._reserved_ports - assert not (number in self._available_ports) - self._available_ports.add(number) - self._reserved_ports.discard(number) - assert not (number in self._reserved_ports) - assert number in self._available_ports diff --git a/testgres/utils.py b/testgres/utils.py index 2ff6f2a0..6603c929 100644 --- a/testgres/utils.py +++ b/testgres/utils.py @@ -6,8 +6,6 @@ import os import sys -import socket -import random from contextlib import contextmanager from packaging.version import Version, InvalidVersion @@ -15,18 +13,27 @@ from six import iteritems -from .exceptions import PortForException from .exceptions import ExecUtilException from .config import testgres_config as tconf from .operations.os_ops import OsOperations from .operations.remote_ops import RemoteOperations +from .operations.local_ops import LocalOperations from .operations.helpers import Helpers as OsHelpers +from .impl.port_manager__generic import PortManager__Generic + # rows returned by PG_CONFIG _pg_config_data = {} +_local_operations = LocalOperations() + +# +# The old, global "port manager" always worked with LOCAL system +# +_old_port_manager = PortManager__Generic(_local_operations) + # ports used by nodes -bound_ports = set() +bound_ports = _old_port_manager._reserved_ports # re-export version type @@ -43,28 +50,7 @@ def internal__reserve_port(): """ Generate a new port and add it to 'bound_ports'. """ - def LOCAL__is_port_free(port: int) -> bool: - """Check if a port is free to use.""" - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: - try: - s.bind(("", port)) - return True - except OSError: - return False - - ports = set(range(1024, 65535)) - assert type(ports) == set # noqa: E721 - assert type(bound_ports) == set # noqa: E721 - ports.difference_update(bound_ports) - - sampled_ports = random.sample(tuple(ports), min(len(ports), 100)) - - for port in sampled_ports: - if LOCAL__is_port_free(port): - bound_ports.add(port) - return port - - raise PortForException("Can't select a port") + return _old_port_manager.reserve_port() def internal__release_port(port): @@ -73,9 +59,7 @@ def internal__release_port(port): """ assert type(port) == int # noqa: E721 - assert port in bound_ports - - bound_ports.discard(port) + return _old_port_manager.release_port(port) reserve_port = internal__reserve_port diff --git a/tests/helpers/global_data.py b/tests/helpers/global_data.py index 51bf4485..07ac083d 100644 --- a/tests/helpers/global_data.py +++ b/tests/helpers/global_data.py @@ -39,7 +39,7 @@ class OsOpsDescrs: class PortManagers: sm_remote_port_manager = PortManager__Generic(OsOpsDescrs.sm_remote_os_ops) - sm_local_port_manager = PortManager__ThisHost() + sm_local_port_manager = PortManager__ThisHost.get_single_instance() sm_local2_port_manager = PortManager__Generic(OsOpsDescrs.sm_local_os_ops) From 6972bfcdab479088ac881375db92944be87487f4 Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Mon, 5 May 2025 15:27:43 +0300 Subject: [PATCH 467/475] [#244] PostgresNode now uses os_ops only (#245) This patch implements the proposal #244 - detach PostgresNode from ConnectionParams object. It will use os_ops object only. conn_params is saved but must be None. It will be removed in the future. --- testgres/node.py | 30 +++++++++++++++--------------- tests/test_testgres_common.py | 1 - tests/test_testgres_remote.py | 1 - 3 files changed, 15 insertions(+), 17 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index dd1a45d3..80ac5ee8 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -107,7 +107,6 @@ from .operations.os_ops import ConnectionParams from .operations.os_ops import OsOperations from .operations.local_ops import LocalOperations -from .operations.remote_ops import RemoteOperations InternalError = pglib.InternalError ProgrammingError = pglib.ProgrammingError @@ -151,7 +150,7 @@ def __init__(self, name=None, base_dir=None, port: typing.Optional[int] = None, - conn_params: ConnectionParams = ConnectionParams(), + conn_params: ConnectionParams = None, bin_dir=None, prefix=None, os_ops: typing.Optional[OsOperations] = None, @@ -171,11 +170,15 @@ def __init__(self, assert os_ops is None or isinstance(os_ops, OsOperations) assert port_manager is None or isinstance(port_manager, PortManager) + if conn_params is not None: + assert type(conn_params) == ConnectionParams # noqa: E721 + + raise InvalidOperationException("conn_params is deprecated, please use os_ops parameter instead.") + # private if os_ops is None: - self._os_ops = __class__._get_os_ops(conn_params) + self._os_ops = __class__._get_os_ops() else: - assert conn_params is None assert isinstance(os_ops, OsOperations) self._os_ops = os_ops pass @@ -200,11 +203,14 @@ def __init__(self, self._should_free_port = False self._port_manager = None else: - if port_manager is not None: + if port_manager is None: + self._port_manager = __class__._get_port_manager(self._os_ops) + elif os_ops is None: + raise InvalidOperationException("When port_manager is not None you have to define os_ops, too.") + else: assert isinstance(port_manager, PortManager) + assert self._os_ops is os_ops self._port_manager = port_manager - else: - self._port_manager = __class__._get_port_manager(self._os_ops) assert self._port_manager is not None assert isinstance(self._port_manager, PortManager) @@ -255,16 +261,11 @@ def __repr__(self): ) @staticmethod - def _get_os_ops(conn_params: ConnectionParams) -> OsOperations: + def _get_os_ops() -> OsOperations: if testgres_config.os_ops: return testgres_config.os_ops - assert type(conn_params) == ConnectionParams # noqa: E721 - - if conn_params.ssh_key: - return RemoteOperations(conn_params) - - return LocalOperations(conn_params) + return LocalOperations() @staticmethod def _get_port_manager(os_ops: OsOperations) -> PortManager: @@ -294,7 +295,6 @@ def clone_with_new_name_and_base_dir(self, name: str, base_dir: str): node = PostgresNode( name=name, base_dir=base_dir, - conn_params=None, bin_dir=self._bin_dir, prefix=self._prefix, os_ops=self._os_ops, diff --git a/tests/test_testgres_common.py b/tests/test_testgres_common.py index b71dc5ad..5b926bc8 100644 --- a/tests/test_testgres_common.py +++ b/tests/test_testgres_common.py @@ -1487,7 +1487,6 @@ def helper__get_node( return PostgresNode( name, port=port, - conn_params=None, os_ops=node_svc.os_ops, port_manager=port_manager if port is None else None ) diff --git a/tests/test_testgres_remote.py b/tests/test_testgres_remote.py index 87cc0269..6a8d068b 100755 --- a/tests/test_testgres_remote.py +++ b/tests/test_testgres_remote.py @@ -173,7 +173,6 @@ def helper__get_node(name=None): return testgres.PostgresNode( name, - conn_params=None, os_ops=svc.os_ops, port_manager=svc.port_manager) From df4f545eb47427f2997dbe7eeb18e80ff64c5686 Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Tue, 6 May 2025 09:39:26 +0300 Subject: [PATCH 468/475] LocalOperations::get_single_instance is added (#252) * LocalOperations::get_single_instance is added This patch forces testgres to use a single instance of LocalOperations that is created with default parameters. Note that, PortManager__ThisHost is used only when PostgresNode uses this single local_ops instance. --- testgres/cache.py | 8 ++++++-- testgres/config.py | 3 ++- testgres/node.py | 23 +++++++++++++++++++---- testgres/operations/local_ops.py | 20 ++++++++++++++++++++ testgres/utils.py | 4 +--- tests/helpers/global_data.py | 2 +- 6 files changed, 49 insertions(+), 11 deletions(-) diff --git a/testgres/cache.py b/testgres/cache.py index 3ac63326..499cce91 100644 --- a/testgres/cache.py +++ b/testgres/cache.py @@ -22,12 +22,16 @@ from .operations.os_ops import OsOperations -def cached_initdb(data_dir, logfile=None, params=None, os_ops: OsOperations = LocalOperations(), bin_path=None, cached=True): +def cached_initdb(data_dir, logfile=None, params=None, os_ops: OsOperations = None, bin_path=None, cached=True): """ Perform initdb or use cached node files. """ - assert os_ops is not None + assert os_ops is None or isinstance(os_ops, OsOperations) + + if os_ops is None: + os_ops = LocalOperations.get_single_instance() + assert isinstance(os_ops, OsOperations) def make_utility_path(name): diff --git a/testgres/config.py b/testgres/config.py index 67d467d3..55d52426 100644 --- a/testgres/config.py +++ b/testgres/config.py @@ -50,8 +50,9 @@ class GlobalConfig(object): _cached_initdb_dir = None """ underlying class attribute for cached_initdb_dir property """ - os_ops = LocalOperations() + os_ops = LocalOperations.get_single_instance() """ OsOperation object that allows work on remote host """ + @property def cached_initdb_dir(self): """ path to a temp directory for cached initdb. """ diff --git a/testgres/node.py b/testgres/node.py index 80ac5ee8..66783e08 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -93,6 +93,8 @@ from .standby import First +from . import utils + from .utils import \ PgVer, \ eprint, \ @@ -265,14 +267,17 @@ def _get_os_ops() -> OsOperations: if testgres_config.os_ops: return testgres_config.os_ops - return LocalOperations() + return LocalOperations.get_single_instance() @staticmethod def _get_port_manager(os_ops: OsOperations) -> PortManager: assert os_ops is not None assert isinstance(os_ops, OsOperations) - if isinstance(os_ops, LocalOperations): + if os_ops is LocalOperations.get_single_instance(): + assert utils._old_port_manager is not None + assert type(utils._old_port_manager) == PortManager__Generic # noqa: E721 + assert utils._old_port_manager._os_ops is os_ops return PortManager__ThisHost.get_single_instance() # TODO: Throw the exception "Please define a port manager." ? @@ -816,10 +821,13 @@ def init(self, initdb_params=None, cached=True, **kwargs): """ # initialize this PostgreSQL node + assert self._os_ops is not None + assert isinstance(self._os_ops, OsOperations) + cached_initdb( data_dir=self.data_dir, logfile=self.utils_log_file, - os_ops=self.os_ops, + os_ops=self._os_ops, params=initdb_params, bin_path=self.bin_dir, cached=False) @@ -2186,7 +2194,14 @@ def _escape_config_value(value): class NodeApp: - def __init__(self, test_path=None, nodes_to_cleanup=None, os_ops=LocalOperations()): + def __init__(self, test_path=None, nodes_to_cleanup=None, os_ops=None): + assert os_ops is None or isinstance(os_ops, OsOperations) + + if os_ops is None: + os_ops = LocalOperations.get_single_instance() + + assert isinstance(os_ops, OsOperations) + if test_path: if os.path.isabs(test_path): self.test_path = test_path diff --git a/testgres/operations/local_ops.py b/testgres/operations/local_ops.py index 74323bb8..b9fd7aef 100644 --- a/testgres/operations/local_ops.py +++ b/testgres/operations/local_ops.py @@ -10,6 +10,7 @@ import psutil import typing +import threading from ..exceptions import ExecUtilException from ..exceptions import InvalidOperationException @@ -28,6 +29,9 @@ class LocalOperations(OsOperations): + sm_single_instance: OsOperations = None + sm_single_instance_guard = threading.Lock() + def __init__(self, conn_params=None): if conn_params is None: conn_params = ConnectionParams() @@ -38,6 +42,22 @@ def __init__(self, conn_params=None): self.remote = False self.username = conn_params.username or getpass.getuser() + @staticmethod + def get_single_instance() -> OsOperations: + assert __class__ == LocalOperations + assert __class__.sm_single_instance_guard is not None + + if __class__.sm_single_instance is not None: + assert type(__class__.sm_single_instance) == __class__ # noqa: E721 + return __class__.sm_single_instance + + with __class__.sm_single_instance_guard: + if __class__.sm_single_instance is None: + __class__.sm_single_instance = __class__() + assert __class__.sm_single_instance is not None + assert type(__class__.sm_single_instance) == __class__ # noqa: E721 + return __class__.sm_single_instance + @staticmethod def _process_output(encoding, temp_file_path): """Process the output of a command from a temporary file.""" diff --git a/testgres/utils.py b/testgres/utils.py index 6603c929..d231eec3 100644 --- a/testgres/utils.py +++ b/testgres/utils.py @@ -25,12 +25,10 @@ # rows returned by PG_CONFIG _pg_config_data = {} -_local_operations = LocalOperations() - # # The old, global "port manager" always worked with LOCAL system # -_old_port_manager = PortManager__Generic(_local_operations) +_old_port_manager = PortManager__Generic(LocalOperations.get_single_instance()) # ports used by nodes bound_ports = _old_port_manager._reserved_ports diff --git a/tests/helpers/global_data.py b/tests/helpers/global_data.py index 07ac083d..f3df41a3 100644 --- a/tests/helpers/global_data.py +++ b/tests/helpers/global_data.py @@ -31,7 +31,7 @@ class OsOpsDescrs: sm_remote_os_ops_descr = OsOpsDescr("remote_ops", sm_remote_os_ops) - sm_local_os_ops = LocalOperations() + sm_local_os_ops = LocalOperations.get_single_instance() sm_local_os_ops_descr = OsOpsDescr("local_ops", sm_local_os_ops) From a0a85065f59ac40a8e8f951e88efa346cfb9d695 Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Tue, 6 May 2025 14:52:14 +0300 Subject: [PATCH 469/475] New OsOperations methods: makedir, rmdir (#253) Signatures: def makedir(self, path: str) def rmdir(self, path: str) It is a part of work for #247. --- testgres/operations/local_ops.py | 8 + testgres/operations/os_ops.py | 8 + testgres/operations/remote_ops.py | 10 ++ tests/test_os_ops_common.py | 268 ++++++++++++++++++++++++++++++ 4 files changed, 294 insertions(+) diff --git a/testgres/operations/local_ops.py b/testgres/operations/local_ops.py index b9fd7aef..d33e8b65 100644 --- a/testgres/operations/local_ops.py +++ b/testgres/operations/local_ops.py @@ -250,6 +250,10 @@ def makedirs(self, path, remove_existing=False): except FileExistsError: pass + def makedir(self, path: str): + assert type(path) == str # noqa: E721 + os.mkdir(path) + # [2025-02-03] Old name of parameter attempts is "retries". def rmdirs(self, path, ignore_errors=True, attempts=3, delay=1): """ @@ -293,6 +297,10 @@ def rmdirs(self, path, ignore_errors=True, attempts=3, delay=1): # OK! return True + def rmdir(self, path: str): + assert type(path) == str # noqa: E721 + os.rmdir(path) + def listdir(self, path): return os.listdir(path) diff --git a/testgres/operations/os_ops.py b/testgres/operations/os_ops.py index d25e76bc..a4e1d9a2 100644 --- a/testgres/operations/os_ops.py +++ b/testgres/operations/os_ops.py @@ -53,9 +53,17 @@ def get_name(self): def makedirs(self, path, remove_existing=False): raise NotImplementedError() + def makedir(self, path: str): + assert type(path) == str # noqa: E721 + raise NotImplementedError() + def rmdirs(self, path, ignore_errors=True): raise NotImplementedError() + def rmdir(self, path: str): + assert type(path) == str # noqa: E721 + raise NotImplementedError() + def listdir(self, path): raise NotImplementedError() diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index e722a2cb..09406f79 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -225,6 +225,11 @@ def makedirs(self, path, remove_existing=False): raise Exception("Couldn't create dir {} because of error {}".format(path, error)) return result + def makedir(self, path: str): + assert type(path) == str # noqa: E721 + cmd = ["mkdir", path] + self.exec_command(cmd) + def rmdirs(self, path, ignore_errors=True): """ Remove a directory in the remote server. @@ -265,6 +270,11 @@ def rmdirs(self, path, ignore_errors=True): return False return True + def rmdir(self, path: str): + assert type(path) == str # noqa: E721 + cmd = ["rmdir", path] + self.exec_command(cmd) + def listdir(self, path): """ List all files and directories in a directory. diff --git a/tests/test_os_ops_common.py b/tests/test_os_ops_common.py index 9c4d8857..149050f9 100644 --- a/tests/test_os_ops_common.py +++ b/tests/test_os_ops_common.py @@ -13,10 +13,14 @@ import socket import threading import typing +import uuid from testgres import InvalidOperationException from testgres import ExecUtilException +from concurrent.futures import ThreadPoolExecutor +from concurrent.futures import Future as ThreadFuture + class TestOsOpsCommon: sm_os_ops_descrs: typing.List[OsOpsDescr] = [ @@ -812,3 +816,267 @@ def LOCAL_server(s: socket.socket): if ok_count == 0: raise RuntimeError("No one free port was found.") + + class tagData_OS_OPS__NUMS: + os_ops_descr: OsOpsDescr + nums: int + + def __init__(self, os_ops_descr: OsOpsDescr, nums: int): + assert isinstance(os_ops_descr, OsOpsDescr) + assert type(nums) == int # noqa: E721 + + self.os_ops_descr = os_ops_descr + self.nums = nums + + sm_test_exclusive_creation__mt__data = [ + tagData_OS_OPS__NUMS(OsOpsDescrs.sm_local_os_ops_descr, 100000), + tagData_OS_OPS__NUMS(OsOpsDescrs.sm_remote_os_ops_descr, 120), + ] + + @pytest.fixture( + params=sm_test_exclusive_creation__mt__data, + ids=[x.os_ops_descr.sign for x in sm_test_exclusive_creation__mt__data] + ) + def data001(self, request: pytest.FixtureRequest) -> tagData_OS_OPS__NUMS: + assert isinstance(request, pytest.FixtureRequest) + return request.param + + def test_mkdir__mt(self, data001: tagData_OS_OPS__NUMS): + assert type(data001) == __class__.tagData_OS_OPS__NUMS # noqa: E721 + + N_WORKERS = 4 + N_NUMBERS = data001.nums + assert type(N_NUMBERS) == int # noqa: E721 + + os_ops = data001.os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + lock_dir_prefix = "test_mkdir_mt--" + uuid.uuid4().hex + + lock_dir = os_ops.mkdtemp(prefix=lock_dir_prefix) + + logging.info("A lock file [{}] is creating ...".format(lock_dir)) + + assert os.path.exists(lock_dir) + + def MAKE_PATH(lock_dir: str, num: int) -> str: + assert type(lock_dir) == str # noqa: E721 + assert type(num) == int # noqa: E721 + return os.path.join(lock_dir, str(num) + ".lock") + + def LOCAL_WORKER(os_ops: OsOperations, + workerID: int, + lock_dir: str, + cNumbers: int, + reservedNumbers: typing.Set[int]) -> None: + assert isinstance(os_ops, OsOperations) + assert type(workerID) == int # noqa: E721 + assert type(lock_dir) == str # noqa: E721 + assert type(cNumbers) == int # noqa: E721 + assert type(reservedNumbers) == set # noqa: E721 + assert cNumbers > 0 + assert len(reservedNumbers) == 0 + + assert os.path.exists(lock_dir) + + def LOG_INFO(template: str, *args: list) -> None: + assert type(template) == str # noqa: E721 + assert type(args) == tuple # noqa: E721 + + msg = template.format(*args) + assert type(msg) == str # noqa: E721 + + logging.info("[Worker #{}] {}".format(workerID, msg)) + return + + LOG_INFO("HELLO! I am here!") + + for num in range(cNumbers): + assert not (num in reservedNumbers) + + file_path = MAKE_PATH(lock_dir, num) + + try: + os_ops.makedir(file_path) + except Exception as e: + LOG_INFO( + "Can't reserve {}. Error ({}): {}", + num, + type(e).__name__, + str(e) + ) + continue + + LOG_INFO("Number {} is reserved!", num) + assert os_ops.path_exists(file_path) + reservedNumbers.add(num) + continue + + n_total = cNumbers + n_ok = len(reservedNumbers) + assert n_ok <= n_total + + LOG_INFO("Finish! OK: {}. FAILED: {}.", n_ok, n_total - n_ok) + return + + # ----------------------- + logging.info("Worker are creating ...") + + threadPool = ThreadPoolExecutor( + max_workers=N_WORKERS, + thread_name_prefix="ex_creator" + ) + + class tadWorkerData: + future: ThreadFuture + reservedNumbers: typing.Set[int] + + workerDatas: typing.List[tadWorkerData] = list() + + nErrors = 0 + + try: + for n in range(N_WORKERS): + logging.info("worker #{} is creating ...".format(n)) + + workerDatas.append(tadWorkerData()) + + workerDatas[n].reservedNumbers = set() + + workerDatas[n].future = threadPool.submit( + LOCAL_WORKER, + os_ops, + n, + lock_dir, + N_NUMBERS, + workerDatas[n].reservedNumbers + ) + + assert workerDatas[n].future is not None + + logging.info("OK. All the workers were created!") + except Exception as e: + nErrors += 1 + logging.error("A problem is detected ({}): {}".format(type(e).__name__, str(e))) + + logging.info("Will wait for stop of all the workers...") + + nWorkers = 0 + + assert type(workerDatas) == list # noqa: E721 + + for i in range(len(workerDatas)): + worker = workerDatas[i].future + + if worker is None: + continue + + nWorkers += 1 + + assert isinstance(worker, ThreadFuture) + + try: + logging.info("Wait for worker #{}".format(i)) + worker.result() + except Exception as e: + nErrors += 1 + logging.error("Worker #{} finished with error ({}): {}".format( + i, + type(e).__name__, + str(e), + )) + continue + + assert nWorkers == N_WORKERS + + if nErrors != 0: + raise RuntimeError("Some problems were detected. Please examine the log messages.") + + logging.info("OK. Let's check worker results!") + + reservedNumbers: typing.Dict[int, int] = dict() + + for i in range(N_WORKERS): + logging.info("Worker #{} is checked ...".format(i)) + + workerNumbers = workerDatas[i].reservedNumbers + assert type(workerNumbers) == set # noqa: E721 + + for n in workerNumbers: + if n < 0 or n >= N_NUMBERS: + nErrors += 1 + logging.error("Unexpected number {}".format(n)) + continue + + if n in reservedNumbers.keys(): + nErrors += 1 + logging.error("Number {} was already reserved by worker #{}".format( + n, + reservedNumbers[n] + )) + else: + reservedNumbers[n] = i + + file_path = MAKE_PATH(lock_dir, n) + if not os_ops.path_exists(file_path): + nErrors += 1 + logging.error("File {} is not found!".format(file_path)) + continue + + continue + + logging.info("OK. Let's check reservedNumbers!") + + for n in range(N_NUMBERS): + if not (n in reservedNumbers.keys()): + nErrors += 1 + logging.error("Number {} is not reserved!".format(n)) + continue + + file_path = MAKE_PATH(lock_dir, n) + if not os_ops.path_exists(file_path): + nErrors += 1 + logging.error("File {} is not found!".format(file_path)) + continue + + # OK! + continue + + logging.info("Verification is finished! Total error count is {}.".format(nErrors)) + + if nErrors == 0: + logging.info("Root lock-directory [{}] will be deleted.".format( + lock_dir + )) + + for n in range(N_NUMBERS): + file_path = MAKE_PATH(lock_dir, n) + try: + os_ops.rmdir(file_path) + except Exception as e: + nErrors += 1 + logging.error("Cannot delete directory [{}]. Error ({}): {}".format( + file_path, + type(e).__name__, + str(e) + )) + continue + + if os_ops.path_exists(file_path): + nErrors += 1 + logging.error("Directory {} is not deleted!".format(file_path)) + continue + + if nErrors == 0: + try: + os_ops.rmdir(lock_dir) + except Exception as e: + nErrors += 1 + logging.error("Cannot delete directory [{}]. Error ({}): {}".format( + lock_dir, + type(e).__name__, + str(e) + )) + + logging.info("Test is finished! Total error count is {}.".format(nErrors)) + return From edd64db5284a6b34e275e022a4cc673e1032a6ea Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Tue, 6 May 2025 15:17:15 +0300 Subject: [PATCH 470/475] OsOperations::get_tempdir() is added (#254) Signature: def get_tempdir(self) -> str --- testgres/operations/local_ops.py | 7 +++++++ testgres/operations/os_ops.py | 3 +++ testgres/operations/remote_ops.py | 28 ++++++++++++++++++++++++++ tests/test_os_ops_common.py | 33 +++++++++++++++++++++++++++++++ 4 files changed, 71 insertions(+) diff --git a/testgres/operations/local_ops.py b/testgres/operations/local_ops.py index d33e8b65..ccf1ab82 100644 --- a/testgres/operations/local_ops.py +++ b/testgres/operations/local_ops.py @@ -528,3 +528,10 @@ def is_port_free(self, number: int) -> bool: return True except OSError: return False + + def get_tempdir(self) -> str: + r = tempfile.gettempdir() + assert r is not None + assert type(r) == str # noqa: E721 + assert os.path.exists(r) + return r diff --git a/testgres/operations/os_ops.py b/testgres/operations/os_ops.py index a4e1d9a2..45e4f71c 100644 --- a/testgres/operations/os_ops.py +++ b/testgres/operations/os_ops.py @@ -130,3 +130,6 @@ def get_process_children(self, pid): def is_port_free(self, number: int): assert type(number) == int # noqa: E721 raise NotImplementedError() + + def get_tempdir(self) -> str: + raise NotImplementedError() diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index 09406f79..a478b453 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -659,6 +659,34 @@ def is_port_free(self, number: int) -> bool: out=output ) + def get_tempdir(self) -> str: + command = ["mktemp", "-u", "-d"] + + exec_exitcode, exec_output, exec_error = self.exec_command( + command, + verbose=True, + encoding=get_default_encoding(), + ignore_errors=True + ) + + assert type(exec_exitcode) == int # noqa: E721 + assert type(exec_output) == str # noqa: E721 + assert type(exec_error) == str # noqa: E721 + + if exec_exitcode != 0: + RaiseError.CommandExecutionError( + cmd=command, + exit_code=exec_exitcode, + message="Could not detect a temporary directory.", + error=exec_error, + out=exec_output) + + temp_subdir = exec_output.strip() + assert type(temp_subdir) == str # noqa: E721 + temp_dir = os.path.dirname(temp_subdir) + assert type(temp_dir) == str # noqa: E721 + return temp_dir + @staticmethod def _is_port_free__process_0(error: str) -> bool: assert type(error) == str # noqa: E721 diff --git a/tests/test_os_ops_common.py b/tests/test_os_ops_common.py index 149050f9..5ae3a61f 100644 --- a/tests/test_os_ops_common.py +++ b/tests/test_os_ops_common.py @@ -817,6 +817,39 @@ def LOCAL_server(s: socket.socket): if ok_count == 0: raise RuntimeError("No one free port was found.") + def test_get_tmpdir(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + + dir = os_ops.get_tempdir() + assert type(dir) == str # noqa: E721 + assert os_ops.path_exists(dir) + assert os.path.exists(dir) + + file_path = os.path.join(dir, "testgres--" + uuid.uuid4().hex + ".tmp") + + os_ops.write(file_path, "1234", binary=False) + + assert os_ops.path_exists(file_path) + assert os.path.exists(file_path) + + d = os_ops.read(file_path, binary=False) + + assert d == "1234" + + os_ops.remove_file(file_path) + + assert not os_ops.path_exists(file_path) + assert not os.path.exists(file_path) + + def test_get_tmpdir__compare_with_py_info(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + + actual_dir = os_ops.get_tempdir() + assert actual_dir is not None + assert type(actual_dir) == str # noqa: E721 + expected_dir = str(tempfile.tempdir) + assert actual_dir == expected_dir + class tagData_OS_OPS__NUMS: os_ops_descr: OsOpsDescr nums: int From 3bb59c64b45826f2a50cfb7d423de1c86721946b Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Wed, 7 May 2025 22:10:08 +0300 Subject: [PATCH 471/475] [#235] test_pg_ctl_wait_option detects a port conflict (#257) This patch must fix a problem in test_pg_ctl_wait_option when his PostgreSQL instance conflicts with another one. For this, we added two new things: - PostgresNodeLogReader - PostgresNodeUtils PostgresNodeLogReader reads server logs. PostgresNodeUtils provides an utility to detect a port conflict. PostgresNode::start also uses these new classes. --- testgres/node.py | 208 +++++++++++++++++++++++++++------- tests/test_testgres_common.py | 37 +++++- 2 files changed, 200 insertions(+), 45 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index 66783e08..9a2f4e77 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -784,28 +784,6 @@ def _collect_special_files(self): return result - def _collect_log_files(self): - # dictionary of log files + size in bytes - - files = [ - self.pg_log_file - ] # yapf: disable - - result = {} - - for f in files: - # skip missing files - if not self.os_ops.path_exists(f): - continue - - file_size = self.os_ops.get_file_size(f) - assert type(file_size) == int # noqa: E721 - assert file_size >= 0 - - result[f] = file_size - - return result - def init(self, initdb_params=None, cached=True, **kwargs): """ Perform initdb for this node. @@ -1062,22 +1040,6 @@ def slow_start(self, replica=False, dbname='template1', username=None, max_attem OperationalError}, max_attempts=max_attempts) - def _detect_port_conflict(self, log_files0, log_files1): - assert type(log_files0) == dict # noqa: E721 - assert type(log_files1) == dict # noqa: E721 - - for file in log_files1.keys(): - read_pos = 0 - - if file in log_files0.keys(): - read_pos = log_files0[file] # the previous size - - file_content = self.os_ops.read_binary(file, read_pos) - file_content_s = file_content.decode() - if 'Is another postmaster already running on port' in file_content_s: - return True - return False - def start(self, params=[], wait=True, exec_env=None): """ Starts the PostgreSQL node using pg_ctl if node has not been started. @@ -1137,8 +1099,7 @@ def LOCAL__raise_cannot_start_node__std(from_exception): assert isinstance(self._port_manager, PortManager) assert __class__._C_MAX_START_ATEMPTS > 1 - log_files0 = self._collect_log_files() - assert type(log_files0) == dict # noqa: E721 + log_reader = PostgresNodeLogReader(self, from_beginnig=False) nAttempt = 0 timeout = 1 @@ -1154,11 +1115,11 @@ def LOCAL__raise_cannot_start_node__std(from_exception): if nAttempt == __class__._C_MAX_START_ATEMPTS: LOCAL__raise_cannot_start_node(e, "Cannot start node after multiple attempts.") - log_files1 = self._collect_log_files() - if not self._detect_port_conflict(log_files0, log_files1): + is_it_port_conflict = PostgresNodeUtils.delect_port_conflict(log_reader) + + if not is_it_port_conflict: LOCAL__raise_cannot_start_node__std(e) - log_files0 = log_files1 logging.warning( "Detected a conflict with using the port {0}. Trying another port after a {1}-second sleep...".format(self._port, timeout) ) @@ -2192,6 +2153,167 @@ def _escape_config_value(value): return result +class PostgresNodeLogReader: + class LogInfo: + position: int + + def __init__(self, position: int): + self.position = position + + # -------------------------------------------------------------------- + class LogDataBlock: + _file_name: str + _position: int + _data: str + + def __init__( + self, + file_name: str, + position: int, + data: str + ): + assert type(file_name) == str # noqa: E721 + assert type(position) == int # noqa: E721 + assert type(data) == str # noqa: E721 + assert file_name != "" + assert position >= 0 + self._file_name = file_name + self._position = position + self._data = data + + @property + def file_name(self) -> str: + assert type(self._file_name) == str # noqa: E721 + assert self._file_name != "" + return self._file_name + + @property + def position(self) -> int: + assert type(self._position) == int # noqa: E721 + assert self._position >= 0 + return self._position + + @property + def data(self) -> str: + assert type(self._data) == str # noqa: E721 + return self._data + + # -------------------------------------------------------------------- + _node: PostgresNode + _logs: typing.Dict[str, LogInfo] + + # -------------------------------------------------------------------- + def __init__(self, node: PostgresNode, from_beginnig: bool): + assert node is not None + assert isinstance(node, PostgresNode) + assert type(from_beginnig) == bool # noqa: E721 + + self._node = node + + if from_beginnig: + self._logs = dict() + else: + self._logs = self._collect_logs() + + assert type(self._logs) == dict # noqa: E721 + return + + def read(self) -> typing.List[LogDataBlock]: + assert self._node is not None + assert isinstance(self._node, PostgresNode) + + cur_logs: typing.Dict[__class__.LogInfo] = self._collect_logs() + assert cur_logs is not None + assert type(cur_logs) == dict # noqa: E721 + + assert type(self._logs) == dict # noqa: E721 + + result = list() + + for file_name, cur_log_info in cur_logs.items(): + assert type(file_name) == str # noqa: E721 + assert type(cur_log_info) == __class__.LogInfo # noqa: E721 + + read_pos = 0 + + if file_name in self._logs.keys(): + prev_log_info = self._logs[file_name] + assert type(prev_log_info) == __class__.LogInfo # noqa: E721 + read_pos = prev_log_info.position # the previous size + + file_content_b = self._node.os_ops.read_binary(file_name, read_pos) + assert type(file_content_b) == bytes # noqa: E721 + + # + # A POTENTIAL PROBLEM: file_content_b may contain an incompleted UTF-8 symbol. + # + file_content_s = file_content_b.decode() + assert type(file_content_s) == str # noqa: E721 + + next_read_pos = read_pos + len(file_content_b) + + # It is a research/paranoja check. + # When we will process partial UTF-8 symbol, it must be adjusted. + assert cur_log_info.position <= next_read_pos + + cur_log_info.position = next_read_pos + + block = __class__.LogDataBlock( + file_name, + read_pos, + file_content_s + ) + + result.append(block) + + # A new check point + self._logs = cur_logs + + return result + + def _collect_logs(self) -> typing.Dict[LogInfo]: + assert self._node is not None + assert isinstance(self._node, PostgresNode) + + files = [ + self._node.pg_log_file + ] # yapf: disable + + result = dict() + + for f in files: + assert type(f) == str # noqa: E721 + + # skip missing files + if not self._node.os_ops.path_exists(f): + continue + + file_size = self._node.os_ops.get_file_size(f) + assert type(file_size) == int # noqa: E721 + assert file_size >= 0 + + result[f] = __class__.LogInfo(file_size) + + return result + + +class PostgresNodeUtils: + @staticmethod + def delect_port_conflict(log_reader: PostgresNodeLogReader) -> bool: + assert type(log_reader) == PostgresNodeLogReader # noqa: E721 + + blocks = log_reader.read() + assert type(blocks) == list # noqa: E721 + + for block in blocks: + assert type(block) == PostgresNodeLogReader.LogDataBlock # noqa: E721 + + if 'Is another postmaster already running on port' in block.data: + return True + + return False + + class NodeApp: def __init__(self, test_path=None, nodes_to_cleanup=None, os_ops=None): diff --git a/tests/test_testgres_common.py b/tests/test_testgres_common.py index 5b926bc8..cf203a67 100644 --- a/tests/test_testgres_common.py +++ b/tests/test_testgres_common.py @@ -5,6 +5,8 @@ from testgres.node import PgVer from testgres.node import PostgresNode +from testgres.node import PostgresNodeLogReader +from testgres.node import PostgresNodeUtils from testgres.utils import get_pg_version2 from testgres.utils import file_tail from testgres.utils import get_bin_path2 @@ -883,8 +885,29 @@ def test_backup_wrong_xlog_method(self, node_svc: PostgresNodeService): def test_pg_ctl_wait_option(self, node_svc: PostgresNodeService): assert isinstance(node_svc, PostgresNodeService) - with __class__.helper__get_node(node_svc) as node: - self.impl__test_pg_ctl_wait_option(node_svc, node) + + C_MAX_ATTEMPT = 5 + + nAttempt = 0 + + while True: + if nAttempt == C_MAX_ATTEMPT: + raise Exception("PostgresSQL did not start.") + + nAttempt += 1 + logging.info("------------------------ NODE #{}".format( + nAttempt + )) + + with __class__.helper__get_node(node_svc, port=12345) as node: + if self.impl__test_pg_ctl_wait_option(node_svc, node): + break + continue + + logging.info("OK. Test is passed. Number of attempts is {}".format( + nAttempt + )) + return def impl__test_pg_ctl_wait_option( self, @@ -899,9 +922,18 @@ def impl__test_pg_ctl_wait_option( node.init() assert node.status() == NodeStatus.Stopped + + node_log_reader = PostgresNodeLogReader(node, from_beginnig=True) + node.start(wait=False) nAttempt = 0 while True: + if PostgresNodeUtils.delect_port_conflict(node_log_reader): + logging.info("Node port {} conflicted with another PostgreSQL instance.".format( + node.port + )) + return False + if nAttempt == C_MAX_ATTEMPTS: # # [2025-03-11] @@ -960,6 +992,7 @@ def impl__test_pg_ctl_wait_option( raise Exception("Unexpected node status: {0}.".format(s1)) logging.info("OK. Node is stopped.") + return True def test_replicate(self, node_svc: PostgresNodeService): assert isinstance(node_svc, PostgresNodeService) From 4c6bb1714b4102504a86cb200c9dd04d7036acaa Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Wed, 7 May 2025 22:16:04 +0300 Subject: [PATCH 472/475] Update README.md The version of supported Python is corrected. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a3b854f8..defbc8b3 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ # testgres -PostgreSQL testing utility. Both Python 2.7 and 3.3+ are supported. +PostgreSQL testing utility. Python 3.8+ is supported. ## Installation From 354de695b13be37ad481f92a084df9809704eea1 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Wed, 7 May 2025 22:22:54 +0300 Subject: [PATCH 473/475] [CI] Python 2 is not supported any more. --- Dockerfile--std-all.tmpl | 5 ----- Dockerfile--std.tmpl | 5 ----- 2 files changed, 10 deletions(-) diff --git a/Dockerfile--std-all.tmpl b/Dockerfile--std-all.tmpl index c41c5a06..d19f52a6 100644 --- a/Dockerfile--std-all.tmpl +++ b/Dockerfile--std-all.tmpl @@ -4,11 +4,6 @@ ARG PYTHON_VERSION # --------------------------------------------- base1 FROM postgres:${PG_VERSION}-alpine as base1 -# --------------------------------------------- base2_with_python-2 -FROM base1 as base2_with_python-2 -RUN apk add --no-cache curl python2 python2-dev build-base musl-dev linux-headers py-virtualenv py-pip -ENV PYTHON_VERSION=2 - # --------------------------------------------- base2_with_python-3 FROM base1 as base2_with_python-3 RUN apk add --no-cache curl python3 python3-dev build-base musl-dev linux-headers py-virtualenv diff --git a/Dockerfile--std.tmpl b/Dockerfile--std.tmpl index 91886ede..67aa30b4 100644 --- a/Dockerfile--std.tmpl +++ b/Dockerfile--std.tmpl @@ -4,11 +4,6 @@ ARG PYTHON_VERSION # --------------------------------------------- base1 FROM postgres:${PG_VERSION}-alpine as base1 -# --------------------------------------------- base2_with_python-2 -FROM base1 as base2_with_python-2 -RUN apk add --no-cache curl python2 python2-dev build-base musl-dev linux-headers py-virtualenv py-pip -ENV PYTHON_VERSION=2 - # --------------------------------------------- base2_with_python-3 FROM base1 as base2_with_python-3 RUN apk add --no-cache curl python3 python3-dev build-base musl-dev linux-headers py-virtualenv From d48477d5e32ab0870229c5f676172111fd6ba3ee Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Mon, 12 May 2025 07:18:23 +0300 Subject: [PATCH 474/475] [#258] Problems in ProbackupTest and TestBasic(ProbackupTest) are fixed (#259) * [#258] Declaration of ProbackupTest::pg_node is corrected ProbackupTest::pg_node is testgres.NodeApp, not testgres.PostgresNode. Asserts are added. * [#258] TestBasic::test_full_backup cleans node object Node cleanup is added. TODO: we should to stop node only and cleanup his data in conftest. --- .../pg_probackup2/tests/test_basic.py | 35 ++++++++++++------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/tests/test_basic.py b/testgres/plugins/pg_probackup2/pg_probackup2/tests/test_basic.py index f22a62bf..2540ddb0 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/tests/test_basic.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/tests/test_basic.py @@ -11,7 +11,7 @@ class ProbackupTest: - pg_node: testgres.PostgresNode + pg_node: testgres.NodeApp @staticmethod def probackup_is_available() -> bool: @@ -75,21 +75,30 @@ def helper__build_backup_dir(self, backup='backup'): @pytest.mark.skipif(not ProbackupTest.probackup_is_available(), reason="Check that PGPROBACKUPBIN is defined and is valid.") class TestBasic(ProbackupTest): def test_full_backup(self): + assert self.pg_node is not None + assert type(self.pg_node) == testgres.NodeApp # noqa: E721 + assert self.pb is not None + assert type(self.pb) == ProbackupApp # noqa: E721 + # Setting up a simple test node node = self.pg_node.make_simple('node', pg_options={"fsync": "off", "synchronous_commit": "off"}) - # Initialize and configure Probackup - self.pb.init() - self.pb.add_instance('node', node) - self.pb.set_archiving('node', node) + assert node is not None + assert type(node) == testgres.PostgresNode # noqa: E721 + + with node: + # Initialize and configure Probackup + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) - # Start the node and initialize pgbench - node.slow_start() - node.pgbench_init(scale=100, no_vacuum=True) + # Start the node and initialize pgbench + node.slow_start() + node.pgbench_init(scale=100, no_vacuum=True) - # Perform backup and validation - backup_id = self.pb.backup_node('node', node) - out = self.pb.validate('node', backup_id) + # Perform backup and validation + backup_id = self.pb.backup_node('node', node) + out = self.pb.validate('node', backup_id) - # Check if the backup is valid - assert f"INFO: Backup {backup_id} is valid" in out + # Check if the backup is valid + assert f"INFO: Backup {backup_id} is valid" in out From 0470d305af4af16edd8ffa56d05a5b90cad1e128 Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Mon, 12 May 2025 18:29:10 +0300 Subject: [PATCH 475/475] conftest is updated (#260) - the calls of logging.root.handle are handled (LogWrapper2) - critical errors are processed --- tests/conftest.py | 200 +++++++++++++++++++++++----------------------- 1 file changed, 100 insertions(+), 100 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 9e74879b..111edc87 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -338,6 +338,7 @@ def helper__build_test_id(item: pytest.Function) -> str: g_error_msg_count_key = pytest.StashKey[int]() g_warning_msg_count_key = pytest.StashKey[int]() +g_critical_msg_count_key = pytest.StashKey[int]() # ///////////////////////////////////////////////////////////////////////////// @@ -413,10 +414,17 @@ def helper__makereport__call( assert type(outcome) == pluggy.Result # noqa: E721 # -------- - item_error_msg_count = item.stash.get(g_error_msg_count_key, 0) - assert type(item_error_msg_count) == int # noqa: E721 - assert item_error_msg_count >= 0 + item_error_msg_count1 = item.stash.get(g_error_msg_count_key, 0) + assert type(item_error_msg_count1) == int # noqa: E721 + assert item_error_msg_count1 >= 0 + item_error_msg_count2 = item.stash.get(g_critical_msg_count_key, 0) + assert type(item_error_msg_count2) == int # noqa: E721 + assert item_error_msg_count2 >= 0 + + item_error_msg_count = item_error_msg_count1 + item_error_msg_count2 + + # -------- item_warning_msg_count = item.stash.get(g_warning_msg_count_key, 0) assert type(item_warning_msg_count) == int # noqa: E721 assert item_warning_msg_count >= 0 @@ -600,103 +608,87 @@ def pytest_runtest_makereport(item: pytest.Function, call: pytest.CallInfo): # ///////////////////////////////////////////////////////////////////////////// -class LogErrorWrapper2: +class LogWrapper2: _old_method: any - _counter: typing.Optional[int] + _err_counter: typing.Optional[int] + _warn_counter: typing.Optional[int] + + _critical_counter: typing.Optional[int] # -------------------------------------------------------------------- def __init__(self): self._old_method = None - self._counter = None + self._err_counter = None + self._warn_counter = None + + self._critical_counter = None # -------------------------------------------------------------------- def __enter__(self): assert self._old_method is None - assert self._counter is None - - self._old_method = logging.error - self._counter = 0 - - logging.error = self - return self - - # -------------------------------------------------------------------- - def __exit__(self, exc_type, exc_val, exc_tb): - assert self._old_method is not None - assert self._counter is not None - - assert logging.error is self - - logging.error = self._old_method - - self._old_method = None - self._counter = None - return False - - # -------------------------------------------------------------------- - def __call__(self, *args, **kwargs): - assert self._old_method is not None - assert self._counter is not None - - assert type(self._counter) == int # noqa: E721 - assert self._counter >= 0 - - r = self._old_method(*args, **kwargs) - - self._counter += 1 - assert self._counter > 0 - - return r - - -# ///////////////////////////////////////////////////////////////////////////// + assert self._err_counter is None + assert self._warn_counter is None + assert self._critical_counter is None -class LogWarningWrapper2: - _old_method: any - _counter: typing.Optional[int] + assert logging.root is not None + assert isinstance(logging.root, logging.RootLogger) - # -------------------------------------------------------------------- - def __init__(self): - self._old_method = None - self._counter = None + self._old_method = logging.root.handle + self._err_counter = 0 + self._warn_counter = 0 - # -------------------------------------------------------------------- - def __enter__(self): - assert self._old_method is None - assert self._counter is None + self._critical_counter = 0 - self._old_method = logging.warning - self._counter = 0 - - logging.warning = self + logging.root.handle = self return self # -------------------------------------------------------------------- def __exit__(self, exc_type, exc_val, exc_tb): assert self._old_method is not None - assert self._counter is not None + assert self._err_counter is not None + assert self._warn_counter is not None + + assert logging.root is not None + assert isinstance(logging.root, logging.RootLogger) - assert logging.warning is self + assert logging.root.handle is self - logging.warning = self._old_method + logging.root.handle = self._old_method self._old_method = None - self._counter = None + self._err_counter = None + self._warn_counter = None + self._critical_counter = None return False # -------------------------------------------------------------------- - def __call__(self, *args, **kwargs): + def __call__(self, record: logging.LogRecord): + assert record is not None + assert isinstance(record, logging.LogRecord) assert self._old_method is not None - assert self._counter is not None - - assert type(self._counter) == int # noqa: E721 - assert self._counter >= 0 - - r = self._old_method(*args, **kwargs) - - self._counter += 1 - assert self._counter > 0 + assert self._err_counter is not None + assert self._warn_counter is not None + assert self._critical_counter is not None + + assert type(self._err_counter) == int # noqa: E721 + assert self._err_counter >= 0 + assert type(self._warn_counter) == int # noqa: E721 + assert self._warn_counter >= 0 + assert type(self._critical_counter) == int # noqa: E721 + assert self._critical_counter >= 0 + + r = self._old_method(record) + + if record.levelno == logging.ERROR: + self._err_counter += 1 + assert self._err_counter > 0 + elif record.levelno == logging.WARNING: + self._warn_counter += 1 + assert self._warn_counter > 0 + elif record.levelno == logging.CRITICAL: + self._critical_counter += 1 + assert self._critical_counter > 0 return r @@ -717,6 +709,13 @@ def pytest_pyfunc_call(pyfuncitem: pytest.Function): assert pyfuncitem is not None assert isinstance(pyfuncitem, pytest.Function) + assert logging.root is not None + assert isinstance(logging.root, logging.RootLogger) + assert logging.root.handle is not None + + debug__log_handle_method = logging.root.handle + assert debug__log_handle_method is not None + debug__log_error_method = logging.error assert debug__log_error_method is not None @@ -725,55 +724,56 @@ def pytest_pyfunc_call(pyfuncitem: pytest.Function): pyfuncitem.stash[g_error_msg_count_key] = 0 pyfuncitem.stash[g_warning_msg_count_key] = 0 + pyfuncitem.stash[g_critical_msg_count_key] = 0 try: - with LogErrorWrapper2() as logErrorWrapper, LogWarningWrapper2() as logWarningWrapper: - assert type(logErrorWrapper) == LogErrorWrapper2 # noqa: E721 - assert logErrorWrapper._old_method is not None - assert type(logErrorWrapper._counter) == int # noqa: E721 - assert logErrorWrapper._counter == 0 - assert logging.error is logErrorWrapper - - assert type(logWarningWrapper) == LogWarningWrapper2 # noqa: E721 - assert logWarningWrapper._old_method is not None - assert type(logWarningWrapper._counter) == int # noqa: E721 - assert logWarningWrapper._counter == 0 - assert logging.warning is logWarningWrapper + with LogWrapper2() as logWrapper: + assert type(logWrapper) == LogWrapper2 # noqa: E721 + assert logWrapper._old_method is not None + assert type(logWrapper._err_counter) == int # noqa: E721 + assert logWrapper._err_counter == 0 + assert type(logWrapper._warn_counter) == int # noqa: E721 + assert logWrapper._warn_counter == 0 + assert type(logWrapper._critical_counter) == int # noqa: E721 + assert logWrapper._critical_counter == 0 + assert logging.root.handle is logWrapper r: pluggy.Result = yield assert r is not None assert type(r) == pluggy.Result # noqa: E721 - assert logErrorWrapper._old_method is not None - assert type(logErrorWrapper._counter) == int # noqa: E721 - assert logErrorWrapper._counter >= 0 - assert logging.error is logErrorWrapper - - assert logWarningWrapper._old_method is not None - assert type(logWarningWrapper._counter) == int # noqa: E721 - assert logWarningWrapper._counter >= 0 - assert logging.warning is logWarningWrapper + assert logWrapper._old_method is not None + assert type(logWrapper._err_counter) == int # noqa: E721 + assert logWrapper._err_counter >= 0 + assert type(logWrapper._warn_counter) == int # noqa: E721 + assert logWrapper._warn_counter >= 0 + assert type(logWrapper._critical_counter) == int # noqa: E721 + assert logWrapper._critical_counter >= 0 + assert logging.root.handle is logWrapper assert g_error_msg_count_key in pyfuncitem.stash assert g_warning_msg_count_key in pyfuncitem.stash + assert g_critical_msg_count_key in pyfuncitem.stash assert pyfuncitem.stash[g_error_msg_count_key] == 0 assert pyfuncitem.stash[g_warning_msg_count_key] == 0 + assert pyfuncitem.stash[g_critical_msg_count_key] == 0 - pyfuncitem.stash[g_error_msg_count_key] = logErrorWrapper._counter - pyfuncitem.stash[g_warning_msg_count_key] = logWarningWrapper._counter + pyfuncitem.stash[g_error_msg_count_key] = logWrapper._err_counter + pyfuncitem.stash[g_warning_msg_count_key] = logWrapper._warn_counter + pyfuncitem.stash[g_critical_msg_count_key] = logWrapper._critical_counter if r.exception is not None: pass - elif logErrorWrapper._counter == 0: - pass - else: - assert logErrorWrapper._counter > 0 + elif logWrapper._err_counter > 0: + r.force_exception(SIGNAL_EXCEPTION()) + elif logWrapper._critical_counter > 0: r.force_exception(SIGNAL_EXCEPTION()) finally: assert logging.error is debug__log_error_method assert logging.warning is debug__log_warning_method + assert logging.root.handle == debug__log_handle_method pass