From fd11545624381d1d868ea76c01b439a1754360f4 Mon Sep 17 00:00:00 2001 From: Viktoriia Shepard Date: Fri, 27 Oct 2023 23:00:54 +0200 Subject: [PATCH 001/216] Change setting user in start_node --- testgres/node.py | 4 +++- testgres/operations/local_ops.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index 84c25327..52e6d2ee 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -659,7 +659,7 @@ def get_control_data(self): return out_dict - def slow_start(self, replica=False, dbname='template1', username=default_username(), max_attempts=0): + def slow_start(self, replica=False, dbname='template1', username=None, max_attempts=0): """ Starts the PostgreSQL instance and then polls the instance until it reaches the expected state (primary or replica). The state is checked @@ -672,6 +672,8 @@ def slow_start(self, replica=False, dbname='template1', username=default_usernam If False, waits for the instance to be in primary mode. Default is False. max_attempts: """ + if not username: + username = default_username() self.start() if replica: diff --git a/testgres/operations/local_ops.py b/testgres/operations/local_ops.py index a692750e..36b14058 100644 --- a/testgres/operations/local_ops.py +++ b/testgres/operations/local_ops.py @@ -117,7 +117,7 @@ def set_env(self, var_name, var_val): # Get environment variables def get_user(self): - return getpass.getuser() + return self.username or getpass.getuser() def get_name(self): return os.name From eafa7f0da736a7dcf9362845db769905efbef306 Mon Sep 17 00:00:00 2001 From: vshepard Date: Thu, 9 Nov 2023 21:11:45 +0100 Subject: [PATCH 002/216] problem with get_pg_version in Homebrew Fix issue 87 --- testgres/utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/testgres/utils.py b/testgres/utils.py index b7df70d1..db75fadc 100644 --- a/testgres/utils.py +++ b/testgres/utils.py @@ -179,6 +179,9 @@ def get_pg_version(): _params = [get_bin_path('postgres'), '--version'] raw_ver = tconf.os_ops.exec_command(_params, encoding='utf-8') + # Remove "(Homebrew)" if present + raw_ver = raw_ver.replace('(Homebrew)', '').strip() + # cook version of PostgreSQL version = raw_ver.strip().split(' ')[-1] \ .partition('devel')[0] \ From 4200b8096f198a36d8bf109d1b8da679b66322ad Mon Sep 17 00:00:00 2001 From: vshepard Date: Wed, 22 Nov 2023 09:34:13 +0100 Subject: [PATCH 003/216] RemoteOperations add_known_host macos fix --- testgres/operations/remote_ops.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index 421c0a6d..0a545834 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -3,6 +3,7 @@ import os import subprocess import tempfile +import platform # we support both pg8000 and psycopg2 try: @@ -42,7 +43,8 @@ def cmdline(self): class RemoteOperations(OsOperations): def __init__(self, conn_params: ConnectionParams): - if os.name != "posix": + + if not platform.system().lower() == "linux": raise EnvironmentError("Remote operations are supported only on Linux!") super().__init__(conn_params.username) @@ -76,16 +78,14 @@ def close_ssh_tunnel(self): print("No active tunnel to close.") def add_known_host(self, host): - cmd = 'ssh-keyscan -H %s >> /home/%s/.ssh/known_hosts' % (host, os.getlogin()) + known_hosts_path = os.path.expanduser("~/.ssh/known_hosts") + cmd = 'ssh-keyscan -H %s >> %s' % (host, known_hosts_path) + try: - subprocess.check_call( - cmd, - shell=True, - ) + subprocess.check_call(cmd, shell=True) logging.info("Successfully added %s to known_hosts." % host) except subprocess.CalledProcessError as e: - raise ExecUtilException(message="Failed to add %s to known_hosts. Error: %s" % (host, str(e)), command=cmd, - exit_code=e.returncode, out=e.stderr) + raise Exception("Failed to add %s to known_hosts. Error: %s" % (host, str(e))) def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False, encoding=None, shell=True, text=False, input=None, stdin=None, stdout=None, From 1a2f6dad098a1963d6ef82c707cc26f4c5fadc55 Mon Sep 17 00:00:00 2001 From: Victoria Shepard <5807469+demonolock@users.noreply.github.com> Date: Tue, 19 Dec 2023 17:05:14 +0100 Subject: [PATCH 004/216] Fix initdb error on Windows (#99) --- setup.py | 8 +- testgres/__init__.py | 4 +- testgres/helpers/__init__.py | 0 testgres/helpers/port_manager.py | 40 +++++++++ testgres/node.py | 8 +- testgres/operations/local_ops.py | 133 +++++++++++++++++------------- testgres/operations/os_ops.py | 8 +- testgres/operations/remote_ops.py | 55 ++++++------ testgres/utils.py | 10 ++- tests/test_remote.py | 9 +- tests/test_simple.py | 47 ++++++++--- tests/test_simple_remote.py | 7 +- 12 files changed, 207 insertions(+), 122 deletions(-) create mode 100644 testgres/helpers/__init__.py create mode 100644 testgres/helpers/port_manager.py mode change 100755 => 100644 tests/test_simple.py diff --git a/setup.py b/setup.py index 16d4c300..e0287659 100755 --- a/setup.py +++ b/setup.py @@ -27,16 +27,16 @@ readme = f.read() setup( - version='1.9.2', + version='1.9.3', name='testgres', - packages=['testgres', 'testgres.operations'], + packages=['testgres', 'testgres.operations', 'testgres.helpers'], description='Testing utility for PostgreSQL and its extensions', url='https://github.com/postgrespro/testgres', long_description=readme, long_description_content_type='text/markdown', license='PostgreSQL', - author='Ildar Musin', - author_email='zildermann@gmail.com', + author='Postgres Professional', + author_email='testgres@postgrespro.ru', keywords=['test', 'testing', 'postgresql'], install_requires=install_requires, classifiers=[], diff --git a/testgres/__init__.py b/testgres/__init__.py index 383daf2d..8d0e38c6 100644 --- a/testgres/__init__.py +++ b/testgres/__init__.py @@ -52,6 +52,8 @@ from .operations.local_ops import LocalOperations from .operations.remote_ops import RemoteOperations +from .helpers.port_manager import PortManager + __all__ = [ "get_new_node", "get_remote_node", @@ -62,6 +64,6 @@ "XLogMethod", "IsolationLevel", "NodeStatus", "ProcessType", "DumpFormat", "PostgresNode", "NodeApp", "reserve_port", "release_port", "bound_ports", "get_bin_path", "get_pg_config", "get_pg_version", - "First", "Any", + "First", "Any", "PortManager", "OsOperations", "LocalOperations", "RemoteOperations", "ConnectionParams" ] diff --git a/testgres/helpers/__init__.py b/testgres/helpers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/testgres/helpers/port_manager.py b/testgres/helpers/port_manager.py new file mode 100644 index 00000000..6afdf8a9 --- /dev/null +++ b/testgres/helpers/port_manager.py @@ -0,0 +1,40 @@ +import socket +import random +from typing import Set, Iterable, Optional + + +class PortForException(Exception): + pass + + +class PortManager: + def __init__(self, ports_range=(1024, 65535)): + self.ports_range = ports_range + + @staticmethod + def is_port_free(port: int) -> bool: + """Check if a port is free to use.""" + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + try: + s.bind(("", port)) + return True + except OSError: + return False + + def find_free_port(self, ports: Optional[Set[int]] = None, exclude_ports: Optional[Iterable[int]] = None) -> int: + """Return a random unused port number.""" + if ports is None: + ports = set(range(1024, 65535)) + + if exclude_ports is None: + exclude_ports = set() + + ports.difference_update(set(exclude_ports)) + + sampled_ports = random.sample(tuple(ports), min(len(ports), 100)) + + for port in sampled_ports: + if self.is_port_free(port): + return port + + raise PortForException("Can't select a port") diff --git a/testgres/node.py b/testgres/node.py index 52e6d2ee..20cf4264 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -623,8 +623,8 @@ def status(self): "-D", self.data_dir, "status" ] # yapf: disable - status_code, out, err = execute_utility(_params, self.utils_log_file, verbose=True) - if 'does not exist' in err: + status_code, out, error = execute_utility(_params, self.utils_log_file, verbose=True) + if error and 'does not exist' in error: return NodeStatus.Uninitialized elif 'no server running' in out: return NodeStatus.Stopped @@ -717,7 +717,7 @@ def start(self, params=[], wait=True): try: exit_status, out, error = execute_utility(_params, self.utils_log_file, verbose=True) - if 'does not exist' in error: + if error and 'does not exist' in error: raise Exception except Exception as e: msg = 'Cannot start node' @@ -791,7 +791,7 @@ def restart(self, params=[]): try: error_code, out, error = execute_utility(_params, self.utils_log_file, verbose=True) - if 'could not start server' in error: + if error and 'could not start server' in error: raise ExecUtilException except ExecUtilException as e: msg = 'Cannot restart node' diff --git a/testgres/operations/local_ops.py b/testgres/operations/local_ops.py index 36b14058..93ebf012 100644 --- a/testgres/operations/local_ops.py +++ b/testgres/operations/local_ops.py @@ -8,8 +8,7 @@ import psutil from ..exceptions import ExecUtilException -from .os_ops import ConnectionParams, OsOperations -from .os_ops import pglib +from .os_ops import ConnectionParams, OsOperations, pglib, get_default_encoding try: from shutil import which as find_executable @@ -22,6 +21,14 @@ error_markers = [b'error', b'Permission denied', b'fatal'] +def has_errors(output): + if output: + if isinstance(output, str): + output = output.encode(get_default_encoding()) + return any(marker in output for marker in error_markers) + return False + + class LocalOperations(OsOperations): def __init__(self, conn_params=None): if conn_params is None: @@ -33,72 +40,80 @@ def __init__(self, conn_params=None): self.remote = False self.username = conn_params.username or self.get_user() - # Command execution - def exec_command(self, cmd, wait_exit=False, verbose=False, - expect_error=False, encoding=None, shell=False, text=False, - input=None, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - get_process=None, timeout=None): - """ - Execute a command in a subprocess. - - Args: - - cmd: The command to execute. - - wait_exit: Whether to wait for the subprocess to exit before returning. - - verbose: Whether to return verbose output. - - expect_error: Whether to raise an error if the subprocess exits with an error status. - - encoding: The encoding to use for decoding the subprocess output. - - shell: Whether to use shell when executing the subprocess. - - text: Whether to return str instead of bytes for the subprocess output. - - input: The input to pass to the subprocess. - - stdout: The stdout to use for the subprocess. - - stderr: The stderr to use for the subprocess. - - proc: The process to use for subprocess creation. - :return: The output of the subprocess. - """ - if os.name == 'nt': - with tempfile.NamedTemporaryFile() as buf: - process = subprocess.Popen(cmd, stdout=buf, stderr=subprocess.STDOUT) - process.communicate() - buf.seek(0) - result = buf.read().decode(encoding) - return result - else: + @staticmethod + def _raise_exec_exception(message, command, exit_code, output): + """Raise an ExecUtilException.""" + raise ExecUtilException(message=message.format(output), + command=command, + exit_code=exit_code, + out=output) + + @staticmethod + def _process_output(encoding, temp_file_path): + """Process the output of a command from a temporary file.""" + with open(temp_file_path, 'rb') as temp_file: + output = temp_file.read() + if encoding: + output = output.decode(encoding) + return output, None # In Windows stderr writing in stdout + + def _run_command(self, cmd, shell, input, stdin, stdout, stderr, get_process, timeout, encoding): + """Execute a command and return the process and its output.""" + if os.name == 'nt' and stdout is None: # Windows + with tempfile.NamedTemporaryFile(mode='w+b', delete=False) as temp_file: + stdout = temp_file + stderr = subprocess.STDOUT + process = subprocess.Popen( + cmd, + shell=shell, + stdin=stdin or subprocess.PIPE if input is not None else None, + stdout=stdout, + stderr=stderr, + ) + if get_process: + return process, None, None + temp_file_path = temp_file.name + + # Wait process finished + process.wait() + + output, error = self._process_output(encoding, temp_file_path) + return process, output, error + else: # Other OS process = subprocess.Popen( cmd, shell=shell, - stdout=stdout, - stderr=stderr, + stdin=stdin or subprocess.PIPE if input is not None else None, + stdout=stdout or subprocess.PIPE, + stderr=stderr or subprocess.PIPE, ) if get_process: - return process - + return process, None, None try: - result, error = process.communicate(input, timeout=timeout) + output, error = process.communicate(input=input.encode(encoding) if input else None, timeout=timeout) + if encoding: + output = output.decode(encoding) + error = error.decode(encoding) + return process, output, error except subprocess.TimeoutExpired: process.kill() raise ExecUtilException("Command timed out after {} seconds.".format(timeout)) - exit_status = process.returncode - - error_found = exit_status != 0 or any(marker in error for marker in error_markers) - if encoding: - result = result.decode(encoding) - error = error.decode(encoding) - - if expect_error: - raise Exception(result, error) - - if exit_status != 0 or error_found: - if exit_status == 0: - exit_status = 1 - raise ExecUtilException(message='Utility exited with non-zero code. Error `{}`'.format(error), - command=cmd, - exit_code=exit_status, - out=result) - if verbose: - return exit_status, result, error - else: - return result + def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False, encoding=None, shell=False, + text=False, input=None, stdin=None, stdout=None, stderr=None, get_process=False, timeout=None): + """ + Execute a command in a subprocess and handle the output based on the provided parameters. + """ + process, output, error = self._run_command(cmd, shell, input, stdin, stdout, stderr, get_process, timeout, encoding) + if get_process: + return process + if process.returncode != 0 or (has_errors(error) and not expect_error): + self._raise_exec_exception('Utility exited with non-zero code. Error `{}`', cmd, process.returncode, error) + + if verbose: + return process.returncode, output, error + else: + return output # Environment setup def environ(self, var_name): @@ -210,7 +225,7 @@ def read(self, filename, encoding=None, binary=False): if binary: return content if isinstance(content, bytes): - return content.decode(encoding or 'utf-8') + return content.decode(encoding or get_default_encoding()) return content def readlines(self, filename, num_lines=0, binary=False, encoding=None): diff --git a/testgres/operations/os_ops.py b/testgres/operations/os_ops.py index 9261cacf..dd6613cf 100644 --- a/testgres/operations/os_ops.py +++ b/testgres/operations/os_ops.py @@ -1,3 +1,5 @@ +import locale + try: import psycopg2 as pglib # noqa: F401 except ImportError: @@ -14,6 +16,10 @@ def __init__(self, host='127.0.0.1', ssh_key=None, username=None): self.username = username +def get_default_encoding(): + return locale.getdefaultlocale()[1] or 'UTF-8' + + class OsOperations: def __init__(self, username=None): self.ssh_key = None @@ -75,7 +81,7 @@ def write(self, filename, data, truncate=False, binary=False, read_and_write=Fal def touch(self, filename): raise NotImplementedError() - def read(self, filename): + def read(self, filename, encoding, binary): raise NotImplementedError() def readlines(self, filename): diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index 0a545834..01251e1c 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -1,4 +1,3 @@ -import locale import logging import os import subprocess @@ -15,12 +14,7 @@ raise ImportError("You must have psycopg2 or pg8000 modules installed") from ..exceptions import ExecUtilException - -from .os_ops import OsOperations, ConnectionParams - -ConsoleEncoding = locale.getdefaultlocale()[1] -if not ConsoleEncoding: - ConsoleEncoding = 'UTF-8' +from .os_ops import OsOperations, ConnectionParams, get_default_encoding error_markers = [b'error', b'Permission denied', b'fatal', b'No such file or directory'] @@ -36,7 +30,7 @@ def kill(self): def cmdline(self): command = "ps -p {} -o cmd --no-headers".format(self.pid) - stdin, stdout, stderr = self.ssh.exec_command(command, verbose=True, encoding=ConsoleEncoding) + stdin, stdout, stderr = self.ssh.exec_command(command, verbose=True, encoding=get_default_encoding()) cmdline = stdout.strip() return cmdline.split() @@ -51,6 +45,10 @@ def __init__(self, conn_params: ConnectionParams): self.conn_params = conn_params self.host = conn_params.host self.ssh_key = conn_params.ssh_key + if self.ssh_key: + self.ssh_cmd = ["-i", self.ssh_key] + else: + self.ssh_cmd = [] self.remote = True self.username = conn_params.username or self.get_user() self.add_known_host(self.host) @@ -97,9 +95,9 @@ def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False, """ ssh_cmd = [] if isinstance(cmd, str): - ssh_cmd = ['ssh', f"{self.username}@{self.host}", '-i', self.ssh_key, cmd] + ssh_cmd = ['ssh', f"{self.username}@{self.host}"] + self.ssh_cmd + [cmd] elif isinstance(cmd, list): - ssh_cmd = ['ssh', f"{self.username}@{self.host}", '-i', self.ssh_key] + cmd + ssh_cmd = ['ssh', f"{self.username}@{self.host}"] + self.ssh_cmd + cmd process = subprocess.Popen(ssh_cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) if get_process: return process @@ -145,7 +143,7 @@ def environ(self, var_name: str) -> str: - var_name (str): The name of the environment variable. """ cmd = "echo ${}".format(var_name) - return self.exec_command(cmd, encoding=ConsoleEncoding).strip() + return self.exec_command(cmd, encoding=get_default_encoding()).strip() def find_executable(self, executable): search_paths = self.environ("PATH") @@ -176,11 +174,11 @@ def set_env(self, var_name: str, var_val: str): # Get environment variables def get_user(self): - return self.exec_command("echo $USER", encoding=ConsoleEncoding).strip() + return self.exec_command("echo $USER", encoding=get_default_encoding()).strip() def get_name(self): cmd = 'python3 -c "import os; print(os.name)"' - return self.exec_command(cmd, encoding=ConsoleEncoding).strip() + return self.exec_command(cmd, encoding=get_default_encoding()).strip() # Work with dirs def makedirs(self, path, remove_existing=False): @@ -227,7 +225,7 @@ def listdir(self, path): return result.splitlines() def path_exists(self, path): - result = self.exec_command("test -e {}; echo $?".format(path), encoding=ConsoleEncoding) + result = self.exec_command("test -e {}; echo $?".format(path), encoding=get_default_encoding()) return int(result.strip()) == 0 @property @@ -248,9 +246,9 @@ def mkdtemp(self, prefix=None): - prefix (str): The prefix of the temporary directory name. """ if prefix: - command = ["ssh", "-i", self.ssh_key, f"{self.username}@{self.host}", f"mktemp -d {prefix}XXXXX"] + command = ["ssh"] + self.ssh_cmd + [f"{self.username}@{self.host}", f"mktemp -d {prefix}XXXXX"] else: - command = ["ssh", "-i", self.ssh_key, f"{self.username}@{self.host}", "mktemp -d"] + command = ["ssh"] + self.ssh_cmd + [f"{self.username}@{self.host}", "mktemp -d"] result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) @@ -264,9 +262,9 @@ def mkdtemp(self, prefix=None): def mkstemp(self, prefix=None): if prefix: - temp_dir = self.exec_command("mktemp {}XXXXX".format(prefix), encoding=ConsoleEncoding) + temp_dir = self.exec_command("mktemp {}XXXXX".format(prefix), encoding=get_default_encoding()) else: - temp_dir = self.exec_command("mktemp", encoding=ConsoleEncoding) + temp_dir = self.exec_command("mktemp", encoding=get_default_encoding()) if temp_dir: if not os.path.isabs(temp_dir): @@ -283,7 +281,9 @@ def copytree(self, src, dst): return self.exec_command("cp -r {} {}".format(src, dst)) # Work with files - def write(self, filename, data, truncate=False, binary=False, read_and_write=False, encoding=ConsoleEncoding): + def write(self, filename, data, truncate=False, binary=False, read_and_write=False, encoding=None): + if not encoding: + encoding = get_default_encoding() mode = "wb" if binary else "w" if not truncate: mode = "ab" if binary else "a" @@ -292,7 +292,7 @@ def write(self, filename, data, truncate=False, binary=False, read_and_write=Fal with tempfile.NamedTemporaryFile(mode=mode, delete=False) as tmp_file: if not truncate: - scp_cmd = ['scp', '-i', self.ssh_key, f"{self.username}@{self.host}:{filename}", tmp_file.name] + scp_cmd = ['scp'] + self.ssh_cmd + [f"{self.username}@{self.host}:{filename}", tmp_file.name] subprocess.run(scp_cmd, check=False) # The file might not exist yet tmp_file.seek(0, os.SEEK_END) @@ -302,18 +302,17 @@ def write(self, filename, data, truncate=False, binary=False, read_and_write=Fal data = data.encode(encoding) if isinstance(data, list): - data = [(s if isinstance(s, str) else s.decode(ConsoleEncoding)).rstrip('\n') + '\n' for s in data] + data = [(s if isinstance(s, str) else s.decode(get_default_encoding())).rstrip('\n') + '\n' for s in data] tmp_file.writelines(data) else: tmp_file.write(data) tmp_file.flush() - - scp_cmd = ['scp', '-i', self.ssh_key, tmp_file.name, f"{self.username}@{self.host}:{filename}"] + scp_cmd = ['scp'] + self.ssh_cmd + [tmp_file.name, f"{self.username}@{self.host}:{filename}"] subprocess.run(scp_cmd, check=True) remote_directory = os.path.dirname(filename) - mkdir_cmd = ['ssh', '-i', self.ssh_key, f"{self.username}@{self.host}", f"mkdir -p {remote_directory}"] + mkdir_cmd = ['ssh'] + self.ssh_cmd + [f"{self.username}@{self.host}", f"mkdir -p {remote_directory}"] subprocess.run(mkdir_cmd, check=True) os.remove(tmp_file.name) @@ -334,7 +333,7 @@ def read(self, filename, binary=False, encoding=None): result = self.exec_command(cmd, encoding=encoding) if not binary and result: - result = result.decode(encoding or ConsoleEncoding) + result = result.decode(encoding or get_default_encoding()) return result @@ -347,7 +346,7 @@ def readlines(self, filename, num_lines=0, binary=False, encoding=None): result = self.exec_command(cmd, encoding=encoding) if not binary and result: - lines = result.decode(encoding or ConsoleEncoding).splitlines() + lines = result.decode(encoding or get_default_encoding()).splitlines() else: lines = result.splitlines() @@ -375,10 +374,10 @@ def kill(self, pid, signal): def get_pid(self): # Get current process id - return int(self.exec_command("echo $$", encoding=ConsoleEncoding)) + return int(self.exec_command("echo $$", encoding=get_default_encoding())) def get_process_children(self, pid): - command = ["ssh", "-i", self.ssh_key, f"{self.username}@{self.host}", f"pgrep -P {pid}"] + command = ["ssh"] + self.ssh_cmd + [f"{self.username}@{self.host}", f"pgrep -P {pid}"] result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) diff --git a/testgres/utils.py b/testgres/utils.py index db75fadc..b21fc2c8 100644 --- a/testgres/utils.py +++ b/testgres/utils.py @@ -4,7 +4,7 @@ from __future__ import print_function import os -import port_for + import sys from contextlib import contextmanager @@ -13,6 +13,7 @@ from six import iteritems +from .helpers.port_manager import PortManager from .exceptions import ExecUtilException from .config import testgres_config as tconf @@ -37,8 +38,8 @@ def reserve_port(): """ Generate a new port and add it to 'bound_ports'. """ - - port = port_for.select_random(exclude_ports=bound_ports) + port_mng = PortManager() + port = port_mng.find_free_port(exclude_ports=bound_ports) bound_ports.add(port) return port @@ -80,7 +81,8 @@ def execute_utility(args, logfile=None, verbose=False): lines = [u'\n'] + ['# ' + line for line in out.splitlines()] + [u'\n'] tconf.os_ops.write(filename=logfile, data=lines) except IOError: - raise ExecUtilException("Problem with writing to logfile `{}` during run command `{}`".format(logfile, args)) + raise ExecUtilException( + "Problem with writing to logfile `{}` during run command `{}`".format(logfile, args)) if verbose: return exit_status, out, error else: diff --git a/tests/test_remote.py b/tests/test_remote.py index 2e0f0676..e0e4a555 100755 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -11,10 +11,9 @@ class TestRemoteOperations: @pytest.fixture(scope="function", autouse=True) def setup(self): - conn_params = ConnectionParams(host=os.getenv('RDBMS_TESTPOOL1_HOST') or '172.18.0.3', - username='dev', - ssh_key=os.getenv( - 'RDBMS_TESTPOOL_SSHKEY') or '../../container_files/postgres/ssh/id_ed25519') + conn_params = ConnectionParams(host=os.getenv('RDBMS_TESTPOOL1_HOST') or '127.0.0.1', + username=os.getenv('USER'), + ssh_key=os.getenv('RDBMS_TESTPOOL_SSHKEY')) self.operations = RemoteOperations(conn_params) def test_exec_command_success(self): @@ -41,7 +40,7 @@ def test_is_executable_true(self): """ Test is_executable for an existing executable. """ - cmd = "postgres" + cmd = os.getenv('PG_CONFIG') response = self.operations.is_executable(cmd) assert response is True diff --git a/tests/test_simple.py b/tests/test_simple.py old mode 100755 new mode 100644 index 45c28a21..9d31d4d9 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -74,6 +74,24 @@ def good_properties(f): return True +def rm_carriage_returns(out): + """ + In Windows we have additional '\r' symbols in output. + Let's get rid of them. + """ + if os.name == 'nt': + if isinstance(out, (int, float, complex)): + return out + elif isinstance(out, tuple): + return tuple(rm_carriage_returns(item) for item in out) + elif isinstance(out, bytes): + return out.replace(b'\r', b'') + else: + return out.replace('\r', '') + else: + return out + + @contextmanager def removing(f): try: @@ -123,7 +141,7 @@ def test_init_after_cleanup(self): node.cleanup() node.init().start().execute('select 1') - @unittest.skipUnless(util_exists('pg_resetwal'), 'might be missing') + @unittest.skipUnless(util_exists('pg_resetwal.exe' if os.name == 'nt' else 'pg_resetwal'), 'pgbench might be missing') @unittest.skipUnless(pg_version_ge('9.6'), 'requires 9.6+') def test_init_unique_system_id(self): # this function exists in PostgreSQL 9.6+ @@ -254,34 +272,34 @@ def test_psql(self): # check returned values (1 arg) res = node.psql('select 1') - self.assertEqual(res, (0, b'1\n', b'')) + self.assertEqual(rm_carriage_returns(res), (0, b'1\n', b'')) # check returned values (2 args) res = node.psql('postgres', 'select 2') - self.assertEqual(res, (0, b'2\n', b'')) + self.assertEqual(rm_carriage_returns(res), (0, b'2\n', b'')) # check returned values (named) res = node.psql(query='select 3', dbname='postgres') - self.assertEqual(res, (0, b'3\n', b'')) + self.assertEqual(rm_carriage_returns(res), (0, b'3\n', b'')) # check returned values (1 arg) res = node.safe_psql('select 4') - self.assertEqual(res, b'4\n') + self.assertEqual(rm_carriage_returns(res), b'4\n') # check returned values (2 args) res = node.safe_psql('postgres', 'select 5') - self.assertEqual(res, b'5\n') + self.assertEqual(rm_carriage_returns(res), b'5\n') # check returned values (named) res = node.safe_psql(query='select 6', dbname='postgres') - self.assertEqual(res, b'6\n') + self.assertEqual(rm_carriage_returns(res), b'6\n') # check feeding input node.safe_psql('create table horns (w int)') node.safe_psql('copy horns from stdin (format csv)', input=b"1\n2\n3\n\\.\n") _sum = node.safe_psql('select sum(w) from horns') - self.assertEqual(_sum, b'6\n') + self.assertEqual(rm_carriage_returns(_sum), b'6\n') # check psql's default args, fails with self.assertRaises(QueryException): @@ -455,7 +473,7 @@ def test_synchronous_replication(self): master.safe_psql( 'insert into abc select generate_series(1, 1000000)') res = standby1.safe_psql('select count(*) from abc') - self.assertEqual(res, b'1000000\n') + self.assertEqual(rm_carriage_returns(res), b'1000000\n') @unittest.skipUnless(pg_version_ge('10'), 'requires 10+') def test_logical_replication(self): @@ -589,7 +607,7 @@ def test_promotion(self): # make standby becomes writable master replica.safe_psql('insert into abc values (1)') res = replica.safe_psql('select * from abc') - self.assertEqual(res, b'1\n') + self.assertEqual(rm_carriage_returns(res), b'1\n') def test_dump(self): query_create = 'create table test as select generate_series(1, 2) as val' @@ -614,6 +632,7 @@ def test_users(self): with get_new_node().init().start() as node: node.psql('create role test_user login') value = node.safe_psql('select 1', username='test_user') + value = rm_carriage_returns(value) self.assertEqual(value, b'1\n') def test_poll_query_until(self): @@ -728,7 +747,7 @@ def test_logging(self): master.restart() self.assertTrue(master._logger.is_alive()) - @unittest.skipUnless(util_exists('pgbench'), 'might be missing') + @unittest.skipUnless(util_exists('pgbench.exe' if os.name == 'nt' else 'pgbench'), 'pgbench might be missing') def test_pgbench(self): with get_new_node().init().start() as node: @@ -744,6 +763,8 @@ def test_pgbench(self): out, _ = proc.communicate() out = out.decode('utf-8') + proc.stdout.close() + self.assertTrue('tps' in out) def test_pg_config(self): @@ -977,7 +998,9 @@ def test_child_pids(self): def test_child_process_dies(self): # test for FileNotFound exception during child_processes() function - with subprocess.Popen(["sleep", "60"]) as process: + cmd = ["timeout", "60"] if os.name == 'nt' else ["sleep", "60"] + + with subprocess.Popen(cmd, shell=True) as process: # shell=True might be needed on Windows self.assertEqual(process.poll(), None) # collect list of processes currently running children = psutil.Process(os.getpid()).children() diff --git a/tests/test_simple_remote.py b/tests/test_simple_remote.py index 1042f3c4..d51820ba 100755 --- a/tests/test_simple_remote.py +++ b/tests/test_simple_remote.py @@ -52,10 +52,9 @@ from testgres.utils import PgVer from testgres.node import ProcessProxy, ConnectionParams -conn_params = ConnectionParams(host=os.getenv('RDBMS_TESTPOOL1_HOST') or '172.18.0.3', - username='dev', - ssh_key=os.getenv( - 'RDBMS_TESTPOOL_SSHKEY') or '../../container_files/postgres/ssh/id_ed25519') +conn_params = ConnectionParams(host=os.getenv('RDBMS_TESTPOOL1_HOST') or '127.0.0.1', + username=os.getenv('USER'), + ssh_key=os.getenv('RDBMS_TESTPOOL_SSHKEY')) os_ops = RemoteOperations(conn_params) testgres_config.set_os_ops(os_ops=os_ops) From 79a8dc5e17f631dd8dc74b1bfa97c910c4c2c7b1 Mon Sep 17 00:00:00 2001 From: homper Date: Fri, 22 Dec 2023 16:23:13 +0100 Subject: [PATCH 005/216] Remove unnecessary output (#88) --- testgres/connection.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/testgres/connection.py b/testgres/connection.py index aeb040ce..882498a9 100644 --- a/testgres/connection.py +++ b/testgres/connection.py @@ -110,8 +110,7 @@ def execute(self, query, *args): res = [tuple(t) for t in res] return res - except Exception as e: - print("Error executing query: {}".format(e)) + except Exception: return None def close(self): From 5218b113fda81ee93e202d4a9ead0891f4ea475d Mon Sep 17 00:00:00 2001 From: Victoria Shepard <5807469+demonolock@users.noreply.github.com> Date: Tue, 2 Jan 2024 17:54:44 +0100 Subject: [PATCH 006/216] Add function pg_update (#97) --- docker-compose.yml | 4 +- testgres/cache.py | 7 +-- testgres/node.py | 84 +++++++++++++++++++++++++------- testgres/operations/local_ops.py | 2 +- testgres/utils.py | 5 +- tests/test_simple.py | 13 +++++ 6 files changed, 90 insertions(+), 25 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 471ab779..86edf9a4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,2 +1,4 @@ -tests: +version: '3.8' +services: + tests: build: . diff --git a/testgres/cache.py b/testgres/cache.py index 21198e83..f17b54b5 100644 --- a/testgres/cache.py +++ b/testgres/cache.py @@ -22,19 +22,20 @@ from .operations.os_ops import OsOperations -def cached_initdb(data_dir, logfile=None, params=None, os_ops: OsOperations = LocalOperations()): +def cached_initdb(data_dir, logfile=None, params=None, os_ops: OsOperations = LocalOperations(), bin_path=None, cached=True): """ Perform initdb or use cached node files. """ def call_initdb(initdb_dir, log=logfile): try: - _params = [get_bin_path("initdb"), "-D", initdb_dir, "-N"] + initdb_path = os.path.join(bin_path, 'initdb') if bin_path else get_bin_path("initdb") + _params = [initdb_path, "-D", initdb_dir, "-N"] execute_utility(_params + (params or []), log) except ExecUtilException as e: raise_from(InitNodeException("Failed to run initdb"), e) - if params or not testgres_config.cache_initdb: + if params or not testgres_config.cache_initdb or not cached: call_initdb(data_dir, logfile) else: # Fetch cached initdb dir diff --git a/testgres/node.py b/testgres/node.py index 20cf4264..0f1dcf98 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -127,7 +127,7 @@ def __repr__(self): class PostgresNode(object): - def __init__(self, name=None, port=None, base_dir=None, conn_params: ConnectionParams = ConnectionParams()): + def __init__(self, name=None, port=None, base_dir=None, conn_params: ConnectionParams = ConnectionParams(), bin_dir=None, prefix=None): """ PostgresNode constructor. @@ -135,12 +135,15 @@ def __init__(self, name=None, port=None, base_dir=None, conn_params: ConnectionP name: node's application name. port: port to accept connections. base_dir: path to node's data directory. + bin_dir: path to node's binary directory. """ # private - self._pg_version = PgVer(get_pg_version()) + self._pg_version = PgVer(get_pg_version(bin_dir)) self._should_free_port = port is None self._base_dir = base_dir + self._bin_dir = bin_dir + self._prefix = prefix self._logger = None self._master = None @@ -281,7 +284,7 @@ def master(self): @property def base_dir(self): if not self._base_dir: - self._base_dir = self.os_ops.mkdtemp(prefix=TMP_NODE) + self._base_dir = self.os_ops.mkdtemp(prefix=self._prefix or TMP_NODE) # NOTE: it's safe to create a new dir if not self.os_ops.path_exists(self._base_dir): @@ -289,6 +292,12 @@ def base_dir(self): return self._base_dir + @property + def bin_dir(self): + if not self._bin_dir: + self._bin_dir = os.path.dirname(get_bin_path("pg_config")) + return self._bin_dir + @property def logs_dir(self): path = os.path.join(self.base_dir, LOGS_DIR) @@ -441,7 +450,7 @@ def _collect_special_files(self): return result - def init(self, initdb_params=None, **kwargs): + def init(self, initdb_params=None, cached=True, **kwargs): """ Perform initdb for this node. @@ -460,7 +469,9 @@ def init(self, initdb_params=None, **kwargs): data_dir=self.data_dir, logfile=self.utils_log_file, os_ops=self.os_ops, - params=initdb_params) + params=initdb_params, + bin_path=self.bin_dir, + cached=False) # initialize default config files self.default_conf(**kwargs) @@ -619,7 +630,7 @@ def status(self): try: _params = [ - get_bin_path("pg_ctl"), + self._get_bin_path('pg_ctl'), "-D", self.data_dir, "status" ] # yapf: disable @@ -645,7 +656,7 @@ def get_control_data(self): """ # this one is tricky (blame PG 9.4) - _params = [get_bin_path("pg_controldata")] + _params = [self._get_bin_path("pg_controldata")] _params += ["-D"] if self._pg_version >= PgVer('9.5') else [] _params += [self.data_dir] @@ -708,7 +719,7 @@ def start(self, params=[], wait=True): return self _params = [ - get_bin_path("pg_ctl"), + self._get_bin_path("pg_ctl"), "-D", self.data_dir, "-l", self.pg_log_file, "-w" if wait else '-W', # --wait or --no-wait @@ -742,7 +753,7 @@ def stop(self, params=[], wait=True): return self _params = [ - get_bin_path("pg_ctl"), + self._get_bin_path("pg_ctl"), "-D", self.data_dir, "-w" if wait else '-W', # --wait or --no-wait "stop" @@ -782,7 +793,7 @@ def restart(self, params=[]): """ _params = [ - get_bin_path("pg_ctl"), + self._get_bin_path("pg_ctl"), "-D", self.data_dir, "-l", self.pg_log_file, "-w", # wait @@ -814,7 +825,7 @@ def reload(self, params=[]): """ _params = [ - get_bin_path("pg_ctl"), + self._get_bin_path("pg_ctl"), "-D", self.data_dir, "reload" ] + params # yapf: disable @@ -835,7 +846,7 @@ def promote(self, dbname=None, username=None): """ _params = [ - get_bin_path("pg_ctl"), + self._get_bin_path("pg_ctl"), "-D", self.data_dir, "-w", # wait "promote" @@ -871,7 +882,7 @@ def pg_ctl(self, params): """ _params = [ - get_bin_path("pg_ctl"), + self._get_bin_path("pg_ctl"), "-D", self.data_dir, "-w" # wait ] + params # yapf: disable @@ -945,7 +956,7 @@ def psql(self, username = username or default_username() psql_params = [ - get_bin_path("psql"), + self._get_bin_path("psql"), "-p", str(self.port), "-h", self.host, "-U", username, @@ -1066,7 +1077,7 @@ def tmpfile(): filename = filename or tmpfile() _params = [ - get_bin_path("pg_dump"), + self._get_bin_path("pg_dump"), "-p", str(self.port), "-h", self.host, "-f", filename, @@ -1094,7 +1105,7 @@ def restore(self, filename, dbname=None, username=None): username = username or default_username() _params = [ - get_bin_path("pg_restore"), + self._get_bin_path("pg_restore"), "-p", str(self.port), "-h", self.host, "-U", username, @@ -1364,7 +1375,7 @@ def pgbench(self, username = username or default_username() _params = [ - get_bin_path("pgbench"), + self._get_bin_path("pgbench"), "-p", str(self.port), "-h", self.host, "-U", username, @@ -1416,7 +1427,7 @@ def pgbench_run(self, dbname=None, username=None, options=[], **kwargs): username = username or default_username() _params = [ - get_bin_path("pgbench"), + self._get_bin_path("pgbench"), "-p", str(self.port), "-h", self.host, "-U", username, @@ -1587,6 +1598,43 @@ def set_auto_conf(self, options, config='postgresql.auto.conf', rm_options={}): self.os_ops.write(path, auto_conf, truncate=True) + def upgrade_from(self, old_node): + """ + Upgrade this node from an old node using pg_upgrade. + + Args: + old_node: An instance of PostgresNode representing the old node. + """ + if not os.path.exists(old_node.data_dir): + raise Exception("Old node must be initialized") + + if not os.path.exists(self.data_dir): + self.init() + + pg_upgrade_binary = self._get_bin_path("pg_upgrade") + + if not os.path.exists(pg_upgrade_binary): + raise Exception("pg_upgrade does not exist in the new node's binary path") + + upgrade_command = [ + pg_upgrade_binary, + "--old-bindir", old_node.bin_dir, + "--new-bindir", self.bin_dir, + "--old-datadir", old_node.data_dir, + "--new-datadir", self.data_dir, + "--old-port", str(old_node.port), + "--new-port", str(self.port), + ] + + return self.os_ops.exec_command(upgrade_command) + + def _get_bin_path(self, filename): + if self.bin_dir: + bin_path = os.path.join(self.bin_dir, filename) + else: + bin_path = get_bin_path(filename) + return bin_path + class NodeApp: diff --git a/testgres/operations/local_ops.py b/testgres/operations/local_ops.py index 93ebf012..ef360d3b 100644 --- a/testgres/operations/local_ops.py +++ b/testgres/operations/local_ops.py @@ -44,7 +44,7 @@ def __init__(self, conn_params=None): def _raise_exec_exception(message, command, exit_code, output): """Raise an ExecUtilException.""" raise ExecUtilException(message=message.format(output), - command=command, + command=' '.join(command) if isinstance(command, list) else command, exit_code=exit_code, out=output) diff --git a/testgres/utils.py b/testgres/utils.py index b21fc2c8..d84bb2b5 100644 --- a/testgres/utils.py +++ b/testgres/utils.py @@ -172,13 +172,14 @@ def cache_pg_config_data(cmd): return cache_pg_config_data("pg_config") -def get_pg_version(): +def get_pg_version(bin_dir=None): """ Return PostgreSQL version provided by postmaster. """ # get raw version (e.g. postgres (PostgreSQL) 9.5.7) - _params = [get_bin_path('postgres'), '--version'] + postgres_path = os.path.join(bin_dir, 'postgres') if bin_dir else get_bin_path('postgres') + _params = [postgres_path, '--version'] raw_ver = tconf.os_ops.exec_command(_params, encoding='utf-8') # Remove "(Homebrew)" if present diff --git a/tests/test_simple.py b/tests/test_simple.py index 9d31d4d9..a013f478 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -1010,6 +1010,19 @@ def test_child_process_dies(self): # try to handle children list -- missing processes will have ptype "ProcessType.Unknown" [ProcessProxy(p) for p in children] + def test_upgrade_node(self): + old_bin_dir = os.path.dirname(get_bin_path("pg_config")) + new_bin_dir = os.path.dirname(get_bin_path("pg_config")) + node_old = get_new_node(prefix='node_old', bin_dir=old_bin_dir) + node_old.init() + node_old.start() + node_old.stop() + node_new = get_new_node(prefix='node_new', bin_dir=new_bin_dir) + node_new.init(cached=False) + res = node_new.upgrade_from(old_node=node_old) + node_new.start() + self.assertTrue(b'Upgrade Complete' in res) + if __name__ == '__main__': if os.environ.get('ALT_CONFIG'): From a65da28b1717efeea4c01fd6df197c056a6c1846 Mon Sep 17 00:00:00 2001 From: Victoria Shepard <5807469+demonolock@users.noreply.github.com> Date: Thu, 18 Jan 2024 13:56:48 +0100 Subject: [PATCH 007/216] Add pg_probackup plugin (#92) --- setup.py | 2 +- testgres/plugins/__init__.py | 9 + testgres/plugins/pg_probackup2/README.md | 65 ++ testgres/plugins/pg_probackup2/__init__.py | 0 .../pg_probackup2/pg_probackup2/__init__.py | 0 .../pg_probackup2/pg_probackup2/app.py | 762 ++++++++++++++++++ .../pg_probackup2/pg_probackup2/gdb.py | 346 ++++++++ .../pg_probackup2/init_helpers.py | 207 +++++ .../pg_probackup2/storage/__init__.py | 0 .../pg_probackup2/storage/fs_backup.py | 101 +++ .../pg_probackup2/storage/s3_backup.py | 134 +++ .../pg_probackup2/tests/basic_test.py | 79 ++ testgres/plugins/pg_probackup2/setup.py | 18 + 13 files changed, 1722 insertions(+), 1 deletion(-) create mode 100644 testgres/plugins/__init__.py create mode 100644 testgres/plugins/pg_probackup2/README.md create mode 100644 testgres/plugins/pg_probackup2/__init__.py create mode 100644 testgres/plugins/pg_probackup2/pg_probackup2/__init__.py create mode 100644 testgres/plugins/pg_probackup2/pg_probackup2/app.py create mode 100644 testgres/plugins/pg_probackup2/pg_probackup2/gdb.py create mode 100644 testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py create mode 100644 testgres/plugins/pg_probackup2/pg_probackup2/storage/__init__.py create mode 100644 testgres/plugins/pg_probackup2/pg_probackup2/storage/fs_backup.py create mode 100644 testgres/plugins/pg_probackup2/pg_probackup2/storage/s3_backup.py create mode 100644 testgres/plugins/pg_probackup2/pg_probackup2/tests/basic_test.py create mode 100644 testgres/plugins/pg_probackup2/setup.py diff --git a/setup.py b/setup.py index e0287659..b006c8bf 100755 --- a/setup.py +++ b/setup.py @@ -27,7 +27,7 @@ readme = f.read() setup( - version='1.9.3', + version='1.10.0', name='testgres', packages=['testgres', 'testgres.operations', 'testgres.helpers'], description='Testing utility for PostgreSQL and its extensions', diff --git a/testgres/plugins/__init__.py b/testgres/plugins/__init__.py new file mode 100644 index 00000000..e60331f0 --- /dev/null +++ b/testgres/plugins/__init__.py @@ -0,0 +1,9 @@ +from pg_probackup2.gdb import GDBobj +from pg_probackup2.app import ProbackupApp, ProbackupException +from pg_probackup2.init_helpers import init_params +from pg_probackup2.storage.fs_backup import FSTestBackupDir +from pg_probackup2.storage.s3_backup import S3TestBackupDir + +__all__ = [ + "ProbackupApp", "ProbackupException", "init_params", "FSTestBackupDir", "S3TestBackupDir", "GDBobj" +] diff --git a/testgres/plugins/pg_probackup2/README.md b/testgres/plugins/pg_probackup2/README.md new file mode 100644 index 00000000..b62bf24b --- /dev/null +++ b/testgres/plugins/pg_probackup2/README.md @@ -0,0 +1,65 @@ +# testgres - pg_probackup2 + +Ccontrol and testing utility for [pg_probackup2](https://github.com/postgrespro/pg_probackup). Python 3.5+ is supported. + + +## Installation + +To install `testgres`, run: + +``` +pip install testgres-pg_probackup +``` + +We encourage you to use `virtualenv` for your testing environment. +The package requires testgres~=1.9.3. + +## Usage + +### Environment + +> Note: by default testgres runs `initdb`, `pg_ctl`, `psql` provided by `PATH`. + +There are several ways to specify a custom postgres installation: + +* export `PG_CONFIG` environment variable pointing to the `pg_config` executable; +* export `PG_BIN` environment variable pointing to the directory with executable files. + +Example: + +```bash +export PG_BIN=$HOME/pg/bin +python my_tests.py +``` + + +### Examples + +Here is an example of what you can do with `testgres-pg_probackup2`: + +```python +# You can see full script here plugins/pg_probackup2/pg_probackup2/tests/basic_test.py +def test_full_backup(self): + # Setting up a simple test node + node = self.pg_node.make_simple('node', pg_options={"fsync": "off", "synchronous_commit": "off"}) + + # Initialize and configure Probackup + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) + + # Start the node and initialize pgbench + node.slow_start() + node.pgbench_init(scale=100, no_vacuum=True) + + # Perform backup and validation + backup_id = self.pb.backup_node('node', node) + out = self.pb.validate('node', backup_id) + + # Check if the backup is valid + self.assertIn(f"INFO: Backup {backup_id} is valid", out) +``` + +## Authors + +[Postgres Professional](https://postgrespro.ru/about) diff --git a/testgres/plugins/pg_probackup2/__init__.py b/testgres/plugins/pg_probackup2/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/__init__.py b/testgres/plugins/pg_probackup2/pg_probackup2/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/app.py b/testgres/plugins/pg_probackup2/pg_probackup2/app.py new file mode 100644 index 00000000..2c31de51 --- /dev/null +++ b/testgres/plugins/pg_probackup2/pg_probackup2/app.py @@ -0,0 +1,762 @@ +import contextlib +import importlib +import json +import os +import re +import subprocess +import sys +import threading +import time +import unittest + +import testgres + +from .storage.fs_backup import TestBackupDir, FSTestBackupDir +from .gdb import GDBobj +from .init_helpers import init_params + +warning = """ +Wrong splint in show_pb +Original Header:f +{header} +Original Body: +{body} +Splitted Header +{header_split} +Splitted Body +{body_split} +""" + + +class ProbackupException(Exception): + def __init__(self, message, cmd): + self.message = message + self.cmd = cmd + + def __str__(self): + return '\n ERROR: {0}\n CMD: {1}'.format(repr(self.message), self.cmd) + + +class ProbackupApp: + + def __init__(self, test_class: unittest.TestCase, + pg_node, pb_log_path, test_env, auto_compress_alg, backup_dir): + self.test_class = test_class + self.pg_node = pg_node + self.pb_log_path = pb_log_path + self.test_env = test_env + self.auto_compress_alg = auto_compress_alg + self.backup_dir = backup_dir + self.probackup_path = init_params.probackup_path + self.probackup_old_path = init_params.probackup_old_path + self.remote = init_params.remote + self.verbose = init_params.verbose + self.archive_compress = init_params.archive_compress + self.test_class.output = None + + def run(self, command, gdb=False, old_binary=False, return_id=True, env=None, + skip_log_directory=False, expect_error=False, use_backup_dir=True): + """ + Run pg_probackup + backup_dir: target directory for making backup + command: commandline options + expect_error: option for ignoring errors and getting error message as a result of running the function + gdb: when True it returns GDBObj(), when tuple('suspend', port) it runs probackup + in suspended gdb mode with attachable gdb port, for local debugging + """ + if isinstance(use_backup_dir, TestBackupDir): + command = [command[0], *use_backup_dir.pb_args, *command[1:]] + elif use_backup_dir: + command = [command[0], *self.backup_dir.pb_args, *command[1:]] + + if not self.probackup_old_path and old_binary: + print('PGPROBACKUPBIN_OLD is not set') + exit(1) + + if old_binary: + binary_path = self.probackup_old_path + else: + binary_path = self.probackup_path + + if not env: + env = self.test_env + + strcommand = ' '.join(str(p) for p in command) + if '--log-level-file' in strcommand and \ + '--log-directory' not in strcommand and \ + not skip_log_directory: + command += ['--log-directory=' + self.pb_log_path] + strcommand += ' ' + command[-1] + + if 'pglz' in strcommand and \ + ' -j' not in strcommand and '--thread' not in strcommand: + command += ['-j', '1'] + strcommand += ' -j 1' + + self.test_class.cmd = binary_path + ' ' + strcommand + if self.verbose: + print(self.test_class.cmd) + + cmdline = [binary_path, *command] + if gdb is True: + # general test flow for using GDBObj + return GDBobj(cmdline, self.test_class) + + try: + result = None + if type(gdb) is tuple and gdb[0] == 'suspend': + # special test flow for manually debug probackup + gdb_port = gdb[1] + cmdline = ['gdbserver'] + ['localhost:' + str(gdb_port)] + cmdline + print("pg_probackup gdb suspended, waiting gdb connection on localhost:{0}".format(gdb_port)) + + self.test_class.output = subprocess.check_output( + cmdline, + stderr=subprocess.STDOUT, + env=env + ).decode('utf-8', errors='replace') + if command[0] == 'backup' and return_id: + # return backup ID + for line in self.test_class.output.splitlines(): + if 'INFO: Backup' and 'completed' in line: + result = line.split()[2] + else: + result = self.test_class.output + if expect_error is True: + assert False, f"Exception was expected, but run finished successful with result: `{result}`\n" \ + f"CMD: {self.test_class.cmd}" + elif expect_error: + assert False, f"Exception was expected {expect_error}, but run finished successful with result: `{result}`\n" \ + f"CMD: {self.test_class.cmd}" + return result + except subprocess.CalledProcessError as e: + self.test_class.output = e.output.decode('utf-8').replace("\r", "") + if expect_error: + return self.test_class.output + else: + raise ProbackupException(self.test_class.output, self.test_class.cmd) + + def init(self, options=None, old_binary=False, skip_log_directory=False, expect_error=False, use_backup_dir=True): + if options is None: + options = [] + return self.run(['init'] + options, + old_binary=old_binary, + skip_log_directory=skip_log_directory, + expect_error=expect_error, + use_backup_dir=use_backup_dir + ) + + def add_instance(self, instance, node, old_binary=False, options=None, expect_error=False): + if options is None: + options = [] + cmd = [ + 'add-instance', + '--instance={0}'.format(instance), + '-D', node.data_dir + ] + + # don`t forget to kill old_binary after remote ssh release + if self.remote and not old_binary: + options = options + [ + '--remote-proto=ssh', + '--remote-host=localhost'] + + return self.run(cmd + options, old_binary=old_binary, expect_error=expect_error) + + def set_config(self, instance, old_binary=False, options=None, expect_error=False): + if options is None: + options = [] + cmd = [ + 'set-config', + '--instance={0}'.format(instance), + ] + + return self.run(cmd + options, old_binary=old_binary, expect_error=expect_error) + + def set_backup(self, instance, backup_id=False, + old_binary=False, options=None, expect_error=False): + if options is None: + options = [] + cmd = [ + 'set-backup', + ] + + if instance: + cmd = cmd + ['--instance={0}'.format(instance)] + + if backup_id: + cmd = cmd + ['-i', backup_id] + + return self.run(cmd + options, old_binary=old_binary, expect_error=expect_error) + + def del_instance(self, instance, old_binary=False, expect_error=False): + + return self.run([ + 'del-instance', + '--instance={0}'.format(instance), + ], + old_binary=old_binary, + expect_error=expect_error + ) + + def backup_node( + self, instance, node, data_dir=False, + backup_type='full', datname=False, options=None, + gdb=False, + old_binary=False, return_id=True, no_remote=False, + env=None, + expect_error=False, + sync=False + ): + if options is None: + options = [] + if not node and not data_dir: + print('You must provide ether node or data_dir for backup') + exit(1) + + if not datname: + datname = 'postgres' + + cmd_list = [ + 'backup', + '--instance={0}'.format(instance), + # "-D", pgdata, + '-p', '%i' % node.port, + '-d', datname + ] + + if data_dir: + cmd_list += ['-D', self._node_dir(data_dir)] + + # don`t forget to kill old_binary after remote ssh release + if self.remote and not old_binary and not no_remote: + options = options + [ + '--remote-proto=ssh', + '--remote-host=localhost'] + + if self.auto_compress_alg and '--compress' in options and \ + self.archive_compress and self.archive_compress != 'zlib': + options = [o if o != '--compress' else f'--compress-algorithm={self.archive_compress}' + for o in options] + + if backup_type: + cmd_list += ['-b', backup_type] + + if not (old_binary or sync): + cmd_list += ['--no-sync'] + + return self.run(cmd_list + options, gdb, old_binary, return_id, env=env, + expect_error=expect_error) + + def backup_replica_node(self, instance, node, data_dir=False, *, + master, backup_type='full', datname=False, + options=None, env=None): + """ + Try to reliably run backup on replica by switching wal at master + at the moment pg_probackup is waiting for archived wal segment + """ + if options is None: + options = [] + assert '--stream' not in options or backup_type == 'page', \ + "backup_replica_node should be used with one of archive-mode or " \ + "page-stream mode" + + options = options.copy() + if not any('--log-level-file' in x for x in options): + options.append('--log-level-file=INFO') + + gdb = self.backup_node( + instance, node, data_dir, + backup_type=backup_type, + datname=datname, + options=options, + env=env, + gdb=True) + gdb.set_breakpoint('wait_wal_lsn') + # we need to break on wait_wal_lsn in pg_stop_backup + gdb.run_until_break() + if backup_type == 'page': + self.switch_wal_segment(master) + if '--stream' not in options: + gdb.continue_execution_until_break() + self.switch_wal_segment(master) + gdb.continue_execution_until_exit() + + output = self.read_pb_log() + self.unlink_pg_log() + parsed_output = re.compile(r'Backup \S+ completed').search(output) + assert parsed_output, f"Expected: `Backup 'backup_id' completed`, but found `{output}`" + backup_id = parsed_output[0].split(' ')[1] + return (backup_id, output) + + def checkdb_node( + self, use_backup_dir=False, instance=False, data_dir=False, + options=None, gdb=False, old_binary=False, + skip_log_directory=False, + expect_error=False + ): + if options is None: + options = [] + cmd_list = ["checkdb"] + + if instance: + cmd_list += ["--instance={0}".format(instance)] + + if data_dir: + cmd_list += ["-D", self._node_dir(data_dir)] + + return self.run(cmd_list + options, gdb, old_binary, + skip_log_directory=skip_log_directory, expect_error=expect_error, + use_backup_dir=use_backup_dir) + + def merge_backup( + self, instance, backup_id, + gdb=False, old_binary=False, options=None, expect_error=False): + if options is None: + options = [] + cmd_list = [ + 'merge', + '--instance={0}'.format(instance), + '-i', backup_id + ] + + return self.run(cmd_list + options, gdb, old_binary, expect_error=expect_error) + + def restore_node( + self, instance, node=None, restore_dir=None, + backup_id=None, old_binary=False, options=None, + gdb=False, + expect_error=False, + sync=False + ): + if options is None: + options = [] + if node: + if isinstance(node, str): + data_dir = node + else: + data_dir = node.data_dir + elif restore_dir: + data_dir = self._node_dir(restore_dir) + else: + raise ValueError("You must provide ether node or base_dir for backup") + + cmd_list = [ + 'restore', + '-D', data_dir, + '--instance={0}'.format(instance) + ] + + # don`t forget to kill old_binary after remote ssh release + if self.remote and not old_binary: + options = options + [ + '--remote-proto=ssh', + '--remote-host=localhost'] + + if backup_id: + cmd_list += ['-i', backup_id] + + if not (old_binary or sync): + cmd_list += ['--no-sync'] + + return self.run(cmd_list + options, gdb=gdb, old_binary=old_binary, expect_error=expect_error) + + def catchup_node( + self, + backup_mode, source_pgdata, destination_node, + options=None, + remote_host='localhost', + expect_error=False, + gdb=False + ): + + if options is None: + options = [] + cmd_list = [ + 'catchup', + '--backup-mode={0}'.format(backup_mode), + '--source-pgdata={0}'.format(source_pgdata), + '--destination-pgdata={0}'.format(destination_node.data_dir) + ] + if self.remote: + cmd_list += ['--remote-proto=ssh', '--remote-host=%s' % remote_host] + if self.verbose: + cmd_list += [ + '--log-level-file=VERBOSE', + '--log-directory={0}'.format(destination_node.logs_dir) + ] + + return self.run(cmd_list + options, gdb=gdb, expect_error=expect_error, use_backup_dir=False) + + def show( + self, instance=None, backup_id=None, + options=None, as_text=False, as_json=True, old_binary=False, + env=None, + expect_error=False, + gdb=False + ): + + if options is None: + options = [] + backup_list = [] + specific_record = {} + cmd_list = [ + 'show', + ] + if instance: + cmd_list += ['--instance={0}'.format(instance)] + + if backup_id: + cmd_list += ['-i', backup_id] + + # AHTUNG, WARNING will break json parsing + if as_json: + cmd_list += ['--format=json', '--log-level-console=error'] + + if as_text: + # You should print it when calling as_text=true + return self.run(cmd_list + options, old_binary=old_binary, env=env, + expect_error=expect_error, gdb=gdb) + + # get show result as list of lines + if as_json: + text_json = str(self.run(cmd_list + options, old_binary=old_binary, env=env, + expect_error=expect_error, gdb=gdb)) + try: + if expect_error: + return text_json + data = json.loads(text_json) + except ValueError: + assert False, f"Couldn't parse {text_json} as json. " \ + f"Check that you don't have additional messages inside the log or use 'as_text=True'" + + for instance_data in data: + # find specific instance if requested + if instance and instance_data['instance'] != instance: + continue + + for backup in reversed(instance_data['backups']): + # find specific backup if requested + if backup_id: + if backup['id'] == backup_id: + return backup + else: + backup_list.append(backup) + + if backup_id is not None: + assert False, "Failed to find backup with ID: {0}".format(backup_id) + + return backup_list + else: + show_splitted = self.run(cmd_list + options, old_binary=old_binary, env=env, + expect_error=expect_error).splitlines() + if instance is not None and backup_id is None: + # cut header(ID, Mode, etc) from show as single string + header = show_splitted[1:2][0] + # cut backup records from show as single list + # with string for every backup record + body = show_splitted[3:] + # inverse list so oldest record come first + body = body[::-1] + # split string in list with string for every header element + header_split = re.split(' +', header) + # Remove empty items + for i in header_split: + if i == '': + header_split.remove(i) + continue + header_split = [ + header_element.rstrip() for header_element in header_split + ] + for backup_record in body: + backup_record = backup_record.rstrip() + # split list with str for every backup record element + backup_record_split = re.split(' +', backup_record) + # Remove empty items + for i in backup_record_split: + if i == '': + backup_record_split.remove(i) + if len(header_split) != len(backup_record_split): + print(warning.format( + header=header, body=body, + header_split=header_split, + body_split=backup_record_split) + ) + exit(1) + new_dict = dict(zip(header_split, backup_record_split)) + backup_list.append(new_dict) + return backup_list + else: + # cut out empty lines and lines started with # + # and other garbage then reconstruct it as dictionary + # print show_splitted + sanitized_show = [item for item in show_splitted if item] + sanitized_show = [ + item for item in sanitized_show if not item.startswith('#') + ] + # print sanitized_show + for line in sanitized_show: + name, var = line.partition(' = ')[::2] + var = var.strip('"') + var = var.strip("'") + specific_record[name.strip()] = var + + if not specific_record: + assert False, "Failed to find backup with ID: {0}".format(backup_id) + + return specific_record + + def show_archive( + self, instance=None, options=None, + as_text=False, as_json=True, old_binary=False, + tli=0, + expect_error=False + ): + if options is None: + options = [] + cmd_list = [ + 'show', + '--archive', + ] + if instance: + cmd_list += ['--instance={0}'.format(instance)] + + # AHTUNG, WARNING will break json parsing + if as_json: + cmd_list += ['--format=json', '--log-level-console=error'] + + if as_text: + # You should print it when calling as_text=true + return self.run(cmd_list + options, old_binary=old_binary, expect_error=expect_error) + + if as_json: + if as_text: + data = self.run(cmd_list + options, old_binary=old_binary, expect_error=expect_error) + else: + data = json.loads(self.run(cmd_list + options, old_binary=old_binary, expect_error=expect_error)) + + if instance: + instance_timelines = None + for instance_name in data: + if instance_name['instance'] == instance: + instance_timelines = instance_name['timelines'] + break + + if tli > 0: + for timeline in instance_timelines: + if timeline['tli'] == tli: + return timeline + + return {} + + if instance_timelines: + return instance_timelines + + return data + else: + show_splitted = self.run(cmd_list + options, old_binary=old_binary, + expect_error=expect_error).splitlines() + print(show_splitted) + exit(1) + + def validate( + self, instance=None, backup_id=None, + options=None, old_binary=False, gdb=False, expect_error=False + ): + if options is None: + options = [] + cmd_list = [ + 'validate', + ] + if instance: + cmd_list += ['--instance={0}'.format(instance)] + if backup_id: + cmd_list += ['-i', backup_id] + + return self.run(cmd_list + options, old_binary=old_binary, gdb=gdb, + expect_error=expect_error) + + def delete( + self, instance, backup_id=None, + options=None, old_binary=False, gdb=False, expect_error=False): + if options is None: + options = [] + cmd_list = [ + 'delete', + ] + + cmd_list += ['--instance={0}'.format(instance)] + if backup_id: + cmd_list += ['-i', backup_id] + + return self.run(cmd_list + options, old_binary=old_binary, gdb=gdb, + expect_error=expect_error) + + def delete_expired( + self, instance, options=None, old_binary=False, expect_error=False): + if options is None: + options = [] + cmd_list = [ + 'delete', + '--instance={0}'.format(instance) + ] + return self.run(cmd_list + options, old_binary=old_binary, expect_error=expect_error) + + def show_config(self, instance, old_binary=False, expect_error=False, gdb=False): + out_dict = {} + cmd_list = [ + 'show-config', + '--instance={0}'.format(instance) + ] + + res = self.run(cmd_list, old_binary=old_binary, expect_error=expect_error, gdb=gdb).splitlines() + for line in res: + if not line.startswith('#'): + name, var = line.partition(' = ')[::2] + out_dict[name] = var + return out_dict + + def run_binary(self, command, asynchronous=False, env=None): + + if not env: + env = self.test_env + + if self.verbose: + print([' '.join(map(str, command))]) + try: + if asynchronous: + return subprocess.Popen( + command, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=env + ) + else: + self.test_class.output = subprocess.check_output( + command, + stderr=subprocess.STDOUT, + env=env + ).decode('utf-8') + return self.test_class.output + except subprocess.CalledProcessError as e: + raise ProbackupException(e.output.decode('utf-8'), command) + + def _node_dir(self, base_dir): + return os.path.join(self.pg_node.test_path, base_dir) + + def set_archiving( + self, instance, node, replica=False, + overwrite=False, compress=True, old_binary=False, + log_level=False, archive_timeout=False, + custom_archive_command=None): + + # parse postgresql.auto.conf + options = {} + if replica: + options['archive_mode'] = 'always' + options['hot_standby'] = 'on' + else: + options['archive_mode'] = 'on' + + if custom_archive_command is None: + archive_command = " ".join([f'"{init_params.probackup_path}"', + 'archive-push', *self.backup_dir.pb_args]) + if os.name == "nt": + archive_command = archive_command.replace("\\", "\\\\") + archive_command += f' --instance={instance}' + + # don`t forget to kill old_binary after remote ssh release + if init_params.remote and not old_binary: + archive_command += ' --remote-proto=ssh --remote-host=localhost' + + if init_params.archive_compress and compress: + archive_command += ' --compress-algorithm=' + init_params.archive_compress + + if overwrite: + archive_command += ' --overwrite' + + archive_command += ' --log-level-console=VERBOSE' + archive_command += ' -j 5' + archive_command += ' --batch-size 10' + archive_command += ' --no-sync' + + if archive_timeout: + archive_command += f' --archive-timeout={archive_timeout}' + + if os.name == 'posix': + archive_command += ' --wal-file-path=%p --wal-file-name=%f' + + elif os.name == 'nt': + archive_command += ' --wal-file-path="%p" --wal-file-name="%f"' + + if log_level: + archive_command += f' --log-level-console={log_level}' + else: # custom_archive_command is not None + archive_command = custom_archive_command + options['archive_command'] = archive_command + + node.set_auto_conf(options) + + def switch_wal_segment(self, node, sleep_seconds=1, and_tx=False): + """ + Execute pg_switch_wal() in given node + + Args: + node: an instance of PostgresNode or NodeConnection class + """ + if isinstance(node, testgres.PostgresNode): + with node.connect('postgres') as con: + if and_tx: + con.execute('select txid_current()') + lsn = con.execute('select pg_switch_wal()')[0][0] + else: + lsn = node.execute('select pg_switch_wal()')[0][0] + + if sleep_seconds > 0: + time.sleep(sleep_seconds) + return lsn + + @contextlib.contextmanager + def switch_wal_after(self, node, seconds, and_tx=True): + tm = threading.Timer(seconds, self.switch_wal_segment, [node, 0, and_tx]) + tm.start() + try: + yield + finally: + tm.cancel() + tm.join() + + def read_pb_log(self): + with open(os.path.join(self.pb_log_path, 'pg_probackup.log')) as fl: + return fl.read() + + def unlink_pg_log(self): + os.unlink(os.path.join(self.pb_log_path, 'pg_probackup.log')) + + def load_backup_class(fs_type): + fs_type = os.environ.get('PROBACKUP_FS_TYPE') + implementation = f"{__package__}.fs_backup.FSTestBackupDir" + if fs_type: + implementation = fs_type + + print("Using ", implementation) + module_name, class_name = implementation.rsplit(sep='.', maxsplit=1) + + module = importlib.import_module(module_name) + + return getattr(module, class_name) + + +# Local or S3 backup +fs_backup_class = FSTestBackupDir +if os.environ.get('PG_PROBACKUP_S3_TEST', os.environ.get('PROBACKUP_S3_TYPE_FULL_TEST')): + root = os.path.realpath(os.path.join(os.path.dirname(__file__), '../..')) + if root not in sys.path: + sys.path.append(root) + from pg_probackup2.storage.s3_backup import S3TestBackupDir + + fs_backup_class = S3TestBackupDir + + def build_backup_dir(self, backup='backup'): + return fs_backup_class(rel_path=self.rel_path, backup=backup) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/gdb.py b/testgres/plugins/pg_probackup2/pg_probackup2/gdb.py new file mode 100644 index 00000000..0b61da65 --- /dev/null +++ b/testgres/plugins/pg_probackup2/pg_probackup2/gdb.py @@ -0,0 +1,346 @@ +import functools +import os +import re +import subprocess +import sys +import unittest +from time import sleep + + +class GdbException(Exception): + def __init__(self, message="False"): + self.message = message + + def __str__(self): + return '\n ERROR: {0}\n'.format(repr(self.message)) + + +class GDBobj: + _gdb_enabled = False + _gdb_ok = False + _gdb_ptrace_ok = False + + def __init__(self, cmd, env, attach=False): + self.verbose = env.verbose + self.output = '' + self._did_quit = False + self.has_breakpoint = False + + # Check gdb flag is set up + if not hasattr(env, "_gdb_decorated") or not env._gdb_decorated: + raise GdbException("Test should be decorated with @needs_gdb") + if not self._gdb_enabled: + raise GdbException("No `PGPROBACKUP_GDB=on` is set.") + if not self._gdb_ok: + if not self._gdb_ptrace_ok: + raise GdbException("set /proc/sys/kernel/yama/ptrace_scope to 0" + " to run GDB tests") + raise GdbException("No gdb usage possible.") + + # Check gdb presense + try: + gdb_version, _ = subprocess.Popen( + ['gdb', '--version'], + stdout=subprocess.PIPE + ).communicate() + except OSError: + raise GdbException("Couldn't find gdb on the path") + + self.base_cmd = [ + 'gdb', + '--interpreter', + 'mi2', + ] + + if attach: + self.cmd = self.base_cmd + ['--pid'] + cmd + else: + self.cmd = self.base_cmd + ['--args'] + cmd + + # Get version + gdb_version_number = re.search( + br"^GNU gdb [^\d]*(\d+)\.(\d)", + gdb_version) + self.major_version = int(gdb_version_number.group(1)) + self.minor_version = int(gdb_version_number.group(2)) + + if self.verbose: + print([' '.join(map(str, self.cmd))]) + + self.proc = subprocess.Popen( + self.cmd, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + bufsize=0, + text=True, + errors='replace', + ) + self.gdb_pid = self.proc.pid + + while True: + line = self.get_line() + + if 'No such process' in line: + raise GdbException(line) + + if not line.startswith('(gdb)'): + pass + else: + break + + def __del__(self): + if not self._did_quit and hasattr(self, "proc"): + try: + self.quit() + except subprocess.TimeoutExpired: + self.kill() + + def get_line(self): + line = self.proc.stdout.readline() + self.output += line + return line + + def kill(self): + self._did_quit = True + self.proc.kill() + self.proc.wait(3) + self.proc.stdin.close() + self.proc.stdout.close() + + def set_breakpoint(self, location): + + result = self._execute('break ' + location) + self.has_breakpoint = True + for line in result: + if line.startswith('~"Breakpoint'): + return + + elif line.startswith('=breakpoint-created'): + return + + elif line.startswith('^error'): # or line.startswith('(gdb)'): + break + + elif line.startswith('&"break'): + pass + + elif line.startswith('&"Function'): + raise GdbException(line) + + elif line.startswith('&"No line'): + raise GdbException(line) + + elif line.startswith('~"Make breakpoint pending on future shared'): + raise GdbException(line) + + raise GdbException( + 'Failed to set breakpoint.\n Output:\n {0}'.format(result) + ) + + def remove_all_breakpoints(self): + if not self.has_breakpoint: + return + + result = self._execute('delete') + self.has_breakpoint = False + for line in result: + + if line.startswith('^done'): + return + + raise GdbException( + 'Failed to remove breakpoints.\n Output:\n {0}'.format(result) + ) + + def run_until_break(self): + result = self._execute('run', False) + for line in result: + if line.startswith('*stopped,reason="breakpoint-hit"'): + return + raise GdbException( + 'Failed to run until breakpoint.\n' + ) + + def continue_execution_until_running(self): + result = self._execute('continue') + + for line in result: + if line.startswith('*running') or line.startswith('^running'): + return + if line.startswith('*stopped,reason="breakpoint-hit"'): + continue + if line.startswith('*stopped,reason="exited-normally"'): + continue + + raise GdbException( + 'Failed to continue execution until running.\n' + ) + + def signal(self, sig): + if 'KILL' in sig: + self.remove_all_breakpoints() + self._execute(f'signal {sig}') + + def continue_execution_until_exit(self): + self.remove_all_breakpoints() + result = self._execute('continue', False) + + for line in result: + if line.startswith('*running'): + continue + if line.startswith('*stopped,reason="breakpoint-hit"'): + continue + if line.startswith('*stopped,reason="exited') or line == '*stopped\n': + self.quit() + return + + raise GdbException( + 'Failed to continue execution until exit.\n' + ) + + def continue_execution_until_error(self): + self.remove_all_breakpoints() + result = self._execute('continue', False) + + for line in result: + if line.startswith('^error'): + return + if line.startswith('*stopped,reason="exited'): + return + if line.startswith( + '*stopped,reason="signal-received",signal-name="SIGABRT"'): + return + + raise GdbException( + 'Failed to continue execution until error.\n') + + def continue_execution_until_break(self, ignore_count=0): + if ignore_count > 0: + result = self._execute( + 'continue ' + str(ignore_count), + False + ) + else: + result = self._execute('continue', False) + + for line in result: + if line.startswith('*stopped,reason="breakpoint-hit"'): + return + if line.startswith('*stopped,reason="exited-normally"'): + break + + raise GdbException( + 'Failed to continue execution until break.\n') + + def show_backtrace(self): + return self._execute("backtrace", running=False) + + def stopped_in_breakpoint(self): + while True: + line = self.get_line() + if self.verbose: + print(line) + if line.startswith('*stopped,reason="breakpoint-hit"'): + return True + + def detach(self): + if not self._did_quit: + self._execute('detach') + + def quit(self): + if not self._did_quit: + self._did_quit = True + self.proc.terminate() + self.proc.wait(3) + self.proc.stdin.close() + self.proc.stdout.close() + + # use for breakpoint, run, continue + def _execute(self, cmd, running=True): + output = [] + self.proc.stdin.flush() + self.proc.stdin.write(cmd + '\n') + self.proc.stdin.flush() + sleep(1) + + # look for command we just send + while True: + line = self.get_line() + if self.verbose: + print(repr(line)) + + if cmd not in line: + continue + else: + break + + while True: + line = self.get_line() + output += [line] + if self.verbose: + print(repr(line)) + if line.startswith('^done') or line.startswith('*stopped'): + break + if line.startswith('^error'): + break + if running and (line.startswith('*running') or line.startswith('^running')): + # if running and line.startswith('*running'): + break + return output + + +def _set_gdb(self): + test_env = os.environ.copy() + self._gdb_enabled = test_env.get('PGPROBACKUP_GDB') == 'ON' + self._gdb_ok = self._gdb_enabled + if not self._gdb_enabled or sys.platform != 'linux': + return + try: + with open('/proc/sys/kernel/yama/ptrace_scope') as f: + ptrace = f.read() + except FileNotFoundError: + self._gdb_ptrace_ok = True + return + self._gdb_ptrace_ok = int(ptrace) == 0 + self._gdb_ok = self._gdb_ok and self._gdb_ptrace_ok + + +def _check_gdb_flag_or_skip_test(): + if not GDBobj._gdb_enabled: + return ("skip", + "Specify PGPROBACKUP_GDB and build without " + "optimizations for run this test" + ) + if GDBobj._gdb_ok: + return None + if not GDBobj._gdb_ptrace_ok: + return ("fail", "set /proc/sys/kernel/yama/ptrace_scope to 0" + " to run GDB tests") + else: + return ("fail", "use of gdb is not possible") + + +def needs_gdb(func): + check = _check_gdb_flag_or_skip_test() + if not check: + @functools.wraps(func) + def ok_wrapped(self): + self._gdb_decorated = True + func(self) + + return ok_wrapped + reason = check[1] + if check[0] == "skip": + return unittest.skip(reason)(func) + elif check[0] == "fail": + @functools.wraps(func) + def fail_wrapper(self): + self.fail(reason) + + return fail_wrapper + else: + raise "Wrong action {0}".format(check) + + +_set_gdb(GDBobj) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py new file mode 100644 index 00000000..7af21eb6 --- /dev/null +++ b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py @@ -0,0 +1,207 @@ +from functools import reduce +import getpass +import os +import re +import shutil +import subprocess +import sys +import testgres + +try: + import lz4.frame # noqa: F401 + + HAVE_LZ4 = True +except ImportError as e: + HAVE_LZ4 = False + LZ4_error = e + +try: + import zstd # noqa: F401 + + HAVE_ZSTD = True +except ImportError as e: + HAVE_ZSTD = False + ZSTD_error = e + +delete_logs = os.getenv('KEEP_LOGS') not in ['1', 'y', 'Y'] + +try: + testgres.configure_testgres( + cache_initdb=False, + cached_initdb_dir=False, + node_cleanup_full=delete_logs) +except Exception as e: + print("Can't configure testgres: {0}".format(e)) + + +class Init(object): + def __init__(self): + if '-v' in sys.argv or '--verbose' in sys.argv: + self.verbose = True + else: + self.verbose = False + + self._pg_config = testgres.get_pg_config() + self.is_enterprise = self._pg_config.get('PGPRO_EDITION', None) == 'enterprise' + self.is_shardman = self._pg_config.get('PGPRO_EDITION', None) == 'shardman' + self.is_pgpro = 'PGPRO_EDITION' in self._pg_config + self.is_nls_enabled = 'enable-nls' in self._pg_config['CONFIGURE'] + self.is_lz4_enabled = '-llz4' in self._pg_config['LIBS'] + version = self._pg_config['VERSION'].rstrip('develalphabetapre') + parts = [*version.split(' ')[1].split('.'), '0', '0'][:3] + parts[0] = re.match(r'\d+', parts[0]).group() + self.pg_config_version = reduce(lambda v, x: v * 100 + int(x), parts, 0) + + test_env = os.environ.copy() + envs_list = [ + 'LANGUAGE', + 'LC_ALL', + 'PGCONNECT_TIMEOUT', + 'PGDATA', + 'PGDATABASE', + 'PGHOSTADDR', + 'PGREQUIRESSL', + 'PGSERVICE', + 'PGSSLMODE', + 'PGUSER', + 'PGPORT', + 'PGHOST' + ] + + for e in envs_list: + test_env.pop(e, None) + + test_env['LC_MESSAGES'] = 'C' + test_env['LC_TIME'] = 'C' + self._test_env = test_env + + # Get the directory from which the script was executed + self.source_path = os.getcwd() + tmp_path = test_env.get('PGPROBACKUP_TMP_DIR') + if tmp_path and os.path.isabs(tmp_path): + self.tmp_path = tmp_path + else: + self.tmp_path = os.path.abspath( + os.path.join(self.source_path, tmp_path or os.path.join('tests', 'tmp_dirs')) + ) + + os.makedirs(self.tmp_path, exist_ok=True) + + self.username = getpass.getuser() + + self.probackup_path = None + if 'PGPROBACKUPBIN' in test_env: + if shutil.which(test_env["PGPROBACKUPBIN"]): + self.probackup_path = test_env["PGPROBACKUPBIN"] + else: + if self.verbose: + print('PGPROBACKUPBIN is not an executable file') + + if not self.probackup_path: + probackup_path_tmp = os.path.join( + testgres.get_pg_config()['BINDIR'], 'pg_probackup') + + if os.path.isfile(probackup_path_tmp): + if not os.access(probackup_path_tmp, os.X_OK): + print('{0} is not an executable file'.format( + probackup_path_tmp)) + else: + self.probackup_path = probackup_path_tmp + + if not self.probackup_path: + probackup_path_tmp = self.source_path + + if os.path.isfile(probackup_path_tmp): + if not os.access(probackup_path_tmp, os.X_OK): + print('{0} is not an executable file'.format( + probackup_path_tmp)) + else: + self.probackup_path = probackup_path_tmp + + if not self.probackup_path: + print('pg_probackup binary is not found') + exit(1) + + if os.name == 'posix': + self.EXTERNAL_DIRECTORY_DELIMITER = ':' + os.environ['PATH'] = os.path.dirname( + self.probackup_path) + ':' + os.environ['PATH'] + + elif os.name == 'nt': + self.EXTERNAL_DIRECTORY_DELIMITER = ';' + os.environ['PATH'] = os.path.dirname( + self.probackup_path) + ';' + os.environ['PATH'] + + self.probackup_old_path = None + if 'PGPROBACKUPBIN_OLD' in test_env: + if (os.path.isfile(test_env['PGPROBACKUPBIN_OLD']) and os.access(test_env['PGPROBACKUPBIN_OLD'], os.X_OK)): + self.probackup_old_path = test_env['PGPROBACKUPBIN_OLD'] + else: + if self.verbose: + print('PGPROBACKUPBIN_OLD is not an executable file') + + self.probackup_version = None + self.old_probackup_version = None + + probackup_version_output = subprocess.check_output( + [self.probackup_path, "--version"], + stderr=subprocess.STDOUT, + ).decode('utf-8') + self.probackup_version = re.search(r"\d+\.\d+\.\d+", + probackup_version_output + ).group(0) + compressions = re.search(r"\(compressions: ([^)]*)\)", + probackup_version_output).group(1) + self.probackup_compressions = {s.strip() for s in compressions.split(',')} + + if self.probackup_old_path: + old_probackup_version_output = subprocess.check_output( + [self.probackup_old_path, "--version"], + stderr=subprocess.STDOUT, + ).decode('utf-8') + self.old_probackup_version = re.search(r"\d+\.\d+\.\d+", + old_probackup_version_output + ).group(0) + + self.remote = test_env.get('PGPROBACKUP_SSH_REMOTE', None) == 'ON' + self.ptrack = test_env.get('PG_PROBACKUP_PTRACK', None) == 'ON' and self.pg_config_version >= 110000 + + self.paranoia = test_env.get('PG_PROBACKUP_PARANOIA', None) == 'ON' + env_compress = test_env.get('ARCHIVE_COMPRESSION', None) + if env_compress: + env_compress = env_compress.lower() + if env_compress in ('on', 'zlib'): + self.compress_suffix = '.gz' + self.archive_compress = 'zlib' + elif env_compress == 'lz4': + if not HAVE_LZ4: + raise LZ4_error + if 'lz4' not in self.probackup_compressions: + raise Exception("pg_probackup is not compiled with lz4 support") + self.compress_suffix = '.lz4' + self.archive_compress = 'lz4' + elif env_compress == 'zstd': + if not HAVE_ZSTD: + raise ZSTD_error + if 'zstd' not in self.probackup_compressions: + raise Exception("pg_probackup is not compiled with zstd support") + self.compress_suffix = '.zst' + self.archive_compress = 'zstd' + else: + self.compress_suffix = '' + self.archive_compress = False + + cfs_compress = test_env.get('PG_PROBACKUP_CFS_COMPRESS', None) + if cfs_compress: + self.cfs_compress = cfs_compress.lower() + else: + self.cfs_compress = self.archive_compress + + os.environ["PGAPPNAME"] = "pg_probackup" + self.delete_logs = delete_logs + + def test_env(self): + return self._test_env.copy() + + +init_params = Init() diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/storage/__init__.py b/testgres/plugins/pg_probackup2/pg_probackup2/storage/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/storage/fs_backup.py b/testgres/plugins/pg_probackup2/pg_probackup2/storage/fs_backup.py new file mode 100644 index 00000000..d076432a --- /dev/null +++ b/testgres/plugins/pg_probackup2/pg_probackup2/storage/fs_backup.py @@ -0,0 +1,101 @@ +""" +Utilities for accessing pg_probackup backup data on file system. +""" +import os +import shutil + +from ..init_helpers import init_params + + +class TestBackupDir: + + def list_instance_backups(self, instance): + raise NotImplementedError() + + def list_files(self, sub_dir, recursive=False): + raise NotImplementedError() + + def list_dirs(self, sub_dir): + raise NotImplementedError() + + def read_file(self, sub_path, *, text=True): + raise NotImplementedError() + + def write_file(self, sub_path, data, *, text=True): + raise NotImplementedError() + + def cleanup(self): + raise NotImplementedError() + + def remove_file(self, sub_path): + raise NotImplementedError() + + def remove_dir(self, sub_path): + raise NotImplementedError() + + def exists(self, sub_path): + raise NotImplementedError() + + +class FSTestBackupDir(TestBackupDir): + is_file_based = True + + """ Backup directory. Usually created by running pg_probackup init -B """ + + def __init__(self, *, rel_path, backup): + self.path = os.path.join(init_params.tmp_path, rel_path, backup) + self.pb_args = ('-B', self.path) + + def list_instance_backups(self, instance): + full_path = os.path.join(self.path, 'backups', instance) + return sorted((x for x in os.listdir(full_path) + if os.path.isfile(os.path.join(full_path, x, 'backup.control')))) + + def list_files(self, sub_dir, recursive=False): + full_path = os.path.join(self.path, sub_dir) + if not recursive: + return [f for f in os.listdir(full_path) + if os.path.isfile(os.path.join(full_path, f))] + files = [] + for rootdir, dirs, files_in_dir in os.walk(full_path): + rootdir = rootdir[len(self.path) + 1:] + files.extend(os.path.join(rootdir, file) for file in files_in_dir) + return files + + def list_dirs(self, sub_dir): + full_path = os.path.join(self.path, sub_dir) + return [f for f in os.listdir(full_path) + if os.path.isdir(os.path.join(full_path, f))] + + def read_file(self, sub_path, *, text=True): + full_path = os.path.join(self.path, sub_path) + with open(full_path, 'r' if text else 'rb') as fin: + return fin.read() + + def write_file(self, sub_path, data, *, text=True): + full_path = os.path.join(self.path, sub_path) + with open(full_path, 'w' if text else 'wb') as fout: + fout.write(data) + + def cleanup(self): + shutil.rmtree(self.path, ignore_errors=True) + + def remove_file(self, sub_path): + os.remove(os.path.join(self.path, sub_path)) + + def remove_dir(self, sub_path): + full_path = os.path.join(self.path, sub_path) + shutil.rmtree(full_path, ignore_errors=True) + + def exists(self, sub_path): + full_path = os.path.join(self.path, sub_path) + return os.path.exists(full_path) + + def __str__(self): + return self.path + + def __repr__(self): + return "FSTestBackupDir" + str(self.path) + + def __fspath__(self): + return self.path diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/storage/s3_backup.py b/testgres/plugins/pg_probackup2/pg_probackup2/storage/s3_backup.py new file mode 100644 index 00000000..c6b764fb --- /dev/null +++ b/testgres/plugins/pg_probackup2/pg_probackup2/storage/s3_backup.py @@ -0,0 +1,134 @@ +import os +import io +import sys + +import minio +from minio import Minio +from minio.deleteobjects import DeleteObject +import urllib3 +from .fs_backup import TestBackupDir + +root = os.path.realpath(os.path.join(os.path.dirname(__file__), '../..')) +if root not in sys.path: + sys.path.append(root) + +# Should fail if either of env vars does not exist +host = os.environ['PG_PROBACKUP_S3_HOST'] +port = os.environ['PG_PROBACKUP_S3_PORT'] +access = os.environ['PG_PROBACKUP_S3_ACCESS_KEY'] +secret = os.environ['PG_PROBACKUP_S3_SECRET_ACCESS_KEY'] +bucket = os.environ['PG_PROBACKUP_S3_BUCKET_NAME'] +path_suffix = os.environ.get("PG_PROBACKUP_TEST_TMP_SUFFIX") +https = os.environ.get("PG_PROBACKUP_S3_HTTPS") + +s3_type = os.environ.get('PG_PROBACKUP_S3_TEST', os.environ.get('PROBACKUP_S3_TYPE_FULL_TEST')) +tmp_path = os.environ.get('PGPROBACKUP_TMP_DIR', default='') + +status_forcelist = [413, # RequestBodyTooLarge + 429, # TooManyRequests + 500, # InternalError + 503, # ServerBusy + ] + + +class S3TestBackupDir(TestBackupDir): + is_file_based = False + + def __init__(self, *, rel_path, backup): + path = "pg_probackup" + if path_suffix: + path += "_" + path_suffix + if tmp_path == '' or os.path.isabs(tmp_path): + self.path = f"{path}{tmp_path}/{rel_path}/{backup}" + else: + self.path = f"{path}/{tmp_path}/{rel_path}/{backup}" + + secure: bool = False + if https in ['ON', 'HTTPS']: + secure = True + self.conn = Minio(host + ":" + port, secure=secure, access_key=access, + secret_key=secret, http_client=urllib3.PoolManager(retries=urllib3.Retry(total=5, + backoff_factor=1, + status_forcelist=status_forcelist))) + if not self.conn.bucket_exists(bucket): + raise Exception(f"Test bucket {bucket} does not exist.") + self.pb_args = ('-B', '/' + self.path, f'--s3={s3_type}') + return + + def list_instance_backups(self, instance): + full_path = os.path.join(self.path, 'backups', instance) + candidates = self.conn.list_objects(bucket, prefix=full_path, recursive=True) + return [os.path.basename(os.path.dirname(x.object_name)) + for x in candidates if x.object_name.endswith('backup.control')] + + def list_files(self, sub_dir, recursive=False): + full_path = os.path.join(self.path, sub_dir) + # Need '/' in the end to find inside the folder + full_path_dir = full_path if full_path[-1] == '/' else full_path + '/' + object_list = self.conn.list_objects(bucket, prefix=full_path_dir, recursive=recursive) + return [obj.object_name.replace(full_path_dir, '', 1) + for obj in object_list + if not obj.is_dir] + + def list_dirs(self, sub_dir): + full_path = os.path.join(self.path, sub_dir) + # Need '/' in the end to find inside the folder + full_path_dir = full_path if full_path[-1] == '/' else full_path + '/' + object_list = self.conn.list_objects(bucket, prefix=full_path_dir, recursive=False) + return [obj.object_name.replace(full_path_dir, '', 1).rstrip('\\/') + for obj in object_list + if obj.is_dir] + + def read_file(self, sub_path, *, text=True): + full_path = os.path.join(self.path, sub_path) + bytes = self.conn.get_object(bucket, full_path).read() + if not text: + return bytes + return bytes.decode('utf-8') + + def write_file(self, sub_path, data, *, text=True): + full_path = os.path.join(self.path, sub_path) + if text: + data = data.encode('utf-8') + self.conn.put_object(bucket, full_path, io.BytesIO(data), length=len(data)) + + def cleanup(self): + self.remove_dir('') + + def remove_file(self, sub_path): + full_path = os.path.join(self.path, sub_path) + self.conn.remove_object(bucket, full_path) + + def remove_dir(self, sub_path): + if sub_path: + full_path = os.path.join(self.path, sub_path) + else: + full_path = self.path + objs = self.conn.list_objects(bucket, prefix=full_path, recursive=True, + include_version=True) + delobjs = (DeleteObject(o.object_name, o.version_id) for o in objs) + errs = list(self.conn.remove_objects(bucket, delobjs)) + if errs: + strerrs = "; ".join(str(err) for err in errs) + raise Exception("There were errors: {0}".format(strerrs)) + + def exists(self, sub_path): + full_path = os.path.join(self.path, sub_path) + try: + self.conn.stat_object(bucket, full_path) + return True + except minio.error.S3Error as s3err: + if s3err.code == 'NoSuchKey': + return False + raise s3err + except Exception as err: + raise err + + def __str__(self): + return '/' + self.path + + def __repr__(self): + return "S3TestBackupDir" + str(self.path) + + def __fspath__(self): + return self.path diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/tests/basic_test.py b/testgres/plugins/pg_probackup2/pg_probackup2/tests/basic_test.py new file mode 100644 index 00000000..f5a82d38 --- /dev/null +++ b/testgres/plugins/pg_probackup2/pg_probackup2/tests/basic_test.py @@ -0,0 +1,79 @@ +import os +import shutil +import unittest +import testgres +from pg_probackup2.app import ProbackupApp +from pg_probackup2.init_helpers import Init, init_params +from pg_probackup2.app import build_backup_dir + + +class TestUtils: + @staticmethod + def get_module_and_function_name(test_id): + try: + module_name = test_id.split('.')[-2] + fname = test_id.split('.')[-1] + except IndexError: + print(f"Couldn't get module name and function name from test_id: `{test_id}`") + module_name, fname = test_id.split('(')[1].split('.')[1], test_id.split('(')[0] + return module_name, fname + + +class ProbackupTest(unittest.TestCase): + def setUp(self): + self.setup_test_environment() + self.setup_test_paths() + self.setup_backup_dir() + self.setup_probackup() + + def setup_test_environment(self): + self.output = None + self.cmd = None + self.nodes_to_cleanup = [] + self.module_name, self.fname = TestUtils.get_module_and_function_name(self.id()) + self.test_env = Init().test_env() + + def setup_test_paths(self): + self.rel_path = os.path.join(self.module_name, self.fname) + self.test_path = os.path.join(init_params.tmp_path, self.rel_path) + os.makedirs(self.test_path) + self.pb_log_path = os.path.join(self.test_path, "pb_log") + + def setup_backup_dir(self): + self.backup_dir = build_backup_dir(self, 'backup') + self.backup_dir.cleanup() + + def setup_probackup(self): + self.pg_node = testgres.NodeApp(self.test_path, self.nodes_to_cleanup) + self.pb = ProbackupApp(self, self.pg_node, self.pb_log_path, self.test_env, + auto_compress_alg='zlib', backup_dir=self.backup_dir) + + def tearDown(self): + if os.path.exists(self.test_path): + shutil.rmtree(self.test_path) + + +class BasicTest(ProbackupTest): + def test_full_backup(self): + # Setting up a simple test node + node = self.pg_node.make_simple('node', pg_options={"fsync": "off", "synchronous_commit": "off"}) + + # Initialize and configure Probackup + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) + + # Start the node and initialize pgbench + node.slow_start() + node.pgbench_init(scale=100, no_vacuum=True) + + # Perform backup and validation + backup_id = self.pb.backup_node('node', node) + out = self.pb.validate('node', backup_id) + + # Check if the backup is valid + self.assertIn(f"INFO: Backup {backup_id} is valid", out) + + +if __name__ == "__main__": + unittest.main() diff --git a/testgres/plugins/pg_probackup2/setup.py b/testgres/plugins/pg_probackup2/setup.py new file mode 100644 index 00000000..371eb078 --- /dev/null +++ b/testgres/plugins/pg_probackup2/setup.py @@ -0,0 +1,18 @@ +try: + from setuptools import setup +except ImportError: + from distutils.core import setup + +setup( + version='0.0.1', + name='testgres_pg_probackup2', + packages=['pg_probackup2', 'pg_probackup2.storage'], + description='Plugin for testgres that manages pg_probackup2', + url='https://github.com/postgrespro/testgres', + long_description_content_type='text/markdown', + license='PostgreSQL', + author='Postgres Professional', + author_email='testgres@postgrespro.ru', + keywords=['pg_probackup', 'testing', 'testgres'], + install_requires=['testgres>=1.9.2'] +) From 0edc937b08c7b562c8b16626d445b0b2a5d78f1a Mon Sep 17 00:00:00 2001 From: Victoria Shepard <5807469+demonolock@users.noreply.github.com> Date: Thu, 18 Jan 2024 15:44:47 +0100 Subject: [PATCH 008/216] Fix get_pg_version for linux mint (#101) issue #100 --- testgres/utils.py | 15 +++++++++------ tests/test_simple.py | 12 +++++++++++- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/testgres/utils.py b/testgres/utils.py index d84bb2b5..745a2555 100644 --- a/testgres/utils.py +++ b/testgres/utils.py @@ -177,20 +177,23 @@ def get_pg_version(bin_dir=None): Return PostgreSQL version provided by postmaster. """ - # get raw version (e.g. postgres (PostgreSQL) 9.5.7) + # Get raw version (e.g., postgres (PostgreSQL) 9.5.7) postgres_path = os.path.join(bin_dir, 'postgres') if bin_dir else get_bin_path('postgres') _params = [postgres_path, '--version'] raw_ver = tconf.os_ops.exec_command(_params, encoding='utf-8') - # Remove "(Homebrew)" if present - raw_ver = raw_ver.replace('(Homebrew)', '').strip() + return parse_pg_version(raw_ver) - # cook version of PostgreSQL - version = raw_ver.strip().split(' ')[-1] \ + +def parse_pg_version(version_out): + # Generalize removal of system-specific suffixes (anything in parentheses) + raw_ver = re.sub(r'\([^)]*\)', '', version_out).strip() + + # Cook version of PostgreSQL + version = raw_ver.split(' ')[-1] \ .partition('devel')[0] \ .partition('beta')[0] \ .partition('rc')[0] - return version diff --git a/tests/test_simple.py b/tests/test_simple.py index a013f478..8cb0d94e 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -48,7 +48,7 @@ # NOTE: those are ugly imports from testgres import bound_ports -from testgres.utils import PgVer +from testgres.utils import PgVer, parse_pg_version from testgres.node import ProcessProxy @@ -1023,6 +1023,16 @@ def test_upgrade_node(self): node_new.start() self.assertTrue(b'Upgrade Complete' in res) + def test_parse_pg_version(self): + # Linux Mint + assert parse_pg_version("postgres (PostgreSQL) 15.5 (Ubuntu 15.5-1.pgdg22.04+1)") == "15.5" + # Linux Ubuntu + assert parse_pg_version("postgres (PostgreSQL) 12.17") == "12.17" + # Windows + assert parse_pg_version("postgres (PostgreSQL) 11.4") == "11.4" + # Macos + assert parse_pg_version("postgres (PostgreSQL) 14.9 (Homebrew)") == "14.9" + if __name__ == '__main__': if os.environ.get('ALT_CONFIG'): From 992f0c8415d92625ffa6d8a1b2186c79791ed91e Mon Sep 17 00:00:00 2001 From: vshepard Date: Tue, 30 Jan 2024 07:23:05 +0100 Subject: [PATCH 009/216] Move file control for tests --- .../pg_probackup2/pg_probackup2/app.py | 14 +- .../pg_probackup2/storage/s3_backup.py | 134 ------------------ 2 files changed, 3 insertions(+), 145 deletions(-) delete mode 100644 testgres/plugins/pg_probackup2/pg_probackup2/storage/s3_backup.py diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/app.py b/testgres/plugins/pg_probackup2/pg_probackup2/app.py index 2c31de51..6b176488 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/app.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/app.py @@ -37,6 +37,9 @@ def __str__(self): return '\n ERROR: {0}\n CMD: {1}'.format(repr(self.message), self.cmd) +# Local or S3 backup +fs_backup_class = FSTestBackupDir + class ProbackupApp: def __init__(self, test_class: unittest.TestCase, @@ -747,16 +750,5 @@ def load_backup_class(fs_type): return getattr(module, class_name) - -# Local or S3 backup -fs_backup_class = FSTestBackupDir -if os.environ.get('PG_PROBACKUP_S3_TEST', os.environ.get('PROBACKUP_S3_TYPE_FULL_TEST')): - root = os.path.realpath(os.path.join(os.path.dirname(__file__), '../..')) - if root not in sys.path: - sys.path.append(root) - from pg_probackup2.storage.s3_backup import S3TestBackupDir - - fs_backup_class = S3TestBackupDir - def build_backup_dir(self, backup='backup'): return fs_backup_class(rel_path=self.rel_path, backup=backup) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/storage/s3_backup.py b/testgres/plugins/pg_probackup2/pg_probackup2/storage/s3_backup.py deleted file mode 100644 index c6b764fb..00000000 --- a/testgres/plugins/pg_probackup2/pg_probackup2/storage/s3_backup.py +++ /dev/null @@ -1,134 +0,0 @@ -import os -import io -import sys - -import minio -from minio import Minio -from minio.deleteobjects import DeleteObject -import urllib3 -from .fs_backup import TestBackupDir - -root = os.path.realpath(os.path.join(os.path.dirname(__file__), '../..')) -if root not in sys.path: - sys.path.append(root) - -# Should fail if either of env vars does not exist -host = os.environ['PG_PROBACKUP_S3_HOST'] -port = os.environ['PG_PROBACKUP_S3_PORT'] -access = os.environ['PG_PROBACKUP_S3_ACCESS_KEY'] -secret = os.environ['PG_PROBACKUP_S3_SECRET_ACCESS_KEY'] -bucket = os.environ['PG_PROBACKUP_S3_BUCKET_NAME'] -path_suffix = os.environ.get("PG_PROBACKUP_TEST_TMP_SUFFIX") -https = os.environ.get("PG_PROBACKUP_S3_HTTPS") - -s3_type = os.environ.get('PG_PROBACKUP_S3_TEST', os.environ.get('PROBACKUP_S3_TYPE_FULL_TEST')) -tmp_path = os.environ.get('PGPROBACKUP_TMP_DIR', default='') - -status_forcelist = [413, # RequestBodyTooLarge - 429, # TooManyRequests - 500, # InternalError - 503, # ServerBusy - ] - - -class S3TestBackupDir(TestBackupDir): - is_file_based = False - - def __init__(self, *, rel_path, backup): - path = "pg_probackup" - if path_suffix: - path += "_" + path_suffix - if tmp_path == '' or os.path.isabs(tmp_path): - self.path = f"{path}{tmp_path}/{rel_path}/{backup}" - else: - self.path = f"{path}/{tmp_path}/{rel_path}/{backup}" - - secure: bool = False - if https in ['ON', 'HTTPS']: - secure = True - self.conn = Minio(host + ":" + port, secure=secure, access_key=access, - secret_key=secret, http_client=urllib3.PoolManager(retries=urllib3.Retry(total=5, - backoff_factor=1, - status_forcelist=status_forcelist))) - if not self.conn.bucket_exists(bucket): - raise Exception(f"Test bucket {bucket} does not exist.") - self.pb_args = ('-B', '/' + self.path, f'--s3={s3_type}') - return - - def list_instance_backups(self, instance): - full_path = os.path.join(self.path, 'backups', instance) - candidates = self.conn.list_objects(bucket, prefix=full_path, recursive=True) - return [os.path.basename(os.path.dirname(x.object_name)) - for x in candidates if x.object_name.endswith('backup.control')] - - def list_files(self, sub_dir, recursive=False): - full_path = os.path.join(self.path, sub_dir) - # Need '/' in the end to find inside the folder - full_path_dir = full_path if full_path[-1] == '/' else full_path + '/' - object_list = self.conn.list_objects(bucket, prefix=full_path_dir, recursive=recursive) - return [obj.object_name.replace(full_path_dir, '', 1) - for obj in object_list - if not obj.is_dir] - - def list_dirs(self, sub_dir): - full_path = os.path.join(self.path, sub_dir) - # Need '/' in the end to find inside the folder - full_path_dir = full_path if full_path[-1] == '/' else full_path + '/' - object_list = self.conn.list_objects(bucket, prefix=full_path_dir, recursive=False) - return [obj.object_name.replace(full_path_dir, '', 1).rstrip('\\/') - for obj in object_list - if obj.is_dir] - - def read_file(self, sub_path, *, text=True): - full_path = os.path.join(self.path, sub_path) - bytes = self.conn.get_object(bucket, full_path).read() - if not text: - return bytes - return bytes.decode('utf-8') - - def write_file(self, sub_path, data, *, text=True): - full_path = os.path.join(self.path, sub_path) - if text: - data = data.encode('utf-8') - self.conn.put_object(bucket, full_path, io.BytesIO(data), length=len(data)) - - def cleanup(self): - self.remove_dir('') - - def remove_file(self, sub_path): - full_path = os.path.join(self.path, sub_path) - self.conn.remove_object(bucket, full_path) - - def remove_dir(self, sub_path): - if sub_path: - full_path = os.path.join(self.path, sub_path) - else: - full_path = self.path - objs = self.conn.list_objects(bucket, prefix=full_path, recursive=True, - include_version=True) - delobjs = (DeleteObject(o.object_name, o.version_id) for o in objs) - errs = list(self.conn.remove_objects(bucket, delobjs)) - if errs: - strerrs = "; ".join(str(err) for err in errs) - raise Exception("There were errors: {0}".format(strerrs)) - - def exists(self, sub_path): - full_path = os.path.join(self.path, sub_path) - try: - self.conn.stat_object(bucket, full_path) - return True - except minio.error.S3Error as s3err: - if s3err.code == 'NoSuchKey': - return False - raise s3err - except Exception as err: - raise err - - def __str__(self): - return '/' + self.path - - def __repr__(self): - return "S3TestBackupDir" + str(self.path) - - def __fspath__(self): - return self.path From 979671de764f2832d502a1e93405aff165ce78dc Mon Sep 17 00:00:00 2001 From: Victoria Shepard <5807469+demonolock@users.noreply.github.com> Date: Tue, 30 Jan 2024 08:31:04 +0100 Subject: [PATCH 010/216] Move file control for tests (#106) --- testgres/plugins/__init__.py | 3 +-- testgres/plugins/pg_probackup2/pg_probackup2/app.py | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/testgres/plugins/__init__.py b/testgres/plugins/__init__.py index e60331f0..8c19a23b 100644 --- a/testgres/plugins/__init__.py +++ b/testgres/plugins/__init__.py @@ -2,8 +2,7 @@ from pg_probackup2.app import ProbackupApp, ProbackupException from pg_probackup2.init_helpers import init_params from pg_probackup2.storage.fs_backup import FSTestBackupDir -from pg_probackup2.storage.s3_backup import S3TestBackupDir __all__ = [ - "ProbackupApp", "ProbackupException", "init_params", "FSTestBackupDir", "S3TestBackupDir", "GDBobj" + "ProbackupApp", "ProbackupException", "init_params", "FSTestBackupDir", "GDBobj" ] diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/app.py b/testgres/plugins/pg_probackup2/pg_probackup2/app.py index 6b176488..a8050f94 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/app.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/app.py @@ -4,7 +4,6 @@ import os import re import subprocess -import sys import threading import time import unittest @@ -37,9 +36,10 @@ def __str__(self): return '\n ERROR: {0}\n CMD: {1}'.format(repr(self.message), self.cmd) -# Local or S3 backup +# Local backup control fs_backup_class = FSTestBackupDir + class ProbackupApp: def __init__(self, test_class: unittest.TestCase, From a078792ccb11854bb0959d84ee5a93a0e956a33b Mon Sep 17 00:00:00 2001 From: z-kasymalieva <149158086+z-kasymalieva@users.noreply.github.com> Date: Tue, 30 Jan 2024 16:34:08 +0300 Subject: [PATCH 011/216] Fix del_instance function to receive options from tests (#105) --- .../plugins/pg_probackup2/pg_probackup2/app.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/app.py b/testgres/plugins/pg_probackup2/pg_probackup2/app.py index a8050f94..40524864 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/app.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/app.py @@ -192,15 +192,13 @@ def set_backup(self, instance, backup_id=False, return self.run(cmd + options, old_binary=old_binary, expect_error=expect_error) - def del_instance(self, instance, old_binary=False, expect_error=False): - - return self.run([ - 'del-instance', - '--instance={0}'.format(instance), - ], - old_binary=old_binary, - expect_error=expect_error - ) + def del_instance(self, instance, options=None, old_binary=False, expect_error=False): + if options is None: + options = [] + cmd = ['del-instance', '--instance={0}'.format(instance)] + options + return self.run(cmd, + old_binary=old_binary, + expect_error=expect_error) def backup_node( self, instance, node, data_dir=False, From 40aa655985debd8ccc3d6d9e13899a31f89834cc Mon Sep 17 00:00:00 2001 From: Oleg Gurev Date: Tue, 30 Jan 2024 16:39:28 +0300 Subject: [PATCH 012/216] set default test locale to en (#107) --- testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py index 7af21eb6..23777e92 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py @@ -52,6 +52,7 @@ def __init__(self): parts[0] = re.match(r'\d+', parts[0]).group() self.pg_config_version = reduce(lambda v, x: v * 100 + int(x), parts, 0) + os.environ['LANGUAGE'] = 'en' # set default locale language to en. All messages will use this locale test_env = os.environ.copy() envs_list = [ 'LANGUAGE', From 70d2f276a6ceda42b7d66b6937e2857d5332f4c6 Mon Sep 17 00:00:00 2001 From: z-kasymalieva <149158086+z-kasymalieva@users.noreply.github.com> Date: Fri, 2 Feb 2024 13:34:13 +0300 Subject: [PATCH 013/216] fix Error executing query (#110) --- testgres/connection.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/testgres/connection.py b/testgres/connection.py index 882498a9..49b74844 100644 --- a/testgres/connection.py +++ b/testgres/connection.py @@ -104,13 +104,13 @@ def rollback(self): def execute(self, query, *args): self.cursor.execute(query, args) try: - res = self.cursor.fetchall() # pg8000 might return tuples - if isinstance(res, tuple): - res = [tuple(t) for t in res] - + res = [tuple(t) for t in self.cursor.fetchall()] return res - except Exception: + except ProgrammingError: + return None + except Exception as e: + print("Error executing query: {}\n {}".format(repr(e), query)) return None def close(self): From 6424451bdc967a090ad4fb1ca155f59418a5269d Mon Sep 17 00:00:00 2001 From: "z.kasymalieva" Date: Fri, 2 Feb 2024 17:27:01 +0300 Subject: [PATCH 014/216] [create_archive_push] archive_push_command in testgres_pg_probackup added --- .../plugins/pg_probackup2/pg_probackup2/app.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/app.py b/testgres/plugins/pg_probackup2/pg_probackup2/app.py index 40524864..2c9953e6 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/app.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/app.py @@ -748,5 +748,19 @@ def load_backup_class(fs_type): return getattr(module, class_name) + def archive_push(self, instance, node, wal_file_name, wal_file_path=None, options=None, expect_error=False): + if options is None: + options = [] + cmd = [ + 'archive-push', + '--instance={0}'.format(instance), + '--wal-file-name={0}'.format(wal_file_name), + ] + if wal_file_path is None: + cmd = cmd + ['--wal-file-path={0}'.format(os.path.join(node.data_dir, 'pg_wal'))] + else: + cmd = cmd + ['--wal-file-path={0}'.format(wal_file_path)] + return self.run(cmd + options, expect_error=expect_error) + def build_backup_dir(self, backup='backup'): return fs_backup_class(rel_path=self.rel_path, backup=backup) From bd37658ef3f615a3ab78231361583f08178fc1c6 Mon Sep 17 00:00:00 2001 From: Victoria Shepard <5807469+demonolock@users.noreply.github.com> Date: Mon, 12 Feb 2024 15:50:48 +0100 Subject: [PATCH 015/216] Abs backup path (#108) --- .../plugins/pg_probackup2/pg_probackup2/storage/fs_backup.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/storage/fs_backup.py b/testgres/plugins/pg_probackup2/pg_probackup2/storage/fs_backup.py index d076432a..6c9d1463 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/storage/fs_backup.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/storage/fs_backup.py @@ -43,7 +43,10 @@ class FSTestBackupDir(TestBackupDir): """ Backup directory. Usually created by running pg_probackup init -B """ def __init__(self, *, rel_path, backup): - self.path = os.path.join(init_params.tmp_path, rel_path, backup) + backup_prefix = os.environ.get('PG_PROBACKUP_TEST_BACKUP_DIR_PREFIX') + if backup_prefix and not os.path.isabs(backup_prefix): + raise Exception(f"PG_PROBACKUP_TEST_BACKUP_DIR_PREFIX must be an absolute path, current value: {backup_prefix}") + self.path = os.path.join(backup_prefix or init_params.tmp_path, rel_path, backup) self.pb_args = ('-B', self.path) def list_instance_backups(self, instance): From 62238ac3db4920f9c1b0caffee22e34cbe6e2aeb Mon Sep 17 00:00:00 2001 From: asavchkov Date: Tue, 13 Feb 2024 15:54:06 +0700 Subject: [PATCH 016/216] Describe the test backup directory prefix --- testgres/plugins/pg_probackup2/README.md | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/testgres/plugins/pg_probackup2/README.md b/testgres/plugins/pg_probackup2/README.md index b62bf24b..c45f2e3a 100644 --- a/testgres/plugins/pg_probackup2/README.md +++ b/testgres/plugins/pg_probackup2/README.md @@ -16,22 +16,14 @@ The package requires testgres~=1.9.3. ## Usage -### Environment +### Environment variables -> Note: by default testgres runs `initdb`, `pg_ctl`, `psql` provided by `PATH`. - -There are several ways to specify a custom postgres installation: - -* export `PG_CONFIG` environment variable pointing to the `pg_config` executable; -* export `PG_BIN` environment variable pointing to the directory with executable files. - -Example: - -```bash -export PG_BIN=$HOME/pg/bin -python my_tests.py -``` +| Variable | Required | Default value | Description | +| - | - | - | - | +| PGPROBACKUP_TMP_DIR | No | tmp_dirs | The root of the temporary directory hierarchy where tests store data and logs. Relative paths start from the `tests` directory. | +| PG_PROBACKUP_TEST_BACKUP_DIR_PREFIX | No | Temporary test hierarchy | Prefix of the test backup directories. Must be an absolute path. Use this variable to store test backups in a location other than the temporary test hierarchy. | +See [Testgres](https://github.com/postgrespro/testgres/tree/master#environment) on how to configure a custom Postgres installation using `PG_CONFIG` and `PG_BIN` environment variables. ### Examples From d98cee279af487f7bcf588ddae73af63052e4958 Mon Sep 17 00:00:00 2001 From: asavchkov Date: Wed, 14 Feb 2024 19:34:27 +0700 Subject: [PATCH 017/216] Update the Readme of the pg_probackup2 package --- testgres/plugins/pg_probackup2/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testgres/plugins/pg_probackup2/README.md b/testgres/plugins/pg_probackup2/README.md index c45f2e3a..5139ab0f 100644 --- a/testgres/plugins/pg_probackup2/README.md +++ b/testgres/plugins/pg_probackup2/README.md @@ -20,7 +20,7 @@ The package requires testgres~=1.9.3. | Variable | Required | Default value | Description | | - | - | - | - | -| PGPROBACKUP_TMP_DIR | No | tmp_dirs | The root of the temporary directory hierarchy where tests store data and logs. Relative paths start from the `tests` directory. | +| PGPROBACKUP_TMP_DIR | No | tests/tmp_dirs | The root of the temporary directory hierarchy where tests store data and logs. Relative paths start from the current working directory. | | PG_PROBACKUP_TEST_BACKUP_DIR_PREFIX | No | Temporary test hierarchy | Prefix of the test backup directories. Must be an absolute path. Use this variable to store test backups in a location other than the temporary test hierarchy. | See [Testgres](https://github.com/postgrespro/testgres/tree/master#environment) on how to configure a custom Postgres installation using `PG_CONFIG` and `PG_BIN` environment variables. From 29d41fba1fd5a235afe9d3d50b98bd6ae906dd99 Mon Sep 17 00:00:00 2001 From: vshepard Date: Thu, 15 Feb 2024 08:04:43 +0100 Subject: [PATCH 018/216] Add s3 env vars --- testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py index 23777e92..f81386aa 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py @@ -201,6 +201,10 @@ def __init__(self): os.environ["PGAPPNAME"] = "pg_probackup" self.delete_logs = delete_logs + # s3 params + self.s3_config_file = test_env.get('PG_PROBACKUP_S3_CONFIG_FILE') + self.s3_type = test_env.get('PG_PROBACKUP_S3_TEST') + def test_env(self): return self._test_env.copy() From 6b15b7b5660850da51528c91aafb6b3101429094 Mon Sep 17 00:00:00 2001 From: z-kasymalieva <149158086+z-kasymalieva@users.noreply.github.com> Date: Tue, 27 Feb 2024 08:23:00 +0300 Subject: [PATCH 019/216] [create_archive_get] Function testgres_pg_probackup2 archive-get added (#112) --- testgres/plugins/pg_probackup2/pg_probackup2/app.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/app.py b/testgres/plugins/pg_probackup2/pg_probackup2/app.py index 2c9953e6..07825673 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/app.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/app.py @@ -762,5 +762,16 @@ def archive_push(self, instance, node, wal_file_name, wal_file_path=None, option cmd = cmd + ['--wal-file-path={0}'.format(wal_file_path)] return self.run(cmd + options, expect_error=expect_error) + def archive_get(self, instance, wal_file_name, wal_file_path, options=None, expect_error=False): + if options is None: + options = [] + cmd = [ + 'archive-get', + '--instance={0}'.format(instance), + '--wal-file-name={0}'.format(wal_file_name), + '--wal-file-path={0}'.format(wal_file_path), + ] + return self.run(cmd + options, expect_error=expect_error) + def build_backup_dir(self, backup='backup'): return fs_backup_class(rel_path=self.rel_path, backup=backup) From 2baaf7c5a05207d4f26fa378b532cbbdd3b1772c Mon Sep 17 00:00:00 2001 From: asavchkov <79832668+asavchkov@users.noreply.github.com> Date: Thu, 7 Mar 2024 04:13:12 +0700 Subject: [PATCH 020/216] Run the archive command through exec (#113) * Run the archive command through exec * Invoke exec only on Linux --- testgres/plugins/pg_probackup2/pg_probackup2/app.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/app.py b/testgres/plugins/pg_probackup2/pg_probackup2/app.py index 07825673..94dcd997 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/app.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/app.py @@ -663,7 +663,11 @@ def set_archiving( if custom_archive_command is None: archive_command = " ".join([f'"{init_params.probackup_path}"', 'archive-push', *self.backup_dir.pb_args]) - if os.name == "nt": + if os.name == 'posix': + # Dash produces a core dump when it gets a SIGQUIT from its + # child process so replace the shell with pg_probackup + archive_command = 'exec ' + archive_command + elif os.name == "nt": archive_command = archive_command.replace("\\", "\\\\") archive_command += f' --instance={instance}' From e4a6a471da8b215b1b07a94a5aa0841b632c34aa Mon Sep 17 00:00:00 2001 From: Victoria Shepard <5807469+demonolock@users.noreply.github.com> Date: Thu, 7 Mar 2024 00:04:45 +0100 Subject: [PATCH 021/216] Add print commands from run (#114) Co-authored-by: vshepard --- testgres/plugins/pg_probackup2/pg_probackup2/app.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/app.py b/testgres/plugins/pg_probackup2/pg_probackup2/app.py index 94dcd997..4e6e91ff 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/app.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/app.py @@ -1,6 +1,7 @@ import contextlib import importlib import json +import logging import os import re import subprocess @@ -101,6 +102,7 @@ def run(self, command, gdb=False, old_binary=False, return_id=True, env=None, print(self.test_class.cmd) cmdline = [binary_path, *command] + logging.info(' '.join(cmdline)) if gdb is True: # general test flow for using GDBObj return GDBobj(cmdline, self.test_class) From 8436008a5459d530bb9686212517d592ffcd75e8 Mon Sep 17 00:00:00 2001 From: Victoria Shepard <5807469+demonolock@users.noreply.github.com> Date: Mon, 11 Mar 2024 10:58:11 +0100 Subject: [PATCH 022/216] Remove init param (#115) --- .../pg_probackup2/init_helpers.py | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py index f81386aa..f392d1b9 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py @@ -148,21 +148,24 @@ def __init__(self): [self.probackup_path, "--version"], stderr=subprocess.STDOUT, ).decode('utf-8') - self.probackup_version = re.search(r"\d+\.\d+\.\d+", - probackup_version_output - ).group(0) - compressions = re.search(r"\(compressions: ([^)]*)\)", - probackup_version_output).group(1) - self.probackup_compressions = {s.strip() for s in compressions.split(',')} + match = re.search(r"\d+\.\d+\.\d+", + probackup_version_output) + self.probackup_version = match.group(0) if match else None + match = re.search(r"\(compressions: ([^)]*)\)", probackup_version_output) + compressions = match.group(1) if match else None + if compressions: + self.probackup_compressions = {s.strip() for s in compressions.split(',')} + else: + self.probackup_compressions = [] if self.probackup_old_path: old_probackup_version_output = subprocess.check_output( [self.probackup_old_path, "--version"], stderr=subprocess.STDOUT, ).decode('utf-8') - self.old_probackup_version = re.search(r"\d+\.\d+\.\d+", - old_probackup_version_output - ).group(0) + match = re.search(r"\d+\.\d+\.\d+", + old_probackup_version_output) + self.old_probackup_version = match.group(0) if match else None self.remote = test_env.get('PGPROBACKUP_SSH_REMOTE', None) == 'ON' self.ptrack = test_env.get('PG_PROBACKUP_PTRACK', None) == 'ON' and self.pg_config_version >= 110000 @@ -202,7 +205,6 @@ def __init__(self): self.delete_logs = delete_logs # s3 params - self.s3_config_file = test_env.get('PG_PROBACKUP_S3_CONFIG_FILE') self.s3_type = test_env.get('PG_PROBACKUP_S3_TEST') def test_env(self): From 19f66abfb1d8f0303d12138ed65f9a8a00d07bda Mon Sep 17 00:00:00 2001 From: Victoria Shepard <5807469+demonolock@users.noreply.github.com> Date: Tue, 12 Mar 2024 11:22:43 +0100 Subject: [PATCH 023/216] Remove logging in run (#116) --- testgres/plugins/pg_probackup2/pg_probackup2/app.py | 2 -- testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py | 3 --- 2 files changed, 5 deletions(-) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/app.py b/testgres/plugins/pg_probackup2/pg_probackup2/app.py index 4e6e91ff..94dcd997 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/app.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/app.py @@ -1,7 +1,6 @@ import contextlib import importlib import json -import logging import os import re import subprocess @@ -102,7 +101,6 @@ def run(self, command, gdb=False, old_binary=False, return_id=True, env=None, print(self.test_class.cmd) cmdline = [binary_path, *command] - logging.info(' '.join(cmdline)) if gdb is True: # general test flow for using GDBObj return GDBobj(cmdline, self.test_class) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py index f392d1b9..e3dd9e4f 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py @@ -204,9 +204,6 @@ def __init__(self): os.environ["PGAPPNAME"] = "pg_probackup" self.delete_logs = delete_logs - # s3 params - self.s3_type = test_env.get('PG_PROBACKUP_S3_TEST') - def test_env(self): return self._test_env.copy() From 356cd5246f596fc26e7b076ea74223a5ff491025 Mon Sep 17 00:00:00 2001 From: Victoria Shepard <5807469+demonolock@users.noreply.github.com> Date: Thu, 14 Mar 2024 09:21:07 +0100 Subject: [PATCH 024/216] Up version testgres_pg_probackup2 --- testgres/plugins/pg_probackup2/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testgres/plugins/pg_probackup2/setup.py b/testgres/plugins/pg_probackup2/setup.py index 371eb078..381d7ae2 100644 --- a/testgres/plugins/pg_probackup2/setup.py +++ b/testgres/plugins/pg_probackup2/setup.py @@ -4,7 +4,7 @@ from distutils.core import setup setup( - version='0.0.1', + version='0.0.2', name='testgres_pg_probackup2', packages=['pg_probackup2', 'pg_probackup2.storage'], description='Plugin for testgres that manages pg_probackup2', From 4c70b80b84da5d0c950b645b28aee85593374239 Mon Sep 17 00:00:00 2001 From: egarbuz <165897130+egarbuz@users.noreply.github.com> Date: Thu, 4 Apr 2024 12:02:39 +0300 Subject: [PATCH 025/216] Adding port and database name into params list for add-instance command (#118) --- testgres/plugins/pg_probackup2/pg_probackup2/app.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/app.py b/testgres/plugins/pg_probackup2/pg_probackup2/app.py index 94dcd997..1ab71109 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/app.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/app.py @@ -149,13 +149,19 @@ def init(self, options=None, old_binary=False, skip_log_directory=False, expect_ use_backup_dir=use_backup_dir ) - def add_instance(self, instance, node, old_binary=False, options=None, expect_error=False): + def add_instance(self, instance, node, old_binary=False, options=None, expect_error=False, datname=False): if options is None: options = [] + + if not datname: + datname = 'postgres' + cmd = [ 'add-instance', '--instance={0}'.format(instance), - '-D', node.data_dir + '-D', node.data_dir, + '--pgport', '%i' % node.port, + '--pgdatabase', datname ] # don`t forget to kill old_binary after remote ssh release From 3dc640fb7d6df7e5bbdd1970376773f3d9b7bd09 Mon Sep 17 00:00:00 2001 From: "e.garbuz" Date: Thu, 4 Apr 2024 14:11:09 +0300 Subject: [PATCH 026/216] Value pg_probackup major_version added to the class Init --- testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py index e3dd9e4f..b6bf90f4 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py @@ -204,6 +204,8 @@ def __init__(self): os.environ["PGAPPNAME"] = "pg_probackup" self.delete_logs = delete_logs + self.major_version = int(self.probackup_version.split('.')[0]) + def test_env(self): return self._test_env.copy() From fba3bd36a6f86c6be42f87f3fb10e873b02dbec3 Mon Sep 17 00:00:00 2001 From: Sofia Kopikova Date: Thu, 4 Apr 2024 19:16:02 +0300 Subject: [PATCH 027/216] Fix error 'Is another postmaster already running on port XXX' Sometimes when we abnormally shutdown node its port stays busy. So I added retry attempts to start() function in case we encounter such error. Test for this case was added (test_the_same_port). --- testgres/node.py | 32 ++++++++++++++++++++++++-------- tests/test_simple.py | 8 ++++++++ 2 files changed, 32 insertions(+), 8 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index 0f1dcf98..c9b34d7b 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -726,14 +726,30 @@ def start(self, params=[], wait=True): "start" ] + params # yapf: disable - try: - exit_status, out, error = execute_utility(_params, self.utils_log_file, verbose=True) - if error and 'does not exist' in error: - raise Exception - except Exception as e: - msg = 'Cannot start node' - files = self._collect_special_files() - raise_from(StartNodeException(msg, files), e) + startup_retries = 5 + while True: + try: + exit_status, out, error = execute_utility(_params, self.utils_log_file, verbose=True) + if error and 'does not exist' in error: + raise Exception + except Exception as e: + files = self._collect_special_files() + if any(len(file) > 1 and 'Is another postmaster already ' + 'running on port' in file[1].decode() for + file in files): + print("Detected an issue with connecting to port {0}. " + "Trying another port after a 5-second sleep...".format(self.port)) + self.port = reserve_port() + options = {} + options['port'] = str(self.port) + self.set_auto_conf(options) + startup_retries -= 1 + time.sleep(5) + continue + + msg = 'Cannot start node' + raise_from(StartNodeException(msg, files), e) + break self._maybe_start_logger() self.is_started = True return self diff --git a/tests/test_simple.py b/tests/test_simple.py index 8cb0d94e..ba23c3ed 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -1033,6 +1033,14 @@ def test_parse_pg_version(self): # Macos assert parse_pg_version("postgres (PostgreSQL) 14.9 (Homebrew)") == "14.9" + def test_the_same_port(self): + with get_new_node() as node: + node.init().start() + + with get_new_node() as node2: + node2.port = node.port + node2.init().start() + if __name__ == '__main__': if os.environ.get('ALT_CONFIG'): From 2a0c37b9e139cd0a2fc40eedc4edff1fa96dbf66 Mon Sep 17 00:00:00 2001 From: "e.garbuz" Date: Mon, 8 Apr 2024 17:57:55 +0300 Subject: [PATCH 028/216] Add check on digit for probackup_version --- testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py index b6bf90f4..109a411e 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py @@ -204,7 +204,9 @@ def __init__(self): os.environ["PGAPPNAME"] = "pg_probackup" self.delete_logs = delete_logs - self.major_version = int(self.probackup_version.split('.')[0]) + self.major_version = 0 + if self.probackup_version.split('.')[0].isdigit: + self.major_version = int(self.probackup_version.split('.')[0]) def test_env(self): return self._test_env.copy() From 5db4ae393e4b4f69b5721b27e9c80fc3dcd5c044 Mon Sep 17 00:00:00 2001 From: "e.garbuz" Date: Tue, 9 Apr 2024 13:01:07 +0300 Subject: [PATCH 029/216] Break when pg_probackup version is incorrect --- testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py index 109a411e..318877af 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py @@ -204,9 +204,11 @@ def __init__(self): os.environ["PGAPPNAME"] = "pg_probackup" self.delete_logs = delete_logs - self.major_version = 0 if self.probackup_version.split('.')[0].isdigit: self.major_version = int(self.probackup_version.split('.')[0]) + else: + print('Pg_probackup version is not correct!') + sys.exit(1) def test_env(self): return self._test_env.copy() From 3a90ecd41293aa08c2db662ce73ecad0f34b8636 Mon Sep 17 00:00:00 2001 From: "e.garbuz" Date: Tue, 9 Apr 2024 13:36:42 +0300 Subject: [PATCH 030/216] Adding print incorrect version --- testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py index 318877af..f8c9f3b3 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py @@ -207,7 +207,7 @@ def __init__(self): if self.probackup_version.split('.')[0].isdigit: self.major_version = int(self.probackup_version.split('.')[0]) else: - print('Pg_probackup version is not correct!') + print('Pg_probackup version \"{}\" is not correct!'.format(self.probackup_version)) sys.exit(1) def test_env(self): From ea05a4afc54a9ed65769921b9221afec378a6df1 Mon Sep 17 00:00:00 2001 From: "e.garbuz" Date: Wed, 10 Apr 2024 09:45:17 +0300 Subject: [PATCH 031/216] Correction of remarks --- testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py index f8c9f3b3..9231ac04 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py @@ -204,10 +204,10 @@ def __init__(self): os.environ["PGAPPNAME"] = "pg_probackup" self.delete_logs = delete_logs - if self.probackup_version.split('.')[0].isdigit: + if self.probackup_version.split('.')[0].isdigit(): self.major_version = int(self.probackup_version.split('.')[0]) else: - print('Pg_probackup version \"{}\" is not correct!'.format(self.probackup_version)) + print('Pg_probackup version \"{}\" is not correct! Expected that the major pg_probackup version should be a number.'.format(self.probackup_version)) sys.exit(1) def test_env(self): From 5b8a8369cbc0517c4aecacfa35b2955d8e9a4a0d Mon Sep 17 00:00:00 2001 From: "e.garbuz" Date: Wed, 10 Apr 2024 14:23:31 +0300 Subject: [PATCH 032/216] Changed error message --- testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py index 9231ac04..73731a6e 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py @@ -207,7 +207,7 @@ def __init__(self): if self.probackup_version.split('.')[0].isdigit(): self.major_version = int(self.probackup_version.split('.')[0]) else: - print('Pg_probackup version \"{}\" is not correct! Expected that the major pg_probackup version should be a number.'.format(self.probackup_version)) + print('Can\'t process pg_probackup version \"{}\": the major version is expected to be a number'.format(self.probackup_version)) sys.exit(1) def test_env(self): From eb0476209523b4809a4a7e20e4f2cd9968dd5cb9 Mon Sep 17 00:00:00 2001 From: Victoria Shepard <5807469+demonolock@users.noreply.github.com> Date: Tue, 7 May 2024 15:53:49 +0300 Subject: [PATCH 033/216] Add options to pg_basebackup (#121) --- testgres/backup.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/testgres/backup.py b/testgres/backup.py index a89e214d..cecb0f7b 100644 --- a/testgres/backup.py +++ b/testgres/backup.py @@ -33,7 +33,8 @@ def __init__(self, node, base_dir=None, username=None, - xlog_method=XLogMethod.fetch): + xlog_method=XLogMethod.fetch, + options=None): """ Create a new backup. @@ -43,6 +44,8 @@ def __init__(self, username: database user name. xlog_method: none | fetch | stream (see docs) """ + if not options: + options = [] self.os_ops = node.os_ops if not node.status(): raise BackupException('Node must be running') @@ -77,6 +80,7 @@ def __init__(self, "-D", data_dir, "-X", xlog_method.value ] # yapf: disable + _params += options execute_utility(_params, self.log_file) def __enter__(self): From 0cf30a7dc0078645052a7fa9afddb5f6999df5d8 Mon Sep 17 00:00:00 2001 From: Victoria Shepard <5807469+demonolock@users.noreply.github.com> Date: Tue, 7 May 2024 20:07:32 +0300 Subject: [PATCH 034/216] Fix port bind (#117) * Fix port bind * Up testgres version - 1.10.1 --- setup.py | 2 +- testgres/operations/remote_ops.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index b006c8bf..412e8823 100755 --- a/setup.py +++ b/setup.py @@ -27,7 +27,7 @@ readme = f.read() setup( - version='1.10.0', + version='1.10.1', name='testgres', packages=['testgres', 'testgres.operations', 'testgres.helpers'], description='Testing utility for PostgreSQL and its extensions', diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index 01251e1c..f182768b 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -4,6 +4,8 @@ import tempfile import platform +from ..utils import reserve_port + # we support both pg8000 and psycopg2 try: import psycopg2 as pglib @@ -392,7 +394,7 @@ def db_connect(self, dbname, user, password=None, host="localhost", port=5432): """ Established SSH tunnel and Connects to a PostgreSQL """ - self.establish_ssh_tunnel(local_port=port, remote_port=5432) + self.establish_ssh_tunnel(local_port=reserve_port(), remote_port=5432) try: conn = pglib.connect( host=host, From 8a25cb375138442c149f93051579a04704bbe38a Mon Sep 17 00:00:00 2001 From: Victoria Shepard <5807469+demonolock@users.noreply.github.com> Date: Tue, 21 May 2024 01:57:29 +0300 Subject: [PATCH 035/216] Add pgbench_with_wait function (#122) --- testgres/node.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/testgres/node.py b/testgres/node.py index c9b34d7b..d1784cb9 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -1371,7 +1371,7 @@ def pgbench(self, username=None, stdout=None, stderr=None, - options=[]): + options=None): """ Spawn a pgbench process. @@ -1385,6 +1385,8 @@ def pgbench(self, Returns: Process created by subprocess.Popen. """ + if options is None: + options = [] # Set default arguments dbname = dbname or default_dbname() @@ -1404,6 +1406,29 @@ def pgbench(self, return proc + def pgbench_with_wait(self, + dbname=None, + username=None, + stdout=None, + stderr=None, + options=None): + """ + Do pgbench command and wait. + + Args: + dbname: database name to connect to. + username: database user name. + stdout: stdout file to be used by Popen. + stderr: stderr file to be used by Popen. + options: additional options for pgbench (list). + """ + if options is None: + options = [] + + with self.pgbench(dbname, username, stdout, stderr, options) as pgbench: + pgbench.wait() + return + def pgbench_init(self, **kwargs): """ Small wrapper for pgbench_run(). From e375302a114cd4df3ceed54d6526f250c44c08e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabr=C3=ADzio=20de=20Royes=20Mello?= Date: Fri, 24 May 2024 17:36:14 -0300 Subject: [PATCH 036/216] Add options to pg_upgrade (#125) --- testgres/node.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/testgres/node.py b/testgres/node.py index d1784cb9..e5e8fd5f 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -1639,7 +1639,7 @@ def set_auto_conf(self, options, config='postgresql.auto.conf', rm_options={}): self.os_ops.write(path, auto_conf, truncate=True) - def upgrade_from(self, old_node): + def upgrade_from(self, old_node, options=None): """ Upgrade this node from an old node using pg_upgrade. @@ -1652,6 +1652,9 @@ def upgrade_from(self, old_node): if not os.path.exists(self.data_dir): self.init() + if not options: + options = [] + pg_upgrade_binary = self._get_bin_path("pg_upgrade") if not os.path.exists(pg_upgrade_binary): @@ -1666,6 +1669,7 @@ def upgrade_from(self, old_node): "--old-port", str(old_node.port), "--new-port", str(self.port), ] + upgrade_command += options return self.os_ops.exec_command(upgrade_command) From 63db2752501f9ac58a05accc686aec3cd2825c20 Mon Sep 17 00:00:00 2001 From: Vyacheslav Makarov <50846161+MakSl@users.noreply.github.com> Date: Tue, 28 May 2024 18:07:38 +0300 Subject: [PATCH 037/216] Added command execution time measurement (#123) This will be useful to keep track of possible performance degradation when code changes. --- testgres/plugins/pg_probackup2/pg_probackup2/app.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/app.py b/testgres/plugins/pg_probackup2/pg_probackup2/app.py index 1ab71109..3d2e0101 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/app.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/app.py @@ -56,6 +56,7 @@ def __init__(self, test_class: unittest.TestCase, self.verbose = init_params.verbose self.archive_compress = init_params.archive_compress self.test_class.output = None + self.execution_time = None def run(self, command, gdb=False, old_binary=False, return_id=True, env=None, skip_log_directory=False, expect_error=False, use_backup_dir=True): @@ -113,11 +114,15 @@ def run(self, command, gdb=False, old_binary=False, return_id=True, env=None, cmdline = ['gdbserver'] + ['localhost:' + str(gdb_port)] + cmdline print("pg_probackup gdb suspended, waiting gdb connection on localhost:{0}".format(gdb_port)) + start_time = time.time() self.test_class.output = subprocess.check_output( cmdline, stderr=subprocess.STDOUT, env=env ).decode('utf-8', errors='replace') + end_time = time.time() + self.execution_time = end_time - start_time + if command[0] == 'backup' and return_id: # return backup ID for line in self.test_class.output.splitlines(): From c1cfc26b4b74c36fd6cccda2ee00fdce60dabc92 Mon Sep 17 00:00:00 2001 From: Victoria Shepard <5807469+demonolock@users.noreply.github.com> Date: Thu, 30 May 2024 11:28:33 +0200 Subject: [PATCH 038/216] Add parsing backup_id in run app.py (#126) --- .../plugins/pg_probackup2/pg_probackup2/app.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/app.py b/testgres/plugins/pg_probackup2/pg_probackup2/app.py index 3d2e0101..1a4ca9e7 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/app.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/app.py @@ -124,10 +124,7 @@ def run(self, command, gdb=False, old_binary=False, return_id=True, env=None, self.execution_time = end_time - start_time if command[0] == 'backup' and return_id: - # return backup ID - for line in self.test_class.output.splitlines(): - if 'INFO: Backup' and 'completed' in line: - result = line.split()[2] + result = self.get_backup_id() else: result = self.test_class.output if expect_error is True: @@ -144,6 +141,19 @@ def run(self, command, gdb=False, old_binary=False, return_id=True, env=None, else: raise ProbackupException(self.test_class.output, self.test_class.cmd) + def get_backup_id(self): + if init_params.major_version > 2: + pattern = re.compile(r"Backup (.*) completed successfully.") + for line in self.test_class.output.splitlines(): + match = pattern.search(line) + if match: + return match.group(1) + else: + for line in self.test_class.output.splitlines(): + if 'INFO: Backup' and 'completed' in line: + return line.split()[2] + return None + def init(self, options=None, old_binary=False, skip_log_directory=False, expect_error=False, use_backup_dir=True): if options is None: options = [] From b4268f8bdaaf67e0f7677c0f801210a0085bb594 Mon Sep 17 00:00:00 2001 From: Victoria Shepard <5807469+demonolock@users.noreply.github.com> Date: Wed, 5 Jun 2024 09:34:35 +0200 Subject: [PATCH 039/216] Revert "Fix port bind (#117)" (#128) This reverts commit 0cf30a7dc0078645052a7fa9afddb5f6999df5d8. --- setup.py | 2 +- testgres/operations/remote_ops.py | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index 412e8823..b006c8bf 100755 --- a/setup.py +++ b/setup.py @@ -27,7 +27,7 @@ readme = f.read() setup( - version='1.10.1', + version='1.10.0', name='testgres', packages=['testgres', 'testgres.operations', 'testgres.helpers'], description='Testing utility for PostgreSQL and its extensions', diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index f182768b..01251e1c 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -4,8 +4,6 @@ import tempfile import platform -from ..utils import reserve_port - # we support both pg8000 and psycopg2 try: import psycopg2 as pglib @@ -394,7 +392,7 @@ def db_connect(self, dbname, user, password=None, host="localhost", port=5432): """ Established SSH tunnel and Connects to a PostgreSQL """ - self.establish_ssh_tunnel(local_port=reserve_port(), remote_port=5432) + self.establish_ssh_tunnel(local_port=port, remote_port=5432) try: conn = pglib.connect( host=host, From 0728583c9b6a9df660cfd209220d8031893d136c Mon Sep 17 00:00:00 2001 From: asavchkov Date: Mon, 24 Jun 2024 17:01:31 +0700 Subject: [PATCH 040/216] Up version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b006c8bf..412e8823 100755 --- a/setup.py +++ b/setup.py @@ -27,7 +27,7 @@ readme = f.read() setup( - version='1.10.0', + version='1.10.1', name='testgres', packages=['testgres', 'testgres.operations', 'testgres.helpers'], description='Testing utility for PostgreSQL and its extensions', From 529b4df245ff73b9db757854174f29bb208a0c4c Mon Sep 17 00:00:00 2001 From: asavchkov Date: Mon, 24 Jun 2024 17:16:50 +0700 Subject: [PATCH 041/216] Up pg_probackup2 version --- testgres/plugins/pg_probackup2/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testgres/plugins/pg_probackup2/setup.py b/testgres/plugins/pg_probackup2/setup.py index 381d7ae2..2ff5a503 100644 --- a/testgres/plugins/pg_probackup2/setup.py +++ b/testgres/plugins/pg_probackup2/setup.py @@ -4,7 +4,7 @@ from distutils.core import setup setup( - version='0.0.2', + version='0.0.3', name='testgres_pg_probackup2', packages=['pg_probackup2', 'pg_probackup2.storage'], description='Plugin for testgres that manages pg_probackup2', From 7847380ed604ddbcc4abf57115777c7064dfd716 Mon Sep 17 00:00:00 2001 From: asavchkov <79832668+asavchkov@users.noreply.github.com> Date: Wed, 26 Jun 2024 04:04:40 +0700 Subject: [PATCH 042/216] Add an SSH port parameter (#131) --- testgres/operations/os_ops.py | 3 ++- testgres/operations/remote_ops.py | 26 +++++++++++++++----------- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/testgres/operations/os_ops.py b/testgres/operations/os_ops.py index dd6613cf..236a08c6 100644 --- a/testgres/operations/os_ops.py +++ b/testgres/operations/os_ops.py @@ -10,8 +10,9 @@ class ConnectionParams: - def __init__(self, host='127.0.0.1', ssh_key=None, username=None): + def __init__(self, host='127.0.0.1', port=None, ssh_key=None, username=None): self.host = host + self.port = port self.ssh_key = ssh_key self.username = username diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index 01251e1c..697b4258 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -44,11 +44,13 @@ def __init__(self, conn_params: ConnectionParams): super().__init__(conn_params.username) self.conn_params = conn_params self.host = conn_params.host + self.port = conn_params.port self.ssh_key = conn_params.ssh_key + self.ssh_args = [] if self.ssh_key: - self.ssh_cmd = ["-i", self.ssh_key] - else: - self.ssh_cmd = [] + self.ssh_args += ["-i", self.ssh_key] + if self.port: + self.ssh_args += ["-p", self.port] self.remote = True self.username = conn_params.username or self.get_user() self.add_known_host(self.host) @@ -95,9 +97,9 @@ def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False, """ ssh_cmd = [] if isinstance(cmd, str): - ssh_cmd = ['ssh', f"{self.username}@{self.host}"] + self.ssh_cmd + [cmd] + ssh_cmd = ['ssh', f"{self.username}@{self.host}"] + self.ssh_args + [cmd] elif isinstance(cmd, list): - ssh_cmd = ['ssh', f"{self.username}@{self.host}"] + self.ssh_cmd + cmd + ssh_cmd = ['ssh', f"{self.username}@{self.host}"] + self.ssh_args + cmd process = subprocess.Popen(ssh_cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) if get_process: return process @@ -246,9 +248,9 @@ def mkdtemp(self, prefix=None): - prefix (str): The prefix of the temporary directory name. """ if prefix: - command = ["ssh"] + self.ssh_cmd + [f"{self.username}@{self.host}", f"mktemp -d {prefix}XXXXX"] + command = ["ssh"] + self.ssh_args + [f"{self.username}@{self.host}", f"mktemp -d {prefix}XXXXX"] else: - command = ["ssh"] + self.ssh_cmd + [f"{self.username}@{self.host}", "mktemp -d"] + command = ["ssh"] + self.ssh_args + [f"{self.username}@{self.host}", "mktemp -d"] result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) @@ -291,8 +293,10 @@ def write(self, filename, data, truncate=False, binary=False, read_and_write=Fal mode = "r+b" if binary else "r+" with tempfile.NamedTemporaryFile(mode=mode, delete=False) as tmp_file: + # For scp the port is specified by a "-P" option + scp_args = ['-P' if x == '-p' else x for x in self.ssh_args] if not truncate: - scp_cmd = ['scp'] + self.ssh_cmd + [f"{self.username}@{self.host}:{filename}", tmp_file.name] + scp_cmd = ['scp'] + scp_args + [f"{self.username}@{self.host}:{filename}", tmp_file.name] subprocess.run(scp_cmd, check=False) # The file might not exist yet tmp_file.seek(0, os.SEEK_END) @@ -308,11 +312,11 @@ def write(self, filename, data, truncate=False, binary=False, read_and_write=Fal tmp_file.write(data) tmp_file.flush() - scp_cmd = ['scp'] + self.ssh_cmd + [tmp_file.name, f"{self.username}@{self.host}:{filename}"] + scp_cmd = ['scp'] + scp_args + [tmp_file.name, f"{self.username}@{self.host}:{filename}"] subprocess.run(scp_cmd, check=True) remote_directory = os.path.dirname(filename) - mkdir_cmd = ['ssh'] + self.ssh_cmd + [f"{self.username}@{self.host}", f"mkdir -p {remote_directory}"] + mkdir_cmd = ['ssh'] + self.ssh_args + [f"{self.username}@{self.host}", f"mkdir -p {remote_directory}"] subprocess.run(mkdir_cmd, check=True) os.remove(tmp_file.name) @@ -377,7 +381,7 @@ def get_pid(self): return int(self.exec_command("echo $$", encoding=get_default_encoding())) def get_process_children(self, pid): - command = ["ssh"] + self.ssh_cmd + [f"{self.username}@{self.host}", f"pgrep -P {pid}"] + command = ["ssh"] + self.ssh_args + [f"{self.username}@{self.host}", f"pgrep -P {pid}"] result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) From 2bf16a5a5b5ec64b93e5ebbd0891ef8edbe25f0f Mon Sep 17 00:00:00 2001 From: asavchkov <79832668+asavchkov@users.noreply.github.com> Date: Thu, 27 Jun 2024 18:18:33 +0700 Subject: [PATCH 043/216] Pass DB port to NodeApp (#132) --- testgres/node.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index e5e8fd5f..f7109b0c 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -127,7 +127,7 @@ def __repr__(self): class PostgresNode(object): - def __init__(self, name=None, port=None, base_dir=None, conn_params: ConnectionParams = ConnectionParams(), bin_dir=None, prefix=None): + def __init__(self, name=None, base_dir=None, port=None, conn_params: ConnectionParams = ConnectionParams(), bin_dir=None, prefix=None): """ PostgresNode constructor. @@ -156,9 +156,9 @@ def __init__(self, name=None, port=None, base_dir=None, conn_params: ConnectionP else: self.os_ops = LocalOperations(conn_params) + self.host = self.os_ops.host self.port = port or reserve_port() - self.host = self.os_ops.host self.ssh_key = self.os_ops.ssh_key # defaults for __exit__() @@ -1690,12 +1690,13 @@ def __init__(self, test_path, nodes_to_cleanup, os_ops=LocalOperations()): def make_empty( self, - base_dir=None): + base_dir=None, + port=None): real_base_dir = os.path.join(self.test_path, base_dir) self.os_ops.rmdirs(real_base_dir, ignore_errors=True) self.os_ops.makedirs(real_base_dir) - node = PostgresNode(base_dir=real_base_dir) + node = PostgresNode(base_dir=real_base_dir, port=port) node.should_rm_dirs = True self.nodes_to_cleanup.append(node) @@ -1704,6 +1705,7 @@ def make_empty( def make_simple( self, base_dir=None, + port=None, set_replication=False, ptrack_enable=False, initdb_params=[], @@ -1711,7 +1713,7 @@ def make_simple( checksum=True): if checksum and '--data-checksums' not in initdb_params: initdb_params.append('--data-checksums') - node = self.make_empty(base_dir) + node = self.make_empty(base_dir, port) node.init( initdb_params=initdb_params, allow_streaming=set_replication) From 4543f80fbc75bac80e2d229f8fe8ba8d2c65f551 Mon Sep 17 00:00:00 2001 From: asavchkov <79832668+asavchkov@users.noreply.github.com> Date: Fri, 28 Jun 2024 18:50:19 +0700 Subject: [PATCH 044/216] Make use of the default SSH user (#133) Make use of the default SSH user --- testgres/node.py | 26 +++++++------------------- testgres/operations/local_ops.py | 6 +----- testgres/operations/os_ops.py | 3 +-- testgres/operations/remote_ops.py | 28 +++++++++++++--------------- 4 files changed, 22 insertions(+), 41 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index f7109b0c..13d13294 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -63,7 +63,6 @@ from .defaults import \ default_dbname, \ - default_username, \ generate_app_name from .exceptions import \ @@ -683,8 +682,6 @@ def slow_start(self, replica=False, dbname='template1', username=None, max_attem If False, waits for the instance to be in primary mode. Default is False. max_attempts: """ - if not username: - username = default_username() self.start() if replica: @@ -694,7 +691,7 @@ def slow_start(self, replica=False, dbname='template1', username=None, max_attem # Call poll_query_until until the expected value is returned self.poll_query_until(query=query, dbname=dbname, - username=username, + username=username or self.os_ops.username, suppress={InternalError, QueryException, ProgrammingError, @@ -967,15 +964,13 @@ def psql(self, >>> psql(query='select 3', ON_ERROR_STOP=1) """ - # Set default arguments dbname = dbname or default_dbname() - username = username or default_username() psql_params = [ self._get_bin_path("psql"), "-p", str(self.port), "-h", self.host, - "-U", username, + "-U", username or self.os_ops.username, "-X", # no .psqlrc "-A", # unaligned output "-t", # print rows only @@ -1087,9 +1082,6 @@ def tmpfile(): fname = self.os_ops.mkstemp(prefix=TMP_DUMP) return fname - # Set default arguments - dbname = dbname or default_dbname() - username = username or default_username() filename = filename or tmpfile() _params = [ @@ -1097,8 +1089,8 @@ def tmpfile(): "-p", str(self.port), "-h", self.host, "-f", filename, - "-U", username, - "-d", dbname, + "-U", username or self.os_ops.username, + "-d", dbname or default_dbname(), "-F", format.value ] # yapf: disable @@ -1118,7 +1110,7 @@ def restore(self, filename, dbname=None, username=None): # Set default arguments dbname = dbname or default_dbname() - username = username or default_username() + username = username or self.os_ops.username _params = [ self._get_bin_path("pg_restore"), @@ -1388,15 +1380,13 @@ def pgbench(self, if options is None: options = [] - # Set default arguments dbname = dbname or default_dbname() - username = username or default_username() _params = [ self._get_bin_path("pgbench"), "-p", str(self.port), "-h", self.host, - "-U", username, + "-U", username or self.os_ops.username ] + options # yapf: disable # should be the last one @@ -1463,15 +1453,13 @@ def pgbench_run(self, dbname=None, username=None, options=[], **kwargs): >>> pgbench_run(time=10) """ - # Set default arguments dbname = dbname or default_dbname() - username = username or default_username() _params = [ self._get_bin_path("pgbench"), "-p", str(self.port), "-h", self.host, - "-U", username, + "-U", username or self.os_ops.username ] + options # yapf: disable for key, value in iteritems(kwargs): diff --git a/testgres/operations/local_ops.py b/testgres/operations/local_ops.py index ef360d3b..313d7060 100644 --- a/testgres/operations/local_ops.py +++ b/testgres/operations/local_ops.py @@ -38,7 +38,7 @@ def __init__(self, conn_params=None): self.host = conn_params.host self.ssh_key = None self.remote = False - self.username = conn_params.username or self.get_user() + self.username = conn_params.username or getpass.getuser() @staticmethod def _raise_exec_exception(message, command, exit_code, output): @@ -130,10 +130,6 @@ def set_env(self, var_name, var_val): # Check if the directory is already in PATH os.environ[var_name] = var_val - # Get environment variables - def get_user(self): - return self.username or getpass.getuser() - def get_name(self): return os.name diff --git a/testgres/operations/os_ops.py b/testgres/operations/os_ops.py index 236a08c6..0b5efff9 100644 --- a/testgres/operations/os_ops.py +++ b/testgres/operations/os_ops.py @@ -45,9 +45,8 @@ def set_env(self, var_name, var_val): # Check if the directory is already in PATH raise NotImplementedError() - # Get environment variables def get_user(self): - raise NotImplementedError() + return self.username def get_name(self): raise NotImplementedError() diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index 697b4258..83965336 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -1,8 +1,9 @@ -import logging +import getpass import os +import logging +import platform import subprocess import tempfile -import platform # we support both pg8000 and psycopg2 try: @@ -52,7 +53,8 @@ def __init__(self, conn_params: ConnectionParams): if self.port: self.ssh_args += ["-p", self.port] self.remote = True - self.username = conn_params.username or self.get_user() + self.username = conn_params.username or getpass.getuser() + self.ssh_dest = f"{self.username}@{self.host}" if conn_params.username else self.host self.add_known_host(self.host) self.tunnel_process = None @@ -97,9 +99,9 @@ def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False, """ ssh_cmd = [] if isinstance(cmd, str): - ssh_cmd = ['ssh', f"{self.username}@{self.host}"] + self.ssh_args + [cmd] + ssh_cmd = ['ssh', self.ssh_dest] + self.ssh_args + [cmd] elif isinstance(cmd, list): - ssh_cmd = ['ssh', f"{self.username}@{self.host}"] + self.ssh_args + cmd + ssh_cmd = ['ssh', self.ssh_dest] + self.ssh_args + cmd process = subprocess.Popen(ssh_cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) if get_process: return process @@ -174,10 +176,6 @@ def set_env(self, var_name: str, var_val: str): """ return self.exec_command("export {}={}".format(var_name, var_val)) - # Get environment variables - def get_user(self): - return self.exec_command("echo $USER", encoding=get_default_encoding()).strip() - def get_name(self): cmd = 'python3 -c "import os; print(os.name)"' return self.exec_command(cmd, encoding=get_default_encoding()).strip() @@ -248,9 +246,9 @@ def mkdtemp(self, prefix=None): - prefix (str): The prefix of the temporary directory name. """ if prefix: - command = ["ssh"] + self.ssh_args + [f"{self.username}@{self.host}", f"mktemp -d {prefix}XXXXX"] + command = ["ssh"] + self.ssh_args + [self.ssh_dest, f"mktemp -d {prefix}XXXXX"] else: - command = ["ssh"] + self.ssh_args + [f"{self.username}@{self.host}", "mktemp -d"] + command = ["ssh"] + self.ssh_args + [self.ssh_dest, "mktemp -d"] result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) @@ -296,7 +294,7 @@ def write(self, filename, data, truncate=False, binary=False, read_and_write=Fal # For scp the port is specified by a "-P" option scp_args = ['-P' if x == '-p' else x for x in self.ssh_args] if not truncate: - scp_cmd = ['scp'] + scp_args + [f"{self.username}@{self.host}:{filename}", tmp_file.name] + scp_cmd = ['scp'] + scp_args + [f"{self.ssh_dest}:{filename}", tmp_file.name] subprocess.run(scp_cmd, check=False) # The file might not exist yet tmp_file.seek(0, os.SEEK_END) @@ -312,11 +310,11 @@ def write(self, filename, data, truncate=False, binary=False, read_and_write=Fal tmp_file.write(data) tmp_file.flush() - scp_cmd = ['scp'] + scp_args + [tmp_file.name, f"{self.username}@{self.host}:{filename}"] + scp_cmd = ['scp'] + scp_args + [tmp_file.name, f"{self.ssh_dest}:{filename}"] subprocess.run(scp_cmd, check=True) remote_directory = os.path.dirname(filename) - mkdir_cmd = ['ssh'] + self.ssh_args + [f"{self.username}@{self.host}", f"mkdir -p {remote_directory}"] + mkdir_cmd = ['ssh'] + self.ssh_args + [self.ssh_dest, f"mkdir -p {remote_directory}"] subprocess.run(mkdir_cmd, check=True) os.remove(tmp_file.name) @@ -381,7 +379,7 @@ def get_pid(self): return int(self.exec_command("echo $$", encoding=get_default_encoding())) def get_process_children(self, pid): - command = ["ssh"] + self.ssh_args + [f"{self.username}@{self.host}", f"pgrep -P {pid}"] + command = ["ssh"] + self.ssh_args + [self.ssh_dest, f"pgrep -P {pid}"] result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) From a128b12cb18ff11c4aef65491dce091e849ef471 Mon Sep 17 00:00:00 2001 From: asavchkov <79832668+asavchkov@users.noreply.github.com> Date: Mon, 1 Jul 2024 15:47:40 +0700 Subject: [PATCH 045/216] Remove SSH tunnel (#136) --- testgres/node.py | 16 ++++-- testgres/operations/os_ops.py | 11 +++- testgres/operations/remote_ops.py | 55 +++---------------- .../pg_probackup2/pg_probackup2/app.py | 9 ++- 4 files changed, 37 insertions(+), 54 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index 13d13294..479ea4ec 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -528,7 +528,9 @@ def get_auth_method(t): u"host\treplication\tall\t127.0.0.1/32\t{}\n".format(auth_host), u"host\treplication\tall\t::1/128\t\t{}\n".format(auth_host), u"host\treplication\tall\t{}/24\t\t{}\n".format(subnet_base, auth_host), - u"host\tall\tall\t{}/24\t\t{}\n".format(subnet_base, auth_host) + u"host\tall\tall\t{}/24\t\t{}\n".format(subnet_base, auth_host), + u"host\tall\tall\tall\t{}\n".format(auth_host), + u"host\treplication\tall\tall\t{}\n".format(auth_host) ] # yapf: disable # write missing lines @@ -1671,9 +1673,15 @@ def _get_bin_path(self, filename): class NodeApp: - def __init__(self, test_path, nodes_to_cleanup, os_ops=LocalOperations()): - self.test_path = test_path - self.nodes_to_cleanup = nodes_to_cleanup + def __init__(self, test_path=None, nodes_to_cleanup=None, os_ops=LocalOperations()): + if test_path: + if os.path.isabs(test_path): + self.test_path = test_path + else: + self.test_path = os.path.join(os_ops.cwd(), test_path) + else: + self.test_path = os_ops.cwd() + self.nodes_to_cleanup = nodes_to_cleanup if nodes_to_cleanup else [] self.os_ops = os_ops def make_empty( diff --git a/testgres/operations/os_ops.py b/testgres/operations/os_ops.py index 0b5efff9..76284049 100644 --- a/testgres/operations/os_ops.py +++ b/testgres/operations/os_ops.py @@ -1,4 +1,6 @@ +import getpass import locale +import sys try: import psycopg2 as pglib # noqa: F401 @@ -24,7 +26,7 @@ def get_default_encoding(): class OsOperations: def __init__(self, username=None): self.ssh_key = None - self.username = username + self.username = username or getpass.getuser() # Command execution def exec_command(self, cmd, **kwargs): @@ -34,6 +36,13 @@ def exec_command(self, cmd, **kwargs): def environ(self, var_name): raise NotImplementedError() + def cwd(self): + if sys.platform == 'linux': + cmd = 'pwd' + elif sys.platform == 'win32': + cmd = 'cd' + return self.exec_command(cmd).decode().rstrip() + def find_executable(self, executable): raise NotImplementedError() diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index 83965336..fa031075 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -1,6 +1,5 @@ import getpass import os -import logging import platform import subprocess import tempfile @@ -55,40 +54,10 @@ def __init__(self, conn_params: ConnectionParams): self.remote = True self.username = conn_params.username or getpass.getuser() self.ssh_dest = f"{self.username}@{self.host}" if conn_params.username else self.host - self.add_known_host(self.host) - self.tunnel_process = None def __enter__(self): return self - def __exit__(self, exc_type, exc_val, exc_tb): - self.close_ssh_tunnel() - - def establish_ssh_tunnel(self, local_port, remote_port): - """ - Establish an SSH tunnel from a local port to a remote PostgreSQL port. - """ - ssh_cmd = ['-N', '-L', f"{local_port}:localhost:{remote_port}"] - self.tunnel_process = self.exec_command(ssh_cmd, get_process=True, timeout=300) - - def close_ssh_tunnel(self): - if hasattr(self, 'tunnel_process'): - self.tunnel_process.terminate() - self.tunnel_process.wait() - del self.tunnel_process - else: - print("No active tunnel to close.") - - def add_known_host(self, host): - known_hosts_path = os.path.expanduser("~/.ssh/known_hosts") - cmd = 'ssh-keyscan -H %s >> %s' % (host, known_hosts_path) - - try: - subprocess.check_call(cmd, shell=True) - logging.info("Successfully added %s to known_hosts." % host) - except subprocess.CalledProcessError as e: - raise Exception("Failed to add %s to known_hosts. Error: %s" % (host, str(e))) - def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False, encoding=None, shell=True, text=False, input=None, stdin=None, stdout=None, stderr=None, get_process=None, timeout=None): @@ -293,6 +262,7 @@ def write(self, filename, data, truncate=False, binary=False, read_and_write=Fal with tempfile.NamedTemporaryFile(mode=mode, delete=False) as tmp_file: # For scp the port is specified by a "-P" option scp_args = ['-P' if x == '-p' else x for x in self.ssh_args] + if not truncate: scp_cmd = ['scp'] + scp_args + [f"{self.ssh_dest}:{filename}", tmp_file.name] subprocess.run(scp_cmd, check=False) # The file might not exist yet @@ -391,18 +361,11 @@ def get_process_children(self, pid): # Database control def db_connect(self, dbname, user, password=None, host="localhost", port=5432): - """ - Established SSH tunnel and Connects to a PostgreSQL - """ - self.establish_ssh_tunnel(local_port=port, remote_port=5432) - try: - conn = pglib.connect( - host=host, - port=port, - database=dbname, - user=user, - password=password, - ) - return conn - except Exception as e: - raise Exception(f"Could not connect to the database. Error: {e}") + conn = pglib.connect( + host=host, + port=port, + database=dbname, + user=user, + password=password, + ) + return conn diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/app.py b/testgres/plugins/pg_probackup2/pg_probackup2/app.py index 1a4ca9e7..ffad24d3 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/app.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/app.py @@ -43,14 +43,14 @@ def __str__(self): class ProbackupApp: def __init__(self, test_class: unittest.TestCase, - pg_node, pb_log_path, test_env, auto_compress_alg, backup_dir): + pg_node, pb_log_path, test_env, auto_compress_alg, backup_dir, probackup_path=None): self.test_class = test_class self.pg_node = pg_node self.pb_log_path = pb_log_path self.test_env = test_env self.auto_compress_alg = auto_compress_alg self.backup_dir = backup_dir - self.probackup_path = init_params.probackup_path + self.probackup_path = probackup_path or init_params.probackup_path self.probackup_old_path = init_params.probackup_old_path self.remote = init_params.remote self.verbose = init_params.verbose @@ -388,6 +388,7 @@ def catchup_node( backup_mode, source_pgdata, destination_node, options=None, remote_host='localhost', + remote_port=None, expect_error=False, gdb=False ): @@ -401,7 +402,9 @@ def catchup_node( '--destination-pgdata={0}'.format(destination_node.data_dir) ] if self.remote: - cmd_list += ['--remote-proto=ssh', '--remote-host=%s' % remote_host] + cmd_list += ['--remote-proto=ssh', f'--remote-host={remote_host}'] + if remote_port: + cmd_list.append(f'--remote-port={remote_port}') if self.verbose: cmd_list += [ '--log-level-file=VERBOSE', From 1d68e91a71f98f440fa429578fa1c289652ba10a Mon Sep 17 00:00:00 2001 From: Victoria Shepard <5807469+demonolock@users.noreply.github.com> Date: Mon, 1 Jul 2024 12:35:39 +0200 Subject: [PATCH 046/216] Fix node cleanup (#135) * Add expect_error to pg_upgrade * Fix node cleanup - rmdirs * Add cleanup parameter - clean full dir --- testgres/node.py | 13 +++++------ testgres/operations/local_ops.py | 37 ++++++++++++++++++++++++++------ 2 files changed, 38 insertions(+), 12 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index 479ea4ec..2ad6ce54 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -914,13 +914,14 @@ def free_port(self): self._should_free_port = False release_port(self.port) - def cleanup(self, max_attempts=3): + def cleanup(self, max_attempts=3, full=False): """ Stop node if needed and remove its data/logs directory. NOTE: take a look at TestgresConfig.node_cleanup_full. Args: max_attempts: how many times should we try to stop()? + full: clean full base dir Returns: This instance of :class:`.PostgresNode`. @@ -929,12 +930,12 @@ def cleanup(self, max_attempts=3): self._try_shutdown(max_attempts) # choose directory to be removed - if testgres_config.node_cleanup_full: + if testgres_config.node_cleanup_full or full: rm_dir = self.base_dir # everything else: rm_dir = self.data_dir # just data, save logs - self.os_ops.rmdirs(rm_dir, ignore_errors=True) + self.os_ops.rmdirs(rm_dir, ignore_errors=False) return self @@ -1629,7 +1630,7 @@ def set_auto_conf(self, options, config='postgresql.auto.conf', rm_options={}): self.os_ops.write(path, auto_conf, truncate=True) - def upgrade_from(self, old_node, options=None): + def upgrade_from(self, old_node, options=None, expect_error=False): """ Upgrade this node from an old node using pg_upgrade. @@ -1657,11 +1658,11 @@ def upgrade_from(self, old_node, options=None): "--old-datadir", old_node.data_dir, "--new-datadir", self.data_dir, "--old-port", str(old_node.port), - "--new-port", str(self.port), + "--new-port", str(self.port) ] upgrade_command += options - return self.os_ops.exec_command(upgrade_command) + return self.os_ops.exec_command(upgrade_command, expect_error=expect_error) def _get_bin_path(self, filename): if self.bin_dir: diff --git a/testgres/operations/local_ops.py b/testgres/operations/local_ops.py index 313d7060..b05a11e2 100644 --- a/testgres/operations/local_ops.py +++ b/testgres/operations/local_ops.py @@ -4,6 +4,7 @@ import stat import subprocess import tempfile +import time import psutil @@ -19,13 +20,18 @@ CMD_TIMEOUT_SEC = 60 error_markers = [b'error', b'Permission denied', b'fatal'] +err_out_markers = [b'Failure'] -def has_errors(output): +def has_errors(output=None, error=None): if output: if isinstance(output, str): output = output.encode(get_default_encoding()) - return any(marker in output for marker in error_markers) + return any(marker in output for marker in err_out_markers) + if error: + if isinstance(error, str): + error = error.encode(get_default_encoding()) + return any(marker in error for marker in error_markers) return False @@ -107,8 +113,8 @@ def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False, process, output, error = self._run_command(cmd, shell, input, stdin, stdout, stderr, get_process, timeout, encoding) if get_process: return process - if process.returncode != 0 or (has_errors(error) and not expect_error): - self._raise_exec_exception('Utility exited with non-zero code. Error `{}`', cmd, process.returncode, error) + if (process.returncode != 0 or has_errors(output=output, error=error)) and not expect_error: + self._raise_exec_exception('Utility exited with non-zero code. Error `{}`', cmd, process.returncode, error or output) if verbose: return process.returncode, output, error @@ -142,8 +148,27 @@ def makedirs(self, path, remove_existing=False): except FileExistsError: pass - def rmdirs(self, path, ignore_errors=True): - return rmtree(path, ignore_errors=ignore_errors) + def rmdirs(self, path, ignore_errors=True, retries=3, delay=1): + """ + Removes a directory and its contents, retrying on failure. + + :param path: Path to the directory. + :param ignore_errors: If True, ignore errors. + :param retries: Number of attempts to remove the directory. + :param delay: Delay between attempts in seconds. + """ + for attempt in range(retries): + try: + rmtree(path, ignore_errors=ignore_errors) + if not os.path.exists(path): + return True + except FileNotFoundError: + return True + except Exception as e: + print(f"Error: Failed to remove directory {path} on attempt {attempt + 1}: {e}") + time.sleep(delay) + print(f"Error: Failed to remove directory {path} after {retries} attempts.") + return False def listdir(self, path): return os.listdir(path) From 8b6b813f2245859dcd7572b458910cb9ad213654 Mon Sep 17 00:00:00 2001 From: Victoria Shepard <5807469+demonolock@users.noreply.github.com> Date: Thu, 4 Jul 2024 02:42:03 +0200 Subject: [PATCH 047/216] Remove polling (#137) * Change print on logging * Normalize error --------- Co-authored-by: vshepard --- testgres/config.py | 6 ++++++ testgres/connection.py | 3 ++- testgres/node.py | 11 +++++------ testgres/operations/local_ops.py | 5 +++-- testgres/operations/remote_ops.py | 10 +++++++++- .../plugins/pg_probackup2/pg_probackup2/app.py | 14 +++++++------- .../pg_probackup2/pg_probackup2/init_helpers.py | 11 ++++++----- .../pg_probackup2/tests/basic_test.py | 3 ++- testgres/utils.py | 1 - 9 files changed, 40 insertions(+), 24 deletions(-) diff --git a/testgres/config.py b/testgres/config.py index b6c43926..63719f1d 100644 --- a/testgres/config.py +++ b/testgres/config.py @@ -2,6 +2,8 @@ import atexit import copy +import logging +import os import tempfile from contextlib import contextmanager @@ -10,6 +12,10 @@ from .operations.os_ops import OsOperations from .operations.local_ops import LocalOperations +log_level = os.getenv('LOGGING_LEVEL', 'WARNING').upper() +log_format = os.getenv('LOGGING_FORMAT', '%(asctime)s - %(levelname)s - %(message)s').upper() +logging.basicConfig(level=log_level, format=log_format) + class GlobalConfig(object): """ diff --git a/testgres/connection.py b/testgres/connection.py index 49b74844..ccedd135 100644 --- a/testgres/connection.py +++ b/testgres/connection.py @@ -1,4 +1,5 @@ # coding: utf-8 +import logging # we support both pg8000 and psycopg2 try: @@ -110,7 +111,7 @@ def execute(self, query, *args): except ProgrammingError: return None except Exception as e: - print("Error executing query: {}\n {}".format(repr(e), query)) + logging.error("Error executing query: {}\n {}".format(repr(e), query)) return None def close(self): diff --git a/testgres/node.py b/testgres/node.py index 2ad6ce54..2ea49529 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -1,5 +1,5 @@ # coding: utf-8 - +import logging import os import random import signal @@ -736,11 +736,10 @@ def start(self, params=[], wait=True): if any(len(file) > 1 and 'Is another postmaster already ' 'running on port' in file[1].decode() for file in files): - print("Detected an issue with connecting to port {0}. " - "Trying another port after a 5-second sleep...".format(self.port)) + logging.warning("Detected an issue with connecting to port {0}. " + "Trying another port after a 5-second sleep...".format(self.port)) self.port = reserve_port() - options = {} - options['port'] = str(self.port) + options = {'port': str(self.port)} self.set_auto_conf(options) startup_retries -= 1 time.sleep(5) @@ -1166,7 +1165,6 @@ def poll_query_until(self, assert sleep_time > 0 attempts = 0 while max_attempts == 0 or attempts < max_attempts: - print(f"Pooling {attempts}") try: res = self.execute(dbname=dbname, query=query, @@ -1190,6 +1188,7 @@ def poll_query_until(self, return # done except tuple(suppress or []): + logging.info(f"Trying execute, attempt {attempts + 1}.\nQuery: {query}") pass # we're suppressing them time.sleep(sleep_time) diff --git a/testgres/operations/local_ops.py b/testgres/operations/local_ops.py index b05a11e2..b518a6cb 100644 --- a/testgres/operations/local_ops.py +++ b/testgres/operations/local_ops.py @@ -1,4 +1,5 @@ import getpass +import logging import os import shutil import stat @@ -165,9 +166,9 @@ def rmdirs(self, path, ignore_errors=True, retries=3, delay=1): except FileNotFoundError: return True except Exception as e: - print(f"Error: Failed to remove directory {path} on attempt {attempt + 1}: {e}") + logging.error(f"Error: Failed to remove directory {path} on attempt {attempt + 1}: {e}") time.sleep(delay) - print(f"Error: Failed to remove directory {path} after {retries} attempts.") + logging.error(f"Error: Failed to remove directory {path} after {retries} attempts.") return False def listdir(self, path): diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index fa031075..f85490ef 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -93,8 +93,10 @@ def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False, if not error: error_found = 0 else: + error = normalize_error(error) error_found = exit_status != 0 or any( - marker in error for marker in [b'error', b'Permission denied', b'fatal', b'No such file or directory']) + marker in error for marker in ['error', 'Permission denied', 'fatal', 'No such file or directory'] + ) if error_found: if isinstance(error, bytes): @@ -369,3 +371,9 @@ def db_connect(self, dbname, user, password=None, host="localhost", port=5432): password=password, ) return conn + + +def normalize_error(error): + if isinstance(error, bytes): + return error.decode() + return error diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/app.py b/testgres/plugins/pg_probackup2/pg_probackup2/app.py index ffad24d3..620dc563 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/app.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/app.py @@ -1,6 +1,7 @@ import contextlib import importlib import json +import logging import os import re import subprocess @@ -74,7 +75,7 @@ def run(self, command, gdb=False, old_binary=False, return_id=True, env=None, command = [command[0], *self.backup_dir.pb_args, *command[1:]] if not self.probackup_old_path and old_binary: - print('PGPROBACKUPBIN_OLD is not set') + logging.error('PGPROBACKUPBIN_OLD is not set') exit(1) if old_binary: @@ -107,12 +108,11 @@ def run(self, command, gdb=False, old_binary=False, return_id=True, env=None, return GDBobj(cmdline, self.test_class) try: - result = None if type(gdb) is tuple and gdb[0] == 'suspend': # special test flow for manually debug probackup gdb_port = gdb[1] cmdline = ['gdbserver'] + ['localhost:' + str(gdb_port)] + cmdline - print("pg_probackup gdb suspended, waiting gdb connection on localhost:{0}".format(gdb_port)) + logging.warning("pg_probackup gdb suspended, waiting gdb connection on localhost:{0}".format(gdb_port)) start_time = time.time() self.test_class.output = subprocess.check_output( @@ -233,7 +233,7 @@ def backup_node( if options is None: options = [] if not node and not data_dir: - print('You must provide ether node or data_dir for backup') + logging.error('You must provide ether node or data_dir for backup') exit(1) if not datname: @@ -502,7 +502,7 @@ def show( if i == '': backup_record_split.remove(i) if len(header_split) != len(backup_record_split): - print(warning.format( + logging.error(warning.format( header=header, body=body, header_split=header_split, body_split=backup_record_split) @@ -581,7 +581,7 @@ def show_archive( else: show_splitted = self.run(cmd_list + options, old_binary=old_binary, expect_error=expect_error).splitlines() - print(show_splitted) + logging.error(show_splitted) exit(1) def validate( @@ -769,7 +769,7 @@ def load_backup_class(fs_type): if fs_type: implementation = fs_type - print("Using ", implementation) + logging.info("Using ", implementation) module_name, class_name = implementation.rsplit(sep='.', maxsplit=1) module = importlib.import_module(module_name) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py index 73731a6e..2d19e980 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py @@ -1,3 +1,4 @@ +import logging from functools import reduce import getpass import os @@ -31,7 +32,7 @@ cached_initdb_dir=False, node_cleanup_full=delete_logs) except Exception as e: - print("Can't configure testgres: {0}".format(e)) + logging.warning("Can't configure testgres: {0}".format(e)) class Init(object): @@ -104,7 +105,7 @@ def __init__(self): if os.path.isfile(probackup_path_tmp): if not os.access(probackup_path_tmp, os.X_OK): - print('{0} is not an executable file'.format( + logging.warning('{0} is not an executable file'.format( probackup_path_tmp)) else: self.probackup_path = probackup_path_tmp @@ -114,13 +115,13 @@ def __init__(self): if os.path.isfile(probackup_path_tmp): if not os.access(probackup_path_tmp, os.X_OK): - print('{0} is not an executable file'.format( + logging.warning('{0} is not an executable file'.format( probackup_path_tmp)) else: self.probackup_path = probackup_path_tmp if not self.probackup_path: - print('pg_probackup binary is not found') + logging.error('pg_probackup binary is not found') exit(1) if os.name == 'posix': @@ -207,7 +208,7 @@ def __init__(self): if self.probackup_version.split('.')[0].isdigit(): self.major_version = int(self.probackup_version.split('.')[0]) else: - print('Can\'t process pg_probackup version \"{}\": the major version is expected to be a number'.format(self.probackup_version)) + logging.error('Can\'t process pg_probackup version \"{}\": the major version is expected to be a number'.format(self.probackup_version)) sys.exit(1) def test_env(self): diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/tests/basic_test.py b/testgres/plugins/pg_probackup2/pg_probackup2/tests/basic_test.py index f5a82d38..b63531ec 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/tests/basic_test.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/tests/basic_test.py @@ -1,3 +1,4 @@ +import logging import os import shutil import unittest @@ -14,7 +15,7 @@ def get_module_and_function_name(test_id): module_name = test_id.split('.')[-2] fname = test_id.split('.')[-1] except IndexError: - print(f"Couldn't get module name and function name from test_id: `{test_id}`") + logging.warning(f"Couldn't get module name and function name from test_id: `{test_id}`") module_name, fname = test_id.split('(')[1].split('.')[1], test_id.split('(')[0] return module_name, fname diff --git a/testgres/utils.py b/testgres/utils.py index 745a2555..a4ee7877 100644 --- a/testgres/utils.py +++ b/testgres/utils.py @@ -228,7 +228,6 @@ def eprint(*args, **kwargs): """ Print stuff to stderr. """ - print(*args, file=sys.stderr, **kwargs) From 5c7cf1808c0657f729da2e4e6a0867513905adfa Mon Sep 17 00:00:00 2001 From: egarbuz <165897130+egarbuz@users.noreply.github.com> Date: Mon, 15 Jul 2024 06:32:52 +0300 Subject: [PATCH 048/216] Use fog pg_probackup3 log level TRACE insted of VERBOSE and don't use -j and --batch-size keys (#138) --- testgres/plugins/pg_probackup2/pg_probackup2/app.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/app.py b/testgres/plugins/pg_probackup2/pg_probackup2/app.py index 620dc563..e656b66d 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/app.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/app.py @@ -705,9 +705,13 @@ def set_archiving( if overwrite: archive_command += ' --overwrite' - archive_command += ' --log-level-console=VERBOSE' - archive_command += ' -j 5' - archive_command += ' --batch-size 10' + if init_params.major_version > 2: + archive_command += ' --log-level-console=trace' + else: + archive_command += ' --log-level-console=VERBOSE' + archive_command += ' -j 5' + archive_command += ' --batch-size 10' + archive_command += ' --no-sync' if archive_timeout: From e1ed1589220a52d108644f30888e066142916ffc Mon Sep 17 00:00:00 2001 From: Victoria Shepard <5807469+demonolock@users.noreply.github.com> Date: Tue, 13 Aug 2024 17:40:22 +0200 Subject: [PATCH 049/216] Fix logger (#140) Co-authored-by: vshepard --- testgres/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testgres/config.py b/testgres/config.py index 63719f1d..67d467d3 100644 --- a/testgres/config.py +++ b/testgres/config.py @@ -13,7 +13,7 @@ from .operations.local_ops import LocalOperations log_level = os.getenv('LOGGING_LEVEL', 'WARNING').upper() -log_format = os.getenv('LOGGING_FORMAT', '%(asctime)s - %(levelname)s - %(message)s').upper() +log_format = os.getenv('LOGGING_FORMAT', '%(asctime)s - %(levelname)s - %(message)s') logging.basicConfig(level=log_level, format=log_format) From 8c193b2c203cda2480331c6c7c52d235ccad70d2 Mon Sep 17 00:00:00 2001 From: Victoria Shepard <5807469+demonolock@users.noreply.github.com> Date: Wed, 14 Aug 2024 22:49:33 +0200 Subject: [PATCH 050/216] Add force node stopping using SIGKILL in case of unsuccessful pg_ctl stop (#139) --- testgres/node.py | 23 +++++++++++++++++++++-- testgres/operations/local_ops.py | 4 ++-- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index 2ea49529..8b30476f 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -330,8 +330,9 @@ def version(self): """ return self._pg_version - def _try_shutdown(self, max_attempts): + def _try_shutdown(self, max_attempts, with_force=False): attempts = 0 + node_pid = self.pid # try stopping server N times while attempts < max_attempts: @@ -341,12 +342,30 @@ def _try_shutdown(self, max_attempts): except ExecUtilException: pass # one more time except Exception: - # TODO: probably should kill stray instance eprint('cannot stop node {}'.format(self.name)) break attempts += 1 + # If force stopping is enabled and PID is valid + if with_force and node_pid != 0: + # If we couldn't stop the node + p_status_output = self.os_ops.exec_command(cmd=f'ps -p {node_pid}', shell=True).decode('utf-8') + if self.status() != NodeStatus.Stopped and p_status_output and str(node_pid) in p_status_output: + try: + eprint(f'Force stopping node {self.name} with PID {node_pid}') + self.os_ops.kill(node_pid, signal.SIGKILL, expect_error=False) + except Exception: + # The node has already stopped + pass + + # Check that node stopped + p_status_output = self.os_ops.exec_command(f'ps -p {node_pid}', shell=True, expect_error=True).decode('utf-8') + if p_status_output and str(node_pid) in p_status_output: + eprint(f'Failed to stop node {self.name}.') + else: + eprint(f'Node {self.name} has been stopped successfully.') + def _assign_master(self, master): """NOTE: this is a private method!""" diff --git a/testgres/operations/local_ops.py b/testgres/operations/local_ops.py index b518a6cb..3d9e490e 100644 --- a/testgres/operations/local_ops.py +++ b/testgres/operations/local_ops.py @@ -293,10 +293,10 @@ def remove_file(self, filename): return os.remove(filename) # Processes control - def kill(self, pid, signal): + def kill(self, pid, signal, expect_error=False): # Kill the process cmd = "kill -{} {}".format(signal, pid) - return self.exec_command(cmd) + return self.exec_command(cmd, expect_error=expect_error) def get_pid(self): # Get current process id From 1d1d3f08e0349067e26d56d84f853b1941f481dc Mon Sep 17 00:00:00 2001 From: dura0ok Date: Fri, 16 Aug 2024 13:59:08 +0300 Subject: [PATCH 051/216] fix locale warning (#141) DeprecationWarning: 'locale.getdefaultlocale' is deprecated and slated for removal in Python 3.15. Use setlocale(), getencoding() and getlocale() instead. rewrite using getlocale --- testgres/operations/os_ops.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/testgres/operations/os_ops.py b/testgres/operations/os_ops.py index 76284049..34242040 100644 --- a/testgres/operations/os_ops.py +++ b/testgres/operations/os_ops.py @@ -20,7 +20,9 @@ def __init__(self, host='127.0.0.1', port=None, ssh_key=None, username=None): def get_default_encoding(): - return locale.getdefaultlocale()[1] or 'UTF-8' + if not hasattr(locale, 'getencoding'): + locale.getencoding = locale.getpreferredencoding + return locale.getencoding() or 'UTF-8' class OsOperations: From 98b028635f5c30d63c94ec07cfdd567fba36aa4c Mon Sep 17 00:00:00 2001 From: Victoria Shepard <5807469+demonolock@users.noreply.github.com> Date: Tue, 27 Aug 2024 09:41:41 +0200 Subject: [PATCH 052/216] Ignore error teardown (#142) --- testgres/node.py | 4 ++-- testgres/operations/local_ops.py | 5 +++-- testgres/operations/remote_ops.py | 4 ++-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index 8b30476f..1404f9cc 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -359,8 +359,8 @@ def _try_shutdown(self, max_attempts, with_force=False): # The node has already stopped pass - # Check that node stopped - p_status_output = self.os_ops.exec_command(f'ps -p {node_pid}', shell=True, expect_error=True).decode('utf-8') + # Check that node stopped - print only column pid without headers + p_status_output = self.os_ops.exec_command(f'ps -o pid= -p {node_pid}', shell=True, ignore_errors=True).decode('utf-8') if p_status_output and str(node_pid) in p_status_output: eprint(f'Failed to stop node {self.name}.') else: diff --git a/testgres/operations/local_ops.py b/testgres/operations/local_ops.py index 3d9e490e..a0a9926d 100644 --- a/testgres/operations/local_ops.py +++ b/testgres/operations/local_ops.py @@ -107,14 +107,15 @@ def _run_command(self, cmd, shell, input, stdin, stdout, stderr, get_process, ti raise ExecUtilException("Command timed out after {} seconds.".format(timeout)) def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False, encoding=None, shell=False, - text=False, input=None, stdin=None, stdout=None, stderr=None, get_process=False, timeout=None): + text=False, input=None, stdin=None, stdout=None, stderr=None, get_process=False, timeout=None, + ignore_errors=False): """ Execute a command in a subprocess and handle the output based on the provided parameters. """ process, output, error = self._run_command(cmd, shell, input, stdin, stdout, stderr, get_process, timeout, encoding) if get_process: return process - if (process.returncode != 0 or has_errors(output=output, error=error)) and not expect_error: + if not ignore_errors and ((process.returncode != 0 or has_errors(output=output, error=error)) and not expect_error): self._raise_exec_exception('Utility exited with non-zero code. Error `{}`', cmd, process.returncode, error or output) if verbose: diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index f85490ef..20095051 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -60,7 +60,7 @@ def __enter__(self): def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False, encoding=None, shell=True, text=False, input=None, stdin=None, stdout=None, - stderr=None, get_process=None, timeout=None): + stderr=None, get_process=None, timeout=None, ignore_errors=False): """ Execute a command in the SSH session. Args: @@ -98,7 +98,7 @@ def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False, marker in error for marker in ['error', 'Permission denied', 'fatal', 'No such file or directory'] ) - if error_found: + if not ignore_errors and error_found: if isinstance(error, bytes): message = b"Utility exited with non-zero code. Error: " + error else: From 78603f85d4332a7ea35877f5f7cb1967156150bd Mon Sep 17 00:00:00 2001 From: Victoria Shepard <5807469+demonolock@users.noreply.github.com> Date: Tue, 17 Sep 2024 19:35:43 +0200 Subject: [PATCH 053/216] Fix node.py: try_shutdown and cleanup (#144) --- testgres/node.py | 59 ++++++++++++++++++++++++------------------------ 1 file changed, 30 insertions(+), 29 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index 1404f9cc..5d95857f 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -334,37 +334,38 @@ def _try_shutdown(self, max_attempts, with_force=False): attempts = 0 node_pid = self.pid - # try stopping server N times - while attempts < max_attempts: - try: - self.stop() - break # OK - except ExecUtilException: - pass # one more time - except Exception: - eprint('cannot stop node {}'.format(self.name)) - break - - attempts += 1 - - # If force stopping is enabled and PID is valid - if with_force and node_pid != 0: - # If we couldn't stop the node - p_status_output = self.os_ops.exec_command(cmd=f'ps -p {node_pid}', shell=True).decode('utf-8') - if self.status() != NodeStatus.Stopped and p_status_output and str(node_pid) in p_status_output: + if node_pid > 0: + # try stopping server N times + while attempts < max_attempts: try: - eprint(f'Force stopping node {self.name} with PID {node_pid}') - self.os_ops.kill(node_pid, signal.SIGKILL, expect_error=False) + self.stop() + break # OK + except ExecUtilException: + pass # one more time except Exception: - # The node has already stopped - pass - - # Check that node stopped - print only column pid without headers - p_status_output = self.os_ops.exec_command(f'ps -o pid= -p {node_pid}', shell=True, ignore_errors=True).decode('utf-8') - if p_status_output and str(node_pid) in p_status_output: - eprint(f'Failed to stop node {self.name}.') - else: - eprint(f'Node {self.name} has been stopped successfully.') + eprint('cannot stop node {}'.format(self.name)) + break + + attempts += 1 + + # If force stopping is enabled and PID is valid + if with_force and node_pid != 0: + # If we couldn't stop the node + p_status_output = self.os_ops.exec_command(cmd=f'ps -p {node_pid}', shell=True).decode('utf-8') + if self.status() != NodeStatus.Stopped and p_status_output and str(node_pid) in p_status_output: + try: + eprint(f'Force stopping node {self.name} with PID {node_pid}') + self.os_ops.kill(node_pid, signal.SIGKILL, expect_error=False) + except Exception: + # The node has already stopped + pass + + # Check that node stopped - print only column pid without headers + p_status_output = self.os_ops.exec_command(f'ps -o pid= -p {node_pid}', shell=True, ignore_errors=True).decode('utf-8') + if p_status_output and str(node_pid) in p_status_output: + eprint(f'Failed to stop node {self.name}.') + else: + eprint(f'Node {self.name} has been stopped successfully.') def _assign_master(self, master): """NOTE: this is a private method!""" From 177724b0265c5977bfd6d190aab7c16f6972b3a0 Mon Sep 17 00:00:00 2001 From: Victoria Shepard <5807469+demonolock@users.noreply.github.com> Date: Fri, 20 Sep 2024 22:36:47 +0200 Subject: [PATCH 054/216] Fix node.py: try_shutdown and cleanup (#145) --- testgres/node.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testgres/node.py b/testgres/node.py index 5d95857f..b9bf9896 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -351,7 +351,7 @@ def _try_shutdown(self, max_attempts, with_force=False): # If force stopping is enabled and PID is valid if with_force and node_pid != 0: # If we couldn't stop the node - p_status_output = self.os_ops.exec_command(cmd=f'ps -p {node_pid}', shell=True).decode('utf-8') + p_status_output = self.os_ops.exec_command(cmd=f'ps -o pid= -p {node_pid}', shell=True, ignore_errors=True).decode('utf-8') if self.status() != NodeStatus.Stopped and p_status_output and str(node_pid) in p_status_output: try: eprint(f'Force stopping node {self.name} with PID {node_pid}') From 6db4520f3b2825ec71166a9ffa6cbe0c95ffefef Mon Sep 17 00:00:00 2001 From: Victoria Shepard <5807469+demonolock@users.noreply.github.com> Date: Sat, 26 Oct 2024 02:16:34 +0300 Subject: [PATCH 055/216] Fix test test_child_pids for PostgreSql 15 and higher (#146) --- tests/test_simple.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_simple.py b/tests/test_simple.py index ba23c3ed..43394718 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -957,6 +957,9 @@ def test_child_pids(self): if pg_version_ge('10'): master_processes.append(ProcessType.LogicalReplicationLauncher) + if pg_version_ge('14'): + master_processes.remove(ProcessType.StatsCollector) + repl_processes = [ ProcessType.Startup, ProcessType.WalReceiver, From 8a5c252ebc5dadd871bb92eec10959a8733167b4 Mon Sep 17 00:00:00 2001 From: Ekaterina Sokolova Date: Wed, 23 Oct 2024 14:57:06 +0300 Subject: [PATCH 056/216] Update configuration for Travis CI. --- .travis.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.travis.yml b/.travis.yml index c06cab3d..6f63a67b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,6 +20,9 @@ notifications: on_failure: always env: + - PYTHON_VERSION=3 PG_VERSION=17 + - PYTHON_VERSION=3 PG_VERSION=16 + - PYTHON_VERSION=3 PG_VERSION=15 - PYTHON_VERSION=3 PG_VERSION=14 - PYTHON_VERSION=3 PG_VERSION=13 - PYTHON_VERSION=3 PG_VERSION=12 @@ -32,3 +35,8 @@ env: # - PYTHON_VERSION=2 PG_VERSION=9.6 # - PYTHON_VERSION=2 PG_VERSION=9.5 # - PYTHON_VERSION=2 PG_VERSION=9.4 + +matrix: + allow_failures: + - env: PYTHON_VERSION=3 PG_VERSION=11 + - env: PYTHON_VERSION=3 PG_VERSION=10 From 569923a004165aa40530444b2154ad98e3f57cc7 Mon Sep 17 00:00:00 2001 From: asavchkov Date: Mon, 28 Oct 2024 17:10:17 +0700 Subject: [PATCH 057/216] Configure the unix socket directory --- testgres/node.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/testgres/node.py b/testgres/node.py index b9bf9896..cb1b8385 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -1751,7 +1751,8 @@ def make_simple( 'log_connections': 'on', 'log_disconnections': 'on', 'restart_after_crash': 'off', - 'autovacuum': 'off'} + 'autovacuum': 'off', + 'unix_socket_directories': '/tmp'} # Allow replication in pg_hba.conf if set_replication: From c0dca6bf8c9f8fa41ad076542ce9f4a0e68daa2d Mon Sep 17 00:00:00 2001 From: dura0ok Date: Mon, 18 Nov 2024 16:32:33 +0700 Subject: [PATCH 058/216] Add PG_PROBACKUP_WAL_TREE_ENABLED env param (#143) --- testgres/plugins/pg_probackup2/pg_probackup2/app.py | 4 ++++ testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py | 1 + 2 files changed, 5 insertions(+) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/app.py b/testgres/plugins/pg_probackup2/pg_probackup2/app.py index e656b66d..899a8092 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/app.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/app.py @@ -54,6 +54,7 @@ def __init__(self, test_class: unittest.TestCase, self.probackup_path = probackup_path or init_params.probackup_path self.probackup_old_path = init_params.probackup_old_path self.remote = init_params.remote + self.wal_tree_enabled = init_params.wal_tree_enabled self.verbose = init_params.verbose self.archive_compress = init_params.archive_compress self.test_class.output = None @@ -185,6 +186,9 @@ def add_instance(self, instance, node, old_binary=False, options=None, expect_er '--remote-proto=ssh', '--remote-host=localhost'] + if self.wal_tree_enabled: + options = options + ['--wal-tree'] + return self.run(cmd + options, old_binary=old_binary, expect_error=expect_error) def set_config(self, instance, old_binary=False, options=None, expect_error=False): diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py index 2d19e980..b7174a7c 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py @@ -170,6 +170,7 @@ def __init__(self): self.remote = test_env.get('PGPROBACKUP_SSH_REMOTE', None) == 'ON' self.ptrack = test_env.get('PG_PROBACKUP_PTRACK', None) == 'ON' and self.pg_config_version >= 110000 + self.wal_tree_enabled = test_env.get('PG_PROBACKUP_WAL_TREE_ENABLED', None) == 'ON' self.paranoia = test_env.get('PG_PROBACKUP_PARANOIA', None) == 'ON' env_compress = test_env.get('ARCHIVE_COMPRESSION', None) From a5cfdb65f56882a11cc38ddd408420467beac6e7 Mon Sep 17 00:00:00 2001 From: vshepard Date: Mon, 18 Nov 2024 11:57:51 +0100 Subject: [PATCH 059/216] Add bin_dir to make_simple --- testgres/node.py | 10 ++++++---- tests/test_simple.py | 19 ++++++++++++++++++- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index cb1b8385..c8c8c087 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -1707,12 +1707,13 @@ def __init__(self, test_path=None, nodes_to_cleanup=None, os_ops=LocalOperations def make_empty( self, base_dir=None, - port=None): + port=None, + bin_dir=None): real_base_dir = os.path.join(self.test_path, base_dir) self.os_ops.rmdirs(real_base_dir, ignore_errors=True) self.os_ops.makedirs(real_base_dir) - node = PostgresNode(base_dir=real_base_dir, port=port) + node = PostgresNode(base_dir=real_base_dir, port=port, bin_dir=bin_dir) node.should_rm_dirs = True self.nodes_to_cleanup.append(node) @@ -1726,10 +1727,11 @@ def make_simple( ptrack_enable=False, initdb_params=[], pg_options={}, - checksum=True): + checksum=True, + bin_dir=None): if checksum and '--data-checksums' not in initdb_params: initdb_params.append('--data-checksums') - node = self.make_empty(base_dir, port) + node = self.make_empty(base_dir, port, bin_dir=bin_dir) node.init( initdb_params=initdb_params, allow_streaming=set_replication) diff --git a/tests/test_simple.py b/tests/test_simple.py index 43394718..8f85a23b 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -23,7 +23,7 @@ BackupException, \ QueryException, \ TimeoutException, \ - TestgresException + TestgresException, NodeApp from testgres import \ TestgresConfig, \ @@ -1044,6 +1044,23 @@ def test_the_same_port(self): node2.port = node.port node2.init().start() + def test_make_simple_with_bin_dir(self): + with get_new_node() as node: + node.init().start() + bin_dir = node.bin_dir + + app = NodeApp() + correct_bin_dir = app.make_simple(base_dir=node.base_dir, bin_dir=bin_dir) + correct_bin_dir.slow_start() + correct_bin_dir.safe_psql("SELECT 1;") + + try: + wrong_bin_dir = app.make_empty(base_dir=node.base_dir, bin_dir="wrong/path") + wrong_bin_dir.slow_start() + raise RuntimeError("Error was expected.") # We should not reach this + except FileNotFoundError: + pass # Expected error + if __name__ == '__main__': if os.environ.get('ALT_CONFIG'): From 1cd6cd72ae761ec0b9f34ef8477c8b925de36965 Mon Sep 17 00:00:00 2001 From: Victoria Shepard <5807469+demonolock@users.noreply.github.com> Date: Mon, 18 Nov 2024 16:22:27 +0100 Subject: [PATCH 060/216] Fix style in test_simple.py (#150) --- tests/test_simple.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_simple.py b/tests/test_simple.py index 8f85a23b..62de1df5 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -1044,7 +1044,7 @@ def test_the_same_port(self): node2.port = node.port node2.init().start() - def test_make_simple_with_bin_dir(self): + def test_simple_with_bin_dir(self): with get_new_node() as node: node.init().start() bin_dir = node.bin_dir @@ -1059,7 +1059,7 @@ def test_make_simple_with_bin_dir(self): wrong_bin_dir.slow_start() raise RuntimeError("Error was expected.") # We should not reach this except FileNotFoundError: - pass # Expected error + pass # Expected error if __name__ == '__main__': From 5ae340c9b95430ac9065d58bb1d09882fddf7c30 Mon Sep 17 00:00:00 2001 From: Kian-Meng Ang Date: Wed, 20 Nov 2024 19:55:47 +0800 Subject: [PATCH 061/216] Fix typos (#152) Found via `codespell -L splitted` --- README.md | 4 ++-- testgres/node.py | 4 ++-- testgres/plugins/pg_probackup2/pg_probackup2/gdb.py | 2 +- tests/test_simple.py | 8 ++++---- tests/test_simple_remote.py | 8 ++++---- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index a2a0ec7e..f0071a90 100644 --- a/README.md +++ b/README.md @@ -59,12 +59,12 @@ with testgres.get_new_node() as node: # ... node stops and its files are about to be removed ``` -There are four API methods for runnig queries: +There are four API methods for running queries: | Command | Description | |----------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------| | `node.psql(query, ...)` | Runs query via `psql` command and returns tuple `(error code, stdout, stderr)`. | -| `node.safe_psql(query, ...)` | Same as `psql()` except that it returns only `stdout`. If an error occures during the execution, an exception will be thrown. | +| `node.safe_psql(query, ...)` | Same as `psql()` except that it returns only `stdout`. If an error occurs during the execution, an exception will be thrown. | | `node.execute(query, ...)` | Connects to PostgreSQL using `psycopg2` or `pg8000` (depends on which one is installed in your system) and returns two-dimensional array with data. | | `node.connect(dbname, ...)` | Returns connection wrapper (`NodeConnection`) capable of running several queries within a single transaction. | diff --git a/testgres/node.py b/testgres/node.py index c8c8c087..4ae30908 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -1143,7 +1143,7 @@ def restore(self, filename, dbname=None, username=None): filename ] # yapf: disable - # try pg_restore if dump is binary formate, and psql if not + # try pg_restore if dump is binary format, and psql if not try: execute_utility(_params, self.utils_log_name) except ExecUtilException: @@ -1286,7 +1286,7 @@ def set_synchronous_standbys(self, standbys): Args: standbys: either :class:`.First` or :class:`.Any` object specifying - sychronization parameters or just a plain list of + synchronization parameters or just a plain list of :class:`.PostgresNode`s replicas which would be equivalent to passing ``First(1, )``. For PostgreSQL 9.5 and below it is only possible to specify a plain list of standbys as diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/gdb.py b/testgres/plugins/pg_probackup2/pg_probackup2/gdb.py index 0b61da65..ceb1f6a9 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/gdb.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/gdb.py @@ -37,7 +37,7 @@ def __init__(self, cmd, env, attach=False): " to run GDB tests") raise GdbException("No gdb usage possible.") - # Check gdb presense + # Check gdb presence try: gdb_version, _ = subprocess.Popen( ['gdb', '--version'], diff --git a/tests/test_simple.py b/tests/test_simple.py index 62de1df5..41203a65 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -501,7 +501,7 @@ def test_logical_replication(self): sub.disable() node1.safe_psql('insert into test values (3, 3)') - # enable and ensure that data successfully transfered + # enable and ensure that data successfully transferred sub.enable() sub.catchup() res = node2.execute('select * from test') @@ -509,7 +509,7 @@ def test_logical_replication(self): # Add new tables. Since we added "all tables" to publication # (default behaviour of publish() method) we don't need - # to explicitely perform pub.add_tables() + # to explicitly perform pub.add_tables() create_table = 'create table test2 (c char)' node1.safe_psql(create_table) node2.safe_psql(create_table) @@ -526,7 +526,7 @@ def test_logical_replication(self): pub.drop() # create new publication and subscription for specific table - # (ommitting copying data as it's already done) + # (omitting copying data as it's already done) pub = node1.publish('newpub', tables=['test']) sub = node2.subscribe(pub, 'newsub', copy_data=False) @@ -535,7 +535,7 @@ def test_logical_replication(self): res = node2.execute('select * from test') self.assertListEqual(res, [(1, 1), (2, 2), (3, 3), (4, 4)]) - # explicitely add table + # explicitly add table with self.assertRaises(ValueError): pub.add_tables([]) # fail pub.add_tables(['test2']) diff --git a/tests/test_simple_remote.py b/tests/test_simple_remote.py index d51820ba..79bdb74c 100755 --- a/tests/test_simple_remote.py +++ b/tests/test_simple_remote.py @@ -480,7 +480,7 @@ def test_logical_replication(self): sub.disable() node1.safe_psql('insert into test values (3, 3)') - # enable and ensure that data successfully transfered + # enable and ensure that data successfully transferred sub.enable() sub.catchup() res = node2.execute('select * from test') @@ -488,7 +488,7 @@ def test_logical_replication(self): # Add new tables. Since we added "all tables" to publication # (default behaviour of publish() method) we don't need - # to explicitely perform pub.add_tables() + # to explicitly perform pub.add_tables() create_table = 'create table test2 (c char)' node1.safe_psql(create_table) node2.safe_psql(create_table) @@ -505,7 +505,7 @@ def test_logical_replication(self): pub.drop() # create new publication and subscription for specific table - # (ommitting copying data as it's already done) + # (omitting copying data as it's already done) pub = node1.publish('newpub', tables=['test']) sub = node2.subscribe(pub, 'newsub', copy_data=False) @@ -514,7 +514,7 @@ def test_logical_replication(self): res = node2.execute('select * from test') self.assertListEqual(res, [(1, 1), (2, 2), (3, 3), (4, 4)]) - # explicitely add table + # explicitly add table with self.assertRaises(ValueError): pub.add_tables([]) # fail pub.add_tables(['test2']) From 644eea8cd6c2e85536a1d18a5a8b6e5d0b949bce Mon Sep 17 00:00:00 2001 From: Vyacheslav Makarov <50846161+MakSl@users.noreply.github.com> Date: Wed, 20 Nov 2024 14:56:33 +0300 Subject: [PATCH 062/216] Refinement of the run function in app.py (#148) --- testgres/plugins/pg_probackup2/pg_probackup2/app.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/app.py b/testgres/plugins/pg_probackup2/pg_probackup2/app.py index 899a8092..57492814 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/app.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/app.py @@ -74,6 +74,8 @@ def run(self, command, gdb=False, old_binary=False, return_id=True, env=None, command = [command[0], *use_backup_dir.pb_args, *command[1:]] elif use_backup_dir: command = [command[0], *self.backup_dir.pb_args, *command[1:]] + else: + command = [command[0], *self.backup_dir.pb_args[2:], *command[1:]] if not self.probackup_old_path and old_binary: logging.error('PGPROBACKUPBIN_OLD is not set') From b5855e417cdba94766158b0d1fe23c90b717a967 Mon Sep 17 00:00:00 2001 From: MetalDream666 <61190185+MetalDream666@users.noreply.github.com> Date: Fri, 22 Nov 2024 12:30:39 +0300 Subject: [PATCH 063/216] Fix broken pipe from gdb after killing process (#151) --- testgres/plugins/pg_probackup2/pg_probackup2/gdb.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/gdb.py b/testgres/plugins/pg_probackup2/pg_probackup2/gdb.py index ceb1f6a9..2424c04d 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/gdb.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/gdb.py @@ -108,6 +108,9 @@ def kill(self): self.proc.stdin.close() self.proc.stdout.close() + def terminate_subprocess(self): + self._execute('kill') + def set_breakpoint(self, location): result = self._execute('break ' + location) From 655a386117c995618720f4eb0ec7cbc990221343 Mon Sep 17 00:00:00 2001 From: asavchkov Date: Fri, 29 Nov 2024 19:00:07 +0700 Subject: [PATCH 064/216] Up version --- setup.py | 2 +- testgres/plugins/pg_probackup2/setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 412e8823..a41094d6 100755 --- a/setup.py +++ b/setup.py @@ -27,7 +27,7 @@ readme = f.read() setup( - version='1.10.1', + version='1.10.3', name='testgres', packages=['testgres', 'testgres.operations', 'testgres.helpers'], description='Testing utility for PostgreSQL and its extensions', diff --git a/testgres/plugins/pg_probackup2/setup.py b/testgres/plugins/pg_probackup2/setup.py index 2ff5a503..ade2d85d 100644 --- a/testgres/plugins/pg_probackup2/setup.py +++ b/testgres/plugins/pg_probackup2/setup.py @@ -4,7 +4,7 @@ from distutils.core import setup setup( - version='0.0.3', + version='0.0.4', name='testgres_pg_probackup2', packages=['pg_probackup2', 'pg_probackup2.storage'], description='Plugin for testgres that manages pg_probackup2', From 2c26debe967920e0f2a74d430fe6410a8a9a0513 Mon Sep 17 00:00:00 2001 From: vshepard Date: Tue, 3 Dec 2024 01:30:03 +0100 Subject: [PATCH 065/216] Fix set_auto_conf with single quotes --- testgres/node.py | 13 +++++++++---- tests/test_simple.py | 27 +++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 4 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index 4ae30908..be5019a4 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -1627,17 +1627,22 @@ def set_auto_conf(self, options, config='postgresql.auto.conf', rm_options={}): name, var = line.partition('=')[::2] name = name.strip() var = var.strip() - var = var.strip('"') - var = var.strip("'") - # remove options specified in rm_options list + # Handle quoted values and remove escaping + if var.startswith("'") and var.endswith("'"): + var = var[1:-1].replace("''", "'") + + # Remove options specified in rm_options list if name in rm_options: continue current_options[name] = var for option in options: - current_options[option] = options[option] + value = options[option] + if isinstance(value, str): + value = value.replace("'", "\\'") + current_options[option] = value auto_conf = '' for option in current_options: diff --git a/tests/test_simple.py b/tests/test_simple.py index 41203a65..a11d7932 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -1061,6 +1061,33 @@ def test_simple_with_bin_dir(self): except FileNotFoundError: pass # Expected error + def test_set_auto_conf(self): + with get_new_node() as node: + node.init().start() + + options = { + "archive_command": "cp '%p' \"/mnt/server/archivedir/%f\"", + 'restore_command': 'cp "/mnt/server/archivedir/%f" \'%p\'', + } + + node.set_auto_conf(options) + node.stop() + node.slow_start() + + auto_conf_path = f"{node.data_dir}/postgresql.auto.conf" + with open(auto_conf_path, "r") as f: + content = f.read() + self.assertIn( + "archive_command = 'cp \\'%p\\' \"/mnt/server/archivedir/%f\"", + content, + "archive_command stored wrong" + ) + self.assertIn( + "restore_command = 'cp \"/mnt/server/archivedir/%f\" \\'%p\\''", + content, + "restore_command stored wrong" + ) + if __name__ == '__main__': if os.environ.get('ALT_CONFIG'): From a4092af44fae5a1a92e9c9fbfec449fd99c8e520 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 3 Dec 2024 16:50:47 +0300 Subject: [PATCH 066/216] Node.set_auto_conf is improved - we do not touch existing values - escaping of '\n', '\r', '\t', '\b' and '\\' is added - translation of bool into 'on|off' is added test_set_auto_conf is updated. --- testgres/node.py | 39 +++++++++++++++++++++++++++--------- tests/test_simple.py | 47 +++++++++++++++++++++++++++++++------------- 2 files changed, 63 insertions(+), 23 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index be5019a4..7469b0d6 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -1626,11 +1626,6 @@ def set_auto_conf(self, options, config='postgresql.auto.conf', rm_options={}): name, var = line.partition('=')[::2] name = name.strip() - var = var.strip() - - # Handle quoted values and remove escaping - if var.startswith("'") and var.endswith("'"): - var = var[1:-1].replace("''", "'") # Remove options specified in rm_options list if name in rm_options: @@ -1640,14 +1635,18 @@ def set_auto_conf(self, options, config='postgresql.auto.conf', rm_options={}): for option in options: value = options[option] - if isinstance(value, str): - value = value.replace("'", "\\'") + valueType = type(value) + + if valueType == str: + value = __class__._escape_config_value(value) + elif valueType == bool: + value = "on" if value else "off" + current_options[option] = value auto_conf = '' for option in current_options: - auto_conf += "{0} = '{1}'\n".format( - option, current_options[option]) + auto_conf += option + " = " + str(current_options[option]) + "\n" for directive in current_directives: auto_conf += directive + "\n" @@ -1695,6 +1694,28 @@ def _get_bin_path(self, filename): bin_path = get_bin_path(filename) return bin_path + def _escape_config_value(value): + result = "'" + + for ch in value: + if (ch == "'"): + result += "\\'" + elif (ch == "\n"): + result += "\\n" + elif (ch == "\r"): + result += "\\r" + elif (ch == "\t"): + result += "\\t" + elif (ch == "\b"): + result += "\\b" + elif (ch == "\\"): + result += "\\\\" + else: + result += ch + + result += "'" + return result + class NodeApp: diff --git a/tests/test_simple.py b/tests/test_simple.py index a11d7932..ffefda6c 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -1062,13 +1062,35 @@ def test_simple_with_bin_dir(self): pass # Expected error def test_set_auto_conf(self): + # elements contain [property id, value, storage value] + testData = [ + ["archive_command", + "cp '%p' \"/mnt/server/archivedir/%f\"", + "'cp \\'%p\\' \"/mnt/server/archivedir/%f\""], + ["restore_command", + 'cp "/mnt/server/archivedir/%f" \'%p\'', + "'cp \"/mnt/server/archivedir/%f\" \\'%p\\''"], + ["log_line_prefix", + "'\n\r\t\b\\\"", + "'\\\'\\n\\r\\t\\b\\\\\""], + ["log_connections", + True, + "on"], + ["log_disconnections", + False, + "off"], + ["autovacuum_max_workers", + 3, + "3"] + ] + with get_new_node() as node: node.init().start() - options = { - "archive_command": "cp '%p' \"/mnt/server/archivedir/%f\"", - 'restore_command': 'cp "/mnt/server/archivedir/%f" \'%p\'', - } + options = {} + + for x in testData: + options[x[0]] = x[1] node.set_auto_conf(options) node.stop() @@ -1077,16 +1099,13 @@ def test_set_auto_conf(self): auto_conf_path = f"{node.data_dir}/postgresql.auto.conf" with open(auto_conf_path, "r") as f: content = f.read() - self.assertIn( - "archive_command = 'cp \\'%p\\' \"/mnt/server/archivedir/%f\"", - content, - "archive_command stored wrong" - ) - self.assertIn( - "restore_command = 'cp \"/mnt/server/archivedir/%f\" \\'%p\\''", - content, - "restore_command stored wrong" - ) + + for x in testData: + self.assertIn( + x[0] + " = " + x[2], + content, + x[0] + " stored wrong" + ) if __name__ == '__main__': From ee7dc91f2f56d80ca975f091c4dd60a004770177 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 3 Dec 2024 19:01:12 +0300 Subject: [PATCH 067/216] Asserts in PostgresNode.set_auto_conf are added Let's control a correct usage of this function. Plus one assert was added in _escape_config_value. --- testgres/node.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/testgres/node.py b/testgres/node.py index 7469b0d6..74a18cdf 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -1634,6 +1634,10 @@ def set_auto_conf(self, options, config='postgresql.auto.conf', rm_options={}): current_options[name] = var for option in options: + assert type(option) == str + assert option != "" + assert option.strip() == option + value = options[option] valueType = type(value) @@ -1695,6 +1699,8 @@ def _get_bin_path(self, filename): return bin_path def _escape_config_value(value): + assert type(value) == str + result = "'" for ch in value: From 9dd8dfec44ef56f732761d31c94cf98764a820a8 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 3 Dec 2024 21:11:00 +0300 Subject: [PATCH 068/216] Flake8 E721 is suppressed --- testgres/node.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index 74a18cdf..d9fb0604 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -1634,7 +1634,7 @@ def set_auto_conf(self, options, config='postgresql.auto.conf', rm_options={}): current_options[name] = var for option in options: - assert type(option) == str + assert type(option) == str # noqa: E721 assert option != "" assert option.strip() == option @@ -1699,7 +1699,7 @@ def _get_bin_path(self, filename): return bin_path def _escape_config_value(value): - assert type(value) == str + assert type(value) == str # noqa: E721 result = "'" From 1335e517e6d8bc16f2d628446c02a03078c5957e Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Wed, 4 Dec 2024 10:00:58 +0300 Subject: [PATCH 069/216] PostgresNode::_escape_config_value is updated (code style) --- testgres/node.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index d9fb0604..1706de11 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -1704,17 +1704,17 @@ def _escape_config_value(value): result = "'" for ch in value: - if (ch == "'"): + if ch == "'": result += "\\'" - elif (ch == "\n"): + elif ch == "\n": result += "\\n" - elif (ch == "\r"): + elif ch == "\r": result += "\\r" - elif (ch == "\t"): + elif ch == "\t": result += "\\t" - elif (ch == "\b"): + elif ch == "\b": result += "\\b" - elif (ch == "\\"): + elif ch == "\\": result += "\\\\" else: result += ch From 6acbeb6517b6b4aac4d23f0a137821ddb6b7928f Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Fri, 6 Dec 2024 12:32:55 +0300 Subject: [PATCH 070/216] NodeApp::make_simple is refactored (tempfile.gettempdir) - [BUG FIX] Windows does not have "/tmp" directory. Let's use tempfile.gettempdir() - Aggregation of standard and custom options to avoid two calls of node.set_auto_conf --- testgres/node.py | 69 +++++++++++++++++++++++++++++++++++------------- 1 file changed, 51 insertions(+), 18 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index 1706de11..0e5bb866 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -5,6 +5,7 @@ import signal import subprocess import threading +import tempfile from queue import Queue import time @@ -1761,6 +1762,8 @@ def make_simple( pg_options={}, checksum=True, bin_dir=None): + assert type(pg_options) == dict # noqa: E721 + if checksum and '--data-checksums' not in initdb_params: initdb_params.append('--data-checksums') node = self.make_empty(base_dir, port, bin_dir=bin_dir) @@ -1773,20 +1776,22 @@ def make_simple( node.major_version = float(node.major_version_str) # Set default parameters - options = {'max_connections': 100, - 'shared_buffers': '10MB', - 'fsync': 'off', - 'wal_level': 'logical', - 'hot_standby': 'off', - 'log_line_prefix': '%t [%p]: [%l-1] ', - 'log_statement': 'none', - 'log_duration': 'on', - 'log_min_duration_statement': 0, - 'log_connections': 'on', - 'log_disconnections': 'on', - 'restart_after_crash': 'off', - 'autovacuum': 'off', - 'unix_socket_directories': '/tmp'} + options = { + 'max_connections': 100, + 'shared_buffers': '10MB', + 'fsync': 'off', + 'wal_level': 'logical', + 'hot_standby': 'off', + 'log_line_prefix': '%t [%p]: [%l-1] ', + 'log_statement': 'none', + 'log_duration': 'on', + 'log_min_duration_statement': 0, + 'log_connections': 'on', + 'log_disconnections': 'on', + 'restart_after_crash': 'off', + 'autovacuum': 'off', + # 'unix_socket_directories': '/tmp', + } # Allow replication in pg_hba.conf if set_replication: @@ -1801,11 +1806,16 @@ def make_simple( else: options['wal_keep_segments'] = '12' - # set default values - node.set_auto_conf(options) - # Apply given parameters - node.set_auto_conf(pg_options) + for x in pg_options: + options[x] = pg_options[x] + + # Define delayed propertyes + if not ("unix_socket_directories" in options.keys()): + options["unix_socket_directories"] = __class__._gettempdir() + + # Set config values + node.set_auto_conf(options) # kludge for testgres # https://github.com/postgrespro/testgres/issues/54 @@ -1814,3 +1824,26 @@ def make_simple( node.set_auto_conf({}, 'postgresql.conf', ['wal_keep_segments']) return node + + def _gettempdir(): + v = tempfile.gettempdir() + + # + # Paranoid checks + # + if type(v) != str: # noqa: E721 + __class__._raise_bugcheck("tempfile.gettempdir returned a value with type {0}.".format(type(v).__name__)) + + if v == "": + __class__._raise_bugcheck("tempfile.gettempdir returned an empty string.") + + if not os.path.exists(v): + __class__._raise_bugcheck("tempfile.gettempdir returned a not exist path [{0}].".format(v)) + + # OK + return v + + def _raise_bugcheck(msg): + assert type(msg) == str # noqa: E721 + assert msg != "" + raise Exception("[BUG CHECK] " + msg) From 9d6f61472817c47477561911c7361d713a1fb1ca Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Fri, 6 Dec 2024 15:22:16 +0300 Subject: [PATCH 071/216] NodeApp::make_simple is updated [comment] --- testgres/node.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testgres/node.py b/testgres/node.py index 0e5bb866..62509790 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -1790,7 +1790,7 @@ def make_simple( 'log_disconnections': 'on', 'restart_after_crash': 'off', 'autovacuum': 'off', - # 'unix_socket_directories': '/tmp', + # unix_socket_directories will be defined later } # Allow replication in pg_hba.conf From 1d44628c65ad9266f303ecbb21080ed790ff01d0 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Fri, 6 Dec 2024 15:47:20 +0300 Subject: [PATCH 072/216] NodeApp::make_simple uses iteritems(pg_options) --- testgres/node.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index 62509790..48a100a9 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -1807,8 +1807,8 @@ def make_simple( options['wal_keep_segments'] = '12' # Apply given parameters - for x in pg_options: - options[x] = pg_options[x] + for option_name, option_value in iteritems(pg_options): + options[option_name] = option_value # Define delayed propertyes if not ("unix_socket_directories" in options.keys()): From d07104f8885f0d0c9332a608a32fc915017ff891 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Fri, 6 Dec 2024 17:43:41 +0300 Subject: [PATCH 073/216] LocalOperations::_run_command is refactored LocalOperations::_run_command delegates its work into two new methods: - _run_command__nt - _run_command__generic --- testgres/operations/local_ops.py | 76 ++++++++++++++++++-------------- 1 file changed, 42 insertions(+), 34 deletions(-) diff --git a/testgres/operations/local_ops.py b/testgres/operations/local_ops.py index a0a9926d..5b7972ae 100644 --- a/testgres/operations/local_ops.py +++ b/testgres/operations/local_ops.py @@ -64,47 +64,55 @@ def _process_output(encoding, temp_file_path): output = output.decode(encoding) return output, None # In Windows stderr writing in stdout - def _run_command(self, cmd, shell, input, stdin, stdout, stderr, get_process, timeout, encoding): - """Execute a command and return the process and its output.""" - if os.name == 'nt' and stdout is None: # Windows - with tempfile.NamedTemporaryFile(mode='w+b', delete=False) as temp_file: - stdout = temp_file - stderr = subprocess.STDOUT - process = subprocess.Popen( - cmd, - shell=shell, - stdin=stdin or subprocess.PIPE if input is not None else None, - stdout=stdout, - stderr=stderr, - ) - if get_process: - return process, None, None - temp_file_path = temp_file.name - - # Wait process finished - process.wait() - - output, error = self._process_output(encoding, temp_file_path) - return process, output, error - else: # Other OS + def _run_command__nt(self, cmd, shell, input, stdin, stdout, stderr, get_process, timeout, encoding): + with tempfile.NamedTemporaryFile(mode='w+b', delete=False) as temp_file: + stdout = temp_file + stderr = subprocess.STDOUT process = subprocess.Popen( cmd, shell=shell, stdin=stdin or subprocess.PIPE if input is not None else None, - stdout=stdout or subprocess.PIPE, - stderr=stderr or subprocess.PIPE, + stdout=stdout, + stderr=stderr, ) if get_process: return process, None, None - try: - output, error = process.communicate(input=input.encode(encoding) if input else None, timeout=timeout) - if encoding: - output = output.decode(encoding) - error = error.decode(encoding) - return process, output, error - except subprocess.TimeoutExpired: - process.kill() - raise ExecUtilException("Command timed out after {} seconds.".format(timeout)) + temp_file_path = temp_file.name + + # Wait process finished + process.wait() + + output, error = self._process_output(encoding, temp_file_path) + return process, output, error + + def _run_command__generic(self, cmd, shell, input, stdin, stdout, stderr, get_process, timeout, encoding): + process = subprocess.Popen( + cmd, + shell=shell, + stdin=stdin or subprocess.PIPE if input is not None else None, + stdout=stdout or subprocess.PIPE, + stderr=stderr or subprocess.PIPE, + ) + if get_process: + return process, None, None + try: + output, error = process.communicate(input=input.encode(encoding) if input else None, timeout=timeout) + if encoding: + output = output.decode(encoding) + error = error.decode(encoding) + return process, output, error + except subprocess.TimeoutExpired: + process.kill() + raise ExecUtilException("Command timed out after {} seconds.".format(timeout)) + + def _run_command(self, cmd, shell, input, stdin, stdout, stderr, get_process, timeout, encoding): + """Execute a command and return the process and its output.""" + if os.name == 'nt' and stdout is None: # Windows + method = __class__._run_command__nt + else: # Other OS + method = __class__._run_command__generic + + return method(self, cmd, shell, input, stdin, stdout, stderr, get_process, timeout, encoding) def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False, encoding=None, shell=False, text=False, input=None, stdin=None, stdout=None, stderr=None, get_process=False, timeout=None, From cf1d227d7fee878247a26f7f1b8636464d52c576 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Sat, 7 Dec 2024 21:40:41 +0300 Subject: [PATCH 074/216] Local and remote test code formatting has been synchronized test_simple.py and test_simple_remote.py have been compared and synchronized. --- tests/test_simple.py | 7 ------- tests/test_simple_remote.py | 2 +- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/tests/test_simple.py b/tests/test_simple.py index ffefda6c..51cdc896 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -153,7 +153,6 @@ def test_init_unique_system_id(self): with scoped_config(cache_initdb=True, cached_initdb_unique=True) as config: - self.assertTrue(config.cache_initdb) self.assertTrue(config.cached_initdb_unique) @@ -376,13 +375,11 @@ def test_backup_multiple(self): with node.backup(xlog_method='fetch') as backup1, \ node.backup(xlog_method='fetch') as backup2: - self.assertNotEqual(backup1.base_dir, backup2.base_dir) with node.backup(xlog_method='fetch') as backup: with backup.spawn_primary('node1', destroy=False) as node1, \ backup.spawn_primary('node2', destroy=False) as node2: - self.assertNotEqual(node1.base_dir, node2.base_dir) def test_backup_exhaust(self): @@ -390,7 +387,6 @@ def test_backup_exhaust(self): node.init(allow_streaming=True).start() with node.backup(xlog_method='fetch') as backup: - # exhaust backup by creating new node with backup.spawn_primary(): pass @@ -778,7 +774,6 @@ def test_pg_config(self): # modify setting for this scope with scoped_config(cache_pg_config=False) as config: - # sanity check for value self.assertFalse(config.cache_pg_config) @@ -810,7 +805,6 @@ def test_config_stack(self): self.assertEqual(c1.cached_initdb_dir, d1) with scoped_config(cached_initdb_dir=d2) as c2: - stack_size = len(testgres.config.config_stack) # try to break a stack @@ -840,7 +834,6 @@ def test_unix_sockets(self): def test_auto_name(self): with get_new_node().init(allow_streaming=True).start() as m: with m.replicate().start() as r: - # check that nodes are running self.assertTrue(m.status()) self.assertTrue(r.status()) diff --git a/tests/test_simple_remote.py b/tests/test_simple_remote.py index 79bdb74c..936c31f2 100755 --- a/tests/test_simple_remote.py +++ b/tests/test_simple_remote.py @@ -94,7 +94,6 @@ def removing(f): class TestgresRemoteTests(unittest.TestCase): - def test_node_repr(self): with get_remote_node(conn_params=conn_params) as node: pattern = r"PostgresNode\(name='.+', port=.+, base_dir='.+'\)" @@ -748,6 +747,7 @@ def test_pg_config(self): # save right before config change c1 = get_pg_config() + # modify setting for this scope with scoped_config(cache_pg_config=False) as config: # sanity check for value From 22c649dcaa8129bc240d49b24f2e363832e70d9a Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Sat, 7 Dec 2024 23:14:00 +0300 Subject: [PATCH 075/216] test_local.py and test_exec_command_failure__expect_error are added test_local.py contains set of test for LocalOperations - test_exec_command_success - test_exec_command_failure - test_exec_command_failure__expect_error Changes in TestRemoteOperations: - test_exec_command_failure exptects an exception - new test test_exec_command_failure__expect_error was added TestRemoteOperations::test_exec_command_failure__expect_error will fail because RemoteOperations::exec_command does not handle the 'expect_error' parameter correctly. --- tests/test_local.py | 46 ++++++++++++++++++++++++++++++++++++++++++++ tests/test_remote.py | 23 ++++++++++++++++++---- 2 files changed, 65 insertions(+), 4 deletions(-) create mode 100644 tests/test_local.py diff --git a/tests/test_local.py b/tests/test_local.py new file mode 100644 index 00000000..1caba74b --- /dev/null +++ b/tests/test_local.py @@ -0,0 +1,46 @@ +import pytest + +from testgres import ExecUtilException +from testgres import LocalOperations + + +class TestLocalOperations: + + @pytest.fixture(scope="function", autouse=True) + def setup(self): + self.operations = LocalOperations() + + def test_exec_command_success(self): + """ + Test exec_command for successful command execution. + """ + cmd = "python3 --version" + response = self.operations.exec_command(cmd, wait_exit=True, shell=True) + + assert b'Python 3.' in response + + def test_exec_command_failure(self): + """ + Test exec_command for command execution failure. + """ + cmd = "nonexistent_command" + while True: + try: + self.operations.exec_command(cmd, wait_exit=True, shell=True) + except ExecUtilException as e: + error = e.message + break + raise Exception("We wait an exception!") + assert error == "Utility exited with non-zero code. Error `b'/bin/sh: 1: nonexistent_command: not found\\n'`" + + def test_exec_command_failure__expect_error(self): + """ + Test exec_command for command execution failure. + """ + cmd = "nonexistent_command" + + exit_status, result, error = self.operations.exec_command(cmd, verbose=True, wait_exit=True, shell=True, expect_error=True) + + assert error == b'/bin/sh: 1: nonexistent_command: not found\n' + assert exit_status == 127 + assert result == b'' diff --git a/tests/test_remote.py b/tests/test_remote.py index e0e4a555..565163f7 100755 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -30,12 +30,27 @@ def test_exec_command_failure(self): Test exec_command for command execution failure. """ cmd = "nonexistent_command" - try: - exit_status, result, error = self.operations.exec_command(cmd, verbose=True, wait_exit=True) - except ExecUtilException as e: - error = e.message + while True: + try: + self.operations.exec_command(cmd, verbose=True, wait_exit=True) + except ExecUtilException as e: + error = e.message + break + raise Exception("We wait an exception!") assert error == b'Utility exited with non-zero code. Error: bash: line 1: nonexistent_command: command not found\n' + def test_exec_command_failure__expect_error(self): + """ + Test exec_command for command execution failure. + """ + cmd = "nonexistent_command" + + exit_status, result, error = self.operations.exec_command(cmd, verbose=True, wait_exit=True, shell=True, expect_error=True) + + assert error == b'bash: line 1: nonexistent_command: command not found\n' + assert exit_status == 127 + assert result == b'' + def test_is_executable_true(self): """ Test is_executable for an existing executable. From 45cfbf738e1f5d7f46f8ab053f4a6a481abf53e7 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Sat, 7 Dec 2024 23:26:04 +0300 Subject: [PATCH 076/216] RemoteOperations::exec_command must not raise an exception when 'expect_error' is True (#159) This commit fixes an issue #159. --- testgres/operations/remote_ops.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index 20095051..88394eb7 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -83,26 +83,26 @@ def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False, exit_status = process.returncode - if encoding: - result = result.decode(encoding) - error = error.decode(encoding) - - if expect_error: - raise Exception(result, error) + assert type(result) == bytes # noqa: E721 + assert type(error) == bytes # noqa: E721 if not error: - error_found = 0 + error_found = False else: - error = normalize_error(error) error_found = exit_status != 0 or any( - marker in error for marker in ['error', 'Permission denied', 'fatal', 'No such file or directory'] + marker in error for marker in [b'error', b'Permission denied', b'fatal', b'No such file or directory'] ) - if not ignore_errors and error_found: - if isinstance(error, bytes): - message = b"Utility exited with non-zero code. Error: " + error - else: - message = f"Utility exited with non-zero code. Error: {error}" + assert type(error_found) == bool # noqa: E721 + + if encoding: + result = result.decode(encoding) + error = error.decode(encoding) + + if not ignore_errors and error_found and not expect_error: + error = normalize_error(error) + assert type(error) == str # noqa: E721 + message = "Utility exited with non-zero code. Error: " + error raise ExecUtilException(message=message, command=cmd, exit_code=exit_status, out=result) if verbose: From 1c64337682288201f4c163ad4f12afdca82e33cb Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Sun, 8 Dec 2024 00:12:41 +0300 Subject: [PATCH 077/216] TestLocalOperations tests skip Windows --- tests/test_local.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/test_local.py b/tests/test_local.py index 1caba74b..3493810f 100644 --- a/tests/test_local.py +++ b/tests/test_local.py @@ -1,4 +1,5 @@ import pytest +import platform from testgres import ExecUtilException from testgres import LocalOperations @@ -10,10 +11,16 @@ class TestLocalOperations: def setup(self): self.operations = LocalOperations() + def skip_if_windows(): + if platform.system().lower() == "windows": + pytest.skip("This test does not support Windows.") + def test_exec_command_success(self): """ Test exec_command for successful command execution. """ + __class__.skip_if_windows() + cmd = "python3 --version" response = self.operations.exec_command(cmd, wait_exit=True, shell=True) @@ -23,6 +30,8 @@ def test_exec_command_failure(self): """ Test exec_command for command execution failure. """ + __class__.skip_if_windows() + cmd = "nonexistent_command" while True: try: @@ -37,6 +46,8 @@ def test_exec_command_failure__expect_error(self): """ Test exec_command for command execution failure. """ + __class__.skip_if_windows() + cmd = "nonexistent_command" exit_status, result, error = self.operations.exec_command(cmd, verbose=True, wait_exit=True, shell=True, expect_error=True) From cb87d0a7b1c87a10c85787591c3d9ba4e6687e76 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Sun, 8 Dec 2024 15:26:58 +0300 Subject: [PATCH 078/216] tests.helpers.RunConditions is added RunConditions contains the code to check the execution condition of tests. It is used in TestLocalOperations. --- tests/__init__.py | 0 tests/helpers/__init__.py | 0 tests/helpers/run_conditions.py | 11 +++++++++++ tests/test_local.py | 13 +++++-------- 4 files changed, 16 insertions(+), 8 deletions(-) create mode 100644 tests/__init__.py create mode 100644 tests/helpers/__init__.py create mode 100644 tests/helpers/run_conditions.py diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/helpers/__init__.py b/tests/helpers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/helpers/run_conditions.py b/tests/helpers/run_conditions.py new file mode 100644 index 00000000..8d57f753 --- /dev/null +++ b/tests/helpers/run_conditions.py @@ -0,0 +1,11 @@ +import pytest +import platform + + +class RunConditions: + # It is not a test kit! + __test__ = False + + def skip_if_windows(): + if platform.system().lower() == "windows": + pytest.skip("This test does not support Windows.") diff --git a/tests/test_local.py b/tests/test_local.py index 3493810f..da26468b 100644 --- a/tests/test_local.py +++ b/tests/test_local.py @@ -1,9 +1,10 @@ import pytest -import platform from testgres import ExecUtilException from testgres import LocalOperations +from .helpers.run_conditions import RunConditions + class TestLocalOperations: @@ -11,15 +12,11 @@ class TestLocalOperations: def setup(self): self.operations = LocalOperations() - def skip_if_windows(): - if platform.system().lower() == "windows": - pytest.skip("This test does not support Windows.") - def test_exec_command_success(self): """ Test exec_command for successful command execution. """ - __class__.skip_if_windows() + RunConditions.skip_if_windows() cmd = "python3 --version" response = self.operations.exec_command(cmd, wait_exit=True, shell=True) @@ -30,7 +27,7 @@ def test_exec_command_failure(self): """ Test exec_command for command execution failure. """ - __class__.skip_if_windows() + RunConditions.skip_if_windows() cmd = "nonexistent_command" while True: @@ -46,7 +43,7 @@ def test_exec_command_failure__expect_error(self): """ Test exec_command for command execution failure. """ - __class__.skip_if_windows() + RunConditions.skip_if_windows() cmd = "nonexistent_command" From f9ddd043aceb3ac86f6b63cd7fba00575e0d44b1 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Sun, 8 Dec 2024 22:06:29 +0300 Subject: [PATCH 079/216] Proposal to fix #154 (v2) - The one way to generate ExecUtilException - RaiseError.UtilityExitedWithNonZeroCode - RaiseError is added - ExecUtilException::error is added (it contains the error data) - ExecUtilException::__str__ is updated - PostgresNode::psql and PostgresNode::safe_psql are updated - TestLocalOperations::test_exec_command_failure is updated - TestRemoteOperations::test_exec_command_failure is updated - TestRemoteOperations::test_makedirs_and_rmdirs_failure is updated --- testgres/exceptions.py | 11 +++++-- testgres/helpers/raise_error.py | 45 ++++++++++++++++++++++++++++ testgres/node.py | 50 ++++++++++++++++--------------- testgres/operations/local_ops.py | 20 +++++++------ testgres/operations/remote_ops.py | 14 ++++++--- tests/test_local.py | 2 +- tests/test_remote.py | 15 ++++++---- 7 files changed, 110 insertions(+), 47 deletions(-) create mode 100644 testgres/helpers/raise_error.py diff --git a/testgres/exceptions.py b/testgres/exceptions.py index ee329031..ff4381f4 100644 --- a/testgres/exceptions.py +++ b/testgres/exceptions.py @@ -9,13 +9,14 @@ class TestgresException(Exception): @six.python_2_unicode_compatible class ExecUtilException(TestgresException): - def __init__(self, message=None, command=None, exit_code=0, out=None): + def __init__(self, message=None, command=None, exit_code=0, out=None, error=None): super(ExecUtilException, self).__init__(message) self.message = message self.command = command self.exit_code = exit_code self.out = out + self.error = error def __str__(self): msg = [] @@ -24,13 +25,17 @@ def __str__(self): msg.append(self.message) if self.command: - msg.append(u'Command: {}'.format(self.command)) + command_s = ' '.join(self.command) if isinstance(self.command, list) else self.command, + msg.append(u'Command: {}'.format(command_s)) if self.exit_code: msg.append(u'Exit code: {}'.format(self.exit_code)) + if self.error: + msg.append(u'---- Error:\n{}'.format(self.error)) + if self.out: - msg.append(u'----\n{}'.format(self.out)) + msg.append(u'---- Out:\n{}'.format(self.out)) return self.convert_and_join(msg) diff --git a/testgres/helpers/raise_error.py b/testgres/helpers/raise_error.py new file mode 100644 index 00000000..c67833dd --- /dev/null +++ b/testgres/helpers/raise_error.py @@ -0,0 +1,45 @@ +from ..exceptions import ExecUtilException + + +class RaiseError: + def UtilityExitedWithNonZeroCode(cmd, exit_code, msg_arg, error, out): + assert type(exit_code) == int # noqa: E721 + + msg_arg_s = __class__._TranslateDataIntoString(msg_arg).strip() + assert type(msg_arg_s) == str # noqa: E721 + + if msg_arg_s == "": + msg_arg_s = "#no_error_message" + + message = "Utility exited with non-zero code. Error: `" + msg_arg_s + "`" + raise ExecUtilException( + message=message, + command=cmd, + exit_code=exit_code, + out=out, + error=error) + + def _TranslateDataIntoString(data): + if type(data) == bytes: # noqa: E721 + return __class__._TranslateDataIntoString__FromBinary(data) + + return str(data) + + def _TranslateDataIntoString__FromBinary(data): + assert type(data) == bytes # noqa: E721 + + try: + return data.decode('utf-8') + except UnicodeDecodeError: + pass + + return "#cannot_decode_text" + + def _BinaryIsASCII(data): + assert type(data) == bytes # noqa: E721 + + for b in data: + if not (b >= 0 and b <= 127): + return False + + return True diff --git a/testgres/node.py b/testgres/node.py index 48a100a9..8300d493 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -3,7 +3,6 @@ import os import random import signal -import subprocess import threading import tempfile from queue import Queue @@ -987,6 +986,25 @@ def psql(self, >>> psql(query='select 3', ON_ERROR_STOP=1) """ + return self._psql( + ignore_errors=True, + query=query, + filename=filename, + dbname=dbname, + username=username, + input=input, + **variables + ) + + def _psql( + self, + ignore_errors, + query=None, + filename=None, + dbname=None, + username=None, + input=None, + **variables): dbname = dbname or default_dbname() psql_params = [ @@ -1017,20 +1035,8 @@ def psql(self, # should be the last one psql_params.append(dbname) - if not self.os_ops.remote: - # start psql process - process = subprocess.Popen(psql_params, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - - # wait until it finishes and get stdout and stderr - out, err = process.communicate(input=input) - return process.returncode, out, err - else: - status_code, out, err = self.os_ops.exec_command(psql_params, verbose=True, input=input) - return status_code, out, err + return self.os_ops.exec_command(psql_params, verbose=True, input=input, ignore_errors=ignore_errors) @method_decorator(positional_args_hack(['dbname', 'query'])) def safe_psql(self, query=None, expect_error=False, **kwargs): @@ -1051,21 +1057,17 @@ def safe_psql(self, query=None, expect_error=False, **kwargs): Returns: psql's output as str. """ + assert type(kwargs) == dict # noqa: E721 + assert not ("ignore_errors" in kwargs.keys()) # force this setting kwargs['ON_ERROR_STOP'] = 1 try: - ret, out, err = self.psql(query=query, **kwargs) + ret, out, err = self._psql(ignore_errors=False, query=query, **kwargs) except ExecUtilException as e: - ret = e.exit_code - out = e.out - err = e.message - if ret: - if expect_error: - out = (err or b'').decode('utf-8') - else: - raise QueryException((err or b'').decode('utf-8'), query) - elif expect_error: + raise QueryException(e.message, query) + + if expect_error: assert False, "Exception was expected, but query finished successfully: `{}` ".format(query) return out diff --git a/testgres/operations/local_ops.py b/testgres/operations/local_ops.py index 5b7972ae..14c408c9 100644 --- a/testgres/operations/local_ops.py +++ b/testgres/operations/local_ops.py @@ -11,6 +11,7 @@ from ..exceptions import ExecUtilException from .os_ops import ConnectionParams, OsOperations, pglib, get_default_encoding +from ..helpers.raise_error import RaiseError try: from shutil import which as find_executable @@ -47,14 +48,6 @@ def __init__(self, conn_params=None): self.remote = False self.username = conn_params.username or getpass.getuser() - @staticmethod - def _raise_exec_exception(message, command, exit_code, output): - """Raise an ExecUtilException.""" - raise ExecUtilException(message=message.format(output), - command=' '.join(command) if isinstance(command, list) else command, - exit_code=exit_code, - out=output) - @staticmethod def _process_output(encoding, temp_file_path): """Process the output of a command from a temporary file.""" @@ -120,11 +113,20 @@ def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False, """ Execute a command in a subprocess and handle the output based on the provided parameters. """ + assert type(expect_error) == bool # noqa: E721 + assert type(ignore_errors) == bool # noqa: E721 + process, output, error = self._run_command(cmd, shell, input, stdin, stdout, stderr, get_process, timeout, encoding) if get_process: return process if not ignore_errors and ((process.returncode != 0 or has_errors(output=output, error=error)) and not expect_error): - self._raise_exec_exception('Utility exited with non-zero code. Error `{}`', cmd, process.returncode, error or output) + RaiseError.UtilityExitedWithNonZeroCode( + cmd=cmd, + exit_code=process.returncode, + msg_arg=error or output, + error=error, + out=output + ) if verbose: return process.returncode, output, error diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index 88394eb7..4340ec11 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -14,6 +14,7 @@ raise ImportError("You must have psycopg2 or pg8000 modules installed") from ..exceptions import ExecUtilException +from ..helpers.raise_error import RaiseError from .os_ops import OsOperations, ConnectionParams, get_default_encoding error_markers = [b'error', b'Permission denied', b'fatal', b'No such file or directory'] @@ -66,6 +67,9 @@ def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False, Args: - cmd (str): The command to be executed. """ + assert type(expect_error) == bool # noqa: E721 + assert type(ignore_errors) == bool # noqa: E721 + ssh_cmd = [] if isinstance(cmd, str): ssh_cmd = ['ssh', self.ssh_dest] + self.ssh_args + [cmd] @@ -100,10 +104,12 @@ def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False, error = error.decode(encoding) if not ignore_errors and error_found and not expect_error: - error = normalize_error(error) - assert type(error) == str # noqa: E721 - message = "Utility exited with non-zero code. Error: " + error - raise ExecUtilException(message=message, command=cmd, exit_code=exit_status, out=result) + RaiseError.UtilityExitedWithNonZeroCode( + cmd=cmd, + exit_code=exit_status, + msg_arg=error, + error=error, + out=result) if verbose: return exit_status, result, error diff --git a/tests/test_local.py b/tests/test_local.py index da26468b..cb96a3bc 100644 --- a/tests/test_local.py +++ b/tests/test_local.py @@ -37,7 +37,7 @@ def test_exec_command_failure(self): error = e.message break raise Exception("We wait an exception!") - assert error == "Utility exited with non-zero code. Error `b'/bin/sh: 1: nonexistent_command: not found\\n'`" + assert error == "Utility exited with non-zero code. Error: `/bin/sh: 1: nonexistent_command: not found`" def test_exec_command_failure__expect_error(self): """ diff --git a/tests/test_remote.py b/tests/test_remote.py index 565163f7..c1a91bc6 100755 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -37,7 +37,7 @@ def test_exec_command_failure(self): error = e.message break raise Exception("We wait an exception!") - assert error == b'Utility exited with non-zero code. Error: bash: line 1: nonexistent_command: command not found\n' + assert error == 'Utility exited with non-zero code. Error: `bash: line 1: nonexistent_command: command not found`' def test_exec_command_failure__expect_error(self): """ @@ -98,11 +98,14 @@ def test_makedirs_and_rmdirs_failure(self): self.operations.makedirs(path) # Test rmdirs - try: - exit_status, result, error = self.operations.rmdirs(path, verbose=True) - except ExecUtilException as e: - error = e.message - assert error == b"Utility exited with non-zero code. Error: rm: cannot remove '/root/test_dir': Permission denied\n" + while True: + try: + self.operations.rmdirs(path, verbose=True) + except ExecUtilException as e: + error = e.message + break + raise Exception("We wait an exception!") + assert error == "Utility exited with non-zero code. Error: `rm: cannot remove '/root/test_dir': Permission denied`" def test_listdir(self): """ From 2bb38dc45b69c4d5d4c93a65fea4b64b81511287 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Mon, 9 Dec 2024 11:30:51 +0300 Subject: [PATCH 080/216] [BUG FIX] PostgresNode::safe_psql did not respect "expect_error" parameter --- testgres/node.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/testgres/node.py b/testgres/node.py index 8300d493..11f73af2 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -1059,13 +1059,22 @@ def safe_psql(self, query=None, expect_error=False, **kwargs): """ assert type(kwargs) == dict # noqa: E721 assert not ("ignore_errors" in kwargs.keys()) + assert not ("expect_error" in kwargs.keys()) # force this setting kwargs['ON_ERROR_STOP'] = 1 try: ret, out, err = self._psql(ignore_errors=False, query=query, **kwargs) except ExecUtilException as e: - raise QueryException(e.message, query) + if not expect_error: + raise QueryException(e.message, query) + + if type(e.error) == bytes: # noqa: E721 + return e.error.decode("utf-8") # throw + + # [2024-12-09] This situation is not expected + assert False + return e.error if expect_error: assert False, "Exception was expected, but query finished successfully: `{}` ".format(query) From 45b8dc024dc39994bed029ebb1d363686ae71cbf Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Mon, 9 Dec 2024 15:07:21 +0300 Subject: [PATCH 081/216] [BUG FIX] A problem in psql/safe_psql and 'input' data was fixed [local_op] Both LocalOperations::exec_command and RemoteOperations::exec_command were updated. --- testgres/node.py | 12 ++++++++++++ testgres/operations/helpers.py | 15 +++++++++++++++ testgres/operations/local_ops.py | 9 ++++++++- testgres/operations/remote_ops.py | 7 ++++++- 4 files changed, 41 insertions(+), 2 deletions(-) create mode 100644 testgres/operations/helpers.py diff --git a/testgres/node.py b/testgres/node.py index 11f73af2..68e9b0eb 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -1005,6 +1005,18 @@ def _psql( username=None, input=None, **variables): + assert type(variables) == dict # noqa: E721 + + # + # We do not support encoding. It may be added later. Ok? + # + if input is None: + pass + elif type(input) == bytes: # noqa: E721 + pass + else: + raise Exception("Input data must be None or bytes.") + dbname = dbname or default_dbname() psql_params = [ diff --git a/testgres/operations/helpers.py b/testgres/operations/helpers.py new file mode 100644 index 00000000..d714d336 --- /dev/null +++ b/testgres/operations/helpers.py @@ -0,0 +1,15 @@ +class Helpers: + def PrepareProcessInput(input, encoding): + if not input: + return None + + if type(input) == str: # noqa: E721 + if encoding is None: + return input.encode() + + assert type(encoding) == str # noqa: E721 + return input.encode(encoding) + + # It is expected! + assert type(input) == bytes # noqa: E721 + return input diff --git a/testgres/operations/local_ops.py b/testgres/operations/local_ops.py index 14c408c9..833d1fd2 100644 --- a/testgres/operations/local_ops.py +++ b/testgres/operations/local_ops.py @@ -11,6 +11,7 @@ from ..exceptions import ExecUtilException from .os_ops import ConnectionParams, OsOperations, pglib, get_default_encoding +from .helpers import Helpers from ..helpers.raise_error import RaiseError try: @@ -58,6 +59,8 @@ def _process_output(encoding, temp_file_path): return output, None # In Windows stderr writing in stdout def _run_command__nt(self, cmd, shell, input, stdin, stdout, stderr, get_process, timeout, encoding): + # TODO: why don't we use the data from input? + with tempfile.NamedTemporaryFile(mode='w+b', delete=False) as temp_file: stdout = temp_file stderr = subprocess.STDOUT @@ -79,6 +82,10 @@ def _run_command__nt(self, cmd, shell, input, stdin, stdout, stderr, get_process return process, output, error def _run_command__generic(self, cmd, shell, input, stdin, stdout, stderr, get_process, timeout, encoding): + input_prepared = None + if not get_process: + input_prepared = Helpers.PrepareProcessInput(input, encoding) # throw + process = subprocess.Popen( cmd, shell=shell, @@ -89,7 +96,7 @@ def _run_command__generic(self, cmd, shell, input, stdin, stdout, stderr, get_pr if get_process: return process, None, None try: - output, error = process.communicate(input=input.encode(encoding) if input else None, timeout=timeout) + output, error = process.communicate(input=input_prepared, timeout=timeout) if encoding: output = output.decode(encoding) error = error.decode(encoding) diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index 4340ec11..cbaeb62b 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -16,6 +16,7 @@ from ..exceptions import ExecUtilException from ..helpers.raise_error import RaiseError from .os_ops import OsOperations, ConnectionParams, get_default_encoding +from .helpers import Helpers error_markers = [b'error', b'Permission denied', b'fatal', b'No such file or directory'] @@ -70,6 +71,10 @@ def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False, assert type(expect_error) == bool # noqa: E721 assert type(ignore_errors) == bool # noqa: E721 + input_prepared = None + if not get_process: + input_prepared = Helpers.PrepareProcessInput(input, encoding) # throw + ssh_cmd = [] if isinstance(cmd, str): ssh_cmd = ['ssh', self.ssh_dest] + self.ssh_args + [cmd] @@ -80,7 +85,7 @@ def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False, return process try: - result, error = process.communicate(input, timeout=timeout) + result, error = process.communicate(input=input_prepared, timeout=timeout) except subprocess.TimeoutExpired: process.kill() raise ExecUtilException("Command timed out after {} seconds.".format(timeout)) From f848a63fd3dce4a50d1a994f2b5ee1ce9e15a7b6 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Mon, 9 Dec 2024 17:50:50 +0300 Subject: [PATCH 082/216] PostgresNode::safe_psql raises InvalidOperationException If expect_error is True and no error is detected, safe_psql raises InvalidOperationException exception. Tests (local, remote) are added. --- testgres/__init__.py | 3 ++- testgres/exceptions.py | 3 +++ testgres/node.py | 5 +++-- tests/test_simple.py | 21 ++++++++++++++++++++- tests/test_simple_remote.py | 20 +++++++++++++++++++- 5 files changed, 47 insertions(+), 5 deletions(-) diff --git a/testgres/__init__.py b/testgres/__init__.py index 8d0e38c6..69d2ab4a 100644 --- a/testgres/__init__.py +++ b/testgres/__init__.py @@ -23,7 +23,8 @@ CatchUpException, \ StartNodeException, \ InitNodeException, \ - BackupException + BackupException, \ + InvalidOperationException from .enums import \ XLogMethod, \ diff --git a/testgres/exceptions.py b/testgres/exceptions.py index ff4381f4..b4d9f76b 100644 --- a/testgres/exceptions.py +++ b/testgres/exceptions.py @@ -103,3 +103,6 @@ class InitNodeException(TestgresException): class BackupException(TestgresException): pass + +class InvalidOperationException(TestgresException): + pass diff --git a/testgres/node.py b/testgres/node.py index 68e9b0eb..ae52f21b 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -73,7 +73,8 @@ TimeoutException, \ InitNodeException, \ TestgresException, \ - BackupException + BackupException, \ + InvalidOperationException from .logger import TestgresLogger @@ -1089,7 +1090,7 @@ def safe_psql(self, query=None, expect_error=False, **kwargs): return e.error if expect_error: - assert False, "Exception was expected, but query finished successfully: `{}` ".format(query) + raise InvalidOperationException("Exception was expected, but query finished successfully: `{}`.".format(query)) return out diff --git a/tests/test_simple.py b/tests/test_simple.py index 51cdc896..c4200c6f 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -23,7 +23,9 @@ BackupException, \ QueryException, \ TimeoutException, \ - TestgresException, NodeApp + TestgresException, \ + InvalidOperationException, \ + NodeApp from testgres import \ TestgresConfig, \ @@ -310,6 +312,23 @@ def test_psql(self): with self.assertRaises(QueryException): node.safe_psql('select 1') + def test_safe_psql__expect_error(self): + with get_new_node().init().start() as node: + err = node.safe_psql('select_or_not_select 1', expect_error=True) + self.assertTrue(type(err) == str) # noqa: E721 + self.assertIn('select_or_not_select', err) + self.assertIn('ERROR: syntax error at or near "select_or_not_select"', err) + + # --------- + with self.assertRaises(InvalidOperationException) as ctx: + node.safe_psql("select 1;", expect_error=True) + + self.assertEqual(str(ctx.exception), "Exception was expected, but query finished successfully: `select 1;`.") + + # --------- + res = node.safe_psql("select 1;", expect_error=False) + self.assertEqual(res, b'1\n') + def test_transactions(self): with get_new_node().init().start() as node: diff --git a/tests/test_simple_remote.py b/tests/test_simple_remote.py index 936c31f2..26ac7c61 100755 --- a/tests/test_simple_remote.py +++ b/tests/test_simple_remote.py @@ -23,7 +23,8 @@ BackupException, \ QueryException, \ TimeoutException, \ - TestgresException + TestgresException, \ + InvalidOperationException from testgres.config import \ TestgresConfig, \ @@ -295,6 +296,23 @@ def test_psql(self): with self.assertRaises(QueryException): node.safe_psql('select 1') + def test_safe_psql__expect_error(self): + with get_remote_node(conn_params=conn_params).init().start() as node: + err = node.safe_psql('select_or_not_select 1', expect_error=True) + self.assertTrue(type(err) == str) # noqa: E721 + self.assertIn('select_or_not_select', err) + self.assertIn('ERROR: syntax error at or near "select_or_not_select"', err) + + # --------- + with self.assertRaises(InvalidOperationException) as ctx: + node.safe_psql("select 1;", expect_error=True) + + self.assertEqual(str(ctx.exception), "Exception was expected, but query finished successfully: `select 1;`.") + + # --------- + res = node.safe_psql("select 1;", expect_error=False) + self.assertEqual(res, b'1\n') + def test_transactions(self): with get_remote_node(conn_params=conn_params).init().start() as node: with node.connect() as con: From db0744e7a575ebd5d7263c2a9fe2a193f7716c48 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Mon, 9 Dec 2024 18:02:12 +0300 Subject: [PATCH 083/216] A problem with InvalidOperationException and flake8 is fixed --- testgres/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testgres/__init__.py b/testgres/__init__.py index 69d2ab4a..665548d6 100644 --- a/testgres/__init__.py +++ b/testgres/__init__.py @@ -61,7 +61,7 @@ "NodeBackup", "testgres_config", "TestgresConfig", "configure_testgres", "scoped_config", "push_config", "pop_config", "NodeConnection", "DatabaseError", "InternalError", "ProgrammingError", "OperationalError", - "TestgresException", "ExecUtilException", "QueryException", "TimeoutException", "CatchUpException", "StartNodeException", "InitNodeException", "BackupException", + "TestgresException", "ExecUtilException", "QueryException", "TimeoutException", "CatchUpException", "StartNodeException", "InitNodeException", "BackupException", "InvalidOperationException", "XLogMethod", "IsolationLevel", "NodeStatus", "ProcessType", "DumpFormat", "PostgresNode", "NodeApp", "reserve_port", "release_port", "bound_ports", "get_bin_path", "get_pg_config", "get_pg_version", From 5bb1510bdb8a48d84bd2b5ea0c4a20f6951a8edf Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Mon, 9 Dec 2024 18:27:29 +0300 Subject: [PATCH 084/216] A code style is fixed [flake8] --- testgres/exceptions.py | 1 + 1 file changed, 1 insertion(+) diff --git a/testgres/exceptions.py b/testgres/exceptions.py index b4d9f76b..d61d4691 100644 --- a/testgres/exceptions.py +++ b/testgres/exceptions.py @@ -104,5 +104,6 @@ class InitNodeException(TestgresException): class BackupException(TestgresException): pass + class InvalidOperationException(TestgresException): pass From 31c7bce1e21e796d996873f3c9ae270e52992f88 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Mon, 9 Dec 2024 19:56:32 +0300 Subject: [PATCH 085/216] [BUG FIX] Wrappers for psql use subprocess.PIPE for stdout and stderr It fixes a problem with Windows. --- testgres/node.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/testgres/node.py b/testgres/node.py index ae52f21b..1037aca2 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -5,6 +5,7 @@ import signal import threading import tempfile +import subprocess from queue import Queue import time @@ -1049,7 +1050,13 @@ def _psql( # should be the last one psql_params.append(dbname) - return self.os_ops.exec_command(psql_params, verbose=True, input=input, ignore_errors=ignore_errors) + return self.os_ops.exec_command( + psql_params, + verbose=True, + input=input, + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + ignore_errors=ignore_errors) @method_decorator(positional_args_hack(['dbname', 'query'])) def safe_psql(self, query=None, expect_error=False, **kwargs): From b013801e5aff06d1daa13e805d6080906f953868 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Mon, 9 Dec 2024 20:23:56 +0300 Subject: [PATCH 086/216] [BUG FIX] TestgresTests::test_safe_psql__expect_error uses rm_carriage_returns It fixes a problem with this test on Windows. --- tests/test_simple.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_simple.py b/tests/test_simple.py index c4200c6f..fade468c 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -327,7 +327,7 @@ def test_safe_psql__expect_error(self): # --------- res = node.safe_psql("select 1;", expect_error=False) - self.assertEqual(res, b'1\n') + self.assertEqual(rm_carriage_returns(res), b'1\n') def test_transactions(self): with get_new_node().init().start() as node: From c49ee4cf5979b83dbcecb085a2b8473f32474271 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 10 Dec 2024 11:22:28 +0300 Subject: [PATCH 087/216] node.py is updated [formatting] The previous order of imports is restored for minimization number of changes. --- testgres/node.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testgres/node.py b/testgres/node.py index 1037aca2..32b1e244 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -3,9 +3,9 @@ import os import random import signal +import subprocess import threading import tempfile -import subprocess from queue import Queue import time From 6a0e71495740caa32cbd54cd4b2a3819e13de539 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 10 Dec 2024 11:47:48 +0300 Subject: [PATCH 088/216] Formatting --- testgres/node.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testgres/node.py b/testgres/node.py index 32b1e244..b5cbab27 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -74,7 +74,7 @@ TimeoutException, \ InitNodeException, \ TestgresException, \ - BackupException, \ + BackupException, \ InvalidOperationException from .logger import TestgresLogger From 3cc19d2afdd390699ec234f83496cece996f2fe0 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 10 Dec 2024 14:55:46 +0300 Subject: [PATCH 089/216] raise_error.py is moved into testgres/operations from testgres/helpers Let's store our things on one place. We use RaiseError only in testgres/operations structures currently. --- testgres/operations/local_ops.py | 2 +- testgres/{helpers => operations}/raise_error.py | 0 testgres/operations/remote_ops.py | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename testgres/{helpers => operations}/raise_error.py (100%) diff --git a/testgres/operations/local_ops.py b/testgres/operations/local_ops.py index 833d1fd2..d6daaa3b 100644 --- a/testgres/operations/local_ops.py +++ b/testgres/operations/local_ops.py @@ -11,8 +11,8 @@ from ..exceptions import ExecUtilException from .os_ops import ConnectionParams, OsOperations, pglib, get_default_encoding +from .raise_error import RaiseError from .helpers import Helpers -from ..helpers.raise_error import RaiseError try: from shutil import which as find_executable diff --git a/testgres/helpers/raise_error.py b/testgres/operations/raise_error.py similarity index 100% rename from testgres/helpers/raise_error.py rename to testgres/operations/raise_error.py diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index cbaeb62b..c48f867b 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -14,8 +14,8 @@ raise ImportError("You must have psycopg2 or pg8000 modules installed") from ..exceptions import ExecUtilException -from ..helpers.raise_error import RaiseError from .os_ops import OsOperations, ConnectionParams, get_default_encoding +from .raise_error import RaiseError from .helpers import Helpers error_markers = [b'error', b'Permission denied', b'fatal', b'No such file or directory'] From 7b70e9e7a0b22fb999e308dab8bccc46e8e22a65 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 10 Dec 2024 16:21:39 +0300 Subject: [PATCH 090/216] Helpers.GetDefaultEncoding is added This function is equal to os_ops.get_default_encoding and is used in: - Helpers.PrepareProcessInput - RaiseError._TranslateDataIntoString__FromBinary --- testgres/operations/helpers.py | 39 +++++++++++++++++++++++++++++- testgres/operations/raise_error.py | 3 ++- 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/testgres/operations/helpers.py b/testgres/operations/helpers.py index d714d336..b50f0baa 100644 --- a/testgres/operations/helpers.py +++ b/testgres/operations/helpers.py @@ -1,11 +1,48 @@ +import locale + + class Helpers: + def _make_get_default_encoding_func(): + # locale.getencoding is added in Python 3.11 + if hasattr(locale, 'getencoding'): + return locale.getencoding + + # It must exist + return locale.getpreferredencoding + + # Prepared pointer on function to get a name of system codepage + _get_default_encoding_func = _make_get_default_encoding_func() + + def GetDefaultEncoding(): + # + # Original idea/source was: + # + # def os_ops.get_default_encoding(): + # if not hasattr(locale, 'getencoding'): + # locale.getencoding = locale.getpreferredencoding + # return locale.getencoding() or 'UTF-8' + # + + assert __class__._get_default_encoding_func is not None + + r = __class__._get_default_encoding_func() + + if r: + assert r is not None + assert type(r) == str # noqa: E721 + assert r != "" + return r + + # Is it an unexpected situation? + return 'UTF-8' + def PrepareProcessInput(input, encoding): if not input: return None if type(input) == str: # noqa: E721 if encoding is None: - return input.encode() + return input.encode(__class__.GetDefaultEncoding()) assert type(encoding) == str # noqa: E721 return input.encode(encoding) diff --git a/testgres/operations/raise_error.py b/testgres/operations/raise_error.py index c67833dd..0e760e74 100644 --- a/testgres/operations/raise_error.py +++ b/testgres/operations/raise_error.py @@ -1,4 +1,5 @@ from ..exceptions import ExecUtilException +from .helpers import Helpers class RaiseError: @@ -29,7 +30,7 @@ def _TranslateDataIntoString__FromBinary(data): assert type(data) == bytes # noqa: E721 try: - return data.decode('utf-8') + return data.decode(Helpers.GetDefaultEncoding()) except UnicodeDecodeError: pass From cd0b5f8671d61214afb2985adb83cd3efb9b852a Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 10 Dec 2024 17:01:57 +0300 Subject: [PATCH 091/216] Code normalization - New debug checks - Normalization --- testgres/operations/local_ops.py | 15 +++++++++++---- testgres/operations/remote_ops.py | 3 +++ 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/testgres/operations/local_ops.py b/testgres/operations/local_ops.py index d6daaa3b..3e8ab8ca 100644 --- a/testgres/operations/local_ops.py +++ b/testgres/operations/local_ops.py @@ -86,6 +86,8 @@ def _run_command__generic(self, cmd, shell, input, stdin, stdout, stderr, get_pr if not get_process: input_prepared = Helpers.PrepareProcessInput(input, encoding) # throw + assert input_prepared is None or (type(input_prepared) == bytes) # noqa: E721 + process = subprocess.Popen( cmd, shell=shell, @@ -93,18 +95,23 @@ def _run_command__generic(self, cmd, shell, input, stdin, stdout, stderr, get_pr stdout=stdout or subprocess.PIPE, stderr=stderr or subprocess.PIPE, ) + assert not (process is None) if get_process: return process, None, None try: output, error = process.communicate(input=input_prepared, timeout=timeout) - if encoding: - output = output.decode(encoding) - error = error.decode(encoding) - return process, output, error except subprocess.TimeoutExpired: process.kill() raise ExecUtilException("Command timed out after {} seconds.".format(timeout)) + assert type(output) == bytes # noqa: E721 + assert type(error) == bytes # noqa: E721 + + if encoding: + output = output.decode(encoding) + error = error.decode(encoding) + return process, output, error + def _run_command(self, cmd, shell, input, stdin, stdout, stderr, get_process, timeout, encoding): """Execute a command and return the process and its output.""" if os.name == 'nt' and stdout is None: # Windows diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index c48f867b..00c50d93 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -75,12 +75,15 @@ def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False, if not get_process: input_prepared = Helpers.PrepareProcessInput(input, encoding) # throw + assert input_prepared is None or (type(input_prepared) == bytes) # noqa: E721 + ssh_cmd = [] if isinstance(cmd, str): ssh_cmd = ['ssh', self.ssh_dest] + self.ssh_args + [cmd] elif isinstance(cmd, list): ssh_cmd = ['ssh', self.ssh_dest] + self.ssh_args + cmd process = subprocess.Popen(ssh_cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + assert not (process is None) if get_process: return process From 1b7bba4479a98c9d207e0e89b16e8a8269e826a8 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 10 Dec 2024 20:37:56 +0300 Subject: [PATCH 092/216] Fix for TestgresRemoteTests::test_child_pids [thanks to Victoria Shepard] This change was extracted from #149. --- tests/test_simple_remote.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_simple_remote.py b/tests/test_simple_remote.py index 26ac7c61..c8dd2964 100755 --- a/tests/test_simple_remote.py +++ b/tests/test_simple_remote.py @@ -940,6 +940,9 @@ def test_child_pids(self): if pg_version_ge('10'): master_processes.append(ProcessType.LogicalReplicationLauncher) + if pg_version_ge('14'): + master_processes.remove(ProcessType.StatsCollector) + repl_processes = [ ProcessType.Startup, ProcessType.WalReceiver, From 0b1b3de1d7ca370e6b4bcdfec0030e4776065714 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 10 Dec 2024 21:42:25 +0300 Subject: [PATCH 093/216] Formatting It is part of PR #149. --- testgres/node.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index b5cbab27..0faf904b 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -739,13 +739,11 @@ def start(self, params=[], wait=True): if self.is_started: return self - _params = [ - self._get_bin_path("pg_ctl"), - "-D", self.data_dir, - "-l", self.pg_log_file, - "-w" if wait else '-W', # --wait or --no-wait - "start" - ] + params # yapf: disable + _params = [self._get_bin_path("pg_ctl"), + "-D", self.data_dir, + "-l", self.pg_log_file, + "-w" if wait else '-W', # --wait or --no-wait + "start"] + params # yapf: disable startup_retries = 5 while True: From e4c2e07b3ff0c4aa16188a60bb53f108e2d5c0ca Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Thu, 12 Dec 2024 14:22:11 +0300 Subject: [PATCH 094/216] reserve_port and release_port are "pointers" to functions for a port numbers management. This need to replace port numbers management in unit tests. --- testgres/utils.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/testgres/utils.py b/testgres/utils.py index a4ee7877..4bd232b1 100644 --- a/testgres/utils.py +++ b/testgres/utils.py @@ -34,7 +34,7 @@ def __init__(self, version: str) -> None: super().__init__(version) -def reserve_port(): +def internal__reserve_port(): """ Generate a new port and add it to 'bound_ports'. """ @@ -45,7 +45,7 @@ def reserve_port(): return port -def release_port(port): +def internal__release_port(port): """ Free port provided by reserve_port(). """ @@ -53,6 +53,10 @@ def release_port(port): bound_ports.discard(port) +reserve_port = internal__reserve_port +release_port = internal__release_port + + def execute_utility(args, logfile=None, verbose=False): """ Execute utility (pg_ctl, pg_dump etc). From 85d2aa3917f8210dce06531b563ec34c42e6b544 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Thu, 12 Dec 2024 14:38:39 +0300 Subject: [PATCH 095/216] TestgresTests::test_the_same_port is corrected --- tests/test_simple.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/tests/test_simple.py b/tests/test_simple.py index fade468c..6b04f8bd 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -1051,10 +1051,18 @@ def test_parse_pg_version(self): def test_the_same_port(self): with get_new_node() as node: node.init().start() + self.assertTrue(node._should_free_port) + self.assertEqual(type(node.port), int) - with get_new_node() as node2: - node2.port = node.port - node2.init().start() + with get_new_node(port=node.port) as node2: + self.assertEqual(type(node2.port), int) + self.assertEqual(node2.port, node.port) + self.assertFalse(node2._should_free_port) + + with self.assertRaises(StartNodeException) as ctx: + node2.init().start() + + self.assertIn("Cannot start node", str(ctx.exception)) def test_simple_with_bin_dir(self): with get_new_node() as node: From 88371d1a610c62591d3b3ec2d5e88b7a2437ffb9 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Thu, 12 Dec 2024 14:46:23 +0300 Subject: [PATCH 096/216] OsOperations::read_binary(self, filename, start_pos) is added It is a specialized function to read binary data from files. --- testgres/operations/local_ops.py | 11 ++++++++ testgres/operations/os_ops.py | 6 +++++ testgres/operations/remote_ops.py | 19 +++++++++++++ tests/test_local.py | 45 +++++++++++++++++++++++++++++++ tests/test_remote.py | 43 +++++++++++++++++++++++++++++ 5 files changed, 124 insertions(+) diff --git a/testgres/operations/local_ops.py b/testgres/operations/local_ops.py index 3e8ab8ca..65dc0965 100644 --- a/testgres/operations/local_ops.py +++ b/testgres/operations/local_ops.py @@ -308,6 +308,17 @@ def readlines(self, filename, num_lines=0, binary=False, encoding=None): buffers * max(2, int(num_lines / max(cur_lines, 1))) ) # Adjust buffer size + def read_binary(self, filename, start_pos): + assert type(filename) == str # noqa: E721 + assert type(start_pos) == int # noqa: E721 + assert start_pos >= 0 + + with open(filename, 'rb') as file: # open in a binary mode + file.seek(start_pos, os.SEEK_SET) + r = file.read() + assert type(r) == bytes # noqa: E721 + return r + def isfile(self, remote_file): return os.path.isfile(remote_file) diff --git a/testgres/operations/os_ops.py b/testgres/operations/os_ops.py index 34242040..82d44a4e 100644 --- a/testgres/operations/os_ops.py +++ b/testgres/operations/os_ops.py @@ -98,6 +98,12 @@ def read(self, filename, encoding, binary): def readlines(self, filename): raise NotImplementedError() + def read_binary(self, filename, start_pos): + assert type(filename) == str # noqa: E721 + assert type(start_pos) == int # noqa: E721 + assert start_pos >= 0 + raise NotImplementedError() + def isfile(self, remote_file): raise NotImplementedError() diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index 00c50d93..9d72731d 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -340,6 +340,16 @@ def readlines(self, filename, num_lines=0, binary=False, encoding=None): return lines + def read_binary(self, filename, start_pos): + assert type(filename) == str # noqa: E721 + assert type(start_pos) == int # noqa: E721 + assert start_pos >= 0 + + cmd = "tail -c +{} {}".format(start_pos + 1, __class__._escape_path(filename)) + r = self.exec_command(cmd) + assert type(r) == bytes # noqa: E721 + return r + def isfile(self, remote_file): stdout = self.exec_command("test -f {}; echo $?".format(remote_file)) result = int(stdout.strip()) @@ -386,6 +396,15 @@ def db_connect(self, dbname, user, password=None, host="localhost", port=5432): ) return conn + def _escape_path(path): + assert type(path) == str # noqa: E721 + assert path != "" # Ok? + + r = "'" + r += path + r += "'" + return r + def normalize_error(error): if isinstance(error, bytes): diff --git a/tests/test_local.py b/tests/test_local.py index cb96a3bc..812b4030 100644 --- a/tests/test_local.py +++ b/tests/test_local.py @@ -1,4 +1,7 @@ +import os + import pytest +import re from testgres import ExecUtilException from testgres import LocalOperations @@ -52,3 +55,45 @@ def test_exec_command_failure__expect_error(self): assert error == b'/bin/sh: 1: nonexistent_command: not found\n' assert exit_status == 127 assert result == b'' + + def test_read_binary__spec(self): + """ + Test LocalOperations::read_binary. + """ + filename = __file__ # current file + + with open(filename, 'rb') as file: # open in a binary mode + response0 = file.read() + + assert type(response0) == bytes # noqa: E721 + + response1 = self.operations.read_binary(filename, 0) + assert type(response1) == bytes # noqa: E721 + assert response1 == response0 + + response2 = self.operations.read_binary(filename, 1) + assert type(response2) == bytes # noqa: E721 + assert len(response2) < len(response1) + assert len(response2) + 1 == len(response1) + assert response2 == response1[1:] + + response3 = self.operations.read_binary(filename, len(response1)) + assert type(response3) == bytes # noqa: E721 + assert len(response3) == 0 + + response4 = self.operations.read_binary(filename, len(response2)) + assert type(response4) == bytes # noqa: E721 + assert len(response4) == 1 + assert response4[0] == response1[len(response1) - 1] + + response5 = self.operations.read_binary(filename, len(response1) + 1) + assert type(response5) == bytes # noqa: E721 + assert len(response5) == 0 + + def test_read_binary__spec__unk_file(self): + """ + Test LocalOperations::read_binary with unknown file. + """ + + with pytest.raises(FileNotFoundError, match=re.escape("[Errno 2] No such file or directory: '/dummy'")): + self.operations.read_binary("/dummy", 0) diff --git a/tests/test_remote.py b/tests/test_remote.py index c1a91bc6..c775f72d 100755 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -1,6 +1,7 @@ import os import pytest +import re from testgres import ExecUtilException from testgres import RemoteOperations @@ -181,6 +182,48 @@ def test_read_binary_file(self): assert isinstance(response, bytes) + def test_read_binary__spec(self): + """ + Test RemoteOperations::read_binary. + """ + filename = __file__ # currnt file + + with open(filename, 'rb') as file: # open in a binary mode + response0 = file.read() + + assert type(response0) == bytes # noqa: E721 + + response1 = self.operations.read_binary(filename, 0) + assert type(response1) == bytes # noqa: E721 + assert response1 == response0 + + response2 = self.operations.read_binary(filename, 1) + assert type(response2) == bytes # noqa: E721 + assert len(response2) < len(response1) + assert len(response2) + 1 == len(response1) + assert response2 == response1[1:] + + response3 = self.operations.read_binary(filename, len(response1)) + assert type(response3) == bytes # noqa: E721 + assert len(response3) == 0 + + response4 = self.operations.read_binary(filename, len(response2)) + assert type(response4) == bytes # noqa: E721 + assert len(response4) == 1 + assert response4[0] == response1[len(response1) - 1] + + response5 = self.operations.read_binary(filename, len(response1) + 1) + assert type(response5) == bytes # noqa: E721 + assert len(response5) == 0 + + def test_read_binary__spec__unk_file(self): + """ + Test RemoteOperations::read_binary with unknown file. + """ + + with pytest.raises(ExecUtilException, match=re.escape("tail: cannot open '/dummy' for reading: No such file or directory")): + self.operations.read_binary("/dummy", 0) + def test_touch(self): """ Test touch for creating a new file or updating access and modification times of an existing file. From 4fe189445b0351692b57fc908d366e4dd82cb9e6 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Thu, 12 Dec 2024 15:16:48 +0300 Subject: [PATCH 097/216] OsOperations::get_file_size(self, filename) is added It is a function to get a size of file. --- testgres/operations/local_ops.py | 5 +++ testgres/operations/os_ops.py | 3 ++ testgres/operations/remote_ops.py | 64 +++++++++++++++++++++++++++++++ tests/test_local.py | 21 ++++++++++ tests/test_remote.py | 21 ++++++++++ 5 files changed, 114 insertions(+) diff --git a/testgres/operations/local_ops.py b/testgres/operations/local_ops.py index 65dc0965..82d1711d 100644 --- a/testgres/operations/local_ops.py +++ b/testgres/operations/local_ops.py @@ -325,6 +325,11 @@ def isfile(self, remote_file): def isdir(self, dirname): return os.path.isdir(dirname) + def get_file_size(self, filename): + assert filename is not None + assert type(filename) == str # noqa: E721 + return os.path.getsize(filename) + def remove_file(self, filename): return os.remove(filename) diff --git a/testgres/operations/os_ops.py b/testgres/operations/os_ops.py index 82d44a4e..2ab41246 100644 --- a/testgres/operations/os_ops.py +++ b/testgres/operations/os_ops.py @@ -107,6 +107,9 @@ def read_binary(self, filename, start_pos): def isfile(self, remote_file): raise NotImplementedError() + def get_file_size(self, filename): + raise NotImplementedError() + # Processes control def kill(self, pid, signal): # Kill the process diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index 9d72731d..9f88140c 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -360,6 +360,70 @@ def isdir(self, dirname): response = self.exec_command(cmd) return response.strip() == b"True" + def get_file_size(self, filename): + C_ERR_SRC = "RemoteOpertions::get_file_size" + + assert filename is not None + assert type(filename) == str # noqa: E721 + cmd = "du -b " + __class__._escape_path(filename) + + s = self.exec_command(cmd, encoding=get_default_encoding()) + assert type(s) == str # noqa: E721 + + if len(s) == 0: + raise Exception( + "[BUG CHECK] Can't get size of file [{2}]. Remote operation returned an empty string. Check point [{0}][{1}].".format( + C_ERR_SRC, + "#001", + filename + ) + ) + + i = 0 + + while i < len(s) and s[i].isdigit(): + assert s[i] >= '0' + assert s[i] <= '9' + i += 1 + + if i == 0: + raise Exception( + "[BUG CHECK] Can't get size of file [{2}]. Remote operation returned a bad formatted string. Check point [{0}][{1}].".format( + C_ERR_SRC, + "#002", + filename + ) + ) + + if i == len(s): + raise Exception( + "[BUG CHECK] Can't get size of file [{2}]. Remote operation returned a bad formatted string. Check point [{0}][{1}].".format( + C_ERR_SRC, + "#003", + filename + ) + ) + + if not s[i].isspace(): + raise Exception( + "[BUG CHECK] Can't get size of file [{2}]. Remote operation returned a bad formatted string. Check point [{0}][{1}].".format( + C_ERR_SRC, + "#004", + filename + ) + ) + + r = 0 + + for i2 in range(0, i): + ch = s[i2] + assert ch >= '0' + assert ch <= '9' + # Here is needed to check overflow or that it is a human-valid result? + r = (r * 10) + ord(ch) - ord('0') + + return r + def remove_file(self, filename): cmd = "rm {}".format(filename) return self.exec_command(cmd) diff --git a/tests/test_local.py b/tests/test_local.py index 812b4030..a8a0bde0 100644 --- a/tests/test_local.py +++ b/tests/test_local.py @@ -97,3 +97,24 @@ def test_read_binary__spec__unk_file(self): with pytest.raises(FileNotFoundError, match=re.escape("[Errno 2] No such file or directory: '/dummy'")): self.operations.read_binary("/dummy", 0) + + def test_get_file_size(self): + """ + Test LocalOperations::get_file_size. + """ + filename = __file__ # current file + + sz0 = os.path.getsize(filename) + assert type(sz0) == int # noqa: E721 + + sz1 = self.operations.get_file_size(filename) + assert type(sz1) == int # noqa: E721 + assert sz1 == sz0 + + def test_get_file_size__unk_file(self): + """ + Test LocalOperations::get_file_size. + """ + + with pytest.raises(FileNotFoundError, match=re.escape("[Errno 2] No such file or directory: '/dummy'")): + self.operations.get_file_size("/dummy") diff --git a/tests/test_remote.py b/tests/test_remote.py index c775f72d..be1a56bb 100755 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -224,6 +224,27 @@ def test_read_binary__spec__unk_file(self): with pytest.raises(ExecUtilException, match=re.escape("tail: cannot open '/dummy' for reading: No such file or directory")): self.operations.read_binary("/dummy", 0) + def test_get_file_size(self): + """ + Test LocalOperations::get_file_size. + """ + filename = __file__ # current file + + sz0 = os.path.getsize(filename) + assert type(sz0) == int # noqa: E721 + + sz1 = self.operations.get_file_size(filename) + assert type(sz1) == int # noqa: E721 + assert sz1 == sz0 + + def test_get_file_size__unk_file(self): + """ + Test LocalOperations::get_file_size. + """ + + with pytest.raises(ExecUtilException, match=re.escape("du: cannot access '/dummy': No such file or directory")): + self.operations.get_file_size("/dummy") + def test_touch(self): """ Test touch for creating a new file or updating access and modification times of an existing file. From 28ac4252e5761394b74bb782e29b986f1ffd31d9 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Thu, 12 Dec 2024 15:28:39 +0300 Subject: [PATCH 098/216] Port numbers management is improved (#164) - We don't release a port number that was defined by client - We only check log files to detect port number conflicts - We use slightly smarter log file checking A test is added. --- testgres/node.py | 89 +++++++++++++++++++++++++------- tests/test_simple.py | 117 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 189 insertions(+), 17 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index 0faf904b..7f5aa648 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -83,13 +83,13 @@ from .standby import First +from . import utils + from .utils import \ PgVer, \ eprint, \ get_bin_path, \ get_pg_version, \ - reserve_port, \ - release_port, \ execute_utility, \ options_string, \ clean_on_error @@ -158,7 +158,7 @@ def __init__(self, name=None, base_dir=None, port=None, conn_params: ConnectionP self.os_ops = LocalOperations(conn_params) self.host = self.os_ops.host - self.port = port or reserve_port() + self.port = port or utils.reserve_port() self.ssh_key = self.os_ops.ssh_key @@ -471,6 +471,28 @@ def _collect_special_files(self): return result + def _collect_log_files(self): + # dictionary of log files + size in bytes + + files = [ + self.pg_log_file + ] # yapf: disable + + result = {} + + for f in files: + # skip missing files + if not self.os_ops.path_exists(f): + continue + + file_size = self.os_ops.get_file_size(f) + assert type(file_size) == int # noqa: E721 + assert file_size >= 0 + + result[f] = file_size + + return result + def init(self, initdb_params=None, cached=True, **kwargs): """ Perform initdb for this node. @@ -722,6 +744,22 @@ def slow_start(self, replica=False, dbname='template1', username=None, max_attem OperationalError}, max_attempts=max_attempts) + def _detect_port_conflict(self, log_files0, log_files1): + assert type(log_files0) == dict # noqa: E721 + assert type(log_files1) == dict # noqa: E721 + + for file in log_files1.keys(): + read_pos = 0 + + if file in log_files0.keys(): + read_pos = log_files0[file] # the previous size + + file_content = self.os_ops.read_binary(file, read_pos) + file_content_s = file_content.decode() + if 'Is another postmaster already running on port' in file_content_s: + return True + return False + def start(self, params=[], wait=True): """ Starts the PostgreSQL node using pg_ctl if node has not been started. @@ -745,27 +783,42 @@ def start(self, params=[], wait=True): "-w" if wait else '-W', # --wait or --no-wait "start"] + params # yapf: disable - startup_retries = 5 + log_files0 = self._collect_log_files() + assert type(log_files0) == dict # noqa: E721 + + nAttempt = 0 + timeout = 1 while True: + nAttempt += 1 try: exit_status, out, error = execute_utility(_params, self.utils_log_file, verbose=True) if error and 'does not exist' in error: raise Exception except Exception as e: - files = self._collect_special_files() - if any(len(file) > 1 and 'Is another postmaster already ' - 'running on port' in file[1].decode() for - file in files): - logging.warning("Detected an issue with connecting to port {0}. " - "Trying another port after a 5-second sleep...".format(self.port)) - self.port = reserve_port() - options = {'port': str(self.port)} - self.set_auto_conf(options) - startup_retries -= 1 - time.sleep(5) - continue + if self._should_free_port and nAttempt < 5: + log_files1 = self._collect_log_files() + if self._detect_port_conflict(log_files0, log_files1): + log_files0 = log_files1 + logging.warning( + "Detected an issue with connecting to port {0}. " + "Trying another port after a {1}-second sleep...".format(self.port, timeout) + ) + time.sleep(timeout) + timeout = min(2 * timeout, 5) + cur_port = self.port + new_port = utils.reserve_port() # throw + try: + options = {'port': str(new_port)} + self.set_auto_conf(options) + except: # noqa: E722 + utils.release_port(new_port) + raise + self.port = new_port + utils.release_port(cur_port) + continue msg = 'Cannot start node' + files = self._collect_special_files() raise_from(StartNodeException(msg, files), e) break self._maybe_start_logger() @@ -930,8 +983,10 @@ def free_port(self): """ if self._should_free_port: + port = self.port self._should_free_port = False - release_port(self.port) + self.port = None + utils.release_port(port) def cleanup(self, max_attempts=3, full=False): """ diff --git a/tests/test_simple.py b/tests/test_simple.py index 6b04f8bd..0a09135c 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -1064,6 +1064,123 @@ def test_the_same_port(self): self.assertIn("Cannot start node", str(ctx.exception)) + class tagPortManagerProxy: + sm_prev_testgres_reserve_port = None + sm_prev_testgres_release_port = None + + sm_DummyPortNumber = None + sm_DummyPortMaxUsage = None + + sm_DummyPortCurrentUsage = None + sm_DummyPortTotalUsage = None + + def __init__(self, dummyPortNumber, dummyPortMaxUsage): + assert type(dummyPortNumber) == int # noqa: E721 + assert type(dummyPortMaxUsage) == int # noqa: E721 + assert dummyPortNumber >= 0 + assert dummyPortMaxUsage >= 0 + + assert __class__.sm_prev_testgres_reserve_port is None + assert __class__.sm_prev_testgres_release_port is None + assert testgres.utils.reserve_port == testgres.utils.internal__reserve_port + assert testgres.utils.release_port == testgres.utils.internal__release_port + + __class__.sm_prev_testgres_reserve_port = testgres.utils.reserve_port + __class__.sm_prev_testgres_release_port = testgres.utils.release_port + + testgres.utils.reserve_port = __class__._proxy__reserve_port + testgres.utils.release_port = __class__._proxy__release_port + + assert testgres.utils.reserve_port == __class__._proxy__reserve_port + assert testgres.utils.release_port == __class__._proxy__release_port + + __class__.sm_DummyPortNumber = dummyPortNumber + __class__.sm_DummyPortMaxUsage = dummyPortMaxUsage + + __class__.sm_DummyPortCurrentUsage = 0 + __class__.sm_DummyPortTotalUsage = 0 + + def __enter__(self): + return self + + def __exit__(self, type, value, traceback): + assert __class__.sm_DummyPortCurrentUsage == 0 + + assert __class__.sm_prev_testgres_reserve_port is not None + assert __class__.sm_prev_testgres_release_port is not None + + assert testgres.utils.reserve_port == __class__._proxy__reserve_port + assert testgres.utils.release_port == __class__._proxy__release_port + + testgres.utils.reserve_port = __class__.sm_prev_testgres_reserve_port + testgres.utils.release_port = __class__.sm_prev_testgres_release_port + + __class__.sm_prev_testgres_reserve_port = None + __class__.sm_prev_testgres_release_port = None + + def _proxy__reserve_port(): + assert type(__class__.sm_DummyPortMaxUsage) == int # noqa: E721 + assert type(__class__.sm_DummyPortTotalUsage) == int # noqa: E721 + assert type(__class__.sm_DummyPortCurrentUsage) == int # noqa: E721 + assert __class__.sm_DummyPortTotalUsage >= 0 + assert __class__.sm_DummyPortCurrentUsage >= 0 + + assert __class__.sm_DummyPortTotalUsage <= __class__.sm_DummyPortMaxUsage + assert __class__.sm_DummyPortCurrentUsage <= __class__.sm_DummyPortTotalUsage + + assert __class__.sm_prev_testgres_reserve_port is not None + + if __class__.sm_DummyPortTotalUsage == __class__.sm_DummyPortMaxUsage: + return __class__.sm_prev_testgres_reserve_port() + + __class__.sm_DummyPortTotalUsage += 1 + __class__.sm_DummyPortCurrentUsage += 1 + return __class__.sm_DummyPortNumber + + def _proxy__release_port(dummyPortNumber): + assert type(dummyPortNumber) == int # noqa: E721 + + assert type(__class__.sm_DummyPortMaxUsage) == int # noqa: E721 + assert type(__class__.sm_DummyPortTotalUsage) == int # noqa: E721 + assert type(__class__.sm_DummyPortCurrentUsage) == int # noqa: E721 + assert __class__.sm_DummyPortTotalUsage >= 0 + assert __class__.sm_DummyPortCurrentUsage >= 0 + + assert __class__.sm_DummyPortTotalUsage <= __class__.sm_DummyPortMaxUsage + assert __class__.sm_DummyPortCurrentUsage <= __class__.sm_DummyPortTotalUsage + + assert __class__.sm_prev_testgres_release_port is not None + + if __class__.sm_DummyPortCurrentUsage > 0 and dummyPortNumber == __class__.sm_DummyPortNumber: + assert __class__.sm_DummyPortTotalUsage > 0 + __class__.sm_DummyPortCurrentUsage -= 1 + return + + return __class__.sm_prev_testgres_release_port(dummyPortNumber) + + def test_port_rereserve_during_node_start(self): + C_COUNT_OF_BAD_PORT_USAGE = 3 + + with get_new_node() as node1: + node1.init().start() + self.assertTrue(node1._should_free_port) + self.assertEqual(type(node1.port), int) # noqa: E721 + node1.safe_psql("SELECT 1;") + + with __class__.tagPortManagerProxy(node1.port, C_COUNT_OF_BAD_PORT_USAGE): + assert __class__.tagPortManagerProxy.sm_DummyPortNumber == node1.port + with get_new_node() as node2: + self.assertTrue(node2._should_free_port) + self.assertEqual(node2.port, node1.port) + + node2.init().start() + + self.assertNotEqual(node2.port, node1.port) + self.assertEqual(__class__.tagPortManagerProxy.sm_DummyPortCurrentUsage, 0) + self.assertEqual(__class__.tagPortManagerProxy.sm_DummyPortTotalUsage, C_COUNT_OF_BAD_PORT_USAGE) + + node2.safe_psql("SELECT 1;") + def test_simple_with_bin_dir(self): with get_new_node() as node: node.init().start() From 663612cb870fbeade43db3a0c6f771127fb650a2 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Sat, 14 Dec 2024 11:07:35 +0300 Subject: [PATCH 099/216] PostgresNode._C_MAX_START_ATEMPTS=5 is added (+ 1 new test) Also - TestgresTests.test_the_same_port is updated - TestgresTests.test_port_rereserve_during_node_start is updated - TestgresTests.test_port_conflict is added --- testgres/node.py | 14 +++++++++-- tests/test_simple.py | 58 ++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 68 insertions(+), 4 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index 7f5aa648..554c226d 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -128,6 +128,9 @@ def __repr__(self): class PostgresNode(object): + # a max number of node start attempts + _C_MAX_START_ATEMPTS = 5 + def __init__(self, name=None, base_dir=None, port=None, conn_params: ConnectionParams = ConnectionParams(), bin_dir=None, prefix=None): """ PostgresNode constructor. @@ -774,6 +777,9 @@ def start(self, params=[], wait=True): Returns: This instance of :class:`.PostgresNode`. """ + + assert __class__._C_MAX_START_ATEMPTS > 1 + if self.is_started: return self @@ -789,13 +795,17 @@ def start(self, params=[], wait=True): nAttempt = 0 timeout = 1 while True: + assert nAttempt >= 0 + assert nAttempt < __class__._C_MAX_START_ATEMPTS nAttempt += 1 try: exit_status, out, error = execute_utility(_params, self.utils_log_file, verbose=True) if error and 'does not exist' in error: raise Exception except Exception as e: - if self._should_free_port and nAttempt < 5: + assert nAttempt > 0 + assert nAttempt <= __class__._C_MAX_START_ATEMPTS + if self._should_free_port and nAttempt < __class__._C_MAX_START_ATEMPTS: log_files1 = self._collect_log_files() if self._detect_port_conflict(log_files0, log_files1): log_files0 = log_files1 @@ -806,7 +816,7 @@ def start(self, params=[], wait=True): time.sleep(timeout) timeout = min(2 * timeout, 5) cur_port = self.port - new_port = utils.reserve_port() # throw + new_port = utils.reserve_port() # can raise try: options = {'port': str(new_port)} self.set_auto_conf(options) diff --git a/tests/test_simple.py b/tests/test_simple.py index 0a09135c..93968466 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -1053,6 +1053,8 @@ def test_the_same_port(self): node.init().start() self.assertTrue(node._should_free_port) self.assertEqual(type(node.port), int) + node_port_copy = node.port + self.assertEqual(node.safe_psql("SELECT 1;"), b'1\n') with get_new_node(port=node.port) as node2: self.assertEqual(type(node2.port), int) @@ -1064,6 +1066,11 @@ def test_the_same_port(self): self.assertIn("Cannot start node", str(ctx.exception)) + # node is still working + self.assertEqual(node.port, node_port_copy) + self.assertTrue(node._should_free_port) + self.assertEqual(node.safe_psql("SELECT 3;"), b'3\n') + class tagPortManagerProxy: sm_prev_testgres_reserve_port = None sm_prev_testgres_release_port = None @@ -1159,13 +1166,16 @@ def _proxy__release_port(dummyPortNumber): return __class__.sm_prev_testgres_release_port(dummyPortNumber) def test_port_rereserve_during_node_start(self): + assert testgres.PostgresNode._C_MAX_START_ATEMPTS == 5 + C_COUNT_OF_BAD_PORT_USAGE = 3 with get_new_node() as node1: node1.init().start() self.assertTrue(node1._should_free_port) self.assertEqual(type(node1.port), int) # noqa: E721 - node1.safe_psql("SELECT 1;") + node1_port_copy = node1.port + self.assertEqual(node1.safe_psql("SELECT 1;"), b'1\n') with __class__.tagPortManagerProxy(node1.port, C_COUNT_OF_BAD_PORT_USAGE): assert __class__.tagPortManagerProxy.sm_DummyPortNumber == node1.port @@ -1176,10 +1186,54 @@ def test_port_rereserve_during_node_start(self): node2.init().start() self.assertNotEqual(node2.port, node1.port) + self.assertTrue(node2._should_free_port) self.assertEqual(__class__.tagPortManagerProxy.sm_DummyPortCurrentUsage, 0) self.assertEqual(__class__.tagPortManagerProxy.sm_DummyPortTotalUsage, C_COUNT_OF_BAD_PORT_USAGE) + self.assertTrue(node2.is_started) + + self.assertEqual(node2.safe_psql("SELECT 2;"), b'2\n') + + # node1 is still working + self.assertEqual(node1.port, node1_port_copy) + self.assertTrue(node1._should_free_port) + self.assertEqual(node1.safe_psql("SELECT 3;"), b'3\n') + + def test_port_conflict(self): + assert testgres.PostgresNode._C_MAX_START_ATEMPTS > 1 + + C_COUNT_OF_BAD_PORT_USAGE = testgres.PostgresNode._C_MAX_START_ATEMPTS + + with get_new_node() as node1: + node1.init().start() + self.assertTrue(node1._should_free_port) + self.assertEqual(type(node1.port), int) # noqa: E721 + node1_port_copy = node1.port + self.assertEqual(node1.safe_psql("SELECT 1;"), b'1\n') + + with __class__.tagPortManagerProxy(node1.port, C_COUNT_OF_BAD_PORT_USAGE): + assert __class__.tagPortManagerProxy.sm_DummyPortNumber == node1.port + with get_new_node() as node2: + self.assertTrue(node2._should_free_port) + self.assertEqual(node2.port, node1.port) + + with self.assertRaises(StartNodeException) as ctx: + node2.init().start() + + self.assertIn("Cannot start node", str(ctx.exception)) + + self.assertEqual(node2.port, node1.port) + self.assertTrue(node2._should_free_port) + self.assertEqual(__class__.tagPortManagerProxy.sm_DummyPortCurrentUsage, 1) + self.assertEqual(__class__.tagPortManagerProxy.sm_DummyPortTotalUsage, C_COUNT_OF_BAD_PORT_USAGE) + self.assertFalse(node2.is_started) + + # node2 must release our dummyPort (node1.port) + self.assertEqual(__class__.tagPortManagerProxy.sm_DummyPortCurrentUsage, 0) - node2.safe_psql("SELECT 1;") + # node1 is still working + self.assertEqual(node1.port, node1_port_copy) + self.assertTrue(node1._should_free_port) + self.assertEqual(node1.safe_psql("SELECT 3;"), b'3\n') def test_simple_with_bin_dir(self): with get_new_node() as node: From 2f3fc40d39562ffa62e77863df8316becd353dee Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Wed, 18 Dec 2024 21:54:32 +0300 Subject: [PATCH 100/216] OsOperations.isdir is added It a correction of base interface - OsOperations. It seems we forgot to add a declaration of the "abstract" method "isdir". --- testgres/operations/os_ops.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/testgres/operations/os_ops.py b/testgres/operations/os_ops.py index 2ab41246..d644509a 100644 --- a/testgres/operations/os_ops.py +++ b/testgres/operations/os_ops.py @@ -107,6 +107,9 @@ def read_binary(self, filename, start_pos): def isfile(self, remote_file): raise NotImplementedError() + def isdir(self, dirname): + raise NotImplementedError() + def get_file_size(self, filename): raise NotImplementedError() From 9f527435b28b3c9655d66ef7a2b5a120d767786d Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Wed, 18 Dec 2024 21:57:37 +0300 Subject: [PATCH 101/216] Tests for LocalOperations methods isdir and isfile are added --- tests/test_local.py | 64 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/tests/test_local.py b/tests/test_local.py index a8a0bde0..e223b090 100644 --- a/tests/test_local.py +++ b/tests/test_local.py @@ -118,3 +118,67 @@ def test_get_file_size__unk_file(self): with pytest.raises(FileNotFoundError, match=re.escape("[Errno 2] No such file or directory: '/dummy'")): self.operations.get_file_size("/dummy") + + def test_isfile_true(self): + """ + Test isfile for an existing file. + """ + filename = __file__ + + response = self.operations.isfile(filename) + + assert response is True + + def test_isfile_false__not_exist(self): + """ + Test isfile for a non-existing file. + """ + filename = os.path.join(os.path.dirname(__file__), "nonexistent_file.txt") + + response = self.operations.isfile(filename) + + assert response is False + + def test_isfile_false__directory(self): + """ + Test isfile for a firectory. + """ + name = os.path.dirname(__file__) + + assert self.operations.isdir(name) + + response = self.operations.isfile(name) + + assert response is False + + def test_isdir_true(self): + """ + Test isdir for an existing directory. + """ + name = os.path.dirname(__file__) + + response = self.operations.isdir(name) + + assert response is True + + def test_isdir_false__not_exist(self): + """ + Test isdir for a non-existing directory. + """ + name = os.path.join(os.path.dirname(__file__), "it_is_nonexistent_directory") + + response = self.operations.isdir(name) + + assert response is False + + def test_isdir_false__file(self): + """ + Test isdir for a file. + """ + name = __file__ + + assert self.operations.isfile(name) + + response = self.operations.isdir(name) + + assert response is False From 38ce127ec63189d4a35d1f73232435538329ef93 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Wed, 18 Dec 2024 21:58:30 +0300 Subject: [PATCH 102/216] Tests for RemoteOperations methods isdir and isfile are updated --- tests/test_remote.py | 50 +++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 47 insertions(+), 3 deletions(-) diff --git a/tests/test_remote.py b/tests/test_remote.py index be1a56bb..67d66549 100755 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -259,18 +259,62 @@ def test_isfile_true(self): """ Test isfile for an existing file. """ - filename = "/etc/hosts" + filename = __file__ response = self.operations.isfile(filename) assert response is True - def test_isfile_false(self): + def test_isfile_false__not_exist(self): """ Test isfile for a non-existing file. """ - filename = "/nonexistent_file.txt" + filename = os.path.join(os.path.dirname(__file__), "nonexistent_file.txt") response = self.operations.isfile(filename) assert response is False + + def test_isfile_false__directory(self): + """ + Test isfile for a firectory. + """ + name = os.path.dirname(__file__) + + assert self.operations.isdir(name) + + response = self.operations.isfile(name) + + assert response is False + + def test_isdir_true(self): + """ + Test isdir for an existing directory. + """ + name = os.path.dirname(__file__) + + response = self.operations.isdir(name) + + assert response is True + + def test_isdir_false__not_exist(self): + """ + Test isdir for a non-existing directory. + """ + name = os.path.join(os.path.dirname(__file__), "it_is_nonexistent_directory") + + response = self.operations.isdir(name) + + assert response is False + + def test_isdir_false__file(self): + """ + Test isdir for a file. + """ + name = __file__ + + assert self.operations.isfile(name) + + response = self.operations.isdir(name) + + assert response is False From 93d1122d0f59575a5b7f7b316f18d2824ecacc62 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 24 Dec 2024 11:51:52 +0300 Subject: [PATCH 103/216] Formatting --- testgres/node.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/testgres/node.py b/testgres/node.py index 554c226d..dff47cf6 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -131,7 +131,8 @@ class PostgresNode(object): # a max number of node start attempts _C_MAX_START_ATEMPTS = 5 - def __init__(self, name=None, base_dir=None, port=None, conn_params: ConnectionParams = ConnectionParams(), bin_dir=None, prefix=None): + def __init__(self, name=None, base_dir=None, port=None, conn_params: ConnectionParams = ConnectionParams(), + bin_dir=None, prefix=None): """ PostgresNode constructor. From 00fd9257158cd9b0aa7450b66762ad9c2c31f08b Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 24 Dec 2024 14:14:49 +0300 Subject: [PATCH 104/216] Node.start is refactored [Victoria Shepard' ideas are used, #149] - Save an orignal text of 'does not exist' error - When we reach maximum retry attempt of restarts - We log an error message - We raise exception "Cannot start node after multiple attempts" - A new port number is not tranlating into string - Reorganization TestgresTests.test_port_conflict is updated. --- testgres/node.py | 100 +++++++++++++++++++++++++++---------------- tests/test_simple.py | 2 +- 2 files changed, 63 insertions(+), 39 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index dff47cf6..7121339f 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -790,48 +790,72 @@ def start(self, params=[], wait=True): "-w" if wait else '-W', # --wait or --no-wait "start"] + params # yapf: disable - log_files0 = self._collect_log_files() - assert type(log_files0) == dict # noqa: E721 + def LOCAL__start_node(): + _, _, error = execute_utility(_params, self.utils_log_file, verbose=True) + assert type(error) == str # noqa: E721 + if error and 'does not exist' in error: + raise Exception(error) - nAttempt = 0 - timeout = 1 - while True: - assert nAttempt >= 0 - assert nAttempt < __class__._C_MAX_START_ATEMPTS - nAttempt += 1 + def LOCAL__raise_cannot_start_node(from_exception, msg): + assert isinstance(from_exception, Exception) + assert type(msg) == str # noqa: E721 + files = self._collect_special_files() + raise_from(StartNodeException(msg, files), from_exception) + + def LOCAL__raise_cannot_start_node__std(from_exception): + assert isinstance(from_exception, Exception) + LOCAL__raise_cannot_start_node(from_exception, 'Cannot start node') + + if not self._should_free_port: try: - exit_status, out, error = execute_utility(_params, self.utils_log_file, verbose=True) - if error and 'does not exist' in error: - raise Exception + LOCAL__start_node() except Exception as e: - assert nAttempt > 0 - assert nAttempt <= __class__._C_MAX_START_ATEMPTS - if self._should_free_port and nAttempt < __class__._C_MAX_START_ATEMPTS: + LOCAL__raise_cannot_start_node__std(e) + else: + assert self._should_free_port + assert __class__._C_MAX_START_ATEMPTS > 1 + + log_files0 = self._collect_log_files() + assert type(log_files0) == dict # noqa: E721 + + nAttempt = 0 + timeout = 1 + while True: + assert nAttempt >= 0 + assert nAttempt < __class__._C_MAX_START_ATEMPTS + nAttempt += 1 + try: + LOCAL__start_node() + except Exception as e: + assert nAttempt > 0 + assert nAttempt <= __class__._C_MAX_START_ATEMPTS + if nAttempt == __class__._C_MAX_START_ATEMPTS: + logging.error("Reached maximum retry attempts. Unable to start node.") + LOCAL__raise_cannot_start_node(e, "Cannot start node after multiple attempts") + log_files1 = self._collect_log_files() - if self._detect_port_conflict(log_files0, log_files1): - log_files0 = log_files1 - logging.warning( - "Detected an issue with connecting to port {0}. " - "Trying another port after a {1}-second sleep...".format(self.port, timeout) - ) - time.sleep(timeout) - timeout = min(2 * timeout, 5) - cur_port = self.port - new_port = utils.reserve_port() # can raise - try: - options = {'port': str(new_port)} - self.set_auto_conf(options) - except: # noqa: E722 - utils.release_port(new_port) - raise - self.port = new_port - utils.release_port(cur_port) - continue - - msg = 'Cannot start node' - files = self._collect_special_files() - raise_from(StartNodeException(msg, files), e) - break + if not self._detect_port_conflict(log_files0, log_files1): + LOCAL__raise_cannot_start_node__std(e) + + log_files0 = log_files1 + logging.warning( + "Detected a conflict with using the port {0}. " + "Trying another port after a {1}-second sleep...".format(self.port, timeout) + ) + time.sleep(timeout) + timeout = min(2 * timeout, 5) + cur_port = self.port + new_port = utils.reserve_port() # can raise + try: + options = {'port': str(new_port)} + self.set_auto_conf(options) + except: # noqa: E722 + utils.release_port(new_port) + raise + self.port = new_port + utils.release_port(cur_port) + continue + break self._maybe_start_logger() self.is_started = True return self diff --git a/tests/test_simple.py b/tests/test_simple.py index 93968466..9cf29c64 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -1219,7 +1219,7 @@ def test_port_conflict(self): with self.assertRaises(StartNodeException) as ctx: node2.init().start() - self.assertIn("Cannot start node", str(ctx.exception)) + self.assertIn("Cannot start node after multiple attempts", str(ctx.exception)) self.assertEqual(node2.port, node1.port) self.assertTrue(node2._should_free_port) From 2fa4426095acda3caadbdd47f6b029d699c0a0aa Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 24 Dec 2024 17:19:04 +0300 Subject: [PATCH 105/216] Formatting --- testgres/operations/remote_ops.py | 1 - 1 file changed, 1 deletion(-) diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index 9f88140c..3aa2d8c4 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -39,7 +39,6 @@ def cmdline(self): class RemoteOperations(OsOperations): def __init__(self, conn_params: ConnectionParams): - if not platform.system().lower() == "linux": raise EnvironmentError("Remote operations are supported only on Linux!") From a33860d0cb625516535cf5814cac2e89562a5f70 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 24 Dec 2024 20:47:20 +0300 Subject: [PATCH 106/216] PSQL passes a database name through the explicit '-d ' parameter --- testgres/node.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index 7121339f..f7d6e839 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -1107,13 +1107,12 @@ def _psql( else: raise Exception("Input data must be None or bytes.") - dbname = dbname or default_dbname() - psql_params = [ self._get_bin_path("psql"), "-p", str(self.port), "-h", self.host, "-U", username or self.os_ops.username, + "-d", dbname or default_dbname(), "-X", # no .psqlrc "-A", # unaligned output "-t", # print rows only @@ -1135,9 +1134,6 @@ def _psql( else: raise QueryException('Query or filename must be provided') - # should be the last one - psql_params.append(dbname) - return self.os_ops.exec_command( psql_params, verbose=True, From 3ca3ff7eef2687c717c3cbcedc04f37b98ec7ed9 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 24 Dec 2024 22:43:39 +0300 Subject: [PATCH 107/216] Ssh command line in RemoteOperations::execute is corrected We use subprocess.list2cmdline(cmd) to pack a user command line. It allows PostgresNode::_psql to build PSQL command line without a "special case" for remote-host. Also RemoteOperations::execute raises exception if 'cmd' parameter has an unknown type. --- testgres/node.py | 5 +---- testgres/operations/remote_ops.py | 5 ++++- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index f7d6e839..3d023399 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -1125,10 +1125,7 @@ def _psql( # select query source if query: - if self.os_ops.remote: - psql_params.extend(("-c", '"{}"'.format(query))) - else: - psql_params.extend(("-c", query)) + psql_params.extend(("-c", query)) elif filename: psql_params.extend(("-f", filename)) else: diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index 3aa2d8c4..fb5dd4b2 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -80,7 +80,10 @@ def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False, if isinstance(cmd, str): ssh_cmd = ['ssh', self.ssh_dest] + self.ssh_args + [cmd] elif isinstance(cmd, list): - ssh_cmd = ['ssh', self.ssh_dest] + self.ssh_args + cmd + ssh_cmd = ['ssh', self.ssh_dest] + self.ssh_args + [subprocess.list2cmdline(cmd)] + else: + raise ValueError("Invalid 'cmd' argument type - {0}".format(type(cmd).__name__)) + process = subprocess.Popen(ssh_cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) assert not (process is None) if get_process: From 7c0f1846205c89acc96139fca50b24b76dad4a28 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Wed, 25 Dec 2024 12:33:38 +0300 Subject: [PATCH 108/216] RemoteOperations is updated (read_binary, get_file_size) get_file_size and get_file_size use a list for command list arguments. It allows to use standard way to escape a filename. Our bicycle "_escape_path" is deleted. --- testgres/operations/remote_ops.py | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index fb5dd4b2..128a2a21 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -347,7 +347,7 @@ def read_binary(self, filename, start_pos): assert type(start_pos) == int # noqa: E721 assert start_pos >= 0 - cmd = "tail -c +{} {}".format(start_pos + 1, __class__._escape_path(filename)) + cmd = ["tail", "-c", "+{}".format(start_pos + 1), filename] r = self.exec_command(cmd) assert type(r) == bytes # noqa: E721 return r @@ -367,7 +367,7 @@ def get_file_size(self, filename): assert filename is not None assert type(filename) == str # noqa: E721 - cmd = "du -b " + __class__._escape_path(filename) + cmd = ["du", "-b", filename] s = self.exec_command(cmd, encoding=get_default_encoding()) assert type(s) == str # noqa: E721 @@ -462,15 +462,6 @@ def db_connect(self, dbname, user, password=None, host="localhost", port=5432): ) return conn - def _escape_path(path): - assert type(path) == str # noqa: E721 - assert path != "" # Ok? - - r = "'" - r += path - r += "'" - return r - def normalize_error(error): if isinstance(error, bytes): From 5b263f3c6231a98de7a72c1bf9445989e00a7f62 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Wed, 25 Dec 2024 13:43:25 +0300 Subject: [PATCH 109/216] Comments are corrected --- tests/test_remote.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_remote.py b/tests/test_remote.py index 67d66549..780ad46e 100755 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -226,7 +226,7 @@ def test_read_binary__spec__unk_file(self): def test_get_file_size(self): """ - Test LocalOperations::get_file_size. + Test RemoteOperations::get_file_size. """ filename = __file__ # current file @@ -239,7 +239,7 @@ def test_get_file_size(self): def test_get_file_size__unk_file(self): """ - Test LocalOperations::get_file_size. + Test RemoteOperations::get_file_size. """ with pytest.raises(ExecUtilException, match=re.escape("du: cannot access '/dummy': No such file or directory")): From 4e23e030cfb8b2531cae5abfd4f471809e7f7371 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Wed, 25 Dec 2024 13:50:05 +0300 Subject: [PATCH 110/216] OsOps::read methods were corrected Now they always read a file as binary. When 'binary' parameter is False we will use 'encoding' parameter to decode bytes into string. Binary read does not allow an usage of 'encoding' parameter (InvalidOperationException is raised). New tests are added. --- testgres/operations/local_ops.py | 36 ++++++++++++++--- testgres/operations/remote_ops.py | 35 +++++++++++++--- tests/test_local.py | 66 ++++++++++++++++++++++++++++++- tests/test_remote.py | 64 ++++++++++++++++++++++++++++++ 4 files changed, 189 insertions(+), 12 deletions(-) diff --git a/testgres/operations/local_ops.py b/testgres/operations/local_ops.py index 82d1711d..d6013ab5 100644 --- a/testgres/operations/local_ops.py +++ b/testgres/operations/local_ops.py @@ -10,6 +10,7 @@ import psutil from ..exceptions import ExecUtilException +from ..exceptions import InvalidOperationException from .os_ops import ConnectionParams, OsOperations, pglib, get_default_encoding from .raise_error import RaiseError from .helpers import Helpers @@ -266,13 +267,36 @@ def touch(self, filename): os.utime(filename, None) def read(self, filename, encoding=None, binary=False): - mode = "rb" if binary else "r" - with open(filename, mode) as file: + assert type(filename) == str # noqa: E721 + assert encoding is None or type(encoding) == str # noqa: E721 + assert type(binary) == bool # noqa: E721 + + if binary: + if encoding is not None: + raise InvalidOperationException("Enconding is not allowed for read binary operation") + + return self._read__binary(filename) + + # python behavior + assert None or "abc" == "abc" + assert "" or "abc" == "abc" + + return self._read__text_with_encoding(filename, encoding or get_default_encoding()) + + def _read__text_with_encoding(self, filename, encoding): + assert type(filename) == str # noqa: E721 + assert type(encoding) == str # noqa: E721 + content = self._read__binary(filename) + assert type(content) == bytes # noqa: E721 + content_s = content.decode(encoding) + assert type(content_s) == str # noqa: E721 + return content_s + + def _read__binary(self, filename): + assert type(filename) == str # noqa: E721 + with open(filename, 'rb') as file: # open in a binary mode content = file.read() - if binary: - return content - if isinstance(content, bytes): - return content.decode(encoding or get_default_encoding()) + assert type(content) == bytes # noqa: E721 return content def readlines(self, filename, num_lines=0, binary=False, encoding=None): diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index 128a2a21..abcb8fe1 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -14,6 +14,7 @@ raise ImportError("You must have psycopg2 or pg8000 modules installed") from ..exceptions import ExecUtilException +from ..exceptions import InvalidOperationException from .os_ops import OsOperations, ConnectionParams, get_default_encoding from .raise_error import RaiseError from .helpers import Helpers @@ -319,13 +320,37 @@ def touch(self, filename): self.exec_command("touch {}".format(filename)) def read(self, filename, binary=False, encoding=None): - cmd = "cat {}".format(filename) - result = self.exec_command(cmd, encoding=encoding) + assert type(filename) == str # noqa: E721 + assert encoding is None or type(encoding) == str # noqa: E721 + assert type(binary) == bool # noqa: E721 - if not binary and result: - result = result.decode(encoding or get_default_encoding()) + if binary: + if encoding is not None: + raise InvalidOperationException("Enconding is not allowed for read binary operation") - return result + return self._read__binary(filename) + + # python behavior + assert None or "abc" == "abc" + assert "" or "abc" == "abc" + + return self._read__text_with_encoding(filename, encoding or get_default_encoding()) + + def _read__text_with_encoding(self, filename, encoding): + assert type(filename) == str # noqa: E721 + assert type(encoding) == str # noqa: E721 + content = self._read__binary(filename) + assert type(content) == bytes # noqa: E721 + content_s = content.decode(encoding) + assert type(content_s) == str # noqa: E721 + return content_s + + def _read__binary(self, filename): + assert type(filename) == str # noqa: E721 + cmd = ["cat", filename] + content = self.exec_command(cmd) + assert type(content) == bytes # noqa: E721 + return content def readlines(self, filename, num_lines=0, binary=False, encoding=None): if num_lines > 0: diff --git a/tests/test_local.py b/tests/test_local.py index e223b090..47a63994 100644 --- a/tests/test_local.py +++ b/tests/test_local.py @@ -4,6 +4,7 @@ import re from testgres import ExecUtilException +from testgres import InvalidOperationException from testgres import LocalOperations from .helpers.run_conditions import RunConditions @@ -56,6 +57,67 @@ def test_exec_command_failure__expect_error(self): assert exit_status == 127 assert result == b'' + def test_read__text(self): + """ + Test LocalOperations::read for text data. + """ + filename = __file__ # current file + + with open(filename, 'r') as file: # open in a text mode + response0 = file.read() + + assert type(response0) == str # noqa: E721 + + response1 = self.operations.read(filename) + assert type(response1) == str # noqa: E721 + assert response1 == response0 + + response2 = self.operations.read(filename, encoding=None, binary=False) + assert type(response2) == str # noqa: E721 + assert response2 == response0 + + response3 = self.operations.read(filename, encoding="") + assert type(response3) == str # noqa: E721 + assert response3 == response0 + + response4 = self.operations.read(filename, encoding="UTF-8") + assert type(response4) == str # noqa: E721 + assert response4 == response0 + + def test_read__binary(self): + """ + Test LocalOperations::read for binary data. + """ + filename = __file__ # current file + + with open(filename, 'rb') as file: # open in a binary mode + response0 = file.read() + + assert type(response0) == bytes # noqa: E721 + + response1 = self.operations.read(filename, binary=True) + assert type(response1) == bytes # noqa: E721 + assert response1 == response0 + + def test_read__binary_and_encoding(self): + """ + Test LocalOperations::read for binary data and encoding. + """ + filename = __file__ # current file + + with pytest.raises( + InvalidOperationException, + match=re.escape("Enconding is not allowed for read binary operation")): + self.operations.read(filename, encoding="", binary=True) + + def test_read__unknown_file(self): + """ + Test LocalOperations::read with unknown file. + """ + + with pytest.raises(FileNotFoundError, match=re.escape("[Errno 2] No such file or directory: '/dummy'")): + self.operations.read("/dummy") + def test_read_binary__spec(self): """ Test LocalOperations::read_binary. @@ -95,7 +157,9 @@ def test_read_binary__spec__unk_file(self): Test LocalOperations::read_binary with unknown file. """ - with pytest.raises(FileNotFoundError, match=re.escape("[Errno 2] No such file or directory: '/dummy'")): + with pytest.raises( + FileNotFoundError, + match=re.escape("[Errno 2] No such file or directory: '/dummy'")): self.operations.read_binary("/dummy", 0) def test_get_file_size(self): diff --git a/tests/test_remote.py b/tests/test_remote.py index 780ad46e..7421ca3a 100755 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -4,6 +4,7 @@ import re from testgres import ExecUtilException +from testgres import InvalidOperationException from testgres import RemoteOperations from testgres import ConnectionParams @@ -182,6 +183,69 @@ def test_read_binary_file(self): assert isinstance(response, bytes) + def test_read__text(self): + """ + Test RemoteOperations::read for text data. + """ + filename = __file__ # current file + + with open(filename, 'r') as file: # open in a text mode + response0 = file.read() + + assert type(response0) == str # noqa: E721 + + response1 = self.operations.read(filename) + assert type(response1) == str # noqa: E721 + assert response1 == response0 + + response2 = self.operations.read(filename, encoding=None, binary=False) + assert type(response2) == str # noqa: E721 + assert response2 == response0 + + response3 = self.operations.read(filename, encoding="") + assert type(response3) == str # noqa: E721 + assert response3 == response0 + + response4 = self.operations.read(filename, encoding="UTF-8") + assert type(response4) == str # noqa: E721 + assert response4 == response0 + + def test_read__binary(self): + """ + Test RemoteOperations::read for binary data. + """ + filename = __file__ # current file + + with open(filename, 'rb') as file: # open in a binary mode + response0 = file.read() + + assert type(response0) == bytes # noqa: E721 + + response1 = self.operations.read(filename, binary=True) + assert type(response1) == bytes # noqa: E721 + assert response1 == response0 + + def test_read__binary_and_encoding(self): + """ + Test RemoteOperations::read for binary data and encoding. + """ + filename = __file__ # current file + + with pytest.raises( + InvalidOperationException, + match=re.escape("Enconding is not allowed for read binary operation")): + self.operations.read(filename, encoding="", binary=True) + + def test_read__unknown_file(self): + """ + Test RemoteOperations::read with unknown file. + """ + + with pytest.raises( + ExecUtilException, + match=re.escape("cat: /dummy: No such file or directory")): + self.operations.read("/dummy") + def test_read_binary__spec(self): """ Test RemoteOperations::read_binary. From 28c91c2e2670f74bae5beaa18a6f3a30f0f0dfa6 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Wed, 25 Dec 2024 15:11:07 +0300 Subject: [PATCH 111/216] [BUG FIX] xxxOperations::_read__text_with_encoding opens a file as text LocalOps uses "open(filename, mode='r', encoding=encoding)" RemoteOps uses "io.TextIOWrapper(io.BytesIO(binaryData), encoding=encoding)" It solves a problem on Windows. --- testgres/operations/local_ops.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/testgres/operations/local_ops.py b/testgres/operations/local_ops.py index d6013ab5..e1c3b9fd 100644 --- a/testgres/operations/local_ops.py +++ b/testgres/operations/local_ops.py @@ -286,11 +286,10 @@ def read(self, filename, encoding=None, binary=False): def _read__text_with_encoding(self, filename, encoding): assert type(filename) == str # noqa: E721 assert type(encoding) == str # noqa: E721 - content = self._read__binary(filename) - assert type(content) == bytes # noqa: E721 - content_s = content.decode(encoding) - assert type(content_s) == str # noqa: E721 - return content_s + with open(filename, mode='r', encoding=encoding) as file: # open in a text mode + content = file.read() + assert type(content) == str # noqa: E721 + return content def _read__binary(self, filename): assert type(filename) == str # noqa: E721 From 2679646fd6f12038887db3d123fc6bdc8d7fb6b8 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Wed, 25 Dec 2024 16:15:00 +0300 Subject: [PATCH 112/216] [BUG FIX] Part for RemoteOps changes... --- testgres/operations/remote_ops.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index abcb8fe1..bb00cfaf 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -3,6 +3,7 @@ import platform import subprocess import tempfile +import io # we support both pg8000 and psycopg2 try: @@ -341,7 +342,9 @@ def _read__text_with_encoding(self, filename, encoding): assert type(encoding) == str # noqa: E721 content = self._read__binary(filename) assert type(content) == bytes # noqa: E721 - content_s = content.decode(encoding) + buf0 = io.BytesIO(content) + buf1 = io.TextIOWrapper(buf0, encoding=encoding) + content_s = buf1.read() assert type(content_s) == str # noqa: E721 return content_s From 6c514bfe0308e912556105db0029541449af44b3 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Wed, 25 Dec 2024 16:15:57 +0300 Subject: [PATCH 113/216] Code normalization --- testgres/operations/local_ops.py | 4 ++-- testgres/operations/remote_ops.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/testgres/operations/local_ops.py b/testgres/operations/local_ops.py index e1c3b9fd..c88c16ca 100644 --- a/testgres/operations/local_ops.py +++ b/testgres/operations/local_ops.py @@ -278,8 +278,8 @@ def read(self, filename, encoding=None, binary=False): return self._read__binary(filename) # python behavior - assert None or "abc" == "abc" - assert "" or "abc" == "abc" + assert (None or "abc") == "abc" + assert ("" or "abc") == "abc" return self._read__text_with_encoding(filename, encoding or get_default_encoding()) diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index bb00cfaf..c0307195 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -332,8 +332,8 @@ def read(self, filename, binary=False, encoding=None): return self._read__binary(filename) # python behavior - assert None or "abc" == "abc" - assert "" or "abc" == "abc" + assert (None or "abc") == "abc" + assert ("" or "abc") == "abc" return self._read__text_with_encoding(filename, encoding or get_default_encoding()) From 0139bbd858d1fe4bf6566ff6c3e633c32b592063 Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Wed, 25 Dec 2024 16:57:42 +0300 Subject: [PATCH 114/216] [windows] TestgresTests is updated - test_the_same_port - test_port_rereserve_during_node_start - test_port_conflict - use rm_carriage_returns --- tests/test_simple.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/test_simple.py b/tests/test_simple.py index 9cf29c64..8148d05d 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -1054,7 +1054,7 @@ def test_the_same_port(self): self.assertTrue(node._should_free_port) self.assertEqual(type(node.port), int) node_port_copy = node.port - self.assertEqual(node.safe_psql("SELECT 1;"), b'1\n') + self.assertEqual(rm_carriage_returns(node.safe_psql("SELECT 1;")), b'1\n') with get_new_node(port=node.port) as node2: self.assertEqual(type(node2.port), int) @@ -1069,7 +1069,7 @@ def test_the_same_port(self): # node is still working self.assertEqual(node.port, node_port_copy) self.assertTrue(node._should_free_port) - self.assertEqual(node.safe_psql("SELECT 3;"), b'3\n') + self.assertEqual(rm_carriage_returns(node.safe_psql("SELECT 3;")), b'3\n') class tagPortManagerProxy: sm_prev_testgres_reserve_port = None @@ -1175,7 +1175,7 @@ def test_port_rereserve_during_node_start(self): self.assertTrue(node1._should_free_port) self.assertEqual(type(node1.port), int) # noqa: E721 node1_port_copy = node1.port - self.assertEqual(node1.safe_psql("SELECT 1;"), b'1\n') + self.assertEqual(rm_carriage_returns(node1.safe_psql("SELECT 1;")), b'1\n') with __class__.tagPortManagerProxy(node1.port, C_COUNT_OF_BAD_PORT_USAGE): assert __class__.tagPortManagerProxy.sm_DummyPortNumber == node1.port @@ -1191,12 +1191,12 @@ def test_port_rereserve_during_node_start(self): self.assertEqual(__class__.tagPortManagerProxy.sm_DummyPortTotalUsage, C_COUNT_OF_BAD_PORT_USAGE) self.assertTrue(node2.is_started) - self.assertEqual(node2.safe_psql("SELECT 2;"), b'2\n') + self.assertEqual(rm_carriage_returns(node2.safe_psql("SELECT 2;")), b'2\n') # node1 is still working self.assertEqual(node1.port, node1_port_copy) self.assertTrue(node1._should_free_port) - self.assertEqual(node1.safe_psql("SELECT 3;"), b'3\n') + self.assertEqual(rm_carriage_returns(node1.safe_psql("SELECT 3;")), b'3\n') def test_port_conflict(self): assert testgres.PostgresNode._C_MAX_START_ATEMPTS > 1 @@ -1208,7 +1208,7 @@ def test_port_conflict(self): self.assertTrue(node1._should_free_port) self.assertEqual(type(node1.port), int) # noqa: E721 node1_port_copy = node1.port - self.assertEqual(node1.safe_psql("SELECT 1;"), b'1\n') + self.assertEqual(rm_carriage_returns(node1.safe_psql("SELECT 1;")), b'1\n') with __class__.tagPortManagerProxy(node1.port, C_COUNT_OF_BAD_PORT_USAGE): assert __class__.tagPortManagerProxy.sm_DummyPortNumber == node1.port @@ -1233,7 +1233,7 @@ def test_port_conflict(self): # node1 is still working self.assertEqual(node1.port, node1_port_copy) self.assertTrue(node1._should_free_port) - self.assertEqual(node1.safe_psql("SELECT 3;"), b'3\n') + self.assertEqual(rm_carriage_returns(node1.safe_psql("SELECT 3;")), b'3\n') def test_simple_with_bin_dir(self): with get_new_node() as node: From 660ab62d96cda35d19ac69db556a986e4a241297 Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Wed, 25 Dec 2024 16:59:44 +0300 Subject: [PATCH 115/216] [windows] PostgresNode.start (LOCAL__start_node) is corrected [BUG FIX] execute_utility may return None in error. --- testgres/node.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testgres/node.py b/testgres/node.py index 3d023399..baf532de 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -792,7 +792,7 @@ def start(self, params=[], wait=True): def LOCAL__start_node(): _, _, error = execute_utility(_params, self.utils_log_file, verbose=True) - assert type(error) == str # noqa: E721 + assert error is None or type(error) == str # noqa: E721 if error and 'does not exist' in error: raise Exception(error) From 6465b4183f63d82edbbb24ab9a41fa36da551185 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Wed, 25 Dec 2024 17:34:35 +0300 Subject: [PATCH 116/216] A comment is added --- testgres/node.py | 1 + 1 file changed, 1 insertion(+) diff --git a/testgres/node.py b/testgres/node.py index baf532de..e203bb7d 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -791,6 +791,7 @@ def start(self, params=[], wait=True): "start"] + params # yapf: disable def LOCAL__start_node(): + # 'error' will be None on Windows _, _, error = execute_utility(_params, self.utils_log_file, verbose=True) assert error is None or type(error) == str # noqa: E721 if error and 'does not exist' in error: From 2c1dd97fac0053e0b61ab917838e5f88ecaead6b Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Wed, 25 Dec 2024 18:11:09 +0300 Subject: [PATCH 117/216] OsOps::read_binary is updated (offset) - the parameter 'start_pos' is renamed with 'offset' - we raise a runtime-error when 'offset' is negative Tests are added. --- testgres/operations/local_ops.py | 10 ++++++---- testgres/operations/os_ops.py | 6 +++--- testgres/operations/remote_ops.py | 10 ++++++---- tests/test_local.py | 10 ++++++++++ tests/test_remote.py | 10 ++++++++++ 5 files changed, 35 insertions(+), 11 deletions(-) diff --git a/testgres/operations/local_ops.py b/testgres/operations/local_ops.py index c88c16ca..8bdb22cd 100644 --- a/testgres/operations/local_ops.py +++ b/testgres/operations/local_ops.py @@ -331,13 +331,15 @@ def readlines(self, filename, num_lines=0, binary=False, encoding=None): buffers * max(2, int(num_lines / max(cur_lines, 1))) ) # Adjust buffer size - def read_binary(self, filename, start_pos): + def read_binary(self, filename, offset): assert type(filename) == str # noqa: E721 - assert type(start_pos) == int # noqa: E721 - assert start_pos >= 0 + assert type(offset) == int # noqa: E721 + + if offset < 0: + raise ValueError("Negative 'offset' is not supported.") with open(filename, 'rb') as file: # open in a binary mode - file.seek(start_pos, os.SEEK_SET) + file.seek(offset, os.SEEK_SET) r = file.read() assert type(r) == bytes # noqa: E721 return r diff --git a/testgres/operations/os_ops.py b/testgres/operations/os_ops.py index d644509a..35525b3c 100644 --- a/testgres/operations/os_ops.py +++ b/testgres/operations/os_ops.py @@ -98,10 +98,10 @@ def read(self, filename, encoding, binary): def readlines(self, filename): raise NotImplementedError() - def read_binary(self, filename, start_pos): + def read_binary(self, filename, offset): assert type(filename) == str # noqa: E721 - assert type(start_pos) == int # noqa: E721 - assert start_pos >= 0 + assert type(offset) == int # noqa: E721 + assert offset >= 0 raise NotImplementedError() def isfile(self, remote_file): diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index c0307195..2f34ecec 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -370,12 +370,14 @@ def readlines(self, filename, num_lines=0, binary=False, encoding=None): return lines - def read_binary(self, filename, start_pos): + def read_binary(self, filename, offset): assert type(filename) == str # noqa: E721 - assert type(start_pos) == int # noqa: E721 - assert start_pos >= 0 + assert type(offset) == int # noqa: E721 - cmd = ["tail", "-c", "+{}".format(start_pos + 1), filename] + if offset < 0: + raise ValueError("Negative 'offset' is not supported.") + + cmd = ["tail", "-c", "+{}".format(offset + 1), filename] r = self.exec_command(cmd) assert type(r) == bytes # noqa: E721 return r diff --git a/tests/test_local.py b/tests/test_local.py index 47a63994..d7adce17 100644 --- a/tests/test_local.py +++ b/tests/test_local.py @@ -162,6 +162,16 @@ def test_read_binary__spec__unk_file(self): match=re.escape("[Errno 2] No such file or directory: '/dummy'")): self.operations.read_binary("/dummy", 0) + def test_read_binary__spec__negative_offset(self): + """ + Test LocalOperations::read_binary with negative offset. + """ + + with pytest.raises( + ValueError, + match=re.escape("Negative 'offset' is not supported.")): + self.operations.read_binary(__file__, -1) + def test_get_file_size(self): """ Test LocalOperations::get_file_size. diff --git a/tests/test_remote.py b/tests/test_remote.py index 7421ca3a..7071a9d9 100755 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -288,6 +288,16 @@ def test_read_binary__spec__unk_file(self): with pytest.raises(ExecUtilException, match=re.escape("tail: cannot open '/dummy' for reading: No such file or directory")): self.operations.read_binary("/dummy", 0) + def test_read_binary__spec__negative_offset(self): + """ + Test RemoteOperations::read_binary with negative offset. + """ + + with pytest.raises( + ValueError, + match=re.escape("Negative 'offset' is not supported.")): + self.operations.read_binary(__file__, -1) + def test_get_file_size(self): """ Test RemoteOperations::get_file_size. From c6e6f1095c072b1828056a54e73054d2d9788f50 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Fri, 27 Dec 2024 15:49:36 +0300 Subject: [PATCH 118/216] PostgresNode::start is refactored We do not translate a new node port into string when we pack it in new option dictionary. --- testgres/node.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testgres/node.py b/testgres/node.py index e203bb7d..6f466ec9 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -848,7 +848,7 @@ def LOCAL__raise_cannot_start_node__std(from_exception): cur_port = self.port new_port = utils.reserve_port() # can raise try: - options = {'port': str(new_port)} + options = {'port': new_port} self.set_auto_conf(options) except: # noqa: E722 utils.release_port(new_port) From 9e2928ea4ef6a17e5cbb1401164194d631b44f1a Mon Sep 17 00:00:00 2001 From: "e.garbuz" Date: Mon, 20 Jan 2025 05:48:22 +0300 Subject: [PATCH 119/216] Add check backup-source for testing pg_probackup3 --- testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py index b7174a7c..078fdbab 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py @@ -172,6 +172,10 @@ def __init__(self): self.ptrack = test_env.get('PG_PROBACKUP_PTRACK', None) == 'ON' and self.pg_config_version >= 110000 self.wal_tree_enabled = test_env.get('PG_PROBACKUP_WAL_TREE_ENABLED', None) == 'ON' + self.bckp_source = test_env.get('PG_PROBACKUP_SOURCE', 'pro').lower() + if self.bckp_source not in ('base', 'direct', 'pro'): + raise Exception("Wrong PG_PROBACKUP_SOURCE value. Available options: base|direct|pro") + self.paranoia = test_env.get('PG_PROBACKUP_PARANOIA', None) == 'ON' env_compress = test_env.get('ARCHIVE_COMPRESSION', None) if env_compress: From ab7de699e9c90be4a83170e412547cb6849a366e Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Fri, 31 Jan 2025 11:09:17 +0300 Subject: [PATCH 120/216] PortManager::find_free_port is updated We will use exclude_ports only when it is not none. Creation of empty exclude_ports is removed. --- testgres/helpers/port_manager.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/testgres/helpers/port_manager.py b/testgres/helpers/port_manager.py index 6afdf8a9..a7e2a85f 100644 --- a/testgres/helpers/port_manager.py +++ b/testgres/helpers/port_manager.py @@ -26,10 +26,8 @@ def find_free_port(self, ports: Optional[Set[int]] = None, exclude_ports: Option if ports is None: ports = set(range(1024, 65535)) - if exclude_ports is None: - exclude_ports = set() - - ports.difference_update(set(exclude_ports)) + if exclude_ports is not None: + ports.difference_update(set(exclude_ports)) sampled_ports = random.sample(tuple(ports), min(len(ports), 100)) From 27c40d28645c677c5c05f2bd9409b3dc21b8c26d Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Fri, 31 Jan 2025 11:25:46 +0300 Subject: [PATCH 121/216] PortManager::find_free_port is updated Asserts are added: - ports must be the "set" - exclude_ports must be iterable Do not convert exclude_ports into "set" [optimization?] --- testgres/helpers/port_manager.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/testgres/helpers/port_manager.py b/testgres/helpers/port_manager.py index a7e2a85f..f59df259 100644 --- a/testgres/helpers/port_manager.py +++ b/testgres/helpers/port_manager.py @@ -26,8 +26,11 @@ def find_free_port(self, ports: Optional[Set[int]] = None, exclude_ports: Option if ports is None: ports = set(range(1024, 65535)) + assert type(ports) == set + if exclude_ports is not None: - ports.difference_update(set(exclude_ports)) + assert isinstance(exclude_ports, Iterable) + ports.difference_update(exclude_ports) sampled_ports = random.sample(tuple(ports), min(len(ports), 100)) From bc893d8b1d36f2073a3629c102c615e21240245a Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Fri, 31 Jan 2025 11:33:01 +0300 Subject: [PATCH 122/216] noqa: E721 --- testgres/helpers/port_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testgres/helpers/port_manager.py b/testgres/helpers/port_manager.py index f59df259..cfc5c096 100644 --- a/testgres/helpers/port_manager.py +++ b/testgres/helpers/port_manager.py @@ -26,7 +26,7 @@ def find_free_port(self, ports: Optional[Set[int]] = None, exclude_ports: Option if ports is None: ports = set(range(1024, 65535)) - assert type(ports) == set + assert type(ports) == set # noqa: E721 if exclude_ports is not None: assert isinstance(exclude_ports, Iterable) From 67beb95ce02bd824a0f0cc75fd52ebe426de666e Mon Sep 17 00:00:00 2001 From: asavchkov Date: Tue, 11 Feb 2025 18:52:41 +0700 Subject: [PATCH 123/216] Up version --- setup.py | 2 +- testgres/plugins/pg_probackup2/setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index a41094d6..16586b88 100755 --- a/setup.py +++ b/setup.py @@ -27,7 +27,7 @@ readme = f.read() setup( - version='1.10.3', + version='1.10.4', name='testgres', packages=['testgres', 'testgres.operations', 'testgres.helpers'], description='Testing utility for PostgreSQL and its extensions', diff --git a/testgres/plugins/pg_probackup2/setup.py b/testgres/plugins/pg_probackup2/setup.py index ade2d85d..619b8d39 100644 --- a/testgres/plugins/pg_probackup2/setup.py +++ b/testgres/plugins/pg_probackup2/setup.py @@ -4,7 +4,7 @@ from distutils.core import setup setup( - version='0.0.4', + version='0.0.5', name='testgres_pg_probackup2', packages=['pg_probackup2', 'pg_probackup2.storage'], description='Plugin for testgres that manages pg_probackup2', From 6fe28a5cea332d2e8f8fbc9374a9f84e4c36e287 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Mon, 17 Feb 2025 13:51:20 +0300 Subject: [PATCH 124/216] [BUG FIX] A problem with socket directory is fixed On non-Windows platform Postgres always looks for socket files in "/tmp" directory. --- testgres/node.py | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/testgres/node.py b/testgres/node.py index 6f466ec9..b85a62f2 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -6,6 +6,7 @@ import subprocess import threading import tempfile +import platform from queue import Queue import time @@ -1925,7 +1926,7 @@ def make_simple( # Define delayed propertyes if not ("unix_socket_directories" in options.keys()): - options["unix_socket_directories"] = __class__._gettempdir() + options["unix_socket_directories"] = __class__._gettempdir_for_socket() # Set config values node.set_auto_conf(options) @@ -1938,6 +1939,33 @@ def make_simple( return node + def _gettempdir_for_socket(): + platform_system_name = platform.system().lower() + + if platform_system_name == "windows": + return __class__._gettempdir() + + # + # [2025-02-17] Hot fix. + # + # Let's use hard coded path as Postgres likes. + # + # pg_config_manual.h: + # + # #ifndef WIN32 + # #define DEFAULT_PGSOCKET_DIR "/tmp" + # #else + # #define DEFAULT_PGSOCKET_DIR "" + # #endif + # + # On the altlinux-10 tempfile.gettempdir() may return + # the path to "private" temp directiry - "/temp/.private//" + # + # But Postgres want to find a socket file in "/tmp" (see above). + # + + return "/tmp" + def _gettempdir(): v = tempfile.gettempdir() From 89c8625252be857d8589141921aa185ac9db2d67 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Mon, 17 Feb 2025 14:02:24 +0300 Subject: [PATCH 125/216] A problem with tests.test_simple.TestgresTests.test_simple_with_bin_dir is fixed testgres generates the exception testgres.exceptions.ExecUtilException, but test traps the exception FileNotFoundError. Error message is: Utility exited with non-zero code. Error: `bash: line 1: wrong/path/postgres: No such file or directory` Command: ('wrong/path/postgres --version',) --- tests/test_simple.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/tests/test_simple.py b/tests/test_simple.py index 8148d05d..4e6fb573 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -1244,13 +1244,19 @@ def test_simple_with_bin_dir(self): correct_bin_dir = app.make_simple(base_dir=node.base_dir, bin_dir=bin_dir) correct_bin_dir.slow_start() correct_bin_dir.safe_psql("SELECT 1;") + correct_bin_dir.stop() + + while True: + try: + app.make_simple(base_dir=node.base_dir, bin_dir="wrong/path") + except FileNotFoundError: + break # Expected error + except ExecUtilException: + break # Expected error - try: - wrong_bin_dir = app.make_empty(base_dir=node.base_dir, bin_dir="wrong/path") - wrong_bin_dir.slow_start() raise RuntimeError("Error was expected.") # We should not reach this - except FileNotFoundError: - pass # Expected error + + return def test_set_auto_conf(self): # elements contain [property id, value, storage value] From 737e0b4fee2249247934b147fe60feb844b3da41 Mon Sep 17 00:00:00 2001 From: asavchkov Date: Tue, 18 Feb 2025 19:23:05 +0700 Subject: [PATCH 126/216] Up version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 16586b88..3f2474dd 100755 --- a/setup.py +++ b/setup.py @@ -27,7 +27,7 @@ readme = f.read() setup( - version='1.10.4', + version='1.10.5', name='testgres', packages=['testgres', 'testgres.operations', 'testgres.helpers'], description='Testing utility for PostgreSQL and its extensions', From 4493d69d80b803d1c48ffd49f2e6d33a35e2f734 Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Wed, 19 Feb 2025 11:10:21 +0300 Subject: [PATCH 127/216] OsOperations::cwd() is corrected (#182) * OsOperations::cwd() is corrected This patch fixes the following problems: - It does not work on Windows - It always returns LOCAL path --- testgres/operations/local_ops.py | 3 +++ testgres/operations/os_ops.py | 7 +------ testgres/operations/remote_ops.py | 4 ++++ tests/test_local.py | 17 +++++++++++++++++ tests/test_remote.py | 10 ++++++++++ 5 files changed, 35 insertions(+), 6 deletions(-) diff --git a/testgres/operations/local_ops.py b/testgres/operations/local_ops.py index 8bdb22cd..fc3e3954 100644 --- a/testgres/operations/local_ops.py +++ b/testgres/operations/local_ops.py @@ -152,6 +152,9 @@ def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False, def environ(self, var_name): return os.environ.get(var_name) + def cwd(self): + return os.getcwd() + def find_executable(self, executable): return find_executable(executable) diff --git a/testgres/operations/os_ops.py b/testgres/operations/os_ops.py index 35525b3c..00880863 100644 --- a/testgres/operations/os_ops.py +++ b/testgres/operations/os_ops.py @@ -1,6 +1,5 @@ import getpass import locale -import sys try: import psycopg2 as pglib # noqa: F401 @@ -39,11 +38,7 @@ def environ(self, var_name): raise NotImplementedError() def cwd(self): - if sys.platform == 'linux': - cmd = 'pwd' - elif sys.platform == 'win32': - cmd = 'cd' - return self.exec_command(cmd).decode().rstrip() + raise NotImplementedError() def find_executable(self, executable): raise NotImplementedError() diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index 2f34ecec..3ebc2e60 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -138,6 +138,10 @@ def environ(self, var_name: str) -> str: cmd = "echo ${}".format(var_name) return self.exec_command(cmd, encoding=get_default_encoding()).strip() + def cwd(self): + cmd = 'pwd' + return self.exec_command(cmd, encoding=get_default_encoding()).rstrip() + def find_executable(self, executable): search_paths = self.environ("PATH") if not search_paths: diff --git a/tests/test_local.py b/tests/test_local.py index d7adce17..568a4bc5 100644 --- a/tests/test_local.py +++ b/tests/test_local.py @@ -256,3 +256,20 @@ def test_isdir_false__file(self): response = self.operations.isdir(name) assert response is False + + def test_cwd(self): + """ + Test cwd. + """ + v = self.operations.cwd() + + assert v is not None + assert type(v) == str # noqa: E721 + + expectedValue = os.getcwd() + assert expectedValue is not None + assert type(expectedValue) == str # noqa: E721 + assert expectedValue != "" # research + + # Comp result + assert v == expectedValue diff --git a/tests/test_remote.py b/tests/test_remote.py index 7071a9d9..30c5d348 100755 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -392,3 +392,13 @@ def test_isdir_false__file(self): response = self.operations.isdir(name) assert response is False + + def test_cwd(self): + """ + Test cwd. + """ + v = self.operations.cwd() + + assert v is not None + assert type(v) == str # noqa: E721 + assert v != "" From 22c476347d4e8bfae832e641e00428ed4b6d14a2 Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Wed, 19 Feb 2025 15:26:56 +0300 Subject: [PATCH 128/216] OsOps::write method is corrected (#183) LocalOperations - [BUG FIX] (read_write=true and truncate=false) writes to begging of a file - Preparation of data is added (verification/encoding/decoding/eol) RemoteOperations - Preparation of data is corrected (verification/encoding/decoding/eol) - Temp file is always opened with "w+"/"w+b" modes. Tests are added. --- testgres/operations/local_ops.py | 53 ++++++++++++++++++------ testgres/operations/remote_ops.py | 35 ++++++++++------ tests/test_local.py | 69 +++++++++++++++++++++++++++++++ tests/test_remote.py | 69 +++++++++++++++++++++++++++++++ 4 files changed, 201 insertions(+), 25 deletions(-) diff --git a/testgres/operations/local_ops.py b/testgres/operations/local_ops.py index fc3e3954..5c79bb7e 100644 --- a/testgres/operations/local_ops.py +++ b/testgres/operations/local_ops.py @@ -235,27 +235,54 @@ def write(self, filename, data, truncate=False, binary=False, read_and_write=Fal Args: filename: The file path where the data will be written. data: The data to be written to the file. - truncate: If True, the file will be truncated before writing ('w' or 'wb' option); - if False (default), data will be appended ('a' or 'ab' option). - binary: If True, the data will be written in binary mode ('wb' or 'ab' option); - if False (default), the data will be written in text mode ('w' or 'a' option). - read_and_write: If True, the file will be opened with read and write permissions ('r+' option); - if False (default), only write permission will be used ('w', 'a', 'wb', or 'ab' option) + truncate: If True, the file will be truncated before writing ('w' option); + if False (default), data will be appended ('a' option). + binary: If True, the data will be written in binary mode ('b' option); + if False (default), the data will be written in text mode. + read_and_write: If True, the file will be opened with read and write permissions ('+' option); + if False (default), only write permission will be used. """ - # If it is a bytes str or list if isinstance(data, bytes) or isinstance(data, list) and all(isinstance(item, bytes) for item in data): binary = True - mode = "wb" if binary else "w" - if not truncate: - mode = "ab" if binary else "a" + + mode = "w" if truncate else "a" + if read_and_write: - mode = "r+b" if binary else "r+" + mode += "+" + + # If it is a bytes str or list + if binary: + mode += "b" + + assert type(mode) == str # noqa: E721 + assert mode != "" with open(filename, mode) as file: if isinstance(data, list): - file.writelines(data) + data2 = [__class__._prepare_line_to_write(s, binary) for s in data] + file.writelines(data2) else: - file.write(data) + data2 = __class__._prepare_data_to_write(data, binary) + file.write(data2) + + def _prepare_line_to_write(data, binary): + data = __class__._prepare_data_to_write(data, binary) + + if binary: + assert type(data) == bytes # noqa: E721 + return data.rstrip(b'\n') + b'\n' + + assert type(data) == str # noqa: E721 + return data.rstrip('\n') + '\n' + + def _prepare_data_to_write(data, binary): + if isinstance(data, bytes): + return data if binary else data.decode() + + if isinstance(data, str): + return data if not binary else data.encode() + + raise InvalidOperationException("Unknown type of data type [{0}].".format(type(data).__name__)) def touch(self, filename): """ diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index 3ebc2e60..f690e063 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -278,10 +278,6 @@ def write(self, filename, data, truncate=False, binary=False, read_and_write=Fal if not encoding: encoding = get_default_encoding() mode = "wb" if binary else "w" - if not truncate: - mode = "ab" if binary else "a" - if read_and_write: - mode = "r+b" if binary else "r+" with tempfile.NamedTemporaryFile(mode=mode, delete=False) as tmp_file: # For scp the port is specified by a "-P" option @@ -292,16 +288,12 @@ def write(self, filename, data, truncate=False, binary=False, read_and_write=Fal subprocess.run(scp_cmd, check=False) # The file might not exist yet tmp_file.seek(0, os.SEEK_END) - if isinstance(data, bytes) and not binary: - data = data.decode(encoding) - elif isinstance(data, str) and binary: - data = data.encode(encoding) - if isinstance(data, list): - data = [(s if isinstance(s, str) else s.decode(get_default_encoding())).rstrip('\n') + '\n' for s in data] - tmp_file.writelines(data) + data2 = [__class__._prepare_line_to_write(s, binary, encoding) for s in data] + tmp_file.writelines(data2) else: - tmp_file.write(data) + data2 = __class__._prepare_data_to_write(data, binary, encoding) + tmp_file.write(data2) tmp_file.flush() scp_cmd = ['scp'] + scp_args + [tmp_file.name, f"{self.ssh_dest}:{filename}"] @@ -313,6 +305,25 @@ def write(self, filename, data, truncate=False, binary=False, read_and_write=Fal os.remove(tmp_file.name) + def _prepare_line_to_write(data, binary, encoding): + data = __class__._prepare_data_to_write(data, binary, encoding) + + if binary: + assert type(data) == bytes # noqa: E721 + return data.rstrip(b'\n') + b'\n' + + assert type(data) == str # noqa: E721 + return data.rstrip('\n') + '\n' + + def _prepare_data_to_write(data, binary, encoding): + if isinstance(data, bytes): + return data if binary else data.decode(encoding) + + if isinstance(data, str): + return data if not binary else data.encode(encoding) + + raise InvalidOperationException("Unknown type of data type [{0}].".format(type(data).__name__)) + def touch(self, filename): """ Create a new file or update the access and modification times of an existing file on the remote server. diff --git a/tests/test_local.py b/tests/test_local.py index 568a4bc5..4051bfb5 100644 --- a/tests/test_local.py +++ b/tests/test_local.py @@ -2,6 +2,7 @@ import pytest import re +import tempfile from testgres import ExecUtilException from testgres import InvalidOperationException @@ -273,3 +274,71 @@ def test_cwd(self): # Comp result assert v == expectedValue + + class tagWriteData001: + def __init__(self, sign, source, cp_rw, cp_truncate, cp_binary, cp_data, result): + self.sign = sign + self.source = source + self.call_param__rw = cp_rw + self.call_param__truncate = cp_truncate + self.call_param__binary = cp_binary + self.call_param__data = cp_data + self.result = result + + sm_write_data001 = [ + tagWriteData001("A001", "1234567890", False, False, False, "ABC", "1234567890ABC"), + tagWriteData001("A002", b"1234567890", False, False, True, b"ABC", b"1234567890ABC"), + + tagWriteData001("B001", "1234567890", False, True, False, "ABC", "ABC"), + tagWriteData001("B002", "1234567890", False, True, False, "ABC1234567890", "ABC1234567890"), + tagWriteData001("B003", b"1234567890", False, True, True, b"ABC", b"ABC"), + tagWriteData001("B004", b"1234567890", False, True, True, b"ABC1234567890", b"ABC1234567890"), + + tagWriteData001("C001", "1234567890", True, False, False, "ABC", "1234567890ABC"), + tagWriteData001("C002", b"1234567890", True, False, True, b"ABC", b"1234567890ABC"), + + tagWriteData001("D001", "1234567890", True, True, False, "ABC", "ABC"), + tagWriteData001("D002", "1234567890", True, True, False, "ABC1234567890", "ABC1234567890"), + tagWriteData001("D003", b"1234567890", True, True, True, b"ABC", b"ABC"), + tagWriteData001("D004", b"1234567890", True, True, True, b"ABC1234567890", b"ABC1234567890"), + + tagWriteData001("E001", "\0001234567890\000", False, False, False, "\000ABC\000", "\0001234567890\000\000ABC\000"), + tagWriteData001("E002", b"\0001234567890\000", False, False, True, b"\000ABC\000", b"\0001234567890\000\000ABC\000"), + + tagWriteData001("F001", "a\nb\n", False, False, False, ["c", "d"], "a\nb\nc\nd\n"), + tagWriteData001("F002", b"a\nb\n", False, False, True, [b"c", b"d"], b"a\nb\nc\nd\n"), + + tagWriteData001("G001", "a\nb\n", False, False, False, ["c\n\n", "d\n"], "a\nb\nc\nd\n"), + tagWriteData001("G002", b"a\nb\n", False, False, True, [b"c\n\n", b"d\n"], b"a\nb\nc\nd\n"), + ] + + @pytest.fixture( + params=sm_write_data001, + ids=[x.sign for x in sm_write_data001], + ) + def write_data001(self, request): + assert isinstance(request, pytest.FixtureRequest) + assert type(request.param) == __class__.tagWriteData001 # noqa: E721 + return request.param + + def test_write(self, write_data001): + assert type(write_data001) == __class__.tagWriteData001 # noqa: E721 + + mode = "w+b" if write_data001.call_param__binary else "w+" + + with tempfile.NamedTemporaryFile(mode=mode, delete=True) as tmp_file: + tmp_file.write(write_data001.source) + tmp_file.flush() + + self.operations.write( + tmp_file.name, + write_data001.call_param__data, + read_and_write=write_data001.call_param__rw, + truncate=write_data001.call_param__truncate, + binary=write_data001.call_param__binary) + + tmp_file.seek(0) + + s = tmp_file.read() + + assert s == write_data001.result diff --git a/tests/test_remote.py b/tests/test_remote.py index 30c5d348..3e6b79dd 100755 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -2,6 +2,7 @@ import pytest import re +import tempfile from testgres import ExecUtilException from testgres import InvalidOperationException @@ -402,3 +403,71 @@ def test_cwd(self): assert v is not None assert type(v) == str # noqa: E721 assert v != "" + + class tagWriteData001: + def __init__(self, sign, source, cp_rw, cp_truncate, cp_binary, cp_data, result): + self.sign = sign + self.source = source + self.call_param__rw = cp_rw + self.call_param__truncate = cp_truncate + self.call_param__binary = cp_binary + self.call_param__data = cp_data + self.result = result + + sm_write_data001 = [ + tagWriteData001("A001", "1234567890", False, False, False, "ABC", "1234567890ABC"), + tagWriteData001("A002", b"1234567890", False, False, True, b"ABC", b"1234567890ABC"), + + tagWriteData001("B001", "1234567890", False, True, False, "ABC", "ABC"), + tagWriteData001("B002", "1234567890", False, True, False, "ABC1234567890", "ABC1234567890"), + tagWriteData001("B003", b"1234567890", False, True, True, b"ABC", b"ABC"), + tagWriteData001("B004", b"1234567890", False, True, True, b"ABC1234567890", b"ABC1234567890"), + + tagWriteData001("C001", "1234567890", True, False, False, "ABC", "1234567890ABC"), + tagWriteData001("C002", b"1234567890", True, False, True, b"ABC", b"1234567890ABC"), + + tagWriteData001("D001", "1234567890", True, True, False, "ABC", "ABC"), + tagWriteData001("D002", "1234567890", True, True, False, "ABC1234567890", "ABC1234567890"), + tagWriteData001("D003", b"1234567890", True, True, True, b"ABC", b"ABC"), + tagWriteData001("D004", b"1234567890", True, True, True, b"ABC1234567890", b"ABC1234567890"), + + tagWriteData001("E001", "\0001234567890\000", False, False, False, "\000ABC\000", "\0001234567890\000\000ABC\000"), + tagWriteData001("E002", b"\0001234567890\000", False, False, True, b"\000ABC\000", b"\0001234567890\000\000ABC\000"), + + tagWriteData001("F001", "a\nb\n", False, False, False, ["c", "d"], "a\nb\nc\nd\n"), + tagWriteData001("F002", b"a\nb\n", False, False, True, [b"c", b"d"], b"a\nb\nc\nd\n"), + + tagWriteData001("G001", "a\nb\n", False, False, False, ["c\n\n", "d\n"], "a\nb\nc\nd\n"), + tagWriteData001("G002", b"a\nb\n", False, False, True, [b"c\n\n", b"d\n"], b"a\nb\nc\nd\n"), + ] + + @pytest.fixture( + params=sm_write_data001, + ids=[x.sign for x in sm_write_data001], + ) + def write_data001(self, request): + assert isinstance(request, pytest.FixtureRequest) + assert type(request.param) == __class__.tagWriteData001 # noqa: E721 + return request.param + + def test_write(self, write_data001): + assert type(write_data001) == __class__.tagWriteData001 # noqa: E721 + + mode = "w+b" if write_data001.call_param__binary else "w+" + + with tempfile.NamedTemporaryFile(mode=mode, delete=True) as tmp_file: + tmp_file.write(write_data001.source) + tmp_file.flush() + + self.operations.write( + tmp_file.name, + write_data001.call_param__data, + read_and_write=write_data001.call_param__rw, + truncate=write_data001.call_param__truncate, + binary=write_data001.call_param__binary) + + tmp_file.seek(0) + + s = tmp_file.read() + + assert s == write_data001.result From 7fd2f07af75391503fb1c79a1bc9ec1f763923af Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Thu, 20 Feb 2025 14:56:02 +0300 Subject: [PATCH 129/216] RemoteOperations::exec_command updated (#185) - Exact enumeration of supported 'cmd' types - Refactoring --- testgres/operations/remote_ops.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index f690e063..a24fce50 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -78,14 +78,17 @@ def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False, assert input_prepared is None or (type(input_prepared) == bytes) # noqa: E721 - ssh_cmd = [] - if isinstance(cmd, str): - ssh_cmd = ['ssh', self.ssh_dest] + self.ssh_args + [cmd] - elif isinstance(cmd, list): - ssh_cmd = ['ssh', self.ssh_dest] + self.ssh_args + [subprocess.list2cmdline(cmd)] + if type(cmd) == str: # noqa: E721 + cmd_s = cmd + elif type(cmd) == list: # noqa: E721 + cmd_s = subprocess.list2cmdline(cmd) else: raise ValueError("Invalid 'cmd' argument type - {0}".format(type(cmd).__name__)) + assert type(cmd_s) == str # noqa: E721 + + ssh_cmd = ['ssh', self.ssh_dest] + self.ssh_args + [cmd_s] + process = subprocess.Popen(ssh_cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) assert not (process is None) if get_process: From e44aa9813daf04e8170f57c277923100aa04eadc Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Fri, 21 Feb 2025 20:04:08 +0300 Subject: [PATCH 130/216] RemoteOperations::exec_command explicitly transfers LANG, LANGUAGE and LC_* envvars to the server side (#187) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * RemoteOperations::exec_command updated - Exact enumeration of supported 'cmd' types - Refactoring * RemoteOperations::exec_command explicitly transfers LANG, LANGUAGE and LC_* envvars to the server side It should help resolve a problem with replacing a LANG variable by ssh-server. History. On our internal tests we got a problem on the Debian 11 and PostgresPro STD-13. One test returned the error from initdb: initdb: error: collations with different collate and ctype values ("en_US.UTF-8" and "C.UTF-8" accordingly) are not supported by ICU - TestRunner set variable LANG="C" - Python set variable LC_CTYPE="C.UTF-8" - Test call inidb through command "ssh test@localhost inidb -D ...." - SSH-server replaces LANG with value "en_US.UTF-8" (from etc/default/locale) - initdb calculate collate through this value of LANG variable and get en_US.UTF-8 So we have that: - ctype is C.UTF-8 - collate is en_US.UTF-8 ICU on the Debuan-11 (uconv v2.1 ICU 67.1) does not suppot this combination and inidb rturns the error. This patch generates a new command line for ssh: ssh test@localhost "LANG=\"...\";LC_xxx=\"...\";" It resolves this problem with initdb and should help resolve other problems with execution of command through SSH. Amen. * New tests in TestgresRemoteTests are added New tests: - test_init__LANG_С - test_init__unk_LANG_and_LC_CTYPE * TestgresRemoteTests.test_init__unk_LANG_and_LC_CTYPE is updated Let's test bad data with '\' and '"' symbols. * Static methods are marked with @staticmethod [thanks to Victoria Shepard] The following methods of RemoteOperations were corrected: - _make_exec_env_list - _does_put_envvar_into_exec_cmd - _quote_envvar * TestRemoteOperations::_quote_envvar is updated (typification) --- testgres/operations/remote_ops.py | 46 +++++++++++++++++- tests/test_simple_remote.py | 79 +++++++++++++++++++++++++++++++ 2 files changed, 124 insertions(+), 1 deletion(-) diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index a24fce50..af4c59f9 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -87,7 +87,12 @@ def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False, assert type(cmd_s) == str # noqa: E721 - ssh_cmd = ['ssh', self.ssh_dest] + self.ssh_args + [cmd_s] + cmd_items = __class__._make_exec_env_list() + cmd_items.append(cmd_s) + + env_cmd_s = ';'.join(cmd_items) + + ssh_cmd = ['ssh', self.ssh_dest] + self.ssh_args + [env_cmd_s] process = subprocess.Popen(ssh_cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) assert not (process is None) @@ -510,6 +515,45 @@ def db_connect(self, dbname, user, password=None, host="localhost", port=5432): ) return conn + @staticmethod + def _make_exec_env_list() -> list[str]: + result = list[str]() + for envvar in os.environ.items(): + if not __class__._does_put_envvar_into_exec_cmd(envvar[0]): + continue + qvalue = __class__._quote_envvar(envvar[1]) + assert type(qvalue) == str # noqa: E721 + result.append(envvar[0] + "=" + qvalue) + continue + + return result + + sm_envs_for_exec_cmd = ["LANG", "LANGUAGE"] + + @staticmethod + def _does_put_envvar_into_exec_cmd(name: str) -> bool: + assert type(name) == str # noqa: E721 + name = name.upper() + if name.startswith("LC_"): + return True + if name in __class__.sm_envs_for_exec_cmd: + return True + return False + + @staticmethod + def _quote_envvar(value: str) -> str: + assert type(value) == str # noqa: E721 + result = "\"" + for ch in value: + if ch == "\"": + result += "\\\"" + elif ch == "\\": + result += "\\\\" + else: + result += ch + result += "\"" + return result + def normalize_error(error): if isinstance(error, bytes): diff --git a/tests/test_simple_remote.py b/tests/test_simple_remote.py index c8dd2964..2b581ac9 100755 --- a/tests/test_simple_remote.py +++ b/tests/test_simple_remote.py @@ -119,6 +119,79 @@ def test_custom_init(self): # there should be no trust entries at all self.assertFalse(any('trust' in s for s in lines)) + def test_init__LANG_С(self): + # PBCKP-1744 + prev_LANG = os.environ.get("LANG") + + try: + os.environ["LANG"] = "C" + + with get_remote_node(conn_params=conn_params) as node: + node.init().start() + finally: + __class__.helper__restore_envvar("LANG", prev_LANG) + + def test_init__unk_LANG_and_LC_CTYPE(self): + # PBCKP-1744 + prev_LANG = os.environ.get("LANG") + prev_LANGUAGE = os.environ.get("LANGUAGE") + prev_LC_CTYPE = os.environ.get("LC_CTYPE") + prev_LC_COLLATE = os.environ.get("LC_COLLATE") + + try: + # TODO: Pass unkData through test parameter. + unkDatas = [ + ("UNKNOWN_LANG", "UNKNOWN_CTYPE"), + ("\"UNKNOWN_LANG\"", "\"UNKNOWN_CTYPE\""), + ("\\UNKNOWN_LANG\\", "\\UNKNOWN_CTYPE\\"), + ("\"UNKNOWN_LANG", "UNKNOWN_CTYPE\""), + ("\\UNKNOWN_LANG", "UNKNOWN_CTYPE\\"), + ("\\", "\\"), + ("\"", "\""), + ] + + for unkData in unkDatas: + logging.info("----------------------") + logging.info("Unk LANG is [{0}]".format(unkData[0])) + logging.info("Unk LC_CTYPE is [{0}]".format(unkData[1])) + + os.environ["LANG"] = unkData[0] + os.environ.pop("LANGUAGE", None) + os.environ["LC_CTYPE"] = unkData[1] + os.environ.pop("LC_COLLATE", None) + + assert os.environ.get("LANG") == unkData[0] + assert not ("LANGUAGE" in os.environ.keys()) + assert os.environ.get("LC_CTYPE") == unkData[1] + assert not ("LC_COLLATE" in os.environ.keys()) + + while True: + try: + with get_remote_node(conn_params=conn_params): + pass + except testgres.exceptions.ExecUtilException as e: + # + # Example of an error message: + # + # warning: setlocale: LC_CTYPE: cannot change locale (UNKNOWN_CTYPE): No such file or directory + # postgres (PostgreSQL) 14.12 + # + errMsg = str(e) + + logging.info("Error message is: {0}".format(errMsg)) + + assert "LC_CTYPE" in errMsg + assert unkData[1] in errMsg + assert "warning: setlocale: LC_CTYPE: cannot change locale (" + unkData[1] + "): No such file or directory" in errMsg + assert "postgres" in errMsg + break + raise Exception("We expected an error!") + finally: + __class__.helper__restore_envvar("LANG", prev_LANG) + __class__.helper__restore_envvar("LANGUAGE", prev_LANGUAGE) + __class__.helper__restore_envvar("LC_CTYPE", prev_LC_CTYPE) + __class__.helper__restore_envvar("LC_COLLATE", prev_LC_COLLATE) + def test_double_init(self): with get_remote_node(conn_params=conn_params).init() as node: # can't initialize node more than once @@ -994,6 +1067,12 @@ def test_child_process_dies(self): # try to handle children list -- missing processes will have ptype "ProcessType.Unknown" [ProcessProxy(p) for p in children] + def helper__restore_envvar(name, prev_value): + if prev_value is None: + os.environ.pop(name, None) + else: + os.environ[name] = prev_value + if __name__ == '__main__': if os_ops.environ('ALT_CONFIG'): From 44b99f08084d706a3cc9a67fa255579def9cb201 Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Fri, 21 Feb 2025 22:11:00 +0300 Subject: [PATCH 131/216] TestRemoteOperations::test_makedirs_and_rmdirs_success is updated (#188) The new checks are added. --- tests/test_remote.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_remote.py b/tests/test_remote.py index 3e6b79dd..4330b92f 100755 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -83,10 +83,12 @@ def test_makedirs_and_rmdirs_success(self): # Test makedirs self.operations.makedirs(path) + assert os.path.exists(path) assert self.operations.path_exists(path) # Test rmdirs self.operations.rmdirs(path) + assert not os.path.exists(path) assert not self.operations.path_exists(path) def test_makedirs_and_rmdirs_failure(self): From 40eaf7de413eb4b4eb94ed32111837f8aaa7507b Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Sun, 23 Feb 2025 19:34:57 +0300 Subject: [PATCH 132/216] Usage of @staticmethod (#189) * Usage of @staticmethod All the known static methods are marked with @staticmethod decorators. * Fix for "ERROR: Unexpected indentation. [docutils]" This commit should fix an unstable error in CI: /pg/testgres/testgres/api.py:docstring of testgres.api.get_remote_node:5: ERROR: Unexpected indentation. [docutils] --- testgres/api.py | 4 +--- testgres/node.py | 3 +++ testgres/operations/helpers.py | 3 +++ testgres/operations/local_ops.py | 2 ++ testgres/operations/raise_error.py | 4 ++++ testgres/operations/remote_ops.py | 2 ++ tests/test_simple.py | 2 ++ 7 files changed, 17 insertions(+), 3 deletions(-) diff --git a/testgres/api.py b/testgres/api.py index e4b1cdd5..6a96ee84 100644 --- a/testgres/api.py +++ b/testgres/api.py @@ -47,8 +47,6 @@ def get_remote_node(name=None, conn_params=None): Simply a wrapper around :class:`.PostgresNode` constructor for remote node. See :meth:`.PostgresNode.__init__` for details. For remote connection you can add the next parameter: - conn_params = ConnectionParams(host='127.0.0.1', - ssh_key=None, - username=default_username()) + conn_params = ConnectionParams(host='127.0.0.1', ssh_key=None, username=default_username()) """ return get_new_node(name=name, conn_params=conn_params) diff --git a/testgres/node.py b/testgres/node.py index b85a62f2..8a712753 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -1939,6 +1939,7 @@ def make_simple( return node + @staticmethod def _gettempdir_for_socket(): platform_system_name = platform.system().lower() @@ -1966,6 +1967,7 @@ def _gettempdir_for_socket(): return "/tmp" + @staticmethod def _gettempdir(): v = tempfile.gettempdir() @@ -1984,6 +1986,7 @@ def _gettempdir(): # OK return v + @staticmethod def _raise_bugcheck(msg): assert type(msg) == str # noqa: E721 assert msg != "" diff --git a/testgres/operations/helpers.py b/testgres/operations/helpers.py index b50f0baa..03e97edc 100644 --- a/testgres/operations/helpers.py +++ b/testgres/operations/helpers.py @@ -2,6 +2,7 @@ class Helpers: + @staticmethod def _make_get_default_encoding_func(): # locale.getencoding is added in Python 3.11 if hasattr(locale, 'getencoding'): @@ -13,6 +14,7 @@ def _make_get_default_encoding_func(): # Prepared pointer on function to get a name of system codepage _get_default_encoding_func = _make_get_default_encoding_func() + @staticmethod def GetDefaultEncoding(): # # Original idea/source was: @@ -36,6 +38,7 @@ def GetDefaultEncoding(): # Is it an unexpected situation? return 'UTF-8' + @staticmethod def PrepareProcessInput(input, encoding): if not input: return None diff --git a/testgres/operations/local_ops.py b/testgres/operations/local_ops.py index 5c79bb7e..91070fe7 100644 --- a/testgres/operations/local_ops.py +++ b/testgres/operations/local_ops.py @@ -265,6 +265,7 @@ def write(self, filename, data, truncate=False, binary=False, read_and_write=Fal data2 = __class__._prepare_data_to_write(data, binary) file.write(data2) + @staticmethod def _prepare_line_to_write(data, binary): data = __class__._prepare_data_to_write(data, binary) @@ -275,6 +276,7 @@ def _prepare_line_to_write(data, binary): assert type(data) == str # noqa: E721 return data.rstrip('\n') + '\n' + @staticmethod def _prepare_data_to_write(data, binary): if isinstance(data, bytes): return data if binary else data.decode() diff --git a/testgres/operations/raise_error.py b/testgres/operations/raise_error.py index 0e760e74..6031b238 100644 --- a/testgres/operations/raise_error.py +++ b/testgres/operations/raise_error.py @@ -3,6 +3,7 @@ class RaiseError: + @staticmethod def UtilityExitedWithNonZeroCode(cmd, exit_code, msg_arg, error, out): assert type(exit_code) == int # noqa: E721 @@ -20,12 +21,14 @@ def UtilityExitedWithNonZeroCode(cmd, exit_code, msg_arg, error, out): out=out, error=error) + @staticmethod def _TranslateDataIntoString(data): if type(data) == bytes: # noqa: E721 return __class__._TranslateDataIntoString__FromBinary(data) return str(data) + @staticmethod def _TranslateDataIntoString__FromBinary(data): assert type(data) == bytes # noqa: E721 @@ -36,6 +39,7 @@ def _TranslateDataIntoString__FromBinary(data): return "#cannot_decode_text" + @staticmethod def _BinaryIsASCII(data): assert type(data) == bytes # noqa: E721 diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index af4c59f9..51f5b2e8 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -313,6 +313,7 @@ def write(self, filename, data, truncate=False, binary=False, read_and_write=Fal os.remove(tmp_file.name) + @staticmethod def _prepare_line_to_write(data, binary, encoding): data = __class__._prepare_data_to_write(data, binary, encoding) @@ -323,6 +324,7 @@ def _prepare_line_to_write(data, binary, encoding): assert type(data) == str # noqa: E721 return data.rstrip('\n') + '\n' + @staticmethod def _prepare_data_to_write(data, binary, encoding): if isinstance(data, bytes): return data if binary else data.decode(encoding) diff --git a/tests/test_simple.py b/tests/test_simple.py index 4e6fb573..a751f0a3 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -1125,6 +1125,7 @@ def __exit__(self, type, value, traceback): __class__.sm_prev_testgres_reserve_port = None __class__.sm_prev_testgres_release_port = None + @staticmethod def _proxy__reserve_port(): assert type(__class__.sm_DummyPortMaxUsage) == int # noqa: E721 assert type(__class__.sm_DummyPortTotalUsage) == int # noqa: E721 @@ -1144,6 +1145,7 @@ def _proxy__reserve_port(): __class__.sm_DummyPortCurrentUsage += 1 return __class__.sm_DummyPortNumber + @staticmethod def _proxy__release_port(dummyPortNumber): assert type(dummyPortNumber) == int # noqa: E721 From 8824946283a305c0ee2b997aeb513efda8a073e7 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Sun, 23 Feb 2025 23:18:57 +0300 Subject: [PATCH 133/216] TestgresRemoteTests::helper__restore_envvar is marked with @staticmethod --- tests/test_simple_remote.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_simple_remote.py b/tests/test_simple_remote.py index 2b581ac9..9d12e618 100755 --- a/tests/test_simple_remote.py +++ b/tests/test_simple_remote.py @@ -1067,6 +1067,7 @@ def test_child_process_dies(self): # try to handle children list -- missing processes will have ptype "ProcessType.Unknown" [ProcessProxy(p) for p in children] + @staticmethod def helper__restore_envvar(name, prev_value): if prev_value is None: os.environ.pop(name, None) From fe03c24d35b8713f01df4d015859d1d099dfcd58 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Mon, 24 Feb 2025 13:49:43 +0300 Subject: [PATCH 134/216] TestgresRemoteTests.test_init__unk_LANG_and_LC_CTYPE is corrected Vanilla PG18 returns "...PostgreSQL 18devel" --- tests/test_simple_remote.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_simple_remote.py b/tests/test_simple_remote.py index 9d12e618..8b44623a 100755 --- a/tests/test_simple_remote.py +++ b/tests/test_simple_remote.py @@ -183,7 +183,7 @@ def test_init__unk_LANG_and_LC_CTYPE(self): assert "LC_CTYPE" in errMsg assert unkData[1] in errMsg assert "warning: setlocale: LC_CTYPE: cannot change locale (" + unkData[1] + "): No such file or directory" in errMsg - assert "postgres" in errMsg + assert ("postgres" in errMsg) or ("PostgreSQL" in errMsg) break raise Exception("We expected an error!") finally: From 50fc4c5fecc9aa722dbc660762a9d120f12e92fa Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Mon, 24 Feb 2025 14:12:50 +0300 Subject: [PATCH 135/216] [BUG FIX] PostgresNode::start is corrected Incorrect code to build a warning message. --- testgres/node.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index 8a712753..512650c1 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -841,8 +841,7 @@ def LOCAL__raise_cannot_start_node__std(from_exception): log_files0 = log_files1 logging.warning( - "Detected a conflict with using the port {0}. " - "Trying another port after a {1}-second sleep...".format(self.port, timeout) + "Detected a conflict with using the port {0}. Trying another port after a {1}-second sleep...".format(self.port, timeout) ) time.sleep(timeout) timeout = min(2 * timeout, 5) From 2b34236e9f8677cd20b75b37840841dbe0968b09 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Mon, 24 Feb 2025 15:03:23 +0300 Subject: [PATCH 136/216] execute_utility2, get_bin_path2, get_pg_config2 are added This the functions with explicit os_ops argument. testgres/utils.py - [add] def execute_utility2(os_ops: OsOperations, args, logfile=None, verbose=False) - [add] def get_bin_path2(os_ops: OsOperations, filename) - [add] def get_pg_config2(os_ops: OsOperations, pg_config_path): ATTENTION get_pg_config does not change tconf.os_ops now testgres/cache.py - cached_initdb - [add] make_utility_path - it is used for pg_resetwal, too. --- testgres/backup.py | 13 +++++++++---- testgres/cache.py | 24 ++++++++++++++++++------ testgres/node.py | 30 +++++++++++++++--------------- testgres/utils.py | 46 +++++++++++++++++++++++++++++++++++----------- 4 files changed, 77 insertions(+), 36 deletions(-) diff --git a/testgres/backup.py b/testgres/backup.py index cecb0f7b..619c0270 100644 --- a/testgres/backup.py +++ b/testgres/backup.py @@ -15,9 +15,11 @@ from .exceptions import BackupException +from .operations.os_ops import OsOperations + from .utils import \ - get_bin_path, \ - execute_utility, \ + get_bin_path2, \ + execute_utility2, \ clean_on_error @@ -44,6 +46,9 @@ def __init__(self, username: database user name. xlog_method: none | fetch | stream (see docs) """ + assert node.os_ops is not None + assert isinstance(node.os_ops, OsOperations) + if not options: options = [] self.os_ops = node.os_ops @@ -73,7 +78,7 @@ def __init__(self, data_dir = os.path.join(self.base_dir, DATA_DIR) _params = [ - get_bin_path("pg_basebackup"), + get_bin_path2(self.os_ops, "pg_basebackup"), "-p", str(node.port), "-h", node.host, "-U", username, @@ -81,7 +86,7 @@ def __init__(self, "-X", xlog_method.value ] # yapf: disable _params += options - execute_utility(_params, self.log_file) + execute_utility2(self.os_ops, _params, self.log_file) def __enter__(self): return self diff --git a/testgres/cache.py b/testgres/cache.py index f17b54b5..61d44868 100644 --- a/testgres/cache.py +++ b/testgres/cache.py @@ -15,8 +15,8 @@ ExecUtilException from .utils import \ - get_bin_path, \ - execute_utility + get_bin_path2, \ + execute_utility2 from .operations.local_ops import LocalOperations from .operations.os_ops import OsOperations @@ -27,11 +27,23 @@ def cached_initdb(data_dir, logfile=None, params=None, os_ops: OsOperations = Lo Perform initdb or use cached node files. """ + assert os_ops is not None + assert isinstance(os_ops, OsOperations) + + def make_utility_path(name): + assert name is not None + assert type(name) == str + + if bin_path: + return os.path.join(bin_path, name) + + return get_bin_path2(os_ops, name) + def call_initdb(initdb_dir, log=logfile): try: - initdb_path = os.path.join(bin_path, 'initdb') if bin_path else get_bin_path("initdb") + initdb_path = make_utility_path("initdb") _params = [initdb_path, "-D", initdb_dir, "-N"] - execute_utility(_params + (params or []), log) + execute_utility2(os_ops, _params + (params or []), log) except ExecUtilException as e: raise_from(InitNodeException("Failed to run initdb"), e) @@ -63,8 +75,8 @@ def call_initdb(initdb_dir, log=logfile): os_ops.write(pg_control, new_pg_control, truncate=True, binary=True, read_and_write=True) # XXX: build new WAL segment with our system id - _params = [get_bin_path("pg_resetwal"), "-D", data_dir, "-f"] - execute_utility(_params, logfile) + _params = [make_utility_path("pg_resetwal"), "-D", data_dir, "-f"] + execute_utility2(os_ops, _params, logfile) except ExecUtilException as e: msg = "Failed to reset WAL for system id" diff --git a/testgres/node.py b/testgres/node.py index 512650c1..56899b90 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -89,9 +89,9 @@ from .utils import \ PgVer, \ eprint, \ - get_bin_path, \ + get_bin_path2, \ get_pg_version, \ - execute_utility, \ + execute_utility2, \ options_string, \ clean_on_error @@ -301,7 +301,7 @@ def base_dir(self): @property def bin_dir(self): if not self._bin_dir: - self._bin_dir = os.path.dirname(get_bin_path("pg_config")) + self._bin_dir = os.path.dirname(get_bin_path2(self.os_ops, "pg_config")) return self._bin_dir @property @@ -684,7 +684,7 @@ def status(self): "-D", self.data_dir, "status" ] # yapf: disable - status_code, out, error = execute_utility(_params, self.utils_log_file, verbose=True) + status_code, out, error = execute_utility2(self.os_ops, _params, self.utils_log_file, verbose=True) if error and 'does not exist' in error: return NodeStatus.Uninitialized elif 'no server running' in out: @@ -710,7 +710,7 @@ def get_control_data(self): _params += ["-D"] if self._pg_version >= PgVer('9.5') else [] _params += [self.data_dir] - data = execute_utility(_params, self.utils_log_file) + data = execute_utility2(self.os_ops, _params, self.utils_log_file) out_dict = {} @@ -793,7 +793,7 @@ def start(self, params=[], wait=True): def LOCAL__start_node(): # 'error' will be None on Windows - _, _, error = execute_utility(_params, self.utils_log_file, verbose=True) + _, _, error = execute_utility2(self.os_ops, _params, self.utils_log_file, verbose=True) assert error is None or type(error) == str # noqa: E721 if error and 'does not exist' in error: raise Exception(error) @@ -882,7 +882,7 @@ def stop(self, params=[], wait=True): "stop" ] + params # yapf: disable - execute_utility(_params, self.utils_log_file) + execute_utility2(self.os_ops, _params, self.utils_log_file) self._maybe_stop_logger() self.is_started = False @@ -924,7 +924,7 @@ def restart(self, params=[]): ] + params # yapf: disable try: - error_code, out, error = execute_utility(_params, self.utils_log_file, verbose=True) + error_code, out, error = execute_utility2(self.os_ops, _params, self.utils_log_file, verbose=True) if error and 'could not start server' in error: raise ExecUtilException except ExecUtilException as e: @@ -953,7 +953,7 @@ def reload(self, params=[]): "reload" ] + params # yapf: disable - execute_utility(_params, self.utils_log_file) + execute_utility2(self.os_ops, _params, self.utils_log_file) return self @@ -975,7 +975,7 @@ def promote(self, dbname=None, username=None): "promote" ] # yapf: disable - execute_utility(_params, self.utils_log_file) + execute_utility2(self.os_ops, _params, self.utils_log_file) # for versions below 10 `promote` is asynchronous so we need to wait # until it actually becomes writable @@ -1010,7 +1010,7 @@ def pg_ctl(self, params): "-w" # wait ] + params # yapf: disable - return execute_utility(_params, self.utils_log_file) + return execute_utility2(self.os_ops, _params, self.utils_log_file) def free_port(self): """ @@ -1230,7 +1230,7 @@ def tmpfile(): "-F", format.value ] # yapf: disable - execute_utility(_params, self.utils_log_file) + execute_utility2(self.os_ops, _params, self.utils_log_file) return filename @@ -1259,7 +1259,7 @@ def restore(self, filename, dbname=None, username=None): # try pg_restore if dump is binary format, and psql if not try: - execute_utility(_params, self.utils_log_name) + execute_utility2(self.os_ops, _params, self.utils_log_name) except ExecUtilException: self.psql(filename=filename, dbname=dbname, username=username) @@ -1612,7 +1612,7 @@ def pgbench_run(self, dbname=None, username=None, options=[], **kwargs): # should be the last one _params.append(dbname) - return execute_utility(_params, self.utils_log_file) + return execute_utility2(self.os_ops, _params, self.utils_log_file) def connect(self, dbname=None, @@ -1809,7 +1809,7 @@ def _get_bin_path(self, filename): if self.bin_dir: bin_path = os.path.join(self.bin_dir, filename) else: - bin_path = get_bin_path(filename) + bin_path = get_bin_path2(self.os_ops, filename) return bin_path def _escape_config_value(value): diff --git a/testgres/utils.py b/testgres/utils.py index 4bd232b1..9645fc3b 100644 --- a/testgres/utils.py +++ b/testgres/utils.py @@ -16,6 +16,8 @@ from .helpers.port_manager import PortManager from .exceptions import ExecUtilException from .config import testgres_config as tconf +from .operations.os_ops import OsOperations +from .operations.remote_ops import RemoteOperations # rows returned by PG_CONFIG _pg_config_data = {} @@ -68,7 +70,14 @@ def execute_utility(args, logfile=None, verbose=False): Returns: stdout of executed utility. """ - exit_status, out, error = tconf.os_ops.exec_command(args, verbose=True) + return execute_utility2(tconf.os_ops, args, logfile, verbose) + + +def execute_utility2(os_ops: OsOperations, args, logfile=None, verbose=False): + assert os_ops is not None + assert isinstance(os_ops, OsOperations) + + exit_status, out, error = os_ops.exec_command(args, verbose=True) # decode result out = '' if not out else out if isinstance(out, bytes): @@ -79,11 +88,11 @@ def execute_utility(args, logfile=None, verbose=False): # write new log entry if possible if logfile: try: - tconf.os_ops.write(filename=logfile, data=args, truncate=True) + os_ops.write(filename=logfile, data=args, truncate=True) if out: # comment-out lines lines = [u'\n'] + ['# ' + line for line in out.splitlines()] + [u'\n'] - tconf.os_ops.write(filename=logfile, data=lines) + os_ops.write(filename=logfile, data=lines) except IOError: raise ExecUtilException( "Problem with writing to logfile `{}` during run command `{}`".format(logfile, args)) @@ -98,25 +107,32 @@ def get_bin_path(filename): Return absolute path to an executable using PG_BIN or PG_CONFIG. This function does nothing if 'filename' is already absolute. """ + return get_bin_path2(tconf.os_ops, filename) + + +def get_bin_path2(os_ops: OsOperations, filename): + assert os_ops is not None + assert isinstance(os_ops, OsOperations) + # check if it's already absolute if os.path.isabs(filename): return filename - if tconf.os_ops.remote: + if isinstance(os_ops, RemoteOperations): pg_config = os.environ.get("PG_CONFIG_REMOTE") or os.environ.get("PG_CONFIG") else: # try PG_CONFIG - get from local machine pg_config = os.environ.get("PG_CONFIG") if pg_config: - bindir = get_pg_config()["BINDIR"] + bindir = get_pg_config(pg_config, os_ops)["BINDIR"] return os.path.join(bindir, filename) # try PG_BIN - pg_bin = tconf.os_ops.environ("PG_BIN") + pg_bin = os_ops.environ("PG_BIN") if pg_bin: return os.path.join(pg_bin, filename) - pg_config_path = tconf.os_ops.find_executable('pg_config') + pg_config_path = os_ops.find_executable('pg_config') if pg_config_path: bindir = get_pg_config(pg_config_path)["BINDIR"] return os.path.join(bindir, filename) @@ -129,12 +145,20 @@ def get_pg_config(pg_config_path=None, os_ops=None): Return output of pg_config (provided that it is installed). NOTE: this function caches the result by default (see GlobalConfig). """ - if os_ops: - tconf.os_ops = os_ops + + if os_ops is None: + os_ops = tconf.os_ops + + return get_pg_config2(os_ops, pg_config_path) + + +def get_pg_config2(os_ops: OsOperations, pg_config_path): + assert os_ops is not None + assert isinstance(os_ops, OsOperations) def cache_pg_config_data(cmd): # execute pg_config and get the output - out = tconf.os_ops.exec_command(cmd, encoding='utf-8') + out = os_ops.exec_command(cmd, encoding='utf-8') data = {} for line in out.splitlines(): @@ -158,7 +182,7 @@ def cache_pg_config_data(cmd): return _pg_config_data # try specified pg_config path or PG_CONFIG - if tconf.os_ops.remote: + if isinstance(os_ops, RemoteOperations): pg_config = pg_config_path or os.environ.get("PG_CONFIG_REMOTE") or os.environ.get("PG_CONFIG") else: # try PG_CONFIG - get from local machine From ed3ef60be4c09b135d86737755c2d4cbb7c5657f Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Mon, 24 Feb 2025 16:38:42 +0300 Subject: [PATCH 137/216] Code style (flake8) --- testgres/cache.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testgres/cache.py b/testgres/cache.py index 61d44868..3ac63326 100644 --- a/testgres/cache.py +++ b/testgres/cache.py @@ -32,11 +32,11 @@ def cached_initdb(data_dir, logfile=None, params=None, os_ops: OsOperations = Lo def make_utility_path(name): assert name is not None - assert type(name) == str + assert type(name) == str # noqa: E721 if bin_path: return os.path.join(bin_path, name) - + return get_bin_path2(os_ops, name) def call_initdb(initdb_dir, log=logfile): From 0eeb705151b23b7dabb0b677ff9371279389a6cf Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Wed, 26 Feb 2025 11:17:29 +0300 Subject: [PATCH 138/216] Tests are based on pytest (#192) * Using pytest [pytest.raises] * Using pytest [pytest.skip] * Using pytest [assertIsNotNone] * Using pytest [assertFalse] * Using pytest [assertTrue] * Using pytest [assertEqual] * Using pytest [assertNotEqual] * Using pytest [assertGreaterEqual] * Using pytest [assertGreater] * Using pytest [assertIn] * Using pytest [assertListEqual] * unittest is not used * Code style (flake8) * Execution signature is removed * run_tests.sh installs pytest * run_tests.sh is updated run tests through pytest explicitly * Total refactoring of tests - TestgresRemoteTests does not use global variables and code - Explicit work with ..testgres folder * Code style (flake8) * Root __init__.py is added It is required for tests. * Code style (flake8) * pytest.ini is added * TestgresTests::test_ports_management is corrected Let's send warning about a garbage in the container "bound_ports" and continue working. * coding: utf-8 * Cleanup * CI runs all the tests of testgres. * Add install ssh (cherry picked from commit fec1e7ac9d6e2bfb43a01f0e370336ba5ed8e971) * Revert "Add install ssh" This reverts commit 537a9acb9dfb26d82251d2d68796a55989be8317. * Revert "CI runs all the tests of testgres." This reverts commit 2d2532c77e8d7521552c0f3511c119e90d55573e. * Test of probackup plugin is restored It works now (was runned with a fresh probackup2 and vanilla 18devel). * The test suite of a probackup plugin is based on pytest * Probackup plugin is updated Probackup plugin tests - They are skipped if PGPROBACKUPBIN is not defined Global variable init_params is None when PGPROBACKUPBIN is not defined or version is not processed * CI test use 4 cores * testgres.plugins.probackup2.Init was restored [thanks to Yuri Sokolov] * pytest.ini is updated [testpaths] Enumeration of all the known folders with tests. * test_child_pids (local, remote) is updated Multiple attempts and logging are added. * test_child_process_dies is updated Multiple attempts are added. --------- Co-authored-by: vshepard --- __init__.py | 0 pytest.ini | 9 + run_tests.sh | 12 +- testgres/plugins/__init__.py | 8 +- .../pg_probackup2/init_helpers.py | 13 +- .../pg_probackup2/tests/__init__.py | 0 .../pg_probackup2/tests/basic_test.py | 80 --- .../pg_probackup2/tests/test_basic.py | 95 +++ tests/helpers/run_conditions.py | 1 + tests/test_local.py | 7 +- tests/test_remote.py | 9 +- tests/test_simple.py | 607 +++++++++-------- tests/test_simple_remote.py | 615 ++++++++++-------- 13 files changed, 839 insertions(+), 617 deletions(-) create mode 100644 __init__.py create mode 100644 pytest.ini create mode 100644 testgres/plugins/pg_probackup2/pg_probackup2/tests/__init__.py delete mode 100644 testgres/plugins/pg_probackup2/pg_probackup2/tests/basic_test.py create mode 100644 testgres/plugins/pg_probackup2/pg_probackup2/tests/test_basic.py diff --git a/__init__.py b/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 00000000..c94eabc2 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,9 @@ +[pytest] +testpaths = ["./tests", "./testgres/plugins/pg_probackup2/pg_probackup2/tests"] +addopts = --strict-markers +markers = +#log_file = logs/pytest.log +log_file_level = NOTSET +log_file_format = %(levelname)8s [%(asctime)s] %(message)s +log_file_date_format=%Y-%m-%d %H:%M:%S + diff --git a/run_tests.sh b/run_tests.sh index 73c459be..e9d58b54 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -22,11 +22,11 @@ export VIRTUAL_ENV_DISABLE_PROMPT=1 source $VENV_PATH/bin/activate # install utilities -$PIP install coverage flake8 psutil Sphinx +$PIP install coverage flake8 psutil Sphinx pytest pytest-xdist psycopg2 six psutil # install testgres' dependencies export PYTHONPATH=$(pwd) -$PIP install . +# $PIP install . # test code quality flake8 . @@ -38,21 +38,19 @@ rm -f $COVERAGE_FILE # run tests (PATH) -time coverage run -a tests/test_simple.py +time coverage run -a -m pytest -l -v -n 4 -k "TestgresTests" # run tests (PG_BIN) time \ PG_BIN=$(dirname $(which pg_config)) \ - ALT_CONFIG=1 \ - coverage run -a tests/test_simple.py + coverage run -a -m pytest -l -v -n 4 -k "TestgresTests" # run tests (PG_CONFIG) time \ PG_CONFIG=$(which pg_config) \ - ALT_CONFIG=1 \ - coverage run -a tests/test_simple.py + coverage run -a -m pytest -l -v -n 4 -k "TestgresTests" # show coverage diff --git a/testgres/plugins/__init__.py b/testgres/plugins/__init__.py index 8c19a23b..824eadc6 100644 --- a/testgres/plugins/__init__.py +++ b/testgres/plugins/__init__.py @@ -1,7 +1,7 @@ -from pg_probackup2.gdb import GDBobj -from pg_probackup2.app import ProbackupApp, ProbackupException -from pg_probackup2.init_helpers import init_params -from pg_probackup2.storage.fs_backup import FSTestBackupDir +from .pg_probackup2.pg_probackup2.gdb import GDBobj +from .pg_probackup2.pg_probackup2.app import ProbackupApp, ProbackupException +from .pg_probackup2.pg_probackup2.init_helpers import init_params +from .pg_probackup2.pg_probackup2.storage.fs_backup import FSTestBackupDir __all__ = [ "ProbackupApp", "ProbackupException", "init_params", "FSTestBackupDir", "GDBobj" diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py index 078fdbab..c4570a39 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py @@ -121,8 +121,7 @@ def __init__(self): self.probackup_path = probackup_path_tmp if not self.probackup_path: - logging.error('pg_probackup binary is not found') - exit(1) + raise Exception('pg_probackup binary is not found') if os.name == 'posix': self.EXTERNAL_DIRECTORY_DELIMITER = ':' @@ -213,11 +212,15 @@ def __init__(self): if self.probackup_version.split('.')[0].isdigit(): self.major_version = int(self.probackup_version.split('.')[0]) else: - logging.error('Can\'t process pg_probackup version \"{}\": the major version is expected to be a number'.format(self.probackup_version)) - sys.exit(1) + raise Exception('Can\'t process pg_probackup version \"{}\": the major version is expected to be a number'.format(self.probackup_version)) def test_env(self): return self._test_env.copy() -init_params = Init() +try: + init_params = Init() +except Exception as e: + logging.error(str(e)) + logging.warning("testgres.plugins.probackup2.init_params is set to None.") + init_params = None diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/tests/__init__.py b/testgres/plugins/pg_probackup2/pg_probackup2/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/tests/basic_test.py b/testgres/plugins/pg_probackup2/pg_probackup2/tests/basic_test.py deleted file mode 100644 index b63531ec..00000000 --- a/testgres/plugins/pg_probackup2/pg_probackup2/tests/basic_test.py +++ /dev/null @@ -1,80 +0,0 @@ -import logging -import os -import shutil -import unittest -import testgres -from pg_probackup2.app import ProbackupApp -from pg_probackup2.init_helpers import Init, init_params -from pg_probackup2.app import build_backup_dir - - -class TestUtils: - @staticmethod - def get_module_and_function_name(test_id): - try: - module_name = test_id.split('.')[-2] - fname = test_id.split('.')[-1] - except IndexError: - logging.warning(f"Couldn't get module name and function name from test_id: `{test_id}`") - module_name, fname = test_id.split('(')[1].split('.')[1], test_id.split('(')[0] - return module_name, fname - - -class ProbackupTest(unittest.TestCase): - def setUp(self): - self.setup_test_environment() - self.setup_test_paths() - self.setup_backup_dir() - self.setup_probackup() - - def setup_test_environment(self): - self.output = None - self.cmd = None - self.nodes_to_cleanup = [] - self.module_name, self.fname = TestUtils.get_module_and_function_name(self.id()) - self.test_env = Init().test_env() - - def setup_test_paths(self): - self.rel_path = os.path.join(self.module_name, self.fname) - self.test_path = os.path.join(init_params.tmp_path, self.rel_path) - os.makedirs(self.test_path) - self.pb_log_path = os.path.join(self.test_path, "pb_log") - - def setup_backup_dir(self): - self.backup_dir = build_backup_dir(self, 'backup') - self.backup_dir.cleanup() - - def setup_probackup(self): - self.pg_node = testgres.NodeApp(self.test_path, self.nodes_to_cleanup) - self.pb = ProbackupApp(self, self.pg_node, self.pb_log_path, self.test_env, - auto_compress_alg='zlib', backup_dir=self.backup_dir) - - def tearDown(self): - if os.path.exists(self.test_path): - shutil.rmtree(self.test_path) - - -class BasicTest(ProbackupTest): - def test_full_backup(self): - # Setting up a simple test node - node = self.pg_node.make_simple('node', pg_options={"fsync": "off", "synchronous_commit": "off"}) - - # Initialize and configure Probackup - self.pb.init() - self.pb.add_instance('node', node) - self.pb.set_archiving('node', node) - - # Start the node and initialize pgbench - node.slow_start() - node.pgbench_init(scale=100, no_vacuum=True) - - # Perform backup and validation - backup_id = self.pb.backup_node('node', node) - out = self.pb.validate('node', backup_id) - - # Check if the backup is valid - self.assertIn(f"INFO: Backup {backup_id} is valid", out) - - -if __name__ == "__main__": - unittest.main() diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/tests/test_basic.py b/testgres/plugins/pg_probackup2/pg_probackup2/tests/test_basic.py new file mode 100644 index 00000000..ba788623 --- /dev/null +++ b/testgres/plugins/pg_probackup2/pg_probackup2/tests/test_basic.py @@ -0,0 +1,95 @@ +from __future__ import annotations + +import os +import shutil +import pytest + +from ...... import testgres +from ...pg_probackup2.app import ProbackupApp +from ...pg_probackup2.init_helpers import Init, init_params +from ..storage.fs_backup import FSTestBackupDir + + +class ProbackupTest: + pg_node: testgres.PostgresNode + + @staticmethod + def probackup_is_available() -> bool: + p = os.environ.get("PGPROBACKUPBIN") + + if p is None: + return False + + if not os.path.exists(p): + return False + + return True + + @pytest.fixture(autouse=True, scope="function") + def implicit_fixture(self, request: pytest.FixtureRequest): + assert isinstance(request, pytest.FixtureRequest) + self.helper__setUp(request) + yield + self.helper__tearDown() + + def helper__setUp(self, request: pytest.FixtureRequest): + assert isinstance(request, pytest.FixtureRequest) + + self.helper__setup_test_environment(request) + self.helper__setup_test_paths() + self.helper__setup_backup_dir() + self.helper__setup_probackup() + + def helper__setup_test_environment(self, request: pytest.FixtureRequest): + assert isinstance(request, pytest.FixtureRequest) + + self.output = None + self.cmd = None + self.nodes_to_cleanup = [] + self.module_name, self.fname = request.node.cls.__name__, request.node.name + self.test_env = Init().test_env() + + def helper__setup_test_paths(self): + self.rel_path = os.path.join(self.module_name, self.fname) + self.test_path = os.path.join(init_params.tmp_path, self.rel_path) + os.makedirs(self.test_path, exist_ok=True) + self.pb_log_path = os.path.join(self.test_path, "pb_log") + + def helper__setup_backup_dir(self): + self.backup_dir = self.helper__build_backup_dir('backup') + self.backup_dir.cleanup() + + def helper__setup_probackup(self): + self.pg_node = testgres.NodeApp(self.test_path, self.nodes_to_cleanup) + self.pb = ProbackupApp(self, self.pg_node, self.pb_log_path, self.test_env, + auto_compress_alg='zlib', backup_dir=self.backup_dir) + + def helper__tearDown(self): + if os.path.exists(self.test_path): + shutil.rmtree(self.test_path) + + def helper__build_backup_dir(self, backup='backup'): + return FSTestBackupDir(rel_path=self.rel_path, backup=backup) + + +@pytest.mark.skipif(not ProbackupTest.probackup_is_available(), reason="Check that PGPROBACKUPBIN is defined and is valid.") +class TestBasic(ProbackupTest): + def test_full_backup(self): + # Setting up a simple test node + node = self.pg_node.make_simple('node', pg_options={"fsync": "off", "synchronous_commit": "off"}) + + # Initialize and configure Probackup + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) + + # Start the node and initialize pgbench + node.slow_start() + node.pgbench_init(scale=100, no_vacuum=True) + + # Perform backup and validation + backup_id = self.pb.backup_node('node', node) + out = self.pb.validate('node', backup_id) + + # Check if the backup is valid + assert f"INFO: Backup {backup_id} is valid" in out diff --git a/tests/helpers/run_conditions.py b/tests/helpers/run_conditions.py index 8d57f753..11357c30 100644 --- a/tests/helpers/run_conditions.py +++ b/tests/helpers/run_conditions.py @@ -1,3 +1,4 @@ +# coding: utf-8 import pytest import platform diff --git a/tests/test_local.py b/tests/test_local.py index 4051bfb5..60a96c18 100644 --- a/tests/test_local.py +++ b/tests/test_local.py @@ -1,12 +1,13 @@ +# coding: utf-8 import os import pytest import re import tempfile -from testgres import ExecUtilException -from testgres import InvalidOperationException -from testgres import LocalOperations +from ..testgres import ExecUtilException +from ..testgres import InvalidOperationException +from ..testgres import LocalOperations from .helpers.run_conditions import RunConditions diff --git a/tests/test_remote.py b/tests/test_remote.py index 4330b92f..8b167e9f 100755 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -1,13 +1,14 @@ +# coding: utf-8 import os import pytest import re import tempfile -from testgres import ExecUtilException -from testgres import InvalidOperationException -from testgres import RemoteOperations -from testgres import ConnectionParams +from ..testgres import ExecUtilException +from ..testgres import InvalidOperationException +from ..testgres import RemoteOperations +from ..testgres import ConnectionParams class TestRemoteOperations: diff --git a/tests/test_simple.py b/tests/test_simple.py index a751f0a3..6c433cd4 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -1,22 +1,22 @@ -#!/usr/bin/env python # coding: utf-8 - import os import re import subprocess import tempfile -import testgres import time import six -import unittest +import pytest import psutil +import platform import logging.config from contextlib import contextmanager from shutil import rmtree -from testgres import \ +from .. import testgres + +from ..testgres import \ InitNodeException, \ StartNodeException, \ ExecUtilException, \ @@ -27,31 +27,32 @@ InvalidOperationException, \ NodeApp -from testgres import \ +from ..testgres import \ TestgresConfig, \ configure_testgres, \ scoped_config, \ pop_config -from testgres import \ +from ..testgres import \ NodeStatus, \ ProcessType, \ IsolationLevel, \ get_new_node -from testgres import \ +from ..testgres import \ get_bin_path, \ get_pg_config, \ get_pg_version -from testgres import \ +from ..testgres import \ First, \ Any # NOTE: those are ugly imports -from testgres import bound_ports -from testgres.utils import PgVer, parse_pg_version -from testgres.node import ProcessProxy +from ..testgres import bound_ports +from ..testgres.utils import PgVer, parse_pg_version +from ..testgres.utils import file_tail +from ..testgres.node import ProcessProxy def pg_version_ge(version): @@ -105,11 +106,11 @@ def removing(f): rmtree(f, ignore_errors=True) -class TestgresTests(unittest.TestCase): +class TestgresTests: def test_node_repr(self): with get_new_node() as node: pattern = r"PostgresNode\(name='.+', port=.+, base_dir='.+'\)" - self.assertIsNotNone(re.match(pattern, str(node))) + assert re.match(pattern, str(node)) is not None def test_custom_init(self): with get_new_node() as node: @@ -126,15 +127,15 @@ def test_custom_init(self): lines = conf.readlines() # check number of lines - self.assertGreaterEqual(len(lines), 6) + assert (len(lines) >= 6) # there should be no trust entries at all - self.assertFalse(any('trust' in s for s in lines)) + assert not (any('trust' in s for s in lines)) def test_double_init(self): with get_new_node().init() as node: # can't initialize node more than once - with self.assertRaises(InitNodeException): + with pytest.raises(expected_exception=InitNodeException): node.init() def test_init_after_cleanup(self): @@ -143,10 +144,11 @@ def test_init_after_cleanup(self): node.cleanup() node.init().start().execute('select 1') - @unittest.skipUnless(util_exists('pg_resetwal.exe' if os.name == 'nt' else 'pg_resetwal'), 'pgbench might be missing') - @unittest.skipUnless(pg_version_ge('9.6'), 'requires 9.6+') def test_init_unique_system_id(self): # this function exists in PostgreSQL 9.6+ + __class__.helper__skip_test_if_util_not_exist("pg_resetwal") + __class__.helper__skip_test_if_pg_version_is_not_ge("9.6") + query = 'select system_identifier from pg_control_system()' with scoped_config(cache_initdb=False): @@ -155,8 +157,8 @@ def test_init_unique_system_id(self): with scoped_config(cache_initdb=True, cached_initdb_unique=True) as config: - self.assertTrue(config.cache_initdb) - self.assertTrue(config.cached_initdb_unique) + assert (config.cache_initdb) + assert (config.cached_initdb_unique) # spawn two nodes; ids must be different with get_new_node().init().start() as node1, \ @@ -166,37 +168,37 @@ def test_init_unique_system_id(self): id2 = node2.execute(query)[0] # ids must increase - self.assertGreater(id1, id0) - self.assertGreater(id2, id1) + assert (id1 > id0) + assert (id2 > id1) def test_node_exit(self): base_dir = None - with self.assertRaises(QueryException): + with pytest.raises(expected_exception=QueryException): with get_new_node().init() as node: base_dir = node.base_dir node.safe_psql('select 1') # we should save the DB for "debugging" - self.assertTrue(os.path.exists(base_dir)) + assert (os.path.exists(base_dir)) rmtree(base_dir, ignore_errors=True) with get_new_node().init() as node: base_dir = node.base_dir # should have been removed by default - self.assertFalse(os.path.exists(base_dir)) + assert not (os.path.exists(base_dir)) def test_double_start(self): with get_new_node().init().start() as node: # can't start node more than once node.start() - self.assertTrue(node.is_started) + assert (node.is_started) def test_uninitialized_start(self): with get_new_node() as node: # node is not initialized yet - with self.assertRaises(StartNodeException): + with pytest.raises(expected_exception=StartNodeException): node.start() def test_restart(self): @@ -205,13 +207,13 @@ def test_restart(self): # restart, ok res = node.execute('select 1') - self.assertEqual(res, [(1, )]) + assert (res == [(1, )]) node.restart() res = node.execute('select 2') - self.assertEqual(res, [(2, )]) + assert (res == [(2, )]) # restart, fail - with self.assertRaises(StartNodeException): + with pytest.raises(expected_exception=StartNodeException): node.append_conf('pg_hba.conf', 'DUMMY') node.restart() @@ -228,106 +230,107 @@ def test_reload(self): # check new value cmm_new = node.execute('show client_min_messages') - self.assertEqual('debug1', cmm_new[0][0].lower()) - self.assertNotEqual(cmm_old, cmm_new) + assert ('debug1' == cmm_new[0][0].lower()) + assert (cmm_old != cmm_new) def test_pg_ctl(self): with get_new_node() as node: node.init().start() status = node.pg_ctl(['status']) - self.assertTrue('PID' in status) + assert ('PID' in status) def test_status(self): - self.assertTrue(NodeStatus.Running) - self.assertFalse(NodeStatus.Stopped) - self.assertFalse(NodeStatus.Uninitialized) + assert (NodeStatus.Running) + assert not (NodeStatus.Stopped) + assert not (NodeStatus.Uninitialized) # check statuses after each operation with get_new_node() as node: - self.assertEqual(node.pid, 0) - self.assertEqual(node.status(), NodeStatus.Uninitialized) + assert (node.pid == 0) + assert (node.status() == NodeStatus.Uninitialized) node.init() - self.assertEqual(node.pid, 0) - self.assertEqual(node.status(), NodeStatus.Stopped) + assert (node.pid == 0) + assert (node.status() == NodeStatus.Stopped) node.start() - self.assertNotEqual(node.pid, 0) - self.assertEqual(node.status(), NodeStatus.Running) + assert (node.pid != 0) + assert (node.status() == NodeStatus.Running) node.stop() - self.assertEqual(node.pid, 0) - self.assertEqual(node.status(), NodeStatus.Stopped) + assert (node.pid == 0) + assert (node.status() == NodeStatus.Stopped) node.cleanup() - self.assertEqual(node.pid, 0) - self.assertEqual(node.status(), NodeStatus.Uninitialized) + assert (node.pid == 0) + assert (node.status() == NodeStatus.Uninitialized) def test_psql(self): with get_new_node().init().start() as node: # check returned values (1 arg) res = node.psql('select 1') - self.assertEqual(rm_carriage_returns(res), (0, b'1\n', b'')) + assert (rm_carriage_returns(res) == (0, b'1\n', b'')) # check returned values (2 args) res = node.psql('postgres', 'select 2') - self.assertEqual(rm_carriage_returns(res), (0, b'2\n', b'')) + assert (rm_carriage_returns(res) == (0, b'2\n', b'')) # check returned values (named) res = node.psql(query='select 3', dbname='postgres') - self.assertEqual(rm_carriage_returns(res), (0, b'3\n', b'')) + assert (rm_carriage_returns(res) == (0, b'3\n', b'')) # check returned values (1 arg) res = node.safe_psql('select 4') - self.assertEqual(rm_carriage_returns(res), b'4\n') + assert (rm_carriage_returns(res) == b'4\n') # check returned values (2 args) res = node.safe_psql('postgres', 'select 5') - self.assertEqual(rm_carriage_returns(res), b'5\n') + assert (rm_carriage_returns(res) == b'5\n') # check returned values (named) res = node.safe_psql(query='select 6', dbname='postgres') - self.assertEqual(rm_carriage_returns(res), b'6\n') + assert (rm_carriage_returns(res) == b'6\n') # check feeding input node.safe_psql('create table horns (w int)') node.safe_psql('copy horns from stdin (format csv)', input=b"1\n2\n3\n\\.\n") _sum = node.safe_psql('select sum(w) from horns') - self.assertEqual(rm_carriage_returns(_sum), b'6\n') + assert (rm_carriage_returns(_sum) == b'6\n') # check psql's default args, fails - with self.assertRaises(QueryException): + with pytest.raises(expected_exception=QueryException): node.psql() node.stop() # check psql on stopped node, fails - with self.assertRaises(QueryException): + with pytest.raises(expected_exception=QueryException): node.safe_psql('select 1') def test_safe_psql__expect_error(self): with get_new_node().init().start() as node: err = node.safe_psql('select_or_not_select 1', expect_error=True) - self.assertTrue(type(err) == str) # noqa: E721 - self.assertIn('select_or_not_select', err) - self.assertIn('ERROR: syntax error at or near "select_or_not_select"', err) + assert (type(err) == str) # noqa: E721 + assert ('select_or_not_select' in err) + assert ('ERROR: syntax error at or near "select_or_not_select"' in err) # --------- - with self.assertRaises(InvalidOperationException) as ctx: + with pytest.raises( + expected_exception=InvalidOperationException, + match="^" + re.escape("Exception was expected, but query finished successfully: `select 1;`.") + "$" + ): node.safe_psql("select 1;", expect_error=True) - self.assertEqual(str(ctx.exception), "Exception was expected, but query finished successfully: `select 1;`.") - # --------- res = node.safe_psql("select 1;", expect_error=False) - self.assertEqual(rm_carriage_returns(res), b'1\n') + assert (rm_carriage_returns(res) == b'1\n') def test_transactions(self): with get_new_node().init().start() as node: @@ -341,12 +344,12 @@ def test_transactions(self): con.begin() con.execute('insert into test values (2)') res = con.execute('select * from test order by val asc') - self.assertListEqual(res, [(1, ), (2, )]) + assert (res == [(1, ), (2, )]) con.rollback() con.begin() res = con.execute('select * from test') - self.assertListEqual(res, [(1, )]) + assert (res == [(1, )]) con.rollback() con.begin() @@ -357,15 +360,15 @@ def test_control_data(self): with get_new_node() as node: # node is not initialized yet - with self.assertRaises(ExecUtilException): + with pytest.raises(expected_exception=ExecUtilException): node.get_control_data() node.init() data = node.get_control_data() # check returned dict - self.assertIsNotNone(data) - self.assertTrue(any('pg_control' in s for s in data.keys())) + assert data is not None + assert (any('pg_control' in s for s in data.keys())) def test_backup_simple(self): with get_new_node() as master: @@ -374,7 +377,7 @@ def test_backup_simple(self): master.init(allow_streaming=True) # node must be running - with self.assertRaises(BackupException): + with pytest.raises(expected_exception=BackupException): master.backup() # it's time to start node @@ -386,7 +389,7 @@ def test_backup_simple(self): with master.backup(xlog_method='stream') as backup: with backup.spawn_primary().start() as slave: res = slave.execute('select * from test order by i asc') - self.assertListEqual(res, [(1, ), (2, ), (3, ), (4, )]) + assert (res == [(1, ), (2, ), (3, ), (4, )]) def test_backup_multiple(self): with get_new_node() as node: @@ -394,12 +397,12 @@ def test_backup_multiple(self): with node.backup(xlog_method='fetch') as backup1, \ node.backup(xlog_method='fetch') as backup2: - self.assertNotEqual(backup1.base_dir, backup2.base_dir) + assert (backup1.base_dir != backup2.base_dir) with node.backup(xlog_method='fetch') as backup: with backup.spawn_primary('node1', destroy=False) as node1, \ backup.spawn_primary('node2', destroy=False) as node2: - self.assertNotEqual(node1.base_dir, node2.base_dir) + assert (node1.base_dir != node2.base_dir) def test_backup_exhaust(self): with get_new_node() as node: @@ -411,15 +414,17 @@ def test_backup_exhaust(self): pass # now let's try to create one more node - with self.assertRaises(BackupException): + with pytest.raises(expected_exception=BackupException): backup.spawn_primary() def test_backup_wrong_xlog_method(self): with get_new_node() as node: node.init(allow_streaming=True).start() - with self.assertRaises(BackupException, - msg='Invalid xlog_method "wrong"'): + with pytest.raises( + expected_exception=BackupException, + match="^" + re.escape('Invalid xlog_method "wrong"') + "$" + ): node.backup(xlog_method='wrong') def test_pg_ctl_wait_option(self): @@ -440,17 +445,18 @@ def test_replicate(self): with node.replicate().start() as replica: res = replica.execute('select 1') - self.assertListEqual(res, [(1, )]) + assert (res == [(1, )]) node.execute('create table test (val int)', commit=True) replica.catchup() res = node.execute('select * from test') - self.assertListEqual(res, []) + assert (res == []) - @unittest.skipUnless(pg_version_ge('9.6'), 'requires 9.6+') def test_synchronous_replication(self): + __class__.helper__skip_test_if_pg_version_is_not_ge("9.6") + with get_new_node() as master: old_version = not pg_version_ge('9.6') @@ -465,12 +471,12 @@ def test_synchronous_replication(self): standby2.start() # check formatting - self.assertEqual( - '1 ("{}", "{}")'.format(standby1.name, standby2.name), - str(First(1, (standby1, standby2)))) # yapf: disable - self.assertEqual( - 'ANY 1 ("{}", "{}")'.format(standby1.name, standby2.name), - str(Any(1, (standby1, standby2)))) # yapf: disable + assert ( + '1 ("{}", "{}")'.format(standby1.name, standby2.name) == str(First(1, (standby1, standby2))) + ) # yapf: disable + assert ( + 'ANY 1 ("{}", "{}")'.format(standby1.name, standby2.name) == str(Any(1, (standby1, standby2))) + ) # yapf: disable # set synchronous_standby_names master.set_synchronous_standbys(First(2, [standby1, standby2])) @@ -488,10 +494,11 @@ def test_synchronous_replication(self): master.safe_psql( 'insert into abc select generate_series(1, 1000000)') res = standby1.safe_psql('select count(*) from abc') - self.assertEqual(rm_carriage_returns(res), b'1000000\n') + assert (rm_carriage_returns(res) == b'1000000\n') - @unittest.skipUnless(pg_version_ge('10'), 'requires 10+') def test_logical_replication(self): + __class__.helper__skip_test_if_pg_version_is_not_ge("10") + with get_new_node() as node1, get_new_node() as node2: node1.init(allow_logical=True) node1.start() @@ -510,7 +517,7 @@ def test_logical_replication(self): # wait until changes apply on subscriber and check them sub.catchup() res = node2.execute('select * from test') - self.assertListEqual(res, [(1, 1), (2, 2)]) + assert (res == [(1, 1), (2, 2)]) # disable and put some new data sub.disable() @@ -520,7 +527,7 @@ def test_logical_replication(self): sub.enable() sub.catchup() res = node2.execute('select * from test') - self.assertListEqual(res, [(1, 1), (2, 2), (3, 3)]) + assert (res == [(1, 1), (2, 2), (3, 3)]) # Add new tables. Since we added "all tables" to publication # (default behaviour of publish() method) we don't need @@ -534,7 +541,7 @@ def test_logical_replication(self): node1.safe_psql('insert into test2 values (\'a\'), (\'b\')') sub.catchup() res = node2.execute('select * from test2') - self.assertListEqual(res, [('a', ), ('b', )]) + assert (res == [('a', ), ('b', )]) # drop subscription sub.drop() @@ -548,20 +555,21 @@ def test_logical_replication(self): node1.safe_psql('insert into test values (4, 4)') sub.catchup() res = node2.execute('select * from test') - self.assertListEqual(res, [(1, 1), (2, 2), (3, 3), (4, 4)]) + assert (res == [(1, 1), (2, 2), (3, 3), (4, 4)]) # explicitly add table - with self.assertRaises(ValueError): + with pytest.raises(expected_exception=ValueError): pub.add_tables([]) # fail pub.add_tables(['test2']) node1.safe_psql('insert into test2 values (\'c\')') sub.catchup() res = node2.execute('select * from test2') - self.assertListEqual(res, [('a', ), ('b', )]) + assert (res == [('a', ), ('b', )]) - @unittest.skipUnless(pg_version_ge('10'), 'requires 10+') def test_logical_catchup(self): """ Runs catchup for 100 times to be sure that it is consistent """ + __class__.helper__skip_test_if_pg_version_is_not_ge("10") + with get_new_node() as node1, get_new_node() as node2: node1.init(allow_logical=True) node1.start() @@ -579,16 +587,14 @@ def test_logical_catchup(self): node1.execute('insert into test values ({0}, {0})'.format(i)) sub.catchup() res = node2.execute('select * from test') - self.assertListEqual(res, [( - i, - i, - )]) + assert (res == [(i, i, )]) node1.execute('delete from test') - @unittest.skipIf(pg_version_ge('10'), 'requires <10') def test_logical_replication_fail(self): + __class__.helper__skip_test_if_pg_version_is_ge("10") + with get_new_node() as node: - with self.assertRaises(InitNodeException): + with pytest.raises(expected_exception=InitNodeException): node.init(allow_logical=True) def test_replication_slots(self): @@ -599,7 +605,7 @@ def test_replication_slots(self): replica.execute('select 1') # cannot create new slot with the same name - with self.assertRaises(TestgresException): + with pytest.raises(expected_exception=TestgresException): node.replicate(slot='slot1') def test_incorrect_catchup(self): @@ -607,7 +613,7 @@ def test_incorrect_catchup(self): node.init(allow_streaming=True).start() # node has no master, can't catch up - with self.assertRaises(TestgresException): + with pytest.raises(expected_exception=TestgresException): node.catchup() def test_promotion(self): @@ -622,7 +628,7 @@ def test_promotion(self): # make standby becomes writable master replica.safe_psql('insert into abc values (1)') res = replica.safe_psql('select * from abc') - self.assertEqual(rm_carriage_returns(res), b'1\n') + assert (rm_carriage_returns(res) == b'1\n') def test_dump(self): query_create = 'create table test as select generate_series(1, 2) as val' @@ -635,20 +641,20 @@ def test_dump(self): with removing(node1.dump(format=format)) as dump: with get_new_node().init().start() as node3: if format == 'directory': - self.assertTrue(os.path.isdir(dump)) + assert (os.path.isdir(dump)) else: - self.assertTrue(os.path.isfile(dump)) + assert (os.path.isfile(dump)) # restore dump node3.restore(filename=dump) res = node3.execute(query_select) - self.assertListEqual(res, [(1, ), (2, )]) + assert (res == [(1, ), (2, )]) def test_users(self): with get_new_node().init().start() as node: node.psql('create role test_user login') value = node.safe_psql('select 1', username='test_user') value = rm_carriage_returns(value) - self.assertEqual(value, b'1\n') + assert (value == b'1\n') def test_poll_query_until(self): with get_new_node() as node: @@ -661,15 +667,15 @@ def test_poll_query_until(self): node.poll_query_until(query=check_time.format(start_time)) end_time = node.execute(get_time)[0][0] - self.assertTrue(end_time - start_time >= 5) + assert (end_time - start_time >= 5) # check 0 columns - with self.assertRaises(QueryException): + with pytest.raises(expected_exception=QueryException): node.poll_query_until( query='select from pg_catalog.pg_class limit 1') # check None, fail - with self.assertRaises(QueryException): + with pytest.raises(expected_exception=QueryException): node.poll_query_until(query='create table abc (val int)') # check None, ok @@ -682,7 +688,7 @@ def test_poll_query_until(self): expected=None) # check arbitrary expected value, fail - with self.assertRaises(TimeoutException): + with pytest.raises(expected_exception=TimeoutException): node.poll_query_until(query='select 3', expected=1, max_attempts=3, @@ -692,17 +698,17 @@ def test_poll_query_until(self): node.poll_query_until(query='select 2', expected=2) # check timeout - with self.assertRaises(TimeoutException): + with pytest.raises(expected_exception=TimeoutException): node.poll_query_until(query='select 1 > 2', max_attempts=3, sleep_time=0.01) # check ProgrammingError, fail - with self.assertRaises(testgres.ProgrammingError): + with pytest.raises(expected_exception=testgres.ProgrammingError): node.poll_query_until(query='dummy1') # check ProgrammingError, ok - with self.assertRaises(TimeoutException): + with pytest.raises(expected_exception=(TimeoutException)): node.poll_query_until(query='dummy2', max_attempts=3, sleep_time=0.01, @@ -754,16 +760,17 @@ def test_logging(self): # check that master's port is found with open(logfile.name, 'r') as log: lines = log.readlines() - self.assertTrue(any(node_name in s for s in lines)) + assert (any(node_name in s for s in lines)) # test logger after stop/start/restart master.stop() master.start() master.restart() - self.assertTrue(master._logger.is_alive()) + assert (master._logger.is_alive()) - @unittest.skipUnless(util_exists('pgbench.exe' if os.name == 'nt' else 'pgbench'), 'pgbench might be missing') def test_pgbench(self): + __class__.helper__skip_test_if_util_not_exist("pgbench") + with get_new_node().init().start() as node: # initialize pgbench DB and run benchmarks @@ -780,13 +787,13 @@ def test_pgbench(self): proc.stdout.close() - self.assertTrue('tps' in out) + assert ('tps' in out) def test_pg_config(self): # check same instances a = get_pg_config() b = get_pg_config() - self.assertEqual(id(a), id(b)) + assert (id(a) == id(b)) # save right before config change c1 = get_pg_config() @@ -794,26 +801,26 @@ def test_pg_config(self): # modify setting for this scope with scoped_config(cache_pg_config=False) as config: # sanity check for value - self.assertFalse(config.cache_pg_config) + assert not (config.cache_pg_config) # save right after config change c2 = get_pg_config() # check different instances after config change - self.assertNotEqual(id(c1), id(c2)) + assert (id(c1) != id(c2)) # check different instances a = get_pg_config() b = get_pg_config() - self.assertNotEqual(id(a), id(b)) + assert (id(a) != id(b)) def test_config_stack(self): # no such option - with self.assertRaises(TypeError): + with pytest.raises(expected_exception=TypeError): configure_testgres(dummy=True) # we have only 1 config in stack - with self.assertRaises(IndexError): + with pytest.raises(expected_exception=IndexError): pop_config() d0 = TestgresConfig.cached_initdb_dir @@ -821,22 +828,22 @@ def test_config_stack(self): d2 = 'dummy_def' with scoped_config(cached_initdb_dir=d1) as c1: - self.assertEqual(c1.cached_initdb_dir, d1) + assert (c1.cached_initdb_dir == d1) with scoped_config(cached_initdb_dir=d2) as c2: stack_size = len(testgres.config.config_stack) # try to break a stack - with self.assertRaises(TypeError): + with pytest.raises(expected_exception=TypeError): with scoped_config(dummy=True): pass - self.assertEqual(c2.cached_initdb_dir, d2) - self.assertEqual(len(testgres.config.config_stack), stack_size) + assert (c2.cached_initdb_dir == d2) + assert (len(testgres.config.config_stack) == stack_size) - self.assertEqual(c1.cached_initdb_dir, d1) + assert (c1.cached_initdb_dir == d1) - self.assertEqual(TestgresConfig.cached_initdb_dir, d0) + assert (TestgresConfig.cached_initdb_dir == d0) def test_unix_sockets(self): with get_new_node() as node: @@ -854,17 +861,15 @@ def test_auto_name(self): with get_new_node().init(allow_streaming=True).start() as m: with m.replicate().start() as r: # check that nodes are running - self.assertTrue(m.status()) - self.assertTrue(r.status()) + assert (m.status()) + assert (r.status()) # check their names - self.assertNotEqual(m.name, r.name) - self.assertTrue('testgres' in m.name) - self.assertTrue('testgres' in r.name) + assert (m.name != r.name) + assert ('testgres' in m.name) + assert ('testgres' in r.name) def test_file_tail(self): - from testgres.utils import file_tail - s1 = "the quick brown fox jumped over that lazy dog\n" s2 = "abc\n" s3 = "def\n" @@ -879,13 +884,13 @@ def test_file_tail(self): f.seek(0) lines = file_tail(f, 3) - self.assertEqual(lines[0], s1) - self.assertEqual(lines[1], s2) - self.assertEqual(lines[2], s3) + assert (lines[0] == s1) + assert (lines[1] == s2) + assert (lines[2] == s3) f.seek(0) lines = file_tail(f, 1) - self.assertEqual(lines[0], s3) + assert (lines[0] == s3) def test_isolation_levels(self): with get_new_node().init().start() as node: @@ -903,24 +908,42 @@ def test_isolation_levels(self): con.begin(IsolationLevel.Serializable).commit() # check wrong level - with self.assertRaises(QueryException): + with pytest.raises(expected_exception=QueryException): con.begin('Garbage').commit() def test_ports_management(self): - # check that no ports have been bound yet - self.assertEqual(len(bound_ports), 0) + assert bound_ports is not None + assert type(bound_ports) == set # noqa: E721 + + if len(bound_ports) != 0: + logging.warning("bound_ports is not empty: {0}".format(bound_ports)) + + stage0__bound_ports = bound_ports.copy() with get_new_node() as node: - # check that we've just bound a port - self.assertEqual(len(bound_ports), 1) + assert bound_ports is not None + assert type(bound_ports) == set # noqa: E721 + + assert node.port is not None + assert type(node.port) == int # noqa: E721 + + logging.info("node port is {0}".format(node.port)) - # check that bound_ports contains our port - port_1 = list(bound_ports)[0] - port_2 = node.port - self.assertEqual(port_1, port_2) + assert node.port in bound_ports + assert node.port not in stage0__bound_ports + + assert stage0__bound_ports <= bound_ports + assert len(stage0__bound_ports) + 1 == len(bound_ports) + + stage1__bound_ports = stage0__bound_ports.copy() + stage1__bound_ports.add(node.port) + + assert stage1__bound_ports == bound_ports # check that port has been freed successfully - self.assertEqual(len(bound_ports), 0) + assert bound_ports is not None + assert type(bound_ports) == set # noqa: E721 + assert bound_ports == stage0__bound_ports def test_exceptions(self): str(StartNodeException('msg', [('file', 'lines')])) @@ -939,22 +962,22 @@ def test_version_management(self): g = PgVer('15.3.1bihabeta1') k = PgVer('15.3.1') - self.assertTrue(a == b) - self.assertTrue(b > c) - self.assertTrue(a > c) - self.assertTrue(d > e) - self.assertTrue(e > f) - self.assertTrue(d > f) - self.assertTrue(h > f) - self.assertTrue(h == i) - self.assertTrue(g == k) - self.assertTrue(g > h) + assert (a == b) + assert (b > c) + assert (a > c) + assert (d > e) + assert (e > f) + assert (d > f) + assert (h > f) + assert (h == i) + assert (g == k) + assert (g > h) version = get_pg_version() with get_new_node() as node: - self.assertTrue(isinstance(version, six.string_types)) - self.assertTrue(isinstance(node.version, PgVer)) - self.assertEqual(node.version, PgVer(version)) + assert (isinstance(version, six.string_types)) + assert (isinstance(node.version, PgVer)) + assert (node.version == PgVer(version)) def test_child_pids(self): master_processes = [ @@ -977,53 +1000,128 @@ def test_child_pids(self): ProcessType.WalReceiver, ] + def LOCAL__test_auxiliary_pids( + node: testgres.PostgresNode, + expectedTypes: list[ProcessType] + ) -> list[ProcessType]: + # returns list of the absence processes + assert node is not None + assert type(node) == testgres.PostgresNode # noqa: E721 + assert expectedTypes is not None + assert type(expectedTypes) == list # noqa: E721 + + pids = node.auxiliary_pids + assert pids is not None # noqa: E721 + assert type(pids) == dict # noqa: E721 + + result = list[ProcessType]() + for ptype in expectedTypes: + if not (ptype in pids): + result.append(ptype) + return result + + def LOCAL__check_auxiliary_pids__multiple_attempts( + node: testgres.PostgresNode, + expectedTypes: list[ProcessType]): + assert node is not None + assert type(node) == testgres.PostgresNode # noqa: E721 + assert expectedTypes is not None + assert type(expectedTypes) == list # noqa: E721 + + nAttempt = 0 + + while nAttempt < 5: + nAttempt += 1 + + logging.info("Test pids of [{0}] node. Attempt #{1}.".format( + node.name, + nAttempt + )) + + if nAttempt > 1: + time.sleep(1) + + absenceList = LOCAL__test_auxiliary_pids(node, expectedTypes) + assert absenceList is not None + assert type(absenceList) == list # noqa: E721 + if len(absenceList) == 0: + logging.info("Bingo!") + return + + logging.info("These processes are not found: {0}.".format(absenceList)) + continue + + raise Exception("Node {0} does not have the following processes: {1}.".format( + node.name, + absenceList + )) + with get_new_node().init().start() as master: # master node doesn't have a source walsender! - with self.assertRaises(TestgresException): + with pytest.raises(expected_exception=TestgresException): master.source_walsender with master.connect() as con: - self.assertGreater(con.pid, 0) + assert (con.pid > 0) with master.replicate().start() as replica: # test __str__ method str(master.child_processes[0]) - master_pids = master.auxiliary_pids - for ptype in master_processes: - self.assertIn(ptype, master_pids) + LOCAL__check_auxiliary_pids__multiple_attempts( + master, + master_processes) + + LOCAL__check_auxiliary_pids__multiple_attempts( + replica, + repl_processes) - replica_pids = replica.auxiliary_pids - for ptype in repl_processes: - self.assertIn(ptype, replica_pids) + master_pids = master.auxiliary_pids # there should be exactly 1 source walsender for replica - self.assertEqual(len(master_pids[ProcessType.WalSender]), 1) + assert (len(master_pids[ProcessType.WalSender]) == 1) pid1 = master_pids[ProcessType.WalSender][0] pid2 = replica.source_walsender.pid - self.assertEqual(pid1, pid2) + assert (pid1 == pid2) replica.stop() # there should be no walsender after we've stopped replica - with self.assertRaises(TestgresException): + with pytest.raises(expected_exception=TestgresException): replica.source_walsender def test_child_process_dies(self): # test for FileNotFound exception during child_processes() function cmd = ["timeout", "60"] if os.name == 'nt' else ["sleep", "60"] - with subprocess.Popen(cmd, shell=True) as process: # shell=True might be needed on Windows - self.assertEqual(process.poll(), None) - # collect list of processes currently running - children = psutil.Process(os.getpid()).children() - # kill a process, so received children dictionary becomes invalid - process.kill() - process.wait() - # try to handle children list -- missing processes will have ptype "ProcessType.Unknown" - [ProcessProxy(p) for p in children] + nAttempt = 0 + + while True: + if nAttempt == 5: + raise Exception("Max attempt number is exceed.") + + nAttempt += 1 + + logging.info("Attempt #{0}".format(nAttempt)) + + with subprocess.Popen(cmd, shell=True) as process: # shell=True might be needed on Windows + r = process.poll() + + if r is not None: + logging.warning("process.pool() returns an unexpected result: {0}.".format(r)) + continue + + assert r is None + # collect list of processes currently running + children = psutil.Process(os.getpid()).children() + # kill a process, so received children dictionary becomes invalid + process.kill() + process.wait() + # try to handle children list -- missing processes will have ptype "ProcessType.Unknown" + [ProcessProxy(p) for p in children] + break def test_upgrade_node(self): old_bin_dir = os.path.dirname(get_bin_path("pg_config")) @@ -1036,7 +1134,7 @@ def test_upgrade_node(self): node_new.init(cached=False) res = node_new.upgrade_from(old_node=node_old) node_new.start() - self.assertTrue(b'Upgrade Complete' in res) + assert (b'Upgrade Complete' in res) def test_parse_pg_version(self): # Linux Mint @@ -1051,25 +1149,26 @@ def test_parse_pg_version(self): def test_the_same_port(self): with get_new_node() as node: node.init().start() - self.assertTrue(node._should_free_port) - self.assertEqual(type(node.port), int) + assert (node._should_free_port) + assert (type(node.port) == int) # noqa: E721 node_port_copy = node.port - self.assertEqual(rm_carriage_returns(node.safe_psql("SELECT 1;")), b'1\n') + assert (rm_carriage_returns(node.safe_psql("SELECT 1;")) == b'1\n') with get_new_node(port=node.port) as node2: - self.assertEqual(type(node2.port), int) - self.assertEqual(node2.port, node.port) - self.assertFalse(node2._should_free_port) - - with self.assertRaises(StartNodeException) as ctx: + assert (type(node2.port) == int) # noqa: E721 + assert (node2.port == node.port) + assert not (node2._should_free_port) + + with pytest.raises( + expected_exception=StartNodeException, + match=re.escape("Cannot start node") + ): node2.init().start() - self.assertIn("Cannot start node", str(ctx.exception)) - # node is still working - self.assertEqual(node.port, node_port_copy) - self.assertTrue(node._should_free_port) - self.assertEqual(rm_carriage_returns(node.safe_psql("SELECT 3;")), b'3\n') + assert (node.port == node_port_copy) + assert (node._should_free_port) + assert (rm_carriage_returns(node.safe_psql("SELECT 3;")) == b'3\n') class tagPortManagerProxy: sm_prev_testgres_reserve_port = None @@ -1174,31 +1273,31 @@ def test_port_rereserve_during_node_start(self): with get_new_node() as node1: node1.init().start() - self.assertTrue(node1._should_free_port) - self.assertEqual(type(node1.port), int) # noqa: E721 + assert (node1._should_free_port) + assert (type(node1.port) == int) # noqa: E721 node1_port_copy = node1.port - self.assertEqual(rm_carriage_returns(node1.safe_psql("SELECT 1;")), b'1\n') + assert (rm_carriage_returns(node1.safe_psql("SELECT 1;")) == b'1\n') with __class__.tagPortManagerProxy(node1.port, C_COUNT_OF_BAD_PORT_USAGE): assert __class__.tagPortManagerProxy.sm_DummyPortNumber == node1.port with get_new_node() as node2: - self.assertTrue(node2._should_free_port) - self.assertEqual(node2.port, node1.port) + assert (node2._should_free_port) + assert (node2.port == node1.port) node2.init().start() - self.assertNotEqual(node2.port, node1.port) - self.assertTrue(node2._should_free_port) - self.assertEqual(__class__.tagPortManagerProxy.sm_DummyPortCurrentUsage, 0) - self.assertEqual(__class__.tagPortManagerProxy.sm_DummyPortTotalUsage, C_COUNT_OF_BAD_PORT_USAGE) - self.assertTrue(node2.is_started) + assert (node2.port != node1.port) + assert (node2._should_free_port) + assert (__class__.tagPortManagerProxy.sm_DummyPortCurrentUsage == 0) + assert (__class__.tagPortManagerProxy.sm_DummyPortTotalUsage == C_COUNT_OF_BAD_PORT_USAGE) + assert (node2.is_started) - self.assertEqual(rm_carriage_returns(node2.safe_psql("SELECT 2;")), b'2\n') + assert (rm_carriage_returns(node2.safe_psql("SELECT 2;")) == b'2\n') # node1 is still working - self.assertEqual(node1.port, node1_port_copy) - self.assertTrue(node1._should_free_port) - self.assertEqual(rm_carriage_returns(node1.safe_psql("SELECT 3;")), b'3\n') + assert (node1.port == node1_port_copy) + assert (node1._should_free_port) + assert (rm_carriage_returns(node1.safe_psql("SELECT 3;")) == b'3\n') def test_port_conflict(self): assert testgres.PostgresNode._C_MAX_START_ATEMPTS > 1 @@ -1207,35 +1306,36 @@ def test_port_conflict(self): with get_new_node() as node1: node1.init().start() - self.assertTrue(node1._should_free_port) - self.assertEqual(type(node1.port), int) # noqa: E721 + assert (node1._should_free_port) + assert (type(node1.port) == int) # noqa: E721 node1_port_copy = node1.port - self.assertEqual(rm_carriage_returns(node1.safe_psql("SELECT 1;")), b'1\n') + assert (rm_carriage_returns(node1.safe_psql("SELECT 1;")) == b'1\n') with __class__.tagPortManagerProxy(node1.port, C_COUNT_OF_BAD_PORT_USAGE): assert __class__.tagPortManagerProxy.sm_DummyPortNumber == node1.port with get_new_node() as node2: - self.assertTrue(node2._should_free_port) - self.assertEqual(node2.port, node1.port) + assert (node2._should_free_port) + assert (node2.port == node1.port) - with self.assertRaises(StartNodeException) as ctx: + with pytest.raises( + expected_exception=StartNodeException, + match=re.escape("Cannot start node after multiple attempts") + ): node2.init().start() - self.assertIn("Cannot start node after multiple attempts", str(ctx.exception)) - - self.assertEqual(node2.port, node1.port) - self.assertTrue(node2._should_free_port) - self.assertEqual(__class__.tagPortManagerProxy.sm_DummyPortCurrentUsage, 1) - self.assertEqual(__class__.tagPortManagerProxy.sm_DummyPortTotalUsage, C_COUNT_OF_BAD_PORT_USAGE) - self.assertFalse(node2.is_started) + assert (node2.port == node1.port) + assert (node2._should_free_port) + assert (__class__.tagPortManagerProxy.sm_DummyPortCurrentUsage == 1) + assert (__class__.tagPortManagerProxy.sm_DummyPortTotalUsage == C_COUNT_OF_BAD_PORT_USAGE) + assert not (node2.is_started) # node2 must release our dummyPort (node1.port) - self.assertEqual(__class__.tagPortManagerProxy.sm_DummyPortCurrentUsage, 0) + assert (__class__.tagPortManagerProxy.sm_DummyPortCurrentUsage == 0) # node1 is still working - self.assertEqual(node1.port, node1_port_copy) - self.assertTrue(node1._should_free_port) - self.assertEqual(rm_carriage_returns(node1.safe_psql("SELECT 3;")), b'3\n') + assert (node1.port == node1_port_copy) + assert (node1._should_free_port) + assert (rm_carriage_returns(node1.safe_psql("SELECT 3;")) == b'3\n') def test_simple_with_bin_dir(self): with get_new_node() as node: @@ -1300,29 +1400,28 @@ def test_set_auto_conf(self): content = f.read() for x in testData: - self.assertIn( - x[0] + " = " + x[2], - content, - x[0] + " stored wrong" - ) - - -if __name__ == '__main__': - if os.environ.get('ALT_CONFIG'): - suite = unittest.TestSuite() - - # Small subset of tests for alternative configs (PG_BIN or PG_CONFIG) - suite.addTest(TestgresTests('test_pg_config')) - suite.addTest(TestgresTests('test_pg_ctl')) - suite.addTest(TestgresTests('test_psql')) - suite.addTest(TestgresTests('test_replicate')) - - print('Running tests for alternative config:') - for t in suite: - print(t) - print() - - runner = unittest.TextTestRunner() - runner.run(suite) - else: - unittest.main() + assert x[0] + " = " + x[2] in content + + @staticmethod + def helper__skip_test_if_util_not_exist(name: str): + assert type(name) == str # noqa: E721 + + if platform.system().lower() == "windows": + name2 = name + ".exe" + else: + name2 = name + + if not util_exists(name2): + pytest.skip('might be missing') + + @staticmethod + def helper__skip_test_if_pg_version_is_not_ge(version: str): + assert type(version) == str # noqa: E721 + if not pg_version_ge(version): + pytest.skip('requires {0}+'.format(version)) + + @staticmethod + def helper__skip_test_if_pg_version_is_ge(version: str): + assert type(version) == str # noqa: E721 + if pg_version_ge(version): + pytest.skip('requires <{0}'.format(version)) diff --git a/tests/test_simple_remote.py b/tests/test_simple_remote.py index 8b44623a..e7cc5e5c 100755 --- a/tests/test_simple_remote.py +++ b/tests/test_simple_remote.py @@ -1,22 +1,21 @@ -#!/usr/bin/env python # coding: utf-8 - import os import re import subprocess import tempfile -import testgres import time import six -import unittest +import pytest import psutil import logging.config from contextlib import contextmanager -from testgres.exceptions import \ +from .. import testgres + +from ..testgres.exceptions import \ InitNodeException, \ StartNodeException, \ ExecUtilException, \ @@ -26,38 +25,33 @@ TestgresException, \ InvalidOperationException -from testgres.config import \ +from ..testgres.config import \ TestgresConfig, \ configure_testgres, \ scoped_config, \ pop_config, testgres_config -from testgres import \ +from ..testgres import \ NodeStatus, \ ProcessType, \ IsolationLevel, \ get_remote_node, \ RemoteOperations -from testgres import \ +from ..testgres import \ get_bin_path, \ get_pg_config, \ get_pg_version -from testgres import \ +from ..testgres import \ First, \ Any # NOTE: those are ugly imports -from testgres import bound_ports -from testgres.utils import PgVer -from testgres.node import ProcessProxy, ConnectionParams - -conn_params = ConnectionParams(host=os.getenv('RDBMS_TESTPOOL1_HOST') or '127.0.0.1', - username=os.getenv('USER'), - ssh_key=os.getenv('RDBMS_TESTPOOL_SSHKEY')) -os_ops = RemoteOperations(conn_params) -testgres_config.set_os_ops(os_ops=os_ops) +from ..testgres import bound_ports +from ..testgres.utils import PgVer +from ..testgres.utils import file_tail +from ..testgres.node import ProcessProxy, ConnectionParams def pg_version_ge(version): @@ -68,16 +62,16 @@ def pg_version_ge(version): def util_exists(util): def good_properties(f): - return (os_ops.path_exists(f) and # noqa: W504 - os_ops.isfile(f) and # noqa: W504 - os_ops.is_executable(f)) # yapf: disable + return (testgres_config.os_ops.path_exists(f) and # noqa: W504 + testgres_config.os_ops.isfile(f) and # noqa: W504 + testgres_config.os_ops.is_executable(f)) # yapf: disable # try to resolve it if good_properties(get_bin_path(util)): return True # check if util is in PATH - for path in os_ops.environ("PATH").split(os_ops.pathsep): + for path in testgres_config.os_ops.environ("PATH").split(testgres_config.os_ops.pathsep): if good_properties(os.path.join(path, util)): return True @@ -87,37 +81,56 @@ def removing(f): try: yield f finally: - if os_ops.isfile(f): - os_ops.remove_file(f) + if testgres_config.os_ops.isfile(f): + testgres_config.os_ops.remove_file(f) + + elif testgres_config.os_ops.isdir(f): + testgres_config.os_ops.rmdirs(f, ignore_errors=True) - elif os_ops.isdir(f): - os_ops.rmdirs(f, ignore_errors=True) +class TestgresRemoteTests: + sm_conn_params = ConnectionParams( + host=os.getenv('RDBMS_TESTPOOL1_HOST') or '127.0.0.1', + username=os.getenv('USER'), + ssh_key=os.getenv('RDBMS_TESTPOOL_SSHKEY')) + + sm_os_ops = RemoteOperations(sm_conn_params) + + @pytest.fixture(autouse=True, scope="class") + def implicit_fixture(self): + prev_ops = testgres_config.os_ops + assert prev_ops is not None + assert __class__.sm_os_ops is not None + testgres_config.set_os_ops(os_ops=__class__.sm_os_ops) + assert testgres_config.os_ops is __class__.sm_os_ops + yield + assert testgres_config.os_ops is __class__.sm_os_ops + testgres_config.set_os_ops(os_ops=prev_ops) + assert testgres_config.os_ops is prev_ops -class TestgresRemoteTests(unittest.TestCase): def test_node_repr(self): - with get_remote_node(conn_params=conn_params) as node: + with __class__.helper__get_node() as node: pattern = r"PostgresNode\(name='.+', port=.+, base_dir='.+'\)" - self.assertIsNotNone(re.match(pattern, str(node))) + assert re.match(pattern, str(node)) is not None def test_custom_init(self): - with get_remote_node(conn_params=conn_params) as node: + with __class__.helper__get_node() as node: # enable page checksums node.init(initdb_params=['-k']).start() - with get_remote_node(conn_params=conn_params) as node: + with __class__.helper__get_node() as node: node.init( allow_streaming=True, initdb_params=['--auth-local=reject', '--auth-host=reject']) hba_file = os.path.join(node.data_dir, 'pg_hba.conf') - lines = os_ops.readlines(hba_file) + lines = node.os_ops.readlines(hba_file) # check number of lines - self.assertGreaterEqual(len(lines), 6) + assert (len(lines) >= 6) # there should be no trust entries at all - self.assertFalse(any('trust' in s for s in lines)) + assert not (any('trust' in s for s in lines)) def test_init__LANG_С(self): # PBCKP-1744 @@ -126,7 +139,7 @@ def test_init__LANG_С(self): try: os.environ["LANG"] = "C" - with get_remote_node(conn_params=conn_params) as node: + with __class__.helper__get_node() as node: node.init().start() finally: __class__.helper__restore_envvar("LANG", prev_LANG) @@ -167,9 +180,9 @@ def test_init__unk_LANG_and_LC_CTYPE(self): while True: try: - with get_remote_node(conn_params=conn_params): + with __class__.helper__get_node(): pass - except testgres.exceptions.ExecUtilException as e: + except ExecUtilException as e: # # Example of an error message: # @@ -193,88 +206,89 @@ def test_init__unk_LANG_and_LC_CTYPE(self): __class__.helper__restore_envvar("LC_COLLATE", prev_LC_COLLATE) def test_double_init(self): - with get_remote_node(conn_params=conn_params).init() as node: + with __class__.helper__get_node().init() as node: # can't initialize node more than once - with self.assertRaises(InitNodeException): + with pytest.raises(expected_exception=InitNodeException): node.init() def test_init_after_cleanup(self): - with get_remote_node(conn_params=conn_params) as node: + with __class__.helper__get_node() as node: node.init().start().execute('select 1') node.cleanup() node.init().start().execute('select 1') - @unittest.skipUnless(util_exists('pg_resetwal'), 'might be missing') - @unittest.skipUnless(pg_version_ge('9.6'), 'requires 9.6+') def test_init_unique_system_id(self): # this function exists in PostgreSQL 9.6+ + __class__.helper__skip_test_if_util_not_exist("pg_resetwal") + __class__.helper__skip_test_if_pg_version_is_not_ge('9.6') + query = 'select system_identifier from pg_control_system()' with scoped_config(cache_initdb=False): - with get_remote_node(conn_params=conn_params).init().start() as node0: + with __class__.helper__get_node().init().start() as node0: id0 = node0.execute(query)[0] with scoped_config(cache_initdb=True, cached_initdb_unique=True) as config: - self.assertTrue(config.cache_initdb) - self.assertTrue(config.cached_initdb_unique) + assert (config.cache_initdb) + assert (config.cached_initdb_unique) # spawn two nodes; ids must be different - with get_remote_node(conn_params=conn_params).init().start() as node1, \ - get_remote_node(conn_params=conn_params).init().start() as node2: + with __class__.helper__get_node().init().start() as node1, \ + __class__.helper__get_node().init().start() as node2: id1 = node1.execute(query)[0] id2 = node2.execute(query)[0] # ids must increase - self.assertGreater(id1, id0) - self.assertGreater(id2, id1) + assert (id1 > id0) + assert (id2 > id1) def test_node_exit(self): - with self.assertRaises(QueryException): - with get_remote_node(conn_params=conn_params).init() as node: + with pytest.raises(expected_exception=QueryException): + with __class__.helper__get_node().init() as node: base_dir = node.base_dir node.safe_psql('select 1') # we should save the DB for "debugging" - self.assertTrue(os_ops.path_exists(base_dir)) - os_ops.rmdirs(base_dir, ignore_errors=True) + assert (__class__.sm_os_ops.path_exists(base_dir)) + __class__.sm_os_ops.rmdirs(base_dir, ignore_errors=True) - with get_remote_node(conn_params=conn_params).init() as node: + with __class__.helper__get_node().init() as node: base_dir = node.base_dir # should have been removed by default - self.assertFalse(os_ops.path_exists(base_dir)) + assert not (__class__.sm_os_ops.path_exists(base_dir)) def test_double_start(self): - with get_remote_node(conn_params=conn_params).init().start() as node: + with __class__.helper__get_node().init().start() as node: # can't start node more than once node.start() - self.assertTrue(node.is_started) + assert (node.is_started) def test_uninitialized_start(self): - with get_remote_node(conn_params=conn_params) as node: + with __class__.helper__get_node() as node: # node is not initialized yet - with self.assertRaises(StartNodeException): + with pytest.raises(expected_exception=StartNodeException): node.start() def test_restart(self): - with get_remote_node(conn_params=conn_params) as node: + with __class__.helper__get_node() as node: node.init().start() # restart, ok res = node.execute('select 1') - self.assertEqual(res, [(1,)]) + assert (res == [(1,)]) node.restart() res = node.execute('select 2') - self.assertEqual(res, [(2,)]) + assert (res == [(2,)]) # restart, fail - with self.assertRaises(StartNodeException): + with pytest.raises(expected_exception=StartNodeException): node.append_conf('pg_hba.conf', 'DUMMY') node.restart() def test_reload(self): - with get_remote_node(conn_params=conn_params) as node: + with __class__.helper__get_node() as node: node.init().start() # change client_min_messages and save old value @@ -286,108 +300,109 @@ def test_reload(self): # check new value cmm_new = node.execute('show client_min_messages') - self.assertEqual('debug1', cmm_new[0][0].lower()) - self.assertNotEqual(cmm_old, cmm_new) + assert ('debug1' == cmm_new[0][0].lower()) + assert (cmm_old != cmm_new) def test_pg_ctl(self): - with get_remote_node(conn_params=conn_params) as node: + with __class__.helper__get_node() as node: node.init().start() status = node.pg_ctl(['status']) - self.assertTrue('PID' in status) + assert ('PID' in status) def test_status(self): - self.assertTrue(NodeStatus.Running) - self.assertFalse(NodeStatus.Stopped) - self.assertFalse(NodeStatus.Uninitialized) + assert (NodeStatus.Running) + assert not (NodeStatus.Stopped) + assert not (NodeStatus.Uninitialized) # check statuses after each operation - with get_remote_node(conn_params=conn_params) as node: - self.assertEqual(node.pid, 0) - self.assertEqual(node.status(), NodeStatus.Uninitialized) + with __class__.helper__get_node() as node: + assert (node.pid == 0) + assert (node.status() == NodeStatus.Uninitialized) node.init() - self.assertEqual(node.pid, 0) - self.assertEqual(node.status(), NodeStatus.Stopped) + assert (node.pid == 0) + assert (node.status() == NodeStatus.Stopped) node.start() - self.assertNotEqual(node.pid, 0) - self.assertEqual(node.status(), NodeStatus.Running) + assert (node.pid != 0) + assert (node.status() == NodeStatus.Running) node.stop() - self.assertEqual(node.pid, 0) - self.assertEqual(node.status(), NodeStatus.Stopped) + assert (node.pid == 0) + assert (node.status() == NodeStatus.Stopped) node.cleanup() - self.assertEqual(node.pid, 0) - self.assertEqual(node.status(), NodeStatus.Uninitialized) + assert (node.pid == 0) + assert (node.status() == NodeStatus.Uninitialized) def test_psql(self): - with get_remote_node(conn_params=conn_params).init().start() as node: + with __class__.helper__get_node().init().start() as node: # check returned values (1 arg) res = node.psql('select 1') - self.assertEqual(res, (0, b'1\n', b'')) + assert (res == (0, b'1\n', b'')) # check returned values (2 args) res = node.psql('postgres', 'select 2') - self.assertEqual(res, (0, b'2\n', b'')) + assert (res == (0, b'2\n', b'')) # check returned values (named) res = node.psql(query='select 3', dbname='postgres') - self.assertEqual(res, (0, b'3\n', b'')) + assert (res == (0, b'3\n', b'')) # check returned values (1 arg) res = node.safe_psql('select 4') - self.assertEqual(res, b'4\n') + assert (res == b'4\n') # check returned values (2 args) res = node.safe_psql('postgres', 'select 5') - self.assertEqual(res, b'5\n') + assert (res == b'5\n') # check returned values (named) res = node.safe_psql(query='select 6', dbname='postgres') - self.assertEqual(res, b'6\n') + assert (res == b'6\n') # check feeding input node.safe_psql('create table horns (w int)') node.safe_psql('copy horns from stdin (format csv)', input=b"1\n2\n3\n\\.\n") _sum = node.safe_psql('select sum(w) from horns') - self.assertEqual(_sum, b'6\n') + assert (_sum == b'6\n') # check psql's default args, fails - with self.assertRaises(QueryException): + with pytest.raises(expected_exception=QueryException): node.psql() node.stop() # check psql on stopped node, fails - with self.assertRaises(QueryException): + with pytest.raises(expected_exception=QueryException): node.safe_psql('select 1') def test_safe_psql__expect_error(self): - with get_remote_node(conn_params=conn_params).init().start() as node: + with __class__.helper__get_node().init().start() as node: err = node.safe_psql('select_or_not_select 1', expect_error=True) - self.assertTrue(type(err) == str) # noqa: E721 - self.assertIn('select_or_not_select', err) - self.assertIn('ERROR: syntax error at or near "select_or_not_select"', err) + assert (type(err) == str) # noqa: E721 + assert ('select_or_not_select' in err) + assert ('ERROR: syntax error at or near "select_or_not_select"' in err) # --------- - with self.assertRaises(InvalidOperationException) as ctx: + with pytest.raises( + expected_exception=InvalidOperationException, + match="^" + re.escape("Exception was expected, but query finished successfully: `select 1;`.") + "$" + ): node.safe_psql("select 1;", expect_error=True) - self.assertEqual(str(ctx.exception), "Exception was expected, but query finished successfully: `select 1;`.") - # --------- res = node.safe_psql("select 1;", expect_error=False) - self.assertEqual(res, b'1\n') + assert (res == b'1\n') def test_transactions(self): - with get_remote_node(conn_params=conn_params).init().start() as node: + with __class__.helper__get_node().init().start() as node: with node.connect() as con: con.begin() con.execute('create table test(val int)') @@ -397,12 +412,12 @@ def test_transactions(self): con.begin() con.execute('insert into test values (2)') res = con.execute('select * from test order by val asc') - self.assertListEqual(res, [(1,), (2,)]) + assert (res == [(1,), (2,)]) con.rollback() con.begin() res = con.execute('select * from test') - self.assertListEqual(res, [(1,)]) + assert (res == [(1,)]) con.rollback() con.begin() @@ -410,25 +425,25 @@ def test_transactions(self): con.commit() def test_control_data(self): - with get_remote_node(conn_params=conn_params) as node: + with __class__.helper__get_node() as node: # node is not initialized yet - with self.assertRaises(ExecUtilException): + with pytest.raises(expected_exception=ExecUtilException): node.get_control_data() node.init() data = node.get_control_data() # check returned dict - self.assertIsNotNone(data) - self.assertTrue(any('pg_control' in s for s in data.keys())) + assert data is not None + assert (any('pg_control' in s for s in data.keys())) def test_backup_simple(self): - with get_remote_node(conn_params=conn_params) as master: + with __class__.helper__get_node() as master: # enable streaming for backups master.init(allow_streaming=True) # node must be running - with self.assertRaises(BackupException): + with pytest.raises(expected_exception=BackupException): master.backup() # it's time to start node @@ -440,23 +455,23 @@ def test_backup_simple(self): with master.backup(xlog_method='stream') as backup: with backup.spawn_primary().start() as slave: res = slave.execute('select * from test order by i asc') - self.assertListEqual(res, [(1,), (2,), (3,), (4,)]) + assert (res == [(1,), (2,), (3,), (4,)]) def test_backup_multiple(self): - with get_remote_node(conn_params=conn_params) as node: + with __class__.helper__get_node() as node: node.init(allow_streaming=True).start() with node.backup(xlog_method='fetch') as backup1, \ node.backup(xlog_method='fetch') as backup2: - self.assertNotEqual(backup1.base_dir, backup2.base_dir) + assert (backup1.base_dir != backup2.base_dir) with node.backup(xlog_method='fetch') as backup: with backup.spawn_primary('node1', destroy=False) as node1, \ backup.spawn_primary('node2', destroy=False) as node2: - self.assertNotEqual(node1.base_dir, node2.base_dir) + assert (node1.base_dir != node2.base_dir) def test_backup_exhaust(self): - with get_remote_node(conn_params=conn_params) as node: + with __class__.helper__get_node() as node: node.init(allow_streaming=True).start() with node.backup(xlog_method='fetch') as backup: @@ -465,19 +480,21 @@ def test_backup_exhaust(self): pass # now let's try to create one more node - with self.assertRaises(BackupException): + with pytest.raises(expected_exception=BackupException): backup.spawn_primary() def test_backup_wrong_xlog_method(self): - with get_remote_node(conn_params=conn_params) as node: + with __class__.helper__get_node() as node: node.init(allow_streaming=True).start() - with self.assertRaises(BackupException, - msg='Invalid xlog_method "wrong"'): + with pytest.raises( + expected_exception=BackupException, + match="^" + re.escape('Invalid xlog_method "wrong"') + "$" + ): node.backup(xlog_method='wrong') def test_pg_ctl_wait_option(self): - with get_remote_node(conn_params=conn_params) as node: + with __class__.helper__get_node() as node: node.init().start(wait=False) while True: try: @@ -489,23 +506,24 @@ def test_pg_ctl_wait_option(self): pass def test_replicate(self): - with get_remote_node(conn_params=conn_params) as node: + with __class__.helper__get_node() as node: node.init(allow_streaming=True).start() with node.replicate().start() as replica: res = replica.execute('select 1') - self.assertListEqual(res, [(1,)]) + assert (res == [(1,)]) node.execute('create table test (val int)', commit=True) replica.catchup() res = node.execute('select * from test') - self.assertListEqual(res, []) + assert (res == []) - @unittest.skipUnless(pg_version_ge('9.6'), 'requires 9.6+') def test_synchronous_replication(self): - with get_remote_node(conn_params=conn_params) as master: + __class__.helper__skip_test_if_pg_version_is_not_ge("9.6") + + with __class__.helper__get_node() as master: old_version = not pg_version_ge('9.6') master.init(allow_streaming=True).start() @@ -519,12 +537,12 @@ def test_synchronous_replication(self): standby2.start() # check formatting - self.assertEqual( - '1 ("{}", "{}")'.format(standby1.name, standby2.name), - str(First(1, (standby1, standby2)))) # yapf: disable - self.assertEqual( - 'ANY 1 ("{}", "{}")'.format(standby1.name, standby2.name), - str(Any(1, (standby1, standby2)))) # yapf: disable + assert ( + '1 ("{}", "{}")'.format(standby1.name, standby2.name) == str(First(1, (standby1, standby2))) + ) # yapf: disable + assert ( + 'ANY 1 ("{}", "{}")'.format(standby1.name, standby2.name) == str(Any(1, (standby1, standby2))) + ) # yapf: disable # set synchronous_standby_names master.set_synchronous_standbys(First(2, [standby1, standby2])) @@ -542,11 +560,12 @@ def test_synchronous_replication(self): master.safe_psql( 'insert into abc select generate_series(1, 1000000)') res = standby1.safe_psql('select count(*) from abc') - self.assertEqual(res, b'1000000\n') + assert (res == b'1000000\n') - @unittest.skipUnless(pg_version_ge('10'), 'requires 10+') def test_logical_replication(self): - with get_remote_node(conn_params=conn_params) as node1, get_remote_node(conn_params=conn_params) as node2: + __class__.helper__skip_test_if_pg_version_is_not_ge("10") + + with __class__.helper__get_node() as node1, __class__.helper__get_node() as node2: node1.init(allow_logical=True) node1.start() node2.init().start() @@ -564,7 +583,7 @@ def test_logical_replication(self): # wait until changes apply on subscriber and check them sub.catchup() res = node2.execute('select * from test') - self.assertListEqual(res, [(1, 1), (2, 2)]) + assert (res == [(1, 1), (2, 2)]) # disable and put some new data sub.disable() @@ -574,7 +593,7 @@ def test_logical_replication(self): sub.enable() sub.catchup() res = node2.execute('select * from test') - self.assertListEqual(res, [(1, 1), (2, 2), (3, 3)]) + assert (res == [(1, 1), (2, 2), (3, 3)]) # Add new tables. Since we added "all tables" to publication # (default behaviour of publish() method) we don't need @@ -588,7 +607,7 @@ def test_logical_replication(self): node1.safe_psql('insert into test2 values (\'a\'), (\'b\')') sub.catchup() res = node2.execute('select * from test2') - self.assertListEqual(res, [('a',), ('b',)]) + assert (res == [('a',), ('b',)]) # drop subscription sub.drop() @@ -602,21 +621,22 @@ def test_logical_replication(self): node1.safe_psql('insert into test values (4, 4)') sub.catchup() res = node2.execute('select * from test') - self.assertListEqual(res, [(1, 1), (2, 2), (3, 3), (4, 4)]) + assert (res == [(1, 1), (2, 2), (3, 3), (4, 4)]) # explicitly add table - with self.assertRaises(ValueError): + with pytest.raises(expected_exception=ValueError): pub.add_tables([]) # fail pub.add_tables(['test2']) node1.safe_psql('insert into test2 values (\'c\')') sub.catchup() res = node2.execute('select * from test2') - self.assertListEqual(res, [('a',), ('b',)]) + assert (res == [('a',), ('b',)]) - @unittest.skipUnless(pg_version_ge('10'), 'requires 10+') def test_logical_catchup(self): """ Runs catchup for 100 times to be sure that it is consistent """ - with get_remote_node(conn_params=conn_params) as node1, get_remote_node(conn_params=conn_params) as node2: + __class__.helper__skip_test_if_pg_version_is_not_ge("10") + + with __class__.helper__get_node() as node1, __class__.helper__get_node() as node2: node1.init(allow_logical=True) node1.start() node2.init().start() @@ -633,39 +653,37 @@ def test_logical_catchup(self): node1.execute('insert into test values ({0}, {0})'.format(i)) sub.catchup() res = node2.execute('select * from test') - self.assertListEqual(res, [( - i, - i, - )]) + assert (res == [(i, i, )]) node1.execute('delete from test') - @unittest.skipIf(pg_version_ge('10'), 'requires <10') def test_logical_replication_fail(self): - with get_remote_node(conn_params=conn_params) as node: - with self.assertRaises(InitNodeException): + __class__.helper__skip_test_if_pg_version_is_ge("10") + + with __class__.helper__get_node() as node: + with pytest.raises(expected_exception=InitNodeException): node.init(allow_logical=True) def test_replication_slots(self): - with get_remote_node(conn_params=conn_params) as node: + with __class__.helper__get_node() as node: node.init(allow_streaming=True).start() with node.replicate(slot='slot1').start() as replica: replica.execute('select 1') # cannot create new slot with the same name - with self.assertRaises(TestgresException): + with pytest.raises(expected_exception=TestgresException): node.replicate(slot='slot1') def test_incorrect_catchup(self): - with get_remote_node(conn_params=conn_params) as node: + with __class__.helper__get_node() as node: node.init(allow_streaming=True).start() # node has no master, can't catch up - with self.assertRaises(TestgresException): + with pytest.raises(expected_exception=TestgresException): node.catchup() def test_promotion(self): - with get_remote_node(conn_params=conn_params) as master: + with __class__.helper__get_node() as master: master.init().start() master.safe_psql('create table abc(id serial)') @@ -676,35 +694,35 @@ def test_promotion(self): # make standby becomes writable master replica.safe_psql('insert into abc values (1)') res = replica.safe_psql('select * from abc') - self.assertEqual(res, b'1\n') + assert (res == b'1\n') def test_dump(self): query_create = 'create table test as select generate_series(1, 2) as val' query_select = 'select * from test order by val asc' - with get_remote_node(conn_params=conn_params).init().start() as node1: + with __class__.helper__get_node().init().start() as node1: node1.execute(query_create) for format in ['plain', 'custom', 'directory', 'tar']: with removing(node1.dump(format=format)) as dump: - with get_remote_node(conn_params=conn_params).init().start() as node3: + with __class__.helper__get_node().init().start() as node3: if format == 'directory': - self.assertTrue(os_ops.isdir(dump)) + assert (node1.os_ops.isdir(dump)) else: - self.assertTrue(os_ops.isfile(dump)) + assert (node1.os_ops.isfile(dump)) # restore dump node3.restore(filename=dump) res = node3.execute(query_select) - self.assertListEqual(res, [(1,), (2,)]) + assert (res == [(1,), (2,)]) def test_users(self): - with get_remote_node(conn_params=conn_params).init().start() as node: + with __class__.helper__get_node().init().start() as node: node.psql('create role test_user login') value = node.safe_psql('select 1', username='test_user') - self.assertEqual(b'1\n', value) + assert (b'1\n' == value) def test_poll_query_until(self): - with get_remote_node(conn_params=conn_params) as node: + with __class__.helper__get_node() as node: node.init().start() get_time = 'select extract(epoch from now())' @@ -714,15 +732,15 @@ def test_poll_query_until(self): node.poll_query_until(query=check_time.format(start_time)) end_time = node.execute(get_time)[0][0] - self.assertTrue(end_time - start_time >= 5) + assert (end_time - start_time >= 5) # check 0 columns - with self.assertRaises(QueryException): + with pytest.raises(expected_exception=QueryException): node.poll_query_until( query='select from pg_catalog.pg_class limit 1') # check None, fail - with self.assertRaises(QueryException): + with pytest.raises(expected_exception=QueryException): node.poll_query_until(query='create table abc (val int)') # check None, ok @@ -735,7 +753,7 @@ def test_poll_query_until(self): expected=None) # check arbitrary expected value, fail - with self.assertRaises(TimeoutException): + with pytest.raises(expected_exception=TimeoutException): node.poll_query_until(query='select 3', expected=1, max_attempts=3, @@ -745,17 +763,17 @@ def test_poll_query_until(self): node.poll_query_until(query='select 2', expected=2) # check timeout - with self.assertRaises(TimeoutException): + with pytest.raises(expected_exception=TimeoutException): node.poll_query_until(query='select 1 > 2', max_attempts=3, sleep_time=0.01) # check ProgrammingError, fail - with self.assertRaises(testgres.ProgrammingError): + with pytest.raises(expected_exception=testgres.ProgrammingError): node.poll_query_until(query='dummy1') # check ProgrammingError, ok - with self.assertRaises(TimeoutException): + with pytest.raises(expected_exception=TimeoutException): node.poll_query_until(query='dummy2', max_attempts=3, sleep_time=0.01, @@ -808,17 +826,18 @@ def test_logging(self): # check that master's port is found with open(logfile.name, 'r') as log: lines = log.readlines() - self.assertTrue(any(node_name in s for s in lines)) + assert (any(node_name in s for s in lines)) # test logger after stop/start/restart master.stop() master.start() master.restart() - self.assertTrue(master._logger.is_alive()) + assert (master._logger.is_alive()) - @unittest.skipUnless(util_exists('pgbench'), 'might be missing') def test_pgbench(self): - with get_remote_node(conn_params=conn_params).init().start() as node: + __class__.helper__skip_test_if_util_not_exist("pgbench") + + with __class__.helper__get_node().init().start() as node: # initialize pgbench DB and run benchmarks node.pgbench_init(scale=2, foreign_keys=True, options=['-q']).pgbench_run(time=2) @@ -828,13 +847,13 @@ def test_pgbench(self): stderr=subprocess.STDOUT, options=['-T3']) out = proc.communicate()[0] - self.assertTrue(b'tps = ' in out) + assert (b'tps = ' in out) def test_pg_config(self): # check same instances a = get_pg_config() b = get_pg_config() - self.assertEqual(id(a), id(b)) + assert (id(a) == id(b)) # save right before config change c1 = get_pg_config() @@ -842,26 +861,26 @@ def test_pg_config(self): # modify setting for this scope with scoped_config(cache_pg_config=False) as config: # sanity check for value - self.assertFalse(config.cache_pg_config) + assert not (config.cache_pg_config) # save right after config change c2 = get_pg_config() # check different instances after config change - self.assertNotEqual(id(c1), id(c2)) + assert (id(c1) != id(c2)) # check different instances a = get_pg_config() b = get_pg_config() - self.assertNotEqual(id(a), id(b)) + assert (id(a) != id(b)) def test_config_stack(self): # no such option - with self.assertRaises(TypeError): + with pytest.raises(expected_exception=TypeError): configure_testgres(dummy=True) # we have only 1 config in stack - with self.assertRaises(IndexError): + with pytest.raises(expected_exception=IndexError): pop_config() d0 = TestgresConfig.cached_initdb_dir @@ -869,54 +888,52 @@ def test_config_stack(self): d2 = 'dummy_def' with scoped_config(cached_initdb_dir=d1) as c1: - self.assertEqual(c1.cached_initdb_dir, d1) + assert (c1.cached_initdb_dir == d1) with scoped_config(cached_initdb_dir=d2) as c2: stack_size = len(testgres.config.config_stack) # try to break a stack - with self.assertRaises(TypeError): + with pytest.raises(expected_exception=TypeError): with scoped_config(dummy=True): pass - self.assertEqual(c2.cached_initdb_dir, d2) - self.assertEqual(len(testgres.config.config_stack), stack_size) + assert (c2.cached_initdb_dir == d2) + assert (len(testgres.config.config_stack) == stack_size) - self.assertEqual(c1.cached_initdb_dir, d1) + assert (c1.cached_initdb_dir == d1) - self.assertEqual(TestgresConfig.cached_initdb_dir, d0) + assert (TestgresConfig.cached_initdb_dir == d0) def test_unix_sockets(self): - with get_remote_node(conn_params=conn_params) as node: + with __class__.helper__get_node() as node: node.init(unix_sockets=False, allow_streaming=True) node.start() res_exec = node.execute('select 1') res_psql = node.safe_psql('select 1') - self.assertEqual(res_exec, [(1,)]) - self.assertEqual(res_psql, b'1\n') + assert (res_exec == [(1,)]) + assert (res_psql == b'1\n') with node.replicate().start() as r: res_exec = r.execute('select 1') res_psql = r.safe_psql('select 1') - self.assertEqual(res_exec, [(1,)]) - self.assertEqual(res_psql, b'1\n') + assert (res_exec == [(1,)]) + assert (res_psql == b'1\n') def test_auto_name(self): - with get_remote_node(conn_params=conn_params).init(allow_streaming=True).start() as m: + with __class__.helper__get_node().init(allow_streaming=True).start() as m: with m.replicate().start() as r: # check that nodes are running - self.assertTrue(m.status()) - self.assertTrue(r.status()) + assert (m.status()) + assert (r.status()) # check their names - self.assertNotEqual(m.name, r.name) - self.assertTrue('testgres' in m.name) - self.assertTrue('testgres' in r.name) + assert (m.name != r.name) + assert ('testgres' in m.name) + assert ('testgres' in r.name) def test_file_tail(self): - from testgres.utils import file_tail - s1 = "the quick brown fox jumped over that lazy dog\n" s2 = "abc\n" s3 = "def\n" @@ -931,16 +948,16 @@ def test_file_tail(self): f.seek(0) lines = file_tail(f, 3) - self.assertEqual(lines[0], s1) - self.assertEqual(lines[1], s2) - self.assertEqual(lines[2], s3) + assert (lines[0] == s1) + assert (lines[1] == s2) + assert (lines[2] == s3) f.seek(0) lines = file_tail(f, 1) - self.assertEqual(lines[0], s3) + assert (lines[0] == s3) def test_isolation_levels(self): - with get_remote_node(conn_params=conn_params).init().start() as node: + with __class__.helper__get_node().init().start() as node: with node.connect() as con: # string levels con.begin('Read Uncommitted').commit() @@ -955,24 +972,24 @@ def test_isolation_levels(self): con.begin(IsolationLevel.Serializable).commit() # check wrong level - with self.assertRaises(QueryException): + with pytest.raises(expected_exception=QueryException): con.begin('Garbage').commit() def test_ports_management(self): # check that no ports have been bound yet - self.assertEqual(len(bound_ports), 0) + assert (len(bound_ports) == 0) - with get_remote_node(conn_params=conn_params) as node: + with __class__.helper__get_node() as node: # check that we've just bound a port - self.assertEqual(len(bound_ports), 1) + assert (len(bound_ports) == 1) # check that bound_ports contains our port port_1 = list(bound_ports)[0] port_2 = node.port - self.assertEqual(port_1, port_2) + assert (port_1 == port_2) # check that port has been freed successfully - self.assertEqual(len(bound_ports), 0) + assert (len(bound_ports) == 0) def test_exceptions(self): str(StartNodeException('msg', [('file', 'lines')])) @@ -987,18 +1004,18 @@ def test_version_management(self): e = PgVer('15rc1') f = PgVer('15beta4') - self.assertTrue(a == b) - self.assertTrue(b > c) - self.assertTrue(a > c) - self.assertTrue(d > e) - self.assertTrue(e > f) - self.assertTrue(d > f) + assert (a == b) + assert (b > c) + assert (a > c) + assert (d > e) + assert (e > f) + assert (d > f) version = get_pg_version() - with get_remote_node(conn_params=conn_params) as node: - self.assertTrue(isinstance(version, six.string_types)) - self.assertTrue(isinstance(node.version, PgVer)) - self.assertEqual(node.version, PgVer(version)) + with __class__.helper__get_node() as node: + assert (isinstance(version, six.string_types)) + assert (isinstance(node.version, PgVer)) + assert (node.version == PgVer(version)) def test_child_pids(self): master_processes = [ @@ -1021,51 +1038,132 @@ def test_child_pids(self): ProcessType.WalReceiver, ] - with get_remote_node(conn_params=conn_params).init().start() as master: + def LOCAL__test_auxiliary_pids( + node: testgres.PostgresNode, + expectedTypes: list[ProcessType] + ) -> list[ProcessType]: + # returns list of the absence processes + assert node is not None + assert type(node) == testgres.PostgresNode # noqa: E721 + assert expectedTypes is not None + assert type(expectedTypes) == list # noqa: E721 + + pids = node.auxiliary_pids + assert pids is not None # noqa: E721 + assert type(pids) == dict # noqa: E721 + + result = list[ProcessType]() + for ptype in expectedTypes: + if not (ptype in pids): + result.append(ptype) + return result + + def LOCAL__check_auxiliary_pids__multiple_attempts( + node: testgres.PostgresNode, + expectedTypes: list[ProcessType]): + assert node is not None + assert type(node) == testgres.PostgresNode # noqa: E721 + assert expectedTypes is not None + assert type(expectedTypes) == list # noqa: E721 + + nAttempt = 0 + + while nAttempt < 5: + nAttempt += 1 + + logging.info("Test pids of [{0}] node. Attempt #{1}.".format( + node.name, + nAttempt + )) + + if nAttempt > 1: + time.sleep(1) + + absenceList = LOCAL__test_auxiliary_pids(node, expectedTypes) + assert absenceList is not None + assert type(absenceList) == list # noqa: E721 + if len(absenceList) == 0: + logging.info("Bingo!") + return + + logging.info("These processes are not found: {0}.".format(absenceList)) + continue + + raise Exception("Node {0} does not have the following processes: {1}.".format( + node.name, + absenceList + )) + + with __class__.helper__get_node().init().start() as master: # master node doesn't have a source walsender! - with self.assertRaises(TestgresException): + with pytest.raises(expected_exception=TestgresException): master.source_walsender with master.connect() as con: - self.assertGreater(con.pid, 0) + assert (con.pid > 0) with master.replicate().start() as replica: # test __str__ method str(master.child_processes[0]) - master_pids = master.auxiliary_pids - for ptype in master_processes: - self.assertIn(ptype, master_pids) + LOCAL__check_auxiliary_pids__multiple_attempts( + master, + master_processes) - replica_pids = replica.auxiliary_pids - for ptype in repl_processes: - self.assertIn(ptype, replica_pids) + LOCAL__check_auxiliary_pids__multiple_attempts( + replica, + repl_processes) + + master_pids = master.auxiliary_pids # there should be exactly 1 source walsender for replica - self.assertEqual(len(master_pids[ProcessType.WalSender]), 1) + assert (len(master_pids[ProcessType.WalSender]) == 1) pid1 = master_pids[ProcessType.WalSender][0] pid2 = replica.source_walsender.pid - self.assertEqual(pid1, pid2) + assert (pid1 == pid2) replica.stop() # there should be no walsender after we've stopped replica - with self.assertRaises(TestgresException): + with pytest.raises(expected_exception=TestgresException): replica.source_walsender + # TODO: Why does not this test work with remote host? def test_child_process_dies(self): - # test for FileNotFound exception during child_processes() function - with subprocess.Popen(["sleep", "60"]) as process: - self.assertEqual(process.poll(), None) - # collect list of processes currently running - children = psutil.Process(os.getpid()).children() - # kill a process, so received children dictionary becomes invalid - process.kill() - process.wait() - # try to handle children list -- missing processes will have ptype "ProcessType.Unknown" - [ProcessProxy(p) for p in children] + nAttempt = 0 + + while True: + if nAttempt == 5: + raise Exception("Max attempt number is exceed.") + + nAttempt += 1 + + logging.info("Attempt #{0}".format(nAttempt)) + + # test for FileNotFound exception during child_processes() function + with subprocess.Popen(["sleep", "60"]) as process: + r = process.poll() + + if r is not None: + logging.warning("process.pool() returns an unexpected result: {0}.".format(r)) + continue + + assert r is None + # collect list of processes currently running + children = psutil.Process(os.getpid()).children() + # kill a process, so received children dictionary becomes invalid + process.kill() + process.wait() + # try to handle children list -- missing processes will have ptype "ProcessType.Unknown" + [ProcessProxy(p) for p in children] + break + + @staticmethod + def helper__get_node(): + assert __class__.sm_conn_params is not None + return get_remote_node(conn_params=__class__.sm_conn_params) @staticmethod def helper__restore_envvar(name, prev_value): @@ -1074,23 +1172,20 @@ def helper__restore_envvar(name, prev_value): else: os.environ[name] = prev_value + @staticmethod + def helper__skip_test_if_util_not_exist(name: str): + assert type(name) == str # noqa: E721 + if not util_exists(name): + pytest.skip('might be missing') -if __name__ == '__main__': - if os_ops.environ('ALT_CONFIG'): - suite = unittest.TestSuite() - - # Small subset of tests for alternative configs (PG_BIN or PG_CONFIG) - suite.addTest(TestgresRemoteTests('test_pg_config')) - suite.addTest(TestgresRemoteTests('test_pg_ctl')) - suite.addTest(TestgresRemoteTests('test_psql')) - suite.addTest(TestgresRemoteTests('test_replicate')) - - print('Running tests for alternative config:') - for t in suite: - print(t) - print() + @staticmethod + def helper__skip_test_if_pg_version_is_not_ge(version: str): + assert type(version) == str # noqa: E721 + if not pg_version_ge(version): + pytest.skip('requires {0}+'.format(version)) - runner = unittest.TextTestRunner() - runner.run(suite) - else: - unittest.main() + @staticmethod + def helper__skip_test_if_pg_version_is_ge(version: str): + assert type(version) == str # noqa: E721 + if pg_version_ge(version): + pytest.skip('requires <{0}'.format(version)) From 9911ed0c928b602c17a3b0845b24910bf291925a Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Thu, 27 Feb 2025 13:48:05 +0300 Subject: [PATCH 139/216] [run_tests.sh] A right way for obtaining of BINDIR and PG_CONFIG is used (#196) A problem was detected in container with Ubuntu 24.04 tests works with "/usr/bin/pg_config" but real pg_config is "/usr/lib/postgresql/17/bin/pg_config" To resovle this problem we will call "pg_config --bindir" and use it result for BINDIR and PG_CONFIG. --- run_tests.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/run_tests.sh b/run_tests.sh index e9d58b54..5cbbac60 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -43,13 +43,13 @@ time coverage run -a -m pytest -l -v -n 4 -k "TestgresTests" # run tests (PG_BIN) time \ - PG_BIN=$(dirname $(which pg_config)) \ + PG_BIN=$(pg_config --bindir) \ coverage run -a -m pytest -l -v -n 4 -k "TestgresTests" # run tests (PG_CONFIG) time \ - PG_CONFIG=$(which pg_config) \ + PG_CONFIG=$(pg_config --bindir)/pg_config \ coverage run -a -m pytest -l -v -n 4 -k "TestgresTests" From 6d67da2170becb944fc768ff4938f88b37e0f2a4 Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Thu, 27 Feb 2025 14:30:43 +0300 Subject: [PATCH 140/216] Update README.md Build status is corrected A correct URL to get a starus image is https://api.travis-ci.com/postgrespro/testgres.svg?branch=master --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f0071a90..a3b854f8 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -[![Build Status](https://travis-ci.com/postgrespro/testgres.svg?branch=master)](https://app.travis-ci.com/github/postgrespro/testgres/branches) +[![Build Status](https://api.travis-ci.com/postgrespro/testgres.svg?branch=master)](https://travis-ci.com/github/postgrespro/testgres) [![codecov](https://codecov.io/gh/postgrespro/testgres/branch/master/graph/badge.svg)](https://codecov.io/gh/postgrespro/testgres) [![PyPI version](https://badge.fury.io/py/testgres.svg)](https://badge.fury.io/py/testgres) From 3cc162715e3bc3c49b944e03038665dbd4445221 Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Fri, 28 Feb 2025 12:04:05 +0300 Subject: [PATCH 141/216] TestgresRemoteTests::test_ports_management is corrected (#198) It is synchronized with TestgresTests::test_ports_management. --- tests/test_simple_remote.py | 36 +++++++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/tests/test_simple_remote.py b/tests/test_simple_remote.py index e7cc5e5c..d4a28a2b 100755 --- a/tests/test_simple_remote.py +++ b/tests/test_simple_remote.py @@ -976,20 +976,38 @@ def test_isolation_levels(self): con.begin('Garbage').commit() def test_ports_management(self): - # check that no ports have been bound yet - assert (len(bound_ports) == 0) + assert bound_ports is not None + assert type(bound_ports) == set # noqa: E721 + + if len(bound_ports) != 0: + logging.warning("bound_ports is not empty: {0}".format(bound_ports)) + + stage0__bound_ports = bound_ports.copy() with __class__.helper__get_node() as node: - # check that we've just bound a port - assert (len(bound_ports) == 1) + assert bound_ports is not None + assert type(bound_ports) == set # noqa: E721 + + assert node.port is not None + assert type(node.port) == int # noqa: E721 + + logging.info("node port is {0}".format(node.port)) + + assert node.port in bound_ports + assert node.port not in stage0__bound_ports + + assert stage0__bound_ports <= bound_ports + assert len(stage0__bound_ports) + 1 == len(bound_ports) + + stage1__bound_ports = stage0__bound_ports.copy() + stage1__bound_ports.add(node.port) - # check that bound_ports contains our port - port_1 = list(bound_ports)[0] - port_2 = node.port - assert (port_1 == port_2) + assert stage1__bound_ports == bound_ports # check that port has been freed successfully - assert (len(bound_ports) == 0) + assert bound_ports is not None + assert type(bound_ports) == set # noqa: E721 + assert bound_ports == stage0__bound_ports def test_exceptions(self): str(StartNodeException('msg', [('file', 'lines')])) From de432edafd63e3f054cefaefcbe69b4b3ae90e4f Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Fri, 28 Feb 2025 19:43:38 +0300 Subject: [PATCH 142/216] execute_utility2 is updated (ignore_errors) (#201) - New parameters "ignore_errors" is added. Default value is False. - Asserts are added. --- testgres/utils.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/testgres/utils.py b/testgres/utils.py index 9645fc3b..76d42b02 100644 --- a/testgres/utils.py +++ b/testgres/utils.py @@ -73,11 +73,13 @@ def execute_utility(args, logfile=None, verbose=False): return execute_utility2(tconf.os_ops, args, logfile, verbose) -def execute_utility2(os_ops: OsOperations, args, logfile=None, verbose=False): +def execute_utility2(os_ops: OsOperations, args, logfile=None, verbose=False, ignore_errors=False): assert os_ops is not None assert isinstance(os_ops, OsOperations) + assert type(verbose) == bool # noqa: E721 + assert type(ignore_errors) == bool # noqa: E721 - exit_status, out, error = os_ops.exec_command(args, verbose=True) + exit_status, out, error = os_ops.exec_command(args, verbose=True, ignore_errors=ignore_errors) # decode result out = '' if not out else out if isinstance(out, bytes): From 22826e001bd32314da0bef24101a916f011a0fdc Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Sat, 1 Mar 2025 09:57:49 +0300 Subject: [PATCH 143/216] PostgresNode::pid is improved (#199) * PostgresNode::pid is improved - We do multiple attempts to read pid file. - We process a case when we see that node is stopped between test and read. - We process a case when pid-file is empty. * PostgresNode::pid is updated Assert is added. * execute_utility2 is updated (ignore_errors) - New parameters "ignore_errors" is added. Default value is False. - Asserts are added. * PostgresNode::_try_shutdown is rewrited (normalization) * PostgresNode::pid uses the data from "pg_ctl status" output. * PostgresNode::_try_shutdown is correct (return None) This method returns nothing (None). --- testgres/consts.py | 4 + testgres/node.py | 247 +++++++++++++++++++++++++++++++++++++-------- 2 files changed, 211 insertions(+), 40 deletions(-) diff --git a/testgres/consts.py b/testgres/consts.py index 98c84af6..89c49ab7 100644 --- a/testgres/consts.py +++ b/testgres/consts.py @@ -35,3 +35,7 @@ # logical replication settings LOGICAL_REPL_MAX_CATCHUP_ATTEMPTS = 60 + +PG_CTL__STATUS__OK = 0 +PG_CTL__STATUS__NODE_IS_STOPPED = 3 +PG_CTL__STATUS__BAD_DATADIR = 4 diff --git a/testgres/node.py b/testgres/node.py index 56899b90..859fe742 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -49,7 +49,9 @@ RECOVERY_CONF_FILE, \ PG_LOG_FILE, \ UTILS_LOG_FILE, \ - PG_PID_FILE + PG_CTL__STATUS__OK, \ + PG_CTL__STATUS__NODE_IS_STOPPED, \ + PG_CTL__STATUS__BAD_DATADIR \ from .consts import \ MAX_LOGICAL_REPLICATION_WORKERS, \ @@ -208,14 +210,136 @@ def pid(self): Return postmaster's PID if node is running, else 0. """ - if self.status(): - pid_file = os.path.join(self.data_dir, PG_PID_FILE) - lines = self.os_ops.readlines(pid_file) - pid = int(lines[0]) if lines else None - return pid + self__data_dir = self.data_dir - # for clarity - return 0 + _params = [ + self._get_bin_path('pg_ctl'), + "-D", self__data_dir, + "status" + ] # yapf: disable + + status_code, out, error = execute_utility2( + self.os_ops, + _params, + self.utils_log_file, + verbose=True, + ignore_errors=True) + + assert type(status_code) == int # noqa: E721 + assert type(out) == str # noqa: E721 + assert type(error) == str # noqa: E721 + + # ----------------- + if status_code == PG_CTL__STATUS__NODE_IS_STOPPED: + return 0 + + # ----------------- + if status_code == PG_CTL__STATUS__BAD_DATADIR: + return 0 + + # ----------------- + if status_code != PG_CTL__STATUS__OK: + errMsg = "Getting of a node status [data_dir is {0}] failed.".format(self__data_dir) + + raise ExecUtilException( + message=errMsg, + command=_params, + exit_code=status_code, + out=out, + error=error, + ) + + # ----------------- + assert status_code == PG_CTL__STATUS__OK + + if out == "": + __class__._throw_error__pg_ctl_returns_an_empty_string( + _params + ) + + C_PID_PREFIX = "(PID: " + + i = out.find(C_PID_PREFIX) + + if i == -1: + __class__._throw_error__pg_ctl_returns_an_unexpected_string( + out, + _params + ) + + assert i > 0 + assert i < len(out) + assert len(C_PID_PREFIX) <= len(out) + assert i <= len(out) - len(C_PID_PREFIX) + + i += len(C_PID_PREFIX) + start_pid_s = i + + while True: + if i == len(out): + __class__._throw_error__pg_ctl_returns_an_unexpected_string( + out, + _params + ) + + ch = out[i] + + if ch == ")": + break + + if ch.isdigit(): + i += 1 + continue + + __class__._throw_error__pg_ctl_returns_an_unexpected_string( + out, + _params + ) + assert False + + if i == start_pid_s: + __class__._throw_error__pg_ctl_returns_an_unexpected_string( + out, + _params + ) + + # TODO: Let's verify a length of pid string. + + pid = int(out[start_pid_s:i]) + + if pid == 0: + __class__._throw_error__pg_ctl_returns_a_zero_pid( + out, + _params + ) + + assert pid != 0 + return pid + + @staticmethod + def _throw_error__pg_ctl_returns_an_empty_string(_params): + errLines = [] + errLines.append("Utility pg_ctl returns empty string.") + errLines.append("Command line is {0}".format(_params)) + raise RuntimeError("\n".join(errLines)) + + @staticmethod + def _throw_error__pg_ctl_returns_an_unexpected_string(out, _params): + errLines = [] + errLines.append("Utility pg_ctl returns an unexpected string:") + errLines.append(out) + errLines.append("------------") + errLines.append("Command line is {0}".format(_params)) + raise RuntimeError("\n".join(errLines)) + + @staticmethod + def _throw_error__pg_ctl_returns_a_zero_pid(out, _params): + errLines = [] + errLines.append("Utility pg_ctl returns a zero pid. Output string is:") + errLines.append(out) + errLines.append("------------") + errLines.append("Command line is {0}".format(_params)) + raise RuntimeError("\n".join(errLines)) @property def auxiliary_pids(self): @@ -338,41 +462,84 @@ def version(self): return self._pg_version def _try_shutdown(self, max_attempts, with_force=False): + assert type(max_attempts) == int # noqa: E721 + assert type(with_force) == bool # noqa: E721 + assert max_attempts > 0 + attempts = 0 + + # try stopping server N times + while attempts < max_attempts: + attempts += 1 + try: + self.stop() + except ExecUtilException: + continue # one more time + except Exception: + eprint('cannot stop node {}'.format(self.name)) + break + + return # OK + + # If force stopping is enabled and PID is valid + if not with_force: + return + node_pid = self.pid + assert node_pid is not None + assert type(node_pid) == int # noqa: E721 - if node_pid > 0: - # try stopping server N times - while attempts < max_attempts: - try: - self.stop() - break # OK - except ExecUtilException: - pass # one more time - except Exception: - eprint('cannot stop node {}'.format(self.name)) - break - - attempts += 1 - - # If force stopping is enabled and PID is valid - if with_force and node_pid != 0: - # If we couldn't stop the node - p_status_output = self.os_ops.exec_command(cmd=f'ps -o pid= -p {node_pid}', shell=True, ignore_errors=True).decode('utf-8') - if self.status() != NodeStatus.Stopped and p_status_output and str(node_pid) in p_status_output: - try: - eprint(f'Force stopping node {self.name} with PID {node_pid}') - self.os_ops.kill(node_pid, signal.SIGKILL, expect_error=False) - except Exception: - # The node has already stopped - pass - - # Check that node stopped - print only column pid without headers - p_status_output = self.os_ops.exec_command(f'ps -o pid= -p {node_pid}', shell=True, ignore_errors=True).decode('utf-8') - if p_status_output and str(node_pid) in p_status_output: - eprint(f'Failed to stop node {self.name}.') - else: - eprint(f'Node {self.name} has been stopped successfully.') + if node_pid == 0: + return + + # TODO: [2025-02-28] It is really the old ugly code. We have to rewrite it! + + ps_command = ['ps', '-o', 'pid=', '-p', str(node_pid)] + + ps_output = self.os_ops.exec_command(cmd=ps_command, shell=True, ignore_errors=True).decode('utf-8') + assert type(ps_output) == str # noqa: E721 + + if ps_output == "": + return + + if ps_output != str(node_pid): + __class__._throw_bugcheck__unexpected_result_of_ps( + ps_output, + ps_command) + + try: + eprint('Force stopping node {0} with PID {1}'.format(self.name, node_pid)) + self.os_ops.kill(node_pid, signal.SIGKILL, expect_error=False) + except Exception: + # The node has already stopped + pass + + # Check that node stopped - print only column pid without headers + ps_output = self.os_ops.exec_command(cmd=ps_command, shell=True, ignore_errors=True).decode('utf-8') + assert type(ps_output) == str # noqa: E721 + + if ps_output == "": + eprint('Node {0} has been stopped successfully.'.format(self.name)) + return + + if ps_output == str(node_pid): + eprint('Failed to stop node {0}.'.format(self.name)) + return + + __class__._throw_bugcheck__unexpected_result_of_ps( + ps_output, + ps_command) + + @staticmethod + def _throw_bugcheck__unexpected_result_of_ps(result, cmd): + assert type(result) == str # noqa: E721 + assert type(cmd) == list # noqa: E721 + errLines = [] + errLines.append("[BUG CHECK] Unexpected result of command ps:") + errLines.append(result) + errLines.append("-----") + errLines.append("Command line is {0}".format(cmd)) + raise RuntimeError("\n".join(errLines)) def _assign_master(self, master): """NOTE: this is a private method!""" From b0f90d94d81a6470e2ef4f904e79ff1114d69bed Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Sat, 1 Mar 2025 11:13:27 +0300 Subject: [PATCH 144/216] [RemoteOperations] A call of mktemp is fixed (#202) When we define a template we have to use "-t" option. It forces mktemp to return a path instead name. The following methods of RemoteOperations are fixed: - mkdtemp - mkstemp --- testgres/operations/remote_ops.py | 46 +++++++++++++++++++------------ 1 file changed, 28 insertions(+), 18 deletions(-) diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index 51f5b2e8..2a4e5c78 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -247,32 +247,42 @@ def mkdtemp(self, prefix=None): - prefix (str): The prefix of the temporary directory name. """ if prefix: - command = ["ssh"] + self.ssh_args + [self.ssh_dest, f"mktemp -d {prefix}XXXXX"] + command = ["mktemp", "-d", "-t", prefix + "XXXXX"] else: - command = ["ssh"] + self.ssh_args + [self.ssh_dest, "mktemp -d"] + command = ["mktemp", "-d"] - result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + exit_status, result, error = self.exec_command(command, verbose=True, encoding=get_default_encoding(), ignore_errors=True) - if result.returncode == 0: - temp_dir = result.stdout.strip() - if not os.path.isabs(temp_dir): - temp_dir = os.path.join('/home', self.username, temp_dir) - return temp_dir - else: - raise ExecUtilException(f"Could not create temporary directory. Error: {result.stderr}") + assert type(result) == str # noqa: E721 + assert type(error) == str # noqa: E721 + + if exit_status != 0: + raise ExecUtilException("Could not create temporary directory. Error code: {0}. Error message: {1}".format(exit_status, error)) + + temp_dir = result.strip() + return temp_dir def mkstemp(self, prefix=None): + """ + Creates a temporary file in the remote server. + Args: + - prefix (str): The prefix of the temporary directory name. + """ if prefix: - temp_dir = self.exec_command("mktemp {}XXXXX".format(prefix), encoding=get_default_encoding()) + command = ["mktemp", "-t", prefix + "XXXXX"] else: - temp_dir = self.exec_command("mktemp", encoding=get_default_encoding()) + command = ["mktemp"] - if temp_dir: - if not os.path.isabs(temp_dir): - temp_dir = os.path.join('/home', self.username, temp_dir.strip()) - return temp_dir - else: - raise ExecUtilException("Could not create temporary directory.") + exit_status, result, error = self.exec_command(command, verbose=True, encoding=get_default_encoding(), ignore_errors=True) + + assert type(result) == str # noqa: E721 + assert type(error) == str # noqa: E721 + + if exit_status != 0: + raise ExecUtilException("Could not create temporary file. Error code: {0}. Error message: {1}".format(exit_status, error)) + + temp_file = result.strip() + return temp_file def copytree(self, src, dst): if not os.path.isabs(dst): From 71772122b6485a71c6831dbe11690d970c809f38 Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Sat, 1 Mar 2025 23:39:42 +0300 Subject: [PATCH 145/216] TestRemoteOperations::test_is_executable_true is corrected (#204) Let's test a real pg_config. --- tests/test_remote.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/test_remote.py b/tests/test_remote.py index 8b167e9f..e457de07 100755 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -8,7 +8,9 @@ from ..testgres import ExecUtilException from ..testgres import InvalidOperationException from ..testgres import RemoteOperations +from ..testgres import LocalOperations from ..testgres import ConnectionParams +from ..testgres import utils as testgres_utils class TestRemoteOperations: @@ -59,7 +61,11 @@ def test_is_executable_true(self): """ Test is_executable for an existing executable. """ - cmd = os.getenv('PG_CONFIG') + local_ops = LocalOperations() + cmd = testgres_utils.get_bin_path2(local_ops, "pg_config") + cmd = local_ops.exec_command([cmd, "--bindir"], encoding="utf-8") + cmd = cmd.rstrip() + cmd = os.path.join(cmd, "pg_config") response = self.operations.is_executable(cmd) assert response is True From 0ffd5f0c4c5ff682ffaec8ee7a6145d55e904aec Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Sat, 1 Mar 2025 23:40:29 +0300 Subject: [PATCH 146/216] Total refactoring of os_ops::execute_command (#203) * Total refactoring of os_ops::execute_command Main - We check only an exit code to detect an error. - If someone utility returns a result through an exit code, a caller side should set ignore_errors=true and process this case itself. - If expect_error is true and no errors occurred, we raise an InvalidOperationException. * The old behaviour of RaiseError.UtilityExitedWithNonZeroCode is restored Let's rollback the new code to avoid problems with probackup2' tests. --- testgres/operations/local_ops.py | 34 +++++----- testgres/operations/raise_error.py | 31 +++++---- testgres/operations/remote_ops.py | 101 +++++++++++++++++++---------- testgres/utils.py | 13 ++-- tests/test_local.py | 5 +- tests/test_remote.py | 10 +-- tests/test_simple_remote.py | 43 ++++++------ 7 files changed, 141 insertions(+), 96 deletions(-) diff --git a/testgres/operations/local_ops.py b/testgres/operations/local_ops.py index 91070fe7..51003174 100644 --- a/testgres/operations/local_ops.py +++ b/testgres/operations/local_ops.py @@ -23,20 +23,6 @@ from distutils import rmtree CMD_TIMEOUT_SEC = 60 -error_markers = [b'error', b'Permission denied', b'fatal'] -err_out_markers = [b'Failure'] - - -def has_errors(output=None, error=None): - if output: - if isinstance(output, str): - output = output.encode(get_default_encoding()) - return any(marker in output for marker in err_out_markers) - if error: - if isinstance(error, str): - error = error.encode(get_default_encoding()) - return any(marker in error for marker in error_markers) - return False class LocalOperations(OsOperations): @@ -134,19 +120,29 @@ def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False, process, output, error = self._run_command(cmd, shell, input, stdin, stdout, stderr, get_process, timeout, encoding) if get_process: return process - if not ignore_errors and ((process.returncode != 0 or has_errors(output=output, error=error)) and not expect_error): + + if expect_error: + if process.returncode == 0: + raise InvalidOperationException("We expected an execution error.") + elif ignore_errors: + pass + elif process.returncode == 0: + pass + else: + assert not expect_error + assert not ignore_errors + assert process.returncode != 0 RaiseError.UtilityExitedWithNonZeroCode( cmd=cmd, exit_code=process.returncode, msg_arg=error or output, error=error, - out=output - ) + out=output) if verbose: return process.returncode, output, error - else: - return output + + return output # Environment setup def environ(self, var_name): diff --git a/testgres/operations/raise_error.py b/testgres/operations/raise_error.py index 6031b238..0d14be5a 100644 --- a/testgres/operations/raise_error.py +++ b/testgres/operations/raise_error.py @@ -7,13 +7,27 @@ class RaiseError: def UtilityExitedWithNonZeroCode(cmd, exit_code, msg_arg, error, out): assert type(exit_code) == int # noqa: E721 - msg_arg_s = __class__._TranslateDataIntoString(msg_arg).strip() + msg_arg_s = __class__._TranslateDataIntoString(msg_arg) assert type(msg_arg_s) == str # noqa: E721 + msg_arg_s = msg_arg_s.strip() if msg_arg_s == "": msg_arg_s = "#no_error_message" - message = "Utility exited with non-zero code. Error: `" + msg_arg_s + "`" + message = "Utility exited with non-zero code (" + str(exit_code) + "). Error: `" + msg_arg_s + "`" + raise ExecUtilException( + message=message, + command=cmd, + exit_code=exit_code, + out=out, + error=error) + + @staticmethod + def CommandExecutionError(cmd, exit_code, message, error, out): + assert type(exit_code) == int # noqa: E721 + assert type(message) == str # noqa: E721 + assert message != "" + raise ExecUtilException( message=message, command=cmd, @@ -23,6 +37,9 @@ def UtilityExitedWithNonZeroCode(cmd, exit_code, msg_arg, error, out): @staticmethod def _TranslateDataIntoString(data): + if data is None: + return "" + if type(data) == bytes: # noqa: E721 return __class__._TranslateDataIntoString__FromBinary(data) @@ -38,13 +55,3 @@ def _TranslateDataIntoString__FromBinary(data): pass return "#cannot_decode_text" - - @staticmethod - def _BinaryIsASCII(data): - assert type(data) == bytes # noqa: E721 - - for b in data: - if not (b >= 0 and b <= 127): - return False - - return True diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index 2a4e5c78..dc392bee 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -100,41 +100,40 @@ def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False, return process try: - result, error = process.communicate(input=input_prepared, timeout=timeout) + output, error = process.communicate(input=input_prepared, timeout=timeout) except subprocess.TimeoutExpired: process.kill() raise ExecUtilException("Command timed out after {} seconds.".format(timeout)) - exit_status = process.returncode - - assert type(result) == bytes # noqa: E721 + assert type(output) == bytes # noqa: E721 assert type(error) == bytes # noqa: E721 - if not error: - error_found = False - else: - error_found = exit_status != 0 or any( - marker in error for marker in [b'error', b'Permission denied', b'fatal', b'No such file or directory'] - ) - - assert type(error_found) == bool # noqa: E721 - if encoding: - result = result.decode(encoding) + output = output.decode(encoding) error = error.decode(encoding) - if not ignore_errors and error_found and not expect_error: + if expect_error: + if process.returncode == 0: + raise InvalidOperationException("We expected an execution error.") + elif ignore_errors: + pass + elif process.returncode == 0: + pass + else: + assert not expect_error + assert not ignore_errors + assert process.returncode != 0 RaiseError.UtilityExitedWithNonZeroCode( cmd=cmd, - exit_code=exit_status, + exit_code=process.returncode, msg_arg=error, error=error, - out=result) + out=output) if verbose: - return exit_status, result, error - else: - return result + return process.returncode, output, error + + return output # Environment setup def environ(self, var_name: str) -> str: @@ -165,8 +164,30 @@ def find_executable(self, executable): def is_executable(self, file): # Check if the file is executable - is_exec = self.exec_command("test -x {} && echo OK".format(file)) - return is_exec == b"OK\n" + command = ["test", "-x", file] + + exit_status, output, error = self.exec_command(cmd=command, encoding=get_default_encoding(), ignore_errors=True, verbose=True) + + assert type(output) == str # noqa: E721 + assert type(error) == str # noqa: E721 + + if exit_status == 0: + return True + + if exit_status == 1: + return False + + errMsg = "Test operation returns an unknown result code: {0}. File name is [{1}].".format( + exit_status, + file) + + RaiseError.CommandExecutionError( + cmd=command, + exit_code=exit_status, + msg_arg=errMsg, + error=error, + out=output + ) def set_env(self, var_name: str, var_val: str): """ @@ -251,15 +272,21 @@ def mkdtemp(self, prefix=None): else: command = ["mktemp", "-d"] - exit_status, result, error = self.exec_command(command, verbose=True, encoding=get_default_encoding(), ignore_errors=True) + exec_exitcode, exec_output, exec_error = self.exec_command(command, verbose=True, encoding=get_default_encoding(), ignore_errors=True) - assert type(result) == str # noqa: E721 - assert type(error) == str # noqa: E721 + assert type(exec_exitcode) == int # noqa: E721 + assert type(exec_output) == str # noqa: E721 + assert type(exec_error) == str # noqa: E721 - if exit_status != 0: - raise ExecUtilException("Could not create temporary directory. Error code: {0}. Error message: {1}".format(exit_status, error)) + if exec_exitcode != 0: + RaiseError.CommandExecutionError( + cmd=command, + exit_code=exec_exitcode, + message="Could not create temporary directory.", + error=exec_error, + out=exec_output) - temp_dir = result.strip() + temp_dir = exec_output.strip() return temp_dir def mkstemp(self, prefix=None): @@ -273,15 +300,21 @@ def mkstemp(self, prefix=None): else: command = ["mktemp"] - exit_status, result, error = self.exec_command(command, verbose=True, encoding=get_default_encoding(), ignore_errors=True) + exec_exitcode, exec_output, exec_error = self.exec_command(command, verbose=True, encoding=get_default_encoding(), ignore_errors=True) - assert type(result) == str # noqa: E721 - assert type(error) == str # noqa: E721 + assert type(exec_exitcode) == int # noqa: E721 + assert type(exec_output) == str # noqa: E721 + assert type(exec_error) == str # noqa: E721 - if exit_status != 0: - raise ExecUtilException("Could not create temporary file. Error code: {0}. Error message: {1}".format(exit_status, error)) + if exec_exitcode != 0: + RaiseError.CommandExecutionError( + cmd=command, + exit_code=exec_exitcode, + message="Could not create temporary file.", + error=exec_error, + out=exec_output) - temp_file = result.strip() + temp_file = exec_output.strip() return temp_file def copytree(self, src, dst): diff --git a/testgres/utils.py b/testgres/utils.py index 76d42b02..093eaff6 100644 --- a/testgres/utils.py +++ b/testgres/utils.py @@ -18,6 +18,7 @@ from .config import testgres_config as tconf from .operations.os_ops import OsOperations from .operations.remote_ops import RemoteOperations +from .operations.helpers import Helpers as OsHelpers # rows returned by PG_CONFIG _pg_config_data = {} @@ -79,13 +80,13 @@ def execute_utility2(os_ops: OsOperations, args, logfile=None, verbose=False, ig assert type(verbose) == bool # noqa: E721 assert type(ignore_errors) == bool # noqa: E721 - exit_status, out, error = os_ops.exec_command(args, verbose=True, ignore_errors=ignore_errors) - # decode result + exit_status, out, error = os_ops.exec_command( + args, + verbose=True, + ignore_errors=ignore_errors, + encoding=OsHelpers.GetDefaultEncoding()) + out = '' if not out else out - if isinstance(out, bytes): - out = out.decode('utf-8') - if isinstance(error, bytes): - error = error.decode('utf-8') # write new log entry if possible if logfile: diff --git a/tests/test_local.py b/tests/test_local.py index 60a96c18..ee5e19a0 100644 --- a/tests/test_local.py +++ b/tests/test_local.py @@ -40,10 +40,11 @@ def test_exec_command_failure(self): try: self.operations.exec_command(cmd, wait_exit=True, shell=True) except ExecUtilException as e: - error = e.message + assert e.message == "Utility exited with non-zero code (127). Error: `/bin/sh: 1: nonexistent_command: not found`" + assert type(e.error) == bytes # noqa: E721 + assert e.error.strip() == b"/bin/sh: 1: nonexistent_command: not found" break raise Exception("We wait an exception!") - assert error == "Utility exited with non-zero code. Error: `/bin/sh: 1: nonexistent_command: not found`" def test_exec_command_failure__expect_error(self): """ diff --git a/tests/test_remote.py b/tests/test_remote.py index e457de07..6114e29e 100755 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -40,10 +40,11 @@ def test_exec_command_failure(self): try: self.operations.exec_command(cmd, verbose=True, wait_exit=True) except ExecUtilException as e: - error = e.message + assert e.message == "Utility exited with non-zero code (127). Error: `bash: line 1: nonexistent_command: command not found`" + assert type(e.error) == bytes # noqa: E721 + assert e.error.strip() == b"bash: line 1: nonexistent_command: command not found" break raise Exception("We wait an exception!") - assert error == 'Utility exited with non-zero code. Error: `bash: line 1: nonexistent_command: command not found`' def test_exec_command_failure__expect_error(self): """ @@ -114,10 +115,11 @@ def test_makedirs_and_rmdirs_failure(self): try: self.operations.rmdirs(path, verbose=True) except ExecUtilException as e: - error = e.message + assert e.message == "Utility exited with non-zero code (1). Error: `rm: cannot remove '/root/test_dir': Permission denied`" + assert type(e.error) == bytes # noqa: E721 + assert e.error.strip() == b"rm: cannot remove '/root/test_dir': Permission denied" break raise Exception("We wait an exception!") - assert error == "Utility exited with non-zero code. Error: `rm: cannot remove '/root/test_dir': Permission denied`" def test_listdir(self): """ diff --git a/tests/test_simple_remote.py b/tests/test_simple_remote.py index d4a28a2b..74b10635 100755 --- a/tests/test_simple_remote.py +++ b/tests/test_simple_remote.py @@ -178,27 +178,32 @@ def test_init__unk_LANG_and_LC_CTYPE(self): assert os.environ.get("LC_CTYPE") == unkData[1] assert not ("LC_COLLATE" in os.environ.keys()) - while True: + assert os.getenv('LANG') == unkData[0] + assert os.getenv('LANGUAGE') is None + assert os.getenv('LC_CTYPE') == unkData[1] + assert os.getenv('LC_COLLATE') is None + + exc: ExecUtilException = None + with __class__.helper__get_node() as node: try: - with __class__.helper__get_node(): - pass - except ExecUtilException as e: - # - # Example of an error message: - # - # warning: setlocale: LC_CTYPE: cannot change locale (UNKNOWN_CTYPE): No such file or directory - # postgres (PostgreSQL) 14.12 - # - errMsg = str(e) - - logging.info("Error message is: {0}".format(errMsg)) - - assert "LC_CTYPE" in errMsg - assert unkData[1] in errMsg - assert "warning: setlocale: LC_CTYPE: cannot change locale (" + unkData[1] + "): No such file or directory" in errMsg - assert ("postgres" in errMsg) or ("PostgreSQL" in errMsg) - break + node.init() # IT RAISES! + except InitNodeException as e: + exc = e.__cause__ + assert exc is not None + assert isinstance(exc, ExecUtilException) + + if exc is None: raise Exception("We expected an error!") + + assert isinstance(exc, ExecUtilException) + + errMsg = str(exc) + logging.info("Error message is {0}: {1}".format(type(exc).__name__, errMsg)) + + assert "warning: setlocale: LC_CTYPE: cannot change locale (" + unkData[1] + ")" in errMsg + assert "initdb: error: invalid locale settings; check LANG and LC_* environment variables" in errMsg + continue + finally: __class__.helper__restore_envvar("LANG", prev_LANG) __class__.helper__restore_envvar("LANGUAGE", prev_LANGUAGE) From 7abca7f5e36ea20c6d8dd5dcc29b34ca5c9090c1 Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Sun, 2 Mar 2025 17:33:10 +0300 Subject: [PATCH 147/216] xxx::test_logging is corrected (local, remote) (#205) - these tests configure logging wrong and create the conflicts with root logger - these tests (local and remote) conflict with each other --- tests/test_simple.py | 142 +++++++++++++++++++++------------- tests/test_simple_remote.py | 147 +++++++++++++++++++++++------------- 2 files changed, 184 insertions(+), 105 deletions(-) diff --git a/tests/test_simple.py b/tests/test_simple.py index 6c433cd4..37c3db44 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -8,8 +8,8 @@ import pytest import psutil import platform - -import logging.config +import logging +import uuid from contextlib import contextmanager from shutil import rmtree @@ -718,55 +718,95 @@ def test_poll_query_until(self): node.poll_query_until('select true') def test_logging(self): - logfile = tempfile.NamedTemporaryFile('w', delete=True) - - log_conf = { - 'version': 1, - 'handlers': { - 'file': { - 'class': 'logging.FileHandler', - 'filename': logfile.name, - 'formatter': 'base_format', - 'level': logging.DEBUG, - }, - }, - 'formatters': { - 'base_format': { - 'format': '%(node)-5s: %(message)s', - }, - }, - 'root': { - 'handlers': ('file', ), - 'level': 'DEBUG', - }, - } - - logging.config.dictConfig(log_conf) - - with scoped_config(use_python_logging=True): - node_name = 'master' - - with get_new_node(name=node_name) as master: - master.init().start() - - # execute a dummy query a few times - for i in range(20): - master.execute('select 1') - time.sleep(0.01) - - # let logging worker do the job - time.sleep(0.1) - - # check that master's port is found - with open(logfile.name, 'r') as log: - lines = log.readlines() - assert (any(node_name in s for s in lines)) - - # test logger after stop/start/restart - master.stop() - master.start() - master.restart() - assert (master._logger.is_alive()) + C_MAX_ATTEMPTS = 50 + # This name is used for testgres logging, too. + C_NODE_NAME = "testgres_tests." + __class__.__name__ + "test_logging-master-" + uuid.uuid4().hex + + logging.info("Node name is [{0}]".format(C_NODE_NAME)) + + with tempfile.NamedTemporaryFile('w', delete=True) as logfile: + formatter = logging.Formatter(fmt="%(node)-5s: %(message)s") + handler = logging.FileHandler(filename=logfile.name) + handler.formatter = formatter + logger = logging.getLogger(C_NODE_NAME) + assert logger is not None + assert len(logger.handlers) == 0 + + try: + # It disables to log on the root level + logger.propagate = False + logger.addHandler(handler) + + with scoped_config(use_python_logging=True): + with get_new_node(name=C_NODE_NAME) as master: + logging.info("Master node is initilizing") + master.init() + + logging.info("Master node is starting") + master.start() + + logging.info("Dummy query is executed a few times") + for _ in range(20): + master.execute('select 1') + time.sleep(0.01) + + # let logging worker do the job + time.sleep(0.1) + + logging.info("Master node log file is checking") + nAttempt = 0 + + while True: + assert nAttempt <= C_MAX_ATTEMPTS + if nAttempt == C_MAX_ATTEMPTS: + raise Exception("Test failed!") + + # let logging worker do the job + time.sleep(0.1) + + nAttempt += 1 + + logging.info("Attempt {0}".format(nAttempt)) + + # check that master's port is found + with open(logfile.name, 'r') as log: + lines = log.readlines() + + assert lines is not None + assert type(lines) == list # noqa: E721 + + def LOCAL__test_lines(): + for s in lines: + if any(C_NODE_NAME in s for s in lines): + logging.info("OK. We found the node_name in a line \"{0}\"".format(s)) + return True + return False + + if LOCAL__test_lines(): + break + + logging.info("Master node log file does not have an expected information.") + continue + + # test logger after stop/start/restart + logging.info("Master node is stopping...") + master.stop() + logging.info("Master node is staring again...") + master.start() + logging.info("Master node is restaring...") + master.restart() + assert (master._logger.is_alive()) + finally: + # It is a hack code to logging cleanup + logging._acquireLock() + assert logging.Logger.manager is not None + assert C_NODE_NAME in logging.Logger.manager.loggerDict.keys() + logging.Logger.manager.loggerDict.pop(C_NODE_NAME, None) + assert not (C_NODE_NAME in logging.Logger.manager.loggerDict.keys()) + assert not (handler in logging._handlers.values()) + logging._releaseLock() + # GO HOME! + return def test_pgbench(self): __class__.helper__skip_test_if_util_not_exist("pgbench") diff --git a/tests/test_simple_remote.py b/tests/test_simple_remote.py index 74b10635..a62085ce 100755 --- a/tests/test_simple_remote.py +++ b/tests/test_simple_remote.py @@ -8,8 +8,8 @@ import six import pytest import psutil - -import logging.config +import logging +import uuid from contextlib import contextmanager @@ -788,56 +788,95 @@ def test_poll_query_until(self): node.poll_query_until('select true') def test_logging(self): - # FAIL - logfile = tempfile.NamedTemporaryFile('w', delete=True) - - log_conf = { - 'version': 1, - 'handlers': { - 'file': { - 'class': 'logging.FileHandler', - 'filename': logfile.name, - 'formatter': 'base_format', - 'level': logging.DEBUG, - }, - }, - 'formatters': { - 'base_format': { - 'format': '%(node)-5s: %(message)s', - }, - }, - 'root': { - 'handlers': ('file',), - 'level': 'DEBUG', - }, - } - - logging.config.dictConfig(log_conf) - - with scoped_config(use_python_logging=True): - node_name = 'master' - - with get_remote_node(name=node_name) as master: - master.init().start() - - # execute a dummy query a few times - for i in range(20): - master.execute('select 1') - time.sleep(0.01) - - # let logging worker do the job - time.sleep(0.1) - - # check that master's port is found - with open(logfile.name, 'r') as log: - lines = log.readlines() - assert (any(node_name in s for s in lines)) - - # test logger after stop/start/restart - master.stop() - master.start() - master.restart() - assert (master._logger.is_alive()) + C_MAX_ATTEMPTS = 50 + # This name is used for testgres logging, too. + C_NODE_NAME = "testgres_tests." + __class__.__name__ + "test_logging-master-" + uuid.uuid4().hex + + logging.info("Node name is [{0}]".format(C_NODE_NAME)) + + with tempfile.NamedTemporaryFile('w', delete=True) as logfile: + formatter = logging.Formatter(fmt="%(node)-5s: %(message)s") + handler = logging.FileHandler(filename=logfile.name) + handler.formatter = formatter + logger = logging.getLogger(C_NODE_NAME) + assert logger is not None + assert len(logger.handlers) == 0 + + try: + # It disables to log on the root level + logger.propagate = False + logger.addHandler(handler) + + with scoped_config(use_python_logging=True): + with __class__.helper__get_node(name=C_NODE_NAME) as master: + logging.info("Master node is initilizing") + master.init() + + logging.info("Master node is starting") + master.start() + + logging.info("Dummy query is executed a few times") + for _ in range(20): + master.execute('select 1') + time.sleep(0.01) + + # let logging worker do the job + time.sleep(0.1) + + logging.info("Master node log file is checking") + nAttempt = 0 + + while True: + assert nAttempt <= C_MAX_ATTEMPTS + if nAttempt == C_MAX_ATTEMPTS: + raise Exception("Test failed!") + + # let logging worker do the job + time.sleep(0.1) + + nAttempt += 1 + + logging.info("Attempt {0}".format(nAttempt)) + + # check that master's port is found + with open(logfile.name, 'r') as log: + lines = log.readlines() + + assert lines is not None + assert type(lines) == list # noqa: E721 + + def LOCAL__test_lines(): + for s in lines: + if any(C_NODE_NAME in s for s in lines): + logging.info("OK. We found the node_name in a line \"{0}\"".format(s)) + return True + return False + + if LOCAL__test_lines(): + break + + logging.info("Master node log file does not have an expected information.") + continue + + # test logger after stop/start/restart + logging.info("Master node is stopping...") + master.stop() + logging.info("Master node is staring again...") + master.start() + logging.info("Master node is restaring...") + master.restart() + assert (master._logger.is_alive()) + finally: + # It is a hack code to logging cleanup + logging._acquireLock() + assert logging.Logger.manager is not None + assert C_NODE_NAME in logging.Logger.manager.loggerDict.keys() + logging.Logger.manager.loggerDict.pop(C_NODE_NAME, None) + assert not (C_NODE_NAME in logging.Logger.manager.loggerDict.keys()) + assert not (handler in logging._handlers.values()) + logging._releaseLock() + # GO HOME! + return def test_pgbench(self): __class__.helper__skip_test_if_util_not_exist("pgbench") @@ -1184,9 +1223,9 @@ def test_child_process_dies(self): break @staticmethod - def helper__get_node(): + def helper__get_node(name=None): assert __class__.sm_conn_params is not None - return get_remote_node(conn_params=__class__.sm_conn_params) + return get_remote_node(name=name, conn_params=__class__.sm_conn_params) @staticmethod def helper__restore_envvar(name, prev_value): From e1a5bb451186c871df56f79f78af78dd63d3381e Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Mon, 3 Mar 2025 12:37:26 +0300 Subject: [PATCH 148/216] Updating of CI-tests (#197) * Test dockerfile for ubuntu 24.04 is added * Cleanup (Dockerfile--ubuntu-24_04) * Cleanup. Including of 'postgres' in 'sudo' group is not required. * [run_tests.sh] A right way for obtaining of BINDIR and PG_CONFIG is used A problem was detected in container with Ubuntu 24.04 tests works with "/usr/bin/pg_config" but real pg_config is "/usr/lib/postgresql/17/bin/pg_config" To resovle this problem we will call "pg_config --bindir" and use it result for BINDIR and PG_CONFIG. * Dockerfile--ubuntu-24_04 is updated Let's use /pg/testgres/run_tests.sh directly. * Dockerfile--ubuntu-24_04 is updated (cleanup) curl is installing twice. * Dockerfile--ubuntu-24_04 is updated (cleanup) [del] musl-dev [del] mc * Dockerfile--ubuntu-24_04 is formatted * CI-test on Ubuntu 24.04 is added. * Dockerfile--std.tmpl is updated (refactoring) /pg/testgres/run_tests.sh is used directly. * Dockerfile--ubuntu-24_04.tmpl is updated * PostgresNode::pid is improved - We do multiple attempts to read pid file. - We process a case when we see that node is stopped between test and read. - We process a case when pid-file is empty. * PostgresNode::pid is updated Assert is added. * execute_utility2 is updated (ignore_errors) - New parameters "ignore_errors" is added. Default value is False. - Asserts are added. * PostgresNode::_try_shutdown is rewrited (normalization) * PostgresNode::pid uses the data from "pg_ctl status" output. * PostgresNode::_try_shutdown is correct (return None) This method returns nothing (None). * [RemoteOperations] A call of mktemp is fixed When we define a template we have to use "-t" option. It forces mktemp to return a path instead name. The following methods of RemoteOperations are fixed: - mkdtemp - mkstemp * Total refactoring of os_ops::execute_command Main - We check only an exit code to detect an error. - If someone utility returns a result through an exit code, a caller side should set ignore_errors=true and process this case itself. - If expect_error is true and no errors occurred, we raise an InvalidOperationException. * Dockerfile--ubuntu-24_04.tmpl is updated The folder "home/postgres" is not required now. * The old behaviour of RaiseError.UtilityExitedWithNonZeroCode is restored Let's rollback the new code to avoid problems with probackup2' tests. * TestRemoteOperations::test_is_executable_true is corrected Let's test a real pg_config. * xxx::test_logging is corrected (local, remote) - these tests configure logging wrong and create the conflicts with root logger - these tests (local and remote) conflict with each other * TEST_FILTER is added * CI on Ubuntu 24.04 runs all the tests --- .travis.yml | 28 ++++------ Dockerfile.tmpl => Dockerfile--std.tmpl | 6 +-- Dockerfile--ubuntu-24_04.tmpl | 69 +++++++++++++++++++++++++ mk_dockerfile.sh | 2 +- run_tests.sh | 7 +-- 5 files changed, 86 insertions(+), 26 deletions(-) rename Dockerfile.tmpl => Dockerfile--std.tmpl (80%) create mode 100644 Dockerfile--ubuntu-24_04.tmpl diff --git a/.travis.yml b/.travis.yml index 6f63a67b..4110835a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,23 +20,17 @@ notifications: on_failure: always env: - - PYTHON_VERSION=3 PG_VERSION=17 - - PYTHON_VERSION=3 PG_VERSION=16 - - PYTHON_VERSION=3 PG_VERSION=15 - - PYTHON_VERSION=3 PG_VERSION=14 - - PYTHON_VERSION=3 PG_VERSION=13 - - PYTHON_VERSION=3 PG_VERSION=12 - - PYTHON_VERSION=3 PG_VERSION=11 - - PYTHON_VERSION=3 PG_VERSION=10 -# - PYTHON_VERSION=3 PG_VERSION=9.6 -# - PYTHON_VERSION=3 PG_VERSION=9.5 -# - PYTHON_VERSION=3 PG_VERSION=9.4 -# - PYTHON_VERSION=2 PG_VERSION=10 -# - PYTHON_VERSION=2 PG_VERSION=9.6 -# - PYTHON_VERSION=2 PG_VERSION=9.5 -# - PYTHON_VERSION=2 PG_VERSION=9.4 + - TEST_PLATFORM=std PYTHON_VERSION=3 PG_VERSION=17 + - TEST_PLATFORM=std PYTHON_VERSION=3 PG_VERSION=16 + - TEST_PLATFORM=std PYTHON_VERSION=3 PG_VERSION=15 + - TEST_PLATFORM=std PYTHON_VERSION=3 PG_VERSION=14 + - TEST_PLATFORM=std PYTHON_VERSION=3 PG_VERSION=13 + - TEST_PLATFORM=std PYTHON_VERSION=3 PG_VERSION=12 + - TEST_PLATFORM=std PYTHON_VERSION=3 PG_VERSION=11 + - TEST_PLATFORM=std PYTHON_VERSION=3 PG_VERSION=10 + - TEST_PLATFORM=ubuntu-24_04 PYTHON_VERSION=3 PG_VERSION=17 matrix: allow_failures: - - env: PYTHON_VERSION=3 PG_VERSION=11 - - env: PYTHON_VERSION=3 PG_VERSION=10 + - env: TEST_PLATFORM=std PYTHON_VERSION=3 PG_VERSION=11 + - env: TEST_PLATFORM=std PYTHON_VERSION=3 PG_VERSION=10 diff --git a/Dockerfile.tmpl b/Dockerfile--std.tmpl similarity index 80% rename from Dockerfile.tmpl rename to Dockerfile--std.tmpl index dc5878b6..d844c9a3 100644 --- a/Dockerfile.tmpl +++ b/Dockerfile--std.tmpl @@ -11,13 +11,9 @@ RUN if [ "${PYTHON_VERSION}" = "3" ] ; then \ fi ENV LANG=C.UTF-8 -RUN mkdir -p /pg -COPY run_tests.sh /run.sh -RUN chmod 755 /run.sh - ADD . /pg/testgres WORKDIR /pg/testgres RUN chown -R postgres:postgres /pg USER postgres -ENTRYPOINT PYTHON_VERSION=${PYTHON_VERSION} /run.sh +ENTRYPOINT PYTHON_VERSION=${PYTHON_VERSION} bash run_tests.sh diff --git a/Dockerfile--ubuntu-24_04.tmpl b/Dockerfile--ubuntu-24_04.tmpl new file mode 100644 index 00000000..99be5343 --- /dev/null +++ b/Dockerfile--ubuntu-24_04.tmpl @@ -0,0 +1,69 @@ +FROM ubuntu:24.04 + +RUN apt update +RUN apt install -y sudo curl ca-certificates postgresql-common + +RUN bash /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh -y + +RUN install -d /usr/share/postgresql-common/pgdg +RUN curl -o /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc --fail https://www.postgresql.org/media/keys/ACCC4CF8.asc + +# It does not work +# RUN sh -c 'echo "deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' + +RUN apt update +RUN apt install -y postgresql-${PG_VERSION} + +RUN apt install -y python3 python3-dev python3-virtualenv +# RUN apt install -y mc + +# It is required for psycopg2 +RUN apt install -y libpq-dev +RUN apt install -y openssh-server + +# [2025-02-26] It adds the user 'postgres' in the group 'sudo' +# [2025-02-27] It is not required. +# RUN adduser postgres sudo + +ADD . /pg/testgres +WORKDIR /pg/testgres +RUN chown -R postgres /pg + +EXPOSE 22 + +RUN ssh-keygen -A + +# It enables execution of "sudo service ssh start" without password +RUN sh -c "echo postgres ALL=NOPASSWD:/usr/sbin/service ssh start" >> /etc/sudoers + +USER postgres + +ENV LANG=C.UTF-8 + +#ENTRYPOINT PYTHON_VERSION=3.12 /run.sh +ENTRYPOINT sh -c " \ +#set -eux; \ +echo HELLO FROM ENTRYPOINT; \ +echo HOME DIR IS [`realpath ~/`]; \ +echo POINT 1; \ +chmod go-w /var/lib/postgresql; \ +echo POINT 1.5; \ +mkdir -p ~/.ssh; \ +echo POINT 2; \ +service ssh enable; \ +echo POINT 3; \ +sudo service ssh start; \ +echo POINT 4; \ +ssh-keyscan -H localhost >> ~/.ssh/known_hosts; \ +echo POINT 5; \ +ssh-keyscan -H 127.0.0.1 >> ~/.ssh/known_hosts; \ +echo POINT 6; \ +ssh-keygen -t rsa -f ~/.ssh/id_rsa -q -N ''; \ +echo ----; \ +cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys; \ +echo ----; \ +chmod 600 ~/.ssh/authorized_keys; \ +echo ----; \ +ls -la ~/.ssh/; \ +echo ----; \ +TEST_FILTER="" PYTHON_VERSION=${PYTHON_VERSION} bash run_tests.sh;" diff --git a/mk_dockerfile.sh b/mk_dockerfile.sh index d2aa3a8a..8f7876a3 100755 --- a/mk_dockerfile.sh +++ b/mk_dockerfile.sh @@ -1,2 +1,2 @@ set -eu -sed -e 's/${PYTHON_VERSION}/'${PYTHON_VERSION}/g -e 's/${PG_VERSION}/'${PG_VERSION}/g Dockerfile.tmpl > Dockerfile +sed -e 's/${PYTHON_VERSION}/'${PYTHON_VERSION}/g -e 's/${PG_VERSION}/'${PG_VERSION}/g Dockerfile--${TEST_PLATFORM}.tmpl > Dockerfile diff --git a/run_tests.sh b/run_tests.sh index 5cbbac60..021f9d9f 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -4,6 +4,7 @@ set -eux +if [ -z ${TEST_FILTER+x} ]; then export TEST_FILTER="TestgresTests"; fi # choose python version echo python version is $PYTHON_VERSION @@ -38,19 +39,19 @@ rm -f $COVERAGE_FILE # run tests (PATH) -time coverage run -a -m pytest -l -v -n 4 -k "TestgresTests" +time coverage run -a -m pytest -l -v -n 4 -k "${TEST_FILTER}" # run tests (PG_BIN) time \ PG_BIN=$(pg_config --bindir) \ - coverage run -a -m pytest -l -v -n 4 -k "TestgresTests" + coverage run -a -m pytest -l -v -n 4 -k "${TEST_FILTER}" # run tests (PG_CONFIG) time \ PG_CONFIG=$(pg_config --bindir)/pg_config \ - coverage run -a -m pytest -l -v -n 4 -k "TestgresTests" + coverage run -a -m pytest -l -v -n 4 -k "${TEST_FILTER}" # show coverage From 3a1d08b3156b56609d02568d34771a6a9a535aa8 Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Tue, 4 Mar 2025 07:06:08 +0300 Subject: [PATCH 149/216] RemoteOperations::path_exists is updated (#206) - command is passed through list - we process all the result codes of test --- testgres/operations/remote_ops.py | 26 ++++++++++++++++++++++++-- tests/test_remote.py | 26 ++++++++++++++++---------- 2 files changed, 40 insertions(+), 12 deletions(-) diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index dc392bee..60d5265c 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -247,8 +247,30 @@ def listdir(self, path): return result.splitlines() def path_exists(self, path): - result = self.exec_command("test -e {}; echo $?".format(path), encoding=get_default_encoding()) - return int(result.strip()) == 0 + command = ["test", "-e", path] + + exit_status, output, error = self.exec_command(cmd=command, encoding=get_default_encoding(), ignore_errors=True, verbose=True) + + assert type(output) == str # noqa: E721 + assert type(error) == str # noqa: E721 + + if exit_status == 0: + return True + + if exit_status == 1: + return False + + errMsg = "Test operation returns an unknown result code: {0}. Path is [{1}].".format( + exit_status, + path) + + RaiseError.CommandExecutionError( + cmd=command, + exit_code=exit_status, + msg_arg=errMsg, + error=error, + out=output + ) @property def pathsep(self): diff --git a/tests/test_remote.py b/tests/test_remote.py index 6114e29e..85e65c24 100755 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -130,23 +130,29 @@ def test_listdir(self): assert isinstance(files, list) - def test_path_exists_true(self): + def test_path_exists_true__directory(self): """ - Test path_exists for an existing path. + Test path_exists for an existing directory. """ - path = "/etc" - response = self.operations.path_exists(path) + assert self.operations.path_exists("/etc") is True - assert response is True + def test_path_exists_true__file(self): + """ + Test path_exists for an existing file. + """ + assert self.operations.path_exists(__file__) is True - def test_path_exists_false(self): + def test_path_exists_false__directory(self): """ - Test path_exists for a non-existing path. + Test path_exists for a non-existing directory. """ - path = "/nonexistent_path" - response = self.operations.path_exists(path) + assert self.operations.path_exists("/nonexistent_path") is False - assert response is False + def test_path_exists_false__file(self): + """ + Test path_exists for a non-existing file. + """ + assert self.operations.path_exists("/etc/nonexistent_path.txt") is False def test_write_text_file(self): """ From ddcaea03a1d69cca22a172aeedbef79ba82f5d40 Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Tue, 4 Mar 2025 10:56:30 +0300 Subject: [PATCH 150/216] os_ops::rmdirs (local, remote) was refactored (#207) * os_ops::rmdirs (local, remote) was refactored LocalOperations::rmdirs - parameter 'retries' was remaned with 'attempts' - if ignore_errors we raise an error RemoteOperations::rmdirs - parameter 'verbose' was removed - method returns bool - we prevent to delete a file * [TestRemoteOperations] New tests for rmdirs are added. * test_pg_ctl_wait_option (local, remote) is corrected --- testgres/operations/local_ops.py | 41 ++++++++++---- testgres/operations/remote_ops.py | 42 +++++++++++--- tests/test_remote.py | 92 +++++++++++++++++++++++++++---- tests/test_simple.py | 65 ++++++++++++++++++---- tests/test_simple_remote.py | 65 ++++++++++++++++++---- 5 files changed, 255 insertions(+), 50 deletions(-) diff --git a/testgres/operations/local_ops.py b/testgres/operations/local_ops.py index 51003174..0fa7d0ad 100644 --- a/testgres/operations/local_ops.py +++ b/testgres/operations/local_ops.py @@ -174,7 +174,8 @@ def makedirs(self, path, remove_existing=False): except FileExistsError: pass - def rmdirs(self, path, ignore_errors=True, retries=3, delay=1): + # [2025-02-03] Old name of parameter attempts is "retries". + def rmdirs(self, path, ignore_errors=True, attempts=3, delay=1): """ Removes a directory and its contents, retrying on failure. @@ -183,18 +184,38 @@ def rmdirs(self, path, ignore_errors=True, retries=3, delay=1): :param retries: Number of attempts to remove the directory. :param delay: Delay between attempts in seconds. """ - for attempt in range(retries): + assert type(path) == str # noqa: E721 + assert type(ignore_errors) == bool # noqa: E721 + assert type(attempts) == int # noqa: E721 + assert type(delay) == int or type(delay) == float # noqa: E721 + assert attempts > 0 + assert delay >= 0 + + attempt = 0 + while True: + assert attempt < attempts + attempt += 1 try: - rmtree(path, ignore_errors=ignore_errors) - if not os.path.exists(path): - return True + rmtree(path) except FileNotFoundError: - return True + pass except Exception as e: - logging.error(f"Error: Failed to remove directory {path} on attempt {attempt + 1}: {e}") - time.sleep(delay) - logging.error(f"Error: Failed to remove directory {path} after {retries} attempts.") - return False + if attempt < attempt: + errMsg = "Failed to remove directory {0} on attempt {1} ({2}): {3}".format( + path, attempt, type(e).__name__, e + ) + logging.warning(errMsg) + time.sleep(delay) + continue + + assert attempt == attempts + if not ignore_errors: + raise + + return False + + # OK! + return True def listdir(self, path): return os.listdir(path) diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index 60d5265c..767df567 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -4,6 +4,7 @@ import subprocess import tempfile import io +import logging # we support both pg8000 and psycopg2 try: @@ -222,20 +223,45 @@ def makedirs(self, path, remove_existing=False): raise Exception("Couldn't create dir {} because of error {}".format(path, error)) return result - def rmdirs(self, path, verbose=False, ignore_errors=True): + def rmdirs(self, path, ignore_errors=True): """ Remove a directory in the remote server. Args: - path (str): The path to the directory to be removed. - - verbose (bool): If True, return exit status, result, and error. - ignore_errors (bool): If True, do not raise error if directory does not exist. """ - cmd = "rm -rf {}".format(path) - exit_status, result, error = self.exec_command(cmd, verbose=True) - if verbose: - return exit_status, result, error - else: - return result + assert type(path) == str # noqa: E721 + assert type(ignore_errors) == bool # noqa: E721 + + # ENOENT = 2 - No such file or directory + # ENOTDIR = 20 - Not a directory + + cmd1 = [ + "if", "[", "-d", path, "]", ";", + "then", "rm", "-rf", path, ";", + "elif", "[", "-e", path, "]", ";", + "then", "{", "echo", "cannot remove '" + path + "': it is not a directory", ">&2", ";", "exit", "20", ";", "}", ";", + "else", "{", "echo", "directory '" + path + "' does not exist", ">&2", ";", "exit", "2", ";", "}", ";", + "fi" + ] + + cmd2 = ["sh", "-c", subprocess.list2cmdline(cmd1)] + + try: + self.exec_command(cmd2, encoding=Helpers.GetDefaultEncoding()) + except ExecUtilException as e: + if e.exit_code == 2: # No such file or directory + return True + + if not ignore_errors: + raise + + errMsg = "Failed to remove directory {0} ({1}): {2}".format( + path, type(e).__name__, e + ) + logging.warning(errMsg) + return False + return True def listdir(self, path): """ diff --git a/tests/test_remote.py b/tests/test_remote.py index 85e65c24..b1c4e58c 100755 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -99,9 +99,9 @@ def test_makedirs_and_rmdirs_success(self): assert not os.path.exists(path) assert not self.operations.path_exists(path) - def test_makedirs_and_rmdirs_failure(self): + def test_makedirs_failure(self): """ - Test makedirs and rmdirs for directory creation and removal failure. + Test makedirs for failure. """ # Try to create a directory in a read-only location path = "/root/test_dir" @@ -110,16 +110,84 @@ def test_makedirs_and_rmdirs_failure(self): with pytest.raises(Exception): self.operations.makedirs(path) - # Test rmdirs - while True: - try: - self.operations.rmdirs(path, verbose=True) - except ExecUtilException as e: - assert e.message == "Utility exited with non-zero code (1). Error: `rm: cannot remove '/root/test_dir': Permission denied`" - assert type(e.error) == bytes # noqa: E721 - assert e.error.strip() == b"rm: cannot remove '/root/test_dir': Permission denied" - break - raise Exception("We wait an exception!") + def test_rmdirs(self): + path = self.operations.mkdtemp() + assert os.path.exists(path) + + assert self.operations.rmdirs(path, ignore_errors=False) is True + assert not os.path.exists(path) + + def test_rmdirs__01_with_subfolder(self): + # folder with subfolder + path = self.operations.mkdtemp() + assert os.path.exists(path) + + dir1 = os.path.join(path, "dir1") + assert not os.path.exists(dir1) + + self.operations.makedirs(dir1) + assert os.path.exists(dir1) + + assert self.operations.rmdirs(path, ignore_errors=False) is True + assert not os.path.exists(path) + assert not os.path.exists(dir1) + + def test_rmdirs__02_with_file(self): + # folder with file + path = self.operations.mkdtemp() + assert os.path.exists(path) + + file1 = os.path.join(path, "file1.txt") + assert not os.path.exists(file1) + + self.operations.touch(file1) + assert os.path.exists(file1) + + assert self.operations.rmdirs(path, ignore_errors=False) is True + assert not os.path.exists(path) + assert not os.path.exists(file1) + + def test_rmdirs__03_with_subfolder_and_file(self): + # folder with subfolder and file + path = self.operations.mkdtemp() + assert os.path.exists(path) + + dir1 = os.path.join(path, "dir1") + assert not os.path.exists(dir1) + + self.operations.makedirs(dir1) + assert os.path.exists(dir1) + + file1 = os.path.join(dir1, "file1.txt") + assert not os.path.exists(file1) + + self.operations.touch(file1) + assert os.path.exists(file1) + + assert self.operations.rmdirs(path, ignore_errors=False) is True + assert not os.path.exists(path) + assert not os.path.exists(dir1) + assert not os.path.exists(file1) + + def test_rmdirs__try_to_delete_nonexist_path(self): + path = "/root/test_dir" + + assert self.operations.rmdirs(path, ignore_errors=False) is True + + def test_rmdirs__try_to_delete_file(self): + path = self.operations.mkstemp() + assert os.path.exists(path) + + with pytest.raises(ExecUtilException) as x: + self.operations.rmdirs(path, ignore_errors=False) + + assert os.path.exists(path) + assert type(x.value) == ExecUtilException # noqa: E721 + assert x.value.message == "Utility exited with non-zero code (20). Error: `cannot remove '" + path + "': it is not a directory`" + assert type(x.value.error) == str # noqa: E721 + assert x.value.error.strip() == "cannot remove '" + path + "': it is not a directory" + assert type(x.value.exit_code) == int # noqa: E721 + assert x.value.exit_code == 20 def test_listdir(self): """ diff --git a/tests/test_simple.py b/tests/test_simple.py index 37c3db44..d9844fed 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -428,16 +428,61 @@ def test_backup_wrong_xlog_method(self): node.backup(xlog_method='wrong') def test_pg_ctl_wait_option(self): - with get_new_node() as node: - node.init().start(wait=False) - while True: - try: - node.stop(wait=False) - break - except ExecUtilException: - # it's ok to get this exception here since node - # could be not started yet - pass + C_MAX_ATTEMPTS = 50 + + node = get_new_node() + assert node.status() == testgres.NodeStatus.Uninitialized + node.init() + assert node.status() == testgres.NodeStatus.Stopped + node.start(wait=False) + nAttempt = 0 + while True: + if nAttempt == C_MAX_ATTEMPTS: + raise Exception("Could not stop node.") + + nAttempt += 1 + + if nAttempt > 1: + logging.info("Wait 1 second.") + time.sleep(1) + logging.info("") + + logging.info("Try to stop node. Attempt #{0}.".format(nAttempt)) + + try: + node.stop(wait=False) + break + except ExecUtilException as e: + # it's ok to get this exception here since node + # could be not started yet + logging.info("Node is not stopped. Exception ({0}): {1}".format(type(e).__name__, e)) + continue + + logging.info("OK. Stop command was executed. Let's wait while our node will stop really.") + nAttempt = 0 + while True: + if nAttempt == C_MAX_ATTEMPTS: + raise Exception("Could not stop node.") + + nAttempt += 1 + if nAttempt > 1: + logging.info("Wait 1 second.") + time.sleep(1) + logging.info("") + + logging.info("Attempt #{0}.".format(nAttempt)) + s1 = node.status() + + if s1 == testgres.NodeStatus.Running: + continue + + if s1 == testgres.NodeStatus.Stopped: + break + + raise Exception("Unexpected node status: {0}.".format(s1)) + + logging.info("OK. Node is stopped.") + node.cleanup() def test_replicate(self): with get_new_node() as node: diff --git a/tests/test_simple_remote.py b/tests/test_simple_remote.py index a62085ce..42527dbc 100755 --- a/tests/test_simple_remote.py +++ b/tests/test_simple_remote.py @@ -499,16 +499,61 @@ def test_backup_wrong_xlog_method(self): node.backup(xlog_method='wrong') def test_pg_ctl_wait_option(self): - with __class__.helper__get_node() as node: - node.init().start(wait=False) - while True: - try: - node.stop(wait=False) - break - except ExecUtilException: - # it's ok to get this exception here since node - # could be not started yet - pass + C_MAX_ATTEMPTS = 50 + + node = __class__.helper__get_node() + assert node.status() == testgres.NodeStatus.Uninitialized + node.init() + assert node.status() == testgres.NodeStatus.Stopped + node.start(wait=False) + nAttempt = 0 + while True: + if nAttempt == C_MAX_ATTEMPTS: + raise Exception("Could not stop node.") + + nAttempt += 1 + + if nAttempt > 1: + logging.info("Wait 1 second.") + time.sleep(1) + logging.info("") + + logging.info("Try to stop node. Attempt #{0}.".format(nAttempt)) + + try: + node.stop(wait=False) + break + except ExecUtilException as e: + # it's ok to get this exception here since node + # could be not started yet + logging.info("Node is not stopped. Exception ({0}): {1}".format(type(e).__name__, e)) + continue + + logging.info("OK. Stop command was executed. Let's wait while our node will stop really.") + nAttempt = 0 + while True: + if nAttempt == C_MAX_ATTEMPTS: + raise Exception("Could not stop node.") + + nAttempt += 1 + if nAttempt > 1: + logging.info("Wait 1 second.") + time.sleep(1) + logging.info("") + + logging.info("Attempt #{0}.".format(nAttempt)) + s1 = node.status() + + if s1 == testgres.NodeStatus.Running: + continue + + if s1 == testgres.NodeStatus.Stopped: + break + + raise Exception("Unexpected node status: {0}.".format(s1)) + + logging.info("OK. Node is stopped.") + node.cleanup() def test_replicate(self): with __class__.helper__get_node() as node: From e47cded2a784bfbc64c9a70aceedfe26f9a3b802 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 11 Mar 2025 10:27:59 +0300 Subject: [PATCH 151/216] [remote_ops] A problem with mktemp on Alpine Linux is fixed Five 'X' in template is not enough - Alpine returns "mktemp: : Invalid argument" error. Six 'X' is OK. --- testgres/operations/remote_ops.py | 4 ++-- tests/test_local.py | 17 +++++++++++++++++ tests/test_remote.py | 17 +++++++++++++++++ 3 files changed, 36 insertions(+), 2 deletions(-) diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index 767df567..11d9cd37 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -316,7 +316,7 @@ def mkdtemp(self, prefix=None): - prefix (str): The prefix of the temporary directory name. """ if prefix: - command = ["mktemp", "-d", "-t", prefix + "XXXXX"] + command = ["mktemp", "-d", "-t", prefix + "XXXXXX"] else: command = ["mktemp", "-d"] @@ -344,7 +344,7 @@ def mkstemp(self, prefix=None): - prefix (str): The prefix of the temporary directory name. """ if prefix: - command = ["mktemp", "-t", prefix + "XXXXX"] + command = ["mktemp", "-t", prefix + "XXXXXX"] else: command = ["mktemp"] diff --git a/tests/test_local.py b/tests/test_local.py index ee5e19a0..826c3f51 100644 --- a/tests/test_local.py +++ b/tests/test_local.py @@ -4,6 +4,7 @@ import pytest import re import tempfile +import logging from ..testgres import ExecUtilException from ..testgres import InvalidOperationException @@ -18,6 +19,22 @@ class TestLocalOperations: def setup(self): self.operations = LocalOperations() + def test_mkdtemp__default(self): + path = self.operations.mkdtemp() + logging.info("Path is [{0}].".format(path)) + assert os.path.exists(path) + os.rmdir(path) + assert not os.path.exists(path) + + def test_mkdtemp__custom(self): + C_TEMPLATE = "abcdef" + path = self.operations.mkdtemp(C_TEMPLATE) + logging.info("Path is [{0}].".format(path)) + assert os.path.exists(path) + assert C_TEMPLATE in os.path.basename(path) + os.rmdir(path) + assert not os.path.exists(path) + def test_exec_command_success(self): """ Test exec_command for successful command execution. diff --git a/tests/test_remote.py b/tests/test_remote.py index b1c4e58c..17c76c2c 100755 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -4,6 +4,7 @@ import pytest import re import tempfile +import logging from ..testgres import ExecUtilException from ..testgres import InvalidOperationException @@ -110,6 +111,22 @@ def test_makedirs_failure(self): with pytest.raises(Exception): self.operations.makedirs(path) + def test_mkdtemp__default(self): + path = self.operations.mkdtemp() + logging.info("Path is [{0}].".format(path)) + assert os.path.exists(path) + os.rmdir(path) + assert not os.path.exists(path) + + def test_mkdtemp__custom(self): + C_TEMPLATE = "abcdef" + path = self.operations.mkdtemp(C_TEMPLATE) + logging.info("Path is [{0}].".format(path)) + assert os.path.exists(path) + assert C_TEMPLATE in os.path.basename(path) + os.rmdir(path) + assert not os.path.exists(path) + def test_rmdirs(self): path = self.operations.mkdtemp() assert os.path.exists(path) From cf6f4cc6f6111a07302e4f6644baccbf58bb158a Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 11 Mar 2025 11:38:34 +0300 Subject: [PATCH 152/216] [remote] Tests are updated to support Alpine Linux --- tests/test_local.py | 16 +++++++++++++--- tests/test_remote.py | 36 ++++++++++++++++++++++++++++-------- tests/test_simple_remote.py | 10 +++++++++- 3 files changed, 50 insertions(+), 12 deletions(-) diff --git a/tests/test_local.py b/tests/test_local.py index 826c3f51..68e7db33 100644 --- a/tests/test_local.py +++ b/tests/test_local.py @@ -57,9 +57,17 @@ def test_exec_command_failure(self): try: self.operations.exec_command(cmd, wait_exit=True, shell=True) except ExecUtilException as e: - assert e.message == "Utility exited with non-zero code (127). Error: `/bin/sh: 1: nonexistent_command: not found`" + assert type(e.exit_code) == int # noqa: E721 + assert e.exit_code == 127 + + assert type(e.message) == str # noqa: E721 assert type(e.error) == bytes # noqa: E721 - assert e.error.strip() == b"/bin/sh: 1: nonexistent_command: not found" + + assert e.message.startswith("Utility exited with non-zero code (127). Error:") + assert "nonexistent_command" in e.message + assert "not found" in e.message + assert b"nonexistent_command" in e.error + assert b"not found" in e.error break raise Exception("We wait an exception!") @@ -73,9 +81,11 @@ def test_exec_command_failure__expect_error(self): exit_status, result, error = self.operations.exec_command(cmd, verbose=True, wait_exit=True, shell=True, expect_error=True) - assert error == b'/bin/sh: 1: nonexistent_command: not found\n' assert exit_status == 127 assert result == b'' + assert type(error) == bytes # noqa: E721 + assert b"nonexistent_command" in error + assert b"not found" in error def test_read__text(self): """ diff --git a/tests/test_remote.py b/tests/test_remote.py index 17c76c2c..1f771c62 100755 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -41,9 +41,17 @@ def test_exec_command_failure(self): try: self.operations.exec_command(cmd, verbose=True, wait_exit=True) except ExecUtilException as e: - assert e.message == "Utility exited with non-zero code (127). Error: `bash: line 1: nonexistent_command: command not found`" + assert type(e.exit_code) == int # noqa: E721 + assert e.exit_code == 127 + + assert type(e.message) == str # noqa: E721 assert type(e.error) == bytes # noqa: E721 - assert e.error.strip() == b"bash: line 1: nonexistent_command: command not found" + + assert e.message.startswith("Utility exited with non-zero code (127). Error:") + assert "nonexistent_command" in e.message + assert "not found" in e.message + assert b"nonexistent_command" in e.error + assert b"not found" in e.error break raise Exception("We wait an exception!") @@ -55,9 +63,11 @@ def test_exec_command_failure__expect_error(self): exit_status, result, error = self.operations.exec_command(cmd, verbose=True, wait_exit=True, shell=True, expect_error=True) - assert error == b'bash: line 1: nonexistent_command: command not found\n' assert exit_status == 127 assert result == b'' + assert type(error) == bytes # noqa: E721 + assert b"nonexistent_command" in error + assert b"not found" in error def test_is_executable_true(self): """ @@ -344,11 +354,13 @@ def test_read__unknown_file(self): Test RemoteOperations::read with unknown file. """ - with pytest.raises( - ExecUtilException, - match=re.escape("cat: /dummy: No such file or directory")): + with pytest.raises(ExecUtilException) as x: self.operations.read("/dummy") + assert "Utility exited with non-zero code (1)." in str(x.value) + assert "No such file or directory" in str(x.value) + assert "/dummy" in str(x.value) + def test_read_binary__spec(self): """ Test RemoteOperations::read_binary. @@ -388,9 +400,13 @@ def test_read_binary__spec__unk_file(self): Test RemoteOperations::read_binary with unknown file. """ - with pytest.raises(ExecUtilException, match=re.escape("tail: cannot open '/dummy' for reading: No such file or directory")): + with pytest.raises(ExecUtilException) as x: self.operations.read_binary("/dummy", 0) + assert "Utility exited with non-zero code (1)." in str(x.value) + assert "No such file or directory" in str(x.value) + assert "/dummy" in str(x.value) + def test_read_binary__spec__negative_offset(self): """ Test RemoteOperations::read_binary with negative offset. @@ -419,9 +435,13 @@ def test_get_file_size__unk_file(self): Test RemoteOperations::get_file_size. """ - with pytest.raises(ExecUtilException, match=re.escape("du: cannot access '/dummy': No such file or directory")): + with pytest.raises(ExecUtilException) as x: self.operations.get_file_size("/dummy") + assert "Utility exited with non-zero code (1)." in str(x.value) + assert "No such file or directory" in str(x.value) + assert "/dummy" in str(x.value) + def test_touch(self): """ Test touch for creating a new file or updating access and modification times of an existing file. diff --git a/tests/test_simple_remote.py b/tests/test_simple_remote.py index 42527dbc..cdad161c 100755 --- a/tests/test_simple_remote.py +++ b/tests/test_simple_remote.py @@ -163,6 +163,8 @@ def test_init__unk_LANG_and_LC_CTYPE(self): ("\"", "\""), ] + errorIsDetected = False + for unkData in unkDatas: logging.info("----------------------") logging.info("Unk LANG is [{0}]".format(unkData[0])) @@ -193,7 +195,10 @@ def test_init__unk_LANG_and_LC_CTYPE(self): assert isinstance(exc, ExecUtilException) if exc is None: - raise Exception("We expected an error!") + logging.warning("We expected an error!") + continue + + errorIsDetected = True assert isinstance(exc, ExecUtilException) @@ -204,6 +209,9 @@ def test_init__unk_LANG_and_LC_CTYPE(self): assert "initdb: error: invalid locale settings; check LANG and LC_* environment variables" in errMsg continue + if not errorIsDetected: + pytest.xfail("All the bad data are processed without errors!") + finally: __class__.helper__restore_envvar("LANG", prev_LANG) __class__.helper__restore_envvar("LANGUAGE", prev_LANGUAGE) From 44f280bc7aa51cce5374fb19c1c234b379e4fbf0 Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Tue, 11 Mar 2025 18:19:12 +0300 Subject: [PATCH 153/216] [CI] A run of all tests (local, remote) on Alpine Linux [PY3, PG17] is added (#210) * Dockerfile to run all tests on Alpine Linux * A run of all the tests on Alpine Linux [PY3, PG17] is added * [CI] The run of [STD, PY3, PG17] is removed It was replaced with [ALPINE, PY3, PG17]. * Test platform "alpine" was renamed with "std.all" --- .travis.yml | 2 +- Dockerfile--std.all.tmpl | 60 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 1 deletion(-) create mode 100644 Dockerfile--std.all.tmpl diff --git a/.travis.yml b/.travis.yml index 4110835a..7fb34808 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,7 +20,6 @@ notifications: on_failure: always env: - - TEST_PLATFORM=std PYTHON_VERSION=3 PG_VERSION=17 - TEST_PLATFORM=std PYTHON_VERSION=3 PG_VERSION=16 - TEST_PLATFORM=std PYTHON_VERSION=3 PG_VERSION=15 - TEST_PLATFORM=std PYTHON_VERSION=3 PG_VERSION=14 @@ -28,6 +27,7 @@ env: - TEST_PLATFORM=std PYTHON_VERSION=3 PG_VERSION=12 - TEST_PLATFORM=std PYTHON_VERSION=3 PG_VERSION=11 - TEST_PLATFORM=std PYTHON_VERSION=3 PG_VERSION=10 + - TEST_PLATFORM=std.all PYTHON_VERSION=3 PG_VERSION=17 - TEST_PLATFORM=ubuntu-24_04 PYTHON_VERSION=3 PG_VERSION=17 matrix: diff --git a/Dockerfile--std.all.tmpl b/Dockerfile--std.all.tmpl new file mode 100644 index 00000000..dfd9ab20 --- /dev/null +++ b/Dockerfile--std.all.tmpl @@ -0,0 +1,60 @@ +FROM postgres:${PG_VERSION}-alpine + +ENV PYTHON=python${PYTHON_VERSION} +RUN if [ "${PYTHON_VERSION}" = "2" ] ; then \ + apk add --no-cache curl python2 python2-dev build-base musl-dev \ + linux-headers py-virtualenv py-pip; \ + fi +RUN if [ "${PYTHON_VERSION}" = "3" ] ; then \ + apk add --no-cache curl python3 python3-dev build-base musl-dev \ + linux-headers py-virtualenv; \ + fi + +#RUN apk add --no-cache mc + +# Full version of "ps" command +RUN apk add --no-cache procps + +RUN apk add --no-cache openssh +RUN apk add --no-cache sudo + +ENV LANG=C.UTF-8 + +RUN addgroup -S sudo +RUN adduser postgres sudo + +EXPOSE 22 +RUN ssh-keygen -A + +ADD . /pg/testgres +WORKDIR /pg/testgres +RUN chown -R postgres:postgres /pg + +# It allows to use sudo without password +RUN sh -c "echo \"postgres ALL=(ALL:ALL) NOPASSWD:ALL\"">>/etc/sudoers + +# THIS CMD IS NEEDED TO CONNECT THROUGH SSH WITHOUT PASSWORD +RUN sh -c "echo "postgres:*" | chpasswd -e" + +USER postgres + +# THIS CMD IS NEEDED TO CONNECT THROUGH SSH WITHOUT PASSWORD +RUN chmod 700 ~/ + +RUN mkdir -p ~/.ssh +#RUN chmod 700 ~/.ssh + +#ENTRYPOINT PYTHON_VERSION=${PYTHON_VERSION} bash run_tests.sh + +ENTRYPOINT sh -c " \ +set -eux; \ +echo HELLO FROM ENTRYPOINT; \ +echo HOME DIR IS [`realpath ~/`]; \ +ssh-keygen -t rsa -f ~/.ssh/id_rsa -q -N ''; \ +cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys; \ +chmod 600 ~/.ssh/authorized_keys; \ +ls -la ~/.ssh/; \ +sudo /usr/sbin/sshd; \ +ssh-keyscan -H localhost >> ~/.ssh/known_hosts; \ +ssh-keyscan -H 127.0.0.1 >> ~/.ssh/known_hosts; \ +TEST_FILTER=\"\" PYTHON_VERSION=${PYTHON_VERSION} bash run_tests.sh;" From 600572857f100c37218d29ec5c4d6fa8c6d9d6c6 Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Tue, 11 Mar 2025 21:00:07 +0300 Subject: [PATCH 154/216] PsUtilProcessProxy is updated (refactoring) (#212) --- testgres/operations/remote_ops.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index 11d9cd37..d6917c82 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -26,17 +26,25 @@ class PsUtilProcessProxy: def __init__(self, ssh, pid): + assert isinstance(ssh, RemoteOperations) + assert type(pid) == int # noqa: E721 self.ssh = ssh self.pid = pid def kill(self): - command = "kill {}".format(self.pid) - self.ssh.exec_command(command) + assert isinstance(self.ssh, RemoteOperations) + assert type(self.pid) == int # noqa: E721 + command = ["kill", str(self.pid)] + self.ssh.exec_command(command, encoding=get_default_encoding()) def cmdline(self): - command = "ps -p {} -o cmd --no-headers".format(self.pid) - stdin, stdout, stderr = self.ssh.exec_command(command, verbose=True, encoding=get_default_encoding()) - cmdline = stdout.strip() + assert isinstance(self.ssh, RemoteOperations) + assert type(self.pid) == int # noqa: E721 + command = ["ps", "-p", str(self.pid), "-o", "cmd", "--no-headers"] + output = self.ssh.exec_command(command, encoding=get_default_encoding()) + assert type(output) == str # noqa: E721 + cmdline = output.strip() + # TODO: This code work wrong if command line contains quoted values. Yes? return cmdline.split() From 1a4655cb9011c3a01cd0e7d56606ba36fff40e43 Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Tue, 11 Mar 2025 21:46:10 +0300 Subject: [PATCH 155/216] Dockerfile for Ubuntu 2024.04 is updated (#211) * Dockerfile for Ubuntu 2024.04 updated - Using "chmod 700 ~/" instead "chmod go-w /var/lib/postgresql" - "~/.ssh" folder is prepared at image level - Entrypoint startup code is cleaned from trace messages * Dockerfile--ubuntu-24_04.tmpl is updated (double quotes) --- Dockerfile--ubuntu-24_04.tmpl | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/Dockerfile--ubuntu-24_04.tmpl b/Dockerfile--ubuntu-24_04.tmpl index 99be5343..fd1136d8 100644 --- a/Dockerfile--ubuntu-24_04.tmpl +++ b/Dockerfile--ubuntu-24_04.tmpl @@ -37,33 +37,22 @@ RUN ssh-keygen -A RUN sh -c "echo postgres ALL=NOPASSWD:/usr/sbin/service ssh start" >> /etc/sudoers USER postgres - ENV LANG=C.UTF-8 +RUN chmod 700 ~/ +RUN mkdir -p ~/.ssh + #ENTRYPOINT PYTHON_VERSION=3.12 /run.sh ENTRYPOINT sh -c " \ #set -eux; \ echo HELLO FROM ENTRYPOINT; \ echo HOME DIR IS [`realpath ~/`]; \ -echo POINT 1; \ -chmod go-w /var/lib/postgresql; \ -echo POINT 1.5; \ -mkdir -p ~/.ssh; \ -echo POINT 2; \ service ssh enable; \ -echo POINT 3; \ sudo service ssh start; \ -echo POINT 4; \ ssh-keyscan -H localhost >> ~/.ssh/known_hosts; \ -echo POINT 5; \ ssh-keyscan -H 127.0.0.1 >> ~/.ssh/known_hosts; \ -echo POINT 6; \ ssh-keygen -t rsa -f ~/.ssh/id_rsa -q -N ''; \ -echo ----; \ cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys; \ -echo ----; \ chmod 600 ~/.ssh/authorized_keys; \ -echo ----; \ ls -la ~/.ssh/; \ -echo ----; \ -TEST_FILTER="" PYTHON_VERSION=${PYTHON_VERSION} bash run_tests.sh;" +TEST_FILTER=\"\" PYTHON_VERSION=${PYTHON_VERSION} bash run_tests.sh;" From 438e84508c7c0b3d9f05336ffdd1df44ba23c36b Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Wed, 12 Mar 2025 08:48:16 +0300 Subject: [PATCH 156/216] test_pg_ctl_wait_option is updated (#213) When node is not stopped, we read and output a content of node log file to provide an additional information about this problem. It should help find a reason of unexpected problem with this test in CI. --- tests/test_simple.py | 13 +++++++++++++ tests/test_simple_remote.py | 13 +++++++++++++ 2 files changed, 26 insertions(+) diff --git a/tests/test_simple.py b/tests/test_simple.py index d9844fed..e13cf095 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -438,6 +438,19 @@ def test_pg_ctl_wait_option(self): nAttempt = 0 while True: if nAttempt == C_MAX_ATTEMPTS: + # + # [2025-03-11] + # We have an unexpected problem with this test in CI + # Let's get an additional information about this test failure. + # + logging.error("Node was not stopped.") + if not node.os_ops.path_exists(node.pg_log_file): + logging.warning("Node log does not exist.") + else: + logging.info("Let's read node log file [{0}]".format(node.pg_log_file)) + logFileData = node.os_ops.read(node.pg_log_file, binary=False) + logging.info("Node log file content:\n{0}".format(logFileData)) + raise Exception("Could not stop node.") nAttempt += 1 diff --git a/tests/test_simple_remote.py b/tests/test_simple_remote.py index cdad161c..d484f1e3 100755 --- a/tests/test_simple_remote.py +++ b/tests/test_simple_remote.py @@ -517,6 +517,19 @@ def test_pg_ctl_wait_option(self): nAttempt = 0 while True: if nAttempt == C_MAX_ATTEMPTS: + # + # [2025-03-11] + # We have an unexpected problem with this test in CI + # Let's get an additional information about this test failure. + # + logging.error("Node was not stopped.") + if not node.os_ops.path_exists(node.pg_log_file): + logging.warning("Node log does not exist.") + else: + logging.info("Let's read node log file [{0}]".format(node.pg_log_file)) + logFileData = node.os_ops.read(node.pg_log_file, binary=False) + logging.info("Node log file content:\n{0}".format(logFileData)) + raise Exception("Could not stop node.") nAttempt += 1 From f2c000c28e4f250c5dc319be557217d8433306cd Mon Sep 17 00:00:00 2001 From: Victoria Shepard <5807469+demonolock@users.noreply.github.com> Date: Wed, 12 Mar 2025 06:52:03 +0100 Subject: [PATCH 157/216] Fix auto conf test (#214) * Fix test_set_auto_conf for Postgresql 10, 11 * Remove allow failures --------- Co-authored-by: vshepard --- .travis.yml | 5 ----- tests/test_simple.py | 7 ++++--- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7fb34808..997945b5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,8 +29,3 @@ env: - TEST_PLATFORM=std PYTHON_VERSION=3 PG_VERSION=10 - TEST_PLATFORM=std.all PYTHON_VERSION=3 PG_VERSION=17 - TEST_PLATFORM=ubuntu-24_04 PYTHON_VERSION=3 PG_VERSION=17 - -matrix: - allow_failures: - - env: TEST_PLATFORM=std PYTHON_VERSION=3 PG_VERSION=11 - - env: TEST_PLATFORM=std PYTHON_VERSION=3 PG_VERSION=10 diff --git a/tests/test_simple.py b/tests/test_simple.py index e13cf095..e886a39c 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -1464,9 +1464,6 @@ def test_set_auto_conf(self): ["archive_command", "cp '%p' \"/mnt/server/archivedir/%f\"", "'cp \\'%p\\' \"/mnt/server/archivedir/%f\""], - ["restore_command", - 'cp "/mnt/server/archivedir/%f" \'%p\'', - "'cp \"/mnt/server/archivedir/%f\" \\'%p\\''"], ["log_line_prefix", "'\n\r\t\b\\\"", "'\\\'\\n\\r\\t\\b\\\\\""], @@ -1480,6 +1477,10 @@ def test_set_auto_conf(self): 3, "3"] ] + if pg_version_ge('12'): + testData.append(["restore_command", + 'cp "/mnt/server/archivedir/%f" \'%p\'', + "'cp \"/mnt/server/archivedir/%f\" \\'%p\\''"]) with get_new_node() as node: node.init().start() From e3eb6ae1a8ed63a6df516e59e406b674f4e5ea96 Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Wed, 12 Mar 2025 21:49:38 +0300 Subject: [PATCH 158/216] Refactoring of dockerfiles (#215) * Refactoring of dockerfiles Let's to try using a pure dockerfile' logic. It is the fist step. - We use docker instead docker-composer - We upgrade and use only "std" plaform. Other plaforms will be upgraded later * It is better to run a docker with "-t" options. This allows the colors in output data. * A problem with PYTHON_VERSION is fixed * Dockerfile--std.all.tmpl is updated * Dockerfile--ubuntu-24_04.tmpl * Dockerfile--std.tmpl is updated (formatting) * docker-compose.yml and mk_dockerfile.sh are deleted * [CI] Platform "std.all" was renamed with "std_all" Let's avoid using a "point" symbol in name of file. This symbol may create a problem in the future if we decide to use configuration docker files without extensions. * [CI] Platform name has the one format Dockerfiles--[-].tmpl - "std_all" -> "std-all" - "ubuntu-24_04" -> "ubuntu_24_04" * Dockerfile--ubuntu_24_04.tmpl is updated (minimization) --- .travis.yml | 9 ++-- ...--std.all.tmpl => Dockerfile--std-all.tmpl | 31 ++++++++------ Dockerfile--std.tmpl | 30 +++++++++----- ...4_04.tmpl => Dockerfile--ubuntu_24_04.tmpl | 41 ++++++++++++------- docker-compose.yml | 4 -- mk_dockerfile.sh | 2 - 6 files changed, 69 insertions(+), 48 deletions(-) rename Dockerfile--std.all.tmpl => Dockerfile--std-all.tmpl (60%) rename Dockerfile--ubuntu-24_04.tmpl => Dockerfile--ubuntu_24_04.tmpl (71%) delete mode 100644 docker-compose.yml delete mode 100755 mk_dockerfile.sh diff --git a/.travis.yml b/.travis.yml index 997945b5..3a889845 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,11 +8,10 @@ services: - docker install: - - ./mk_dockerfile.sh - - docker-compose build + - docker build --build-arg PG_VERSION="${PG_VERSION}" --build-arg PYTHON_VERSION="${PYTHON_VERSION}" -t tests -f Dockerfile--${TEST_PLATFORM}.tmpl . script: - - docker-compose run $(bash <(curl -s https://codecov.io/env)) tests + - docker run $(bash <(curl -s https://codecov.io/env)) -t tests notifications: email: @@ -27,5 +26,5 @@ env: - TEST_PLATFORM=std PYTHON_VERSION=3 PG_VERSION=12 - TEST_PLATFORM=std PYTHON_VERSION=3 PG_VERSION=11 - TEST_PLATFORM=std PYTHON_VERSION=3 PG_VERSION=10 - - TEST_PLATFORM=std.all PYTHON_VERSION=3 PG_VERSION=17 - - TEST_PLATFORM=ubuntu-24_04 PYTHON_VERSION=3 PG_VERSION=17 + - TEST_PLATFORM=std-all PYTHON_VERSION=3 PG_VERSION=17 + - TEST_PLATFORM=ubuntu_24_04 PYTHON_VERSION=3 PG_VERSION=17 diff --git a/Dockerfile--std.all.tmpl b/Dockerfile--std-all.tmpl similarity index 60% rename from Dockerfile--std.all.tmpl rename to Dockerfile--std-all.tmpl index dfd9ab20..c41c5a06 100644 --- a/Dockerfile--std.all.tmpl +++ b/Dockerfile--std-all.tmpl @@ -1,14 +1,21 @@ -FROM postgres:${PG_VERSION}-alpine - -ENV PYTHON=python${PYTHON_VERSION} -RUN if [ "${PYTHON_VERSION}" = "2" ] ; then \ - apk add --no-cache curl python2 python2-dev build-base musl-dev \ - linux-headers py-virtualenv py-pip; \ - fi -RUN if [ "${PYTHON_VERSION}" = "3" ] ; then \ - apk add --no-cache curl python3 python3-dev build-base musl-dev \ - linux-headers py-virtualenv; \ - fi +ARG PG_VERSION +ARG PYTHON_VERSION + +# --------------------------------------------- base1 +FROM postgres:${PG_VERSION}-alpine as base1 + +# --------------------------------------------- base2_with_python-2 +FROM base1 as base2_with_python-2 +RUN apk add --no-cache curl python2 python2-dev build-base musl-dev linux-headers py-virtualenv py-pip +ENV PYTHON_VERSION=2 + +# --------------------------------------------- base2_with_python-3 +FROM base1 as base2_with_python-3 +RUN apk add --no-cache curl python3 python3-dev build-base musl-dev linux-headers py-virtualenv +ENV PYTHON_VERSION=3 + +# --------------------------------------------- final +FROM base2_with_python-${PYTHON_VERSION} as final #RUN apk add --no-cache mc @@ -57,4 +64,4 @@ ls -la ~/.ssh/; \ sudo /usr/sbin/sshd; \ ssh-keyscan -H localhost >> ~/.ssh/known_hosts; \ ssh-keyscan -H 127.0.0.1 >> ~/.ssh/known_hosts; \ -TEST_FILTER=\"\" PYTHON_VERSION=${PYTHON_VERSION} bash run_tests.sh;" +TEST_FILTER=\"\" bash run_tests.sh;" diff --git a/Dockerfile--std.tmpl b/Dockerfile--std.tmpl index d844c9a3..91886ede 100644 --- a/Dockerfile--std.tmpl +++ b/Dockerfile--std.tmpl @@ -1,14 +1,22 @@ -FROM postgres:${PG_VERSION}-alpine +ARG PG_VERSION +ARG PYTHON_VERSION + +# --------------------------------------------- base1 +FROM postgres:${PG_VERSION}-alpine as base1 + +# --------------------------------------------- base2_with_python-2 +FROM base1 as base2_with_python-2 +RUN apk add --no-cache curl python2 python2-dev build-base musl-dev linux-headers py-virtualenv py-pip +ENV PYTHON_VERSION=2 + +# --------------------------------------------- base2_with_python-3 +FROM base1 as base2_with_python-3 +RUN apk add --no-cache curl python3 python3-dev build-base musl-dev linux-headers py-virtualenv +ENV PYTHON_VERSION=3 + +# --------------------------------------------- final +FROM base2_with_python-${PYTHON_VERSION} as final -ENV PYTHON=python${PYTHON_VERSION} -RUN if [ "${PYTHON_VERSION}" = "2" ] ; then \ - apk add --no-cache curl python2 python2-dev build-base musl-dev \ - linux-headers py-virtualenv py-pip; \ - fi -RUN if [ "${PYTHON_VERSION}" = "3" ] ; then \ - apk add --no-cache curl python3 python3-dev build-base musl-dev \ - linux-headers py-virtualenv; \ - fi ENV LANG=C.UTF-8 ADD . /pg/testgres @@ -16,4 +24,4 @@ WORKDIR /pg/testgres RUN chown -R postgres:postgres /pg USER postgres -ENTRYPOINT PYTHON_VERSION=${PYTHON_VERSION} bash run_tests.sh +ENTRYPOINT bash run_tests.sh diff --git a/Dockerfile--ubuntu-24_04.tmpl b/Dockerfile--ubuntu_24_04.tmpl similarity index 71% rename from Dockerfile--ubuntu-24_04.tmpl rename to Dockerfile--ubuntu_24_04.tmpl index fd1136d8..c1ddeab6 100644 --- a/Dockerfile--ubuntu-24_04.tmpl +++ b/Dockerfile--ubuntu_24_04.tmpl @@ -1,7 +1,17 @@ -FROM ubuntu:24.04 +ARG PG_VERSION +ARG PYTHON_VERSION + +# --------------------------------------------- base1 +FROM ubuntu:24.04 as base1 +ARG PG_VERSION + +RUN apt update +RUN apt install -y sudo curl ca-certificates +RUN apt update +RUN apt install -y openssh-server RUN apt update -RUN apt install -y sudo curl ca-certificates postgresql-common +RUN apt install -y postgresql-common RUN bash /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh -y @@ -14,21 +24,12 @@ RUN curl -o /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc --fail http RUN apt update RUN apt install -y postgresql-${PG_VERSION} -RUN apt install -y python3 python3-dev python3-virtualenv # RUN apt install -y mc -# It is required for psycopg2 -RUN apt install -y libpq-dev -RUN apt install -y openssh-server - # [2025-02-26] It adds the user 'postgres' in the group 'sudo' # [2025-02-27] It is not required. # RUN adduser postgres sudo -ADD . /pg/testgres -WORKDIR /pg/testgres -RUN chown -R postgres /pg - EXPOSE 22 RUN ssh-keygen -A @@ -36,13 +37,25 @@ RUN ssh-keygen -A # It enables execution of "sudo service ssh start" without password RUN sh -c "echo postgres ALL=NOPASSWD:/usr/sbin/service ssh start" >> /etc/sudoers -USER postgres +# --------------------------------------------- base2_with_python-3 +FROM base1 as base2_with_python-3 +RUN apt install -y python3 python3-dev python3-virtualenv libpq-dev +ENV PYTHON_VERSION=3 + +# --------------------------------------------- final +FROM base2_with_python-${PYTHON_VERSION} as final + +ADD . /pg/testgres +WORKDIR /pg/testgres +RUN chown -R postgres /pg + ENV LANG=C.UTF-8 +USER postgres + RUN chmod 700 ~/ RUN mkdir -p ~/.ssh -#ENTRYPOINT PYTHON_VERSION=3.12 /run.sh ENTRYPOINT sh -c " \ #set -eux; \ echo HELLO FROM ENTRYPOINT; \ @@ -55,4 +68,4 @@ ssh-keygen -t rsa -f ~/.ssh/id_rsa -q -N ''; \ cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys; \ chmod 600 ~/.ssh/authorized_keys; \ ls -la ~/.ssh/; \ -TEST_FILTER=\"\" PYTHON_VERSION=${PYTHON_VERSION} bash run_tests.sh;" +TEST_FILTER=\"\" bash ./run_tests.sh;" diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index 86edf9a4..00000000 --- a/docker-compose.yml +++ /dev/null @@ -1,4 +0,0 @@ -version: '3.8' -services: - tests: - build: . diff --git a/mk_dockerfile.sh b/mk_dockerfile.sh deleted file mode 100755 index 8f7876a3..00000000 --- a/mk_dockerfile.sh +++ /dev/null @@ -1,2 +0,0 @@ -set -eu -sed -e 's/${PYTHON_VERSION}/'${PYTHON_VERSION}/g -e 's/${PG_VERSION}/'${PG_VERSION}/g Dockerfile--${TEST_PLATFORM}.tmpl > Dockerfile From 230e5620d68ffef92f31fe67b4ed30881950847d Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Fri, 14 Mar 2025 00:10:11 +0300 Subject: [PATCH 159/216] RemoteOperations::listdir is corrected (#217) - It returns list[str] - New asserts are added Tests for listdir are updated. --- testgres/operations/remote_ops.py | 8 ++++++-- tests/test_local.py | 11 +++++++++++ tests/test_remote.py | 4 +++- 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index d6917c82..60161e3c 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -277,8 +277,12 @@ def listdir(self, path): Args: path (str): The path to the directory. """ - result = self.exec_command("ls {}".format(path)) - return result.splitlines() + command = ["ls", path] + output = self.exec_command(cmd=command, encoding=get_default_encoding()) + assert type(output) == str # noqa: E721 + result = output.splitlines() + assert type(result) == list # noqa: E721 + return result def path_exists(self, path): command = ["test", "-e", path] diff --git a/tests/test_local.py b/tests/test_local.py index 68e7db33..3ae93f76 100644 --- a/tests/test_local.py +++ b/tests/test_local.py @@ -87,6 +87,17 @@ def test_exec_command_failure__expect_error(self): assert b"nonexistent_command" in error assert b"not found" in error + def test_listdir(self): + """ + Test listdir for listing directory contents. + """ + path = "/etc" + files = self.operations.listdir(path) + assert isinstance(files, list) + for f in files: + assert f is not None + assert type(f) == str # noqa: E721 + def test_read__text(self): """ Test LocalOperations::read for text data. diff --git a/tests/test_remote.py b/tests/test_remote.py index 1f771c62..2c37e2c1 100755 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -222,8 +222,10 @@ def test_listdir(self): """ path = "/etc" files = self.operations.listdir(path) - assert isinstance(files, list) + for f in files: + assert f is not None + assert type(f) == str # noqa: E721 def test_path_exists_true__directory(self): """ From cc4361c6fce77df6380aed8e023241b2cdde915f Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Fri, 14 Mar 2025 18:44:28 +0300 Subject: [PATCH 160/216] get_process_children is updated --- testgres/operations/local_ops.py | 1 + testgres/operations/remote_ops.py | 7 ++++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/testgres/operations/local_ops.py b/testgres/operations/local_ops.py index 0fa7d0ad..6ae1cf2b 100644 --- a/testgres/operations/local_ops.py +++ b/testgres/operations/local_ops.py @@ -418,6 +418,7 @@ def get_pid(self): return os.getpid() def get_process_children(self, pid): + assert type(pid) == int # noqa: E721 return psutil.Process(pid).children() # Database control diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index 60161e3c..625a184b 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -599,15 +599,16 @@ def get_pid(self): return int(self.exec_command("echo $$", encoding=get_default_encoding())) def get_process_children(self, pid): - command = ["ssh"] + self.ssh_args + [self.ssh_dest, f"pgrep -P {pid}"] + assert type(pid) == int # noqa: E721 + command = ["ssh"] + self.ssh_args + [self.ssh_dest, "pgrep", "-P", str(pid)] result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) if result.returncode == 0: children = result.stdout.strip().splitlines() return [PsUtilProcessProxy(self, int(child_pid.strip())) for child_pid in children] - else: - raise ExecUtilException(f"Error in getting process children. Error: {result.stderr}") + + raise ExecUtilException(f"Error in getting process children. Error: {result.stderr}") # Database control def db_connect(self, dbname, user, password=None, host="localhost", port=5432): From bab1d8ec8671261787689dec99a4a3ff10499fd7 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Fri, 14 Mar 2025 18:49:58 +0300 Subject: [PATCH 161/216] os_ops::readlines is updated (revision) --- testgres/operations/local_ops.py | 16 +++++++++++++++- testgres/operations/remote_ops.py | 27 +++++++++++++++++++++++---- 2 files changed, 38 insertions(+), 5 deletions(-) diff --git a/testgres/operations/local_ops.py b/testgres/operations/local_ops.py index 6ae1cf2b..93a64787 100644 --- a/testgres/operations/local_ops.py +++ b/testgres/operations/local_ops.py @@ -352,12 +352,26 @@ def readlines(self, filename, num_lines=0, binary=False, encoding=None): Read lines from a local file. If num_lines is greater than 0, only the last num_lines lines will be read. """ + assert type(num_lines) == int # noqa: E721 + assert type(filename) == str # noqa: E721 + assert type(binary) == bool # noqa: E721 + assert encoding is None or type(encoding) == str # noqa: E721 assert num_lines >= 0 + + if binary: + assert encoding is None + pass + elif encoding is None: + encoding = get_default_encoding() + assert type(encoding) == str # noqa: E721 + else: + assert type(encoding) == str # noqa: E721 + pass + mode = 'rb' if binary else 'r' if num_lines == 0: with open(filename, mode, encoding=encoding) as file: # open in binary mode return file.readlines() - else: bufsize = 8192 buffers = 1 diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index 625a184b..e1ad6dac 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -484,18 +484,37 @@ def _read__binary(self, filename): return content def readlines(self, filename, num_lines=0, binary=False, encoding=None): + assert type(num_lines) == int # noqa: E721 + assert type(filename) == str # noqa: E721 + assert type(binary) == bool # noqa: E721 + assert encoding is None or type(encoding) == str # noqa: E721 + if num_lines > 0: - cmd = "tail -n {} {}".format(num_lines, filename) + cmd = ["tail", "-n", str(num_lines), filename] + else: + cmd = ["cat", filename] + + if binary: + assert encoding is None + pass + elif encoding is None: + encoding = get_default_encoding() + assert type(encoding) == str # noqa: E721 else: - cmd = "cat {}".format(filename) + assert type(encoding) == str # noqa: E721 + pass result = self.exec_command(cmd, encoding=encoding) + assert result is not None - if not binary and result: - lines = result.decode(encoding or get_default_encoding()).splitlines() + if binary: + assert type(result) == bytes # noqa: E721 + lines = result.splitlines() else: + assert type(result) == str # noqa: E721 lines = result.splitlines() + assert type(lines) == list # noqa: E721 return lines def read_binary(self, filename, offset): From c07f1127f110bf7dac28ef6f28cad363d404d551 Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Sun, 16 Mar 2025 14:07:04 +0300 Subject: [PATCH 162/216] Initialization of Helpers._get_default_encoding_func is corrected [py3.9] (#221) Python 3.9 does not undestand the following code: _get_default_encoding_func = _make_get_default_encoding_func() ERROR - TypeError: 'staticmethod' object is not callable https://app.travis-ci.com/github/postgrespro/testgres/jobs/631402370 The code: _get_default_encoding_func = _make_get_default_encoding_func.__func__() is processed without problems. --- testgres/operations/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testgres/operations/helpers.py b/testgres/operations/helpers.py index 03e97edc..ebbf0f73 100644 --- a/testgres/operations/helpers.py +++ b/testgres/operations/helpers.py @@ -12,7 +12,7 @@ def _make_get_default_encoding_func(): return locale.getpreferredencoding # Prepared pointer on function to get a name of system codepage - _get_default_encoding_func = _make_get_default_encoding_func() + _get_default_encoding_func = _make_get_default_encoding_func.__func__() @staticmethod def GetDefaultEncoding(): From ea114966771982f0a2fabaaeedd8c89bf8ec1feb Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Sun, 16 Mar 2025 18:04:31 +0300 Subject: [PATCH 163/216] testgres.utils.get_pg_version2 is added This a version of get_pg_version that requires and uses an explicit os_ops object. --- testgres/utils.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/testgres/utils.py b/testgres/utils.py index 093eaff6..a988effe 100644 --- a/testgres/utils.py +++ b/testgres/utils.py @@ -203,19 +203,29 @@ def cache_pg_config_data(cmd): return cache_pg_config_data("pg_config") -def get_pg_version(bin_dir=None): +def get_pg_version2(os_ops: OsOperations, bin_dir=None): """ Return PostgreSQL version provided by postmaster. """ + assert os_ops is not None + assert isinstance(os_ops, OsOperations) # Get raw version (e.g., postgres (PostgreSQL) 9.5.7) - postgres_path = os.path.join(bin_dir, 'postgres') if bin_dir else get_bin_path('postgres') + postgres_path = os.path.join(bin_dir, 'postgres') if bin_dir else get_bin_path2(os_ops, 'postgres') _params = [postgres_path, '--version'] - raw_ver = tconf.os_ops.exec_command(_params, encoding='utf-8') + raw_ver = os_ops.exec_command(_params, encoding='utf-8') return parse_pg_version(raw_ver) +def get_pg_version(bin_dir=None): + """ + Return PostgreSQL version provided by postmaster. + """ + + return get_pg_version2(tconf.os_ops, bin_dir) + + def parse_pg_version(version_out): # Generalize removal of system-specific suffixes (anything in parentheses) raw_ver = re.sub(r'\([^)]*\)', '', version_out).strip() From ac0a2bbd373b5d5852bfb3b3ff0ff6caec2c6633 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Mon, 17 Mar 2025 16:10:55 +0300 Subject: [PATCH 164/216] PostgresNode is updated [os_ops and clone_with_new_name_and_base_dir] 1) Constructor of PostgresNode can get an explicit os_ops object 2) PostgresNode::os_ops property is added 3) New method PostgresNode::clone_with_new_name_and_base_dir is added It is used to right clone an object in NodeBackup::spawn_primary --- testgres/backup.py | 15 +++++++++++-- testgres/node.py | 54 +++++++++++++++++++++++++++++++++++++++------- 2 files changed, 59 insertions(+), 10 deletions(-) diff --git a/testgres/backup.py b/testgres/backup.py index 619c0270..388697b7 100644 --- a/testgres/backup.py +++ b/testgres/backup.py @@ -147,8 +147,19 @@ def spawn_primary(self, name=None, destroy=True): base_dir = self._prepare_dir(destroy) # Build a new PostgresNode - NodeClass = self.original_node.__class__ - with clean_on_error(NodeClass(name=name, base_dir=base_dir, conn_params=self.original_node.os_ops.conn_params)) as node: + assert self.original_node is not None + + if (hasattr(self.original_node, "clone_with_new_name_and_base_dir")): + node = self.original_node.clone_with_new_name_and_base_dir(name=name, base_dir=base_dir) + else: + # For backward compatibility + NodeClass = self.original_node.__class__ + node = NodeClass(name=name, base_dir=base_dir, conn_params=self.original_node.os_ops.conn_params) + + assert node is not None + assert type(node) == self.original_node.__class__ # noqa: E721 + + with clean_on_error(node) as node: # New nodes should always remove dir tree node._should_rm_dirs = True diff --git a/testgres/node.py b/testgres/node.py index 859fe742..6d2417c4 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -100,6 +100,7 @@ from .backup import NodeBackup from .operations.os_ops import ConnectionParams +from .operations.os_ops import OsOperations from .operations.local_ops import LocalOperations from .operations.remote_ops import RemoteOperations @@ -135,7 +136,7 @@ class PostgresNode(object): _C_MAX_START_ATEMPTS = 5 def __init__(self, name=None, base_dir=None, port=None, conn_params: ConnectionParams = ConnectionParams(), - bin_dir=None, prefix=None): + bin_dir=None, prefix=None, os_ops=None): """ PostgresNode constructor. @@ -157,17 +158,20 @@ def __init__(self, name=None, base_dir=None, port=None, conn_params: ConnectionP # basic self.name = name or generate_app_name() - if testgres_config.os_ops: - self.os_ops = testgres_config.os_ops - elif conn_params.ssh_key: - self.os_ops = RemoteOperations(conn_params) + if os_ops is None: + os_ops = __class__._get_os_ops(conn_params) else: - self.os_ops = LocalOperations(conn_params) + assert conn_params is None + pass - self.host = self.os_ops.host + assert os_ops is not None + assert isinstance(os_ops, OsOperations) + self._os_ops = os_ops + + self.host = os_ops.host self.port = port or utils.reserve_port() - self.ssh_key = self.os_ops.ssh_key + self.ssh_key = os_ops.ssh_key # defaults for __exit__() self.cleanup_on_good_exit = testgres_config.node_cleanup_on_good_exit @@ -204,6 +208,40 @@ def __repr__(self): return "{}(name='{}', port={}, base_dir='{}')".format( self.__class__.__name__, self.name, self.port, self.base_dir) + @staticmethod + def _get_os_ops(conn_params: ConnectionParams) -> OsOperations: + if testgres_config.os_ops: + return testgres_config.os_ops + + assert type(conn_params) == ConnectionParams # noqa: E721 + + if conn_params.ssh_key: + return RemoteOperations(conn_params) + + return LocalOperations(conn_params) + + def clone_with_new_name_and_base_dir(self, name: str, base_dir: str): + assert name is None or type(name) == str # noqa: E721 + assert base_dir is None or type(base_dir) == str # noqa: E721 + + assert __class__ == PostgresNode + + node = PostgresNode( + name=name, + base_dir=base_dir, + conn_params=None, + bin_dir=self._bin_dir, + prefix=self._prefix, + os_ops=self._os_ops) + + return node + + @property + def os_ops(self) -> OsOperations: + assert self._os_ops is not None + assert isinstance(self._os_ops, OsOperations) + return self._os_ops + @property def pid(self): """ From dcb7f24592b157297606c29985621a28ebeebeaa Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Mon, 17 Mar 2025 16:34:30 +0300 Subject: [PATCH 165/216] TestTestgresCommon - generic tests for local and remote modes. This commit replaces #216 --- run_tests.sh | 4 +- tests/test_simple.py | 979 ---------------------------- tests/test_simple_remote.py | 993 +---------------------------- tests/test_testgres_common.py | 1131 +++++++++++++++++++++++++++++++++ 4 files changed, 1142 insertions(+), 1965 deletions(-) create mode 100644 tests/test_testgres_common.py diff --git a/run_tests.sh b/run_tests.sh index 021f9d9f..0fecde60 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -4,7 +4,9 @@ set -eux -if [ -z ${TEST_FILTER+x} ]; then export TEST_FILTER="TestgresTests"; fi +if [ -z ${TEST_FILTER+x} ]; \ +then export TEST_FILTER="TestgresTests or (TestTestgresCommon and (not remote_ops))"; \ +fi # choose python version echo python version is $PYTHON_VERSION diff --git a/tests/test_simple.py b/tests/test_simple.py index e886a39c..f648e558 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -2,29 +2,16 @@ import os import re import subprocess -import tempfile -import time -import six import pytest import psutil import platform import logging -import uuid - -from contextlib import contextmanager -from shutil import rmtree from .. import testgres from ..testgres import \ - InitNodeException, \ StartNodeException, \ ExecUtilException, \ - BackupException, \ - QueryException, \ - TimeoutException, \ - TestgresException, \ - InvalidOperationException, \ NodeApp from ..testgres import \ @@ -34,9 +21,6 @@ pop_config from ..testgres import \ - NodeStatus, \ - ProcessType, \ - IsolationLevel, \ get_new_node from ..testgres import \ @@ -44,14 +28,9 @@ get_pg_config, \ get_pg_version -from ..testgres import \ - First, \ - Any - # NOTE: those are ugly imports from ..testgres import bound_ports from ..testgres.utils import PgVer, parse_pg_version -from ..testgres.utils import file_tail from ..testgres.node import ProcessProxy @@ -95,17 +74,6 @@ def rm_carriage_returns(out): return out -@contextmanager -def removing(f): - try: - yield f - finally: - if os.path.isfile(f): - os.remove(f) - elif os.path.isdir(f): - rmtree(f, ignore_errors=True) - - class TestgresTests: def test_node_repr(self): with get_new_node() as node: @@ -132,740 +100,6 @@ def test_custom_init(self): # there should be no trust entries at all assert not (any('trust' in s for s in lines)) - def test_double_init(self): - with get_new_node().init() as node: - # can't initialize node more than once - with pytest.raises(expected_exception=InitNodeException): - node.init() - - def test_init_after_cleanup(self): - with get_new_node() as node: - node.init().start().execute('select 1') - node.cleanup() - node.init().start().execute('select 1') - - def test_init_unique_system_id(self): - # this function exists in PostgreSQL 9.6+ - __class__.helper__skip_test_if_util_not_exist("pg_resetwal") - __class__.helper__skip_test_if_pg_version_is_not_ge("9.6") - - query = 'select system_identifier from pg_control_system()' - - with scoped_config(cache_initdb=False): - with get_new_node().init().start() as node0: - id0 = node0.execute(query)[0] - - with scoped_config(cache_initdb=True, - cached_initdb_unique=True) as config: - assert (config.cache_initdb) - assert (config.cached_initdb_unique) - - # spawn two nodes; ids must be different - with get_new_node().init().start() as node1, \ - get_new_node().init().start() as node2: - - id1 = node1.execute(query)[0] - id2 = node2.execute(query)[0] - - # ids must increase - assert (id1 > id0) - assert (id2 > id1) - - def test_node_exit(self): - base_dir = None - - with pytest.raises(expected_exception=QueryException): - with get_new_node().init() as node: - base_dir = node.base_dir - node.safe_psql('select 1') - - # we should save the DB for "debugging" - assert (os.path.exists(base_dir)) - rmtree(base_dir, ignore_errors=True) - - with get_new_node().init() as node: - base_dir = node.base_dir - - # should have been removed by default - assert not (os.path.exists(base_dir)) - - def test_double_start(self): - with get_new_node().init().start() as node: - # can't start node more than once - node.start() - assert (node.is_started) - - def test_uninitialized_start(self): - with get_new_node() as node: - # node is not initialized yet - with pytest.raises(expected_exception=StartNodeException): - node.start() - - def test_restart(self): - with get_new_node() as node: - node.init().start() - - # restart, ok - res = node.execute('select 1') - assert (res == [(1, )]) - node.restart() - res = node.execute('select 2') - assert (res == [(2, )]) - - # restart, fail - with pytest.raises(expected_exception=StartNodeException): - node.append_conf('pg_hba.conf', 'DUMMY') - node.restart() - - def test_reload(self): - with get_new_node() as node: - node.init().start() - - # change client_min_messages and save old value - cmm_old = node.execute('show client_min_messages') - node.append_conf(client_min_messages='DEBUG1') - - # reload config - node.reload() - - # check new value - cmm_new = node.execute('show client_min_messages') - assert ('debug1' == cmm_new[0][0].lower()) - assert (cmm_old != cmm_new) - - def test_pg_ctl(self): - with get_new_node() as node: - node.init().start() - - status = node.pg_ctl(['status']) - assert ('PID' in status) - - def test_status(self): - assert (NodeStatus.Running) - assert not (NodeStatus.Stopped) - assert not (NodeStatus.Uninitialized) - - # check statuses after each operation - with get_new_node() as node: - assert (node.pid == 0) - assert (node.status() == NodeStatus.Uninitialized) - - node.init() - - assert (node.pid == 0) - assert (node.status() == NodeStatus.Stopped) - - node.start() - - assert (node.pid != 0) - assert (node.status() == NodeStatus.Running) - - node.stop() - - assert (node.pid == 0) - assert (node.status() == NodeStatus.Stopped) - - node.cleanup() - - assert (node.pid == 0) - assert (node.status() == NodeStatus.Uninitialized) - - def test_psql(self): - with get_new_node().init().start() as node: - - # check returned values (1 arg) - res = node.psql('select 1') - assert (rm_carriage_returns(res) == (0, b'1\n', b'')) - - # check returned values (2 args) - res = node.psql('postgres', 'select 2') - assert (rm_carriage_returns(res) == (0, b'2\n', b'')) - - # check returned values (named) - res = node.psql(query='select 3', dbname='postgres') - assert (rm_carriage_returns(res) == (0, b'3\n', b'')) - - # check returned values (1 arg) - res = node.safe_psql('select 4') - assert (rm_carriage_returns(res) == b'4\n') - - # check returned values (2 args) - res = node.safe_psql('postgres', 'select 5') - assert (rm_carriage_returns(res) == b'5\n') - - # check returned values (named) - res = node.safe_psql(query='select 6', dbname='postgres') - assert (rm_carriage_returns(res) == b'6\n') - - # check feeding input - node.safe_psql('create table horns (w int)') - node.safe_psql('copy horns from stdin (format csv)', - input=b"1\n2\n3\n\\.\n") - _sum = node.safe_psql('select sum(w) from horns') - assert (rm_carriage_returns(_sum) == b'6\n') - - # check psql's default args, fails - with pytest.raises(expected_exception=QueryException): - node.psql() - - node.stop() - - # check psql on stopped node, fails - with pytest.raises(expected_exception=QueryException): - node.safe_psql('select 1') - - def test_safe_psql__expect_error(self): - with get_new_node().init().start() as node: - err = node.safe_psql('select_or_not_select 1', expect_error=True) - assert (type(err) == str) # noqa: E721 - assert ('select_or_not_select' in err) - assert ('ERROR: syntax error at or near "select_or_not_select"' in err) - - # --------- - with pytest.raises( - expected_exception=InvalidOperationException, - match="^" + re.escape("Exception was expected, but query finished successfully: `select 1;`.") + "$" - ): - node.safe_psql("select 1;", expect_error=True) - - # --------- - res = node.safe_psql("select 1;", expect_error=False) - assert (rm_carriage_returns(res) == b'1\n') - - def test_transactions(self): - with get_new_node().init().start() as node: - - with node.connect() as con: - con.begin() - con.execute('create table test(val int)') - con.execute('insert into test values (1)') - con.commit() - - con.begin() - con.execute('insert into test values (2)') - res = con.execute('select * from test order by val asc') - assert (res == [(1, ), (2, )]) - con.rollback() - - con.begin() - res = con.execute('select * from test') - assert (res == [(1, )]) - con.rollback() - - con.begin() - con.execute('drop table test') - con.commit() - - def test_control_data(self): - with get_new_node() as node: - - # node is not initialized yet - with pytest.raises(expected_exception=ExecUtilException): - node.get_control_data() - - node.init() - data = node.get_control_data() - - # check returned dict - assert data is not None - assert (any('pg_control' in s for s in data.keys())) - - def test_backup_simple(self): - with get_new_node() as master: - - # enable streaming for backups - master.init(allow_streaming=True) - - # node must be running - with pytest.raises(expected_exception=BackupException): - master.backup() - - # it's time to start node - master.start() - - # fill node with some data - master.psql('create table test as select generate_series(1, 4) i') - - with master.backup(xlog_method='stream') as backup: - with backup.spawn_primary().start() as slave: - res = slave.execute('select * from test order by i asc') - assert (res == [(1, ), (2, ), (3, ), (4, )]) - - def test_backup_multiple(self): - with get_new_node() as node: - node.init(allow_streaming=True).start() - - with node.backup(xlog_method='fetch') as backup1, \ - node.backup(xlog_method='fetch') as backup2: - assert (backup1.base_dir != backup2.base_dir) - - with node.backup(xlog_method='fetch') as backup: - with backup.spawn_primary('node1', destroy=False) as node1, \ - backup.spawn_primary('node2', destroy=False) as node2: - assert (node1.base_dir != node2.base_dir) - - def test_backup_exhaust(self): - with get_new_node() as node: - node.init(allow_streaming=True).start() - - with node.backup(xlog_method='fetch') as backup: - # exhaust backup by creating new node - with backup.spawn_primary(): - pass - - # now let's try to create one more node - with pytest.raises(expected_exception=BackupException): - backup.spawn_primary() - - def test_backup_wrong_xlog_method(self): - with get_new_node() as node: - node.init(allow_streaming=True).start() - - with pytest.raises( - expected_exception=BackupException, - match="^" + re.escape('Invalid xlog_method "wrong"') + "$" - ): - node.backup(xlog_method='wrong') - - def test_pg_ctl_wait_option(self): - C_MAX_ATTEMPTS = 50 - - node = get_new_node() - assert node.status() == testgres.NodeStatus.Uninitialized - node.init() - assert node.status() == testgres.NodeStatus.Stopped - node.start(wait=False) - nAttempt = 0 - while True: - if nAttempt == C_MAX_ATTEMPTS: - # - # [2025-03-11] - # We have an unexpected problem with this test in CI - # Let's get an additional information about this test failure. - # - logging.error("Node was not stopped.") - if not node.os_ops.path_exists(node.pg_log_file): - logging.warning("Node log does not exist.") - else: - logging.info("Let's read node log file [{0}]".format(node.pg_log_file)) - logFileData = node.os_ops.read(node.pg_log_file, binary=False) - logging.info("Node log file content:\n{0}".format(logFileData)) - - raise Exception("Could not stop node.") - - nAttempt += 1 - - if nAttempt > 1: - logging.info("Wait 1 second.") - time.sleep(1) - logging.info("") - - logging.info("Try to stop node. Attempt #{0}.".format(nAttempt)) - - try: - node.stop(wait=False) - break - except ExecUtilException as e: - # it's ok to get this exception here since node - # could be not started yet - logging.info("Node is not stopped. Exception ({0}): {1}".format(type(e).__name__, e)) - continue - - logging.info("OK. Stop command was executed. Let's wait while our node will stop really.") - nAttempt = 0 - while True: - if nAttempt == C_MAX_ATTEMPTS: - raise Exception("Could not stop node.") - - nAttempt += 1 - if nAttempt > 1: - logging.info("Wait 1 second.") - time.sleep(1) - logging.info("") - - logging.info("Attempt #{0}.".format(nAttempt)) - s1 = node.status() - - if s1 == testgres.NodeStatus.Running: - continue - - if s1 == testgres.NodeStatus.Stopped: - break - - raise Exception("Unexpected node status: {0}.".format(s1)) - - logging.info("OK. Node is stopped.") - node.cleanup() - - def test_replicate(self): - with get_new_node() as node: - node.init(allow_streaming=True).start() - - with node.replicate().start() as replica: - res = replica.execute('select 1') - assert (res == [(1, )]) - - node.execute('create table test (val int)', commit=True) - - replica.catchup() - - res = node.execute('select * from test') - assert (res == []) - - def test_synchronous_replication(self): - __class__.helper__skip_test_if_pg_version_is_not_ge("9.6") - - with get_new_node() as master: - old_version = not pg_version_ge('9.6') - - master.init(allow_streaming=True).start() - - if not old_version: - master.append_conf('synchronous_commit = remote_apply') - - # create standby - with master.replicate() as standby1, master.replicate() as standby2: - standby1.start() - standby2.start() - - # check formatting - assert ( - '1 ("{}", "{}")'.format(standby1.name, standby2.name) == str(First(1, (standby1, standby2))) - ) # yapf: disable - assert ( - 'ANY 1 ("{}", "{}")'.format(standby1.name, standby2.name) == str(Any(1, (standby1, standby2))) - ) # yapf: disable - - # set synchronous_standby_names - master.set_synchronous_standbys(First(2, [standby1, standby2])) - master.restart() - - # the following part of the test is only applicable to newer - # versions of PostgresQL - if not old_version: - master.safe_psql('create table abc(a int)') - - # Create a large transaction that will take some time to apply - # on standby to check that it applies synchronously - # (If set synchronous_commit to 'on' or other lower level then - # standby most likely won't catchup so fast and test will fail) - master.safe_psql( - 'insert into abc select generate_series(1, 1000000)') - res = standby1.safe_psql('select count(*) from abc') - assert (rm_carriage_returns(res) == b'1000000\n') - - def test_logical_replication(self): - __class__.helper__skip_test_if_pg_version_is_not_ge("10") - - with get_new_node() as node1, get_new_node() as node2: - node1.init(allow_logical=True) - node1.start() - node2.init().start() - - create_table = 'create table test (a int, b int)' - node1.safe_psql(create_table) - node2.safe_psql(create_table) - - # create publication / create subscription - pub = node1.publish('mypub') - sub = node2.subscribe(pub, 'mysub') - - node1.safe_psql('insert into test values (1, 1), (2, 2)') - - # wait until changes apply on subscriber and check them - sub.catchup() - res = node2.execute('select * from test') - assert (res == [(1, 1), (2, 2)]) - - # disable and put some new data - sub.disable() - node1.safe_psql('insert into test values (3, 3)') - - # enable and ensure that data successfully transferred - sub.enable() - sub.catchup() - res = node2.execute('select * from test') - assert (res == [(1, 1), (2, 2), (3, 3)]) - - # Add new tables. Since we added "all tables" to publication - # (default behaviour of publish() method) we don't need - # to explicitly perform pub.add_tables() - create_table = 'create table test2 (c char)' - node1.safe_psql(create_table) - node2.safe_psql(create_table) - sub.refresh() - - # put new data - node1.safe_psql('insert into test2 values (\'a\'), (\'b\')') - sub.catchup() - res = node2.execute('select * from test2') - assert (res == [('a', ), ('b', )]) - - # drop subscription - sub.drop() - pub.drop() - - # create new publication and subscription for specific table - # (omitting copying data as it's already done) - pub = node1.publish('newpub', tables=['test']) - sub = node2.subscribe(pub, 'newsub', copy_data=False) - - node1.safe_psql('insert into test values (4, 4)') - sub.catchup() - res = node2.execute('select * from test') - assert (res == [(1, 1), (2, 2), (3, 3), (4, 4)]) - - # explicitly add table - with pytest.raises(expected_exception=ValueError): - pub.add_tables([]) # fail - pub.add_tables(['test2']) - node1.safe_psql('insert into test2 values (\'c\')') - sub.catchup() - res = node2.execute('select * from test2') - assert (res == [('a', ), ('b', )]) - - def test_logical_catchup(self): - """ Runs catchup for 100 times to be sure that it is consistent """ - __class__.helper__skip_test_if_pg_version_is_not_ge("10") - - with get_new_node() as node1, get_new_node() as node2: - node1.init(allow_logical=True) - node1.start() - node2.init().start() - - create_table = 'create table test (key int primary key, val int); ' - node1.safe_psql(create_table) - node1.safe_psql('alter table test replica identity default') - node2.safe_psql(create_table) - - # create publication / create subscription - sub = node2.subscribe(node1.publish('mypub'), 'mysub') - - for i in range(0, 100): - node1.execute('insert into test values ({0}, {0})'.format(i)) - sub.catchup() - res = node2.execute('select * from test') - assert (res == [(i, i, )]) - node1.execute('delete from test') - - def test_logical_replication_fail(self): - __class__.helper__skip_test_if_pg_version_is_ge("10") - - with get_new_node() as node: - with pytest.raises(expected_exception=InitNodeException): - node.init(allow_logical=True) - - def test_replication_slots(self): - with get_new_node() as node: - node.init(allow_streaming=True).start() - - with node.replicate(slot='slot1').start() as replica: - replica.execute('select 1') - - # cannot create new slot with the same name - with pytest.raises(expected_exception=TestgresException): - node.replicate(slot='slot1') - - def test_incorrect_catchup(self): - with get_new_node() as node: - node.init(allow_streaming=True).start() - - # node has no master, can't catch up - with pytest.raises(expected_exception=TestgresException): - node.catchup() - - def test_promotion(self): - with get_new_node() as master: - master.init().start() - master.safe_psql('create table abc(id serial)') - - with master.replicate().start() as replica: - master.stop() - replica.promote() - - # make standby becomes writable master - replica.safe_psql('insert into abc values (1)') - res = replica.safe_psql('select * from abc') - assert (rm_carriage_returns(res) == b'1\n') - - def test_dump(self): - query_create = 'create table test as select generate_series(1, 2) as val' - query_select = 'select * from test order by val asc' - - with get_new_node().init().start() as node1: - - node1.execute(query_create) - for format in ['plain', 'custom', 'directory', 'tar']: - with removing(node1.dump(format=format)) as dump: - with get_new_node().init().start() as node3: - if format == 'directory': - assert (os.path.isdir(dump)) - else: - assert (os.path.isfile(dump)) - # restore dump - node3.restore(filename=dump) - res = node3.execute(query_select) - assert (res == [(1, ), (2, )]) - - def test_users(self): - with get_new_node().init().start() as node: - node.psql('create role test_user login') - value = node.safe_psql('select 1', username='test_user') - value = rm_carriage_returns(value) - assert (value == b'1\n') - - def test_poll_query_until(self): - with get_new_node() as node: - node.init().start() - - get_time = 'select extract(epoch from now())' - check_time = 'select extract(epoch from now()) - {} >= 5' - - start_time = node.execute(get_time)[0][0] - node.poll_query_until(query=check_time.format(start_time)) - end_time = node.execute(get_time)[0][0] - - assert (end_time - start_time >= 5) - - # check 0 columns - with pytest.raises(expected_exception=QueryException): - node.poll_query_until( - query='select from pg_catalog.pg_class limit 1') - - # check None, fail - with pytest.raises(expected_exception=QueryException): - node.poll_query_until(query='create table abc (val int)') - - # check None, ok - node.poll_query_until(query='create table def()', - expected=None) # returns nothing - - # check 0 rows equivalent to expected=None - node.poll_query_until( - query='select * from pg_catalog.pg_class where true = false', - expected=None) - - # check arbitrary expected value, fail - with pytest.raises(expected_exception=TimeoutException): - node.poll_query_until(query='select 3', - expected=1, - max_attempts=3, - sleep_time=0.01) - - # check arbitrary expected value, ok - node.poll_query_until(query='select 2', expected=2) - - # check timeout - with pytest.raises(expected_exception=TimeoutException): - node.poll_query_until(query='select 1 > 2', - max_attempts=3, - sleep_time=0.01) - - # check ProgrammingError, fail - with pytest.raises(expected_exception=testgres.ProgrammingError): - node.poll_query_until(query='dummy1') - - # check ProgrammingError, ok - with pytest.raises(expected_exception=(TimeoutException)): - node.poll_query_until(query='dummy2', - max_attempts=3, - sleep_time=0.01, - suppress={testgres.ProgrammingError}) - - # check 1 arg, ok - node.poll_query_until('select true') - - def test_logging(self): - C_MAX_ATTEMPTS = 50 - # This name is used for testgres logging, too. - C_NODE_NAME = "testgres_tests." + __class__.__name__ + "test_logging-master-" + uuid.uuid4().hex - - logging.info("Node name is [{0}]".format(C_NODE_NAME)) - - with tempfile.NamedTemporaryFile('w', delete=True) as logfile: - formatter = logging.Formatter(fmt="%(node)-5s: %(message)s") - handler = logging.FileHandler(filename=logfile.name) - handler.formatter = formatter - logger = logging.getLogger(C_NODE_NAME) - assert logger is not None - assert len(logger.handlers) == 0 - - try: - # It disables to log on the root level - logger.propagate = False - logger.addHandler(handler) - - with scoped_config(use_python_logging=True): - with get_new_node(name=C_NODE_NAME) as master: - logging.info("Master node is initilizing") - master.init() - - logging.info("Master node is starting") - master.start() - - logging.info("Dummy query is executed a few times") - for _ in range(20): - master.execute('select 1') - time.sleep(0.01) - - # let logging worker do the job - time.sleep(0.1) - - logging.info("Master node log file is checking") - nAttempt = 0 - - while True: - assert nAttempt <= C_MAX_ATTEMPTS - if nAttempt == C_MAX_ATTEMPTS: - raise Exception("Test failed!") - - # let logging worker do the job - time.sleep(0.1) - - nAttempt += 1 - - logging.info("Attempt {0}".format(nAttempt)) - - # check that master's port is found - with open(logfile.name, 'r') as log: - lines = log.readlines() - - assert lines is not None - assert type(lines) == list # noqa: E721 - - def LOCAL__test_lines(): - for s in lines: - if any(C_NODE_NAME in s for s in lines): - logging.info("OK. We found the node_name in a line \"{0}\"".format(s)) - return True - return False - - if LOCAL__test_lines(): - break - - logging.info("Master node log file does not have an expected information.") - continue - - # test logger after stop/start/restart - logging.info("Master node is stopping...") - master.stop() - logging.info("Master node is staring again...") - master.start() - logging.info("Master node is restaring...") - master.restart() - assert (master._logger.is_alive()) - finally: - # It is a hack code to logging cleanup - logging._acquireLock() - assert logging.Logger.manager is not None - assert C_NODE_NAME in logging.Logger.manager.loggerDict.keys() - logging.Logger.manager.loggerDict.pop(C_NODE_NAME, None) - assert not (C_NODE_NAME in logging.Logger.manager.loggerDict.keys()) - assert not (handler in logging._handlers.values()) - logging._releaseLock() - # GO HOME! - return - def test_pgbench(self): __class__.helper__skip_test_if_util_not_exist("pgbench") @@ -955,60 +189,6 @@ def test_unix_sockets(self): r.execute('select 1') r.safe_psql('select 1') - def test_auto_name(self): - with get_new_node().init(allow_streaming=True).start() as m: - with m.replicate().start() as r: - # check that nodes are running - assert (m.status()) - assert (r.status()) - - # check their names - assert (m.name != r.name) - assert ('testgres' in m.name) - assert ('testgres' in r.name) - - def test_file_tail(self): - s1 = "the quick brown fox jumped over that lazy dog\n" - s2 = "abc\n" - s3 = "def\n" - - with tempfile.NamedTemporaryFile(mode='r+', delete=True) as f: - sz = 0 - while sz < 3 * 8192: - sz += len(s1) - f.write(s1) - f.write(s2) - f.write(s3) - - f.seek(0) - lines = file_tail(f, 3) - assert (lines[0] == s1) - assert (lines[1] == s2) - assert (lines[2] == s3) - - f.seek(0) - lines = file_tail(f, 1) - assert (lines[0] == s3) - - def test_isolation_levels(self): - with get_new_node().init().start() as node: - with node.connect() as con: - # string levels - con.begin('Read Uncommitted').commit() - con.begin('Read Committed').commit() - con.begin('Repeatable Read').commit() - con.begin('Serializable').commit() - - # enum levels - con.begin(IsolationLevel.ReadUncommitted).commit() - con.begin(IsolationLevel.ReadCommitted).commit() - con.begin(IsolationLevel.RepeatableRead).commit() - con.begin(IsolationLevel.Serializable).commit() - - # check wrong level - with pytest.raises(expected_exception=QueryException): - con.begin('Garbage').commit() - def test_ports_management(self): assert bound_ports is not None assert type(bound_ports) == set # noqa: E721 @@ -1043,153 +223,6 @@ def test_ports_management(self): assert type(bound_ports) == set # noqa: E721 assert bound_ports == stage0__bound_ports - def test_exceptions(self): - str(StartNodeException('msg', [('file', 'lines')])) - str(ExecUtilException('msg', 'cmd', 1, 'out')) - str(QueryException('msg', 'query')) - - def test_version_management(self): - a = PgVer('10.0') - b = PgVer('10') - c = PgVer('9.6.5') - d = PgVer('15.0') - e = PgVer('15rc1') - f = PgVer('15beta4') - h = PgVer('15.3biha') - i = PgVer('15.3') - g = PgVer('15.3.1bihabeta1') - k = PgVer('15.3.1') - - assert (a == b) - assert (b > c) - assert (a > c) - assert (d > e) - assert (e > f) - assert (d > f) - assert (h > f) - assert (h == i) - assert (g == k) - assert (g > h) - - version = get_pg_version() - with get_new_node() as node: - assert (isinstance(version, six.string_types)) - assert (isinstance(node.version, PgVer)) - assert (node.version == PgVer(version)) - - def test_child_pids(self): - master_processes = [ - ProcessType.AutovacuumLauncher, - ProcessType.BackgroundWriter, - ProcessType.Checkpointer, - ProcessType.StatsCollector, - ProcessType.WalSender, - ProcessType.WalWriter, - ] - - if pg_version_ge('10'): - master_processes.append(ProcessType.LogicalReplicationLauncher) - - if pg_version_ge('14'): - master_processes.remove(ProcessType.StatsCollector) - - repl_processes = [ - ProcessType.Startup, - ProcessType.WalReceiver, - ] - - def LOCAL__test_auxiliary_pids( - node: testgres.PostgresNode, - expectedTypes: list[ProcessType] - ) -> list[ProcessType]: - # returns list of the absence processes - assert node is not None - assert type(node) == testgres.PostgresNode # noqa: E721 - assert expectedTypes is not None - assert type(expectedTypes) == list # noqa: E721 - - pids = node.auxiliary_pids - assert pids is not None # noqa: E721 - assert type(pids) == dict # noqa: E721 - - result = list[ProcessType]() - for ptype in expectedTypes: - if not (ptype in pids): - result.append(ptype) - return result - - def LOCAL__check_auxiliary_pids__multiple_attempts( - node: testgres.PostgresNode, - expectedTypes: list[ProcessType]): - assert node is not None - assert type(node) == testgres.PostgresNode # noqa: E721 - assert expectedTypes is not None - assert type(expectedTypes) == list # noqa: E721 - - nAttempt = 0 - - while nAttempt < 5: - nAttempt += 1 - - logging.info("Test pids of [{0}] node. Attempt #{1}.".format( - node.name, - nAttempt - )) - - if nAttempt > 1: - time.sleep(1) - - absenceList = LOCAL__test_auxiliary_pids(node, expectedTypes) - assert absenceList is not None - assert type(absenceList) == list # noqa: E721 - if len(absenceList) == 0: - logging.info("Bingo!") - return - - logging.info("These processes are not found: {0}.".format(absenceList)) - continue - - raise Exception("Node {0} does not have the following processes: {1}.".format( - node.name, - absenceList - )) - - with get_new_node().init().start() as master: - - # master node doesn't have a source walsender! - with pytest.raises(expected_exception=TestgresException): - master.source_walsender - - with master.connect() as con: - assert (con.pid > 0) - - with master.replicate().start() as replica: - - # test __str__ method - str(master.child_processes[0]) - - LOCAL__check_auxiliary_pids__multiple_attempts( - master, - master_processes) - - LOCAL__check_auxiliary_pids__multiple_attempts( - replica, - repl_processes) - - master_pids = master.auxiliary_pids - - # there should be exactly 1 source walsender for replica - assert (len(master_pids[ProcessType.WalSender]) == 1) - pid1 = master_pids[ProcessType.WalSender][0] - pid2 = replica.source_walsender.pid - assert (pid1 == pid2) - - replica.stop() - - # there should be no walsender after we've stopped replica - with pytest.raises(expected_exception=TestgresException): - replica.source_walsender - def test_child_process_dies(self): # test for FileNotFound exception during child_processes() function cmd = ["timeout", "60"] if os.name == 'nt' else ["sleep", "60"] @@ -1512,15 +545,3 @@ def helper__skip_test_if_util_not_exist(name: str): if not util_exists(name2): pytest.skip('might be missing') - - @staticmethod - def helper__skip_test_if_pg_version_is_not_ge(version: str): - assert type(version) == str # noqa: E721 - if not pg_version_ge(version): - pytest.skip('requires {0}+'.format(version)) - - @staticmethod - def helper__skip_test_if_pg_version_is_ge(version: str): - assert type(version) == str # noqa: E721 - if pg_version_ge(version): - pytest.skip('requires <{0}'.format(version)) diff --git a/tests/test_simple_remote.py b/tests/test_simple_remote.py index d484f1e3..c16fe53f 100755 --- a/tests/test_simple_remote.py +++ b/tests/test_simple_remote.py @@ -2,28 +2,19 @@ import os import re import subprocess -import tempfile -import time -import six import pytest import psutil import logging -import uuid -from contextlib import contextmanager +from .helpers.os_ops_descrs import OsOpsDescrs +from .helpers.os_ops_descrs import OsOperations from .. import testgres from ..testgres.exceptions import \ InitNodeException, \ - StartNodeException, \ - ExecUtilException, \ - BackupException, \ - QueryException, \ - TimeoutException, \ - TestgresException, \ - InvalidOperationException + ExecUtilException from ..testgres.config import \ TestgresConfig, \ @@ -31,33 +22,13 @@ scoped_config, \ pop_config, testgres_config -from ..testgres import \ - NodeStatus, \ - ProcessType, \ - IsolationLevel, \ - get_remote_node, \ - RemoteOperations - from ..testgres import \ get_bin_path, \ - get_pg_config, \ - get_pg_version - -from ..testgres import \ - First, \ - Any + get_pg_config # NOTE: those are ugly imports from ..testgres import bound_ports -from ..testgres.utils import PgVer -from ..testgres.utils import file_tail -from ..testgres.node import ProcessProxy, ConnectionParams - - -def pg_version_ge(version): - cur_ver = PgVer(get_pg_version()) - min_ver = PgVer(version) - return cur_ver >= min_ver +from ..testgres.node import ProcessProxy def util_exists(util): @@ -76,25 +47,8 @@ def good_properties(f): return True -@contextmanager -def removing(f): - try: - yield f - finally: - if testgres_config.os_ops.isfile(f): - testgres_config.os_ops.remove_file(f) - - elif testgres_config.os_ops.isdir(f): - testgres_config.os_ops.rmdirs(f, ignore_errors=True) - - class TestgresRemoteTests: - sm_conn_params = ConnectionParams( - host=os.getenv('RDBMS_TESTPOOL1_HOST') or '127.0.0.1', - username=os.getenv('USER'), - ssh_key=os.getenv('RDBMS_TESTPOOL_SSHKEY')) - - sm_os_ops = RemoteOperations(sm_conn_params) + sm_os_ops = OsOpsDescrs.sm_remote_os_ops @pytest.fixture(autouse=True, scope="class") def implicit_fixture(self): @@ -218,732 +172,6 @@ def test_init__unk_LANG_and_LC_CTYPE(self): __class__.helper__restore_envvar("LC_CTYPE", prev_LC_CTYPE) __class__.helper__restore_envvar("LC_COLLATE", prev_LC_COLLATE) - def test_double_init(self): - with __class__.helper__get_node().init() as node: - # can't initialize node more than once - with pytest.raises(expected_exception=InitNodeException): - node.init() - - def test_init_after_cleanup(self): - with __class__.helper__get_node() as node: - node.init().start().execute('select 1') - node.cleanup() - node.init().start().execute('select 1') - - def test_init_unique_system_id(self): - # this function exists in PostgreSQL 9.6+ - __class__.helper__skip_test_if_util_not_exist("pg_resetwal") - __class__.helper__skip_test_if_pg_version_is_not_ge('9.6') - - query = 'select system_identifier from pg_control_system()' - - with scoped_config(cache_initdb=False): - with __class__.helper__get_node().init().start() as node0: - id0 = node0.execute(query)[0] - - with scoped_config(cache_initdb=True, - cached_initdb_unique=True) as config: - assert (config.cache_initdb) - assert (config.cached_initdb_unique) - - # spawn two nodes; ids must be different - with __class__.helper__get_node().init().start() as node1, \ - __class__.helper__get_node().init().start() as node2: - id1 = node1.execute(query)[0] - id2 = node2.execute(query)[0] - - # ids must increase - assert (id1 > id0) - assert (id2 > id1) - - def test_node_exit(self): - with pytest.raises(expected_exception=QueryException): - with __class__.helper__get_node().init() as node: - base_dir = node.base_dir - node.safe_psql('select 1') - - # we should save the DB for "debugging" - assert (__class__.sm_os_ops.path_exists(base_dir)) - __class__.sm_os_ops.rmdirs(base_dir, ignore_errors=True) - - with __class__.helper__get_node().init() as node: - base_dir = node.base_dir - - # should have been removed by default - assert not (__class__.sm_os_ops.path_exists(base_dir)) - - def test_double_start(self): - with __class__.helper__get_node().init().start() as node: - # can't start node more than once - node.start() - assert (node.is_started) - - def test_uninitialized_start(self): - with __class__.helper__get_node() as node: - # node is not initialized yet - with pytest.raises(expected_exception=StartNodeException): - node.start() - - def test_restart(self): - with __class__.helper__get_node() as node: - node.init().start() - - # restart, ok - res = node.execute('select 1') - assert (res == [(1,)]) - node.restart() - res = node.execute('select 2') - assert (res == [(2,)]) - - # restart, fail - with pytest.raises(expected_exception=StartNodeException): - node.append_conf('pg_hba.conf', 'DUMMY') - node.restart() - - def test_reload(self): - with __class__.helper__get_node() as node: - node.init().start() - - # change client_min_messages and save old value - cmm_old = node.execute('show client_min_messages') - node.append_conf(client_min_messages='DEBUG1') - - # reload config - node.reload() - - # check new value - cmm_new = node.execute('show client_min_messages') - assert ('debug1' == cmm_new[0][0].lower()) - assert (cmm_old != cmm_new) - - def test_pg_ctl(self): - with __class__.helper__get_node() as node: - node.init().start() - - status = node.pg_ctl(['status']) - assert ('PID' in status) - - def test_status(self): - assert (NodeStatus.Running) - assert not (NodeStatus.Stopped) - assert not (NodeStatus.Uninitialized) - - # check statuses after each operation - with __class__.helper__get_node() as node: - assert (node.pid == 0) - assert (node.status() == NodeStatus.Uninitialized) - - node.init() - - assert (node.pid == 0) - assert (node.status() == NodeStatus.Stopped) - - node.start() - - assert (node.pid != 0) - assert (node.status() == NodeStatus.Running) - - node.stop() - - assert (node.pid == 0) - assert (node.status() == NodeStatus.Stopped) - - node.cleanup() - - assert (node.pid == 0) - assert (node.status() == NodeStatus.Uninitialized) - - def test_psql(self): - with __class__.helper__get_node().init().start() as node: - # check returned values (1 arg) - res = node.psql('select 1') - assert (res == (0, b'1\n', b'')) - - # check returned values (2 args) - res = node.psql('postgres', 'select 2') - assert (res == (0, b'2\n', b'')) - - # check returned values (named) - res = node.psql(query='select 3', dbname='postgres') - assert (res == (0, b'3\n', b'')) - - # check returned values (1 arg) - res = node.safe_psql('select 4') - assert (res == b'4\n') - - # check returned values (2 args) - res = node.safe_psql('postgres', 'select 5') - assert (res == b'5\n') - - # check returned values (named) - res = node.safe_psql(query='select 6', dbname='postgres') - assert (res == b'6\n') - - # check feeding input - node.safe_psql('create table horns (w int)') - node.safe_psql('copy horns from stdin (format csv)', - input=b"1\n2\n3\n\\.\n") - _sum = node.safe_psql('select sum(w) from horns') - assert (_sum == b'6\n') - - # check psql's default args, fails - with pytest.raises(expected_exception=QueryException): - node.psql() - - node.stop() - - # check psql on stopped node, fails - with pytest.raises(expected_exception=QueryException): - node.safe_psql('select 1') - - def test_safe_psql__expect_error(self): - with __class__.helper__get_node().init().start() as node: - err = node.safe_psql('select_or_not_select 1', expect_error=True) - assert (type(err) == str) # noqa: E721 - assert ('select_or_not_select' in err) - assert ('ERROR: syntax error at or near "select_or_not_select"' in err) - - # --------- - with pytest.raises( - expected_exception=InvalidOperationException, - match="^" + re.escape("Exception was expected, but query finished successfully: `select 1;`.") + "$" - ): - node.safe_psql("select 1;", expect_error=True) - - # --------- - res = node.safe_psql("select 1;", expect_error=False) - assert (res == b'1\n') - - def test_transactions(self): - with __class__.helper__get_node().init().start() as node: - with node.connect() as con: - con.begin() - con.execute('create table test(val int)') - con.execute('insert into test values (1)') - con.commit() - - con.begin() - con.execute('insert into test values (2)') - res = con.execute('select * from test order by val asc') - assert (res == [(1,), (2,)]) - con.rollback() - - con.begin() - res = con.execute('select * from test') - assert (res == [(1,)]) - con.rollback() - - con.begin() - con.execute('drop table test') - con.commit() - - def test_control_data(self): - with __class__.helper__get_node() as node: - # node is not initialized yet - with pytest.raises(expected_exception=ExecUtilException): - node.get_control_data() - - node.init() - data = node.get_control_data() - - # check returned dict - assert data is not None - assert (any('pg_control' in s for s in data.keys())) - - def test_backup_simple(self): - with __class__.helper__get_node() as master: - # enable streaming for backups - master.init(allow_streaming=True) - - # node must be running - with pytest.raises(expected_exception=BackupException): - master.backup() - - # it's time to start node - master.start() - - # fill node with some data - master.psql('create table test as select generate_series(1, 4) i') - - with master.backup(xlog_method='stream') as backup: - with backup.spawn_primary().start() as slave: - res = slave.execute('select * from test order by i asc') - assert (res == [(1,), (2,), (3,), (4,)]) - - def test_backup_multiple(self): - with __class__.helper__get_node() as node: - node.init(allow_streaming=True).start() - - with node.backup(xlog_method='fetch') as backup1, \ - node.backup(xlog_method='fetch') as backup2: - assert (backup1.base_dir != backup2.base_dir) - - with node.backup(xlog_method='fetch') as backup: - with backup.spawn_primary('node1', destroy=False) as node1, \ - backup.spawn_primary('node2', destroy=False) as node2: - assert (node1.base_dir != node2.base_dir) - - def test_backup_exhaust(self): - with __class__.helper__get_node() as node: - node.init(allow_streaming=True).start() - - with node.backup(xlog_method='fetch') as backup: - # exhaust backup by creating new node - with backup.spawn_primary(): - pass - - # now let's try to create one more node - with pytest.raises(expected_exception=BackupException): - backup.spawn_primary() - - def test_backup_wrong_xlog_method(self): - with __class__.helper__get_node() as node: - node.init(allow_streaming=True).start() - - with pytest.raises( - expected_exception=BackupException, - match="^" + re.escape('Invalid xlog_method "wrong"') + "$" - ): - node.backup(xlog_method='wrong') - - def test_pg_ctl_wait_option(self): - C_MAX_ATTEMPTS = 50 - - node = __class__.helper__get_node() - assert node.status() == testgres.NodeStatus.Uninitialized - node.init() - assert node.status() == testgres.NodeStatus.Stopped - node.start(wait=False) - nAttempt = 0 - while True: - if nAttempt == C_MAX_ATTEMPTS: - # - # [2025-03-11] - # We have an unexpected problem with this test in CI - # Let's get an additional information about this test failure. - # - logging.error("Node was not stopped.") - if not node.os_ops.path_exists(node.pg_log_file): - logging.warning("Node log does not exist.") - else: - logging.info("Let's read node log file [{0}]".format(node.pg_log_file)) - logFileData = node.os_ops.read(node.pg_log_file, binary=False) - logging.info("Node log file content:\n{0}".format(logFileData)) - - raise Exception("Could not stop node.") - - nAttempt += 1 - - if nAttempt > 1: - logging.info("Wait 1 second.") - time.sleep(1) - logging.info("") - - logging.info("Try to stop node. Attempt #{0}.".format(nAttempt)) - - try: - node.stop(wait=False) - break - except ExecUtilException as e: - # it's ok to get this exception here since node - # could be not started yet - logging.info("Node is not stopped. Exception ({0}): {1}".format(type(e).__name__, e)) - continue - - logging.info("OK. Stop command was executed. Let's wait while our node will stop really.") - nAttempt = 0 - while True: - if nAttempt == C_MAX_ATTEMPTS: - raise Exception("Could not stop node.") - - nAttempt += 1 - if nAttempt > 1: - logging.info("Wait 1 second.") - time.sleep(1) - logging.info("") - - logging.info("Attempt #{0}.".format(nAttempt)) - s1 = node.status() - - if s1 == testgres.NodeStatus.Running: - continue - - if s1 == testgres.NodeStatus.Stopped: - break - - raise Exception("Unexpected node status: {0}.".format(s1)) - - logging.info("OK. Node is stopped.") - node.cleanup() - - def test_replicate(self): - with __class__.helper__get_node() as node: - node.init(allow_streaming=True).start() - - with node.replicate().start() as replica: - res = replica.execute('select 1') - assert (res == [(1,)]) - - node.execute('create table test (val int)', commit=True) - - replica.catchup() - - res = node.execute('select * from test') - assert (res == []) - - def test_synchronous_replication(self): - __class__.helper__skip_test_if_pg_version_is_not_ge("9.6") - - with __class__.helper__get_node() as master: - old_version = not pg_version_ge('9.6') - - master.init(allow_streaming=True).start() - - if not old_version: - master.append_conf('synchronous_commit = remote_apply') - - # create standby - with master.replicate() as standby1, master.replicate() as standby2: - standby1.start() - standby2.start() - - # check formatting - assert ( - '1 ("{}", "{}")'.format(standby1.name, standby2.name) == str(First(1, (standby1, standby2))) - ) # yapf: disable - assert ( - 'ANY 1 ("{}", "{}")'.format(standby1.name, standby2.name) == str(Any(1, (standby1, standby2))) - ) # yapf: disable - - # set synchronous_standby_names - master.set_synchronous_standbys(First(2, [standby1, standby2])) - master.restart() - - # the following part of the test is only applicable to newer - # versions of PostgresQL - if not old_version: - master.safe_psql('create table abc(a int)') - - # Create a large transaction that will take some time to apply - # on standby to check that it applies synchronously - # (If set synchronous_commit to 'on' or other lower level then - # standby most likely won't catchup so fast and test will fail) - master.safe_psql( - 'insert into abc select generate_series(1, 1000000)') - res = standby1.safe_psql('select count(*) from abc') - assert (res == b'1000000\n') - - def test_logical_replication(self): - __class__.helper__skip_test_if_pg_version_is_not_ge("10") - - with __class__.helper__get_node() as node1, __class__.helper__get_node() as node2: - node1.init(allow_logical=True) - node1.start() - node2.init().start() - - create_table = 'create table test (a int, b int)' - node1.safe_psql(create_table) - node2.safe_psql(create_table) - - # create publication / create subscription - pub = node1.publish('mypub') - sub = node2.subscribe(pub, 'mysub') - - node1.safe_psql('insert into test values (1, 1), (2, 2)') - - # wait until changes apply on subscriber and check them - sub.catchup() - res = node2.execute('select * from test') - assert (res == [(1, 1), (2, 2)]) - - # disable and put some new data - sub.disable() - node1.safe_psql('insert into test values (3, 3)') - - # enable and ensure that data successfully transferred - sub.enable() - sub.catchup() - res = node2.execute('select * from test') - assert (res == [(1, 1), (2, 2), (3, 3)]) - - # Add new tables. Since we added "all tables" to publication - # (default behaviour of publish() method) we don't need - # to explicitly perform pub.add_tables() - create_table = 'create table test2 (c char)' - node1.safe_psql(create_table) - node2.safe_psql(create_table) - sub.refresh() - - # put new data - node1.safe_psql('insert into test2 values (\'a\'), (\'b\')') - sub.catchup() - res = node2.execute('select * from test2') - assert (res == [('a',), ('b',)]) - - # drop subscription - sub.drop() - pub.drop() - - # create new publication and subscription for specific table - # (omitting copying data as it's already done) - pub = node1.publish('newpub', tables=['test']) - sub = node2.subscribe(pub, 'newsub', copy_data=False) - - node1.safe_psql('insert into test values (4, 4)') - sub.catchup() - res = node2.execute('select * from test') - assert (res == [(1, 1), (2, 2), (3, 3), (4, 4)]) - - # explicitly add table - with pytest.raises(expected_exception=ValueError): - pub.add_tables([]) # fail - pub.add_tables(['test2']) - node1.safe_psql('insert into test2 values (\'c\')') - sub.catchup() - res = node2.execute('select * from test2') - assert (res == [('a',), ('b',)]) - - def test_logical_catchup(self): - """ Runs catchup for 100 times to be sure that it is consistent """ - __class__.helper__skip_test_if_pg_version_is_not_ge("10") - - with __class__.helper__get_node() as node1, __class__.helper__get_node() as node2: - node1.init(allow_logical=True) - node1.start() - node2.init().start() - - create_table = 'create table test (key int primary key, val int); ' - node1.safe_psql(create_table) - node1.safe_psql('alter table test replica identity default') - node2.safe_psql(create_table) - - # create publication / create subscription - sub = node2.subscribe(node1.publish('mypub'), 'mysub') - - for i in range(0, 100): - node1.execute('insert into test values ({0}, {0})'.format(i)) - sub.catchup() - res = node2.execute('select * from test') - assert (res == [(i, i, )]) - node1.execute('delete from test') - - def test_logical_replication_fail(self): - __class__.helper__skip_test_if_pg_version_is_ge("10") - - with __class__.helper__get_node() as node: - with pytest.raises(expected_exception=InitNodeException): - node.init(allow_logical=True) - - def test_replication_slots(self): - with __class__.helper__get_node() as node: - node.init(allow_streaming=True).start() - - with node.replicate(slot='slot1').start() as replica: - replica.execute('select 1') - - # cannot create new slot with the same name - with pytest.raises(expected_exception=TestgresException): - node.replicate(slot='slot1') - - def test_incorrect_catchup(self): - with __class__.helper__get_node() as node: - node.init(allow_streaming=True).start() - - # node has no master, can't catch up - with pytest.raises(expected_exception=TestgresException): - node.catchup() - - def test_promotion(self): - with __class__.helper__get_node() as master: - master.init().start() - master.safe_psql('create table abc(id serial)') - - with master.replicate().start() as replica: - master.stop() - replica.promote() - - # make standby becomes writable master - replica.safe_psql('insert into abc values (1)') - res = replica.safe_psql('select * from abc') - assert (res == b'1\n') - - def test_dump(self): - query_create = 'create table test as select generate_series(1, 2) as val' - query_select = 'select * from test order by val asc' - - with __class__.helper__get_node().init().start() as node1: - - node1.execute(query_create) - for format in ['plain', 'custom', 'directory', 'tar']: - with removing(node1.dump(format=format)) as dump: - with __class__.helper__get_node().init().start() as node3: - if format == 'directory': - assert (node1.os_ops.isdir(dump)) - else: - assert (node1.os_ops.isfile(dump)) - # restore dump - node3.restore(filename=dump) - res = node3.execute(query_select) - assert (res == [(1,), (2,)]) - - def test_users(self): - with __class__.helper__get_node().init().start() as node: - node.psql('create role test_user login') - value = node.safe_psql('select 1', username='test_user') - assert (b'1\n' == value) - - def test_poll_query_until(self): - with __class__.helper__get_node() as node: - node.init().start() - - get_time = 'select extract(epoch from now())' - check_time = 'select extract(epoch from now()) - {} >= 5' - - start_time = node.execute(get_time)[0][0] - node.poll_query_until(query=check_time.format(start_time)) - end_time = node.execute(get_time)[0][0] - - assert (end_time - start_time >= 5) - - # check 0 columns - with pytest.raises(expected_exception=QueryException): - node.poll_query_until( - query='select from pg_catalog.pg_class limit 1') - - # check None, fail - with pytest.raises(expected_exception=QueryException): - node.poll_query_until(query='create table abc (val int)') - - # check None, ok - node.poll_query_until(query='create table def()', - expected=None) # returns nothing - - # check 0 rows equivalent to expected=None - node.poll_query_until( - query='select * from pg_catalog.pg_class where true = false', - expected=None) - - # check arbitrary expected value, fail - with pytest.raises(expected_exception=TimeoutException): - node.poll_query_until(query='select 3', - expected=1, - max_attempts=3, - sleep_time=0.01) - - # check arbitrary expected value, ok - node.poll_query_until(query='select 2', expected=2) - - # check timeout - with pytest.raises(expected_exception=TimeoutException): - node.poll_query_until(query='select 1 > 2', - max_attempts=3, - sleep_time=0.01) - - # check ProgrammingError, fail - with pytest.raises(expected_exception=testgres.ProgrammingError): - node.poll_query_until(query='dummy1') - - # check ProgrammingError, ok - with pytest.raises(expected_exception=TimeoutException): - node.poll_query_until(query='dummy2', - max_attempts=3, - sleep_time=0.01, - suppress={testgres.ProgrammingError}) - - # check 1 arg, ok - node.poll_query_until('select true') - - def test_logging(self): - C_MAX_ATTEMPTS = 50 - # This name is used for testgres logging, too. - C_NODE_NAME = "testgres_tests." + __class__.__name__ + "test_logging-master-" + uuid.uuid4().hex - - logging.info("Node name is [{0}]".format(C_NODE_NAME)) - - with tempfile.NamedTemporaryFile('w', delete=True) as logfile: - formatter = logging.Formatter(fmt="%(node)-5s: %(message)s") - handler = logging.FileHandler(filename=logfile.name) - handler.formatter = formatter - logger = logging.getLogger(C_NODE_NAME) - assert logger is not None - assert len(logger.handlers) == 0 - - try: - # It disables to log on the root level - logger.propagate = False - logger.addHandler(handler) - - with scoped_config(use_python_logging=True): - with __class__.helper__get_node(name=C_NODE_NAME) as master: - logging.info("Master node is initilizing") - master.init() - - logging.info("Master node is starting") - master.start() - - logging.info("Dummy query is executed a few times") - for _ in range(20): - master.execute('select 1') - time.sleep(0.01) - - # let logging worker do the job - time.sleep(0.1) - - logging.info("Master node log file is checking") - nAttempt = 0 - - while True: - assert nAttempt <= C_MAX_ATTEMPTS - if nAttempt == C_MAX_ATTEMPTS: - raise Exception("Test failed!") - - # let logging worker do the job - time.sleep(0.1) - - nAttempt += 1 - - logging.info("Attempt {0}".format(nAttempt)) - - # check that master's port is found - with open(logfile.name, 'r') as log: - lines = log.readlines() - - assert lines is not None - assert type(lines) == list # noqa: E721 - - def LOCAL__test_lines(): - for s in lines: - if any(C_NODE_NAME in s for s in lines): - logging.info("OK. We found the node_name in a line \"{0}\"".format(s)) - return True - return False - - if LOCAL__test_lines(): - break - - logging.info("Master node log file does not have an expected information.") - continue - - # test logger after stop/start/restart - logging.info("Master node is stopping...") - master.stop() - logging.info("Master node is staring again...") - master.start() - logging.info("Master node is restaring...") - master.restart() - assert (master._logger.is_alive()) - finally: - # It is a hack code to logging cleanup - logging._acquireLock() - assert logging.Logger.manager is not None - assert C_NODE_NAME in logging.Logger.manager.loggerDict.keys() - logging.Logger.manager.loggerDict.pop(C_NODE_NAME, None) - assert not (C_NODE_NAME in logging.Logger.manager.loggerDict.keys()) - assert not (handler in logging._handlers.values()) - logging._releaseLock() - # GO HOME! - return - def test_pgbench(self): __class__.helper__skip_test_if_util_not_exist("pgbench") @@ -1031,60 +259,6 @@ def test_unix_sockets(self): assert (res_exec == [(1,)]) assert (res_psql == b'1\n') - def test_auto_name(self): - with __class__.helper__get_node().init(allow_streaming=True).start() as m: - with m.replicate().start() as r: - # check that nodes are running - assert (m.status()) - assert (r.status()) - - # check their names - assert (m.name != r.name) - assert ('testgres' in m.name) - assert ('testgres' in r.name) - - def test_file_tail(self): - s1 = "the quick brown fox jumped over that lazy dog\n" - s2 = "abc\n" - s3 = "def\n" - - with tempfile.NamedTemporaryFile(mode='r+', delete=True) as f: - sz = 0 - while sz < 3 * 8192: - sz += len(s1) - f.write(s1) - f.write(s2) - f.write(s3) - - f.seek(0) - lines = file_tail(f, 3) - assert (lines[0] == s1) - assert (lines[1] == s2) - assert (lines[2] == s3) - - f.seek(0) - lines = file_tail(f, 1) - assert (lines[0] == s3) - - def test_isolation_levels(self): - with __class__.helper__get_node().init().start() as node: - with node.connect() as con: - # string levels - con.begin('Read Uncommitted').commit() - con.begin('Read Committed').commit() - con.begin('Repeatable Read').commit() - con.begin('Serializable').commit() - - # enum levels - con.begin(IsolationLevel.ReadUncommitted).commit() - con.begin(IsolationLevel.ReadCommitted).commit() - con.begin(IsolationLevel.RepeatableRead).commit() - con.begin(IsolationLevel.Serializable).commit() - - # check wrong level - with pytest.raises(expected_exception=QueryException): - con.begin('Garbage').commit() - def test_ports_management(self): assert bound_ports is not None assert type(bound_ports) == set # noqa: E721 @@ -1119,145 +293,6 @@ def test_ports_management(self): assert type(bound_ports) == set # noqa: E721 assert bound_ports == stage0__bound_ports - def test_exceptions(self): - str(StartNodeException('msg', [('file', 'lines')])) - str(ExecUtilException('msg', 'cmd', 1, 'out')) - str(QueryException('msg', 'query')) - - def test_version_management(self): - a = PgVer('10.0') - b = PgVer('10') - c = PgVer('9.6.5') - d = PgVer('15.0') - e = PgVer('15rc1') - f = PgVer('15beta4') - - assert (a == b) - assert (b > c) - assert (a > c) - assert (d > e) - assert (e > f) - assert (d > f) - - version = get_pg_version() - with __class__.helper__get_node() as node: - assert (isinstance(version, six.string_types)) - assert (isinstance(node.version, PgVer)) - assert (node.version == PgVer(version)) - - def test_child_pids(self): - master_processes = [ - ProcessType.AutovacuumLauncher, - ProcessType.BackgroundWriter, - ProcessType.Checkpointer, - ProcessType.StatsCollector, - ProcessType.WalSender, - ProcessType.WalWriter, - ] - - if pg_version_ge('10'): - master_processes.append(ProcessType.LogicalReplicationLauncher) - - if pg_version_ge('14'): - master_processes.remove(ProcessType.StatsCollector) - - repl_processes = [ - ProcessType.Startup, - ProcessType.WalReceiver, - ] - - def LOCAL__test_auxiliary_pids( - node: testgres.PostgresNode, - expectedTypes: list[ProcessType] - ) -> list[ProcessType]: - # returns list of the absence processes - assert node is not None - assert type(node) == testgres.PostgresNode # noqa: E721 - assert expectedTypes is not None - assert type(expectedTypes) == list # noqa: E721 - - pids = node.auxiliary_pids - assert pids is not None # noqa: E721 - assert type(pids) == dict # noqa: E721 - - result = list[ProcessType]() - for ptype in expectedTypes: - if not (ptype in pids): - result.append(ptype) - return result - - def LOCAL__check_auxiliary_pids__multiple_attempts( - node: testgres.PostgresNode, - expectedTypes: list[ProcessType]): - assert node is not None - assert type(node) == testgres.PostgresNode # noqa: E721 - assert expectedTypes is not None - assert type(expectedTypes) == list # noqa: E721 - - nAttempt = 0 - - while nAttempt < 5: - nAttempt += 1 - - logging.info("Test pids of [{0}] node. Attempt #{1}.".format( - node.name, - nAttempt - )) - - if nAttempt > 1: - time.sleep(1) - - absenceList = LOCAL__test_auxiliary_pids(node, expectedTypes) - assert absenceList is not None - assert type(absenceList) == list # noqa: E721 - if len(absenceList) == 0: - logging.info("Bingo!") - return - - logging.info("These processes are not found: {0}.".format(absenceList)) - continue - - raise Exception("Node {0} does not have the following processes: {1}.".format( - node.name, - absenceList - )) - - with __class__.helper__get_node().init().start() as master: - - # master node doesn't have a source walsender! - with pytest.raises(expected_exception=TestgresException): - master.source_walsender - - with master.connect() as con: - assert (con.pid > 0) - - with master.replicate().start() as replica: - - # test __str__ method - str(master.child_processes[0]) - - LOCAL__check_auxiliary_pids__multiple_attempts( - master, - master_processes) - - LOCAL__check_auxiliary_pids__multiple_attempts( - replica, - repl_processes) - - master_pids = master.auxiliary_pids - - # there should be exactly 1 source walsender for replica - assert (len(master_pids[ProcessType.WalSender]) == 1) - pid1 = master_pids[ProcessType.WalSender][0] - pid2 = replica.source_walsender.pid - assert (pid1 == pid2) - - replica.stop() - - # there should be no walsender after we've stopped replica - with pytest.raises(expected_exception=TestgresException): - replica.source_walsender - # TODO: Why does not this test work with remote host? def test_child_process_dies(self): nAttempt = 0 @@ -1290,8 +325,8 @@ def test_child_process_dies(self): @staticmethod def helper__get_node(name=None): - assert __class__.sm_conn_params is not None - return get_remote_node(name=name, conn_params=__class__.sm_conn_params) + assert isinstance(__class__.sm_os_ops, OsOperations) + return testgres.PostgresNode(name, conn_params=None, os_ops=__class__.sm_os_ops) @staticmethod def helper__restore_envvar(name, prev_value): @@ -1305,15 +340,3 @@ def helper__skip_test_if_util_not_exist(name: str): assert type(name) == str # noqa: E721 if not util_exists(name): pytest.skip('might be missing') - - @staticmethod - def helper__skip_test_if_pg_version_is_not_ge(version: str): - assert type(version) == str # noqa: E721 - if not pg_version_ge(version): - pytest.skip('requires {0}+'.format(version)) - - @staticmethod - def helper__skip_test_if_pg_version_is_ge(version: str): - assert type(version) == str # noqa: E721 - if pg_version_ge(version): - pytest.skip('requires <{0}'.format(version)) diff --git a/tests/test_testgres_common.py b/tests/test_testgres_common.py new file mode 100644 index 00000000..49740b61 --- /dev/null +++ b/tests/test_testgres_common.py @@ -0,0 +1,1131 @@ +from .helpers.os_ops_descrs import OsOpsDescr +from .helpers.os_ops_descrs import OsOpsDescrs +from .helpers.os_ops_descrs import OsOperations + +from ..testgres.node import PgVer +from ..testgres.node import PostgresNode +from ..testgres.utils import get_pg_version2 +from ..testgres.utils import file_tail +from ..testgres.utils import get_bin_path2 +from ..testgres import ProcessType +from ..testgres import NodeStatus +from ..testgres import IsolationLevel +from ..testgres import TestgresException +from ..testgres import InitNodeException +from ..testgres import StartNodeException +from ..testgres import QueryException +from ..testgres import ExecUtilException +from ..testgres import TimeoutException +from ..testgres import InvalidOperationException +from ..testgres import BackupException +from ..testgres import ProgrammingError +from ..testgres import scoped_config +from ..testgres import First, Any + +from contextlib import contextmanager + +import pytest +import six +import logging +import time +import tempfile +import uuid +import os +import re + + +@contextmanager +def removing(os_ops: OsOperations, f): + assert isinstance(os_ops, OsOperations) + + try: + yield f + finally: + if os_ops.isfile(f): + os_ops.remove_file(f) + + elif os_ops.isdir(f): + os_ops.rmdirs(f, ignore_errors=True) + + +class TestTestgresCommon: + sm_os_ops_descrs: list[OsOpsDescr] = [ + OsOpsDescrs.sm_local_os_ops_descr, + OsOpsDescrs.sm_remote_os_ops_descr + ] + + @pytest.fixture( + params=[descr.os_ops for descr in sm_os_ops_descrs], + ids=[descr.sign for descr in sm_os_ops_descrs] + ) + def os_ops(self, request: pytest.FixtureRequest) -> OsOperations: + assert isinstance(request, pytest.FixtureRequest) + assert isinstance(request.param, OsOperations) + return request.param + + def test_version_management(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + + a = PgVer('10.0') + b = PgVer('10') + c = PgVer('9.6.5') + d = PgVer('15.0') + e = PgVer('15rc1') + f = PgVer('15beta4') + h = PgVer('15.3biha') + i = PgVer('15.3') + g = PgVer('15.3.1bihabeta1') + k = PgVer('15.3.1') + + assert (a == b) + assert (b > c) + assert (a > c) + assert (d > e) + assert (e > f) + assert (d > f) + assert (h > f) + assert (h == i) + assert (g == k) + assert (g > h) + + version = get_pg_version2(os_ops) + + with __class__.helper__get_node(os_ops) as node: + assert (isinstance(version, six.string_types)) + assert (isinstance(node.version, PgVer)) + assert (node.version == PgVer(version)) + + def test_double_init(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + + with __class__.helper__get_node(os_ops).init() as node: + # can't initialize node more than once + with pytest.raises(expected_exception=InitNodeException): + node.init() + + def test_init_after_cleanup(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + + with __class__.helper__get_node(os_ops) as node: + node.init().start().execute('select 1') + node.cleanup() + node.init().start().execute('select 1') + + def test_init_unique_system_id(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + + # this function exists in PostgreSQL 9.6+ + current_version = get_pg_version2(os_ops) + + __class__.helper__skip_test_if_util_not_exist(os_ops, "pg_resetwal") + __class__.helper__skip_test_if_pg_version_is_not_ge(current_version, '9.6') + + query = 'select system_identifier from pg_control_system()' + + with scoped_config(cache_initdb=False): + with __class__.helper__get_node(os_ops).init().start() as node0: + id0 = node0.execute(query)[0] + + with scoped_config(cache_initdb=True, + cached_initdb_unique=True) as config: + assert (config.cache_initdb) + assert (config.cached_initdb_unique) + + # spawn two nodes; ids must be different + with __class__.helper__get_node(os_ops).init().start() as node1, \ + __class__.helper__get_node(os_ops).init().start() as node2: + id1 = node1.execute(query)[0] + id2 = node2.execute(query)[0] + + # ids must increase + assert (id1 > id0) + assert (id2 > id1) + + def test_node_exit(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + + with pytest.raises(expected_exception=QueryException): + with __class__.helper__get_node(os_ops).init() as node: + base_dir = node.base_dir + node.safe_psql('select 1') + + # we should save the DB for "debugging" + assert (os_ops.path_exists(base_dir)) + os_ops.rmdirs(base_dir, ignore_errors=True) + + with __class__.helper__get_node(os_ops).init() as node: + base_dir = node.base_dir + + # should have been removed by default + assert not (os_ops.path_exists(base_dir)) + + def test_double_start(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + + with __class__.helper__get_node(os_ops).init().start() as node: + # can't start node more than once + node.start() + assert (node.is_started) + + def test_uninitialized_start(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + + with __class__.helper__get_node(os_ops) as node: + # node is not initialized yet + with pytest.raises(expected_exception=StartNodeException): + node.start() + + def test_restart(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + + with __class__.helper__get_node(os_ops) as node: + node.init().start() + + # restart, ok + res = node.execute('select 1') + assert (res == [(1,)]) + node.restart() + res = node.execute('select 2') + assert (res == [(2,)]) + + # restart, fail + with pytest.raises(expected_exception=StartNodeException): + node.append_conf('pg_hba.conf', 'DUMMY') + node.restart() + + def test_reload(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + + with __class__.helper__get_node(os_ops) as node: + node.init().start() + + # change client_min_messages and save old value + cmm_old = node.execute('show client_min_messages') + node.append_conf(client_min_messages='DEBUG1') + + # reload config + node.reload() + + # check new value + cmm_new = node.execute('show client_min_messages') + assert ('debug1' == cmm_new[0][0].lower()) + assert (cmm_old != cmm_new) + + def test_pg_ctl(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + + with __class__.helper__get_node(os_ops) as node: + node.init().start() + + status = node.pg_ctl(['status']) + assert ('PID' in status) + + def test_status(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + + assert (NodeStatus.Running) + assert not (NodeStatus.Stopped) + assert not (NodeStatus.Uninitialized) + + # check statuses after each operation + with __class__.helper__get_node(os_ops) as node: + assert (node.pid == 0) + assert (node.status() == NodeStatus.Uninitialized) + + node.init() + + assert (node.pid == 0) + assert (node.status() == NodeStatus.Stopped) + + node.start() + + assert (node.pid != 0) + assert (node.status() == NodeStatus.Running) + + node.stop() + + assert (node.pid == 0) + assert (node.status() == NodeStatus.Stopped) + + node.cleanup() + + assert (node.pid == 0) + assert (node.status() == NodeStatus.Uninitialized) + + def test_child_pids(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + + master_processes = [ + ProcessType.AutovacuumLauncher, + ProcessType.BackgroundWriter, + ProcessType.Checkpointer, + ProcessType.StatsCollector, + ProcessType.WalSender, + ProcessType.WalWriter, + ] + + postgresVersion = get_pg_version2(os_ops) + + if __class__.helper__pg_version_ge(postgresVersion, '10'): + master_processes.append(ProcessType.LogicalReplicationLauncher) + + if __class__.helper__pg_version_ge(postgresVersion, '14'): + master_processes.remove(ProcessType.StatsCollector) + + repl_processes = [ + ProcessType.Startup, + ProcessType.WalReceiver, + ] + + def LOCAL__test_auxiliary_pids( + node: PostgresNode, + expectedTypes: list[ProcessType] + ) -> list[ProcessType]: + # returns list of the absence processes + assert node is not None + assert type(node) == PostgresNode # noqa: E721 + assert expectedTypes is not None + assert type(expectedTypes) == list # noqa: E721 + + pids = node.auxiliary_pids + assert pids is not None # noqa: E721 + assert type(pids) == dict # noqa: E721 + + result = list[ProcessType]() + for ptype in expectedTypes: + if not (ptype in pids): + result.append(ptype) + return result + + def LOCAL__check_auxiliary_pids__multiple_attempts( + node: PostgresNode, + expectedTypes: list[ProcessType]): + assert node is not None + assert type(node) == PostgresNode # noqa: E721 + assert expectedTypes is not None + assert type(expectedTypes) == list # noqa: E721 + + nAttempt = 0 + + while nAttempt < 5: + nAttempt += 1 + + logging.info("Test pids of [{0}] node. Attempt #{1}.".format( + node.name, + nAttempt + )) + + if nAttempt > 1: + time.sleep(1) + + absenceList = LOCAL__test_auxiliary_pids(node, expectedTypes) + assert absenceList is not None + assert type(absenceList) == list # noqa: E721 + if len(absenceList) == 0: + logging.info("Bingo!") + return + + logging.info("These processes are not found: {0}.".format(absenceList)) + continue + + raise Exception("Node {0} does not have the following processes: {1}.".format( + node.name, + absenceList + )) + + with __class__.helper__get_node(os_ops).init().start() as master: + + # master node doesn't have a source walsender! + with pytest.raises(expected_exception=TestgresException): + master.source_walsender + + with master.connect() as con: + assert (con.pid > 0) + + with master.replicate().start() as replica: + + # test __str__ method + str(master.child_processes[0]) + + LOCAL__check_auxiliary_pids__multiple_attempts( + master, + master_processes) + + LOCAL__check_auxiliary_pids__multiple_attempts( + replica, + repl_processes) + + master_pids = master.auxiliary_pids + + # there should be exactly 1 source walsender for replica + assert (len(master_pids[ProcessType.WalSender]) == 1) + pid1 = master_pids[ProcessType.WalSender][0] + pid2 = replica.source_walsender.pid + assert (pid1 == pid2) + + replica.stop() + + # there should be no walsender after we've stopped replica + with pytest.raises(expected_exception=TestgresException): + replica.source_walsender + + def test_exceptions(self): + str(StartNodeException('msg', [('file', 'lines')])) + str(ExecUtilException('msg', 'cmd', 1, 'out')) + str(QueryException('msg', 'query')) + + def test_auto_name(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + + with __class__.helper__get_node(os_ops).init(allow_streaming=True).start() as m: + with m.replicate().start() as r: + # check that nodes are running + assert (m.status()) + assert (r.status()) + + # check their names + assert (m.name != r.name) + assert ('testgres' in m.name) + assert ('testgres' in r.name) + + def test_file_tail(self): + s1 = "the quick brown fox jumped over that lazy dog\n" + s2 = "abc\n" + s3 = "def\n" + + with tempfile.NamedTemporaryFile(mode='r+', delete=True) as f: + sz = 0 + while sz < 3 * 8192: + sz += len(s1) + f.write(s1) + f.write(s2) + f.write(s3) + + f.seek(0) + lines = file_tail(f, 3) + assert (lines[0] == s1) + assert (lines[1] == s2) + assert (lines[2] == s3) + + f.seek(0) + lines = file_tail(f, 1) + assert (lines[0] == s3) + + def test_isolation_levels(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + with __class__.helper__get_node(os_ops).init().start() as node: + with node.connect() as con: + # string levels + con.begin('Read Uncommitted').commit() + con.begin('Read Committed').commit() + con.begin('Repeatable Read').commit() + con.begin('Serializable').commit() + + # enum levels + con.begin(IsolationLevel.ReadUncommitted).commit() + con.begin(IsolationLevel.ReadCommitted).commit() + con.begin(IsolationLevel.RepeatableRead).commit() + con.begin(IsolationLevel.Serializable).commit() + + # check wrong level + with pytest.raises(expected_exception=QueryException): + con.begin('Garbage').commit() + + def test_users(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + with __class__.helper__get_node(os_ops).init().start() as node: + node.psql('create role test_user login') + value = node.safe_psql('select 1', username='test_user') + value = __class__.helper__rm_carriage_returns(value) + assert (value == b'1\n') + + def test_poll_query_until(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + with __class__.helper__get_node(os_ops) as node: + node.init().start() + + get_time = 'select extract(epoch from now())' + check_time = 'select extract(epoch from now()) - {} >= 5' + + start_time = node.execute(get_time)[0][0] + node.poll_query_until(query=check_time.format(start_time)) + end_time = node.execute(get_time)[0][0] + + assert (end_time - start_time >= 5) + + # check 0 columns + with pytest.raises(expected_exception=QueryException): + node.poll_query_until( + query='select from pg_catalog.pg_class limit 1') + + # check None, fail + with pytest.raises(expected_exception=QueryException): + node.poll_query_until(query='create table abc (val int)') + + # check None, ok + node.poll_query_until(query='create table def()', + expected=None) # returns nothing + + # check 0 rows equivalent to expected=None + node.poll_query_until( + query='select * from pg_catalog.pg_class where true = false', + expected=None) + + # check arbitrary expected value, fail + with pytest.raises(expected_exception=TimeoutException): + node.poll_query_until(query='select 3', + expected=1, + max_attempts=3, + sleep_time=0.01) + + # check arbitrary expected value, ok + node.poll_query_until(query='select 2', expected=2) + + # check timeout + with pytest.raises(expected_exception=TimeoutException): + node.poll_query_until(query='select 1 > 2', + max_attempts=3, + sleep_time=0.01) + + # check ProgrammingError, fail + with pytest.raises(expected_exception=ProgrammingError): + node.poll_query_until(query='dummy1') + + # check ProgrammingError, ok + with pytest.raises(expected_exception=(TimeoutException)): + node.poll_query_until(query='dummy2', + max_attempts=3, + sleep_time=0.01, + suppress={ProgrammingError}) + + # check 1 arg, ok + node.poll_query_until('select true') + + def test_logging(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + C_MAX_ATTEMPTS = 50 + # This name is used for testgres logging, too. + C_NODE_NAME = "testgres_tests." + __class__.__name__ + "test_logging-master-" + uuid.uuid4().hex + + logging.info("Node name is [{0}]".format(C_NODE_NAME)) + + with tempfile.NamedTemporaryFile('w', delete=True) as logfile: + formatter = logging.Formatter(fmt="%(node)-5s: %(message)s") + handler = logging.FileHandler(filename=logfile.name) + handler.formatter = formatter + logger = logging.getLogger(C_NODE_NAME) + assert logger is not None + assert len(logger.handlers) == 0 + + try: + # It disables to log on the root level + logger.propagate = False + logger.addHandler(handler) + + with scoped_config(use_python_logging=True): + with __class__.helper__get_node(os_ops, name=C_NODE_NAME) as master: + logging.info("Master node is initilizing") + master.init() + + logging.info("Master node is starting") + master.start() + + logging.info("Dummy query is executed a few times") + for _ in range(20): + master.execute('select 1') + time.sleep(0.01) + + # let logging worker do the job + time.sleep(0.1) + + logging.info("Master node log file is checking") + nAttempt = 0 + + while True: + assert nAttempt <= C_MAX_ATTEMPTS + if nAttempt == C_MAX_ATTEMPTS: + raise Exception("Test failed!") + + # let logging worker do the job + time.sleep(0.1) + + nAttempt += 1 + + logging.info("Attempt {0}".format(nAttempt)) + + # check that master's port is found + with open(logfile.name, 'r') as log: + lines = log.readlines() + + assert lines is not None + assert type(lines) == list # noqa: E721 + + def LOCAL__test_lines(): + for s in lines: + if any(C_NODE_NAME in s for s in lines): + logging.info("OK. We found the node_name in a line \"{0}\"".format(s)) + return True + return False + + if LOCAL__test_lines(): + break + + logging.info("Master node log file does not have an expected information.") + continue + + # test logger after stop/start/restart + logging.info("Master node is stopping...") + master.stop() + logging.info("Master node is staring again...") + master.start() + logging.info("Master node is restaring...") + master.restart() + assert (master._logger.is_alive()) + finally: + # It is a hack code to logging cleanup + logging._acquireLock() + assert logging.Logger.manager is not None + assert C_NODE_NAME in logging.Logger.manager.loggerDict.keys() + logging.Logger.manager.loggerDict.pop(C_NODE_NAME, None) + assert not (C_NODE_NAME in logging.Logger.manager.loggerDict.keys()) + assert not (handler in logging._handlers.values()) + logging._releaseLock() + # GO HOME! + return + + def test_psql(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + with __class__.helper__get_node(os_ops).init().start() as node: + + # check returned values (1 arg) + res = node.psql('select 1') + assert (__class__.helper__rm_carriage_returns(res) == (0, b'1\n', b'')) + + # check returned values (2 args) + res = node.psql('postgres', 'select 2') + assert (__class__.helper__rm_carriage_returns(res) == (0, b'2\n', b'')) + + # check returned values (named) + res = node.psql(query='select 3', dbname='postgres') + assert (__class__.helper__rm_carriage_returns(res) == (0, b'3\n', b'')) + + # check returned values (1 arg) + res = node.safe_psql('select 4') + assert (__class__.helper__rm_carriage_returns(res) == b'4\n') + + # check returned values (2 args) + res = node.safe_psql('postgres', 'select 5') + assert (__class__.helper__rm_carriage_returns(res) == b'5\n') + + # check returned values (named) + res = node.safe_psql(query='select 6', dbname='postgres') + assert (__class__.helper__rm_carriage_returns(res) == b'6\n') + + # check feeding input + node.safe_psql('create table horns (w int)') + node.safe_psql('copy horns from stdin (format csv)', + input=b"1\n2\n3\n\\.\n") + _sum = node.safe_psql('select sum(w) from horns') + assert (__class__.helper__rm_carriage_returns(_sum) == b'6\n') + + # check psql's default args, fails + with pytest.raises(expected_exception=QueryException): + node.psql() + + node.stop() + + # check psql on stopped node, fails + with pytest.raises(expected_exception=QueryException): + node.safe_psql('select 1') + + def test_safe_psql__expect_error(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + with __class__.helper__get_node(os_ops).init().start() as node: + err = node.safe_psql('select_or_not_select 1', expect_error=True) + assert (type(err) == str) # noqa: E721 + assert ('select_or_not_select' in err) + assert ('ERROR: syntax error at or near "select_or_not_select"' in err) + + # --------- + with pytest.raises( + expected_exception=InvalidOperationException, + match="^" + re.escape("Exception was expected, but query finished successfully: `select 1;`.") + "$" + ): + node.safe_psql("select 1;", expect_error=True) + + # --------- + res = node.safe_psql("select 1;", expect_error=False) + assert (__class__.helper__rm_carriage_returns(res) == b'1\n') + + def test_transactions(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + with __class__.helper__get_node(os_ops).init().start() as node: + + with node.connect() as con: + con.begin() + con.execute('create table test(val int)') + con.execute('insert into test values (1)') + con.commit() + + con.begin() + con.execute('insert into test values (2)') + res = con.execute('select * from test order by val asc') + assert (res == [(1, ), (2, )]) + con.rollback() + + con.begin() + res = con.execute('select * from test') + assert (res == [(1, )]) + con.rollback() + + con.begin() + con.execute('drop table test') + con.commit() + + def test_control_data(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + with __class__.helper__get_node(os_ops) as node: + + # node is not initialized yet + with pytest.raises(expected_exception=ExecUtilException): + node.get_control_data() + + node.init() + data = node.get_control_data() + + # check returned dict + assert data is not None + assert (any('pg_control' in s for s in data.keys())) + + def test_backup_simple(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + with __class__.helper__get_node(os_ops) as master: + + # enable streaming for backups + master.init(allow_streaming=True) + + # node must be running + with pytest.raises(expected_exception=BackupException): + master.backup() + + # it's time to start node + master.start() + + # fill node with some data + master.psql('create table test as select generate_series(1, 4) i') + + with master.backup(xlog_method='stream') as backup: + with backup.spawn_primary().start() as slave: + res = slave.execute('select * from test order by i asc') + assert (res == [(1, ), (2, ), (3, ), (4, )]) + + def test_backup_multiple(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + with __class__.helper__get_node(os_ops) as node: + node.init(allow_streaming=True).start() + + with node.backup(xlog_method='fetch') as backup1, \ + node.backup(xlog_method='fetch') as backup2: + assert (backup1.base_dir != backup2.base_dir) + + with node.backup(xlog_method='fetch') as backup: + with backup.spawn_primary('node1', destroy=False) as node1, \ + backup.spawn_primary('node2', destroy=False) as node2: + assert (node1.base_dir != node2.base_dir) + + def test_backup_exhaust(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + with __class__.helper__get_node(os_ops) as node: + node.init(allow_streaming=True).start() + + with node.backup(xlog_method='fetch') as backup: + # exhaust backup by creating new node + with backup.spawn_primary(): + pass + + # now let's try to create one more node + with pytest.raises(expected_exception=BackupException): + backup.spawn_primary() + + def test_backup_wrong_xlog_method(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + with __class__.helper__get_node(os_ops) as node: + node.init(allow_streaming=True).start() + + with pytest.raises( + expected_exception=BackupException, + match="^" + re.escape('Invalid xlog_method "wrong"') + "$" + ): + node.backup(xlog_method='wrong') + + def test_pg_ctl_wait_option(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + C_MAX_ATTEMPTS = 50 + + node = __class__.helper__get_node(os_ops) + assert node.status() == NodeStatus.Uninitialized + node.init() + assert node.status() == NodeStatus.Stopped + node.start(wait=False) + nAttempt = 0 + while True: + if nAttempt == C_MAX_ATTEMPTS: + # + # [2025-03-11] + # We have an unexpected problem with this test in CI + # Let's get an additional information about this test failure. + # + logging.error("Node was not stopped.") + if not node.os_ops.path_exists(node.pg_log_file): + logging.warning("Node log does not exist.") + else: + logging.info("Let's read node log file [{0}]".format(node.pg_log_file)) + logFileData = node.os_ops.read(node.pg_log_file, binary=False) + logging.info("Node log file content:\n{0}".format(logFileData)) + + raise Exception("Could not stop node.") + + nAttempt += 1 + + if nAttempt > 1: + logging.info("Wait 1 second.") + time.sleep(1) + logging.info("") + + logging.info("Try to stop node. Attempt #{0}.".format(nAttempt)) + + try: + node.stop(wait=False) + break + except ExecUtilException as e: + # it's ok to get this exception here since node + # could be not started yet + logging.info("Node is not stopped. Exception ({0}): {1}".format(type(e).__name__, e)) + continue + + logging.info("OK. Stop command was executed. Let's wait while our node will stop really.") + nAttempt = 0 + while True: + if nAttempt == C_MAX_ATTEMPTS: + raise Exception("Could not stop node.") + + nAttempt += 1 + if nAttempt > 1: + logging.info("Wait 1 second.") + time.sleep(1) + logging.info("") + + logging.info("Attempt #{0}.".format(nAttempt)) + s1 = node.status() + + if s1 == NodeStatus.Running: + continue + + if s1 == NodeStatus.Stopped: + break + + raise Exception("Unexpected node status: {0}.".format(s1)) + + logging.info("OK. Node is stopped.") + node.cleanup() + + def test_replicate(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + with __class__.helper__get_node(os_ops) as node: + node.init(allow_streaming=True).start() + + with node.replicate().start() as replica: + res = replica.execute('select 1') + assert (res == [(1, )]) + + node.execute('create table test (val int)', commit=True) + + replica.catchup() + + res = node.execute('select * from test') + assert (res == []) + + def test_synchronous_replication(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + + current_version = get_pg_version2(os_ops) + + __class__.helper__skip_test_if_pg_version_is_not_ge(current_version, "9.6") + + with __class__.helper__get_node(os_ops) as master: + old_version = not __class__.helper__pg_version_ge(current_version, '9.6') + + master.init(allow_streaming=True).start() + + if not old_version: + master.append_conf('synchronous_commit = remote_apply') + + # create standby + with master.replicate() as standby1, master.replicate() as standby2: + standby1.start() + standby2.start() + + # check formatting + assert ( + '1 ("{}", "{}")'.format(standby1.name, standby2.name) == str(First(1, (standby1, standby2))) + ) # yapf: disable + assert ( + 'ANY 1 ("{}", "{}")'.format(standby1.name, standby2.name) == str(Any(1, (standby1, standby2))) + ) # yapf: disable + + # set synchronous_standby_names + master.set_synchronous_standbys(First(2, [standby1, standby2])) + master.restart() + + # the following part of the test is only applicable to newer + # versions of PostgresQL + if not old_version: + master.safe_psql('create table abc(a int)') + + # Create a large transaction that will take some time to apply + # on standby to check that it applies synchronously + # (If set synchronous_commit to 'on' or other lower level then + # standby most likely won't catchup so fast and test will fail) + master.safe_psql( + 'insert into abc select generate_series(1, 1000000)') + res = standby1.safe_psql('select count(*) from abc') + assert (__class__.helper__rm_carriage_returns(res) == b'1000000\n') + + def test_logical_replication(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + + current_version = get_pg_version2(os_ops) + + __class__.helper__skip_test_if_pg_version_is_not_ge(current_version, "10") + + with __class__.helper__get_node(os_ops) as node1, __class__.helper__get_node(os_ops) as node2: + node1.init(allow_logical=True) + node1.start() + node2.init().start() + + create_table = 'create table test (a int, b int)' + node1.safe_psql(create_table) + node2.safe_psql(create_table) + + # create publication / create subscription + pub = node1.publish('mypub') + sub = node2.subscribe(pub, 'mysub') + + node1.safe_psql('insert into test values (1, 1), (2, 2)') + + # wait until changes apply on subscriber and check them + sub.catchup() + res = node2.execute('select * from test') + assert (res == [(1, 1), (2, 2)]) + + # disable and put some new data + sub.disable() + node1.safe_psql('insert into test values (3, 3)') + + # enable and ensure that data successfully transferred + sub.enable() + sub.catchup() + res = node2.execute('select * from test') + assert (res == [(1, 1), (2, 2), (3, 3)]) + + # Add new tables. Since we added "all tables" to publication + # (default behaviour of publish() method) we don't need + # to explicitly perform pub.add_tables() + create_table = 'create table test2 (c char)' + node1.safe_psql(create_table) + node2.safe_psql(create_table) + sub.refresh() + + # put new data + node1.safe_psql('insert into test2 values (\'a\'), (\'b\')') + sub.catchup() + res = node2.execute('select * from test2') + assert (res == [('a', ), ('b', )]) + + # drop subscription + sub.drop() + pub.drop() + + # create new publication and subscription for specific table + # (omitting copying data as it's already done) + pub = node1.publish('newpub', tables=['test']) + sub = node2.subscribe(pub, 'newsub', copy_data=False) + + node1.safe_psql('insert into test values (4, 4)') + sub.catchup() + res = node2.execute('select * from test') + assert (res == [(1, 1), (2, 2), (3, 3), (4, 4)]) + + # explicitly add table + with pytest.raises(expected_exception=ValueError): + pub.add_tables([]) # fail + pub.add_tables(['test2']) + node1.safe_psql('insert into test2 values (\'c\')') + sub.catchup() + res = node2.execute('select * from test2') + assert (res == [('a', ), ('b', )]) + + def test_logical_catchup(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + """ Runs catchup for 100 times to be sure that it is consistent """ + + current_version = get_pg_version2(os_ops) + + __class__.helper__skip_test_if_pg_version_is_not_ge(current_version, "10") + + with __class__.helper__get_node(os_ops) as node1, __class__.helper__get_node(os_ops) as node2: + node1.init(allow_logical=True) + node1.start() + node2.init().start() + + create_table = 'create table test (key int primary key, val int); ' + node1.safe_psql(create_table) + node1.safe_psql('alter table test replica identity default') + node2.safe_psql(create_table) + + # create publication / create subscription + sub = node2.subscribe(node1.publish('mypub'), 'mysub') + + for i in range(0, 100): + node1.execute('insert into test values ({0}, {0})'.format(i)) + sub.catchup() + res = node2.execute('select * from test') + assert (res == [(i, i, )]) + node1.execute('delete from test') + + def test_logical_replication_fail(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + + current_version = get_pg_version2(os_ops) + + __class__.helper__skip_test_if_pg_version_is_ge(current_version, "10") + + with __class__.helper__get_node(os_ops) as node: + with pytest.raises(expected_exception=InitNodeException): + node.init(allow_logical=True) + + def test_replication_slots(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + with __class__.helper__get_node(os_ops) as node: + node.init(allow_streaming=True).start() + + with node.replicate(slot='slot1').start() as replica: + replica.execute('select 1') + + # cannot create new slot with the same name + with pytest.raises(expected_exception=TestgresException): + node.replicate(slot='slot1') + + def test_incorrect_catchup(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + with __class__.helper__get_node(os_ops) as node: + node.init(allow_streaming=True).start() + + # node has no master, can't catch up + with pytest.raises(expected_exception=TestgresException): + node.catchup() + + def test_promotion(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + with __class__.helper__get_node(os_ops) as master: + master.init().start() + master.safe_psql('create table abc(id serial)') + + with master.replicate().start() as replica: + master.stop() + replica.promote() + + # make standby becomes writable master + replica.safe_psql('insert into abc values (1)') + res = replica.safe_psql('select * from abc') + assert (__class__.helper__rm_carriage_returns(res) == b'1\n') + + def test_dump(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + query_create = 'create table test as select generate_series(1, 2) as val' + query_select = 'select * from test order by val asc' + + with __class__.helper__get_node(os_ops).init().start() as node1: + + node1.execute(query_create) + for format in ['plain', 'custom', 'directory', 'tar']: + with removing(os_ops, node1.dump(format=format)) as dump: + with __class__.helper__get_node(os_ops).init().start() as node3: + if format == 'directory': + assert (os.path.isdir(dump)) + else: + assert (os.path.isfile(dump)) + # restore dump + node3.restore(filename=dump) + res = node3.execute(query_select) + assert (res == [(1, ), (2, )]) + + @staticmethod + def helper__get_node(os_ops: OsOperations, name=None): + assert isinstance(os_ops, OsOperations) + return PostgresNode(name, conn_params=None, os_ops=os_ops) + + @staticmethod + def helper__skip_test_if_pg_version_is_not_ge(ver1: str, ver2: str): + assert type(ver1) == str # noqa: E721 + assert type(ver2) == str # noqa: E721 + if not __class__.helper__pg_version_ge(ver1, ver2): + pytest.skip('requires {0}+'.format(ver2)) + + @staticmethod + def helper__skip_test_if_pg_version_is_ge(ver1: str, ver2: str): + assert type(ver1) == str # noqa: E721 + assert type(ver2) == str # noqa: E721 + if __class__.helper__pg_version_ge(ver1, ver2): + pytest.skip('requires <{0}'.format(ver2)) + + @staticmethod + def helper__pg_version_ge(ver1: str, ver2: str) -> bool: + assert type(ver1) == str # noqa: E721 + assert type(ver2) == str # noqa: E721 + v1 = PgVer(ver1) + v2 = PgVer(ver2) + return v1 >= v2 + + @staticmethod + def helper__rm_carriage_returns(out): + """ + In Windows we have additional '\r' symbols in output. + Let's get rid of them. + """ + if isinstance(out, (int, float, complex)): + return out + + if isinstance(out, tuple): + return tuple(__class__.helper__rm_carriage_returns(item) for item in out) + + if isinstance(out, bytes): + return out.replace(b'\r', b'') + + assert type(out) == str # noqa: E721 + return out.replace('\r', '') + + @staticmethod + def helper__skip_test_if_util_not_exist(os_ops: OsOperations, name: str): + assert isinstance(os_ops, OsOperations) + assert type(name) == str # noqa: E721 + if not __class__.helper__util_exists(os_ops, name): + pytest.skip('might be missing') + + @staticmethod + def helper__util_exists(os_ops: OsOperations, util): + assert isinstance(os_ops, OsOperations) + + def good_properties(f): + return (os_ops.path_exists(f) and # noqa: W504 + os_ops.isfile(f) and # noqa: W504 + os_ops.is_executable(f)) # yapf: disable + + # try to resolve it + if good_properties(get_bin_path2(os_ops, util)): + return True + + # check if util is in PATH + for path in os_ops.environ("PATH").split(os.pathsep): + if good_properties(os.path.join(path, util)): + return True From 0b2c629318ea98333b6824fcade369efa349df34 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Mon, 17 Mar 2025 17:07:41 +0300 Subject: [PATCH 166/216] os_ops_descrs.py is added --- tests/helpers/os_ops_descrs.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 tests/helpers/os_ops_descrs.py diff --git a/tests/helpers/os_ops_descrs.py b/tests/helpers/os_ops_descrs.py new file mode 100644 index 00000000..02297adb --- /dev/null +++ b/tests/helpers/os_ops_descrs.py @@ -0,0 +1,32 @@ +from ...testgres.operations.os_ops import OsOperations +from ...testgres.operations.os_ops import ConnectionParams +from ...testgres.operations.local_ops import LocalOperations +from ...testgres.operations.remote_ops import RemoteOperations + +import os + + +class OsOpsDescr: + os_ops: OsOperations + sign: str + + def __init__(self, os_ops: OsOperations, sign: str): + assert isinstance(os_ops, OsOperations) + assert type(sign) == str # noqa: E721 + self.os_ops = os_ops + self.sign = sign + + +class OsOpsDescrs: + sm_remote_conn_params = ConnectionParams( + host=os.getenv('RDBMS_TESTPOOL1_HOST') or '127.0.0.1', + username=os.getenv('USER'), + ssh_key=os.getenv('RDBMS_TESTPOOL_SSHKEY')) + + sm_remote_os_ops = RemoteOperations(sm_remote_conn_params) + + sm_remote_os_ops_descr = OsOpsDescr(sm_remote_os_ops, "remote_ops") + + sm_local_os_ops = LocalOperations() + + sm_local_os_ops_descr = OsOpsDescr(sm_local_os_ops, "local_ops") From b597bf893633dd5e6701c925f8fd70b1f7fda128 Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Tue, 18 Mar 2025 19:25:11 +0300 Subject: [PATCH 167/216] Warnings with pytest are fixed (#223) 1) [pytest.ini] testpaths has another format. It is a spaces separated list. pytest warning: PytestConfigWarning: No files were found in testpaths; consider removing or adjusting your testpaths configuration. Searching recursively from the current directory instead. 2) pytest tries to find the test function in TestgresException class. Let's rename it to avoid this problem. pytest warning: PytestCollectionWarning: cannot collect test class 'TestgresException' because it has a __init__ constructor (from: tests/test_simple.py) class TestgresException(Exception): Of course, we can add __test__=False in TestgresException but it is not a good solution. --- pytest.ini | 2 +- tests/test_testgres_common.py | 14 +++++++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/pytest.ini b/pytest.ini index c94eabc2..9f5fa375 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,5 +1,5 @@ [pytest] -testpaths = ["./tests", "./testgres/plugins/pg_probackup2/pg_probackup2/tests"] +testpaths = tests testgres/plugins/pg_probackup2/pg_probackup2/tests addopts = --strict-markers markers = #log_file = logs/pytest.log diff --git a/tests/test_testgres_common.py b/tests/test_testgres_common.py index 49740b61..f42964c8 100644 --- a/tests/test_testgres_common.py +++ b/tests/test_testgres_common.py @@ -10,7 +10,11 @@ from ..testgres import ProcessType from ..testgres import NodeStatus from ..testgres import IsolationLevel -from ..testgres import TestgresException + +# New name prevents to collect test-functions in TestgresException and fixes +# the problem with pytest warning. +from ..testgres import TestgresException as testgres_TestgresException + from ..testgres import InitNodeException from ..testgres import StartNodeException from ..testgres import QueryException @@ -336,7 +340,7 @@ def LOCAL__check_auxiliary_pids__multiple_attempts( with __class__.helper__get_node(os_ops).init().start() as master: # master node doesn't have a source walsender! - with pytest.raises(expected_exception=TestgresException): + with pytest.raises(expected_exception=testgres_TestgresException): master.source_walsender with master.connect() as con: @@ -366,7 +370,7 @@ def LOCAL__check_auxiliary_pids__multiple_attempts( replica.stop() # there should be no walsender after we've stopped replica - with pytest.raises(expected_exception=TestgresException): + with pytest.raises(expected_exception=testgres_TestgresException): replica.source_walsender def test_exceptions(self): @@ -1013,7 +1017,7 @@ def test_replication_slots(self, os_ops: OsOperations): replica.execute('select 1') # cannot create new slot with the same name - with pytest.raises(expected_exception=TestgresException): + with pytest.raises(expected_exception=testgres_TestgresException): node.replicate(slot='slot1') def test_incorrect_catchup(self, os_ops: OsOperations): @@ -1022,7 +1026,7 @@ def test_incorrect_catchup(self, os_ops: OsOperations): node.init(allow_streaming=True).start() # node has no master, can't catch up - with pytest.raises(expected_exception=TestgresException): + with pytest.raises(expected_exception=testgres_TestgresException): node.catchup() def test_promotion(self, os_ops: OsOperations): From 87dbecb3a1c190099b43fc1510c92a6baa35fea3 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 18 Mar 2025 19:27:44 +0300 Subject: [PATCH 168/216] OsOperations::remove_file is added It seems to me we forgot to add it. --- testgres/operations/os_ops.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/testgres/operations/os_ops.py b/testgres/operations/os_ops.py index 00880863..f20a7a30 100644 --- a/testgres/operations/os_ops.py +++ b/testgres/operations/os_ops.py @@ -108,6 +108,10 @@ def isdir(self, dirname): def get_file_size(self, filename): raise NotImplementedError() + def remove_file(self, filename): + assert type(filename) == str # noqa: E721 + raise NotImplementedError() + # Processes control def kill(self, pid, signal): # Kill the process From 25c6a2fe3d0557f0a60d719c74e2f044cf22f6ac Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Tue, 18 Mar 2025 21:34:25 +0300 Subject: [PATCH 169/216] Log files (#224) Let's start writing test events into files to provide an access to the tests execution information. --- .gitignore | 1 + tests/conftest.py | 510 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 511 insertions(+) create mode 100644 tests/conftest.py diff --git a/.gitignore b/.gitignore index 038d1952..238181b5 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ dist/ build/ docs/build/ +logs/ env/ venv/ diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 00000000..e37c3c77 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,510 @@ +# ///////////////////////////////////////////////////////////////////////////// +# PyTest Configuration + +import _pytest.outcomes + +import pluggy +import pytest +import _pytest +import os +import logging +import pathlib +import math +import datetime + +# ///////////////////////////////////////////////////////////////////////////// + +C_ROOT_DIR__RELATIVE = ".." + +# ///////////////////////////////////////////////////////////////////////////// +# TestConfigPropNames + + +class TestConfigPropNames: + TEST_CFG__LOG_DIR = "TEST_CFG__LOG_DIR" + + +# ///////////////////////////////////////////////////////////////////////////// +# TestStartupData__Helper + + +class TestStartupData__Helper: + sm_StartTS = datetime.datetime.now() + + # -------------------------------------------------------------------- + def GetStartTS() -> datetime.datetime: + assert type(__class__.sm_StartTS) == datetime.datetime # noqa: E721 + return __class__.sm_StartTS + + # -------------------------------------------------------------------- + def CalcRootDir() -> str: + r = os.path.abspath(__file__) + r = os.path.dirname(r) + r = os.path.join(r, C_ROOT_DIR__RELATIVE) + r = os.path.abspath(r) + return r + + # -------------------------------------------------------------------- + def CalcCurrentTestWorkerSignature() -> str: + currentPID = os.getpid() + assert type(currentPID) + + startTS = __class__.sm_StartTS + assert type(startTS) + + result = "pytest-{0:04d}{1:02d}{2:02d}_{3:02d}{4:02d}{5:02d}".format( + startTS.year, + startTS.month, + startTS.day, + startTS.hour, + startTS.minute, + startTS.second, + ) + + gwid = os.environ.get("PYTEST_XDIST_WORKER") + + if gwid is not None: + result += "--xdist_" + str(gwid) + + result += "--" + "pid" + str(currentPID) + return result + + +# ///////////////////////////////////////////////////////////////////////////// +# TestStartupData + + +class TestStartupData: + sm_RootDir: str = TestStartupData__Helper.CalcRootDir() + sm_CurrentTestWorkerSignature: str = ( + TestStartupData__Helper.CalcCurrentTestWorkerSignature() + ) + + # -------------------------------------------------------------------- + def GetRootDir() -> str: + assert type(__class__.sm_RootDir) == str # noqa: E721 + return __class__.sm_RootDir + + # -------------------------------------------------------------------- + def GetCurrentTestWorkerSignature() -> str: + assert type(__class__.sm_CurrentTestWorkerSignature) == str # noqa: E721 + return __class__.sm_CurrentTestWorkerSignature + + +# /////////////////////////////////////////////////////////////////////////////# ///////////////////////////////////////////////////////////////////////////// +# Fixtures + + +# ///////////////////////////////////////////////////////////////////////////// +# TEST_PROCESS_STATS + + +class TEST_PROCESS_STATS: + cTotalTests: int = 0 + cNotExecutedTests: int = 0 + cExecutedTests: int = 0 + cPassedTests: int = 0 + cFailedTests: int = 0 + cXFailedTests: int = 0 + cSkippedTests: int = 0 + cNotXFailedTests: int = 0 + cUnexpectedTests: int = 0 + + FailedTests = list[str]() + XFailedTests = list[str]() + NotXFailedTests = list[str]() + + # -------------------------------------------------------------------- + def incrementTotalTestCount() -> None: + __class__.cTotalTests += 1 + + # -------------------------------------------------------------------- + def incrementNotExecutedTestCount() -> None: + __class__.cNotExecutedTests += 1 + + # -------------------------------------------------------------------- + def incrementExecutedTestCount() -> int: + __class__.cExecutedTests += 1 + return __class__.cExecutedTests + + # -------------------------------------------------------------------- + def incrementPassedTestCount() -> None: + __class__.cPassedTests += 1 + + # -------------------------------------------------------------------- + def incrementFailedTestCount(testID: str) -> None: + assert type(testID) == str # noqa: E721 + assert type(__class__.FailedTests) == list # noqa: E721 + + __class__.FailedTests.append(testID) # raise? + __class__.cFailedTests += 1 + + # -------------------------------------------------------------------- + def incrementXFailedTestCount(testID: str) -> None: + assert type(testID) == str # noqa: E721 + assert type(__class__.XFailedTests) == list # noqa: E721 + + __class__.XFailedTests.append(testID) # raise? + __class__.cXFailedTests += 1 + + # -------------------------------------------------------------------- + def incrementSkippedTestCount() -> None: + __class__.cSkippedTests += 1 + + # -------------------------------------------------------------------- + def incrementNotXFailedTests(testID: str) -> None: + assert type(testID) == str # noqa: E721 + assert type(__class__.NotXFailedTests) == list # noqa: E721 + + __class__.NotXFailedTests.append(testID) # raise? + __class__.cNotXFailedTests += 1 + + # -------------------------------------------------------------------- + def incrementUnexpectedTests() -> None: + __class__.cUnexpectedTests += 1 + + +# ///////////////////////////////////////////////////////////////////////////// + + +def timedelta_to_human_text(delta: datetime.timedelta) -> str: + assert isinstance(delta, datetime.timedelta) + + C_SECONDS_IN_MINUTE = 60 + C_SECONDS_IN_HOUR = 60 * C_SECONDS_IN_MINUTE + + v = delta.seconds + + cHours = int(v / C_SECONDS_IN_HOUR) + v = v - cHours * C_SECONDS_IN_HOUR + cMinutes = int(v / C_SECONDS_IN_MINUTE) + cSeconds = v - cMinutes * C_SECONDS_IN_MINUTE + + result = "" if delta.days == 0 else "{0} day(s) ".format(delta.days) + + result = result + "{:02d}:{:02d}:{:02d}.{:06d}".format( + cHours, cMinutes, cSeconds, delta.microseconds + ) + + return result + + +# ///////////////////////////////////////////////////////////////////////////// + + +def helper__makereport__setup( + item: pytest.Function, call: pytest.CallInfo, outcome: pluggy.Result +): + assert item is not None + assert call is not None + assert outcome is not None + assert type(item) == pytest.Function # noqa: E721 + assert type(call) == pytest.CallInfo # noqa: E721 + assert type(outcome) == pluggy.Result # noqa: E721 + + # logging.info("pytest_runtest_makereport - setup") + + TEST_PROCESS_STATS.incrementTotalTestCount() + + rep: pytest.TestReport = outcome.get_result() + assert rep is not None + assert type(rep) == pytest.TestReport # noqa: E721 + + if rep.outcome == "skipped": + TEST_PROCESS_STATS.incrementNotExecutedTestCount() + return + + assert rep.outcome == "passed" + + testNumber = TEST_PROCESS_STATS.incrementExecutedTestCount() + + testID = "" + + if item.cls is not None: + testID = item.cls.__module__ + "." + item.cls.__name__ + "::" + + testID = testID + item.name + + if testNumber > 1: + logging.info("") + + logging.info("******************************************************") + logging.info("* START TEST {0}".format(testID)) + logging.info("*") + logging.info("* Path : {0}".format(item.path)) + logging.info("* Number: {0}".format(testNumber)) + logging.info("*") + + +# ------------------------------------------------------------------------ +def helper__makereport__call( + item: pytest.Function, call: pytest.CallInfo, outcome: pluggy.Result +): + assert item is not None + assert call is not None + assert outcome is not None + assert type(item) == pytest.Function # noqa: E721 + assert type(call) == pytest.CallInfo # noqa: E721 + assert type(outcome) == pluggy.Result # noqa: E721 + + # logging.info("pytest_runtest_makereport - call") + + rep = outcome.get_result() + assert rep is not None + assert type(rep) == pytest.TestReport # noqa: E721 + + # -------- + testID = "" + + if item.cls is not None: + testID = item.cls.__module__ + "." + item.cls.__name__ + "::" + + testID = testID + item.name + + # -------- + assert call.start <= call.stop + + startDT = datetime.datetime.fromtimestamp(call.start) + assert type(startDT) == datetime.datetime # noqa: E721 + stopDT = datetime.datetime.fromtimestamp(call.stop) + assert type(stopDT) == datetime.datetime # noqa: E721 + + testDurration = stopDT - startDT + assert type(testDurration) == datetime.timedelta # noqa: E721 + + # -------- + exitStatus = None + if rep.outcome == "skipped": + assert call.excinfo is not None # research + assert call.excinfo.value is not None # research + + if type(call.excinfo.value) == _pytest.outcomes.Skipped: # noqa: E721 + assert not hasattr(rep, "wasxfail") + + TEST_PROCESS_STATS.incrementSkippedTestCount() + + exitStatus = "SKIPPED" + reasonText = str(call.excinfo.value) + reasonMsg = "SKIP REASON: {0}" + + elif type(call.excinfo.value) == _pytest.outcomes.XFailed: # noqa: E721 + TEST_PROCESS_STATS.incrementXFailedTestCount(testID) + + exitStatus = "XFAILED" + reasonText = str(call.excinfo.value) + reasonMsg = "XFAIL REASON: {0}" + else: + exitStatus = "XFAILED" + assert hasattr(rep, "wasxfail") + assert rep.wasxfail is not None + assert type(rep.wasxfail) == str # noqa: E721 + + TEST_PROCESS_STATS.incrementXFailedTestCount(testID) + + reasonText = rep.wasxfail + reasonMsg = "XFAIL REASON: {0}" + + logging.error(call.excinfo.value) + + if reasonText != "": + logging.info("*") + logging.info("* " + reasonMsg.format(reasonText)) + + elif rep.outcome == "failed": + assert call.excinfo is not None + assert call.excinfo.value is not None + + TEST_PROCESS_STATS.incrementFailedTestCount(testID) + + logging.error(call.excinfo.value) + exitStatus = "FAILED" + elif rep.outcome == "passed": + assert call.excinfo is None + + if hasattr(rep, "wasxfail"): + assert type(rep.wasxfail) == str # noqa: E721 + + TEST_PROCESS_STATS.incrementNotXFailedTests(testID) + + warnMsg = "Test is marked as xfail" + + if rep.wasxfail != "": + warnMsg += " [" + rep.wasxfail + "]" + + logging.warning(warnMsg) + exitStatus = "NOT XFAILED" + else: + assert not hasattr(rep, "wasxfail") + + TEST_PROCESS_STATS.incrementPassedTestCount() + exitStatus = "PASSED" + else: + TEST_PROCESS_STATS.incrementUnexpectedTests() + exitStatus = "UNEXPECTED [{0}]".format(rep.outcome) + assert False + + # -------- + logging.info("*") + logging.info("* DURATION : {0}".format(timedelta_to_human_text(testDurration))) + logging.info("*") + logging.info("* EXIT STATUS : {0}".format(exitStatus)) + logging.info("*") + logging.info("* STOP TEST {0}".format(testID)) + + +# ///////////////////////////////////////////////////////////////////////////// + + +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_makereport(item: pytest.Function, call: pytest.CallInfo): + assert item is not None + assert call is not None + assert type(item) == pytest.Function # noqa: E721 + assert type(call) == pytest.CallInfo # noqa: E721 + + # logging.info("[pytest_runtest_makereport][#001][{0}][{1}]".format(item.name, call.when)) + + outcome: pluggy.Result = yield + assert outcome is not None + assert type(outcome) == pluggy.Result # noqa: E721 + + # logging.info("[pytest_runtest_makereport][#002][{0}][{1}]".format(item.name, call.when)) + + rep: pytest.TestReport = outcome.get_result() + assert rep is not None + assert type(rep) == pytest.TestReport # noqa: E721 + + if call.when == "collect": + return + + if call.when == "setup": + helper__makereport__setup(item, call, outcome) + return + + if call.when == "call": + helper__makereport__call(item, call, outcome) + return + + if call.when == "teardown": + return + + assert False + + +# ///////////////////////////////////////////////////////////////////////////// + + +def helper__calc_W(n: int) -> int: + assert n > 0 + + x = int(math.log10(n)) + assert type(x) == int # noqa: E721 + assert x >= 0 + x += 1 + return x + + +# ------------------------------------------------------------------------ +def helper__print_test_list(tests: list[str]) -> None: + assert type(tests) == list # noqa: E721 + + assert helper__calc_W(9) == 1 + assert helper__calc_W(10) == 2 + assert helper__calc_W(11) == 2 + assert helper__calc_W(99) == 2 + assert helper__calc_W(100) == 3 + assert helper__calc_W(101) == 3 + assert helper__calc_W(999) == 3 + assert helper__calc_W(1000) == 4 + assert helper__calc_W(1001) == 4 + + W = helper__calc_W(len(tests)) + + templateLine = "{0:0" + str(W) + "d}. {1}" + + nTest = 0 + + while nTest < len(tests): + testID = tests[nTest] + assert type(testID) == str # noqa: E721 + nTest += 1 + logging.info(templateLine.format(nTest, testID)) + + +# ///////////////////////////////////////////////////////////////////////////// + + +@pytest.fixture(autouse=True, scope="session") +def run_after_tests(request: pytest.FixtureRequest): + assert isinstance(request, pytest.FixtureRequest) + + yield + + logging.info("--------------------------- [FAILED TESTS]") + logging.info("") + + assert len(TEST_PROCESS_STATS.FailedTests) == TEST_PROCESS_STATS.cFailedTests + + if len(TEST_PROCESS_STATS.FailedTests) > 0: + helper__print_test_list(TEST_PROCESS_STATS.FailedTests) + logging.info("") + + logging.info("--------------------------- [XFAILED TESTS]") + logging.info("") + + assert len(TEST_PROCESS_STATS.XFailedTests) == TEST_PROCESS_STATS.cXFailedTests + + if len(TEST_PROCESS_STATS.XFailedTests) > 0: + helper__print_test_list(TEST_PROCESS_STATS.XFailedTests) + logging.info("") + + logging.info("--------------------------- [NOT XFAILED TESTS]") + logging.info("") + + assert ( + len(TEST_PROCESS_STATS.NotXFailedTests) == TEST_PROCESS_STATS.cNotXFailedTests + ) + + if len(TEST_PROCESS_STATS.NotXFailedTests) > 0: + helper__print_test_list(TEST_PROCESS_STATS.NotXFailedTests) + logging.info("") + + logging.info("--------------------------- [SUMMARY STATISTICS]") + logging.info("") + logging.info("[TESTS]") + logging.info(" TOTAL : {0}".format(TEST_PROCESS_STATS.cTotalTests)) + logging.info(" EXECUTED : {0}".format(TEST_PROCESS_STATS.cExecutedTests)) + logging.info(" NOT EXECUTED: {0}".format(TEST_PROCESS_STATS.cNotExecutedTests)) + logging.info("") + logging.info(" PASSED : {0}".format(TEST_PROCESS_STATS.cPassedTests)) + logging.info(" FAILED : {0}".format(TEST_PROCESS_STATS.cFailedTests)) + logging.info(" XFAILED : {0}".format(TEST_PROCESS_STATS.cXFailedTests)) + logging.info(" NOT XFAILED : {0}".format(TEST_PROCESS_STATS.cNotXFailedTests)) + logging.info(" SKIPPED : {0}".format(TEST_PROCESS_STATS.cSkippedTests)) + logging.info(" UNEXPECTED : {0}".format(TEST_PROCESS_STATS.cUnexpectedTests)) + logging.info("") + + +# ///////////////////////////////////////////////////////////////////////////// + + +@pytest.hookimpl(trylast=True) +def pytest_configure(config: pytest.Config) -> None: + assert isinstance(config, pytest.Config) + + log_name = TestStartupData.GetCurrentTestWorkerSignature() + log_name += ".log" + + if TestConfigPropNames.TEST_CFG__LOG_DIR in os.environ: + log_path_v = os.environ[TestConfigPropNames.TEST_CFG__LOG_DIR] + log_path = pathlib.Path(log_path_v) + else: + log_path = config.rootpath.joinpath("logs") + + log_path.mkdir(exist_ok=True) + + logging_plugin = config.pluginmanager.get_plugin("logging-plugin") + logging_plugin.set_log_path(str(log_path / log_name)) + + +# ///////////////////////////////////////////////////////////////////////////// From 81a5eb43b7398cd3c436ffe32b21e20c4c9712b4 Mon Sep 17 00:00:00 2001 From: dura0ok Date: Wed, 19 Mar 2025 23:08:05 +0700 Subject: [PATCH 170/216] add FUSE support to plugin pg_probackup2 (#184) --- .../pg_probackup2/pg_probackup2/app.py | 43 ++++++++++++++++--- 1 file changed, 37 insertions(+), 6 deletions(-) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/app.py b/testgres/plugins/pg_probackup2/pg_probackup2/app.py index 57492814..d47cf51f 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/app.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/app.py @@ -45,6 +45,7 @@ class ProbackupApp: def __init__(self, test_class: unittest.TestCase, pg_node, pb_log_path, test_env, auto_compress_alg, backup_dir, probackup_path=None): + self.process = None self.test_class = test_class self.pg_node = pg_node self.pb_log_path = pb_log_path @@ -60,8 +61,35 @@ def __init__(self, test_class: unittest.TestCase, self.test_class.output = None self.execution_time = None + def form_daemon_process(self, cmdline, env): + def stream_output(stream: subprocess.PIPE) -> None: + try: + for line in iter(stream.readline, ''): + print(line) + self.test_class.output += line + finally: + stream.close() + + self.process = subprocess.Popen( + cmdline, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=env + ) + logging.info(f"Process started in background with PID: {self.process.pid}") + + if self.process.stdout and self.process.stderr: + stdout_thread = threading.Thread(target=stream_output, args=(self.process.stdout,), daemon=True) + stderr_thread = threading.Thread(target=stream_output, args=(self.process.stderr,), daemon=True) + + stdout_thread.start() + stderr_thread.start() + + return self.process.pid + def run(self, command, gdb=False, old_binary=False, return_id=True, env=None, - skip_log_directory=False, expect_error=False, use_backup_dir=True): + skip_log_directory=False, expect_error=False, use_backup_dir=True, daemonize=False): """ Run pg_probackup backup_dir: target directory for making backup @@ -118,11 +146,14 @@ def run(self, command, gdb=False, old_binary=False, return_id=True, env=None, logging.warning("pg_probackup gdb suspended, waiting gdb connection on localhost:{0}".format(gdb_port)) start_time = time.time() - self.test_class.output = subprocess.check_output( - cmdline, - stderr=subprocess.STDOUT, - env=env - ).decode('utf-8', errors='replace') + if daemonize: + return self.form_daemon_process(cmdline, env) + else: + self.test_class.output = subprocess.check_output( + cmdline, + stderr=subprocess.STDOUT, + env=env + ).decode('utf-8', errors='replace') end_time = time.time() self.execution_time = end_time - start_time From ddfaff401f0f55b5a51bb0490d12dda1631acbe3 Mon Sep 17 00:00:00 2001 From: asavchkov Date: Thu, 20 Mar 2025 10:13:51 +0700 Subject: [PATCH 171/216] pg-probackup2 version 0.0.6 --- testgres/plugins/pg_probackup2/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testgres/plugins/pg_probackup2/setup.py b/testgres/plugins/pg_probackup2/setup.py index 619b8d39..8bcfe7b4 100644 --- a/testgres/plugins/pg_probackup2/setup.py +++ b/testgres/plugins/pg_probackup2/setup.py @@ -4,7 +4,7 @@ from distutils.core import setup setup( - version='0.0.5', + version='0.0.6', name='testgres_pg_probackup2', packages=['pg_probackup2', 'pg_probackup2.storage'], description='Plugin for testgres that manages pg_probackup2', From f0bf7a8994b5dd39e5c2f841381d5e4be5e22297 Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Fri, 21 Mar 2025 14:15:24 +0300 Subject: [PATCH 172/216] CI files are updated (#225) * [CI] work with 'time' is corrected AltLinux 10 does not support the sequential "time coverage run ...". Because this OS does not has a builtin command 'time' in bash. https://forum.altlinux.org/index.php?topic=48342.0 We will install 'time' manually and use another command " time coverage run ..." that works without problems but it requires to install 'time' on Ubuntu 2024.04, too. AlpineLinux processes a new command line without any problems. * [CI] An initization of python virtualenv is simplified Let's avoid creating useless environment variables. --- Dockerfile--ubuntu_24_04.tmpl | 1 + run_tests.sh | 27 +++++++++------------------ 2 files changed, 10 insertions(+), 18 deletions(-) diff --git a/Dockerfile--ubuntu_24_04.tmpl b/Dockerfile--ubuntu_24_04.tmpl index c1ddeab6..3bdc6640 100644 --- a/Dockerfile--ubuntu_24_04.tmpl +++ b/Dockerfile--ubuntu_24_04.tmpl @@ -9,6 +9,7 @@ RUN apt update RUN apt install -y sudo curl ca-certificates RUN apt update RUN apt install -y openssh-server +RUN apt install -y time RUN apt update RUN apt install -y postgresql-common diff --git a/run_tests.sh b/run_tests.sh index 0fecde60..a40a97cf 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -8,24 +8,17 @@ if [ -z ${TEST_FILTER+x} ]; \ then export TEST_FILTER="TestgresTests or (TestTestgresCommon and (not remote_ops))"; \ fi -# choose python version -echo python version is $PYTHON_VERSION -VIRTUALENV="virtualenv --python=/usr/bin/python$PYTHON_VERSION" -PIP="pip$PYTHON_VERSION" - # fail early echo check that pg_config is in PATH command -v pg_config -# prepare environment -VENV_PATH=/tmp/testgres_venv +# prepare python environment +VENV_PATH="/tmp/testgres_venv" rm -rf $VENV_PATH -$VIRTUALENV $VENV_PATH +virtualenv --python="/usr/bin/python${PYTHON_VERSION}" "${VENV_PATH}" export VIRTUAL_ENV_DISABLE_PROMPT=1 -source $VENV_PATH/bin/activate - -# install utilities -$PIP install coverage flake8 psutil Sphinx pytest pytest-xdist psycopg2 six psutil +source "${VENV_PATH}/bin/activate" +pip install coverage flake8 psutil Sphinx pytest pytest-xdist psycopg2 six psutil # install testgres' dependencies export PYTHONPATH=$(pwd) @@ -45,15 +38,13 @@ time coverage run -a -m pytest -l -v -n 4 -k "${TEST_FILTER}" # run tests (PG_BIN) -time \ - PG_BIN=$(pg_config --bindir) \ - coverage run -a -m pytest -l -v -n 4 -k "${TEST_FILTER}" +PG_BIN=$(pg_config --bindir) \ +time coverage run -a -m pytest -l -v -n 4 -k "${TEST_FILTER}" # run tests (PG_CONFIG) -time \ - PG_CONFIG=$(pg_config --bindir)/pg_config \ - coverage run -a -m pytest -l -v -n 4 -k "${TEST_FILTER}" +PG_CONFIG=$(pg_config --bindir)/pg_config \ +time coverage run -a -m pytest -l -v -n 4 -k "${TEST_FILTER}" # show coverage From 76fa94cfa6d2645abbb59a027074bd97b9f389ee Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Mon, 24 Mar 2025 13:04:41 +0300 Subject: [PATCH 173/216] [CI] Run tests on AltLinux 10 and 11 (#219) This patch adds an automated tests of testgres on AltLinux 10 and 11. We will execute only "local" tests because AltLinux has an unexpected problem with SSH connection - it is created too slowly. --- .travis.yml | 2 + Dockerfile--altlinux_10.tmpl | 118 +++++++++++++++++++++++++++++++++++ Dockerfile--altlinux_11.tmpl | 118 +++++++++++++++++++++++++++++++++++ 3 files changed, 238 insertions(+) create mode 100644 Dockerfile--altlinux_10.tmpl create mode 100644 Dockerfile--altlinux_11.tmpl diff --git a/.travis.yml b/.travis.yml index 3a889845..7557a2ce 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,3 +28,5 @@ env: - TEST_PLATFORM=std PYTHON_VERSION=3 PG_VERSION=10 - TEST_PLATFORM=std-all PYTHON_VERSION=3 PG_VERSION=17 - TEST_PLATFORM=ubuntu_24_04 PYTHON_VERSION=3 PG_VERSION=17 + - TEST_PLATFORM=altlinux_10 PYTHON_VERSION=3 PG_VERSION=17 + - TEST_PLATFORM=altlinux_11 PYTHON_VERSION=3 PG_VERSION=17 diff --git a/Dockerfile--altlinux_10.tmpl b/Dockerfile--altlinux_10.tmpl new file mode 100644 index 00000000..e60e9320 --- /dev/null +++ b/Dockerfile--altlinux_10.tmpl @@ -0,0 +1,118 @@ +ARG PG_VERSION +ARG PYTHON_VERSION + +# --------------------------------------------- base1 +FROM alt:p10 as base1 +ARG PG_VERSION + +RUN apt-get update +RUN apt-get install -y sudo curl ca-certificates +RUN apt-get update +RUN apt-get install -y openssh-server openssh-clients +RUN apt-get install -y time + +# RUN apt-get install -y mc + +RUN apt-get install -y libsqlite3-devel + +EXPOSE 22 + +RUN ssh-keygen -A + +# --------------------------------------------- postgres +FROM base1 as base1_with_dev_tools + +RUN apt-get update + +RUN apt-get install -y git +RUN apt-get install -y gcc +RUN apt-get install -y make + +RUN apt-get install -y meson +RUN apt-get install -y flex +RUN apt-get install -y bison + +RUN apt-get install -y pkg-config +RUN apt-get install -y libssl-devel +RUN apt-get install -y libicu-devel +RUN apt-get install -y libzstd-devel +RUN apt-get install -y zlib-devel +RUN apt-get install -y liblz4-devel +RUN apt-get install -y libzstd-devel +RUN apt-get install -y libxml2-devel + +# --------------------------------------------- postgres +FROM base1_with_dev_tools as base1_with_pg-17 + +RUN git clone https://github.com/postgres/postgres.git -b REL_17_STABLE /pg/postgres/source + +WORKDIR /pg/postgres/source + +RUN ./configure --prefix=/pg/postgres/install --with-zlib --with-openssl --without-readline --with-lz4 --with-zstd --with-libxml +RUN make -j 4 install +RUN make -j 4 -C contrib install + +# SETUP PG_CONFIG +# When pg_config symlink in /usr/local/bin it returns a real (right) result of --bindir +RUN ln -s /pg/postgres/install/bin/pg_config -t /usr/local/bin + +# SETUP PG CLIENT LIBRARY +# libpq.so.5 is enough +RUN ln -s /pg/postgres/install/lib/libpq.so.5.17 /usr/lib64/libpq.so.5 + +# --------------------------------------------- base2_with_python-3 +FROM base1_with_pg-${PG_VERSION} as base2_with_python-3 +RUN apt-get install -y python3 +RUN apt-get install -y python3-dev +RUN apt-get install -y python3-module-virtualenv +RUN apt-get install -y python3-modules-sqlite3 + +# AltLinux does not have "generic" virtualenv utility. Let's create it. +RUN if [[ -f "/usr/bin/virtualenv" ]] ; then \ + echo AAA; \ + elif [[ -f "/usr/bin/virtualenv3" ]] ; then \ + ln -s /usr/bin/virtualenv3 /usr/bin/virtualenv; \ + else \ + echo "/usr/bin/virtualenv is not created!"; \ + exit 1; \ + fi + +ENV PYTHON_VERSION=3 + +# --------------------------------------------- final +FROM base2_with_python-${PYTHON_VERSION} as final + +RUN adduser test -G wheel + +# It enables execution of "sudo service ssh start" without password +RUN sh -c "echo \"WHEEL_USERS ALL=(ALL:ALL) NOPASSWD: ALL\"" >> /etc/sudoers + +ADD . /pg/testgres +WORKDIR /pg/testgres +RUN chown -R test /pg/testgres + +ENV LANG=C.UTF-8 + +USER test + +RUN chmod 700 ~/ +RUN mkdir -p ~/.ssh + +# +# Altlinux 10 and 11 too slowly create a new SSH connection (x6). +# +# So, we exclude the "remote" tests until this problem has been resolved. +# + +ENTRYPOINT sh -c " \ +set -eux; \ +echo HELLO FROM ENTRYPOINT; \ +echo HOME DIR IS [`realpath ~/`]; \ +sudo /usr/sbin/sshd; \ +ssh-keyscan -H localhost >> ~/.ssh/known_hosts; \ +ssh-keyscan -H 127.0.0.1 >> ~/.ssh/known_hosts; \ +ssh-keygen -t rsa -f ~/.ssh/id_rsa -q -N ''; \ +cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys; \ +chmod 600 ~/.ssh/authorized_keys; \ +ls -la ~/.ssh/; \ +TEST_FILTER=\"TestgresTests or (TestTestgresCommon and (not remote_ops))\" bash ./run_tests.sh;" diff --git a/Dockerfile--altlinux_11.tmpl b/Dockerfile--altlinux_11.tmpl new file mode 100644 index 00000000..4b591632 --- /dev/null +++ b/Dockerfile--altlinux_11.tmpl @@ -0,0 +1,118 @@ +ARG PG_VERSION +ARG PYTHON_VERSION + +# --------------------------------------------- base1 +FROM alt:p11 as base1 +ARG PG_VERSION + +RUN apt-get update +RUN apt-get install -y sudo curl ca-certificates +RUN apt-get update +RUN apt-get install -y openssh-server openssh-clients +RUN apt-get install -y time + +# RUN apt-get install -y mc + +RUN apt-get install -y libsqlite3-devel + +EXPOSE 22 + +RUN ssh-keygen -A + +# --------------------------------------------- postgres +FROM base1 as base1_with_dev_tools + +RUN apt-get update + +RUN apt-get install -y git +RUN apt-get install -y gcc +RUN apt-get install -y make + +RUN apt-get install -y meson +RUN apt-get install -y flex +RUN apt-get install -y bison + +RUN apt-get install -y pkg-config +RUN apt-get install -y libssl-devel +RUN apt-get install -y libicu-devel +RUN apt-get install -y libzstd-devel +RUN apt-get install -y zlib-devel +RUN apt-get install -y liblz4-devel +RUN apt-get install -y libzstd-devel +RUN apt-get install -y libxml2-devel + +# --------------------------------------------- postgres +FROM base1_with_dev_tools as base1_with_pg-17 + +RUN git clone https://github.com/postgres/postgres.git -b REL_17_STABLE /pg/postgres/source + +WORKDIR /pg/postgres/source + +RUN ./configure --prefix=/pg/postgres/install --with-zlib --with-openssl --without-readline --with-lz4 --with-zstd --with-libxml +RUN make -j 4 install +RUN make -j 4 -C contrib install + +# SETUP PG_CONFIG +# When pg_config symlink in /usr/local/bin it returns a real (right) result of --bindir +RUN ln -s /pg/postgres/install/bin/pg_config -t /usr/local/bin + +# SETUP PG CLIENT LIBRARY +# libpq.so.5 is enough +RUN ln -s /pg/postgres/install/lib/libpq.so.5.17 /usr/lib64/libpq.so.5 + +# --------------------------------------------- base2_with_python-3 +FROM base1_with_pg-${PG_VERSION} as base2_with_python-3 +RUN apt-get install -y python3 +RUN apt-get install -y python3-dev +RUN apt-get install -y python3-module-virtualenv +RUN apt-get install -y python3-modules-sqlite3 + +# AltLinux does not have "generic" virtualenv utility. Let's create it. +RUN if [[ -f "/usr/bin/virtualenv" ]] ; then \ + echo AAA; \ + elif [[ -f "/usr/bin/virtualenv3" ]] ; then \ + ln -s /usr/bin/virtualenv3 /usr/bin/virtualenv; \ + else \ + echo "/usr/bin/virtualenv is not created!"; \ + exit 1; \ + fi + +ENV PYTHON_VERSION=3 + +# --------------------------------------------- final +FROM base2_with_python-${PYTHON_VERSION} as final + +RUN adduser test -G wheel + +# It enables execution of "sudo service ssh start" without password +RUN sh -c "echo \"WHEEL_USERS ALL=(ALL:ALL) NOPASSWD: ALL\"" >> /etc/sudoers + +ADD . /pg/testgres +WORKDIR /pg/testgres +RUN chown -R test /pg/testgres + +ENV LANG=C.UTF-8 + +USER test + +RUN chmod 700 ~/ +RUN mkdir -p ~/.ssh + +# +# Altlinux 10 and 11 too slowly create a new SSH connection (x6). +# +# So, we exclude the "remote" tests until this problem has been resolved. +# + +ENTRYPOINT sh -c " \ +set -eux; \ +echo HELLO FROM ENTRYPOINT; \ +echo HOME DIR IS [`realpath ~/`]; \ +sudo /usr/sbin/sshd; \ +ssh-keyscan -H localhost >> ~/.ssh/known_hosts; \ +ssh-keyscan -H 127.0.0.1 >> ~/.ssh/known_hosts; \ +ssh-keygen -t rsa -f ~/.ssh/id_rsa -q -N ''; \ +cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys; \ +chmod 600 ~/.ssh/authorized_keys; \ +ls -la ~/.ssh/; \ +TEST_FILTER=\"TestgresTests or (TestTestgresCommon and (not remote_ops))\" bash ./run_tests.sh;" From bc18e5b9362d9634c468c616e40282cf7e30e5da Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Fri, 28 Mar 2025 15:55:58 +0300 Subject: [PATCH 174/216] conftest.py is updated [refactoring] (#226) New code can process a failure in fixtures and builds a list of these cases (achtung tests). --- tests/conftest.py | 146 +++++++++++++++++++++++++++++++--------------- 1 file changed, 98 insertions(+), 48 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index e37c3c77..0f65838e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,17 +1,18 @@ # ///////////////////////////////////////////////////////////////////////////// # PyTest Configuration -import _pytest.outcomes - import pluggy import pytest -import _pytest import os import logging import pathlib import math import datetime +import _pytest.outcomes +import _pytest.unittest +import _pytest.logging + # ///////////////////////////////////////////////////////////////////////////// C_ROOT_DIR__RELATIVE = ".." @@ -91,10 +92,6 @@ def GetCurrentTestWorkerSignature() -> str: return __class__.sm_CurrentTestWorkerSignature -# /////////////////////////////////////////////////////////////////////////////# ///////////////////////////////////////////////////////////////////////////// -# Fixtures - - # ///////////////////////////////////////////////////////////////////////////// # TEST_PROCESS_STATS @@ -109,10 +106,12 @@ class TEST_PROCESS_STATS: cSkippedTests: int = 0 cNotXFailedTests: int = 0 cUnexpectedTests: int = 0 + cAchtungTests: int = 0 FailedTests = list[str]() XFailedTests = list[str]() NotXFailedTests = list[str]() + AchtungTests = list[str]() # -------------------------------------------------------------------- def incrementTotalTestCount() -> None: @@ -163,6 +162,14 @@ def incrementNotXFailedTests(testID: str) -> None: def incrementUnexpectedTests() -> None: __class__.cUnexpectedTests += 1 + # -------------------------------------------------------------------- + def incrementAchtungTestCount(testID: str) -> None: + assert type(testID) == str # noqa: E721 + assert type(__class__.AchtungTests) == list # noqa: E721 + + __class__.AchtungTests.append(testID) # raise? + __class__.cAchtungTests += 1 + # ///////////////////////////////////////////////////////////////////////////// @@ -198,10 +205,13 @@ def helper__makereport__setup( assert item is not None assert call is not None assert outcome is not None - assert type(item) == pytest.Function # noqa: E721 + # it may be pytest.Function or _pytest.unittest.TestCaseFunction + assert isinstance(item, pytest.Function) assert type(call) == pytest.CallInfo # noqa: E721 assert type(outcome) == pluggy.Result # noqa: E721 + C_LINE1 = "******************************************************" + # logging.info("pytest_runtest_makereport - setup") TEST_PROCESS_STATS.incrementTotalTestCount() @@ -214,10 +224,6 @@ def helper__makereport__setup( TEST_PROCESS_STATS.incrementNotExecutedTestCount() return - assert rep.outcome == "passed" - - testNumber = TEST_PROCESS_STATS.incrementExecutedTestCount() - testID = "" if item.cls is not None: @@ -225,15 +231,35 @@ def helper__makereport__setup( testID = testID + item.name - if testNumber > 1: - logging.info("") + if rep.outcome == "passed": + testNumber = TEST_PROCESS_STATS.incrementExecutedTestCount() + + logging.info(C_LINE1) + logging.info("* START TEST {0}".format(testID)) + logging.info("*") + logging.info("* Path : {0}".format(item.path)) + logging.info("* Number: {0}".format(testNumber)) + logging.info("*") + return + + assert rep.outcome != "passed" + + TEST_PROCESS_STATS.incrementAchtungTestCount(testID) - logging.info("******************************************************") - logging.info("* START TEST {0}".format(testID)) + logging.info(C_LINE1) + logging.info("* ACHTUNG TEST {0}".format(testID)) logging.info("*") logging.info("* Path : {0}".format(item.path)) - logging.info("* Number: {0}".format(testNumber)) + logging.info("* Outcome is [{0}]".format(rep.outcome)) + + if rep.outcome == "failed": + assert call.excinfo is not None + assert call.excinfo.value is not None + logging.info("*") + logging.error(call.excinfo.value) + logging.info("*") + return # ------------------------------------------------------------------------ @@ -243,12 +269,11 @@ def helper__makereport__call( assert item is not None assert call is not None assert outcome is not None - assert type(item) == pytest.Function # noqa: E721 + # it may be pytest.Function or _pytest.unittest.TestCaseFunction + assert isinstance(item, pytest.Function) assert type(call) == pytest.CallInfo # noqa: E721 assert type(outcome) == pluggy.Result # noqa: E721 - # logging.info("pytest_runtest_makereport - call") - rep = outcome.get_result() assert rep is not None assert type(rep) == pytest.TestReport # noqa: E721 @@ -341,7 +366,8 @@ def helper__makereport__call( else: TEST_PROCESS_STATS.incrementUnexpectedTests() exitStatus = "UNEXPECTED [{0}]".format(rep.outcome) - assert False + # [2025-03-28] It may create a useless problem in new environment. + # assert False # -------- logging.info("*") @@ -350,6 +376,7 @@ def helper__makereport__call( logging.info("* EXIT STATUS : {0}".format(exitStatus)) logging.info("*") logging.info("* STOP TEST {0}".format(testID)) + logging.info("*") # ///////////////////////////////////////////////////////////////////////////// @@ -359,17 +386,14 @@ def helper__makereport__call( def pytest_runtest_makereport(item: pytest.Function, call: pytest.CallInfo): assert item is not None assert call is not None - assert type(item) == pytest.Function # noqa: E721 + # it may be pytest.Function or _pytest.unittest.TestCaseFunction + assert isinstance(item, pytest.Function) assert type(call) == pytest.CallInfo # noqa: E721 - # logging.info("[pytest_runtest_makereport][#001][{0}][{1}]".format(item.name, call.when)) - outcome: pluggy.Result = yield assert outcome is not None assert type(outcome) == pluggy.Result # noqa: E721 - # logging.info("[pytest_runtest_makereport][#002][{0}][{1}]".format(item.name, call.when)) - rep: pytest.TestReport = outcome.get_result() assert rep is not None assert type(rep) == pytest.TestReport # noqa: E721 @@ -440,41 +464,61 @@ def run_after_tests(request: pytest.FixtureRequest): yield - logging.info("--------------------------- [FAILED TESTS]") - logging.info("") - - assert len(TEST_PROCESS_STATS.FailedTests) == TEST_PROCESS_STATS.cFailedTests - - if len(TEST_PROCESS_STATS.FailedTests) > 0: - helper__print_test_list(TEST_PROCESS_STATS.FailedTests) - logging.info("") + C_LINE1 = "---------------------------" - logging.info("--------------------------- [XFAILED TESTS]") - logging.info("") + def LOCAL__print_line1_with_header(header: str): + assert type(C_LINE1) == str # noqa: E721 + assert type(header) == str # noqa: E721 + assert header != "" + logging.info(C_LINE1 + " [" + header + "]") - assert len(TEST_PROCESS_STATS.XFailedTests) == TEST_PROCESS_STATS.cXFailedTests + def LOCAL__print_test_list(header: str, test_count: int, test_list: list[str]): + assert type(header) == str # noqa: E721 + assert type(test_count) == int # noqa: E721 + assert type(test_list) == list # noqa: E721 + assert header != "" + assert test_count >= 0 + assert len(test_list) == test_count - if len(TEST_PROCESS_STATS.XFailedTests) > 0: - helper__print_test_list(TEST_PROCESS_STATS.XFailedTests) + LOCAL__print_line1_with_header(header) logging.info("") + if len(test_list) > 0: + helper__print_test_list(test_list) + logging.info("") + + # fmt: off + LOCAL__print_test_list( + "ACHTUNG TESTS", + TEST_PROCESS_STATS.cAchtungTests, + TEST_PROCESS_STATS.AchtungTests, + ) - logging.info("--------------------------- [NOT XFAILED TESTS]") - logging.info("") + LOCAL__print_test_list( + "FAILED TESTS", + TEST_PROCESS_STATS.cFailedTests, + TEST_PROCESS_STATS.FailedTests + ) - assert ( - len(TEST_PROCESS_STATS.NotXFailedTests) == TEST_PROCESS_STATS.cNotXFailedTests + LOCAL__print_test_list( + "XFAILED TESTS", + TEST_PROCESS_STATS.cXFailedTests, + TEST_PROCESS_STATS.XFailedTests, ) - if len(TEST_PROCESS_STATS.NotXFailedTests) > 0: - helper__print_test_list(TEST_PROCESS_STATS.NotXFailedTests) - logging.info("") + LOCAL__print_test_list( + "NOT XFAILED TESTS", + TEST_PROCESS_STATS.cNotXFailedTests, + TEST_PROCESS_STATS.NotXFailedTests, + ) + # fmt: on - logging.info("--------------------------- [SUMMARY STATISTICS]") + LOCAL__print_line1_with_header("SUMMARY STATISTICS") logging.info("") logging.info("[TESTS]") logging.info(" TOTAL : {0}".format(TEST_PROCESS_STATS.cTotalTests)) logging.info(" EXECUTED : {0}".format(TEST_PROCESS_STATS.cExecutedTests)) logging.info(" NOT EXECUTED: {0}".format(TEST_PROCESS_STATS.cNotExecutedTests)) + logging.info(" ACHTUNG : {0}".format(TEST_PROCESS_STATS.cAchtungTests)) logging.info("") logging.info(" PASSED : {0}".format(TEST_PROCESS_STATS.cPassedTests)) logging.info(" FAILED : {0}".format(TEST_PROCESS_STATS.cFailedTests)) @@ -503,7 +547,13 @@ def pytest_configure(config: pytest.Config) -> None: log_path.mkdir(exist_ok=True) - logging_plugin = config.pluginmanager.get_plugin("logging-plugin") + logging_plugin: _pytest.logging.LoggingPlugin = config.pluginmanager.get_plugin( + "logging-plugin" + ) + + assert logging_plugin is not None + assert isinstance(logging_plugin, _pytest.logging.LoggingPlugin) + logging_plugin.set_log_path(str(log_path / log_name)) From a5d6df452371206da124309ef7cddaf44d023299 Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Fri, 28 Mar 2025 23:36:42 +0300 Subject: [PATCH 175/216] [BUG FIX] PostgresNode must use get_pg_version2 (#227) --- testgres/node.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index 6d2417c4..2b5fc6d1 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -92,7 +92,7 @@ PgVer, \ eprint, \ get_bin_path2, \ - get_pg_version, \ + get_pg_version2, \ execute_utility2, \ options_string, \ clean_on_error @@ -148,16 +148,6 @@ def __init__(self, name=None, base_dir=None, port=None, conn_params: ConnectionP """ # private - self._pg_version = PgVer(get_pg_version(bin_dir)) - self._should_free_port = port is None - self._base_dir = base_dir - self._bin_dir = bin_dir - self._prefix = prefix - self._logger = None - self._master = None - - # basic - self.name = name or generate_app_name() if os_ops is None: os_ops = __class__._get_os_ops(conn_params) else: @@ -168,6 +158,17 @@ def __init__(self, name=None, base_dir=None, port=None, conn_params: ConnectionP assert isinstance(os_ops, OsOperations) self._os_ops = os_ops + self._pg_version = PgVer(get_pg_version2(os_ops, bin_dir)) + self._should_free_port = port is None + self._base_dir = base_dir + self._bin_dir = bin_dir + self._prefix = prefix + self._logger = None + self._master = None + + # basic + self.name = name or generate_app_name() + self.host = os_ops.host self.port = port or utils.reserve_port() From 56ae1a8ceb31a34cbf382d7884411389e293d83c Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Fri, 28 Mar 2025 23:38:48 +0300 Subject: [PATCH 176/216] get_pg_version2 is updated --- testgres/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testgres/utils.py b/testgres/utils.py index a988effe..62e95ff6 100644 --- a/testgres/utils.py +++ b/testgres/utils.py @@ -212,8 +212,8 @@ def get_pg_version2(os_ops: OsOperations, bin_dir=None): # Get raw version (e.g., postgres (PostgreSQL) 9.5.7) postgres_path = os.path.join(bin_dir, 'postgres') if bin_dir else get_bin_path2(os_ops, 'postgres') - _params = [postgres_path, '--version'] - raw_ver = os_ops.exec_command(_params, encoding='utf-8') + cmd = [postgres_path, '--version'] + raw_ver = os_ops.exec_command(cmd, encoding='utf-8') return parse_pg_version(raw_ver) From ee441caa91ebd96f0a1ac8244da36269a0d70d98 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Fri, 28 Mar 2025 23:39:40 +0300 Subject: [PATCH 177/216] get_pg_config2 is updated --- testgres/utils.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/testgres/utils.py b/testgres/utils.py index 62e95ff6..92383571 100644 --- a/testgres/utils.py +++ b/testgres/utils.py @@ -185,11 +185,15 @@ def cache_pg_config_data(cmd): return _pg_config_data # try specified pg_config path or PG_CONFIG + if pg_config_path: + return cache_pg_config_data(pg_config_path) + if isinstance(os_ops, RemoteOperations): - pg_config = pg_config_path or os.environ.get("PG_CONFIG_REMOTE") or os.environ.get("PG_CONFIG") + pg_config = os.environ.get("PG_CONFIG_REMOTE") or os.environ.get("PG_CONFIG") else: # try PG_CONFIG - get from local machine - pg_config = pg_config_path or os.environ.get("PG_CONFIG") + pg_config = os.environ.get("PG_CONFIG") + if pg_config: return cache_pg_config_data(pg_config) From 939ca6dfd29d6c6680ffd4f6441fbcae94da66f4 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Fri, 28 Mar 2025 23:40:33 +0300 Subject: [PATCH 178/216] TestTestgresCommon::test_get_pg_config2 is added --- tests/test_testgres_common.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/test_testgres_common.py b/tests/test_testgres_common.py index f42964c8..2440b8f0 100644 --- a/tests/test_testgres_common.py +++ b/tests/test_testgres_common.py @@ -5,6 +5,7 @@ from ..testgres.node import PgVer from ..testgres.node import PostgresNode from ..testgres.utils import get_pg_version2 +from ..testgres.utils import get_pg_config2 from ..testgres.utils import file_tail from ..testgres.utils import get_bin_path2 from ..testgres import ProcessType @@ -1064,6 +1065,31 @@ def test_dump(self, os_ops: OsOperations): res = node3.execute(query_select) assert (res == [(1, ), (2, )]) + def test_get_pg_config2(self, os_ops: OsOperations): + # check same instances + a = get_pg_config2(os_ops, None) + b = get_pg_config2(os_ops, None) + assert (id(a) == id(b)) + + # save right before config change + c1 = get_pg_config2(os_ops, None) + + # modify setting for this scope + with scoped_config(cache_pg_config=False) as config: + # sanity check for value + assert not (config.cache_pg_config) + + # save right after config change + c2 = get_pg_config2(os_ops, None) + + # check different instances after config change + assert (id(c1) != id(c2)) + + # check different instances + a = get_pg_config2(os_ops, None) + b = get_pg_config2(os_ops, None) + assert (id(a) != id(b)) + @staticmethod def helper__get_node(os_ops: OsOperations, name=None): assert isinstance(os_ops, OsOperations) From ca545891c7961b11b481f7befdc6e918289f454a Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Mon, 31 Mar 2025 13:45:52 +0300 Subject: [PATCH 179/216] pytest_runtest_makereport is updated (cleanup) (#228) Local 'rep' is not used. --- tests/conftest.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 0f65838e..e27eaeb3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -394,10 +394,6 @@ def pytest_runtest_makereport(item: pytest.Function, call: pytest.CallInfo): assert outcome is not None assert type(outcome) == pluggy.Result # noqa: E721 - rep: pytest.TestReport = outcome.get_result() - assert rep is not None - assert type(rep) == pytest.TestReport # noqa: E721 - if call.when == "collect": return From 712de460eb9a49998b88f32f14c096d77705dfd4 Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Mon, 31 Mar 2025 14:42:48 +0300 Subject: [PATCH 180/216] helper__build_test_id is added (conftest refactoring) (#229) --- tests/conftest.py | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index e27eaeb3..c6306454 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -199,6 +199,22 @@ def timedelta_to_human_text(delta: datetime.timedelta) -> str: # ///////////////////////////////////////////////////////////////////////////// +def helper__build_test_id(item: pytest.Function) -> str: + assert item is not None + assert isinstance(item, pytest.Function) + + testID = "" + + if item.cls is not None: + testID = item.cls.__module__ + "." + item.cls.__name__ + "::" + + testID = testID + item.name + + return testID + +# ///////////////////////////////////////////////////////////////////////////// + + def helper__makereport__setup( item: pytest.Function, call: pytest.CallInfo, outcome: pluggy.Result ): @@ -224,12 +240,7 @@ def helper__makereport__setup( TEST_PROCESS_STATS.incrementNotExecutedTestCount() return - testID = "" - - if item.cls is not None: - testID = item.cls.__module__ + "." + item.cls.__name__ + "::" - - testID = testID + item.name + testID = helper__build_test_id(item) if rep.outcome == "passed": testNumber = TEST_PROCESS_STATS.incrementExecutedTestCount() @@ -279,12 +290,7 @@ def helper__makereport__call( assert type(rep) == pytest.TestReport # noqa: E721 # -------- - testID = "" - - if item.cls is not None: - testID = item.cls.__module__ + "." + item.cls.__name__ + "::" - - testID = testID + item.name + testID = helper__build_test_id(item) # -------- assert call.start <= call.stop From cf19df91f33e44a7a130e04bca8b7d8cc1af155c Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Mon, 31 Mar 2025 14:50:58 +0300 Subject: [PATCH 181/216] pytest_runtest_makereport is updated (refactoring+documentation) --- tests/conftest.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index c6306454..ae528536 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -390,6 +390,12 @@ def helper__makereport__call( @pytest.hookimpl(hookwrapper=True) def pytest_runtest_makereport(item: pytest.Function, call: pytest.CallInfo): + # + # https://docs.pytest.org/en/7.1.x/how-to/writing_hook_functions.html#hookwrapper-executing-around-other-hooks + # + # Note that hook wrappers don’t return results themselves, + # they merely perform tracing or other side effects around the actual hook implementations. + # assert item is not None assert call is not None # it may be pytest.Function or _pytest.unittest.TestCaseFunction @@ -400,6 +406,8 @@ def pytest_runtest_makereport(item: pytest.Function, call: pytest.CallInfo): assert outcome is not None assert type(outcome) == pluggy.Result # noqa: E721 + assert type(call.when) == str + if call.when == "collect": return @@ -414,7 +422,9 @@ def pytest_runtest_makereport(item: pytest.Function, call: pytest.CallInfo): if call.when == "teardown": return - assert False + errMsg = "[pytest_runtest_makereport] unknown 'call.when' value: [{0}].".format(call.when) + + raise RuntimeError(errMsg) # ///////////////////////////////////////////////////////////////////////////// From d9db881901a202e76651c0940efd91e372fd56c8 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Mon, 31 Mar 2025 17:31:38 +0300 Subject: [PATCH 182/216] [FIX] Formatting [flake8] --- tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index ae528536..196dbf39 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -406,7 +406,7 @@ def pytest_runtest_makereport(item: pytest.Function, call: pytest.CallInfo): assert outcome is not None assert type(outcome) == pluggy.Result # noqa: E721 - assert type(call.when) == str + assert type(call.when) == str # noqa: E721 if call.when == "collect": return From 4502b86c3e0ffa4178411951074878cacbd42ca2 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Mon, 31 Mar 2025 17:34:59 +0300 Subject: [PATCH 183/216] helper__makereport__call is updated [revision] --- tests/conftest.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 196dbf39..ee03d1c3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -316,14 +316,14 @@ def helper__makereport__call( exitStatus = "SKIPPED" reasonText = str(call.excinfo.value) - reasonMsg = "SKIP REASON: {0}" + reasonMsgTempl = "SKIP REASON: {0}" elif type(call.excinfo.value) == _pytest.outcomes.XFailed: # noqa: E721 TEST_PROCESS_STATS.incrementXFailedTestCount(testID) exitStatus = "XFAILED" reasonText = str(call.excinfo.value) - reasonMsg = "XFAIL REASON: {0}" + reasonMsgTempl = "XFAIL REASON: {0}" else: exitStatus = "XFAILED" assert hasattr(rep, "wasxfail") @@ -333,13 +333,16 @@ def helper__makereport__call( TEST_PROCESS_STATS.incrementXFailedTestCount(testID) reasonText = rep.wasxfail - reasonMsg = "XFAIL REASON: {0}" + reasonMsgTempl = "XFAIL REASON: {0}" logging.error(call.excinfo.value) + assert type(reasonText) == str # noqa: E721 + if reasonText != "": + assert type(reasonMsgTempl) == str # noqa: E721 logging.info("*") - logging.info("* " + reasonMsg.format(reasonText)) + logging.info("* " + reasonMsgTempl.format(reasonText)) elif rep.outcome == "failed": assert call.excinfo is not None From 2090fbce9f6095da51364e03ab21763171ed650b Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Mon, 31 Mar 2025 23:02:12 +0300 Subject: [PATCH 184/216] conftest is updated [formatting+comments] --- tests/conftest.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index ee03d1c3..9e8ea368 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -399,6 +399,8 @@ def pytest_runtest_makereport(item: pytest.Function, call: pytest.CallInfo): # Note that hook wrappers don’t return results themselves, # they merely perform tracing or other side effects around the actual hook implementations. # + # https://docs.pytest.org/en/7.1.x/reference/reference.html#test-running-runtest-hooks + # assert item is not None assert call is not None # it may be pytest.Function or _pytest.unittest.TestCaseFunction @@ -425,7 +427,9 @@ def pytest_runtest_makereport(item: pytest.Function, call: pytest.CallInfo): if call.when == "teardown": return - errMsg = "[pytest_runtest_makereport] unknown 'call.when' value: [{0}].".format(call.when) + errMsg = "[pytest_runtest_makereport] unknown 'call.when' value: [{0}].".format( + call.when + ) raise RuntimeError(errMsg) From b91714142fb77d1af7450bfa58abf2cecf529c4d Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Tue, 1 Apr 2025 09:01:54 +0300 Subject: [PATCH 185/216] PostgresNode::start is updated [no logging.error] It does not uses logging.error when it can't reallocate port number. It throws exception only. Why? Our new test infrastructure will process logging.error and will increment an error counter. As result - some tests start failing. --- testgres/node.py | 3 +-- tests/test_simple.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index 2b5fc6d1..c8ae4204 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -1038,8 +1038,7 @@ def LOCAL__raise_cannot_start_node__std(from_exception): assert nAttempt > 0 assert nAttempt <= __class__._C_MAX_START_ATEMPTS if nAttempt == __class__._C_MAX_START_ATEMPTS: - logging.error("Reached maximum retry attempts. Unable to start node.") - LOCAL__raise_cannot_start_node(e, "Cannot start node after multiple attempts") + LOCAL__raise_cannot_start_node(e, "Cannot start node after multiple attempts.") log_files1 = self._collect_log_files() if not self._detect_port_conflict(log_files0, log_files1): diff --git a/tests/test_simple.py b/tests/test_simple.py index f648e558..6ca52cb0 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -450,7 +450,7 @@ def test_port_conflict(self): with pytest.raises( expected_exception=StartNodeException, - match=re.escape("Cannot start node after multiple attempts") + match=re.escape("Cannot start node after multiple attempts.") ): node2.init().start() From a9137dfeabfd0ac14b177811fef5c73296803734 Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Tue, 1 Apr 2025 16:15:30 +0300 Subject: [PATCH 186/216] [conftest] Advanced processing of logging (#230) * [conftest] Advanced processing of logging This patch does the following things: - it processes the calls of logging.error as test errors - it prints the number of errors/warnings for each test - it prints the total stats of errors/warnings/duration --- tests/conftest.py | 462 +++++++++++++++++++++++++++++++--- tests/test_conftest.py--devel | 80 ++++++ 2 files changed, 507 insertions(+), 35 deletions(-) create mode 100644 tests/test_conftest.py--devel diff --git a/tests/conftest.py b/tests/conftest.py index 9e8ea368..ff3b3cb4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,6 +8,7 @@ import pathlib import math import datetime +import typing import _pytest.outcomes import _pytest.unittest @@ -105,71 +106,169 @@ class TEST_PROCESS_STATS: cXFailedTests: int = 0 cSkippedTests: int = 0 cNotXFailedTests: int = 0 + cWarningTests: int = 0 cUnexpectedTests: int = 0 cAchtungTests: int = 0 - FailedTests = list[str]() - XFailedTests = list[str]() + FailedTests = list[str, int]() + XFailedTests = list[str, int]() NotXFailedTests = list[str]() + WarningTests = list[str, int]() AchtungTests = list[str]() + cTotalDuration: datetime.timedelta = datetime.timedelta() + + cTotalErrors: int = 0 + cTotalWarnings: int = 0 + # -------------------------------------------------------------------- def incrementTotalTestCount() -> None: + assert type(__class__.cTotalTests) == int # noqa: E721 + assert __class__.cTotalTests >= 0 + __class__.cTotalTests += 1 + assert __class__.cTotalTests > 0 + # -------------------------------------------------------------------- def incrementNotExecutedTestCount() -> None: + assert type(__class__.cNotExecutedTests) == int # noqa: E721 + assert __class__.cNotExecutedTests >= 0 + __class__.cNotExecutedTests += 1 + assert __class__.cNotExecutedTests > 0 + # -------------------------------------------------------------------- def incrementExecutedTestCount() -> int: + assert type(__class__.cExecutedTests) == int # noqa: E721 + assert __class__.cExecutedTests >= 0 + __class__.cExecutedTests += 1 + + assert __class__.cExecutedTests > 0 return __class__.cExecutedTests # -------------------------------------------------------------------- def incrementPassedTestCount() -> None: + assert type(__class__.cPassedTests) == int # noqa: E721 + assert __class__.cPassedTests >= 0 + __class__.cPassedTests += 1 + assert __class__.cPassedTests > 0 + # -------------------------------------------------------------------- - def incrementFailedTestCount(testID: str) -> None: + def incrementFailedTestCount(testID: str, errCount: int) -> None: assert type(testID) == str # noqa: E721 + assert type(errCount) == int # noqa: E721 + assert errCount > 0 assert type(__class__.FailedTests) == list # noqa: E721 + assert type(__class__.cFailedTests) == int # noqa: E721 + assert __class__.cFailedTests >= 0 - __class__.FailedTests.append(testID) # raise? + __class__.FailedTests.append((testID, errCount)) # raise? __class__.cFailedTests += 1 + assert len(__class__.FailedTests) > 0 + assert __class__.cFailedTests > 0 + assert len(__class__.FailedTests) == __class__.cFailedTests + + # -------- + assert type(__class__.cTotalErrors) == int # noqa: E721 + assert __class__.cTotalErrors >= 0 + + __class__.cTotalErrors += errCount + + assert __class__.cTotalErrors > 0 + # -------------------------------------------------------------------- - def incrementXFailedTestCount(testID: str) -> None: + def incrementXFailedTestCount(testID: str, errCount: int) -> None: assert type(testID) == str # noqa: E721 + assert type(errCount) == int # noqa: E721 + assert errCount >= 0 assert type(__class__.XFailedTests) == list # noqa: E721 + assert type(__class__.cXFailedTests) == int # noqa: E721 + assert __class__.cXFailedTests >= 0 - __class__.XFailedTests.append(testID) # raise? + __class__.XFailedTests.append((testID, errCount)) # raise? __class__.cXFailedTests += 1 + assert len(__class__.XFailedTests) > 0 + assert __class__.cXFailedTests > 0 + assert len(__class__.XFailedTests) == __class__.cXFailedTests + # -------------------------------------------------------------------- def incrementSkippedTestCount() -> None: + assert type(__class__.cSkippedTests) == int # noqa: E721 + assert __class__.cSkippedTests >= 0 + __class__.cSkippedTests += 1 + assert __class__.cSkippedTests > 0 + # -------------------------------------------------------------------- def incrementNotXFailedTests(testID: str) -> None: assert type(testID) == str # noqa: E721 assert type(__class__.NotXFailedTests) == list # noqa: E721 + assert type(__class__.cNotXFailedTests) == int # noqa: E721 + assert __class__.cNotXFailedTests >= 0 __class__.NotXFailedTests.append(testID) # raise? __class__.cNotXFailedTests += 1 + assert len(__class__.NotXFailedTests) > 0 + assert __class__.cNotXFailedTests > 0 + assert len(__class__.NotXFailedTests) == __class__.cNotXFailedTests + + # -------------------------------------------------------------------- + def incrementWarningTestCount(testID: str, warningCount: int) -> None: + assert type(testID) == str # noqa: E721 + assert type(warningCount) == int # noqa: E721 + assert testID != "" + assert warningCount > 0 + assert type(__class__.WarningTests) == list # noqa: E721 + assert type(__class__.cWarningTests) == int # noqa: E721 + assert __class__.cWarningTests >= 0 + + __class__.WarningTests.append((testID, warningCount)) # raise? + __class__.cWarningTests += 1 + + assert len(__class__.WarningTests) > 0 + assert __class__.cWarningTests > 0 + assert len(__class__.WarningTests) == __class__.cWarningTests + + # -------- + assert type(__class__.cTotalWarnings) == int # noqa: E721 + assert __class__.cTotalWarnings >= 0 + + __class__.cTotalWarnings += warningCount + + assert __class__.cTotalWarnings > 0 + # -------------------------------------------------------------------- def incrementUnexpectedTests() -> None: + assert type(__class__.cUnexpectedTests) == int # noqa: E721 + assert __class__.cUnexpectedTests >= 0 + __class__.cUnexpectedTests += 1 + assert __class__.cUnexpectedTests > 0 + # -------------------------------------------------------------------- def incrementAchtungTestCount(testID: str) -> None: assert type(testID) == str # noqa: E721 assert type(__class__.AchtungTests) == list # noqa: E721 + assert type(__class__.cAchtungTests) == int # noqa: E721 + assert __class__.cAchtungTests >= 0 __class__.AchtungTests.append(testID) # raise? __class__.cAchtungTests += 1 + assert len(__class__.AchtungTests) > 0 + assert __class__.cAchtungTests > 0 + assert len(__class__.AchtungTests) == __class__.cAchtungTests + # ///////////////////////////////////////////////////////////////////////////// @@ -212,6 +311,12 @@ def helper__build_test_id(item: pytest.Function) -> str: return testID + +# ///////////////////////////////////////////////////////////////////////////// + +g_error_msg_count_key = pytest.StashKey[int]() +g_warning_msg_count_key = pytest.StashKey[int]() + # ///////////////////////////////////////////////////////////////////////////// @@ -285,6 +390,16 @@ def helper__makereport__call( assert type(call) == pytest.CallInfo # noqa: E721 assert type(outcome) == pluggy.Result # noqa: E721 + # -------- + item_error_msg_count = item.stash.get(g_error_msg_count_key, 0) + assert type(item_error_msg_count) == int # noqa: E721 + assert item_error_msg_count >= 0 + + item_warning_msg_count = item.stash.get(g_warning_msg_count_key, 0) + assert type(item_warning_msg_count) == int # noqa: E721 + assert item_warning_msg_count >= 0 + + # -------- rep = outcome.get_result() assert rep is not None assert type(rep) == pytest.TestReport # noqa: E721 @@ -312,30 +427,35 @@ def helper__makereport__call( if type(call.excinfo.value) == _pytest.outcomes.Skipped: # noqa: E721 assert not hasattr(rep, "wasxfail") - TEST_PROCESS_STATS.incrementSkippedTestCount() - exitStatus = "SKIPPED" reasonText = str(call.excinfo.value) reasonMsgTempl = "SKIP REASON: {0}" - elif type(call.excinfo.value) == _pytest.outcomes.XFailed: # noqa: E721 - TEST_PROCESS_STATS.incrementXFailedTestCount(testID) + TEST_PROCESS_STATS.incrementSkippedTestCount() + elif type(call.excinfo.value) == _pytest.outcomes.XFailed: # noqa: E721 exitStatus = "XFAILED" reasonText = str(call.excinfo.value) reasonMsgTempl = "XFAIL REASON: {0}" + + TEST_PROCESS_STATS.incrementXFailedTestCount(testID, item_error_msg_count) + else: exitStatus = "XFAILED" assert hasattr(rep, "wasxfail") assert rep.wasxfail is not None assert type(rep.wasxfail) == str # noqa: E721 - TEST_PROCESS_STATS.incrementXFailedTestCount(testID) - reasonText = rep.wasxfail reasonMsgTempl = "XFAIL REASON: {0}" - logging.error(call.excinfo.value) + if type(call.excinfo.value) == SIGNAL_EXCEPTION: # noqa: E721 + pass + else: + logging.error(call.excinfo.value) + item_error_msg_count += 1 + + TEST_PROCESS_STATS.incrementXFailedTestCount(testID, item_error_msg_count) assert type(reasonText) == str # noqa: E721 @@ -348,9 +468,16 @@ def helper__makereport__call( assert call.excinfo is not None assert call.excinfo.value is not None - TEST_PROCESS_STATS.incrementFailedTestCount(testID) + if type(call.excinfo.value) == SIGNAL_EXCEPTION: # noqa: E721 + assert item_error_msg_count > 0 + pass + else: + logging.error(call.excinfo.value) + item_error_msg_count += 1 + + assert item_error_msg_count > 0 + TEST_PROCESS_STATS.incrementFailedTestCount(testID, item_error_msg_count) - logging.error(call.excinfo.value) exitStatus = "FAILED" elif rep.outcome == "passed": assert call.excinfo is None @@ -360,12 +487,12 @@ def helper__makereport__call( TEST_PROCESS_STATS.incrementNotXFailedTests(testID) - warnMsg = "Test is marked as xfail" + warnMsg = "NOTE: Test is marked as xfail" if rep.wasxfail != "": warnMsg += " [" + rep.wasxfail + "]" - logging.warning(warnMsg) + logging.info(warnMsg) exitStatus = "NOT XFAILED" else: assert not hasattr(rep, "wasxfail") @@ -378,11 +505,25 @@ def helper__makereport__call( # [2025-03-28] It may create a useless problem in new environment. # assert False + # -------- + if item_warning_msg_count > 0: + TEST_PROCESS_STATS.incrementWarningTestCount(testID, item_warning_msg_count) + + # -------- + assert type(TEST_PROCESS_STATS.cTotalDuration) == datetime.timedelta # noqa: E721 + assert type(testDurration) == datetime.timedelta # noqa: E721 + + TEST_PROCESS_STATS.cTotalDuration += testDurration + + assert testDurration <= TEST_PROCESS_STATS.cTotalDuration + # -------- logging.info("*") - logging.info("* DURATION : {0}".format(timedelta_to_human_text(testDurration))) + logging.info("* DURATION : {0}".format(timedelta_to_human_text(testDurration))) logging.info("*") - logging.info("* EXIT STATUS : {0}".format(exitStatus)) + logging.info("* EXIT STATUS : {0}".format(exitStatus)) + logging.info("* ERROR COUNT : {0}".format(item_error_msg_count)) + logging.info("* WARNING COUNT: {0}".format(item_warning_msg_count)) logging.info("*") logging.info("* STOP TEST {0}".format(testID)) logging.info("*") @@ -437,6 +578,186 @@ def pytest_runtest_makereport(item: pytest.Function, call: pytest.CallInfo): # ///////////////////////////////////////////////////////////////////////////// +class LogErrorWrapper2: + _old_method: any + _counter: typing.Optional[int] + + # -------------------------------------------------------------------- + def __init__(self): + self._old_method = None + self._counter = None + + # -------------------------------------------------------------------- + def __enter__(self): + assert self._old_method is None + assert self._counter is None + + self._old_method = logging.error + self._counter = 0 + + logging.error = self + return self + + # -------------------------------------------------------------------- + def __exit__(self, exc_type, exc_val, exc_tb): + assert self._old_method is not None + assert self._counter is not None + + assert logging.error is self + + logging.error = self._old_method + + self._old_method = None + self._counter = None + return False + + # -------------------------------------------------------------------- + def __call__(self, *args, **kwargs): + assert self._old_method is not None + assert self._counter is not None + + assert type(self._counter) == int # noqa: E721 + assert self._counter >= 0 + + r = self._old_method(*args, **kwargs) + + self._counter += 1 + assert self._counter > 0 + + return r + + +# ///////////////////////////////////////////////////////////////////////////// + + +class LogWarningWrapper2: + _old_method: any + _counter: typing.Optional[int] + + # -------------------------------------------------------------------- + def __init__(self): + self._old_method = None + self._counter = None + + # -------------------------------------------------------------------- + def __enter__(self): + assert self._old_method is None + assert self._counter is None + + self._old_method = logging.warning + self._counter = 0 + + logging.warning = self + return self + + # -------------------------------------------------------------------- + def __exit__(self, exc_type, exc_val, exc_tb): + assert self._old_method is not None + assert self._counter is not None + + assert logging.warning is self + + logging.warning = self._old_method + + self._old_method = None + self._counter = None + return False + + # -------------------------------------------------------------------- + def __call__(self, *args, **kwargs): + assert self._old_method is not None + assert self._counter is not None + + assert type(self._counter) == int # noqa: E721 + assert self._counter >= 0 + + r = self._old_method(*args, **kwargs) + + self._counter += 1 + assert self._counter > 0 + + return r + + +# ///////////////////////////////////////////////////////////////////////////// + + +class SIGNAL_EXCEPTION(Exception): + def __init__(self): + pass + + +# ///////////////////////////////////////////////////////////////////////////// + + +@pytest.hookimpl(hookwrapper=True) +def pytest_pyfunc_call(pyfuncitem: pytest.Function): + assert pyfuncitem is not None + assert isinstance(pyfuncitem, pytest.Function) + + debug__log_error_method = logging.error + assert debug__log_error_method is not None + + debug__log_warning_method = logging.warning + assert debug__log_warning_method is not None + + pyfuncitem.stash[g_error_msg_count_key] = 0 + pyfuncitem.stash[g_warning_msg_count_key] = 0 + + try: + with LogErrorWrapper2() as logErrorWrapper, LogWarningWrapper2() as logWarningWrapper: + assert type(logErrorWrapper) == LogErrorWrapper2 # noqa: E721 + assert logErrorWrapper._old_method is not None + assert type(logErrorWrapper._counter) == int # noqa: E721 + assert logErrorWrapper._counter == 0 + assert logging.error is logErrorWrapper + + assert type(logWarningWrapper) == LogWarningWrapper2 # noqa: E721 + assert logWarningWrapper._old_method is not None + assert type(logWarningWrapper._counter) == int # noqa: E721 + assert logWarningWrapper._counter == 0 + assert logging.warning is logWarningWrapper + + r: pluggy.Result = yield + + assert r is not None + assert type(r) == pluggy.Result # noqa: E721 + + assert logErrorWrapper._old_method is not None + assert type(logErrorWrapper._counter) == int # noqa: E721 + assert logErrorWrapper._counter >= 0 + assert logging.error is logErrorWrapper + + assert logWarningWrapper._old_method is not None + assert type(logWarningWrapper._counter) == int # noqa: E721 + assert logWarningWrapper._counter >= 0 + assert logging.warning is logWarningWrapper + + assert g_error_msg_count_key in pyfuncitem.stash + assert g_warning_msg_count_key in pyfuncitem.stash + + assert pyfuncitem.stash[g_error_msg_count_key] == 0 + assert pyfuncitem.stash[g_warning_msg_count_key] == 0 + + pyfuncitem.stash[g_error_msg_count_key] = logErrorWrapper._counter + pyfuncitem.stash[g_warning_msg_count_key] = logWarningWrapper._counter + + if r.exception is not None: + pass + elif logErrorWrapper._counter == 0: + pass + else: + assert logErrorWrapper._counter > 0 + r.force_exception(SIGNAL_EXCEPTION()) + finally: + assert logging.error is debug__log_error_method + assert logging.warning is debug__log_warning_method + pass + + +# ///////////////////////////////////////////////////////////////////////////// + + def helper__calc_W(n: int) -> int: assert n > 0 @@ -467,11 +788,42 @@ def helper__print_test_list(tests: list[str]) -> None: nTest = 0 - while nTest < len(tests): - testID = tests[nTest] - assert type(testID) == str # noqa: E721 + for t in tests: + assert type(t) == str # noqa: E721 + assert t != "" nTest += 1 - logging.info(templateLine.format(nTest, testID)) + logging.info(templateLine.format(nTest, t)) + + +# ------------------------------------------------------------------------ +def helper__print_test_list2(tests: list[str, int]) -> None: + assert type(tests) == list # noqa: E721 + + assert helper__calc_W(9) == 1 + assert helper__calc_W(10) == 2 + assert helper__calc_W(11) == 2 + assert helper__calc_W(99) == 2 + assert helper__calc_W(100) == 3 + assert helper__calc_W(101) == 3 + assert helper__calc_W(999) == 3 + assert helper__calc_W(1000) == 4 + assert helper__calc_W(1001) == 4 + + W = helper__calc_W(len(tests)) + + templateLine = "{0:0" + str(W) + "d}. {1} ({2})" + + nTest = 0 + + for t in tests: + assert type(t) == tuple # noqa: E721 + assert len(t) == 2 + assert type(t[0]) == str # noqa: E721 + assert type(t[1]) == int # noqa: E721 + assert t[0] != "" + assert t[1] >= 0 + nTest += 1 + logging.info(templateLine.format(nTest, t[0], t[1])) # ///////////////////////////////////////////////////////////////////////////// @@ -505,6 +857,22 @@ def LOCAL__print_test_list(header: str, test_count: int, test_list: list[str]): helper__print_test_list(test_list) logging.info("") + def LOCAL__print_test_list2( + header: str, test_count: int, test_list: list[str, int] + ): + assert type(header) == str # noqa: E721 + assert type(test_count) == int # noqa: E721 + assert type(test_list) == list # noqa: E721 + assert header != "" + assert test_count >= 0 + assert len(test_list) == test_count + + LOCAL__print_line1_with_header(header) + logging.info("") + if len(test_list) > 0: + helper__print_test_list2(test_list) + logging.info("") + # fmt: off LOCAL__print_test_list( "ACHTUNG TESTS", @@ -512,13 +880,13 @@ def LOCAL__print_test_list(header: str, test_count: int, test_list: list[str]): TEST_PROCESS_STATS.AchtungTests, ) - LOCAL__print_test_list( + LOCAL__print_test_list2( "FAILED TESTS", TEST_PROCESS_STATS.cFailedTests, TEST_PROCESS_STATS.FailedTests ) - LOCAL__print_test_list( + LOCAL__print_test_list2( "XFAILED TESTS", TEST_PROCESS_STATS.cXFailedTests, TEST_PROCESS_STATS.XFailedTests, @@ -529,22 +897,46 @@ def LOCAL__print_test_list(header: str, test_count: int, test_list: list[str]): TEST_PROCESS_STATS.cNotXFailedTests, TEST_PROCESS_STATS.NotXFailedTests, ) + + LOCAL__print_test_list2( + "WARNING TESTS", + TEST_PROCESS_STATS.cWarningTests, + TEST_PROCESS_STATS.WarningTests, + ) # fmt: on LOCAL__print_line1_with_header("SUMMARY STATISTICS") logging.info("") logging.info("[TESTS]") - logging.info(" TOTAL : {0}".format(TEST_PROCESS_STATS.cTotalTests)) - logging.info(" EXECUTED : {0}".format(TEST_PROCESS_STATS.cExecutedTests)) - logging.info(" NOT EXECUTED: {0}".format(TEST_PROCESS_STATS.cNotExecutedTests)) - logging.info(" ACHTUNG : {0}".format(TEST_PROCESS_STATS.cAchtungTests)) + logging.info(" TOTAL : {0}".format(TEST_PROCESS_STATS.cTotalTests)) + logging.info(" EXECUTED : {0}".format(TEST_PROCESS_STATS.cExecutedTests)) + logging.info(" NOT EXECUTED : {0}".format(TEST_PROCESS_STATS.cNotExecutedTests)) + logging.info(" ACHTUNG : {0}".format(TEST_PROCESS_STATS.cAchtungTests)) + logging.info("") + logging.info(" PASSED : {0}".format(TEST_PROCESS_STATS.cPassedTests)) + logging.info(" FAILED : {0}".format(TEST_PROCESS_STATS.cFailedTests)) + logging.info(" XFAILED : {0}".format(TEST_PROCESS_STATS.cXFailedTests)) + logging.info(" NOT XFAILED : {0}".format(TEST_PROCESS_STATS.cNotXFailedTests)) + logging.info(" SKIPPED : {0}".format(TEST_PROCESS_STATS.cSkippedTests)) + logging.info(" WITH WARNINGS: {0}".format(TEST_PROCESS_STATS.cWarningTests)) + logging.info(" UNEXPECTED : {0}".format(TEST_PROCESS_STATS.cUnexpectedTests)) + logging.info("") + + assert type(TEST_PROCESS_STATS.cTotalDuration) == datetime.timedelta # noqa: E721 + + LOCAL__print_line1_with_header("TIME") + logging.info("") + logging.info( + " TOTAL DURATION: {0}".format( + timedelta_to_human_text(TEST_PROCESS_STATS.cTotalDuration) + ) + ) + logging.info("") + + LOCAL__print_line1_with_header("TOTAL INFORMATION") logging.info("") - logging.info(" PASSED : {0}".format(TEST_PROCESS_STATS.cPassedTests)) - logging.info(" FAILED : {0}".format(TEST_PROCESS_STATS.cFailedTests)) - logging.info(" XFAILED : {0}".format(TEST_PROCESS_STATS.cXFailedTests)) - logging.info(" NOT XFAILED : {0}".format(TEST_PROCESS_STATS.cNotXFailedTests)) - logging.info(" SKIPPED : {0}".format(TEST_PROCESS_STATS.cSkippedTests)) - logging.info(" UNEXPECTED : {0}".format(TEST_PROCESS_STATS.cUnexpectedTests)) + logging.info(" TOTAL ERROR COUNT : {0}".format(TEST_PROCESS_STATS.cTotalErrors)) + logging.info(" TOTAL WARNING COUNT: {0}".format(TEST_PROCESS_STATS.cTotalWarnings)) logging.info("") diff --git a/tests/test_conftest.py--devel b/tests/test_conftest.py--devel new file mode 100644 index 00000000..67c1dafe --- /dev/null +++ b/tests/test_conftest.py--devel @@ -0,0 +1,80 @@ +import pytest +import logging + + +class TestConfest: + def test_failed(self): + raise Exception("TEST EXCEPTION!") + + def test_ok(self): + pass + + @pytest.mark.skip() + def test_mark_skip__no_reason(self): + pass + + @pytest.mark.xfail() + def test_mark_xfail__no_reason(self): + raise Exception("XFAIL EXCEPTION") + + @pytest.mark.xfail() + def test_mark_xfail__no_reason___no_error(self): + pass + + @pytest.mark.skip(reason="reason") + def test_mark_skip__with_reason(self): + pass + + @pytest.mark.xfail(reason="reason") + def test_mark_xfail__with_reason(self): + raise Exception("XFAIL EXCEPTION") + + @pytest.mark.xfail(reason="reason") + def test_mark_xfail__with_reason___no_error(self): + pass + + def test_exc_skip__no_reason(self): + pytest.skip() + + def test_exc_xfail__no_reason(self): + pytest.xfail() + + def test_exc_skip__with_reason(self): + pytest.skip(reason="SKIP REASON") + + def test_exc_xfail__with_reason(self): + pytest.xfail(reason="XFAIL EXCEPTION") + + def test_log_error(self): + logging.error("IT IS A LOG ERROR!") + + def test_log_error_and_exc(self): + logging.error("IT IS A LOG ERROR!") + + raise Exception("TEST EXCEPTION!") + + def test_log_error_and_warning(self): + logging.error("IT IS A LOG ERROR!") + logging.warning("IT IS A LOG WARNING!") + logging.error("IT IS THE SECOND LOG ERROR!") + logging.warning("IT IS THE SECOND LOG WARNING!") + + @pytest.mark.xfail() + def test_log_error_and_xfail_mark_without_reason(self): + logging.error("IT IS A LOG ERROR!") + + @pytest.mark.xfail(reason="It is a reason message") + def test_log_error_and_xfail_mark_with_reason(self): + logging.error("IT IS A LOG ERROR!") + + @pytest.mark.xfail() + def test_two_log_error_and_xfail_mark_without_reason(self): + logging.error("IT IS THE FIRST LOG ERROR!") + logging.info("----------") + logging.error("IT IS THE SECOND LOG ERROR!") + + @pytest.mark.xfail(reason="It is a reason message") + def test_two_log_error_and_xfail_mark_with_reason(self): + logging.error("IT IS THE FIRST LOG ERROR!") + logging.info("----------") + logging.error("IT IS THE SECOND LOG ERROR!") From 5a19e0b4d677bee7264c4ce3d7a323fc9f1e5559 Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Wed, 2 Apr 2025 14:33:51 +0300 Subject: [PATCH 187/216] TestOsOpsCommon is added [generic os_ops tests] (#231) Plus: - [BUG FIX] LocalOperations::is_executable is corrected - [FIX] OsOperations::mkstemp is added --- testgres/operations/local_ops.py | 3 +- testgres/operations/os_ops.py | 3 + tests/test_local.py | 327 ---------------- tests/test_os_ops_common.py | 650 +++++++++++++++++++++++++++++++ tests/test_remote.py | 523 ------------------------- 5 files changed, 655 insertions(+), 851 deletions(-) create mode 100644 tests/test_os_ops_common.py diff --git a/testgres/operations/local_ops.py b/testgres/operations/local_ops.py index 93a64787..35e94210 100644 --- a/testgres/operations/local_ops.py +++ b/testgres/operations/local_ops.py @@ -156,7 +156,8 @@ def find_executable(self, executable): def is_executable(self, file): # Check if the file is executable - return os.stat(file).st_mode & stat.S_IXUSR + assert stat.S_IXUSR != 0 + return (os.stat(file).st_mode & stat.S_IXUSR) == stat.S_IXUSR def set_env(self, var_name, var_val): # Check if the directory is already in PATH diff --git a/testgres/operations/os_ops.py b/testgres/operations/os_ops.py index f20a7a30..3c606871 100644 --- a/testgres/operations/os_ops.py +++ b/testgres/operations/os_ops.py @@ -77,6 +77,9 @@ def pathsep(self): def mkdtemp(self, prefix=None): raise NotImplementedError() + def mkstemp(self, prefix=None): + raise NotImplementedError() + def copytree(self, src, dst): raise NotImplementedError() diff --git a/tests/test_local.py b/tests/test_local.py index 3ae93f76..7b5e488d 100644 --- a/tests/test_local.py +++ b/tests/test_local.py @@ -3,15 +3,9 @@ import pytest import re -import tempfile -import logging -from ..testgres import ExecUtilException -from ..testgres import InvalidOperationException from ..testgres import LocalOperations -from .helpers.run_conditions import RunConditions - class TestLocalOperations: @@ -19,138 +13,6 @@ class TestLocalOperations: def setup(self): self.operations = LocalOperations() - def test_mkdtemp__default(self): - path = self.operations.mkdtemp() - logging.info("Path is [{0}].".format(path)) - assert os.path.exists(path) - os.rmdir(path) - assert not os.path.exists(path) - - def test_mkdtemp__custom(self): - C_TEMPLATE = "abcdef" - path = self.operations.mkdtemp(C_TEMPLATE) - logging.info("Path is [{0}].".format(path)) - assert os.path.exists(path) - assert C_TEMPLATE in os.path.basename(path) - os.rmdir(path) - assert not os.path.exists(path) - - def test_exec_command_success(self): - """ - Test exec_command for successful command execution. - """ - RunConditions.skip_if_windows() - - cmd = "python3 --version" - response = self.operations.exec_command(cmd, wait_exit=True, shell=True) - - assert b'Python 3.' in response - - def test_exec_command_failure(self): - """ - Test exec_command for command execution failure. - """ - RunConditions.skip_if_windows() - - cmd = "nonexistent_command" - while True: - try: - self.operations.exec_command(cmd, wait_exit=True, shell=True) - except ExecUtilException as e: - assert type(e.exit_code) == int # noqa: E721 - assert e.exit_code == 127 - - assert type(e.message) == str # noqa: E721 - assert type(e.error) == bytes # noqa: E721 - - assert e.message.startswith("Utility exited with non-zero code (127). Error:") - assert "nonexistent_command" in e.message - assert "not found" in e.message - assert b"nonexistent_command" in e.error - assert b"not found" in e.error - break - raise Exception("We wait an exception!") - - def test_exec_command_failure__expect_error(self): - """ - Test exec_command for command execution failure. - """ - RunConditions.skip_if_windows() - - cmd = "nonexistent_command" - - exit_status, result, error = self.operations.exec_command(cmd, verbose=True, wait_exit=True, shell=True, expect_error=True) - - assert exit_status == 127 - assert result == b'' - assert type(error) == bytes # noqa: E721 - assert b"nonexistent_command" in error - assert b"not found" in error - - def test_listdir(self): - """ - Test listdir for listing directory contents. - """ - path = "/etc" - files = self.operations.listdir(path) - assert isinstance(files, list) - for f in files: - assert f is not None - assert type(f) == str # noqa: E721 - - def test_read__text(self): - """ - Test LocalOperations::read for text data. - """ - filename = __file__ # current file - - with open(filename, 'r') as file: # open in a text mode - response0 = file.read() - - assert type(response0) == str # noqa: E721 - - response1 = self.operations.read(filename) - assert type(response1) == str # noqa: E721 - assert response1 == response0 - - response2 = self.operations.read(filename, encoding=None, binary=False) - assert type(response2) == str # noqa: E721 - assert response2 == response0 - - response3 = self.operations.read(filename, encoding="") - assert type(response3) == str # noqa: E721 - assert response3 == response0 - - response4 = self.operations.read(filename, encoding="UTF-8") - assert type(response4) == str # noqa: E721 - assert response4 == response0 - - def test_read__binary(self): - """ - Test LocalOperations::read for binary data. - """ - filename = __file__ # current file - - with open(filename, 'rb') as file: # open in a binary mode - response0 = file.read() - - assert type(response0) == bytes # noqa: E721 - - response1 = self.operations.read(filename, binary=True) - assert type(response1) == bytes # noqa: E721 - assert response1 == response0 - - def test_read__binary_and_encoding(self): - """ - Test LocalOperations::read for binary data and encoding. - """ - filename = __file__ # current file - - with pytest.raises( - InvalidOperationException, - match=re.escape("Enconding is not allowed for read binary operation")): - self.operations.read(filename, encoding="", binary=True) - def test_read__unknown_file(self): """ Test LocalOperations::read with unknown file. @@ -159,40 +21,6 @@ def test_read__unknown_file(self): with pytest.raises(FileNotFoundError, match=re.escape("[Errno 2] No such file or directory: '/dummy'")): self.operations.read("/dummy") - def test_read_binary__spec(self): - """ - Test LocalOperations::read_binary. - """ - filename = __file__ # current file - - with open(filename, 'rb') as file: # open in a binary mode - response0 = file.read() - - assert type(response0) == bytes # noqa: E721 - - response1 = self.operations.read_binary(filename, 0) - assert type(response1) == bytes # noqa: E721 - assert response1 == response0 - - response2 = self.operations.read_binary(filename, 1) - assert type(response2) == bytes # noqa: E721 - assert len(response2) < len(response1) - assert len(response2) + 1 == len(response1) - assert response2 == response1[1:] - - response3 = self.operations.read_binary(filename, len(response1)) - assert type(response3) == bytes # noqa: E721 - assert len(response3) == 0 - - response4 = self.operations.read_binary(filename, len(response2)) - assert type(response4) == bytes # noqa: E721 - assert len(response4) == 1 - assert response4[0] == response1[len(response1) - 1] - - response5 = self.operations.read_binary(filename, len(response1) + 1) - assert type(response5) == bytes # noqa: E721 - assert len(response5) == 0 - def test_read_binary__spec__unk_file(self): """ Test LocalOperations::read_binary with unknown file. @@ -203,29 +31,6 @@ def test_read_binary__spec__unk_file(self): match=re.escape("[Errno 2] No such file or directory: '/dummy'")): self.operations.read_binary("/dummy", 0) - def test_read_binary__spec__negative_offset(self): - """ - Test LocalOperations::read_binary with negative offset. - """ - - with pytest.raises( - ValueError, - match=re.escape("Negative 'offset' is not supported.")): - self.operations.read_binary(__file__, -1) - - def test_get_file_size(self): - """ - Test LocalOperations::get_file_size. - """ - filename = __file__ # current file - - sz0 = os.path.getsize(filename) - assert type(sz0) == int # noqa: E721 - - sz1 = self.operations.get_file_size(filename) - assert type(sz1) == int # noqa: E721 - assert sz1 == sz0 - def test_get_file_size__unk_file(self): """ Test LocalOperations::get_file_size. @@ -234,70 +39,6 @@ def test_get_file_size__unk_file(self): with pytest.raises(FileNotFoundError, match=re.escape("[Errno 2] No such file or directory: '/dummy'")): self.operations.get_file_size("/dummy") - def test_isfile_true(self): - """ - Test isfile for an existing file. - """ - filename = __file__ - - response = self.operations.isfile(filename) - - assert response is True - - def test_isfile_false__not_exist(self): - """ - Test isfile for a non-existing file. - """ - filename = os.path.join(os.path.dirname(__file__), "nonexistent_file.txt") - - response = self.operations.isfile(filename) - - assert response is False - - def test_isfile_false__directory(self): - """ - Test isfile for a firectory. - """ - name = os.path.dirname(__file__) - - assert self.operations.isdir(name) - - response = self.operations.isfile(name) - - assert response is False - - def test_isdir_true(self): - """ - Test isdir for an existing directory. - """ - name = os.path.dirname(__file__) - - response = self.operations.isdir(name) - - assert response is True - - def test_isdir_false__not_exist(self): - """ - Test isdir for a non-existing directory. - """ - name = os.path.join(os.path.dirname(__file__), "it_is_nonexistent_directory") - - response = self.operations.isdir(name) - - assert response is False - - def test_isdir_false__file(self): - """ - Test isdir for a file. - """ - name = __file__ - - assert self.operations.isfile(name) - - response = self.operations.isdir(name) - - assert response is False - def test_cwd(self): """ Test cwd. @@ -314,71 +55,3 @@ def test_cwd(self): # Comp result assert v == expectedValue - - class tagWriteData001: - def __init__(self, sign, source, cp_rw, cp_truncate, cp_binary, cp_data, result): - self.sign = sign - self.source = source - self.call_param__rw = cp_rw - self.call_param__truncate = cp_truncate - self.call_param__binary = cp_binary - self.call_param__data = cp_data - self.result = result - - sm_write_data001 = [ - tagWriteData001("A001", "1234567890", False, False, False, "ABC", "1234567890ABC"), - tagWriteData001("A002", b"1234567890", False, False, True, b"ABC", b"1234567890ABC"), - - tagWriteData001("B001", "1234567890", False, True, False, "ABC", "ABC"), - tagWriteData001("B002", "1234567890", False, True, False, "ABC1234567890", "ABC1234567890"), - tagWriteData001("B003", b"1234567890", False, True, True, b"ABC", b"ABC"), - tagWriteData001("B004", b"1234567890", False, True, True, b"ABC1234567890", b"ABC1234567890"), - - tagWriteData001("C001", "1234567890", True, False, False, "ABC", "1234567890ABC"), - tagWriteData001("C002", b"1234567890", True, False, True, b"ABC", b"1234567890ABC"), - - tagWriteData001("D001", "1234567890", True, True, False, "ABC", "ABC"), - tagWriteData001("D002", "1234567890", True, True, False, "ABC1234567890", "ABC1234567890"), - tagWriteData001("D003", b"1234567890", True, True, True, b"ABC", b"ABC"), - tagWriteData001("D004", b"1234567890", True, True, True, b"ABC1234567890", b"ABC1234567890"), - - tagWriteData001("E001", "\0001234567890\000", False, False, False, "\000ABC\000", "\0001234567890\000\000ABC\000"), - tagWriteData001("E002", b"\0001234567890\000", False, False, True, b"\000ABC\000", b"\0001234567890\000\000ABC\000"), - - tagWriteData001("F001", "a\nb\n", False, False, False, ["c", "d"], "a\nb\nc\nd\n"), - tagWriteData001("F002", b"a\nb\n", False, False, True, [b"c", b"d"], b"a\nb\nc\nd\n"), - - tagWriteData001("G001", "a\nb\n", False, False, False, ["c\n\n", "d\n"], "a\nb\nc\nd\n"), - tagWriteData001("G002", b"a\nb\n", False, False, True, [b"c\n\n", b"d\n"], b"a\nb\nc\nd\n"), - ] - - @pytest.fixture( - params=sm_write_data001, - ids=[x.sign for x in sm_write_data001], - ) - def write_data001(self, request): - assert isinstance(request, pytest.FixtureRequest) - assert type(request.param) == __class__.tagWriteData001 # noqa: E721 - return request.param - - def test_write(self, write_data001): - assert type(write_data001) == __class__.tagWriteData001 # noqa: E721 - - mode = "w+b" if write_data001.call_param__binary else "w+" - - with tempfile.NamedTemporaryFile(mode=mode, delete=True) as tmp_file: - tmp_file.write(write_data001.source) - tmp_file.flush() - - self.operations.write( - tmp_file.name, - write_data001.call_param__data, - read_and_write=write_data001.call_param__rw, - truncate=write_data001.call_param__truncate, - binary=write_data001.call_param__binary) - - tmp_file.seek(0) - - s = tmp_file.read() - - assert s == write_data001.result diff --git a/tests/test_os_ops_common.py b/tests/test_os_ops_common.py new file mode 100644 index 00000000..c3944c3b --- /dev/null +++ b/tests/test_os_ops_common.py @@ -0,0 +1,650 @@ +# coding: utf-8 +from .helpers.os_ops_descrs import OsOpsDescr +from .helpers.os_ops_descrs import OsOpsDescrs +from .helpers.os_ops_descrs import OsOperations +from .helpers.run_conditions import RunConditions + +import os + +import pytest +import re +import tempfile +import logging + +from ..testgres import InvalidOperationException +from ..testgres import ExecUtilException + + +class TestOsOpsCommon: + sm_os_ops_descrs: list[OsOpsDescr] = [ + OsOpsDescrs.sm_local_os_ops_descr, + OsOpsDescrs.sm_remote_os_ops_descr + ] + + @pytest.fixture( + params=[descr.os_ops for descr in sm_os_ops_descrs], + ids=[descr.sign for descr in sm_os_ops_descrs] + ) + def os_ops(self, request: pytest.FixtureRequest) -> OsOperations: + assert isinstance(request, pytest.FixtureRequest) + assert isinstance(request.param, OsOperations) + return request.param + + def test_exec_command_success(self, os_ops: OsOperations): + """ + Test exec_command for successful command execution. + """ + assert isinstance(os_ops, OsOperations) + + RunConditions.skip_if_windows() + + cmd = ["sh", "-c", "python3 --version"] + + response = os_ops.exec_command(cmd) + + assert b'Python 3.' in response + + def test_exec_command_failure(self, os_ops: OsOperations): + """ + Test exec_command for command execution failure. + """ + assert isinstance(os_ops, OsOperations) + + RunConditions.skip_if_windows() + + cmd = ["sh", "-c", "nonexistent_command"] + + while True: + try: + os_ops.exec_command(cmd) + except ExecUtilException as e: + assert type(e.exit_code) == int # noqa: E721 + assert e.exit_code == 127 + + assert type(e.message) == str # noqa: E721 + assert type(e.error) == bytes # noqa: E721 + + assert e.message.startswith("Utility exited with non-zero code (127). Error:") + assert "nonexistent_command" in e.message + assert "not found" in e.message + assert b"nonexistent_command" in e.error + assert b"not found" in e.error + break + raise Exception("We wait an exception!") + + def test_exec_command_failure__expect_error(self, os_ops: OsOperations): + """ + Test exec_command for command execution failure. + """ + assert isinstance(os_ops, OsOperations) + + RunConditions.skip_if_windows() + + cmd = ["sh", "-c", "nonexistent_command"] + + exit_status, result, error = os_ops.exec_command(cmd, verbose=True, expect_error=True) + + assert exit_status == 127 + assert result == b'' + assert type(error) == bytes # noqa: E721 + assert b"nonexistent_command" in error + assert b"not found" in error + + def test_is_executable_true(self, os_ops: OsOperations): + """ + Test is_executable for an existing executable. + """ + assert isinstance(os_ops, OsOperations) + + RunConditions.skip_if_windows() + + response = os_ops.is_executable("/bin/sh") + + assert response is True + + def test_is_executable_false(self, os_ops: OsOperations): + """ + Test is_executable for a non-executable. + """ + assert isinstance(os_ops, OsOperations) + + response = os_ops.is_executable(__file__) + + assert response is False + + def test_makedirs_and_rmdirs_success(self, os_ops: OsOperations): + """ + Test makedirs and rmdirs for successful directory creation and removal. + """ + assert isinstance(os_ops, OsOperations) + + RunConditions.skip_if_windows() + + cmd = "pwd" + pwd = os_ops.exec_command(cmd, wait_exit=True, encoding='utf-8').strip() + + path = "{}/test_dir".format(pwd) + + # Test makedirs + os_ops.makedirs(path) + assert os.path.exists(path) + assert os_ops.path_exists(path) + + # Test rmdirs + os_ops.rmdirs(path) + assert not os.path.exists(path) + assert not os_ops.path_exists(path) + + def test_makedirs_failure(self, os_ops: OsOperations): + """ + Test makedirs for failure. + """ + # Try to create a directory in a read-only location + assert isinstance(os_ops, OsOperations) + + RunConditions.skip_if_windows() + + path = "/root/test_dir" + + # Test makedirs + with pytest.raises(Exception): + os_ops.makedirs(path) + + def test_listdir(self, os_ops: OsOperations): + """ + Test listdir for listing directory contents. + """ + assert isinstance(os_ops, OsOperations) + + RunConditions.skip_if_windows() + + path = "/etc" + files = os_ops.listdir(path) + assert isinstance(files, list) + for f in files: + assert f is not None + assert type(f) == str # noqa: E721 + + def test_path_exists_true__directory(self, os_ops: OsOperations): + """ + Test path_exists for an existing directory. + """ + assert isinstance(os_ops, OsOperations) + + RunConditions.skip_if_windows() + + assert os_ops.path_exists("/etc") is True + + def test_path_exists_true__file(self, os_ops: OsOperations): + """ + Test path_exists for an existing file. + """ + assert isinstance(os_ops, OsOperations) + + RunConditions.skip_if_windows() + + assert os_ops.path_exists(__file__) is True + + def test_path_exists_false__directory(self, os_ops: OsOperations): + """ + Test path_exists for a non-existing directory. + """ + assert isinstance(os_ops, OsOperations) + + RunConditions.skip_if_windows() + + assert os_ops.path_exists("/nonexistent_path") is False + + def test_path_exists_false__file(self, os_ops: OsOperations): + """ + Test path_exists for a non-existing file. + """ + assert isinstance(os_ops, OsOperations) + + RunConditions.skip_if_windows() + + assert os_ops.path_exists("/etc/nonexistent_path.txt") is False + + def test_mkdtemp__default(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + + path = os_ops.mkdtemp() + logging.info("Path is [{0}].".format(path)) + assert os.path.exists(path) + os.rmdir(path) + assert not os.path.exists(path) + + def test_mkdtemp__custom(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + + C_TEMPLATE = "abcdef" + path = os_ops.mkdtemp(C_TEMPLATE) + logging.info("Path is [{0}].".format(path)) + assert os.path.exists(path) + assert C_TEMPLATE in os.path.basename(path) + os.rmdir(path) + assert not os.path.exists(path) + + def test_rmdirs(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + + path = os_ops.mkdtemp() + assert os.path.exists(path) + + assert os_ops.rmdirs(path, ignore_errors=False) is True + assert not os.path.exists(path) + + def test_rmdirs__01_with_subfolder(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + + # folder with subfolder + path = os_ops.mkdtemp() + assert os.path.exists(path) + + dir1 = os.path.join(path, "dir1") + assert not os.path.exists(dir1) + + os_ops.makedirs(dir1) + assert os.path.exists(dir1) + + assert os_ops.rmdirs(path, ignore_errors=False) is True + assert not os.path.exists(path) + assert not os.path.exists(dir1) + + def test_rmdirs__02_with_file(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + + # folder with file + path = os_ops.mkdtemp() + assert os.path.exists(path) + + file1 = os.path.join(path, "file1.txt") + assert not os.path.exists(file1) + + os_ops.touch(file1) + assert os.path.exists(file1) + + assert os_ops.rmdirs(path, ignore_errors=False) is True + assert not os.path.exists(path) + assert not os.path.exists(file1) + + def test_rmdirs__03_with_subfolder_and_file(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + + # folder with subfolder and file + path = os_ops.mkdtemp() + assert os.path.exists(path) + + dir1 = os.path.join(path, "dir1") + assert not os.path.exists(dir1) + + os_ops.makedirs(dir1) + assert os.path.exists(dir1) + + file1 = os.path.join(dir1, "file1.txt") + assert not os.path.exists(file1) + + os_ops.touch(file1) + assert os.path.exists(file1) + + assert os_ops.rmdirs(path, ignore_errors=False) is True + assert not os.path.exists(path) + assert not os.path.exists(dir1) + assert not os.path.exists(file1) + + def test_write_text_file(self, os_ops: OsOperations): + """ + Test write for writing data to a text file. + """ + assert isinstance(os_ops, OsOperations) + + RunConditions.skip_if_windows() + + filename = os_ops.mkstemp() + data = "Hello, world!" + + os_ops.write(filename, data, truncate=True) + os_ops.write(filename, data) + + response = os_ops.read(filename) + + assert response == data + data + + os_ops.remove_file(filename) + + def test_write_binary_file(self, os_ops: OsOperations): + """ + Test write for writing data to a binary file. + """ + assert isinstance(os_ops, OsOperations) + + RunConditions.skip_if_windows() + + filename = "/tmp/test_file.bin" + data = b"\x00\x01\x02\x03" + + os_ops.write(filename, data, binary=True, truncate=True) + + response = os_ops.read(filename, binary=True) + + assert response == data + + def test_read_text_file(self, os_ops: OsOperations): + """ + Test read for reading data from a text file. + """ + assert isinstance(os_ops, OsOperations) + + RunConditions.skip_if_windows() + + filename = "/etc/hosts" + + response = os_ops.read(filename) + + assert isinstance(response, str) + + def test_read_binary_file(self, os_ops: OsOperations): + """ + Test read for reading data from a binary file. + """ + assert isinstance(os_ops, OsOperations) + + RunConditions.skip_if_windows() + + filename = "/usr/bin/python3" + + response = os_ops.read(filename, binary=True) + + assert isinstance(response, bytes) + + def test_read__text(self, os_ops: OsOperations): + """ + Test OsOperations::read for text data. + """ + assert isinstance(os_ops, OsOperations) + + filename = __file__ # current file + + with open(filename, 'r') as file: # open in a text mode + response0 = file.read() + + assert type(response0) == str # noqa: E721 + + response1 = os_ops.read(filename) + assert type(response1) == str # noqa: E721 + assert response1 == response0 + + response2 = os_ops.read(filename, encoding=None, binary=False) + assert type(response2) == str # noqa: E721 + assert response2 == response0 + + response3 = os_ops.read(filename, encoding="") + assert type(response3) == str # noqa: E721 + assert response3 == response0 + + response4 = os_ops.read(filename, encoding="UTF-8") + assert type(response4) == str # noqa: E721 + assert response4 == response0 + + def test_read__binary(self, os_ops: OsOperations): + """ + Test OsOperations::read for binary data. + """ + filename = __file__ # current file + + with open(filename, 'rb') as file: # open in a binary mode + response0 = file.read() + + assert type(response0) == bytes # noqa: E721 + + response1 = os_ops.read(filename, binary=True) + assert type(response1) == bytes # noqa: E721 + assert response1 == response0 + + def test_read__binary_and_encoding(self, os_ops: OsOperations): + """ + Test OsOperations::read for binary data and encoding. + """ + assert isinstance(os_ops, OsOperations) + + filename = __file__ # current file + + with pytest.raises( + InvalidOperationException, + match=re.escape("Enconding is not allowed for read binary operation")): + os_ops.read(filename, encoding="", binary=True) + + def test_read_binary__spec(self, os_ops: OsOperations): + """ + Test OsOperations::read_binary. + """ + assert isinstance(os_ops, OsOperations) + + filename = __file__ # currnt file + + with open(filename, 'rb') as file: # open in a binary mode + response0 = file.read() + + assert type(response0) == bytes # noqa: E721 + + response1 = os_ops.read_binary(filename, 0) + assert type(response1) == bytes # noqa: E721 + assert response1 == response0 + + response2 = os_ops.read_binary(filename, 1) + assert type(response2) == bytes # noqa: E721 + assert len(response2) < len(response1) + assert len(response2) + 1 == len(response1) + assert response2 == response1[1:] + + response3 = os_ops.read_binary(filename, len(response1)) + assert type(response3) == bytes # noqa: E721 + assert len(response3) == 0 + + response4 = os_ops.read_binary(filename, len(response2)) + assert type(response4) == bytes # noqa: E721 + assert len(response4) == 1 + assert response4[0] == response1[len(response1) - 1] + + response5 = os_ops.read_binary(filename, len(response1) + 1) + assert type(response5) == bytes # noqa: E721 + assert len(response5) == 0 + + def test_read_binary__spec__negative_offset(self, os_ops: OsOperations): + """ + Test OsOperations::read_binary with negative offset. + """ + assert isinstance(os_ops, OsOperations) + + with pytest.raises( + ValueError, + match=re.escape("Negative 'offset' is not supported.")): + os_ops.read_binary(__file__, -1) + + def test_get_file_size(self, os_ops: OsOperations): + """ + Test OsOperations::get_file_size. + """ + assert isinstance(os_ops, OsOperations) + + filename = __file__ # current file + + sz0 = os.path.getsize(filename) + assert type(sz0) == int # noqa: E721 + + sz1 = os_ops.get_file_size(filename) + assert type(sz1) == int # noqa: E721 + assert sz1 == sz0 + + def test_isfile_true(self, os_ops: OsOperations): + """ + Test isfile for an existing file. + """ + assert isinstance(os_ops, OsOperations) + + filename = __file__ + + response = os_ops.isfile(filename) + + assert response is True + + def test_isfile_false__not_exist(self, os_ops: OsOperations): + """ + Test isfile for a non-existing file. + """ + assert isinstance(os_ops, OsOperations) + + filename = os.path.join(os.path.dirname(__file__), "nonexistent_file.txt") + + response = os_ops.isfile(filename) + + assert response is False + + def test_isfile_false__directory(self, os_ops: OsOperations): + """ + Test isfile for a firectory. + """ + assert isinstance(os_ops, OsOperations) + + name = os.path.dirname(__file__) + + assert os_ops.isdir(name) + + response = os_ops.isfile(name) + + assert response is False + + def test_isdir_true(self, os_ops: OsOperations): + """ + Test isdir for an existing directory. + """ + assert isinstance(os_ops, OsOperations) + + name = os.path.dirname(__file__) + + response = os_ops.isdir(name) + + assert response is True + + def test_isdir_false__not_exist(self, os_ops: OsOperations): + """ + Test isdir for a non-existing directory. + """ + assert isinstance(os_ops, OsOperations) + + name = os.path.join(os.path.dirname(__file__), "it_is_nonexistent_directory") + + response = os_ops.isdir(name) + + assert response is False + + def test_isdir_false__file(self, os_ops: OsOperations): + """ + Test isdir for a file. + """ + assert isinstance(os_ops, OsOperations) + + name = __file__ + + assert os_ops.isfile(name) + + response = os_ops.isdir(name) + + assert response is False + + def test_cwd(self, os_ops: OsOperations): + """ + Test cwd. + """ + assert isinstance(os_ops, OsOperations) + + v = os_ops.cwd() + + assert v is not None + assert type(v) == str # noqa: E721 + assert v != "" + + class tagWriteData001: + def __init__(self, sign, source, cp_rw, cp_truncate, cp_binary, cp_data, result): + self.sign = sign + self.source = source + self.call_param__rw = cp_rw + self.call_param__truncate = cp_truncate + self.call_param__binary = cp_binary + self.call_param__data = cp_data + self.result = result + + sm_write_data001 = [ + tagWriteData001("A001", "1234567890", False, False, False, "ABC", "1234567890ABC"), + tagWriteData001("A002", b"1234567890", False, False, True, b"ABC", b"1234567890ABC"), + + tagWriteData001("B001", "1234567890", False, True, False, "ABC", "ABC"), + tagWriteData001("B002", "1234567890", False, True, False, "ABC1234567890", "ABC1234567890"), + tagWriteData001("B003", b"1234567890", False, True, True, b"ABC", b"ABC"), + tagWriteData001("B004", b"1234567890", False, True, True, b"ABC1234567890", b"ABC1234567890"), + + tagWriteData001("C001", "1234567890", True, False, False, "ABC", "1234567890ABC"), + tagWriteData001("C002", b"1234567890", True, False, True, b"ABC", b"1234567890ABC"), + + tagWriteData001("D001", "1234567890", True, True, False, "ABC", "ABC"), + tagWriteData001("D002", "1234567890", True, True, False, "ABC1234567890", "ABC1234567890"), + tagWriteData001("D003", b"1234567890", True, True, True, b"ABC", b"ABC"), + tagWriteData001("D004", b"1234567890", True, True, True, b"ABC1234567890", b"ABC1234567890"), + + tagWriteData001("E001", "\0001234567890\000", False, False, False, "\000ABC\000", "\0001234567890\000\000ABC\000"), + tagWriteData001("E002", b"\0001234567890\000", False, False, True, b"\000ABC\000", b"\0001234567890\000\000ABC\000"), + + tagWriteData001("F001", "a\nb\n", False, False, False, ["c", "d"], "a\nb\nc\nd\n"), + tagWriteData001("F002", b"a\nb\n", False, False, True, [b"c", b"d"], b"a\nb\nc\nd\n"), + + tagWriteData001("G001", "a\nb\n", False, False, False, ["c\n\n", "d\n"], "a\nb\nc\nd\n"), + tagWriteData001("G002", b"a\nb\n", False, False, True, [b"c\n\n", b"d\n"], b"a\nb\nc\nd\n"), + ] + + @pytest.fixture( + params=sm_write_data001, + ids=[x.sign for x in sm_write_data001], + ) + def write_data001(self, request): + assert isinstance(request, pytest.FixtureRequest) + assert type(request.param) == __class__.tagWriteData001 # noqa: E721 + return request.param + + def test_write(self, write_data001: tagWriteData001, os_ops: OsOperations): + assert type(write_data001) == __class__.tagWriteData001 # noqa: E721 + assert isinstance(os_ops, OsOperations) + + mode = "w+b" if write_data001.call_param__binary else "w+" + + with tempfile.NamedTemporaryFile(mode=mode, delete=True) as tmp_file: + tmp_file.write(write_data001.source) + tmp_file.flush() + + os_ops.write( + tmp_file.name, + write_data001.call_param__data, + read_and_write=write_data001.call_param__rw, + truncate=write_data001.call_param__truncate, + binary=write_data001.call_param__binary) + + tmp_file.seek(0) + + s = tmp_file.read() + + assert s == write_data001.result + + def test_touch(self, os_ops: OsOperations): + """ + Test touch for creating a new file or updating access and modification times of an existing file. + """ + assert isinstance(os_ops, OsOperations) + + filename = os_ops.mkstemp() + + # TODO: this test does not check the result of 'touch' command! + + os_ops.touch(filename) + + assert os_ops.isfile(filename) + + os_ops.remove_file(filename) diff --git a/tests/test_remote.py b/tests/test_remote.py index 2c37e2c1..565b2d20 100755 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -2,16 +2,10 @@ import os import pytest -import re -import tempfile -import logging from ..testgres import ExecUtilException -from ..testgres import InvalidOperationException from ..testgres import RemoteOperations -from ..testgres import LocalOperations from ..testgres import ConnectionParams -from ..testgres import utils as testgres_utils class TestRemoteOperations: @@ -23,179 +17,6 @@ def setup(self): ssh_key=os.getenv('RDBMS_TESTPOOL_SSHKEY')) self.operations = RemoteOperations(conn_params) - def test_exec_command_success(self): - """ - Test exec_command for successful command execution. - """ - cmd = "python3 --version" - response = self.operations.exec_command(cmd, wait_exit=True) - - assert b'Python 3.' in response - - def test_exec_command_failure(self): - """ - Test exec_command for command execution failure. - """ - cmd = "nonexistent_command" - while True: - try: - self.operations.exec_command(cmd, verbose=True, wait_exit=True) - except ExecUtilException as e: - assert type(e.exit_code) == int # noqa: E721 - assert e.exit_code == 127 - - assert type(e.message) == str # noqa: E721 - assert type(e.error) == bytes # noqa: E721 - - assert e.message.startswith("Utility exited with non-zero code (127). Error:") - assert "nonexistent_command" in e.message - assert "not found" in e.message - assert b"nonexistent_command" in e.error - assert b"not found" in e.error - break - raise Exception("We wait an exception!") - - def test_exec_command_failure__expect_error(self): - """ - Test exec_command for command execution failure. - """ - cmd = "nonexistent_command" - - exit_status, result, error = self.operations.exec_command(cmd, verbose=True, wait_exit=True, shell=True, expect_error=True) - - assert exit_status == 127 - assert result == b'' - assert type(error) == bytes # noqa: E721 - assert b"nonexistent_command" in error - assert b"not found" in error - - def test_is_executable_true(self): - """ - Test is_executable for an existing executable. - """ - local_ops = LocalOperations() - cmd = testgres_utils.get_bin_path2(local_ops, "pg_config") - cmd = local_ops.exec_command([cmd, "--bindir"], encoding="utf-8") - cmd = cmd.rstrip() - cmd = os.path.join(cmd, "pg_config") - response = self.operations.is_executable(cmd) - - assert response is True - - def test_is_executable_false(self): - """ - Test is_executable for a non-executable. - """ - cmd = "python" - response = self.operations.is_executable(cmd) - - assert response is False - - def test_makedirs_and_rmdirs_success(self): - """ - Test makedirs and rmdirs for successful directory creation and removal. - """ - cmd = "pwd" - pwd = self.operations.exec_command(cmd, wait_exit=True, encoding='utf-8').strip() - - path = "{}/test_dir".format(pwd) - - # Test makedirs - self.operations.makedirs(path) - assert os.path.exists(path) - assert self.operations.path_exists(path) - - # Test rmdirs - self.operations.rmdirs(path) - assert not os.path.exists(path) - assert not self.operations.path_exists(path) - - def test_makedirs_failure(self): - """ - Test makedirs for failure. - """ - # Try to create a directory in a read-only location - path = "/root/test_dir" - - # Test makedirs - with pytest.raises(Exception): - self.operations.makedirs(path) - - def test_mkdtemp__default(self): - path = self.operations.mkdtemp() - logging.info("Path is [{0}].".format(path)) - assert os.path.exists(path) - os.rmdir(path) - assert not os.path.exists(path) - - def test_mkdtemp__custom(self): - C_TEMPLATE = "abcdef" - path = self.operations.mkdtemp(C_TEMPLATE) - logging.info("Path is [{0}].".format(path)) - assert os.path.exists(path) - assert C_TEMPLATE in os.path.basename(path) - os.rmdir(path) - assert not os.path.exists(path) - - def test_rmdirs(self): - path = self.operations.mkdtemp() - assert os.path.exists(path) - - assert self.operations.rmdirs(path, ignore_errors=False) is True - assert not os.path.exists(path) - - def test_rmdirs__01_with_subfolder(self): - # folder with subfolder - path = self.operations.mkdtemp() - assert os.path.exists(path) - - dir1 = os.path.join(path, "dir1") - assert not os.path.exists(dir1) - - self.operations.makedirs(dir1) - assert os.path.exists(dir1) - - assert self.operations.rmdirs(path, ignore_errors=False) is True - assert not os.path.exists(path) - assert not os.path.exists(dir1) - - def test_rmdirs__02_with_file(self): - # folder with file - path = self.operations.mkdtemp() - assert os.path.exists(path) - - file1 = os.path.join(path, "file1.txt") - assert not os.path.exists(file1) - - self.operations.touch(file1) - assert os.path.exists(file1) - - assert self.operations.rmdirs(path, ignore_errors=False) is True - assert not os.path.exists(path) - assert not os.path.exists(file1) - - def test_rmdirs__03_with_subfolder_and_file(self): - # folder with subfolder and file - path = self.operations.mkdtemp() - assert os.path.exists(path) - - dir1 = os.path.join(path, "dir1") - assert not os.path.exists(dir1) - - self.operations.makedirs(dir1) - assert os.path.exists(dir1) - - file1 = os.path.join(dir1, "file1.txt") - assert not os.path.exists(file1) - - self.operations.touch(file1) - assert os.path.exists(file1) - - assert self.operations.rmdirs(path, ignore_errors=False) is True - assert not os.path.exists(path) - assert not os.path.exists(dir1) - assert not os.path.exists(file1) - def test_rmdirs__try_to_delete_nonexist_path(self): path = "/root/test_dir" @@ -216,141 +37,6 @@ def test_rmdirs__try_to_delete_file(self): assert type(x.value.exit_code) == int # noqa: E721 assert x.value.exit_code == 20 - def test_listdir(self): - """ - Test listdir for listing directory contents. - """ - path = "/etc" - files = self.operations.listdir(path) - assert isinstance(files, list) - for f in files: - assert f is not None - assert type(f) == str # noqa: E721 - - def test_path_exists_true__directory(self): - """ - Test path_exists for an existing directory. - """ - assert self.operations.path_exists("/etc") is True - - def test_path_exists_true__file(self): - """ - Test path_exists for an existing file. - """ - assert self.operations.path_exists(__file__) is True - - def test_path_exists_false__directory(self): - """ - Test path_exists for a non-existing directory. - """ - assert self.operations.path_exists("/nonexistent_path") is False - - def test_path_exists_false__file(self): - """ - Test path_exists for a non-existing file. - """ - assert self.operations.path_exists("/etc/nonexistent_path.txt") is False - - def test_write_text_file(self): - """ - Test write for writing data to a text file. - """ - filename = "/tmp/test_file.txt" - data = "Hello, world!" - - self.operations.write(filename, data, truncate=True) - self.operations.write(filename, data) - - response = self.operations.read(filename) - - assert response == data + data - - def test_write_binary_file(self): - """ - Test write for writing data to a binary file. - """ - filename = "/tmp/test_file.bin" - data = b"\x00\x01\x02\x03" - - self.operations.write(filename, data, binary=True, truncate=True) - - response = self.operations.read(filename, binary=True) - - assert response == data - - def test_read_text_file(self): - """ - Test read for reading data from a text file. - """ - filename = "/etc/hosts" - - response = self.operations.read(filename) - - assert isinstance(response, str) - - def test_read_binary_file(self): - """ - Test read for reading data from a binary file. - """ - filename = "/usr/bin/python3" - - response = self.operations.read(filename, binary=True) - - assert isinstance(response, bytes) - - def test_read__text(self): - """ - Test RemoteOperations::read for text data. - """ - filename = __file__ # current file - - with open(filename, 'r') as file: # open in a text mode - response0 = file.read() - - assert type(response0) == str # noqa: E721 - - response1 = self.operations.read(filename) - assert type(response1) == str # noqa: E721 - assert response1 == response0 - - response2 = self.operations.read(filename, encoding=None, binary=False) - assert type(response2) == str # noqa: E721 - assert response2 == response0 - - response3 = self.operations.read(filename, encoding="") - assert type(response3) == str # noqa: E721 - assert response3 == response0 - - response4 = self.operations.read(filename, encoding="UTF-8") - assert type(response4) == str # noqa: E721 - assert response4 == response0 - - def test_read__binary(self): - """ - Test RemoteOperations::read for binary data. - """ - filename = __file__ # current file - - with open(filename, 'rb') as file: # open in a binary mode - response0 = file.read() - - assert type(response0) == bytes # noqa: E721 - - response1 = self.operations.read(filename, binary=True) - assert type(response1) == bytes # noqa: E721 - assert response1 == response0 - - def test_read__binary_and_encoding(self): - """ - Test RemoteOperations::read for binary data and encoding. - """ - filename = __file__ # current file - - with pytest.raises( - InvalidOperationException, - match=re.escape("Enconding is not allowed for read binary operation")): - self.operations.read(filename, encoding="", binary=True) - def test_read__unknown_file(self): """ Test RemoteOperations::read with unknown file. @@ -363,40 +49,6 @@ def test_read__unknown_file(self): assert "No such file or directory" in str(x.value) assert "/dummy" in str(x.value) - def test_read_binary__spec(self): - """ - Test RemoteOperations::read_binary. - """ - filename = __file__ # currnt file - - with open(filename, 'rb') as file: # open in a binary mode - response0 = file.read() - - assert type(response0) == bytes # noqa: E721 - - response1 = self.operations.read_binary(filename, 0) - assert type(response1) == bytes # noqa: E721 - assert response1 == response0 - - response2 = self.operations.read_binary(filename, 1) - assert type(response2) == bytes # noqa: E721 - assert len(response2) < len(response1) - assert len(response2) + 1 == len(response1) - assert response2 == response1[1:] - - response3 = self.operations.read_binary(filename, len(response1)) - assert type(response3) == bytes # noqa: E721 - assert len(response3) == 0 - - response4 = self.operations.read_binary(filename, len(response2)) - assert type(response4) == bytes # noqa: E721 - assert len(response4) == 1 - assert response4[0] == response1[len(response1) - 1] - - response5 = self.operations.read_binary(filename, len(response1) + 1) - assert type(response5) == bytes # noqa: E721 - assert len(response5) == 0 - def test_read_binary__spec__unk_file(self): """ Test RemoteOperations::read_binary with unknown file. @@ -409,29 +61,6 @@ def test_read_binary__spec__unk_file(self): assert "No such file or directory" in str(x.value) assert "/dummy" in str(x.value) - def test_read_binary__spec__negative_offset(self): - """ - Test RemoteOperations::read_binary with negative offset. - """ - - with pytest.raises( - ValueError, - match=re.escape("Negative 'offset' is not supported.")): - self.operations.read_binary(__file__, -1) - - def test_get_file_size(self): - """ - Test RemoteOperations::get_file_size. - """ - filename = __file__ # current file - - sz0 = os.path.getsize(filename) - assert type(sz0) == int # noqa: E721 - - sz1 = self.operations.get_file_size(filename) - assert type(sz1) == int # noqa: E721 - assert sz1 == sz0 - def test_get_file_size__unk_file(self): """ Test RemoteOperations::get_file_size. @@ -443,155 +72,3 @@ def test_get_file_size__unk_file(self): assert "Utility exited with non-zero code (1)." in str(x.value) assert "No such file or directory" in str(x.value) assert "/dummy" in str(x.value) - - def test_touch(self): - """ - Test touch for creating a new file or updating access and modification times of an existing file. - """ - filename = "/tmp/test_file.txt" - - self.operations.touch(filename) - - assert self.operations.isfile(filename) - - def test_isfile_true(self): - """ - Test isfile for an existing file. - """ - filename = __file__ - - response = self.operations.isfile(filename) - - assert response is True - - def test_isfile_false__not_exist(self): - """ - Test isfile for a non-existing file. - """ - filename = os.path.join(os.path.dirname(__file__), "nonexistent_file.txt") - - response = self.operations.isfile(filename) - - assert response is False - - def test_isfile_false__directory(self): - """ - Test isfile for a firectory. - """ - name = os.path.dirname(__file__) - - assert self.operations.isdir(name) - - response = self.operations.isfile(name) - - assert response is False - - def test_isdir_true(self): - """ - Test isdir for an existing directory. - """ - name = os.path.dirname(__file__) - - response = self.operations.isdir(name) - - assert response is True - - def test_isdir_false__not_exist(self): - """ - Test isdir for a non-existing directory. - """ - name = os.path.join(os.path.dirname(__file__), "it_is_nonexistent_directory") - - response = self.operations.isdir(name) - - assert response is False - - def test_isdir_false__file(self): - """ - Test isdir for a file. - """ - name = __file__ - - assert self.operations.isfile(name) - - response = self.operations.isdir(name) - - assert response is False - - def test_cwd(self): - """ - Test cwd. - """ - v = self.operations.cwd() - - assert v is not None - assert type(v) == str # noqa: E721 - assert v != "" - - class tagWriteData001: - def __init__(self, sign, source, cp_rw, cp_truncate, cp_binary, cp_data, result): - self.sign = sign - self.source = source - self.call_param__rw = cp_rw - self.call_param__truncate = cp_truncate - self.call_param__binary = cp_binary - self.call_param__data = cp_data - self.result = result - - sm_write_data001 = [ - tagWriteData001("A001", "1234567890", False, False, False, "ABC", "1234567890ABC"), - tagWriteData001("A002", b"1234567890", False, False, True, b"ABC", b"1234567890ABC"), - - tagWriteData001("B001", "1234567890", False, True, False, "ABC", "ABC"), - tagWriteData001("B002", "1234567890", False, True, False, "ABC1234567890", "ABC1234567890"), - tagWriteData001("B003", b"1234567890", False, True, True, b"ABC", b"ABC"), - tagWriteData001("B004", b"1234567890", False, True, True, b"ABC1234567890", b"ABC1234567890"), - - tagWriteData001("C001", "1234567890", True, False, False, "ABC", "1234567890ABC"), - tagWriteData001("C002", b"1234567890", True, False, True, b"ABC", b"1234567890ABC"), - - tagWriteData001("D001", "1234567890", True, True, False, "ABC", "ABC"), - tagWriteData001("D002", "1234567890", True, True, False, "ABC1234567890", "ABC1234567890"), - tagWriteData001("D003", b"1234567890", True, True, True, b"ABC", b"ABC"), - tagWriteData001("D004", b"1234567890", True, True, True, b"ABC1234567890", b"ABC1234567890"), - - tagWriteData001("E001", "\0001234567890\000", False, False, False, "\000ABC\000", "\0001234567890\000\000ABC\000"), - tagWriteData001("E002", b"\0001234567890\000", False, False, True, b"\000ABC\000", b"\0001234567890\000\000ABC\000"), - - tagWriteData001("F001", "a\nb\n", False, False, False, ["c", "d"], "a\nb\nc\nd\n"), - tagWriteData001("F002", b"a\nb\n", False, False, True, [b"c", b"d"], b"a\nb\nc\nd\n"), - - tagWriteData001("G001", "a\nb\n", False, False, False, ["c\n\n", "d\n"], "a\nb\nc\nd\n"), - tagWriteData001("G002", b"a\nb\n", False, False, True, [b"c\n\n", b"d\n"], b"a\nb\nc\nd\n"), - ] - - @pytest.fixture( - params=sm_write_data001, - ids=[x.sign for x in sm_write_data001], - ) - def write_data001(self, request): - assert isinstance(request, pytest.FixtureRequest) - assert type(request.param) == __class__.tagWriteData001 # noqa: E721 - return request.param - - def test_write(self, write_data001): - assert type(write_data001) == __class__.tagWriteData001 # noqa: E721 - - mode = "w+b" if write_data001.call_param__binary else "w+" - - with tempfile.NamedTemporaryFile(mode=mode, delete=True) as tmp_file: - tmp_file.write(write_data001.source) - tmp_file.flush() - - self.operations.write( - tmp_file.name, - write_data001.call_param__data, - read_and_write=write_data001.call_param__rw, - truncate=write_data001.call_param__truncate, - binary=write_data001.call_param__binary) - - tmp_file.seek(0) - - s = tmp_file.read() - - assert s == write_data001.result From 1e4a1e37aa8fba5f5254a5a8fd00b2028cfaf4de Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Wed, 2 Apr 2025 17:58:33 +0300 Subject: [PATCH 188/216] Refactoring of tests (#232) * TestLocalOperations is refactored * TestRemoteOperations is refactored --- tests/test_local.py | 31 ++++++++++++++++------------- tests/test_remote.py | 47 ++++++++++++++++++++++++-------------------- 2 files changed, 43 insertions(+), 35 deletions(-) diff --git a/tests/test_local.py b/tests/test_local.py index 7b5e488d..e82df989 100644 --- a/tests/test_local.py +++ b/tests/test_local.py @@ -1,27 +1,27 @@ # coding: utf-8 +from .helpers.os_ops_descrs import OsOpsDescrs +from .helpers.os_ops_descrs import OsOperations + import os import pytest import re -from ..testgres import LocalOperations - class TestLocalOperations: + @pytest.fixture + def os_ops(self): + return OsOpsDescrs.sm_local_os_ops - @pytest.fixture(scope="function", autouse=True) - def setup(self): - self.operations = LocalOperations() - - def test_read__unknown_file(self): + def test_read__unknown_file(self, os_ops: OsOperations): """ Test LocalOperations::read with unknown file. """ with pytest.raises(FileNotFoundError, match=re.escape("[Errno 2] No such file or directory: '/dummy'")): - self.operations.read("/dummy") + os_ops.read("/dummy") - def test_read_binary__spec__unk_file(self): + def test_read_binary__spec__unk_file(self, os_ops: OsOperations): """ Test LocalOperations::read_binary with unknown file. """ @@ -29,21 +29,24 @@ def test_read_binary__spec__unk_file(self): with pytest.raises( FileNotFoundError, match=re.escape("[Errno 2] No such file or directory: '/dummy'")): - self.operations.read_binary("/dummy", 0) + os_ops.read_binary("/dummy", 0) - def test_get_file_size__unk_file(self): + def test_get_file_size__unk_file(self, os_ops: OsOperations): """ Test LocalOperations::get_file_size. """ + assert isinstance(os_ops, OsOperations) with pytest.raises(FileNotFoundError, match=re.escape("[Errno 2] No such file or directory: '/dummy'")): - self.operations.get_file_size("/dummy") + os_ops.get_file_size("/dummy") - def test_cwd(self): + def test_cwd(self, os_ops: OsOperations): """ Test cwd. """ - v = self.operations.cwd() + assert isinstance(os_ops, OsOperations) + + v = os_ops.cwd() assert v is not None assert type(v) == str # noqa: E721 diff --git a/tests/test_remote.py b/tests/test_remote.py index 565b2d20..a37e258e 100755 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -1,33 +1,35 @@ # coding: utf-8 -import os -import pytest +from .helpers.os_ops_descrs import OsOpsDescrs +from .helpers.os_ops_descrs import OsOperations from ..testgres import ExecUtilException -from ..testgres import RemoteOperations -from ..testgres import ConnectionParams + +import os +import pytest class TestRemoteOperations: + @pytest.fixture + def os_ops(self): + return OsOpsDescrs.sm_remote_os_ops - @pytest.fixture(scope="function", autouse=True) - def setup(self): - conn_params = ConnectionParams(host=os.getenv('RDBMS_TESTPOOL1_HOST') or '127.0.0.1', - username=os.getenv('USER'), - ssh_key=os.getenv('RDBMS_TESTPOOL_SSHKEY')) - self.operations = RemoteOperations(conn_params) + def test_rmdirs__try_to_delete_nonexist_path(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) - def test_rmdirs__try_to_delete_nonexist_path(self): path = "/root/test_dir" - assert self.operations.rmdirs(path, ignore_errors=False) is True + assert os_ops.rmdirs(path, ignore_errors=False) is True + + def test_rmdirs__try_to_delete_file(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) - def test_rmdirs__try_to_delete_file(self): - path = self.operations.mkstemp() + path = os_ops.mkstemp() + assert type(path) == str # noqa: E721 assert os.path.exists(path) with pytest.raises(ExecUtilException) as x: - self.operations.rmdirs(path, ignore_errors=False) + os_ops.rmdirs(path, ignore_errors=False) assert os.path.exists(path) assert type(x.value) == ExecUtilException # noqa: E721 @@ -37,37 +39,40 @@ def test_rmdirs__try_to_delete_file(self): assert type(x.value.exit_code) == int # noqa: E721 assert x.value.exit_code == 20 - def test_read__unknown_file(self): + def test_read__unknown_file(self, os_ops: OsOperations): """ Test RemoteOperations::read with unknown file. """ + assert isinstance(os_ops, OsOperations) with pytest.raises(ExecUtilException) as x: - self.operations.read("/dummy") + os_ops.read("/dummy") assert "Utility exited with non-zero code (1)." in str(x.value) assert "No such file or directory" in str(x.value) assert "/dummy" in str(x.value) - def test_read_binary__spec__unk_file(self): + def test_read_binary__spec__unk_file(self, os_ops: OsOperations): """ Test RemoteOperations::read_binary with unknown file. """ + assert isinstance(os_ops, OsOperations) with pytest.raises(ExecUtilException) as x: - self.operations.read_binary("/dummy", 0) + os_ops.read_binary("/dummy", 0) assert "Utility exited with non-zero code (1)." in str(x.value) assert "No such file or directory" in str(x.value) assert "/dummy" in str(x.value) - def test_get_file_size__unk_file(self): + def test_get_file_size__unk_file(self, os_ops: OsOperations): """ Test RemoteOperations::get_file_size. """ + assert isinstance(os_ops, OsOperations) with pytest.raises(ExecUtilException) as x: - self.operations.get_file_size("/dummy") + os_ops.get_file_size("/dummy") assert "Utility exited with non-zero code (1)." in str(x.value) assert "No such file or directory" in str(x.value) From 46598b476888606c7b197f263f4d828253cf9d1d Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Wed, 2 Apr 2025 20:48:19 +0300 Subject: [PATCH 189/216] Test files and classes were renamed (#233) CI for AltLinux (10, 11) tests all the "local" cases. --- Dockerfile--altlinux_10.tmpl | 2 +- Dockerfile--altlinux_11.tmpl | 2 +- run_tests.sh | 2 +- tests/{test_local.py => test_os_ops_local.py} | 2 +- tests/{test_remote.py => test_os_ops_remote.py} | 2 +- tests/{test_simple.py => test_testgres_local.py} | 2 +- tests/{test_simple_remote.py => test_testgres_remote.py} | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) rename tests/{test_local.py => test_os_ops_local.py} (98%) rename tests/{test_remote.py => test_os_ops_remote.py} (98%) rename tests/{test_simple.py => test_testgres_local.py} (99%) rename tests/{test_simple_remote.py => test_testgres_remote.py} (99%) diff --git a/Dockerfile--altlinux_10.tmpl b/Dockerfile--altlinux_10.tmpl index e60e9320..a75e35a0 100644 --- a/Dockerfile--altlinux_10.tmpl +++ b/Dockerfile--altlinux_10.tmpl @@ -115,4 +115,4 @@ ssh-keygen -t rsa -f ~/.ssh/id_rsa -q -N ''; \ cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys; \ chmod 600 ~/.ssh/authorized_keys; \ ls -la ~/.ssh/; \ -TEST_FILTER=\"TestgresTests or (TestTestgresCommon and (not remote_ops))\" bash ./run_tests.sh;" +TEST_FILTER=\"TestTestgresLocal or TestOsOpsLocal or local_ops\" bash ./run_tests.sh;" diff --git a/Dockerfile--altlinux_11.tmpl b/Dockerfile--altlinux_11.tmpl index 4b591632..5b43da20 100644 --- a/Dockerfile--altlinux_11.tmpl +++ b/Dockerfile--altlinux_11.tmpl @@ -115,4 +115,4 @@ ssh-keygen -t rsa -f ~/.ssh/id_rsa -q -N ''; \ cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys; \ chmod 600 ~/.ssh/authorized_keys; \ ls -la ~/.ssh/; \ -TEST_FILTER=\"TestgresTests or (TestTestgresCommon and (not remote_ops))\" bash ./run_tests.sh;" +TEST_FILTER=\"TestTestgresLocal or TestOsOpsLocal or local_ops\" bash ./run_tests.sh;" diff --git a/run_tests.sh b/run_tests.sh index a40a97cf..8202aff5 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -5,7 +5,7 @@ set -eux if [ -z ${TEST_FILTER+x} ]; \ -then export TEST_FILTER="TestgresTests or (TestTestgresCommon and (not remote_ops))"; \ +then export TEST_FILTER="TestTestgresLocal or (TestTestgresCommon and (not remote_ops))"; \ fi # fail early diff --git a/tests/test_local.py b/tests/test_os_ops_local.py similarity index 98% rename from tests/test_local.py rename to tests/test_os_ops_local.py index e82df989..2e3c30b7 100644 --- a/tests/test_local.py +++ b/tests/test_os_ops_local.py @@ -8,7 +8,7 @@ import re -class TestLocalOperations: +class TestOsOpsLocal: @pytest.fixture def os_ops(self): return OsOpsDescrs.sm_local_os_ops diff --git a/tests/test_remote.py b/tests/test_os_ops_remote.py similarity index 98% rename from tests/test_remote.py rename to tests/test_os_ops_remote.py index a37e258e..58b09242 100755 --- a/tests/test_remote.py +++ b/tests/test_os_ops_remote.py @@ -9,7 +9,7 @@ import pytest -class TestRemoteOperations: +class TestOsOpsRemote: @pytest.fixture def os_ops(self): return OsOpsDescrs.sm_remote_os_ops diff --git a/tests/test_simple.py b/tests/test_testgres_local.py similarity index 99% rename from tests/test_simple.py rename to tests/test_testgres_local.py index 6ca52cb0..01f975a0 100644 --- a/tests/test_simple.py +++ b/tests/test_testgres_local.py @@ -74,7 +74,7 @@ def rm_carriage_returns(out): return out -class TestgresTests: +class TestTestgresLocal: def test_node_repr(self): with get_new_node() as node: pattern = r"PostgresNode\(name='.+', port=.+, base_dir='.+'\)" diff --git a/tests/test_simple_remote.py b/tests/test_testgres_remote.py similarity index 99% rename from tests/test_simple_remote.py rename to tests/test_testgres_remote.py index c16fe53f..2142e5ba 100755 --- a/tests/test_simple_remote.py +++ b/tests/test_testgres_remote.py @@ -47,7 +47,7 @@ def good_properties(f): return True -class TestgresRemoteTests: +class TestTestgresRemote: sm_os_ops = OsOpsDescrs.sm_remote_os_ops @pytest.fixture(autouse=True, scope="class") From ecd5427b7db2181a8b36b01797576d837e05eb46 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Wed, 2 Apr 2025 21:55:28 +0300 Subject: [PATCH 190/216] PostgresNode::source_walsender is updated [revision] --- testgres/node.py | 4 +++- tests/test_testgres_common.py | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/testgres/node.py b/testgres/node.py index c8ae4204..4ab98ea1 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -431,9 +431,11 @@ def source_walsender(self): where application_name = %s """ - if not self.master: + if self.master is None: raise TestgresException("Node doesn't have a master") + assert type(self.master) == PostgresNode + # master should be on the same host assert self.master.host == self.host diff --git a/tests/test_testgres_common.py b/tests/test_testgres_common.py index 2440b8f0..7cfb203e 100644 --- a/tests/test_testgres_common.py +++ b/tests/test_testgres_common.py @@ -348,6 +348,7 @@ def LOCAL__check_auxiliary_pids__multiple_attempts( assert (con.pid > 0) with master.replicate().start() as replica: + assert type(replica) == PostgresNode # test __str__ method str(master.child_processes[0]) From afee4a55a04893240fe674db7b134b646b005531 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Wed, 2 Apr 2025 22:03:55 +0300 Subject: [PATCH 191/216] [FIX] formatting (flake8) --- testgres/node.py | 2 +- tests/test_testgres_common.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index 4ab98ea1..1f8fca6e 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -434,7 +434,7 @@ def source_walsender(self): if self.master is None: raise TestgresException("Node doesn't have a master") - assert type(self.master) == PostgresNode + assert type(self.master) == PostgresNode # noqa: E721 # master should be on the same host assert self.master.host == self.host diff --git a/tests/test_testgres_common.py b/tests/test_testgres_common.py index 7cfb203e..4e23c4af 100644 --- a/tests/test_testgres_common.py +++ b/tests/test_testgres_common.py @@ -348,7 +348,7 @@ def LOCAL__check_auxiliary_pids__multiple_attempts( assert (con.pid > 0) with master.replicate().start() as replica: - assert type(replica) == PostgresNode + assert type(replica) == PostgresNode # noqa: E721 # test __str__ method str(master.child_processes[0]) From 6e7d315ab45a3065d1440819a369f7781c54a87e Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Sat, 5 Apr 2025 20:23:55 +0300 Subject: [PATCH 192/216] [FIX] The call of RaiseError.CommandExecutionError is corrected [message arg] --- testgres/operations/remote_ops.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index e1ad6dac..a3ecf637 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -193,7 +193,7 @@ def is_executable(self, file): RaiseError.CommandExecutionError( cmd=command, exit_code=exit_status, - msg_arg=errMsg, + message=errMsg, error=error, out=output ) @@ -305,7 +305,7 @@ def path_exists(self, path): RaiseError.CommandExecutionError( cmd=command, exit_code=exit_status, - msg_arg=errMsg, + message=errMsg, error=error, out=output ) From 14bc733db60712c99df20faea3efe659f9d9dafb Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Sun, 6 Apr 2025 19:14:21 +0300 Subject: [PATCH 193/216] PortManager (#234) Main - Old PortManager was deleted - PostresNode uses new "abstract" interface PortManager to reserve/release port number. - PostgresNode.free_port always resets port number - OsOperations::is_free_port is added - PostgresNode.start raises exception (InvalidOperationException) if node is None Refactoring - PostgresNode::port is RO-property now. It throws if port is None - PostgresNode::host is RO-property now - PostgresNode::ssh_key is RO-property now - PostgresNode::name is RO-property now --- Dockerfile--altlinux_10.tmpl | 2 +- Dockerfile--altlinux_11.tmpl | 2 +- Dockerfile--ubuntu_24_04.tmpl | 1 + run_tests.sh | 2 +- setup.py | 2 +- testgres/__init__.py | 6 +- testgres/exceptions.py | 4 + testgres/helpers/__init__.py | 0 testgres/helpers/port_manager.py | 41 --- testgres/node.py | 167 +++++++-- testgres/operations/local_ops.py | 11 + testgres/operations/os_ops.py | 4 + testgres/operations/remote_ops.py | 48 +++ testgres/port_manager.py | 102 +++++ testgres/utils.py | 32 +- tests/helpers/global_data.py | 78 ++++ tests/helpers/os_ops_descrs.py | 32 -- tests/test_os_ops_common.py | 105 +++++- tests/test_os_ops_local.py | 4 +- tests/test_os_ops_remote.py | 4 +- tests/test_testgres_common.py | 592 ++++++++++++++++++++++-------- tests/test_testgres_local.py | 57 --- tests/test_testgres_remote.py | 128 +------ 23 files changed, 993 insertions(+), 431 deletions(-) delete mode 100644 testgres/helpers/__init__.py delete mode 100644 testgres/helpers/port_manager.py create mode 100644 testgres/port_manager.py create mode 100644 tests/helpers/global_data.py delete mode 100644 tests/helpers/os_ops_descrs.py diff --git a/Dockerfile--altlinux_10.tmpl b/Dockerfile--altlinux_10.tmpl index a75e35a0..d78b05f5 100644 --- a/Dockerfile--altlinux_10.tmpl +++ b/Dockerfile--altlinux_10.tmpl @@ -115,4 +115,4 @@ ssh-keygen -t rsa -f ~/.ssh/id_rsa -q -N ''; \ cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys; \ chmod 600 ~/.ssh/authorized_keys; \ ls -la ~/.ssh/; \ -TEST_FILTER=\"TestTestgresLocal or TestOsOpsLocal or local_ops\" bash ./run_tests.sh;" +TEST_FILTER=\"TestTestgresLocal or TestOsOpsLocal or local\" bash ./run_tests.sh;" diff --git a/Dockerfile--altlinux_11.tmpl b/Dockerfile--altlinux_11.tmpl index 5b43da20..5c88585d 100644 --- a/Dockerfile--altlinux_11.tmpl +++ b/Dockerfile--altlinux_11.tmpl @@ -115,4 +115,4 @@ ssh-keygen -t rsa -f ~/.ssh/id_rsa -q -N ''; \ cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys; \ chmod 600 ~/.ssh/authorized_keys; \ ls -la ~/.ssh/; \ -TEST_FILTER=\"TestTestgresLocal or TestOsOpsLocal or local_ops\" bash ./run_tests.sh;" +TEST_FILTER=\"TestTestgresLocal or TestOsOpsLocal or local\" bash ./run_tests.sh;" diff --git a/Dockerfile--ubuntu_24_04.tmpl b/Dockerfile--ubuntu_24_04.tmpl index 3bdc6640..7a559776 100644 --- a/Dockerfile--ubuntu_24_04.tmpl +++ b/Dockerfile--ubuntu_24_04.tmpl @@ -10,6 +10,7 @@ RUN apt install -y sudo curl ca-certificates RUN apt update RUN apt install -y openssh-server RUN apt install -y time +RUN apt install -y netcat-traditional RUN apt update RUN apt install -y postgresql-common diff --git a/run_tests.sh b/run_tests.sh index 8202aff5..65c17dbf 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -5,7 +5,7 @@ set -eux if [ -z ${TEST_FILTER+x} ]; \ -then export TEST_FILTER="TestTestgresLocal or (TestTestgresCommon and (not remote_ops))"; \ +then export TEST_FILTER="TestTestgresLocal or (TestTestgresCommon and (not remote))"; \ fi # fail early diff --git a/setup.py b/setup.py index 3f2474dd..b47a1d8a 100755 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ setup( version='1.10.5', name='testgres', - packages=['testgres', 'testgres.operations', 'testgres.helpers'], + packages=['testgres', 'testgres.operations'], description='Testing utility for PostgreSQL and its extensions', url='https://github.com/postgrespro/testgres', long_description=readme, diff --git a/testgres/__init__.py b/testgres/__init__.py index 665548d6..339ae62e 100644 --- a/testgres/__init__.py +++ b/testgres/__init__.py @@ -34,6 +34,7 @@ DumpFormat from .node import PostgresNode, NodeApp +from .node import PortManager from .utils import \ reserve_port, \ @@ -53,8 +54,6 @@ from .operations.local_ops import LocalOperations from .operations.remote_ops import RemoteOperations -from .helpers.port_manager import PortManager - __all__ = [ "get_new_node", "get_remote_node", @@ -64,7 +63,8 @@ "TestgresException", "ExecUtilException", "QueryException", "TimeoutException", "CatchUpException", "StartNodeException", "InitNodeException", "BackupException", "InvalidOperationException", "XLogMethod", "IsolationLevel", "NodeStatus", "ProcessType", "DumpFormat", "PostgresNode", "NodeApp", + "PortManager", "reserve_port", "release_port", "bound_ports", "get_bin_path", "get_pg_config", "get_pg_version", - "First", "Any", "PortManager", + "First", "Any", "OsOperations", "LocalOperations", "RemoteOperations", "ConnectionParams" ] diff --git a/testgres/exceptions.py b/testgres/exceptions.py index d61d4691..20c1a8cf 100644 --- a/testgres/exceptions.py +++ b/testgres/exceptions.py @@ -7,6 +7,10 @@ class TestgresException(Exception): pass +class PortForException(TestgresException): + pass + + @six.python_2_unicode_compatible class ExecUtilException(TestgresException): def __init__(self, message=None, command=None, exit_code=0, out=None, error=None): diff --git a/testgres/helpers/__init__.py b/testgres/helpers/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/testgres/helpers/port_manager.py b/testgres/helpers/port_manager.py deleted file mode 100644 index cfc5c096..00000000 --- a/testgres/helpers/port_manager.py +++ /dev/null @@ -1,41 +0,0 @@ -import socket -import random -from typing import Set, Iterable, Optional - - -class PortForException(Exception): - pass - - -class PortManager: - def __init__(self, ports_range=(1024, 65535)): - self.ports_range = ports_range - - @staticmethod - def is_port_free(port: int) -> bool: - """Check if a port is free to use.""" - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: - try: - s.bind(("", port)) - return True - except OSError: - return False - - def find_free_port(self, ports: Optional[Set[int]] = None, exclude_ports: Optional[Iterable[int]] = None) -> int: - """Return a random unused port number.""" - if ports is None: - ports = set(range(1024, 65535)) - - assert type(ports) == set # noqa: E721 - - if exclude_ports is not None: - assert isinstance(exclude_ports, Iterable) - ports.difference_update(exclude_ports) - - sampled_ports = random.sample(tuple(ports), min(len(ports), 100)) - - for port in sampled_ports: - if self.is_port_free(port): - return port - - raise PortForException("Can't select a port") diff --git a/testgres/node.py b/testgres/node.py index 1f8fca6e..5039fc43 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -1,4 +1,6 @@ # coding: utf-8 +from __future__ import annotations + import logging import os import random @@ -10,6 +12,7 @@ from queue import Queue import time +import typing try: from collections.abc import Iterable @@ -80,14 +83,16 @@ BackupException, \ InvalidOperationException +from .port_manager import PortManager +from .port_manager import PortManager__ThisHost +from .port_manager import PortManager__Generic + from .logger import TestgresLogger from .pubsub import Publication, Subscription from .standby import First -from . import utils - from .utils import \ PgVer, \ eprint, \ @@ -126,17 +131,31 @@ def __getattr__(self, name): return getattr(self.process, name) def __repr__(self): - return '{}(ptype={}, process={})'.format(self.__class__.__name__, - str(self.ptype), - repr(self.process)) + return '{}(ptype={}, process={})'.format( + self.__class__.__name__, + str(self.ptype), + repr(self.process)) class PostgresNode(object): # a max number of node start attempts _C_MAX_START_ATEMPTS = 5 - def __init__(self, name=None, base_dir=None, port=None, conn_params: ConnectionParams = ConnectionParams(), - bin_dir=None, prefix=None, os_ops=None): + _name: typing.Optional[str] + _port: typing.Optional[int] + _should_free_port: bool + _os_ops: OsOperations + _port_manager: PortManager + + def __init__(self, + name=None, + base_dir=None, + port: typing.Optional[int] = None, + conn_params: ConnectionParams = ConnectionParams(), + bin_dir=None, + prefix=None, + os_ops: typing.Optional[OsOperations] = None, + port_manager: typing.Optional[PortManager] = None): """ PostgresNode constructor. @@ -145,21 +164,26 @@ def __init__(self, name=None, base_dir=None, port=None, conn_params: ConnectionP port: port to accept connections. base_dir: path to node's data directory. bin_dir: path to node's binary directory. + os_ops: None or correct OS operation object. + port_manager: None or correct port manager object. """ + assert port is None or type(port) == int # noqa: E721 + assert os_ops is None or isinstance(os_ops, OsOperations) + assert port_manager is None or isinstance(port_manager, PortManager) # private if os_ops is None: - os_ops = __class__._get_os_ops(conn_params) + self._os_ops = __class__._get_os_ops(conn_params) else: assert conn_params is None + assert isinstance(os_ops, OsOperations) + self._os_ops = os_ops pass - assert os_ops is not None - assert isinstance(os_ops, OsOperations) - self._os_ops = os_ops + assert self._os_ops is not None + assert isinstance(self._os_ops, OsOperations) - self._pg_version = PgVer(get_pg_version2(os_ops, bin_dir)) - self._should_free_port = port is None + self._pg_version = PgVer(get_pg_version2(self._os_ops, bin_dir)) self._base_dir = base_dir self._bin_dir = bin_dir self._prefix = prefix @@ -167,12 +191,29 @@ def __init__(self, name=None, base_dir=None, port=None, conn_params: ConnectionP self._master = None # basic - self.name = name or generate_app_name() + self._name = name or generate_app_name() + + if port is not None: + assert type(port) == int # noqa: E721 + assert port_manager is None + self._port = port + self._should_free_port = False + self._port_manager = None + else: + if port_manager is not None: + assert isinstance(port_manager, PortManager) + self._port_manager = port_manager + else: + self._port_manager = __class__._get_port_manager(self._os_ops) - self.host = os_ops.host - self.port = port or utils.reserve_port() + assert self._port_manager is not None + assert isinstance(self._port_manager, PortManager) - self.ssh_key = os_ops.ssh_key + self._port = self._port_manager.reserve_port() # raises + assert type(self._port) == int # noqa: E721 + self._should_free_port = True + + assert type(self._port) == int # noqa: E721 # defaults for __exit__() self.cleanup_on_good_exit = testgres_config.node_cleanup_on_good_exit @@ -207,7 +248,11 @@ def __exit__(self, type, value, traceback): def __repr__(self): return "{}(name='{}', port={}, base_dir='{}')".format( - self.__class__.__name__, self.name, self.port, self.base_dir) + self.__class__.__name__, + self.name, + str(self._port) if self._port is not None else "None", + self.base_dir + ) @staticmethod def _get_os_ops(conn_params: ConnectionParams) -> OsOperations: @@ -221,19 +266,39 @@ def _get_os_ops(conn_params: ConnectionParams) -> OsOperations: return LocalOperations(conn_params) + @staticmethod + def _get_port_manager(os_ops: OsOperations) -> PortManager: + assert os_ops is not None + assert isinstance(os_ops, OsOperations) + + if isinstance(os_ops, LocalOperations): + return PortManager__ThisHost() + + # TODO: Throw the exception "Please define a port manager." ? + return PortManager__Generic(os_ops) + def clone_with_new_name_and_base_dir(self, name: str, base_dir: str): assert name is None or type(name) == str # noqa: E721 assert base_dir is None or type(base_dir) == str # noqa: E721 assert __class__ == PostgresNode + if self._port_manager is None: + raise InvalidOperationException("PostgresNode without PortManager can't be cloned.") + + assert self._port_manager is not None + assert isinstance(self._port_manager, PortManager) + assert self._os_ops is not None + assert isinstance(self._os_ops, OsOperations) + node = PostgresNode( name=name, base_dir=base_dir, conn_params=None, bin_dir=self._bin_dir, prefix=self._prefix, - os_ops=self._os_ops) + os_ops=self._os_ops, + port_manager=self._port_manager) return node @@ -243,6 +308,33 @@ def os_ops(self) -> OsOperations: assert isinstance(self._os_ops, OsOperations) return self._os_ops + @property + def name(self) -> str: + if self._name is None: + raise InvalidOperationException("PostgresNode name is not defined.") + assert type(self._name) == str # noqa: E721 + return self._name + + @property + def host(self) -> str: + assert self._os_ops is not None + assert isinstance(self._os_ops, OsOperations) + return self._os_ops.host + + @property + def port(self) -> int: + if self._port is None: + raise InvalidOperationException("PostgresNode port is not defined.") + + assert type(self._port) == int # noqa: E721 + return self._port + + @property + def ssh_key(self) -> typing.Optional[str]: + assert self._os_ops is not None + assert isinstance(self._os_ops, OsOperations) + return self._os_ops.ssh_key + @property def pid(self): """ @@ -993,6 +1085,11 @@ def start(self, params=[], wait=True): if self.is_started: return self + if self._port is None: + raise InvalidOperationException("Can't start PostgresNode. Port is not defined.") + + assert type(self._port) == int # noqa: E721 + _params = [self._get_bin_path("pg_ctl"), "-D", self.data_dir, "-l", self.pg_log_file, @@ -1023,6 +1120,8 @@ def LOCAL__raise_cannot_start_node__std(from_exception): LOCAL__raise_cannot_start_node__std(e) else: assert self._should_free_port + assert self._port_manager is not None + assert isinstance(self._port_manager, PortManager) assert __class__._C_MAX_START_ATEMPTS > 1 log_files0 = self._collect_log_files() @@ -1048,20 +1147,20 @@ def LOCAL__raise_cannot_start_node__std(from_exception): log_files0 = log_files1 logging.warning( - "Detected a conflict with using the port {0}. Trying another port after a {1}-second sleep...".format(self.port, timeout) + "Detected a conflict with using the port {0}. Trying another port after a {1}-second sleep...".format(self._port, timeout) ) time.sleep(timeout) timeout = min(2 * timeout, 5) - cur_port = self.port - new_port = utils.reserve_port() # can raise + cur_port = self._port + new_port = self._port_manager.reserve_port() # can raise try: options = {'port': new_port} self.set_auto_conf(options) except: # noqa: E722 - utils.release_port(new_port) + self._port_manager.release_port(new_port) raise - self.port = new_port - utils.release_port(cur_port) + self._port = new_port + self._port_manager.release_port(cur_port) continue break self._maybe_start_logger() @@ -1222,14 +1321,22 @@ def pg_ctl(self, params): def free_port(self): """ Reclaim port owned by this node. - NOTE: does not free auto selected ports. + NOTE: this method does not release manually defined port but reset it. """ + assert type(self._should_free_port) == bool # noqa: E721 + + if not self._should_free_port: + self._port = None + else: + assert type(self._port) == int # noqa: E721 + + assert self._port_manager is not None + assert isinstance(self._port_manager, PortManager) - if self._should_free_port: - port = self.port + port = self._port self._should_free_port = False - self.port = None - utils.release_port(port) + self._port = None + self._port_manager.release_port(port) def cleanup(self, max_attempts=3, full=False): """ diff --git a/testgres/operations/local_ops.py b/testgres/operations/local_ops.py index 35e94210..39c81405 100644 --- a/testgres/operations/local_ops.py +++ b/testgres/operations/local_ops.py @@ -6,6 +6,7 @@ import subprocess import tempfile import time +import socket import psutil @@ -436,6 +437,16 @@ def get_process_children(self, pid): assert type(pid) == int # noqa: E721 return psutil.Process(pid).children() + def is_port_free(self, number: int) -> bool: + assert type(number) == int # noqa: E721 + + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + try: + s.bind(("", number)) + return True + except OSError: + return False + # Database control def db_connect(self, dbname, user, password=None, host="localhost", port=5432): conn = pglib.connect( diff --git a/testgres/operations/os_ops.py b/testgres/operations/os_ops.py index 3c606871..489a7cb2 100644 --- a/testgres/operations/os_ops.py +++ b/testgres/operations/os_ops.py @@ -127,6 +127,10 @@ def get_pid(self): def get_process_children(self, pid): raise NotImplementedError() + def is_port_free(self, number: int): + assert type(number) == int # noqa: E721 + raise NotImplementedError() + # Database control def db_connect(self, dbname, user, password=None, host="localhost", port=5432): raise NotImplementedError() diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index a3ecf637..ee747e52 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -629,6 +629,54 @@ def get_process_children(self, pid): raise ExecUtilException(f"Error in getting process children. Error: {result.stderr}") + def is_port_free(self, number: int) -> bool: + assert type(number) == int # noqa: E721 + + cmd = ["nc", "-w", "5", "-z", "-v", "localhost", str(number)] + + exit_status, output, error = self.exec_command(cmd=cmd, encoding=get_default_encoding(), ignore_errors=True, verbose=True) + + assert type(output) == str # noqa: E721 + assert type(error) == str # noqa: E721 + + if exit_status == 0: + return __class__._is_port_free__process_0(error) + + if exit_status == 1: + return __class__._is_port_free__process_1(error) + + errMsg = "nc returns an unknown result code: {0}".format(exit_status) + + RaiseError.CommandExecutionError( + cmd=cmd, + exit_code=exit_status, + message=errMsg, + error=error, + out=output + ) + + @staticmethod + def _is_port_free__process_0(error: str) -> bool: + assert type(error) == str # noqa: E721 + # + # Example of error text: + # "Connection to localhost (127.0.0.1) 1024 port [tcp/*] succeeded!\n" + # + # May be here is needed to check error message? + # + return False + + @staticmethod + def _is_port_free__process_1(error: str) -> bool: + assert type(error) == str # noqa: E721 + # + # Example of error text: + # "nc: connect to localhost (127.0.0.1) port 1024 (tcp) failed: Connection refused\n" + # + # May be here is needed to check error message? + # + return True + # Database control def db_connect(self, dbname, user, password=None, host="localhost", port=5432): conn = pglib.connect( diff --git a/testgres/port_manager.py b/testgres/port_manager.py new file mode 100644 index 00000000..164661e7 --- /dev/null +++ b/testgres/port_manager.py @@ -0,0 +1,102 @@ +from .operations.os_ops import OsOperations + +from .exceptions import PortForException + +from . import utils + +import threading +import random + + +class PortManager: + def __init__(self): + super().__init__() + + def reserve_port(self) -> int: + raise NotImplementedError("PortManager::reserve_port is not implemented.") + + def release_port(self, number: int) -> None: + assert type(number) == int # noqa: E721 + raise NotImplementedError("PortManager::release_port is not implemented.") + + +class PortManager__ThisHost(PortManager): + sm_single_instance: PortManager = None + sm_single_instance_guard = threading.Lock() + + def __init__(self): + pass + + def __new__(cls) -> PortManager: + assert __class__ == PortManager__ThisHost + assert __class__.sm_single_instance_guard is not None + + if __class__.sm_single_instance is None: + with __class__.sm_single_instance_guard: + __class__.sm_single_instance = super().__new__(cls) + assert __class__.sm_single_instance + assert type(__class__.sm_single_instance) == __class__ # noqa: E721 + return __class__.sm_single_instance + + def reserve_port(self) -> int: + return utils.reserve_port() + + def release_port(self, number: int) -> None: + assert type(number) == int # noqa: E721 + return utils.release_port(number) + + +class PortManager__Generic(PortManager): + _os_ops: OsOperations + _guard: object + # TODO: is there better to use bitmap fot _available_ports? + _available_ports: set[int] + _reserved_ports: set[int] + + def __init__(self, os_ops: OsOperations): + assert os_ops is not None + assert isinstance(os_ops, OsOperations) + self._os_ops = os_ops + self._guard = threading.Lock() + self._available_ports = set[int](range(1024, 65535)) + self._reserved_ports = set[int]() + + def reserve_port(self) -> int: + assert self._guard is not None + assert type(self._available_ports) == set # noqa: E721t + assert type(self._reserved_ports) == set # noqa: E721 + + with self._guard: + t = tuple(self._available_ports) + assert len(t) == len(self._available_ports) + sampled_ports = random.sample(t, min(len(t), 100)) + t = None + + for port in sampled_ports: + assert not (port in self._reserved_ports) + assert port in self._available_ports + + if not self._os_ops.is_port_free(port): + continue + + self._reserved_ports.add(port) + self._available_ports.discard(port) + assert port in self._reserved_ports + assert not (port in self._available_ports) + return port + + raise PortForException("Can't select a port.") + + def release_port(self, number: int) -> None: + assert type(number) == int # noqa: E721 + + assert self._guard is not None + assert type(self._reserved_ports) == set # noqa: E721 + + with self._guard: + assert number in self._reserved_ports + assert not (number in self._available_ports) + self._available_ports.add(number) + self._reserved_ports.discard(number) + assert not (number in self._reserved_ports) + assert number in self._available_ports diff --git a/testgres/utils.py b/testgres/utils.py index 92383571..10ae81b6 100644 --- a/testgres/utils.py +++ b/testgres/utils.py @@ -6,6 +6,8 @@ import os import sys +import socket +import random from contextlib import contextmanager from packaging.version import Version, InvalidVersion @@ -13,7 +15,7 @@ from six import iteritems -from .helpers.port_manager import PortManager +from .exceptions import PortForException from .exceptions import ExecUtilException from .config import testgres_config as tconf from .operations.os_ops import OsOperations @@ -41,11 +43,28 @@ def internal__reserve_port(): """ Generate a new port and add it to 'bound_ports'. """ - port_mng = PortManager() - port = port_mng.find_free_port(exclude_ports=bound_ports) - bound_ports.add(port) + def LOCAL__is_port_free(port: int) -> bool: + """Check if a port is free to use.""" + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + try: + s.bind(("", port)) + return True + except OSError: + return False - return port + ports = set(range(1024, 65535)) + assert type(ports) == set # noqa: E721 + assert type(bound_ports) == set # noqa: E721 + ports.difference_update(bound_ports) + + sampled_ports = random.sample(tuple(ports), min(len(ports), 100)) + + for port in sampled_ports: + if LOCAL__is_port_free(port): + bound_ports.add(port) + return port + + raise PortForException("Can't select a port") def internal__release_port(port): @@ -53,6 +72,9 @@ def internal__release_port(port): Free port provided by reserve_port(). """ + assert type(port) == int # noqa: E721 + assert port in bound_ports + bound_ports.discard(port) diff --git a/tests/helpers/global_data.py b/tests/helpers/global_data.py new file mode 100644 index 00000000..c21d7dd8 --- /dev/null +++ b/tests/helpers/global_data.py @@ -0,0 +1,78 @@ +from ...testgres.operations.os_ops import OsOperations +from ...testgres.operations.os_ops import ConnectionParams +from ...testgres.operations.local_ops import LocalOperations +from ...testgres.operations.remote_ops import RemoteOperations + +from ...testgres.node import PortManager +from ...testgres.node import PortManager__ThisHost +from ...testgres.node import PortManager__Generic + +import os + + +class OsOpsDescr: + sign: str + os_ops: OsOperations + + def __init__(self, sign: str, os_ops: OsOperations): + assert type(sign) == str # noqa: E721 + assert isinstance(os_ops, OsOperations) + self.sign = sign + self.os_ops = os_ops + + +class OsOpsDescrs: + sm_remote_conn_params = ConnectionParams( + host=os.getenv('RDBMS_TESTPOOL1_HOST') or '127.0.0.1', + username=os.getenv('USER'), + ssh_key=os.getenv('RDBMS_TESTPOOL_SSHKEY')) + + sm_remote_os_ops = RemoteOperations(sm_remote_conn_params) + + sm_remote_os_ops_descr = OsOpsDescr("remote_ops", sm_remote_os_ops) + + sm_local_os_ops = LocalOperations() + + sm_local_os_ops_descr = OsOpsDescr("local_ops", sm_local_os_ops) + + +class PortManagers: + sm_remote_port_manager = PortManager__Generic(OsOpsDescrs.sm_remote_os_ops) + + sm_local_port_manager = PortManager__ThisHost() + + sm_local2_port_manager = PortManager__Generic(OsOpsDescrs.sm_local_os_ops) + + +class PostgresNodeService: + sign: str + os_ops: OsOperations + port_manager: PortManager + + def __init__(self, sign: str, os_ops: OsOperations, port_manager: PortManager): + assert type(sign) == str # noqa: E721 + assert isinstance(os_ops, OsOperations) + assert isinstance(port_manager, PortManager) + self.sign = sign + self.os_ops = os_ops + self.port_manager = port_manager + + +class PostgresNodeServices: + sm_remote = PostgresNodeService( + "remote", + OsOpsDescrs.sm_remote_os_ops, + PortManagers.sm_remote_port_manager + ) + + sm_local = PostgresNodeService( + "local", + OsOpsDescrs.sm_local_os_ops, + PortManagers.sm_local_port_manager + ) + + sm_local2 = PostgresNodeService( + "local2", + OsOpsDescrs.sm_local_os_ops, + PortManagers.sm_local2_port_manager + ) diff --git a/tests/helpers/os_ops_descrs.py b/tests/helpers/os_ops_descrs.py deleted file mode 100644 index 02297adb..00000000 --- a/tests/helpers/os_ops_descrs.py +++ /dev/null @@ -1,32 +0,0 @@ -from ...testgres.operations.os_ops import OsOperations -from ...testgres.operations.os_ops import ConnectionParams -from ...testgres.operations.local_ops import LocalOperations -from ...testgres.operations.remote_ops import RemoteOperations - -import os - - -class OsOpsDescr: - os_ops: OsOperations - sign: str - - def __init__(self, os_ops: OsOperations, sign: str): - assert isinstance(os_ops, OsOperations) - assert type(sign) == str # noqa: E721 - self.os_ops = os_ops - self.sign = sign - - -class OsOpsDescrs: - sm_remote_conn_params = ConnectionParams( - host=os.getenv('RDBMS_TESTPOOL1_HOST') or '127.0.0.1', - username=os.getenv('USER'), - ssh_key=os.getenv('RDBMS_TESTPOOL_SSHKEY')) - - sm_remote_os_ops = RemoteOperations(sm_remote_conn_params) - - sm_remote_os_ops_descr = OsOpsDescr(sm_remote_os_ops, "remote_ops") - - sm_local_os_ops = LocalOperations() - - sm_local_os_ops_descr = OsOpsDescr(sm_local_os_ops, "local_ops") diff --git a/tests/test_os_ops_common.py b/tests/test_os_ops_common.py index c3944c3b..7d183775 100644 --- a/tests/test_os_ops_common.py +++ b/tests/test_os_ops_common.py @@ -1,7 +1,7 @@ # coding: utf-8 -from .helpers.os_ops_descrs import OsOpsDescr -from .helpers.os_ops_descrs import OsOpsDescrs -from .helpers.os_ops_descrs import OsOperations +from .helpers.global_data import OsOpsDescr +from .helpers.global_data import OsOpsDescrs +from .helpers.global_data import OsOperations from .helpers.run_conditions import RunConditions import os @@ -10,6 +10,8 @@ import re import tempfile import logging +import socket +import threading from ..testgres import InvalidOperationException from ..testgres import ExecUtilException @@ -648,3 +650,100 @@ def test_touch(self, os_ops: OsOperations): assert os_ops.isfile(filename) os_ops.remove_file(filename) + + def test_is_port_free__true(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + + C_LIMIT = 128 + + ports = set(range(1024, 65535)) + assert type(ports) == set # noqa: E721 + + ok_count = 0 + no_count = 0 + + for port in ports: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + try: + s.bind(("", port)) + except OSError: + continue + + r = os_ops.is_port_free(port) + + if r: + ok_count += 1 + logging.info("OK. Port {} is free.".format(port)) + else: + no_count += 1 + logging.warning("NO. Port {} is not free.".format(port)) + + if ok_count == C_LIMIT: + return + + if no_count == C_LIMIT: + raise RuntimeError("To many false positive test attempts.") + + if ok_count == 0: + raise RuntimeError("No one free port was found.") + + def test_is_port_free__false(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + + C_LIMIT = 10 + + ports = set(range(1024, 65535)) + assert type(ports) == set # noqa: E721 + + def LOCAL_server(s: socket.socket): + assert s is not None + assert type(s) == socket.socket # noqa: E721 + + try: + while True: + r = s.accept() + + if r is None: + break + except Exception as e: + assert e is not None + pass + + ok_count = 0 + no_count = 0 + + for port in ports: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + try: + s.bind(("", port)) + except OSError: + continue + + th = threading.Thread(target=LOCAL_server, args=[s]) + + s.listen(10) + + assert type(th) == threading.Thread # noqa: E721 + th.start() + + try: + r = os_ops.is_port_free(port) + finally: + s.shutdown(2) + th.join() + + if not r: + ok_count += 1 + logging.info("OK. Port {} is not free.".format(port)) + else: + no_count += 1 + logging.warning("NO. Port {} does not accept connection.".format(port)) + + if ok_count == C_LIMIT: + return + + if no_count == C_LIMIT: + raise RuntimeError("To many false positive test attempts.") + + if ok_count == 0: + raise RuntimeError("No one free port was found.") diff --git a/tests/test_os_ops_local.py b/tests/test_os_ops_local.py index 2e3c30b7..f60c3fc9 100644 --- a/tests/test_os_ops_local.py +++ b/tests/test_os_ops_local.py @@ -1,6 +1,6 @@ # coding: utf-8 -from .helpers.os_ops_descrs import OsOpsDescrs -from .helpers.os_ops_descrs import OsOperations +from .helpers.global_data import OsOpsDescrs +from .helpers.global_data import OsOperations import os diff --git a/tests/test_os_ops_remote.py b/tests/test_os_ops_remote.py index 58b09242..338e49f3 100755 --- a/tests/test_os_ops_remote.py +++ b/tests/test_os_ops_remote.py @@ -1,7 +1,7 @@ # coding: utf-8 -from .helpers.os_ops_descrs import OsOpsDescrs -from .helpers.os_ops_descrs import OsOperations +from .helpers.global_data import OsOpsDescrs +from .helpers.global_data import OsOperations from ..testgres import ExecUtilException diff --git a/tests/test_testgres_common.py b/tests/test_testgres_common.py index 4e23c4af..b286a1c6 100644 --- a/tests/test_testgres_common.py +++ b/tests/test_testgres_common.py @@ -1,6 +1,7 @@ -from .helpers.os_ops_descrs import OsOpsDescr -from .helpers.os_ops_descrs import OsOpsDescrs -from .helpers.os_ops_descrs import OsOperations +from .helpers.global_data import PostgresNodeService +from .helpers.global_data import PostgresNodeServices +from .helpers.global_data import OsOperations +from .helpers.global_data import PortManager from ..testgres.node import PgVer from ..testgres.node import PostgresNode @@ -37,6 +38,8 @@ import uuid import os import re +import subprocess +import typing @contextmanager @@ -54,22 +57,25 @@ def removing(os_ops: OsOperations, f): class TestTestgresCommon: - sm_os_ops_descrs: list[OsOpsDescr] = [ - OsOpsDescrs.sm_local_os_ops_descr, - OsOpsDescrs.sm_remote_os_ops_descr + sm_node_svcs: list[PostgresNodeService] = [ + PostgresNodeServices.sm_local, + PostgresNodeServices.sm_local2, + PostgresNodeServices.sm_remote, ] @pytest.fixture( - params=[descr.os_ops for descr in sm_os_ops_descrs], - ids=[descr.sign for descr in sm_os_ops_descrs] + params=sm_node_svcs, + ids=[descr.sign for descr in sm_node_svcs] ) - def os_ops(self, request: pytest.FixtureRequest) -> OsOperations: + def node_svc(self, request: pytest.FixtureRequest) -> PostgresNodeService: assert isinstance(request, pytest.FixtureRequest) - assert isinstance(request.param, OsOperations) + assert isinstance(request.param, PostgresNodeService) + assert isinstance(request.param.os_ops, OsOperations) + assert isinstance(request.param.port_manager, PortManager) return request.param - def test_version_management(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) + def test_version_management(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) a = PgVer('10.0') b = PgVer('10') @@ -93,42 +99,42 @@ def test_version_management(self, os_ops: OsOperations): assert (g == k) assert (g > h) - version = get_pg_version2(os_ops) + version = get_pg_version2(node_svc.os_ops) - with __class__.helper__get_node(os_ops) as node: + with __class__.helper__get_node(node_svc) as node: assert (isinstance(version, six.string_types)) assert (isinstance(node.version, PgVer)) assert (node.version == PgVer(version)) - def test_double_init(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) + def test_double_init(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) - with __class__.helper__get_node(os_ops).init() as node: + with __class__.helper__get_node(node_svc).init() as node: # can't initialize node more than once with pytest.raises(expected_exception=InitNodeException): node.init() - def test_init_after_cleanup(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) + def test_init_after_cleanup(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) - with __class__.helper__get_node(os_ops) as node: + with __class__.helper__get_node(node_svc) as node: node.init().start().execute('select 1') node.cleanup() node.init().start().execute('select 1') - def test_init_unique_system_id(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) + def test_init_unique_system_id(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) # this function exists in PostgreSQL 9.6+ - current_version = get_pg_version2(os_ops) + current_version = get_pg_version2(node_svc.os_ops) - __class__.helper__skip_test_if_util_not_exist(os_ops, "pg_resetwal") + __class__.helper__skip_test_if_util_not_exist(node_svc.os_ops, "pg_resetwal") __class__.helper__skip_test_if_pg_version_is_not_ge(current_version, '9.6') query = 'select system_identifier from pg_control_system()' with scoped_config(cache_initdb=False): - with __class__.helper__get_node(os_ops).init().start() as node0: + with __class__.helper__get_node(node_svc).init().start() as node0: id0 = node0.execute(query)[0] with scoped_config(cache_initdb=True, @@ -137,8 +143,8 @@ def test_init_unique_system_id(self, os_ops: OsOperations): assert (config.cached_initdb_unique) # spawn two nodes; ids must be different - with __class__.helper__get_node(os_ops).init().start() as node1, \ - __class__.helper__get_node(os_ops).init().start() as node2: + with __class__.helper__get_node(node_svc).init().start() as node1, \ + __class__.helper__get_node(node_svc).init().start() as node2: id1 = node1.execute(query)[0] id2 = node2.execute(query)[0] @@ -146,44 +152,44 @@ def test_init_unique_system_id(self, os_ops: OsOperations): assert (id1 > id0) assert (id2 > id1) - def test_node_exit(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) + def test_node_exit(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) with pytest.raises(expected_exception=QueryException): - with __class__.helper__get_node(os_ops).init() as node: + with __class__.helper__get_node(node_svc).init() as node: base_dir = node.base_dir node.safe_psql('select 1') # we should save the DB for "debugging" - assert (os_ops.path_exists(base_dir)) - os_ops.rmdirs(base_dir, ignore_errors=True) + assert (node_svc.os_ops.path_exists(base_dir)) + node_svc.os_ops.rmdirs(base_dir, ignore_errors=True) - with __class__.helper__get_node(os_ops).init() as node: + with __class__.helper__get_node(node_svc).init() as node: base_dir = node.base_dir # should have been removed by default - assert not (os_ops.path_exists(base_dir)) + assert not (node_svc.os_ops.path_exists(base_dir)) - def test_double_start(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) + def test_double_start(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) - with __class__.helper__get_node(os_ops).init().start() as node: + with __class__.helper__get_node(node_svc).init().start() as node: # can't start node more than once node.start() assert (node.is_started) - def test_uninitialized_start(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) + def test_uninitialized_start(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) - with __class__.helper__get_node(os_ops) as node: + with __class__.helper__get_node(node_svc) as node: # node is not initialized yet with pytest.raises(expected_exception=StartNodeException): node.start() - def test_restart(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) + def test_restart(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) - with __class__.helper__get_node(os_ops) as node: + with __class__.helper__get_node(node_svc) as node: node.init().start() # restart, ok @@ -198,10 +204,10 @@ def test_restart(self, os_ops: OsOperations): node.append_conf('pg_hba.conf', 'DUMMY') node.restart() - def test_reload(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) + def test_reload(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) - with __class__.helper__get_node(os_ops) as node: + with __class__.helper__get_node(node_svc) as node: node.init().start() # change client_min_messages and save old value @@ -216,24 +222,24 @@ def test_reload(self, os_ops: OsOperations): assert ('debug1' == cmm_new[0][0].lower()) assert (cmm_old != cmm_new) - def test_pg_ctl(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) + def test_pg_ctl(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) - with __class__.helper__get_node(os_ops) as node: + with __class__.helper__get_node(node_svc) as node: node.init().start() status = node.pg_ctl(['status']) assert ('PID' in status) - def test_status(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) + def test_status(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) assert (NodeStatus.Running) assert not (NodeStatus.Stopped) assert not (NodeStatus.Uninitialized) # check statuses after each operation - with __class__.helper__get_node(os_ops) as node: + with __class__.helper__get_node(node_svc) as node: assert (node.pid == 0) assert (node.status() == NodeStatus.Uninitialized) @@ -257,8 +263,8 @@ def test_status(self, os_ops: OsOperations): assert (node.pid == 0) assert (node.status() == NodeStatus.Uninitialized) - def test_child_pids(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) + def test_child_pids(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) master_processes = [ ProcessType.AutovacuumLauncher, @@ -269,7 +275,7 @@ def test_child_pids(self, os_ops: OsOperations): ProcessType.WalWriter, ] - postgresVersion = get_pg_version2(os_ops) + postgresVersion = get_pg_version2(node_svc.os_ops) if __class__.helper__pg_version_ge(postgresVersion, '10'): master_processes.append(ProcessType.LogicalReplicationLauncher) @@ -338,7 +344,7 @@ def LOCAL__check_auxiliary_pids__multiple_attempts( absenceList )) - with __class__.helper__get_node(os_ops).init().start() as master: + with __class__.helper__get_node(node_svc).init().start() as master: # master node doesn't have a source walsender! with pytest.raises(expected_exception=testgres_TestgresException): @@ -380,10 +386,10 @@ def test_exceptions(self): str(ExecUtilException('msg', 'cmd', 1, 'out')) str(QueryException('msg', 'query')) - def test_auto_name(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) + def test_auto_name(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) - with __class__.helper__get_node(os_ops).init(allow_streaming=True).start() as m: + with __class__.helper__get_node(node_svc).init(allow_streaming=True).start() as m: with m.replicate().start() as r: # check that nodes are running assert (m.status()) @@ -417,9 +423,9 @@ def test_file_tail(self): lines = file_tail(f, 1) assert (lines[0] == s3) - def test_isolation_levels(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) - with __class__.helper__get_node(os_ops).init().start() as node: + def test_isolation_levels(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc).init().start() as node: with node.connect() as con: # string levels con.begin('Read Uncommitted').commit() @@ -437,17 +443,17 @@ def test_isolation_levels(self, os_ops: OsOperations): with pytest.raises(expected_exception=QueryException): con.begin('Garbage').commit() - def test_users(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) - with __class__.helper__get_node(os_ops).init().start() as node: + def test_users(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc).init().start() as node: node.psql('create role test_user login') value = node.safe_psql('select 1', username='test_user') value = __class__.helper__rm_carriage_returns(value) assert (value == b'1\n') - def test_poll_query_until(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) - with __class__.helper__get_node(os_ops) as node: + def test_poll_query_until(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc) as node: node.init().start() get_time = 'select extract(epoch from now())' @@ -507,8 +513,8 @@ def test_poll_query_until(self, os_ops: OsOperations): # check 1 arg, ok node.poll_query_until('select true') - def test_logging(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) + def test_logging(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) C_MAX_ATTEMPTS = 50 # This name is used for testgres logging, too. C_NODE_NAME = "testgres_tests." + __class__.__name__ + "test_logging-master-" + uuid.uuid4().hex @@ -529,7 +535,7 @@ def test_logging(self, os_ops: OsOperations): logger.addHandler(handler) with scoped_config(use_python_logging=True): - with __class__.helper__get_node(os_ops, name=C_NODE_NAME) as master: + with __class__.helper__get_node(node_svc, name=C_NODE_NAME) as master: logging.info("Master node is initilizing") master.init() @@ -599,9 +605,9 @@ def LOCAL__test_lines(): # GO HOME! return - def test_psql(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) - with __class__.helper__get_node(os_ops).init().start() as node: + def test_psql(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc).init().start() as node: # check returned values (1 arg) res = node.psql('select 1') @@ -636,17 +642,20 @@ def test_psql(self, os_ops: OsOperations): # check psql's default args, fails with pytest.raises(expected_exception=QueryException): - node.psql() + r = node.psql() # raises! + logging.error("node.psql returns [{}]".format(r)) node.stop() # check psql on stopped node, fails with pytest.raises(expected_exception=QueryException): - node.safe_psql('select 1') + # [2025-04-03] This call does not raise exception! I do not know why. + r = node.safe_psql('select 1') # raises! + logging.error("node.safe_psql returns [{}]".format(r)) - def test_safe_psql__expect_error(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) - with __class__.helper__get_node(os_ops).init().start() as node: + def test_safe_psql__expect_error(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc).init().start() as node: err = node.safe_psql('select_or_not_select 1', expect_error=True) assert (type(err) == str) # noqa: E721 assert ('select_or_not_select' in err) @@ -663,9 +672,9 @@ def test_safe_psql__expect_error(self, os_ops: OsOperations): res = node.safe_psql("select 1;", expect_error=False) assert (__class__.helper__rm_carriage_returns(res) == b'1\n') - def test_transactions(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) - with __class__.helper__get_node(os_ops).init().start() as node: + def test_transactions(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc).init().start() as node: with node.connect() as con: con.begin() @@ -688,9 +697,9 @@ def test_transactions(self, os_ops: OsOperations): con.execute('drop table test') con.commit() - def test_control_data(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) - with __class__.helper__get_node(os_ops) as node: + def test_control_data(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc) as node: # node is not initialized yet with pytest.raises(expected_exception=ExecUtilException): @@ -703,9 +712,9 @@ def test_control_data(self, os_ops: OsOperations): assert data is not None assert (any('pg_control' in s for s in data.keys())) - def test_backup_simple(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) - with __class__.helper__get_node(os_ops) as master: + def test_backup_simple(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc) as master: # enable streaming for backups master.init(allow_streaming=True) @@ -725,9 +734,9 @@ def test_backup_simple(self, os_ops: OsOperations): res = slave.execute('select * from test order by i asc') assert (res == [(1, ), (2, ), (3, ), (4, )]) - def test_backup_multiple(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) - with __class__.helper__get_node(os_ops) as node: + def test_backup_multiple(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc) as node: node.init(allow_streaming=True).start() with node.backup(xlog_method='fetch') as backup1, \ @@ -739,9 +748,9 @@ def test_backup_multiple(self, os_ops: OsOperations): backup.spawn_primary('node2', destroy=False) as node2: assert (node1.base_dir != node2.base_dir) - def test_backup_exhaust(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) - with __class__.helper__get_node(os_ops) as node: + def test_backup_exhaust(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc) as node: node.init(allow_streaming=True).start() with node.backup(xlog_method='fetch') as backup: @@ -753,9 +762,9 @@ def test_backup_exhaust(self, os_ops: OsOperations): with pytest.raises(expected_exception=BackupException): backup.spawn_primary() - def test_backup_wrong_xlog_method(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) - with __class__.helper__get_node(os_ops) as node: + def test_backup_wrong_xlog_method(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc) as node: node.init(allow_streaming=True).start() with pytest.raises( @@ -764,11 +773,11 @@ def test_backup_wrong_xlog_method(self, os_ops: OsOperations): ): node.backup(xlog_method='wrong') - def test_pg_ctl_wait_option(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) + def test_pg_ctl_wait_option(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) C_MAX_ATTEMPTS = 50 - node = __class__.helper__get_node(os_ops) + node = __class__.helper__get_node(node_svc) assert node.status() == NodeStatus.Uninitialized node.init() assert node.status() == NodeStatus.Stopped @@ -835,9 +844,9 @@ def test_pg_ctl_wait_option(self, os_ops: OsOperations): logging.info("OK. Node is stopped.") node.cleanup() - def test_replicate(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) - with __class__.helper__get_node(os_ops) as node: + def test_replicate(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc) as node: node.init(allow_streaming=True).start() with node.replicate().start() as replica: @@ -851,14 +860,14 @@ def test_replicate(self, os_ops: OsOperations): res = node.execute('select * from test') assert (res == []) - def test_synchronous_replication(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) + def test_synchronous_replication(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) - current_version = get_pg_version2(os_ops) + current_version = get_pg_version2(node_svc.os_ops) __class__.helper__skip_test_if_pg_version_is_not_ge(current_version, "9.6") - with __class__.helper__get_node(os_ops) as master: + with __class__.helper__get_node(node_svc) as master: old_version = not __class__.helper__pg_version_ge(current_version, '9.6') master.init(allow_streaming=True).start() @@ -897,14 +906,14 @@ def test_synchronous_replication(self, os_ops: OsOperations): res = standby1.safe_psql('select count(*) from abc') assert (__class__.helper__rm_carriage_returns(res) == b'1000000\n') - def test_logical_replication(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) + def test_logical_replication(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) - current_version = get_pg_version2(os_ops) + current_version = get_pg_version2(node_svc.os_ops) __class__.helper__skip_test_if_pg_version_is_not_ge(current_version, "10") - with __class__.helper__get_node(os_ops) as node1, __class__.helper__get_node(os_ops) as node2: + with __class__.helper__get_node(node_svc) as node1, __class__.helper__get_node(node_svc) as node2: node1.init(allow_logical=True) node1.start() node2.init().start() @@ -971,15 +980,15 @@ def test_logical_replication(self, os_ops: OsOperations): res = node2.execute('select * from test2') assert (res == [('a', ), ('b', )]) - def test_logical_catchup(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) + def test_logical_catchup(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) """ Runs catchup for 100 times to be sure that it is consistent """ - current_version = get_pg_version2(os_ops) + current_version = get_pg_version2(node_svc.os_ops) __class__.helper__skip_test_if_pg_version_is_not_ge(current_version, "10") - with __class__.helper__get_node(os_ops) as node1, __class__.helper__get_node(os_ops) as node2: + with __class__.helper__get_node(node_svc) as node1, __class__.helper__get_node(node_svc) as node2: node1.init(allow_logical=True) node1.start() node2.init().start() @@ -999,20 +1008,20 @@ def test_logical_catchup(self, os_ops: OsOperations): assert (res == [(i, i, )]) node1.execute('delete from test') - def test_logical_replication_fail(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) + def test_logical_replication_fail(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) - current_version = get_pg_version2(os_ops) + current_version = get_pg_version2(node_svc.os_ops) __class__.helper__skip_test_if_pg_version_is_ge(current_version, "10") - with __class__.helper__get_node(os_ops) as node: + with __class__.helper__get_node(node_svc) as node: with pytest.raises(expected_exception=InitNodeException): node.init(allow_logical=True) - def test_replication_slots(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) - with __class__.helper__get_node(os_ops) as node: + def test_replication_slots(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc) as node: node.init(allow_streaming=True).start() with node.replicate(slot='slot1').start() as replica: @@ -1022,18 +1031,18 @@ def test_replication_slots(self, os_ops: OsOperations): with pytest.raises(expected_exception=testgres_TestgresException): node.replicate(slot='slot1') - def test_incorrect_catchup(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) - with __class__.helper__get_node(os_ops) as node: + def test_incorrect_catchup(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc) as node: node.init(allow_streaming=True).start() # node has no master, can't catch up with pytest.raises(expected_exception=testgres_TestgresException): node.catchup() - def test_promotion(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) - with __class__.helper__get_node(os_ops) as master: + def test_promotion(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc) as master: master.init().start() master.safe_psql('create table abc(id serial)') @@ -1046,17 +1055,17 @@ def test_promotion(self, os_ops: OsOperations): res = replica.safe_psql('select * from abc') assert (__class__.helper__rm_carriage_returns(res) == b'1\n') - def test_dump(self, os_ops: OsOperations): - assert isinstance(os_ops, OsOperations) + def test_dump(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) query_create = 'create table test as select generate_series(1, 2) as val' query_select = 'select * from test order by val asc' - with __class__.helper__get_node(os_ops).init().start() as node1: + with __class__.helper__get_node(node_svc).init().start() as node1: node1.execute(query_create) for format in ['plain', 'custom', 'directory', 'tar']: - with removing(os_ops, node1.dump(format=format)) as dump: - with __class__.helper__get_node(os_ops).init().start() as node3: + with removing(node_svc.os_ops, node1.dump(format=format)) as dump: + with __class__.helper__get_node(node_svc).init().start() as node3: if format == 'directory': assert (os.path.isdir(dump)) else: @@ -1066,14 +1075,16 @@ def test_dump(self, os_ops: OsOperations): res = node3.execute(query_select) assert (res == [(1, ), (2, )]) - def test_get_pg_config2(self, os_ops: OsOperations): + def test_get_pg_config2(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + # check same instances - a = get_pg_config2(os_ops, None) - b = get_pg_config2(os_ops, None) + a = get_pg_config2(node_svc.os_ops, None) + b = get_pg_config2(node_svc.os_ops, None) assert (id(a) == id(b)) # save right before config change - c1 = get_pg_config2(os_ops, None) + c1 = get_pg_config2(node_svc.os_ops, None) # modify setting for this scope with scoped_config(cache_pg_config=False) as config: @@ -1081,20 +1092,315 @@ def test_get_pg_config2(self, os_ops: OsOperations): assert not (config.cache_pg_config) # save right after config change - c2 = get_pg_config2(os_ops, None) + c2 = get_pg_config2(node_svc.os_ops, None) # check different instances after config change assert (id(c1) != id(c2)) # check different instances - a = get_pg_config2(os_ops, None) - b = get_pg_config2(os_ops, None) + a = get_pg_config2(node_svc.os_ops, None) + b = get_pg_config2(node_svc.os_ops, None) assert (id(a) != id(b)) + def test_pgbench(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + + __class__.helper__skip_test_if_util_not_exist(node_svc.os_ops, "pgbench") + + with __class__.helper__get_node(node_svc).init().start() as node: + # initialize pgbench DB and run benchmarks + node.pgbench_init( + scale=2, + foreign_keys=True, + options=['-q'] + ).pgbench_run(time=2) + + # run TPC-B benchmark + proc = node.pgbench(stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=['-T3']) + out = proc.communicate()[0] + assert (b'tps = ' in out) + + def test_unix_sockets(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + + with __class__.helper__get_node(node_svc) as node: + node.init(unix_sockets=False, allow_streaming=True) + node.start() + + res_exec = node.execute('select 1') + assert (res_exec == [(1,)]) + res_psql = node.safe_psql('select 1') + assert (res_psql == b'1\n') + + with node.replicate() as r: + assert type(r) == PostgresNode # noqa: E721 + r.start() + res_exec = r.execute('select 1') + assert (res_exec == [(1,)]) + res_psql = r.safe_psql('select 1') + assert (res_psql == b'1\n') + + def test_the_same_port(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + + with __class__.helper__get_node(node_svc) as node: + node.init().start() + assert (node._should_free_port) + assert (type(node.port) == int) # noqa: E721 + node_port_copy = node.port + r = node.safe_psql("SELECT 1;") + assert (__class__.helper__rm_carriage_returns(r) == b'1\n') + + with __class__.helper__get_node(node_svc, port=node.port) as node2: + assert (type(node2.port) == int) # noqa: E721 + assert (node2.port == node.port) + assert not (node2._should_free_port) + + with pytest.raises( + expected_exception=StartNodeException, + match=re.escape("Cannot start node") + ): + node2.init().start() + + # node is still working + assert (node.port == node_port_copy) + assert (node._should_free_port) + r = node.safe_psql("SELECT 3;") + assert (__class__.helper__rm_carriage_returns(r) == b'3\n') + + class tagPortManagerProxy(PortManager): + m_PrevPortManager: PortManager + + m_DummyPortNumber: int + m_DummyPortMaxUsage: int + + m_DummyPortCurrentUsage: int + m_DummyPortTotalUsage: int + + def __init__(self, prevPortManager: PortManager, dummyPortNumber: int, dummyPortMaxUsage: int): + assert isinstance(prevPortManager, PortManager) + assert type(dummyPortNumber) == int # noqa: E721 + assert type(dummyPortMaxUsage) == int # noqa: E721 + assert dummyPortNumber >= 0 + assert dummyPortMaxUsage >= 0 + + super().__init__() + + self.m_PrevPortManager = prevPortManager + + self.m_DummyPortNumber = dummyPortNumber + self.m_DummyPortMaxUsage = dummyPortMaxUsage + + self.m_DummyPortCurrentUsage = 0 + self.m_DummyPortTotalUsage = 0 + + def __enter__(self): + return self + + def __exit__(self, type, value, traceback): + assert self.m_DummyPortCurrentUsage == 0 + + assert self.m_PrevPortManager is not None + + def reserve_port(self) -> int: + assert type(self.m_DummyPortMaxUsage) == int # noqa: E721 + assert type(self.m_DummyPortTotalUsage) == int # noqa: E721 + assert type(self.m_DummyPortCurrentUsage) == int # noqa: E721 + assert self.m_DummyPortTotalUsage >= 0 + assert self.m_DummyPortCurrentUsage >= 0 + + assert self.m_DummyPortTotalUsage <= self.m_DummyPortMaxUsage + assert self.m_DummyPortCurrentUsage <= self.m_DummyPortTotalUsage + + assert self.m_PrevPortManager is not None + assert isinstance(self.m_PrevPortManager, PortManager) + + if self.m_DummyPortTotalUsage == self.m_DummyPortMaxUsage: + return self.m_PrevPortManager.reserve_port() + + self.m_DummyPortTotalUsage += 1 + self.m_DummyPortCurrentUsage += 1 + return self.m_DummyPortNumber + + def release_port(self, dummyPortNumber: int): + assert type(dummyPortNumber) == int # noqa: E721 + + assert type(self.m_DummyPortMaxUsage) == int # noqa: E721 + assert type(self.m_DummyPortTotalUsage) == int # noqa: E721 + assert type(self.m_DummyPortCurrentUsage) == int # noqa: E721 + assert self.m_DummyPortTotalUsage >= 0 + assert self.m_DummyPortCurrentUsage >= 0 + + assert self.m_DummyPortTotalUsage <= self.m_DummyPortMaxUsage + assert self.m_DummyPortCurrentUsage <= self.m_DummyPortTotalUsage + + assert self.m_PrevPortManager is not None + assert isinstance(self.m_PrevPortManager, PortManager) + + if self.m_DummyPortCurrentUsage > 0 and dummyPortNumber == self.m_DummyPortNumber: + assert self.m_DummyPortTotalUsage > 0 + self.m_DummyPortCurrentUsage -= 1 + return + + return self.m_PrevPortManager.release_port(dummyPortNumber) + + def test_port_rereserve_during_node_start(self, node_svc: PostgresNodeService): + assert type(node_svc) == PostgresNodeService # noqa: E721 + assert PostgresNode._C_MAX_START_ATEMPTS == 5 + + C_COUNT_OF_BAD_PORT_USAGE = 3 + + with __class__.helper__get_node(node_svc) as node1: + node1.init().start() + assert node1._should_free_port + assert type(node1.port) == int # noqa: E721 + node1_port_copy = node1.port + assert __class__.helper__rm_carriage_returns(node1.safe_psql("SELECT 1;")) == b'1\n' + + with __class__.tagPortManagerProxy(node_svc.port_manager, node1.port, C_COUNT_OF_BAD_PORT_USAGE) as proxy: + assert proxy.m_DummyPortNumber == node1.port + with __class__.helper__get_node(node_svc, port_manager=proxy) as node2: + assert node2._should_free_port + assert node2.port == node1.port + + node2.init().start() + + assert node2.port != node1.port + assert node2._should_free_port + assert proxy.m_DummyPortCurrentUsage == 0 + assert proxy.m_DummyPortTotalUsage == C_COUNT_OF_BAD_PORT_USAGE + assert node2.is_started + r = node2.safe_psql("SELECT 2;") + assert __class__.helper__rm_carriage_returns(r) == b'2\n' + + # node1 is still working + assert node1.port == node1_port_copy + assert node1._should_free_port + r = node1.safe_psql("SELECT 3;") + assert __class__.helper__rm_carriage_returns(r) == b'3\n' + + def test_port_conflict(self, node_svc: PostgresNodeService): + assert type(node_svc) == PostgresNodeService # noqa: E721 + assert PostgresNode._C_MAX_START_ATEMPTS > 1 + + C_COUNT_OF_BAD_PORT_USAGE = PostgresNode._C_MAX_START_ATEMPTS + + with __class__.helper__get_node(node_svc) as node1: + node1.init().start() + assert node1._should_free_port + assert type(node1.port) == int # noqa: E721 + node1_port_copy = node1.port + assert __class__.helper__rm_carriage_returns(node1.safe_psql("SELECT 1;")) == b'1\n' + + with __class__.tagPortManagerProxy(node_svc.port_manager, node1.port, C_COUNT_OF_BAD_PORT_USAGE) as proxy: + assert proxy.m_DummyPortNumber == node1.port + with __class__.helper__get_node(node_svc, port_manager=proxy) as node2: + assert node2._should_free_port + assert node2.port == node1.port + + with pytest.raises( + expected_exception=StartNodeException, + match=re.escape("Cannot start node after multiple attempts.") + ): + node2.init().start() + + assert node2.port == node1.port + assert node2._should_free_port + assert proxy.m_DummyPortCurrentUsage == 1 + assert proxy.m_DummyPortTotalUsage == C_COUNT_OF_BAD_PORT_USAGE + assert not node2.is_started + + # node2 must release our dummyPort (node1.port) + assert (proxy.m_DummyPortCurrentUsage == 0) + + # node1 is still working + assert node1.port == node1_port_copy + assert node1._should_free_port + r = node1.safe_psql("SELECT 3;") + assert __class__.helper__rm_carriage_returns(r) == b'3\n' + + def test_try_to_get_port_after_free_manual_port(self, node_svc: PostgresNodeService): + assert type(node_svc) == PostgresNodeService # noqa: E721 + + assert node_svc.port_manager is not None + assert isinstance(node_svc.port_manager, PortManager) + + with __class__.helper__get_node(node_svc) as node1: + assert node1 is not None + assert type(node1) == PostgresNode # noqa: E721 + assert node1.port is not None + assert type(node1.port) == int # noqa: E721 + with __class__.helper__get_node(node_svc, port=node1.port, port_manager=None) as node2: + assert node2 is not None + assert type(node1) == PostgresNode # noqa: E721 + assert node2 is not node1 + assert node2.port is not None + assert type(node2.port) == int # noqa: E721 + assert node2.port == node1.port + + logging.info("Release node2 port") + node2.free_port() + + logging.info("try to get node2.port...") + with pytest.raises( + InvalidOperationException, + match="^" + re.escape("PostgresNode port is not defined.") + "$" + ): + p = node2.port + assert p is None + + def test_try_to_start_node_after_free_manual_port(self, node_svc: PostgresNodeService): + assert type(node_svc) == PostgresNodeService # noqa: E721 + + assert node_svc.port_manager is not None + assert isinstance(node_svc.port_manager, PortManager) + + with __class__.helper__get_node(node_svc) as node1: + assert node1 is not None + assert type(node1) == PostgresNode # noqa: E721 + assert node1.port is not None + assert type(node1.port) == int # noqa: E721 + with __class__.helper__get_node(node_svc, port=node1.port, port_manager=None) as node2: + assert node2 is not None + assert type(node1) == PostgresNode # noqa: E721 + assert node2 is not node1 + assert node2.port is not None + assert type(node2.port) == int # noqa: E721 + assert node2.port == node1.port + + logging.info("Release node2 port") + node2.free_port() + + logging.info("node2 is trying to start...") + with pytest.raises( + InvalidOperationException, + match="^" + re.escape("Can't start PostgresNode. Port is not defined.") + "$" + ): + node2.start() + @staticmethod - def helper__get_node(os_ops: OsOperations, name=None): - assert isinstance(os_ops, OsOperations) - return PostgresNode(name, conn_params=None, os_ops=os_ops) + def helper__get_node( + node_svc: PostgresNodeService, + name: typing.Optional[str] = None, + port: typing.Optional[int] = None, + port_manager: typing.Optional[PortManager] = None + ) -> PostgresNode: + assert isinstance(node_svc, PostgresNodeService) + assert isinstance(node_svc.os_ops, OsOperations) + assert isinstance(node_svc.port_manager, PortManager) + + if port_manager is None: + port_manager = node_svc.port_manager + + return PostgresNode( + name, + port=port, + conn_params=None, + os_ops=node_svc.os_ops, + port_manager=port_manager if port is None else None + ) @staticmethod def helper__skip_test_if_pg_version_is_not_ge(ver1: str, ver2: str): diff --git a/tests/test_testgres_local.py b/tests/test_testgres_local.py index 01f975a0..bef80d0f 100644 --- a/tests/test_testgres_local.py +++ b/tests/test_testgres_local.py @@ -100,27 +100,6 @@ def test_custom_init(self): # there should be no trust entries at all assert not (any('trust' in s for s in lines)) - def test_pgbench(self): - __class__.helper__skip_test_if_util_not_exist("pgbench") - - with get_new_node().init().start() as node: - - # initialize pgbench DB and run benchmarks - node.pgbench_init(scale=2, foreign_keys=True, - options=['-q']).pgbench_run(time=2) - - # run TPC-B benchmark - proc = node.pgbench(stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - options=['-T3']) - - out, _ = proc.communicate() - out = out.decode('utf-8') - - proc.stdout.close() - - assert ('tps' in out) - def test_pg_config(self): # check same instances a = get_pg_config() @@ -177,18 +156,6 @@ def test_config_stack(self): assert (TestgresConfig.cached_initdb_dir == d0) - def test_unix_sockets(self): - with get_new_node() as node: - node.init(unix_sockets=False, allow_streaming=True) - node.start() - - node.execute('select 1') - node.safe_psql('select 1') - - with node.replicate().start() as r: - r.execute('select 1') - r.safe_psql('select 1') - def test_ports_management(self): assert bound_ports is not None assert type(bound_ports) == set # noqa: E721 @@ -277,30 +244,6 @@ def test_parse_pg_version(self): # Macos assert parse_pg_version("postgres (PostgreSQL) 14.9 (Homebrew)") == "14.9" - def test_the_same_port(self): - with get_new_node() as node: - node.init().start() - assert (node._should_free_port) - assert (type(node.port) == int) # noqa: E721 - node_port_copy = node.port - assert (rm_carriage_returns(node.safe_psql("SELECT 1;")) == b'1\n') - - with get_new_node(port=node.port) as node2: - assert (type(node2.port) == int) # noqa: E721 - assert (node2.port == node.port) - assert not (node2._should_free_port) - - with pytest.raises( - expected_exception=StartNodeException, - match=re.escape("Cannot start node") - ): - node2.init().start() - - # node is still working - assert (node.port == node_port_copy) - assert (node._should_free_port) - assert (rm_carriage_returns(node.safe_psql("SELECT 3;")) == b'3\n') - class tagPortManagerProxy: sm_prev_testgres_reserve_port = None sm_prev_testgres_release_port = None diff --git a/tests/test_testgres_remote.py b/tests/test_testgres_remote.py index 2142e5ba..ef4bd0c8 100755 --- a/tests/test_testgres_remote.py +++ b/tests/test_testgres_remote.py @@ -1,14 +1,12 @@ # coding: utf-8 import os import re -import subprocess import pytest -import psutil import logging -from .helpers.os_ops_descrs import OsOpsDescrs -from .helpers.os_ops_descrs import OsOperations +from .helpers.global_data import PostgresNodeService +from .helpers.global_data import PostgresNodeServices from .. import testgres @@ -27,8 +25,6 @@ get_pg_config # NOTE: those are ugly imports -from ..testgres import bound_ports -from ..testgres.node import ProcessProxy def util_exists(util): @@ -48,17 +44,17 @@ def good_properties(f): class TestTestgresRemote: - sm_os_ops = OsOpsDescrs.sm_remote_os_ops - @pytest.fixture(autouse=True, scope="class") def implicit_fixture(self): + cur_os_ops = PostgresNodeServices.sm_remote.os_ops + assert cur_os_ops is not None + prev_ops = testgres_config.os_ops assert prev_ops is not None - assert __class__.sm_os_ops is not None - testgres_config.set_os_ops(os_ops=__class__.sm_os_ops) - assert testgres_config.os_ops is __class__.sm_os_ops + testgres_config.set_os_ops(os_ops=cur_os_ops) + assert testgres_config.os_ops is cur_os_ops yield - assert testgres_config.os_ops is __class__.sm_os_ops + assert testgres_config.os_ops is cur_os_ops testgres_config.set_os_ops(os_ops=prev_ops) assert testgres_config.os_ops is prev_ops @@ -172,21 +168,6 @@ def test_init__unk_LANG_and_LC_CTYPE(self): __class__.helper__restore_envvar("LC_CTYPE", prev_LC_CTYPE) __class__.helper__restore_envvar("LC_COLLATE", prev_LC_COLLATE) - def test_pgbench(self): - __class__.helper__skip_test_if_util_not_exist("pgbench") - - with __class__.helper__get_node().init().start() as node: - # initialize pgbench DB and run benchmarks - node.pgbench_init(scale=2, foreign_keys=True, - options=['-q']).pgbench_run(time=2) - - # run TPC-B benchmark - proc = node.pgbench(stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - options=['-T3']) - out = proc.communicate()[0] - assert (b'tps = ' in out) - def test_pg_config(self): # check same instances a = get_pg_config() @@ -243,90 +224,19 @@ def test_config_stack(self): assert (TestgresConfig.cached_initdb_dir == d0) - def test_unix_sockets(self): - with __class__.helper__get_node() as node: - node.init(unix_sockets=False, allow_streaming=True) - node.start() - - res_exec = node.execute('select 1') - res_psql = node.safe_psql('select 1') - assert (res_exec == [(1,)]) - assert (res_psql == b'1\n') - - with node.replicate().start() as r: - res_exec = r.execute('select 1') - res_psql = r.safe_psql('select 1') - assert (res_exec == [(1,)]) - assert (res_psql == b'1\n') - - def test_ports_management(self): - assert bound_ports is not None - assert type(bound_ports) == set # noqa: E721 - - if len(bound_ports) != 0: - logging.warning("bound_ports is not empty: {0}".format(bound_ports)) - - stage0__bound_ports = bound_ports.copy() - - with __class__.helper__get_node() as node: - assert bound_ports is not None - assert type(bound_ports) == set # noqa: E721 - - assert node.port is not None - assert type(node.port) == int # noqa: E721 - - logging.info("node port is {0}".format(node.port)) - - assert node.port in bound_ports - assert node.port not in stage0__bound_ports - - assert stage0__bound_ports <= bound_ports - assert len(stage0__bound_ports) + 1 == len(bound_ports) - - stage1__bound_ports = stage0__bound_ports.copy() - stage1__bound_ports.add(node.port) - - assert stage1__bound_ports == bound_ports - - # check that port has been freed successfully - assert bound_ports is not None - assert type(bound_ports) == set # noqa: E721 - assert bound_ports == stage0__bound_ports - - # TODO: Why does not this test work with remote host? - def test_child_process_dies(self): - nAttempt = 0 - - while True: - if nAttempt == 5: - raise Exception("Max attempt number is exceed.") - - nAttempt += 1 - - logging.info("Attempt #{0}".format(nAttempt)) - - # test for FileNotFound exception during child_processes() function - with subprocess.Popen(["sleep", "60"]) as process: - r = process.poll() - - if r is not None: - logging.warning("process.pool() returns an unexpected result: {0}.".format(r)) - continue - - assert r is None - # collect list of processes currently running - children = psutil.Process(os.getpid()).children() - # kill a process, so received children dictionary becomes invalid - process.kill() - process.wait() - # try to handle children list -- missing processes will have ptype "ProcessType.Unknown" - [ProcessProxy(p) for p in children] - break - @staticmethod def helper__get_node(name=None): - assert isinstance(__class__.sm_os_ops, OsOperations) - return testgres.PostgresNode(name, conn_params=None, os_ops=__class__.sm_os_ops) + svc = PostgresNodeServices.sm_remote + + assert isinstance(svc, PostgresNodeService) + assert isinstance(svc.os_ops, testgres.OsOperations) + assert isinstance(svc.port_manager, testgres.PortManager) + + return testgres.PostgresNode( + name, + conn_params=None, + os_ops=svc.os_ops, + port_manager=svc.port_manager) @staticmethod def helper__restore_envvar(name, prev_value): From 307ef5fc523d2156d711b1493ee8d7d98f24ba82 Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Mon, 7 Apr 2025 07:31:53 +0300 Subject: [PATCH 194/216] Test refactoring (#236) * TestUtils is added * TestTestgresCommon::test_node_repr is added * test_get_pg_config2 moved to TestUtils * TestTestgresCommon.test_custom_init is added * TestConfig is added --- tests/test_config.py | 41 +++++++++++++++ tests/test_testgres_common.py | 54 ++++++++++---------- tests/test_testgres_local.py | 96 ++++------------------------------- tests/test_testgres_remote.py | 73 +++----------------------- tests/test_utils.py | 62 ++++++++++++++++++++++ 5 files changed, 145 insertions(+), 181 deletions(-) create mode 100644 tests/test_config.py create mode 100644 tests/test_utils.py diff --git a/tests/test_config.py b/tests/test_config.py new file mode 100644 index 00000000..05702e9a --- /dev/null +++ b/tests/test_config.py @@ -0,0 +1,41 @@ +from ..testgres import TestgresConfig +from ..testgres import configure_testgres +from ..testgres import scoped_config +from ..testgres import pop_config + +from .. import testgres + +import pytest + + +class TestConfig: + def test_config_stack(self): + # no such option + with pytest.raises(expected_exception=TypeError): + configure_testgres(dummy=True) + + # we have only 1 config in stack + with pytest.raises(expected_exception=IndexError): + pop_config() + + d0 = TestgresConfig.cached_initdb_dir + d1 = 'dummy_abc' + d2 = 'dummy_def' + + with scoped_config(cached_initdb_dir=d1) as c1: + assert (c1.cached_initdb_dir == d1) + + with scoped_config(cached_initdb_dir=d2) as c2: + stack_size = len(testgres.config.config_stack) + + # try to break a stack + with pytest.raises(expected_exception=TypeError): + with scoped_config(dummy=True): + pass + + assert (c2.cached_initdb_dir == d2) + assert (len(testgres.config.config_stack) == stack_size) + + assert (c1.cached_initdb_dir == d1) + + assert (TestgresConfig.cached_initdb_dir == d0) diff --git a/tests/test_testgres_common.py b/tests/test_testgres_common.py index b286a1c6..c384dfb2 100644 --- a/tests/test_testgres_common.py +++ b/tests/test_testgres_common.py @@ -6,7 +6,6 @@ from ..testgres.node import PgVer from ..testgres.node import PostgresNode from ..testgres.utils import get_pg_version2 -from ..testgres.utils import get_pg_config2 from ..testgres.utils import file_tail from ..testgres.utils import get_bin_path2 from ..testgres import ProcessType @@ -106,6 +105,32 @@ def test_version_management(self, node_svc: PostgresNodeService): assert (isinstance(node.version, PgVer)) assert (node.version == PgVer(version)) + def test_node_repr(self, node_svc: PostgresNodeService): + with __class__.helper__get_node(node_svc).init() as node: + pattern = r"PostgresNode\(name='.+', port=.+, base_dir='.+'\)" + assert re.match(pattern, str(node)) is not None + + def test_custom_init(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + + with __class__.helper__get_node(node_svc) as node: + # enable page checksums + node.init(initdb_params=['-k']).start() + + with __class__.helper__get_node(node_svc) as node: + node.init( + allow_streaming=True, + initdb_params=['--auth-local=reject', '--auth-host=reject']) + + hba_file = os.path.join(node.data_dir, 'pg_hba.conf') + lines = node.os_ops.readlines(hba_file) + + # check number of lines + assert (len(lines) >= 6) + + # there should be no trust entries at all + assert not (any('trust' in s for s in lines)) + def test_double_init(self, node_svc: PostgresNodeService): assert isinstance(node_svc, PostgresNodeService) @@ -1075,33 +1100,6 @@ def test_dump(self, node_svc: PostgresNodeService): res = node3.execute(query_select) assert (res == [(1, ), (2, )]) - def test_get_pg_config2(self, node_svc: PostgresNodeService): - assert isinstance(node_svc, PostgresNodeService) - - # check same instances - a = get_pg_config2(node_svc.os_ops, None) - b = get_pg_config2(node_svc.os_ops, None) - assert (id(a) == id(b)) - - # save right before config change - c1 = get_pg_config2(node_svc.os_ops, None) - - # modify setting for this scope - with scoped_config(cache_pg_config=False) as config: - # sanity check for value - assert not (config.cache_pg_config) - - # save right after config change - c2 = get_pg_config2(node_svc.os_ops, None) - - # check different instances after config change - assert (id(c1) != id(c2)) - - # check different instances - a = get_pg_config2(node_svc.os_ops, None) - b = get_pg_config2(node_svc.os_ops, None) - assert (id(a) != id(b)) - def test_pgbench(self, node_svc: PostgresNodeService): assert isinstance(node_svc, PostgresNodeService) diff --git a/tests/test_testgres_local.py b/tests/test_testgres_local.py index bef80d0f..9dbd455b 100644 --- a/tests/test_testgres_local.py +++ b/tests/test_testgres_local.py @@ -9,28 +9,18 @@ from .. import testgres -from ..testgres import \ - StartNodeException, \ - ExecUtilException, \ - NodeApp - -from ..testgres import \ - TestgresConfig, \ - configure_testgres, \ - scoped_config, \ - pop_config - -from ..testgres import \ - get_new_node - -from ..testgres import \ - get_bin_path, \ - get_pg_config, \ - get_pg_version +from ..testgres import StartNodeException +from ..testgres import ExecUtilException +from ..testgres import NodeApp +from ..testgres import scoped_config +from ..testgres import get_new_node +from ..testgres import get_bin_path +from ..testgres import get_pg_config +from ..testgres import get_pg_version # NOTE: those are ugly imports -from ..testgres import bound_ports -from ..testgres.utils import PgVer, parse_pg_version +from ..testgres.utils import bound_ports +from ..testgres.utils import PgVer from ..testgres.node import ProcessProxy @@ -75,31 +65,6 @@ def rm_carriage_returns(out): class TestTestgresLocal: - def test_node_repr(self): - with get_new_node() as node: - pattern = r"PostgresNode\(name='.+', port=.+, base_dir='.+'\)" - assert re.match(pattern, str(node)) is not None - - def test_custom_init(self): - with get_new_node() as node: - # enable page checksums - node.init(initdb_params=['-k']).start() - - with get_new_node() as node: - node.init( - allow_streaming=True, - initdb_params=['--auth-local=reject', '--auth-host=reject']) - - hba_file = os.path.join(node.data_dir, 'pg_hba.conf') - with open(hba_file, 'r') as conf: - lines = conf.readlines() - - # check number of lines - assert (len(lines) >= 6) - - # there should be no trust entries at all - assert not (any('trust' in s for s in lines)) - def test_pg_config(self): # check same instances a = get_pg_config() @@ -125,37 +90,6 @@ def test_pg_config(self): b = get_pg_config() assert (id(a) != id(b)) - def test_config_stack(self): - # no such option - with pytest.raises(expected_exception=TypeError): - configure_testgres(dummy=True) - - # we have only 1 config in stack - with pytest.raises(expected_exception=IndexError): - pop_config() - - d0 = TestgresConfig.cached_initdb_dir - d1 = 'dummy_abc' - d2 = 'dummy_def' - - with scoped_config(cached_initdb_dir=d1) as c1: - assert (c1.cached_initdb_dir == d1) - - with scoped_config(cached_initdb_dir=d2) as c2: - stack_size = len(testgres.config.config_stack) - - # try to break a stack - with pytest.raises(expected_exception=TypeError): - with scoped_config(dummy=True): - pass - - assert (c2.cached_initdb_dir == d2) - assert (len(testgres.config.config_stack) == stack_size) - - assert (c1.cached_initdb_dir == d1) - - assert (TestgresConfig.cached_initdb_dir == d0) - def test_ports_management(self): assert bound_ports is not None assert type(bound_ports) == set # noqa: E721 @@ -234,16 +168,6 @@ def test_upgrade_node(self): node_new.start() assert (b'Upgrade Complete' in res) - def test_parse_pg_version(self): - # Linux Mint - assert parse_pg_version("postgres (PostgreSQL) 15.5 (Ubuntu 15.5-1.pgdg22.04+1)") == "15.5" - # Linux Ubuntu - assert parse_pg_version("postgres (PostgreSQL) 12.17") == "12.17" - # Windows - assert parse_pg_version("postgres (PostgreSQL) 11.4") == "11.4" - # Macos - assert parse_pg_version("postgres (PostgreSQL) 14.9 (Homebrew)") == "14.9" - class tagPortManagerProxy: sm_prev_testgres_reserve_port = None sm_prev_testgres_release_port = None diff --git a/tests/test_testgres_remote.py b/tests/test_testgres_remote.py index ef4bd0c8..e38099b7 100755 --- a/tests/test_testgres_remote.py +++ b/tests/test_testgres_remote.py @@ -1,6 +1,5 @@ # coding: utf-8 import os -import re import pytest import logging @@ -10,19 +9,14 @@ from .. import testgres -from ..testgres.exceptions import \ - InitNodeException, \ - ExecUtilException +from ..testgres.exceptions import InitNodeException +from ..testgres.exceptions import ExecUtilException -from ..testgres.config import \ - TestgresConfig, \ - configure_testgres, \ - scoped_config, \ - pop_config, testgres_config +from ..testgres.config import scoped_config +from ..testgres.config import testgres_config -from ..testgres import \ - get_bin_path, \ - get_pg_config +from ..testgres import get_bin_path +from ..testgres import get_pg_config # NOTE: those are ugly imports @@ -58,30 +52,6 @@ def implicit_fixture(self): testgres_config.set_os_ops(os_ops=prev_ops) assert testgres_config.os_ops is prev_ops - def test_node_repr(self): - with __class__.helper__get_node() as node: - pattern = r"PostgresNode\(name='.+', port=.+, base_dir='.+'\)" - assert re.match(pattern, str(node)) is not None - - def test_custom_init(self): - with __class__.helper__get_node() as node: - # enable page checksums - node.init(initdb_params=['-k']).start() - - with __class__.helper__get_node() as node: - node.init( - allow_streaming=True, - initdb_params=['--auth-local=reject', '--auth-host=reject']) - - hba_file = os.path.join(node.data_dir, 'pg_hba.conf') - lines = node.os_ops.readlines(hba_file) - - # check number of lines - assert (len(lines) >= 6) - - # there should be no trust entries at all - assert not (any('trust' in s for s in lines)) - def test_init__LANG_С(self): # PBCKP-1744 prev_LANG = os.environ.get("LANG") @@ -193,37 +163,6 @@ def test_pg_config(self): b = get_pg_config() assert (id(a) != id(b)) - def test_config_stack(self): - # no such option - with pytest.raises(expected_exception=TypeError): - configure_testgres(dummy=True) - - # we have only 1 config in stack - with pytest.raises(expected_exception=IndexError): - pop_config() - - d0 = TestgresConfig.cached_initdb_dir - d1 = 'dummy_abc' - d2 = 'dummy_def' - - with scoped_config(cached_initdb_dir=d1) as c1: - assert (c1.cached_initdb_dir == d1) - - with scoped_config(cached_initdb_dir=d2) as c2: - stack_size = len(testgres.config.config_stack) - - # try to break a stack - with pytest.raises(expected_exception=TypeError): - with scoped_config(dummy=True): - pass - - assert (c2.cached_initdb_dir == d2) - assert (len(testgres.config.config_stack) == stack_size) - - assert (c1.cached_initdb_dir == d1) - - assert (TestgresConfig.cached_initdb_dir == d0) - @staticmethod def helper__get_node(name=None): svc = PostgresNodeServices.sm_remote diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 00000000..d4a4c9ad --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,62 @@ +from .helpers.global_data import OsOpsDescr +from .helpers.global_data import OsOpsDescrs +from .helpers.global_data import OsOperations + +from ..testgres.utils import parse_pg_version +from ..testgres.utils import get_pg_config2 +from ..testgres import scoped_config + +import pytest + + +class TestUtils: + sm_os_ops_descrs: list[OsOpsDescr] = [ + OsOpsDescrs.sm_local_os_ops_descr, + OsOpsDescrs.sm_remote_os_ops_descr + ] + + @pytest.fixture( + params=[descr.os_ops for descr in sm_os_ops_descrs], + ids=[descr.sign for descr in sm_os_ops_descrs] + ) + def os_ops(self, request: pytest.FixtureRequest) -> OsOperations: + assert isinstance(request, pytest.FixtureRequest) + assert isinstance(request.param, OsOperations) + return request.param + + def test_parse_pg_version(self): + # Linux Mint + assert parse_pg_version("postgres (PostgreSQL) 15.5 (Ubuntu 15.5-1.pgdg22.04+1)") == "15.5" + # Linux Ubuntu + assert parse_pg_version("postgres (PostgreSQL) 12.17") == "12.17" + # Windows + assert parse_pg_version("postgres (PostgreSQL) 11.4") == "11.4" + # Macos + assert parse_pg_version("postgres (PostgreSQL) 14.9 (Homebrew)") == "14.9" + + def test_get_pg_config2(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + + # check same instances + a = get_pg_config2(os_ops, None) + b = get_pg_config2(os_ops, None) + assert (id(a) == id(b)) + + # save right before config change + c1 = get_pg_config2(os_ops, None) + + # modify setting for this scope + with scoped_config(cache_pg_config=False) as config: + # sanity check for value + assert not (config.cache_pg_config) + + # save right after config change + c2 = get_pg_config2(os_ops, None) + + # check different instances after config change + assert (id(c1) != id(c2)) + + # check different instances + a = get_pg_config2(os_ops, None) + b = get_pg_config2(os_ops, None) + assert (id(a) != id(b)) From da2c493473fb124612a8e2f1baa74ac6b6ff980e Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Mon, 7 Apr 2025 12:38:03 +0300 Subject: [PATCH 195/216] OsOperation::db_connect is removed (#237) * OsOperation::db_connect is removed OsOperation does not work with databases. It provides an only OS functional. TODO: CI must explicitly test pg8000 and psycopg2. --- testgres/connection.py | 12 +++++++----- testgres/operations/local_ops.py | 13 +------------ testgres/operations/os_ops.py | 12 ------------ testgres/operations/remote_ops.py | 20 -------------------- 4 files changed, 8 insertions(+), 49 deletions(-) diff --git a/testgres/connection.py b/testgres/connection.py index ccedd135..b8dc49a9 100644 --- a/testgres/connection.py +++ b/testgres/connection.py @@ -42,11 +42,13 @@ def __init__(self, self._node = node - self._connection = node.os_ops.db_connect(dbname=dbname, - user=username, - password=password, - host=node.host, - port=node.port) + self._connection = pglib.connect( + database=dbname, + user=username, + password=password, + host=node.host, + port=node.port + ) self._connection.autocommit = autocommit self._cursor = self.connection.cursor() diff --git a/testgres/operations/local_ops.py b/testgres/operations/local_ops.py index 39c81405..9785d462 100644 --- a/testgres/operations/local_ops.py +++ b/testgres/operations/local_ops.py @@ -12,7 +12,7 @@ from ..exceptions import ExecUtilException from ..exceptions import InvalidOperationException -from .os_ops import ConnectionParams, OsOperations, pglib, get_default_encoding +from .os_ops import ConnectionParams, OsOperations, get_default_encoding from .raise_error import RaiseError from .helpers import Helpers @@ -446,14 +446,3 @@ def is_port_free(self, number: int) -> bool: return True except OSError: return False - - # Database control - def db_connect(self, dbname, user, password=None, host="localhost", port=5432): - conn = pglib.connect( - host=host, - port=port, - database=dbname, - user=user, - password=password, - ) - return conn diff --git a/testgres/operations/os_ops.py b/testgres/operations/os_ops.py index 489a7cb2..d25e76bc 100644 --- a/testgres/operations/os_ops.py +++ b/testgres/operations/os_ops.py @@ -1,14 +1,6 @@ import getpass import locale -try: - import psycopg2 as pglib # noqa: F401 -except ImportError: - try: - import pg8000 as pglib # noqa: F401 - except ImportError: - raise ImportError("You must have psycopg2 or pg8000 modules installed") - class ConnectionParams: def __init__(self, host='127.0.0.1', port=None, ssh_key=None, username=None): @@ -130,7 +122,3 @@ def get_process_children(self, pid): def is_port_free(self, number: int): assert type(number) == int # noqa: E721 raise NotImplementedError() - - # Database control - def db_connect(self, dbname, user, password=None, host="localhost", port=5432): - raise NotImplementedError() diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index ee747e52..25d02f38 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -6,15 +6,6 @@ import io import logging -# we support both pg8000 and psycopg2 -try: - import psycopg2 as pglib -except ImportError: - try: - import pg8000 as pglib - except ImportError: - raise ImportError("You must have psycopg2 or pg8000 modules installed") - from ..exceptions import ExecUtilException from ..exceptions import InvalidOperationException from .os_ops import OsOperations, ConnectionParams, get_default_encoding @@ -677,17 +668,6 @@ def _is_port_free__process_1(error: str) -> bool: # return True - # Database control - def db_connect(self, dbname, user, password=None, host="localhost", port=5432): - conn = pglib.connect( - host=host, - port=port, - database=dbname, - user=user, - password=password, - ) - return conn - @staticmethod def _make_exec_env_list() -> list[str]: result = list[str]() From 24014744bd031e34c2e2490127c40613edec01d5 Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Thu, 10 Apr 2025 09:07:05 +0300 Subject: [PATCH 196/216] A support of Python v3.8 (#238) * A support of Python 3.8 [typing] Python 3.8 does not support tuple[...], list[...], set[...] and so on. We will use the analogues from typing package: typing.Tuple[...], typing.List[...] and typing.Set[...]. * [CI] Jobs for testing with python 3.8.0, 3.8.latest, 3.9.latest, 3.10.latest and 3.11.latest [std2-all][alpine] are added --- .travis.yml | 5 ++ Dockerfile--std2-all.tmpl | 96 +++++++++++++++++++++++++++++++ run_tests2.sh | 68 ++++++++++++++++++++++ testgres/operations/remote_ops.py | 5 +- testgres/port_manager.py | 9 +-- tests/conftest.py | 22 ++++--- tests/test_os_ops_common.py | 3 +- tests/test_testgres_common.py | 10 ++-- tests/test_utils.py | 3 +- 9 files changed, 199 insertions(+), 22 deletions(-) create mode 100644 Dockerfile--std2-all.tmpl create mode 100755 run_tests2.sh diff --git a/.travis.yml b/.travis.yml index 7557a2ce..55b7afa9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,6 +19,11 @@ notifications: on_failure: always env: + - TEST_PLATFORM=std2-all PYTHON_VERSION=3.8.0 PG_VERSION=17 + - TEST_PLATFORM=std2-all PYTHON_VERSION=3.8 PG_VERSION=17 + - TEST_PLATFORM=std2-all PYTHON_VERSION=3.9 PG_VERSION=17 + - TEST_PLATFORM=std2-all PYTHON_VERSION=3.10 PG_VERSION=17 + - TEST_PLATFORM=std2-all PYTHON_VERSION=3.11 PG_VERSION=17 - TEST_PLATFORM=std PYTHON_VERSION=3 PG_VERSION=16 - TEST_PLATFORM=std PYTHON_VERSION=3 PG_VERSION=15 - TEST_PLATFORM=std PYTHON_VERSION=3 PG_VERSION=14 diff --git a/Dockerfile--std2-all.tmpl b/Dockerfile--std2-all.tmpl new file mode 100644 index 00000000..10d8280c --- /dev/null +++ b/Dockerfile--std2-all.tmpl @@ -0,0 +1,96 @@ +ARG PG_VERSION +ARG PYTHON_VERSION + +# --------------------------------------------- base1 +FROM postgres:${PG_VERSION}-alpine as base1 + +# --------------------------------------------- base2_with_python-3 +FROM base1 as base2_with_python-3 +RUN apk add --no-cache curl python3 python3-dev build-base musl-dev linux-headers + +# For pyenv +RUN apk add patch +RUN apk add git +RUN apk add xz-dev +RUN apk add zip +RUN apk add zlib-dev +RUN apk add libffi-dev +RUN apk add readline-dev +RUN apk add openssl openssl-dev +RUN apk add sqlite-dev +RUN apk add bzip2-dev + +# --------------------------------------------- base3_with_python-3.8.0 +FROM base2_with_python-3 as base3_with_python-3.8.0 +ENV PYTHON_VERSION=3.8.0 + +# --------------------------------------------- base3_with_python-3.8 +FROM base2_with_python-3 as base3_with_python-3.8 +ENV PYTHON_VERSION=3.8 + +# --------------------------------------------- base3_with_python-3.9 +FROM base2_with_python-3 as base3_with_python-3.9 +ENV PYTHON_VERSION=3.9 + +# --------------------------------------------- base3_with_python-3.10 +FROM base2_with_python-3 as base3_with_python-3.10 +ENV PYTHON_VERSION=3.10 + +# --------------------------------------------- base3_with_python-3.11 +FROM base2_with_python-3 as base3_with_python-3.11 +ENV PYTHON_VERSION=3.11 + +# --------------------------------------------- final +FROM base3_with_python-${PYTHON_VERSION} as final + +#RUN apk add --no-cache mc + +# Full version of "ps" command +RUN apk add --no-cache procps + +RUN apk add --no-cache openssh +RUN apk add --no-cache sudo + +ENV LANG=C.UTF-8 + +RUN addgroup -S sudo +RUN adduser postgres sudo + +EXPOSE 22 +RUN ssh-keygen -A + +ADD . /pg/testgres +WORKDIR /pg/testgres +RUN chown -R postgres:postgres /pg + +# It allows to use sudo without password +RUN sh -c "echo \"postgres ALL=(ALL:ALL) NOPASSWD:ALL\"">>/etc/sudoers + +# THIS CMD IS NEEDED TO CONNECT THROUGH SSH WITHOUT PASSWORD +RUN sh -c "echo "postgres:*" | chpasswd -e" + +USER postgres + +RUN curl https://raw.githubusercontent.com/pyenv/pyenv-installer/master/bin/pyenv-installer | bash + +RUN ~/.pyenv/bin/pyenv install ${PYTHON_VERSION} + +# THIS CMD IS NEEDED TO CONNECT THROUGH SSH WITHOUT PASSWORD +RUN chmod 700 ~/ + +RUN mkdir -p ~/.ssh +#RUN chmod 700 ~/.ssh + +ENTRYPOINT sh -c " \ +set -eux; \ +echo HELLO FROM ENTRYPOINT; \ +echo HOME DIR IS [`realpath ~/`]; \ +ssh-keygen -t rsa -f ~/.ssh/id_rsa -q -N ''; \ +cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys; \ +chmod 600 ~/.ssh/authorized_keys; \ +ls -la ~/.ssh/; \ +sudo /usr/sbin/sshd; \ +ssh-keyscan -H localhost >> ~/.ssh/known_hosts; \ +ssh-keyscan -H 127.0.0.1 >> ~/.ssh/known_hosts; \ +export PATH=\"~/.pyenv/bin:$PATH\"; \ +TEST_FILTER=\"\" bash run_tests2.sh;" diff --git a/run_tests2.sh b/run_tests2.sh new file mode 100755 index 00000000..173b19dc --- /dev/null +++ b/run_tests2.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash + +# Copyright (c) 2017-2025 Postgres Professional + +set -eux + +eval "$(pyenv init -)" +eval "$(pyenv virtualenv-init -)" + +pyenv virtualenv --force ${PYTHON_VERSION} cur +pyenv activate cur + +if [ -z ${TEST_FILTER+x} ]; \ +then export TEST_FILTER="TestTestgresLocal or (TestTestgresCommon and (not remote))"; \ +fi + +# fail early +echo check that pg_config is in PATH +command -v pg_config + +# prepare python environment +VENV_PATH="/tmp/testgres_venv" +rm -rf $VENV_PATH +python -m venv "${VENV_PATH}" +export VIRTUAL_ENV_DISABLE_PROMPT=1 +source "${VENV_PATH}/bin/activate" +pip install coverage flake8 psutil Sphinx pytest pytest-xdist psycopg2 six psutil + +# install testgres' dependencies +export PYTHONPATH=$(pwd) +# $PIP install . + +# test code quality +flake8 . + + +# remove existing coverage file +export COVERAGE_FILE=.coverage +rm -f $COVERAGE_FILE + + +# run tests (PATH) +time coverage run -a -m pytest -l -v -n 4 -k "${TEST_FILTER}" + + +# run tests (PG_BIN) +PG_BIN=$(pg_config --bindir) \ +time coverage run -a -m pytest -l -v -n 4 -k "${TEST_FILTER}" + + +# run tests (PG_CONFIG) +PG_CONFIG=$(pg_config --bindir)/pg_config \ +time coverage run -a -m pytest -l -v -n 4 -k "${TEST_FILTER}" + + +# show coverage +coverage report + +# build documentation +cd docs +make html +cd .. + +# attempt to fix codecov +set +eux + +# send coverage stats to Codecov +bash <(curl -s https://codecov.io/bash) diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index 25d02f38..33b61ac2 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -5,6 +5,7 @@ import tempfile import io import logging +import typing from ..exceptions import ExecUtilException from ..exceptions import InvalidOperationException @@ -669,8 +670,8 @@ def _is_port_free__process_1(error: str) -> bool: return True @staticmethod - def _make_exec_env_list() -> list[str]: - result = list[str]() + def _make_exec_env_list() -> typing.List[str]: + result: typing.List[str] = list() for envvar in os.environ.items(): if not __class__._does_put_envvar_into_exec_cmd(envvar[0]): continue diff --git a/testgres/port_manager.py b/testgres/port_manager.py index 164661e7..e2530470 100644 --- a/testgres/port_manager.py +++ b/testgres/port_manager.py @@ -6,6 +6,7 @@ import threading import random +import typing class PortManager: @@ -50,16 +51,16 @@ class PortManager__Generic(PortManager): _os_ops: OsOperations _guard: object # TODO: is there better to use bitmap fot _available_ports? - _available_ports: set[int] - _reserved_ports: set[int] + _available_ports: typing.Set[int] + _reserved_ports: typing.Set[int] def __init__(self, os_ops: OsOperations): assert os_ops is not None assert isinstance(os_ops, OsOperations) self._os_ops = os_ops self._guard = threading.Lock() - self._available_ports = set[int](range(1024, 65535)) - self._reserved_ports = set[int]() + self._available_ports: typing.Set[int] = set(range(1024, 65535)) + self._reserved_ports: typing.Set[int] = set() def reserve_port(self) -> int: assert self._guard is not None diff --git a/tests/conftest.py b/tests/conftest.py index ff3b3cb4..6f2f9e41 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -26,6 +26,10 @@ class TestConfigPropNames: TEST_CFG__LOG_DIR = "TEST_CFG__LOG_DIR" +# ///////////////////////////////////////////////////////////////////////////// + +T_TUPLE__str_int = typing.Tuple[str, int] + # ///////////////////////////////////////////////////////////////////////////// # TestStartupData__Helper @@ -110,11 +114,11 @@ class TEST_PROCESS_STATS: cUnexpectedTests: int = 0 cAchtungTests: int = 0 - FailedTests = list[str, int]() - XFailedTests = list[str, int]() - NotXFailedTests = list[str]() - WarningTests = list[str, int]() - AchtungTests = list[str]() + FailedTests: typing.List[T_TUPLE__str_int] = list() + XFailedTests: typing.List[T_TUPLE__str_int] = list() + NotXFailedTests: typing.List[str] = list() + WarningTests: typing.List[T_TUPLE__str_int] = list() + AchtungTests: typing.List[str] = list() cTotalDuration: datetime.timedelta = datetime.timedelta() @@ -769,7 +773,7 @@ def helper__calc_W(n: int) -> int: # ------------------------------------------------------------------------ -def helper__print_test_list(tests: list[str]) -> None: +def helper__print_test_list(tests: typing.List[str]) -> None: assert type(tests) == list # noqa: E721 assert helper__calc_W(9) == 1 @@ -796,7 +800,7 @@ def helper__print_test_list(tests: list[str]) -> None: # ------------------------------------------------------------------------ -def helper__print_test_list2(tests: list[str, int]) -> None: +def helper__print_test_list2(tests: typing.List[T_TUPLE__str_int]) -> None: assert type(tests) == list # noqa: E721 assert helper__calc_W(9) == 1 @@ -843,7 +847,7 @@ def LOCAL__print_line1_with_header(header: str): assert header != "" logging.info(C_LINE1 + " [" + header + "]") - def LOCAL__print_test_list(header: str, test_count: int, test_list: list[str]): + def LOCAL__print_test_list(header: str, test_count: int, test_list: typing.List[str]): assert type(header) == str # noqa: E721 assert type(test_count) == int # noqa: E721 assert type(test_list) == list # noqa: E721 @@ -858,7 +862,7 @@ def LOCAL__print_test_list(header: str, test_count: int, test_list: list[str]): logging.info("") def LOCAL__print_test_list2( - header: str, test_count: int, test_list: list[str, int] + header: str, test_count: int, test_list: typing.List[T_TUPLE__str_int] ): assert type(header) == str # noqa: E721 assert type(test_count) == int # noqa: E721 diff --git a/tests/test_os_ops_common.py b/tests/test_os_ops_common.py index 7d183775..ecfff5b2 100644 --- a/tests/test_os_ops_common.py +++ b/tests/test_os_ops_common.py @@ -12,13 +12,14 @@ import logging import socket import threading +import typing from ..testgres import InvalidOperationException from ..testgres import ExecUtilException class TestOsOpsCommon: - sm_os_ops_descrs: list[OsOpsDescr] = [ + sm_os_ops_descrs: typing.List[OsOpsDescr] = [ OsOpsDescrs.sm_local_os_ops_descr, OsOpsDescrs.sm_remote_os_ops_descr ] diff --git a/tests/test_testgres_common.py b/tests/test_testgres_common.py index c384dfb2..e1252de2 100644 --- a/tests/test_testgres_common.py +++ b/tests/test_testgres_common.py @@ -56,7 +56,7 @@ def removing(os_ops: OsOperations, f): class TestTestgresCommon: - sm_node_svcs: list[PostgresNodeService] = [ + sm_node_svcs: typing.List[PostgresNodeService] = [ PostgresNodeServices.sm_local, PostgresNodeServices.sm_local2, PostgresNodeServices.sm_remote, @@ -315,8 +315,8 @@ def test_child_pids(self, node_svc: PostgresNodeService): def LOCAL__test_auxiliary_pids( node: PostgresNode, - expectedTypes: list[ProcessType] - ) -> list[ProcessType]: + expectedTypes: typing.List[ProcessType] + ) -> typing.List[ProcessType]: # returns list of the absence processes assert node is not None assert type(node) == PostgresNode # noqa: E721 @@ -327,7 +327,7 @@ def LOCAL__test_auxiliary_pids( assert pids is not None # noqa: E721 assert type(pids) == dict # noqa: E721 - result = list[ProcessType]() + result: typing.List[ProcessType] = list() for ptype in expectedTypes: if not (ptype in pids): result.append(ptype) @@ -335,7 +335,7 @@ def LOCAL__test_auxiliary_pids( def LOCAL__check_auxiliary_pids__multiple_attempts( node: PostgresNode, - expectedTypes: list[ProcessType]): + expectedTypes: typing.List[ProcessType]): assert node is not None assert type(node) == PostgresNode # noqa: E721 assert expectedTypes is not None diff --git a/tests/test_utils.py b/tests/test_utils.py index d4a4c9ad..c05bd2fe 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -7,10 +7,11 @@ from ..testgres import scoped_config import pytest +import typing class TestUtils: - sm_os_ops_descrs: list[OsOpsDescr] = [ + sm_os_ops_descrs: typing.List[OsOpsDescr] = [ OsOpsDescrs.sm_local_os_ops_descr, OsOpsDescrs.sm_remote_os_ops_descr ] From ac782bb3354c596171c9d0498494fbb4b828463b Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Tue, 15 Apr 2025 14:50:35 +0300 Subject: [PATCH 197/216] [New] OsOps::execute_command supports a transfer of environment variables (exec_env) (#239) * [New] OsOps::execute_command supports a transfer of environment variables (exec_env) New feature allows to pass environment variables to an executed program. If variable in exec_env has None value, then this variable will be unset. PostgresNode::start and PostgresNode::slow_start supports exec_env. --- testgres/node.py | 12 +++--- testgres/operations/local_ops.py | 66 ++++++++++++++++++++++++++++--- testgres/operations/remote_ops.py | 44 ++++++++++++++++----- testgres/utils.py | 13 +++++- tests/test_os_ops_common.py | 64 ++++++++++++++++++++++++++++++ 5 files changed, 177 insertions(+), 22 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index 5039fc43..3a294044 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -1020,7 +1020,7 @@ def get_control_data(self): return out_dict - def slow_start(self, replica=False, dbname='template1', username=None, max_attempts=0): + def slow_start(self, replica=False, dbname='template1', username=None, max_attempts=0, exec_env=None): """ Starts the PostgreSQL instance and then polls the instance until it reaches the expected state (primary or replica). The state is checked @@ -1033,7 +1033,9 @@ def slow_start(self, replica=False, dbname='template1', username=None, max_attem If False, waits for the instance to be in primary mode. Default is False. max_attempts: """ - self.start() + assert exec_env is None or type(exec_env) == dict # noqa: E721 + + self.start(exec_env=exec_env) if replica: query = 'SELECT pg_is_in_recovery()' @@ -1065,7 +1067,7 @@ def _detect_port_conflict(self, log_files0, log_files1): return True return False - def start(self, params=[], wait=True): + def start(self, params=[], wait=True, exec_env=None): """ Starts the PostgreSQL node using pg_ctl if node has not been started. By default, it waits for the operation to complete before returning. @@ -1079,7 +1081,7 @@ def start(self, params=[], wait=True): Returns: This instance of :class:`.PostgresNode`. """ - + assert exec_env is None or type(exec_env) == dict # noqa: E721 assert __class__._C_MAX_START_ATEMPTS > 1 if self.is_started: @@ -1098,7 +1100,7 @@ def start(self, params=[], wait=True): def LOCAL__start_node(): # 'error' will be None on Windows - _, _, error = execute_utility2(self.os_ops, _params, self.utils_log_file, verbose=True) + _, _, error = execute_utility2(self.os_ops, _params, self.utils_log_file, verbose=True, exec_env=exec_env) assert error is None or type(error) == str # noqa: E721 if error and 'does not exist' in error: raise Exception(error) diff --git a/testgres/operations/local_ops.py b/testgres/operations/local_ops.py index 9785d462..74323bb8 100644 --- a/testgres/operations/local_ops.py +++ b/testgres/operations/local_ops.py @@ -9,6 +9,7 @@ import socket import psutil +import typing from ..exceptions import ExecUtilException from ..exceptions import InvalidOperationException @@ -46,9 +47,34 @@ def _process_output(encoding, temp_file_path): output = output.decode(encoding) return output, None # In Windows stderr writing in stdout - def _run_command__nt(self, cmd, shell, input, stdin, stdout, stderr, get_process, timeout, encoding): + def _run_command__nt(self, cmd, shell, input, stdin, stdout, stderr, get_process, timeout, encoding, exec_env=None): + assert exec_env is None or type(exec_env) == dict # noqa: E721 + # TODO: why don't we use the data from input? + extParams: typing.Dict[str, str] = dict() + + if exec_env is None: + pass + elif len(exec_env) == 0: + pass + else: + env = os.environ.copy() + assert type(env) == dict # noqa: E721 + for v in exec_env.items(): + assert type(v) == tuple # noqa: E721 + assert len(v) == 2 + assert type(v[0]) == str # noqa: E721 + assert v[0] != "" + + if v[1] is None: + env.pop(v[0], None) + else: + assert type(v[1]) == str # noqa: E721 + env[v[0]] = v[1] + + extParams["env"] = env + with tempfile.NamedTemporaryFile(mode='w+b', delete=False) as temp_file: stdout = temp_file stderr = subprocess.STDOUT @@ -58,6 +84,7 @@ def _run_command__nt(self, cmd, shell, input, stdin, stdout, stderr, get_process stdin=stdin or subprocess.PIPE if input is not None else None, stdout=stdout, stderr=stderr, + **extParams, ) if get_process: return process, None, None @@ -69,19 +96,45 @@ def _run_command__nt(self, cmd, shell, input, stdin, stdout, stderr, get_process output, error = self._process_output(encoding, temp_file_path) return process, output, error - def _run_command__generic(self, cmd, shell, input, stdin, stdout, stderr, get_process, timeout, encoding): + def _run_command__generic(self, cmd, shell, input, stdin, stdout, stderr, get_process, timeout, encoding, exec_env=None): + assert exec_env is None or type(exec_env) == dict # noqa: E721 + input_prepared = None if not get_process: input_prepared = Helpers.PrepareProcessInput(input, encoding) # throw assert input_prepared is None or (type(input_prepared) == bytes) # noqa: E721 + extParams: typing.Dict[str, str] = dict() + + if exec_env is None: + pass + elif len(exec_env) == 0: + pass + else: + env = os.environ.copy() + assert type(env) == dict # noqa: E721 + for v in exec_env.items(): + assert type(v) == tuple # noqa: E721 + assert len(v) == 2 + assert type(v[0]) == str # noqa: E721 + assert v[0] != "" + + if v[1] is None: + env.pop(v[0], None) + else: + assert type(v[1]) == str # noqa: E721 + env[v[0]] = v[1] + + extParams["env"] = env + process = subprocess.Popen( cmd, shell=shell, stdin=stdin or subprocess.PIPE if input is not None else None, stdout=stdout or subprocess.PIPE, stderr=stderr or subprocess.PIPE, + **extParams ) assert not (process is None) if get_process: @@ -100,25 +153,26 @@ def _run_command__generic(self, cmd, shell, input, stdin, stdout, stderr, get_pr error = error.decode(encoding) return process, output, error - def _run_command(self, cmd, shell, input, stdin, stdout, stderr, get_process, timeout, encoding): + def _run_command(self, cmd, shell, input, stdin, stdout, stderr, get_process, timeout, encoding, exec_env=None): """Execute a command and return the process and its output.""" if os.name == 'nt' and stdout is None: # Windows method = __class__._run_command__nt else: # Other OS method = __class__._run_command__generic - return method(self, cmd, shell, input, stdin, stdout, stderr, get_process, timeout, encoding) + return method(self, cmd, shell, input, stdin, stdout, stderr, get_process, timeout, encoding, exec_env=exec_env) def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False, encoding=None, shell=False, text=False, input=None, stdin=None, stdout=None, stderr=None, get_process=False, timeout=None, - ignore_errors=False): + ignore_errors=False, exec_env=None): """ Execute a command in a subprocess and handle the output based on the provided parameters. """ assert type(expect_error) == bool # noqa: E721 assert type(ignore_errors) == bool # noqa: E721 + assert exec_env is None or type(exec_env) == dict # noqa: E721 - process, output, error = self._run_command(cmd, shell, input, stdin, stdout, stderr, get_process, timeout, encoding) + process, output, error = self._run_command(cmd, shell, input, stdin, stdout, stderr, get_process, timeout, encoding, exec_env=exec_env) if get_process: return process diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index 33b61ac2..e722a2cb 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -64,7 +64,8 @@ def __enter__(self): def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False, encoding=None, shell=True, text=False, input=None, stdin=None, stdout=None, - stderr=None, get_process=None, timeout=None, ignore_errors=False): + stderr=None, get_process=None, timeout=None, ignore_errors=False, + exec_env=None): """ Execute a command in the SSH session. Args: @@ -72,6 +73,7 @@ def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False, """ assert type(expect_error) == bool # noqa: E721 assert type(ignore_errors) == bool # noqa: E721 + assert exec_env is None or type(exec_env) == dict # noqa: E721 input_prepared = None if not get_process: @@ -88,7 +90,7 @@ def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False, assert type(cmd_s) == str # noqa: E721 - cmd_items = __class__._make_exec_env_list() + cmd_items = __class__._make_exec_env_list(exec_env=exec_env) cmd_items.append(cmd_s) env_cmd_s = ';'.join(cmd_items) @@ -670,14 +672,38 @@ def _is_port_free__process_1(error: str) -> bool: return True @staticmethod - def _make_exec_env_list() -> typing.List[str]: - result: typing.List[str] = list() + def _make_exec_env_list(exec_env: typing.Dict) -> typing.List[str]: + env: typing.Dict[str, str] = dict() + + # ---------------------------------- SYSTEM ENV for envvar in os.environ.items(): - if not __class__._does_put_envvar_into_exec_cmd(envvar[0]): - continue - qvalue = __class__._quote_envvar(envvar[1]) - assert type(qvalue) == str # noqa: E721 - result.append(envvar[0] + "=" + qvalue) + if __class__._does_put_envvar_into_exec_cmd(envvar[0]): + env[envvar[0]] = envvar[1] + + # ---------------------------------- EXEC (LOCAL) ENV + if exec_env is None: + pass + else: + for envvar in exec_env.items(): + assert type(envvar) == tuple # noqa: E721 + assert len(envvar) == 2 + assert type(envvar[0]) == str # noqa: E721 + env[envvar[0]] = envvar[1] + + # ---------------------------------- FINAL BUILD + result: typing.List[str] = list() + for envvar in env.items(): + assert type(envvar) == tuple # noqa: E721 + assert len(envvar) == 2 + assert type(envvar[0]) == str # noqa: E721 + + if envvar[1] is None: + result.append("unset " + envvar[0]) + else: + assert type(envvar[1]) == str # noqa: E721 + qvalue = __class__._quote_envvar(envvar[1]) + assert type(qvalue) == str # noqa: E721 + result.append(envvar[0] + "=" + qvalue) continue return result diff --git a/testgres/utils.py b/testgres/utils.py index 10ae81b6..2ff6f2a0 100644 --- a/testgres/utils.py +++ b/testgres/utils.py @@ -96,17 +96,26 @@ def execute_utility(args, logfile=None, verbose=False): return execute_utility2(tconf.os_ops, args, logfile, verbose) -def execute_utility2(os_ops: OsOperations, args, logfile=None, verbose=False, ignore_errors=False): +def execute_utility2( + os_ops: OsOperations, + args, + logfile=None, + verbose=False, + ignore_errors=False, + exec_env=None, +): assert os_ops is not None assert isinstance(os_ops, OsOperations) assert type(verbose) == bool # noqa: E721 assert type(ignore_errors) == bool # noqa: E721 + assert exec_env is None or type(exec_env) == dict # noqa: E721 exit_status, out, error = os_ops.exec_command( args, verbose=True, ignore_errors=ignore_errors, - encoding=OsHelpers.GetDefaultEncoding()) + encoding=OsHelpers.GetDefaultEncoding(), + exec_env=exec_env) out = '' if not out else out diff --git a/tests/test_os_ops_common.py b/tests/test_os_ops_common.py index ecfff5b2..17c3151c 100644 --- a/tests/test_os_ops_common.py +++ b/tests/test_os_ops_common.py @@ -93,6 +93,70 @@ def test_exec_command_failure__expect_error(self, os_ops: OsOperations): assert b"nonexistent_command" in error assert b"not found" in error + def test_exec_command_with_exec_env(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + + RunConditions.skip_if_windows() + + C_ENV_NAME = "TESTGRES_TEST__EXEC_ENV_20250414" + + cmd = ["sh", "-c", "echo ${}".format(C_ENV_NAME)] + + exec_env = {C_ENV_NAME: "Hello!"} + + response = os_ops.exec_command(cmd, exec_env=exec_env) + assert response is not None + assert type(response) == bytes # noqa: E721 + assert response == b'Hello!\n' + + response = os_ops.exec_command(cmd) + assert response is not None + assert type(response) == bytes # noqa: E721 + assert response == b'\n' + + def test_exec_command__test_unset(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + + RunConditions.skip_if_windows() + + C_ENV_NAME = "LANG" + + cmd = ["sh", "-c", "echo ${}".format(C_ENV_NAME)] + + response1 = os_ops.exec_command(cmd) + assert response1 is not None + assert type(response1) == bytes # noqa: E721 + + if response1 == b'\n': + logging.warning("Environment variable {} is not defined.".format(C_ENV_NAME)) + return + + exec_env = {C_ENV_NAME: None} + response2 = os_ops.exec_command(cmd, exec_env=exec_env) + assert response2 is not None + assert type(response2) == bytes # noqa: E721 + assert response2 == b'\n' + + response3 = os_ops.exec_command(cmd) + assert response3 is not None + assert type(response3) == bytes # noqa: E721 + assert response3 == response1 + + def test_exec_command__test_unset_dummy_var(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + + RunConditions.skip_if_windows() + + C_ENV_NAME = "TESTGRES_TEST__DUMMY_VAR_20250414" + + cmd = ["sh", "-c", "echo ${}".format(C_ENV_NAME)] + + exec_env = {C_ENV_NAME: None} + response2 = os_ops.exec_command(cmd, exec_env=exec_env) + assert response2 is not None + assert type(response2) == bytes # noqa: E721 + assert response2 == b'\n' + def test_is_executable_true(self, os_ops: OsOperations): """ Test is_executable for an existing executable. From daa2b7b146dc5ed41c34f7d75ec0b8dcb8db00bd Mon Sep 17 00:00:00 2001 From: Victoria Shepard <5807469+demonolock@users.noreply.github.com> Date: Tue, 15 Apr 2025 20:32:33 +0200 Subject: [PATCH 198/216] Add maintain command (#175) --- .../plugins/pg_probackup2/pg_probackup2/app.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/app.py b/testgres/plugins/pg_probackup2/pg_probackup2/app.py index d47cf51f..5166e9b8 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/app.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/app.py @@ -842,5 +842,22 @@ def archive_get(self, instance, wal_file_name, wal_file_path, options=None, expe ] return self.run(cmd + options, expect_error=expect_error) + def maintain( + self, instance=None, backup_id=None, + options=None, old_binary=False, gdb=False, expect_error=False + ): + if options is None: + options = [] + cmd_list = [ + 'maintain', + ] + if instance: + cmd_list += ['--instance={0}'.format(instance)] + if backup_id: + cmd_list += ['-i', backup_id] + + return self.run(cmd_list + options, old_binary=old_binary, gdb=gdb, + expect_error=expect_error) + def build_backup_dir(self, backup='backup'): return fs_backup_class(rel_path=self.rel_path, backup=backup) From 330fd9abaf081a47ecd8f88767f18cb0bf3dcd2c Mon Sep 17 00:00:00 2001 From: Alexey Savchkov Date: Mon, 21 Apr 2025 20:09:27 +0700 Subject: [PATCH 199/216] Up versions --- setup.py | 2 +- testgres/plugins/pg_probackup2/setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index b47a1d8a..2c44b18f 100755 --- a/setup.py +++ b/setup.py @@ -27,7 +27,7 @@ readme = f.read() setup( - version='1.10.5', + version='1.11.0', name='testgres', packages=['testgres', 'testgres.operations'], description='Testing utility for PostgreSQL and its extensions', diff --git a/testgres/plugins/pg_probackup2/setup.py b/testgres/plugins/pg_probackup2/setup.py index 8bcfe7b4..7a3212e4 100644 --- a/testgres/plugins/pg_probackup2/setup.py +++ b/testgres/plugins/pg_probackup2/setup.py @@ -4,7 +4,7 @@ from distutils.core import setup setup( - version='0.0.6', + version='0.1.0', name='testgres_pg_probackup2', packages=['pg_probackup2', 'pg_probackup2.storage'], description='Plugin for testgres that manages pg_probackup2', From 6e5e4f5a9eb7f7a02df7056dcded7e0b68a6d1da Mon Sep 17 00:00:00 2001 From: Victoria Shepard <5807469+demonolock@users.noreply.github.com> Date: Thu, 24 Apr 2025 14:00:30 +0000 Subject: [PATCH 200/216] remove __init__.py from the root --- __init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 __init__.py diff --git a/__init__.py b/__init__.py deleted file mode 100644 index e69de29b..00000000 From 1a662f138f9e03b750b91455113ea3b58c0c986e Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Fri, 25 Apr 2025 15:33:13 +0300 Subject: [PATCH 201/216] [FIX] Tests include testgres by right way through import (#241) When we do not have root __init__.py tests must to import testgres through "import testgres" not through "from import testgres" --- .../pg_probackup2/tests/test_basic.py | 2 +- tests/helpers/global_data.py | 16 ++++---- tests/test_config.py | 10 ++--- tests/test_os_ops_common.py | 4 +- tests/test_os_ops_remote.py | 2 +- tests/test_testgres_common.py | 40 +++++++++---------- tests/test_testgres_local.py | 24 +++++------ tests/test_testgres_remote.py | 14 +++---- tests/test_utils.py | 6 +-- 9 files changed, 59 insertions(+), 59 deletions(-) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/tests/test_basic.py b/testgres/plugins/pg_probackup2/pg_probackup2/tests/test_basic.py index ba788623..f22a62bf 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/tests/test_basic.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/tests/test_basic.py @@ -4,7 +4,7 @@ import shutil import pytest -from ...... import testgres +import testgres from ...pg_probackup2.app import ProbackupApp from ...pg_probackup2.init_helpers import Init, init_params from ..storage.fs_backup import FSTestBackupDir diff --git a/tests/helpers/global_data.py b/tests/helpers/global_data.py index c21d7dd8..51bf4485 100644 --- a/tests/helpers/global_data.py +++ b/tests/helpers/global_data.py @@ -1,11 +1,11 @@ -from ...testgres.operations.os_ops import OsOperations -from ...testgres.operations.os_ops import ConnectionParams -from ...testgres.operations.local_ops import LocalOperations -from ...testgres.operations.remote_ops import RemoteOperations - -from ...testgres.node import PortManager -from ...testgres.node import PortManager__ThisHost -from ...testgres.node import PortManager__Generic +from testgres.operations.os_ops import OsOperations +from testgres.operations.os_ops import ConnectionParams +from testgres.operations.local_ops import LocalOperations +from testgres.operations.remote_ops import RemoteOperations + +from testgres.node import PortManager +from testgres.node import PortManager__ThisHost +from testgres.node import PortManager__Generic import os diff --git a/tests/test_config.py b/tests/test_config.py index 05702e9a..a80a11f1 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,9 +1,9 @@ -from ..testgres import TestgresConfig -from ..testgres import configure_testgres -from ..testgres import scoped_config -from ..testgres import pop_config +from testgres import TestgresConfig +from testgres import configure_testgres +from testgres import scoped_config +from testgres import pop_config -from .. import testgres +import testgres import pytest diff --git a/tests/test_os_ops_common.py b/tests/test_os_ops_common.py index 17c3151c..9c4d8857 100644 --- a/tests/test_os_ops_common.py +++ b/tests/test_os_ops_common.py @@ -14,8 +14,8 @@ import threading import typing -from ..testgres import InvalidOperationException -from ..testgres import ExecUtilException +from testgres import InvalidOperationException +from testgres import ExecUtilException class TestOsOpsCommon: diff --git a/tests/test_os_ops_remote.py b/tests/test_os_ops_remote.py index 338e49f3..65830218 100755 --- a/tests/test_os_ops_remote.py +++ b/tests/test_os_ops_remote.py @@ -3,7 +3,7 @@ from .helpers.global_data import OsOpsDescrs from .helpers.global_data import OsOperations -from ..testgres import ExecUtilException +from testgres import ExecUtilException import os import pytest diff --git a/tests/test_testgres_common.py b/tests/test_testgres_common.py index e1252de2..f4e5996f 100644 --- a/tests/test_testgres_common.py +++ b/tests/test_testgres_common.py @@ -3,29 +3,29 @@ from .helpers.global_data import OsOperations from .helpers.global_data import PortManager -from ..testgres.node import PgVer -from ..testgres.node import PostgresNode -from ..testgres.utils import get_pg_version2 -from ..testgres.utils import file_tail -from ..testgres.utils import get_bin_path2 -from ..testgres import ProcessType -from ..testgres import NodeStatus -from ..testgres import IsolationLevel +from testgres.node import PgVer +from testgres.node import PostgresNode +from testgres.utils import get_pg_version2 +from testgres.utils import file_tail +from testgres.utils import get_bin_path2 +from testgres import ProcessType +from testgres import NodeStatus +from testgres import IsolationLevel # New name prevents to collect test-functions in TestgresException and fixes # the problem with pytest warning. -from ..testgres import TestgresException as testgres_TestgresException - -from ..testgres import InitNodeException -from ..testgres import StartNodeException -from ..testgres import QueryException -from ..testgres import ExecUtilException -from ..testgres import TimeoutException -from ..testgres import InvalidOperationException -from ..testgres import BackupException -from ..testgres import ProgrammingError -from ..testgres import scoped_config -from ..testgres import First, Any +from testgres import TestgresException as testgres_TestgresException + +from testgres import InitNodeException +from testgres import StartNodeException +from testgres import QueryException +from testgres import ExecUtilException +from testgres import TimeoutException +from testgres import InvalidOperationException +from testgres import BackupException +from testgres import ProgrammingError +from testgres import scoped_config +from testgres import First, Any from contextlib import contextmanager diff --git a/tests/test_testgres_local.py b/tests/test_testgres_local.py index 9dbd455b..1dd98fe3 100644 --- a/tests/test_testgres_local.py +++ b/tests/test_testgres_local.py @@ -7,21 +7,21 @@ import platform import logging -from .. import testgres +import testgres -from ..testgres import StartNodeException -from ..testgres import ExecUtilException -from ..testgres import NodeApp -from ..testgres import scoped_config -from ..testgres import get_new_node -from ..testgres import get_bin_path -from ..testgres import get_pg_config -from ..testgres import get_pg_version +from testgres import StartNodeException +from testgres import ExecUtilException +from testgres import NodeApp +from testgres import scoped_config +from testgres import get_new_node +from testgres import get_bin_path +from testgres import get_pg_config +from testgres import get_pg_version # NOTE: those are ugly imports -from ..testgres.utils import bound_ports -from ..testgres.utils import PgVer -from ..testgres.node import ProcessProxy +from testgres.utils import bound_ports +from testgres.utils import PgVer +from testgres.node import ProcessProxy def pg_version_ge(version): diff --git a/tests/test_testgres_remote.py b/tests/test_testgres_remote.py index e38099b7..87cc0269 100755 --- a/tests/test_testgres_remote.py +++ b/tests/test_testgres_remote.py @@ -7,16 +7,16 @@ from .helpers.global_data import PostgresNodeService from .helpers.global_data import PostgresNodeServices -from .. import testgres +import testgres -from ..testgres.exceptions import InitNodeException -from ..testgres.exceptions import ExecUtilException +from testgres.exceptions import InitNodeException +from testgres.exceptions import ExecUtilException -from ..testgres.config import scoped_config -from ..testgres.config import testgres_config +from testgres.config import scoped_config +from testgres.config import testgres_config -from ..testgres import get_bin_path -from ..testgres import get_pg_config +from testgres import get_bin_path +from testgres import get_pg_config # NOTE: those are ugly imports diff --git a/tests/test_utils.py b/tests/test_utils.py index c05bd2fe..39e9dda0 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -2,9 +2,9 @@ from .helpers.global_data import OsOpsDescrs from .helpers.global_data import OsOperations -from ..testgres.utils import parse_pg_version -from ..testgres.utils import get_pg_config2 -from ..testgres import scoped_config +from testgres.utils import parse_pg_version +from testgres.utils import get_pg_config2 +from testgres import scoped_config import pytest import typing From 94d75725bb4d3bfb6667804e075e9b4d46e062d6 Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Mon, 28 Apr 2025 08:37:16 +0300 Subject: [PATCH 202/216] [#240] Using of node.psql with other host and port (#242) * [FIX] Tests include testgres by right way through import When we do not have root __init__.py tests must to import testgres through "import testgres" not through "from import testgres" * [#240] Using of node.psql with other host and port This patch adds the support of using other host and port in the following methods: - PostgresNode.psql (explicit new args: host and port) - PostgresNode.safe_psql (indirectly through **kwargs) It allows to run psql utility from one PostgreSQL instance to work with another one. If explicit host and port are not defined (are None), PostgresNode will use own ones. This patch closes #240. --- testgres/node.py | 29 +++++++++++- tests/test_testgres_common.py | 83 +++++++++++++++++++++++++++++++++++ 2 files changed, 110 insertions(+), 2 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index 3a294044..41504e89 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -1372,6 +1372,8 @@ def psql(self, dbname=None, username=None, input=None, + host: typing.Optional[str] = None, + port: typing.Optional[int] = None, **variables): """ Execute a query using psql. @@ -1382,6 +1384,8 @@ def psql(self, dbname: database name to connect to. username: database user name. input: raw input to be passed. + host: an explicit host of server. + port: an explicit port of server. **variables: vars to be set before execution. Returns: @@ -1393,6 +1397,10 @@ def psql(self, >>> psql(query='select 3', ON_ERROR_STOP=1) """ + assert host is None or type(host) == str # noqa: E721 + assert port is None or type(port) == int # noqa: E721 + assert type(variables) == dict # noqa: E721 + return self._psql( ignore_errors=True, query=query, @@ -1400,6 +1408,8 @@ def psql(self, dbname=dbname, username=username, input=input, + host=host, + port=port, **variables ) @@ -1411,7 +1421,11 @@ def _psql( dbname=None, username=None, input=None, + host: typing.Optional[str] = None, + port: typing.Optional[int] = None, **variables): + assert host is None or type(host) == str # noqa: E721 + assert port is None or type(port) == int # noqa: E721 assert type(variables) == dict # noqa: E721 # @@ -1424,10 +1438,21 @@ def _psql( else: raise Exception("Input data must be None or bytes.") + if host is None: + host = self.host + + if port is None: + port = self.port + + assert host is not None + assert port is not None + assert type(host) == str # noqa: E721 + assert type(port) == int # noqa: E721 + psql_params = [ self._get_bin_path("psql"), - "-p", str(self.port), - "-h", self.host, + "-p", str(port), + "-h", host, "-U", username or self.os_ops.username, "-d", dbname or default_dbname(), "-X", # no .psqlrc diff --git a/tests/test_testgres_common.py b/tests/test_testgres_common.py index f4e5996f..21fa00df 100644 --- a/tests/test_testgres_common.py +++ b/tests/test_testgres_common.py @@ -678,6 +678,89 @@ def test_psql(self, node_svc: PostgresNodeService): r = node.safe_psql('select 1') # raises! logging.error("node.safe_psql returns [{}]".format(r)) + def test_psql__another_port(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc).init() as node1: + with __class__.helper__get_node(node_svc).init() as node2: + node1.start() + node2.start() + assert node1.port != node2.port + assert node1.host == node2.host + + node1.stop() + + logging.info("test table in node2 is creating ...") + node2.safe_psql( + dbname="postgres", + query="create table test (id integer);" + ) + + logging.info("try to find test table through node1.psql ...") + res = node1.psql( + dbname="postgres", + query="select count(*) from pg_class where relname='test'", + host=node2.host, + port=node2.port, + ) + assert (__class__.helper__rm_carriage_returns(res) == (0, b'1\n', b'')) + + def test_psql__another_bad_host(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc).init() as node: + logging.info("try to execute node1.psql ...") + res = node.psql( + dbname="postgres", + query="select count(*) from pg_class where relname='test'", + host="DUMMY_HOST_NAME", + port=node.port, + ) + + res2 = __class__.helper__rm_carriage_returns(res) + + assert res2[0] != 0 + assert b"DUMMY_HOST_NAME" in res[2] + + def test_safe_psql__another_port(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc).init() as node1: + with __class__.helper__get_node(node_svc).init() as node2: + node1.start() + node2.start() + assert node1.port != node2.port + assert node1.host == node2.host + + node1.stop() + + logging.info("test table in node2 is creating ...") + node2.safe_psql( + dbname="postgres", + query="create table test (id integer);" + ) + + logging.info("try to find test table through node1.psql ...") + res = node1.safe_psql( + dbname="postgres", + query="select count(*) from pg_class where relname='test'", + host=node2.host, + port=node2.port, + ) + assert (__class__.helper__rm_carriage_returns(res) == b'1\n') + + def test_safe_psql__another_bad_host(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc).init() as node: + logging.info("try to execute node1.psql ...") + + with pytest.raises(expected_exception=Exception) as x: + node.safe_psql( + dbname="postgres", + query="select count(*) from pg_class where relname='test'", + host="DUMMY_HOST_NAME", + port=node.port, + ) + + assert "DUMMY_HOST_NAME" in str(x.value) + def test_safe_psql__expect_error(self, node_svc: PostgresNodeService): assert isinstance(node_svc, PostgresNodeService) with __class__.helper__get_node(node_svc).init().start() as node: From 2f550d8787130551e7c72d07070bf3dcc11dd586 Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Thu, 1 May 2025 23:21:48 +0300 Subject: [PATCH 203/216] Testgres tests create log dir in exact place (#243) When we do not define TEST_CFG__LOG_DIR it is expected the testgres tests will create a log directory in a root of testgres project folder. We used config.rootpath for detect this folder in pytest_configure function. It was occurred that config.rootpath can point to another (unexpected) place. So we will use exact code to calculate testgres project folder (see TestStartupData.GetRootLogDir) to avid this problem. --- tests/conftest.py | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 6f2f9e41..9e74879b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -50,6 +50,17 @@ def CalcRootDir() -> str: r = os.path.abspath(r) return r + # -------------------------------------------------------------------- + def CalcRootLogDir() -> str: + if TestConfigPropNames.TEST_CFG__LOG_DIR in os.environ: + resultPath = os.environ[TestConfigPropNames.TEST_CFG__LOG_DIR] + else: + rootDir = __class__.CalcRootDir() + resultPath = os.path.join(rootDir, "logs") + + assert type(resultPath) == str # noqa: E721 + return resultPath + # -------------------------------------------------------------------- def CalcCurrentTestWorkerSignature() -> str: currentPID = os.getpid() @@ -86,11 +97,18 @@ class TestStartupData: TestStartupData__Helper.CalcCurrentTestWorkerSignature() ) + sm_RootLogDir: str = TestStartupData__Helper.CalcRootLogDir() + # -------------------------------------------------------------------- def GetRootDir() -> str: assert type(__class__.sm_RootDir) == str # noqa: E721 return __class__.sm_RootDir + # -------------------------------------------------------------------- + def GetRootLogDir() -> str: + assert type(__class__.sm_RootLogDir) == str # noqa: E721 + return __class__.sm_RootLogDir + # -------------------------------------------------------------------- def GetCurrentTestWorkerSignature() -> str: assert type(__class__.sm_CurrentTestWorkerSignature) == str # noqa: E721 @@ -954,13 +972,9 @@ def pytest_configure(config: pytest.Config) -> None: log_name = TestStartupData.GetCurrentTestWorkerSignature() log_name += ".log" - if TestConfigPropNames.TEST_CFG__LOG_DIR in os.environ: - log_path_v = os.environ[TestConfigPropNames.TEST_CFG__LOG_DIR] - log_path = pathlib.Path(log_path_v) - else: - log_path = config.rootpath.joinpath("logs") + log_dir = TestStartupData.GetRootLogDir() - log_path.mkdir(exist_ok=True) + pathlib.Path(log_dir).mkdir(exist_ok=True) logging_plugin: _pytest.logging.LoggingPlugin = config.pluginmanager.get_plugin( "logging-plugin" @@ -969,7 +983,7 @@ def pytest_configure(config: pytest.Config) -> None: assert logging_plugin is not None assert isinstance(logging_plugin, _pytest.logging.LoggingPlugin) - logging_plugin.set_log_path(str(log_path / log_name)) + logging_plugin.set_log_path(os.path.join(log_dir, log_name)) # ///////////////////////////////////////////////////////////////////////////// From 5f8f5dd2e5684a340f640141f01b3edd5ebca5a9 Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Sun, 4 May 2025 06:10:29 +0300 Subject: [PATCH 204/216] [test] TestTestgresLocal.test_upgrade_node is corrected (#246) Let's "release" all our test nodes correctly. --- tests/test_testgres_local.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/test_testgres_local.py b/tests/test_testgres_local.py index 1dd98fe3..53c9e0f6 100644 --- a/tests/test_testgres_local.py +++ b/tests/test_testgres_local.py @@ -158,15 +158,15 @@ def test_child_process_dies(self): def test_upgrade_node(self): old_bin_dir = os.path.dirname(get_bin_path("pg_config")) new_bin_dir = os.path.dirname(get_bin_path("pg_config")) - node_old = get_new_node(prefix='node_old', bin_dir=old_bin_dir) - node_old.init() - node_old.start() - node_old.stop() - node_new = get_new_node(prefix='node_new', bin_dir=new_bin_dir) - node_new.init(cached=False) - res = node_new.upgrade_from(old_node=node_old) - node_new.start() - assert (b'Upgrade Complete' in res) + with get_new_node(prefix='node_old', bin_dir=old_bin_dir) as node_old: + node_old.init() + node_old.start() + node_old.stop() + with get_new_node(prefix='node_new', bin_dir=new_bin_dir) as node_new: + node_new.init(cached=False) + res = node_new.upgrade_from(old_node=node_old) + node_new.start() + assert (b'Upgrade Complete' in res) class tagPortManagerProxy: sm_prev_testgres_reserve_port = None From 0b331e6839002da9b51fb3ca6ca7db228373dff0 Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Sun, 4 May 2025 16:01:58 +0300 Subject: [PATCH 205/216] Releasing of reserved port in tests (#248) * [test] TestTestgresLocal.test_pg_ctl_wait_option is corrected Let's "release" all our test nodes correctly. * [test] TestTestgresLocal.test_simple_with_bin_dir is corrected Let's "release" all our test nodes correctly. --- tests/test_testgres_common.py | 15 ++++++++++++--- tests/test_testgres_local.py | 8 ++++---- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/tests/test_testgres_common.py b/tests/test_testgres_common.py index 21fa00df..b71dc5ad 100644 --- a/tests/test_testgres_common.py +++ b/tests/test_testgres_common.py @@ -883,10 +883,20 @@ def test_backup_wrong_xlog_method(self, node_svc: PostgresNodeService): def test_pg_ctl_wait_option(self, node_svc: PostgresNodeService): assert isinstance(node_svc, PostgresNodeService) - C_MAX_ATTEMPTS = 50 + with __class__.helper__get_node(node_svc) as node: + self.impl__test_pg_ctl_wait_option(node_svc, node) - node = __class__.helper__get_node(node_svc) + def impl__test_pg_ctl_wait_option( + self, + node_svc: PostgresNodeService, + node: PostgresNode + ) -> None: + assert isinstance(node_svc, PostgresNodeService) + assert isinstance(node, PostgresNode) assert node.status() == NodeStatus.Uninitialized + + C_MAX_ATTEMPTS = 50 + node.init() assert node.status() == NodeStatus.Stopped node.start(wait=False) @@ -950,7 +960,6 @@ def test_pg_ctl_wait_option(self, node_svc: PostgresNodeService): raise Exception("Unexpected node status: {0}.".format(s1)) logging.info("OK. Node is stopped.") - node.cleanup() def test_replicate(self, node_svc: PostgresNodeService): assert isinstance(node_svc, PostgresNodeService) diff --git a/tests/test_testgres_local.py b/tests/test_testgres_local.py index 53c9e0f6..63e5f37e 100644 --- a/tests/test_testgres_local.py +++ b/tests/test_testgres_local.py @@ -341,10 +341,10 @@ def test_simple_with_bin_dir(self): bin_dir = node.bin_dir app = NodeApp() - correct_bin_dir = app.make_simple(base_dir=node.base_dir, bin_dir=bin_dir) - correct_bin_dir.slow_start() - correct_bin_dir.safe_psql("SELECT 1;") - correct_bin_dir.stop() + with app.make_simple(base_dir=node.base_dir, bin_dir=bin_dir) as correct_bin_dir: + correct_bin_dir.slow_start() + correct_bin_dir.safe_psql("SELECT 1;") + correct_bin_dir.stop() while True: try: From c3b25b22f6004d592e1f53d0366486d367fb3b48 Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Sun, 4 May 2025 22:39:18 +0300 Subject: [PATCH 206/216] [#249] Fix of port number leak in NodeBackup::spawn_replica (#250) This patch has the following changes: 1) It adds a new argument release_resources to PostgresNode::cleanup method. Default value is False. 2) It fixes a port number leak in NodeBackup::spawn_replica through explicit call of PostgresNode::cleanup(release_resources=True). Closes #249. --- testgres/backup.py | 11 ++++++++--- testgres/node.py | 12 +++++++++--- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/testgres/backup.py b/testgres/backup.py index 388697b7..857c46d4 100644 --- a/testgres/backup.py +++ b/testgres/backup.py @@ -184,14 +184,19 @@ def spawn_replica(self, name=None, destroy=True, slot=None): """ # Build a new PostgresNode - with clean_on_error(self.spawn_primary(name=name, - destroy=destroy)) as node: + node = self.spawn_primary(name=name, destroy=destroy) + assert node is not None + try: # Assign it a master and a recovery file (private magic) node._assign_master(self.original_node) node._create_recovery_conf(username=self.username, slot=slot) + except: # noqa: E722 + # TODO: Pass 'final=True' ? + node.cleanup(release_resources=True) + raise - return node + return node def cleanup(self): """ diff --git a/testgres/node.py b/testgres/node.py index 41504e89..defc0b40 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -231,8 +231,6 @@ def __enter__(self): return self def __exit__(self, type, value, traceback): - self.free_port() - # NOTE: Ctrl+C does not count! got_exception = type is not None and type != KeyboardInterrupt @@ -246,6 +244,8 @@ def __exit__(self, type, value, traceback): else: self._try_shutdown(attempts) + self._release_resources() + def __repr__(self): return "{}(name='{}', port={}, base_dir='{}')".format( self.__class__.__name__, @@ -663,6 +663,9 @@ def _try_shutdown(self, max_attempts, with_force=False): ps_output, ps_command) + def _release_resources(self): + self.free_port() + @staticmethod def _throw_bugcheck__unexpected_result_of_ps(result, cmd): assert type(result) == str # noqa: E721 @@ -1340,7 +1343,7 @@ def free_port(self): self._port = None self._port_manager.release_port(port) - def cleanup(self, max_attempts=3, full=False): + def cleanup(self, max_attempts=3, full=False, release_resources=False): """ Stop node if needed and remove its data/logs directory. NOTE: take a look at TestgresConfig.node_cleanup_full. @@ -1363,6 +1366,9 @@ def cleanup(self, max_attempts=3, full=False): self.os_ops.rmdirs(rm_dir, ignore_errors=False) + if release_resources: + self._release_resources() + return self @method_decorator(positional_args_hack(['dbname', 'query'])) From a683c65ae222e1980aa141f8d215c9fc3eac9383 Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Mon, 5 May 2025 15:12:45 +0300 Subject: [PATCH 207/216] [Refactoring] Default port manager functions now use PortManager__Generic and LocalOperations (#251) * [Refactoring] Default port manager functions now use PortManager__Generic and LocalOperations This patch deletes a duplication of port manager code. Now utils.reserve_port and utils.release_port works through _old_port_manager - it is a global instance of PortManager__Generic that uses a global instance of LocalOperations. This commit is a part of work for #247. * [BUG FIX] PortManager__ThisHost::__new__ had MT-problem After MT-lock we must to check __class__.sm_single_instance again. Refactoring - PortManager__ThisHost::__new__ is replaced with an explicit PortManager__ThisHost::get_single_instance() - PortManager__ThisHost::__init__ is deleted --- setup.py | 2 +- testgres/impl/port_manager__generic.py | 64 ++++++++++++++++ testgres/impl/port_manager__this_host.py | 33 +++++++++ testgres/node.py | 6 +- testgres/port_manager.py | 93 ------------------------ testgres/utils.py | 42 ++++------- tests/helpers/global_data.py | 2 +- 7 files changed, 115 insertions(+), 127 deletions(-) create mode 100755 testgres/impl/port_manager__generic.py create mode 100755 testgres/impl/port_manager__this_host.py diff --git a/setup.py b/setup.py index 2c44b18f..0b209181 100755 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ setup( version='1.11.0', name='testgres', - packages=['testgres', 'testgres.operations'], + packages=['testgres', 'testgres.operations', 'testgres.impl'], description='Testing utility for PostgreSQL and its extensions', url='https://github.com/postgrespro/testgres', long_description=readme, diff --git a/testgres/impl/port_manager__generic.py b/testgres/impl/port_manager__generic.py new file mode 100755 index 00000000..a51af2bd --- /dev/null +++ b/testgres/impl/port_manager__generic.py @@ -0,0 +1,64 @@ +from ..operations.os_ops import OsOperations + +from ..port_manager import PortManager +from ..exceptions import PortForException + +import threading +import random +import typing + + +class PortManager__Generic(PortManager): + _os_ops: OsOperations + _guard: object + # TODO: is there better to use bitmap fot _available_ports? + _available_ports: typing.Set[int] + _reserved_ports: typing.Set[int] + + def __init__(self, os_ops: OsOperations): + assert os_ops is not None + assert isinstance(os_ops, OsOperations) + self._os_ops = os_ops + self._guard = threading.Lock() + self._available_ports: typing.Set[int] = set(range(1024, 65535)) + self._reserved_ports: typing.Set[int] = set() + + def reserve_port(self) -> int: + assert self._guard is not None + assert type(self._available_ports) == set # noqa: E721t + assert type(self._reserved_ports) == set # noqa: E721 + + with self._guard: + t = tuple(self._available_ports) + assert len(t) == len(self._available_ports) + sampled_ports = random.sample(t, min(len(t), 100)) + t = None + + for port in sampled_ports: + assert not (port in self._reserved_ports) + assert port in self._available_ports + + if not self._os_ops.is_port_free(port): + continue + + self._reserved_ports.add(port) + self._available_ports.discard(port) + assert port in self._reserved_ports + assert not (port in self._available_ports) + return port + + raise PortForException("Can't select a port.") + + def release_port(self, number: int) -> None: + assert type(number) == int # noqa: E721 + + assert self._guard is not None + assert type(self._reserved_ports) == set # noqa: E721 + + with self._guard: + assert number in self._reserved_ports + assert not (number in self._available_ports) + self._available_ports.add(number) + self._reserved_ports.discard(number) + assert not (number in self._reserved_ports) + assert number in self._available_ports diff --git a/testgres/impl/port_manager__this_host.py b/testgres/impl/port_manager__this_host.py new file mode 100755 index 00000000..0d56f356 --- /dev/null +++ b/testgres/impl/port_manager__this_host.py @@ -0,0 +1,33 @@ +from ..port_manager import PortManager + +from .. import utils + +import threading + + +class PortManager__ThisHost(PortManager): + sm_single_instance: PortManager = None + sm_single_instance_guard = threading.Lock() + + @staticmethod + def get_single_instance() -> PortManager: + assert __class__ == PortManager__ThisHost + assert __class__.sm_single_instance_guard is not None + + if __class__.sm_single_instance is not None: + assert type(__class__.sm_single_instance) == __class__ # noqa: E721 + return __class__.sm_single_instance + + with __class__.sm_single_instance_guard: + if __class__.sm_single_instance is None: + __class__.sm_single_instance = __class__() + assert __class__.sm_single_instance is not None + assert type(__class__.sm_single_instance) == __class__ # noqa: E721 + return __class__.sm_single_instance + + def reserve_port(self) -> int: + return utils.reserve_port() + + def release_port(self, number: int) -> None: + assert type(number) == int # noqa: E721 + return utils.release_port(number) diff --git a/testgres/node.py b/testgres/node.py index defc0b40..dd1a45d3 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -84,8 +84,8 @@ InvalidOperationException from .port_manager import PortManager -from .port_manager import PortManager__ThisHost -from .port_manager import PortManager__Generic +from .impl.port_manager__this_host import PortManager__ThisHost +from .impl.port_manager__generic import PortManager__Generic from .logger import TestgresLogger @@ -272,7 +272,7 @@ def _get_port_manager(os_ops: OsOperations) -> PortManager: assert isinstance(os_ops, OsOperations) if isinstance(os_ops, LocalOperations): - return PortManager__ThisHost() + return PortManager__ThisHost.get_single_instance() # TODO: Throw the exception "Please define a port manager." ? return PortManager__Generic(os_ops) diff --git a/testgres/port_manager.py b/testgres/port_manager.py index e2530470..1ae696c8 100644 --- a/testgres/port_manager.py +++ b/testgres/port_manager.py @@ -1,14 +1,3 @@ -from .operations.os_ops import OsOperations - -from .exceptions import PortForException - -from . import utils - -import threading -import random -import typing - - class PortManager: def __init__(self): super().__init__() @@ -19,85 +8,3 @@ def reserve_port(self) -> int: def release_port(self, number: int) -> None: assert type(number) == int # noqa: E721 raise NotImplementedError("PortManager::release_port is not implemented.") - - -class PortManager__ThisHost(PortManager): - sm_single_instance: PortManager = None - sm_single_instance_guard = threading.Lock() - - def __init__(self): - pass - - def __new__(cls) -> PortManager: - assert __class__ == PortManager__ThisHost - assert __class__.sm_single_instance_guard is not None - - if __class__.sm_single_instance is None: - with __class__.sm_single_instance_guard: - __class__.sm_single_instance = super().__new__(cls) - assert __class__.sm_single_instance - assert type(__class__.sm_single_instance) == __class__ # noqa: E721 - return __class__.sm_single_instance - - def reserve_port(self) -> int: - return utils.reserve_port() - - def release_port(self, number: int) -> None: - assert type(number) == int # noqa: E721 - return utils.release_port(number) - - -class PortManager__Generic(PortManager): - _os_ops: OsOperations - _guard: object - # TODO: is there better to use bitmap fot _available_ports? - _available_ports: typing.Set[int] - _reserved_ports: typing.Set[int] - - def __init__(self, os_ops: OsOperations): - assert os_ops is not None - assert isinstance(os_ops, OsOperations) - self._os_ops = os_ops - self._guard = threading.Lock() - self._available_ports: typing.Set[int] = set(range(1024, 65535)) - self._reserved_ports: typing.Set[int] = set() - - def reserve_port(self) -> int: - assert self._guard is not None - assert type(self._available_ports) == set # noqa: E721t - assert type(self._reserved_ports) == set # noqa: E721 - - with self._guard: - t = tuple(self._available_ports) - assert len(t) == len(self._available_ports) - sampled_ports = random.sample(t, min(len(t), 100)) - t = None - - for port in sampled_ports: - assert not (port in self._reserved_ports) - assert port in self._available_ports - - if not self._os_ops.is_port_free(port): - continue - - self._reserved_ports.add(port) - self._available_ports.discard(port) - assert port in self._reserved_ports - assert not (port in self._available_ports) - return port - - raise PortForException("Can't select a port.") - - def release_port(self, number: int) -> None: - assert type(number) == int # noqa: E721 - - assert self._guard is not None - assert type(self._reserved_ports) == set # noqa: E721 - - with self._guard: - assert number in self._reserved_ports - assert not (number in self._available_ports) - self._available_ports.add(number) - self._reserved_ports.discard(number) - assert not (number in self._reserved_ports) - assert number in self._available_ports diff --git a/testgres/utils.py b/testgres/utils.py index 2ff6f2a0..6603c929 100644 --- a/testgres/utils.py +++ b/testgres/utils.py @@ -6,8 +6,6 @@ import os import sys -import socket -import random from contextlib import contextmanager from packaging.version import Version, InvalidVersion @@ -15,18 +13,27 @@ from six import iteritems -from .exceptions import PortForException from .exceptions import ExecUtilException from .config import testgres_config as tconf from .operations.os_ops import OsOperations from .operations.remote_ops import RemoteOperations +from .operations.local_ops import LocalOperations from .operations.helpers import Helpers as OsHelpers +from .impl.port_manager__generic import PortManager__Generic + # rows returned by PG_CONFIG _pg_config_data = {} +_local_operations = LocalOperations() + +# +# The old, global "port manager" always worked with LOCAL system +# +_old_port_manager = PortManager__Generic(_local_operations) + # ports used by nodes -bound_ports = set() +bound_ports = _old_port_manager._reserved_ports # re-export version type @@ -43,28 +50,7 @@ def internal__reserve_port(): """ Generate a new port and add it to 'bound_ports'. """ - def LOCAL__is_port_free(port: int) -> bool: - """Check if a port is free to use.""" - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: - try: - s.bind(("", port)) - return True - except OSError: - return False - - ports = set(range(1024, 65535)) - assert type(ports) == set # noqa: E721 - assert type(bound_ports) == set # noqa: E721 - ports.difference_update(bound_ports) - - sampled_ports = random.sample(tuple(ports), min(len(ports), 100)) - - for port in sampled_ports: - if LOCAL__is_port_free(port): - bound_ports.add(port) - return port - - raise PortForException("Can't select a port") + return _old_port_manager.reserve_port() def internal__release_port(port): @@ -73,9 +59,7 @@ def internal__release_port(port): """ assert type(port) == int # noqa: E721 - assert port in bound_ports - - bound_ports.discard(port) + return _old_port_manager.release_port(port) reserve_port = internal__reserve_port diff --git a/tests/helpers/global_data.py b/tests/helpers/global_data.py index 51bf4485..07ac083d 100644 --- a/tests/helpers/global_data.py +++ b/tests/helpers/global_data.py @@ -39,7 +39,7 @@ class OsOpsDescrs: class PortManagers: sm_remote_port_manager = PortManager__Generic(OsOpsDescrs.sm_remote_os_ops) - sm_local_port_manager = PortManager__ThisHost() + sm_local_port_manager = PortManager__ThisHost.get_single_instance() sm_local2_port_manager = PortManager__Generic(OsOpsDescrs.sm_local_os_ops) From 6972bfcdab479088ac881375db92944be87487f4 Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Mon, 5 May 2025 15:27:43 +0300 Subject: [PATCH 208/216] [#244] PostgresNode now uses os_ops only (#245) This patch implements the proposal #244 - detach PostgresNode from ConnectionParams object. It will use os_ops object only. conn_params is saved but must be None. It will be removed in the future. --- testgres/node.py | 30 +++++++++++++++--------------- tests/test_testgres_common.py | 1 - tests/test_testgres_remote.py | 1 - 3 files changed, 15 insertions(+), 17 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index dd1a45d3..80ac5ee8 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -107,7 +107,6 @@ from .operations.os_ops import ConnectionParams from .operations.os_ops import OsOperations from .operations.local_ops import LocalOperations -from .operations.remote_ops import RemoteOperations InternalError = pglib.InternalError ProgrammingError = pglib.ProgrammingError @@ -151,7 +150,7 @@ def __init__(self, name=None, base_dir=None, port: typing.Optional[int] = None, - conn_params: ConnectionParams = ConnectionParams(), + conn_params: ConnectionParams = None, bin_dir=None, prefix=None, os_ops: typing.Optional[OsOperations] = None, @@ -171,11 +170,15 @@ def __init__(self, assert os_ops is None or isinstance(os_ops, OsOperations) assert port_manager is None or isinstance(port_manager, PortManager) + if conn_params is not None: + assert type(conn_params) == ConnectionParams # noqa: E721 + + raise InvalidOperationException("conn_params is deprecated, please use os_ops parameter instead.") + # private if os_ops is None: - self._os_ops = __class__._get_os_ops(conn_params) + self._os_ops = __class__._get_os_ops() else: - assert conn_params is None assert isinstance(os_ops, OsOperations) self._os_ops = os_ops pass @@ -200,11 +203,14 @@ def __init__(self, self._should_free_port = False self._port_manager = None else: - if port_manager is not None: + if port_manager is None: + self._port_manager = __class__._get_port_manager(self._os_ops) + elif os_ops is None: + raise InvalidOperationException("When port_manager is not None you have to define os_ops, too.") + else: assert isinstance(port_manager, PortManager) + assert self._os_ops is os_ops self._port_manager = port_manager - else: - self._port_manager = __class__._get_port_manager(self._os_ops) assert self._port_manager is not None assert isinstance(self._port_manager, PortManager) @@ -255,16 +261,11 @@ def __repr__(self): ) @staticmethod - def _get_os_ops(conn_params: ConnectionParams) -> OsOperations: + def _get_os_ops() -> OsOperations: if testgres_config.os_ops: return testgres_config.os_ops - assert type(conn_params) == ConnectionParams # noqa: E721 - - if conn_params.ssh_key: - return RemoteOperations(conn_params) - - return LocalOperations(conn_params) + return LocalOperations() @staticmethod def _get_port_manager(os_ops: OsOperations) -> PortManager: @@ -294,7 +295,6 @@ def clone_with_new_name_and_base_dir(self, name: str, base_dir: str): node = PostgresNode( name=name, base_dir=base_dir, - conn_params=None, bin_dir=self._bin_dir, prefix=self._prefix, os_ops=self._os_ops, diff --git a/tests/test_testgres_common.py b/tests/test_testgres_common.py index b71dc5ad..5b926bc8 100644 --- a/tests/test_testgres_common.py +++ b/tests/test_testgres_common.py @@ -1487,7 +1487,6 @@ def helper__get_node( return PostgresNode( name, port=port, - conn_params=None, os_ops=node_svc.os_ops, port_manager=port_manager if port is None else None ) diff --git a/tests/test_testgres_remote.py b/tests/test_testgres_remote.py index 87cc0269..6a8d068b 100755 --- a/tests/test_testgres_remote.py +++ b/tests/test_testgres_remote.py @@ -173,7 +173,6 @@ def helper__get_node(name=None): return testgres.PostgresNode( name, - conn_params=None, os_ops=svc.os_ops, port_manager=svc.port_manager) From df4f545eb47427f2997dbe7eeb18e80ff64c5686 Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Tue, 6 May 2025 09:39:26 +0300 Subject: [PATCH 209/216] LocalOperations::get_single_instance is added (#252) * LocalOperations::get_single_instance is added This patch forces testgres to use a single instance of LocalOperations that is created with default parameters. Note that, PortManager__ThisHost is used only when PostgresNode uses this single local_ops instance. --- testgres/cache.py | 8 ++++++-- testgres/config.py | 3 ++- testgres/node.py | 23 +++++++++++++++++++---- testgres/operations/local_ops.py | 20 ++++++++++++++++++++ testgres/utils.py | 4 +--- tests/helpers/global_data.py | 2 +- 6 files changed, 49 insertions(+), 11 deletions(-) diff --git a/testgres/cache.py b/testgres/cache.py index 3ac63326..499cce91 100644 --- a/testgres/cache.py +++ b/testgres/cache.py @@ -22,12 +22,16 @@ from .operations.os_ops import OsOperations -def cached_initdb(data_dir, logfile=None, params=None, os_ops: OsOperations = LocalOperations(), bin_path=None, cached=True): +def cached_initdb(data_dir, logfile=None, params=None, os_ops: OsOperations = None, bin_path=None, cached=True): """ Perform initdb or use cached node files. """ - assert os_ops is not None + assert os_ops is None or isinstance(os_ops, OsOperations) + + if os_ops is None: + os_ops = LocalOperations.get_single_instance() + assert isinstance(os_ops, OsOperations) def make_utility_path(name): diff --git a/testgres/config.py b/testgres/config.py index 67d467d3..55d52426 100644 --- a/testgres/config.py +++ b/testgres/config.py @@ -50,8 +50,9 @@ class GlobalConfig(object): _cached_initdb_dir = None """ underlying class attribute for cached_initdb_dir property """ - os_ops = LocalOperations() + os_ops = LocalOperations.get_single_instance() """ OsOperation object that allows work on remote host """ + @property def cached_initdb_dir(self): """ path to a temp directory for cached initdb. """ diff --git a/testgres/node.py b/testgres/node.py index 80ac5ee8..66783e08 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -93,6 +93,8 @@ from .standby import First +from . import utils + from .utils import \ PgVer, \ eprint, \ @@ -265,14 +267,17 @@ def _get_os_ops() -> OsOperations: if testgres_config.os_ops: return testgres_config.os_ops - return LocalOperations() + return LocalOperations.get_single_instance() @staticmethod def _get_port_manager(os_ops: OsOperations) -> PortManager: assert os_ops is not None assert isinstance(os_ops, OsOperations) - if isinstance(os_ops, LocalOperations): + if os_ops is LocalOperations.get_single_instance(): + assert utils._old_port_manager is not None + assert type(utils._old_port_manager) == PortManager__Generic # noqa: E721 + assert utils._old_port_manager._os_ops is os_ops return PortManager__ThisHost.get_single_instance() # TODO: Throw the exception "Please define a port manager." ? @@ -816,10 +821,13 @@ def init(self, initdb_params=None, cached=True, **kwargs): """ # initialize this PostgreSQL node + assert self._os_ops is not None + assert isinstance(self._os_ops, OsOperations) + cached_initdb( data_dir=self.data_dir, logfile=self.utils_log_file, - os_ops=self.os_ops, + os_ops=self._os_ops, params=initdb_params, bin_path=self.bin_dir, cached=False) @@ -2186,7 +2194,14 @@ def _escape_config_value(value): class NodeApp: - def __init__(self, test_path=None, nodes_to_cleanup=None, os_ops=LocalOperations()): + def __init__(self, test_path=None, nodes_to_cleanup=None, os_ops=None): + assert os_ops is None or isinstance(os_ops, OsOperations) + + if os_ops is None: + os_ops = LocalOperations.get_single_instance() + + assert isinstance(os_ops, OsOperations) + if test_path: if os.path.isabs(test_path): self.test_path = test_path diff --git a/testgres/operations/local_ops.py b/testgres/operations/local_ops.py index 74323bb8..b9fd7aef 100644 --- a/testgres/operations/local_ops.py +++ b/testgres/operations/local_ops.py @@ -10,6 +10,7 @@ import psutil import typing +import threading from ..exceptions import ExecUtilException from ..exceptions import InvalidOperationException @@ -28,6 +29,9 @@ class LocalOperations(OsOperations): + sm_single_instance: OsOperations = None + sm_single_instance_guard = threading.Lock() + def __init__(self, conn_params=None): if conn_params is None: conn_params = ConnectionParams() @@ -38,6 +42,22 @@ def __init__(self, conn_params=None): self.remote = False self.username = conn_params.username or getpass.getuser() + @staticmethod + def get_single_instance() -> OsOperations: + assert __class__ == LocalOperations + assert __class__.sm_single_instance_guard is not None + + if __class__.sm_single_instance is not None: + assert type(__class__.sm_single_instance) == __class__ # noqa: E721 + return __class__.sm_single_instance + + with __class__.sm_single_instance_guard: + if __class__.sm_single_instance is None: + __class__.sm_single_instance = __class__() + assert __class__.sm_single_instance is not None + assert type(__class__.sm_single_instance) == __class__ # noqa: E721 + return __class__.sm_single_instance + @staticmethod def _process_output(encoding, temp_file_path): """Process the output of a command from a temporary file.""" diff --git a/testgres/utils.py b/testgres/utils.py index 6603c929..d231eec3 100644 --- a/testgres/utils.py +++ b/testgres/utils.py @@ -25,12 +25,10 @@ # rows returned by PG_CONFIG _pg_config_data = {} -_local_operations = LocalOperations() - # # The old, global "port manager" always worked with LOCAL system # -_old_port_manager = PortManager__Generic(_local_operations) +_old_port_manager = PortManager__Generic(LocalOperations.get_single_instance()) # ports used by nodes bound_ports = _old_port_manager._reserved_ports diff --git a/tests/helpers/global_data.py b/tests/helpers/global_data.py index 07ac083d..f3df41a3 100644 --- a/tests/helpers/global_data.py +++ b/tests/helpers/global_data.py @@ -31,7 +31,7 @@ class OsOpsDescrs: sm_remote_os_ops_descr = OsOpsDescr("remote_ops", sm_remote_os_ops) - sm_local_os_ops = LocalOperations() + sm_local_os_ops = LocalOperations.get_single_instance() sm_local_os_ops_descr = OsOpsDescr("local_ops", sm_local_os_ops) From a0a85065f59ac40a8e8f951e88efa346cfb9d695 Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Tue, 6 May 2025 14:52:14 +0300 Subject: [PATCH 210/216] New OsOperations methods: makedir, rmdir (#253) Signatures: def makedir(self, path: str) def rmdir(self, path: str) It is a part of work for #247. --- testgres/operations/local_ops.py | 8 + testgres/operations/os_ops.py | 8 + testgres/operations/remote_ops.py | 10 ++ tests/test_os_ops_common.py | 268 ++++++++++++++++++++++++++++++ 4 files changed, 294 insertions(+) diff --git a/testgres/operations/local_ops.py b/testgres/operations/local_ops.py index b9fd7aef..d33e8b65 100644 --- a/testgres/operations/local_ops.py +++ b/testgres/operations/local_ops.py @@ -250,6 +250,10 @@ def makedirs(self, path, remove_existing=False): except FileExistsError: pass + def makedir(self, path: str): + assert type(path) == str # noqa: E721 + os.mkdir(path) + # [2025-02-03] Old name of parameter attempts is "retries". def rmdirs(self, path, ignore_errors=True, attempts=3, delay=1): """ @@ -293,6 +297,10 @@ def rmdirs(self, path, ignore_errors=True, attempts=3, delay=1): # OK! return True + def rmdir(self, path: str): + assert type(path) == str # noqa: E721 + os.rmdir(path) + def listdir(self, path): return os.listdir(path) diff --git a/testgres/operations/os_ops.py b/testgres/operations/os_ops.py index d25e76bc..a4e1d9a2 100644 --- a/testgres/operations/os_ops.py +++ b/testgres/operations/os_ops.py @@ -53,9 +53,17 @@ def get_name(self): def makedirs(self, path, remove_existing=False): raise NotImplementedError() + def makedir(self, path: str): + assert type(path) == str # noqa: E721 + raise NotImplementedError() + def rmdirs(self, path, ignore_errors=True): raise NotImplementedError() + def rmdir(self, path: str): + assert type(path) == str # noqa: E721 + raise NotImplementedError() + def listdir(self, path): raise NotImplementedError() diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index e722a2cb..09406f79 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -225,6 +225,11 @@ def makedirs(self, path, remove_existing=False): raise Exception("Couldn't create dir {} because of error {}".format(path, error)) return result + def makedir(self, path: str): + assert type(path) == str # noqa: E721 + cmd = ["mkdir", path] + self.exec_command(cmd) + def rmdirs(self, path, ignore_errors=True): """ Remove a directory in the remote server. @@ -265,6 +270,11 @@ def rmdirs(self, path, ignore_errors=True): return False return True + def rmdir(self, path: str): + assert type(path) == str # noqa: E721 + cmd = ["rmdir", path] + self.exec_command(cmd) + def listdir(self, path): """ List all files and directories in a directory. diff --git a/tests/test_os_ops_common.py b/tests/test_os_ops_common.py index 9c4d8857..149050f9 100644 --- a/tests/test_os_ops_common.py +++ b/tests/test_os_ops_common.py @@ -13,10 +13,14 @@ import socket import threading import typing +import uuid from testgres import InvalidOperationException from testgres import ExecUtilException +from concurrent.futures import ThreadPoolExecutor +from concurrent.futures import Future as ThreadFuture + class TestOsOpsCommon: sm_os_ops_descrs: typing.List[OsOpsDescr] = [ @@ -812,3 +816,267 @@ def LOCAL_server(s: socket.socket): if ok_count == 0: raise RuntimeError("No one free port was found.") + + class tagData_OS_OPS__NUMS: + os_ops_descr: OsOpsDescr + nums: int + + def __init__(self, os_ops_descr: OsOpsDescr, nums: int): + assert isinstance(os_ops_descr, OsOpsDescr) + assert type(nums) == int # noqa: E721 + + self.os_ops_descr = os_ops_descr + self.nums = nums + + sm_test_exclusive_creation__mt__data = [ + tagData_OS_OPS__NUMS(OsOpsDescrs.sm_local_os_ops_descr, 100000), + tagData_OS_OPS__NUMS(OsOpsDescrs.sm_remote_os_ops_descr, 120), + ] + + @pytest.fixture( + params=sm_test_exclusive_creation__mt__data, + ids=[x.os_ops_descr.sign for x in sm_test_exclusive_creation__mt__data] + ) + def data001(self, request: pytest.FixtureRequest) -> tagData_OS_OPS__NUMS: + assert isinstance(request, pytest.FixtureRequest) + return request.param + + def test_mkdir__mt(self, data001: tagData_OS_OPS__NUMS): + assert type(data001) == __class__.tagData_OS_OPS__NUMS # noqa: E721 + + N_WORKERS = 4 + N_NUMBERS = data001.nums + assert type(N_NUMBERS) == int # noqa: E721 + + os_ops = data001.os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + lock_dir_prefix = "test_mkdir_mt--" + uuid.uuid4().hex + + lock_dir = os_ops.mkdtemp(prefix=lock_dir_prefix) + + logging.info("A lock file [{}] is creating ...".format(lock_dir)) + + assert os.path.exists(lock_dir) + + def MAKE_PATH(lock_dir: str, num: int) -> str: + assert type(lock_dir) == str # noqa: E721 + assert type(num) == int # noqa: E721 + return os.path.join(lock_dir, str(num) + ".lock") + + def LOCAL_WORKER(os_ops: OsOperations, + workerID: int, + lock_dir: str, + cNumbers: int, + reservedNumbers: typing.Set[int]) -> None: + assert isinstance(os_ops, OsOperations) + assert type(workerID) == int # noqa: E721 + assert type(lock_dir) == str # noqa: E721 + assert type(cNumbers) == int # noqa: E721 + assert type(reservedNumbers) == set # noqa: E721 + assert cNumbers > 0 + assert len(reservedNumbers) == 0 + + assert os.path.exists(lock_dir) + + def LOG_INFO(template: str, *args: list) -> None: + assert type(template) == str # noqa: E721 + assert type(args) == tuple # noqa: E721 + + msg = template.format(*args) + assert type(msg) == str # noqa: E721 + + logging.info("[Worker #{}] {}".format(workerID, msg)) + return + + LOG_INFO("HELLO! I am here!") + + for num in range(cNumbers): + assert not (num in reservedNumbers) + + file_path = MAKE_PATH(lock_dir, num) + + try: + os_ops.makedir(file_path) + except Exception as e: + LOG_INFO( + "Can't reserve {}. Error ({}): {}", + num, + type(e).__name__, + str(e) + ) + continue + + LOG_INFO("Number {} is reserved!", num) + assert os_ops.path_exists(file_path) + reservedNumbers.add(num) + continue + + n_total = cNumbers + n_ok = len(reservedNumbers) + assert n_ok <= n_total + + LOG_INFO("Finish! OK: {}. FAILED: {}.", n_ok, n_total - n_ok) + return + + # ----------------------- + logging.info("Worker are creating ...") + + threadPool = ThreadPoolExecutor( + max_workers=N_WORKERS, + thread_name_prefix="ex_creator" + ) + + class tadWorkerData: + future: ThreadFuture + reservedNumbers: typing.Set[int] + + workerDatas: typing.List[tadWorkerData] = list() + + nErrors = 0 + + try: + for n in range(N_WORKERS): + logging.info("worker #{} is creating ...".format(n)) + + workerDatas.append(tadWorkerData()) + + workerDatas[n].reservedNumbers = set() + + workerDatas[n].future = threadPool.submit( + LOCAL_WORKER, + os_ops, + n, + lock_dir, + N_NUMBERS, + workerDatas[n].reservedNumbers + ) + + assert workerDatas[n].future is not None + + logging.info("OK. All the workers were created!") + except Exception as e: + nErrors += 1 + logging.error("A problem is detected ({}): {}".format(type(e).__name__, str(e))) + + logging.info("Will wait for stop of all the workers...") + + nWorkers = 0 + + assert type(workerDatas) == list # noqa: E721 + + for i in range(len(workerDatas)): + worker = workerDatas[i].future + + if worker is None: + continue + + nWorkers += 1 + + assert isinstance(worker, ThreadFuture) + + try: + logging.info("Wait for worker #{}".format(i)) + worker.result() + except Exception as e: + nErrors += 1 + logging.error("Worker #{} finished with error ({}): {}".format( + i, + type(e).__name__, + str(e), + )) + continue + + assert nWorkers == N_WORKERS + + if nErrors != 0: + raise RuntimeError("Some problems were detected. Please examine the log messages.") + + logging.info("OK. Let's check worker results!") + + reservedNumbers: typing.Dict[int, int] = dict() + + for i in range(N_WORKERS): + logging.info("Worker #{} is checked ...".format(i)) + + workerNumbers = workerDatas[i].reservedNumbers + assert type(workerNumbers) == set # noqa: E721 + + for n in workerNumbers: + if n < 0 or n >= N_NUMBERS: + nErrors += 1 + logging.error("Unexpected number {}".format(n)) + continue + + if n in reservedNumbers.keys(): + nErrors += 1 + logging.error("Number {} was already reserved by worker #{}".format( + n, + reservedNumbers[n] + )) + else: + reservedNumbers[n] = i + + file_path = MAKE_PATH(lock_dir, n) + if not os_ops.path_exists(file_path): + nErrors += 1 + logging.error("File {} is not found!".format(file_path)) + continue + + continue + + logging.info("OK. Let's check reservedNumbers!") + + for n in range(N_NUMBERS): + if not (n in reservedNumbers.keys()): + nErrors += 1 + logging.error("Number {} is not reserved!".format(n)) + continue + + file_path = MAKE_PATH(lock_dir, n) + if not os_ops.path_exists(file_path): + nErrors += 1 + logging.error("File {} is not found!".format(file_path)) + continue + + # OK! + continue + + logging.info("Verification is finished! Total error count is {}.".format(nErrors)) + + if nErrors == 0: + logging.info("Root lock-directory [{}] will be deleted.".format( + lock_dir + )) + + for n in range(N_NUMBERS): + file_path = MAKE_PATH(lock_dir, n) + try: + os_ops.rmdir(file_path) + except Exception as e: + nErrors += 1 + logging.error("Cannot delete directory [{}]. Error ({}): {}".format( + file_path, + type(e).__name__, + str(e) + )) + continue + + if os_ops.path_exists(file_path): + nErrors += 1 + logging.error("Directory {} is not deleted!".format(file_path)) + continue + + if nErrors == 0: + try: + os_ops.rmdir(lock_dir) + except Exception as e: + nErrors += 1 + logging.error("Cannot delete directory [{}]. Error ({}): {}".format( + lock_dir, + type(e).__name__, + str(e) + )) + + logging.info("Test is finished! Total error count is {}.".format(nErrors)) + return From edd64db5284a6b34e275e022a4cc673e1032a6ea Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Tue, 6 May 2025 15:17:15 +0300 Subject: [PATCH 211/216] OsOperations::get_tempdir() is added (#254) Signature: def get_tempdir(self) -> str --- testgres/operations/local_ops.py | 7 +++++++ testgres/operations/os_ops.py | 3 +++ testgres/operations/remote_ops.py | 28 ++++++++++++++++++++++++++ tests/test_os_ops_common.py | 33 +++++++++++++++++++++++++++++++ 4 files changed, 71 insertions(+) diff --git a/testgres/operations/local_ops.py b/testgres/operations/local_ops.py index d33e8b65..ccf1ab82 100644 --- a/testgres/operations/local_ops.py +++ b/testgres/operations/local_ops.py @@ -528,3 +528,10 @@ def is_port_free(self, number: int) -> bool: return True except OSError: return False + + def get_tempdir(self) -> str: + r = tempfile.gettempdir() + assert r is not None + assert type(r) == str # noqa: E721 + assert os.path.exists(r) + return r diff --git a/testgres/operations/os_ops.py b/testgres/operations/os_ops.py index a4e1d9a2..45e4f71c 100644 --- a/testgres/operations/os_ops.py +++ b/testgres/operations/os_ops.py @@ -130,3 +130,6 @@ def get_process_children(self, pid): def is_port_free(self, number: int): assert type(number) == int # noqa: E721 raise NotImplementedError() + + def get_tempdir(self) -> str: + raise NotImplementedError() diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py index 09406f79..a478b453 100644 --- a/testgres/operations/remote_ops.py +++ b/testgres/operations/remote_ops.py @@ -659,6 +659,34 @@ def is_port_free(self, number: int) -> bool: out=output ) + def get_tempdir(self) -> str: + command = ["mktemp", "-u", "-d"] + + exec_exitcode, exec_output, exec_error = self.exec_command( + command, + verbose=True, + encoding=get_default_encoding(), + ignore_errors=True + ) + + assert type(exec_exitcode) == int # noqa: E721 + assert type(exec_output) == str # noqa: E721 + assert type(exec_error) == str # noqa: E721 + + if exec_exitcode != 0: + RaiseError.CommandExecutionError( + cmd=command, + exit_code=exec_exitcode, + message="Could not detect a temporary directory.", + error=exec_error, + out=exec_output) + + temp_subdir = exec_output.strip() + assert type(temp_subdir) == str # noqa: E721 + temp_dir = os.path.dirname(temp_subdir) + assert type(temp_dir) == str # noqa: E721 + return temp_dir + @staticmethod def _is_port_free__process_0(error: str) -> bool: assert type(error) == str # noqa: E721 diff --git a/tests/test_os_ops_common.py b/tests/test_os_ops_common.py index 149050f9..5ae3a61f 100644 --- a/tests/test_os_ops_common.py +++ b/tests/test_os_ops_common.py @@ -817,6 +817,39 @@ def LOCAL_server(s: socket.socket): if ok_count == 0: raise RuntimeError("No one free port was found.") + def test_get_tmpdir(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + + dir = os_ops.get_tempdir() + assert type(dir) == str # noqa: E721 + assert os_ops.path_exists(dir) + assert os.path.exists(dir) + + file_path = os.path.join(dir, "testgres--" + uuid.uuid4().hex + ".tmp") + + os_ops.write(file_path, "1234", binary=False) + + assert os_ops.path_exists(file_path) + assert os.path.exists(file_path) + + d = os_ops.read(file_path, binary=False) + + assert d == "1234" + + os_ops.remove_file(file_path) + + assert not os_ops.path_exists(file_path) + assert not os.path.exists(file_path) + + def test_get_tmpdir__compare_with_py_info(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + + actual_dir = os_ops.get_tempdir() + assert actual_dir is not None + assert type(actual_dir) == str # noqa: E721 + expected_dir = str(tempfile.tempdir) + assert actual_dir == expected_dir + class tagData_OS_OPS__NUMS: os_ops_descr: OsOpsDescr nums: int From 3bb59c64b45826f2a50cfb7d423de1c86721946b Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Wed, 7 May 2025 22:10:08 +0300 Subject: [PATCH 212/216] [#235] test_pg_ctl_wait_option detects a port conflict (#257) This patch must fix a problem in test_pg_ctl_wait_option when his PostgreSQL instance conflicts with another one. For this, we added two new things: - PostgresNodeLogReader - PostgresNodeUtils PostgresNodeLogReader reads server logs. PostgresNodeUtils provides an utility to detect a port conflict. PostgresNode::start also uses these new classes. --- testgres/node.py | 208 +++++++++++++++++++++++++++------- tests/test_testgres_common.py | 37 +++++- 2 files changed, 200 insertions(+), 45 deletions(-) diff --git a/testgres/node.py b/testgres/node.py index 66783e08..9a2f4e77 100644 --- a/testgres/node.py +++ b/testgres/node.py @@ -784,28 +784,6 @@ def _collect_special_files(self): return result - def _collect_log_files(self): - # dictionary of log files + size in bytes - - files = [ - self.pg_log_file - ] # yapf: disable - - result = {} - - for f in files: - # skip missing files - if not self.os_ops.path_exists(f): - continue - - file_size = self.os_ops.get_file_size(f) - assert type(file_size) == int # noqa: E721 - assert file_size >= 0 - - result[f] = file_size - - return result - def init(self, initdb_params=None, cached=True, **kwargs): """ Perform initdb for this node. @@ -1062,22 +1040,6 @@ def slow_start(self, replica=False, dbname='template1', username=None, max_attem OperationalError}, max_attempts=max_attempts) - def _detect_port_conflict(self, log_files0, log_files1): - assert type(log_files0) == dict # noqa: E721 - assert type(log_files1) == dict # noqa: E721 - - for file in log_files1.keys(): - read_pos = 0 - - if file in log_files0.keys(): - read_pos = log_files0[file] # the previous size - - file_content = self.os_ops.read_binary(file, read_pos) - file_content_s = file_content.decode() - if 'Is another postmaster already running on port' in file_content_s: - return True - return False - def start(self, params=[], wait=True, exec_env=None): """ Starts the PostgreSQL node using pg_ctl if node has not been started. @@ -1137,8 +1099,7 @@ def LOCAL__raise_cannot_start_node__std(from_exception): assert isinstance(self._port_manager, PortManager) assert __class__._C_MAX_START_ATEMPTS > 1 - log_files0 = self._collect_log_files() - assert type(log_files0) == dict # noqa: E721 + log_reader = PostgresNodeLogReader(self, from_beginnig=False) nAttempt = 0 timeout = 1 @@ -1154,11 +1115,11 @@ def LOCAL__raise_cannot_start_node__std(from_exception): if nAttempt == __class__._C_MAX_START_ATEMPTS: LOCAL__raise_cannot_start_node(e, "Cannot start node after multiple attempts.") - log_files1 = self._collect_log_files() - if not self._detect_port_conflict(log_files0, log_files1): + is_it_port_conflict = PostgresNodeUtils.delect_port_conflict(log_reader) + + if not is_it_port_conflict: LOCAL__raise_cannot_start_node__std(e) - log_files0 = log_files1 logging.warning( "Detected a conflict with using the port {0}. Trying another port after a {1}-second sleep...".format(self._port, timeout) ) @@ -2192,6 +2153,167 @@ def _escape_config_value(value): return result +class PostgresNodeLogReader: + class LogInfo: + position: int + + def __init__(self, position: int): + self.position = position + + # -------------------------------------------------------------------- + class LogDataBlock: + _file_name: str + _position: int + _data: str + + def __init__( + self, + file_name: str, + position: int, + data: str + ): + assert type(file_name) == str # noqa: E721 + assert type(position) == int # noqa: E721 + assert type(data) == str # noqa: E721 + assert file_name != "" + assert position >= 0 + self._file_name = file_name + self._position = position + self._data = data + + @property + def file_name(self) -> str: + assert type(self._file_name) == str # noqa: E721 + assert self._file_name != "" + return self._file_name + + @property + def position(self) -> int: + assert type(self._position) == int # noqa: E721 + assert self._position >= 0 + return self._position + + @property + def data(self) -> str: + assert type(self._data) == str # noqa: E721 + return self._data + + # -------------------------------------------------------------------- + _node: PostgresNode + _logs: typing.Dict[str, LogInfo] + + # -------------------------------------------------------------------- + def __init__(self, node: PostgresNode, from_beginnig: bool): + assert node is not None + assert isinstance(node, PostgresNode) + assert type(from_beginnig) == bool # noqa: E721 + + self._node = node + + if from_beginnig: + self._logs = dict() + else: + self._logs = self._collect_logs() + + assert type(self._logs) == dict # noqa: E721 + return + + def read(self) -> typing.List[LogDataBlock]: + assert self._node is not None + assert isinstance(self._node, PostgresNode) + + cur_logs: typing.Dict[__class__.LogInfo] = self._collect_logs() + assert cur_logs is not None + assert type(cur_logs) == dict # noqa: E721 + + assert type(self._logs) == dict # noqa: E721 + + result = list() + + for file_name, cur_log_info in cur_logs.items(): + assert type(file_name) == str # noqa: E721 + assert type(cur_log_info) == __class__.LogInfo # noqa: E721 + + read_pos = 0 + + if file_name in self._logs.keys(): + prev_log_info = self._logs[file_name] + assert type(prev_log_info) == __class__.LogInfo # noqa: E721 + read_pos = prev_log_info.position # the previous size + + file_content_b = self._node.os_ops.read_binary(file_name, read_pos) + assert type(file_content_b) == bytes # noqa: E721 + + # + # A POTENTIAL PROBLEM: file_content_b may contain an incompleted UTF-8 symbol. + # + file_content_s = file_content_b.decode() + assert type(file_content_s) == str # noqa: E721 + + next_read_pos = read_pos + len(file_content_b) + + # It is a research/paranoja check. + # When we will process partial UTF-8 symbol, it must be adjusted. + assert cur_log_info.position <= next_read_pos + + cur_log_info.position = next_read_pos + + block = __class__.LogDataBlock( + file_name, + read_pos, + file_content_s + ) + + result.append(block) + + # A new check point + self._logs = cur_logs + + return result + + def _collect_logs(self) -> typing.Dict[LogInfo]: + assert self._node is not None + assert isinstance(self._node, PostgresNode) + + files = [ + self._node.pg_log_file + ] # yapf: disable + + result = dict() + + for f in files: + assert type(f) == str # noqa: E721 + + # skip missing files + if not self._node.os_ops.path_exists(f): + continue + + file_size = self._node.os_ops.get_file_size(f) + assert type(file_size) == int # noqa: E721 + assert file_size >= 0 + + result[f] = __class__.LogInfo(file_size) + + return result + + +class PostgresNodeUtils: + @staticmethod + def delect_port_conflict(log_reader: PostgresNodeLogReader) -> bool: + assert type(log_reader) == PostgresNodeLogReader # noqa: E721 + + blocks = log_reader.read() + assert type(blocks) == list # noqa: E721 + + for block in blocks: + assert type(block) == PostgresNodeLogReader.LogDataBlock # noqa: E721 + + if 'Is another postmaster already running on port' in block.data: + return True + + return False + + class NodeApp: def __init__(self, test_path=None, nodes_to_cleanup=None, os_ops=None): diff --git a/tests/test_testgres_common.py b/tests/test_testgres_common.py index 5b926bc8..cf203a67 100644 --- a/tests/test_testgres_common.py +++ b/tests/test_testgres_common.py @@ -5,6 +5,8 @@ from testgres.node import PgVer from testgres.node import PostgresNode +from testgres.node import PostgresNodeLogReader +from testgres.node import PostgresNodeUtils from testgres.utils import get_pg_version2 from testgres.utils import file_tail from testgres.utils import get_bin_path2 @@ -883,8 +885,29 @@ def test_backup_wrong_xlog_method(self, node_svc: PostgresNodeService): def test_pg_ctl_wait_option(self, node_svc: PostgresNodeService): assert isinstance(node_svc, PostgresNodeService) - with __class__.helper__get_node(node_svc) as node: - self.impl__test_pg_ctl_wait_option(node_svc, node) + + C_MAX_ATTEMPT = 5 + + nAttempt = 0 + + while True: + if nAttempt == C_MAX_ATTEMPT: + raise Exception("PostgresSQL did not start.") + + nAttempt += 1 + logging.info("------------------------ NODE #{}".format( + nAttempt + )) + + with __class__.helper__get_node(node_svc, port=12345) as node: + if self.impl__test_pg_ctl_wait_option(node_svc, node): + break + continue + + logging.info("OK. Test is passed. Number of attempts is {}".format( + nAttempt + )) + return def impl__test_pg_ctl_wait_option( self, @@ -899,9 +922,18 @@ def impl__test_pg_ctl_wait_option( node.init() assert node.status() == NodeStatus.Stopped + + node_log_reader = PostgresNodeLogReader(node, from_beginnig=True) + node.start(wait=False) nAttempt = 0 while True: + if PostgresNodeUtils.delect_port_conflict(node_log_reader): + logging.info("Node port {} conflicted with another PostgreSQL instance.".format( + node.port + )) + return False + if nAttempt == C_MAX_ATTEMPTS: # # [2025-03-11] @@ -960,6 +992,7 @@ def impl__test_pg_ctl_wait_option( raise Exception("Unexpected node status: {0}.".format(s1)) logging.info("OK. Node is stopped.") + return True def test_replicate(self, node_svc: PostgresNodeService): assert isinstance(node_svc, PostgresNodeService) From 4c6bb1714b4102504a86cb200c9dd04d7036acaa Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Wed, 7 May 2025 22:16:04 +0300 Subject: [PATCH 213/216] Update README.md The version of supported Python is corrected. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a3b854f8..defbc8b3 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ # testgres -PostgreSQL testing utility. Both Python 2.7 and 3.3+ are supported. +PostgreSQL testing utility. Python 3.8+ is supported. ## Installation From 354de695b13be37ad481f92a084df9809704eea1 Mon Sep 17 00:00:00 2001 From: "d.kovalenko" Date: Wed, 7 May 2025 22:22:54 +0300 Subject: [PATCH 214/216] [CI] Python 2 is not supported any more. --- Dockerfile--std-all.tmpl | 5 ----- Dockerfile--std.tmpl | 5 ----- 2 files changed, 10 deletions(-) diff --git a/Dockerfile--std-all.tmpl b/Dockerfile--std-all.tmpl index c41c5a06..d19f52a6 100644 --- a/Dockerfile--std-all.tmpl +++ b/Dockerfile--std-all.tmpl @@ -4,11 +4,6 @@ ARG PYTHON_VERSION # --------------------------------------------- base1 FROM postgres:${PG_VERSION}-alpine as base1 -# --------------------------------------------- base2_with_python-2 -FROM base1 as base2_with_python-2 -RUN apk add --no-cache curl python2 python2-dev build-base musl-dev linux-headers py-virtualenv py-pip -ENV PYTHON_VERSION=2 - # --------------------------------------------- base2_with_python-3 FROM base1 as base2_with_python-3 RUN apk add --no-cache curl python3 python3-dev build-base musl-dev linux-headers py-virtualenv diff --git a/Dockerfile--std.tmpl b/Dockerfile--std.tmpl index 91886ede..67aa30b4 100644 --- a/Dockerfile--std.tmpl +++ b/Dockerfile--std.tmpl @@ -4,11 +4,6 @@ ARG PYTHON_VERSION # --------------------------------------------- base1 FROM postgres:${PG_VERSION}-alpine as base1 -# --------------------------------------------- base2_with_python-2 -FROM base1 as base2_with_python-2 -RUN apk add --no-cache curl python2 python2-dev build-base musl-dev linux-headers py-virtualenv py-pip -ENV PYTHON_VERSION=2 - # --------------------------------------------- base2_with_python-3 FROM base1 as base2_with_python-3 RUN apk add --no-cache curl python3 python3-dev build-base musl-dev linux-headers py-virtualenv From d48477d5e32ab0870229c5f676172111fd6ba3ee Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Mon, 12 May 2025 07:18:23 +0300 Subject: [PATCH 215/216] [#258] Problems in ProbackupTest and TestBasic(ProbackupTest) are fixed (#259) * [#258] Declaration of ProbackupTest::pg_node is corrected ProbackupTest::pg_node is testgres.NodeApp, not testgres.PostgresNode. Asserts are added. * [#258] TestBasic::test_full_backup cleans node object Node cleanup is added. TODO: we should to stop node only and cleanup his data in conftest. --- .../pg_probackup2/tests/test_basic.py | 35 ++++++++++++------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/tests/test_basic.py b/testgres/plugins/pg_probackup2/pg_probackup2/tests/test_basic.py index f22a62bf..2540ddb0 100644 --- a/testgres/plugins/pg_probackup2/pg_probackup2/tests/test_basic.py +++ b/testgres/plugins/pg_probackup2/pg_probackup2/tests/test_basic.py @@ -11,7 +11,7 @@ class ProbackupTest: - pg_node: testgres.PostgresNode + pg_node: testgres.NodeApp @staticmethod def probackup_is_available() -> bool: @@ -75,21 +75,30 @@ def helper__build_backup_dir(self, backup='backup'): @pytest.mark.skipif(not ProbackupTest.probackup_is_available(), reason="Check that PGPROBACKUPBIN is defined and is valid.") class TestBasic(ProbackupTest): def test_full_backup(self): + assert self.pg_node is not None + assert type(self.pg_node) == testgres.NodeApp # noqa: E721 + assert self.pb is not None + assert type(self.pb) == ProbackupApp # noqa: E721 + # Setting up a simple test node node = self.pg_node.make_simple('node', pg_options={"fsync": "off", "synchronous_commit": "off"}) - # Initialize and configure Probackup - self.pb.init() - self.pb.add_instance('node', node) - self.pb.set_archiving('node', node) + assert node is not None + assert type(node) == testgres.PostgresNode # noqa: E721 + + with node: + # Initialize and configure Probackup + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) - # Start the node and initialize pgbench - node.slow_start() - node.pgbench_init(scale=100, no_vacuum=True) + # Start the node and initialize pgbench + node.slow_start() + node.pgbench_init(scale=100, no_vacuum=True) - # Perform backup and validation - backup_id = self.pb.backup_node('node', node) - out = self.pb.validate('node', backup_id) + # Perform backup and validation + backup_id = self.pb.backup_node('node', node) + out = self.pb.validate('node', backup_id) - # Check if the backup is valid - assert f"INFO: Backup {backup_id} is valid" in out + # Check if the backup is valid + assert f"INFO: Backup {backup_id} is valid" in out From 0470d305af4af16edd8ffa56d05a5b90cad1e128 Mon Sep 17 00:00:00 2001 From: Dmitry Kovalenko Date: Mon, 12 May 2025 18:29:10 +0300 Subject: [PATCH 216/216] conftest is updated (#260) - the calls of logging.root.handle are handled (LogWrapper2) - critical errors are processed --- tests/conftest.py | 200 +++++++++++++++++++++++----------------------- 1 file changed, 100 insertions(+), 100 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 9e74879b..111edc87 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -338,6 +338,7 @@ def helper__build_test_id(item: pytest.Function) -> str: g_error_msg_count_key = pytest.StashKey[int]() g_warning_msg_count_key = pytest.StashKey[int]() +g_critical_msg_count_key = pytest.StashKey[int]() # ///////////////////////////////////////////////////////////////////////////// @@ -413,10 +414,17 @@ def helper__makereport__call( assert type(outcome) == pluggy.Result # noqa: E721 # -------- - item_error_msg_count = item.stash.get(g_error_msg_count_key, 0) - assert type(item_error_msg_count) == int # noqa: E721 - assert item_error_msg_count >= 0 + item_error_msg_count1 = item.stash.get(g_error_msg_count_key, 0) + assert type(item_error_msg_count1) == int # noqa: E721 + assert item_error_msg_count1 >= 0 + item_error_msg_count2 = item.stash.get(g_critical_msg_count_key, 0) + assert type(item_error_msg_count2) == int # noqa: E721 + assert item_error_msg_count2 >= 0 + + item_error_msg_count = item_error_msg_count1 + item_error_msg_count2 + + # -------- item_warning_msg_count = item.stash.get(g_warning_msg_count_key, 0) assert type(item_warning_msg_count) == int # noqa: E721 assert item_warning_msg_count >= 0 @@ -600,103 +608,87 @@ def pytest_runtest_makereport(item: pytest.Function, call: pytest.CallInfo): # ///////////////////////////////////////////////////////////////////////////// -class LogErrorWrapper2: +class LogWrapper2: _old_method: any - _counter: typing.Optional[int] + _err_counter: typing.Optional[int] + _warn_counter: typing.Optional[int] + + _critical_counter: typing.Optional[int] # -------------------------------------------------------------------- def __init__(self): self._old_method = None - self._counter = None + self._err_counter = None + self._warn_counter = None + + self._critical_counter = None # -------------------------------------------------------------------- def __enter__(self): assert self._old_method is None - assert self._counter is None - - self._old_method = logging.error - self._counter = 0 - - logging.error = self - return self - - # -------------------------------------------------------------------- - def __exit__(self, exc_type, exc_val, exc_tb): - assert self._old_method is not None - assert self._counter is not None - - assert logging.error is self - - logging.error = self._old_method - - self._old_method = None - self._counter = None - return False - - # -------------------------------------------------------------------- - def __call__(self, *args, **kwargs): - assert self._old_method is not None - assert self._counter is not None - - assert type(self._counter) == int # noqa: E721 - assert self._counter >= 0 - - r = self._old_method(*args, **kwargs) - - self._counter += 1 - assert self._counter > 0 - - return r - - -# ///////////////////////////////////////////////////////////////////////////// + assert self._err_counter is None + assert self._warn_counter is None + assert self._critical_counter is None -class LogWarningWrapper2: - _old_method: any - _counter: typing.Optional[int] + assert logging.root is not None + assert isinstance(logging.root, logging.RootLogger) - # -------------------------------------------------------------------- - def __init__(self): - self._old_method = None - self._counter = None + self._old_method = logging.root.handle + self._err_counter = 0 + self._warn_counter = 0 - # -------------------------------------------------------------------- - def __enter__(self): - assert self._old_method is None - assert self._counter is None + self._critical_counter = 0 - self._old_method = logging.warning - self._counter = 0 - - logging.warning = self + logging.root.handle = self return self # -------------------------------------------------------------------- def __exit__(self, exc_type, exc_val, exc_tb): assert self._old_method is not None - assert self._counter is not None + assert self._err_counter is not None + assert self._warn_counter is not None + + assert logging.root is not None + assert isinstance(logging.root, logging.RootLogger) - assert logging.warning is self + assert logging.root.handle is self - logging.warning = self._old_method + logging.root.handle = self._old_method self._old_method = None - self._counter = None + self._err_counter = None + self._warn_counter = None + self._critical_counter = None return False # -------------------------------------------------------------------- - def __call__(self, *args, **kwargs): + def __call__(self, record: logging.LogRecord): + assert record is not None + assert isinstance(record, logging.LogRecord) assert self._old_method is not None - assert self._counter is not None - - assert type(self._counter) == int # noqa: E721 - assert self._counter >= 0 - - r = self._old_method(*args, **kwargs) - - self._counter += 1 - assert self._counter > 0 + assert self._err_counter is not None + assert self._warn_counter is not None + assert self._critical_counter is not None + + assert type(self._err_counter) == int # noqa: E721 + assert self._err_counter >= 0 + assert type(self._warn_counter) == int # noqa: E721 + assert self._warn_counter >= 0 + assert type(self._critical_counter) == int # noqa: E721 + assert self._critical_counter >= 0 + + r = self._old_method(record) + + if record.levelno == logging.ERROR: + self._err_counter += 1 + assert self._err_counter > 0 + elif record.levelno == logging.WARNING: + self._warn_counter += 1 + assert self._warn_counter > 0 + elif record.levelno == logging.CRITICAL: + self._critical_counter += 1 + assert self._critical_counter > 0 return r @@ -717,6 +709,13 @@ def pytest_pyfunc_call(pyfuncitem: pytest.Function): assert pyfuncitem is not None assert isinstance(pyfuncitem, pytest.Function) + assert logging.root is not None + assert isinstance(logging.root, logging.RootLogger) + assert logging.root.handle is not None + + debug__log_handle_method = logging.root.handle + assert debug__log_handle_method is not None + debug__log_error_method = logging.error assert debug__log_error_method is not None @@ -725,55 +724,56 @@ def pytest_pyfunc_call(pyfuncitem: pytest.Function): pyfuncitem.stash[g_error_msg_count_key] = 0 pyfuncitem.stash[g_warning_msg_count_key] = 0 + pyfuncitem.stash[g_critical_msg_count_key] = 0 try: - with LogErrorWrapper2() as logErrorWrapper, LogWarningWrapper2() as logWarningWrapper: - assert type(logErrorWrapper) == LogErrorWrapper2 # noqa: E721 - assert logErrorWrapper._old_method is not None - assert type(logErrorWrapper._counter) == int # noqa: E721 - assert logErrorWrapper._counter == 0 - assert logging.error is logErrorWrapper - - assert type(logWarningWrapper) == LogWarningWrapper2 # noqa: E721 - assert logWarningWrapper._old_method is not None - assert type(logWarningWrapper._counter) == int # noqa: E721 - assert logWarningWrapper._counter == 0 - assert logging.warning is logWarningWrapper + with LogWrapper2() as logWrapper: + assert type(logWrapper) == LogWrapper2 # noqa: E721 + assert logWrapper._old_method is not None + assert type(logWrapper._err_counter) == int # noqa: E721 + assert logWrapper._err_counter == 0 + assert type(logWrapper._warn_counter) == int # noqa: E721 + assert logWrapper._warn_counter == 0 + assert type(logWrapper._critical_counter) == int # noqa: E721 + assert logWrapper._critical_counter == 0 + assert logging.root.handle is logWrapper r: pluggy.Result = yield assert r is not None assert type(r) == pluggy.Result # noqa: E721 - assert logErrorWrapper._old_method is not None - assert type(logErrorWrapper._counter) == int # noqa: E721 - assert logErrorWrapper._counter >= 0 - assert logging.error is logErrorWrapper - - assert logWarningWrapper._old_method is not None - assert type(logWarningWrapper._counter) == int # noqa: E721 - assert logWarningWrapper._counter >= 0 - assert logging.warning is logWarningWrapper + assert logWrapper._old_method is not None + assert type(logWrapper._err_counter) == int # noqa: E721 + assert logWrapper._err_counter >= 0 + assert type(logWrapper._warn_counter) == int # noqa: E721 + assert logWrapper._warn_counter >= 0 + assert type(logWrapper._critical_counter) == int # noqa: E721 + assert logWrapper._critical_counter >= 0 + assert logging.root.handle is logWrapper assert g_error_msg_count_key in pyfuncitem.stash assert g_warning_msg_count_key in pyfuncitem.stash + assert g_critical_msg_count_key in pyfuncitem.stash assert pyfuncitem.stash[g_error_msg_count_key] == 0 assert pyfuncitem.stash[g_warning_msg_count_key] == 0 + assert pyfuncitem.stash[g_critical_msg_count_key] == 0 - pyfuncitem.stash[g_error_msg_count_key] = logErrorWrapper._counter - pyfuncitem.stash[g_warning_msg_count_key] = logWarningWrapper._counter + pyfuncitem.stash[g_error_msg_count_key] = logWrapper._err_counter + pyfuncitem.stash[g_warning_msg_count_key] = logWrapper._warn_counter + pyfuncitem.stash[g_critical_msg_count_key] = logWrapper._critical_counter if r.exception is not None: pass - elif logErrorWrapper._counter == 0: - pass - else: - assert logErrorWrapper._counter > 0 + elif logWrapper._err_counter > 0: + r.force_exception(SIGNAL_EXCEPTION()) + elif logWrapper._critical_counter > 0: r.force_exception(SIGNAL_EXCEPTION()) finally: assert logging.error is debug__log_error_method assert logging.warning is debug__log_warning_method + assert logging.root.handle == debug__log_handle_method pass