[#329] get_pg_node_state looks for postmaster to avoid "PID file is empty" (#339)

* os_ops v2.1.0 us used

* [#329] get_pg_node_state looks for postmaster to avoid "PID file is empty" error

This patch suggests a solution to fix an error "PID file is empty" during getting a server state data.

When pg_ctl returns this error, we are trying to find postmaster process for our data directory.

If we find it then we wait a few seconds and do the next attempt to get server state data.

Closes #329.

* run_tests.sh is updated (-vvv)

* linux.InternalPlatformUtils is refactored (normalization)

Author: dmitry-lipetsk

pyproject.toml

@@ -62,7 +62,7 @@ dependencies = [
     "six>=1.9.0",
     "psutil",
     "packaging",
-    "testgres.os_ops>=2.0.2,<3.0.0",
+    "testgres.os_ops>=2.1.0,<3.0.0",
 ]
 
 [project.urls]

run_tests.sh

@@ -26,21 +26,21 @@ rm -f $COVERAGE_FILE
 pip install coverage
 
 # run tests (PATH)
-time coverage run -a -m pytest -l -v -n 4 -k "${TEST_FILTER}"
+time coverage run -a -m pytest -l -vvv -n 4 -k "${TEST_FILTER}"
 
 # run tests (PG_BIN)
 PG_BIN=$(pg_config --bindir) \
-time coverage run -a -m pytest -l -v -n 4 -k "${TEST_FILTER}"
+time coverage run -a -m pytest -l -vvv -n 4 -k "${TEST_FILTER}"
 
 # run tests (PG_CONFIG)
 PG_CONFIG=$(pg_config --bindir)/pg_config \
-time coverage run -a -m pytest -l -v -n 4 -k "${TEST_FILTER}"
+time coverage run -a -m pytest -l -vvv -n 4 -k "${TEST_FILTER}"
 
 # test pg8000
 pip uninstall -y psycopg2
 pip install pg8000
 PG_CONFIG=$(pg_config --bindir)/pg_config \
-time coverage run -a -m pytest -l -v -n 4 -k "${TEST_FILTER}"
+time coverage run -a -m pytest -l -vvv -n 4 -k "${TEST_FILTER}"
 
 # show coverage
 coverage report

src/impl/internal_utils.py

@@ -0,0 +1,20 @@
+import logging
+
+
+def send_log(level: int, msg: str) -> None:
+    assert type(level) == int  # noqa: E721
+    assert type(msg) == str  # noqa: E721
+
+    return logging.log(level, "[testgres] " + msg)
+
+
+def send_log_info(msg: str) -> None:
+    assert type(msg) == str  # noqa: E721
+
+    return send_log(logging.INFO, msg)
+
+
+def send_log_debug(msg: str) -> None:
+    assert type(msg) == str  # noqa: E721
+
+    return send_log(logging.DEBUG, msg)

src/impl/platforms/internal_platform_utils.py

@@ -0,0 +1,62 @@
+from __future__ import annotations
+
+import enum
+import typing
+
+from testgres.operations.os_ops import OsOperations
+
+
+class InternalPlatformUtils:
+    class FindPostmasterResultCode(enum.Enum):
+        ok = 0
+        not_found = 1,
+        not_implemented = 2
+        many_processes = 3
+        has_problems = 4
+
+    class FindPostmasterResult:
+        code: InternalPlatformUtils.FindPostmasterResultCode
+        pid: typing.Optional[int]
+
+        def __init__(
+            self,
+            code: InternalPlatformUtils.FindPostmasterResultCode,
+            pid: typing.Optional[int]
+        ):
+            assert type(code) == InternalPlatformUtils.FindPostmasterResultCode  # noqa: E721
+            assert pid is None or type(pid) == int  # noqa: E721
+            self.code = code
+            self.pid = pid
+            return
+
+        @staticmethod
+        def create_ok(pid: int) -> InternalPlatformUtils.FindPostmasterResult:
+            assert type(pid) == int  # noqa: E721
+            return __class__(InternalPlatformUtils.FindPostmasterResultCode.ok, pid)
+
+        @staticmethod
+        def create_not_found() -> InternalPlatformUtils.FindPostmasterResult:
+            return __class__(InternalPlatformUtils.FindPostmasterResultCode.not_found, None)
+
+        @staticmethod
+        def create_not_implemented() -> InternalPlatformUtils.FindPostmasterResult:
+            return __class__(InternalPlatformUtils.FindPostmasterResultCode.not_implemented, None)
+
+        @staticmethod
+        def create_many_processes() -> InternalPlatformUtils.FindPostmasterResult:
+            return __class__(InternalPlatformUtils.FindPostmasterResultCode.many_processes, None)
+
+        @staticmethod
+        def create_has_problems() -> InternalPlatformUtils.FindPostmasterResult:
+            return __class__(InternalPlatformUtils.FindPostmasterResultCode.has_problems, None)
+
+    def FindPostmaster(
+        self,
+        os_ops: OsOperations,
+        bin_dir: str,
+        data_dir: str
+    ) -> FindPostmasterResult:
+        assert isinstance(os_ops, OsOperations)
+        assert type(bin_dir) == str  # noqa: E721
+        assert type(data_dir) == str  # noqa: E721
+        raise NotImplementedError("InternalPlatformUtils::FindPostmaster is not implemented.")

src/impl/platforms/internal_platform_utils_factory.py

@@ -0,0 +1,23 @@
+from .internal_platform_utils import InternalPlatformUtils
+
+from testgres.operations.os_ops import OsOperations
+
+
+def create_internal_platform_utils(
+    os_ops: OsOperations
+) -> InternalPlatformUtils:
+    assert isinstance(os_ops, OsOperations)
+
+    platform_name = os_ops.get_platform()
+    assert type(platform_name) == str  # noqa: E721
+
+    if platform_name == "linux":
+        from .linux import internal_platform_utils as x
+        return x.InternalPlatformUtils()
+
+    if platform_name == "win32":
+        from .win32 import internal_platform_utils as x
+        return x.InternalPlatformUtils()
+
+    # not implemented
+    return InternalPlatformUtils()

src/impl/platforms/linux/internal_platform_utils.py

@@ -0,0 +1,120 @@
+from __future__ import annotations
+
+from .. import internal_platform_utils as base
+from ... import internal_utils
+
+from testgres.operations.os_ops import OsOperations
+from testgres.operations.exceptions import ExecUtilException
+
+import re
+import shlex
+
+
+class InternalPlatformUtils(base.InternalPlatformUtils):
+    C_BASH_EXE = "/bin/bash"
+
+    sm_exec_env = {
+        "LANG": "en_US.UTF-8",
+        "LC_ALL": "en_US.UTF-8",
+    }
+
+    # --------------------------------------------------------------------
+    def FindPostmaster(
+        self,
+        os_ops: OsOperations,
+        bin_dir: str,
+        data_dir: str
+    ) -> InternalPlatformUtils.FindPostmasterResult:
+        assert isinstance(os_ops, OsOperations)
+        assert type(bin_dir) == str  # noqa: E721
+        assert type(data_dir) == str  # noqa: E721
+        assert type(__class__.C_BASH_EXE) == str  # noqa: E721
+        assert type(__class__.sm_exec_env) == dict  # noqa: E721
+        assert len(__class__.C_BASH_EXE) > 0
+        assert len(bin_dir) > 0
+        assert len(data_dir) > 0
+
+        pg_path_e = re.escape(os_ops.build_path(bin_dir, "postgres"))
+        data_dir_e = re.escape(data_dir)
+
+        assert type(pg_path_e) == str  # noqa: E721
+        assert type(data_dir_e) == str  # noqa: E721
+
+        regexp = r"^\s*[0-9]+\s+" + pg_path_e + r"(\s+.*)?\s+\-[D]\s+" + data_dir_e + r"(\s+.*)?"
+
+        cmd = [
+            __class__.C_BASH_EXE,
+            "-c",
+            "ps -ewwo \"pid=,args=\" | grep -E " + shlex.quote(regexp),
+        ]
+
+        exit_status, output_b, error_b = os_ops.exec_command(
+            cmd=cmd,
+            ignore_errors=True,
+            verbose=True,
+            exec_env=__class__.sm_exec_env,
+        )
+
+        assert type(output_b) == bytes  # noqa: E721
+        assert type(error_b) == bytes  # noqa: E721
+
+        output = output_b.decode("utf-8")
+        error = error_b.decode("utf-8")
+
+        assert type(output) == str  # noqa: E721
+        assert type(error) == str  # noqa: E721
+
+        if exit_status == 1:
+            return __class__.FindPostmasterResult.create_not_found()
+
+        if exit_status != 0:
+            errMsg = f"test command returned an unexpected exit code: {exit_status}"
+            raise ExecUtilException(
+                message=errMsg,
+                command=cmd,
+                exit_code=exit_status,
+                out=output,
+                error=error,
+            )
+
+        lines = output.splitlines()
+        assert type(lines) == list  # noqa: E721
+
+        if len(lines) == 0:
+            return __class__.FindPostmasterResult.create_not_found()
+
+        if len(lines) > 1:
+            msgs = []
+            msgs.append("Many processes like a postmaster are found: {}.".format(len(lines)))
+
+            for i in range(len(lines)):
+                assert type(lines[i]) == str  # noqa: E721
+                lines.append("[{}] '{}'".format(i, lines[i]))
+                continue
+
+            internal_utils.send_log_debug("\n".join(lines))
+            return __class__.FindPostmasterResult.create_many_processes()
+
+        def is_space_or_tab(ch) -> bool:
+            assert type(ch) == str  # noqa: E721
+            return ch == " " or ch == "\t"
+
+        line = lines[0]
+        start = 0
+        while start < len(line) and is_space_or_tab(line[start]):
+            start += 1
+
+        pos = start
+        while pos < len(line) and line[pos].isnumeric():
+            pos += 1
+
+        if pos == start:
+            return __class__.FindPostmasterResult.create_has_problems()
+
+        if pos != len(line) and not line[pos].isspace():
+            return __class__.FindPostmasterResult.create_has_problems()
+
+        pid = int(line[start:pos])
+        assert type(pid) == int  # noqa: E721
+
+        return __class__.FindPostmasterResult.create_ok(pid)

src/impl/platforms/win32/internal_platform_utils.py

@@ -0,0 +1,17 @@
+from __future__ import annotations
+
+from .. import internal_platform_utils as base
+from testgres.operations.os_ops import OsOperations
+
+
+class InternalPlatformUtils(base.InternalPlatformUtils):
+    def FindPostmaster(
+        self,
+        os_ops: OsOperations,
+        bin_dir: str,
+        data_dir: str
+    ) -> InternalPlatformUtils.FindPostmasterResult:
+        assert isinstance(os_ops, OsOperations)
+        assert type(bin_dir) == str  # noqa: E721
+        assert type(data_dir) == str  # noqa: E721
+        return __class__.FindPostmasterResult.create_not_implemented()