diff --git a/.dockerignore b/.dockerignore index ff03e5de..89ad10bd 100644 --- a/.dockerignore +++ b/.dockerignore @@ -2,3 +2,6 @@ dist env venv *.egg-info +logs +.vscode +.pytest_cache diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100755 index 00000000..0a241998 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,7 @@ +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + open-pull-requests-limit: 5 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..3c9295c1 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,290 @@ +# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python + +name: CI + +on: + push: + branches: [ "master" ] + paths-ignore: + - "**/*.md" + + pull_request: + branches: [ "master" ] + paths-ignore: + - "**/*.md" + +jobs: + lint: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13", "3.14"] + + steps: + - name: Checkout + uses: actions/checkout@v7 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v7 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install flake8 flake8-pyproject ruff==0.15.22 + - name: Lint with flake8 + run: | + flake8 . + - name: Lint with ruff + run: | + ruff check . + + build-check: + runs-on: ubuntu-latest + needs: lint + steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: Set up Python 3.12 + uses: actions/setup-python@v7 + with: + python-version: "3.12" + cache: 'pip' + + - name: Install build tools + run: | + python -m pip install --upgrade pip + python -m pip install build twine + + - name: Build package + run: | + python -m build + + - name: Check package metadata + run: | + twine check dist/* + + test: + runs-on: ubuntu-latest + needs: build-check + strategy: + fail-fast: false + matrix: + include: + - platform: "alpine" + python: "3.7" + postgres: "17" + case_suffix: "py3_07_xx-pg17_xx" + - platform: "alpine" + python: "3.8.0" + postgres: "17" + case_suffix: "py3_08_00-pg17_xx" + - platform: "alpine" + python: "3.8" + postgres: "17" + case_suffix: "py3_08_xx-pg17_xx" + - platform: "alpine" + python: "3.9" + postgres: "17" + case_suffix: "py3_09_xx-pg17_xx" + - platform: "alpine" + python: "3.10" + postgres: "17" + case_suffix: "py3_10_xx-pg17_xx" + - platform: "alpine" + python: "3.11" + postgres: "17" + case_suffix: "py3_11_xx-pg17_xx" + - platform: "alpine" + python: "3.12" + postgres: "17" + case_suffix: "py3_12_xx-pg17_xx" + - platform: "alpine" + python: "3.13" + postgres: "17" + case_suffix: "py3_13_xx-pg17_xx" + - platform: "alpine" + python: "3.14" + postgres: "17" + case_suffix: "py3_14_xx-pg17_xx" + - platform: "alpine" + python: "3.12" + postgres: "10" + case_suffix: "py3_12_xx-pg10_xx" + - platform: "alpine" + python: "3.12" + postgres: "11" + case_suffix: "py3_12_xx-pg11_xx" + - platform: "alpine" + python: "3.12" + postgres: "12" + case_suffix: "py3_12_xx-pg12_xx" + - platform: "alpine" + python: "3.12" + postgres: "13" + case_suffix: "py3_12_xx-pg13_xx" + - platform: "alpine" + python: "3.12" + postgres: "14" + case_suffix: "py3_12_xx-pg14_xx" + - platform: "alpine" + python: "3.12" + postgres: "15" + case_suffix: "py3_12_xx-pg15_xx" + - platform: "alpine" + python: "3.12" + postgres: "16" + case_suffix: "py3_12_xx-pg16_xx" + - platform: "alpine" + python: "3.12" + postgres: "18" + case_suffix: "py3_12_xx-pg18_xx" + - platform: "ubuntu_24_04" + python: "3" + postgres: "17" + case_suffix: "py3_xx_xx-pg17_xx" + - platform: "ubuntu_26_04" + python: "3.12" + postgres: "17" + case_suffix: "py3_12_xx-pg17_xx" + - platform: "altlinux_10" + python: "3" + postgres: "17" + case_suffix: "py3_xx_xx-pg17_xx" + - platform: "altlinux_11" + python: "3" + postgres: "17" + case_suffix: "py3_xx_xx-pg17_xx" + - platform: "astralinux_1_7" + python: "3" + postgres: "17" + case_suffix: "py3_xx_xx-pg17_xx" + - platform: "rockylinux_8" + python: "3" + postgres: "17" + case_suffix: "py3_xx_xx-pg17_xx" + - platform: "rockylinux_9" + python: "3" + postgres: "17" + case_suffix: "py3_xx_xx-pg17_xx" + - platform: "rockylinux_10" + python: "3" + postgres: "17" + case_suffix: "py3_xx_xx-pg17_xx" + + name: "test: ${{ matrix.platform }} | ${{ matrix.case_suffix }}" + + env: + BASE_SIGN: "${{ matrix.platform }}-${{ matrix.case_suffix }}" + + DOCKER_HIGHLOAD_FLAGS: >- + --init + --sysctl net.core.somaxconn=4096 + --sysctl net.ipv4.tcp_max_syn_backlog=4096 + --ulimit nofile=524288:524288 + --ulimit nproc=65535:65535 + --sysctl net.ipv4.tcp_tw_reuse=1 + --sysctl net.ipv4.ip_local_port_range="1024 65535" + + steps: + - name: Prepare variables + run: | + echo "RUN_CFG__NOW=$(date +'%Y%m%d_%H%M%S')" >> $GITHUB_ENV + echo "RUN_CFG__LOGS_DIR=logs-${{ env.BASE_SIGN }}" >> $GITHUB_ENV + echo "RUN_CFG__DOCKER_IMAGE_NAME=tests-${{ env.BASE_SIGN }}" >> $GITHUB_ENV + echo "RUN_CFG__NETWORK_NAME=network-${{ env.BASE_SIGN }}" >> $GITHUB_ENV + echo "RUN_CFG__MACHINE1_NAME=machine1-${{ env.BASE_SIGN }}" >> $GITHUB_ENV + echo "RUN_CFG__MACHINE2_NAME=machine2-${{ env.BASE_SIGN }}" >> $GITHUB_ENV + echo "RUN_CFG__SSH_KEY_NAME=id_ed25519-${{ env.BASE_SIGN }}" >> $GITHUB_ENV + echo "---------- [$GITHUB_ENV]" + cat $GITHUB_ENV + - name: Checkout + uses: actions/checkout@v7 + - name: Prepare logs folder on the host + run: mkdir -p "${{ env.RUN_CFG__LOGS_DIR }}" + - name: Adjust logs folder permission + run: chmod -R 777 "${{ env.RUN_CFG__LOGS_DIR }}" + - name: Build local image + run: docker build --build-arg PG_VERSION="${{ matrix.postgres }}" --build-arg PYTHON_VERSION="${{ matrix.python }}" -t "${{ env.RUN_CFG__DOCKER_IMAGE_NAME }}" -f Dockerfile--${{ matrix.platform }}.tmpl . + - name: Generate temporary SSH Key pair for CI + run: | + # Generate a key without a password directly on the GitHub Actions host + ssh-keygen -q -t ed25519 -N "" -f ${{ github.workspace }}/${{ env.RUN_CFG__SSH_KEY_NAME }} + + - name: Setup network + run: | + docker network create ${{ env.RUN_CFG__NETWORK_NAME }} + + - name: Run machine2 (for remote operations) + run: | + # + # "- sleep infinity" is a right command. Docker ignores "-" and runs "sleep infinity". + # + docker run -d -t \ + ${{ env.DOCKER_HIGHLOAD_FLAGS }} \ + --network ${{ env.RUN_CFG__NETWORK_NAME }} \ + --name ${{ env.RUN_CFG__MACHINE2_NAME }} \ + "${{ env.RUN_CFG__DOCKER_IMAGE_NAME }}" \ + sleep infinity + + echo "Waiting for container ${{ env.RUN_CFG__MACHINE2_NAME }} to start ..." + + # Set the timeout using the attempt counter + MAX_ATTEMPTS=50 + ATTEMPT=0 + + until [ "$(docker container inspect --format '{{.State.Running}}' "${{ env.RUN_CFG__MACHINE2_NAME }}")" = "true" ]; do + ATTEMPT=$((ATTEMPT + 1)) + if [ $ATTEMPT -ge $MAX_ATTEMPTS ]; then + echo "Error: Container did not start within the allotted time (timeout)!" + exit 1 + fi + sleep 0.2 + done + + echo "Container successfully launched in $((ATTEMPT * 2 / 10)) second(s)!" + + # + # Setup authorized_keys + # + cat ${{ github.workspace }}/${{ env.RUN_CFG__SSH_KEY_NAME }}.pub | \ + docker exec -i -u test ${{ env.RUN_CFG__MACHINE2_NAME }} sh -c " \ + cat >> /home/test/.ssh/authorized_keys && \ + chmod 600 /home/test/.ssh/authorized_keys \ + " + + # + # Log status + # + docker exec -u root ${{ env.RUN_CFG__MACHINE2_NAME }} sh -c " + echo '--- INSIDE CONTAINER OS ---' && \ + cat /etc/os-release && \ + echo '--- INSIDE CONTAINER .SSH ---' && \ + ls -la /home/test/.ssh/ \ + " + + - name: Run machine1 (main test container) + run: | + # Launch the main container, passing the private key to the .ssh folder of the test user. + # Pass environment variables so Python tests know where to start. + # Added the --rm flag so that the container is automatically deleted after the tests are completed + docker run --rm -t \ + ${{ env.DOCKER_HIGHLOAD_FLAGS }} \ + --network ${{ env.RUN_CFG__NETWORK_NAME }} \ + --name ${{ env.RUN_CFG__MACHINE1_NAME }} \ + -v ${{ github.workspace }}/${{ env.RUN_CFG__LOGS_DIR }}:/home/test/testgres/logs \ + -v ${{ github.workspace }}/${{ env.RUN_CFG__SSH_KEY_NAME }}:/home/test/testgres/id_ed25519_test:ro \ + -e TEST_CFG__REMOTE_HOST="${{ env.RUN_CFG__MACHINE2_NAME }}" \ + -e TEST_CFG__REMOTE_PORT="22" \ + -e TEST_CFG__REMOTE_USERNAME="test" \ + -e TEST_CFG__REMOTE_SSH_KEY="/home/test/testgres/id_ed25519_test" \ + "${{ env.RUN_CFG__DOCKER_IMAGE_NAME }}" + - name: Upload Logs + uses: actions/upload-artifact@v7 + if: always() # IT IS IMPORTANT! + with: + name: testgres--test_logs--${{ env.RUN_CFG__NOW }}-${{ env.BASE_SIGN }}-id${{ github.run_id }} + path: "${{ env.RUN_CFG__LOGS_DIR }}/" diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml new file mode 100644 index 00000000..b1782e54 --- /dev/null +++ b/.github/workflows/python-publish.yml @@ -0,0 +1,70 @@ +# This workflow will upload a Python Package to PyPI when a release is created +# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python#publishing-to-package-registries + +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +name: Upload Python Package + +on: + release: + types: [published] + +permissions: + contents: read + +jobs: + release-build: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v7 + + - uses: actions/setup-python@v7 + with: + python-version: "3.x" + + - name: Build release distributions + run: | + # NOTE: put your own distribution build steps here. + python -m pip install build + python -m build + + - name: Upload distributions + uses: actions/upload-artifact@v7 + with: + name: release-dists + path: dist/ + + pypi-publish: + runs-on: ubuntu-latest + needs: + - release-build + permissions: + # IMPORTANT: this permission is mandatory for trusted publishing + id-token: write + + # Dedicated environments with protections for publishing are strongly recommended. + # For more information, see: https://docs.github.com/en/actions/deployment/targeting-different-environments/using-environments-for-deployment#deployment-protection-rules + environment: + name: pypi + # OPTIONAL: uncomment and update to include your PyPI project URL in the deployment status: + # url: https://pypi.org/p/YOURPROJECT + # + # ALTERNATIVE: if your GitHub Release name is the PyPI project version string + # ALTERNATIVE: exactly, uncomment the following line instead: + # url: https://pypi.org/project/YOURPROJECT/${{ github.event.release.name }} + + steps: + - name: Retrieve release distributions + uses: actions/download-artifact@v8 + with: + name: release-dists + path: dist/ + + - name: Publish release distributions to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + packages-dir: dist/ diff --git a/.gitignore b/.gitignore index 038d1952..238181b5 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ dist/ build/ docs/build/ +logs/ env/ venv/ diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index c06cab3d..00000000 --- a/.travis.yml +++ /dev/null @@ -1,34 +0,0 @@ -os: linux - -dist: bionic - -language: python - -services: - - docker - -install: - - ./mk_dockerfile.sh - - docker-compose build - -script: - - docker-compose run $(bash <(curl -s https://codecov.io/env)) tests - -notifications: - email: - on_success: change - on_failure: always - -env: - - PYTHON_VERSION=3 PG_VERSION=14 - - PYTHON_VERSION=3 PG_VERSION=13 - - PYTHON_VERSION=3 PG_VERSION=12 - - PYTHON_VERSION=3 PG_VERSION=11 - - PYTHON_VERSION=3 PG_VERSION=10 -# - PYTHON_VERSION=3 PG_VERSION=9.6 -# - PYTHON_VERSION=3 PG_VERSION=9.5 -# - PYTHON_VERSION=3 PG_VERSION=9.4 -# - PYTHON_VERSION=2 PG_VERSION=10 -# - PYTHON_VERSION=2 PG_VERSION=9.6 -# - PYTHON_VERSION=2 PG_VERSION=9.5 -# - PYTHON_VERSION=2 PG_VERSION=9.4 diff --git a/Dockerfile--alpine.tmpl b/Dockerfile--alpine.tmpl new file mode 100644 index 00000000..d7cbc15d --- /dev/null +++ b/Dockerfile--alpine.tmpl @@ -0,0 +1,150 @@ +ARG PG_VERSION=17 +ARG PYTHON_VERSION=3.12 + +# --------------------------------------------- base1 +FROM postgres:${PG_VERSION}-alpine AS base1 + +RUN apk add --no-cache \ + coreutils \ + bash \ + mc \ + procps \ + openssh \ + sshpass \ + sudo \ + git + +RUN sed -i 's/#MaxStartups 10:30:100/MaxStartups 2000:30:2000/' /etc/ssh/sshd_config && \ + sed -i 's/#MaxSessions 10/MaxSessions 500/' /etc/ssh/sshd_config && \ + sed -i 's/#MaxAuthTries 6/MaxAuthTries 20/' /etc/ssh/sshd_config + +# --------------------------------------------- base2_with_python-3 +FROM base1 AS base2_with_python-3 + +RUN apk add --no-cache \ + curl \ + python3 \ + python3-dev \ + build-base \ + musl-dev \ + linux-headers \ + # For pyenv \ + patch \ + xz-dev \ + zip \ + zlib-dev \ + libffi-dev \ + readline-dev \ + openssl openssl-dev \ + sqlite-dev \ + bzip2-dev + +ENV PYTHON_BINARY=python3 + +# --------------------------------------------- base3_with_python-3.7 +FROM base2_with_python-3 AS base3_with_python-3.7 +ENV PYTHON_VERSION=3.7 + +# --------------------------------------------- base3_with_python-3.8.0 +FROM base2_with_python-3 AS base3_with_python-3.8.0 +ENV PYTHON_VERSION=3.8.0 + +# --------------------------------------------- base3_with_python-3.8 +FROM base2_with_python-3 AS base3_with_python-3.8 +ENV PYTHON_VERSION=3.8 + +# --------------------------------------------- base3_with_python-3.9 +FROM base2_with_python-3 AS base3_with_python-3.9 +ENV PYTHON_VERSION=3.9 + +# --------------------------------------------- base3_with_python-3.10 +FROM base2_with_python-3 AS base3_with_python-3.10 +ENV PYTHON_VERSION=3.10 + +# --------------------------------------------- base3_with_python-3.11 +FROM base2_with_python-3 AS base3_with_python-3.11 +ENV PYTHON_VERSION=3.11 + +# --------------------------------------------- base3_with_python-3.12 +FROM base2_with_python-3 AS base3_with_python-3.12 +ENV PYTHON_VERSION=3.12 + +# --------------------------------------------- base3_with_python-3.13 +FROM base2_with_python-3 AS base3_with_python-3.13 +ENV PYTHON_VERSION=3.13 + +# --------------------------------------------- base3_with_python-3.14 +FROM base2_with_python-3 AS base3_with_python-3.14 +ENV PYTHON_VERSION=3.14 + +# --------------------------------------------- final +FROM base3_with_python-${PYTHON_VERSION} AS final + +EXPOSE 22 +RUN ssh-keygen -A + +RUN adduser -D test && addgroup -S sudo && adduser test sudo + +# It allows to use sudo without password +RUN echo "test ALL=(ALL:ALL) NOPASSWD:ALL" >> /etc/sudoers + +# THIS CMD IS NEEDED TO CONNECT THROUGH SSH WITHOUT PASSWORD +RUN echo "test:*" | chpasswd -e + +USER test +RUN curl https://raw.githubusercontent.com/pyenv/pyenv-installer/master/bin/pyenv-installer | bash +RUN ~/.pyenv/bin/pyenv install ${PYTHON_VERSION} +USER root + +COPY --chown=test:test . /home/test/testgres +WORKDIR /home/test/testgres + +ENV LANG=C.UTF-8 + +RUN chmod 700 /home/test/ && \ + mkdir -p /home/test/.ssh && \ + echo 'Host *' > /home/test/.ssh/config && \ + echo ' ControlMaster auto' >> /home/test/.ssh/config && \ + echo ' ControlPath /home/test/.ssh/master-%r@%h:%p' >> /home/test/.ssh/config && \ + echo ' ControlPersist 30m' >> /home/test/.ssh/config && \ + echo ' StrictHostKeyChecking no' >> /home/test/.ssh/config && \ + echo ' UserKnownHostsFile /dev/null' >> /home/test/.ssh/config && \ + # echo ' GSSAPIAuthentication no' >> /home/test/.ssh/config && \ + chown -R test:test /home/test/.ssh && \ + chmod 700 /home/test/.ssh && \ + chmod 600 /home/test/.ssh/config + +# +# \"$@\" +# - quote is important! +# - "DUMMY-DUMMY-DUMMY" will be ignored. Do not ask me "why?". AXEZ. +# +ENTRYPOINT ["sh", "-c", " \ + set -eux; \ + echo 'SYSTEM START: PREPARING SSH'; \ + /usr/sbin/sshd; \ + ls -la /home/test/.ssh/; \ + \"$@\" \ +", "DUMMY-DUMMY-DUMMY"] + +# Run tests by default (master machine role) +CMD ["bash", "-c", " \ +set -eux; \ +echo \"HOME DIR IS [`realpath ~/`]\"; \ +echo \"WORK DIR IS [$(pwd)]\"; \ +if [ ! -f /home/test/.ssh/id_rsa ]; then \ + su test -c \"ssh-keygen -t rsa -f /home/test/.ssh/id_rsa -q -N ''\"; \ + su test -c \"cat /home/test/.ssh/id_rsa.pub >> /home/test/.ssh/authorized_keys\"; \ + chmod 600 /home/test/.ssh/authorized_keys; \ +fi; \ +ls -la /home/test/.ssh/; \ +if [ -n \"${TEST_CFG__REMOTE_SSH_KEY:-}\" ]; then \ + cp \"${TEST_CFG__REMOTE_SSH_KEY}\" \"${TEST_CFG__REMOTE_SSH_KEY}_ci\"; \ + export TEST_CFG__REMOTE_SSH_KEY=\"${TEST_CFG__REMOTE_SSH_KEY}_ci\"; \ + chown test:test \"${TEST_CFG__REMOTE_SSH_KEY}\"; \ + chmod 600 \"${TEST_CFG__REMOTE_SSH_KEY}\"; \ + ls -la /home/test/.ssh/; \ +fi; \ +ls -la ./; \ +su test -c \"TEST_FILTER='' PATH=\"/home/test/.pyenv/bin:$PATH\" bash ./run_tests2.sh\"; \ +"] diff --git a/Dockerfile--altlinux_10.tmpl b/Dockerfile--altlinux_10.tmpl new file mode 100644 index 00000000..2ec2c7ba --- /dev/null +++ b/Dockerfile--altlinux_10.tmpl @@ -0,0 +1,146 @@ +ARG PG_VERSION=17 +ARG PYTHON_VERSION=3 + +# --------------------------------------------- base1 +FROM alt:p10 AS base1 + +RUN apt-get update && apt-get install -y \ + sudo \ + curl \ + ca-certificates \ + openssh-server \ + openssh-clients \ + sshpass \ + time \ + su \ + git \ + libsqlite3-devel \ + && apt-get clean + +RUN sed -i 's/#MaxStartups 10:30:100/MaxStartups 2000:30:2000/' /etc/openssh/sshd_config && \ + sed -i 's/#MaxSessions 10/MaxSessions 500/' /etc/openssh/sshd_config && \ + sed -i 's/#MaxAuthTries 6/MaxAuthTries 20/' /etc/openssh/sshd_config + +# --------------------------------------------- postgres +FROM base1 AS base1_with_dev_tools + +RUN apt-get update && apt-get install -y \ + gcc \ + make \ + meson \ + flex \ + bison \ + pkg-config \ + libssl-devel \ + libicu-devel \ + libzstd-devel \ + zlib-devel \ + liblz4-devel \ + libxml2-devel \ + && apt-get clean + +# --------------------------------------------- postgres +FROM base1_with_dev_tools AS base1_with_pg-17 + +RUN git clone https://github.com/postgres/postgres.git -b REL_17_STABLE /pg/postgres/source + +WORKDIR /pg/postgres/source + +RUN ./configure --prefix=/pg/postgres/install --with-zlib --with-openssl --without-readline --with-lz4 --with-zstd --with-libxml +RUN make -j 4 install +RUN make -j 4 -C contrib install + +# SETUP PG_CONFIG +# When pg_config symlink in /usr/local/bin it returns a real (right) result of --bindir +RUN ln -s /pg/postgres/install/bin/pg_config -t /usr/local/bin + +# SETUP PG CLIENT LIBRARY +# libpq.so.5 is enough +RUN ln -s /pg/postgres/install/lib/libpq.so.5.17 /usr/lib64/libpq.so.5 + +# --------------------------------------------- base2_with_python-3 +FROM base1_with_pg-${PG_VERSION} AS base2_with_python-3 + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-dev \ + python3-modules-sqlite3 \ + && apt-get clean + +ENV PYTHON_BINARY=python3 + +# --------------------------------------------- final +FROM base2_with_python-${PYTHON_VERSION} AS final + +EXPOSE 22 +RUN ssh-keygen -A + +RUN adduser test -G wheel + +# It enables execution of "sudo service ssh start" without password +RUN echo "test ALL=(ALL:ALL) NOPASSWD: ALL" >> /etc/sudoers + +# +# Altlinux 10 and 11 too slowly create a new SSH connection (x6). +# +# AI: SPEED UP SSH 6 TIMES (REMOVE REVERSE DNS LOOKUP TIMEOUTS) +# +RUN echo "UseDNS no" >> /etc/openssh/sshd_config + +COPY --chown=test:test . /home/test/testgres +WORKDIR /home/test/testgres + +ENV LANG=C.UTF-8 + +RUN chmod 700 /home/test/ && \ + mkdir -p /home/test/.ssh && \ + echo 'Host *' > /home/test/.ssh/config && \ + echo ' ControlMaster auto' >> /home/test/.ssh/config && \ + echo ' ControlPath /home/test/.ssh/master-%r@%h:%p' >> /home/test/.ssh/config && \ + echo ' ControlPersist 30m' >> /home/test/.ssh/config && \ + echo ' StrictHostKeyChecking no' >> /home/test/.ssh/config && \ + echo ' UserKnownHostsFile /dev/null' >> /home/test/.ssh/config && \ + echo ' GSSAPIAuthentication no' >> /home/test/.ssh/config && \ + chown -R test:test /home/test/.ssh && \ + chmod 700 /home/test/.ssh && \ + chmod 600 /home/test/.ssh/config + +# +# \"$@\" +# - quote is important! +# - "DUMMY-DUMMY-DUMMY" will be ignored. Do not ask me "why?". AXEZ. +# +ENTRYPOINT ["sh", "-c", " \ + set -eux; \ + echo 'SYSTEM START: PREPARING SSH'; \ + (ulimit -n 1024 && /usr/sbin/sshd); \ + ls -la /home/test/.ssh/; \ + \"$@\" \ +", "DUMMY-DUMMY-DUMMY"] + +# Run tests by default (master machine role) +CMD ["bash", "-c", " \ +set -eux; \ +echo \"HOME DIR IS [`realpath ~/`]\"; \ +echo \"WORK DIR IS [$(pwd)]\"; \ +if [ ! -f /home/test/.ssh/id_rsa ]; then \ + su -c \"ssh-keygen -t rsa -f /home/test/.ssh/id_rsa -q -N ''\" test; \ + su -c \"cat /home/test/.ssh/id_rsa.pub >> /home/test/.ssh/authorized_keys\" test; \ + chmod 600 /home/test/.ssh/authorized_keys; \ +fi; \ +ls -la /home/test/.ssh/; \ +if [ -n \"${TEST_CFG__REMOTE_SSH_KEY:-}\" ]; then \ + cp \"${TEST_CFG__REMOTE_SSH_KEY}\" \"${TEST_CFG__REMOTE_SSH_KEY}_ci\"; \ + export TEST_CFG__REMOTE_SSH_KEY=\"${TEST_CFG__REMOTE_SSH_KEY}_ci\"; \ + chown test:test \"${TEST_CFG__REMOTE_SSH_KEY}\"; \ + chmod 600 \"${TEST_CFG__REMOTE_SSH_KEY}\"; \ + ls -la /home/test/.ssh/; \ +fi; \ +ls -la ./; \ +# AUTOMATIC TRANSFER: Save all TEST_CFG__xxx variables to a temporary file \ +(env | grep -E '^(TEST_CFG__|PYTHON_BINARY)' | sed 's/^/export /' || echo '') > ./test_cfg_env.sh; \ +chown test:test ./test_cfg_env.sh; \ +cat ./test_cfg_env.sh; \ +# Run su -, pull in the file with variables and start the tests \ +su - test -c \"cd /home/test/testgres && source ./test_cfg_env.sh && TEST_FILTER='' bash ./run_tests.sh\"; \ +"] diff --git a/Dockerfile--altlinux_11.tmpl b/Dockerfile--altlinux_11.tmpl new file mode 100644 index 00000000..f3a3cc9e --- /dev/null +++ b/Dockerfile--altlinux_11.tmpl @@ -0,0 +1,147 @@ +ARG PG_VERSION=17 +ARG PYTHON_VERSION=3 + +# --------------------------------------------- base1 +FROM alt:p11 AS base1 + +RUN apt-get update && apt-get install -y \ + sudo \ + curl \ + ca-certificates \ + openssh-server \ + openssh-clients \ + sshpass \ + time \ + su \ + git \ + procps \ + libsqlite3-devel \ + && apt-get clean + +RUN sed -i 's/#MaxStartups 10:30:100/MaxStartups 2000:30:2000/' /etc/openssh/sshd_config && \ + sed -i 's/#MaxSessions 10/MaxSessions 500/' /etc/openssh/sshd_config && \ + sed -i 's/#MaxAuthTries 6/MaxAuthTries 20/' /etc/openssh/sshd_config + +# --------------------------------------------- postgres +FROM base1 AS base1_with_dev_tools + +RUN apt-get update && apt-get install -y \ + gcc \ + make \ + meson \ + flex \ + bison \ + pkg-config \ + libssl-devel \ + libicu-devel \ + libzstd-devel \ + zlib-devel \ + liblz4-devel \ + libxml2-devel \ + && apt-get clean + +# --------------------------------------------- postgres +FROM base1_with_dev_tools AS base1_with_pg-17 + +RUN git clone https://github.com/postgres/postgres.git -b REL_17_STABLE /pg/postgres/source + +WORKDIR /pg/postgres/source + +RUN ./configure --prefix=/pg/postgres/install --with-zlib --with-openssl --without-readline --with-lz4 --with-zstd --with-libxml +RUN make -j 4 install +RUN make -j 4 -C contrib install + +# SETUP PG_CONFIG +# When pg_config symlink in /usr/local/bin it returns a real (right) result of --bindir +RUN ln -s /pg/postgres/install/bin/pg_config -t /usr/local/bin + +# SETUP PG CLIENT LIBRARY +# libpq.so.5 is enough +RUN ln -s /pg/postgres/install/lib/libpq.so.5.17 /usr/lib64/libpq.so.5 + +# --------------------------------------------- base2_with_python-3 +FROM base1_with_pg-${PG_VERSION} AS base2_with_python-3 + +RUN apt-get update && apt-get install -y \ + python3 \ + python3-dev \ + python3-modules-sqlite3 \ + && apt-get clean + +ENV PYTHON_BINARY=python3 + +# --------------------------------------------- final +FROM base2_with_python-${PYTHON_VERSION} AS final + +EXPOSE 22 +RUN ssh-keygen -A + +RUN adduser test -G wheel + +# It enables execution of "sudo service ssh start" without password +RUN echo "test ALL=(ALL:ALL) NOPASSWD: ALL" >> /etc/sudoers + +# +# Altlinux 10 and 11 too slowly create a new SSH connection (x6). +# +# AI: SPEED UP SSH 6 TIMES (REMOVE REVERSE DNS LOOKUP TIMEOUTS) +# +RUN echo "UseDNS no" >> /etc/openssh/sshd_config + +COPY --chown=test:test . /home/test/testgres +WORKDIR /home/test/testgres + +ENV LANG=C.UTF-8 + +RUN chmod 700 /home/test/ && \ + mkdir -p /home/test/.ssh && \ + echo 'Host *' > /home/test/.ssh/config && \ + echo ' ControlMaster auto' >> /home/test/.ssh/config && \ + echo ' ControlPath /home/test/.ssh/master-%r@%h:%p' >> /home/test/.ssh/config && \ + echo ' ControlPersist 30m' >> /home/test/.ssh/config && \ + echo ' StrictHostKeyChecking no' >> /home/test/.ssh/config && \ + echo ' UserKnownHostsFile /dev/null' >> /home/test/.ssh/config && \ + echo ' GSSAPIAuthentication no' >> /home/test/.ssh/config && \ + chown -R test:test /home/test/.ssh && \ + chmod 700 /home/test/.ssh && \ + chmod 600 /home/test/.ssh/config + +# +# \"$@\" +# - quote is important! +# - "DUMMY-DUMMY-DUMMY" will be ignored. Do not ask me "why?". AXEZ. +# +ENTRYPOINT ["sh", "-c", " \ + set -eux; \ + echo 'SYSTEM START: PREPARING SSH'; \ + (ulimit -n 1024 && /usr/sbin/sshd); \ + ls -la /home/test/.ssh/; \ + \"$@\" \ +", "DUMMY-DUMMY-DUMMY"] + +# Run tests by default (master machine role) +CMD ["bash", "-c", " \ +set -eux; \ +echo \"HOME DIR IS [`realpath ~/`]\"; \ +echo \"WORK DIR IS [$(pwd)]\"; \ +if [ ! -f /home/test/.ssh/id_rsa ]; then \ + su -c \"ssh-keygen -t rsa -f /home/test/.ssh/id_rsa -q -N ''\" test; \ + su -c \"cat /home/test/.ssh/id_rsa.pub >> /home/test/.ssh/authorized_keys\" test; \ + chmod 600 /home/test/.ssh/authorized_keys; \ +fi; \ +ls -la /home/test/.ssh/; \ +if [ -n \"${TEST_CFG__REMOTE_SSH_KEY:-}\" ]; then \ + cp \"${TEST_CFG__REMOTE_SSH_KEY}\" \"${TEST_CFG__REMOTE_SSH_KEY}_ci\"; \ + export TEST_CFG__REMOTE_SSH_KEY=\"${TEST_CFG__REMOTE_SSH_KEY}_ci\"; \ + chown test:test \"${TEST_CFG__REMOTE_SSH_KEY}\"; \ + chmod 600 \"${TEST_CFG__REMOTE_SSH_KEY}\"; \ + ls -la /home/test/.ssh/; \ +fi; \ +ls -la ./; \ +# AUTOMATIC TRANSFER: Save all TEST_CFG__xxx variables to a temporary file \ +(env | grep -E '^(TEST_CFG__|PYTHON_BINARY)' | sed 's/^/export /' || echo '') > ./test_cfg_env.sh; \ +chown test:test ./test_cfg_env.sh; \ +cat ./test_cfg_env.sh; \ +# Run su -, pull in the file with variables and start the tests \ +su - test -c \"cd /home/test/testgres && source ./test_cfg_env.sh && TEST_FILTER='' bash ./run_tests.sh\"; \ +"] diff --git a/Dockerfile--astralinux_1_7.tmpl b/Dockerfile--astralinux_1_7.tmpl new file mode 100644 index 00000000..7d731e8b --- /dev/null +++ b/Dockerfile--astralinux_1_7.tmpl @@ -0,0 +1,142 @@ +ARG PG_VERSION=17 +ARG PYTHON_VERSION=3 + +# --------------------------------------------- base1 +FROM packpack/packpack:astra-1.7 AS base1 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y --no-install-recommends \ + sudo \ + curl \ + ca-certificates \ + openssh-server \ + sshpass \ + iproute2 \ + git \ + time \ + && rm -rf /var/lib/apt/lists/* + +RUN sed -i 's/#MaxStartups 10:30:100/MaxStartups 2000:30:2000/' /etc/ssh/sshd_config && \ + sed -i 's/#MaxSessions 10/MaxSessions 500/' /etc/ssh/sshd_config && \ + sed -i 's/#MaxAuthTries 6/MaxAuthTries 20/' /etc/ssh/sshd_config + +# --------------------------------------------- postgres +FROM base1 AS base1_with_dev_tools + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y --no-install-recommends \ + gcc \ + make \ + meson \ + flex \ + bison \ + pkg-config \ + libssl-dev \ + libicu-dev \ + libzstd-dev \ + zlib1g-dev \ + liblz4-dev \ + libxml2-dev \ + && rm -rf /var/lib/apt/lists/* + +# --------------------------------------------- postgres +FROM base1_with_dev_tools AS base1_with_pg-17 + +RUN curl -fsSL https://ftp.postgresql.org/pub/source/v17.7/postgresql-17.7.tar.bz2 -o postgresql.tar.bz2 \ + && mkdir -p /pg/postgres/source \ + && tar -xjf postgresql.tar.bz2 -C /pg/postgres/source --strip-components=1 \ + && rm postgresql.tar.bz2 + +WORKDIR /pg/postgres/source + +RUN ./configure --prefix=/pg/postgres/install --with-zlib --with-openssl --without-readline --with-lz4 --with-zstd --with-libxml +RUN make -j 4 install +RUN make -j 4 -C contrib install + +# SETUP PG_CONFIG +# When pg_config symlink in /usr/local/bin it returns a real (right) result of --bindir +RUN ln -s /pg/postgres/install/bin/pg_config -t /usr/local/bin + +# SETUP PG CLIENT LIBRARY +# libpq.so.5 is enough +RUN ln -s /pg/postgres/install/lib/libpq.so.5.17 /usr/lib/libpq.so.5 + +# --------------------------------------------- base2_with_python-3 +FROM base1_with_pg-${PG_VERSION} AS base2_with_python-3 + +RUN apt-get update && apt-get install -y --no-install-recommends \ + python3 \ + python3-dev \ + python3-venv \ + && rm -rf /var/lib/apt/lists/* + +ENV PYTHON_BINARY=python3 + +# --------------------------------------------- final +FROM base2_with_python-${PYTHON_VERSION} AS final + +EXPOSE 22 +RUN ssh-keygen -A + +RUN useradd -m test + +# It enables execution of "sudo service ssh start" without password +# MY OLD: +# RUN sh -c "echo test ALL=NOPASSWD:ALL" >> /etc/sudoers +# AI: +RUN echo "test ALL=(ALL:ALL) NOPASSWD:ALL" >> /etc/sudoers + +# THIS CMD IS NEEDED TO CONNECT THROUGH SSH WITHOUT PASSWORD +RUN echo "test:*" | chpasswd -e && \ + sed -i 's/UsePAM yes/UsePAM no/' /etc/ssh/sshd_config + +COPY --chown=test:test . /home/test/testgres +WORKDIR /home/test/testgres + +ENV LANG=C.UTF-8 + +RUN chmod 700 /home/test/ && \ + mkdir -p /home/test/.ssh && \ + chown -R test:test /home/test/.ssh + +# +# \"$@\" +# - quote is important! +# - "DUMMY-DUMMY-DUMMY" will be ignored. Do not ask me "why?". AXEZ. +# +ENTRYPOINT ["sh", "-c", " \ + set -eux; \ + echo 'SYSTEM START: PREPARING SSH'; \ + service ssh start; \ + ls -la /home/test/.ssh/; \ + \"$@\" \ +", "DUMMY-DUMMY-DUMMY"] + +# Run tests by default (master machine role) +CMD ["bash", "-c", " \ +set -eux; \ +echo \"HOME DIR IS [`realpath ~/`]\"; \ +echo \"WORK DIR IS [$(pwd)]\"; \ +if [ ! -f /home/test/.ssh/id_rsa ]; then \ + su test -c \"ssh-keygen -t rsa -f /home/test/.ssh/id_rsa -q -N ''\"; \ + su test -c \"cat /home/test/.ssh/id_rsa.pub >> /home/test/.ssh/authorized_keys\"; \ + chmod 600 /home/test/.ssh/authorized_keys; \ +fi; \ +ls -la /home/test/.ssh/; \ +su test -c \"ssh-keyscan -H localhost >> /home/test/.ssh/known_hosts\"; \ +su test -c \"ssh-keyscan -H 127.0.0.1 >> /home/test/.ssh/known_hosts\"; \ +if [ -n \"${TEST_CFG__REMOTE_HOST:-}\" ]; then \ + su test -c \"ssh-keyscan -H ${TEST_CFG__REMOTE_HOST} >> /home/test/.ssh/known_hosts\"; \ +fi; \ +if [ -n \"${TEST_CFG__REMOTE_SSH_KEY:-}\" ]; then \ + cp \"${TEST_CFG__REMOTE_SSH_KEY}\" \"${TEST_CFG__REMOTE_SSH_KEY}_ci\"; \ + export TEST_CFG__REMOTE_SSH_KEY=\"${TEST_CFG__REMOTE_SSH_KEY}_ci\"; \ + chown test:test \"${TEST_CFG__REMOTE_SSH_KEY}\"; \ + chmod 600 \"${TEST_CFG__REMOTE_SSH_KEY}\"; \ + ls -la /home/test/.ssh/; \ +fi; \ +ls -la ./; \ +su test -c \"TEST_FILTER='' bash ./run_tests.sh\"; \ +"] diff --git a/Dockerfile--rockylinux_10.tmpl b/Dockerfile--rockylinux_10.tmpl new file mode 100644 index 00000000..c35aca7e --- /dev/null +++ b/Dockerfile--rockylinux_10.tmpl @@ -0,0 +1,161 @@ +# +# NOTE: RockyLinux-10 creates zomby processes! +# +ARG PG_VERSION=17 +ARG PYTHON_VERSION=3 + +# --------------------------------------------- base1 +FROM rockylinux/rockylinux:10 AS base1 + +# STEP 1: Fix the root cause (unstable mirrors). Switch to the official CDN. +RUN sed -i 's/mirrorlist=/ #mirrorlist=/g' /etc/yum.repos.d/rocky*.repo && \ + sed -i 's/#baseurl=http:\/\/dl.rockylinux.org/baseurl=https:\/\/dl.rockylinux.org/g' /etc/yum.repos.d/rocky*.repo + +# In RHEL 9/10, the repository for dev packages is called CRB (instead of powertools) +RUN dnf install -y 'dnf-command(config-manager)' && \ + dnf config-manager --set-enabled crb + +# Consolidating system utilities, including iproute (to prevent tests from failing) +# Added the --allowerasing flag so dnf can seamlessly replace curl-minimal with full-fledged curl +RUN dnf install -y --setopt=retries=10 --setopt=max_parallel_downloads=10 --allowerasing \ + sudo \ + curl \ + ca-certificates \ + openssh-server \ + openssh-clients \ + sshpass \ + time \ + util-linux \ + procps-ng \ + git \ + iproute \ + bzip2 \ + && dnf clean all + +# In RHEL 9/10, to generate host keys inside Docker, you need to call this binary +RUN /usr/libexec/openssh/sshd-keygen rsa && \ + /usr/libexec/openssh/sshd-keygen ed25519 + +RUN sed -i 's/#MaxStartups 10:30:100/MaxStartups 2000:30:2000/' /etc/ssh/sshd_config && \ + sed -i 's/#MaxSessions 10/MaxSessions 500/' /etc/ssh/sshd_config && \ + sed -i 's/#MaxAuthTries 6/MaxAuthTries 20/' /etc/ssh/sshd_config + +# --------------------------------------------- postgres dev tools +FROM base1 AS base1_with_dev_tools + +RUN dnf install -y --setopt=retries=10 --setopt=max_parallel_downloads=10 --allowerasing \ + gcc \ + make \ + meson \ + flex \ + bison \ + pkg-config \ + openssl-devel \ + libicu-devel \ + libzstd-devel \ + zlib-devel \ + lz4-devel \ + libxml2-devel \ + perl-interpreter \ + perl-FindBin \ + perl-File-Compare \ + && dnf clean all + +# --------------------------------------------- postgres build +FROM base1_with_dev_tools AS base1_with_pg-17 + +RUN curl -fsSL https://ftp.postgresql.org/pub/source/v17.7/postgresql-17.7.tar.bz2 -o postgresql.tar.bz2 \ + && mkdir -p /pg/postgres/source \ + && tar -xjf postgresql.tar.bz2 -C /pg/postgres/source --strip-components=1 \ + && rm postgresql.tar.bz2 + +WORKDIR /pg/postgres/source + +RUN ./configure --prefix=/pg/postgres/install --with-zlib --with-openssl --without-readline --with-lz4 --with-zstd --with-libxml +RUN make -j 4 install +RUN make -j 4 -C contrib install + +# SETUP PG_CONFIG +RUN ln -s /pg/postgres/install/bin/pg_config -t /usr/local/bin + +# SETUP PG CLIENT LIBRARY +RUN ln -s /pg/postgres/install/lib/libpq.so.5 /usr/lib64/libpq.so.5 && \ + echo "/pg/postgres/install/lib" > /etc/ld.so.conf.d/postgres.conf && ldconfig + +# --------------------------------------------- base2_with_python-3 +FROM base1_with_pg-${PG_VERSION} AS base2_with_python-3 + +# Rocky 9/10 installs python3.9 or higher (depending on the repository) +RUN dnf install -y --setopt=retries=10 --setopt=max_parallel_downloads=10 --allowerasing \ + python3 \ + python3-devel \ + && dnf clean all + +ENV PYTHON_BINARY=/usr/bin/python3 + +# --------------------------------------------- final +FROM base2_with_python-${PYTHON_VERSION} AS final + +EXPOSE 22 + +RUN useradd -m test && usermod -aG wheel test + +# Enable sudo without a password +RUN echo "test ALL=(ALL:ALL) NOPASSWD:ALL" >> /etc/sudoers + +# HACK FOR RHEL 9/10: Allow legacy key types and disable UsePAM. +# Additionally, allow passwordless root/test authentication for testing. +RUN echo "test:*" | chpasswd -e && \ + sed -i 's/UsePAM yes/UsePAM no/' /etc/ssh/sshd_config && \ + echo "PubkeyAcceptedKeyTypes +ssh-rsa" >> /etc/ssh/sshd_config && \ + echo "PermitEmptyPasswords yes" >> /etc/ssh/sshd_config + +COPY --chown=test:test . /home/test/testgres +WORKDIR /home/test/testgres + +ENV LANG=C.UTF-8 + +RUN chmod 700 /home/test/ && \ + mkdir -p /home/test/.ssh && \ + echo 'Host *' > /home/test/.ssh/config && \ + echo ' ControlMaster auto' >> /home/test/.ssh/config && \ + echo ' ControlPath /home/test/.ssh/master-%r@%h:%p' >> /home/test/.ssh/config && \ + echo ' ControlPersist 30m' >> /home/test/.ssh/config && \ + echo ' StrictHostKeyChecking no' >> /home/test/.ssh/config && \ + echo ' UserKnownHostsFile /dev/null' >> /home/test/.ssh/config && \ + echo ' GSSAPIAuthentication no' >> /home/test/.ssh/config && \ + chown -R test:test /home/test/.ssh && \ + chmod 700 /home/test/.ssh && \ + chmod 600 /home/test/.ssh/config + +# Our ENTRYPOINT with a stub +ENTRYPOINT ["sh", "-c", " \ + set -eux; \ + echo 'SYSTEM START: PREPARING SSH'; \ + /usr/sbin/sshd; \ + ls -la /home/test/.ssh/; \ + \"$@\" \ +", "DUMMY-DUMMY-DUMMY"] + +# In CMD, we change the key generation from the deprecated -t rsa to the modern -t ed25519! +# This ensures that Rocky 9/10 will accept this key without any cryptographic policy issues. +CMD ["bash", "-c", " \ +set -eux; \ +echo \"HOME DIR IS [`realpath ~/`]\"; \ +echo \"WORK DIR IS [$(pwd)]\"; \ +if [ ! -f /home/test/.ssh/id_ed25519 ]; then \ + su test -c \"ssh-keygen -t ed25519 -f /home/test/.ssh/id_ed25519 -q -N ''\"; \ + su test -c \"cat /home/test/.ssh/id_ed25519.pub >> /home/test/.ssh/authorized_keys\"; \ + chmod 600 /home/test/.ssh/authorized_keys; \ +fi; \ +ls -la /home/test/.ssh/; \ +if [ -n \"${TEST_CFG__REMOTE_SSH_KEY:-}\" ]; then \ + cp \"${TEST_CFG__REMOTE_SSH_KEY}\" \"${TEST_CFG__REMOTE_SSH_KEY}_ci\"; \ + export TEST_CFG__REMOTE_SSH_KEY=\"${TEST_CFG__REMOTE_SSH_KEY}_ci\"; \ + chown test:test \"${TEST_CFG__REMOTE_SSH_KEY}\"; \ + chmod 600 \"${TEST_CFG__REMOTE_SSH_KEY}\"; \ + ls -la /home/test/.ssh/; \ +fi; \ +ls -la ./; \ +su test -c \"TEST_FILTER='' bash ./run_tests.sh\"; \ +"] diff --git a/Dockerfile--rockylinux_8.tmpl b/Dockerfile--rockylinux_8.tmpl new file mode 100644 index 00000000..d9432ddc --- /dev/null +++ b/Dockerfile--rockylinux_8.tmpl @@ -0,0 +1,150 @@ +ARG PG_VERSION=17 +ARG PYTHON_VERSION=3 + +# --------------------------------------------- base1 +FROM rockylinux:8 AS base1 + +# Enable the PowerTools repository (called crb or powertools in Rocky/Alma), +# since without it, dnf won't find the python3-devel packages for building extensions. +RUN dnf install -y 'dnf-command(config-manager)' && \ + dnf config-manager --set-enabled powertools + +# Combine the installation of system utilities, SSH into one layer +RUN dnf install -y --setopt=retries=10 --setopt=max_parallel_downloads=10 \ + sudo \ + curl \ + ca-certificates \ + openssh-server \ + openssh-clients \ + sshpass \ + time \ + util-linux \ + procps-ng \ + git \ + iproute \ + bzip2 \ + && dnf clean all + +RUN sed -i 's/#MaxStartups 10:30:100/MaxStartups 2000:30:2000/' /etc/ssh/sshd_config && \ + sed -i 's/#MaxSessions 10/MaxSessions 500/' /etc/ssh/sshd_config && \ + sed -i 's/#MaxAuthTries 6/MaxAuthTries 20/' /etc/ssh/sshd_config + +# --------------------------------------------- postgres dev tools +FROM base1 AS base1_with_dev_tools + +RUN dnf install -y --setopt=retries=10 --setopt=max_parallel_downloads=10 \ + gcc \ + make \ + meson \ + flex \ + bison \ + pkg-config \ + openssl-devel \ + libicu-devel \ + libzstd-devel \ + zlib-devel \ + lz4-devel \ + libxml2-devel \ + && dnf clean all + +# --------------------------------------------- postgres build +FROM base1_with_dev_tools AS base1_with_pg-17 + +RUN curl -fsSL https://ftp.postgresql.org/pub/source/v17.7/postgresql-17.7.tar.bz2 -o postgresql.tar.bz2 \ + && mkdir -p /pg/postgres/source \ + && tar -xjf postgresql.tar.bz2 -C /pg/postgres/source --strip-components=1 \ + && rm postgresql.tar.bz2 + +WORKDIR /pg/postgres/source + +RUN ./configure --prefix=/pg/postgres/install --with-zlib --with-openssl --without-readline --with-lz4 --with-zstd --with-libxml +RUN make -j 4 install +RUN make -j 4 -C contrib install + +# SETUP PG_CONFIG +# When pg_config symlink in /usr/local/bin it returns a real (right) result of --bindir +RUN ln -s /pg/postgres/install/bin/pg_config -t /usr/local/bin + +# SETUP PG CLIENT LIBRARY +# libpq.so.5 is enough +RUN ln -s /pg/postgres/install/lib/libpq.so.5 /usr/lib/libpq.so.5 + +# --------------------------------------------- base2_with_python-3 +FROM base1_with_pg-${PG_VERSION} AS base2_with_python-3 + +RUN dnf install -y --setopt=retries=10 --setopt=max_parallel_downloads=10 \ + python39 \ + python39-devel \ + && dnf clean all + +ENV PYTHON_BINARY=/usr/bin/python3.9 + +# --------------------------------------------- final +FROM base2_with_python-${PYTHON_VERSION} AS final + +EXPOSE 22 +# In RHEL/CentOS, SSH host keys are generated via sshd-keygen +RUN ssh-keygen -A + +# In the RedHat family, the equivalent of the wheel/sudo group is the wheel group +RUN useradd -m test && usermod -aG wheel test + +# Enable sudo without a password +RUN echo "test ALL=(ALL:ALL) NOPASSWD:ALL" >> /etc/sudoers + +# Setting a blank password and disabling UsePAM (like in Astra), +# since SELinux/PAM in RHEL blocks blank passwords over SSH. +# +# On local tests it produces: +# WARNING: 'UsePAM no' is not supported in RHEL and may cause several problems. +# +# But without "sed -i 's/UsePAM yes/UsePAM no/' /etc/ssh/sshd_config" remote tests +# does not work! +# +RUN echo "test:*" | chpasswd -e && \ + sed -i 's/UsePAM yes/UsePAM no/' /etc/ssh/sshd_config + +COPY --chown=test:test . /home/test/testgres +WORKDIR /home/test/testgres + +ENV LANG=C.UTF-8 + +RUN chmod 700 /home/test/ && \ + mkdir -p /home/test/.ssh && \ + chown -R test:test /home/test/.ssh + +# Our proven ENTRYPOINT with a DUMMY plug +ENTRYPOINT ["sh", "-c", " \ + set -eux; \ + echo 'SYSTEM START: PREPARING SSH'; \ + /usr/sbin/sshd; \ + ls -la /home/test/.ssh/; \ + \"$@\" \ +", "DUMMY-DUMMY-DUMMY"] + +# Standard CMD for tests +CMD ["bash", "-c", " \ +set -eux; \ +echo \"HOME DIR IS [`realpath ~/`]\"; \ +echo \"WORK DIR IS [$(pwd)]\"; \ +if [ ! -f /home/test/.ssh/id_rsa ]; then \ + su test -c \"ssh-keygen -t rsa -f /home/test/.ssh/id_rsa -q -N ''\"; \ + su test -c \"cat /home/test/.ssh/id_rsa.pub >> /home/test/.ssh/authorized_keys\"; \ + chmod 600 /home/test/.ssh/authorized_keys; \ +fi; \ +ls -la /home/test/.ssh/; \ +su test -c \"ssh-keyscan -H localhost >> /home/test/.ssh/known_hosts\"; \ +su test -c \"ssh-keyscan -H 127.0.0.1 >> /home/test/.ssh/known_hosts\"; \ +if [ -n \"${TEST_CFG__REMOTE_HOST:-}\" ]; then \ + su test -c \"ssh-keyscan -H ${TEST_CFG__REMOTE_HOST} >> /home/test/.ssh/known_hosts\"; \ +fi; \ +if [ -n \"${TEST_CFG__REMOTE_SSH_KEY:-}\" ]; then \ + cp \"${TEST_CFG__REMOTE_SSH_KEY}\" \"${TEST_CFG__REMOTE_SSH_KEY}_ci\"; \ + export TEST_CFG__REMOTE_SSH_KEY=\"${TEST_CFG__REMOTE_SSH_KEY}_ci\"; \ + chown test:test \"${TEST_CFG__REMOTE_SSH_KEY}\"; \ + chmod 600 \"${TEST_CFG__REMOTE_SSH_KEY}\"; \ + ls -la /home/test/.ssh/; \ +fi; \ +ls -la ./; \ +su test -c \"TEST_FILTER='' bash ./run_tests.sh\"; \ +"] diff --git a/Dockerfile--rockylinux_9.tmpl b/Dockerfile--rockylinux_9.tmpl new file mode 100644 index 00000000..cf976601 --- /dev/null +++ b/Dockerfile--rockylinux_9.tmpl @@ -0,0 +1,161 @@ +# +# NOTE: RockyLinux-9 creates zomby processes! +# +ARG PG_VERSION=17 +ARG PYTHON_VERSION=3 + +# --------------------------------------------- base1 +FROM rockylinux/rockylinux:9 AS base1 + +# STEP 1: Fix the root cause (unstable mirrors). Switch to the official CDN. +RUN sed -i 's/mirrorlist=/ #mirrorlist=/g' /etc/yum.repos.d/rocky*.repo && \ + sed -i 's/#baseurl=http:\/\/dl.rockylinux.org/baseurl=https:\/\/dl.rockylinux.org/g' /etc/yum.repos.d/rocky*.repo + +# In RHEL 9/10, the repository for dev packages is called CRB (instead of powertools) +RUN dnf install -y 'dnf-command(config-manager)' && \ + dnf config-manager --set-enabled crb + +# Consolidating system utilities, including iproute (to prevent tests from failing) +# Added the --allowerasing flag so dnf can seamlessly replace curl-minimal with full-fledged curl +RUN dnf install -y --setopt=retries=10 --setopt=max_parallel_downloads=10 --allowerasing \ + sudo \ + curl \ + ca-certificates \ + openssh-server \ + openssh-clients \ + sshpass \ + time \ + util-linux \ + procps-ng \ + git \ + iproute \ + bzip2 \ + && dnf clean all + +# In RHEL 9/10, to generate host keys inside Docker, you need to call this binary +RUN /usr/libexec/openssh/sshd-keygen rsa && \ + /usr/libexec/openssh/sshd-keygen ed25519 + +RUN sed -i 's/#MaxStartups 10:30:100/MaxStartups 2000:30:2000/' /etc/ssh/sshd_config && \ + sed -i 's/#MaxSessions 10/MaxSessions 500/' /etc/ssh/sshd_config && \ + sed -i 's/#MaxAuthTries 6/MaxAuthTries 20/' /etc/ssh/sshd_config + +# --------------------------------------------- postgres dev tools +FROM base1 AS base1_with_dev_tools + +RUN dnf install -y --setopt=retries=10 --setopt=max_parallel_downloads=10 --allowerasing \ + gcc \ + make \ + meson \ + flex \ + bison \ + pkg-config \ + openssl-devel \ + libicu-devel \ + libzstd-devel \ + zlib-devel \ + lz4-devel \ + libxml2-devel \ + perl-interpreter \ + perl-FindBin \ + perl-File-Compare \ + && dnf clean all + +# --------------------------------------------- postgres build +FROM base1_with_dev_tools AS base1_with_pg-17 + +RUN curl -fsSL https://ftp.postgresql.org/pub/source/v17.7/postgresql-17.7.tar.bz2 -o postgresql.tar.bz2 \ + && mkdir -p /pg/postgres/source \ + && tar -xjf postgresql.tar.bz2 -C /pg/postgres/source --strip-components=1 \ + && rm postgresql.tar.bz2 + +WORKDIR /pg/postgres/source + +RUN ./configure --prefix=/pg/postgres/install --with-zlib --with-openssl --without-readline --with-lz4 --with-zstd --with-libxml +RUN make -j 4 install +RUN make -j 4 -C contrib install + +# SETUP PG_CONFIG +RUN ln -s /pg/postgres/install/bin/pg_config -t /usr/local/bin + +# SETUP PG CLIENT LIBRARY +RUN ln -s /pg/postgres/install/lib/libpq.so.5 /usr/lib64/libpq.so.5 && \ + echo "/pg/postgres/install/lib" > /etc/ld.so.conf.d/postgres.conf && ldconfig + +# --------------------------------------------- base2_with_python-3 +FROM base1_with_pg-${PG_VERSION} AS base2_with_python-3 + +# Rocky 9/10 installs python3.9 or higher (depending on the repository) +RUN dnf install -y --setopt=retries=10 --setopt=max_parallel_downloads=10 --allowerasing \ + python3 \ + python3-devel \ + && dnf clean all + +ENV PYTHON_BINARY=/usr/bin/python3 + +# --------------------------------------------- final +FROM base2_with_python-${PYTHON_VERSION} AS final + +EXPOSE 22 + +RUN useradd -m test && usermod -aG wheel test + +# Enable sudo without a password +RUN echo "test ALL=(ALL:ALL) NOPASSWD:ALL" >> /etc/sudoers + +# HACK FOR RHEL 9/10: Allow legacy key types and disable UsePAM. +# Additionally, allow passwordless root/test authentication for testing. +RUN echo "test:*" | chpasswd -e && \ + sed -i 's/UsePAM yes/UsePAM no/' /etc/ssh/sshd_config && \ + echo "PubkeyAcceptedKeyTypes +ssh-rsa" >> /etc/ssh/sshd_config && \ + echo "PermitEmptyPasswords yes" >> /etc/ssh/sshd_config + +COPY --chown=test:test . /home/test/testgres +WORKDIR /home/test/testgres + +ENV LANG=C.UTF-8 + +RUN chmod 700 /home/test/ && \ + mkdir -p /home/test/.ssh && \ + echo 'Host *' > /home/test/.ssh/config && \ + echo ' ControlMaster auto' >> /home/test/.ssh/config && \ + echo ' ControlPath /home/test/.ssh/master-%r@%h:%p' >> /home/test/.ssh/config && \ + echo ' ControlPersist 30m' >> /home/test/.ssh/config && \ + echo ' StrictHostKeyChecking no' >> /home/test/.ssh/config && \ + echo ' UserKnownHostsFile /dev/null' >> /home/test/.ssh/config && \ + echo ' GSSAPIAuthentication no' >> /home/test/.ssh/config && \ + chown -R test:test /home/test/.ssh && \ + chmod 700 /home/test/.ssh && \ + chmod 600 /home/test/.ssh/config + +# Our ENTRYPOINT with a stub +ENTRYPOINT ["sh", "-c", " \ + set -eux; \ + echo 'SYSTEM START: PREPARING SSH'; \ + /usr/sbin/sshd; \ + ls -la /home/test/.ssh/; \ + \"$@\" \ +", "DUMMY-DUMMY-DUMMY"] + +# In CMD, we change the key generation from the deprecated -t rsa to the modern -t ed25519! +# This ensures that Rocky 9/10 will accept this key without any cryptographic policy issues. +CMD ["bash", "-c", " \ +set -eux; \ +echo \"HOME DIR IS [`realpath ~/`]\"; \ +echo \"WORK DIR IS [$(pwd)]\"; \ +if [ ! -f /home/test/.ssh/id_ed25519 ]; then \ + su test -c \"ssh-keygen -t ed25519 -f /home/test/.ssh/id_ed25519 -q -N ''\"; \ + su test -c \"cat /home/test/.ssh/id_ed25519.pub >> /home/test/.ssh/authorized_keys\"; \ + chmod 600 /home/test/.ssh/authorized_keys; \ +fi; \ +ls -la /home/test/.ssh/; \ +if [ -n \"${TEST_CFG__REMOTE_SSH_KEY:-}\" ]; then \ + cp \"${TEST_CFG__REMOTE_SSH_KEY}\" \"${TEST_CFG__REMOTE_SSH_KEY}_ci\"; \ + export TEST_CFG__REMOTE_SSH_KEY=\"${TEST_CFG__REMOTE_SSH_KEY}_ci\"; \ + chown test:test \"${TEST_CFG__REMOTE_SSH_KEY}\"; \ + chmod 600 \"${TEST_CFG__REMOTE_SSH_KEY}\"; \ + ls -la /home/test/.ssh/; \ +fi; \ +ls -la ./; \ +su test -c \"TEST_FILTER='' bash ./run_tests.sh\"; \ +"] diff --git a/Dockerfile--ubuntu_24_04.tmpl b/Dockerfile--ubuntu_24_04.tmpl new file mode 100644 index 00000000..340999f1 --- /dev/null +++ b/Dockerfile--ubuntu_24_04.tmpl @@ -0,0 +1,111 @@ +ARG PG_VERSION=17 +ARG PYTHON_VERSION=3 + +# --------------------------------------------- base1 +FROM ubuntu:24.04 AS base1 +ARG PG_VERSION + +# Disable interactive apt questions so that the build doesn't hang when setting time zones +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y --no-install-recommends \ + sudo \ + curl \ + ca-certificates \ + openssh-server \ + sshpass \ + time \ + netcat-traditional \ + iproute2 \ + git \ + postgresql-common \ + libpq-dev \ + build-essential \ + && rm -rf /var/lib/apt/lists/* + +RUN sed -i 's/#MaxStartups 10:30:100/MaxStartups 2000:30:2000/' /etc/ssh/sshd_config && \ + sed -i 's/#MaxSessions 10/MaxSessions 500/' /etc/ssh/sshd_config && \ + sed -i 's/#MaxAuthTries 6/MaxAuthTries 20/' /etc/ssh/sshd_config + +RUN bash /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh -y + +RUN install -d /usr/share/postgresql-common/pgdg +RUN curl -o /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc --fail https://www.postgresql.org/media/keys/ACCC4CF8.asc + +RUN apt-get update && apt-get install -y --no-install-recommends \ + postgresql-${PG_VERSION} + +# --------------------------------------------- base2_with_python-3 +FROM base1 AS base2_with_python-3 + +RUN apt-get update && apt-get install -y --no-install-recommends \ + python3 \ + python3-dev \ + python3-venv \ + && rm -rf /var/lib/apt/lists/* + +ENV PYTHON_BINARY=python3 + +# --------------------------------------------- final +FROM base2_with_python-${PYTHON_VERSION} AS final + +EXPOSE 22 +RUN ssh-keygen -A + +RUN useradd -m test && usermod -aG postgres test + +# It enables execution of "sudo service ssh start" without password +RUN echo "test ALL=(ALL:ALL) NOPASSWD:ALL" >> /etc/sudoers + +COPY --chown=test:test . /home/test/testgres +WORKDIR /home/test/testgres + +ENV LANG=C.UTF-8 + +RUN chmod 700 /home/test/ && \ + mkdir -p /home/test/.ssh && \ + echo 'Host *' > /home/test/.ssh/config && \ + echo ' ControlMaster auto' >> /home/test/.ssh/config && \ + echo ' ControlPath /home/test/.ssh/master-%r@%h:%p' >> /home/test/.ssh/config && \ + echo ' ControlPersist 30m' >> /home/test/.ssh/config && \ + echo ' StrictHostKeyChecking no' >> /home/test/.ssh/config && \ + echo ' UserKnownHostsFile /dev/null' >> /home/test/.ssh/config && \ + echo ' GSSAPIAuthentication no' >> /home/test/.ssh/config && \ + chown -R test:test /home/test/.ssh && \ + chmod 700 /home/test/.ssh && \ + chmod 600 /home/test/.ssh/config + +# +# \"$@\" +# - quote is important! +# - "DUMMY-DUMMY-DUMMY" will be ignored. Do not ask me "why?". AXEZ. +# +ENTRYPOINT ["sh", "-c", " \ + set -eux; \ + echo 'SYSTEM START: PREPARING SSH'; \ + service ssh start; \ + ls -la /home/test/.ssh/; \ + \"$@\" \ +", "DUMMY-DUMMY-DUMMY"] + +# Run tests by default (master machine role) +CMD ["bash", "-c", " \ +set -eux; \ +echo \"HOME DIR IS [`realpath ~/`]\"; \ +echo \"WORK DIR IS [$(pwd)]\"; \ +if [ ! -f /home/test/.ssh/id_rsa ]; then \ + su test -c \"ssh-keygen -t rsa -f /home/test/.ssh/id_rsa -q -N ''\"; \ + su test -c \"cat /home/test/.ssh/id_rsa.pub >> /home/test/.ssh/authorized_keys\"; \ + chmod 600 /home/test/.ssh/authorized_keys; \ +fi; \ +ls -la /home/test/.ssh/; \ +if [ -n \"${TEST_CFG__REMOTE_SSH_KEY:-}\" ]; then \ + cp \"${TEST_CFG__REMOTE_SSH_KEY}\" \"${TEST_CFG__REMOTE_SSH_KEY}_ci\"; \ + export TEST_CFG__REMOTE_SSH_KEY=\"${TEST_CFG__REMOTE_SSH_KEY}_ci\"; \ + chown test:test \"${TEST_CFG__REMOTE_SSH_KEY}\"; \ + chmod 600 \"${TEST_CFG__REMOTE_SSH_KEY}\"; \ + ls -la /home/test/.ssh/; \ +fi; \ +ls -la ./; \ +su test -c \"TEST_FILTER='' bash ./run_tests.sh\"; \ +"] diff --git a/Dockerfile--ubuntu_26_04.tmpl b/Dockerfile--ubuntu_26_04.tmpl new file mode 100644 index 00000000..3b30cc12 --- /dev/null +++ b/Dockerfile--ubuntu_26_04.tmpl @@ -0,0 +1,131 @@ +ARG PG_VERSION=17 +ARG PYTHON_VERSION=3.12 + +# --------------------------------------------- base1 +FROM ubuntu:26.04 AS base1 +ARG PG_VERSION + +# Disable interactive apt questions so that the build doesn't hang when setting time zones +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y --no-install-recommends \ + sudo \ + curl \ + ca-certificates \ + openssh-server \ + sshpass \ + time \ + netcat-traditional \ + iproute2 \ + git \ + postgresql-common \ + libpq-dev \ + build-essential \ + # FIX FOR TestOsOpsCommon::test_mkdir__mt[remote_ops] \ + # INFO [2026-07-14 11:12:31] [Worker #3] Number 0 is reserved! \ + # INFO [2026-07-14 11:12:31] [Worker #0] Number 0 is reserved! \ + # Returning classic GNU coreutils instead of the default Rust/uutils \ + && apt-get install -y --allow-remove-essential coreutils-from-gnu coreutils-from-uutils- \ + && rm -rf /var/lib/apt/lists/* + +RUN sed -i 's/#MaxStartups 10:30:100/MaxStartups 2000:30:2000/' /etc/ssh/sshd_config && \ + sed -i 's/#MaxSessions 10/MaxSessions 500/' /etc/ssh/sshd_config && \ + sed -i 's/#MaxAuthTries 6/MaxAuthTries 20/' /etc/ssh/sshd_config + +RUN bash /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh -y + +RUN install -d /usr/share/postgresql-common/pgdg +RUN curl -o /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc --fail https://www.postgresql.org/media/keys/ACCC4CF8.asc + +RUN apt-get update && apt-get install -y --no-install-recommends \ + postgresql-${PG_VERSION} + +# --------------------------------------------- base2_with_python-3 +FROM base1 AS base2_with_python-3 + +RUN apt-get update && apt-get install -y --no-install-recommends \ + python3 \ + python3-dev \ + python3-venv \ + && rm -rf /var/lib/apt/lists/* + +ENV PYTHON_BINARY=python3 + +# --------------------------------------------- base2_with_python-3.12 +FROM base1 AS base2_with_python-3.12 + +RUN apt-get update && apt-get install -y --no-install-recommends \ + software-properties-common \ + gnupg \ + && add-apt-repository -y ppa:deadsnakes/ppa \ + && apt-get update && apt-get install -y --no-install-recommends \ + python3.12 \ + python3.12-venv \ + python3.12-dev \ + && rm -rf /var/lib/apt/lists/* + +ENV PYTHON_BINARY=/usr/bin/python3.12 + +# --------------------------------------------- final +FROM base2_with_python-${PYTHON_VERSION} AS final + +EXPOSE 22 +RUN ssh-keygen -A + +RUN useradd -m test && usermod -aG postgres test + +# It enables execution of "sudo service ssh start" without password +RUN echo "test ALL=(ALL:ALL) NOPASSWD:ALL" >> /etc/sudoers + +COPY --chown=test:test . /home/test/testgres +WORKDIR /home/test/testgres + +ENV LANG=C.UTF-8 + +RUN chmod 700 /home/test/ && \ + mkdir -p /home/test/.ssh && \ + echo 'Host *' > /home/test/.ssh/config && \ + echo ' ControlMaster auto' >> /home/test/.ssh/config && \ + echo ' ControlPath /home/test/.ssh/master-%r@%h:%p' >> /home/test/.ssh/config && \ + echo ' ControlPersist 30m' >> /home/test/.ssh/config && \ + echo ' StrictHostKeyChecking no' >> /home/test/.ssh/config && \ + echo ' UserKnownHostsFile /dev/null' >> /home/test/.ssh/config && \ + echo ' GSSAPIAuthentication no' >> /home/test/.ssh/config && \ + chown -R test:test /home/test/.ssh && \ + chmod 700 /home/test/.ssh && \ + chmod 600 /home/test/.ssh/config + +# +# \"$@\" +# - quote is important! +# - "DUMMY-DUMMY-DUMMY" will be ignored. Do not ask me "why?". AXEZ. +# +ENTRYPOINT ["sh", "-c", " \ + set -eux; \ + echo 'SYSTEM START: PREPARING SSH'; \ + service ssh start; \ + ls -la /home/test/.ssh/; \ + \"$@\" \ +", "DUMMY-DUMMY-DUMMY"] + +# Run tests by default (master machine role) +CMD ["bash", "-c", " \ +set -eux; \ +echo \"HOME DIR IS [`realpath ~/`]\"; \ +echo \"WORK DIR IS [$(pwd)]\"; \ +if [ ! -f /home/test/.ssh/id_rsa ]; then \ + su test -c \"ssh-keygen -t rsa -f /home/test/.ssh/id_rsa -q -N ''\"; \ + su test -c \"cat /home/test/.ssh/id_rsa.pub >> /home/test/.ssh/authorized_keys\"; \ + chmod 600 /home/test/.ssh/authorized_keys; \ +fi; \ +ls -la /home/test/.ssh/; \ +if [ -n \"${TEST_CFG__REMOTE_SSH_KEY:-}\" ]; then \ + cp \"${TEST_CFG__REMOTE_SSH_KEY}\" \"${TEST_CFG__REMOTE_SSH_KEY}_ci\"; \ + export TEST_CFG__REMOTE_SSH_KEY=\"${TEST_CFG__REMOTE_SSH_KEY}_ci\"; \ + chown test:test \"${TEST_CFG__REMOTE_SSH_KEY}\"; \ + chmod 600 \"${TEST_CFG__REMOTE_SSH_KEY}\"; \ + ls -la /home/test/.ssh/; \ +fi; \ +ls -la ./; \ +su test -c \"TEST_FILTER='' bash ./run_tests.sh\"; \ +"] diff --git a/Dockerfile.tmpl b/Dockerfile.tmpl deleted file mode 100644 index dc5878b6..00000000 --- a/Dockerfile.tmpl +++ /dev/null @@ -1,23 +0,0 @@ -FROM postgres:${PG_VERSION}-alpine - -ENV PYTHON=python${PYTHON_VERSION} -RUN if [ "${PYTHON_VERSION}" = "2" ] ; then \ - apk add --no-cache curl python2 python2-dev build-base musl-dev \ - linux-headers py-virtualenv py-pip; \ - fi -RUN if [ "${PYTHON_VERSION}" = "3" ] ; then \ - apk add --no-cache curl python3 python3-dev build-base musl-dev \ - linux-headers py-virtualenv; \ - fi -ENV LANG=C.UTF-8 - -RUN mkdir -p /pg -COPY run_tests.sh /run.sh -RUN chmod 755 /run.sh - -ADD . /pg/testgres -WORKDIR /pg/testgres -RUN chown -R postgres:postgres /pg - -USER postgres -ENTRYPOINT PYTHON_VERSION=${PYTHON_VERSION} /run.sh diff --git a/LICENSE b/LICENSE index 7e9cc712..cd042e7e 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ testgres is released under the PostgreSQL License, a liberal Open Source license, similar to the BSD or MIT licenses. -Copyright (c) 2016-2023, Postgres Professional +Copyright (c) 2016-2026, Postgres Professional Permission to use, copy, modify, and distribute this software and its documentation for any purpose, without fee, and without a written agreement is hereby granted, provided that the above copyright notice and this paragraph and the following two paragraphs appear in all copies. diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index 8adcedf3..00000000 --- a/MANIFEST.in +++ /dev/null @@ -1,8 +0,0 @@ -include LICENSE -include README.md -include setup.cfg - -recursive-include testgres *.py -recursive-include tests *.py - -global-exclude *.pyc diff --git a/README.md b/README.md index a2a0ec7e..d615d367 100644 --- a/README.md +++ b/README.md @@ -1,74 +1,75 @@ -[![Build Status](https://travis-ci.com/postgrespro/testgres.svg?branch=master)](https://app.travis-ci.com/github/postgrespro/testgres/branches) -[![codecov](https://codecov.io/gh/postgrespro/testgres/branch/master/graph/badge.svg)](https://codecov.io/gh/postgrespro/testgres) -[![PyPI version](https://badge.fury.io/py/testgres.svg)](https://badge.fury.io/py/testgres) +[![CI Status](https://img.shields.io/github/actions/workflow/status/postgrespro/testgres/.github/workflows/ci.yml?label=CI)](https://github.com/postgrespro/testgres/actions/workflows/ci.yml) +[![PyPI package version](https://badge.fury.io/py/testgres.svg)](https://badge.fury.io/py/testgres) +[![PyPI python versions](https://img.shields.io/pypi/pyversions/testgres)](https://pypi.org/project/testgres) +[![PyPI downloads](https://img.shields.io/pypi/dm/testgres)](https://pypi.org/project/testgres) [Documentation](https://postgrespro.github.io/testgres/) # testgres -PostgreSQL testing utility. Both Python 2.7 and 3.3+ are supported. - +Utility for orchestrating temporary PostgreSQL clusters in Python tests. Supports Python 3.7.3 and newer. ## Installation -To install `testgres`, run: +Install `testgres` from PyPI: -``` +```sh pip install testgres ``` -We encourage you to use `virtualenv` for your testing environment. - +Use a dedicated virtual environment for isolated test dependencies. ## Usage ### Environment -> Note: by default testgres runs `initdb`, `pg_ctl`, `psql` provided by `PATH`. +> Note: by default `testgres` invokes `initdb`, `pg_ctl`, and `psql` binaries found in `PATH`. -There are several ways to specify a custom postgres installation: +Specify a custom PostgreSQL installation in one of the following ways: -* export `PG_CONFIG` environment variable pointing to the `pg_config` executable; -* export `PG_BIN` environment variable pointing to the directory with executable files. +- Set the `PG_CONFIG` environment variable to point to the `pg_config` executable. +- Set the `PG_BIN` environment variable to point to the directory with PostgreSQL binaries. Example: -```bash -export PG_BIN=$HOME/pg_10/bin +```sh +export PG_BIN=$HOME/pg_16/bin python my_tests.py ``` - ### Examples -Here is an example of what you can do with `testgres`: +Create a temporary node, run queries, and let `testgres` clean up automatically: ```python -# create a node with random name, port, etc +# create a node with a random name, port, and data directory with testgres.get_new_node() as node: - # run inidb + # run initdb node.init() # start PostgreSQL node.start() - # execute a query in a default DB + # execute a query in the default database print(node.execute('select 1')) -# ... node stops and its files are about to be removed +# the node is stopped and its files are removed automatically ``` -There are four API methods for runnig queries: +### Query helpers + +`testgres` provides four helpers for executing queries against the node: | Command | Description | -|----------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------| -| `node.psql(query, ...)` | Runs query via `psql` command and returns tuple `(error code, stdout, stderr)`. | -| `node.safe_psql(query, ...)` | Same as `psql()` except that it returns only `stdout`. If an error occures during the execution, an exception will be thrown. | -| `node.execute(query, ...)` | Connects to PostgreSQL using `psycopg2` or `pg8000` (depends on which one is installed in your system) and returns two-dimensional array with data. | -| `node.connect(dbname, ...)` | Returns connection wrapper (`NodeConnection`) capable of running several queries within a single transaction. | +|---------|-------------| +| `node.psql(query, ...)` | Runs the query via `psql` and returns a tuple `(returncode, stdout, stderr)`. | +| `node.safe_psql(query, ...)` | Same as `psql()` but returns only `stdout` and raises if the command fails. | +| `node.execute(query, ...)` | Connects via `psycopg2` or `pg8000` (whichever is available) and returns a list of tuples. | +| `node.connect(dbname, ...)` | Returns a `NodeConnection` wrapper for executing multiple statements within a transaction. | + +Example of transactional usage: -The last one is the most powerful: you can use `begin(isolation_level)`, `commit()` and `rollback()`: ```python with node.connect() as con: con.begin('serializable') @@ -76,16 +77,13 @@ with node.connect() as con: con.rollback() ``` - ### Logging -By default, `cleanup()` removes all temporary files (DB files, logs etc) that were created by testgres' API methods. -If you'd like to keep logs, execute `configure_testgres(node_cleanup_full=False)` before running any tests. +By default `cleanup()` removes all temporary files (data directories, logs, and so on) created by the API. Call `configure_testgres(node_cleanup_full=False)` before starting nodes if you want to keep logs for inspection. -> Note: context managers (aka `with`) call `stop()` and `cleanup()` automatically. +> Note: context managers (the `with` statement) call `stop()` and `cleanup()` automatically. -`testgres` supports [python logging](https://docs.python.org/3.6/library/logging.html), -which means that you can aggregate logs from several nodes into one file: +`testgres` integrates with the standard [Python logging](https://docs.python.org/3/library/logging.html) module, so you can aggregate logs from multiple nodes: ```python import logging @@ -93,12 +91,11 @@ import logging # write everything to /tmp/testgres.log logging.basicConfig(filename='/tmp/testgres.log') -# enable logging, and create two different nodes +# enable logging and create two nodes testgres.configure_testgres(use_python_logging=True) node1 = testgres.get_new_node().init().start() node2 = testgres.get_new_node().init().start() -# execute a few queries node1.execute('select 1') node2.execute('select 2') @@ -106,104 +103,103 @@ node2.execute('select 2') testgres.configure_testgres(use_python_logging=False) ``` -Look at `tests/test_simple.py` file for a complete example of the logging -configuration. - +See `tests/test_simple.py` for a complete logging example. -### Backup & replication +### Backup and replication -It's quite easy to create a backup and start a new replica: +Creating backups and spawning replicas is straightforward: ```python with testgres.get_new_node('master') as master: master.init().start() - # create a backup with master.backup() as backup: - - # create and start a new replica replica = backup.spawn_replica('replica').start() - - # catch up with master node replica.catchup() - # execute a dummy query print(replica.execute('postgres', 'select 1')) ``` ### Benchmarks -`testgres` is also capable of running benchmarks using `pgbench`: +Use `pgbench` through `testgres` to run quick benchmarks: ```python with testgres.get_new_node('master') as master: - # start a new node master.init().start() - # initialize default DB and run bench for 10 seconds - res = master.pgbench_init(scale=2).pgbench_run(time=10) - print(res) + result = master.pgbench_init(scale=2).pgbench_run(time=10) + print(result) ``` - ### Custom configuration -It's often useful to extend default configuration provided by `testgres`. - -`testgres` has `default_conf()` function that helps control some basic -options. The `append_conf()` function can be used to add custom -lines to configuration lines: +`testgres` ships with sensible defaults. Adjust them as needed with `default_conf()` and `append_conf()`: ```python -ext_conf = "shared_preload_libraries = 'postgres_fdw'" +extra_conf = "shared_preload_libraries = 'postgres_fdw'" -# initialize a new node with testgres.get_new_node().init() as master: - - # ... do something ... - - # reset main config file - master.default_conf(fsync=True, - allow_streaming=True) - - # add a new config line - master.append_conf('postgresql.conf', ext_conf) + master.default_conf(fsync=True, allow_streaming=True) + master.append_conf('postgresql.conf', extra_conf) ``` -Note that `default_conf()` is called by `init()` function; both of them overwrite -the configuration file, which means that they should be called before `append_conf()`. +`default_conf()` is called by `init()` and rewrites the configuration file. Apply `append_conf()` afterwards to keep custom lines. ### Remote mode -Testgres supports the creation of PostgreSQL nodes on a remote host. This is useful when you want to run distributed tests involving multiple nodes spread across different machines. -To use this feature, you need to use the RemoteOperations class. This feature is only supported with Linux. -Here is an example of how you might set this up: +You can provision nodes on a remote host (Linux only) by wiring `RemoteOperations` into the configuration: ```python from testgres import ConnectionParams, RemoteOperations, TestgresConfig, get_remote_node -# Set up connection params conn_params = ConnectionParams( - host='your_host', # replace with your host - username='user_name', # replace with your username - ssh_key='path_to_ssh_key' # replace with your SSH key path + host='example.com', + username='postgres', + ssh_key='/path/to/ssh/key' ) os_ops = RemoteOperations(conn_params) -# Add remote testgres config before test TestgresConfig.set_os_ops(os_ops=os_ops) -# Proceed with your test -def test_basic_query(self): +def test_basic_query(): with get_remote_node(conn_params=conn_params) as node: node.init().start() - res = node.execute('SELECT 1') - self.assertEqual(res, [(1,)]) + assert node.execute('SELECT 1') == [(1,)] ``` +### Pytest integration + +Use fixtures to create and clean up nodes automatically when testing with `pytest`: + +```python +import pytest +import testgres + +@pytest.fixture +def pg_node(): + node = testgres.get_new_node().init().start() + try: + yield node + finally: + node.stop() + node.cleanup() + +def test_simple(pg_node): + assert pg_node.execute('select 1')[0][0] == 1 +``` + +This pattern keeps tests concise and ensures that every node is stopped and removed even if the test fails. + +### Scaling tips + +- Run tests in parallel with `pytest -n auto` (requires `pytest-xdist`). Ensure each node uses a distinct port by setting `PGPORT` in the fixture or by passing the `port` argument to `get_new_node()`. +- Always call `node.cleanup()` after each test, or rely on context managers/fixtures that do it for you, to avoid leftover data directories. +- Prefer `node.safe_psql()` for lightweight assertions that should fail fast; use `node.execute()` when you need structured Python results. + ## Authors [Ildar Musin](https://github.com/zilder) [Dmitry Ivanov](https://github.com/funbringer) [Ildus Kurbangaliev](https://github.com/ildus) -[Yury Zhuravlev](https://github.com/stalkerg) +[Yury Zhuravlev](https://github.com/stalkerg) diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index 471ab779..00000000 --- a/docker-compose.yml +++ /dev/null @@ -1,2 +0,0 @@ -tests: - build: . diff --git a/docs/Makefile b/docs/Makefile index f33f6be0..d6818981 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -17,4 +17,5 @@ help: # Catch-all target: route all unknown targets to Sphinx using the new # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). %: Makefile + @pip install --force-reinstall .. @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) \ No newline at end of file diff --git a/docs/source/conf.py b/docs/source/conf.py index 688a850f..44eee57b 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -14,18 +14,27 @@ # import os import sys -sys.path.insert(0, os.path.abspath('../..')) +import testgres + +assert testgres.__path__ is not None +assert len(testgres.__path__) == 1 +assert type(testgres.__path__[0]) is str +p = os.path.dirname(testgres.__path__[0]) +assert type(p) is str +sys.path.insert(0, os.path.abspath(p)) # -- Project information ----------------------------------------------------- project = u'testgres' -copyright = u'2016-2023, Postgres Professional' +package_name = u'testgres' +copyright = u'2016-2026, Postgres Professional' author = u'Postgres Professional' -# The short X.Y version -version = u'' # The full version, including alpha/beta/rc tags -release = u'1.5' +release = testgres.__version__ + +# The short X.Y version +version = '.'.join(release.split('.')[:2]) # -- General configuration --------------------------------------------------- @@ -55,7 +64,7 @@ # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. -language = None +language = "en" # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. diff --git a/docs/source/index.rst b/docs/source/index.rst index 566d9a50..c2104a65 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -2,7 +2,7 @@ Testgres documentation ====================== -Testgres is a PostgreSQL testing framework. +Utility for orchestrating temporary PostgreSQL clusters in Python tests. Supports Python 3.7.17 and newer. Installation ============ @@ -21,9 +21,9 @@ Usage Environment ----------- -Note: by default testgres runs ``initdb``, ``pg_ctl``, ``psql`` provided by ``PATH``. +Note: by default ``testgres`` runs ``initdb``, ``pg_ctl``, and ``psql`` found in ``PATH``. -There are several ways to specify a custom postgres installation: +There are several ways to specify a custom PostgreSQL installation: - export ``PG_CONFIG`` environment variable pointing to the ``pg_config`` executable; - export ``PG_BIN`` environment variable pointing to the directory with executable files. @@ -32,29 +32,76 @@ Example: .. code-block:: bash - export PG_BIN=$HOME/pg_10/bin + export PG_BIN=$HOME/pg_16/bin python my_tests.py Examples -------- -Here is an example of what you can do with ``testgres``: +Create a temporary node, run queries, and let ``testgres`` clean up automatically: .. code-block:: python - # create a node with random name, port, etc + # create a node with a random name, port, and data directory with testgres.get_new_node() as node: - # run inidb + # run initdb node.init() # start PostgreSQL node.start() - # execute a query in a default DB + # execute a query in the default database print(node.execute('select 1')) - # ... node stops and its files are about to be removed + # the node is stopped and its files are removed automatically + +Query helpers +------------- + +``testgres`` provides four helpers for executing queries against the node: + +========================== ======================================================= +Command Description +========================== ======================================================= +``node.psql(query, ...)`` Runs the query via ``psql`` and returns ``(code, out, err)``. +``node.safe_psql(...)`` Returns only ``stdout`` and raises if the command fails. +``node.execute(...)`` Uses ``psycopg2``/``pg8000`` and returns a list of tuples. +``node.connect(...)`` Returns a ``NodeConnection`` for transactional usage. +========================== ======================================================= + +Example: + +.. code-block:: python + + with node.connect() as con: + con.begin('serializable') + print(con.execute('select %s', 1)) + con.rollback() + +Logging +------- + +By default ``cleanup()`` removes all temporary files (data directories, logs, and so on) created by the API. Call ``configure_testgres(node_cleanup_full=False)`` before starting nodes if you want to keep logs for inspection. + +Note: context managers (the ``with`` statement) call ``stop()`` and ``cleanup()`` automatically. + +``testgres`` integrates with the standard `Python logging `_ module, so you can aggregate logs from multiple nodes: + +.. code-block:: python + + import logging + + logging.basicConfig(filename='/tmp/testgres.log') + + testgres.configure_testgres(use_python_logging=True) + node1 = testgres.get_new_node().init().start() + node2 = testgres.get_new_node().init().start() + + node1.execute('select 1') + node2.execute('select 2') + + testgres.configure_testgres(use_python_logging=False) Backup & replication -------------------- @@ -72,12 +119,90 @@ It's quite easy to create a backup and start a new replica: # create and start a new replica replica = backup.spawn_replica('replica').start() - # catch up with master node replica.catchup() - # execute a dummy query print(replica.execute('postgres', 'select 1')) +Benchmarks +---------- + +Use ``pgbench`` through ``testgres`` to run quick benchmarks: + +.. code-block:: python + + with testgres.get_new_node('master') as master: + master.init().start() + + result = master.pgbench_init(scale=2).pgbench_run(time=10) + print(result) + +Custom configuration +-------------------- + +``testgres`` ships with sensible defaults. Adjust them as needed with ``default_conf()`` and ``append_conf()``: + +.. code-block:: python + + extra_conf = "shared_preload_libraries = 'postgres_fdw'" + + with testgres.get_new_node().init() as master: + master.default_conf(fsync=True, allow_streaming=True) + master.append_conf('postgresql.conf', extra_conf) + +``default_conf()`` is called by ``init()`` and rewrites the configuration file. Apply ``append_conf()`` afterwards to keep custom lines. + +Remote mode +----------- + +Provision nodes on a remote host (Linux only) by wiring ``RemoteOperations`` into the configuration: + +.. code-block:: python + + from testgres import ConnectionParams, RemoteOperations, TestgresConfig, get_remote_node + + conn_params = ConnectionParams( + host='example.com', + username='postgres', + ssh_key='/path/to/ssh/key' + ) + os_ops = RemoteOperations(conn_params) + + TestgresConfig.set_os_ops(os_ops=os_ops) + + def test_basic_query(): + with get_remote_node(conn_params=conn_params) as node: + node.init().start() + assert node.execute('SELECT 1') == [(1,)] + +Pytest integration +------------------ + +Use fixtures to create and clean up nodes automatically when testing with ``pytest``: + +.. code-block:: python + + import pytest + import testgres + + @pytest.fixture + def pg_node(): + node = testgres.get_new_node().init().start() + try: + yield node + finally: + node.stop() + node.cleanup() + + def test_simple(pg_node): + assert pg_node.execute('select 1')[0][0] == 1 + +Scaling tips +------------ + +* Run tests in parallel with ``pytest -n auto`` (requires ``pytest-xdist``). Set unique ports by passing ``port`` to ``get_new_node()`` or exporting ``PGPORT`` in the fixture. +* Always call ``node.cleanup()`` after each test, or rely on context managers/fixtures that do it for you, to avoid leftover data directories. +* Prefer ``safe_psql()`` for quick assertions, and ``execute()`` when you need Python data structures. + Modules ======= diff --git a/hooks/README.md b/hooks/README.md deleted file mode 100644 index 607a38ea..00000000 --- a/hooks/README.md +++ /dev/null @@ -1,7 +0,0 @@ -### What's this? - -This is a set of git hooks to be executed on special events, e.g. before you commit your changes. To install them, just execute the `install.sh` script, and you're good to go! - -### What do they do? - -Currently there's only one hook (`pre-commit`) which formats changed python files with `yapf`. diff --git a/hooks/install.sh b/hooks/install.sh deleted file mode 100755 index d1ea0366..00000000 --- a/hooks/install.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash -DIR=$(dirname $0) -ln -s -f ../../hooks/pre-commit "$DIR/../.git/hooks/" diff --git a/hooks/pre-commit b/hooks/pre-commit deleted file mode 100755 index 52531d14..00000000 --- a/hooks/pre-commit +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash - -set -e - -# capture the changed files that have been staged -changed_files=$(git diff --staged --name-only) - -for file in ${changed_files} -do - if [[ "${file##*.}" == "py" ]]; then - if command -v yapf > /dev/null; then - echo "Run yapf on ${file}" - yapf ${file} -i - git add ${file} - fi - - if command -v flake8 > /dev/null; then - echo "Run flake8 on ${file}" - flake8 ${file} - fi - fi -done - diff --git a/mk_dockerfile.sh b/mk_dockerfile.sh deleted file mode 100755 index d2aa3a8a..00000000 --- a/mk_dockerfile.sh +++ /dev/null @@ -1,2 +0,0 @@ -set -eu -sed -e 's/${PYTHON_VERSION}/'${PYTHON_VERSION}/g -e 's/${PG_VERSION}/'${PG_VERSION}/g Dockerfile.tmpl > Dockerfile diff --git a/publish_package.sh b/publish_package.sh deleted file mode 100755 index 9cc56e94..00000000 --- a/publish_package.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env bash - -set -eux - -# prepare environment -venv_path=.venv -rm -rf "$venv_path" -virtualenv "$venv_path" -export VIRTUAL_ENV_DISABLE_PROMPT=1 -. "$venv_path"/bin/activate - -# install utilities -pip3 install setuptools twine - -# create distribution of the package -python3 setup.py sdist bdist_wheel - -# upload dist -twine upload dist/* - -set +eux - diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..1a790e80 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,72 @@ +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" + +[tool.setuptools.package-dir] +"testgres" = "src" + +[tool.setuptools.dynamic] +version = {attr = "testgres.__version__"} + +[tool.flake8] +extend-ignore = ["E501"] +exclude = [".git", "__pycache__", "env", "venv"] + +# Pytest settings +[tool.pytest.ini_options] + +testpaths = ["tests"] +log_file_level = "NOTSET" +log_file_format = "%(levelname)8s [%(asctime)s] %(message)s" +log_file_date_format = "%Y-%m-%d %H:%M:%S" + +[project] +name = "testgres" +dynamic = ["version"] + +description = "Testing utility for PostgreSQL and its extensions" +readme = "README.md" + +# [2026-01-05] +# This old format is used to ensure compatibility with Python 3.7. +license = {text = "PostgreSQL"} + +authors = [ + {name = "Postgres Professional", email = "testgres@postgrespro.ru"}, +] + +keywords = [ + 'test', + 'testing', + 'postgresql', +] + +requires-python = ">=3.7.3" + +classifiers = [ + "Intended Audience :: Developers", + "Operating System :: Unix", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Topic :: Software Development :: Libraries", + "Topic :: Software Development :: Testing", +] + +dependencies = [ + "pg8000", + "port-for>=0.4", + "six>=1.9.0", + "psutil", + "packaging", + "testgres.os_ops>=3.2.0,<4.0.0", +] + +[project.urls] +"HomePage" = "https://github.com/postgrespro/testgres" diff --git a/run_tests.sh b/run_tests.sh index 73c459be..0f98b60d 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -1,68 +1,167 @@ #!/usr/bin/env bash -# Copyright (c) 2017-2023 Postgres Professional - set -eux +if [ -z ${TEST_FILTER+x} ]; \ +then export TEST_FILTER="TestTestgresLocal or (TestTestgresCommon and (not remote))"; \ +fi -# choose python version -echo python version is $PYTHON_VERSION -VIRTUALENV="virtualenv --python=/usr/bin/python$PYTHON_VERSION" -PIP="pip$PYTHON_VERSION" +echo NPROC: $(nproc) # fail early echo check that pg_config is in PATH command -v pg_config -# prepare environment -VENV_PATH=/tmp/testgres_venv +# prepare python environment +VENV_PATH="/tmp/testgres_venv" rm -rf $VENV_PATH -$VIRTUALENV $VENV_PATH +${PYTHON_BINARY} -m venv "${VENV_PATH}" export VIRTUAL_ENV_DISABLE_PROMPT=1 -source $VENV_PATH/bin/activate - -# install utilities -$PIP install coverage flake8 psutil Sphinx - -# install testgres' dependencies -export PYTHONPATH=$(pwd) -$PIP install . - -# test code quality -flake8 . - +source "${VENV_PATH}/bin/activate" +pip install --upgrade pip setuptools wheel +pip install -r tests/requirements.txt # remove existing coverage file export COVERAGE_FILE=.coverage rm -f $COVERAGE_FILE +pip install coverage + +if [ -n "${TEST_CFG__REMOTE_HOST:-}" ] && [ -n "${TEST_CFG__REMOTE_USERNAME:-}" ]; then + cmd_str="ssh" + + if [ -n "${TEST_CFG__REMOTE_PASSWORD:-}" ]; then + cmd_str="sshpass -p \"$TEST_CFG__REMOTE_PASSWORD\" $cmd_str" + fi + + [ -n "${TEST_CFG__REMOTE_SSH_KEY:-}" ] && cmd_str="$cmd_str -i \"$TEST_CFG__REMOTE_SSH_KEY\"" + [ -n "${TEST_CFG__REMOTE_PORT:-}" ] && cmd_str="$cmd_str -p \"$TEST_CFG__REMOTE_PORT\"" + + REMOTE_SSH_PREFIX="$cmd_str \"$TEST_CFG__REMOTE_USERNAME@$TEST_CFG__REMOTE_HOST\"" +else + REMOTE_SSH_PREFIX="" +fi + +exec_command() { + local cmd="$1" + local prefix="$2" + + eval "$prefix $cmd" +} + +show_fs_state__impl() { + local prefix="$1" + local host_label="$2" + + set +x + echo "------------- ${host_label} FS STATE" + set -x + exec_command "df -T" "$prefix" +} + +check_leftover_ports__impl() { + local prefix="$1" + local host_label="$2" + local ports_dir="/tmp/testgres/ports" + + set +x + echo "------------- Checking ${host_label} ports lock directory" + set -x + + # Check command: will print FOUND if the directory exists and is not empty + local check_cmd="if [ -d '${ports_dir}' ] && [ \"\$(ls -A '${ports_dir}' 2>/dev/null)\" ]; then echo 'FOUND'; fi" + + # Temporarily disable bash's instant drop (set +e) to safely intercept the result + set +e + local result + result=$(exec_command "$check_cmd" "$prefix") + set -e + + set +x + if [ "$result" = "FOUND" ]; then + echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" + echo "ERROR: Leftover ports detected in $ports_dir on $host_label machine!" + echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" + set -x + + # We display a list of frozen ports so that the culprits can be identified + exec_command "ls -la '$ports_dir'" "$prefix" + + # We hard-drop the entire control script + # sleep 3600 + exit 1 + else + echo "Clear. No leftover port locks." + fi + set -x +} + +fs_verification__impl() { + show_fs_state__impl "$1" "$2" + + check_leftover_ports__impl "$1" "$2" +} + +fs_verification() { + fs_verification__impl "" "LOCAL" + + if [ -n "$REMOTE_SSH_PREFIX" ]; then + fs_verification__impl "$REMOTE_SSH_PREFIX" "REMOTE" + fi +} + +# ---------------------------------------- PATH + +fs_verification # run tests (PATH) -time coverage run -a tests/test_simple.py +time coverage run -a -m pytest -l -vvv -n 4 -k "${TEST_FILTER}" + +# ---------------------------------------- PG_BIN +fs_verification # run tests (PG_BIN) -time \ - PG_BIN=$(dirname $(which pg_config)) \ - ALT_CONFIG=1 \ - coverage run -a tests/test_simple.py +PG_BIN=$(pg_config --bindir) \ +time coverage run -a -m pytest -l -vvv -n 4 -k "${TEST_FILTER}" +# ---------------------------------------- PG_CONFIG + +fs_verification # run tests (PG_CONFIG) -time \ - PG_CONFIG=$(which pg_config) \ - ALT_CONFIG=1 \ - coverage run -a tests/test_simple.py +PG_CONFIG=$(pg_config --bindir)/pg_config \ +time coverage run -a -m pytest -l -vvv -n 4 -k "${TEST_FILTER}" + +# ---------------------------------------- pg8000 +fs_verification + +# test pg8000 +pip uninstall -y psycopg2 +pip install pg8000 +PG_CONFIG=$(pg_config --bindir)/pg_config \ +time coverage run -a -m pytest -l -vvv -n 4 -k "${TEST_FILTER}" + +# ---------------------------------------- finish + +fs_verification + +# ---------------------------------------- coverage -# show coverage coverage report +pip uninstall -y coverage + # build documentation +pip install Sphinx + cd docs make html cd .. +pip uninstall -y Sphinx + # attempt to fix codecov set +eux diff --git a/run_tests2.sh b/run_tests2.sh new file mode 100755 index 00000000..82b1b983 --- /dev/null +++ b/run_tests2.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +set -eux + +eval "$(pyenv init -)" +eval "$(pyenv virtualenv-init -)" + +pyenv virtualenv --force ${PYTHON_VERSION} cur +pyenv activate cur + +./run_tests.sh diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index ba6a57fc..00000000 --- a/setup.cfg +++ /dev/null @@ -1,6 +0,0 @@ -[metadata] -description-file = README.md - -[flake8] -ignore = E501 -exclude = .git,__pycache__,env,venv,testgres/__init__.py diff --git a/setup.py b/setup.py deleted file mode 100755 index 16d4c300..00000000 --- a/setup.py +++ /dev/null @@ -1,43 +0,0 @@ -import sys - -try: - from setuptools import setup -except ImportError: - from distutils.core import setup - -# Basic dependencies -install_requires = [ - "pg8000", - "port-for>=0.4", - "six>=1.9.0", - "psutil", - "packaging" -] - -# Add compatibility enum class -if sys.version_info < (3, 4): - install_requires.append("enum34") - -# Add compatibility ipaddress module -if sys.version_info < (3, 3): - install_requires.append("ipaddress") - -# Get contents of README file -with open('README.md', 'r') as f: - readme = f.read() - -setup( - version='1.9.2', - name='testgres', - packages=['testgres', 'testgres.operations'], - description='Testing utility for PostgreSQL and its extensions', - url='https://github.com/postgrespro/testgres', - long_description=readme, - long_description_content_type='text/markdown', - license='PostgreSQL', - author='Ildar Musin', - author_email='zildermann@gmail.com', - keywords=['test', 'testing', 'postgresql'], - install_requires=install_requires, - classifiers=[], -) diff --git a/testgres/__init__.py b/src/__init__.py similarity index 60% rename from testgres/__init__.py rename to src/__init__.py index 383daf2d..af917a1b 100644 --- a/testgres/__init__.py +++ b/src/__init__.py @@ -19,11 +19,13 @@ TestgresException, \ ExecUtilException, \ QueryException, \ + QueryTimeoutException, \ TimeoutException, \ CatchUpException, \ StartNodeException, \ InitNodeException, \ - BackupException + BackupException, \ + InvalidOperationException from .enums import \ XLogMethod, \ @@ -32,15 +34,18 @@ ProcessType, \ DumpFormat -from .node import PostgresNode, NodeApp +from .node import PostgresNode +from .node import PortManager +from .node_app import NodeApp from .utils import \ reserve_port, \ release_port, \ - bound_ports, \ get_bin_path, \ + get_bin_dir, \ get_pg_config, \ - get_pg_version + get_pg_version, \ + parse_pg_version from .standby import \ First, \ @@ -48,9 +53,11 @@ from .config import testgres_config -from .operations.os_ops import OsOperations, ConnectionParams -from .operations.local_ops import LocalOperations -from .operations.remote_ops import RemoteOperations +from testgres.operations.os_ops import OsOperations, ConnectionParams +from testgres.operations.local_ops import LocalOperations +from testgres.operations.remote_ops import RemoteOperations + +__version__ = "1.15.2" __all__ = [ "get_new_node", @@ -58,10 +65,14 @@ "NodeBackup", "testgres_config", "TestgresConfig", "configure_testgres", "scoped_config", "push_config", "pop_config", "NodeConnection", "DatabaseError", "InternalError", "ProgrammingError", "OperationalError", - "TestgresException", "ExecUtilException", "QueryException", "TimeoutException", "CatchUpException", "StartNodeException", "InitNodeException", "BackupException", + "TestgresException", "ExecUtilException", "QueryException", + "QueryTimeoutException", + "TimeoutException", "CatchUpException", "StartNodeException", "InitNodeException", "BackupException", "InvalidOperationException", "XLogMethod", "IsolationLevel", "NodeStatus", "ProcessType", "DumpFormat", - "PostgresNode", "NodeApp", - "reserve_port", "release_port", "bound_ports", "get_bin_path", "get_pg_config", "get_pg_version", + "NodeApp", + "PostgresNode", + "PortManager", + "reserve_port", "release_port", "get_bin_path", "get_bin_dir", "get_pg_config", "get_pg_version", "parse_pg_version", "First", "Any", "OsOperations", "LocalOperations", "RemoteOperations", "ConnectionParams" ] diff --git a/testgres/api.py b/src/api.py similarity index 77% rename from testgres/api.py rename to src/api.py index e4b1cdd5..d5e3d91d 100644 --- a/testgres/api.py +++ b/src/api.py @@ -31,6 +31,10 @@ [(3,)] """ from .node import PostgresNode +from testgres.operations.remote_ops import ConnectionParams +from testgres.operations.remote_ops import RemoteOperations + +import typing def get_new_node(name=None, base_dir=None, **kwargs): @@ -42,13 +46,16 @@ def get_new_node(name=None, base_dir=None, **kwargs): return PostgresNode(name=name, base_dir=base_dir, **kwargs) -def get_remote_node(name=None, conn_params=None): +def get_remote_node(name=None, conn_params: typing.Optional[ConnectionParams] = None): """ Simply a wrapper around :class:`.PostgresNode` constructor for remote node. See :meth:`.PostgresNode.__init__` for details. For remote connection you can add the next parameter: - conn_params = ConnectionParams(host='127.0.0.1', - ssh_key=None, - username=default_username()) + conn_params = ConnectionParams(host='127.0.0.1', ssh_key=None, username=default_username()) """ - return get_new_node(name=name, conn_params=conn_params) + + if conn_params is None: + raise ValueError("Argument 'conn_params' is None.") + + os_ops = RemoteOperations(conn_params) + return PostgresNode(name=name, os_ops=os_ops) diff --git a/testgres/backup.py b/src/backup.py similarity index 74% rename from testgres/backup.py rename to src/backup.py index a89e214d..d91db5a0 100644 --- a/testgres/backup.py +++ b/src/backup.py @@ -1,7 +1,5 @@ # coding: utf-8 -import os - from six import raise_from from .enums import XLogMethod @@ -15,9 +13,11 @@ from .exceptions import BackupException +from testgres.operations.os_ops import OsOperations + from .utils import \ - get_bin_path, \ - execute_utility, \ + get_bin_path2, \ + execute_utility2, \ clean_on_error @@ -27,13 +27,16 @@ class NodeBackup(object): """ @property def log_file(self): - return os.path.join(self.base_dir, BACKUP_LOG_FILE) + assert self.os_ops is not None + assert isinstance(self.os_ops, OsOperations) + return self.os_ops.build_path(self.base_dir, BACKUP_LOG_FILE) def __init__(self, node, base_dir=None, username=None, - xlog_method=XLogMethod.fetch): + xlog_method=XLogMethod.fetch, + options=None): """ Create a new backup. @@ -43,6 +46,11 @@ def __init__(self, username: database user name. xlog_method: none | fetch | stream (see docs) """ + assert node.os_ops is not None + assert isinstance(node.os_ops, OsOperations) + + if not options: + options = [] self.os_ops = node.os_ops if not node.status(): raise BackupException('Node must be running') @@ -67,17 +75,18 @@ def __init__(self, # private self._available = True - data_dir = os.path.join(self.base_dir, DATA_DIR) + data_dir = self.os_ops.build_path(self.base_dir, DATA_DIR) _params = [ - get_bin_path("pg_basebackup"), + get_bin_path2(self.os_ops, "pg_basebackup"), "-p", str(node.port), "-h", node.host, "-U", username, "-D", data_dir, "-X", xlog_method.value ] # yapf: disable - execute_utility(_params, self.log_file) + _params += options + execute_utility2(self.os_ops, _params, self.log_file) def __enter__(self): return self @@ -103,10 +112,13 @@ def _prepare_dir(self, destroy): available = not destroy if available: + assert self.os_ops is not None + assert isinstance(self.os_ops, OsOperations) + dest_base_dir = self.os_ops.mkdtemp(prefix=TMP_NODE) - data1 = os.path.join(self.base_dir, DATA_DIR) - data2 = os.path.join(dest_base_dir, DATA_DIR) + data1 = self.os_ops.build_path(self.base_dir, DATA_DIR) + data2 = self.os_ops.build_path(dest_base_dir, DATA_DIR) try: # Copy backup to new data dir @@ -138,12 +150,14 @@ def spawn_primary(self, name=None, destroy=True): base_dir = self._prepare_dir(destroy) # Build a new PostgresNode - NodeClass = self.original_node.__class__ - with clean_on_error(NodeClass(name=name, base_dir=base_dir, conn_params=self.original_node.os_ops.conn_params)) as node: + assert self.original_node is not None + + node = self.original_node.clone_with_new_name_and_base_dir(name=name, base_dir=base_dir) - # New nodes should always remove dir tree - node._should_rm_dirs = True + assert node is not None + assert type(node) is self.original_node.__class__ + with clean_on_error(node) as node: # Set a new port node.append_conf(filename=PG_CONF_FILE, line='\n') node.append_conf(filename=PG_CONF_FILE, port=node.port) @@ -164,14 +178,19 @@ def spawn_replica(self, name=None, destroy=True, slot=None): """ # Build a new PostgresNode - with clean_on_error(self.spawn_primary(name=name, - destroy=destroy)) as node: + node = self.spawn_primary(name=name, destroy=destroy) + assert node is not None + try: # Assign it a master and a recovery file (private magic) node._assign_master(self.original_node) node._create_recovery_conf(username=self.username, slot=slot) + except: # noqa: E722 + # TODO: Pass 'final=True' ? + node.cleanup(release_resources=True) + raise - return node + return node def cleanup(self): """ diff --git a/testgres/cache.py b/src/cache.py similarity index 64% rename from testgres/cache.py rename to src/cache.py index 21198e83..72ec5698 100644 --- a/testgres/cache.py +++ b/src/cache.py @@ -1,7 +1,5 @@ # coding: utf-8 -import os - from six import raise_from from .config import testgres_config @@ -15,26 +13,43 @@ ExecUtilException from .utils import \ - get_bin_path, \ - execute_utility + get_bin_path2, \ + execute_utility2 -from .operations.local_ops import LocalOperations -from .operations.os_ops import OsOperations +from testgres.operations.local_ops import LocalOperations +from testgres.operations.os_ops import OsOperations -def cached_initdb(data_dir, logfile=None, params=None, os_ops: OsOperations = LocalOperations()): +def cached_initdb(data_dir, logfile=None, params=None, os_ops: OsOperations = None, bin_path=None, cached=True): """ Perform initdb or use cached node files. """ + assert os_ops is None or isinstance(os_ops, OsOperations) + + if os_ops is None: + os_ops = LocalOperations.get_single_instance() + + assert isinstance(os_ops, OsOperations) + + def make_utility_path(name): + assert name is not None + assert type(name) is str + + if bin_path: + return os_ops.build_path(bin_path, name) + + return get_bin_path2(os_ops, name) + def call_initdb(initdb_dir, log=logfile): try: - _params = [get_bin_path("initdb"), "-D", initdb_dir, "-N"] - execute_utility(_params + (params or []), log) + initdb_path = make_utility_path("initdb") + _params = [initdb_path, "-D", initdb_dir, "-N"] + execute_utility2(os_ops, _params + (params or []), log) except ExecUtilException as e: raise_from(InitNodeException("Failed to run initdb"), e) - if params or not testgres_config.cache_initdb: + if params or not testgres_config.cache_initdb or not cached: call_initdb(data_dir, logfile) else: # Fetch cached initdb dir @@ -55,15 +70,15 @@ def call_initdb(initdb_dir, log=logfile): # XXX: write new unique system id to control file # Some users might rely upon unique system ids, but # our initdb caching mechanism breaks this contract. - pg_control = os.path.join(data_dir, XLOG_CONTROL_FILE) + pg_control = os_ops.build_path(data_dir, XLOG_CONTROL_FILE) system_id = generate_system_id() cur_pg_control = os_ops.read(pg_control, binary=True) new_pg_control = system_id + cur_pg_control[len(system_id):] os_ops.write(pg_control, new_pg_control, truncate=True, binary=True, read_and_write=True) # XXX: build new WAL segment with our system id - _params = [get_bin_path("pg_resetwal"), "-D", data_dir, "-f"] - execute_utility(_params, logfile) + _params = [make_utility_path("pg_resetwal"), "-D", data_dir, "-f"] + execute_utility2(os_ops, _params, logfile) except ExecUtilException as e: msg = "Failed to reset WAL for system id" diff --git a/testgres/config.py b/src/config.py similarity index 92% rename from testgres/config.py rename to src/config.py index b6c43926..1d09ccb8 100644 --- a/testgres/config.py +++ b/src/config.py @@ -2,13 +2,19 @@ import atexit import copy +import logging +import os import tempfile from contextlib import contextmanager from .consts import TMP_CACHE -from .operations.os_ops import OsOperations -from .operations.local_ops import LocalOperations +from testgres.operations.os_ops import OsOperations +from testgres.operations.local_ops import LocalOperations + +log_level = os.getenv('LOGGING_LEVEL', 'WARNING').upper() +log_format = os.getenv('LOGGING_FORMAT', '%(asctime)s - %(levelname)s - %(message)s') +logging.basicConfig(level=log_level, format=log_format) class GlobalConfig(object): @@ -44,8 +50,9 @@ class GlobalConfig(object): _cached_initdb_dir = None """ underlying class attribute for cached_initdb_dir property """ - os_ops = LocalOperations() + os_ops = LocalOperations.get_single_instance() """ OsOperation object that allows work on remote host """ + @property def cached_initdb_dir(self): """ path to a temp directory for cached initdb. """ diff --git a/testgres/connection.py b/src/connection.py similarity index 81% rename from testgres/connection.py rename to src/connection.py index aeb040ce..b16edb23 100644 --- a/testgres/connection.py +++ b/src/connection.py @@ -1,4 +1,5 @@ # coding: utf-8 +import logging # we support both pg8000 and psycopg2 try: @@ -13,7 +14,7 @@ from .defaults import \ default_dbname, \ - default_username + default_username2 from .exceptions import QueryException @@ -37,15 +38,17 @@ def __init__(self, # Set default arguments dbname = dbname or default_dbname() - username = username or default_username() + username = username or default_username2(node.os_ops) self._node = node - self._connection = node.os_ops.db_connect(dbname=dbname, - user=username, - password=password, - host=node.host, - port=node.port) + self._connection = pglib.connect( + database=dbname, + user=username, + password=password, + host=node.host, + port=node.port + ) self._connection.autocommit = autocommit self._cursor = self.connection.cursor() @@ -104,14 +107,13 @@ def rollback(self): def execute(self, query, *args): self.cursor.execute(query, args) try: - res = self.cursor.fetchall() # pg8000 might return tuples - if isinstance(res, tuple): - res = [tuple(t) for t in res] - + res = [tuple(t) for t in self.cursor.fetchall()] return res + except ProgrammingError: + return None except Exception as e: - print("Error executing query: {}".format(e)) + logging.error("Error executing query: {}\n {}".format(repr(e), query)) return None def close(self): diff --git a/testgres/consts.py b/src/consts.py similarity index 83% rename from testgres/consts.py rename to src/consts.py index 98c84af6..d3589205 100644 --- a/testgres/consts.py +++ b/src/consts.py @@ -10,6 +10,10 @@ TMP_CACHE = 'tgsc_' TMP_BACKUP = 'tgsb_' +TMP_TESTGRES = "testgres" + +TMP_TESTGRES_PORTS = TMP_TESTGRES + "/ports" + # path to control file XLOG_CONTROL_FILE = "global/pg_control" @@ -35,3 +39,7 @@ # logical replication settings LOGICAL_REPL_MAX_CATCHUP_ATTEMPTS = 60 + +PG_CTL__STATUS__OK = 0 +PG_CTL__STATUS__NODE_IS_STOPPED = 3 +PG_CTL__STATUS__BAD_DATADIR = 4 diff --git a/testgres/decorators.py b/src/decorators.py similarity index 100% rename from testgres/decorators.py rename to src/decorators.py diff --git a/testgres/defaults.py b/src/defaults.py similarity index 59% rename from testgres/defaults.py rename to src/defaults.py index d77361d7..0f79feef 100644 --- a/testgres/defaults.py +++ b/src/defaults.py @@ -1,6 +1,9 @@ import datetime import struct import uuid +import typing + +from testgres.operations.os_ops import OsOperations from .config import testgres_config as tconf @@ -13,11 +16,30 @@ def default_dbname(): return 'postgres' -def default_username(): +def default_username(os_ops: typing.Optional[OsOperations] = None) -> str: """ Return default username (current user). """ - return tconf.os_ops.get_user() + assert os_ops is None or isinstance(os_ops, OsOperations) + + if os_ops is None: + os_ops = tconf.os_ops + + assert isinstance(os_ops, OsOperations) + result = default_username2(os_ops) + assert type(result) is str + return result + + +def default_username2(os_ops: OsOperations) -> str: + """ + Return default username (current user). + """ + assert isinstance(os_ops, OsOperations) + + result = os_ops.get_user() + assert type(result) is str + return result def generate_app_name(): diff --git a/testgres/enums.py b/src/enums.py similarity index 97% rename from testgres/enums.py rename to src/enums.py index d07d8068..a483c5b4 100644 --- a/testgres/enums.py +++ b/src/enums.py @@ -29,7 +29,7 @@ class NodeStatus(IntEnum): Status of a PostgresNode """ - Running, Stopped, Uninitialized = range(3) + Running, Stopped, Uninitialized, Zombie = range(4) # for Python 3.x def __bool__(self): diff --git a/src/exceptions.py b/src/exceptions.py new file mode 100644 index 00000000..743b11b4 --- /dev/null +++ b/src/exceptions.py @@ -0,0 +1,300 @@ +# coding: utf-8 + +import six +import typing + +from testgres.operations.exceptions import TestgresException +from testgres.operations.exceptions import ExecUtilException +from testgres.operations.exceptions import InvalidOperationException + + +class PortForException(TestgresException): + _message: typing.Optional[str] + + def __init__( + self, + message: typing.Optional[str] = None, + ): + assert message is None or type(message) is str + super().__init__(message) + self._message = message + return + + @property + def message(self) -> str: + assert self._message is None or type(self._message) is str + if self._message is None: + return "" + return self._message + + def __repr__(self) -> str: + args = [] + + if self._message is not None: + args.append(("message", self._message)) + + result = "{}(".format(type(self).__name__) + sep = "" + for a in args: + result += sep + a[0] + "=" + repr(a[1]) + sep = ", " + continue + result += ")" + return result + + +@six.python_2_unicode_compatible +class QueryException(TestgresException): + _description: typing.Optional[str] + _query: typing.Optional[str] + + def __init__( + self, + message: typing.Optional[str] = None, + query: typing.Optional[str] = None + ): + assert message is None or type(message) is str + assert query is None or type(query) is str + + super().__init__(message) + + self._description = message + self._query = query + return + + @property + def message(self) -> str: + assert self._description is None or type(self._description) is str + assert self._query is None or type(self._query) is str + + msg = [] + + if self._description: + msg.append(self._description) + + if self._query: + msg.append(u'Query: {}'.format(self._query)) + + r = six.text_type('\n').join(msg) + assert type(r) is str + return r + + @property + def description(self) -> typing.Optional[str]: + assert self._description is None or type(self._description) is str + return self._description + + @property + def query(self) -> typing.Optional[str]: + assert self._query is None or type(self._query) is str + return self._query + + def __repr__(self) -> str: + args = [] + + if self._description is not None: + args.append(("message", self._description)) + + if self._query is not None: + args.append(("query", self._query)) + + result = "{}(".format(type(self).__name__) + sep = "" + for a in args: + result += sep + a[0] + "=" + repr(a[1]) + sep = ", " + continue + result += ")" + return result + + +class QueryTimeoutException(QueryException): + def __init__( + self, + message: typing.Optional[str] = None, + query: typing.Optional[str] = None + ): + assert message is None or type(message) is str + assert query is None or type(query) is str + + super().__init__(message, query) + return + + +# [2026-01-10] To backward compatibility. +TimeoutException = QueryTimeoutException + + +# [2026-01-10] It inherits TestgresException now, not QueryException +class CatchUpException(TestgresException): + _message: typing.Optional[str] + + def __init__( + self, + message: typing.Optional[str] = None, + ): + assert message is None or type(message) is str + super().__init__(message) + self._message = message + return + + @property + def message(self) -> str: + assert self._message is None or type(self._message) is str + if self._message is None: + return "" + return self._message + + def __repr__(self) -> str: + args = [] + + if self._message is not None: + args.append(("message", self._message)) + + result = "{}(".format(type(self).__name__) + sep = "" + for a in args: + result += sep + a[0] + "=" + repr(a[1]) + sep = ", " + continue + result += ")" + return result + + +@six.python_2_unicode_compatible +class StartNodeException(TestgresException): + _description: typing.Optional[str] + _files: typing.Optional[typing.Iterable] + + def __init__( + self, + message: typing.Optional[str] = None, + files: typing.Optional[typing.Iterable] = None + ): + assert message is None or type(message) is str + assert files is None or isinstance(files, typing.Iterable) + + super().__init__(message) + + self._description = message + self._files = files + return + + @property + def message(self) -> str: + assert self._description is None or type(self._description) is str + assert self._files is None or isinstance(self._files, typing.Iterable) + + msg = [] + + if self._description: + msg.append(self._description) + + for f, lines in self._files or []: + assert type(f) is str + assert type(lines) in [str, bytes] + msg.append(u'{}\n----\n{}\n'.format(f, lines)) + + return six.text_type('\n').join(msg) + + @property + def description(self) -> typing.Optional[str]: + assert self._description is None or type(self._description) is str + return self._description + + @property + def files(self) -> typing.Optional[typing.Iterable]: + assert self._files is None or isinstance(self._files, typing.Iterable) + return self._files + + def __repr__(self) -> str: + args = [] + + if self._description is not None: + args.append(("message", self._description)) + + if self._files is not None: + args.append(("files", self._files)) + + result = "{}(".format(type(self).__name__) + sep = "" + for a in args: + result += sep + a[0] + "=" + repr(a[1]) + sep = ", " + continue + result += ")" + return result + + +class InitNodeException(TestgresException): + _message: typing.Optional[str] + + def __init__( + self, + message: typing.Optional[str] = None, + ): + assert message is None or type(message) is str + super().__init__(message) + self._message = message + return + + @property + def message(self) -> str: + assert self._message is None or type(self._message) is str + if self._message is None: + return "" + return self._message + + def __repr__(self) -> str: + args = [] + + if self._message is not None: + args.append(("message", self._message)) + + result = "{}(".format(type(self).__name__) + sep = "" + for a in args: + result += sep + a[0] + "=" + repr(a[1]) + sep = ", " + continue + result += ")" + return result + + +class BackupException(TestgresException): + _message: typing.Optional[str] + + def __init__( + self, + message: typing.Optional[str] = None, + ): + assert message is None or type(message) is str + super().__init__(message) + self._message = message + return + + @property + def message(self) -> str: + assert self._message is None or type(self._message) is str + if self._message is None: + return "" + return self._message + + def __repr__(self) -> str: + args = [] + + if self._message is not None: + args.append(("message", self._message)) + + result = "{}(".format(type(self).__name__) + sep = "" + for a in args: + result += sep + a[0] + "=" + repr(a[1]) + sep = ", " + continue + result += ")" + return result + + +assert ExecUtilException.__name__ == "ExecUtilException" +assert InvalidOperationException.__name__ == "InvalidOperationException" diff --git a/src/impl/file_line_reader.py b/src/impl/file_line_reader.py new file mode 100755 index 00000000..6bd37cf7 --- /dev/null +++ b/src/impl/file_line_reader.py @@ -0,0 +1,87 @@ +# coding: utf-8 + +from . import internal_utils + +import typing + +from testgres.operations.os_ops import OsOperations + + +class FileLineReader: + _os_ops: OsOperations + _file_name: str + _file_encoding: str + _file_pos: int + _buffer_pos: int + _buffer: bytes + + # -------------------------------------------------------------------- + def __init__( + self, + os_ops: OsOperations, + file_name: str, + file_encoding: str = "utf-8", + file_pos: int = 0 + ): + assert isinstance(os_ops, OsOperations) + assert type(file_encoding) is str + self._os_ops = os_ops + self._file_name = file_name + self._file_encoding = file_encoding + self._file_pos = file_pos + self._buffer_pos = 0 + self._buffer = internal_utils.read_line_to_pos__bin( + os_ops, + file_name, + file_pos, + ) + return + + # interface ---------------------------------------------------------- + def read_line(self) -> typing.Optional[str]: + assert isinstance(self._os_ops, OsOperations) + assert type(self._buffer_pos) is int + assert type(self._buffer) is bytes + assert self._buffer_pos >= 0 + assert self._buffer_pos <= len(self._buffer) + + scan_pos = self._buffer_pos + + while True: + sz1 = len(self._buffer) + + if scan_pos == sz1: + block = self._os_ops.read_binary( + self._file_name, + self._file_pos, + ) + assert type(block) is bytes + self._buffer += block + self._file_pos += len(block) + + x = self._buffer.find(b'\n', scan_pos) + + sz2 = len(self._buffer) + + if x == -1: + if scan_pos == sz2: + return None + + if self._buffer_pos == 0: + scan_pos = sz2 + else: + assert self._buffer_pos > 0 + self._buffer = self._buffer[self._buffer_pos:] + scan_pos = sz2 - self._buffer_pos + self._buffer_pos = 0 + continue + + assert x >= 0 + assert x < sz2 + + b = self._buffer[self._buffer_pos:(x+1)] + + s = b.decode(self._file_encoding) + + self._buffer_pos = x + 1 + return s diff --git a/src/impl/internal_utils.py b/src/impl/internal_utils.py new file mode 100644 index 00000000..05d5a465 --- /dev/null +++ b/src/impl/internal_utils.py @@ -0,0 +1,92 @@ +from testgres.operations.os_ops import OsOperations + +import logging +import typing + + +def send_log(level: int, msg: str) -> None: + assert type(level) is int + assert type(msg) is str + + return logging.log(level, "[testgres] " + msg) + + +def send_log_info(msg: str) -> None: + assert type(msg) is str + + return send_log(logging.INFO, msg) + + +def send_log_debug(msg: str) -> None: + assert type(msg) is str + + return send_log(logging.DEBUG, msg) + + +def read_line_to_pos__bin( + os_ops: OsOperations, + filename: str, + position: int, +) -> bytes: + assert type(filename) is str + assert type(position) is int + assert len(filename) > 0 + assert position >= 0 + assert os_ops is not None + assert isinstance(os_ops, OsOperations) + + if position == 0: + return b'' + + assert position > 0 + + read_position = position + result_blocks: typing.List[bytes] = [] + + C_BACK_READ_BLOCK_SIZE = 4096 + + while read_position > 0: + if read_position < C_BACK_READ_BLOCK_SIZE: + block_sz = read_position + else: + block_sz = C_BACK_READ_BLOCK_SIZE + + assert block_sz > 0 + assert block_sz <= C_BACK_READ_BLOCK_SIZE + + read_position -= block_sz + + assert read_position < position + assert read_position >= 0 + + block = os_ops.read_binary(filename, read_position, block_sz) + + assert type(block) is bytes + + if len(block) != block_sz: + err_msg = "[BUG CHECK] Readed block has bad size ({}). Expected size is ({}). File name {}.".format( + len(block), + block_sz, + filename, + ) + raise RuntimeError(err_msg) + + assert len(block) == block_sz + + x = block.rfind(b"\n", 0, block_sz) + + if x == -1: + result_blocks.append(block) + continue + + if x == block_sz - 1: + break + + block = block[x + 1:] + result_blocks.append(block) + break + + result = b''.join(reversed(result_blocks)) + assert type(result) is bytes + assert len(result) <= (position - read_position) + return result diff --git a/src/impl/platforms/internal_platform_utils.py b/src/impl/platforms/internal_platform_utils.py new file mode 100644 index 00000000..3ae684ed --- /dev/null +++ b/src/impl/platforms/internal_platform_utils.py @@ -0,0 +1,71 @@ +from __future__ import annotations + +import enum +import typing + +from testgres.operations.os_ops import OsOperations + + +class InternalPlatformUtils: + class FindPostmasterResultCode(enum.Enum): + ok = 0 + not_found = 1, + not_implemented = 2 + many_processes = 3 + has_problems = 4 + + class FindPostmasterResult: + code: InternalPlatformUtils.FindPostmasterResultCode + pid: typing.Optional[int] + + def __init__( + self, + code: InternalPlatformUtils.FindPostmasterResultCode, + pid: typing.Optional[int] + ): + assert type(code) is InternalPlatformUtils.FindPostmasterResultCode + assert pid is None or type(pid) is int + self.code = code + self.pid = pid + return + + @staticmethod + def create_ok(pid: int) -> InternalPlatformUtils.FindPostmasterResult: + assert type(pid) is int + return __class__(InternalPlatformUtils.FindPostmasterResultCode.ok, pid) + + @staticmethod + def create_not_found() -> InternalPlatformUtils.FindPostmasterResult: + return __class__(InternalPlatformUtils.FindPostmasterResultCode.not_found, None) + + @staticmethod + def create_not_implemented() -> InternalPlatformUtils.FindPostmasterResult: + return __class__(InternalPlatformUtils.FindPostmasterResultCode.not_implemented, None) + + @staticmethod + def create_many_processes() -> InternalPlatformUtils.FindPostmasterResult: + return __class__(InternalPlatformUtils.FindPostmasterResultCode.many_processes, None) + + @staticmethod + def create_has_problems() -> InternalPlatformUtils.FindPostmasterResult: + return __class__(InternalPlatformUtils.FindPostmasterResultCode.has_problems, None) + + def FindPostmaster( + self, + os_ops: OsOperations, + bin_dir: str, + data_dir: str + ) -> FindPostmasterResult: + assert isinstance(os_ops, OsOperations) + assert type(bin_dir) is str + assert type(data_dir) is str + raise NotImplementedError("InternalPlatformUtils::FindPostmaster is not implemented.") + + def ProcessIsZombi_soft_check( + self, + os_ops: OsOperations, + pid: int, + ) -> typing.Optional[bool]: + assert isinstance(os_ops, OsOperations) + assert type(pid) is int + raise NotImplementedError("InternalPlatformUtils::ProcessIsZombi_soft_ver is not implemented.") diff --git a/src/impl/platforms/internal_platform_utils_factory.py b/src/impl/platforms/internal_platform_utils_factory.py new file mode 100644 index 00000000..1098185e --- /dev/null +++ b/src/impl/platforms/internal_platform_utils_factory.py @@ -0,0 +1,23 @@ +from .internal_platform_utils import InternalPlatformUtils + +from testgres.operations.os_ops import OsOperations + + +def create_internal_platform_utils( + os_ops: OsOperations +) -> InternalPlatformUtils: + assert isinstance(os_ops, OsOperations) + + platform_name = os_ops.get_platform() + assert type(platform_name) is str + + if platform_name == "linux": + from .linux import internal_platform_utils as x + return x.InternalPlatformUtils() + + if platform_name == "win32": + from .win32 import internal_platform_utils as x + return x.InternalPlatformUtils() + + # not implemented + return InternalPlatformUtils() diff --git a/src/impl/platforms/linux/internal_platform_utils.py b/src/impl/platforms/linux/internal_platform_utils.py new file mode 100644 index 00000000..95d9d15a --- /dev/null +++ b/src/impl/platforms/linux/internal_platform_utils.py @@ -0,0 +1,429 @@ +from __future__ import annotations + +from .. import internal_platform_utils as base +from ... import internal_utils +from ....raise_error import RaiseError + +from testgres.operations.os_ops import OsOperations +from testgres.operations.exceptions import ExecUtilException + +import re +import shlex +import typing +import time + + +class InternalPlatformUtils(base.InternalPlatformUtils): + C_MAX_FIND_POSTMASTER_ATTEMPTS = 5 + C_BASH_EXE = "/bin/bash" + + sm_exec_env = { + "LANG": "en_US.UTF-8", + "LC_ALL": "en_US.UTF-8", + } + + # -------------------------------------------------------------------- + def FindPostmaster( + self, + os_ops: OsOperations, + bin_dir: str, + data_dir: str + ) -> InternalPlatformUtils.FindPostmasterResult: + assert isinstance(os_ops, OsOperations) + assert type(bin_dir) is str + assert type(data_dir) is str + assert type(__class__.C_BASH_EXE) is str + assert type(__class__.sm_exec_env) is dict + assert len(__class__.C_BASH_EXE) > 0 + assert len(bin_dir) > 0 + assert len(data_dir) > 0 + + failures: typing.List[Exception] = [] + + postmaster_pid: typing.Optional[int] = None + + nAttempts = 0 + + while True: + nAttempts += 1 + + try: + postmaster_pid = __class__._FindPostmaster( + os_ops, + bin_dir, + data_dir, + ) + except Exception as e: + failures.append(e) + + log_msg = "FindPostmaster (bin_dir={!r}, data_dir={!r}) detects a problem. Exception {}:\n{}".format( + bin_dir, + data_dir, + type(e).__name__, + e, + ) + internal_utils.send_log_debug(log_msg) + + if nAttempts < __class__.C_MAX_FIND_POSTMASTER_ATTEMPTS: + time.sleep(0.05) + continue + + __class__._find_postmaster__throw_error__fail( + bin_dir=bin_dir, + data_dir=data_dir, + failures=failures, + ) + + break + + if postmaster_pid is None: + return InternalPlatformUtils.FindPostmasterResult.create_not_found() + + assert type(postmaster_pid) is int + + return InternalPlatformUtils.FindPostmasterResult.create_ok(postmaster_pid) + + # -------------------------------------------------------------------- + @staticmethod + def _FindPostmaster( + os_ops: OsOperations, + bin_dir: str, + data_dir: str + ) -> typing.Optional[int]: + assert isinstance(os_ops, OsOperations) + assert type(bin_dir) is str + assert type(data_dir) is str + assert type(__class__.C_BASH_EXE) is str + assert type(__class__.sm_exec_env) is dict + assert len(__class__.C_BASH_EXE) > 0 + assert len(bin_dir) > 0 + assert len(data_dir) > 0 + + pg_path_e = re.escape(os_ops.build_path(bin_dir, "postgres")) + data_dir_e = re.escape(data_dir) + + assert type(pg_path_e) is str + assert type(data_dir_e) is str + + regexp = r"^\s*[0-9]+\s+[0-9]+\s+" + pg_path_e + r"(\s+.*)?\s+\-[D]\s+" + data_dir_e + r"(\s+.*)?" + + cmd = [ + __class__.C_BASH_EXE, + "-c", + "ps -ewwo \"pid=,ppid=,args=\" | grep -E " + shlex.quote(regexp), + ] + + exec_r = os_ops.exec_command( + cmd=cmd, + ignore_errors=True, + verbose=True, + exec_env=__class__.sm_exec_env, + ) + + assert type(exec_r) is tuple + assert len(exec_r) == 3 + + exit_status, output_b, error_b = exec_r + + assert type(exit_status) is int + assert type(output_b) is bytes + assert type(error_b) is bytes + + if exit_status == 1: + return None + + output = output_b.decode("utf-8") + error = error_b.decode("utf-8") + + assert type(output) is str + assert type(error) is str + + if exit_status != 0: + errMsg = f"test command returned an unexpected exit code: {exit_status}" + raise ExecUtilException( + message=errMsg, + command=cmd, + exit_code=exit_status, + out=output, + error=error, + ) + + lines = output.splitlines() + assert type(lines) is list + + if len(lines) == 0: + # ACHTUNG! + raise RuntimeError("Command returns 0 error code without output.") + + # parse result lines + pid_to_ppid: __class__.T_PID_TO_PPID = {} + + for i_line in range(len(lines)): + assert type(lines[i_line]) is str + + parts = lines[i_line].split() + assert type(parts) is list + + if len(parts) < 2: + __class__._find_postmaster__throw_error__bad_line_format( + lines, + i_line, + "no usefull data", + ) + + if not parts[0].isdigit(): + __class__._find_postmaster__throw_error__bad_line_format( + lines, + i_line, + "bad pid", + ) + + if not parts[1].isdigit(): + __class__._find_postmaster__throw_error__bad_line_format( + lines, + i_line, + "bad ppid", + ) + + pid = int(parts[0]) + ppid = int(parts[1]) + + if pid not in pid_to_ppid: + pid_to_ppid[pid] = ppid + continue + + other_ppid = pid_to_ppid[pid] + assert type(other_ppid) is int + + if ppid == other_ppid: + log_msg = "FindPostmaster (data_dir={!r}) get pid ({}) with ppid ({}) more than one time.".format( + data_dir, + pid, + ppid, + ) + internal_utils.send_log_debug(log_msg) + continue + + # ACTUNG ppid is changed --> restart + __class__._find_postmaster__throw_error__ppid_is_changed( + pid, + ppid, + other_ppid, + lines, + ) + + assert len(pid_to_ppid) <= len(lines) + + true_postmasters = [ + pid for pid, ppid in pid_to_ppid.items() + if ppid not in pid_to_ppid + ] + + if len(true_postmasters) == 0: + __class__._find_postmaster__throw_error__cycle( + pid_to_ppid, + ) + + if len(true_postmasters) == 1: + true_pid = true_postmasters[0] + + if len(pid_to_ppid) > 1: + msg = "Many processes like a postmaster for data dir [{}] are found ({}).".format( + data_dir, + len(true_postmasters), + ) + + msg += " List (ppid->pid): {}.".format( + __class__._make_text_from_pid_to_ppid(pid_to_ppid), + ) + + msg += " True postmaster PID is {}.".format(true_pid) + internal_utils.send_log_debug(msg) + + return true_pid + + assert len(true_postmasters) > 1 + + __class__._find_postmaster__throw_error__many_postmasters( + true_postmasters, + pid_to_ppid, + ) + + def ProcessIsZombi_soft_check( + self, + os_ops: OsOperations, + pid: int, + ) -> typing.Optional[bool]: + assert isinstance(os_ops, OsOperations) + assert type(pid) is int + + proc_stat_file = os_ops.build_path("/proc", str(pid), "stat") + + if not os_ops.path_exists(proc_stat_file): + return False + + result: typing.Optional[bool] = None + + try: + # Read one line from /proc/PID/stat + stat_content = os_ops.read_binary(proc_stat_file, 0).decode("utf-8", errors="ignore") + + # We look for the closing parenthesis of the process name to ensure that + # we start from it and not depend on spaces inside the parentheses! + r_paren_idx = stat_content.rfind(")") + + if r_paren_idx == -1: + pass + elif len(stat_content) <= r_paren_idx + 2: + pass + else: + # The status goes exactly one space after the closing bracket + assert (r_paren_idx + 2) < len(stat_content) + proc_status = stat_content[r_paren_idx + 2] + result = proc_status == "Z" + except Exception as e: + # If the file disappeared right during reading, it means the process is completely erased + if __class__._is_file_not_found_exception(e): + result = False + + return result + + @staticmethod + def _is_file_not_found_exception(e: Exception) -> bool: + if isinstance(e, FileNotFoundError): + return True + + if isinstance(e, ExecUtilException): + if e.exit_code == 2: + return True + + return False + + T_PID_TO_PPID = typing.Dict[int, int] + + @staticmethod + def _make_text_from_pid_to_ppid(pid_to_ppid: T_PID_TO_PPID) -> str: + result = "" + sep = "" + for pid, ppid in pid_to_ppid.items(): + result += sep + " {}->{}".format(ppid, pid) + sep = ", " + continue + return result + + @staticmethod + def _find_postmaster__throw_error__bad_line_format( + lines: typing.List[str], + i_line: int, + hint: str, + ) -> typing.NoReturn: + assert type(lines) is list + assert type(i_line) is int + assert type(hint) is int + + error_lines: typing.List[str] = [] + error_lines.append( + "Line {} has bad format. Hint: {}.".format( + i_line + 1, + hint, + ), + ) + error_lines.append( + "Problem line is:" + ) + error_lines.append( + " " + repr(lines[i_line]), + ) + error_lines.append( + "All the lines is:", + ) + for i in range(len(lines)): + error_lines.append( + " {}. {!r}".format(i + 1, lines[i]), + ) + continue + + raise RuntimeError("\n".join(error_lines)) + + @staticmethod + def _find_postmaster__throw_error__ppid_is_changed( + pid: int, + ppid: int, + other_ppid: int, + lines: typing.List[str], + ) -> typing.NoReturn: + assert type(pid) is int + assert type(ppid) is int + assert type(other_ppid) is int + assert type(lines) is list + + error_lines: typing.List[str] = [] + error_lines.append( + "Parent of process ({}) is changed from {} to {}.".format( + pid, + other_ppid, + ppid, + ), + ) + error_lines.append( + "All the lines is:", + ) + for i in range(len(lines)): + error_lines.append( + " {}. {!r}".format(i + 1, lines[i]), + ) + continue + + raise RuntimeError("\n".join(error_lines)) + + @staticmethod + def _find_postmaster__throw_error__cycle( + pid_to_ppid: T_PID_TO_PPID, + ) -> typing.NoReturn: + msg = "Cycle in processes postgres process tree. " + + msg += " List (ppid->pid): {},".format( + __class__._make_text_from_pid_to_ppid(pid_to_ppid), + ) + + msg += " List size is {}.".format( + len(pid_to_ppid), + ) + raise RuntimeError(msg) + + @staticmethod + def _find_postmaster__throw_error__many_postmasters( + postmaster_pids: typing.List[int], + pid_to_ppid: T_PID_TO_PPID, + ) -> typing.NoReturn: + msg = "Many processes like a postmaster are found ({}): {}.".format( + len(postmaster_pids), + ", ".join(map(str, postmaster_pids)), + ) + + msg += " Trees (ppid->pid): {}.".format( + __class__._make_text_from_pid_to_ppid(pid_to_ppid), + ) + + msg += " Total process count is {}.".format(len(pid_to_ppid)) + raise RuntimeError(msg) + + @staticmethod + def _find_postmaster__throw_error__fail( + bin_dir: str, + data_dir: str, + failures: typing.List[Exception], + ) -> typing.NoReturn: + assert type(bin_dir) is str + assert type(data_dir) is str + assert type(failures) is list + + method_name = "InternalPlatformUtils::FindPostmaster(bin_dir={!r}, data_dir={!r})".format( + bin_dir, + data_dir, + ) + + RaiseError.function_did_multiple_attempts_without_stable_result( + method_name, + failures, + ) diff --git a/src/impl/platforms/win32/internal_platform_utils.py b/src/impl/platforms/win32/internal_platform_utils.py new file mode 100644 index 00000000..c6df6378 --- /dev/null +++ b/src/impl/platforms/win32/internal_platform_utils.py @@ -0,0 +1,27 @@ +from __future__ import annotations + +from .. import internal_platform_utils as base +from testgres.operations.os_ops import OsOperations +import typing + + +class InternalPlatformUtils(base.InternalPlatformUtils): + def FindPostmaster( + self, + os_ops: OsOperations, + bin_dir: str, + data_dir: str + ) -> InternalPlatformUtils.FindPostmasterResult: + assert isinstance(os_ops, OsOperations) + assert type(bin_dir) is str + assert type(data_dir) is str + return __class__.FindPostmasterResult.create_not_implemented() + + def ProcessIsZombi_soft_check( + self, + os_ops: OsOperations, + pid: int, + ) -> typing.Optional[bool]: + assert isinstance(os_ops, OsOperations) + assert type(pid) is int + return None diff --git a/src/impl/port_manager__generic.py b/src/impl/port_manager__generic.py new file mode 100755 index 00000000..3d5b1490 --- /dev/null +++ b/src/impl/port_manager__generic.py @@ -0,0 +1,97 @@ +from testgres.operations.os_ops import OsOperations + +from ..port_manager import PortManager +from ..exceptions import PortForException + +import threading +import random +import typing +import logging + + +class PortManager__Generic(PortManager): + _C_MIN_PORT_NUMBER = 1024 + _C_MAX_PORT_NUMBER = 65535 + + _os_ops: OsOperations + _guard: object + # TODO: is there better to use bitmap fot _available_ports? + _available_ports: typing.Set[int] + _reserved_ports: typing.Set[int] + + def __init__(self, os_ops: OsOperations): + assert __class__._C_MIN_PORT_NUMBER <= __class__._C_MAX_PORT_NUMBER + + assert os_ops is not None + assert isinstance(os_ops, OsOperations) + self._os_ops = os_ops + self._guard = threading.Lock() + + self._available_ports = set( + range(__class__._C_MIN_PORT_NUMBER, __class__._C_MAX_PORT_NUMBER + 1) + ) + assert len(self._available_ports) == ( + (__class__._C_MAX_PORT_NUMBER - __class__._C_MIN_PORT_NUMBER) + 1 + ) + self._reserved_ports = set() + return + + def reserve_port(self) -> int: + assert self._guard is not None + assert type(self._available_ports) is set + assert type(self._reserved_ports) is set + + with self._guard: + t = tuple(self._available_ports) + assert len(t) == len(self._available_ports) + sampled_ports = random.sample(t, min(len(t), 100)) + t = None + + for port in sampled_ports: + assert type(port) is int + assert port not in self._reserved_ports + assert port in self._available_ports + + assert port >= __class__._C_MIN_PORT_NUMBER + assert port <= __class__._C_MAX_PORT_NUMBER + + if not self._os_ops.is_port_free(port): + continue + + self._reserved_ports.add(port) + self._available_ports.discard(port) + assert port in self._reserved_ports + assert port not in self._available_ports + __class__.helper__send_debug_msg("Port {} is reserved.", port) + return port + + raise PortForException("Can't select a port.") + + def release_port(self, number: int) -> None: + assert type(number) is int + assert number >= __class__._C_MIN_PORT_NUMBER + assert number <= __class__._C_MAX_PORT_NUMBER + + assert self._guard is not None + assert type(self._reserved_ports) is set + + with self._guard: + assert number in self._reserved_ports + assert number not in self._available_ports + self._available_ports.add(number) + self._reserved_ports.discard(number) + assert number not in self._reserved_ports + assert number in self._available_ports + __class__.helper__send_debug_msg("Port {} is released.", number) + return + + @staticmethod + def helper__send_debug_msg(msg_template: str, *args) -> None: + assert msg_template is not None + assert args is not None + assert type(msg_template) is str + assert type(args) is tuple + assert msg_template != "" + s = "[port manager] " + s += msg_template.format(*args) + logging.debug(s) diff --git a/src/impl/port_manager__generic2.py b/src/impl/port_manager__generic2.py new file mode 100755 index 00000000..b2b5f5c2 --- /dev/null +++ b/src/impl/port_manager__generic2.py @@ -0,0 +1,166 @@ +from testgres.operations.os_ops import OsOperations + +from ..port_manager import PortManager +from ..exceptions import PortForException +from .. import consts + +import threading +import random +import typing +import logging + + +class OsLockFsObj: + _os_ops: typing.Optional[OsOperations] + _path: typing.Optional[str] + + def __init__(self, os_ops: OsOperations, path: str): + assert isinstance(os_ops, OsOperations) + assert type(path) is str + + os_ops.makedir(path) # throw + + self._os_ops = os_ops + self._path = path + return + + def release(self) -> None: + assert type(self._path) is str + assert isinstance(self._os_ops, OsOperations) + assert self._os_ops.path_exists(self._path) + + self._os_ops.rmdir(self._path) # throw + + self._path = None + self._os_ops = None + return + + +class PortManager__Generic2(PortManager): + _C_MIN_PORT_NUMBER = 1024 + _C_MAX_PORT_NUMBER = 65535 + + _os_ops: OsOperations + _guard: typing.Any + + # TODO: is there better to use bitmap fot _available_ports? + _available_ports: typing.Set[int] + _reserved_ports: typing.Dict[int, OsLockFsObj] + + _lock_dir: typing.Optional[str] + + def __init__(self, os_ops: OsOperations): + assert __class__._C_MIN_PORT_NUMBER <= __class__._C_MAX_PORT_NUMBER + + assert os_ops is not None + assert isinstance(os_ops, OsOperations) + self._os_ops = os_ops + self._guard = threading.Lock() + + self._available_ports = set( + range(__class__._C_MIN_PORT_NUMBER, __class__._C_MAX_PORT_NUMBER + 1) + ) + assert len(self._available_ports) == ( + (__class__._C_MAX_PORT_NUMBER - __class__._C_MIN_PORT_NUMBER) + 1 + ) + self._reserved_ports = dict() + self._lock_dir = None + return + + def reserve_port(self) -> int: + assert self._guard is not None + assert type(self._available_ports) is set + assert type(self._reserved_ports) is dict + assert isinstance(self._os_ops, OsOperations) + + with self._guard: + if self._lock_dir is None: + temp_dir = self._os_ops.get_tempdir() + assert type(temp_dir) is str + lock_dir = self._os_ops.build_path(temp_dir, consts.TMP_TESTGRES_PORTS) + assert type(lock_dir) is str + self._os_ops.makedirs(lock_dir) + self._lock_dir = lock_dir + + assert self._lock_dir is not None + assert type(self._lock_dir) is str + + t = tuple(self._available_ports) + assert len(t) == len(self._available_ports) + sampled_ports = random.sample(t, min(len(t), 100)) + t = None + + for port in sampled_ports: + assert type(port) is int + assert port not in self._reserved_ports + assert port in self._available_ports + + assert port >= __class__._C_MIN_PORT_NUMBER + assert port <= __class__._C_MAX_PORT_NUMBER + + if not self._os_ops.is_port_free(port): + continue + + try: + lock_path = self.helper__make_lock_path(port) + lock_obj = OsLockFsObj(self._os_ops, lock_path) # raise + except: # noqa: E722 + continue + + assert isinstance(lock_obj, OsLockFsObj) + assert self._os_ops.path_exists(lock_path) + + try: + self._reserved_ports[port] = lock_obj + except: # noqa: E722 + assert port not in self._reserved_ports + lock_obj.release() + raise + + self._available_ports.discard(port) + assert port in self._reserved_ports + assert port not in self._available_ports + __class__.helper__send_debug_msg("Port {} is reserved.", port) + return port + + raise PortForException("Can't select a port.") + + def release_port(self, number: int) -> None: + assert type(number) is int + assert number >= __class__._C_MIN_PORT_NUMBER + assert number <= __class__._C_MAX_PORT_NUMBER + + assert self._guard is not None + assert type(self._reserved_ports) is dict + + with self._guard: + assert number in self._reserved_ports + assert number not in self._available_ports + self._available_ports.add(number) + lock_obj = self._reserved_ports.pop(number) + assert number not in self._reserved_ports + assert number in self._available_ports + assert isinstance(lock_obj, OsLockFsObj) + lock_obj.release() + __class__.helper__send_debug_msg("Port {} is released.", number) + return + + @staticmethod + def helper__send_debug_msg(msg_template: str, *args) -> None: + assert msg_template is not None + assert args is not None + assert type(msg_template) is str + assert type(args) is tuple + assert msg_template != "" + s = "[port manager] " + s += msg_template.format(*args) + logging.debug(s) + + def helper__make_lock_path(self, port_number: int) -> str: + assert type(port_number) is int + # You have to call the reserve_port at first! + assert type(self._lock_dir) is str + + result = self._os_ops.build_path(self._lock_dir, str(port_number) + ".lock") + assert type(result) is str + return result diff --git a/src/impl/port_manager__this_host.py b/src/impl/port_manager__this_host.py new file mode 100755 index 00000000..6c3ef41d --- /dev/null +++ b/src/impl/port_manager__this_host.py @@ -0,0 +1,34 @@ +from ..port_manager import PortManager + +from .. import utils + +import threading +import typing + + +class PortManager__ThisHost(PortManager): + sm_single_instance: typing.Optional[PortManager] = None + sm_single_instance_guard = threading.Lock() + + @staticmethod + def get_single_instance() -> PortManager: + assert __class__ == PortManager__ThisHost + assert __class__.sm_single_instance_guard is not None + + if __class__.sm_single_instance is not None: + assert type(__class__.sm_single_instance) is __class__ + return __class__.sm_single_instance + + with __class__.sm_single_instance_guard: + if __class__.sm_single_instance is None: + __class__.sm_single_instance = __class__() + assert __class__.sm_single_instance is not None + assert type(__class__.sm_single_instance) is __class__ + return __class__.sm_single_instance + + def reserve_port(self) -> int: + return utils.reserve_port() + + def release_port(self, number: int) -> None: + assert type(number) is int + return utils.release_port(number) diff --git a/src/logger.py b/src/logger.py new file mode 100644 index 00000000..333da71c --- /dev/null +++ b/src/logger.py @@ -0,0 +1,116 @@ +# coding: utf-8 + +from .config import testgres_config as tconf +from .exceptions import ExecUtilException +from .impl.file_line_reader import FileLineReader + +from testgres.operations.os_ops import OsOperations + +import logging +import threading +import time +import typing + + +class TestgresLogger(threading.Thread): + _C_SLEEP_MIN = 0.01 + _C_SLEEP_MAX = 60 + + """ + Helper class to implement reading from log files. + """ + def __init__( + self, + node_name: str, + log_file_name: str, + log_file_encoding: str = "utf-8", + os_ops: typing.Optional[OsOperations] = None, + ): + assert type(node_name) is str + assert type(log_file_name) is str + assert type(log_file_encoding) is str + assert os_ops is None or isinstance(os_ops, OsOperations) + + threading.Thread.__init__(self, daemon=True) + + if os_ops is None: + os_ops = tconf.os_ops + + assert os_ops is None or isinstance(os_ops, OsOperations) + + self._os_ops = os_ops + self._node_name = node_name + self._log_file_name = log_file_name + self._log_file_encoding = log_file_encoding + self._stop_event = threading.Event() + self._logger = logging.getLogger(node_name) + self._logger.setLevel(logging.INFO) + return + + def run(self): + # open log file for reading + file_line_reader = FileLineReader( + self._os_ops, + self._log_file_name, + self._log_file_encoding, + ) + + sleep_time = __class__._C_SLEEP_MIN + + try: + # work until we're asked to stop + while not self._stop_event.is_set(): + line = None + + while True: + try: + line = file_line_reader.read_line() # raise + except Exception as e: + if __class__._is_file_not_found_exception(e): + if not self._os_ops.path_exists(self._log_file_name): + break + raise + break + + if line is None: + time.sleep(sleep_time) + sleep_time = min(__class__._C_SLEEP_MAX, 2 * sleep_time) + continue + + assert type(line) is str + + sleep_time = __class__._C_SLEEP_MIN + + # do we have new lines? + line = line.strip() + + extra = {'node': self._node_name} + self._logger.info(line, extra=extra) + continue + except Exception as e: + self._logger.error(e) + raise + finally: + # don't forget to clear event + # [2026-07-10] legacy cargo cult, thread is single-use only. + # self._stop_event.clear() + pass + return + + def stop(self, wait=True): + self._stop_event.set() + + if wait: + self.join() + return + + @staticmethod + def _is_file_not_found_exception(e: Exception) -> bool: + if isinstance(e, FileNotFoundError): + return True + + if isinstance(e, ExecUtilException): + if e.exit_code == 2: + return True + + return False diff --git a/src/node.py b/src/node.py new file mode 100644 index 00000000..b369c73a --- /dev/null +++ b/src/node.py @@ -0,0 +1,2661 @@ +# coding: utf-8 +from __future__ import annotations + +import logging +import signal +import subprocess + +import time +import typing + +try: + from collections.abc import Iterable +except ImportError: + from collections import Iterable + +# we support both pg8000 and psycopg2 +try: + import psycopg2 as pglib +except ImportError: + try: + import pg8000 as pglib + except ImportError: + raise ImportError("You must have psycopg2 or pg8000 modules installed") + +from six import raise_from, iteritems, text_type + +from .enums import \ + NodeStatus, \ + ProcessType, \ + DumpFormat + +from .cache import cached_initdb + +from .config import testgres_config + +from .connection import NodeConnection + +from .consts import \ + DATA_DIR, \ + LOGS_DIR, \ + TMP_NODE, \ + TMP_DUMP, \ + PG_CONF_FILE, \ + PG_AUTO_CONF_FILE, \ + HBA_CONF_FILE, \ + RECOVERY_CONF_FILE, \ + PG_LOG_FILE, \ + UTILS_LOG_FILE + +from .consts import \ + MAX_LOGICAL_REPLICATION_WORKERS, \ + MAX_REPLICATION_SLOTS, \ + MAX_WORKER_PROCESSES, \ + MAX_WAL_SENDERS, \ + WAL_KEEP_SEGMENTS, \ + WAL_KEEP_SIZE + +from .decorators import \ + method_decorator, \ + positional_args_hack + +from .defaults import \ + default_dbname, \ + generate_app_name + +from .exceptions import \ + CatchUpException, \ + ExecUtilException, \ + QueryException, \ + QueryTimeoutException, \ + StartNodeException, \ + TimeoutException, \ + InitNodeException, \ + TestgresException, \ + BackupException, \ + InvalidOperationException + +from .port_manager import PortManager +from .impl.port_manager__this_host import PortManager__ThisHost +from .impl.port_manager__generic2 import PortManager__Generic2 +from .impl import internal_utils + +from .logger import TestgresLogger + +from .pubsub import Publication, Subscription + +from .standby import First + +from . import utils + +from .utils import \ + PgVer, \ + eprint, \ + get_pg_version2, \ + execute_utility2, \ + options_string, \ + clean_on_error + +from .raise_error import RaiseError + +from .backup import NodeBackup + +from testgres.operations.os_ops import OsOperations +from testgres.operations.local_ops import LocalOperations + +InternalError = pglib.InternalError +ProgrammingError = pglib.ProgrammingError +OperationalError = pglib.OperationalError + + +assert TimeoutException == QueryTimeoutException + + +class ProcessProxy(object): + """ + Wrapper for psutil.Process + + Attributes: + process: wrapped psutill.Process object + ptype: instance of ProcessType + """ + + _process: typing.Any + _ptype: ProcessType + + def __init__(self, process, ptype: typing.Optional[ProcessType] = None): + assert process is not None + assert ptype is None or type(ptype) is ProcessType + self._process = process + + if ptype is not None: + self._ptype = ptype + else: + self._ptype = ProcessType.from_process(process) + assert type(self._ptype) is ProcessType + return + + def __getattr__(self, name): + return getattr(self.process, name) + + def __repr__(self): + return '{}(ptype={}, process={})'.format( + self.__class__.__name__, + str(self.ptype), + repr(self.process)) + + @property + def process(self) -> typing.Any: + assert self._process is not None + return self._process + + @property + def ptype(self) -> ProcessType: + assert type(self._ptype) is ProcessType + return self._ptype + + +class PostgresNode(object): + # a max number of node start attempts + _C_MAX_START_ATEMPTS = 5 + + _C_MAX_GET_CHILDREN_ATTEMPTS = 5 + + _C_PM_PID__IS_NOT_DETECTED = -1 + + _name: typing.Optional[str] + _host: str + _port: typing.Optional[int] + _bin_dir: str + _should_free_port: bool + _os_ops: OsOperations + _port_manager: typing.Optional[PortManager] + _manually_started_pm_pid: typing.Optional[int] + + def __init__( + self, + name=None, + base_dir=None, + port: typing.Optional[int] = None, + bin_dir: typing.Optional[str] = None, + prefix=None, + os_ops: typing.Optional[OsOperations] = None, + port_manager: typing.Optional[PortManager] = None, + host: typing.Optional[str] = None, + ): + """ + PostgresNode constructor. + + Args: + name: node's application name. + port: port to accept connections. + base_dir: path to node's data directory. + bin_dir: path to node's binary directory. + os_ops: None or correct OS operation object. + port_manager: None or correct port manager object. + host: None or valid address of node host. + """ + assert port is None or type(port) is int + assert bin_dir is None or type(bin_dir) is str + assert os_ops is None or isinstance(os_ops, OsOperations) + assert port_manager is None or isinstance(port_manager, PortManager) + assert host is None or type(host) is str + + # private + if os_ops is None: + self._os_ops = __class__._get_os_ops() + else: + assert isinstance(os_ops, OsOperations) + self._os_ops = os_ops + pass + + assert self._os_ops is not None + assert isinstance(self._os_ops, OsOperations) + + if bin_dir is not None: + self._bin_dir = bin_dir + else: + self._bin_dir = utils.get_bin_dir(self._os_ops) + + assert type(self._bin_dir) is str + + self._pg_version = PgVer(get_pg_version2(self._os_ops, self._bin_dir)) + self._base_dir = base_dir + self._prefix = prefix + self._logger = None + self._master = None + + # basic + self._name = name or generate_app_name() + + if host is not None: + assert type(host) is str + self._host = host + else: + self._host = self._os_ops.host + assert type(self._host) is str + + if self._host == "": + raise RuntimeError("PostgresNode host is empty.") + + if port is not None: + assert type(port) is int + assert port_manager is None + self._port = port + self._should_free_port = False + self._port_manager = None + else: + if port_manager is None: + self._port_manager = __class__._get_port_manager(self._os_ops) + elif os_ops is None: + raise InvalidOperationException("When port_manager is not None you have to define os_ops, too.") + else: + assert isinstance(port_manager, PortManager) + assert self._os_ops is os_ops + self._port_manager = port_manager + + assert self._port_manager is not None + assert isinstance(self._port_manager, PortManager) + + self._port = self._port_manager.reserve_port() # raises + assert type(self._port) is int + self._should_free_port = True + + assert type(self._port) is int + + # defaults for __exit__() + self.cleanup_on_good_exit = testgres_config.node_cleanup_on_good_exit + self.cleanup_on_bad_exit = testgres_config.node_cleanup_on_bad_exit + self.shutdown_max_attempts = 3 + + # Node state + self._manually_started_pm_pid = None + + def __enter__(self): + return self + + def __exit__(self, type, value, traceback): + # NOTE: Ctrl+C does not count! + got_exception = type is not None and type is not KeyboardInterrupt + + c1 = self.cleanup_on_good_exit and not got_exception + c2 = self.cleanup_on_bad_exit and got_exception + + attempts = self.shutdown_max_attempts + + if c1 or c2: + self.cleanup(attempts) + else: + self._try_shutdown(attempts) + + self._release_resources() + + def __repr__(self): + return "{}(name='{}', port={}, base_dir='{}')".format( + self.__class__.__name__, + self.name, + str(self._port) if self._port is not None else "None", + self.base_dir + ) + + @staticmethod + def _get_os_ops() -> OsOperations: + if testgres_config.os_ops: + return testgres_config.os_ops + + return LocalOperations.get_single_instance() + + @staticmethod + def _get_port_manager(os_ops: OsOperations) -> PortManager: + assert os_ops is not None + assert isinstance(os_ops, OsOperations) + + if os_ops is LocalOperations.get_single_instance(): + assert utils._old_port_manager is not None + assert type(utils._old_port_manager) is PortManager__Generic2 + assert utils._old_port_manager._os_ops is os_ops + return PortManager__ThisHost.get_single_instance() + + # TODO: Throw the exception "Please define a port manager." ? + return PortManager__Generic2(os_ops) + + def clone_with_new_name_and_base_dir(self, name: str, base_dir: str): + assert name is None or type(name) is str + assert base_dir is None or type(base_dir) is str + + assert __class__ == PostgresNode + + if self._port_manager is None: + raise InvalidOperationException("PostgresNode without PortManager can't be cloned.") + + assert self._port_manager is not None + assert isinstance(self._port_manager, PortManager) + assert self._os_ops is not None + assert isinstance(self._os_ops, OsOperations) + assert type(self._host) is str + + node = PostgresNode( + name=name, + base_dir=base_dir, + bin_dir=self._bin_dir, + prefix=self._prefix, + os_ops=self._os_ops, + port_manager=self._port_manager, + host=self._host, + ) + + return node + + @property + def os_ops(self) -> OsOperations: + assert self._os_ops is not None + assert isinstance(self._os_ops, OsOperations) + return self._os_ops + + @property + def port_manager(self) -> typing.Optional[PortManager]: + assert self._port_manager is None or isinstance(self._port_manager, PortManager) + return self._port_manager + + @property + def name(self) -> str: + if self._name is None: + raise InvalidOperationException("PostgresNode name is not defined.") + assert type(self._name) is str + return self._name + + @property + def host(self) -> str: + assert self._host is not None + assert type(self._host) is str + return self._host + + @property + def port(self) -> int: + if self._port is None: + raise InvalidOperationException("PostgresNode port is not defined.") + + assert type(self._port) is int + return self._port + + @property + def ssh_key(self) -> typing.Optional[str]: + assert self._os_ops is not None + assert isinstance(self._os_ops, OsOperations) + return self._os_ops.ssh_key + + @property + def pid(self) -> int: + """ + Return postmaster's PID if node is running, else 0. + """ + + x = self._get_node_state() + assert type(x) is utils.PostgresNodeState + + if x.pid is None: + assert x.node_status != NodeStatus.Running + return 0 + + assert x.node_status == NodeStatus.Running + assert type(x.pid) is int + return x.pid + + @property + def is_started(self) -> bool: + if self._manually_started_pm_pid is None: + return False + + assert type(self._manually_started_pm_pid) is int + return True + + @property + def auxiliary_pids(self) -> typing.Dict[ProcessType, typing.List[int]]: + """ + Returns a dict of { ProcessType : PID }. + """ + + result = {} + + for process in self.auxiliary_processes: + assert type(process) is ProcessProxy + if process.ptype not in result: + result[process.ptype] = [] + + result[process.ptype].append(process.pid) + + return result + + @property + def auxiliary_processes(self) -> typing.List[ProcessProxy]: + """ + Returns a list of auxiliary processes. + Each process is represented by :class:`.ProcessProxy` object. + """ + def is_aux(process: ProcessProxy) -> bool: + assert type(process) is ProcessProxy + return process.ptype != ProcessType.Unknown + + return list(filter(is_aux, self.child_processes)) + + @property + def child_processes(self) -> typing.List[ProcessProxy]: + """ + Returns a list of all child processes. + Each process is represented by :class:`.ProcessProxy` object. + """ + + # get a list of postmaster's children + x = self._get_node_state() + assert type(x) is utils.PostgresNodeState + if x.pid is None: + assert x.node_status != NodeStatus.Running + RaiseError.node_err__cant_enumerate_child_processes( + x.node_status, + ) + + assert x.node_status != NodeStatus.Stopped + assert type(x.pid) is int + return self._get_child_processes(x.pid) + + def _get_child_processes(self, pid: int) -> typing.List[ProcessProxy]: + assert type(pid) is int + assert isinstance(self._os_ops, OsOperations) + + C_MAX_ATTEMPT_COUNT = __class__. _C_MAX_GET_CHILDREN_ATTEMPTS + assert type(C_MAX_ATTEMPT_COUNT) is int + assert C_MAX_ATTEMPT_COUNT > 0 + + failures: typing.List[Exception] = [] + + nAttempt = 0 + + while True: + assert nAttempt < C_MAX_ATTEMPT_COUNT + + nAttempt += 1 + + # get a list of postmaster's children + children = self._os_ops.get_process_children(pid) + assert type(children) is list + + result: typing.List[ProcessProxy] = [] + + for p in children: + assert hasattr(p, "pid") + try: + proxy = ProcessProxy(p) # raise + except Exception as e: + internal_utils.send_log_debug( + "Failed to process a node child process [pid: {}]. Exception ({}): {}".format( + p.pid, + type(e).__name__, + e, + ) + ) + failures.append(e) + break + + assert type(proxy) is ProcessProxy + result.append(proxy) + continue + + if len(result) == len(children): + return result + + assert len(result) < len(children) + + if nAttempt < C_MAX_ATTEMPT_COUNT: + time.sleep(0.05) + continue + break + + assert nAttempt == C_MAX_ATTEMPT_COUNT + assert len(failures) == C_MAX_ATTEMPT_COUNT + + method_name = "PostgresNode::_get_child_processes(pid={!r})".format( + pid, + ) + + RaiseError.function_did_multiple_attempts_without_stable_result( + method_name, + failures, + ) + + @property + def source_walsender(self): + """ + Returns master's walsender feeding this replica. + """ + + sql = """ + select pid + from pg_catalog.pg_stat_replication + where application_name = %s + """ + + if self.master is None: + raise TestgresException("Node doesn't have a master") + + assert type(self.master) is PostgresNode + + # master should be on the same host + assert self.master.host == self._host + + with self.master.connect() as con: + for row in con.execute(sql, self.name): + for child in self.master.auxiliary_processes: + if child.pid == int(row[0]): + return child + + msg = "Master doesn't send WAL to {}".format(self.name) + raise TestgresException(msg) + + @property + def master(self): + return self._master + + @property + def base_dir(self): + if not self._base_dir: + self._base_dir = self._os_ops.mkdtemp(prefix=self._prefix or TMP_NODE) + + # NOTE: it's safe to create a new dir + if not self._os_ops.path_exists(self._base_dir): + self._os_ops.makedirs(self._base_dir) + + return self._base_dir + + @property + def bin_dir(self) -> str: + assert type(self._bin_dir) is str + return self._bin_dir + + @property + def logs_dir(self): + assert self._os_ops is not None + assert isinstance(self._os_ops, OsOperations) + + path = self._os_ops.build_path(self.base_dir, LOGS_DIR) + assert type(path) is str + + # NOTE: it's safe to create a new dir + if not self._os_ops.path_exists(path): + self._os_ops.makedirs(path) + + return path + + @property + def data_dir(self): + assert self._os_ops is not None + assert isinstance(self._os_ops, OsOperations) + + # NOTE: we can't run initdb without user's args + path = self._os_ops.build_path(self.base_dir, DATA_DIR) + assert type(path) is str + return path + + @property + def utils_log_file(self): + assert self._os_ops is not None + assert isinstance(self._os_ops, OsOperations) + + path = self._os_ops.build_path(self.logs_dir, UTILS_LOG_FILE) + assert type(path) is str + return path + + @property + def pg_log_file(self): + assert self._os_ops is not None + assert isinstance(self._os_ops, OsOperations) + + path = self._os_ops.build_path(self.logs_dir, PG_LOG_FILE) + assert type(path) is str + return path + + # NOTE: for compatibility + @property + def utils_log_name(self) -> str: + return self.utils_log_file + + # NOTE: for compatibility + @property + def pg_log_name(self) -> str: + return self.pg_log_file + + @property + def version(self): + """ + Return PostgreSQL version for this node. + + Returns: + Instance of :class:`distutils.version.LooseVersion`. + """ + return self._pg_version + + def _try_shutdown(self, max_attempts, with_force=False): + assert type(max_attempts) is int + assert type(with_force) is bool + assert max_attempts > 0 + + try: + self._try_shutdown_internal(max_attempts, with_force) + finally: + self._maybe_stop_logger() + + def _try_shutdown_internal(self, max_attempts, with_force): + attempts = 0 + + # try stopping server N times + while attempts < max_attempts: + attempts += 1 + try: + self.stop() + except ExecUtilException: + continue # one more time + except Exception: + eprint('cannot stop node {}'.format(self.name)) + break + + return # OK + + # If force stopping is enabled and PID is valid + if not with_force: + return + + node_pid = self.pid + assert node_pid is not None + assert type(node_pid) is int + + if node_pid == 0: + return + + # TODO: [2025-02-28] It is really the old ugly code. We have to rewrite it! + + ps_command = ['ps', '-o', 'pid=', '-p', str(node_pid)] + + ps_output = self._os_ops.exec_command(cmd=ps_command, shell=True, ignore_errors=True).decode('utf-8') + assert type(ps_output) is str + + if ps_output == "": + return + + if ps_output != str(node_pid): + __class__._throw_bugcheck__unexpected_result_of_ps( + ps_output, + ps_command) + + try: + eprint('Force stopping node {0} with PID {1}'.format(self.name, node_pid)) + self._os_ops.kill(node_pid, signal.SIGKILL) + except Exception: + # The node has already stopped + pass + + # Check that node stopped - print only column pid without headers + ps_output = self._os_ops.exec_command(cmd=ps_command, shell=True, ignore_errors=True).decode('utf-8') + assert type(ps_output) is str + + if ps_output == "": + eprint('Node {0} has been stopped successfully.'.format(self.name)) + return + + if ps_output == str(node_pid): + eprint('Failed to stop node {0}.'.format(self.name)) + return + + __class__._throw_bugcheck__unexpected_result_of_ps( + ps_output, + ps_command) + + @staticmethod + def _throw_bugcheck__unexpected_result_of_ps(result, cmd): + assert type(result) is str + assert type(cmd) is list + errLines = [] + errLines.append("[BUG CHECK] Unexpected result of command ps:") + errLines.append(result) + errLines.append("-----") + errLines.append("Command line is {0}".format(cmd)) + raise RuntimeError("\n".join(errLines)) + + def _assign_master(self, master): + """NOTE: this is a private method!""" + + # now this node has a master + self._master = master + + def _create_recovery_conf(self, username, slot=None): + """NOTE: this is a private method!""" + + # fetch master of this node + master = self.master + assert master is not None + + conninfo = { + "application_name": self.name, + "port": master.port, + "user": username + } # yapf: disable + + # host is tricky + try: + import ipaddress + ipaddress.ip_address(master.host) + conninfo["hostaddr"] = master.host + except ValueError: + conninfo["host"] = master.host + + line = ( + "primary_conninfo='{}'\n" + ).format(options_string(**conninfo)) # yapf: disable + # Since 12 recovery.conf had disappeared + if self.version >= PgVer('12'): + assert self._os_ops is not None + assert isinstance(self._os_ops, OsOperations) + + signal_name = self._os_ops.build_path(self.data_dir, "standby.signal") + assert type(signal_name) is str + self._os_ops.touch(signal_name) + else: + line += "standby_mode=on\n" + + if slot: + # Connect to master for some additional actions + with master.connect(username=username) as con: + # check if slot already exists + res = con.execute( + """ + select exists ( + select from pg_catalog.pg_replication_slots + where slot_name = %s + ) + """, slot) + + if res[0][0]: + raise TestgresException( + "Slot '{}' already exists".format(slot)) + + # TODO: we should drop this slot after replica's cleanup() + con.execute( + """ + select pg_catalog.pg_create_physical_replication_slot(%s) + """, slot) + + line += "primary_slot_name={}\n".format(slot) + + if self.version >= PgVer('12'): + self.append_conf(line=line) + else: + self.append_conf(filename=RECOVERY_CONF_FILE, line=line) + + def _maybe_start_logger(self): + if testgres_config.use_python_logging: + # spawn new logger if it doesn't exist or is stopped + if not self._logger or not self._logger.is_alive(): + self._logger = TestgresLogger( + self.name, + self.pg_log_file, + os_ops=self._os_ops, + ) + self._logger.start() + + def _maybe_stop_logger(self): + if self._logger: + self._logger.stop() + + def _collect_special_files(self) -> typing.List[typing.Tuple[str, bytes]]: + result = [] + + # list of important files + last N lines + assert self._os_ops is not None + assert isinstance(self._os_ops, OsOperations) + + files = [ + (self._os_ops.build_path(self.data_dir, PG_CONF_FILE), 0), + (self._os_ops.build_path(self.data_dir, PG_AUTO_CONF_FILE), 0), + (self._os_ops.build_path(self.data_dir, RECOVERY_CONF_FILE), 0), + (self._os_ops.build_path(self.data_dir, HBA_CONF_FILE), 0), + (self.pg_log_file, testgres_config.error_log_lines) + ] # yapf: disable + + for f, num_lines in files: + # skip missing files + if not self._os_ops.path_exists(f): + continue + + file_lines = self._os_ops.readlines(f, num_lines, binary=True, encoding=None) + lines = b''.join(file_lines) + + # fill list + result.append((f, lines)) + + return result + + def init(self, initdb_params=None, cached=True, **kwargs): + """ + Perform initdb for this node. + + Args: + initdb_params: parameters for initdb (list). + fsync: should this node use fsync to keep data safe? + unix_sockets: should we enable UNIX sockets? + allow_streaming: should this node add a hba entry for replication? + + Returns: + This instance of :class:`.PostgresNode` + """ + + # initialize this PostgreSQL node + assert self._os_ops is not None + assert isinstance(self._os_ops, OsOperations) + + cached_initdb( + data_dir=self.data_dir, + logfile=self.utils_log_file, + os_ops=self._os_ops, + params=initdb_params, + bin_path=self.bin_dir, + cached=False) + + # initialize default config files + self.default_conf(**kwargs) + + return self + + def default_conf(self, + fsync=False, + unix_sockets=True, + allow_streaming=True, + allow_logical=False, + log_statement='all'): + """ + Apply default settings to this node. + + Args: + fsync: should this node use fsync to keep data safe? + unix_sockets: should we enable UNIX sockets? + allow_streaming: (ignored) should this node add a hba entry for replication? + allow_logical: can this node be used as a logical replication publisher? + log_statement: one of ('all', 'off', 'mod', 'ddl'). + + Returns: + This instance of :class:`.PostgresNode`. + """ + + assert self._os_ops is not None + assert isinstance(self._os_ops, OsOperations) + + # hba file is updated + self._default_conf__hba() + + postgres_conf = self._os_ops.build_path(self.data_dir, PG_CONF_FILE) + + # overwrite config file + self._os_ops.write(postgres_conf, '', truncate=True) + + self.append_conf(fsync=fsync, + max_worker_processes=MAX_WORKER_PROCESSES, + log_statement=log_statement, + listen_addresses=self._host, + port=self.port) # yapf:disable + + # common replication settings + if allow_streaming or allow_logical: + self.append_conf(max_replication_slots=MAX_REPLICATION_SLOTS, + max_wal_senders=MAX_WAL_SENDERS) # yapf: disable + + # binary replication + if allow_streaming: + # select a proper wal_level for PostgreSQL + wal_level = 'replica' if self._pg_version >= PgVer('9.6') else 'hot_standby' + + if self._pg_version < PgVer('13'): + self.append_conf(hot_standby=True, + wal_keep_segments=WAL_KEEP_SEGMENTS, + wal_level=wal_level) # yapf: disable + else: + self.append_conf(hot_standby=True, + wal_keep_size=WAL_KEEP_SIZE, + wal_level=wal_level) # yapf: disable + + # logical replication + if allow_logical: + if self._pg_version < PgVer('10'): + raise InitNodeException("Logical replication is only " + "available on PostgreSQL 10 and newer") + + self.append_conf( + max_logical_replication_workers=MAX_LOGICAL_REPLICATION_WORKERS, + wal_level='logical') + + # disable UNIX sockets if asked to + if not unix_sockets: + self.append_conf(unix_socket_directories='') + + return self + + def _default_conf__hba(self) -> None: + hba_conf = self._os_ops.build_path(self.data_dir, HBA_CONF_FILE) + + # filter lines in hba file + # get rid of comments and blank lines + hba_conf_file = self._os_ops.readlines(hba_conf, binary=False) + + assert type(hba_conf_file) is list + + hba_conf_file_finished_with_eol = True + if len(hba_conf_file) > 0: + last_line = hba_conf_file[-1] + assert type(last_line) is str + hba_conf_file_finished_with_eol = last_line.endswith("\n") + + # Normalize function: turns a string into a list of pure words + def normalize_line(line_str): + return line_str.strip().split() + + # We collect a list of rules that already exist in the file (in the form of word lists) + existing_normalized = [] + for s in hba_conf_file: + s_clean = s.strip() + if s_clean and not s_clean.startswith("#"): + existing_normalized.append(normalize_line(s_clean)) + continue + + # get auth method for host or local users + def get_auth_method(t): + for x in existing_normalized: + assert type(x) is list + if len(x) > 0 and x[0] == t: + return x[-1] + continue + return 'trust' + + # get auth methods + auth_local = get_auth_method('local') + auth_host = get_auth_method('host') + + # Basic rules that we want to see in the file + raw_rules = [ + ("local", "replication", "all", "", auth_local), + ("host", "replication", "all", "0.0.0.0/0", auth_host), + ("host", "replication", "all", "::/0", auth_host), + ("local", "all", "all", "", auth_local), + ("host", "all", "all", "0.0.0.0/0", auth_host), + ("host", "all", "all", "::/0", auth_host), + ] + + add_rules = [] + + for type_hba, db, user, addr, method in raw_rules: + # We check if such a rule already exists in the file (by meaning, not by tabs!) + target_words = [type_hba, db, user, method] + if addr: + target_words.insert(3, addr) + + if target_words in existing_normalized: + continue # ะขะฐะบะพะต ะฟั€ะฐะฒะธะปะพ ัƒะถะต ะตัั‚ัŒ, ะฟั€ะพะฟัƒัะบะฐะตะผ! + + # Beautiful, smooth enterprise formatting with spaces! + # Text will be left-aligned and aligned strictly within columns. + formatted_rule = "{:<8} {:<16} {:<16} {:<24} {}\n".format( + type_hba, db, user, addr if addr else "", method + ) + add_rules.append(formatted_rule) + continue + + if len(add_rules) > 0: + add_lines = [] + if not hba_conf_file_finished_with_eol: + add_lines.append("\n") + + add_lines.append("\n") + add_lines.append("# Testgres default configuration\n") + add_lines += add_rules + + # We add only real, beautifully formatted new items + self._os_ops.write(hba_conf, add_lines, truncate=False) + return + + @method_decorator(positional_args_hack(['filename', 'line'])) + def append_conf(self, line='', filename=PG_CONF_FILE, **kwargs): + """ + Append line to a config file. + + Args: + line: string to be appended to config. + filename: config file (postgresql.conf by default). + **kwargs: named config options. + + Returns: + This instance of :class:`.PostgresNode`. + + Examples: + >>> append_conf(fsync=False) + >>> append_conf('log_connections = yes') + >>> append_conf(random_page_cost=1.5, fsync=True, ...) + >>> append_conf('postgresql.conf', 'synchronous_commit = off') + """ + + lines = [line] + + for option, value in iteritems(kwargs): + if isinstance(value, bool): + value = 'on' if value else 'off' + elif not str(value).replace('.', '', 1).isdigit(): + value = "'{}'".format(value) + if value == '*': + lines.append("{} = '*'".format(option)) + else: + # format a new config line + lines.append('{} = {}'.format(option, value)) + + config_name = self._os_ops.build_path(self.data_dir, filename) + conf_text = '' + for line in lines: + conf_text += text_type(line) + '\n' + self._os_ops.write(config_name, conf_text) + + return self + + def status(self): + """ + Check this node's status. + + Returns: + An instance of :class:`.NodeStatus`. + """ + x = self._get_node_state() + assert type(x) is utils.PostgresNodeState + return x.node_status + + def _get_node_state(self) -> utils.PostgresNodeState: + if self._base_dir is None: + return utils.PostgresNodeState( + node_status=NodeStatus.Uninitialized, + pid=None, + ) + + return utils.get_pg_node_state( + self._os_ops, + self.bin_dir, + self.data_dir, + self.utils_log_file + ) + + def get_control_data(self): + """ + Return contents of pg_control file. + """ + + # this one is tricky (blame PG 9.4) + _params = [self._get_bin_path("pg_controldata")] + _params += ["-D"] if self._pg_version >= PgVer('9.5') else [] + _params += [self.data_dir] + + data = execute_utility2(self._os_ops, _params, self.utils_log_file) + + out_dict = {} + + for line in data.splitlines(): + key, _, value = line.partition(':') + out_dict[key.strip()] = value.strip() + + return out_dict + + def slow_start( + self, + replica: bool = False, + dbname: typing.Optional[str] = 'template1', + username: typing.Optional[str] = None, + max_attempts: int = 0, + exec_env: typing.Optional[typing.Dict[str, str]] = None, + ): + """ + Starts the PostgreSQL instance and then polls the instance + until it reaches the expected state (primary or replica). The state is checked + using the pg_is_in_recovery() function. + + Args: + dbname: + username: + replica: If True, waits for the instance to be in recovery (i.e., replica mode). + If False, waits for the instance to be in primary mode. Default is False. + max_attempts: + """ + assert dbname is None or type(dbname) is str + assert username is None or type(username) is str + assert exec_env is None or type(exec_env) is dict + + self.start(exec_env=exec_env) + + try: + if replica: + query = 'SELECT pg_is_in_recovery()' + else: + query = 'SELECT not pg_is_in_recovery()' + + # Call poll_query_until until the expected value is returned + suppressed_exceptions = { + InternalError, + QueryException, + ProgrammingError, + OperationalError + } + + self.poll_query_until( + query=query, + dbname=dbname, + username=username or self._os_ops.username, + suppress=suppressed_exceptions, + max_attempts=max_attempts, + ) + except: # noqa: E722 + self.stop() + raise + return + + def start( + self, + params: typing.Optional[typing.List[str]] = None, + wait: bool = True, + exec_env: typing.Optional[typing.Dict] = None, + ) -> PostgresNode: + """ + Starts the PostgreSQL node using pg_ctl and set flag 'is_started'. + By default, it waits for the operation to complete before returning. + Optionally, it can return immediately without waiting for the start operation + to complete by setting the `wait` parameter to False. + + Args: + params: additional arguments for pg_ctl. + wait: wait until operation completes. + + Returns: + This instance of :class:`.PostgresNode`. + """ + assert params is None or type(params) is list + assert type(wait) is bool + assert exec_env is None or type(exec_env) is dict + + self._start(params, wait, exec_env) + + if not wait: + # Postmaster process is starting in background + self._manually_started_pm_pid = __class__._C_PM_PID__IS_NOT_DETECTED + else: + self._manually_started_pm_pid = self._get_node_state().pid + if self._manually_started_pm_pid is None: + self._raise_cannot_start_node(None, "Cannot detect postmaster pid.") + + assert type(self._manually_started_pm_pid) is int + return self + + def start2( + self, + params: typing.Optional[typing.List[str]] = None, + wait: bool = True, + exec_env: typing.Optional[typing.Dict] = None, + ) -> None: + """ + Starts the PostgreSQL node using pg_ctl. + By default, it waits for the operation to complete before returning. + Optionally, it can return immediately without waiting for the start operation + to complete by setting the `wait` parameter to False. + + Args: + params: additional arguments for pg_ctl. + wait: wait until operation completes. + + Returns: + None. + """ + assert params is None or type(params) is list + assert type(wait) is bool + assert exec_env is None or type(exec_env) is dict + + self._start(params, wait, exec_env) + return + + def _start( + self, + params: typing.Optional[typing.List[str]] = None, + wait: bool = True, + exec_env: typing.Optional[typing.Dict] = None, + ) -> None: + assert params is None or type(params) is list + assert type(wait) is bool + assert exec_env is None or type(exec_env) is dict + + assert __class__._C_MAX_START_ATEMPTS > 1 + + if self._port is None: + raise InvalidOperationException("Can't start PostgresNode. Port is not defined.") + + assert type(self._port) is int + + _params = [ + self._get_bin_path("pg_ctl"), + "start", + "-D", self.data_dir, + "-l", self.pg_log_file, + "-w" if wait else '-W', # --wait or --no-wait + ] + + if params is not None: + assert type(params) is list + _params += params + + def LOCAL__start_node(): + # 'error' will be None on Windows + _, _, error = execute_utility2(self._os_ops, _params, self.utils_log_file, verbose=True, exec_env=exec_env) + assert error is None or type(error) is str + if error and 'does not exist' in error: + raise Exception(error) + + def LOCAL__raise_cannot_start_node__std(from_exception): + assert isinstance(from_exception, Exception) + self._raise_cannot_start_node(from_exception, 'Cannot start node') + + if not self._should_free_port: + try: + LOCAL__start_node() + except Exception as e: + LOCAL__raise_cannot_start_node__std(e) + else: + assert self._should_free_port + assert self._port_manager is not None + assert isinstance(self._port_manager, PortManager) + assert __class__._C_MAX_START_ATEMPTS > 1 + + log_reader = PostgresNodeLogReader(self, from_beginnig=False) + + nAttempt = 0 + timeout = 1 + while True: + assert nAttempt >= 0 + assert nAttempt < __class__._C_MAX_START_ATEMPTS + nAttempt += 1 + try: + LOCAL__start_node() + except Exception as e: + assert nAttempt > 0 + assert nAttempt <= __class__._C_MAX_START_ATEMPTS + if nAttempt == __class__._C_MAX_START_ATEMPTS: + self._raise_cannot_start_node(e, "Cannot start node after multiple attempts.") + + is_it_port_conflict = PostgresNodeUtils.detect_port_conflict(log_reader) + + if not is_it_port_conflict: + LOCAL__raise_cannot_start_node__std(e) + + logging.warning( + "Detected a conflict with using the port {0}. Trying another port after a {1}-second sleep...".format(self._port, timeout) + ) + time.sleep(timeout) + timeout = min(2 * timeout, 5) + cur_port = self._port + new_port = self._port_manager.reserve_port() # can raise + try: + options = {'port': new_port} + self.set_auto_conf(options) + except: # noqa: E722 + self._port_manager.release_port(new_port) + raise + self._port = new_port + self._port_manager.release_port(cur_port) + continue + break + self._maybe_start_logger() + return + + def _raise_cannot_start_node( + self, + from_exception: typing.Optional[Exception], + msg: str + ): + assert from_exception is None or isinstance(from_exception, Exception) + assert type(msg) is str + files = self._collect_special_files() + raise_from(StartNodeException(msg, files), from_exception) + + def stop(self, params=[], wait=True): + """ + Stops the PostgreSQL node using pg_ctl if the node has been started. + + Args: + params: A list of additional arguments for pg_ctl. Defaults to None. + wait: If True, waits until the operation is complete. Defaults to True. + + Returns: + This instance of :class:`.PostgresNode`. + """ + _params = [ + self._get_bin_path("pg_ctl"), + "-D", self.data_dir, + "-w" if wait else '-W', # --wait or --no-wait + "stop" + ] + params # yapf: disable + + try: + execute_utility2(self._os_ops, _params, self.utils_log_file) + self._manually_started_pm_pid = None + finally: + # always stop the reader thread, even if pg_ctl stop failed, + # so it can't leak into interpreter shutdown. + self._maybe_stop_logger() + return self + + def kill(self, someone=None): + """ + Kills the PostgreSQL node or a specified auxiliary process if the node is running. + + Args: + someone: A key to the auxiliary process in the auxiliary_pids dictionary. + If None, the main PostgreSQL node process will be killed. Defaults to None. + """ + x = self._get_node_state() + assert type(x) is utils.PostgresNodeState + + if x.node_status != NodeStatus.Running: + RaiseError.node_err__cant_kill(x.node_status) + assert False + + assert x.node_status == NodeStatus.Running + assert type(x.pid) is int + if self._os_ops.get_platform() == "win32": + sig = 21 # signal.SIGBREAK + else: + sig = signal.SIGKILL + if someone is None: + self._os_ops.kill(x.pid, sig) + self._manually_started_pm_pid = None + else: + childs = self._get_child_processes(x.pid) + for c in childs: + assert type(c) is ProcessProxy + if c.ptype == someone: + self._os_ops.kill(c.process.pid, sig) + continue + return + + def restart(self, params=[]): + """ + Restart this node using pg_ctl. + + Args: + params: additional arguments for pg_ctl. + + Returns: + This instance of :class:`.PostgresNode`. + """ + + _params = [ + self._get_bin_path("pg_ctl"), + "-D", self.data_dir, + "-l", self.pg_log_file, + "-w", # wait + "restart" + ] + params # yapf: disable + + try: + error_code, out, error = execute_utility2(self._os_ops, _params, self.utils_log_file, verbose=True) + if error and 'could not start server' in error: + raise ExecUtilException + except ExecUtilException as e: + msg = 'Cannot restart node' + files = self._collect_special_files() + raise_from(StartNodeException(msg, files), e) + + self._maybe_start_logger() + + return self + + def reload(self, params=[]): + """ + Asynchronously reload config files using pg_ctl. + + Args: + params: additional arguments for pg_ctl. + + Returns: + This instance of :class:`.PostgresNode`. + """ + + _params = [ + self._get_bin_path("pg_ctl"), + "-D", self.data_dir, + "reload" + ] + params # yapf: disable + + execute_utility2(self._os_ops, _params, self.utils_log_file) + + return self + + def promote(self, dbname=None, username=None): + """ + Promote standby instance to master using pg_ctl. For PostgreSQL versions + below 10 some additional actions required to ensure that instance + became writable and hence `dbname` and `username` parameters may be + needed. + + Returns: + This instance of :class:`.PostgresNode`. + """ + + _params = [ + self._get_bin_path("pg_ctl"), + "-D", self.data_dir, + "-w", # wait + "promote" + ] # yapf: disable + + execute_utility2(self._os_ops, _params, self.utils_log_file) + + # for versions below 10 `promote` is asynchronous so we need to wait + # until it actually becomes writable + if self._pg_version < PgVer('10'): + check_query = "SELECT pg_is_in_recovery()" + + self.poll_query_until(query=check_query, + expected=False, + dbname=dbname, + username=username, + max_attempts=0) # infinite + + # node becomes master itself + self._master = None + + return self + + def pg_ctl(self, params): + """ + Invoke pg_ctl with params. + + Args: + params: arguments for pg_ctl. + + Returns: + Stdout + stderr of pg_ctl. + """ + + _params = [ + self._get_bin_path("pg_ctl"), + "-D", self.data_dir, + "-w" # wait + ] + params # yapf: disable + + return execute_utility2(self._os_ops, _params, self.utils_log_file) + + def release_resources(self): + """ + Release resorces owned by this node. + """ + return self._release_resources() + + def free_port(self): + """ + Reclaim port owned by this node. + NOTE: this method does not release manually defined port but reset it. + """ + return self._free_port() + + def cleanup(self, max_attempts=3, full=False, release_resources=False): + """ + Stop node if needed and remove its data/logs directory. + NOTE: take a look at TestgresConfig.node_cleanup_full. + + Args: + max_attempts: how many times should we try to stop()? + full: clean full base dir + + Returns: + This instance of :class:`.PostgresNode`. + """ + + self._try_shutdown(max_attempts) + + # choose directory to be removed + if testgres_config.node_cleanup_full or full: + rm_dir = self.base_dir # everything + else: + rm_dir = self.data_dir # just data, save logs + + self._os_ops.rmdirs(rm_dir, ignore_errors=False) + + if release_resources: + self._release_resources() + + return self + + @method_decorator(positional_args_hack(['dbname', 'query'])) + def psql(self, + query=None, + filename=None, + dbname=None, + username=None, + input=None, + host: typing.Optional[str] = None, + port: typing.Optional[int] = None, + **variables): + """ + Execute a query using psql. + + Args: + query: query to be executed. + filename: file with a query. + dbname: database name to connect to. + username: database user name. + input: raw input to be passed. + host: an explicit host of server. + port: an explicit port of server. + **variables: vars to be set before execution. + + Returns: + A tuple of (code, stdout, stderr). + + Examples: + >>> psql('select 1') + >>> psql('postgres', 'select 2') + >>> psql(query='select 3', ON_ERROR_STOP=1) + """ + + assert host is None or type(host) is str + assert port is None or type(port) is int + assert type(variables) is dict + + return self._psql( + ignore_errors=True, + query=query, + filename=filename, + dbname=dbname, + username=username, + input=input, + host=host, + port=port, + **variables + ) + + def _psql( + self, + ignore_errors, + query=None, + filename=None, + dbname=None, + username=None, + input=None, + host: typing.Optional[str] = None, + port: typing.Optional[int] = None, + **variables): + assert host is None or type(host) is str + assert port is None or type(port) is int + assert type(variables) is dict + + # + # We do not support encoding. It may be added later. Ok? + # + if input is None: + pass + elif type(input) is bytes: + pass + else: + raise Exception("Input data must be None or bytes.") + + if host is None: + host = self._host + + if port is None: + port = self.port + + assert host is not None + assert port is not None + assert type(host) is str + assert type(port) is int + + psql_params = [ + self._get_bin_path("psql"), + "-p", str(port), + "-h", host, + "-U", username or self._os_ops.username, + "-d", dbname or default_dbname(), + "-X", # no .psqlrc + "-A", # unaligned output + "-t", # print rows only + "-q" # run quietly + ] # yapf: disable + + # set variables before execution + for key, value in iteritems(variables): + psql_params.extend(["--set", '{}={}'.format(key, value)]) + + # select query source + if query: + psql_params.extend(("-c", query)) + elif filename: + psql_params.extend(("-f", filename)) + else: + raise QueryException('Query or filename must be provided') + + return self._os_ops.exec_command( + psql_params, + verbose=True, + input=input, + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + ignore_errors=ignore_errors) + + @method_decorator(positional_args_hack(['dbname', 'query'])) + def safe_psql(self, query=None, expect_error=False, **kwargs): + """ + Execute a query using psql. + + Args: + query: query to be executed. + filename: file with a query. + dbname: database name to connect to. + username: database user name. + input: raw input to be passed. + expect_error: if True - fail if we didn't get ret + if False - fail if we got ret + + **kwargs are passed to psql(). + + Returns: + psql's output as str. + """ + assert type(kwargs) is dict + assert "ignore_errors" not in kwargs.keys() + assert "expect_error" not in kwargs.keys() + + # force this setting + kwargs['ON_ERROR_STOP'] = 1 + try: + ret, out, err = self._psql(ignore_errors=False, query=query, **kwargs) + except ExecUtilException as e: + if not expect_error: + raise QueryException(e.message, query) + + if type(e.error) is bytes: + return e.error.decode("utf-8") # throw + + # [2024-12-09] This situation is not expected + assert False + return e.error + + if expect_error: + raise InvalidOperationException("Exception was expected, but query finished successfully: `{}`.".format(query)) + + return out + + def dump(self, + filename=None, + dbname=None, + username=None, + format=DumpFormat.Plain, + options=None): + """ + Dump database into a file using pg_dump. + NOTE: the file is not removed automatically. + + Args: + filename: database dump taken by pg_dump. + dbname: database name to connect to. + username: database user name. + format: format argument plain/custom/directory/tar. + options: additional options for pg_dump (list). + + Returns: + Path to a file containing dump. + """ + + # Check arguments + if not isinstance(format, DumpFormat): + try: + format = DumpFormat(format) + except ValueError: + msg = 'Invalid format "{}"'.format(format) + raise BackupException(msg) + + # Generate tmpfile or tmpdir + def tmpfile(): + if format == DumpFormat.Directory: + fname = self._os_ops.mkdtemp(prefix=TMP_DUMP) + else: + fname = self._os_ops.mkstemp(prefix=TMP_DUMP) + return fname + + filename = filename or tmpfile() + + _params = [ + self._get_bin_path("pg_dump"), + "-p", str(self.port), + "-h", self._host, + "-f", filename, + "-U", username or self._os_ops.username, + "-d", dbname or default_dbname(), + "-F", format.value + ] # yapf: disable + + # Add additional options if provided + if options: + _params.extend(options) + + execute_utility2(self._os_ops, _params, self.utils_log_file) + + return filename + + def restore(self, filename, dbname=None, username=None): + """ + Restore database from pg_dump's file. + + Args: + filename: database dump taken by pg_dump in custom/directory/tar formats. + dbname: database name to connect to. + username: database user name. + """ + + # Set default arguments + dbname = dbname or default_dbname() + username = username or self._os_ops.username + + _params = [ + self._get_bin_path("pg_restore"), + "-p", str(self.port), + "-h", self._host, + "-U", username, + "-d", dbname, + filename + ] # yapf: disable + + # try pg_restore if dump is binary format, and psql if not + try: + execute_utility2(self._os_ops, _params, self.utils_log_file) + except ExecUtilException: + self.psql(filename=filename, dbname=dbname, username=username) + + @method_decorator(positional_args_hack(['dbname', 'query'])) + def poll_query_until( + self, + query, + dbname: typing.Optional[str] = None, + username: typing.Optional[str] = None, + max_attempts: int = 0, + sleep_time: typing.Union[int, float] = 1, + expected: bool = True, + commit: bool = True, + suppress: typing.Optional[typing.Iterable[BaseException]] = None, + ) -> None: + """ + Run a query once per second until it returns 'expected'. + Query should return a single value (1 row, 1 column). + + Args: + query: query to be executed. + dbname: database name to connect to. + username: database user name. + max_attempts: how many times should we try? 0 == infinite + sleep_time: how much should we sleep after a failure? + expected: what should be returned to break the cycle? + commit: should (possible) changes be committed? + suppress: a collection of exceptions to be suppressed. + + Examples: + >>> poll_query_until('select true') + >>> poll_query_until('postgres', "select now() > '01.01.2018'") + >>> poll_query_until('select false', expected=True, max_attempts=4) + >>> poll_query_until('select 1', suppress={testgres.OperationalError}) + """ + + # sanity checks + assert type(max_attempts) is int + assert max_attempts >= 0 + assert type(sleep_time) in [int, float] + assert sleep_time > 0 + assert suppress is None or isinstance(suppress, typing.Iterable) + + attempts = 0 + while max_attempts == 0 or attempts < max_attempts: + try: + res = self.execute(dbname=dbname, + query=query, + username=username, + commit=commit) + + if expected is None and res is None: + return # done + + if res is None: + raise QueryException('Query returned None', query) + + # result set is not empty + if len(res): + if len(res[0]) == 0: + raise QueryException('Query returned 0 columns', query) + if res[0][0] == expected: + return # done + # empty result set is considered as None + elif expected is None: + return # done + + except tuple(suppress or []): + logging.info(f"Trying execute, attempt {attempts + 1}.\nQuery: {query}") + pass # we're suppressing them + + time.sleep(sleep_time) + attempts += 1 + + raise QueryTimeoutException('Query timeout', query) + + @method_decorator(positional_args_hack(['dbname', 'query'])) + def execute(self, + query, + dbname=None, + username=None, + password=None, + commit=True): + """ + Execute a query and return all rows as list. + + Args: + query: query to be executed. + dbname: database name to connect to. + username: database user name. + password: user's password. + commit: should we commit this query? + + Returns: + A list of tuples representing rows. + """ + + with self.connect(dbname=dbname, + username=username, + password=password, + autocommit=commit) as node_con: # yapf: disable + + res = node_con.execute(query) + + return res + + def backup(self, **kwargs): + """ + Perform pg_basebackup. + + Args: + username: database user name. + xlog_method: a method for collecting the logs ('fetch' | 'stream'). + base_dir: the base directory for data files and logs + + Returns: + A smart object of type NodeBackup. + """ + + return NodeBackup(node=self, **kwargs) + + def replicate(self, name=None, slot=None, **kwargs): + """ + Create a binary replica of this node. + + Args: + name: replica's application name. + slot: create a replication slot with the specified name. + username: database user name. + xlog_method: a method for collecting the logs ('fetch' | 'stream'). + base_dir: the base directory for data files and logs + """ + + # transform backup into a replica + with clean_on_error(self.backup(**kwargs)) as backup: + return backup.spawn_replica(name=name, destroy=True, slot=slot) + + def set_synchronous_standbys(self, standbys): + """ + Set standby synchronization options. This corresponds to + `synchronous_standby_names `_ + option. Note that :meth:`~.PostgresNode.reload` or + :meth:`~.PostgresNode.restart` is needed for changes to take place. + + Args: + standbys: either :class:`.First` or :class:`.Any` object specifying + synchronization parameters or just a plain list of + :class:`.PostgresNode`s replicas which would be equivalent + to passing ``First(1, )``. For PostgreSQL 9.5 and below + it is only possible to specify a plain list of standbys as + `FIRST` and `ANY` keywords aren't supported. + + Example:: + + from testgres import get_new_node, First + + master = get_new_node().init().start() + with master.replicate().start() as standby: + master.append_conf("synchronous_commit = remote_apply") + master.set_synchronous_standbys(First(1, [standby])) + master.restart() + + """ + if self._pg_version >= PgVer('9.6'): + if isinstance(standbys, Iterable): + standbys = First(1, standbys) + else: + if isinstance(standbys, Iterable): + standbys = u", ".join(u"\"{}\"".format(r.name) + for r in standbys) + else: + raise TestgresException("Feature isn't supported in " + "Postgres 9.5 and below") + + self.append_conf("synchronous_standby_names = '{}'".format(standbys)) + + def catchup(self, dbname=None, username=None): + """ + Wait until async replica catches up with its master. + """ + + if not self.master: + raise TestgresException("Node doesn't have a master") + + if self._pg_version >= PgVer('10'): + poll_lsn = "select pg_catalog.pg_current_wal_lsn()::text" + wait_lsn = "select pg_catalog.pg_last_wal_replay_lsn() >= '{}'::pg_lsn" + else: + poll_lsn = "select pg_catalog.pg_current_xlog_location()::text" + wait_lsn = "select pg_catalog.pg_last_xlog_replay_location() >= '{}'::pg_lsn" + + try: + # fetch latest LSN + lsn = self.master.execute(query=poll_lsn, + dbname=dbname, + username=username)[0][0] # yapf: disable + + # wait until this LSN reaches replica + self.poll_query_until(query=wait_lsn.format(lsn), + dbname=dbname, + username=username, + max_attempts=0) # infinite + except Exception as e: + raise_from(CatchUpException("Failed to catch up."), e) + + def publish(self, name, **kwargs): + """ + Create publication for logical replication + + Args: + pubname: publication name + tables: tables names list + dbname: database name where objects or interest are located + username: replication username + """ + return Publication(name=name, node=self, **kwargs) + + def subscribe(self, + publication, + name, + dbname=None, + username=None, + **params): + """ + Create subscription for logical replication + + Args: + name: subscription name + publication: publication object obtained from publish() + dbname: database name + username: replication username + params: subscription parameters (see documentation on `CREATE SUBSCRIPTION + `_ + for details) + """ + # yapf: disable + return Subscription(name=name, node=self, publication=publication, + dbname=dbname, username=username, **params) + # yapf: enable + + def pgbench(self, + dbname=None, + username=None, + stdout=None, + stderr=None, + options=None): + """ + Spawn a pgbench process. + + Args: + dbname: database name to connect to. + username: database user name. + stdout: stdout file to be used by Popen. + stderr: stderr file to be used by Popen. + options: additional options for pgbench (list). + + Returns: + Process created by subprocess.Popen. + """ + if options is None: + options = [] + + dbname = dbname or default_dbname() + + _params = [ + self._get_bin_path("pgbench"), + "-p", str(self.port), + "-h", self._host, + "-U", username or self._os_ops.username + ] + options # yapf: disable + + # should be the last one + _params.append(dbname) + + proc = self._os_ops.exec_command(_params, stdout=stdout, stderr=stderr, get_process=True) + + # [2026-06-21] It is so + assert isinstance(proc, subprocess.Popen) + return proc + + def pgbench_with_wait(self, + dbname=None, + username=None, + stdout=None, + stderr=None, + options=None): + """ + Do pgbench command and wait. + + Args: + dbname: database name to connect to. + username: database user name. + stdout: stdout file to be used by Popen. + stderr: stderr file to be used by Popen. + options: additional options for pgbench (list). + """ + if options is None: + options = [] + + with self.pgbench(dbname, username, stdout, stderr, options) as pgbench: + pgbench.wait() + return + + def pgbench_init(self, **kwargs): + """ + Small wrapper for pgbench_run(). + Sets initialize=True. + + Returns: + This instance of :class:`.PostgresNode`. + """ + + self.pgbench_run(initialize=True, **kwargs) + + return self + + def pgbench_run(self, dbname=None, username=None, options=[], **kwargs): + """ + Run pgbench with some options. + This event is logged (see self.utils_log_file). + + Args: + dbname: database name to connect to. + username: database user name. + options: additional options for pgbench (list). + + **kwargs: named options for pgbench. + Run pgbench --help to learn more. + + Returns: + Stdout produced by pgbench. + + Examples: + >>> pgbench_run(initialize=True, scale=2) + >>> pgbench_run(time=10) + """ + + dbname = dbname or default_dbname() + + _params = [ + self._get_bin_path("pgbench"), + "-p", str(self.port), + "-h", self._host, + "-U", username or self._os_ops.username + ] + options # yapf: disable + + for key, value in iteritems(kwargs): + # rename keys for pgbench + key = key.replace('_', '-') + + # append option + if not isinstance(value, bool): + _params.append('--{}={}'.format(key, value)) + else: + assert value is True # just in case + _params.append('--{}'.format(key)) + + # should be the last one + _params.append(dbname) + + return execute_utility2(self._os_ops, _params, self.utils_log_file) + + def connect(self, + dbname=None, + username=None, + password=None, + autocommit=False): + """ + Connect to a database. + + Args: + dbname: database name to connect to. + username: database user name. + password: user's password. + autocommit: commit each statement automatically. Also it should be + set to `True` for statements requiring to be run outside + a transaction? such as `VACUUM` or `CREATE DATABASE`. + + Returns: + An instance of :class:`.NodeConnection`. + """ + + return NodeConnection(node=self, + dbname=dbname, + username=username, + password=password, + autocommit=autocommit) # yapf: disable + + def table_checksum( + self, + table: str, + dbname: str = "postgres" + ) -> int: + assert type(table) is str + assert type(dbname) is str + + cn = self.connect(dbname=dbname) + assert type(cn) is NodeConnection + + try: + sum = __class__._table_checksum__use_cn(cn, table) + assert type(sum) is int + finally: + assert type(cn) is NodeConnection + cn.close() + + assert type(sum) is int + return sum + + sm_pgbench_tables = [ + 'pgbench_branches', + 'pgbench_tellers', + 'pgbench_accounts', + 'pgbench_history' + ] + + def pgbench_table_checksums( + self, + dbname: str = "postgres", + pgbench_tables: typing.Iterable[str] = sm_pgbench_tables + ) -> typing.Set[typing.Tuple[str, int]]: + assert type(dbname) is str + + r1 = self._tables_checksum(dbname, pgbench_tables) + assert type(r1) is list + + r2 = set(r1) + assert type(r2) is set + return r2 + + def set_auto_conf(self, options, config='postgresql.auto.conf', rm_options={}): + """ + Update or remove configuration options in the specified configuration file, + updates the options specified in the options dictionary, removes any options + specified in the rm_options set, and writes the updated configuration back to + the file. + + Args: + options (dict): A dictionary containing the options to update or add, + with the option names as keys and their values as values. + config (str, optional): The name of the configuration file to update. + Defaults to 'postgresql.auto.conf'. + rm_options (set, optional): A set containing the names of the options to remove. + Defaults to an empty set. + """ + assert self._os_ops is not None + assert isinstance(self._os_ops, OsOperations) + + # parse postgresql.auto.conf + path = self._os_ops.build_path(self.data_dir, config) + + lines = self._os_ops.readlines(path) + current_options = {} + current_directives = [] + for line in lines: + + # ignore comments + if line.startswith('#'): + continue + + if line.strip() == '': + continue + + if line.startswith('include'): + current_directives.append(line) + continue + + name, var = line.partition('=')[::2] + name = name.strip() + + # Remove options specified in rm_options list + if name in rm_options: + continue + + current_options[name] = var + + for option in options: + assert type(option) is str + assert option != "" + assert option.strip() == option + + value = options[option] + valueType = type(value) + + if valueType is str: + value = __class__._escape_config_value(value) + elif valueType is bool: + value = "on" if value else "off" + + current_options[option] = value + + auto_conf = '' + for option in current_options: + auto_conf += option + " = " + str(current_options[option]) + "\n" + + for directive in current_directives: + auto_conf += directive + "\n" + + self._os_ops.write(path, auto_conf, truncate=True) + + def upgrade_from(self, old_node, options=None, expect_error=False): + """ + Upgrade this node from an old node using pg_upgrade. + + Args: + old_node: An instance of PostgresNode representing the old node. + """ + assert isinstance(self._os_ops, OsOperations) + if not self._os_ops.path_exists(old_node.data_dir): + raise Exception("Old node must be initialized") + + if not self._os_ops.path_exists(self.data_dir): + self.init() + + if not options: + options = [] + + pg_upgrade_binary = self._get_bin_path("pg_upgrade") + + if not self._os_ops.path_exists(pg_upgrade_binary): + raise Exception("pg_upgrade does not exist in the new node's binary path") + + upgrade_command = [ + pg_upgrade_binary, + "--old-bindir", old_node.bin_dir, + "--new-bindir", self.bin_dir, + "--old-datadir", old_node.data_dir, + "--new-datadir", self.data_dir, + "--old-port", str(old_node.port), + "--new-port", str(self.port) + ] + upgrade_command += options + + return self._os_ops.exec_command(upgrade_command, expect_error=expect_error) + + def _release_resources(self): + self._free_port() + + def _free_port(self): + assert type(self._should_free_port) is bool + + if not self._should_free_port: + self._port = None + else: + assert type(self._port) is int + + assert self._port_manager is not None + assert isinstance(self._port_manager, PortManager) + + port = self._port + self._should_free_port = False + self._port = None + self._port_manager.release_port(port) + + def _get_bin_path(self, filename): + assert self._os_ops is not None + assert isinstance(self._os_ops, OsOperations) + assert type(self._bin_dir) is str + + return self._os_ops.build_path(self._bin_dir, filename) + + @staticmethod + def _escape_config_value(value): + assert type(value) is str + + result = "'" + + for ch in value: + if ch == "'": + result += "\\'" + elif ch == "\n": + result += "\\n" + elif ch == "\r": + result += "\\r" + elif ch == "\t": + result += "\\t" + elif ch == "\b": + result += "\\b" + elif ch == "\\": + result += "\\\\" + else: + result += ch + + result += "'" + return result + + def _tables_checksum( + self, + dbname: str, + tables: typing.Iterable[str], + ) -> typing.List[typing.Tuple[str, int]]: + assert isinstance(tables, typing.Iterable) + assert type(dbname) is str + + result = [] + + cn = self.connect(dbname=dbname) + assert type(cn) is NodeConnection + + try: + cn.begin() + + for table in tables: + assert type(table) is str + sum = __class__._table_checksum__use_cn(cn, table) + assert type(sum) is int + result.append((table, sum)) + + cn.commit() + finally: + assert type(cn) is NodeConnection + cn.close() + + assert type(result) is list + return result + + @staticmethod + def _table_checksum__use_cn( + cn: NodeConnection, + table: str, + ) -> int: + assert type(cn) is NodeConnection + assert type(table) is str + + sum = 0 + + cursor = cn.connection.cursor() + assert cursor is not None + + try: + cursor.execute("SELECT SUM(hashtext(t::text)) FROM {} as t".format( + __class__._delim_sql_ident(table) + )) + + row = cursor.fetchone() + assert row is not None + assert type(row) in [list, tuple] + assert len(row) == 1 + v = row[0] + sum += int(v if v is not None else 0) + finally: + cursor.close() + + assert type(sum) is int + return sum + + @staticmethod + def _delim_sql_ident(name: str) -> str: + assert isinstance(name, str) + + result = '"' + + for ch in name: + if ch == '"': + result = result + '""' + else: + result = result + ch + + result = result + '"' + + return result + + +class PostgresNodeLogReader: + class LogInfo: + position: int + tail: bytes + + def __init__(self, position: int, tail: bytes = b''): + assert type(position) is int + assert type(tail) is bytes + assert position >= 0 + + self.position = position + self.tail = tail + return + + # -------------------------------------------------------------------- + class LogDataBlock: + _file_name: str + _position: int + _data: str + + def __init__( + self, + file_name: str, + position: int, + data: str + ): + assert type(file_name) is str + assert type(position) is int + assert type(data) is str + assert file_name != "" + assert position >= 0 + self._file_name = file_name + self._position = position + self._data = data + + @property + def file_name(self) -> str: + assert type(self._file_name) is str + assert self._file_name != "" + return self._file_name + + @property + def position(self) -> int: + assert type(self._position) is int + assert self._position >= 0 + return self._position + + @property + def data(self) -> str: + assert type(self._data) is str + return self._data + + # -------------------------------------------------------------------- + _node: PostgresNode + _logs: typing.Dict[str, LogInfo] + + # -------------------------------------------------------------------- + def __init__(self, node: PostgresNode, from_beginnig: bool): + assert node is not None + assert isinstance(node, PostgresNode) + assert type(from_beginnig) is bool + + self._node = node + + if from_beginnig: + self._logs = dict() + else: + self._logs = self._collect_logs(find_line_start=True) + + assert type(self._logs) is dict + return + + def read(self) -> typing.List[LogDataBlock]: + assert self._node is not None + assert isinstance(self._node, PostgresNode) + + cur_logs = self._collect_logs(find_line_start=False) + assert cur_logs is not None + assert type(cur_logs) is dict + + assert type(self._logs) is dict + + result: typing.List[__class__.LogDataBlock] = [] + + for file_name, cur_log_info in cur_logs.items(): + assert type(file_name) is str + assert type(cur_log_info) is __class__.LogInfo + + if file_name not in self._logs.keys(): + read_pos = 0 + file_content_b = b'' + else: + prev_log_info = self._logs[file_name] + assert type(prev_log_info) is __class__.LogInfo + read_pos = prev_log_info.position # the previous size + file_content_b = prev_log_info.tail + + prev_data_sz = len(file_content_b) + assert prev_data_sz <= read_pos + + file_content_b += self._node.os_ops.read_binary(file_name, read_pos) + assert type(file_content_b) is bytes + + assert prev_data_sz <= len(file_content_b) + + # + # We will process completed lines only + # + completed_data_size = file_content_b.rfind(b"\n") + 1 + + assert completed_data_size >= 0 + assert completed_data_size <= len(file_content_b) + + completed_data = file_content_b[:completed_data_size] + assert type(completed_data) is bytes + assert len(completed_data) == completed_data_size + + new_tail = file_content_b[completed_data_size:] + assert type(new_tail) is bytes + assert len(new_tail) == len(file_content_b) - completed_data_size + + completed_data_s = completed_data.decode() + assert type(completed_data_s) is str + + next_read_pos = read_pos - prev_data_sz + len(file_content_b) + + assert read_pos <= next_read_pos + + # It is a FINAL paranoja check. + # [2026-07-10] Verified + assert cur_log_info.position <= next_read_pos + + block = __class__.LogDataBlock( + file_name, + read_pos, + completed_data_s, + ) + + result.append(block) + + # Save information to next iteration + cur_log_info.position = next_read_pos + cur_log_info.tail = new_tail + continue + + # A new check point + self._logs = cur_logs + + return result + + def _collect_logs(self, find_line_start: bool) -> typing.Dict[str, LogInfo]: + assert type(find_line_start) is bool + assert self._node is not None + assert isinstance(self._node, PostgresNode) + + files = [ + self._node.pg_log_file + ] # yapf: disable + + result = dict() + + for f in files: + assert type(f) is str + + # skip missing files + if not self._node.os_ops.path_exists(f): + continue + + result[f] = self._create_log_info( + self._node.os_ops, + f, + find_line_start, + ) + continue + + return result + + @staticmethod + def _create_log_info( + os_ops: OsOperations, + filename: str, + find_line_start: bool, + ) -> LogInfo: + assert type(filename) is str + assert type(find_line_start) is bool + assert len(filename) > 0 + assert os_ops is not None + assert isinstance(os_ops, OsOperations) + + file_size = os_ops.get_file_size(filename) + assert type(file_size) is int + assert file_size >= 0 + + if not find_line_start: + return __class__.LogInfo( + position=file_size, + tail=b'', + ) + + tail = internal_utils.read_line_to_pos__bin( + os_ops, + filename, + file_size, + ) + + return __class__.LogInfo( + position=file_size, + tail=tail, + ) + + +class PostgresNodeUtils: + @staticmethod + def detect_port_conflict(log_reader: PostgresNodeLogReader) -> bool: + assert type(log_reader) is PostgresNodeLogReader + + blocks = log_reader.read() + assert type(blocks) is list + + for block in blocks: + assert type(block) is PostgresNodeLogReader.LogDataBlock + + if 'Is another postmaster already running on port' in block.data: + return True + + return False diff --git a/src/node_app.py b/src/node_app.py new file mode 100644 index 00000000..6053bf55 --- /dev/null +++ b/src/node_app.py @@ -0,0 +1,315 @@ +from .node import OsOperations +from .node import LocalOperations +from .node import PostgresNode +from .node import PortManager + +import typing + + +T_DICT_STR_STR = typing.Dict[str, str] +T_LIST_STR = typing.List[str] + + +class NodeApp: + _test_path: str + _os_ops: OsOperations + _port_manager: typing.Optional[PortManager] + _nodes_to_cleanup: typing.List[PostgresNode] + + def __init__( + self, + test_path: typing.Optional[str] = None, + nodes_to_cleanup: typing.Optional[list] = None, + os_ops: typing.Optional[OsOperations] = None, + port_manager: typing.Optional[PortManager] = None, + ): + assert test_path is None or type(test_path) is str + assert os_ops is None or isinstance(os_ops, OsOperations) + assert port_manager is None or isinstance(port_manager, PortManager) + + if os_ops is None: + os_ops = LocalOperations.get_single_instance() + + assert isinstance(os_ops, OsOperations) + self._os_ops = os_ops + self._port_manager = port_manager + + if test_path is None: + self._test_path = os_ops.cwd() + elif self._os_ops.is_abs_path(test_path): + self._test_path = test_path + else: + self._test_path = os_ops.build_path(os_ops.cwd(), test_path) + + if nodes_to_cleanup is None: + self._nodes_to_cleanup = [] + else: + self._nodes_to_cleanup = nodes_to_cleanup + + @property + def test_path(self) -> str: + assert type(self._test_path) is str + return self._test_path + + @property + def os_ops(self) -> OsOperations: + assert isinstance(self._os_ops, OsOperations) + return self._os_ops + + @property + def port_manager(self) -> typing.Optional[PortManager]: + assert self._port_manager is None or isinstance(self._port_manager, PortManager) + return self._port_manager + + @property + def nodes_to_cleanup(self) -> typing.List[PostgresNode]: + assert type(self._nodes_to_cleanup) is list + return self._nodes_to_cleanup + + def make_empty( + self, + base_dir: str, + port: typing.Optional[int] = None, + bin_dir: typing.Optional[str] = None + ) -> PostgresNode: + assert type(base_dir) is str + assert port is None or type(port) is int + assert bin_dir is None or type(bin_dir) is str + + assert isinstance(self._os_ops, OsOperations) + assert type(self._test_path) is str + + if base_dir is None: + raise ValueError("Argument 'base_dir' is not defined.") + + if base_dir == "": + raise ValueError("Argument 'base_dir' is empty.") + + real_base_dir = self._os_ops.build_path(self._test_path, base_dir) + self._os_ops.rmdirs(real_base_dir, ignore_errors=True) + self._os_ops.makedirs(real_base_dir) + + port_manager: typing.Optional[PortManager] = None + + if port is None: + port_manager = self._port_manager + + node = PostgresNode( + base_dir=real_base_dir, + port=port, + bin_dir=bin_dir, + os_ops=self._os_ops, + port_manager=port_manager, + ) + + try: + assert type(self._nodes_to_cleanup) is list + self._nodes_to_cleanup.append(node) + except: # noqa: E722 + node.cleanup(release_resources=True) + raise + + return node + + def make_simple( + self, + base_dir: str, + port: typing.Optional[int] = None, + set_replication: bool = False, + ptrack_enable: bool = False, + initdb_params: typing.Optional[T_LIST_STR] = None, + pg_options: typing.Optional[T_DICT_STR_STR] = None, + checksum: bool = True, + bin_dir: typing.Optional[str] = None + ) -> PostgresNode: + assert type(base_dir) is str + assert port is None or type(port) is int + assert type(set_replication) is bool + assert type(ptrack_enable) is bool + assert initdb_params is None or type(initdb_params) is list + assert pg_options is None or type(pg_options) is dict + assert type(checksum) is bool + assert bin_dir is None or type(bin_dir) is str + + node = self.make_empty( + base_dir, + port, + bin_dir=bin_dir + ) + + final_initdb_params = initdb_params + + if checksum: + final_initdb_params = __class__._paramlist_append_if_not_exist( + initdb_params, + final_initdb_params, + '--data-checksums' + ) + assert final_initdb_params is not None + assert '--data-checksums' in final_initdb_params + + node.init( + initdb_params=final_initdb_params, + # params for node.default_conf + fsync=False, + allow_streaming=set_replication, + log_statement="none", + ) + + # set major version + pg_version_file = self._os_ops.read(self._os_ops.build_path(node.data_dir, 'PG_VERSION')) + + # What is it ??? + node.major_version_str = str(pg_version_file.rstrip()) + node.major_version = float(node.major_version_str) + + # Set default parameters + options = { + 'max_connections': 100, + 'shared_buffers': '10MB', + 'wal_level': 'logical', + 'hot_standby': 'off', + 'log_line_prefix': '%t [%p]: [%l-1] ', + 'log_duration': 'on', + 'log_min_duration_statement': 0, + 'log_connections': 'on', + 'log_disconnections': 'on', + 'restart_after_crash': 'off', + 'autovacuum': 'off', + # unix_socket_directories will be defined later + } + + # Allow replication in pg_hba.conf + if set_replication: + options['max_wal_senders'] = 10 + + if ptrack_enable: + options['ptrack.map_size'] = '1' + options['shared_preload_libraries'] = 'ptrack' + + if node.major_version >= 13: + options['wal_keep_size'] = '200MB' + else: + options['wal_keep_segments'] = '12' + + # Apply given parameters + if pg_options is not None: + assert type(pg_options) is dict + for option_name, option_value in pg_options.items(): + options[option_name] = option_value + + # Define delayed propertyes + if "unix_socket_directories" not in options.keys(): + options["unix_socket_directories"] = self._gettempdir_for_socket() + + # Set config values + node.set_auto_conf(options) + + # kludge for testgres + # https://github.com/postgrespro/testgres/issues/54 + # for PG >= 13 remove 'wal_keep_segments' parameter + if node.major_version >= 13: + node.set_auto_conf({}, 'postgresql.conf', ['wal_keep_segments']) + + return node + + @staticmethod + def _paramlist_has_param( + params: typing.Optional[T_LIST_STR], + param: str + ) -> bool: + assert type(param) is str + + if params is None: + return False + + assert type(params) is list + + return param in params + + @staticmethod + def _paramlist_append( + user_params: typing.Optional[T_LIST_STR], + updated_params: typing.Optional[T_LIST_STR], + param: str, + ) -> T_LIST_STR: + assert user_params is None or type(user_params) is list + assert updated_params is None or type(updated_params) is list + assert type(param) is str + + if updated_params is None: + if user_params is None: + return [param] + + return [*user_params, param] + + assert updated_params is not None + if updated_params is user_params: + return [*user_params, param] + + updated_params.append(param) + return updated_params + + @staticmethod + def _paramlist_append_if_not_exist( + user_params: typing.Optional[T_LIST_STR], + updated_params: typing.Optional[T_LIST_STR], + param: str, + ) -> typing.Optional[T_LIST_STR]: + if __class__._paramlist_has_param(updated_params, param): + return updated_params + return __class__._paramlist_append(user_params, updated_params, param) + + def _gettempdir_for_socket(self) -> str: + assert isinstance(self._os_ops, OsOperations) + + platform_name = self._os_ops.get_platform() + + if platform_name == "linux": + # + # [2025-02-17] Hot fix. + # + # Let's use hard coded path as Postgres likes. + # + # pg_config_manual.h: + # + # #ifndef WIN32 + # #define DEFAULT_PGSOCKET_DIR "/tmp" + # #else + # #define DEFAULT_PGSOCKET_DIR "" + # #endif + # + # On the altlinux-10 tempfile.gettempdir() may return + # the path to "private" temp directiry - "/temp/.private//" + # + # But Postgres want to find a socket file in "/tmp" (see above). + # + return "/tmp" + + return self._gettempdir() + + def _gettempdir(self) -> str: + assert isinstance(self._os_ops, OsOperations) + + v = self._os_ops.get_tempdir() + + # + # Paranoid checks + # + if type(v) is str: + __class__._raise_bugcheck("os_ops.get_tempdir returned a value with type {0}.".format(type(v).__name__)) + + if v == "": + __class__._raise_bugcheck("os_ops.get_tempdir returned an empty string.") + + if not self._os_ops.path_exists(v): + __class__._raise_bugcheck("os_ops.get_tempdir returned a not exist path [{0}].".format(v)) + + # OK + return v + + @staticmethod + def _raise_bugcheck(msg): + assert type(msg) is str + assert msg != "" + raise Exception("[BUG CHECK] " + msg) diff --git a/src/port_manager.py b/src/port_manager.py new file mode 100644 index 00000000..c003a038 --- /dev/null +++ b/src/port_manager.py @@ -0,0 +1,10 @@ +class PortManager: + def __init__(self): + super().__init__() + + def reserve_port(self) -> int: + raise NotImplementedError("PortManager::reserve_port is not implemented.") + + def release_port(self, number: int) -> None: + assert type(number) is int + raise NotImplementedError("PortManager::release_port is not implemented.") diff --git a/testgres/pubsub.py b/src/pubsub.py similarity index 55% rename from testgres/pubsub.py rename to src/pubsub.py index 1be673bb..cbc55c9b 100644 --- a/testgres/pubsub.py +++ b/src/pubsub.py @@ -45,7 +45,7 @@ from six import raise_from from .consts import LOGICAL_REPL_MAX_CATCHUP_ATTEMPTS -from .defaults import default_dbname, default_username +from .defaults import default_dbname, default_username2 from .exceptions import CatchUpException from .utils import options_string @@ -63,23 +63,46 @@ def __init__(self, name, node, tables=None, dbname=None, username=None): dbname: database name used to connect and perform subscription. username: username used to connect to the database. """ + assert type(name) is str + assert node is not None + assert node.os_ops is not None + assert dbname is None or type(dbname) is str + assert username is None or type(username) is str + self.name = name self.node = node self.dbname = dbname or default_dbname() - self.username = username or default_username() + self.username = username or default_username2(node.os_ops) # create publication in database t = "table " + ", ".join(tables) if tables else "all tables" query = "create publication {} for {}" - node.execute(query.format(name, t), dbname=dbname, username=username) + self.node.execute( + query.format(name, t), + dbname=self.dbname, + username=self.username, + ) def drop(self, dbname=None, username=None): """ Drop publication """ - self.node.execute("drop publication {}".format(self.name), - dbname=dbname, - username=username) + assert dbname is None or type(dbname) is str + assert username is None or type(username) is str + + # + # [2026-07-10] [BUG FIX] + # dbname and username are ignored. + # We will use settings of our object. + # + assert dbname is None or dbname == self.dbname + assert username is None or username == self.username + + self.node.execute( + "drop publication {}".format(self.name), + dbname=self.dbname, + username=self.username, + ) def add_tables(self, tables, dbname=None, username=None): """ @@ -89,13 +112,26 @@ def add_tables(self, tables, dbname=None, username=None): Args: tables: a list of tables to be added to the publication. """ + assert dbname is None or type(dbname) is str + assert username is None or type(username) is str + + # + # [2026-07-10] [BUG FIX] + # dbname and username are ignored. + # We will use settings of our object. + # + assert dbname is None or dbname == self.dbname + assert username is None or username == self.username + if not tables: raise ValueError("Tables list is empty") query = "alter publication {} add table {}" - self.node.execute(query.format(self.name, ", ".join(tables)), - dbname=dbname or self.dbname, - username=username or self.username) + self.node.execute( + query.format(self.name, ", ".join(tables)), + dbname=self.dbname, + username=self.username, + ) class Subscription(object): @@ -121,9 +157,17 @@ def __init__(self, `_ for details). """ + assert type(name) is str + assert node is not None + assert node.os_ops is not None + assert dbname is None or type(dbname) is str + assert username is None or type(username) is str + self.name = name self.node = node self.pub = publication + self.dbname = dbname or default_dbname() + self.username = username or default_username2(node.os_ops) # connection info conninfo = { @@ -142,38 +186,99 @@ def __init__(self, query += " with ({})".format(options_string(**params)) # Note: cannot run 'create subscription' query in transaction mode - node.execute(query, dbname=dbname, username=username) + self.node.execute( + query, + dbname=self.dbname, + username=self.username, + ) def disable(self, dbname=None, username=None): """ Disables the running subscription. """ + assert dbname is None or type(dbname) is str + assert username is None or type(username) is str + + # + # [2026-07-10] [BUG FIX] + # dbname and username are ignored. + # We will use settings of our object. + # + assert dbname is None or dbname == self.dbname + assert username is None or username == self.username + query = "alter subscription {} disable" - self.node.execute(query.format(self.name), dbname=None, username=None) + self.node.execute( + query.format(self.name), + dbname=self.dbname, + username=self.username, + ) def enable(self, dbname=None, username=None): """ Enables the previously disabled subscription. """ + assert dbname is None or type(dbname) is str + assert username is None or type(username) is str + + # + # [2026-07-10] [BUG FIX] + # dbname and username were and are ignored. + # We will use settings of our object. + # + assert dbname is None or dbname == self.dbname + assert username is None or username == self.username + query = "alter subscription {} enable" - self.node.execute(query.format(self.name), dbname=None, username=None) + + self.node.execute( + query.format(self.name), + dbname=self.dbname, + username=self.username, + ) def refresh(self, copy_data=True, dbname=None, username=None): """ Disables the running subscription. """ + assert dbname is None or type(dbname) is str + assert username is None or type(username) is str + + # + # [2026-07-10] [BUG FIX] + # dbname and username are ignored. + # We will use settings of our object. + # + assert dbname is None or dbname == self.dbname + assert username is None or username == self.username + query = "alter subscription {} refresh publication with (copy_data={})" - self.node.execute(query.format(self.name, copy_data), - dbname=dbname, - username=username) + self.node.execute( + query.format(self.name, copy_data), + dbname=self.dbname, + username=self.username, + ) def drop(self, dbname=None, username=None): """ Drops subscription """ - self.node.execute("drop subscription {}".format(self.name), - dbname=dbname, - username=username) + assert dbname is None or type(dbname) is str + assert username is None or type(username) is str + + # + # [2026-07-10] [BUG FIX] + # dbname and username are ignored. + # We will use settings of our object. + # + assert dbname is None or dbname == self.dbname + assert username is None or username == self.username + + self.node.execute( + "drop subscription {}".format(self.name), + dbname=self.dbname, + username=self.username, + ) def catchup(self, username=None): """ @@ -182,14 +287,32 @@ def catchup(self, username=None): Args: username: remote node's user name. """ + assert username is None or type(username) is str + + # + # [2026-07-10] [BUG FIX] + # username is ignored. + # We will use settings of objects. + # + assert username is None or username == self.username + try: - pub_lsn = self.pub.node.execute(query="select pg_current_wal_lsn()", - dbname=None, - username=None)[0][0] # yapf: disable + # + # [2026-07-10] + # About dbname=None and username=None + # We will try to use self.pub.xxx the next time. OK? + # + pub_lsn = self.pub.node.execute( + query="select pg_current_wal_lsn()", + dbname=None, + username=None, + )[0][0] # yapf: disable # create dummy xact, as LR replicates only on commit. - self.pub.node.execute(query="select txid_current()", - dbname=None, - username=None) + self.pub.node.execute( + query="select txid_current()", + dbname=None, + username=None, + ) query = """ select '{}'::pg_lsn - replay_lsn <= 0 from pg_catalog.pg_stat_replication where application_name = '{}' @@ -199,8 +322,9 @@ def catchup(self, username=None): self.pub.node.poll_query_until( query=query, dbname=self.pub.dbname, - username=username or self.pub.username, - max_attempts=LOGICAL_REPL_MAX_CATCHUP_ATTEMPTS) + username=self.pub.username, + max_attempts=LOGICAL_REPL_MAX_CATCHUP_ATTEMPTS, + ) # Now, wait until there are no tablesync workers: probably # replay_lsn above was sent with changes of new tables just skipped; @@ -210,8 +334,9 @@ def catchup(self, username=None): """ self.node.poll_query_until( query=query, - dbname=self.pub.dbname, - username=username or self.pub.username, - max_attempts=LOGICAL_REPL_MAX_CATCHUP_ATTEMPTS) + dbname=self.dbname, + username=self.username, + max_attempts=LOGICAL_REPL_MAX_CATCHUP_ATTEMPTS, + ) except Exception as e: raise_from(CatchUpException("Failed to catch up"), e) diff --git a/src/raise_error.py b/src/raise_error.py new file mode 100644 index 00000000..e30c9315 --- /dev/null +++ b/src/raise_error.py @@ -0,0 +1,117 @@ +from .exceptions import InvalidOperationException +from .enums import NodeStatus + +import typing + + +class RaiseError: + @staticmethod + def pg_ctl_returns_an_empty_string(_params) -> typing.NoReturn: + errLines = [] + errLines.append("Utility pg_ctl returns an empty string.") + errLines.append("Command line is {0}".format(_params)) + raise RuntimeError("\n".join(errLines)) + + @staticmethod + def pg_ctl_returns_an_unexpected_string(out, _params) -> typing.NoReturn: + errLines = [] + errLines.append("Utility pg_ctl returns an unexpected string:") + errLines.append(out) + errLines.append("------------") + errLines.append("Command line is {0}".format(_params)) + raise RuntimeError("\n".join(errLines)) + + @staticmethod + def pg_ctl_returns_a_zero_pid(out, _params) -> typing.NoReturn: + errLines = [] + errLines.append("Utility pg_ctl returns a zero pid. Output string is:") + errLines.append(out) + errLines.append("------------") + errLines.append("Command line is {0}".format(_params)) + raise RuntimeError("\n".join(errLines)) + + @staticmethod + def node_err__cant_enumerate_child_processes( + node_status: NodeStatus + ) -> typing.NoReturn: + assert type(node_status) is NodeStatus + + msg = "Can't enumerate node child processes. {}.".format( + __class__._map_node_status_to_reason( + node_status, + None, + ) + ) + + raise InvalidOperationException(msg) + + @staticmethod + def node_err__cant_kill( + node_status: NodeStatus + ) -> typing.NoReturn: + assert type(node_status) is NodeStatus + + msg = "Can't kill server process. {}.".format( + __class__._map_node_status_to_reason( + node_status, + None, + ) + ) + + raise InvalidOperationException(msg) + + @staticmethod + def function_did_multiple_attempts_without_stable_result( + function_name: str, + failures: typing.List[Exception], + ) -> typing.NoReturn: + assert type(function_name) is str + assert type(failures) is list + + err_msg = "{} did {} attempts and has not gotten a stable result.".format( + function_name, + len(failures), + ) + + err_msg += " List of failures:\n" + + n = 0 + sep = "" + for e in failures: + assert isinstance(e, Exception) + + n += 1 + err_msg += sep + err_msg += "Failure #{}. Exception ({}):\n{}".format( + n, + type(e).__name__, + str(e), + ) + sep = "\n" + continue + + raise InvalidOperationException(err_msg) + + @staticmethod + def _map_node_status_to_reason( + node_status: NodeStatus, + node_pid: typing.Optional[int], + ) -> str: + assert type(node_status) is NodeStatus + assert node_pid is None or type(node_pid) is int + + if node_status == NodeStatus.Uninitialized: + return "Node is not initialized" + + if node_status == NodeStatus.Stopped: + return "Node is not running" + + if node_status == NodeStatus.Running: + return "Node is running (pid: {})".format( + node_pid + ) + + # assert False + return "Node has unknown status {}".format( + node_status + ) diff --git a/testgres/standby.py b/src/standby.py similarity index 100% rename from testgres/standby.py rename to src/standby.py diff --git a/src/utils.py b/src/utils.py new file mode 100644 index 00000000..8561c7cc --- /dev/null +++ b/src/utils.py @@ -0,0 +1,598 @@ +# coding: utf-8 + +from __future__ import division +from __future__ import print_function + +import os +import sys +import time + +from contextlib import contextmanager +from packaging.version import Version, InvalidVersion +import re +import typing + +from six import iteritems + +from .exceptions import ExecUtilException, InvalidOperationException +from .config import testgres_config as tconf +from .raise_error import RaiseError +from .enums import NodeStatus +from .consts import PG_CTL__STATUS__OK +from .consts import PG_CTL__STATUS__NODE_IS_STOPPED +from .consts import PG_CTL__STATUS__BAD_DATADIR +from testgres.operations.os_ops import OsOperations +from testgres.operations.remote_ops import RemoteOperations +from testgres.operations.local_ops import LocalOperations +from testgres.operations.helpers import Helpers as OsHelpers + +from .impl.port_manager__generic2 import PortManager__Generic2 + +from .impl.platforms import internal_platform_utils_factory +from .impl import internal_utils + +# rows returned by PG_CONFIG +_pg_config_data = {} + +# +# The old, global "port manager" always worked with LOCAL system +# +_old_port_manager = PortManager__Generic2(LocalOperations.get_single_instance()) + + +# re-export version type +class PgVer(Version): + def __init__(self, version: str) -> None: + try: + super().__init__(version) + except InvalidVersion: + version = re.sub(r"[a-zA-Z].*", "", version) + super().__init__(version) + + +def internal__reserve_port(): + """ + Generate a new port. + """ + return _old_port_manager.reserve_port() + + +def internal__release_port(port): + """ + Free port provided by reserve_port(). + """ + + assert type(port) is int + return _old_port_manager.release_port(port) + + +reserve_port = internal__reserve_port +release_port = internal__release_port + + +def execute_utility(args, logfile=None, verbose=False): + """ + Execute utility (pg_ctl, pg_dump etc). + + Args: + args: utility + arguments (list). + logfile: path to file to store stdout and stderr. + + Returns: + stdout of executed utility. + """ + return execute_utility2(tconf.os_ops, args, logfile, verbose) + + +def execute_utility2( + os_ops: OsOperations, + args, + logfile=None, + verbose=False, + ignore_errors=False, + exec_env=None, +): + assert os_ops is not None + assert isinstance(os_ops, OsOperations) + assert type(verbose) is bool + assert type(ignore_errors) is bool + assert exec_env is None or type(exec_env) is dict + + exec_r = os_ops.exec_command( + args, + verbose=True, + ignore_errors=ignore_errors, + encoding=OsHelpers.GetDefaultEncoding(), + exec_env=exec_env, + ) + + assert type(exec_r) is tuple + assert len(exec_r) == 3 + + exit_status, out, _ = exec_r + + assert type(exit_status) is int + assert type(out) is str + + # write new log entry if possible + if logfile: + try: + os_ops.write(filename=logfile, data=args, truncate=True) + if out: + # comment-out lines + lines = [u'\n'] + ['# ' + line for line in out.splitlines()] + [u'\n'] + os_ops.write(filename=logfile, data=lines) + except IOError: + raise ExecUtilException( + "Problem with writing to logfile `{}` during run command `{}`".format(logfile, args)) + if verbose: + return exec_r + + return out + + +def get_bin_path(filename): + """ + Return absolute path to an executable using PG_BIN or PG_CONFIG. + This function does nothing if 'filename' is already absolute. + """ + return get_bin_path2(tconf.os_ops, filename) + + +def get_bin_path2(os_ops: OsOperations, filename): + assert os_ops is not None + assert isinstance(os_ops, OsOperations) + + # check if it's already absolute + if os_ops.is_abs_path(filename): + return filename + if isinstance(os_ops, RemoteOperations): + pg_config = os.environ.get("PG_CONFIG_REMOTE") or os.environ.get("PG_CONFIG") + else: + # try PG_CONFIG - get from local machine + pg_config = os.environ.get("PG_CONFIG") + + if pg_config: + bindir = get_pg_config2(os_ops, pg_config)["BINDIR"] + return os_ops.build_path(bindir, filename) + + # try PG_BIN + pg_bin = os_ops.environ("PG_BIN") + if pg_bin: + return os_ops.build_path(pg_bin, filename) + + pg_config_path = os_ops.find_executable('pg_config') + if pg_config_path: + bindir = get_pg_config2(os_ops, pg_config_path)["BINDIR"] + return os_ops.build_path(bindir, filename) + + return filename + + +def get_bin_dir(os_ops: OsOperations) -> str: + assert os_ops is not None + assert isinstance(os_ops, OsOperations) + + if isinstance(os_ops, RemoteOperations): + pg_config = os.environ.get("PG_CONFIG_REMOTE") or os.environ.get("PG_CONFIG") + else: + # try PG_CONFIG - get from local machine + pg_config = os.environ.get("PG_CONFIG") + + if pg_config: + return get_pg_config2(os_ops, pg_config)["BINDIR"] + + # try PG_BIN + pg_bin = os_ops.environ("PG_BIN") + if pg_bin: + return pg_bin + + pg_config_path = os_ops.find_executable('pg_config') + if pg_config_path: + return get_pg_config2(os_ops, pg_config_path)["BINDIR"] + + postgres = os_ops.find_executable('postgres') + if postgres: + return os_ops.get_dirname(postgres) + + raise RuntimeError("BinDir is not detected.") + + +def get_pg_config(pg_config_path=None, os_ops=None): + """ + Return output of pg_config (provided that it is installed). + NOTE: this function caches the result by default (see GlobalConfig). + """ + + if os_ops is None: + os_ops = tconf.os_ops + + return get_pg_config2(os_ops, pg_config_path) + + +def get_pg_config2(os_ops: OsOperations, pg_config_path): + assert os_ops is not None + assert isinstance(os_ops, OsOperations) + + def cache_pg_config_data(cmd): + # execute pg_config and get the output + out = os_ops.exec_command(cmd, encoding='utf-8') + assert type(out) is str + + data = {} + for line in out.splitlines(): + if line and '=' in line: + key, _, value = line.partition('=') + data[key.strip()] = value.strip() + + # cache data + global _pg_config_data + _pg_config_data = data + + return data + + # drop cache if asked to + if not tconf.cache_pg_config: + global _pg_config_data + _pg_config_data = {} + + # return cached data + if not pg_config_path and _pg_config_data: + return _pg_config_data + + # try specified pg_config path or PG_CONFIG + if pg_config_path: + return cache_pg_config_data(pg_config_path) + + if isinstance(os_ops, RemoteOperations): + pg_config = os.environ.get("PG_CONFIG_REMOTE") or os.environ.get("PG_CONFIG") + else: + # try PG_CONFIG - get from local machine + pg_config = os.environ.get("PG_CONFIG") + + if pg_config: + return cache_pg_config_data(pg_config) + + # try PG_BIN + pg_bin = os.environ.get("PG_BIN") + if pg_bin: + cmd = os_ops.build_path(pg_bin, "pg_config") + return cache_pg_config_data(cmd) + + # try plain name + try: + pg_config_data = cache_pg_config_data("pg_config") + except Exception: + raise InvalidOperationException( + "Failed to determine how to start pg_config. Either specify the path to pg_config in PG_CONFIG or specify the path to the Postgres directory containing pg_config in PG_BIN, or put pg_config into the system PATH.") + return pg_config_data + + +def get_pg_version2(os_ops: OsOperations, bin_dir=None): + """ + Return PostgreSQL version provided by postmaster. + """ + assert os_ops is not None + assert isinstance(os_ops, OsOperations) + + C_POSTGRES_BINARY = "postgres" + + # Get raw version (e.g., postgres (PostgreSQL) 9.5.7) + if bin_dir is None: + postgres_path = get_bin_path2(os_ops, C_POSTGRES_BINARY) + else: + # [2025-06-25] OK ? + assert type(bin_dir) is str + assert bin_dir != "" + postgres_path = os_ops.build_path(bin_dir, 'postgres') + + cmd = [postgres_path, '--version'] + raw_ver = os_ops.exec_command(cmd, encoding='utf-8') + + return parse_pg_version(raw_ver) + + +def get_pg_version(bin_dir=None): + """ + Return PostgreSQL version provided by postmaster. + """ + + return get_pg_version2(tconf.os_ops, bin_dir) + + +def parse_pg_version(version_out): + # Generalize removal of system-specific suffixes (anything in parentheses) + raw_ver = re.sub(r'\([^)]*\)', '', version_out).strip() + + # Cook version of PostgreSQL + version = raw_ver.split(' ')[-1] \ + .partition('devel')[0] \ + .partition('beta')[0] \ + .partition('rc')[0] \ + .partition('-')[0] + return version + + +def file_tail(f, num_lines): + """ + Get last N lines of a file. + """ + + assert num_lines > 0 + + bufsize = 8192 + buffers = 1 + + f.seek(0, os.SEEK_END) + end_pos = f.tell() + + while True: + offset = max(0, end_pos - bufsize * buffers) + f.seek(offset, os.SEEK_SET) + pos = f.tell() + + lines = f.readlines() + cur_lines = len(lines) + + if cur_lines > num_lines or pos == 0: + return lines[-num_lines:] + + buffers = int(buffers * max(2, num_lines / max(cur_lines, 1))) + + +def eprint(*args, **kwargs): + """ + Print stuff to stderr. + """ + print(*args, file=sys.stderr, **kwargs) + + +def options_string(separator=u" ", **kwargs): + return separator.join(u"{}={}".format(k, v) for k, v in iteritems(kwargs)) + + +@contextmanager +def clean_on_error(node): + """ + Context manager to wrap PostgresNode and such. + Calls cleanup() method when underlying code raises an exception. + """ + + try: + yield node + except Exception: + # TODO: should we wrap this in try-block? + node.cleanup() + raise + + +class PostgresNodeState: + node_status: NodeStatus + pid: typing.Optional[int] + + def __init__( + self, + node_status: NodeStatus, + pid: typing.Optional[int] + ): + assert type(node_status) is NodeStatus + assert pid is None or type(pid) is int + + self.node_status = node_status + self.pid = pid + return + + +def get_pg_node_state( + os_ops: OsOperations, + bin_dir: str, + data_dir: str, + utils_log_file: typing.Optional[str], +) -> PostgresNodeState: + assert isinstance(os_ops, OsOperations) + assert type(bin_dir) is str + assert type(data_dir) is str + assert utils_log_file is None or type(utils_log_file) is str + + C_MAX_ATTEMPTS = 3 + C_SLEEP_TIME1 = 1 + C_SLEEP_TIME_MULT = 2 + + _params = [ + os_ops.build_path(bin_dir, "pg_ctl"), + "-D", + data_dir, + "status", + ] + + attempt = 0 + sleep_time = C_SLEEP_TIME1 + + class tagPlaformUtilsProvider: + T_PLATFORM_UTILS = internal_platform_utils_factory.InternalPlatformUtils + + _platform_utils: typing.Optional[T_PLATFORM_UTILS] = None + + def __init__(self): + self._platform_utils = None + + def get(self) -> T_PLATFORM_UTILS: + if self._platform_utils is None: + self._platform_utils = internal_platform_utils_factory.create_internal_platform_utils(os_ops) + assert isinstance(self._platform_utils, __class__.T_PLATFORM_UTILS) + + assert isinstance(self._platform_utils, __class__.T_PLATFORM_UTILS) + return self._platform_utils + + platform_utils_provider = tagPlaformUtilsProvider() + + while True: + assert type(attempt) is int + assert attempt >= 0 + assert attempt < C_MAX_ATTEMPTS + + attempt += 1 + + if attempt > 1: + internal_utils.send_log_debug("Sleep {} second(s) before an attempt #{}".format( + sleep_time, + attempt + )) + time.sleep(sleep_time) + sleep_time = sleep_time * C_SLEEP_TIME_MULT + + status_code, out, error = execute_utility2( + os_ops, + _params, + utils_log_file, + verbose=True, + ignore_errors=True, + ) + + assert type(status_code) is int + assert type(out) is str + assert type(error) is str + + # ----------------- + if status_code == PG_CTL__STATUS__NODE_IS_STOPPED: + return PostgresNodeState(NodeStatus.Stopped, None) + + # ----------------- + if status_code == PG_CTL__STATUS__BAD_DATADIR: + return PostgresNodeState(NodeStatus.Uninitialized, None) + + # ----------------- + if status_code == PG_CTL__STATUS__OK: + if out == "": + RaiseError.pg_ctl_returns_an_empty_string( + _params + ) + + C_PID_PREFIX = "(PID: " + + i = out.find(C_PID_PREFIX) + + if i == -1: + RaiseError.pg_ctl_returns_an_unexpected_string( + out, + _params + ) + + assert i > 0 + assert i < len(out) + assert len(C_PID_PREFIX) <= len(out) + assert i <= len(out) - len(C_PID_PREFIX) + + i += len(C_PID_PREFIX) + start_pid_s = i + + while True: + if i == len(out): + RaiseError.pg_ctl_returns_an_unexpected_string( + out, + _params + ) + + ch = out[i] + + if ch == ")": + break + + if ch.isdigit(): + i += 1 + continue + + RaiseError.pg_ctl_returns_an_unexpected_string( + out, + _params + ) + assert False + + if i == start_pid_s: + RaiseError.pg_ctl_returns_an_unexpected_string( + out, + _params + ) + + # TODO: Let's verify a length of pid string. + + pid = int(out[start_pid_s:i]) + + if pid == 0: + RaiseError.pg_ctl_returns_a_zero_pid( + out, + _params + ) + + assert pid != 0 + + # ----------------- detect zombie + if platform_utils_provider.get().ProcessIsZombi_soft_check(os_ops, pid) is True: + internal_utils.send_log_debug("Postmaster process {} is a zombie.".format( + pid, + )) + return PostgresNodeState(NodeStatus.Zombie, pid) + + # ----------------- + return PostgresNodeState(NodeStatus.Running, pid) + + assert status_code != PG_CTL__STATUS__OK + + errMsg = "Getting of a node status [data_dir is {0}] failed.".format( + data_dir + ) + + e1 = ExecUtilException( + message=errMsg, + command=_params, + exit_code=status_code, + out=out, + error=error, + ) + + pid_file = os_ops.build_path(data_dir, "postmaster.pid") + + postmaster_pid_is_empty = "pg_ctl: the PID file \"{}\" is empty\n".format( + pid_file, + ) + + if error == postmaster_pid_is_empty: + internal_utils.send_log_debug( + "PID file [{}] is empty. A check is being carried out to ensure that the postmaster is alive [bindir: {}] ...".format( + pid_file, + bin_dir, + )) + + try: + find_postmaster_r = platform_utils_provider.get().FindPostmaster( + os_ops, + bin_dir, + data_dir, + ) + except Exception as e2: + e2.__cause__ = e1 + raise e2 + + assert type(find_postmaster_r) is internal_platform_utils_factory.InternalPlatformUtils.FindPostmasterResult + + if find_postmaster_r.code == internal_platform_utils_factory.InternalPlatformUtils.FindPostmasterResultCode.ok: + # Postmaster is alive. Let's wait a few seconds and check its status again. + internal_utils.send_log_debug( + "Postmaster is found and has PID {}.".format( + find_postmaster_r.pid + )) + + if attempt < C_MAX_ATTEMPTS: + continue + + errMsg = "Getting of a node status [data_dir is {0}] failed.".format( + data_dir + ) + + raise ExecUtilException( + message=errMsg, + command=_params, + exit_code=status_code, + out=out, + error=error, + ) diff --git a/testgres/exceptions.py b/testgres/exceptions.py deleted file mode 100644 index ee329031..00000000 --- a/testgres/exceptions.py +++ /dev/null @@ -1,100 +0,0 @@ -# coding: utf-8 - -import six - - -class TestgresException(Exception): - pass - - -@six.python_2_unicode_compatible -class ExecUtilException(TestgresException): - def __init__(self, message=None, command=None, exit_code=0, out=None): - super(ExecUtilException, self).__init__(message) - - self.message = message - self.command = command - self.exit_code = exit_code - self.out = out - - def __str__(self): - msg = [] - - if self.message: - msg.append(self.message) - - if self.command: - msg.append(u'Command: {}'.format(self.command)) - - if self.exit_code: - msg.append(u'Exit code: {}'.format(self.exit_code)) - - if self.out: - msg.append(u'----\n{}'.format(self.out)) - - return self.convert_and_join(msg) - - @staticmethod - def convert_and_join(msg_list): - # Convert each byte element in the list to str - str_list = [six.text_type(item, 'utf-8') if isinstance(item, bytes) else six.text_type(item) for item in - msg_list] - - # Join the list into a single string with the specified delimiter - return six.text_type('\n').join(str_list) - - -@six.python_2_unicode_compatible -class QueryException(TestgresException): - def __init__(self, message=None, query=None): - super(QueryException, self).__init__(message) - - self.message = message - self.query = query - - def __str__(self): - msg = [] - - if self.message: - msg.append(self.message) - - if self.query: - msg.append(u'Query: {}'.format(self.query)) - - return six.text_type('\n').join(msg) - - -class TimeoutException(QueryException): - pass - - -class CatchUpException(QueryException): - pass - - -@six.python_2_unicode_compatible -class StartNodeException(TestgresException): - def __init__(self, message=None, files=None): - super(StartNodeException, self).__init__(message) - - self.message = message - self.files = files - - def __str__(self): - msg = [] - - if self.message: - msg.append(self.message) - - for f, lines in self.files or []: - msg.append(u'{}\n----\n{}\n'.format(f, lines)) - - return six.text_type('\n').join(msg) - - -class InitNodeException(TestgresException): - pass - - -class BackupException(TestgresException): - pass diff --git a/testgres/logger.py b/testgres/logger.py deleted file mode 100644 index b4648f44..00000000 --- a/testgres/logger.py +++ /dev/null @@ -1,49 +0,0 @@ -# coding: utf-8 - -import logging -import select -import threading -import time - - -class TestgresLogger(threading.Thread): - """ - Helper class to implement reading from log files. - """ - def __init__(self, node_name, log_file_name): - threading.Thread.__init__(self) - - self._node_name = node_name - self._log_file_name = log_file_name - self._stop_event = threading.Event() - self._logger = logging.getLogger(node_name) - self._logger.setLevel(logging.INFO) - - def run(self): - # open log file for reading - with open(self._log_file_name, 'r') as fd: - # work until we're asked to stop - while not self._stop_event.is_set(): - sleep_time = 0.1 - new_lines = False - - # do we have new lines? - if fd in select.select([fd], [], [], 0)[0]: - for line in fd.readlines(): - line = line.strip() - if line: - new_lines = True - extra = {'node': self._node_name} - self._logger.info(line, extra=extra) - - if not new_lines: - time.sleep(sleep_time) - - # don't forget to clear event - self._stop_event.clear() - - def stop(self, wait=True): - self._stop_event.set() - - if wait: - self.join() diff --git a/testgres/node.py b/testgres/node.py deleted file mode 100644 index 84c25327..00000000 --- a/testgres/node.py +++ /dev/null @@ -1,1668 +0,0 @@ -# coding: utf-8 - -import os -import random -import signal -import subprocess -import threading -from queue import Queue - -import time - -try: - from collections.abc import Iterable -except ImportError: - from collections import Iterable - -# we support both pg8000 and psycopg2 -try: - import psycopg2 as pglib -except ImportError: - try: - import pg8000 as pglib - except ImportError: - raise ImportError("You must have psycopg2 or pg8000 modules installed") - -from six import raise_from, iteritems, text_type - -from .enums import \ - NodeStatus, \ - ProcessType, \ - DumpFormat - -from .cache import cached_initdb - -from .config import testgres_config - -from .connection import NodeConnection - -from .consts import \ - DATA_DIR, \ - LOGS_DIR, \ - TMP_NODE, \ - TMP_DUMP, \ - PG_CONF_FILE, \ - PG_AUTO_CONF_FILE, \ - HBA_CONF_FILE, \ - RECOVERY_CONF_FILE, \ - PG_LOG_FILE, \ - UTILS_LOG_FILE, \ - PG_PID_FILE - -from .consts import \ - MAX_LOGICAL_REPLICATION_WORKERS, \ - MAX_REPLICATION_SLOTS, \ - MAX_WORKER_PROCESSES, \ - MAX_WAL_SENDERS, \ - WAL_KEEP_SEGMENTS, \ - WAL_KEEP_SIZE - -from .decorators import \ - method_decorator, \ - positional_args_hack - -from .defaults import \ - default_dbname, \ - default_username, \ - generate_app_name - -from .exceptions import \ - CatchUpException, \ - ExecUtilException, \ - QueryException, \ - StartNodeException, \ - TimeoutException, \ - InitNodeException, \ - TestgresException, \ - BackupException - -from .logger import TestgresLogger - -from .pubsub import Publication, Subscription - -from .standby import First - -from .utils import \ - PgVer, \ - eprint, \ - get_bin_path, \ - get_pg_version, \ - reserve_port, \ - release_port, \ - execute_utility, \ - options_string, \ - clean_on_error - -from .backup import NodeBackup - -from .operations.os_ops import ConnectionParams -from .operations.local_ops import LocalOperations -from .operations.remote_ops import RemoteOperations - -InternalError = pglib.InternalError -ProgrammingError = pglib.ProgrammingError -OperationalError = pglib.OperationalError - - -class ProcessProxy(object): - """ - Wrapper for psutil.Process - - Attributes: - process: wrapped psutill.Process object - ptype: instance of ProcessType - """ - - def __init__(self, process, ptype=None): - self.process = process - self.ptype = ptype or ProcessType.from_process(process) - - def __getattr__(self, name): - return getattr(self.process, name) - - def __repr__(self): - return '{}(ptype={}, process={})'.format(self.__class__.__name__, - str(self.ptype), - repr(self.process)) - - -class PostgresNode(object): - def __init__(self, name=None, port=None, base_dir=None, conn_params: ConnectionParams = ConnectionParams()): - """ - PostgresNode constructor. - - Args: - name: node's application name. - port: port to accept connections. - base_dir: path to node's data directory. - """ - - # private - self._pg_version = PgVer(get_pg_version()) - self._should_free_port = port is None - self._base_dir = base_dir - self._logger = None - self._master = None - - # basic - self.name = name or generate_app_name() - if testgres_config.os_ops: - self.os_ops = testgres_config.os_ops - elif conn_params.ssh_key: - self.os_ops = RemoteOperations(conn_params) - else: - self.os_ops = LocalOperations(conn_params) - - self.port = port or reserve_port() - - self.host = self.os_ops.host - self.ssh_key = self.os_ops.ssh_key - - # defaults for __exit__() - self.cleanup_on_good_exit = testgres_config.node_cleanup_on_good_exit - self.cleanup_on_bad_exit = testgres_config.node_cleanup_on_bad_exit - self.shutdown_max_attempts = 3 - - # NOTE: for compatibility - self.utils_log_name = self.utils_log_file - self.pg_log_name = self.pg_log_file - - # Node state - self.is_started = False - - def __enter__(self): - return self - - def __exit__(self, type, value, traceback): - self.free_port() - - # NOTE: Ctrl+C does not count! - got_exception = type is not None and type != KeyboardInterrupt - - c1 = self.cleanup_on_good_exit and not got_exception - c2 = self.cleanup_on_bad_exit and got_exception - - attempts = self.shutdown_max_attempts - - if c1 or c2: - self.cleanup(attempts) - else: - self._try_shutdown(attempts) - - def __repr__(self): - return "{}(name='{}', port={}, base_dir='{}')".format( - self.__class__.__name__, self.name, self.port, self.base_dir) - - @property - def pid(self): - """ - Return postmaster's PID if node is running, else 0. - """ - - if self.status(): - pid_file = os.path.join(self.data_dir, PG_PID_FILE) - lines = self.os_ops.readlines(pid_file) - pid = int(lines[0]) if lines else None - return pid - - # for clarity - return 0 - - @property - def auxiliary_pids(self): - """ - Returns a dict of { ProcessType : PID }. - """ - - result = {} - - for process in self.auxiliary_processes: - if process.ptype not in result: - result[process.ptype] = [] - - result[process.ptype].append(process.pid) - - return result - - @property - def auxiliary_processes(self): - """ - Returns a list of auxiliary processes. - Each process is represented by :class:`.ProcessProxy` object. - """ - def is_aux(process): - return process.ptype != ProcessType.Unknown - - return list(filter(is_aux, self.child_processes)) - - @property - def child_processes(self): - """ - Returns a list of all child processes. - Each process is represented by :class:`.ProcessProxy` object. - """ - - # get a list of postmaster's children - children = self.os_ops.get_process_children(self.pid) - - return [ProcessProxy(p) for p in children] - - @property - def source_walsender(self): - """ - Returns master's walsender feeding this replica. - """ - - sql = """ - select pid - from pg_catalog.pg_stat_replication - where application_name = %s - """ - - if not self.master: - raise TestgresException("Node doesn't have a master") - - # master should be on the same host - assert self.master.host == self.host - - with self.master.connect() as con: - for row in con.execute(sql, self.name): - for child in self.master.auxiliary_processes: - if child.pid == int(row[0]): - return child - - msg = "Master doesn't send WAL to {}".format(self.name) - raise TestgresException(msg) - - @property - def master(self): - return self._master - - @property - def base_dir(self): - if not self._base_dir: - self._base_dir = self.os_ops.mkdtemp(prefix=TMP_NODE) - - # NOTE: it's safe to create a new dir - if not self.os_ops.path_exists(self._base_dir): - self.os_ops.makedirs(self._base_dir) - - return self._base_dir - - @property - def logs_dir(self): - path = os.path.join(self.base_dir, LOGS_DIR) - - # NOTE: it's safe to create a new dir - if not self.os_ops.path_exists(path): - self.os_ops.makedirs(path) - - return path - - @property - def data_dir(self): - # NOTE: we can't run initdb without user's args - return os.path.join(self.base_dir, DATA_DIR) - - @property - def utils_log_file(self): - return os.path.join(self.logs_dir, UTILS_LOG_FILE) - - @property - def pg_log_file(self): - return os.path.join(self.logs_dir, PG_LOG_FILE) - - @property - def version(self): - """ - Return PostgreSQL version for this node. - - Returns: - Instance of :class:`distutils.version.LooseVersion`. - """ - return self._pg_version - - def _try_shutdown(self, max_attempts): - attempts = 0 - - # try stopping server N times - while attempts < max_attempts: - try: - self.stop() - break # OK - except ExecUtilException: - pass # one more time - except Exception: - # TODO: probably should kill stray instance - eprint('cannot stop node {}'.format(self.name)) - break - - attempts += 1 - - def _assign_master(self, master): - """NOTE: this is a private method!""" - - # now this node has a master - self._master = master - - def _create_recovery_conf(self, username, slot=None): - """NOTE: this is a private method!""" - - # fetch master of this node - master = self.master - assert master is not None - - conninfo = { - "application_name": self.name, - "port": master.port, - "user": username - } # yapf: disable - - # host is tricky - try: - import ipaddress - ipaddress.ip_address(master.host) - conninfo["hostaddr"] = master.host - except ValueError: - conninfo["host"] = master.host - - line = ( - "primary_conninfo='{}'\n" - ).format(options_string(**conninfo)) # yapf: disable - # Since 12 recovery.conf had disappeared - if self.version >= PgVer('12'): - signal_name = os.path.join(self.data_dir, "standby.signal") - self.os_ops.touch(signal_name) - else: - line += "standby_mode=on\n" - - if slot: - # Connect to master for some additional actions - with master.connect(username=username) as con: - # check if slot already exists - res = con.execute( - """ - select exists ( - select from pg_catalog.pg_replication_slots - where slot_name = %s - ) - """, slot) - - if res[0][0]: - raise TestgresException( - "Slot '{}' already exists".format(slot)) - - # TODO: we should drop this slot after replica's cleanup() - con.execute( - """ - select pg_catalog.pg_create_physical_replication_slot(%s) - """, slot) - - line += "primary_slot_name={}\n".format(slot) - - if self.version >= PgVer('12'): - self.append_conf(line=line) - else: - self.append_conf(filename=RECOVERY_CONF_FILE, line=line) - - def _maybe_start_logger(self): - if testgres_config.use_python_logging: - # spawn new logger if it doesn't exist or is stopped - if not self._logger or not self._logger.is_alive(): - self._logger = TestgresLogger(self.name, self.pg_log_file) - self._logger.start() - - def _maybe_stop_logger(self): - if self._logger: - self._logger.stop() - - def _collect_special_files(self): - result = [] - - # list of important files + last N lines - files = [ - (os.path.join(self.data_dir, PG_CONF_FILE), 0), - (os.path.join(self.data_dir, PG_AUTO_CONF_FILE), 0), - (os.path.join(self.data_dir, RECOVERY_CONF_FILE), 0), - (os.path.join(self.data_dir, HBA_CONF_FILE), 0), - (self.pg_log_file, testgres_config.error_log_lines) - ] # yapf: disable - - for f, num_lines in files: - # skip missing files - if not self.os_ops.path_exists(f): - continue - - file_lines = self.os_ops.readlines(f, num_lines, binary=True, encoding=None) - lines = b''.join(file_lines) - - # fill list - result.append((f, lines)) - - return result - - def init(self, initdb_params=None, **kwargs): - """ - Perform initdb for this node. - - Args: - initdb_params: parameters for initdb (list). - fsync: should this node use fsync to keep data safe? - unix_sockets: should we enable UNIX sockets? - allow_streaming: should this node add a hba entry for replication? - - Returns: - This instance of :class:`.PostgresNode` - """ - - # initialize this PostgreSQL node - cached_initdb( - data_dir=self.data_dir, - logfile=self.utils_log_file, - os_ops=self.os_ops, - params=initdb_params) - - # initialize default config files - self.default_conf(**kwargs) - - return self - - def default_conf(self, - fsync=False, - unix_sockets=True, - allow_streaming=True, - allow_logical=False, - log_statement='all'): - """ - Apply default settings to this node. - - Args: - fsync: should this node use fsync to keep data safe? - unix_sockets: should we enable UNIX sockets? - allow_streaming: should this node add a hba entry for replication? - allow_logical: can this node be used as a logical replication publisher? - log_statement: one of ('all', 'off', 'mod', 'ddl'). - - Returns: - This instance of :class:`.PostgresNode`. - """ - - postgres_conf = os.path.join(self.data_dir, PG_CONF_FILE) - hba_conf = os.path.join(self.data_dir, HBA_CONF_FILE) - - # filter lines in hba file - # get rid of comments and blank lines - hba_conf_file = self.os_ops.readlines(hba_conf) - lines = [ - s for s in hba_conf_file - if len(s.strip()) > 0 and not s.startswith('#') - ] - - # write filtered lines - self.os_ops.write(hba_conf, lines, truncate=True) - - # replication-related settings - if allow_streaming: - # get auth method for host or local users - def get_auth_method(t): - return next((s.split()[-1] - for s in lines if s.startswith(t)), 'trust') - - # get auth methods - auth_local = get_auth_method('local') - auth_host = get_auth_method('host') - subnet_base = ".".join(self.os_ops.host.split('.')[:-1] + ['0']) - - new_lines = [ - u"local\treplication\tall\t\t\t{}\n".format(auth_local), - u"host\treplication\tall\t127.0.0.1/32\t{}\n".format(auth_host), - u"host\treplication\tall\t::1/128\t\t{}\n".format(auth_host), - u"host\treplication\tall\t{}/24\t\t{}\n".format(subnet_base, auth_host), - u"host\tall\tall\t{}/24\t\t{}\n".format(subnet_base, auth_host) - ] # yapf: disable - - # write missing lines - self.os_ops.write(hba_conf, new_lines) - - # overwrite config file - self.os_ops.write(postgres_conf, '', truncate=True) - - self.append_conf(fsync=fsync, - max_worker_processes=MAX_WORKER_PROCESSES, - log_statement=log_statement, - listen_addresses=self.host, - port=self.port) # yapf:disable - - # common replication settings - if allow_streaming or allow_logical: - self.append_conf(max_replication_slots=MAX_REPLICATION_SLOTS, - max_wal_senders=MAX_WAL_SENDERS) # yapf: disable - - # binary replication - if allow_streaming: - # select a proper wal_level for PostgreSQL - wal_level = 'replica' if self._pg_version >= PgVer('9.6') else 'hot_standby' - - if self._pg_version < PgVer('13'): - self.append_conf(hot_standby=True, - wal_keep_segments=WAL_KEEP_SEGMENTS, - wal_level=wal_level) # yapf: disable - else: - self.append_conf(hot_standby=True, - wal_keep_size=WAL_KEEP_SIZE, - wal_level=wal_level) # yapf: disable - - # logical replication - if allow_logical: - if self._pg_version < PgVer('10'): - raise InitNodeException("Logical replication is only " - "available on PostgreSQL 10 and newer") - - self.append_conf( - max_logical_replication_workers=MAX_LOGICAL_REPLICATION_WORKERS, - wal_level='logical') - - # disable UNIX sockets if asked to - if not unix_sockets: - self.append_conf(unix_socket_directories='') - - return self - - @method_decorator(positional_args_hack(['filename', 'line'])) - def append_conf(self, line='', filename=PG_CONF_FILE, **kwargs): - """ - Append line to a config file. - - Args: - line: string to be appended to config. - filename: config file (postgresql.conf by default). - **kwargs: named config options. - - Returns: - This instance of :class:`.PostgresNode`. - - Examples: - >>> append_conf(fsync=False) - >>> append_conf('log_connections = yes') - >>> append_conf(random_page_cost=1.5, fsync=True, ...) - >>> append_conf('postgresql.conf', 'synchronous_commit = off') - """ - - lines = [line] - - for option, value in iteritems(kwargs): - if isinstance(value, bool): - value = 'on' if value else 'off' - elif not str(value).replace('.', '', 1).isdigit(): - value = "'{}'".format(value) - if value == '*': - lines.append("{} = '*'".format(option)) - else: - # format a new config line - lines.append('{} = {}'.format(option, value)) - - config_name = os.path.join(self.data_dir, filename) - conf_text = '' - for line in lines: - conf_text += text_type(line) + '\n' - self.os_ops.write(config_name, conf_text) - - return self - - def status(self): - """ - Check this node's status. - - Returns: - An instance of :class:`.NodeStatus`. - """ - - try: - _params = [ - get_bin_path("pg_ctl"), - "-D", self.data_dir, - "status" - ] # yapf: disable - status_code, out, err = execute_utility(_params, self.utils_log_file, verbose=True) - if 'does not exist' in err: - return NodeStatus.Uninitialized - elif 'no server running' in out: - return NodeStatus.Stopped - return NodeStatus.Running - - except ExecUtilException as e: - # Node is not running - if e.exit_code == 3: - return NodeStatus.Stopped - - # Node has no file dir - elif e.exit_code == 4: - return NodeStatus.Uninitialized - - def get_control_data(self): - """ - Return contents of pg_control file. - """ - - # this one is tricky (blame PG 9.4) - _params = [get_bin_path("pg_controldata")] - _params += ["-D"] if self._pg_version >= PgVer('9.5') else [] - _params += [self.data_dir] - - data = execute_utility(_params, self.utils_log_file) - - out_dict = {} - - for line in data.splitlines(): - key, _, value = line.partition(':') - out_dict[key.strip()] = value.strip() - - return out_dict - - def slow_start(self, replica=False, dbname='template1', username=default_username(), max_attempts=0): - """ - Starts the PostgreSQL instance and then polls the instance - until it reaches the expected state (primary or replica). The state is checked - using the pg_is_in_recovery() function. - - Args: - dbname: - username: - replica: If True, waits for the instance to be in recovery (i.e., replica mode). - If False, waits for the instance to be in primary mode. Default is False. - max_attempts: - """ - self.start() - - if replica: - query = 'SELECT pg_is_in_recovery()' - else: - query = 'SELECT not pg_is_in_recovery()' - # Call poll_query_until until the expected value is returned - self.poll_query_until(query=query, - dbname=dbname, - username=username, - suppress={InternalError, - QueryException, - ProgrammingError, - OperationalError}, - max_attempts=max_attempts) - - def start(self, params=[], wait=True): - """ - Starts the PostgreSQL node using pg_ctl if node has not been started. - By default, it waits for the operation to complete before returning. - Optionally, it can return immediately without waiting for the start operation - to complete by setting the `wait` parameter to False. - - Args: - params: additional arguments for pg_ctl. - wait: wait until operation completes. - - Returns: - This instance of :class:`.PostgresNode`. - """ - if self.is_started: - return self - - _params = [ - get_bin_path("pg_ctl"), - "-D", self.data_dir, - "-l", self.pg_log_file, - "-w" if wait else '-W', # --wait or --no-wait - "start" - ] + params # yapf: disable - - try: - exit_status, out, error = execute_utility(_params, self.utils_log_file, verbose=True) - if 'does not exist' in error: - raise Exception - except Exception as e: - msg = 'Cannot start node' - files = self._collect_special_files() - raise_from(StartNodeException(msg, files), e) - self._maybe_start_logger() - self.is_started = True - return self - - def stop(self, params=[], wait=True): - """ - Stops the PostgreSQL node using pg_ctl if the node has been started. - - Args: - params: A list of additional arguments for pg_ctl. Defaults to None. - wait: If True, waits until the operation is complete. Defaults to True. - - Returns: - This instance of :class:`.PostgresNode`. - """ - if not self.is_started: - return self - - _params = [ - get_bin_path("pg_ctl"), - "-D", self.data_dir, - "-w" if wait else '-W', # --wait or --no-wait - "stop" - ] + params # yapf: disable - - execute_utility(_params, self.utils_log_file) - - self._maybe_stop_logger() - self.is_started = False - return self - - def kill(self, someone=None): - """ - Kills the PostgreSQL node or a specified auxiliary process if the node is running. - - Args: - someone: A key to the auxiliary process in the auxiliary_pids dictionary. - If None, the main PostgreSQL node process will be killed. Defaults to None. - """ - if self.is_started: - sig = signal.SIGKILL if os.name != 'nt' else signal.SIGBREAK - if someone is None: - os.kill(self.pid, sig) - else: - os.kill(self.auxiliary_pids[someone][0], sig) - self.is_started = False - - def restart(self, params=[]): - """ - Restart this node using pg_ctl. - - Args: - params: additional arguments for pg_ctl. - - Returns: - This instance of :class:`.PostgresNode`. - """ - - _params = [ - get_bin_path("pg_ctl"), - "-D", self.data_dir, - "-l", self.pg_log_file, - "-w", # wait - "restart" - ] + params # yapf: disable - - try: - error_code, out, error = execute_utility(_params, self.utils_log_file, verbose=True) - if 'could not start server' in error: - raise ExecUtilException - except ExecUtilException as e: - msg = 'Cannot restart node' - files = self._collect_special_files() - raise_from(StartNodeException(msg, files), e) - - self._maybe_start_logger() - - return self - - def reload(self, params=[]): - """ - Asynchronously reload config files using pg_ctl. - - Args: - params: additional arguments for pg_ctl. - - Returns: - This instance of :class:`.PostgresNode`. - """ - - _params = [ - get_bin_path("pg_ctl"), - "-D", self.data_dir, - "reload" - ] + params # yapf: disable - - execute_utility(_params, self.utils_log_file) - - return self - - def promote(self, dbname=None, username=None): - """ - Promote standby instance to master using pg_ctl. For PostgreSQL versions - below 10 some additional actions required to ensure that instance - became writable and hence `dbname` and `username` parameters may be - needed. - - Returns: - This instance of :class:`.PostgresNode`. - """ - - _params = [ - get_bin_path("pg_ctl"), - "-D", self.data_dir, - "-w", # wait - "promote" - ] # yapf: disable - - execute_utility(_params, self.utils_log_file) - - # for versions below 10 `promote` is asynchronous so we need to wait - # until it actually becomes writable - if self._pg_version < PgVer('10'): - check_query = "SELECT pg_is_in_recovery()" - - self.poll_query_until(query=check_query, - expected=False, - dbname=dbname, - username=username, - max_attempts=0) # infinite - - # node becomes master itself - self._master = None - - return self - - def pg_ctl(self, params): - """ - Invoke pg_ctl with params. - - Args: - params: arguments for pg_ctl. - - Returns: - Stdout + stderr of pg_ctl. - """ - - _params = [ - get_bin_path("pg_ctl"), - "-D", self.data_dir, - "-w" # wait - ] + params # yapf: disable - - return execute_utility(_params, self.utils_log_file) - - def free_port(self): - """ - Reclaim port owned by this node. - NOTE: does not free auto selected ports. - """ - - if self._should_free_port: - self._should_free_port = False - release_port(self.port) - - def cleanup(self, max_attempts=3): - """ - Stop node if needed and remove its data/logs directory. - NOTE: take a look at TestgresConfig.node_cleanup_full. - - Args: - max_attempts: how many times should we try to stop()? - - Returns: - This instance of :class:`.PostgresNode`. - """ - - self._try_shutdown(max_attempts) - - # choose directory to be removed - if testgres_config.node_cleanup_full: - rm_dir = self.base_dir # everything - else: - rm_dir = self.data_dir # just data, save logs - - self.os_ops.rmdirs(rm_dir, ignore_errors=True) - - return self - - @method_decorator(positional_args_hack(['dbname', 'query'])) - def psql(self, - query=None, - filename=None, - dbname=None, - username=None, - input=None, - **variables): - """ - Execute a query using psql. - - Args: - query: query to be executed. - filename: file with a query. - dbname: database name to connect to. - username: database user name. - input: raw input to be passed. - **variables: vars to be set before execution. - - Returns: - A tuple of (code, stdout, stderr). - - Examples: - >>> psql('select 1') - >>> psql('postgres', 'select 2') - >>> psql(query='select 3', ON_ERROR_STOP=1) - """ - - # Set default arguments - dbname = dbname or default_dbname() - username = username or default_username() - - psql_params = [ - get_bin_path("psql"), - "-p", str(self.port), - "-h", self.host, - "-U", username, - "-X", # no .psqlrc - "-A", # unaligned output - "-t", # print rows only - "-q" # run quietly - ] # yapf: disable - - # set variables before execution - for key, value in iteritems(variables): - psql_params.extend(["--set", '{}={}'.format(key, value)]) - - # select query source - if query: - if self.os_ops.remote: - psql_params.extend(("-c", '"{}"'.format(query))) - else: - psql_params.extend(("-c", query)) - elif filename: - psql_params.extend(("-f", filename)) - else: - raise QueryException('Query or filename must be provided') - - # should be the last one - psql_params.append(dbname) - if not self.os_ops.remote: - # start psql process - process = subprocess.Popen(psql_params, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - - # wait until it finishes and get stdout and stderr - out, err = process.communicate(input=input) - return process.returncode, out, err - else: - status_code, out, err = self.os_ops.exec_command(psql_params, verbose=True, input=input) - - return status_code, out, err - - @method_decorator(positional_args_hack(['dbname', 'query'])) - def safe_psql(self, query=None, expect_error=False, **kwargs): - """ - Execute a query using psql. - - Args: - query: query to be executed. - filename: file with a query. - dbname: database name to connect to. - username: database user name. - input: raw input to be passed. - expect_error: if True - fail if we didn't get ret - if False - fail if we got ret - - **kwargs are passed to psql(). - - Returns: - psql's output as str. - """ - - # force this setting - kwargs['ON_ERROR_STOP'] = 1 - try: - ret, out, err = self.psql(query=query, **kwargs) - except ExecUtilException as e: - ret = e.exit_code - out = e.out - err = e.message - if ret: - if expect_error: - out = (err or b'').decode('utf-8') - else: - raise QueryException((err or b'').decode('utf-8'), query) - elif expect_error: - assert False, "Exception was expected, but query finished successfully: `{}` ".format(query) - - return out - - def dump(self, - filename=None, - dbname=None, - username=None, - format=DumpFormat.Plain): - """ - Dump database into a file using pg_dump. - NOTE: the file is not removed automatically. - - Args: - filename: database dump taken by pg_dump. - dbname: database name to connect to. - username: database user name. - format: format argument plain/custom/directory/tar. - - Returns: - Path to a file containing dump. - """ - - # Check arguments - if not isinstance(format, DumpFormat): - try: - format = DumpFormat(format) - except ValueError: - msg = 'Invalid format "{}"'.format(format) - raise BackupException(msg) - - # Generate tmpfile or tmpdir - def tmpfile(): - if format == DumpFormat.Directory: - fname = self.os_ops.mkdtemp(prefix=TMP_DUMP) - else: - fname = self.os_ops.mkstemp(prefix=TMP_DUMP) - return fname - - # Set default arguments - dbname = dbname or default_dbname() - username = username or default_username() - filename = filename or tmpfile() - - _params = [ - get_bin_path("pg_dump"), - "-p", str(self.port), - "-h", self.host, - "-f", filename, - "-U", username, - "-d", dbname, - "-F", format.value - ] # yapf: disable - - execute_utility(_params, self.utils_log_file) - - return filename - - def restore(self, filename, dbname=None, username=None): - """ - Restore database from pg_dump's file. - - Args: - filename: database dump taken by pg_dump in custom/directory/tar formats. - dbname: database name to connect to. - username: database user name. - """ - - # Set default arguments - dbname = dbname or default_dbname() - username = username or default_username() - - _params = [ - get_bin_path("pg_restore"), - "-p", str(self.port), - "-h", self.host, - "-U", username, - "-d", dbname, - filename - ] # yapf: disable - - # try pg_restore if dump is binary formate, and psql if not - try: - execute_utility(_params, self.utils_log_name) - except ExecUtilException: - self.psql(filename=filename, dbname=dbname, username=username) - - @method_decorator(positional_args_hack(['dbname', 'query'])) - def poll_query_until(self, - query, - dbname=None, - username=None, - max_attempts=0, - sleep_time=1, - expected=True, - commit=True, - suppress=None): - """ - Run a query once per second until it returns 'expected'. - Query should return a single value (1 row, 1 column). - - Args: - query: query to be executed. - dbname: database name to connect to. - username: database user name. - max_attempts: how many times should we try? 0 == infinite - sleep_time: how much should we sleep after a failure? - expected: what should be returned to break the cycle? - commit: should (possible) changes be committed? - suppress: a collection of exceptions to be suppressed. - - Examples: - >>> poll_query_until('select true') - >>> poll_query_until('postgres', "select now() > '01.01.2018'") - >>> poll_query_until('select false', expected=True, max_attempts=4) - >>> poll_query_until('select 1', suppress={testgres.OperationalError}) - """ - - # sanity checks - assert max_attempts >= 0 - assert sleep_time > 0 - attempts = 0 - while max_attempts == 0 or attempts < max_attempts: - print(f"Pooling {attempts}") - try: - res = self.execute(dbname=dbname, - query=query, - username=username, - commit=commit) - - if expected is None and res is None: - return # done - - if res is None: - raise QueryException('Query returned None', query) - - # result set is not empty - if len(res): - if len(res[0]) == 0: - raise QueryException('Query returned 0 columns', query) - if res[0][0] == expected: - return # done - # empty result set is considered as None - elif expected is None: - return # done - - except tuple(suppress or []): - pass # we're suppressing them - - time.sleep(sleep_time) - attempts += 1 - - raise TimeoutException('Query timeout') - - @method_decorator(positional_args_hack(['dbname', 'query'])) - def execute(self, - query, - dbname=None, - username=None, - password=None, - commit=True): - """ - Execute a query and return all rows as list. - - Args: - query: query to be executed. - dbname: database name to connect to. - username: database user name. - password: user's password. - commit: should we commit this query? - - Returns: - A list of tuples representing rows. - """ - - with self.connect(dbname=dbname, - username=username, - password=password, - autocommit=commit) as node_con: # yapf: disable - - res = node_con.execute(query) - - return res - - def backup(self, **kwargs): - """ - Perform pg_basebackup. - - Args: - username: database user name. - xlog_method: a method for collecting the logs ('fetch' | 'stream'). - base_dir: the base directory for data files and logs - - Returns: - A smart object of type NodeBackup. - """ - - return NodeBackup(node=self, **kwargs) - - def replicate(self, name=None, slot=None, **kwargs): - """ - Create a binary replica of this node. - - Args: - name: replica's application name. - slot: create a replication slot with the specified name. - username: database user name. - xlog_method: a method for collecting the logs ('fetch' | 'stream'). - base_dir: the base directory for data files and logs - """ - - # transform backup into a replica - with clean_on_error(self.backup(**kwargs)) as backup: - return backup.spawn_replica(name=name, destroy=True, slot=slot) - - def set_synchronous_standbys(self, standbys): - """ - Set standby synchronization options. This corresponds to - `synchronous_standby_names `_ - option. Note that :meth:`~.PostgresNode.reload` or - :meth:`~.PostgresNode.restart` is needed for changes to take place. - - Args: - standbys: either :class:`.First` or :class:`.Any` object specifying - sychronization parameters or just a plain list of - :class:`.PostgresNode`s replicas which would be equivalent - to passing ``First(1, )``. For PostgreSQL 9.5 and below - it is only possible to specify a plain list of standbys as - `FIRST` and `ANY` keywords aren't supported. - - Example:: - - from testgres import get_new_node, First - - master = get_new_node().init().start() - with master.replicate().start() as standby: - master.append_conf("synchronous_commit = remote_apply") - master.set_synchronous_standbys(First(1, [standby])) - master.restart() - - """ - if self._pg_version >= PgVer('9.6'): - if isinstance(standbys, Iterable): - standbys = First(1, standbys) - else: - if isinstance(standbys, Iterable): - standbys = u", ".join(u"\"{}\"".format(r.name) - for r in standbys) - else: - raise TestgresException("Feature isn't supported in " - "Postgres 9.5 and below") - - self.append_conf("synchronous_standby_names = '{}'".format(standbys)) - - def catchup(self, dbname=None, username=None): - """ - Wait until async replica catches up with its master. - """ - - if not self.master: - raise TestgresException("Node doesn't have a master") - - if self._pg_version >= PgVer('10'): - poll_lsn = "select pg_catalog.pg_current_wal_lsn()::text" - wait_lsn = "select pg_catalog.pg_last_wal_replay_lsn() >= '{}'::pg_lsn" - else: - poll_lsn = "select pg_catalog.pg_current_xlog_location()::text" - wait_lsn = "select pg_catalog.pg_last_xlog_replay_location() >= '{}'::pg_lsn" - - try: - # fetch latest LSN - lsn = self.master.execute(query=poll_lsn, - dbname=dbname, - username=username)[0][0] # yapf: disable - - # wait until this LSN reaches replica - self.poll_query_until(query=wait_lsn.format(lsn), - dbname=dbname, - username=username, - max_attempts=0) # infinite - except Exception as e: - raise_from(CatchUpException("Failed to catch up", poll_lsn), e) - - def publish(self, name, **kwargs): - """ - Create publication for logical replication - - Args: - pubname: publication name - tables: tables names list - dbname: database name where objects or interest are located - username: replication username - """ - return Publication(name=name, node=self, **kwargs) - - def subscribe(self, - publication, - name, - dbname=None, - username=None, - **params): - """ - Create subscription for logical replication - - Args: - name: subscription name - publication: publication object obtained from publish() - dbname: database name - username: replication username - params: subscription parameters (see documentation on `CREATE SUBSCRIPTION - `_ - for details) - """ - # yapf: disable - return Subscription(name=name, node=self, publication=publication, - dbname=dbname, username=username, **params) - # yapf: enable - - def pgbench(self, - dbname=None, - username=None, - stdout=None, - stderr=None, - options=[]): - """ - Spawn a pgbench process. - - Args: - dbname: database name to connect to. - username: database user name. - stdout: stdout file to be used by Popen. - stderr: stderr file to be used by Popen. - options: additional options for pgbench (list). - - Returns: - Process created by subprocess.Popen. - """ - - # Set default arguments - dbname = dbname or default_dbname() - username = username or default_username() - - _params = [ - get_bin_path("pgbench"), - "-p", str(self.port), - "-h", self.host, - "-U", username, - ] + options # yapf: disable - - # should be the last one - _params.append(dbname) - - proc = self.os_ops.exec_command(_params, stdout=stdout, stderr=stderr, wait_exit=True, get_process=True) - - return proc - - def pgbench_init(self, **kwargs): - """ - Small wrapper for pgbench_run(). - Sets initialize=True. - - Returns: - This instance of :class:`.PostgresNode`. - """ - - self.pgbench_run(initialize=True, **kwargs) - - return self - - def pgbench_run(self, dbname=None, username=None, options=[], **kwargs): - """ - Run pgbench with some options. - This event is logged (see self.utils_log_file). - - Args: - dbname: database name to connect to. - username: database user name. - options: additional options for pgbench (list). - - **kwargs: named options for pgbench. - Run pgbench --help to learn more. - - Returns: - Stdout produced by pgbench. - - Examples: - >>> pgbench_run(initialize=True, scale=2) - >>> pgbench_run(time=10) - """ - - # Set default arguments - dbname = dbname or default_dbname() - username = username or default_username() - - _params = [ - get_bin_path("pgbench"), - "-p", str(self.port), - "-h", self.host, - "-U", username, - ] + options # yapf: disable - - for key, value in iteritems(kwargs): - # rename keys for pgbench - key = key.replace('_', '-') - - # append option - if not isinstance(value, bool): - _params.append('--{}={}'.format(key, value)) - else: - assert value is True # just in case - _params.append('--{}'.format(key)) - - # should be the last one - _params.append(dbname) - - return execute_utility(_params, self.utils_log_file) - - def connect(self, - dbname=None, - username=None, - password=None, - autocommit=False): - """ - Connect to a database. - - Args: - dbname: database name to connect to. - username: database user name. - password: user's password. - autocommit: commit each statement automatically. Also it should be - set to `True` for statements requiring to be run outside - a transaction? such as `VACUUM` or `CREATE DATABASE`. - - Returns: - An instance of :class:`.NodeConnection`. - """ - - return NodeConnection(node=self, - dbname=dbname, - username=username, - password=password, - autocommit=autocommit) # yapf: disable - - def table_checksum(self, table, dbname="postgres"): - con = self.connect(dbname=dbname) - - curname = "cur_" + str(random.randint(0, 2 ** 48)) - - con.execute(""" - DECLARE %s NO SCROLL CURSOR FOR - SELECT t::text FROM %s as t - """ % (curname, table)) - - que = Queue(maxsize=50) - sum = 0 - - rows = con.execute("FETCH FORWARD 2000 FROM %s" % curname) - if not rows: - return 0 - que.put(rows) - - th = None - if len(rows) == 2000: - def querier(): - try: - while True: - rows = con.execute("FETCH FORWARD 2000 FROM %s" % curname) - if not rows: - break - que.put(rows) - except Exception as e: - que.put(e) - else: - que.put(None) - - th = threading.Thread(target=querier) - th.start() - else: - que.put(None) - - while True: - rows = que.get() - if rows is None: - break - if isinstance(rows, Exception): - raise rows - # hash uses SipHash since Python3.4, therefore it is good enough - for row in rows: - sum += hash(row[0]) - - if th is not None: - th.join() - - con.execute("CLOSE %s; ROLLBACK;" % curname) - - con.close() - return sum - - def pgbench_table_checksums(self, dbname="postgres", - pgbench_tables=('pgbench_branches', - 'pgbench_tellers', - 'pgbench_accounts', - 'pgbench_history') - ): - return {(table, self.table_checksum(table, dbname)) - for table in pgbench_tables} - - def set_auto_conf(self, options, config='postgresql.auto.conf', rm_options={}): - """ - Update or remove configuration options in the specified configuration file, - updates the options specified in the options dictionary, removes any options - specified in the rm_options set, and writes the updated configuration back to - the file. - - Args: - options (dict): A dictionary containing the options to update or add, - with the option names as keys and their values as values. - config (str, optional): The name of the configuration file to update. - Defaults to 'postgresql.auto.conf'. - rm_options (set, optional): A set containing the names of the options to remove. - Defaults to an empty set. - """ - # parse postgresql.auto.conf - path = os.path.join(self.data_dir, config) - - lines = self.os_ops.readlines(path) - current_options = {} - current_directives = [] - for line in lines: - - # ignore comments - if line.startswith('#'): - continue - - if line.strip() == '': - continue - - if line.startswith('include'): - current_directives.append(line) - continue - - name, var = line.partition('=')[::2] - name = name.strip() - var = var.strip() - var = var.strip('"') - var = var.strip("'") - - # remove options specified in rm_options list - if name in rm_options: - continue - - current_options[name] = var - - for option in options: - current_options[option] = options[option] - - auto_conf = '' - for option in current_options: - auto_conf += "{0} = '{1}'\n".format( - option, current_options[option]) - - for directive in current_directives: - auto_conf += directive + "\n" - - self.os_ops.write(path, auto_conf, truncate=True) - - -class NodeApp: - - def __init__(self, test_path, nodes_to_cleanup, os_ops=LocalOperations()): - self.test_path = test_path - self.nodes_to_cleanup = nodes_to_cleanup - self.os_ops = os_ops - - def make_empty( - self, - base_dir=None): - real_base_dir = os.path.join(self.test_path, base_dir) - self.os_ops.rmdirs(real_base_dir, ignore_errors=True) - self.os_ops.makedirs(real_base_dir) - - node = PostgresNode(base_dir=real_base_dir) - node.should_rm_dirs = True - self.nodes_to_cleanup.append(node) - - return node - - def make_simple( - self, - base_dir=None, - set_replication=False, - ptrack_enable=False, - initdb_params=[], - pg_options={}, - checksum=True): - if checksum and '--data-checksums' not in initdb_params: - initdb_params.append('--data-checksums') - node = self.make_empty(base_dir) - node.init( - initdb_params=initdb_params, allow_streaming=set_replication) - - # set major version - pg_version_file = self.os_ops.read(os.path.join(node.data_dir, 'PG_VERSION')) - node.major_version_str = str(pg_version_file.rstrip()) - node.major_version = float(node.major_version_str) - - # Set default parameters - options = {'max_connections': 100, - 'shared_buffers': '10MB', - 'fsync': 'off', - 'wal_level': 'logical', - 'hot_standby': 'off', - 'log_line_prefix': '%t [%p]: [%l-1] ', - 'log_statement': 'none', - 'log_duration': 'on', - 'log_min_duration_statement': 0, - 'log_connections': 'on', - 'log_disconnections': 'on', - 'restart_after_crash': 'off', - 'autovacuum': 'off'} - - # Allow replication in pg_hba.conf - if set_replication: - options['max_wal_senders'] = 10 - - if ptrack_enable: - options['ptrack.map_size'] = '1' - options['shared_preload_libraries'] = 'ptrack' - - if node.major_version >= 13: - options['wal_keep_size'] = '200MB' - else: - options['wal_keep_segments'] = '12' - - # set default values - node.set_auto_conf(options) - - # Apply given parameters - node.set_auto_conf(pg_options) - - # kludge for testgres - # https://github.com/postgrespro/testgres/issues/54 - # for PG >= 13 remove 'wal_keep_segments' parameter - if node.major_version >= 13: - node.set_auto_conf({}, 'postgresql.conf', ['wal_keep_segments']) - - return node diff --git a/testgres/operations/local_ops.py b/testgres/operations/local_ops.py deleted file mode 100644 index a692750e..00000000 --- a/testgres/operations/local_ops.py +++ /dev/null @@ -1,280 +0,0 @@ -import getpass -import os -import shutil -import stat -import subprocess -import tempfile - -import psutil - -from ..exceptions import ExecUtilException -from .os_ops import ConnectionParams, OsOperations -from .os_ops import pglib - -try: - from shutil import which as find_executable - from shutil import rmtree -except ImportError: - from distutils.spawn import find_executable - from distutils import rmtree - -CMD_TIMEOUT_SEC = 60 -error_markers = [b'error', b'Permission denied', b'fatal'] - - -class LocalOperations(OsOperations): - def __init__(self, conn_params=None): - if conn_params is None: - conn_params = ConnectionParams() - super(LocalOperations, self).__init__(conn_params.username) - self.conn_params = conn_params - self.host = conn_params.host - self.ssh_key = None - self.remote = False - self.username = conn_params.username or self.get_user() - - # Command execution - def exec_command(self, cmd, wait_exit=False, verbose=False, - expect_error=False, encoding=None, shell=False, text=False, - input=None, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - get_process=None, timeout=None): - """ - Execute a command in a subprocess. - - Args: - - cmd: The command to execute. - - wait_exit: Whether to wait for the subprocess to exit before returning. - - verbose: Whether to return verbose output. - - expect_error: Whether to raise an error if the subprocess exits with an error status. - - encoding: The encoding to use for decoding the subprocess output. - - shell: Whether to use shell when executing the subprocess. - - text: Whether to return str instead of bytes for the subprocess output. - - input: The input to pass to the subprocess. - - stdout: The stdout to use for the subprocess. - - stderr: The stderr to use for the subprocess. - - proc: The process to use for subprocess creation. - :return: The output of the subprocess. - """ - if os.name == 'nt': - with tempfile.NamedTemporaryFile() as buf: - process = subprocess.Popen(cmd, stdout=buf, stderr=subprocess.STDOUT) - process.communicate() - buf.seek(0) - result = buf.read().decode(encoding) - return result - else: - process = subprocess.Popen( - cmd, - shell=shell, - stdout=stdout, - stderr=stderr, - ) - if get_process: - return process - - try: - result, error = process.communicate(input, timeout=timeout) - except subprocess.TimeoutExpired: - process.kill() - raise ExecUtilException("Command timed out after {} seconds.".format(timeout)) - exit_status = process.returncode - - error_found = exit_status != 0 or any(marker in error for marker in error_markers) - - if encoding: - result = result.decode(encoding) - error = error.decode(encoding) - - if expect_error: - raise Exception(result, error) - - if exit_status != 0 or error_found: - if exit_status == 0: - exit_status = 1 - raise ExecUtilException(message='Utility exited with non-zero code. Error `{}`'.format(error), - command=cmd, - exit_code=exit_status, - out=result) - if verbose: - return exit_status, result, error - else: - return result - - # Environment setup - def environ(self, var_name): - return os.environ.get(var_name) - - def find_executable(self, executable): - return find_executable(executable) - - def is_executable(self, file): - # Check if the file is executable - return os.stat(file).st_mode & stat.S_IXUSR - - def set_env(self, var_name, var_val): - # Check if the directory is already in PATH - os.environ[var_name] = var_val - - # Get environment variables - def get_user(self): - return getpass.getuser() - - def get_name(self): - return os.name - - # Work with dirs - def makedirs(self, path, remove_existing=False): - if remove_existing: - shutil.rmtree(path, ignore_errors=True) - try: - os.makedirs(path) - except FileExistsError: - pass - - def rmdirs(self, path, ignore_errors=True): - return rmtree(path, ignore_errors=ignore_errors) - - def listdir(self, path): - return os.listdir(path) - - def path_exists(self, path): - return os.path.exists(path) - - @property - def pathsep(self): - os_name = self.get_name() - if os_name == "posix": - pathsep = ":" - elif os_name == "nt": - pathsep = ";" - else: - raise Exception("Unsupported operating system: {}".format(os_name)) - return pathsep - - def mkdtemp(self, prefix=None): - return tempfile.mkdtemp(prefix='{}'.format(prefix)) - - def mkstemp(self, prefix=None): - fd, filename = tempfile.mkstemp(prefix=prefix) - os.close(fd) # Close the file descriptor immediately after creating the file - return filename - - def copytree(self, src, dst): - return shutil.copytree(src, dst) - - # Work with files - def write(self, filename, data, truncate=False, binary=False, read_and_write=False): - """ - Write data to a file locally - Args: - filename: The file path where the data will be written. - data: The data to be written to the file. - truncate: If True, the file will be truncated before writing ('w' or 'wb' option); - if False (default), data will be appended ('a' or 'ab' option). - binary: If True, the data will be written in binary mode ('wb' or 'ab' option); - if False (default), the data will be written in text mode ('w' or 'a' option). - read_and_write: If True, the file will be opened with read and write permissions ('r+' option); - if False (default), only write permission will be used ('w', 'a', 'wb', or 'ab' option) - """ - # If it is a bytes str or list - if isinstance(data, bytes) or isinstance(data, list) and all(isinstance(item, bytes) for item in data): - binary = True - mode = "wb" if binary else "w" - if not truncate: - mode = "ab" if binary else "a" - if read_and_write: - mode = "r+b" if binary else "r+" - - with open(filename, mode) as file: - if isinstance(data, list): - file.writelines(data) - else: - file.write(data) - - def touch(self, filename): - """ - Create a new file or update the access and modification times of an existing file. - Args: - filename (str): The name of the file to touch. - - This method behaves as the 'touch' command in Unix. It's equivalent to calling 'touch filename' in the shell. - """ - # cross-python touch(). It is vulnerable to races, but who cares? - with open(filename, "a"): - os.utime(filename, None) - - def read(self, filename, encoding=None, binary=False): - mode = "rb" if binary else "r" - with open(filename, mode) as file: - content = file.read() - if binary: - return content - if isinstance(content, bytes): - return content.decode(encoding or 'utf-8') - return content - - def readlines(self, filename, num_lines=0, binary=False, encoding=None): - """ - Read lines from a local file. - If num_lines is greater than 0, only the last num_lines lines will be read. - """ - assert num_lines >= 0 - mode = 'rb' if binary else 'r' - if num_lines == 0: - with open(filename, mode, encoding=encoding) as file: # open in binary mode - return file.readlines() - - else: - bufsize = 8192 - buffers = 1 - - with open(filename, mode, encoding=encoding) as file: # open in binary mode - file.seek(0, os.SEEK_END) - end_pos = file.tell() - - while True: - offset = max(0, end_pos - bufsize * buffers) - file.seek(offset, os.SEEK_SET) - pos = file.tell() - lines = file.readlines() - cur_lines = len(lines) - - if cur_lines >= num_lines or pos == 0: - return lines[-num_lines:] # get last num_lines from lines - - buffers = int( - buffers * max(2, int(num_lines / max(cur_lines, 1))) - ) # Adjust buffer size - - def isfile(self, remote_file): - return os.path.isfile(remote_file) - - def isdir(self, dirname): - return os.path.isdir(dirname) - - def remove_file(self, filename): - return os.remove(filename) - - # Processes control - def kill(self, pid, signal): - # Kill the process - cmd = "kill -{} {}".format(signal, pid) - return self.exec_command(cmd) - - def get_pid(self): - # Get current process id - return os.getpid() - - def get_process_children(self, pid): - return psutil.Process(pid).children() - - # Database control - def db_connect(self, dbname, user, password=None, host="localhost", port=5432): - conn = pglib.connect( - host=host, - port=port, - database=dbname, - user=user, - password=password, - ) - return conn diff --git a/testgres/operations/os_ops.py b/testgres/operations/os_ops.py deleted file mode 100644 index 9261cacf..00000000 --- a/testgres/operations/os_ops.py +++ /dev/null @@ -1,101 +0,0 @@ -try: - import psycopg2 as pglib # noqa: F401 -except ImportError: - try: - import pg8000 as pglib # noqa: F401 - except ImportError: - raise ImportError("You must have psycopg2 or pg8000 modules installed") - - -class ConnectionParams: - def __init__(self, host='127.0.0.1', ssh_key=None, username=None): - self.host = host - self.ssh_key = ssh_key - self.username = username - - -class OsOperations: - def __init__(self, username=None): - self.ssh_key = None - self.username = username - - # Command execution - def exec_command(self, cmd, **kwargs): - raise NotImplementedError() - - # Environment setup - def environ(self, var_name): - raise NotImplementedError() - - def find_executable(self, executable): - raise NotImplementedError() - - def is_executable(self, file): - # Check if the file is executable - raise NotImplementedError() - - def set_env(self, var_name, var_val): - # Check if the directory is already in PATH - raise NotImplementedError() - - # Get environment variables - def get_user(self): - raise NotImplementedError() - - def get_name(self): - raise NotImplementedError() - - # Work with dirs - def makedirs(self, path, remove_existing=False): - raise NotImplementedError() - - def rmdirs(self, path, ignore_errors=True): - raise NotImplementedError() - - def listdir(self, path): - raise NotImplementedError() - - def path_exists(self, path): - raise NotImplementedError() - - @property - def pathsep(self): - raise NotImplementedError() - - def mkdtemp(self, prefix=None): - raise NotImplementedError() - - def copytree(self, src, dst): - raise NotImplementedError() - - # Work with files - def write(self, filename, data, truncate=False, binary=False, read_and_write=False): - raise NotImplementedError() - - def touch(self, filename): - raise NotImplementedError() - - def read(self, filename): - raise NotImplementedError() - - def readlines(self, filename): - raise NotImplementedError() - - def isfile(self, remote_file): - raise NotImplementedError() - - # Processes control - def kill(self, pid, signal): - # Kill the process - raise NotImplementedError() - - def get_pid(self): - # Get current process id - raise NotImplementedError() - - def get_process_children(self, pid): - raise NotImplementedError() - - # Database control - def db_connect(self, dbname, user, password=None, host="localhost", port=5432): - raise NotImplementedError() diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py deleted file mode 100644 index 421c0a6d..00000000 --- a/testgres/operations/remote_ops.py +++ /dev/null @@ -1,407 +0,0 @@ -import locale -import logging -import os -import subprocess -import tempfile - -# we support both pg8000 and psycopg2 -try: - import psycopg2 as pglib -except ImportError: - try: - import pg8000 as pglib - except ImportError: - raise ImportError("You must have psycopg2 or pg8000 modules installed") - -from ..exceptions import ExecUtilException - -from .os_ops import OsOperations, ConnectionParams - -ConsoleEncoding = locale.getdefaultlocale()[1] -if not ConsoleEncoding: - ConsoleEncoding = 'UTF-8' - -error_markers = [b'error', b'Permission denied', b'fatal', b'No such file or directory'] - - -class PsUtilProcessProxy: - def __init__(self, ssh, pid): - self.ssh = ssh - self.pid = pid - - def kill(self): - command = "kill {}".format(self.pid) - self.ssh.exec_command(command) - - def cmdline(self): - command = "ps -p {} -o cmd --no-headers".format(self.pid) - stdin, stdout, stderr = self.ssh.exec_command(command, verbose=True, encoding=ConsoleEncoding) - cmdline = stdout.strip() - return cmdline.split() - - -class RemoteOperations(OsOperations): - def __init__(self, conn_params: ConnectionParams): - if os.name != "posix": - raise EnvironmentError("Remote operations are supported only on Linux!") - - super().__init__(conn_params.username) - self.conn_params = conn_params - self.host = conn_params.host - self.ssh_key = conn_params.ssh_key - self.remote = True - self.username = conn_params.username or self.get_user() - self.add_known_host(self.host) - self.tunnel_process = None - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - self.close_ssh_tunnel() - - def establish_ssh_tunnel(self, local_port, remote_port): - """ - Establish an SSH tunnel from a local port to a remote PostgreSQL port. - """ - ssh_cmd = ['-N', '-L', f"{local_port}:localhost:{remote_port}"] - self.tunnel_process = self.exec_command(ssh_cmd, get_process=True, timeout=300) - - def close_ssh_tunnel(self): - if hasattr(self, 'tunnel_process'): - self.tunnel_process.terminate() - self.tunnel_process.wait() - del self.tunnel_process - else: - print("No active tunnel to close.") - - def add_known_host(self, host): - cmd = 'ssh-keyscan -H %s >> /home/%s/.ssh/known_hosts' % (host, os.getlogin()) - try: - subprocess.check_call( - cmd, - shell=True, - ) - logging.info("Successfully added %s to known_hosts." % host) - except subprocess.CalledProcessError as e: - raise ExecUtilException(message="Failed to add %s to known_hosts. Error: %s" % (host, str(e)), command=cmd, - exit_code=e.returncode, out=e.stderr) - - def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False, - encoding=None, shell=True, text=False, input=None, stdin=None, stdout=None, - stderr=None, get_process=None, timeout=None): - """ - Execute a command in the SSH session. - Args: - - cmd (str): The command to be executed. - """ - ssh_cmd = [] - if isinstance(cmd, str): - ssh_cmd = ['ssh', f"{self.username}@{self.host}", '-i', self.ssh_key, cmd] - elif isinstance(cmd, list): - ssh_cmd = ['ssh', f"{self.username}@{self.host}", '-i', self.ssh_key] + cmd - process = subprocess.Popen(ssh_cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - if get_process: - return process - - try: - result, error = process.communicate(input, timeout=timeout) - except subprocess.TimeoutExpired: - process.kill() - raise ExecUtilException("Command timed out after {} seconds.".format(timeout)) - - exit_status = process.returncode - - if encoding: - result = result.decode(encoding) - error = error.decode(encoding) - - if expect_error: - raise Exception(result, error) - - if not error: - error_found = 0 - else: - error_found = exit_status != 0 or any( - marker in error for marker in [b'error', b'Permission denied', b'fatal', b'No such file or directory']) - - if error_found: - if isinstance(error, bytes): - message = b"Utility exited with non-zero code. Error: " + error - else: - message = f"Utility exited with non-zero code. Error: {error}" - raise ExecUtilException(message=message, command=cmd, exit_code=exit_status, out=result) - - if verbose: - return exit_status, result, error - else: - return result - - # Environment setup - def environ(self, var_name: str) -> str: - """ - Get the value of an environment variable. - Args: - - var_name (str): The name of the environment variable. - """ - cmd = "echo ${}".format(var_name) - return self.exec_command(cmd, encoding=ConsoleEncoding).strip() - - def find_executable(self, executable): - search_paths = self.environ("PATH") - if not search_paths: - return None - - search_paths = search_paths.split(self.pathsep) - for path in search_paths: - remote_file = os.path.join(path, executable) - if self.isfile(remote_file): - return remote_file - - return None - - def is_executable(self, file): - # Check if the file is executable - is_exec = self.exec_command("test -x {} && echo OK".format(file)) - return is_exec == b"OK\n" - - def set_env(self, var_name: str, var_val: str): - """ - Set the value of an environment variable. - Args: - - var_name (str): The name of the environment variable. - - var_val (str): The value to be set for the environment variable. - """ - return self.exec_command("export {}={}".format(var_name, var_val)) - - # Get environment variables - def get_user(self): - return self.exec_command("echo $USER", encoding=ConsoleEncoding).strip() - - def get_name(self): - cmd = 'python3 -c "import os; print(os.name)"' - return self.exec_command(cmd, encoding=ConsoleEncoding).strip() - - # Work with dirs - def makedirs(self, path, remove_existing=False): - """ - Create a directory in the remote server. - Args: - - path (str): The path to the directory to be created. - - remove_existing (bool): If True, the existing directory at the path will be removed. - """ - if remove_existing: - cmd = "rm -rf {} && mkdir -p {}".format(path, path) - else: - cmd = "mkdir -p {}".format(path) - try: - exit_status, result, error = self.exec_command(cmd, verbose=True) - except ExecUtilException as e: - raise Exception("Couldn't create dir {} because of error {}".format(path, e.message)) - if exit_status != 0: - raise Exception("Couldn't create dir {} because of error {}".format(path, error)) - return result - - def rmdirs(self, path, verbose=False, ignore_errors=True): - """ - Remove a directory in the remote server. - Args: - - path (str): The path to the directory to be removed. - - verbose (bool): If True, return exit status, result, and error. - - ignore_errors (bool): If True, do not raise error if directory does not exist. - """ - cmd = "rm -rf {}".format(path) - exit_status, result, error = self.exec_command(cmd, verbose=True) - if verbose: - return exit_status, result, error - else: - return result - - def listdir(self, path): - """ - List all files and directories in a directory. - Args: - path (str): The path to the directory. - """ - result = self.exec_command("ls {}".format(path)) - return result.splitlines() - - def path_exists(self, path): - result = self.exec_command("test -e {}; echo $?".format(path), encoding=ConsoleEncoding) - return int(result.strip()) == 0 - - @property - def pathsep(self): - os_name = self.get_name() - if os_name == "posix": - pathsep = ":" - elif os_name == "nt": - pathsep = ";" - else: - raise Exception("Unsupported operating system: {}".format(os_name)) - return pathsep - - def mkdtemp(self, prefix=None): - """ - Creates a temporary directory in the remote server. - Args: - - prefix (str): The prefix of the temporary directory name. - """ - if prefix: - command = ["ssh", "-i", self.ssh_key, f"{self.username}@{self.host}", f"mktemp -d {prefix}XXXXX"] - else: - command = ["ssh", "-i", self.ssh_key, f"{self.username}@{self.host}", "mktemp -d"] - - result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) - - if result.returncode == 0: - temp_dir = result.stdout.strip() - if not os.path.isabs(temp_dir): - temp_dir = os.path.join('/home', self.username, temp_dir) - return temp_dir - else: - raise ExecUtilException(f"Could not create temporary directory. Error: {result.stderr}") - - def mkstemp(self, prefix=None): - if prefix: - temp_dir = self.exec_command("mktemp {}XXXXX".format(prefix), encoding=ConsoleEncoding) - else: - temp_dir = self.exec_command("mktemp", encoding=ConsoleEncoding) - - if temp_dir: - if not os.path.isabs(temp_dir): - temp_dir = os.path.join('/home', self.username, temp_dir.strip()) - return temp_dir - else: - raise ExecUtilException("Could not create temporary directory.") - - def copytree(self, src, dst): - if not os.path.isabs(dst): - dst = os.path.join('~', dst) - if self.isdir(dst): - raise FileExistsError("Directory {} already exists.".format(dst)) - return self.exec_command("cp -r {} {}".format(src, dst)) - - # Work with files - def write(self, filename, data, truncate=False, binary=False, read_and_write=False, encoding=ConsoleEncoding): - mode = "wb" if binary else "w" - if not truncate: - mode = "ab" if binary else "a" - if read_and_write: - mode = "r+b" if binary else "r+" - - with tempfile.NamedTemporaryFile(mode=mode, delete=False) as tmp_file: - if not truncate: - scp_cmd = ['scp', '-i', self.ssh_key, f"{self.username}@{self.host}:{filename}", tmp_file.name] - subprocess.run(scp_cmd, check=False) # The file might not exist yet - tmp_file.seek(0, os.SEEK_END) - - if isinstance(data, bytes) and not binary: - data = data.decode(encoding) - elif isinstance(data, str) and binary: - data = data.encode(encoding) - - if isinstance(data, list): - data = [(s if isinstance(s, str) else s.decode(ConsoleEncoding)).rstrip('\n') + '\n' for s in data] - tmp_file.writelines(data) - else: - tmp_file.write(data) - - tmp_file.flush() - - scp_cmd = ['scp', '-i', self.ssh_key, tmp_file.name, f"{self.username}@{self.host}:{filename}"] - subprocess.run(scp_cmd, check=True) - - remote_directory = os.path.dirname(filename) - mkdir_cmd = ['ssh', '-i', self.ssh_key, f"{self.username}@{self.host}", f"mkdir -p {remote_directory}"] - subprocess.run(mkdir_cmd, check=True) - - os.remove(tmp_file.name) - - def touch(self, filename): - """ - Create a new file or update the access and modification times of an existing file on the remote server. - - Args: - filename (str): The name of the file to touch. - - This method behaves as the 'touch' command in Unix. It's equivalent to calling 'touch filename' in the shell. - """ - self.exec_command("touch {}".format(filename)) - - def read(self, filename, binary=False, encoding=None): - cmd = "cat {}".format(filename) - result = self.exec_command(cmd, encoding=encoding) - - if not binary and result: - result = result.decode(encoding or ConsoleEncoding) - - return result - - def readlines(self, filename, num_lines=0, binary=False, encoding=None): - if num_lines > 0: - cmd = "tail -n {} {}".format(num_lines, filename) - else: - cmd = "cat {}".format(filename) - - result = self.exec_command(cmd, encoding=encoding) - - if not binary and result: - lines = result.decode(encoding or ConsoleEncoding).splitlines() - else: - lines = result.splitlines() - - return lines - - def isfile(self, remote_file): - stdout = self.exec_command("test -f {}; echo $?".format(remote_file)) - result = int(stdout.strip()) - return result == 0 - - def isdir(self, dirname): - cmd = "if [ -d {} ]; then echo True; else echo False; fi".format(dirname) - response = self.exec_command(cmd) - return response.strip() == b"True" - - def remove_file(self, filename): - cmd = "rm {}".format(filename) - return self.exec_command(cmd) - - # Processes control - def kill(self, pid, signal): - # Kill the process - cmd = "kill -{} {}".format(signal, pid) - return self.exec_command(cmd) - - def get_pid(self): - # Get current process id - return int(self.exec_command("echo $$", encoding=ConsoleEncoding)) - - def get_process_children(self, pid): - command = ["ssh", "-i", self.ssh_key, f"{self.username}@{self.host}", f"pgrep -P {pid}"] - - result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) - - if result.returncode == 0: - children = result.stdout.strip().splitlines() - return [PsUtilProcessProxy(self, int(child_pid.strip())) for child_pid in children] - else: - raise ExecUtilException(f"Error in getting process children. Error: {result.stderr}") - - # Database control - def db_connect(self, dbname, user, password=None, host="localhost", port=5432): - """ - Established SSH tunnel and Connects to a PostgreSQL - """ - self.establish_ssh_tunnel(local_port=port, remote_port=5432) - try: - conn = pglib.connect( - host=host, - port=port, - database=dbname, - user=user, - password=password, - ) - return conn - except Exception as e: - raise Exception(f"Could not connect to the database. Error: {e}") diff --git a/testgres/utils.py b/testgres/utils.py deleted file mode 100644 index b7df70d1..00000000 --- a/testgres/utils.py +++ /dev/null @@ -1,242 +0,0 @@ -# coding: utf-8 - -from __future__ import division -from __future__ import print_function - -import os -import port_for -import sys - -from contextlib import contextmanager -from packaging.version import Version, InvalidVersion -import re - -from six import iteritems - -from .exceptions import ExecUtilException -from .config import testgres_config as tconf - -# rows returned by PG_CONFIG -_pg_config_data = {} - -# ports used by nodes -bound_ports = set() - - -# re-export version type -class PgVer(Version): - def __init__(self, version: str) -> None: - try: - super().__init__(version) - except InvalidVersion: - version = re.sub(r"[a-zA-Z].*", "", version) - super().__init__(version) - - -def reserve_port(): - """ - Generate a new port and add it to 'bound_ports'. - """ - - port = port_for.select_random(exclude_ports=bound_ports) - bound_ports.add(port) - - return port - - -def release_port(port): - """ - Free port provided by reserve_port(). - """ - - bound_ports.discard(port) - - -def execute_utility(args, logfile=None, verbose=False): - """ - Execute utility (pg_ctl, pg_dump etc). - - Args: - args: utility + arguments (list). - logfile: path to file to store stdout and stderr. - - Returns: - stdout of executed utility. - """ - exit_status, out, error = tconf.os_ops.exec_command(args, verbose=True) - # decode result - out = '' if not out else out - if isinstance(out, bytes): - out = out.decode('utf-8') - if isinstance(error, bytes): - error = error.decode('utf-8') - - # write new log entry if possible - if logfile: - try: - tconf.os_ops.write(filename=logfile, data=args, truncate=True) - if out: - # comment-out lines - lines = [u'\n'] + ['# ' + line for line in out.splitlines()] + [u'\n'] - tconf.os_ops.write(filename=logfile, data=lines) - except IOError: - raise ExecUtilException("Problem with writing to logfile `{}` during run command `{}`".format(logfile, args)) - if verbose: - return exit_status, out, error - else: - return out - - -def get_bin_path(filename): - """ - Return absolute path to an executable using PG_BIN or PG_CONFIG. - This function does nothing if 'filename' is already absolute. - """ - # check if it's already absolute - if os.path.isabs(filename): - return filename - if tconf.os_ops.remote: - pg_config = os.environ.get("PG_CONFIG_REMOTE") or os.environ.get("PG_CONFIG") - else: - # try PG_CONFIG - get from local machine - pg_config = os.environ.get("PG_CONFIG") - - if pg_config: - bindir = get_pg_config()["BINDIR"] - return os.path.join(bindir, filename) - - # try PG_BIN - pg_bin = tconf.os_ops.environ("PG_BIN") - if pg_bin: - return os.path.join(pg_bin, filename) - - pg_config_path = tconf.os_ops.find_executable('pg_config') - if pg_config_path: - bindir = get_pg_config(pg_config_path)["BINDIR"] - return os.path.join(bindir, filename) - - return filename - - -def get_pg_config(pg_config_path=None, os_ops=None): - """ - Return output of pg_config (provided that it is installed). - NOTE: this function caches the result by default (see GlobalConfig). - """ - if os_ops: - tconf.os_ops = os_ops - - def cache_pg_config_data(cmd): - # execute pg_config and get the output - out = tconf.os_ops.exec_command(cmd, encoding='utf-8') - - data = {} - for line in out.splitlines(): - if line and '=' in line: - key, _, value = line.partition('=') - data[key.strip()] = value.strip() - - # cache data - global _pg_config_data - _pg_config_data = data - - return data - - # drop cache if asked to - if not tconf.cache_pg_config: - global _pg_config_data - _pg_config_data = {} - - # return cached data - if not pg_config_path and _pg_config_data: - return _pg_config_data - - # try specified pg_config path or PG_CONFIG - if tconf.os_ops.remote: - pg_config = pg_config_path or os.environ.get("PG_CONFIG_REMOTE") or os.environ.get("PG_CONFIG") - else: - # try PG_CONFIG - get from local machine - pg_config = pg_config_path or os.environ.get("PG_CONFIG") - if pg_config: - return cache_pg_config_data(pg_config) - - # try PG_BIN - pg_bin = os.environ.get("PG_BIN") - if pg_bin: - cmd = os.path.join(pg_bin, "pg_config") - return cache_pg_config_data(cmd) - - # try plain name - return cache_pg_config_data("pg_config") - - -def get_pg_version(): - """ - Return PostgreSQL version provided by postmaster. - """ - - # get raw version (e.g. postgres (PostgreSQL) 9.5.7) - _params = [get_bin_path('postgres'), '--version'] - raw_ver = tconf.os_ops.exec_command(_params, encoding='utf-8') - - # cook version of PostgreSQL - version = raw_ver.strip().split(' ')[-1] \ - .partition('devel')[0] \ - .partition('beta')[0] \ - .partition('rc')[0] - - return version - - -def file_tail(f, num_lines): - """ - Get last N lines of a file. - """ - - assert num_lines > 0 - - bufsize = 8192 - buffers = 1 - - f.seek(0, os.SEEK_END) - end_pos = f.tell() - - while True: - offset = max(0, end_pos - bufsize * buffers) - f.seek(offset, os.SEEK_SET) - pos = f.tell() - - lines = f.readlines() - cur_lines = len(lines) - - if cur_lines > num_lines or pos == 0: - return lines[-num_lines:] - - buffers = int(buffers * max(2, num_lines / max(cur_lines, 1))) - - -def eprint(*args, **kwargs): - """ - Print stuff to stderr. - """ - - print(*args, file=sys.stderr, **kwargs) - - -def options_string(separator=u" ", **kwargs): - return separator.join(u"{}={}".format(k, v) for k, v in iteritems(kwargs)) - - -@contextmanager -def clean_on_error(node): - """ - Context manager to wrap PostgresNode and such. - Calls cleanup() method when underlying code raises an exception. - """ - - try: - yield node - except Exception: - # TODO: should we wrap this in try-block? - node.cleanup() - raise diff --git a/tests/README.md b/tests/README.md index d89efc7e..66952854 100644 --- a/tests/README.md +++ b/tests/README.md @@ -7,14 +7,13 @@ virtualenv venv source venv/bin/activate -# Install local version of testgres -pip install -U . +pip install -r tests/requirements.txt # Set path to PostgreSQL export PG_BIN=/path/to/pg/bin # Run tests -./tests/test_simple.py +pytest -l -v -n 4 tests ``` #### All configurations + coverage @@ -22,37 +21,10 @@ export PG_BIN=/path/to/pg/bin ```bash # Set path to PostgreSQL and python version export PATH=/path/to/pg/bin:$PATH -export PYTHON_VERSION=3 # or 2 + +# Set path of python binary +export PYTHON_BINARY=python3 # Run tests ./run_tests.sh ``` - - -#### Remote host tests - -1. Start remote host or docker container -2. Make sure that you run ssh -```commandline -sudo apt-get install openssh-server -sudo systemctl start sshd -``` -3. You need to connect to the remote host at least once to add it to the known hosts file -4. Generate ssh keys -5. Set up params for tests - - -```commandline -conn_params = ConnectionParams( - host='remote_host', - username='username', - ssh_key=/path/to/your/ssh/key' -) -os_ops = RemoteOperations(conn_params) -``` -If you have different path to `PG_CONFIG` on your local and remote host you can set up `PG_CONFIG_REMOTE`, this value will be -using during work with remote host. - -`test_remote` - Tests for RemoteOperations class. - -`test_simple_remote` - Tests that create node and check it. The same as `test_simple`, but for remote node. \ No newline at end of file diff --git a/testgres/operations/__init__.py b/tests/__init__.py similarity index 100% rename from testgres/operations/__init__.py rename to tests/__init__.py diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 00000000..a1adc157 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,1157 @@ +# ///////////////////////////////////////////////////////////////////////////// +# PyTest Configuration + +import pluggy +import pytest +import os +import logging +import pathlib +import math +import datetime +import typing +import enum + +import _pytest.outcomes +import _pytest.unittest +import _pytest.logging + +from packaging.version import Version + +# ///////////////////////////////////////////////////////////////////////////// + +C_ROOT_DIR__RELATIVE = ".." + +# ///////////////////////////////////////////////////////////////////////////// + +T_TUPLE__str_int = typing.Tuple[str, int] + +# ///////////////////////////////////////////////////////////////////////////// +# T_PLUGGY_RESULT + +if Version(pluggy.__version__) <= Version("1.2"): + T_PLUGGY_RESULT = pluggy._result._Result +else: + T_PLUGGY_RESULT = pluggy.Result + +# ///////////////////////////////////////////////////////////////////////////// + +g_error_msg_count_key = pytest.StashKey[int]() +g_warning_msg_count_key = pytest.StashKey[int]() +g_critical_msg_count_key = pytest.StashKey[int]() + +# ///////////////////////////////////////////////////////////////////////////// +# T_TEST_PROCESS_KIND + + +class T_TEST_PROCESS_KIND(enum.Enum): + Master = 1 + Worker = 2 + + +# ///////////////////////////////////////////////////////////////////////////// +# T_TEST_PROCESS_MODE + + +class T_TEST_PROCESS_MODE(enum.Enum): + Collect = 1 + ExecTests = 2 + + +# ///////////////////////////////////////////////////////////////////////////// + +g_test_process_kind: typing.Optional[T_TEST_PROCESS_KIND] = None +g_test_process_mode: typing.Optional[T_TEST_PROCESS_MODE] = None + +g_worker_log_is_created: typing.Optional[bool] = None + +# ///////////////////////////////////////////////////////////////////////////// +# TestConfigPropNames + + +class TestConfigPropNames: + TEST_CFG__LOG_DIR = "TEST_CFG__LOG_DIR" + + +# ///////////////////////////////////////////////////////////////////////////// +# TestStartupData__Helper + + +class TestStartupData__Helper: + sm_StartTS = datetime.datetime.now() + + # -------------------------------------------------------------------- + @staticmethod + def GetStartTS() -> datetime.datetime: + assert type(__class__.sm_StartTS) is datetime.datetime + return __class__.sm_StartTS + + # -------------------------------------------------------------------- + @staticmethod + def CalcRootDir() -> str: + r = os.path.abspath(__file__) + r = os.path.dirname(r) + r = os.path.join(r, C_ROOT_DIR__RELATIVE) + r = os.path.abspath(r) + return r + + # -------------------------------------------------------------------- + @staticmethod + def CalcRootLogDir() -> str: + if TestConfigPropNames.TEST_CFG__LOG_DIR in os.environ: + resultPath = os.environ[TestConfigPropNames.TEST_CFG__LOG_DIR] + else: + rootDir = __class__.CalcRootDir() + resultPath = os.path.join(rootDir, "logs") + + assert type(resultPath) is str + return resultPath + + # -------------------------------------------------------------------- + @staticmethod + def CalcCurrentTestWorkerSignature() -> str: + currentPID = os.getpid() + assert type(currentPID) is int + + startTS = __class__.sm_StartTS + assert type(startTS) is datetime.datetime + + result = "pytest-{0:04d}{1:02d}{2:02d}_{3:02d}{4:02d}{5:02d}".format( + startTS.year, + startTS.month, + startTS.day, + startTS.hour, + startTS.minute, + startTS.second, + ) + + gwid = os.environ.get("PYTEST_XDIST_WORKER") + + if gwid is not None: + result += "--xdist_" + str(gwid) + + result += "--" + "pid" + str(currentPID) + return result + + +# ///////////////////////////////////////////////////////////////////////////// +# TestStartupData + + +class TestStartupData: + sm_RootDir: str = TestStartupData__Helper.CalcRootDir() + sm_CurrentTestWorkerSignature: str = ( + TestStartupData__Helper.CalcCurrentTestWorkerSignature() + ) + + sm_RootLogDir: str = TestStartupData__Helper.CalcRootLogDir() + + # -------------------------------------------------------------------- + @staticmethod + def GetRootDir() -> str: + assert type(__class__.sm_RootDir) is str + return __class__.sm_RootDir + + # -------------------------------------------------------------------- + @staticmethod + def GetRootLogDir() -> str: + assert type(__class__.sm_RootLogDir) is str + return __class__.sm_RootLogDir + + # -------------------------------------------------------------------- + @staticmethod + def GetCurrentTestWorkerSignature() -> str: + assert type(__class__.sm_CurrentTestWorkerSignature) is str + return __class__.sm_CurrentTestWorkerSignature + + +# ///////////////////////////////////////////////////////////////////////////// +# TEST_PROCESS_STATS + + +class TEST_PROCESS_STATS: + cTotalTests: int = 0 + cNotExecutedTests: int = 0 + cExecutedTests: int = 0 + cPassedTests: int = 0 + cFailedTests: int = 0 + cXFailedTests: int = 0 + cSkippedTests: int = 0 + cNotXFailedTests: int = 0 + cWarningTests: int = 0 + cUnexpectedTests: int = 0 + cAchtungTests: int = 0 + + FailedTests: typing.List[T_TUPLE__str_int] = list() + XFailedTests: typing.List[T_TUPLE__str_int] = list() + NotXFailedTests: typing.List[str] = list() + WarningTests: typing.List[T_TUPLE__str_int] = list() + AchtungTests: typing.List[str] = list() + + cTotalDuration: datetime.timedelta = datetime.timedelta() + + cTotalErrors: int = 0 + cTotalWarnings: int = 0 + + # -------------------------------------------------------------------- + @staticmethod + def incrementTotalTestCount() -> None: + assert type(__class__.cTotalTests) is int + assert __class__.cTotalTests >= 0 + + __class__.cTotalTests += 1 + + assert __class__.cTotalTests > 0 + + # -------------------------------------------------------------------- + @staticmethod + def incrementNotExecutedTestCount() -> None: + assert type(__class__.cNotExecutedTests) is int + assert __class__.cNotExecutedTests >= 0 + + __class__.cNotExecutedTests += 1 + + assert __class__.cNotExecutedTests > 0 + + # -------------------------------------------------------------------- + @staticmethod + def incrementExecutedTestCount() -> int: + assert type(__class__.cExecutedTests) is int + assert __class__.cExecutedTests >= 0 + + __class__.cExecutedTests += 1 + + assert __class__.cExecutedTests > 0 + return __class__.cExecutedTests + + # -------------------------------------------------------------------- + @staticmethod + def incrementPassedTestCount() -> None: + assert type(__class__.cPassedTests) is int + assert __class__.cPassedTests >= 0 + + __class__.cPassedTests += 1 + + assert __class__.cPassedTests > 0 + + # -------------------------------------------------------------------- + @staticmethod + def incrementFailedTestCount(testID: str, errCount: int) -> None: + assert type(testID) is str + assert type(errCount) is int + assert errCount > 0 + assert type(__class__.FailedTests) is list + assert type(__class__.cFailedTests) is int + assert __class__.cFailedTests >= 0 + + __class__.FailedTests.append((testID, errCount)) # raise? + __class__.cFailedTests += 1 + + assert len(__class__.FailedTests) > 0 + assert __class__.cFailedTests > 0 + assert len(__class__.FailedTests) == __class__.cFailedTests + + # -------- + assert type(__class__.cTotalErrors) is int + assert __class__.cTotalErrors >= 0 + + __class__.cTotalErrors += errCount + + assert __class__.cTotalErrors > 0 + + # -------------------------------------------------------------------- + @staticmethod + def incrementXFailedTestCount(testID: str, errCount: int) -> None: + assert type(testID) is str + assert type(errCount) is int + assert errCount >= 0 + assert type(__class__.XFailedTests) is list + assert type(__class__.cXFailedTests) is int + assert __class__.cXFailedTests >= 0 + + __class__.XFailedTests.append((testID, errCount)) # raise? + __class__.cXFailedTests += 1 + + assert len(__class__.XFailedTests) > 0 + assert __class__.cXFailedTests > 0 + assert len(__class__.XFailedTests) == __class__.cXFailedTests + + # -------------------------------------------------------------------- + @staticmethod + def incrementSkippedTestCount() -> None: + assert type(__class__.cSkippedTests) is int + assert __class__.cSkippedTests >= 0 + + __class__.cSkippedTests += 1 + + assert __class__.cSkippedTests > 0 + + # -------------------------------------------------------------------- + @staticmethod + def incrementNotXFailedTests(testID: str) -> None: + assert type(testID) is str + assert type(__class__.NotXFailedTests) is list + assert type(__class__.cNotXFailedTests) is int + assert __class__.cNotXFailedTests >= 0 + + __class__.NotXFailedTests.append(testID) # raise? + __class__.cNotXFailedTests += 1 + + assert len(__class__.NotXFailedTests) > 0 + assert __class__.cNotXFailedTests > 0 + assert len(__class__.NotXFailedTests) == __class__.cNotXFailedTests + + # -------------------------------------------------------------------- + @staticmethod + def incrementWarningTestCount(testID: str, warningCount: int) -> None: + assert type(testID) is str + assert type(warningCount) is int + assert testID != "" + assert warningCount > 0 + assert type(__class__.WarningTests) is list + assert type(__class__.cWarningTests) is int + assert __class__.cWarningTests >= 0 + + __class__.WarningTests.append((testID, warningCount)) # raise? + __class__.cWarningTests += 1 + + assert len(__class__.WarningTests) > 0 + assert __class__.cWarningTests > 0 + assert len(__class__.WarningTests) == __class__.cWarningTests + + # -------- + assert type(__class__.cTotalWarnings) is int + assert __class__.cTotalWarnings >= 0 + + __class__.cTotalWarnings += warningCount + + assert __class__.cTotalWarnings > 0 + + # -------------------------------------------------------------------- + @staticmethod + def incrementUnexpectedTests() -> None: + assert type(__class__.cUnexpectedTests) is int + assert __class__.cUnexpectedTests >= 0 + + __class__.cUnexpectedTests += 1 + + assert __class__.cUnexpectedTests > 0 + + # -------------------------------------------------------------------- + @staticmethod + def incrementAchtungTestCount(testID: str) -> None: + assert type(testID) is str + assert type(__class__.AchtungTests) is list + assert type(__class__.cAchtungTests) is int + assert __class__.cAchtungTests >= 0 + + __class__.AchtungTests.append(testID) # raise? + __class__.cAchtungTests += 1 + + assert len(__class__.AchtungTests) > 0 + assert __class__.cAchtungTests > 0 + assert len(__class__.AchtungTests) == __class__.cAchtungTests + + +# ///////////////////////////////////////////////////////////////////////////// + + +def timedelta_to_human_text(delta: datetime.timedelta) -> str: + assert isinstance(delta, datetime.timedelta) + + C_SECONDS_IN_MINUTE = 60 + C_SECONDS_IN_HOUR = 60 * C_SECONDS_IN_MINUTE + + v = delta.seconds + + cHours = int(v / C_SECONDS_IN_HOUR) + v = v - cHours * C_SECONDS_IN_HOUR + cMinutes = int(v / C_SECONDS_IN_MINUTE) + cSeconds = v - cMinutes * C_SECONDS_IN_MINUTE + + result = "" if delta.days == 0 else "{0} day(s) ".format(delta.days) + + result = result + "{:02d}:{:02d}:{:02d}.{:06d}".format( + cHours, cMinutes, cSeconds, delta.microseconds + ) + + return result + + +# ///////////////////////////////////////////////////////////////////////////// + + +def helper__build_test_id(item: pytest.Function) -> str: + assert item is not None + assert isinstance(item, pytest.Function) + + testID = "" + + if item.cls is not None: + testID = item.cls.__module__ + "." + item.cls.__name__ + "::" + + testID = testID + item.name + + return testID + + +# ///////////////////////////////////////////////////////////////////////////// + + +def helper__makereport__setup( + item: pytest.Function, call: pytest.CallInfo, outcome: T_PLUGGY_RESULT +): + assert item is not None + assert call is not None + assert outcome is not None + # it may be pytest.Function or _pytest.unittest.TestCaseFunction + assert isinstance(item, pytest.Function) + assert type(call) is pytest.CallInfo + assert type(outcome) is T_PLUGGY_RESULT + + C_LINE1 = "******************************************************" + + # logging.info("pytest_runtest_makereport - setup") + + TEST_PROCESS_STATS.incrementTotalTestCount() + + rep: pytest.TestReport = outcome.get_result() + assert rep is not None + assert type(rep) is pytest.TestReport + + if rep.outcome == "skipped": + TEST_PROCESS_STATS.incrementNotExecutedTestCount() + return + + testID = helper__build_test_id(item) + + if rep.outcome == "passed": + testNumber = TEST_PROCESS_STATS.incrementExecutedTestCount() + + logging.info(C_LINE1) + logging.info("* START TEST {0}".format(testID)) + logging.info("*") + logging.info("* Path : {0}".format(item.path)) + logging.info("* Number: {0}".format(testNumber)) + logging.info("*") + return + + assert rep.outcome != "passed" + + TEST_PROCESS_STATS.incrementAchtungTestCount(testID) + + logging.info(C_LINE1) + logging.info("* ACHTUNG TEST {0}".format(testID)) + logging.info("*") + logging.info("* Path : {0}".format(item.path)) + logging.info("* Outcome is [{0}]".format(rep.outcome)) + + if rep.outcome == "failed": + assert call.excinfo is not None + assert call.excinfo.value is not None + logging.info("*") + logging.error(call.excinfo.value) + + logging.info("*") + return + + +# ------------------------------------------------------------------------ +class ExitStatusNames: + FAILED = "FAILED" + PASSED = "PASSED" + XFAILED = "XFAILED" + NOT_XFAILED = "NOT XFAILED" + SKIPPED = "SKIPPED" + UNEXPECTED = "UNEXPECTED" + + +# ------------------------------------------------------------------------ +def helper__makereport__call( + item: pytest.Function, call: pytest.CallInfo, outcome: T_PLUGGY_RESULT +): + assert item is not None + assert call is not None + assert outcome is not None + # it may be pytest.Function or _pytest.unittest.TestCaseFunction + assert isinstance(item, pytest.Function) + assert type(call) is pytest.CallInfo + assert type(outcome) is T_PLUGGY_RESULT + + # -------- + item_error_msg_count1 = item.stash.get(g_error_msg_count_key, 0) + assert type(item_error_msg_count1) is int + assert item_error_msg_count1 >= 0 + + item_error_msg_count2 = item.stash.get(g_critical_msg_count_key, 0) + assert type(item_error_msg_count2) is int + assert item_error_msg_count2 >= 0 + + item_error_msg_count = item_error_msg_count1 + item_error_msg_count2 + + # -------- + item_warning_msg_count = item.stash.get(g_warning_msg_count_key, 0) + assert type(item_warning_msg_count) is int + assert item_warning_msg_count >= 0 + + # -------- + rep = outcome.get_result() + assert rep is not None + assert type(rep) is pytest.TestReport + + # -------- + testID = helper__build_test_id(item) + + # -------- + assert call.start <= call.stop + + startDT = datetime.datetime.fromtimestamp(call.start) + assert type(startDT) is datetime.datetime + stopDT = datetime.datetime.fromtimestamp(call.stop) + assert type(stopDT) is datetime.datetime + + testDurration = stopDT - startDT + assert type(testDurration) is datetime.timedelta + + # -------- + exitStatus = None + exitStatusInfo = None + if rep.outcome == "skipped": + assert call.excinfo is not None # research + assert call.excinfo.value is not None # research + + if type(call.excinfo.value) is _pytest.outcomes.Skipped: + assert not hasattr(rep, "wasxfail") + + exitStatus = ExitStatusNames.SKIPPED + reasonText = str(call.excinfo.value) + reasonMsgTempl = "SKIP REASON: {0}" + + TEST_PROCESS_STATS.incrementSkippedTestCount() + + elif type(call.excinfo.value) is _pytest.outcomes.XFailed: + exitStatus = ExitStatusNames.XFAILED + reasonText = str(call.excinfo.value) + reasonMsgTempl = "XFAIL REASON: {0}" + + TEST_PROCESS_STATS.incrementXFailedTestCount(testID, item_error_msg_count) + + else: + exitStatus = ExitStatusNames.XFAILED + assert hasattr(rep, "wasxfail") + assert rep.wasxfail is not None + assert type(rep.wasxfail) is str + + reasonText = rep.wasxfail + reasonMsgTempl = "XFAIL REASON: {0}" + + if type(call.excinfo.value) is SIGNAL_EXCEPTION: + pass + else: + logging.error(call.excinfo.value) + item_error_msg_count += 1 + + TEST_PROCESS_STATS.incrementXFailedTestCount(testID, item_error_msg_count) + + assert type(reasonText) is str + + if reasonText != "": + assert type(reasonMsgTempl) is str + logging.info("*") + logging.info("* " + reasonMsgTempl.format(reasonText)) + + elif rep.outcome == "failed": + assert call.excinfo is not None + assert call.excinfo.value is not None + + if type(call.excinfo.value) is SIGNAL_EXCEPTION: + assert item_error_msg_count > 0 + pass + else: + logging.error(call.excinfo.value) + item_error_msg_count += 1 + + assert item_error_msg_count > 0 + TEST_PROCESS_STATS.incrementFailedTestCount(testID, item_error_msg_count) + + exitStatus = ExitStatusNames.FAILED + elif rep.outcome == "passed": + assert call.excinfo is None + + if hasattr(rep, "wasxfail"): + assert type(rep.wasxfail) is str + + TEST_PROCESS_STATS.incrementNotXFailedTests(testID) + + warnMsg = "NOTE: Test is marked as xfail" + + if rep.wasxfail != "": + warnMsg += " [" + rep.wasxfail + "]" + + logging.info(warnMsg) + exitStatus = ExitStatusNames.NOT_XFAILED + else: + assert not hasattr(rep, "wasxfail") + + TEST_PROCESS_STATS.incrementPassedTestCount() + exitStatus = ExitStatusNames.PASSED + else: + TEST_PROCESS_STATS.incrementUnexpectedTests() + exitStatus = ExitStatusNames.UNEXPECTED + exitStatusInfo = rep.outcome + # [2025-03-28] It may create a useless problem in new environment. + # assert False + + # -------- + if item_warning_msg_count > 0: + TEST_PROCESS_STATS.incrementWarningTestCount(testID, item_warning_msg_count) + + # -------- + assert exitStatus is not None + assert type(exitStatus) is str + + if exitStatus == ExitStatusNames.FAILED: + assert item_error_msg_count > 0 + pass + + # -------- + assert type(TEST_PROCESS_STATS.cTotalDuration) is datetime.timedelta + assert type(testDurration) is datetime.timedelta + + TEST_PROCESS_STATS.cTotalDuration += testDurration + + assert testDurration <= TEST_PROCESS_STATS.cTotalDuration + + # -------- + exitStatusLineData = exitStatus + + if exitStatusInfo is not None: + exitStatusLineData += " [{}]".format(exitStatusInfo) + + # -------- + logging.info("*") + logging.info("* DURATION : {0}".format(timedelta_to_human_text(testDurration))) + logging.info("*") + logging.info("* EXIT STATUS : {0}".format(exitStatusLineData)) + logging.info("* ERROR COUNT : {0}".format(item_error_msg_count)) + logging.info("* WARNING COUNT: {0}".format(item_warning_msg_count)) + logging.info("*") + logging.info("* STOP TEST {0}".format(testID)) + logging.info("*") + + +# ///////////////////////////////////////////////////////////////////////////// + + +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_makereport(item: pytest.Function, call: pytest.CallInfo): + # + # https://docs.pytest.org/en/7.1.x/how-to/writing_hook_functions.html#hookwrapper-executing-around-other-hooks + # + # Note that hook wrappers donโ€™t return results themselves, + # they merely perform tracing or other side effects around the actual hook implementations. + # + # https://docs.pytest.org/en/7.1.x/reference/reference.html#test-running-runtest-hooks + # + assert item is not None + assert call is not None + # it may be pytest.Function or _pytest.unittest.TestCaseFunction + assert isinstance(item, pytest.Function) + assert type(call) is pytest.CallInfo + + outcome = yield + assert outcome is not None + assert type(outcome) is T_PLUGGY_RESULT + + assert type(call.when) is str + + if call.when == "collect": + return + + if call.when == "setup": + helper__makereport__setup(item, call, outcome) + return + + if call.when == "call": + helper__makereport__call(item, call, outcome) + return + + if call.when == "teardown": + return + + errMsg = "[pytest_runtest_makereport] unknown 'call.when' value: [{0}].".format( + call.when + ) + + raise RuntimeError(errMsg) + + +# ///////////////////////////////////////////////////////////////////////////// + + +class LogWrapper2: + _old_method: typing.Any + _err_counter: typing.Optional[int] + _warn_counter: typing.Optional[int] + + _critical_counter: typing.Optional[int] + + # -------------------------------------------------------------------- + def __init__(self): + self._old_method = None + self._err_counter = None + self._warn_counter = None + + self._critical_counter = None + + # -------------------------------------------------------------------- + def __enter__(self): + assert self._old_method is None + assert self._err_counter is None + assert self._warn_counter is None + + assert self._critical_counter is None + + assert logging.root is not None + assert isinstance(logging.root, logging.RootLogger) + + self._old_method = logging.root.handle + self._err_counter = 0 + self._warn_counter = 0 + + self._critical_counter = 0 + + logging.root.handle = self + return self + + # -------------------------------------------------------------------- + def __exit__(self, exc_type, exc_val, exc_tb): + assert self._old_method is not None + assert self._err_counter is not None + assert self._warn_counter is not None + + assert logging.root is not None + assert isinstance(logging.root, logging.RootLogger) + + assert logging.root.handle is self + + logging.root.handle = self._old_method + + self._old_method = None + self._err_counter = None + self._warn_counter = None + self._critical_counter = None + return False + + # -------------------------------------------------------------------- + def __call__(self, record: logging.LogRecord): + assert record is not None + assert isinstance(record, logging.LogRecord) + assert self._old_method is not None + assert self._err_counter is not None + assert self._warn_counter is not None + assert self._critical_counter is not None + + assert type(self._err_counter) is int + assert self._err_counter >= 0 + assert type(self._warn_counter) is int + assert self._warn_counter >= 0 + assert type(self._critical_counter) is int + assert self._critical_counter >= 0 + + r = self._old_method(record) + + if record.levelno == logging.ERROR: + self._err_counter += 1 + assert self._err_counter > 0 + elif record.levelno == logging.WARNING: + self._warn_counter += 1 + assert self._warn_counter > 0 + elif record.levelno == logging.CRITICAL: + self._critical_counter += 1 + assert self._critical_counter > 0 + + return r + + +# ///////////////////////////////////////////////////////////////////////////// + + +class SIGNAL_EXCEPTION(Exception): + def __init__(self): + pass + + +# ///////////////////////////////////////////////////////////////////////////// + + +@pytest.hookimpl(hookwrapper=True) +def pytest_pyfunc_call(pyfuncitem: pytest.Function): + assert pyfuncitem is not None + assert isinstance(pyfuncitem, pytest.Function) + + assert logging.root is not None + assert isinstance(logging.root, logging.RootLogger) + assert logging.root.handle is not None + + debug__log_handle_method = logging.root.handle + assert debug__log_handle_method is not None + + debug__log_error_method = logging.error + assert debug__log_error_method is not None + + debug__log_warning_method = logging.warning + assert debug__log_warning_method is not None + + pyfuncitem.stash[g_error_msg_count_key] = 0 + pyfuncitem.stash[g_warning_msg_count_key] = 0 + pyfuncitem.stash[g_critical_msg_count_key] = 0 + + try: + with LogWrapper2() as logWrapper: + assert type(logWrapper) is LogWrapper2 + assert logWrapper._old_method is not None + assert type(logWrapper._err_counter) is int + assert logWrapper._err_counter == 0 + assert type(logWrapper._warn_counter) is int + assert logWrapper._warn_counter == 0 + assert type(logWrapper._critical_counter) is int + assert logWrapper._critical_counter == 0 + assert logging.root.handle is logWrapper + + r = yield + + assert r is not None + assert type(r) is T_PLUGGY_RESULT + + assert logWrapper._old_method is not None + assert type(logWrapper._err_counter) is int + assert logWrapper._err_counter >= 0 + assert type(logWrapper._warn_counter) is int + assert logWrapper._warn_counter >= 0 + assert type(logWrapper._critical_counter) is int + assert logWrapper._critical_counter >= 0 + assert logging.root.handle is logWrapper + + assert g_error_msg_count_key in pyfuncitem.stash + assert g_warning_msg_count_key in pyfuncitem.stash + assert g_critical_msg_count_key in pyfuncitem.stash + + assert pyfuncitem.stash[g_error_msg_count_key] == 0 + assert pyfuncitem.stash[g_warning_msg_count_key] == 0 + assert pyfuncitem.stash[g_critical_msg_count_key] == 0 + + pyfuncitem.stash[g_error_msg_count_key] = logWrapper._err_counter + pyfuncitem.stash[g_warning_msg_count_key] = logWrapper._warn_counter + pyfuncitem.stash[g_critical_msg_count_key] = logWrapper._critical_counter + + if r.exception is not None: + pass + elif logWrapper._err_counter > 0: + r.force_exception(SIGNAL_EXCEPTION()) + elif logWrapper._critical_counter > 0: + r.force_exception(SIGNAL_EXCEPTION()) + finally: + assert logging.error is debug__log_error_method + assert logging.warning is debug__log_warning_method + assert logging.root.handle == debug__log_handle_method + pass + + +# ///////////////////////////////////////////////////////////////////////////// + + +def helper__calc_W(n: int) -> int: + assert n > 0 + + x = int(math.log10(n)) + assert type(x) is int + assert x >= 0 + x += 1 + return x + + +# ------------------------------------------------------------------------ +def helper__print_test_list(tests: typing.List[str]) -> None: + assert type(tests) is list + + assert helper__calc_W(9) == 1 + assert helper__calc_W(10) == 2 + assert helper__calc_W(11) == 2 + assert helper__calc_W(99) == 2 + assert helper__calc_W(100) == 3 + assert helper__calc_W(101) == 3 + assert helper__calc_W(999) == 3 + assert helper__calc_W(1000) == 4 + assert helper__calc_W(1001) == 4 + + W = helper__calc_W(len(tests)) + + templateLine = "{0:0" + str(W) + "d}. {1}" + + nTest = 0 + + for t in tests: + assert type(t) is str + assert t != "" + nTest += 1 + logging.info(templateLine.format(nTest, t)) + + +# ------------------------------------------------------------------------ +def helper__print_test_list2(tests: typing.List[T_TUPLE__str_int]) -> None: + assert type(tests) is list + + assert helper__calc_W(9) == 1 + assert helper__calc_W(10) == 2 + assert helper__calc_W(11) == 2 + assert helper__calc_W(99) == 2 + assert helper__calc_W(100) == 3 + assert helper__calc_W(101) == 3 + assert helper__calc_W(999) == 3 + assert helper__calc_W(1000) == 4 + assert helper__calc_W(1001) == 4 + + W = helper__calc_W(len(tests)) + + templateLine = "{0:0" + str(W) + "d}. {1} ({2})" + + nTest = 0 + + for t in tests: + assert type(t) is tuple + assert len(t) == 2 + assert type(t[0]) is str + assert type(t[1]) is int + assert t[0] != "" + assert t[1] >= 0 + nTest += 1 + logging.info(templateLine.format(nTest, t[0], t[1])) + + +# ///////////////////////////////////////////////////////////////////////////// +# SUMMARY BUILDER + + +@pytest.hookimpl(trylast=True) +def pytest_sessionfinish(): + # + # NOTE: It should execute after logging.pytest_sessionfinish + # + + global g_test_process_kind # noqa: F824 + global g_test_process_mode # noqa: F824 + global g_worker_log_is_created # noqa: F824 + + assert g_test_process_kind is not None + assert type(g_test_process_kind) is T_TEST_PROCESS_KIND + + if g_test_process_kind == T_TEST_PROCESS_KIND.Master: + return + + assert g_test_process_kind == T_TEST_PROCESS_KIND.Worker + + assert g_test_process_mode is not None + assert type(g_test_process_mode) is T_TEST_PROCESS_MODE + + if g_test_process_mode == T_TEST_PROCESS_MODE.Collect: + return + + assert g_test_process_mode == T_TEST_PROCESS_MODE.ExecTests + + assert type(g_worker_log_is_created) is bool + assert g_worker_log_is_created + + C_LINE1 = "---------------------------" + + def LOCAL__print_line1_with_header(header: str): + assert type(C_LINE1) is str + assert type(header) is str + assert header != "" + logging.info(C_LINE1 + " [" + header + "]") + + def LOCAL__print_test_list( + header: str, test_count: int, test_list: typing.List[str] + ): + assert type(header) is str + assert type(test_count) is int + assert type(test_list) is list + assert header != "" + assert test_count >= 0 + assert len(test_list) == test_count + + LOCAL__print_line1_with_header(header) + logging.info("") + if len(test_list) > 0: + helper__print_test_list(test_list) + logging.info("") + + def LOCAL__print_test_list2( + header: str, test_count: int, test_list: typing.List[T_TUPLE__str_int] + ): + assert type(header) is str + assert type(test_count) is int + assert type(test_list) is list + assert header != "" + assert test_count >= 0 + assert len(test_list) == test_count + + LOCAL__print_line1_with_header(header) + logging.info("") + if len(test_list) > 0: + helper__print_test_list2(test_list) + logging.info("") + + # fmt: off + LOCAL__print_test_list( + "ACHTUNG TESTS", + TEST_PROCESS_STATS.cAchtungTests, + TEST_PROCESS_STATS.AchtungTests, + ) + + LOCAL__print_test_list2( + "FAILED TESTS", + TEST_PROCESS_STATS.cFailedTests, + TEST_PROCESS_STATS.FailedTests + ) + + LOCAL__print_test_list2( + "XFAILED TESTS", + TEST_PROCESS_STATS.cXFailedTests, + TEST_PROCESS_STATS.XFailedTests, + ) + + LOCAL__print_test_list( + "NOT XFAILED TESTS", + TEST_PROCESS_STATS.cNotXFailedTests, + TEST_PROCESS_STATS.NotXFailedTests, + ) + + LOCAL__print_test_list2( + "WARNING TESTS", + TEST_PROCESS_STATS.cWarningTests, + TEST_PROCESS_STATS.WarningTests, + ) + # fmt: on + + LOCAL__print_line1_with_header("SUMMARY STATISTICS") + logging.info("") + logging.info("[TESTS]") + logging.info(" TOTAL : {0}".format(TEST_PROCESS_STATS.cTotalTests)) + logging.info(" EXECUTED : {0}".format(TEST_PROCESS_STATS.cExecutedTests)) + logging.info(" NOT EXECUTED : {0}".format(TEST_PROCESS_STATS.cNotExecutedTests)) + logging.info(" ACHTUNG : {0}".format(TEST_PROCESS_STATS.cAchtungTests)) + logging.info("") + logging.info(" PASSED : {0}".format(TEST_PROCESS_STATS.cPassedTests)) + logging.info(" FAILED : {0}".format(TEST_PROCESS_STATS.cFailedTests)) + logging.info(" XFAILED : {0}".format(TEST_PROCESS_STATS.cXFailedTests)) + logging.info(" NOT XFAILED : {0}".format(TEST_PROCESS_STATS.cNotXFailedTests)) + logging.info(" SKIPPED : {0}".format(TEST_PROCESS_STATS.cSkippedTests)) + logging.info(" WITH WARNINGS: {0}".format(TEST_PROCESS_STATS.cWarningTests)) + logging.info(" UNEXPECTED : {0}".format(TEST_PROCESS_STATS.cUnexpectedTests)) + logging.info("") + + assert type(TEST_PROCESS_STATS.cTotalDuration) is datetime.timedelta + + LOCAL__print_line1_with_header("TIME") + logging.info("") + logging.info( + " TOTAL DURATION: {0}".format( + timedelta_to_human_text(TEST_PROCESS_STATS.cTotalDuration) + ) + ) + logging.info("") + + LOCAL__print_line1_with_header("TOTAL INFORMATION") + logging.info("") + logging.info(" TOTAL ERROR COUNT : {0}".format(TEST_PROCESS_STATS.cTotalErrors)) + logging.info(" TOTAL WARNING COUNT: {0}".format(TEST_PROCESS_STATS.cTotalWarnings)) + logging.info("") + + +# ///////////////////////////////////////////////////////////////////////////// + + +def helper__detect_test_process_kind(config: pytest.Config) -> T_TEST_PROCESS_KIND: + assert isinstance(config, pytest.Config) + + # + # xdist' master process registers DSession plugin. + # + p = config.pluginmanager.get_plugin("dsession") + + if p is not None: + return T_TEST_PROCESS_KIND.Master + + return T_TEST_PROCESS_KIND.Worker + + +# ------------------------------------------------------------------------ +def helper__detect_test_process_mode(config: pytest.Config) -> T_TEST_PROCESS_MODE: + assert isinstance(config, pytest.Config) + + if config.getvalue("collectonly"): + return T_TEST_PROCESS_MODE.Collect + + return T_TEST_PROCESS_MODE.ExecTests + + +# ------------------------------------------------------------------------ +@pytest.hookimpl(trylast=True) +def helper__pytest_configure__logging(config: pytest.Config) -> None: + assert isinstance(config, pytest.Config) + + log_name = TestStartupData.GetCurrentTestWorkerSignature() + log_name += ".log" + + log_dir = TestStartupData.GetRootLogDir() + + pathlib.Path(log_dir).mkdir(exist_ok=True) + + logging_plugin = config.pluginmanager.get_plugin("logging-plugin") + + assert logging_plugin is not None + assert isinstance(logging_plugin, _pytest.logging.LoggingPlugin) + + log_file_path = os.path.join(log_dir, log_name) + assert log_file_path is not None + assert type(log_file_path) is str + + logging_plugin.set_log_path(log_file_path) + return + + +# ------------------------------------------------------------------------ +@pytest.hookimpl(trylast=True) +def pytest_configure(config: pytest.Config) -> None: + assert isinstance(config, pytest.Config) + + global g_test_process_kind + global g_test_process_mode + global g_worker_log_is_created + + assert g_test_process_kind is None + assert g_test_process_mode is None + assert g_worker_log_is_created is None + + g_test_process_mode = helper__detect_test_process_mode(config) + g_test_process_kind = helper__detect_test_process_kind(config) + + assert type(g_test_process_kind) is T_TEST_PROCESS_KIND + assert type(g_test_process_mode) is T_TEST_PROCESS_MODE + + if g_test_process_kind == T_TEST_PROCESS_KIND.Master: + pass + else: + assert g_test_process_kind == T_TEST_PROCESS_KIND.Worker + + if g_test_process_mode == T_TEST_PROCESS_MODE.Collect: + g_worker_log_is_created = False + else: + assert g_test_process_mode == T_TEST_PROCESS_MODE.ExecTests + helper__pytest_configure__logging(config) + g_worker_log_is_created = True + + return + + +# ///////////////////////////////////////////////////////////////////////////// diff --git a/tests/helpers/__init__.py b/tests/helpers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/helpers/global_data.py b/tests/helpers/global_data.py new file mode 100644 index 00000000..c98ae9b4 --- /dev/null +++ b/tests/helpers/global_data.py @@ -0,0 +1,92 @@ +from testgres.operations.os_ops import OsOperations +from testgres.operations.os_ops import ConnectionParams +from testgres.operations.local_ops import LocalOperations +from testgres.operations.remote_ops import RemoteOperations + +from src.node import PortManager +from src.node import PortManager__ThisHost +from src.node import PortManager__Generic2 + +import os +import typing + + +class OsOpsDescr: + sign: str + os_ops: OsOperations + + def __init__(self, sign: str, os_ops: OsOperations): + assert type(sign) is str + assert isinstance(os_ops, OsOperations) + self.sign = sign + self.os_ops = os_ops + + +def _to_int_or_none(v: typing.Any) -> typing.Optional[int]: + return v if v is None else int(v) + + +class OsOpsDescrs: + sm_remote_conn_params = ConnectionParams( + host=os.getenv('TEST_CFG__REMOTE_HOST', '127.0.0.1'), + port=_to_int_or_none(os.getenv('TEST_CFG__REMOTE_PORT')), + username=os.getenv('TEST_CFG__REMOTE_USERNAME'), + ssh_key=os.getenv('TEST_CFG__REMOTE_SSH_KEY'), + password=os.getenv('TEST_CFG__REMOTE_PASSWORD'), + ) + + sm_remote_os_ops = RemoteOperations(sm_remote_conn_params) + + sm_remote_os_ops_descr = OsOpsDescr("remote_ops", sm_remote_os_ops) + + sm_local_os_ops = LocalOperations.get_single_instance() + + sm_local_os_ops_descr = OsOpsDescr("local_ops", sm_local_os_ops) + + +class PortManagers: + sm_remote_port_manager = PortManager__Generic2(OsOpsDescrs.sm_remote_os_ops) + + sm_local_port_manager = PortManager__ThisHost.get_single_instance() + + sm_local2_port_manager = PortManager__Generic2(OsOpsDescrs.sm_local_os_ops) + + +class PostgresNodeService: + sign: str + os_ops: OsOperations + port_manager: PortManager + + def __init__(self, sign: str, os_ops: OsOperations, port_manager: PortManager): + assert type(sign) is str + assert isinstance(os_ops, OsOperations) + assert isinstance(port_manager, PortManager) + self.sign = sign + self.os_ops = os_ops + self.port_manager = port_manager + + +class PostgresNodeServices: + sm_remote = PostgresNodeService( + "remote", + OsOpsDescrs.sm_remote_os_ops, + PortManagers.sm_remote_port_manager + ) + + sm_local = PostgresNodeService( + "local", + OsOpsDescrs.sm_local_os_ops, + PortManagers.sm_local_port_manager + ) + + sm_local2 = PostgresNodeService( + "local2", + OsOpsDescrs.sm_local_os_ops, + PortManagers.sm_local2_port_manager + ) + + sm_locals_and_remotes = [ + sm_local, + sm_local2, + sm_remote, + ] diff --git a/tests/helpers/local_check.py b/tests/helpers/local_check.py new file mode 100755 index 00000000..b704b339 --- /dev/null +++ b/tests/helpers/local_check.py @@ -0,0 +1,166 @@ +# coding: utf-8 +from .os_ops_helpers import OsOpsHelpers +from .os_ops_helpers import OsOperations + +import os + + +class LocalCheck: + @staticmethod + def check_path_exists( + os_ops: OsOperations, + path: str, + ) -> None: + assert isinstance(os_ops, OsOperations) + assert type(path) is str + + if not OsOpsHelpers.is_localhost(os_ops): + return + + if os.path.exists(path): + return + + err_msg = "[LocalCheck] Local path [{}] does not exist.".format( + path, + ) + raise RuntimeError(err_msg) + + # -------------------------------------------------------------------- + @staticmethod + def check_path_does_not_exists( + os_ops: OsOperations, + path: str, + ) -> None: + assert isinstance(os_ops, OsOperations) + assert type(path) is str + + if not OsOpsHelpers.is_localhost(os_ops): + return + + if not os.path.exists(path): + return + + err_msg = "[LocalCheck] Local path [{}] exists.".format( + path, + ) + raise RuntimeError(err_msg) + + # -------------------------------------------------------------------- + @staticmethod + def check_isdir( + os_ops: OsOperations, + path: str, + ) -> None: + assert isinstance(os_ops, OsOperations) + assert type(path) is str + + if not OsOpsHelpers.is_localhost(os_ops): + return + + if os.path.isdir(path): + return + + err_msg = "[LocalCheck] Local path [{}] is not dir.".format( + path, + ) + raise RuntimeError(err_msg) + + # -------------------------------------------------------------------- + @staticmethod + def check_not_isdir( + os_ops: OsOperations, + path: str, + ) -> None: + assert isinstance(os_ops, OsOperations) + assert type(path) is str + + if not OsOpsHelpers.is_localhost(os_ops): + return + + if not os.path.isdir(path): + return + + err_msg = "[LocalCheck] Local path [{}] is dir.".format( + path, + ) + raise RuntimeError(err_msg) + + # -------------------------------------------------------------------- + @staticmethod + def check_isfile( + os_ops: OsOperations, + path: str, + ) -> None: + assert isinstance(os_ops, OsOperations) + assert type(path) is str + + if not OsOpsHelpers.is_localhost(os_ops): + return + + if os.path.isfile(path): + return + + err_msg = "[LocalCheck] Local path [{}] is not file.".format( + path, + ) + raise RuntimeError(err_msg) + + # -------------------------------------------------------------------- + @staticmethod + def check_not_isfile( + os_ops: OsOperations, + path: str, + ) -> None: + assert isinstance(os_ops, OsOperations) + assert type(path) is str + + if not OsOpsHelpers.is_localhost(os_ops): + return + + if not os.path.isfile(path): + return + + err_msg = "[LocalCheck] Local path [{}] is file.".format( + path, + ) + raise RuntimeError(err_msg) + + # -------------------------------------------------------------------- + @staticmethod + def check_path_is_abs( + os_ops: OsOperations, + path: str, + ) -> None: + assert isinstance(os_ops, OsOperations) + assert type(path) is str + + if not OsOpsHelpers.is_localhost(os_ops): + return + + if os.path.isabs(path): + return + + err_msg = "[LocalCheck] Local path [{}] is not abs.".format( + path, + ) + raise RuntimeError(err_msg) + + # -------------------------------------------------------------------- + @staticmethod + def check_path_is_not_abs( + os_ops: OsOperations, + path: str, + ) -> None: + assert isinstance(os_ops, OsOperations) + assert type(path) is str + + if not OsOpsHelpers.is_localhost(os_ops): + return + + if not os.path.isabs(path): + return + + err_msg = "[LocalCheck] Local path [{}] is abs.".format( + path, + ) + raise RuntimeError(err_msg) diff --git a/tests/helpers/os_ops_helpers.py b/tests/helpers/os_ops_helpers.py new file mode 100755 index 00000000..56337674 --- /dev/null +++ b/tests/helpers/os_ops_helpers.py @@ -0,0 +1,18 @@ +# coding: utf-8 +from testgres.operations.local_ops import OsOperations + + +class OsOpsHelpers: + @staticmethod + def is_localhost(os_ops: OsOperations) -> bool: + assert isinstance(os_ops, OsOperations) + + host = os_ops.host + + if host == "127.0.0.1": + return True + + if host == "localhost": + return True + + return False diff --git a/tests/helpers/pg_cfg_os_ops.py b/tests/helpers/pg_cfg_os_ops.py new file mode 100644 index 00000000..543c45ad --- /dev/null +++ b/tests/helpers/pg_cfg_os_ops.py @@ -0,0 +1,221 @@ +from __future__ import annotations + +from testgres.postgres_configuration.os.abstract import configuration_os_ops as abs_pg_cfg_os_ops +from testgres.operations.os_ops import OsOperations + +import datetime +import typing + + +class PgCfgOsFile(abs_pg_cfg_os_ops.ConfigurationOsFile): + class tagData: + os_ops: OsOperations + file_path: str + encoding: str + next_line: int + file_lines: typing.Optional[typing.List[str]] + + def __init__( + self, + os_ops: OsOperations, + file_path: str, + encoding: str, + ): + assert isinstance(os_ops, OsOperations) + assert type(file_path) is str + assert type(encoding) is str + + self.os_ops = os_ops + self.file_path = file_path + self.encoding = encoding + self.file_lines = None + self.next_line = 0 + self.file_lines = None + return + + # -------------------------------------------------------------------- + _data: typing.Optional[tagData] + + # -------------------------------------------------------------------- + def __init__( + self, + os_ops: OsOperations, + file_path: str, + encoding: str, + ): + assert isinstance(os_ops, OsOperations) + assert type(file_path) is str + + super().__init__() + + self._data = __class__.tagData( + os_ops, + file_path, + encoding, + ) + return + + # -------------------------------------------------------------------- + def __enter__(self) -> PgCfgOsFile: + assert isinstance(self._data, __class__.tagData) + return self + + # -------------------------------------------------------------------- + def __exit__(self, exc_type, exc_val, exc_tb) -> bool: + if self._data is not None: + self.Close() + + assert self._data is None + return False + + # -------------------------------------------------------------------- + @property + def Name(self) -> str: + assert type(self._data) is __class__.tagData + assert type(self._data.file_path) is str + return self._data.file_path + + # -------------------------------------------------------------------- + @property + def IsClosed(self) -> bool: + return self._data is None + + # -------------------------------------------------------------------- + def ReadLine(self) -> typing.Optional[str]: + assert type(self._data) is __class__.tagData + assert isinstance(self._data.os_ops, OsOperations) + + if self._data.file_lines is None: + assert self._data.next_line == 0 + content = self._data.os_ops.read( + self._data.file_path, + encoding=self._data.encoding, + binary=False, + ) + assert type(content) is str + self._data.file_lines = content.splitlines(keepends=True) + + assert type(self._data.file_lines) is list + + assert self._data.next_line <= len(self._data.file_lines) + + if self._data.next_line == len(self._data.file_lines): + return None + + r = self._data.file_lines[self._data.next_line] + assert type(r) is str + + self._data.next_line = self._data.next_line + 1 + return r + + # -------------------------------------------------------------------- + def Overwrite(self, text: str) -> None: + assert type(self._data) is __class__.tagData + assert isinstance(self._data.os_ops, OsOperations) + + self._data.os_ops.write( + self._data.file_path, + text, + truncate=True, + encoding=self._data.encoding, + binary=False, + ) + return + + # -------------------------------------------------------------------- + def Close(self) -> None: + assert type(self._data) is __class__.tagData + self._data = None + return + + # -------------------------------------------------------------------- + def GetModificationTS(self) -> datetime.datetime: + assert type(self._data) is __class__.tagData + assert isinstance(self._data.os_ops, OsOperations) + + stat = self._data.os_ops.get_file_stat(self._data.file_path) + assert stat is not None + assert type(stat) is dict + assert OsOperations.C_FILE_STAT_PROP__MTIME in stat + + mtime = stat[OsOperations.C_FILE_STAT_PROP__MTIME] + assert mtime is not None + assert type(mtime) is datetime.datetime + return mtime + + +class PgCfgOsOps(abs_pg_cfg_os_ops.ConfigurationOsOps): + _os_ops: OsOperations + _file_encoding: str + + def __init__( + self, + os_ops: OsOperations, + file_encoding: str, + ): + assert isinstance(os_ops, OsOperations) + assert type(file_encoding) is str + self._os_ops = os_ops + self._file_encoding = file_encoding + return + + def Path_IsAbs(self, a: str) -> bool: + assert type(a) is str + assert isinstance(self._os_ops, OsOperations) + return self._os_ops.is_abs_path(a) + + def Path_Join(self, a: str, *p: str) -> str: + assert type(a) is str + assert type(p) is tuple + assert isinstance(self._os_ops, OsOperations) + return self._os_ops.build_path(a, *p) + + def Path_NormPath(self, a: str) -> str: + assert type(a) is str + assert isinstance(self._os_ops, OsOperations) + return self._os_ops.get_path_normpath(a) + + def Path_AbsPath(self, a: str) -> str: + assert type(a) is str + assert isinstance(self._os_ops, OsOperations) + return self._os_ops.get_abs_path(a) + + def Path_NormCase(self, a: str) -> str: + assert type(a) is str + assert isinstance(self._os_ops, OsOperations) + return self._os_ops.get_path_normcase(a) + + def Path_DirName(self, a: str) -> str: + assert type(a) is str + assert isinstance(self._os_ops, OsOperations) + return self._os_ops.get_dirname(a) + + def Path_BaseName(self, a: str) -> str: + assert type(a) is str + assert isinstance(self._os_ops, OsOperations) + return self._os_ops.get_path_basename(a) + + def Remove(self, a: str) -> None: + assert type(a) is str + assert isinstance(self._os_ops, OsOperations) + self._os_ops.remove_file(a) + return + + def OpenFileToRead(self, filePath: str) -> abs_pg_cfg_os_ops.ConfigurationOsFile: + assert type(filePath) is str + assert isinstance(self._os_ops, OsOperations) + f = PgCfgOsFile(self._os_ops, filePath, self._file_encoding) + return f + + def OpenFileToWrite(self, filePath: str) -> abs_pg_cfg_os_ops.ConfigurationOsFile: + assert type(filePath) is str + assert isinstance(self._os_ops, OsOperations) + f = PgCfgOsFile(self._os_ops, filePath, self._file_encoding) + return f + + def CreateFile(self, filePath: str) -> abs_pg_cfg_os_ops.ConfigurationOsFile: + assert type(filePath) is str + assert isinstance(self._os_ops, OsOperations) + f = PgCfgOsFile(self._os_ops, filePath, self._file_encoding) + self._os_ops.create_file(filePath) + return f diff --git a/tests/helpers/pg_node_utils.py b/tests/helpers/pg_node_utils.py new file mode 100644 index 00000000..05811735 --- /dev/null +++ b/tests/helpers/pg_node_utils.py @@ -0,0 +1,189 @@ +from src import PostgresNode +from src import PortManager +from src import OsOperations +from src import NodeStatus +from src.node import PostgresNodeLogReader + +from tests.helpers.utils import Utils as HelperUtils +from tests.helpers.utils import T_WAIT_TIME + +from tests.helpers.global_data import PostgresNodeService + +import typing + + +class PostgresNodeUtils: + class PostgresNodeUtilsException(Exception): + pass + + class PortConflictNodeException(PostgresNodeUtilsException): + _data_dir: str + _port: int + + def __init__(self, data_dir: str, port: int): + assert type(data_dir) is str + assert type(port) is int + + super().__init__() + + self._data_dir = data_dir + self._port = port + return + + @property + def data_dir(self) -> str: + assert type(self._data_dir) is str + return self._data_dir + + @property + def port(self) -> int: + assert type(self._port) is int + return self._port + + @property + def message(self) -> str: + assert type(self._data_dir) is str + assert type(self._port) is int + + r = "PostgresNode [data:{}][port: {}] conflicts with port of another instance.".format( + self._data_dir, + self._port, + ) + assert type(r) is str + return r + + def __str__(self) -> str: + r = self.message + assert type(r) is str + return r + + def __repr__(self) -> str: + # It must be overrided! + assert type(self) is __class__ + r = "{}({}, {})".format( + __class__.__name__, + repr(self._data_dir), + repr(self._port), + ) + assert type(r) is str + return r + + # -------------------------------------------------------------------- + class StartNodeException(PostgresNodeUtilsException): + _data_dir: str + _files: typing.Optional[typing.Iterable] + + def __init__( + self, + data_dir: str, + files: typing.Optional[typing.Iterable] = None + ): + assert type(data_dir) is str + assert files is None or isinstance(files, typing.Iterable) + + super().__init__() + + self._data_dir = data_dir + self._files = files + return + + @property + def message(self) -> str: + assert self._data_dir is None or type(self._data_dir) is str + assert self._files is None or isinstance(self._files, typing.Iterable) + + msg_parts = [] + + msg_parts.append("PostgresNode [data_dir: {}] is not started.".format( + self._data_dir + )) + + for f, lines in self._files or []: + assert type(f) is str + assert type(lines) in [str, bytes] + msg_parts.append(u'{}\n----\n{}\n'.format(f, lines)) + + return "\n".join(msg_parts) + + @property + def data_dir(self) -> typing.Optional[str]: + assert type(self._data_dir) is str + return self._data_dir + + @property + def files(self) -> typing.Optional[typing.Iterable]: + assert self._files is None or isinstance(self._files, typing.Iterable) + return self._files + + def __repr__(self) -> str: + assert type(self._data_dir) is str + assert self._files is None or isinstance(self._files, typing.Iterable) + + r = "{}({}, {})".format( + __class__.__name__, + repr(self._data_dir), + repr(self._files), + ) + assert type(r) is str + return r + + # -------------------------------------------------------------------- + @staticmethod + def get_node( + node_svc: PostgresNodeService, + name: typing.Optional[str] = None, + port: typing.Optional[int] = None, + port_manager: typing.Optional[PortManager] = None + ) -> PostgresNode: + assert isinstance(node_svc, PostgresNodeService) + assert isinstance(node_svc.os_ops, OsOperations) + assert isinstance(node_svc.port_manager, PortManager) + + if port_manager is None: + port_manager = node_svc.port_manager + + return PostgresNode( + name, + port=port, + os_ops=node_svc.os_ops, + port_manager=port_manager if port is None else None + ) + + # -------------------------------------------------------------------- + @staticmethod + def wait_for_running_state( + node: PostgresNode, + node_log_reader: PostgresNodeLogReader, + timeout: T_WAIT_TIME, + ): + assert type(node) is PostgresNode + assert type(node_log_reader) is PostgresNodeLogReader + assert type(timeout) in [int, float] + assert node_log_reader._node is node + assert timeout > 0 + + for _ in HelperUtils.WaitUntil( + timeout=timeout + ): + s = node.status() + + if s == NodeStatus.Running: + return + + assert s == NodeStatus.Stopped + + blocks = node_log_reader.read() + assert type(blocks) is list + + for block in blocks: + assert type(block) is PostgresNodeLogReader.LogDataBlock + + if 'Is another postmaster already running on port' in block.data: + raise __class__.PortConflictNodeException(node.data_dir, node.port) + + if 'database system is shut down' in block.data: + raise __class__.StartNodeException( + node.data_dir, + node._collect_special_files(), + ) + continue diff --git a/tests/helpers/run_conditions.py b/tests/helpers/run_conditions.py new file mode 100644 index 00000000..f847d879 --- /dev/null +++ b/tests/helpers/run_conditions.py @@ -0,0 +1,13 @@ +# coding: utf-8 +import pytest +import platform + + +class RunConditions: + # It is not a test kit! + __test__ = False + + @staticmethod + def skip_if_windows(): + if platform.system().lower() == "windows": + pytest.skip("This test does not support Windows.") diff --git a/tests/helpers/utils.py b/tests/helpers/utils.py new file mode 100644 index 00000000..50badf01 --- /dev/null +++ b/tests/helpers/utils.py @@ -0,0 +1,65 @@ +import typing +import time +import logging + + +T_WAIT_TIME = typing.Union[int, float] + + +class Utils: + @staticmethod + def PrintAndSleep(wait: T_WAIT_TIME): + assert type(wait) in [int, float] + logging.info("Wait for {} second(s)".format(wait)) + time.sleep(wait) + return + + @staticmethod + def WaitUntil( + error_message: str = "Did not complete", + timeout: T_WAIT_TIME = 30, + interval: T_WAIT_TIME = 1, + notification_interval: T_WAIT_TIME = 5, + ): + """ + Loop until the timeout is reached. If the timeout is reached, raise an + exception with the given error message. + + Source of idea: pgbouncer + """ + assert type(timeout) in [int, float] + assert type(interval) in [int, float] + assert type(notification_interval) in [int, float] + assert timeout >= 0 + assert interval >= 0 + assert notification_interval >= 0 + + start_ts = time.monotonic() + end_ts = start_ts + timeout + last_printed_progress = start_ts + last_iteration_ts = start_ts + + yield + attempt = 1 + + while end_ts > time.monotonic(): + if (timeout > 5 and time.monotonic() - last_printed_progress) > notification_interval: + last_printed_progress = time.monotonic() + + m = "{} in {} seconds and {} attempts - will retry".format( + error_message, + time.monotonic() - start_ts, + attempt, + ) + logging.info(m) + + interval_remaining = last_iteration_ts + interval - time.monotonic() + if interval_remaining > 0: + time.sleep(interval_remaining) + + last_iteration_ts = time.monotonic() + yield + attempt += 1 + continue + + raise TimeoutError(error_message + " in time") diff --git a/tests/requirements.txt b/tests/requirements.txt new file mode 100644 index 00000000..c4adb2c6 --- /dev/null +++ b/tests/requirements.txt @@ -0,0 +1,8 @@ +psutil +pytest +pytest-env +pytest-xdist +psycopg2 +six +testgres.os_ops>=3.2.0,<4.0.0 +testgres.postgres_configuration>=0.2.2,<1.0.0 diff --git a/tests/test_api.py b/tests/test_api.py new file mode 100644 index 00000000..af2813e7 --- /dev/null +++ b/tests/test_api.py @@ -0,0 +1,30 @@ +from src import api as testgres_api +from src.node import PostgresNode + +from tests.helpers.global_data import OsOpsDescrs + + +class TestAPI: + def test_001__get_new_node(self): + C_NODE_NAME = "abc" + + with testgres_api.get_new_node(name=C_NODE_NAME) as node: + assert type(node) is PostgresNode + assert node.name == C_NODE_NAME + node.init() + node.slow_start() + node.stop() + return + + def test_001__get_remote_node(self): + C_NODE_NAME = "abc" + + conn_params = OsOpsDescrs.sm_remote_conn_params + + with testgres_api.get_remote_node(name=C_NODE_NAME, conn_params=conn_params) as node: + assert type(node) is PostgresNode + assert node.name == C_NODE_NAME + node.init() + node.slow_start() + node.stop() + return diff --git a/tests/test_config.py b/tests/test_config.py new file mode 100644 index 00000000..3969b2c2 --- /dev/null +++ b/tests/test_config.py @@ -0,0 +1,41 @@ +from src import TestgresConfig +from src import configure_testgres +from src import scoped_config +from src import pop_config + +import src as testgres + +import pytest + + +class TestConfig: + def test_config_stack(self): + # no such option + with pytest.raises(expected_exception=TypeError): + configure_testgres(dummy=True) + + # we have only 1 config in stack + with pytest.raises(expected_exception=IndexError): + pop_config() + + d0 = TestgresConfig.cached_initdb_dir + d1 = 'dummy_abc' + d2 = 'dummy_def' + + with scoped_config(cached_initdb_dir=d1) as c1: + assert (c1.cached_initdb_dir == d1) + + with scoped_config(cached_initdb_dir=d2) as c2: + stack_size = len(testgres.config.config_stack) + + # try to break a stack + with pytest.raises(expected_exception=TypeError): + with scoped_config(dummy=True): + pass + + assert (c2.cached_initdb_dir == d2) + assert (len(testgres.config.config_stack) == stack_size) + + assert (c1.cached_initdb_dir == d1) + + assert (TestgresConfig.cached_initdb_dir == d0) diff --git a/tests/test_conftest.py--devel b/tests/test_conftest.py--devel new file mode 100644 index 00000000..67c1dafe --- /dev/null +++ b/tests/test_conftest.py--devel @@ -0,0 +1,80 @@ +import pytest +import logging + + +class TestConfest: + def test_failed(self): + raise Exception("TEST EXCEPTION!") + + def test_ok(self): + pass + + @pytest.mark.skip() + def test_mark_skip__no_reason(self): + pass + + @pytest.mark.xfail() + def test_mark_xfail__no_reason(self): + raise Exception("XFAIL EXCEPTION") + + @pytest.mark.xfail() + def test_mark_xfail__no_reason___no_error(self): + pass + + @pytest.mark.skip(reason="reason") + def test_mark_skip__with_reason(self): + pass + + @pytest.mark.xfail(reason="reason") + def test_mark_xfail__with_reason(self): + raise Exception("XFAIL EXCEPTION") + + @pytest.mark.xfail(reason="reason") + def test_mark_xfail__with_reason___no_error(self): + pass + + def test_exc_skip__no_reason(self): + pytest.skip() + + def test_exc_xfail__no_reason(self): + pytest.xfail() + + def test_exc_skip__with_reason(self): + pytest.skip(reason="SKIP REASON") + + def test_exc_xfail__with_reason(self): + pytest.xfail(reason="XFAIL EXCEPTION") + + def test_log_error(self): + logging.error("IT IS A LOG ERROR!") + + def test_log_error_and_exc(self): + logging.error("IT IS A LOG ERROR!") + + raise Exception("TEST EXCEPTION!") + + def test_log_error_and_warning(self): + logging.error("IT IS A LOG ERROR!") + logging.warning("IT IS A LOG WARNING!") + logging.error("IT IS THE SECOND LOG ERROR!") + logging.warning("IT IS THE SECOND LOG WARNING!") + + @pytest.mark.xfail() + def test_log_error_and_xfail_mark_without_reason(self): + logging.error("IT IS A LOG ERROR!") + + @pytest.mark.xfail(reason="It is a reason message") + def test_log_error_and_xfail_mark_with_reason(self): + logging.error("IT IS A LOG ERROR!") + + @pytest.mark.xfail() + def test_two_log_error_and_xfail_mark_without_reason(self): + logging.error("IT IS THE FIRST LOG ERROR!") + logging.info("----------") + logging.error("IT IS THE SECOND LOG ERROR!") + + @pytest.mark.xfail(reason="It is a reason message") + def test_two_log_error_and_xfail_mark_with_reason(self): + logging.error("IT IS THE FIRST LOG ERROR!") + logging.info("----------") + logging.error("IT IS THE SECOND LOG ERROR!") diff --git a/tests/test_os_ops_common.py b/tests/test_os_ops_common.py new file mode 100644 index 00000000..67c88566 --- /dev/null +++ b/tests/test_os_ops_common.py @@ -0,0 +1,3921 @@ +# coding: utf-8 +from __future__ import annotations + +from .helpers.global_data import OsOpsDescr +from .helpers.global_data import OsOpsDescrs +from .helpers.global_data import OsOperations +from .helpers.run_conditions import RunConditions +from .helpers.local_check import LocalCheck +from .helpers.local_check import OsOpsHelpers + +import os +import sys + +import pytest +import re +import logging +import typing +import uuid +import subprocess +import psutil +import time +import signal as os_signal +import dataclasses +import random +import datetime +import threading +import queue + +from src import InvalidOperationException +from src import ExecUtilException + +from concurrent.futures import ThreadPoolExecutor +from concurrent.futures import Future as ThreadFuture + + +class TestOsOpsCommon: + sm_os_ops_descrs: typing.List[OsOpsDescr] = [ + OsOpsDescrs.sm_local_os_ops_descr, + OsOpsDescrs.sm_remote_os_ops_descr + ] + + @pytest.fixture( + params=[ + pytest.param( + descr, + id=descr.sign, + ) + for descr in sm_os_ops_descrs + ], + ) + def os_ops_descr(self, request: pytest.FixtureRequest) -> OsOpsDescr: + assert isinstance(request, pytest.FixtureRequest) + assert isinstance(request.param, OsOpsDescr) + return request.param + + @dataclasses.dataclass + class tagNameWithSurprize: + sign: str + value: str + + sm_names_with_surprize: typing.List[tagNameWithSurprize] = [ + tagNameWithSurprize( + sign="std", + value="exclusive_new_file.txt", + ), + tagNameWithSurprize( + sign="with_one_double_quote", + value="exclusive_new_file\".txt", + ), + tagNameWithSurprize( + sign="with_two_double_quote", + value="\"exclusive_new_file\".txt", + ), + tagNameWithSurprize( + sign="with_one_single_quote", + value="exclusive_new_file\'.txt", + ), + tagNameWithSurprize( + sign="with_two_single_quote", + value="\'exclusive_new_file\'.txt", + ), + tagNameWithSurprize( + sign="with_single_quote_and_double_quote", + value="\'exclusive_new_file\".txt", + ), + tagNameWithSurprize( + sign="with_double_quote_and_single_quote", + value="\"exclusive_new_file\'.txt", + ), + ] + + @pytest.fixture( + params=[ + pytest.param( + x, + id=x.sign, + ) + for x in sm_names_with_surprize + ] + ) + def name_with_surprize(self, request: pytest.FixtureRequest) -> tagNameWithSurprize: + assert isinstance(request, pytest.FixtureRequest) + assert type(request.param).__name__ == "tagNameWithSurprize" + return request.param + + sm_false_true: typing.List[bool] = [False, True] + + @pytest.fixture( + params=[ + pytest.param( + x, + id="test_clone" if x else "test_orig", + ) + for x in sm_false_true + ] + ) + def use_clone(self, request: pytest.FixtureRequest) -> bool: + assert isinstance(request, pytest.FixtureRequest) + assert type(request.param) is bool + return request.param + + def test_prop__remote(self, os_ops_descr: OsOpsDescr): + assert type(os_ops_descr) is OsOpsDescr + assert isinstance(os_ops_descr.os_ops, OsOperations) + + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + v = os_ops.remote + assert v is not None or type(v) is bool + + if type(os_ops).__name__ == "RemoteOperations": + assert v is True + elif type(os_ops).__name__ == "LocalOperations": + assert v is False + else: + raise RuntimeError("[BUG CHECK] Unknown os_ops type: {}.".format( + type(os_ops).__name__, + )) + return + + def test_prop__host(self, os_ops_descr: OsOpsDescr): + assert type(os_ops_descr) is OsOpsDescr + assert isinstance(os_ops_descr.os_ops, OsOperations) + + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + v = os_ops.host + assert v is not None or type(v) is str + return + + def test_prop__port(self, os_ops_descr: OsOpsDescr): + assert type(os_ops_descr) is OsOpsDescr + assert isinstance(os_ops_descr.os_ops, OsOperations) + + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + v = os_ops.port + assert v is None or type(v) is int + return + + def test_prop__ssh_key(self, os_ops_descr: OsOpsDescr): + assert type(os_ops_descr) is OsOpsDescr + assert isinstance(os_ops_descr.os_ops, OsOperations) + + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + v = os_ops.ssh_key + assert v is None or type(v) is str + return + + def test_prop__username(self, os_ops_descr: OsOpsDescr): + assert type(os_ops_descr) is OsOpsDescr + assert isinstance(os_ops_descr.os_ops, OsOperations) + + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + v = os_ops.username + assert v is None or type(v) is str + return + + def test_get_platform(self, os_ops_descr: OsOpsDescr): + assert type(os_ops_descr) is OsOpsDescr + assert isinstance(os_ops_descr.os_ops, OsOperations) + + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + p = os_ops.get_platform() + assert p is not None + assert type(p) is str + assert p == sys.platform + return + + def test_get_platform__is_known(self, os_ops_descr: OsOpsDescr): + assert type(os_ops_descr) is OsOpsDescr + assert isinstance(os_ops_descr.os_ops, OsOperations) + + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + p = os_ops.get_platform() + assert p is not None + assert type(p) is str + assert p in {"win32", "linux"} + return + + def test_create_clone( + self, + os_ops_descr: OsOpsDescr, + use_clone: bool, + ): + assert type(os_ops_descr) is OsOpsDescr + assert type(use_clone) is bool + + os_ops = __class__.helper__get_os_ops(use_clone, os_ops_descr) + assert isinstance(os_ops, OsOperations) + + env1_name = "env1_" + uuid.uuid4().bytes.hex() + env1_val = "abc" + env2_name = "env1_" + uuid.uuid4().bytes.hex() + + os_ops.set_env(env1_name, env1_val) + + try: + clone = os_ops.create_clone() + assert clone is not None + assert clone is not os_ops + assert type(clone) is type(os_ops) + + assert clone.remote == os_ops.remote + assert clone.username == os_ops.username + assert clone.ssh_key == os_ops.ssh_key + assert clone.host == os_ops.host + assert clone.port == os_ops.port + + assert clone.get_name() == os_ops.get_name() + assert clone.get_platform() == os_ops.get_platform() + + assert clone.environ("PATH") == os_ops.environ("PATH") + + v1_orig = os_ops.environ(env1_name) + v1_clone = clone.environ(env1_name) + assert v1_orig == env1_val + assert v1_orig == v1_clone + + assert clone.environ(env2_name) == os_ops.environ(env2_name) + finally: + os_ops.reset_env(env1_name, None) + + return + + def test_exec_command_success(self, os_ops_descr: OsOpsDescr): + """ + Test exec_command for successful command execution. + """ + assert type(os_ops_descr) is OsOpsDescr + assert isinstance(os_ops_descr.os_ops, OsOperations) + + RunConditions.skip_if_windows() + + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + cmd = ["sh", "-c", "python3 --version"] + + response = os_ops.exec_command(cmd) + assert type(response) is bytes + assert b'Python 3.' in response + return + + def test_exec_command_failure(self, os_ops_descr: OsOpsDescr): + """ + Test exec_command for command execution failure. + """ + assert type(os_ops_descr) is OsOpsDescr + assert isinstance(os_ops_descr.os_ops, OsOperations) + + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + RunConditions.skip_if_windows() + + cmd = ["sh", "-c", "nonexistent_command"] + + while True: + try: + os_ops.exec_command(cmd) + except ExecUtilException as e: + assert type(e.exit_code) is int + assert e.exit_code == 127 + + assert type(e.message) is str + assert type(e.error) is bytes + + assert e.message.startswith("Utility exited with non-zero code (127). Error:") + assert "nonexistent_command" in e.message + assert "not found" in e.message + assert b"nonexistent_command" in e.error + assert b"not found" in e.error + break + raise Exception("We wait an exception!") + return + + def test_exec_command_failure__expect_error(self, os_ops_descr: OsOpsDescr): + """ + Test exec_command for command execution failure. + """ + assert type(os_ops_descr) is OsOpsDescr + assert isinstance(os_ops_descr.os_ops, OsOperations) + + RunConditions.skip_if_windows() + + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + cmd = ["sh", "-c", "nonexistent_command"] + + exec_r = os_ops.exec_command(cmd, verbose=True, expect_error=True) + assert type(exec_r) is tuple + assert len(exec_r) == 3 + + exit_status, result, error = exec_r + assert type(exit_status) is int + assert type(result) is bytes + assert type(error) is bytes + + assert exit_status == 127 + assert result == b'' + assert b"nonexistent_command" in error + assert b"not found" in error + return + + def test_exec_command_with_exec_env(self, os_ops_descr: OsOpsDescr): + assert type(os_ops_descr) is OsOpsDescr + assert isinstance(os_ops_descr.os_ops, OsOperations) + + RunConditions.skip_if_windows() + + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + C_ENV_NAME = "TESTGRES_TEST__EXEC_ENV_20250414" + + cmd = ["sh", "-c", "echo ${}".format(C_ENV_NAME)] + + exec_env = {C_ENV_NAME: "Hello!"} + + response = os_ops.exec_command(cmd, exec_env=exec_env) + assert response is not None + assert type(response) is bytes + assert response == b'Hello!\n' + + response = os_ops.exec_command(cmd) + assert response is not None + assert type(response) is bytes + assert response == b'\n' + return + + def test_exec_command_with_exec_env__2(self, os_ops_descr: OsOpsDescr): + assert type(os_ops_descr) is OsOpsDescr + assert isinstance(os_ops_descr.os_ops, OsOperations) + + RunConditions.skip_if_windows() + + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + C_ENV_NAME = "TESTGRES_TEST__EXEC_ENV_20250414" + + tmp_file_content = "echo ${{{}}}".format(C_ENV_NAME) + + logging.info("content is [{}]".format(tmp_file_content)) + + tmp_file = os_ops.mkstemp() + assert type(tmp_file) is str + assert tmp_file != "" + + logging.info("file is [{}]".format(tmp_file)) + assert os_ops.path_exists(tmp_file) + + os_ops.write(tmp_file, tmp_file_content) + + cmd = ["sh", tmp_file] + + exec_env = {C_ENV_NAME: "Hello!"} + + response = os_ops.exec_command(cmd, exec_env=exec_env) + assert response is not None + assert type(response) is bytes + assert response == b'Hello!\n' + + response = os_ops.exec_command(cmd) + assert response is not None + assert type(response) is bytes + assert response == b'\n' + + os_ops.remove_file(tmp_file) + assert not os_ops.path_exists(tmp_file) + return + + def test_exec_command_with_cwd(self, os_ops_descr: OsOpsDescr): + assert type(os_ops_descr) is OsOpsDescr + assert isinstance(os_ops_descr.os_ops, OsOperations) + + RunConditions.skip_if_windows() + + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + cmd = ["pwd"] + + response = os_ops.exec_command(cmd, cwd="/tmp") + assert response is not None + assert type(response) is bytes + assert response == b'/tmp\n' + + response = os_ops.exec_command(cmd) + assert response is not None + assert type(response) is bytes + assert response != b'/tmp\n' + return + + def test_exec_command__test_unset(self, os_ops_descr: OsOpsDescr): + assert type(os_ops_descr) is OsOpsDescr + assert isinstance(os_ops_descr.os_ops, OsOperations) + + RunConditions.skip_if_windows() + + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + C_ENV_NAME = "LANG" + + cmd = ["sh", "-c", "echo ${}".format(C_ENV_NAME)] + + response1 = os_ops.exec_command(cmd) + assert response1 is not None + assert type(response1) is bytes + + if response1 == b'\n': + logging.warning("Environment variable {} is not defined.".format(C_ENV_NAME)) + return + + exec_env = {C_ENV_NAME: None} + response2 = os_ops.exec_command(cmd, exec_env=exec_env) + assert response2 is not None + assert type(response2) is bytes + assert response2 == b'\n' + + response3 = os_ops.exec_command(cmd) + assert response3 is not None + assert type(response3) is bytes + assert response3 == response1 + return + + def test_exec_command__test_unset_dummy_var( + self, + os_ops_descr: OsOpsDescr, + ): + assert type(os_ops_descr) is OsOpsDescr + assert isinstance(os_ops_descr.os_ops, OsOperations) + + RunConditions.skip_if_windows() + + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + C_ENV_NAME = "TESTGRES_TEST__DUMMY_VAR_20250414" + + cmd = ["sh", "-c", "echo ${}".format(C_ENV_NAME)] + + exec_env = {C_ENV_NAME: None} + response2 = os_ops.exec_command(cmd, exec_env=exec_env) + assert response2 is not None + assert type(response2) is bytes + assert response2 == b'\n' + return + + def test_is_executable_true( + self, + os_ops_descr: OsOpsDescr, + ): + """ + Test is_executable for an existing executable. + """ + assert type(os_ops_descr) is OsOpsDescr + assert isinstance(os_ops_descr.os_ops, OsOperations) + + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + RunConditions.skip_if_windows() + + response = os_ops.is_executable("/bin/sh") + + assert response is True + return + + def test_is_executable_true_2( + self, + os_ops_descr: OsOpsDescr, + name_with_surprize: tagNameWithSurprize, + ): + """ + Test is_executable for an existing executable. + """ + assert type(os_ops_descr) is OsOpsDescr + assert isinstance(os_ops_descr.os_ops, OsOperations) + assert type(name_with_surprize) is __class__.tagNameWithSurprize + + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + RunConditions.skip_if_windows() + + assert os_ops.is_executable("/bin/sh") is True + + tmpdir = os_ops.mkdtemp(name_with_surprize.value) + + cmd = ["sh", "-c", "cp -p /bin/sh " + os_ops.quote_path(tmpdir)] + + os_ops.exec_command(cmd) + + target = os_ops.build_path(tmpdir, "sh") + + assert os_ops.path_exists(target) + + response = os_ops.is_executable(target) + assert response is True + + os_ops.remove_file(target) + os_ops.rmdir(tmpdir) + return + + def test_is_executable_false( + self, + os_ops_descr: OsOpsDescr, + name_with_surprize: tagNameWithSurprize, + ): + """ + Test is_executable for a non-executable. + """ + assert type(os_ops_descr) is OsOpsDescr + assert type(name_with_surprize) is __class__.tagNameWithSurprize + assert isinstance(os_ops_descr.os_ops, OsOperations) + + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + tmp_dir = os_ops.mkdtemp() + filename = os_ops.build_path(tmp_dir, name_with_surprize.value + ".no_exe") + + os_ops.touch(filename) + + response = os_ops.is_executable(filename) + assert response is False + + os_ops.remove_file(filename) + os_ops.rmdir(tmp_dir) + return + + def test_makedirs_and_rmdirs_success( + self, + os_ops_descr: OsOpsDescr, + name_with_surprize: tagNameWithSurprize, + ): + """ + Test makedirs and rmdirs for successful directory creation and removal. + """ + assert type(os_ops_descr) is OsOpsDescr + assert type(name_with_surprize) is __class__.tagNameWithSurprize + assert isinstance(os_ops_descr.os_ops, OsOperations) + + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + RunConditions.skip_if_windows() + + path = "/tmp/{}-{}".format( + name_with_surprize.value, + uuid.uuid4().bytes.hex() + ) + + # Test makedirs + os_ops.makedirs(path) + LocalCheck.check_path_exists(os_ops, path) + assert os_ops.path_exists(path) + + # Test rmdirs + os_ops.rmdirs(path) + LocalCheck.check_path_does_not_exists(os_ops, path) + assert not os_ops.path_exists(path) + return + + def test_makedirs_failure( + self, + os_ops_descr: OsOpsDescr, + name_with_surprize: tagNameWithSurprize, + ): + """ + Test makedirs for failure. + """ + # Try to create a directory in a read-only location + assert type(os_ops_descr) is OsOpsDescr + assert type(name_with_surprize) is __class__.tagNameWithSurprize + assert isinstance(os_ops_descr.os_ops, OsOperations) + + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + RunConditions.skip_if_windows() + + path = "/root/test_dir-{}-{}".format( + name_with_surprize.value, + uuid.uuid4().bytes.hex(), + ) + + # Test makedirs + with pytest.raises(Exception) as x: + os_ops.makedirs(path) + + if type(os_ops).__name__ == "LocalOperations": + assert type(x.value) is PermissionError + elif type(os_ops).__name__ == "RemoteOperations": + assert type(x.value) is ExecUtilException + else: + __class__.helper__bug_check__unknown_os_ops_type(os_ops) + return + + def test_listdir( + self, + os_ops_descr: OsOpsDescr, + ): + """ + Test listdir for listing directory contents. + """ + assert type(os_ops_descr) is OsOpsDescr + assert isinstance(os_ops_descr.os_ops, OsOperations) + + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + RunConditions.skip_if_windows() + + path = "/etc" + files = os_ops.listdir(path) + assert isinstance(files, list) + for f in files: + assert f is not None + assert type(f) is str + return + + def test_listdir_2( + self, + os_ops_descr: OsOpsDescr, + name_with_surprize: tagNameWithSurprize, + ): + """ + Test listdir for listing directory contents. + """ + assert type(os_ops_descr) is OsOpsDescr + assert isinstance(os_ops_descr.os_ops, OsOperations) + + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + RunConditions.skip_if_windows() + + path = os_ops.mkdtemp(name_with_surprize.value) + files = os_ops.listdir(path) + assert isinstance(files, list) + assert len(files) == 0 + + os_ops.rmdir(path) + return + + def test_path_exists_true__directory( + self, + os_ops_descr: OsOpsDescr, + name_with_surprize: tagNameWithSurprize, + ): + """ + Test path_exists for an existing directory. + """ + assert type(os_ops_descr) is OsOpsDescr + assert type(name_with_surprize) is __class__.tagNameWithSurprize + assert isinstance(os_ops_descr.os_ops, OsOperations) + + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + tmp_dir = os_ops.mkdtemp() + assert os_ops.path_exists(tmp_dir) is True + + tmp_dir2 = os_ops.build_path(tmp_dir, name_with_surprize.value) + os_ops.makedir(tmp_dir2) + assert os_ops.path_exists(tmp_dir2) is True + + os_ops.rmdir(tmp_dir2) + os_ops.rmdir(tmp_dir) + return + + def test_path_exists_true__file( + self, + os_ops_descr: OsOpsDescr, + name_with_surprize: tagNameWithSurprize, + ): + """ + Test path_exists for an existing file. + """ + assert type(os_ops_descr) is OsOpsDescr + assert type(name_with_surprize) is __class__.tagNameWithSurprize + assert isinstance(os_ops_descr.os_ops, OsOperations) + + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + tmp_dir = os_ops.mkdtemp() + assert os_ops.path_exists(tmp_dir) is True + + filename = os_ops.build_path(tmp_dir, name_with_surprize.value) + + data = "abc" + os_ops.write(filename, data, binary=False) + assert os_ops.read(filename, binary=False) == data + + LocalCheck.check_path_exists(os_ops, filename) + assert os_ops.path_exists(filename) is True + + os_ops.remove_file(filename) + os_ops.rmdir(tmp_dir) + return + + def test_path_exists_false__directory( + self, + os_ops_descr: OsOpsDescr, + ): + """ + Test path_exists for a non-existing directory. + """ + assert type(os_ops_descr) is OsOpsDescr + assert isinstance(os_ops_descr.os_ops, OsOperations) + + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + RunConditions.skip_if_windows() + + assert os_ops.path_exists("/nonexistent_path") is False + return + + def test_path_exists_false__file( + self, + os_ops_descr: OsOpsDescr, + ): + """ + Test path_exists for a non-existing file. + """ + assert type(os_ops_descr) is OsOpsDescr + assert isinstance(os_ops_descr.os_ops, OsOperations) + + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + RunConditions.skip_if_windows() + + assert os_ops.path_exists("/etc/nonexistent_path.txt") is False + return + + def test_mkdtemp__default( + self, + os_ops_descr: OsOpsDescr, + ): + assert type(os_ops_descr) is OsOpsDescr + assert isinstance(os_ops_descr.os_ops, OsOperations) + + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + path = os_ops.mkdtemp() + logging.info("Path is [{0}].".format(path)) + LocalCheck.check_path_exists(os_ops, path) + assert os_ops.path_exists(path) + assert os_ops.isdir(path) + os_ops.rmdir(path) + LocalCheck.check_path_does_not_exists(os_ops, path) + assert not os_ops.path_exists(path) + return + + def test_mkdtemp__custom( + self, + os_ops_descr: OsOpsDescr, + name_with_surprize: tagNameWithSurprize, + ): + assert type(os_ops_descr) is OsOpsDescr + assert isinstance(os_ops_descr.os_ops, OsOperations) + assert type(name_with_surprize) is __class__.tagNameWithSurprize + + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + C_TEMPLATE = name_with_surprize.value + path = os_ops.mkdtemp(C_TEMPLATE) + logging.info("Path is [{0}].".format(path)) + LocalCheck.check_path_exists(os_ops, path) + assert os_ops.path_exists(path) + assert os_ops.isdir(path) + assert C_TEMPLATE in os_ops.get_path_basename(path) + os_ops.rmdir(path) + LocalCheck.check_path_does_not_exists(os_ops, path) + assert not os_ops.path_exists(path) + return + + def test_rmdirs( + self, + os_ops_descr: OsOpsDescr, + name_with_surprize: tagNameWithSurprize, + ): + assert type(os_ops_descr) is OsOpsDescr + assert isinstance(os_ops_descr.os_ops, OsOperations) + assert type(name_with_surprize) is __class__.tagNameWithSurprize + + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + tmpdir = os_ops.get_tempdir() + + path = os_ops.build_path( + tmpdir, + "testgres-os_ops-test_rmdirs-{}-{}".format( + name_with_surprize.value, + uuid.uuid4().bytes.hex(), + ) + ) + + local_detecter_is_created = False + if OsOpsHelpers.is_localhost(os_ops): + pass + elif sys.platform != os_ops.get_platform(): + pass + elif not os.path.exists(tmpdir): + pass + else: + # We will check a real work with another host + assert not os.path.exists(path) + os.mkdir(path) + assert os.path.exists(path) + local_detecter_is_created = True + logging.info("Local detecter is created [{}]".format(path)) + + cmd = ["sh", "-c", "mkdir " + os_ops.quote_path(path)] + os_ops.exec_command(cmd) + + LocalCheck.check_path_exists(os_ops, path) + assert os_ops.path_exists(path) + assert os_ops.isdir(path) + + assert os_ops.rmdirs(path, ignore_errors=False) is True + LocalCheck.check_path_does_not_exists(os_ops, path) + assert not os_ops.path_exists(path) + + if local_detecter_is_created: + assert os.path.exists(path) + os.rmdir(path) + logging.info("Local detecter is deleted [{}]".format(path)) + + return + + def test_rmdirs__01_with_subfolder( + self, + os_ops_descr: OsOpsDescr, + name_with_surprize: tagNameWithSurprize, + ): + assert type(os_ops_descr) is OsOpsDescr + assert isinstance(os_ops_descr.os_ops, OsOperations) + assert type(name_with_surprize) is __class__.tagNameWithSurprize + + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + # folder with subfolder + path = os_ops.mkdtemp() + LocalCheck.check_path_exists(os_ops, path) + assert os_ops.path_exists(path) + + dir1 = os_ops.build_path(path, name_with_surprize.value) + LocalCheck.check_path_does_not_exists(os_ops, dir1) + assert not os_ops.path_exists(dir1) + + os_ops.makedir(dir1) + LocalCheck.check_path_exists(os_ops, dir1) + assert os_ops.path_exists(dir1) + assert os_ops.isdir(dir1) + + assert os_ops.rmdirs(path, ignore_errors=False) is True + LocalCheck.check_path_does_not_exists(os_ops, path) + LocalCheck.check_path_does_not_exists(os_ops, dir1) + assert not os_ops.path_exists(path) + assert not os_ops.path_exists(dir1) + return + + def test_rmdirs__02_with_file( + self, + os_ops_descr: OsOpsDescr, + name_with_surprize: tagNameWithSurprize, + ): + assert type(os_ops_descr) is OsOpsDescr + assert isinstance(os_ops_descr.os_ops, OsOperations) + assert type(name_with_surprize) is __class__.tagNameWithSurprize + + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + # folder with file + path = os_ops.mkdtemp() + LocalCheck.check_path_exists(os_ops, path) + assert os_ops.path_exists(path) + + file1 = os_ops.build_path(path, name_with_surprize.value) + LocalCheck.check_path_does_not_exists(os_ops, file1) + assert not os_ops.path_exists(file1) + + os_ops.touch(file1) + LocalCheck.check_path_exists(os_ops, file1) + assert os_ops.path_exists(file1) + assert os_ops.isfile(file1) + + assert os_ops.rmdirs(path, ignore_errors=False) is True + LocalCheck.check_path_does_not_exists(os_ops, path) + LocalCheck.check_path_does_not_exists(os_ops, file1) + assert not os_ops.path_exists(path) + assert not os_ops.path_exists(file1) + return + + def test_rmdirs__03_with_subfolder_and_file( + self, + os_ops_descr: OsOpsDescr, + name_with_surprize: tagNameWithSurprize, + ): + assert type(os_ops_descr) is OsOpsDescr + assert isinstance(os_ops_descr.os_ops, OsOperations) + assert type(name_with_surprize) is __class__.tagNameWithSurprize + + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + # folder with subfolder and file + path = os_ops.mkdtemp() + LocalCheck.check_path_exists(os_ops, path) + assert os_ops.path_exists(path) + + dir1 = os_ops.build_path(path, name_with_surprize.value) + LocalCheck.check_path_does_not_exists(os_ops, dir1) + assert not os_ops.path_exists(dir1) + + os_ops.makedirs(dir1) + LocalCheck.check_path_exists(os_ops, dir1) + assert os_ops.path_exists(dir1) + assert os_ops.isdir(dir1) + assert not os_ops.isfile(dir1) + + file1 = os_ops.build_path(dir1, name_with_surprize.value) + LocalCheck.check_path_does_not_exists(os_ops, file1) + assert not os_ops.path_exists(file1) + + os_ops.touch(file1) + LocalCheck.check_path_exists(os_ops, file1) + assert os_ops.path_exists(file1) + assert os_ops.isfile(file1) + assert not os_ops.isdir(file1) + + assert os_ops.rmdirs(path, ignore_errors=False) is True + LocalCheck.check_path_does_not_exists(os_ops, path) + LocalCheck.check_path_does_not_exists(os_ops, dir1) + LocalCheck.check_path_does_not_exists(os_ops, file1) + assert not os_ops.path_exists(path) + assert not os_ops.path_exists(dir1) + assert not os_ops.path_exists(file1) + return + + def test_write_text_file( + self, + os_ops_descr: OsOpsDescr, + name_with_surprize: tagNameWithSurprize, + ): + """ + Test write for writing data to a text file. + """ + assert type(os_ops_descr) is OsOpsDescr + assert isinstance(os_ops_descr.os_ops, OsOperations) + assert type(name_with_surprize) is __class__.tagNameWithSurprize + + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + RunConditions.skip_if_windows() + + filename = os_ops.mkstemp(name_with_surprize.value) + data = "Hello, world!" + + os_ops.write(filename, data, truncate=True) + os_ops.write(filename, data) + + response = os_ops.read(filename) + + assert response == data + data + + os_ops.remove_file(filename) + return + + def test_write_binary_file( + self, + os_ops_descr: OsOpsDescr, + name_with_surprize: tagNameWithSurprize, + ): + """ + Test write for writing data to a binary file. + """ + assert type(os_ops_descr) is OsOpsDescr + assert isinstance(os_ops_descr.os_ops, OsOperations) + assert type(name_with_surprize) is __class__.tagNameWithSurprize + + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + RunConditions.skip_if_windows() + + filename = os_ops.mkstemp(name_with_surprize.value) + data = b"\x00\x01\x02\x03" + + os_ops.write(filename, data, binary=True, truncate=True) + + response = os_ops.read(filename, binary=True) + assert response == data + + os_ops.remove_file(filename) + return + + def test_read_text_file( + self, + os_ops_descr: OsOpsDescr, + name_with_surprize: tagNameWithSurprize, + ): + """ + Test read for reading data from a text file. + """ + assert type(os_ops_descr) is OsOpsDescr + assert isinstance(os_ops_descr.os_ops, OsOperations) + assert type(name_with_surprize) is __class__.tagNameWithSurprize + + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + RunConditions.skip_if_windows() + + filename = os_ops.mkstemp(name_with_surprize.value) + + C_DATA = "\nabc\n321\n\n" + os_ops.write(filename, C_DATA) + + response = os_ops.read(filename) + assert isinstance(response, str) + assert response == C_DATA + return + + def test_read_binary_file( + self, + os_ops_descr: OsOpsDescr, + ): + """ + Test read for reading data from a binary file. + """ + assert type(os_ops_descr) is OsOpsDescr + assert isinstance(os_ops_descr.os_ops, OsOperations) + + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + RunConditions.skip_if_windows() + + filename = "/usr/bin/python3" + + response = os_ops.read(filename, binary=True) + + assert isinstance(response, bytes) + return + + def test_read__text( + self, + os_ops_descr: OsOpsDescr, + ): + """ + Test OsOperations::read for text data. + """ + assert type(os_ops_descr) is OsOpsDescr + assert isinstance(os_ops_descr.os_ops, OsOperations) + + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + with open(__file__, 'r', encoding="utf-8") as file: + response0 = file.read() + + assert type(response0) is str + + filename = os_ops.mkstemp( + "testgres-os_ops-test_read__text", + ) + + os_ops.write( + filename, + response0, + binary=False, + ) + + response1 = os_ops.read(filename) + assert type(response1) is str + assert response1 == response0 + + response2 = os_ops.read(filename, encoding=None, binary=False) + assert type(response2) is str + assert response2 == response0 + + response3 = os_ops.read(filename, encoding="") + assert type(response3) is str + assert response3 == response0 + + response4 = os_ops.read(filename, encoding="UTF-8") + assert type(response4) is str + assert response4 == response0 + + os_ops.remove_file(filename) + return + + def test_read__binary( + self, + os_ops_descr: OsOpsDescr, + name_with_surprize: tagNameWithSurprize, + ): + """ + Test OsOperations::read for binary data. + """ + assert type(os_ops_descr) is OsOpsDescr + assert isinstance(os_ops_descr.os_ops, OsOperations) + assert type(name_with_surprize) is __class__.tagNameWithSurprize + + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + with open(__file__, 'rb') as file: + response0 = file.read() + + assert type(response0) is bytes + + filename = os_ops.mkstemp( + name_with_surprize.value, + ) + + os_ops.write( + filename, + response0, + binary=True, + ) + + response1 = os_ops.read(filename, binary=True) + assert type(response1) is bytes + assert response1 == response0 + + os_ops.remove_file(filename) + return + + def test_read__binary_and_encoding( + self, + os_ops_descr: OsOpsDescr, + ): + """ + Test OsOperations::read for binary data and encoding. + """ + assert type(os_ops_descr) is OsOpsDescr + assert isinstance(os_ops_descr.os_ops, OsOperations) + + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + filename = os_ops.mkstemp() + + with pytest.raises( + InvalidOperationException, + match=re.escape("Enconding is not allowed for read binary operation")): + os_ops.read(filename, encoding="", binary=True) + + os_ops.remove_file(filename) + return + + def test_read_binary__spec( + self, + os_ops_descr: OsOpsDescr, + name_with_surprize: tagNameWithSurprize, + ): + """ + Test OsOperations::read_binary. + """ + assert type(os_ops_descr) is OsOpsDescr + assert isinstance(os_ops_descr.os_ops, OsOperations) + assert type(name_with_surprize) is __class__.tagNameWithSurprize + + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + with open(__file__, 'rb') as file: + response0 = file.read() + + assert type(response0) is bytes + + filename = os_ops.mkstemp( + name_with_surprize.value, + ) + + os_ops.write( + filename, + response0, + binary=True, + ) + + response1 = os_ops.read_binary(filename, 0) + assert type(response1) is bytes + assert response1 == response0 + + response2 = os_ops.read_binary(filename, 1) + assert type(response2) is bytes + assert len(response2) < len(response1) + assert len(response2) + 1 == len(response1) + assert response2 == response1[1:] + + response3 = os_ops.read_binary(filename, len(response1)) + assert type(response3) is bytes + assert len(response3) == 0 + + response4 = os_ops.read_binary(filename, len(response2)) + assert type(response4) is bytes + assert len(response4) == 1 + assert response4[0] == response1[len(response1) - 1] + + response5 = os_ops.read_binary(filename, len(response1) + 1) + assert type(response5) is bytes + assert len(response5) == 0 + + os_ops.remove_file(filename) + return + + def test_read_binary__spec__negative_offset( + self, + os_ops_descr: OsOpsDescr, + name_with_surprize: tagNameWithSurprize, + ): + """ + Test OsOperations::read_binary with negative offset. + """ + assert type(os_ops_descr) is OsOpsDescr + assert isinstance(os_ops_descr.os_ops, OsOperations) + assert type(name_with_surprize) is __class__.tagNameWithSurprize + + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + filename = os_ops.mkstemp(name_with_surprize.value) + + with pytest.raises( + ValueError, + match=re.escape("Negative 'offset' is not supported.")): + os_ops.read_binary(filename, -1) + + os_ops.remove_file(filename) + return + + def test_get_file_size( + self, + os_ops_descr: OsOpsDescr, + name_with_surprize: tagNameWithSurprize, + ): + """ + Test OsOperations::get_file_size. + """ + assert type(os_ops_descr) is OsOpsDescr + assert isinstance(os_ops_descr.os_ops, OsOperations) + assert type(name_with_surprize) is __class__.tagNameWithSurprize + + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + filename = os_ops.mkstemp(name_with_surprize.value) + sz = os_ops.get_file_size(filename) + assert type(sz) is int + assert sz == 0 + + os_ops.write(filename, b"\x02\x01\x00", binary=True) + sz = os_ops.get_file_size(filename) + assert type(sz) is int + assert sz == 3 + + os_ops.write(filename, b"\x04", binary=True, truncate=False) + sz = os_ops.get_file_size(filename) + assert type(sz) is int + assert sz == 4 + + os_ops.remove_file(filename) + return + + def test_isfile_true( + self, + os_ops_descr: OsOpsDescr, + name_with_surprize: tagNameWithSurprize, + ): + """ + Test isfile for an existing file. + """ + assert type(os_ops_descr) is OsOpsDescr + assert isinstance(os_ops_descr.os_ops, OsOperations) + assert type(name_with_surprize) is __class__.tagNameWithSurprize + + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + filename = os_ops.mkstemp(name_with_surprize.value) + + LocalCheck.check_isfile(os_ops, filename) + response = os_ops.isfile(filename) + assert response is True + + os_ops.remove_file(filename) + return + + def test_isfile_false__not_exist( + self, + os_ops_descr: OsOpsDescr, + name_with_surprize: tagNameWithSurprize, + ): + """ + Test isfile for a non-existing file. + """ + assert type(os_ops_descr) is OsOpsDescr + assert isinstance(os_ops_descr.os_ops, OsOperations) + assert type(name_with_surprize) is __class__.tagNameWithSurprize + + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + tmpdir = os_ops.get_tempdir() + + filedir = os_ops.build_path( + tmpdir, + "test_isfile_false__not_exist-" + uuid.uuid4().bytes.hex(), + ) + + os_ops.makedir(filedir) + + filename = os_ops.build_path( + filedir, + name_with_surprize.value, + ) + + LocalCheck.check_path_does_not_exists(os_ops, filename) + assert not os_ops.path_exists(filename) + + local_detecter_is_created = False + if OsOpsHelpers.is_localhost(os_ops): + pass + elif sys.platform != os_ops.get_platform(): + pass + elif not os.path.exists(tmpdir): + pass + else: + # We will check a real work with another host + assert not os.path.exists(filedir) + assert not os.path.exists(filename) + + os.mkdir(filedir) + + with open(filename, "a"): + os.utime(filename, None) + assert os.path.exists(filename) + assert os.path.isfile(filename) + local_detecter_is_created = True + logging.info("Local detecter is created [{}]".format(filename)) + + response = os_ops.isfile(filename) + assert response is False + + if local_detecter_is_created: + assert os.path.exists(filename) + os.remove(filename) + assert not os.path.exists(filename) + os.rmdir(filedir) + assert not os.path.exists(filedir) + logging.info("Local detecter is deleted [{}]".format(filename)) + + os_ops.rmdir(filedir) + return + + def test_isfile_false__directory( + self, + os_ops_descr: OsOpsDescr, + ): + """ + Test isfile for a firectory. + """ + assert type(os_ops_descr) is OsOpsDescr + assert isinstance(os_ops_descr.os_ops, OsOperations) + + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + tmpdir = os_ops.get_tempdir() + LocalCheck.check_path_exists(os_ops, tmpdir) + LocalCheck.check_isdir(os_ops, tmpdir) + LocalCheck.check_not_isfile(os_ops, tmpdir) + assert os_ops.path_exists(tmpdir) + assert os_ops.isdir(tmpdir) + + response = os_ops.isfile(tmpdir) + assert response is False + return + + def test_isdir_true( + self, + os_ops_descr: OsOpsDescr, + name_with_surprize: tagNameWithSurprize, + ): + """ + Test isdir for an existing directory. + """ + assert type(os_ops_descr) is OsOpsDescr + assert isinstance(os_ops_descr.os_ops, OsOperations) + assert type(name_with_surprize) is __class__.tagNameWithSurprize + + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + tmpdir = os_ops.mkdtemp(name_with_surprize.value) + LocalCheck.check_path_exists(os_ops, tmpdir) + LocalCheck.check_isdir(os_ops, tmpdir) + LocalCheck.check_not_isfile(os_ops, tmpdir) + assert os_ops.path_exists(tmpdir) + + response = os_ops.isdir(tmpdir) + assert response is True + + os_ops.rmdir(tmpdir) + return + + def test_isdir_false__not_exist( + self, + os_ops_descr: OsOpsDescr, + ): + """ + Test isdir for a non-existing directory. + """ + assert type(os_ops_descr) is OsOpsDescr + assert isinstance(os_ops_descr.os_ops, OsOperations) + + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + tmpdir = os_ops.get_tempdir() + LocalCheck.check_path_exists(os_ops, tmpdir) + LocalCheck.check_isdir(os_ops, tmpdir) + LocalCheck.check_not_isfile(os_ops, tmpdir) + assert os_ops.path_exists(tmpdir) + assert os_ops.isdir(tmpdir) is True + + name = os_ops.build_path( + tmpdir, + "it_is_nonexistent_directory-{}".format(uuid.uuid4().bytes.hex()), + ) + + response = os_ops.isdir(name) + assert response is False + return + + def test_isdir_false__file( + self, + os_ops_descr: OsOpsDescr, + ): + """ + Test isdir for a file. + """ + assert type(os_ops_descr) is OsOpsDescr + assert isinstance(os_ops_descr.os_ops, OsOperations) + + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + name = os_ops.mkstemp() + LocalCheck.check_path_exists(os_ops, name) + LocalCheck.check_isfile(os_ops, name) + LocalCheck.check_not_isdir(os_ops, name) + assert os_ops.path_exists(name) is True + assert os_ops.isfile(name) is True + + response = os_ops.isdir(name) + assert response is False + + os_ops.remove_file(name) + LocalCheck.check_path_does_not_exists(os_ops, name) + assert os_ops.path_exists(name) is False + return + + def test_cwd( + self, + os_ops_descr: OsOpsDescr, + ): + """ + Test cwd. + """ + assert type(os_ops_descr) is OsOpsDescr + assert isinstance(os_ops_descr.os_ops, OsOperations) + + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + v = os_ops.cwd() + + assert v is not None + assert type(v) is str + assert v != "" + return + + class tagWriteData001: + def __init__(self, sign, source, cp_rw, cp_truncate, cp_binary, cp_data, result): + self.sign = sign + self.source = source + self.call_param__rw = cp_rw + self.call_param__truncate = cp_truncate + self.call_param__binary = cp_binary + self.call_param__data = cp_data + self.result = result + return + + sm_write_data001 = [ + tagWriteData001("A001", "1234567890", False, False, False, "ABC", "1234567890ABC"), + tagWriteData001("A002", b"1234567890", False, False, True, b"ABC", b"1234567890ABC"), + + tagWriteData001("B001", "1234567890", False, True, False, "ABC", "ABC"), + tagWriteData001("B002", "1234567890", False, True, False, "ABC1234567890", "ABC1234567890"), + tagWriteData001("B003", b"1234567890", False, True, True, b"ABC", b"ABC"), + tagWriteData001("B004", b"1234567890", False, True, True, b"ABC1234567890", b"ABC1234567890"), + + tagWriteData001("C001", "1234567890", True, False, False, "ABC", "1234567890ABC"), + tagWriteData001("C002", b"1234567890", True, False, True, b"ABC", b"1234567890ABC"), + + tagWriteData001("D001", "1234567890", True, True, False, "ABC", "ABC"), + tagWriteData001("D002", "1234567890", True, True, False, "ABC1234567890", "ABC1234567890"), + tagWriteData001("D003", b"1234567890", True, True, True, b"ABC", b"ABC"), + tagWriteData001("D004", b"1234567890", True, True, True, b"ABC1234567890", b"ABC1234567890"), + + tagWriteData001("E001", "\0001234567890\000", False, False, False, "\000ABC\000", "\0001234567890\000\000ABC\000"), + tagWriteData001("E002", b"\0001234567890\000", False, False, True, b"\000ABC\000", b"\0001234567890\000\000ABC\000"), + + tagWriteData001("F001", "a\nb\n", False, False, False, ["c", "d"], "a\nb\ncd"), + tagWriteData001("F002", b"a\nb\n", False, False, True, [b"c", b"d"], b"a\nb\ncd"), + + tagWriteData001("G001", "a\nb\n", False, False, False, ["c\n\n", "d\n"], "a\nb\nc\n\nd\n"), + tagWriteData001("G002", b"a\nb\n", False, False, True, [b"c\n\n", b"d\n"], b"a\nb\nc\n\nd\n"), + + tagWriteData001("H001", "a\nb\n\000", False, False, False, ["c\n\n", "d\n"], "a\nb\n\000c\n\nd\n"), + tagWriteData001("H002", b"a\nb\n\000", False, False, True, [b"c\n\n", b"d\n"], b"a\nb\n\000c\n\nd\n"), + + tagWriteData001("J001", "a\nb\n\000", False, False, False, ["c\n\n\x00", "d\n"], "a\nb\n\000c\n\n\x00d\n"), + tagWriteData001("J002", b"a\nb\n\000", False, False, True, [b"c\n\n\x00", b"d\n"], b"a\nb\n\000c\n\n\x00d\n"), + ] + + @pytest.fixture( + params=sm_write_data001, + ids=[x.sign for x in sm_write_data001], + ) + def write_data001(self, request: pytest.FixtureRequest): + assert isinstance(request, pytest.FixtureRequest) + assert type(request.param) is __class__.tagWriteData001 + return request.param + + def test_write( + self, + write_data001: tagWriteData001, + os_ops_descr: OsOpsDescr, + ): + assert type(os_ops_descr) is OsOpsDescr + assert isinstance(os_ops_descr.os_ops, OsOperations) + assert type(write_data001) is __class__.tagWriteData001 + + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + tmp_file = os_ops.mkstemp("testgres-os_ops-test_write") + + os_ops.write( + tmp_file, + write_data001.source, + binary=write_data001.call_param__binary, + ) + s = os_ops.read( + tmp_file, + binary=write_data001.call_param__binary, + ) + assert s == write_data001.source + + os_ops.write( + tmp_file, + write_data001.call_param__data, + read_and_write=write_data001.call_param__rw, + truncate=write_data001.call_param__truncate, + binary=write_data001.call_param__binary, + ) + s = os_ops.read( + tmp_file, + binary=write_data001.call_param__binary, + ) + assert s == write_data001.result + + os_ops.remove_file(tmp_file) + return + + def test_touch( + self, + os_ops_descr: OsOpsDescr, + name_with_surprize: tagNameWithSurprize, + ): + """ + Test touch for creating a new file or updating access and modification times of an existing file. + """ + assert type(os_ops_descr) is OsOpsDescr + assert isinstance(os_ops_descr.os_ops, OsOperations) + assert type(name_with_surprize) is __class__.tagNameWithSurprize + + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + tmpdir = os_ops.mkdtemp() + + filename = os_ops.build_path(tmpdir, name_with_surprize.value) + + os_ops.touch(filename) + + assert os_ops.path_exists(filename) + assert os_ops.isfile(filename) + + stat1 = os_ops.get_file_stat(filename) + assert type(stat1) is dict + + time.sleep(1.1) + + os_ops.touch(filename) + + stat2 = os_ops.get_file_stat(filename) + assert type(stat2) is dict + + mtime1 = stat1[os_ops.C_FILE_STAT_PROP__MTIME] + mtime2 = stat2[os_ops.C_FILE_STAT_PROP__MTIME] + + assert type(mtime1) is datetime.datetime + assert type(mtime2) is datetime.datetime + + assert mtime1 < mtime2 + + os_ops.remove_file(filename) + os_ops.rmdir(tmpdir) + return + + def test_is_port_free__true( + self, + os_ops_descr: OsOpsDescr, + ): + assert type(os_ops_descr) is OsOpsDescr + assert isinstance(os_ops_descr.os_ops, OsOperations) + + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + C_SAMPLES = 128 + C_LIMIT = 10 + + ports = random.sample(range(1024, 65536), C_SAMPLES) + assert type(ports) is list + + ok_count = 0 + no_count = 0 + + py_test_code_templ = """ +import socket +with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + try: + s.bind(("", {0})) + except OSError: + exit(123) +print(str({0})) +exit(0) +""" + for port in ports: + logging.info("Try to bind port {}...".format(port)) + + py_test_code = py_test_code_templ.format(port) + + try: + r = os_ops.exec_command( + ["python3", "-c", py_test_code], + encoding="utf-8", + ) + except ExecUtilException as e: + if e.exit_code == 123: + logging.info("Fails") + continue + raise + + assert r == str(port) + "\n" + + logging.info("Try to check via is_port_free ...") + r = os_ops.is_port_free(port) + + if r: + ok_count += 1 + logging.info("OK. Port {} is free.".format(port)) + else: + no_count += 1 + logging.warning("NO. Port {} is not free.".format(port)) + + if ok_count == C_LIMIT: + return + continue + + if ok_count == 0: + raise RuntimeError("No one free port was found.") + return + + def test_is_port_free__false( + self, + os_ops_descr: OsOpsDescr, + ): + assert type(os_ops_descr) is OsOpsDescr + assert isinstance(os_ops_descr.os_ops, OsOperations) + + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + C_LIMIT = 5 + C_SAMPLES = C_LIMIT * 2 + + ports = random.sample(range(1024, 65536), C_SAMPLES) + + ok_count = 0 + no_count = 0 + + py_server_code_templ = """ +import socket, time +s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) +try: + s.bind(("", {0})) + s.listen(1) + print("READY", flush=True) + time.sleep(30) # Keep the port busy for 30 seconds +except OSError: + exit(123) +exit(0) +""" + for port in ports: + logging.info("Try to occupy port {} on target machine...".format(port)) + + py_server_code = py_server_code_templ.format(port) + + # Start a background process on the target machine + p = os_ops.exec_command( + ["python3", "-u", "-c", py_server_code], + get_process=True, + encoding="utf-8", + ) + assert isinstance(p, subprocess.Popen) + assert p.stdout is not None + + try: + # Read the first line from the process's stdout. + # If it's READY, the socket was successfully bound. + # If the process crashed (code 123), readline() will return empty. + ready_line = p.stdout.readline() + + if ready_line != "READY\n": + logging.info("Port {} is already busy or failed to bind, skipping...".format(port)) + continue # it jumps into finally + + logging.info("Port {} is occupied. Verifying via is_port_free...".format(port)) + + # MAIN CHECK: The port is currently busy, so is_port_free should return False! + r = os_ops.is_port_free(port) + assert type(r) is bool + + if not r: + ok_count += 1 + logging.info("OK. Port {} is correctly detected as NOT free.".format(port)) + else: + no_count += 1 + logging.warning("NO. Port {} was detected as free, but it is busy!".format(port)) + finally: + # We guarantee that the process will be terminated and the port will be released on the target machine. + p.terminate() + p.wait() + + if ok_count == C_LIMIT: + return + continue + + if ok_count == 0: + raise RuntimeError("No one free port was found.") + return + + def test_get_tempdir( + self, + os_ops_descr: OsOpsDescr, + ): + assert type(os_ops_descr) is OsOpsDescr + assert isinstance(os_ops_descr.os_ops, OsOperations) + + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + dir = os_ops.get_tempdir() + assert type(dir) is str + LocalCheck.check_path_exists(os_ops, dir) + assert os_ops.path_exists(dir) is True + assert os_ops.isdir(dir) is True + + file_path = os_ops.build_path( + dir, + "testgres--" + uuid.uuid4().hex + ".tmp", + ) + + os_ops.write(file_path, "1234", binary=False) + + LocalCheck.check_path_exists(os_ops, file_path) + LocalCheck.check_isfile(os_ops, file_path) + assert os_ops.path_exists(file_path) is True + assert os_ops.isfile(file_path) is True + assert os_ops.get_file_size(file_path) == 4 + + d = os_ops.read(file_path, binary=False) + assert d == "1234" + + os_ops.remove_file(file_path) + LocalCheck.check_path_does_not_exists(os_ops, file_path) + assert os_ops.path_exists(file_path) is False + return + + def test_get_tempdir__compare_with_py_info( + self, + os_ops_descr: OsOpsDescr, + ): + assert type(os_ops_descr) is OsOpsDescr + assert isinstance(os_ops_descr.os_ops, OsOperations) + + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + actual_dir = os_ops.get_tempdir() + assert actual_dir is not None + assert type(actual_dir) is str + + # -------- + cmd = ["python3", "-c", "import tempfile;print(tempfile.gettempdir());"] + + expected_dir_b = os_ops.exec_command(cmd) + assert type(expected_dir_b) is bytes + expected_dir = expected_dir_b.decode() + assert type(expected_dir) is str + assert actual_dir + "\n" == expected_dir + return + + class tagData_OS_OPS__NUMS: + os_ops_descr: OsOpsDescr + nums: int + + def __init__(self, os_ops_descr: OsOpsDescr, nums: int): + assert type(os_ops_descr) is OsOpsDescr + assert type(nums) is int + + self.os_ops_descr = os_ops_descr + self.nums = nums + return + + sm_test_exclusive_creation__mt__data = [ + tagData_OS_OPS__NUMS(OsOpsDescrs.sm_local_os_ops_descr, 100000), + tagData_OS_OPS__NUMS(OsOpsDescrs.sm_remote_os_ops_descr, 120), + ] + + @pytest.fixture( + params=sm_test_exclusive_creation__mt__data, + ids=[x.os_ops_descr.sign for x in sm_test_exclusive_creation__mt__data] + ) + def data001(self, request: pytest.FixtureRequest) -> tagData_OS_OPS__NUMS: + assert isinstance(request, pytest.FixtureRequest) + return request.param + + def test_mkdir__mt(self, data001: tagData_OS_OPS__NUMS): + assert type(data001) is __class__.tagData_OS_OPS__NUMS + + N_WORKERS = 4 + N_NUMBERS = data001.nums + assert type(N_NUMBERS) is int + + os_ops = data001.os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + lock_dir_prefix = "test_mkdir_mt--" + uuid.uuid4().hex + + lock_dir = os_ops.mkdtemp(prefix=lock_dir_prefix) + + logging.info("A lock file [{}] is creating ...".format(lock_dir)) + + LocalCheck.check_path_exists(os_ops, lock_dir) + assert os_ops.path_exists(lock_dir) is True + + def MAKE_PATH(os_ops: OsOperations, lock_dir: str, num: int) -> str: + assert isinstance(os_ops, OsOperations) + assert type(lock_dir) is str + assert type(num) is int + return os_ops.build_path(lock_dir, str(num) + ".lock") + + def LOCAL_WORKER(os_ops: OsOperations, + workerID: int, + lock_dir: str, + cNumbers: int, + reservedNumbers: typing.Set[int]) -> None: + assert isinstance(os_ops, OsOperations) + assert type(workerID) is int + assert type(lock_dir) is str + assert type(cNumbers) is int + assert type(reservedNumbers) is set + assert cNumbers > 0 + assert len(reservedNumbers) == 0 + + assert os_ops.path_exists(lock_dir) + + def LOG_INFO(template: str, *args) -> None: + assert type(template) is str + assert type(args) is tuple + + msg = template.format(*args) + assert type(msg) is str + + logging.info("[Worker #{}] {}".format(workerID, msg)) + return + + LOG_INFO("HELLO! I am here!") + + for num in range(cNumbers): + assert num not in reservedNumbers + + file_path = MAKE_PATH(os_ops, lock_dir, num) + + try: + os_ops.makedir(file_path) + except Exception as e: + LOG_INFO( + "Can't reserve {}. Error ({}): {}", + num, + type(e).__name__, + str(e) + ) + continue + + LOG_INFO("Number {} is reserved!", num) + assert os_ops.path_exists(file_path) + reservedNumbers.add(num) + continue + + n_total = cNumbers + n_ok = len(reservedNumbers) + assert n_ok <= n_total + + LOG_INFO("Finish! OK: {}. FAILED: {}.", n_ok, n_total - n_ok) + return + + # ----------------------- + logging.info("Worker are creating ...") + + threadPool = ThreadPoolExecutor( + max_workers=N_WORKERS, + thread_name_prefix="ex_creator" + ) + + class tadWorkerData: + future: ThreadFuture + reservedNumbers: typing.Set[int] + + workerDatas: typing.List[tadWorkerData] = list() + + nErrors = 0 + + try: + for n in range(N_WORKERS): + logging.info("worker #{} is creating ...".format(n)) + + workerDatas.append(tadWorkerData()) + + workerDatas[n].reservedNumbers = set() + + workerDatas[n].future = threadPool.submit( + LOCAL_WORKER, + os_ops, + n, + lock_dir, + N_NUMBERS, + workerDatas[n].reservedNumbers + ) + + assert workerDatas[n].future is not None + + logging.info("OK. All the workers were created!") + except Exception as e: + nErrors += 1 + logging.error("A problem is detected ({}): {}".format(type(e).__name__, str(e))) + + logging.info("Will wait for stop of all the workers...") + + nWorkers = 0 + + assert type(workerDatas) is list + + for i in range(len(workerDatas)): + worker = workerDatas[i].future + + if worker is None: + continue + + nWorkers += 1 + + assert isinstance(worker, ThreadFuture) + + try: + logging.info("Wait for worker #{}".format(i)) + worker.result() + except Exception as e: + nErrors += 1 + logging.error("Worker #{} finished with error ({}): {}".format( + i, + type(e).__name__, + str(e), + )) + continue + + assert nWorkers == N_WORKERS + + if nErrors != 0: + raise RuntimeError("Some problems were detected. Please examine the log messages.") + + logging.info("OK. Let's check worker results!") + + reservedNumbers: typing.Dict[int, int] = dict() + + for i in range(N_WORKERS): + logging.info("Worker #{} is checked ...".format(i)) + + workerNumbers = workerDatas[i].reservedNumbers + assert type(workerNumbers) is set + + for n in workerNumbers: + if n < 0 or n >= N_NUMBERS: + nErrors += 1 + logging.error("Unexpected number {}".format(n)) + continue + + if n in reservedNumbers.keys(): + nErrors += 1 + logging.error("Number {} was already reserved by worker #{}".format( + n, + reservedNumbers[n] + )) + else: + reservedNumbers[n] = i + + file_path = MAKE_PATH(os_ops, lock_dir, n) + if not os_ops.path_exists(file_path): + nErrors += 1 + logging.error("File {} is not found!".format(file_path)) + continue + + continue + + logging.info("OK. Let's check reservedNumbers!") + + for n in range(N_NUMBERS): + if n not in reservedNumbers.keys(): + nErrors += 1 + logging.error("Number {} is not reserved!".format(n)) + continue + + file_path = MAKE_PATH(os_ops, lock_dir, n) + if not os_ops.path_exists(file_path): + nErrors += 1 + logging.error("File {} is not found!".format(file_path)) + continue + + # OK! + continue + + logging.info("Verification is finished! Total error count is {}.".format(nErrors)) + + if nErrors == 0: + logging.info("Root lock-directory [{}] will be deleted.".format( + lock_dir + )) + + for n in range(N_NUMBERS): + file_path = MAKE_PATH(os_ops, lock_dir, n) + try: + os_ops.rmdir(file_path) + except Exception as e: + nErrors += 1 + logging.error("Cannot delete directory [{}]. Error ({}): {}".format( + file_path, + type(e).__name__, + str(e) + )) + continue + + if os_ops.path_exists(file_path): + nErrors += 1 + logging.error("Directory {} is not deleted!".format(file_path)) + continue + + if nErrors == 0: + try: + os_ops.rmdir(lock_dir) + except Exception as e: + nErrors += 1 + logging.error("Cannot delete directory [{}]. Error ({}): {}".format( + lock_dir, + type(e).__name__, + str(e) + )) + + logging.info("Test is finished! Total error count is {}.".format(nErrors)) + return + + @dataclasses.dataclass + class T_KILL_SIGNAL_DESCR: + sign: str + signal: typing.Union[int, os_signal.Signals] + signal_num_s: str + + sm_kill_signal_ids: typing.List[T_KILL_SIGNAL_DESCR] = [ + T_KILL_SIGNAL_DESCR("SIGINT", os_signal.SIGINT, "2"), + # T_KILL_SIGNAL_DESCR("SIGQUIT", os_signal.SIGQUIT, "3"), # it creates coredump + T_KILL_SIGNAL_DESCR("SIGKILL", os_signal.SIGKILL, "9"), + T_KILL_SIGNAL_DESCR("SIGTERM", os_signal.SIGTERM, "15"), + T_KILL_SIGNAL_DESCR("2", 2, "2"), + # T_KILL_SIGNAL_DESCR("3", 3, "3"), # it creates coredump + T_KILL_SIGNAL_DESCR("9", 9, "9"), + T_KILL_SIGNAL_DESCR("15", 15, "15"), + ] + + @pytest.fixture( + params=sm_kill_signal_ids, + ids=["signal: {}".format(x.sign) for x in sm_kill_signal_ids], + ) + def kill_signal_id( + self, + request: pytest.FixtureRequest, + ) -> T_KILL_SIGNAL_DESCR: + assert isinstance(request, pytest.FixtureRequest) + assert type(request.param).__name__ == "T_KILL_SIGNAL_DESCR" + return request.param + + def test_kill_signal( + self, + kill_signal_id: T_KILL_SIGNAL_DESCR, + ): + assert type(kill_signal_id) is __class__.T_KILL_SIGNAL_DESCR + assert "{}".format(kill_signal_id.signal) == kill_signal_id.signal_num_s + assert "{}".format(int(kill_signal_id.signal)) == kill_signal_id.signal_num_s + return + + def test_kill( + self, + os_ops_descr: OsOpsDescr, + kill_signal_id: T_KILL_SIGNAL_DESCR, + ): + """ + Test listdir for listing directory contents. + """ + assert type(os_ops_descr) is OsOpsDescr + assert isinstance(os_ops_descr.os_ops, OsOperations) + assert type(kill_signal_id) is __class__.T_KILL_SIGNAL_DESCR + + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + cmd = [ + "python3", + "-u", + "-c", + "import os, time; print(os.getpid());time.sleep(300);print('EXIT')" + ] + + logging.info("Local test process is creating ...") + proc = os_ops.exec_command( + cmd, + encoding="utf-8", + get_process=True, + ) + + assert proc is not None + assert type(proc) is subprocess.Popen + assert proc.stdout is not None + line = proc.stdout.readline() + assert line is not None + assert type(line) is str + logging.info("proc output: {!r}".format(line)) + line = line.rstrip() + assert line != "" + proc_pid = int(line) + assert type(proc_pid) is int + logging.info("Test process pid is {}".format(proc_pid)) + + logging.info("Check this test process ...") + assert os_ops.get_process_children(proc_pid) == [] + + logging.info("Kill this test process ...") + os_ops.kill(proc_pid, kill_signal_id.signal) + + logging.info("Wait for finish ...") + proc.wait() + + logging.info("Try to get this test process ...") + + attempt = 0 + while True: + if attempt == 20: + raise RuntimeError("Process did not die.") + + attempt += 1 + + if attempt > 1: + logging.info("Sleep 1 seconds...") + time.sleep(1) + + try: + os_ops.get_process_children(proc_pid) + except Exception as e: + if type(os_ops).__name__ == "LocalOperations": + if isinstance(e, psutil.ZombieProcess): + logging.info("Exception {}: {}".format( + type(e).__name__, + str(e), + )) + break + if isinstance(e, psutil.NoSuchProcess): + logging.info("OK. Process died.") + break + raise + + if type(os_ops).__name__ == "RemoteOperations": + if isinstance(e, ExecUtilException): + assert e.exit_code == 1 + logging.info("OK. Process died.") + break + raise + + logging.error("Unknown os_ops object: {}".format( + type(os_ops).__name__, + )) + raise + else: + logging.info("Process is alive!") + continue + return + + def test_kill__unk_pid( + self, + os_ops_descr: OsOpsDescr, + kill_signal_id: T_KILL_SIGNAL_DESCR, + ): + """ + Test listdir for listing directory contents. + """ + assert type(os_ops_descr) is OsOpsDescr + assert isinstance(os_ops_descr.os_ops, OsOperations) + assert type(kill_signal_id) is __class__.T_KILL_SIGNAL_DESCR + + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + cmd = [ + "python3", + "-u", + "-c", + """ +import os, sys +print('a:' + str(os.getpid()), file=sys.stdout) +print('a2', file=sys.stdout) +print('b', file=sys.stderr) +""" + ] + + logging.info("Local test process is creating ...") + proc = os_ops.exec_command( + cmd, + encoding="utf-8", + get_process=True, + ) + + assert proc is not None + assert type(proc) is subprocess.Popen + + proc_pid = proc.pid + assert type(proc_pid) is int + assert proc.stdout is not None + line = proc.stdout.readline() + assert line is not None + assert type(line) is str + logging.info("proc output: {!r}".format(line)) + line = line.rstrip() + assert line != "" + assert line.startswith("a:") + line = line[2:] + assert line != "" + proc_pid = int(line) + assert type(proc_pid) is int + logging.info("Test process pid is {}".format(proc_pid)) + + logging.info("Wait for finish ...") + pout = proc.stdout.read() + assert proc.stderr is not None + perr = proc.stderr.read() + proc.wait() + logging.info("STDOUT: {}".format(pout)) + logging.info("STDERR: {}".format(perr)) + assert type(pout) is str + assert type(perr) is str + assert pout == "a2\n" + assert perr == "b\n" + assert type(proc.returncode) is int + assert proc.returncode == 0 + + logging.info("Try to get this test process ...") + + attempt = 0 + while True: + if attempt == 20: + raise RuntimeError("Process did not die.") + + attempt += 1 + + if attempt > 1: + logging.info("Sleep 1 seconds...") + time.sleep(1) + + try: + os_ops.get_process_children(proc_pid) + except Exception as e: + if type(os_ops).__name__ == "LocalOperations": + if isinstance(e, psutil.ZombieProcess): + logging.info("Exception {}: {}".format( + type(e).__name__, + str(e), + )) + break + if isinstance(e, psutil.NoSuchProcess): + logging.info("OK. Process died.") + break + raise + + if type(os_ops).__name__ == "RemoteOperations": + if isinstance(e, ExecUtilException): + assert e.exit_code == 1 + logging.info("OK. Process died.") + break + raise + + logging.error("Unknown os_ops object: {}".format( + type(os_ops).__name__, + )) + raise + else: + logging.info("Process is alive!") + continue + + # -------------------- + with pytest.raises(expected_exception=Exception) as x: + os_ops.kill(proc_pid, kill_signal_id.signal) + + assert x is not None + assert isinstance(x.value, Exception) + assert not isinstance(x.value, AssertionError) + + logging.info("Our error is [{}]".format(str(x.value))) + logging.info("Our exception has type [{}]".format(type(x.value).__name__)) + + if type(os_ops).__name__ == "LocalOperations": + assert type(x.value) is ProcessLookupError + assert "No such process" in str(x.value) + elif type(os_ops).__name__ == "RemoteOperations": + assert type(x.value) is ExecUtilException + assert "No such process" in str(x.value) + else: + __class__.helper__bug_check__unknown_os_ops_type(os_ops) + + return + + def test_get_dirname( + self, + os_ops_descr: OsOpsDescr, + ): + assert type(os_ops_descr) is OsOpsDescr + assert isinstance(os_ops_descr.os_ops, OsOperations) + + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + expected_dirname = "abc" + + p = os_ops.build_path(expected_dirname, "file1.txt") + assert type(p) is str + assert p != "" + + actual_dirname = os_ops.get_dirname(p) + assert type(actual_dirname) is str + assert actual_dirname != "" + assert actual_dirname == expected_dirname + return + + def test_is_abs_path__yes( + self, + os_ops_descr: OsOpsDescr, + ): + assert type(os_ops_descr) is OsOpsDescr + assert isinstance(os_ops_descr.os_ops, OsOperations) + + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + p = os_ops.get_tempdir() + assert type(p) is str + assert p != "" + LocalCheck.check_path_exists(os_ops, p) + LocalCheck.check_isdir(os_ops, p) + LocalCheck.check_path_is_abs(os_ops, p) + assert os_ops.path_exists(p) is True + assert os_ops.isdir(p) is True + + actual_value = os_ops.is_abs_path(p) + assert type(actual_value) is bool + assert actual_value is True + return + + def test_is_abs_path__no(self, os_ops_descr: OsOpsDescr): + assert type(os_ops_descr) is OsOpsDescr + assert isinstance(os_ops_descr.os_ops, OsOperations) + + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + p = "." + assert not os.path.isabs(p) + LocalCheck.check_path_is_not_abs(os_ops, p) + + actual_value = os_ops.is_abs_path(p) + assert type(actual_value) is bool + assert actual_value is False + return + + # -------------------------------------------------------------------- + def test_get_abs_path( + self, + os_ops_descr: OsOpsDescr, + ): + assert type(os_ops_descr) is OsOpsDescr + assert isinstance(os_ops_descr.os_ops, OsOperations) + + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + def LOCAL__check(value, expected) -> bool: + logging.info("Source path: [{}]".format(value)) + actual = os_ops.get_abs_path(value) + if actual == expected: + logging.info("Result is OK: [{}].".format( + actual, + )) + else: + logging.error("Result is BAD: [{}]. Expected: [{}].".format( + actual, + expected, + )) + logging.info("") + return False + + logging.info("------------- test empty string") + cwd = os_ops.cwd() + LOCAL__check("", cwd) + + logging.info("------------- test cwd") + LOCAL__check(".", cwd) + + path = os_ops.build_path(cwd, ".") + LOCAL__check(path, cwd) + + cwd = os_ops.cwd() + expected_r = os_ops.build_path(cwd, "abc") + LOCAL__check("abc", expected_r) + + cwd = os_ops.cwd() + expected_r = os_ops.build_path(cwd, "abc") + LOCAL__check("./abc", expected_r) + + cwd = os_ops.cwd() + expected_r = os_ops.build_path(os_ops.get_dirname(cwd), "abc") + LOCAL__check("../abc", expected_r) + + cwd = os_ops.cwd() + expected_r = os_ops.build_path(cwd, "abc1.txt") + LOCAL__check("abc1.txt", expected_r) + + logging.info("------------- test cwd parent") + cwd = os_ops.cwd() + expected_r = os_ops.get_dirname(cwd) + LOCAL__check("..", expected_r) + + logging.info("------------- test file") + file = os_ops.mkstemp() + LOCAL__check(file, file) + os_ops.remove_file(file) + + logging.info("------------- test dir") + dir = os_ops.mkdtemp() + LOCAL__check(dir, dir) + + dirname = os_ops.get_path_basename(dir) + path = os_ops.build_path(dir, "..", dirname) + LOCAL__check(path, dir) + + dirname = os_ops.get_path_basename(dir) + path = os_ops.build_path(dir, "..", dirname, "abc.txt") + expected_r = os_ops.build_path(dir, "abc.txt") + LOCAL__check(path, expected_r) + + os_ops.rmdir(dir) + + logging.info("------------- unknown path") + expected_r = os_ops.build_path(cwd, "abc/file.txt") + LOCAL__check("./abc/file.txt", expected_r) + + logging.info("------------- home dir") + LOCAL__check("/~", "/~") + + logging.info("------------- test root over-traversal") + # ะ˜ะท ะปัŽะฑะพะณะพ ะผะตัั‚ะฐ ัะธัั‚ะตะผั‹ (ะดะฐะถะต ะณะปัƒะฑะพะบะพะณะพ) 15 ะฟะตั€ะตั…ะพะดะพะฒ ะฒะฒะตั€ั… ะฒั‹ะฒะตะดัƒั‚ ะฒ ะบะพั€ะตะฝัŒ + many_dots = os_ops.build_path(*([".."] * 15)) + LOCAL__check(many_dots, "/") + + # ะšะพั€ะตะฝัŒ + ะตั‰ะต ั€ะฐะท ะฒะฒะตั€ั… + ะฟะฐะฟะบะฐ + path = os_ops.build_path("/", "..", "abc") + LOCAL__check(path, "/abc") + + logging.info("------------- test multiple slashes") + + # TODO: Double slash at the beginning. In POSIX it sometimes + # has a special meaning, let's check it out. + # os.path.abs returns "//abc" + # r = os_ops.get_abs_path("//abc") + # LOCAL__check("//abc", "/abc") + + # Slashes in the middle of a relative path + expected_r = os_ops.build_path(cwd, "abc", "def") + LOCAL__check("abc///def", expected_r) + + # Relative path ending with a slash + expected_r = os_ops.build_path(cwd, "abc") + LOCAL__check("abc/", expected_r) + + logging.info("------------- test raw tilde") + exec_r = os_ops.exec_command(["sh", "-c", "cd ~;pwd"], encoding="utf-8") + assert type(exec_r) is str + expected_r = exec_r.rstrip() + LOCAL__check("~", expected_r) + + LOCAL__check("~/", expected_r) + + # Tilda with a ROOT user + LOCAL__check("~root", "/root") + LOCAL__check("~root/", "/root") + LOCAL__check("~root", "/root") + LOCAL__check("~root/abc/", "/root/abc") + + logging.info("------------- test spaces and special chars") + # Folder with quotes, and spaces. + weird_name = "my folder VAR 'single' \"double\"" + expected_r = os_ops.build_path(cwd, weird_name) + LOCAL__check(weird_name, expected_r) + + # TODO: Folder with dollar signs, quotes, and spaces. + # weird_name = "my folder $VAR 'single' \"double\"" + # expected_r = os_ops.build_path(cwd, weird_name) + # LOCAL__check(weird_name, expected_r) + + for n in __class__.sm_names_with_surprize: + logging.info("--------------------- test names with surprizes [{}]".format(n.sign)) + expected_r = os_ops.build_path(cwd, n.value) + LOCAL__check(n.value, expected_r) + + logging.info("OK. GO HOME!") + return + + # -------------------------------------------------------------------- + @dataclasses.dataclass + class tagGetPathBaseNameData: + sign: str + value: str + result: str + + sm_GetPathBaseNameDatas: typing.List[tagGetPathBaseNameData] = [ + tagGetPathBaseNameData( + sign="empty", + value="", + result="", + ), + tagGetPathBaseNameData( + sign="relative_curdir", + value=".", + result=".", + ), + tagGetPathBaseNameData( + sign="relative_parentdir", + value="..", + result="..", + ), + tagGetPathBaseNameData( + sign="a", + value="a", + result="a", + ), + tagGetPathBaseNameData( + sign="a.txt", + value="a.txt", + result="a.txt", + ), + tagGetPathBaseNameData( + sign="root__a.txt", + value="/a.txt", + result="a.txt", + ), + tagGetPathBaseNameData( + sign="curdir__a.txt", + value="./a.txt", + result="a.txt", + ), + tagGetPathBaseNameData( + sign="parentdir__a.txt", + value="../a.txt", + result="a.txt", + ), + tagGetPathBaseNameData( + sign="path001", + value="a/b/c/my-file-name.txt", + result="my-file-name.txt", + ), + tagGetPathBaseNameData( + sign="path002", + value="a/b/c/my-file-name", + result="my-file-name", + ), + ] + + @pytest.fixture( + params=[ + pytest.param( + x, + id=x.sign, + ) + for x in sm_GetPathBaseNameDatas + ] + ) + def fx_get_path_basename_data( + self, + request: pytest.FixtureRequest, + ) -> tagGetPathBaseNameData: + assert isinstance(request, pytest.FixtureRequest) + assert type(request.param).__name__ == "tagGetPathBaseNameData" + return request.param + + def test_get_path_basename( + self, + os_ops_descr: OsOpsDescr, + fx_get_path_basename_data: tagGetPathBaseNameData, + ): + assert type(os_ops_descr) is OsOpsDescr + assert isinstance(os_ops_descr.os_ops, OsOperations) + assert type(fx_get_path_basename_data) is __class__.tagGetPathBaseNameData + + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + actual_value = os_ops.get_path_basename(fx_get_path_basename_data.value) + assert type(actual_value) is str + assert actual_value == fx_get_path_basename_data.result + return + + # -------------------------------------------------------------------- + def test_get_process_children__no_children( + self, + os_ops_descr: OsOpsDescr, + ): + assert type(os_ops_descr) is OsOpsDescr + assert isinstance(os_ops_descr.os_ops, OsOperations) + + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + sh_cmd = ["sh", "-c", "python3 -u -c 'import time;import os; print(os.getpid()); time.sleep(60)'"] + + p1 = os_ops.exec_command( + sh_cmd, + get_process=True, + encoding="utf-8", + ) + + assert isinstance(p1, subprocess.Popen) + assert p1.stdout is not None + + line = p1.stdout.readline() + assert line is not None + assert type(line) is str + line = line.rstrip() + assert line != "" + logging.info("pid is {}".format(line)) + + pid = int(line.rstrip()) + + childs = os_ops.get_process_children(pid) + assert childs is not None + assert type(childs) is list + assert len(childs) == 0 + return + + def test_get_process_children__with_child( + self, + os_ops_descr: OsOpsDescr, + ): + assert type(os_ops_descr) is OsOpsDescr + assert isinstance(os_ops_descr.os_ops, OsOperations) + + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + script = ( + "import time, os, subprocess; " + "s = str(os.getpid()); " + "p = subprocess.Popen('exec sleep 60', shell=True, stdout=subprocess.PIPE); " + "s += ':' + str(p.pid); " + "print(s, flush=True); " + "time.sleep(60)" + ) + sh_cmd = ["python3", "-u", "-c", script] + + p1 = os_ops.exec_command( + sh_cmd, + get_process=True, + encoding="utf-8", + ) + + assert isinstance(p1, subprocess.Popen) + assert p1.stdout is not None + + line = p1.stdout.readline() + assert line is not None + line = line.rstrip() + assert line != "" + + # "PARENT_PID:CHILD_PID" + parent_pid_str, expected_child_pid_str = line.split(":") + parent_pid = int(parent_pid_str) + expected_child_pid = int(expected_child_pid_str) + + logging.info(f"Parent PID from stdout: {parent_pid}") + logging.info(f"Expected Child PID from stdout: {expected_child_pid}") + + # A short pause to ensure registration in the OS + # time.sleep(0.5) + + childs = os_ops.get_process_children(parent_pid) + + assert childs is not None + assert isinstance(childs, list) + assert len(childs) == 1 + + actual_child_pid = childs[0].pid + logging.info(f"Actual Child PID from get_process_children: {actual_child_pid}") + + assert actual_child_pid == expected_child_pid + + p1.terminate() + p1.wait() + return + + def test_get_process_children__with_three_children( + self, + os_ops_descr: OsOpsDescr, + ): + assert type(os_ops_descr) is OsOpsDescr + assert isinstance(os_ops_descr.os_ops, OsOperations) + + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + script = ( + "import time, os, subprocess; " + "s = str(os.getpid()); " + "p1 = subprocess.Popen('exec sleep 60', shell=True, stdout=subprocess.PIPE); " + "p2 = subprocess.Popen('exec sleep 60', shell=True, stdout=subprocess.PIPE); " + "p3 = subprocess.Popen('exec sleep 60', shell=True, stdout=subprocess.PIPE); " + "s += ':' + str(p1.pid) + ':' + str(p2.pid) + ':' + str(p3.pid); " + "print(s, flush=True); " + "time.sleep(60)" + ) + sh_cmd = ["python3", "-u", "-c", script] + + p = os_ops.exec_command( + sh_cmd, + get_process=True, + encoding="utf-8", + ) + + assert isinstance(p, subprocess.Popen) + assert p.stdout is not None + + line = p.stdout.readline() + assert line is not None + line = line.rstrip() + assert line != "" + + parts = [int(x) for x in line.split(":")] + parent_pid = parts[0] + expected_child_pids = set(parts[1:]) + + logging.info(f"Parent PID: {parent_pid}") + logging.info(f"Expected Child PIDs: {expected_child_pids}") + + # A short pause to ensure registration in the OS + # time.sleep(0.5) + + childs = os_ops.get_process_children(parent_pid) + + assert childs is not None + assert isinstance(childs, list) + assert len(childs) == 3 + + actual_child_pids = {child.pid for child in childs} + logging.info(f"Actual Child PIDs: {actual_child_pids}") + + assert actual_child_pids == expected_child_pids + + p.terminate() + p.wait() + return + + def test_get_process_children__bad_pid( + self, + os_ops_descr: OsOpsDescr, + ): + assert type(os_ops_descr) is OsOpsDescr + assert isinstance(os_ops_descr.os_ops, OsOperations) + + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + C_BAD_PID = 999999876 # OK? )) + + with pytest.raises(expected_exception=Exception) as x: + os_ops.get_process_children(999999876) + + if type(os_ops).__name__ == "LocalOperations": + assert type(x.value) is psutil.NoSuchProcess + assert x.value.pid == C_BAD_PID + elif type(os_ops).__name__ == "RemoteOperations": + assert type(x.value) is ExecUtilException + msg1 = "Failed to get process children. Reason: No such process with PID {}.".format( + C_BAD_PID, + ) + assert msg1 in str(x) + assert x.value.exit_code == 1 + else: + raise RuntimeError("[BUG CHECK] Unknown os_ops type [{}]".format( + type(os_ops).__name__, + )) + return + + @dataclasses.dataclass + class tagReadLinesData_TXT: + sign: str + source: str + result: typing.List[str] + + sm_ReadLinesData_TXT: typing.List[tagReadLinesData_TXT] = [ + tagReadLinesData_TXT( + sign="empty", + source="", + result=[], + ), + tagReadLinesData_TXT( + sign="eol", + source="\n", + result=["\n"], + ), + tagReadLinesData_TXT( + sign="null_char", + source="\x00", + result=["\x00"], + ), + tagReadLinesData_TXT( + sign="null_char_and_eol", + source="\x00\n", + result=["\x00\n"], + ), + tagReadLinesData_TXT( + sign="one_char", + source="-", + result=["-"], + ), + tagReadLinesData_TXT( + sign="one_char_and_eol", + source="-\n", + result=["-\n"], + ), + tagReadLinesData_TXT( + sign="one_char_and_eol", + source="-\n", + result=["-\n"], + ), + tagReadLinesData_TXT( + sign="two_empty_lines", + source="\n\n", + result=["\n", "\n"], + ), + tagReadLinesData_TXT( + sign="two_lines_without_final_eol", + source="\n123", + result=["\n", "123"], + ), + tagReadLinesData_TXT( + sign="A0001", + source="12\x0034\n\x00123\nabcdefg\x00\x00", + result=["12\x0034\n", "\x00123\n", "abcdefg\x00\x00"], + ), + ] + + @pytest.fixture( + params=[ + pytest.param( + x, + id=x.sign, + ) + for x in sm_ReadLinesData_TXT + ] + ) + def readlines_data_txt(self, request: pytest.FixtureRequest) -> tagReadLinesData_TXT: + assert isinstance(request, pytest.FixtureRequest) + return request.param + + def test_readlines__TXT( + self, + os_ops_descr: OsOpsDescr, + readlines_data_txt: tagReadLinesData_TXT, + name_with_surprize: tagNameWithSurprize, + ): + assert type(os_ops_descr) is OsOpsDescr + assert isinstance(os_ops_descr.os_ops, OsOperations) + assert type(readlines_data_txt) is __class__.tagReadLinesData_TXT + assert type(name_with_surprize) is __class__.tagNameWithSurprize + + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + tmpfile = os_ops.mkstemp(name_with_surprize.value) + + os_ops.write(tmpfile, readlines_data_txt.source, binary=False) + + lines = os_ops.readlines(tmpfile) + + assert type(lines) is list + + assert lines == readlines_data_txt.result + return + + def test_readlines__BIN( + self, + os_ops_descr: OsOpsDescr, + readlines_data_txt: tagReadLinesData_TXT, + name_with_surprize: tagNameWithSurprize, + ): + assert type(os_ops_descr) is OsOpsDescr + assert isinstance(os_ops_descr.os_ops, OsOperations) + assert isinstance(readlines_data_txt, __class__.tagReadLinesData_TXT) + assert type(name_with_surprize) is __class__.tagNameWithSurprize + + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + tmpfile = os_ops.mkstemp(name_with_surprize.value) + + os_ops.write(tmpfile, readlines_data_txt.source, binary=True) + + lines = os_ops.readlines(tmpfile, binary=True) + + assert type(lines) is list + + result_bin = [s.encode() for s in readlines_data_txt.result] + + assert lines == result_bin + return + + def test_prove_environment_isolation( + self, + os_ops_descr: OsOpsDescr + ): + # + # Author: Marg G. (mark@google.com) + # + + assert type(os_ops_descr) is OsOpsDescr + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + logging.info("=================== COKANUM PROOF START ===================") + logging.info(f"Target environment type: [{os_ops_descr.sign}]") + + # 1. ะ—ะฐะฑะธั€ะฐะตะผ OS-RELEASE + try: + # ะ˜ัะฟะพะปัŒะทัƒะตะผ cat, ะบะพั‚ะพั€ั‹ะน ะผั‹ ัƒะถะต ะฟั€ะพะฒะตั€ะธะปะธ + os_release = os_ops.read("/etc/os-release") + assert type(os_release) is str + # ะ’ั‹ั‚ะฐัะบะธะฒะฐะตะผ ั‚ะพะปัŒะบะพ PRETTY_NAME ะดะปั ะบะพะผะฟะฐะบั‚ะฝะพัั‚ะธ ะฒ ะปะพะณะฐั… + pretty_name = "Unknown Linux" + for line in os_release.splitlines(): + if line.startswith("PRETTY_NAME="): + pretty_name = line.split("=")[1].strip('"') + break + logging.info(f"OS Platform detected : {pretty_name}") + except Exception as e: + logging.error(f"Failed to read os-release: {e}") + + # 2. ะŸั€ะพัั‚ะพ ะฒั‹ะฒะพะดะธะผ ัะตั‚ะตะฒะพะน ะปะฐะฝะดัˆะฐั„ั‚ "ะบะฐะบ ะตัั‚ัŒ" ะธ ะฝะต ะฟะฐั€ะธะผัั! + try: + cmd = ["ip", "address"] + ip_output = os_ops.exec_command(cmd, encoding="utf-8") + assert type(ip_output) is str + logging.info("Network interfaces info:") + # ะŸะตั‡ะฐั‚ะฐะตะผ ะฒััŽ ะฟั€ะพัั‚ั‹ะฝัŽ ั†ะตะปะธะบะพะผ, ะดะพะฑะฐะฒะธะฒ ะพั‚ัั‚ัƒะฟั‹ ะดะปั ะบั€ะฐัะพั‚ั‹ + for line in ip_output.splitlines(): + logging.info(f" {line}") + except Exception as e: + logging.error(f"Failed to get ip address output: {e}") + + logging.info("=================== COKANUM PROOF END =====================") + + # ะขะตัั‚ ะฒัะตะณะดะฐ ัƒัะฟะตัˆะฝั‹ะน, ะตะณะพ ั†ะตะปัŒ โ€” ะพัั‚ะฐะฒะธั‚ัŒ ะธัั‚ะพั€ะธั‡ะตัะบะธะน ัะปะตะด ะฒ ะปะพะณะฐั… ะณะธั‚ั…ะฐะฑะฐ + assert True + return + + def test_get_file_stat__common( + self, + os_ops_descr: OsOpsDescr, + name_with_surprize: tagNameWithSurprize, + ): + # + # Author: Marg G. (mark@google.com) + # + + assert type(os_ops_descr) is OsOpsDescr + assert type(name_with_surprize) is __class__.tagNameWithSurprize + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + # ะ“ะพั‚ะพะฒะธะผ ะฟัƒั‚ะธ ะธ ะดะฐะฝะฝั‹ะต + tmp_dir = os_ops.mkdtemp() + assert type(tmp_dir) is str + assert tmp_dir != "" + + filename = os_ops.build_path(tmp_dir, name_with_surprize.value) + initial_data = "Hello" + append_data = " World!!!" + + # ะ—ะฐะฟะธัั‹ะฒะฐะตะผ ะฝะฐั‡ะฐะปัŒะฝั‹ะต ะดะฐะฝะฝั‹ะต ะธ ะฟั€ะพะฒะตั€ัะตะผ ะธัั…ะพะดะฝั‹ะน stat + os_ops.write(filename, initial_data, truncate=True) + + stat1 = os_ops.get_file_stat(filename) + assert type(stat1) is dict + + # ะŸั€ะพะฒะตั€ัะตะผ ะฝะฐะปะธั‡ะธะต ะธ ั‚ะธะฟั‹ ัะฒะพะนัั‚ะฒ ั‡ะตั€ะตะท ะบะพะฝัั‚ะฐะฝั‚ั‹ + assert OsOperations.C_FILE_STAT_PROP__SIZE in stat1 + assert OsOperations.C_FILE_STAT_PROP__MTIME in stat1 + assert type(stat1[OsOperations.C_FILE_STAT_PROP__SIZE]) is int + assert isinstance(stat1[OsOperations.C_FILE_STAT_PROP__MTIME], datetime.datetime) + + # ะŸั€ะพะฒะตั€ัะตะผ ั‚ะพั‡ะฝั‹ะน ั€ะฐะทะผะตั€ + assert stat1[OsOperations.C_FILE_STAT_PROP__SIZE] == len(initial_data) + # ะŸั€ะพะฒะตั€ัะตะผ, ั‡ั‚ะพ ั‚ะฐะนะผะทะพะฝั‹ ัะฑั€ะพัะธะปะธััŒ ะฒ UTC + assert stat1[OsOperations.C_FILE_STAT_PROP__MTIME].tzinfo == datetime.timezone.utc + + logging.info("stat1.size: {}".format(stat1[OsOperations.C_FILE_STAT_PROP__SIZE])) + logging.info("stat1.mtime: {}".format(stat1[OsOperations.C_FILE_STAT_PROP__MTIME])) + + # ะ”ะตะปะฐะตะผ ะผะธะบั€ะพ-ะฟะฐัƒะทัƒ, ั‡ั‚ะพะฑั‹ ะพะฟะตั€ะฐั†ะธะพะฝะฝะฐั ัะธัั‚ะตะผะฐ ะทะฐั„ะธะบัะธั€ะพะฒะฐะปะฐ ะธะทะผะตะฝะตะฝะธะต ะฒั€ะตะผะตะฝะธ + time.sleep(1.1) + + # ะจะฐะณ 2: ะ”ะพะฟะธัั‹ะฒะฐะตะผ ะดะฐะฝะฝั‹ะต (ะธะทะผะตะฝัะตะผ ั€ะฐะทะผะตั€ ะธ mtime) + os_ops.write(filename, append_data, truncate=False) + + stat2 = os_ops.get_file_stat(filename) + assert type(stat2) is dict + + logging.info("stat2.size: {}".format(stat2[OsOperations.C_FILE_STAT_PROP__SIZE])) + logging.info("stat2.mtime: {}".format(stat2[OsOperations.C_FILE_STAT_PROP__MTIME])) + + # ะŸั€ะพะฒะตั€ัะตะผ, ั‡ั‚ะพ ะฝะพะฒั‹ะน ั€ะฐะทะผะตั€ ั€ะฐะฒะตะฝ ััƒะผะผะต ะดะฒัƒั… ัั‚ั€ะพะบ + expected_size = len(initial_data) + len(append_data) + assert stat2[OsOperations.C_FILE_STAT_PROP__SIZE] == expected_size + + # ะŸั€ะพะฒะตั€ัะตะผ, ั‡ั‚ะพ ะดะฐั‚ะฐ ะผะพะดะธั„ะธะบะฐั†ะธะธ ั‡ะตัั‚ะฝะพ ัะดะฒะธะฝัƒะปะฐััŒ ะฒะฟะตั€ะตะด + assert stat2[OsOperations.C_FILE_STAT_PROP__MTIME] > stat1[OsOperations.C_FILE_STAT_PROP__MTIME] + + logging.info("SUCCESS. File stat size and mtime verified successfully.") + + file_content = os_ops.read(filename, binary=False) + assert file_content == initial_data + append_data + + # ะŸั€ะพะฒะตั€ะบะฐ ะณั€ะฐะฝะธั‡ะฝะพะณะพ ัƒัะปะพะฒะธั (ะฝะตััƒั‰ะตัั‚ะฒัƒัŽั‰ะธะน ั„ะฐะนะป) + # ะœะตั‚ะพะด ะพะฑัะทะฐะฝ ะฒั‹ะบะธะดั‹ะฒะฐั‚ัŒ ะพัˆะธะฑะบัƒ (FileNotFoundError ะธะปะธ ExecUtilException) + fake_filename = os_ops.build_path(tmp_dir, "does_not_exist.txt") + + with pytest.raises(Exception) as x: + os_ops.get_file_stat(fake_filename) + + assert x is not None + assert not isinstance(x.value, AssertionError) + logging.info("SUCCESS. Error on missing file verified. Exception: {}".format(type(x.value).__name__)) + + # ------- + logging.info("cleanup") + os_ops.remove_file(filename) + os_ops.rmdir(tmp_dir) + return + + def test_path_normpath( + self, + os_ops_descr: OsOpsDescr, + ): + assert type(os_ops_descr) is OsOpsDescr + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + def LOCAL__check(value, expected) -> bool: + logging.info("Source path: [{}]".format(value)) + actual = os_ops.get_path_normpath(value) + if actual == expected: + logging.info("Result is OK: [{}].".format( + actual, + )) + else: + logging.error("Result is BAD: [{}]. Expected: [{}].".format( + actual, + expected, + )) + logging.info("") + return False + + logging.info("------------- test empty string") + LOCAL__check("", ".") + + logging.info("------------- test one char") + LOCAL__check("a", "a") + + logging.info("------------- test path") + LOCAL__check("a/b/c", "a/b/c") + return + + def test_path_normcase( + self, + os_ops_descr: OsOpsDescr, + ): + assert type(os_ops_descr) is OsOpsDescr + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + def LOCAL__check(value, expected) -> bool: + logging.info("Source path: [{}]".format(value)) + actual = os_ops.get_path_normcase(value) + if actual == expected: + logging.info("Result is OK: [{}].".format( + actual, + )) + else: + logging.error("Result is BAD: [{}]. Expected: [{}].".format( + actual, + expected, + )) + logging.info("") + return False + + logging.info("------------- test empty string") + LOCAL__check("", "") + + logging.info("------------- test one char") + LOCAL__check("a", "a") + + logging.info("------------- test path") + LOCAL__check("a/b/c", "a/b/c") + return + + def test_quote_path( + self, + os_ops_descr: OsOpsDescr, + ): + assert type(os_ops_descr) is OsOpsDescr + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + def LOCAL__check(value, expected) -> bool: + logging.info("Source path: [{}]".format(value)) + actual = os_ops.quote_path(value) + if actual == expected: + logging.info("Result is OK: [{}].".format( + actual, + )) + else: + logging.error("Result is BAD: [{}]. Expected: [{}].".format( + actual, + expected, + )) + logging.info("") + return False + + logging.info("------------- test empty string") + LOCAL__check("", "''") + + logging.info("------------- test one char") + LOCAL__check("a", "a") + + logging.info("------------- test path") + LOCAL__check("a/b/c", "a/b/c") + + logging.info("------------- test single quote") + LOCAL__check("'", "''\"'\"''") + + logging.info("------------- test double quote") + LOCAL__check("\"", "'\"'") + + logging.info("------------- test tilde") + LOCAL__check("~", "~") + + logging.info("------------- test tilde and slash") + LOCAL__check("~/", "~") + + logging.info("------------- test tilde and slash and path") + LOCAL__check("~/abc", "~/abc") + + logging.info("------------- test tilde and slash and path_with_spaces") + LOCAL__check("~/a b c", "~/'a b c'") + + logging.info("------------- test tilde and slash and path_with_spaces_and_final_slash") + LOCAL__check("~/a b c/", "~/'a b c/'") + + logging.info("------------- test tilde and slash and path_with_dquote") + LOCAL__check("~/a\"b c/", "~/'a\"b c/'") + + logging.info("------------- test tilde_with_user") + LOCAL__check("~root", "~root") + + logging.info("------------- test tilde_with_user and slash") + LOCAL__check("~root/", "~root") + + logging.info("------------- test tilde_with_user and path") + LOCAL__check("~root/a b c d e f ", "~root/'a b c d e f '") + + logging.info("------------- test tilde_with_user and path_and_final_slash") + LOCAL__check("~root/a b c d e f /", "~root/'a b c d e f /'") + + return + + def test_join_command_arguments( + self, + os_ops_descr: OsOpsDescr, + ): + assert type(os_ops_descr) is OsOpsDescr + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + def LOCAL__check(value, expected) -> bool: + logging.info("Source path: [{}]".format(value)) + actual = os_ops.join_command_arguments(value) + if actual == expected: + logging.info("Result is OK: [{}].".format( + actual, + )) + else: + logging.error("Result is BAD: [{}]. Expected: [{}].".format( + actual, + expected, + )) + logging.info("") + return False + + logging.info("------------- test empty string") + LOCAL__check(["cmd", ""], "cmd ''") + + logging.info("------------- test one char") + LOCAL__check(["cmd", "a"], "cmd a") + + logging.info("------------- test path") + LOCAL__check(["cmd", "a/b/c"], "cmd a/b/c") + + logging.info("------------- test single quote") + LOCAL__check(["cmd", "'"], "cmd ''\"'\"''") + + logging.info("------------- test double quote") + LOCAL__check(["cmd", "\""], "cmd '\"'") + + logging.info("------------- test tilde") + LOCAL__check(["cmd", "~"], "cmd ~") + + logging.info("------------- test tilde and slash") + LOCAL__check(["cmd", "~/"], "cmd ~") + + logging.info("------------- test tilde and slash and path") + LOCAL__check(["cmd", "~/abc"], "cmd ~/abc") + + logging.info("------------- test tilde and slash and path_with_spaces") + LOCAL__check(["cmd", "~/a b c"], "cmd ~/'a b c'") + + logging.info("------------- test tilde and slash and path_with_spaces_and_final_slash") + LOCAL__check(["cmd", "~/a b c/"], "cmd ~/'a b c/'") + + logging.info("------------- test tilde and slash and path_with_dquote") + LOCAL__check(["cmd", "~/a\"b c/"], "cmd ~/'a\"b c/'") + + logging.info("------------- test tilde_with_user") + LOCAL__check(["cmd", "~root"], "cmd ~root") + + logging.info("------------- test tilde_with_user and slash") + LOCAL__check(["cmd", "~root/"], "cmd ~root") + + logging.info("------------- test tilde_with_user and path") + LOCAL__check(["cmd", "~root/a b c d e f "], "cmd ~root/'a b c d e f '") + + logging.info("------------- test tilde_with_user and path_and_final_slash") + LOCAL__check(["cmd", "~root/a b c d e f /"], "cmd ~root/'a b c d e f /'") + + return + + def test_copytree__empty( + self, + os_ops_descr: OsOpsDescr, + name_with_surprize: tagNameWithSurprize, + ): + assert type(os_ops_descr) is OsOpsDescr + assert type(name_with_surprize) is __class__.tagNameWithSurprize + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + tmpdir = os_ops.mkdtemp(name_with_surprize.value) + + src = os_ops.build_path(tmpdir, "src") + dst = os_ops.build_path(tmpdir, "dst") + + os_ops.makedir(src) + copytree_r = os_ops.copytree(src, dst) + assert copytree_r == dst + + assert os_ops.path_exists(src) + assert os_ops.path_exists(dst) + + os_ops.rmdirs(tmpdir) + assert not os_ops.path_exists(tmpdir) + return + + def test_copytree__empty__relative( + self, + os_ops_descr: OsOpsDescr, + name_with_surprize: tagNameWithSurprize, + ): + assert type(os_ops_descr) is OsOpsDescr + assert type(name_with_surprize) is __class__.tagNameWithSurprize + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + cwd = os_ops.cwd() + + tmpdir = os_ops.mkdtemp(name_with_surprize.value) + + src = os_ops.build_path(tmpdir, "src") + dst = "copytree--" + uuid.uuid4().bytes.hex() + + dst_a = os_ops.build_path(cwd, dst) + + os_ops.makedir(src) + copytree_r = os_ops.copytree(src, dst) + assert copytree_r == dst + + assert os_ops.path_exists(src) + assert os_ops.path_exists(dst) + assert os_ops.path_exists(dst_a) + + os_ops.rmdirs(dst) + assert not os_ops.path_exists(dst) + + os_ops.rmdirs(tmpdir) + assert not os_ops.path_exists(tmpdir) + return + + def test_copytree__with_content( + self, + os_ops_descr: OsOpsDescr, + name_with_surprize: tagNameWithSurprize, + ): + assert type(os_ops_descr) is OsOpsDescr + assert type(name_with_surprize) is __class__.tagNameWithSurprize + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + tmpdir = os_ops.mkdtemp(name_with_surprize.value) + + src = os_ops.build_path(tmpdir, "src") + dst = os_ops.build_path(tmpdir, "dst") + + os_ops.makedir(src) + + src_file1 = os_ops.build_path(src, "file1.dat") + os_ops.write(src_file1, "abc") + src_dir1 = os_ops.build_path(src, "dir1") + os_ops.makedir(src_dir1) + src_dir1_file2 = os_ops.build_path(src_dir1, "file2") + os_ops.write(src_dir1_file2, "cba") + + copytree_r = os_ops.copytree(src, dst) + assert copytree_r == dst + + assert os_ops.path_exists(src) + assert os_ops.path_exists(dst) + + dst_file1 = os_ops.build_path(dst, "file1.dat") + assert os_ops.read(dst_file1, binary=False) == "abc" + dst_dir1 = os_ops.build_path(dst, "dir1") + assert os_ops.path_exists(dst_dir1) + dst_dir1_file2 = os_ops.build_path(dst_dir1, "file2") + assert os_ops.path_exists(dst_dir1_file2) + assert os_ops.read(dst_dir1_file2, binary=False) == "cba" + + os_ops.rmdirs(tmpdir) + assert not os_ops.path_exists(tmpdir) + return + + def test_copytree__with_content__relative( + self, + os_ops_descr: OsOpsDescr, + name_with_surprize: tagNameWithSurprize, + ): + assert type(os_ops_descr) is OsOpsDescr + assert type(name_with_surprize) is __class__.tagNameWithSurprize + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + cwd = os_ops.cwd() + + tmpdir = os_ops.mkdtemp(name_with_surprize.value) + + src = os_ops.build_path(tmpdir, "src") + dst = "copytree--" + uuid.uuid4().bytes.hex() + + dst_a = os_ops.build_path(cwd, dst) + + logging.info("src : [{}]".format(src)) + logging.info("dst : [{}]".format(dst)) + logging.info("dst_a: [{}]".format(dst_a)) + + os_ops.makedir(src) + + src_file1 = os_ops.build_path(src, "file1.dat") + os_ops.write(src_file1, "abc") + src_dir1 = os_ops.build_path(src, "dir1") + os_ops.makedir(src_dir1) + src_dir1_file2 = os_ops.build_path(src_dir1, "file2") + os_ops.write(src_dir1_file2, "cba") + + copytree_r = os_ops.copytree(src, dst) + assert copytree_r == dst + + assert os_ops.path_exists(src) + assert os_ops.path_exists(dst) + assert os_ops.path_exists(dst_a) + + dst_file1 = os_ops.build_path(dst, "file1.dat") + assert os_ops.read(dst_file1, binary=False) == "abc" + dst_dir1 = os_ops.build_path(dst, "dir1") + assert os_ops.path_exists(dst_dir1) + dst_dir1_file2 = os_ops.build_path(dst_dir1, "file2") + assert os_ops.path_exists(dst_dir1_file2) + assert os_ops.read(dst_dir1_file2, binary=False) == "cba" + + os_ops.rmdirs(dst) + assert not os_ops.path_exists(dst) + assert not os_ops.path_exists(dst_a) + + os_ops.rmdirs(tmpdir) + assert not os_ops.path_exists(tmpdir) + return + + def test_create_file__exclusive( + self, + os_ops_descr: OsOpsDescr, + name_with_surprize: tagNameWithSurprize, + ): + assert type(os_ops_descr) is OsOpsDescr + assert type(name_with_surprize) is __class__.tagNameWithSurprize + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + tmp_dir = os_ops.mkdtemp() + filename = os_ops.build_path(tmp_dir, name_with_surprize.value) + + # Step 1: The first attempt should be successful - the file is created from scratch + os_ops.create_file(filename) + assert os_ops.get_file_stat(filename)[OsOperations.C_FILE_STAT_PROP__SIZE] == 0 + logging.info("SUCCESS. New file created successfully.") + + # Step 2: The second attempt MUST throw an exception because the file already exists + with pytest.raises(Exception) as x: + os_ops.create_file(filename) + + assert x is not None + assert not isinstance(x.value, AssertionError) + logging.info("SUCCESS. Blocked recreation of an existing file. Exception: {}".format(type(x.value).__name__)) + + os_ops.remove_file(filename) + os_ops.rmdir(tmp_dir) + return + + def test_environ( + self, + os_ops_descr: OsOpsDescr, + ): + assert type(os_ops_descr) is OsOpsDescr + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + v = os_ops.environ("PATH") + assert type(v) is str + assert v != "" + assert not v.endswith("\n") + + v = os_ops.environ("DUMMY") + assert v is None + + C_AAAAAA = "AAAAAAAAAAAAA" + v = os_ops.environ("DUMMY; echo \"{}\";".format(C_AAAAAA)) + assert v is None + + # Adding a real nightmare for argument escaping to the test: + # Spaces, dollar signs, single and double quotes, ampersands. + evil_var_name = "MY_VAR 'single' \"double\" $PATH && rm -rf / ;" + + v = os_ops.environ(evil_var_name) + # printenv should honestly say that there is no such variable (exit_code 1) + # and return None without failing according to shell syntax. + assert v is None + + return + + def test_set_env__persistence( + self, + os_ops_descr: OsOpsDescr, + use_clone: bool, + ): + assert type(os_ops_descr) is OsOpsDescr + assert type(use_clone) is bool + + os_ops = __class__.helper__get_os_ops(use_clone, os_ops_descr) + assert isinstance(os_ops, OsOperations) + + var_name = "TEST_SET_ENV_MAGIC_VAR" + + assert os_ops.environ(var_name) is None + + try: + var_value = "Cokanum_Detected_123" + + # Step 1: Set the variable + os_ops.set_env(var_name, var_value) + + # Step 2: In a SEPARATE command, we try to read it using our new environ() + # The old remote_ops is guaranteed to return None and crash! + fetched_value = os_ops.environ(var_name) + + assert fetched_value == var_value, "Env variable persistence failed! Got: {}".format(fetched_value) + logging.info("SUCCESS. Environment variable persistence verified across commands.") + + printenv = os_ops.find_executable("printenv") + assert type(printenv) is str + assert printenv != "" + + cmd1 = [printenv, var_name] + + exec_r = os_ops.exec_command( + cmd1, + encoding="utf-8", + ) + assert type(exec_r) is str + exec_r = exec_r.rstrip() + assert fetched_value == var_value + + exec_r = os_ops.exec_command( + cmd1, + encoding="utf-8", + exec_env={ + var_name: "ABC", + } + ) + assert type(exec_r) is str + exec_r = exec_r.rstrip() + assert exec_r == "ABC" + + exec_r = os_ops.exec_command( + cmd1, + encoding="utf-8", + exec_env={ + var_name: None, + }, + ignore_errors=True, + verbose=True, + ) + assert type(exec_r) is tuple + assert len(exec_r) == 3 + assert exec_r[0] == 1 + finally: + os_ops.reset_env(var_name, None) + + # ---------------- + x = os_ops.environ("PATH") + + try: + os_ops.set_env("PATH", None) + + cmd2 = [printenv, "PATH"] + + exec_r = os_ops.exec_command( + cmd2, + encoding="utf-8", + ignore_errors=True, + verbose=True, + ) + assert type(exec_r) is tuple + assert len(exec_r) == 3 + assert exec_r[0] == 1 + finally: + os_ops.reset_env("PATH", x) + assert os_ops.environ("PATH") == x + + return + + def test_reset_env( + self, + os_ops_descr: OsOpsDescr, + use_clone: bool, + ): + assert type(os_ops_descr) is OsOpsDescr + assert type(use_clone) is bool + + os_ops = __class__.helper__get_os_ops(use_clone, os_ops_descr) + assert isinstance(os_ops, OsOperations) + + C_VAR_NAME = "PATH" + + clone = __class__.helper__create_clone_and_formal_check_it(os_ops) + + origin = os_ops.environ(C_VAR_NAME) + assert origin is not None + assert type(origin) is str + newvalue = origin + os_ops.pathsep + "aaaa" + + os_ops.set_env(C_VAR_NAME, newvalue) + + assert os_ops.environ(C_VAR_NAME) == newvalue + + os_ops.reset_env(C_VAR_NAME, origin) + + assert os_ops.environ(C_VAR_NAME) == origin + assert clone.environ(C_VAR_NAME) == origin + return + + def test_set_env__evil( + self, + os_ops_descr: OsOpsDescr, + ): + assert type(os_ops_descr) is OsOpsDescr + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + # --- INJECTION TEST VIA VARIABLE VALUE --- + # We stuff a hellish mixture of quotes, + # ampersands, and destructive shell commands into the variable value. + evil_value = "clean_val' && echo 'HACKED' && rm -rf / ; \" double_q" + evil_var = "TEST_EVIL_EXPORT_VAR" + + # Step 1: Store this crazy value in the state + os_ops.set_env(evil_var, evil_value) + + # Step 2: Call any harmless command (e.g., pwd or printenv) + # If quoting inside exec_command fails, the shell will execute the "echo 'HACKED'" chunk + # or crash with a quoting syntax error. + try: + fetched_evil = os_ops.environ(evil_var) + # printenv should return the string EXACTLY, without distortion and code execution + assert fetched_evil == evil_value, f"Evil env corruption! Got: {fetched_evil}" + logging.info("SUCCESS. Remote export variable value is completely bulletproof against shell injections.") + finally: + # Be sure to clean up after yourself + os_ops.set_env(evil_var, None) + return + + def test_set_env__thread_safety( + self, + os_ops_descr: OsOpsDescr, + use_clone: bool, + ): + """ + Test thread safety and data isolation of set_env and environ methods + when executed concurrently from multiple threads. + """ + assert type(os_ops_descr) is OsOpsDescr + assert type(use_clone) is bool + + os_ops = __class__.helper__get_os_ops(use_clone, os_ops_descr) + assert isinstance(os_ops, OsOperations) + + C_NUM_THREADS = 2 + + if type(os_ops).__name__ == "RemoteOperations": + C_NUM_ITERATIONS = 200 + else: + C_NUM_ITERATIONS = 2000 + + logging.info("NUM_ITERATIONS: {}".format(C_NUM_ITERATIONS)) + + # Queue for collecting exceptions from background threads + exceptions_queue = queue.Queue() + + # The function that each thread will run + def thread_worker( + thread_num: int, + var_name: str, + var_value: str, + iterations: int, + ): + logging.info("Hello from thread [{}].".format( + thread_num, + )) + + try: + nPass = 0 + while nPass < iterations: + if nPass > 0 and (nPass % 100) == 0: + logging.info("thread [{}]: {}".format( + thread_num, + nPass, + )) + + nPass += 1 + + # 1. The thread writes ITS own isolated variable + os_ops.set_env(var_name, var_value) + + # 2. The thread reads ITS own variable + fetched = os_ops.environ(var_name) + + # We check that someone else's thread hasn't overwritten our data + assert fetched == var_value, \ + "Thread isolation broken! Expected {!r}, got {!r}.".format( + var_value, + fetched, + ) + + # 3. Clean up after yourself + os_ops.reset_env(var_name, None) + assert os_ops.environ(var_name) is None + continue + + logging.info("thread [{}] finished ({})".format( + thread_num, + nPass, + )) + except Exception as e: + # If something goes wrong, we pass the error to the main test thread + exceptions_queue.put(e) + return + + total_error_count = 0 + + threads: typing.List[typing.Optional[threading.Thread]] = [None] * C_NUM_THREADS + assert len(threads) == C_NUM_THREADS + + for i in range(C_NUM_THREADS): + thread_name = "THREAD_{}_MAGIC_VAR".format(i) + thread_val = "Value_From_Thread_{}".format(i) + + assert threads[i] is None + + threads[i] = threading.Thread( + target=thread_worker, + args=(i, thread_name, thread_val, C_NUM_ITERATIONS), + ) + continue + + logging.info("Start threads...") + cActiveThreads = 0 + + try: + while cActiveThreads < C_NUM_THREADS: + logging.info("Start thread [{}] ...".format(cActiveThreads)) + + thread = threads[cActiveThreads] + assert thread is not None + assert isinstance(thread, threading.Thread) + thread.start() + cActiveThreads += 1 + continue + except Exception as e: + logging.error("Failed to start thread [{}]. Exception ({}): {}.".format( + cActiveThreads, + type(e).__name__, + e, + )) + + logging.info("Wait for finish of threads...") + cStoppedThread = 0 + + while cStoppedThread < cActiveThreads: + try: + logging.info("Wait for thread [{}] ...".format(cStoppedThread)) + thread = threads[cStoppedThread] + assert thread is not None + assert isinstance(thread, threading.Thread) + thread.join() + except Exception as e: + logging.error("Failed to stop thread [{}]. Exception ({}): {}.".format( + cStoppedThread, + type(e).__name__, + e, + )) + cStoppedThread += 1 + continue + + logging.info("Check errors queue...") + + # We check if any of the internal threads have crashed. + while not exceptions_queue.empty(): + total_error_count += 1 + err_msg = exceptions_queue.get() + logging.error(err_msg) + continue + + if total_error_count == 0: + logging.info("SUCCESS. Concurrent thread safety and environment isolation verified successfully.") + else: + logging.info("Total number of errors: {}".format(total_error_count)) + return + + @staticmethod + def helper__get_os_ops( + use_clone: bool, + os_ops_descr: OsOpsDescr, + ) -> OsOperations: + assert type(os_ops_descr) is OsOpsDescr + assert isinstance(os_ops_descr.os_ops, OsOperations) + + if (use_clone): + os_ops = __class__.helper__create_clone_and_formal_check_it( + os_ops_descr.os_ops, + ) + else: + os_ops = os_ops_descr.os_ops + + assert isinstance(os_ops, OsOperations) + return os_ops + + @staticmethod + def helper__create_clone_and_formal_check_it( + os_ops: OsOperations, + ) -> OsOperations: + assert isinstance(os_ops, OsOperations) + + clone = os_ops.create_clone() + assert clone is not None + assert type(clone) is type(os_ops) + + # it is safe + assert clone.remote == os_ops.remote + assert clone.username == os_ops.username + assert clone.ssh_key == os_ops.ssh_key + assert clone.host == os_ops.host + assert clone.port == os_ops.port + + return clone + + @staticmethod + def helper__bug_check__unknown_os_ops_type( + os_ops: OsOperations, + ) -> typing.NoReturn: + assert isinstance(os_ops, OsOperations) + + err_msg = "[BUG CHECK] Unknown os_ops type [{}].".format( + type(os_ops).__name__, + ) + raise RuntimeError(err_msg) diff --git a/tests/test_os_ops_local.py b/tests/test_os_ops_local.py new file mode 100644 index 00000000..f3d2a29c --- /dev/null +++ b/tests/test_os_ops_local.py @@ -0,0 +1,96 @@ +# coding: utf-8 +from .helpers.global_data import OsOpsDescr +from .helpers.global_data import OsOpsDescrs +from .helpers.global_data import OsOperations + +import os + +import pytest +import re + + +class TestOsOpsLocal: + @pytest.fixture + def os_ops_descr(self) -> OsOpsDescr: + assert type(OsOpsDescrs.sm_local_os_ops_descr) is OsOpsDescr + return OsOpsDescrs.sm_local_os_ops_descr + + def test_read__unknown_file( + self, + os_ops_descr: OsOpsDescr, + ): + """ + Test LocalOperations::read with unknown file. + """ + assert type(os_ops_descr) is OsOpsDescr + assert isinstance(os_ops_descr.os_ops, OsOperations) + + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + with pytest.raises(FileNotFoundError, match=re.escape("[Errno 2] No such file or directory: '/dummy'")): + os_ops.read("/dummy") + return + + def test_read_binary__spec__unk_file( + self, + os_ops_descr: OsOpsDescr, + ): + """ + Test LocalOperations::read_binary with unknown file. + """ + assert type(os_ops_descr) is OsOpsDescr + assert isinstance(os_ops_descr.os_ops, OsOperations) + + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + with pytest.raises( + FileNotFoundError, + match=re.escape("[Errno 2] No such file or directory: '/dummy'")): + os_ops.read_binary("/dummy", 0) + return + + def test_get_file_size__unk_file( + self, + os_ops_descr: OsOpsDescr, + ): + """ + Test LocalOperations::get_file_size. + """ + assert type(os_ops_descr) is OsOpsDescr + assert isinstance(os_ops_descr.os_ops, OsOperations) + + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + with pytest.raises(FileNotFoundError, match=re.escape("[Errno 2] No such file or directory: '/dummy'")): + os_ops.get_file_size("/dummy") + return + + def test_cwd( + self, + os_ops_descr: OsOpsDescr, + ): + """ + Test cwd. + """ + assert type(os_ops_descr) is OsOpsDescr + assert isinstance(os_ops_descr.os_ops, OsOperations) + + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + v = os_ops.cwd() + + assert v is not None + assert type(v) is str + + expectedValue = os.getcwd() + assert expectedValue is not None + assert type(expectedValue) is str + assert expectedValue != "" # research + + # Comp result + assert v == expectedValue + return diff --git a/tests/test_os_ops_remote.py b/tests/test_os_ops_remote.py new file mode 100755 index 00000000..7d4db8e9 --- /dev/null +++ b/tests/test_os_ops_remote.py @@ -0,0 +1,125 @@ +# coding: utf-8 + +from .helpers.global_data import OsOpsDescr +from .helpers.global_data import OsOpsDescrs +from .helpers.global_data import OsOperations +from .helpers.local_check import LocalCheck + +from src import ExecUtilException + +import pytest + + +class TestOsOpsRemote: + @pytest.fixture + def os_ops_descr(self) -> OsOpsDescr: + assert type(OsOpsDescrs.sm_remote_os_ops_descr) is OsOpsDescr + return OsOpsDescrs.sm_remote_os_ops_descr + + def test_rmdirs__try_to_delete_nonexist_path( + self, + os_ops_descr: OsOpsDescr, + ): + assert type(os_ops_descr) is OsOpsDescr + assert isinstance(os_ops_descr.os_ops, OsOperations) + + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + path = "/root/test_dir" + + assert os_ops.rmdirs(path, ignore_errors=False) is True + return + + def test_rmdirs__try_to_delete_file( + self, + os_ops_descr: OsOpsDescr, + ): + assert type(os_ops_descr) is OsOpsDescr + assert isinstance(os_ops_descr.os_ops, OsOperations) + + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + path = os_ops.mkstemp() + assert type(path) is str + LocalCheck.check_path_exists(os_ops, path) + assert os_ops.path_exists(path) + + with pytest.raises(ExecUtilException) as x: + os_ops.rmdirs(path, ignore_errors=False) + + LocalCheck.check_path_exists(os_ops, path) + assert os_ops.path_exists(path) + assert type(x.value) is ExecUtilException + assert type(x.value.description) is str + assert x.value.description == "Utility exited with non-zero code (20). Error: `cannot remove " + path + ": it is not a directory`" + assert x.value.message.startswith(x.value.description) + assert type(x.value.error) is str + assert x.value.error.strip() == "cannot remove " + path + ": it is not a directory" + assert type(x.value.exit_code) is int + assert x.value.exit_code == 20 + return + + def test_read__unknown_file( + self, + os_ops_descr: OsOpsDescr, + ): + """ + Test RemoteOperations::read with unknown file. + """ + assert type(os_ops_descr) is OsOpsDescr + assert isinstance(os_ops_descr.os_ops, OsOperations) + + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + with pytest.raises(ExecUtilException) as x: + os_ops.read("/dummy") + + assert "Utility exited with non-zero code (1)." in str(x.value) + assert "No such file or directory" in str(x.value) + assert "/dummy" in str(x.value) + return + + def test_read_binary__spec__unk_file( + self, + os_ops_descr: OsOpsDescr, + ): + """ + Test RemoteOperations::read_binary with unknown file. + """ + assert type(os_ops_descr) is OsOpsDescr + assert isinstance(os_ops_descr.os_ops, OsOperations) + + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + with pytest.raises(ExecUtilException) as x: + os_ops.read_binary("/dummy", 0) + + assert "Utility exited with non-zero code (1)." in str(x.value) + assert "No such file or directory" in str(x.value) + assert "/dummy" in str(x.value) + return + + def test_get_file_size__unk_file( + self, + os_ops_descr: OsOpsDescr, + ): + """ + Test RemoteOperations::get_file_size. + """ + assert type(os_ops_descr) is OsOpsDescr + assert isinstance(os_ops_descr.os_ops, OsOperations) + + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + with pytest.raises(ExecUtilException) as x: + os_ops.get_file_size("/dummy") + + assert "Utility exited with non-zero code (1)." in str(x.value) + assert "No such file or directory" in str(x.value) + assert "/dummy" in str(x.value) + return diff --git a/tests/test_raise_error.py b/tests/test_raise_error.py new file mode 100644 index 00000000..4e2a5a15 --- /dev/null +++ b/tests/test_raise_error.py @@ -0,0 +1,100 @@ +from src import InvalidOperationException +from src import NodeStatus +from src.raise_error import RaiseError + +import pytest +import typing + + +class TestRaiseError: + class tagTestData001: + node_status: NodeStatus + expected_msg: str + + def __init__( + self, + node_status: NodeStatus, + expected_msg: str, + ): + assert type(node_status) is NodeStatus + assert type(expected_msg) is str + self.node_status = node_status + self.expected_msg = expected_msg + return + + @property + def sign(self) -> str: + assert type(self.node_status) is NodeStatus + + msg = "status: {}".format(self.node_status) + return msg + + sm_Data001: typing.List[tagTestData001] = [ + tagTestData001( + NodeStatus.Uninitialized, + "Can't enumerate node child processes. Node is not initialized.", + ), + tagTestData001( + NodeStatus.Stopped, + "Can't enumerate node child processes. Node is not running.", + ), + ] + + @pytest.fixture( + params=sm_Data001, + ids=[x.sign for x in sm_Data001], + ) + def data001(self, request: pytest.FixtureRequest) -> tagTestData001: + assert isinstance(request, pytest.FixtureRequest) + assert type(request.param).__name__ == "tagTestData001" + return request.param + + def test_001__node_err__cant_enumerate_child_processes( + self, + data001: tagTestData001, + ): + assert type(data001) is __class__.tagTestData001 + + with pytest.raises(expected_exception=InvalidOperationException) as x: + RaiseError.node_err__cant_enumerate_child_processes( + data001.node_status + ) + + assert x is not None + assert str(x.value) == data001.expected_msg + return + + sm_Data002: typing.List[tagTestData001] = [ + tagTestData001( + NodeStatus.Uninitialized, + "Can't kill server process. Node is not initialized.", + ), + tagTestData001( + NodeStatus.Stopped, + "Can't kill server process. Node is not running.", + ), + ] + + @pytest.fixture( + params=sm_Data002, + ids=[x.sign for x in sm_Data002], + ) + def data002(self, request: pytest.FixtureRequest) -> tagTestData001: + assert isinstance(request, pytest.FixtureRequest) + assert type(request.param).__name__ == "tagTestData001" + return request.param + + def test_002__node_err__cant_kill( + self, + data002: tagTestData001, + ): + assert type(data002) is __class__.tagTestData001 + + with pytest.raises(expected_exception=InvalidOperationException) as x: + RaiseError.node_err__cant_kill( + data002.node_status + ) + + assert x is not None + assert str(x.value) == data002.expected_msg + return diff --git a/tests/test_remote.py b/tests/test_remote.py deleted file mode 100755 index 2e0f0676..00000000 --- a/tests/test_remote.py +++ /dev/null @@ -1,195 +0,0 @@ -import os - -import pytest - -from testgres import ExecUtilException -from testgres import RemoteOperations -from testgres import ConnectionParams - - -class TestRemoteOperations: - - @pytest.fixture(scope="function", autouse=True) - def setup(self): - conn_params = ConnectionParams(host=os.getenv('RDBMS_TESTPOOL1_HOST') or '172.18.0.3', - username='dev', - ssh_key=os.getenv( - 'RDBMS_TESTPOOL_SSHKEY') or '../../container_files/postgres/ssh/id_ed25519') - self.operations = RemoteOperations(conn_params) - - def test_exec_command_success(self): - """ - Test exec_command for successful command execution. - """ - cmd = "python3 --version" - response = self.operations.exec_command(cmd, wait_exit=True) - - assert b'Python 3.' in response - - def test_exec_command_failure(self): - """ - Test exec_command for command execution failure. - """ - cmd = "nonexistent_command" - try: - exit_status, result, error = self.operations.exec_command(cmd, verbose=True, wait_exit=True) - except ExecUtilException as e: - error = e.message - assert error == b'Utility exited with non-zero code. Error: bash: line 1: nonexistent_command: command not found\n' - - def test_is_executable_true(self): - """ - Test is_executable for an existing executable. - """ - cmd = "postgres" - response = self.operations.is_executable(cmd) - - assert response is True - - def test_is_executable_false(self): - """ - Test is_executable for a non-executable. - """ - cmd = "python" - response = self.operations.is_executable(cmd) - - assert response is False - - def test_makedirs_and_rmdirs_success(self): - """ - Test makedirs and rmdirs for successful directory creation and removal. - """ - cmd = "pwd" - pwd = self.operations.exec_command(cmd, wait_exit=True, encoding='utf-8').strip() - - path = "{}/test_dir".format(pwd) - - # Test makedirs - self.operations.makedirs(path) - assert self.operations.path_exists(path) - - # Test rmdirs - self.operations.rmdirs(path) - assert not self.operations.path_exists(path) - - def test_makedirs_and_rmdirs_failure(self): - """ - Test makedirs and rmdirs for directory creation and removal failure. - """ - # Try to create a directory in a read-only location - path = "/root/test_dir" - - # Test makedirs - with pytest.raises(Exception): - self.operations.makedirs(path) - - # Test rmdirs - try: - exit_status, result, error = self.operations.rmdirs(path, verbose=True) - except ExecUtilException as e: - error = e.message - assert error == b"Utility exited with non-zero code. Error: rm: cannot remove '/root/test_dir': Permission denied\n" - - def test_listdir(self): - """ - Test listdir for listing directory contents. - """ - path = "/etc" - files = self.operations.listdir(path) - - assert isinstance(files, list) - - def test_path_exists_true(self): - """ - Test path_exists for an existing path. - """ - path = "/etc" - response = self.operations.path_exists(path) - - assert response is True - - def test_path_exists_false(self): - """ - Test path_exists for a non-existing path. - """ - path = "/nonexistent_path" - response = self.operations.path_exists(path) - - assert response is False - - def test_write_text_file(self): - """ - Test write for writing data to a text file. - """ - filename = "/tmp/test_file.txt" - data = "Hello, world!" - - self.operations.write(filename, data, truncate=True) - self.operations.write(filename, data) - - response = self.operations.read(filename) - - assert response == data + data - - def test_write_binary_file(self): - """ - Test write for writing data to a binary file. - """ - filename = "/tmp/test_file.bin" - data = b"\x00\x01\x02\x03" - - self.operations.write(filename, data, binary=True, truncate=True) - - response = self.operations.read(filename, binary=True) - - assert response == data - - def test_read_text_file(self): - """ - Test read for reading data from a text file. - """ - filename = "/etc/hosts" - - response = self.operations.read(filename) - - assert isinstance(response, str) - - def test_read_binary_file(self): - """ - Test read for reading data from a binary file. - """ - filename = "/usr/bin/python3" - - response = self.operations.read(filename, binary=True) - - assert isinstance(response, bytes) - - def test_touch(self): - """ - Test touch for creating a new file or updating access and modification times of an existing file. - """ - filename = "/tmp/test_file.txt" - - self.operations.touch(filename) - - assert self.operations.isfile(filename) - - def test_isfile_true(self): - """ - Test isfile for an existing file. - """ - filename = "/etc/hosts" - - response = self.operations.isfile(filename) - - assert response is True - - def test_isfile_false(self): - """ - Test isfile for a non-existing file. - """ - filename = "/nonexistent_file.txt" - - response = self.operations.isfile(filename) - - assert response is False diff --git a/tests/test_simple.py b/tests/test_simple.py deleted file mode 100755 index 45c28a21..00000000 --- a/tests/test_simple.py +++ /dev/null @@ -1,1009 +0,0 @@ -#!/usr/bin/env python -# coding: utf-8 - -import os -import re -import subprocess -import tempfile -import testgres -import time -import six -import unittest -import psutil - -import logging.config - -from contextlib import contextmanager -from shutil import rmtree - -from testgres import \ - InitNodeException, \ - StartNodeException, \ - ExecUtilException, \ - BackupException, \ - QueryException, \ - TimeoutException, \ - TestgresException - -from testgres import \ - TestgresConfig, \ - configure_testgres, \ - scoped_config, \ - pop_config - -from testgres import \ - NodeStatus, \ - ProcessType, \ - IsolationLevel, \ - get_new_node - -from testgres import \ - get_bin_path, \ - get_pg_config, \ - get_pg_version - -from testgres import \ - First, \ - Any - -# NOTE: those are ugly imports -from testgres import bound_ports -from testgres.utils import PgVer -from testgres.node import ProcessProxy - - -def pg_version_ge(version): - cur_ver = PgVer(get_pg_version()) - min_ver = PgVer(version) - return cur_ver >= min_ver - - -def util_exists(util): - def good_properties(f): - return (os.path.exists(f) and # noqa: W504 - os.path.isfile(f) and # noqa: W504 - os.access(f, os.X_OK)) # yapf: disable - - # try to resolve it - if good_properties(get_bin_path(util)): - return True - - # check if util is in PATH - for path in os.environ["PATH"].split(os.pathsep): - if good_properties(os.path.join(path, util)): - return True - - -@contextmanager -def removing(f): - try: - yield f - finally: - if os.path.isfile(f): - os.remove(f) - elif os.path.isdir(f): - rmtree(f, ignore_errors=True) - - -class TestgresTests(unittest.TestCase): - def test_node_repr(self): - with get_new_node() as node: - pattern = r"PostgresNode\(name='.+', port=.+, base_dir='.+'\)" - self.assertIsNotNone(re.match(pattern, str(node))) - - def test_custom_init(self): - with get_new_node() as node: - # enable page checksums - node.init(initdb_params=['-k']).start() - - with get_new_node() as node: - node.init( - allow_streaming=True, - initdb_params=['--auth-local=reject', '--auth-host=reject']) - - hba_file = os.path.join(node.data_dir, 'pg_hba.conf') - with open(hba_file, 'r') as conf: - lines = conf.readlines() - - # check number of lines - self.assertGreaterEqual(len(lines), 6) - - # there should be no trust entries at all - self.assertFalse(any('trust' in s for s in lines)) - - def test_double_init(self): - with get_new_node().init() as node: - # can't initialize node more than once - with self.assertRaises(InitNodeException): - node.init() - - def test_init_after_cleanup(self): - with get_new_node() as node: - node.init().start().execute('select 1') - node.cleanup() - node.init().start().execute('select 1') - - @unittest.skipUnless(util_exists('pg_resetwal'), 'might be missing') - @unittest.skipUnless(pg_version_ge('9.6'), 'requires 9.6+') - def test_init_unique_system_id(self): - # this function exists in PostgreSQL 9.6+ - query = 'select system_identifier from pg_control_system()' - - with scoped_config(cache_initdb=False): - with get_new_node().init().start() as node0: - id0 = node0.execute(query)[0] - - with scoped_config(cache_initdb=True, - cached_initdb_unique=True) as config: - - self.assertTrue(config.cache_initdb) - self.assertTrue(config.cached_initdb_unique) - - # spawn two nodes; ids must be different - with get_new_node().init().start() as node1, \ - get_new_node().init().start() as node2: - - id1 = node1.execute(query)[0] - id2 = node2.execute(query)[0] - - # ids must increase - self.assertGreater(id1, id0) - self.assertGreater(id2, id1) - - def test_node_exit(self): - base_dir = None - - with self.assertRaises(QueryException): - with get_new_node().init() as node: - base_dir = node.base_dir - node.safe_psql('select 1') - - # we should save the DB for "debugging" - self.assertTrue(os.path.exists(base_dir)) - rmtree(base_dir, ignore_errors=True) - - with get_new_node().init() as node: - base_dir = node.base_dir - - # should have been removed by default - self.assertFalse(os.path.exists(base_dir)) - - def test_double_start(self): - with get_new_node().init().start() as node: - # can't start node more than once - node.start() - self.assertTrue(node.is_started) - - def test_uninitialized_start(self): - with get_new_node() as node: - # node is not initialized yet - with self.assertRaises(StartNodeException): - node.start() - - def test_restart(self): - with get_new_node() as node: - node.init().start() - - # restart, ok - res = node.execute('select 1') - self.assertEqual(res, [(1, )]) - node.restart() - res = node.execute('select 2') - self.assertEqual(res, [(2, )]) - - # restart, fail - with self.assertRaises(StartNodeException): - node.append_conf('pg_hba.conf', 'DUMMY') - node.restart() - - def test_reload(self): - with get_new_node() as node: - node.init().start() - - # change client_min_messages and save old value - cmm_old = node.execute('show client_min_messages') - node.append_conf(client_min_messages='DEBUG1') - - # reload config - node.reload() - - # check new value - cmm_new = node.execute('show client_min_messages') - self.assertEqual('debug1', cmm_new[0][0].lower()) - self.assertNotEqual(cmm_old, cmm_new) - - def test_pg_ctl(self): - with get_new_node() as node: - node.init().start() - - status = node.pg_ctl(['status']) - self.assertTrue('PID' in status) - - def test_status(self): - self.assertTrue(NodeStatus.Running) - self.assertFalse(NodeStatus.Stopped) - self.assertFalse(NodeStatus.Uninitialized) - - # check statuses after each operation - with get_new_node() as node: - self.assertEqual(node.pid, 0) - self.assertEqual(node.status(), NodeStatus.Uninitialized) - - node.init() - - self.assertEqual(node.pid, 0) - self.assertEqual(node.status(), NodeStatus.Stopped) - - node.start() - - self.assertNotEqual(node.pid, 0) - self.assertEqual(node.status(), NodeStatus.Running) - - node.stop() - - self.assertEqual(node.pid, 0) - self.assertEqual(node.status(), NodeStatus.Stopped) - - node.cleanup() - - self.assertEqual(node.pid, 0) - self.assertEqual(node.status(), NodeStatus.Uninitialized) - - def test_psql(self): - with get_new_node().init().start() as node: - - # check returned values (1 arg) - res = node.psql('select 1') - self.assertEqual(res, (0, b'1\n', b'')) - - # check returned values (2 args) - res = node.psql('postgres', 'select 2') - self.assertEqual(res, (0, b'2\n', b'')) - - # check returned values (named) - res = node.psql(query='select 3', dbname='postgres') - self.assertEqual(res, (0, b'3\n', b'')) - - # check returned values (1 arg) - res = node.safe_psql('select 4') - self.assertEqual(res, b'4\n') - - # check returned values (2 args) - res = node.safe_psql('postgres', 'select 5') - self.assertEqual(res, b'5\n') - - # check returned values (named) - res = node.safe_psql(query='select 6', dbname='postgres') - self.assertEqual(res, b'6\n') - - # check feeding input - node.safe_psql('create table horns (w int)') - node.safe_psql('copy horns from stdin (format csv)', - input=b"1\n2\n3\n\\.\n") - _sum = node.safe_psql('select sum(w) from horns') - self.assertEqual(_sum, b'6\n') - - # check psql's default args, fails - with self.assertRaises(QueryException): - node.psql() - - node.stop() - - # check psql on stopped node, fails - with self.assertRaises(QueryException): - node.safe_psql('select 1') - - def test_transactions(self): - with get_new_node().init().start() as node: - - with node.connect() as con: - con.begin() - con.execute('create table test(val int)') - con.execute('insert into test values (1)') - con.commit() - - con.begin() - con.execute('insert into test values (2)') - res = con.execute('select * from test order by val asc') - self.assertListEqual(res, [(1, ), (2, )]) - con.rollback() - - con.begin() - res = con.execute('select * from test') - self.assertListEqual(res, [(1, )]) - con.rollback() - - con.begin() - con.execute('drop table test') - con.commit() - - def test_control_data(self): - with get_new_node() as node: - - # node is not initialized yet - with self.assertRaises(ExecUtilException): - node.get_control_data() - - node.init() - data = node.get_control_data() - - # check returned dict - self.assertIsNotNone(data) - self.assertTrue(any('pg_control' in s for s in data.keys())) - - def test_backup_simple(self): - with get_new_node() as master: - - # enable streaming for backups - master.init(allow_streaming=True) - - # node must be running - with self.assertRaises(BackupException): - master.backup() - - # it's time to start node - master.start() - - # fill node with some data - master.psql('create table test as select generate_series(1, 4) i') - - with master.backup(xlog_method='stream') as backup: - with backup.spawn_primary().start() as slave: - res = slave.execute('select * from test order by i asc') - self.assertListEqual(res, [(1, ), (2, ), (3, ), (4, )]) - - def test_backup_multiple(self): - with get_new_node() as node: - node.init(allow_streaming=True).start() - - with node.backup(xlog_method='fetch') as backup1, \ - node.backup(xlog_method='fetch') as backup2: - - self.assertNotEqual(backup1.base_dir, backup2.base_dir) - - with node.backup(xlog_method='fetch') as backup: - with backup.spawn_primary('node1', destroy=False) as node1, \ - backup.spawn_primary('node2', destroy=False) as node2: - - self.assertNotEqual(node1.base_dir, node2.base_dir) - - def test_backup_exhaust(self): - with get_new_node() as node: - node.init(allow_streaming=True).start() - - with node.backup(xlog_method='fetch') as backup: - - # exhaust backup by creating new node - with backup.spawn_primary(): - pass - - # now let's try to create one more node - with self.assertRaises(BackupException): - backup.spawn_primary() - - def test_backup_wrong_xlog_method(self): - with get_new_node() as node: - node.init(allow_streaming=True).start() - - with self.assertRaises(BackupException, - msg='Invalid xlog_method "wrong"'): - node.backup(xlog_method='wrong') - - def test_pg_ctl_wait_option(self): - with get_new_node() as node: - node.init().start(wait=False) - while True: - try: - node.stop(wait=False) - break - except ExecUtilException: - # it's ok to get this exception here since node - # could be not started yet - pass - - def test_replicate(self): - with get_new_node() as node: - node.init(allow_streaming=True).start() - - with node.replicate().start() as replica: - res = replica.execute('select 1') - self.assertListEqual(res, [(1, )]) - - node.execute('create table test (val int)', commit=True) - - replica.catchup() - - res = node.execute('select * from test') - self.assertListEqual(res, []) - - @unittest.skipUnless(pg_version_ge('9.6'), 'requires 9.6+') - def test_synchronous_replication(self): - with get_new_node() as master: - old_version = not pg_version_ge('9.6') - - master.init(allow_streaming=True).start() - - if not old_version: - master.append_conf('synchronous_commit = remote_apply') - - # create standby - with master.replicate() as standby1, master.replicate() as standby2: - standby1.start() - standby2.start() - - # check formatting - self.assertEqual( - '1 ("{}", "{}")'.format(standby1.name, standby2.name), - str(First(1, (standby1, standby2)))) # yapf: disable - self.assertEqual( - 'ANY 1 ("{}", "{}")'.format(standby1.name, standby2.name), - str(Any(1, (standby1, standby2)))) # yapf: disable - - # set synchronous_standby_names - master.set_synchronous_standbys(First(2, [standby1, standby2])) - master.restart() - - # the following part of the test is only applicable to newer - # versions of PostgresQL - if not old_version: - master.safe_psql('create table abc(a int)') - - # Create a large transaction that will take some time to apply - # on standby to check that it applies synchronously - # (If set synchronous_commit to 'on' or other lower level then - # standby most likely won't catchup so fast and test will fail) - master.safe_psql( - 'insert into abc select generate_series(1, 1000000)') - res = standby1.safe_psql('select count(*) from abc') - self.assertEqual(res, b'1000000\n') - - @unittest.skipUnless(pg_version_ge('10'), 'requires 10+') - def test_logical_replication(self): - with get_new_node() as node1, get_new_node() as node2: - node1.init(allow_logical=True) - node1.start() - node2.init().start() - - create_table = 'create table test (a int, b int)' - node1.safe_psql(create_table) - node2.safe_psql(create_table) - - # create publication / create subscription - pub = node1.publish('mypub') - sub = node2.subscribe(pub, 'mysub') - - node1.safe_psql('insert into test values (1, 1), (2, 2)') - - # wait until changes apply on subscriber and check them - sub.catchup() - res = node2.execute('select * from test') - self.assertListEqual(res, [(1, 1), (2, 2)]) - - # disable and put some new data - sub.disable() - node1.safe_psql('insert into test values (3, 3)') - - # enable and ensure that data successfully transfered - sub.enable() - sub.catchup() - res = node2.execute('select * from test') - self.assertListEqual(res, [(1, 1), (2, 2), (3, 3)]) - - # Add new tables. Since we added "all tables" to publication - # (default behaviour of publish() method) we don't need - # to explicitely perform pub.add_tables() - create_table = 'create table test2 (c char)' - node1.safe_psql(create_table) - node2.safe_psql(create_table) - sub.refresh() - - # put new data - node1.safe_psql('insert into test2 values (\'a\'), (\'b\')') - sub.catchup() - res = node2.execute('select * from test2') - self.assertListEqual(res, [('a', ), ('b', )]) - - # drop subscription - sub.drop() - pub.drop() - - # create new publication and subscription for specific table - # (ommitting copying data as it's already done) - pub = node1.publish('newpub', tables=['test']) - sub = node2.subscribe(pub, 'newsub', copy_data=False) - - node1.safe_psql('insert into test values (4, 4)') - sub.catchup() - res = node2.execute('select * from test') - self.assertListEqual(res, [(1, 1), (2, 2), (3, 3), (4, 4)]) - - # explicitely add table - with self.assertRaises(ValueError): - pub.add_tables([]) # fail - pub.add_tables(['test2']) - node1.safe_psql('insert into test2 values (\'c\')') - sub.catchup() - res = node2.execute('select * from test2') - self.assertListEqual(res, [('a', ), ('b', )]) - - @unittest.skipUnless(pg_version_ge('10'), 'requires 10+') - def test_logical_catchup(self): - """ Runs catchup for 100 times to be sure that it is consistent """ - with get_new_node() as node1, get_new_node() as node2: - node1.init(allow_logical=True) - node1.start() - node2.init().start() - - create_table = 'create table test (key int primary key, val int); ' - node1.safe_psql(create_table) - node1.safe_psql('alter table test replica identity default') - node2.safe_psql(create_table) - - # create publication / create subscription - sub = node2.subscribe(node1.publish('mypub'), 'mysub') - - for i in range(0, 100): - node1.execute('insert into test values ({0}, {0})'.format(i)) - sub.catchup() - res = node2.execute('select * from test') - self.assertListEqual(res, [( - i, - i, - )]) - node1.execute('delete from test') - - @unittest.skipIf(pg_version_ge('10'), 'requires <10') - def test_logical_replication_fail(self): - with get_new_node() as node: - with self.assertRaises(InitNodeException): - node.init(allow_logical=True) - - def test_replication_slots(self): - with get_new_node() as node: - node.init(allow_streaming=True).start() - - with node.replicate(slot='slot1').start() as replica: - replica.execute('select 1') - - # cannot create new slot with the same name - with self.assertRaises(TestgresException): - node.replicate(slot='slot1') - - def test_incorrect_catchup(self): - with get_new_node() as node: - node.init(allow_streaming=True).start() - - # node has no master, can't catch up - with self.assertRaises(TestgresException): - node.catchup() - - def test_promotion(self): - with get_new_node() as master: - master.init().start() - master.safe_psql('create table abc(id serial)') - - with master.replicate().start() as replica: - master.stop() - replica.promote() - - # make standby becomes writable master - replica.safe_psql('insert into abc values (1)') - res = replica.safe_psql('select * from abc') - self.assertEqual(res, b'1\n') - - def test_dump(self): - query_create = 'create table test as select generate_series(1, 2) as val' - query_select = 'select * from test order by val asc' - - with get_new_node().init().start() as node1: - - node1.execute(query_create) - for format in ['plain', 'custom', 'directory', 'tar']: - with removing(node1.dump(format=format)) as dump: - with get_new_node().init().start() as node3: - if format == 'directory': - self.assertTrue(os.path.isdir(dump)) - else: - self.assertTrue(os.path.isfile(dump)) - # restore dump - node3.restore(filename=dump) - res = node3.execute(query_select) - self.assertListEqual(res, [(1, ), (2, )]) - - def test_users(self): - with get_new_node().init().start() as node: - node.psql('create role test_user login') - value = node.safe_psql('select 1', username='test_user') - self.assertEqual(value, b'1\n') - - def test_poll_query_until(self): - with get_new_node() as node: - node.init().start() - - get_time = 'select extract(epoch from now())' - check_time = 'select extract(epoch from now()) - {} >= 5' - - start_time = node.execute(get_time)[0][0] - node.poll_query_until(query=check_time.format(start_time)) - end_time = node.execute(get_time)[0][0] - - self.assertTrue(end_time - start_time >= 5) - - # check 0 columns - with self.assertRaises(QueryException): - node.poll_query_until( - query='select from pg_catalog.pg_class limit 1') - - # check None, fail - with self.assertRaises(QueryException): - node.poll_query_until(query='create table abc (val int)') - - # check None, ok - node.poll_query_until(query='create table def()', - expected=None) # returns nothing - - # check 0 rows equivalent to expected=None - node.poll_query_until( - query='select * from pg_catalog.pg_class where true = false', - expected=None) - - # check arbitrary expected value, fail - with self.assertRaises(TimeoutException): - node.poll_query_until(query='select 3', - expected=1, - max_attempts=3, - sleep_time=0.01) - - # check arbitrary expected value, ok - node.poll_query_until(query='select 2', expected=2) - - # check timeout - with self.assertRaises(TimeoutException): - node.poll_query_until(query='select 1 > 2', - max_attempts=3, - sleep_time=0.01) - - # check ProgrammingError, fail - with self.assertRaises(testgres.ProgrammingError): - node.poll_query_until(query='dummy1') - - # check ProgrammingError, ok - with self.assertRaises(TimeoutException): - node.poll_query_until(query='dummy2', - max_attempts=3, - sleep_time=0.01, - suppress={testgres.ProgrammingError}) - - # check 1 arg, ok - node.poll_query_until('select true') - - def test_logging(self): - logfile = tempfile.NamedTemporaryFile('w', delete=True) - - log_conf = { - 'version': 1, - 'handlers': { - 'file': { - 'class': 'logging.FileHandler', - 'filename': logfile.name, - 'formatter': 'base_format', - 'level': logging.DEBUG, - }, - }, - 'formatters': { - 'base_format': { - 'format': '%(node)-5s: %(message)s', - }, - }, - 'root': { - 'handlers': ('file', ), - 'level': 'DEBUG', - }, - } - - logging.config.dictConfig(log_conf) - - with scoped_config(use_python_logging=True): - node_name = 'master' - - with get_new_node(name=node_name) as master: - master.init().start() - - # execute a dummy query a few times - for i in range(20): - master.execute('select 1') - time.sleep(0.01) - - # let logging worker do the job - time.sleep(0.1) - - # check that master's port is found - with open(logfile.name, 'r') as log: - lines = log.readlines() - self.assertTrue(any(node_name in s for s in lines)) - - # test logger after stop/start/restart - master.stop() - master.start() - master.restart() - self.assertTrue(master._logger.is_alive()) - - @unittest.skipUnless(util_exists('pgbench'), 'might be missing') - def test_pgbench(self): - with get_new_node().init().start() as node: - - # initialize pgbench DB and run benchmarks - node.pgbench_init(scale=2, foreign_keys=True, - options=['-q']).pgbench_run(time=2) - - # run TPC-B benchmark - proc = node.pgbench(stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - options=['-T3']) - - out, _ = proc.communicate() - out = out.decode('utf-8') - - self.assertTrue('tps' in out) - - def test_pg_config(self): - # check same instances - a = get_pg_config() - b = get_pg_config() - self.assertEqual(id(a), id(b)) - - # save right before config change - c1 = get_pg_config() - - # modify setting for this scope - with scoped_config(cache_pg_config=False) as config: - - # sanity check for value - self.assertFalse(config.cache_pg_config) - - # save right after config change - c2 = get_pg_config() - - # check different instances after config change - self.assertNotEqual(id(c1), id(c2)) - - # check different instances - a = get_pg_config() - b = get_pg_config() - self.assertNotEqual(id(a), id(b)) - - def test_config_stack(self): - # no such option - with self.assertRaises(TypeError): - configure_testgres(dummy=True) - - # we have only 1 config in stack - with self.assertRaises(IndexError): - pop_config() - - d0 = TestgresConfig.cached_initdb_dir - d1 = 'dummy_abc' - d2 = 'dummy_def' - - with scoped_config(cached_initdb_dir=d1) as c1: - self.assertEqual(c1.cached_initdb_dir, d1) - - with scoped_config(cached_initdb_dir=d2) as c2: - - stack_size = len(testgres.config.config_stack) - - # try to break a stack - with self.assertRaises(TypeError): - with scoped_config(dummy=True): - pass - - self.assertEqual(c2.cached_initdb_dir, d2) - self.assertEqual(len(testgres.config.config_stack), stack_size) - - self.assertEqual(c1.cached_initdb_dir, d1) - - self.assertEqual(TestgresConfig.cached_initdb_dir, d0) - - def test_unix_sockets(self): - with get_new_node() as node: - node.init(unix_sockets=False, allow_streaming=True) - node.start() - - node.execute('select 1') - node.safe_psql('select 1') - - with node.replicate().start() as r: - r.execute('select 1') - r.safe_psql('select 1') - - def test_auto_name(self): - with get_new_node().init(allow_streaming=True).start() as m: - with m.replicate().start() as r: - - # check that nodes are running - self.assertTrue(m.status()) - self.assertTrue(r.status()) - - # check their names - self.assertNotEqual(m.name, r.name) - self.assertTrue('testgres' in m.name) - self.assertTrue('testgres' in r.name) - - def test_file_tail(self): - from testgres.utils import file_tail - - s1 = "the quick brown fox jumped over that lazy dog\n" - s2 = "abc\n" - s3 = "def\n" - - with tempfile.NamedTemporaryFile(mode='r+', delete=True) as f: - sz = 0 - while sz < 3 * 8192: - sz += len(s1) - f.write(s1) - f.write(s2) - f.write(s3) - - f.seek(0) - lines = file_tail(f, 3) - self.assertEqual(lines[0], s1) - self.assertEqual(lines[1], s2) - self.assertEqual(lines[2], s3) - - f.seek(0) - lines = file_tail(f, 1) - self.assertEqual(lines[0], s3) - - def test_isolation_levels(self): - with get_new_node().init().start() as node: - with node.connect() as con: - # string levels - con.begin('Read Uncommitted').commit() - con.begin('Read Committed').commit() - con.begin('Repeatable Read').commit() - con.begin('Serializable').commit() - - # enum levels - con.begin(IsolationLevel.ReadUncommitted).commit() - con.begin(IsolationLevel.ReadCommitted).commit() - con.begin(IsolationLevel.RepeatableRead).commit() - con.begin(IsolationLevel.Serializable).commit() - - # check wrong level - with self.assertRaises(QueryException): - con.begin('Garbage').commit() - - def test_ports_management(self): - # check that no ports have been bound yet - self.assertEqual(len(bound_ports), 0) - - with get_new_node() as node: - # check that we've just bound a port - self.assertEqual(len(bound_ports), 1) - - # check that bound_ports contains our port - port_1 = list(bound_ports)[0] - port_2 = node.port - self.assertEqual(port_1, port_2) - - # check that port has been freed successfully - self.assertEqual(len(bound_ports), 0) - - def test_exceptions(self): - str(StartNodeException('msg', [('file', 'lines')])) - str(ExecUtilException('msg', 'cmd', 1, 'out')) - str(QueryException('msg', 'query')) - - def test_version_management(self): - a = PgVer('10.0') - b = PgVer('10') - c = PgVer('9.6.5') - d = PgVer('15.0') - e = PgVer('15rc1') - f = PgVer('15beta4') - h = PgVer('15.3biha') - i = PgVer('15.3') - g = PgVer('15.3.1bihabeta1') - k = PgVer('15.3.1') - - self.assertTrue(a == b) - self.assertTrue(b > c) - self.assertTrue(a > c) - self.assertTrue(d > e) - self.assertTrue(e > f) - self.assertTrue(d > f) - self.assertTrue(h > f) - self.assertTrue(h == i) - self.assertTrue(g == k) - self.assertTrue(g > h) - - version = get_pg_version() - with get_new_node() as node: - self.assertTrue(isinstance(version, six.string_types)) - self.assertTrue(isinstance(node.version, PgVer)) - self.assertEqual(node.version, PgVer(version)) - - def test_child_pids(self): - master_processes = [ - ProcessType.AutovacuumLauncher, - ProcessType.BackgroundWriter, - ProcessType.Checkpointer, - ProcessType.StatsCollector, - ProcessType.WalSender, - ProcessType.WalWriter, - ] - - if pg_version_ge('10'): - master_processes.append(ProcessType.LogicalReplicationLauncher) - - repl_processes = [ - ProcessType.Startup, - ProcessType.WalReceiver, - ] - - with get_new_node().init().start() as master: - - # master node doesn't have a source walsender! - with self.assertRaises(TestgresException): - master.source_walsender - - with master.connect() as con: - self.assertGreater(con.pid, 0) - - with master.replicate().start() as replica: - - # test __str__ method - str(master.child_processes[0]) - - master_pids = master.auxiliary_pids - for ptype in master_processes: - self.assertIn(ptype, master_pids) - - replica_pids = replica.auxiliary_pids - for ptype in repl_processes: - self.assertIn(ptype, replica_pids) - - # there should be exactly 1 source walsender for replica - self.assertEqual(len(master_pids[ProcessType.WalSender]), 1) - pid1 = master_pids[ProcessType.WalSender][0] - pid2 = replica.source_walsender.pid - self.assertEqual(pid1, pid2) - - replica.stop() - - # there should be no walsender after we've stopped replica - with self.assertRaises(TestgresException): - replica.source_walsender - - def test_child_process_dies(self): - # test for FileNotFound exception during child_processes() function - with subprocess.Popen(["sleep", "60"]) as process: - self.assertEqual(process.poll(), None) - # collect list of processes currently running - children = psutil.Process(os.getpid()).children() - # kill a process, so received children dictionary becomes invalid - process.kill() - process.wait() - # try to handle children list -- missing processes will have ptype "ProcessType.Unknown" - [ProcessProxy(p) for p in children] - - -if __name__ == '__main__': - if os.environ.get('ALT_CONFIG'): - suite = unittest.TestSuite() - - # Small subset of tests for alternative configs (PG_BIN or PG_CONFIG) - suite.addTest(TestgresTests('test_pg_config')) - suite.addTest(TestgresTests('test_pg_ctl')) - suite.addTest(TestgresTests('test_psql')) - suite.addTest(TestgresTests('test_replicate')) - - print('Running tests for alternative config:') - for t in suite: - print(t) - print() - - runner = unittest.TextTestRunner() - runner.run(suite) - else: - unittest.main() diff --git a/tests/test_simple_remote.py b/tests/test_simple_remote.py deleted file mode 100755 index 1042f3c4..00000000 --- a/tests/test_simple_remote.py +++ /dev/null @@ -1,996 +0,0 @@ -#!/usr/bin/env python -# coding: utf-8 - -import os -import re -import subprocess -import tempfile - -import testgres -import time -import six -import unittest -import psutil - -import logging.config - -from contextlib import contextmanager - -from testgres.exceptions import \ - InitNodeException, \ - StartNodeException, \ - ExecUtilException, \ - BackupException, \ - QueryException, \ - TimeoutException, \ - TestgresException - -from testgres.config import \ - TestgresConfig, \ - configure_testgres, \ - scoped_config, \ - pop_config, testgres_config - -from testgres import \ - NodeStatus, \ - ProcessType, \ - IsolationLevel, \ - get_remote_node, \ - RemoteOperations - -from testgres import \ - get_bin_path, \ - get_pg_config, \ - get_pg_version - -from testgres import \ - First, \ - Any - -# NOTE: those are ugly imports -from testgres import bound_ports -from testgres.utils import PgVer -from testgres.node import ProcessProxy, ConnectionParams - -conn_params = ConnectionParams(host=os.getenv('RDBMS_TESTPOOL1_HOST') or '172.18.0.3', - username='dev', - ssh_key=os.getenv( - 'RDBMS_TESTPOOL_SSHKEY') or '../../container_files/postgres/ssh/id_ed25519') -os_ops = RemoteOperations(conn_params) -testgres_config.set_os_ops(os_ops=os_ops) - - -def pg_version_ge(version): - cur_ver = PgVer(get_pg_version()) - min_ver = PgVer(version) - return cur_ver >= min_ver - - -def util_exists(util): - def good_properties(f): - return (os_ops.path_exists(f) and # noqa: W504 - os_ops.isfile(f) and # noqa: W504 - os_ops.is_executable(f)) # yapf: disable - - # try to resolve it - if good_properties(get_bin_path(util)): - return True - - # check if util is in PATH - for path in os_ops.environ("PATH").split(os_ops.pathsep): - if good_properties(os.path.join(path, util)): - return True - - -@contextmanager -def removing(f): - try: - yield f - finally: - if os_ops.isfile(f): - os_ops.remove_file(f) - - elif os_ops.isdir(f): - os_ops.rmdirs(f, ignore_errors=True) - - -class TestgresRemoteTests(unittest.TestCase): - - def test_node_repr(self): - with get_remote_node(conn_params=conn_params) as node: - pattern = r"PostgresNode\(name='.+', port=.+, base_dir='.+'\)" - self.assertIsNotNone(re.match(pattern, str(node))) - - def test_custom_init(self): - with get_remote_node(conn_params=conn_params) as node: - # enable page checksums - node.init(initdb_params=['-k']).start() - - with get_remote_node(conn_params=conn_params) as node: - node.init( - allow_streaming=True, - initdb_params=['--auth-local=reject', '--auth-host=reject']) - - hba_file = os.path.join(node.data_dir, 'pg_hba.conf') - lines = os_ops.readlines(hba_file) - - # check number of lines - self.assertGreaterEqual(len(lines), 6) - - # there should be no trust entries at all - self.assertFalse(any('trust' in s for s in lines)) - - def test_double_init(self): - with get_remote_node(conn_params=conn_params).init() as node: - # can't initialize node more than once - with self.assertRaises(InitNodeException): - node.init() - - def test_init_after_cleanup(self): - with get_remote_node(conn_params=conn_params) as node: - node.init().start().execute('select 1') - node.cleanup() - node.init().start().execute('select 1') - - @unittest.skipUnless(util_exists('pg_resetwal'), 'might be missing') - @unittest.skipUnless(pg_version_ge('9.6'), 'requires 9.6+') - def test_init_unique_system_id(self): - # this function exists in PostgreSQL 9.6+ - query = 'select system_identifier from pg_control_system()' - - with scoped_config(cache_initdb=False): - with get_remote_node(conn_params=conn_params).init().start() as node0: - id0 = node0.execute(query)[0] - - with scoped_config(cache_initdb=True, - cached_initdb_unique=True) as config: - self.assertTrue(config.cache_initdb) - self.assertTrue(config.cached_initdb_unique) - - # spawn two nodes; ids must be different - with get_remote_node(conn_params=conn_params).init().start() as node1, \ - get_remote_node(conn_params=conn_params).init().start() as node2: - id1 = node1.execute(query)[0] - id2 = node2.execute(query)[0] - - # ids must increase - self.assertGreater(id1, id0) - self.assertGreater(id2, id1) - - def test_node_exit(self): - with self.assertRaises(QueryException): - with get_remote_node(conn_params=conn_params).init() as node: - base_dir = node.base_dir - node.safe_psql('select 1') - - # we should save the DB for "debugging" - self.assertTrue(os_ops.path_exists(base_dir)) - os_ops.rmdirs(base_dir, ignore_errors=True) - - with get_remote_node(conn_params=conn_params).init() as node: - base_dir = node.base_dir - - # should have been removed by default - self.assertFalse(os_ops.path_exists(base_dir)) - - def test_double_start(self): - with get_remote_node(conn_params=conn_params).init().start() as node: - # can't start node more than once - node.start() - self.assertTrue(node.is_started) - - def test_uninitialized_start(self): - with get_remote_node(conn_params=conn_params) as node: - # node is not initialized yet - with self.assertRaises(StartNodeException): - node.start() - - def test_restart(self): - with get_remote_node(conn_params=conn_params) as node: - node.init().start() - - # restart, ok - res = node.execute('select 1') - self.assertEqual(res, [(1,)]) - node.restart() - res = node.execute('select 2') - self.assertEqual(res, [(2,)]) - - # restart, fail - with self.assertRaises(StartNodeException): - node.append_conf('pg_hba.conf', 'DUMMY') - node.restart() - - def test_reload(self): - with get_remote_node(conn_params=conn_params) as node: - node.init().start() - - # change client_min_messages and save old value - cmm_old = node.execute('show client_min_messages') - node.append_conf(client_min_messages='DEBUG1') - - # reload config - node.reload() - - # check new value - cmm_new = node.execute('show client_min_messages') - self.assertEqual('debug1', cmm_new[0][0].lower()) - self.assertNotEqual(cmm_old, cmm_new) - - def test_pg_ctl(self): - with get_remote_node(conn_params=conn_params) as node: - node.init().start() - - status = node.pg_ctl(['status']) - self.assertTrue('PID' in status) - - def test_status(self): - self.assertTrue(NodeStatus.Running) - self.assertFalse(NodeStatus.Stopped) - self.assertFalse(NodeStatus.Uninitialized) - - # check statuses after each operation - with get_remote_node(conn_params=conn_params) as node: - self.assertEqual(node.pid, 0) - self.assertEqual(node.status(), NodeStatus.Uninitialized) - - node.init() - - self.assertEqual(node.pid, 0) - self.assertEqual(node.status(), NodeStatus.Stopped) - - node.start() - - self.assertNotEqual(node.pid, 0) - self.assertEqual(node.status(), NodeStatus.Running) - - node.stop() - - self.assertEqual(node.pid, 0) - self.assertEqual(node.status(), NodeStatus.Stopped) - - node.cleanup() - - self.assertEqual(node.pid, 0) - self.assertEqual(node.status(), NodeStatus.Uninitialized) - - def test_psql(self): - with get_remote_node(conn_params=conn_params).init().start() as node: - # check returned values (1 arg) - res = node.psql('select 1') - self.assertEqual(res, (0, b'1\n', b'')) - - # check returned values (2 args) - res = node.psql('postgres', 'select 2') - self.assertEqual(res, (0, b'2\n', b'')) - - # check returned values (named) - res = node.psql(query='select 3', dbname='postgres') - self.assertEqual(res, (0, b'3\n', b'')) - - # check returned values (1 arg) - res = node.safe_psql('select 4') - self.assertEqual(res, b'4\n') - - # check returned values (2 args) - res = node.safe_psql('postgres', 'select 5') - self.assertEqual(res, b'5\n') - - # check returned values (named) - res = node.safe_psql(query='select 6', dbname='postgres') - self.assertEqual(res, b'6\n') - - # check feeding input - node.safe_psql('create table horns (w int)') - node.safe_psql('copy horns from stdin (format csv)', - input=b"1\n2\n3\n\\.\n") - _sum = node.safe_psql('select sum(w) from horns') - self.assertEqual(_sum, b'6\n') - - # check psql's default args, fails - with self.assertRaises(QueryException): - node.psql() - - node.stop() - - # check psql on stopped node, fails - with self.assertRaises(QueryException): - node.safe_psql('select 1') - - def test_transactions(self): - with get_remote_node(conn_params=conn_params).init().start() as node: - with node.connect() as con: - con.begin() - con.execute('create table test(val int)') - con.execute('insert into test values (1)') - con.commit() - - con.begin() - con.execute('insert into test values (2)') - res = con.execute('select * from test order by val asc') - self.assertListEqual(res, [(1,), (2,)]) - con.rollback() - - con.begin() - res = con.execute('select * from test') - self.assertListEqual(res, [(1,)]) - con.rollback() - - con.begin() - con.execute('drop table test') - con.commit() - - def test_control_data(self): - with get_remote_node(conn_params=conn_params) as node: - # node is not initialized yet - with self.assertRaises(ExecUtilException): - node.get_control_data() - - node.init() - data = node.get_control_data() - - # check returned dict - self.assertIsNotNone(data) - self.assertTrue(any('pg_control' in s for s in data.keys())) - - def test_backup_simple(self): - with get_remote_node(conn_params=conn_params) as master: - # enable streaming for backups - master.init(allow_streaming=True) - - # node must be running - with self.assertRaises(BackupException): - master.backup() - - # it's time to start node - master.start() - - # fill node with some data - master.psql('create table test as select generate_series(1, 4) i') - - with master.backup(xlog_method='stream') as backup: - with backup.spawn_primary().start() as slave: - res = slave.execute('select * from test order by i asc') - self.assertListEqual(res, [(1,), (2,), (3,), (4,)]) - - def test_backup_multiple(self): - with get_remote_node(conn_params=conn_params) as node: - node.init(allow_streaming=True).start() - - with node.backup(xlog_method='fetch') as backup1, \ - node.backup(xlog_method='fetch') as backup2: - self.assertNotEqual(backup1.base_dir, backup2.base_dir) - - with node.backup(xlog_method='fetch') as backup: - with backup.spawn_primary('node1', destroy=False) as node1, \ - backup.spawn_primary('node2', destroy=False) as node2: - self.assertNotEqual(node1.base_dir, node2.base_dir) - - def test_backup_exhaust(self): - with get_remote_node(conn_params=conn_params) as node: - node.init(allow_streaming=True).start() - - with node.backup(xlog_method='fetch') as backup: - # exhaust backup by creating new node - with backup.spawn_primary(): - pass - - # now let's try to create one more node - with self.assertRaises(BackupException): - backup.spawn_primary() - - def test_backup_wrong_xlog_method(self): - with get_remote_node(conn_params=conn_params) as node: - node.init(allow_streaming=True).start() - - with self.assertRaises(BackupException, - msg='Invalid xlog_method "wrong"'): - node.backup(xlog_method='wrong') - - def test_pg_ctl_wait_option(self): - with get_remote_node(conn_params=conn_params) as node: - node.init().start(wait=False) - while True: - try: - node.stop(wait=False) - break - except ExecUtilException: - # it's ok to get this exception here since node - # could be not started yet - pass - - def test_replicate(self): - with get_remote_node(conn_params=conn_params) as node: - node.init(allow_streaming=True).start() - - with node.replicate().start() as replica: - res = replica.execute('select 1') - self.assertListEqual(res, [(1,)]) - - node.execute('create table test (val int)', commit=True) - - replica.catchup() - - res = node.execute('select * from test') - self.assertListEqual(res, []) - - @unittest.skipUnless(pg_version_ge('9.6'), 'requires 9.6+') - def test_synchronous_replication(self): - with get_remote_node(conn_params=conn_params) as master: - old_version = not pg_version_ge('9.6') - - master.init(allow_streaming=True).start() - - if not old_version: - master.append_conf('synchronous_commit = remote_apply') - - # create standby - with master.replicate() as standby1, master.replicate() as standby2: - standby1.start() - standby2.start() - - # check formatting - self.assertEqual( - '1 ("{}", "{}")'.format(standby1.name, standby2.name), - str(First(1, (standby1, standby2)))) # yapf: disable - self.assertEqual( - 'ANY 1 ("{}", "{}")'.format(standby1.name, standby2.name), - str(Any(1, (standby1, standby2)))) # yapf: disable - - # set synchronous_standby_names - master.set_synchronous_standbys(First(2, [standby1, standby2])) - master.restart() - - # the following part of the test is only applicable to newer - # versions of PostgresQL - if not old_version: - master.safe_psql('create table abc(a int)') - - # Create a large transaction that will take some time to apply - # on standby to check that it applies synchronously - # (If set synchronous_commit to 'on' or other lower level then - # standby most likely won't catchup so fast and test will fail) - master.safe_psql( - 'insert into abc select generate_series(1, 1000000)') - res = standby1.safe_psql('select count(*) from abc') - self.assertEqual(res, b'1000000\n') - - @unittest.skipUnless(pg_version_ge('10'), 'requires 10+') - def test_logical_replication(self): - with get_remote_node(conn_params=conn_params) as node1, get_remote_node(conn_params=conn_params) as node2: - node1.init(allow_logical=True) - node1.start() - node2.init().start() - - create_table = 'create table test (a int, b int)' - node1.safe_psql(create_table) - node2.safe_psql(create_table) - - # create publication / create subscription - pub = node1.publish('mypub') - sub = node2.subscribe(pub, 'mysub') - - node1.safe_psql('insert into test values (1, 1), (2, 2)') - - # wait until changes apply on subscriber and check them - sub.catchup() - res = node2.execute('select * from test') - self.assertListEqual(res, [(1, 1), (2, 2)]) - - # disable and put some new data - sub.disable() - node1.safe_psql('insert into test values (3, 3)') - - # enable and ensure that data successfully transfered - sub.enable() - sub.catchup() - res = node2.execute('select * from test') - self.assertListEqual(res, [(1, 1), (2, 2), (3, 3)]) - - # Add new tables. Since we added "all tables" to publication - # (default behaviour of publish() method) we don't need - # to explicitely perform pub.add_tables() - create_table = 'create table test2 (c char)' - node1.safe_psql(create_table) - node2.safe_psql(create_table) - sub.refresh() - - # put new data - node1.safe_psql('insert into test2 values (\'a\'), (\'b\')') - sub.catchup() - res = node2.execute('select * from test2') - self.assertListEqual(res, [('a',), ('b',)]) - - # drop subscription - sub.drop() - pub.drop() - - # create new publication and subscription for specific table - # (ommitting copying data as it's already done) - pub = node1.publish('newpub', tables=['test']) - sub = node2.subscribe(pub, 'newsub', copy_data=False) - - node1.safe_psql('insert into test values (4, 4)') - sub.catchup() - res = node2.execute('select * from test') - self.assertListEqual(res, [(1, 1), (2, 2), (3, 3), (4, 4)]) - - # explicitely add table - with self.assertRaises(ValueError): - pub.add_tables([]) # fail - pub.add_tables(['test2']) - node1.safe_psql('insert into test2 values (\'c\')') - sub.catchup() - res = node2.execute('select * from test2') - self.assertListEqual(res, [('a',), ('b',)]) - - @unittest.skipUnless(pg_version_ge('10'), 'requires 10+') - def test_logical_catchup(self): - """ Runs catchup for 100 times to be sure that it is consistent """ - with get_remote_node(conn_params=conn_params) as node1, get_remote_node(conn_params=conn_params) as node2: - node1.init(allow_logical=True) - node1.start() - node2.init().start() - - create_table = 'create table test (key int primary key, val int); ' - node1.safe_psql(create_table) - node1.safe_psql('alter table test replica identity default') - node2.safe_psql(create_table) - - # create publication / create subscription - sub = node2.subscribe(node1.publish('mypub'), 'mysub') - - for i in range(0, 100): - node1.execute('insert into test values ({0}, {0})'.format(i)) - sub.catchup() - res = node2.execute('select * from test') - self.assertListEqual(res, [( - i, - i, - )]) - node1.execute('delete from test') - - @unittest.skipIf(pg_version_ge('10'), 'requires <10') - def test_logical_replication_fail(self): - with get_remote_node(conn_params=conn_params) as node: - with self.assertRaises(InitNodeException): - node.init(allow_logical=True) - - def test_replication_slots(self): - with get_remote_node(conn_params=conn_params) as node: - node.init(allow_streaming=True).start() - - with node.replicate(slot='slot1').start() as replica: - replica.execute('select 1') - - # cannot create new slot with the same name - with self.assertRaises(TestgresException): - node.replicate(slot='slot1') - - def test_incorrect_catchup(self): - with get_remote_node(conn_params=conn_params) as node: - node.init(allow_streaming=True).start() - - # node has no master, can't catch up - with self.assertRaises(TestgresException): - node.catchup() - - def test_promotion(self): - with get_remote_node(conn_params=conn_params) as master: - master.init().start() - master.safe_psql('create table abc(id serial)') - - with master.replicate().start() as replica: - master.stop() - replica.promote() - - # make standby becomes writable master - replica.safe_psql('insert into abc values (1)') - res = replica.safe_psql('select * from abc') - self.assertEqual(res, b'1\n') - - def test_dump(self): - query_create = 'create table test as select generate_series(1, 2) as val' - query_select = 'select * from test order by val asc' - - with get_remote_node(conn_params=conn_params).init().start() as node1: - - node1.execute(query_create) - for format in ['plain', 'custom', 'directory', 'tar']: - with removing(node1.dump(format=format)) as dump: - with get_remote_node(conn_params=conn_params).init().start() as node3: - if format == 'directory': - self.assertTrue(os_ops.isdir(dump)) - else: - self.assertTrue(os_ops.isfile(dump)) - # restore dump - node3.restore(filename=dump) - res = node3.execute(query_select) - self.assertListEqual(res, [(1,), (2,)]) - - def test_users(self): - with get_remote_node(conn_params=conn_params).init().start() as node: - node.psql('create role test_user login') - value = node.safe_psql('select 1', username='test_user') - self.assertEqual(b'1\n', value) - - def test_poll_query_until(self): - with get_remote_node(conn_params=conn_params) as node: - node.init().start() - - get_time = 'select extract(epoch from now())' - check_time = 'select extract(epoch from now()) - {} >= 5' - - start_time = node.execute(get_time)[0][0] - node.poll_query_until(query=check_time.format(start_time)) - end_time = node.execute(get_time)[0][0] - - self.assertTrue(end_time - start_time >= 5) - - # check 0 columns - with self.assertRaises(QueryException): - node.poll_query_until( - query='select from pg_catalog.pg_class limit 1') - - # check None, fail - with self.assertRaises(QueryException): - node.poll_query_until(query='create table abc (val int)') - - # check None, ok - node.poll_query_until(query='create table def()', - expected=None) # returns nothing - - # check 0 rows equivalent to expected=None - node.poll_query_until( - query='select * from pg_catalog.pg_class where true = false', - expected=None) - - # check arbitrary expected value, fail - with self.assertRaises(TimeoutException): - node.poll_query_until(query='select 3', - expected=1, - max_attempts=3, - sleep_time=0.01) - - # check arbitrary expected value, ok - node.poll_query_until(query='select 2', expected=2) - - # check timeout - with self.assertRaises(TimeoutException): - node.poll_query_until(query='select 1 > 2', - max_attempts=3, - sleep_time=0.01) - - # check ProgrammingError, fail - with self.assertRaises(testgres.ProgrammingError): - node.poll_query_until(query='dummy1') - - # check ProgrammingError, ok - with self.assertRaises(TimeoutException): - node.poll_query_until(query='dummy2', - max_attempts=3, - sleep_time=0.01, - suppress={testgres.ProgrammingError}) - - # check 1 arg, ok - node.poll_query_until('select true') - - def test_logging(self): - # FAIL - logfile = tempfile.NamedTemporaryFile('w', delete=True) - - log_conf = { - 'version': 1, - 'handlers': { - 'file': { - 'class': 'logging.FileHandler', - 'filename': logfile.name, - 'formatter': 'base_format', - 'level': logging.DEBUG, - }, - }, - 'formatters': { - 'base_format': { - 'format': '%(node)-5s: %(message)s', - }, - }, - 'root': { - 'handlers': ('file',), - 'level': 'DEBUG', - }, - } - - logging.config.dictConfig(log_conf) - - with scoped_config(use_python_logging=True): - node_name = 'master' - - with get_remote_node(name=node_name) as master: - master.init().start() - - # execute a dummy query a few times - for i in range(20): - master.execute('select 1') - time.sleep(0.01) - - # let logging worker do the job - time.sleep(0.1) - - # check that master's port is found - with open(logfile.name, 'r') as log: - lines = log.readlines() - self.assertTrue(any(node_name in s for s in lines)) - - # test logger after stop/start/restart - master.stop() - master.start() - master.restart() - self.assertTrue(master._logger.is_alive()) - - @unittest.skipUnless(util_exists('pgbench'), 'might be missing') - def test_pgbench(self): - with get_remote_node(conn_params=conn_params).init().start() as node: - # initialize pgbench DB and run benchmarks - node.pgbench_init(scale=2, foreign_keys=True, - options=['-q']).pgbench_run(time=2) - - # run TPC-B benchmark - proc = node.pgbench(stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - options=['-T3']) - out = proc.communicate()[0] - self.assertTrue(b'tps = ' in out) - - def test_pg_config(self): - # check same instances - a = get_pg_config() - b = get_pg_config() - self.assertEqual(id(a), id(b)) - - # save right before config change - c1 = get_pg_config() - # modify setting for this scope - with scoped_config(cache_pg_config=False) as config: - # sanity check for value - self.assertFalse(config.cache_pg_config) - - # save right after config change - c2 = get_pg_config() - - # check different instances after config change - self.assertNotEqual(id(c1), id(c2)) - - # check different instances - a = get_pg_config() - b = get_pg_config() - self.assertNotEqual(id(a), id(b)) - - def test_config_stack(self): - # no such option - with self.assertRaises(TypeError): - configure_testgres(dummy=True) - - # we have only 1 config in stack - with self.assertRaises(IndexError): - pop_config() - - d0 = TestgresConfig.cached_initdb_dir - d1 = 'dummy_abc' - d2 = 'dummy_def' - - with scoped_config(cached_initdb_dir=d1) as c1: - self.assertEqual(c1.cached_initdb_dir, d1) - - with scoped_config(cached_initdb_dir=d2) as c2: - stack_size = len(testgres.config.config_stack) - - # try to break a stack - with self.assertRaises(TypeError): - with scoped_config(dummy=True): - pass - - self.assertEqual(c2.cached_initdb_dir, d2) - self.assertEqual(len(testgres.config.config_stack), stack_size) - - self.assertEqual(c1.cached_initdb_dir, d1) - - self.assertEqual(TestgresConfig.cached_initdb_dir, d0) - - def test_unix_sockets(self): - with get_remote_node(conn_params=conn_params) as node: - node.init(unix_sockets=False, allow_streaming=True) - node.start() - - res_exec = node.execute('select 1') - res_psql = node.safe_psql('select 1') - self.assertEqual(res_exec, [(1,)]) - self.assertEqual(res_psql, b'1\n') - - with node.replicate().start() as r: - res_exec = r.execute('select 1') - res_psql = r.safe_psql('select 1') - self.assertEqual(res_exec, [(1,)]) - self.assertEqual(res_psql, b'1\n') - - def test_auto_name(self): - with get_remote_node(conn_params=conn_params).init(allow_streaming=True).start() as m: - with m.replicate().start() as r: - # check that nodes are running - self.assertTrue(m.status()) - self.assertTrue(r.status()) - - # check their names - self.assertNotEqual(m.name, r.name) - self.assertTrue('testgres' in m.name) - self.assertTrue('testgres' in r.name) - - def test_file_tail(self): - from testgres.utils import file_tail - - s1 = "the quick brown fox jumped over that lazy dog\n" - s2 = "abc\n" - s3 = "def\n" - - with tempfile.NamedTemporaryFile(mode='r+', delete=True) as f: - sz = 0 - while sz < 3 * 8192: - sz += len(s1) - f.write(s1) - f.write(s2) - f.write(s3) - - f.seek(0) - lines = file_tail(f, 3) - self.assertEqual(lines[0], s1) - self.assertEqual(lines[1], s2) - self.assertEqual(lines[2], s3) - - f.seek(0) - lines = file_tail(f, 1) - self.assertEqual(lines[0], s3) - - def test_isolation_levels(self): - with get_remote_node(conn_params=conn_params).init().start() as node: - with node.connect() as con: - # string levels - con.begin('Read Uncommitted').commit() - con.begin('Read Committed').commit() - con.begin('Repeatable Read').commit() - con.begin('Serializable').commit() - - # enum levels - con.begin(IsolationLevel.ReadUncommitted).commit() - con.begin(IsolationLevel.ReadCommitted).commit() - con.begin(IsolationLevel.RepeatableRead).commit() - con.begin(IsolationLevel.Serializable).commit() - - # check wrong level - with self.assertRaises(QueryException): - con.begin('Garbage').commit() - - def test_ports_management(self): - # check that no ports have been bound yet - self.assertEqual(len(bound_ports), 0) - - with get_remote_node(conn_params=conn_params) as node: - # check that we've just bound a port - self.assertEqual(len(bound_ports), 1) - - # check that bound_ports contains our port - port_1 = list(bound_ports)[0] - port_2 = node.port - self.assertEqual(port_1, port_2) - - # check that port has been freed successfully - self.assertEqual(len(bound_ports), 0) - - def test_exceptions(self): - str(StartNodeException('msg', [('file', 'lines')])) - str(ExecUtilException('msg', 'cmd', 1, 'out')) - str(QueryException('msg', 'query')) - - def test_version_management(self): - a = PgVer('10.0') - b = PgVer('10') - c = PgVer('9.6.5') - d = PgVer('15.0') - e = PgVer('15rc1') - f = PgVer('15beta4') - - self.assertTrue(a == b) - self.assertTrue(b > c) - self.assertTrue(a > c) - self.assertTrue(d > e) - self.assertTrue(e > f) - self.assertTrue(d > f) - - version = get_pg_version() - with get_remote_node(conn_params=conn_params) as node: - self.assertTrue(isinstance(version, six.string_types)) - self.assertTrue(isinstance(node.version, PgVer)) - self.assertEqual(node.version, PgVer(version)) - - def test_child_pids(self): - master_processes = [ - ProcessType.AutovacuumLauncher, - ProcessType.BackgroundWriter, - ProcessType.Checkpointer, - ProcessType.StatsCollector, - ProcessType.WalSender, - ProcessType.WalWriter, - ] - - if pg_version_ge('10'): - master_processes.append(ProcessType.LogicalReplicationLauncher) - - repl_processes = [ - ProcessType.Startup, - ProcessType.WalReceiver, - ] - - with get_remote_node(conn_params=conn_params).init().start() as master: - - # master node doesn't have a source walsender! - with self.assertRaises(TestgresException): - master.source_walsender - - with master.connect() as con: - self.assertGreater(con.pid, 0) - - with master.replicate().start() as replica: - - # test __str__ method - str(master.child_processes[0]) - - master_pids = master.auxiliary_pids - for ptype in master_processes: - self.assertIn(ptype, master_pids) - - replica_pids = replica.auxiliary_pids - for ptype in repl_processes: - self.assertIn(ptype, replica_pids) - - # there should be exactly 1 source walsender for replica - self.assertEqual(len(master_pids[ProcessType.WalSender]), 1) - pid1 = master_pids[ProcessType.WalSender][0] - pid2 = replica.source_walsender.pid - self.assertEqual(pid1, pid2) - - replica.stop() - - # there should be no walsender after we've stopped replica - with self.assertRaises(TestgresException): - replica.source_walsender - - def test_child_process_dies(self): - # test for FileNotFound exception during child_processes() function - with subprocess.Popen(["sleep", "60"]) as process: - self.assertEqual(process.poll(), None) - # collect list of processes currently running - children = psutil.Process(os.getpid()).children() - # kill a process, so received children dictionary becomes invalid - process.kill() - process.wait() - # try to handle children list -- missing processes will have ptype "ProcessType.Unknown" - [ProcessProxy(p) for p in children] - - -if __name__ == '__main__': - if os_ops.environ('ALT_CONFIG'): - suite = unittest.TestSuite() - - # Small subset of tests for alternative configs (PG_BIN or PG_CONFIG) - suite.addTest(TestgresRemoteTests('test_pg_config')) - suite.addTest(TestgresRemoteTests('test_pg_ctl')) - suite.addTest(TestgresRemoteTests('test_psql')) - suite.addTest(TestgresRemoteTests('test_replicate')) - - print('Running tests for alternative config:') - for t in suite: - print(t) - print() - - runner = unittest.TextTestRunner() - runner.run(suite) - else: - unittest.main() diff --git a/tests/test_testgres_common.py b/tests/test_testgres_common.py new file mode 100644 index 00000000..9ad0c2e2 --- /dev/null +++ b/tests/test_testgres_common.py @@ -0,0 +1,3115 @@ +from __future__ import annotations + +from .helpers.global_data import OsOpsDescrs +from .helpers.global_data import OsOpsDescr +from .helpers.global_data import PostgresNodeService +from .helpers.global_data import PostgresNodeServices +from .helpers.global_data import OsOperations +from .helpers.global_data import PortManager +from .helpers.pg_cfg_os_ops import PgCfgOsOps + +from src import __version__ as testgres_version +from src.node import PgVer +from src.node import PostgresNode +from src.node import NodeConnection +from src.node import PostgresNodeLogReader +from src.node import PostgresNodeUtils +from src.node import ProcessProxy +from src.utils import get_pg_version2 +from src.utils import file_tail +from src.utils import get_bin_path2 +from src.utils import execute_utility2 +from src.defaults import default_username +from src.defaults import default_username2 +from src.config import testgres_config as tconf +from src import ProcessType +from src import NodeStatus +from src import IsolationLevel +from src import NodeApp +from src import enums + +# New name prevents to collect test-functions in TestgresException and fixes +# the problem with pytest warning. +from src import TestgresException as testgres_TestgresException + +from src import InitNodeException +from src import StartNodeException +from src import QueryException +from src import ExecUtilException +from src import QueryTimeoutException +from src import InvalidOperationException +from src import BackupException +from src import ProgrammingError +from src import scoped_config +from src import First, Any + +from contextlib import contextmanager + +import pytest +import six +import logging +import time +import tempfile +import uuid +import os +import re +import subprocess +import typing +import types +import psutil +import testgres.postgres_configuration as testgres_pgconf + +from packaging.version import Version + + +@contextmanager +def removing(os_ops: OsOperations, f): + assert isinstance(os_ops, OsOperations) + + try: + yield f + finally: + if os_ops.isfile(f): + os_ops.remove_file(f) + + elif os_ops.isdir(f): + os_ops.rmdirs(f, ignore_errors=True) + + +class TestTestgresCommon: + sm_os_ops_descrs: typing.List[OsOpsDescr] = [ + OsOpsDescrs.sm_local_os_ops_descr, + OsOpsDescrs.sm_remote_os_ops_descr + ] + + @pytest.fixture( + params=[ + pytest.param( + descr, + id=descr.sign, + ) + for descr in sm_os_ops_descrs + ], + ) + def os_ops_descr(self, request: pytest.FixtureRequest) -> OsOpsDescr: + assert isinstance(request, pytest.FixtureRequest) + assert isinstance(request.param, OsOpsDescr) + return request.param + + sm_node_svcs: typing.List[PostgresNodeService] = [ + PostgresNodeServices.sm_local, + PostgresNodeServices.sm_local2, + PostgresNodeServices.sm_remote, + ] + + @pytest.fixture( + params=sm_node_svcs, + ids=[descr.sign for descr in sm_node_svcs] + ) + def node_svc(self, request: pytest.FixtureRequest) -> PostgresNodeService: + assert isinstance(request, pytest.FixtureRequest) + assert isinstance(request.param, PostgresNodeService) + assert isinstance(request.param.os_ops, OsOperations) + assert isinstance(request.param.port_manager, PortManager) + return request.param + + def test_testgres_version(self): + assert type(testgres_version) is str + + v = Version(testgres_version) + + # Author: Mark G. + assert v.major == 1 + assert v.minor == 15 + assert v.micro == 2 + + assert str(v) == testgres_version + return + + def test_version_management(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + + a = PgVer('10.0') + b = PgVer('10') + c = PgVer('9.6.5') + d = PgVer('15.0') + e = PgVer('15rc1') + f = PgVer('15beta4') + h = PgVer('15.3biha') + i = PgVer('15.3') + g = PgVer('15.3.1bihabeta1') + k = PgVer('15.3.1') + + assert (a == b) + assert (b > c) + assert (a > c) + assert (d > e) + assert (e > f) + assert (d > f) + assert (h > f) + assert (h == i) + assert (g == k) + assert (g > h) + + version = get_pg_version2(node_svc.os_ops) + + with __class__.helper__get_node(node_svc) as node: + assert (isinstance(version, six.string_types)) + assert (isinstance(node.version, PgVer)) + assert (node.version == PgVer(version)) + + def test_default_username( + self, + os_ops_descr: OsOpsDescr, + ): + assert type(os_ops_descr) is OsOpsDescr + assert isinstance(os_ops_descr.os_ops, OsOperations) + + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + assert default_username(os_ops) == os_ops.get_user() + assert default_username(os_ops) == os_ops.username + + assert default_username() == tconf.os_ops.username + assert default_username() == tconf.os_ops.get_user() + return + + def test_default_username2( + self, + os_ops_descr: OsOpsDescr, + ): + assert type(os_ops_descr) is OsOpsDescr + assert isinstance(os_ops_descr.os_ops, OsOperations) + + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + assert default_username2(os_ops) == os_ops.get_user() + assert default_username2(os_ops) == os_ops.username + return + + def test_node_constructor__default(self): + with PostgresNode() as node: + assert node._os_ops is not None + assert isinstance(node._os_ops, OsOperations) + assert node._port_manager is not None + assert isinstance(node._port_manager, PortManager) + assert node._name is not None + assert type(node._name) is str + assert node._name != "" + assert node._base_dir is None + return + + def test_node_constructor__host(self): + C_HOST = "AbCdE" + + unique_id = uuid.uuid4().hex + + with PostgresNode(host=C_HOST) as node: + assert node._host == C_HOST + assert node.host == C_HOST + assert isinstance(node.os_ops, OsOperations) + + tmpdir = node.os_ops.get_tempdir() + nodedir2 = node.os_ops.build_path(tmpdir, "node2--" + unique_id) + + C_NODE2_NAME = "node2" + + with node.clone_with_new_name_and_base_dir( + name=C_NODE2_NAME, + base_dir=nodedir2, + ) as node2: + assert node2 is not None + assert node2 is not node + + assert node2._host == C_HOST + assert node2.host == C_HOST + + assert node2._name == C_NODE2_NAME + assert node2._base_dir == nodedir2 + assert node2._port != node._port + assert node2._os_ops is node._os_ops + assert node2._port_manager is node._port_manager + return + + def test_node_repr(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + + with __class__.helper__get_node(node_svc).init() as node: + pattern = r"PostgresNode\(name='.+', port=.+, base_dir='.+'\)" + assert re.match(pattern, str(node)) is not None + + def test_custom_init(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + + with __class__.helper__get_node(node_svc) as node: + # enable page checksums + node.init(initdb_params=['-k']).start() + return + + def test_custom_init__hba(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + + with __class__.helper__get_node(node_svc) as node: + node.init( + allow_streaming=True, + initdb_params=['--auth-local=reject', '--auth-host=reject']) + + hba_file = os.path.join(node.data_dir, 'pg_hba.conf') + lines = node.os_ops.readlines(hba_file) + + # check number of lines + assert (len(lines) >= 6) + + # Normalize function: turns a string into a list of pure words + def normalize_line(line_str): + return line_str.strip().split() + + # We collect a list of rules that already exist in the file (in the form of word lists) + existing_normalized = [] + for s in lines: + s_clean = s.strip() + if s_clean and not s_clean.startswith("#"): + existing_normalized.append(normalize_line(s_clean)) + continue + + # there should be no trust entries at all + for s in lines: + if len(s) > 0 and s[0] in ["host", "local"]: + assert s[-1] == "reject" + continue + return + + def test_double_init(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + + with __class__.helper__get_node(node_svc).init() as node: + # can't initialize node more than once + with pytest.raises(expected_exception=InitNodeException): + node.init() + + def test_init_after_cleanup(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + + with __class__.helper__get_node(node_svc) as node: + node.init().start().execute('select 1') + node.cleanup() + node.init().start().execute('select 1') + + def test_init_unique_system_id(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + + # this function exists in PostgreSQL 9.6+ + current_version = get_pg_version2(node_svc.os_ops) + + __class__.helper__skip_test_if_util_not_exist(node_svc.os_ops, "pg_resetwal") + __class__.helper__skip_test_if_pg_version_is_not_ge(current_version, '9.6') + + query = 'select system_identifier from pg_control_system()' + + with scoped_config(cache_initdb=False): + with __class__.helper__get_node(node_svc).init().start() as node0: + id0 = node0.execute(query)[0] + + with scoped_config(cache_initdb=True, + cached_initdb_unique=True) as config: + assert (config.cache_initdb) + assert (config.cached_initdb_unique) + + # spawn two nodes; ids must be different + with __class__.helper__get_node(node_svc).init().start() as node1, \ + __class__.helper__get_node(node_svc).init().start() as node2: + id1 = node1.execute(query)[0] + id2 = node2.execute(query)[0] + + # ids must increase + assert (id1 > id0) + assert (id2 > id1) + + def test_node_exit(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + + with pytest.raises(expected_exception=QueryException): + with __class__.helper__get_node(node_svc).init() as node: + base_dir = node.base_dir + node.safe_psql('select 1') + + # we should save the DB for "debugging" + assert (node_svc.os_ops.path_exists(base_dir)) + node_svc.os_ops.rmdirs(base_dir, ignore_errors=True) + + with __class__.helper__get_node(node_svc).init() as node: + base_dir = node.base_dir + + # should have been removed by default + assert not (node_svc.os_ops.path_exists(base_dir)) + + def test_double_start(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + + with __class__.helper__get_node(node_svc) as node: + node.init() + assert not node.is_started + assert node.status() == NodeStatus.Stopped + node.start() + assert node.is_started + assert node.status() == NodeStatus.Running + + with pytest.raises(expected_exception=StartNodeException) as x: + # can't start node more than once + node.start() + + assert x is not None + assert type(x.value) is StartNodeException + assert type(x.value.description) is str + assert type(x.value.message) is str + + assert x.value.description == "Cannot start node" + assert x.value.message.startswith(x.value.description) + + assert node.is_started + assert node.status() == NodeStatus.Running + + return + + def test_start__manually_stop__start_again(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + + with __class__.helper__get_node(node_svc) as node: + node.init() + assert not node.is_started + + logging.info("Start node") + node.start() + assert node.is_started + assert node.status() == NodeStatus.Running + + logging.info("Stop node manually via pg_ctl") + stop_cmd = [ + node.os_ops.build_path(node.bin_dir, "pg_ctl"), + "stop", + "-D", + node.data_dir, + ] + + execute_utility2( + node.os_ops, + stop_cmd, + node.utils_log_file + ) + + assert node.is_started + assert node.status() == NodeStatus.Stopped + + logging.info("Start node again") + node.start() + assert node.is_started + assert node.status() == NodeStatus.Running + + assert not node.is_started + assert node.status() == NodeStatus.Uninitialized + return + + def test_uninitialized_start(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + + with __class__.helper__get_node(node_svc) as node: + # node is not initialized yet + assert node.status() == NodeStatus.Uninitialized + + with pytest.raises(expected_exception=StartNodeException): + node.start() + + assert node.status() == NodeStatus.Uninitialized + return + + def test_start2(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + + with __class__.helper__get_node(node_svc) as node: + node.init() + assert not node.is_started + assert node.status() == NodeStatus.Stopped + node.start2() + assert not node.is_started + assert node.status() == NodeStatus.Running + + with pytest.raises(expected_exception=StartNodeException) as x: + # can't start node more than once + node.start2() + + assert x is not None + assert type(x.value) is StartNodeException + assert type(x.value.description) is str + assert type(x.value.message) is str + + assert x.value.description == "Cannot start node" + assert x.value.message.startswith(x.value.description) + + assert not node.is_started + assert node.status() == NodeStatus.Running + + return + + def test_restart(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + + with __class__.helper__get_node(node_svc) as node: + node.init() + + nRestartAttempt = 0 + + while True: + nRestartAttempt += 1 + + logging.info("Attempt #{}".format(nRestartAttempt)) + + node.start() + + # restart, ok + res = node.execute('select 1') + assert (res == [(1,)]) + + node_log_reader = PostgresNodeLogReader( + node, + from_beginnig=False, + ) + + try: + node.restart() + except StartNodeException as e: + logging.info("Exception ({}): {}".format( + type(e).__name__, + e, + )) + + if nRestartAttempt == 5: + raise + + if not PostgresNodeUtils.detect_port_conflict(node_log_reader): + raise + + logging.info("Node port {} conflicted with another PostgreSQL instance.".format( + node.port + )) + + logging.info("Wait for node stop") + + nStopAttemtp = 0 + + while True: + if nStopAttemtp == 5: + raise RuntimeError("Node is not stopped!") + + nStopAttemtp += 1 + + time.sleep(1) + + node_status = node.status() + + logging.info("Node status is {}".format(node_status)) + + if node_status == NodeStatus.Stopped: + break + continue + + # node is stopped. try again + continue + + assert node.status() == NodeStatus.Running + break + + res = node.execute('select 2') + assert (res == [(2,)]) + + assert node.status() == NodeStatus.Running + + # restart, fail + with pytest.raises(expected_exception=StartNodeException): + node.append_conf('pg_hba.conf', 'DUMMY') + node.restart() + + assert node.status() == NodeStatus.Stopped + return + + def test_double_stop(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + + with __class__.helper__get_node(node_svc) as node: + node.init() + assert not node.is_started + node.start() + assert node.is_started + node.stop() + assert not node.is_started + + with pytest.raises(expected_exception=Exception) as x: + # can't start node more than once + node.stop() + + assert x is not None + assert "Is server running?" in str(x.value) + + assert not node.is_started + + return + + def test_reload(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + + with __class__.helper__get_node(node_svc) as node: + node.init().start() + + # change client_min_messages and save old value + cmm_old = node.execute('show client_min_messages') + node.append_conf(client_min_messages='DEBUG1') + + # reload config + node.reload() + + # check new value + cmm_new = node.execute('show client_min_messages') + assert ('debug1' == cmm_new[0][0].lower()) + assert (cmm_old != cmm_new) + + def test_pg_ctl(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + + with __class__.helper__get_node(node_svc) as node: + node.init().start() + + status = node.pg_ctl(['status']) + assert ('PID' in status) + + def test_status(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + + assert (NodeStatus.Running) + assert not (NodeStatus.Stopped) + assert not (NodeStatus.Uninitialized) + + # check statuses after each operation + with __class__.helper__get_node(node_svc) as node: + assert (node.pid == 0) + assert (node.status() == NodeStatus.Uninitialized) + + node.init() + + assert (node.pid == 0) + assert (node.status() == NodeStatus.Stopped) + + node.start() + + assert (node.pid != 0) + assert (node.status() == NodeStatus.Running) + + node.stop() + + assert (node.pid == 0) + assert (node.status() == NodeStatus.Stopped) + + node.cleanup() + + assert (node.pid == 0) + assert (node.status() == NodeStatus.Uninitialized) + + def test_status__empty_postmaster_pid(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + + assert (NodeStatus.Running) + assert not (NodeStatus.Stopped) + assert not (NodeStatus.Uninitialized) + + # check statuses after each operation + with __class__.helper__get_node(node_svc) as node: + assert (node.pid == 0) + assert (node.status() == NodeStatus.Uninitialized) + + node.init() + + postmaster_pid_file = node.os_ops.build_path(node.data_dir, "postmaster.pid") + + node.os_ops.write( + postmaster_pid_file, + "" + ) + + with pytest.raises(expected_exception=ExecUtilException) as x: + node.status() + + expected_msg = "pg_ctl: the PID file \"{}\" is empty\n".format( + postmaster_pid_file + ) + + assert expected_msg == x.value.error + return + + sm_false_true = [False, True] + + @pytest.fixture( + params=[ + pytest.param( + x, + id="sleep_after_clean={}".format(x), + ) + for x in sm_false_true + ] + ) + def sleep_after_clean(self, request: pytest.FixtureRequest) -> bool: + assert isinstance(request, pytest.FixtureRequest) + assert type(request.param) is bool + return request.param + + def test_status__force_clean_postmaster_pid( + self, + node_svc: PostgresNodeService, + sleep_after_clean: bool, + ): + assert isinstance(node_svc, PostgresNodeService) + + assert (NodeStatus.Running) + assert not (NodeStatus.Stopped) + assert not (NodeStatus.Uninitialized) + + # check statuses after each operation + with __class__.helper__get_node(node_svc) as node: + assert (node.pid == 0) + assert (node.status() == NodeStatus.Uninitialized) + + node.init() + node.start() + + assert node.status() == NodeStatus.Running + logging.info("Postmaster PID is {}.".format(node.pid)) + + postmaster_pid_file = node.os_ops.build_path(node.data_dir, "postmaster.pid") + + logging.info("Clean postmaster pid file [{}].".format( + postmaster_pid_file + )) + + logging.info("Clean pid file...") + node.os_ops.write( + postmaster_pid_file, + "", + truncate=True, + ) + + if sleep_after_clean: + # server removes pid file and shutdown within 60 seconds. + logging.info("SLEEP 65 sec!") + time.sleep(65) + + logging.info("Check node status...") + node_status: typing.Optional[NodeStatus] + try: + node_status = node.status() + except ExecUtilException as e: + logging.info("Catch exception ({}): {}".format( + type(e).__name__, + str(e), + )) + + expected_msg = "pg_ctl: the PID file \"{}\" is empty\n".format( + postmaster_pid_file + ) + assert expected_msg == e.error + else: + assert node_status is not None + + logging.info("Node Status is {}".format(node_status.name)) + + if node_status == NodeStatus.Stopped: + pass + elif node_status == NodeStatus.Zombie: + logging.warning("Zombie is detected!") + else: + raise RuntimeError("Unknown node status: {}.".format(node_status)) + return + + def test_kill__is_not_initialized( + self, + node_svc: PostgresNodeService + ): + assert isinstance(node_svc, PostgresNodeService) + + with __class__.helper__get_node(node_svc) as node: + assert isinstance(node, PostgresNode) + assert (node.pid == 0) + assert (node.status() == NodeStatus.Uninitialized) + + with pytest.raises(expected_exception=InvalidOperationException) as x: + node.kill() + + assert x is not None + assert str(x.value) == "Can't kill server process. Node is not initialized." + return + + def test_kill__is_not_running( + self, + node_svc: PostgresNodeService + ): + assert isinstance(node_svc, PostgresNodeService) + + with __class__.helper__get_node(node_svc) as node: + assert isinstance(node, PostgresNode) + assert (node.pid == 0) + assert (node.status() == NodeStatus.Uninitialized) + + node.init() + + try: + with pytest.raises(expected_exception=InvalidOperationException) as x: + node.kill() + + assert x is not None + assert str(x.value) == "Can't kill server process. Node is not running." + finally: + try: + node.cleanup(release_resources=True) + except Exception as e: + logging.error("Exception ({}): {}".format( + type(e).__name__, + e, + )) + return + + def test_kill__ok( + self, + node_svc: PostgresNodeService + ): + assert isinstance(node_svc, PostgresNodeService) + + node = __class__.helper__get_node(node_svc) + + try: + assert isinstance(node, PostgresNode) + assert (node.pid == 0) + assert (node.status() == NodeStatus.Uninitialized) + + node.init() + assert not node.is_started + node.slow_start() + assert node.is_started + + assert node.status() == NodeStatus.Running + + node.kill() + assert not node.is_started + + attempt = 0 + + while True: + if attempt == 60: + raise RuntimeError("Node is not stopped.") + + attempt += 1 + + if attempt > 1: + time.sleep(1) + + s = node.status() + + logging.info("Node status is {}".format(s.name)) + + if s == NodeStatus.Running: + continue + + if s == NodeStatus.Stopped: + logging.info("Node stopped") + break + + if s == NodeStatus.Zombie: + logging.info("Node is zombie") + break + + logging.error("Node has unknown status: {}.".format(s.name)) + break + finally: + if node.is_started: + node.stop() + + node.cleanup(release_resources=True) + return + + def test_kill_backgroud_writer__ok( + self, + node_svc: PostgresNodeService + ): + assert isinstance(node_svc, PostgresNodeService) + + with __class__.helper__get_node(node_svc) as node: + assert isinstance(node, PostgresNode) + assert (node.pid == 0) + assert (node.status() == NodeStatus.Uninitialized) + + node.init() + assert not node.is_started + node.slow_start() + assert node.is_started + node_pid = node.pid + assert type(node_pid) is int + + # --- We expect BackgroundWriter to appear under load ------------------------ + bw_attempt = 0 + while True: + aux_pids = node.auxiliary_pids + assert type(aux_pids) is dict + + if ProcessType.BackgroundWriter in aux_pids: + break + + bw_attempt += 1 + # We give the server up to 3 seconds to start all background workers. + if bw_attempt == 30: + raise RuntimeError("BackgroundWriter process did not start in time under heavy load.") + + time.sleep(0.1) + continue + + # ---------------------------------------------------------------------------- + aux_pids = node.auxiliary_pids + assert type(aux_pids) is dict + assert ProcessType.BackgroundWriter in aux_pids + bw_pids = aux_pids[ProcessType.BackgroundWriter] + assert type(bw_pids) is list + assert len(bw_pids) == 1 + bw_pid = bw_pids[0] + assert type(bw_pid) is int + node.kill(ProcessType.BackgroundWriter) + assert node.is_started + + attempt = 0 + + while True: + if attempt == 60: + raise RuntimeError("Node is not stopped.") + + attempt += 1 + + if attempt > 1: + time.sleep(1) + + try: + psutil.Process(bw_pid) + except psutil.NoSuchProcess: + logging.info("Process is not found") + break + + logging.info("Process is still alive.") + continue + + assert node.is_started + assert node.pid == node_pid + return + + def test_child_processes__is_not_initialized( + self, + node_svc: PostgresNodeService + ): + assert isinstance(node_svc, PostgresNodeService) + + with __class__.helper__get_node(node_svc) as node: + assert isinstance(node, PostgresNode) + assert (node.pid == 0) + assert (node.status() == NodeStatus.Uninitialized) + + with pytest.raises(expected_exception=InvalidOperationException) as x: + node.child_processes + + assert x is not None + assert str(x.value) == "Can't enumerate node child processes. Node is not initialized." + return + + def test_child_processes__is_not_running( + self, + node_svc: PostgresNodeService + ): + assert isinstance(node_svc, PostgresNodeService) + + with __class__.helper__get_node(node_svc) as node: + assert isinstance(node, PostgresNode) + assert (node.pid == 0) + assert (node.status() == NodeStatus.Uninitialized) + + node.init() + + try: + with pytest.raises(expected_exception=InvalidOperationException) as x: + node.child_processes + + assert x is not None + assert str(x.value) == "Can't enumerate node child processes. Node is not running." + finally: + try: + node.cleanup(release_resources=True) + except Exception as e: + logging.error("Exception ({}): {}".format( + type(e).__name__, + e, + )) + return + + def test_child_processes__ok( + self, + node_svc: PostgresNodeService + ): + assert isinstance(node_svc, PostgresNodeService) + + with __class__.helper__get_node(node_svc) as node: + assert isinstance(node, PostgresNode) + assert (node.pid == 0) + assert (node.status() == NodeStatus.Uninitialized) + + node.init() + + try: + node.slow_start() + + children = node.child_processes + assert children is not None + assert type(children) is list + + logging.info("Children count is {}".format(len(children))) + logging.info("") + + def LOCAL__safe_call_cmdline(p: ProcessProxy) -> str: + assert type(p) is ProcessProxy + try: + return p.cmdline() + except Exception as e: + return "Exception ({}): {}".format( + type(e).__name__, + e, + ) + + for i in range(len(children)): + logging.info("------ check child [{}]".format(i)) + child = children[i] + + try: + assert child is not None + assert type(child) is ProcessProxy + assert hasattr(child, "process") + assert hasattr(child, "ptype") + assert hasattr(child, "pid") + assert hasattr(child, "cmdline") + assert child.process is not None + assert child.ptype is not None + assert child.pid is not None + assert type(child.ptype) is ProcessType + assert type(child.pid) is int + assert type(child.cmdline) is types.MethodType + + logging.info("ptype is {}".format(child.ptype)) + logging.info("pid is {}".format(child.pid)) + logging.info("cmdline is [{}]".format(LOCAL__safe_call_cmdline(child))) + except Exception as e: + logging.error("Exception ({}): {}".format( + type(e).__name__, + e, + )) + continue + finally: + try: + node.cleanup(release_resources=True) + except Exception as e: + logging.error("Exception ({}): {}".format( + type(e).__name__, + e, + )) + return + + def test_child_pids(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + + master_processes = [ + ProcessType.AutovacuumLauncher, + ProcessType.BackgroundWriter, + ProcessType.Checkpointer, + ProcessType.StatsCollector, + ProcessType.WalSender, + ProcessType.WalWriter, + ] + + postgresVersion = get_pg_version2(node_svc.os_ops) + + if __class__.helper__pg_version_ge(postgresVersion, '10'): + master_processes.append(ProcessType.LogicalReplicationLauncher) + + if __class__.helper__pg_version_ge(postgresVersion, '14'): + master_processes.remove(ProcessType.StatsCollector) + + repl_processes = [ + ProcessType.Startup, + ProcessType.WalReceiver, + ] + + def LOCAL__test_auxiliary_pids( + node: PostgresNode, + expectedTypes: typing.List[ProcessType] + ) -> typing.List[ProcessType]: + # returns list of the absence processes + assert node is not None + assert type(node) is PostgresNode + assert expectedTypes is not None + assert type(expectedTypes) is list + + pids = node.auxiliary_pids + assert pids is not None + assert type(pids) is dict + + result: typing.List[ProcessType] = list() + for ptype in expectedTypes: + if ptype not in pids: + result.append(ptype) + return result + + def LOCAL__check_auxiliary_pids__multiple_attempts( + node: PostgresNode, + expectedTypes: typing.List[ProcessType], + ): + assert node is not None + assert type(node) is PostgresNode + assert expectedTypes is not None + assert type(expectedTypes) is list + + nAttempt = 0 + + while True: + nAttempt += 1 + + logging.info("Test pids of [{0}] node. Attempt #{1}.".format( + node.name, + nAttempt + )) + + if nAttempt > 1: + time.sleep(1) + + absenceList = LOCAL__test_auxiliary_pids(node, expectedTypes) + assert absenceList is not None + assert type(absenceList) is list + if len(absenceList) == 0: + logging.info("Bingo!") + break + + if nAttempt == 5: + raise Exception("Node {0} does not have the following processes: {1}.".format( + node.name, + absenceList, + )) + + logging.info("These processes are not found: {0}.".format(absenceList)) + continue + return + + with __class__.helper__get_node(node_svc).init().start() as master: + + # master node doesn't have a source walsender! + with pytest.raises(expected_exception=testgres_TestgresException): + master.source_walsender + + with master.connect() as con: + assert (con.pid > 0) + + with master.replicate().start() as replica: + assert type(replica) is PostgresNode + + # test __str__ method + str(master.child_processes[0]) + + LOCAL__check_auxiliary_pids__multiple_attempts( + master, + master_processes) + + LOCAL__check_auxiliary_pids__multiple_attempts( + replica, + repl_processes) + + master_pids = master.auxiliary_pids + + # there should be exactly 1 source walsender for replica + assert (len(master_pids[ProcessType.WalSender]) == 1) + pid1 = master_pids[ProcessType.WalSender][0] + pid2 = replica.source_walsender.pid + assert (pid1 == pid2) + + replica.stop() + + # there should be no walsender after we've stopped replica + with pytest.raises(expected_exception=testgres_TestgresException): + replica.source_walsender + + def test_exceptions(self): + str(StartNodeException('msg', [('file', 'lines')])) + str(ExecUtilException('msg', 'cmd', 1, 'out')) + str(QueryException('msg', 'query')) + + def test_auto_name(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + + with __class__.helper__get_node(node_svc).init(allow_streaming=True).start() as m: + with m.replicate().start() as r: + # check that nodes are running + assert (m.status()) + assert (r.status()) + + # check their names + assert (m.name != r.name) + assert ('testgres' in m.name) + assert ('testgres' in r.name) + + def test_file_tail(self): + s1 = "the quick brown fox jumped over that lazy dog\n" + s2 = "abc\n" + s3 = "def\n" + + with tempfile.NamedTemporaryFile(mode='r+', delete=True) as f: + sz = 0 + while sz < 3 * 8192: + sz += len(s1) + f.write(s1) + f.write(s2) + f.write(s3) + + f.seek(0) + lines = file_tail(f, 3) + assert (lines[0] == s1) + assert (lines[1] == s2) + assert (lines[2] == s3) + + f.seek(0) + lines = file_tail(f, 1) + assert (lines[0] == s3) + + def test_isolation_levels(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc).init().start() as node: + with node.connect() as con: + # string levels + con.begin('Read Uncommitted').commit() + con.begin('Read Committed').commit() + con.begin('Repeatable Read').commit() + con.begin('Serializable').commit() + + # enum levels + con.begin(IsolationLevel.ReadUncommitted).commit() + con.begin(IsolationLevel.ReadCommitted).commit() + con.begin(IsolationLevel.RepeatableRead).commit() + con.begin(IsolationLevel.Serializable).commit() + + # check wrong level + with pytest.raises(expected_exception=QueryException): + con.begin('Garbage').commit() + + def test_users(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc).init().start() as node: + node.psql('create role test_user login') + value = node.safe_psql('select 1', username='test_user') + value = __class__.helper__rm_carriage_returns(value) + assert (value == b'1\n') + + def test_poll_query_until(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc) as node: + node.init().start() + + get_time = 'select extract(epoch from now())' + check_time = 'select extract(epoch from now()) - {} >= 5' + + start_time = node.execute(get_time)[0][0] + node.poll_query_until(query=check_time.format(start_time)) + end_time = node.execute(get_time)[0][0] + + assert (end_time - start_time >= 5) + + # check 0 columns + with pytest.raises(expected_exception=QueryException): + node.poll_query_until( + query='select from pg_catalog.pg_class limit 1') + + # check None, fail + with pytest.raises(expected_exception=QueryException): + node.poll_query_until(query='create table abc (val int)') + + # check None, ok + node.poll_query_until(query='create table def()', + expected=None) # returns nothing + + # check 0 rows equivalent to expected=None + node.poll_query_until( + query='select * from pg_catalog.pg_class where true = false', + expected=None) + + # check arbitrary expected value, fail + with pytest.raises(expected_exception=QueryTimeoutException): + node.poll_query_until(query='select 3', + expected=1, + max_attempts=3, + sleep_time=0.01) + + # check arbitrary expected value, ok + node.poll_query_until(query='select 2', expected=2) + + # check timeout + with pytest.raises(expected_exception=QueryTimeoutException): + node.poll_query_until(query='select 1 > 2', + max_attempts=3, + sleep_time=0.01) + + # check ProgrammingError, fail + with pytest.raises(expected_exception=ProgrammingError): + node.poll_query_until(query='dummy1') + + # check ProgrammingError, ok + with pytest.raises(expected_exception=(QueryTimeoutException)): + node.poll_query_until(query='dummy2', + max_attempts=3, + sleep_time=0.01, + suppress={ProgrammingError}) + + # check 1 arg, ok + node.poll_query_until('select true') + + def test_logging(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + C_MAX_ATTEMPTS = 50 + # This name is used for testgres logging, too. + C_NODE_NAME = "testgres_tests." + __class__.__name__ + "test_logging-master-" + uuid.uuid4().hex + + logging.info("Node name is [{0}]".format(C_NODE_NAME)) + + with tempfile.NamedTemporaryFile('w', delete=True) as logfile: + formatter = logging.Formatter(fmt="%(node)-5s: %(message)s") + handler = logging.FileHandler(filename=logfile.name) + handler.formatter = formatter + logger = logging.getLogger(C_NODE_NAME) + assert logger is not None + assert len(logger.handlers) == 0 + + try: + # It disables to log on the root level + logger.propagate = False + logger.addHandler(handler) + + with scoped_config(use_python_logging=True): + with __class__.helper__get_node(node_svc, name=C_NODE_NAME) as master: + logging.info("Master node is initilizing") + master.init() + + logging.info("Master node is starting") + master.start() + + logging.info("Dummy query is executed a few times") + for _ in range(20): + master.execute('select 1') + time.sleep(0.01) + + # let logging worker do the job + time.sleep(0.1) + + logging.info("Master node log file is checking") + nAttempt = 0 + + while True: + assert nAttempt <= C_MAX_ATTEMPTS + if nAttempt == C_MAX_ATTEMPTS: + raise Exception("Test failed!") + + # let logging worker do the job + time.sleep(0.1) + + nAttempt += 1 + + logging.info("Attempt {0}".format(nAttempt)) + + # check that master's port is found + with open(logfile.name, 'r') as log: + lines = log.readlines() + + assert lines is not None + assert type(lines) is list + + def LOCAL__test_lines(lines: typing.Iterable[str]) -> bool: + assert isinstance(lines, typing.Iterable) + for s in lines: + assert type(s) is str + if C_NODE_NAME in s: + logging.info("OK. We found the node_name in a line \"{0}\"".format(s)) + return True + return False + + if LOCAL__test_lines(lines): + break + + logging.info("Master node log file does not have an expected information.") + continue + + # test logger after stop/start/restart + logging.info("Master node is stopping...") + master.stop() + logging.info("Master node is staring again...") + master.start() + logging.info("Master node is restaring...") + master.restart() + assert (master._logger.is_alive()) + finally: + # It is a hack code to logging cleanup + with logging._lock: + assert logging.Logger.manager is not None + assert C_NODE_NAME in logging.Logger.manager.loggerDict.keys() + logging.Logger.manager.loggerDict.pop(C_NODE_NAME, None) + assert C_NODE_NAME not in logging.Logger.manager.loggerDict.keys() + assert handler not in logging._handlers.values() + # GO HOME! + return + + def test_psql(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc).init().start() as node: + + # check returned values (1 arg) + res = node.psql('select 1') + assert (__class__.helper__rm_carriage_returns(res) == (0, b'1\n', b'')) + + # check returned values (2 args) + res = node.psql('postgres', 'select 2') + assert (__class__.helper__rm_carriage_returns(res) == (0, b'2\n', b'')) + + # check returned values (named) + res = node.psql(query='select 3', dbname='postgres') + assert (__class__.helper__rm_carriage_returns(res) == (0, b'3\n', b'')) + + # check returned values (1 arg) + res = node.safe_psql('select 4') + assert (__class__.helper__rm_carriage_returns(res) == b'4\n') + + # check returned values (2 args) + res = node.safe_psql('postgres', 'select 5') + assert (__class__.helper__rm_carriage_returns(res) == b'5\n') + + # check returned values (named) + res = node.safe_psql(query='select 6', dbname='postgres') + assert (__class__.helper__rm_carriage_returns(res) == b'6\n') + + # check feeding input + node.safe_psql('create table horns (w int)') + node.safe_psql('copy horns from stdin (format csv)', + input=b"1\n2\n3\n\\.\n") + _sum = node.safe_psql('select sum(w) from horns') + assert (__class__.helper__rm_carriage_returns(_sum) == b'6\n') + + # check psql's default args, fails + with pytest.raises(expected_exception=QueryException): + r = node.psql() # raises! + logging.error("node.psql returns [{}]".format(r)) + + node.stop() + + # check psql on stopped node, fails + with pytest.raises(expected_exception=QueryException): + # [2025-04-03] This call does not raise exception! I do not know why. + r = node.safe_psql('select 1') # raises! + logging.error("node.safe_psql returns [{}]".format(r)) + + def test_psql__another_port(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc).init() as node1: + with __class__.helper__get_node(node_svc).init() as node2: + node1.start() + node2.start() + assert node1.port != node2.port + assert node1.host == node2.host + + node1.stop() + + logging.info("test table in node2 is creating ...") + node2.safe_psql( + dbname="postgres", + query="create table test (id integer);" + ) + + logging.info("try to find test table through node1.psql ...") + res = node1.psql( + dbname="postgres", + query="select count(*) from pg_class where relname='test'", + host=node2.host, + port=node2.port, + ) + assert (__class__.helper__rm_carriage_returns(res) == (0, b'1\n', b'')) + + def test_psql__another_bad_host(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc).init() as node: + logging.info("try to execute node1.psql ...") + res = node.psql( + dbname="postgres", + query="select count(*) from pg_class where relname='test'", + host="DUMMY_HOST_NAME", + port=node.port, + ) + + res2 = __class__.helper__rm_carriage_returns(res) + + assert res2[0] != 0 + assert b"DUMMY_HOST_NAME" in res[2] + + def test_safe_psql__another_port(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc).init() as node1: + with __class__.helper__get_node(node_svc).init() as node2: + node1.start() + node2.start() + assert node1.port != node2.port + assert node1.host == node2.host + + node1.stop() + + logging.info("test table in node2 is creating ...") + node2.safe_psql( + dbname="postgres", + query="create table test (id integer);" + ) + + logging.info("try to find test table through node1.psql ...") + res = node1.safe_psql( + dbname="postgres", + query="select count(*) from pg_class where relname='test'", + host=node2.host, + port=node2.port, + ) + assert (__class__.helper__rm_carriage_returns(res) == b'1\n') + + def test_safe_psql__another_bad_host(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc).init() as node: + logging.info("try to execute node1.psql ...") + + with pytest.raises(expected_exception=Exception) as x: + node.safe_psql( + dbname="postgres", + query="select count(*) from pg_class where relname='test'", + host="DUMMY_HOST_NAME", + port=node.port, + ) + + assert "DUMMY_HOST_NAME" in str(x.value) + + def test_safe_psql__expect_error(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc).init().start() as node: + err = node.safe_psql('select_or_not_select 1', expect_error=True) + assert (type(err) is str) + assert ('select_or_not_select' in err) + assert ('ERROR: syntax error at or near "select_or_not_select"' in err) + + # --------- + with pytest.raises( + expected_exception=InvalidOperationException, + match="^" + re.escape("Exception was expected, but query finished successfully: `select 1;`.") + "$" + ): + node.safe_psql("select 1;", expect_error=True) + + # --------- + res = node.safe_psql("select 1;", expect_error=False) + assert (__class__.helper__rm_carriage_returns(res) == b'1\n') + + def test_transactions(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc).init().start() as node: + + with node.connect() as con: + con.begin() + con.execute('create table test(val int)') + con.execute('insert into test values (1)') + con.commit() + + con.begin() + con.execute('insert into test values (2)') + res = con.execute('select * from test order by val asc') + assert (res == [(1, ), (2, )]) + con.rollback() + + con.begin() + res = con.execute('select * from test') + assert (res == [(1, )]) + con.rollback() + + con.begin() + con.execute('drop table test') + con.commit() + + def test_control_data(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc) as node: + + # node is not initialized yet + with pytest.raises(expected_exception=ExecUtilException): + node.get_control_data() + + node.init() + data = node.get_control_data() + + # check returned dict + assert data is not None + assert (any('pg_control' in s for s in data.keys())) + + def test_backup_simple(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc) as master: + + # enable streaming for backups + master.init(allow_streaming=True) + + # node must be running + with pytest.raises(expected_exception=BackupException): + master.backup() + + # it's time to start node + master.start() + + # fill node with some data + master.psql('create table test as select generate_series(1, 4) i') + + with master.backup(xlog_method='stream') as backup: + with backup.spawn_primary().start() as slave: + res = slave.execute('select * from test order by i asc') + assert (res == [(1, ), (2, ), (3, ), (4, )]) + + def test_backup_multiple(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc) as node: + node.init(allow_streaming=True).start() + + with node.backup(xlog_method='fetch') as backup1, \ + node.backup(xlog_method='fetch') as backup2: + assert (backup1.base_dir != backup2.base_dir) + + with node.backup(xlog_method='fetch') as backup: + with backup.spawn_primary('node1', destroy=False) as node1, \ + backup.spawn_primary('node2', destroy=False) as node2: + assert (node1.base_dir != node2.base_dir) + + def test_backup_exhaust(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc) as node: + node.init(allow_streaming=True).start() + + with node.backup(xlog_method='fetch') as backup: + # exhaust backup by creating new node + with backup.spawn_primary(): + pass + + # now let's try to create one more node + with pytest.raises(expected_exception=BackupException): + backup.spawn_primary() + + def test_backup_wrong_xlog_method(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc) as node: + node.init(allow_streaming=True).start() + + with pytest.raises( + expected_exception=BackupException, + match="^" + re.escape('Invalid xlog_method "wrong"') + "$" + ): + node.backup(xlog_method='wrong') + + def test_pg_ctl_wait_option(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + + C_MAX_ATTEMPT = 5 + + nAttempt = 0 + + while True: + if nAttempt == C_MAX_ATTEMPT: + raise Exception("PostgresSQL did not start.") + + nAttempt += 1 + logging.info("------------------------ attempt #{}".format( + nAttempt + )) + + if nAttempt > 1: + logging.info("Sleep 3 seconds") + time.sleep(3) + + port = node_svc.port_manager.reserve_port() + assert type(port) is int + ok = False + try: + with __class__.helper__get_node(node_svc, port=port) as node: + if self.impl__test_pg_ctl_wait_option(node_svc, node): + ok = True + finally: + node_svc.port_manager.release_port(port) + + if ok: + break + + continue + + logging.info("OK. Test is passed. Number of attempts is {}".format( + nAttempt + )) + return + + def impl__test_pg_ctl_wait_option( + self, + node_svc: PostgresNodeService, + node: PostgresNode + ) -> bool: + assert isinstance(node_svc, PostgresNodeService) + assert isinstance(node, PostgresNode) + assert node.status() == NodeStatus.Uninitialized + + C_MAX_ATTEMPTS = 50 + + logging.info("init node") + node.init() + assert node.status() == NodeStatus.Stopped + logging.info("node is inited") + + node_log_reader = PostgresNodeLogReader(node, from_beginnig=True) + + logging.info("start node") + + try: + node.start(wait=False) + except StartNodeException as e: + logging.info("Exception ({}): {}".format( + type(e).__name__, + e, + )) + return False + logging.info("node is started") + + nAttempt = 0 + while True: + if PostgresNodeUtils.detect_port_conflict(node_log_reader): + logging.info("Node port {} conflicted with another PostgreSQL instance.".format( + node.port + )) + return False + + if nAttempt == C_MAX_ATTEMPTS: + # + # [2025-03-11] + # We have an unexpected problem with this test in CI + # Let's get an additional information about this test failure. + # + logging.error("Node was not stopped.") + if not node.os_ops.path_exists(node.pg_log_file): + logging.warning("Node log does not exist.") + else: + logging.info("Let's read node log file [{0}]".format(node.pg_log_file)) + logFileData = node.os_ops.read(node.pg_log_file, binary=False) + logging.info("Node log file content:\n{0}".format(logFileData)) + + raise Exception("Could not stop node.") + + nAttempt += 1 + + if nAttempt > 1: + logging.info("Wait 1 second.") + time.sleep(1) + logging.info("") + + logging.info("Try to stop node. Attempt #{0}.".format(nAttempt)) + + try: + node.stop(wait=False) + break + except ExecUtilException as e: + # it's ok to get this exception here since node + # could be not started yet + logging.info("Node is not stopped. Exception ({0}): {1}".format(type(e).__name__, e)) + continue + + logging.info("OK. Stop command was executed. Let's wait while our node will stop really.") + nAttempt = 0 + while True: + if nAttempt == C_MAX_ATTEMPTS: + raise Exception("Could not stop node.") + + nAttempt += 1 + if nAttempt > 1: + logging.info("Wait 1 second.") + time.sleep(1) + logging.info("") + + logging.info("Attempt #{0}.".format(nAttempt)) + s1 = node.status() + + logging.info("Node status is {}.".format(s1.name)) + + if s1 == NodeStatus.Running: + continue + + if s1 == NodeStatus.Zombie: + # [2026-07-12] We will wait for final stop (stabilization). OK? + continue + + if s1 == NodeStatus.Stopped: + break + + raise Exception("Unexpected node status: {0}.".format(s1)) + + logging.info("OK. Node is stopped.") + return True + + def test_replicate(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc) as node: + node.init(allow_streaming=True).start() + + with node.replicate().start() as replica: + res = replica.execute('select 1') + assert (res == [(1, )]) + + node.execute('create table test (val int)', commit=True) + + replica.catchup() + + res = node.execute('select * from test') + assert (res == []) + + def test_synchronous_replication(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + + current_version = get_pg_version2(node_svc.os_ops) + + __class__.helper__skip_test_if_pg_version_is_not_ge(current_version, "9.6") + + with __class__.helper__get_node(node_svc) as master: + master.init(allow_streaming=True).start() + + master.append_conf('synchronous_commit = remote_apply') + + # create standby + with master.replicate() as standby1, master.replicate() as standby2: + standby1.start() + standby2.start() + + # check formatting + assert ( + '1 ("{}", "{}")'.format(standby1.name, standby2.name) == str(First(1, (standby1, standby2))) + ) # yapf: disable + assert ( + 'ANY 1 ("{}", "{}")'.format(standby1.name, standby2.name) == str(Any(1, (standby1, standby2))) + ) # yapf: disable + + # set synchronous_standby_names + master.set_synchronous_standbys(First(2, [standby1, standby2])) + master.reload() + + master.safe_psql('create table abc(a int)') + + # Create a large transaction that will take some time to apply + # on standby to check that it applies synchronously + # (If set synchronous_commit to 'on' or other lower level then + # standby most likely won't catchup so fast and test will fail) + master.safe_psql( + 'insert into abc select generate_series(1, 1000000)', + ) + res = standby1.safe_psql('select count(*) from abc') + assert (__class__.helper__rm_carriage_returns(res) == b'1000000\n') + return + + def test_logical_replication(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + + current_version = get_pg_version2(node_svc.os_ops) + + __class__.helper__skip_test_if_pg_version_is_not_ge(current_version, "10") + + with __class__.helper__get_node(node_svc) as node1, __class__.helper__get_node(node_svc) as node2: + node1.init(allow_logical=True) + node1.start() + node2.init().start() + + create_table = 'create table test (a int, b int)' + node1.safe_psql(create_table) + node2.safe_psql(create_table) + + # create publication / create subscription + pub = node1.publish('mypub') + sub = node2.subscribe(pub, 'mysub') + + node1.safe_psql('insert into test values (1, 1), (2, 2)') + + # wait until changes apply on subscriber and check them + sub.catchup() + res = node2.execute('select * from test') + assert (res == [(1, 1), (2, 2)]) + + # disable and put some new data + sub.disable() + node1.safe_psql('insert into test values (3, 3)') + + # enable and ensure that data successfully transferred + sub.enable() + sub.catchup() + res = node2.execute('select * from test') + assert (res == [(1, 1), (2, 2), (3, 3)]) + + # Add new tables. Since we added "all tables" to publication + # (default behaviour of publish() method) we don't need + # to explicitly perform pub.add_tables() + create_table = 'create table test2 (c char)' + node1.safe_psql(create_table) + node2.safe_psql(create_table) + sub.refresh() + + # put new data + node1.safe_psql('insert into test2 values (\'a\'), (\'b\')') + sub.catchup() + res = node2.execute('select * from test2') + assert (res == [('a', ), ('b', )]) + + # drop subscription + sub.drop() + pub.drop() + + # create new publication and subscription for specific table + # (omitting copying data as it's already done) + pub = node1.publish('newpub', tables=['test']) + sub = node2.subscribe(pub, 'newsub', copy_data=False) + + node1.safe_psql('insert into test values (4, 4)') + sub.catchup() + res = node2.execute('select * from test') + assert (res == [(1, 1), (2, 2), (3, 3), (4, 4)]) + + # explicitly add table + with pytest.raises(expected_exception=ValueError): + pub.add_tables([]) # fail + pub.add_tables(['test2']) + node1.safe_psql('insert into test2 values (\'c\')') + sub.catchup() + res = node2.execute('select * from test2') + assert (res == [('a', ), ('b', )]) + + def test_logical_catchup(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + """ Runs catchup for 100 times to be sure that it is consistent """ + + current_version = get_pg_version2(node_svc.os_ops) + + __class__.helper__skip_test_if_pg_version_is_not_ge(current_version, "10") + + with __class__.helper__get_node(node_svc) as node1, __class__.helper__get_node(node_svc) as node2: + node1.init(allow_logical=True) + node1.start() + node2.init().start() + + create_table = 'create table test (key int primary key, val int); ' + node1.safe_psql(create_table) + node1.safe_psql('alter table test replica identity default') + node2.safe_psql(create_table) + + # create publication / create subscription + sub = node2.subscribe(node1.publish('mypub'), 'mysub') + + for i in range(0, 100): + node1.execute('insert into test values ({0}, {0})'.format(i)) + sub.catchup() + res = node2.execute('select * from test') + assert (res == [(i, i, )]) + node1.execute('delete from test') + + def test_logical_replication_fail(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + + current_version = get_pg_version2(node_svc.os_ops) + + __class__.helper__skip_test_if_pg_version_is_ge(current_version, "10") + + with __class__.helper__get_node(node_svc) as node: + with pytest.raises(expected_exception=InitNodeException): + node.init(allow_logical=True) + + def test_replication_slots(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc) as node: + node.init(allow_streaming=True).start() + + with node.replicate(slot='slot1').start() as replica: + replica.execute('select 1') + + # cannot create new slot with the same name + with pytest.raises(expected_exception=testgres_TestgresException): + node.replicate(slot='slot1') + + def test_incorrect_catchup(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc) as node: + node.init(allow_streaming=True).start() + + # node has no master, can't catch up + with pytest.raises(expected_exception=testgres_TestgresException): + node.catchup() + + def test_promotion(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc) as master: + master.init().start() + master.safe_psql('create table abc(id serial)') + + with master.replicate().start() as replica: + master.stop() + replica.promote() + + # make standby becomes writable master + replica.safe_psql('insert into abc values (1)') + res = replica.safe_psql('select * from abc') + assert (__class__.helper__rm_carriage_returns(res) == b'1\n') + + @pytest.fixture( + params=[ + enums.DumpFormat.Plain, + enums.DumpFormat.Custom, + enums.DumpFormat.Directory, + enums.DumpFormat.Tar + ] + ) + def dump_fmt(self, request: pytest.FixtureRequest) -> enums.DumpFormat: + assert type(request.param) is enums.DumpFormat + return request.param + + def test_dump(self, node_svc: PostgresNodeService, dump_fmt: enums.DumpFormat): + assert isinstance(node_svc, PostgresNodeService) + assert type(dump_fmt) is enums.DumpFormat + query_create = 'create table test as select generate_series(1, 2) as val' + query_select = 'select * from test order by val asc' + + with __class__.helper__get_node(node_svc).init().start() as node1: + node1.execute(query_create) + with removing(node_svc.os_ops, node1.dump(format=dump_fmt)) as dump: + with __class__.helper__get_node(node_svc).init().start() as node3: + if dump_fmt == enums.DumpFormat.Directory: + assert (node_svc.os_ops.isdir(dump)) + else: + assert (node_svc.os_ops.isfile(dump)) + # restore dump + node3.restore(filename=dump) + res = node3.execute(query_select) + assert (res == [(1, ), (2, )]) + + def test_dump_with_options(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + query_create = 'create table test_options as select generate_series(1, 5) as val' + + with __class__.helper__get_node(node_svc).init().start() as node1: + node1.execute(query_create) + + # Test dump with --schema-only option + with removing(node_svc.os_ops, node1.dump(options=['--schema-only'])) as dump: + with __class__.helper__get_node(node_svc).init().start() as node2: + assert (node_svc.os_ops.isfile(dump)) + # restore schema-only dump + node2.restore(filename=dump) + + # Check that table exists but has no data + res = node2.execute("SELECT COUNT(*) FROM test_options") + assert (res == [(0,)]) # Table exists but empty + + # Verify table structure exists + res = node2.execute(""" + SELECT COUNT(*) FROM information_schema.tables + WHERE table_name = 'test_options' + """) + assert (res == [(1,)]) # Table structure exists + + def test_pgbench(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + + __class__.helper__skip_test_if_util_not_exist(node_svc.os_ops, "pgbench") + + with __class__.helper__get_node(node_svc).init().start() as node: + # initialize pgbench DB and run benchmarks + node.pgbench_init( + scale=2, + foreign_keys=True, + options=['-q'] + ).pgbench_run(time=2) + + # run TPC-B benchmark + proc = node.pgbench(stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + options=['-T3']) + out = proc.communicate()[0] + assert (b'tps = ' in out) + + def test_unix_sockets(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + + with __class__.helper__get_node(node_svc) as node: + node.init(unix_sockets=False, allow_streaming=True) + node.start() + + res_exec = node.execute('select 1') + assert (res_exec == [(1,)]) + res_psql = node.safe_psql('select 1') + assert (res_psql == b'1\n') + + with node.replicate() as r: + assert type(r) is PostgresNode + r.start() + res_exec = r.execute('select 1') + assert (res_exec == [(1,)]) + res_psql = r.safe_psql('select 1') + assert (res_psql == b'1\n') + + def test_the_same_port(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + + with __class__.helper__get_node(node_svc) as node: + node.init().start() + assert (node._should_free_port) + assert (type(node.port) is int) + node_port_copy = node.port + r = node.safe_psql("SELECT 1;") + assert (__class__.helper__rm_carriage_returns(r) == b'1\n') + + with __class__.helper__get_node(node_svc, port=node.port) as node2: + assert (type(node2.port) is int) + assert (node2.port == node.port) + assert (not node2._should_free_port) + assert (node2.status() == NodeStatus.Uninitialized) + + node2.init() + + with pytest.raises( + expected_exception=StartNodeException, + match=re.escape("Cannot start node") + ): + node2.start() + + assert (node2.status() == NodeStatus.Stopped) + + # node is still working + assert (node.port == node_port_copy) + assert (node._should_free_port) + r = node.safe_psql("SELECT 3;") + assert (__class__.helper__rm_carriage_returns(r) == b'3\n') + + class tagPortManagerProxy(PortManager): + m_PrevPortManager: PortManager + + m_DummyPortNumber: int + m_DummyPortMaxUsage: int + + m_DummyPortCurrentUsage: int + m_DummyPortTotalUsage: int + + def __init__(self, prevPortManager: PortManager, dummyPortNumber: int, dummyPortMaxUsage: int): + assert isinstance(prevPortManager, PortManager) + assert type(dummyPortNumber) is int + assert type(dummyPortMaxUsage) is int + assert dummyPortNumber >= 0 + assert dummyPortMaxUsage >= 0 + + super().__init__() + + self.m_PrevPortManager = prevPortManager + + self.m_DummyPortNumber = dummyPortNumber + self.m_DummyPortMaxUsage = dummyPortMaxUsage + + self.m_DummyPortCurrentUsage = 0 + self.m_DummyPortTotalUsage = 0 + + def __enter__(self): + return self + + def __exit__(self, type, value, traceback): + assert self.m_DummyPortCurrentUsage == 0 + + assert self.m_PrevPortManager is not None + + def reserve_port(self) -> int: + assert type(self.m_DummyPortMaxUsage) is int + assert type(self.m_DummyPortTotalUsage) is int + assert type(self.m_DummyPortCurrentUsage) is int + assert self.m_DummyPortTotalUsage >= 0 + assert self.m_DummyPortCurrentUsage >= 0 + + assert self.m_DummyPortTotalUsage <= self.m_DummyPortMaxUsage + assert self.m_DummyPortCurrentUsage <= self.m_DummyPortTotalUsage + + assert self.m_PrevPortManager is not None + assert isinstance(self.m_PrevPortManager, PortManager) + + if self.m_DummyPortTotalUsage == self.m_DummyPortMaxUsage: + return self.m_PrevPortManager.reserve_port() + + self.m_DummyPortTotalUsage += 1 + self.m_DummyPortCurrentUsage += 1 + return self.m_DummyPortNumber + + def release_port(self, number: int) -> None: + assert type(number) is int + + assert type(self.m_DummyPortMaxUsage) is int + assert type(self.m_DummyPortTotalUsage) is int + assert type(self.m_DummyPortCurrentUsage) is int + assert self.m_DummyPortTotalUsage >= 0 + assert self.m_DummyPortCurrentUsage >= 0 + + assert self.m_DummyPortTotalUsage <= self.m_DummyPortMaxUsage + assert self.m_DummyPortCurrentUsage <= self.m_DummyPortTotalUsage + + assert self.m_PrevPortManager is not None + assert isinstance(self.m_PrevPortManager, PortManager) + + if self.m_DummyPortCurrentUsage > 0 and number == self.m_DummyPortNumber: + assert self.m_DummyPortTotalUsage > 0 + self.m_DummyPortCurrentUsage -= 1 + return + + return self.m_PrevPortManager.release_port(number) + + def test_port_rereserve_during_node_start(self, node_svc: PostgresNodeService): + assert type(node_svc) is PostgresNodeService + assert PostgresNode._C_MAX_START_ATEMPTS == 5 + + C_COUNT_OF_BAD_PORT_USAGE = 3 + + with __class__.helper__get_node(node_svc) as node1: + node1.init().start() + assert node1._should_free_port + assert type(node1.port) is int + node1_port_copy = node1.port + assert __class__.helper__rm_carriage_returns(node1.safe_psql("SELECT 1;")) == b'1\n' + + with __class__.tagPortManagerProxy(node_svc.port_manager, node1.port, C_COUNT_OF_BAD_PORT_USAGE) as proxy: + assert proxy.m_DummyPortNumber == node1.port + with __class__.helper__get_node(node_svc, port_manager=proxy) as node2: + assert node2._should_free_port + assert node2.port == node1.port + + node2.init().start() + + assert node2.port != node1.port + assert node2._should_free_port + assert proxy.m_DummyPortCurrentUsage == 0 + assert proxy.m_DummyPortTotalUsage == C_COUNT_OF_BAD_PORT_USAGE + assert node2.is_started + r = node2.safe_psql("SELECT 2;") + assert __class__.helper__rm_carriage_returns(r) == b'2\n' + + # node1 is still working + assert node1.port == node1_port_copy + assert node1._should_free_port + r = node1.safe_psql("SELECT 3;") + assert __class__.helper__rm_carriage_returns(r) == b'3\n' + + def test_port_conflict(self, node_svc: PostgresNodeService): + assert type(node_svc) is PostgresNodeService + assert PostgresNode._C_MAX_START_ATEMPTS > 1 + + C_COUNT_OF_BAD_PORT_USAGE = PostgresNode._C_MAX_START_ATEMPTS + + with __class__.helper__get_node(node_svc) as node1: + node1.init().start() + assert node1._should_free_port + assert type(node1.port) is int + node1_port_copy = node1.port + assert __class__.helper__rm_carriage_returns(node1.safe_psql("SELECT 1;")) == b'1\n' + + with __class__.tagPortManagerProxy(node_svc.port_manager, node1.port, C_COUNT_OF_BAD_PORT_USAGE) as proxy: + assert proxy.m_DummyPortNumber == node1.port + with __class__.helper__get_node(node_svc, port_manager=proxy) as node2: + assert node2._should_free_port + assert node2.port == node1.port + + node2.init() + assert node2.status() == NodeStatus.Stopped + + with pytest.raises( + expected_exception=StartNodeException, + match=re.escape("Cannot start node after multiple attempts.") + ): + node2.start() + + assert node2.port == node1.port + assert node2._should_free_port + assert proxy.m_DummyPortCurrentUsage == 1 + assert proxy.m_DummyPortTotalUsage == C_COUNT_OF_BAD_PORT_USAGE + assert not node2.is_started + assert node2.status() == NodeStatus.Stopped + + # node2 must release our dummyPort (node1.port) + assert (proxy.m_DummyPortCurrentUsage == 0) + + # node1 is still working + assert node1.port == node1_port_copy + assert node1._should_free_port + r = node1.safe_psql("SELECT 3;") + assert __class__.helper__rm_carriage_returns(r) == b'3\n' + + def test_try_to_get_port_after_free_manual_port(self, node_svc: PostgresNodeService): + assert type(node_svc) is PostgresNodeService + + assert node_svc.port_manager is not None + assert isinstance(node_svc.port_manager, PortManager) + + with __class__.helper__get_node(node_svc) as node1: + assert node1 is not None + assert type(node1) is PostgresNode + assert node1.port is not None + assert type(node1.port) is int + with __class__.helper__get_node(node_svc, port=node1.port, port_manager=None) as node2: + assert node2 is not None + assert type(node1) is PostgresNode + assert node2 is not node1 + assert node2.port is not None + assert type(node2.port) is int + assert node2.port == node1.port + + logging.info("Release node2 port") + node2.free_port() + + logging.info("try to get node2.port...") + with pytest.raises( + InvalidOperationException, + match="^" + re.escape("PostgresNode port is not defined.") + "$" + ): + p = node2.port + assert p is None + + def test_try_to_start_node_after_free_manual_port(self, node_svc: PostgresNodeService): + assert type(node_svc) is PostgresNodeService + + assert node_svc.port_manager is not None + assert isinstance(node_svc.port_manager, PortManager) + + with __class__.helper__get_node(node_svc) as node1: + assert node1 is not None + assert type(node1) is PostgresNode + assert node1.port is not None + assert type(node1.port) is int + with __class__.helper__get_node(node_svc, port=node1.port, port_manager=None) as node2: + assert node2 is not None + assert type(node1) is PostgresNode + assert node2 is not node1 + assert node2.port is not None + assert type(node2.port) is int + assert node2.port == node1.port + + logging.info("Release node2 port") + node2.free_port() + + logging.info("node2 is trying to start...") + with pytest.raises( + InvalidOperationException, + match="^" + re.escape("Can't start PostgresNode. Port is not defined.") + "$" + ): + node2.start() + + def test_node__os_ops(self, node_svc: PostgresNodeService): + assert type(node_svc) is PostgresNodeService + + assert node_svc.os_ops is not None + assert isinstance(node_svc.os_ops, OsOperations) + + with PostgresNode(name="node", os_ops=node_svc.os_ops, port_manager=node_svc.port_manager) as node: + # retest + assert node_svc.os_ops is not None + assert isinstance(node_svc.os_ops, OsOperations) + + assert node.os_ops is node_svc.os_ops + # one more time + assert node.os_ops is node_svc.os_ops + + def test_node__port_manager(self, node_svc: PostgresNodeService): + assert type(node_svc) is PostgresNodeService + + assert node_svc.port_manager is not None + assert isinstance(node_svc.port_manager, PortManager) + + with PostgresNode(name="node", os_ops=node_svc.os_ops, port_manager=node_svc.port_manager) as node: + # retest + assert node_svc.port_manager is not None + assert isinstance(node_svc.port_manager, PortManager) + + assert node.port_manager is node_svc.port_manager + # one more time + assert node.port_manager is node_svc.port_manager + + def test_node__port_manager_and_explicit_port(self, node_svc: PostgresNodeService): + assert type(node_svc) is PostgresNodeService + + assert isinstance(node_svc.os_ops, OsOperations) + assert node_svc.port_manager is not None + assert isinstance(node_svc.port_manager, PortManager) + + port = node_svc.port_manager.reserve_port() + assert type(port) is int + + try: + with PostgresNode(name="node", port=port, os_ops=node_svc.os_ops) as node: + # retest + assert isinstance(node_svc.os_ops, OsOperations) + assert node_svc.port_manager is not None + assert isinstance(node_svc.port_manager, PortManager) + + assert node.port_manager is None + assert node.os_ops is node_svc.os_ops + + # one more time + assert node.port_manager is None + assert node.os_ops is node_svc.os_ops + finally: + node_svc.port_manager.release_port(port) + + def test_node__no_port_manager(self, node_svc: PostgresNodeService): + assert type(node_svc) is PostgresNodeService + + assert isinstance(node_svc.os_ops, OsOperations) + assert node_svc.port_manager is not None + assert isinstance(node_svc.port_manager, PortManager) + + port = node_svc.port_manager.reserve_port() + assert type(port) is int + + try: + with PostgresNode(name="node", port=port, os_ops=node_svc.os_ops, port_manager=None) as node: + # retest + assert isinstance(node_svc.os_ops, OsOperations) + assert node_svc.port_manager is not None + assert isinstance(node_svc.port_manager, PortManager) + + assert node.port_manager is None + assert node.os_ops is node_svc.os_ops + + # one more time + assert node.port_manager is None + assert node.os_ops is node_svc.os_ops + finally: + node_svc.port_manager.release_port(port) + + class tagTableChecksumTestData: + record_count: int + + def __init__( + self, + record_count: int, + ): + assert type(record_count) is int + self.record_count = record_count + return + + sm_TableCheckSumTestDatas = [ + tagTableChecksumTestData(0), + tagTableChecksumTestData(1), + tagTableChecksumTestData(2), + tagTableChecksumTestData(3), + tagTableChecksumTestData(987), + tagTableChecksumTestData(999), + tagTableChecksumTestData(1000), + tagTableChecksumTestData(1001), + tagTableChecksumTestData(1999), + tagTableChecksumTestData(19999), + tagTableChecksumTestData(199999), + tagTableChecksumTestData(1999999), + ] + + @pytest.fixture( + params=sm_TableCheckSumTestDatas, + ids=[x.record_count for x in sm_TableCheckSumTestDatas], + ) + def table_checksum_test_data( + self, + request: pytest.FixtureRequest + ) -> tagTableChecksumTestData: + assert isinstance(request, pytest.FixtureRequest) + assert type(request.param).__name__ == "tagTableChecksumTestData" + return request.param + + def test_node__table_checksum( + self, + node_svc: PostgresNodeService, + table_checksum_test_data: tagTableChecksumTestData, + ): + assert type(node_svc) is PostgresNodeService + assert type(table_checksum_test_data) is __class__.tagTableChecksumTestData + assert node_svc.port_manager is not None + assert isinstance(node_svc.port_manager, PortManager) + + with __class__.helper__get_node(node_svc) as node: + assert node is not None + assert type(node) is PostgresNode + assert node.port is not None + assert type(node.port) is int + assert type(table_checksum_test_data.record_count) is int + assert table_checksum_test_data.record_count >= 0 + + node.init() + node.slow_start() + + C_DB = "postgres" + + with node.connect(dbname=C_DB) as cn: + assert type(cn) is NodeConnection + + cn.execute("create table t (id integer, data varchar(32));") + cn.commit() + + if table_checksum_test_data.record_count > 0: + cn.execute("insert into t (id, data) select x, x from generate_series(1, {}) x".format( + table_checksum_test_data.record_count + )) + cn.commit() + + with cn.connection.cursor() as cursor: + assert cursor is not None + cursor.execute("SELECT hashtext(t::text) FROM \"t\" as t;") + + checksum1 = 0 + record_count = 0 + while True: + row = cursor.fetchone() + if row is None: + break + assert type(row) in [list, tuple] + assert len(row) == 1 + record_count += 1 + checksum1 += int(row[0]) + pass + + assert record_count == table_checksum_test_data.record_count + + checksum2 = node.table_checksum("t", C_DB) + assert type(checksum2) is int + + assert checksum1 == checksum2 + pass + return + + def test_node__pgbench_table_checksums__one_table( + self, + node_svc: PostgresNodeService, + table_checksum_test_data: tagTableChecksumTestData, + ): + assert type(node_svc) is PostgresNodeService + assert type(table_checksum_test_data) is __class__.tagTableChecksumTestData + assert node_svc.port_manager is not None + assert isinstance(node_svc.port_manager, PortManager) + + with __class__.helper__get_node(node_svc) as node: + assert node is not None + assert type(node) is PostgresNode + assert node.port is not None + assert type(node.port) is int + assert type(table_checksum_test_data.record_count) is int + assert table_checksum_test_data.record_count >= 0 + + node.init() + node.slow_start() + + C_DB = "postgres" + + with node.connect(dbname=C_DB) as cn: + assert type(cn) is NodeConnection + + cn.execute("create table t (id integer, data varchar(32));") + cn.commit() + + if table_checksum_test_data.record_count > 0: + cn.execute("insert into t (id, data) select x, x from generate_series(1, {}) x".format( + table_checksum_test_data.record_count + )) + cn.commit() + + with cn.connection.cursor() as cursor: + assert cursor is not None + cursor.execute("SELECT hashtext(t::text) FROM \"t\" as t;") + + checksum1 = 0 + record_count = 0 + while True: + row = cursor.fetchone() + if row is None: + break + assert type(row) in [list, tuple] + assert len(row) == 1 + record_count += 1 + checksum1 += int(row[0]) + pass + + assert record_count == table_checksum_test_data.record_count + + actual_result = node.pgbench_table_checksums(C_DB, ["t"]) + assert type(actual_result) is set + actual1 = actual_result.pop() + assert type(actual1) is tuple + assert len(actual1) == 2 + assert type(actual1[0]) is str + assert type(actual1[1]) is int + + assert checksum1 == actual1[1] + pass + return + + def test_node__pgbench_table_checksums__pbckp_2278(self, node_svc: PostgresNodeService): + assert type(node_svc) is PostgresNodeService + + assert node_svc.port_manager is not None + assert isinstance(node_svc.port_manager, PortManager) + + with __class__.helper__get_node(node_svc) as node: + assert node is not None + assert type(node) is PostgresNode + assert node.port is not None + assert type(node.port) is int + + node.init() + node.slow_start() + + logging.info("init pgbench database") + node.pgbench_init(scale=20) + + nPass = 0 + while nPass < 3: + nPass += 1 + logging.info("------------------- pass: {}".format(nPass)) + + if not __class__.helper__call_and_check_pgbench_table_checksums(node): + raise RuntimeError("pgbench_table_checksums created a problem. Please, check a test log.") + continue + + return + + @staticmethod + def helper__call_and_check_pgbench_table_checksums( + node: PostgresNode + ) -> bool: + assert node is not None + assert type(node) is PostgresNode + assert node.status() == NodeStatus.Running + + # We will check + # 1) the structure of result + # 2) the release of cursor locks + + logging.info("run pgbench_table_checksums") + full_checksums = node.pgbench_table_checksums() + assert full_checksums is not None + assert type(full_checksums) is set + assert len(full_checksums) == 4 + + expectedTables: typing.Dict[str, bool] = { + 'pgbench_branches': False, + 'pgbench_tellers': False, + 'pgbench_accounts': False, + 'pgbench_history': False, + } + + ok = True + + for tcs in full_checksums: + assert type(tcs) is tuple + assert len(tcs) == 2 + assert type(tcs[0]) is str + assert type(tcs[1]) is int + + tableName = tcs[0] + if tableName not in expectedTables: + ok = False + logging.error("pgbench_table_checksums returns unknown table [{}].".format( + tableName + )) + continue + + if expectedTables[tableName]: + ok = False + logging.error("pgbench_table_checksums returns table [{}] more than one time.".format( + tableName + )) + continue + + expectedTables[tableName] = True + continue + + C_SQL = """select x.granted, x.mode +from pg_locks x join pg_class c on x.relation=c.oid +where c.relname=%s;""" + + cn = node.connect(dbname="postgres") + assert type(cn) is NodeConnection + + try: + for tcs in full_checksums: + tableName = tcs[0] + recs = cn.execute(C_SQL, tableName) + assert type(recs) is list + if len(recs) == 0: + logging.info("Table [{}] does not have a lock. It is ok.".format( + tableName, + )) + else: + ok = False + assert len(recs) == 1 + rec = recs[0] + assert type(rec) is tuple + assert len(rec) == 2 + logging.error("Table [{}] has a lock [granted: {}][mode: {}].".format( + tableName, + rec[0], + rec[1], + )) + continue + finally: + try: + cn.close() + except Exception as e: + logging.error("Can't close connection. Exception ({}): {}".format( + type(e).__name__, + e, + )) + return ok + + class tag_rmdirs_protector: + _os_ops: OsOperations + _cwd: str + _old_rmdirs: typing.Optional[typing.Callable] + _cwd: str + + def __init__(self, os_ops: OsOperations): + self._os_ops = os_ops + self._cwd = os.path.abspath(os_ops.cwd()) + self._old_rmdirs = os_ops.rmdirs + return + + def __enter__(self): + assert self._os_ops.rmdirs == self._old_rmdirs + self._os_ops.rmdirs = self.proxy__rmdirs + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + assert self._os_ops.rmdirs == self.proxy__rmdirs + assert isinstance(self._old_rmdirs, typing.Callable) + self._os_ops.rmdirs = self._old_rmdirs + return False + + def proxy__rmdirs(self, path, ignore_errors=True): + raise Exception("Call of rmdirs is not expected!") + + def test_node_app__make_empty__base_dir_is_None(self, node_svc: PostgresNodeService): + assert type(node_svc) is PostgresNodeService + + assert isinstance(node_svc.os_ops, OsOperations) + assert node_svc.port_manager is not None + assert isinstance(node_svc.port_manager, PortManager) + + tmp_dir = node_svc.os_ops.mkdtemp() + assert tmp_dir is not None + assert type(tmp_dir) is str + logging.info("temp directory is [{}]".format(tmp_dir)) + + # ----------- + os_ops = node_svc.os_ops.create_clone() + assert os_ops is not node_svc.os_ops + + # ----------- + with __class__.tag_rmdirs_protector(os_ops): + node_app = NodeApp(test_path=tmp_dir, os_ops=os_ops) + assert node_app.os_ops is os_ops + + with pytest.raises(expected_exception=BaseException) as x: + node_app.make_empty(base_dir=None) # type: ignore + + if type(x.value) is AssertionError: + pass + else: + assert type(x.value) is ValueError + assert str(x.value) == "Argument 'base_dir' is not defined." + + # ----------- + logging.info("temp directory [{}] is deleting".format(tmp_dir)) + node_svc.os_ops.rmdir(tmp_dir) + + def test_node_app__make_empty__base_dir_is_Empty(self, node_svc: PostgresNodeService): + assert type(node_svc) is PostgresNodeService + + assert isinstance(node_svc.os_ops, OsOperations) + assert node_svc.port_manager is not None + assert isinstance(node_svc.port_manager, PortManager) + + tmp_dir = node_svc.os_ops.mkdtemp() + assert tmp_dir is not None + assert type(tmp_dir) is str + logging.info("temp directory is [{}]".format(tmp_dir)) + + # ----------- + os_ops = node_svc.os_ops.create_clone() + assert os_ops is not node_svc.os_ops + + # ----------- + with __class__.tag_rmdirs_protector(os_ops): + node_app = NodeApp(test_path=tmp_dir, os_ops=os_ops) + assert node_app.os_ops is os_ops + + with pytest.raises(expected_exception=ValueError) as x: + node_app.make_empty(base_dir="") + + assert str(x.value) == "Argument 'base_dir' is empty." + + # ----------- + logging.info("temp directory [{}] is deleting".format(tmp_dir)) + node_svc.os_ops.rmdir(tmp_dir) + + def test_node_app__make_empty(self, node_svc: PostgresNodeService): + assert type(node_svc) is PostgresNodeService + + assert isinstance(node_svc.os_ops, OsOperations) + assert node_svc.port_manager is not None + assert isinstance(node_svc.port_manager, PortManager) + + tmp_dir = node_svc.os_ops.mkdtemp() + assert tmp_dir is not None + assert type(tmp_dir) is str + logging.info("temp directory is [{}]".format(tmp_dir)) + + # ----------- + node_app = NodeApp( + test_path=tmp_dir, + os_ops=node_svc.os_ops, + port_manager=node_svc.port_manager + ) + + assert node_app.os_ops is node_svc.os_ops + assert node_app.port_manager is node_svc.port_manager + assert type(node_app.nodes_to_cleanup) is list + assert len(node_app.nodes_to_cleanup) == 0 + + node: typing.Optional[PostgresNode] = None + try: + node = node_app.make_simple("node") + assert node is not None + assert isinstance(node, PostgresNode) + assert node.os_ops is node_svc.os_ops + assert node.port_manager is node_svc.port_manager + + assert type(node_app.nodes_to_cleanup) is list + assert len(node_app.nodes_to_cleanup) == 1 + assert node_app.nodes_to_cleanup[0] is node + + node.slow_start() + finally: + if node is not None: + node.stop() + node.release_resources() + + if node is not None: + node.cleanup(release_resources=True) + + # ----------- + logging.info("temp directory [{}] is deleting".format(tmp_dir)) + node_svc.os_ops.rmdir(tmp_dir) + + def test_node_app__make_simple__checksum(self, node_svc: PostgresNodeService): + assert type(node_svc) is PostgresNodeService + + assert isinstance(node_svc.os_ops, OsOperations) + assert node_svc.port_manager is not None + assert isinstance(node_svc.port_manager, PortManager) + + tmp_dir = node_svc.os_ops.mkdtemp() + assert tmp_dir is not None + assert type(tmp_dir) is str + + logging.info("temp directory is [{}]".format(tmp_dir)) + node_app = NodeApp(test_path=tmp_dir, os_ops=node_svc.os_ops) + + C_NODE = "node" + + # ----------- + def LOCAL__test(checksum: bool, initdb_params: typing.Optional[list]): + initdb_params0 = initdb_params + initdb_params0_copy = initdb_params0.copy() if initdb_params0 is not None else None + + with node_app.make_simple(C_NODE, checksum=checksum, initdb_params=initdb_params): + assert initdb_params is initdb_params0 + if initdb_params0 is not None: + assert initdb_params0 == initdb_params0_copy + + assert initdb_params is initdb_params0 + if initdb_params0 is not None: + assert initdb_params0 == initdb_params0_copy + + # ----------- + LOCAL__test(checksum=False, initdb_params=None) + LOCAL__test(checksum=True, initdb_params=None) + + # ----------- + params = [] + LOCAL__test(checksum=False, initdb_params=params) + LOCAL__test(checksum=True, initdb_params=params) + + # ----------- + params = ["--no-sync"] + LOCAL__test(checksum=False, initdb_params=params) + LOCAL__test(checksum=True, initdb_params=params) + + # ----------- + params = ["--data-checksums"] + LOCAL__test(checksum=False, initdb_params=params) + LOCAL__test(checksum=True, initdb_params=params) + + # ----------- + logging.info("temp directory [{}] is deleting".format(tmp_dir)) + node_svc.os_ops.rmdir(tmp_dir) + + def test_node_app__make_empty_with_explicit_port(self, node_svc: PostgresNodeService): + assert type(node_svc) is PostgresNodeService + + assert isinstance(node_svc.os_ops, OsOperations) + assert node_svc.port_manager is not None + assert isinstance(node_svc.port_manager, PortManager) + + C_MAX_ATTEMPTS = 5 + + tmp_dir = node_svc.os_ops.mkdtemp() + assert tmp_dir is not None + assert type(tmp_dir) is str + logging.info("temp directory is [{}]".format(tmp_dir)) + + # ----------- + node_app = NodeApp( + test_path=tmp_dir, + os_ops=node_svc.os_ops, + port_manager=node_svc.port_manager + ) + + assert node_app.os_ops is node_svc.os_ops + assert node_app.port_manager is not None + assert node_app.port_manager is node_svc.port_manager + assert type(node_app.nodes_to_cleanup) is list + assert len(node_app.nodes_to_cleanup) == 0 + + attempt = 0 + ports = [] + try: + while True: + if attempt == C_MAX_ATTEMPTS: + raise RuntimeError("Node did not start.") + + attempt += 1 + + logging.info("------------- attempt #{}".format( + attempt + )) + + port = node_app.port_manager.reserve_port() + assert type(port) is int + assert port is not ports + + try: + ports.append(port) + except: # noqa: E722 + node_app.port_manager.release_port(port) + raise + + assert len(ports) == attempt + node_name = "node_{}".format(attempt) + + logging.info("Node [{}] is creating...".format(node_name)) + node = node_app.make_simple(node_name, port=port) + assert node is not None + assert isinstance(node, PostgresNode) + assert node.os_ops is node_svc.os_ops + assert node.port_manager is None # <--------- + assert node.port == port + assert node._should_free_port == False # noqa: E712 + + assert type(node_app.nodes_to_cleanup) is list + assert len(node_app.nodes_to_cleanup) == attempt + assert node_app.nodes_to_cleanup[-1] is node + + assert node.status() == NodeStatus.Stopped + logging.info("Node is created") + + logging.info("Try to start a node...") + try: + node.slow_start() + except StartNodeException as e: + logging.info("Exception ({}): {}".format( + type(e).__name__, + e + )) + assert node.status() == NodeStatus.Stopped + continue + + assert node.status() == NodeStatus.Running + logging.info("Node is started") + + logging.info("Stop node") + node.stop() + assert node.status() == NodeStatus.Stopped + logging.info("Node is stopped") + + logging.info("OK. Go home.") + assert node is not None + assert isinstance(node, PostgresNode) + assert node._port is not None + assert node._port == port + assert not node._should_free_port + break + finally: + assert node_app.port_manager is not None + while len(ports) > 0: + node_app.port_manager.release_port(ports.pop()) + + # ----------- + logging.info("temp directory [{}] is deleting".format(tmp_dir)) + node_svc.os_ops.rmdirs(tmp_dir) + return + + def test_node_app__make_empty_and_pgconf(self, node_svc: PostgresNodeService): + assert type(node_svc) is PostgresNodeService + + assert type(node_svc) is PostgresNodeService + + assert isinstance(node_svc.os_ops, OsOperations) + assert node_svc.port_manager is not None + assert isinstance(node_svc.port_manager, PortManager) + + tmp_dir = node_svc.os_ops.mkdtemp() + assert tmp_dir is not None + assert type(tmp_dir) is str + logging.info("temp directory is [{}]".format(tmp_dir)) + + # ----------- + node_app = NodeApp( + test_path=tmp_dir, + os_ops=node_svc.os_ops, + port_manager=node_svc.port_manager + ) + + # TODO: We have to use node_svc.os_ops here + pgConfOsOps = PgCfgOsOps( + node_svc.os_ops, + "utf-8", + ) + + with node_app.make_simple("abc") as node: + node_conf = testgres_pgconf.PostgresConfiguration(node.data_dir, pgConfOsOps) + + logging.info("Configuration is readed ...") + testgres_pgconf.PostgresConfigurationReader.LoadConfiguration(node_conf) + + logging.info("Configuration is checked ...") + prop__port = node_conf.GetOptionValue("port") + assert type(prop__port) is int + assert prop__port == node.port + # presets are checked + prop__fsync = node_conf.GetOptionValue("fsync") + assert prop__fsync == "off" or prop__fsync is False + prop__log_statement = node_conf.GetOptionValue("log_statement") + assert type(prop__log_statement) is str + assert prop__log_statement == "none" + prop__wal_level = node_conf.GetOptionValue("wal_level") + assert type(prop__wal_level) is str + assert prop__wal_level == "logical" + + logging.info("Configuration is written ...") + testgres_pgconf.PostgresConfigurationWriter.WriteConfiguration(node_conf) + + logging.info("Node is started ...") + node.slow_start() + + assert node.status() == NodeStatus.Running + return + + @staticmethod + def helper__get_node( + node_svc: PostgresNodeService, + name: typing.Optional[str] = None, + port: typing.Optional[int] = None, + port_manager: typing.Optional[PortManager] = None + ) -> PostgresNode: + assert isinstance(node_svc, PostgresNodeService) + assert isinstance(node_svc.os_ops, OsOperations) + assert isinstance(node_svc.port_manager, PortManager) + + if port_manager is None: + port_manager = node_svc.port_manager + + return PostgresNode( + name, + port=port, + os_ops=node_svc.os_ops, + port_manager=port_manager if port is None else None + ) + + @staticmethod + def helper__skip_test_if_pg_version_is_not_ge(ver1: str, ver2: str): + assert type(ver1) is str + assert type(ver2) is str + if not __class__.helper__pg_version_ge(ver1, ver2): + pytest.skip('requires {0}+'.format(ver2)) + + @staticmethod + def helper__skip_test_if_pg_version_is_ge(ver1: str, ver2: str): + assert type(ver1) is str + assert type(ver2) is str + if __class__.helper__pg_version_ge(ver1, ver2): + pytest.skip('requires <{0}'.format(ver2)) + + @staticmethod + def helper__pg_version_ge(ver1: str, ver2: str) -> bool: + assert type(ver1) is str + assert type(ver2) is str + v1 = PgVer(ver1) + v2 = PgVer(ver2) + return v1 >= v2 + + @staticmethod + def helper__rm_carriage_returns(out): + """ + In Windows we have additional '\r' symbols in output. + Let's get rid of them. + """ + if isinstance(out, (int, float, complex)): + return out + + if isinstance(out, tuple): + return tuple(__class__.helper__rm_carriage_returns(item) for item in out) + + if isinstance(out, bytes): + return out.replace(b'\r', b'') + + assert type(out) is str + return out.replace('\r', '') + + @staticmethod + def helper__skip_test_if_util_not_exist(os_ops: OsOperations, name: str): + assert isinstance(os_ops, OsOperations) + assert type(name) is str + if not __class__.helper__util_exists(os_ops, name): + pytest.skip('might be missing') + + @staticmethod + def helper__util_exists(os_ops: OsOperations, util): + assert isinstance(os_ops, OsOperations) + + def good_properties(f): + return (os_ops.path_exists(f) and # noqa: W504 + os_ops.isfile(f) and # noqa: W504 + os_ops.is_executable(f)) # yapf: disable + + # try to resolve it + if good_properties(get_bin_path2(os_ops, util)): + return True + + # check if util is in PATH + for path in os_ops.environ("PATH").split(os.pathsep): + if good_properties(os.path.join(path, util)): + return True diff --git a/tests/test_testgres_local.py b/tests/test_testgres_local.py new file mode 100644 index 00000000..012ec12d --- /dev/null +++ b/tests/test_testgres_local.py @@ -0,0 +1,384 @@ +# coding: utf-8 +import os +import re +import subprocess +import pytest +import psutil +import platform +import logging + +import src as testgres + +from src import StartNodeException +from src import ExecUtilException +from src import NodeApp +from src import NodeStatus +from src import scoped_config +from src import get_new_node +from src import get_bin_path +from src import get_pg_config +from src import get_pg_version + +# NOTE: those are ugly imports +from src.utils import PgVer +from src.node import ProcessProxy + + +def pg_version_ge(version): + cur_ver = PgVer(get_pg_version()) + min_ver = PgVer(version) + return cur_ver >= min_ver + + +def util_exists(util): + def good_properties(f): + return (os.path.exists(f) and # noqa: W504 + os.path.isfile(f) and # noqa: W504 + os.access(f, os.X_OK)) # yapf: disable + + # try to resolve it + if good_properties(get_bin_path(util)): + return True + + # check if util is in PATH + for path in os.environ["PATH"].split(os.pathsep): + if good_properties(os.path.join(path, util)): + return True + + +def rm_carriage_returns(out): + """ + In Windows we have additional '\r' symbols in output. + Let's get rid of them. + """ + if os.name == 'nt': + if isinstance(out, (int, float, complex)): + return out + elif isinstance(out, tuple): + return tuple(rm_carriage_returns(item) for item in out) + elif isinstance(out, bytes): + return out.replace(b'\r', b'') + else: + return out.replace('\r', '') + else: + return out + + +class TestTestgresLocal: + def test_pg_config(self): + # check same instances + a = get_pg_config() + b = get_pg_config() + assert (id(a) == id(b)) + + # save right before config change + c1 = get_pg_config() + + # modify setting for this scope + with scoped_config(cache_pg_config=False) as config: + # sanity check for value + assert not (config.cache_pg_config) + + # save right after config change + c2 = get_pg_config() + + # check different instances after config change + assert (id(c1) != id(c2)) + + # check different instances + a = get_pg_config() + b = get_pg_config() + assert (id(a) != id(b)) + + def test_child_process_dies(self): + # test for FileNotFound exception during child_processes() function + cmd = ["timeout", "60"] if os.name == 'nt' else ["sleep", "60"] + + nAttempt = 0 + + while True: + if nAttempt == 5: + raise Exception("Max attempt number is exceed.") + + nAttempt += 1 + + logging.info("Attempt #{0}".format(nAttempt)) + + with subprocess.Popen(cmd, shell=True) as process: # shell=True might be needed on Windows + r = process.poll() + + if r is not None: + logging.warning("process.pool() returns an unexpected result: {0}.".format(r)) + continue + + assert r is None + # collect list of processes currently running + children = psutil.Process(os.getpid()).children() + # kill a process, so received children dictionary becomes invalid + process.kill() + process.wait() + # try to handle children list -- missing processes will have ptype "ProcessType.Unknown" + [ProcessProxy(p) for p in children] + break + + def test_upgrade_node(self): + old_bin_dir = os.path.dirname(get_bin_path("pg_config")) + new_bin_dir = os.path.dirname(get_bin_path("pg_config")) + with get_new_node(prefix='node_old', bin_dir=old_bin_dir) as node_old: + node_old.init() + node_old.start() + node_old.stop() + with get_new_node(prefix='node_new', bin_dir=new_bin_dir) as node_new: + node_new.init(cached=False) + res = node_new.upgrade_from(old_node=node_old) + node_new.start() + assert (b'Upgrade Complete' in res) + + class tagPortManagerProxy: + sm_prev_testgres_reserve_port = None + sm_prev_testgres_release_port = None + + sm_DummyPortNumber = None + sm_DummyPortMaxUsage = None + + sm_DummyPortCurrentUsage = None + sm_DummyPortTotalUsage = None + + def __init__(self, dummyPortNumber, dummyPortMaxUsage): + assert type(dummyPortNumber) is int + assert type(dummyPortMaxUsage) is int + assert dummyPortNumber >= 0 + assert dummyPortMaxUsage >= 0 + + assert __class__.sm_prev_testgres_reserve_port is None + assert __class__.sm_prev_testgres_release_port is None + assert testgres.utils.reserve_port == testgres.utils.internal__reserve_port + assert testgres.utils.release_port == testgres.utils.internal__release_port + + __class__.sm_prev_testgres_reserve_port = testgres.utils.reserve_port + __class__.sm_prev_testgres_release_port = testgres.utils.release_port + + testgres.utils.reserve_port = __class__._proxy__reserve_port + testgres.utils.release_port = __class__._proxy__release_port + + assert testgres.utils.reserve_port == __class__._proxy__reserve_port + assert testgres.utils.release_port == __class__._proxy__release_port + + __class__.sm_DummyPortNumber = dummyPortNumber + __class__.sm_DummyPortMaxUsage = dummyPortMaxUsage + + __class__.sm_DummyPortCurrentUsage = 0 + __class__.sm_DummyPortTotalUsage = 0 + + def __enter__(self): + return self + + def __exit__(self, type, value, traceback): + assert __class__.sm_DummyPortCurrentUsage == 0 + + assert __class__.sm_prev_testgres_reserve_port is not None + assert __class__.sm_prev_testgres_release_port is not None + + assert testgres.utils.reserve_port == __class__._proxy__reserve_port + assert testgres.utils.release_port == __class__._proxy__release_port + + testgres.utils.reserve_port = __class__.sm_prev_testgres_reserve_port + testgres.utils.release_port = __class__.sm_prev_testgres_release_port + + __class__.sm_prev_testgres_reserve_port = None + __class__.sm_prev_testgres_release_port = None + + @staticmethod + def _proxy__reserve_port(): + assert type(__class__.sm_DummyPortMaxUsage) is int + assert type(__class__.sm_DummyPortTotalUsage) is int + assert type(__class__.sm_DummyPortCurrentUsage) is int + assert __class__.sm_DummyPortTotalUsage >= 0 + assert __class__.sm_DummyPortCurrentUsage >= 0 + + assert __class__.sm_DummyPortTotalUsage <= __class__.sm_DummyPortMaxUsage + assert __class__.sm_DummyPortCurrentUsage <= __class__.sm_DummyPortTotalUsage + + assert __class__.sm_prev_testgres_reserve_port is not None + + if __class__.sm_DummyPortTotalUsage == __class__.sm_DummyPortMaxUsage: + return __class__.sm_prev_testgres_reserve_port() + + __class__.sm_DummyPortTotalUsage += 1 + __class__.sm_DummyPortCurrentUsage += 1 + return __class__.sm_DummyPortNumber + + @staticmethod + def _proxy__release_port(dummyPortNumber): + assert type(dummyPortNumber) is int + + assert type(__class__.sm_DummyPortMaxUsage) is int + assert type(__class__.sm_DummyPortTotalUsage) is int + assert type(__class__.sm_DummyPortCurrentUsage) is int + assert __class__.sm_DummyPortTotalUsage >= 0 + assert __class__.sm_DummyPortCurrentUsage >= 0 + + assert __class__.sm_DummyPortTotalUsage <= __class__.sm_DummyPortMaxUsage + assert __class__.sm_DummyPortCurrentUsage <= __class__.sm_DummyPortTotalUsage + + assert __class__.sm_prev_testgres_release_port is not None + + if __class__.sm_DummyPortCurrentUsage > 0 and dummyPortNumber == __class__.sm_DummyPortNumber: + assert __class__.sm_DummyPortTotalUsage > 0 + __class__.sm_DummyPortCurrentUsage -= 1 + return + + return __class__.sm_prev_testgres_release_port(dummyPortNumber) + + def test_port_rereserve_during_node_start(self): + assert testgres.PostgresNode._C_MAX_START_ATEMPTS == 5 + + C_COUNT_OF_BAD_PORT_USAGE = 3 + + with get_new_node() as node1: + node1.init().start() + assert (node1._should_free_port) + assert (type(node1.port) is int) + node1_port_copy = node1.port + assert (rm_carriage_returns(node1.safe_psql("SELECT 1;")) == b'1\n') + + with __class__.tagPortManagerProxy(node1.port, C_COUNT_OF_BAD_PORT_USAGE): + assert __class__.tagPortManagerProxy.sm_DummyPortNumber == node1.port + with get_new_node() as node2: + assert (node2._should_free_port) + assert (node2.port == node1.port) + + node2.init().start() + + assert (node2.port != node1.port) + assert (node2._should_free_port) + assert (__class__.tagPortManagerProxy.sm_DummyPortCurrentUsage == 0) + assert (__class__.tagPortManagerProxy.sm_DummyPortTotalUsage == C_COUNT_OF_BAD_PORT_USAGE) + assert (node2.is_started) + + assert (rm_carriage_returns(node2.safe_psql("SELECT 2;")) == b'2\n') + + # node1 is still working + assert (node1.port == node1_port_copy) + assert (node1._should_free_port) + assert (rm_carriage_returns(node1.safe_psql("SELECT 3;")) == b'3\n') + + def test_port_conflict(self): + assert testgres.PostgresNode._C_MAX_START_ATEMPTS > 1 + + C_COUNT_OF_BAD_PORT_USAGE = testgres.PostgresNode._C_MAX_START_ATEMPTS + + with get_new_node() as node1: + node1.init().start() + assert (node1._should_free_port) + assert (type(node1.port) is int) + node1_port_copy = node1.port + assert (rm_carriage_returns(node1.safe_psql("SELECT 1;")) == b'1\n') + + with __class__.tagPortManagerProxy(node1.port, C_COUNT_OF_BAD_PORT_USAGE): + assert __class__.tagPortManagerProxy.sm_DummyPortNumber == node1.port + with get_new_node() as node2: + assert (node2._should_free_port) + assert (node2.port == node1.port) + + node2.init() + assert (node2.status() == NodeStatus.Stopped) + with pytest.raises( + expected_exception=StartNodeException, + match=re.escape("Cannot start node after multiple attempts.") + ): + node2.start() + + assert (node2.port == node1.port) + assert (node2._should_free_port) + assert (__class__.tagPortManagerProxy.sm_DummyPortCurrentUsage == 1) + assert (__class__.tagPortManagerProxy.sm_DummyPortTotalUsage == C_COUNT_OF_BAD_PORT_USAGE) + assert (not node2.is_started) + assert (node2.status() == NodeStatus.Stopped) + + # node2 must release our dummyPort (node1.port) + assert (__class__.tagPortManagerProxy.sm_DummyPortCurrentUsage == 0) + + # node1 is still working + assert (node1.port == node1_port_copy) + assert (node1._should_free_port) + assert (node1.status() == NodeStatus.Running) + assert (rm_carriage_returns(node1.safe_psql("SELECT 3;")) == b'3\n') + + def test_simple_with_bin_dir(self): + with get_new_node() as node: + node.init().start() + bin_dir = node.bin_dir + + app = NodeApp() + with app.make_simple(base_dir=node.base_dir, bin_dir=bin_dir) as correct_bin_dir: + correct_bin_dir.slow_start() + correct_bin_dir.safe_psql("SELECT 1;") + correct_bin_dir.stop() + + while True: + try: + app.make_simple(base_dir=node.base_dir, bin_dir="wrong/path") + except FileNotFoundError: + break # Expected error + except ExecUtilException: + break # Expected error + + raise RuntimeError("Error was expected.") # We should not reach this + + return + + def test_set_auto_conf(self): + # elements contain [property id, value, storage value] + testData = [ + ["archive_command", + "cp '%p' \"/mnt/server/archivedir/%f\"", + "'cp \\'%p\\' \"/mnt/server/archivedir/%f\""], + ["log_line_prefix", + "'\n\r\t\b\\\"", + "'\\\'\\n\\r\\t\\b\\\\\""], + ["log_connections", + True, + "on"], + ["log_disconnections", + False, + "off"], + ["autovacuum_max_workers", + 3, + "3"] + ] + if pg_version_ge('12'): + testData.append(["restore_command", + 'cp "/mnt/server/archivedir/%f" \'%p\'', + "'cp \"/mnt/server/archivedir/%f\" \\'%p\\''"]) + + with get_new_node() as node: + node.init().start() + + options = {} + + for x in testData: + options[x[0]] = x[1] + + node.set_auto_conf(options) + node.stop() + node.slow_start() + + auto_conf_path = f"{node.data_dir}/postgresql.auto.conf" + with open(auto_conf_path, "r") as f: + content = f.read() + + for x in testData: + assert x[0] + " = " + x[2] in content + + @staticmethod + def helper__skip_test_if_util_not_exist(name: str): + assert type(name) is str + + if platform.system().lower() == "windows": + name2 = name + ".exe" + else: + name2 = name + + if not util_exists(name2): + pytest.skip('might be missing') diff --git a/tests/test_testgres_remote.py b/tests/test_testgres_remote.py new file mode 100755 index 00000000..3f97ffa6 --- /dev/null +++ b/tests/test_testgres_remote.py @@ -0,0 +1,216 @@ +# coding: utf-8 +import os + +import pytest +import logging +import typing + +from .helpers.global_data import PostgresNodeService +from .helpers.global_data import PostgresNodeServices + +import src as testgres + +from src.exceptions import InitNodeException +from src.exceptions import ExecUtilException + +from src.config import scoped_config +from src.config import testgres_config + +from src import get_bin_path +from src import get_pg_config + +# NOTE: those are ugly imports + +from packaging.version import Version + + +def util_exists(util): + def good_properties(f): + return (testgres_config.os_ops.path_exists(f) and # noqa: W504 + testgres_config.os_ops.isfile(f) and # noqa: W504 + testgres_config.os_ops.is_executable(f)) # yapf: disable + + # try to resolve it + if good_properties(get_bin_path(util)): + return True + + # check if util is in PATH + for path in testgres_config.os_ops.environ("PATH").split(testgres_config.os_ops.pathsep): + if good_properties(os.path.join(path, util)): + return True + + +class TestTestgresRemote: + @pytest.fixture(autouse=True, scope="class") + def implicit_fixture(self): + cur_os_ops = PostgresNodeServices.sm_remote.os_ops + assert cur_os_ops is not None + + prev_ops = testgres_config.os_ops + assert prev_ops is not None + testgres_config.set_os_ops(os_ops=cur_os_ops) + assert testgres_config.os_ops is cur_os_ops + yield + assert testgres_config.os_ops is cur_os_ops + testgres_config.set_os_ops(os_ops=prev_ops) + assert testgres_config.os_ops is prev_ops + + def test_init__LANG_ะก(self): + # PBCKP-1744 + prev_LANG = os.environ.get("LANG") + + try: + os.environ["LANG"] = "C" + + with __class__.helper__get_node() as node: + node.init().start() + finally: + __class__.helper__restore_envvar("LANG", prev_LANG) + + def test_init__unk_LANG_and_LC_CTYPE(self): + # PBCKP-1744 + prev_LANG = os.environ.get("LANG") + prev_LANGUAGE = os.environ.get("LANGUAGE") + prev_LC_CTYPE = os.environ.get("LC_CTYPE") + prev_LC_COLLATE = os.environ.get("LC_COLLATE") + + node = __class__.helper__get_node() + + node_version = node.version + assert node_version is not None + assert type(node_version) is testgres.utils.PgVer + assert isinstance(node_version, Version) + + if node.version < Version("11"): + node.cleanup(release_resources=True) + pytest.skip("This test does not work on old PG10-.") + + try: + # TODO: Pass unkData through test parameter. + unkDatas = [ + ("UNKNOWN_LANG", "UNKNOWN_CTYPE"), + ("\"UNKNOWN_LANG\"", "\"UNKNOWN_CTYPE\""), + ("\\UNKNOWN_LANG\\", "\\UNKNOWN_CTYPE\\"), + ("\"UNKNOWN_LANG", "UNKNOWN_CTYPE\""), + ("\\UNKNOWN_LANG", "UNKNOWN_CTYPE\\"), + ("\\", "\\"), + ("\"", "\""), + ] + + errorIsDetected = False + + for unkData in unkDatas: + logging.info("----------------------") + logging.info("Unk LANG is [{0}]".format(unkData[0])) + logging.info("Unk LC_CTYPE is [{0}]".format(unkData[1])) + + os.environ["LANG"] = unkData[0] + os.environ.pop("LANGUAGE", None) + os.environ["LC_CTYPE"] = unkData[1] + os.environ.pop("LC_COLLATE", None) + + assert os.environ.get("LANG") == unkData[0] + assert "LANGUAGE" not in os.environ.keys() + assert os.environ.get("LC_CTYPE") == unkData[1] + assert "LC_COLLATE" not in os.environ.keys() + + assert os.getenv('LANG') == unkData[0] + assert os.getenv('LANGUAGE') is None + assert os.getenv('LC_CTYPE') == unkData[1] + assert os.getenv('LC_COLLATE') is None + + node.cleanup() + + exc: typing.Optional[BaseException] = None + try: + node.init() # IT RAISES! + except InitNodeException as e: + exc = e.__cause__ + assert exc is not None + assert isinstance(exc, ExecUtilException) + + if exc is None: + logging.warning("We expected an error!") + continue + + errorIsDetected = True + + assert isinstance(exc, ExecUtilException) + + errMsg = str(exc) + logging.info("Error message is {0}: {1}".format(type(exc).__name__, errMsg)) + + assert type(exc.error) is str + + # Check an optional message from OS. + expectedMsg1 = "warning: setlocale: LC_CTYPE: cannot change locale (" + unkData[1] + ")" + + if expectedMsg1 not in exc.error: + logging.warning("Msg does not contain {!r}.".format(expectedMsg1)) + + # Check a mandatory message from initdb. + expectedMsg2 = "initdb: error: invalid locale settings; check LANG and LC_* environment variables" + assert expectedMsg2 in exc.error + continue + + node.cleanup(release_resources=True) + + if not errorIsDetected: + pytest.xfail("All the bad data are processed without errors!") + + finally: + __class__.helper__restore_envvar("LANG", prev_LANG) + __class__.helper__restore_envvar("LANGUAGE", prev_LANGUAGE) + __class__.helper__restore_envvar("LC_CTYPE", prev_LC_CTYPE) + __class__.helper__restore_envvar("LC_COLLATE", prev_LC_COLLATE) + + def test_pg_config(self): + # check same instances + a = get_pg_config() + b = get_pg_config() + assert (id(a) == id(b)) + + # save right before config change + c1 = get_pg_config() + + # modify setting for this scope + with scoped_config(cache_pg_config=False) as config: + # sanity check for value + assert not (config.cache_pg_config) + + # save right after config change + c2 = get_pg_config() + + # check different instances after config change + assert (id(c1) != id(c2)) + + # check different instances + a = get_pg_config() + b = get_pg_config() + assert (id(a) != id(b)) + + @staticmethod + def helper__get_node(name=None): + svc = PostgresNodeServices.sm_remote + + assert isinstance(svc, PostgresNodeService) + assert isinstance(svc.os_ops, testgres.OsOperations) + assert isinstance(svc.port_manager, testgres.PortManager) + + return testgres.PostgresNode( + name, + os_ops=svc.os_ops, + port_manager=svc.port_manager) + + @staticmethod + def helper__restore_envvar(name, prev_value): + if prev_value is None: + os.environ.pop(name, None) + else: + os.environ[name] = prev_value + + @staticmethod + def helper__skip_test_if_util_not_exist(name: str): + assert type(name) is str + if not util_exists(name): + pytest.skip('might be missing') diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 00000000..92505752 --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,66 @@ +from .helpers.global_data import OsOpsDescr +from .helpers.global_data import OsOpsDescrs +from .helpers.global_data import OsOperations + +from src.utils import parse_pg_version +from src.utils import get_pg_config2 +from src import scoped_config + +import pytest +import typing + + +class TestUtils: + sm_os_ops_descrs: typing.List[OsOpsDescr] = [ + OsOpsDescrs.sm_local_os_ops_descr, + OsOpsDescrs.sm_remote_os_ops_descr + ] + + @pytest.fixture( + params=[descr.os_ops for descr in sm_os_ops_descrs], + ids=[descr.sign for descr in sm_os_ops_descrs] + ) + def os_ops(self, request: pytest.FixtureRequest) -> OsOperations: + assert isinstance(request, pytest.FixtureRequest) + assert isinstance(request.param, OsOperations) + return request.param + + def test_parse_pg_version(self): + # Linux Mint + assert parse_pg_version("postgres (PostgreSQL) 15.5 (Ubuntu 15.5-1.pgdg22.04+1)") == "15.5" + # Linux Ubuntu + assert parse_pg_version("postgres (PostgreSQL) 12.17") == "12.17" + # Windows + assert parse_pg_version("postgres (PostgreSQL) 11.4") == "11.4" + # Macos + assert parse_pg_version("postgres (PostgreSQL) 14.9 (Homebrew)") == "14.9" + # Postgres Pro trial + assert parse_pg_version("postgres (PostgreSQL) 18.4-TRIAL") == "18.4" + assert parse_pg_version("PostgreSQL 18.4-TRIAL") == "18.4" + + def test_get_pg_config2(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + + # check same instances + a = get_pg_config2(os_ops, None) + b = get_pg_config2(os_ops, None) + assert (id(a) == id(b)) + + # save right before config change + c1 = get_pg_config2(os_ops, None) + + # modify setting for this scope + with scoped_config(cache_pg_config=False) as config: + # sanity check for value + assert not (config.cache_pg_config) + + # save right after config change + c2 = get_pg_config2(os_ops, None) + + # check different instances after config change + assert (id(c1) != id(c2)) + + # check different instances + a = get_pg_config2(os_ops, None) + b = get_pg_config2(os_ops, None) + assert (id(a) != id(b)) diff --git a/tests/units/__init__.py b/tests/units/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/units/exceptions/BackupException/__init__.py b/tests/units/exceptions/BackupException/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/units/exceptions/BackupException/test_set001__constructor.py b/tests/units/exceptions/BackupException/test_set001__constructor.py new file mode 100644 index 00000000..c0b55001 --- /dev/null +++ b/tests/units/exceptions/BackupException/test_set001__constructor.py @@ -0,0 +1,24 @@ +from src.exceptions import BackupException +from src.exceptions import TestgresException as testgres__TestgresException + + +class TestSet001_Constructor: + def test_001__default(self): + e = BackupException() + assert type(e) is BackupException + assert isinstance(e, testgres__TestgresException) + assert e.source is None + assert e.message == "" + assert str(e) == "" + assert repr(e) == "BackupException()" + return + + def test_002__message(self): + e = BackupException(message="abc\n123") + assert type(e) is BackupException + assert isinstance(e, testgres__TestgresException) + assert e.source is None + assert e.message == "abc\n123" + assert str(e) == "abc\n123" + assert repr(e) == "BackupException(message='abc\\n123')" + return diff --git a/tests/units/exceptions/CatchUpException/__init__.py b/tests/units/exceptions/CatchUpException/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/units/exceptions/CatchUpException/test_set001__constructor.py b/tests/units/exceptions/CatchUpException/test_set001__constructor.py new file mode 100644 index 00000000..0e701264 --- /dev/null +++ b/tests/units/exceptions/CatchUpException/test_set001__constructor.py @@ -0,0 +1,24 @@ +from src.exceptions import CatchUpException +from src.exceptions import TestgresException as testgres__TestgresException + + +class TestSet001_Constructor: + def test_001__default(self): + e = CatchUpException() + assert type(e) is CatchUpException + assert isinstance(e, testgres__TestgresException) + assert e.source is None + assert e.message == "" + assert str(e) == "" + assert repr(e) == "CatchUpException()" + return + + def test_002__message(self): + e = CatchUpException(message="abc\n123") + assert type(e) is CatchUpException + assert isinstance(e, testgres__TestgresException) + assert e.source is None + assert e.message == "abc\n123" + assert str(e) == "abc\n123" + assert repr(e) == "CatchUpException(message='abc\\n123')" + return diff --git a/tests/units/exceptions/InitNodeException/__init__.py b/tests/units/exceptions/InitNodeException/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/units/exceptions/InitNodeException/test_set001__constructor.py b/tests/units/exceptions/InitNodeException/test_set001__constructor.py new file mode 100644 index 00000000..91999c00 --- /dev/null +++ b/tests/units/exceptions/InitNodeException/test_set001__constructor.py @@ -0,0 +1,24 @@ +from src.exceptions import InitNodeException +from src.exceptions import TestgresException as testgres__TestgresException + + +class TestSet001_Constructor: + def test_001__default(self): + e = InitNodeException() + assert type(e) is InitNodeException + assert isinstance(e, testgres__TestgresException) + assert e.source is None + assert e.message == "" + assert str(e) == "" + assert repr(e) == "InitNodeException()" + return + + def test_002__message(self): + e = InitNodeException(message="abc\n123") + assert type(e) is InitNodeException + assert isinstance(e, testgres__TestgresException) + assert e.source is None + assert e.message == "abc\n123" + assert str(e) == "abc\n123" + assert repr(e) == "InitNodeException(message='abc\\n123')" + return diff --git a/tests/units/exceptions/PortForException/__init__.py b/tests/units/exceptions/PortForException/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/units/exceptions/PortForException/test_set001__constructor.py b/tests/units/exceptions/PortForException/test_set001__constructor.py new file mode 100644 index 00000000..f7e1f2ca --- /dev/null +++ b/tests/units/exceptions/PortForException/test_set001__constructor.py @@ -0,0 +1,24 @@ +from src.exceptions import PortForException +from src.exceptions import TestgresException as testgres__TestgresException + + +class TestSet001_Constructor: + def test_001__default(self): + e = PortForException() + assert type(e) is PortForException + assert isinstance(e, testgres__TestgresException) + assert e.source is None + assert e.message == "" + assert str(e) == "" + assert repr(e) == "PortForException()" + return + + def test_002__message(self): + e = PortForException(message="abc\n123") + assert type(e) is PortForException + assert isinstance(e, testgres__TestgresException) + assert e.source is None + assert e.message == "abc\n123" + assert str(e) == "abc\n123" + assert repr(e) == "PortForException(message='abc\\n123')" + return diff --git a/tests/units/exceptions/QueryException/__init__.py b/tests/units/exceptions/QueryException/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/units/exceptions/QueryException/test_set001__constructor.py b/tests/units/exceptions/QueryException/test_set001__constructor.py new file mode 100644 index 00000000..655e1eb7 --- /dev/null +++ b/tests/units/exceptions/QueryException/test_set001__constructor.py @@ -0,0 +1,52 @@ +from src.exceptions import QueryException +from src.exceptions import TestgresException as testgres__TestgresException + + +class TestSet001_Constructor: + def test_001__default(self): + e = QueryException() + assert type(e) is QueryException + assert isinstance(e, testgres__TestgresException) + assert e.source is None + assert e.message == "" + assert e.description is None + assert e.query is None + assert str(e) == "" + assert repr(e) == "QueryException()" + return + + def test_002__message(self): + e = QueryException(message="abc\n123") + assert type(e) is QueryException + assert isinstance(e, testgres__TestgresException) + assert e.source is None + assert e.message == "abc\n123" + assert e.description == "abc\n123" + assert e.query is None + assert str(e) == "abc\n123" + assert repr(e) == "QueryException(message='abc\\n123')" + return + + def test_003__query(self): + e = QueryException(query="cba\n321") + assert type(e) is QueryException + assert isinstance(e, testgres__TestgresException) + assert e.source is None + assert e.message == "Query: cba\n321" + assert e.description is None + assert e.query == "cba\n321" + assert str(e) == "Query: cba\n321" + assert repr(e) == "QueryException(query='cba\\n321')" + return + + def test_004__all(self): + e = QueryException(message="mmm", query="cba\n321") + assert type(e) is QueryException + assert isinstance(e, testgres__TestgresException) + assert e.source is None + assert e.message == "mmm\nQuery: cba\n321" + assert e.description == "mmm" + assert e.query == "cba\n321" + assert str(e) == "mmm\nQuery: cba\n321" + assert repr(e) == "QueryException(message='mmm', query='cba\\n321')" + return diff --git a/tests/units/exceptions/QueryTimeoutException/__init__.py b/tests/units/exceptions/QueryTimeoutException/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/units/exceptions/QueryTimeoutException/test_set001__constructor.py b/tests/units/exceptions/QueryTimeoutException/test_set001__constructor.py new file mode 100644 index 00000000..362edeb8 --- /dev/null +++ b/tests/units/exceptions/QueryTimeoutException/test_set001__constructor.py @@ -0,0 +1,57 @@ +from src.exceptions import QueryTimeoutException +from src.exceptions import QueryException +from src.exceptions import TestgresException as testgres__TestgresException + + +class TestSet001_Constructor: + def test_001__default(self): + e = QueryTimeoutException() + assert type(e) is QueryTimeoutException + assert isinstance(e, QueryException) + assert isinstance(e, testgres__TestgresException) + assert e.source is None + assert e.message == "" + assert e.description is None + assert e.query is None + assert str(e) == "" + assert repr(e) == "QueryTimeoutException()" + return + + def test_002__message(self): + e = QueryTimeoutException(message="abc\n123") + assert type(e) is QueryTimeoutException + assert isinstance(e, QueryException) + assert isinstance(e, testgres__TestgresException) + assert e.source is None + assert e.message == "abc\n123" + assert e.description == "abc\n123" + assert e.query is None + assert str(e) == "abc\n123" + assert repr(e) == "QueryTimeoutException(message='abc\\n123')" + return + + def test_003__query(self): + e = QueryTimeoutException(query="cba\n321") + assert type(e) is QueryTimeoutException + assert isinstance(e, QueryException) + assert isinstance(e, testgres__TestgresException) + assert e.source is None + assert e.message == "Query: cba\n321" + assert e.description is None + assert e.query == "cba\n321" + assert str(e) == "Query: cba\n321" + assert repr(e) == "QueryTimeoutException(query='cba\\n321')" + return + + def test_004__all(self): + e = QueryTimeoutException(message="mmm", query="cba\n321") + assert type(e) is QueryTimeoutException + assert isinstance(e, QueryException) + assert isinstance(e, testgres__TestgresException) + assert e.source is None + assert e.message == "mmm\nQuery: cba\n321" + assert e.description == "mmm" + assert e.query == "cba\n321" + assert str(e) == "mmm\nQuery: cba\n321" + assert repr(e) == "QueryTimeoutException(message='mmm', query='cba\\n321')" + return diff --git a/tests/units/exceptions/StartNodeException/__init__.py b/tests/units/exceptions/StartNodeException/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/units/exceptions/StartNodeException/test_set001__constructor.py b/tests/units/exceptions/StartNodeException/test_set001__constructor.py new file mode 100644 index 00000000..b66e7c79 --- /dev/null +++ b/tests/units/exceptions/StartNodeException/test_set001__constructor.py @@ -0,0 +1,52 @@ +from src.exceptions import StartNodeException +from src.exceptions import TestgresException as testgres__TestgresException + + +class TestSet001_Constructor: + def test_001__default(self): + e = StartNodeException() + assert type(e) is StartNodeException + assert isinstance(e, testgres__TestgresException) + assert e.source is None + assert e.message == "" + assert e.description is None + assert e.files is None + assert str(e) == "" + assert repr(e) == "StartNodeException()" + return + + def test_002__message(self): + e = StartNodeException(message="abc\n123") + assert type(e) is StartNodeException + assert isinstance(e, testgres__TestgresException) + assert e.source is None + assert e.message == "abc\n123" + assert e.description == "abc\n123" + assert e.files is None + assert str(e) == "abc\n123" + assert repr(e) == "StartNodeException(message='abc\\n123')" + return + + def test_003__files(self): + e = StartNodeException(files=[("f\n1", b'line1\nline2')]) + assert type(e) is StartNodeException + assert isinstance(e, testgres__TestgresException) + assert e.source is None + assert e.message == "f\n1\n----\nb'line1\\nline2'\n" + assert e.description is None + assert e.files == [("f\n1", b'line1\nline2')] + assert str(e) == "f\n1\n----\nb'line1\\nline2'\n" + assert repr(e) == "StartNodeException(files=[('f\\n1', b'line1\\nline2')])" + return + + def test_004__all(self): + e = StartNodeException(message="mmm", files=[("f\n1", b'line1\nline2')]) + assert type(e) is StartNodeException + assert isinstance(e, testgres__TestgresException) + assert e.source is None + assert e.message == "mmm\nf\n1\n----\nb'line1\\nline2'\n" + assert e.description == "mmm" + assert e.files == [("f\n1", b'line1\nline2')] + assert str(e) == "mmm\nf\n1\n----\nb'line1\\nline2'\n" + assert repr(e) == "StartNodeException(message='mmm', files=[('f\\n1', b'line1\\nline2')])" + return diff --git a/tests/units/exceptions/TimeoutException/__init__.py b/tests/units/exceptions/TimeoutException/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/units/exceptions/TimeoutException/test_set001.py b/tests/units/exceptions/TimeoutException/test_set001.py new file mode 100644 index 00000000..a4b9ad0e --- /dev/null +++ b/tests/units/exceptions/TimeoutException/test_set001.py @@ -0,0 +1,11 @@ +from src.exceptions import QueryTimeoutException +from src.exceptions import TimeoutException +from src.exceptions import QueryException + + +class TestSet001: + def test_001__default(self): + # It is an alias + assert TimeoutException == QueryTimeoutException + assert issubclass(TimeoutException, QueryException) + return diff --git a/tests/units/exceptions/__init__.py b/tests/units/exceptions/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/units/impl/__init__.py b/tests/units/impl/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/units/impl/platforms/__init__.py b/tests/units/impl/platforms/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/units/impl/platforms/internal_platform_utils/InternalPlatformUtils/__init__.py b/tests/units/impl/platforms/internal_platform_utils/InternalPlatformUtils/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/units/impl/platforms/internal_platform_utils/InternalPlatformUtils/test_set010__FindPostmaster.py b/tests/units/impl/platforms/internal_platform_utils/InternalPlatformUtils/test_set010__FindPostmaster.py new file mode 100755 index 00000000..4696cdff --- /dev/null +++ b/tests/units/impl/platforms/internal_platform_utils/InternalPlatformUtils/test_set010__FindPostmaster.py @@ -0,0 +1,104 @@ +from __future__ import annotations + +from tests.helpers.global_data import PostgresNodeService +from tests.helpers.global_data import PostgresNodeServices +from tests.helpers.global_data import OsOperations +from tests.helpers.global_data import PortManager +from tests.helpers.pg_node_utils import PostgresNodeUtils as PostgresNodeTestUtils + +from src import PostgresNode +from src import NodeStatus +from src.impl.platforms.internal_platform_utils_factory import create_internal_platform_utils +from src.impl.platforms.internal_platform_utils_factory import InternalPlatformUtils + +import pytest +import typing + + +class TestSet010__FindPostmaster: + @pytest.fixture( + params=PostgresNodeServices.sm_locals_and_remotes, + ids=[descr.sign for descr in PostgresNodeServices.sm_locals_and_remotes] + ) + def node_svc(self, request: pytest.FixtureRequest) -> PostgresNodeService: + assert isinstance(request, pytest.FixtureRequest) + assert isinstance(request.param, PostgresNodeService) + assert isinstance(request.param.os_ops, OsOperations) + assert isinstance(request.param.port_manager, PortManager) + return request.param + + class tagData001: + wait: typing.Optional[bool] + + def __init__(self, wait: typing.Optional[bool]): + assert wait is None or type(wait) is bool + self.wait = wait + return + + def test_001__ok( + self, + node_svc: PostgresNodeService, + ): + assert isinstance(node_svc, PostgresNodeService) + + with PostgresNodeTestUtils.get_node(node_svc) as node: + assert type(node) is PostgresNode + node.init() + assert not node.is_started + assert node.status() == NodeStatus.Stopped + + node.start() + assert node.is_started + assert node.status() == NodeStatus.Running + + # Internals + assert type(node._manually_started_pm_pid) is int + assert node._manually_started_pm_pid != 0 + assert node._manually_started_pm_pid != node._C_PM_PID__IS_NOT_DETECTED + assert node._manually_started_pm_pid == node.pid + + platform_utils = create_internal_platform_utils(node.os_ops) + assert platform_utils is not None + assert isinstance(platform_utils, InternalPlatformUtils) + + r = platform_utils.FindPostmaster( + node.os_ops, + node.bin_dir, + node.data_dir + ) + + assert r is not None + assert type(r) is InternalPlatformUtils.FindPostmasterResult + assert r.code == InternalPlatformUtils.FindPostmasterResultCode.ok + assert r.pid == node.pid + + return + + def test_002__not_found( + self, + node_svc: PostgresNodeService, + ): + assert isinstance(node_svc, PostgresNodeService) + + with PostgresNodeTestUtils.get_node(node_svc) as node: + assert type(node) is PostgresNode + node.init() + assert not node.is_started + assert node.status() == NodeStatus.Stopped + + platform_utils = create_internal_platform_utils(node.os_ops) + assert platform_utils is not None + assert isinstance(platform_utils, InternalPlatformUtils) + + r = platform_utils.FindPostmaster( + node.os_ops, + node.bin_dir, + node.data_dir + ) + + assert r is not None + assert type(r) is InternalPlatformUtils.FindPostmasterResult + assert r.code == InternalPlatformUtils.FindPostmasterResultCode.not_found + assert r.pid is None + + return diff --git a/tests/units/impl/platforms/internal_platform_utils/__init__.py b/tests/units/impl/platforms/internal_platform_utils/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/units/impl/test_file_line_reader.py b/tests/units/impl/test_file_line_reader.py new file mode 100755 index 00000000..0118aebf --- /dev/null +++ b/tests/units/impl/test_file_line_reader.py @@ -0,0 +1,223 @@ +from __future__ import annotations + +from ...helpers.global_data import OsOpsDescrs +from ...helpers.global_data import OsOpsDescr +from ...helpers.global_data import OsOperations + +from src.impl.file_line_reader import FileLineReader + +import pytest +import typing +import dataclasses +import logging + + +class TestFileLineReader: + sm_os_ops_descrs: typing.List[OsOpsDescr] = [ + OsOpsDescrs.sm_local_os_ops_descr, + OsOpsDescrs.sm_remote_os_ops_descr + ] + + @pytest.fixture( + params=[ + pytest.param( + descr, + id=descr.sign, + ) + for descr in sm_os_ops_descrs + ], + ) + def os_ops_descr(self, request: pytest.FixtureRequest) -> OsOpsDescr: + assert isinstance(request, pytest.FixtureRequest) + assert isinstance(request.param, OsOpsDescr) + return request.param + + # -------------------------------------------------------------------- + @dataclasses.dataclass + class tagStep: + write_data: bytes + read_lines: typing.List[typing.Optional[str]] + + # -------------------------------------------------------------------- + sm_Steps001: typing.List[tagStep] = [ + tagStep( + b"", + [None, None, None] + ), + tagStep( + b"a", + [None, None] + ), + tagStep( + b"b", + [None, None] + ), + tagStep( + b"c\n", + ["abc\n", None] + ), + tagStep( + b"d", + [None, None] + ), + tagStep( + b"efg\n1\n\n3", + ["defg\n", "1\n", "\n", None, None] + ), + tagStep( + b" \n 1\n", + ["3 \n", " 1\n", None, None] + ), + # russian text ma: b'\xd0\xbc\xd0\xb0' + tagStep( + b'\xd0', + [None] + ), + tagStep( + b'\xbc', + [None] + ), + tagStep( + b'\xd0', + [None] + ), + tagStep( + b'\xb0', + [None] + ), + tagStep( + b'\n', # FINISH + ["\u043c\u0430\n", None] + ), + + ] + + # ------------------------------------------------------------------- + def test_001__from_beginnig( + self, + os_ops_descr: OsOpsDescr, + ): + assert type(os_ops_descr) is OsOpsDescr + assert isinstance(os_ops_descr.os_ops, OsOperations) + + __class__.helper__player( + os_ops_descr, + __class__.sm_Steps001, + 0, + ) + return + + # -------------------------------------------------------------------- + sm_Steps002: typing.List[tagStep] = [ + tagStep( + b"abc\ndefg\n", + ["abc\n", "defg\n", None] + ), + ] + + # ------------------------------------------------------------------- + def test_002__from_1( + self, + os_ops_descr: OsOpsDescr, + ): + assert type(os_ops_descr) is OsOpsDescr + assert isinstance(os_ops_descr.os_ops, OsOperations) + + __class__.helper__player( + os_ops_descr, + __class__.sm_Steps002, + 1, + ) + return + + # -------------------------------------------------------------------- + sm_Steps003: typing.List[tagStep] = [ + tagStep( + b"abc\ndefg\n", + ["defg\n", None] + ), + ] + + # ------------------------------------------------------------------- + def test_003__from_second_line( + self, + os_ops_descr: OsOpsDescr, + ): + assert type(os_ops_descr) is OsOpsDescr + assert isinstance(os_ops_descr.os_ops, OsOperations) + + __class__.helper__player( + os_ops_descr, + __class__.sm_Steps003, + 4, + ) + return + + # -------------------------------------------------------------------- + @staticmethod + def helper__player( + os_ops_descr: OsOpsDescr, + steps: typing.List[tagStep], + initial_pos: int, + ): + assert type(os_ops_descr) is OsOpsDescr + assert isinstance(os_ops_descr.os_ops, OsOperations) + assert type(steps) is list + assert type(initial_pos) is int + assert initial_pos >= 0 + + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops_descr.os_ops, OsOperations) + + tmpdir = os_ops.mkdtemp() + filename = os_ops.build_path(tmpdir, "my.log") + assert not os_ops.path_exists(filename) + + os_ops.touch(filename) + assert os_ops.path_exists(filename) + assert os_ops.get_file_size(filename) == 0 + + file_line_reader: typing.Optional[FileLineReader] = None + + # ----------------------- + nStep = 0 + + for step in steps: + nStep += 1 + + logging.info("-------------------- step: {}".format(nStep)) + + logging.info("write: {}".format(step.write_data)) + os_ops.write(filename, step.write_data, binary=True) + + if file_line_reader is None: + file_line_reader = FileLineReader( + os_ops, + filename, + file_encoding="utf-8", + file_pos=initial_pos, + ) + + nRead = 0 + for expected_line in step.read_lines: + nRead += 1 + logging.info("read [{}]. expected line is {!r}".format( + nRead, + expected_line, + )) + actual_line = file_line_reader.read_line() + + if actual_line != expected_line: + err_msg = "Read bad line {!r}. Expected {!r}".format( + actual_line, + expected_line, + ) + raise RuntimeError(err_msg) + continue + continue + + assert file_line_reader.read_line() is None + + os_ops.rmdirs(tmpdir) + assert not os_ops.path_exists(tmpdir) + return diff --git a/tests/units/node/PostgresNode/__init__.py b/tests/units/node/PostgresNode/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/units/node/PostgresNode/test_setM001__start.py b/tests/units/node/PostgresNode/test_setM001__start.py new file mode 100644 index 00000000..c0acc721 --- /dev/null +++ b/tests/units/node/PostgresNode/test_setM001__start.py @@ -0,0 +1,217 @@ +from __future__ import annotations + +from tests.helpers.global_data import PostgresNodeService +from tests.helpers.global_data import PostgresNodeServices +from tests.helpers.global_data import OsOperations +from tests.helpers.global_data import PortManager +from tests.helpers.utils import Utils as HelperUtils +from tests.helpers.pg_node_utils import PostgresNodeUtils as PostgresNodeTestUtils + +from src import PostgresNode +from src import NodeStatus +from src import NodeConnection + +from src.node import PostgresNodeLogReader + +import pytest +import typing +import logging + + +class TestSet001__start: + @pytest.fixture( + params=PostgresNodeServices.sm_locals_and_remotes, + ids=[descr.sign for descr in PostgresNodeServices.sm_locals_and_remotes] + ) + def node_svc(self, request: pytest.FixtureRequest) -> PostgresNodeService: + assert isinstance(request, pytest.FixtureRequest) + assert isinstance(request.param, PostgresNodeService) + assert isinstance(request.param.os_ops, OsOperations) + assert isinstance(request.param.port_manager, PortManager) + return request.param + + class tagData001: + wait: typing.Optional[bool] + + def __init__(self, wait: typing.Optional[bool]): + assert wait is None or type(wait) is bool + self.wait = wait + return + + sm_Data001: typing.List[tagData001] = [ + tagData001(None), + tagData001(True) + ] + + @pytest.fixture( + params=sm_Data001, + ids=["wait={}".format(x.wait) for x in sm_Data001] + ) + def data001(self, request: pytest.FixtureRequest) -> tagData001: + assert isinstance(request, pytest.FixtureRequest) + assert type(request.param).__name__ == "tagData001" + return request.param + + def test_001__wait_true( + self, + node_svc: PostgresNodeService, + data001: tagData001 + ): + assert isinstance(node_svc, PostgresNodeService) + assert type(data001) is __class__.tagData001 + assert data001.wait is None or type(data001.wait) is bool + + with PostgresNodeTestUtils.get_node(node_svc) as node: + assert type(node) is PostgresNode + node.init() + assert not node.is_started + assert node.status() == NodeStatus.Stopped + + kwargs = {} + + if data001.wait is not None: + assert data001.wait == True # noqa: E712 + kwargs["wait"] = data001.wait + + node.start(**kwargs) + assert node.is_started + assert node.status() == NodeStatus.Running + + # Internals + assert type(node._manually_started_pm_pid) is int + assert node._manually_started_pm_pid != 0 + assert node._manually_started_pm_pid != node._C_PM_PID__IS_NOT_DETECTED + assert node._manually_started_pm_pid == node.pid + return + + def test_002__wait_false(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + + C_MAX_ATTEMPTS = 3 + + attempt = 0 + + while True: + assert type(attempt) is int + assert attempt >= 0 + assert attempt <= C_MAX_ATTEMPTS + + if attempt == C_MAX_ATTEMPTS: + raise RuntimeError("Node is not started") + + attempt += 1 + + logging.info("------------- attempt #{}".format(attempt)) + + if attempt > 1: + HelperUtils.PrintAndSleep(5) + + with PostgresNodeTestUtils.get_node(node_svc) as node: + assert type(node) is PostgresNode + node.init() + assert not node.is_started + assert node.status() == NodeStatus.Stopped + + node_log_reader = PostgresNodeLogReader(node, from_beginnig=False) + node.start(wait=False) + assert node.is_started + assert node.status() in [NodeStatus.Stopped, NodeStatus.Running] + + # Internals + assert type(node._manually_started_pm_pid) is int + assert node._manually_started_pm_pid == node._C_PM_PID__IS_NOT_DETECTED + + logging.info("Wait for running state ...") + + try: + PostgresNodeTestUtils.wait_for_running_state( + node=node, + node_log_reader=node_log_reader, + timeout=60, + ) + except PostgresNodeTestUtils.PortConflictNodeException as e: + logging.warning("Exception {}: {}".format( + type(e).__name__, + e, + )) + continue + + logging.info("Node is running.") + assert node.status() == NodeStatus.Running + return + + def test_003__exec_env( + self, + node_svc: PostgresNodeService, + ): + assert isinstance(node_svc, PostgresNodeService) + + with PostgresNodeTestUtils.get_node(node_svc) as node: + assert type(node) is PostgresNode + node.init() + assert not node.is_started + assert node.status() == NodeStatus.Stopped + + C_ENV_NAME = "MYTESTVAR" + C_ENV_VALUE = "abcdefg" + + envs = { + C_ENV_NAME: C_ENV_VALUE + } + + node.start(exec_env=envs) + assert node.is_started + assert node.status() == NodeStatus.Running + + with node.connect(dbname="postgres") as cn: + assert type(cn) is NodeConnection + + cn.execute("CREATE TEMP TABLE cmd_out(content text);") + cn.commit() + cn.execute("COPY cmd_out FROM PROGRAM 'bash -c \'\'echo ${}\'\'';".format( + C_ENV_NAME, + )) + cn.commit() + recs = cn.execute("select content from cmd_out;") + assert type(recs) is list + assert len(recs) == 1 + assert type(recs[0]) is tuple + rec = recs[0] + assert len(rec) == 1 + assert rec[0] == C_ENV_VALUE + logging.info("Env has value [{}]. It is OK!".find(rec[0])) + return + + def test_004__params_is_None( + self, + node_svc: PostgresNodeService, + ): + assert isinstance(node_svc, PostgresNodeService) + + with PostgresNodeTestUtils.get_node(node_svc) as node: + assert type(node) is PostgresNode + node.init() + assert not node.is_started + assert node.status() == NodeStatus.Stopped + + node.start(params=None) + assert node.is_started + assert node.status() == NodeStatus.Running + return + + def test_005__params_is_empty( + self, + node_svc: PostgresNodeService, + ): + assert isinstance(node_svc, PostgresNodeService) + + with PostgresNodeTestUtils.get_node(node_svc) as node: + assert type(node) is PostgresNode + node.init() + assert not node.is_started + assert node.status() == NodeStatus.Stopped + + node.start(params=[]) + assert node.is_started + assert node.status() == NodeStatus.Running + return diff --git a/tests/units/node/PostgresNode/test_setM002__start2.py b/tests/units/node/PostgresNode/test_setM002__start2.py new file mode 100644 index 00000000..286988f8 --- /dev/null +++ b/tests/units/node/PostgresNode/test_setM002__start2.py @@ -0,0 +1,211 @@ +from __future__ import annotations + +from tests.helpers.global_data import PostgresNodeService +from tests.helpers.global_data import PostgresNodeServices +from tests.helpers.global_data import OsOperations +from tests.helpers.global_data import PortManager +from tests.helpers.utils import Utils as HelperUtils +from tests.helpers.pg_node_utils import PostgresNodeUtils as PostgresNodeTestUtils + +from src import PostgresNode +from src import NodeStatus +from src import NodeConnection + +from src.node import PostgresNodeLogReader + +import pytest +import typing +import logging + + +class TestSet002__start2: + @pytest.fixture( + params=PostgresNodeServices.sm_locals_and_remotes, + ids=[descr.sign for descr in PostgresNodeServices.sm_locals_and_remotes] + ) + def node_svc(self, request: pytest.FixtureRequest) -> PostgresNodeService: + assert isinstance(request, pytest.FixtureRequest) + assert isinstance(request.param, PostgresNodeService) + assert isinstance(request.param.os_ops, OsOperations) + assert isinstance(request.param.port_manager, PortManager) + return request.param + + class tagData001: + wait: typing.Optional[bool] + + def __init__(self, wait: typing.Optional[bool]): + assert wait is None or type(wait) is bool + self.wait = wait + return + + sm_Data001: typing.List[tagData001] = [ + tagData001(None), + tagData001(True) + ] + + @pytest.fixture( + params=sm_Data001, + ids=["wait={}".format(x.wait) for x in sm_Data001] + ) + def data001(self, request: pytest.FixtureRequest) -> tagData001: + assert isinstance(request, pytest.FixtureRequest) + assert type(request.param).__name__ == "tagData001" + return request.param + + def test_001__wait_true( + self, + node_svc: PostgresNodeService, + data001: tagData001 + ): + assert isinstance(node_svc, PostgresNodeService) + assert type(data001) is __class__.tagData001 + assert data001.wait is None or type(data001.wait) is bool + + with PostgresNodeTestUtils.get_node(node_svc) as node: + assert type(node) is PostgresNode + node.init() + assert not node.is_started + assert node.status() == NodeStatus.Stopped + + kwargs = {} + + if data001.wait is not None: + assert data001.wait == True # noqa: E712 + kwargs["wait"] = data001.wait + + node.start2(**kwargs) + assert not node.is_started + assert node.status() == NodeStatus.Running + + # Internals + assert node._manually_started_pm_pid is None + return + + def test_002__wait_false(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + + C_MAX_ATTEMPTS = 3 + + attempt = 0 + + while True: + assert type(attempt) is int + assert attempt >= 0 + assert attempt <= C_MAX_ATTEMPTS + + if attempt == C_MAX_ATTEMPTS: + raise RuntimeError("Node is not started") + + logging.info("------------- attempt #{}".format(attempt)) + + if attempt > 1: + HelperUtils.PrintAndSleep(5) + + with PostgresNodeTestUtils.get_node(node_svc) as node: + assert type(node) is PostgresNode + node.init() + assert not node.is_started + assert node.status() == NodeStatus.Stopped + + node_log_reader = PostgresNodeLogReader(node, from_beginnig=False) + node.start2(wait=False) + assert not node.is_started + assert node.status() in [NodeStatus.Stopped, NodeStatus.Running] + + # Internals + assert node._manually_started_pm_pid is None + + logging.info("Wait for running state ...") + + try: + PostgresNodeTestUtils.wait_for_running_state( + node=node, + node_log_reader=node_log_reader, + timeout=60, + ) + except PostgresNodeTestUtils.PortConflictNodeException as e: + logging.warning("Exception {}: {}".format( + type(e).__name__, + e, + )) + continue + + logging.info("Node is running.") + assert node.status() == NodeStatus.Running + return + + def test_003__exec_env( + self, + node_svc: PostgresNodeService, + ): + assert isinstance(node_svc, PostgresNodeService) + + with PostgresNodeTestUtils.get_node(node_svc) as node: + assert type(node) is PostgresNode + node.init() + assert not node.is_started + assert node.status() == NodeStatus.Stopped + + C_ENV_NAME = "MYTESTVAR" + C_ENV_VALUE = "abcdefg" + + envs = { + C_ENV_NAME: C_ENV_VALUE + } + + node.start2(exec_env=envs) + assert not node.is_started + assert node.status() == NodeStatus.Running + + with node.connect(dbname="postgres") as cn: + assert type(cn) is NodeConnection + + cn.execute("CREATE TEMP TABLE cmd_out(content text);") + cn.commit() + cn.execute("COPY cmd_out FROM PROGRAM 'bash -c \'\'echo ${}\'\'';".format( + C_ENV_NAME, + )) + cn.commit() + recs = cn.execute("select content from cmd_out;") + assert type(recs) is list + assert len(recs) == 1 + assert type(recs[0]) is tuple + rec = recs[0] + assert len(rec) == 1 + assert rec[0] == C_ENV_VALUE + logging.info("Env has value [{}]. It is OK!".find(rec[0])) + return + + def test_004__params_is_None( + self, + node_svc: PostgresNodeService, + ): + assert isinstance(node_svc, PostgresNodeService) + + with PostgresNodeTestUtils.get_node(node_svc) as node: + assert type(node) is PostgresNode + node.init() + assert not node.is_started + assert node.status() == NodeStatus.Stopped + + node.start(params=None) + assert node.is_started + assert node.status() == NodeStatus.Running + return + + def test_005__params_is_empty( + self, + node_svc: PostgresNodeService, + ): + assert isinstance(node_svc, PostgresNodeService) + + with PostgresNodeTestUtils.get_node(node_svc) as node: + assert type(node) is PostgresNode + node.init() + assert not node.is_started + assert node.status() == NodeStatus.Stopped + + node.start(params=[]) + assert node.is_started + assert node.status() == NodeStatus.Running + return diff --git a/tests/units/node/PostgresNodeLogReader/__init__.py b/tests/units/node/PostgresNodeLogReader/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/units/node/PostgresNodeLogReader/test_setM001__helper_create_log_info.py b/tests/units/node/PostgresNodeLogReader/test_setM001__helper_create_log_info.py new file mode 100644 index 00000000..6c846037 --- /dev/null +++ b/tests/units/node/PostgresNodeLogReader/test_setM001__helper_create_log_info.py @@ -0,0 +1,112 @@ +from __future__ import annotations + +from ....helpers.global_data import OsOpsDescrs +from ....helpers.global_data import OsOpsDescr +from ....helpers.global_data import OsOperations + +from src.node import PostgresNodeLogReader + +import pytest +import typing +import random + + +class TestSetM001__helper_create_log_info: + sm_os_ops_descrs: typing.List[OsOpsDescr] = [ + OsOpsDescrs.sm_local_os_ops_descr, + OsOpsDescrs.sm_remote_os_ops_descr + ] + + @pytest.fixture( + params=[ + pytest.param( + descr, + id=descr.sign, + ) + for descr in sm_os_ops_descrs + ], + ) + def os_ops_descr(self, request: pytest.FixtureRequest) -> OsOpsDescr: + assert isinstance(request, pytest.FixtureRequest) + assert isinstance(request.param, OsOpsDescr) + return request.param + + def test_001__common(self, os_ops_descr: OsOpsDescr): + assert type(os_ops_descr) is OsOpsDescr + os_ops = os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + filename = os_ops.mkstemp("data_for_create_log_info") + + # Scenario 0: The log file ends with a normal line feed + C_DATA0 = b"" + os_ops.write(filename, C_DATA0, binary=True, truncate=True) + + log_info1 = PostgresNodeLogReader._create_log_info(os_ops, filename, find_line_start=True) + assert log_info1.tail == b"" + assert log_info1.position == len(C_DATA0) + + # Scenario 1: The log file ends with a normal line feed + C_DATA1 = b"Line 1\nLine 2\n" + os_ops.write(filename, C_DATA1, binary=True, truncate=True) + + log_info1 = PostgresNodeLogReader._create_log_info(os_ops, filename, find_line_start=True) + # Since the file ends with \n, the tail must be empty! + assert log_info1.tail == b"" + assert log_info1.position == len(C_DATA1) + + # Scenario 2: The log file contains an unterminated line (our UTF-8 trap) + C_DATA2 = b"Line 1\nLine 2\nIncomplete UTF8 \xd0" + os_ops.write(filename, C_DATA2, binary=True, truncate=True) + + log_info2 = PostgresNodeLogReader._create_log_info(os_ops, filename, find_line_start=True) + # The tail must contain exactly the piece after the last \n + assert log_info2.tail == b"Incomplete UTF8 \xd0" + assert log_info2.position == len(C_DATA2) + + # Scenario 3: The file has no line breaks at all (one long line) + C_DATA3 = b"Just one long line without newlines" + os_ops.write(filename, C_DATA3, binary=True, truncate=True) + + log_info3 = PostgresNodeLogReader._create_log_info(os_ops, filename, find_line_start=True) + # Should take the entire file in tail, and set the position to the file size + assert log_info3.tail == C_DATA3 + assert log_info3.position == len(C_DATA3) + + # 4. Large data (two segments) + allowed_bytes = bytes([b for b in range(256) if b != 10]) + C_DATA4 = bytes(random.choices(allowed_bytes, k=5000)) + os_ops.write(filename, C_DATA4, binary=True, truncate=True) + + log_info = PostgresNodeLogReader._create_log_info(os_ops, filename, find_line_start=True) + assert log_info.tail == C_DATA4 + assert log_info.position == len(C_DATA4) + + # 5. Large data (many segments) + allowed_bytes = bytes([b for b in range(256) if b != 10]) + C_DATA5 = bytes(random.choices(allowed_bytes, k=999983)) + os_ops.write(filename, C_DATA5, binary=True, truncate=True) + + log_info = PostgresNodeLogReader._create_log_info(os_ops, filename, find_line_start=True) + assert log_info.tail == C_DATA5 + assert log_info.position == len(C_DATA5) + + # 6. Large data (first_line + many segments) + allowed_bytes = bytes([b for b in range(256) if b != 10]) + os_ops.write(filename, b'abcd\n', binary=True, truncate=True) + os_ops.write(filename, C_DATA5, binary=True, truncate=False) + + log_info = PostgresNodeLogReader._create_log_info(os_ops, filename, find_line_start=True) + assert log_info.tail == C_DATA5 + assert log_info.position == 5 + len(C_DATA5) + + # Scenario 7: The log file has one line with the normal end + os_ops.write(filename, b"\n", binary=True, truncate=True) + + log_info = PostgresNodeLogReader._create_log_info(os_ops, filename, find_line_start=True) + assert log_info.tail == b"" + assert log_info.position == 1 + + # Cleanup + os_ops.remove_file(filename) + return diff --git a/tests/units/node/__init__.py b/tests/units/node/__init__.py new file mode 100644 index 00000000..e69de29b