From b20ac4ed47641eb65ffab55408b8edcc0d1ad931 Mon Sep 17 00:00:00 2001 From: Jelte Fennema-Nio Date: Wed, 13 Aug 2025 10:58:56 -0700 Subject: [PATCH v1 04/12] Add support for pytest test suites Specify --enable-pytest/-Dpytest=enabled at configure time. This contains no Postgres test logic -- it is just a "vanilla" pytest skeleton. This contains a custom pytest plugin to generate TAP output. This plugin is used by the Meson mtest runner, to show relevant information for failed tests. The pytest-tap plugin would have been preferable, but it's now in maintenance mode, and it has problems with accidentally suppressing important collection failures. Co-authored-by: Jacob Champion --- .github/workflows/pg-ci.yml | 16 ++- .gitignore | 3 + configure | 107 +++++++++++++++- configure.ac | 24 +++- doc/src/sgml/regress.sgml | 51 +++++++- meson.build | 100 +++++++++++++++ meson_options.txt | 8 +- pyproject.toml | 24 ++++ src/Makefile.global.in | 41 ++++++ src/makefiles/meson.build | 2 + src/makefiles/pgxs.mk | 10 ++ src/test/pytest/README | 69 +++++++++++ src/test/pytest/pgtap.py | 241 ++++++++++++++++++++++++++++++++++++ src/tools/ci/pytest-asan | 14 +++ src/tools/testwrap | 6 +- 15 files changed, 707 insertions(+), 9 deletions(-) create mode 100644 pyproject.toml create mode 100644 src/test/pytest/README create mode 100644 src/test/pytest/pgtap.py create mode 100755 src/tools/ci/pytest-asan diff --git a/.github/workflows/pg-ci.yml b/.github/workflows/pg-ci.yml index 9ceef9e9c1d..8d3c25124b5 100644 --- a/.github/workflows/pg-ci.yml +++ b/.github/workflows/pg-ci.yml @@ -91,6 +91,7 @@ env: -Dreadline=enabled -Dssl=openssl -Dtap_tests=enabled + -Dpytest=enabled -Dzlib=enabled -Dzstd=enabled @@ -502,7 +503,7 @@ jobs: run: | ./configure \ --enable-cassert --enable-injection-points --enable-debug \ - --enable-tap-tests --enable-nls \ + --enable-tap-tests --enable-pytest --enable-nls \ --with-segsize-blocks=6 \ --with-libnuma \ --with-liburing \ @@ -604,6 +605,7 @@ jobs: --buildtype=debug \ --pkg-config-path /usr/lib/i386-linux-gnu/pkgconfig/ \ -DPERL=perl5.40-i386-linux-gnu \ + -DPYTEST=pytest-i386 \ -Dlibnuma=disabled \ build @@ -685,6 +687,10 @@ jobs: - *ccache_restore_branch_step - *linux_prepare_workspace_step + # The pytest suites load the asan-instrumented libpq into an + # uninstrumented python via dlopen(), which the asan runtime refuses + # unless it's the very first library in the process. The pytest-asan + # wrapper LD_PRELOADs the runtime to make that so. - name: Configure shell: *su_postgres_shell run: | @@ -693,6 +699,7 @@ jobs: -Duuid=e2fs \ --buildtype=debug \ -Dllvm=enabled \ + -DPYTEST="$(pwd)/src/tools/ci/pytest-asan" \ build - name: Build @@ -747,6 +754,8 @@ jobs: p5.34-io-tty p5.34-ipc-run python312 + py312-packaging + py312-pytest tcl zstd @@ -815,6 +824,7 @@ jobs: sh src/tools/ci/ci_macports_packages.sh $MACOS_PACKAGE_LIST # system python doesn't provide headers sudo /opt/local/bin/port select python3 python312 + sudo /opt/local/bin/port select pytest pytest312 # Make macports install visible to subsequent steps echo /opt/local/sbin >> "$GITHUB_PATH" echo /opt/local/bin >> "$GITHUB_PATH" @@ -907,6 +917,7 @@ jobs: -Dplpython=enabled -Dssl=openssl -Dtap_tests=enabled + -Dpytest=enabled defaults: run: @@ -994,7 +1005,7 @@ jobs: run: | # meson is not preinstalled on windows-2022. Install via pip echo ::group::pip - python -m pip install --upgrade meson + python -m pip install --upgrade meson pytest if (!$?) { throw 'cmdfail' } echo ::endgroup:: @@ -1134,6 +1145,7 @@ jobs: ${MINGW_PACKAGE_PREFIX}-meson \ ${MINGW_PACKAGE_PREFIX}-perl \ ${MINGW_PACKAGE_PREFIX}-pkgconf \ + ${MINGW_PACKAGE_PREFIX}-python-pytest \ ${MINGW_PACKAGE_PREFIX}-readline \ ${MINGW_PACKAGE_PREFIX}-zlib \ ${MINGW_PACKAGE_PREFIX}-zstd diff --git a/.gitignore b/.gitignore index 4e911395fe3..a550ce6194b 100644 --- a/.gitignore +++ b/.gitignore @@ -31,6 +31,7 @@ win32ver.rc *.exe lib*dll.def lib*.pc +__pycache__/ # Local excludes in root directory /GNUmakefile @@ -43,3 +44,5 @@ lib*.pc /Release/ /tmp_install/ /portlock/ +/.venv/ +/uv.lock diff --git a/configure b/configure index d42a7a794ff..5379872b1b8 100755 --- a/configure +++ b/configure @@ -630,6 +630,7 @@ vpath_build PG_SYSROOT PG_VERSION_NUM LDFLAGS_EX_BE +PYTEST PROVE DBTOEPUB FOP @@ -773,6 +774,7 @@ CFLAGS CC enable_injection_points PG_TEST_EXTRA +enable_pytest enable_tap_tests enable_dtrace DTRACEFLAGS @@ -851,6 +853,7 @@ enable_profiling enable_coverage enable_dtrace enable_tap_tests +enable_pytest enable_injection_points with_blocksize with_segsize @@ -1551,7 +1554,10 @@ Optional Features: --enable-profiling build with profiling enabled --enable-coverage build with coverage testing instrumentation --enable-dtrace build with DTrace support - --enable-tap-tests enable TAP tests (requires Perl and IPC::Run) + --enable-tap-tests enable (Perl-based) TAP tests (requires Perl and + IPC::Run) + --enable-pytest enable (Python-based) pytest suites (requires + Python) --enable-injection-points enable injection points (for testing) --enable-depend turn on automatic dependency tracking @@ -3634,7 +3640,7 @@ fi # -# TAP tests +# Test frameworks # @@ -3662,6 +3668,32 @@ fi + +# Check whether --enable-pytest was given. +if test "${enable_pytest+set}" = set; then : + enableval=$enable_pytest; + case $enableval in + yes) + : + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --enable-pytest option" "$LINENO" 5 + ;; + esac + +else + enable_pytest=no + +fi + + + + + + # # Injection points # @@ -19493,6 +19525,77 @@ $as_echo "$modulestderr" >&6; } fi fi +if test "$enable_pytest" = yes; then + if test -z "$PYTEST"; then + for ac_prog in pytest py.test +do + # Extract the first word of "$ac_prog", so it can be a program name with args. +set dummy $ac_prog; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_path_PYTEST+:} false; then : + $as_echo_n "(cached) " >&6 +else + case $PYTEST in + [\\/]* | ?:[\\/]*) + ac_cv_path_PYTEST="$PYTEST" # Let the user override the test with a path. + ;; + *) + as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then + ac_cv_path_PYTEST="$as_dir/$ac_word$ac_exec_ext" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + + ;; +esac +fi +PYTEST=$ac_cv_path_PYTEST +if test -n "$PYTEST"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $PYTEST" >&5 +$as_echo "$PYTEST" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + + test -n "$PYTEST" && break +done + +else + # Report the value of PYTEST in configure's output in all cases. + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for PYTEST" >&5 +$as_echo_n "checking for PYTEST... " >&6; } + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $PYTEST" >&5 +$as_echo "$PYTEST" >&6; } +fi + + if test -z "$PYTEST"; then + # Try python -m pytest as a fallback + { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether python -m pytest works" >&5 +$as_echo_n "checking whether python -m pytest works... " >&6; } + if "$PYTHON" -m pytest --version >&5 2>&1; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + PYTEST="$PYTHON -m pytest" + else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + as_fn_error $? "pytest not found" "$LINENO" 5 + fi + fi +fi + # If compiler will take -Wl,--as-needed (or various platform-specific # spellings thereof) then add that to LDFLAGS. This is much easier than # trying to filter LIBS to the minimum for each executable. diff --git a/configure.ac b/configure.ac index a331749fcb5..0b51fe4626c 100644 --- a/configure.ac +++ b/configure.ac @@ -226,11 +226,16 @@ AC_SUBST(DTRACEFLAGS)]) AC_SUBST(enable_dtrace) # -# TAP tests +# Test frameworks # PGAC_ARG_BOOL(enable, tap-tests, no, - [enable TAP tests (requires Perl and IPC::Run)]) + [enable (Perl-based) TAP tests (requires Perl and IPC::Run)]) AC_SUBST(enable_tap_tests) + +PGAC_ARG_BOOL(enable, pytest, no, + [enable (Python-based) pytest suites (requires Python)]) +AC_SUBST(enable_pytest) + AC_ARG_VAR(PG_TEST_EXTRA, [enable selected extra tests (overridden at runtime by PG_TEST_EXTRA environment variable)]) @@ -2501,6 +2506,21 @@ if test "$enable_tap_tests" = yes; then fi fi +if test "$enable_pytest" = yes; then + PGAC_PATH_PROGS(PYTEST, [pytest py.test]) + if test -z "$PYTEST"; then + # Try python -m pytest as a fallback + AC_MSG_CHECKING([whether python -m pytest works]) + if "$PYTHON" -m pytest --version >&AS_MESSAGE_LOG_FD 2>&1; then + AC_MSG_RESULT([yes]) + PYTEST="$PYTHON -m pytest" + else + AC_MSG_RESULT([no]) + AC_MSG_ERROR([pytest not found]) + fi + fi +fi + # If compiler will take -Wl,--as-needed (or various platform-specific # spellings thereof) then add that to LDFLAGS. This is much easier than # trying to filter LIBS to the minimum for each executable. diff --git a/doc/src/sgml/regress.sgml b/doc/src/sgml/regress.sgml index c74941bfbf2..7d44cfff4d5 100644 --- a/doc/src/sgml/regress.sgml +++ b/doc/src/sgml/regress.sgml @@ -928,7 +928,7 @@ float4:out:.*-.*-cygwin.*=float4-misrounded-input.out - TAP Tests + Perl TAP Tests Various tests, particularly the client program tests @@ -1017,6 +1017,55 @@ PG_TEST_NOCLEAN=1 make -C src/bin/pg_dump check + + Pytest Tests + + + Tests in pyt directories use the Python + pytest framework. + + + + The pytest tests require PostgreSQL to be + configured with the option (or + for Meson builds). You also need + pytest installed. You can either install it + system-wide, or create a virtual environment in the source directory: + +python -m venv .venv +source .venv/bin/activate +pip install . + + Alternatively, if you have uv installed: + +uv sync +source .venv/bin/activate + + Remember to activate the virtual environment before running + configure or meson setup. + + + + With Meson builds, a directory's pytest tests belong to that directory's + test suite, the same way its TAP tests do, so they run together with it: + +meson test --suite recovery + + With autoconf-based builds, a directory's pytest tests run as part of its + check target, and against an existing installation as + part of its installcheck target, e.g.: + +make -C src/test/recovery check + + + + + For more information on writing pytest tests, see the + src/test/pytest/README file. + + + + Test Coverage Examination diff --git a/meson.build b/meson.build index c303598a0a4..bcd4e6dc486 100644 --- a/meson.build +++ b/meson.build @@ -1827,6 +1827,47 @@ endif +############################################################### +# Library: pytest +############################################################### + +pytest_enabled = false +pytest_version = '' +pytest_cmd = ['pytest'] # dummy, overwritten when pytest is found +# The mtest runner passes our pgtap plugin with -p, and that plugin lives in +# the directory below. pyproject.toml puts the same directory on pytest's +# pythonpath, but versions below 8.4 only apply that setting after loading the +# plugins passed with -p, so on those the plugin is only importable if +# PYTHONPATH is set here too. This won't help people manually running pytest +# outside of meson/make, but we expect those to use a recent enough version of +# pytest anyway (and if not they can manually configure PYTHONPATH too). +pytest_env = {'PYTHONPATH': meson.project_source_root() / 'src' / 'test' / 'pytest'} + +pytestopt = get_option('pytest') +if not pytestopt.disabled() + pytest = find_program(get_option('PYTEST'), native: true, required: false) + + if pytest.found() + pytest_enabled = true + pytest_version = run_command(pytest, '--version', check: false).stdout().strip().split(' ')[-1] + pytest_cmd = [pytest.full_path()] + else + # Try python -m pytest as a fallback + pytest_check = run_command(python, '-m', 'pytest', '--version', check: false) + if pytest_check.returncode() == 0 + pytest_enabled = true + pytest_version = pytest_check.stdout().strip().split(' ')[-1] + pytest_cmd = [python.full_path(), '-m', 'pytest'] + endif + endif + + if not pytest_enabled and pytestopt.enabled() + error('pytest not found') + endif +endif + + + ############################################################### # Library: zstd ############################################################### @@ -4132,6 +4173,64 @@ foreach test_dir : tests ) endforeach install_suites += test_group + elif kind == 'pytest' + testwrap_pytest = testwrap_base + if not pytest_enabled + testwrap_pytest += ['--skip', 'pytest not enabled'] + endif + + test_command = pytest_cmd + + test_command += [ + '-c', meson.project_source_root() / 'pyproject.toml', + '--verbose', + '-p', 'pgtap', # enable our test reporter plugin + '-ra', # show skipped and xfailed tests too + ] + + # Add temporary install, the build directory for non-installed binaries and + # also test/ for non-installed test binaries built separately. + env = test_env + env.prepend('PATH', temp_install_bindir, test_dir['bd'], test_dir['bd'] / 'test') + temp_install_datadir = '@0@@1@'.format(test_install_destdir, dir_prefix / dir_data) + env.set('share_contrib_dir', temp_install_datadir / 'contrib') + env.prepend('PYTHONPATH', pytest_env['PYTHONPATH']) + + foreach name, value : t.get('env', {}) + env.set(name, value) + endforeach + + test_group = test_dir['name'] + test_kwargs = { + 'protocol': 'tap', + 'suite': test_group, + 'timeout': test_timeout, + 'depends': test_deps + t.get('deps', []), + 'env': env, + } + t.get('test_kwargs', {}) + + foreach onetest : t['tests'] + # Make test names prettier, remove pyt/ and .py + onetest_p = onetest + if onetest_p.startswith('pyt/') + onetest_p = onetest.split('pyt/')[1] + endif + if onetest_p.endswith('.py') + onetest_p = fs.stem(onetest_p) + endif + + test(test_dir['name'] / onetest_p, + python, + kwargs: test_kwargs, + args: testwrap_pytest + [ + '--testgroup', test_dir['name'], + '--testname', onetest_p, + '--', test_command, + test_dir['sd'] / onetest, + ], + ) + endforeach + install_suites += test_group else error('unknown kind @0@ of test in @1@'.format(kind, test_dir['sd'])) endif @@ -4324,6 +4423,7 @@ summary( 'bison': '@0@ @1@'.format(bison.full_path(), bison_version), 'dtrace': dtrace, 'flex': '@0@ @1@'.format(flex.full_path(), flex_version), + 'pytest': pytest_enabled ? ' '.join(pytest_cmd) + ' ' + pytest_version : not_found_dep, }, section: 'Programs', ) diff --git a/meson_options.txt b/meson_options.txt index 6a793f3e479..cb4825c3575 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -41,7 +41,10 @@ option('cassert', type: 'boolean', value: false, description: 'Enable assertion checks (for debugging)') option('tap_tests', type: 'feature', value: 'auto', - description: 'Enable TAP tests') + description: 'Enable (Perl-based) TAP tests') + +option('pytest', type: 'feature', value: 'auto', + description: 'Enable (Python-based) pytest suites') option('injection_points', type: 'boolean', value: false, description: 'Enable injection points') @@ -195,6 +198,9 @@ option('PERL', type: 'string', value: 'perl', option('PROVE', type: 'string', value: 'prove', description: 'Path to prove binary') +option('PYTEST', type: 'array', value: ['pytest', 'py.test'], + description: 'Path to pytest binary') + option('PYTHON', type: 'array', value: ['python3', 'python'], description: 'Path to python binary') diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000000..aec4e7e9804 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,24 @@ +[project] +name = "postgresql-hackers-tooling" +version = "0.1.0" +description = "Python infrastructure for PostgreSQL" +requires-python = ">=3.9" +dependencies = [ + # We need at least 7.2 for the new-style hook wrappers + # (@pytest.hookimpl(wrapper=True)). Some OSes worth considering when we + # have a reason to bump this: + # - Debian Bookworm ships pytest 7.2.1 + # - Ubuntu 24.04 ships pytest 7.4.4 + # - RHEL9 ships 7.4.2 in its CRB repository. + "pytest >= 7.2", + + # Any other dependencies are effectively optional (added below). We import + # these libraries using pytest.importorskip(). So tests will be skipped if + # they are not available. +] + +[tool.pytest.ini_options] +minversion = "7.2" + +# Common test code can be found here. +pythonpath = ["src/test/pytest"] diff --git a/src/Makefile.global.in b/src/Makefile.global.in index ea0b218717a..3e24407ba65 100644 --- a/src/Makefile.global.in +++ b/src/Makefile.global.in @@ -211,6 +211,7 @@ enable_dtrace = @enable_dtrace@ enable_coverage = @enable_coverage@ enable_injection_points = @enable_injection_points@ enable_tap_tests = @enable_tap_tests@ +enable_pytest = @enable_pytest@ python_includespec = @python_includespec@ python_libdir = @python_libdir@ @@ -356,6 +357,7 @@ MSGFMT = @MSGFMT@ MSGFMT_FLAGS = @MSGFMT_FLAGS@ MSGMERGE = @MSGMERGE@ OPENSSL = @OPENSSL@ +PYTEST = @PYTEST@ PYTHON = @PYTHON@ TAR = @TAR@ XGETTEXT = @XGETTEXT@ @@ -520,6 +522,45 @@ prove_installcheck = @echo "TAP tests not enabled. Try configuring with --enable prove_check = $(prove_installcheck) endif +ifeq ($(enable_pytest),yes) + +# We also configure the same PYTHONPATH in the pytest settings in +# pyproject.toml, but pytest versions below 8.4 only actually use that value +# after plugin loading. So we need to configure it here too. This won't help +# people manually running pytest outside of meson/make, but we expect those to +# use a recent enough version of pytest anyway (and if not they can manually +# configure PYTHONPATH too). +define pytest_installcheck +echo "# +++ pytest install-check in $(subdir) +++" && \ +rm -rf '$(CURDIR)'/tmp_check && \ +$(MKDIR_P) '$(CURDIR)'/tmp_check && \ +cd $(srcdir) && \ + TESTLOGDIR='$(CURDIR)/tmp_check/log' \ + TESTDATADIR='$(CURDIR)/tmp_check' \ + PYTHONPATH='$(abs_top_srcdir)/src/test/pytest:$$PYTHONPATH' \ + PATH="$(bindir):$$PATH" \ + top_builddir='$(CURDIR)/$(top_builddir)' \ + $(PYTEST) -c '$(abs_top_srcdir)/pyproject.toml' --verbose -ra ./pyt/ +endef + +define pytest_check +echo "# +++ pytest check in $(subdir) +++" && \ +rm -rf '$(CURDIR)'/tmp_check && \ +$(MKDIR_P) '$(CURDIR)'/tmp_check && \ +cd $(srcdir) && \ + TESTLOGDIR='$(CURDIR)/tmp_check/log' \ + TESTDATADIR='$(CURDIR)/tmp_check' \ + PYTHONPATH='$(abs_top_srcdir)/src/test/pytest:$$PYTHONPATH' \ + $(with_temp_install) \ + top_builddir='$(CURDIR)/$(top_builddir)' \ + $(PYTEST) -c '$(abs_top_srcdir)/pyproject.toml' --verbose -ra ./pyt/ +endef + +else +pytest_installcheck = @echo "pytest is not enabled. Try configuring with --enable-pytest" +pytest_check = $(pytest_installcheck) +endif + # Installation. install_bin = @install_bin@ diff --git a/src/makefiles/meson.build b/src/makefiles/meson.build index 2401025d1cd..663f49de381 100644 --- a/src/makefiles/meson.build +++ b/src/makefiles/meson.build @@ -56,6 +56,8 @@ pgxs_kv = { 'enable_nls': libintl.found() ? 'yes' : 'no', 'enable_injection_points': get_option('injection_points') ? 'yes' : 'no', 'enable_tap_tests': tap_tests_enabled ? 'yes' : 'no', + 'enable_pytest': pytest_enabled ? 'yes' : 'no', + 'PYTEST': pytest_enabled ? ' '.join(pytest_cmd) : '', 'enable_debug': get_option('debug') ? 'yes' : 'no', 'enable_coverage': 'no', 'enable_dtrace': dtrace.found() ? 'yes' : 'no', diff --git a/src/makefiles/pgxs.mk b/src/makefiles/pgxs.mk index 039cee3dfe5..34921149315 100644 --- a/src/makefiles/pgxs.mk +++ b/src/makefiles/pgxs.mk @@ -47,6 +47,7 @@ # REGRESS -- list of regression test cases (without suffix) # REGRESS_OPTS -- additional switches to pass to pg_regress # TAP_TESTS -- switch to enable TAP tests +# PYTEST_TESTS -- switch to enable pytest tests # ISOLATION -- list of isolation test cases # ISOLATION_OPTS -- additional switches to pass to pg_isolation_regress # NO_INSTALL -- don't define an install target, useful for test modules @@ -380,6 +381,9 @@ endif ifdef TAP_TESTS rm -rf tmp_check/ endif +ifdef PYTEST_TESTS + rm -rf tmp_check/ +endif ifdef ISOLATION rm -rf output_iso/ tmp_check_iso/ endif @@ -438,6 +442,9 @@ endif ifdef TAP_TESTS $(prove_installcheck) endif +ifdef PYTEST_TESTS + $(pytest_installcheck) +endif endif # NO_INSTALLCHECK # Runs independently of any installation @@ -456,6 +463,9 @@ endif ifdef TAP_TESTS $(prove_check) endif +ifdef PYTEST_TESTS + $(pytest_check) +endif endif # PGXS ifndef NO_TEMP_INSTALL diff --git a/src/test/pytest/README b/src/test/pytest/README new file mode 100644 index 00000000000..c15e0556dd2 --- /dev/null +++ b/src/test/pytest/README @@ -0,0 +1,69 @@ +src/test/pytest/README + +Pytest-based tests +================== + +This directory contains the infrastructure for Python-based tests using +pytest. + +The tests themselves live in pyt/ directories next to the code they cover, the +same way the Perl tests live in t/ directories, in files named test_.py. +Every function in such a file whose name starts with test_ is run as a test. A +new file must also be listed in the directory's meson.build, in the 'pytest' +entry of its tests dict. + +If you're unfamiliar with pytest it's worth reading two pages of its +documentation first: getting started[1] and fixtures[2]. + + +Running the tests +================= + +NOTE: You must have given the --enable-pytest argument to configure (or +-Dpytest=enabled for Meson builds). You also need to have pytest installed. + +If you don't have pytest installed system-wide, you can create a virtual +environment: + + python3 -m venv .venv + source .venv/bin/activate # On Windows: .venv\Scripts\activate + pip install . # Installs pytest and other dependencies + +Or using uv[3]: + + uv sync + source .venv/bin/activate # On Windows: .venv\Scripts\activate + +Remember to activate the virtual environment before running configure/meson +setup. + +With Meson builds, you can run: + meson test --suite pytest + +With autoconf based builds, a directory's pytest tests run as part of its +check target, so for all of them use check-world, and for one directory e.g.: + make -C src/test/recovery check + +They run against an already installed build with installcheck instead, which +skips building a temporary installation and finds everything through the +pg_config on PATH: + make -C src/test/recovery installcheck + +You can run specific test files and/or use pytest's -k option to select tests: + pytest src/test/recovery/pyt/test_049_wait_for_lsn.py + pytest -k "wait_for_lsn" + + +Directory structure +=================== + +pgtap.py + A pytest plugin to output results in TAP format + + +References +========== + +[1] https://docs.pytest.org/en/stable/getting-started.html +[2] https://docs.pytest.org/en/stable/explanation/fixtures.html +[3] https://docs.astral.sh/uv/ diff --git a/src/test/pytest/pgtap.py b/src/test/pytest/pgtap.py new file mode 100644 index 00000000000..7dc68c41783 --- /dev/null +++ b/src/test/pytest/pgtap.py @@ -0,0 +1,241 @@ +# Copyright (c) 2025, PostgreSQL Global Development Group + +""" +A pytest plugin that reports results as TAP, the format meson's test runner +understands. It is enabled with -p pgtap, which meson does for every pytest +test it runs. + +The plugin takes over the standard streams at startup: pytest's own output is +written to a log file under TESTLOGDIR, and the original stdout carries the +TAP stream, with failure details going to stderr where meson shows them. That +also means a test that prints does not corrupt the protocol. +""" + +from __future__ import annotations + +import os +import sys +from typing import TYPE_CHECKING, Any + +import pytest + +if TYPE_CHECKING: + from _pytest._code.code import ExceptionRepr + +# +# Helpers +# + + +class TAP: + """ + A basic API for reporting via the TAP 12 protocol[1]. + + Reporting a newer version like "TAP version 14"[2] here is not even + possible, despite every meson version accepting it, because meson requires + that line to be the literal first line of output and testwrap prints a + comment like this before it: + + # executing test ... + + We could of course modify testwrap to report TAP version 14, but that would + not buy us much currently. The main feature that could be useful to us is + subtests, but no meson version implements those[3]: before meson 1.0.0 + their indented lines were hard parse errors that failed the test, and since + then[4] they are simply ignored with a warning. + + [1] https://testanything.org/tap-specification.html + [2] https://testanything.org/tap-version-14-specification.html + [3] https://github.com/mesonbuild/meson/issues/15768 + [4] https://github.com/mesonbuild/meson/commit/d0054f2c3c3497e22069d1efb5b1d985d75fe5ca + """ + + def __init__(self) -> None: + self.count = 0 + + def expect(self, num: int) -> None: + self.print(f"1..{num}") + + def print(self, *args: Any) -> None: + print(*args, file=sys.__stdout__) + + def ok(self, name: str) -> None: + self.count += 1 + self.print("ok", self.count, "-", name) + + def skip(self, name: str, reason: str) -> None: + self.count += 1 + self.print("ok", self.count, "-", name, "# skip", reason) + + def fail(self, name: str, details: str) -> None: + self.count += 1 + self.print("not ok", self.count, "-", name) + + # mtest has some odd behavior around TAP tests where it won't print + # diagnostics on failure if they're part of the stdout stream, so we + # might as well just dump the details directly to stderr instead. + print(details, file=sys.__stderr__) + + +tap = TAP() + + +class TestNotes: + """ + Annotations for a single test. The existing pytest hooks keep interesting + information somewhat separated across the different stages + (setup/test/teardown), so this class is used to correlate them. + """ + + skipped: bool = False + skip_reason: str | None = None + + failed: bool = False + details: str | None = None + + +# Register a custom key in the stash dictionary for keeping our TestNotes. +notes_key = pytest.StashKey[TestNotes]() + + +# +# Hook Implementations +# + + +@pytest.hookimpl(tryfirst=True) +def pytest_configure(config: pytest.Config) -> None: + """ + Hijacks the standard streams as soon as possible during pytest startup. The + pytest-formatted output gets logged to file instead, and we'll use the + original sys.__stdout__/__stderr__ streams for the TAP protocol. + """ + logdir = os.getenv("TESTLOGDIR") + if not logdir: + raise RuntimeError("pgtap requires the TESTLOGDIR envvar to be set") + + os.makedirs(logdir) + logpath = os.path.join(logdir, "pytest.log") + # Deliberately not closed: this file replaces the standard streams for the + # remaining lifetime of the process. + sys.stdout = sys.stderr = open(logpath, "a", buffering=1) # noqa: SIM115 + + +@pytest.hookimpl(trylast=True) +def pytest_sessionfinish( + session: pytest.Session, exitstatus: int | pytest.ExitCode +) -> None: + """ + Suppresses nonzero exit codes due to failed tests. (In that case, we want + Meson to report a failure count, not a generic ERROR.) + """ + if exitstatus == pytest.ExitCode.TESTS_FAILED: + session.exitstatus = pytest.ExitCode.OK + + +@pytest.hookimpl +def pytest_collectreport(report: pytest.CollectReport) -> None: + # Include collection failures directly in Meson error output. + if report.failed: + print(report.longreprtext, file=sys.__stderr__) + + +@pytest.hookimpl +def pytest_internalerror( + excrepr: ExceptionRepr, excinfo: pytest.ExceptionInfo[BaseException] +) -> None: + # Include internal errors directly in Meson error output. + print(excrepr, file=sys.__stderr__) + + +# +# Hook Wrappers +# +# In pytest parlance, a "wrapper" for a hook can inspect and optionally modify +# existing hooks' behavior, but it does not replace the hook chain. This is done +# through a generator-style API which chains the hooks together (see the use of +# `yield`). +# + + +@pytest.hookimpl(wrapper=True) +def pytest_collection(session: pytest.Session): + """Reports the number of gathered tests after collection is finished.""" + result = yield + tap.expect(session.testscollected) + return result + + +@pytest.hookimpl(wrapper=True) +def pytest_runtest_makereport(item: pytest.Item, call: pytest.CallInfo[None]): + """ + Annotates a test item with our TestNotes and grabs relevant information for + reporting. + + This is called multiple times per test, so it's not correct to print the TAP + result here. (A test and its teardown stage can both fail, and we want to + see the details for both.) We instead combine all the information for use by + our pytest_runtest_protocol wrapper later on. + """ + report = yield + + if notes_key not in item.stash: + item.stash[notes_key] = TestNotes() + notes = item.stash[notes_key] + + if report.passed: + pass # no annotation needed + + elif report.skipped: + notes.skipped = True + _, _, notes.skip_reason = report.longrepr + + elif report.failed: + notes.failed = True + + # The first failing report (a test and its teardown can both fail) + # writes the header; later ones append to what is already there. + details = notes.details or "{:_^72}\n\n".format(f" {report.head_line} ") + + if report.when in ("setup", "teardown"): + details += "\n{:_^72}\n\n".format( + f" Error during {report.when} of {report.head_line} " + ) + + details += report.longreprtext + "\n" + + # Include captured stdout/stderr/log in failure output + for section_name, section_content in report.sections: + if section_content.strip(): + details += "\n{:-^72}\n".format(f" {section_name} ") + details += section_content + "\n" + + notes.details = details + + else: + raise RuntimeError("pytest_runtest_makereport received unknown test status") + + return report + + +@pytest.hookimpl(wrapper=True) +def pytest_runtest_protocol(item: pytest.Item, nextitem: pytest.Item | None): + """ + Reports the TAP result for this test item using our gathered TestNotes. + """ + result = yield + + assert notes_key in item.stash, "pgtap didn't annotate a test item?" + notes = item.stash[notes_key] + + if notes.failed: + # notes.failed implies the makereport hook populated notes.details. + assert notes.details is not None + tap.fail(item.nodeid, notes.details) + elif notes.skipped: + assert notes.skip_reason is not None + tap.skip(item.nodeid, notes.skip_reason) + else: + tap.ok(item.nodeid) + + return result diff --git a/src/tools/ci/pytest-asan b/src/tools/ci/pytest-asan new file mode 100755 index 00000000000..0034694b698 --- /dev/null +++ b/src/tools/ci/pytest-asan @@ -0,0 +1,14 @@ +#!/bin/sh +# +# Run pytest with the AddressSanitizer runtime preloaded. +# +# Our asan CI job builds libpq (and the other libraries the tests load) with +# -fsanitize=address, but pytest itself runs in a plain, non-instrumented +# Python interpreter that only dlopen()s those libraries. asan has to be the +# very first thing mapped into such a process, which we arrange with +# LD_PRELOAD. meson's PYTEST option is only a program path, not a full command, +# so the wrapping can't live in the option itself and needs a script like this. +# +# We ask the compiler for the runtime's path rather than hard-coding it, so this +# keeps working across compiler versions and distros. +exec env LD_PRELOAD="$(${CC:-gcc} -print-file-name=libasan.so)" pytest "$@" diff --git a/src/tools/testwrap b/src/tools/testwrap index e91296ecd15..346f86b8ea3 100755 --- a/src/tools/testwrap +++ b/src/tools/testwrap @@ -42,7 +42,11 @@ open(os.path.join(testdir, 'test.start'), 'x') env_dict = {**os.environ, 'TESTDATADIR': os.path.join(testdir, 'data'), - 'TESTLOGDIR': os.path.join(testdir, 'log')} + 'TESTLOGDIR': os.path.join(testdir, 'log'), + # Prevent emitting terminal capability sequences that pollute the + # TAP output stream (i.e.\033[?1034h). This happens on OpenBSD with + # pytest for unknown reasons. + 'TERM': ''} # The configuration time value of PG_TEST_EXTRA is supplied via argument -- 2.54.0