From febbb867830000a8cfcf350e4197077c9ff2ba7c Mon Sep 17 00:00:00 2001 From: Jacob Champion Date: Wed, 13 Aug 2025 10:58:56 -0700 Subject: [PATCH v11 1/5] Add support for pytest test suites Specify --enable-pytest/-Dpytest=enabled at configure time. This contains no Postgres test logic -- it is just a "vanilla" pytest skeleton. This contains a custom pytest plugin to generate TAP output. This plugin is used by the Meson mtest runner, to show relevant information for failed tests. The pytest-tap plugin would have been preferable, but it's now in maintenance mode, and it has problems with accidentally suppressing important collection failures. Co-authored-by: Jelte Fennema-Nio --- .cirrus.tasks.yml | 11 +- .gitignore | 3 + configure | 166 +++++++++++++++++++++++++++++- configure.ac | 24 ++++- meson.build | 100 ++++++++++++++++++ meson_options.txt | 8 +- pyproject.toml | 21 ++++ src/Makefile.global.in | 29 ++++++ src/makefiles/meson.build | 2 + src/test/Makefile | 1 + src/test/meson.build | 1 + src/test/pytest/Makefile | 20 ++++ src/test/pytest/README | 1 + src/test/pytest/meson.build | 15 +++ src/test/pytest/pgtap.py | 197 ++++++++++++++++++++++++++++++++++++ src/tools/testwrap | 6 +- 16 files changed, 597 insertions(+), 8 deletions(-) create mode 100644 pyproject.toml create mode 100644 src/test/pytest/Makefile create mode 100644 src/test/pytest/README create mode 100644 src/test/pytest/meson.build create mode 100644 src/test/pytest/pgtap.py diff --git a/.cirrus.tasks.yml b/.cirrus.tasks.yml index 2a821593ce5..c9db12d53b9 100644 --- a/.cirrus.tasks.yml +++ b/.cirrus.tasks.yml @@ -44,6 +44,7 @@ env: -Dldap=enabled -Dssl=openssl -Dtap_tests=enabled + -Dpytest=enabled -Dplperl=enabled -Dplpython=enabled -Ddocs=enabled @@ -315,6 +316,7 @@ task: -Dlibcurl=enabled -Dnls=enabled -Dpam=enabled + -DPYTEST=pytest-3.12 setup_additional_packages_script: | #pkgin -y install ... @@ -518,14 +520,15 @@ task: set -e ./configure \ --enable-cassert --enable-injection-points --enable-debug \ - --enable-tap-tests --enable-nls \ + --enable-tap-tests --enable-pytest --enable-nls \ --with-segsize-blocks=6 \ --with-libnuma \ --with-liburing \ \ ${LINUX_CONFIGURE_FEATURES} \ \ - CLANG="ccache clang" + CLANG="ccache clang" \ + PYTEST="env LD_PRELOAD=/lib/x86_64-linux-gnu/libasan.so.8 pytest" EOF build_script: su postgres -c "make -s -j${BUILD_JOBS} world-bin" upload_caches: ccache @@ -663,6 +666,8 @@ task: p5.34-io-tty p5.34-ipc-run python312 + py312-packaging + py312-pytest tcl zstd @@ -712,6 +717,7 @@ task: sh src/tools/ci/ci_macports_packages.sh $MACOS_PACKAGE_LIST # system python doesn't provide headers sudo /opt/local/bin/port select python3 python312 + sudo /opt/local/bin/port select pytest pytest312 # Make macports install visible for subsequent steps echo PATH=/opt/local/sbin/:/opt/local/bin/:$PATH >> $CIRRUS_ENV upload_caches: macports @@ -785,6 +791,7 @@ task: -Dldap=enabled -Dssl=openssl -Dtap_tests=enabled + -Dpytest=enabled -Dplperl=enabled -Dplpython=enabled diff --git a/.gitignore b/.gitignore index 4e911395fe3..a550ce6194b 100644 --- a/.gitignore +++ b/.gitignore @@ -31,6 +31,7 @@ win32ver.rc *.exe lib*dll.def lib*.pc +__pycache__/ # Local excludes in root directory /GNUmakefile @@ -43,3 +44,5 @@ lib*.pc /Release/ /tmp_install/ /portlock/ +/.venv/ +/uv.lock diff --git a/configure b/configure index a10a2c85c6a..cdd5feff014 100755 --- a/configure +++ b/configure @@ -630,6 +630,8 @@ vpath_build PG_SYSROOT PG_VERSION_NUM LDFLAGS_EX_BE +UV +PYTEST PROVE DBTOEPUB FOP @@ -773,6 +775,7 @@ CFLAGS CC enable_injection_points PG_TEST_EXTRA +enable_pytest enable_tap_tests enable_dtrace DTRACEFLAGS @@ -851,6 +854,7 @@ enable_profiling enable_coverage enable_dtrace enable_tap_tests +enable_pytest enable_injection_points with_blocksize with_segsize @@ -1551,7 +1555,10 @@ Optional Features: --enable-profiling build with profiling enabled --enable-coverage build with coverage testing instrumentation --enable-dtrace build with DTrace support - --enable-tap-tests enable TAP tests (requires Perl and IPC::Run) + --enable-tap-tests enable (Perl-based) TAP tests (requires Perl and + IPC::Run) + --enable-pytest enable (Python-based) pytest suites (requires + Python) --enable-injection-points enable injection points (for testing) --enable-depend turn on automatic dependency tracking @@ -3633,7 +3640,7 @@ fi # -# TAP tests +# Test frameworks # @@ -3661,6 +3668,32 @@ fi + +# Check whether --enable-pytest was given. +if test "${enable_pytest+set}" = set; then : + enableval=$enable_pytest; + case $enableval in + yes) + : + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --enable-pytest option" "$LINENO" 5 + ;; + esac + +else + enable_pytest=no + +fi + + + + + + # # Injection points # @@ -19275,6 +19308,135 @@ $as_echo "$modulestderr" >&6; } fi fi +if test "$enable_pytest" = yes; then + if test -z "$PYTEST"; then + for ac_prog in pytest py.test +do + # Extract the first word of "$ac_prog", so it can be a program name with args. +set dummy $ac_prog; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_path_PYTEST+:} false; then : + $as_echo_n "(cached) " >&6 +else + case $PYTEST in + [\\/]* | ?:[\\/]*) + ac_cv_path_PYTEST="$PYTEST" # Let the user override the test with a path. + ;; + *) + as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then + ac_cv_path_PYTEST="$as_dir/$ac_word$ac_exec_ext" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + + ;; +esac +fi +PYTEST=$ac_cv_path_PYTEST +if test -n "$PYTEST"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $PYTEST" >&5 +$as_echo "$PYTEST" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + + test -n "$PYTEST" && break +done + +else + # Report the value of PYTEST in configure's output in all cases. + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for PYTEST" >&5 +$as_echo_n "checking for PYTEST... " >&6; } + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $PYTEST" >&5 +$as_echo "$PYTEST" >&6; } +fi + + if test -z "$PYTEST"; then + # If pytest not found, try installing with uv + if test -z "$UV"; then + for ac_prog in uv +do + # Extract the first word of "$ac_prog", so it can be a program name with args. +set dummy $ac_prog; ac_word=$2 +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +$as_echo_n "checking for $ac_word... " >&6; } +if ${ac_cv_path_UV+:} false; then : + $as_echo_n "(cached) " >&6 +else + case $UV in + [\\/]* | ?:[\\/]*) + ac_cv_path_UV="$UV" # Let the user override the test with a path. + ;; + *) + as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + test -z "$as_dir" && as_dir=. + for ac_exec_ext in '' $ac_executable_extensions; do + if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then + ac_cv_path_UV="$as_dir/$ac_word$ac_exec_ext" + $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + + ;; +esac +fi +UV=$ac_cv_path_UV +if test -n "$UV"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $UV" >&5 +$as_echo "$UV" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + + test -n "$UV" && break +done + +else + # Report the value of UV in configure's output in all cases. + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for UV" >&5 +$as_echo_n "checking for UV... " >&6; } + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $UV" >&5 +$as_echo "$UV" >&6; } +fi + + if test -n "$UV"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether uv can install pytest dependencies" >&5 +$as_echo_n "checking whether uv can install pytest dependencies... " >&6; } + if "$UV" pip install "$srcdir" >&5 2>&1; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + PYTEST="$UV run pytest" + else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + as_fn_error $? "pytest not found and uv failed to install dependencies" "$LINENO" 5 + fi + else + as_fn_error $? "pytest not found" "$LINENO" 5 + fi + fi +fi + # If compiler will take -Wl,--as-needed (or various platform-specific # spellings thereof) then add that to LDFLAGS. This is much easier than # trying to filter LIBS to the minimum for each executable. diff --git a/configure.ac b/configure.ac index 814e64a967e..f3768336474 100644 --- a/configure.ac +++ b/configure.ac @@ -225,11 +225,16 @@ AC_SUBST(DTRACEFLAGS)]) AC_SUBST(enable_dtrace) # -# TAP tests +# Test frameworks # PGAC_ARG_BOOL(enable, tap-tests, no, - [enable TAP tests (requires Perl and IPC::Run)]) + [enable (Perl-based) TAP tests (requires Perl and IPC::Run)]) AC_SUBST(enable_tap_tests) + +PGAC_ARG_BOOL(enable, pytest, no, + [enable (Python-based) pytest suites (requires Python)]) +AC_SUBST(enable_pytest) + AC_ARG_VAR(PG_TEST_EXTRA, [enable selected extra tests (overridden at runtime by PG_TEST_EXTRA environment variable)]) @@ -2453,6 +2458,21 @@ if test "$enable_tap_tests" = yes; then fi fi +if test "$enable_pytest" = yes; then + PGAC_PATH_PROGS(PYTEST, [pytest py.test]) + if test -z "$PYTEST"; then + # Try python -m pytest as a fallback + AC_MSG_CHECKING([whether python -m pytest works]) + if "$PYTHON" -m pytest --version >&AS_MESSAGE_LOG_FD 2>&1; then + AC_MSG_RESULT([yes]) + PYTEST="$PYTHON -m pytest" + else + AC_MSG_RESULT([no]) + AC_MSG_ERROR([pytest not found]) + fi + fi +fi + # If compiler will take -Wl,--as-needed (or various platform-specific # spellings thereof) then add that to LDFLAGS. This is much easier than # trying to filter LIBS to the minimum for each executable. diff --git a/meson.build b/meson.build index df907b62da3..47fe9db5e50 100644 --- a/meson.build +++ b/meson.build @@ -1764,6 +1764,47 @@ endif +############################################################### +# Library: pytest +############################################################### + +pytest_enabled = false +pytest_version = '' +pytest_cmd = ['pytest'] # dummy, overwritten when pytest is found +# We also configure the same PYTHONPATH in the pytest settings in +# pyproject.toml, but pytest versions below 8.4 only actually use that +# value after plugin loading. On lower versions pytest will throw an error even +# when just running 'pytest --version'. So we need to configure it here too. +# This won't help people manually running pytest outside of meson/make, but we +# expect those to use a recent enough version of pytest anyway (and if not they +# can manually configure PYTHONPATH too). +pytest_env = {'PYTHONPATH': meson.project_source_root() / 'src' / 'test' / 'pytest'} + +pytestopt = get_option('pytest') +if not pytestopt.disabled() + pytest = find_program(get_option('PYTEST'), native: true, required: false) + + if pytest.found() + pytest_enabled = true + pytest_version = run_command(pytest, '--version', env: pytest_env, check: false).stdout().strip().split(' ')[-1] + pytest_cmd = [pytest.full_path()] + else + # Try python -m pytest as a fallback + pytest_check = run_command(python, '-m', 'pytest', '--version', env: pytest_env, check: false) + if pytest_check.returncode() == 0 + pytest_enabled = true + pytest_version = pytest_check.stdout().strip().split(' ')[-1] + pytest_cmd = [python.full_path(), '-m', 'pytest'] + endif + endif + + if not pytest_enabled and pytestopt.enabled() + error('pytest not found') + endif +endif + + + ############################################################### # Library: zstd ############################################################### @@ -3856,6 +3897,64 @@ foreach test_dir : tests ) endforeach install_suites += test_group + elif kind == 'pytest' + testwrap_pytest = testwrap_base + if not pytest_enabled + testwrap_pytest += ['--skip', 'pytest not enabled'] + endif + + test_command = pytest_cmd + + test_command += [ + '-c', meson.project_source_root() / 'pyproject.toml', + '--verbose', + '-p', 'pgtap', # enable our test reporter plugin + '-ra', # show skipped and xfailed tests too + ] + + # Add temporary install, the build directory for non-installed binaries and + # also test/ for non-installed test binaries built separately. + env = test_env + env.prepend('PATH', temp_install_bindir, test_dir['bd'], test_dir['bd'] / 'test') + temp_install_datadir = '@0@@1@'.format(test_install_destdir, dir_prefix / dir_data) + env.set('share_contrib_dir', temp_install_datadir / 'contrib') + env.prepend('PYTHONPATH', pytest_env['PYTHONPATH']) + + foreach name, value : t.get('env', {}) + env.set(name, value) + endforeach + + test_group = test_dir['name'] + test_kwargs = { + 'protocol': 'tap', + 'suite': test_group, + 'timeout': 1000, + 'depends': test_deps + t.get('deps', []), + 'env': env, + } + t.get('test_kwargs', {}) + + foreach onetest : t['tests'] + # Make test names prettier, remove pyt/ and .py + onetest_p = onetest + if onetest_p.startswith('pyt/') + onetest_p = onetest.split('pyt/')[1] + endif + if onetest_p.endswith('.py') + onetest_p = fs.stem(onetest_p) + endif + + test(test_dir['name'] / onetest_p, + python, + kwargs: test_kwargs, + args: testwrap_pytest + [ + '--testgroup', test_dir['name'], + '--testname', onetest_p, + '--', test_command, + test_dir['sd'] / onetest, + ], + ) + endforeach + install_suites += test_group else error('unknown kind @0@ of test in @1@'.format(kind, test_dir['sd'])) endif @@ -4029,6 +4128,7 @@ summary( 'bison': '@0@ @1@'.format(bison.full_path(), bison_version), 'dtrace': dtrace, 'flex': '@0@ @1@'.format(flex.full_path(), flex_version), + 'pytest': pytest_enabled ? ' '.join(pytest_cmd) + ' ' + pytest_version : not_found_dep, }, section: 'Programs', ) diff --git a/meson_options.txt b/meson_options.txt index 6a793f3e479..cb4825c3575 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -41,7 +41,10 @@ option('cassert', type: 'boolean', value: false, description: 'Enable assertion checks (for debugging)') option('tap_tests', type: 'feature', value: 'auto', - description: 'Enable TAP tests') + description: 'Enable (Perl-based) TAP tests') + +option('pytest', type: 'feature', value: 'auto', + description: 'Enable (Python-based) pytest suites') option('injection_points', type: 'boolean', value: false, description: 'Enable injection points') @@ -195,6 +198,9 @@ option('PERL', type: 'string', value: 'perl', option('PROVE', type: 'string', value: 'prove', description: 'Path to prove binary') +option('PYTEST', type: 'array', value: ['pytest', 'py.test'], + description: 'Path to pytest binary') + option('PYTHON', type: 'array', value: ['python3', 'python'], description: 'Path to python binary') diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000000..60abb4d0655 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,21 @@ +[project] +name = "postgresql-hackers-tooling" +version = "0.1.0" +description = "Pytest infrastructure for PostgreSQL" +requires-python = ">=3.6" +dependencies = [ + # pytest 7.0 was the last version which supported Python 3.6, but the BSDs + # have started putting 8.x into ports, so we support both. (pytest 8 can be + # used throughout once we drop support for Python 3.7.) + "pytest >= 7.0, < 10", + + # Any other dependencies are effectively optional (added below). We import + # these libraries using pytest.importorskip(). So tests will be skipped if + # they are not available. +] + +[tool.pytest.ini_options] +minversion = "7.0" + +# Common test code can be found here. +pythonpath = ["src/test/pytest"] diff --git a/src/Makefile.global.in b/src/Makefile.global.in index 947a2d79e29..572c24c3f55 100644 --- a/src/Makefile.global.in +++ b/src/Makefile.global.in @@ -211,6 +211,7 @@ enable_dtrace = @enable_dtrace@ enable_coverage = @enable_coverage@ enable_injection_points = @enable_injection_points@ enable_tap_tests = @enable_tap_tests@ +enable_pytest = @enable_pytest@ python_includespec = @python_includespec@ python_libdir = @python_libdir@ @@ -356,6 +357,7 @@ MSGFMT = @MSGFMT@ MSGFMT_FLAGS = @MSGFMT_FLAGS@ MSGMERGE = @MSGMERGE@ OPENSSL = @OPENSSL@ +PYTEST = @PYTEST@ PYTHON = @PYTHON@ TAR = @TAR@ XGETTEXT = @XGETTEXT@ @@ -510,6 +512,33 @@ prove_installcheck = @echo "TAP tests not enabled. Try configuring with --enable prove_check = $(prove_installcheck) endif +ifeq ($(enable_pytest),yes) + +pytest_installcheck = @echo "Installcheck is not currently supported for pytest." + +# We also configure the same PYTHONPATH in the pytest settings in +# pyproject.toml, but pytest versions below 8.4 only actually use that value +# after plugin loading. So we need to configure it here too. This won't help +# people manually running pytest outside of meson/make, but we expect those to +# use a recent enough version of pytest anyway (and if not they can manually +# configure PYTHONPATH too). +define pytest_check +echo "# +++ pytest check in $(subdir) +++" && \ +rm -rf '$(CURDIR)'/tmp_check && \ +$(MKDIR_P) '$(CURDIR)'/tmp_check && \ +cd $(srcdir) && \ + TESTLOGDIR='$(CURDIR)/tmp_check/log' \ + TESTDATADIR='$(CURDIR)/tmp_check' \ + PYTHONPATH='$(abs_top_srcdir)/src/test/pytest:$$PYTHONPATH' \ + $(with_temp_install) \ + $(PYTEST) -c '$(abs_top_srcdir)/pyproject.toml' --verbose -ra ./pyt/ +endef + +else +pytest_installcheck = @echo "pytest is not enabled. Try configuring with --enable-pytest" +pytest_check = $(pytest_installcheck) +endif + # Installation. install_bin = @install_bin@ diff --git a/src/makefiles/meson.build b/src/makefiles/meson.build index 77f7a729cc2..6815178b35e 100644 --- a/src/makefiles/meson.build +++ b/src/makefiles/meson.build @@ -56,6 +56,8 @@ pgxs_kv = { 'enable_nls': libintl.found() ? 'yes' : 'no', 'enable_injection_points': get_option('injection_points') ? 'yes' : 'no', 'enable_tap_tests': tap_tests_enabled ? 'yes' : 'no', + 'enable_pytest': pytest_enabled ? 'yes' : 'no', + 'PYTEST': pytest_enabled ? ' '.join(pytest_cmd) : '', 'enable_debug': get_option('debug') ? 'yes' : 'no', 'enable_coverage': 'no', 'enable_dtrace': dtrace.found() ? 'yes' : 'no', diff --git a/src/test/Makefile b/src/test/Makefile index 3eb0a06abb4..0be9771d71f 100644 --- a/src/test/Makefile +++ b/src/test/Makefile @@ -18,6 +18,7 @@ SUBDIRS = \ modules \ perl \ postmaster \ + pytest \ recovery \ regress \ subscription diff --git a/src/test/meson.build b/src/test/meson.build index cd45cbf57fb..09175f0eaea 100644 --- a/src/test/meson.build +++ b/src/test/meson.build @@ -5,6 +5,7 @@ subdir('isolation') subdir('authentication') subdir('postmaster') +subdir('pytest') subdir('recovery') subdir('subscription') subdir('modules') diff --git a/src/test/pytest/Makefile b/src/test/pytest/Makefile new file mode 100644 index 00000000000..2bdca96ccbe --- /dev/null +++ b/src/test/pytest/Makefile @@ -0,0 +1,20 @@ +#------------------------------------------------------------------------- +# +# Makefile for pytest +# +# Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group +# Portions Copyright (c) 1994, Regents of the University of California +# +# src/test/pytest/Makefile +# +#------------------------------------------------------------------------- + +subdir = src/test/pytest +top_builddir = ../../.. +include $(top_builddir)/src/Makefile.global + +check: + $(pytest_check) + +clean distclean maintainer-clean: + rm -rf tmp_check diff --git a/src/test/pytest/README b/src/test/pytest/README new file mode 100644 index 00000000000..1333ed77b7e --- /dev/null +++ b/src/test/pytest/README @@ -0,0 +1 @@ +TODO diff --git a/src/test/pytest/meson.build b/src/test/pytest/meson.build new file mode 100644 index 00000000000..b1f6061b307 --- /dev/null +++ b/src/test/pytest/meson.build @@ -0,0 +1,15 @@ +# Copyright (c) 2025, PostgreSQL Global Development Group + +if not pytest_enabled + subdir_done() +endif + +tests += { + 'name': 'pytest', + 'sd': meson.current_source_dir(), + 'bd': meson.current_build_dir(), + 'pytest': { + 'tests': [ + ], + }, +} diff --git a/src/test/pytest/pgtap.py b/src/test/pytest/pgtap.py new file mode 100644 index 00000000000..2ae16b624d5 --- /dev/null +++ b/src/test/pytest/pgtap.py @@ -0,0 +1,197 @@ +# Copyright (c) 2025, PostgreSQL Global Development Group + +import os +import sys + +import pytest + +# +# Helpers +# + + +class TAP: + """ + A basic API for reporting via the TAP 12 protocol. + + https://testanything.org/tap-specification.html + """ + + def __init__(self): + self.count = 0 + + def expect(self, num: int): + self.print(f"1..{num}") + + def print(self, *args): + print(*args, file=sys.__stdout__) + + def ok(self, name: str): + self.count += 1 + self.print("ok", self.count, "-", name) + + def skip(self, name: str, reason: str): + self.count += 1 + self.print("ok", self.count, "-", name, "# skip", reason) + + def fail(self, name: str, details: str): + self.count += 1 + self.print("not ok", self.count, "-", name) + + # mtest has some odd behavior around TAP tests where it won't print + # diagnostics on failure if they're part of the stdout stream, so we + # might as well just dump the details directly to stderr instead. + print(details, file=sys.__stderr__) + + +tap = TAP() + + +class TestNotes: + """ + Annotations for a single test. The existing pytest hooks keep interesting + information somewhat separated across the different stages + (setup/test/teardown), so this class is used to correlate them. + """ + + skipped = False + skip_reason = None + + failed = False + details = "" + + +# Register a custom key in the stash dictionary for keeping our TestNotes. +notes_key = pytest.StashKey[TestNotes]() + + +# +# Hook Implementations +# + + +@pytest.hookimpl(tryfirst=True) +def pytest_configure(config): + """ + Hijacks the standard streams as soon as possible during pytest startup. The + pytest-formatted output gets logged to file instead, and we'll use the + original sys.__stdout__/__stderr__ streams for the TAP protocol. + """ + logdir = os.getenv("TESTLOGDIR") + if not logdir: + raise RuntimeError("pgtap requires the TESTLOGDIR envvar to be set") + + os.makedirs(logdir) + logpath = os.path.join(logdir, "pytest.log") + sys.stdout = sys.stderr = open(logpath, "a", buffering=1) + + +@pytest.hookimpl(trylast=True) +def pytest_sessionfinish(session, exitstatus): + """ + Suppresses nonzero exit codes due to failed tests. (In that case, we want + Meson to report a failure count, not a generic ERROR.) + """ + if exitstatus == pytest.ExitCode.TESTS_FAILED: + session.exitstatus = pytest.ExitCode.OK + + +@pytest.hookimpl +def pytest_collectreport(report): + # Include collection failures directly in Meson error output. + if report.failed: + print(report.longreprtext, file=sys.__stderr__) + + +@pytest.hookimpl +def pytest_internalerror(excrepr, excinfo): + # Include internal errors directly in Meson error output. + print(excrepr, file=sys.__stderr__) + + +# +# Hook Wrappers +# +# In pytest parlance, a "wrapper" for a hook can inspect and optionally modify +# existing hooks' behavior, but it does not replace the hook chain. This is done +# through a generator-style API which chains the hooks together (see the use of +# `yield`). +# + + +@pytest.hookimpl(hookwrapper=True) +def pytest_collection(session): + """Reports the number of gathered tests after collection is finished.""" + res = yield + tap.expect(session.testscollected) + return res + + +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_makereport(item, call): + """ + Annotates a test item with our TestNotes and grabs relevant information for + reporting. + + This is called multiple times per test, so it's not correct to print the TAP + result here. (A test and its teardown stage can both fail, and we want to + see the details for both.) We instead combine all the information for use by + our pytest_runtest_protocol wrapper later on. + """ + res = yield + + if notes_key not in item.stash: + item.stash[notes_key] = TestNotes() + notes = item.stash[notes_key] + + report = res.get_result() + if report.passed: + pass # no annotation needed + + elif report.skipped: + notes.skipped = True + _, _, notes.skip_reason = report.longrepr + + elif report.failed: + notes.failed = True + + if not notes.details: + notes.details += "{:_^72}\n\n".format(f" {report.head_line} ") + + if report.when in ("setup", "teardown"): + notes.details += "\n{:_^72}\n\n".format( + f" Error during {report.when} of {report.head_line} " + ) + + notes.details += report.longreprtext + "\n" + + # Include captured stdout/stderr/log in failure output + for section_name, section_content in report.sections: + if section_content.strip(): + notes.details += "\n{:-^72}\n".format(f" {section_name} ") + notes.details += section_content + "\n" + + else: + raise RuntimeError("pytest_runtest_makereport received unknown test status") + + return res + + +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_protocol(item, nextitem): + """ + Reports the TAP result for this test item using our gathered TestNotes. + """ + res = yield + + assert notes_key in item.stash, "pgtap didn't annotate a test item?" + notes = item.stash[notes_key] + + if notes.failed: + tap.fail(item.nodeid, notes.details) + elif notes.skipped: + tap.skip(item.nodeid, notes.skip_reason) + else: + tap.ok(item.nodeid) + + return res diff --git a/src/tools/testwrap b/src/tools/testwrap index e91296ecd15..346f86b8ea3 100755 --- a/src/tools/testwrap +++ b/src/tools/testwrap @@ -42,7 +42,11 @@ open(os.path.join(testdir, 'test.start'), 'x') env_dict = {**os.environ, 'TESTDATADIR': os.path.join(testdir, 'data'), - 'TESTLOGDIR': os.path.join(testdir, 'log')} + 'TESTLOGDIR': os.path.join(testdir, 'log'), + # Prevent emitting terminal capability sequences that pollute the + # TAP output stream (i.e.\033[?1034h). This happens on OpenBSD with + # pytest for unknown reasons. + 'TERM': ''} # The configuration time value of PG_TEST_EXTRA is supplied via argument base-commit: 9b9eaf08ab2dc22c691b22e59f1574e0f1bcc822 -- 2.52.0