From 82966219688878a568f1fb53dd200df12a1f6e4f Mon Sep 17 00:00:00 2001 From: Jelte Fennema-Nio Date: Mon, 27 Jul 2026 00:11:59 +0200 Subject: [PATCH v1 06/12] POC: amcheck: port TAP test 001_verify_heapam to pytest MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This test becomes much faster on all platforms in CI: platform perl pytest diff ----------- ---------------- ---------------- ---------------- windows 20.1s (±0.2) 3.9s (±0.0) -16.2s (-81%) mingw 14.4s (±0.3) 2.6s (±0.0) -11.7s (-82%) linux-64 11.3s (±0.1) 4.0s (±0.0) -7.3s (-65%) macos 9.0s (±1.2) 2.5s (±0.2) -6.5s (-72%) linux-32 4.0s (±0.1) 1.9s (±0.1) -2.0s (-51%) Timings are means of 5 runs of each form, interleaved on one CI runner with nothing else running on it; ± is the standard deviation across the 5 runs. LOC (no comments or blanks, with tokei): 194 -> 136 (-30%). --- contrib/amcheck/Makefile | 1 + contrib/amcheck/meson.build | 6 +- contrib/amcheck/pyt/test_001_verify_heapam.py | 197 +++++++++++++ contrib/amcheck/t/001_verify_heapam.pl | 273 ------------------ 4 files changed, 203 insertions(+), 274 deletions(-) create mode 100644 contrib/amcheck/pyt/test_001_verify_heapam.py delete mode 100644 contrib/amcheck/t/001_verify_heapam.pl diff --git a/contrib/amcheck/Makefile b/contrib/amcheck/Makefile index 1b7a63cbaa4..fd54b6b317e 100644 --- a/contrib/amcheck/Makefile +++ b/contrib/amcheck/Makefile @@ -17,6 +17,7 @@ REGRESS = check check_btree check_gin check_heap EXTRA_INSTALL = contrib/pg_walinspect TAP_TESTS = 1 +PYTEST_TESTS = 1 ifdef USE_PGXS PG_CONFIG = pg_config diff --git a/contrib/amcheck/meson.build b/contrib/amcheck/meson.build index d5137ef691d..18885cc48df 100644 --- a/contrib/amcheck/meson.build +++ b/contrib/amcheck/meson.build @@ -42,9 +42,13 @@ tests += { 'check_heap', ], }, + 'pytest': { + 'tests': [ + 'pyt/test_001_verify_heapam.py', + ], + }, 'tap': { 'tests': [ - 't/001_verify_heapam.pl', 't/002_cic.pl', 't/003_cic_2pc.pl', 't/004_verify_nbtree_unique.pl', diff --git a/contrib/amcheck/pyt/test_001_verify_heapam.py b/contrib/amcheck/pyt/test_001_verify_heapam.py new file mode 100644 index 00000000000..e6432cffb5d --- /dev/null +++ b/contrib/amcheck/pyt/test_001_verify_heapam.py @@ -0,0 +1,197 @@ +# Copyright (c) 2021-2026, PostgreSQL Global Development Group + +"""Port of contrib/amcheck/t/001_verify_heapam.pl. + +Exercises verify_heapam(): an uncorrupted table (and a sequence, which is a heap +under the hood) report nothing across all option combinations, while a table +whose first page has had its line pointers corrupted is reported. Data checksums +are disabled so the hand-corrupted page reads back without a checksum error. +""" + +import itertools +import re +import struct + +# Each line-pointer check in verify_heapam.c that the corrupted first page is +# expected to trip. The values packed into the page are chosen to hit all of +# them. +LINE_POINTER_ERRORS = [ + r"line pointer redirection to item at offset \d+ precedes minimum offset \d+", + r"line pointer redirection to item at offset \d+ exceeds maximum offset \d+", + r"line pointer to page offset \d+ is not maximally aligned", + r"line pointer length \d+ is less than the minimum tuple header size \d+", + r"line pointer to page offset \d+ with length \d+ ends beyond maximum page offset \d+", +] + + +def relpath(node, relname): + """Returns the filesystem path for the named relation.""" + return node.datadir / node.sql("SELECT pg_relation_filepath($1)", relname) + + +def verify_heapam(node, relation, **options): + """Run verify_heapam() on ``relation`` and return the messages it reports. + + Options are given as keyword arguments and bound as query parameters, so + callers say ``skip="all-visible"`` and nothing here has to quote SQL. Only + the msg column is selected: the block and offset numbers depend on the + page layout and nothing here asserts on them. + """ + args = "".join(f", {name} := ${i}" for i, name in enumerate(options, start=2)) + rows = node.sql( + f"SELECT msg FROM verify_heapam($1{args})", + relation, + *options.values(), + simplify_result=False, + ) + return [msg for (msg,) in rows] + + +def assert_line_pointer_errors(msgs): + """Assert every line-pointer check fired at least once.""" + for pattern in LINE_POINTER_ERRORS: + assert any(re.search(pattern, m) for m in msgs), f"no message matched {pattern}" + + +def fresh_test_table(node, relname): + """(Re)create and populate a test table of the given name.""" + node.sql_batch( + f"DROP TABLE IF EXISTS {relname} CASCADE", + f"CREATE TABLE {relname} (a integer, b text)", + f"ALTER TABLE {relname} SET (autovacuum_enabled=false)", + f"ALTER TABLE {relname} ALTER b SET STORAGE external", + f"INSERT INTO {relname} (a, b) " + f"(SELECT gs, repeat('b',gs*10) FROM generate_series(1,1000) gs)", + # A couple of locked/updated rows under savepoints, to exercise + # multixact and update-chain handling. + "BEGIN", + "SAVEPOINT s1", + f"SELECT 1 FROM {relname} WHERE a = 42 FOR UPDATE", + f"UPDATE {relname} SET b = b WHERE a = 42", + "RELEASE s1", + "SAVEPOINT s1", + f"SELECT 1 FROM {relname} WHERE a = 42 FOR UPDATE", + f"UPDATE {relname} SET b = b WHERE a = 42", + "COMMIT", + ) + + +def corrupt_first_page(node, relname): + """Stops the test node, corrupts the first page of the named relation, and + restarts the node. + """ + path = relpath(node, relname) + node.stop() + with open(path, "r+b") as f: + # Corrupt some line pointers. The values are chosen to hit the + # various line-pointer-corruption checks in verify_heapam.c + # on both little-endian and big-endian architectures. + f.seek(32) + f.write( + struct.pack( + "<6L", + 0xAAA15550, + 0xAAA0D550, + 0x00010000, + 0x00008000, + 0x0000800F, + 0x001E8000, + ) + ) + node.start() + + +def check_all_options_uncorrupted(node, relname): + """Check various options are stable (don't abort) and do not report + corruption when running verify_heapam on an uncorrupted test table. + + The relname *must* be an uncorrupted table, or this will fail. + """ + combinations = itertools.product( + (True, False), # on_error_stop + (True, False), # check_toast + ("none", "all-frozen", "all-visible"), # skip + (None, 0), # startblock + (None, 0), # endblock + ) + for on_error_stop, check_toast, skip, startblock, endblock in combinations: + options = dict( + on_error_stop=on_error_stop, + check_toast=check_toast, + skip=skip, + startblock=startblock, + endblock=endblock, + ) + assert verify_heapam(node, relname, **options) == [], options + + +def test_verify_heapam(create_pg): + # + # Test set-up + # + # Data checksums are off so the hand-corrupted page below reads back + # without tripping a checksum error first. + node = create_pg( + "test", initdb_opts=["--no-data-checksums"], conf={"autovacuum": False} + ) + node.sql("CREATE EXTENSION amcheck") + + # + # Check a table with data loaded but no corruption, freezing, etc. + # + fresh_test_table(node, "test") + check_all_options_uncorrupted(node, "test") + + # + # Check a corrupt table + # + fresh_test_table(node, "test") + corrupt_first_page(node, "test") + assert_line_pointer_errors(verify_heapam(node, "test")) + assert_line_pointer_errors(verify_heapam(node, "test", skip="all-visible")) + assert_line_pointer_errors(verify_heapam(node, "test", skip="all-frozen")) + assert_line_pointer_errors(verify_heapam(node, "test", check_toast=False)) + assert_line_pointer_errors(verify_heapam(node, "test", startblock=0, endblock=0)) + + # + # Check a corrupt table with all-frozen data + # + fresh_test_table(node, "test") + node.sql("VACUUM (FREEZE, DISABLE_PAGE_SKIPPING) test") + assert verify_heapam(node, "test") == [], "all-frozen not corrupted table" + corrupt_first_page(node, "test") + assert_line_pointer_errors(verify_heapam(node, "test")) + + # Skipping all-frozen pages skips the corrupted (frozen) page. + assert verify_heapam(node, "test", skip="all-frozen") == [], ( + "all-frozen corrupted table skipping all-frozen" + ) + + # + # Check a sequence with no corruption. The current implementation of + # sequences doesn't require its own test setup, since sequences are really + # just heap tables under-the-hood. To guard against future implementation + # changes made without remembering to update verify_heapam, we create and + # exercise a sequence, checking along the way that it passes corruption + # checks. + # + # Create a test sequence of the given name. + node.sql_batch( + "DROP SEQUENCE IF EXISTS test_seq CASCADE", + "CREATE SEQUENCE test_seq INCREMENT BY 13 MINVALUE 17 START WITH 23", + "SELECT nextval('test_seq')", + "SELECT setval('test_seq', currval('test_seq') + nextval('test_seq'))", + ) + check_all_options_uncorrupted(node, "test_seq") + + # Call SQL functions to increment the sequence + node.sql("SELECT nextval('test_seq')") + check_all_options_uncorrupted(node, "test_seq") + + # Call SQL functions to set the sequence + node.sql("SELECT setval('test_seq', 102)") + check_all_options_uncorrupted(node, "test_seq") + + # Call SQL functions to reset the sequence + node.sql("ALTER SEQUENCE test_seq RESTART WITH 51") + check_all_options_uncorrupted(node, "test_seq") diff --git a/contrib/amcheck/t/001_verify_heapam.pl b/contrib/amcheck/t/001_verify_heapam.pl deleted file mode 100644 index e3fee19ae5d..00000000000 --- a/contrib/amcheck/t/001_verify_heapam.pl +++ /dev/null @@ -1,273 +0,0 @@ - -# Copyright (c) 2021-2026, PostgreSQL Global Development Group - -use strict; -use warnings FATAL => 'all'; - -use PostgreSQL::Test::Cluster; -use PostgreSQL::Test::Utils; - -use Test::More; - -my $node; - -# -# Test set-up -# -$node = PostgreSQL::Test::Cluster->new('test'); -$node->init(no_data_checksums => 1); -$node->append_conf('postgresql.conf', 'autovacuum=off'); -$node->start; -$node->safe_psql('postgres', q(CREATE EXTENSION amcheck)); - -# -# Check a table with data loaded but no corruption, freezing, etc. -# -fresh_test_table('test'); -check_all_options_uncorrupted('test', 'plain'); - -# -# Check a corrupt table -# -fresh_test_table('test'); -corrupt_first_page('test'); -detects_heap_corruption("verify_heapam('test')", "plain corrupted table"); -detects_heap_corruption( - "verify_heapam('test', skip := 'all-visible')", - "plain corrupted table skipping all-visible"); -detects_heap_corruption( - "verify_heapam('test', skip := 'all-frozen')", - "plain corrupted table skipping all-frozen"); -detects_heap_corruption( - "verify_heapam('test', check_toast := false)", - "plain corrupted table skipping toast"); -detects_heap_corruption( - "verify_heapam('test', startblock := 0, endblock := 0)", - "plain corrupted table checking only block zero"); - -# -# Check a corrupt table with all-frozen data -# -fresh_test_table('test'); -$node->safe_psql('postgres', q(VACUUM (FREEZE, DISABLE_PAGE_SKIPPING) test)); -detects_no_corruption("verify_heapam('test')", - "all-frozen not corrupted table"); -corrupt_first_page('test'); -detects_heap_corruption("verify_heapam('test')", - "all-frozen corrupted table"); -detects_no_corruption( - "verify_heapam('test', skip := 'all-frozen')", - "all-frozen corrupted table skipping all-frozen"); - -# -# Check a sequence with no corruption. The current implementation of sequences -# doesn't require its own test setup, since sequences are really just heap -# tables under-the-hood. To guard against future implementation changes made -# without remembering to update verify_heapam, we create and exercise a -# sequence, checking along the way that it passes corruption checks. -# -fresh_test_sequence('test_seq'); -check_all_options_uncorrupted('test_seq', 'plain'); -advance_test_sequence('test_seq'); -check_all_options_uncorrupted('test_seq', 'plain'); -set_test_sequence('test_seq'); -check_all_options_uncorrupted('test_seq', 'plain'); -reset_test_sequence('test_seq'); -check_all_options_uncorrupted('test_seq', 'plain'); - -# Returns the filesystem path for the named relation. -sub relation_filepath -{ - my ($relname) = @_; - - my $pgdata = $node->data_dir; - my $rel = $node->safe_psql('postgres', - qq(SELECT pg_relation_filepath('$relname'))); - die "path not found for relation $relname" unless defined $rel; - return "$pgdata/$rel"; -} - -# (Re)create and populate a test table of the given name. -sub fresh_test_table -{ - my ($relname) = @_; - - return $node->safe_psql( - 'postgres', qq( - DROP TABLE IF EXISTS $relname CASCADE; - CREATE TABLE $relname (a integer, b text); - ALTER TABLE $relname SET (autovacuum_enabled=false); - ALTER TABLE $relname ALTER b SET STORAGE external; - INSERT INTO $relname (a, b) - (SELECT gs, repeat('b',gs*10) FROM generate_series(1,1000) gs); - BEGIN; - SAVEPOINT s1; - SELECT 1 FROM $relname WHERE a = 42 FOR UPDATE; - UPDATE $relname SET b = b WHERE a = 42; - RELEASE s1; - SAVEPOINT s1; - SELECT 1 FROM $relname WHERE a = 42 FOR UPDATE; - UPDATE $relname SET b = b WHERE a = 42; - COMMIT; - )); -} - -# Create a test sequence of the given name. -sub fresh_test_sequence -{ - my ($seqname) = @_; - - return $node->safe_psql( - 'postgres', qq( - DROP SEQUENCE IF EXISTS $seqname CASCADE; - CREATE SEQUENCE $seqname - INCREMENT BY 13 - MINVALUE 17 - START WITH 23; - SELECT nextval('$seqname'); - SELECT setval('$seqname', currval('$seqname') + nextval('$seqname')); - )); -} - -# Call SQL functions to increment the sequence -sub advance_test_sequence -{ - my ($seqname) = @_; - - return $node->safe_psql( - 'postgres', qq( - SELECT nextval('$seqname'); - )); -} - -# Call SQL functions to set the sequence -sub set_test_sequence -{ - my ($seqname) = @_; - - return $node->safe_psql( - 'postgres', qq( - SELECT setval('$seqname', 102); - )); -} - -# Call SQL functions to reset the sequence -sub reset_test_sequence -{ - my ($seqname) = @_; - - return $node->safe_psql( - 'postgres', qq( - ALTER SEQUENCE $seqname RESTART WITH 51 - )); -} - -# Stops the test node, corrupts the first page of the named relation, and -# restarts the node. -sub corrupt_first_page -{ - my ($relname) = @_; - my $relpath = relation_filepath($relname); - - $node->stop; - - my $fh; - open($fh, '+<', $relpath) - or BAIL_OUT("open failed: $!"); - binmode $fh; - - # Corrupt some line pointers. The values are chosen to hit the - # various line-pointer-corruption checks in verify_heapam.c - # on both little-endian and big-endian architectures. - sysseek($fh, 32, 0) - or BAIL_OUT("sysseek failed: $!"); - syswrite( - $fh, - pack("L*", - 0xAAA15550, 0xAAA0D550, 0x00010000, - 0x00008000, 0x0000800F, 0x001e8000) - ) or BAIL_OUT("syswrite failed: $!"); - close($fh) - or BAIL_OUT("close failed: $!"); - - $node->start; -} - -sub detects_heap_corruption -{ - local $Test::Builder::Level = $Test::Builder::Level + 1; - - my ($function, $testname) = @_; - - detects_corruption( - $function, - $testname, - qr/line pointer redirection to item at offset \d+ precedes minimum offset \d+/, - qr/line pointer redirection to item at offset \d+ exceeds maximum offset \d+/, - qr/line pointer to page offset \d+ is not maximally aligned/, - qr/line pointer length \d+ is less than the minimum tuple header size \d+/, - qr/line pointer to page offset \d+ with length \d+ ends beyond maximum page offset \d+/, - ); -} - -sub detects_corruption -{ - local $Test::Builder::Level = $Test::Builder::Level + 1; - - my ($function, $testname, @re) = @_; - - my $result = $node->safe_psql('postgres', qq(SELECT * FROM $function)); - like($result, $_, $testname) for (@re); -} - -sub detects_no_corruption -{ - local $Test::Builder::Level = $Test::Builder::Level + 1; - - my ($function, $testname) = @_; - - my $result = $node->safe_psql('postgres', qq(SELECT * FROM $function)); - is($result, '', $testname); -} - -# Check various options are stable (don't abort) and do not report corruption -# when running verify_heapam on an uncorrupted test table. -# -# The relname *must* be an uncorrupted table, or this will fail. -# -# The prefix is used to identify the test, along with the options, -# and should be unique. -sub check_all_options_uncorrupted -{ - local $Test::Builder::Level = $Test::Builder::Level + 1; - - my ($relname, $prefix) = @_; - - for my $stop (qw(true false)) - { - for my $check_toast (qw(true false)) - { - for my $skip ("'none'", "'all-frozen'", "'all-visible'") - { - for my $startblock (qw(NULL 0)) - { - for my $endblock (qw(NULL 0)) - { - my $opts = - "on_error_stop := $stop, " - . "check_toast := $check_toast, " - . "skip := $skip, " - . "startblock := $startblock, " - . "endblock := $endblock"; - - detects_no_corruption( - "verify_heapam('$relname', $opts)", - "$prefix: $opts"); - } - } - } - } - } -} - -done_testing(); -- 2.54.0