From efd17f34929d0f00477788fffe88b6fe5be27653 Mon Sep 17 00:00:00 2001 From: Melanie Plageman Date: Thu, 23 Apr 2026 11:13:23 -0400 Subject: [PATCH v1_PGMASTER 4/7] Test that VM clear registers VM buffers WAL summarizer only includes registered buffers, so you could end up with data corruption in incremental backups if VM clears were meant to be covered by the incremental backup. Author: Melanie Plageman Reported-by: Andres Freund Original repro by Andres Backpatch through: 17 when incremental backup was introduced --- src/bin/pg_combinebackup/meson.build | 1 + .../pg_combinebackup/t/012_vm_consistency.pl | 270 ++++++++++++++++++ 2 files changed, 271 insertions(+) create mode 100644 src/bin/pg_combinebackup/t/012_vm_consistency.pl diff --git a/src/bin/pg_combinebackup/meson.build b/src/bin/pg_combinebackup/meson.build index a35b86f3f59..ba1c8cfa3d0 100644 --- a/src/bin/pg_combinebackup/meson.build +++ b/src/bin/pg_combinebackup/meson.build @@ -39,6 +39,7 @@ tests += { 't/009_no_full_file.pl', 't/010_hardlink.pl', 't/011_ib_truncation.pl', + 't/012_vm_consistency.pl', ], } } diff --git a/src/bin/pg_combinebackup/t/012_vm_consistency.pl b/src/bin/pg_combinebackup/t/012_vm_consistency.pl new file mode 100644 index 00000000000..cb03f40bfea --- /dev/null +++ b/src/bin/pg_combinebackup/t/012_vm_consistency.pl @@ -0,0 +1,270 @@ +# Copyright (c) 2021-2026, PostgreSQL Global Development Group +# +# Test that heap operations clearing visibility map bits (INSERT, UPDATE, +# DELETE, SELECT FOR UPDATE, COPY) are correctly registering visibility map +# buffers, since incremental backups rely on WAL summarizer which only tracks +# registered buffers. +# +# Each operation is tested in a separate vacuum-modify-backup-validate +# cycle to isolate failures. + +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $tempdir = PostgreSQL::Test::Utils::tempdir_short(); +my $mode = $ENV{PG_TEST_PG_COMBINEBACKUP_MODE} || '--copy'; + +# Set up primary with WAL summarization enabled. +my $primary = PostgreSQL::Test::Cluster->new('primary'); +$primary->init(has_archiving => 1, allows_streaming => 1); +$primary->append_conf('postgresql.conf', <start; + +$primary->safe_psql('postgres', q{CREATE EXTENSION pg_visibility}); + +# Create tables for each test case. +$primary->safe_psql('postgres', <safe_psql('postgres', + "SELECT all_visible, all_frozen FROM pg_visibility_map_summary('$table')"); + my @vals = split(/\|/, $result); + return @vals; +} + +# Helper: take a full backup. Returns ($path, $name). +sub take_full_backup +{ + $backup_num++; + my $name = "full_$backup_num"; + my $path = $primary->backup_dir . "/$name"; + $primary->command_ok( + [ + 'pg_basebackup', '--no-sync', + '--pgdata' => $path, + '--checkpoint' => 'fast', + ], + "full backup $backup_num"); + return ($path, $name); +} + +# Helper: take an incremental backup. Returns ($path, $name). +sub take_incr_backup +{ + my ($prior_path) = @_; + $backup_num++; + my $name = "incr_$backup_num"; + my $path = $primary->backup_dir . "/$name"; + $primary->command_ok( + [ + 'pg_basebackup', '--no-sync', + '--pgdata' => $path, + '--checkpoint' => 'fast', + '--incremental' => $prior_path . '/backup_manifest', + ], + "incremental backup $backup_num"); + return ($path, $name); +} + +# Helper: combine two backups, restore, validate VM for the given table, +# then stop the restored server. Checks both all_visible and all_frozen. +sub validate_restored_vm +{ + my ($full_path, $full_name, $incr_path, $incr_name, $table, $label) = @_; + + $restore_num++; + + my $restored = PostgreSQL::Test::Cluster->new("restored_$restore_num"); + $restored->init_from_backup($primary, $incr_name, + combine_with_prior => [$full_name], + combine_mode => $mode); + $restored->append_conf('postgresql.conf', <start; + $restored->safe_psql('postgres', + q{CREATE EXTENSION IF NOT EXISTS pg_visibility}); + + my ($primary_visible, $primary_frozen) = get_vm_summary($primary, $table); + my ($restored_visible, $restored_frozen) = get_vm_summary($restored, $table); + + is($restored_visible, $primary_visible, + "$label: restored all_visible count matches primary"); + is($restored_frozen, $primary_frozen, + "$label: restored all_frozen count matches primary"); + + my $corrupt_tids = $restored->safe_psql('postgres', + "SELECT count(*) FROM pg_check_visible('$table')"); + is($corrupt_tids, '0', + "$label: no VM corruption detected by pg_check_visible"); + + $restored->stop; +} + +## +## Test 1: INSERT clears all-visible and all-frozen bits +## + +$primary->safe_psql('postgres', q{VACUUM (FREEZE) vm_insert_test}); +my ($pre_ins_vis, $pre_ins_frz) = get_vm_summary($primary, 'vm_insert_test'); +cmp_ok($pre_ins_vis, '>', 0, + "INSERT test: pages are all-visible after vacuum"); +cmp_ok($pre_ins_frz, '>', 0, + "INSERT test: pages are all-frozen after vacuum"); + +my ($full1, $full1_name) = take_full_backup(); + +$primary->safe_psql('postgres', + q{INSERT INTO vm_insert_test VALUES (1)}); + +my ($post_ins_vis, $post_ins_frz) = get_vm_summary($primary, 'vm_insert_test'); +cmp_ok($post_ins_vis, '<', $pre_ins_vis, + "INSERT test: all-visible cleared after insert"); +cmp_ok($post_ins_frz, '<', $pre_ins_frz, + "INSERT test: all-frozen cleared after insert"); + +my ($incr1, $incr1_name) = take_incr_backup($full1); +validate_restored_vm($full1, $full1_name, $incr1, $incr1_name, + 'vm_insert_test', 'INSERT test'); + +## +## Test 2: DELETE clears all-visible and all-frozen bits +## + +$primary->safe_psql('postgres', q{VACUUM (FREEZE) vm_delete_test}); +my ($pre_del_vis, $pre_del_frz) = get_vm_summary($primary, 'vm_delete_test'); +cmp_ok($pre_del_vis, '>', 0, + "DELETE test: pages are all-visible after vacuum"); +cmp_ok($pre_del_frz, '>', 0, + "DELETE test: pages are all-frozen after vacuum"); + +my ($full2, $full2_name) = take_full_backup(); + +$primary->safe_psql('postgres', + q{DELETE FROM vm_delete_test WHERE id = 1}); + +my ($post_del_vis, $post_del_frz) = get_vm_summary($primary, 'vm_delete_test'); +cmp_ok($post_del_vis, '<', $pre_del_vis, + "DELETE test: all-visible cleared after delete"); +cmp_ok($post_del_frz, '<', $pre_del_frz, + "DELETE test: all-frozen cleared after delete"); + +my ($incr2, $incr2_name) = take_incr_backup($full2); +validate_restored_vm($full2, $full2_name, $incr2, $incr2_name, + 'vm_delete_test', 'DELETE test'); + +## +## Test 3: UPDATE clears all-visible and all-frozen bits +## + +$primary->safe_psql('postgres', q{VACUUM (FREEZE) vm_update_test}); +my ($pre_upd_vis, $pre_upd_frz) = get_vm_summary($primary, 'vm_update_test'); +cmp_ok($pre_upd_vis, '>', 0, + "UPDATE test: pages are all-visible after vacuum"); +cmp_ok($pre_upd_frz, '>', 0, + "UPDATE test: pages are all-frozen after vacuum"); + +my ($full3, $full3_name) = take_full_backup(); + +# This should update the same page +$primary->safe_psql('postgres', + q{UPDATE vm_update_test SET id = 0 WHERE id = 1}); +# At least some of these updates should be cross-page +$primary->safe_psql('postgres', + q{UPDATE vm_update_test SET id = 99 WHERE id > 50}); + +my ($post_upd_vis, $post_upd_frz) = get_vm_summary($primary, 'vm_update_test'); +cmp_ok($post_upd_vis, '<', $pre_upd_vis, + "UPDATE test: all-visible cleared after update"); +cmp_ok($post_upd_frz, '<', $pre_upd_frz, + "UPDATE test: all-frozen cleared after update"); + +my ($incr3, $incr3_name) = take_incr_backup($full3); +validate_restored_vm($full3, $full3_name, $incr3, $incr3_name, + 'vm_update_test', 'UPDATE test'); + +## +## Test 4: SELECT FOR UPDATE clears all-frozen bit (but not all-visible) +## +## heap_lock_tuple only clears VISIBILITYMAP_ALL_FROZEN, not ALL_VISIBLE +## + +$primary->safe_psql('postgres', q{VACUUM (FREEZE) vm_lock_test}); +my ($pre_lock_vis, $pre_lock_frz) = get_vm_summary($primary, 'vm_lock_test'); +cmp_ok($pre_lock_vis, '>', 0, + "LOCK test: pages are all-visible after vacuum"); +cmp_ok($pre_lock_frz, '>', 0, + "LOCK test: pages are all-frozen after vacuum"); + +my ($full4, $full4_name) = take_full_backup(); + +$primary->safe_psql('postgres', + q{SELECT * FROM vm_lock_test WHERE id = 1 FOR UPDATE}); + +my ($post_lock_vis, $post_lock_frz) = get_vm_summary($primary, 'vm_lock_test'); +is($post_lock_vis, $pre_lock_vis, + "LOCK test: all-visible unchanged after SELECT FOR UPDATE"); +cmp_ok($post_lock_frz, '<', $pre_lock_frz, + "LOCK test: all-frozen cleared after SELECT FOR UPDATE"); + +my ($incr4, $incr4_name) = take_incr_backup($full4); +validate_restored_vm($full4, $full4_name, $incr4, $incr4_name, + 'vm_lock_test', 'LOCK test'); + +## +## Test 5: COPY clears all-visible and all-frozen bits +## + +$primary->safe_psql('postgres', q{VACUUM (FREEZE) vm_copy_test}); +my ($pre_copy_vis, $pre_copy_frz) = get_vm_summary($primary, 'vm_copy_test'); +cmp_ok($pre_copy_vis, '>', 0, + "COPY test: pages are all-visible after vacuum"); +cmp_ok($pre_copy_frz, '>', 0, + "COPY test: pages are all-frozen after vacuum"); + +my ($full5, $full5_name) = take_full_backup(); + +$primary->safe_psql('postgres', + "COPY vm_copy_test FROM PROGRAM 'echo 42'"); + +my ($post_copy_vis, $post_copy_frz) = get_vm_summary($primary, 'vm_copy_test'); +cmp_ok($post_copy_vis, '<', $pre_copy_vis, + "COPY test: all-visible cleared after COPY"); +cmp_ok($post_copy_frz, '<', $pre_copy_frz, + "COPY test: all-frozen cleared after COPY"); + +my ($incr5, $incr5_name) = take_incr_backup($full5); +validate_restored_vm($full5, $full5_name, $incr5, $incr5_name, + 'vm_copy_test', 'COPY test'); + +$primary->stop; + +done_testing(); -- 2.43.0