diff --git a/src/test/modules/test_fileops/meson.build b/src/test/modules/test_fileops/meson.build index e14676912a4..f6e4d7fc69c 100644 --- a/src/test/modules/test_fileops/meson.build +++ b/src/test/modules/test_fileops/meson.build @@ -44,6 +44,10 @@ tests += { 't/018_fileops_rename.pl', 't/023_fileops_inline_abort_ctx.pl', 't/024_fileops_recovery_apply_failure.pl', + 't/026_fileops_abort_checkpoint_order.pl', + 't/027_fileops_crash_loser_standby.pl', + 't/028_fileops_recovery_apply_failure_count.pl', + 't/029_fileops_insert_holdoff_gdb.pl', ], }, } diff --git a/src/test/modules/test_fileops/t/026_fileops_abort_checkpoint_order.pl b/src/test/modules/test_fileops/t/026_fileops_abort_checkpoint_order.pl new file mode 100644 index 00000000000..0e68ae43398 --- /dev/null +++ b/src/test/modules/test_fileops/t/026_fileops_abort_checkpoint_order.pl @@ -0,0 +1,75 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group + +# A checkpoint must not move the redo pointer past an abort record whose +# FILEOPS inverses are not yet durable. Pause the aborting backend after the +# abort record is flushed and before it applies the inverses, request a +# CHECKPOINT, then crash. Recovery must replay the abort record and remove +# the file, for ROLLBACK and for ROLLBACK PREPARED. +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; +use Time::HiRes qw(usleep); + +plan skip_all => 'injection points not supported by this build' + unless $ENV{enable_injection_points} eq 'yes'; + +my $node = PostgreSQL::Test::Cluster->new('n'); +$node->init(allows_streaming => 1); +$node->append_conf('postgresql.conf', + "max_prepared_transactions = 5\ncheckpoint_timeout = 1h\n"); +$node->start; +$node->safe_psql('postgres', + 'CREATE EXTENSION test_fileops; CREATE EXTENSION injection_points'); + +for my $case ('rollback', 'rollback prepared') +{ + (my $f = "ck-$case.dat") =~ s/ /-/g; + $node->safe_psql('postgres', + "SELECT injection_points_attach('fileops-abort-after-flush', 'wait')"); + my $before = $node->safe_psql('postgres', + 'SELECT checkpoint_lsn FROM pg_control_checkpoint()'); + + my $a = $node->background_psql('postgres'); + if ($case eq 'rollback') + { + $a->query_until(qr/go/, qq{\\echo go +BEGIN; SELECT test_fileops_create('$f', 384); ROLLBACK; +}); + } + else + { + $node->safe_psql('postgres', + "BEGIN; SELECT test_fileops_create('$f', 384); PREPARE TRANSACTION 'p';"); + $a->query_until(qr/go/, qq{\\echo go +ROLLBACK PREPARED 'p'; +}); + } + $node->wait_for_event('client backend', 'fileops-abort-after-flush'); + ok(-e $node->data_dir . "/$f", "$case: file still there at the pause"); + + my $c = $node->background_psql('postgres'); + $c->query_until(qr/go/, "\\echo go\nCHECKPOINT;\n"); + + # Give the checkpoint time to finish if nothing holds it back. + my $after = $before; + for (1 .. 50) + { + $after = $node->safe_psql('postgres', + 'SELECT checkpoint_lsn FROM pg_control_checkpoint()'); + last if $after ne $before; + usleep(100_000); + } + note "$case: checkpoint completed during the pause: " + . ($after ne $before ? 'yes' : 'no'); + + $node->stop('immediate'); + eval { $a->quit; }; + eval { $c->quit; }; + $node->start; + ok(!-e $node->data_dir . "/$f", "$case: crash recovery removed the file"); +} + +$node->stop; +done_testing(); diff --git a/src/test/modules/test_fileops/t/027_fileops_crash_loser_standby.pl b/src/test/modules/test_fileops/t/027_fileops_crash_loser_standby.pl new file mode 100644 index 00000000000..04d77a3f360 --- /dev/null +++ b/src/test/modules/test_fileops/t/027_fileops_crash_loser_standby.pl @@ -0,0 +1,63 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group + +# A transaction is in flight with a FILEOPS create when the primary crashes. +# The primary's startup undo phase reverses it. A crashed transaction leaves +# no abort record, so the question is whether the standby, replaying the +# primary's WAL, ends with the same files. +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $primary = PostgreSQL::Test::Cluster->new('primary'); +$primary->init(allows_streaming => 1); +$primary->append_conf('postgresql.conf', "restart_after_crash = on\n"); +$primary->start; +$primary->safe_psql('postgres', 'CREATE EXTENSION test_fileops'); +$primary->backup('b'); +my $standby = PostgreSQL::Test::Cluster->new('standby'); +$standby->init_from_backup($primary, 'b', has_streaming => 1); +$standby->start; + +for my $how ('immediate stop', 'backend killed') +{ + (my $f = "loser-$how.dat") =~ s/ /-/g; + my $a = $primary->background_psql('postgres'); + $a->query_safe("BEGIN; SELECT test_fileops_create('$f', 384); SELECT test_fileops_mkdir('$f.d', 448);"); + # make it durable and replicated + $primary->safe_psql('postgres', 'CREATE TABLE IF NOT EXISTS flush_me (a int)'); + $primary->wait_for_catchup($standby); + ok(-e $standby->data_dir . "/$f", "$how: standby has the in-flight file"); + + if ($how eq 'immediate stop') + { + $primary->stop('immediate'); + eval { $a->quit; }; + $primary->start; + } + else + { + my $pid = $a->query_safe('SELECT pg_backend_pid()'); + chomp $pid; + kill 'KILL', $pid; + eval { $a->quit; }; + $primary->poll_query_until('postgres', 'SELECT 1', '1'); + } + ok(!-e $primary->data_dir . "/$f", "$how: primary reversed the loser"); + $primary->safe_psql('postgres', 'CHECKPOINT'); + $primary->wait_for_catchup($standby); + ok(!-e $standby->data_dir . "/$f", "$how: standby reversed the loser file"); + ok(!-e $standby->data_dir . "/$f.d", "$how: standby reversed the loser dir"); + +} + +# The standby converges only when it is promoted and runs its own undo phase. +$standby->promote; +for my $how ('immediate-stop', 'backend-killed') +{ + ok(!-e $standby->data_dir . "/loser-$how.dat", "promoted standby: loser-$how.dat absent"); +} +$standby->stop; +$primary->stop; +done_testing(); diff --git a/src/test/modules/test_fileops/t/028_fileops_recovery_apply_failure_count.pl b/src/test/modules/test_fileops/t/028_fileops_recovery_apply_failure_count.pl new file mode 100644 index 00000000000..f3fa4f47b05 --- /dev/null +++ b/src/test/modules/test_fileops/t/028_fileops_recovery_apply_failure_count.pl @@ -0,0 +1,63 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group + +# FILEOPS WRITE inverses vs. a real lstat() error (no injection point), with +# no revert worker: each failing inverse is tried once by the startup undo +# phase and counted in its final WARNING. +# +# Two crash losers overwrite three files (two in one transaction, one in the +# other). Before the restart the files' directory loses its search +# permission, so all three inverses fail with EACCES. The WARNING must say +# 3 records, not 1 or 2: test_fileops/024 has a single failure, so it cannot +# tell a count from a flag. +# +# Must not run as root (root ignores the directory permission). +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +plan skip_all => 'running as root: EACCES cannot be produced' if $> == 0; + +my $node = PostgreSQL::Test::Cluster->new('n'); +$node->init(allows_streaming => 1); +$node->append_conf('postgresql.conf', "autovacuum = off\n"); +$node->start; +$node->safe_psql('postgres', 'CREATE EXTENSION test_fileops'); + +my $dir = PostgreSQL::Test::Utils::tempdir('eacces'); +for my $f ('w1', 'w2', 'w3') +{ + open(my $fh, '>', "$dir/$f.dat") or die "open: $!"; + print $fh 'X' x 1024; + close($fh); +} + +my $a = $node->background_psql('postgres', on_error_stop => 0); +$a->query_safe("BEGIN; SELECT test_fileops_write('$dir/w1.dat', 0, 'LOSER-A1');" + . " SELECT test_fileops_write('$dir/w2.dat', 0, 'LOSER-A2');"); +my $b = $node->background_psql('postgres', on_error_stop => 0); +$b->query_safe("BEGIN; SELECT test_fileops_write('$dir/w3.dat', 0, 'LOSER-B3');"); +$node->safe_psql('postgres', 'SELECT pg_switch_wal()'); +my $offset = -s $node->logfile; +$node->stop('immediate'); +eval { $a->quit; }; +eval { $b->quit; }; + +chmod(0600, $dir) or die "chmod $dir: $!"; +$node->start; +chmod(0700, $dir) or die "chmod $dir: $!"; + +my $log = slurp_file($node->logfile, $offset); +my $warns = () = $log =~ /FILEOPS UNDO WRITE: could not stat/g; +is($warns, 3, 'three inverses failed with a real EACCES'); +like($log, + qr/UNDO recovery complete: 2 transactions processed, 0 records applied, 3 records could not be applied/, + 'final WARNING counts 3 failed records'); +# Tried once and not retried: the bytes stay as the losers left them. +is(substr(slurp_file("$dir/w1.dat"), 0, 8), 'LOSER-A1', 'w1 not restored'); +is(substr(slurp_file("$dir/w3.dat"), 0, 8), 'LOSER-B3', 'w3 not restored'); +note "RESULT stat_warnings=$warns"; + +$node->stop; +done_testing(); diff --git a/src/test/modules/test_fileops/t/029_fileops_insert_holdoff_gdb.pl b/src/test/modules/test_fileops/t/029_fileops_insert_holdoff_gdb.pl new file mode 100644 index 00000000000..65658c5fa02 --- /dev/null +++ b/src/test/modules/test_fileops/t/029_fileops_insert_holdoff_gdb.pl @@ -0,0 +1,92 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group + +# InsertXactUndoData() holds off a checkpoint's redo-pointer phase between +# inserting an UNDO batch and publishing the chain head. Make that interval +# deterministic with gdb: stop the backend in PublishXactUndoChain(), after +# the batch is in the WAL and before the head is visible, request a +# CHECKPOINT, then kill the backend so the server goes through crash +# recovery. The loser's file must be removed. +# +# Without the hold-off, the checkpoint completes during the stop, its +# XLOG_UNDO_INFLIGHT record lacks this chain, and the batch lies before the +# redo pointer: nothing reverses the create. Another commit flushes the +# create's WAL record first, so this is outside the documented window. +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; +use IPC::Run; +use Time::HiRes qw(usleep); + +plan skip_all => 'gdb not found' unless -x '/usr/bin/gdb'; + +my $node = PostgreSQL::Test::Cluster->new('n'); +$node->init(allows_streaming => 1); +$node->append_conf('postgresql.conf', + "checkpoint_timeout = 1h\nrestart_after_crash = on\n"); +$node->start; +$node->safe_psql('postgres', 'CREATE EXTENSION test_fileops'); + +my $a = $node->background_psql('postgres'); +$a->query_safe('BEGIN; SELECT txid_current();'); +my $pid = $a->query_safe('SELECT pg_backend_pid()'); +chomp $pid; + +my $flag = "$PostgreSQL::Test::Utils::tmp_check/at_publish"; +my $cmds = "$PostgreSQL::Test::Utils::tmp_check/gdb.cmds"; +open(my $fh, '>', $cmds) or die; +print $fh <<"EOF"; +set pagination off +set confirm off +break PublishXactUndoChain +continue +shell touch $flag +shell sleep 6 +kill +quit +EOF +close $fh; +my ($gout, $gerr) = ('', ''); +my $gdb = IPC::Run::start([ '/usr/bin/gdb', '-q', '-batch', '-x', $cmds, '-p', $pid ], + '>', \$gout, '2>', \$gerr); +sleep 2; # let gdb attach and set the breakpoint + +$a->query_until(qr/go/, "\\echo go\nSELECT test_fileops_create('holdoff.dat', 384);\n"); +for (1 .. 100) { last if -e $flag; usleep(100_000); } +ok(-e $flag, 'backend stopped between batch insert and chain publish'); + +# Make the create's WAL and UNDO records durable while the backend is still +# stopped: another transaction's commit flushes everything inserted before +# it. Without this the records may still be in WAL buffers at the kill, and +# the case would fall in the documented window. +my $ins = $node->safe_psql('postgres', 'SELECT pg_current_wal_insert_lsn()'); +$node->safe_psql('postgres', 'CREATE TABLE flush_me (a int)'); +my $durable = $node->safe_psql('postgres', + "SELECT pg_current_wal_flush_lsn() >= '$ins'::pg_lsn"); +is($durable, 't', 'the create is durable before the checkpoint'); + +my $before = $node->safe_psql('postgres', + 'SELECT checkpoint_lsn FROM pg_control_checkpoint()'); +my $c = $node->background_psql('postgres', on_error_stop => 0); +$c->query_until(qr/go/, "\\echo go\nCHECKPOINT;\n"); +my $after = $before; +for (1 .. 40) +{ + $after = $node->safe_psql('postgres', + 'SELECT checkpoint_lsn FROM pg_control_checkpoint()'); + last if $after ne $before; + usleep(100_000); +} +note 'checkpoint completed while the backend was stopped: ' + . ($after ne $before ? 'yes' : 'no'); + +$gdb->finish; +note "gdb: $gout $gerr"; +eval { $a->quit; }; +eval { $c->quit; }; +$node->poll_query_until('postgres', 'SELECT 1', '1'); +ok(!-e $node->data_dir . '/holdoff.dat', 'crash recovery removed the loser file'); + +$node->stop; +done_testing();