From 0f0e7b64ef569a4bc70f6e247bd138141b30f456 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Sat, 12 Sep 2026 00:04:45 +0300 Subject: [PATCH 1/1] Rewind walreceiver flushedUpto when restarting from an earlier LSN. RequestXLogStreaming() only initialized flushedUpto on first start or a timeline change. If recovery later requested the same timeline from an earlier position, WaitForWALToBecomeAvailable() still treated RecPtr < flushedUpto as meaning the WAL was already on disk. Startup then reread the same corrupt bytes, shut down walreceiver, and never issued START_REPLICATION. Reset flushedUpto when the requested start is behind the previous flush pointer so streaming can replace the bad record. pg_last_wal_receive_lsn() can move backward in that case. --- src/backend/replication/walreceiverfuncs.c | 15 ++- src/include/replication/walreceiver.h | 3 +- src/test/recovery/meson.build | 1 + src/test/recovery/t/053_stream_repair.pl | 109 +++++++++++++++++++++ 4 files changed, 124 insertions(+), 4 deletions(-) create mode 100644 src/test/recovery/t/053_stream_repair.pl diff --git a/src/backend/replication/walreceiverfuncs.c b/src/backend/replication/walreceiverfuncs.c index 45b9d4f09f2..742426b6e4e 100644 --- a/src/backend/replication/walreceiverfuncs.c +++ b/src/backend/replication/walreceiverfuncs.c @@ -314,10 +314,19 @@ RequestXLogStreaming(TimeLineID tli, XLogRecPtr recptr, const char *conninfo, walrcv->startTime = now; /* - * If this is the first startup of walreceiver (on this timeline), - * initialize flushedUpto and latestChunkStart to the starting point. + * If this is the first startup of walreceiver (on this timeline), or if + * we are restarting from an earlier position, initialize flushedUpto and + * latestChunkStart to the starting point. + * + * Restarting from an earlier position can happen after recovery detects + * a corrupt record in WAL that was previously streamed. In that case, + * flushedUpto must not make recovery believe that the requested WAL is + * already available, or it will read the same corrupt bytes again and + * shut down walreceiver before it can replace them. */ - if (!XLogRecPtrIsValid(walrcv->receiveStart) || walrcv->receivedTLI != tli) + if (!XLogRecPtrIsValid(walrcv->receiveStart) || + walrcv->receivedTLI != tli || + recptr < walrcv->flushedUpto) { walrcv->flushedUpto = recptr; walrcv->receivedTLI = tli; diff --git a/src/include/replication/walreceiver.h b/src/include/replication/walreceiver.h index 9b9bd916314..6c2837d13ff 100644 --- a/src/include/replication/walreceiver.h +++ b/src/include/replication/walreceiver.h @@ -92,7 +92,8 @@ typedef struct * and receivedTLI is the timeline it came from. At the first startup of * walreceiver, these are set to receiveStart and receiveStartTLI. After * that, walreceiver updates these whenever it flushes the received WAL to - * disk. + * disk. RequestXLogStreaming() also resets them when streaming is + * restarted from an earlier position on the same timeline. */ XLogRecPtr flushedUpto; TimeLineID receivedTLI; diff --git a/src/test/recovery/meson.build b/src/test/recovery/meson.build index 36d789720a3..4c6d5c28db5 100644 --- a/src/test/recovery/meson.build +++ b/src/test/recovery/meson.build @@ -61,6 +61,7 @@ tests += { 't/050_redo_segment_missing.pl', 't/051_effective_wal_level.pl', 't/052_checkpoint_segment_missing.pl', + 't/053_stream_repair.pl', ], }, } diff --git a/src/test/recovery/t/053_stream_repair.pl b/src/test/recovery/t/053_stream_repair.pl new file mode 100644 index 00000000000..1470609f7ef --- /dev/null +++ b/src/test/recovery/t/053_stream_repair.pl @@ -0,0 +1,109 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group + +# Test that streaming replication can replace corrupt WAL that walreceiver +# previously reported as flushed. + +use strict; +use warnings FATAL => 'all'; + +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +# pack() of xl_prev must match on-disk endianness. 'Q' is not available in +# all Perl builds, so split the 64-bit LSN into two 32-bit fields. +my $BIG_ENDIAN = pack('L', 0x12345678) eq pack('N', 0x12345678); + +my $primary = PostgreSQL::Test::Cluster->new('primary'); +$primary->init(allows_streaming => 1); +$primary->append_conf( + 'postgresql.conf', qq( +autovacuum = off +wal_keep_size = 1GB +)); +$primary->start; + +$primary->backup('backup'); + +my $standby = PostgreSQL::Test::Cluster->new('standby'); +$standby->init_from_backup($primary, 'backup', has_streaming => 1); +$standby->append_conf( + 'postgresql.conf', qq( +recovery_prefetch = off +wal_retrieve_retry_interval = '100ms' +)); +$standby->start; + +$primary->wait_for_replay_catchup($standby); + +# Pause before generating the WAL that will be corrupted. Moving to a new +# segment ensures that the startup process has not cached the page containing +# that record. +$standby->safe_psql('postgres', 'SELECT pg_wal_replay_pause()'); +$standby->poll_query_until( + 'postgres', + "SELECT pg_get_wal_replay_pause_state() = 'paused'") + or die "timed out while waiting for recovery to pause"; +$primary->safe_psql('postgres', 'SELECT pg_switch_wal()'); + +# Capture the start of an unreplayed record, then stream well past it so +# walreceiver's flushedUpto is ahead of the later corruption. +my $record_start = int( + $primary->safe_psql( + 'postgres', "SELECT pg_current_wal_insert_lsn() - '0/0'")); +$primary->emit_wal(1024); +$primary->safe_psql('postgres', 'SELECT pg_switch_wal()'); +$primary->emit_wal(8192); +my $target_lsn = $primary->lsn('flush'); +$primary->wait_for_catchup($standby, 'flush', $target_lsn); + +# Stop walreceiver without restarting the postmaster, preserving its +# flushedUpto high-water mark in shared memory. +my $walreceiver_pid = $standby->safe_psql( + 'postgres', 'SELECT pid FROM pg_stat_wal_receiver'); +kill 'TERM', $walreceiver_pid + or die "could not terminate walreceiver $walreceiver_pid: $!"; +$standby->poll_query_until( + 'postgres', + 'SELECT NOT EXISTS (SELECT FROM pg_stat_wal_receiver)') + or die "timed out while waiting for walreceiver to stop"; + +# Overwrite the unreplayed record header with a plausible but incorrect +# prev-link. The primary retains the correct copy. +my $wal_segment_size = int( + $standby->safe_psql( + 'postgres', + "SELECT setting FROM pg_settings WHERE name = 'wal_segment_size'")); +my $tli = int( + $standby->safe_psql( + 'postgres', 'SELECT timeline_id FROM pg_control_checkpoint()')); +# XLogRecord: xl_tot_len, xl_xid, xl_prev, xl_info, xl_rmid, pad, xl_crc. +# xl_tot_len=24 and xl_prev=0xdeadbeef fail ValidXLogRecordHeader. +$standby->write_wal( + $tli, $record_start, + $wal_segment_size, + pack( + 'IIIICCBBI', + 24, 0, + $BIG_ENDIAN ? 0 : 0xdeadbeef, + $BIG_ENDIAN ? 0xdeadbeef : 0, + 0, 0, 0, 0, 0)); + +$standby->safe_psql('postgres', 'SELECT pg_wal_replay_resume()'); + +$standby->poll_query_until( + 'postgres', + "SELECT pg_last_wal_replay_lsn() >= '$target_lsn'") + or die "standby did not replace corrupt WAL and catch up"; + +like( + slurp_file($standby->logfile), + qr/record with incorrect prev-link 0\/DEADBEEF/, + 'standby observed the injected corrupt record'); + +pass('standby replaced corrupt WAL from an earlier streaming position'); + +$standby->stop; +$primary->stop; + +done_testing(); -- 2.50.1 (Apple Git-155)