From e800e8488b5eef070771425a8d10da1606f17c33 Mon Sep 17 00:00:00 2001 From: Manu Date: Tue, 3 Nov 2026 02:31:11 -0300 Subject: [PATCH 3/3] Summarize only replayed WAL on a standby During recovery, GetLatestLSN() let the WAL summarizer read up to the walreceiver's flush position whenever that was ahead of replay. That WAL is not part of the server's history until it is replayed: recovery can stop at a recovery target before reaching it, and if the server is then promoted, the summaries already written for the old timeline cover WAL from a history the server abandoned. Incremental backups trust those summaries; a relation truncation recorded there discards the references to blocks modified before the switch point, and the restored blocks silently come from the prior backup. Summarize only up to the replay position during recovery. Backups taken on a standby start at a restartpoint, which is never ahead of replay, so they need no summaries beyond it. With this, a standby promoted at a recovery target no longer writes old-timeline summaries that cross the switch point. Such a summary can still be on disk from before a minor update, and the check added by the previous commit still refuses it. The summarizer is no longer ahead of replay at promotion either, so the check added by "Fix assertion failure in WAL summarizer after promotion" becomes defensive, and t/004_promote_past_target.pl now promotes while the summarizer waits at replay. t/014_summary_past_switch.pl now checks that the summarizer stops at replay and that the incremental backups restore the promoted server's rows; t/015_truncation_after_target.pl, from shihao zhong's report, uses a recovery target that is not at a checkpoint's redo pointer. Reported-by: shihao zhong Co-authored-by: shihao zhong Tested-by: Sai Pranav Discussion: https://postgr.es/m/CAHGQGwFbpzR2Wkrku9J8w6E+-rzD6Ovazs8n_kxvzfccfnJ0tg@mail.gmail.com --- src/backend/backup/basebackup_incremental.c | 14 ++-- src/backend/postmaster/walsummarizer.c | 35 +++----- src/bin/pg_combinebackup/meson.build | 1 + .../t/015_summary_past_switch.pl | 54 +++++++----- .../t/016_truncation_after_target.pl | 82 +++++++++++++++++++ .../t/004_promote_past_target.pl | 51 ++++++++---- 6 files changed, 172 insertions(+), 65 deletions(-) create mode 100644 src/bin/pg_combinebackup/t/016_truncation_after_target.pl diff --git a/src/backend/backup/basebackup_incremental.c b/src/backend/backup/basebackup_incremental.c index 50b5fcf5ff2..8222aab27c2 100644 --- a/src/backend/backup/basebackup_incremental.c +++ b/src/backend/backup/basebackup_incremental.c @@ -570,12 +570,14 @@ PrepareForIncrementalBackup(IncrementalBackupInfo *ib, /* * A summary that ends after the point where we switched away from * this timeline also covers WAL that this server may never have - * replayed: a standby summarizes the WAL it has received, ahead of - * replay, and recovery can stop at a recovery target before the end - * of it. Block references from that WAL only make us copy more than - * needed, but a relation truncation recorded there would discard the - * references to blocks modified before the switch. Remember these - * summaries, so that the merge below can refuse such a truncation. + * replayed. A standby now summarizes only replayed WAL, but a + * summary written by an older release, which followed the walreceiver + * ahead of replay, may still be on disk, and recovery can have + * stopped at a recovery target before the end of it. Block + * references from that WAL only make us copy more than needed, but a + * relation truncation recorded there would discard the references to + * blocks modified before the switch. Remember these summaries, so + * that the merge below can refuse such a truncation. */ if (tle->tli != backup_state->starttli) { diff --git a/src/backend/postmaster/walsummarizer.c b/src/backend/postmaster/walsummarizer.c index 25ca511bc01..a1ded62c3e9 100644 --- a/src/backend/postmaster/walsummarizer.c +++ b/src/backend/postmaster/walsummarizer.c @@ -841,10 +841,6 @@ GetLatestLSN(TimeLineID *tli) } else { - XLogRecPtr flush_lsn; - TimeLineID flush_tli; - XLogRecPtr replay_lsn; - TimeLineID replay_tli; TimeLineID insert_tli; /* @@ -863,26 +859,19 @@ GetLatestLSN(TimeLineID *tli) } /* - * What we really want to know is how much WAL has been flushed to - * disk, but the only flush position available is the one provided by - * the walreceiver, which may not be running, because this could be - * crash recovery or recovery via restore_command. So use either the - * WAL receiver's flush position or the replay position, whichever is - * further ahead, on the theory that if the WAL has been replayed then - * it must also have been flushed to disk. + * Otherwise, summarize only WAL that has been replayed. A + * walreceiver may have flushed WAL beyond that point, but it is not + * yet part of this server's history: recovery may stop at a recovery + * target before reaching it, and if the server is then promoted, that + * WAL belongs to a timeline this server abandoned. A summary + * covering it would describe changes, such as relation truncations, + * that never happened here, and incremental backups would trust it. + * + * Backups taken during recovery start at a restartpoint, which is + * never ahead of replay, so they don't need summaries of anything + * further. */ - flush_lsn = GetWalRcvFlushRecPtr(NULL, &flush_tli); - replay_lsn = GetXLogReplayRecPtr(&replay_tli); - if (flush_lsn > replay_lsn) - { - *tli = flush_tli; - return flush_lsn; - } - else - { - *tli = replay_tli; - return replay_lsn; - } + return GetXLogReplayRecPtr(tli); } } diff --git a/src/bin/pg_combinebackup/meson.build b/src/bin/pg_combinebackup/meson.build index b3d28ca866c..e551636f71e 100644 --- a/src/bin/pg_combinebackup/meson.build +++ b/src/bin/pg_combinebackup/meson.build @@ -43,6 +43,7 @@ tests += { 't/013_summary_order.pl', 't/014_vm_truncation.pl', 't/015_summary_past_switch.pl', + 't/016_truncation_after_target.pl', ], } } diff --git a/src/bin/pg_combinebackup/t/015_summary_past_switch.pl b/src/bin/pg_combinebackup/t/015_summary_past_switch.pl index b1c0bf8c171..5af1a85e5e2 100644 --- a/src/bin/pg_combinebackup/t/015_summary_past_switch.pl +++ b/src/bin/pg_combinebackup/t/015_summary_past_switch.pl @@ -1,10 +1,10 @@ # Copyright (c) 2026, PostgreSQL Global Development Group # -# A streaming standby summarizes the WAL it receives, ahead of replay. When -# it is promoted at a recovery target inside a WAL summary, the old-timeline -# summary covering the target also covers WAL that the promoted server never -# replayed. Block references from that WAL are harmless, but a truncation -# recorded there must not be applied to an incremental backup. +# A streaming standby promoted at a recovery target inside a WAL summary, +# after receiving WAL past the target, including a truncation. A standby only +# summarizes replayed WAL, so the old-timeline summaries end at the switch +# point, and incremental backups after promotion restore the promoted +# server's rows. use strict; use warnings FATAL => 'all'; @@ -57,8 +57,8 @@ $primary->safe_psql('postgres', 'CHECKPOINT'); my $end_lsn = $primary->safe_psql('postgres', 'SELECT redo_lsn FROM pg_control_checkpoint()'); -# Start a standby that pauses at the target and has summarized everything -# the primary wrote, then promote it. +# Start a standby that pauses at the target and has received everything the +# primary wrote, then promote it. sub promoted_standby { my ($name, $target) = @_; @@ -73,11 +73,17 @@ EOF $node->poll_query_until('postgres', "SELECT pg_get_wal_replay_pause_state() = 'paused'") or die "timed out waiting for $name to pause"; - $node->poll_query_until('postgres', <= '$end_lsn') -EOF - or die "timed out waiting for $name to summarize up to $end_lsn"; + $node->poll_query_until('postgres', + "SELECT pg_last_wal_receive_lsn() >= '$end_lsn'") + or die "timed out waiting for $name to receive up to $end_lsn"; + $node->poll_query_until('postgres', + "SELECT pending_lsn >= pg_last_wal_replay_lsn() FROM pg_get_wal_summarizer_state()") + or die "timed out waiting for $name to summarize up to replay"; + is( $node->safe_psql( + 'postgres', + "SELECT pending_lsn <= pg_last_wal_replay_lsn() FROM pg_get_wal_summarizer_state()"), + 't', + "$name: the summarizer does not read past replay"); $node->promote; return $node; } @@ -90,8 +96,7 @@ $primary->stop; # hint bits would WAL-log their blocks again on the new timeline. my $contents_sql = "SELECT id, right(pad, 1) FROM t ORDER BY id"; -# First standby: the summary past the switch only adds block references, so -# the incremental backup works and restores the promoted server's rows. +# First standby: only new rows follow its target. $standby1->command_ok( [ 'pg_basebackup', '--no-sync', @@ -114,17 +119,28 @@ is($restored->safe_psql('postgres', $contents_sql), $expected1, 'restored rows match the first standby'); $restored->stop; -# Second standby: the summary past the switch records the truncation, which -# never happened on this server, so the incremental backup must refuse it. -$standby2->command_fails_like( +# Second standby: the truncation after its target was received but never +# replayed or summarized, so the incremental backup restores the rows. +$standby2->command_ok( [ 'pg_basebackup', '--no-sync', '--pgdata' => $primary->backup_dir . '/incr2', '--checkpoint' => 'fast', '--incremental' => "$full_path/backup_manifest", ], - qr/cannot use WAL summary file .* for an incremental backup/, - 'incremental backup refuses a truncation past the switch point'); + 'incremental backup when a truncation follows the target'); +my $expected2 = "1|x\n2|x\n3|o\n4|o\n5|x"; +is($standby2->safe_psql('postgres', $contents_sql), + $expected2, 'second standby has the rows replayed before the target'); $standby2->stop; +my $restored2 = PostgreSQL::Test::Cluster->new('restored2'); +$restored2->init_from_backup( + $primary, 'incr2', + combine_with_prior => ['full'], + combine_mode => $mode); +$restored2->start; +is($restored2->safe_psql('postgres', $contents_sql), + $expected2, 'restored rows match the second standby'); +$restored2->stop; done_testing(); diff --git a/src/bin/pg_combinebackup/t/016_truncation_after_target.pl b/src/bin/pg_combinebackup/t/016_truncation_after_target.pl new file mode 100644 index 00000000000..c30a7e88199 --- /dev/null +++ b/src/bin/pg_combinebackup/t/016_truncation_after_target.pl @@ -0,0 +1,82 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group +# +# A standby can summarize WAL past its recovery target. If that WAL +# truncates a table, an incremental backup taken after promotion must still +# include blocks modified before the target. + +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $primary = PostgreSQL::Test::Cluster->new('primary'); +$primary->init(allows_streaming => 1); +$primary->append_conf('postgresql.conf', <start; + +# One row per block. +my $pad_size = $primary->safe_psql('postgres', 'SHOW block_size') / 2; +$primary->safe_psql('postgres', <backup('full'); +my $full_path = $primary->backup_dir . '/full'; + +my $standby = PostgreSQL::Test::Cluster->new('standby'); +$standby->init_from_backup($primary, 'full', has_streaming => 1); +$standby->append_conf('postgresql.conf', <safe_psql('postgres', 'UPDATE t SET v = 1 WHERE id = 4'); +my $target = + $primary->safe_psql('postgres', 'SELECT pg_current_wal_insert_lsn()'); +$standby->append_conf('postgresql.conf', "recovery_target_lsn = '$target'\n"); +$standby->start; + +# Truncate the table on the old timeline only. +$primary->safe_psql('postgres', 'DELETE FROM t WHERE id > 1'); +$primary->safe_psql('postgres', 'VACUUM t'); +$primary->safe_psql('postgres', 'SELECT pg_switch_wal()'); +$primary->safe_psql('postgres', 'CHECKPOINT'); + +$standby->poll_query_until('postgres', + "SELECT pg_get_wal_replay_pause_state() = 'paused'") + or die "standby did not pause at the target"; + +# Give the summarizer time to get past the target, if it is going to. +sleep(5); + +$primary->stop; +$standby->safe_psql('postgres', 'SELECT pg_promote()'); +$standby->poll_query_until('postgres', 'SELECT NOT pg_is_in_recovery()') + or die "standby was not promoted"; + +# Back up before reading t, so that setting hint bits can't hide the problem. +$standby->command_ok( + [ + 'pg_basebackup', '--no-sync', + '--pgdata' => $primary->backup_dir . '/incr', + '--checkpoint' => 'fast', + '--incremental' => "$full_path/backup_manifest", + ], + 'incremental backup after promotion'); +$standby->stop; + +my $restored = PostgreSQL::Test::Cluster->new('restored'); +$restored->init_from_backup($primary, 'incr', combine_with_prior => ['full']); +$restored->start; +is($restored->safe_psql('postgres', 'SELECT id, v FROM t ORDER BY id'), + "1|0\n2|0\n3|0\n4|1", 'restored backup has the update'); +$restored->stop; + +done_testing(); diff --git a/src/bin/pg_walsummary/t/004_promote_past_target.pl b/src/bin/pg_walsummary/t/004_promote_past_target.pl index 7672183c730..87314881cda 100644 --- a/src/bin/pg_walsummary/t/004_promote_past_target.pl +++ b/src/bin/pg_walsummary/t/004_promote_past_target.pl @@ -1,11 +1,13 @@ # Copyright (c) 2021-2026, PostgreSQL Global Development Group # -# A standby that stops at a recovery target can have summarized WAL beyond -# that target, because the WAL summarizer reads whatever the walreceiver has -# flushed. When such a standby is promoted, the old timeline ends at a point -# the summarizer has already passed. The summarizer must back up to the switch -# point and continue on the new timeline, rather than report a summary that -# ends before it started (which used to trip an assertion). +# Promote a standby that stopped at a recovery target while its WAL +# summarizer was waiting for more WAL. The summarizer must notice that the +# old timeline became historic and continue on the new timeline from the +# switch point. A standby summarizes only replayed WAL, so the summarizer +# waits at the switch point here, although the standby has received WAL past +# it. When a standby summarized what the walreceiver had flushed, the switch +# point could precede the start of the summary being written, which used to +# trip an assertion. use strict; use warnings FATAL => 'all'; @@ -24,13 +26,14 @@ $primary->start; $primary->safe_psql('postgres', 'CREATE TABLE t (i int)'); $primary->backup('backup'); -# The recovery target is the redo pointer of a checkpoint, so that the -# standby's summarizer completes a summary that starts exactly there before -# promotion; a second checkpoint lets it summarize past the target. +# The recovery target is the redo pointer of a checkpoint; the standby also +# receives a second checkpoint past it. $primary->safe_psql('postgres', 'INSERT INTO t VALUES (1); CHECKPOINT;'); my $target_lsn = $primary->safe_psql('postgres', 'SELECT redo_lsn FROM pg_control_checkpoint()'); $primary->safe_psql('postgres', 'INSERT INTO t VALUES (2); CHECKPOINT;'); +my $end_lsn = + $primary->safe_psql('postgres', 'SELECT pg_current_wal_flush_lsn()'); my $standby = PostgreSQL::Test::Cluster->new('standby'); $standby->init_from_backup($primary, 'backup', has_streaming => 1); @@ -42,33 +45,47 @@ log_min_messages = debug1 EOM $standby->start; -# Wait until replay is paused at the target, the summarizer has summarized -# past it on timeline 1, and the summarizer is waiting for more WAL. +# Wait until replay is paused at the target, the standby has received the +# WAL past it, and the summarizer has reached replay and is waiting for more +# WAL. $standby->poll_query_until('postgres', "SELECT pg_get_wal_replay_pause_state() = 'paused'") or die "timed out waiting for recovery to pause"; -$standby->poll_query_until('postgres', <poll_query_until('postgres', + "SELECT pg_last_wal_receive_lsn() >= '$end_lsn'") + or die "timed out waiting for the standby to receive up to $end_lsn"; +$standby->poll_query_until('postgres', + "SELECT pending_lsn >= pg_last_wal_replay_lsn() FROM pg_get_wal_summarizer_state()" +) or die "timed out waiting for summarization to reach replay"; $standby->poll_query_until('postgres', <safe_psql( + 'postgres', + "SELECT pending_lsn <= pg_last_wal_replay_lsn() FROM pg_get_wal_summarizer_state()" + ), + 't', + 'the summarizer does not read past replay'); my $log_offset = -s $standby->logfile; $standby->promote; $standby->safe_psql('postgres', 'CHECKPOINT'); -# Summarization must resume on timeline 2 exactly at the switch point. +# Summarization must end on timeline 1 and resume on timeline 2 exactly at +# the switch point. my $result = $standby->poll_query_until('postgres', <safe_psql( + 'postgres', + "SELECT max(end_lsn) FROM pg_available_wal_summaries() WHERE tli = 1"), + $target_lsn, + "timeline 1 summaries end at $target_lsn"); ok( !$standby->log_contains( qr/WAL summarizer process .* was terminated by signal/, $log_offset), -- 2.37.1 (Apple Git-137.1)