From 67eb940057474a133dbdb20e62a1e29cf011a650 Mon Sep 17 00:00:00 2001 From: Zsolt Parragi Date: Sat, 15 Aug 2026 13:42:41 +0000 Subject: [PATCH v3] basebackup: do not verify checksums on pages from before enabling Enabling data checksums in a running cluster changes the state to "on" before the checkpoint which flushes the pages the worker rewrote. A base backup which started before that transition absorbs the barrier mid-run and starts verifying pages whose on-disk copies legitimately lack checksums, and whose LSNs predate the backup start, so the LSN check does not skip them either. The backup fails with bogus corruption warnings. The same applies to checksums being disabled and re-enabled while the backup runs: hint bits set while checksums were off reach disk without a checksum update and without moving the page LSN, tripping verification once the re-enabling completes. To fix, verify checksums only while they have been continuously enabled since the checkpoint the backup started from: track the location of the last XLOG2_CHECKSUMS record inserted or replayed, and verify only when the state is "on" and the last change predates the backup start. The starting checkpoint then guarantees that every page flushed before it has a checksum written, and any later change disables verification for the rest of the backup. A standby loses the tracked location when restarting, while pg_control already carries the new state, so it could reach consistency below the record and serve base backups with the location unknown. To prevent this, replaying XLOG2_CHECKSUMS advances minRecoveryPoint to the record, like XLOG_PARAMETER_CHANGE does. The tests hold the enabling between the state change and its final checkpoint with injection points, straddling it with backups on the primary and across a standby crash-restart. --- src/backend/access/transam/xlog.c | 58 +++++ src/backend/backup/basebackup.c | 50 +++- src/include/access/xlog.h | 1 + src/test/modules/test_checksums/meson.build | 2 + .../test_checksums/t/010_backup_straddle.pl | 192 ++++++++++++++++ .../test_checksums/t/011_standby_straddle.pl | 216 ++++++++++++++++++ 6 files changed, 510 insertions(+), 9 deletions(-) create mode 100644 src/test/modules/test_checksums/t/010_backup_straddle.pl create mode 100644 src/test/modules/test_checksums/t/011_standby_straddle.pl diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..d5e7295e1b9 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -560,6 +560,14 @@ typedef struct XLogCtlData uint32 data_checksum_version; slock_t info_lck; /* locks shared variables shown above */ + + /* + * lastChecksumChangeRecPtr points to the end of the last XLOG2_CHECKSUMS + * record inserted or replayed, i.e. the last change of + * data_checksum_version. InvalidXLogRecPtr if the state hasn't changed + * since the server started. + */ + pg_atomic_uint64 lastChecksumChangeRecPtr; } XLogCtlData; /* @@ -4734,6 +4742,23 @@ DataChecksumsNeedVerify(void) return (LocalDataChecksumState == PG_DATA_CHECKSUM_VERSION); } +/* + * GetLastChecksumChangeRecPtr + * Returns the location of the last data checksum state change + * + * Offline state changes by pg_checksums leave no trace here; callers must + * also inspect the current state. + * + * No barrier semantics are needed: pages reach disk under a new checksum + * state only after their writer absorbed the procsignal barrier for the + * change, which is emitted after the new location became visible. + */ +XLogRecPtr +GetLastChecksumChangeRecPtr(void) +{ + return pg_atomic_read_u64(&XLogCtl->lastChecksumChangeRecPtr); +} + /* * SetDataChecksumsOnInProgress * Sets the data checksum state to "inprogress-on" to enable checksums @@ -4844,6 +4869,8 @@ SetDataChecksumsOn(void) MyProc->delayChkptFlags &= ~DELAY_CHKPT_START; END_CRIT_SECTION(); + INJECTION_POINT("datachecksums-on-before-checkpoint", NULL); + RequestCheckpoint(CHECKPOINT_FORCE | CHECKPOINT_WAIT | CHECKPOINT_FAST); WaitForProcSignalBarrier(barrier); } @@ -5434,6 +5461,7 @@ XLOGShmemInit(void *arg) pg_atomic_init_u64(&XLogCtl->logWriteResult, InvalidXLogRecPtr); pg_atomic_init_u64(&XLogCtl->logFlushResult, InvalidXLogRecPtr); pg_atomic_init_u64(&XLogCtl->unloggedLSN, InvalidXLogRecPtr); + pg_atomic_init_u64(&XLogCtl->lastChecksumChangeRecPtr, InvalidXLogRecPtr); } /* @@ -8741,6 +8769,7 @@ XLogChecksums(uint32 new_type) XLogRegisterData((char *) &xlrec, sizeof(xl_checksum_state)); recptr = XLogInsert(RM_XLOG2_ID, XLOG2_CHECKSUMS); + pg_atomic_write_u64(&XLogCtl->lastChecksumChangeRecPtr, recptr); XLogFlush(recptr); } @@ -9242,15 +9271,44 @@ xlog2_redo(XLogReaderState *record) if (info == XLOG2_CHECKSUMS) { xl_checksum_state state; + XLogRecPtr lsn = record->EndRecPtr; memcpy(&state, XLogRecGetData(record), sizeof(xl_checksum_state)); + /* advertise the location before the new state becomes visible */ + pg_atomic_write_u64(&XLogCtl->lastChecksumChangeRecPtr, lsn); + SpinLockAcquire(&XLogCtl->info_lck); XLogCtl->data_checksum_version = state.new_checksum_state; SpinLockRelease(&XLogCtl->info_lck); LWLockAcquire(ControlFileLock, LW_EXCLUSIVE); ControlFile->data_checksum_version = state.new_checksum_state; + + /* + * Update minRecoveryPoint to ensure that if recovery is aborted, we + * recover back up to this point before allowing hot standby again. + * The new state is durable in pg_control while its location is only + * tracked in shared memory; a standby becoming consistent below this + * record would let base backups resume checksum verification with + * the location unknown. The local copies cannot be updated as long + * as crash recovery is happening and we expect all the WAL to be + * replayed. + */ + if (InArchiveRecovery) + { + LocalMinRecoveryPoint = ControlFile->minRecoveryPoint; + LocalMinRecoveryPointTLI = ControlFile->minRecoveryPointTLI; + } + if (XLogRecPtrIsValid(LocalMinRecoveryPoint) && LocalMinRecoveryPoint < lsn) + { + TimeLineID replayTLI; + + (void) GetCurrentReplayRecPtr(&replayTLI); + ControlFile->minRecoveryPoint = lsn; + ControlFile->minRecoveryPointTLI = replayTLI; + } + UpdateControlFile(); LWLockRelease(ControlFileLock); diff --git a/src/backend/backup/basebackup.c b/src/backend/backup/basebackup.c index fe5ce23aaba..3e00cd0dd6e 100644 --- a/src/backend/backup/basebackup.c +++ b/src/backend/backup/basebackup.c @@ -45,6 +45,7 @@ #include "storage/reinit.h" #include "utils/builtins.h" #include "utils/guc.h" +#include "utils/injection_point.h" #include "utils/ps_status.h" #include "utils/relcache.h" #include "utils/resowner.h" @@ -107,6 +108,7 @@ static off_t read_file_data_into_buffer(bbsink *sink, int *checksum_failures); static void push_to_sink(bbsink *sink, pg_checksum_context *checksum_ctx, size_t *bytes_done, void *data, size_t length); +static bool backup_checksums_verifiable(XLogRecPtr start_lsn); static bool verify_page_checksum(Page page, XLogRecPtr start_lsn, BlockNumber blkno, uint16 *expected_checksum); @@ -325,6 +327,12 @@ perform_base_backup(basebackup_options *opt, bbsink *sink, /* notify basebackup sink about start of backup */ bbsink_begin_backup(sink, &state, SINK_BUFFER_LENGTH); + /* + * Allow tests to hold the backup after the starting checkpoint but + * before any file data is sent. + */ + INJECTION_POINT("basebackup-before-send-files", NULL); + /* Send off our tablespaces one by one */ foreach(lc, state.tablespaces) { @@ -1609,13 +1617,14 @@ sendFile(bbsink *sink, const char *readfilename, const char *tarfilename, Assert((sink->bbs_buffer_length % BLCKSZ) == 0); /* - * If we weren't told not to verify checksums, and if checksums are - * enabled for this cluster, and if this is a relation file, then verify - * the checksum. We cannot at this point check if checksums are enabled - * or disabled as that might change, thus we check at each point where we + * Verify checksums unless the client requested otherwise, but only for + * relation files, and only while checksums have been continuously enabled + * since the checkpoint this backup started from. Checksums can still be + * disabled while the backup runs, thus we check at each point where we * could be validating a checksum. */ - if (!noverify_checksums && RelFileNumberIsValid(relfilenumber)) + if (!noverify_checksums && RelFileNumberIsValid(relfilenumber) && + backup_checksums_verifiable(sink->bbs_state->startptr)) verify_checksum = true; /* @@ -1748,7 +1757,9 @@ sendFile(bbsink *sink, const char *readfilename, const char *tarfilename, * If the amount of data we were able to read was not a multiple of * BLCKSZ, we cannot verify checksums, which are block-level. */ - if (verify_checksum && DataChecksumsNeedVerify() && (cnt % BLCKSZ != 0)) + if (verify_checksum && + backup_checksums_verifiable(sink->bbs_state->startptr) && + (cnt % BLCKSZ != 0)) { ereport(WARNING, (errmsg("could not verify checksum in file \"%s\", block " @@ -1876,7 +1887,7 @@ read_file_data_into_buffer(bbsink *sink, const char *readfilename, int fd, * The data checksum state can change at any point, so we need to * re-check before each page. */ - if (!DataChecksumsNeedVerify()) + if (!backup_checksums_verifiable(sink->bbs_state->startptr)) return cnt; page = sink->bbs_buffer + BLCKSZ * i; @@ -1905,7 +1916,7 @@ read_file_data_into_buffer(bbsink *sink, const char *readfilename, int fd, * The data checksum state may also have changed concurrently so check * again. */ - if (!DataChecksumsNeedVerify()) + if (!backup_checksums_verifiable(sink->bbs_state->startptr)) return cnt; reread_cnt = basebackup_read_file(fd, sink->bbs_buffer + BLCKSZ * i, @@ -1996,6 +2007,27 @@ push_to_sink(bbsink *sink, pg_checksum_context *checksum_ctx, } } +/* + * Check whether data checksums can be verified for a backup started at + * start_lsn. + * + * Checksums are verified only while they have been continuously enabled + * since the checkpoint the backup started from: the state must be "on" and + * the last state change must predate the backup start. Such a checkpoint + * guarantees that every page flushed before it has a checksum written. Any + * later state change ends verification for the rest of the backup: pages + * written while checksums were off can lack checksums yet keep LSNs older + * than the backup start, and re-enabling completes before the rewritten + * pages are flushed, so observing the "on" state again is not enough to + * resume. + */ +static bool +backup_checksums_verifiable(XLogRecPtr start_lsn) +{ + return DataChecksumsNeedVerify() && + GetLastChecksumChangeRecPtr() <= start_lsn; +} + /* * Try to verify the checksum for the provided page, if it seems appropriate * to do so. @@ -2021,7 +2053,7 @@ verify_page_checksum(Page page, XLogRecPtr start_lsn, BlockNumber blkno, if (PageIsNew(page) || PageGetLSN(page) >= start_lsn) return true; - if (!DataChecksumsNeedVerify()) + if (!backup_checksums_verifiable(start_lsn)) return true; /* Perform the actual checksum calculation. */ diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h index 4dd98624204..8a22314fb6f 100644 --- a/src/include/access/xlog.h +++ b/src/include/access/xlog.h @@ -249,6 +249,7 @@ extern uint64 GetSystemIdentifier(void); extern char *GetMockAuthenticationNonce(void); extern bool DataChecksumsNeedWrite(void); extern bool DataChecksumsNeedVerify(void); +extern XLogRecPtr GetLastChecksumChangeRecPtr(void); extern bool DataChecksumsOn(void); extern bool DataChecksumsOff(void); extern bool DataChecksumsInProgressOn(void); diff --git a/src/test/modules/test_checksums/meson.build b/src/test/modules/test_checksums/meson.build index 9b1421a9b91..fb7129d796f 100644 --- a/src/test/modules/test_checksums/meson.build +++ b/src/test/modules/test_checksums/meson.build @@ -33,6 +33,8 @@ tests += { 't/007_pgbench_standby.pl', 't/008_pitr.pl', 't/009_fpi.pl', + 't/010_backup_straddle.pl', + 't/011_standby_straddle.pl', ], }, } diff --git a/src/test/modules/test_checksums/t/010_backup_straddle.pl b/src/test/modules/test_checksums/t/010_backup_straddle.pl new file mode 100644 index 00000000000..db50cd81fcd --- /dev/null +++ b/src/test/modules/test_checksums/t/010_backup_straddle.pl @@ -0,0 +1,192 @@ + +# Copyright (c) 2026, PostgreSQL Global Development Group + +# Test base backups running while the data checksum state changes. The +# transition to "on" happens before the checkpoint which flushes the rewritten +# pages, so a backup straddling it reads on-disk pages which legitimately lack +# checksums and carry LSNs older than the backup start. The same applies to +# checksums being disabled and re-enabled while a backup runs: hint bit +# updates made while checksums were off reach disk without a checksum update +# and without moving the page LSN, so once the re-enabling completes the +# backup would resume verification and misjudge those pages until the +# rewritten versions are flushed. +# +# Both scenarios hold the two sides with injection points: the backup after +# its starting checkpoint but before it sends any file data, and the enabling +# after the state changed to "on" but before the checkpoint which flushes the +# rewritten pages. The backup is thus guaranteed to read the stale on-disk +# pages after the state change, without any timing assumptions. + +use strict; +use warnings FATAL => 'all'; + +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; +use IPC::Run; + +use FindBin; +use lib $FindBin::RealBin; + +use DataChecksums::Utils; + +if ($ENV{enable_injection_points} ne 'yes') +{ + plan skip_all => 'Injection points not supported by this build'; +} + +my $node = PostgreSQL::Test::Cluster->new('backup_node'); +$node->init(no_data_checksums => 1, allows_streaming => 1); +# The pages rewritten while enabling must stay dirty in shared buffers until +# the final checkpoint, otherwise they reach disk with checksums on their own +# and nothing is left to misjudge. Autovacuum is disabled so that nothing +# sets hint bits behind our back, and wal_log_hints (implied by +# allows_streaming) must be off so that setting them does not move the page +# LSNs past the backup start. +$node->append_conf('postgresql.conf', 'shared_buffers = 128MB'); +$node->append_conf('postgresql.conf', 'autovacuum = off'); +$node->append_conf('postgresql.conf', 'wal_log_hints = off'); +$node->start; + +$node->safe_psql('postgres', 'CREATE EXTENSION injection_points;'); + +# A body of relation pages for the backup to misjudge. The scan pulls the +# table into shared buffers so that enabling doesn't read it through a ring +# buffer, which would write the pages back out. +$node->safe_psql('postgres', + "CREATE TABLE t AS SELECT generate_series(1,100000) AS a;"); +$node->safe_psql('postgres', "SELECT count(*) FROM t;"); +test_checksum_state($node, 'off'); + +$node->safe_psql('postgres', + "SELECT injection_points_attach('basebackup-before-send-files','wait');"); +$node->safe_psql('postgres', + "SELECT injection_points_attach('datachecksums-on-before-checkpoint','wait');" +); + +my $backupdir = $node->backup_dir . '/straddle'; +my ($out, $err) = ('', ''); +my $backup = IPC::Run::start( + [ + 'pg_basebackup', '-D', $backupdir, + '--wal-method=none', '--no-sync', + '--checkpoint=fast', + '-d', $node->connstr('postgres') + ], + '>', \$out, '2>', \$err, + IPC::Run::timeout(180)); + +$node->wait_for_event('walsender', 'basebackup-before-send-files'); + +# Enable checksums while the backup is held, then release the backup once the +# enabling has reached the "on" state and is held before its checkpoint. +enable_data_checksums($node); +$node->wait_for_event('datachecksums launcher', + 'datachecksums-on-before-checkpoint'); + +$node->safe_psql('postgres', + "SELECT injection_points_wakeup('basebackup-before-send-files');"); +$node->safe_psql('postgres', + "SELECT injection_points_detach('basebackup-before-send-files');"); + +ok($backup->finish, 'backup straddling enable completion succeeds') + or diag("stderr: $err"); + +$node->safe_psql('postgres', + "SELECT injection_points_wakeup('datachecksums-on-before-checkpoint');"); +$node->safe_psql('postgres', + "SELECT injection_points_detach('datachecksums-on-before-checkpoint');"); + +wait_for_checksum_state($node, 'on'); +$node->poll_query_until('postgres', + "SELECT count(*) = 0 FROM pg_catalog.pg_stat_activity " + . "WHERE backend_type = 'datachecksums launcher';"); + +my $result = $node->safe_psql('postgres', + "SELECT coalesce(sum(checksum_failures), 0) FROM pg_catalog.pg_stat_database;" +); +is($result, '0', 'no spurious checksum failures after enable'); + +# A backup started once enabling has completed must verify, and pass +$node->command_ok( + [ + 'pg_basebackup', '-D', $node->backup_dir . '/after_enable', + '--wal-method=none', '--no-sync', '--checkpoint=fast' + ], + 'backup after enable completion succeeds'); + +# Now test a backup which straddles checksums being disabled and re-enabled. +# Recreate the table since the earlier scan set its hint bits and the rewrite +# gave the pages checksums; the new contents are not read here, leaving the +# hint bits unset until checksums are off. +$node->safe_psql('postgres', "DROP TABLE t;"); +$node->safe_psql('postgres', + "CREATE TABLE t AS SELECT generate_series(1,100000) AS a;"); + +$node->safe_psql('postgres', + "SELECT injection_points_attach('basebackup-before-send-files','wait');"); + +$backupdir = $node->backup_dir . '/onoffon'; +($out, $err) = ('', ''); +$backup = IPC::Run::start( + [ + 'pg_basebackup', '-D', $backupdir, + '--wal-method=none', '--no-sync', + '--checkpoint=fast', + '-d', $node->connstr('postgres') + ], + '>', \$out, '2>', \$err, + IPC::Run::timeout(180)); + +$node->wait_for_event('walsender', 'basebackup-before-send-files'); + +disable_data_checksums($node, wait => 1); + +# With checksums off, the scan sets hint bits without WAL logging them, and +# the checkpoint flushes the modified pages without updating their checksums. +# The on-disk pages now carry stale checksums and LSNs older than the backup +# start. +$node->safe_psql('postgres', "SELECT count(*) FROM t;"); +$node->safe_psql('postgres', "CHECKPOINT;"); + +$node->safe_psql('postgres', + "SELECT injection_points_attach('datachecksums-on-before-checkpoint','wait');" +); + +enable_data_checksums($node); +$node->wait_for_event('datachecksums launcher', + 'datachecksums-on-before-checkpoint'); + +$node->safe_psql('postgres', + "SELECT injection_points_wakeup('basebackup-before-send-files');"); +$node->safe_psql('postgres', + "SELECT injection_points_detach('basebackup-before-send-files');"); + +ok($backup->finish, 'backup straddling disable and re-enable succeeds') + or diag("stderr: $err"); + +$node->safe_psql('postgres', + "SELECT injection_points_wakeup('datachecksums-on-before-checkpoint');"); +$node->safe_psql('postgres', + "SELECT injection_points_detach('datachecksums-on-before-checkpoint');"); + +wait_for_checksum_state($node, 'on'); +$node->poll_query_until('postgres', + "SELECT count(*) = 0 FROM pg_catalog.pg_stat_activity " + . "WHERE backend_type = 'datachecksums launcher';"); + +$result = $node->safe_psql('postgres', + "SELECT coalesce(sum(checksum_failures), 0) FROM pg_catalog.pg_stat_database;" +); +is($result, '0', 'no spurious checksum failures after disable and re-enable'); + +# A backup started once re-enabling has completed must verify, and pass +$node->command_ok( + [ + 'pg_basebackup', '-D', $node->backup_dir . '/after_onoffon', + '--wal-method=none', '--no-sync', '--checkpoint=fast' + ], + 'backup after re-enable completion succeeds'); + +$node->stop; +done_testing(); diff --git a/src/test/modules/test_checksums/t/011_standby_straddle.pl b/src/test/modules/test_checksums/t/011_standby_straddle.pl new file mode 100644 index 00000000000..fd6d635c519 --- /dev/null +++ b/src/test/modules/test_checksums/t/011_standby_straddle.pl @@ -0,0 +1,216 @@ + +# Copyright (c) 2026, PostgreSQL Global Development Group + +# Test that a standby does not become available for base backups before it +# has re-replayed the latest data checksum state change after a restart. +# +# When a standby replays XLOG2_CHECKSUMS it writes the new state to +# pg_control, but the location of the change is only tracked in shared +# memory. If the standby restarts before a restartpoint covers the record, +# and reaches consistency below it, base backups would resume checksum +# verification with the change location unknown, while the pages rewritten +# before the change may not have reached disk. The redo routine must +# therefore advance minRecoveryPoint to the record, so that consistency (and +# with it hot standby and base backups) is withheld until the change location +# is known again. +# +# The test holds the enabling on the primary between the state change and its +# final checkpoint, waits for the standby to replay the state change, and +# crashes the standby so that the rewritten pages never reach its disk. WAL +# from the enabling onwards is removed from the standby's pg_wal and +# streaming is disabled, so that replay after the restart stalls below the +# state change record. Restarted this way, the standby must refuse +# connections; once streaming is re-enabled and the record replayed again, +# base backups must succeed without spurious checksum failures. + +use strict; +use warnings FATAL => 'all'; + +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +use FindBin; +use lib $FindBin::RealBin; + +use DataChecksums::Utils; + +if ($ENV{enable_injection_points} ne 'yes') +{ + plan skip_all => 'Injection points not supported by this build'; +} + +my $node_primary = PostgreSQL::Test::Cluster->new('straddle_primary'); +$node_primary->init(no_data_checksums => 1, allows_streaming => 1); +# The pages rewritten while enabling must stay dirty in shared buffers until +# the final checkpoint, and their standby copies must stay dirty as well, so +# that the on-disk pages legitimately lack checksums. wal_log_hints (implied +# by allows_streaming) is turned off to keep the page LSNs put during setup. +$node_primary->append_conf('postgresql.conf', 'shared_buffers = 128MB'); +$node_primary->append_conf('postgresql.conf', 'autovacuum = off'); +$node_primary->append_conf('postgresql.conf', 'wal_log_hints = off'); +$node_primary->start; + +$node_primary->safe_psql('postgres', 'CREATE EXTENSION injection_points;'); + +my $slotname = 'physical_slot'; +$node_primary->safe_psql('postgres', + "SELECT pg_create_physical_replication_slot('$slotname');"); + +# A body of relation pages whose standby copies will lack checksums +$node_primary->safe_psql('postgres', + "CREATE TABLE t AS SELECT generate_series(1,100000) AS a;"); + +my $backup_name = 'straddle_backup'; +$node_primary->backup($backup_name); + +my $node_standby = PostgreSQL::Test::Cluster->new('straddle_standby'); +$node_standby->init_from_backup($node_primary, $backup_name, + has_streaming => 1); +# The background writer must not flush the replayed page rewrites behind our +# back, or nothing is left to protect and minRecoveryPoint could move on its +# own. +$node_standby->append_conf( + 'postgresql.conf', qq[ +primary_slot_name = '$slotname' +bgwriter_lru_maxpages = 0 +]); +$node_standby->start; + +$node_primary->wait_for_catchup($node_standby, 'replay', + $node_primary->lsn('insert')); +test_checksum_state($node_primary, 'off'); +test_checksum_state($node_standby, 'off'); + +# Pin the restartpoint the later base backups will start from: replay a +# primary checkpoint and force a restartpoint on it. No further checkpoint +# record reaches the standby until the enabling is released, so this remains +# the standby's backup starting checkpoint throughout. +$node_primary->safe_psql('postgres', 'CHECKPOINT;'); +$node_primary->wait_for_catchup($node_standby, 'replay', + $node_primary->lsn('insert')); +$node_standby->safe_psql('postgres', 'CHECKPOINT;'); + +# Put everything the enabling writes into fresh WAL segments, so that the +# standby's copies of them can be removed later, and remember where the +# enabling era begins. +$node_primary->safe_psql('postgres', 'SELECT pg_switch_wal();'); +my $enable_start_lsn = + $node_primary->safe_psql('postgres', 'SELECT pg_current_wal_insert_lsn();'); +my $enable_start_seg = $node_primary->safe_psql('postgres', + "SELECT pg_walfile_name('$enable_start_lsn');"); + +# Enable checksums, holding the launcher after the state change but before +# the final checkpoint, so the rewritten pages stay dirty everywhere. +$node_primary->safe_psql('postgres', + "SELECT injection_points_attach('datachecksums-on-before-checkpoint','wait');" +); +enable_data_checksums($node_primary); +$node_primary->wait_for_event('datachecksums launcher', + 'datachecksums-on-before-checkpoint'); + +# The standby has now replayed the state change: its pg_control says "on" +# while the rewritten pages are only dirty in its shared buffers. +$node_primary->wait_for_catchup($node_standby, 'replay', + $node_primary->lsn('insert')); +wait_for_checksum_state($node_standby, 'on'); + +# Crash the standby, losing the dirty rewritten pages. +$node_standby->stop('immediate'); + +# The state change record must have dragged minRecoveryPoint along with it, +# otherwise the standby can become consistent below the record after the +# restart. +my ($stdout, $stderr) = + run_command([ 'pg_controldata', $node_standby->data_dir ]); +my ($min_recovery) = + $stdout =~ /Minimum recovery ending location:\s*([0-9A-F]+\/[0-9A-F]+)/; +die "could not parse pg_controldata output" unless defined $min_recovery; +is( $node_primary->safe_psql( + 'postgres', + "SELECT '$min_recovery'::pg_lsn > '$enable_start_lsn'::pg_lsn;"), + 't', + 'minRecoveryPoint advanced past the checksum state change'); + +# Remove the enabling-era WAL from the standby and cut it off from the +# primary, so that replay after the restart stalls below the state change. +# The replication slot retains the removed segments on the primary. +my $wal_dir = $node_standby->data_dir . '/pg_wal'; +opendir(my $dh, $wal_dir) or die "could not open $wal_dir: $!"; +foreach my $segment (readdir($dh)) +{ + next unless $segment =~ /^[0-9A-F]{24}$/; + next unless $segment ge $enable_start_seg; + unlink("$wal_dir/$segment") + or die "could not unlink $wal_dir/$segment: $!"; +} +closedir($dh); +$node_standby->append_conf('postgresql.conf', "primary_conninfo = ''"); + +# The standby must not reach consistency until it has re-replayed the state +# change, so startup must not complete within the timeout. Reaching hot +# standby below the record would make pg_ctl return success here. +my $started; +{ + local $ENV{PGCTLTIMEOUT} = 10; + $started = $node_standby->start(fail_ok => 1); +} +is($started, 0, + 'standby withholds consistency until the state change is replayed again'); + +my ($ret, $out, $err) = $node_standby->psql('postgres', 'SELECT 1;'); +isnt($ret, 0, 'standby refuses connections while below the state change'); + +# Reconnect the standby; streaming provides the removed WAL again, replay +# passes the state change and the standby becomes consistent. +$node_standby->enable_streaming($node_primary); +$node_standby->reload; +$node_standby->poll_query_until('postgres', 'SELECT true;'); +$node_primary->wait_for_catchup($node_standby, 'replay', + $node_primary->lsn('insert')); + +# The rewritten pages are again only dirty in shared buffers, so the on-disk +# pages still lack checksums. A base backup must skip verification and pass. +$node_standby->command_ok( + [ + 'pg_basebackup', '-D', $node_standby->backup_dir . '/underway', + '--wal-method=none', '--no-sync', '--checkpoint=fast' + ], + 'backup from standby while enabling is underway succeeds'); + +my $result = $node_standby->safe_psql('postgres', + "SELECT coalesce(sum(checksum_failures), 0) FROM pg_catalog.pg_stat_database;" +); +is($result, '0', 'no spurious checksum failures while enabling is underway'); + +# Release the enabling; its final checkpoint flushes the rewritten pages. +$node_primary->safe_psql('postgres', + "SELECT injection_points_wakeup('datachecksums-on-before-checkpoint');"); +$node_primary->safe_psql('postgres', + "SELECT injection_points_detach('datachecksums-on-before-checkpoint');"); +wait_for_checksum_state($node_primary, 'on'); +$node_primary->poll_query_until('postgres', + "SELECT count(*) = 0 FROM pg_catalog.pg_stat_activity " + . "WHERE backend_type = 'datachecksums launcher';"); + +# A restartpoint on the final checkpoint lets verification resume, and a +# backup started from it must again pass. +$node_primary->wait_for_catchup($node_standby, 'replay', + $node_primary->lsn('insert')); +$node_standby->safe_psql('postgres', 'CHECKPOINT;'); + +$node_standby->command_ok( + [ + 'pg_basebackup', '-D', $node_standby->backup_dir . '/after_enable', + '--wal-method=none', '--no-sync', '--checkpoint=fast' + ], + 'backup from standby after enable completion succeeds'); + +$result = $node_standby->safe_psql('postgres', + "SELECT coalesce(sum(checksum_failures), 0) FROM pg_catalog.pg_stat_database;" +); +is($result, '0', 'no spurious checksum failures after enable completion'); + +$node_standby->stop; +$node_primary->stop; +done_testing(); -- 2.54.0