From a25bb13a9911ebd9d5496c5ca311fa6d3d9d5176 Mon Sep 17 00:00:00 2001 From: Zsolt Parragi Date: Fri, 14 Aug 2026 18:40:22 +0000 Subject: [PATCH v4 3/4] pg_rewind: Check the data checksum states of source and target pg_rewind installs the source's control file on the target, but every block it does not copy keeps the target's content. With checksums enabled on the target and disabled on the source, the rewound server claims enabled checksums while the blocks copied from the source have none, and fails checksum verification as soon as it reads them; in the worst case every connection attempt dies on an unverifiable catalog page. Comparing the control files is not enough. Replay on the rewound server resumes from the last common checkpoint and adopts the data checksum state recorded there, so an offline disable on the source after the divergence leaves both control files saying "off" while the rewound server still resumes with verification enabled. Check the state carried by the divergence checkpoint as well, which pg_rewind already reads. Refuse both cases, and an interrupted online transition on either side, same as pg_checksums does. The opposite mismatch, checksums on the source only, is allowed with a warning: recovery under the backup label adopts the divergence state, so the rewound server keeps checksums disabled (or converges through the replayed WAL if the source enabled them online), and no page can fail verification. --- doc/src/sgml/ref/pg_rewind.sgml | 14 ++ src/bin/pg_rewind/parsexlog.c | 4 +- src/bin/pg_rewind/pg_rewind.c | 93 +++++++++++- src/bin/pg_rewind/pg_rewind.h | 1 + src/test/modules/test_checksums/meson.build | 2 + .../test_checksums/t/026_rewind_state.pl | 133 +++++++++++++++++ .../t/027_rewind_standby_target.pl | 141 ++++++++++++++++++ 7 files changed, 386 insertions(+), 2 deletions(-) create mode 100644 src/test/modules/test_checksums/t/026_rewind_state.pl create mode 100644 src/test/modules/test_checksums/t/027_rewind_standby_target.pl diff --git a/doc/src/sgml/ref/pg_rewind.sgml b/doc/src/sgml/ref/pg_rewind.sgml index b95e3868e9d..ac3d0c9328f 100644 --- a/doc/src/sgml/ref/pg_rewind.sgml +++ b/doc/src/sgml/ref/pg_rewind.sgml @@ -124,6 +124,20 @@ PostgreSQL documentation on, but is enabled by default. + + The data checksum states of the source and target server must be + compatible. pg_rewind refuses to run if an + online checksum state transition was interrupted on either server, or + if data checksums are enabled on the target server, or were enabled at + the point of divergence, while the source server runs without them: the + rewound server would fail checksum verification on the blocks copied + from the source. The opposite combination is allowed with a warning; + the rewound server keeps data checksums disabled unless the source + server enabled them online after the point of divergence. See + for how to change the state + consistently across a replication setup. + + Warning: Failures While Rewinding diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c index 023e23b063c..6e87b00f8c2 100644 --- a/src/bin/pg_rewind/parsexlog.c +++ b/src/bin/pg_rewind/parsexlog.c @@ -167,7 +167,8 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex, void findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, XLogRecPtr *lastchkptrec, TimeLineID *lastchkpttli, - XLogRecPtr *lastchkptredo, const char *restoreCommand) + XLogRecPtr *lastchkptredo, uint32 *lastchkptdatachecksums, + const char *restoreCommand) { /* Walk backwards, starting from the given record */ XLogRecord *record; @@ -255,6 +256,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, *lastchkptrec = searchptr; *lastchkpttli = checkPoint.ThisTimeLineID; *lastchkptredo = checkPoint.redo; + *lastchkptdatachecksums = checkPoint.dataChecksumState; break; } diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 2e86fd158d0..6c0f11e01ba 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -144,6 +144,7 @@ main(int argc, char **argv) XLogRecPtr chkptrec; TimeLineID chkpttli; XLogRecPtr chkptredo; + uint32 chkptdatachecksums; TimeLineID source_tli; TimeLineID target_tli; XLogRecPtr target_wal_endrec; @@ -471,10 +472,51 @@ main(int argc, char **argv) keepwal_init(); findLastCheckpoint(datadir_target, divergerec, lastcommontliIndex, - &chkptrec, &chkpttli, &chkptredo, restore_command); + &chkptrec, &chkpttli, &chkptredo, &chkptdatachecksums, + restore_command); pg_log_info("rewinding from last common checkpoint at %X/%08X on timeline %u", LSN_FORMAT_ARGS(chkptrec), chkpttli); + /* + * Replay on the rewound server resumes from the last common checkpoint + * and adopts the data checksum state recorded there, not the state in the + * control file installed from the source. If checksums were enabled at + * the divergence point but the source runs without them, the rewound + * server would verify checksums while the blocks copied from the source + * have none. The control files cannot reveal this: an offline disable on + * the source after the divergence leaves both of them saying "off". + * + * Only the fully enabled state needs checking. An in-progress state at + * the divergence point means a transition was still running there, and + * all of its page rewrites are logged after that point, so replay brings + * the rewound server to whatever state the copied WAL ends in. + */ + if (chkptdatachecksums == PG_DATA_CHECKSUM_VERSION && + ControlFile_source.data_checksum_version != PG_DATA_CHECKSUM_VERSION) + { + pg_log_error("data checksums were enabled at the point of divergence but are disabled on the source server"); + pg_log_error_detail("Blocks copied from the source would have no checksums, but the rewound server would resume with checksum verification enabled."); + pg_log_error_hint("Either enable data checksums on the source server or recreate the target server from a base backup."); + exit(1); + } + + /* + * The same comparison is needed against the target: every block the + * rewind does not copy keeps the target's content. The control files + * cannot reveal this case either, in the other direction: a standby's + * checkpoints are written by its upstream primary, so a standby whose + * checksums were disabled offline still has "on" checkpoints in its WAL, + * while both control files may agree. + */ + if (chkptdatachecksums == PG_DATA_CHECKSUM_VERSION && + ControlFile_target.data_checksum_version != PG_DATA_CHECKSUM_VERSION) + { + pg_log_error("data checksums were enabled at the point of divergence but are disabled on the target server"); + pg_log_error_detail("Blocks kept from the target would have no checksums, but the rewound server would resume with checksum verification enabled."); + pg_log_error_hint("Either enable data checksums on the target server with pg_checksums or recreate the target server from a base backup."); + exit(1); + } + /* Initialize the hash table to track the status of each file */ filehash_init(); @@ -770,6 +812,55 @@ sanityChecks(void) pg_fatal("target server needs to use either data checksums or \"wal_log_hints = on\""); } + /* + * The rewound target keeps its control file fields from the source, but + * every block the rewind does not copy keeps the target's content. If + * checksums are enabled on the target and disabled on the source, the + * result would claim enabled checksums while the blocks copied from the + * source have none, and the target would fail checksum verification as + * soon as it reads them. Refuse that combination, and refuse an + * interrupted online transition on either side, same as pg_checksums. + * + * The opposite mismatch is allowed: recovery under the backup label + * written by pg_rewind adopts the checksum state as of the divergence + * point, so a target whose own checkpoints carry no checksums stays + * without them, or converges through the replayed WAL if the source + * enabled them online. Only warn about it, so that an offline change on + * the source is not overlooked. + * + * That reasoning does not hold when the target is a standby, whose + * checkpoints were written by its upstream primary; see the checks after + * findLastCheckpoint(). + */ + if (ControlFile_target.data_checksum_version == PG_DATA_CHECKSUM_INPROGRESS_ON || + ControlFile_target.data_checksum_version == PG_DATA_CHECKSUM_INPROGRESS_OFF) + { + pg_log_error("an online data checksum state transition was interrupted on the target server"); + pg_log_error_hint("Start the server and shut it down cleanly to reset the state, then retry; on a standby, let replication complete the transition first."); + exit(1); + } + if (ControlFile_source.data_checksum_version == PG_DATA_CHECKSUM_INPROGRESS_ON || + ControlFile_source.data_checksum_version == PG_DATA_CHECKSUM_INPROGRESS_OFF) + { + pg_log_error("an online data checksum state transition is incomplete on the source server"); + pg_log_error_hint("Let the transition complete, or reset the state with a clean restart, then retry."); + exit(1); + } + if (ControlFile_target.data_checksum_version == PG_DATA_CHECKSUM_VERSION && + ControlFile_source.data_checksum_version == PG_DATA_CHECKSUM_OFF) + { + pg_log_error("data checksums are enabled on the target server but disabled on the source server"); + pg_log_error_detail("Blocks copied from the source would have no checksums, and the target would fail checksum verification after the rewind."); + pg_log_error_hint("Either disable data checksums on the target server or enable them on the source server, then retry."); + exit(1); + } + if (ControlFile_target.data_checksum_version == PG_DATA_CHECKSUM_OFF && + ControlFile_source.data_checksum_version == PG_DATA_CHECKSUM_VERSION) + { + pg_log_warning("data checksums are disabled on the target server but enabled on the source server"); + pg_log_warning_detail("The rewound server will keep data checksums disabled unless the source server enabled them online after the point of divergence."); + } + /* * Target cluster better not be running. This doesn't guard against * someone starting the cluster concurrently. Also, this is probably more diff --git a/src/bin/pg_rewind/pg_rewind.h b/src/bin/pg_rewind/pg_rewind.h index 9a981f7f246..9187c3bf72f 100644 --- a/src/bin/pg_rewind/pg_rewind.h +++ b/src/bin/pg_rewind/pg_rewind.h @@ -39,6 +39,7 @@ extern void findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex, XLogRecPtr *lastchkptrec, TimeLineID *lastchkpttli, XLogRecPtr *lastchkptredo, + uint32 *lastchkptdatachecksums, const char *restoreCommand); extern XLogRecPtr readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex, const char *restoreCommand); diff --git a/src/test/modules/test_checksums/meson.build b/src/test/modules/test_checksums/meson.build index 0269d4c6e38..406cc946b0d 100644 --- a/src/test/modules/test_checksums/meson.build +++ b/src/test/modules/test_checksums/meson.build @@ -49,6 +49,8 @@ tests += { 't/023_concurrent_checkpoint_enable.pl', 't/024_enable_crash_after_checkpoint.pl', 't/025_cascade_divergence.pl', + 't/026_rewind_state.pl', + 't/027_rewind_standby_target.pl', ], }, } diff --git a/src/test/modules/test_checksums/t/026_rewind_state.pl b/src/test/modules/test_checksums/t/026_rewind_state.pl new file mode 100644 index 00000000000..b7adc968c4d --- /dev/null +++ b/src/test/modules/test_checksums/t/026_rewind_state.pl @@ -0,0 +1,133 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group + +# pg_rewind checks the data checksum states of source and target. +# Checksums enabled on the target, or at the point of divergence, with +# a source running without them are refused: the rewound server would +# verify checksums on blocks copied from a source that has none. The +# opposite mismatch only warns; replay keeps the target's own state. +use strict; +use warnings FATAL => 'all'; + +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +use FindBin; +use lib $FindBin::RealBin; + +use DataChecksums::Utils; + +# Scenarios 1 and 2: checksums on from initdb. wal_log_hints keeps the +# target eligible for pg_rewind after checksums are disabled on it. +my $node_a = PostgreSQL::Test::Cluster->new('node_a'); +$node_a->init(allows_streaming => 1); +$node_a->append_conf( + 'postgresql.conf', qq[ +autovacuum = off +wal_keep_size = '1GB' +wal_log_hints = on +]); +$node_a->start; +$node_a->safe_psql('postgres', + "CREATE TABLE t AS SELECT generate_series(1,10000) AS a;"); + +$node_a->backup('backup'); +my $node_b = PostgreSQL::Test::Cluster->new('node_b'); +$node_b->init_from_backup($node_a, 'backup', has_streaming => 1); +$node_b->start; +$node_a->wait_for_catchup($node_b); + +# Failover to B, and divergence on A. +$node_b->promote; +$node_b->safe_psql('postgres', "INSERT INTO t VALUES (0);"); +$node_a->safe_psql('postgres', "INSERT INTO t VALUES (-1);"); +$node_a->stop('fast'); + +# Offline disable on the source only: enabled target, disabled source. +$node_b->stop; +$node_b->checksum_disable_offline; +$node_b->start; +test_checksum_state($node_b, 'off'); + +my @rewind_cmd = ( + 'pg_rewind', + '--target-pgdata' => $node_a->data_dir, + '--source-server' => $node_b->connstr('postgres')); + +command_fails_like( + \@rewind_cmd, + qr/data checksums are enabled on the target server but disabled on the source server/, + 'refuses an enabled target with a disabled source'); + +# The refusal happens before any modification: the target still starts +# on its own timeline. +$node_a->start; +is($node_a->safe_psql('postgres', "SELECT count(*) FROM t;"), + '10001', 'target untouched by the refused rewind'); +$node_a->stop('fast'); + +# Scenario 2: disabling the target too makes the control files match, +# but checksums were still enabled at the point of divergence, which is +# the state replay on the rewound server would resume with. +$node_a->checksum_disable_offline; +command_fails_like( + \@rewind_cmd, + qr/data checksums were enabled at the point of divergence but are disabled on the source server/, + 'refuses when checksums were enabled at the point of divergence'); + +# Scenario 3: fresh pair without checksums; offline enable on the +# source only. Allowed with a warning, and the rewound server keeps +# checksums disabled. +my $node_c = PostgreSQL::Test::Cluster->new('node_c'); +$node_c->init(allows_streaming => 1, no_data_checksums => 1); +$node_c->append_conf( + 'postgresql.conf', qq[ +autovacuum = off +wal_keep_size = '1GB' +wal_log_hints = on +]); +$node_c->start; +$node_c->safe_psql('postgres', + "CREATE TABLE t AS SELECT generate_series(1,10000) AS a;"); + +$node_c->backup('backup'); +my $node_d = PostgreSQL::Test::Cluster->new('node_d'); +$node_d->init_from_backup($node_c, 'backup', has_streaming => 1); +$node_d->start; +$node_c->wait_for_catchup($node_d); + +$node_d->promote; +$node_d->safe_psql('postgres', "INSERT INTO t VALUES (0);"); +$node_c->safe_psql('postgres', "INSERT INTO t VALUES (-1);"); +$node_c->stop('fast'); + +$node_d->stop; +$node_d->checksum_enable_offline; +$node_d->start; +test_checksum_state($node_d, 'on'); + +my ($stdout, $stderr) = run_command( + [ + 'pg_rewind', + '--target-pgdata' => $node_c->data_dir, + '--source-server' => $node_d->connstr('postgres'), + ]); +like( + $stderr, + qr/data checksums are disabled on the target server but enabled on the source server/, + 'warns for a disabled target with an enabled source'); +like($stderr, qr/Done!/, 'rewind completed despite the warning'); + +# The rewound server follows D and keeps its own state. +$node_c->append_conf('postgresql.conf', 'port = ' . $node_c->port); +$node_c->enable_streaming($node_d); +$node_c->set_standby_mode; +$node_c->start; +$node_d->wait_for_catchup($node_c); +is($node_c->safe_psql('postgres', "SELECT count(*) FROM t;"), + '10001', 'rewound server readable as a standby'); +test_checksum_state($node_c, 'off'); + +$node_c->stop; +$node_d->stop; +done_testing(); diff --git a/src/test/modules/test_checksums/t/027_rewind_standby_target.pl b/src/test/modules/test_checksums/t/027_rewind_standby_target.pl new file mode 100644 index 00000000000..3f5a3c6be8b --- /dev/null +++ b/src/test/modules/test_checksums/t/027_rewind_standby_target.pl @@ -0,0 +1,141 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group + +# Test that pg_rewind compares the data checksum state at the point of +# divergence against the target as well as the source. +# +# A standby's checkpoints are written by its upstream primary, so a standby +# whose checksums were disabled offline still has "on" checkpoints in its +# WAL. Rewinding it onto a promoted sibling passes every control file +# check (target off + source on only warns), yet recovery after the rewind +# adopts the divergence checkpoint's state and the rewound server would +# verify checksums over the pages it wrote while it was locally "off". +# pg_rewind must refuse; after an offline enable on the target, which +# rewrites all of its pages, the rewind goes through. +use strict; +use warnings FATAL => 'all'; + +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +use FindBin; +use lib $FindBin::RealBin; + +use DataChecksums::Utils; + +my $primary = PostgreSQL::Test::Cluster->new('primary'); +$primary->init(allows_streaming => 1); +$primary->append_conf( + 'postgresql.conf', qq[ +autovacuum = off +wal_keep_size = '1GB' +wal_log_hints = on +]); +$primary->start; +$primary->safe_psql('postgres', + "CREATE TABLE t0 AS SELECT generate_series(1,10000) AS a;"); + +$primary->backup('backup'); + +my $lagging = PostgreSQL::Test::Cluster->new('lagging'); +$lagging->init_from_backup($primary, 'backup', has_streaming => 1); +$lagging->start; + +my $failover = PostgreSQL::Test::Cluster->new('failover'); +$failover->init_from_backup($primary, 'backup', has_streaming => 1); +$failover->start; + +$primary->wait_for_catchup($lagging); +$primary->wait_for_catchup($failover); + +test_checksum_state($primary, 'on'); +test_checksum_state($lagging, 'on'); +test_checksum_state($failover, 'on'); + +# Offline-disable checksums on the lagging standby only. This is a +# divergence 012_offline_standby.pl declares survivable: the standby keeps +# its own state, warns, and stays readable. +$lagging->stop; +system_or_bail('pg_checksums', '--disable', '--pgdata', $lagging->data_dir); +$lagging->start; +test_checksum_state($lagging, 'off'); + +# Give the lagging standby plenty of pages to write while it is locally +# "off", and force a restartpoint so they reach disk without checksums. +# The other standby replays the same WAL with checksums on. +$primary->safe_psql('postgres', + "CREATE TABLE t1 AS SELECT generate_series(1,200000) AS a;"); +$primary->safe_psql('postgres', "CHECKPOINT;"); +$primary->wait_for_catchup($lagging); +$primary->wait_for_catchup($failover); +$lagging->safe_psql('postgres', "CHECKPOINT;"); + +# Take the "failover" standby out of the picture. It is promoted from +# this position later, so everything replayed from here on is the part of +# the WAL that diverges and that pg_rewind will roll back. t1 is already +# replayed everywhere, so its blocks are *not* rolled back. +$failover->stop; + +$primary->safe_psql('postgres', + "CREATE TABLE t2 AS SELECT generate_series(1,1000) AS a;"); +$primary->wait_for_catchup($lagging); + +# Failover: promote the other standby, which forks the timeline behind the +# replay position of the lagging standby. +$primary->stop('fast'); +$failover->start; +$failover->promote; +$failover->safe_psql('postgres', + "CREATE TABLE t3 AS SELECT generate_series(1,1000) AS a;"); +$failover->safe_psql('postgres', "CHECKPOINT;"); + +# Re-attaching the lagging standby with pg_rewind must be refused: the +# divergence checkpoint says "on" while the target is "off", so the rewound +# server would verify checksums over the blocks it keeps. +$lagging->stop; + +command_fails_like( + [ + 'pg_rewind', + '--target-pgdata' => $lagging->data_dir, + '--source-server' => $failover->connstr('postgres'), + ], + qr/data checksums were enabled at the point of divergence but are disabled on the target server/, + 'pg_rewind refuses a target whose divergence checkpoint has checksums'); + +# Enabling checksums offline on the target rewrites all of its pages, after +# which the rewind is safe. +system_or_bail('pg_checksums', '--enable', '--pgdata', $lagging->data_dir); + +my ($stdout, $stderr) = run_command( + [ + 'pg_rewind', + '--target-pgdata' => $lagging->data_dir, + '--source-server' => $failover->connstr('postgres'), + ]); +like($stderr, qr/Done!/, 'pg_rewind completes after the offline enable'); + +$lagging->append_conf('postgresql.conf', 'port = ' . $lagging->port); +$lagging->enable_streaming($failover); +$lagging->set_standby_mode; +$lagging->start; + +my ($rc, $out, $err) = + $lagging->psql('postgres', + "SELECT setting FROM pg_settings " . "WHERE name = 'data_checksums';"); +is($rc, 0, 'rewound standby accepts connections') or diag("stderr: $err"); +is($out, 'on', 'rewound standby resumes with checksums on'); + +($rc, $out, $err) = $lagging->psql('postgres', "SELECT count(*) FROM t1;"); +is($rc, 0, 'blocks kept from the target stay readable') + or diag("stderr: $err"); + +my $log = PostgreSQL::Test::Utils::slurp_file($lagging->logfile); +unlike( + $log, + qr/page verification failed/, + 'no checksum verification failures on the rewound standby'); + +$lagging->stop('immediate'); +$failover->stop; +done_testing(); -- 2.39.3 (Apple Git-146)