From 2097faf164d5f57222187e5e644bd53e0cb9bbb1 Mon Sep 17 00:00:00 2001 From: Zsolt Parragi Date: Fri, 14 Aug 2026 17:06:02 +0000 Subject: [PATCH v4 1/4] Do not adopt data checksum state from another node during replay Offline pg_checksums changes are local to one node, but replay adopted the data checksum state carried by checkpoint records unconditionally. After an offline enable on the primary, a standby whose pages were never rewritten started verifying checksums it does not have; after an offline change on a standby, the next replayed checkpoint silently reverted it. Instead, cross-check the replayed state against the local one and warn once per divergent value. When the states match again, say so in the log and re-arm the warning. The control file now tracks this node's state alone, which makes the moment it is written out part of the design: it may only claim "on" once every page on disk carries a checksum, or a crash-restart would verify pages that were flushed before the transition rewrote them. XLOG2_CHECKSUMS replay therefore persists every state but "on" as soon as it is replayed, none of them verifying anything, and leaves "on" to the flush of the next restartpoint. A restartpoint only persists a state its flush ran under from beginning to end, a promotion writes a full checkpoint instead of an end-of-recovery record while the state is still unpersisted, and a shutdown restartpoint with no new checkpoint record to work from flushes before catching the control file up rather than leaving it behind. Enabling checksums on the primary defers the same write to after the checkpoint that flushes the rewritten pages: the ones the worker found in shared buffers do not go out through its ring buffer. A checkpoint persists the state its own redo point ran under, under the same rule a restartpoint follows, since the checkpoint that licenses "on" also moves the redo point past the record announcing it: without that, a crash before the deferred write would resume above the record and resolve a finished transition as interrupted. Checkpoint records replayed below the consistency point are not cross-checked, the persisted state being legitimately newer than what they carry. A base backup copies the control file at an arbitrary moment, so its state can be newer than the redo point replay starts from. Adopt the state carried by the starting checkpoint record under backup-label recovery; for a shutdown checkpoint, which replay does not see, take it in StartupXLOG. This also covers pg_rewind, which installs the source's control file while replay begins at the last common checkpoint. Backups taken from a standby are the exception: their starting checkpoint was written by the upstream primary, so they keep the state of the control file that was copied with them. The documented procedure for offline changes in a replication setup becomes the lockstep one: stop all nodes, run pg_checksums on each of them, then restart. Tests cover the lockstep procedure, divergent offline changes on either node and down a cascading chain, crash-restarts and promotions around online transitions, checkpoints racing an online enable, a base backup taken during an online enable, and pg_rewind across one. --- doc/src/sgml/ref/pg_checksums.sgml | 32 +- doc/src/sgml/wal.sgml | 8 + src/backend/access/transam/xlog.c | 358 ++++++++++++++++-- src/test/modules/test_checksums/Makefile | 2 +- src/test/modules/test_checksums/meson.build | 14 + .../test_checksums/t/012_offline_standby.pl | 213 +++++++++++ .../modules/test_checksums/t/013_rewind.pl | 198 ++++++++++ .../modules/test_checksums/t/014_lockstep.pl | 89 +++++ .../t/015_backup_online_enable.pl | 71 ++++ .../t/016_backup_from_standby.pl | 100 +++++ .../t/017_standby_crash_after_disable.pl | 125 ++++++ .../t/018_promote_enable_crash.pl | 134 +++++++ .../test_checksums/t/019_restartpoint_race.pl | 138 +++++++ .../t/020_primary_enable_crash.pl | 118 ++++++ .../t/021_standby_shutdown_catchup.pl | 93 +++++ .../t/022_resident_enable_crash.pl | 138 +++++++ .../t/023_concurrent_checkpoint_enable.pl | 114 ++++++ .../t/024_enable_crash_after_checkpoint.pl | 101 +++++ .../t/025_cascade_divergence.pl | 115 ++++++ 19 files changed, 2122 insertions(+), 39 deletions(-) create mode 100644 src/test/modules/test_checksums/t/012_offline_standby.pl create mode 100644 src/test/modules/test_checksums/t/013_rewind.pl create mode 100644 src/test/modules/test_checksums/t/014_lockstep.pl create mode 100644 src/test/modules/test_checksums/t/015_backup_online_enable.pl create mode 100644 src/test/modules/test_checksums/t/016_backup_from_standby.pl create mode 100644 src/test/modules/test_checksums/t/017_standby_crash_after_disable.pl create mode 100644 src/test/modules/test_checksums/t/018_promote_enable_crash.pl create mode 100644 src/test/modules/test_checksums/t/019_restartpoint_race.pl create mode 100644 src/test/modules/test_checksums/t/020_primary_enable_crash.pl create mode 100644 src/test/modules/test_checksums/t/021_standby_shutdown_catchup.pl create mode 100644 src/test/modules/test_checksums/t/022_resident_enable_crash.pl create mode 100644 src/test/modules/test_checksums/t/023_concurrent_checkpoint_enable.pl create mode 100644 src/test/modules/test_checksums/t/024_enable_crash_after_checkpoint.pl create mode 100644 src/test/modules/test_checksums/t/025_cascade_divergence.pl diff --git a/doc/src/sgml/ref/pg_checksums.sgml b/doc/src/sgml/ref/pg_checksums.sgml index ae66fad3f0f..bf07da09754 100644 --- a/doc/src/sgml/ref/pg_checksums.sgml +++ b/doc/src/sgml/ref/pg_checksums.sgml @@ -242,15 +242,29 @@ PostgreSQL documentation data directory must not be started or else data loss may occur. - When using a replication setup with tools which perform direct copies - of relation file blocks (for example ), - enabling or disabling checksums can lead to page corruptions in the - shape of incorrect checksums if the operation is not done consistently - across all nodes. When enabling or disabling checksums in a replication - setup, it is thus recommended to stop all the clusters before switching - them all consistently. Destroying all standbys, performing the operation - on the primary and finally recreating the standbys from scratch is also - safe. + Enabling or disabling checksums with + pg_checksums changes only the local data + directory; the new state does not propagate over replication. In a + replication setup the same change must be applied to every node: stop all + nodes, run pg_checksums on each of them, and + only then restart them. Tools that copy relation file blocks directly + between nodes, such as , likewise require + both nodes to be in the same data checksum state. + + + If the change is applied inconsistently, each node keeps its own state, + and a standby logs a warning when the state recorded in the replayed WAL + differs from its own. The same warning can appear transiently while a + standby catches up over WAL written before a consistent change; it stops + once a checkpoint record carrying the new state has been replayed. + + + A standby whose data directory was never checksummed must not have + checksums enabled by catching up this way. Converge the cluster by + running pg_checksums on it while stopped, by + enabling checksums online, or by recreating it from a base backup. Note + that an online enable only starts from a primary whose checksums are off, + so if they are already enabled there, disable them online first. If pg_checksums is aborted or killed while diff --git a/doc/src/sgml/wal.sgml b/doc/src/sgml/wal.sgml index ec62d17fbcc..db18a8c516e 100644 --- a/doc/src/sgml/wal.sgml +++ b/doc/src/sgml/wal.sgml @@ -317,6 +317,14 @@ verify checksums, on an offline cluster. + + An offline change only affects the data directory it is run on; the + new state does not propagate over replication. In a replication setup + the same change must be applied to all nodes while all of them are + stopped, as described in . A standby + whose state diverges logs a warning but keeps its local setting. + + diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index de4c96e135f..fa24b00e7d4 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -556,7 +556,7 @@ typedef struct XLogCtlData */ XLogRecPtr lastFpwDisableRecPtr; - /* last data_checksum_version we've seen */ + /* current data checksum state of this node */ uint32 data_checksum_version; slock_t info_lck; /* locks shared variables shown above */ @@ -690,6 +690,14 @@ static ChecksumStateType LocalDataChecksumState = 0; */ int data_checksums = 0; +/* + * Whether replay of the next checkpoint-family record must adopt the data + * checksum state it carries. Set when recovery starts from a base backup, + * where the state at the redo point takes precedence over the control file + * copied with the backup later. + */ +static bool adoptChecksumStateFromNextCheckpoint = false; + /* For WALInsertLockAcquire/Release functions */ static int MyLockNo = 0; static bool holdingAllLocks = false; @@ -730,6 +738,8 @@ static void ValidateXLOGDirectoryStructure(void); static void CleanupBackupHistory(void); static void UpdateMinRecoveryPoint(XLogRecPtr lsn, bool force); static bool PerformRecoveryXLogAction(void); +static void CheckReplayedDataChecksumState(uint32 replayed_version); +static void AdoptReplayedDataChecksumState(uint32 new_version); static void InitControlFile(uint64 sysidentifier, uint32 data_checksum_version); static void WriteControlFile(void); static void ReadControlFile(void); @@ -4827,6 +4837,7 @@ void SetDataChecksumsOn(void) { uint64 barrier; + bool persist; SpinLockAcquire(&XLogCtl->info_lck); @@ -4856,15 +4867,6 @@ SetDataChecksumsOn(void) XLogCtl->data_checksum_version = PG_DATA_CHECKSUM_VERSION; SpinLockRelease(&XLogCtl->info_lck); - /* - * Update the controlfile before waiting since if we have an immediate - * shutdown while waiting we want to come back up with checksums enabled. - */ - LWLockAcquire(ControlFileLock, LW_EXCLUSIVE); - ControlFile->data_checksum_version = PG_DATA_CHECKSUM_VERSION; - UpdateControlFile(); - LWLockRelease(ControlFileLock); - barrier = EmitProcSignalBarrier(PROCSIGNAL_BARRIER_CHECKSUM_ON); MyProc->delayChkptFlags &= ~DELAY_CHKPT_START; @@ -4873,6 +4875,35 @@ SetDataChecksumsOn(void) INJECTION_POINT("datachecksums-on-before-checkpoint", NULL); RequestCheckpoint(CHECKPOINT_FORCE | CHECKPOINT_WAIT | CHECKPOINT_FAST); + + INJECTION_POINT("datachecksums-on-after-checkpoint", NULL); + + /* + * Persist "on" only now that the checkpoint has flushed the pages the + * transition rewrote. Crash recovery initializes verification from the + * control file but resumes from a checkpoint that can predate the + * transition, so an "on" persisted earlier would have replay verify pages + * whose rewrite never reached disk: the pages the worker found in shared + * buffers are not written back by its ring buffer. + * + * The checkpoint above normally persists the state itself; this covers the + * case where it started before the record written above and left the field + * alone. Crashing before this point is safe, as replay then re-establishes + * "on" from the full page images of the rewrite. Skip the write if the + * state moved on meanwhile, since whatever moved it persists its own. + */ + SpinLockAcquire(&XLogCtl->info_lck); + persist = (XLogCtl->data_checksum_version == PG_DATA_CHECKSUM_VERSION); + SpinLockRelease(&XLogCtl->info_lck); + + if (persist) + { + LWLockAcquire(ControlFileLock, LW_EXCLUSIVE); + ControlFile->data_checksum_version = PG_DATA_CHECKSUM_VERSION; + UpdateControlFile(); + LWLockRelease(ControlFileLock); + } + WaitForProcSignalBarrier(barrier); } @@ -5001,6 +5032,124 @@ SetLocalDataChecksumState(uint32 data_checksum_version) data_checksums = data_checksum_version; } +/* + * Cross-check the data checksum state carried by a replayed checkpoint record + * against the state of this node. + * + * The state in the record belongs to whichever node wrote the WAL, and must + * not be adopted: an offline state change made with pg_checksums on one node + * of a replication set generates no WAL, and adopting would leak it into the + * other nodes through replay. Backup label recovery is the one exception, + * see AdoptReplayedDataChecksumState(). XLOG_CHECKPOINT_ONLINE needs no call + * here, as an online checkpoint's state already traveled in the preceding + * XLOG_CHECKPOINT_REDO record. + * + * Only archive recovery can see a lasting mismatch, as only there can the WAL + * and the control file come from different nodes or different times. In + * crash recovery a mismatch means replay resumed from a restartpoint + * predating an already-applied XLOG2_CHECKSUMS record, and replaying forward + * re-establishes the same state. + */ +static void +CheckReplayedDataChecksumState(uint32 replayed_version) +{ + /* + * Warn once per remote value, so a lasting mismatch does not flood the + * log. Matching states re-arm the warning. Backend-local state is + * enough: replay only runs in the startup process, and restarting it + * re-arms as well. + */ + static uint32 last_warned_version = PG_UINT32_MAX; + uint32 local_version; + + if (!ArchiveRecoveryRequested) + return; + + /* + * Re-replayed WAL below the consistency point was already cross-checked + * before minRecoveryPoint was last persisted, and the persisted state can + * legitimately be newer than what checkpoint records there carry: + * XLOG2_CHECKSUMS replay persists most states ahead of the restartpoint + * horizon. In particular the checkpoint record recovery restarts from is + * such a re-replay. + */ + if (!reachedConsistency) + return; + + SpinLockAcquire(&XLogCtl->info_lck); + local_version = XLogCtl->data_checksum_version; + SpinLockRelease(&XLogCtl->info_lck); + + if (replayed_version == local_version) + { + /* + * Report convergence if this process warned before. Nothing else + * tells the operator that running pg_checksums on the other nodes, or + * a rebuild, took effect. A restart in between loses the context, + * but it re-arms the warning too. + */ + if (last_warned_version != PG_UINT32_MAX) + ereport(LOG, + errmsg("data checksum state \"%s\" of this node now agrees with the replayed WAL", + get_checksum_state_string(local_version))); + + last_warned_version = PG_UINT32_MAX; + return; + } + + /* the nodes legitimately differ while an online transition runs */ + if (replayed_version == PG_DATA_CHECKSUM_INPROGRESS_ON || + replayed_version == PG_DATA_CHECKSUM_INPROGRESS_OFF || + local_version == PG_DATA_CHECKSUM_INPROGRESS_ON || + local_version == PG_DATA_CHECKSUM_INPROGRESS_OFF) + return; + + if (replayed_version == last_warned_version) + return; + last_warned_version = replayed_version; + + ereport(WARNING, + errmsg("data checksum state \"%s\" of this node does not match the state \"%s\" in the replayed WAL", + get_checksum_state_string(local_version), + get_checksum_state_string(replayed_version)), + errdetail("The data checksum state was most likely changed with pg_checksums on another node."), + errhint("Apply the same change with pg_checksums on the primary and all standby servers, or rebuild this server from a base backup.")); +} + +/* + * Adopt the data checksum state found at the redo point of backup label + * recovery. Persist it immediately so that a crash before the first + * restartpoint does not resurrect the state copied with the backup; a crash + * at this point restarts from the same redo point, so the control file does + * not run ahead of the replay position. If the value is unchanged the + * control file already carries it, so both the barrier and the persist are + * skipped. + */ +static void +AdoptReplayedDataChecksumState(uint32 new_version) +{ + bool changed = false; + + SpinLockAcquire(&XLogCtl->info_lck); + if (XLogCtl->data_checksum_version != new_version) + { + XLogCtl->data_checksum_version = new_version; + SetLocalDataChecksumState(new_version); + changed = true; + } + SpinLockRelease(&XLogCtl->info_lck); + + if (!changed) + return; + + EmitAndWaitDataChecksumsBarrier(new_version); + + LWLockAcquire(ControlFileLock, LW_EXCLUSIVE); + ControlFile->data_checksum_version = new_version; + UpdateControlFile(); + LWLockRelease(ControlFileLock); +} + /* guc hook */ const char * show_data_checksums(void) @@ -6031,6 +6180,39 @@ StartupXLOG(void) SetCommitTsLimit(checkPoint.oldestCommitTsXid, checkPoint.newestCommitTsXid); + /* + * When recovery starts from a base backup, the control file was copied at + * an arbitrary moment and its data checksum state may differ from the + * state at the redo point, which is what the WAL from there on was + * written under. Adopt the state of the starting checkpoint: a shutdown + * checkpoint is not replayed, so take it from the record read above; the + * redo point of an online checkpoint is its CHECKPOINT_REDO record, so + * let the replay of that record adopt it. Check backupStartPoint in + * addition to the label: on a crash restart during backup recovery the + * label file is already renamed away, but the start point persists until + * the backup end record. + * + * Not for a base backup taken from a standby, though. Its starting + * checkpoint is the standby's last restartpoint, a record written by the + * upstream primary, whose state is not the one the copied files were + * written under. The copied control file is right there, as a standby + * persists its state only at restartpoint horizons and so never claims + * more than what reached disk. Such backups are recognized by + * backupEndPoint together with backupEndRequired; backupEndPoint is only + * set for "BACKUP FROM: standby" labels and persists across a crash + * restart. pg_rewind writes a standby label as well, but no + * backupEndPoint, and its recovery must keep adopting. + */ + if ((haveBackupLabel || XLogRecPtrIsValid(ControlFile->backupStartPoint)) && + !(XLogRecPtrIsValid(ControlFile->backupEndPoint) && + ControlFile->backupEndRequired)) + { + if (wasShutdown) + AdoptReplayedDataChecksumState(checkPoint.dataChecksumState); + else + adoptChecksumStateFromNextCheckpoint = true; + } + /* * Clear out any old relcache cache files. This is *necessary* if we do * any WAL replay, since that would probably result in the cache files @@ -6814,6 +6996,23 @@ static bool PerformRecoveryXLogAction(void) { bool promoted = false; + bool flushForChecksums; + uint32 checksum_state; + + /* + * The end-of-recovery record persists the data checksum state without + * flushing the buffer pool, but the control file may only claim "on" once + * every page on disk carries a checksum. If replay entered that state + * without a restartpoint following it, the pages rewritten by the + * transition are still only in the buffer pool, so take the full + * checkpoint below instead of the lightweight record. + */ + SpinLockAcquire(&XLogCtl->info_lck); + checksum_state = XLogCtl->data_checksum_version; + SpinLockRelease(&XLogCtl->info_lck); + + flushForChecksums = (checksum_state == PG_DATA_CHECKSUM_VERSION && + ControlFile->data_checksum_version != checksum_state); /* * Perform a checkpoint to update all our recovery activity to disk. @@ -6829,7 +7028,7 @@ PerformRecoveryXLogAction(void) * fully out of recovery mode and already accepting queries. */ if (ArchiveRecoveryRequested && IsUnderPostmaster && - PromoteIsTriggered()) + PromoteIsTriggered() && !flushForChecksums) { promoted = true; @@ -7823,6 +8022,32 @@ CreateCheckPoint(int flags) ControlFile->minRecoveryPoint = InvalidXLogRecPtr; ControlFile->minRecoveryPointTLI = 0; + /* + * Persist the data checksum state this node runs under. Only the + * top-level field tracks this node; ControlFile->checkPointCopy above is + * a historical record used to resume replay. + * + * checkPoint.dataChecksumState was sampled while holding the WAL insert + * locks, so it is the state in effect at the redo point. If it was "on", + * the XLOG2_CHECKSUMS record announcing that precedes the redo point and + * every page the transition rewrote was dirtied before it, so + * CheckPointGuts() has just written all of them out. Recording the state + * here is what keeps a finished transition from being resolved as + * interrupted when this checkpoint is the one crash recovery resumes + * from: replay never sees the record announcing it. + * + * Persist it only if the state did not change while the flush was in + * progress. If it changed in between, the pages written out straddle two + * states, and the newer one could claim checksums that pages already on + * disk do not carry; leave the field to the next checkpoint then. + * SetDataChecksumsOff() persists the states that are safe to enter + * without a flush already. + */ + SpinLockAcquire(&XLogCtl->info_lck); + if (checkPoint.dataChecksumState == XLogCtl->data_checksum_version) + ControlFile->data_checksum_version = checkPoint.dataChecksumState; + SpinLockRelease(&XLogCtl->info_lck); + /* * Persist unloggedLSN value. It's reset on crash recovery, so this goes * unused on non-shutdown checkpoints, but seems useful to store it always @@ -7967,7 +8192,7 @@ CreateEndOfRecoveryRecord(void) ControlFile->minRecoveryPoint = recptr; ControlFile->minRecoveryPointTLI = xlrec.ThisTimeLineID; - /* start with the latest checksum version (as of the end of recovery) */ + /* persist the data checksum state this node ended recovery with */ SpinLockAcquire(&XLogCtl->info_lck); ControlFile->data_checksum_version = XLogCtl->data_checksum_version; SpinLockRelease(&XLogCtl->info_lck); @@ -8174,6 +8399,7 @@ CreateRestartPoint(int flags) XLogRecPtr endptr; XLogSegNo _logSegNo; TimestampTz xtime; + uint32 checksum_state; /* Concurrent checkpoint/restartpoint cannot happen */ Assert(!IsUnderPostmaster || MyBackendType == B_CHECKPOINTER); @@ -8220,8 +8446,39 @@ CreateRestartPoint(int flags) UpdateMinRecoveryPoint(InvalidXLogRecPtr, true); if (flags & CHECKPOINT_IS_SHUTDOWN) { + bool catchUpChecksums; + + /* + * There is no new restartpoint to persist the data checksum state + * with, but a cleanly stopped node should not leave the control + * file behind the state replay reached: pg_checksums and + * pg_rewind read it, and an in-progress state there makes them + * refuse to run. Catching it up needs the same guarantee a + * restartpoint gives: that every page on disk carries a checksum, + * so flush the buffer pool first. Only a transition to + * "on" that no restartpoint followed can get here; the other + * states are already persisted by XLOG2_CHECKSUMS replay. Replay + * has ended by now, so the state cannot change under us. + */ + SpinLockAcquire(&XLogCtl->info_lck); + checksum_state = XLogCtl->data_checksum_version; + SpinLockRelease(&XLogCtl->info_lck); + + catchUpChecksums = + (checksum_state != ControlFile->data_checksum_version && + XLogRecPtrIsValid(lastCheckPointRecPtr)); + + if (catchUpChecksums) + { + MemSet(&CheckpointStats, 0, sizeof(CheckpointStats)); + CheckpointStats.ckpt_start_t = GetCurrentTimestamp(); + CheckPointGuts(lastCheckPoint.redo, flags); + } + LWLockAcquire(ControlFileLock, LW_EXCLUSIVE); ControlFile->state = DB_SHUTDOWNED_IN_RECOVERY; + if (catchUpChecksums) + ControlFile->data_checksum_version = checksum_state; UpdateControlFile(); LWLockRelease(ControlFileLock); } @@ -8262,6 +8519,16 @@ CreateRestartPoint(int flags) /* Update the process title */ update_checkpoint_display(flags, true, false); + /* + * Note the data checksum state the flush below starts under. Replay runs + * concurrently and can change the state while the flush is in progress, + * in which case the flush covers pages written under both states; see + * where the state is persisted further down. + */ + SpinLockAcquire(&XLogCtl->info_lck); + checksum_state = XLogCtl->data_checksum_version; + SpinLockRelease(&XLogCtl->info_lck); + CheckPointGuts(lastCheckPoint.redo, flags); /* @@ -8321,8 +8588,21 @@ CreateRestartPoint(int flags) ControlFile->state = DB_SHUTDOWNED_IN_RECOVERY; } - /* we shall start with the latest checksum version */ - ControlFile->data_checksum_version = lastCheckPoint.dataChecksumState; + /* + * Persist the data checksum state of this node. Not the state of the + * replayed checkpoint: that one belongs to the node that wrote it and + * may differ after an offline change on either side. + * ControlFile->checkPointCopy above keeps the replayed value on + * purpose, being a historical record used to resume replay rather + * than a tracker of node state. + * + * Persist only if the flush above ran under one state throughout; see + * CreateCheckPoint() for why. + */ + SpinLockAcquire(&XLogCtl->info_lck); + if (checksum_state == XLogCtl->data_checksum_version) + ControlFile->data_checksum_version = checksum_state; + SpinLockRelease(&XLogCtl->info_lck); UpdateControlFile(); } @@ -8966,11 +9246,18 @@ xlog_redo(XLogReaderState *record) /* ControlFile->checkPointCopy always tracks the latest ckpt XID */ LWLockAcquire(ControlFileLock, LW_EXCLUSIVE); ControlFile->checkPointCopy.nextXid = checkPoint.nextXid; - ControlFile->data_checksum_version = checkPoint.dataChecksumState; UpdateControlFile(); LWLockRelease(ControlFileLock); + if (adoptChecksumStateFromNextCheckpoint) + { + adoptChecksumStateFromNextCheckpoint = false; + AdoptReplayedDataChecksumState(checkPoint.dataChecksumState); + } + else + CheckReplayedDataChecksumState(checkPoint.dataChecksumState); + /* * We should've already switched to the new TLI before replaying this * record. @@ -9206,19 +9493,16 @@ xlog_redo(XLogReaderState *record) else if (info == XLOG_CHECKPOINT_REDO) { xl_checkpoint_redo redo_rec; - bool new_state = false; memcpy(&redo_rec, XLogRecGetData(record), sizeof(xl_checkpoint_redo)); - SpinLockAcquire(&XLogCtl->info_lck); - XLogCtl->data_checksum_version = redo_rec.data_checksum_version; - SetLocalDataChecksumState(redo_rec.data_checksum_version); - if (redo_rec.data_checksum_version != ControlFile->data_checksum_version) - new_state = true; - SpinLockRelease(&XLogCtl->info_lck); - - if (new_state) - EmitAndWaitDataChecksumsBarrier(redo_rec.data_checksum_version); + if (adoptChecksumStateFromNextCheckpoint) + { + adoptChecksumStateFromNextCheckpoint = false; + AdoptReplayedDataChecksumState(redo_rec.data_checksum_version); + } + else + CheckReplayedDataChecksumState(redo_rec.data_checksum_version); } else if (info == XLOG_LOGICAL_DECODING_STATUS_CHANGE) { @@ -9288,17 +9572,33 @@ xlog2_redo(XLogReaderState *record) SpinLockAcquire(&XLogCtl->info_lck); XLogCtl->data_checksum_version = state.new_checksum_state; + SetLocalDataChecksumState(state.new_checksum_state); SpinLockRelease(&XLogCtl->info_lck); LWLockAcquire(ControlFileLock, LW_EXCLUSIVE); - ControlFile->data_checksum_version = state.new_checksum_state; + + /* + * Persist the new state, except when it is "on". Only "on" verifies + * checksums during reads, and between the last restartpoint and this + * record there may be pages on disk flushed under the old state; a + * crash-restart initializes verification from the control file and + * replay reads those pages back, so the control file may only say + * "on" once everything written under the transition has been flushed, + * as restartpoints and the end of recovery do. + * The opposite direction cannot wait for the restartpoint: once this + * record is replayed, evicted pages are written without checksums, + * and a control file still saying "on" would fail verification on + * exactly those pages after a crash. + */ + if (state.new_checksum_state != PG_DATA_CHECKSUM_VERSION) + ControlFile->data_checksum_version = state.new_checksum_state; /* * Update minRecoveryPoint to ensure that if recovery is aborted, we * recover back up to this point before allowing hot standby again. - * The new state is durable in pg_control while its location is only - * tracked in shared memory; a standby becoming consistent below this - * record would let base backups resume checksum verification with the + * The change location is only tracked in shared memory and is lost + * over a restart; a standby becoming consistent below this record + * would let base backups resume checksum verification with the * location unknown. The local copies cannot be updated as long as * crash recovery is happening and we expect all the WAL to be * replayed. diff --git a/src/test/modules/test_checksums/Makefile b/src/test/modules/test_checksums/Makefile index 71455cd5577..80f54bf6d8c 100644 --- a/src/test/modules/test_checksums/Makefile +++ b/src/test/modules/test_checksums/Makefile @@ -9,7 +9,7 @@ # #------------------------------------------------------------------------- -EXTRA_INSTALL = src/test/modules/injection_points +EXTRA_INSTALL = contrib/pg_buffercache src/test/modules/injection_points export enable_injection_points diff --git a/src/test/modules/test_checksums/meson.build b/src/test/modules/test_checksums/meson.build index fb7129d796f..0269d4c6e38 100644 --- a/src/test/modules/test_checksums/meson.build +++ b/src/test/modules/test_checksums/meson.build @@ -35,6 +35,20 @@ tests += { 't/009_fpi.pl', 't/010_backup_straddle.pl', 't/011_standby_straddle.pl', + 't/012_offline_standby.pl', + 't/013_rewind.pl', + 't/014_lockstep.pl', + 't/015_backup_online_enable.pl', + 't/016_backup_from_standby.pl', + 't/017_standby_crash_after_disable.pl', + 't/018_promote_enable_crash.pl', + 't/019_restartpoint_race.pl', + 't/020_primary_enable_crash.pl', + 't/021_standby_shutdown_catchup.pl', + 't/022_resident_enable_crash.pl', + 't/023_concurrent_checkpoint_enable.pl', + 't/024_enable_crash_after_checkpoint.pl', + 't/025_cascade_divergence.pl', ], }, } diff --git a/src/test/modules/test_checksums/t/012_offline_standby.pl b/src/test/modules/test_checksums/t/012_offline_standby.pl new file mode 100644 index 00000000000..94c476293b7 --- /dev/null +++ b/src/test/modules/test_checksums/t/012_offline_standby.pl @@ -0,0 +1,213 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group + +# Offline checksum changes with pg_checksums are local to one node. A +# standby must neither adopt the state of the primary from replayed +# checkpoint records, nor lose its own offline change to them. +use strict; +use warnings FATAL => 'all'; + +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +use FindBin; +use lib $FindBin::RealBin; + +use DataChecksums::Utils; + +my $primary = PostgreSQL::Test::Cluster->new('primary'); +$primary->init(allows_streaming => 1, no_data_checksums => 1); +$primary->append_conf('postgresql.conf', 'autovacuum = off'); +$primary->start; +$primary->safe_psql('postgres', + "CREATE TABLE t AS SELECT generate_series(1,10000) AS a;"); + +$primary->backup('backup'); +my $standby = PostgreSQL::Test::Cluster->new('standby'); +$standby->init_from_backup($primary, 'backup', has_streaming => 1); +$standby->start; +$primary->wait_for_catchup($standby); + +test_checksum_state($primary, 'off'); +test_checksum_state($standby, 'off'); + +# Scenario 1: enable offline on the primary only. The standby must +# stay off, warn about the mismatch, and remain readable. +$standby->stop; +$primary->stop; +$primary->checksum_enable_offline; +$primary->start; +$standby->start; + +test_checksum_state($primary, 'on'); +test_checksum_state($standby, 'off'); + +my $logstart = -s $standby->logfile; +$primary->safe_psql('postgres', "INSERT INTO t VALUES (0);"); +$primary->safe_psql('postgres', "CHECKPOINT;"); +$primary->wait_for_catchup($standby); + +test_checksum_state($standby, 'off'); +is($standby->safe_psql('postgres', "SELECT count(*) FROM t;"), + '10001', 'standby readable after offline enable on the primary'); + +$standby->wait_for_log(qr/does not match the state "on" in the replayed WAL/, + $logstart); + +$primary->safe_psql('postgres', "CHECKPOINT;"); +$primary->wait_for_catchup($standby); +my $log = PostgreSQL::Test::Utils::slurp_file($standby->logfile, $logstart); +my @warnings = $log =~ /(does not match the state)/g; +is(scalar(@warnings), 1, 'mismatch warned once per remote value'); + +# Matching states re-arm the warning: undo the divergence on the primary, +# then diverge again to the same value, all without restarting the standby. +$primary->stop; +$primary->checksum_disable_offline; +$logstart = -s $standby->logfile; +$primary->start; +$primary->safe_psql('postgres', "CHECKPOINT;"); +$primary->wait_for_catchup($standby); +$log = PostgreSQL::Test::Utils::slurp_file($standby->logfile, $logstart); +unlike( + $log, + qr/does not match the state/, + 'no warning while the states match again'); +like($log, qr/now agrees with the replayed WAL/, + 'convergence is reported once the states match again'); + +$primary->stop; +$primary->checksum_enable_offline; +$primary->start; +$primary->safe_psql('postgres', "CHECKPOINT;"); +$primary->wait_for_catchup($standby); +$standby->wait_for_log(qr/does not match the state "on" in the replayed WAL/, + $logstart); +test_checksum_state($standby, 'off'); + +# The local state survives both clean and immediate restarts. +$standby->restart; +test_checksum_state($standby, 'off'); +$standby->stop('immediate'); +$standby->start; +test_checksum_state($standby, 'off'); + +# Converge the cluster: enable offline on the standby too. +$standby->stop; +command_checks_all( + [ 'pg_checksums', '--enable', '-D', $standby->data_dir ], + 0, + [qr/appears to be a standby/], + [], + 'standby-role notice on offline enable'); +$standby->start; +test_checksum_state($standby, 'on'); +$primary->wait_for_catchup($standby); +is($standby->safe_psql('postgres', "SELECT count(*) FROM t;"), + '10001', 'standby readable after converging'); + +# Scenario 2: disable offline on the standby only. The replayed +# checkpoint records of the still-enabled primary must not override it. +$standby->stop; +$standby->checksum_disable_offline; +$standby->start; +test_checksum_state($standby, 'off'); +test_checksum_state($primary, 'on'); + +$primary->safe_psql('postgres', "CHECKPOINT;"); +$primary->wait_for_catchup($standby); +test_checksum_state($standby, 'off'); + +# Restartpoints must persist the local state, not the replayed copy. +$standby->safe_psql('postgres', "CHECKPOINT;"); +$standby->restart; +test_checksum_state($standby, 'off'); +$standby->stop('immediate'); +$standby->start; +test_checksum_state($standby, 'off'); + +is($standby->safe_psql('postgres', "SELECT count(*) FROM t;"), + '10001', 'standby readable with checksums disabled locally'); + +# Scenario 3: crash-restart right after an online transition, before the +# next restartpoint. Replay then resumes from an older restartpoint whose +# checkpoint records still carry the pre-transition state. Those must +# still match the state seeded from the control file, and the transition +# itself must be re-established by re-replaying the XLOG2_CHECKSUMS record, +# without a spurious mismatch warning along the way. + +# Converge first: bring the standby back to "on" offline. +$standby->stop; +$standby->checksum_enable_offline; +$standby->start; +test_checksum_state($standby, 'on'); +test_checksum_state($primary, 'on'); + +disable_data_checksums($primary, wait => 'off'); +$primary->wait_for_catchup($standby); +wait_for_checksum_state($standby, 'off'); + +# Crash-restart the standby immediately, before any restartpoint has had a +# chance to persist the new state to its control file. +$logstart = -s $standby->logfile; +$standby->stop('immediate'); +$standby->start; +$primary->wait_for_catchup($standby); + +test_checksum_state($standby, 'off'); +is($standby->safe_psql('postgres', "SELECT count(*) FROM t;"), + '10001', + 'standby readable after crash-restart across an online transition'); + +$log = PostgreSQL::Test::Utils::slurp_file($standby->logfile, $logstart); +unlike( + $log, + qr/does not match the state/, + 'no spurious mismatch warning after crash-restart across an online transition' +); + +# Scenario 4: a standby stopped while replaying an interrupted online +# transition keeps the interrupted state in its own control file, and +# pg_checksums must refuse to touch it. A primary is never caught this +# way, as its checksums launcher resolves inprogress-on back to off from +# its exit cleanup. A standby has no launcher; it carries forward +# whatever the last replayed record left it in. + +# Block an online enable on the primary at inprogress-on with a +# blocking temp table, same trick as in 004_offline.pl. +my $bsession = $primary->background_psql('postgres'); +$bsession->query_safe('CREATE TEMPORARY TABLE tt (a integer);'); +enable_data_checksums($primary, wait => 'inprogress-on'); + +# The standby picks up the in-progress state from the XLOG2_CHECKSUMS record. +wait_for_checksum_state($standby, 'inprogress-on'); + +# Stop the standby cleanly; its restartpoint persists inprogress-on to +# its own control file, since nothing on a standby resolves it away. +$standby->stop; + +command_fails_like( + [ 'pg_checksums', '--enable', '-D', $standby->data_dir ], + qr/online data checksum state transition was interrupted/, + 'pg_checksums --enable refuses a standby stopped mid-transition'); +command_fails_like( + [ 'pg_checksums', '--check', '-D', $standby->data_dir ], + qr/online data checksum state transition was interrupted/, + 'pg_checksums --check refuses a standby stopped mid-transition'); +command_fails_like( + [ 'pg_checksums', '--disable', '-D', $standby->data_dir ], + qr/online data checksum state transition was interrupted/, + 'pg_checksums --disable refuses a standby stopped mid-transition'); + +$standby->start; +$bsession->quit; +wait_for_checksum_state($primary, 'on'); +$primary->wait_for_catchup($standby); +wait_for_checksum_state($standby, 'on'); + +is( $standby->safe_psql('postgres', "SELECT count(*) FROM t;"), + '10001', 'standby readable once the transition completes'); + +$standby->stop; +$primary->stop; +done_testing(); diff --git a/src/test/modules/test_checksums/t/013_rewind.pl b/src/test/modules/test_checksums/t/013_rewind.pl new file mode 100644 index 00000000000..42fee544f62 --- /dev/null +++ b/src/test/modules/test_checksums/t/013_rewind.pl @@ -0,0 +1,198 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group + +# Test pg_rewind across an online data checksum enable. +# +# A clean switchover leaves the shutdown checkpoint of the old primary +# as the last common checkpoint between the two nodes. When data +# checksums are enabled online on the new primary before the old one is +# rewound, pg_rewind installs the control file of the new primary, which +# already claims checksums are fully enabled, while replay begins at the +# shutdown checkpoint whose record still carries the old state. Replay +# of the WAL stretch from before the enable must run with checksums off, +# as recorded in the checkpoint, else it would verify pages which never +# had checksums written and fail recovery. +# +# The new primary requests a checkpoint right after promotion, and +# replaying its CHECKPOINT_REDO record would repair the state before +# any interesting WAL is reached. Hold the checkpointer on the standby +# in a restartpoint over the promotion, like in the recovery test +# 041_checkpoint_at_promote, so that the post-promotion writes end up +# in WAL before the first checkpoint of the new timeline. +use strict; +use warnings FATAL => 'all'; + +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +use FindBin; +use lib $FindBin::RealBin; + +use DataChecksums::Utils; + +if ($ENV{enable_injection_points} ne 'yes') +{ + plan skip_all => 'Injection points not supported by this build'; +} + +# Old primary. full_page_writes is off so that the updates done on the +# promoted node do not carry full page images, forcing replay on the +# rewound node to read the pages from disk. wal_log_hints is required +# by pg_rewind on a cluster without data checksums. +my $node_a = PostgreSQL::Test::Cluster->new('node_a'); +$node_a->init(allows_streaming => 1, no_data_checksums => 1); +$node_a->append_conf( + 'postgresql.conf', qq[ +autovacuum = off +full_page_writes = off +wal_log_hints = on +wal_keep_size = '1GB' +log_checkpoints = on +]); +$node_a->start; + +if (!$node_a->check_extension('injection_points')) +{ + plan skip_all => 'Extension injection_points not installed'; +} + +$node_a->safe_psql('postgres', + "CREATE TABLE t AS SELECT generate_series(1,10000) AS a;"); +$node_a->safe_psql('postgres', "CREATE TABLE t_div (a int);"); +$node_a->safe_psql('postgres', "CREATE EXTENSION injection_points;"); + +# Set the hint bits on t before taking the backup, so that reads on the +# promoted node do not emit full page images for its pages later. +$node_a->safe_psql('postgres', "CHECKPOINT;"); +$node_a->safe_psql('postgres', "SELECT count(*) FROM t;"); + +$node_a->backup('backup'); +my $node_b = PostgreSQL::Test::Cluster->new('node_b'); +$node_b->init_from_backup($node_a, 'backup', has_streaming => 1); +$node_b->start; + +$node_a->wait_for_catchup($node_b, 'replay', $node_a->lsn('insert')); +test_checksum_state($node_a, 'off'); +test_checksum_state($node_b, 'off'); + +# Hold the next restartpoint on the standby. +$node_b->safe_psql('postgres', + "SELECT injection_points_attach('create-restart-point', 'wait');"); + +# Give the restartpoint a checkpoint record to work on, then start it +# in a background session; it will block on the injection point with +# the checkpointer busy until released. +$node_a->safe_psql('postgres', "CHECKPOINT;"); +$node_a->wait_for_catchup($node_b, 'replay', $node_a->lsn('insert')); + +my $bg_psql = $node_b->background_psql('postgres', on_error_stop => 0); +$bg_psql->query_until( + qr/starting_restartpoint/, q( + \echo starting_restartpoint + CHECKPOINT; +)); +$node_b->wait_for_event('checkpointer', 'create-restart-point'); + +# Clean switchover: the shutdown checkpoint of A streams to B and +# becomes the last common checkpoint. +$node_a->stop('fast'); + +my ($stdout, $stderr) = run_command([ 'pg_controldata', $node_a->data_dir ]); +$stdout =~ /^Latest checkpoint location:\s*([0-9A-F\/]+)$/m + or die "checkpoint location missing from pg_controldata output"; +my $shutdown_ckpt = $1; +$node_b->poll_query_until('postgres', + "SELECT pg_last_wal_replay_lsn() > '$shutdown_ckpt'::pg_lsn;") + or die "standby never replayed the shutdown checkpoint"; + +my $logstart = -s $node_b->logfile; +$node_b->promote; + +# Accidental restart of the old primary, diverging its timeline. +$node_a->start; +$node_a->safe_psql('postgres', "INSERT INTO t_div VALUES (1);"); +$node_a->stop('fast'); + +# Updates on the new primary before its first checkpoint; replay of +# these on the rewound node has to read the pages from disk. +$node_b->safe_psql('postgres', "UPDATE t SET a = a WHERE a % 25 = 0;"); +ok( !$node_b->log_contains("checkpoint complete", $logstart), + "no checkpoint on the new timeline before the updates"); + +# Release the checkpointer; the queued post-promotion checkpoint runs +# after the updates. +$node_b->safe_psql('postgres', + "SELECT injection_points_wakeup('create-restart-point');"); +$node_b->safe_psql('postgres', + "SELECT injection_points_detach('create-restart-point');"); +$bg_psql->quit; + +enable_data_checksums($node_b, wait => 'on'); +test_checksum_state($node_b, 'on'); + +# pg_rewind refuses to run with full_page_writes disabled on the +# source; the updates it was disabled for are already in WAL. +$node_b->safe_psql('postgres', "ALTER SYSTEM SET full_page_writes = on;"); +$node_b->safe_psql('postgres', "SELECT pg_reload_conf();"); + +command_ok( + [ + 'pg_rewind', + '--target-pgdata' => $node_a->data_dir, + '--source-server' => $node_b->connstr('postgres'), + ], + 'pg_rewind from the new primary'); + +# Replay on the rewound node must start at the shutdown checkpoint of +# the switchover, with the control file of the new primary. +my $backup_label = slurp_file($node_a->data_dir . '/backup_label'); +$backup_label =~ /^CHECKPOINT LOCATION: ([0-9A-F\/]+)$/m + or die "checkpoint location missing from backup_label"; +is($1, $shutdown_ckpt, 'replay starts at the switchover checkpoint'); + +($stdout, $stderr) = run_command( + [ + 'pg_waldump', + '-p' => $node_a->data_dir . '/pg_wal', + '-t' => 1, + '-s' => $shutdown_ckpt, + '-n' => 1, + ]); +like($stdout, qr/CHECKPOINT_SHUTDOWN/, + 'last common checkpoint is a shutdown checkpoint'); + +($stdout, $stderr) = run_command([ 'pg_controldata', $node_a->data_dir ]); +like( + $stdout, + qr/^Data page checksum version:\s*1$/m, + 'rewound node received a control file with checksums on'); + +# Start the rewound node as a standby of the new primary. Replay runs +# through the pre-enable WAL stretch and the online enable. +# +# The rewind replaced the configuration files with those of the new +# primary, so put the port back. +my $connstr = $node_b->connstr; +$node_a->append_conf( + 'postgresql.conf', qq[ +port = @{[$node_a->port]} +primary_conninfo = '$connstr application_name=@{[$node_a->name]}' +]); +$node_a->set_standby_mode; +$node_a->start; + +$node_b->wait_for_catchup($node_a, 'replay', $node_b->lsn('insert')); +test_checksum_state($node_a, 'on'); + +is($node_a->safe_psql('postgres', "SELECT count(*) FROM t;"), + '10000', 'data readable on the rewound node'); +is($node_a->safe_psql('postgres', "SELECT count(*) FROM t_div;"), + '0', 'divergent insert was rewound'); + +$node_a->stop('fast'); +command_ok([ 'pg_checksums', '--check', '-D', $node_a->data_dir ], + 'checksums valid on the rewound node'); + +$node_b->stop; + +done_testing(); diff --git a/src/test/modules/test_checksums/t/014_lockstep.pl b/src/test/modules/test_checksums/t/014_lockstep.pl new file mode 100644 index 00000000000..89c3b9da2cc --- /dev/null +++ b/src/test/modules/test_checksums/t/014_lockstep.pl @@ -0,0 +1,89 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group + +# The lockstep procedure for offline checksum changes in a replication +# setup: stop all nodes, run pg_checksums on all of them, restart. +# Replay of checkpoint records written before the change must not +# revert the state of the standby. +use strict; +use warnings FATAL => 'all'; + +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +use FindBin; +use lib $FindBin::RealBin; + +use DataChecksums::Utils; + +my $primary = PostgreSQL::Test::Cluster->new('primary'); +$primary->init(allows_streaming => 1, no_data_checksums => 1); +$primary->append_conf( + 'postgresql.conf', qq[ +autovacuum = off +wal_keep_size = '1GB' +]); +$primary->start; +$primary->safe_psql('postgres', + "CREATE TABLE t AS SELECT generate_series(1,10000) AS a;"); + +$primary->backup('backup'); +my $standby = PostgreSQL::Test::Cluster->new('standby'); +$standby->init_from_backup($primary, 'backup', has_streaming => 1); +$standby->start; +$primary->wait_for_catchup($standby); + +# Stop the standby first: the WAL written after this point is replayed +# only after the offline switch, and every checkpoint record in it +# still carries the old state. +$standby->stop; +$primary->safe_psql('postgres', "UPDATE t SET a = a WHERE a % 10 = 0;"); +$primary->safe_psql('postgres', "CHECKPOINT;"); +$primary->safe_psql('postgres', "UPDATE t SET a = a WHERE a % 10 = 1;"); +$primary->safe_psql('postgres', "CHECKPOINT;"); +$primary->stop; + +# The lockstep procedure. +$primary->checksum_enable_offline; +$standby->checksum_enable_offline; +$primary->start; +$standby->start; + +# The standby replays the pre-switch checkpoints; its state must not +# revert to off. +$primary->wait_for_catchup($standby); +$standby->wait_for_log(qr/does not match the state "off" in the replayed WAL/, + 0); +test_checksum_state($standby, 'on'); +test_checksum_state($primary, 'on'); + +is($standby->safe_psql('postgres', "SELECT count(*) FROM t;"), + '10000', 'standby readable after lockstep enable'); + +# Crash the standby and replay the same stretch again. +$standby->stop('immediate'); +$standby->start; +$primary->wait_for_catchup($standby); +test_checksum_state($standby, 'on'); +is($standby->safe_psql('postgres', "SELECT count(*) FROM t;"), + '10000', 'standby readable after crash restart'); + +# Once a post-switch checkpoint has been replayed the states match and +# no warning may be logged. +my $logstart = -s $standby->logfile; +$primary->safe_psql('postgres', "CHECKPOINT;"); +$primary->wait_for_catchup($standby); +my $log = PostgreSQL::Test::Utils::slurp_file($standby->logfile, $logstart); +unlike( + $log, + qr/does not match the state/, + 'no mismatch warning once the states match'); + +# Every page the standby wrote in this window must carry a checksum. +$standby->safe_psql('postgres', 'CHECKPOINT;'); +$standby->stop; +command_ok([ 'pg_checksums', '--check', '-D', $standby->data_dir ], + 'checksums valid on the standby'); + +$primary->stop; +done_testing(); diff --git a/src/test/modules/test_checksums/t/015_backup_online_enable.pl b/src/test/modules/test_checksums/t/015_backup_online_enable.pl new file mode 100644 index 00000000000..8ea6679dc63 --- /dev/null +++ b/src/test/modules/test_checksums/t/015_backup_online_enable.pl @@ -0,0 +1,71 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group + +# A base backup taken while an online checksum enable is running. The +# control file copied with the backup, and the redo point of the backup +# checkpoint, both carry the in-progress state; replay of the +# XLOG2_CHECKSUMS records completes the transition on the new standby. +use strict; +use warnings FATAL => 'all'; + +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +use FindBin; +use lib $FindBin::RealBin; + +use DataChecksums::Utils; + +my $primary = PostgreSQL::Test::Cluster->new('primary'); +$primary->init(allows_streaming => 1, no_data_checksums => 1); +$primary->append_conf('postgresql.conf', 'autovacuum = off'); +$primary->start; +$primary->safe_psql('postgres', + "CREATE TABLE t AS SELECT generate_series(1,10000) AS a;"); + +# Hold the enable at inprogress-on with a pre-existing temp table the +# checksum worker has to wait out, same trick as in 004_offline.pl and +# scenario 4 of 012_offline_standby.pl. +my $bsession = $primary->background_psql('postgres'); +$bsession->query_safe('CREATE TEMPORARY TABLE tt (a integer);'); + +enable_data_checksums($primary, wait => 'inprogress-on'); + +# The backup's checkpoint is taken while the primary sits at +# inprogress-on, so both the control file and the redo point's +# XLOG_CHECKPOINT_REDO record carry that state. +$primary->backup('backup'); +my $standby = PostgreSQL::Test::Cluster->new('standby'); +$standby->init_from_backup($primary, 'backup', has_streaming => 1); +$standby->start; + +# Backup label recovery adopts the redo point's state immediately, before +# any further WAL is replayed. +wait_for_checksum_state($standby, 'inprogress-on'); + +# Release the barrier; the transition completes on the primary and +# replicates. Nothing before this point can legitimately disagree: the +# standby starts out at the same inprogress-on state as the primary, so +# there is nothing yet to warn about. +$bsession->quit; +wait_for_checksum_state($primary, 'on'); +$primary->wait_for_catchup($standby); +wait_for_checksum_state($standby, 'on'); + +is($standby->safe_psql('postgres', "SELECT count(*) FROM t;"), + '10000', 'standby readable after the transition completed'); + +my $log = PostgreSQL::Test::Utils::slurp_file($standby->logfile); +unlike( + $log, + qr/does not match the state/, + 'no mismatch warning at any point during a backup taken mid-transition'); + +# No extra CHECKPOINT needed: the transition's completion checkpoint wrote +# every page with a checksum, and the shutdown restartpoint flushes the rest. +$standby->stop; +command_ok([ 'pg_checksums', '--check', '-D', $standby->data_dir ], + 'checksums valid on the standby'); + +$primary->stop; +done_testing(); diff --git a/src/test/modules/test_checksums/t/016_backup_from_standby.pl b/src/test/modules/test_checksums/t/016_backup_from_standby.pl new file mode 100644 index 00000000000..3d01259e424 --- /dev/null +++ b/src/test/modules/test_checksums/t/016_backup_from_standby.pl @@ -0,0 +1,100 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group + +# Test that a base backup taken from a standby keeps the standby's data +# checksum state. +# +# A backup taken on a standby uses the last restartpoint as its starting +# checkpoint (do_pg_backup_start()), so the record at the redo point was +# written by the upstream primary and carries the primary's state. Recovery +# from such a backup must not adopt that state: the files were copied from +# the standby, and with the standby diverged to "off" under an "on" primary +# the new node would come up verifying checksums its files do not have. + +use strict; +use warnings FATAL => 'all'; + +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +use FindBin; +use lib $FindBin::RealBin; + +use DataChecksums::Utils; + +my $primary = PostgreSQL::Test::Cluster->new('primary'); +$primary->init(allows_streaming => 1, no_data_checksums => 1); +$primary->append_conf('postgresql.conf', 'autovacuum = off'); +$primary->start; +$primary->safe_psql('postgres', + "CREATE TABLE t AS SELECT generate_series(1,10000) AS a;"); + +$primary->backup('backup'); +my $standby = PostgreSQL::Test::Cluster->new('standby'); +$standby->init_from_backup($primary, 'backup', has_streaming => 1); +$standby->start; +$primary->wait_for_catchup($standby); + +test_checksum_state($primary, 'off'); +test_checksum_state($standby, 'off'); + +# Offline enable on the primary only. Per 012_offline_standby.pl this is a +# divergence the standby is expected to survive: it keeps its own "off" +# state, warns once, and stays readable. +$standby->stop; +$primary->stop; +system_or_bail('pg_checksums', '--enable', '--pgdata', $primary->data_dir); +$primary->start; +$standby->start; +$primary->wait_for_catchup($standby); + +test_checksum_state($primary, 'on'); +test_checksum_state($standby, 'off'); + +# Make sure the standby has written pages under its own "off" state, so its +# files really do lack checksums. +$primary->safe_psql('postgres', + "CREATE TABLE t2 AS SELECT generate_series(1,50000) AS a;"); +$primary->safe_psql('postgres', "CHECKPOINT;"); +$primary->wait_for_catchup($standby); +$standby->safe_psql('postgres', "SELECT count(*) FROM t2;"); + +# Now take a base backup *from the standby* and bring the copy up. +$standby->backup('from_standby'); +my $newnode = PostgreSQL::Test::Cluster->new('newnode'); +$newnode->init_from_backup($standby, 'from_standby', has_streaming => 1); +$newnode->append_conf('postgresql.conf', + "primary_conninfo = '" . $primary->connstr . "'"); +$newnode->start; + +# The new node's files all came from a cluster running with checksums off. +# Anything but "off" here means it adopted the primary's state through the +# checkpoint record at the redo point. +my ($rc, $stdout, $stderr) = $newnode->psql('postgres', + "SELECT setting FROM pg_settings WHERE name = 'data_checksums';"); +is($rc, 0, 'the node copied from the standby accepts connections') + or diag("stderr: $stderr"); +is($stdout, 'off', 'backup of an "off" standby comes up with checksums off'); + +# And it must be able to read the pages the standby wrote without checksums. +($rc, $stdout, $stderr) = + $newnode->psql('postgres', "SELECT count(*) FROM t2;"); +is($rc, 0, 'pages copied from the standby are readable') + or diag("stderr: $stderr"); + +my $log = PostgreSQL::Test::Utils::slurp_file($newnode->logfile); +unlike( + $log, + qr/page verification failed/, + 'no checksum verification failures on the node copied from the standby'); + +$newnode->stop('immediate'); + +($stdout, $stderr) = run_command([ 'pg_controldata', $newnode->data_dir ]); +my ($ctl_state) = $stdout =~ /Data page checksum version:\s+(\d+)/; +note("newnode control file data checksum version: $ctl_state"); + +$standby->stop; +$primary->stop; + +done_testing(); diff --git a/src/test/modules/test_checksums/t/017_standby_crash_after_disable.pl b/src/test/modules/test_checksums/t/017_standby_crash_after_disable.pl new file mode 100644 index 00000000000..315b27b93d4 --- /dev/null +++ b/src/test/modules/test_checksums/t/017_standby_crash_after_disable.pl @@ -0,0 +1,125 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group + +# Test that a standby crashing after a replayed online disable, but before +# its next restartpoint, restarts cleanly. +# +# Pages dirtied before the XLOG2_CHECKSUMS record and evicted after it are +# written out under the new "off" state, without checksums. The control +# file must follow the record immediately in this direction: a crash-restart +# initializes checksum verification from the control file, and one still +# saying "on" would fail verification on exactly those pages while replaying +# records older than the state change. + +use strict; +use warnings FATAL => 'all'; + +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +use FindBin; +use lib $FindBin::RealBin; + +use DataChecksums::Utils; + +my $primary = PostgreSQL::Test::Cluster->new('primary'); +$primary->init(allows_streaming => 1); +$primary->append_conf( + 'postgresql.conf', qq( +autovacuum = off +checkpoint_timeout = 1h +max_wal_size = 10GB +)); +$primary->start; + +test_checksum_state($primary, 'on'); + +$primary->safe_psql('postgres', + "CREATE TABLE t AS SELECT generate_series(1,1000) AS a;"); + +$primary->backup('backup'); +my $standby = PostgreSQL::Test::Cluster->new('standby'); +$standby->init_from_backup($primary, 'backup', has_streaming => 1); + +# Small buffer pool so that replaying a bulk load evicts the pages we care +# about, and no restartpoints so the control file stays where it is. +$standby->append_conf( + 'postgresql.conf', qq( +shared_buffers = 1MB +checkpoint_timeout = 1h +max_wal_size = 10GB +bgwriter_delay = 10000 +)); +$standby->start; +$primary->wait_for_catchup($standby); + +test_checksum_state($standby, 'on'); + +# No full page images, so that replay has to read the pages of "t" back from +# disk instead of overwriting them from the WAL. +$primary->append_conf('postgresql.conf', 'full_page_writes = off'); +$primary->reload; +$primary->safe_psql('postgres', 'CHECKPOINT;'); + +# Establish the restartpoint that recovery will resume from after the crash. +$standby->safe_psql('postgres', 'CHECKPOINT;'); +$primary->wait_for_catchup($standby); + +# Dirty the pages of "t" on the standby, *before* the state change. They are +# not flushed: the standby has no restartpoint from here on. +$primary->safe_psql('postgres', 'UPDATE t SET a = a + 1;'); +$primary->wait_for_catchup($standby); + +# Online disable. The standby replays it and switches to "off", but its +# control file is not updated. +$primary->safe_psql('postgres', 'SELECT pg_disable_data_checksums();'); +$primary->wait_for_catchup($standby); +wait_for_checksum_state($standby, 'off'); + +my ($ctl_before) = run_command([ 'pg_controldata', $standby->data_dir ]); +my ($ctl_state) = $ctl_before =~ /Data page checksum version:\s+(\d+)/; +note( + "standby control file data_checksum_version after the disable: " + . "$ctl_state (0 = off, 1 = on)"); + +# Force the standby to evict the dirty pages of "t" now that it is "off": +# they are written back without checksums. +$primary->safe_psql('postgres', + "CREATE TABLE filler AS SELECT generate_series(1,300000) AS a;"); +$primary->wait_for_catchup($standby); + +# Crash the standby before it gets a chance to run a restartpoint. +$standby->stop('immediate'); + +my $started = $standby->start(fail_ok => 1); +ok($started, + 'standby restarts after crashing between a checksum state ' + . 'change and the next restartpoint'); + +my $log = PostgreSQL::Test::Utils::slurp_file($standby->logfile); +unlike( + $log, + qr/page verification failed/, + 'no checksum verification failures while replaying'); +unlike($log, qr/invalid page in block/, 'no invalid pages while replaying'); + +if ($started) +{ + my ($rc, $stdout, $stderr) = + $standby->psql('postgres', 'SELECT count(*) FROM t;'); + is($rc, 0, 'the pages written while "off" are readable') + or diag("stderr: $stderr"); + $standby->stop('immediate'); +} +else +{ + my @lines = grep { /FATAL|PANIC|invalid page|verification failed/ } + split(/\n/, $log); + diag("standby log tail:\n" . join("\n", @lines[ -12 .. -1 ])) + if @lines >= 12; + fail('the pages written while "off" are readable'); +} + +$primary->stop; + +done_testing(); diff --git a/src/test/modules/test_checksums/t/018_promote_enable_crash.pl b/src/test/modules/test_checksums/t/018_promote_enable_crash.pl new file mode 100644 index 00000000000..97f84da5685 --- /dev/null +++ b/src/test/modules/test_checksums/t/018_promote_enable_crash.pl @@ -0,0 +1,134 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group + +# Test a standby promoted right after replaying an online enable, before any +# restartpoint has flushed the rewritten pages. +# +# The end-of-recovery record persists the checksum state without flushing the +# buffer pool, while the control file still points at a restartpoint older than +# the transition. Persisting "on" there would make a crash before the +# post-promotion checkpoint verify checksums over the pages that were flushed +# while the state was still "off"; promotion must take the full checkpoint +# instead. + +use strict; +use warnings FATAL => 'all'; + +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +use FindBin; +use lib $FindBin::RealBin; + +use DataChecksums::Utils; + +my $primary = PostgreSQL::Test::Cluster->new('primary'); +$primary->init(allows_streaming => 1, no_data_checksums => 1); +$primary->append_conf( + 'postgresql.conf', qq( +autovacuum = off +checkpoint_timeout = 1h +max_wal_size = 10GB +full_page_writes = off +)); +$primary->start; + +$primary->safe_psql('postgres', 'CREATE EXTENSION pg_buffercache;'); +$primary->safe_psql('postgres', + 'CREATE TABLE t AS SELECT generate_series(1,10000) AS a;'); + +$primary->backup('backup'); +my $standby = PostgreSQL::Test::Cluster->new('standby'); +$standby->init_from_backup($primary, 'backup', has_streaming => 1); +$standby->append_conf( + 'postgresql.conf', qq( +shared_buffers = 512MB +checkpoint_timeout = 1h +max_wal_size = 10GB +bgwriter_lru_maxpages = 0 +)); +$standby->start; +$primary->wait_for_catchup($standby); + +test_checksum_state($primary, 'off'); +test_checksum_state($standby, 'off'); + +# Establish the restartpoint that recovery will resume from after the crash. +$primary->safe_psql('postgres', 'CHECKPOINT;'); +$primary->wait_for_catchup($standby); +$standby->safe_psql('postgres', 'CHECKPOINT;'); + +# Dirty the pages of "t" without full page images, then push them out to disk +# on the standby while checksums are still off. +$primary->safe_psql('postgres', 'UPDATE t SET a = a + 1;'); +$primary->wait_for_catchup($standby); + +my $relpath = $standby->safe_psql('postgres', + "SELECT pg_relation_filepath('t'::regclass);"); +my $evicted = $standby->safe_psql('postgres', + "SELECT pg_buffercache_evict_relation('t'::regclass);"); +note("evict_relation on the standby: $evicted, relpath $relpath"); + +# Online enable on the primary; the standby replays it into its own buffers, +# where the rewritten pages stay dirty (no restartpoint, no bgwriter). +enable_data_checksums($primary, wait => 'on'); +$primary->wait_for_catchup($standby); +wait_for_checksum_state($standby, 'on'); + +my ($ctl) = run_command([ 'pg_controldata', $standby->data_dir ]); +my ($before) = $ctl =~ /Data page checksum version:\s+(\d+)/; +note("standby control file before the promotion: $before"); + +# Promote. The end-of-recovery record persists the live state without any +# buffer flush; the checkpoint requested afterwards is not immediate. +$standby->promote; +$standby->poll_query_until('postgres', 'SELECT NOT pg_is_in_recovery();') + or die 'timed out waiting for the promotion'; +$standby->stop('immediate'); + +($ctl) = run_command([ 'pg_controldata', $standby->data_dir ]); +my ($after) = $ctl =~ /Data page checksum version:\s+(\d+)/; +my ($ckpt) = $ctl =~ /Latest checkpoint location:\s+(\S+)/; +my ($redo) = $ctl =~ /Latest checkpoint's REDO location:\s+(\S+)/; +note( + "standby control file after the promotion: $after, checkpoint $ckpt, redo $redo" +); + +my $page; +open(my $fh, '<', $standby->data_dir . '/' . $relpath) or die $!; +binmode $fh; +read($fh, $page, 8192); +close($fh); +my ($pd_checksum) = unpack('x8 v', $page); +note("on-disk pd_checksum of t block 0: $pd_checksum"); + +my $started = $standby->start(fail_ok => 1); +ok($started, + 'promoted node restarts after crashing before its first checkpoint'); + +my $log = PostgreSQL::Test::Utils::slurp_file($standby->logfile); +unlike( + $log, + qr/page verification failed/, + 'no checksum verification failures while replaying'); +unlike($log, qr/invalid page in block/, 'no invalid pages while replaying'); + +if ($started) +{ + my ($rc, $stdout, $stderr) = + $standby->psql('postgres', 'SELECT count(*) FROM t;'); + is($rc, 0, 'table readable after the crash restart') + or diag("stderr: $stderr"); + $standby->stop('immediate'); +} +else +{ + my @lines = grep { /FATAL|PANIC|invalid page|verification failed/ } + split(/\n/, $log); + diag("log tail:\n" . join("\n", @lines)); + fail('table readable after the crash restart'); +} + +$primary->stop; + +done_testing(); diff --git a/src/test/modules/test_checksums/t/019_restartpoint_race.pl b/src/test/modules/test_checksums/t/019_restartpoint_race.pl new file mode 100644 index 00000000000..65aa834f366 --- /dev/null +++ b/src/test/modules/test_checksums/t/019_restartpoint_race.pl @@ -0,0 +1,138 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group + +# Test a restartpoint whose flush races the replay of an online enable. +# +# Replay keeps running while CheckPointGuts() writes out the buffer pool, so +# the state at the end of the flush can be newer than the one the flush ran +# under. Persisting it would claim checksums for pages the flush wrote out +# before the transition, against a redo pointer that predates it. + +use strict; +use warnings FATAL => 'all'; + +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +use FindBin; +use lib $FindBin::RealBin; + +use DataChecksums::Utils; + +if ($ENV{enable_injection_points} ne 'yes') +{ + plan skip_all => 'Injection points not supported by this build'; +} + +my $primary = PostgreSQL::Test::Cluster->new('primary'); +$primary->init(allows_streaming => 1, no_data_checksums => 1); +$primary->append_conf( + 'postgresql.conf', qq( +autovacuum = off +checkpoint_timeout = 1h +max_wal_size = 10GB +full_page_writes = off +)); +$primary->start; + +$primary->safe_psql('postgres', 'CREATE EXTENSION injection_points;'); +$primary->safe_psql('postgres', 'CREATE EXTENSION pg_buffercache;'); +$primary->safe_psql('postgres', + 'CREATE TABLE t AS SELECT generate_series(1,10000) AS a;'); + +$primary->backup('backup'); +my $standby = PostgreSQL::Test::Cluster->new('standby'); +$standby->init_from_backup($primary, 'backup', has_streaming => 1); +$standby->append_conf( + 'postgresql.conf', qq( +shared_buffers = 512MB +checkpoint_timeout = 1h +max_wal_size = 10GB +bgwriter_lru_maxpages = 0 +)); +$standby->start; +$primary->wait_for_catchup($standby); + +# C1: the checkpoint the pending restartpoint will target. +$primary->safe_psql('postgres', 'CHECKPOINT;'); +$primary->wait_for_catchup($standby); + +# Dirty the pages of "t" without full page images and push them to disk on +# the standby while it is still "off". +$primary->safe_psql('postgres', 'UPDATE t SET a = a + 1;'); +$primary->wait_for_catchup($standby); + +my $relpath = $standby->safe_psql('postgres', + "SELECT pg_relation_filepath('t'::regclass);"); +my $evicted = $standby->safe_psql('postgres', + "SELECT pg_buffercache_evict_relation('t'::regclass);"); +note("evict_relation on the standby: $evicted, relpath $relpath"); + +# Start a restartpoint for C1 and hold it right after CheckPointGuts(). +$standby->safe_psql('postgres', + "SELECT injection_points_attach('create-restart-point','wait');"); +my $bg = $standby->background_psql('postgres'); +$bg->query_until(qr//, "\\echo restartpoint\nCHECKPOINT;\n"); +$standby->poll_query_until('postgres', + "SELECT count(*) > 0 FROM pg_stat_activity WHERE wait_event = 'create-restart-point';" +) or die 'timed out waiting for the restartpoint injection point'; + +# The startup process keeps replaying while the checkpointer is held: the +# whole online enable lands, and the rewritten pages stay dirty. +enable_data_checksums($primary, wait => 'on'); +$primary->wait_for_catchup($standby); +wait_for_checksum_state($standby, 'on'); + +# Let the restartpoint finish; it persists the state it samples now. +$standby->safe_psql('postgres', + "SELECT injection_points_wakeup('create-restart-point');"); +$standby->poll_query_until('postgres', + "SELECT count(*) = 0 FROM pg_stat_activity WHERE wait_event = 'create-restart-point';" +) or die 'timed out waiting for the restartpoint to finish'; +$standby->safe_psql('postgres', + "SELECT injection_points_detach('create-restart-point');"); + +$standby->stop('immediate'); + +my ($ctl) = run_command([ 'pg_controldata', $standby->data_dir ]); +my ($after) = $ctl =~ /Data page checksum version:\s+(\d+)/; +my ($redo) = $ctl =~ /Latest checkpoint's REDO location:\s+(\S+)/; +note("standby control file after the restartpoint: $after, redo $redo"); + +my $page; +open(my $fh, '<', $standby->data_dir . '/' . $relpath) or die $!; +binmode $fh; +read($fh, $page, 8192); +close($fh); +my ($pd_checksum) = unpack('x8 v', $page); +note("on-disk pd_checksum of t block 0: $pd_checksum"); + +my $started = $standby->start(fail_ok => 1); +ok($started, 'standby restarts after a restartpoint that raced the enable'); + +my $log = PostgreSQL::Test::Utils::slurp_file($standby->logfile); +unlike( + $log, + qr/page verification failed/, + 'no checksum verification failures while replaying'); +unlike($log, qr/invalid page in block/, 'no invalid pages while replaying'); + +if ($started) +{ + my ($rc, $stdout, $stderr) = + $standby->psql('postgres', 'SELECT count(*) FROM t;'); + is($rc, 0, 'table readable after the crash restart') + or diag("stderr: $stderr"); + $standby->stop('immediate'); +} +else +{ + my @lines = grep { /FATAL|PANIC|invalid page|verification failed/ } + split(/\n/, $log); + diag("log tail:\n" . join("\n", @lines)); + fail('table readable after the crash restart'); +} + +$primary->stop; + +done_testing(); diff --git a/src/test/modules/test_checksums/t/020_primary_enable_crash.pl b/src/test/modules/test_checksums/t/020_primary_enable_crash.pl new file mode 100644 index 00000000000..196914bc40c --- /dev/null +++ b/src/test/modules/test_checksums/t/020_primary_enable_crash.pl @@ -0,0 +1,118 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group + +# Test a primary crashing inside SetDataChecksumsOn(), just before the forced +# checkpoint that flushes the rewritten pages, with crash recovery resuming +# from a checkpoint older than the transition. +# +# The control file must still say "inprogress-on" there. Replay does not +# adopt the state of the checkpoint record it resumes from, so an "on" written +# before the flush would stay in effect while replay reads pages whose rewrite +# never reached disk. Here the pages went out through the rewriting worker's +# ring buffer, so only the control file state discriminates; see +# 022_resident_enable_crash.pl for the case where they did not. + +use strict; +use warnings FATAL => 'all'; + +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +use FindBin; +use lib $FindBin::RealBin; + +use DataChecksums::Utils; + +if ($ENV{enable_injection_points} ne 'yes') +{ + plan skip_all => 'Injection points not supported by this build'; +} + +my $node = PostgreSQL::Test::Cluster->new('primary_enable_crash'); +$node->init(no_data_checksums => 1); +$node->append_conf( + 'postgresql.conf', qq( +autovacuum = off +checkpoint_timeout = 1h +max_wal_size = 10GB +full_page_writes = off +shared_buffers = 512MB +bgwriter_lru_maxpages = 0 +)); +$node->start; + +$node->safe_psql('postgres', 'CREATE EXTENSION injection_points;'); +$node->safe_psql('postgres', 'CREATE EXTENSION pg_buffercache;'); + +test_checksum_state($node, 'off'); + +$node->safe_psql('postgres', + 'CREATE TABLE t AS SELECT generate_series(1,10000) AS a;'); + +# Establish the checkpoint that crash recovery will resume from. +$node->safe_psql('postgres', 'CHECKPOINT;'); + +# Dirty the pages of "t" without full page images, then push them out to disk +# while checksums are still off. +$node->safe_psql('postgres', 'UPDATE t SET a = a + 1;'); +my $relpath = + $node->safe_psql('postgres', "SELECT pg_relation_filepath('t'::regclass);"); +my $evicted = $node->safe_psql('postgres', + "SELECT pg_buffercache_evict_relation('t'::regclass);"); +note("evict_relation on t: $evicted, relpath $relpath"); + +# Stop the enable right after the control file has been updated to "on" but +# before the checkpoint that flushes the rewritten pages. +$node->safe_psql('postgres', + "SELECT injection_points_attach('datachecksums-on-before-checkpoint','wait');" +); +$node->safe_psql('postgres', 'SELECT pg_enable_data_checksums();'); + +$node->poll_query_until('postgres', + "SELECT count(*) > 0 FROM pg_stat_activity WHERE wait_event = 'datachecksums-on-before-checkpoint';" +) or die 'timed out waiting for the injection point'; + +my ($ctl) = run_command([ 'pg_controldata', $node->data_dir ]); +my ($ctl_state) = $ctl =~ /Data page checksum version:\s+(\d+)/; +note("control file data_checksum_version before the crash: $ctl_state"); +is($ctl_state, '3', + 'control file still says "inprogress-on" before the checkpoint'); + +$node->stop('immediate'); + +# Show the on-disk checksum field of the first page of "t". +my $page; +open(my $fh, '<', $node->data_dir . '/' . $relpath) or die $!; +binmode $fh; +read($fh, $page, 8192); +close($fh); +my ($pd_checksum) = unpack('x8 v', $page); +note("on-disk pd_checksum of t block 0: $pd_checksum"); + +my $started = $node->start(fail_ok => 1); +ok($started, 'primary restarts after crashing inside the online enable'); + +my $log = PostgreSQL::Test::Utils::slurp_file($node->logfile); +unlike( + $log, + qr/page verification failed/, + 'no checksum verification failures while replaying'); +unlike($log, qr/invalid page in block/, 'no invalid pages while replaying'); + +if ($started) +{ + my ($rc, $stdout, $stderr) = + $node->psql('postgres', 'SELECT count(*) FROM t;'); + is($rc, 0, 'table readable after the crash restart') + or diag("stderr: $stderr"); + $node->stop('immediate'); +} +else +{ + my @lines = grep { /FATAL|PANIC|invalid page|verification failed/ } + split(/\n/, $log); + diag("log tail:\n" . join("\n", @lines)); + fail('table readable after the crash restart'); +} + +done_testing(); diff --git a/src/test/modules/test_checksums/t/021_standby_shutdown_catchup.pl b/src/test/modules/test_checksums/t/021_standby_shutdown_catchup.pl new file mode 100644 index 00000000000..699dae4542a --- /dev/null +++ b/src/test/modules/test_checksums/t/021_standby_shutdown_catchup.pl @@ -0,0 +1,93 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group + +# Test a standby stopped after replaying an online enable, but before the +# checkpoint record that follows it on the primary. +# +# The shutdown restartpoint has no new checkpoint record to work from and is +# skipped, so the control file would keep the in-progress state the transition +# passed through, even though replay left the node at "on" and rewrote every +# page. pg_checksums reads that field and refuses to run on an in-progress +# state, so the shutdown has to flush and catch it up instead. + +use strict; +use warnings FATAL => 'all'; + +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +use FindBin; +use lib $FindBin::RealBin; + +use DataChecksums::Utils; + +if ($ENV{enable_injection_points} ne 'yes') +{ + plan skip_all => 'Injection points not supported by this build'; +} + +my $primary = PostgreSQL::Test::Cluster->new('primary'); +$primary->init(allows_streaming => 1, no_data_checksums => 1); +$primary->append_conf( + 'postgresql.conf', qq( +autovacuum = off +checkpoint_timeout = 1h +max_wal_size = 10GB +)); +$primary->start; + +$primary->safe_psql('postgres', 'CREATE EXTENSION injection_points;'); +$primary->safe_psql('postgres', + 'CREATE TABLE t AS SELECT generate_series(1,10000) AS a;'); + +$primary->backup('backup'); +my $standby = PostgreSQL::Test::Cluster->new('standby'); +$standby->init_from_backup($primary, 'backup', has_streaming => 1); +$standby->append_conf( + 'postgresql.conf', qq( +checkpoint_timeout = 1h +max_wal_size = 10GB +)); +$standby->start; +$primary->wait_for_catchup($standby); + +# Anchor the standby's last restartpoint here, so that nothing replayed from +# now on gives the shutdown restartpoint a newer checkpoint record to use. +$primary->safe_psql('postgres', 'CHECKPOINT;'); +$primary->wait_for_catchup($standby); +$standby->safe_psql('postgres', 'CHECKPOINT;'); + +# Hold the enable right after the state change record, before the checkpoint +# it requests once the transition is complete. +$primary->safe_psql('postgres', + "SELECT injection_points_attach('datachecksums-on-before-checkpoint','wait');" +); +enable_data_checksums($primary); +$primary->poll_query_until('postgres', + "SELECT count(*) > 0 FROM pg_stat_activity WHERE wait_event = 'datachecksums-on-before-checkpoint';" +) or die 'timed out waiting for the injection point'; +wait_for_checksum_state($primary, 'on'); + +# Flush the state change record out to the standby without writing a +# checkpoint record of any kind. +$primary->safe_psql('postgres', 'CREATE TABLE flush_marker (a int);'); +$primary->wait_for_catchup($standby); +wait_for_checksum_state($standby, 'on'); + +$standby->stop; + +my ($ctl) = run_command([ 'pg_controldata', $standby->data_dir ]); +my ($state) = $ctl =~ /Data page checksum version:\s+(\d+)/; +is($state, '1', + 'shutdown catches the control file up with the replayed state'); + +command_ok([ 'pg_checksums', '--check', '-D', $standby->data_dir ], + 'pg_checksums verifies the stopped standby'); + +$primary->safe_psql('postgres', + "SELECT injection_points_wakeup('datachecksums-on-before-checkpoint');"); +$primary->safe_psql('postgres', + "SELECT injection_points_detach('datachecksums-on-before-checkpoint');"); +$primary->stop; + +done_testing(); diff --git a/src/test/modules/test_checksums/t/022_resident_enable_crash.pl b/src/test/modules/test_checksums/t/022_resident_enable_crash.pl new file mode 100644 index 00000000000..95f230ba39e --- /dev/null +++ b/src/test/modules/test_checksums/t/022_resident_enable_crash.pl @@ -0,0 +1,138 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group + +# Test a primary crashing inside SetDataChecksumsOn(), just before the forced +# checkpoint that flushes the rewritten pages, when those pages are resident +# in shared buffers. +# +# The rewriting worker reads through a BAS_VACUUM ring, which writes the pages +# back as the ring recycles, but a page already resident in shared buffers is +# not read through the ring: ReadBufferExtended() hands back the existing +# buffer, and it stays dirty until a checkpoint. The control file may +# therefore not say "on" before that checkpoint has run, or crash recovery +# would resume from a checkpoint older than the transition with verification +# already enabled, and replay records that read those still-unchecksummed +# pages. full_page_writes is off so that the records do not simply overwrite +# the pages with a full page image. + +use strict; +use warnings FATAL => 'all'; + +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +use FindBin; +use lib $FindBin::RealBin; + +use DataChecksums::Utils; + +if ($ENV{enable_injection_points} ne 'yes') +{ + plan skip_all => 'Injection points not supported by this build'; +} + +my $node = PostgreSQL::Test::Cluster->new('resident_enable_crash'); +$node->init(no_data_checksums => 1); +$node->append_conf( + 'postgresql.conf', qq( +autovacuum = off +checkpoint_timeout = 1h +max_wal_size = 10GB +wal_level = replica +full_page_writes = off +wal_log_hints = off +shared_buffers = 512MB +bgwriter_lru_maxpages = 0 +)); +$node->start; + +$node->safe_psql('postgres', 'CREATE EXTENSION injection_points;'); +$node->safe_psql('postgres', 'CREATE EXTENSION pg_buffercache;'); + +test_checksum_state($node, 'off'); + +$node->safe_psql('postgres', + 'CREATE TABLE t AS SELECT generate_series(1,100000) AS a;'); +my $relpath = + $node->safe_psql('postgres', "SELECT pg_relation_filepath('t'::regclass);"); + +# Establish the checkpoint that crash recovery will resume from, with the +# pages of "t" written out while checksums are still off. +$node->safe_psql('postgres', 'CHECKPOINT;'); + +# Dirty those pages again without emitting full page images, and leave them in +# shared buffers. Replay of these records has to read the pages from disk. +$node->safe_psql('postgres', 'UPDATE t SET a = a + 1;'); + +my $dirty_before = $node->safe_psql('postgres', + "SELECT count(*) FROM pg_buffercache " + . "WHERE relfilenode = pg_relation_filenode('t'::regclass) " + . "AND relforknumber = 0 AND isdirty;"); +cmp_ok($dirty_before, '>', 0, + 'pages of t are resident and dirty before enabling checksums'); + +# Hold the enabling right before the checkpoint that flushes the rewritten +# pages. +$node->safe_psql('postgres', + "SELECT injection_points_attach('datachecksums-on-before-checkpoint','wait');" +); + +enable_data_checksums($node); +$node->wait_for_event('datachecksums launcher', + 'datachecksums-on-before-checkpoint'); + +my ($ctl) = run_command([ 'pg_controldata', $node->data_dir ]); +my ($ctl_state) = $ctl =~ /Data page checksum version:\s+(\d+)/; +note("control file data_checksum_version before the crash: $ctl_state"); +is($ctl_state, '3', + 'control file still says "inprogress-on" before the checkpoint'); + +# The rewritten pages must still be sitting dirty in shared buffers, or the +# window this test is about does not exist. +my $dirty_after = $node->safe_psql('postgres', + "SELECT count(*) FROM pg_buffercache " + . "WHERE relfilenode = pg_relation_filenode('t'::regclass) " + . "AND relforknumber = 0 AND isdirty;"); +cmp_ok($dirty_after, '>', 0, + 'rewritten pages of t are still dirty before the checkpoint'); + +$node->stop('immediate'); + +# The on-disk copy is the one written before the transition, without a +# checksum. +my $page; +open(my $fh, '<', $node->data_dir . '/' . $relpath) or die $!; +binmode $fh; +read($fh, $page, 8192); +close($fh); +my ($pd_checksum) = unpack('x8 v', $page); +note("on-disk pd_checksum of t block 0: $pd_checksum"); +is($pd_checksum, 0, 'block 0 of t on disk carries no checksum'); + +my $started = $node->start(fail_ok => 1); +ok($started, 'primary restarts after crashing inside the online enable'); + +my $log = PostgreSQL::Test::Utils::slurp_file($node->logfile); +unlike( + $log, + qr/page verification failed/, + 'no checksum verification failures while replaying'); +unlike($log, qr/invalid page in block/, 'no invalid pages while replaying'); + +if ($started) +{ + my ($rc, $stdout, $stderr) = + $node->psql('postgres', 'SELECT count(*) FROM t;'); + is($rc, 0, 'table readable after the crash restart') + or diag("stderr: $stderr"); + $node->stop('immediate'); +} +else +{ + my @lines = grep { /FATAL|PANIC|invalid page|verification failed/ } + split(/\n/, $log); + diag("log tail:\n" . join("\n", @lines)); + fail('table readable after the crash restart'); +} + +done_testing(); diff --git a/src/test/modules/test_checksums/t/023_concurrent_checkpoint_enable.pl b/src/test/modules/test_checksums/t/023_concurrent_checkpoint_enable.pl new file mode 100644 index 00000000000..2d1b78fc5cd --- /dev/null +++ b/src/test/modules/test_checksums/t/023_concurrent_checkpoint_enable.pl @@ -0,0 +1,114 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group + +# A checkpoint that runs between the XLOG2_CHECKSUMS("on") record and the +# control file write at the end of SetDataChecksumsOn() must not make a crash +# throw the completed transition away. +# +# SetDataChecksumsOn() writes the record, flips shared memory to "on", emits +# the barrier and only then requests the checkpoint that flushes the rewritten +# pages; the control file is written after that checkpoint returns. A crash in +# that window is harmless only as long as recovery still starts before the +# record. Any checkpoint completing in the window moves the redo point past +# the record, so recovery would never see it, would come up with the control +# file's "inprogress-on" and StartupXLOG() would demote that to "off", even +# though every page on disk carries a checksum by then. +# +# CreateCheckPoint() therefore persists the state the checkpoint ran under, the +# same way CreateRestartPoint() does. The window is naturally reachable: +# checkpoint_timeout, max_wal_size, an explicit CHECKPOINT, pg_basebackup or +# pg_backup_start can all fire there. Here it is made deterministic by holding +# the launcher at the datachecksums-on-before-checkpoint injection point and +# checkpointing from another session. + +use strict; +use warnings FATAL => 'all'; + +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +use FindBin; +use lib $FindBin::RealBin; + +use DataChecksums::Utils; + +if ($ENV{enable_injection_points} ne 'yes') +{ + plan skip_all => 'Injection points not supported by this build'; +} + +my $node = PostgreSQL::Test::Cluster->new('concurrent_checkpoint'); +$node->init(no_data_checksums => 1); +$node->append_conf( + 'postgresql.conf', qq( +autovacuum = off +checkpoint_timeout = 1h +max_wal_size = 10GB +)); +$node->start; + +$node->safe_psql('postgres', 'CREATE EXTENSION injection_points;'); + +test_checksum_state($node, 'off'); + +$node->safe_psql('postgres', + 'CREATE TABLE t AS SELECT generate_series(1,10000) AS a;'); +my $relpath = + $node->safe_psql('postgres', "SELECT pg_relation_filepath('t'::regclass);"); + +# Hold the launcher after the record, the shared memory flip and the barrier, +# but before the checkpoint SetDataChecksumsOn() requests itself. +$node->safe_psql('postgres', + "SELECT injection_points_attach('datachecksums-on-before-checkpoint','wait');" +); + +enable_data_checksums($node); +$node->wait_for_event('datachecksums launcher', + 'datachecksums-on-before-checkpoint'); + +# Every backend already sees "on" and writes checksums. +test_checksum_state($node, 'on'); + +# ... while the control file still says "inprogress-on". +my ($ctl) = run_command([ 'pg_controldata', $node->data_dir ]); +my ($before) = $ctl =~ /Data page checksum version:\s+(\d+)/; +is($before, '3', 'control file says "inprogress-on" inside the window'); + +# A concurrent checkpoint. It flushes the rewritten pages and moves the redo +# point past the XLOG2_CHECKSUMS("on") record, so it has to record the "on" +# state in the control file as well. +$node->safe_psql('postgres', 'CHECKPOINT;'); + +($ctl) = run_command([ 'pg_controldata', $node->data_dir ]); +my ($after) = $ctl =~ /Data page checksum version:\s+(\d+)/; +is($after, '1', + 'the concurrent checkpoint records "on" in the control file'); + +$node->stop('immediate'); + +# The checkpoint flushed the rewritten pages, so they carry a checksum on +# disk: the transition really did complete. +my $page; +open(my $fh, '<', $node->data_dir . '/' . $relpath) or die $!; +binmode $fh; +read($fh, $page, 8192); +close($fh); +my ($pd_checksum) = unpack('x8 v', $page); +note("on-disk pd_checksum of t block 0: $pd_checksum"); +isnt($pd_checksum, 0, 'block 0 of t on disk carries a checksum'); + +my $log_offset = -s $node->logfile; +$node->start; + +# The transition is complete on disk, so the cluster has to come back "on". +test_checksum_state($node, 'on'); + +my $log = PostgreSQL::Test::Utils::slurp_file($node->logfile, $log_offset); +unlike( + $log, + qr/enabling data checksums was interrupted/, + 'the completed transition is not reported as interrupted'); + +$node->stop; + +done_testing(); diff --git a/src/test/modules/test_checksums/t/024_enable_crash_after_checkpoint.pl b/src/test/modules/test_checksums/t/024_enable_crash_after_checkpoint.pl new file mode 100644 index 00000000000..e75f17394cb --- /dev/null +++ b/src/test/modules/test_checksums/t/024_enable_crash_after_checkpoint.pl @@ -0,0 +1,101 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group + +# A crash between the checkpoint an online enable requests and the control +# file write that follows it must not lose the transition. +# +# The last steps of SetDataChecksumsOn() are +# +# WAL record -> shmem -> barrier -> checkpoint -> persist +# +# The checkpoint is what makes "on" safe to persist, but it also moves the +# redo point above the XLOG2_CHECKSUMS record that carries the new state. +# Crash recovery started from that checkpoint therefore never replays the +# record, so without the state the checkpoint itself persists, a crash in the +# remaining window would bring the cluster back at "inprogress-on", which +# StartupXLOG() resolves to "off", discarding a transition whose pages are all +# on disk with a checksum. +# +# t/023 exercises the same window through a checkpoint requested by another +# session. This test closes it from the other side: the transition's own +# checkpoint is the one that moves the redo point, so persisting the state +# from CreateCheckPoint() is what has to save it, not the write below the +# injection point. + +use strict; +use warnings FATAL => 'all'; + +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +use FindBin; +use lib $FindBin::RealBin; + +use DataChecksums::Utils; + +if ($ENV{enable_injection_points} ne 'yes') +{ + plan skip_all => 'Injection points not supported by this build'; +} + +my $node = PostgreSQL::Test::Cluster->new('crash_after_checkpoint'); +$node->init(no_data_checksums => 1); +$node->append_conf( + 'postgresql.conf', qq( +autovacuum = off +checkpoint_timeout = 1h +max_wal_size = 10GB +)); +$node->start; + +$node->safe_psql('postgres', 'CREATE EXTENSION injection_points;'); + +test_checksum_state($node, 'off'); + +$node->safe_psql('postgres', + 'CREATE TABLE t AS SELECT generate_series(1,10000) AS a;'); + +# Hold the launcher after the checkpoint that licenses "on" has completed and +# before the state reaches the control file. +$node->safe_psql('postgres', + "SELECT injection_points_attach('datachecksums-on-after-checkpoint','wait');" +); + +enable_data_checksums($node); +$node->wait_for_event('datachecksums launcher', + 'datachecksums-on-after-checkpoint'); + +# The transition is complete as far as the running cluster is concerned. +test_checksum_state($node, 'on'); + +# The checkpoint has already recorded it, so the pending write below the +# injection point has nothing left to do. +my ($ctl) = run_command([ 'pg_controldata', $node->data_dir ]); +my ($version) = $ctl =~ /Data page checksum version:\s+(\d+)/; +is($version, '1', + 'the requested checkpoint recorded "on" in the control file'); + +# Crash before the control file write that follows the checkpoint. +$node->stop('immediate'); + +$node->start; + +# Every page on disk carries a checksum and the checkpoint that flushed them +# completed, so the cluster has to come back verifying them. +test_checksum_state($node, 'on'); + +is($node->safe_psql('postgres', 'SELECT count(*) FROM t;'), + '10000', 'relation readable after the crash'); + +my $log = PostgreSQL::Test::Utils::slurp_file($node->logfile); +unlike( + $log, + qr/enabling data checksums was interrupted/, + 'the completed transition is not reported as interrupted'); + +# The data directory must be one the offline tools accept. +$node->stop; +$node->command_ok([ 'pg_checksums', '--check', '-D', $node->data_dir ], + 'pg_checksums accepts the data directory'); + +done_testing(); diff --git a/src/test/modules/test_checksums/t/025_cascade_divergence.pl b/src/test/modules/test_checksums/t/025_cascade_divergence.pl new file mode 100644 index 00000000000..b4b474fa583 --- /dev/null +++ b/src/test/modules/test_checksums/t/025_cascade_divergence.pl @@ -0,0 +1,115 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group + +# An offline data checksum state change made on one node of a cascading setup +# is reported all the way down the chain. +# +# CheckReplayedDataChecksumState() is reached from xlog_redo() when a +# checkpoint record (XLOG_CHECKPOINT_SHUTDOWN or XLOG_CHECKPOINT_REDO) is +# replayed after consistency has been reached. Those records are written by +# the root primary only and are relayed verbatim by every intermediate +# standby, so a cascaded standby that was changed offline learns about the +# divergence too. An intermediate standby's own restartpoints are not +# WAL-logged and cannot surface it. +# +# Note that this makes detection checkpoint-driven: an idle or read-only +# primary surfaces the divergence no sooner than its next checkpoint, so up to +# checkpoint_timeout may pass before the operator sees anything. Closing that +# window would require comparing the state when a walreceiver connects. + +use strict; +use warnings FATAL => 'all'; + +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +use FindBin; +use lib $FindBin::RealBin; + +use DataChecksums::Utils; + +# checkpoint_timeout is deliberately long so that the only checkpoint record in +# this test is the one requested explicitly below. +my $conf = qq( +autovacuum = off +checkpoint_timeout = 1h +max_wal_size = 10GB +); + +my $primary = PostgreSQL::Test::Cluster->new('cascade_primary'); +$primary->init(allows_streaming => 1, no_data_checksums => 1); +$primary->append_conf('postgresql.conf', $conf); +$primary->start; +$primary->safe_psql('postgres', + 'CREATE TABLE t AS SELECT generate_series(1,1000) AS a;'); + +$primary->backup('backup'); +my $standby = PostgreSQL::Test::Cluster->new('cascade_standby'); +$standby->init_from_backup($primary, 'backup', has_streaming => 1); +$standby->append_conf('postgresql.conf', $conf); +$standby->start; + +$standby->backup('backup'); +my $cascade = PostgreSQL::Test::Cluster->new('cascade_cascade'); +$cascade->init_from_backup($standby, 'backup', has_streaming => 1); +$cascade->append_conf('postgresql.conf', $conf); +$cascade->start; + +$primary->wait_for_catchup($standby); +$standby->wait_for_catchup($cascade); + +test_checksum_state($primary, 'off'); +test_checksum_state($standby, 'off'); +test_checksum_state($cascade, 'off'); + +# Change the cascaded standby offline, and only it. +$cascade->stop; +$cascade->command_ok([ 'pg_checksums', '--enable', '-D', $cascade->data_dir ], + 'pg_checksums enables checksums on the cascaded standby only'); + +my $logstart = -s $cascade->logfile; +$cascade->start; + +test_checksum_state($cascade, 'on'); + +# Ordinary WAL traffic carries no state, so the cascaded standby keeps +# streaming from a chain whose state is "off" without noticing. +$primary->safe_psql('postgres', 'INSERT INTO t VALUES (1);'); +$primary->wait_for_catchup($standby); +$standby->wait_for_catchup($cascade); + +is($cascade->safe_psql('postgres', 'SELECT count(*) FROM t;'), + '1001', 'cascaded standby keeps streaming from a divergent chain'); + +# Restartpoints on the intermediate standby are not WAL-logged, so this does +# not surface anything on the cascaded standby either. +$standby->safe_psql('postgres', 'CHECKPOINT;'); +$primary->safe_psql('postgres', 'INSERT INTO t VALUES (2);'); +$primary->wait_for_catchup($standby); +$standby->wait_for_catchup($cascade); + +my $log = PostgreSQL::Test::Utils::slurp_file($cascade->logfile, $logstart); +unlike( + $log, + qr/data checksum state .* does not match/, + 'an intermediate restartpoint does not surface the divergence'); + +# A checkpoint on the root primary does: the record is relayed down the whole +# chain, so both the direct and the cascaded standby compare it against their +# own state. wait_for_catchup() below waits for replay, not just receipt, so +# the record has been through xlog_redo() on both nodes once it returns. +$primary->safe_psql('postgres', 'CHECKPOINT;'); +$primary->wait_for_catchup($standby); +$standby->wait_for_catchup($cascade); + +$log = PostgreSQL::Test::Utils::slurp_file($cascade->logfile, $logstart); +like( + $log, + qr/data checksum state .* does not match/, + 'the cascaded standby reports the divergence on the primary checkpoint'); + +$cascade->stop; +$standby->stop; +$primary->stop; + +done_testing(); -- 2.39.3 (Apple Git-146)