From 185f1d75df91c405e509a09b9dc3030404784ac2 Mon Sep 17 00:00:00 2001 From: Andrey Borodin Date: Wed, 29 Jul 2026 16:10:26 +0500 Subject: [PATCH v3 1/2] Prevent early data access after crash recovery After crash recovery, locally committed WAL might not yet have reached the configured synchronous standbys. Allowing clients to connect immediately can expose changes that a subsequent failover could lose. Add post_recovery_sync_level to keep a session out until the end-of-recovery WAL location reaches the synchronous standbys at its configured level. The setting can be assigned per role or database, while separate shared completion state preserves the distinction between remote_write, remote flush, and remote_apply. Allow WAL sender connections so standbys can catch up. Treat an empty synchronous_standby_names as an explicit operator bypass, and remember each completed level until the next postmaster start. Discussion: https://postgr.es/m/0B44E464-BA62-4056-9465-3320DD2D0302@yandex-team.ru --- doc/src/sgml/config.sgml | 41 +++++++++ src/backend/access/transam/xlog.c | 2 + src/backend/replication/syncrep.c | 87 +++++++++++++++++++ src/backend/utils/init/postinit.c | 12 +++ src/backend/utils/misc/guc_parameters.dat | 7 ++ src/backend/utils/misc/postgresql.conf.sample | 2 + src/include/replication/syncrep.h | 4 + src/include/replication/walsender_private.h | 8 ++ src/test/recovery/t/007_sync_rep.pl | 78 +++++++++++++++++ 9 files changed, 241 insertions(+) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index aa7b1bd75d2..ef950f88a04 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -5054,6 +5054,47 @@ restore_command = 'copy "C:\\server\\archivedir\\%f" "%p"' # Windows + + post_recovery_sync_level (enum) + + post_recovery_sync_level configuration parameter + + + + + Specifies how far the end-of-recovery WAL location must be processed + by the synchronous standbys before ordinary client connections are + allowed after server startup. Valid values are + remote_apply, on, + remote_write, local, and + off (the default). The meanings of these values + are the same as for . + Values local and off disable + this check. + + + WAL sender connections are allowed while the check is pending, so + that standbys can connect and make progress. Once a synchronization + level has been reached, sessions requiring that level or a weaker + level remain able to connect until the next server start, even if + those standbys subsequently disconnect. Setting + to an empty value + explicitly bypasses the check. + + + This parameter can be set differently for individual roles or + databases. For example, a role configured with + remote_apply can remain unable to connect after a + role configured with remote_write is allowed. + Only superusers and users with the appropriate SET + privilege can change this setting. The value is consulted during + connection establishment; changing it in an established session does + not affect that session. + + + + synchronous_standby_names (string) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..a6be81f1bb7 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -83,6 +83,7 @@ #include "replication/slot.h" #include "replication/slotsync.h" #include "replication/snapbuild.h" +#include "replication/syncrep.h" #include "replication/walreceiver.h" #include "replication/walsender.h" #include "storage/bufmgr.h" @@ -6284,6 +6285,7 @@ StartupXLOG(void) */ endOfRecoveryInfo = FinishWalRecovery(); EndOfLog = endOfRecoveryInfo->endOfLog; + SyncRepInitPostRecovery(EndOfLog); EndOfLogTLI = endOfRecoveryInfo->endOfLogTLI; abortedRecPtr = endOfRecoveryInfo->abortedRecPtr; missingContrecPtr = endOfRecoveryInfo->missingContrecPtr; diff --git a/src/backend/replication/syncrep.c b/src/backend/replication/syncrep.c index d870f09e0a0..2864bac3b8b 100644 --- a/src/backend/replication/syncrep.c +++ b/src/backend/replication/syncrep.c @@ -89,6 +89,7 @@ /* User-settable parameters for sync rep */ char *SyncRepStandbyNames; +int post_recovery_sync_level = SYNCHRONOUS_COMMIT_OFF; #define SyncStandbysDefined() \ (SyncRepStandbyNames != NULL && SyncRepStandbyNames[0] != '\0') @@ -1145,3 +1146,89 @@ assign_synchronous_commit(int newval, void *extra) break; } } + +void +SyncRepInitPostRecovery(XLogRecPtr recovery_end_lsn) +{ + LWLockAcquire(SyncRepLock, LW_EXCLUSIVE); + WalSndCtl->post_recovery_sync_lsn = recovery_end_lsn; + memset(WalSndCtl->post_recovery_sync_complete, false, + sizeof(WalSndCtl->post_recovery_sync_complete)); + LWLockRelease(SyncRepLock); +} + +/* + * Return whether the end-of-recovery LSN has reached the configured + * synchronous standbys at the level requested by this session. Once a level + * is reached, it remains complete until the next postmaster start. + */ +bool +SyncRepPostRecoveryComplete(void) +{ + bool bypassed = false; + bool can_complete = false; + bool complete; + int mode; + + switch (post_recovery_sync_level) + { + case SYNCHRONOUS_COMMIT_REMOTE_WRITE: + mode = SYNC_REP_WAIT_WRITE; + break; + case SYNCHRONOUS_COMMIT_REMOTE_FLUSH: + mode = SYNC_REP_WAIT_FLUSH; + break; + case SYNCHRONOUS_COMMIT_REMOTE_APPLY: + mode = SYNC_REP_WAIT_APPLY; + break; + default: + return true; + } + + LWLockAcquire(SyncRepLock, LW_SHARED); + complete = WalSndCtl->post_recovery_sync_complete[mode]; + if (!complete && + (WalSndCtl->sync_standbys_status & SYNC_STANDBY_INIT) != 0) + { + if ((WalSndCtl->sync_standbys_status & SYNC_STANDBY_DEFINED) == 0) + can_complete = true; + else if (WalSndCtl->lsn[mode] >= + WalSndCtl->post_recovery_sync_lsn) + can_complete = true; + } + LWLockRelease(SyncRepLock); + + if (can_complete) + { + LWLockAcquire(SyncRepLock, LW_EXCLUSIVE); + complete = WalSndCtl->post_recovery_sync_complete[mode]; + if (!complete && + (WalSndCtl->sync_standbys_status & SYNC_STANDBY_INIT) != 0) + { + if ((WalSndCtl->sync_standbys_status & SYNC_STANDBY_DEFINED) == 0) + { + memset(WalSndCtl->post_recovery_sync_complete, true, + sizeof(WalSndCtl->post_recovery_sync_complete)); + bypassed = complete = true; + } + else if (WalSndCtl->lsn[mode] >= + WalSndCtl->post_recovery_sync_lsn) + complete = true; + + WalSndCtl->post_recovery_sync_complete[mode] = complete; + } + LWLockRelease(SyncRepLock); + } + + /* + * Only the process that changes the shared state to complete reports + * this. Later connections observe the completed state, so this message + * is logged only once. + */ + if (bypassed) + ereport(LOG, + (errmsg("post-recovery synchronization wait skipped"), + errdetail("\"synchronous_standby_names\" is empty."))); + + return complete; +} diff --git a/src/backend/utils/init/postinit.c b/src/backend/utils/init/postinit.c index 3d8c9bdebd5..1c24d594e0a 100644 --- a/src/backend/utils/init/postinit.c +++ b/src/backend/utils/init/postinit.c @@ -43,6 +43,7 @@ #include "postmaster/postmaster.h" #include "replication/slot.h" #include "replication/slotsync.h" +#include "replication/syncrep.h" #include "replication/walsender.h" #include "storage/aio_subsys.h" #include "storage/bufmgr.h" @@ -1254,6 +1255,17 @@ InitPostgres(const char *in_dbname, Oid dboid, if (PostAuthDelay > 0) pg_usleep(PostAuthDelay * 1000000L); + /* Check whether this role's post-recovery synchronization is complete */ + if (AmRegularBackendProcess() && + !am_walsender && + !SyncRepPostRecoveryComplete()) + { + ereport(FATAL, + (errcode(ERRCODE_CANNOT_CONNECT_NOW), + errmsg("cannot connect until synchronous replication is established with standbys"), + errdetail("The end-of-recovery WAL location has not been replicated according to \"post_recovery_sync_level\"."))); + } + /* * Initialize various default states that can't be set up until we've * selected the active user and gotten the right GUC settings. diff --git a/src/backend/utils/misc/guc_parameters.dat b/src/backend/utils/misc/guc_parameters.dat index adb72361ce0..bae9e178746 100644 --- a/src/backend/utils/misc/guc_parameters.dat +++ b/src/backend/utils/misc/guc_parameters.dat @@ -2395,6 +2395,13 @@ max => 'INT_MAX / 1000000', }, +{ name => 'post_recovery_sync_level', type => 'enum', context => 'PGC_SUSET', group => 'REPLICATION_PRIMARY', + short_desc => 'Sets the synchronization level required before a session can connect after recovery.', + variable => 'post_recovery_sync_level', + boot_val => 'SYNCHRONOUS_COMMIT_OFF', + options => 'synchronous_commit_options', +}, + # Not for general use { name => 'pre_auth_delay', type => 'int', context => 'PGC_SIGHUP', group => 'DEVELOPER_OPTIONS', short_desc => 'Sets the amount of time to wait before authentication on connection startup.', diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index 7958653077b..a51c7b1d185 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -370,6 +370,8 @@ # These settings are ignored on a standby server. +#post_recovery_sync_level = off # off, local, remote_write, + # remote_apply, or on #synchronous_standby_names = '' # standby servers that provide sync rep # method to choose sync standbys, number of sync standbys, # and comma-separated list of application_name diff --git a/src/include/replication/syncrep.h b/src/include/replication/syncrep.h index b42b5862a70..f6cf18b636e 100644 --- a/src/include/replication/syncrep.h +++ b/src/include/replication/syncrep.h @@ -72,6 +72,7 @@ typedef struct SyncRepConfigData } SyncRepConfigData; extern PGDLLIMPORT SyncRepConfigData *SyncRepConfig; +extern PGDLLIMPORT int post_recovery_sync_level; /* user-settable parameters for synchronous replication */ extern PGDLLIMPORT char *SyncRepStandbyNames; @@ -104,4 +105,7 @@ extern void syncrep_yyerror(SyncRepConfigData **syncrep_parse_result_p, char **s extern void syncrep_scanner_init(const char *str, yyscan_t *yyscannerp); extern void syncrep_scanner_finish(yyscan_t yyscanner); +extern void SyncRepInitPostRecovery(XLogRecPtr recovery_end_lsn); +extern bool SyncRepPostRecoveryComplete(void); + #endif /* _SYNCREP_H */ diff --git a/src/include/replication/walsender_private.h b/src/include/replication/walsender_private.h index b0c80deeb24..7b16de72ef3 100644 --- a/src/include/replication/walsender_private.h +++ b/src/include/replication/walsender_private.h @@ -95,6 +95,14 @@ typedef struct */ XLogRecPtr lsn[NUM_SYNC_REP_WAIT_MODE]; + /* + * End-of-recovery LSN that must reach the synchronous standbys before + * sessions requiring each synchronization level are allowed, and whether + * each condition has been satisfied. Protected by SyncRepLock. + */ + XLogRecPtr post_recovery_sync_lsn; + bool post_recovery_sync_complete[NUM_SYNC_REP_WAIT_MODE]; + /* * Status of data related to the synchronous standbys. Waiting backends * can't reload the config file safely, so checkpointer updates this value diff --git a/src/test/recovery/t/007_sync_rep.pl b/src/test/recovery/t/007_sync_rep.pl index 8eab64998cd..224f1a82e06 100644 --- a/src/test/recovery/t/007_sync_rep.pl +++ b/src/test/recovery/t/007_sync_rep.pl @@ -218,4 +218,82 @@ standby4|1|quorum), 'all standbys are considered as candidates for quorum sync standbys', 'ANY 2(*)'); +# Check that ordinary connections are rejected after crash recovery until +# the end-of-recovery LSN reaches the configured synchronous standby. +for my $standby ( + $node_standby_1, $node_standby_2, $node_standby_3, $node_standby_4) +{ + $standby->stop; +} + +$node_primary->safe_psql( + 'postgres', q[ + SET synchronous_commit = 'local'; + CREATE ROLE post_recovery_bypass LOGIN; + ALTER ROLE post_recovery_bypass + SET post_recovery_sync_level = 'off'; + ALTER SYSTEM SET post_recovery_sync_level = 'remote_write'; + ALTER SYSTEM SET synchronous_standby_names = + 'ANY 2 (standby1, standby2)'; + CREATE TABLE post_recovery_sync_test (a int); + INSERT INTO post_recovery_sync_test VALUES (1);]); +$node_primary->stop('immediate'); +$node_primary->start; + +my $startup_stderr = ''; +isnt( + $node_primary->psql( + 'postgres', 'SELECT 1', stderr => \$startup_stderr), + 0, + 'connection is rejected while post-recovery sync is pending'); +like( + $startup_stderr, + qr/cannot connect until synchronous replication is established/, + 'post-recovery sync rejection reports the reason'); + +is( + $node_primary->safe_psql( + 'postgres', 'SELECT 1', connstr => 'user=post_recovery_bypass'), + '1', + 'role with post-recovery synchronization disabled can connect'); + +$node_standby_1->start; +$node_standby_1->poll_query_until( + 'postgres', 'SELECT count(*) = 1 FROM post_recovery_sync_test') + or die "first standby did not catch up"; +pass('first standby replays WAL generated before primary restart'); + +$startup_stderr = ''; +isnt( + $node_primary->psql( + 'postgres', 'SELECT 1', stderr => \$startup_stderr), + 0, + 'ordinary connection remains rejected with only one standby caught up'); + +$node_standby_2->start; +$node_primary->poll_query_until( + 'postgres', 'SELECT count(*) = 1 FROM post_recovery_sync_test') + or die "ordinary connections did not open after quorum caught up"; +pass('ordinary connections open after standby quorum catches up'); + +$node_standby_2->poll_query_until( + 'postgres', 'SELECT count(*) = 1 FROM post_recovery_sync_test') + or die "second standby did not catch up"; +pass('second standby replays WAL generated before primary restart'); + +# An empty synchronous_standby_names is an explicit way for an operator to +# bypass the startup check when no standby can catch up. +$node_primary->safe_psql( + 'postgres', q[ + ALTER SYSTEM SET synchronous_standby_names = ''; + SELECT pg_reload_conf();]); +$node_standby_1->stop; +$node_standby_2->stop; +$node_primary->stop('immediate'); +$node_primary->start; +is( + $node_primary->safe_psql('postgres', 'SELECT 1'), + '1', + 'empty synchronous_standby_names bypasses post-recovery sync'); + done_testing(); -- 2.50.1 (Apple Git-155)