From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 From: Jingtang Zhang Date: Sun, 08 Nov 2026 17:53:21 +0800 Subject: [PATCH v2] Periodically refresh database frozen IDs during VACUUM Prioritizing tables that need freezing does not promptly advance the database's frozen-ID horizon when that horizon is only recomputed after an entire batch of tables. With many tables, wraparound protection can remain active even after the oldest tables have been vacuumed. Track table-level freezing progress in shared database statistics and periodically refresh database horizons between tables, for both autovacuum and manual multi-table VACUUM. Batch refreshes at ten-minute intervals, with periodic attempts gated by the shared transaction-ID limit state. Use conditional locking so a worker can continue processing tables when another updater holds the database lock. Record only the progress observed before each completed refresh, leaving concurrent progress pending and avoiding acknowledgment across a statistics reset. Retain the existing final refreshes, and honor the option to skip database statistics updates. --- src/backend/access/heap/vacuumlazy.c | 3 + src/backend/access/transam/varsup.c | 7 ++- src/backend/commands/vacuum.c | 68 +++++++++++++++++++++- src/backend/postmaster/autovacuum.c | 16 ++--- src/backend/storage/lmgr/lmgr.c | 13 +++++ src/backend/utils/activity/pgstat_database.c | 87 +++++++++++++++++++++++++++- src/include/commands/vacuum.h | 2 +- src/include/pgstat.h | 14 ++++- src/include/storage/lmgr.h | 1 + 9 files changed, 196 insertions(+), 15 deletions(-) diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index 3e2fc8f0b24cc31eed78c0223d489d8f041ab820..f9c87dfb775cd05d6aaf15e8945c51ca94f73b05 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -965,6 +965,9 @@ heap_vacuum_rel(Relation rel, const VacuumParams *params, vacrel->NewRelfrozenXid, vacrel->NewRelminMxid, &frozenxid_updated, &minmulti_updated, false); + if (frozenxid_updated || minmulti_updated) + pgstat_report_frozenxid_progress(); + /* * Report results to the cumulative stats system, too. * diff --git a/src/backend/access/transam/varsup.c b/src/backend/access/transam/varsup.c index 912f7cb1d2c4527d4c74a1a7ac19b69117fbead8..7ed11665fb09995fe9339a7c3bc630be0915dee0 100644 --- a/src/backend/access/transam/varsup.c +++ b/src/backend/access/transam/varsup.c @@ -511,6 +511,10 @@ SetTransactionIdLimit(TransactionId oldest_datfrozenxid, Oid oldest_datoid) * wrap limit. Also do it if oldestXid is old enough to be forcing * autovacuums or other actions; this ensures we update our state as soon * as possible once extra overhead is being incurred. + * + * Outside a transaction, only check the shared XID state. Checking whether + * the oldest database still exists requires catalog access and is deferred + * until this function is called within a transaction. */ bool ForceTransactionIdLimitUpdate(void) @@ -534,7 +538,8 @@ ForceTransactionIdLimitUpdate(void) return true; /* this shouldn't happen anymore either */ if (TransactionIdFollowsOrEquals(nextXid, xidVacLimit)) return true; /* past xidVacLimit, don't delay updating */ - if (!SearchSysCacheExists1(DATABASEOID, ObjectIdGetDatum(oldestXidDB))) + if (IsTransactionState() && + !SearchSysCacheExists1(DATABASEOID, ObjectIdGetDatum(oldestXidDB))) return true; /* could happen, per comments above */ return false; } diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c index 3a5c1efd6acd8def1ad50195d7aa4756f2f94768..72872609ce79c16dec0e81eba6fc69187bfcb5e9 100644 --- a/src/backend/commands/vacuum.c +++ b/src/backend/commands/vacuum.c @@ -123,6 +123,7 @@ int VacuumCostBalanceLocal = 0; static List *expand_vacuum_rel(VacuumRelation *vrel, MemoryContext vac_context, int options); static List *get_all_vacuum_rels(MemoryContext vac_context, int options); +static bool vacuum_datfrozenxid_update_due(void); static void vac_truncate_clog(TransactionId frozenXID, MultiXactId minMulti, TransactionId lastSaneFrozenXid, @@ -671,6 +672,17 @@ vacuum(List *relations, const VacuumParams *params, BufferAccessStrategy bstrate } } + /* Publish freezing progress after the table's transactions end. */ + if ((params->options & VACOPT_VACUUM) && + !(params->options & VACOPT_SKIP_DATABASE_STATS) && + vacuum_datfrozenxid_update_due() && + ForceTransactionIdLimitUpdate()) + { + StartTransactionCommand(); + vac_update_datfrozenxid(true); + CommitTransactionCommand(); + } + /* * Ensure VacuumFailsafeActive has been reset before vacuuming the * next relation. @@ -709,8 +721,10 @@ vacuum(List *relations, const VacuumParams *params, BufferAccessStrategy bstrate { /* * Update pg_database.datfrozenxid, and truncate pg_xact if possible. + * Autovacuum's final update is handled by do_autovacuum(). */ - vac_update_datfrozenxid(); + if (!AmAutoVacuumWorkerProcess()) + vac_update_datfrozenxid(false); } } @@ -1615,6 +1629,37 @@ vac_update_relstats(Relation relation, } +/* + * Check for freezing progress not covered by a completed database update, + * and a sufficiently old completion time. + * All VACUUMs in a database share the interval. A clock adjustment must not + * delay updates forever. + */ +static bool +vacuum_datfrozenxid_update_due(void) +{ + PgStat_StatDBEntry *dbentry; + bool due = false; + bool may_free; + + dbentry = pgstat_fetch_stat_dbentry_ext(MyDatabaseId, &may_free); + if (dbentry == NULL) + return false; + + if (dbentry->frozenxid_updated > dbentry->last_frozenxid_updated) + { + TimestampTz now = GetCurrentTimestamp(); + + due = dbentry->last_frozenxid_update_time == 0 || + dbentry->last_frozenxid_update_time > now || + TimestampDifferenceExceedsSeconds(dbentry->last_frozenxid_update_time, now, + VAC_DATFROZENXID_UPDATE_INTERVAL); + } + if (may_free) + pfree(dbentry); + return due; +} + /* * vac_update_datfrozenxid() -- update pg_database.datfrozenxid for our DB * @@ -1627,6 +1672,10 @@ vac_update_relstats(Relation relation, * If we are able to advance either pg_database value, also try to * truncate pg_xact and pg_multixact. * + * If nowait is true, skip the update when the database frozen-ID + * lock cannot be acquired immediately. Once acquired, the lock is + * held until transaction end. + * * We violate transaction semantics here by overwriting the database's * existing pg_database tuple with the new values. This is reasonably * safe since the new values are correct whether or not this transaction @@ -1634,7 +1683,7 @@ vac_update_relstats(Relation relation, * behind after a VACUUM. */ void -vac_update_datfrozenxid(void) +vac_update_datfrozenxid(bool nowait) { HeapTuple tuple; Form_pg_database dbform; @@ -1649,6 +1698,8 @@ vac_update_datfrozenxid(void) bool dirty = false; ScanKeyData key[1]; void *inplace_state; + PgStat_Counter saved_frozenxid_updated; + TimestampTz saved_update_time; /* * Restrict this task to one backend per database. This avoids race @@ -1656,7 +1707,16 @@ vac_update_datfrozenxid(void) * avoids calling vac_truncate_clog() with a datfrozenxid preceding a * datfrozenxid passed to an earlier vac_truncate_clog() call. */ - LockDatabaseFrozenIds(ExclusiveLock); + if (nowait) + { + if (!ConditionalLockDatabaseFrozenIds(ExclusiveLock)) + return; + } + else + LockDatabaseFrozenIds(ExclusiveLock); + + /* Only acknowledge progress that precedes this scan. */ + pgstat_prepare_frozenxid_update(&saved_frozenxid_updated, &saved_update_time); /* * Initialize the "min" calculation with @@ -1834,6 +1894,8 @@ vac_update_datfrozenxid(void) if (dirty || ForceTransactionIdLimitUpdate()) vac_truncate_clog(newFrozenXid, newMinMulti, lastSaneFrozenXid, lastSaneMinMulti); + + pgstat_report_frozenxid_update(saved_frozenxid_updated, saved_update_time); } diff --git a/src/backend/postmaster/autovacuum.c b/src/backend/postmaster/autovacuum.c index 0ec01c5da6f2a6b22f2b04e32aab1905e3220ae4..954ea2555e2f5802a3348c218ae2c806fcb450c2 100644 --- a/src/backend/postmaster/autovacuum.c +++ b/src/backend/postmaster/autovacuum.c @@ -2685,8 +2685,8 @@ deleted: MemoryContextSwitchTo(TopTransactionContext); /* - * Update pg_database.datfrozenxid, and truncate pg_xact if possible. We - * only need to do this once, not after each table. + * Force a final update of pg_database.datfrozenxid, and truncate pg_xact + * if possible. * * Even if we didn't vacuum anything, it may still be important to do * this, because one indirect effect of vac_update_datfrozenxid() is to @@ -2705,7 +2705,7 @@ deleted: * worker has already done it, or will do so when it finishes. */ if (did_vacuum || !found_concurrent_worker) - vac_update_datfrozenxid(); + vac_update_datfrozenxid(false); /* Finally close out the last transaction. */ CommitTransactionCommand(); @@ -2913,13 +2913,15 @@ table_recheck_autovac(Oid relid, HTAB *table_toast_map, /* * Select VACUUM options. Note we don't say VACOPT_PROCESS_TOAST, so - * that vacuum() skips toast relations. Also note we tell vacuum() to - * skip vac_update_datfrozenxid(); we'll do that separately. + * that vacuum() skips toast relations. + * + * We will periodically update database frozen IDs if any table's + * frozen IDs have been updated, and force a final update at the end + * of our list. */ tab->at_params.options = (dovacuum ? (VACOPT_VACUUM | - VACOPT_PROCESS_MAIN | - VACOPT_SKIP_DATABASE_STATS) : 0) | + VACOPT_PROCESS_MAIN) : 0) | (doanalyze ? VACOPT_ANALYZE : 0) | (!wraparound ? VACOPT_SKIP_LOCKED : 0); diff --git a/src/backend/storage/lmgr/lmgr.c b/src/backend/storage/lmgr/lmgr.c index 2ccf7237fee941af8522d133dab2c302806ed742..bc79d14e7687eb3a93db978416fa4df81460f8a3 100644 --- a/src/backend/storage/lmgr/lmgr.c +++ b/src/backend/storage/lmgr/lmgr.c @@ -497,6 +497,19 @@ LockDatabaseFrozenIds(LOCKMODE lockmode) (void) LockAcquire(&tag, lockmode, false, false); } +/* + * Acquire the database frozen-ID lock without waiting for another updater. + */ +bool +ConditionalLockDatabaseFrozenIds(LOCKMODE lockmode) +{ + LOCKTAG tag; + + SET_LOCKTAG_DATABASE_FROZEN_IDS(tag, MyDatabaseId); + + return (LockAcquire(&tag, lockmode, false, true) != LOCKACQUIRE_NOT_AVAIL); +} + /* * LockPage * diff --git a/src/backend/utils/activity/pgstat_database.c b/src/backend/utils/activity/pgstat_database.c index 7f3bc0165931cc11f00ebe6c3feaede65f85eda8..9737cfff34ecc85bfbd36875c8b54f8b4a812a36 100644 --- a/src/backend/utils/activity/pgstat_database.c +++ b/src/backend/utils/activity/pgstat_database.c @@ -74,6 +74,76 @@ pgstat_report_autovac(Oid dboid) pgstat_unlock_entry(entry_ref); } +/* + * Count a table's frozen-ID update immediately so that concurrent VACUUMs + * can coordinate database updates. + */ +void +pgstat_report_frozenxid_progress(void) +{ + PgStat_EntryRef *entry_ref; + PgStatShared_Database *dbentry; + + entry_ref = pgstat_get_entry_ref_locked(PGSTAT_KIND_DATABASE, + MyDatabaseId, InvalidOid, false); + dbentry = (PgStatShared_Database *) entry_ref->shared_stats; + dbentry->stats.frozenxid_updated++; + pgstat_unlock_entry(entry_ref); +} + +/* + * Initialize a zero database update time to an expired value, so a failed scan + * can be retried immediately. Save the progress count and update time under + * the same lock, so a statistics reset cannot separate them. The caller must + * already hold the database frozen-ID lock. + */ +void +pgstat_prepare_frozenxid_update(PgStat_Counter *frozenxid_updated, + TimestampTz *last_update_time) +{ + PgStat_EntryRef *entry_ref; + PgStatShared_Database *dbentry; + + entry_ref = pgstat_get_entry_ref_locked(PGSTAT_KIND_DATABASE, + MyDatabaseId, InvalidOid, false); + dbentry = (PgStatShared_Database *) entry_ref->shared_stats; + + if (dbentry->stats.last_frozenxid_update_time == 0) + dbentry->stats.last_frozenxid_update_time = GetCurrentTimestamp() - + (int64) VAC_DATFROZENXID_UPDATE_INTERVAL * USECS_PER_SEC; + + *frozenxid_updated = dbentry->stats.frozenxid_updated; + *last_update_time = dbentry->stats.last_frozenxid_update_time; + + pgstat_unlock_entry(entry_ref); +} + +/* + * Record completion even when the database's frozen IDs did not advance. + * Only acknowledge progress saved before the scan, leaving concurrent updates + * outstanding. A changed update time invalidates the saved count. + * The caller must still hold the database frozen-ID lock. + */ +void +pgstat_report_frozenxid_update(PgStat_Counter frozenxid_updated, + TimestampTz last_update_time) +{ + PgStat_EntryRef *entry_ref; + PgStatShared_Database *dbentry; + + entry_ref = pgstat_get_entry_ref_locked(PGSTAT_KIND_DATABASE, + MyDatabaseId, InvalidOid, false); + dbentry = (PgStatShared_Database *) entry_ref->shared_stats; + + if (dbentry->stats.last_frozenxid_update_time == last_update_time) + { + dbentry->stats.last_frozenxid_updated = frozenxid_updated; + dbentry->stats.last_frozenxid_update_time = GetCurrentTimestamp(); + } + + pgstat_unlock_entry(entry_ref); +} + /* * Report a Hot Standby recovery conflict. */ @@ -286,9 +356,19 @@ pgstat_report_disconnect(Oid dboid) */ PgStat_StatDBEntry * pgstat_fetch_stat_dbentry(Oid dboid) +{ + return pgstat_fetch_stat_dbentry_ext(dboid, NULL); +} + +/* + * Like pgstat_fetch_stat_dbentry(), but also return whether the caller can + * pfree() the result if desired. + */ +PgStat_StatDBEntry * +pgstat_fetch_stat_dbentry_ext(Oid dboid, bool *may_free) { return (PgStat_StatDBEntry *) - pgstat_fetch_entry(PGSTAT_KIND_DATABASE, dboid, InvalidOid, NULL); + pgstat_fetch_entry(PGSTAT_KIND_DATABASE, dboid, InvalidOid, may_free); } void @@ -460,8 +540,11 @@ pgstat_database_flush_cb(PgStat_EntryRef *entry_ref, bool nowait) PGSTAT_ACCUM_DBCOUNT(tuples_updated); PGSTAT_ACCUM_DBCOUNT(tuples_deleted); - /* last_autovac_time is reported immediately */ + /* These fields are reported immediately. */ Assert(pendingent->last_autovac_time == 0); + Assert(pendingent->frozenxid_updated == 0); + Assert(pendingent->last_frozenxid_updated == 0); + Assert(pendingent->last_frozenxid_update_time == 0); PGSTAT_ACCUM_DBCOUNT(conflict_tablespace); PGSTAT_ACCUM_DBCOUNT(conflict_lock); diff --git a/src/include/commands/vacuum.h b/src/include/commands/vacuum.h index 89fa1fcd261ca5e121622d93c65358666ddf6c36..ecb1ad474031942db5b248b1807267c93a3306e1 100644 --- a/src/include/commands/vacuum.h +++ b/src/include/commands/vacuum.h @@ -394,7 +394,7 @@ extern void vac_update_relstats(Relation relation, extern bool vacuum_get_cutoffs(Relation rel, const VacuumParams *params, struct VacuumCutoffs *cutoffs); extern bool vacuum_xid_failsafe_check(const struct VacuumCutoffs *cutoffs); -extern void vac_update_datfrozenxid(void); +extern void vac_update_datfrozenxid(bool nowait); extern void vacuum_delay_point(bool is_analyze); extern bool vacuum_is_permitted_for_relation(Oid relid, Form_pg_class reltuple, uint32 options, bool missing_ok); diff --git a/src/include/pgstat.h b/src/include/pgstat.h index 9b99062f8534d30a0972ed9f5ad2270892bd7d8b..e7ed2cbfcabba3d6f911c7d7cc4d53cd306ffb61 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -275,7 +275,7 @@ typedef struct PgStat_TableXactStatus * ------------------------------------------------------------ */ -#define PGSTAT_FILE_FORMAT_ID 0x01A5BCBD +#define PGSTAT_FILE_FORMAT_ID 0x01A5BCBE typedef struct PgStat_ArchiverStats { @@ -433,6 +433,9 @@ typedef struct PgStat_StatDBEntry PgStat_Counter tuples_updated; PgStat_Counter tuples_deleted; TimestampTz last_autovac_time; + PgStat_Counter frozenxid_updated; + PgStat_Counter last_frozenxid_updated; + TimestampTz last_frozenxid_update_time; PgStat_Counter conflict_tablespace; PgStat_Counter conflict_lock; PgStat_Counter conflict_snapshot; @@ -729,8 +732,16 @@ extern PgStat_Lock *pgstat_fetch_stat_lock(void); * Functions in pgstat_database.c */ +/* Interval for periodic database frozen-ID updates, in seconds. */ +#define VAC_DATFROZENXID_UPDATE_INTERVAL (10 * 60) + extern void pgstat_drop_database(Oid databaseid); extern void pgstat_report_autovac(Oid dboid); +extern void pgstat_report_frozenxid_progress(void); +extern void pgstat_prepare_frozenxid_update(PgStat_Counter *frozenxid_updated, + TimestampTz *last_update_time); +extern void pgstat_report_frozenxid_update(PgStat_Counter frozenxid_updated, + TimestampTz last_update_time); extern void pgstat_report_recovery_conflict(int reason); extern void pgstat_report_deadlock(void); extern void pgstat_prepare_report_checksum_failure(Oid dboid); @@ -749,6 +760,7 @@ extern void pgstat_update_parallel_workers_stats(PgStat_Counter workers_to_launc (pgStatTransactionIdleTime += (n)) extern PgStat_StatDBEntry *pgstat_fetch_stat_dbentry(Oid dboid); +extern PgStat_StatDBEntry *pgstat_fetch_stat_dbentry_ext(Oid dboid, bool *may_free); /* diff --git a/src/include/storage/lmgr.h b/src/include/storage/lmgr.h index 2a985ce5e154d5b41afca77557d85bbf83de0384..dcd4ad6a1057431d6ac9df8194f70093bd6b2e67 100644 --- a/src/include/storage/lmgr.h +++ b/src/include/storage/lmgr.h @@ -64,6 +64,7 @@ extern int RelationExtensionLockWaiterCount(Relation relation); /* Lock to recompute pg_database.datfrozenxid in the current database */ extern void LockDatabaseFrozenIds(LOCKMODE lockmode); +extern bool ConditionalLockDatabaseFrozenIds(LOCKMODE lockmode); /* Lock a page (currently only used within indexes) */ extern void LockPage(Relation relation, BlockNumber blkno, LOCKMODE lockmode); base-commit: b7eee09eab738e085d6b62d35788f542dd233877 -- 2.43.7