From 894e6a8781423da21b61875f65ddd29e0de71c17 Mon Sep 17 00:00:00 2001 From: Bertrand Drouvot Date: Fri, 17 Jul 2026 13:25:10 +0000 Subject: [PATCH v4 4/6] pgstat: move lock statistics to new per-backend infrastructure PGSTAT_KIND_BACKEND stores each backend's lock and I/O statistics together in one variable-numbered entry keyed by ProcNumber. Move lock statistics into a dedicated fixed array associated with the lock statistics kind and indexed by ProcNumber. The global lock stats hold data for backends that have exited, while the array holds statistics for active backends. Build pg_stat_lock snapshots by copying the global stats and adding every active per-backend entry. Transfer the current process's entry into the global stats before invalidating it at process exit or ProcNumber reuse. Transfer all entries before a clean shutdown writes the statistics file. Remove lock counters from PGSTAT_KIND_BACKEND and use the new infrastructure in pg_stat_get_backend_lock(). This also makes lock statistics available for auxiliary and shared-memory-only workers. Update the documentation to describe the new behavior. Author: Bertrand Drouvot Reviewed-by: Michael Paquier Reviewed-by: Andres Freund Discussion: https://postgr.es/m/anCjoFkGah/uB7lw%40bdtpg --- doc/src/sgml/monitoring.sgml | 8 +- src/backend/utils/activity/pgstat.c | 10 +- src/backend/utils/activity/pgstat_backend.c | 72 --------- src/backend/utils/activity/pgstat_lock.c | 170 +++++++++++++++++--- src/backend/utils/adt/pgstatfuncs.c | 34 +++- src/include/pgstat.h | 12 +- src/include/utils/pgstat_internal.h | 21 ++- src/tools/pgindent/typedefs.list | 1 + 8 files changed, 205 insertions(+), 123 deletions(-) 3.7% doc/src/sgml/ 73.9% src/backend/utils/activity/ 10.1% src/backend/utils/adt/ 7.0% src/include/utils/ 4.8% src/include/ diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index f52b859c141..0602d80e599 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -5849,10 +5849,6 @@ description | Waiting for a newly initialized WAL file to reach durable storage Returns lock statistics about the backend with the specified process ID. The output fields are exactly the same as the ones in the pg_stat_lock view. - - - The function does not return lock statistics for the checkpointer, - the background writer, the startup process and the autovacuum launcher. @@ -6025,7 +6021,9 @@ description | Waiting for a newly initialized WAL file to reach durable storage lock: Reset all the counters shown in the - pg_stat_lock view. + pg_stat_lock view, as well as per-backend + lock statistics returned by + pg_stat_get_backend_lock. diff --git a/src/backend/utils/activity/pgstat.c b/src/backend/utils/activity/pgstat.c index 9ea09243ccc..1affde4df05 100644 --- a/src/backend/utils/activity/pgstat.c +++ b/src/backend/utils/activity/pgstat.c @@ -479,6 +479,11 @@ static const PgStat_KindInfo pgstat_kind_builtin_infos[PGSTAT_KIND_BUILTIN_SIZE] .init_shmem_cb = pgstat_lock_init_shmem_cb, .reset_all_cb = pgstat_lock_reset_all_cb, .snapshot_cb = pgstat_lock_snapshot_cb, + + .per_backend_entry_size = sizeof(PgStatShared_LockBackendEntry), + .per_backend_data_off = offsetof(PgStatShared_LockBackendEntry, stats), + .per_backend_data_len = sizeof(PgStat_Lock), + .per_backend_acc_cb = pgstat_lock_per_backend_acc_cb, }, [PGSTAT_KIND_SLRU] = { @@ -632,6 +637,7 @@ pgstat_before_server_shutdown(int code, Datum arg) { /* Transfer all live per-backend stats before writing the stats file */ pgstat_wal_acc_all_backends(); + pgstat_lock_acc_all_backends(); pgStatLocal.shmem->is_shutdown = true; pgstat_write_statsfile(); @@ -676,8 +682,9 @@ pgstat_shutdown_hook(int code, Datum arg) if (!pgstat_drop_entry(PGSTAT_KIND_BACKEND, InvalidOid, MyProcNumber, false)) pgstat_request_entry_refs_gc(); - /* Accumulate per-backend WAL stats into the global stats */ + /* Accumulate per-backend stats into the global stats */ pgstat_wal_acc_backend_cb(); + pgstat_lock_acc_backend_cb(); pgstat_detach_shmem(); @@ -705,6 +712,7 @@ pgstat_initialize(void) * the current process's entries. */ pgstat_wal_acc_backend_cb(); + pgstat_lock_acc_backend_cb(); /* * Activate per-backend statistics entries here. This also covers diff --git a/src/backend/utils/activity/pgstat_backend.c b/src/backend/utils/activity/pgstat_backend.c index 76b970a8c41..8dda2cd88c1 100644 --- a/src/backend/utils/activity/pgstat_backend.c +++ b/src/backend/utils/activity/pgstat_backend.c @@ -39,7 +39,6 @@ */ static PgStat_BackendPending PendingBackendStats; static bool backend_has_iostats = false; -static bool backend_has_lockstats = false; /* * Utility routines to report I/O stats for backends, kept here to avoid @@ -79,39 +78,6 @@ pgstat_count_backend_io_op(IOObject io_object, IOContext io_context, pgstat_report_fixed = true; } -/* - * Utility routines to report lock stats for backends, kept here to avoid - * exposing PendingBackendStats to the outside world. - */ -void -pgstat_count_backend_lock_waits(uint8 locktag_type, PgStat_Counter usecs) -{ - if (!pgstat_tracks_backend_bktype(MyBackendType)) - return; - - Assert(locktag_type <= LOCKTAG_LAST_TYPE); - - PendingBackendStats.pending_lock.stats[locktag_type].waits++; - PendingBackendStats.pending_lock.stats[locktag_type].wait_time += usecs; - - backend_has_lockstats = true; - pgstat_report_fixed = true; -} - -void -pgstat_count_backend_lock_fastpath_exceeded(uint8 locktag_type) -{ - if (!pgstat_tracks_backend_bktype(MyBackendType)) - return; - - Assert(locktag_type <= LOCKTAG_LAST_TYPE); - - PendingBackendStats.pending_lock.stats[locktag_type].fastpath_exceeded++; - - backend_has_lockstats = true; - pgstat_report_fixed = true; -} - /* * Returns statistics of a backend by proc number. */ @@ -236,36 +202,6 @@ pgstat_flush_backend_entry_io(PgStat_EntryRef *entry_ref) backend_has_iostats = false; } -/* - * Flush out locally pending backend lock statistics. Locking is managed - * by the caller. - */ -static void -pgstat_flush_backend_entry_lock(PgStat_EntryRef *entry_ref) -{ - PgStatShared_Backend *shbackendent; - PgStat_PendingLock *bktype_shstats; - - if (!backend_has_lockstats) - return; - - shbackendent = (PgStatShared_Backend *) entry_ref->shared_stats; - bktype_shstats = &shbackendent->stats.lock_stats; - - for (int i = 0; i <= LOCKTAG_LAST_TYPE; i++) - { -#define LOCKSTAT_ACC(fld) \ - (bktype_shstats->stats[i].fld += PendingBackendStats.pending_lock.stats[i].fld) - LOCKSTAT_ACC(waits); - LOCKSTAT_ACC(wait_time); - LOCKSTAT_ACC(fastpath_exceeded); -#undef LOCKSTAT_ACC - } - - MemSet(&PendingBackendStats.pending_lock, 0, sizeof(PgStat_PendingLock)); - backend_has_lockstats = false; -} - /* * Flush out locally pending backend statistics * @@ -285,10 +221,6 @@ pgstat_flush_backend(bool nowait, uint32 flags) if ((flags & PGSTAT_BACKEND_FLUSH_IO) && backend_has_iostats) has_pending_data = true; - /* Some lock data pending? */ - if ((flags & PGSTAT_BACKEND_FLUSH_LOCK) && backend_has_lockstats) - has_pending_data = true; - if (!has_pending_data) return false; @@ -301,9 +233,6 @@ pgstat_flush_backend(bool nowait, uint32 flags) if (flags & PGSTAT_BACKEND_FLUSH_IO) pgstat_flush_backend_entry_io(entry_ref); - if (flags & PGSTAT_BACKEND_FLUSH_LOCK) - pgstat_flush_backend_entry_lock(entry_ref); - pgstat_unlock_entry(entry_ref); return false; @@ -342,7 +271,6 @@ pgstat_create_backend(ProcNumber procnum) MemSet(&PendingBackendStats, 0, sizeof(PgStat_BackendPending)); backend_has_iostats = false; - backend_has_lockstats = false; } /* diff --git a/src/backend/utils/activity/pgstat_lock.c b/src/backend/utils/activity/pgstat_lock.c index c20c7599683..ab226dfd893 100644 --- a/src/backend/utils/activity/pgstat_lock.c +++ b/src/backend/utils/activity/pgstat_lock.c @@ -8,6 +8,12 @@ * access / storage implementation and the details about individual types * of statistics. * + * Lock statistics use a per-backend array to avoid double-counting. Each + * backend flushes lock stats (waits, wait_time, fastpath_exceeded per lock + * tag type) to its own ProcNumber-indexed entry. The global pg_stat_lock view + * aggregates the global stats (which holds stats from exited backends) plus + * all active per-backend entries. + * * Copyright (c) 2021-2026, PostgreSQL Global Development Group * * IDENTIFICATION @@ -40,7 +46,7 @@ pgstat_lock_flush(bool nowait) } /* - * Flush out locally pending lock statistics + * Flush out locally pending lock statistics to the per-backend array entry. * * If no stats have been recorded, this function returns false. * @@ -50,31 +56,27 @@ pgstat_lock_flush(bool nowait) bool pgstat_lock_flush_cb(bool nowait) { - LWLock *lckstat_lock; - PgStatShared_Lock *shstats; + PgStatShared_LockBackendEntry *entry; if (!have_lockstats) return false; - shstats = &pgStatLocal.shmem->lock; - lckstat_lock = &shstats->lock; + entry = pgstat_lock_my_per_backend_entry(PGSTAT_KIND_LOCK, nowait); - if (!nowait) - LWLockAcquire(lckstat_lock, LW_EXCLUSIVE); - else if (!LWLockConditionalAcquire(lckstat_lock, LW_EXCLUSIVE)) - return true; + if (entry == NULL) + return nowait; for (int i = 0; i <= LOCKTAG_LAST_TYPE; i++) { #define LOCKSTAT_ACC(fld) \ - (shstats->stats.stats[i].fld += PendingLockStats.stats[i].fld) + (entry->stats.stats[i].fld += PendingLockStats.stats[i].fld) LOCKSTAT_ACC(waits); LOCKSTAT_ACC(wait_time); LOCKSTAT_ACC(fastpath_exceeded); #undef LOCKSTAT_ACC } - LWLockRelease(lckstat_lock); + LWLockRelease(&entry->header.lock); memset(&PendingLockStats, 0, sizeof(PendingLockStats)); have_lockstats = false; @@ -93,28 +95,117 @@ pgstat_lock_init_shmem_cb(void *stats) void pgstat_lock_reset_all_cb(TimestampTz ts) { - LWLock *lckstat_lock = &pgStatLocal.shmem->lock.lock; + PgStatShared_Lock *shmem = &pgStatLocal.shmem->lock; + + /* + * Hold the kind lock while resetting both the global stats and active + * entries. Transfers hold the same lock, so pre-reset counters cannot be + * moved into the global stats after it is reset. + */ + LWLockAcquire(&shmem->lock, LW_EXCLUSIVE); + memset(&shmem->stats, 0, sizeof(shmem->stats)); + shmem->stats.stat_reset_timestamp = ts; - LWLockAcquire(lckstat_lock, LW_EXCLUSIVE); + /* Reset all per-backend entries, since they contribute to the global view */ + for (ProcNumber procnum = 0; procnum < NumPerBackendStatSlots; procnum++) + { + PgStatShared_LockBackendEntry *entry; - pgStatLocal.shmem->lock.stats.stat_reset_timestamp = ts; + entry = pgstat_get_per_backend_entry(PGSTAT_KIND_LOCK, procnum); - memset(pgStatLocal.shmem->lock.stats.stats, 0, - sizeof(pgStatLocal.shmem->lock.stats.stats)); + LWLockAcquire(&entry->header.lock, LW_EXCLUSIVE); + if (entry->header.backend_type != B_INVALID) + { + memset(&entry->stats.stats, 0, sizeof(entry->stats.stats)); + entry->stats.stat_reset_timestamp = ts; + } + LWLockRelease(&entry->header.lock); + } - LWLockRelease(lckstat_lock); + LWLockRelease(&shmem->lock); } +/* + * Build lock stats snapshot by aggregating global stats and all active + * per-backend entries. + */ void pgstat_lock_snapshot_cb(void) { - LWLock *lckstat_lock = &pgStatLocal.shmem->lock.lock; + PgStatShared_Lock *shmem = &pgStatLocal.shmem->lock; + PgStat_Lock *snap = &pgStatLocal.snapshot.lock; - LWLockAcquire(lckstat_lock, LW_SHARED); + /* + * Prevent entries from moving to the global stats between copying it and + * scanning the per-backend array. + */ + LWLockAcquire(&shmem->lock, LW_SHARED); + memcpy(snap, &shmem->stats, sizeof(PgStat_Lock)); - pgStatLocal.snapshot.lock = pgStatLocal.shmem->lock.stats; + /* Add in all active per-backend entries */ + pgstat_per_backend_snapshot(PGSTAT_KIND_LOCK, snap); - LWLockRelease(lckstat_lock); + LWLockRelease(&shmem->lock); +} + +/* + * Accumulate one per-backend lock entry into a snapshot or the global stats. + */ +void +pgstat_lock_per_backend_acc_cb(void *dst, void *entry) +{ + PgStat_Lock *stats = dst; + PgStatShared_LockBackendEntry *e = (PgStatShared_LockBackendEntry *) entry; + + for (int j = 0; j <= LOCKTAG_LAST_TYPE; j++) + { + stats->stats[j].waits += e->stats.stats[j].waits; + stats->stats[j].wait_time += e->stats.stats[j].wait_time; + stats->stats[j].fastpath_exceeded += e->stats.stats[j].fastpath_exceeded; + } +} + +#define LOCK_ACCUMULATE_COUNTERS(dst, src) \ +do { \ + for (int _i = 0; _i <= LOCKTAG_LAST_TYPE; _i++) \ + { \ + (dst)[_i].waits += (src)[_i].waits; \ + (dst)[_i].wait_time += (src)[_i].wait_time; \ + (dst)[_i].fastpath_exceeded += (src)[_i].fastpath_exceeded; \ + } \ +} while (0) + +/* + * Accumulate a backend's lock stats into the global stats, then invalidate the + * entry. + * + * Called at backend exit after the final flush, or when a ProcNumber is + * being reused. + */ +void +pgstat_lock_acc_backend_cb(void) +{ + pgstat_acc_my_per_backend(PGSTAT_KIND_LOCK, &pgStatLocal.shmem->lock.lock); +} + +/* + * Accumulate all per-backend lock stats entries into the global stats and + * invalidate them. Called at clean server shutdown to ensure all flushed data + * is preserved in the stats file. + */ +void +pgstat_lock_acc_all_backends(void) +{ + pgstat_acc_all_per_backend(PGSTAT_KIND_LOCK, &pgStatLocal.shmem->lock.lock); +} + +/* + * Returns per-backend lock statistics for the given ProcNumber. + */ +PgStat_Lock * +pgstat_fetch_stat_backend_lock(ProcNumber procnum) +{ + return (PgStat_Lock *) pgstat_fetch_per_backend(PGSTAT_KIND_LOCK, procnum); } /* @@ -131,8 +222,6 @@ pgstat_count_lock_fastpath_exceeded(uint8 locktag_type) PendingLockStats.stats[locktag_type].fastpath_exceeded++; have_lockstats = true; pgstat_report_fixed = true; - - pgstat_count_backend_lock_fastpath_exceeded(locktag_type); } /* @@ -149,6 +238,39 @@ pgstat_count_lock_waits(uint8 locktag_type, PgStat_Counter usecs) PendingLockStats.stats[locktag_type].wait_time += usecs; have_lockstats = true; pgstat_report_fixed = true; +} + +/* + * Reset a backend's lock stats. Accumulate the entry's counters into the + * global stats, then zero the stats and set the reset timestamp. + */ +void +pgstat_lock_reset_backend_cb(ProcNumber procnum, TimestampTz ts) +{ + PgStatShared_Lock *shmem = &pgStatLocal.shmem->lock; + PgStatShared_LockBackendEntry *entry; + + entry = pgstat_get_per_backend_entry(PGSTAT_KIND_LOCK, procnum); + if (entry == NULL) + return; + + LWLockAcquire(&shmem->lock, LW_EXCLUSIVE); + LWLockAcquire(&entry->header.lock, LW_EXCLUSIVE); + + if (entry->header.backend_type == B_INVALID) + { + LWLockRelease(&entry->header.lock); + LWLockRelease(&shmem->lock); + return; + } + + /* Accumulate current stats into global before zeroing */ + LOCK_ACCUMULATE_COUNTERS(shmem->stats.stats, entry->stats.stats); + + /* Zero stats and set reset timestamp */ + memset(&entry->stats.stats, 0, sizeof(entry->stats.stats)); + entry->stats.stat_reset_timestamp = ts; - pgstat_count_backend_lock_waits(locktag_type, usecs); + LWLockRelease(&entry->header.lock); + LWLockRelease(&shmem->lock); } diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index ef541205db4..7922519f426 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -1872,19 +1872,36 @@ pg_stat_get_backend_lock(PG_FUNCTION_ARGS) { int pid; ReturnSetInfo *rsinfo; - PgStat_Backend *backend_stats; + PGPROC *proc; + ProcNumber procnum; + PgBackendStatus *beentry; + PgStat_Lock *lock_stats; InitMaterializedSRF(fcinfo, 0); rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; pid = PG_GETARG_INT32(0); - backend_stats = pgstat_fetch_stat_backend_by_pid(pid, NULL); - if (!backend_stats) + proc = BackendPidGetProc(pid); + + if (!proc) + proc = AuxiliaryPidGetProc(pid); + if (!proc) return (Datum) 0; - pg_stat_lock_build_tuples(rsinfo, backend_stats->lock_stats.stats, - backend_stats->stat_reset_timestamp); + procnum = GetNumberFromPGProc(proc); + beentry = pgstat_get_beentry_by_proc_number(procnum); + + if (!beentry || beentry->st_procpid != pid) + return (Datum) 0; + + lock_stats = pgstat_fetch_stat_backend_lock(procnum); + + if (!lock_stats) + return (Datum) 0; + + pg_stat_lock_build_tuples(rsinfo, lock_stats->stats, + lock_stats->stat_reset_timestamp); return (Datum) 0; } @@ -2212,13 +2229,14 @@ pg_stat_reset_backend_stats(PG_FUNCTION_ARGS) PG_RETURN_VOID(); /* - * Accumulate the backend's WAL stats into the global stats, then zero the - * entry. + * Accumulate the backend's WAL and lock stats into the global stats, then + * zero the entries. */ ts = GetCurrentTimestamp(); pgstat_wal_reset_backend_cb(procNumber, ts); + pgstat_lock_reset_backend_cb(procNumber, ts); - /* Reset IO and Lock stats still in PGSTAT_KIND_BACKEND */ + /* Reset IO stats still in PGSTAT_KIND_BACKEND */ pgstat_reset(PGSTAT_KIND_BACKEND, InvalidOid, procNumber); PG_RETURN_VOID(); diff --git a/src/include/pgstat.h b/src/include/pgstat.h index 9a71b8d3cd8..960f0001bd6 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -593,7 +593,6 @@ typedef struct PgStat_Backend { TimestampTz stat_reset_timestamp; PgStat_BktypeIO io_stats; - PgStat_PendingLock lock_stats; } PgStat_Backend; /* --------- @@ -606,12 +605,6 @@ typedef struct PgStat_BackendPending * Backend statistics store the same amount of IO data as PGSTAT_KIND_IO. */ PgStat_PendingIO pending_io; - - /* - * Backend statistics store the same amount of lock data as - * PGSTAT_KIND_LOCK. - */ - PgStat_PendingLock pending_lock; } PgStat_BackendPending; /* @@ -664,9 +657,6 @@ extern void pgstat_count_backend_io_op(IOObject io_object, IOOp io_op, uint32 cnt, uint64 bytes); -/* used by pgstat_lock.c for lock stats tracked in backends */ -extern void pgstat_count_backend_lock_waits(uint8 locktag_type, PgStat_Counter usecs); -extern void pgstat_count_backend_lock_fastpath_exceeded(uint8 locktag_type); extern PgStat_Backend *pgstat_fetch_stat_backend(ProcNumber procNumber); extern PgStat_Backend *pgstat_fetch_stat_backend_by_pid(int pid, @@ -723,6 +713,8 @@ extern void pgstat_count_lock_fastpath_exceeded(uint8 locktag_type); extern void pgstat_count_lock_waits(uint8 locktag_type, PgStat_Counter usecs); extern PgStat_Lock *pgstat_fetch_stat_lock(void); +extern PgStat_Lock *pgstat_fetch_stat_backend_lock(ProcNumber procnum); +extern void pgstat_lock_reset_backend_cb(ProcNumber procnum, TimestampTz ts); /* * Functions in pgstat_database.c diff --git a/src/include/utils/pgstat_internal.h b/src/include/utils/pgstat_internal.h index d0c9379b42b..474be775430 100644 --- a/src/include/utils/pgstat_internal.h +++ b/src/include/utils/pgstat_internal.h @@ -502,7 +502,10 @@ typedef struct PgStatShared_IO typedef struct PgStatShared_Lock { - /* lock protects ->stats */ + /* + * The lock protects global stats and serializes snapshots and resets with + * transfers from per-backend entries. + */ LWLock lock; PgStat_Lock stats; } PgStatShared_Lock; @@ -532,6 +535,16 @@ typedef struct PgStatShared_PerBackendEntry BackendType backend_type; } PgStatShared_PerBackendEntry; +/* + * Per-backend entry for lock statistics, stored in an array indexed by + * ProcNumber. + */ +typedef struct PgStatShared_LockBackendEntry +{ + PgStatShared_PerBackendEntry header; + PgStat_Lock stats; +} PgStatShared_LockBackendEntry; + /* * Per-backend entry for WAL statistics, stored in an array indexed by * ProcNumber. @@ -811,8 +824,7 @@ extern void pgstat_archiver_snapshot_cb(void); /* flags for pgstat_flush_backend() */ #define PGSTAT_BACKEND_FLUSH_IO (1 << 0) /* Flush I/O statistics */ -#define PGSTAT_BACKEND_FLUSH_LOCK (1 << 2) /* Flush lock statistics */ -#define PGSTAT_BACKEND_FLUSH_ALL (PGSTAT_BACKEND_FLUSH_IO | PGSTAT_BACKEND_FLUSH_LOCK) +#define PGSTAT_BACKEND_FLUSH_ALL (PGSTAT_BACKEND_FLUSH_IO) extern bool pgstat_flush_backend(bool nowait, uint32 flags); extern bool pgstat_backend_flush_cb(bool nowait); @@ -878,6 +890,9 @@ extern bool pgstat_lock_flush_cb(bool nowait); extern void pgstat_lock_init_shmem_cb(void *stats); extern void pgstat_lock_reset_all_cb(TimestampTz ts); extern void pgstat_lock_snapshot_cb(void); +extern void pgstat_lock_acc_backend_cb(void); +extern void pgstat_lock_acc_all_backends(void); +extern void pgstat_lock_per_backend_acc_cb(void *dst, void *entry); /* * Functions in pgstat_relation.c diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 95b87d32a62..549141f592d 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2302,6 +2302,7 @@ PgStatShared_HashEntry PgStatShared_Index PgStatShared_IO PgStatShared_Lock +PgStatShared_LockBackendEntry PgStatShared_PerBackendEntry PgStatShared_Relation PgStatShared_ReplSlot -- 2.34.1