From 38d72b38a1ddb141d784fe31f3d40e26ce981800 Mon Sep 17 00:00:00 2001 From: Bertrand Drouvot Date: Fri, 17 Jul 2026 13:26:07 +0000 Subject: [PATCH v2 5/6] pgstat: move IO statistics to new per-backend infrastructure PGSTAT_KIND_BACKEND stores each backend's IO statistics in one variable-numbered entry keyed by ProcNumber. Move IO statistics into a dedicated ProcNumber keyed dshash associated with the fixed IO statistics kind. The global IO stats hold data for backends that have exited, while the dshash holds statistics for live backends. Build pg_stat_io snapshots by copying the global stats and adding every live per-backend entry. Transfer the current process's entry into the global stats before deleting it at process exit or ProcNumber reuse. Transfer all entries before a clean shutdown writes the statistics file. Remove IO counters from PGSTAT_KIND_BACKEND and make use of the new infrastructure in pg_stat_get_backend_io(). This also makes IO statistics available for auxiliary and shared memory-only workers. Update the documentation to describe the resulting new behavior. Author: Bertrand Drouvot Reviewed-by: Michael Paquier Discussion: https://postgr.es/m/anCjoFkGah/uB7lw%40bdtpg --- doc/src/sgml/monitoring.sgml | 9 +- src/backend/replication/walsender.c | 2 - src/backend/utils/activity/pgstat.c | 9 +- src/backend/utils/activity/pgstat_backend.c | 155 +------------ src/backend/utils/activity/pgstat_io.c | 225 ++++++++++++++----- src/backend/utils/activity/pgstat_relation.c | 2 - src/backend/utils/activity/pgstat_shmem.c | 9 +- src/backend/utils/activity/pgstat_wal.c | 1 - src/backend/utils/adt/pgstatfuncs.c | 47 ++-- src/include/pgstat.h | 34 +-- src/include/utils/pgstat_internal.h | 28 ++- src/test/regress/expected/stats.out | 23 +- src/test/regress/sql/stats.sql | 11 +- src/tools/pgindent/typedefs.list | 3 +- 14 files changed, 268 insertions(+), 290 deletions(-) 69.3% src/backend/utils/activity/ 8.5% src/backend/utils/adt/ 4.9% src/include/utils/ 5.5% src/include/ 4.3% src/test/regress/expected/ 3.2% src/test/regress/sql/ diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index 4a625ba5259..cf0c26194cb 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -5797,12 +5797,6 @@ description | Waiting for a newly initialized WAL file to reach durable storage Returns I/O statistics about the backend with the specified process ID. The output fields are exactly the same as the ones in the pg_stat_io view. - - - The function does not return I/O statistics for the checkpointer, - the background writer, the startup process and the autovacuum launcher - as they are already visible in the pg_stat_io - view and there is only one of each. @@ -5982,7 +5976,8 @@ description | Waiting for a newly initialized WAL file to reach durable storage io: Reset all the counters shown in the - pg_stat_io view. + pg_stat_io view, as well as per-backend + I/O statistics returned by pg_stat_get_backend_io. diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index c65dd324325..6d4c15cad4c 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -2083,7 +2083,6 @@ WalSndWaitForWal(XLogRecPtr loc) WALSENDER_STATS_FLUSH_INTERVAL)) { pgstat_flush_io(false); - (void) pgstat_flush_backend(false, PGSTAT_BACKEND_FLUSH_IO); last_flush = now; } @@ -3177,7 +3176,6 @@ WalSndLoop(WalSndSendDataCallback send_data) WALSENDER_STATS_FLUSH_INTERVAL)) { pgstat_flush_io(false); - (void) pgstat_flush_backend(false, PGSTAT_BACKEND_FLUSH_IO); last_flush = now; } diff --git a/src/backend/utils/activity/pgstat.c b/src/backend/utils/activity/pgstat.c index 2f5b3710644..281104096c3 100644 --- a/src/backend/utils/activity/pgstat.c +++ b/src/backend/utils/activity/pgstat.c @@ -395,7 +395,6 @@ static const PgStat_KindInfo pgstat_kind_builtin_infos[PGSTAT_KIND_BUILTIN_SIZE] .shared_data_off = offsetof(PgStatShared_Backend, stats), .shared_data_len = sizeof(((PgStatShared_Backend *) 0)->stats), - .flush_static_cb = pgstat_backend_flush_cb, .reset_timestamp_cb = pgstat_backend_reset_timestamp_cb, }, @@ -464,6 +463,11 @@ static const PgStat_KindInfo pgstat_kind_builtin_infos[PGSTAT_KIND_BUILTIN_SIZE] .init_shmem_cb = pgstat_io_init_shmem_cb, .reset_all_cb = pgstat_io_reset_all_cb, .snapshot_cb = pgstat_io_snapshot_cb, + + .per_backend_data_off = offsetof(PgStatShared_IOBackendEntry, stats), + .per_backend_data_len = sizeof(PgStat_BackendIO), + .per_backend_hash_handle_off = offsetof(PgStatShared_IO, backend_hash_handle), + .per_backend_acc_cb = pgstat_io_per_backend_acc_cb, }, [PGSTAT_KIND_LOCK] = { @@ -640,6 +644,7 @@ pgstat_before_server_shutdown(int code, Datum arg) /* Transfer all live per-backend stats before writing the stats file. */ pgstat_wal_acc_all_backends(); pgstat_lock_acc_all_backends(); + pgstat_io_acc_all_backends(); pgStatLocal.shmem->is_shutdown = true; pgstat_write_statsfile(); @@ -687,6 +692,7 @@ pgstat_shutdown_hook(int code, Datum arg) /* Accumulate per-backend stats into the global stats */ pgstat_wal_acc_backend_cb(); pgstat_lock_acc_backend_cb(); + pgstat_io_acc_backend_cb(); pgstat_detach_shmem(); @@ -715,6 +721,7 @@ pgstat_initialize(void) */ pgstat_wal_acc_backend_cb(); pgstat_lock_acc_backend_cb(); + pgstat_io_acc_backend_cb(); /* * Create and cache per-backend statistics entries here. This also covers diff --git a/src/backend/utils/activity/pgstat_backend.c b/src/backend/utils/activity/pgstat_backend.c index 8dda2cd88c1..b5996d15d8f 100644 --- a/src/backend/utils/activity/pgstat_backend.c +++ b/src/backend/utils/activity/pgstat_backend.c @@ -11,9 +11,7 @@ * This statistics kind uses a proc number as object ID for the hash table * of pgstats. Entries are created each time a process is spawned, and are * dropped when the process exits. These are not written to the pgstats file - * on disk. Pending statistics are managed without direct interactions with - * PgStat_EntryRef->pending, relying on PendingBackendStats instead so as it - * is possible to report data within critical sections. + * on disk. * * Copyright (c) 2001-2026, PostgreSQL Global Development Group * @@ -24,60 +22,10 @@ #include "postgres.h" -#include "access/xlog.h" -#include "executor/instrument.h" -#include "storage/bufmgr.h" #include "storage/proc.h" #include "storage/procarray.h" -#include "utils/memutils.h" #include "utils/pgstat_internal.h" -/* - * Backend statistics counts waiting to be flushed out. These counters may be - * reported within critical sections so we use static memory in order to avoid - * memory allocation. - */ -static PgStat_BackendPending PendingBackendStats; -static bool backend_has_iostats = false; - -/* - * Utility routines to report I/O stats for backends, kept here to avoid - * exposing PendingBackendStats to the outside world. - */ -void -pgstat_count_backend_io_op_time(IOObject io_object, IOContext io_context, - IOOp io_op, instr_time io_time) -{ - Assert(track_io_timing || track_wal_io_timing); - - if (!pgstat_tracks_backend_bktype(MyBackendType)) - return; - - Assert(pgstat_tracks_io_op(MyBackendType, io_object, io_context, io_op)); - - INSTR_TIME_ADD(PendingBackendStats.pending_io.pending_times[io_object][io_context][io_op], - io_time); - - backend_has_iostats = true; - pgstat_report_fixed = true; -} - -void -pgstat_count_backend_io_op(IOObject io_object, IOContext io_context, - IOOp io_op, uint32 cnt, uint64 bytes) -{ - if (!pgstat_tracks_backend_bktype(MyBackendType)) - return; - - Assert(pgstat_tracks_io_op(MyBackendType, io_object, io_context, io_op)); - - PendingBackendStats.pending_io.counts[io_object][io_context][io_op] += cnt; - PendingBackendStats.pending_io.bytes[io_object][io_context][io_op] += bytes; - - backend_has_iostats = true; - pgstat_report_fixed = true; -} - /* * Returns statistics of a backend by proc number. */ @@ -151,104 +99,6 @@ pgstat_fetch_stat_backend_by_pid(int pid, BackendType *bktype) return backend_stats; } -/* - * Flush out locally pending backend IO statistics. Locking is managed - * by the caller. - */ -static void -pgstat_flush_backend_entry_io(PgStat_EntryRef *entry_ref) -{ - PgStatShared_Backend *shbackendent; - PgStat_BktypeIO *bktype_shstats; - PgStat_PendingIO pending_io; - - /* - * This function can be called even if nothing at all has happened for IO - * statistics. In this case, avoid unnecessarily modifying the stats - * entry. - */ - if (!backend_has_iostats) - return; - - shbackendent = (PgStatShared_Backend *) entry_ref->shared_stats; - bktype_shstats = &shbackendent->stats.io_stats; - pending_io = PendingBackendStats.pending_io; - - for (int io_object = 0; io_object < IOOBJECT_NUM_TYPES; io_object++) - { - for (int io_context = 0; io_context < IOCONTEXT_NUM_TYPES; io_context++) - { - for (int io_op = 0; io_op < IOOP_NUM_TYPES; io_op++) - { - instr_time time; - - bktype_shstats->counts[io_object][io_context][io_op] += - pending_io.counts[io_object][io_context][io_op]; - bktype_shstats->bytes[io_object][io_context][io_op] += - pending_io.bytes[io_object][io_context][io_op]; - time = pending_io.pending_times[io_object][io_context][io_op]; - - bktype_shstats->times[io_object][io_context][io_op] += - INSTR_TIME_GET_MICROSEC(time); - } - } - } - - /* - * Clear out the statistics buffer, so it can be re-used. - */ - MemSet(&PendingBackendStats.pending_io, 0, sizeof(PgStat_PendingIO)); - - backend_has_iostats = false; -} - -/* - * Flush out locally pending backend statistics - * - * "flags" parameter controls which statistics to flush. Returns true - * if some statistics could not be flushed due to lock contention. - */ -bool -pgstat_flush_backend(bool nowait, uint32 flags) -{ - PgStat_EntryRef *entry_ref; - bool has_pending_data = false; - - if (!pgstat_tracks_backend_bktype(MyBackendType)) - return false; - - /* Some IO data pending? */ - if ((flags & PGSTAT_BACKEND_FLUSH_IO) && backend_has_iostats) - has_pending_data = true; - - if (!has_pending_data) - return false; - - entry_ref = pgstat_get_entry_ref_locked(PGSTAT_KIND_BACKEND, InvalidOid, - MyProcNumber, nowait); - if (!entry_ref) - return true; - - /* Flush requested statistics */ - if (flags & PGSTAT_BACKEND_FLUSH_IO) - pgstat_flush_backend_entry_io(entry_ref); - - pgstat_unlock_entry(entry_ref); - - return false; -} - -/* - * Callback to flush out locally pending backend statistics. - * - * If some stats could not be flushed due to lock contention, return true. - */ -bool -pgstat_backend_flush_cb(bool nowait) -{ - return pgstat_flush_backend(nowait, PGSTAT_BACKEND_FLUSH_ALL); -} - /* * Create backend statistics entry for proc number. */ @@ -268,9 +118,6 @@ pgstat_create_backend(ProcNumber procnum) */ memset(&shstatent->stats, 0, sizeof(shstatent->stats)); pgstat_unlock_entry(entry_ref); - - MemSet(&PendingBackendStats, 0, sizeof(PgStat_BackendPending)); - backend_has_iostats = false; } /* diff --git a/src/backend/utils/activity/pgstat_io.c b/src/backend/utils/activity/pgstat_io.c index 8ec1aad5078..5e65bc5cb4b 100644 --- a/src/backend/utils/activity/pgstat_io.c +++ b/src/backend/utils/activity/pgstat_io.c @@ -7,6 +7,11 @@ * from pgstat.c to enforce the line between the statistics access / storage * implementation and the details about individual types of statistics. * + * IO statistics use a per-backend dshash to avoid double-counting. Each + * process flushes IO stats to its own entry in the dshash (keyed by + * ProcNumber). The global pg_stat_io view aggregates the global stats + * (which holds stats from exited processes) plus all live per-backend entries. + * * Copyright (c) 2021-2026, PostgreSQL Global Development Group * * IDENTIFICATION @@ -68,9 +73,6 @@ pgstat_count_io_op(IOObject io_object, IOContext io_context, IOOp io_op, PendingIOStats.counts[io_object][io_context][io_op] += cnt; PendingIOStats.bytes[io_object][io_context][io_op] += bytes; - /* Add the per-backend counts */ - pgstat_count_backend_io_op(io_object, io_context, io_op, cnt, bytes); - have_iostats = true; pgstat_report_fixed = true; } @@ -143,10 +145,6 @@ pgstat_count_io_op_time(IOObject io_object, IOContext io_context, IOOp io_op, INSTR_TIME_ADD(PendingIOStats.pending_times[io_object][io_context][io_op], io_time); - - /* Add the per-backend count */ - pgstat_count_backend_io_op_time(io_object, io_context, io_op, - io_time); } pgstat_count_io_op(io_object, io_context, io_op, cnt, bytes); @@ -170,7 +168,7 @@ pgstat_flush_io(bool nowait) } /* - * Flush out locally pending IO statistics + * Flush out locally pending IO statistics to the per-backend dshash entry. * * If no stats have been recorded, this function returns false. * @@ -180,20 +178,18 @@ pgstat_flush_io(bool nowait) bool pgstat_io_flush_cb(bool nowait) { - LWLock *bktype_lock; + PgStatShared_IOBackendEntry *entry; PgStat_BktypeIO *bktype_shstats; if (!have_iostats) return false; - bktype_lock = &pgStatLocal.shmem->io.locks[MyBackendType]; - bktype_shstats = - &pgStatLocal.shmem->io.stats.stats[MyBackendType]; + entry = pgstat_lock_my_per_backend_entry(PGSTAT_KIND_IO, nowait); - if (!nowait) - LWLockAcquire(bktype_lock, LW_EXCLUSIVE); - else if (!LWLockConditionalAcquire(bktype_lock, LW_EXCLUSIVE)) - return true; + if (entry == NULL) + return nowait; + + bktype_shstats = &entry->stats.stats; for (int io_object = 0; io_object < IOOBJECT_NUM_TYPES; io_object++) { @@ -217,12 +213,9 @@ pgstat_io_flush_cb(bool nowait) } } - Assert(pgstat_bktype_io_stats_valid(bktype_shstats, MyBackendType)); - - LWLockRelease(bktype_lock); + LWLockRelease(&entry->header.lock); memset(&PendingIOStats, 0, sizeof(PendingIOStats)); - have_iostats = false; return false; @@ -271,55 +264,70 @@ pgstat_io_init_shmem_cb(void *stats) { PgStatShared_IO *stat_shmem = (PgStatShared_IO *) stats; - for (int i = 0; i < BACKEND_NUM_TYPES; i++) - LWLockInitialize(&stat_shmem->locks[i], LWTRANCHE_PGSTATS_DATA); + LWLockInitialize(&stat_shmem->lock, LWTRANCHE_PGSTATS_DATA); } void pgstat_io_reset_all_cb(TimestampTz ts) { - for (int i = 0; i < BACKEND_NUM_TYPES; i++) - { - LWLock *bktype_lock = &pgStatLocal.shmem->io.locks[i]; - PgStat_BktypeIO *bktype_shstats = &pgStatLocal.shmem->io.stats.stats[i]; + PgStatShared_IO *shmem = &pgStatLocal.shmem->io; + dshash_seq_status hstat; + PgStatShared_IOBackendEntry *entry; + dshash_table *hash; - LWLockAcquire(bktype_lock, LW_EXCLUSIVE); + hash = pgstat_per_backend_attach(PGSTAT_KIND_IO); - /* - * Use the lock in the first BackendType's PgStat_BktypeIO to protect - * the reset timestamp as well. - */ - if (i == 0) - pgStatLocal.shmem->io.stats.stat_reset_timestamp = ts; + /* + * Hold the kind lock while resetting both the global stats and live + * entries. Transfers hold the same lock, so pre-reset counters cannot be + * moved into the global stats after it is reset. + */ + LWLockAcquire(&shmem->lock, LW_EXCLUSIVE); + memset(&shmem->stats, 0, sizeof(shmem->stats)); + shmem->stats.stat_reset_timestamp = ts; - memset(bktype_shstats, 0, sizeof(*bktype_shstats)); - LWLockRelease(bktype_lock); + /* Reset all per-backend entries */ + if (hash != NULL) + { + dshash_seq_init(&hstat, hash, true); + while ((entry = dshash_seq_next(&hstat)) != NULL) + { + LWLockAcquire(&entry->header.lock, LW_EXCLUSIVE); + memset(&entry->stats.stats, 0, sizeof(PgStat_BktypeIO)); + entry->stats.stat_reset_timestamp = ts; + LWLockRelease(&entry->header.lock); + } + dshash_seq_term(&hstat); } + + LWLockRelease(&shmem->lock); } +/* + * Build IO stats snapshot by aggregating global stats and all live + * per-backend entries. + */ void pgstat_io_snapshot_cb(void) { - for (int i = 0; i < BACKEND_NUM_TYPES; i++) - { - LWLock *bktype_lock = &pgStatLocal.shmem->io.locks[i]; - PgStat_BktypeIO *bktype_shstats = &pgStatLocal.shmem->io.stats.stats[i]; - PgStat_BktypeIO *bktype_snap = &pgStatLocal.snapshot.io.stats[i]; + PgStatShared_IO *shmem = &pgStatLocal.shmem->io; + PgStat_IO *snap = &pgStatLocal.snapshot.io; + dshash_table *hash; - LWLockAcquire(bktype_lock, LW_SHARED); + hash = pgstat_per_backend_attach(PGSTAT_KIND_IO); - /* - * Use the lock in the first BackendType's PgStat_BktypeIO to protect - * the reset timestamp as well. - */ - if (i == 0) - pgStatLocal.snapshot.io.stat_reset_timestamp = - pgStatLocal.shmem->io.stats.stat_reset_timestamp; + /* + * Prevent entries from moving to the global stats between copying it and + * scanning the per-backend hash. + */ + LWLockAcquire(&shmem->lock, LW_SHARED); + memcpy(snap, &shmem->stats, sizeof(PgStat_IO)); - /* using struct assignment due to better type safety */ - *bktype_snap = *bktype_shstats; - LWLockRelease(bktype_lock); - } + /* Add in all live per-backend entries */ + if (hash != NULL) + pgstat_per_backend_snapshot(PGSTAT_KIND_IO, hash, snap); + + LWLockRelease(&shmem->lock); } /* @@ -581,3 +589,116 @@ pgstat_tracks_io_op(BackendType bktype, IOObject io_object, return true; } + +/* + * Accumulate IO counters from src into dst. + */ +static inline void +pgstat_io_accumulate_counters(PgStat_BktypeIO *dst, const PgStat_BktypeIO *src) +{ + for (int io_object = 0; io_object < IOOBJECT_NUM_TYPES; io_object++) + { + for (int io_context = 0; io_context < IOCONTEXT_NUM_TYPES; io_context++) + { + for (int io_op = 0; io_op < IOOP_NUM_TYPES; io_op++) + { + dst->counts[io_object][io_context][io_op] += + src->counts[io_object][io_context][io_op]; + dst->bytes[io_object][io_context][io_op] += + src->bytes[io_object][io_context][io_op]; + dst->times[io_object][io_context][io_op] += + src->times[io_object][io_context][io_op]; + } + } + } +} + +/* + * Accumulate one per-backend IO entry into a snapshot or the global stats. + */ +void +pgstat_io_per_backend_acc_cb(void *dst, void *entry) +{ + PgStat_IO *stats = dst; + PgStatShared_IOBackendEntry *e = (PgStatShared_IOBackendEntry *) entry; + BackendType bktype = e->header.backend_type; + + if (bktype == B_INVALID) + return; + + pgstat_io_accumulate_counters(&stats->stats[bktype], &e->stats.stats); +} + +/* + * Accumulate a backend's IO stats into the global stats, then remove the + * entry from the dshash. + * + * Called at backend exit after the final flush, or when a ProcNumber is + * being reused. + */ +void +pgstat_io_acc_backend_cb(void) +{ + pgstat_acc_my_per_backend(PGSTAT_KIND_IO, &pgStatLocal.shmem->io.lock); +} + +/* + * Accumulate all remaining per-backend IO stats entries into the global stats + * and remove them. Called at clean server shutdown to ensure all flushed data + * is preserved in the stats file. + */ +void +pgstat_io_acc_all_backends(void) +{ + pgstat_acc_all_per_backend(PGSTAT_KIND_IO, &pgStatLocal.shmem->io.lock); +} + +/* + * Returns per-backend IO statistics for the given ProcNumber. + */ +PgStat_BackendIO * +pgstat_fetch_stat_backend_io(ProcNumber procnum) +{ + return (PgStat_BackendIO *) pgstat_fetch_per_backend(PGSTAT_KIND_IO, procnum); +} + +/* + * Reset a backend's IO stats. Accumulate the entry's counters into the + * global stats, then zero the stats and set the reset timestamp. + */ +void +pgstat_io_reset_backend_cb(ProcNumber procnum, TimestampTz ts) +{ + PgStatShared_IO *shmem = &pgStatLocal.shmem->io; + dshash_table *hash; + PgStatShared_IOBackendEntry *entry; + + hash = pgstat_per_backend_attach(PGSTAT_KIND_IO); + + if (hash == NULL) + return; + + LWLockAcquire(&shmem->lock, LW_EXCLUSIVE); + + entry = dshash_find(hash, &procnum, true); + + if (entry == NULL) + { + LWLockRelease(&shmem->lock); + return; + } + + LWLockAcquire(&entry->header.lock, LW_EXCLUSIVE); + + /* Accumulate current stats into global before zeroing */ + pgstat_io_accumulate_counters(&shmem->stats.stats[entry->header.backend_type], + &entry->stats.stats); + + /* Zero stats and set reset timestamp */ + memset(&entry->stats, 0, sizeof(entry->stats)); + entry->stats.stat_reset_timestamp = ts; + + LWLockRelease(&entry->header.lock); + dshash_release_lock(hash, entry); + LWLockRelease(&shmem->lock); +} diff --git a/src/backend/utils/activity/pgstat_relation.c b/src/backend/utils/activity/pgstat_relation.c index beea9188773..e4eccaef595 100644 --- a/src/backend/utils/activity/pgstat_relation.c +++ b/src/backend/utils/activity/pgstat_relation.c @@ -316,7 +316,6 @@ pgstat_report_vacuum(Relation rel, PgStat_Counter livetuples, * VACUUM command has processed all tables and committed. */ pgstat_flush_io(false); - (void) pgstat_flush_backend(false, PGSTAT_BACKEND_FLUSH_IO); } /* @@ -411,7 +410,6 @@ pgstat_report_analyze(Relation rel, /* see pgstat_report_vacuum() */ pgstat_flush_io(false); - (void) pgstat_flush_backend(false, PGSTAT_BACKEND_FLUSH_IO); } /* diff --git a/src/backend/utils/activity/pgstat_shmem.c b/src/backend/utils/activity/pgstat_shmem.c index a7e67f706c4..993024eaea5 100644 --- a/src/backend/utils/activity/pgstat_shmem.c +++ b/src/backend/utils/activity/pgstat_shmem.c @@ -118,12 +118,11 @@ pgstat_dsa_init_size(void) /* * The dshash header / initial buckets array needs to fit into "plain" * shared memory, but it's beneficial to not need dsm segments - * immediately. A size of 256kB seems works well and is not - * disproportional compared to other constant sized shared memory - * allocations. NB: To avoid DSMs further, the user can configure - * min_dynamic_shared_memory. + * immediately. A size of 1MB works well and is not disproportional + * compared to other constant sized shared memory allocations. NB: To + * avoid DSMs further, the user can configure min_dynamic_shared_memory. */ - sz = 256 * 1024; + sz = 1024 * 1024; Assert(dsa_minimum_size() <= sz); return MAXALIGN(sz); } diff --git a/src/backend/utils/activity/pgstat_wal.c b/src/backend/utils/activity/pgstat_wal.c index ece4d91ed70..99fd20a04c2 100644 --- a/src/backend/utils/activity/pgstat_wal.c +++ b/src/backend/utils/activity/pgstat_wal.c @@ -60,7 +60,6 @@ pgstat_report_wal(bool force) /* flush IO stats */ pgstat_flush_io(nowait); - (void) pgstat_flush_backend(nowait, PGSTAT_BACKEND_FLUSH_IO); } /* diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index 502b281198e..a054c7bda69 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -1671,32 +1671,47 @@ Datum pg_stat_get_backend_io(PG_FUNCTION_ARGS) { ReturnSetInfo *rsinfo; - BackendType bktype; int pid; - PgStat_Backend *backend_stats; - PgStat_BktypeIO *bktype_stats; + PGPROC *proc; + ProcNumber procnum; + PgBackendStatus *beentry; + BackendType bktype; + PgStat_BackendIO *backend_io; InitMaterializedSRF(fcinfo, 0); rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; pid = PG_GETARG_INT32(0); - backend_stats = pgstat_fetch_stat_backend_by_pid(pid, &bktype); - if (!backend_stats) + proc = BackendPidGetProc(pid); + + if (!proc) + proc = AuxiliaryPidGetProc(pid); + if (!proc) + return (Datum) 0; + + procnum = GetNumberFromPGProc(proc); + beentry = pgstat_get_beentry_by_proc_number(procnum); + + if (!beentry || beentry->st_procpid != pid) return (Datum) 0; - bktype_stats = &backend_stats->io_stats; + bktype = beentry->st_backendType; + backend_io = pgstat_fetch_stat_backend_io(procnum); + + if (!backend_io) + return (Datum) 0; /* * In Assert builds, we can afford an extra loop through all of the * counters (in pg_stat_io_build_tuples()), checking that only expected * stats are non-zero, since it keeps the non-Assert code cleaner. */ - Assert(pgstat_bktype_io_stats_valid(bktype_stats, bktype)); + Assert(pgstat_bktype_io_stats_valid(&backend_io->stats, bktype)); + + pg_stat_io_build_tuples(rsinfo, &backend_io->stats, bktype, + backend_io->stat_reset_timestamp); - /* save tuples with data from this PgStat_BktypeIO */ - pg_stat_io_build_tuples(rsinfo, bktype_stats, bktype, - backend_stats->stat_reset_timestamp); return (Datum) 0; } @@ -2223,20 +2238,14 @@ pg_stat_reset_backend_stats(PG_FUNCTION_ARGS) if (!beentry) PG_RETURN_VOID(); - /* Check if the backend type tracks statistics */ - if (!pgstat_tracks_backend_bktype(beentry->st_backendType)) - PG_RETURN_VOID(); - /* - * Accumulate the backend's WAL and lock stats into the global stats, then - * zero the entries. + * Accumulate the backend's WAL, lock and IO stats into the global stats, + * then zero the entries. */ ts = GetCurrentTimestamp(); pgstat_wal_reset_backend_cb(procNumber, ts); pgstat_lock_reset_backend_cb(procNumber, ts); - - /* Reset IO stats still in PGSTAT_KIND_BACKEND */ - pgstat_reset(PGSTAT_KIND_BACKEND, InvalidOid, procNumber); + pgstat_io_reset_backend_cb(procNumber, ts); PG_RETURN_VOID(); } diff --git a/src/include/pgstat.h b/src/include/pgstat.h index 2c1f42b923e..cd6bfc9a33b 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -218,7 +218,7 @@ typedef struct PgStat_TableXactStatus * ------------------------------------------------------------ */ -#define PGSTAT_FILE_FORMAT_ID 0x01A5BCBD +#define PGSTAT_FILE_FORMAT_ID 0x01A5BCBE typedef struct PgStat_ArchiverStats { @@ -333,6 +333,12 @@ typedef struct PgStat_BktypeIO PgStat_Counter times[IOOBJECT_NUM_TYPES][IOCONTEXT_NUM_TYPES][IOOP_NUM_TYPES]; } PgStat_BktypeIO; +typedef struct PgStat_BackendIO +{ + TimestampTz stat_reset_timestamp; + PgStat_BktypeIO stats; +} PgStat_BackendIO; + typedef struct PgStat_PendingIO { uint64 bytes[IOOBJECT_NUM_TYPES][IOCONTEXT_NUM_TYPES][IOOP_NUM_TYPES]; @@ -535,21 +541,8 @@ typedef struct PgStat_WalStats typedef struct PgStat_Backend { TimestampTz stat_reset_timestamp; - PgStat_BktypeIO io_stats; } PgStat_Backend; -/* --------- - * PgStat_BackendPending Non-flushed backend stats. - * --------- - */ -typedef struct PgStat_BackendPending -{ - /* - * Backend statistics store the same amount of IO data as PGSTAT_KIND_IO. - */ - PgStat_PendingIO pending_io; -} PgStat_BackendPending; - /* * Functions in pgstat.c */ @@ -590,17 +583,6 @@ extern PgStat_ArchiverStats *pgstat_fetch_stat_archiver(void); * Functions in pgstat_backend.c */ -/* used by pgstat_io.c for I/O stats tracked in backends */ -extern void pgstat_count_backend_io_op_time(IOObject io_object, - IOContext io_context, - IOOp io_op, - instr_time io_time); -extern void pgstat_count_backend_io_op(IOObject io_object, - IOContext io_context, - IOOp io_op, uint32 cnt, - uint64 bytes); - - extern PgStat_Backend *pgstat_fetch_stat_backend(ProcNumber procNumber); extern PgStat_Backend *pgstat_fetch_stat_backend_by_pid(int pid, BackendType *bktype); @@ -637,6 +619,8 @@ extern void pgstat_count_io_op_time(IOObject io_object, IOContext io_context, uint32 cnt, uint64 bytes); extern PgStat_IO *pgstat_fetch_stat_io(void); +extern PgStat_BackendIO *pgstat_fetch_stat_backend_io(ProcNumber procnum); +extern void pgstat_io_reset_backend_cb(ProcNumber procnum, TimestampTz ts); extern const char *pgstat_get_io_context_name(IOContext io_context); extern const char *pgstat_get_io_object_name(IOObject io_object); diff --git a/src/include/utils/pgstat_internal.h b/src/include/utils/pgstat_internal.h index 800ef6c4c27..95d52e1a26e 100644 --- a/src/include/utils/pgstat_internal.h +++ b/src/include/utils/pgstat_internal.h @@ -477,12 +477,13 @@ typedef struct PgStatShared_Checkpointer /* Shared-memory ready PgStat_IO */ typedef struct PgStatShared_IO { + LWLock lock; + PgStat_IO stats; + /* - * locks[i] protects stats.stats[i]. locks[0] also protects - * stats.stat_reset_timestamp. + * Per-backend dshash, keyed by ProcNumber. */ - LWLock locks[BACKEND_NUM_TYPES]; - PgStat_IO stats; + dshash_table_handle backend_hash_handle; } PgStatShared_IO; typedef struct PgStatShared_Lock @@ -513,6 +514,16 @@ typedef struct PgStatShared_PerBackendEntry LWLock lock; } PgStatShared_PerBackendEntry; +/* + * Per-backend entry for IO statistics, stored in a dshash keyed by + * ProcNumber. + */ +typedef struct PgStatShared_IOBackendEntry +{ + PgStatShared_PerBackendEntry header; + PgStat_BackendIO stats; +} PgStatShared_IOBackendEntry; + /* * Per-backend entry for lock statistics, stored in a dshash keyed by * ProcNumber. @@ -782,12 +793,6 @@ extern void pgstat_archiver_snapshot_cb(void); * Functions in pgstat_backend.c */ -/* flags for pgstat_flush_backend() */ -#define PGSTAT_BACKEND_FLUSH_IO (1 << 0) /* Flush I/O statistics */ -#define PGSTAT_BACKEND_FLUSH_ALL (PGSTAT_BACKEND_FLUSH_IO) - -extern bool pgstat_flush_backend(bool nowait, uint32 flags); -extern bool pgstat_backend_flush_cb(bool nowait); extern void pgstat_backend_reset_timestamp_cb(PgStatShared_Common *header, TimestampTz ts); @@ -841,6 +846,9 @@ extern bool pgstat_io_flush_cb(bool nowait); extern void pgstat_io_init_shmem_cb(void *stats); extern void pgstat_io_reset_all_cb(TimestampTz ts); extern void pgstat_io_snapshot_cb(void); +extern void pgstat_io_acc_backend_cb(void); +extern void pgstat_io_acc_all_backends(void); +extern void pgstat_io_per_backend_acc_cb(void *dst, void *entry); /* * Functions in pgstat_lock.c diff --git a/src/test/regress/expected/stats.out b/src/test/regress/expected/stats.out index 9db59e25210..4255c1234f4 100644 --- a/src/test/regress/expected/stats.out +++ b/src/test/regress/expected/stats.out @@ -1871,13 +1871,20 @@ SELECT :io_stats_post_reset < :io_stats_pre_reset; SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_post_reset FROM pg_stat_get_backend_io(pg_backend_pid()) \gset --- pg_stat_reset_shared() did not reset backend IO stats -SELECT :my_io_stats_pre_reset <= :my_io_stats_post_reset; +-- pg_stat_reset_shared() also resets per-backend IO stats +SELECT :my_io_stats_pre_reset > :my_io_stats_post_reset; ?column? ---------- t (1 row) +SELECT bool_and(stats_reset IS NOT NULL) AS backend_io_reset_timestamp_set + FROM pg_stat_get_backend_io(pg_backend_pid()); + backend_io_reset_timestamp_set +-------------------------------- + t +(1 row) + -- but pg_stat_reset_backend_stats() does SELECT pg_stat_reset_backend_stats(pg_backend_pid()); pg_stat_reset_backend_stats @@ -1904,11 +1911,13 @@ SELECT pg_stat_get_backend_io(0); ------------------------ (0 rows) --- Auxiliary processes return no data. -SELECT pg_stat_get_backend_io(:checkpointer_pid); - pg_stat_get_backend_io ------------------------- -(0 rows) +-- Auxiliary processes now return data +SELECT count(*) > 0 AS checkpointer_has_io_stats + FROM pg_stat_get_backend_io(:checkpointer_pid); + checkpointer_has_io_stats +--------------------------- + t +(1 row) -- test BRIN index doesn't block HOT update CREATE TABLE brin_hot ( diff --git a/src/test/regress/sql/stats.sql b/src/test/regress/sql/stats.sql index 176bd008673..12cac8ebb0b 100644 --- a/src/test/regress/sql/stats.sql +++ b/src/test/regress/sql/stats.sql @@ -878,8 +878,10 @@ SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + SELECT :io_stats_post_reset < :io_stats_pre_reset; SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_post_reset FROM pg_stat_get_backend_io(pg_backend_pid()) \gset --- pg_stat_reset_shared() did not reset backend IO stats -SELECT :my_io_stats_pre_reset <= :my_io_stats_post_reset; +-- pg_stat_reset_shared() also resets per-backend IO stats +SELECT :my_io_stats_pre_reset > :my_io_stats_post_reset; +SELECT bool_and(stats_reset IS NOT NULL) AS backend_io_reset_timestamp_set + FROM pg_stat_get_backend_io(pg_backend_pid()); -- but pg_stat_reset_backend_stats() does SELECT pg_stat_reset_backend_stats(pg_backend_pid()); SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_post_backend_reset @@ -889,8 +891,9 @@ SELECT :my_io_stats_pre_reset > :my_io_stats_post_backend_reset; -- Check invalid input for pg_stat_get_backend_io() SELECT pg_stat_get_backend_io(NULL); SELECT pg_stat_get_backend_io(0); --- Auxiliary processes return no data. -SELECT pg_stat_get_backend_io(:checkpointer_pid); +-- Auxiliary processes now return data +SELECT count(*) > 0 AS checkpointer_has_io_stats + FROM pg_stat_get_backend_io(:checkpointer_pid); -- test BRIN index doesn't block HOT update CREATE TABLE brin_hot ( diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index bf960d8647b..48a1deb9bca 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2327,6 +2327,7 @@ PgStatShared_Function PgStatShared_HashEntry PgStatShared_Index PgStatShared_IO +PgStatShared_IOBackendEntry PgStatShared_Lock PgStatShared_LockBackendEntry PgStatShared_PerBackendEntry @@ -2338,7 +2339,7 @@ PgStatShared_Wal PgStatShared_WalBackendEntry PgStat_ArchiverStats PgStat_Backend -PgStat_BackendPending +PgStat_BackendIO PgStat_BackendSubEntry PgStat_BgWriterStats PgStat_BktypeIO -- 2.34.1