From 766aafba3598dc9844dbaa4e2baff9174c6341cc Mon Sep 17 00:00:00 2001 From: Bertrand Drouvot Date: Fri, 17 Jul 2026 13:26:07 +0000 Subject: [PATCH v4 5/6] pgstat: move I/O statistics to new per-backend infrastructure PGSTAT_KIND_BACKEND stores each backend's I/O statistics in one variable-numbered entry keyed by ProcNumber. Move I/O statistics into a dedicated fixed array associated with the I/O statistics kind and indexed by ProcNumber. The global I/O stats hold data for backends that have exited, while the array holds statistics for active backends. Build pg_stat_io snapshots by copying the global stats and adding every active per-backend entry. Transfer the current process's entry into the global stats before invalidating it at process exit or ProcNumber reuse. Transfer all entries before a clean shutdown writes the statistics file. Remove I/O counters from PGSTAT_KIND_BACKEND and use the new infrastructure in pg_stat_get_backend_io(). This also makes I/O statistics available for auxiliary and shared-memory-only workers. Update the documentation to describe the resulting behavior. Author: Bertrand Drouvot Reviewed-by: Michael Paquier Reviewed-by: Andres Freund Discussion: https://postgr.es/m/anCjoFkGah/uB7lw%40bdtpg --- doc/src/sgml/monitoring.sgml | 9 +- src/backend/replication/walsender.c | 2 - src/backend/utils/activity/pgstat.c | 9 +- src/backend/utils/activity/pgstat_backend.c | 155 +------------- src/backend/utils/activity/pgstat_io.c | 213 ++++++++++++++----- src/backend/utils/activity/pgstat_relation.c | 2 - src/backend/utils/activity/pgstat_wal.c | 1 - src/backend/utils/adt/pgstatfuncs.c | 47 ++-- src/include/pgstat.h | 37 ++-- src/include/utils/pgstat_internal.h | 25 ++- src/test/regress/expected/stats.out | 25 ++- src/test/regress/sql/stats.sql | 13 +- src/tools/pgindent/typedefs.list | 3 +- 13 files changed, 255 insertions(+), 286 deletions(-) 66.9% src/backend/utils/activity/ 8.7% src/backend/utils/adt/ 5.2% src/include/utils/ 6.0% src/include/ 5.0% src/test/regress/expected/ 3.9% src/test/regress/sql/ diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index 0602d80e599..6bb840e19cd 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -5812,12 +5812,6 @@ description | Waiting for a newly initialized WAL file to reach durable storage Returns I/O statistics about the backend with the specified process ID. The output fields are exactly the same as the ones in the pg_stat_io view. - - - The function does not return I/O statistics for the checkpointer, - the background writer, the startup process and the autovacuum launcher - as they are already visible in the pg_stat_io - view and there is only one of each. @@ -6015,7 +6009,8 @@ description | Waiting for a newly initialized WAL file to reach durable storage io: Reset all the counters shown in the - pg_stat_io view. + pg_stat_io view, as well as per-backend + I/O statistics returned by pg_stat_get_backend_io. diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c index 9f21c7c8dfe..d4ab9c26f8e 100644 --- a/src/backend/replication/walsender.c +++ b/src/backend/replication/walsender.c @@ -2091,7 +2091,6 @@ WalSndWaitForWal(XLogRecPtr loc) WALSENDER_STATS_FLUSH_INTERVAL)) { pgstat_flush_io(false); - (void) pgstat_flush_backend(false, PGSTAT_BACKEND_FLUSH_IO); last_flush = now; } @@ -3210,7 +3209,6 @@ WalSndLoop(WalSndSendDataCallback send_data) WALSENDER_STATS_FLUSH_INTERVAL)) { pgstat_flush_io(false); - (void) pgstat_flush_backend(false, PGSTAT_BACKEND_FLUSH_IO); last_flush = now; } diff --git a/src/backend/utils/activity/pgstat.c b/src/backend/utils/activity/pgstat.c index 1affde4df05..411733e72f4 100644 --- a/src/backend/utils/activity/pgstat.c +++ b/src/backend/utils/activity/pgstat.c @@ -393,7 +393,6 @@ static const PgStat_KindInfo pgstat_kind_builtin_infos[PGSTAT_KIND_BUILTIN_SIZE] .shared_data_off = offsetof(PgStatShared_Backend, stats), .shared_data_len = sizeof(((PgStatShared_Backend *) 0)->stats), - .flush_static_cb = pgstat_backend_flush_cb, .reset_timestamp_cb = pgstat_backend_reset_timestamp_cb, }, @@ -462,6 +461,11 @@ static const PgStat_KindInfo pgstat_kind_builtin_infos[PGSTAT_KIND_BUILTIN_SIZE] .init_shmem_cb = pgstat_io_init_shmem_cb, .reset_all_cb = pgstat_io_reset_all_cb, .snapshot_cb = pgstat_io_snapshot_cb, + + .per_backend_entry_size = sizeof(PgStatShared_IOBackendEntry), + .per_backend_data_off = offsetof(PgStatShared_IOBackendEntry, stats), + .per_backend_data_len = sizeof(PgStat_BackendIO), + .per_backend_acc_cb = pgstat_io_per_backend_acc_cb, }, [PGSTAT_KIND_LOCK] = { @@ -638,6 +642,7 @@ pgstat_before_server_shutdown(int code, Datum arg) /* Transfer all live per-backend stats before writing the stats file */ pgstat_wal_acc_all_backends(); pgstat_lock_acc_all_backends(); + pgstat_io_acc_all_backends(); pgStatLocal.shmem->is_shutdown = true; pgstat_write_statsfile(); @@ -685,6 +690,7 @@ pgstat_shutdown_hook(int code, Datum arg) /* Accumulate per-backend stats into the global stats */ pgstat_wal_acc_backend_cb(); pgstat_lock_acc_backend_cb(); + pgstat_io_acc_backend_cb(); pgstat_detach_shmem(); @@ -713,6 +719,7 @@ pgstat_initialize(void) */ pgstat_wal_acc_backend_cb(); pgstat_lock_acc_backend_cb(); + pgstat_io_acc_backend_cb(); /* * Activate per-backend statistics entries here. This also covers diff --git a/src/backend/utils/activity/pgstat_backend.c b/src/backend/utils/activity/pgstat_backend.c index 8dda2cd88c1..b5996d15d8f 100644 --- a/src/backend/utils/activity/pgstat_backend.c +++ b/src/backend/utils/activity/pgstat_backend.c @@ -11,9 +11,7 @@ * This statistics kind uses a proc number as object ID for the hash table * of pgstats. Entries are created each time a process is spawned, and are * dropped when the process exits. These are not written to the pgstats file - * on disk. Pending statistics are managed without direct interactions with - * PgStat_EntryRef->pending, relying on PendingBackendStats instead so as it - * is possible to report data within critical sections. + * on disk. * * Copyright (c) 2001-2026, PostgreSQL Global Development Group * @@ -24,60 +22,10 @@ #include "postgres.h" -#include "access/xlog.h" -#include "executor/instrument.h" -#include "storage/bufmgr.h" #include "storage/proc.h" #include "storage/procarray.h" -#include "utils/memutils.h" #include "utils/pgstat_internal.h" -/* - * Backend statistics counts waiting to be flushed out. These counters may be - * reported within critical sections so we use static memory in order to avoid - * memory allocation. - */ -static PgStat_BackendPending PendingBackendStats; -static bool backend_has_iostats = false; - -/* - * Utility routines to report I/O stats for backends, kept here to avoid - * exposing PendingBackendStats to the outside world. - */ -void -pgstat_count_backend_io_op_time(IOObject io_object, IOContext io_context, - IOOp io_op, instr_time io_time) -{ - Assert(track_io_timing || track_wal_io_timing); - - if (!pgstat_tracks_backend_bktype(MyBackendType)) - return; - - Assert(pgstat_tracks_io_op(MyBackendType, io_object, io_context, io_op)); - - INSTR_TIME_ADD(PendingBackendStats.pending_io.pending_times[io_object][io_context][io_op], - io_time); - - backend_has_iostats = true; - pgstat_report_fixed = true; -} - -void -pgstat_count_backend_io_op(IOObject io_object, IOContext io_context, - IOOp io_op, uint32 cnt, uint64 bytes) -{ - if (!pgstat_tracks_backend_bktype(MyBackendType)) - return; - - Assert(pgstat_tracks_io_op(MyBackendType, io_object, io_context, io_op)); - - PendingBackendStats.pending_io.counts[io_object][io_context][io_op] += cnt; - PendingBackendStats.pending_io.bytes[io_object][io_context][io_op] += bytes; - - backend_has_iostats = true; - pgstat_report_fixed = true; -} - /* * Returns statistics of a backend by proc number. */ @@ -151,104 +99,6 @@ pgstat_fetch_stat_backend_by_pid(int pid, BackendType *bktype) return backend_stats; } -/* - * Flush out locally pending backend IO statistics. Locking is managed - * by the caller. - */ -static void -pgstat_flush_backend_entry_io(PgStat_EntryRef *entry_ref) -{ - PgStatShared_Backend *shbackendent; - PgStat_BktypeIO *bktype_shstats; - PgStat_PendingIO pending_io; - - /* - * This function can be called even if nothing at all has happened for IO - * statistics. In this case, avoid unnecessarily modifying the stats - * entry. - */ - if (!backend_has_iostats) - return; - - shbackendent = (PgStatShared_Backend *) entry_ref->shared_stats; - bktype_shstats = &shbackendent->stats.io_stats; - pending_io = PendingBackendStats.pending_io; - - for (int io_object = 0; io_object < IOOBJECT_NUM_TYPES; io_object++) - { - for (int io_context = 0; io_context < IOCONTEXT_NUM_TYPES; io_context++) - { - for (int io_op = 0; io_op < IOOP_NUM_TYPES; io_op++) - { - instr_time time; - - bktype_shstats->counts[io_object][io_context][io_op] += - pending_io.counts[io_object][io_context][io_op]; - bktype_shstats->bytes[io_object][io_context][io_op] += - pending_io.bytes[io_object][io_context][io_op]; - time = pending_io.pending_times[io_object][io_context][io_op]; - - bktype_shstats->times[io_object][io_context][io_op] += - INSTR_TIME_GET_MICROSEC(time); - } - } - } - - /* - * Clear out the statistics buffer, so it can be re-used. - */ - MemSet(&PendingBackendStats.pending_io, 0, sizeof(PgStat_PendingIO)); - - backend_has_iostats = false; -} - -/* - * Flush out locally pending backend statistics - * - * "flags" parameter controls which statistics to flush. Returns true - * if some statistics could not be flushed due to lock contention. - */ -bool -pgstat_flush_backend(bool nowait, uint32 flags) -{ - PgStat_EntryRef *entry_ref; - bool has_pending_data = false; - - if (!pgstat_tracks_backend_bktype(MyBackendType)) - return false; - - /* Some IO data pending? */ - if ((flags & PGSTAT_BACKEND_FLUSH_IO) && backend_has_iostats) - has_pending_data = true; - - if (!has_pending_data) - return false; - - entry_ref = pgstat_get_entry_ref_locked(PGSTAT_KIND_BACKEND, InvalidOid, - MyProcNumber, nowait); - if (!entry_ref) - return true; - - /* Flush requested statistics */ - if (flags & PGSTAT_BACKEND_FLUSH_IO) - pgstat_flush_backend_entry_io(entry_ref); - - pgstat_unlock_entry(entry_ref); - - return false; -} - -/* - * Callback to flush out locally pending backend statistics. - * - * If some stats could not be flushed due to lock contention, return true. - */ -bool -pgstat_backend_flush_cb(bool nowait) -{ - return pgstat_flush_backend(nowait, PGSTAT_BACKEND_FLUSH_ALL); -} - /* * Create backend statistics entry for proc number. */ @@ -268,9 +118,6 @@ pgstat_create_backend(ProcNumber procnum) */ memset(&shstatent->stats, 0, sizeof(shstatent->stats)); pgstat_unlock_entry(entry_ref); - - MemSet(&PendingBackendStats, 0, sizeof(PgStat_BackendPending)); - backend_has_iostats = false; } /* diff --git a/src/backend/utils/activity/pgstat_io.c b/src/backend/utils/activity/pgstat_io.c index 3c8fcb20aba..85178ff1d54 100644 --- a/src/backend/utils/activity/pgstat_io.c +++ b/src/backend/utils/activity/pgstat_io.c @@ -7,6 +7,11 @@ * from pgstat.c to enforce the line between the statistics access / storage * implementation and the details about individual types of statistics. * + * I/O statistics use a per-backend array to avoid double-counting. Each + * process flushes I/O stats to its own ProcNumber-indexed entry. The global + * pg_stat_io view aggregates the global stats (which holds stats from exited + * processes) plus all active per-backend entries. + * * Copyright (c) 2021-2026, PostgreSQL Global Development Group * * IDENTIFICATION @@ -68,9 +73,6 @@ pgstat_count_io_op(IOObject io_object, IOContext io_context, IOOp io_op, PendingIOStats.counts[io_object][io_context][io_op] += cnt; PendingIOStats.bytes[io_object][io_context][io_op] += bytes; - /* Add the per-backend counts */ - pgstat_count_backend_io_op(io_object, io_context, io_op, cnt, bytes); - have_iostats = true; pgstat_report_fixed = true; } @@ -143,10 +145,6 @@ pgstat_count_io_op_time(IOObject io_object, IOContext io_context, IOOp io_op, INSTR_TIME_ADD(PendingIOStats.pending_times[io_object][io_context][io_op], io_time); - - /* Add the per-backend count */ - pgstat_count_backend_io_op_time(io_object, io_context, io_op, - io_time); } pgstat_count_io_op(io_object, io_context, io_op, cnt, bytes); @@ -170,7 +168,7 @@ pgstat_flush_io(bool nowait) } /* - * Flush out locally pending IO statistics + * Flush out locally pending I/O statistics to the per-backend array entry. * * If no stats have been recorded, this function returns false. * @@ -180,20 +178,18 @@ pgstat_flush_io(bool nowait) bool pgstat_io_flush_cb(bool nowait) { - LWLock *bktype_lock; + PgStatShared_IOBackendEntry *entry; PgStat_BktypeIO *bktype_shstats; if (!have_iostats) return false; - bktype_lock = &pgStatLocal.shmem->io.locks[MyBackendType]; - bktype_shstats = - &pgStatLocal.shmem->io.stats.stats[MyBackendType]; + entry = pgstat_lock_my_per_backend_entry(PGSTAT_KIND_IO, nowait); - if (!nowait) - LWLockAcquire(bktype_lock, LW_EXCLUSIVE); - else if (!LWLockConditionalAcquire(bktype_lock, LW_EXCLUSIVE)) - return true; + if (entry == NULL) + return nowait; + + bktype_shstats = &entry->stats.stats; for (int io_object = 0; io_object < IOOBJECT_NUM_TYPES; io_object++) { @@ -217,12 +213,9 @@ pgstat_io_flush_cb(bool nowait) } } - Assert(pgstat_bktype_io_stats_valid(bktype_shstats, MyBackendType)); - - LWLockRelease(bktype_lock); + LWLockRelease(&entry->header.lock); memset(&PendingIOStats, 0, sizeof(PendingIOStats)); - have_iostats = false; return false; @@ -271,55 +264,63 @@ pgstat_io_init_shmem_cb(void *stats) { PgStatShared_IO *stat_shmem = (PgStatShared_IO *) stats; - for (int i = 0; i < BACKEND_NUM_TYPES; i++) - LWLockInitialize(&stat_shmem->locks[i], LWTRANCHE_PGSTATS_DATA); + LWLockInitialize(&stat_shmem->lock, LWTRANCHE_PGSTATS_DATA); } void pgstat_io_reset_all_cb(TimestampTz ts) { - for (int i = 0; i < BACKEND_NUM_TYPES; i++) - { - LWLock *bktype_lock = &pgStatLocal.shmem->io.locks[i]; - PgStat_BktypeIO *bktype_shstats = &pgStatLocal.shmem->io.stats.stats[i]; + PgStatShared_IO *shmem = &pgStatLocal.shmem->io; - LWLockAcquire(bktype_lock, LW_EXCLUSIVE); + /* + * Hold the kind lock while resetting both the global stats and active + * entries. Transfers hold the same lock, so pre-reset counters cannot be + * moved into the global stats after it is reset. + */ + LWLockAcquire(&shmem->lock, LW_EXCLUSIVE); + memset(&shmem->stats, 0, sizeof(shmem->stats)); + shmem->stats.stat_reset_timestamp = ts; - /* - * Use the lock in the first BackendType's PgStat_BktypeIO to protect - * the reset timestamp as well. - */ - if (i == 0) - pgStatLocal.shmem->io.stats.stat_reset_timestamp = ts; + /* Reset all per-backend entries */ + for (ProcNumber procnum = 0; procnum < NumPerBackendStatSlots; procnum++) + { + PgStatShared_IOBackendEntry *entry; + + entry = pgstat_get_per_backend_entry(PGSTAT_KIND_IO, procnum); - memset(bktype_shstats, 0, sizeof(*bktype_shstats)); - LWLockRelease(bktype_lock); + LWLockAcquire(&entry->header.lock, LW_EXCLUSIVE); + if (entry->header.backend_type != B_INVALID) + { + memset(&entry->stats.stats, 0, sizeof(PgStat_BktypeIO)); + entry->stats.stat_reset_timestamp = ts; + } + LWLockRelease(&entry->header.lock); } + + LWLockRelease(&shmem->lock); } +/* + * Build an I/O stats snapshot by aggregating global stats and all active + * per-backend entries. + */ void pgstat_io_snapshot_cb(void) { - for (int i = 0; i < BACKEND_NUM_TYPES; i++) - { - LWLock *bktype_lock = &pgStatLocal.shmem->io.locks[i]; - PgStat_BktypeIO *bktype_shstats = &pgStatLocal.shmem->io.stats.stats[i]; - PgStat_BktypeIO *bktype_snap = &pgStatLocal.snapshot.io.stats[i]; + PgStatShared_IO *shmem = &pgStatLocal.shmem->io; + PgStat_IO *snap = &pgStatLocal.snapshot.io; - LWLockAcquire(bktype_lock, LW_SHARED); + /* + * Prevent entries from moving to the global stats between copying it and + * scanning the per-backend array. + */ + LWLockAcquire(&shmem->lock, LW_SHARED); + memcpy(snap, &shmem->stats, sizeof(PgStat_IO)); - /* - * Use the lock in the first BackendType's PgStat_BktypeIO to protect - * the reset timestamp as well. - */ - if (i == 0) - pgStatLocal.snapshot.io.stat_reset_timestamp = - pgStatLocal.shmem->io.stats.stat_reset_timestamp; + /* Add in all active per-backend entries */ + pgstat_per_backend_snapshot(PGSTAT_KIND_IO, snap); - /* using struct assignment due to better type safety */ - *bktype_snap = *bktype_shstats; - LWLockRelease(bktype_lock); - } + LWLockRelease(&shmem->lock); } /* @@ -584,3 +585,111 @@ pgstat_tracks_io_op(BackendType bktype, IOObject io_object, return true; } + +/* + * Accumulate I/O counters from src into dst. + */ +static inline void +pgstat_io_accumulate_counters(PgStat_BktypeIO *dst, const PgStat_BktypeIO *src) +{ + for (int io_object = 0; io_object < IOOBJECT_NUM_TYPES; io_object++) + { + for (int io_context = 0; io_context < IOCONTEXT_NUM_TYPES; io_context++) + { + for (int io_op = 0; io_op < IOOP_NUM_TYPES; io_op++) + { + dst->counts[io_object][io_context][io_op] += + src->counts[io_object][io_context][io_op]; + dst->bytes[io_object][io_context][io_op] += + src->bytes[io_object][io_context][io_op]; + dst->times[io_object][io_context][io_op] += + src->times[io_object][io_context][io_op]; + } + } + } +} + +/* + * Accumulate one per-backend I/O entry into a snapshot or the global stats. + */ +void +pgstat_io_per_backend_acc_cb(void *dst, void *entry) +{ + PgStat_IO *stats = dst; + PgStatShared_IOBackendEntry *e = (PgStatShared_IOBackendEntry *) entry; + BackendType bktype = e->header.backend_type; + + if (bktype == B_INVALID) + return; + + pgstat_io_accumulate_counters(&stats->stats[bktype], &e->stats.stats); +} + +/* + * Accumulate a backend's I/O stats into the global stats, then invalidate the + * entry. + * + * Called at backend exit after the final flush, or when a ProcNumber is + * being reused. + */ +void +pgstat_io_acc_backend_cb(void) +{ + pgstat_acc_my_per_backend(PGSTAT_KIND_IO, &pgStatLocal.shmem->io.lock); +} + +/* + * Accumulate all remaining per-backend I/O stats entries into the global + * stats and invalidate them. Called at clean server shutdown to ensure all + * flushed data is preserved in the stats file. + */ +void +pgstat_io_acc_all_backends(void) +{ + pgstat_acc_all_per_backend(PGSTAT_KIND_IO, &pgStatLocal.shmem->io.lock); +} + +/* + * Return per-backend I/O statistics for the given ProcNumber. + */ +PgStat_BackendIO * +pgstat_fetch_stat_backend_io(ProcNumber procnum) +{ + return (PgStat_BackendIO *) pgstat_fetch_per_backend(PGSTAT_KIND_IO, procnum); +} + +/* + * Reset a backend's I/O stats. Accumulate the entry's counters into the + * global stats, then zero the stats and set the reset timestamp. + */ +void +pgstat_io_reset_backend_cb(ProcNumber procnum, TimestampTz ts) +{ + PgStatShared_IO *shmem = &pgStatLocal.shmem->io; + PgStatShared_IOBackendEntry *entry; + + entry = pgstat_get_per_backend_entry(PGSTAT_KIND_IO, procnum); + if (entry == NULL) + return; + + LWLockAcquire(&shmem->lock, LW_EXCLUSIVE); + LWLockAcquire(&entry->header.lock, LW_EXCLUSIVE); + + if (entry->header.backend_type == B_INVALID) + { + LWLockRelease(&entry->header.lock); + LWLockRelease(&shmem->lock); + return; + } + + /* Accumulate current stats into global before zeroing */ + pgstat_io_accumulate_counters(&shmem->stats.stats[entry->header.backend_type], + &entry->stats.stats); + + /* Zero stats and set reset timestamp */ + memset(&entry->stats, 0, sizeof(entry->stats)); + entry->stats.stat_reset_timestamp = ts; + + LWLockRelease(&entry->header.lock); + LWLockRelease(&shmem->lock); +} diff --git a/src/backend/utils/activity/pgstat_relation.c b/src/backend/utils/activity/pgstat_relation.c index 5c70543ba88..4c3d54c350a 100644 --- a/src/backend/utils/activity/pgstat_relation.c +++ b/src/backend/utils/activity/pgstat_relation.c @@ -321,7 +321,6 @@ pgstat_report_vacuum(Relation rel, PgStat_Counter livetuples, * VACUUM command has processed all tables and committed. */ pgstat_flush_io(false); - (void) pgstat_flush_backend(false, PGSTAT_BACKEND_FLUSH_IO); } /* @@ -417,7 +416,6 @@ pgstat_report_analyze(Relation rel, /* see pgstat_report_vacuum() */ pgstat_flush_io(false); - (void) pgstat_flush_backend(false, PGSTAT_BACKEND_FLUSH_IO); } /* diff --git a/src/backend/utils/activity/pgstat_wal.c b/src/backend/utils/activity/pgstat_wal.c index 8009ac38801..064f1adc518 100644 --- a/src/backend/utils/activity/pgstat_wal.c +++ b/src/backend/utils/activity/pgstat_wal.c @@ -60,7 +60,6 @@ pgstat_report_wal(bool force) /* flush IO stats */ pgstat_flush_io(nowait); - (void) pgstat_flush_backend(nowait, PGSTAT_BACKEND_FLUSH_IO); } /* diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index 7922519f426..b4ede4be5a8 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -1671,32 +1671,47 @@ Datum pg_stat_get_backend_io(PG_FUNCTION_ARGS) { ReturnSetInfo *rsinfo; - BackendType bktype; int pid; - PgStat_Backend *backend_stats; - PgStat_BktypeIO *bktype_stats; + PGPROC *proc; + ProcNumber procnum; + PgBackendStatus *beentry; + BackendType bktype; + PgStat_BackendIO *backend_io; InitMaterializedSRF(fcinfo, 0); rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; pid = PG_GETARG_INT32(0); - backend_stats = pgstat_fetch_stat_backend_by_pid(pid, &bktype); - if (!backend_stats) + proc = BackendPidGetProc(pid); + + if (!proc) + proc = AuxiliaryPidGetProc(pid); + if (!proc) + return (Datum) 0; + + procnum = GetNumberFromPGProc(proc); + beentry = pgstat_get_beentry_by_proc_number(procnum); + + if (!beentry || beentry->st_procpid != pid) return (Datum) 0; - bktype_stats = &backend_stats->io_stats; + bktype = beentry->st_backendType; + backend_io = pgstat_fetch_stat_backend_io(procnum); + + if (!backend_io) + return (Datum) 0; /* * In Assert builds, we can afford an extra loop through all of the * counters (in pg_stat_io_build_tuples()), checking that only expected * stats are non-zero, since it keeps the non-Assert code cleaner. */ - Assert(pgstat_bktype_io_stats_valid(bktype_stats, bktype)); + Assert(pgstat_bktype_io_stats_valid(&backend_io->stats, bktype)); + + pg_stat_io_build_tuples(rsinfo, &backend_io->stats, bktype, + backend_io->stat_reset_timestamp); - /* save tuples with data from this PgStat_BktypeIO */ - pg_stat_io_build_tuples(rsinfo, bktype_stats, bktype, - backend_stats->stat_reset_timestamp); return (Datum) 0; } @@ -2224,20 +2239,14 @@ pg_stat_reset_backend_stats(PG_FUNCTION_ARGS) if (!beentry) PG_RETURN_VOID(); - /* Check if the backend type tracks statistics */ - if (!pgstat_tracks_backend_bktype(beentry->st_backendType)) - PG_RETURN_VOID(); - /* - * Accumulate the backend's WAL and lock stats into the global stats, then - * zero the entries. + * Accumulate the backend's WAL, lock and I/O stats into the global stats, + * then zero the entries. */ ts = GetCurrentTimestamp(); pgstat_wal_reset_backend_cb(procNumber, ts); pgstat_lock_reset_backend_cb(procNumber, ts); - - /* Reset IO stats still in PGSTAT_KIND_BACKEND */ - pgstat_reset(PGSTAT_KIND_BACKEND, InvalidOid, procNumber); + pgstat_io_reset_backend_cb(procNumber, ts); PG_RETURN_VOID(); } diff --git a/src/include/pgstat.h b/src/include/pgstat.h index 960f0001bd6..bc071c2e9aa 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -275,7 +275,7 @@ typedef struct PgStat_TableXactStatus * ------------------------------------------------------------ */ -#define PGSTAT_FILE_FORMAT_ID 0x01A5BCBD +#define PGSTAT_FILE_FORMAT_ID 0x01A5BCBE typedef struct PgStat_ArchiverStats { @@ -390,6 +390,15 @@ typedef struct PgStat_BktypeIO PgStat_Counter times[IOOBJECT_NUM_TYPES][IOCONTEXT_NUM_TYPES][IOOP_NUM_TYPES]; } PgStat_BktypeIO; +/* + * I/O statistics for one process, with an independent reset timestamp. + */ +typedef struct PgStat_BackendIO +{ + TimestampTz stat_reset_timestamp; + PgStat_BktypeIO stats; +} PgStat_BackendIO; + typedef struct PgStat_PendingIO { uint64 bytes[IOOBJECT_NUM_TYPES][IOCONTEXT_NUM_TYPES][IOOP_NUM_TYPES]; @@ -592,21 +601,8 @@ typedef struct PgStat_WalStats typedef struct PgStat_Backend { TimestampTz stat_reset_timestamp; - PgStat_BktypeIO io_stats; } PgStat_Backend; -/* --------- - * PgStat_BackendPending Non-flushed backend stats. - * --------- - */ -typedef struct PgStat_BackendPending -{ - /* - * Backend statistics store the same amount of IO data as PGSTAT_KIND_IO. - */ - PgStat_PendingIO pending_io; -} PgStat_BackendPending; - /* * Functions in pgstat.c */ @@ -647,17 +643,6 @@ extern PgStat_ArchiverStats *pgstat_fetch_stat_archiver(void); * Functions in pgstat_backend.c */ -/* used by pgstat_io.c for I/O stats tracked in backends */ -extern void pgstat_count_backend_io_op_time(IOObject io_object, - IOContext io_context, - IOOp io_op, - instr_time io_time); -extern void pgstat_count_backend_io_op(IOObject io_object, - IOContext io_context, - IOOp io_op, uint32 cnt, - uint64 bytes); - - extern PgStat_Backend *pgstat_fetch_stat_backend(ProcNumber procNumber); extern PgStat_Backend *pgstat_fetch_stat_backend_by_pid(int pid, BackendType *bktype); @@ -694,6 +679,8 @@ extern void pgstat_count_io_op_time(IOObject io_object, IOContext io_context, uint32 cnt, uint64 bytes); extern PgStat_IO *pgstat_fetch_stat_io(void); +extern PgStat_BackendIO *pgstat_fetch_stat_backend_io(ProcNumber procnum); +extern void pgstat_io_reset_backend_cb(ProcNumber procnum, TimestampTz ts); extern const char *pgstat_get_io_context_name(IOContext io_context); extern const char *pgstat_get_io_object_name(IOObject io_object); diff --git a/src/include/utils/pgstat_internal.h b/src/include/utils/pgstat_internal.h index 474be775430..393f09c477a 100644 --- a/src/include/utils/pgstat_internal.h +++ b/src/include/utils/pgstat_internal.h @@ -493,10 +493,10 @@ typedef struct PgStatShared_Checkpointer typedef struct PgStatShared_IO { /* - * locks[i] protects stats.stats[i]. locks[0] also protects - * stats.stat_reset_timestamp. + * The lock protects global stats and serializes snapshots and resets with + * transfers from per-backend entries. */ - LWLock locks[BACKEND_NUM_TYPES]; + LWLock lock; PgStat_IO stats; } PgStatShared_IO; @@ -535,6 +535,16 @@ typedef struct PgStatShared_PerBackendEntry BackendType backend_type; } PgStatShared_PerBackendEntry; +/* + * Per-backend entry for I/O statistics, stored in an array indexed by + * ProcNumber. + */ +typedef struct PgStatShared_IOBackendEntry +{ + PgStatShared_PerBackendEntry header; + PgStat_BackendIO stats; +} PgStatShared_IOBackendEntry; + /* * Per-backend entry for lock statistics, stored in an array indexed by * ProcNumber. @@ -822,12 +832,6 @@ extern void pgstat_archiver_snapshot_cb(void); * Functions in pgstat_backend.c */ -/* flags for pgstat_flush_backend() */ -#define PGSTAT_BACKEND_FLUSH_IO (1 << 0) /* Flush I/O statistics */ -#define PGSTAT_BACKEND_FLUSH_ALL (PGSTAT_BACKEND_FLUSH_IO) - -extern bool pgstat_flush_backend(bool nowait, uint32 flags); -extern bool pgstat_backend_flush_cb(bool nowait); extern void pgstat_backend_reset_timestamp_cb(PgStatShared_Common *header, TimestampTz ts); @@ -881,6 +885,9 @@ extern bool pgstat_io_flush_cb(bool nowait); extern void pgstat_io_init_shmem_cb(void *stats); extern void pgstat_io_reset_all_cb(TimestampTz ts); extern void pgstat_io_snapshot_cb(void); +extern void pgstat_io_acc_backend_cb(void); +extern void pgstat_io_acc_all_backends(void); +extern void pgstat_io_per_backend_acc_cb(void *dst, void *entry); /* * Functions in pgstat_lock.c diff --git a/src/test/regress/expected/stats.out b/src/test/regress/expected/stats.out index e3dd0ce89b0..9f1f635234d 100644 --- a/src/test/regress/expected/stats.out +++ b/src/test/regress/expected/stats.out @@ -1956,14 +1956,21 @@ SELECT :io_stats_post_reset < :io_stats_pre_reset; SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_post_reset FROM pg_stat_get_backend_io(pg_backend_pid()) \gset --- pg_stat_reset_shared() did not reset backend IO stats -SELECT :my_io_stats_pre_reset <= :my_io_stats_post_reset; +-- pg_stat_reset_shared() also resets per-backend IO stats +SELECT :my_io_stats_pre_reset > :my_io_stats_post_reset; ?column? ---------- t (1 row) --- but pg_stat_reset_backend_stats() does +SELECT bool_and(stats_reset IS NOT NULL) AS backend_io_reset_timestamp_set + FROM pg_stat_get_backend_io(pg_backend_pid()); + backend_io_reset_timestamp_set +-------------------------------- + t +(1 row) + +-- pg_stat_reset_backend_stats() also resets per-backend IO stats SELECT pg_stat_reset_backend_stats(pg_backend_pid()); pg_stat_reset_backend_stats ----------------------------- @@ -1989,11 +1996,13 @@ SELECT pg_stat_get_backend_io(0); ------------------------ (0 rows) --- Auxiliary processes return no data. -SELECT pg_stat_get_backend_io(:checkpointer_pid); - pg_stat_get_backend_io ------------------------- -(0 rows) +-- Auxiliary processes now return data +SELECT count(*) > 0 AS checkpointer_has_io_stats + FROM pg_stat_get_backend_io(:checkpointer_pid); + checkpointer_has_io_stats +--------------------------- + t +(1 row) -- test BRIN index doesn't block HOT update CREATE TABLE brin_hot ( diff --git a/src/test/regress/sql/stats.sql b/src/test/regress/sql/stats.sql index f4931fd84de..3f88c04f0aa 100644 --- a/src/test/regress/sql/stats.sql +++ b/src/test/regress/sql/stats.sql @@ -924,9 +924,11 @@ SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + SELECT :io_stats_post_reset < :io_stats_pre_reset; SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_post_reset FROM pg_stat_get_backend_io(pg_backend_pid()) \gset --- pg_stat_reset_shared() did not reset backend IO stats -SELECT :my_io_stats_pre_reset <= :my_io_stats_post_reset; --- but pg_stat_reset_backend_stats() does +-- pg_stat_reset_shared() also resets per-backend IO stats +SELECT :my_io_stats_pre_reset > :my_io_stats_post_reset; +SELECT bool_and(stats_reset IS NOT NULL) AS backend_io_reset_timestamp_set + FROM pg_stat_get_backend_io(pg_backend_pid()); +-- pg_stat_reset_backend_stats() also resets per-backend IO stats SELECT pg_stat_reset_backend_stats(pg_backend_pid()); SELECT sum(evictions) + sum(reuses) + sum(extends) + sum(fsyncs) + sum(reads) + sum(writes) + sum(writebacks) + sum(hits) AS my_io_stats_post_backend_reset FROM pg_stat_get_backend_io(pg_backend_pid()) \gset @@ -935,8 +937,9 @@ SELECT :my_io_stats_pre_reset > :my_io_stats_post_backend_reset; -- Check invalid input for pg_stat_get_backend_io() SELECT pg_stat_get_backend_io(NULL); SELECT pg_stat_get_backend_io(0); --- Auxiliary processes return no data. -SELECT pg_stat_get_backend_io(:checkpointer_pid); +-- Auxiliary processes now return data +SELECT count(*) > 0 AS checkpointer_has_io_stats + FROM pg_stat_get_backend_io(:checkpointer_pid); -- test BRIN index doesn't block HOT update CREATE TABLE brin_hot ( diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 549141f592d..f5e559945f2 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2301,6 +2301,7 @@ PgStatShared_Function PgStatShared_HashEntry PgStatShared_Index PgStatShared_IO +PgStatShared_IOBackendEntry PgStatShared_Lock PgStatShared_LockBackendEntry PgStatShared_PerBackendEntry @@ -2312,7 +2313,7 @@ PgStatShared_Wal PgStatShared_WalBackendEntry PgStat_ArchiverStats PgStat_Backend -PgStat_BackendPending +PgStat_BackendIO PgStat_BackendSubEntry PgStat_BgWriterStats PgStat_BktypeIO -- 2.34.1