From 6d3e3a04ade066e39e79777a6e4054aa57d81dd0 Mon Sep 17 00:00:00 2001 From: Bertrand Drouvot Date: Thu, 11 Jun 2026 09:45:19 +0000 Subject: [PATCH v1] Add per-backend AIO statistics This commit adds per-backend AIO statistics, providing per-backend AIO behavior details. This data can be retrieved with a new system function called pg_stat_get_backend_aio(), that returns the following counters based on the PID provided in input: - started: total number of AIO operations initiated - executed_sync: IOs that were executed synchronously (fallback path) - executed_async: IOs that were submitted asynchronously to the IO method - completed_self: IO completions processed by the issuing backend itself - completed_other: IO completions processed on behalf of another backend - handle_waits: times the backend had to wait for a free AIO handle - submitted: number of submitted calls to the IO method These counters are useful for understanding and tuning AIO behavior: - executed_async / started. A ratio near zero means the backend is falling back to synchronous execution (TOAST chunk fetches, temp buffers, ...). - a non-zero handle_waits means the backend exhausted all its AIO handles. That could mean that io_max_concurrency is too low. - completed_self vs completed_other reveals cross-backend completion patterns. That helps see how IO completion work is distributed and could help interpret per backend IO statistics values. - executed_async / submitted gives the average batch size per submit call. This commit is straight-forward, relying on the infrastructure provided by 9aea73fc61d4 (backend-level pgstats). XXX: Bump catalog version. No need to touch PGSTAT_FILE_FORMAT_ID as backend statistics are never written to disk. Author: Bertrand Drouvot Reviewed-by: Discussion: --- doc/src/sgml/monitoring.sgml | 70 +++++++++++++ src/backend/storage/aio/aio.c | 13 +++ src/backend/utils/activity/pgstat_backend.c | 109 ++++++++++++++++++++ src/backend/utils/adt/pgstatfuncs.c | 60 +++++++++++ src/include/catalog/pg_proc.dat | 7 ++ src/include/pgstat.h | 25 +++++ src/include/utils/pgstat_internal.h | 3 +- src/test/modules/test_aio/t/001_aio.pl | 32 ++++++ src/tools/pgindent/typedefs.list | 1 + 9 files changed, 319 insertions(+), 1 deletion(-) 24.4% doc/src/sgml/ 3.2% src/backend/storage/aio/ 25.3% src/backend/utils/activity/ 20.5% src/backend/utils/adt/ 4.9% src/include/catalog/ 10.0% src/include/ 11.1% src/test/modules/test_aio/t/ diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index 12b9ee20d4a..1c566cf0e0e 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -5641,6 +5641,76 @@ description | Waiting for a newly initialized WAL file to reach durable storage + + + + pg_stat_get_backend_aio + + pg_stat_get_backend_aio ( integer ) + record + + + Returns AIO (Asynchronous I/O) statistics about the + backend with the specified process ID. The returned values are: + + + + started: Total IOs initiated. + + + + + executed_sync: IOs executed synchronously. + + + + + executed_async: IOs submitted asynchronously. + + + + + completed_self: IO completions processed by this + backend. + + + + + completed_other: IO completions processed on + behalf of other backends. + + + + + handle_waits: Times waited for a free AIO handle. + + + + + submitted: Number of submit calls to the IO + method. Compare with executed_async to determine + average batch size. + + + + + stats_reset: Timestamp of last stats reset. + + + + + + The completed_other column is only meaningful + when io_method is set to io_uring; + with worker mode, IO completions are processed by + IO worker processes which do not track these statistics. + + + The function does not return AIO statistics for the checkpointer, + the background writer, the startup process and the autovacuum launcher. + + + diff --git a/src/backend/storage/aio/aio.c b/src/backend/storage/aio/aio.c index 8f7e26607b9..507f1727d41 100644 --- a/src/backend/storage/aio/aio.c +++ b/src/backend/storage/aio/aio.c @@ -40,6 +40,7 @@ #include "lib/ilist.h" #include "miscadmin.h" +#include "pgstat.h" #include "port/atomics.h" #include "storage/aio.h" #include "storage/aio_internal.h" @@ -458,6 +459,8 @@ pgaio_io_stage(PgAioHandle *ioh, PgAioOp op) "staged (synchronous: %d, in_batch: %d)", needs_synchronous, pgaio_my_backend->in_batchmode); + pgstat_count_backend_aio_start(needs_synchronous); + if (!needs_synchronous) { pgaio_my_backend->staged_ios[pgaio_my_backend->num_staged_ios++] = ioh; @@ -544,6 +547,12 @@ pgaio_io_process_completion(PgAioHandle *ioh, int result) /* condition variable broadcast ensures state is visible before wakeup */ ConditionVariableBroadcast(&ioh->cv); + /* Track AIO completion stats */ + if (ioh->owner_procno == MyProcNumber) + pgstat_count_backend_aio_complete_self(); + else + pgstat_count_backend_aio_complete_other(); + /* contains call to pgaio_io_call_complete_local() */ if (ioh->owner_procno == MyProcNumber) pgaio_io_reclaim(ioh); @@ -762,6 +771,8 @@ pgaio_io_wait_for_free(void) { int reclaimed = 0; + pgstat_count_backend_aio_handle_wait(); + pgaio_debug(DEBUG2, "waiting for free IO with %d pending, %u in-flight, %u idle IOs", pgaio_my_backend->num_staged_ios, dclist_count(&pgaio_my_backend->in_flight_ios), @@ -1150,6 +1161,8 @@ pgaio_submit_staged(void) Assert(total_submitted == did_submit); + pgstat_count_backend_aio_submitted(); + pgaio_my_backend->num_staged_ios = 0; pgaio_debug(DEBUG4, diff --git a/src/backend/utils/activity/pgstat_backend.c b/src/backend/utils/activity/pgstat_backend.c index b736b2ccc6f..4ad755f7f18 100644 --- a/src/backend/utils/activity/pgstat_backend.c +++ b/src/backend/utils/activity/pgstat_backend.c @@ -40,6 +40,7 @@ static PgStat_BackendPending PendingBackendStats; static bool backend_has_iostats = false; static bool backend_has_lockstats = false; +static bool backend_has_aiostats = false; /* * WAL usage counters saved from pgWalUsage at the previous call to @@ -120,6 +121,74 @@ pgstat_count_backend_lock_fastpath_exceeded(uint8 locktag_type) pgstat_report_fixed = true; } +/* + * Utility routines to report AIO stats for backends, kept here to avoid + * exposing PendingBackendStats to the outside world. + */ +void +pgstat_count_backend_aio_start(bool synchronous) +{ + if (!pgstat_tracks_backend_bktype(MyBackendType)) + return; + + PendingBackendStats.aio_counters.started++; + if (synchronous) + PendingBackendStats.aio_counters.executed_sync++; + else + PendingBackendStats.aio_counters.executed_async++; + + backend_has_aiostats = true; + pgstat_report_fixed = true; +} + +void +pgstat_count_backend_aio_complete_self(void) +{ + if (!pgstat_tracks_backend_bktype(MyBackendType)) + return; + + PendingBackendStats.aio_counters.completed_self++; + + backend_has_aiostats = true; + pgstat_report_fixed = true; +} + +void +pgstat_count_backend_aio_complete_other(void) +{ + if (!pgstat_tracks_backend_bktype(MyBackendType)) + return; + + PendingBackendStats.aio_counters.completed_other++; + + backend_has_aiostats = true; + pgstat_report_fixed = true; +} + +void +pgstat_count_backend_aio_handle_wait(void) +{ + if (!pgstat_tracks_backend_bktype(MyBackendType)) + return; + + PendingBackendStats.aio_counters.handle_waits++; + + backend_has_aiostats = true; + pgstat_report_fixed = true; +} + +void +pgstat_count_backend_aio_submitted(void) +{ + if (!pgstat_tracks_backend_bktype(MyBackendType)) + return; + + PendingBackendStats.aio_counters.submitted++; + + backend_has_aiostats = true; + pgstat_report_fixed = true; +} + /* * Returns statistics of a backend by proc number. */ @@ -326,6 +395,38 @@ pgstat_flush_backend_entry_lock(PgStat_EntryRef *entry_ref) backend_has_lockstats = false; } +/* + * Flush out locally pending backend AIO statistics. Locking is managed + * by the caller. + */ +static void +pgstat_flush_backend_entry_aio(PgStat_EntryRef *entry_ref) +{ + PgStatShared_Backend *shbackendent; + PgStat_AioCounters *bktype_shstats; + + if (!backend_has_aiostats) + return; + + shbackendent = (PgStatShared_Backend *) entry_ref->shared_stats; + bktype_shstats = &shbackendent->stats.aio_counters; + +#define AIOSTAT_ACC(fld) \ + (bktype_shstats->fld += PendingBackendStats.aio_counters.fld) + AIOSTAT_ACC(started); + AIOSTAT_ACC(executed_sync); + AIOSTAT_ACC(executed_async); + AIOSTAT_ACC(completed_self); + AIOSTAT_ACC(completed_other); + AIOSTAT_ACC(handle_waits); + AIOSTAT_ACC(submitted); +#undef AIOSTAT_ACC + + MemSet(&PendingBackendStats.aio_counters, 0, sizeof(PgStat_AioCounters)); + + backend_has_aiostats = false; +} + /* * Flush out locally pending backend statistics * @@ -354,6 +455,10 @@ pgstat_flush_backend(bool nowait, uint32 flags) if ((flags & PGSTAT_BACKEND_FLUSH_LOCK) && backend_has_lockstats) has_pending_data = true; + /* Some AIO data pending? */ + if ((flags & PGSTAT_BACKEND_FLUSH_AIO) && backend_has_aiostats) + has_pending_data = true; + if (!has_pending_data) return false; @@ -372,6 +477,9 @@ pgstat_flush_backend(bool nowait, uint32 flags) if (flags & PGSTAT_BACKEND_FLUSH_LOCK) pgstat_flush_backend_entry_lock(entry_ref); + if (flags & PGSTAT_BACKEND_FLUSH_AIO) + pgstat_flush_backend_entry_aio(entry_ref); + pgstat_unlock_entry(entry_ref); return false; @@ -411,6 +519,7 @@ pgstat_create_backend(ProcNumber procnum) MemSet(&PendingBackendStats, 0, sizeof(PgStat_BackendPending)); backend_has_iostats = false; backend_has_lockstats = false; + backend_has_aiostats = false; /* * Initialize prevBackendWalUsage with pgWalUsage so that diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index 565d0e70768..fb62a56f3fe 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -1722,6 +1722,66 @@ pg_stat_get_backend_wal(PG_FUNCTION_ARGS) return (pg_stat_wal_build_tuple(bktype_stats, backend_stats->stat_reset_timestamp)); } +/* + * Returns AIO statistics for a backend with given PID. + */ +Datum +pg_stat_get_backend_aio(PG_FUNCTION_ARGS) +{ +#define PG_STAT_BACKEND_AIO_COLS 8 + TupleDesc tupdesc; + Datum values[PG_STAT_BACKEND_AIO_COLS] = {0}; + bool nulls[PG_STAT_BACKEND_AIO_COLS] = {0}; + int pid; + PgStat_Backend *backend_stats; + PgStat_AioCounters aio_counters; + + pid = PG_GETARG_INT32(0); + backend_stats = pgstat_fetch_stat_backend_by_pid(pid, NULL); + + if (!backend_stats) + PG_RETURN_NULL(); + + aio_counters = backend_stats->aio_counters; + + /* Initialise attributes information in the tuple descriptor */ + tupdesc = CreateTemplateTupleDesc(PG_STAT_BACKEND_AIO_COLS); + TupleDescInitEntry(tupdesc, (AttrNumber) 1, "started", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 2, "executed_sync", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 3, "executed_async", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 4, "completed_self", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 5, "completed_other", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 6, "handle_waits", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 7, "submitted", + INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 8, "stats_reset", + TIMESTAMPTZOID, -1, 0); + TupleDescFinalize(tupdesc); + BlessTupleDesc(tupdesc); + + /* Fill values */ + values[0] = Int64GetDatum(aio_counters.started); + values[1] = Int64GetDatum(aio_counters.executed_sync); + values[2] = Int64GetDatum(aio_counters.executed_async); + values[3] = Int64GetDatum(aio_counters.completed_self); + values[4] = Int64GetDatum(aio_counters.completed_other); + values[5] = Int64GetDatum(aio_counters.handle_waits); + values[6] = Int64GetDatum(aio_counters.submitted); + + if (backend_stats->stat_reset_timestamp != 0) + values[7] = TimestampTzGetDatum(backend_stats->stat_reset_timestamp); + else + nulls[7] = true; + + PG_RETURN_DATUM(HeapTupleGetDatum(heap_form_tuple(tupdesc, values, nulls))); +} + /* * Returns statistics of WAL activity */ diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 3cb84359adf..a089860ada4 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -6108,6 +6108,13 @@ proargmodes => '{i,o,o,o,o,o}', proargnames => '{backend_pid,locktype,waits,wait_time,fastpath_exceeded,stats_reset}', prosrc => 'pg_stat_get_backend_lock' }, +{ oid => '9082', descr => 'statistics: backend AIO activity', + proname => 'pg_stat_get_backend_aio', provolatile => 'v', proparallel => 'r', + prorettype => 'record', proargtypes => 'int4', + proallargtypes => '{int4,int8,int8,int8,int8,int8,int8,int8,timestamptz}', + proargmodes => '{i,o,o,o,o,o,o,o,o}', + proargnames => '{backend_pid,started,executed_sync,executed_async,completed_self,completed_other,handle_waits,submitted,stats_reset}', + prosrc => 'pg_stat_get_backend_aio' }, { oid => '6248', descr => 'statistics: information about WAL prefetching', proname => 'pg_stat_get_recovery_prefetch', prorows => '1', proretset => 't', provolatile => 'v', prorettype => 'record', proargtypes => '', diff --git a/src/include/pgstat.h b/src/include/pgstat.h index 58a44857f13..64afb2bc082 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -514,6 +514,21 @@ typedef struct PgStat_WalStats TimestampTz stat_reset_timestamp; } PgStat_WalStats; +/* ------- + * PgStat_AioCounters AIO activity counters + * ------- + */ +typedef struct PgStat_AioCounters +{ + PgStat_Counter started; + PgStat_Counter executed_sync; + PgStat_Counter executed_async; + PgStat_Counter completed_self; + PgStat_Counter completed_other; + PgStat_Counter handle_waits; + PgStat_Counter submitted; +} PgStat_AioCounters; + /* ------- * PgStat_Backend Backend statistics * ------- @@ -524,6 +539,7 @@ typedef struct PgStat_Backend PgStat_BktypeIO io_stats; PgStat_WalCounters wal_counters; PgStat_PendingLock lock_stats; + PgStat_AioCounters aio_counters; } PgStat_Backend; /* --------- @@ -542,6 +558,8 @@ typedef struct PgStat_BackendPending * PGSTAT_KIND_LOCK. */ PgStat_PendingLock pending_lock; + /* Store the AIO statistics counters */ + PgStat_AioCounters aio_counters; } PgStat_BackendPending; /* @@ -598,6 +616,13 @@ extern void pgstat_count_backend_io_op(IOObject io_object, extern void pgstat_count_backend_lock_waits(uint8 locktag_type, PgStat_Counter usecs); extern void pgstat_count_backend_lock_fastpath_exceeded(uint8 locktag_type); +/* used by aio.c for AIO stats tracked in backends */ +extern void pgstat_count_backend_aio_start(bool synchronous); +extern void pgstat_count_backend_aio_complete_self(void); +extern void pgstat_count_backend_aio_complete_other(void); +extern void pgstat_count_backend_aio_handle_wait(void); +extern void pgstat_count_backend_aio_submitted(void); + extern PgStat_Backend *pgstat_fetch_stat_backend(ProcNumber procNumber); extern PgStat_Backend *pgstat_fetch_stat_backend_by_pid(int pid, BackendType *bktype); diff --git a/src/include/utils/pgstat_internal.h b/src/include/utils/pgstat_internal.h index b3dc3ff7d8b..b2092fb42b9 100644 --- a/src/include/utils/pgstat_internal.h +++ b/src/include/utils/pgstat_internal.h @@ -706,7 +706,8 @@ extern void pgstat_archiver_snapshot_cb(void); #define PGSTAT_BACKEND_FLUSH_IO (1 << 0) /* Flush I/O statistics */ #define PGSTAT_BACKEND_FLUSH_WAL (1 << 1) /* Flush WAL statistics */ #define PGSTAT_BACKEND_FLUSH_LOCK (1 << 2) /* Flush lock statistics */ -#define PGSTAT_BACKEND_FLUSH_ALL (PGSTAT_BACKEND_FLUSH_IO | PGSTAT_BACKEND_FLUSH_WAL | PGSTAT_BACKEND_FLUSH_LOCK) +#define PGSTAT_BACKEND_FLUSH_AIO (1 << 3) /* Flush AIO statistics */ +#define PGSTAT_BACKEND_FLUSH_ALL (PGSTAT_BACKEND_FLUSH_IO | PGSTAT_BACKEND_FLUSH_WAL | PGSTAT_BACKEND_FLUSH_LOCK | PGSTAT_BACKEND_FLUSH_AIO) extern bool pgstat_flush_backend(bool nowait, uint32 flags); extern bool pgstat_backend_flush_cb(bool nowait); diff --git a/src/test/modules/test_aio/t/001_aio.pl b/src/test/modules/test_aio/t/001_aio.pl index 63cadd64c15..8ecf3d3ad91 100644 --- a/src/test/modules/test_aio/t/001_aio.pl +++ b/src/test/modules/test_aio/t/001_aio.pl @@ -1842,6 +1842,37 @@ read_buffers('$table', 0, 4)|, $psql_c->quit(); } +# Test per-backend AIO statistics counters +sub test_aio_stats +{ + my $io_method = shift; + my $node = shift; + + my $psql = $node->background_psql('postgres', on_error_stop => 0); + + # Reset backend stats, evict relation, then read it back to force + # physical IO through the AIO layer. + $psql->query_safe(qq(SELECT pg_stat_reset_backend_stats(pg_backend_pid()))); + $psql->query_safe(qq(SELECT evict_rel('tbl_ok'))); + $psql->query_safe(qq(SELECT count(*) FROM tbl_ok)); + $psql->query_safe(qq(SELECT pg_stat_force_next_flush())); + + # started must be > 0 after physical IO + my $started = $psql->query_safe( + qq(SELECT started FROM pg_stat_get_backend_aio(pg_backend_pid()))); + cmp_ok($started, '>', 0, + "$io_method: AIO stats: started > 0 after physical IO"); + + # invariant: started = executed_sync + executed_async + my $consistent = $psql->query_safe( + qq(SELECT started = executed_sync + executed_async + FROM pg_stat_get_backend_aio(pg_backend_pid()))); + is($consistent, 't', + "$io_method: AIO stats: started = executed_sync + executed_async"); + + $psql->quit(); +} + # Run all tests that for the specified node / io_method sub test_io_method { @@ -1878,6 +1909,7 @@ CHECKPOINT; test_ignore_checksum($io_method, $node); test_checksum_createdb($io_method, $node); test_read_buffers($io_method, $node); + test_aio_stats($io_method, $node); # generic injection tests SKIP: diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 117e7379f10..6ce3077fa5b 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2329,6 +2329,7 @@ PgStatShared_ReplSlot PgStatShared_SLRU PgStatShared_Subscription PgStatShared_Wal +PgStat_AioCounters PgStat_ArchiverStats PgStat_Backend PgStat_BackendPending -- 2.34.1