From 4e5e3c6a54bbd98f4862ea07d47a4021f61b831e Mon Sep 17 00:00:00 2001 From: Lukas Fittl Date: Mon, 16 Nov 2026 16:40:13 -0800 Subject: [PATCH vnocfbot] pgstat: Allow variable stats kinds to be flushed at statement end Pending statistics are only flushed once a backend is idle outside a transaction, so a long-running transaction's statistics stay invisible to other sessions until it commits. These semantics are not appropriate for statement-level statistics, such as the proposed pg_stat_statements rework on top of the cumulative statistics system. Add a flush_at_statement_end option for variable-numbered kinds whose pending statistics never depend on the transaction's outcome, and whose code keeps no pointers to entry references or pending statistics across statements. Such kinds are also flushed inside a transaction, at the end of top-level statements and when the backend goes idle, at most once per PGSTAT_MIN_INTERVAL (1s), and after PGSTAT_IDLE_INTERVAL (10s) if it stays idle. Flushed entries are deleted and their references released right away, so that dropped entries aren't kept allocated under high churn, at a slight cost when the same statement runs again. --- NOTE: This contains pg_stat_statements changes for reference since its a sketch, but these would be separated out for a commit. --- contrib/pg_stat_statements/meson.build | 1 + .../pg_stat_statements/pg_stat_statements.c | 1 + .../t/014_statement_flush.pl | 77 ++++++++ doc/src/sgml/pgstatstatements.sgml | 14 +- src/backend/tcop/postgres.c | 41 ++++- src/backend/utils/activity/pgstat.c | 168 +++++++++++++++++- src/backend/utils/activity/pgstat_shmem.c | 17 ++ src/include/utils/pgstat_internal.h | 14 ++ 8 files changed, 316 insertions(+), 17 deletions(-) create mode 100644 contrib/pg_stat_statements/t/014_statement_flush.pl diff --git a/contrib/pg_stat_statements/meson.build b/contrib/pg_stat_statements/meson.build index 557f6eceaaa..dd51b4865dc 100644 --- a/contrib/pg_stat_statements/meson.build +++ b/contrib/pg_stat_statements/meson.build @@ -74,6 +74,7 @@ tests += { 't/010_restart.pl', 't/011_eviction.pl', 't/013_qtext_memory.pl', + 't/014_statement_flush.pl', ], }, } diff --git a/contrib/pg_stat_statements/pg_stat_statements.c b/contrib/pg_stat_statements/pg_stat_statements.c index 887e4ca51b0..4af152bc1be 100644 --- a/contrib/pg_stat_statements/pg_stat_statements.c +++ b/contrib/pg_stat_statements/pg_stat_statements.c @@ -474,6 +474,7 @@ static const PgStat_KindInfo pgss_stats_kind = { .own_hash = true, .track_entry_count = true, .release_refs_after_flush = true, + .flush_at_statement_end = true, .shared_size = sizeof(pgssSharedEntry), .shared_data_off = offsetof(pgssSharedEntry, entry), .shared_data_len = sizeof(pgssEntry), diff --git a/contrib/pg_stat_statements/t/014_statement_flush.pl b/contrib/pg_stat_statements/t/014_statement_flush.pl new file mode 100644 index 00000000000..ff3878e1bae --- /dev/null +++ b/contrib/pg_stat_statements/t/014_statement_flush.pl @@ -0,0 +1,77 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group + +# Tests that the statistics of executions become visible to other sessions +# while the executing session is still inside a transaction block, running +# statements in it or idle in it, rather than only once it is idle outside a +# transaction. +# +# Column aliases are not part of query identifiers, so the statements of the +# tests differ by their number of columns instead. + +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $node = PostgreSQL::Test::Cluster->new('main'); +$node->init; +$node->append_conf('postgresql.conf', + "shared_preload_libraries = 'pg_stat_statements'"); +$node->start; + +$node->safe_psql('postgres', 'CREATE EXTENSION pg_stat_statements'); + +# Has the statement with this text been counted, as seen by another session? +sub counted_query +{ + my ($text) = @_; + return "SELECT coalesce(sum(calls), 0) > 0 FROM pg_stat_statements " + . "WHERE query = '$text'"; +} + +# A session idle in a transaction block flushes its statistics. +my $session = $node->background_psql('postgres'); +$session->query_safe("BEGIN"); +$session->query_safe("SELECT 'idle_in_xact', 1, 1"); +ok( $node->poll_query_until( + 'postgres', counted_query('SELECT $1, $2, $3')), + 'statement visible while its session is idle in a transaction block'); +$session->query_safe("COMMIT"); + +# A session running a multi-statement query in a transaction block, never +# idle until it ends, flushes its statistics at statement end. Its state +# stays active meanwhile, so it can't have flushed them while idle. +my $pid = $session->query_safe("SELECT pg_backend_pid()"); +$session->query_until(qr/started/, + "\\echo started\n" + . "BEGIN \\; SELECT 'busy_in_xact', 1, 1, 1 \\; SELECT pg_sleep(1.5) \\; " + . "SELECT pg_sleep(5) \\; COMMIT;\n"); +ok( $node->poll_query_until( + 'postgres', + "SELECT EXISTS (SELECT FROM pg_stat_activity WHERE pid = $pid AND state = 'active') AND (" + . counted_query('SELECT $1, $2, $3, $4') . ")"), + 'statement visible while its multi-statement query is running'); +$session->query_safe("SELECT 1"); + +# Each execution is added exactly once, though the statistics of a +# transaction block are flushed in several parts. +$session->query_safe("BEGIN"); +for (1 .. 3) +{ + $session->query_safe("SELECT 'repeated', 1, 1, 1, 1"); + $session->query_safe("SELECT pg_sleep(1.1)"); +} +$session->query_safe("COMMIT"); +$session->query_safe("SELECT pg_stat_force_next_flush()"); +is( $node->safe_psql( + 'postgres', + "SELECT calls FROM pg_stat_statements WHERE query = 'SELECT \$1, \$2, \$3, \$4, \$5'" + ), + '3', + 'statement counted once per execution'); + +$session->quit; +$node->stop; + +done_testing(); diff --git a/doc/src/sgml/pgstatstatements.sgml b/doc/src/sgml/pgstatstatements.sgml index 7d287d49a22..c159b08c015 100644 --- a/doc/src/sgml/pgstatstatements.sgml +++ b/doc/src/sgml/pgstatstatements.sgml @@ -748,18 +748,20 @@ calls | 2 The statistics of an execution are kept in the local memory of its - session, and added to its statement's entry when the session next reports - cumulative statistics, normally within a second once it is idle outside a - transaction block (see ). Until - then, a statement whose executions are all still in local memory does - not appear in the view. A session reading the + session, and added to its statement's entry at the end of a later + statement or when the session next waits for a command, at most about + once a second, and otherwise within ten seconds if it stays idle. This + also happens inside a transaction block, so a long transaction does not + delay it. Statements nested in functions and procedures are added once + the outer statement completes. Until then, a statement whose executions + are all still in local memory does not appear in the view. A session reading the pg_stat_statements view first adds its own such statistics. The memory of a discarded statement is released once the sessions that - executed it have reported their statistics. When statements are + executed it have added their statistics. When statements are discarded so fast that as many wait for that as there can be tracked statements, new statements are not tracked until the memory is released, normally within a second; statements already tracked are still counted. diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c index d6761aeea9f..fcbd1074962 100644 --- a/src/backend/tcop/postgres.c +++ b/src/backend/tcop/postgres.c @@ -1305,6 +1305,9 @@ exec_simple_query(const char *query_string) PortalDrop(portal, false); + /* Flush the stats kinds that allow it, if due */ + pgstat_report_stat(false); + if (lnext(parsetree_list, parsetree_item) == NULL) { /* @@ -2333,6 +2336,9 @@ exec_execute_message(const char *portal_name, long max_rows) /* Done executing; remove the params error callback */ error_context_stack = error_context_stack->previous; + /* Flush the stats kinds that allow it, if due, before any commit below */ + pgstat_report_stat(false); + if (completed) { if (is_xact_command || (MyXactFlags & XACT_FLAGS_NEEDIMMEDIATECOMMIT)) @@ -3685,13 +3691,26 @@ ProcessInterrupts(void) /* * If there are pending stats updates and we currently are truly idle - * (matching the conditions in PostgresMain(), report stats now. + * (matching the conditions in PostgresMain(), report stats now. If idle + * in a transaction block that hasn't failed, flush the stats kinds that + * allow it, and retry later if some remain. */ - if (IdleStatsUpdateTimeoutPending && - DoingCommandRead && !IsTransactionOrTransactionBlock()) + if (IdleStatsUpdateTimeoutPending && DoingCommandRead) { - IdleStatsUpdateTimeoutPending = false; - pgstat_report_stat(true); + if (!IsTransactionOrTransactionBlock()) + { + IdleStatsUpdateTimeoutPending = false; + pgstat_report_stat(true); + } + else if (!IsAbortedTransactionBlockState()) + { + long stats_timeout; + + IdleStatsUpdateTimeoutPending = false; + stats_timeout = pgstat_report_stat(false); + if (stats_timeout > 0) + enable_timeout_after(IDLE_STATS_UPDATE_TIMEOUT, stats_timeout); + } } if (ProcSignalBarrierPending) @@ -4757,9 +4776,21 @@ PostgresMain(const char *dbname, const char *username) } else if (IsTransactionOrTransactionBlock()) { + long stats_timeout; + set_ps_display("idle in transaction"); pgstat_report_activity(STATE_IDLEINTRANSACTION, NULL); + /* + * Flush the stats kinds that allow it inside a transaction, + * or arm the idle stats timeout to do so later. + */ + stats_timeout = pgstat_report_stat(false); + if (stats_timeout > 0 && + !get_timeout_active(IDLE_STATS_UPDATE_TIMEOUT)) + enable_timeout_after(IDLE_STATS_UPDATE_TIMEOUT, + stats_timeout); + /* Start the idle-in-transaction timer */ if (IdleInTransactionSessionTimeout > 0 && (IdleInTransactionSessionTimeout < TransactionTimeout || TransactionTimeout == 0)) diff --git a/src/backend/utils/activity/pgstat.c b/src/backend/utils/activity/pgstat.c index 68864426af7..cbbb9844291 100644 --- a/src/backend/utils/activity/pgstat.c +++ b/src/backend/utils/activity/pgstat.c @@ -188,6 +188,8 @@ static void pgstat_init_snapshot_fixed(void); static void pgstat_reset_after_failure(void); static bool pgstat_flush_pending_entries(bool nowait); +static bool pgstat_flush_statement_entries(void); +static long pgstat_report_statement_stats(void); static void pgstat_prep_snapshot(void); static void pgstat_build_snapshot(void); @@ -265,6 +267,15 @@ static PgStat_EntryRef *pgStatCurrentFlushEntry = NULL; /* Generation number of the current pgstat_flush_pending_entries() pass. */ static uint64 pgStatFlushGeneration = 0; +/* Number of pending entries of kinds with flush_at_statement_end */ +static int pgStatNumStatementPending = 0; + +/* When the kinds with flush_at_statement_end were last flushed */ +static TimestampTz pgStatLastStatementFlush = 0; + +/* Statement start timestamp seen by pgstat_report_statement_stats() */ +static TimestampTz pgStatLastStatementTimestamp = 0; + /* * Force-clear existing snapshot before next use when stats_fetch_consistency * is changed. @@ -761,12 +772,15 @@ pgstat_initialize(void) * Whenever pending stats updates remain at the end of pgstat_report_stat() a * suggested idle timeout is returned. Currently this is always * PGSTAT_IDLE_INTERVAL (10000ms). Callers can use the returned time to set up - * a timeout after which to call pgstat_report_stat(true), but are not + * a timeout after which to call pgstat_report_stat() again, but are not * required to do so. * - * A nonforced flush only happens outside a transaction, so transaction stop - * time is close enough. A forced flush can happen during a transaction and - * uses the current time instead. + * Inside a transaction, a nonforced call only flushes the kinds with + * flush_at_statement_end, see pgstat_report_statement_stats(). + * + * A nonforced full flush only happens outside a transaction, so transaction + * stop time is close enough. A forced flush can happen during a transaction + * and uses the current time instead. */ long pgstat_report_stat(bool force) @@ -778,7 +792,17 @@ pgstat_report_stat(bool force) bool nowait; pgstat_assert_is_up(); - Assert(force || !IsTransactionOrTransactionBlock()); + + /* + * A nonforced call from ProcessInterrupts() can arrive while a flush is + * in progress. Retry later. + */ + if (!force && pgStatFlushInProgress) + return PGSTAT_IDLE_INTERVAL; + + /* Inside a transaction, only flush the kinds that allow it */ + if (!force && IsTransactionOrTransactionBlock()) + return pgstat_report_statement_stats(); /* a prior flush must have cleared this, even if it errored out */ Assert(!pgStatFlushInProgress); @@ -896,6 +920,7 @@ pgstat_report_stat(bool force) PG_END_TRY(); last_flush = now; + pgStatLastStatementFlush = now; /* Let the caller know when to retry after a partial flush. */ if (partial_flush) @@ -936,6 +961,45 @@ pgstat_force_next_flush(void) pgStatForceNextFlush = true; } +/* + * Flush the kinds with flush_at_statement_end if PGSTAT_MIN_INTERVAL has + * passed since their last flush. Returns PGSTAT_IDLE_INTERVAL if any of their + * statistics remain pending, else 0. + * + * To avoid a clock read per statement, "now" is the start of the last + * statement, which advances with each client message. If it hasn't advanced + * since the previous call, as for the later statements of a multi-statement + * query string, or when the idle timeout fires, read the clock. + */ +static long +pgstat_report_statement_stats(void) +{ + TimestampTz now; + + if (pgStatNumStatementPending == 0 || pgStatLocal.shmem->is_shutdown) + return 0; + + now = GetCurrentStatementStartTimestamp(); + if (now == pgStatLastStatementTimestamp) + now = GetCurrentTimestamp(); + else + pgStatLastStatementTimestamp = now; + + if (!TimestampDifferenceExceeds(pgStatLastStatementFlush, now, + PGSTAT_MIN_INTERVAL)) + return PGSTAT_IDLE_INTERVAL; + + Assert(!pgStatFlushInProgress); + Assert(pgStatCurrentFlushEntry == NULL); + + pgStatLastStatementFlush = now; + + if (pgstat_flush_statement_entries()) + return PGSTAT_IDLE_INTERVAL; + + return 0; +} + /* * Only for use by pgstat_reset_counters() */ @@ -1453,6 +1517,12 @@ pgstat_delete_pending_entry(PgStat_EntryRef *entry_ref) entry_ref->flushed_generation = 0; dlist_delete(&entry_ref->pending_node); + + if (kind_info->flush_at_statement_end) + { + Assert(pgStatNumStatementPending > 0); + pgStatNumStatementPending--; + } } /* @@ -1560,6 +1630,9 @@ pgstat_prep_pending_from_entry_ref(PgStat_EntryRef *entry_ref) entry_ref->pending = MemoryContextAllocZero(pgStatPendingContext, entrysize); entry_ref->flushed_generation = 0; dlist_push_tail(&pgStatPending, &entry_ref->pending_node); + + if (pgstat_get_kind_info(kind)->flush_at_statement_end) + pgStatNumStatementPending++; } else if (pgStatFlushInProgress && entry_ref->flushed_generation == pgStatFlushGeneration) @@ -1629,10 +1702,17 @@ pgstat_flush_pending_entries(bool nowait) * Only delete a fully flushed entry at a transaction boundary. * Earlier in the transaction it may still be reused as more stats * accumulate, and code holding a pointer into it relies on the entry - * staying put. + * staying put. Kinds with flush_at_statement_end keep no such + * pointers, so delete and release their entries right away. */ if (result == PGSTAT_FLUSH_DONE && xact_boundary) pgstat_delete_pending_entry(entry_ref); + else if (result == PGSTAT_FLUSH_DONE && + pgstat_get_kind_info(entry_ref->shared_entry->key.kind)->flush_at_statement_end) + { + pgstat_delete_pending_entry(entry_ref); + pgstat_release_flushed_entry_ref(entry_ref); + } else have_pending = true; @@ -1644,6 +1724,73 @@ pgstat_flush_pending_entries(bool nowait) return have_pending; } +/* + * Flush the pending entries of kinds with flush_at_statement_end without + * waiting for locks, deleting and releasing them once flushed. Returns + * whether any remain pending. + */ +static bool +pgstat_flush_statement_entries(void) +{ + bool have_pending = false; + bool xact_boundary = !IsTransactionOrTransactionBlock(); + dlist_node *cur; + + Assert(pgStatNumStatementPending > 0); + + Assert(!pgStatFlushInProgress); + + pgStatFlushInProgress = true; + PG_TRY(); + { + /* See pgstat_flush_pending_entries() about iterating the list */ + cur = dlist_head_node(&pgStatPending); + pgStatFlushGeneration++; + + while (cur) + { + PgStat_EntryRef *entry_ref = + dlist_container(PgStat_EntryRef, pending_node, cur); + PgStat_Kind kind = entry_ref->shared_entry->key.kind; + PgStat_FlushResult result; + dlist_node *next; + + CHECK_FOR_INTERRUPTS(); + + if (!pgstat_get_kind_info(kind)->flush_at_statement_end) + { + cur = dlist_has_next(&pgStatPending, cur) ? + dlist_next_node(&pgStatPending, cur) : NULL; + continue; + } + + result = pgstat_flush_entry_ref(entry_ref, true, xact_boundary); + Assert(result != PGSTAT_FLUSH_PARTIAL); + + next = dlist_has_next(&pgStatPending, cur) ? + dlist_next_node(&pgStatPending, cur) : NULL; + + if (result == PGSTAT_FLUSH_DONE) + { + pgstat_delete_pending_entry(entry_ref); + pgstat_release_flushed_entry_ref(entry_ref); + } + else + have_pending = true; + + cur = next; + } + } + PG_FINALLY(); + { + pgStatCurrentFlushEntry = NULL; + pgStatFlushInProgress = false; + } + PG_END_TRY(); + + return have_pending; +} + /* ------------------------------------------------------------ * Helper / infrastructure functions @@ -1773,6 +1920,10 @@ pgstat_register_kind(PgStat_Kind kind, const PgStat_KindInfo *kind_info) ereport(ERROR, (errmsg("failed to register custom cumulative statistics \"%s\" with ID %u", kind_info->name, kind), errhint("Custom cumulative statistics cannot release references after their flush for fixed-numbered objects."))); + if (kind_info->flush_at_statement_end) + ereport(ERROR, + (errmsg("failed to register custom cumulative statistics \"%s\" with ID %u", kind_info->name, kind), + errhint("Custom cumulative statistics cannot be flushed at statement end for fixed-numbered objects."))); } else { @@ -1781,6 +1932,11 @@ pgstat_register_kind(PgStat_Kind kind, const PgStat_KindInfo *kind_info) (errmsg("failed to register custom cumulative statistics \"%s\" with ID %u", kind_info->name, kind), errhint("Custom cumulative statistics require a \"%s\" callback when pending size is set.", "flush_pending_cb"))); + if (kind_info->flush_at_statement_end && kind_info->flush_pending_cb == NULL) + ereport(ERROR, + (errmsg("failed to register custom cumulative statistics \"%s\" with ID %u", kind_info->name, kind), + errhint("Custom cumulative statistics require a \"%s\" callback to be flushed at statement end.", + "flush_pending_cb"))); } /* diff --git a/src/backend/utils/activity/pgstat_shmem.c b/src/backend/utils/activity/pgstat_shmem.c index ca66c66d567..6779dfdac7b 100644 --- a/src/backend/utils/activity/pgstat_shmem.c +++ b/src/backend/utils/activity/pgstat_shmem.c @@ -1069,6 +1069,23 @@ pgstat_release_flushed_entry_refs(void) pgstat_release_matching_entry_refs(false, match_flushed_entry_ref, 0); } +/* + * Release a reference whose pending statistics were just flushed, if its kind + * has release_refs_after_flush. Only for kinds with flush_at_statement_end, + * which allow this inside a transaction. + */ +void +pgstat_release_flushed_entry_ref(PgStat_EntryRef *entry_ref) +{ + PgStat_HashKey key = entry_ref->shared_entry->key; + + Assert(entry_ref->pending == NULL); + Assert(pgstat_get_kind_info(key.kind)->flush_at_statement_end); + + if (pgstat_kind_releases_refs(key.kind)) + pgstat_release_entry_ref(key, entry_ref, false); +} + /* * Remember that this backend created the entry of "entry_ref", if its kind * has release_refs_after_flush, see pgstat_release_created_entry_refs(). diff --git a/src/include/utils/pgstat_internal.h b/src/include/utils/pgstat_internal.h index 1310f0bab6c..6efc945f69f 100644 --- a/src/include/utils/pgstat_internal.h +++ b/src/include/utils/pgstat_internal.h @@ -309,6 +309,19 @@ typedef struct PgStat_KindInfo */ bool release_refs_after_flush:1; + /* + * Also flush pending statistics of this kind inside a transaction, at the + * end of each top-level statement and when the backend goes idle, at most + * once per PGSTAT_MIN_INTERVAL, and after PGSTAT_IDLE_INTERVAL if it + * stays idle. For variable-numbered stats only. + * + * Such a kind must not have its pending stats depend on the outcome of the + * transaction (flush_pending_cb never returns PGSTAT_FLUSH_PARTIAL), and + * never keep a pointer to an entry reference or its pending statistics + * across statements, as flushed entries are deleted and released. + */ + bool flush_at_statement_end:1; + /* * The size of an entry pointed to by PgStatShared_HashEntry->body. For * fixed-numbered statistics, this is the size of an entry in @@ -931,6 +944,7 @@ extern void pgstat_reset_matching_entries(bool (*do_reset) (PgStatShared_HashEnt extern void pgstat_request_entry_refs_gc(void); extern void pgstat_release_flushed_entry_refs(void); extern void pgstat_release_created_entry_refs(void); +extern void pgstat_release_flushed_entry_ref(PgStat_EntryRef *entry_ref); extern dsa_pointer pgstat_alloc_entry_body(PgStat_Kind kind); extern PgStatShared_Common *pgstat_init_entry(PgStat_Kind kind, PgStatShared_HashEntry *shhashent, -- 2.47.1