From 64ffd5ba130ccbdb354527719e75768335bb3b2e Mon Sep 17 00:00:00 2001 From: AyoubKAZ Date: Wed, 9 Sep 2026 03:54:52 +0200 Subject: [PATCH v7 3/3] Add VFD cache footprint metrics to pg_stat_vfdcache Extend pg_stat_vfdcache with three additional columns: open_entries number of currently open file descriptors inside VFD cache allocated_entries total number of allocated slots in the cache cache_bytes total memory allocated by the VFD cache memory context These are live gauges. Each backend publishes its current VFD cache footprint into backend shared stats during backend stats flush, and SQL accessors sum those backend shared stats entries to produce cluster-wide totals. The memory footprint is measured directly from the dedicated VFD memory context using MemoryContextMemAllocated(), avoiding manual bookkeeping. --- doc/src/sgml/monitoring.sgml | 32 +++++++ src/backend/catalog/system_views.sql | 3 + src/backend/storage/file/fd.c | 36 ++++++++ src/backend/utils/activity/pgstat_backend.c | 52 ++++++++++++ src/backend/utils/adt/pgstatfuncs.c | 93 +++++++++++++++++++++ src/include/catalog/pg_proc.dat | 18 ++++ src/include/pgstat.h | 15 ++++ src/include/storage/fd.h | 3 + src/include/utils/pgstat_internal.h | 6 +- src/test/regress/expected/rules.out | 3 + src/test/regress/expected/stats.out | 10 +++ src/test/regress/sql/stats.sql | 6 ++ 12 files changed, 276 insertions(+), 1 deletion(-) diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index f7c41c2d960..a2a618bd945 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -3963,6 +3963,38 @@ description | Waiting for a newly initialized WAL file to reach durable storage + + + open_entries bigint + + + Sum of VFD slots currently holding an open kernel file descriptor + across all active backends (equivalent to summing each backend's + nfile counter) + + + + + + allocated_entries bigint + + + Sum of allocated VFD cache entries across all active backends, + including entries whose file descriptor has been closed by the LRU + eviction policy but whose slot has not yet been freed + + + + + + cache_bytes bigint + + + Sum of memory allocated by the VFD cache memory context across all active backends, + in bytes + + + max_open_fds integer diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 844fea1391f..2b5a92b1e70 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1573,6 +1573,9 @@ CREATE VIEW pg_stat_vfdcache AS SELECT pg_stat_get_vfd_hits() AS hits, pg_stat_get_vfd_misses() AS misses, + pg_stat_get_vfd_cache_open_entries() AS open_entries, + pg_stat_get_vfd_cache_allocated_entries() AS allocated_entries, + pg_stat_get_vfd_cache_bytes() AS cache_bytes, pg_stat_get_vfd_max_open_fds() AS max_open_fds, CASE WHEN pg_stat_get_vfd_hits() + pg_stat_get_vfd_misses() = 0 diff --git a/src/backend/storage/file/fd.c b/src/backend/storage/file/fd.c index d7ecaf5e6c5..e05197095cf 100644 --- a/src/backend/storage/file/fd.c +++ b/src/backend/storage/file/fd.c @@ -1511,6 +1511,42 @@ FileAccess(File file) return 0; } +/* + * Return the number of VFD slots currently holding an open file descriptor. + * This is the nfile counter, which tracks FDs actually open in the kernel. + */ +uint64 +GetVfdCacheOpenEntries(void) +{ + return (uint64) nfile; +} + +/* + * Return the number of currently allocated VFD entries for this backend, + * excluding slot 0 which is used as freelist/LRU header. + */ +uint64 +GetVfdCacheAllocatedEntries(void) +{ + if (SizeVfdCache == 0) + return 0; + + return (uint64) (SizeVfdCache - 1); +} + +/* + * Return the current memory footprint of the VFD cache for this backend, + * measured as the total memory allocated by its dedicated memory context. + */ +uint64 +GetVfdCacheBytes(void) +{ + if (VfdCxt == NULL) + return 0; + + return (uint64) MemoryContextMemAllocated(VfdCxt, false); +} + /* * Called whenever a temporary file is deleted to report its size. */ diff --git a/src/backend/utils/activity/pgstat_backend.c b/src/backend/utils/activity/pgstat_backend.c index b736b2ccc6f..a493b559911 100644 --- a/src/backend/utils/activity/pgstat_backend.c +++ b/src/backend/utils/activity/pgstat_backend.c @@ -27,6 +27,7 @@ #include "access/xlog.h" #include "executor/instrument.h" #include "storage/bufmgr.h" +#include "storage/fd.h" #include "storage/proc.h" #include "storage/procarray.h" #include "utils/memutils.h" @@ -49,6 +50,12 @@ static bool backend_has_lockstats = false; */ static WalUsage prevBackendWalUsage; +/* + * Last per-backend VFD cache gauges published to backend stats. + * Used to detect whether anything changed since the last flush. + */ +static PgStat_BackendVfdCacheStats prevBackendVfdCacheStats; + /* * Utility routines to report I/O stats for backends, kept here to avoid * exposing PendingBackendStats to the outside world. @@ -253,6 +260,21 @@ pgstat_backend_wal_have_pending(void) return (pgWalUsage.wal_records != prevBackendWalUsage.wal_records); } +/* + * Determine whether VFD cache gauges have changed since the last flush. + */ +static inline bool +pgstat_backend_vfdcache_have_pending(void) +{ + PgStat_Counter cur_open = (PgStat_Counter) GetVfdCacheOpenEntries(); + PgStat_Counter cur_alloc = (PgStat_Counter) GetVfdCacheAllocatedEntries(); + PgStat_Counter cur_bytes = (PgStat_Counter) GetVfdCacheBytes(); + + return cur_open != prevBackendVfdCacheStats.open_entries || + cur_alloc != prevBackendVfdCacheStats.allocated_entries || + cur_bytes != prevBackendVfdCacheStats.cache_bytes; +} + /* * Flush out locally pending backend WAL statistics. Locking is managed * by the caller. @@ -326,6 +348,27 @@ pgstat_flush_backend_entry_lock(PgStat_EntryRef *entry_ref) backend_has_lockstats = false; } +/* + * Flush out locally pending backend VFD cache gauges. Locking is managed + * by the caller. No need to check for pending data here; the caller does + * that before acquiring the lock. + */ +static void +pgstat_flush_backend_entry_vfdcache(PgStat_EntryRef *entry_ref) +{ + PgStatShared_Backend *shbackendent; + PgStat_BackendVfdCacheStats *bktype_shstats; + + shbackendent = (PgStatShared_Backend *) entry_ref->shared_stats; + bktype_shstats = &shbackendent->stats.vfdcache_stats; + + bktype_shstats->open_entries = (PgStat_Counter) GetVfdCacheOpenEntries(); + bktype_shstats->allocated_entries = (PgStat_Counter) GetVfdCacheAllocatedEntries(); + bktype_shstats->cache_bytes = (PgStat_Counter) GetVfdCacheBytes(); + + prevBackendVfdCacheStats = *bktype_shstats; +} + /* * Flush out locally pending backend statistics * @@ -354,6 +397,11 @@ pgstat_flush_backend(bool nowait, uint32 flags) if ((flags & PGSTAT_BACKEND_FLUSH_LOCK) && backend_has_lockstats) has_pending_data = true; + /* Some VFD cache data pending? */ + if ((flags & PGSTAT_BACKEND_FLUSH_VFDCACHE) && + pgstat_backend_vfdcache_have_pending()) + has_pending_data = true; + if (!has_pending_data) return false; @@ -372,6 +420,9 @@ pgstat_flush_backend(bool nowait, uint32 flags) if (flags & PGSTAT_BACKEND_FLUSH_LOCK) pgstat_flush_backend_entry_lock(entry_ref); + if (flags & PGSTAT_BACKEND_FLUSH_VFDCACHE) + pgstat_flush_backend_entry_vfdcache(entry_ref); + pgstat_unlock_entry(entry_ref); return false; @@ -411,6 +462,7 @@ pgstat_create_backend(ProcNumber procnum) MemSet(&PendingBackendStats, 0, sizeof(PgStat_BackendPending)); backend_has_iostats = false; backend_has_lockstats = false; + MemSet(&prevBackendVfdCacheStats, 0, sizeof(prevBackendVfdCacheStats)); /* * Initialize prevBackendWalUsage with pgWalUsage so that diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index 3499dd40700..aa6b16f38c4 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -15,6 +15,7 @@ #include "postgres.h" #include "access/htup_details.h" +#include "access/xact.h" #include "access/xlog.h" #include "access/xlogprefetcher.h" #include "catalog/catalog.h" @@ -1411,6 +1412,98 @@ pg_stat_get_vfd_misses(PG_FUNCTION_ARGS) PG_RETURN_INT64(pgstat_fetch_stat_vfdcache()->vfd_misses); } +/* + * Sum per-backend VFD cache gauges across all currently active backends. + * Results are cached for the duration of the current SQL statement to + * avoid redundant scans when the view calls multiple getter functions. + */ +static void +pgstat_get_vfd_backend_sums(PgStat_Counter *open_sum, + PgStat_Counter *alloc_sum, + PgStat_Counter *bytes_sum) +{ + static TimestampTz cached_stmt_start_ts = 0; + static PgStat_Counter cached_open_sum = 0; + static PgStat_Counter cached_alloc_sum = 0; + static PgStat_Counter cached_bytes_sum = 0; + TimestampTz stmt_start_ts = GetCurrentStatementStartTimestamp(); + int num_backends = pgstat_fetch_stat_numbackends(); + + if (cached_stmt_start_ts == stmt_start_ts) + { + *open_sum = cached_open_sum; + *alloc_sum = cached_alloc_sum; + *bytes_sum = cached_bytes_sum; + return; + } + + *open_sum = 0; + *alloc_sum = 0; + *bytes_sum = 0; + + for (int curr_backend = 1; curr_backend <= num_backends; curr_backend++) + { + LocalPgBackendStatus *local_beentry; + PgBackendStatus *beentry; + PgStat_Backend *backend_stats; + + local_beentry = pgstat_get_local_beentry_by_index(curr_backend); + beentry = &local_beentry->backendStatus; + + if (!pgstat_tracks_backend_bktype(beentry->st_backendType)) + continue; + + backend_stats = pgstat_fetch_stat_backend(local_beentry->proc_number); + if (!backend_stats) + continue; + + *open_sum += backend_stats->vfdcache_stats.open_entries; + *alloc_sum += backend_stats->vfdcache_stats.allocated_entries; + *bytes_sum += backend_stats->vfdcache_stats.cache_bytes; + } + + cached_open_sum = *open_sum; + cached_alloc_sum = *alloc_sum; + cached_bytes_sum = *bytes_sum; + cached_stmt_start_ts = stmt_start_ts; +} + +Datum +pg_stat_get_vfd_cache_open_entries(PG_FUNCTION_ARGS) +{ + PgStat_Counter open_sum; + PgStat_Counter alloc_sum; + PgStat_Counter bytes_sum; + + pgstat_get_vfd_backend_sums(&open_sum, &alloc_sum, &bytes_sum); + + PG_RETURN_INT64(open_sum); +} + +Datum +pg_stat_get_vfd_cache_allocated_entries(PG_FUNCTION_ARGS) +{ + PgStat_Counter open_sum; + PgStat_Counter alloc_sum; + PgStat_Counter bytes_sum; + + pgstat_get_vfd_backend_sums(&open_sum, &alloc_sum, &bytes_sum); + + PG_RETURN_INT64(alloc_sum); +} + +Datum +pg_stat_get_vfd_cache_bytes(PG_FUNCTION_ARGS) +{ + PgStat_Counter open_sum; + PgStat_Counter alloc_sum; + PgStat_Counter bytes_sum; + + pgstat_get_vfd_backend_sums(&open_sum, &alloc_sum, &bytes_sum); + + PG_RETURN_INT64(bytes_sum); +} + Datum pg_stat_get_vfd_max_open_fds(PG_FUNCTION_ARGS) { diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index d5b5f08bf5c..f7a4ef14d7a 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -12802,5 +12802,23 @@ provolatile => 'v', proparallel => 'r', prorettype => 'int4', proargtypes => '', prosrc => 'pg_stat_get_vfd_max_open_fds' }, +{ oid => '9562', + descr => 'statistics: number of VFD slots with an open file descriptor across backends', + proname => 'pg_stat_get_vfd_cache_open_entries', + provolatile => 'v', proparallel => 'r', + prorettype => 'int8', proargtypes => '', + prosrc => 'pg_stat_get_vfd_cache_open_entries' }, +{ oid => '9563', + descr => 'statistics: total number of allocated VFD cache entries across backends', + proname => 'pg_stat_get_vfd_cache_allocated_entries', + provolatile => 'v', proparallel => 'r', + prorettype => 'int8', proargtypes => '', + prosrc => 'pg_stat_get_vfd_cache_allocated_entries' }, +{ oid => '9567', + descr => 'statistics: total memory footprint of VFD cache across backends in bytes', + proname => 'pg_stat_get_vfd_cache_bytes', + provolatile => 'v', proparallel => 'r', + prorettype => 'int8', proargtypes => '', + prosrc => 'pg_stat_get_vfd_cache_bytes' }, ] diff --git a/src/include/pgstat.h b/src/include/pgstat.h index e8395664df2..73cd735de06 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -319,6 +319,20 @@ typedef struct PgStat_VfdCacheStats TimestampTz stat_reset_timestamp; } PgStat_VfdCacheStats; +/* ------- + * PgStat_BackendVfdCacheStats Per-backend VFD cache gauges + * + * These are live gauges (not cumulative counters) snapshotted from each + * backend's fd.c variables during stats flush. + * ------- + */ +typedef struct PgStat_BackendVfdCacheStats +{ + PgStat_Counter open_entries; /* FDs currently open (= nfile) */ + PgStat_Counter allocated_entries; /* total allocated VFD slots */ + PgStat_Counter cache_bytes; /* memory footprint in bytes */ +} PgStat_BackendVfdCacheStats; + /* * Types related to counting IO operations */ @@ -584,6 +598,7 @@ typedef struct PgStat_Backend PgStat_BktypeIO io_stats; PgStat_WalCounters wal_counters; PgStat_PendingLock lock_stats; + PgStat_BackendVfdCacheStats vfdcache_stats; } PgStat_Backend; /* --------- diff --git a/src/include/storage/fd.h b/src/include/storage/fd.h index 8ac466fd346..672e18ca19e 100644 --- a/src/include/storage/fd.h +++ b/src/include/storage/fd.h @@ -149,6 +149,9 @@ extern char *FilePathName(File file); extern int FileGetRawDesc(File file); extern int FileGetRawFlags(File file); extern mode_t FileGetRawMode(File file); +extern uint64 GetVfdCacheOpenEntries(void); +extern uint64 GetVfdCacheAllocatedEntries(void); +extern uint64 GetVfdCacheBytes(void); /* Operations used for sharing named temporary files */ extern File PathNameCreateTemporaryFile(const char *path, bool error_on_failure); diff --git a/src/include/utils/pgstat_internal.h b/src/include/utils/pgstat_internal.h index e24a1c84c9f..8b08990cc1b 100644 --- a/src/include/utils/pgstat_internal.h +++ b/src/include/utils/pgstat_internal.h @@ -724,7 +724,11 @@ extern void pgstat_archiver_snapshot_cb(void); #define PGSTAT_BACKEND_FLUSH_IO (1 << 0) /* Flush I/O statistics */ #define PGSTAT_BACKEND_FLUSH_WAL (1 << 1) /* Flush WAL statistics */ #define PGSTAT_BACKEND_FLUSH_LOCK (1 << 2) /* Flush lock statistics */ -#define PGSTAT_BACKEND_FLUSH_ALL (PGSTAT_BACKEND_FLUSH_IO | PGSTAT_BACKEND_FLUSH_WAL | PGSTAT_BACKEND_FLUSH_LOCK) +#define PGSTAT_BACKEND_FLUSH_VFDCACHE (1 << 3) /* Flush VFD cache gauges */ +#define PGSTAT_BACKEND_FLUSH_ALL (PGSTAT_BACKEND_FLUSH_IO | \ + PGSTAT_BACKEND_FLUSH_WAL | \ + PGSTAT_BACKEND_FLUSH_LOCK | \ + PGSTAT_BACKEND_FLUSH_VFDCACHE) extern bool pgstat_flush_backend(bool nowait, uint32 flags); extern bool pgstat_backend_flush_cb(bool nowait); diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index cdf21d82924..3d38a1bf6b8 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -2428,6 +2428,9 @@ pg_stat_user_tables| SELECT relid, WHERE ((schemaname <> ALL (ARRAY['pg_catalog'::name, 'information_schema'::name])) AND (schemaname !~ '^pg_toast'::text)); pg_stat_vfdcache| SELECT pg_stat_get_vfd_hits() AS hits, pg_stat_get_vfd_misses() AS misses, + pg_stat_get_vfd_cache_open_entries() AS open_entries, + pg_stat_get_vfd_cache_allocated_entries() AS allocated_entries, + pg_stat_get_vfd_cache_bytes() AS cache_bytes, pg_stat_get_vfd_max_open_fds() AS max_open_fds, CASE WHEN ((pg_stat_get_vfd_hits() + pg_stat_get_vfd_misses()) = 0) THEN NULL::double precision diff --git a/src/test/regress/expected/stats.out b/src/test/regress/expected/stats.out index 64f9cbc6e0c..21deb48c0b0 100644 --- a/src/test/regress/expected/stats.out +++ b/src/test/regress/expected/stats.out @@ -1295,6 +1295,16 @@ SELECT (hits + misses) > :vfd_accesses_before FROM pg_stat_vfdcache; t (1 row) +-- Test that VFD cache footprint metrics are non-negative and consistent +SELECT open_entries <= allocated_entries, + allocated_entries > 0, + cache_bytes > 0 +FROM pg_stat_vfdcache; + ?column? | ?column? | ?column? +----------+----------+---------- + t | t | t +(1 row) + -- Test error case for reset_shared with unknown stats type SELECT pg_stat_reset_shared('unknown'); ERROR: unrecognized reset target: "unknown" diff --git a/src/test/regress/sql/stats.sql b/src/test/regress/sql/stats.sql index f460b2125b3..2f745ef78bb 100644 --- a/src/test/regress/sql/stats.sql +++ b/src/test/regress/sql/stats.sql @@ -594,6 +594,12 @@ DROP TABLE test_vfd_activity; SELECT pg_stat_force_next_flush(); SELECT (hits + misses) > :vfd_accesses_before FROM pg_stat_vfdcache; +-- Test that VFD cache footprint metrics are non-negative and consistent +SELECT open_entries <= allocated_entries, + allocated_entries > 0, + cache_bytes > 0 +FROM pg_stat_vfdcache; + -- Test error case for reset_shared with unknown stats type SELECT pg_stat_reset_shared('unknown'); -- 2.34.1