From 37de112185d8d444f06d55a09837885d7fa24090 Mon Sep 17 00:00:00 2001 From: AyoubKAZ Date: Sat, 4 Jul 2026 19:39:25 +0200 Subject: [PATCH 2/2] Add VFD cache footprint metrics to pg_stat_vfdcache Extend pg_stat_vfdcache with three additional columns: open_entries number of currently open file descriptors inside VFD cache allocated_entries total number of allocated slots in the cache cache_bytes total memory used by VFD cache, including sizeof(Vfd) for allocated slots and the heap-allocated fileName strings These are live gauges. Each backend publishes its current VFD cache footprint into backend shared stats during backend stats flush, and SQL accessors sum those backend shared stats entries to produce cluster-wide totals. The memory footprint is computed incrementally inside fd.c using a running counter, avoiding expensive iterations or averages, and accurately accounting for filename string allocations. --- doc/src/sgml/monitoring.sgml | 33 ++++++++ src/backend/catalog/system_views.sql | 3 + src/backend/storage/file/fd.c | 48 ++++++++++- src/backend/utils/activity/pgstat_backend.c | 52 ++++++++++++ src/backend/utils/adt/pgstatfuncs.c | 93 +++++++++++++++++++++ src/include/catalog/pg_proc.dat | 18 ++++ src/include/pgstat.h | 15 ++++ src/include/storage/fd.h | 3 + src/include/utils/pgstat_internal.h | 6 +- src/test/regress/expected/rules.out | 3 + 10 files changed, 272 insertions(+), 2 deletions(-) diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index 67acd370215..1d63d17a2b8 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -3802,6 +3802,39 @@ description | Waiting for a newly initialized WAL file to reach durable storage + + + open_entries bigint + + + Sum of VFD slots currently holding an open kernel file descriptor + across all active backends (equivalent to summing each backend's + nfile counter) + + + + + + allocated_entries bigint + + + Sum of allocated VFD cache entries across all active backends, + including entries whose file descriptor has been closed by the LRU + eviction policy but whose slot has not yet been freed + + + + + + cache_bytes bigint + + + Sum of memory used by VFD cache entries across all active backends, + in bytes; includes sizeof(Vfd) for each allocated + slot plus the heap-allocated file name string for each populated slot + + + max_open_fds integer diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 42a82ecb07d..712542d0fdd 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1573,6 +1573,9 @@ CREATE VIEW pg_stat_vfdcache AS SELECT pg_stat_get_vfd_hits() AS hits, pg_stat_get_vfd_misses() AS misses, + pg_stat_get_vfd_cache_open_entries() AS open_entries, + pg_stat_get_vfd_cache_allocated_entries() AS allocated_entries, + pg_stat_get_vfd_cache_bytes() AS cache_bytes, pg_stat_get_vfd_max_open_fds() AS max_open_fds, CASE WHEN pg_stat_get_vfd_hits() + pg_stat_get_vfd_misses() = 0 diff --git a/src/backend/storage/file/fd.c b/src/backend/storage/file/fd.c index 4954e639397..80f0a695011 100644 --- a/src/backend/storage/file/fd.c +++ b/src/backend/storage/file/fd.c @@ -225,6 +225,14 @@ static Size SizeVfdCache = 0; */ static int nfile = 0; +/* + * Running total of memory used by the VFD cache for this backend, + * including sizeof(Vfd) for each allocated slot plus the malloc'd + * fileName string for each populated slot. Updated incrementally in + * AllocateVfd(), PathNameOpenFilePerm(), and FreeVfd(). + */ +static uint64 VfdCacheFootprintBytes = 0; + /* * Flag to tell whether it's worth scanning VfdCache looking for temp files * to close @@ -1444,8 +1452,10 @@ AllocateVfd(void) VfdCache[0].nextFree = SizeVfdCache; /* - * Record the new size + * Record the new size and account for the newly allocated Vfd + * structs. */ + VfdCacheFootprintBytes += (newCacheSize - SizeVfdCache) * sizeof(Vfd); SizeVfdCache = newCacheSize; } @@ -1466,6 +1476,7 @@ FreeVfd(File file) if (vfdP->fileName != NULL) { + VfdCacheFootprintBytes -= strlen(vfdP->fileName) + 1; free(vfdP->fileName); vfdP->fileName = NULL; } @@ -1512,6 +1523,40 @@ FileAccess(File file) return 0; } +/* + * Return the number of VFD slots currently holding an open file descriptor. + * This is the nfile counter, which tracks FDs actually open in the kernel. + */ +uint64 +GetVfdCacheOpenEntries(void) +{ + return (uint64) nfile; +} + +/* + * Return the number of currently allocated VFD entries for this backend, + * excluding slot 0 which is used as freelist/LRU header. + */ +uint64 +GetVfdCacheAllocatedEntries(void) +{ + if (SizeVfdCache == 0) + return 0; + + return (uint64) (SizeVfdCache - 1); +} + +/* + * Return the current memory footprint of the VFD cache for this backend. + * This includes sizeof(Vfd) for each allocated slot and the malloc'd + * fileName bytes for each currently-populated slot. + */ +uint64 +GetVfdCacheBytes(void) +{ + return VfdCacheFootprintBytes; +} + /* * Called whenever a temporary file is deleted to report its size. */ @@ -1624,6 +1669,7 @@ PathNameOpenFilePerm(const char *fileName, int fileFlags, mode_t fileMode) vfdP->fd)); vfdP->fileName = fnamecopy; + VfdCacheFootprintBytes += strlen(fnamecopy) + 1; /* Saved flags are adjusted to be OK for re-opening file */ vfdP->fileFlags = fileFlags & ~(O_CREAT | O_TRUNC | O_EXCL); vfdP->fileMode = fileMode; diff --git a/src/backend/utils/activity/pgstat_backend.c b/src/backend/utils/activity/pgstat_backend.c index b736b2ccc6f..a493b559911 100644 --- a/src/backend/utils/activity/pgstat_backend.c +++ b/src/backend/utils/activity/pgstat_backend.c @@ -27,6 +27,7 @@ #include "access/xlog.h" #include "executor/instrument.h" #include "storage/bufmgr.h" +#include "storage/fd.h" #include "storage/proc.h" #include "storage/procarray.h" #include "utils/memutils.h" @@ -49,6 +50,12 @@ static bool backend_has_lockstats = false; */ static WalUsage prevBackendWalUsage; +/* + * Last per-backend VFD cache gauges published to backend stats. + * Used to detect whether anything changed since the last flush. + */ +static PgStat_BackendVfdCacheStats prevBackendVfdCacheStats; + /* * Utility routines to report I/O stats for backends, kept here to avoid * exposing PendingBackendStats to the outside world. @@ -253,6 +260,21 @@ pgstat_backend_wal_have_pending(void) return (pgWalUsage.wal_records != prevBackendWalUsage.wal_records); } +/* + * Determine whether VFD cache gauges have changed since the last flush. + */ +static inline bool +pgstat_backend_vfdcache_have_pending(void) +{ + PgStat_Counter cur_open = (PgStat_Counter) GetVfdCacheOpenEntries(); + PgStat_Counter cur_alloc = (PgStat_Counter) GetVfdCacheAllocatedEntries(); + PgStat_Counter cur_bytes = (PgStat_Counter) GetVfdCacheBytes(); + + return cur_open != prevBackendVfdCacheStats.open_entries || + cur_alloc != prevBackendVfdCacheStats.allocated_entries || + cur_bytes != prevBackendVfdCacheStats.cache_bytes; +} + /* * Flush out locally pending backend WAL statistics. Locking is managed * by the caller. @@ -326,6 +348,27 @@ pgstat_flush_backend_entry_lock(PgStat_EntryRef *entry_ref) backend_has_lockstats = false; } +/* + * Flush out locally pending backend VFD cache gauges. Locking is managed + * by the caller. No need to check for pending data here; the caller does + * that before acquiring the lock. + */ +static void +pgstat_flush_backend_entry_vfdcache(PgStat_EntryRef *entry_ref) +{ + PgStatShared_Backend *shbackendent; + PgStat_BackendVfdCacheStats *bktype_shstats; + + shbackendent = (PgStatShared_Backend *) entry_ref->shared_stats; + bktype_shstats = &shbackendent->stats.vfdcache_stats; + + bktype_shstats->open_entries = (PgStat_Counter) GetVfdCacheOpenEntries(); + bktype_shstats->allocated_entries = (PgStat_Counter) GetVfdCacheAllocatedEntries(); + bktype_shstats->cache_bytes = (PgStat_Counter) GetVfdCacheBytes(); + + prevBackendVfdCacheStats = *bktype_shstats; +} + /* * Flush out locally pending backend statistics * @@ -354,6 +397,11 @@ pgstat_flush_backend(bool nowait, uint32 flags) if ((flags & PGSTAT_BACKEND_FLUSH_LOCK) && backend_has_lockstats) has_pending_data = true; + /* Some VFD cache data pending? */ + if ((flags & PGSTAT_BACKEND_FLUSH_VFDCACHE) && + pgstat_backend_vfdcache_have_pending()) + has_pending_data = true; + if (!has_pending_data) return false; @@ -372,6 +420,9 @@ pgstat_flush_backend(bool nowait, uint32 flags) if (flags & PGSTAT_BACKEND_FLUSH_LOCK) pgstat_flush_backend_entry_lock(entry_ref); + if (flags & PGSTAT_BACKEND_FLUSH_VFDCACHE) + pgstat_flush_backend_entry_vfdcache(entry_ref); + pgstat_unlock_entry(entry_ref); return false; @@ -411,6 +462,7 @@ pgstat_create_backend(ProcNumber procnum) MemSet(&PendingBackendStats, 0, sizeof(PgStat_BackendPending)); backend_has_iostats = false; backend_has_lockstats = false; + MemSet(&prevBackendVfdCacheStats, 0, sizeof(prevBackendVfdCacheStats)); /* * Initialize prevBackendWalUsage with pgWalUsage so that diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index 714918d292c..55835645bae 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -15,6 +15,7 @@ #include "postgres.h" #include "access/htup_details.h" +#include "access/xact.h" #include "access/xlog.h" #include "access/xlogprefetcher.h" #include "catalog/catalog.h" @@ -1349,6 +1350,98 @@ pg_stat_get_vfd_misses(PG_FUNCTION_ARGS) PG_RETURN_INT64(pgstat_fetch_stat_vfdcache()->vfd_misses); } +/* + * Sum per-backend VFD cache gauges across all currently active backends. + * Results are cached for the duration of the current SQL statement to + * avoid redundant scans when the view calls multiple getter functions. + */ +static void +pgstat_get_vfd_backend_sums(PgStat_Counter *open_sum, + PgStat_Counter *alloc_sum, + PgStat_Counter *bytes_sum) +{ + static TimestampTz cached_stmt_start_ts = 0; + static PgStat_Counter cached_open_sum = 0; + static PgStat_Counter cached_alloc_sum = 0; + static PgStat_Counter cached_bytes_sum = 0; + TimestampTz stmt_start_ts = GetCurrentStatementStartTimestamp(); + int num_backends = pgstat_fetch_stat_numbackends(); + + if (cached_stmt_start_ts == stmt_start_ts) + { + *open_sum = cached_open_sum; + *alloc_sum = cached_alloc_sum; + *bytes_sum = cached_bytes_sum; + return; + } + + *open_sum = 0; + *alloc_sum = 0; + *bytes_sum = 0; + + for (int curr_backend = 1; curr_backend <= num_backends; curr_backend++) + { + LocalPgBackendStatus *local_beentry; + PgBackendStatus *beentry; + PgStat_Backend *backend_stats; + + local_beentry = pgstat_get_local_beentry_by_index(curr_backend); + beentry = &local_beentry->backendStatus; + + if (!pgstat_tracks_backend_bktype(beentry->st_backendType)) + continue; + + backend_stats = pgstat_fetch_stat_backend(local_beentry->proc_number); + if (!backend_stats) + continue; + + *open_sum += backend_stats->vfdcache_stats.open_entries; + *alloc_sum += backend_stats->vfdcache_stats.allocated_entries; + *bytes_sum += backend_stats->vfdcache_stats.cache_bytes; + } + + cached_open_sum = *open_sum; + cached_alloc_sum = *alloc_sum; + cached_bytes_sum = *bytes_sum; + cached_stmt_start_ts = stmt_start_ts; +} + +Datum +pg_stat_get_vfd_cache_open_entries(PG_FUNCTION_ARGS) +{ + PgStat_Counter open_sum; + PgStat_Counter alloc_sum; + PgStat_Counter bytes_sum; + + pgstat_get_vfd_backend_sums(&open_sum, &alloc_sum, &bytes_sum); + + PG_RETURN_INT64(open_sum); +} + +Datum +pg_stat_get_vfd_cache_allocated_entries(PG_FUNCTION_ARGS) +{ + PgStat_Counter open_sum; + PgStat_Counter alloc_sum; + PgStat_Counter bytes_sum; + + pgstat_get_vfd_backend_sums(&open_sum, &alloc_sum, &bytes_sum); + + PG_RETURN_INT64(alloc_sum); +} + +Datum +pg_stat_get_vfd_cache_bytes(PG_FUNCTION_ARGS) +{ + PgStat_Counter open_sum; + PgStat_Counter alloc_sum; + PgStat_Counter bytes_sum; + + pgstat_get_vfd_backend_sums(&open_sum, &alloc_sum, &bytes_sum); + + PG_RETURN_INT64(bytes_sum); +} + Datum pg_stat_get_vfd_max_open_fds(PG_FUNCTION_ARGS) { diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 52a9579f873..9435286a171 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -12747,5 +12747,23 @@ provolatile => 'v', proparallel => 'r', prorettype => 'int4', proargtypes => '', prosrc => 'pg_stat_get_vfd_max_open_fds' }, +{ oid => '9562', + descr => 'statistics: number of VFD slots with an open file descriptor across backends', + proname => 'pg_stat_get_vfd_cache_open_entries', + provolatile => 'v', proparallel => 'r', + prorettype => 'int8', proargtypes => '', + prosrc => 'pg_stat_get_vfd_cache_open_entries' }, +{ oid => '9563', + descr => 'statistics: total number of allocated VFD cache entries across backends', + proname => 'pg_stat_get_vfd_cache_allocated_entries', + provolatile => 'v', proparallel => 'r', + prorettype => 'int8', proargtypes => '', + prosrc => 'pg_stat_get_vfd_cache_allocated_entries' }, +{ oid => '9567', + descr => 'statistics: total memory footprint of VFD cache across backends in bytes', + proname => 'pg_stat_get_vfd_cache_bytes', + provolatile => 'v', proparallel => 'r', + prorettype => 'int8', proargtypes => '', + prosrc => 'pg_stat_get_vfd_cache_bytes' }, ] diff --git a/src/include/pgstat.h b/src/include/pgstat.h index 5791894782b..7c7a18c22bb 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -286,6 +286,20 @@ typedef struct PgStat_VfdCacheStats TimestampTz stat_reset_timestamp; } PgStat_VfdCacheStats; +/* ------- + * PgStat_BackendVfdCacheStats Per-backend VFD cache gauges + * + * These are live gauges (not cumulative counters) snapshotted from each + * backend's fd.c variables during stats flush. + * ------- + */ +typedef struct PgStat_BackendVfdCacheStats +{ + PgStat_Counter open_entries; /* FDs currently open (= nfile) */ + PgStat_Counter allocated_entries; /* total allocated VFD slots */ + PgStat_Counter cache_bytes; /* memory footprint in bytes */ +} PgStat_BackendVfdCacheStats; + /* * Types related to counting IO operations */ @@ -537,6 +551,7 @@ typedef struct PgStat_Backend PgStat_BktypeIO io_stats; PgStat_WalCounters wal_counters; PgStat_PendingLock lock_stats; + PgStat_BackendVfdCacheStats vfdcache_stats; } PgStat_Backend; /* --------- diff --git a/src/include/storage/fd.h b/src/include/storage/fd.h index 8ac466fd346..672e18ca19e 100644 --- a/src/include/storage/fd.h +++ b/src/include/storage/fd.h @@ -149,6 +149,9 @@ extern char *FilePathName(File file); extern int FileGetRawDesc(File file); extern int FileGetRawFlags(File file); extern mode_t FileGetRawMode(File file); +extern uint64 GetVfdCacheOpenEntries(void); +extern uint64 GetVfdCacheAllocatedEntries(void); +extern uint64 GetVfdCacheBytes(void); /* Operations used for sharing named temporary files */ extern File PathNameCreateTemporaryFile(const char *path, bool error_on_failure); diff --git a/src/include/utils/pgstat_internal.h b/src/include/utils/pgstat_internal.h index 790de4436d9..eb64a96af20 100644 --- a/src/include/utils/pgstat_internal.h +++ b/src/include/utils/pgstat_internal.h @@ -716,7 +716,11 @@ extern void pgstat_archiver_snapshot_cb(void); #define PGSTAT_BACKEND_FLUSH_IO (1 << 0) /* Flush I/O statistics */ #define PGSTAT_BACKEND_FLUSH_WAL (1 << 1) /* Flush WAL statistics */ #define PGSTAT_BACKEND_FLUSH_LOCK (1 << 2) /* Flush lock statistics */ -#define PGSTAT_BACKEND_FLUSH_ALL (PGSTAT_BACKEND_FLUSH_IO | PGSTAT_BACKEND_FLUSH_WAL | PGSTAT_BACKEND_FLUSH_LOCK) +#define PGSTAT_BACKEND_FLUSH_VFDCACHE (1 << 3) /* Flush VFD cache gauges */ +#define PGSTAT_BACKEND_FLUSH_ALL (PGSTAT_BACKEND_FLUSH_IO | \ + PGSTAT_BACKEND_FLUSH_WAL | \ + PGSTAT_BACKEND_FLUSH_LOCK | \ + PGSTAT_BACKEND_FLUSH_VFDCACHE) extern bool pgstat_flush_backend(bool nowait, uint32 flags); extern bool pgstat_backend_flush_cb(bool nowait); diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index d195291a27a..9c9df43bc2f 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -2425,6 +2425,9 @@ pg_stat_user_tables| SELECT relid, WHERE ((schemaname <> ALL (ARRAY['pg_catalog'::name, 'information_schema'::name])) AND (schemaname !~ '^pg_toast'::text)); pg_stat_vfdcache| SELECT pg_stat_get_vfd_hits() AS hits, pg_stat_get_vfd_misses() AS misses, + pg_stat_get_vfd_cache_open_entries() AS open_entries, + pg_stat_get_vfd_cache_allocated_entries() AS allocated_entries, + pg_stat_get_vfd_cache_bytes() AS cache_bytes, pg_stat_get_vfd_max_open_fds() AS max_open_fds, CASE WHEN ((pg_stat_get_vfd_hits() + pg_stat_get_vfd_misses()) = 0) THEN NULL::double precision -- 2.34.1