From aa677769c5d6c4d5a0d002cbe6da125da932eef3 Mon Sep 17 00:00:00 2001 From: Khoa Nguyen Date: Wed, 8 Jul 2026 17:23:49 -0700 Subject: [PATCH v3] pg_buffercache: Add pg_buffercache_relations() function This function returns an aggregation of buffer contents, grouped on a per-relfilenode basis. This is often useful to understand which tables or indexes are currently in cache, and can show cache disruptions due to query activity when sampled over time. The existing pg_buffercache() function can be utilized for this by grouping the result, but passing a large amount of buffer entries (one per page) back to the tuplestore and then aggregating it can be prohibitively expensive with large buffer counts. Even on a small shared buffers (128MB) the new function is 10x faster. Similar to the existing summary functions this new function does not hold buffer partition or buffer header locks whilst gathering its statistics. The aggregation hash is bounded by work_mem. If the number of distinct relation-fork combinations exceeds that budget, the function errors out. This addresses the concern that a large number of relations could otherwise consume unbounded memory. Author: Lukas Fittl Author: Khoa Nguyen Reviewed by: Jakub Wartak Reviewed by: Bertrand Drouvot Reviewed by: Haibo Yan Reviewed by: Masahiko Sawada Reviewed by: Paul A Jungwirth Discussion: https://www.postgresql.org/message-id/flat/CAP53Pkx0=ph0vG_M20yVAoK11yGSTZP=53-rZt36OCP4hBPaDQ@mail.gmail.com --- .../expected/pg_buffercache.out | 74 +++++++++ .../pg_buffercache--1.6--1.7.sql | 16 ++ contrib/pg_buffercache/pg_buffercache_pages.c | 153 +++++++++++++++++- contrib/pg_buffercache/sql/pg_buffercache.sql | 57 +++++++ doc/src/sgml/pgbuffercache.sgml | 141 ++++++++++++++++ src/tools/pgindent/typedefs.list | 2 + 6 files changed, 442 insertions(+), 1 deletion(-) diff --git a/contrib/pg_buffercache/expected/pg_buffercache.out b/contrib/pg_buffercache/expected/pg_buffercache.out index 886dea770f6..2c11ce5c384 100644 --- a/contrib/pg_buffercache/expected/pg_buffercache.out +++ b/contrib/pg_buffercache/expected/pg_buffercache.out @@ -33,6 +33,12 @@ SELECT count(*) > 0 FROM pg_buffercache_usage_counts() WHERE buffers >= 0; t (1 row) +SELECT count(*) > 0 FROM pg_buffercache_relations() WHERE buffers >= 0; + ?column? +---------- + t +(1 row) + -- Check that the functions / views can't be accessed by default. To avoid -- having to create a dedicated user, use the pg_database_owner pseudo-role. SET ROLE pg_database_owner; @@ -46,6 +52,8 @@ SELECT * FROM pg_buffercache_summary(); ERROR: permission denied for function pg_buffercache_summary SELECT * FROM pg_buffercache_usage_counts(); ERROR: permission denied for function pg_buffercache_usage_counts +SELECT * FROM pg_buffercache_relations(); +ERROR: permission denied for function pg_buffercache_relations RESET role; -- Check that pg_monitor is allowed to query view / function SET ROLE pg_monitor; @@ -73,6 +81,12 @@ SELECT count(*) > 0 FROM pg_buffercache_usage_counts(); t (1 row) +SELECT count(*) > 0 FROM pg_buffercache_relations(); + ?column? +---------- + t +(1 row) + RESET role; ------ ---- Test pg_buffercache_evict* and pg_buffercache_mark_dirty* functions @@ -179,4 +193,64 @@ SELECT pg_buffercache_mark_dirty_all() IS NOT NULL; t (1 row) +------ +---- Test work_mem constraint on pg_buffercache_relations +------ +-- Need at least 16MB shared_buffers for this test to run. Checking for 24MB +select buffers_used+buffers_unused > 3000 from pg_buffercache_summary(); + ?column? +---------- + t +(1 row) + +CREATE SCHEMA IF NOT EXISTS buffercache_test; +-- Create N number of single page table. +DO $$ +DECLARE i int; +BEGIN + FOR i IN 1..2000 LOOP + EXECUTE format('CREATE UNLOGGED TABLE IF NOT EXISTS buffercache_test.t%s AS SELECT 1::int AS a', i); + EXECUTE format('SELECT * FROM buffercache_test.t%s', i); + -- group the number of ddls per txn to avoid max_locks_per_transaction issue + IF i % 1000 = 0 THEN + COMMIT; + RAISE NOTICE 'committed at t%', i; + END IF; + END LOOP; + COMMIT; +END $$; +NOTICE: committed at t1000 +NOTICE: committed at t2000 +-- 2000 tables each with 80B per entry will need at least ~157kB of work_mem. Expect to fail with 64kB +set work_mem='64kB'; +SELECT count(*) FROM pg_buffercache_relations(); +ERROR: number of relations exceeds "work_mem" limit +DETAIL: Aggregating buffer cache statistics required more than 64 kB. +HINT: Increase "work_mem" and retry. +-- Plenty of headroom to pass unless there are other tables, beyond 6500, in shared_buffers. +set work_mem='512kB'; +SELECT count(*) > 2000 FROM pg_buffercache_relations(); + ?column? +---------- + t +(1 row) + +reset work_mem; +-- Drop N number of single page table. Drop schema cascade with too many tables +-- can cause max_locks_per_transaction issue; +DO $$ +DECLARE i int; +BEGIN + FOR i IN 1..2000 LOOP + EXECUTE format('DROP TABLE IF EXISTS buffercache_test.t%s', i); + IF i % 1000 = 0 THEN + COMMIT; + RAISE NOTICE 'dropped up to t%', i; + END IF; + END LOOP; + COMMIT; +END $$; +NOTICE: dropped up to t1000 +NOTICE: dropped up to t2000 +drop schema buffercache_test cascade; DROP ROLE regress_buffercache_normal; diff --git a/contrib/pg_buffercache/pg_buffercache--1.6--1.7.sql b/contrib/pg_buffercache/pg_buffercache--1.6--1.7.sql index 9a7bf66dab5..e0dea3edd6f 100644 --- a/contrib/pg_buffercache/pg_buffercache--1.6--1.7.sql +++ b/contrib/pg_buffercache/pg_buffercache--1.6--1.7.sql @@ -54,3 +54,19 @@ CREATE FUNCTION pg_buffercache_mark_dirty_all( OUT buffers_skipped int4) AS 'MODULE_PATHNAME', 'pg_buffercache_mark_dirty_all' LANGUAGE C PARALLEL SAFE VOLATILE; + +CREATE FUNCTION pg_buffercache_relations( + OUT relfilenode oid, + OUT reltablespace oid, + OUT reldatabase oid, + OUT relforknumber int2, + OUT buffers int4, + OUT buffers_dirty int4, + OUT buffers_pinned int4, + OUT usagecount_total int8) +RETURNS SETOF record +AS 'MODULE_PATHNAME', 'pg_buffercache_relations' +LANGUAGE C PARALLEL SAFE; + +REVOKE ALL ON FUNCTION pg_buffercache_relations() FROM PUBLIC; +GRANT EXECUTE ON FUNCTION pg_buffercache_relations() TO pg_monitor; diff --git a/contrib/pg_buffercache/pg_buffercache_pages.c b/contrib/pg_buffercache/pg_buffercache_pages.c index 89b86855243..726684bb144 100644 --- a/contrib/pg_buffercache/pg_buffercache_pages.c +++ b/contrib/pg_buffercache/pg_buffercache_pages.c @@ -12,6 +12,9 @@ #include "access/relation.h" #include "catalog/pg_type.h" #include "funcapi.h" +#include "miscadmin.h" +#include "common/hashfn.h" + #include "port/pg_numa.h" #include "storage/buf_internals.h" #include "storage/bufmgr.h" @@ -22,6 +25,7 @@ #define NUM_BUFFERCACHE_PAGES_ELEM 9 #define NUM_BUFFERCACHE_SUMMARY_ELEM 5 #define NUM_BUFFERCACHE_USAGE_COUNTS_ELEM 4 +#define NUM_BUFFERCACHE_RELATIONS_ELEM 8 #define NUM_BUFFERCACHE_EVICT_ELEM 2 #define NUM_BUFFERCACHE_EVICT_RELATION_ELEM 3 #define NUM_BUFFERCACHE_EVICT_ALL_ELEM 3 @@ -91,6 +95,29 @@ typedef struct BufferCacheOsPagesRec *record; } BufferCacheOsPagesContext; +/* + * Hash key for pg_buffercache_relations — groups by relation file. + */ +typedef struct +{ + RelFileLocator locator; + ForkNumber forknum; +} BufferRelStatsKey; + +/* + * Hash entry for pg_buffercache_relations — accumulates per-relation + * buffer statistics. + */ +typedef struct +{ + BufferRelStatsKey key; + uint32 status; /* for simplehash */ + int32 buffers; + int32 buffers_dirty; + int32 buffers_pinned; + int64 usagecount_total; +} BufferRelStatsEntry; + /* * Function returning data from the shared buffer cache - buffer number, @@ -107,7 +134,20 @@ PG_FUNCTION_INFO_V1(pg_buffercache_evict_all); PG_FUNCTION_INFO_V1(pg_buffercache_mark_dirty); PG_FUNCTION_INFO_V1(pg_buffercache_mark_dirty_relation); PG_FUNCTION_INFO_V1(pg_buffercache_mark_dirty_all); - +PG_FUNCTION_INFO_V1(pg_buffercache_relations); + +#define SH_PREFIX relstats +#define SH_ELEMENT_TYPE BufferRelStatsEntry +#define SH_KEY_TYPE BufferRelStatsKey +#define SH_KEY key +#define SH_HASH_KEY(tb, key) \ + hash_bytes((const unsigned char *) &(key), sizeof(BufferRelStatsKey)) +#define SH_EQUAL(tb, a, b) \ + (memcmp(&(a), &(b), sizeof(BufferRelStatsKey)) == 0) +#define SH_SCOPE static inline +#define SH_DECLARE +#define SH_DEFINE +#include "lib/simplehash.h" /* Only need to touch memory once per backend process lifetime */ static bool firstNumaTouch = true; @@ -958,3 +998,114 @@ pg_buffercache_mark_dirty_all(PG_FUNCTION_ARGS) PG_RETURN_DATUM(result); } + +/* + * pg_buffercache_relations + * + * Produces a set of rows that summarize buffer cache usage per relation-fork + * combination. This enables monitoring scripts to only get the summary stats, + * instead of accumulating in a query with the full buffer information. + */ +Datum +pg_buffercache_relations(PG_FUNCTION_ARGS) +{ + ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; + relstats_hash *relstats; + relstats_iterator iter; + BufferRelStatsEntry *entry; + Datum values[NUM_BUFFERCACHE_RELATIONS_ELEM]; + bool nulls[NUM_BUFFERCACHE_RELATIONS_ELEM] = {0}; + Size entry_sz_est; + Size max_num_entries_est; + Size work_mem_bytes = (Size) work_mem * 1024L; + + InitMaterializedSRF(fcinfo, 0); + + /* Create a hash table to aggregate stats by relation-fork */ + relstats = relstats_create(CurrentMemoryContext, 128, NULL); + + /* + * Hashtable created with initial 128 entries, we can use that same 128 to + * estimate the size. Also using 128 element to estimate help us amortize + * the cost of the table header. + */ + entry_sz_est = relstats_estimate_space(128) / 128; + max_num_entries_est = work_mem_bytes / (entry_sz_est); + + /* Single pass over all buffers */ + for (int i = 0; i < NBuffers; i++) + { + BufferDesc *bufHdr; + uint64 buf_state; + BufferRelStatsKey key = {0}; + bool found; + + CHECK_FOR_INTERRUPTS(); + + /* + * Read buffer state without locking, same as pg_buffercache_summary + * and pg_buffercache_usage_counts. Locking wouldn't provide a + * meaningfully more consistent result since buffers can change state + * immediately after we release the lock. + */ + bufHdr = GetBufferDescriptor(i); + buf_state = pg_atomic_read_u64(&bufHdr->state); + + /* Skip unused/invalid buffers */ + if (!(buf_state & BM_VALID)) + continue; + + key.locator = BufTagGetRelFileLocator(&bufHdr->tag); + key.forknum = BufTagGetForkNum(&bufHdr->tag); + + entry = relstats_insert(relstats, key, &found); + + if (!found) + { + if (relstats->members >= max_num_entries_est) + { + ereport(ERROR, + (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("number of relations exceeds \"work_mem\" limit"), + errdetail("Aggregating buffer cache statistics required more than %d kB.", work_mem), + errhint("Increase \"work_mem\" and retry."))); + } + entry->buffers = 0; + entry->buffers_dirty = 0; + entry->buffers_pinned = 0; + entry->usagecount_total = 0; + } + + entry->buffers++; + entry->usagecount_total += BUF_STATE_GET_USAGECOUNT(buf_state); + + if (buf_state & BM_DIRTY) + entry->buffers_dirty++; + + if (BUF_STATE_GET_REFCOUNT(buf_state) > 0) + entry->buffers_pinned++; + } + + /* Emit one row per hash entry */ + relstats_start_iterate(relstats, &iter); + while ((entry = relstats_iterate(relstats, &iter)) != NULL) + { + CHECK_FOR_INTERRUPTS(); + + values[0] = ObjectIdGetDatum(entry->key.locator.relNumber); + values[1] = ObjectIdGetDatum(entry->key.locator.spcOid); + values[2] = ObjectIdGetDatum(entry->key.locator.dbOid); + values[3] = Int16GetDatum(entry->key.forknum); + values[4] = Int32GetDatum(entry->buffers); + values[5] = Int32GetDatum(entry->buffers_dirty); + values[6] = Int32GetDatum(entry->buffers_pinned); + values[7] = Int64GetDatum(entry->usagecount_total); + + tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, + values, nulls); + } + + relstats_destroy(relstats); + + return (Datum) 0; +} diff --git a/contrib/pg_buffercache/sql/pg_buffercache.sql b/contrib/pg_buffercache/sql/pg_buffercache.sql index 127d604905c..54bf5552160 100644 --- a/contrib/pg_buffercache/sql/pg_buffercache.sql +++ b/contrib/pg_buffercache/sql/pg_buffercache.sql @@ -18,6 +18,8 @@ from pg_buffercache_summary(); SELECT count(*) > 0 FROM pg_buffercache_usage_counts() WHERE buffers >= 0; +SELECT count(*) > 0 FROM pg_buffercache_relations() WHERE buffers >= 0; + -- Check that the functions / views can't be accessed by default. To avoid -- having to create a dedicated user, use the pg_database_owner pseudo-role. SET ROLE pg_database_owner; @@ -26,6 +28,7 @@ SELECT * FROM pg_buffercache_os_pages; SELECT * FROM pg_buffercache_pages() AS p (wrong int); SELECT * FROM pg_buffercache_summary(); SELECT * FROM pg_buffercache_usage_counts(); +SELECT * FROM pg_buffercache_relations(); RESET role; -- Check that pg_monitor is allowed to query view / function @@ -34,6 +37,7 @@ SELECT count(*) > 0 FROM pg_buffercache; SELECT count(*) > 0 FROM pg_buffercache_os_pages; SELECT buffers_used + buffers_unused > 0 FROM pg_buffercache_summary(); SELECT count(*) > 0 FROM pg_buffercache_usage_counts(); +SELECT count(*) > 0 FROM pg_buffercache_relations(); RESET role; @@ -86,4 +90,57 @@ DROP TABLE shared_pg_buffercache; SELECT pg_buffercache_mark_dirty(1) IS NOT NULL; SELECT pg_buffercache_mark_dirty_all() IS NOT NULL; + +------ +---- Test work_mem constraint on pg_buffercache_relations +------ + +-- Need at least 16MB shared_buffers for this test to run. Checking for 24MB +select buffers_used+buffers_unused > 3000 from pg_buffercache_summary(); + +CREATE SCHEMA IF NOT EXISTS buffercache_test; + +-- Create N number of single page table. +DO $$ +DECLARE i int; +BEGIN + FOR i IN 1..2000 LOOP + EXECUTE format('CREATE UNLOGGED TABLE IF NOT EXISTS buffercache_test.t%s AS SELECT 1::int AS a', i); + EXECUTE format('SELECT * FROM buffercache_test.t%s', i); + -- group the number of ddls per txn to avoid max_locks_per_transaction issue + IF i % 1000 = 0 THEN + COMMIT; + RAISE NOTICE 'committed at t%', i; + END IF; + END LOOP; + COMMIT; +END $$; + +-- 2000 tables each with 80B per entry will need at least ~157kB of work_mem. Expect to fail with 64kB +set work_mem='64kB'; +SELECT count(*) FROM pg_buffercache_relations(); + +-- Plenty of headroom to pass unless there are other tables, beyond 6500, in shared_buffers. +set work_mem='512kB'; +SELECT count(*) > 2000 FROM pg_buffercache_relations(); + +reset work_mem; + +-- Drop N number of single page table. Drop schema cascade with too many tables +-- can cause max_locks_per_transaction issue; +DO $$ +DECLARE i int; +BEGIN + FOR i IN 1..2000 LOOP + EXECUTE format('DROP TABLE IF EXISTS buffercache_test.t%s', i); + IF i % 1000 = 0 THEN + COMMIT; + RAISE NOTICE 'dropped up to t%', i; + END IF; + END LOOP; + COMMIT; +END $$; + +drop schema buffercache_test cascade; + DROP ROLE regress_buffercache_normal; diff --git a/doc/src/sgml/pgbuffercache.sgml b/doc/src/sgml/pgbuffercache.sgml index 1e9aee10275..c8f65da3cf9 100644 --- a/doc/src/sgml/pgbuffercache.sgml +++ b/doc/src/sgml/pgbuffercache.sgml @@ -31,6 +31,10 @@ pg_buffercache_usage_counts + + pg_buffercache_relations + + pg_buffercache_evict @@ -63,6 +67,7 @@ pg_buffercache_numa views), the pg_buffercache_summary() function, the pg_buffercache_usage_counts() function, the + pg_buffercache_relations() function, the pg_buffercache_evict() function, the pg_buffercache_evict_relation() function, the pg_buffercache_evict_all() function, the @@ -102,6 +107,12 @@ count. + + The pg_buffercache_relations() function returns a + set of rows summarizing buffer cache usage aggregated by relation and fork + number. + + By default, use of the above functions is restricted to superusers and roles with privileges of the pg_monitor role. Access may be @@ -564,6 +575,136 @@ + + The <function>pg_buffercache_relations()</function> Function + + + The definitions of the columns exposed by the function are shown in + . + + + + <function>pg_buffercache_relations()</function> Output Columns + + + + + Column Type + + + Description + + + + + + + + relfilenode oid + (references pg_class.relfilenode) + + + Filenode number of the relation + + + + + + reltablespace oid + (references pg_tablespace.oid) + + + Tablespace OID of the relation + + + + + + reldatabase oid + (references pg_database.oid) + + + Database OID of the relation + + + + + + relforknumber smallint + + + Fork number within the relation; see + common/relpath.h + + + + + + buffers int4 + + + Number of buffers for the relation + + + + + + buffers_dirty int4 + + + Number of dirty buffers for the relation + + + + + + buffers_pinned int4 + + + Number of pinned buffers for the relation + + + + + + usagecount_total int8 + + + Total usage count of the relation's buffers + + + + +
+ + + The pg_buffercache_relations() function returns a + set of rows summarizing the state of all shared buffers, aggregated by + relation and fork number. Similar and more detailed information is + provided by the pg_buffercache view, but + pg_buffercache_relations() is significantly + cheaper. + + + + Like the pg_buffercache view, + pg_buffercache_relations() does not acquire buffer + manager locks. Therefore concurrent activity can lead to minor inaccuracies + in the result. + + + + pg_buffercache_relations() aggregates its result in + an in-memory hash table. To bound memory usage, the function estimates + the space required and raises an error if it would exceed + . This can only happen when the buffer + cache holds a very large number of distinct relation and fork + combinations. If it occurs, increase work_mem for the + session and retry. + + + + The <function>pg_buffercache_evict()</function> Function diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 77e3c04144e..19a87f702ee 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -357,6 +357,8 @@ BufferHeapTupleTableSlot BufferLockMode BufferLookupEnt BufferManagerRelation +BufferRelStatsEntry +BufferRelStatsKey BufferStrategyControl BufferTag BufferUsage -- 2.50.1 (Apple Git-155)