From cb99234f9d5f708eba068867aea156af12280be9 Mon Sep 17 00:00:00 2001 From: Ashutosh Bapat Date: Thu, 4 Jun 2026 15:10:56 +0530 Subject: [PATCH v20260724 2/7] Decouple GUC shared_buffers and size of the buffer pool A single variable NBuffers holds the value of the GUC 'shared_buffers' and the size of the buffer pool in number of buffers. With the introduction of resizable shared buffer pool feature, the value of 'shared_buffers' GUC and the size of the buffer pool can be different. This commit prepares for the same by decoupling the two. A new variable NBuffersGUC holds the value of the GUC 'shared_buffers'. The variable NBuffers continues to hold the size of the buffer pool. It is set to the value of NBuffersGUC during initialization. Because of this decoupling, the code which references NBuffers as the value of the GUC 'shared_buffers' is clearly differentiated from the code which references NBuffers as the size of the buffer pool. Also change comments to use term "size of the buffer pool" or "number of buffers" instead of referencing variable NBuffers. Author: Ashutosh Bapat --- src/backend/access/heap/heapam.c | 15 ++++++++------- src/backend/access/transam/slru.c | 2 +- src/backend/access/transam/xlog.c | 8 ++++---- src/backend/optimizer/path/costsize.c | 8 ++++---- src/backend/postmaster/checkpointer.c | 11 ++++++----- src/backend/storage/aio/aio_init.c | 2 +- src/backend/storage/buffer/buf_init.c | 17 +++++++++++++---- src/backend/storage/buffer/buf_table.c | 14 ++++++++------ src/backend/storage/buffer/bufmgr.c | 13 +++++++------ src/backend/storage/buffer/freelist.c | 7 ++++--- src/backend/utils/init/globals.c | 2 +- src/backend/utils/misc/guc_parameters.dat | 2 +- src/include/miscadmin.h | 2 +- src/include/storage/buf.h | 2 +- src/include/storage/buf_internals.h | 4 ++-- src/include/storage/bufmgr.h | 3 ++- 16 files changed, 64 insertions(+), 48 deletions(-) diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c index 8b488cfd8f6..8eb84bd51d6 100644 --- a/src/backend/access/heap/heapam.c +++ b/src/backend/access/heap/heapam.c @@ -383,13 +383,14 @@ initscan(HeapScanDesc scan, ScanKey key, bool keep_startblock) scan->rs_nblocks = RelationGetNumberOfBlocks(scan->rs_base.rs_rd); /* - * If the table is large relative to NBuffers, use a bulk-read access - * strategy and enable synchronized scanning (see syncscan.c). Although - * the thresholds for these features could be different, we make them the - * same so that there are only two behaviors to tune rather than four. - * (However, some callers need to be able to disable one or both of these - * behaviors, independently of the size of the table; also there is a GUC - * variable that can disable synchronized scanning.) + * If the table is large relative to the size of the buffer pool, use a + * bulk-read access strategy and enable synchronized scanning (see + * syncscan.c). Although the thresholds for these features could be + * different, we make them the same so that there are only two behaviors + * to tune rather than four. (However, some callers need to be able to + * disable one or both of these behaviors, independently of the size of + * the table; also there is a GUC variable that can disable synchronized + * scanning.) * * Note that table_block_parallelscan_initialize has a very similar test; * if you change this, consider changing that one, too. diff --git a/src/backend/access/transam/slru.c b/src/backend/access/transam/slru.c index 47dd52d6749..b47ace37974 100644 --- a/src/backend/access/transam/slru.c +++ b/src/backend/access/transam/slru.c @@ -236,7 +236,7 @@ SimpleLruAutotuneBuffers(int divisor, int max) { return Min(max - (max % SLRU_BANK_SIZE), Max(SLRU_BANK_SIZE, - NBuffers / divisor - (NBuffers / divisor) % SLRU_BANK_SIZE)); + NBuffersGUC / divisor - (NBuffersGUC / divisor) % SLRU_BANK_SIZE)); } /* diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index f8b939853e9..26e4929e167 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -5017,14 +5017,14 @@ GetFakeLSNForUnloggedRel(void) * and a minimum of 8 blocks (which was the default value prior to PostgreSQL * 9.1, when auto-tuning was added). * - * This should not be called until NBuffers has received its final value. + * This should not be called until NBuffersGUC has received its final value. */ static int XLOGChooseNumBuffers(void) { int xbuffers; - xbuffers = NBuffers / 32; + xbuffers = NBuffersGUC / 32; if (xbuffers > (wal_segment_size / XLOG_BLCKSZ)) xbuffers = (wal_segment_size / XLOG_BLCKSZ); if (xbuffers < 8) @@ -5297,8 +5297,8 @@ XLOGShmemRequest(void *arg) /* * If the value of wal_buffers is -1, use the preferred auto-tune value. * This isn't an amazingly clean place to do this, but we must wait till - * NBuffers has received its final value, and must do it before using the - * value of XLOGbuffers to do anything important. + * NBuffersGUC has received its final value, and must do it before using + * the value of XLOGbuffers to do anything important. * * We prefer to report this value's source as PGC_S_DYNAMIC_DEFAULT. * However, if the DBA explicitly set wal_buffers = -1 in the config file, diff --git a/src/backend/optimizer/path/costsize.c b/src/backend/optimizer/path/costsize.c index ac523ecf9a8..8be50631e9c 100644 --- a/src/backend/optimizer/path/costsize.c +++ b/src/backend/optimizer/path/costsize.c @@ -19,10 +19,10 @@ * is normally considerably less than random_page_cost. (However, if the * database is fully cached in RAM, it is reasonable to set them equal.) * - * We also use a rough estimate "effective_cache_size" of the number of - * disk pages in Postgres + OS-level disk cache. (We can't simply use - * NBuffers for this purpose because that would ignore the effects of - * the kernel's disk cache.) + * We also use a rough estimate "effective_cache_size" of the number of disk + * pages in Postgres + OS-level disk cache. (We can't simply use size of the + * buffer pool for this purpose because that would ignore the effects of the + * kernel's disk cache.) * * Obviously, taking constants for these values is an oversimplification, * but it's tough enough to get any useful estimates even at this level of diff --git a/src/backend/postmaster/checkpointer.c b/src/backend/postmaster/checkpointer.c index f6351f1eb10..0db3c9e1a38 100644 --- a/src/backend/postmaster/checkpointer.c +++ b/src/backend/postmaster/checkpointer.c @@ -963,12 +963,13 @@ CheckpointerShmemRequest(void *arg) Size size; /* - * The size of the requests[] array is arbitrarily set equal to NBuffers. - * But there is a cap of MAX_CHECKPOINT_REQUESTS to prevent accumulating - * too many checkpoint requests in the ring buffer. + * The size of the requests[] array is arbitrarily set equal to the + * initial size of buffer pool. But there is a cap of + * MAX_CHECKPOINT_REQUESTS to prevent accumulating too many checkpoint + * requests in the ring buffer. */ size = offsetof(CheckpointerShmemStruct, requests); - size = add_size(size, mul_size(Min(NBuffers, + size = add_size(size, mul_size(Min(NBuffersGUC, MAX_CHECKPOINT_REQUESTS), sizeof(CheckpointerRequest))); ShmemRequestStruct(.name = "Checkpointer Data", @@ -985,7 +986,7 @@ static void CheckpointerShmemInit(void *arg) { SpinLockInit(&CheckpointerShmem->ckpt_lck); - CheckpointerShmem->max_requests = Min(NBuffers, MAX_CHECKPOINT_REQUESTS); + CheckpointerShmem->max_requests = Min(NBuffersGUC, MAX_CHECKPOINT_REQUESTS); CheckpointerShmem->head = CheckpointerShmem->tail = 0; ConditionVariableInit(&CheckpointerShmem->start_cv); ConditionVariableInit(&CheckpointerShmem->done_cv); diff --git a/src/backend/storage/aio/aio_init.c b/src/backend/storage/aio/aio_init.c index de50e6a8a31..81825dd1b73 100644 --- a/src/backend/storage/aio/aio_init.c +++ b/src/backend/storage/aio/aio_init.c @@ -109,7 +109,7 @@ AioChooseMaxConcurrency(void) /* Similar logic to LimitAdditionalPins() */ max_backends = MaxBackends + NUM_AUXILIARY_PROCS; - max_proportional_pins = NBuffers / max_backends; + max_proportional_pins = NBuffersGUC / max_backends; max_proportional_pins = Max(max_proportional_pins, 1); diff --git a/src/backend/storage/buffer/buf_init.c b/src/backend/storage/buffer/buf_init.c index 1407c930c56..9ddf6551fcd 100644 --- a/src/backend/storage/buffer/buf_init.c +++ b/src/backend/storage/buffer/buf_init.c @@ -77,21 +77,21 @@ static void BufferManagerShmemRequest(void *arg) { ShmemRequestStruct(.name = "Buffer Descriptors", - .size = NBuffers * sizeof(BufferDescPadded), + .size = NBuffersGUC * sizeof(BufferDescPadded), /* Align descriptors to a cacheline boundary. */ .alignment = PG_CACHE_LINE_SIZE, .ptr = (void **) &BufferDescriptors, ); ShmemRequestStruct(.name = "Buffer Blocks", - .size = NBuffers * (Size) BLCKSZ, + .size = NBuffersGUC * (Size) BLCKSZ, /* Align buffer pool on IO page size boundary. */ .alignment = PG_IO_ALIGN_SIZE, .ptr = (void **) &BufferBlocks, ); ShmemRequestStruct(.name = "Buffer IO Condition Variables", - .size = NBuffers * sizeof(ConditionVariableMinimallyPadded), + .size = NBuffersGUC * sizeof(ConditionVariableMinimallyPadded), /* Align descriptors to a cacheline boundary. */ .alignment = PG_CACHE_LINE_SIZE, .ptr = (void **) &BufferIOCVArray, @@ -105,7 +105,7 @@ BufferManagerShmemRequest(void *arg) * painful. */ ShmemRequestStruct(.name = "Checkpoint BufferIds", - .size = NBuffers * sizeof(CkptSortItem), + .size = NBuffersGUC * sizeof(CkptSortItem), .ptr = (void **) &CkptBufferIds, ); } @@ -119,6 +119,12 @@ BufferManagerShmemRequest(void *arg) static void BufferManagerShmemInit(void *arg) { + /* + * Set the size of the buffer pool, now that it's allocated and ready to + * be initialized. + */ + NBuffers = NBuffersGUC; + /* * Initialize all the buffer headers. */ @@ -147,6 +153,9 @@ BufferManagerShmemInit(void *arg) static void BufferManagerShmemAttach(void *arg) { + /* Update the size of the buffer pool. */ + NBuffers = NBuffersGUC; + /* Initialize per-backend file flush context */ WritebackContextInit(&BackendWritebackContext, &backend_flush_after); diff --git a/src/backend/storage/buffer/buf_table.c b/src/backend/storage/buffer/buf_table.c index 347bf267d73..5c8ccbee13f 100644 --- a/src/backend/storage/buffer/buf_table.c +++ b/src/backend/storage/buffer/buf_table.c @@ -42,7 +42,7 @@ const ShmemCallbacks BufTableShmemCallbacks = { /* * Register shmem hash table for mapping buffers. - * size is the desired hash table size (possibly more than NBuffers) + * size is the desired hash table size (possibly more than the size of the buffer pool). */ void BufTableShmemRequest(void *arg) @@ -54,12 +54,14 @@ BufTableShmemRequest(void *arg) * * Since we can't tolerate running out of lookup table entries, we must be * sure to specify an adequate table size here. The maximum steady-state - * usage is of course NBuffers entries, but BufferAlloc() tries to insert - * a new entry before deleting the old. In principle this could be - * happening in each partition concurrently, so we could need as many as - * NBuffers + NUM_BUFFER_PARTITIONS entries. + * usage is of course as many entries as the number of buffers in the + * pool, but BufferAlloc() tries to insert a new entry before deleting the + * old. In principle this could be happening in each partition + * concurrently, so we could need as many as (number of buffers in the + * pool) + NUM_BUFFER_PARTITIONS entries. Since we are still requesting + * shared memory, use the GUC value instead of the actual size. */ - size = NBuffers + NUM_BUFFER_PARTITIONS; + size = NBuffersGUC + NUM_BUFFER_PARTITIONS; ShmemRequestHash(.name = "Shared Buffer Lookup Table", .nelems = size, diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c index 3eea17115b5..db7f34d2274 100644 --- a/src/backend/storage/buffer/bufmgr.c +++ b/src/backend/storage/buffer/bufmgr.c @@ -223,6 +223,7 @@ int io_max_combine_limit = DEFAULT_IO_COMBINE_LIMIT; int checkpoint_flush_after = DEFAULT_CHECKPOINT_FLUSH_AFTER; int bgwriter_flush_after = DEFAULT_BGWRITER_FLUSH_AFTER; int backend_flush_after = DEFAULT_BACKEND_FLUSH_AFTER; +int NBuffers = 0; /* number of buffers in the buffer pool */ /* local state for LockBufferForCleanup */ static BufferDesc *PinCountWaitBuf = NULL; @@ -239,11 +240,11 @@ static BufferDesc *PinCountWaitBuf = NULL; * and, if so, in what mode. * * - * To avoid - as we used to - requiring an array with NBuffers entries to keep - * track of local buffers, we use a small sequentially searched array - * (PrivateRefCountArrayKeys, with the corresponding data stored in - * PrivateRefCountArray) and an overflow hash table (PrivateRefCountHash) to - * keep track of backend local pins. + * To avoid - as we used to - requiring an array, with as many entries as the + * size of buffer pool, to keep track of local buffers, we use a small + * sequentially searched array (PrivateRefCountArrayKeys, with the corresponding + * data stored in PrivateRefCountArray) and an overflow hash table + * (PrivateRefCountHash) to keep track of backend local pins. * * Until no more than REFCOUNT_ARRAY_ENTRIES buffers are pinned at once, all * refcounts are kept track of in the array; after that, new array entries @@ -3642,7 +3643,7 @@ BufferSync(int flags) set_bits, 0, 0); - /* Check for barrier events in case NBuffers is large. */ + /* Check for barrier events in case the buffer pool is large. */ if (ProcSignalBarrierPending) ProcessProcSignalBarrier(); } diff --git a/src/backend/storage/buffer/freelist.c b/src/backend/storage/buffer/freelist.c index fdb5bad7910..4d5ee52ddc0 100644 --- a/src/backend/storage/buffer/freelist.c +++ b/src/backend/storage/buffer/freelist.c @@ -37,7 +37,8 @@ typedef struct /* * clock-sweep hand: index of next buffer to consider grabbing. Note that * this isn't a concrete buffer - we only ever increase the value. So, to - * get an actual buffer, it needs to be used modulo NBuffers. + * get an actual buffer, it needs to be used modulo size of the buffer + * pool. */ pg_atomic_uint32 nextVictimBuffer; @@ -522,10 +523,10 @@ GetAccessStrategyWithSize(BufferAccessStrategyType btype, int ring_size_kb) if (ring_buffers == 0) return NULL; - /* Cap to 1/8th of shared_buffers */ + /* Cap to 1/8th of number of buffers in the buffer pool. */ ring_buffers = Min(NBuffers / 8, ring_buffers); - /* NBuffers should never be less than 16, so this shouldn't happen */ + /* Buffer pool should always have more than 16 buffers. */ Assert(ring_buffers > 0); /* Allocate the object and initialize all elements to zeroes */ diff --git a/src/backend/utils/init/globals.c b/src/backend/utils/init/globals.c index bbd28d14d99..ccf845e87b9 100644 --- a/src/backend/utils/init/globals.c +++ b/src/backend/utils/init/globals.c @@ -141,7 +141,7 @@ int max_parallel_maintenance_workers = 2; * MaxBackends is computed by PostmasterMain after modules have had a chance to * register background workers. */ -int NBuffers = 16384; +int NBuffersGUC = 16384; int MaxConnections = 100; int max_worker_processes = 8; int max_parallel_workers = 8; diff --git a/src/backend/utils/misc/guc_parameters.dat b/src/backend/utils/misc/guc_parameters.dat index adb72361ce0..dccc4c82507 100644 --- a/src/backend/utils/misc/guc_parameters.dat +++ b/src/backend/utils/misc/guc_parameters.dat @@ -2716,7 +2716,7 @@ { name => 'shared_buffers', type => 'int', context => 'PGC_POSTMASTER', group => 'RESOURCES_MEM', short_desc => 'Sets the number of shared memory buffers used by the server.', flags => 'GUC_UNIT_BLOCKS', - variable => 'NBuffers', + variable => 'NBuffersGUC', boot_val => '16384', min => '16', max => 'INT_MAX / 2', diff --git a/src/include/miscadmin.h b/src/include/miscadmin.h index 7170a4bff98..b9449e9aced 100644 --- a/src/include/miscadmin.h +++ b/src/include/miscadmin.h @@ -175,7 +175,7 @@ extern PGDLLIMPORT bool ExitOnAnyError; extern PGDLLIMPORT char *DataDir; extern PGDLLIMPORT int data_directory_mode; -extern PGDLLIMPORT int NBuffers; +extern PGDLLIMPORT int NBuffersGUC; extern PGDLLIMPORT int MaxBackends; extern PGDLLIMPORT int MaxConnections; extern PGDLLIMPORT int max_worker_processes; diff --git a/src/include/storage/buf.h b/src/include/storage/buf.h index b21445522b1..a12d6b9082a 100644 --- a/src/include/storage/buf.h +++ b/src/include/storage/buf.h @@ -17,7 +17,7 @@ /* * Buffer identifiers. * - * Zero is invalid, positive is the index of a shared buffer (1..NBuffers), + * Zero is invalid, positive is the index of a shared buffer (1..{size of shared buffer pool}), * negative is the index of a local buffer (-1 .. -NLocBuffer). */ typedef int Buffer; diff --git a/src/include/storage/buf_internals.h b/src/include/storage/buf_internals.h index e4ff5619b79..a7606c6e92b 100644 --- a/src/include/storage/buf_internals.h +++ b/src/include/storage/buf_internals.h @@ -135,8 +135,8 @@ StaticAssertDecl(MAX_BACKENDS_BITS <= (BUF_LOCK_BITS - 2), /* * The maximum allowed value of usage_count represents a tradeoff between - * accuracy and speed of the clock-sweep buffer management algorithm. A - * large value (comparable to NBuffers) would approximate LRU semantics. + * accuracy and speed of the clock-sweep buffer management algorithm. A large + * value (comparable to the size of buffer pool) would approximate LRU semantics. * But it can take as many as BM_MAX_USAGE_COUNT+1 complete cycles of the * clock-sweep hand to find a free buffer, so in practice we don't want the * value to be very large. diff --git a/src/include/storage/bufmgr.h b/src/include/storage/bufmgr.h index 6837b35fc6d..f1f6e601f51 100644 --- a/src/include/storage/bufmgr.h +++ b/src/include/storage/bufmgr.h @@ -159,13 +159,14 @@ typedef struct ReadBuffersOperation ReadBuffersOperation; typedef struct WritebackContext WritebackContext; /* in globals.c ... this duplicates miscadmin.h */ -extern PGDLLIMPORT int NBuffers; +extern PGDLLIMPORT int NBuffersGUC; /* in bufmgr.c */ extern PGDLLIMPORT bool zero_damaged_pages; extern PGDLLIMPORT int bgwriter_lru_maxpages; extern PGDLLIMPORT double bgwriter_lru_multiplier; extern PGDLLIMPORT bool track_io_timing; +extern PGDLLIMPORT int NBuffers; #define DEFAULT_EFFECTIVE_IO_CONCURRENCY 16 #define DEFAULT_MAINTENANCE_IO_CONCURRENCY 16 -- 2.34.1