From b2c15f6a20db0fcced93f9c665f26b4697794d1c Mon Sep 17 00:00:00 2001 From: Greg Burd Date: Mon, 6 Jul 2026 19:22:45 -0400 Subject: [PATCH v20260922a 9/9] Replace the usage_count clock sweep with a cooling-stage evictor Replace the 0..5 usage_count buffer-replacement policy with a cooling-stage clock (the LeanStore / 2Q-A1 model): a buffer is either HOT (recently used) or COOL (an eviction candidate), with "pinned" being the existing refcount. There is no per-buffer access counter. - A demand-loaded page is admitted COOL (probationary), not HOT. A second access via PinBuffer promotes it COOL -> HOT (the rescue). A page touched once -- a sequential scan -- therefore fills and drains the COOL stage and is evicted from it without displacing the HOT working set. - The foreground sweep in StrategyGetBuffer() reclaims an already-COOL, unpinned buffer, pinning it with a CAS so a racing PinBuffer always wins. When it passes a HOT buffer it demotes it to COOL and keeps scanning, so a HOT buffer survives the visit that cools it and is only reclaimed if it is still COOL when the hand comes around again. That gives every buffer one full sweep of grace in which a new access can promote it back to HOT. - A strategy (ring) access deliberately does not promote, which is the cooling-state form of the existing rule that ring buffers must not evict others from the pool: a buffer the ring keeps recycling stays COOL and so stays reusable by GetBufferFromRing(), while an access from outside the ring promotes it to HOT and thereby removes it from the ring's reuse set. GetBufferFromRing() tests that state directly, replacing the stock "usage_count > 1 means someone else touched it" test. The usage_count field is reinterpreted in place as the one-bit cooling state. The 64-bit buffer-state layout -- refcount, flag and lock offsets and their StaticAsserts -- is unchanged; only the meaning of the field and the instructions that touch it change. BM_MAX_USAGE_COUNT becomes BUF_COOLSTATE_HOT (1), so the pin fast path saturates at HOT. Local (temp-table) buffers get the same two-state treatment. contrib/pg_buffercache reports the cooling state in its usagecount column (0 = COOL, 1 = HOT), and pg_buffercache_summary and pg_buffercache_usage_counts() follow, so the usage-count histogram now has two populated buckets instead of six. This is a replacement-policy change only. It does not alter the background writer's pacing or write limits, and it keeps BufferAccessStrategy rings. --- contrib/pg_buffercache/pg_buffercache_pages.c | 6 +-- src/backend/storage/buffer/bufmgr.c | 44 +++++++++------- src/backend/storage/buffer/freelist.c | 38 ++++++++------ src/backend/storage/buffer/localbuf.c | 13 ++--- src/include/storage/buf_internals.h | 51 ++++++++++++++++--- 5 files changed, 100 insertions(+), 52 deletions(-) diff --git a/contrib/pg_buffercache/pg_buffercache_pages.c b/contrib/pg_buffercache/pg_buffercache_pages.c index e12542da..2370a449 100644 --- a/contrib/pg_buffercache/pg_buffercache_pages.c +++ b/contrib/pg_buffercache/pg_buffercache_pages.c @@ -166,7 +166,7 @@ pg_buffercache_pages(PG_FUNCTION_ARGS) reldatabase = bufHdr->tag.dbOid; forknum = BufTagGetForkNum(&bufHdr->tag); blocknum = bufHdr->tag.blockNum; - usagecount = BUF_STATE_GET_USAGECOUNT(buf_state); + usagecount = BUF_STATE_GET_COOLSTATE(buf_state); pinning_backends = BUF_STATE_GET_REFCOUNT(buf_state); if (buf_state & BM_DIRTY) @@ -610,7 +610,7 @@ pg_buffercache_summary(PG_FUNCTION_ARGS) if (buf_state & BM_VALID) { buffers_used++; - usagecount_total += BUF_STATE_GET_USAGECOUNT(buf_state); + usagecount_total += BUF_STATE_GET_COOLSTATE(buf_state); if (buf_state & BM_DIRTY) buffers_dirty++; @@ -660,7 +660,7 @@ pg_buffercache_usage_counts(PG_FUNCTION_ARGS) CHECK_FOR_INTERRUPTS(); - usage_count = BUF_STATE_GET_USAGECOUNT(buf_state); + usage_count = BUF_STATE_GET_COOLSTATE(buf_state); usage_counts[usage_count]++; if (buf_state & BM_DIRTY) diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c index a601c505..fb87c687 100644 --- a/src/backend/storage/buffer/bufmgr.c +++ b/src/backend/storage/buffer/bufmgr.c @@ -2333,7 +2333,10 @@ BufferAlloc(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, * checkpoints, except for their "init" forks, which need to be treated * just like permanent relations. */ - set_bits |= BM_TAG_VALID | BUF_USAGECOUNT_ONE; + set_bits |= BM_TAG_VALID; + /* Admit the newly loaded page COOL (probation); a second access via + * PinBuffer promotes it to HOT. This is what makes a one-touch scan + * self-evicting -- see the cooling-state notes in buf_internals.h. */ if (relpersistence == RELPERSISTENCE_PERMANENT || forkNum == INIT_FORKNUM) set_bits |= BM_PERMANENT; @@ -3002,7 +3005,9 @@ ExtendBufferedRelShared(BufferManagerRelation bmr, victim_buf_hdr->tag = tag; - set_bits |= BM_TAG_VALID | BUF_USAGECOUNT_ONE; + set_bits |= BM_TAG_VALID; + /* Admit COOL (probation); see the comment at the other admission + * site and the cooling-state notes in buf_internals.h. */ if (bmr.relpersistence == RELPERSISTENCE_PERMANENT || fork == INIT_FORKNUM) set_bits |= BM_PERMANENT; @@ -3332,21 +3337,22 @@ PinBuffer(BufferDesc *buf, BufferAccessStrategy strategy, /* increase refcount */ buf_state += BUF_REFCOUNT_ONE; - if (strategy == NULL) - { - /* Default case: increase usagecount unless already max. */ - if (BUF_STATE_GET_USAGECOUNT(buf_state) < BM_MAX_USAGE_COUNT) - buf_state += BUF_USAGECOUNT_ONE; - } - else - { - /* - * Ring buffers shouldn't evict others from pool. Thus we - * don't make usagecount more than 1. - */ - if (BUF_STATE_GET_USAGECOUNT(buf_state) == 0) - buf_state += BUF_USAGECOUNT_ONE; - } + /* + * Accessing a resident buffer promotes it to HOT (the 2Q rescue): a + * page admitted COOL on probation joins the hot working set on its + * second touch. The cooling state saturates at BUF_COOLSTATE_HOT, + * so this never overflows the field. + * + * A strategy (ring) access deliberately does not promote, which is + * the cooling-state form of the stock rule that ring buffers must + * not evict others from the pool: a buffer the ring keeps recycling + * stays COOL and so stays reusable by GetBufferFromRing(), while an + * access from outside the ring promotes it to HOT and thereby takes + * it out of the ring's reuse set. + */ + if (strategy == NULL && + BUF_STATE_GET_COOLSTATE(buf_state) < BUF_COOLSTATE_HOT) + buf_state += BUF_COOLSTATE_ONE; if (pg_atomic_compare_exchange_u64(&buf->state, &old_buf_state, buf_state)) @@ -4214,7 +4220,7 @@ BgBufferSync(WritebackContext *wb_context) * Returns a bitmask containing the following flag bits: * BUF_WRITTEN: we wrote the buffer. * BUF_REUSABLE: buffer is available for replacement, ie, it has - * pin count 0 and usage count 0. + * pin count 0 and is COOL (an eviction candidate). * * (BUF_WRITTEN could be set in error if FlushBuffer finds the buffer clean * after locking it, but we don't care all that much.) @@ -4243,7 +4249,7 @@ SyncOneBuffer(int buf_id, bool skip_recently_used, WritebackContext *wb_context) buf_state = LockBufHdr(bufHdr); if (BUF_STATE_GET_REFCOUNT(buf_state) == 0 && - BUF_STATE_GET_USAGECOUNT(buf_state) == 0) + BUF_STATE_GET_COOLSTATE(buf_state) == BUF_COOLSTATE_COOL) { result |= BUF_REUSABLE; } diff --git a/src/backend/storage/buffer/freelist.c b/src/backend/storage/buffer/freelist.c index 8e3f7c31..987b7648 100644 --- a/src/backend/storage/buffer/freelist.c +++ b/src/backend/storage/buffer/freelist.c @@ -632,12 +632,7 @@ StrategyGetBufferPartition(ClockSweep *sweep, BufferAccessStrategy strategy, { local_buf_state = old_buf_state; - /* - * If the buffer is pinned or has a nonzero usage_count, we cannot - * use it; decrement the usage_count (unless pinned) and keep - * scanning. - */ - + /* If the buffer is pinned we cannot use it; keep scanning. */ if (BUF_STATE_GET_REFCOUNT(local_buf_state) != 0) { if (--trycounter == 0) @@ -661,9 +656,17 @@ StrategyGetBufferPartition(ClockSweep *sweep, BufferAccessStrategy strategy, continue; } - if (BUF_STATE_GET_USAGECOUNT(local_buf_state) != 0) + if (BUF_STATE_GET_COOLSTATE(local_buf_state) != BUF_COOLSTATE_COOL) { - local_buf_state -= BUF_USAGECOUNT_ONE; + /* + * HOT buffer: cool it in place this tick and keep scanning. We + * do NOT claim it now -- a demoted buffer only becomes a victim + * on a later tick, so a HOT buffer always survives the pass that + * cools it and gets a full sweep of grace in which a new access + * can promote it back to HOT. Cooling is progress toward a + * victim, so reset trycounter. + */ + local_buf_state &= ~BUF_USAGECOUNT_MASK; /* HOT -> COOL */ if (pg_atomic_compare_exchange_u64(&buf->state, &old_buf_state, local_buf_state)) @@ -674,7 +677,7 @@ StrategyGetBufferPartition(ClockSweep *sweep, BufferAccessStrategy strategy, } else { - /* pin the buffer if the CAS succeeds */ + /* COOL and unpinned: claim it. Pin if the CAS succeeds. */ local_buf_state += BUF_REFCOUNT_ONE; if (pg_atomic_compare_exchange_u64(&buf->state, &old_buf_state, @@ -1398,14 +1401,17 @@ GetBufferFromRing(BufferAccessStrategy strategy, uint64 *buf_state) /* * If the buffer is pinned we cannot use it under any circumstances. * - * If usage_count is 0 or 1 then the buffer is fair game (we expect 1, - * since our own previous usage of the ring element would have left it - * there, but it might've been decremented by clock-sweep since then). - * A higher usage_count indicates someone else has touched the buffer, - * so we shouldn't re-use it. + * If it is unpinned but has been promoted to HOT, another backend + * touched it since we last cycled past this ring slot, so it has + * joined the working set and we must not recycle it -- tell the caller + * to get a fresh victim from the sweep instead. This is the 1-bit + * cooling-state analog of the stock ring's "usage_count > 1 means + * someone else touched it" test: a slot the ring keeps reusing and + * nobody else pins stays COOL, and an out-of-ring PinBuffer() is + * exactly what promotes it to HOT. */ - if (BUF_STATE_GET_REFCOUNT(local_buf_state) != 0 - || BUF_STATE_GET_USAGECOUNT(local_buf_state) > 1) + if (BUF_STATE_GET_REFCOUNT(local_buf_state) != 0 || + BUF_STATE_GET_COOLSTATE(local_buf_state) != BUF_COOLSTATE_COOL) break; /* See equivalent code in PinBuffer() */ diff --git a/src/backend/storage/buffer/localbuf.c b/src/backend/storage/buffer/localbuf.c index 4870c8e1..f2434112 100644 --- a/src/backend/storage/buffer/localbuf.c +++ b/src/backend/storage/buffer/localbuf.c @@ -167,7 +167,7 @@ LocalBufferAlloc(SMgrRelation smgr, ForkNumber forkNum, BlockNumber blockNum, buf_state = pg_atomic_read_u64(&bufHdr->state); buf_state &= ~(BUF_FLAG_MASK | BUF_USAGECOUNT_MASK); - buf_state |= BM_TAG_VALID | BUF_USAGECOUNT_ONE; + buf_state |= BM_TAG_VALID; /* admit COOL (probation) */ pg_atomic_unlocked_write_u64(&bufHdr->state, buf_state); *foundPtr = false; @@ -248,9 +248,10 @@ GetLocalVictimBuffer(void) { uint64 buf_state = pg_atomic_read_u64(&bufHdr->state); - if (BUF_STATE_GET_USAGECOUNT(buf_state) > 0) + if (BUF_STATE_GET_COOLSTATE(buf_state) != BUF_COOLSTATE_COOL) { - buf_state -= BUF_USAGECOUNT_ONE; + /* HOT: demote it to COOL and keep scanning. */ + buf_state &= ~BUF_USAGECOUNT_MASK; pg_atomic_unlocked_write_u64(&bufHdr->state, buf_state); trycounter = NLocBuffer; } @@ -454,7 +455,7 @@ ExtendBufferedRelLocal(BufferManagerRelation bmr, victim_buf_hdr->tag = tag; - buf_state |= BM_TAG_VALID | BUF_USAGECOUNT_ONE; + buf_state |= BM_TAG_VALID; /* admit COOL (probation) */ pg_atomic_unlocked_write_u64(&victim_buf_hdr->state, buf_state); @@ -839,9 +840,9 @@ PinLocalBuffer(BufferDesc *buf_hdr, bool adjust_usagecount) NLocalPinnedBuffers++; buf_state += BUF_REFCOUNT_ONE; if (adjust_usagecount && - BUF_STATE_GET_USAGECOUNT(buf_state) < BM_MAX_USAGE_COUNT) + BUF_STATE_GET_COOLSTATE(buf_state) < BUF_COOLSTATE_HOT) { - buf_state += BUF_USAGECOUNT_ONE; + buf_state += BUF_COOLSTATE_ONE; } pg_atomic_unlocked_write_u64(&buf_hdr->state, buf_state); diff --git a/src/include/storage/buf_internals.h b/src/include/storage/buf_internals.h index d39bd7e8..4b0f178a 100644 --- a/src/include/storage/buf_internals.h +++ b/src/include/storage/buf_internals.h @@ -67,6 +67,36 @@ StaticAssertDecl(BUF_REFCOUNT_BITS + BUF_USAGECOUNT_BITS + BUF_FLAG_BITS + BUF_L #define BUF_USAGECOUNT_ONE \ (UINT64CONST(1) << BUF_REFCOUNT_BITS) +/* + * Cooling state (LeanStore / 2Q-A1 cooling-stage clock sweep). + * + * The field historically used for the 0..5 usage_count now holds a single + * cooling-state bit: HOT (recently accessed, not an eviction candidate) or + * COOL (an eviction candidate). We reuse BUF_USAGECOUNT_ONE as the unit so + * the buffer-state bit geography -- refcount, flag, and lock offsets, and the + * 64-bit StaticAsserts -- is unchanged; only the meaning of the field and the + * instructions that touch it change. + * + * A demand-loaded page is admitted COOL (probation); a second access promotes + * it to HOT (the rescue). The sweep reclaims an already-COOL buffer and + * demotes a HOT one to COOL as it passes, so a HOT buffer survives the visit + * that cools it and is only reclaimed if it is still COOL when the hand comes + * around again. A page touched once -- a sequential scan -- therefore fills + * and drains the COOL stage without displacing the HOT working set: scan + * resistance intrinsic to the replacement algorithm. + */ +#define BUF_COOLSTATE_COOL 0 +#define BUF_COOLSTATE_HOT 1 +#define BUF_COOLSTATE_ONE BUF_USAGECOUNT_ONE + +/* + * The cooling state is one bit, so the field must be at least that wide. + * Assert it here so a future change to BUF_USAGECOUNT_BITS cannot silently + * narrow the field out from under the cooling state. + */ +StaticAssertDecl(BUF_USAGECOUNT_BITS >= 1, + "cooling state needs at least one bit in the usagecount field"); + /* flags related definitions */ #define BUF_FLAG_SHIFT \ (BUF_REFCOUNT_BITS + BUF_USAGECOUNT_BITS) @@ -92,6 +122,13 @@ StaticAssertDecl(BUF_REFCOUNT_BITS + BUF_USAGECOUNT_BITS + BUF_FLAG_BITS + BUF_L #define BUF_STATE_GET_USAGECOUNT(state) \ ((uint32)(((state) & BUF_USAGECOUNT_MASK) >> BUF_USAGECOUNT_SHIFT)) +/* + * Cooling state (HOT/COOL) from buffer state. The field holds only + * BUF_COOLSTATE_COOL or BUF_COOLSTATE_HOT, so this is the whole field. + */ +#define BUF_STATE_GET_COOLSTATE(state) \ + ((uint32) (((state) & BUF_USAGECOUNT_MASK) >> BUF_USAGECOUNT_SHIFT)) + /* * Flags for buffer descriptors * @@ -134,17 +171,15 @@ StaticAssertDecl(MAX_BACKENDS_BITS <= (BUF_LOCK_BITS - 2), /* - * The maximum allowed value of usage_count represents a tradeoff between - * accuracy and speed of the clock-sweep buffer management algorithm. A - * large value (comparable to NBuffers) would approximate LRU semantics. - * But it can take as many as BM_MAX_USAGE_COUNT+1 complete cycles of the - * clock-sweep hand to find a free buffer, so in practice we don't want the - * value to be very large. + * The cooling state is a single bit (HOT/COOL); the maximum value stored in + * the field is therefore BUF_COOLSTATE_HOT. Retained under the historical + * name BM_MAX_USAGE_COUNT so the pin fast path ("promote unless already at + * max") reads naturally. */ -#define BM_MAX_USAGE_COUNT 5 +#define BM_MAX_USAGE_COUNT BUF_COOLSTATE_HOT StaticAssertDecl(BM_MAX_USAGE_COUNT < (UINT64CONST(1) << BUF_USAGECOUNT_BITS), - "BM_MAX_USAGE_COUNT doesn't fit in BUF_USAGECOUNT_BITS bits"); + "cooling state doesn't fit in BUF_USAGECOUNT_BITS bits"); /* * Buffer tag identifies which disk block the buffer contains. -- 2.50.1