From 7fd0f0019ee668fac4b6ec3af7da53286081596f Mon Sep 17 00:00:00 2001 From: Ashutosh Bapat Date: Mon, 27 Jul 2026 11:14:32 +0530 Subject: [PATCH] Follow-up changes since last email on hackers Please note that the stress tests added by this or the earlier commits are not necessarily meant to be committed to the core. But they are in the patch so that reviewers have some readily available stress tests. Move the SQL wrapper around pg_resize_shared_buffers() to an extension. Address stress test utility common for all buffer pool resizing stress tests. Use randomised sequence of sizes when resizing. Test synchronization between buffer pool resizing and DropRelationBuffers, DropRelationsAllBuffers, DropDatabaseBuffers, CHECKPOINT, FlushRelationBuffers, FlushRelationsAllBuffers, monitoring and diagnostic functions in pg_buffercache and pg_prewarm. FlushDatabaseBuffers() is not covered by any stress test: it is only called during WAL replay of xl_dbase_create_file_copy_rec (in dbcommands.c). The primary CREATE DATABASE and ALTER DATABASE SET TABLESPACE paths use RequestCheckpoint() instead, so the function is not reachable from a live SQL workload. A bug fix in BufferSync because of which checkpoint didn't advance post a buffer pool shrink and blocked any other activity that involved ProcSignalBarrier. Use separate number ranges for stress tests and other tests. Add database checker at the end of all stress tests. Enable running tests only when PG_TEST_EXTRA has bufmgr_stress. Also skip all tests if have_resizable_shmem is OFF. Change pg_resize_shared_buffers() to throw an error when the server does not support resizable shmem. Fix when CFI is called in some pg_buffercache functions that scan the buffer pool. Skip pg_resize_shared_buffers tests when resizable shmem is not supported. Add a test to make sure that pg_resize_shared_buffers() causes an error when invoked in a server which does not support resizable shmem. --- contrib/pg_buffercache/pg_buffercache_pages.c | 24 +- doc/src/sgml/regress.sgml | 11 + src/backend/catalog/storage.c | 4 + src/backend/storage/buffer/buf_resize.c | 16 + src/backend/storage/buffer/bufmgr.c | 63 +- src/test/buffermgr/Makefile | 7 +- src/test/buffermgr/README | 28 +- src/test/buffermgr/buffermgr_test--1.0.sql | 52 ++ src/test/buffermgr/buffermgr_test.control | 3 + src/test/buffermgr/meson.build | 22 +- src/test/buffermgr/t/001_resize_buffer.pl | 193 ------ ...rance.pl => 001_resize_fault_tolerance.pl} | 11 +- ...ze.pl => 002_client_join_buffer_resize.pl} | 15 +- ...ize_failures.pl => 003_resize_failures.pl} | 7 + ...logger.pl => 004_resize_with_syslogger.pl} | 7 + .../buffermgr/t/005_resize_unsupported.pl | 51 ++ .../buffermgr/t/010_stress_resize_buffer.pl | 34 + .../t/011_stress_drop_relation_buffers.pl | 99 +++ .../t/012_stress_drop_database_buffers.pl | 82 +++ src/test/buffermgr/t/013_stress_checkpoint.pl | 47 ++ .../t/014_stress_flush_relation_buffers.pl | 68 ++ .../buffermgr/t/015_stress_pg_buffercache.pl | 60 ++ src/test/buffermgr/t/016_stress_pg_prewarm.pl | 73 ++ src/test/buffermgr/t/StressUtil.pm | 628 ++++++++++++++++++ 24 files changed, 1361 insertions(+), 244 deletions(-) create mode 100644 src/test/buffermgr/buffermgr_test--1.0.sql create mode 100644 src/test/buffermgr/buffermgr_test.control delete mode 100644 src/test/buffermgr/t/001_resize_buffer.pl rename src/test/buffermgr/t/{003_resize_fault_tolerance.pl => 001_resize_fault_tolerance.pl} (99%) rename src/test/buffermgr/t/{004_client_join_buffer_resize.pl => 002_client_join_buffer_resize.pl} (96%) rename src/test/buffermgr/t/{005_resize_failures.pl => 003_resize_failures.pl} (96%) rename src/test/buffermgr/t/{006_resize_with_syslogger.pl => 004_resize_with_syslogger.pl} (87%) create mode 100644 src/test/buffermgr/t/005_resize_unsupported.pl create mode 100644 src/test/buffermgr/t/010_stress_resize_buffer.pl create mode 100644 src/test/buffermgr/t/011_stress_drop_relation_buffers.pl create mode 100644 src/test/buffermgr/t/012_stress_drop_database_buffers.pl create mode 100644 src/test/buffermgr/t/013_stress_checkpoint.pl create mode 100644 src/test/buffermgr/t/014_stress_flush_relation_buffers.pl create mode 100644 src/test/buffermgr/t/015_stress_pg_buffercache.pl create mode 100644 src/test/buffermgr/t/016_stress_pg_prewarm.pl create mode 100644 src/test/buffermgr/t/StressUtil.pm diff --git a/contrib/pg_buffercache/pg_buffercache_pages.c b/contrib/pg_buffercache/pg_buffercache_pages.c index 312343fd7bf..7335ccae150 100644 --- a/contrib/pg_buffercache/pg_buffercache_pages.c +++ b/contrib/pg_buffercache/pg_buffercache_pages.c @@ -150,8 +150,6 @@ pg_buffercache_pages(PG_FUNCTION_ARGS) Datum values[NUM_BUFFERCACHE_PAGES_ELEM]; bool nulls[NUM_BUFFERCACHE_PAGES_ELEM]; - CHECK_FOR_INTERRUPTS(); - bufHdr = GetBufferDescriptor(i); /* Lock each buffer header before inspecting. */ buf_state = LockBufHdr(bufHdr); @@ -220,6 +218,12 @@ pg_buffercache_pages(PG_FUNCTION_ARGS) } tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, values, nulls); + + /* + * Check for interrupts here, at the end of the loop, so that the buffer + * index i remains valid till the next iteration. + */ + CHECK_FOR_INTERRUPTS(); } return (Datum) 0; @@ -457,8 +461,6 @@ pg_buffercache_os_pages_internal(FunctionCallInfo fcinfo, bool include_numa) char *startptr_buff, *endptr_buff; - CHECK_FOR_INTERRUPTS(); - bufHdr = GetBufferDescriptor(i); /* Lock each buffer header before inspecting. */ @@ -488,6 +490,12 @@ pg_buffercache_os_pages_internal(FunctionCallInfo fcinfo, bool include_numa) ++idx; ++page_num; } + + /* + * Check for interrupts here, at the end of the loop, so that the + * buffer index i remains valid till the next iteration. + */ + CHECK_FOR_INTERRUPTS(); } Assert(idx <= max_entries); @@ -599,8 +607,6 @@ pg_buffercache_summary(PG_FUNCTION_ARGS) BufferDesc *bufHdr; uint64 buf_state; - CHECK_FOR_INTERRUPTS(); - /* * This function summarizes the state of all headers. Locking the * buffer headers wouldn't provide an improved result as the state of @@ -623,6 +629,12 @@ pg_buffercache_summary(PG_FUNCTION_ARGS) if (BUF_STATE_GET_REFCOUNT(buf_state) > 0) buffers_pinned++; + + /* + * Check for interrupts here, at the end of the loop, so that the buffer + * index i remains valid till the next iteration. + */ + CHECK_FOR_INTERRUPTS(); } memset(nulls, 0, sizeof(nulls)); diff --git a/doc/src/sgml/regress.sgml b/doc/src/sgml/regress.sgml index c74941bfbf2..30092fd820d 100644 --- a/doc/src/sgml/regress.sgml +++ b/doc/src/sgml/regress.sgml @@ -275,6 +275,17 @@ make check-world PG_TEST_EXTRA='kerberos ldap ssl load_balance libpq_encryption' The following values are currently supported: + + bufmgr_stress + + + Runs the shared_buffers resize stress tests under + src/test/buffermgr. Not enabled by default because + they are long-running and resource-intensive. + + + + checksum, checksum_extended diff --git a/src/backend/catalog/storage.c b/src/backend/catalog/storage.c index e443a4993c5..5c36820d58f 100644 --- a/src/backend/catalog/storage.c +++ b/src/backend/catalog/storage.c @@ -33,6 +33,7 @@ #include "storage/proc.h" #include "storage/smgr.h" #include "utils/hsearch.h" +#include "utils/injection_point.h" #include "utils/memutils.h" #include "utils/rel.h" @@ -383,6 +384,9 @@ RelationTruncate(Relation rel, BlockNumber nblocks) * * (See also visibilitymap.c if changing this code.) */ + + /* Load the injection point before entering the critical section */ + INJECTION_POINT_LOAD("drop-relation-buffers-scan"); START_CRIT_SECTION(); if (RelationNeedsWAL(rel)) diff --git a/src/backend/storage/buffer/buf_resize.c b/src/backend/storage/buffer/buf_resize.c index c758493f353..90f5fb1c71d 100644 --- a/src/backend/storage/buffer/buf_resize.c +++ b/src/backend/storage/buffer/buf_resize.c @@ -36,6 +36,7 @@ static volatile sig_atomic_t safe_exit = true; +#ifdef HAVE_RESIZABLE_SHMEM static bool resize_shared_buffers_internal(void); static void buf_resize_shmem_exit(int code, Datum arg); @@ -97,6 +98,7 @@ buf_resize_shmem_resize(int currentNBuffers, int targetNBuffers) elog(LOG, "all backends acknowledged PROCSIGNAL_BARRIER_BUFFER_POOL_RESIZE barrier"); return true; } +#endif /* * C implementation of SQL interface to update the shared buffers according to @@ -188,8 +190,19 @@ buf_resize_shmem_resize(int currentNBuffers, int targetNBuffers) Datum pg_resize_shared_buffers(PG_FUNCTION_ARGS) { +#ifndef HAVE_RESIZABLE_SHMEM + ereport(ERROR, + errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("resizing shared buffer pool is not supported on this platform")); + pg_unreachable(); +#else bool success = false; + if (shared_memory_type != SHMEM_TYPE_MMAP) + ereport(ERROR, + errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("resizing shared buffer pool is not supported on this platform")); + /* * Register the exit hook before claiming resizer_pid, so that if we exit * after claiming resizer_pid, the hook is in place to reset it. @@ -269,8 +282,10 @@ pg_resize_shared_buffers(PG_FUNCTION_ARGS) elog(WARNING, "shared buffer resizing to %d buffers failed", NBuffersGUC); PG_RETURN_BOOL(success); +#endif } +#ifdef HAVE_RESIZABLE_SHMEM /* * Workhorse function for the C implementation. */ @@ -404,6 +419,7 @@ buf_resize_shmem_exit(int code, Datum arg) (void) pg_atomic_compare_exchange_u32(&BufferControl->resizer_pid, &expected_pid, 0); } +#endif /* * Process and acknowledge PROCSIGNAL_BARRIER_NEW_BUFFER_ALLOC. diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c index cebc85624b8..bb436734585 100644 --- a/src/backend/storage/buffer/bufmgr.c +++ b/src/backend/storage/buffer/bufmgr.c @@ -64,6 +64,7 @@ #include "storage/read_stream.h" #include "storage/smgr.h" #include "storage/standby.h" +#include "utils/injection_point.h" #include "utils/memdebug.h" #include "utils/ps_status.h" #include "utils/rel.h" @@ -3793,36 +3794,35 @@ BufferSync(int flags) /* * The buffer pool might have been shrunk between the time the - * checkpoint collected the buffer ids and now. Ignore any buffers - * that are out of range now. Those buffers must have been written - * when they were evicted during resizing. + * checkpoint collected the buffer ids and now. Skip any buffers that + * are out of range now; they were written when they were evicted + * during resizing. */ - if (buf_id >= NBuffers) - continue; - - bufHdr = GetBufferDescriptor(buf_id); - - num_processed++; - - /* - * We don't need to acquire the lock here, because we're only looking - * at a single bit. It's possible that someone else writes the buffer - * and clears the flag right after we check, but that doesn't matter - * since SyncOneBuffer will then do nothing. However, there is a - * further race condition: it's conceivable that between the time we - * examine the bit here and the time SyncOneBuffer acquires the lock, - * someone else not only wrote the buffer but replaced it with another - * page and dirtied it. In that improbable case, SyncOneBuffer will - * write the buffer though we didn't need to. It doesn't seem worth - * guarding against this, though. - */ - if (pg_atomic_read_u64(&bufHdr->state) & BM_CHECKPOINT_NEEDED) + if (buf_id < NBuffers) { - if (SyncOneBuffer(buf_id, false, &wb_context) & BUF_WRITTEN) + bufHdr = GetBufferDescriptor(buf_id); + + /* + * We don't need to acquire the lock here, because we're only + * looking at a single bit. It's possible that someone else writes + * the buffer and clears the flag right after we check, but that + * doesn't matter since SyncOneBuffer will then do nothing. + * However, there is a further race condition: it's conceivable + * that between the time we examine the bit here and the time + * SyncOneBuffer acquires the lock, someone else not only wrote + * the buffer but replaced it with another page and dirtied it. + * In that improbable case, SyncOneBuffer will write the buffer + * though we didn't need to. It doesn't seem worth guarding + * against this, though. + */ + if (pg_atomic_read_u64(&bufHdr->state) & BM_CHECKPOINT_NEEDED) { - TRACE_POSTGRESQL_BUFFER_SYNC_WRITTEN(buf_id); - PendingCheckpointerStats.buffers_written++; - num_written++; + if (SyncOneBuffer(buf_id, false, &wb_context) & BUF_WRITTEN) + { + TRACE_POSTGRESQL_BUFFER_SYNC_WRITTEN(buf_id); + PendingCheckpointerStats.buffers_written++; + num_written++; + } } } @@ -3833,6 +3833,7 @@ BufferSync(int flags) ts_stat->progress += ts_stat->progress_slice; ts_stat->num_scanned++; ts_stat->index++; + num_processed++; /* Have all the buffers from the tablespace been processed? */ if (ts_stat->num_scanned == ts_stat->num_to_scan) @@ -4976,6 +4977,8 @@ DropRelationBuffers(SMgrRelation smgr_reln, ForkNumber *forkNum, if (j >= nforks) UnlockBufHdr(bufHdr); } + + INJECTION_POINT_CACHED("drop-relation-buffers-scan", NULL); } /* --------------------------------------------------------------------- @@ -5143,6 +5146,8 @@ DropRelationsAllBuffers(SMgrRelation *smgr_reln, int nlocators) UnlockBufHdr(bufHdr); } + INJECTION_POINT("drop-relations-all-buffers-scan", NULL); + pfree(locators); pfree(rels); } @@ -5243,6 +5248,8 @@ DropDatabaseBuffers(Oid dbid) else UnlockBufHdr(bufHdr); } + + INJECTION_POINT("drop-database-buffers-scan", NULL); } /* --------------------------------------------------------------------- @@ -5340,6 +5347,7 @@ FlushRelationBuffers(Relation rel) else UnlockBufHdr(bufHdr); } + INJECTION_POINT("flush-relation-buffers-scan", NULL); } /* --------------------------------------------------------------------- @@ -5435,6 +5443,7 @@ FlushRelationsAllBuffers(SMgrRelation *smgrs, int nrels) else UnlockBufHdr(bufHdr); } + INJECTION_POINT("flush-relations-all-buffers-scan", NULL); pfree(srels); } diff --git a/src/test/buffermgr/Makefile b/src/test/buffermgr/Makefile index 24c245c900a..92a430e736b 100644 --- a/src/test/buffermgr/Makefile +++ b/src/test/buffermgr/Makefile @@ -9,10 +9,15 @@ # #------------------------------------------------------------------------- -EXTRA_INSTALL = contrib/pg_buffercache \ +EXTRA_INSTALL = contrib/amcheck \ + contrib/pg_buffercache \ + contrib/pg_prewarm \ src/test/modules/injection_points \ src/test/modules/test_shmem +EXTENSION = buffermgr_test +DATA = buffermgr_test--1.0.sql + REGRESS = buffer_resize # Custom configuration for buffer manager tests diff --git a/src/test/buffermgr/README b/src/test/buffermgr/README index c375ad80989..55bcf0a800c 100644 --- a/src/test/buffermgr/README +++ b/src/test/buffermgr/README @@ -3,8 +3,34 @@ src/test/buffermgr/README Regression tests for buffer manager =================================== -This directory contains a test suite for resizing buffer manager without restarting the server. +This directory contains a test suite for resizing buffer manager without +restarting the server. +Some of the TAP tests rely on a helper extension buffermgr_test. It bundles SQL +helpers (such as pg_resize_shared_buffers_sql) used by the tests. + +Stress tests +------------------ +These tests exercise the synchronization between the buffer manager resizing and the code that uses buffer manager. They run the SQL commands that exercise the specific subsystem functionality in parallel with the resizing of buffer manager, both in a tight loop so as to increase the chances of hitting race conditions. Optionally they run pgbench to keep the buffer pool busy. + +1. simple pgbench: 010_stress_resize_buffer.pl + +2. checkpoint stress test: TODO: Palak Chaturvedi. Disable automatic checkpointing. Run pgbench. Run checkpoint and resize buffer pool in a tight loop. + +3. pg_buffercache stress test: TODO: Run pgbench. Run pg_buffercache queries and resize buffer pool in a tight loop. + +4. pg_prewarm stress test: TODO: Need to figure out exactly what to test. + +5. Relation/Database buffer scan stress test: TODO: +5.a: Create a table with a lot of data (spanning more than NBuffers/32 pages) and truncate it within the same transaction, insert the data again and drop the table. This should exercise both DropRelationBuffers() and DropRelationsAllBuffers(). Run this in a tight loop along with resizing buffer pool. Run pgbench so that the table being created and dropped has its pages interspersed with other pages in the buffer pool. +5.b: Similarly Create and drop a database in a tight loop along with resizing buffer pool. Run pgbench so that the database being created and dropped has its pages interspersed with other pages in the buffer pool. You may want to populate some tables in the template database so that the database being created and dropped is prepopulated with some data. +5.c: This is more nuanced. Run commands exercising FlushRelationBuffers(), FlushRelationsAllBuffers() and FlushDatabaseBuffers() in a tight loop along with resizing buffer pool. Run pgbench so that the relations/databases being flushed have their pages interspersed with other pages in the buffer pool. The exact commands to run are TBD and need experimentation. + +Injection point tests +------------------ +Any issues discovered during the stress tests can be converted into injection +point tests. Injection points allow to simulate a scenario precisely and +deterministically. Running the tests ================= diff --git a/src/test/buffermgr/buffermgr_test--1.0.sql b/src/test/buffermgr/buffermgr_test--1.0.sql new file mode 100644 index 00000000000..51901508523 --- /dev/null +++ b/src/test/buffermgr/buffermgr_test--1.0.sql @@ -0,0 +1,52 @@ +-- Helper functions used by TAP tests in src/test/buffermgr/t. +-- + +\echo Use "CREATE EXTENSION buffermgr_test" to load this file. \quit + +-- Retries pg_resize_shared_buffers() until it succeeds, then confirms the +-- new value is in effect. Returns the number of retries taken along with +-- the wall-clock times immediately before and after the retry loop. +-- +-- The new size is expected to be set in shared_buffers GUC before calling this +-- function. +create function pg_resize_shared_buffers_sql( + new_size int, + out num_tries int, + out started_at timestamptz, + out ended_at timestamptz) +returns record as $$ +declare + success boolean := false; + tries int := 0; + cur_setting text; + target text := new_size::text; + pending_pattern text := '%(pending: ' || target || ')%'; +begin + select setting into cur_setting + from pg_settings where name = 'shared_buffers'; + if cur_setting <> target and cur_setting not like pending_pattern then + raise exception 'shared_buffers change not visible to this backend: setting is %, expected % or matching %', + cur_setting, target, pending_pattern; + end if; + + started_at := clock_timestamp(); + while not success loop + tries := tries + 1; + select pg_resize_shared_buffers() into success; + if not success then + perform pg_sleep(0.1); + end if; + end loop; + ended_at := clock_timestamp(); + + select setting into cur_setting + from pg_settings where name = 'shared_buffers'; + if cur_setting <> target then + raise exception 'shared_buffers resize did not take effect: expected %, got %', + target, cur_setting; + end if; + + num_tries := tries; + return; +end; +$$ language plpgsql; diff --git a/src/test/buffermgr/buffermgr_test.control b/src/test/buffermgr/buffermgr_test.control new file mode 100644 index 00000000000..2d13d889ec7 --- /dev/null +++ b/src/test/buffermgr/buffermgr_test.control @@ -0,0 +1,3 @@ +comment = 'Helpers for src/test/buffermgr TAP tests' +default_version = '1.0' +relocatable = true diff --git a/src/test/buffermgr/meson.build b/src/test/buffermgr/meson.build index 7a6d5e29f8d..3c64d2740ee 100644 --- a/src/test/buffermgr/meson.build +++ b/src/test/buffermgr/meson.build @@ -1,5 +1,10 @@ # Copyright (c) 2022-2025, PostgreSQL Global Development Group +test_install_data += files( + 'buffermgr_test.control', + 'buffermgr_test--1.0.sql', +) + tests += { 'name': 'buffermgr', 'sd': meson.current_source_dir(), @@ -15,11 +20,18 @@ tests += { 'enable_injection_points': get_option('injection_points') ? 'yes' : 'no', }, 'tests': [ - 't/001_resize_buffer.pl', - 't/003_resize_fault_tolerance.pl', - 't/004_client_join_buffer_resize.pl', - 't/005_resize_failures.pl', - 't/006_resize_with_syslogger.pl', + 't/001_resize_fault_tolerance.pl', + 't/002_client_join_buffer_resize.pl', + 't/003_resize_failures.pl', + 't/004_resize_with_syslogger.pl', + 't/005_resize_unsupported.pl', + 't/010_stress_resize_buffer.pl', + 't/011_stress_drop_relation_buffers.pl', + 't/012_stress_drop_database_buffers.pl', + 't/013_stress_checkpoint.pl', + 't/014_stress_flush_relation_buffers.pl', + 't/015_stress_pg_buffercache.pl', + 't/016_stress_pg_prewarm.pl', ], }, } diff --git a/src/test/buffermgr/t/001_resize_buffer.pl b/src/test/buffermgr/t/001_resize_buffer.pl deleted file mode 100644 index fb5a42be26a..00000000000 --- a/src/test/buffermgr/t/001_resize_buffer.pl +++ /dev/null @@ -1,193 +0,0 @@ -# Copyright (c) 2025-2025, PostgreSQL Global Development Group -# -# Minimal test testing shared_buffer resizing under load - -use strict; -use warnings; -use IPC::Run; -use PostgreSQL::Test::Cluster; -use PostgreSQL::Test::Utils; -use Test::More; - -# Function to check if pgbench is still running. -# -# Relying on IPC::Run's pumpable status to check if pgbench is still running has -# been proven unreliable. Instead we rely on existence of pgbench processes in -# pg_stat_activity. Since we use -C with pgbench, there can be a non-zero -# chance that no pgbench process is running even thought pgbench is running. But -# that's a very rare possibility that can be ignored. -sub pgbench_processes_active -{ - my ($node, $application_name) = @_; - - my $result = $node->safe_psql('postgres', - "SELECT count(*) FROM pg_stat_activity WHERE application_name = '$application_name';"); - return int($result) > 0; -} - -my $resize_sql_func_def = q{ -create or replace function pg_resize_shared_buffers_sql(new_size int, out num_tries int) returns int as $$ -declare - success boolean := false; - tries int := 0; - cur_setting text; - pending_pattern text; - target text := new_size::text; -begin - -- Wait until pg_settings reports the new value as pending, - -- i.e. " (pending: )". - pending_pattern := '%(pending: ' || target || ')%'; - loop - select setting into cur_setting - from pg_settings where name = 'shared_buffers'; - exit when cur_setting like pending_pattern or cur_setting = target; - perform pg_sleep(0.1); - raise notice 'Current setting: %', cur_setting; - end loop; - - -- pg_resize_shared_buffers() returns true on success; retry until it succeeds. - while not success loop - tries := tries + 1; - select pg_resize_shared_buffers() into success; - if not success then - perform pg_sleep(0.1); - end if; - raise notice 'pg_resize_shared_buffers() attempt %: success = %', tries, success; - end loop; - - -- Confirm the new value is in effect (no longer pending). - select setting into cur_setting - from pg_settings where name = 'shared_buffers'; - if cur_setting <> target then - raise exception 'shared_buffers resize did not take effect: expected %, got %', - target, cur_setting; - end if; - - num_tries := tries; - return; -end; -$$ language plpgsql; -}; - -# Function to resize buffer pool and verify the change. -sub apply_and_verify_buffer_change -{ - my ($node, $new_size) = @_; - - # Use the new pg_resize_shared_buffers() interface which handles everything synchronously - $node->safe_psql('postgres', "ALTER SYSTEM SET shared_buffers = '$new_size'"); - $node->safe_psql('postgres', "SELECT pg_reload_conf()"); - $node->safe_psql('postgres', "SELECT pg_resize_shared_buffers_sql($new_size)"); - - # Any failure in resizing the buffer pool will cause the test to timeout. So - # if we reach here, the resize was successful. Just declare it as a - # successful test so that we can see progress in the test output. - ok(1, "Buffer pool resized to $new_size"); -} - -my @buffer_sizes = (128, 28, 16 * 1024, 32 * 1024, 1024, 512, 16, 24, 256, 128 * 1024, 16 * 1204); - -# Initialize a cluster and start pgbench in the background for concurrent load. -my $node = PostgreSQL::Test::Cluster->new('main'); -$node->init; - -# Permit resizing up to 1GB for this test and let the server start with 128MB. -$node->append_conf('postgresql.conf', qq{ -max_shared_buffers = } . (sort { $b <=> $a } @buffer_sizes)[0] . qq{ -shared_buffers = 16 -log_statement = none -restart_after_crash = off -}); - -$node->start; -$node->safe_psql('postgres', "CREATE EXTENSION pg_buffercache"); -$node->safe_psql('postgres', $resize_sql_func_def); - -my $pgb_scale = 10; -my $pgb_duration = 120; -my $pgb_num_clients = 10; -# make it easy to identify pgbench processes in pg_stat_activity -my $application_name = 'pgbench_buffer_resize_test'; -$node->pgbench( - "--initialize --init-steps=dtpvg --scale=$pgb_scale --quiet", - 0, - [qr{^$}], - [ # stderr patterns to verify initialization stages - qr{dropping old tables}, - qr{creating tables}, - qr{done in \d+\.\d\d s } - ], - "pgbench initialization (scale=$pgb_scale)" -); -my ($pgbench_stdin, $pgbench_stdout, $pgbench_stderr) = ('', '', ''); -# Use --exit-on-abort so that the test stops on the first server crash or error, -# thus making it easy to debug the failure. Use -C to increase the chances of a -# new backend being created while resizing the buffer pool. -my $pgbench_process = IPC::Run::start( - [ - 'pgbench', - '-p', $node->port, - '-h', $node->host, - '-T', $pgb_duration, - '-c', $pgb_num_clients, - '-C', - '--exit-on-abort', - '--continue-on-error', - "dbname=postgres application_name=$application_name" - ], - '<' => \$pgbench_stdin, - '>' => \$pgbench_stdout, - '2>' => \$pgbench_stderr -); - -ok($pgbench_process, "pgbench started successfully"); - -# Resize buffer pool to various sizes while pgbench is running in the -# background. We use smaller sizes to induce frequent buffer eviction and -# allocation. Also smaller buffer pool means frequent wraparound in background -# writer, default buffer allocation strategy and checkpointer. -# -# TODO: These are pseudo-randomly picked sizes, but we can do better. -my $tests_completed = 0; - -# Reset background writer stats before starting the resize cycle -$node->safe_psql('postgres', "SELECT pg_stat_reset_shared('bgwriter')"); - -# Resize as many times as possible while pgbench is running. -while (pgbench_processes_active($node, $application_name)) -{ - for my $target_size (@buffer_sizes) - { - # Stop if pgbench finished - if (!pgbench_processes_active($node, $application_name)) - { - last; - } - - apply_and_verify_buffer_change($node, $target_size); - $tests_completed++; - - # Wait for the resized buffer pool to stabilize. - sleep(1); - } -} - -ok($tests_completed > scalar(@buffer_sizes), "All buffer size transitions were tested"); -note "Completed $tests_completed buffer resize operations while pgbench was running"; - -# Check that the background writer did some work during the resize cycle -is($node->safe_psql('postgres', "SELECT buffers_clean > 0 FROM pg_stat_bgwriter"), 't', "Background writer ran during resize cycle"); - -# Make sure that pgbench finishes -$pgbench_process->signal('TERM'); -ok((IPC::Run::finish $pgbench_process), "pgbench finished successfully"); - -# Log any error output from pgbench for debugging -diag("pgbench stderr:\n$pgbench_stderr"); -diag("pgbench stdout:\n$pgbench_stdout"); - -# Ensure database is still functional after all the buffer changes -$node->connect_ok("dbname=postgres", - "Database remains accessible after $tests_completed buffer resize operations"); - -done_testing(); diff --git a/src/test/buffermgr/t/003_resize_fault_tolerance.pl b/src/test/buffermgr/t/001_resize_fault_tolerance.pl similarity index 99% rename from src/test/buffermgr/t/003_resize_fault_tolerance.pl rename to src/test/buffermgr/t/001_resize_fault_tolerance.pl index 366929f4c45..87c95dbedf1 100644 --- a/src/test/buffermgr/t/003_resize_fault_tolerance.pl +++ b/src/test/buffermgr/t/001_resize_fault_tolerance.pl @@ -29,6 +29,13 @@ $node->append_conf('postgresql.conf', 'max_shared_buffers = 32'); $node->append_conf('postgresql.conf', 'restart_after_crash = on'); $node->start; +# Bail out if this build does not support resizable shared memory, which +# also means that resizing buffer pool is not supported. +if ($node->safe_psql('postgres', 'SHOW have_resizable_shmem') ne 'on') +{ + plan skip_all => "resizable shared memory not supported by this build"; +} + # Load injection points extension for test coordination $node->safe_psql('postgres', "CREATE EXTENSION injection_points"); @@ -832,8 +839,4 @@ else done_testing(); # Few more tests to add but may be somewhere else -# TODO: test when there are backends that have not attached to the shared memory # TODO: test that a non-superuser cannot run pg_resize_shared_buffers() -# TODO: the resize_sql_func_def in 001_resize_buffer may be useful in other -# tests (not necessarily this one). Maybe we can use it in other tests where we -# are looping in TAP test code. diff --git a/src/test/buffermgr/t/004_client_join_buffer_resize.pl b/src/test/buffermgr/t/002_client_join_buffer_resize.pl similarity index 96% rename from src/test/buffermgr/t/004_client_join_buffer_resize.pl rename to src/test/buffermgr/t/002_client_join_buffer_resize.pl index fda0f01bb27..6cca03d1816 100644 --- a/src/test/buffermgr/t/004_client_join_buffer_resize.pl +++ b/src/test/buffermgr/t/002_client_join_buffer_resize.pl @@ -78,20 +78,21 @@ max_parallel_workers_per_gather = 0 }); $node->start; +# Bail out if this build does not support resizable shared memory, which +# also means that resizing buffer pool is not supported. +if ($node->safe_psql('postgres', 'SHOW have_resizable_shmem') ne 'on') +{ + plan skip_all => "resizable shared memory not supported by this build"; +} + # Enable injection points $node->safe_psql('postgres', "CREATE EXTENSION injection_points"); +$node->safe_psql('postgres', "CREATE EXTENSION pg_buffercache"); # Get the block size (this is fixed for the binary) my $block_size = $node->safe_psql('postgres', "SHOW block_size"); # Try to create pg_buffercache extension for buffer analysis -eval { - $node->safe_psql('postgres', "CREATE EXTENSION pg_buffercache"); -}; -if ($@) { - $node->stop; - plan skip_all => 'pg_buffercache extension not available - cannot verify buffer usage'; -} # Create a small test table, and fetch its properties for later reference if required. $node->safe_psql('postgres', qq{ diff --git a/src/test/buffermgr/t/005_resize_failures.pl b/src/test/buffermgr/t/003_resize_failures.pl similarity index 96% rename from src/test/buffermgr/t/005_resize_failures.pl rename to src/test/buffermgr/t/003_resize_failures.pl index 820aa07f006..21986733f74 100644 --- a/src/test/buffermgr/t/005_resize_failures.pl +++ b/src/test/buffermgr/t/003_resize_failures.pl @@ -24,6 +24,13 @@ $node->append_conf('postgresql.conf', "shared_buffers = $initial_nbuffers"); $node->append_conf('postgresql.conf', "max_shared_buffers = $max_nbuffers"); $node->start; +# Bail out if this build does not support resizable shared memory, which +# also means that resizing buffer pool is not supported. +if ($node->safe_psql('postgres', 'SHOW have_resizable_shmem') ne 'on') +{ + plan skip_all => "resizable shared memory not supported by this build"; +} + # pg_buffercache lets us locate the bufferid holding a given page. $node->safe_psql('postgres', "CREATE EXTENSION pg_buffercache"); if ($have_injection_points) diff --git a/src/test/buffermgr/t/006_resize_with_syslogger.pl b/src/test/buffermgr/t/004_resize_with_syslogger.pl similarity index 87% rename from src/test/buffermgr/t/006_resize_with_syslogger.pl rename to src/test/buffermgr/t/004_resize_with_syslogger.pl index 75b047ad984..e546b4c390e 100644 --- a/src/test/buffermgr/t/006_resize_with_syslogger.pl +++ b/src/test/buffermgr/t/004_resize_with_syslogger.pl @@ -25,6 +25,13 @@ logging_collector = on }); $node->start; +# Bail out if this build does not support resizable shared memory, which +# also means that resizing buffer pool is not supported. +if ($node->safe_psql('postgres', 'SHOW have_resizable_shmem') ne 'on') +{ + plan skip_all => "resizable shared memory not supported by this build"; +} + # Check that the syslogger is running by writing a log marker and waiting for it # to appear in the log file. sub check_syslogger_running diff --git a/src/test/buffermgr/t/005_resize_unsupported.pl b/src/test/buffermgr/t/005_resize_unsupported.pl new file mode 100644 index 00000000000..fcaf3311fea --- /dev/null +++ b/src/test/buffermgr/t/005_resize_unsupported.pl @@ -0,0 +1,51 @@ +# Copyright (c) 2026-2026, PostgreSQL Global Development Group +# +# Test that pg_resize_shared_buffers() errors out when resizable shared +# memory is not supported. + +use strict; +use warnings; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $initial_nbuffers = 256; +my $max_nbuffers = 512; +my $node = PostgreSQL::Test::Cluster->new('main'); +$node->init; +$node->append_conf( + 'postgresql.conf', qq{ +shared_buffers = $initial_nbuffers +max_shared_buffers = $max_nbuffers +}); +$node->start; + +if ($node->safe_psql('postgres', 'SHOW have_resizable_shmem') eq 'on') +{ + # The builds that support resizable shared memory, usually, will not support + # the feature when SysV shared memory is used. + $node->safe_psql('postgres', + "ALTER SYSTEM SET shared_memory_type = 'sysv'"); + $node->restart; +} + +is( $node->safe_psql('postgres', 'SHOW have_resizable_shmem'), + 'off', + 'have_resizable_shmem reports off'); + +my $target_nbuffers = $initial_nbuffers / 2; +$node->safe_psql('postgres', "ALTER SYSTEM SET shared_buffers = '$target_nbuffers'"); +$node->safe_psql('postgres', "SELECT pg_reload_conf()"); + +my ($ret, $stdout, $stderr) = + $node->psql('postgres', "SELECT pg_resize_shared_buffers()"); +isnt($ret, 0, + 'pg_resize_shared_buffers fails when resizable shared memory is unsupported' +); +like( + $stderr, + qr/resizing shared buffer pool is not supported on this platform/, + 'error message reports that resizing shared buffer pool is unsupported' +); + +done_testing(); diff --git a/src/test/buffermgr/t/010_stress_resize_buffer.pl b/src/test/buffermgr/t/010_stress_resize_buffer.pl new file mode 100644 index 00000000000..70b33f83e97 --- /dev/null +++ b/src/test/buffermgr/t/010_stress_resize_buffer.pl @@ -0,0 +1,34 @@ +# Copyright (c) 2025-2026, PostgreSQL Global Development Group +# +# Minimal stress test: resize shared_buffers repeatedly against regular pgbench +# workload. + +use strict; +use warnings; +use FindBin; +use lib $FindBin::RealBin; +use Test::More; +use StressUtil; + +if (!$ENV{PG_TEST_EXTRA} || $ENV{PG_TEST_EXTRA} !~ /\bbufmgr_stress\b/) +{ + plan skip_all => "bufmgr_stress not enabled in PG_TEST_EXTRA"; +} + +# A mix of small and large sizes exercises the resize logic in a variety +# of scenarios. +my @buffer_sizes = + (128, 28, 16 * 1024, 32 * 1024, 1024, 512, 16, 24, 256, 128 * 1024); + +my $stress = StressUtil->new( + buffer_sizes => \@buffer_sizes, + application_name => 'pgbench_buffer_resize_test', + pgbench_clients => 10, + pgbench_scale => 10, + pgbench_duration => 120,); + +$stress->setup; + +$stress->run; + +done_testing(); diff --git a/src/test/buffermgr/t/011_stress_drop_relation_buffers.pl b/src/test/buffermgr/t/011_stress_drop_relation_buffers.pl new file mode 100644 index 00000000000..a3053a87d69 --- /dev/null +++ b/src/test/buffermgr/t/011_stress_drop_relation_buffers.pl @@ -0,0 +1,99 @@ +# Copyright (c) 2025-2026, PostgreSQL Global Development Group +# +# Stress test execution of DropRelationBuffers(), DropRelationsAllBuffers() and +# FlushRelationsAllBuffers() concurrently with shared_buffers resizing. + +use strict; +use warnings; +use FindBin; +use lib $FindBin::RealBin; +use List::Util qw(max); +use PostgreSQL::Test::Utils; +use Test::More; +use StressUtil; + +if (!$ENV{PG_TEST_EXTRA} || $ENV{PG_TEST_EXTRA} !~ /\bbufmgr_stress\b/) +{ + plan skip_all => "bufmgr_stress not enabled in PG_TEST_EXTRA"; +} + +# A mix of small and large sizes exercises the resize logic in a variety +# of scenarios. At any time during the run buffer pool should be large enough to +# let a new backend join while other backends are performing COPY, which seems +# to pin many buffers at a time; avoid too small sizes. +my @buffer_sizes = (512, 1024, 4096, 16 * 1024, 32 * 1024, 128 * 1024); + +# The injection points verify that the buffer pool scan is exercised as expected +my $stress = StressUtil->new( + buffer_sizes => \@buffer_sizes, + application_name => 'pgbench_drop_relation_buffers_test', + injection_points => [ + 'drop-relation-buffers-scan', + 'drop-relations-all-buffers-scan', + 'flush-relations-all-buffers-scan', + ], + pgbench_clients => 10, + pgbench_scale => 10, + pgbench_duration => 120,); +$stress->setup; + +# Force execution of FlushRelationsAllBuffers() by skipping WAL logging DMLs to +# a newly created table. +my $node = $stress->node; +$node->append_conf( + 'postgresql.conf', qq{ +wal_level = minimal +max_wal_senders = 0 +wal_skip_threshold = 0 +}); +$node->restart; + +# Test specific load preparation. +# +# DropRelationBuffers() and DropRelationsAllBuffers() scan the buffer pool +# only when the size of the relation exceeds NBuffers/32. Create a +# relation larger than max(@buffer_sizes)/32 so a scan always runs. Dump +# it once so pgbench clients can COPY it back in each iteration instead of +# regenerating the data. +my $tempdir = PostgreSQL::Test::Utils::tempdir; +my $refdata_path = "$tempdir/refdata.bin"; +$node->safe_psql( + 'postgres', qq{ + CREATE UNLOGGED TABLE refdata_source AS + SELECT g AS a, repeat('x', 1900)::bytea AS b + FROM generate_series(1, 16800) g; +}); +$node->safe_psql('postgres', + "COPY refdata_source TO '$refdata_path' WITH (FORMAT binary)"); +my $max_nbuffers = max @buffer_sizes; +my $required_pages = int($max_nbuffers / 32) + 1; +my $refdata_pages = $node->safe_psql('postgres', + "SELECT (pg_relation_size('refdata_source') / current_setting('block_size')::bigint)::int" +); +cmp_ok($refdata_pages, '>', $required_pages, + "refdata_source spans more than NBuffers/32 pages at the largest tested pool size" +); + +# Workload script fed to pgbench. +# +# Each client picks table names from a disjoint numeric range so table +# names from concurrent clients do not collide. The :pgbench_id prefix +# further disambiguates across the two pgbench flavors (persistent / +# per_transaction) that StressUtil runs in parallel. +my $workload_sql = qq{ +\\set tid :pgbench_id * 1000000000 + :client_id * 100000000 + random(1, 100000000) +BEGIN; +CREATE TABLE t_:tid (a int, b bytea); +COPY t_:tid FROM '$refdata_path' WITH (FORMAT binary); +TRUNCATE t_:tid; -- hit DropRelationBuffers() +COPY t_:tid FROM '$refdata_path' WITH (FORMAT binary); +COMMIT; -- hit FlushRelationsAllBuffers() +DROP TABLE t_:tid; -- hit DropRelationsAllBuffers() +}; + +$stress->run( + workload_sql => $workload_sql, + workload_weight => 10, + default_load_weight => 1,); + +done_testing(); diff --git a/src/test/buffermgr/t/012_stress_drop_database_buffers.pl b/src/test/buffermgr/t/012_stress_drop_database_buffers.pl new file mode 100644 index 00000000000..77afb1f950d --- /dev/null +++ b/src/test/buffermgr/t/012_stress_drop_database_buffers.pl @@ -0,0 +1,82 @@ +# Copyright (c) 2025-2026, PostgreSQL Global Development Group +# +# Exercise the buffer pool scan that drops buffers belonging to a given +# database (DropDatabaseBuffers()) concurrently with shared_buffers +# resizing. + +use strict; +use warnings; +use FindBin; +use lib $FindBin::RealBin; +use List::Util qw(max); +use Test::More; +use StressUtil; + +if (!$ENV{PG_TEST_EXTRA} || $ENV{PG_TEST_EXTRA} !~ /\bbufmgr_stress\b/) +{ + plan skip_all => "bufmgr_stress not enabled in PG_TEST_EXTRA"; +} + +# A mix of small and large sizes exercises the resize logic in a variety of +# scenarios. Avoid very small sizes because concurrent CREATE DATABASE clones +# can pin more buffers than a very small pool provides, which can cause +# unrelated failures (in particular, per-transaction pgbench connections +# get "no unpinned buffers available" when opening a fresh backend). +my @buffer_sizes = (256, 512, 1024, 4096, 16 * 1024, 32 * 1024, 128 * 1024); + +my $stress = StressUtil->new( + buffer_sizes => \@buffer_sizes, + application_name => 'pgbench_drop_database_buffers_test', + injection_points => ['drop-database-buffers-scan'], + + # Concurrent CREATE DATABASE clones pin many buffers per client; raising + # this can exhaust the small pool sizes resulting in the + # pg_resize_shared_buffers() query failing with and fail with error "no + # unpinned buffers available". + pgbench_clients => 4, + pgbench_scale => 10, + pgbench_duration => 120,); + +$stress->setup; + +# Populate a template database with enough data. Size the seed table to roughly +# 1/8 of the largest buffer pool we exercise. At the largest pool it still +# occupies ~12% so that DropDatabaseBuffers() finds enough fraction of buffers +# to drop. At smaller pool sizes the template exceeds the pool, thus covering +# all the combinations of scanning buffer pool and dropping buffers. +# +# The 1900-byte payload makes sure that each row remains in the heap. +my $node = $stress->node; +my $seed_template = 'seedtemplate'; +my $max_buffer_pool = max @buffer_sizes; +my $seed_row_count = 4 * $max_buffer_pool / 8; +$node->safe_psql('postgres', "CREATE DATABASE $seed_template"); +$node->safe_psql( + $seed_template, qq{ + CREATE TABLE seedtab (a int, b bytea); + INSERT INTO seedtab + SELECT g, repeat('x', 1900)::bytea + FROM generate_series(1, $seed_row_count) g; +}); +$node->safe_psql('postgres', + "UPDATE pg_database SET datistemplate = true, datallowconn = false " + . "WHERE datname = '$seed_template'"); + +# Workload script fed to pgbench. +# +# Each client picks database names from a disjoint numeric range so +# database names from concurrent clients do not collide. The :pgbench_id +# prefix further disambiguates across the two pgbench flavors +# (persistent / per_transaction) that StressUtil runs in parallel. +my $workload_sql = qq{ +\\set tid :pgbench_id * 1000000000 + :client_id * 100000000 + random(1, 100000000) +CREATE DATABASE d_:tid TEMPLATE $seed_template; +DROP DATABASE d_:tid; +}; + +$stress->run( + workload_sql => $workload_sql, + workload_weight => 10, + default_load_weight => 1,); + +done_testing(); diff --git a/src/test/buffermgr/t/013_stress_checkpoint.pl b/src/test/buffermgr/t/013_stress_checkpoint.pl new file mode 100644 index 00000000000..8c7d6bb5841 --- /dev/null +++ b/src/test/buffermgr/t/013_stress_checkpoint.pl @@ -0,0 +1,47 @@ +# Copyright (c) 2025-2026, PostgreSQL Global Development Group +# +# Test synchronization between shared_buffers resize and CHECKPOINT. The test +# issues explicit CHECKPOINTs so that the frequency of heckpoints can be +# controlled so as increase the likelihood of concurrent checkpoint with every +# resize. + +use strict; +use warnings; +use FindBin; +use lib $FindBin::RealBin; +use Test::More; +use StressUtil; + +if (!$ENV{PG_TEST_EXTRA} || $ENV{PG_TEST_EXTRA} !~ /\bbufmgr_stress\b/) +{ + plan skip_all => "bufmgr_stress not enabled in PG_TEST_EXTRA"; +} + +# A mix of small and large sizes exercises the resize logic in a variety +# of scenarios. +my @buffer_sizes = + (128, 28, 16 * 1024, 32 * 1024, 1024, 512, 16, 24, 256, 128 * 1024); + +my $stress = StressUtil->new( + buffer_sizes => \@buffer_sizes, + application_name => 'pgbench_checkpoint_stress_test', + pgbench_clients => 10, + pgbench_scale => 10, + pgbench_duration => 120,); + +$stress->setup; + +# Disable implicit checkpoints +$stress->node->append_conf( + 'postgresql.conf', qq{ +checkpoint_timeout = 1h +max_wal_size = 100GB +}); +$stress->node->reload; + +$stress->run( + workload_sql => "CHECKPOINT;\n", + workload_weight => 1, + default_load_weight => 10,); + +done_testing(); diff --git a/src/test/buffermgr/t/014_stress_flush_relation_buffers.pl b/src/test/buffermgr/t/014_stress_flush_relation_buffers.pl new file mode 100644 index 00000000000..52e326ef391 --- /dev/null +++ b/src/test/buffermgr/t/014_stress_flush_relation_buffers.pl @@ -0,0 +1,68 @@ +# Copyright (c) 2025-2026, PostgreSQL Global Development Group +# +# Stress FlushRelationBuffers() concurrently with shared_buffers +# resizing. + +use strict; +use warnings; +use FindBin; +use lib $FindBin::RealBin; +use Test::More; +use StressUtil; + +if (!$ENV{PG_TEST_EXTRA} || $ENV{PG_TEST_EXTRA} !~ /\bbufmgr_stress\b/) +{ + plan skip_all => "bufmgr_stress not enabled in PG_TEST_EXTRA"; +} + +# A mix of small and large sizes exercises the resize logic in a variety +# of scenarios. At any time during the run the buffer pool must be large +# enough to let a new backend join while other backends are CLUSTERing a +# small table, which pins several buffers at once; avoid too small sizes. +my @buffer_sizes = (512, 1024, 4096, 16 * 1024, 32 * 1024, 128 * 1024); + +# The injection point verifies that the buffer pool scan is exercised in +# FlushRelationBuffers(). If the function stops scanning the buffer pool +# the test is useless, so we assert that it fires at least once per +# workload transaction. +my $stress = StressUtil->new( + buffer_sizes => \@buffer_sizes, + application_name => 'pgbench_flush_relation_buffers_test', + injection_points => ['flush-relation-buffers-scan'], + pgbench_clients => 10, + pgbench_scale => 10, + pgbench_duration => 120,); + +$stress->setup; + +my $node = $stress->node; +my $ts2_dir = $node->basedir . '/tblsp2'; +mkdir $ts2_dir or die "mkdir $ts2_dir: $!"; +$node->safe_psql('postgres', "CREATE TABLESPACE ts2 LOCATION '$ts2_dir'"); + +# Workload script fed to pgbench. +# +# Each client picks table names from a disjoint numeric range so table +# names from concurrent clients do not collide. The :pgbench_id prefix +# further disambiguates across the two pgbench flavors (persistent / +# per_transaction) that StressUtil runs in parallel. +my $workload_sql = qq{ +\\set tid :pgbench_id * 1000000000 + :client_id * 100000000 + random(1, 100000000) +BEGIN; +CREATE TABLE t_:tid (a int PRIMARY KEY, b bytea); +INSERT INTO t_:tid + SELECT g, repeat('x', 1900)::bytea + FROM generate_series(1, 100) g; +-- hit FlushRelationBuffers() +ALTER TABLE t_:tid SET TABLESPACE ts2; +ALTER TABLE t_:tid SET TABLESPACE pg_default; +DROP TABLE t_:tid; +COMMIT; +}; + +$stress->run( + workload_sql => $workload_sql, + workload_weight => 10, + default_load_weight => 1,); + +done_testing(); diff --git a/src/test/buffermgr/t/015_stress_pg_buffercache.pl b/src/test/buffermgr/t/015_stress_pg_buffercache.pl new file mode 100644 index 00000000000..4d823e1fb9d --- /dev/null +++ b/src/test/buffermgr/t/015_stress_pg_buffercache.pl @@ -0,0 +1,60 @@ +# Copyright (c) 2025-2026, PostgreSQL Global Development Group +# +# Stress the pg_buffercache diagnostic and monitoring functions +# concurrently with shared_buffers resizing. +# +# Destructive functions like pg_buffercache_evict_* and +# pg_buffercache_mark_dirty_* are excluded because they might cause failures +# unrelated to the test. +# +# TODO: This test fails because pg_buffercache_os_pages_internal() expects the +# buffer pool size to be constant. Fix is on the way. + +use strict; +use warnings; +use FindBin; +use lib $FindBin::RealBin; +use Test::More; +use StressUtil; + +if (!$ENV{PG_TEST_EXTRA} || $ENV{PG_TEST_EXTRA} !~ /\bbufmgr_stress\b/) +{ + plan skip_all => "bufmgr_stress not enabled in PG_TEST_EXTRA"; +} + +# A mix of small and large sizes exercises the resize logic in a variety +# of scenarios. At any time during the run buffer pool should be large +# enough to let a new backend join while other backends are scanning the +# buffer pool via pg_buffercache; avoid too small sizes. +my @buffer_sizes = (512, 1024, 4096, 16 * 1024, 32 * 1024, 128 * 1024); + +my $stress = StressUtil->new( + buffer_sizes => \@buffer_sizes, + application_name => 'pgbench_pg_buffercache_test', + pgbench_clients => 10, + pgbench_scale => 10, + pgbench_duration => 120,); +$stress->setup; + +$stress->node->safe_psql('postgres', 'CREATE EXTENSION pg_buffercache'); + +# Test specific workload. Include NUMA view only if the server supports NUMA. +my $workload_sql = qq{ +SELECT count(*) FROM pg_buffercache; +SELECT * FROM pg_buffercache_summary(); +SELECT count(*) FROM pg_buffercache_usage_counts(); +SELECT count(*) FROM pg_buffercache_os_pages; +}; +if ($stress->node->safe_psql('postgres', 'SELECT pg_numa_available()') eq 't') +{ + $workload_sql .= "SELECT count(*) FROM pg_buffercache_numa;\n"; +} + +# Use the default workload only to populate the buffer pool, but main workload +# is the pg_buffercache queries. +$stress->run( + workload_sql => $workload_sql, + workload_weight => 10, + default_load_weight => 1,); + +done_testing(); diff --git a/src/test/buffermgr/t/016_stress_pg_prewarm.pl b/src/test/buffermgr/t/016_stress_pg_prewarm.pl new file mode 100644 index 00000000000..23161f8865a --- /dev/null +++ b/src/test/buffermgr/t/016_stress_pg_prewarm.pl @@ -0,0 +1,73 @@ +# Copyright (c) 2025-2026, PostgreSQL Global Development Group +# +# Stress test the autoprewarm concurrently with shared_buffers resizing. +# +# TODO: This test fails because apw_dump_now() assumes NBuffers is +# constant. Fix is on the way. + +use strict; +use warnings; +use FindBin; +use lib $FindBin::RealBin; +use Test::More; +use StressUtil; + +if (!$ENV{PG_TEST_EXTRA} || $ENV{PG_TEST_EXTRA} !~ /\bbufmgr_stress\b/) +{ + plan skip_all => "bufmgr_stress not enabled in PG_TEST_EXTRA"; +} + +my @buffer_sizes = (512, 1024, 4096, 16 * 1024, 32 * 1024, 128 * 1024); + +my $stress = StressUtil->new( + buffer_sizes => \@buffer_sizes, + application_name => 'pgbench_pg_prewarm_test', + pgbench_clients => 10, + pgbench_scale => 10, + pgbench_duration => 120,); +$stress->setup; + +my $node = $stress->node; + +# Configure the autoprewarm background worker to run as frequently as possible. +$node->append_conf( + 'postgresql.conf', qq{ +shared_preload_libraries = 'pg_prewarm' +pg_prewarm.autoprewarm = on +pg_prewarm.autoprewarm_interval = 1s +}); +$node->restart; + +$node->safe_psql('postgres', 'CREATE EXTENSION pg_prewarm'); + +# Dump the buffer pool through additional load to increase the likelihood of it +# happening concurrently with a resize. Wrap the call in an exception block to +# swallow "dump file is being used by PID N" errors. +my $workload_sql = qq{ +DO \$\$ +BEGIN + PERFORM autoprewarm_dump_now(); +EXCEPTION WHEN OTHERS THEN + NULL; +END +\$\$; +}; + +$stress->run( + workload_sql => $workload_sql, + workload_weight => 10, + default_load_weight => 1,); + +my $dumpfile = $node->data_dir . '/autoprewarm.blocks'; +ok(-s $dumpfile, "buffer pool was dumped at least once"); + +# Restart to confirm that the dump file can be read and the buffer pool can be +# prewarmed. +my $log_offset = -s $node->logfile; +$node->restart; +$node->wait_for_log( + qr/autoprewarm successfully prewarmed \d+ of \d+ previously-loaded blocks/, + $log_offset); +pass("autoprewarm prewarmed shared buffers after restart"); + +done_testing(); diff --git a/src/test/buffermgr/t/StressUtil.pm b/src/test/buffermgr/t/StressUtil.pm new file mode 100644 index 00000000000..6a942421e09 --- /dev/null +++ b/src/test/buffermgr/t/StressUtil.pm @@ -0,0 +1,628 @@ +# Copyright (c) 2025-2026, PostgreSQL Global Development Group + +=pod + +=head1 NAME + +StressUtil - shared driver for the buffermgr shared_buffers resize stress tests + +=head1 SYNOPSIS + + use StressUtil; + + # Configure a stress test driver + my $stress = StressUtil->new( + buffer_sizes => [128, 28, ...], + application_name => 'pgbench_..._test', + pgbench_clients => 10, + pgbench_scale => 10, + pgbench_duration => 120, + injection_points => [...], # optional + ); + + # Setup the cluster and pgbench workload + $stress->setup; + + # -- per-test prep goes here; may use $stress->node -- + + # Run the stress test with optional custom workload and perform post-stress + # checks + $stress->run( + default_load_weight => 1, # required if workload_sql set + workload_sql => $sql, # optional + workload_weight => 10, # required if workload_sql set + ); + +=head1 DESCRIPTION + +StressUtil provides common routines for the shared_buffers resize stress tests. +These are the routines for setting up the cluster and pgbench database, resizing +shared_buffers in a tight loop while a pgbench workload runs concurrently, and +performing post-stress checks. + +=cut + +package StressUtil; + +use strict; +use warnings FATAL => 'all'; + +use IPC::Run; +use List::Util qw(max min shuffle); +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +=pod + +=head1 METHODS + +=over + +=item StressUtil->new(%opts) + +Construct a stress-test object which can be used to run the stress test with the +given specifications. Named options: + +=over + +=item buffer_sizes + +Array reference of shared_buffers values (in number of buffers) that the resize +loop cycles through. Required. + +=item application_name + +application_name string set on the pgbench connection; the resize loop +polls pg_stat_activity for this value to detect when pgbench has +exited. Required. + +=item injection_points + +Array reference of injection point names used to detect whether a code path is +hit during stress test. They are attached with the B action. Number of +times the notice message appears in the server error log indicates the number of +times a certain code path is hit during the stress test. Defaults to the empty +list. Used only when the build supports injection points. + +=item pgbench_clients + +Number of pgbench client connections. Split evenly across the persistent and +per-transaction pgbench processes, so must be at least B<2>. May be overridden +at run time by the C environment variable. + +=item pgbench_scale + +pgbench scale factor. May be overridden at run time by the +C environment variable. + +=item pgbench_duration + +pgbench duration in seconds. May be overridden at run time by the +C environment variable so that the test never +hits the timeout in a successful run. + +=back + +=cut + +sub new +{ + my ($class, %opts) = @_; + for my $required ( + qw(buffer_sizes + application_name + pgbench_clients + pgbench_scale + pgbench_duration)) + { + defined $opts{$required} or die "$required required"; + } + + # Env vars override the test-specified values. + my $duration = + int($ENV{PG_TEST_RESIZE_STRESS_SECONDS} // $opts{pgbench_duration}); + my $clients = + int($ENV{PG_TEST_RESIZE_STRESS_CLIENTS} // $opts{pgbench_clients}); + my $scale = + int($ENV{PG_TEST_RESIZE_STRESS_SCALE} // $opts{pgbench_scale}); + + my $timeout_cap = int(($ENV{PG_TEST_TIMEOUT_DEFAULT} // 0) * 0.8); + if ($timeout_cap > 0 && $duration > $timeout_cap) + { + note "clamping pgbench duration from $duration to $timeout_cap"; + $duration = $timeout_cap; + } + + $clients >= 2 + or die "pgbench_clients must be at least 2 " + . "(split between persistent and per-transaction pgbench)"; + + my $self = { + buffer_sizes => $opts{buffer_sizes}, + application_name => $opts{application_name}, + pgbench_clients => $clients, + pgbench_scale => $scale, + pgbench_duration => $duration, + injection_points => $opts{injection_points} // [], + node => undef, + injection_points_supported => undef, + }; + return bless $self, $class; +} + +=pod + +=item $stress->node + +Return the underlying C node. Valid only +after setup(). + +=cut + +sub node { return $_[0]->{node}; } + +=pod + +=item $stress->setup + +Create and initialize PostgreSQL cluster and other necessary objects required +for the stress test. + +=cut + +sub setup +{ + my ($self) = @_; + + my $node = PostgreSQL::Test::Cluster->new('main'); + $node->init; + $self->{node} = $node; + + my $max_buffer_pool = max @{ $self->{buffer_sizes} }; + my $initial_buffers = min @{ $self->{buffer_sizes} }; + my $ips_supported = ($ENV{enable_injection_points} // 'no') eq 'yes'; + my $use_ips = $ips_supported && @{ $self->{injection_points} }; + $self->{injection_points_supported} = $ips_supported; + + $node->append_conf( + 'postgresql.conf', qq{ +max_shared_buffers = $max_buffer_pool +shared_buffers = $initial_buffers +log_statement = none +restart_after_crash = off +}); + + # Route injection-point NOTICEs to the server log, not to the pgbench + # client which does not expect them. + if ($use_ips) + { + $node->append_conf( + 'postgresql.conf', qq{ +shared_preload_libraries = injection_points +log_min_messages = notice +client_min_messages = warning +}); + } + + $node->start; + + # Bail out if this build does not support resizable shared memory, which + # also means that resizing buffer pool is not supported. + if ($node->safe_psql('postgres', 'SHOW have_resizable_shmem') ne 'on') + { + plan skip_all => + "resizable shared memory not supported by this build"; + } + + $node->safe_psql('postgres', "CREATE EXTENSION buffermgr_test"); + $node->safe_psql('postgres', "CREATE EXTENSION amcheck"); + + if ($use_ips) + { + $node->safe_psql('postgres', "CREATE EXTENSION injection_points"); + } + + # Create a table to capture the history of resizes + $node->safe_psql( + 'postgres', qq{ +CREATE TABLE resize_log( + size int NOT NULL, + started_at timestamptz NOT NULL, + ended_at timestamptz NOT NULL, + num_tries int NOT NULL); +}); + + # Reset the bgwriter stats so we can assert that it ran during the test. + $node->safe_psql('postgres', "SELECT pg_stat_reset_shared('bgwriter')"); +} + +=pod + +=item $stress->run(%opts) + +Run the stress test. This resizes shared_buffers repeatedly while a pgbench +workload runs concurrently. Perform post-stress checks. + +Two pgbench processes run in parallel: one keeps its connections open for the +whole run (persistent), the other reconnects for every transaction. The +C count is split evenly between them. Each pgbench is passed a +distinct C variable and gets a distinct application_name so custom +workloads can build names that do not collide across the two. + +pgbench always runs its built-in tpcb-like workload; an optional custom +workload is added if requested. + +Named options: + +=over + +=item workload_sql + +Contents of a custom pgbench workload script. Optional. + +=item workload_weight + +Weight of the custom workload script relative to the built-in +tpcb-like script. Required when C is set; must not be +set otherwise. + +=item default_load_weight + +Weight of the built-in tpcb-like workload relative to the custom +workload script. Required when C is set; must not be +set otherwise. + +=back + +=cut + +sub run +{ + my ($self, %opts) = @_; + my $node = $self->{node} or die "setup() must be called first"; + my $ips = $self->{injection_points}; + my $use_ips = $self->{injection_points_supported} && @$ips; + + my @pgbench_args; + + my $workload_path; + if (defined $opts{workload_sql}) + { + my $default_weight = $opts{default_load_weight} + // die "default_load_weight required when workload_sql is set"; + my $workload_weight = $opts{workload_weight} + // die "workload_weight required when workload_sql is set"; + + push @pgbench_args, '-b', "tpcb-like\@$default_weight"; + + $workload_path = $node->basedir . '/workload.sql'; + open(my $wfh, '>', $workload_path) + or die "cannot write $workload_path: $!"; + print $wfh $opts{workload_sql}; + close($wfh); + push @pgbench_args, '-f', "$workload_path\@$workload_weight"; + } + elsif (defined $opts{default_load_weight} + || defined $opts{workload_weight}) + { + die "default_load_weight and workload_weight require workload_sql"; + } + + # Attach injection points just before starting the workload. + if ($use_ips) + { + for my $ip (@$ips) + { + $node->safe_psql('postgres', + "SELECT injection_points_attach('$ip', 'notice')"); + } + } + + my $log_offset = -s $node->logfile; + + my @procs = _start_pgbench_workloads($self, \@pgbench_args); + + _wait_for_pgbench_ready($node, $self->{application_name}); + + my $tests_completed = _run_resize_loop($self); + + _run_post_checks($self, \@procs, $log_offset, $workload_path, + $tests_completed); +} + +=back + +=cut + +# Resize the buffer pool and log the outcome to resize_log. +sub _apply_and_verify_buffer_change +{ + my ($node, $new_size) = @_; + + $node->safe_psql('postgres', + "ALTER SYSTEM SET shared_buffers = '$new_size'"); + $node->safe_psql('postgres', "SELECT pg_reload_conf()"); + + # Start a new backend so that it inherits the reloaded GUC directly from the + # postmaster. + $node->safe_psql( + 'postgres', qq{ + INSERT INTO resize_log(size, started_at, ended_at, num_tries) + SELECT $new_size, started_at, ended_at, num_tries + FROM pg_resize_shared_buffers_sql($new_size) + }); + + # A resize failure causes the test to time out, so reaching here means + # success. + ok(1, "buffer pool resized to $new_size"); +} + +# Return true if either pgbench workload is still running, false otherwise. +# +# IPC::Run's pumpable status is unreliable; check pg_stat_activity instead. +sub _pgbench_processes_active +{ + my ($node, $application_name) = @_; + + my $result = $node->safe_psql('postgres', + "SELECT count(*) FROM pg_stat_activity " + . "WHERE application_name LIKE '${application_name}%'"); + return int($result) > 0; +} + +# Wait until at least one pgbench workload has registered in +# pg_stat_activity, so the resize loop's first _pgbench_processes_active +# check does not race pgbench startup and exit immediately. +sub _wait_for_pgbench_ready +{ + my ($node, $application_name) = @_; + + $node->poll_query_until('postgres', + "SELECT count(*) >= 1 FROM pg_stat_activity " + . "WHERE application_name LIKE '${application_name}%'") + or die "timed out waiting for pgbench workloads to connect"; +} + +# Initialize pgbench, then start two pgbench workloads in parallel: one +# persistent, one with -C (per_transaction). Clients are split evenly. Each +# pgbench gets a distinct application_name and a distinct :pgbench_id script +# variable so custom workloads can build non-colliding names across the two. +# +# Returns a list of per-process hashes with keys process, stdout_ref, +# stderr_ref, pgbench_id. +sub _start_pgbench_workloads +{ + my ($self, $extra_args) = @_; + my $node = $self->{node}; + my $scale = $self->{pgbench_scale}; + my $app_name = $self->{application_name}; + my $total_clients = $self->{pgbench_clients}; + + $node->pgbench( + "--initialize --init-steps=dtpvg --scale=$scale --quiet", + 0, + [qr{^$}], + [ + qr{dropping old tables}, + qr{creating tables}, + qr{done in \d+\.\d\d s } + ], + "pgbench initialization (scale=$scale)"); + + my @flavors = ( + { pgbench_id => 1, extra => [] }, + { pgbench_id => 2, extra => ['-C'] },); + my $clients_1 = int($total_clients / 2); + my $clients_2 = $total_clients - $clients_1; + $flavors[0]->{clients} = $clients_1; + $flavors[1]->{clients} = $clients_2; + + my @procs; + for my $f (@flavors) + { + my $id = $f->{pgbench_id}; + my $flavor_app = "${app_name}_${id}"; + my ($stdin, $stdout, $stderr) = ('', '', ''); + my $process = IPC::Run::start( + [ + 'pgbench', + '-p', $node->port, + '-h', $node->host, + '-T', $self->{pgbench_duration}, + '-c', $f->{clients}, + '-D', "pgbench_id=$id", + # stop on first server crash, so that conditions at the time of + # crash are preserved for diagnosis. + '--exit-on-abort', + '--continue-on-error', + @{ $f->{extra} }, + @$extra_args, + "dbname=postgres application_name=$flavor_app" + ], + '<' => \$stdin, + '>' => \$stdout, + '2>' => \$stderr); + + ok($process, "pgbench started successfully (pgbench_id=$id)"); + push @procs, + { + process => $process, + stdout_ref => \$stdout, + stderr_ref => \$stderr, + pgbench_id => $id, + }; + } + return @procs; +} + +# Resize as many times as possible while pgbench is running, cycling +# through $self->{buffer_sizes} in a shuffled order without ever picking +# the same size twice in a row. Returns the number of resizes performed. +sub _run_resize_loop +{ + my ($self) = @_; + my $node = $self->{node}; + my $app_name = $self->{application_name}; + my $buffer_sizes = $self->{buffer_sizes}; + my @queue; + my $last_picked; + my $tests_completed = 0; + + while (_pgbench_processes_active($node, $app_name)) + { + if (!@queue) + { + @queue = shuffle(@$buffer_sizes); + if (defined $last_picked + && @queue > 1 + && $queue[0] == $last_picked) + { + @queue[0, 1] = @queue[1, 0]; + } + } + $last_picked = shift @queue; + _apply_and_verify_buffer_change($node, $last_picked); + $tests_completed++; + } + return $tests_completed; +} + +# Assert the resize loop ran through at least one full sequence and +# every size in @$buffer_sizes was picked at least once. +sub _assert_all_sizes_used +{ + my ($node, $buffer_sizes, $tests_completed) = @_; + + cmp_ok($tests_completed, '>', scalar(@$buffer_sizes), + "all buffer size transitions were tested"); + note + "completed $tests_completed buffer resize operations while pgbench was running"; + + my $ndistinct = $node->safe_psql('postgres', + "SELECT count(DISTINCT size) FROM resize_log"); + is($ndistinct, scalar(@$buffer_sizes), + "every buffer size was exercised at least once"); +} + +# Make sure that the pgbench workloads have ended and perform post-stress +# checks. +sub _run_post_checks +{ + my ($self, $procs, $log_offset, $workload_path, $tests_completed) = @_; + + my $node = $self->{node}; + my $ips = $self->{injection_points}; + + for my $p (@$procs) + { + my $id = $p->{pgbench_id}; + $p->{process}->signal('TERM'); + ok((IPC::Run::finish $p->{process}), + "pgbench finished successfully (pgbench_id=$id)"); + note("pgbench_id=$id stderr:\n" . ${ $p->{stderr_ref} }) + if ${ $p->{stderr_ref} } ne ''; + note("pgbench_id=$id stdout:\n" . ${ $p->{stdout_ref} }); + } + + _assert_all_sizes_used($node, $self->{buffer_sizes}, $tests_completed); + + # Log resize latency distribution and max retry count, for post-mortem + note $node->safe_psql( + 'postgres', + q{SELECT format('resize stats: n=%s, min=%s, avg=%s, max=%s, max_tries=%s', + count(*), + min(ended_at - started_at), + avg(ended_at - started_at), + max(ended_at - started_at), + max(num_tries)) + FROM resize_log}); + + # Checkpointer activity, for post-mortem. + note $node->safe_psql( + 'postgres', + q{SELECT format('checkpointer stats: timed=%s, requested=%s, buffers_written=%s', + num_timed, num_requested, buffers_written) + FROM pg_stat_checkpointer}); + + is( $node->safe_psql( + 'postgres', "SELECT buffers_clean > 0 FROM pg_stat_bgwriter"), + 't', + "background writer ran during resize cycle"); + + # Server error log is expected to be crash free + $node->log_check("no PANIC or SIGBUS during stress run", + $log_offset, log_unlike => [ qr/PANIC/, qr/signal 7/ ]); + + # pg_dumpall reads every table and catalog in every database; An error free + # dump indicates that the database remained non-corrupt after the stress + # run. We are not interested in the dump output, so discard it to /dev/null. + $node->command_ok( + [ 'pg_dumpall', '--no-sync', '-f', '/dev/null' ], + "pg_dumpall succeeds after stress run"); + + # pg_dumpall does not scan indexes; run bt_index_parent_check over every + # btree index to catch index-level corruption. + $node->safe_psql( + 'postgres', q{ + SELECT bt_index_parent_check(c.oid, true, true) + FROM pg_class c + JOIN pg_index i ON i.indexrelid = c.oid + WHERE c.relkind = 'i' + AND c.relam = (SELECT oid FROM pg_am WHERE amname = 'btree') + }); + ok(1, "all btree indexes verified"); + + # Verify tables + my $heap_findings = $node->safe_psql( + 'postgres', q{ + SELECT count(*) + FROM (SELECT c.oid AS rel + FROM pg_class c + WHERE c.relkind IN ('r', 'S') + AND c.relpersistence = 'p') r, + LATERAL verify_heapam(r.rel, check_toast => true) v + }); + is($heap_findings, '0', "verify_heapam found no corruption"); + + if (@$ips) + { + SKIP: + { + skip "injection points not supported by this build" + unless $self->{injection_points_supported}; + + my $workload_txns = 0; + for my $p (@$procs) + { + my $n; + if (defined $workload_path) + { + ($n) = ${ $p->{stdout_ref} } =~ + m{SQL script \d+:\s+\Q$workload_path\E.*?number of transactions actually processed:\s*(\d+)}s; + } + else + { + ($n) = ${ $p->{stdout_ref} } =~ + m{number of transactions actually processed:\s*(\d+)}; + } + ok( defined $n, + "transaction count found in pgbench stdout (pgbench_id=" + . $p->{pgbench_id} . ")"); + $workload_txns += $n if defined $n; + } + + my $log_content = slurp_file($node->logfile, $log_offset); + for my $ip (@$ips) + { + my $count = () = $log_content =~ + /notice triggered for injection point $ip\b/g; + cmp_ok($count, '>=', $workload_txns, + "injection point $ip fired at least $workload_txns times" + ); + } + } + } +} + +1;