From df141f9e4588ca45e8430d3accf55f4cfe3d3a9f Mon Sep 17 00:00:00 2001 From: Daniil Davidov Date: Sun, 23 Nov 2025 01:08:14 +0700 Subject: [PATCH v24 4/5] Tests for parallel autovacuum --- src/backend/access/heap/vacuumlazy.c | 9 + src/backend/commands/vacuumparallel.c | 22 ++ src/backend/postmaster/autovacuum.c | 38 +++ src/include/postmaster/autovacuum.h | 1 + src/test/modules/Makefile | 1 + src/test/modules/meson.build | 1 + src/test/modules/test_autovacuum/.gitignore | 2 + src/test/modules/test_autovacuum/Makefile | 28 ++ src/test/modules/test_autovacuum/meson.build | 36 +++ .../t/001_parallel_autovacuum.pl | 299 ++++++++++++++++++ .../test_autovacuum/test_autovacuum--1.0.sql | 12 + .../modules/test_autovacuum/test_autovacuum.c | 31 ++ .../test_autovacuum/test_autovacuum.control | 3 + 13 files changed, 483 insertions(+) create mode 100644 src/test/modules/test_autovacuum/.gitignore create mode 100644 src/test/modules/test_autovacuum/Makefile create mode 100644 src/test/modules/test_autovacuum/meson.build create mode 100644 src/test/modules/test_autovacuum/t/001_parallel_autovacuum.pl create mode 100644 src/test/modules/test_autovacuum/test_autovacuum--1.0.sql create mode 100644 src/test/modules/test_autovacuum/test_autovacuum.c create mode 100644 src/test/modules/test_autovacuum/test_autovacuum.control diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index 2bcdbdcfcf3..4a3b826dde5 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -152,6 +152,7 @@ #include "storage/latch.h" #include "storage/lmgr.h" #include "storage/read_stream.h" +#include "utils/injection_point.h" #include "utils/lsyscache.h" #include "utils/pg_rusage.h" #include "utils/timestamp.h" @@ -872,6 +873,14 @@ heap_vacuum_rel(Relation rel, const VacuumParams params, lazy_check_wraparound_failsafe(vacrel); dead_items_alloc(vacrel, params.nworkers); +#ifdef USE_INJECTION_POINTS + /* + * Trigger injection point, if parallel autovacuum is about to be started. + */ + if (AmAutoVacuumWorkerProcess() && ParallelVacuumIsActive(vacrel)) + INJECTION_POINT("autovacuum-start-parallel-vacuum", NULL); +#endif + /* * Call lazy_scan_heap to perform all required heap pruning, index * vacuuming, and heap vacuuming (plus related processing) diff --git a/src/backend/commands/vacuumparallel.c b/src/backend/commands/vacuumparallel.c index 13304c40b59..82618ab3ac5 100644 --- a/src/backend/commands/vacuumparallel.c +++ b/src/backend/commands/vacuumparallel.c @@ -47,6 +47,7 @@ #include "storage/bufmgr.h" #include "storage/proc.h" #include "tcop/tcopprot.h" +#include "utils/injection_point.h" #include "utils/lsyscache.h" #include "utils/rel.h" @@ -653,6 +654,14 @@ parallel_vacuum_update_shared_delay_params(void) VacuumUpdateCosts(); shared_params_generation_local = params_generation; + + elog(DEBUG2, + "parallel autovacuum worker cost params: cost_limit=%d, cost_delay=%g, cost_page_miss=%d, cost_page_dirty=%d, cost_page_hit=%d", + vacuum_cost_limit, + vacuum_cost_delay, + VacuumCostPageMiss, + VacuumCostPageDirty, + VacuumCostPageHit); } /* @@ -919,6 +928,19 @@ parallel_vacuum_process_all_indexes(ParallelVacuumState *pvs, int num_index_scan pvs->pcxt->nworkers_launched, nworkers))); } +#ifdef USE_INJECTION_POINTS + /* + * To be able to exercise whether all reserved parallel workers are being + * released anyway, allow injection points to trigger a failure at this + * point. + * + * This injection point is also used to wait until parallel workers + * finishes their part of index processing. + */ + if (nworkers > 0) + INJECTION_POINT("autovacuum-leader-before-indexes-processing", NULL); +#endif + /* Vacuum the indexes that can be processed by only leader process */ parallel_vacuum_process_unsafe_indexes(pvs); diff --git a/src/backend/postmaster/autovacuum.c b/src/backend/postmaster/autovacuum.c index cc3456e205d..1c51210883e 100644 --- a/src/backend/postmaster/autovacuum.c +++ b/src/backend/postmaster/autovacuum.c @@ -1409,7 +1409,18 @@ avl_sigusr2_handler(SIGNAL_ARGS) static void autovacuum_worker_before_shmem_exit(int code, Datum arg) { + int nreserved_old = av_nworkers_reserved; + AutoVacuumReleaseAllParallelWorkers(); + + if (nreserved_old > 0) + { + elog(DEBUG2, + ngettext("autovacuum worker before_shmem_exit: %d parallel worker has been released", + "autovacuum worker before_shmem_exit: %d parallel workers has been released", + nreserved_old - av_nworkers_reserved), + nreserved_old - av_nworkers_reserved); + } } /* @@ -2495,12 +2506,20 @@ do_autovacuum(void) } PG_CATCH(); { + int nreserved_workers = av_nworkers_reserved; + /* * Parallel autovacuum can reserve parallel workers. Make sure * that all reserved workers are released. */ AutoVacuumReleaseAllParallelWorkers(); + if (nreserved_workers > 0) + ereport(DEBUG2, + (errmsg("%d parallel autovacuum workers has been released after occured error", + nreserved_workers), + errhidecontext(true))); + /* * Abort the transaction, start a new one, and proceed with the * next table in our list. @@ -3465,6 +3484,21 @@ AutoVacuumReleaseAllParallelWorkers(void) Assert(av_nworkers_reserved == 0); } +/* + * Get number of free autovacuum parallel workers. + */ +int32 +AutoVacuumGetFreeParallelWorkers(void) +{ + int32 nfree_workers; + + LWLockAcquire(AutovacuumLock, LW_SHARED); + nfree_workers = AutoVacuumShmem->av_freeParallelWorkers; + LWLockRelease(AutovacuumLock); + + return nfree_workers; +} + /* * autovac_init * This is called at postmaster initialization. @@ -3633,5 +3667,9 @@ adjust_free_parallel_workers(int prev_max_parallel_workers) AutoVacuumShmem->av_freeParallelWorkers = Max(nfree_workers, 0); AutoVacuumShmem->av_maxParallelWorkers = autovacuum_max_parallel_workers; + elog(DEBUG2, + "number of free parallel autovacuum workers is set to %u due to config reload", + AutoVacuumShmem->av_freeParallelWorkers); + LWLockRelease(AutovacuumLock); } diff --git a/src/include/postmaster/autovacuum.h b/src/include/postmaster/autovacuum.h index f3783afb51b..d60010a43b4 100644 --- a/src/include/postmaster/autovacuum.h +++ b/src/include/postmaster/autovacuum.h @@ -66,6 +66,7 @@ extern bool AutoVacuumRequestWork(AutoVacuumWorkItemType type, extern void AutoVacuumReserveParallelWorkers(int *nworkers); extern void AutoVacuumReleaseParallelWorkers(int nworkers); extern void AutoVacuumReleaseAllParallelWorkers(void); +extern int32 AutoVacuumGetFreeParallelWorkers(void); /* shared memory stuff */ extern Size AutoVacuumShmemSize(void); diff --git a/src/test/modules/Makefile b/src/test/modules/Makefile index 4ac5c84db43..01fe0041c97 100644 --- a/src/test/modules/Makefile +++ b/src/test/modules/Makefile @@ -16,6 +16,7 @@ SUBDIRS = \ plsample \ spgist_name_ops \ test_aio \ + test_autovacuum \ test_binaryheap \ test_bitmapset \ test_bloomfilter \ diff --git a/src/test/modules/meson.build b/src/test/modules/meson.build index e2b3eef4136..9dcdc68bc87 100644 --- a/src/test/modules/meson.build +++ b/src/test/modules/meson.build @@ -16,6 +16,7 @@ subdir('plsample') subdir('spgist_name_ops') subdir('ssl_passphrase_callback') subdir('test_aio') +subdir('test_autovacuum') subdir('test_binaryheap') subdir('test_bitmapset') subdir('test_bloomfilter') diff --git a/src/test/modules/test_autovacuum/.gitignore b/src/test/modules/test_autovacuum/.gitignore new file mode 100644 index 00000000000..716e17f5a2a --- /dev/null +++ b/src/test/modules/test_autovacuum/.gitignore @@ -0,0 +1,2 @@ +# Generated subdirectories +/tmp_check/ diff --git a/src/test/modules/test_autovacuum/Makefile b/src/test/modules/test_autovacuum/Makefile new file mode 100644 index 00000000000..32254c53a5d --- /dev/null +++ b/src/test/modules/test_autovacuum/Makefile @@ -0,0 +1,28 @@ +# src/test/modules/test_autovacuum/Makefile + +PGFILEDESC = "test_autovacuum - test code for parallel autovacuum" + +MODULE_big = test_autovacuum +OBJS = \ + $(WIN32RES) \ + test_autovacuum.o + +EXTENSION = test_autovacuum +DATA = test_autovacuum--1.0.sql + +TAP_TESTS = 1 + +EXTRA_INSTALL = src/test/modules/injection_points + +export enable_injection_points + +ifdef USE_PGXS +PG_CONFIG = pg_config +PGXS := $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) +else +subdir = src/test/modules/test_autovacuum +top_builddir = ../../../.. +include $(top_builddir)/src/Makefile.global +include $(top_srcdir)/contrib/contrib-global.mk +endif diff --git a/src/test/modules/test_autovacuum/meson.build b/src/test/modules/test_autovacuum/meson.build new file mode 100644 index 00000000000..969af8bd52a --- /dev/null +++ b/src/test/modules/test_autovacuum/meson.build @@ -0,0 +1,36 @@ +# Copyright (c) 2024-2026, PostgreSQL Global Development Group + +test_autovacuum_sources = files( + 'test_autovacuum.c', +) + +if host_system == 'windows' + test_autovacuum_sources += rc_lib_gen.process(win32ver_rc, extra_args: [ + '--NAME', 'test_autovacuum', + '--FILEDESC', 'test_autovacuum - test code for parallel autovacuum',]) +endif + +test_autovacuum = shared_module('test_autovacuum', + test_autovacuum_sources, + kwargs: pg_test_mod_args, +) +test_install_libs += test_autovacuum + +test_install_data += files( + 'test_autovacuum.control', + 'test_autovacuum--1.0.sql', +) + +tests += { + 'name': 'test_autovacuum', + 'sd': meson.current_source_dir(), + 'bd': meson.current_build_dir(), + 'tap': { + 'env': { + 'enable_injection_points': get_option('injection_points') ? 'yes' : 'no', + }, + 'tests': [ + 't/001_parallel_autovacuum.pl', + ], + }, +} diff --git a/src/test/modules/test_autovacuum/t/001_parallel_autovacuum.pl b/src/test/modules/test_autovacuum/t/001_parallel_autovacuum.pl new file mode 100644 index 00000000000..7f8b5a7b4d3 --- /dev/null +++ b/src/test/modules/test_autovacuum/t/001_parallel_autovacuum.pl @@ -0,0 +1,299 @@ +# Test parallel autovacuum behavior + +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +if ($ENV{enable_injection_points} ne 'yes') +{ + plan skip_all => 'Injection points not supported by this build'; +} + +# Before each test we should disable autovacuum for 'test_autovac' table and +# generate some dead tuples in it. + +sub prepare_for_next_test +{ + my ($node, $test_number) = @_; + + $node->safe_psql('postgres', qq{ + ALTER TABLE test_autovac SET (autovacuum_enabled = false); + }); + + $node->safe_psql('postgres', qq{ + UPDATE test_autovac SET col_1 = $test_number; + }); +} + + +my $psql_out; + +my $node = PostgreSQL::Test::Cluster->new('node1'); +$node->init; + +# Configure postgres, so it can launch parallel autovacuum workers, log all +# information we are interested in and autovacuum works frequently +$node->append_conf('postgresql.conf', qq{ + max_worker_processes = 20 + max_parallel_workers = 20 + max_parallel_maintenance_workers = 20 + autovacuum_max_parallel_workers = 20 + log_min_messages = debug2 + autovacuum_naptime = '1s' + min_parallel_index_scan_size = 0 + shared_preload_libraries=test_autovacuum +}); +$node->start; + +# Check if the extension injection_points is available, as it may be +# possible that this script is run with installcheck, where the module +# would not be installed by default. +if (!$node->check_extension('injection_points')) +{ + plan skip_all => 'Extension injection_points not installed'; +} + +# Create all functions needed for testing +$node->safe_psql('postgres', qq{ + CREATE EXTENSION test_autovacuum; + CREATE EXTENSION injection_points; +}); + +my $indexes_num = 4; +my $initial_rows_num = 10_000; +my $autovacuum_parallel_workers = 2; + +# Create table and fill it with some data +$node->safe_psql('postgres', qq{ + CREATE TABLE test_autovac ( + id SERIAL PRIMARY KEY, + col_1 INTEGER, col_2 INTEGER, col_3 INTEGER, col_4 INTEGER + ) WITH (autovacuum_parallel_workers = $autovacuum_parallel_workers, + log_autovacuum_min_duration = 0); + + INSERT INTO test_autovac + SELECT + g AS col1, + g + 1 AS col2, + g + 2 AS col3, + g + 3 AS col4 + FROM generate_series(1, $initial_rows_num) AS g; +}); + +# Create specified number of b-tree indexes on the table +$node->safe_psql('postgres', qq{ + DO \$\$ + DECLARE + i INTEGER; + BEGIN + FOR i IN 1..$indexes_num LOOP + EXECUTE format('CREATE INDEX idx_col_\%s ON test_autovac (col_\%s);', i, i); + END LOOP; + END \$\$; +}); + +# Test 1 : +# Our table has enough indexes and appropriate reloptions, so autovacuum must +# be able to process it in parallel mode. Just check if it can. +# Also check whether all requested workers: +# 1) launched +# 2) correctly released + +prepare_for_next_test($node, 1); + +$node->safe_psql('postgres', qq{ + ALTER TABLE test_autovac SET (autovacuum_enabled = true); +}); + +# Wait until the parallel autovacuum on table is completed. At the same time, +# we check that the required number of parallel workers has been started. +$log_start = $node->wait_for_log( + qr/parallel workers: index vacuum: 2 planned, 2 reserved, 2 launched/, + $log_start +); + +$psql_out = $node->safe_psql('postgres', qq{ + SELECT get_parallel_autovacuum_free_workers(); +}); +is($psql_out, 20, 'All parallel workers has been released by the leader'); + +# Test 2: +# Check whether parallel autovacuum leader can propagate cost-based parameters +# to parallel workers. + +prepare_for_next_test($node, 2); + +$node->safe_psql('postgres', qq{ + SELECT injection_points_attach('autovacuum-start-parallel-vacuum', 'wait'); + SELECT injection_points_attach('autovacuum-leader-before-indexes-processing', 'wait'); + + ALTER TABLE test_autovac SET (autovacuum_parallel_workers = 1, autovacuum_enabled = true); +}); + +# Wait until parallel autovacuum is inited +$node->wait_for_event( + 'autovacuum worker', + 'autovacuum-start-parallel-vacuum' +); + +# Reload config - leader worker must update its own parameters during indexes +# processing +$node->safe_psql('postgres', qq{ + ALTER SYSTEM SET vacuum_cost_limit = 500; + ALTER SYSTEM SET vacuum_cost_page_miss = 10; + ALTER SYSTEM SET vacuum_cost_page_dirty = 10; + ALTER SYSTEM SET vacuum_cost_page_hit = 10; + SELECT pg_reload_conf(); +}); + +$node->safe_psql('postgres', qq{ + SELECT injection_points_wakeup('autovacuum-start-parallel-vacuum'); +}); + +# Now wait until parallel autovacuum leader completes processing table (i.e. +# guaranteed to call vacuum_delay_point) and launches parallel worker. +$node->wait_for_event( + 'autovacuum worker', + 'autovacuum-leader-before-indexes-processing' +); + +# Check whether parallel worker successfully updated all parameters during +# index processing +$log_start = $node->wait_for_log( + qr/parallel autovacuum worker cost params: cost_limit=500, cost_delay=2, / . + qr/cost_page_miss=10, cost_page_dirty=10, cost_page_hit=10/, + $log_start +); + +# Cleanup +$node->safe_psql('postgres', qq{ + SELECT injection_points_wakeup('autovacuum-leader-before-indexes-processing'); + + SELECT injection_points_detach('autovacuum-start-parallel-vacuum'); + SELECT injection_points_detach('autovacuum-leader-before-indexes-processing'); + + ALTER TABLE test_autovac SET (autovacuum_parallel_workers = $autovacuum_parallel_workers); +}); + +# Test 3: +# Test adjustment of free parallel workers number when changing +# autovacuum_max_parallel_workers parameter + +prepare_for_next_test($node, 4); + +$node->safe_psql('postgres', qq{ + SELECT injection_points_attach('autovacuum-leader-before-indexes-processing', 'wait'); + ALTER TABLE test_autovac SET (autovacuum_enabled = true); +}); + +$node->wait_for_event( + 'autovacuum worker', + 'autovacuum-leader-before-indexes-processing' +); + +$node->safe_psql('postgres', qq{ + ALTER SYSTEM SET autovacuum_max_parallel_workers = 1; + SELECT pg_reload_conf(); +}); + +# Since 2 parallel workers already launched and will be released in the future, +# we are expecting that : +# 1) number of free workers will be '0' after config reload +# 2) number of free workers will be '1' after releasing workers + +# Check statement (1) +$log_start = $node->wait_for_log( + qr/number of free parallel autovacuum workers is set to 0 due to config reload/, + $log_start +); + +$node->safe_psql('postgres', qq{ + SELECT injection_points_wakeup('autovacuum-leader-before-indexes-processing'); +}); + +# Wait until the end of parallel processing +$log_start = $node->wait_for_log( + qr/parallel workers: index vacuum: 2 planned, 2 reserved, 2 launched/, + $log_start +); + +# Check statement (2) +$psql_out = $node->safe_psql('postgres', qq{ + SELECT get_parallel_autovacuum_free_workers(); +}); +is($psql_out, 1, 'Number of free parallel workers is consistent'); + +# Cleanup +$node->safe_psql('postgres', qq{ + SELECT injection_points_detach('autovacuum-leader-before-indexes-processing'); + ALTER SYSTEM SET autovacuum_max_parallel_workers = 10; + SELECT pg_reload_conf(); +}); + +# Test 4: +# We want parallel autovacuum workers to be released even if leader gets an +# error. At first, simulate situation, when leader exits due to an ERROR. + +prepare_for_next_test($node, 4); + +$node->safe_psql('postgres', qq{ + SELECT injection_points_attach('autovacuum-leader-before-indexes-processing', 'error'); + ALTER TABLE test_autovac SET (autovacuum_enabled = true); +}); + +$log_start = $node->wait_for_log( + qr/error triggered for injection point / . + qr/autovacuum-leader-before-indexes-processing/, + $log_start +); + +$log_start = $node->wait_for_log( + qr/2 parallel autovacuum workers has been released after occured error/, + $log_start +); + +# Cleanup +$node->safe_psql('postgres', qq{ + SELECT injection_points_detach('autovacuum-leader-before-indexes-processing'); +}); + +# Test 5: +# Same as above test, but simulate situation, when leader exits due to FATAL. + +prepare_for_next_test($node, 5); + +$node->safe_psql('postgres', qq{ + SELECT injection_points_attach('autovacuum-leader-before-indexes-processing', 'wait'); + ALTER TABLE test_autovac SET (autovacuum_enabled = true); +}); + +# Wait until parallel workers are reserved autovacuum and kill the leader +$node->wait_for_event( + 'autovacuum worker', + 'autovacuum-leader-before-indexes-processing' +); + +my $av_pid = $node->safe_psql('postgres', qq{ + SELECT pid FROM pg_stat_activity + WHERE backend_type = 'autovacuum worker' + AND wait_event = 'autovacuum-leader-before-indexes-processing' + LIMIT 1; +}); + +$node->safe_psql('postgres', qq{ + SELECT pg_terminate_backend('$av_pid'); +}); + +$log_start = $node->wait_for_log( + qr/autovacuum worker before_shmem_exit: 2 parallel workers has been released/, + $log_start +); + +# Cleanup +$node->safe_psql('postgres', qq{ + SELECT injection_points_detach('autovacuum-leader-before-indexes-processing'); +}); + +$node->stop; +done_testing(); diff --git a/src/test/modules/test_autovacuum/test_autovacuum--1.0.sql b/src/test/modules/test_autovacuum/test_autovacuum--1.0.sql new file mode 100644 index 00000000000..e5646e0def5 --- /dev/null +++ b/src/test/modules/test_autovacuum/test_autovacuum--1.0.sql @@ -0,0 +1,12 @@ +/* src/test/modules/test_autovacuum/test_autovacuum--1.0.sql */ + +-- complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION test_autovacuum" to load this file. \quit + +/* + * Functions for expecting shared autovacuum state + */ + +CREATE FUNCTION get_parallel_autovacuum_free_workers() +RETURNS INTEGER STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; diff --git a/src/test/modules/test_autovacuum/test_autovacuum.c b/src/test/modules/test_autovacuum/test_autovacuum.c new file mode 100644 index 00000000000..dd5c839e851 --- /dev/null +++ b/src/test/modules/test_autovacuum/test_autovacuum.c @@ -0,0 +1,31 @@ +/*------------------------------------------------------------------------- + * + * test_autovacuum.c + * Helpers to write tests for parallel autovacuum + * + * Copyright (c) 2020-2026, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/test/modules/test_autovacuum/test_autovacuum.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "fmgr.h" +#include "postmaster/autovacuum.h" +#include "utils/injection_point.h" + +PG_MODULE_MAGIC; + +PG_FUNCTION_INFO_V1(get_parallel_autovacuum_free_workers); +Datum +get_parallel_autovacuum_free_workers(PG_FUNCTION_ARGS) +{ + int32 nfree_workers; + + nfree_workers = AutoVacuumGetFreeParallelWorkers(); + + PG_RETURN_INT32(nfree_workers); +} diff --git a/src/test/modules/test_autovacuum/test_autovacuum.control b/src/test/modules/test_autovacuum/test_autovacuum.control new file mode 100644 index 00000000000..1b7fad258f0 --- /dev/null +++ b/src/test/modules/test_autovacuum/test_autovacuum.control @@ -0,0 +1,3 @@ +comment = 'Test code for parallel autovacuum' +default_version = '1.0' +module_pathname = '$libdir/test_autovacuum' -- 2.43.0