From b04988c28d71b02121e1f1d189f1217df9f6b261 Mon Sep 17 00:00:00 2001 From: Colin McDonnell Date: Fri, 20 Nov 2026 17:29:38 +0000 Subject: [PATCH v2] Retry a failed worker start in the emergency autovacuum launcher When autovacuum is off, the launcher that the postmaster starts to prevent wraparound requests one worker and exits. Only the launcher's main loop retries a start that the postmaster reports as failed, or gives back the WorkerInfo of a worker that never started, so a failed start in emergency mode left the WorkerInfo in av_startingWorker for good. Each later emergency launcher then found no free worker slot, and nothing was vacuumed to prevent wraparound until a restart. Since autovacuum workers got a pool of exactly autovacuum_worker_slots postmaster child slots, the start can fail without any fork() failure: the worker that just finished asks for the next launcher before it exits, and that launcher can take the WorkerInfo the worker gave back while the postmaster still holds the worker's child slot. With autovacuum_worker_slots = 1 that happens after almost every database. Make the emergency launcher wait for its worker to start before exiting, retrying a failed start and timing out a stuck one as the main loop does. If a WorkerInfo is already reserved when it gets there, as after an error, it waits for that one instead of reserving another. Backpatch-through: 18 --- src/backend/postmaster/autovacuum.c | 136 +++++++++++++++++++++++++--- 1 file changed, 121 insertions(+), 15 deletions(-) diff --git a/src/backend/postmaster/autovacuum.c b/src/backend/postmaster/autovacuum.c index 29f8493..9c5d269 100644 --- a/src/backend/postmaster/autovacuum.c +++ b/src/backend/postmaster/autovacuum.c @@ -360,6 +360,8 @@ avl_dbase *avl_dbase_array; static WorkerInfo MyWorkerInfo = NULL; static Oid do_start_worker(void); +static void do_start_emergency_worker(void); +static void cancel_starting_worker(void); static void ProcessAutoVacLauncherInterrupts(void); pg_noreturn static void AutoVacLauncherShutdown(void); static void launcher_determine_sleep(bool canlaunch, bool recursing, @@ -585,12 +587,12 @@ AutoVacLauncherMain(const void *startup_data, size_t startup_data_len) /* * In emergency mode, just start a worker (unless shutdown was requested) - * and go away. + * and go away once it is running. */ if (!AutoVacuumingActive()) { if (!ShutdownRequestPending) - do_start_worker(); + do_start_emergency_worker(); proc_exit(0); /* done */ } @@ -713,19 +715,7 @@ AutoVacLauncherMain(const void *startup_data, size_t startup_data_len) * recheck the launch time). */ if (AutoVacuumShmem->av_startingWorker != NULL) - { - worker = AutoVacuumShmem->av_startingWorker; - worker->wi_dboid = InvalidOid; - worker->wi_tableoid = InvalidOid; - worker->wi_sharedrel = false; - worker->wi_proc = NULL; - worker->wi_launchtime = 0; - dclist_push_head(&AutoVacuumShmem->av_freeWorkers, - &worker->wi_links); - AutoVacuumShmem->av_startingWorker = NULL; - ereport(WARNING, - errmsg("autovacuum worker took too long to start; canceled")); - } + cancel_starting_worker(); } else can_launch = false; @@ -1323,6 +1313,122 @@ do_start_worker(void) return retval; } +/* + * do_start_emergency_worker + * + * Start a worker from an emergency launcher, and wait until it has taken its + * WorkerInfo out of av_startingWorker. A normal launcher's main loop retries + * a start that the postmaster reports as failed, and gives back the WorkerInfo + * of a worker that never starts. An emergency launcher exits after this, so + * it does both here. Otherwise a failed start would leave the WorkerInfo in + * av_startingWorker, where nothing gives it back while autovacuum is off, and + * every later emergency launcher would find no free worker slot. + * + * If a WorkerInfo is already reserved, as when an error brought us back here + * after reserving it, wait for that one instead of reserving another. + */ +static void +do_start_emergency_worker(void) +{ + int waittime = Min(autovacuum_naptime, 60) * 1000; + bool have_reserved; + + LWLockAcquire(AutovacuumLock, LW_SHARED); + have_reserved = (AutoVacuumShmem->av_startingWorker != NULL); + LWLockRelease(AutovacuumLock); + + if (!have_reserved) + { + /* with nothing reserved, a report of a failed start is stale */ + AutoVacuumShmem->av_signal[AutoVacForkFailed] = false; + + if (!OidIsValid(do_start_worker())) + return; + } + + for (;;) + { + WorkerInfo reserved; + + if (ShutdownRequestPending) + return; + if (ProcSignalBarrierPending) + ProcessProcSignalBarrier(); + if (LogMemoryContextPending) + ProcessLogMemoryContextInterrupt(); + ProcessCatchupInterrupt(); + + LWLockAcquire(AutovacuumLock, LW_EXCLUSIVE); + reserved = AutoVacuumShmem->av_startingWorker; + if (reserved == NULL) + { + /* the worker is running */ + LWLockRelease(AutovacuumLock); + return; + } + if (TimestampDifferenceExceeds(reserved->wi_launchtime, + GetCurrentTimestamp(), waittime)) + { + cancel_starting_worker(); + LWLockRelease(AutovacuumLock); + return; + } + LWLockRelease(AutovacuumLock); + + if (AutoVacuumShmem->av_signal[AutoVacForkFailed]) + { + /* + * As in the main loop, sleep for a little while, then ask again + * if the worker still hasn't shown up. + */ + AutoVacuumShmem->av_signal[AutoVacForkFailed] = false; + pg_usleep(1000000L); /* 1s */ + + LWLockAcquire(AutovacuumLock, LW_SHARED); + have_reserved = (AutoVacuumShmem->av_startingWorker != NULL); + LWLockRelease(AutovacuumLock); + + if (have_reserved) + SendPostmasterSignal(PMSIGNAL_START_AUTOVAC_WORKER); + continue; + } + + /* + * The worker signals us once it has started, and the postmaster + * signals us if it could not start it. + */ + (void) WaitLatch(MyLatch, + WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH, + 1000L, + WAIT_EVENT_AUTOVACUUM_MAIN); + ResetLatch(MyLatch); + } +} + +/* + * cancel_starting_worker + * + * Give the WorkerInfo of a worker that never started back to the free list. + * The caller must hold AutovacuumLock exclusively. + */ +static void +cancel_starting_worker(void) +{ + WorkerInfo worker = AutoVacuumShmem->av_startingWorker; + + Assert(LWLockHeldByMeInMode(AutovacuumLock, LW_EXCLUSIVE)); + + worker->wi_dboid = InvalidOid; + worker->wi_tableoid = InvalidOid; + worker->wi_sharedrel = false; + worker->wi_proc = NULL; + worker->wi_launchtime = 0; + dclist_push_head(&AutoVacuumShmem->av_freeWorkers, &worker->wi_links); + AutoVacuumShmem->av_startingWorker = NULL; + ereport(WARNING, + errmsg("autovacuum worker took too long to start; canceled")); +} + /* * launch_worker * -- 2.43.0