From ae8fe640c7dc7b62a0f2a7cd40b00714b730af53 Mon Sep 17 00:00:00 2001 From: Colin McDonnell Date: Thu, 19 Nov 2026 07:33:52 +0000 Subject: [PATCH] Retry a failed worker start in the emergency autovacuum launcher When autovacuum is off, the launcher that the postmaster starts to prevent wraparound requests one worker and exits. Only the launcher's main loop retries a start that the postmaster reports as failed, or gives back the WorkerInfo of a worker that never started, so a failed start in emergency mode left the WorkerInfo in av_startingWorker for good. Each later emergency launcher then found no free worker slot, and nothing was vacuumed to prevent wraparound until a restart. Since autovacuum workers got a pool of exactly autovacuum_worker_slots postmaster child slots, the start can fail without any fork() failure: the worker that just finished asks for the next launcher before it exits, and that launcher can take the WorkerInfo the worker gave back while the postmaster still holds the worker's child slot. With autovacuum_worker_slots = 1 that happens after almost every database. Make the emergency launcher wait for its worker to start before exiting, retrying a failed start and timing out a stuck one as the main loop does. --- src/backend/postmaster/autovacuum.c | 111 ++++++++++++++++++++++++---- 1 file changed, 95 insertions(+), 16 deletions(-) diff --git a/src/backend/postmaster/autovacuum.c b/src/backend/postmaster/autovacuum.c index c4c44c9..da763fb 100644 --- a/src/backend/postmaster/autovacuum.c +++ b/src/backend/postmaster/autovacuum.c @@ -360,6 +360,8 @@ avl_dbase *avl_dbase_array; static WorkerInfo MyWorkerInfo = NULL; static Oid do_start_worker(void); +static void wait_for_emergency_worker(void); +static void cancel_starting_worker(void); static void ProcessAutoVacLauncherInterrupts(void); pg_noreturn static void AutoVacLauncherShutdown(void); static void launcher_determine_sleep(bool canlaunch, bool recursing, @@ -585,12 +587,12 @@ AutoVacLauncherMain(const void *startup_data, size_t startup_data_len) /* * In emergency mode, just start a worker (unless shutdown was requested) - * and go away. + * and go away once it is running. */ if (!AutoVacuumingActive()) { - if (!ShutdownRequestPending) - do_start_worker(); + if (!ShutdownRequestPending && OidIsValid(do_start_worker())) + wait_for_emergency_worker(); proc_exit(0); /* done */ } @@ -713,19 +715,7 @@ AutoVacLauncherMain(const void *startup_data, size_t startup_data_len) * recheck the launch time). */ if (AutoVacuumShmem->av_startingWorker != NULL) - { - worker = AutoVacuumShmem->av_startingWorker; - worker->wi_dboid = InvalidOid; - worker->wi_tableoid = InvalidOid; - worker->wi_sharedrel = false; - worker->wi_proc = NULL; - worker->wi_launchtime = 0; - dclist_push_head(&AutoVacuumShmem->av_freeWorkers, - &worker->wi_links); - AutoVacuumShmem->av_startingWorker = NULL; - ereport(WARNING, - errmsg("autovacuum worker took too long to start; canceled")); - } + cancel_starting_worker(); } else can_launch = false; @@ -1381,6 +1371,95 @@ launch_worker(TimestampTz now) } } +/* + * wait_for_emergency_worker + * + * In emergency mode, wait for the worker that do_start_worker requested to + * take its WorkerInfo out of av_startingWorker, retrying the request if the + * postmaster reports that it could not start the worker. This does what the + * main loop does for a normal launcher. Without it, a failed start would + * leave the WorkerInfo in av_startingWorker, where nothing reclaims it while + * autovacuum is off: each later emergency launcher would find no free worker + * slot, and no database would be vacuumed to prevent wraparound until a + * restart. + */ +static void +wait_for_emergency_worker(void) +{ + int waittime = Min(autovacuum_naptime, 60) * 1000; + + for (;;) + { + WorkerInfo worker; + + LWLockAcquire(AutovacuumLock, LW_EXCLUSIVE); + worker = AutoVacuumShmem->av_startingWorker; + if (worker == NULL) + { + /* the worker is running */ + LWLockRelease(AutovacuumLock); + return; + } + if (TimestampDifferenceExceeds(worker->wi_launchtime, + GetCurrentTimestamp(), waittime)) + { + cancel_starting_worker(); + LWLockRelease(AutovacuumLock); + return; + } + LWLockRelease(AutovacuumLock); + + if (AutoVacuumShmem->av_signal[AutoVacForkFailed]) + { + /* as in the main loop, sleep for a little while and ask again */ + AutoVacuumShmem->av_signal[AutoVacForkFailed] = false; + pg_usleep(1000000L); /* 1s */ + SendPostmasterSignal(PMSIGNAL_START_AUTOVAC_WORKER); + continue; + } + + /* + * The worker signals us once it has started, and the postmaster + * signals us if it could not start it. + */ + (void) WaitLatch(MyLatch, + WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH, + 1000L, + WAIT_EVENT_AUTOVACUUM_MAIN); + ResetLatch(MyLatch); + + if (ShutdownRequestPending) + return; + if (ProcSignalBarrierPending) + ProcessProcSignalBarrier(); + if (LogMemoryContextPending) + ProcessLogMemoryContextInterrupt(); + ProcessCatchupInterrupt(); + } +} + +/* + * cancel_starting_worker + * + * Give the WorkerInfo of a worker that never started back to the free list. + * The caller must hold AutovacuumLock exclusively. + */ +static void +cancel_starting_worker(void) +{ + WorkerInfo worker = AutoVacuumShmem->av_startingWorker; + + worker->wi_dboid = InvalidOid; + worker->wi_tableoid = InvalidOid; + worker->wi_sharedrel = false; + worker->wi_proc = NULL; + worker->wi_launchtime = 0; + dclist_push_head(&AutoVacuumShmem->av_freeWorkers, &worker->wi_links); + AutoVacuumShmem->av_startingWorker = NULL; + ereport(WARNING, + errmsg("autovacuum worker took too long to start; canceled")); +} + /* * Called from postmaster to signal a failure to fork a process to become * worker. The postmaster should kill(SIGUSR2) the launcher shortly -- 2.43.0