From 68e3005c14ba116e372a1724dad079914108ab2d Mon Sep 17 00:00:00 2001 From: Nathan Bossart Date: Wed, 5 Jan 2022 19:24:22 +0000 Subject: [PATCH v6 1/6] Introduce custodian. The custodian process is a new auxiliary process that is intended to help offload tasks could otherwise delay startup and checkpointing. This commit simply adds the new process; it does not yet do anything useful. --- src/backend/postmaster/Makefile | 1 + src/backend/postmaster/auxprocess.c | 8 + src/backend/postmaster/custodian.c | 252 ++++++++++++++++++++++++ src/backend/postmaster/postmaster.c | 44 ++++- src/backend/storage/ipc/ipci.c | 3 + src/backend/storage/lmgr/proc.c | 1 + src/backend/utils/activity/wait_event.c | 3 + src/backend/utils/init/miscinit.c | 3 + src/include/miscadmin.h | 3 + src/include/postmaster/custodian.h | 20 ++ src/include/storage/proc.h | 11 +- src/include/utils/wait_event.h | 1 + 12 files changed, 345 insertions(+), 5 deletions(-) create mode 100644 src/backend/postmaster/custodian.c create mode 100644 src/include/postmaster/custodian.h diff --git a/src/backend/postmaster/Makefile b/src/backend/postmaster/Makefile index 3a794e54d6..e1e1d1123f 100644 --- a/src/backend/postmaster/Makefile +++ b/src/backend/postmaster/Makefile @@ -18,6 +18,7 @@ OBJS = \ bgworker.o \ bgwriter.o \ checkpointer.o \ + custodian.o \ fork_process.o \ interrupt.o \ pgarch.o \ diff --git a/src/backend/postmaster/auxprocess.c b/src/backend/postmaster/auxprocess.c index 39ac4490db..620a0b1bae 100644 --- a/src/backend/postmaster/auxprocess.c +++ b/src/backend/postmaster/auxprocess.c @@ -20,6 +20,7 @@ #include "pgstat.h" #include "postmaster/auxprocess.h" #include "postmaster/bgwriter.h" +#include "postmaster/custodian.h" #include "postmaster/startup.h" #include "postmaster/walwriter.h" #include "replication/walreceiver.h" @@ -74,6 +75,9 @@ AuxiliaryProcessMain(AuxProcType auxtype) case CheckpointerProcess: MyBackendType = B_CHECKPOINTER; break; + case CustodianProcess: + MyBackendType = B_CUSTODIAN; + break; case WalWriterProcess: MyBackendType = B_WAL_WRITER; break; @@ -153,6 +157,10 @@ AuxiliaryProcessMain(AuxProcType auxtype) CheckpointerMain(); proc_exit(1); + case CustodianProcess: + CustodianMain(); + proc_exit(1); + case WalWriterProcess: WalWriterMain(); proc_exit(1); diff --git a/src/backend/postmaster/custodian.c b/src/backend/postmaster/custodian.c new file mode 100644 index 0000000000..db00282658 --- /dev/null +++ b/src/backend/postmaster/custodian.c @@ -0,0 +1,252 @@ +/*------------------------------------------------------------------------- + * + * custodian.c + * + * The custodian process handles a variety of non-critical tasks that might + * otherwise delay startup, checkpointing, etc. Offloaded tasks should not + * be synchronous (e.g., checkpointing shouldn't wait for the custodian to + * complete a task before proceeding). Also, ensure that any offloaded + * tasks are either not required during single-user mode or are performed + * separately during single-user mode. + * + * The custodian is not an essential process and can shutdown quickly when + * requested. The custodian will wake up approximately once every 5 + * minutes to perform its tasks, but backends can (and should) set its + * latch to wake it up sooner. + * + * + * Copyright (c) 2022, PostgreSQL Global Development Group + * + * + * IDENTIFICATION + * src/backend/postmaster/custodian.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include + +#include "libpq/pqsignal.h" +#include "pgstat.h" +#include "postmaster/custodian.h" +#include "postmaster/interrupt.h" +#include "storage/bufmgr.h" +#include "storage/condition_variable.h" +#include "storage/fd.h" +#include "storage/proc.h" +#include "storage/procsignal.h" +#include "storage/smgr.h" +#include "utils/memutils.h" + +#define CUSTODIAN_TIMEOUT_S (300) /* 5 minutes */ + +typedef struct +{ + slock_t cust_lck; + int cust_flags; +} CustodianShmemStruct; + +static CustodianShmemStruct *CustodianShmem; + +/* + * Main entry point for custodian process + * + * This is invoked from AuxiliaryProcessMain, which has already created the + * basic execution environment, but not enabled signals yet. + */ +void +CustodianMain(void) +{ + sigjmp_buf local_sigjmp_buf; + MemoryContext custodian_context; + + /* + * Properly accept or ignore signals that might be sent to us. + */ + pqsignal(SIGHUP, SignalHandlerForConfigReload); + pqsignal(SIGINT, SignalHandlerForShutdownRequest); + pqsignal(SIGTERM, SignalHandlerForShutdownRequest); + /* SIGQUIT handler was already set up by InitPostmasterChild */ + pqsignal(SIGALRM, SIG_IGN); + pqsignal(SIGPIPE, SIG_IGN); + pqsignal(SIGUSR1, procsignal_sigusr1_handler); + pqsignal(SIGUSR2, SIG_IGN); + + /* + * Reset some signals that are accepted by postmaster but not here + */ + pqsignal(SIGCHLD, SIG_DFL); + + /* + * Create a memory context that we will do all our work in. We do this so + * that we can reset the context during error recovery and thereby avoid + * possible memory leaks. + */ + custodian_context = AllocSetContextCreate(TopMemoryContext, + "Custodian", + ALLOCSET_DEFAULT_SIZES); + MemoryContextSwitchTo(custodian_context); + + /* + * If an exception is encountered, processing resumes here. As with other + * auxiliary processes, we cannot use PG_TRY because this is the bottom of + * the exception stack. + */ + if (sigsetjmp(local_sigjmp_buf, 1) != 0) + { + /* Since not using PG_TRY, must reset error stack by hand */ + error_context_stack = NULL; + + /* Prevent interrupts while cleaning up */ + HOLD_INTERRUPTS(); + + /* Report the error to the server log */ + EmitErrorReport(); + + /* + * These operations are really just a minimal subset of + * AbortTransaction(). We don't have very many resources to worry + * about. + */ + LWLockReleaseAll(); + ConditionVariableCancelSleep(); + AbortBufferIO(); + UnlockBuffers(); + ReleaseAuxProcessResources(false); + AtEOXact_Buffers(false); + AtEOXact_SMgr(); + AtEOXact_Files(false); + AtEOXact_HashTables(false); + + /* + * Now return to normal top-level context and clear ErrorContext for + * next time. + */ + MemoryContextSwitchTo(custodian_context); + FlushErrorState(); + + /* Flush any leaked data in the top-level context */ + MemoryContextResetAndDeleteChildren(custodian_context); + + /* Now we can allow interrupts again */ + RESUME_INTERRUPTS(); + + /* + * Sleep at least 1 second after any error. A write error is likely + * to be repeated, and we don't want to be filling the error logs as + * fast as we can. + */ + pg_usleep(1000000L); + + /* + * Close all open files after any error. This is helpful on Windows, + * where holding deleted files open causes various strange errors. + * It's not clear we need it elsewhere, but shouldn't hurt. + */ + smgrcloseall(); + + /* Report wait end here, when there is no further possibility of wait */ + pgstat_report_wait_end(); + } + + /* We can now handle ereport(ERROR) */ + PG_exception_stack = &local_sigjmp_buf; + + /* + * Unblock signals (they were blocked when the postmaster forked us) + */ + PG_SETMASK(&UnBlockSig); + + /* + * Advertise out latch that backends can use to wake us up while we're + * sleeping. + */ + ProcGlobal->custodianLatch = &MyProc->procLatch; + + /* + * On startup and after an exception, we won't know exactly what tasks need + * to be performed, so request all of them. + */ + SpinLockAcquire(&CustodianShmem->cust_lck); + CustodianShmem->cust_flags = 0xFFFFFFFF; + SpinLockRelease(&CustodianShmem->cust_lck); + + /* + * Loop forever + */ + for (;;) + { + pg_time_t start_time; + pg_time_t end_time; + int elapsed_secs; + int cur_timeout; + int flags; + + /* Clear any already-pending wakeups */ + ResetLatch(MyLatch); + + HandleMainLoopInterrupts(); + + start_time = (pg_time_t) time(NULL); + + /* Obtain requested tasks */ + SpinLockAcquire(&CustodianShmem->cust_lck); + flags = CustodianShmem->cust_flags; + CustodianShmem->cust_flags = 0; + SpinLockRelease(&CustodianShmem->cust_lck); + + /* TODO: offloaded tasks go here */ + + /* Calculate how long to sleep */ + end_time = (pg_time_t) time(NULL); + elapsed_secs = end_time - start_time; + if (elapsed_secs >= CUSTODIAN_TIMEOUT_S) + continue; /* no sleep for us */ + cur_timeout = CUSTODIAN_TIMEOUT_S - elapsed_secs; + + (void) WaitLatch(MyLatch, + WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH, + cur_timeout * 1000L /* convert to ms */ , + WAIT_EVENT_CUSTODIAN_MAIN); + } + + pg_unreachable(); +} + +Size +CustodianShmemSize(void) +{ + return sizeof(CustodianShmemStruct); +} + +void +CustodianShmemInit(void) +{ + Size size = CustodianShmemSize(); + bool found; + + CustodianShmem = (CustodianShmemStruct *) + ShmemInitStruct("Custodian Data", size, &found); + + if (!found) + { + memset(CustodianShmem, 0, size); + SpinLockInit(&CustodianShmem->cust_lck); + } +} + +/* + * RequestCustodian + * Called to request a custodian task. + */ +void +RequestCustodian(int flags) +{ + SpinLockAcquire(&CustodianShmem->cust_lck); + CustodianShmem->cust_flags |= flags; + SpinLockRelease(&CustodianShmem->cust_lck); + + if (ProcGlobal->custodianLatch) + SetLatch(ProcGlobal->custodianLatch); +} diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c index dde4bc25b1..5162ee9dec 100644 --- a/src/backend/postmaster/postmaster.c +++ b/src/backend/postmaster/postmaster.c @@ -251,6 +251,7 @@ bool remove_temp_files_after_crash = true; static pid_t StartupPID = 0, BgWriterPID = 0, CheckpointerPID = 0, + CustodianPID = 0, WalWriterPID = 0, WalReceiverPID = 0, AutoVacPID = 0, @@ -548,6 +549,7 @@ static void ShmemBackendArrayRemove(Backend *bn); #define StartArchiver() StartChildProcess(ArchiverProcess) #define StartBackgroundWriter() StartChildProcess(BgWriterProcess) #define StartCheckpointer() StartChildProcess(CheckpointerProcess) +#define StartCustodian() StartChildProcess(CustodianProcess) #define StartWalWriter() StartChildProcess(WalWriterProcess) #define StartWalReceiver() StartChildProcess(WalReceiverProcess) @@ -1823,13 +1825,16 @@ ServerLoop(void) /* * If no background writer process is running, and we are not in a * state that prevents it, start one. It doesn't matter if this - * fails, we'll just try again later. Likewise for the checkpointer. + * fails, we'll just try again later. Likewise for the checkpointer + * and custodian. */ if (pmState == PM_RUN || pmState == PM_RECOVERY || pmState == PM_HOT_STANDBY || pmState == PM_STARTUP) { if (CheckpointerPID == 0) CheckpointerPID = StartCheckpointer(); + if (CustodianPID == 0) + CustodianPID = StartCustodian(); if (BgWriterPID == 0) BgWriterPID = StartBackgroundWriter(); } @@ -2769,6 +2774,8 @@ SIGHUP_handler(SIGNAL_ARGS) signal_child(BgWriterPID, SIGHUP); if (CheckpointerPID != 0) signal_child(CheckpointerPID, SIGHUP); + if (CustodianPID != 0) + signal_child(CustodianPID, SIGHUP); if (WalWriterPID != 0) signal_child(WalWriterPID, SIGHUP); if (WalReceiverPID != 0) @@ -3089,6 +3096,8 @@ reaper(SIGNAL_ARGS) */ if (CheckpointerPID == 0) CheckpointerPID = StartCheckpointer(); + if (CustodianPID == 0) + CustodianPID = StartCustodian(); if (BgWriterPID == 0) BgWriterPID = StartBackgroundWriter(); if (WalWriterPID == 0) @@ -3182,6 +3191,20 @@ reaper(SIGNAL_ARGS) continue; } + /* + * Was it the custodian? Normal exit can be ignored; we'll start a + * new one at the next iteration of the postmaster's main loop, if + * necessary. Any other exit condition is treated as a crash. + */ + if (pid == CustodianPID) + { + CustodianPID = 0; + if (!EXIT_STATUS_0(exitstatus)) + HandleChildCrash(pid, exitstatus, + _("custodian process")); + continue; + } + /* * Was it the wal writer? Normal exit can be ignored; we'll start a * new one at the next iteration of the postmaster's main loop, if @@ -3639,6 +3662,18 @@ HandleChildCrash(int pid, int exitstatus, const char *procname) signal_child(CheckpointerPID, (SendStop ? SIGSTOP : SIGQUIT)); } + /* Take care of the custodian too */ + if (pid == CustodianPID) + CustodianPID = 0; + else if (CustodianPID != 0 && take_action) + { + ereport(DEBUG2, + (errmsg_internal("sending %s to process %d", + (SendStop ? "SIGSTOP" : "SIGQUIT"), + (int) CustodianPID))); + signal_child(CustodianPID, (SendStop ? SIGSTOP : SIGQUIT)); + } + /* Take care of the walwriter too */ if (pid == WalWriterPID) WalWriterPID = 0; @@ -3816,6 +3851,9 @@ PostmasterStateMachine(void) /* and the bgwriter too */ if (BgWriterPID != 0) signal_child(BgWriterPID, SIGTERM); + /* and the custodian too */ + if (CustodianPID != 0) + signal_child(CustodianPID, SIGTERM); /* and the walwriter too */ if (WalWriterPID != 0) signal_child(WalWriterPID, SIGTERM); @@ -3853,6 +3891,7 @@ PostmasterStateMachine(void) BgWriterPID == 0 && (CheckpointerPID == 0 || (!FatalError && Shutdown < ImmediateShutdown)) && + CustodianPID == 0 && WalWriterPID == 0 && AutoVacPID == 0) { @@ -3942,6 +3981,7 @@ PostmasterStateMachine(void) Assert(WalReceiverPID == 0); Assert(BgWriterPID == 0); Assert(CheckpointerPID == 0); + Assert(CustodianPID == 0); Assert(WalWriterPID == 0); Assert(AutoVacPID == 0); /* syslogger is not considered here */ @@ -4135,6 +4175,8 @@ TerminateChildren(int signal) signal_child(BgWriterPID, signal); if (CheckpointerPID != 0) signal_child(CheckpointerPID, signal); + if (CustodianPID != 0) + signal_child(CustodianPID, signal); if (WalWriterPID != 0) signal_child(WalWriterPID, signal); if (WalReceiverPID != 0) diff --git a/src/backend/storage/ipc/ipci.c b/src/backend/storage/ipc/ipci.c index 1a6f527051..b19d743cab 100644 --- a/src/backend/storage/ipc/ipci.c +++ b/src/backend/storage/ipc/ipci.c @@ -30,6 +30,7 @@ #include "postmaster/autovacuum.h" #include "postmaster/bgworker_internals.h" #include "postmaster/bgwriter.h" +#include "postmaster/custodian.h" #include "postmaster/postmaster.h" #include "replication/logicallauncher.h" #include "replication/origin.h" @@ -129,6 +130,7 @@ CalculateShmemSize(int *num_semaphores) size = add_size(size, PMSignalShmemSize()); size = add_size(size, ProcSignalShmemSize()); size = add_size(size, CheckpointerShmemSize()); + size = add_size(size, CustodianShmemSize()); size = add_size(size, AutoVacuumShmemSize()); size = add_size(size, ReplicationSlotsShmemSize()); size = add_size(size, ReplicationOriginShmemSize()); @@ -277,6 +279,7 @@ CreateSharedMemoryAndSemaphores(void) PMSignalShmemInit(); ProcSignalShmemInit(); CheckpointerShmemInit(); + CustodianShmemInit(); AutoVacuumShmemInit(); ReplicationSlotsShmemInit(); ReplicationOriginShmemInit(); diff --git a/src/backend/storage/lmgr/proc.c b/src/backend/storage/lmgr/proc.c index 37aaab1338..f297f489c9 100644 --- a/src/backend/storage/lmgr/proc.c +++ b/src/backend/storage/lmgr/proc.c @@ -180,6 +180,7 @@ InitProcGlobal(void) ProcGlobal->startupBufferPinWaitBufId = -1; ProcGlobal->walwriterLatch = NULL; ProcGlobal->checkpointerLatch = NULL; + ProcGlobal->custodianLatch = NULL; pg_atomic_init_u32(&ProcGlobal->procArrayGroupFirst, INVALID_PGPROCNO); pg_atomic_init_u32(&ProcGlobal->clogGroupFirst, INVALID_PGPROCNO); diff --git a/src/backend/utils/activity/wait_event.c b/src/backend/utils/activity/wait_event.c index 87c15b9c6f..469768c4e4 100644 --- a/src/backend/utils/activity/wait_event.c +++ b/src/backend/utils/activity/wait_event.c @@ -224,6 +224,9 @@ pgstat_get_wait_activity(WaitEventActivity w) case WAIT_EVENT_CHECKPOINTER_MAIN: event_name = "CheckpointerMain"; break; + case WAIT_EVENT_CUSTODIAN_MAIN: + event_name = "CustodianMain"; + break; case WAIT_EVENT_LOGICAL_APPLY_MAIN: event_name = "LogicalApplyMain"; break; diff --git a/src/backend/utils/init/miscinit.c b/src/backend/utils/init/miscinit.c index b25bd0e583..66bf42e5b1 100644 --- a/src/backend/utils/init/miscinit.c +++ b/src/backend/utils/init/miscinit.c @@ -273,6 +273,9 @@ GetBackendTypeDesc(BackendType backendType) case B_CHECKPOINTER: backendDesc = "checkpointer"; break; + case B_CUSTODIAN: + backendDesc = "custodian"; + break; case B_STARTUP: backendDesc = "startup"; break; diff --git a/src/include/miscadmin.h b/src/include/miscadmin.h index 0af130fbc5..ffe9404c68 100644 --- a/src/include/miscadmin.h +++ b/src/include/miscadmin.h @@ -330,6 +330,7 @@ typedef enum BackendType B_BG_WORKER, B_BG_WRITER, B_CHECKPOINTER, + B_CUSTODIAN, B_STARTUP, B_WAL_RECEIVER, B_WAL_SENDER, @@ -433,6 +434,7 @@ typedef enum BgWriterProcess, ArchiverProcess, CheckpointerProcess, + CustodianProcess, WalWriterProcess, WalReceiverProcess, @@ -445,6 +447,7 @@ extern PGDLLIMPORT AuxProcType MyAuxProcType; #define AmBackgroundWriterProcess() (MyAuxProcType == BgWriterProcess) #define AmArchiverProcess() (MyAuxProcType == ArchiverProcess) #define AmCheckpointerProcess() (MyAuxProcType == CheckpointerProcess) +#define AmCustodianProcess() (MyAuxProcType == CustodianProcess) #define AmWalWriterProcess() (MyAuxProcType == WalWriterProcess) #define AmWalReceiverProcess() (MyAuxProcType == WalReceiverProcess) diff --git a/src/include/postmaster/custodian.h b/src/include/postmaster/custodian.h new file mode 100644 index 0000000000..c95a7c7de6 --- /dev/null +++ b/src/include/postmaster/custodian.h @@ -0,0 +1,20 @@ +/*------------------------------------------------------------------------- + * + * custodian.h + * Exports from postmaster/custodian.c. + * + * Copyright (c) 2022, PostgreSQL Global Development Group + * + * src/include/postmaster/custodian.h + * + *------------------------------------------------------------------------- + */ +#ifndef _CUSTODIAN_H +#define _CUSTODIAN_H + +extern void CustodianMain(void) pg_attribute_noreturn(); +extern Size CustodianShmemSize(void); +extern void CustodianShmemInit(void); +extern void RequestCustodian(int flags); + +#endif /* _CUSTODIAN_H */ diff --git a/src/include/storage/proc.h b/src/include/storage/proc.h index 2579e619eb..467421e371 100644 --- a/src/include/storage/proc.h +++ b/src/include/storage/proc.h @@ -394,6 +394,8 @@ typedef struct PROC_HDR Latch *walwriterLatch; /* Checkpointer process's latch */ Latch *checkpointerLatch; + /* Custodian process's latch */ + Latch *custodianLatch; /* Current shared estimate of appropriate spins_per_delay value */ int spins_per_delay; /* Buffer id of the buffer that Startup process waits for pin on, or -1 */ @@ -411,11 +413,12 @@ extern PGDLLIMPORT PGPROC *PreparedXactProcs; * We set aside some extra PGPROC structures for auxiliary processes, * ie things that aren't full-fledged backends but need shmem access. * - * Background writer, checkpointer, WAL writer and archiver run during normal - * operation. Startup process and WAL receiver also consume 2 slots, but WAL - * writer is launched only after startup has exited, so we only need 5 slots. + * Background writer, checkpointer, custodian, WAL writer and archiver run + * during normal operation. Startup process and WAL receiver also consume 2 + * slots, but WAL writer is launched only after startup has exited, so we only + * need 6 slots. */ -#define NUM_AUXILIARY_PROCS 5 +#define NUM_AUXILIARY_PROCS 6 /* configurable options */ extern PGDLLIMPORT int DeadlockTimeout; diff --git a/src/include/utils/wait_event.h b/src/include/utils/wait_event.h index b578e2ec75..7524e197e5 100644 --- a/src/include/utils/wait_event.h +++ b/src/include/utils/wait_event.h @@ -40,6 +40,7 @@ typedef enum WAIT_EVENT_BGWRITER_HIBERNATE, WAIT_EVENT_BGWRITER_MAIN, WAIT_EVENT_CHECKPOINTER_MAIN, + WAIT_EVENT_CUSTODIAN_MAIN, WAIT_EVENT_LOGICAL_APPLY_MAIN, WAIT_EVENT_LOGICAL_LAUNCHER_MAIN, WAIT_EVENT_RECOVERY_WAL_STREAM, -- 2.25.1