From f6517f54aa1996f74600715e6bb67f9bf64576b9 Mon Sep 17 00:00:00 2001 From: Hayato Kuroda Date: Thu, 12 Nov 2026 11:22:12 +0900 Subject: [PATCH v2] Simplify parallel apply worker error reporting Parallel apply workers used a dedicated shared-memory queue to send ErrorResponse messages to the leader apply worker. The leader parsed those messages only to detect worker failure and report a generic error. However, commit 952bab8b74d4 stopped the leader from re-reporting errors reported by parallel apply workers. Therefore, transporting ErrorResponse messages from parallel apply workers to the leader is no longer necessary. Replace that queue with a status field in the worker's existing shared state. On exit, the worker records whether it stopped normally at the leader's request or failed, and then notifies the leader with a process signal. The leader scans the shared status of its parallel apply workers and exits if any worker failed. --- src/backend/libpq/pqmq.c | 7 +- .../replication/logical/applyparallelworker.c | 245 ++++-------------- src/backend/replication/logical/launcher.c | 20 -- src/backend/replication/logical/worker.c | 9 - src/backend/storage/ipc/procsignal.c | 4 +- src/backend/tcop/postgres.c | 4 +- src/include/replication/logicalworker.h | 7 +- src/include/replication/worker_internal.h | 17 +- src/include/storage/procsignal.h | 2 +- src/tools/pgindent/typedefs.list | 1 + 10 files changed, 76 insertions(+), 240 deletions(-) diff --git a/src/backend/libpq/pqmq.c b/src/backend/libpq/pqmq.c index d038a9da515..e9caf2382e5 100644 --- a/src/backend/libpq/pqmq.c +++ b/src/backend/libpq/pqmq.c @@ -20,7 +20,6 @@ #include "libpq/pqmq.h" #include "miscadmin.h" #include "pgstat.h" -#include "replication/logicalworker.h" #include "storage/latch.h" #include "tcop/tcopprot.h" #include "utils/builtins.h" @@ -173,11 +172,7 @@ mq_putmessage(char msgtype, const char *s, size_t len) if (pq_mq_parallel_leader_pid != 0) { - if (IsLogicalParallelApplyWorker()) - SendProcSignal(pq_mq_parallel_leader_pid, - PROCSIG_PARALLEL_APPLY_MESSAGE, - pq_mq_parallel_leader_proc_number); - else if (AmRepackWorker()) + if (AmRepackWorker()) SendProcSignal(pq_mq_parallel_leader_pid, PROCSIG_REPACK_MESSAGE, pq_mq_parallel_leader_proc_number); diff --git a/src/backend/replication/logical/applyparallelworker.c b/src/backend/replication/logical/applyparallelworker.c index a6d58ad0161..bfdcc427e3a 100644 --- a/src/backend/replication/logical/applyparallelworker.c +++ b/src/backend/replication/logical/applyparallelworker.c @@ -52,9 +52,7 @@ * * The dynamic shared memory segment contains: (a) a shm_mq that is used to * send changes in the transaction from leader apply worker to parallel apply - * worker; (b) another shm_mq that is used to send errors (and other messages - * reported via elog/ereport) from the parallel apply worker to leader apply - * worker; (c) necessary information to be shared among parallel apply workers + * worker; (b) necessary information to be shared among parallel apply workers * and the leader apply worker (i.e. members of ParallelApplyWorkerShared). * * Locking Considerations @@ -158,7 +156,6 @@ #include "postgres.h" #include "libpq/pqformat.h" -#include "libpq/pqmq.h" #include "pgstat.h" #include "postmaster/interrupt.h" #include "replication/logicallauncher.h" @@ -184,19 +181,10 @@ */ #define PARALLEL_APPLY_KEY_SHARED 1 #define PARALLEL_APPLY_KEY_MQ 2 -#define PARALLEL_APPLY_KEY_ERROR_QUEUE 3 /* Queue size of DSM, 16 MB for now. */ #define DSM_QUEUE_SIZE (16 * 1024 * 1024) -/* - * Error queue size of DSM. It is desirable to make it large enough that a - * typical ErrorResponse can be sent without blocking. That way, a worker that - * errors out can write the whole message into the queue and terminate without - * waiting for the user backend. - */ -#define DSM_ERROR_QUEUE_SIZE (16 * 1024) - /* * There are three fields in each message received by the parallel apply * worker: start_lsn, end_lsn and send_time. Because we have updated these @@ -242,10 +230,9 @@ static List *ParallelApplyWorkerPool = NIL; ParallelApplyWorkerShared *MyParallelShared = NULL; /* - * Is there a message sent by a parallel apply worker that the leader apply - * worker needs to receive? + * Has a parallel apply worker exited? */ -volatile sig_atomic_t ParallelApplyMessagePending = false; +volatile sig_atomic_t ParallelApplyWorkerExitPending = false; /* * Cache the parallel apply worker information required for applying the @@ -258,6 +245,8 @@ static ParallelApplyWorkerInfo *stream_apply_worker = NULL; static List *subxactlist = NIL; static void pa_free_worker_info(ParallelApplyWorkerInfo *winfo); +static ParallelApplyWorkerStatus pa_get_worker_status(ParallelApplyWorkerShared *wshared); +pg_noreturn static void pa_report_worker_error(void); static ParallelTransState pa_get_xact_state(ParallelApplyWorkerShared *wshared); static PartialFileSetState pa_get_fileset_state(void); @@ -323,7 +312,7 @@ pa_can_start(void) * Set up a dynamic shared memory segment. * * We set up a control region that contains a fixed-size worker info - * (ParallelApplyWorkerShared), a message queue, and an error queue. + * (ParallelApplyWorkerShared) and a message queue. * * Returns true on success, false on failure. */ @@ -337,7 +326,6 @@ pa_setup_dsm(ParallelApplyWorkerInfo *winfo) ParallelApplyWorkerShared *shared; shm_mq *mq; Size queue_size = DSM_QUEUE_SIZE; - Size error_queue_size = DSM_ERROR_QUEUE_SIZE; /* * Estimate how much shared memory we need. @@ -345,16 +333,14 @@ pa_setup_dsm(ParallelApplyWorkerInfo *winfo) * Because the TOC machinery may choose to insert padding of oddly-sized * requests, we must estimate each chunk separately. * - * We need one key to register the location of the header, and two other - * keys to track the locations of the message queue and the error message - * queue. + * We need one key to register the location of the header and a key to + * track the location of the message queue. */ shm_toc_initialize_estimator(&e); shm_toc_estimate_chunk(&e, sizeof(ParallelApplyWorkerShared)); shm_toc_estimate_chunk(&e, queue_size); - shm_toc_estimate_chunk(&e, error_queue_size); - shm_toc_estimate_keys(&e, 3); + shm_toc_estimate_keys(&e, 2); segsize = shm_toc_estimate(&e); /* Create the shared memory segment and establish a table of contents. */ @@ -373,6 +359,7 @@ pa_setup_dsm(ParallelApplyWorkerInfo *winfo) pg_atomic_init_u32(&(shared->pending_stream_count), 0); shared->last_commit_end = InvalidXLogRecPtr; shared->fileset_state = FS_EMPTY; + shared->worker_status = PA_WORKER_RUNNING; shm_toc_insert(toc, PARALLEL_APPLY_KEY_SHARED, shared); @@ -384,15 +371,6 @@ pa_setup_dsm(ParallelApplyWorkerInfo *winfo) /* Attach the queue. */ winfo->mq_handle = shm_mq_attach(mq, seg, NULL); - /* Set up error queue for the worker. */ - mq = shm_mq_create(shm_toc_allocate(toc, error_queue_size), - error_queue_size); - shm_toc_insert(toc, PARALLEL_APPLY_KEY_ERROR_QUEUE, mq); - shm_mq_set_receiver(mq, MyProc); - - /* Attach the queue. */ - winfo->error_mq_handle = shm_mq_attach(mq, seg, NULL); - /* Return results to caller. */ winfo->dsm_seg = seg; winfo->shared = shared; @@ -604,9 +582,6 @@ pa_free_worker_info(ParallelApplyWorkerInfo *winfo) if (winfo->mq_handle) shm_mq_detach(winfo->mq_handle); - if (winfo->error_mq_handle) - shm_mq_detach(winfo->error_mq_handle); - /* Unlink the files with serialized changes. */ if (winfo->serialize_changes) stream_cleanup_files(MyLogicalRepWorker->subid, winfo->shared->xid); @@ -620,26 +595,6 @@ pa_free_worker_info(ParallelApplyWorkerInfo *winfo) pfree(winfo); } -/* - * Detach the error queue for all parallel apply workers. - */ -void -pa_detach_all_error_mq(void) -{ - ListCell *lc; - - foreach(lc, ParallelApplyWorkerPool) - { - ParallelApplyWorkerInfo *winfo = (ParallelApplyWorkerInfo *) lfirst(lc); - - if (winfo->error_mq_handle) - { - shm_mq_detach(winfo->error_mq_handle); - winfo->error_mq_handle = NULL; - } - } -} - /* * Check if there are any pending spooled messages. */ @@ -847,9 +802,7 @@ LogicalParallelApplyLoop(shm_mq_handle *mqh) } /* - * Make sure the leader apply worker tries to read from our error queue one more - * time. This guards against the case where we exit uncleanly without sending - * an ErrorResponse, for example because some code calls proc_exit directly. + * Set the worker status and notify the leader apply worker about the shutdown. * * Also explicitly detach from dsm segment to invoke on_dsm_detach callbacks, * if any. See ParallelWorkerShutdown for details. @@ -857,8 +810,18 @@ LogicalParallelApplyLoop(shm_mq_handle *mqh) static void pa_shutdown(int code, Datum arg) { + SpinLockAcquire(&MyParallelShared->mutex); + + /* Update the worker status based on the shutdown code and request */ + if (code == 0 && ShutdownRequestPending) + MyParallelShared->worker_status = PA_WORKER_STOPPED; + else + MyParallelShared->worker_status = PA_WORKER_FAILED; + + SpinLockRelease(&MyParallelShared->mutex); + SendProcSignal(MyLogicalRepWorker->leader_pid, - PROCSIG_PARALLEL_APPLY_MESSAGE, + PROCSIG_PARALLEL_APPLY_WORKER_EXIT, INVALID_PROC_NUMBER); dsm_detach((dsm_segment *) DatumGetPointer(arg)); @@ -876,7 +839,6 @@ ParallelApplyWorkerMain(Datum main_arg) shm_toc *toc; shm_mq *mq; shm_mq_handle *mqh; - shm_mq_handle *error_mqh; ReplOriginId originid; int worker_slot = DatumGetInt32(main_arg); char originname[NAMEDATALEN]; @@ -945,17 +907,6 @@ ParallelApplyWorkerMain(Datum main_arg) MyParallelShared->logicalrep_worker_slot_no = worker_slot; SpinLockRelease(&MyParallelShared->mutex); - /* - * Attach to the error queue. - */ - mq = shm_toc_lookup(toc, PARALLEL_APPLY_KEY_ERROR_QUEUE, false); - shm_mq_set_sender(mq, MyProc); - error_mqh = shm_mq_attach(mq, seg, NULL); - - pq_redirect_to_shm_mq(seg, error_mqh); - pq_set_parallel_leader(MyLogicalRepWorker->leader_pid, - INVALID_PROC_NUMBER); - MyLogicalRepWorker->last_send_time = MyLogicalRepWorker->last_recv_time = MyLogicalRepWorker->reply_time = 0; @@ -999,144 +950,48 @@ ParallelApplyWorkerMain(Datum main_arg) } /* - * Handle receipt of an interrupt indicating a parallel apply worker message. - * - * Note: this is called within a signal handler! All we can do is set a flag - * that will cause the next CHECK_FOR_INTERRUPTS() to invoke - * ProcessParallelApplyMessages(). + * Handle receipt of an interrupt indicating that a parallel apply worker + * has exited. */ void -HandleParallelApplyMessageInterrupt(void) +HandleParallelApplyWorkerExitInterrupt(void) { InterruptPending = true; - ParallelApplyMessagePending = true; + ParallelApplyWorkerExitPending = true; /* latch will be set by procsignal_sigusr1_handler */ } -/* - * Process a single protocol message received from a single parallel apply - * worker. - */ -static void -ProcessParallelApplyMessage(StringInfo msg) +pg_noreturn static void +pa_report_worker_error(void) { - char msgtype; - - msgtype = pq_getmsgbyte(msg); + error_context_stack = apply_error_context_stack; - switch (msgtype) - { - case PqMsg_ErrorResponse: - - /* - * The parallel apply worker has already reported the error with - * the context of the transaction that failed. Report the error - * of the leader apply worker with the error context callbacks - * that were in effect. - */ - error_context_stack = apply_error_context_stack; - - /* - * The actual error must have been reported by the parallel apply - * worker. - */ - ereport(ERROR, - (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), - errmsg("logical replication parallel apply worker exited due to error"), - errdetail("The leader apply worker will now exit and terminate any other parallel apply workers."))); - - /* - * Don't need to do anything about NoticeResponse and - * NotificationResponse as the logical replication worker doesn't - * need to send messages to the client. - */ - case PqMsg_NoticeResponse: - case PqMsg_NotificationResponse: - break; - - default: - elog(ERROR, "unrecognized message type received from logical replication parallel apply worker: %c (message length %d bytes)", - msgtype, msg->len); - } + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("logical replication parallel apply worker exited due to error"), + errdetail("The leader apply worker will now exit and terminate any other parallel apply workers."))); } /* - * Handle any queued protocol messages received from parallel apply workers. + * Process parallel apply worker exits. */ void -ProcessParallelApplyMessages(void) +ProcessParallelApplyWorkerExits(void) { - ListCell *lc; - MemoryContext oldcontext; + ParallelApplyWorkerExitPending = false; - static MemoryContext hpam_context = NULL; - - /* - * This is invoked from ProcessInterrupts(), and since some of the - * functions it calls contain CHECK_FOR_INTERRUPTS(), there is a potential - * for recursive calls if more signals are received while this runs. It's - * unclear that recursive entry would be safe, and it doesn't seem useful - * even if it is safe, so let's block interrupts until done. - */ - HOLD_INTERRUPTS(); - - /* - * Moreover, CurrentMemoryContext might be pointing almost anywhere. We - * don't want to risk leaking data into long-lived contexts, so let's do - * our work here in a private context that we can reset on each use. - */ - if (!hpam_context) /* first time through? */ - hpam_context = AllocSetContextCreate(TopMemoryContext, - "ProcessParallelApplyMessages", - ALLOCSET_DEFAULT_SIZES); - else - MemoryContextReset(hpam_context); - - oldcontext = MemoryContextSwitchTo(hpam_context); - - ParallelApplyMessagePending = false; - - foreach(lc, ParallelApplyWorkerPool) + foreach_ptr(ParallelApplyWorkerInfo, winfo, ParallelApplyWorkerPool) { - shm_mq_result res; - Size nbytes; - void *data; - ParallelApplyWorkerInfo *winfo = (ParallelApplyWorkerInfo *) lfirst(lc); - - /* - * The leader will detach from the error queue and set it to NULL - * before preparing to stop all parallel apply workers, so we don't - * need to handle error messages anymore. See - * logicalrep_worker_detach. - */ - if (!winfo->error_mq_handle) - continue; - - res = shm_mq_receive(winfo->error_mq_handle, &nbytes, &data, true); - - if (res == SHM_MQ_WOULD_BLOCK) - continue; - else if (res == SHM_MQ_SUCCESS) + switch (pa_get_worker_status(winfo->shared)) { - StringInfoData msg; - - initStringInfo(&msg); - appendBinaryStringInfo(&msg, data, nbytes); - ProcessParallelApplyMessage(&msg); - pfree(msg.data); + case PA_WORKER_RUNNING: + break; + case PA_WORKER_STOPPED: + continue; + case PA_WORKER_FAILED: + pa_report_worker_error(); } - else - ereport(ERROR, - (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), - errmsg("lost connection to the logical replication parallel apply worker"))); } - - MemoryContextSwitchTo(oldcontext); - - /* Might as well clear the context on our way out */ - MemoryContextReset(hpam_context); - - RESUME_INTERRUPTS(); } /* @@ -1331,6 +1186,18 @@ pa_get_xact_state(ParallelApplyWorkerShared *wshared) return xact_state; } +static ParallelApplyWorkerStatus +pa_get_worker_status(ParallelApplyWorkerShared *wshared) +{ + ParallelApplyWorkerStatus worker_status; + + SpinLockAcquire(&wshared->mutex); + worker_status = wshared->worker_status; + SpinLockRelease(&wshared->mutex); + + return worker_status; +} + /* * Cache the parallel apply worker information. */ diff --git a/src/backend/replication/logical/launcher.c b/src/backend/replication/logical/launcher.c index 5057a928adb..dd0d9a09b69 100644 --- a/src/backend/replication/logical/launcher.c +++ b/src/backend/replication/logical/launcher.c @@ -701,18 +701,6 @@ logicalrep_pa_worker_stop(ParallelApplyWorkerInfo *winfo) Assert(slot_no >= 0 && slot_no < max_logical_replication_workers); - /* - * Detach from the error_mq_handle for the parallel apply worker before - * stopping it. This prevents the leader apply worker from trying to - * receive the message from the error queue that might already be detached - * by the parallel apply worker. - */ - if (winfo->error_mq_handle) - { - shm_mq_detach(winfo->error_mq_handle); - winfo->error_mq_handle = NULL; - } - LWLockAcquire(LogicalRepWorkerLock, LW_SHARED); worker = &LogicalRepCtx->workers[slot_no]; @@ -811,14 +799,6 @@ logicalrep_worker_detach(void) List *workers; ListCell *lc; - /* - * Detach from the error_mq_handle for all parallel apply workers - * before terminating them. This prevents the leader apply worker from - * receiving the worker termination message and sending it to logs - * when the same is already done by the parallel worker. - */ - pa_detach_all_error_mq(); - LWLockAcquire(LogicalRepWorkerLock, LW_SHARED); workers = logicalrep_workers_find(MyLogicalRepWorker->subid, true, false); diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c index 95044620e68..743f3d1002d 100644 --- a/src/backend/replication/logical/worker.c +++ b/src/backend/replication/logical/worker.c @@ -6183,15 +6183,6 @@ IsLogicalWorker(void) return MyLogicalRepWorker != NULL; } -/* - * Is current process a logical replication parallel apply worker? - */ -bool -IsLogicalParallelApplyWorker(void) -{ - return IsLogicalWorker() && am_parallel_apply_worker(); -} - /* * Start skipping changes of the transaction if the given LSN matches the * LSN specified by subscription's skiplsn. diff --git a/src/backend/storage/ipc/procsignal.c b/src/backend/storage/ipc/procsignal.c index eb0f4d6fcdf..1e522ca79a7 100644 --- a/src/backend/storage/ipc/procsignal.c +++ b/src/backend/storage/ipc/procsignal.c @@ -715,8 +715,8 @@ procsignal_sigusr1_handler(SIGNAL_ARGS) if (CheckProcSignal(PROCSIG_LOG_MEMORY_CONTEXT)) HandleLogMemoryContextInterrupt(); - if (CheckProcSignal(PROCSIG_PARALLEL_APPLY_MESSAGE)) - HandleParallelApplyMessageInterrupt(); + if (CheckProcSignal(PROCSIG_PARALLEL_APPLY_WORKER_EXIT)) + HandleParallelApplyWorkerExitInterrupt(); if (CheckProcSignal(PROCSIG_REPACK_MESSAGE)) HandleRepackMessageInterrupt(); diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c index d6761aeea9f..ce709d299f2 100644 --- a/src/backend/tcop/postgres.c +++ b/src/backend/tcop/postgres.c @@ -3703,8 +3703,8 @@ ProcessInterrupts(void) if (LogMemoryContextPending) ProcessLogMemoryContextInterrupt(); - if (ParallelApplyMessagePending) - ProcessParallelApplyMessages(); + if (ParallelApplyWorkerExitPending) + ProcessParallelApplyWorkerExits(); if (SlotSyncShutdownPending) ProcessSlotSyncMessage(); diff --git a/src/include/replication/logicalworker.h b/src/include/replication/logicalworker.h index 7d748a28da8..ae62455148b 100644 --- a/src/include/replication/logicalworker.h +++ b/src/include/replication/logicalworker.h @@ -14,7 +14,7 @@ #include -extern PGDLLIMPORT volatile sig_atomic_t ParallelApplyMessagePending; +extern PGDLLIMPORT volatile sig_atomic_t ParallelApplyWorkerExitPending; extern void ApplyWorkerMain(Datum main_arg); extern void ParallelApplyWorkerMain(Datum main_arg); @@ -22,10 +22,9 @@ extern void TableSyncWorkerMain(Datum main_arg); extern void SequenceSyncWorkerMain(Datum main_arg); extern bool IsLogicalWorker(void); -extern bool IsLogicalParallelApplyWorker(void); -extern void HandleParallelApplyMessageInterrupt(void); -extern void ProcessParallelApplyMessages(void); +extern void HandleParallelApplyWorkerExitInterrupt(void); +extern void ProcessParallelApplyWorkerExits(void); extern void LogicalRepWorkersWakeupAtCommit(Oid subid); diff --git a/src/include/replication/worker_internal.h b/src/include/replication/worker_internal.h index 88cb7c1e252..a311e2ab471 100644 --- a/src/include/replication/worker_internal.h +++ b/src/include/replication/worker_internal.h @@ -147,6 +147,13 @@ typedef enum PartialFileSetState FS_READY, } PartialFileSetState; +typedef enum ParallelApplyWorkerStatus +{ + PA_WORKER_RUNNING, + PA_WORKER_STOPPED, + PA_WORKER_FAILED, +} ParallelApplyWorkerStatus; + /* * Struct for sharing information between leader apply worker and parallel * apply workers. @@ -196,6 +203,9 @@ typedef struct ParallelApplyWorkerShared */ PartialFileSetState fileset_state; FileSet fileset; + + /* Current execution status of the parallel apply worker. */ + ParallelApplyWorkerStatus worker_status; } ParallelApplyWorkerShared; /* @@ -209,12 +219,6 @@ typedef struct ParallelApplyWorkerInfo */ shm_mq_handle *mq_handle; - /* - * This queue is used to transfer error messages from the parallel apply - * worker to the leader apply worker. - */ - shm_mq_handle *error_mq_handle; - dsm_segment *dsm_seg; /* @@ -331,7 +335,6 @@ extern void set_apply_error_context_origin(char *originname); /* Parallel apply worker setup and interactions */ extern void pa_allocate_worker(TransactionId xid); extern ParallelApplyWorkerInfo *pa_find_worker(TransactionId xid); -extern void pa_detach_all_error_mq(void); extern bool pa_send_data(ParallelApplyWorkerInfo *winfo, Size nbytes, const void *data); diff --git a/src/include/storage/procsignal.h b/src/include/storage/procsignal.h index aaa158bfd66..0c0717f1883 100644 --- a/src/include/storage/procsignal.h +++ b/src/include/storage/procsignal.h @@ -35,7 +35,7 @@ typedef enum PROCSIG_WALSND_INIT_STOPPING, /* ask walsenders to prepare for shutdown */ PROCSIG_BARRIER, /* global barrier interrupt */ PROCSIG_LOG_MEMORY_CONTEXT, /* ask backend to log the memory contexts */ - PROCSIG_PARALLEL_APPLY_MESSAGE, /* Message from parallel apply workers */ + PROCSIG_PARALLEL_APPLY_WORKER_EXIT, /* parallel apply worker exited */ PROCSIG_SLOTSYNC_MESSAGE, /* ask slot synchronization to stop */ PROCSIG_REPACK_MESSAGE, /* Message from repack worker */ PROCSIG_RECOVERY_CONFLICT, /* backend is blocking recovery, check diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f10eb469266..78d48d57292 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -2135,6 +2135,7 @@ ParallelAppendState ParallelApplyWorkerEntry ParallelApplyWorkerInfo ParallelApplyWorkerShared +ParallelApplyWorkerStatus ParallelBitmapHeapState ParallelBlockTableScanDesc ParallelBlockTableScanWorker -- 2.52.0