From ef22bd69c93c5e9dd647a7c27bf405c402aec281 Mon Sep 17 00:00:00 2001 From: Manu Date: Wed, 21 Oct 2026 14:23:10 -0300 Subject: [PATCH v1] Terminate sessions that do not absorb a ProcSignalBarrier in time WaitForProcSignalBarrier() waits until every process has absorbed the barrier. A client backend or WAL sender that is blocked sending data to a client that has stopped reading never does: while blocked in a client write it services only ProcDiePending (see ProcessClientWriteInterrupt()), not the barrier and not a recovery conflict signal. Whatever waits on the barrier then waits for as long as that client stays stuck. That includes the end of recovery and the replay of the record that enables logical decoding when wal_level=replica, so a standby can neither promote nor keep replaying, as well as changing data checksums online, and DROP DATABASE and the other users of PROCSIGNAL_BARRIER_SMGRRELEASE. Add procsignal_barrier_timeout (default 30s, 0 disables). Once a barrier wait has lasted that long, a client backend or WAL sender that still has not absorbed the barrier is sent SIGTERM, as pg_terminate_backend() would do, and the wait goes on until it exits. Other kinds of process are never signaled. A LOG message names each process terminated this way. This applies to every waiter, not only to the startup process, since the hang is the same on a primary. Reported-by: shihao zhong Suggested-by: Masahiko Sawada Discussion: https://postgr.es/m/CAA4eK1LC09Tq3vTr04ppXRbyQnMb66_pgyuES2C_5ZoWfYRUrQ@mail.gmail.com --- doc/src/sgml/config.sgml | 34 ++ src/backend/storage/ipc/procsignal.c | 92 +++++- src/backend/utils/misc/guc_parameters.dat | 10 + src/backend/utils/misc/guc_tables.c | 1 + src/backend/utils/misc/postgresql.conf.sample | 3 + src/include/storage/procsignal.h | 3 + src/test/recovery/meson.build | 1 + .../t/061_procsignal_barrier_timeout.pl | 293 ++++++++++++++++++ 8 files changed, 435 insertions(+), 2 deletions(-) create mode 100644 src/test/recovery/t/061_procsignal_barrier_timeout.pl diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index f7db299..2a6082e 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -11836,6 +11836,40 @@ dynamic_library_path = '/usr/local/lib/postgresql:$libdir' + + procsignal_barrier_timeout (integer) + + procsignal_barrier_timeout configuration parameter + + + + + Some operations change state that every running process must take + into account before the operation can continue, and wait until all + processes have done so. Among them are the end of recovery and the + replay of WAL on a standby when logical decoding is being enabled, + enabling or disabling data checksums online (see + ) and + DROP DATABASE. A session can only do this + between other work; one that is blocked sending data to a client that + has stopped reading cannot, and the operation would wait for it + indefinitely. This parameter sets how long the operation waits + before it terminates such a client backend or WAL sender process, + as pg_terminate_backend would. Other processes + are never terminated by this mechanism. A message is written to the + server log for each process terminated this way. + + + If this value is specified without units, it is taken as + milliseconds. A value of zero disables the timeout, so the + operation waits as long as it takes. The default is 30 seconds + (30s). This parameter can only be set in the + postgresql.conf file or on the server command + line. + + + + max_locks_per_transaction (integer) diff --git a/src/backend/storage/ipc/procsignal.c b/src/backend/storage/ipc/procsignal.c index eb0f4d6..0cb83e8 100644 --- a/src/backend/storage/ipc/procsignal.c +++ b/src/backend/storage/ipc/procsignal.c @@ -38,6 +38,7 @@ #include "storage/subsystems.h" #include "tcop/tcopprot.h" #include "utils/memutils.h" +#include "utils/timestamp.h" #include "utils/wait_event.h" /* @@ -119,9 +120,13 @@ const ShmemCallbacks ProcSignalShmemCallbacks = { NON_EXEC_STATIC ProcSignalHeader *ProcSignal = NULL; +/* GUC variable */ +int ProcSignalBarrierTimeout = 30000; + static ProcSignalSlot *MyProcSignalSlot = NULL; static bool CheckProcSignal(ProcSignalReason reason); +static void TerminateBarrierStraggler(int procno); static void CleanupProcSignalState(int status, Datum arg); static void ResetProcSignalBarrierBits(uint32 flags); @@ -437,6 +442,8 @@ EmitProcSignalBarrier(ProcSignalBarrierType type) void WaitForProcSignalBarrier(uint64 generation) { + TimestampTz wait_start = 0; + Assert(generation <= pg_atomic_read_u64(&ProcSignal->psh_barrierGeneration)); elog(DEBUG1, @@ -448,6 +455,7 @@ WaitForProcSignalBarrier(uint64 generation) { ProcSignalSlot *slot = &ProcSignal->psh_slot[i]; uint64 oldval; + bool terminate_tried = false; /* * It's important that we check only pss_barrierGeneration here and @@ -458,9 +466,47 @@ WaitForProcSignalBarrier(uint64 generation) oldval = pg_atomic_read_u64(&slot->pss_barrierGeneration); while (oldval < generation) { + long sleep_ms = 5000; + bool report_on_timeout = true; + + /* + * Once we have waited procsignal_barrier_timeout in total, try + * once to terminate the process that has not absorbed the + * barrier, then keep waiting for it to exit. A process that + * exits advertises PG_UINT64_MAX and wakes us up (see + * CleanupProcSignalState()). + * + * XXX: This applies to every caller, not only to the startup + * process. A process blocked on its client (see + * TerminateBarrierStraggler()) can hold up the end of recovery + * and the replay of WAL just as well as DROP DATABASE or + * pg_enable_data_checksums() on a primary. + */ + if (ProcSignalBarrierTimeout > 0 && !terminate_tried) + { + TimestampTz now = GetCurrentTimestamp(); + long elapsed_ms; + + if (wait_start == 0) + wait_start = now; + elapsed_ms = TimestampDifferenceMilliseconds(wait_start, now); + + if (elapsed_ms >= ProcSignalBarrierTimeout) + { + TerminateBarrierStraggler(i); + terminate_tried = true; + } + else if (ProcSignalBarrierTimeout - elapsed_ms < sleep_ms) + { + sleep_ms = ProcSignalBarrierTimeout - elapsed_ms; + report_on_timeout = false; + } + } + if (ConditionVariableTimedSleep(&slot->pss_barrierCV, - 5000, - WAIT_EVENT_PROC_SIGNAL_BARRIER)) + sleep_ms, + WAIT_EVENT_PROC_SIGNAL_BARRIER) && + report_on_timeout) ereport(LOG, (errmsg("still waiting for backend with PID %d to accept ProcSignalBarrier", (int) pg_atomic_read_u32(&slot->pss_pid)))); @@ -484,6 +530,48 @@ WaitForProcSignalBarrier(uint64 generation) pg_memory_barrier(); } +/* + * TerminateBarrierStraggler - terminate the process using ProcSignal slot + * 'procno', which has not absorbed a ProcSignalBarrier within + * procsignal_barrier_timeout. + * + * Only client backends and walsenders are terminated, with SIGTERM, as + * pg_terminate_backend() would do. Those can block indefinitely on a client, + * for example sending data to a client that has stopped reading. While + * blocked in a client write, a backend services only ProcDiePending (see + * ProcessClientWriteInterrupt()), so SIGTERM is what releases it; a recovery + * conflict signal would not. Other processes are never signaled; the caller + * keeps waiting for them. + */ +static void +TerminateBarrierStraggler(int procno) +{ + ProcSignalSlot *slot = &ProcSignal->psh_slot[procno]; + PGPROC *proc = GetPGProcByNumber(procno); + pid_t pid = (pid_t) pg_atomic_read_u32(&slot->pss_pid); + + /* The slot is indexed by ProcNumber, so 'proc' is its owner */ + if (pid == 0 || pid == MyProcPid || proc->pid != pid) + return; + if (proc->backendType != B_BACKEND && proc->backendType != B_WAL_SENDER) + return; + + ereport(LOG, + (errmsg("terminating process %d because it has not accepted ProcSignalBarrier within %d ms", + (int) pid, ProcSignalBarrierTimeout))); + + /* + * As in pg_signal_backend(), the process could exit and its PID be reused + * before we send the signal; that seems too unlikely to worry about. + * Signal the whole process group if we have setsid(). + */ +#ifdef HAVE_SETSID + (void) kill(-pid, SIGTERM); +#else + (void) kill(pid, SIGTERM); +#endif +} + /* * Handle receipt of an interrupt indicating a global barrier event. * diff --git a/src/backend/utils/misc/guc_parameters.dat b/src/backend/utils/misc/guc_parameters.dat index 380679a..c579b07 100644 --- a/src/backend/utils/misc/guc_parameters.dat +++ b/src/backend/utils/misc/guc_parameters.dat @@ -2423,6 +2423,16 @@ check_hook => 'check_primary_slot_name', }, +{ name => 'procsignal_barrier_timeout', type => 'int', context => 'PGC_SIGHUP', group => 'LOCK_MANAGEMENT', + short_desc => 'Sets the time to wait for a session to accept a global state change before terminating it.', + long_desc => '0 disables the timeout.', + flags => 'GUC_UNIT_MS', + variable => 'ProcSignalBarrierTimeout', + boot_val => '30000', + min => '0', + max => 'INT_MAX', +}, + { name => 'quote_all_identifiers', type => 'bool', context => 'PGC_USERSET', group => 'COMPAT_OPTIONS_PREVIOUS', short_desc => 'When generating SQL fragments, quote all identifiers.', variable => 'quote_all_identifiers', diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index 342aaee..8c03a4f 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -88,6 +88,7 @@ #include "storage/predicate.h" #include "storage/proc.h" #include "storage/procnumber.h" +#include "storage/procsignal.h" #include "storage/standby.h" #include "tcop/backend_startup.h" #include "tcop/tcopprot.h" diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index 01f98de..9786f25 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -876,6 +876,9 @@ #------------------------------------------------------------------------------ #deadlock_timeout = 1s +#procsignal_barrier_timeout = 30s # terminate a session that has not + # accepted a global state change + # after this long; 0 disables #max_locks_per_transaction = 128 # min 10 # (change requires restart) #max_pred_locks_per_transaction = 64 # min 10 diff --git a/src/include/storage/procsignal.h b/src/include/storage/procsignal.h index aaa158b..9f048c2 100644 --- a/src/include/storage/procsignal.h +++ b/src/include/storage/procsignal.h @@ -66,6 +66,9 @@ typedef enum */ #define MAX_CANCEL_KEY_LENGTH 32 +/* GUC variable */ +extern PGDLLIMPORT int ProcSignalBarrierTimeout; + /* * prototypes for functions in procsignal.c */ diff --git a/src/test/recovery/meson.build b/src/test/recovery/meson.build index b6ac9dd..4942a0c 100644 --- a/src/test/recovery/meson.build +++ b/src/test/recovery/meson.build @@ -69,6 +69,7 @@ tests += { 't/058_shutdown_crash_restart.pl', 't/059_remote_apply_status_interval.pl', 't/060_multixact_crashed_updater.pl', + 't/061_procsignal_barrier_timeout.pl', ], }, } diff --git a/src/test/recovery/t/061_procsignal_barrier_timeout.pl b/src/test/recovery/t/061_procsignal_barrier_timeout.pl new file mode 100644 index 0000000..7a370ec --- /dev/null +++ b/src/test/recovery/t/061_procsignal_barrier_timeout.pl @@ -0,0 +1,293 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group + +# Test procsignal_barrier_timeout. Operations that wait for every process to +# absorb a ProcSignalBarrier must not wait forever behind a session that is +# blocked writing to a client that has stopped reading: after the timeout such +# a session is terminated. Sessions that are merely idle, sleeping or reading +# slowly must not be. + +use strict; +use warnings FATAL => 'all'; + +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; +use Time::HiRes qw(time usleep); + +my $copy_query = + "COPY (SELECT generate_series(1, 1000000000), repeat('x', 200)) TO STDOUT"; +my $terminated_re = + 'because it has not accepted ProcSignalBarrier within 2000 ms'; + +# Start a psql running $sql, with its output captured but not read. +sub start_psql +{ + my ($node, $dbname, $appname, $sql, $out) = @_; + my $err = ''; + + return IPC::Run::start( + [ + 'psql', '--no-psqlrc', '--quiet', + '--dbname' => $node->connstr($dbname) + . " application_name=$appname", + '--command' => $sql + ], + '>' => $out, + '2>' => \$err); +} + +# Return the PID of the session with the given application_name, once its +# wait_event is $wait_event (or once it exists, if $wait_event is undef). +sub session_pid +{ + my ($node, $appname, $wait_event) = @_; + my $cond = defined $wait_event ? "wait_event = '$wait_event'" : 'true'; + + $node->poll_query_until('postgres', + "SELECT $cond FROM pg_stat_activity WHERE application_name = '$appname'" + ) or die "timed out waiting for session \"$appname\""; + return $node->safe_psql('postgres', + "SELECT pid FROM pg_stat_activity WHERE application_name = '$appname'"); +} + +# Start a COPY TO STDOUT whose output is never read. psql blocks writing to +# its stdout and stops reading from the server, so the backend ends up blocked +# in a client write, where it cannot absorb a ProcSignalBarrier. +sub start_stuck_session +{ + my ($node, $dbname, $appname) = @_; + my $out = ''; + my $h = start_psql($node, $dbname, $appname, $copy_query, \$out); + + return ($h, session_pid($node, $appname, 'ClientWrite')); +} + +sub session_alive +{ + my ($node, $pid) = @_; + + return $node->safe_psql('postgres', + "SELECT count(*) FROM pg_stat_activity WHERE pid = $pid") eq '1'; +} + +# Check that the session $pid was terminated by the barrier timeout. +sub check_terminated +{ + my ($node, $pid, $log_offset, $what) = @_; + + ok( $node->log_contains( + "terminating process $pid $terminated_re", $log_offset), + "$what: blocked session was terminated after the timeout"); + ok( $node->log_contains( + qr/\[$pid\].*FATAL: terminating connection due to administrator command/, + $log_offset), + "$what: blocked session exited"); + ok($node->poll_query_until('postgres', + "SELECT count(*) = 0 FROM pg_stat_activity WHERE pid = $pid"), + "$what: blocked session is gone"); +} + +my $primary = PostgreSQL::Test::Cluster->new('primary'); +$primary->init(allows_streaming => 1, no_data_checksums => 1); +$primary->append_conf('postgresql.conf', 'procsignal_barrier_timeout = 2s'); +$primary->start; +$primary->safe_psql('postgres', 'CREATE DATABASE other'); +$primary->safe_psql('postgres', 'CREATE DATABASE target1'); +$primary->safe_psql('postgres', 'CREATE DATABASE target2'); +$primary->safe_psql('postgres', 'CREATE DATABASE target3'); +$primary->safe_psql('postgres', 'CREATE DATABASE target4'); + +my ($h, $pid, $log_offset, $start, $elapsed); + +# +# DROP DATABASE waits for every process to close its files of the dropped +# database, including a session connected to another database. +# +($h, $pid) = start_stuck_session($primary, 'other', 'stuck_dropdb'); +$log_offset = -s $primary->logfile; +$start = time; +$primary->safe_psql('postgres', 'DROP DATABASE target1'); +$elapsed = time - $start; +is( $primary->safe_psql( + 'postgres', + "SELECT count(*) FROM pg_database WHERE datname = 'target1'"), + '0', + 'DROP DATABASE: completed with a blocked session on another database'); +cmp_ok($elapsed, '>=', 2, 'DROP DATABASE: waited for the timeout'); +check_terminated($primary, $pid, $log_offset, 'DROP DATABASE'); +$h->kill_kill; + +# +# Enabling data checksums online. +# +($h, $pid) = start_stuck_session($primary, 'postgres', 'stuck_checksums'); +$log_offset = -s $primary->logfile; +$primary->safe_psql('postgres', 'SELECT pg_enable_data_checksums()'); +ok( $primary->poll_query_until( + 'postgres', + "SELECT setting FROM pg_settings WHERE name = 'data_checksums'", 'on'), + 'pg_enable_data_checksums(): reached "on" with a blocked session'); +check_terminated($primary, $pid, $log_offset, 'pg_enable_data_checksums()'); +$h->kill_kill; + +# +# A WAL sender blocked sending a base backup to a client that has stopped +# reading. +# +my ($bb_out, $bb_err) = ('', ''); +$h = IPC::Run::start( + [ + 'pg_basebackup', + '--pgdata' => '-', + '--format' => 'tar', + '--wal-method' => 'none', + '--checkpoint' => 'fast', + '--dbname' => $primary->connstr('postgres') + . ' application_name=stuck_walsender' + ], + '>' => \$bb_out, + '2>' => \$bb_err); +$pid = session_pid($primary, 'stuck_walsender', 'ClientWrite'); +is( $primary->safe_psql( + 'postgres', "SELECT backend_type FROM pg_stat_activity WHERE pid = $pid"), + 'walsender', + 'base backup is served by a WAL sender'); +$log_offset = -s $primary->logfile; +$primary->safe_psql('postgres', 'DROP DATABASE target4'); +check_terminated($primary, $pid, $log_offset, 'WAL sender'); +$h->kill_kill; + +# +# Negative control: idle sessions, a session in pg_sleep() and a client that +# reads slowly but keeps reading are not terminated, and do not delay the +# operations. +# +my $idle = $primary->background_psql('postgres'); +my $idle_pid = $idle->query_safe('SELECT pg_backend_pid()'); +my $sleep_out = ''; +my $sleeper = start_psql($primary, 'postgres', 'sleeper', + 'SELECT pg_sleep(600)', \$sleep_out); +my $sleeper_pid = session_pid($primary, 'sleeper', 'PgSleep'); +my $slow_out = ''; +my $slow = start_psql($primary, 'other', 'slow_reader', $copy_query, + \$slow_out); +my $slow_pid = session_pid($primary, 'slow_reader', 'ClientWrite'); + +# Run $sql to completion while reading a little from the slow client now +# and then. Returns the elapsed time. +sub run_while_reading_slowly +{ + my ($sql) = @_; + my $out = ''; + my $t0 = time; + my $op = start_psql($primary, 'postgres', 'op', $sql, \$out); + + while ($op->pumpable) + { + $op->pump_nb; + $slow->pump_nb; + $slow_out = ''; + usleep(50_000); + } + $op->finish; + return time - $t0; +} + +$log_offset = -s $primary->logfile; +$elapsed = run_while_reading_slowly('DROP DATABASE target2'); +is( $primary->safe_psql( + 'postgres', + "SELECT count(*) FROM pg_database WHERE datname = 'target2'"), + '0', + 'negative control: DROP DATABASE completed'); +cmp_ok($elapsed, '<', 2, + 'negative control: DROP DATABASE did not wait for the timeout'); +$elapsed = run_while_reading_slowly('SELECT pg_disable_data_checksums()'); +ok( $primary->poll_query_until( + 'postgres', + "SELECT setting FROM pg_settings WHERE name = 'data_checksums'", 'off' + ), + 'negative control: data checksums disabled'); +ok(!$primary->log_contains('terminating process', $log_offset), + 'negative control: no session was terminated'); +ok( session_alive($primary, $idle_pid) + && session_alive($primary, $sleeper_pid) + && session_alive($primary, $slow_pid), + 'negative control: idle, sleeping and slowly reading sessions survived'); +$idle->quit; +$sleeper->kill_kill; +$slow->kill_kill; + +# +# With procsignal_barrier_timeout = 0, the operation keeps waiting for the +# blocked session. +# +$primary->append_conf('postgresql.conf', 'procsignal_barrier_timeout = 0'); +$primary->reload; +$primary->poll_query_until('postgres', 'SHOW procsignal_barrier_timeout', + '0') + or die 'timed out waiting for the reload'; +($h, $pid) = start_stuck_session($primary, 'other', 'stuck_disabled'); +$log_offset = -s $primary->logfile; +my $dropper = $primary->background_psql('postgres'); +$dropper->query_until(qr/dropping/, "\\echo dropping\nDROP DATABASE target3;\n"); +$primary->wait_for_log( + qr/still waiting for backend with PID $pid to accept ProcSignalBarrier/, + $log_offset); +ok(session_alive($primary, $pid), + 'timeout disabled: blocked session is not terminated'); +is( $primary->safe_psql( + 'postgres', + "SELECT count(*) FROM pg_database WHERE datname = 'target3'"), + '1', + 'timeout disabled: DROP DATABASE is still waiting'); +$primary->safe_psql('postgres', "SELECT pg_terminate_backend($pid)"); +$dropper->quit; +is( $primary->safe_psql( + 'postgres', + "SELECT count(*) FROM pg_database WHERE datname = 'target3'"), + '0', + 'timeout disabled: DROP DATABASE completed once the session was terminated' +); +$h->kill_kill; +$primary->append_conf('postgresql.conf', 'procsignal_barrier_timeout = 2s'); +$primary->reload; + +# +# Standby: replay of the record that enables logical decoding with +# wal_level = replica, and the end of recovery when logical decoding is +# enabled, wait for every process to update its cached state. +# +$primary->backup('backup'); +my $standby = PostgreSQL::Test::Cluster->new('standby'); +$standby->init_from_backup($primary, 'backup', has_streaming => 1); +$standby->append_conf('postgresql.conf', 'procsignal_barrier_timeout = 2s'); +$standby->start; +$primary->wait_for_replay_catchup($standby); + +($h, $pid) = start_stuck_session($standby, 'postgres', 'stuck_replay'); +$log_offset = -s $standby->logfile; +$primary->safe_psql('postgres', + "SELECT pg_create_logical_replication_slot('primary_slot', 'pgoutput')"); +$primary->wait_for_replay_catchup($standby); +is($standby->safe_psql('postgres', 'SHOW effective_wal_level'), + 'logical', 'replay: logical decoding enabled with a blocked session'); +check_terminated($standby, $pid, $log_offset, 'replay'); +$h->kill_kill; + +$standby->create_logical_slot_on_standby($primary, 'standby_slot', + 'postgres'); +$idle = $standby->background_psql('postgres'); +$idle_pid = $idle->query_safe('SELECT pg_backend_pid()'); +($h, $pid) = start_stuck_session($standby, 'postgres', 'stuck_promote'); +$log_offset = -s $standby->logfile; +$standby->promote; +is($standby->safe_psql('postgres', 'SELECT pg_is_in_recovery()'), + 'f', 'promotion: completed with a blocked session'); +check_terminated($standby, $pid, $log_offset, 'promotion'); +ok(session_alive($standby, $idle_pid), + 'promotion: idle session survived'); +$idle->quit; +$h->kill_kill; + +done_testing(); -- 2.56.0