#!/bin/bash
# WAIT FOR LSN deadlocks a hot standby.  The dependency
#
#     backend B -- waits for replay --> startup S
#
# lives in the LSN-wait subsystem and is invisible to deadlock.c, which only
# reconstructs heavyweight-lock edges.  Two shapes close a cycle with it:
#
#   2-way (default)  B -> S -> B
#     B takes AccessShareLock(t) in an EARLIER statement, then WAIT FORs an LSN
#     whose replay needs AccessExclusiveLock(t).  Startup blocks behind B.
#
#   3-way            B -> S -> C -> B
#     B holds a transaction advisory lock and WAIT FORs.  C holds
#     AccessShareLock(t) and waits for B's advisory lock.  Startup waits for
#     AccessExclusiveLock(t) behind C.  Here startup's probe *does* reach a real
#     lock waiter (C), and C *does* run the detector -- but C cannot see B -> S,
#     so it finds nothing.  This is why a relation-lock-only guard on WAIT FOR
#     would not be a complete fix: B never touches t.
#
# Nothing detects either shape: the recovery deadlock probe returns immediately
# at postgres.c's `GetAwaitedLock() == NULL` for B, which sleeps on a latch
# rather than in the lock manager.
#
#   ./c3_repro.sh          -1, 2-way -> permanent hang
#   ./c3_repro.sh 3s       3s, 2-way -> startup cancels B, replay resumes
#   ./c3_repro.sh -1 3     -1, 3-way -> permanent hang
#   ./c3_repro.sh 3s 3     3s, 3-way -> startup cancels C, replay resumes
set -u
DELAY=${1:--1}
WAYS=${2:-2}                           # 2 = B->S->B, 3 = B->S->C->B
INST=${3:-/Users/qqj/dev/postgres/inst}
B=/tmp/c3repro; S=/tmp/c3sock          # short socket path: macOS caps at 103 bytes
rm -rf $B $S; mkdir -p $B $S
trap 'for d in p s; do $INST/bin/pg_ctl -D $B/$d stop -m immediate; done >/dev/null 2>&1; rm -rf $S' EXIT

pri() { $INST/bin/psql -X -h $S -p 5501 -d postgres -At -F' | ' -c "$1"; }
sby() { $INST/bin/psql -X -h $S -p 5502 -d postgres -At -F' | ' -c "$1"; }
waiters() { sby "SELECT pid, backend_type, wait_event_type, wait_event, left(query,32)
                 FROM pg_stat_activity
                 WHERE backend_type = 'startup' OR wait_event = 'WaitForWalReplay'
                       OR wait_event_type = 'Lock'"; }

$INST/bin/initdb -D $B/p -A trust --no-sync >/dev/null
printf "port=5501\nunix_socket_directories='$S'\nlisten_addresses=''\n" >> $B/p/postgresql.conf
$INST/bin/pg_ctl -D $B/p -l $B/p.log -w start >/dev/null
pri "CREATE TABLE t(i int)" >/dev/null

$INST/bin/pg_basebackup -h $S -p 5501 -D $B/s -R -c fast --no-sync >/dev/null
cat >> $B/s/postgresql.conf <<EOF
port=5502
unix_socket_directories='$S'
listen_addresses=''
max_standby_streaming_delay=$DELAY
deadlock_timeout=1s
log_recovery_conflict_waits=on
EOF
$INST/bin/pg_ctl -D $B/s -l $B/s.log -w start >/dev/null

# Two standby sessions on fds 3 (B) and 4 (C); C is idle in the 2-way case.
mkfifo $B/b.fifo $B/c.fifo
$INST/bin/psql -X -h $S -p 5502 -d postgres > $B/b.out 2>&1 < $B/b.fifo &
exec 3> $B/b.fifo
$INST/bin/psql -X -h $S -p 5502 -d postgres > $B/c.out 2>&1 < $B/c.fifo &
exec 4> $B/c.fifo

# Take the locks in an EARLIER statement and leave the transaction open.  READ
# COMMITTED releases the snapshot but keeps the locks, which is exactly why
# WAIT FOR's snapshot check does not catch this.
if [ "$WAYS" = 3 ]; then
    echo "BEGIN; SELECT pg_advisory_xact_lock(42);" >&3   # B holds advisory L
    echo "BEGIN; SELECT count(*) FROM t;"           >&4   # C holds AccessShareLock(t)
    sleep 1
    echo "SELECT pg_advisory_xact_lock(42);"        >&4   # C -> B, blocks
else
    echo "BEGIN; SELECT count(*) FROM t;"           >&3   # B holds AccessShareLock(t)
fi
sleep 1

# Primary: AccessExclusiveLock on t, logged for the standby.  Target is past it.
pri "DROP TABLE t" >/dev/null
L=$(pri "SELECT pg_current_wal_insert_lsn()")
sleep 2                                # standby streams it; startup blocks

echo "== startup is already stalled on the lock:"; waiters

echo "WAIT FOR LSN '$L';" >&3          # B -> S closes the cycle
sleep 10

echo "== after 10s, ${WAYS}-way, max_standby_streaming_delay = $DELAY"
echo "target             $L"
echo "replay stuck at    $(sby 'SELECT pg_last_wal_replay_lsn()')"
waiters
echo "B session          $(tr '\n' ' ' < $B/b.out)"
[ "$WAYS" = 3 ] && echo "C session          $(tr '\n' ' ' < $B/c.out)"
echo "deadlock detected  $(grep -c 'deadlock detected' $B/s.log)"
grep -h 'still waiting\|Conflicting\|canceling' $B/s.log | tail -3