#!/usr/bin/env bash # v1-0001 under a hammer: what happens if the TOAST relation is rewritten # over and over? # # Thom asked (2026-09-23) whether another rewrite can sneak in between # UnlockRelationOid() and stop_repack_decoding_worker(). Shihao answered that # it does not matter, because the old worker is discarded and nothing has been # copied yet. This does not argue with that: it measures it. A loop of # VACUUM FULL on the TOAST relation runs for the whole startup of REPACK, so # it lands inside that window many times over. # # Three things are measured: # 1. whether the result is still correct (the UPDATE is not lost), # 2. how many times the worker was restarted (the DEBUG1 the patch adds), # 3. whether REPACK finishes, and how fast: the loop in the patch has no # retry cap, so it matters whether it converges or spins. # # hammer.sh [build] [hammer_seconds] set -u B=${1:-/home/manu/pgtoast-i-fix} SECS=${2:-20} D=/home/manu/pgprog/data_hammer P=55703 LOG=/home/manu/pgprog/hammer.log REPACK_OUT=/home/manu/pgprog/hammer-repack.out "$B/bin/pg_ctl" -D "$D" -m immediate -w stop >/dev/null 2>&1 # The log lives OUTSIDE $D and pg_ctl -l appends: without this, the restart # count below carries over the previous run (it happened: 446 restarts # reported on a build that does not even have that message). rm -rf "$D" "$LOG" "$B/bin/initdb" -D "$D" -U postgres --no-sync -A trust >/dev/null 2>&1 cat >> "$D/postgresql.conf" <<'EOF' wal_level = logical max_replication_slots = 10 max_wal_senders = 10 log_min_messages = debug1 log_line_prefix = '%m [%p] ' EOF "$B/bin/pg_ctl" -D "$D" -o "-p $P" -l "$LOG" -w start >/dev/null 2>&1 q() { "$B/bin/psql" -p $P -U postgres -qtAX -c "$1" 2>&1; } q "CREATE TABLE test (id int PRIMARY KEY, big text)" >/dev/null q "ALTER TABLE test ALTER COLUMN big SET STORAGE EXTERNAL" >/dev/null q "INSERT INTO test SELECT g, repeat('old', 3000) FROM generate_series(1,3) g" >/dev/null TOAST=$(q "SELECT 'pg_toast.' || c2.relname FROM pg_class c1 JOIN pg_class c2 ON c2.oid = c1.reltoastrelid WHERE c1.relname='test'") echo "== toast: $TOAST build: $B" # A: an open transaction, so the worker has someone to wait for during setup ( "$B/bin/psql" -p $P -U postgres -qtAX \ -c "BEGIN" -c "SELECT pg_current_xact_id()" -c "SELECT pg_sleep(4)" -c "COMMIT" >/dev/null 2>&1 ) & sleep 0.5 # C: the hammer, rewriting the TOAST relation without pause ( until_t=$((SECONDS + SECS)) while [ $SECONDS -lt $until_t ]; do "$B/bin/psql" -p $P -U postgres -qtAX -c "VACUUM FULL $TOAST" >/dev/null 2>&1; done ) & HAMMER=$! # B: the REPACK that has to survive the hammer t0=$SECONDS ( q "REPACK (CONCURRENTLY) test" > "$REPACK_OUT" 2>&1 ) & REPACK=$! wait %1 2>/dev/null # let A commit q "UPDATE test SET big = repeat('NEW', 4000) WHERE id IN (1,2,3)" >/dev/null 2>&1 # Bound the wait on REPACK: if it never returns, that is exactly the data point waited=0 while kill -0 $REPACK 2>/dev/null && [ $waited -lt $((SECS + 60)) ]; do sleep 1; waited=$((waited+1)); done if kill -0 $REPACK 2>/dev/null; then echo "== REPACK DID NOT FINISH after ${waited}s (the hammer ran for ${SECS}s)"; finished=no else echo "== REPACK finished in ~$((SECONDS - t0))s"; finished=yes fi wait $HAMMER 2>/dev/null wait 2>/dev/null value=$(q "SELECT DISTINCT left(big,9) FROM test") restarts=$(grep -c 'restarting REPACK decoding worker' "$LOG") echo "== worker restarts: $restarts" echo "== REPACK output: $(head -2 "$REPACK_OUT" | tr '\n' ' ')" echo "== final value: $value (expected NEWNEWNEW)" # Reading the control run (a build WITHOUT the patch): there, 0 restarts is # normal (the message does not exist) and a correct value proves nothing, # because this scenario is not a reliable reproducer of the lost update -- # that is what the earlier race script is for, and it wins 5 times out of 5. # Here the control only tells us how long a REPACK takes when it ignores the # rewrites. [ "$finished" = yes ] && [ "$value" = "NEWNEWNEW" ] \ && echo "== result correct and REPACK finished" \ || echo "== CHECK: see $LOG" "$B/bin/pg_ctl" -D "$D" -m immediate -w stop >/dev/null 2>&1