Checks of v5 (0001 + 0002) v5: master 3e9bc39c5d7 + v5 ctl5: the same with src/backend reverted (v5's test, no fix) v5r19: REL_19_STABLE a4052efe2aa + v5 All --enable-cassert --enable-injection-points. ===== output of v5_all.sh ===== == build: /home/manu/pgtoast-i-ctl5 (postgres (PostgreSQL) 20devel), deadlock_timeout 1s race 1 REPACK ok 2.5s value oldoldold race 2 REPACK ok 2.5s value oldoldold race 3 REPACK ok 2.5s value oldoldold race 4 REPACK ok 2.5s value oldoldold race 5 REPACK ok 2.5s value oldoldold hammer 20s REPACK ok 3.5s value NEWNEWNEW none 1 REPACK ok 2.5s value NEWNEWNEW none 2 REPACK ok 2.5s value NEWNEWNEW none 3 REPACK ok 2.5s value NEWNEWNEW xidlock REPACK ok 2.0s value NEWNEWNEW,x session 1 (REINDEX TABLE pg_toast.pg_toast_31349): ok xidrewrite REPACK ok 2.0s value oldoldold,x session 1 (CLUSTER pg_toast.pg_toast_31377 USING pg_toast_31377_index): ok xidalter REPACK deadlock detected 1.0s value NEWNEWNEW,x DETAIL: Process 2026188 waits for ShareLock on transaction 4430; blocked by process 2026146. HINT: See server log for query details. session 1 (ALTER TABLE test ADD COLUMN extra int): ok log: 2026-09-25 16:29:54.813 -03 [2026188] ERROR: deadlock detected log: 2026-09-25 16:29:54.816 -03 [2026187] ERROR: deadlock detected == build: /home/manu/pgtoast-i-v5 (postgres (PostgreSQL) 20devel), deadlock_timeout 1s race 1 REPACK deadlock detected 3.6s value NEWNEWNEW DETAIL: Process 2026286 waits for ShareLock on transaction 668; blocked by process 2026340. HINT: See server log for query details. race 2 REPACK deadlock detected 3.7s value NEWNEWNEW DETAIL: Process 2026398 waits for ShareLock on transaction 675; blocked by process 2026401. HINT: See server log for query details. race 3 REPACK deadlock detected 3.7s value NEWNEWNEW DETAIL: Process 2026492 waits for ShareLock on transaction 682; blocked by process 2026495. HINT: See server log for query details. race 4 REPACK deadlock detected 3.7s value NEWNEWNEW DETAIL: Process 2026575 waits for ShareLock on transaction 689; blocked by process 2026625. HINT: See server log for query details. race 5 REPACK deadlock detected 3.7s value NEWNEWNEW DETAIL: Process 2026684 waits for ShareLock on transaction 696; blocked by process 2026717. HINT: See server log for query details. hammer 20s REPACK deadlock detected 4.5s value NEWNEWNEW DETAIL: Process 2026819 waits for ShareLock on transaction 704; blocked by process 2026823. HINT: See server log for query details. none 1 REPACK ok 2.5s value NEWNEWNEW none 2 REPACK ok 2.5s value NEWNEWNEW none 3 REPACK ok 2.5s value NEWNEWNEW xidlock REPACK deadlock detected 1.0s value NEWNEWNEW,x DETAIL: Process 2033502 waits for ShareLock on transaction 3783; blocked by process 2033490. HINT: See server log for query details. session 1 (REINDEX TABLE pg_toast.pg_toast_28785): ok xidrewrite REPACK deadlock detected 1.0s value NEWNEWNEW,x DETAIL: Process 2033578 waits for ShareLock on transaction 3789; blocked by process 2033554. HINT: See server log for query details. session 1 (CLUSTER pg_toast.pg_toast_28800 USING pg_toast_28800_index): ok xidalter REPACK deadlock detected 1.0s value NEWNEWNEW,x DETAIL: Process 2033692 waits for ShareLock on transaction 3795; blocked by process 2033686. HINT: See server log for query details. session 1 (ALTER TABLE test ADD COLUMN extra int): ok log: 2026-09-25 16:30:00.653 -03 [2026286] ERROR: deadlock detected log: 2026-09-25 16:30:00.655 -03 [2026285] ERROR: deadlock detected log: 2026-09-25 16:30:04.858 -03 [2026398] ERROR: deadlock detected == build: /home/manu/pgtoast-i-ctl5 (postgres (PostgreSQL) 20devel), deadlock_timeout 5s race 1 REPACK ok 2.5s value oldoldold race 2 REPACK ok 2.5s value oldoldold race 3 REPACK ok 2.5s value oldoldold race 4 REPACK ok 2.5s value oldoldold race 5 REPACK ok 2.5s value oldoldold hammer 20s REPACK ok 3.5s value NEWNEWNEW none 1 REPACK ok 2.5s value NEWNEWNEW none 2 REPACK ok 2.5s value NEWNEWNEW none 3 REPACK ok 2.5s value NEWNEWNEW xidlock REPACK ok 2.0s value NEWNEWNEW,x session 1 (REINDEX TABLE pg_toast.pg_toast_32397): ok xidrewrite REPACK ok 2.0s value oldoldold,x session 1 (CLUSTER pg_toast.pg_toast_32422 USING pg_toast_32422_index): ok xidalter REPACK deadlock detected 5.0s value NEWNEWNEW,x DETAIL: Process 2042720 waits for ShareLock on transaction 4692; blocked by process 2042715. HINT: See server log for query details. session 1 (ALTER TABLE test ADD COLUMN extra int): ok log: 2026-09-25 16:32:02.791 -03 [2042720] ERROR: deadlock detected log: 2026-09-25 16:32:02.794 -03 [2042719] ERROR: deadlock detected == build: /home/manu/pgtoast-i-v5 (postgres (PostgreSQL) 20devel), deadlock_timeout 5s race 1 REPACK ok 6.0s value NEWNEWNEW race 2 REPACK ok 6.0s value NEWNEWNEW race 3 REPACK ok 6.0s value NEWNEWNEW race 4 REPACK ok 6.0s value NEWNEWNEW race 5 REPACK ok 6.0s value NEWNEWNEW hammer 20s REPACK ok 5.1s value NEWNEWNEW none 1 REPACK ok 2.5s value NEWNEWNEW none 2 REPACK ok 2.5s value NEWNEWNEW none 3 REPACK ok 2.5s value NEWNEWNEW xidlock REPACK deadlock detected 5.0s value NEWNEWNEW,x DETAIL: Process 2049799 waits for ShareLock on transaction 3479; blocked by process 2049792. HINT: See server log for query details. session 1 (REINDEX TABLE pg_toast.pg_toast_27577): ok xidrewrite REPACK deadlock detected 5.0s value NEWNEWNEW,x DETAIL: Process 2049978 waits for ShareLock on transaction 3485; blocked by process 2049971. HINT: See server log for query details. session 1 (CLUSTER pg_toast.pg_toast_27592 USING pg_toast_27592_index): ok xidalter REPACK deadlock detected 5.0s value NEWNEWNEW,x DETAIL: Process 2050106 waits for ShareLock on transaction 3491; blocked by process 2050100. HINT: See server log for query details. session 1 (ALTER TABLE test ADD COLUMN extra int): ok log: 2026-09-25 16:32:10.940 -03 [2042937] ERROR: deadlock detected log: 2026-09-25 16:32:18.487 -03 [2043020] ERROR: deadlock detected log: 2026-09-25 16:32:26.034 -03 [2043162] ERROR: deadlock detected == injection_points check, ctl5 ok 1 - injection_points 69 ms ok 2 - hashagg 10 ms ok 3 - reindex_conc 10 ms ok 4 - vacuum 8 ms # All 4 tests passed. ok 1 - basic 43 ms ok 2 - inplace 588 ms ok 3 - reindex_concurrently_deferred 95 ms ok 4 - repack 167 ms ok 5 - repack_commit_race 492 ms ok 6 - repack_decode 248 ms ok 7 - repack_missingval 128 ms ok 8 - repack_temporal 140 ms ok 9 - repack_temporal_multirange 139 ms not ok 10 - repack_toast 497 ms ok 11 - ri_fastpath_reindex 317 ms ok 12 - ri_fastpath_snapshot 33 ms ok 13 - syscache-update-pruned 516 ms ok 14 - wait_cleanup 61 ms ok 15 - heap_lock_update 119 ms ok 16 - on_conflict_probe_window 146 ms # 1 of 16 tests failed. == injection_points check, v5 ok 1 - injection_points 70 ms ok 2 - hashagg 12 ms ok 3 - reindex_conc 10 ms ok 4 - vacuum 10 ms # All 4 tests passed. ok 1 - basic 57 ms ok 2 - inplace 605 ms ok 3 - reindex_concurrently_deferred 101 ms ok 4 - repack 172 ms ok 5 - repack_commit_race 497 ms ok 6 - repack_decode 249 ms ok 7 - repack_missingval 130 ms ok 8 - repack_temporal 140 ms ok 9 - repack_temporal_multirange 140 ms ok 10 - repack_toast 506 ms ok 11 - ri_fastpath_reindex 326 ms ok 12 - ri_fastpath_snapshot 59 ms ok 13 - syscache-update-pruned 441 ms ok 14 - wait_cleanup 65 ms ok 15 - heap_lock_update 122 ms ok 16 - on_conflict_probe_window 185 ms # All 16 tests passed. == injection_points check, v5r19 ok 1 - injection_points 70 ms ok 2 - hashagg 12 ms ok 3 - reindex_conc 11 ms ok 4 - vacuum 9 ms # All 4 tests passed. ok 1 - basic 49 ms ok 2 - inplace 493 ms ok 3 - reindex_concurrently_deferred 92 ms ok 4 - repack 174 ms ok 5 - repack_commit_race 457 ms ok 6 - repack_decode 251 ms ok 7 - repack_missingval 131 ms ok 8 - repack_temporal 142 ms ok 9 - repack_temporal_multirange 140 ms ok 10 - repack_toast 511 ms ok 11 - ri_fastpath_reindex 241 ms ok 12 - ri_fastpath_snapshot 59 ms ok 13 - syscache-update-pruned 427 ms ok 14 - wait_cleanup 63 ms ok 15 - heap_lock_update 104 ms ok 16 - on_conflict_probe_window 149 ms # All 16 tests passed. == repack_toast alone on v5 repack_toast, normal: 30 passed, 0 failed of 30 repack_toast, one-cpu: 30 passed, 0 failed of 30 == core regression + isolation, v5 # All 239 tests passed. # All 133 tests passed. == core regression + isolation, v5r19 # All 239 tests passed. # All 133 tests passed. V5-ALL-DONE ===== repack_toast.diffs on ctl5 ===== diff -U3 /home/manu/pgtoast-ctl5/src/test/modules/injection_points/expected/repack_toast.out /home/manu/pgtoast-ctl5/src/test/modules/injection_points/output_iso/results/repack_toast.out --- /home/manu/pgtoast-ctl5/src/test/modules/injection_points/expected/repack_toast.out 2026-09-25 16:27:31.345804460 -0300 +++ /home/manu/pgtoast-ctl5/src/test/modules/injection_points/output_iso/results/repack_toast.out 2026-09-25 16:33:39.158553380 -0300 @@ -153,7 +153,6 @@ $$; step s3_rewrite_toast: <... completed> -ERROR: canceling statement due to lock timeout step s3_noop: step s2_commit: COMMIT; @@ -264,9 +263,29 @@ 4 (1 row) -i|d1_j|d1_k|i|d2_j|d2_k|d1_j_tst|d2_j_tst|d1_k_tst|d2_k_tst --+----+----+-+----+----+--------+--------+--------+-------- -(0 rows) + i|d1_j |d1_k | i|d2_j |d2_k |d1_j_tst|d2_j_tst|d1_k_tst|d2_k_tst +--+------------+------------+--+------------+------------+--------+--------+--------+-------- + | | | 3|MTLOQXVAQMNK|e93028bdc1aa| | 21233| | 0 + 3|NAEPUVPXSSYJ|e93028bdc1aa| | | | 21212| | 0| + | | | 4|a9b7ba70783b|1bd69c7df311| | 21235| | 0 + 4|ENOUIRMPNQBG|1bd69c7df311| | | | 21213| | 0| + | | | 5|08f90c1a4171|a35fe7f7fe82| | 21237| | 0 + 5|OXXFSAYTLVSM|a35fe7f7fe82| | | | 21214| | 0| + | | | 6|CTSKKFLJLRKC|a8c6dd982010| | 21240| | 21239 + 6|UNTFMJLQFINS|a8c6dd982010| | | | 21215| | 0| + | | | 7|DSWDHEUAMEJS|WOTJBBSSWUXQ| | 21244| | 21243 + 7|UCVTCQCOMSSV|708be71b9ab6| | | | 21216| | 0| + | | |13|KOMFHVXDHWMF|276756f75e1e| | 21234| | 0 +13|XQRQLOJAIWIK|276756f75e1e| | | | 21222| | 0| + | | |14|a9b7ba70783b|276756f75e1e| | 21236| | 0 +14|CWSRBLODIBBP|276756f75e1e| | | | 21223| | 0| + | | |15|08f90c1a4171|276756f75e1e| | 21238| | 0 +15|MOEWNGRTFMJW|276756f75e1e| | | | 21224| | 0| + | | |16|ELUKGVABNCQN|276756f75e1e| | 21242| | 21241 +16|HBTFGNMCAHJF|276756f75e1e| | | | 21225| | 0| + | | |17|OKQUBIQHPEII|APAYNYNUEWFD| | 21246| | 21245 +17|SGFDJRVYLYUF|276756f75e1e| | | | 21226| | 0| +(20 rows) injection_points_detach ----------------------- ===== build_v5.sh ===== #!/bin/bash # Three builds for the v5 review, --enable-cassert --enable-injection-points: # v5 current master + v5-0001 + v5-0002 # ctl5 the same, with src/backend reverted: v5's test without the fix # v5r19 current REL_19_STABLE + v5-0001 + v5-0002 (does it apply there?) set -eu SRC=$HOME/Proyectos/postgresql git -C $SRC fetch -q origin P=$(cd "$(dirname "$0")" && pwd)/shihao-v5 build() { # name base [revert] local name=$1 base=$2 revert=${3:-} local tree=$HOME/pgtoast-$name if [ ! -d $tree ]; then git -C $SRC worktree add -q --detach $tree $base git -C $tree am -q $P/v5-0001-*.patch $P/v5-0002-*.patch if [ -n "$revert" ]; then git -C $tree checkout -q HEAD~2 -- src/backend fi fi cd $tree ./configure --prefix=$HOME/pgtoast-i-$name --enable-cassert --enable-injection-points \ --enable-tap-tests > $HOME/pgtoast-configure-$name.log 2>&1 make -j"$(nproc)" -s > $HOME/pgtoast-make-$name.log 2>&1 make -s install > $HOME/pgtoast-install-$name.log 2>&1 make -C src/test/modules/injection_points -s install >> $HOME/pgtoast-install-$name.log 2>&1 echo "$name: base $(git -C $tree rev-parse --short HEAD~2), warnings=$(grep -c 'warning:' $HOME/pgtoast-make-$name.log), backend diff vs base: $(git -C $tree diff --stat HEAD~2 -- src/backend | tail -1)" } build v5 origin/master & build ctl5 origin/master revert & build v5r19 origin/REL_19_STABLE & wait echo BUILD-DONE ===== v5_all.sh ===== #!/bin/bash # Everything for the v5 review, in one run: # 1. behaviour checks (v3_check.sh) on ctl5 (v5's test, no fix) and on v5, # at deadlock_timeout 1s and 5s # 2. the injection_points tests on ctl5, v5 and v5r19 (REL_19_STABLE): # repack_toast should fail on ctl5 only # 3. repack_toast alone RUNS times on v5, then RUNS times with the whole # run pinned to one CPU, to look for timing-dependent output (the new # permutation relies on lock_timeout = 10 and the (*) marker) # 4. core regression + isolation on v5 and v5r19 set -u A=$(cd "$(dirname "$0")" && pwd) RUNS=${RUNS:-30} for dt in 1s 5s; do for b in ctl5 v5; do DT=$dt bash $A/v3_check.sh $HOME/pgtoast-i-$b echo done done for b in ctl5 v5 v5r19; do echo "== injection_points check, $b" make -C $HOME/pgtoast-$b/src/test/modules/injection_points check 2>&1 \ | grep -E '^(ok|not ok|# All|# [0-9]+ of)|tests (passed|failed)' | tail -30 [ $b = ctl5 ] && cp $HOME/pgtoast-ctl5/src/test/modules/injection_points/output_iso/regression.diffs \ $A/v5_ctl5_repack_toast.diffs 2>/dev/null done rt() { # label prefix... local label=$1; shift local pass=0 fail=0 for i in $(seq $RUNS); do if "$@" make -C $HOME/pgtoast-v5/src/test/modules/injection_points check \ REGRESS= ISOLATION=repack_toast > /tmp/claude-1000/rt5_$i.log 2>&1; then pass=$((pass + 1)) else fail=$((fail + 1)); cp /tmp/claude-1000/rt5_$i.log $A/v5_repack_toast_fail_${label}_$i.log cp $HOME/pgtoast-v5/src/test/modules/injection_points/output_iso/regression.diffs \ $A/v5_repack_toast_fail_${label}_$i.diffs 2>/dev/null fi done echo "repack_toast, $label: $pass passed, $fail failed of $RUNS" } echo "== repack_toast alone on v5" rt normal env rt one-cpu taskset -c 0 for b in v5 v5r19; do echo "== core regression + isolation, $b" make -C $HOME/pgtoast-$b -s check 2>&1 | grep -E '# All|# [0-9]+ of|not ok' | tail -5 make -C $HOME/pgtoast-$b/src/test/isolation -s check 2>&1 | grep -E '# All|# [0-9]+ of|not ok' | tail -5 done echo V5-ALL-DONE ===== v3_check.sh (behaviour checks, unchanged since the v3 review) ===== #!/usr/bin/env bash # The same checks as v2_check.sh, for v3 of the fix: REPACK (CONCURRENTLY) # now locks the TOAST relation before the decoding worker starts, so a # rewrite of it waits for REPACK. Every case runs against a build and # reports REPACK's own outcome (ok or its error), how long it took, and the # final value of the updated rows. # # New in v3_check.sh: # xidalter a transaction that already has an XID runs ALTER TABLE on the # table itself while the worker waits for it (v3's commit # message says master already deadlocks here). # # race Thom's case without an injection point: an open transaction # keeps the worker waiting, VACUUM FULL rewrites the TOAST # relation meanwhile, an UPDATE of the TOASTed column commits # right after the transaction ends. N attempts. # hammer VACUUM FULL of the TOAST relation in a loop for SECS seconds, # over the whole startup of REPACK (the v1 retry loop spun here). # none the same, with no rewrite at all: the normal path. # xidlock a transaction that already has an XID locks the TOAST # relation (REINDEX of it) while the worker waits for it: taking # the TOAST lock before starting the worker deadlocks here. # xidrewrite the same, but the transaction rewrites the TOAST relation # (CLUSTER of it) before committing. # # v2_check.sh [case ...] set -u B=$1; shift CASES=${*:-race hammer none xidlock xidrewrite xidalter} N=${N:-5} SECS=${SECS:-20} D=${D:-$HOME/pgprog/data_v3check} P=${P:-55711} LOG=$HOME/pgprog/v3check.log OUT=$HOME/pgprog/v3check-repack.out "$B/bin/pg_ctl" -D "$D" -m immediate -w stop >/dev/null 2>&1 rm -rf "$D" "$LOG" "$B/bin/initdb" -D "$D" -U postgres --no-sync -A trust >/dev/null 2>&1 cat >> "$D/postgresql.conf" </dev/null 2>&1 q() { "$B/bin/psql" -p $P -U postgres -qtAX -c "$1" 2>&1; } fresh() { # a new table with three TOASTed rows; sets TOAST q "DROP TABLE IF EXISTS test" >/dev/null q "CREATE TABLE test (id int PRIMARY KEY, big text)" >/dev/null q "ALTER TABLE test ALTER COLUMN big SET STORAGE EXTERNAL" >/dev/null q "INSERT INTO test SELECT g, repeat('old', 3000) FROM generate_series(1,3) g" >/dev/null TOAST=$(q "SELECT 'pg_toast.' || c2.relname FROM pg_class c1 JOIN pg_class c2 ON c2.oid = c1.reltoastrelid WHERE c1.relname='test'") } start_repack() { # in the background; sets REPACK and T0 T0=$(date +%s.%N) ( q "REPACK (CONCURRENTLY) test" > "$OUT" 2>&1; date +%s.%N > "$OUT.end" ) & REPACK=$! } report() { # label local limit=$((SECS + 60)) waited=0 while kill -0 $REPACK 2>/dev/null && [ $waited -lt $limit ]; do sleep 1; waited=$((waited+1)); done wait 2>/dev/null local secs=$(echo "$(cat "$OUT.end" 2>/dev/null || date +%s.%N) - $T0" | bc) local repack=$(grep -m1 -E 'ERROR|FATAL' "$OUT" | sed 's/^.*\(ERROR\|FATAL\): *//') local value=$(q "SELECT string_agg(DISTINCT left(big, 9), ',') FROM test") printf ' %-12s REPACK %-50s %5.1fs value %s\n' "$1" "${repack:-ok}" "$secs" "$value" [ -z "$repack" ] || grep -E '^(DETAIL|HINT):' "$OUT" | sed 's/^/ /' } open_xact() { # seconds: a transaction with an XID, closed after the sleep ( "$B/bin/psql" -p $P -U postgres -qtAX \ -c "BEGIN" -c "SELECT pg_current_xact_id()" -c "SELECT pg_sleep($1)" -c "COMMIT" >/dev/null 2>&1 ) & XACT=$! } update_new() { q "UPDATE test SET big = repeat('NEW', 4000) WHERE id IN (1,2,3)" >/dev/null } echo "== build: $B ($("$B/bin/postgres" --version)), deadlock_timeout ${DT:-1s}" for c in $CASES; do case $c in race) for i in $(seq 1 $N); do fresh; open_xact 3; sleep 0.5 start_repack; sleep 1 q "VACUUM FULL $TOAST" >/dev/null wait $XACT; update_new report "race $i" done ;; hammer) fresh; open_xact 4; sleep 0.5 ( until_t=$((SECONDS + SECS)) while [ $SECONDS -lt $until_t ]; do q "VACUUM FULL $TOAST" >/dev/null; done ) & HAMMER=$! start_repack wait $XACT; update_new report "hammer ${SECS}s" wait $HAMMER 2>/dev/null ;; none) for i in $(seq 1 3); do fresh; open_xact 3; sleep 0.5 start_repack wait $XACT; update_new report "none $i" done ;; xidlock|xidrewrite|xidalter) fresh # LOCK TABLE is refused on a TOAST relation, so use commands that # lock it for real: REINDEX takes ShareLock on it without a rewrite, # CLUSTER rewrites it (new relfilenumber). xidalter locks the table # itself instead. if [ $c = xidlock ]; then stmt="REINDEX TABLE $TOAST" elif [ $c = xidalter ]; then stmt="ALTER TABLE test ADD COLUMN extra int" else idx=$(q "SELECT c.relname FROM pg_index i JOIN pg_class c ON c.oid = i.indexrelid WHERE i.indrelid = '$TOAST'::regclass") stmt="CLUSTER $TOAST USING $idx" fi ( "$B/bin/psql" -p $P -U postgres -qtAX \ -c "BEGIN" -c "INSERT INTO test VALUES (100, 'x')" -c "SELECT pg_sleep(1.5)" \ -c "$stmt" -c "SELECT pg_sleep(1)" -c "COMMIT" > "$OUT.s1" 2>&1 ) & S1=$! sleep 0.5; start_repack wait $S1 s1=$(grep -m1 -E 'ERROR' "$OUT.s1" | sed 's/^.*ERROR: *//') update_new report "$c" echo " session 1 ($stmt): ${s1:-ok}" ;; esac done grep -E 'deadlock detected' "$LOG" | head -3 | sed 's/^/ log: /' "$B/bin/pg_ctl" -D "$D" -m immediate -w stop >/dev/null 2>&1