Re: pg19b1: stuck in LockBuffer

(gdb) fr 6
#6 0x0000564cb7d3e9c0 in BufferLockAcquire (buffer=<optimized out>, buf_hdr=0x7f437f064a40, mode=BUFFER_LOCK_SHARE) at ./build/../src/backend/storage/buffer/bufmgr.c:5997
5997 ./build/../src/backend/storage/buffer/bufmgr.c: No such file or directory.
(gdb) p *buf_hdr
$4 = {tag = {spcOid = 1663, dbOid = 59856131, relNumber = 120237979, forkNum = MAIN_FORKNUM, blockNum = 97}, buf_id = 14933, state = {value = 9007206830964744}, wait_backend_pgprocno = -1, io_wref = {aio_index = 4294967295,
generation_upper = 0, generation_lower = 35962}, lock_waiters = {head = 204, tail = 137}}
(gdb) p *entry
$5 = {buffer = 14934, status = 0 '\000', data = {refcount = 2, lockmode = BUFFER_LOCK_UNLOCK}}

From another backend:
(gdb) fr 8
#8 _bt_lockbuf (access=3, buf=14934, rel=0x7f437c307208) at ./build/../src/backend/access/nbtree/nbtpage.c:1070
1070 ./build/../src/backend/access/nbtree/nbtpage.c: No such file or directory.
(gdb) p *rel
$6 = {rd_locator = {spcOid = 1663, dbOid = 59856131, relNumber = 120237979}, rd_smgr = 0x564cb9b85c20, rd_refcnt = 1, rd_backend = -1, rd_islocaltemp = false, rd_isnailed = false, rd_isvalid = true, rd_indexvalid = false,
rd_statvalid = false, rd_createSubid = 0, rd_newRelfilelocatorSubid = 0, rd_firstRelfilelocatorSubid = 0, rd_droppedSubid = 0, rd_rel = 0x7f437c8d85a8, rd_att = 0x7f437c828468, rd_id = 2658, rd_lockInfo = {lockRelId = {
relId = 2658, dbId = 59856131}}, rd_rules = 0x0, rd_rulescxt = 0x0, trigdesc = 0x0, rd_rsdesc = 0x0, rd_fkeylist = 0x0, rd_fkeyvalid = false, rd_partkey = 0x0, rd_partkeycxt = 0x0, rd_partdesc = 0x0, rd_pdcxt = 0x0,
rd_partdesc_nodetached = 0x0, rd_pddcxt = 0x0, rd_partdesc_nodetached_xmin = 0, rd_partcheck = 0x0, rd_partcheckvalid = false, rd_partcheckcxt = 0x0, rd_indexlist = 0x0, rd_pkindex = 0, rd_ispkdeferrable = false,
rd_replidindex = 0, rd_statlist = 0x0, rd_attrsvalid = false, rd_keyattr = 0x0, rd_pkattr = 0x0, rd_idattr = 0x0, rd_hotblockingattr = 0x0, rd_summarizedattr = 0x0, rd_pubdesc = 0x0, rd_options = 0x0, rd_amhandler = 330,
rd_tableam = 0x0, rd_index = 0x7f437c8f5e20, rd_indextuple = 0x7f437c8f5de8, rd_indexcxt = 0x564cb9a7c7f0, rd_indam = 0x564cb825daa0 <amroutine>, rd_opfamily = 0x564cb9a7c910, rd_opcintype = 0x564cb9a7c920,
rd_support = 0x564cb9a7c930, rd_supportinfo = 0x564cb9a65560, rd_indoption = 0x564cb9a65978, rd_indexprs = 0x0, rd_indpred = 0x0, rd_exclops = 0x0, rd_exclprocs = 0x0, rd_exclstrats = 0x0, rd_indcollation = 0x564cb9a65968,
rd_opcoptions = 0x564cb9a65988, rd_amcache = 0x564cb9a7cb80, rd_fdwroutine = 0x0, rd_toastoid = 0, pgstat_enabled = true, pgstat_info = 0x564cb9aa1980}

ts=# SELECT relname FROM pg_class WHERE pg_relation_filenode(oid) = 120237979;
relname | pg_attribute_relid_attnam_index

#0 __futex_abstimed_wait_common (futex_word=futex_word(at)entry=0x7f4389e3fd80, expected=expected(at)entry=0, clockid=clockid(at)entry=0, abstime=abstime(at)entry=0x0, private=<optimized out>, cancel=cancel(at)entry=true)
at ./nptl/futex-internal.c:103
err = -512
clockbit = 256
op = <optimized out>
#1 0x00007f438d6981bb in __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word(at)entry=0x7f4389e3fd80, expected=expected(at)entry=0, clockid=clockid(at)entry=0, abstime=abstime(at)entry=0x0, private=<optimized out>)
at ./nptl/futex-internal.c:139
No locals.
#2 0x00007f438d6a2f0f in do_futex_wait (sem=sem(at)entry=0x7f4389e3fd80, abstime=0x0, clockid=0) at ./nptl/sem_waitcommon.c:111
err = <optimized out>
#3 0x00007f438d6a2fa0 in __new_sem_wait_slow64 (sem=sem(at)entry=0x7f4389e3fd80, abstime=0x0, clockid=0) at ./nptl/sem_waitcommon.c:183
_buffer = {__routine = 0x7f438d6a2ef0 <__sem_wait_cleanup>, __arg = 0x7f4389e3fd80, __canceltype = 2147434608, __prev = 0x0}
err = <optimized out>
d = 0
#4 0x00007f438d6a3009 in __new_sem_wait (sem=sem(at)entry=0x7f4389e3fd80) at ./nptl/sem_wait.c:42
No locals.
#5 0x0000564cb7cc3ca2 in PGSemaphoreLock (sema=0x7f4389e3fd80) at ./build/src/backend/port/pg_sema.c:320
errStatus = <optimized out>
__func__ = "PGSemaphoreLock"
#6 0x0000564cb7d3e9c0 in BufferLockAcquire (buffer=<optimized out>, buf_hdr=0x7f437f064a40, mode=BUFFER_LOCK_SHARE) at ./build/../src/backend/storage/buffer/bufmgr.c:5997
wait_event = 67108866
mustwait = true
entry = 0x564cb8295a80 <PrivateRefCountArray>
extraWaits = 0
#7 0x0000564cb7a14def in LockBuffer (mode=BUFFER_LOCK_SHARE, buffer=14934) at ./build/../src/include/storage/bufmgr.h:339
No locals.
...

On Wed, Jul 29, 2026 at 09:48:49PM -0500, Justin Pryzby wrote:
> Several backends are stuck at:
> #7 0x0000564cb7a14def in LockBuffer (mode=BUFFER_LOCK_SHARE, buffer=14934) at ./build/../src/include/storage/bufmgr.h:339
>
> process 2124119 #7 0x0000564cb7a14def in LockBuffer (mode=BUFFER_LOCK_SHARE, buffer=14934) at ./build/../src/include/storage/bufmgr.h:339
> process 2099630 #7 0x0000564cb7a13909 in LockBuffer (mode=BUFFER_LOCK_EXCLUSIVE, buffer=14934) at ./build/../src/include/storage/bufmgr.h:339
> process 2391859 #7 0x0000564cb7a13909 in LockBuffer (mode=BUFFER_LOCK_EXCLUSIVE, buffer=14934) at ./build/../src/include/storage/bufmgr.h:339
> process 2131445 #7 0x0000564cb7a13909 in LockBuffer (mode=BUFFER_LOCK_EXCLUSIVE, buffer=14934) at ./build/../src/include/storage/bufmgr.h:339
> process 2131446 #7 0x0000564cb7a13909 in LockBuffer (mode=BUFFER_LOCK_EXCLUSIVE, buffer=14934) at ./build/../src/include/storage/bufmgr.h:339
> process 2103100 #7 0x0000564cb7a13909 in LockBuffer (mode=BUFFER_LOCK_EXCLUSIVE, buffer=14934) at ./build/../src/include/storage/bufmgr.h:339
>
> The earliest stuck backend (by query_start) is:
> 2124119 | 2026-07-29 20:00:39.925199-05 | | BufferShared | autovacuum worker | autovacuum: VACUUM ANALYZE pg_catalog.pg_attribute
>
> This is 19~beta1-1~20260603.2015.g378668d50b4.pgdg12+1 on debian12 / linux 6.1.0-21-amd64
> pg_upgraded on Jun 3.
>
> (gdb) bt
> #0 __futex_abstimed_wait_common (futex_word=futex_word(at)entry=0x7f4389e3fd80, expected=expected(at)entry=0, clockid=clockid(at)entry=0, abstime=abstime(at)entry=0x0, private=<optimized out>, cancel=cancel(at)entry=true)
> at ./nptl/futex-internal.c:103
> #1 0x00007f438d6981bb in __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word(at)entry=0x7f4389e3fd80, expected=expected(at)entry=0, clockid=clockid(at)entry=0, abstime=abstime(at)entry=0x0, private=<optimized out>)
> at ./nptl/futex-internal.c:139
> #2 0x00007f438d6a2f0f in do_futex_wait (sem=sem(at)entry=0x7f4389e3fd80, abstime=0x0, clockid=0) at ./nptl/sem_waitcommon.c:111
> #3 0x00007f438d6a2fa0 in __new_sem_wait_slow64 (sem=sem(at)entry=0x7f4389e3fd80, abstime=0x0, clockid=0) at ./nptl/sem_waitcommon.c:183
> #4 0x00007f438d6a3009 in __new_sem_wait (sem=sem(at)entry=0x7f4389e3fd80) at ./nptl/sem_wait.c:42
> #5 0x0000564cb7cc3ca2 in PGSemaphoreLock (sema=0x7f4389e3fd80) at ./build/src/backend/port/pg_sema.c:320
> #6 0x0000564cb7d3e9c0 in BufferLockAcquire (buffer=<optimized out>, buf_hdr=0x7f437f064a40, mode=BUFFER_LOCK_SHARE) at ./build/../src/backend/storage/buffer/bufmgr.c:5997
> #7 0x0000564cb7a14def in LockBuffer (mode=BUFFER_LOCK_SHARE, buffer=14934) at ./build/../src/include/storage/bufmgr.h:339
> #8 _bt_lockbuf (access=1, buf=14934, rel=<optimized out>) at ./build/../src/backend/access/nbtree/nbtpage.c:1070
> #9 _bt_getbuf (access=1, blkno=97, rel=<optimized out>) at ./build/../src/backend/access/nbtree/nbtpage.c:858
> #10 _bt_rightsib_halfdeadflag (leafrightsib=97, rel=<optimized out>) at ./build/../src/backend/access/nbtree/nbtpage.c:1791
> #11 _bt_mark_page_halfdead (stack=0x564cb9afc908, leafbuf=3597, heaprel=0x7f437c8d9ca0, rel=<optimized out>) at ./build/../src/backend/access/nbtree/nbtpage.c:2160
> #12 _bt_pagedel (rel=<optimized out>, leafbuf=3597, vstate=0x7fff720f3ed0) at ./build/../src/backend/access/nbtree/nbtpage.c:2031
> #13 0x0000564cb7a1d5c6 in btvacuumpage (vstate=vstate(at)entry=0x7fff720f3ed0, buf=<optimized out>) at ./build/../src/backend/access/nbtree/nbtree.c:1712
> #14 0x0000564cb7a1de18 in btvacuumscan (info=info(at)entry=0x7fff720f40e0, stats=stats(at)entry=0x564cb9a4f128, callback=callback(at)entry=0x564cb7b77b80 <vac_tid_reaped>, callback_state=callback_state(at)entry=0x564cb9a4fa38,
> cycleid=<optimized out>) at ./build/../src/backend/access/nbtree/nbtree.c:1366
> #15 0x0000564cb7a1dfdf in btbulkdelete (info=0x7fff720f40e0, stats=0x564cb9a4f128, callback=0x564cb7b77b80 <vac_tid_reaped>, callback_state=0x564cb9a4fa38) at ./build/../src/backend/access/nbtree/nbtree.c:1138
> #16 0x0000564cb7b7a982 in vac_bulkdel_one_index (ivinfo=ivinfo(at)entry=0x7fff720f40e0, istat=istat(at)entry=0x0, dead_items=<optimized out>, dead_items_info=0x564cb9a4fa20) at ./build/../src/backend/commands/vacuum.c:2667
> #17 0x0000564cb7a07b8d in lazy_vacuum_one_index (vacrel=0x564cb9a4f208, reltuples=<optimized out>, istat=0x0, indrel=<optimized out>) at ./build/../src/backend/access/heap/vacuumlazy.c:3041
> #18 lazy_vacuum_all_indexes (vacrel=vacrel(at)entry=0x564cb9a4f208) at ./build/../src/backend/access/heap/vacuumlazy.c:2536
> #19 0x0000564cb7a0a01f in lazy_vacuum (vacrel=0x564cb9a4f208) at ./build/../src/backend/access/heap/vacuumlazy.c:2454
> #20 lazy_scan_heap (vacrel=0x564cb9a4f208) at ./build/../src/backend/access/heap/vacuumlazy.c:1604
> #21 heap_vacuum_rel (rel=<optimized out>, params=<optimized out>, bstrategy=<optimized out>) at ./build/../src/backend/access/heap/vacuumlazy.c:881
> #22 0x0000564cb7b7827b in table_relation_vacuum (bstrategy=0x564cb9a9c0e0, params=0x7fff720f4870, rel=0x7f437c8d9ca0) at ./build/../src/include/access/tableam.h:1780
> #23 vacuum_rel (relid=1249, relation=<optimized out>, params=..., bstrategy=bstrategy(at)entry=0x564cb9a9c0e0, isTopLevel=isTopLevel(at)entry=true) at ./build/../src/backend/commands/vacuum.c:2310
> #24 0x0000564cb7b7981c in vacuum (relations=<optimized out>, params=params(at)entry=0x564cb9a93e80, bstrategy=bstrategy(at)entry=0x564cb9a9c0e0, vac_context=vac_context(at)entry=0x564cb9ac4090, isTopLevel=isTopLevel(at)entry=true)
> at ./build/../src/backend/commands/vacuum.c:633
> #25 0x0000564cb7cc7ff1 in autovacuum_do_vac_analyze (bstrategy=<optimized out>, tab=0x564cb9a93e78) at ./build/../src/backend/postmaster/autovacuum.c:3367
> #26 do_autovacuum () at ./build/../src/backend/postmaster/autovacuum.c:2517
> #27 0x0000564cb7cc8557 in AutoVacWorkerMain (startup_data=<optimized out>, startup_data_len=<optimized out>) at ./build/../src/backend/postmaster/autovacuum.c:1637
> #28 0x0000564cb7ccdb64 in postmaster_child_launch (child_type=child_type(at)entry=B_AUTOVAC_WORKER, child_slot=404, startup_data=startup_data(at)entry=0x0, startup_data_len=startup_data_len(at)entry=0,
> client_sock=client_sock(at)entry=0x0) at ./build/../src/backend/postmaster/launch_backend.c:268
> #29 0x0000564cb7ccfb6d in StartChildProcess (type=B_AUTOVAC_WORKER) at ./build/../src/backend/postmaster/postmaster.c:4030
> #30 0x0000564cb7cd1e7c in StartAutovacuumWorker () at ./build/../src/backend/postmaster/postmaster.c:4094
> #31 process_pm_pmsignal () at ./build/../src/backend/postmaster/postmaster.c:3851
> #32 ServerLoop () at ./build/../src/backend/postmaster/postmaster.c:1721
> #33 0x0000564cb7cd3044 in PostmasterMain (argc=argc(at)entry=11, argv=argv(at)entry=0x564cb9959180) at ./build/../src/backend/postmaster/postmaster.c:1415
> #34 0x0000564cb79ac67c in main (argc=11, argv=0x564cb9959180) at ./build/../src/backend/main/main.c:231
>
> I don't need to restart this immediately, so might be able to collect
> more info.
>
> --
> Justin

--
Justin Pryzby
System Administrator
Telsasoft
+1-952-707-8581