Re: 9.1.3 backends getting stuck in 'startup'

It's tough to catch the thing in the act on the new server as they always
eventually clear..though they still sit in startup for up to 30 seconds on
occasion.

A few here that look interesting:

#0 0x00007f62b6150583 in __select_nocancel () at
../sysdeps/unix/syscall-template.S:82
#1 0x00007f62b82948ca in pg_usleep (microsec=<value optimized out>) at
/build/buildd/postgresql-9.1-9.1.3/build/../src/port/pgsleep.c:43
#2 0x00007f62b8195119 in s_lock (lock=0x7f60a5ff3540 "", file=0x7f62b83a7408
"/build/buildd/postgresql-9.1-9.1.3/build/../src/backend/storage/lmgr/lwlock.c",
line=<value optimized out>) at
/build/buildd/postgresql-9.1-9.1.3/build/../src/backend/storage/lmgr/s_lock.c:113
#3 0x00007f62b8194cb7 in LWLockRelease (lockid=38) at
/build/buildd/postgresql-9.1-9.1.3/build/../src/backend/storage/lmgr/lwlock.c:593
#4 0x00007f62b81808dd in BufferAlloc (smgr=0x7f62b95a2910,
relpersistence=<value optimized out>, forkNum=<value optimized out>,
blockNum=<value optimized out>, mode=<value optimized out>, strategy=<value
optimized out>, hit=0x7fff9045d92f "") at
/build/buildd/postgresql-9.1-9.1.3/build/../src/backend/storage/buffer/bufmgr.c:745
#5 ReadBuffer_common (smgr=0x7f62b95a2910, relpersistence=<value optimized
out>, forkNum=<value optimized out>, blockNum=<value optimized out>,
mode=<value optimized out>, strategy=<value optimized out>, hit=0x7fff9045d92f
"") at
/build/buildd/postgresql-9.1-9.1.3/build/../src/backend/storage/buffer/bufmgr.c:325
#6 0x00007f62b81811e4 in ReadBufferExtended (reln=0x7f60a58bea30,
forkNum=MAIN_FORKNUM, blockNum=14895, mode=RBM_NORMAL, strategy=0x18a994d6) at
/build/buildd/postgresql-9.1-9.1.3/build/../src/backend/storage/buffer/bufmgr.c:246
#7 0x00007f62b7fd5110 in index_getnext (scan=0x7f62b978dc90, direction=<value
optimized out>) at
/build/buildd/postgresql-9.1-9.1.3/build/../src/backend/access/index/indexam.c:514
#8 0x00007f62b80e2946 in IndexNext (node=0x7f62b978cb08) at
/build/buildd/postgresql-9.1-9.1.3/build/../src/backend/executor/nodeIndexscan.c:78
#9 0x00007f62b80d77b7 in ExecScanFetch (node=0x7f62b978cb08, accessMtd=<value
optimized out>, recheckMtd=<value optimized out>) at
/build/buildd/postgresql-9.1-9.1.3/build/../src/backend/executor/execScan.c:82
#10 ExecScan (node=0x7f62b978cb08, accessMtd=<value optimized out>,
recheckMtd=<value optimized out>) at
/build/buildd/postgresql-9.1-9.1.3/build/../src/backend/executor/execScan.c:132
#11 0x00007f62b80d0068 in ExecProcNode (node=0x7f62b978cb08) at
/build/buildd/postgresql-9.1-9.1.3/build/../src/backend/executor/execProcnode.c:398
#12 0x00007f62b80e51c7 in ExecMergeJoin (node=0x7f62b978c6b8) at
/build/buildd/postgresql-9.1-9.1.3/build/../src/backend/executor/nodeMergejoin.c:1285
#13 0x00007f62b80cffc8 in ExecProcNode (node=0x7f62b978c6b8) at
/build/buildd/postgresql-9.1-9.1.3/build/../src/backend/executor/execProcnode.c:443
#14 0x00007f62b80dba99 in ExecAppend (node=0x7f62b9783ee0) at
/build/buildd/postgresql-9.1-9.1.3/build/../src/backend/executor/nodeAppend.c:209
#15 0x00007f62b80d00a8 in ExecProcNode (node=0x7f62b9783ee0) at
/build/buildd/postgresql-9.1-9.1.3/build/../src/backend/executor/execProcnode.c:375
#16 0x00007f62b80dda58 in agg_fill_hash_table (node=0x7f62b97837d0) at
/build/buildd/postgresql-9.1-9.1.3/build/../src/backend/executor/nodeAgg.c:1237
#17 ExecAgg (node=0x7f62b97837d0) at
/build/buildd/postgresql-9.1-9.1.3/build/../src/backend/executor/nodeAgg.c:1007
#18 0x00007f62b80cff78 in ExecProcNode (node=0x7f62b97837d0) at
/build/buildd/postgresql-9.1-9.1.3/build/../src/backend/executor/execProcnode.c:466
#19 0x00007f62b80ceed2 in ExecutePlan (queryDesc=0x7f62b9783130,
direction=NoMovementScanDirection, count=0) at
/build/buildd/postgresql-9.1-9.1.3/build/../src/backend/executor/execMain.c:1439
#20 standard_ExecutorRun (queryDesc=0x7f62b9783130,
direction=NoMovementScanDirection, count=0) at
/build/buildd/postgresql-9.1-9.1.3/build/../src/backend/executor/execMain.c:313
#21 0x00007f62b81a45c7 in PortalRunSelect (portal=0x7f62b94afdf0,
forward=<value optimized out>, count=0, dest=0x7f62b9543080) at
/build/buildd/postgresql-9.1-9.1.3/build/../src/backend/tcop/pquery.c:943
#22 0x00007f62b81a5a30 in PortalRun (portal=<value optimized out>,
count=<value optimized out>, isTopLevel=<value optimized out>, dest=<value
optimized out>, altdest=<value optimized out>, completionTag=<value optimized
out>) at /build/buildd/postgresql-9.1-9.1.3/build/../src/backend/tcop/pquery.c:787
#23 0x00007f62b81a310c in exec_execute_message (argc=<value optimized out>,
argv=<value optimized out>, username=<value optimized out>) at
/build/buildd/postgresql-9.1-9.1.3/build/../src/backend/tcop/postgres.c:1963
#24 PostgresMain (argc=<value optimized out>, argv=<value optimized out>,
username=<value optimized out>) at
/build/buildd/postgresql-9.1-9.1.3/build/../src/backend/tcop/postgres.c:3983
#25 0x00007f62b8161ad3 in BackendRun () at
/build/buildd/postgresql-9.1-9.1.3/build/../src/backend/postmaster/postmaster.c:3606
#26 BackendStartup () at
/build/buildd/postgresql-9.1-9.1.3/build/../src/backend/postmaster/postmaster.c:3291
#27 ServerLoop () at
/build/buildd/postgresql-9.1-9.1.3/build/../src/backend/postmaster/postmaster.c:1455
#28 0x00007f62b816457c in PostmasterMain (argc=-1186270896,
argv=0x7f62b9471190) at
/build/buildd/postgresql-9.1-9.1.3/build/../src/backend/postmaster/postmaster.c:1116
#29 0x00007f62b8102ec3 in main (argc=5, argv=0x7f62b9471170) at
/build/buildd/postgresql-9.1-9.1.3/build/../src/backend/main/main.c:199
=================
[Thread debugging using libthread_db enabled]
PinBuffer (buf=0x7f60ac8748f0, strategy=0x0)
at
/build/buildd/postgresql-9.1-9.1.3/build/../src/backend/storage/buffer/bufmgr.c:1060
1060
/build/buildd/postgresql-9.1-9.1.3/build/../src/backend/storage/buffer/bufmgr.c:
No such file or directory.
in
/build/buildd/postgresql-9.1-9.1.3/build/../src/backend/storage/buffer/bufmgr.c
#0 PinBuffer (buf=0x7f60ac8748f0, strategy=0x0) at
/build/buildd/postgresql-9.1-9.1.3/build/../src/backend/storage/buffer/bufmgr.c:1060
#1 0x00007f62b81808d2 in BufferAlloc (smgr=0x7f62b961be00,
relpersistence=<value optimized out>, forkNum=<value optimized out>,
blockNum=<value optimized out>, mode=<value optimized out>, strategy=<value
optimized out>, hit=0x7fff9045d98f "") at
/build/buildd/postgresql-9.1-9.1.3/build/../src/backend/storage/buffer/bufmgr.c:742
#2 ReadBuffer_common (smgr=0x7f62b961be00, relpersistence=<value optimized
out>, forkNum=<value optimized out>, blockNum=<value optimized out>,
mode=<value optimized out>, strategy=<value optimized out>, hit=0x7fff9045d98f
"") at
/build/buildd/postgresql-9.1-9.1.3/build/../src/backend/storage/buffer/bufmgr.c:325
#3 0x00007f62b81811e4 in ReadBufferExtended (reln=0x7f62b9787d78,
forkNum=MAIN_FORKNUM, blockNum=865, mode=RBM_NORMAL, strategy=0xa01962db) at
/build/buildd/postgresql-9.1-9.1.3/build/../src/backend/storage/buffer/bufmgr.c:246
#4 0x00007f62b7fc97bf in heapgetpage (scan=0x7f62b9874730, page=865) at
/build/buildd/postgresql-9.1-9.1.3/build/../src/backend/access/heap/heapam.c:223
#5 0x00007f62b7fc9cea in heapgettup_pagemode (scan=0x7f62b9874730, dir=<value
optimized out>, nkeys=<value optimized out>, key=0x0) at
/build/buildd/postgresql-9.1-9.1.3/build/../src/backend/access/heap/heapam.c:817
#6 0x00007f62b7fca786 in heap_getnext (scan=0x7f62b9874730,
direction=NoMovementScanDirection) at
/build/buildd/postgresql-9.1-9.1.3/build/../src/backend/access/heap/heapam.c:1342
#7 0x00007f62b80e7f8b in SeqNext (node=<value optimized out>) at
/build/buildd/postgresql-9.1-9.1.3/build/../src/backend/executor/nodeSeqscan.c:66
#8 0x00007f62b80d769a in ExecScanFetch (node=0x7f62b9873800, accessMtd=<value
optimized out>, recheckMtd=<value optimized out>) at
/build/buildd/postgresql-9.1-9.1.3/build/../src/backend/executor/execScan.c:82
#9 ExecScan (node=0x7f62b9873800, accessMtd=<value optimized out>,
recheckMtd=<value optimized out>) at
/build/buildd/postgresql-9.1-9.1.3/build/../src/backend/executor/execScan.c:167
#10 0x00007f62b80d0078 in ExecProcNode (node=0x7f62b9873800) at
/build/buildd/postgresql-9.1-9.1.3/build/../src/backend/executor/execProcnode.c:394
#11 0x00007f62b80e8a69 in ExecSort (node=0x7f62b9873560) at
/build/buildd/postgresql-9.1-9.1.3/build/../src/backend/executor/nodeSort.c:103
#12 0x00007f62b80cff98 in ExecProcNode (node=0x7f62b9873560) at
/build/buildd/postgresql-9.1-9.1.3/build/../src/backend/executor/execProcnode.c:458
#13 0x00007f62b80ceed2 in ExecutePlan (queryDesc=0x7f62b9871970,
direction=NoMovementScanDirection, count=0) at
/build/buildd/postgresql-9.1-9.1.3/build/../src/backend/executor/execMain.c:1439
#14 standard_ExecutorRun (queryDesc=0x7f62b9871970,
direction=NoMovementScanDirection, count=0) at
/build/buildd/postgresql-9.1-9.1.3/build/../src/backend/executor/execMain.c:313
#15 0x00007f62b81a45c7 in PortalRunSelect (portal=0x7f62b94afdf0,
forward=<value optimized out>, count=0, dest=0x7f62b9543080) at
/build/buildd/postgresql-9.1-9.1.3/build/../src/backend/tcop/pquery.c:943
#16 0x00007f62b81a5a30 in PortalRun (portal=<value optimized out>,
count=<value optimized out>, isTopLevel=<value optimized out>, dest=<value
optimized out>, altdest=<value optimized out>, completionTag=<value optimized
out>) at /build/buildd/postgresql-9.1-9.1.3/build/../src/backend/tcop/pquery.c:787
#17 0x00007f62b81a310c in exec_execute_message (argc=<value optimized out>,
argv=<value optimized out>, username=<value optimized out>) at
/build/buildd/postgresql-9.1-9.1.3/build/../src/backend/tcop/postgres.c:1963
#18 PostgresMain (argc=<value optimized out>, argv=<value optimized out>,
username=<value optimized out>) at
/build/buildd/postgresql-9.1-9.1.3/build/../src/backend/tcop/postgres.c:3983
#19 0x00007f62b8161ad3 in BackendRun () at
/build/buildd/postgresql-9.1-9.1.3/build/../src/backend/postmaster/postmaster.c:3606
#20 BackendStartup () at
/build/buildd/postgresql-9.1-9.1.3/build/../src/backend/postmaster/postmaster.c:3291
#21 ServerLoop () at
/build/buildd/postgresql-9.1-9.1.3/build/../src/backend/postmaster/postmaster.c:1455
#22 0x00007f62b816457c in PostmasterMain (argc=-1186270896,
argv=0x7f62b9471190) at
/build/buildd/postgresql-9.1-9.1.3/build/../src/backend/postmaster/postmaster.c:1116
#23 0x00007f62b8102ec3 in main (argc=5, argv=0x7f62b9471170) at
/build/buildd/postgresql-9.1-9.1.3/build/../src/backend/main/main.c:199

--
Jeff Frost <jeff(at)pgexperts(dot)com>
CTO, PostgreSQL Experts, Inc.
Phone: 1-888-PG-EXPRT x506
FAX: 415-762-5122
http://www.pgexperts.com/