Re: Deadlock in XLogInsert at AIX

One more assertion failure:

ExceptionalCondition(conditionName = "!(OldPageRqstPtr <=
XLogCtl->InitializedUpTo)", errorType = "FailedAssertion", fileName =
"xlog.c", lineNumber = 1887), line 54 in "assert.c"

(dbx) p OldPageRqstPtr
153551667200
(dbx) p XLogCtl->InitializedUpTo
153551667200
(dbx) p InitializedUpTo
153551659008

I slightly modify xlog.c code - store value of XLogCtl->InitializedUpTo
in local variable:

1870 LWLockAcquire(WALBufMappingLock, LW_EXCLUSIVE);
1871
1872 /*
1873 * Now that we have the lock, check if someone
initialized the page
1874 * already.
1875 */
1876 while (upto >= XLogCtl->InitializedUpTo || opportunistic)
1877 {
1878 XLogRecPtr InitializedUpTo =
XLogCtl->InitializedUpTo;
1879 nextidx = XLogRecPtrToBufIdx(InitializedUpTo);
1880
1881 /*
1882 * Get ending-offset of the buffer page we need
to replace (this may
1883 * be zero if the buffer hasn't been used yet).
Fall through if it's
1884 * already written out.
1885 */
1886 OldPageRqstPtr = XLogCtl->xlblocks[nextidx];
1887 Assert(OldPageRqstPtr <= XLogCtl->InitializedUpTo);

And, as you can see, XLogCtl->InitializedUpTo is not equal to saved
value InitializedUpTo.
But we are under exclusive WALBufMappingLock and InitializedUpTo is
updated only under this lock.
So it means that LW-locks doesn't work!
I inspected code of pg_atomic_compare_exchange_u32_impl and didn't sync
in prologue:

(dbx) listi pg_atomic_compare_exchange_u32_impl
0x1000817bc (pg_atomic_compare_exchange_u32_impl+0x1c)
e88100b0 ld r4,0xb0(r1)
0x1000817c0 (pg_atomic_compare_exchange_u32_impl+0x20)
e86100b8 ld r3,0xb8(r1)
0x1000817c4 (pg_atomic_compare_exchange_u32_impl+0x24)
800100c0 lwz r0,0xc0(r1)
0x1000817c8 (pg_atomic_compare_exchange_u32_impl+0x28) 7c0007b4
extsw r0,r0
0x1000817cc (pg_atomic_compare_exchange_u32_impl+0x2c)
e8a30002 lwa r5,0x0(r3)
0x1000817d0 (pg_atomic_compare_exchange_u32_impl+0x30) 7cc02028
lwarx r6,r0,r4,0x0
0x1000817d4 (pg_atomic_compare_exchange_u32_impl+0x34)
7c053040 cmpl cr0,0x0,r5,r6
0x1000817d8 (pg_atomic_compare_exchange_u32_impl+0x38)
4082000c bne 0x1000817e4
(pg_atomic_compare_exchange_u32_impl+0x44)
0x1000817dc (pg_atomic_compare_exchange_u32_impl+0x3c) 7c00212d
stwcx. r0,r0,r4
0x1000817e0 (pg_atomic_compare_exchange_u32_impl+0x40)
40e2fff0 bne+ 0x1000817d0
(pg_atomic_compare_exchange_u32_impl+0x30)
0x1000817e4 (pg_atomic_compare_exchange_u32_impl+0x44)
60c00000 ori r0,r6,0x0
0x1000817e8 (pg_atomic_compare_exchange_u32_impl+0x48)
90030000 stw r0,0x0(r3)
0x1000817ec (pg_atomic_compare_exchange_u32_impl+0x4c)
7c000026 mfcr r0
0x1000817f0 (pg_atomic_compare_exchange_u32_impl+0x50) 54001ffe
rlwinm r0,r0,0x3,0x1f,0x1f
0x1000817f4 (pg_atomic_compare_exchange_u32_impl+0x54) 78000620
rldicl r0,r0,0x0,0x19
0x1000817f8 (pg_atomic_compare_exchange_u32_impl+0x58)
98010070 stb r0,0x70(r1)
0x1000817fc (pg_atomic_compare_exchange_u32_impl+0x5c) 4c00012c
isync
0x100081800 (pg_atomic_compare_exchange_u32_impl+0x60)
88610070 lbz r3,0x70(r1)
0x100081804 (pg_atomic_compare_exchange_u32_impl+0x64)
48000004 b 0x100081808
(pg_atomic_compare_exchange_u32_impl+0x68)
0x100081808 (pg_atomic_compare_exchange_u32_impl+0x68)
38210080 addi r1,0x80(r1)
0x10008180c (pg_atomic_compare_exchange_u32_impl+0x6c)
4e800020 blr

Source code of pg_atomic_compare_exchange_u32_impl is the following:

static inline bool
pg_atomic_compare_exchange_u32_impl(volatile pg_atomic_uint32 *ptr,
uint32 *expected, uint32 newval)
{
bool ret;

/*
* atomics.h specifies sequential consistency ("full barrier
semantics")
* for this interface. Since "lwsync" provides acquire/release
* consistency only, do not use it here. GCC atomics observe the same
* restriction; see its rs6000_pre_atomic_barrier().
*/
__asm__ __volatile__ (" sync \n" ::: "memory");

/*
* XXX: __compare_and_swap is defined to take signed parameters,
but that
* shouldn't matter since we don't perform any arithmetic operations.
*/
ret = __compare_and_swap((volatile int*)&ptr->value,
(int *)expected, (int)newval);

/*
* xlc's documentation tells us:
* "If __compare_and_swap is used as a locking primitive, insert a
call to
* the __isync built-in function at the start of any critical
sections."
*
* The critical section begins immediately after __compare_and_swap().
*/
__isync();

return ret;
}

and if I compile this fuctions standalone, I get the following assembler
code:

.pg_atomic_compare_exchange_u32_impl: # 0x0000000000000000 (H.4.NO_SYMBOL)
stdu SP,-128(SP)
std r3,176(SP)
std r4,184(SP)
std r5,192(SP)
ld r0,192(SP)
stw r0,192(SP)
sync
ld r4,176(SP)
ld r3,184(SP)
lwz r0,192(SP)
extsw r0,r0
lwa r5,0(r3)
__L30: # 0x0000000000000030
(H.4.NO_SYMBOL+0x030)
lwarx r6,r0,r4
cmpl 0,0,r5,r6
bc BO_IF_NOT,CR0_EQ,__L44
stwcx. r0,r0,r4
.machine "any"
bc BO_IF_NOT_3,CR0_EQ,__L30
__L44: # 0x0000000000000044
(H.4.NO_SYMBOL+0x044)
ori r0,r6,0x0000
stw r0,0(r3)
mfcr r0
rlwinm r0,r0,3,31,31
rldicl r0,r0,0,56
stb r0,112(SP)
isync
lbz r3,112(SP)
addi SP,SP,128
bclr BO_ALWAYS,CR0_LT

sync is here!

--
Konstantin Knizhnik
Postgres Professional: http://www.postgrespro.com
The Russian Postgres Company