From 973e2ef7711542e0bdd35cd5d320ff5618e61945 Mon Sep 17 00:00:00 2001 From: Zhiguo Zhou Date: Sat, 10 Oct 2026 15:48:51 +0800 Subject: [PATCH v8] Use 128-bit CAS for WAL insert reservation Update the current and previous WAL reservation positions with one CAS, replacing insertpos_lck while preserving the record prev-link. Add optional native 128-bit compare-exchange to the atomics layer. Provide initialization and reads of the low and high 64-bit halves. Separate reads need not form a consistent pair, which the reservation CAS validates. Store the current position in the low half and the previous position in the high half, regardless of byte order. Check for native 128-bit CAS in Autoconf and Meson alongside the existing atomic checks. Keep the WAL spinlock path when it is unavailable, and report the compiler-target capability during configuration. x86-64 builds may need -mcx16 to enable this path. Extend the atomic operations regression test and add a recovery test covering concurrent insertion, segment switches, checkpoints and WAL decoding across crash restarts. Reviewed-by: Andrew Kim Tested-by: Andrew Kim --- config/c-compiler.m4 | 22 +++ configure | 49 +++++- configure.ac | 9 +- meson.build | 35 +++- src/backend/access/transam/xlog.c | 177 ++++++++++++++++----- src/include/pg_config.h.in | 4 + src/include/port/atomics.h | 62 ++++++++ src/include/port/atomics/generic-gcc.h | 41 +++++ src/include/port/atomics/generic.h | 31 ++++ src/test/recovery/meson.build | 1 + src/test/recovery/t/061_wal_reservation.pl | 146 +++++++++++++++++ src/test/regress/regress.c | 52 ++++++ src/tools/pgindent/typedefs.list | 1 + 13 files changed, 583 insertions(+), 47 deletions(-) create mode 100644 src/test/recovery/t/061_wal_reservation.pl diff --git a/config/c-compiler.m4 b/config/c-compiler.m4 index f05b744667c..8bafcd21555 100644 --- a/config/c-compiler.m4 +++ b/config/c-compiler.m4 @@ -600,6 +600,28 @@ if test x"$pgac_cv_gcc_sync_int64_cas" = x"yes"; then AC_DEFINE(HAVE_GCC__SYNC_INT64_CAS, 1, [Define to 1 if you have __sync_val_compare_and_swap(int64_t *, int64_t, int64_t).]) fi])# PGAC_HAVE_GCC__SYNC_INT64_CAS +# PGAC_HAVE_GCC__SYNC_INT128_CAS +# ------------------------------ +# Check for native 128-bit __sync compare-and-swap. Require the compiler's +# native-CAS indicator and a successful link, excluding library emulation. +AC_DEFUN([PGAC_HAVE_GCC__SYNC_INT128_CAS], +[AC_CACHE_CHECK(for native builtin __sync int128 atomic operations, pgac_cv_gcc_sync_int128_cas, +[AC_LINK_IFELSE([AC_LANG_PROGRAM([ +#ifndef PG_INT128_TYPE +#error no usable 128-bit integer type +#endif +#ifndef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_16 +#error no native 128-bit compare-and-swap +#endif +], + [volatile unsigned PG_INT128_TYPE val __attribute__((aligned(16))) = 0; + return __sync_val_compare_and_swap(&val, 0, 37) != 0;])], + [pgac_cv_gcc_sync_int128_cas="yes"], + [pgac_cv_gcc_sync_int128_cas="no"])]) +if test x"$pgac_cv_gcc_sync_int128_cas" = x"yes"; then + AC_DEFINE(HAVE_GCC__SYNC_INT128_CAS, 1, [Define to 1 if you have native __sync_val_compare_and_swap for 128-bit integers.]) +fi])# PGAC_HAVE_GCC__SYNC_INT128_CAS + # PGAC_HAVE_GCC__ATOMIC_INT32_CAS # ------------------------------- # Check if the C compiler understands __atomic_compare_exchange_n() for 32bit diff --git a/configure b/configure index f3b135c95ca..d96b036c7e5 100755 --- a/configure +++ b/configure @@ -17594,7 +17594,7 @@ _ACEOF fi # Check for various atomic operations now that we have checked how to declare -# 64bit integers. +# 64bit and 128bit integers. { $as_echo "$as_me:${as_lineno-$LINENO}: checking for builtin __sync char locking functions" >&5 $as_echo_n "checking for builtin __sync char locking functions... " >&6; } if ${pgac_cv_gcc_sync_char_tas+:} false; then : @@ -17724,6 +17724,45 @@ if test x"$pgac_cv_gcc_sync_int64_cas" = x"yes"; then $as_echo "#define HAVE_GCC__SYNC_INT64_CAS 1" >>confdefs.h +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for native builtin __sync int128 atomic operations" >&5 +$as_echo_n "checking for native builtin __sync int128 atomic operations... " >&6; } +if ${pgac_cv_gcc_sync_int128_cas+:} false; then : + $as_echo_n "(cached) " >&6 +else + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +#ifndef PG_INT128_TYPE +#error no usable 128-bit integer type +#endif +#ifndef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_16 +#error no native 128-bit compare-and-swap +#endif + +int +main () +{ +volatile unsigned PG_INT128_TYPE val __attribute__((aligned(16))) = 0; + return __sync_val_compare_and_swap(&val, 0, 37) != 0; + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + pgac_cv_gcc_sync_int128_cas="yes" +else + pgac_cv_gcc_sync_int128_cas="no" +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $pgac_cv_gcc_sync_int128_cas" >&5 +$as_echo "$pgac_cv_gcc_sync_int128_cas" >&6; } +if test x"$pgac_cv_gcc_sync_int128_cas" = x"yes"; then + +$as_echo "#define HAVE_GCC__SYNC_INT128_CAS 1" >>confdefs.h + fi { $as_echo "$as_me:${as_lineno-$LINENO}: checking for builtin __atomic int32 atomic operations" >&5 $as_echo_n "checking for builtin __atomic int32 atomic operations... " >&6; } @@ -17792,6 +17831,14 @@ $as_echo "#define HAVE_GCC__ATOMIC_INT64_CAS 1" >>confdefs.h fi +if test x"$pgac_cv_gcc_sync_int128_cas" = x"yes"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: native 128-bit CAS is available for the compiler target" >&5 +$as_echo "$as_me: native 128-bit CAS is available for the compiler target" >&6;} +else + { $as_echo "$as_me:${as_lineno-$LINENO}: native 128-bit CAS is unavailable; WAL reservation will use spinlock fallback" >&5 +$as_echo "$as_me: native 128-bit CAS is unavailable; WAL reservation will use spinlock fallback" >&6;} +fi + # Check for __get_cpuid() and __cpuid() { $as_echo "$as_me:${as_lineno-$LINENO}: checking for __get_cpuid" >&5 diff --git a/configure.ac b/configure.ac index 18bbe596612..9a69be07d09 100644 --- a/configure.ac +++ b/configure.ac @@ -2106,14 +2106,21 @@ AC_DEFINE_UNQUOTED(MAXIMUM_ALIGNOF, $MAX_ALIGNOF, [Define as the maximum alignme PGAC_TYPE_128BIT_INT # Check for various atomic operations now that we have checked how to declare -# 64bit integers. +# 64bit and 128bit integers. PGAC_HAVE_GCC__SYNC_CHAR_TAS PGAC_HAVE_GCC__SYNC_INT32_TAS PGAC_HAVE_GCC__SYNC_INT32_CAS PGAC_HAVE_GCC__SYNC_INT64_CAS +PGAC_HAVE_GCC__SYNC_INT128_CAS PGAC_HAVE_GCC__ATOMIC_INT32_CAS PGAC_HAVE_GCC__ATOMIC_INT64_CAS +if test x"$pgac_cv_gcc_sync_int128_cas" = x"yes"; then + AC_MSG_NOTICE([native 128-bit CAS is available for the compiler target]) +else + AC_MSG_NOTICE([native 128-bit CAS is unavailable; WAL reservation will use spinlock fallback]) +fi + # Check for __get_cpuid() and __cpuid() AC_CACHE_CHECK([for __get_cpuid], [pgac_cv__get_cpuid], diff --git a/meson.build b/meson.build index 3238c849e1b..919a40e762d 100644 --- a/meson.build +++ b/meson.build @@ -2452,6 +2452,23 @@ int64_t expect = 0; __atomic_compare_exchange_n(&val, &expect, 37, 0, __ATOMIC_SEQ_CST, __ATOMIC_RELAXED);'''}, ] +have_sync_int128_cas = false +if cdata.has('PG_INT128_TYPE') + # Require native CAS, rather than accepting library-emulated operations. + atomic_checks += [ + {'name': 'HAVE_GCC__SYNC_INT128_CAS', + 'desc': 'native __sync_val_compare_and_swap(int128)', + 'test': ''' +#ifndef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_16 +#error no native 128-bit compare-and-swap +#endif +volatile unsigned @0@ val __attribute__((aligned(16))) = 0; +return __sync_val_compare_and_swap(&val, 0, 37) != 0;'''.format(cdata.get('PG_INT128_TYPE'))}, + ] +else + cdata.set('HAVE_GCC__SYNC_INT128_CAS', false) +endif + foreach check : atomic_checks test = ''' #include @@ -2460,13 +2477,21 @@ int main(void) @0@ }'''.format(check['test']) - cdata.set(check['name'], - cc.links(test, - name: check['desc'], - args: test_c_args) ? 1 : false - ) + have_atomic = cc.links(test, + name: check['desc'], + args: test_c_args) + cdata.set(check['name'], have_atomic ? 1 : false) + if check['name'] == 'HAVE_GCC__SYNC_INT128_CAS' + have_sync_int128_cas = have_atomic + endif endforeach +if have_sync_int128_cas + message('native 128-bit CAS is available for the compiler target') +else + message('native 128-bit CAS is unavailable; WAL reservation will use spinlock fallback') +endif + ############################################################### # Check for the availability of XSAVE intrinsics. diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index fd37ea4f607..4738a3ea2e3 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -398,28 +398,39 @@ typedef union WALInsertLockPadded static SessionBackupState sessionBackupState = SESSION_BACKUP_NONE; /* - * Shared state data for WAL insertion. + * WAL insert reservation positions, updated together by 128-bit CAS. + * + * CurrBytePos is the end of reserved WAL, where the next record will be + * inserted. PrevBytePos is the start of the previously reserved record, + * copied to the next record's prev-link. Both are "usable byte positions" + * rather than XLogRecPtrs (see XLogBytePosToRecPtr()). + * + * Store CurrBytePos in the low 64 bits and PrevBytePos in the high 64 bits, + * regardless of byte order. Without native 128-bit CAS, or when + * XL_FORCE_SPINLOCK_RESERVATION is defined, use insertpos_lck instead. */ +#if defined(PG_HAVE_ATOMIC_U128_SUPPORT) && !defined(XL_FORCE_SPINLOCK_RESERVATION) +#define XL_LOCKFREE_RESERVATION 1 +#endif /* lock-free reservation available */ + +/* Shared state data for WAL insertion. */ typedef struct XLogCtlInsert { +#ifdef XL_LOCKFREE_RESERVATION + /* CurrBytePos and PrevBytePos, advanced together by 128-bit CAS */ + pg_atomic_uint128 CurrPrevPos; +#else slock_t insertpos_lck; /* protects CurrBytePos and PrevBytePos */ - - /* - * CurrBytePos is the end of reserved WAL. The next record will be - * inserted at that position. PrevBytePos is the start position of the - * previously inserted (or rather, reserved) record - it is copied to the - * prev-link of the next record. These are stored as "usable byte - * positions" rather than XLogRecPtrs (see XLogBytePosToRecPtr()). - */ uint64 CurrBytePos; uint64 PrevBytePos; +#endif /* - * Make sure the above heavily-contended spinlock and byte positions are - * on their own cache line. In particular, the RedoRecPtr and full page - * write variables below should be on a different cache line. They are - * read on every WAL insertion, but updated rarely, and we don't want - * those reads to steal the cache line containing Curr/PrevBytePos. + * Make sure the above heavily-contended reservation state is on its own + * cache line. In particular, the RedoRecPtr and full page write variables + * below should be on a different cache line. They are read on every WAL + * insertion, but updated rarely, and we don't want those reads to steal + * the cache line containing Curr/PrevBytePos. */ char pad[PG_CACHE_LINE_SIZE]; @@ -451,6 +462,24 @@ typedef struct XLogCtlInsert WALInsertLockPadded *WALInsertLocks; } XLogCtlInsert; +/* Read CurrBytePos without needing a snapshot of both reservation positions. */ +static inline uint64 +GetInsertCurrBytePos(XLogCtlInsert *Insert) +{ + uint64 bytepos; + +#ifdef XL_LOCKFREE_RESERVATION + bytepos = pg_atomic_read_u128_low64(&Insert->CurrPrevPos); + pg_read_barrier(); +#else + SpinLockAcquire(&Insert->insertpos_lck); + bytepos = Insert->CurrBytePos; + SpinLockRelease(&Insert->insertpos_lck); +#endif + + return bytepos; +} + /* * Total shared-memory state for XLOG. */ @@ -882,8 +911,8 @@ XLogInsertRecord(XLogRecData *rdata, * record to the shared WAL buffer cache is a two-step process: * * 1. Reserve the right amount of space from the WAL. The current head of - * reserved space is kept in Insert->CurrBytePos, and is protected by - * insertpos_lck. + * reserved space is advanced together with the previous record's start, + * using a double-width CAS or insertpos_lck. * * 2. Copy the record to the reserved WAL space. This involves finding the * correct WAL buffer containing the reserved space, and copying the @@ -1192,8 +1221,8 @@ XLogInsertRecord(XLogRecData *rdata, * * This is the performance critical part of XLogInsert that must be serialized * across backends. The rest can happen mostly in parallel. Try to keep this - * section as short as possible, insertpos_lck can be heavily contended on a - * busy system. + * section as short as possible, since reservation can be heavily contended + * on a busy system. * * NB: The space calculation here must match the code in CopyXLogRecordToWAL, * where we actually copy the record to the reserved space. @@ -1217,15 +1246,37 @@ ReserveXLogInsertLocation(int size, XLogRecPtr *StartPos, XLogRecPtr *EndPos, Assert(size > SizeOfXLogRecord); /* - * The duration the spinlock needs to be held is minimized by minimizing - * the calculations that have to be done while holding the lock. The - * current tip of reserved WAL is kept in CurrBytePos, as a byte position - * that only counts "usable" bytes in WAL, that is, it excludes all WAL - * page headers. The mapping between "usable" byte positions and physical - * positions (XLogRecPtrs) can be done outside the locked region, and - * because the usable byte position doesn't include any headers, reserving - * X bytes from WAL is almost as simple as "CurrBytePos += X". + * Keep calculations outside the serialized reservation step where + * possible. The current tip of reserved WAL is kept in CurrBytePos, as a + * byte position that only counts "usable" bytes in WAL, that is, it + * excludes all WAL page headers. The mapping between "usable" byte + * positions and physical positions (XLogRecPtrs) can be done after + * reserving the bytes, and because the usable byte position doesn't + * include any headers, reserving X bytes from WAL is almost as simple as + * "CurrBytePos += X". */ +#ifdef XL_LOCKFREE_RESERVATION + { + uint128 old_cp; + uint128 new_cp; + + /* + * Read each field atomically. If they come from different + * reservations, the CAS fails and returns a consistent pair. + */ + startbytepos = pg_atomic_read_u128_low64(&Insert->CurrPrevPos); + prevbytepos = pg_atomic_read_u128_high64(&Insert->CurrPrevPos); + old_cp = ((uint128) prevbytepos << 64) | startbytepos; + do + { + startbytepos = (uint64) old_cp; + endbytepos = startbytepos + size; + prevbytepos = (uint64) (old_cp >> 64); + new_cp = ((uint128) startbytepos << 64) | endbytepos; + } while (!pg_atomic_compare_exchange_u128(&Insert->CurrPrevPos, + &old_cp, new_cp)); + } +#else SpinLockAcquire(&Insert->insertpos_lck); startbytepos = Insert->CurrBytePos; @@ -1235,6 +1286,7 @@ ReserveXLogInsertLocation(int size, XLogRecPtr *StartPos, XLogRecPtr *EndPos, Insert->PrevBytePos = startbytepos; SpinLockRelease(&Insert->insertpos_lck); +#endif *StartPos = XLogBytePosToRecPtr(startbytepos); *EndPos = XLogBytePosToEndRecPtr(endbytepos); @@ -1270,11 +1322,47 @@ ReserveXLogSwitch(XLogRecPtr *StartPos, XLogRecPtr *EndPos, XLogRecPtr *PrevPtr) uint32 segleft; /* - * These calculations are a bit heavy-weight to be done while holding a - * spinlock, but since we're holding all the WAL insertion locks, there - * are no other inserters competing for it. GetXLogInsertRecPtr() does - * compete for it, but that's not called very frequently. + * Holding all the WAL insertion locks excludes other reserving inserters. + * In the spinlock path, readers such as GetXLogInsertRecPtr() can still + * compete for insertpos_lck, but those calls are not very frequent. */ +#ifdef XL_LOCKFREE_RESERVATION + { + uint128 old_cp; + uint128 new_cp; + + startbytepos = pg_atomic_read_u128_low64(&Insert->CurrPrevPos); + prevbytepos = pg_atomic_read_u128_high64(&Insert->CurrPrevPos); + old_cp = ((uint128) prevbytepos << 64) | startbytepos; + do + { + startbytepos = (uint64) old_cp; + + ptr = XLogBytePosToEndRecPtr(startbytepos); + if (XLogSegmentOffset(ptr, wal_segment_size) == 0) + { + *EndPos = *StartPos = ptr; + return false; + } + + endbytepos = startbytepos + size; + prevbytepos = (uint64) (old_cp >> 64); + + *StartPos = XLogBytePosToRecPtr(startbytepos); + *EndPos = XLogBytePosToEndRecPtr(endbytepos); + + segleft = wal_segment_size - XLogSegmentOffset(*EndPos, wal_segment_size); + if (segleft != wal_segment_size) + { + /* consume the rest of the segment */ + *EndPos += segleft; + endbytepos = XLogRecPtrToBytePos(*EndPos); + } + new_cp = ((uint128) startbytepos << 64) | endbytepos; + } while (!pg_atomic_compare_exchange_u128(&Insert->CurrPrevPos, + &old_cp, new_cp)); + } +#else SpinLockAcquire(&Insert->insertpos_lck); startbytepos = Insert->CurrBytePos; @@ -1304,6 +1392,7 @@ ReserveXLogSwitch(XLogRecPtr *StartPos, XLogRecPtr *EndPos, XLogRecPtr *PrevPtr) Insert->PrevBytePos = startbytepos; SpinLockRelease(&Insert->insertpos_lck); +#endif *PrevPtr = XLogBytePosToRecPtr(prevbytepos); @@ -1620,9 +1709,7 @@ WaitXLogInsertionsToFinish(XLogRecPtr upto) return inserted; /* Read the current insert position */ - SpinLockAcquire(&Insert->insertpos_lck); - bytepos = Insert->CurrBytePos; - SpinLockRelease(&Insert->insertpos_lck); + bytepos = GetInsertCurrBytePos(Insert); reservedUpto = XLogBytePosToEndRecPtr(bytepos); /* @@ -5641,7 +5728,11 @@ XLOGShmemInit(void *arg) XLogCtl->data_checksum_is_local = ControlFile->data_checksum_is_local; SetLocalDataChecksumState(XLogCtl->data_checksum_version); +#ifdef XL_LOCKFREE_RESERVATION + AssertPointerAlignment(&XLogCtl->Insert.CurrPrevPos, 16); +#else SpinLockInit(&XLogCtl->Insert.insertpos_lck); +#endif SpinLockInit(&XLogCtl->info_lck); pg_atomic_init_u64(&XLogCtl->logInsertResult, InvalidXLogRecPtr); pg_atomic_init_u64(&XLogCtl->logWriteResult, InvalidXLogRecPtr); @@ -6728,8 +6819,18 @@ StartupXLOG(void) * previous incarnation. */ Insert = &XLogCtl->Insert; +#ifdef XL_LOCKFREE_RESERVATION + { + uint128 positions; + + positions = (uint128) XLogRecPtrToBytePos(endOfRecoveryInfo->lastRec) << 64; + positions |= XLogRecPtrToBytePos(EndOfLog); + pg_atomic_init_u128(&Insert->CurrPrevPos, positions); + } +#else Insert->PrevBytePos = XLogRecPtrToBytePos(endOfRecoveryInfo->lastRec); Insert->CurrBytePos = XLogRecPtrToBytePos(EndOfLog); +#endif /* * Tricky point here: lastPage contains the *last* block that the LastRec @@ -7824,7 +7925,7 @@ CreateCheckPoint(int flags) if (shutdown) { - XLogRecPtr curInsert = XLogBytePosToRecPtr(Insert->CurrBytePos); + XLogRecPtr curInsert = XLogBytePosToRecPtr(GetInsertCurrBytePos(Insert)); /* * Compute new REDO record ptr = location of next XLOG record. @@ -10632,9 +10733,7 @@ GetXLogInsertRecPtr(void) XLogCtlInsert *Insert = &XLogCtl->Insert; uint64 current_bytepos; - SpinLockAcquire(&Insert->insertpos_lck); - current_bytepos = Insert->CurrBytePos; - SpinLockRelease(&Insert->insertpos_lck); + current_bytepos = GetInsertCurrBytePos(Insert); return XLogBytePosToRecPtr(current_bytepos); } @@ -10648,9 +10747,7 @@ GetXLogInsertEndRecPtr(void) XLogCtlInsert *Insert = &XLogCtl->Insert; uint64 current_bytepos; - SpinLockAcquire(&Insert->insertpos_lck); - current_bytepos = Insert->CurrBytePos; - SpinLockRelease(&Insert->insertpos_lck); + current_bytepos = GetInsertCurrBytePos(Insert); return XLogBytePosToEndRecPtr(current_bytepos); } diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 2e5ed6f5693..7d6859d13aa 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -152,6 +152,10 @@ /* Define to 1 if you have __sync_lock_test_and_set(char *) and friends. */ #undef HAVE_GCC__SYNC_CHAR_TAS +/* Define to 1 if you have native __sync_val_compare_and_swap for 128-bit + integers. */ +#undef HAVE_GCC__SYNC_INT128_CAS + /* Define to 1 if you have __sync_val_compare_and_swap(int *, int, int). */ #undef HAVE_GCC__SYNC_INT32_CAS diff --git a/src/include/port/atomics.h b/src/include/port/atomics.h index a605ea81d07..51b2892f6d0 100644 --- a/src/include/port/atomics.h +++ b/src/include/port/atomics.h @@ -611,6 +611,68 @@ pg_atomic_monotonic_advance_u64(volatile pg_atomic_uint64 *ptr, uint64 target) return currval; } +/* + * The 128-bit operations are available only with native support. There is + * no simulated implementation; callers must provide their own fallback. + */ +#ifdef PG_HAVE_ATOMIC_U128_SUPPORT + +/* + * pg_atomic_init_u128 - initialize atomic variable before concurrent use. + * + * No barrier semantics. + */ +static inline void +pg_atomic_init_u128(volatile pg_atomic_uint128 *ptr, uint128 val) +{ + AssertPointerAlignment(ptr, 16); + pg_atomic_init_u128_impl(ptr, val); +} + +/* + * pg_atomic_compare_exchange_u128 - compare and exchange a 128-bit value. + * + * Has the same semantics as pg_atomic_compare_exchange_u32(). + */ +static inline bool +pg_atomic_compare_exchange_u128(volatile pg_atomic_uint128 *ptr, + uint128 *expected, uint128 newval) +{ + AssertPointerAlignment(ptr, 16); + AssertPointerAlignment(expected, alignof(uint128)); + return pg_atomic_compare_exchange_u128_impl(ptr, expected, newval); +} + +/* + * pg_atomic_read_u128_low64 - atomically read bits 63..0. + * + * The bit range is independent of byte order. Reading both halves does + * not give an atomic snapshot of the 128-bit value; use compare-exchange + * to validate a value assembled from separate reads. + * + * No barrier semantics. + */ +static inline uint64 +pg_atomic_read_u128_low64(volatile pg_atomic_uint128 *ptr) +{ + AssertPointerAlignment(ptr, 16); + return pg_atomic_read_u128_low64_impl(ptr); +} + +/* + * pg_atomic_read_u128_high64 - atomically read bits 127..64. + * + * Has the same semantics as pg_atomic_read_u128_low64(). + */ +static inline uint64 +pg_atomic_read_u128_high64(volatile pg_atomic_uint128 *ptr) +{ + AssertPointerAlignment(ptr, 16); + return pg_atomic_read_u128_high64_impl(ptr); +} + +#endif /* PG_HAVE_ATOMIC_U128_SUPPORT */ + #undef INSIDE_ATOMICS_H #endif /* ATOMICS_H */ diff --git a/src/include/port/atomics/generic-gcc.h b/src/include/port/atomics/generic-gcc.h index 5bfce82f687..f8b9f77b6f2 100644 --- a/src/include/port/atomics/generic-gcc.h +++ b/src/include/port/atomics/generic-gcc.h @@ -107,6 +107,47 @@ typedef struct pg_atomic_uint64 #endif /* defined(HAVE_GCC__ATOMIC_INT64_CAS) || defined(HAVE_GCC__SYNC_INT64_CAS) */ +/* Optional native 128-bit atomics, with independently readable 64-bit halves. */ +#if !defined(PG_HAVE_ATOMIC_U128_SUPPORT) && defined(HAVE_INT128) && \ + defined(HAVE_GCC__SYNC_INT128_CAS) && \ + defined(PG_HAVE_ATOMIC_U64_SUPPORT) && !defined(PG_DISABLE_64_BIT_ATOMICS) +#define PG_HAVE_ATOMIC_U128_SUPPORT +typedef union pg_atomic_uint128 +{ + alignas(16) volatile uint128 value; + struct + { +#ifdef WORDS_BIGENDIAN + pg_atomic_uint64 high; + pg_atomic_uint64 low; +#else + pg_atomic_uint64 low; + pg_atomic_uint64 high; +#endif + } parts; +} pg_atomic_uint128; + +StaticAssertDecl(sizeof(pg_atomic_uint64) == sizeof(uint64), + "native 64-bit atomics must fit in a 128-bit atomic"); +#endif + +#if !defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U128) && \ + defined(PG_HAVE_ATOMIC_U128_SUPPORT) && defined(HAVE_GCC__SYNC_INT128_CAS) +#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U128 +static inline bool +pg_atomic_compare_exchange_u128_impl(volatile pg_atomic_uint128 *ptr, + uint128 *expected, uint128 newval) +{ + bool ret; + uint128 current; + + current = __sync_val_compare_and_swap(&ptr->value, *expected, newval); + ret = current == *expected; + *expected = current; + return ret; +} +#endif + #ifdef PG_HAVE_ATOMIC_FLAG_SUPPORT #if defined(HAVE_GCC__SYNC_CHAR_TAS) || defined(HAVE_GCC__SYNC_INT32_TAS) diff --git a/src/include/port/atomics/generic.h b/src/include/port/atomics/generic.h index daa772e9a6d..5b3eff31daa 100644 --- a/src/include/port/atomics/generic.h +++ b/src/include/port/atomics/generic.h @@ -428,3 +428,34 @@ pg_atomic_write_membarrier_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 val) (void) pg_atomic_exchange_u64_impl(ptr, val); } #endif + +#ifdef PG_HAVE_ATOMIC_U128_SUPPORT + +#ifndef PG_HAVE_ATOMIC_INIT_U128 +#define PG_HAVE_ATOMIC_INIT_U128 +static inline void +pg_atomic_init_u128_impl(volatile pg_atomic_uint128 *ptr, uint128 val) +{ + ptr->value = val; +} +#endif + +#ifndef PG_HAVE_ATOMIC_READ_U128_LOW64 +#define PG_HAVE_ATOMIC_READ_U128_LOW64 +static inline uint64 +pg_atomic_read_u128_low64_impl(volatile pg_atomic_uint128 *ptr) +{ + return pg_atomic_read_u64_impl(&ptr->parts.low); +} +#endif + +#ifndef PG_HAVE_ATOMIC_READ_U128_HIGH64 +#define PG_HAVE_ATOMIC_READ_U128_HIGH64 +static inline uint64 +pg_atomic_read_u128_high64_impl(volatile pg_atomic_uint128 *ptr) +{ + return pg_atomic_read_u64_impl(&ptr->parts.high); +} +#endif + +#endif /* PG_HAVE_ATOMIC_U128_SUPPORT */ diff --git a/src/test/recovery/meson.build b/src/test/recovery/meson.build index b6ac9dd07a7..4c1a35fb87b 100644 --- a/src/test/recovery/meson.build +++ b/src/test/recovery/meson.build @@ -69,6 +69,7 @@ tests += { 't/058_shutdown_crash_restart.pl', 't/059_remote_apply_status_interval.pl', 't/060_multixact_crashed_updater.pl', + 't/061_wal_reservation.pl', ], }, } diff --git a/src/test/recovery/t/061_wal_reservation.pl b/src/test/recovery/t/061_wal_reservation.pl new file mode 100644 index 00000000000..f6627ff6cf8 --- /dev/null +++ b/src/test/recovery/t/061_wal_reservation.pl @@ -0,0 +1,146 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group + +# Exercise WAL reservation with concurrent inserters, segment switches and +# checkpoints. Check recovery of records written after a checkpoint and WAL +# insertion after restart, then decode the WAL from both runs. + +use strict; +use warnings FATAL => 'all'; +use IPC::Run; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $node = PostgreSQL::Test::Cluster->new('main'); +$node->init(extra => ['--wal-segsize=1']); +$node->append_conf( + 'postgresql.conf', qq{ +autovacuum = off +checkpoint_timeout = '30min' +wal_buffers = '64kB' +wal_keep_size = '64MB' +}); +$node->start; +$node->safe_psql('postgres', + 'CREATE TABLE records (writer int, seq int, PRIMARY KEY (writer, seq))'); +$node->safe_psql('postgres', + 'CREATE TABLE recovery_records (id int PRIMARY KEY)'); + +# An empty segment needs no further switch record. +$node->safe_psql('postgres', 'SELECT pg_switch_wal()'); +my $start_lsn = + $node->safe_psql('postgres', 'SELECT pg_current_wal_insert_lsn()'); +$node->safe_psql('postgres', 'SELECT pg_switch_wal()'); +is( $node->safe_psql('postgres', 'SELECT pg_current_wal_insert_lsn()'), + $start_lsn, + 'switching an empty segment does not advance the insert position'); + +my $writer_script = $node->basedir . '/writer.sql'; +append_to_file( + $writer_script, q{ +\set seq :seq + 1 +BEGIN; +SELECT pg_advisory_xact_lock_shared(12345); +INSERT INTO records VALUES (:client_id, :seq); +SELECT pg_logical_emit_message(true, 'reservation', + repeat('x', CASE :client_id % 3 WHEN 0 THEN 16 WHEN 1 THEN 256 ELSE 8192 END)); +COMMIT; +}); +my $switch_script = $node->basedir . '/switch.sql'; +append_to_file( + $switch_script, q{ +SELECT pg_advisory_lock_shared(12345); +SELECT pg_switch_wal(); +CHECKPOINT; +SELECT pg_advisory_unlock_shared(12345); +}); + +# Hold all clients at a start barrier so both kinds of work are ready before +# releasing them. Timeouts cover both the barrier and the concurrent commands. +my $gate = $node->background_psql('postgres'); +$gate->query_safe('SELECT pg_advisory_lock(12345)'); +my ($writer_out, $writer_err, $switch_out, $switch_err) = ('', '', '', ''); +my $writers = IPC::Run::start( + [ + 'pgbench', '-n', '-c', '8', '-j', '2', '-t', '100', '-D', 'seq=0', + '-f', $writer_script, '-d', $node->connstr('postgres') + ], + '>' => \$writer_out, + '2>' => \$writer_err, + IPC::Run::timeout($PostgreSQL::Test::Utils::timeout_default)); +my $switches = IPC::Run::start( + [ + 'pgbench', '-n', '-c', '1', '-t', '8', '-f', $switch_script, + '-d', $node->connstr('postgres') + ], + '>' => \$switch_out, + '2>' => \$switch_err, + IPC::Run::timeout($PostgreSQL::Test::Utils::timeout_default)); +$node->poll_query_until('postgres', + q{SELECT count(*) = 9 FROM pg_locks WHERE locktype = 'advisory' AND NOT granted} +) or die 'WAL inserters did not reach the start barrier'; +$gate->query_safe('SELECT pg_advisory_unlock(12345)'); +$gate->quit; +ok($writers->finish, 'concurrent WAL writers complete') or diag($writer_err); +ok($switches->finish, 'concurrent switches and checkpoints complete') + or diag($switch_err); + +# The switcher may have checkpointed all of the concurrent inserts. Write +# another row after a completed checkpoint so recovery must read its WAL. +$node->safe_psql('postgres', 'CHECKPOINT'); +$node->safe_psql('postgres', 'INSERT INTO recovery_records VALUES (1)'); +my $recovery_lsn = + $node->safe_psql('postgres', 'SELECT pg_current_wal_insert_lsn()'); +$node->stop('immediate'); +$node->start; +is( $node->safe_psql( + 'postgres', + "SELECT pg_last_wal_replay_lsn() >= '$recovery_lsn'::pg_lsn"), + 't', + 'recovery replays through the post-checkpoint commit'); +is($node->safe_psql('postgres', 'SELECT id FROM recovery_records'), + '1', 'the post-checkpoint row survives crash recovery'); +is($node->safe_psql('postgres', 'SELECT count(*) FROM records'), + '800', 'all committed rows survive crash recovery'); +is( $node->safe_psql( + 'postgres', + 'SELECT count(*) FROM (SELECT writer FROM records GROUP BY writer HAVING count(*) = 100) s' + ), + '8', + 'every writer has all its committed rows after recovery'); + +# Exercise the reservation positions initialized by recovery. A second +# restart and the WAL scan below also check records written after restart. +$node->safe_psql('postgres', 'INSERT INTO recovery_records VALUES (2)'); +my $end_lsn = + $node->safe_psql('postgres', 'SELECT pg_current_wal_insert_lsn()'); +$node->stop('immediate'); +$node->start; +is( $node->safe_psql( + 'postgres', "SELECT pg_last_wal_replay_lsn() >= '$end_lsn'::pg_lsn"), + 't', + 'recovery replays through the commit written after restart'); +is( $node->safe_psql( + 'postgres', + q{SELECT string_agg(id::text, ',' ORDER BY id) FROM recovery_records} + ), + '1,2', + 'rows written before and after restart survive crash recovery'); + +# Decoding validates record CRCs and predecessor links across page and segment +# boundaries, including the switch records and records written after restart. +# Keep the inspected WAL range on disk through the checkpoints and restarts. +$node->command_checks_all( + [ + 'pg_waldump', '--quiet', + '--path', $node->data_dir . '/pg_wal', + '--start', $start_lsn, + '--end', $end_lsn + ], + 0, + [], + [], + 'WAL range can be decoded after crash recovery'); +$node->stop; + +done_testing(); diff --git a/src/test/regress/regress.c b/src/test/regress/regress.c index 6e61c0d201b..3e26a9f94af 100644 --- a/src/test/regress/regress.c +++ b/src/test/regress/regress.c @@ -750,6 +750,54 @@ test_atomic_uint64(void) EXPECT_EQ_U64(pg_atomic_fetch_and_u64(&var, ~0), 0); } +#ifdef PG_HAVE_ATOMIC_U128_SUPPORT +static void +test_atomic_uint128(void) +{ + pg_atomic_uint128 var; + uint128 original; + uint128 changed; + uint128 expected; + uint64 original_low = UINT64CONST(0x0123456789abcdef); + uint64 original_high = UINT64CONST(0xfedcba9876543210); + int i; + + original = ((uint128) original_high << 64) | original_low; + changed = PG_UINT64_MAX; + + pg_atomic_init_u128(&var, original); + EXPECT_EQ_U64(pg_atomic_read_u128_low64(&var), original_low); + EXPECT_EQ_U64(pg_atomic_read_u128_high64(&var), original_high); + + /* A failed exchange must return the complete current value. */ + expected = original ^ ((uint128) 1 << 127); + EXPECT_TRUE(!pg_atomic_compare_exchange_u128(&var, &expected, changed)); + EXPECT_TRUE(expected == original); + EXPECT_EQ_U64(pg_atomic_read_u128_low64(&var), original_low); + EXPECT_EQ_U64(pg_atomic_read_u128_high64(&var), original_high); + + /* Allow for spurious failures, as with the other CAS operations. */ + for (i = 0; i < 100; i++) + { + expected = original; + if (pg_atomic_compare_exchange_u128(&var, &expected, changed)) + break; + } + if (i == 100) + elog(ERROR, "atomic_compare_exchange_u128() never succeeded"); + EXPECT_TRUE(expected == original); + EXPECT_EQ_U64(pg_atomic_read_u128_low64(&var), PG_UINT64_MAX); + EXPECT_EQ_U64(pg_atomic_read_u128_high64(&var), 0); + + /* Neither half may change when the comparison fails. */ + expected = original; + EXPECT_TRUE(!pg_atomic_compare_exchange_u128(&var, &expected, original)); + EXPECT_TRUE(expected == changed); + EXPECT_EQ_U64(pg_atomic_read_u128_low64(&var), PG_UINT64_MAX); + EXPECT_EQ_U64(pg_atomic_read_u128_high64(&var), 0); +} +#endif /* PG_HAVE_ATOMIC_U128_SUPPORT */ + /* * Perform, fairly minimal, testing of the spinlock implementation. * @@ -829,6 +877,10 @@ test_atomic_ops(PG_FUNCTION_ARGS) test_atomic_uint64(); +#ifdef PG_HAVE_ATOMIC_U128_SUPPORT + test_atomic_uint128(); +#endif + /* * Arguably this shouldn't be tested as part of this function, but it's * closely enough related that that seems ok for now. diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index f10eb469266..a94fbd7e819 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -4009,6 +4009,7 @@ pendingPosition pending_label pgParameterStatus pg_atomic_flag +pg_atomic_uint128 pg_atomic_uint32 pg_atomic_uint64 pg_be_sasl_mech