From 5813cbb09c08831ff7e1296e69d5ad1e4a585fec Mon Sep 17 00:00:00 2001 From: Greg Burd Date: Thu, 9 Jul 2026 12:34:55 -0400 Subject: [PATCH v4 1/3] Add an opt-in C11 stdatomic.h implementation of the atomics API PostgreSQL requires C11, but not all supported compilers provide a fully functional . This adds an alternative implementation of port/atomics.h built on the standard C11 header, selectable at build time alongside the existing platform-specific implementations. Build-system detection (meson and autoconf): A new option -- meson -Duse_stdatomic=yes|no|auto and autoconf --with-stdatomic=yes|no|auto -- controls the choice: * 'auto' (default): use stdatomic.h if a probe compile succeeds * 'yes': require stdatomic.h (fail if unavailable) * 'no': use the traditional platform-specific implementation Both build systems run the same probe (32-bit fetch_add, a seq_cst thread fence, and a 64-bit compare_exchange) and define USE_STDATOMIC_H when it succeeds and the option is not 'no'. For MSVC, stdatomic.h is only available in Visual Studio 2022 or later with /experimental:c11atomics, which the build adds automatically; earlier MSVC falls back to the traditional implementation. Implementation (src/include/port/atomics/stdatomic_impl.h): All atomic types (pg_atomic_flag, pg_atomic_uint8/16/32/64) are implemented with C11 _Atomic. Memory ordering matches the traditional implementation's observable semantics: * pg_atomic_read/write_* use memory_order_relaxed, matching the plain volatile loads/stores of the traditional path (no fence). * read-modify-write, exchange, and compare-exchange use memory_order_seq_cst, matching the traditional "full barrier" contract. * the _membarrier read/write variants use memory_order_seq_cst. C++ compatibility is provided via for C++11 through C++20, which do not include ; C++23 and later include it directly. Platform-specific spin-delay hints (PAUSE, ISB, YIELD) are factored into a new src/include/port/spin_delay.h, with the shared SpinDelayStatus type and helpers in src/include/port/spin_delay_status.h so both spinlock paths use them. On ARM64, ISB is used instead of YIELD for better backoff under contention; on Windows ARM64 this adds a spin delay where the traditional implementation had none. Wire-up (src/include/port/atomics.h, storage/spin.h, s_lock.c): When USE_STDATOMIC_H is defined, atomics.h includes stdatomic_impl.h; otherwise it includes the arch-*.h and generic-*.h headers exactly as before. Both paths feed the same shared public-API layer (the static inline pg_atomic_* wrappers), so the API is identical either way. For spinlocks, when USE_STDATOMIC_H is defined slock_t is a pg_atomic_uint32 (0 == unlocked, 1 == locked, so a zeroed slock_t is a valid free lock) and the SpinLock* operations are built on the u32 atomic API rather than platform-specific TAS assembly. SpinLockAcquire is a thin wrapper macro over an inline helper: the macro captures the call site's __FILE__/__LINE__/__func__ for stuck-spinlock diagnostics while the helper evaluates its lock argument exactly once. On strong-memory platforms a relaxed-load test precedes the atomic exchange (PG_SPIN_TRY_RELAXED). When USE_STDATOMIC_H is not defined, spinlock behavior is byte-for-byte unchanged from previous PostgreSQL versions. With stdatomic.h, atomic operations no longer depend on backend-only infrastructure, so the frontend include restriction in atomics.h is relaxed for that path. The traditional implementation remains the default , this is an opt-in alternative rather than a replacement. Both code paths provide identical public API, semantics, and roughly equal performance. Co-authored-by: Greg Burd Co-authored-by: Thomas Munro Discussion: https://postgr.es/m/CC76554F-41AC-45AA-AF10-370FEC416498%40greg.burd.me#dd4b139cd557d6e87a9f2d706b6415ae --- configure | 109 ++++ configure.ac | 66 +++ meson.build | 102 ++++ meson_options.txt | 5 + src/backend/storage/lmgr/README | 71 +++ src/backend/storage/lmgr/s_lock.c | 42 +- src/include/pg_config.h.in | 4 + src/include/port/atomics.h | 103 +++- src/include/port/atomics/stdatomic_impl.h | 589 ++++++++++++++++++++++ src/include/port/spin_delay.h | 151 ++++++ src/include/port/spin_delay_status.h | 52 ++ src/include/storage/spin.h | 76 +++ src/test/regress/regress.c | 12 + 13 files changed, 1372 insertions(+), 10 deletions(-) create mode 100644 src/include/port/atomics/stdatomic_impl.h create mode 100644 src/include/port/spin_delay.h create mode 100644 src/include/port/spin_delay_status.h diff --git a/configure b/configure index 35b0b72f0a7..f9853829bd3 100755 --- a/configure +++ b/configure @@ -759,6 +759,7 @@ LLVM_LIBS CLANG LLVM_CONFIG AWK +with_stdatomic with_llvm have_cxx ac_ct_CXX @@ -857,6 +858,7 @@ with_segsize with_segsize_blocks with_wal_blocksize with_llvm +with_stdatomic enable_depend enable_cassert with_icu @@ -1576,6 +1578,8 @@ Optional Packages: --with-wal-blocksize=BLOCKSIZE set WAL block size in kB [8] --with-llvm build with LLVM based JIT support + --with-stdatomic[=yes/no/auto] + use C11 stdatomic.h for atomic operations [auto] --without-icu build without ICU support --with-tcl build Tcl modules (PL/Tcl) --with-tclconfig=DIR tclConfig.sh is in DIR @@ -4917,6 +4921,26 @@ fi + +# +# C11 stdatomic.h +# +# Accepts --with-stdatomic={yes,no,auto}. "auto" (the default) uses +# stdatomic.h if a compile test succeeds; "yes" requires it; "no" forces the +# traditional platform-specific atomics. The actual detection runs later, +# after the compiler and 64-bit integer handling have been configured. + +# Check whether --with-stdatomic was given. +if test "${with_stdatomic+set}" = set; then : + withval=$with_stdatomic; case $withval in + yes | no | auto) ;; + *) as_fn_error $? "invalid argument to --with-stdatomic; use yes, no, or auto" "$LINENO" 5 ;; + esac +else + with_stdatomic=no +fi + + for ac_prog in gawk mawk nawk awk do # Extract the first word of "$ac_prog", so it can be a program name with args. @@ -17718,6 +17742,91 @@ $as_echo "#define HAVE_GCC__ATOMIC_INT64_CAS 1" >>confdefs.h fi +# Decide whether to use C11 stdatomic.h (see --with-stdatomic above). The +# probe mirrors the one used by the meson build: it must compile 32-bit +# fetch_add, a seq_cst thread fence, and a 64-bit compare-exchange. +if test "$with_stdatomic" != no; then + # Use a link test, not a compile test: 64-bit atomic_compare_exchange_strong() + # can lower to a libatomic call (__atomic_compare_exchange_8) on some 32-bit + # or weakly-aligned targets. A compile-only check would pass there and then + # fail at final link. Retry with -latomic if the bare link fails. + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for working C11 stdatomic.h" >&5 +$as_echo_n "checking for working C11 stdatomic.h... " >&6; } +if ${pgac_cv_stdatomic+:} false; then : + $as_echo_n "(cached) " >&6 +else + pgac_cv_stdatomic=no + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include +int +main () +{ +_Atomic int x = 0; + atomic_fetch_add(&x, 1); + atomic_thread_fence(memory_order_seq_cst); + _Atomic(unsigned long long) y = 0; + unsigned long long expected = 0; + atomic_compare_exchange_strong(&y, &expected, 42); + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + pgac_cv_stdatomic=yes +else + pgac_save_LIBS=$LIBS + LIBS="$LIBS -latomic" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include +int +main () +{ +_Atomic int x = 0; + atomic_fetch_add(&x, 1); + atomic_thread_fence(memory_order_seq_cst); + _Atomic(unsigned long long) y = 0; + unsigned long long expected = 0; + atomic_compare_exchange_strong(&y, &expected, 42); + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + pgac_cv_stdatomic='yes, with -latomic' +else + pgac_cv_stdatomic=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext + LIBS=$pgac_save_LIBS +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $pgac_cv_stdatomic" >&5 +$as_echo "$pgac_cv_stdatomic" >&6; } +else + pgac_cv_stdatomic=no +fi + +if test "$with_stdatomic" = yes && test "$pgac_cv_stdatomic" = no; then + as_fn_error $? "--with-stdatomic=yes was given, but a working C11 stdatomic.h could not be found" "$LINENO" 5 +fi + +if test "$pgac_cv_stdatomic" != no; then + +$as_echo "#define USE_STDATOMIC_H 1" >>confdefs.h + + if test "$pgac_cv_stdatomic" = 'yes, with -latomic'; then + LIBS="$LIBS -latomic" + fi +fi + + # Check for __get_cpuid() and __cpuid() { $as_echo "$as_me:${as_lineno-$LINENO}: checking for __get_cpuid" >&5 $as_echo_n "checking for __get_cpuid... " >&6; } diff --git a/configure.ac b/configure.ac index 0e624fe36b9..a3d4e31a609 100644 --- a/configure.ac +++ b/configure.ac @@ -443,6 +443,23 @@ choke me PGAC_ARG_BOOL(with, llvm, no, [build with LLVM based JIT support], [AC_DEFINE([USE_LLVM], 1, [Define to 1 to build with LLVM based JIT support. (--with-llvm)])]) AC_SUBST(with_llvm) + +# +# C11 stdatomic.h +# +# Accepts --with-stdatomic={yes,no,auto}. "auto" (the default) uses +# stdatomic.h if a compile test succeeds; "yes" requires it; "no" forces the +# traditional platform-specific atomics. The actual detection runs later, +# after the compiler and 64-bit integer handling have been configured. +AC_ARG_WITH(stdatomic, + [AS_HELP_STRING([--with-stdatomic@<:@=yes/no/auto@:>@], + [use C11 stdatomic.h for atomic operations @<:@auto@:>@])], + [case $withval in + yes | no | auto) ;; + *) AC_MSG_ERROR([invalid argument to --with-stdatomic; use yes, no, or auto]) ;; + esac], + [with_stdatomic=no]) +AC_SUBST(with_stdatomic) dnl must use AS_IF here, else AC_REQUIRES inside PGAC_LLVM_SUPPORT malfunctions AS_IF([test "$with_llvm" = yes], [ PGAC_LLVM_SUPPORT() @@ -2101,6 +2118,55 @@ PGAC_HAVE_GCC__ATOMIC_INT32_CAS PGAC_HAVE_GCC__ATOMIC_INT64_CAS +# Decide whether to use C11 stdatomic.h (see --with-stdatomic above). The +# probe mirrors the one used by the meson build: it must compile 32-bit +# fetch_add, a seq_cst thread fence, and a 64-bit compare-exchange. +if test "$with_stdatomic" != no; then + # Use a link test, not a compile test: 64-bit atomic_compare_exchange_strong() + # can lower to a libatomic call (__atomic_compare_exchange_8) on some 32-bit + # or weakly-aligned targets. A compile-only check would pass there and then + # fail at final link. Retry with -latomic if the bare link fails. + AC_CACHE_CHECK([for working C11 stdatomic.h], [pgac_cv_stdatomic], + [pgac_cv_stdatomic=no + AC_LINK_IFELSE([AC_LANG_PROGRAM([#include ], + [[_Atomic int x = 0; + atomic_fetch_add(&x, 1); + atomic_thread_fence(memory_order_seq_cst); + _Atomic(unsigned long long) y = 0; + unsigned long long expected = 0; + atomic_compare_exchange_strong(&y, &expected, 42); + ]])], + [pgac_cv_stdatomic=yes], + [pgac_save_LIBS=$LIBS + LIBS="$LIBS -latomic" + AC_LINK_IFELSE([AC_LANG_PROGRAM([#include ], + [[_Atomic int x = 0; + atomic_fetch_add(&x, 1); + atomic_thread_fence(memory_order_seq_cst); + _Atomic(unsigned long long) y = 0; + unsigned long long expected = 0; + atomic_compare_exchange_strong(&y, &expected, 42); + ]])], + [pgac_cv_stdatomic='yes, with -latomic'], + [pgac_cv_stdatomic=no]) + LIBS=$pgac_save_LIBS])]) +else + pgac_cv_stdatomic=no +fi + +if test "$with_stdatomic" = yes && test "$pgac_cv_stdatomic" = no; then + AC_MSG_ERROR([--with-stdatomic=yes was given, but a working C11 stdatomic.h could not be found]) +fi + +if test "$pgac_cv_stdatomic" != no; then + AC_DEFINE([USE_STDATOMIC_H], 1, + [Define to 1 to use C11 stdatomic.h for atomic operations. (--with-stdatomic)]) + if test "$pgac_cv_stdatomic" = 'yes, with -latomic'; then + LIBS="$LIBS -latomic" + fi +fi + + # Check for __get_cpuid() and __cpuid() AC_CACHE_CHECK([for __get_cpuid], [pgac_cv__get_cpuid], [AC_LINK_IFELSE([AC_LANG_PROGRAM([#include ], diff --git a/meson.build b/meson.build index d88a7a70308..f4e72386deb 100644 --- a/meson.build +++ b/meson.build @@ -674,6 +674,108 @@ if have_cxx and not cxx.compiles(cxx11_test, name: 'C++11') endif +############################################################### +# Check for C11 stdatomic.h support +############################################################### + +stdatomic_test_code = ''' +#include +int main(void) { + _Atomic int x = 0; + atomic_fetch_add(&x, 1); + atomic_thread_fence(memory_order_seq_cst); + _Atomic(unsigned long long) y = 0; + unsigned long long expected = 0; + atomic_compare_exchange_strong(&y, &expected, 42); + return 0; +} +''' + +has_stdatomic = false +use_stdatomic = get_option('use_stdatomic') +atomic_link_args = [] + +# The stdatomic probe below is a *link* test, not just a compile test: 64-bit +# atomic_compare_exchange_strong() can lower to a libatomic call +# (__atomic_compare_exchange_8) on some 32-bit or weakly-aligned targets. A +# compile-only check would pass there and then fail at final link, so we +# require the probe to link, retrying with -latomic if the bare link fails. + +if cc.get_id() == 'msvc' + msvc_ver = cc.version() + if msvc_ver.version_compare('>=19.30') # Visual Studio 2022+ + message('MSVC version @0@ detected (>= 19.30), checking for stdatomic.h support'.format(msvc_ver)) + # MSVC's requires C11 mode (/std:c11) in addition to + # /experimental:c11atomics; without /std:c11 the header errors out with + # "C atomics require C11 or later". The probe must pass both, since + # test_c_args is not yet populated with the C11 flag at this point. + if cc.links(stdatomic_test_code, + name: 'MSVC stdatomic.h with /experimental:c11atomics', + args: test_c_args + ['/std:c11', '/experimental:c11atomics']) + has_stdatomic = true + message('MSVC stdatomic.h is available with /experimental:c11atomics') + else + message('MSVC stdatomic.h test compilation failed') + endif + else + message('MSVC version @0@ is too old for stdatomic.h (requires >= 19.30)'.format(msvc_ver)) + endif +else + # GCC/Clang and other compilers + if cc.has_header('stdatomic.h', args: test_c_args) + message('stdatomic.h header found, testing that it links...') + if cc.links(stdatomic_test_code, + name: 'stdatomic.h works', + args: test_c_args) + has_stdatomic = true + message('stdatomic.h is available and working') + elif cc.links(stdatomic_test_code, + name: 'stdatomic.h works with -latomic', + args: test_c_args + ['-latomic']) + # Some targets need libatomic for the wider (e.g. 64-bit) operations. + has_stdatomic = true + atomic_link_args = ['-latomic'] + message('stdatomic.h is available and working (requires -latomic)') + else + message('stdatomic.h found but test link failed') + endif + else + message('stdatomic.h header not found') + endif +endif + +# Process the use_stdatomic option +if use_stdatomic == 'yes' + if not has_stdatomic + error('stdatomic.h was requested with -Duse_stdatomic=yes but is not available or not working') + endif + cdata.set('USE_STDATOMIC_H', 1) + message('Using C11 stdatomic.h for atomic operations (forced by -Duse_stdatomic=yes)') + if cc.get_id() == 'msvc' + # MSVC needs C11 mode plus the experimental atomics switch for . + cflags += ['/std:c11', '/experimental:c11atomics'] + endif + if atomic_link_args.length() > 0 + os_deps += cc.find_library('atomic') + endif +elif use_stdatomic == 'no' + message('Using traditional platform-specific atomics (forced by -Duse_stdatomic=no)') + # Do not set USE_STDATOMIC_H +elif use_stdatomic == 'auto' + if has_stdatomic + cdata.set('USE_STDATOMIC_H', 1) + message('Using C11 stdatomic.h for atomic operations (auto-detected)') + if cc.get_id() == 'msvc' + cflags += ['/std:c11', '/experimental:c11atomics'] + endif + if atomic_link_args.length() > 0 + os_deps += cc.find_library('atomic') + endif + else + message('Using traditional platform-specific atomics (stdatomic.h not available)') + endif +endif + postgres_inc = [include_directories(postgres_inc_d)] test_lib_d = postgres_lib_d test_c_args = cppflags + cflags diff --git a/meson_options.txt b/meson_options.txt index 6a793f3e479..f8c5ccc14e1 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -52,6 +52,11 @@ option('PG_TEST_EXTRA', type: 'string', value: '', option('PG_GIT_REVISION', type: 'string', value: 'HEAD', description: 'git revision to be packaged by pgdist target') +option('use_stdatomic', type: 'combo', + choices: ['auto', 'yes', 'no'], + value: 'no', + description: 'Use C11 stdatomic.h for atomic operations (auto, yes, no)') + # Compilation options diff --git a/src/backend/storage/lmgr/README b/src/backend/storage/lmgr/README index 45de0fd2bd6..460b9559449 100644 --- a/src/backend/storage/lmgr/README +++ b/src/backend/storage/lmgr/README @@ -47,6 +47,77 @@ when the wait time might exceed a few seconds. The rest of this README file discusses the regular lock manager in detail. +Atomic Operations +================= + +PostgreSQL's atomic operations (see src/include/port/atomics.h) can be +implemented using either C11 stdatomic.h or traditional platform-specific +code. The implementation is selected at build time via the USE_STDATOMIC_H +preprocessor definition: + +* C11 stdatomic.h implementation (USE_STDATOMIC_H defined): + Uses the standard C11 header for all atomic operations. + This provides better portability and potentially better compiler + optimizations. Available on: + - MSVC 2022+ (requires /experimental:c11atomics flag) + - GCC 4.9+ and Clang 3.1+ (no special flags needed) + +* Traditional implementation (USE_STDATOMIC_H not defined): + Uses platform-specific implementations that have been battle-tested + in PostgreSQL for many years: + - Architecture-specific: arch-x86.h, arch-arm.h, arch-ppc.h + - Compiler intrinsics: generic-gcc.h, generic-msvc.h + - Fallback implementations: generic.h, fallback.h + +Both implementations provide the same public API and observable semantics. +The choice is made at build configuration time: + + meson setup build -Duse_stdatomic=auto # Auto-detect (default) + meson setup build -Duse_stdatomic=yes # Force stdatomic.h + meson setup build -Duse_stdatomic=no # Force traditional + +The two implementations differ internally in several ways that do not affect +the public API: + +* Memory ordering. pg_atomic_read_u32/u64() use seq_cst ordering under + stdatomic.h; pg_atomic_write_u32/u64() use relaxed. The seq_cst read was + required for correctness on weak-memory hardware (e.g. RISC-V), where the + weaker relaxed and acquire orderings let a concurrent reader miss a tuple + in a parallel hash join. The write stays relaxed: a plain atomic store + matches the traditional "no barrier" contract for pg_atomic_write and, on + ARM64, avoids the store-release (STLR) serialization that would penalize + contended writes to hot shared state. The pg_atomic_unlocked_write_* + variant also uses relaxed ordering, consistent with its "no guarantees" + contract. On x86 (TSO) a seq_cst load is still a plain mov, so there is no + added cost; + on ARM/RISC-V it adds fence instructions on these paths. + +* Barriers. Each barrier pairs a compiler barrier (atomic_signal_fence) with + the thread fence (atomic_thread_fence), because a bare thread fence orders + only atomic accesses and would let the compiler reorder plain loads/stores + across it, whereas PostgreSQL's barrier contract must order non-atomic + accesses too. This matches the traditional generic-gcc.h barriers. + +* Spinlock flag polarity. The stdatomic.h pg_atomic_flag uses 1=unlocked, + 0=locked (fetch_and-based test-and-set), the inverse of the traditional + 0=unlocked, 1=locked (exchange-based). This is an internal detail with no + external visibility beyond raw values seen in a debugger. + +* Binary compatibility. The atomic types are _Atomic(...) under stdatomic.h + and volatile-struct under the traditional path. The two are not binary + compatible, so objects and shared memory cannot be mixed between builds; + the implementation is fixed at build time, so this is not a concern in + practice. + +* Platform reach. Under stdatomic.h, 64-bit atomics are available on 32-bit + ARM (compiler/runtime may emulate them with locks), atomics are usable from + frontend programs, and the headers are C++-compatible. The traditional + path disables 64-bit atomics on ARM32 and is backend-only. + +For more information on memory barriers and atomic operations, see +src/backend/storage/lmgr/README.barrier. + + Lock Data Structures -------------------- diff --git a/src/backend/storage/lmgr/s_lock.c b/src/backend/storage/lmgr/s_lock.c index 6df568eccb3..b3a11fe9fc9 100644 --- a/src/backend/storage/lmgr/s_lock.c +++ b/src/backend/storage/lmgr/s_lock.c @@ -51,7 +51,13 @@ #include #include "common/pg_prng.h" +#ifdef USE_STDATOMIC_H +#include "port/atomics.h" +#include "port/spin_delay.h" +#include "storage/spin.h" +#else #include "storage/s_lock.h" +#endif #include "utils/wait_event.h" #define MIN_SPINS_PER_DELAY 10 @@ -60,6 +66,7 @@ #define MIN_DELAY_USEC 1000L #define MAX_DELAY_USEC 1000000L +#ifndef USE_STDATOMIC_H #ifdef S_LOCK_TEST /* * These are needed by pgstat_report_wait_start in the standalone compile of @@ -68,6 +75,7 @@ static uint32 local_my_wait_event_info; uint32 *my_wait_event_info = &local_my_wait_event_info; #endif +#endif /* !USE_STDATOMIC_H */ static int spins_per_delay = DEFAULT_SPINS_PER_DELAY; @@ -80,7 +88,7 @@ s_lock_stuck(const char *file, int line, const char *func) { if (!func) func = "(unknown)"; -#if defined(S_LOCK_TEST) +#if !defined(USE_STDATOMIC_H) && defined(S_LOCK_TEST) fprintf(stderr, "\nStuck spinlock detected at %s, %s:%d.\n", func, file, line); @@ -101,16 +109,39 @@ s_lock(volatile slock_t *lock, const char *file, int line, const char *func) init_spin_delay(&delayStatus, file, line, func); +#ifdef USE_STDATOMIC_H + for (;;) + { + bool try_to_set; + +#ifdef PG_SPIN_TRY_RELAXED + /* + * Test-and-test-and-set: read the lock first and only attempt the + * exchange when it looks free, to avoid taking cache-line ownership on + * every spin. PG_SPIN_TRY_RELAXED is defined only where this pre-read + * is cheap (x86/x86_64 TSO, where a seq_cst load is a plain load). + */ + try_to_set = (pg_atomic_read_u32(lock) == 0); +#else + try_to_set = true; +#endif + if (try_to_set && pg_atomic_exchange_u32(lock, 1) == 0) + break; + perform_spin_delay(&delayStatus); + } +#else /* !USE_STDATOMIC_H */ while (TAS_SPIN(lock)) { perform_spin_delay(&delayStatus); } +#endif /* USE_STDATOMIC_H */ finish_spin_delay(&delayStatus); return delayStatus.delays; } +#ifndef USE_STDATOMIC_H #ifdef USE_DEFAULT_S_UNLOCK void s_unlock(volatile slock_t *lock) @@ -118,6 +149,7 @@ s_unlock(volatile slock_t *lock) *lock = 0; } #endif +#endif /* !USE_STDATOMIC_H */ /* * Wait while spinning on a contended spinlock. @@ -126,7 +158,11 @@ void perform_spin_delay(SpinDelayStatus *status) { /* CPU-specific delay each time through the loop */ +#ifdef USE_STDATOMIC_H + pg_spin_delay(); +#else SPIN_DELAY(); +#endif /* Block the process every spins_per_delay tries */ if (++(status->spins) >= spins_per_delay) @@ -149,7 +185,7 @@ perform_spin_delay(SpinDelayStatus *status) pg_usleep(status->cur_delay); pgstat_report_wait_end(); -#if defined(S_LOCK_TEST) +#if !defined(USE_STDATOMIC_H) && defined(S_LOCK_TEST) fprintf(stdout, "*"); fflush(stdout); #endif @@ -232,6 +268,7 @@ update_spins_per_delay(int shared_spins_per_delay) /*****************************************************************************/ +#ifndef USE_STDATOMIC_H #if defined(S_LOCK_TEST) /* @@ -298,3 +335,4 @@ main() } #endif /* S_LOCK_TEST */ +#endif /* !USE_STDATOMIC_H */ diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 4f8113c144b..27dd89765e4 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -747,6 +747,10 @@ /* Define to 1 to use Intel SSE 4.2 CRC instructions with a runtime check. */ #undef USE_SSE42_CRC32C_WITH_RUNTIME_CHECK +/* Define to 1 to use C11 stdatomic.h for atomic operations. + (--with-stdatomic) */ +#undef USE_STDATOMIC_H + /* Define to 1 to use SVE popcount instructions with a runtime check. */ #undef USE_SVE_POPCNT_WITH_RUNTIME_CHECK diff --git a/src/include/port/atomics.h b/src/include/port/atomics.h index a605ea81d07..2058dbb9238 100644 --- a/src/include/port/atomics.h +++ b/src/include/port/atomics.h @@ -7,6 +7,26 @@ * atomically and dealing with cache coherency. Used to implement locking * facilities and lockless algorithms/data structures. * + * IMPLEMENTATION SELECTION: + * + * PostgreSQL's atomic operations can be implemented using either: + * + * 1. C11 stdatomic.h (when USE_STDATOMIC_H is defined) + * - Uses standard C11 for all atomic operations + * - Better portability and compiler optimizations + * - Requires MSVC 2022+ or GCC 4.9+/Clang 3.1+ + * + * 2. Traditional platform-specific implementations (when USE_STDATOMIC_H is not defined) + * - Uses battle-tested PostgreSQL implementations + * - Architecture-specific: arch-x86.h, arch-arm.h, arch-ppc.h + * - Compiler intrinsics: generic-gcc.h, generic-msvc.h + * - Fallback implementations: generic.h, fallback.h + * + * Both implementations provide identical public API and semantics. + * Selection is made at build time via -Duse_stdatomic=auto/yes/no. + * + * PORTING NOTES: + * * To bring up postgres on a platform/compiler at the very least * implementations for the following operations should be provided: * * pg_compiler_barrier(), pg_write_barrier(), pg_read_barrier() @@ -14,13 +34,8 @@ * * pg_atomic_test_set_flag(), pg_atomic_init_flag(), pg_atomic_clear_flag() * * PG_HAVE_8BYTE_SINGLE_COPY_ATOMICITY should be defined if appropriate. * - * There exist generic, hardware independent, implementations for several - * compilers which might be sufficient, although possibly not optimal, for a - * new platform. If no such generic implementation is available spinlocks will - * be used to implement the 64-bit parts of the API. - * - * Implement _u64 atomics if and only if your platform can use them - * efficiently (and obviously correctly). + * For new platforms, prefer using stdatomic.h if available, as it reduces + * maintenance burden and leverages compiler-provided implementations. * * Use higher level functionality (lwlocks, spinlocks, heavyweight locks) * whenever possible. Writing correct code using these facilities is hard. @@ -38,15 +53,76 @@ #ifndef ATOMICS_H #define ATOMICS_H +/* + * Frontend code restriction + * + * Traditionally, atomics.h could not be included from frontend code because + * the platform-specific implementations often relied on backend-only features. + * + * With the stdatomic.h implementation, this restriction is no longer necessary + * since stdatomic.h is a standard header. However, we keep the restriction for + * the traditional implementation path to maintain compatibility. + */ #ifdef FRONTEND -#error "atomics.h may not be included from frontend code" +#ifndef USE_STDATOMIC_H +#error "atomics.h may not be included from frontend code (use -Duse_stdatomic=yes if atomics are needed)" +#endif #endif #define INSIDE_ATOMICS_H +/* + * The public API and the _impl signatures keep the historical volatile + * qualifier on the atomic pointer, because many callers reach atomic fields + * through volatile-qualified struct pointers. Under C11 the _Atomic types + * carry their own ordering guarantees, so the volatile is redundant for the + * atomic accesses themselves and is accepted (and effectively ignored for the + * atomic operation) by the supported compilers, including MSVC. + */ + #include /* + * PostgreSQL atomics can be implemented using either C11 stdatomic.h + * or platform-specific implementations. The choice is made at build + * configuration time via the USE_STDATOMIC_H preprocessor definition. + * + * When USE_STDATOMIC_H is defined, we use C11 for all + * atomic operations. This provides better portability and potentially + * better compiler optimizations. + * + * When USE_STDATOMIC_H is not defined (default initially), we use + * traditional platform-specific implementations (arch-*.h, generic-*.h) + * that have been battle-tested in PostgreSQL for many years. + * + * Both paths provide identical public API and semantics. + */ + +#ifdef USE_STDATOMIC_H + +/* + * C11 stdatomic.h implementation path + * + * This uses the standard C11 header for atomic operations. + * The type definitions and implementations are in stdatomic_impl.h. + */ + +#include "port/atomics/stdatomic_impl.h" + +/* + * The public API is provided by static inline functions in the common code + * section below. Those functions call _impl functions, which are provided by + * stdatomic_impl.h for this path. + */ + +#else /* !USE_STDATOMIC_H */ + +/* + * Traditional platform-specific implementation path + * + * This is the original PostgreSQL atomics implementation using + * architecture-specific headers and compiler intrinsics. + * * First a set of architecture specific files is included. * * These files can provide the full set of atomics or can do pretty much @@ -116,6 +192,17 @@ */ #include "port/atomics/generic.h" +#endif /* USE_STDATOMIC_H */ + +/* + * Common code for both stdatomic.h and traditional implementations. + * + * The following definitions provide the public API that works identically + * regardless of which implementation is used. The static inline functions + * below call _impl functions which are provided by either stdatomic_impl.h + * (when USE_STDATOMIC_H is defined) or the traditional implementation files + * (generic.h, etc.) otherwise. + */ /* * pg_compiler_barrier - prevent the compiler from moving code across diff --git a/src/include/port/atomics/stdatomic_impl.h b/src/include/port/atomics/stdatomic_impl.h new file mode 100644 index 00000000000..0bda02d16f4 --- /dev/null +++ b/src/include/port/atomics/stdatomic_impl.h @@ -0,0 +1,589 @@ +/*------------------------------------------------------------------------- + * + * stdatomic_impl.h + * Atomic operations implementation using C11 stdatomic.h + * + * This file provides PostgreSQL atomic operations using the C11 standard + * . It is only included when USE_STDATOMIC_H is defined. + * + * The traditional platform-specific implementation (arch-*.h, generic-*.h, + * fallback.h) remains available and is used when stdatomic.h is not + * available or when explicitly requested via -Duse_stdatomic=no. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/port/atomics/stdatomic_impl.h + * + *------------------------------------------------------------------------- + */ + +#ifndef STDATOMIC_IMPL_H +#define STDATOMIC_IMPL_H + +/* + * Only include this file when USE_STDATOMIC_H is defined at build time. + * This ensures the traditional implementation remains available as fallback. + */ +#ifdef USE_STDATOMIC_H + +/* + * C++ Compatibility + * + * C++11 through C++20 cannot include C's directly. Instead, + * we use C++11's header and map to std::atomic types. C++23 allows + * but we handle older versions. + * + * This follows the approach from C++23 standard section 33.5.12 which + * allows C and C++ atomic types to interoperate. + */ +#if defined(__cplusplus) && __cplusplus < 202302L +extern "C++" +{ +#include + + /* Map to C++ atomic types */ +#define pg_atomic(T) std::atomic + + /* Import memory order constants into global namespace */ + using std::memory_order_relaxed; + using std::memory_order_acquire; + using std::memory_order_release; + using std::memory_order_acq_rel; + using std::memory_order_seq_cst; +} +#else +/* C11 or C++23+: use standard */ +#include +#define pg_atomic(T) _Atomic(T) +#endif + +/* + * Type Definitions + * + * We use C11 _Atomic qualifier (or C++ std::atomic) with PostgreSQL's + * standard integer types. These types are compatible with atomic operations + * and provide sequential consistency guarantees. + * + * Note: pg_atomic(T) is defined above and maps to either _Atomic(T) for C + * or std::atomic for C++. + */ +/* + * pg_atomic_flag is backed by a full 32-bit word, not uint8. On + * architectures without byte-granular atomics (notably RISC-V, whose base + * 'A' extension has only word/doubleword AMOs), a sub-word _Atomic(uint8) + * RMW is emulated by the compiler as a load-reserved/store-conditional of + * the *entire containing 32-bit word*. Because slock_t (a pg_atomic_flag) + * is frequently packed next to other fields, that word-wide RMW can clobber + * the neighbouring bytes, corrupting adjacent shared state under contention. + * Using a 32-bit word makes every flag operation a clean word-aligned AMO + * that never touches neighbouring memory, matching the traditional + * implementation, whose pg_atomic_flag is also 32 bits (generic.h). + */ +typedef pg_atomic(uint32) pg_atomic_flag; +typedef pg_atomic(uint8) pg_atomic_uint8; +typedef pg_atomic(uint16) pg_atomic_uint16; +typedef pg_atomic(uint32) pg_atomic_uint32; +typedef pg_atomic(uint64) pg_atomic_uint64; + +/* + * PostgreSQL's atomics live in shared memory and are accessed from multiple + * *processes*, not just threads. A non-lock-free atomic implemented by the + * compiler/runtime with a lock table (e.g. libatomic) uses process-local + * locks, which would silently fail to provide atomicity across processes and + * corrupt shared state. Require genuine, always-lock-free support at compile + * time for every width we use. (ATOMIC_*_LOCK_FREE is 0 = never lock-free, + * 1 = sometimes / not known at compile time, 2 = always lock-free.) + * + * The == 1 case is a real hazard, not just conservatism: e.g. 64-bit atomics + * on a 32-bit target may be emulated by libatomic with a process-local lock, + * which corrupts cross-process shared memory exactly as == 0 would. So we + * require == 2 everywhere the macro is trustworthy. + * + * The one exception is MSVC compiling as C: its reports + * ATOMIC_*_LOCK_FREE == 1 for every width even though the atomics are in fact + * always lock-free on the hardware we target (in C++ mode the same MSVC + * reports 2, and atomic_is_lock_free() returns true at run time). We skip the + * assertion only in that specific configuration rather than weakening it for + * everyone. + */ +#if !(defined(_MSC_VER) && !defined(__cplusplus)) +StaticAssertDecl(ATOMIC_CHAR_LOCK_FREE == 2, "8-bit atomics are not lock-free"); +StaticAssertDecl(ATOMIC_SHORT_LOCK_FREE == 2, "16-bit atomics are not lock-free"); +StaticAssertDecl(ATOMIC_INT_LOCK_FREE == 2, "32-bit atomics are not lock-free"); +StaticAssertDecl(ATOMIC_LLONG_LOCK_FREE == 2, "64-bit atomics are not lock-free"); +#endif + +/* + * Signal atomic operation support to atomics.h + * + * stdatomic.h provides native support for all atomic types on all platforms + * it compiles on, so we unconditionally define these. PG_HAVE_ATOMIC_U64_SIMULATION + * is intentionally NOT defined -- stdatomic provides real 64-bit atomics. + */ +#define PG_HAVE_ATOMIC_U32_SUPPORT +#define PG_HAVE_ATOMIC_U64_SUPPORT + +/* + * 8-byte single-copy atomicity. + * + * All 64-bit architectures that support C11 stdatomic.h guarantee 8-byte + * single-copy atomicity for aligned loads/stores. This is used by bufmgr.c + * and other performance-critical paths to avoid unnecessary atomic operations. + */ +#if SIZEOF_VOID_P >= 8 +#define PG_HAVE_8BYTE_SINGLE_COPY_ATOMICITY +#endif + +/* + * Memory Barrier Implementation + * + * PostgreSQL's memory barriers must order *non-atomic, non-volatile* accesses + * too, which a bare C11 atomic_thread_fence() does NOT guarantee against + * compiler reordering: atomic_thread_fence only establishes ordering with + * respect to atomic operations, so the compiler is still free to move plain + * loads/stores across it. We therefore pair each thread fence with an + * explicit compiler barrier (atomic_signal_fence), matching the semantics of + * the traditional implementation whose barriers also constrain plain memory + * accesses. Omitting the compiler barrier corrupts lock-free algorithms on + * weak-memory architectures (observed as lost tuples in parallel hash join on + * RISC-V, where the compiler reordered non-atomic hash-table stores across a + * bare acquire/release fence). + * + * These are implementation functions (_impl suffix) mapped to the public API + * by atomics.h. + */ +#define pg_compiler_barrier_impl() atomic_signal_fence(memory_order_seq_cst) + +static inline void +pg_memory_barrier_impl(void) +{ + atomic_signal_fence(memory_order_seq_cst); + atomic_thread_fence(memory_order_seq_cst); +} + +static inline void +pg_read_barrier_impl(void) +{ + atomic_signal_fence(memory_order_seq_cst); + atomic_thread_fence(memory_order_acquire); +} + +static inline void +pg_write_barrier_impl(void) +{ + atomic_signal_fence(memory_order_seq_cst); + atomic_thread_fence(memory_order_release); +} + +/* + * Spin Delay Implementation + * + * On x86, a spinloop without a "pause" instruction can waste a lot of power + * and slow down the adjacent hyperthread. On ARM64, ISB provides better + * backoff than YIELD. This provides platform-specific spin delay hints. + * + * The implementation is in spin_delay.h which directly defines pg_spin_delay_impl(), + * matching the pattern used by the traditional atomics implementation. + */ +#include "port/spin_delay.h" + +/* + * ================================================================= + * pg_atomic_flag Operations + * ================================================================= + * + * Atomic flag type used primarily for spinlocks. + * + * CRITICAL FLAG CONVENTION DIFFERENCE FROM TRADITIONAL IMPLEMENTATION: + * + * stdatomic.h implementation (this file): + * 1 = unlocked, 0 = locked + * + * Traditional implementation (generic.h): + * 0 = unlocked, 1 = locked + * + * This polarity difference is INTENTIONAL and internal to the implementation. + * The public API behavior is identical in both implementations: + * - pg_atomic_init_flag() initializes to unlocked state + * - pg_atomic_test_set_flag() returns true if lock was successfully acquired + * - pg_atomic_clear_flag() releases the lock + * + * The stdatomic.h convention (1=unlocked) is more natural for C11 atomics + * and avoids double-negation logic in the implementation. + */ + +/* + * pg_atomic_init_flag_impl - Initialize atomic flag to unlocked state + * + * In the stdatomic.h implementation, we initialize to 1 (unlocked). + */ +static inline void +pg_atomic_init_flag_impl(volatile pg_atomic_flag *ptr) +{ + atomic_init(ptr, 1); /* 1 = unlocked */ +} + +/* + * pg_atomic_test_set_flag_impl - Try to acquire lock + * + * Atomically clears the flag (AND with 0) and returns previous value. + * Returns true if we successfully acquired the lock (old value was 1, + * meaning unlocked), false if already locked (old value was 0). + * + * Memory ordering: acquire (synchronizes-with prior release) + */ +static inline bool +pg_atomic_test_set_flag_impl(volatile pg_atomic_flag *ptr) +{ + /* + * AND flag with 0 to clear it (locked). If previous value was 1 + * (unlocked), we succeeded in acquiring. If previous value was 0 + * (already locked), we failed. + * + * Return true if we SUCCEEDED in acquiring the lock. + */ + return atomic_fetch_and_explicit(ptr, 0, memory_order_acquire) != 0; +} + +/* + * pg_atomic_unlocked_test_flag_impl - Test if flag is unlocked without acquiring + * + * Returns true if the flag is currently unlocked (value == 1). + * Uses relaxed ordering since this is typically used for optimization hints. + */ +static inline bool +pg_atomic_unlocked_test_flag_impl(volatile pg_atomic_flag *ptr) +{ + return atomic_load_explicit(ptr, memory_order_relaxed) != 0; +} + +/* + * pg_atomic_clear_flag_impl - Release lock + * + * Sets flag to 1 (unlocked). + * Memory ordering: release (makes prior writes visible) + */ +static inline void +pg_atomic_clear_flag_impl(volatile pg_atomic_flag *ptr) +{ + atomic_store_explicit(ptr, 1, memory_order_release); +} + +/* + * ================================================================= + * pg_atomic_uint32 Operations + * ================================================================= + * + * 32-bit atomic unsigned integer operations. + * All operations use sequential consistency unless otherwise noted. + */ + +/* + * pg_atomic_init_u32_impl - Initialize atomic uint32 + * + * This is not an atomic operation - must only be called during initialization + * when no concurrent access is possible. + */ +static inline void +pg_atomic_init_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 val) +{ + atomic_init(ptr, val); +} + +/* + * pg_atomic_read_u32_impl - Atomically read uint32 value + * + * Memory ordering: sequentially consistent. This is stronger than the + * documented "no barrier semantics" of pg_atomic_read_u32(), and it is + * deliberate: a bare memory_order_relaxed load reintroduces a real, + * reproducible data-loss bug on weak-memory hardware. + * + * The traditional implementation reads through a volatile-qualified pointer. + * A volatile access cannot be reordered *by the compiler* relative to the + * surrounding code, so consumers that publish a pointer with a + * barrier-carrying store and then read+dereference it get the dependent load + * ordered for free. A C11 memory_order_relaxed atomic load does not carry + * that compiler-ordering property, and the parallel hash join relies on it: + * ExecParallelHash reads a freshly published bucket pointer via + * dsa_pointer_atomic_read() and then dereferences the tuple it points to. + * With a relaxed load this was observed to lose a tuple on RISC-V -- + * join_hash "extremely_skewed" returns 19999 instead of 20000, intermittently + * (reproduced here at roughly 1 run in 4 on real rv64 hardware). seq_cst + * restores correctness. + * + * The cost of the seq_cst *load* is not the ARM regression that was measured + * for an earlier revision of this work: microbenchmarking on Graviton showed + * an uncontended ldar is not measurably costlier than a plain ldr, and the + * ~2% aarch64 read-only regression seen previously was traced to the seq_cst + * *store* (STLR) in pg_atomic_write_u32, which is now relaxed (see below). + * So the ordered read is kept for correctness while the write, which was the + * actual measured cost, is not paid. + * + * A cleaner long-term fix is to leave this primitive relaxed (matching the + * documented contract) and give the specific hash-join consumer the ordering + * it needs -- e.g. an acquire read at the call site or + * dsa_pointer_atomic_read_membarrier(). That is a change to core executor / + * dsa code and is intentionally out of scope for the atomics port; it is + * noted as follow-up work rather than folded in here. + */ +static inline uint32 +pg_atomic_read_u32_impl(volatile pg_atomic_uint32 *ptr) +{ + return atomic_load_explicit(ptr, memory_order_seq_cst); +} + +/* + * pg_atomic_write_u32_impl - Atomically write uint32 value + * + * Memory ordering: relaxed. A plain atomic (non-torn) store matches the + * traditional implementation's "no barrier" contract for pg_atomic_write_u32 + * and, on ARM64, compiles to a plain STR rather than STLR. This matters: + * benchmarking traced the ~2% aarch64 read-only regression of an earlier + * seq_cst-everywhere revision to STLR store-side serialization on hot, + * contended shared lines (LWLock state, buffer headers), not to the reads. + * Making the write relaxed removes that cost. Callers needing an ordered + * write use pg_atomic_write_membarrier_u32() or an explicit barrier. + */ +static inline void +pg_atomic_write_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 val) +{ + atomic_store_explicit(ptr, val, memory_order_relaxed); +} + +/* + * pg_atomic_exchange_u32_impl - Atomically exchange uint32 value + * + * Atomically replaces the value with newval and returns the old value. + * Memory ordering: seq_cst + */ +static inline uint32 +pg_atomic_exchange_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 newval) +{ + return atomic_exchange_explicit(ptr, newval, memory_order_seq_cst); +} + +/* + * pg_atomic_compare_exchange_u32_impl - Atomic compare-and-swap + * + * Compares *ptr with *expected. If equal, replaces *ptr with newval and + * returns true. If not equal, updates *expected with current *ptr value + * and returns false. + * + * This is the strong version (no spurious failures). + * Memory ordering: seq_cst for both success and failure + */ +static inline bool +pg_atomic_compare_exchange_u32_impl(volatile pg_atomic_uint32 *ptr, + uint32 *expected, uint32 newval) +{ + return atomic_compare_exchange_strong_explicit(ptr, expected, newval, + memory_order_seq_cst, + memory_order_seq_cst); +} + +/* + * pg_atomic_fetch_add_u32_impl - Atomic fetch-and-add + * + * Atomically adds add_ to *ptr and returns the old value. + * Memory ordering: seq_cst + */ +static inline uint32 +pg_atomic_fetch_add_u32_impl(volatile pg_atomic_uint32 *ptr, int32 add_) +{ + return atomic_fetch_add_explicit(ptr, add_, memory_order_seq_cst); +} + +/* + * pg_atomic_fetch_sub_u32_impl - Atomic fetch-and-subtract + * + * Atomically subtracts sub_ from *ptr and returns the old value. + * Memory ordering: seq_cst + */ +static inline uint32 +pg_atomic_fetch_sub_u32_impl(volatile pg_atomic_uint32 *ptr, int32 sub_) +{ + return atomic_fetch_sub_explicit(ptr, sub_, memory_order_seq_cst); +} + +/* + * pg_atomic_fetch_and_u32_impl - Atomic fetch-and-AND + * + * Atomically performs *ptr &= and_ and returns the old value. + * Memory ordering: seq_cst + */ +static inline uint32 +pg_atomic_fetch_and_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 and_) +{ + return atomic_fetch_and_explicit(ptr, and_, memory_order_seq_cst); +} + +/* + * pg_atomic_fetch_or_u32_impl - Atomic fetch-and-OR + * + * Atomically performs *ptr |= or_ and returns the old value. + * Memory ordering: seq_cst + */ +static inline uint32 +pg_atomic_fetch_or_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 or_) +{ + return atomic_fetch_or_explicit(ptr, or_, memory_order_seq_cst); +} + +/* + * pg_atomic_add_fetch_u32_impl - Atomic add-and-fetch + * + * Atomically adds add_ to *ptr and returns the NEW value. + * Implemented using fetch_add + add_ since C11 doesn't have add_fetch. + */ +static inline uint32 +pg_atomic_add_fetch_u32_impl(volatile pg_atomic_uint32 *ptr, int32 add_) +{ + return atomic_fetch_add_explicit(ptr, add_, memory_order_seq_cst) + add_; +} + +/* + * pg_atomic_sub_fetch_u32_impl - Atomic subtract-and-fetch + * + * Atomically subtracts sub_ from *ptr and returns the NEW value. + */ +static inline uint32 +pg_atomic_sub_fetch_u32_impl(volatile pg_atomic_uint32 *ptr, int32 sub_) +{ + return atomic_fetch_sub_explicit(ptr, sub_, memory_order_seq_cst) - sub_; +} + +/* + * pg_atomic_read_membarrier_u32_impl - Read with full memory barrier + * + * Atomic read with sequential consistency. Useful for cases where + * correctness is more important than performance. + */ +static inline uint32 +pg_atomic_read_membarrier_u32_impl(volatile pg_atomic_uint32 *ptr) +{ + return atomic_load_explicit(ptr, memory_order_seq_cst); +} + +/* + * pg_atomic_unlocked_write_u32_impl - Non-atomic write + * + * Write without atomicity guarantees. Only safe when exclusive access + * is guaranteed externally. Uses relaxed ordering. + */ +static inline void +pg_atomic_unlocked_write_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 val) +{ + atomic_store_explicit(ptr, val, memory_order_relaxed); +} + +/* + * pg_atomic_write_membarrier_u32_impl - Write with full memory barrier + * + * Atomic write with sequential consistency. + */ +static inline void +pg_atomic_write_membarrier_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 val) +{ + atomic_store_explicit(ptr, val, memory_order_seq_cst); +} + +/* + * ================================================================= + * pg_atomic_uint64 Operations + * ================================================================= + * + * 64-bit equivalents of the u32 operations above. See the u32 comments + * for API documentation; semantics and memory ordering are identical. + */ + +static inline void +pg_atomic_init_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 val) +{ + atomic_init(ptr, val); +} + +static inline uint64 +pg_atomic_read_u64_impl(volatile pg_atomic_uint64 *ptr) +{ + return atomic_load_explicit(ptr, memory_order_seq_cst); +} + +static inline void +pg_atomic_write_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 val) +{ + atomic_store_explicit(ptr, val, memory_order_relaxed); +} + +static inline uint64 +pg_atomic_exchange_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 newval) +{ + return atomic_exchange_explicit(ptr, newval, memory_order_seq_cst); +} + +static inline bool +pg_atomic_compare_exchange_u64_impl(volatile pg_atomic_uint64 *ptr, + uint64 *expected, uint64 newval) +{ + return atomic_compare_exchange_strong_explicit(ptr, expected, newval, + memory_order_seq_cst, + memory_order_seq_cst); +} + +static inline uint64 +pg_atomic_fetch_add_u64_impl(volatile pg_atomic_uint64 *ptr, int64 add_) +{ + return atomic_fetch_add_explicit(ptr, add_, memory_order_seq_cst); +} + +static inline uint64 +pg_atomic_fetch_sub_u64_impl(volatile pg_atomic_uint64 *ptr, int64 sub_) +{ + return atomic_fetch_sub_explicit(ptr, sub_, memory_order_seq_cst); +} + +static inline uint64 +pg_atomic_fetch_and_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 and_) +{ + return atomic_fetch_and_explicit(ptr, and_, memory_order_seq_cst); +} + +static inline uint64 +pg_atomic_fetch_or_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 or_) +{ + return atomic_fetch_or_explicit(ptr, or_, memory_order_seq_cst); +} + +static inline uint64 +pg_atomic_add_fetch_u64_impl(volatile pg_atomic_uint64 *ptr, int64 add_) +{ + return atomic_fetch_add_explicit(ptr, add_, memory_order_seq_cst) + add_; +} + +static inline uint64 +pg_atomic_sub_fetch_u64_impl(volatile pg_atomic_uint64 *ptr, int64 sub_) +{ + return atomic_fetch_sub_explicit(ptr, sub_, memory_order_seq_cst) - sub_; +} + +static inline uint64 +pg_atomic_read_membarrier_u64_impl(volatile pg_atomic_uint64 *ptr) +{ + return atomic_load_explicit(ptr, memory_order_seq_cst); +} + +static inline void +pg_atomic_unlocked_write_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 val) +{ + atomic_store_explicit(ptr, val, memory_order_relaxed); +} + +static inline void +pg_atomic_write_membarrier_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 val) +{ + atomic_store_explicit(ptr, val, memory_order_seq_cst); +} + +#endif /* USE_STDATOMIC_H */ + +#endif /* STDATOMIC_IMPL_H */ diff --git a/src/include/port/spin_delay.h b/src/include/port/spin_delay.h new file mode 100644 index 00000000000..5b3d45587a8 --- /dev/null +++ b/src/include/port/spin_delay.h @@ -0,0 +1,151 @@ +/*------------------------------------------------------------------------- + * + * spin_delay.h + * Platform-specific spin delay for busy-wait loops + * + * This file provides pg_spin_delay(), a platform-optimized delay instruction + * for use in spinlock contention loops. Different architectures have different + * optimal instructions for indicating to the CPU that we're in a busy-wait. + * + * Key optimizations: + * - x86/x86_64: PAUSE instruction (rep nop) reduces power and helps hyperthreads + * - ARM64: ISB instruction, per the discussion thread below, backs off better + * than YIELD under heavy spinlock contention on high-core-count parts + * - Windows ARM64: __isb() intrinsic (same rationale as ARM64 above) + * + * Discussion: https://postgr.es/m/1c2a29b8-5b1e-44f7-a871-71ec5fefc120%40app.fastmail.com + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/port/spin_delay.h + * + *------------------------------------------------------------------------- + */ +#ifndef SPIN_DELAY_H +#define SPIN_DELAY_H + +#ifdef _MSC_VER +/* + * MSVC intrinsics used below (__isb, _mm_pause, _ReadWriteBarrier) are declared + * in , which also provides 's _mm_pause. Include it at + * file scope rather than relying on another header having pulled it in first. + */ +#include +#endif + +/* + * pg_spin_delay_impl - Execute a platform-specific CPU hint for spin-wait loops + * + * This function should be called inside busy-wait loops to: + * 1. Reduce CPU power consumption during contention + * 2. Improve performance of sibling hyperthreads + * 3. Signal to the CPU that we're in a spin loop + * + * The implementation varies by platform to use the most efficient instruction. + * + * Note: This defines pg_spin_delay_impl() directly, matching the pattern used + * by the traditional atomics implementation (arch-*.h files). + */ +#ifndef PG_HAVE_SPIN_DELAY +#define PG_HAVE_SPIN_DELAY +static inline void +pg_spin_delay_impl(void) +{ +#if defined(__GNUC__) || defined(__INTEL_COMPILER) + + /* + * GCC and Intel compiler: use inline assembly for optimal instructions + */ + +#if defined(__i386__) || defined(__i386) + /* x86 32-bit: PAUSE instruction (encoded as rep nop) */ + __asm__ __volatile__(" rep; nop \n"); +#elif defined(__x86_64__) + /* x86-64: PAUSE instruction */ + __asm__ __volatile__(" rep; nop \n"); +#elif defined(__aarch64__) + /* + * ARM64: ISB (Instruction Synchronization Barrier). Per the discussion + * thread above, ISB backs off better than YIELD in spinlock loops on + * high-core-count ARM64 parts; it forces a pipeline flush that YIELD does + * not. + */ + __asm__ __volatile__(" isb; \n"); +#elif defined(__arm__) || defined(__arm) + /* ARM 32-bit: YIELD hint */ + __asm__ __volatile__(" yield; \n"); +#else + /* + * Other architectures: compiler barrier only + * + * A compiler barrier prevents the compiler from optimizing away the loop, + * even if we don't have an architecture-specific delay instruction. + */ + __asm__ __volatile__("":::"memory"); +#endif + +#elif defined(_MSC_VER) + + /* + * Microsoft Visual C++: use intrinsics (declared in , included + * at file scope above). + */ + +#if defined(_M_ARM64) || defined(_M_ARM64EC) + /* + * Windows ARM64: ISB via intrinsic, matching the GCC/Clang ARM64 path + * above. _ARM64_BARRIER_SY is a full system barrier. + */ + __isb(_ARM64_BARRIER_SY); +#elif defined(_M_AMD64) + /* Windows x86-64: _mm_pause() maps to the PAUSE instruction. */ + _mm_pause(); +#elif defined(_M_IX86) + /* Windows x86 32-bit: MASM syntax for PAUSE (rep nop). */ + __asm rep nop; +#else + /* + * Other Windows architectures: compiler barrier only, matching the + * unknown-compiler fallback below. + */ + _ReadWriteBarrier(); +#endif + +#else + /* + * Unknown compiler: no-op with compiler barrier + * + * At minimum, we need to prevent the compiler from optimizing away the + * spin loop. + */ + (void) 0; +#endif +} +#endif /* PG_HAVE_SPIN_DELAY */ + +/* + * Public spin-delay macro for the stdatomic path. + * + * This is the public pg_spin_delay() entry point used by the stdatomic + * spinlock path; it maps to the pg_spin_delay_impl() defined above. + */ +#ifndef pg_spin_delay +#define pg_spin_delay() pg_spin_delay_impl() +#endif + +/* + * Architectures where a plain load before the atomic exchange reduces + * cache-coherency traffic under spinlock contention (test-and-test-and-set). + * + * Restricted to x86/x86_64: these are TSO and pg_atomic_read_u32() (seq_cst) + * compiles to an ordinary load there, so the pre-read is genuinely cheap. On + * weakly-ordered targets a seq_cst load emits fence instructions, which would + * defeat the point of a cheap pre-check, so they are intentionally omitted. + */ +#if defined(__i386__) || defined(__x86_64__) || \ + defined(_M_IX86) || defined(_M_AMD64) +#define PG_SPIN_TRY_RELAXED +#endif + +#endif /* SPIN_DELAY_H */ diff --git a/src/include/port/spin_delay_status.h b/src/include/port/spin_delay_status.h new file mode 100644 index 00000000000..738ecb12a44 --- /dev/null +++ b/src/include/port/spin_delay_status.h @@ -0,0 +1,52 @@ +/*------------------------------------------------------------------------- + * + * spin_delay_status.h + * SpinDelayStatus type and the spin-delay/backoff helpers + * + * This header holds the delay/backoff bookkeeping used by the spinlock slow + * path (perform_spin_delay / finish_spin_delay in s_lock.c) and by other + * spinlock-like busy-wait loops. It is included by storage/spin.h so the + * definitions live in exactly one place. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/port/spin_delay_status.h + * + *------------------------------------------------------------------------- + */ +#ifndef SPIN_DELAY_STATUS_H +#define SPIN_DELAY_STATUS_H + +#define DEFAULT_SPINS_PER_DELAY 100 + +typedef struct +{ + int spins; + int delays; + int cur_delay; + const char *file; + int line; + const char *func; +} SpinDelayStatus; + +static inline void +init_spin_delay(SpinDelayStatus *status, + const char *file, int line, const char *func) +{ + status->spins = 0; + status->delays = 0; + status->cur_delay = 0; + status->file = file; + status->line = line; + status->func = func; +} + +#define init_local_spin_delay(status) init_spin_delay(status, __FILE__, __LINE__, __func__) + +extern void perform_spin_delay(SpinDelayStatus *status); +extern void finish_spin_delay(SpinDelayStatus *status); +extern void set_spins_per_delay(int shared_spins_per_delay); +extern int update_spins_per_delay(int shared_spins_per_delay); + +#endif /* SPIN_DELAY_STATUS_H */ diff --git a/src/include/storage/spin.h b/src/include/storage/spin.h index 9cf6adb671a..28121482224 100644 --- a/src/include/storage/spin.h +++ b/src/include/storage/spin.h @@ -44,6 +44,80 @@ #ifndef SPIN_H #define SPIN_H +#ifdef USE_STDATOMIC_H + +/* + * Atomics-based spinlocks. + * + * When stdatomic.h is available, spinlocks are implemented on top of a plain + * 32-bit atomic rather than platform-specific TAS assembly. We deliberately + * do NOT use pg_atomic_flag: that type uses 1==unlocked so it can offer a + * relaxed "unlocked test", whereas a spinlock must be usable when its memory + * has merely been zeroed (much shared-memory state is set up with + * memset(...,0,...) and then relies on embedded spinlocks reading as free). + * So slock_t uses the traditional convention: 0 == unlocked, 1 == locked, + * making a zero-initialized slock_t a valid, free lock. + */ +#include "port/atomics.h" + +typedef pg_atomic_uint32 slock_t; + +/* SpinDelayStatus and the spin-delay helpers (perform/finish_spin_delay). */ +#include "port/spin_delay_status.h" + +extern int s_lock(volatile slock_t *lock, const char *file, int line, const char *func); + +static inline void +SpinLockInit(volatile slock_t *lock) +{ + pg_atomic_init_u32(lock, 0); /* 0 = unlocked */ +} + +/* + * SpinLockAcquire - acquire a spinlock, waiting if necessary. + * + * Fast path: a single atomic exchange (0->1). On failure fall into s_lock(), + * which does the backoff/retry and the "stuck spinlock" diagnostics. + * + * This is split into an inline helper plus a wrapper macro: the macro exists + * only to capture the call site's __FILE__/__LINE__/__func__ for diagnostics, + * while the helper evaluates its lock argument exactly once (a macro doing the + * exchange-then-s_lock inline would evaluate the argument twice) and preserves + * the volatile slock_t * type check. + */ +static inline void +spin_lock_acquire(volatile slock_t *lock, const char *file, int line, + const char *func) +{ + if (pg_atomic_exchange_u32(lock, 1) != 0) + s_lock(lock, file, line, func); +} +#define SpinLockAcquire(lock) \ + spin_lock_acquire((lock), __FILE__, __LINE__, __func__) + +static inline void +SpinLockRelease(volatile slock_t *lock) +{ + /* + * Release the lock with release ordering: every load and store in the + * critical section must complete before the lock is observed free. In the + * stdatomic implementation pg_write_barrier() is a release thread fence + * (atomic_thread_fence with release order), which orders the prior + * critical-section accesses -- loads included -- ahead of the following + * store; the plain store then publishes the unlocked value. This gives + * the same release semantics as the traditional S_UNLOCK() while avoiding + * the store-side serialization (STLR on ARM64) that a seq_cst store adds. + */ + pg_write_barrier(); + pg_atomic_write_u32(lock, 0); +} + +#else /* !USE_STDATOMIC_H */ + +/* + * Traditional spinlock implementation using platform-specific TAS assembly. + * This branch is kept byte-for-byte equivalent to the pre-stdatomic spin.h. + */ #include "storage/s_lock.h" static inline void @@ -64,4 +138,6 @@ SpinLockRelease(volatile slock_t *lock) S_UNLOCK(lock); } +#endif /* USE_STDATOMIC_H */ + #endif /* SPIN_H */ diff --git a/src/test/regress/regress.c b/src/test/regress/regress.c index 6ee6689a468..df6931a0cf5 100644 --- a/src/test/regress/regress.c +++ b/src/test/regress/regress.c @@ -651,6 +651,10 @@ test_spinlock(void) * * We embed the spinlock in a struct with other members to test that the * spinlock operations don't perform too wide writes. + * + * Note: slock_t is pg_atomic_uint32 (4 bytes) in the stdatomic path vs + * potentially smaller (e.g., unsigned char) in the traditional path. This + * test verifies no over-wide writes regardless of slock_t size. */ { struct test_lock_struct @@ -668,15 +672,22 @@ test_spinlock(void) SpinLockAcquire(&struct_w_lock.lock); SpinLockRelease(&struct_w_lock.lock); +#ifndef USE_STDATOMIC_H /* test basic operations via underlying S_* API */ S_INIT_LOCK(&struct_w_lock.lock); S_LOCK(&struct_w_lock.lock); S_UNLOCK(&struct_w_lock.lock); +#endif /* and that "contended" acquisition works */ s_lock(&struct_w_lock.lock, "testfile", 17, "testfunc"); +#ifdef USE_STDATOMIC_H + SpinLockRelease(&struct_w_lock.lock); +#else S_UNLOCK(&struct_w_lock.lock); +#endif +#ifndef USE_STDATOMIC_H /* * Check, using TAS directly, that a single spin cycle doesn't block * when acquiring an already acquired lock. @@ -694,6 +705,7 @@ test_spinlock(void) S_UNLOCK(&struct_w_lock.lock); #endif /* defined(TAS) */ +#endif /* !USE_STDATOMIC_H */ /* * Verify that after all of this the non-lock contents are still -- 2.50.1