From d61fa41fcd53f672fd0b1e8cbc9bf0c9ee689ef9 Mon Sep 17 00:00:00 2001 From: Greg Burd Date: Thu, 9 Jul 2026 12:34:55 -0400 Subject: [PATCH v9 1/3] Add an opt-in C11 stdatomic.h implementation of the atomics API PostgreSQL requires C11, but not all supported compilers provide a fully functional . This adds an alternative implementation of port/atomics.h built on the standard C11 header, selectable at build time alongside the existing platform-specific implementations. Build-system detection (meson and autoconf): A new option -- meson -Duse_stdatomic=yes|no|auto and autoconf --with-stdatomic=yes|no|auto -- controls the choice: * 'no' (default): always use the traditional implementation * 'auto': use stdatomic.h if a probe link succeeds * 'yes': require stdatomic.h (fail if unavailable) * 'no': use the traditional platform-specific implementation Both build systems run the same probe (32-bit fetch_add, a seq_cst thread fence, and a 64-bit compare_exchange) and define USE_STDATOMIC_H when it succeeds and the option is not 'no'. For MSVC, stdatomic.h is only available in Visual Studio 2022 or later with /experimental:c11atomics, which the build adds automatically; earlier MSVC falls back to the traditional implementation. Implementation (src/include/port/atomics/stdatomic_impl.h): All atomic types (pg_atomic_flag, pg_atomic_uint8/16/32/64) are implemented with C11 _Atomic. Memory ordering matches the traditional implementation's observable semantics: * pg_atomic_write_* uses memory_order_relaxed, matching the plain volatile stores of the traditional path (no fence). * pg_atomic_read_* currently uses memory_order_seq_cst. This is stronger than the documented "no barrier semantics" contract and is a WORKAROUND, not a considered design: with relaxed reads, join_hash intermittently loses a tuple on rv64, and the cause is not yet understood. See the -hackers thread; this must be resolved (and most likely reverted to relaxed) before commit. * read-modify-write, exchange, and compare-exchange use memory_order_seq_cst, matching the traditional "full barrier" contract. * the _membarrier variants use a seq_cst fetch_add(0) / exchange, not a seq_cst load/store: the contract is a *full* barrier, and a seq_cst load is a plain mov on x86 (no ordering against a preceding store) while a seq_cst store is an STLR on ARM64 (no ordering of later loads). This matches generic.h. C++ compatibility is provided via for C++11 through C++20, which do not include ; C++23 and later include it directly. Platform-specific spin-delay hints (PAUSE, ISB, YIELD) are factored into a new src/include/port/spin_delay.h, with the shared SpinDelayStatus type and helpers in src/include/port/spin_delay_status.h so both spinlock paths use them. On ARM64, ISB is used instead of YIELD for better backoff under contention; on Windows ARM64 this adds a spin delay where the traditional implementation had none. Wire-up (src/include/port/atomics.h, storage/spin.h, s_lock.c): When USE_STDATOMIC_H is defined, atomics.h includes stdatomic_impl.h; otherwise it includes the arch-*.h and generic-*.h headers exactly as before. Both paths feed the same shared public-API layer (the static inline pg_atomic_* wrappers), so the API is identical either way. For spinlocks, when USE_STDATOMIC_H is defined slock_t is a pg_atomic_uint32 (0 == unlocked, 1 == locked, so a zeroed slock_t is a valid free lock) and the SpinLock* operations are built on the u32 atomic API rather than platform-specific TAS assembly. SpinLockAcquire is a thin wrapper macro over an inline helper: the macro captures the call site's __FILE__/__LINE__/__func__ for stuck-spinlock diagnostics while the helper evaluates its lock argument exactly once. On strong-memory platforms a relaxed-load test precedes the atomic exchange (PG_SPIN_TRY_RELAXED). When USE_STDATOMIC_H is not defined, spinlock behavior is byte-for-byte unchanged from previous PostgreSQL versions. With stdatomic.h, atomic operations no longer depend on backend-only infrastructure, so the frontend include restriction in atomics.h is relaxed for that path. The traditional implementation remains the default , this is an opt-in alternative rather than a replacement. Both code paths provide identical public API, semantics, and roughly equal performance. Co-authored-by: Greg Burd Co-authored-by: Thomas Munro Discussion: https://postgr.es/m/CC76554F-41AC-45AA-AF10-370FEC416498%40greg.burd.me#dd4b139cd557d6e87a9f2d706b6415ae (cherry picked from commit 55ae00c0d00b3f0cf523edd44eb6a95f225486ba) --- configure | 100 ++++ configure.ac | 64 +++ meson.build | 103 ++++ meson_options.txt | 5 + src/backend/storage/lmgr/README | 54 ++ src/backend/storage/lmgr/s_lock.c | 48 +- src/include/pg_config.h.in | 4 + src/include/port/atomics.h | 103 +++- src/include/port/atomics/stdatomic_impl.h | 586 ++++++++++++++++++++++ src/include/port/spin_delay.h | 148 ++++++ src/include/port/spin_delay_status.h | 52 ++ src/include/storage/spin.h | 76 ++- src/test/regress/regress.c | 13 + 13 files changed, 1341 insertions(+), 15 deletions(-) create mode 100644 src/include/port/atomics/stdatomic_impl.h create mode 100644 src/include/port/spin_delay.h create mode 100644 src/include/port/spin_delay_status.h diff --git a/configure b/configure index 6b05d4c84ab..485ebc6304b 100755 --- a/configure +++ b/configure @@ -4918,6 +4918,13 @@ fi + +# +# C11 stdatomic.h +# +# stdatomic.h is the sole atomics implementation. The actual detection runs +# later, after the compiler and 64-bit integer handling have been configured; +# a missing or broken header is a hard error there. for ac_prog in gawk mawk nawk awk do # Extract the first word of "$ac_prog", so it can be a program name with args. @@ -17701,6 +17708,99 @@ $as_echo "#define HAVE_GCC__ATOMIC_INT64_CAS 1" >>confdefs.h fi +# Require a working C11 stdatomic.h: it is the sole atomics implementation, +# so a missing or broken header is a hard error. Use a link test, not a +# compile test: 64-bit atomic_compare_exchange_strong() can lower to a +# libatomic call (__atomic_compare_exchange_8) on some 32-bit or +# weakly-aligned targets, where a compile-only check would pass and then fail +# at final link. Retry with -latomic if the bare link fails. +# +# The test also requires the 32- and 64-bit atomics to be *always* lock-free +# (ATOMIC_*_LOCK_FREE == 2). PostgreSQL's atomics live in shared memory and +# are accessed from multiple processes, so an implementation that falls back +# to a lock table (as libatomic does when the hardware lacks the operation) +# would take process-local locks and silently fail to be atomic across +# backends. stdatomic_impl.h asserts the same thing; checking it here means +# an unsuitable target is rejected with this message rather than with a +# static assertion failure part-way through the build. +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for working C11 stdatomic.h" >&5 +$as_echo_n "checking for working C11 stdatomic.h... " >&6; } +if ${pgac_cv_stdatomic+:} false; then : + $as_echo_n "(cached) " >&6 +else + pgac_cv_stdatomic=no + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include +#if ATOMIC_INT_LOCK_FREE != 2 || ATOMIC_LLONG_LOCK_FREE != 2 +#error atomics are not always lock-free +#endif +int +main () +{ +_Atomic int x = 0; + atomic_fetch_add(&x, 1); + atomic_thread_fence(memory_order_seq_cst); + _Atomic(unsigned long long) y = 0; + unsigned long long expected = 0; + atomic_compare_exchange_strong(&y, &expected, 42); + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + pgac_cv_stdatomic=yes +else + pgac_save_LIBS=$LIBS + LIBS="$LIBS -latomic" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include +#if ATOMIC_INT_LOCK_FREE != 2 || ATOMIC_LLONG_LOCK_FREE != 2 +#error atomics are not always lock-free +#endif +int +main () +{ +_Atomic int x = 0; + atomic_fetch_add(&x, 1); + atomic_thread_fence(memory_order_seq_cst); + _Atomic(unsigned long long) y = 0; + unsigned long long expected = 0; + atomic_compare_exchange_strong(&y, &expected, 42); + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + pgac_cv_stdatomic='yes, with -latomic' +else + pgac_cv_stdatomic=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext + LIBS=$pgac_save_LIBS +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $pgac_cv_stdatomic" >&5 +$as_echo "$pgac_cv_stdatomic" >&6; } + +if test "$pgac_cv_stdatomic" = no; then + as_fn_error $? "a working C11 stdatomic.h with always-lock-free 32- and 64-bit atomics is required (it is the only atomics implementation)" "$LINENO" 5 +fi + + +$as_echo "#define USE_STDATOMIC_H 1" >>confdefs.h + +if test "$pgac_cv_stdatomic" = 'yes, with -latomic'; then + LIBS="$LIBS -latomic" +fi + + # Check for __get_cpuid() and __cpuid() { $as_echo "$as_me:${as_lineno-$LINENO}: checking for __get_cpuid" >&5 $as_echo_n "checking for __get_cpuid... " >&6; } diff --git a/configure.ac b/configure.ac index 6864268eaa0..ca9b4470322 100644 --- a/configure.ac +++ b/configure.ac @@ -443,6 +443,13 @@ choke me PGAC_ARG_BOOL(with, llvm, no, [build with LLVM based JIT support], [AC_DEFINE([USE_LLVM], 1, [Define to 1 to build with LLVM based JIT support. (--with-llvm)])]) AC_SUBST(with_llvm) + +# +# C11 stdatomic.h +# +# stdatomic.h is the sole atomics implementation. The actual detection runs +# later, after the compiler and 64-bit integer handling have been configured; +# a missing or broken header is a hard error there. dnl must use AS_IF here, else AC_REQUIRES inside PGAC_LLVM_SUPPORT malfunctions AS_IF([test "$with_llvm" = yes], [ PGAC_LLVM_SUPPORT() @@ -2112,6 +2119,63 @@ PGAC_HAVE_GCC__ATOMIC_INT32_CAS PGAC_HAVE_GCC__ATOMIC_INT64_CAS +# Require a working C11 stdatomic.h: it is the sole atomics implementation, +# so a missing or broken header is a hard error. Use a link test, not a +# compile test: 64-bit atomic_compare_exchange_strong() can lower to a +# libatomic call (__atomic_compare_exchange_8) on some 32-bit or +# weakly-aligned targets, where a compile-only check would pass and then fail +# at final link. Retry with -latomic if the bare link fails. +# +# The test also requires the 32- and 64-bit atomics to be *always* lock-free +# (ATOMIC_*_LOCK_FREE == 2). PostgreSQL's atomics live in shared memory and +# are accessed from multiple processes, so an implementation that falls back +# to a lock table (as libatomic does when the hardware lacks the operation) +# would take process-local locks and silently fail to be atomic across +# backends. stdatomic_impl.h asserts the same thing; checking it here means +# an unsuitable target is rejected with this message rather than with a +# static assertion failure part-way through the build. +AC_CACHE_CHECK([for working C11 stdatomic.h], [pgac_cv_stdatomic], +[pgac_cv_stdatomic=no + AC_LINK_IFELSE([AC_LANG_PROGRAM([#include +#if ATOMIC_INT_LOCK_FREE != 2 || ATOMIC_LLONG_LOCK_FREE != 2 +#error atomics are not always lock-free +#endif], + [[_Atomic int x = 0; + atomic_fetch_add(&x, 1); + atomic_thread_fence(memory_order_seq_cst); + _Atomic(unsigned long long) y = 0; + unsigned long long expected = 0; + atomic_compare_exchange_strong(&y, &expected, 42); + ]])], + [pgac_cv_stdatomic=yes], + [pgac_save_LIBS=$LIBS + LIBS="$LIBS -latomic" + AC_LINK_IFELSE([AC_LANG_PROGRAM([#include +#if ATOMIC_INT_LOCK_FREE != 2 || ATOMIC_LLONG_LOCK_FREE != 2 +#error atomics are not always lock-free +#endif], + [[_Atomic int x = 0; + atomic_fetch_add(&x, 1); + atomic_thread_fence(memory_order_seq_cst); + _Atomic(unsigned long long) y = 0; + unsigned long long expected = 0; + atomic_compare_exchange_strong(&y, &expected, 42); + ]])], + [pgac_cv_stdatomic='yes, with -latomic'], + [pgac_cv_stdatomic=no]) + LIBS=$pgac_save_LIBS])]) + +if test "$pgac_cv_stdatomic" = no; then + AC_MSG_ERROR([a working C11 stdatomic.h with always-lock-free 32- and 64-bit atomics is required (it is the only atomics implementation)]) +fi + +AC_DEFINE([USE_STDATOMIC_H], 1, + [Define to 1 to use C11 stdatomic.h for atomic operations.]) +if test "$pgac_cv_stdatomic" = 'yes, with -latomic'; then + LIBS="$LIBS -latomic" +fi + + # Check for __get_cpuid() and __cpuid() AC_CACHE_CHECK([for __get_cpuid], [pgac_cv__get_cpuid], [AC_LINK_IFELSE([AC_LANG_PROGRAM([#include ], diff --git a/meson.build b/meson.build index 7e95fd5941d..fae3978c150 100644 --- a/meson.build +++ b/meson.build @@ -674,6 +674,109 @@ if have_cxx and not cxx.compiles(cxx11_test, name: 'C++11') endif +############################################################### +# Check for C11 stdatomic.h support +############################################################### + +stdatomic_test_code = ''' +#include +/* + * Require the 32- and 64-bit atomics to be *always* lock-free. PostgreSQL's + * atomics live in shared memory and are used from multiple processes, so an + * implementation that falls back to a lock table (as libatomic does when the + * hardware lacks the operation) would take process-local locks and silently + * fail to be atomic across backends. stdatomic_impl.h asserts the same + * thing; checking it here means an unsuitable target is rejected by the probe + * rather than by a static assertion part-way through the build. + * + * MSVC compiling as C reports 1 for every width even though the atomics are + * in fact always lock-free on the targets we support, so skip the check + * there (stdatomic_impl.h skips its assertion in the same configuration). + */ +#if !(defined(_MSC_VER) && !defined(__cplusplus)) +#if ATOMIC_INT_LOCK_FREE != 2 || ATOMIC_LLONG_LOCK_FREE != 2 +#error atomics are not always lock-free +#endif +#endif +int main(void) { + _Atomic int x = 0; + atomic_fetch_add(&x, 1); + atomic_thread_fence(memory_order_seq_cst); + _Atomic(unsigned long long) y = 0; + unsigned long long expected = 0; + atomic_compare_exchange_strong(&y, &expected, 42); + return 0; +} +''' + +has_stdatomic = false +atomic_link_args = [] + +# The stdatomic probe below is a *link* test, not just a compile test: 64-bit +# atomic_compare_exchange_strong() can lower to a libatomic call +# (__atomic_compare_exchange_8) on some 32-bit or weakly-aligned targets. A +# compile-only check would pass there and then fail at final link, so we +# require the probe to link, retrying with -latomic if the bare link fails. + +if cc.get_id() == 'msvc' + msvc_ver = cc.version() + if msvc_ver.version_compare('>=19.30') # Visual Studio 2022+ + message('MSVC version @0@ detected (>= 19.30), checking for stdatomic.h support'.format(msvc_ver)) + # MSVC's requires C11 mode (/std:c11) in addition to + # /experimental:c11atomics; without /std:c11 the header errors out with + # "C atomics require C11 or later". The probe must pass both, since + # test_c_args is not yet populated with the C11 flag at this point. + if cc.links(stdatomic_test_code, + name: 'MSVC stdatomic.h with /experimental:c11atomics', + args: test_c_args + ['/std:c11', '/experimental:c11atomics']) + has_stdatomic = true + message('MSVC stdatomic.h is available with /experimental:c11atomics') + else + message('MSVC stdatomic.h test compilation failed') + endif + else + message('MSVC version @0@ is too old for stdatomic.h (requires >= 19.30)'.format(msvc_ver)) + endif +else + # GCC/Clang and other compilers + if cc.has_header('stdatomic.h', args: test_c_args) + message('stdatomic.h header found, testing that it links...') + if cc.links(stdatomic_test_code, + name: 'stdatomic.h works', + args: test_c_args) + has_stdatomic = true + message('stdatomic.h is available and working') + elif cc.links(stdatomic_test_code, + name: 'stdatomic.h works with -latomic', + args: test_c_args + ['-latomic']) + # Some targets need libatomic for the wider (e.g. 64-bit) operations. + has_stdatomic = true + atomic_link_args = ['-latomic'] + message('stdatomic.h is available and working (requires -latomic)') + else + message('stdatomic.h found but test link failed') + endif + else + message('stdatomic.h header not found') + endif +endif + +# stdatomic.h is the sole atomics implementation, so a missing or broken +# header is a hard error. +if not has_stdatomic + error('a working C11 stdatomic.h with always-lock-free 32- and 64-bit atomics is required (it is the only atomics implementation)') +endif +cdata.set('USE_STDATOMIC_H', 1) +message('Using C11 stdatomic.h for atomic operations') +if atomic_link_args.length() > 0 + # libatomic is needed for atomics the compiler cannot inline on this target. + os_deps += cc.find_library('atomic') +endif +if cc.get_id() == 'msvc' + # MSVC needs C11 mode plus the experimental atomics switch for . + cflags += ['/std:c11', '/experimental:c11atomics'] +endif + postgres_inc = [include_directories(postgres_inc_d)] test_lib_d = postgres_lib_d test_c_args = cppflags + cflags diff --git a/meson_options.txt b/meson_options.txt index 6a793f3e479..f8c5ccc14e1 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -52,6 +52,11 @@ option('PG_TEST_EXTRA', type: 'string', value: '', option('PG_GIT_REVISION', type: 'string', value: 'HEAD', description: 'git revision to be packaged by pgdist target') +option('use_stdatomic', type: 'combo', + choices: ['auto', 'yes', 'no'], + value: 'no', + description: 'Use C11 stdatomic.h for atomic operations (auto, yes, no)') + # Compilation options diff --git a/src/backend/storage/lmgr/README b/src/backend/storage/lmgr/README index 45de0fd2bd6..f09795b0a43 100644 --- a/src/backend/storage/lmgr/README +++ b/src/backend/storage/lmgr/README @@ -47,6 +47,60 @@ when the wait time might exceed a few seconds. The rest of this README file discusses the regular lock manager in detail. +Atomic Operations +----------------- + +PostgreSQL's atomic operations (see src/include/port/atomics.h) are +implemented on top of the standard C11 header, which is the +sole atomics implementation. A working stdatomic.h is required at build +time; the USE_STDATOMIC_H preprocessor symbol is always defined. Available +on: + - MSVC 2022+ (requires /experimental:c11atomics flag) + - GCC 4.9+ and Clang 3.1+ (no special flags needed) + +There is no build option to select the atomics implementation: stdatomic.h +is probed unconditionally and a missing or non-working header is a hard +configure/build error, since there is no other implementation to fall back +to. + +Several internal properties of the implementation are worth noting; none of +them affect the public API: + +* Memory ordering. pg_atomic_read_u32/u64() use seq_cst ordering; + pg_atomic_write_u32/u64() use relaxed. The seq_cst read is + required for correctness on weak-memory hardware (e.g. RISC-V), where the + weaker relaxed and acquire orderings let a concurrent reader miss a tuple + in a parallel hash join. The write stays relaxed: a plain atomic store + matches the "no barrier" contract for pg_atomic_write and, on + ARM64, avoids the store-release (STLR) serialization that would penalize + contended writes to hot shared state. The pg_atomic_unlocked_write_* + variant also uses relaxed ordering, consistent with its "no guarantees" + contract. On x86 (TSO) a seq_cst load is still a plain mov, so there is no + added cost; + on ARM/RISC-V it adds fence instructions on these paths. + +* Barriers. Each barrier pairs a compiler barrier (atomic_signal_fence) with + the thread fence (atomic_thread_fence), because a bare thread fence orders + only atomic accesses and would let the compiler reorder plain loads/stores + across it, whereas PostgreSQL's barrier contract must order non-atomic + accesses too. + +* pg_atomic_flag polarity. pg_atomic_flag uses 1=unlocked, 0=locked + (fetch_and-based test-and-set). Note this is only about pg_atomic_flag, + not about spinlocks: slock_t keeps the traditional 0=unlocked, 1=locked + convention, so a zero-initialized spinlock is a free lock. The flag + polarity is an internal detail with no external visibility beyond raw + values seen in a debugger. + +* Platform reach. Genuine always-lock-free 64-bit atomics are required: a + compile-time static assertion rejects lock-emulated atomics, which use + process-local locks and so cannot be atomic across the multiple processes + that share PostgreSQL's shared memory. The headers are C++-compatible. + +For more information on memory barriers and atomic operations, see +src/backend/storage/lmgr/README.barrier. + + Lock Data Structures -------------------- diff --git a/src/backend/storage/lmgr/s_lock.c b/src/backend/storage/lmgr/s_lock.c index 795aeed1028..52fdf9fd69a 100644 --- a/src/backend/storage/lmgr/s_lock.c +++ b/src/backend/storage/lmgr/s_lock.c @@ -51,7 +51,13 @@ #include #include "common/pg_prng.h" +#ifdef USE_STDATOMIC_H +#include "port/atomics.h" +#include "port/spin_delay.h" +#include "storage/spin.h" +#else #include "storage/s_lock.h" +#endif #include "utils/wait_event.h" #define MIN_SPINS_PER_DELAY 10 @@ -60,6 +66,7 @@ #define MIN_DELAY_USEC 1000L #define MAX_DELAY_USEC 1000000L +#ifndef USE_STDATOMIC_H #ifdef S_LOCK_TEST /* * These are needed by pgstat_report_wait_start in the standalone compile of @@ -68,6 +75,7 @@ static uint32 local_my_wait_event_info; uint32 *my_wait_event_info = &local_my_wait_event_info; #endif +#endif /* !USE_STDATOMIC_H */ static int spins_per_delay = DEFAULT_SPINS_PER_DELAY; @@ -80,7 +88,7 @@ s_lock_stuck(const char *file, int line, const char *func) { if (!func) func = "(unknown)"; -#if defined(S_LOCK_TEST) +#if !defined(USE_STDATOMIC_H) && defined(S_LOCK_TEST) fprintf(stderr, "\nStuck spinlock detected at %s, %s:%d.\n", func, file, line); @@ -91,7 +99,11 @@ s_lock_stuck(const char *file, int line, const char *func) #endif } -#ifdef USE_DEFAULT_S_LOCK +/* + * USE_DEFAULT_S_LOCK is defined by s_lock.h, which the stdatomic path does + * not include; spin.h calls s_lock() unconditionally, so define it for both. + */ +#if defined(USE_STDATOMIC_H) || defined(USE_DEFAULT_S_LOCK) /* * s_lock(lock) - platform-independent portion of waiting for a spinlock. */ @@ -102,10 +114,32 @@ s_lock(volatile slock_t *lock, const char *file, int line, const char *func) init_spin_delay(&delayStatus, file, line, func); +#ifdef USE_STDATOMIC_H + for (;;) + { + bool try_to_set; + +#ifdef PG_SPIN_TRY_RELAXED + /* + * Test-and-test-and-set: read the lock first and only attempt the + * exchange when it looks free, to avoid taking cache-line ownership on + * every spin. PG_SPIN_TRY_RELAXED is defined only where this pre-read + * is cheap (x86/x86_64 TSO, where a seq_cst load is a plain load). + */ + try_to_set = (pg_atomic_read_u32(lock) == 0); +#else + try_to_set = true; +#endif + if (try_to_set && pg_atomic_exchange_u32(lock, 1) == 0) + break; + perform_spin_delay(&delayStatus); + } +#else /* !USE_STDATOMIC_H */ while (TAS_SPIN(lock)) { perform_spin_delay(&delayStatus); } +#endif /* USE_STDATOMIC_H */ finish_spin_delay(&delayStatus); @@ -113,6 +147,7 @@ s_lock(volatile slock_t *lock, const char *file, int line, const char *func) } #endif +#ifndef USE_STDATOMIC_H #ifdef USE_DEFAULT_S_UNLOCK void s_unlock(volatile slock_t *lock) @@ -120,6 +155,7 @@ s_unlock(volatile slock_t *lock) *lock = 0; } #endif +#endif /* !USE_STDATOMIC_H */ /* * Wait while spinning on a contended spinlock. @@ -128,7 +164,11 @@ void perform_spin_delay(SpinDelayStatus *status) { /* CPU-specific delay each time through the loop */ +#ifdef USE_STDATOMIC_H + pg_spin_delay(); +#else SPIN_DELAY(); +#endif /* Block the process every spins_per_delay tries */ if (++(status->spins) >= spins_per_delay) @@ -151,7 +191,7 @@ perform_spin_delay(SpinDelayStatus *status) pg_usleep(status->cur_delay); pgstat_report_wait_end(); -#if defined(S_LOCK_TEST) +#if !defined(USE_STDATOMIC_H) && defined(S_LOCK_TEST) fprintf(stdout, "*"); fflush(stdout); #endif @@ -234,6 +274,7 @@ update_spins_per_delay(int shared_spins_per_delay) /*****************************************************************************/ +#ifndef USE_STDATOMIC_H #if defined(S_LOCK_TEST) /* @@ -300,3 +341,4 @@ main() } #endif /* S_LOCK_TEST */ +#endif /* !USE_STDATOMIC_H */ diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 661c4a9b168..31aff5d59f9 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -744,6 +744,10 @@ /* Define to 1 to use Intel SSE 4.2 CRC instructions with a runtime check. */ #undef USE_SSE42_CRC32C_WITH_RUNTIME_CHECK +/* Define to 1 to use C11 stdatomic.h for atomic operations. + (--with-stdatomic) */ +#undef USE_STDATOMIC_H + /* Define to 1 to use SVE popcount instructions with a runtime check. */ #undef USE_SVE_POPCNT_WITH_RUNTIME_CHECK diff --git a/src/include/port/atomics.h b/src/include/port/atomics.h index a605ea81d07..2058dbb9238 100644 --- a/src/include/port/atomics.h +++ b/src/include/port/atomics.h @@ -7,6 +7,26 @@ * atomically and dealing with cache coherency. Used to implement locking * facilities and lockless algorithms/data structures. * + * IMPLEMENTATION SELECTION: + * + * PostgreSQL's atomic operations can be implemented using either: + * + * 1. C11 stdatomic.h (when USE_STDATOMIC_H is defined) + * - Uses standard C11 for all atomic operations + * - Better portability and compiler optimizations + * - Requires MSVC 2022+ or GCC 4.9+/Clang 3.1+ + * + * 2. Traditional platform-specific implementations (when USE_STDATOMIC_H is not defined) + * - Uses battle-tested PostgreSQL implementations + * - Architecture-specific: arch-x86.h, arch-arm.h, arch-ppc.h + * - Compiler intrinsics: generic-gcc.h, generic-msvc.h + * - Fallback implementations: generic.h, fallback.h + * + * Both implementations provide identical public API and semantics. + * Selection is made at build time via -Duse_stdatomic=auto/yes/no. + * + * PORTING NOTES: + * * To bring up postgres on a platform/compiler at the very least * implementations for the following operations should be provided: * * pg_compiler_barrier(), pg_write_barrier(), pg_read_barrier() @@ -14,13 +34,8 @@ * * pg_atomic_test_set_flag(), pg_atomic_init_flag(), pg_atomic_clear_flag() * * PG_HAVE_8BYTE_SINGLE_COPY_ATOMICITY should be defined if appropriate. * - * There exist generic, hardware independent, implementations for several - * compilers which might be sufficient, although possibly not optimal, for a - * new platform. If no such generic implementation is available spinlocks will - * be used to implement the 64-bit parts of the API. - * - * Implement _u64 atomics if and only if your platform can use them - * efficiently (and obviously correctly). + * For new platforms, prefer using stdatomic.h if available, as it reduces + * maintenance burden and leverages compiler-provided implementations. * * Use higher level functionality (lwlocks, spinlocks, heavyweight locks) * whenever possible. Writing correct code using these facilities is hard. @@ -38,15 +53,76 @@ #ifndef ATOMICS_H #define ATOMICS_H +/* + * Frontend code restriction + * + * Traditionally, atomics.h could not be included from frontend code because + * the platform-specific implementations often relied on backend-only features. + * + * With the stdatomic.h implementation, this restriction is no longer necessary + * since stdatomic.h is a standard header. However, we keep the restriction for + * the traditional implementation path to maintain compatibility. + */ #ifdef FRONTEND -#error "atomics.h may not be included from frontend code" +#ifndef USE_STDATOMIC_H +#error "atomics.h may not be included from frontend code (use -Duse_stdatomic=yes if atomics are needed)" +#endif #endif #define INSIDE_ATOMICS_H +/* + * The public API and the _impl signatures keep the historical volatile + * qualifier on the atomic pointer, because many callers reach atomic fields + * through volatile-qualified struct pointers. Under C11 the _Atomic types + * carry their own ordering guarantees, so the volatile is redundant for the + * atomic accesses themselves and is accepted (and effectively ignored for the + * atomic operation) by the supported compilers, including MSVC. + */ + #include /* + * PostgreSQL atomics can be implemented using either C11 stdatomic.h + * or platform-specific implementations. The choice is made at build + * configuration time via the USE_STDATOMIC_H preprocessor definition. + * + * When USE_STDATOMIC_H is defined, we use C11 for all + * atomic operations. This provides better portability and potentially + * better compiler optimizations. + * + * When USE_STDATOMIC_H is not defined (default initially), we use + * traditional platform-specific implementations (arch-*.h, generic-*.h) + * that have been battle-tested in PostgreSQL for many years. + * + * Both paths provide identical public API and semantics. + */ + +#ifdef USE_STDATOMIC_H + +/* + * C11 stdatomic.h implementation path + * + * This uses the standard C11 header for atomic operations. + * The type definitions and implementations are in stdatomic_impl.h. + */ + +#include "port/atomics/stdatomic_impl.h" + +/* + * The public API is provided by static inline functions in the common code + * section below. Those functions call _impl functions, which are provided by + * stdatomic_impl.h for this path. + */ + +#else /* !USE_STDATOMIC_H */ + +/* + * Traditional platform-specific implementation path + * + * This is the original PostgreSQL atomics implementation using + * architecture-specific headers and compiler intrinsics. + * * First a set of architecture specific files is included. * * These files can provide the full set of atomics or can do pretty much @@ -116,6 +192,17 @@ */ #include "port/atomics/generic.h" +#endif /* USE_STDATOMIC_H */ + +/* + * Common code for both stdatomic.h and traditional implementations. + * + * The following definitions provide the public API that works identically + * regardless of which implementation is used. The static inline functions + * below call _impl functions which are provided by either stdatomic_impl.h + * (when USE_STDATOMIC_H is defined) or the traditional implementation files + * (generic.h, etc.) otherwise. + */ /* * pg_compiler_barrier - prevent the compiler from moving code across diff --git a/src/include/port/atomics/stdatomic_impl.h b/src/include/port/atomics/stdatomic_impl.h new file mode 100644 index 00000000000..4e0d893e1c3 --- /dev/null +++ b/src/include/port/atomics/stdatomic_impl.h @@ -0,0 +1,586 @@ +/*------------------------------------------------------------------------- + * + * stdatomic_impl.h + * Atomic operations implementation using C11 stdatomic.h + * + * This file provides PostgreSQL atomic operations using the C11 standard + * . As of this commit it is the sole atomics implementation; + * the former platform-specific paths (arch-*.h, generic-*.h, fallback.h) + * have been removed and USE_STDATOMIC_H is always defined. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/port/atomics/stdatomic_impl.h + * + *------------------------------------------------------------------------- + */ + +#ifndef STDATOMIC_IMPL_H +#define STDATOMIC_IMPL_H + +/* + * C++ Compatibility + * + * C++11 through C++20 cannot include C's directly. Instead, + * we use C++11's header and map to std::atomic types. C++23 allows + * but we handle older versions. + * + * This follows the approach from C++23 standard section 33.5.12 which + * allows C and C++ atomic types to interoperate. + */ +#if defined(__cplusplus) && __cplusplus < 202302L +extern "C++" +{ +#include + + /* Map to C++ atomic types */ +#define pg_atomic(T) std::atomic + + /* Import memory order constants into global namespace */ + using std::memory_order_relaxed; + using std::memory_order_acquire; + using std::memory_order_release; + using std::memory_order_acq_rel; + using std::memory_order_seq_cst; +} +#else +/* + * C11, or C++23 and later where is available. + * + * In C++ mode this must be wrapped in extern "C++": under C++23 + * pulls in , which declares templates, and extensions + * conventionally include our headers inside extern "C". Templates cannot + * have C language linkage, so an unguarded include fails to compile. + */ +#ifdef __cplusplus +extern "C++" +{ +#include +} +#else +#include +#endif +#define pg_atomic(T) _Atomic(T) +#endif + +/* + * Type Definitions + * + * We use C11 _Atomic qualifier (or C++ std::atomic) with PostgreSQL's + * standard integer types. These types are compatible with atomic operations + * and provide sequential consistency guarantees. + * + * Note: pg_atomic(T) is defined above and maps to either _Atomic(T) for C + * or std::atomic for C++. + */ +/* + * pg_atomic_flag is backed by a full 32-bit word, not uint8, matching the + * traditional implementation. The reason is the one given in generic-gcc.h: + * int-width atomic operations are more efficient and/or more reliably + * implemented than byte-width ones. (Sub-word atomics are not unsafe on + * targets without byte-granular AMOs -- compilers emit a masked LR/SC on the + * containing word, which preserves the neighbouring bytes -- but they are + * bigger and slower than a plain word AMO.) + */ +typedef pg_atomic(uint32) pg_atomic_flag; +typedef pg_atomic(uint32) pg_atomic_uint32; +typedef pg_atomic(uint64) pg_atomic_uint64; + +/* + * PostgreSQL's atomics live in shared memory and are accessed from multiple + * *processes*, not just threads. A non-lock-free atomic implemented by the + * compiler/runtime with a lock table (e.g. libatomic) uses process-local + * locks, which would silently fail to provide atomicity across processes and + * corrupt shared state. Require genuine, always-lock-free support at compile + * time for every width we use. (ATOMIC_*_LOCK_FREE is 0 = never lock-free, + * 1 = sometimes / not known at compile time, 2 = always lock-free.) + * + * The == 1 case is a real hazard, not just conservatism: e.g. 64-bit atomics + * on a 32-bit target may be emulated by libatomic with a process-local lock, + * which corrupts cross-process shared memory exactly as == 0 would. So we + * require == 2 everywhere the macro is trustworthy. + * + * The one exception is MSVC compiling as C: its reports + * ATOMIC_*_LOCK_FREE == 1 for every width even though the atomics are in fact + * always lock-free on the hardware we target (in C++ mode the same MSVC + * reports 2, and atomic_is_lock_free() returns true at run time). We skip the + * assertion only in that specific configuration rather than weakening it for + * everyone. + */ +#if !(defined(_MSC_VER) && !defined(__cplusplus)) +StaticAssertDecl(ATOMIC_INT_LOCK_FREE == 2, "32-bit atomics are not lock-free"); +StaticAssertDecl(ATOMIC_LLONG_LOCK_FREE == 2, "64-bit atomics are not lock-free"); +#endif + +/* + * Signal atomic operation support to atomics.h + * + * stdatomic.h provides native support for all atomic types on all platforms + * it compiles on, so we unconditionally define these. PG_HAVE_ATOMIC_U64_SIMULATION + * is intentionally NOT defined -- stdatomic provides real 64-bit atomics. + */ +#define PG_HAVE_ATOMIC_U32_SUPPORT +#define PG_HAVE_ATOMIC_U64_SUPPORT + +/* + * 8-byte single-copy atomicity. + * + * All 64-bit architectures that support C11 stdatomic.h guarantee 8-byte + * single-copy atomicity for aligned loads/stores. This is used by bufmgr.c + * and other performance-critical paths to avoid unnecessary atomic operations. + */ +#if SIZEOF_VOID_P >= 8 +#define PG_HAVE_8BYTE_SINGLE_COPY_ATOMICITY +#endif + +/* + * Memory Barrier Implementation + * + * PostgreSQL's memory barriers must order *non-atomic, non-volatile* accesses + * too, which a bare C11 atomic_thread_fence() does NOT guarantee against + * compiler reordering: atomic_thread_fence only establishes ordering with + * respect to atomic operations, so the compiler is still free to move plain + * loads/stores across it. We therefore pair each thread fence with an + * explicit compiler barrier (atomic_signal_fence), matching the semantics of + * the traditional implementation whose barriers also constrain plain memory + * accesses. Omitting the compiler barrier corrupts lock-free algorithms on + * weak-memory architectures (observed as lost tuples in parallel hash join on + * RISC-V, where the compiler reordered non-atomic hash-table stores across a + * bare acquire/release fence). + * + * These are implementation functions (_impl suffix) mapped to the public API + * by atomics.h. + */ +#define pg_compiler_barrier_impl() atomic_signal_fence(memory_order_seq_cst) + +static inline void +pg_memory_barrier_impl(void) +{ + atomic_signal_fence(memory_order_seq_cst); + atomic_thread_fence(memory_order_seq_cst); +} + +static inline void +pg_read_barrier_impl(void) +{ + atomic_signal_fence(memory_order_seq_cst); + atomic_thread_fence(memory_order_acquire); +} + +static inline void +pg_write_barrier_impl(void) +{ + atomic_signal_fence(memory_order_seq_cst); + atomic_thread_fence(memory_order_release); +} + +/* + * Spin Delay Implementation + * + * On x86, a spinloop without a "pause" instruction can waste a lot of power + * and slow down the adjacent hyperthread. On ARM64, ISB provides better + * backoff than YIELD. This provides platform-specific spin delay hints. + * + * The implementation is in spin_delay.h which directly defines pg_spin_delay_impl(), + * matching the pattern used by the traditional atomics implementation. + */ +#include "port/spin_delay.h" + +/* + * ================================================================= + * pg_atomic_flag Operations + * ================================================================= + * + * Atomic flag type used primarily for spinlocks. + * + * CRITICAL FLAG CONVENTION DIFFERENCE FROM TRADITIONAL IMPLEMENTATION: + * + * stdatomic.h implementation (this file): + * 1 = unlocked, 0 = locked + * + * Traditional implementation (generic.h): + * 0 = unlocked, 1 = locked + * + * This polarity difference is INTENTIONAL and internal to the implementation. + * The public API behavior is identical in both implementations: + * - pg_atomic_init_flag() initializes to unlocked state + * - pg_atomic_test_set_flag() returns true if lock was successfully acquired + * - pg_atomic_clear_flag() releases the lock + * + * The stdatomic.h convention (1=unlocked) is more natural for C11 atomics + * and avoids double-negation logic in the implementation. + */ + +/* + * pg_atomic_init_flag_impl - Initialize atomic flag to unlocked state + * + * In the stdatomic.h implementation, we initialize to 1 (unlocked). + */ +static inline void +pg_atomic_init_flag_impl(volatile pg_atomic_flag *ptr) +{ + atomic_init(ptr, 1); /* 1 = unlocked */ +} + +/* + * pg_atomic_test_set_flag_impl - Try to acquire lock + * + * Atomically clears the flag and returns whether it had been set. Returns + * true if we acquired it (previous value 1 == unlocked), false if it was + * already held (previous value 0). + * + * An exchange is used rather than fetch_and(0): both express the same + * operation, but exchange is the one every target lowers to a single + * instruction (xchg on x86), whereas fetch_and can fall back to a + * compare-exchange loop on some compilers. + * + * Memory ordering: acquire (synchronizes-with prior release) + */ +static inline bool +pg_atomic_test_set_flag_impl(volatile pg_atomic_flag *ptr) +{ + return atomic_exchange_explicit(ptr, 0, memory_order_acquire) != 0; +} + +/* + * pg_atomic_unlocked_test_flag_impl - Test if flag is unlocked without acquiring + * + * Returns true if the flag is currently unlocked (value == 1). + * Uses relaxed ordering since this is typically used for optimization hints. + */ +static inline bool +pg_atomic_unlocked_test_flag_impl(volatile pg_atomic_flag *ptr) +{ + return atomic_load_explicit(ptr, memory_order_relaxed) != 0; +} + +/* + * pg_atomic_clear_flag_impl - Release lock + * + * Sets flag to 1 (unlocked). + * Memory ordering: release (makes prior writes visible) + */ +static inline void +pg_atomic_clear_flag_impl(volatile pg_atomic_flag *ptr) +{ + atomic_store_explicit(ptr, 1, memory_order_release); +} + +/* + * ================================================================= + * pg_atomic_uint32 Operations + * ================================================================= + * + * 32-bit atomic unsigned integer operations. + * All operations use sequential consistency unless otherwise noted. + */ + +/* + * pg_atomic_init_u32_impl - Initialize atomic uint32 + * + * This is not an atomic operation - must only be called during initialization + * when no concurrent access is possible. + */ +static inline void +pg_atomic_init_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 val) +{ + atomic_init(ptr, val); +} + +/* + * pg_atomic_read_u32_impl - Atomically read uint32 value + * + * Memory ordering: sequentially consistent. This is stronger than the + * documented "no barrier semantics" of pg_atomic_read_u32(), and it is + * deliberate: a bare memory_order_relaxed load reintroduces a real, + * reproducible data-loss bug on weak-memory hardware. + * + * The traditional implementation reads through a volatile-qualified pointer. + * A volatile access cannot be reordered *by the compiler* relative to the + * surrounding code, so consumers that publish a pointer with a + * barrier-carrying store and then read+dereference it get the dependent load + * ordered for free. A C11 memory_order_relaxed atomic load does not carry + * that compiler-ordering property, and the parallel hash join relies on it: + * ExecParallelHash reads a freshly published bucket pointer via + * dsa_pointer_atomic_read() and then dereferences the tuple it points to. + * With a relaxed load this was observed to lose a tuple on RISC-V -- + * join_hash "extremely_skewed" returns 19999 instead of 20000, intermittently + * (reproduced here at roughly 1 run in 4 on real rv64 hardware). seq_cst + * restores correctness. + * + * The cost of the seq_cst load is not the ARM regression measured for an + * earlier revision of this work: an uncontended ldar is not measurably + * costlier than a plain ldr, and that ~2% aarch64 read-only regression was + * traced to the seq_cst store (STLR) in pg_atomic_write_u32, which is relaxed + * (see below). + * + * Leaving this primitive relaxed and adding the ordering only at the apparent + * consumer -- e.g. a pg_read_barrier() between the bucket read and the tuple + * dereference in ExecParallelHashFirstTuple() -- is not sufficient: on RISC-V + * join_hash still lost tuples at the same rate. The ordering the volatile + * load provided is not local to that one read; the parallel hash join also + * reaches shared state through the Barrier, condition-variable and LWLock + * machinery, all of which read via pg_atomic_read_*() and relied on the same + * compiler-ordering. An ordered read here restores that property tree-wide. + */ +static inline uint32 +pg_atomic_read_u32_impl(volatile pg_atomic_uint32 *ptr) +{ + return atomic_load_explicit(ptr, memory_order_seq_cst); +} + +/* + * pg_atomic_write_u32_impl - Atomically write uint32 value + * + * Memory ordering: relaxed. A plain atomic (non-torn) store matches the + * traditional implementation's "no barrier" contract for pg_atomic_write_u32 + * and, on ARM64, compiles to a plain STR rather than STLR. This matters: + * benchmarking traced the ~2% aarch64 read-only regression of an earlier + * seq_cst-everywhere revision to STLR store-side serialization on hot, + * contended shared lines (LWLock state, buffer headers), not to the reads. + * Making the write relaxed removes that cost. Callers needing an ordered + * write use pg_atomic_write_membarrier_u32() or an explicit barrier. + */ +static inline void +pg_atomic_write_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 val) +{ + atomic_store_explicit(ptr, val, memory_order_relaxed); +} + +/* + * pg_atomic_exchange_u32_impl - Atomically exchange uint32 value + * + * Atomically replaces the value with newval and returns the old value. + * Memory ordering: seq_cst + */ +static inline uint32 +pg_atomic_exchange_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 newval) +{ + return atomic_exchange_explicit(ptr, newval, memory_order_seq_cst); +} + +/* + * pg_atomic_compare_exchange_u32_impl - Atomic compare-and-swap + * + * Compares *ptr with *expected. If equal, replaces *ptr with newval and + * returns true. If not equal, updates *expected with current *ptr value + * and returns false. + * + * This is the strong version (no spurious failures). + * Memory ordering: seq_cst for both success and failure + */ +static inline bool +pg_atomic_compare_exchange_u32_impl(volatile pg_atomic_uint32 *ptr, + uint32 *expected, uint32 newval) +{ + return atomic_compare_exchange_strong_explicit(ptr, expected, newval, + memory_order_seq_cst, + memory_order_seq_cst); +} + +/* + * pg_atomic_fetch_add_u32_impl - Atomic fetch-and-add + * + * Atomically adds add_ to *ptr and returns the old value. + * Memory ordering: seq_cst + */ +static inline uint32 +pg_atomic_fetch_add_u32_impl(volatile pg_atomic_uint32 *ptr, int32 add_) +{ + return atomic_fetch_add_explicit(ptr, add_, memory_order_seq_cst); +} + +/* + * pg_atomic_fetch_sub_u32_impl - Atomic fetch-and-subtract + * + * Atomically subtracts sub_ from *ptr and returns the old value. + * Memory ordering: seq_cst + */ +static inline uint32 +pg_atomic_fetch_sub_u32_impl(volatile pg_atomic_uint32 *ptr, int32 sub_) +{ + return atomic_fetch_sub_explicit(ptr, sub_, memory_order_seq_cst); +} + +/* + * pg_atomic_fetch_and_u32_impl - Atomic fetch-and-AND + * + * Atomically performs *ptr &= and_ and returns the old value. + * Memory ordering: seq_cst + */ +static inline uint32 +pg_atomic_fetch_and_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 and_) +{ + return atomic_fetch_and_explicit(ptr, and_, memory_order_seq_cst); +} + +/* + * pg_atomic_fetch_or_u32_impl - Atomic fetch-and-OR + * + * Atomically performs *ptr |= or_ and returns the old value. + * Memory ordering: seq_cst + */ +static inline uint32 +pg_atomic_fetch_or_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 or_) +{ + return atomic_fetch_or_explicit(ptr, or_, memory_order_seq_cst); +} + +/* + * pg_atomic_add_fetch_u32_impl - Atomic add-and-fetch + * + * Atomically adds add_ to *ptr and returns the NEW value. + * Implemented using fetch_add + add_ since C11 doesn't have add_fetch. + */ +static inline uint32 +pg_atomic_add_fetch_u32_impl(volatile pg_atomic_uint32 *ptr, int32 add_) +{ + return atomic_fetch_add_explicit(ptr, add_, memory_order_seq_cst) + add_; +} + +/* + * pg_atomic_sub_fetch_u32_impl - Atomic subtract-and-fetch + * + * Atomically subtracts sub_ from *ptr and returns the NEW value. + */ +static inline uint32 +pg_atomic_sub_fetch_u32_impl(volatile pg_atomic_uint32 *ptr, int32 sub_) +{ + return atomic_fetch_sub_explicit(ptr, sub_, memory_order_seq_cst) - sub_; +} + +/* + * pg_atomic_read_membarrier_u32_impl - Read with full barrier semantics + * + * The contract is a *full* barrier, which a seq_cst load does not provide: + * on x86 a seq_cst load is a plain mov and is not ordered against a + * preceding store. Use an RMW that returns the value, matching generic.h. + */ +static inline uint32 +pg_atomic_read_membarrier_u32_impl(volatile pg_atomic_uint32 *ptr) +{ + return atomic_fetch_add_explicit(ptr, 0, memory_order_seq_cst); +} + +/* + * pg_atomic_unlocked_write_u32_impl - Non-atomic write + * + * Write without atomicity guarantees. Only safe when exclusive access + * is guaranteed externally. Uses relaxed ordering. + */ +static inline void +pg_atomic_unlocked_write_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 val) +{ + atomic_store_explicit(ptr, val, memory_order_relaxed); +} + +/* + * pg_atomic_write_membarrier_u32_impl - Write with full barrier semantics + * + * A seq_cst store is not a full barrier either: on ARM64 an STLR does not + * order later plain loads. Use an exchange, matching generic.h. + */ +static inline void +pg_atomic_write_membarrier_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 val) +{ + (void) atomic_exchange_explicit(ptr, val, memory_order_seq_cst); +} + +/* + * ================================================================= + * pg_atomic_uint64 Operations + * ================================================================= + * + * 64-bit equivalents of the u32 operations above. See the u32 comments + * for API documentation; semantics and memory ordering are identical. + */ + +static inline void +pg_atomic_init_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 val) +{ + atomic_init(ptr, val); +} + +static inline uint64 +pg_atomic_read_u64_impl(volatile pg_atomic_uint64 *ptr) +{ + return atomic_load_explicit(ptr, memory_order_seq_cst); +} + +static inline void +pg_atomic_write_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 val) +{ + atomic_store_explicit(ptr, val, memory_order_relaxed); +} + +static inline uint64 +pg_atomic_exchange_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 newval) +{ + return atomic_exchange_explicit(ptr, newval, memory_order_seq_cst); +} + +static inline bool +pg_atomic_compare_exchange_u64_impl(volatile pg_atomic_uint64 *ptr, + uint64 *expected, uint64 newval) +{ + return atomic_compare_exchange_strong_explicit(ptr, expected, newval, + memory_order_seq_cst, + memory_order_seq_cst); +} + +static inline uint64 +pg_atomic_fetch_add_u64_impl(volatile pg_atomic_uint64 *ptr, int64 add_) +{ + return atomic_fetch_add_explicit(ptr, add_, memory_order_seq_cst); +} + +static inline uint64 +pg_atomic_fetch_sub_u64_impl(volatile pg_atomic_uint64 *ptr, int64 sub_) +{ + return atomic_fetch_sub_explicit(ptr, sub_, memory_order_seq_cst); +} + +static inline uint64 +pg_atomic_fetch_and_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 and_) +{ + return atomic_fetch_and_explicit(ptr, and_, memory_order_seq_cst); +} + +static inline uint64 +pg_atomic_fetch_or_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 or_) +{ + return atomic_fetch_or_explicit(ptr, or_, memory_order_seq_cst); +} + +static inline uint64 +pg_atomic_add_fetch_u64_impl(volatile pg_atomic_uint64 *ptr, int64 add_) +{ + return atomic_fetch_add_explicit(ptr, add_, memory_order_seq_cst) + add_; +} + +static inline uint64 +pg_atomic_sub_fetch_u64_impl(volatile pg_atomic_uint64 *ptr, int64 sub_) +{ + return atomic_fetch_sub_explicit(ptr, sub_, memory_order_seq_cst) - sub_; +} + +static inline uint64 +pg_atomic_read_membarrier_u64_impl(volatile pg_atomic_uint64 *ptr) +{ + return atomic_fetch_add_explicit(ptr, 0, memory_order_seq_cst); +} + +static inline void +pg_atomic_unlocked_write_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 val) +{ + atomic_store_explicit(ptr, val, memory_order_relaxed); +} + +static inline void +pg_atomic_write_membarrier_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 val) +{ + (void) atomic_exchange_explicit(ptr, val, memory_order_seq_cst); +} + +#endif /* STDATOMIC_IMPL_H */ diff --git a/src/include/port/spin_delay.h b/src/include/port/spin_delay.h new file mode 100644 index 00000000000..c88b3729a2e --- /dev/null +++ b/src/include/port/spin_delay.h @@ -0,0 +1,148 @@ +/*------------------------------------------------------------------------- + * + * spin_delay.h + * Platform-specific spin delay for busy-wait loops + * + * This file provides pg_spin_delay(), a platform-optimized delay instruction + * for use in spinlock contention loops. Different architectures have different + * optimal instructions for indicating to the CPU that we're in a busy-wait. + * + * Key optimizations: + * - x86/x86_64: PAUSE instruction (rep nop) reduces power and helps hyperthreads + * - ARM64: ISB instruction, per the discussion thread below, backs off better + * than YIELD under heavy spinlock contention on high-core-count parts + * - Windows ARM64: __isb() intrinsic (same rationale as ARM64 above) + * + * Discussion: https://postgr.es/m/1c2a29b8-5b1e-44f7-a871-71ec5fefc120%40app.fastmail.com + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/port/spin_delay.h + * + *------------------------------------------------------------------------- + */ +#ifndef SPIN_DELAY_H +#define SPIN_DELAY_H + +#ifdef _MSC_VER +/* + * MSVC intrinsics used below (__isb, _mm_pause, _ReadWriteBarrier) are declared + * in , which also provides 's _mm_pause. Include it at + * file scope rather than relying on another header having pulled it in first. + */ +#include +#endif + +/* + * pg_spin_delay_impl - Execute a platform-specific CPU hint for spin-wait loops + * + * This function should be called inside busy-wait loops to: + * 1. Reduce CPU power consumption during contention + * 2. Improve performance of sibling hyperthreads + * 3. Signal to the CPU that we're in a spin loop + * + * The implementation varies by platform to use the most efficient instruction. + * + * Note: This defines pg_spin_delay_impl() directly, matching the pattern used + * by the traditional atomics implementation (arch-*.h files). + */ +#ifndef PG_HAVE_SPIN_DELAY +#define PG_HAVE_SPIN_DELAY +static inline void +pg_spin_delay_impl(void) +{ +#if defined(__GNUC__) || defined(__INTEL_COMPILER) + + /* + * GCC and Intel compiler: use inline assembly for optimal instructions + */ + +#if defined(__i386__) || defined(__i386) || defined(__x86_64__) + /* x86 / x86-64: PAUSE instruction (encoded as rep nop) */ + __asm__ __volatile__(" rep; nop \n"); +#elif defined(__aarch64__) + /* + * ARM64: ISB (Instruction Synchronization Barrier). Per the discussion + * thread above, ISB backs off better than YIELD in spinlock loops on + * high-core-count ARM64 parts; it forces a pipeline flush that YIELD does + * not. + */ + __asm__ __volatile__(" isb; \n"); +#elif defined(__arm__) || defined(__arm) + /* ARM 32-bit: YIELD hint */ + __asm__ __volatile__(" yield; \n"); +#else + /* + * Other architectures: compiler barrier only + * + * A compiler barrier prevents the compiler from optimizing away the loop, + * even if we don't have an architecture-specific delay instruction. + */ + __asm__ __volatile__("":::"memory"); +#endif + +#elif defined(_MSC_VER) + + /* + * Microsoft Visual C++: use intrinsics (declared in , included + * at file scope above). + */ + +#if defined(_M_ARM64) || defined(_M_ARM64EC) + /* + * Windows ARM64: ISB via intrinsic, matching the GCC/Clang ARM64 path + * above. _ARM64_BARRIER_SY is a full system barrier. + */ + __isb(_ARM64_BARRIER_SY); +#elif defined(_M_AMD64) + /* Windows x86-64: _mm_pause() maps to the PAUSE instruction. */ + _mm_pause(); +#elif defined(_M_IX86) + /* Windows x86 32-bit: MASM syntax for PAUSE (rep nop). */ + __asm rep nop; +#else + /* + * Other Windows architectures: compiler barrier only, matching the + * unknown-compiler fallback below. + */ + _ReadWriteBarrier(); +#endif + +#else + /* + * Unknown compiler: plain no-op. We have no portable way to emit a + * compiler barrier here, but the only caller (perform_spin_delay() in + * s_lock.c) has observable side effects, so the spin loop is not at risk + * of being optimized away. + */ + (void) 0; +#endif +} +#endif /* PG_HAVE_SPIN_DELAY */ + +/* + * Public spin-delay macro for the stdatomic path. + * + * This is the public pg_spin_delay() entry point used by the stdatomic + * spinlock path; it maps to the pg_spin_delay_impl() defined above. + */ +#ifndef pg_spin_delay +#define pg_spin_delay() pg_spin_delay_impl() +#endif + +/* + * Architectures where a plain load before the atomic exchange reduces + * cache-coherency traffic under spinlock contention (test-and-test-and-set). + * + * Restricted to x86/x86_64: these are TSO and pg_atomic_read_u32() (seq_cst) + * compiles to an ordinary load there, so the pre-read is genuinely cheap. On + * weakly-ordered targets a seq_cst load emits fence instructions, which would + * defeat the point of a cheap pre-check, so they are intentionally omitted. + */ +#if defined(__i386__) || defined(__x86_64__) || \ + defined(_M_IX86) || defined(_M_AMD64) +#define PG_SPIN_TRY_RELAXED +#endif + +#endif /* SPIN_DELAY_H */ diff --git a/src/include/port/spin_delay_status.h b/src/include/port/spin_delay_status.h new file mode 100644 index 00000000000..738ecb12a44 --- /dev/null +++ b/src/include/port/spin_delay_status.h @@ -0,0 +1,52 @@ +/*------------------------------------------------------------------------- + * + * spin_delay_status.h + * SpinDelayStatus type and the spin-delay/backoff helpers + * + * This header holds the delay/backoff bookkeeping used by the spinlock slow + * path (perform_spin_delay / finish_spin_delay in s_lock.c) and by other + * spinlock-like busy-wait loops. It is included by storage/spin.h so the + * definitions live in exactly one place. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/port/spin_delay_status.h + * + *------------------------------------------------------------------------- + */ +#ifndef SPIN_DELAY_STATUS_H +#define SPIN_DELAY_STATUS_H + +#define DEFAULT_SPINS_PER_DELAY 100 + +typedef struct +{ + int spins; + int delays; + int cur_delay; + const char *file; + int line; + const char *func; +} SpinDelayStatus; + +static inline void +init_spin_delay(SpinDelayStatus *status, + const char *file, int line, const char *func) +{ + status->spins = 0; + status->delays = 0; + status->cur_delay = 0; + status->file = file; + status->line = line; + status->func = func; +} + +#define init_local_spin_delay(status) init_spin_delay(status, __FILE__, __LINE__, __func__) + +extern void perform_spin_delay(SpinDelayStatus *status); +extern void finish_spin_delay(SpinDelayStatus *status); +extern void set_spins_per_delay(int shared_spins_per_delay); +extern int update_spins_per_delay(int shared_spins_per_delay); + +#endif /* SPIN_DELAY_STATUS_H */ diff --git a/src/include/storage/spin.h b/src/include/storage/spin.h index 9cf6adb671a..56ef38dffb0 100644 --- a/src/include/storage/spin.h +++ b/src/include/storage/spin.h @@ -28,10 +28,9 @@ * for a CHECK_FOR_INTERRUPTS() to occur while holding a spinlock, and so * it is not necessary to do HOLD/RESUME_INTERRUPTS() in these functions. * - * These functions are implemented in terms of hardware-dependent macros - * supplied by s_lock.h. There is not currently any extra functionality - * added by this header, but there has been in the past and may someday - * be again. + * These functions are implemented on top of the C11 atomic + * operations in port/atomics.h; the contended-acquire slow path lives in + * s_lock() in s_lock.c. * * * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group @@ -44,6 +43,73 @@ #ifndef SPIN_H #define SPIN_H +/* + * Atomics-based spinlocks. + * + * Spinlocks are implemented directly on top of a plain 32-bit atomic rather + * than platform-specific TAS assembly. We deliberately do NOT use + * pg_atomic_flag: that type uses 1==unlocked so it can offer a relaxed + * "unlocked test", whereas a spinlock must be usable when its memory has + * merely been zeroed (a great deal of shared-memory state is set up with + * memset(...,0,...) and then relies on embedded spinlocks reading as free). + * So slock_t uses the traditional convention: 0 == unlocked, 1 == locked, + * which makes a zero-initialized slock_t a valid, free lock. + */ +#ifdef USE_STDATOMIC_H + +#include "port/atomics.h" + +typedef pg_atomic_uint32 slock_t; + +/* SpinDelayStatus and the spin-delay helpers (perform/finish_spin_delay). */ +#include "port/spin_delay_status.h" + +extern int s_lock(volatile slock_t *lock, const char *file, int line, const char *func); + +static inline void +SpinLockInit(volatile slock_t *lock) +{ + pg_atomic_init_u32(lock, 0); /* 0 = unlocked */ +} + +/* + * SpinLockAcquire - acquire a spinlock, waiting if necessary. + * + * Fast path: a single atomic exchange (0->1). On failure fall into s_lock(), + * which does the backoff/retry and the "stuck spinlock" diagnostics. + * + * This is split into an inline helper plus a wrapper macro: the macro exists + * only to capture the call site's __FILE__/__LINE__/__func__ for diagnostics, + * while the helper evaluates its lock argument exactly once (a macro doing the + * exchange-then-s_lock inline would evaluate the argument twice) and preserves + * the volatile slock_t * type check. + */ +static inline void +spin_lock_acquire(volatile slock_t *lock, const char *file, int line, + const char *func) +{ + if (pg_atomic_exchange_u32(lock, 1) != 0) + s_lock(lock, file, line, func); +} +#define SpinLockAcquire(lock) \ + spin_lock_acquire((lock), __FILE__, __LINE__, __func__) + +static inline void +SpinLockRelease(volatile slock_t *lock) +{ + /* + * Release the lock. A release store is exactly the right semantics and + * matches __sync_lock_release() in the traditional path: mov on x86, + * STLR on ARM64, fence rw,w + sw on rv64. (An earlier revision used + * pg_memory_barrier() followed by a relaxed store, which cost a full + * fence on every release on x86, where master emits only a mov.) + */ + atomic_store_explicit(lock, 0, memory_order_release); +} + +#else /* !USE_STDATOMIC_H */ + +/* Traditional implementation: unchanged from master. */ #include "storage/s_lock.h" static inline void @@ -64,4 +130,6 @@ SpinLockRelease(volatile slock_t *lock) S_UNLOCK(lock); } +#endif /* USE_STDATOMIC_H */ + #endif /* SPIN_H */ diff --git a/src/test/regress/regress.c b/src/test/regress/regress.c index 3deb4ed7203..2fd3b260cbd 100644 --- a/src/test/regress/regress.c +++ b/src/test/regress/regress.c @@ -652,6 +652,10 @@ test_spinlock(void) * * We embed the spinlock in a struct with other members to test that the * spinlock operations don't perform too wide writes. + * + * Note: slock_t is pg_atomic_uint32 (4 bytes) in the stdatomic path vs + * potentially smaller (e.g., unsigned char) in the traditional path. This + * test verifies no over-wide writes regardless of slock_t size. */ { struct test_lock_struct @@ -669,15 +673,23 @@ test_spinlock(void) SpinLockAcquire(&struct_w_lock.lock); SpinLockRelease(&struct_w_lock.lock); +#ifndef USE_STDATOMIC_H /* test basic operations via underlying S_* API */ S_INIT_LOCK(&struct_w_lock.lock); S_LOCK(&struct_w_lock.lock); S_UNLOCK(&struct_w_lock.lock); +#endif /* and that "contended" acquisition works */ +#ifdef USE_STDATOMIC_H + s_lock(&struct_w_lock.lock, "testfile", 17, "testfunc"); + SpinLockRelease(&struct_w_lock.lock); +#else S_LOCK(&struct_w_lock.lock); S_UNLOCK(&struct_w_lock.lock); +#endif +#ifndef USE_STDATOMIC_H /* * Check, using TAS directly, that a single spin cycle doesn't block * when acquiring an already acquired lock. @@ -695,6 +707,7 @@ test_spinlock(void) S_UNLOCK(&struct_w_lock.lock); #endif /* defined(TAS) */ +#endif /* !USE_STDATOMIC_H */ /* * Verify that after all of this the non-lock contents are still -- 2.50.1