From 73086c4665e37de1f08df59b88137f3180307183 Mon Sep 17 00:00:00 2001 From: Greg Burd Date: Fri, 23 Oct 2026 08:04:43 -0400 Subject: [PATCH v12] Use C11 to implement port/atomics.h and spinlocks PostgreSQL requires C11, which provides . Reimplement port/atomics.h on top of it, and storage/spin.h on top of that, and remove the hand-written platform-specific implementations: src/include/port/atomics/arch-arm.h src/include/port/atomics/arch-ppc.h src/include/port/atomics/arch-x86.h src/include/port/atomics/fallback.h src/include/port/atomics/generic-gcc.h src/include/port/atomics/generic-msvc.h src/include/port/atomics/generic.h src/include/storage/s_lock.h The public pg_atomic_* and SpinLock* APIs are unchanged. Build requirements (meson and autoconf): configure and meson now require a that works and whose 32- and 64-bit atomics are always lock-free (ATOMIC_INT_LOCK_FREE and ATOMIC_LLONG_LOCK_FREE == 2), and fail with a clear message otherwise. The probe is a link test that retries with -latomic, because a 64-bit compare-exchange can lower to a libatomic call on some 32-bit targets. Lock-freedom is required because PostgreSQL's atomics live in shared memory and are used from multiple processes: a lock-table fallback of the kind libatomic uses would take process-local locks and silently fail to be atomic across backends. MSVC needs Visual Studio 2022 or later with /experimental:c11atomics, which the build adds; MSVC compiling as C reports ATOMIC_*_LOCK_FREE as 1 although the atomics are lock-free, so the lock-freedom check is skipped there. Implementation (src/include/port/atomics/stdatomic_impl.h): pg_atomic_flag, pg_atomic_uint32 and pg_atomic_uint64 are C11 _Atomic types; pg_atomic_flag is backed by a 32-bit word. Memory ordering: * pg_atomic_write_* is memory_order_relaxed, like the plain store of the previous implementation. * pg_atomic_read_* is memory_order_seq_cst, stronger than the documented "no barrier semantics" contract. With a relaxed read, join_hash intermittently loses a tuple on rv64 built with gcc 13.3 (gcc 15.2 and clang 21 are unaffected, and restoring the previous spinlock does not help). The stronger ordering is kept until that is understood. * read-modify-write, exchange and compare-exchange are seq_cst, matching the "full barrier" contract. * the _membarrier variants use a seq_cst fetch_add(0) and exchange, not a seq_cst load and store, because the contract is a full barrier: a seq_cst load is a plain mov on x86 with no ordering against a preceding store, and a seq_cst store is an STLR on ARM64 that does not order later loads. * pg_atomic_test_set_flag is an exchange. The read/write/memory barriers pair atomic_thread_fence() with atomic_signal_fence(), because the C11 model orders only atomic accesses while our barriers must also order plain ones (the problem c5d34f4a addressed in generic-gcc.h with an asm memory clobber). For C++, is used for C++11 through C++20, which cannot include , and for C++23 and later; both includes are wrapped in extern "C++" so that extensions including our headers inside extern "C" still build. Spinlocks (storage/spin.h, s_lock.c): slock_t is a pg_atomic_uint32 (0 == unlocked, so a zeroed slock_t is a valid free lock). SpinLockAcquire is an exchange with a fallback to s_lock(); SpinLockRelease is a release store. SpinLockAcquire is a wrapper macro over an inline helper so the caller's __FILE__/__LINE__/__func__ reach stuck-spinlock reports while the lock argument is evaluated once. In the contended path in s_lock(), x86 reads the lock before retrying the exchange (test-and-test-and-set), to avoid taking cache-line ownership on every spin. The standalone s_lock_test program, which exercised the removed TAS macros, is removed; regress.c's test_spinlock() continues to cover spinlocks. The CPU spin-delay hints (PAUSE, ISB, YIELD), which has no equivalent for, move to src/include/port/spin_delay.h, with SpinDelayStatus and its helpers in src/include/port/spin_delay_status.h. On ARM64, ISB is used instead of YIELD for better backoff under contention; on Windows ARM64 this adds a spin delay where there was none. Co-authored-by: Thomas Munro Co-authored-by: Greg Burd Reviewed-by: Shihao Zhong Discussion: https://postgr.es/m/CA+hUKGKFvu3zyvv3aaj5hHs9VtWcjFAmisOwOc7aOZNc5AF3NA@mail.gmail.com --- configure | 96 +++ configure.ac | 60 ++ meson.build | 104 +++ src/backend/storage/lmgr/.gitignore | 1 - src/backend/storage/lmgr/Makefile | 9 - src/backend/storage/lmgr/README | 54 ++ src/backend/storage/lmgr/s_lock.c | 123 +--- src/include/pg_config.h.in | 4 + src/include/port/atomics.h | 104 +-- src/include/port/atomics/arch-arm.h | 32 - src/include/port/atomics/arch-ppc.h | 256 -------- src/include/port/atomics/arch-x86.h | 189 ------ src/include/port/atomics/fallback.h | 42 -- src/include/port/atomics/generic-gcc.h | 326 ---------- src/include/port/atomics/generic-msvc.h | 115 ---- src/include/port/atomics/generic.h | 430 ------------- src/include/port/atomics/stdatomic_impl.h | 586 +++++++++++++++++ src/include/port/spin_delay.h | 148 +++++ src/include/port/spin_delay_status.h | 52 ++ src/include/storage/s_lock.h | 734 ---------------------- src/include/storage/spin.h | 60 +- src/test/regress/regress.c | 30 +- src/tools/pginclude/headerscheck | 10 - 23 files changed, 1200 insertions(+), 2365 deletions(-) delete mode 100644 src/include/port/atomics/arch-arm.h delete mode 100644 src/include/port/atomics/arch-ppc.h delete mode 100644 src/include/port/atomics/arch-x86.h delete mode 100644 src/include/port/atomics/fallback.h delete mode 100644 src/include/port/atomics/generic-gcc.h delete mode 100644 src/include/port/atomics/generic-msvc.h delete mode 100644 src/include/port/atomics/generic.h create mode 100644 src/include/port/atomics/stdatomic_impl.h create mode 100644 src/include/port/spin_delay.h create mode 100644 src/include/port/spin_delay_status.h delete mode 100644 src/include/storage/s_lock.h diff --git a/configure b/configure index 6b05d4c84ab..8cadb8e6bdd 100755 --- a/configure +++ b/configure @@ -4918,6 +4918,7 @@ fi + for ac_prog in gawk mawk nawk awk do # Extract the first word of "$ac_prog", so it can be a program name with args. @@ -17701,6 +17702,101 @@ $as_echo "#define HAVE_GCC__ATOMIC_INT64_CAS 1" >>confdefs.h fi +# Require a working C11 stdatomic.h: it is the sole atomics implementation, +# so a missing or broken header is a hard error. +# Use a link test, not a compile test: 64-bit atomic_compare_exchange_strong() +# can lower to a libatomic call (__atomic_compare_exchange_8) on some 32-bit +# or weakly-aligned targets. A compile-only check would pass there and then +# fail at final link. Retry with -latomic if the bare link fails. +# +# The probe also requires the 32- and 64-bit atomics to be *always* +# lock-free. PostgreSQL's atomics live in shared memory and are used from +# multiple processes, so a lock-table fallback (as libatomic uses when the +# hardware lacks the operation) would take process-local locks and silently +# fail to be atomic across backends. stdatomic_impl.h asserts the same +# thing; checking it here means such a target is rejected by configure with +# the message below rather than failing a static assertion later in the build. +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for working C11 stdatomic.h" >&5 +$as_echo_n "checking for working C11 stdatomic.h... " >&6; } +if ${pgac_cv_stdatomic+:} false; then : + $as_echo_n "(cached) " >&6 +else + pgac_cv_stdatomic=no + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include +#if !(defined(_MSC_VER) && !defined(__cplusplus)) +#if ATOMIC_INT_LOCK_FREE != 2 || ATOMIC_LLONG_LOCK_FREE != 2 +#error atomics are not always lock-free +#endif +#endif +int +main () +{ +_Atomic int x = 0; + atomic_fetch_add(&x, 1); + atomic_thread_fence(memory_order_seq_cst); + _Atomic(unsigned long long) y = 0; + unsigned long long expected = 0; + atomic_compare_exchange_strong(&y, &expected, 42); + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + pgac_cv_stdatomic=yes +else + pgac_save_LIBS=$LIBS + LIBS="$LIBS -latomic" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include +#if !(defined(_MSC_VER) && !defined(__cplusplus)) +#if ATOMIC_INT_LOCK_FREE != 2 || ATOMIC_LLONG_LOCK_FREE != 2 +#error atomics are not always lock-free +#endif +#endif +int +main () +{ +_Atomic int x = 0; + atomic_fetch_add(&x, 1); + atomic_thread_fence(memory_order_seq_cst); + _Atomic(unsigned long long) y = 0; + unsigned long long expected = 0; + atomic_compare_exchange_strong(&y, &expected, 42); + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + pgac_cv_stdatomic='yes, with -latomic' +else + pgac_cv_stdatomic=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext + LIBS=$pgac_save_LIBS +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $pgac_cv_stdatomic" >&5 +$as_echo "$pgac_cv_stdatomic" >&6; } +if test "$pgac_cv_stdatomic" = no; then + as_fn_error $? "a working C11 stdatomic.h with always-lock-free 32- and 64-bit atomics is required (it is the only atomics implementation)" "$LINENO" 5 +fi + + +$as_echo "#define USE_STDATOMIC_H 1" >>confdefs.h + +if test "$pgac_cv_stdatomic" = 'yes, with -latomic'; then + LIBS="$LIBS -latomic" +fi + + # Check for __get_cpuid() and __cpuid() { $as_echo "$as_me:${as_lineno-$LINENO}: checking for __get_cpuid" >&5 $as_echo_n "checking for __get_cpuid... " >&6; } diff --git a/configure.ac b/configure.ac index 6864268eaa0..ba5bfce2f75 100644 --- a/configure.ac +++ b/configure.ac @@ -443,6 +443,7 @@ choke me PGAC_ARG_BOOL(with, llvm, no, [build with LLVM based JIT support], [AC_DEFINE([USE_LLVM], 1, [Define to 1 to build with LLVM based JIT support. (--with-llvm)])]) AC_SUBST(with_llvm) + dnl must use AS_IF here, else AC_REQUIRES inside PGAC_LLVM_SUPPORT malfunctions AS_IF([test "$with_llvm" = yes], [ PGAC_LLVM_SUPPORT() @@ -2112,6 +2113,65 @@ PGAC_HAVE_GCC__ATOMIC_INT32_CAS PGAC_HAVE_GCC__ATOMIC_INT64_CAS +# Require a working C11 stdatomic.h: it is the sole atomics implementation, +# so a missing or broken header is a hard error. +# Use a link test, not a compile test: 64-bit atomic_compare_exchange_strong() +# can lower to a libatomic call (__atomic_compare_exchange_8) on some 32-bit +# or weakly-aligned targets. A compile-only check would pass there and then +# fail at final link. Retry with -latomic if the bare link fails. +# +# The probe also requires the 32- and 64-bit atomics to be *always* +# lock-free. PostgreSQL's atomics live in shared memory and are used from +# multiple processes, so a lock-table fallback (as libatomic uses when the +# hardware lacks the operation) would take process-local locks and silently +# fail to be atomic across backends. stdatomic_impl.h asserts the same +# thing; checking it here means such a target is rejected by configure with +# the message below rather than failing a static assertion later in the build. +AC_CACHE_CHECK([for working C11 stdatomic.h], [pgac_cv_stdatomic], +[pgac_cv_stdatomic=no + AC_LINK_IFELSE([AC_LANG_PROGRAM([#include +#if !(defined(_MSC_VER) && !defined(__cplusplus)) +#if ATOMIC_INT_LOCK_FREE != 2 || ATOMIC_LLONG_LOCK_FREE != 2 +#error atomics are not always lock-free +#endif +#endif], + [[_Atomic int x = 0; + atomic_fetch_add(&x, 1); + atomic_thread_fence(memory_order_seq_cst); + _Atomic(unsigned long long) y = 0; + unsigned long long expected = 0; + atomic_compare_exchange_strong(&y, &expected, 42); + ]])], + [pgac_cv_stdatomic=yes], + [pgac_save_LIBS=$LIBS + LIBS="$LIBS -latomic" + AC_LINK_IFELSE([AC_LANG_PROGRAM([#include +#if !(defined(_MSC_VER) && !defined(__cplusplus)) +#if ATOMIC_INT_LOCK_FREE != 2 || ATOMIC_LLONG_LOCK_FREE != 2 +#error atomics are not always lock-free +#endif +#endif], + [[_Atomic int x = 0; + atomic_fetch_add(&x, 1); + atomic_thread_fence(memory_order_seq_cst); + _Atomic(unsigned long long) y = 0; + unsigned long long expected = 0; + atomic_compare_exchange_strong(&y, &expected, 42); + ]])], + [pgac_cv_stdatomic='yes, with -latomic'], + [pgac_cv_stdatomic=no]) + LIBS=$pgac_save_LIBS])]) +if test "$pgac_cv_stdatomic" = no; then + AC_MSG_ERROR([a working C11 stdatomic.h with always-lock-free 32- and 64-bit atomics is required (it is the only atomics implementation)]) +fi + +AC_DEFINE([USE_STDATOMIC_H], 1, + [Define to 1 to use C11 stdatomic.h for atomic operations.]) +if test "$pgac_cv_stdatomic" = 'yes, with -latomic'; then + LIBS="$LIBS -latomic" +fi + + # Check for __get_cpuid() and __cpuid() AC_CACHE_CHECK([for __get_cpuid], [pgac_cv__get_cpuid], [AC_LINK_IFELSE([AC_LANG_PROGRAM([#include ], diff --git a/meson.build b/meson.build index 7e95fd5941d..846457a874c 100644 --- a/meson.build +++ b/meson.build @@ -674,6 +674,110 @@ if have_cxx and not cxx.compiles(cxx11_test, name: 'C++11') endif +############################################################### +# Check for C11 stdatomic.h support +############################################################### + +stdatomic_test_code = ''' +#include +/* + * Require the 32- and 64-bit atomics to be *always* lock-free. PostgreSQL's + * atomics live in shared memory and are used from multiple processes, so an + * implementation that falls back to a lock table (as libatomic does when the + * hardware lacks the operation) would take process-local locks and silently + * fail to be atomic across backends. stdatomic_impl.h asserts the same + * thing; checking it in the probe means such a target simply doesn't select + * the stdatomic path rather than failing a static assertion later in the + * build. + * + * MSVC compiling as C reports 1 for every width even though the atomics are + * in fact always lock-free on the targets we support, so skip the check + * there (stdatomic_impl.h skips its assertion in the same configuration). + */ +#if !(defined(_MSC_VER) && !defined(__cplusplus)) +#if ATOMIC_INT_LOCK_FREE != 2 || ATOMIC_LLONG_LOCK_FREE != 2 +#error atomics are not always lock-free +#endif +#endif +int main(void) { + _Atomic int x = 0; + atomic_fetch_add(&x, 1); + atomic_thread_fence(memory_order_seq_cst); + _Atomic(unsigned long long) y = 0; + unsigned long long expected = 0; + atomic_compare_exchange_strong(&y, &expected, 42); + return 0; +} +''' + +has_stdatomic = false +atomic_link_args = [] + +# The stdatomic probe below is a *link* test, not just a compile test: 64-bit +# atomic_compare_exchange_strong() can lower to a libatomic call +# (__atomic_compare_exchange_8) on some 32-bit or weakly-aligned targets. A +# compile-only check would pass there and then fail at final link, so we +# require the probe to link, retrying with -latomic if the bare link fails. + +if cc.get_id() == 'msvc' + msvc_ver = cc.version() + if msvc_ver.version_compare('>=19.30') # Visual Studio 2022+ + message('MSVC version @0@ detected (>= 19.30), checking for stdatomic.h support'.format(msvc_ver)) + # MSVC's requires C11 mode (/std:c11) in addition to + # /experimental:c11atomics; without /std:c11 the header errors out with + # "C atomics require C11 or later". The probe must pass both, since + # test_c_args is not yet populated with the C11 flag at this point. + if cc.links(stdatomic_test_code, + name: 'MSVC stdatomic.h with /experimental:c11atomics', + args: test_c_args + ['/std:c11', '/experimental:c11atomics']) + has_stdatomic = true + message('MSVC stdatomic.h is available with /experimental:c11atomics') + else + message('MSVC stdatomic.h test compilation failed') + endif + else + message('MSVC version @0@ is too old for stdatomic.h (requires >= 19.30)'.format(msvc_ver)) + endif +else + # GCC/Clang and other compilers + if cc.has_header('stdatomic.h', args: test_c_args) + message('stdatomic.h header found, testing that it links...') + if cc.links(stdatomic_test_code, + name: 'stdatomic.h works', + args: test_c_args) + has_stdatomic = true + message('stdatomic.h is available and working') + elif cc.links(stdatomic_test_code, + name: 'stdatomic.h works with -latomic', + args: test_c_args + ['-latomic']) + # Some targets need libatomic for the wider (e.g. 64-bit) operations. + has_stdatomic = true + atomic_link_args = ['-latomic'] + message('stdatomic.h is available and working (requires -latomic)') + else + message('stdatomic.h found but test link failed') + endif + else + message('stdatomic.h header not found') + endif +endif + +# stdatomic.h is the sole atomics implementation, so a missing or broken +# header (or one whose atomics are not always lock-free) is a hard error. +if not has_stdatomic + error('a working C11 stdatomic.h with always-lock-free 32- and 64-bit atomics is required (it is the only atomics implementation)') +endif +cdata.set('USE_STDATOMIC_H', 1) +message('Using C11 stdatomic.h for atomic operations') +if atomic_link_args.length() > 0 + # libatomic is needed for atomics the compiler cannot inline on this target. + os_deps += cc.find_library('atomic') +endif +if cc.get_id() == 'msvc' + # MSVC needs C11 mode plus the experimental atomics switch for . + cflags += ['/std:c11', '/experimental:c11atomics'] +endif + postgres_inc = [include_directories(postgres_inc_d)] test_lib_d = postgres_lib_d test_c_args = cppflags + cflags diff --git a/src/backend/storage/lmgr/.gitignore b/src/backend/storage/lmgr/.gitignore index 8e5b734f152..209c8be7223 100644 --- a/src/backend/storage/lmgr/.gitignore +++ b/src/backend/storage/lmgr/.gitignore @@ -1,2 +1 @@ /lwlocknames.h -/s_lock_test diff --git a/src/backend/storage/lmgr/Makefile b/src/backend/storage/lmgr/Makefile index a5fbc24ddad..a60e70ef0e1 100644 --- a/src/backend/storage/lmgr/Makefile +++ b/src/backend/storage/lmgr/Makefile @@ -24,17 +24,8 @@ OBJS = \ include $(top_srcdir)/src/backend/common.mk -s_lock_test: s_lock.c $(top_builddir)/src/common/libpgcommon.a $(top_builddir)/src/port/libpgport.a - $(CC) $(CPPFLAGS) $(CFLAGS) -DS_LOCK_TEST=1 $(srcdir)/s_lock.c \ - -L $(top_builddir)/src/common -lpgcommon \ - -L $(top_builddir)/src/port -lpgport -lm -o s_lock_test - lwlocknames.h: ../../../include/storage/lwlocklist.h ../../utils/activity/wait_event_names.txt generate-lwlocknames.pl $(PERL) $(srcdir)/generate-lwlocknames.pl $^ -check: s_lock_test - ./s_lock_test - clean: - rm -f s_lock_test rm -f lwlocknames.h diff --git a/src/backend/storage/lmgr/README b/src/backend/storage/lmgr/README index 45de0fd2bd6..f09795b0a43 100644 --- a/src/backend/storage/lmgr/README +++ b/src/backend/storage/lmgr/README @@ -47,6 +47,60 @@ when the wait time might exceed a few seconds. The rest of this README file discusses the regular lock manager in detail. +Atomic Operations +----------------- + +PostgreSQL's atomic operations (see src/include/port/atomics.h) are +implemented on top of the standard C11 header, which is the +sole atomics implementation. A working stdatomic.h is required at build +time; the USE_STDATOMIC_H preprocessor symbol is always defined. Available +on: + - MSVC 2022+ (requires /experimental:c11atomics flag) + - GCC 4.9+ and Clang 3.1+ (no special flags needed) + +There is no build option to select the atomics implementation: stdatomic.h +is probed unconditionally and a missing or non-working header is a hard +configure/build error, since there is no other implementation to fall back +to. + +Several internal properties of the implementation are worth noting; none of +them affect the public API: + +* Memory ordering. pg_atomic_read_u32/u64() use seq_cst ordering; + pg_atomic_write_u32/u64() use relaxed. The seq_cst read is + required for correctness on weak-memory hardware (e.g. RISC-V), where the + weaker relaxed and acquire orderings let a concurrent reader miss a tuple + in a parallel hash join. The write stays relaxed: a plain atomic store + matches the "no barrier" contract for pg_atomic_write and, on + ARM64, avoids the store-release (STLR) serialization that would penalize + contended writes to hot shared state. The pg_atomic_unlocked_write_* + variant also uses relaxed ordering, consistent with its "no guarantees" + contract. On x86 (TSO) a seq_cst load is still a plain mov, so there is no + added cost; + on ARM/RISC-V it adds fence instructions on these paths. + +* Barriers. Each barrier pairs a compiler barrier (atomic_signal_fence) with + the thread fence (atomic_thread_fence), because a bare thread fence orders + only atomic accesses and would let the compiler reorder plain loads/stores + across it, whereas PostgreSQL's barrier contract must order non-atomic + accesses too. + +* pg_atomic_flag polarity. pg_atomic_flag uses 1=unlocked, 0=locked + (fetch_and-based test-and-set). Note this is only about pg_atomic_flag, + not about spinlocks: slock_t keeps the traditional 0=unlocked, 1=locked + convention, so a zero-initialized spinlock is a free lock. The flag + polarity is an internal detail with no external visibility beyond raw + values seen in a debugger. + +* Platform reach. Genuine always-lock-free 64-bit atomics are required: a + compile-time static assertion rejects lock-emulated atomics, which use + process-local locks and so cannot be atomic across the multiple processes + that share PostgreSQL's shared memory. The headers are C++-compatible. + +For more information on memory barriers and atomic operations, see +src/backend/storage/lmgr/README.barrier. + + Lock Data Structures -------------------- diff --git a/src/backend/storage/lmgr/s_lock.c b/src/backend/storage/lmgr/s_lock.c index 795aeed1028..8caaa562ced 100644 --- a/src/backend/storage/lmgr/s_lock.c +++ b/src/backend/storage/lmgr/s_lock.c @@ -51,7 +51,9 @@ #include #include "common/pg_prng.h" -#include "storage/s_lock.h" +#include "port/atomics.h" +#include "port/spin_delay.h" +#include "storage/spin.h" #include "utils/wait_event.h" #define MIN_SPINS_PER_DELAY 10 @@ -60,15 +62,6 @@ #define MIN_DELAY_USEC 1000L #define MAX_DELAY_USEC 1000000L -#ifdef S_LOCK_TEST -/* - * These are needed by pgstat_report_wait_start in the standalone compile of - * s_lock_test. - */ -static uint32 local_my_wait_event_info; -uint32 *my_wait_event_info = &local_my_wait_event_info; -#endif - static int spins_per_delay = DEFAULT_SPINS_PER_DELAY; @@ -80,18 +73,10 @@ s_lock_stuck(const char *file, int line, const char *func) { if (!func) func = "(unknown)"; -#if defined(S_LOCK_TEST) - fprintf(stderr, - "\nStuck spinlock detected at %s, %s:%d.\n", - func, file, line); - exit(1); -#else elog(PANIC, "stuck spinlock detected at %s, %s:%d", func, file, line); -#endif } -#ifdef USE_DEFAULT_S_LOCK /* * s_lock(lock) - platform-independent portion of waiting for a spinlock. */ @@ -102,8 +87,23 @@ s_lock(volatile slock_t *lock, const char *file, int line, const char *func) init_spin_delay(&delayStatus, file, line, func); - while (TAS_SPIN(lock)) + for (;;) { + bool try_to_set; + +#ifdef PG_SPIN_TRY_RELAXED + /* + * Test-and-test-and-set: read the lock first and only attempt the + * exchange when it looks free, to avoid taking cache-line ownership on + * every spin. PG_SPIN_TRY_RELAXED is defined only where this pre-read + * is cheap (x86/x86_64 TSO, where a seq_cst load is a plain load). + */ + try_to_set = (pg_atomic_read_u32(lock) == 0); +#else + try_to_set = true; +#endif + if (try_to_set && pg_atomic_exchange_u32(lock, 1) == 0) + break; perform_spin_delay(&delayStatus); } @@ -111,15 +111,6 @@ s_lock(volatile slock_t *lock, const char *file, int line, const char *func) return delayStatus.delays; } -#endif - -#ifdef USE_DEFAULT_S_UNLOCK -void -s_unlock(volatile slock_t *lock) -{ - *lock = 0; -} -#endif /* * Wait while spinning on a contended spinlock. @@ -128,7 +119,7 @@ void perform_spin_delay(SpinDelayStatus *status) { /* CPU-specific delay each time through the loop */ - SPIN_DELAY(); + pg_spin_delay(); /* Block the process every spins_per_delay tries */ if (++(status->spins) >= spins_per_delay) @@ -151,11 +142,6 @@ perform_spin_delay(SpinDelayStatus *status) pg_usleep(status->cur_delay); pgstat_report_wait_end(); -#if defined(S_LOCK_TEST) - fprintf(stdout, "*"); - fflush(stdout); -#endif - /* increase delay by a random fraction between 1X and 2X */ status->cur_delay += (int) (status->cur_delay * pg_prng_double(&pg_global_prng_state) + 0.5); @@ -231,72 +217,3 @@ update_spins_per_delay(int shared_spins_per_delay) */ return (shared_spins_per_delay * 15 + spins_per_delay) / 16; } - - -/*****************************************************************************/ -#if defined(S_LOCK_TEST) - -/* - * test program for verifying a port's spinlock support. - */ - -struct test_lock_struct -{ - char pad1; - slock_t lock; - char pad2; -}; - -volatile struct test_lock_struct test_lock; - -int -main() -{ - pg_prng_seed(&pg_global_prng_state, (uint64) time(NULL)); - - test_lock.pad1 = test_lock.pad2 = 0x44; - - S_INIT_LOCK(&test_lock.lock); - - if (test_lock.pad1 != 0x44 || test_lock.pad2 != 0x44) - { - printf("S_LOCK_TEST: failed, declared datatype is wrong size\n"); - return 1; - } - - S_LOCK(&test_lock.lock); - - if (test_lock.pad1 != 0x44 || test_lock.pad2 != 0x44) - { - printf("S_LOCK_TEST: failed, declared datatype is wrong size\n"); - return 1; - } - - S_UNLOCK(&test_lock.lock); - - if (test_lock.pad1 != 0x44 || test_lock.pad2 != 0x44) - { - printf("S_LOCK_TEST: failed, declared datatype is wrong size\n"); - return 1; - } - - S_LOCK(&test_lock.lock); - - if (test_lock.pad1 != 0x44 || test_lock.pad2 != 0x44) - { - printf("S_LOCK_TEST: failed, declared datatype is wrong size\n"); - return 1; - } - - printf("S_LOCK_TEST: this will print %d stars and then\n", NUM_DELAYS); - printf(" exit with a 'stuck spinlock' message\n"); - printf(" if S_LOCK() and TAS() are working.\n"); - fflush(stdout); - - S_LOCK(&test_lock.lock); - - printf("S_LOCK_TEST: failed, lock not locked\n"); - return 1; -} - -#endif /* S_LOCK_TEST */ diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 661c4a9b168..31aff5d59f9 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -744,6 +744,10 @@ /* Define to 1 to use Intel SSE 4.2 CRC instructions with a runtime check. */ #undef USE_SSE42_CRC32C_WITH_RUNTIME_CHECK +/* Define to 1 to use C11 stdatomic.h for atomic operations. + (--with-stdatomic) */ +#undef USE_STDATOMIC_H + /* Define to 1 to use SVE popcount instructions with a runtime check. */ #undef USE_SVE_POPCNT_WITH_RUNTIME_CHECK diff --git a/src/include/port/atomics.h b/src/include/port/atomics.h index a605ea81d07..0377a65a4af 100644 --- a/src/include/port/atomics.h +++ b/src/include/port/atomics.h @@ -7,20 +7,17 @@ * atomically and dealing with cache coherency. Used to implement locking * facilities and lockless algorithms/data structures. * - * To bring up postgres on a platform/compiler at the very least - * implementations for the following operations should be provided: - * * pg_compiler_barrier(), pg_write_barrier(), pg_read_barrier() - * * pg_atomic_compare_exchange_u32(), pg_atomic_fetch_add_u32() - * * pg_atomic_test_set_flag(), pg_atomic_init_flag(), pg_atomic_clear_flag() - * * PG_HAVE_8BYTE_SINGLE_COPY_ATOMICITY should be defined if appropriate. + * IMPLEMENTATION: * - * There exist generic, hardware independent, implementations for several - * compilers which might be sufficient, although possibly not optimal, for a - * new platform. If no such generic implementation is available spinlocks will - * be used to implement the 64-bit parts of the API. + * PostgreSQL's atomic operations are implemented on top of C11 + * (see stdatomic_impl.h). This standard header supplies all + * atomic types and operations, so no architecture-specific assembly or + * compiler intrinsics are required here. A working C11 stdatomic.h is a + * hard build requirement (USE_STDATOMIC_H is always defined). * - * Implement _u64 atomics if and only if your platform can use them - * efficiently (and obviously correctly). + * The CPU spin-delay instruction, which stdatomic.h does not cover, is in + * port/spin_delay.h. The memory-ordering and flag-width choices are + * documented in port/atomics/stdatomic_impl.h. * * Use higher level functionality (lwlocks, spinlocks, heavyweight locks) * whenever possible. Writing correct code using these facilities is hard. @@ -38,84 +35,27 @@ #ifndef ATOMICS_H #define ATOMICS_H -#ifdef FRONTEND -#error "atomics.h may not be included from frontend code" -#endif - -#define INSIDE_ATOMICS_H - -#include - /* - * First a set of architecture specific files is included. - * - * These files can provide the full set of atomics or can do pretty much - * nothing if all the compilers commonly used on these platforms provide - * usable generics. - * - * Don't add an inline assembly of the actual atomic operations if all the - * common implementations of your platform provide intrinsics. Intrinsics are - * much easier to understand and potentially support more architectures. - * - * It will often make sense to define memory barrier semantics here, since - * e.g. generic compiler intrinsics for x86 memory barriers can't know that - * postgres doesn't need x86 read/write barriers do anything more than a - * compiler barrier. - * + * The public API and the _impl signatures keep the historical volatile + * qualifier on the atomic pointer, because many callers reach atomic fields + * through volatile-qualified struct pointers. Passing a volatile-qualified + * _Atomic pointer to the C11 generic operations is well defined: the access + * remains fully atomic, and the qualifier is retained purely for source + * compatibility. */ -#if defined(__arm__) || defined(__aarch64__) -#include "port/atomics/arch-arm.h" -#elif defined(__i386__) || defined(__x86_64__) -#include "port/atomics/arch-x86.h" -#elif defined(__powerpc__) || defined(__powerpc64__) -#include "port/atomics/arch-ppc.h" -#endif - -/* - * Compiler specific, but architecture independent implementations. - * - * Provide architecture independent implementations of the atomic - * facilities. At the very least compiler barriers should be provided, but a - * full implementation of - * * pg_compiler_barrier(), pg_write_barrier(), pg_read_barrier() - * * pg_atomic_compare_exchange_u32(), pg_atomic_fetch_add_u32() - * using compiler intrinsics are a good idea. - */ -/* - * gcc or compatible, including clang and icc. - */ -#if defined(__GNUC__) || defined(__INTEL_COMPILER) -#include "port/atomics/generic-gcc.h" -#elif defined(_MSC_VER) -#include "port/atomics/generic-msvc.h" -#else -/* Unknown compiler. */ -#endif - -/* Fail if we couldn't find implementations of required facilities. */ -#if !defined(PG_HAVE_ATOMIC_U32_SUPPORT) -#error "could not find an implementation of pg_atomic_uint32" -#endif -#if !defined(pg_compiler_barrier_impl) -#error "could not find an implementation of pg_compiler_barrier" -#endif -#if !defined(pg_memory_barrier_impl) -#error "could not find an implementation of pg_memory_barrier_impl" -#endif +#include /* - * Provide a spinlock-based implementation of the 64 bit variants, if - * necessary. + * All atomic types and _impl operations come from C11 , wrapped + * in stdatomic_impl.h. */ -#include "port/atomics/fallback.h" +#include "port/atomics/stdatomic_impl.h" /* - * Provide additional operations using supported infrastructure. These are - * expected to be efficient if the underlying atomic operations are efficient. + * The public API below is provided by static inline functions that call the + * _impl functions supplied by stdatomic_impl.h. */ -#include "port/atomics/generic.h" - /* * pg_compiler_barrier - prevent the compiler from moving code across @@ -611,6 +551,4 @@ pg_atomic_monotonic_advance_u64(volatile pg_atomic_uint64 *ptr, uint64 target) return currval; } -#undef INSIDE_ATOMICS_H - #endif /* ATOMICS_H */ diff --git a/src/include/port/atomics/arch-arm.h b/src/include/port/atomics/arch-arm.h deleted file mode 100644 index 90280c7b751..00000000000 --- a/src/include/port/atomics/arch-arm.h +++ /dev/null @@ -1,32 +0,0 @@ -/*------------------------------------------------------------------------- - * - * arch-arm.h - * Atomic operations considerations specific to ARM - * - * Portions Copyright (c) 2013-2026, PostgreSQL Global Development Group - * - * NOTES: - * - * src/include/port/atomics/arch-arm.h - * - *------------------------------------------------------------------------- - */ - -/* intentionally no include guards, should only be included by atomics.h */ -#ifndef INSIDE_ATOMICS_H -#error "should be included via atomics.h" -#endif - -/* - * 64 bit atomics on ARM32 are implemented using kernel fallbacks and thus - * might be slow, so disable entirely. On ARM64 that problem doesn't exist. - */ -#if !defined(__aarch64__) -#define PG_DISABLE_64_BIT_ATOMICS -#else -/* - * Architecture Reference Manual for ARMv8 states aligned read/write to/from - * general purpose register is atomic. - */ -#define PG_HAVE_8BYTE_SINGLE_COPY_ATOMICITY -#endif /* __aarch64__ */ diff --git a/src/include/port/atomics/arch-ppc.h b/src/include/port/atomics/arch-ppc.h deleted file mode 100644 index 65ced2c6d3a..00000000000 --- a/src/include/port/atomics/arch-ppc.h +++ /dev/null @@ -1,256 +0,0 @@ -/*------------------------------------------------------------------------- - * - * arch-ppc.h - * Atomic operations considerations specific to PowerPC - * - * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group - * Portions Copyright (c) 1994, Regents of the University of California - * - * NOTES: - * - * src/include/port/atomics/arch-ppc.h - * - *------------------------------------------------------------------------- - */ - -#if defined(__GNUC__) - -/* - * lwsync orders loads with respect to each other, and similarly with stores. - * But a load can be performed before a subsequent store, so sync must be used - * for a full memory barrier. - */ -#define pg_memory_barrier_impl() __asm__ __volatile__ ("sync" : : : "memory") -#define pg_read_barrier_impl() __asm__ __volatile__ ("lwsync" : : : "memory") -#define pg_write_barrier_impl() __asm__ __volatile__ ("lwsync" : : : "memory") -#endif - -#define PG_HAVE_ATOMIC_U32_SUPPORT -typedef struct pg_atomic_uint32 -{ - volatile uint32 value; -} pg_atomic_uint32; - -/* 64bit atomics are only supported in 64bit mode */ -#if SIZEOF_VOID_P >= 8 -#define PG_HAVE_ATOMIC_U64_SUPPORT -typedef struct pg_atomic_uint64 -{ - alignas(8) volatile uint64 value; -} pg_atomic_uint64; - -#endif - -/* - * This mimics gcc __atomic_compare_exchange_n(..., __ATOMIC_SEQ_CST), but - * code generation differs at the end. __atomic_compare_exchange_n(): - * 100: isync - * 104: mfcr r3 - * 108: rlwinm r3,r3,3,31,31 - * 10c: bne 120 <.eb+0x10> - * 110: clrldi r3,r3,63 - * 114: addi r1,r1,112 - * 118: blr - * 11c: nop - * 120: clrldi r3,r3,63 - * 124: stw r9,0(r4) - * 128: addi r1,r1,112 - * 12c: blr - * - * This: - * f0: isync - * f4: mfcr r9 - * f8: rldicl. r3,r9,35,63 - * fc: bne 104 <.eb> - * 100: stw r10,0(r4) - * 104: addi r1,r1,112 - * 108: blr - * - * This implementation may or may not have materially different performance. - * It's not exploiting the fact that cr0 still holds the relevant comparison - * bits, set during the __asm__. One could fix that by moving more code into - * the __asm__. (That would remove the freedom to eliminate dead stores when - * the caller ignores "expected", but few callers do.) - * - * Recognizing constant "newval" would be superfluous, because there's no - * immediate-operand version of stwcx. - */ -#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32 -static inline bool -pg_atomic_compare_exchange_u32_impl(volatile pg_atomic_uint32 *ptr, - uint32 *expected, uint32 newval) -{ - uint32 found; - uint32 condition_register; - bool ret; - -#ifdef HAVE_I_CONSTRAINT__BUILTIN_CONSTANT_P - if (__builtin_constant_p(*expected) && - (int32) *expected <= PG_INT16_MAX && - (int32) *expected >= PG_INT16_MIN) - __asm__ __volatile__( - " sync \n" - " lwarx %0,0,%5,1 \n" - " cmpwi %0,%3 \n" - " bne $+12 \n" /* branch to lwsync */ - " stwcx. %4,0,%5 \n" - " bne $-16 \n" /* branch to lwarx */ - " lwsync \n" - " mfcr %1 \n" -: "=&r"(found), "=r"(condition_register), "+m"(ptr->value) -: "i"(*expected), "r"(newval), "r"(&ptr->value) -: "memory", "cc"); - else -#endif - __asm__ __volatile__( - " sync \n" - " lwarx %0,0,%5,1 \n" - " cmpw %0,%3 \n" - " bne $+12 \n" /* branch to lwsync */ - " stwcx. %4,0,%5 \n" - " bne $-16 \n" /* branch to lwarx */ - " lwsync \n" - " mfcr %1 \n" -: "=&r"(found), "=r"(condition_register), "+m"(ptr->value) -: "r"(*expected), "r"(newval), "r"(&ptr->value) -: "memory", "cc"); - - ret = (condition_register >> 29) & 1; /* test eq bit of cr0 */ - if (!ret) - *expected = found; - return ret; -} - -/* - * This mirrors gcc __sync_fetch_and_add(). - * - * Like tas(), use constraint "=&b" to avoid allocating r0. - */ -#define PG_HAVE_ATOMIC_FETCH_ADD_U32 -static inline uint32 -pg_atomic_fetch_add_u32_impl(volatile pg_atomic_uint32 *ptr, int32 add_) -{ - uint32 _t; - uint32 res; - -#ifdef HAVE_I_CONSTRAINT__BUILTIN_CONSTANT_P - if (__builtin_constant_p(add_) && - add_ <= PG_INT16_MAX && add_ >= PG_INT16_MIN) - __asm__ __volatile__( - " sync \n" - " lwarx %1,0,%4,1 \n" - " addi %0,%1,%3 \n" - " stwcx. %0,0,%4 \n" - " bne $-12 \n" /* branch to lwarx */ - " lwsync \n" -: "=&r"(_t), "=&b"(res), "+m"(ptr->value) -: "i"(add_), "r"(&ptr->value) -: "memory", "cc"); - else -#endif - __asm__ __volatile__( - " sync \n" - " lwarx %1,0,%4,1 \n" - " add %0,%1,%3 \n" - " stwcx. %0,0,%4 \n" - " bne $-12 \n" /* branch to lwarx */ - " lwsync \n" -: "=&r"(_t), "=&r"(res), "+m"(ptr->value) -: "r"(add_), "r"(&ptr->value) -: "memory", "cc"); - - return res; -} - -#ifdef PG_HAVE_ATOMIC_U64_SUPPORT - -#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64 -static inline bool -pg_atomic_compare_exchange_u64_impl(volatile pg_atomic_uint64 *ptr, - uint64 *expected, uint64 newval) -{ - uint64 found; - uint32 condition_register; - bool ret; - - AssertPointerAlignment(expected, 8); - - /* Like u32, but s/lwarx/ldarx/; s/stwcx/stdcx/; s/cmpw/cmpd/ */ -#ifdef HAVE_I_CONSTRAINT__BUILTIN_CONSTANT_P - if (__builtin_constant_p(*expected) && - (int64) *expected <= PG_INT16_MAX && - (int64) *expected >= PG_INT16_MIN) - __asm__ __volatile__( - " sync \n" - " ldarx %0,0,%5,1 \n" - " cmpdi %0,%3 \n" - " bne $+12 \n" /* branch to lwsync */ - " stdcx. %4,0,%5 \n" - " bne $-16 \n" /* branch to ldarx */ - " lwsync \n" - " mfcr %1 \n" -: "=&r"(found), "=r"(condition_register), "+m"(ptr->value) -: "i"(*expected), "r"(newval), "r"(&ptr->value) -: "memory", "cc"); - else -#endif - __asm__ __volatile__( - " sync \n" - " ldarx %0,0,%5,1 \n" - " cmpd %0,%3 \n" - " bne $+12 \n" /* branch to lwsync */ - " stdcx. %4,0,%5 \n" - " bne $-16 \n" /* branch to ldarx */ - " lwsync \n" - " mfcr %1 \n" -: "=&r"(found), "=r"(condition_register), "+m"(ptr->value) -: "r"(*expected), "r"(newval), "r"(&ptr->value) -: "memory", "cc"); - - ret = (condition_register >> 29) & 1; /* test eq bit of cr0 */ - if (!ret) - *expected = found; - return ret; -} - -#define PG_HAVE_ATOMIC_FETCH_ADD_U64 -static inline uint64 -pg_atomic_fetch_add_u64_impl(volatile pg_atomic_uint64 *ptr, int64 add_) -{ - uint64 _t; - uint64 res; - - /* Like u32, but s/lwarx/ldarx/; s/stwcx/stdcx/ */ -#ifdef HAVE_I_CONSTRAINT__BUILTIN_CONSTANT_P - if (__builtin_constant_p(add_) && - add_ <= PG_INT16_MAX && add_ >= PG_INT16_MIN) - __asm__ __volatile__( - " sync \n" - " ldarx %1,0,%4,1 \n" - " addi %0,%1,%3 \n" - " stdcx. %0,0,%4 \n" - " bne $-12 \n" /* branch to ldarx */ - " lwsync \n" -: "=&r"(_t), "=&b"(res), "+m"(ptr->value) -: "i"(add_), "r"(&ptr->value) -: "memory", "cc"); - else -#endif - __asm__ __volatile__( - " sync \n" - " ldarx %1,0,%4,1 \n" - " add %0,%1,%3 \n" - " stdcx. %0,0,%4 \n" - " bne $-12 \n" /* branch to ldarx */ - " lwsync \n" -: "=&r"(_t), "=&r"(res), "+m"(ptr->value) -: "r"(add_), "r"(&ptr->value) -: "memory", "cc"); - - return res; -} - -#endif /* PG_HAVE_ATOMIC_U64_SUPPORT */ - -/* per architecture manual doubleword accesses have single copy atomicity */ -#define PG_HAVE_8BYTE_SINGLE_COPY_ATOMICITY diff --git a/src/include/port/atomics/arch-x86.h b/src/include/port/atomics/arch-x86.h deleted file mode 100644 index 231831dcf59..00000000000 --- a/src/include/port/atomics/arch-x86.h +++ /dev/null @@ -1,189 +0,0 @@ -/*------------------------------------------------------------------------- - * - * arch-x86.h - * Atomic operations considerations specific to intel x86 - * - * Note that we actually require a 486 upwards because the 386 doesn't have - * support for xadd and cmpxchg. Given that the 386 isn't supported anywhere - * anymore that's not much of a restriction luckily. - * - * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group - * Portions Copyright (c) 1994, Regents of the University of California - * - * NOTES: - * - * src/include/port/atomics/arch-x86.h - * - *------------------------------------------------------------------------- - */ - -/* - * Both 32 and 64 bit x86 do not allow loads to be reordered with other loads, - * or stores to be reordered with other stores, but a load can be performed - * before a subsequent store. - * - * Technically, some x86-ish chips support uncached memory access and/or - * special instructions that are weakly ordered. In those cases we'd need - * the read and write barriers to be lfence and sfence. But since we don't - * do those things, a compiler barrier should be enough. - * - * "lock; addl" has worked for longer than "mfence". It's also rumored to be - * faster in many scenarios. - */ - -#if defined(__GNUC__) || defined(__INTEL_COMPILER) -#if defined(__i386__) -#define pg_memory_barrier_impl() \ - __asm__ __volatile__ ("lock; addl $0,0(%%esp)" : : : "memory", "cc") -#elif defined(__x86_64__) -#define pg_memory_barrier_impl() \ - __asm__ __volatile__ ("lock; addl $0,0(%%rsp)" : : : "memory", "cc") -#endif -#endif /* defined(__GNUC__) || defined(__INTEL_COMPILER) */ - -#define pg_read_barrier_impl() pg_compiler_barrier_impl() -#define pg_write_barrier_impl() pg_compiler_barrier_impl() - -/* - * Provide implementation for atomics using inline assembly on x86 gcc. It's - * nice to support older gcc's and the compare/exchange implementation here is - * actually more efficient than the * __sync variant. - */ -#if defined(__GNUC__) || defined(__INTEL_COMPILER) - -#define PG_HAVE_ATOMIC_FLAG_SUPPORT -typedef struct pg_atomic_flag -{ - volatile char value; -} pg_atomic_flag; - -#define PG_HAVE_ATOMIC_U32_SUPPORT -typedef struct pg_atomic_uint32 -{ - volatile uint32 value; -} pg_atomic_uint32; - -/* - * It's too complicated to write inline asm for 64bit types on 32bit and the - * 486 can't do it anyway. - */ -#ifdef __x86_64__ -#define PG_HAVE_ATOMIC_U64_SUPPORT -typedef struct pg_atomic_uint64 -{ - /* alignment guaranteed due to being on a 64bit platform */ - volatile uint64 value; -} pg_atomic_uint64; -#endif /* __x86_64__ */ - -#define PG_HAVE_ATOMIC_TEST_SET_FLAG -static inline bool -pg_atomic_test_set_flag_impl(volatile pg_atomic_flag *ptr) -{ - char _res = 1; - - __asm__ __volatile__( - " lock \n" - " xchgb %0,%1 \n" -: "+q"(_res), "+m"(ptr->value) -: -: "memory"); - return _res == 0; -} - -#define PG_HAVE_ATOMIC_CLEAR_FLAG -static inline void -pg_atomic_clear_flag_impl(volatile pg_atomic_flag *ptr) -{ - /* - * On a TSO architecture like x86 it's sufficient to use a compiler - * barrier to achieve release semantics. - */ - __asm__ __volatile__("" ::: "memory"); - ptr->value = 0; -} - -#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32 -static inline bool -pg_atomic_compare_exchange_u32_impl(volatile pg_atomic_uint32 *ptr, - uint32 *expected, uint32 newval) -{ - char ret; - - /* - * Perform cmpxchg and use the zero flag which it implicitly sets when - * equal to measure the success. - */ - __asm__ __volatile__( - " lock \n" - " cmpxchgl %4,%5 \n" - " setz %2 \n" -: "=a" (*expected), "=m"(ptr->value), "=q" (ret) -: "a" (*expected), "r" (newval), "m"(ptr->value) -: "memory", "cc"); - return (bool) ret; -} - -#define PG_HAVE_ATOMIC_FETCH_ADD_U32 -static inline uint32 -pg_atomic_fetch_add_u32_impl(volatile pg_atomic_uint32 *ptr, int32 add_) -{ - uint32 res; - __asm__ __volatile__( - " lock \n" - " xaddl %0,%1 \n" -: "=q"(res), "=m"(ptr->value) -: "0" (add_), "m"(ptr->value) -: "memory", "cc"); - return res; -} - -#ifdef __x86_64__ - -#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64 -static inline bool -pg_atomic_compare_exchange_u64_impl(volatile pg_atomic_uint64 *ptr, - uint64 *expected, uint64 newval) -{ - char ret; - - AssertPointerAlignment(expected, 8); - - /* - * Perform cmpxchg and use the zero flag which it implicitly sets when - * equal to measure the success. - */ - __asm__ __volatile__( - " lock \n" - " cmpxchgq %4,%5 \n" - " setz %2 \n" -: "=a" (*expected), "=m"(ptr->value), "=q" (ret) -: "a" (*expected), "r" (newval), "m"(ptr->value) -: "memory", "cc"); - return (bool) ret; -} - -#define PG_HAVE_ATOMIC_FETCH_ADD_U64 -static inline uint64 -pg_atomic_fetch_add_u64_impl(volatile pg_atomic_uint64 *ptr, int64 add_) -{ - uint64 res; - __asm__ __volatile__( - " lock \n" - " xaddq %0,%1 \n" -: "=q"(res), "=m"(ptr->value) -: "0" (add_), "m"(ptr->value) -: "memory", "cc"); - return res; -} - -#endif /* __x86_64__ */ - -#endif /* defined(__GNUC__) || defined(__INTEL_COMPILER) */ - -/* - * 8 byte reads / writes have single-copy atomicity on all x86-64 cpus. - */ -#if defined(__x86_64__) -#define PG_HAVE_8BYTE_SINGLE_COPY_ATOMICITY -#endif /* 8 byte single-copy atomicity */ diff --git a/src/include/port/atomics/fallback.h b/src/include/port/atomics/fallback.h deleted file mode 100644 index f37225189fc..00000000000 --- a/src/include/port/atomics/fallback.h +++ /dev/null @@ -1,42 +0,0 @@ -/*------------------------------------------------------------------------- - * - * fallback.h - * Fallback for platforms without 64 bit atomics support. Slower - * than native atomics support, but not unusably slow. - * - * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group - * Portions Copyright (c) 1994, Regents of the University of California - * - * src/include/port/atomics/fallback.h - * - *------------------------------------------------------------------------- - */ - -/* intentionally no include guards, should only be included by atomics.h */ -#ifndef INSIDE_ATOMICS_H -# error "should be included via atomics.h" -#endif - - -#if !defined(PG_HAVE_ATOMIC_U64_SUPPORT) - -#define PG_HAVE_ATOMIC_U64_SIMULATION - -#define PG_HAVE_ATOMIC_U64_SUPPORT -typedef struct pg_atomic_uint64 -{ - int sema; - volatile uint64 value; -} pg_atomic_uint64; - -#define PG_HAVE_ATOMIC_INIT_U64 -extern void pg_atomic_init_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 val_); - -#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64 -extern bool pg_atomic_compare_exchange_u64_impl(volatile pg_atomic_uint64 *ptr, - uint64 *expected, uint64 newval); - -#define PG_HAVE_ATOMIC_FETCH_ADD_U64 -extern uint64 pg_atomic_fetch_add_u64_impl(volatile pg_atomic_uint64 *ptr, int64 add_); - -#endif /* PG_HAVE_ATOMIC_U64_SUPPORT */ diff --git a/src/include/port/atomics/generic-gcc.h b/src/include/port/atomics/generic-gcc.h deleted file mode 100644 index 5bfce82f687..00000000000 --- a/src/include/port/atomics/generic-gcc.h +++ /dev/null @@ -1,326 +0,0 @@ -/*------------------------------------------------------------------------- - * - * generic-gcc.h - * Atomic operations, implemented using gcc (or compatible) intrinsics. - * - * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group - * Portions Copyright (c) 1994, Regents of the University of California - * - * NOTES: - * - * Documentation: - * * Legacy __sync Built-in Functions for Atomic Memory Access - * https://gcc.gnu.org/onlinedocs/gcc-4.8.2/gcc/_005f_005fsync-Builtins.html - * * Built-in functions for memory model aware atomic operations - * https://gcc.gnu.org/onlinedocs/gcc-4.8.2/gcc/_005f_005fatomic-Builtins.html - * - * src/include/port/atomics/generic-gcc.h - * - *------------------------------------------------------------------------- - */ - -/* intentionally no include guards, should only be included by atomics.h */ -#ifndef INSIDE_ATOMICS_H -#error "should be included via atomics.h" -#endif - -/* - * An empty asm block should be a sufficient compiler barrier. - */ -#define pg_compiler_barrier_impl() __asm__ __volatile__("" ::: "memory") - -/* - * If we're on GCC, we should be able to get a memory barrier - * out of this compiler built-in. But we prefer to rely on platform specific - * definitions where possible, and use this only as a fallback. - */ -#if !defined(pg_memory_barrier_impl) -# if defined(HAVE_GCC__ATOMIC_INT32_CAS) -# define pg_memory_barrier_impl() __atomic_thread_fence(__ATOMIC_SEQ_CST) -# elif defined(__GNUC__) -# define pg_memory_barrier_impl() __sync_synchronize() -# endif -#endif /* !defined(pg_memory_barrier_impl) */ - -#if !defined(pg_read_barrier_impl) && defined(HAVE_GCC__ATOMIC_INT32_CAS) -/* acquire semantics include read barrier semantics */ -# define pg_read_barrier_impl() do \ -{ \ - pg_compiler_barrier_impl(); \ - __atomic_thread_fence(__ATOMIC_ACQUIRE); \ -} while (0) -#endif - -#if !defined(pg_write_barrier_impl) && defined(HAVE_GCC__ATOMIC_INT32_CAS) -/* release semantics include write barrier semantics */ -# define pg_write_barrier_impl() do \ -{ \ - pg_compiler_barrier_impl(); \ - __atomic_thread_fence(__ATOMIC_RELEASE); \ -} while (0) -#endif - - -/* generic gcc based atomic flag implementation */ -#if !defined(PG_HAVE_ATOMIC_FLAG_SUPPORT) \ - && (defined(HAVE_GCC__SYNC_INT32_TAS) || defined(HAVE_GCC__SYNC_CHAR_TAS)) - -#define PG_HAVE_ATOMIC_FLAG_SUPPORT -typedef struct pg_atomic_flag -{ - /* - * If we have a choice, use int-width TAS, because that is more efficient - * and/or more reliably implemented on most non-Intel platforms. (Note - * that this code isn't used on x86[_64]; see arch-x86.h for that.) - */ -#ifdef HAVE_GCC__SYNC_INT32_TAS - volatile int value; -#else - volatile char value; -#endif -} pg_atomic_flag; - -#endif /* !ATOMIC_FLAG_SUPPORT && SYNC_INT32_TAS */ - -/* generic gcc based atomic uint32 implementation */ -#if !defined(PG_HAVE_ATOMIC_U32_SUPPORT) \ - && (defined(HAVE_GCC__ATOMIC_INT32_CAS) || defined(HAVE_GCC__SYNC_INT32_CAS)) - -#define PG_HAVE_ATOMIC_U32_SUPPORT -typedef struct pg_atomic_uint32 -{ - volatile uint32 value; -} pg_atomic_uint32; - -#endif /* defined(HAVE_GCC__ATOMIC_INT32_CAS) || defined(HAVE_GCC__SYNC_INT32_CAS) */ - -/* generic gcc based atomic uint64 implementation */ -#if !defined(PG_HAVE_ATOMIC_U64_SUPPORT) \ - && !defined(PG_DISABLE_64_BIT_ATOMICS) \ - && (defined(HAVE_GCC__ATOMIC_INT64_CAS) || defined(HAVE_GCC__SYNC_INT64_CAS)) - -#define PG_HAVE_ATOMIC_U64_SUPPORT -typedef struct pg_atomic_uint64 -{ - alignas(8) volatile uint64 value; -} pg_atomic_uint64; - -#endif /* defined(HAVE_GCC__ATOMIC_INT64_CAS) || defined(HAVE_GCC__SYNC_INT64_CAS) */ - -#ifdef PG_HAVE_ATOMIC_FLAG_SUPPORT - -#if defined(HAVE_GCC__SYNC_CHAR_TAS) || defined(HAVE_GCC__SYNC_INT32_TAS) - -#ifndef PG_HAVE_ATOMIC_TEST_SET_FLAG -#define PG_HAVE_ATOMIC_TEST_SET_FLAG -static inline bool -pg_atomic_test_set_flag_impl(volatile pg_atomic_flag *ptr) -{ - /* NB: only an acquire barrier, not a full one */ - /* some platform only support a 1 here */ - return __sync_lock_test_and_set(&ptr->value, 1) == 0; -} -#endif - -#endif /* defined(HAVE_GCC__SYNC_*_TAS) */ - -#ifndef PG_HAVE_ATOMIC_UNLOCKED_TEST_FLAG -#define PG_HAVE_ATOMIC_UNLOCKED_TEST_FLAG -static inline bool -pg_atomic_unlocked_test_flag_impl(volatile pg_atomic_flag *ptr) -{ - return ptr->value == 0; -} -#endif - -#ifndef PG_HAVE_ATOMIC_CLEAR_FLAG -#define PG_HAVE_ATOMIC_CLEAR_FLAG -static inline void -pg_atomic_clear_flag_impl(volatile pg_atomic_flag *ptr) -{ - __sync_lock_release(&ptr->value); -} -#endif - -#ifndef PG_HAVE_ATOMIC_INIT_FLAG -#define PG_HAVE_ATOMIC_INIT_FLAG -static inline void -pg_atomic_init_flag_impl(volatile pg_atomic_flag *ptr) -{ - pg_atomic_clear_flag_impl(ptr); -} -#endif - -#endif /* defined(PG_HAVE_ATOMIC_FLAG_SUPPORT) */ - -/* prefer __atomic, it has a better API */ -#if !defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32) && defined(HAVE_GCC__ATOMIC_INT32_CAS) -#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32 -static inline bool -pg_atomic_compare_exchange_u32_impl(volatile pg_atomic_uint32 *ptr, - uint32 *expected, uint32 newval) -{ - /* FIXME: we can probably use a lower consistency model */ - return __atomic_compare_exchange_n(&ptr->value, expected, newval, false, - __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); -} -#endif - -#if !defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32) && defined(HAVE_GCC__SYNC_INT32_CAS) -#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32 -static inline bool -pg_atomic_compare_exchange_u32_impl(volatile pg_atomic_uint32 *ptr, - uint32 *expected, uint32 newval) -{ - bool ret; - uint32 current; - current = __sync_val_compare_and_swap(&ptr->value, *expected, newval); - ret = current == *expected; - *expected = current; - return ret; -} -#endif - -/* - * __sync_lock_test_and_set() only supports setting the value to 1 on some - * platforms, so we only provide an __atomic implementation for - * pg_atomic_exchange. - * - * We assume the availability of 32-bit __atomic_compare_exchange_n() implies - * the availability of 32-bit __atomic_exchange_n(). - */ -#if !defined(PG_HAVE_ATOMIC_EXCHANGE_U32) && defined(HAVE_GCC__ATOMIC_INT32_CAS) -#define PG_HAVE_ATOMIC_EXCHANGE_U32 -static inline uint32 -pg_atomic_exchange_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 newval) -{ - return __atomic_exchange_n(&ptr->value, newval, __ATOMIC_SEQ_CST); -} -#endif - -/* if we have 32-bit __sync_val_compare_and_swap, assume we have these too: */ - -#if !defined(PG_HAVE_ATOMIC_FETCH_ADD_U32) && defined(HAVE_GCC__SYNC_INT32_CAS) -#define PG_HAVE_ATOMIC_FETCH_ADD_U32 -static inline uint32 -pg_atomic_fetch_add_u32_impl(volatile pg_atomic_uint32 *ptr, int32 add_) -{ - return __sync_fetch_and_add(&ptr->value, add_); -} -#endif - -#if !defined(PG_HAVE_ATOMIC_FETCH_SUB_U32) && defined(HAVE_GCC__SYNC_INT32_CAS) -#define PG_HAVE_ATOMIC_FETCH_SUB_U32 -static inline uint32 -pg_atomic_fetch_sub_u32_impl(volatile pg_atomic_uint32 *ptr, int32 sub_) -{ - return __sync_fetch_and_sub(&ptr->value, sub_); -} -#endif - -#if !defined(PG_HAVE_ATOMIC_FETCH_AND_U32) && defined(HAVE_GCC__SYNC_INT32_CAS) -#define PG_HAVE_ATOMIC_FETCH_AND_U32 -static inline uint32 -pg_atomic_fetch_and_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 and_) -{ - return __sync_fetch_and_and(&ptr->value, and_); -} -#endif - -#if !defined(PG_HAVE_ATOMIC_FETCH_OR_U32) && defined(HAVE_GCC__SYNC_INT32_CAS) -#define PG_HAVE_ATOMIC_FETCH_OR_U32 -static inline uint32 -pg_atomic_fetch_or_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 or_) -{ - return __sync_fetch_and_or(&ptr->value, or_); -} -#endif - - -#if !defined(PG_DISABLE_64_BIT_ATOMICS) - -#if !defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64) && defined(HAVE_GCC__ATOMIC_INT64_CAS) -#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64 -static inline bool -pg_atomic_compare_exchange_u64_impl(volatile pg_atomic_uint64 *ptr, - uint64 *expected, uint64 newval) -{ - AssertPointerAlignment(expected, 8); - return __atomic_compare_exchange_n(&ptr->value, expected, newval, false, - __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); -} -#endif - -#if !defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64) && defined(HAVE_GCC__SYNC_INT64_CAS) -#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64 -static inline bool -pg_atomic_compare_exchange_u64_impl(volatile pg_atomic_uint64 *ptr, - uint64 *expected, uint64 newval) -{ - bool ret; - uint64 current; - - AssertPointerAlignment(expected, 8); - current = __sync_val_compare_and_swap(&ptr->value, *expected, newval); - ret = current == *expected; - *expected = current; - return ret; -} -#endif - -/* - * __sync_lock_test_and_set() only supports setting the value to 1 on some - * platforms, so we only provide an __atomic implementation for - * pg_atomic_exchange. - * - * We assume the availability of 64-bit __atomic_compare_exchange_n() implies - * the availability of 64-bit __atomic_exchange_n(). - */ -#if !defined(PG_HAVE_ATOMIC_EXCHANGE_U64) && defined(HAVE_GCC__ATOMIC_INT64_CAS) -#define PG_HAVE_ATOMIC_EXCHANGE_U64 -static inline uint64 -pg_atomic_exchange_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 newval) -{ - return __atomic_exchange_n(&ptr->value, newval, __ATOMIC_SEQ_CST); -} -#endif - -/* if we have 64-bit __sync_val_compare_and_swap, assume we have these too: */ - -#if !defined(PG_HAVE_ATOMIC_FETCH_ADD_U64) && defined(HAVE_GCC__SYNC_INT64_CAS) -#define PG_HAVE_ATOMIC_FETCH_ADD_U64 -static inline uint64 -pg_atomic_fetch_add_u64_impl(volatile pg_atomic_uint64 *ptr, int64 add_) -{ - return __sync_fetch_and_add(&ptr->value, add_); -} -#endif - -#if !defined(PG_HAVE_ATOMIC_FETCH_SUB_U64) && defined(HAVE_GCC__SYNC_INT64_CAS) -#define PG_HAVE_ATOMIC_FETCH_SUB_U64 -static inline uint64 -pg_atomic_fetch_sub_u64_impl(volatile pg_atomic_uint64 *ptr, int64 sub_) -{ - return __sync_fetch_and_sub(&ptr->value, sub_); -} -#endif - -#if !defined(PG_HAVE_ATOMIC_FETCH_AND_U64) && defined(HAVE_GCC__SYNC_INT64_CAS) -#define PG_HAVE_ATOMIC_FETCH_AND_U64 -static inline uint64 -pg_atomic_fetch_and_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 and_) -{ - return __sync_fetch_and_and(&ptr->value, and_); -} -#endif - -#if !defined(PG_HAVE_ATOMIC_FETCH_OR_U64) && defined(HAVE_GCC__SYNC_INT64_CAS) -#define PG_HAVE_ATOMIC_FETCH_OR_U64 -static inline uint64 -pg_atomic_fetch_or_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 or_) -{ - return __sync_fetch_and_or(&ptr->value, or_); -} -#endif - -#endif /* !defined(PG_DISABLE_64_BIT_ATOMICS) */ diff --git a/src/include/port/atomics/generic-msvc.h b/src/include/port/atomics/generic-msvc.h deleted file mode 100644 index 6d09cd0e043..00000000000 --- a/src/include/port/atomics/generic-msvc.h +++ /dev/null @@ -1,115 +0,0 @@ -/*------------------------------------------------------------------------- - * - * generic-msvc.h - * Atomic operations support when using MSVC - * - * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group - * Portions Copyright (c) 1994, Regents of the University of California - * - * NOTES: - * - * Documentation: - * * Interlocked Variable Access - * http://msdn.microsoft.com/en-us/library/ms684122%28VS.85%29.aspx - * - * src/include/port/atomics/generic-msvc.h - * - *------------------------------------------------------------------------- - */ -#include - -/* intentionally no include guards, should only be included by atomics.h */ -#ifndef INSIDE_ATOMICS_H -#error "should be included via atomics.h" -#endif - -#pragma intrinsic(_ReadWriteBarrier) -#define pg_compiler_barrier_impl() _ReadWriteBarrier() - -#ifndef pg_memory_barrier_impl -#define pg_memory_barrier_impl() MemoryBarrier() -#endif - -#define PG_HAVE_ATOMIC_U32_SUPPORT -typedef struct pg_atomic_uint32 -{ - volatile uint32 value; -} pg_atomic_uint32; - -#define PG_HAVE_ATOMIC_U64_SUPPORT -typedef struct pg_atomic_uint64 -{ - alignas(8) volatile uint64 value; -} pg_atomic_uint64; - - -#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32 -static inline bool -pg_atomic_compare_exchange_u32_impl(volatile pg_atomic_uint32 *ptr, - uint32 *expected, uint32 newval) -{ - bool ret; - uint32 current; - current = InterlockedCompareExchange(&ptr->value, newval, *expected); - ret = current == *expected; - *expected = current; - return ret; -} - -#define PG_HAVE_ATOMIC_EXCHANGE_U32 -static inline uint32 -pg_atomic_exchange_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 newval) -{ - return InterlockedExchange(&ptr->value, newval); -} - -#define PG_HAVE_ATOMIC_FETCH_ADD_U32 -static inline uint32 -pg_atomic_fetch_add_u32_impl(volatile pg_atomic_uint32 *ptr, int32 add_) -{ - return InterlockedExchangeAdd(&ptr->value, add_); -} - -/* - * The non-intrinsics versions are only available in vista upwards, so use the - * intrinsic version. Only supported on >486, but we require XP as a minimum - * baseline, which doesn't support the 486, so we don't need to add checks for - * that case. - */ -#pragma intrinsic(_InterlockedCompareExchange64) - -#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64 -static inline bool -pg_atomic_compare_exchange_u64_impl(volatile pg_atomic_uint64 *ptr, - uint64 *expected, uint64 newval) -{ - bool ret; - uint64 current; - current = _InterlockedCompareExchange64(&ptr->value, newval, *expected); - ret = current == *expected; - *expected = current; - return ret; -} - -/* Only implemented on 64bit builds */ -#ifdef _WIN64 - -#pragma intrinsic(_InterlockedExchange64) - -#define PG_HAVE_ATOMIC_EXCHANGE_U64 -static inline uint64 -pg_atomic_exchange_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 newval) -{ - return _InterlockedExchange64(&ptr->value, newval); -} - -#pragma intrinsic(_InterlockedExchangeAdd64) - -#define PG_HAVE_ATOMIC_FETCH_ADD_U64 -static inline uint64 -pg_atomic_fetch_add_u64_impl(volatile pg_atomic_uint64 *ptr, int64 add_) -{ - return _InterlockedExchangeAdd64(&ptr->value, add_); -} - -#endif /* _WIN64 */ diff --git a/src/include/port/atomics/generic.h b/src/include/port/atomics/generic.h deleted file mode 100644 index daa772e9a6d..00000000000 --- a/src/include/port/atomics/generic.h +++ /dev/null @@ -1,430 +0,0 @@ -/*------------------------------------------------------------------------- - * - * generic.h - * Implement higher level operations based on some lower level atomic - * operations. - * - * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group - * Portions Copyright (c) 1994, Regents of the University of California - * - * src/include/port/atomics/generic.h - * - *------------------------------------------------------------------------- - */ - -/* intentionally no include guards, should only be included by atomics.h */ -#ifndef INSIDE_ATOMICS_H -# error "should be included via atomics.h" -#endif - -/* - * If read or write barriers are undefined, we upgrade them to full memory - * barriers. - */ -#if !defined(pg_read_barrier_impl) -# define pg_read_barrier_impl pg_memory_barrier_impl -#endif -#if !defined(pg_write_barrier_impl) -# define pg_write_barrier_impl pg_memory_barrier_impl -#endif - -/* provide fallback */ -#if !defined(PG_HAVE_ATOMIC_FLAG_SUPPORT) && defined(PG_HAVE_ATOMIC_U32_SUPPORT) -#define PG_HAVE_ATOMIC_FLAG_SUPPORT -typedef pg_atomic_uint32 pg_atomic_flag; -#endif - -#ifndef PG_HAVE_ATOMIC_READ_U32 -#define PG_HAVE_ATOMIC_READ_U32 -static inline uint32 -pg_atomic_read_u32_impl(volatile pg_atomic_uint32 *ptr) -{ - return ptr->value; -} -#endif - -#ifndef PG_HAVE_ATOMIC_WRITE_U32 -#define PG_HAVE_ATOMIC_WRITE_U32 -static inline void -pg_atomic_write_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 val) -{ - ptr->value = val; -} -#endif - -#ifndef PG_HAVE_ATOMIC_UNLOCKED_WRITE_U32 -#define PG_HAVE_ATOMIC_UNLOCKED_WRITE_U32 -static inline void -pg_atomic_unlocked_write_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 val) -{ - ptr->value = val; -} -#endif - -/* - * provide fallback for test_and_set using atomic_exchange if available - */ -#if !defined(PG_HAVE_ATOMIC_TEST_SET_FLAG) && defined(PG_HAVE_ATOMIC_EXCHANGE_U32) - -#define PG_HAVE_ATOMIC_INIT_FLAG -static inline void -pg_atomic_init_flag_impl(volatile pg_atomic_flag *ptr) -{ - pg_atomic_write_u32_impl(ptr, 0); -} - -#define PG_HAVE_ATOMIC_TEST_SET_FLAG -static inline bool -pg_atomic_test_set_flag_impl(volatile pg_atomic_flag *ptr) -{ - return pg_atomic_exchange_u32_impl(ptr, 1) == 0; -} - -#define PG_HAVE_ATOMIC_UNLOCKED_TEST_FLAG -static inline bool -pg_atomic_unlocked_test_flag_impl(volatile pg_atomic_flag *ptr) -{ - return pg_atomic_read_u32_impl(ptr) == 0; -} - - -#define PG_HAVE_ATOMIC_CLEAR_FLAG -static inline void -pg_atomic_clear_flag_impl(volatile pg_atomic_flag *ptr) -{ - /* XXX: release semantics suffice? */ - pg_memory_barrier_impl(); - pg_atomic_write_u32_impl(ptr, 0); -} - -/* - * provide fallback for test_and_set using atomic_compare_exchange if - * available. - */ -#elif !defined(PG_HAVE_ATOMIC_TEST_SET_FLAG) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32) - -#define PG_HAVE_ATOMIC_INIT_FLAG -static inline void -pg_atomic_init_flag_impl(volatile pg_atomic_flag *ptr) -{ - pg_atomic_write_u32_impl(ptr, 0); -} - -#define PG_HAVE_ATOMIC_TEST_SET_FLAG -static inline bool -pg_atomic_test_set_flag_impl(volatile pg_atomic_flag *ptr) -{ - uint32 value = 0; - return pg_atomic_compare_exchange_u32_impl(ptr, &value, 1); -} - -#define PG_HAVE_ATOMIC_UNLOCKED_TEST_FLAG -static inline bool -pg_atomic_unlocked_test_flag_impl(volatile pg_atomic_flag *ptr) -{ - return pg_atomic_read_u32_impl(ptr) == 0; -} - -#define PG_HAVE_ATOMIC_CLEAR_FLAG -static inline void -pg_atomic_clear_flag_impl(volatile pg_atomic_flag *ptr) -{ - /* XXX: release semantics suffice? */ - pg_memory_barrier_impl(); - pg_atomic_write_u32_impl(ptr, 0); -} - -#elif !defined(PG_HAVE_ATOMIC_TEST_SET_FLAG) -# error "No pg_atomic_test_and_set provided" -#endif /* !defined(PG_HAVE_ATOMIC_TEST_SET_FLAG) */ - - -#ifndef PG_HAVE_ATOMIC_INIT_U32 -#define PG_HAVE_ATOMIC_INIT_U32 -static inline void -pg_atomic_init_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 val_) -{ - ptr->value = val_; -} -#endif - -#if !defined(PG_HAVE_ATOMIC_EXCHANGE_U32) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32) -#define PG_HAVE_ATOMIC_EXCHANGE_U32 -static inline uint32 -pg_atomic_exchange_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 xchg_) -{ - uint32 old; - old = ptr->value; /* ok if read is not atomic */ - while (!pg_atomic_compare_exchange_u32_impl(ptr, &old, xchg_)) - /* skip */; - return old; -} -#endif - -#if !defined(PG_HAVE_ATOMIC_FETCH_ADD_U32) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32) -#define PG_HAVE_ATOMIC_FETCH_ADD_U32 -static inline uint32 -pg_atomic_fetch_add_u32_impl(volatile pg_atomic_uint32 *ptr, int32 add_) -{ - uint32 old; - old = ptr->value; /* ok if read is not atomic */ - while (!pg_atomic_compare_exchange_u32_impl(ptr, &old, old + add_)) - /* skip */; - return old; -} -#endif - -#if !defined(PG_HAVE_ATOMIC_FETCH_SUB_U32) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32) -#define PG_HAVE_ATOMIC_FETCH_SUB_U32 -static inline uint32 -pg_atomic_fetch_sub_u32_impl(volatile pg_atomic_uint32 *ptr, int32 sub_) -{ - return pg_atomic_fetch_add_u32_impl(ptr, -sub_); -} -#endif - -#if !defined(PG_HAVE_ATOMIC_FETCH_AND_U32) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32) -#define PG_HAVE_ATOMIC_FETCH_AND_U32 -static inline uint32 -pg_atomic_fetch_and_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 and_) -{ - uint32 old; - old = ptr->value; /* ok if read is not atomic */ - while (!pg_atomic_compare_exchange_u32_impl(ptr, &old, old & and_)) - /* skip */; - return old; -} -#endif - -#if !defined(PG_HAVE_ATOMIC_FETCH_OR_U32) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32) -#define PG_HAVE_ATOMIC_FETCH_OR_U32 -static inline uint32 -pg_atomic_fetch_or_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 or_) -{ - uint32 old; - old = ptr->value; /* ok if read is not atomic */ - while (!pg_atomic_compare_exchange_u32_impl(ptr, &old, old | or_)) - /* skip */; - return old; -} -#endif - -#if !defined(PG_HAVE_ATOMIC_ADD_FETCH_U32) && defined(PG_HAVE_ATOMIC_FETCH_ADD_U32) -#define PG_HAVE_ATOMIC_ADD_FETCH_U32 -static inline uint32 -pg_atomic_add_fetch_u32_impl(volatile pg_atomic_uint32 *ptr, int32 add_) -{ - return pg_atomic_fetch_add_u32_impl(ptr, add_) + add_; -} -#endif - -#if !defined(PG_HAVE_ATOMIC_SUB_FETCH_U32) && defined(PG_HAVE_ATOMIC_FETCH_SUB_U32) -#define PG_HAVE_ATOMIC_SUB_FETCH_U32 -static inline uint32 -pg_atomic_sub_fetch_u32_impl(volatile pg_atomic_uint32 *ptr, int32 sub_) -{ - return pg_atomic_fetch_sub_u32_impl(ptr, sub_) - sub_; -} -#endif - -#if !defined(PG_HAVE_ATOMIC_READ_MEMBARRIER_U32) && defined(PG_HAVE_ATOMIC_FETCH_ADD_U32) -#define PG_HAVE_ATOMIC_READ_MEMBARRIER_U32 -static inline uint32 -pg_atomic_read_membarrier_u32_impl(volatile pg_atomic_uint32 *ptr) -{ - return pg_atomic_fetch_add_u32_impl(ptr, 0); -} -#endif - -#if !defined(PG_HAVE_ATOMIC_WRITE_MEMBARRIER_U32) && defined(PG_HAVE_ATOMIC_EXCHANGE_U32) -#define PG_HAVE_ATOMIC_WRITE_MEMBARRIER_U32 -static inline void -pg_atomic_write_membarrier_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 val) -{ - (void) pg_atomic_exchange_u32_impl(ptr, val); -} -#endif - -#if !defined(PG_HAVE_ATOMIC_EXCHANGE_U64) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64) -#define PG_HAVE_ATOMIC_EXCHANGE_U64 -static inline uint64 -pg_atomic_exchange_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 xchg_) -{ - uint64 old; - old = ptr->value; /* ok if read is not atomic */ - while (!pg_atomic_compare_exchange_u64_impl(ptr, &old, xchg_)) - /* skip */; - return old; -} -#endif - -#ifndef PG_HAVE_ATOMIC_WRITE_U64 -#define PG_HAVE_ATOMIC_WRITE_U64 - -#if defined(PG_HAVE_8BYTE_SINGLE_COPY_ATOMICITY) && \ - !defined(PG_HAVE_ATOMIC_U64_SIMULATION) - -static inline void -pg_atomic_write_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 val) -{ - /* - * On this platform aligned 64bit writes are guaranteed to be atomic, - * except if using the fallback implementation, where can't guarantee the - * required alignment. - */ - AssertPointerAlignment(ptr, 8); - ptr->value = val; -} - -#else - -static inline void -pg_atomic_write_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 val) -{ - /* - * 64 bit writes aren't safe on all platforms. In the generic - * implementation implement them as an atomic exchange. - */ - pg_atomic_exchange_u64_impl(ptr, val); -} - -#endif /* PG_HAVE_8BYTE_SINGLE_COPY_ATOMICITY && !PG_HAVE_ATOMIC_U64_SIMULATION */ -#endif /* PG_HAVE_ATOMIC_WRITE_U64 */ - -#ifndef PG_HAVE_ATOMIC_UNLOCKED_WRITE_U64 -#define PG_HAVE_ATOMIC_UNLOCKED_WRITE_U64 -static inline void -pg_atomic_unlocked_write_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 val) -{ - ptr->value = val; -} -#endif - -#ifndef PG_HAVE_ATOMIC_READ_U64 -#define PG_HAVE_ATOMIC_READ_U64 - -#if defined(PG_HAVE_8BYTE_SINGLE_COPY_ATOMICITY) && \ - !defined(PG_HAVE_ATOMIC_U64_SIMULATION) - -static inline uint64 -pg_atomic_read_u64_impl(volatile pg_atomic_uint64 *ptr) -{ - /* - * On this platform aligned 64-bit reads are guaranteed to be atomic. - */ - AssertPointerAlignment(ptr, 8); - return ptr->value; -} - -#else - -static inline uint64 -pg_atomic_read_u64_impl(volatile pg_atomic_uint64 *ptr) -{ - uint64 old = 0; - - /* - * 64-bit reads aren't atomic on all platforms. In the generic - * implementation implement them as a compare/exchange with 0. That'll - * fail or succeed, but always return the old value. Possibly might store - * a 0, but only if the previous value also was a 0 - i.e. harmless. - */ - pg_atomic_compare_exchange_u64_impl(ptr, &old, 0); - - return old; -} -#endif /* PG_HAVE_8BYTE_SINGLE_COPY_ATOMICITY && !PG_HAVE_ATOMIC_U64_SIMULATION */ -#endif /* PG_HAVE_ATOMIC_READ_U64 */ - -#ifndef PG_HAVE_ATOMIC_INIT_U64 -#define PG_HAVE_ATOMIC_INIT_U64 -static inline void -pg_atomic_init_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 val_) -{ - ptr->value = val_; -} -#endif - -#if !defined(PG_HAVE_ATOMIC_FETCH_ADD_U64) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64) -#define PG_HAVE_ATOMIC_FETCH_ADD_U64 -static inline uint64 -pg_atomic_fetch_add_u64_impl(volatile pg_atomic_uint64 *ptr, int64 add_) -{ - uint64 old; - old = ptr->value; /* ok if read is not atomic */ - while (!pg_atomic_compare_exchange_u64_impl(ptr, &old, old + add_)) - /* skip */; - return old; -} -#endif - -#if !defined(PG_HAVE_ATOMIC_FETCH_SUB_U64) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64) -#define PG_HAVE_ATOMIC_FETCH_SUB_U64 -static inline uint64 -pg_atomic_fetch_sub_u64_impl(volatile pg_atomic_uint64 *ptr, int64 sub_) -{ - return pg_atomic_fetch_add_u64_impl(ptr, -sub_); -} -#endif - -#if !defined(PG_HAVE_ATOMIC_FETCH_AND_U64) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64) -#define PG_HAVE_ATOMIC_FETCH_AND_U64 -static inline uint64 -pg_atomic_fetch_and_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 and_) -{ - uint64 old; - old = ptr->value; /* ok if read is not atomic */ - while (!pg_atomic_compare_exchange_u64_impl(ptr, &old, old & and_)) - /* skip */; - return old; -} -#endif - -#if !defined(PG_HAVE_ATOMIC_FETCH_OR_U64) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64) -#define PG_HAVE_ATOMIC_FETCH_OR_U64 -static inline uint64 -pg_atomic_fetch_or_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 or_) -{ - uint64 old; - old = ptr->value; /* ok if read is not atomic */ - while (!pg_atomic_compare_exchange_u64_impl(ptr, &old, old | or_)) - /* skip */; - return old; -} -#endif - -#if !defined(PG_HAVE_ATOMIC_ADD_FETCH_U64) && defined(PG_HAVE_ATOMIC_FETCH_ADD_U64) -#define PG_HAVE_ATOMIC_ADD_FETCH_U64 -static inline uint64 -pg_atomic_add_fetch_u64_impl(volatile pg_atomic_uint64 *ptr, int64 add_) -{ - return pg_atomic_fetch_add_u64_impl(ptr, add_) + add_; -} -#endif - -#if !defined(PG_HAVE_ATOMIC_SUB_FETCH_U64) && defined(PG_HAVE_ATOMIC_FETCH_SUB_U64) -#define PG_HAVE_ATOMIC_SUB_FETCH_U64 -static inline uint64 -pg_atomic_sub_fetch_u64_impl(volatile pg_atomic_uint64 *ptr, int64 sub_) -{ - return pg_atomic_fetch_sub_u64_impl(ptr, sub_) - sub_; -} -#endif - -#if !defined(PG_HAVE_ATOMIC_READ_MEMBARRIER_U64) && defined(PG_HAVE_ATOMIC_FETCH_ADD_U64) -#define PG_HAVE_ATOMIC_READ_MEMBARRIER_U64 -static inline uint64 -pg_atomic_read_membarrier_u64_impl(volatile pg_atomic_uint64 *ptr) -{ - return pg_atomic_fetch_add_u64_impl(ptr, 0); -} -#endif - -#if !defined(PG_HAVE_ATOMIC_WRITE_MEMBARRIER_U64) && defined(PG_HAVE_ATOMIC_EXCHANGE_U64) -#define PG_HAVE_ATOMIC_WRITE_MEMBARRIER_U64 -static inline void -pg_atomic_write_membarrier_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 val) -{ - (void) pg_atomic_exchange_u64_impl(ptr, val); -} -#endif diff --git a/src/include/port/atomics/stdatomic_impl.h b/src/include/port/atomics/stdatomic_impl.h new file mode 100644 index 00000000000..4e0d893e1c3 --- /dev/null +++ b/src/include/port/atomics/stdatomic_impl.h @@ -0,0 +1,586 @@ +/*------------------------------------------------------------------------- + * + * stdatomic_impl.h + * Atomic operations implementation using C11 stdatomic.h + * + * This file provides PostgreSQL atomic operations using the C11 standard + * . As of this commit it is the sole atomics implementation; + * the former platform-specific paths (arch-*.h, generic-*.h, fallback.h) + * have been removed and USE_STDATOMIC_H is always defined. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/port/atomics/stdatomic_impl.h + * + *------------------------------------------------------------------------- + */ + +#ifndef STDATOMIC_IMPL_H +#define STDATOMIC_IMPL_H + +/* + * C++ Compatibility + * + * C++11 through C++20 cannot include C's directly. Instead, + * we use C++11's header and map to std::atomic types. C++23 allows + * but we handle older versions. + * + * This follows the approach from C++23 standard section 33.5.12 which + * allows C and C++ atomic types to interoperate. + */ +#if defined(__cplusplus) && __cplusplus < 202302L +extern "C++" +{ +#include + + /* Map to C++ atomic types */ +#define pg_atomic(T) std::atomic + + /* Import memory order constants into global namespace */ + using std::memory_order_relaxed; + using std::memory_order_acquire; + using std::memory_order_release; + using std::memory_order_acq_rel; + using std::memory_order_seq_cst; +} +#else +/* + * C11, or C++23 and later where is available. + * + * In C++ mode this must be wrapped in extern "C++": under C++23 + * pulls in , which declares templates, and extensions + * conventionally include our headers inside extern "C". Templates cannot + * have C language linkage, so an unguarded include fails to compile. + */ +#ifdef __cplusplus +extern "C++" +{ +#include +} +#else +#include +#endif +#define pg_atomic(T) _Atomic(T) +#endif + +/* + * Type Definitions + * + * We use C11 _Atomic qualifier (or C++ std::atomic) with PostgreSQL's + * standard integer types. These types are compatible with atomic operations + * and provide sequential consistency guarantees. + * + * Note: pg_atomic(T) is defined above and maps to either _Atomic(T) for C + * or std::atomic for C++. + */ +/* + * pg_atomic_flag is backed by a full 32-bit word, not uint8, matching the + * traditional implementation. The reason is the one given in generic-gcc.h: + * int-width atomic operations are more efficient and/or more reliably + * implemented than byte-width ones. (Sub-word atomics are not unsafe on + * targets without byte-granular AMOs -- compilers emit a masked LR/SC on the + * containing word, which preserves the neighbouring bytes -- but they are + * bigger and slower than a plain word AMO.) + */ +typedef pg_atomic(uint32) pg_atomic_flag; +typedef pg_atomic(uint32) pg_atomic_uint32; +typedef pg_atomic(uint64) pg_atomic_uint64; + +/* + * PostgreSQL's atomics live in shared memory and are accessed from multiple + * *processes*, not just threads. A non-lock-free atomic implemented by the + * compiler/runtime with a lock table (e.g. libatomic) uses process-local + * locks, which would silently fail to provide atomicity across processes and + * corrupt shared state. Require genuine, always-lock-free support at compile + * time for every width we use. (ATOMIC_*_LOCK_FREE is 0 = never lock-free, + * 1 = sometimes / not known at compile time, 2 = always lock-free.) + * + * The == 1 case is a real hazard, not just conservatism: e.g. 64-bit atomics + * on a 32-bit target may be emulated by libatomic with a process-local lock, + * which corrupts cross-process shared memory exactly as == 0 would. So we + * require == 2 everywhere the macro is trustworthy. + * + * The one exception is MSVC compiling as C: its reports + * ATOMIC_*_LOCK_FREE == 1 for every width even though the atomics are in fact + * always lock-free on the hardware we target (in C++ mode the same MSVC + * reports 2, and atomic_is_lock_free() returns true at run time). We skip the + * assertion only in that specific configuration rather than weakening it for + * everyone. + */ +#if !(defined(_MSC_VER) && !defined(__cplusplus)) +StaticAssertDecl(ATOMIC_INT_LOCK_FREE == 2, "32-bit atomics are not lock-free"); +StaticAssertDecl(ATOMIC_LLONG_LOCK_FREE == 2, "64-bit atomics are not lock-free"); +#endif + +/* + * Signal atomic operation support to atomics.h + * + * stdatomic.h provides native support for all atomic types on all platforms + * it compiles on, so we unconditionally define these. PG_HAVE_ATOMIC_U64_SIMULATION + * is intentionally NOT defined -- stdatomic provides real 64-bit atomics. + */ +#define PG_HAVE_ATOMIC_U32_SUPPORT +#define PG_HAVE_ATOMIC_U64_SUPPORT + +/* + * 8-byte single-copy atomicity. + * + * All 64-bit architectures that support C11 stdatomic.h guarantee 8-byte + * single-copy atomicity for aligned loads/stores. This is used by bufmgr.c + * and other performance-critical paths to avoid unnecessary atomic operations. + */ +#if SIZEOF_VOID_P >= 8 +#define PG_HAVE_8BYTE_SINGLE_COPY_ATOMICITY +#endif + +/* + * Memory Barrier Implementation + * + * PostgreSQL's memory barriers must order *non-atomic, non-volatile* accesses + * too, which a bare C11 atomic_thread_fence() does NOT guarantee against + * compiler reordering: atomic_thread_fence only establishes ordering with + * respect to atomic operations, so the compiler is still free to move plain + * loads/stores across it. We therefore pair each thread fence with an + * explicit compiler barrier (atomic_signal_fence), matching the semantics of + * the traditional implementation whose barriers also constrain plain memory + * accesses. Omitting the compiler barrier corrupts lock-free algorithms on + * weak-memory architectures (observed as lost tuples in parallel hash join on + * RISC-V, where the compiler reordered non-atomic hash-table stores across a + * bare acquire/release fence). + * + * These are implementation functions (_impl suffix) mapped to the public API + * by atomics.h. + */ +#define pg_compiler_barrier_impl() atomic_signal_fence(memory_order_seq_cst) + +static inline void +pg_memory_barrier_impl(void) +{ + atomic_signal_fence(memory_order_seq_cst); + atomic_thread_fence(memory_order_seq_cst); +} + +static inline void +pg_read_barrier_impl(void) +{ + atomic_signal_fence(memory_order_seq_cst); + atomic_thread_fence(memory_order_acquire); +} + +static inline void +pg_write_barrier_impl(void) +{ + atomic_signal_fence(memory_order_seq_cst); + atomic_thread_fence(memory_order_release); +} + +/* + * Spin Delay Implementation + * + * On x86, a spinloop without a "pause" instruction can waste a lot of power + * and slow down the adjacent hyperthread. On ARM64, ISB provides better + * backoff than YIELD. This provides platform-specific spin delay hints. + * + * The implementation is in spin_delay.h which directly defines pg_spin_delay_impl(), + * matching the pattern used by the traditional atomics implementation. + */ +#include "port/spin_delay.h" + +/* + * ================================================================= + * pg_atomic_flag Operations + * ================================================================= + * + * Atomic flag type used primarily for spinlocks. + * + * CRITICAL FLAG CONVENTION DIFFERENCE FROM TRADITIONAL IMPLEMENTATION: + * + * stdatomic.h implementation (this file): + * 1 = unlocked, 0 = locked + * + * Traditional implementation (generic.h): + * 0 = unlocked, 1 = locked + * + * This polarity difference is INTENTIONAL and internal to the implementation. + * The public API behavior is identical in both implementations: + * - pg_atomic_init_flag() initializes to unlocked state + * - pg_atomic_test_set_flag() returns true if lock was successfully acquired + * - pg_atomic_clear_flag() releases the lock + * + * The stdatomic.h convention (1=unlocked) is more natural for C11 atomics + * and avoids double-negation logic in the implementation. + */ + +/* + * pg_atomic_init_flag_impl - Initialize atomic flag to unlocked state + * + * In the stdatomic.h implementation, we initialize to 1 (unlocked). + */ +static inline void +pg_atomic_init_flag_impl(volatile pg_atomic_flag *ptr) +{ + atomic_init(ptr, 1); /* 1 = unlocked */ +} + +/* + * pg_atomic_test_set_flag_impl - Try to acquire lock + * + * Atomically clears the flag and returns whether it had been set. Returns + * true if we acquired it (previous value 1 == unlocked), false if it was + * already held (previous value 0). + * + * An exchange is used rather than fetch_and(0): both express the same + * operation, but exchange is the one every target lowers to a single + * instruction (xchg on x86), whereas fetch_and can fall back to a + * compare-exchange loop on some compilers. + * + * Memory ordering: acquire (synchronizes-with prior release) + */ +static inline bool +pg_atomic_test_set_flag_impl(volatile pg_atomic_flag *ptr) +{ + return atomic_exchange_explicit(ptr, 0, memory_order_acquire) != 0; +} + +/* + * pg_atomic_unlocked_test_flag_impl - Test if flag is unlocked without acquiring + * + * Returns true if the flag is currently unlocked (value == 1). + * Uses relaxed ordering since this is typically used for optimization hints. + */ +static inline bool +pg_atomic_unlocked_test_flag_impl(volatile pg_atomic_flag *ptr) +{ + return atomic_load_explicit(ptr, memory_order_relaxed) != 0; +} + +/* + * pg_atomic_clear_flag_impl - Release lock + * + * Sets flag to 1 (unlocked). + * Memory ordering: release (makes prior writes visible) + */ +static inline void +pg_atomic_clear_flag_impl(volatile pg_atomic_flag *ptr) +{ + atomic_store_explicit(ptr, 1, memory_order_release); +} + +/* + * ================================================================= + * pg_atomic_uint32 Operations + * ================================================================= + * + * 32-bit atomic unsigned integer operations. + * All operations use sequential consistency unless otherwise noted. + */ + +/* + * pg_atomic_init_u32_impl - Initialize atomic uint32 + * + * This is not an atomic operation - must only be called during initialization + * when no concurrent access is possible. + */ +static inline void +pg_atomic_init_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 val) +{ + atomic_init(ptr, val); +} + +/* + * pg_atomic_read_u32_impl - Atomically read uint32 value + * + * Memory ordering: sequentially consistent. This is stronger than the + * documented "no barrier semantics" of pg_atomic_read_u32(), and it is + * deliberate: a bare memory_order_relaxed load reintroduces a real, + * reproducible data-loss bug on weak-memory hardware. + * + * The traditional implementation reads through a volatile-qualified pointer. + * A volatile access cannot be reordered *by the compiler* relative to the + * surrounding code, so consumers that publish a pointer with a + * barrier-carrying store and then read+dereference it get the dependent load + * ordered for free. A C11 memory_order_relaxed atomic load does not carry + * that compiler-ordering property, and the parallel hash join relies on it: + * ExecParallelHash reads a freshly published bucket pointer via + * dsa_pointer_atomic_read() and then dereferences the tuple it points to. + * With a relaxed load this was observed to lose a tuple on RISC-V -- + * join_hash "extremely_skewed" returns 19999 instead of 20000, intermittently + * (reproduced here at roughly 1 run in 4 on real rv64 hardware). seq_cst + * restores correctness. + * + * The cost of the seq_cst load is not the ARM regression measured for an + * earlier revision of this work: an uncontended ldar is not measurably + * costlier than a plain ldr, and that ~2% aarch64 read-only regression was + * traced to the seq_cst store (STLR) in pg_atomic_write_u32, which is relaxed + * (see below). + * + * Leaving this primitive relaxed and adding the ordering only at the apparent + * consumer -- e.g. a pg_read_barrier() between the bucket read and the tuple + * dereference in ExecParallelHashFirstTuple() -- is not sufficient: on RISC-V + * join_hash still lost tuples at the same rate. The ordering the volatile + * load provided is not local to that one read; the parallel hash join also + * reaches shared state through the Barrier, condition-variable and LWLock + * machinery, all of which read via pg_atomic_read_*() and relied on the same + * compiler-ordering. An ordered read here restores that property tree-wide. + */ +static inline uint32 +pg_atomic_read_u32_impl(volatile pg_atomic_uint32 *ptr) +{ + return atomic_load_explicit(ptr, memory_order_seq_cst); +} + +/* + * pg_atomic_write_u32_impl - Atomically write uint32 value + * + * Memory ordering: relaxed. A plain atomic (non-torn) store matches the + * traditional implementation's "no barrier" contract for pg_atomic_write_u32 + * and, on ARM64, compiles to a plain STR rather than STLR. This matters: + * benchmarking traced the ~2% aarch64 read-only regression of an earlier + * seq_cst-everywhere revision to STLR store-side serialization on hot, + * contended shared lines (LWLock state, buffer headers), not to the reads. + * Making the write relaxed removes that cost. Callers needing an ordered + * write use pg_atomic_write_membarrier_u32() or an explicit barrier. + */ +static inline void +pg_atomic_write_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 val) +{ + atomic_store_explicit(ptr, val, memory_order_relaxed); +} + +/* + * pg_atomic_exchange_u32_impl - Atomically exchange uint32 value + * + * Atomically replaces the value with newval and returns the old value. + * Memory ordering: seq_cst + */ +static inline uint32 +pg_atomic_exchange_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 newval) +{ + return atomic_exchange_explicit(ptr, newval, memory_order_seq_cst); +} + +/* + * pg_atomic_compare_exchange_u32_impl - Atomic compare-and-swap + * + * Compares *ptr with *expected. If equal, replaces *ptr with newval and + * returns true. If not equal, updates *expected with current *ptr value + * and returns false. + * + * This is the strong version (no spurious failures). + * Memory ordering: seq_cst for both success and failure + */ +static inline bool +pg_atomic_compare_exchange_u32_impl(volatile pg_atomic_uint32 *ptr, + uint32 *expected, uint32 newval) +{ + return atomic_compare_exchange_strong_explicit(ptr, expected, newval, + memory_order_seq_cst, + memory_order_seq_cst); +} + +/* + * pg_atomic_fetch_add_u32_impl - Atomic fetch-and-add + * + * Atomically adds add_ to *ptr and returns the old value. + * Memory ordering: seq_cst + */ +static inline uint32 +pg_atomic_fetch_add_u32_impl(volatile pg_atomic_uint32 *ptr, int32 add_) +{ + return atomic_fetch_add_explicit(ptr, add_, memory_order_seq_cst); +} + +/* + * pg_atomic_fetch_sub_u32_impl - Atomic fetch-and-subtract + * + * Atomically subtracts sub_ from *ptr and returns the old value. + * Memory ordering: seq_cst + */ +static inline uint32 +pg_atomic_fetch_sub_u32_impl(volatile pg_atomic_uint32 *ptr, int32 sub_) +{ + return atomic_fetch_sub_explicit(ptr, sub_, memory_order_seq_cst); +} + +/* + * pg_atomic_fetch_and_u32_impl - Atomic fetch-and-AND + * + * Atomically performs *ptr &= and_ and returns the old value. + * Memory ordering: seq_cst + */ +static inline uint32 +pg_atomic_fetch_and_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 and_) +{ + return atomic_fetch_and_explicit(ptr, and_, memory_order_seq_cst); +} + +/* + * pg_atomic_fetch_or_u32_impl - Atomic fetch-and-OR + * + * Atomically performs *ptr |= or_ and returns the old value. + * Memory ordering: seq_cst + */ +static inline uint32 +pg_atomic_fetch_or_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 or_) +{ + return atomic_fetch_or_explicit(ptr, or_, memory_order_seq_cst); +} + +/* + * pg_atomic_add_fetch_u32_impl - Atomic add-and-fetch + * + * Atomically adds add_ to *ptr and returns the NEW value. + * Implemented using fetch_add + add_ since C11 doesn't have add_fetch. + */ +static inline uint32 +pg_atomic_add_fetch_u32_impl(volatile pg_atomic_uint32 *ptr, int32 add_) +{ + return atomic_fetch_add_explicit(ptr, add_, memory_order_seq_cst) + add_; +} + +/* + * pg_atomic_sub_fetch_u32_impl - Atomic subtract-and-fetch + * + * Atomically subtracts sub_ from *ptr and returns the NEW value. + */ +static inline uint32 +pg_atomic_sub_fetch_u32_impl(volatile pg_atomic_uint32 *ptr, int32 sub_) +{ + return atomic_fetch_sub_explicit(ptr, sub_, memory_order_seq_cst) - sub_; +} + +/* + * pg_atomic_read_membarrier_u32_impl - Read with full barrier semantics + * + * The contract is a *full* barrier, which a seq_cst load does not provide: + * on x86 a seq_cst load is a plain mov and is not ordered against a + * preceding store. Use an RMW that returns the value, matching generic.h. + */ +static inline uint32 +pg_atomic_read_membarrier_u32_impl(volatile pg_atomic_uint32 *ptr) +{ + return atomic_fetch_add_explicit(ptr, 0, memory_order_seq_cst); +} + +/* + * pg_atomic_unlocked_write_u32_impl - Non-atomic write + * + * Write without atomicity guarantees. Only safe when exclusive access + * is guaranteed externally. Uses relaxed ordering. + */ +static inline void +pg_atomic_unlocked_write_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 val) +{ + atomic_store_explicit(ptr, val, memory_order_relaxed); +} + +/* + * pg_atomic_write_membarrier_u32_impl - Write with full barrier semantics + * + * A seq_cst store is not a full barrier either: on ARM64 an STLR does not + * order later plain loads. Use an exchange, matching generic.h. + */ +static inline void +pg_atomic_write_membarrier_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 val) +{ + (void) atomic_exchange_explicit(ptr, val, memory_order_seq_cst); +} + +/* + * ================================================================= + * pg_atomic_uint64 Operations + * ================================================================= + * + * 64-bit equivalents of the u32 operations above. See the u32 comments + * for API documentation; semantics and memory ordering are identical. + */ + +static inline void +pg_atomic_init_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 val) +{ + atomic_init(ptr, val); +} + +static inline uint64 +pg_atomic_read_u64_impl(volatile pg_atomic_uint64 *ptr) +{ + return atomic_load_explicit(ptr, memory_order_seq_cst); +} + +static inline void +pg_atomic_write_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 val) +{ + atomic_store_explicit(ptr, val, memory_order_relaxed); +} + +static inline uint64 +pg_atomic_exchange_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 newval) +{ + return atomic_exchange_explicit(ptr, newval, memory_order_seq_cst); +} + +static inline bool +pg_atomic_compare_exchange_u64_impl(volatile pg_atomic_uint64 *ptr, + uint64 *expected, uint64 newval) +{ + return atomic_compare_exchange_strong_explicit(ptr, expected, newval, + memory_order_seq_cst, + memory_order_seq_cst); +} + +static inline uint64 +pg_atomic_fetch_add_u64_impl(volatile pg_atomic_uint64 *ptr, int64 add_) +{ + return atomic_fetch_add_explicit(ptr, add_, memory_order_seq_cst); +} + +static inline uint64 +pg_atomic_fetch_sub_u64_impl(volatile pg_atomic_uint64 *ptr, int64 sub_) +{ + return atomic_fetch_sub_explicit(ptr, sub_, memory_order_seq_cst); +} + +static inline uint64 +pg_atomic_fetch_and_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 and_) +{ + return atomic_fetch_and_explicit(ptr, and_, memory_order_seq_cst); +} + +static inline uint64 +pg_atomic_fetch_or_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 or_) +{ + return atomic_fetch_or_explicit(ptr, or_, memory_order_seq_cst); +} + +static inline uint64 +pg_atomic_add_fetch_u64_impl(volatile pg_atomic_uint64 *ptr, int64 add_) +{ + return atomic_fetch_add_explicit(ptr, add_, memory_order_seq_cst) + add_; +} + +static inline uint64 +pg_atomic_sub_fetch_u64_impl(volatile pg_atomic_uint64 *ptr, int64 sub_) +{ + return atomic_fetch_sub_explicit(ptr, sub_, memory_order_seq_cst) - sub_; +} + +static inline uint64 +pg_atomic_read_membarrier_u64_impl(volatile pg_atomic_uint64 *ptr) +{ + return atomic_fetch_add_explicit(ptr, 0, memory_order_seq_cst); +} + +static inline void +pg_atomic_unlocked_write_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 val) +{ + atomic_store_explicit(ptr, val, memory_order_relaxed); +} + +static inline void +pg_atomic_write_membarrier_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 val) +{ + (void) atomic_exchange_explicit(ptr, val, memory_order_seq_cst); +} + +#endif /* STDATOMIC_IMPL_H */ diff --git a/src/include/port/spin_delay.h b/src/include/port/spin_delay.h new file mode 100644 index 00000000000..c88b3729a2e --- /dev/null +++ b/src/include/port/spin_delay.h @@ -0,0 +1,148 @@ +/*------------------------------------------------------------------------- + * + * spin_delay.h + * Platform-specific spin delay for busy-wait loops + * + * This file provides pg_spin_delay(), a platform-optimized delay instruction + * for use in spinlock contention loops. Different architectures have different + * optimal instructions for indicating to the CPU that we're in a busy-wait. + * + * Key optimizations: + * - x86/x86_64: PAUSE instruction (rep nop) reduces power and helps hyperthreads + * - ARM64: ISB instruction, per the discussion thread below, backs off better + * than YIELD under heavy spinlock contention on high-core-count parts + * - Windows ARM64: __isb() intrinsic (same rationale as ARM64 above) + * + * Discussion: https://postgr.es/m/1c2a29b8-5b1e-44f7-a871-71ec5fefc120%40app.fastmail.com + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/port/spin_delay.h + * + *------------------------------------------------------------------------- + */ +#ifndef SPIN_DELAY_H +#define SPIN_DELAY_H + +#ifdef _MSC_VER +/* + * MSVC intrinsics used below (__isb, _mm_pause, _ReadWriteBarrier) are declared + * in , which also provides 's _mm_pause. Include it at + * file scope rather than relying on another header having pulled it in first. + */ +#include +#endif + +/* + * pg_spin_delay_impl - Execute a platform-specific CPU hint for spin-wait loops + * + * This function should be called inside busy-wait loops to: + * 1. Reduce CPU power consumption during contention + * 2. Improve performance of sibling hyperthreads + * 3. Signal to the CPU that we're in a spin loop + * + * The implementation varies by platform to use the most efficient instruction. + * + * Note: This defines pg_spin_delay_impl() directly, matching the pattern used + * by the traditional atomics implementation (arch-*.h files). + */ +#ifndef PG_HAVE_SPIN_DELAY +#define PG_HAVE_SPIN_DELAY +static inline void +pg_spin_delay_impl(void) +{ +#if defined(__GNUC__) || defined(__INTEL_COMPILER) + + /* + * GCC and Intel compiler: use inline assembly for optimal instructions + */ + +#if defined(__i386__) || defined(__i386) || defined(__x86_64__) + /* x86 / x86-64: PAUSE instruction (encoded as rep nop) */ + __asm__ __volatile__(" rep; nop \n"); +#elif defined(__aarch64__) + /* + * ARM64: ISB (Instruction Synchronization Barrier). Per the discussion + * thread above, ISB backs off better than YIELD in spinlock loops on + * high-core-count ARM64 parts; it forces a pipeline flush that YIELD does + * not. + */ + __asm__ __volatile__(" isb; \n"); +#elif defined(__arm__) || defined(__arm) + /* ARM 32-bit: YIELD hint */ + __asm__ __volatile__(" yield; \n"); +#else + /* + * Other architectures: compiler barrier only + * + * A compiler barrier prevents the compiler from optimizing away the loop, + * even if we don't have an architecture-specific delay instruction. + */ + __asm__ __volatile__("":::"memory"); +#endif + +#elif defined(_MSC_VER) + + /* + * Microsoft Visual C++: use intrinsics (declared in , included + * at file scope above). + */ + +#if defined(_M_ARM64) || defined(_M_ARM64EC) + /* + * Windows ARM64: ISB via intrinsic, matching the GCC/Clang ARM64 path + * above. _ARM64_BARRIER_SY is a full system barrier. + */ + __isb(_ARM64_BARRIER_SY); +#elif defined(_M_AMD64) + /* Windows x86-64: _mm_pause() maps to the PAUSE instruction. */ + _mm_pause(); +#elif defined(_M_IX86) + /* Windows x86 32-bit: MASM syntax for PAUSE (rep nop). */ + __asm rep nop; +#else + /* + * Other Windows architectures: compiler barrier only, matching the + * unknown-compiler fallback below. + */ + _ReadWriteBarrier(); +#endif + +#else + /* + * Unknown compiler: plain no-op. We have no portable way to emit a + * compiler barrier here, but the only caller (perform_spin_delay() in + * s_lock.c) has observable side effects, so the spin loop is not at risk + * of being optimized away. + */ + (void) 0; +#endif +} +#endif /* PG_HAVE_SPIN_DELAY */ + +/* + * Public spin-delay macro for the stdatomic path. + * + * This is the public pg_spin_delay() entry point used by the stdatomic + * spinlock path; it maps to the pg_spin_delay_impl() defined above. + */ +#ifndef pg_spin_delay +#define pg_spin_delay() pg_spin_delay_impl() +#endif + +/* + * Architectures where a plain load before the atomic exchange reduces + * cache-coherency traffic under spinlock contention (test-and-test-and-set). + * + * Restricted to x86/x86_64: these are TSO and pg_atomic_read_u32() (seq_cst) + * compiles to an ordinary load there, so the pre-read is genuinely cheap. On + * weakly-ordered targets a seq_cst load emits fence instructions, which would + * defeat the point of a cheap pre-check, so they are intentionally omitted. + */ +#if defined(__i386__) || defined(__x86_64__) || \ + defined(_M_IX86) || defined(_M_AMD64) +#define PG_SPIN_TRY_RELAXED +#endif + +#endif /* SPIN_DELAY_H */ diff --git a/src/include/port/spin_delay_status.h b/src/include/port/spin_delay_status.h new file mode 100644 index 00000000000..738ecb12a44 --- /dev/null +++ b/src/include/port/spin_delay_status.h @@ -0,0 +1,52 @@ +/*------------------------------------------------------------------------- + * + * spin_delay_status.h + * SpinDelayStatus type and the spin-delay/backoff helpers + * + * This header holds the delay/backoff bookkeeping used by the spinlock slow + * path (perform_spin_delay / finish_spin_delay in s_lock.c) and by other + * spinlock-like busy-wait loops. It is included by storage/spin.h so the + * definitions live in exactly one place. + * + * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/port/spin_delay_status.h + * + *------------------------------------------------------------------------- + */ +#ifndef SPIN_DELAY_STATUS_H +#define SPIN_DELAY_STATUS_H + +#define DEFAULT_SPINS_PER_DELAY 100 + +typedef struct +{ + int spins; + int delays; + int cur_delay; + const char *file; + int line; + const char *func; +} SpinDelayStatus; + +static inline void +init_spin_delay(SpinDelayStatus *status, + const char *file, int line, const char *func) +{ + status->spins = 0; + status->delays = 0; + status->cur_delay = 0; + status->file = file; + status->line = line; + status->func = func; +} + +#define init_local_spin_delay(status) init_spin_delay(status, __FILE__, __LINE__, __func__) + +extern void perform_spin_delay(SpinDelayStatus *status); +extern void finish_spin_delay(SpinDelayStatus *status); +extern void set_spins_per_delay(int shared_spins_per_delay); +extern int update_spins_per_delay(int shared_spins_per_delay); + +#endif /* SPIN_DELAY_STATUS_H */ diff --git a/src/include/storage/s_lock.h b/src/include/storage/s_lock.h deleted file mode 100644 index 7e049dfac18..00000000000 --- a/src/include/storage/s_lock.h +++ /dev/null @@ -1,734 +0,0 @@ -/*------------------------------------------------------------------------- - * - * s_lock.h - * Implementation of spinlocks. - * - * NOTE: none of the macros in this file are intended to be called directly. - * Call them through the macros in spin.h. - * - * The following hardware-dependent macros must be provided for each - * supported platform: - * - * void S_INIT_LOCK(slock_t *lock) - * Initialize a spinlock (to the unlocked state). - * - * int S_LOCK(slock_t *lock) - * Acquire a spinlock, waiting if necessary. - * Time out and abort() if unable to acquire the lock in a - * "reasonable" amount of time --- typically ~ 1 minute. - * Should return number of "delays"; see s_lock.c - * - * void S_UNLOCK(slock_t *lock) - * Unlock a previously acquired lock. - * - * void SPIN_DELAY(void) - * Delay operation to occur inside spinlock wait loop. - * - * Note to implementors: there are default implementations for all these - * macros at the bottom of the file. Check if your platform can use - * these or needs to override them. - * - * Usually, S_LOCK() is implemented in terms of even lower-level macros - * TAS() and TAS_SPIN(): - * - * int TAS(slock_t *lock) - * Atomic test-and-set instruction. Attempt to acquire the lock, - * but do *not* wait. Returns 0 if successful, nonzero if unable - * to acquire the lock. - * - * int TAS_SPIN(slock_t *lock) - * Like TAS(), but this version is used when waiting for a lock - * previously found to be contended. By default, this is the - * same as TAS(), but on some architectures it's better to poll a - * contended lock using an unlocked instruction and retry the - * atomic test-and-set only when it appears free. - * - * TAS() and TAS_SPIN() are NOT part of the API, and should never be called - * directly. A platform must provide either S_LOCK() or a TAS() for the - * default S_LOCK() to be built on. Currently, all supported platforms do - * the latter, so that is probably the best place to start if adding a new - * one. - * - * It is the responsibility of these macros to make sure that the compiler - * does not re-order accesses to shared memory to precede the actual lock - * acquisition, or follow the lock release. - * - * On platforms with weak memory ordering, the TAS(), TAS_SPIN(), and - * S_UNLOCK() macros must further include hardware-level memory fence - * instructions to prevent similar re-ordering at the hardware level. - * TAS() and TAS_SPIN() must guarantee that loads and stores issued after - * the macro are not executed until the lock has been obtained. Conversely, - * S_UNLOCK() must guarantee that loads and stores issued before the macro - * have been executed before the lock is released. - * - * On most supported platforms, TAS() uses a tas() function written - * in assembly language to execute a hardware atomic-test-and-set - * instruction. Equivalent compiler intrinsics are another popular option. - * - * - * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group - * Portions Copyright (c) 1994, Regents of the University of California - * - * src/include/storage/s_lock.h - * - *------------------------------------------------------------------------- - */ -#ifndef S_LOCK_H -#define S_LOCK_H - -#ifdef FRONTEND -#error "s_lock.h may not be included from frontend code" -#endif - -#if defined(__GNUC__) || defined(__INTEL_COMPILER) -/************************************************************************* - * All the gcc inlines - * Gcc consistently defines the CPU as __cpu__. - * Other compilers use __cpu or __cpu__ so we test for both in those cases. - */ - -/*---------- - * Standard gcc asm format (assuming "volatile slock_t *lock"): - - __asm__ __volatile__( - " instruction \n" - " instruction \n" - " instruction \n" -: "=r"(_res), "+m"(*lock) // return register, in/out lock value -: "r"(lock) // lock pointer, in input register -: "memory", "cc"); // show clobbered registers here - - * The output-operands list (after first colon) should always include - * "+m"(*lock), whether or not the asm code actually refers to this - * operand directly. This ensures that gcc believes the value in the - * lock variable is used and set by the asm code. Also, the clobbers - * list (after third colon) should always include "memory"; this prevents - * gcc from thinking it can cache the values of shared-memory fields - * across the asm code. Add "cc" if your asm code changes the condition - * code register, and also list any temp registers the code uses. - * - * If you need branch target labels within the asm block, include "%=" - * in the label names to make them distinct across multiple asm blocks - * within a source file. - *---------- - */ - - -#ifdef __i386__ /* 32-bit i386 */ - -typedef unsigned char slock_t; - -#define TAS(lock) tas(lock) - -static inline int -tas(volatile slock_t *lock) -{ - slock_t _res = 1; - - /* - * Use a non-locking test before asserting the bus lock. Note that the - * extra test appears to be a small loss on some x86 platforms and a small - * win on others; it's by no means clear that we should keep it. - * - * When this was last tested, we didn't have separate TAS() and TAS_SPIN() - * macros. Nowadays it probably would be better to do a non-locking test - * in TAS_SPIN() but not in TAS(), like on x86_64, but no-one's done the - * testing to verify that. Without some empirical evidence, better to - * leave it alone. - */ - __asm__ __volatile__( - " cmpb $0,%1 \n" - " jne TAS%=_out \n" - " lock \n" - " xchgb %0,%1 \n" - "TAS%=_out: \n" -: "+q"(_res), "+m"(*lock) -: /* no inputs */ -: "memory", "cc"); - return (int) _res; -} - -#define SPIN_DELAY() spin_delay() - -static inline void -spin_delay(void) -{ - /* - * This sequence is equivalent to the PAUSE instruction ("rep" is - * ignored by old IA32 processors if the following instruction is - * not a string operation); the IA-32 Architecture Software - * Developer's Manual, Vol. 3, Section 7.7.2 describes why using - * PAUSE in the inner loop of a spin lock is necessary for good - * performance: - * - * The PAUSE instruction improves the performance of IA-32 - * processors supporting Hyper-Threading Technology when - * executing spin-wait loops and other routines where one - * thread is accessing a shared lock or semaphore in a tight - * polling loop. When executing a spin-wait loop, the - * processor can suffer a severe performance penalty when - * exiting the loop because it detects a possible memory order - * violation and flushes the core processor's pipeline. The - * PAUSE instruction provides a hint to the processor that the - * code sequence is a spin-wait loop. The processor uses this - * hint to avoid the memory order violation and prevent the - * pipeline flush. In addition, the PAUSE instruction - * de-pipelines the spin-wait loop to prevent it from - * consuming execution resources excessively. - */ - __asm__ __volatile__( - " rep; nop \n"); -} - -#endif /* __i386__ */ - - -#ifdef __x86_64__ /* AMD Opteron, Intel EM64T */ - -typedef unsigned char slock_t; - -#define TAS(lock) tas(lock) - -/* - * On Intel EM64T, it's a win to use a non-locking test before the xchg proper, - * but only when spinning. - * - * See also Implementing Scalable Atomic Locks for Multi-Core Intel(tm) EM64T - * and IA32, by Michael Chynoweth and Mary R. Lee. As of this writing, it is - * available at: - * http://software.intel.com/en-us/articles/implementing-scalable-atomic-locks-for-multi-core-intel-em64t-and-ia32-architectures - */ -#define TAS_SPIN(lock) (*(lock) ? 1 : TAS(lock)) - -static inline int -tas(volatile slock_t *lock) -{ - slock_t _res = 1; - - __asm__ __volatile__( - " lock \n" - " xchgb %0,%1 \n" -: "+q"(_res), "+m"(*lock) -: /* no inputs */ -: "memory", "cc"); - return (int) _res; -} - -#define SPIN_DELAY() spin_delay() - -static inline void -spin_delay(void) -{ - /* - * Adding a PAUSE in the spin delay loop is demonstrably a no-op on - * Opteron, but it may be of some use on EM64T, so we keep it. - */ - __asm__ __volatile__( - " rep; nop \n"); -} - -#endif /* __x86_64__ */ - - -/* - * On ARM and ARM64, we use __sync_lock_test_and_set(int *, int) if available. - * - * We use the int-width variant of the builtin because it works on more chips - * than other widths. - */ -#if defined(__arm__) || defined(__aarch64__) -#ifdef HAVE_GCC__SYNC_INT32_TAS - -#define TAS(lock) tas(lock) - -typedef int slock_t; - -static inline int -tas(volatile slock_t *lock) -{ - return __sync_lock_test_and_set(lock, 1); -} - -#define S_UNLOCK(lock) __sync_lock_release(lock) - -#if defined(__aarch64__) - -/* - * On ARM64, it's a win to use a non-locking test before the TAS proper. It - * may be a win on 32-bit ARM, too, but nobody's tested it yet. - */ -#define TAS_SPIN(lock) (*(lock) ? 1 : TAS(lock)) - -#define SPIN_DELAY() spin_delay() - -static inline void -spin_delay(void) -{ - /* - * Using an ISB instruction to delay in spinlock loops appears beneficial - * on high-core-count ARM64 processors. It seems mostly a wash for smaller - * gear, and ISB doesn't exist at all on pre-v7 ARM chips. - */ - __asm__ __volatile__( - " isb; \n"); -} - -#endif /* __aarch64__ */ -#endif /* HAVE_GCC__SYNC_INT32_TAS */ -#endif /* __arm__ || __aarch64__ */ - - -/* S/390 and S/390x Linux (32- and 64-bit zSeries) */ -#if defined(__s390__) || defined(__s390x__) - -typedef unsigned int slock_t; - -#define TAS(lock) tas(lock) - -static inline int -tas(volatile slock_t *lock) -{ - int _res = 0; - - __asm__ __volatile__( - " cs %0,%3,0(%2) \n" -: "+d"(_res), "+m"(*lock) -: "a"(lock), "d"(1) -: "memory", "cc"); - return _res; -} - -#endif /* __s390__ || __s390x__ */ - - -#if defined(__sparc__) /* Sparc */ -/* - * Solaris has always run sparc processors in TSO (total store) mode, but - * linux didn't use to and the *BSDs still don't. So, be careful about - * acquire/release semantics. The CPU will treat superfluous members as - * NOPs, so it's just code space. - */ - -typedef unsigned char slock_t; - -#define TAS(lock) tas(lock) - -static inline int -tas(volatile slock_t *lock) -{ - slock_t _res; - - /* - * "cas" would be better than "ldstub", but it is only present on - * sparcv8plus and later, while some platforms still support sparcv7 or - * sparcv8. Also, "cas" requires that the system be running in TSO mode. - */ - __asm__ __volatile__( - " ldstub [%2], %0 \n" -: "=r"(_res), "+m"(*lock) -: "r"(lock) -: "memory"); -#if defined(__sparcv7) || defined(__sparc_v7__) - /* - * No stbar or membar available, luckily no actually produced hardware - * requires a barrier. - */ -#elif defined(__sparcv8) || defined(__sparc_v8__) - /* stbar is available (and required for both PSO, RMO), membar isn't */ - __asm__ __volatile__ ("stbar \n":::"memory"); -#else - /* - * #LoadStore (RMO) | #LoadLoad (RMO) together are the appropriate acquire - * barrier for sparcv8+ upwards. - */ - __asm__ __volatile__ ("membar #LoadStore | #LoadLoad \n":::"memory"); -#endif - return (int) _res; -} - -#if defined(__sparcv7) || defined(__sparc_v7__) -/* - * No stbar or membar available, luckily no actually produced hardware - * requires a barrier. We fall through to the default gcc definition of - * S_UNLOCK in this case. - */ -#elif defined(__sparcv8) || defined(__sparc_v8__) -/* stbar is available (and required for both PSO, RMO), membar isn't */ -#define S_UNLOCK(lock) \ -do \ -{ \ - __asm__ __volatile__ ("stbar \n":::"memory"); \ - *((volatile slock_t *) (lock)) = 0; \ -} while (0) -#else -/* - * #LoadStore (RMO) | #StoreStore (RMO, PSO) together are the appropriate - * release barrier for sparcv8+ upwards. - */ -#define S_UNLOCK(lock) \ -do \ -{ \ - __asm__ __volatile__ ("membar #LoadStore | #StoreStore \n":::"memory"); \ - *((volatile slock_t *) (lock)) = 0; \ -} while (0) -#endif - -#endif /* __sparc__ */ - - -/* PowerPC */ -#if defined(__powerpc__) || defined(__powerpc64__) - -typedef unsigned int slock_t; - -#define TAS(lock) tas(lock) - -/* On PPC, it's a win to use a non-locking test before the lwarx */ -#define TAS_SPIN(lock) (*(lock) ? 1 : TAS(lock)) - -/* - * The second operand of addi can hold a constant zero or a register number, - * hence constraint "=&b" to avoid allocating r0. "b" stands for "address - * base register"; most operands having this register-or-zero property are - * address bases, e.g. the second operand of lwax. - * - * NOTE: per the Enhanced PowerPC Architecture manual, v1.0 dated 7-May-2002, - * an isync is a sufficient synchronization barrier after a lwarx/stwcx loop. - * But if the spinlock is in ordinary memory, we can use lwsync instead for - * better performance. - */ -static inline int -tas(volatile slock_t *lock) -{ - slock_t _t; - int _res; - - __asm__ __volatile__( -" lwarx %0,0,%3,1 \n" -" cmpwi %0,0 \n" -" bne TAS%=_fail \n" -" addi %0,%0,1 \n" -" stwcx. %0,0,%3 \n" -" beq TAS%=_ok \n" -"TAS%=_fail: \n" -" li %1,1 \n" -" b TAS%=_out \n" -"TAS%=_ok: \n" -" lwsync \n" -" li %1,0 \n" -"TAS%=_out: \n" -: "=&b"(_t), "=r"(_res), "+m"(*lock) -: "r"(lock) -: "memory", "cc"); - return _res; -} - -/* - * PowerPC S_UNLOCK is almost standard but requires a "sync" instruction. - * But we can use lwsync instead for better performance. - */ -#define S_UNLOCK(lock) \ -do \ -{ \ - __asm__ __volatile__ (" lwsync \n" ::: "memory"); \ - *((volatile slock_t *) (lock)) = 0; \ -} while (0) - -#endif /* powerpc */ - - -#if defined(__mips__) && !defined(__sgi) /* non-SGI MIPS */ - -typedef unsigned int slock_t; - -#define TAS(lock) tas(lock) - -/* - * Original MIPS-I processors lacked the LL/SC instructions, but if we are - * so unfortunate as to be running on one of those, we expect that the kernel - * will handle the illegal-instruction traps and emulate them for us. On - * anything newer (and really, MIPS-I is extinct) LL/SC is the only sane - * choice because any other synchronization method must involve a kernel - * call. Unfortunately, many toolchains still default to MIPS-I as the - * codegen target; if the symbol __mips shows that that's the case, we - * have to force the assembler to accept LL/SC. - * - * R10000 and up processors require a separate SYNC, which has the same - * issues as LL/SC. - */ -#if __mips < 2 -#define MIPS_SET_MIPS2 " .set mips2 \n" -#else -#define MIPS_SET_MIPS2 -#endif - -static inline int -tas(volatile slock_t *lock) -{ - volatile slock_t *_l = lock; - int _res; - int _tmp; - - __asm__ __volatile__( - " .set push \n" - MIPS_SET_MIPS2 - " .set noreorder \n" - " .set nomacro \n" - " ll %0, %2 \n" - " or %1, %0, 1 \n" - " sc %1, %2 \n" - " xori %1, 1 \n" - " or %0, %0, %1 \n" - " sync \n" - " .set pop " -: "=&r" (_res), "=&r" (_tmp), "+R" (*_l) -: /* no inputs */ -: "memory"); - return _res; -} - -/* MIPS S_UNLOCK is almost standard but requires a "sync" instruction */ -#define S_UNLOCK(lock) \ -do \ -{ \ - __asm__ __volatile__( \ - " .set push \n" \ - MIPS_SET_MIPS2 \ - " .set noreorder \n" \ - " .set nomacro \n" \ - " sync \n" \ - " .set pop " \ -: /* no outputs */ \ -: /* no inputs */ \ -: "memory"); \ - *((volatile slock_t *) (lock)) = 0; \ -} while (0) - -#endif /* __mips__ && !__sgi */ - - - -/* - * If we have no platform-specific knowledge, but we found that the compiler - * provides __sync_lock_test_and_set(), use that. Prefer the int-width - * version over the char-width version if we have both, on the rather dubious - * grounds that that's known to be more likely to work in the ARM ecosystem. - * (But we dealt with ARM above.) - */ -#if !defined(TAS) - -#if defined(HAVE_GCC__SYNC_INT32_TAS) - -#define TAS(lock) tas(lock) - -typedef int slock_t; - -static inline int -tas(volatile slock_t *lock) -{ - return __sync_lock_test_and_set(lock, 1); -} - -#define S_UNLOCK(lock) __sync_lock_release(lock) - -#elif defined(HAVE_GCC__SYNC_CHAR_TAS) - -#define TAS(lock) tas(lock) - -typedef char slock_t; - -static inline int -tas(volatile slock_t *lock) -{ - return __sync_lock_test_and_set(lock, 1); -} - -#define S_UNLOCK(lock) __sync_lock_release(lock) - -#endif /* HAVE_GCC__SYNC_INT32_TAS */ - -#endif /* !defined(TAS) */ - - -/* - * Default implementation of S_UNLOCK() for gcc/icc. - * - * Note that this implementation is unsafe for any platform that can reorder - * a memory access (either load or store) after a following store. That - * happens not to be possible on x86 and most legacy architectures (some are - * single-processor!), but many modern systems have weaker memory ordering. - * Those that do must define their own version of S_UNLOCK() rather than - * relying on this one. - */ -#if !defined(S_UNLOCK) -#define S_UNLOCK(lock) \ - do { __asm__ __volatile__("" : : : "memory"); *(lock) = 0; } while (0) -#endif - -#endif /* defined(__GNUC__) || defined(__INTEL_COMPILER) */ - - -/* - * --------------------------------------------------------------------- - * Platforms that use non-gcc inline assembly: - * --------------------------------------------------------------------- - */ - -#if !defined(TAS) /* We didn't trigger above, let's try here */ - -#ifdef _MSC_VER -typedef LONG slock_t; - -#define TAS(lock) (InterlockedCompareExchange(lock, 1, 0)) - -#define SPIN_DELAY() spin_delay() - -#ifdef __aarch64__ -static __forceinline void -spin_delay(void) -{ - /* - * Research indicates ISB is better than __yield() on AArch64. See - * https://postgr.es/m/1c2a29b8-5b1e-44f7-a871-71ec5fefc120%40app.fastmail.com. - */ - __isb(_ARM64_BARRIER_SY); -} -#elif defined(_WIN64) -static __forceinline void -spin_delay(void) -{ - /* - * If using Visual C++ on Win64, inline assembly is unavailable. - * Use a _mm_pause intrinsic instead of rep nop. - */ - _mm_pause(); -} -#else -static __forceinline void -spin_delay(void) -{ - /* See comment for gcc code. Same code, MASM syntax */ - __asm rep nop; -} -#endif - -#include - -#ifdef __aarch64__ - -/* _ReadWriteBarrier() is insufficient on non-TSO architectures. */ -#pragma intrinsic(_InterlockedExchange) -#define S_UNLOCK(lock) _InterlockedExchange(lock, 0) - -#else - -#pragma intrinsic(_ReadWriteBarrier) -#define S_UNLOCK(lock) \ - do { _ReadWriteBarrier(); (*(lock)) = 0; } while (0) - -#endif -#endif - - -#endif /* !defined(TAS) */ - - -/* - * Default Definitions - override these above as needed. - */ - -/* - * Make sure S_LOCK is defined, either explicitly for the platform or via a TAS - * macro for the platform. - */ -#if !defined(S_LOCK) -#ifdef TAS -#define USE_DEFAULT_S_LOCK -extern int s_lock(volatile slock_t *lock, const char *file, int line, const char *func); -#define S_LOCK(lock) \ - (TAS(lock) ? s_lock((lock), __FILE__, __LINE__, __func__) : 0) -#else -#error PostgreSQL does not have spinlock support on this platform. Please report this to pgsql-bugs@lists.postgresql.org. -#endif /* TAS */ -#endif /* S_LOCK */ - -#if !defined(S_UNLOCK) -/* - * Our default implementation of S_UNLOCK is essentially *(lock) = 0. This - * is unsafe if the platform can reorder a memory access (either load or - * store) after a following store; platforms where this is possible must - * define their own S_UNLOCK. But CPU reordering is not the only concern: - * if we simply defined S_UNLOCK() as an inline macro, the compiler might - * reorder instructions from inside the critical section to occur after the - * lock release. Since the compiler probably can't know what the external - * function s_unlock is doing, putting the same logic there should be adequate. - * A sufficiently-smart globally optimizing compiler could break that - * assumption, though, and the cost of a function call for every spinlock - * release may hurt performance significantly, so we use this implementation - * only for platforms where we don't know of a suitable intrinsic. For the - * most part, those are relatively obscure platform/compiler combinations to - * which the PostgreSQL project does not have access. - */ -#define USE_DEFAULT_S_UNLOCK -extern void s_unlock(volatile slock_t *lock); -#define S_UNLOCK(lock) s_unlock(lock) -#endif /* S_UNLOCK */ - -#if !defined(S_INIT_LOCK) -#define S_INIT_LOCK(lock) S_UNLOCK(lock) -#endif /* S_INIT_LOCK */ - -#if !defined(SPIN_DELAY) -#define SPIN_DELAY() ((void) 0) -#endif /* SPIN_DELAY */ - -/* - * TAS_SPIN is only needed by the default S_LOCK's helper function (s_lock()), - * so we only provide a default when there is a TAS to base it on. - */ -#if !defined(TAS_SPIN) && defined(TAS) -#define TAS_SPIN(lock) TAS(lock) -#endif /* ! TAS_SPIN && TAS */ - - -/* - * Platform-independent out-of-line support routines - */ - -/* Support for dynamic adjustment of spins_per_delay */ -#define DEFAULT_SPINS_PER_DELAY 100 - -extern void set_spins_per_delay(int shared_spins_per_delay); -extern int update_spins_per_delay(int shared_spins_per_delay); - -/* - * Support for spin delay which is useful in various places where - * spinlock-like procedures take place. - */ -typedef struct -{ - int spins; - int delays; - int cur_delay; - const char *file; - int line; - const char *func; -} SpinDelayStatus; - -static inline void -init_spin_delay(SpinDelayStatus *status, - const char *file, int line, const char *func) -{ - status->spins = 0; - status->delays = 0; - status->cur_delay = 0; - status->file = file; - status->line = line; - status->func = func; -} - -#define init_local_spin_delay(status) init_spin_delay(status, __FILE__, __LINE__, __func__) -extern void perform_spin_delay(SpinDelayStatus *status); -extern void finish_spin_delay(SpinDelayStatus *status); - -#endif /* S_LOCK_H */ diff --git a/src/include/storage/spin.h b/src/include/storage/spin.h index 9cf6adb671a..5e10355032d 100644 --- a/src/include/storage/spin.h +++ b/src/include/storage/spin.h @@ -28,10 +28,9 @@ * for a CHECK_FOR_INTERRUPTS() to occur while holding a spinlock, and so * it is not necessary to do HOLD/RESUME_INTERRUPTS() in these functions. * - * These functions are implemented in terms of hardware-dependent macros - * supplied by s_lock.h. There is not currently any extra functionality - * added by this header, but there has been in the past and may someday - * be again. + * These functions are implemented on top of the C11 atomic + * operations in port/atomics.h; the contended-acquire slow path lives in + * s_lock() in s_lock.c. * * * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group @@ -44,24 +43,67 @@ #ifndef SPIN_H #define SPIN_H -#include "storage/s_lock.h" +/* + * Atomics-based spinlocks. + * + * Spinlocks are implemented directly on top of a plain 32-bit atomic rather + * than platform-specific TAS assembly. We deliberately do NOT use + * pg_atomic_flag: that type uses 1==unlocked so it can offer a relaxed + * "unlocked test", whereas a spinlock must be usable when its memory has + * merely been zeroed (a great deal of shared-memory state is set up with + * memset(...,0,...) and then relies on embedded spinlocks reading as free). + * So slock_t uses the traditional convention: 0 == unlocked, 1 == locked, + * which makes a zero-initialized slock_t a valid, free lock. + */ +#include "port/atomics.h" + +typedef pg_atomic_uint32 slock_t; + +/* SpinDelayStatus and the spin-delay helpers (perform/finish_spin_delay). */ +#include "port/spin_delay_status.h" + +extern int s_lock(volatile slock_t *lock, const char *file, int line, const char *func); static inline void SpinLockInit(volatile slock_t *lock) { - S_INIT_LOCK(lock); + pg_atomic_init_u32(lock, 0); /* 0 = unlocked */ } +/* + * SpinLockAcquire - acquire a spinlock, waiting if necessary. + * + * Fast path: a single atomic exchange (0->1). On failure fall into s_lock(), + * which does the backoff/retry and the "stuck spinlock" diagnostics. + * + * This is split into an inline helper plus a wrapper macro: the macro exists + * only to capture the call site's __FILE__/__LINE__/__func__ for diagnostics, + * while the helper evaluates its lock argument exactly once (a macro doing the + * exchange-then-s_lock inline would evaluate the argument twice) and preserves + * the volatile slock_t * type check. + */ static inline void -SpinLockAcquire(volatile slock_t *lock) +spin_lock_acquire(volatile slock_t *lock, const char *file, int line, + const char *func) { - S_LOCK(lock); + if (pg_atomic_exchange_u32(lock, 1) != 0) + s_lock(lock, file, line, func); } +#define SpinLockAcquire(lock) \ + spin_lock_acquire((lock), __FILE__, __LINE__, __func__) static inline void SpinLockRelease(volatile slock_t *lock) { - S_UNLOCK(lock); + /* + * Release the lock. A release store is exactly the right semantics and + * matches __sync_lock_release() in the traditional path: mov on x86, + * STLR on ARM64, fence rw,w + sw on rv64. (An earlier revision used + * pg_memory_barrier() followed by a relaxed store, which cost a full + * fence on every release on x86, where master emits only a mov.) + */ + atomic_store_explicit(lock, 0, memory_order_release); } + #endif /* SPIN_H */ diff --git a/src/test/regress/regress.c b/src/test/regress/regress.c index 3deb4ed7203..ba6afe2bc79 100644 --- a/src/test/regress/regress.c +++ b/src/test/regress/regress.c @@ -651,7 +651,8 @@ test_spinlock(void) * Basic tests for spinlocks, as well as the underlying operations. * * We embed the spinlock in a struct with other members to test that the - * spinlock operations don't perform too wide writes. + * spinlock operations don't perform too wide writes. (slock_t is a + * pg_atomic_uint32, i.e. 4 bytes.) */ { struct test_lock_struct @@ -669,32 +670,9 @@ test_spinlock(void) SpinLockAcquire(&struct_w_lock.lock); SpinLockRelease(&struct_w_lock.lock); - /* test basic operations via underlying S_* API */ - S_INIT_LOCK(&struct_w_lock.lock); - S_LOCK(&struct_w_lock.lock); - S_UNLOCK(&struct_w_lock.lock); - /* and that "contended" acquisition works */ - S_LOCK(&struct_w_lock.lock); - S_UNLOCK(&struct_w_lock.lock); - - /* - * Check, using TAS directly, that a single spin cycle doesn't block - * when acquiring an already acquired lock. - */ -#ifdef TAS - S_LOCK(&struct_w_lock.lock); - - if (!TAS(&struct_w_lock.lock)) - elog(ERROR, "acquired already held spinlock"); - -#ifdef TAS_SPIN - if (!TAS_SPIN(&struct_w_lock.lock)) - elog(ERROR, "acquired already held spinlock"); -#endif /* defined(TAS_SPIN) */ - - S_UNLOCK(&struct_w_lock.lock); -#endif /* defined(TAS) */ + s_lock(&struct_w_lock.lock, "testfile", 17, "testfunc"); + SpinLockRelease(&struct_w_lock.lock); /* * Verify that after all of this the non-lock contents are still diff --git a/src/tools/pginclude/headerscheck b/src/tools/pginclude/headerscheck index 785d6f867ad..378100acf2c 100755 --- a/src/tools/pginclude/headerscheck +++ b/src/tools/pginclude/headerscheck @@ -107,16 +107,6 @@ do test "$f" = src/include/port/win32ntdll.h && continue test "$f" = src/port/pthread-win32.h && continue - # Likewise, these files are platform-specific, and the one - # relevant to our platform will be included by atomics.h. - test "$f" = src/include/port/atomics/arch-arm.h && continue - test "$f" = src/include/port/atomics/arch-ppc.h && continue - test "$f" = src/include/port/atomics/arch-x86.h && continue - test "$f" = src/include/port/atomics/fallback.h && continue - test "$f" = src/include/port/atomics/generic.h && continue - test "$f" = src/include/port/atomics/generic-gcc.h && continue - test "$f" = src/include/port/atomics/generic-msvc.h && continue - # sepgsql.h depends on headers that aren't there on most platforms. test "$f" = contrib/sepgsql/sepgsql.h && continue