From fc39412f51d2726c696eb866a3b43a483fc4f7ee Mon Sep 17 00:00:00 2001 From: Greg Burd Date: Sat, 11 Jul 2026 18:42:50 -0400 Subject: [PATCH v4 3/3] [NOT FOR MERGE... yet.] Remove traditional atomics; use C11 stdatomic.h exclusively This commit is included to show the intended long-term destination of the series, and to let reviewers and the buildfarm exercise the stdatomic-only end state -- but it is deliberately NOT proposed for commit yet. The plan is to ship the earlier patches (which add C11 as a *selectable* implementation alongside the traditional platform-specific code), let the stdatomic path prove itself in the field across the buildfarm and at least one release, and only THEN remove the traditional implementation via a commit like this one. Removing the battle-tested arch-*/generic-* code on day one is not being asked for here; keeping both paths until stdatomic has earned trust is. Everything below describes what this (eventual) removal commit does. This collapses the dual-path atomics into a single stdatomic-only implementation, mirroring the destination state of the series. Rather than selecting between C11 and the platform-specific implementations at build time, stdatomic.h is now the sole backend for PostgreSQL's atomic operations and USE_STDATOMIC_H is effectively always-on. The platform-specific implementation files are deleted outright: - src/include/port/atomics/arch-arm.h - src/include/port/atomics/arch-ppc.h - src/include/port/atomics/arch-x86.h - src/include/port/atomics/generic.h - src/include/port/atomics/generic-gcc.h - src/include/port/atomics/generic-msvc.h - src/include/port/atomics/fallback.h atomics.h drops its #ifdef USE_STDATOMIC_H / #else conditional and the now-dead includes of the arch-*/generic-*/fallback headers, keeping only the stdatomic_impl.h include and the common public-API layer of static inline pg_atomic_* wrappers. storage/spin.h and s_lock.c lose their traditional-spinlock (#else) branches, so only the pg_atomic_flag-based spinlock path remains; the s_lock.h includes that served only that path are removed. Detection is retained but now hard-errors when a working C11 stdatomic.h is unavailable, in both meson.build and configure.ac, instead of silently falling back. The only remaining hand-written platform hint is the CPU spin-delay instruction in src/include/port/spin_delay.h, which stdatomic.h does not cover; it is intentionally left in place along with spin_delay_status.h and stdatomic_impl.h. This depends on the prior commits in the series (build-time detection, stdatomic_impl.h, conditional wire-up, documentation, and the test_atomics module). Co-authored-by: Greg Burd Co-authored-by: Thomas Munro --- configure | 80 +-- configure.ac | 79 +-- meson.build | 45 +- meson_options.txt | 5 - src/backend/storage/lmgr/.gitignore | 1 - src/backend/storage/lmgr/Makefile | 9 - src/backend/storage/lmgr/README | 59 +- src/backend/storage/lmgr/s_lock.c | 119 ---- src/include/port/atomics.h | 173 +---- src/include/port/atomics/arch-arm.h | 32 - src/include/port/atomics/arch-ppc.h | 256 -------- src/include/port/atomics/arch-x86.h | 189 ------ src/include/port/atomics/fallback.h | 42 -- src/include/port/atomics/generic-gcc.h | 326 ---------- src/include/port/atomics/generic-msvc.h | 115 ---- src/include/port/atomics/generic.h | 430 ------------ src/include/port/atomics/stdatomic_impl.h | 16 +- src/include/storage/s_lock.h | 753 ---------------------- src/include/storage/spin.h | 49 +- src/test/regress/regress.c | 38 +- src/tools/pginclude/headerscheck | 10 - 21 files changed, 123 insertions(+), 2703 deletions(-) delete mode 100644 src/include/port/atomics/arch-arm.h delete mode 100644 src/include/port/atomics/arch-ppc.h delete mode 100644 src/include/port/atomics/arch-x86.h delete mode 100644 src/include/port/atomics/fallback.h delete mode 100644 src/include/port/atomics/generic-gcc.h delete mode 100644 src/include/port/atomics/generic-msvc.h delete mode 100644 src/include/port/atomics/generic.h delete mode 100644 src/include/storage/s_lock.h diff --git a/configure b/configure index f9853829bd3..8e0f521e247 100755 --- a/configure +++ b/configure @@ -759,7 +759,6 @@ LLVM_LIBS CLANG LLVM_CONFIG AWK -with_stdatomic with_llvm have_cxx ac_ct_CXX @@ -858,7 +857,6 @@ with_segsize with_segsize_blocks with_wal_blocksize with_llvm -with_stdatomic enable_depend enable_cassert with_icu @@ -1578,8 +1576,6 @@ Optional Packages: --with-wal-blocksize=BLOCKSIZE set WAL block size in kB [8] --with-llvm build with LLVM based JIT support - --with-stdatomic[=yes/no/auto] - use C11 stdatomic.h for atomic operations [auto] --without-icu build without ICU support --with-tcl build Tcl modules (PL/Tcl) --with-tclconfig=DIR tclConfig.sh is in DIR @@ -4925,22 +4921,9 @@ fi # # C11 stdatomic.h # -# Accepts --with-stdatomic={yes,no,auto}. "auto" (the default) uses -# stdatomic.h if a compile test succeeds; "yes" requires it; "no" forces the -# traditional platform-specific atomics. The actual detection runs later, -# after the compiler and 64-bit integer handling have been configured. - -# Check whether --with-stdatomic was given. -if test "${with_stdatomic+set}" = set; then : - withval=$with_stdatomic; case $withval in - yes | no | auto) ;; - *) as_fn_error $? "invalid argument to --with-stdatomic; use yes, no, or auto" "$LINENO" 5 ;; - esac -else - with_stdatomic=no -fi - - +# stdatomic.h is the sole atomics implementation. The actual detection runs +# later, after the compiler and 64-bit integer handling have been configured; +# a missing or broken header is a hard error there. for ac_prog in gawk mawk nawk awk do # Extract the first word of "$ac_prog", so it can be a program name with args. @@ -17742,32 +17725,30 @@ $as_echo "#define HAVE_GCC__ATOMIC_INT64_CAS 1" >>confdefs.h fi -# Decide whether to use C11 stdatomic.h (see --with-stdatomic above). The -# probe mirrors the one used by the meson build: it must compile 32-bit -# fetch_add, a seq_cst thread fence, and a 64-bit compare-exchange. -if test "$with_stdatomic" != no; then - # Use a link test, not a compile test: 64-bit atomic_compare_exchange_strong() - # can lower to a libatomic call (__atomic_compare_exchange_8) on some 32-bit - # or weakly-aligned targets. A compile-only check would pass there and then - # fail at final link. Retry with -latomic if the bare link fails. - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for working C11 stdatomic.h" >&5 +# Require a working C11 stdatomic.h: it is the sole atomics implementation, +# so a missing or broken header is a hard error. Use a link test, not a +# compile test: 64-bit atomic_compare_exchange_strong() can lower to a +# libatomic call (__atomic_compare_exchange_8) on some 32-bit or +# weakly-aligned targets, where a compile-only check would pass and then fail +# at final link. Retry with -latomic if the bare link fails. +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for working C11 stdatomic.h" >&5 $as_echo_n "checking for working C11 stdatomic.h... " >&6; } if ${pgac_cv_stdatomic+:} false; then : $as_echo_n "(cached) " >&6 else pgac_cv_stdatomic=no - cat confdefs.h - <<_ACEOF >conftest.$ac_ext + cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include int main () { _Atomic int x = 0; - atomic_fetch_add(&x, 1); - atomic_thread_fence(memory_order_seq_cst); - _Atomic(unsigned long long) y = 0; - unsigned long long expected = 0; - atomic_compare_exchange_strong(&y, &expected, 42); + atomic_fetch_add(&x, 1); + atomic_thread_fence(memory_order_seq_cst); + _Atomic(unsigned long long) y = 0; + unsigned long long expected = 0; + atomic_compare_exchange_strong(&y, &expected, 42); ; return 0; @@ -17777,19 +17758,19 @@ if ac_fn_c_try_link "$LINENO"; then : pgac_cv_stdatomic=yes else pgac_save_LIBS=$LIBS - LIBS="$LIBS -latomic" - cat confdefs.h - <<_ACEOF >conftest.$ac_ext + LIBS="$LIBS -latomic" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include int main () { _Atomic int x = 0; - atomic_fetch_add(&x, 1); - atomic_thread_fence(memory_order_seq_cst); - _Atomic(unsigned long long) y = 0; - unsigned long long expected = 0; - atomic_compare_exchange_strong(&y, &expected, 42); + atomic_fetch_add(&x, 1); + atomic_thread_fence(memory_order_seq_cst); + _Atomic(unsigned long long) y = 0; + unsigned long long expected = 0; + atomic_compare_exchange_strong(&y, &expected, 42); ; return 0; @@ -17802,28 +17783,23 @@ else fi rm -f core conftest.err conftest.$ac_objext \ conftest$ac_exeext conftest.$ac_ext - LIBS=$pgac_save_LIBS + LIBS=$pgac_save_LIBS fi rm -f core conftest.err conftest.$ac_objext \ conftest$ac_exeext conftest.$ac_ext fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $pgac_cv_stdatomic" >&5 $as_echo "$pgac_cv_stdatomic" >&6; } -else - pgac_cv_stdatomic=no -fi -if test "$with_stdatomic" = yes && test "$pgac_cv_stdatomic" = no; then - as_fn_error $? "--with-stdatomic=yes was given, but a working C11 stdatomic.h could not be found" "$LINENO" 5 +if test "$pgac_cv_stdatomic" = no; then + as_fn_error $? "a working C11 stdatomic.h is required (it is the only atomics implementation)" "$LINENO" 5 fi -if test "$pgac_cv_stdatomic" != no; then $as_echo "#define USE_STDATOMIC_H 1" >>confdefs.h - if test "$pgac_cv_stdatomic" = 'yes, with -latomic'; then - LIBS="$LIBS -latomic" - fi +if test "$pgac_cv_stdatomic" = 'yes, with -latomic'; then + LIBS="$LIBS -latomic" fi diff --git a/configure.ac b/configure.ac index a3d4e31a609..da7a6e1014d 100644 --- a/configure.ac +++ b/configure.ac @@ -447,19 +447,9 @@ AC_SUBST(with_llvm) # # C11 stdatomic.h # -# Accepts --with-stdatomic={yes,no,auto}. "auto" (the default) uses -# stdatomic.h if a compile test succeeds; "yes" requires it; "no" forces the -# traditional platform-specific atomics. The actual detection runs later, -# after the compiler and 64-bit integer handling have been configured. -AC_ARG_WITH(stdatomic, - [AS_HELP_STRING([--with-stdatomic@<:@=yes/no/auto@:>@], - [use C11 stdatomic.h for atomic operations @<:@auto@:>@])], - [case $withval in - yes | no | auto) ;; - *) AC_MSG_ERROR([invalid argument to --with-stdatomic; use yes, no, or auto]) ;; - esac], - [with_stdatomic=no]) -AC_SUBST(with_stdatomic) +# stdatomic.h is the sole atomics implementation. The actual detection runs +# later, after the compiler and 64-bit integer handling have been configured; +# a missing or broken header is a hard error there. dnl must use AS_IF here, else AC_REQUIRES inside PGAC_LLVM_SUPPORT malfunctions AS_IF([test "$with_llvm" = yes], [ PGAC_LLVM_SUPPORT() @@ -2118,16 +2108,25 @@ PGAC_HAVE_GCC__ATOMIC_INT32_CAS PGAC_HAVE_GCC__ATOMIC_INT64_CAS -# Decide whether to use C11 stdatomic.h (see --with-stdatomic above). The -# probe mirrors the one used by the meson build: it must compile 32-bit -# fetch_add, a seq_cst thread fence, and a 64-bit compare-exchange. -if test "$with_stdatomic" != no; then - # Use a link test, not a compile test: 64-bit atomic_compare_exchange_strong() - # can lower to a libatomic call (__atomic_compare_exchange_8) on some 32-bit - # or weakly-aligned targets. A compile-only check would pass there and then - # fail at final link. Retry with -latomic if the bare link fails. - AC_CACHE_CHECK([for working C11 stdatomic.h], [pgac_cv_stdatomic], - [pgac_cv_stdatomic=no +# Require a working C11 stdatomic.h: it is the sole atomics implementation, +# so a missing or broken header is a hard error. Use a link test, not a +# compile test: 64-bit atomic_compare_exchange_strong() can lower to a +# libatomic call (__atomic_compare_exchange_8) on some 32-bit or +# weakly-aligned targets, where a compile-only check would pass and then fail +# at final link. Retry with -latomic if the bare link fails. +AC_CACHE_CHECK([for working C11 stdatomic.h], [pgac_cv_stdatomic], +[pgac_cv_stdatomic=no + AC_LINK_IFELSE([AC_LANG_PROGRAM([#include ], + [[_Atomic int x = 0; + atomic_fetch_add(&x, 1); + atomic_thread_fence(memory_order_seq_cst); + _Atomic(unsigned long long) y = 0; + unsigned long long expected = 0; + atomic_compare_exchange_strong(&y, &expected, 42); + ]])], + [pgac_cv_stdatomic=yes], + [pgac_save_LIBS=$LIBS + LIBS="$LIBS -latomic" AC_LINK_IFELSE([AC_LANG_PROGRAM([#include ], [[_Atomic int x = 0; atomic_fetch_add(&x, 1); @@ -2136,34 +2135,18 @@ if test "$with_stdatomic" != no; then unsigned long long expected = 0; atomic_compare_exchange_strong(&y, &expected, 42); ]])], - [pgac_cv_stdatomic=yes], - [pgac_save_LIBS=$LIBS - LIBS="$LIBS -latomic" - AC_LINK_IFELSE([AC_LANG_PROGRAM([#include ], - [[_Atomic int x = 0; - atomic_fetch_add(&x, 1); - atomic_thread_fence(memory_order_seq_cst); - _Atomic(unsigned long long) y = 0; - unsigned long long expected = 0; - atomic_compare_exchange_strong(&y, &expected, 42); - ]])], - [pgac_cv_stdatomic='yes, with -latomic'], - [pgac_cv_stdatomic=no]) - LIBS=$pgac_save_LIBS])]) -else - pgac_cv_stdatomic=no -fi + [pgac_cv_stdatomic='yes, with -latomic'], + [pgac_cv_stdatomic=no]) + LIBS=$pgac_save_LIBS])]) -if test "$with_stdatomic" = yes && test "$pgac_cv_stdatomic" = no; then - AC_MSG_ERROR([--with-stdatomic=yes was given, but a working C11 stdatomic.h could not be found]) +if test "$pgac_cv_stdatomic" = no; then + AC_MSG_ERROR([a working C11 stdatomic.h is required (it is the only atomics implementation)]) fi -if test "$pgac_cv_stdatomic" != no; then - AC_DEFINE([USE_STDATOMIC_H], 1, - [Define to 1 to use C11 stdatomic.h for atomic operations. (--with-stdatomic)]) - if test "$pgac_cv_stdatomic" = 'yes, with -latomic'; then - LIBS="$LIBS -latomic" - fi +AC_DEFINE([USE_STDATOMIC_H], 1, + [Define to 1 to use C11 stdatomic.h for atomic operations.]) +if test "$pgac_cv_stdatomic" = 'yes, with -latomic'; then + LIBS="$LIBS -latomic" fi diff --git a/meson.build b/meson.build index f4e72386deb..9132df40b16 100644 --- a/meson.build +++ b/meson.build @@ -692,7 +692,6 @@ int main(void) { ''' has_stdatomic = false -use_stdatomic = get_option('use_stdatomic') atomic_link_args = [] # The stdatomic probe below is a *link* test, not just a compile test: 64-bit @@ -744,36 +743,20 @@ else endif endif -# Process the use_stdatomic option -if use_stdatomic == 'yes' - if not has_stdatomic - error('stdatomic.h was requested with -Duse_stdatomic=yes but is not available or not working') - endif - cdata.set('USE_STDATOMIC_H', 1) - message('Using C11 stdatomic.h for atomic operations (forced by -Duse_stdatomic=yes)') - if cc.get_id() == 'msvc' - # MSVC needs C11 mode plus the experimental atomics switch for . - cflags += ['/std:c11', '/experimental:c11atomics'] - endif - if atomic_link_args.length() > 0 - os_deps += cc.find_library('atomic') - endif -elif use_stdatomic == 'no' - message('Using traditional platform-specific atomics (forced by -Duse_stdatomic=no)') - # Do not set USE_STDATOMIC_H -elif use_stdatomic == 'auto' - if has_stdatomic - cdata.set('USE_STDATOMIC_H', 1) - message('Using C11 stdatomic.h for atomic operations (auto-detected)') - if cc.get_id() == 'msvc' - cflags += ['/std:c11', '/experimental:c11atomics'] - endif - if atomic_link_args.length() > 0 - os_deps += cc.find_library('atomic') - endif - else - message('Using traditional platform-specific atomics (stdatomic.h not available)') - endif +# stdatomic.h is the sole atomics implementation, so a missing or broken +# header is a hard error. +if not has_stdatomic + error('a working C11 stdatomic.h is required (it is the only atomics implementation)') +endif +cdata.set('USE_STDATOMIC_H', 1) +message('Using C11 stdatomic.h for atomic operations') +if atomic_link_args.length() > 0 + # libatomic is needed for atomics the compiler cannot inline on this target. + os_deps += cc.find_library('atomic') +endif +if cc.get_id() == 'msvc' + # MSVC needs C11 mode plus the experimental atomics switch for . + cflags += ['/std:c11', '/experimental:c11atomics'] endif postgres_inc = [include_directories(postgres_inc_d)] diff --git a/meson_options.txt b/meson_options.txt index f8c5ccc14e1..6a793f3e479 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -52,11 +52,6 @@ option('PG_TEST_EXTRA', type: 'string', value: '', option('PG_GIT_REVISION', type: 'string', value: 'HEAD', description: 'git revision to be packaged by pgdist target') -option('use_stdatomic', type: 'combo', - choices: ['auto', 'yes', 'no'], - value: 'no', - description: 'Use C11 stdatomic.h for atomic operations (auto, yes, no)') - # Compilation options diff --git a/src/backend/storage/lmgr/.gitignore b/src/backend/storage/lmgr/.gitignore index 8e5b734f152..209c8be7223 100644 --- a/src/backend/storage/lmgr/.gitignore +++ b/src/backend/storage/lmgr/.gitignore @@ -1,2 +1 @@ /lwlocknames.h -/s_lock_test diff --git a/src/backend/storage/lmgr/Makefile b/src/backend/storage/lmgr/Makefile index a5fbc24ddad..a60e70ef0e1 100644 --- a/src/backend/storage/lmgr/Makefile +++ b/src/backend/storage/lmgr/Makefile @@ -24,17 +24,8 @@ OBJS = \ include $(top_srcdir)/src/backend/common.mk -s_lock_test: s_lock.c $(top_builddir)/src/common/libpgcommon.a $(top_builddir)/src/port/libpgport.a - $(CC) $(CPPFLAGS) $(CFLAGS) -DS_LOCK_TEST=1 $(srcdir)/s_lock.c \ - -L $(top_builddir)/src/common -lpgcommon \ - -L $(top_builddir)/src/port -lpgport -lm -o s_lock_test - lwlocknames.h: ../../../include/storage/lwlocklist.h ../../utils/activity/wait_event_names.txt generate-lwlocknames.pl $(PERL) $(srcdir)/generate-lwlocknames.pl $^ -check: s_lock_test - ./s_lock_test - clean: - rm -f s_lock_test rm -f lwlocknames.h diff --git a/src/backend/storage/lmgr/README b/src/backend/storage/lmgr/README index 460b9559449..b55911396db 100644 --- a/src/backend/storage/lmgr/README +++ b/src/backend/storage/lmgr/README @@ -50,41 +50,28 @@ The rest of this README file discusses the regular lock manager in detail. Atomic Operations ================= -PostgreSQL's atomic operations (see src/include/port/atomics.h) can be -implemented using either C11 stdatomic.h or traditional platform-specific -code. The implementation is selected at build time via the USE_STDATOMIC_H -preprocessor definition: - -* C11 stdatomic.h implementation (USE_STDATOMIC_H defined): - Uses the standard C11 header for all atomic operations. - This provides better portability and potentially better compiler - optimizations. Available on: +PostgreSQL's atomic operations (see src/include/port/atomics.h) are +implemented on top of the standard C11 header, which is the +sole atomics implementation. A working stdatomic.h is required at build +time; the USE_STDATOMIC_H preprocessor symbol is always defined. Available +on: - MSVC 2022+ (requires /experimental:c11atomics flag) - GCC 4.9+ and Clang 3.1+ (no special flags needed) -* Traditional implementation (USE_STDATOMIC_H not defined): - Uses platform-specific implementations that have been battle-tested - in PostgreSQL for many years: - - Architecture-specific: arch-x86.h, arch-arm.h, arch-ppc.h - - Compiler intrinsics: generic-gcc.h, generic-msvc.h - - Fallback implementations: generic.h, fallback.h +There is no build option to select the atomics implementation: stdatomic.h +is probed unconditionally and a missing or non-working header is a hard +configure/build error, since there is no other implementation to fall back +to. -Both implementations provide the same public API and observable semantics. -The choice is made at build configuration time: +Several internal properties of the implementation are worth noting; none of +them affect the public API: - meson setup build -Duse_stdatomic=auto # Auto-detect (default) - meson setup build -Duse_stdatomic=yes # Force stdatomic.h - meson setup build -Duse_stdatomic=no # Force traditional - -The two implementations differ internally in several ways that do not affect -the public API: - -* Memory ordering. pg_atomic_read_u32/u64() use seq_cst ordering under - stdatomic.h; pg_atomic_write_u32/u64() use relaxed. The seq_cst read was +* Memory ordering. pg_atomic_read_u32/u64() use seq_cst ordering; + pg_atomic_write_u32/u64() use relaxed. The seq_cst read is required for correctness on weak-memory hardware (e.g. RISC-V), where the weaker relaxed and acquire orderings let a concurrent reader miss a tuple in a parallel hash join. The write stays relaxed: a plain atomic store - matches the traditional "no barrier" contract for pg_atomic_write and, on + matches the "no barrier" contract for pg_atomic_write and, on ARM64, avoids the store-release (STLR) serialization that would penalize contended writes to hot shared state. The pg_atomic_unlocked_write_* variant also uses relaxed ordering, consistent with its "no guarantees" @@ -96,23 +83,15 @@ the public API: the thread fence (atomic_thread_fence), because a bare thread fence orders only atomic accesses and would let the compiler reorder plain loads/stores across it, whereas PostgreSQL's barrier contract must order non-atomic - accesses too. This matches the traditional generic-gcc.h barriers. + accesses too. -* Spinlock flag polarity. The stdatomic.h pg_atomic_flag uses 1=unlocked, - 0=locked (fetch_and-based test-and-set), the inverse of the traditional - 0=unlocked, 1=locked (exchange-based). This is an internal detail with no +* Spinlock flag polarity. The pg_atomic_flag uses 1=unlocked, + 0=locked (fetch_and-based test-and-set). This is an internal detail with no external visibility beyond raw values seen in a debugger. -* Binary compatibility. The atomic types are _Atomic(...) under stdatomic.h - and volatile-struct under the traditional path. The two are not binary - compatible, so objects and shared memory cannot be mixed between builds; - the implementation is fixed at build time, so this is not a concern in - practice. - -* Platform reach. Under stdatomic.h, 64-bit atomics are available on 32-bit +* Platform reach. 64-bit atomics are available on 32-bit ARM (compiler/runtime may emulate them with locks), atomics are usable from - frontend programs, and the headers are C++-compatible. The traditional - path disables 64-bit atomics on ARM32 and is backend-only. + frontend programs, and the headers are C++-compatible. For more information on memory barriers and atomic operations, see src/backend/storage/lmgr/README.barrier. diff --git a/src/backend/storage/lmgr/s_lock.c b/src/backend/storage/lmgr/s_lock.c index b3a11fe9fc9..8caaa562ced 100644 --- a/src/backend/storage/lmgr/s_lock.c +++ b/src/backend/storage/lmgr/s_lock.c @@ -51,13 +51,9 @@ #include #include "common/pg_prng.h" -#ifdef USE_STDATOMIC_H #include "port/atomics.h" #include "port/spin_delay.h" #include "storage/spin.h" -#else -#include "storage/s_lock.h" -#endif #include "utils/wait_event.h" #define MIN_SPINS_PER_DELAY 10 @@ -66,17 +62,6 @@ #define MIN_DELAY_USEC 1000L #define MAX_DELAY_USEC 1000000L -#ifndef USE_STDATOMIC_H -#ifdef S_LOCK_TEST -/* - * These are needed by pgstat_report_wait_start in the standalone compile of - * s_lock_test. - */ -static uint32 local_my_wait_event_info; -uint32 *my_wait_event_info = &local_my_wait_event_info; -#endif -#endif /* !USE_STDATOMIC_H */ - static int spins_per_delay = DEFAULT_SPINS_PER_DELAY; @@ -88,15 +73,8 @@ s_lock_stuck(const char *file, int line, const char *func) { if (!func) func = "(unknown)"; -#if !defined(USE_STDATOMIC_H) && defined(S_LOCK_TEST) - fprintf(stderr, - "\nStuck spinlock detected at %s, %s:%d.\n", - func, file, line); - exit(1); -#else elog(PANIC, "stuck spinlock detected at %s, %s:%d", func, file, line); -#endif } /* @@ -109,7 +87,6 @@ s_lock(volatile slock_t *lock, const char *file, int line, const char *func) init_spin_delay(&delayStatus, file, line, func); -#ifdef USE_STDATOMIC_H for (;;) { bool try_to_set; @@ -129,28 +106,12 @@ s_lock(volatile slock_t *lock, const char *file, int line, const char *func) break; perform_spin_delay(&delayStatus); } -#else /* !USE_STDATOMIC_H */ - while (TAS_SPIN(lock)) - { - perform_spin_delay(&delayStatus); - } -#endif /* USE_STDATOMIC_H */ finish_spin_delay(&delayStatus); return delayStatus.delays; } -#ifndef USE_STDATOMIC_H -#ifdef USE_DEFAULT_S_UNLOCK -void -s_unlock(volatile slock_t *lock) -{ - *lock = 0; -} -#endif -#endif /* !USE_STDATOMIC_H */ - /* * Wait while spinning on a contended spinlock. */ @@ -158,11 +119,7 @@ void perform_spin_delay(SpinDelayStatus *status) { /* CPU-specific delay each time through the loop */ -#ifdef USE_STDATOMIC_H pg_spin_delay(); -#else - SPIN_DELAY(); -#endif /* Block the process every spins_per_delay tries */ if (++(status->spins) >= spins_per_delay) @@ -185,11 +142,6 @@ perform_spin_delay(SpinDelayStatus *status) pg_usleep(status->cur_delay); pgstat_report_wait_end(); -#if !defined(USE_STDATOMIC_H) && defined(S_LOCK_TEST) - fprintf(stdout, "*"); - fflush(stdout); -#endif - /* increase delay by a random fraction between 1X and 2X */ status->cur_delay += (int) (status->cur_delay * pg_prng_double(&pg_global_prng_state) + 0.5); @@ -265,74 +217,3 @@ update_spins_per_delay(int shared_spins_per_delay) */ return (shared_spins_per_delay * 15 + spins_per_delay) / 16; } - - -/*****************************************************************************/ -#ifndef USE_STDATOMIC_H -#if defined(S_LOCK_TEST) - -/* - * test program for verifying a port's spinlock support. - */ - -struct test_lock_struct -{ - char pad1; - slock_t lock; - char pad2; -}; - -volatile struct test_lock_struct test_lock; - -int -main() -{ - pg_prng_seed(&pg_global_prng_state, (uint64) time(NULL)); - - test_lock.pad1 = test_lock.pad2 = 0x44; - - S_INIT_LOCK(&test_lock.lock); - - if (test_lock.pad1 != 0x44 || test_lock.pad2 != 0x44) - { - printf("S_LOCK_TEST: failed, declared datatype is wrong size\n"); - return 1; - } - - S_LOCK(&test_lock.lock); - - if (test_lock.pad1 != 0x44 || test_lock.pad2 != 0x44) - { - printf("S_LOCK_TEST: failed, declared datatype is wrong size\n"); - return 1; - } - - S_UNLOCK(&test_lock.lock); - - if (test_lock.pad1 != 0x44 || test_lock.pad2 != 0x44) - { - printf("S_LOCK_TEST: failed, declared datatype is wrong size\n"); - return 1; - } - - S_LOCK(&test_lock.lock); - - if (test_lock.pad1 != 0x44 || test_lock.pad2 != 0x44) - { - printf("S_LOCK_TEST: failed, declared datatype is wrong size\n"); - return 1; - } - - printf("S_LOCK_TEST: this will print %d stars and then\n", NUM_DELAYS); - printf(" exit with a 'stuck spinlock' message\n"); - printf(" if S_LOCK() and TAS() are working.\n"); - fflush(stdout); - - s_lock(&test_lock.lock, __FILE__, __LINE__, __func__); - - printf("S_LOCK_TEST: failed, lock not locked\n"); - return 1; -} - -#endif /* S_LOCK_TEST */ -#endif /* !USE_STDATOMIC_H */ diff --git a/src/include/port/atomics.h b/src/include/port/atomics.h index 2058dbb9238..9533b1e42bf 100644 --- a/src/include/port/atomics.h +++ b/src/include/port/atomics.h @@ -7,35 +7,18 @@ * atomically and dealing with cache coherency. Used to implement locking * facilities and lockless algorithms/data structures. * - * IMPLEMENTATION SELECTION: + * IMPLEMENTATION: * - * PostgreSQL's atomic operations can be implemented using either: + * PostgreSQL's atomic operations are implemented on top of C11 + * (see stdatomic_impl.h). This standard header supplies all + * atomic types and operations, so no architecture-specific assembly or + * compiler intrinsics are required here. A working C11 stdatomic.h is a + * hard build requirement (USE_STDATOMIC_H is always defined); the former + * platform-specific implementations (arch-*.h, generic-*.h, fallback.h) + * have been removed. * - * 1. C11 stdatomic.h (when USE_STDATOMIC_H is defined) - * - Uses standard C11 for all atomic operations - * - Better portability and compiler optimizations - * - Requires MSVC 2022+ or GCC 4.9+/Clang 3.1+ - * - * 2. Traditional platform-specific implementations (when USE_STDATOMIC_H is not defined) - * - Uses battle-tested PostgreSQL implementations - * - Architecture-specific: arch-x86.h, arch-arm.h, arch-ppc.h - * - Compiler intrinsics: generic-gcc.h, generic-msvc.h - * - Fallback implementations: generic.h, fallback.h - * - * Both implementations provide identical public API and semantics. - * Selection is made at build time via -Duse_stdatomic=auto/yes/no. - * - * PORTING NOTES: - * - * To bring up postgres on a platform/compiler at the very least - * implementations for the following operations should be provided: - * * pg_compiler_barrier(), pg_write_barrier(), pg_read_barrier() - * * pg_atomic_compare_exchange_u32(), pg_atomic_fetch_add_u32() - * * pg_atomic_test_set_flag(), pg_atomic_init_flag(), pg_atomic_clear_flag() - * * PG_HAVE_8BYTE_SINGLE_COPY_ATOMICITY should be defined if appropriate. - * - * For new platforms, prefer using stdatomic.h if available, as it reduces - * maintenance burden and leverages compiler-provided implementations. + * The only remaining hand-written platform hint is the CPU spin-delay + * instruction in port/spin_delay.h, which stdatomic.h does not cover. * * Use higher level functionality (lwlocks, spinlocks, heavyweight locks) * whenever possible. Writing correct code using these facilities is hard. @@ -54,21 +37,10 @@ #define ATOMICS_H /* - * Frontend code restriction - * - * Traditionally, atomics.h could not be included from frontend code because - * the platform-specific implementations often relied on backend-only features. - * - * With the stdatomic.h implementation, this restriction is no longer necessary - * since stdatomic.h is a standard header. However, we keep the restriction for - * the traditional implementation path to maintain compatibility. + * With the stdatomic.h implementation there is no longer any restriction on + * including atomics.h from frontend code, since stdatomic.h is a standard + * header with no backend-only dependencies. */ -#ifdef FRONTEND -#ifndef USE_STDATOMIC_H -#error "atomics.h may not be included from frontend code (use -Duse_stdatomic=yes if atomics are needed)" -#endif -#endif - #define INSIDE_ATOMICS_H /* @@ -83,125 +55,14 @@ #include /* - * PostgreSQL atomics can be implemented using either C11 stdatomic.h - * or platform-specific implementations. The choice is made at build - * configuration time via the USE_STDATOMIC_H preprocessor definition. - * - * When USE_STDATOMIC_H is defined, we use C11 for all - * atomic operations. This provides better portability and potentially - * better compiler optimizations. - * - * When USE_STDATOMIC_H is not defined (default initially), we use - * traditional platform-specific implementations (arch-*.h, generic-*.h) - * that have been battle-tested in PostgreSQL for many years. - * - * Both paths provide identical public API and semantics. - */ - -#ifdef USE_STDATOMIC_H - -/* - * C11 stdatomic.h implementation path - * - * This uses the standard C11 header for atomic operations. - * The type definitions and implementations are in stdatomic_impl.h. + * All atomic types and _impl operations come from C11 , wrapped + * in stdatomic_impl.h. */ - #include "port/atomics/stdatomic_impl.h" /* - * The public API is provided by static inline functions in the common code - * section below. Those functions call _impl functions, which are provided by - * stdatomic_impl.h for this path. - */ - -#else /* !USE_STDATOMIC_H */ - -/* - * Traditional platform-specific implementation path - * - * This is the original PostgreSQL atomics implementation using - * architecture-specific headers and compiler intrinsics. - * - * First a set of architecture specific files is included. - * - * These files can provide the full set of atomics or can do pretty much - * nothing if all the compilers commonly used on these platforms provide - * usable generics. - * - * Don't add an inline assembly of the actual atomic operations if all the - * common implementations of your platform provide intrinsics. Intrinsics are - * much easier to understand and potentially support more architectures. - * - * It will often make sense to define memory barrier semantics here, since - * e.g. generic compiler intrinsics for x86 memory barriers can't know that - * postgres doesn't need x86 read/write barriers do anything more than a - * compiler barrier. - * - */ -#if defined(__arm__) || defined(__aarch64__) -#include "port/atomics/arch-arm.h" -#elif defined(__i386__) || defined(__x86_64__) -#include "port/atomics/arch-x86.h" -#elif defined(__powerpc__) || defined(__powerpc64__) -#include "port/atomics/arch-ppc.h" -#endif - -/* - * Compiler specific, but architecture independent implementations. - * - * Provide architecture independent implementations of the atomic - * facilities. At the very least compiler barriers should be provided, but a - * full implementation of - * * pg_compiler_barrier(), pg_write_barrier(), pg_read_barrier() - * * pg_atomic_compare_exchange_u32(), pg_atomic_fetch_add_u32() - * using compiler intrinsics are a good idea. - */ -/* - * gcc or compatible, including clang and icc. - */ -#if defined(__GNUC__) || defined(__INTEL_COMPILER) -#include "port/atomics/generic-gcc.h" -#elif defined(_MSC_VER) -#include "port/atomics/generic-msvc.h" -#else -/* Unknown compiler. */ -#endif - -/* Fail if we couldn't find implementations of required facilities. */ -#if !defined(PG_HAVE_ATOMIC_U32_SUPPORT) -#error "could not find an implementation of pg_atomic_uint32" -#endif -#if !defined(pg_compiler_barrier_impl) -#error "could not find an implementation of pg_compiler_barrier" -#endif -#if !defined(pg_memory_barrier_impl) -#error "could not find an implementation of pg_memory_barrier_impl" -#endif - - -/* - * Provide a spinlock-based implementation of the 64 bit variants, if - * necessary. - */ -#include "port/atomics/fallback.h" - -/* - * Provide additional operations using supported infrastructure. These are - * expected to be efficient if the underlying atomic operations are efficient. - */ -#include "port/atomics/generic.h" - -#endif /* USE_STDATOMIC_H */ - -/* - * Common code for both stdatomic.h and traditional implementations. - * - * The following definitions provide the public API that works identically - * regardless of which implementation is used. The static inline functions - * below call _impl functions which are provided by either stdatomic_impl.h - * (when USE_STDATOMIC_H is defined) or the traditional implementation files - * (generic.h, etc.) otherwise. + * The public API below is provided by static inline functions that call the + * _impl functions supplied by stdatomic_impl.h. */ /* diff --git a/src/include/port/atomics/arch-arm.h b/src/include/port/atomics/arch-arm.h deleted file mode 100644 index 90280c7b751..00000000000 --- a/src/include/port/atomics/arch-arm.h +++ /dev/null @@ -1,32 +0,0 @@ -/*------------------------------------------------------------------------- - * - * arch-arm.h - * Atomic operations considerations specific to ARM - * - * Portions Copyright (c) 2013-2026, PostgreSQL Global Development Group - * - * NOTES: - * - * src/include/port/atomics/arch-arm.h - * - *------------------------------------------------------------------------- - */ - -/* intentionally no include guards, should only be included by atomics.h */ -#ifndef INSIDE_ATOMICS_H -#error "should be included via atomics.h" -#endif - -/* - * 64 bit atomics on ARM32 are implemented using kernel fallbacks and thus - * might be slow, so disable entirely. On ARM64 that problem doesn't exist. - */ -#if !defined(__aarch64__) -#define PG_DISABLE_64_BIT_ATOMICS -#else -/* - * Architecture Reference Manual for ARMv8 states aligned read/write to/from - * general purpose register is atomic. - */ -#define PG_HAVE_8BYTE_SINGLE_COPY_ATOMICITY -#endif /* __aarch64__ */ diff --git a/src/include/port/atomics/arch-ppc.h b/src/include/port/atomics/arch-ppc.h deleted file mode 100644 index 65ced2c6d3a..00000000000 --- a/src/include/port/atomics/arch-ppc.h +++ /dev/null @@ -1,256 +0,0 @@ -/*------------------------------------------------------------------------- - * - * arch-ppc.h - * Atomic operations considerations specific to PowerPC - * - * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group - * Portions Copyright (c) 1994, Regents of the University of California - * - * NOTES: - * - * src/include/port/atomics/arch-ppc.h - * - *------------------------------------------------------------------------- - */ - -#if defined(__GNUC__) - -/* - * lwsync orders loads with respect to each other, and similarly with stores. - * But a load can be performed before a subsequent store, so sync must be used - * for a full memory barrier. - */ -#define pg_memory_barrier_impl() __asm__ __volatile__ ("sync" : : : "memory") -#define pg_read_barrier_impl() __asm__ __volatile__ ("lwsync" : : : "memory") -#define pg_write_barrier_impl() __asm__ __volatile__ ("lwsync" : : : "memory") -#endif - -#define PG_HAVE_ATOMIC_U32_SUPPORT -typedef struct pg_atomic_uint32 -{ - volatile uint32 value; -} pg_atomic_uint32; - -/* 64bit atomics are only supported in 64bit mode */ -#if SIZEOF_VOID_P >= 8 -#define PG_HAVE_ATOMIC_U64_SUPPORT -typedef struct pg_atomic_uint64 -{ - alignas(8) volatile uint64 value; -} pg_atomic_uint64; - -#endif - -/* - * This mimics gcc __atomic_compare_exchange_n(..., __ATOMIC_SEQ_CST), but - * code generation differs at the end. __atomic_compare_exchange_n(): - * 100: isync - * 104: mfcr r3 - * 108: rlwinm r3,r3,3,31,31 - * 10c: bne 120 <.eb+0x10> - * 110: clrldi r3,r3,63 - * 114: addi r1,r1,112 - * 118: blr - * 11c: nop - * 120: clrldi r3,r3,63 - * 124: stw r9,0(r4) - * 128: addi r1,r1,112 - * 12c: blr - * - * This: - * f0: isync - * f4: mfcr r9 - * f8: rldicl. r3,r9,35,63 - * fc: bne 104 <.eb> - * 100: stw r10,0(r4) - * 104: addi r1,r1,112 - * 108: blr - * - * This implementation may or may not have materially different performance. - * It's not exploiting the fact that cr0 still holds the relevant comparison - * bits, set during the __asm__. One could fix that by moving more code into - * the __asm__. (That would remove the freedom to eliminate dead stores when - * the caller ignores "expected", but few callers do.) - * - * Recognizing constant "newval" would be superfluous, because there's no - * immediate-operand version of stwcx. - */ -#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32 -static inline bool -pg_atomic_compare_exchange_u32_impl(volatile pg_atomic_uint32 *ptr, - uint32 *expected, uint32 newval) -{ - uint32 found; - uint32 condition_register; - bool ret; - -#ifdef HAVE_I_CONSTRAINT__BUILTIN_CONSTANT_P - if (__builtin_constant_p(*expected) && - (int32) *expected <= PG_INT16_MAX && - (int32) *expected >= PG_INT16_MIN) - __asm__ __volatile__( - " sync \n" - " lwarx %0,0,%5,1 \n" - " cmpwi %0,%3 \n" - " bne $+12 \n" /* branch to lwsync */ - " stwcx. %4,0,%5 \n" - " bne $-16 \n" /* branch to lwarx */ - " lwsync \n" - " mfcr %1 \n" -: "=&r"(found), "=r"(condition_register), "+m"(ptr->value) -: "i"(*expected), "r"(newval), "r"(&ptr->value) -: "memory", "cc"); - else -#endif - __asm__ __volatile__( - " sync \n" - " lwarx %0,0,%5,1 \n" - " cmpw %0,%3 \n" - " bne $+12 \n" /* branch to lwsync */ - " stwcx. %4,0,%5 \n" - " bne $-16 \n" /* branch to lwarx */ - " lwsync \n" - " mfcr %1 \n" -: "=&r"(found), "=r"(condition_register), "+m"(ptr->value) -: "r"(*expected), "r"(newval), "r"(&ptr->value) -: "memory", "cc"); - - ret = (condition_register >> 29) & 1; /* test eq bit of cr0 */ - if (!ret) - *expected = found; - return ret; -} - -/* - * This mirrors gcc __sync_fetch_and_add(). - * - * Like tas(), use constraint "=&b" to avoid allocating r0. - */ -#define PG_HAVE_ATOMIC_FETCH_ADD_U32 -static inline uint32 -pg_atomic_fetch_add_u32_impl(volatile pg_atomic_uint32 *ptr, int32 add_) -{ - uint32 _t; - uint32 res; - -#ifdef HAVE_I_CONSTRAINT__BUILTIN_CONSTANT_P - if (__builtin_constant_p(add_) && - add_ <= PG_INT16_MAX && add_ >= PG_INT16_MIN) - __asm__ __volatile__( - " sync \n" - " lwarx %1,0,%4,1 \n" - " addi %0,%1,%3 \n" - " stwcx. %0,0,%4 \n" - " bne $-12 \n" /* branch to lwarx */ - " lwsync \n" -: "=&r"(_t), "=&b"(res), "+m"(ptr->value) -: "i"(add_), "r"(&ptr->value) -: "memory", "cc"); - else -#endif - __asm__ __volatile__( - " sync \n" - " lwarx %1,0,%4,1 \n" - " add %0,%1,%3 \n" - " stwcx. %0,0,%4 \n" - " bne $-12 \n" /* branch to lwarx */ - " lwsync \n" -: "=&r"(_t), "=&r"(res), "+m"(ptr->value) -: "r"(add_), "r"(&ptr->value) -: "memory", "cc"); - - return res; -} - -#ifdef PG_HAVE_ATOMIC_U64_SUPPORT - -#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64 -static inline bool -pg_atomic_compare_exchange_u64_impl(volatile pg_atomic_uint64 *ptr, - uint64 *expected, uint64 newval) -{ - uint64 found; - uint32 condition_register; - bool ret; - - AssertPointerAlignment(expected, 8); - - /* Like u32, but s/lwarx/ldarx/; s/stwcx/stdcx/; s/cmpw/cmpd/ */ -#ifdef HAVE_I_CONSTRAINT__BUILTIN_CONSTANT_P - if (__builtin_constant_p(*expected) && - (int64) *expected <= PG_INT16_MAX && - (int64) *expected >= PG_INT16_MIN) - __asm__ __volatile__( - " sync \n" - " ldarx %0,0,%5,1 \n" - " cmpdi %0,%3 \n" - " bne $+12 \n" /* branch to lwsync */ - " stdcx. %4,0,%5 \n" - " bne $-16 \n" /* branch to ldarx */ - " lwsync \n" - " mfcr %1 \n" -: "=&r"(found), "=r"(condition_register), "+m"(ptr->value) -: "i"(*expected), "r"(newval), "r"(&ptr->value) -: "memory", "cc"); - else -#endif - __asm__ __volatile__( - " sync \n" - " ldarx %0,0,%5,1 \n" - " cmpd %0,%3 \n" - " bne $+12 \n" /* branch to lwsync */ - " stdcx. %4,0,%5 \n" - " bne $-16 \n" /* branch to ldarx */ - " lwsync \n" - " mfcr %1 \n" -: "=&r"(found), "=r"(condition_register), "+m"(ptr->value) -: "r"(*expected), "r"(newval), "r"(&ptr->value) -: "memory", "cc"); - - ret = (condition_register >> 29) & 1; /* test eq bit of cr0 */ - if (!ret) - *expected = found; - return ret; -} - -#define PG_HAVE_ATOMIC_FETCH_ADD_U64 -static inline uint64 -pg_atomic_fetch_add_u64_impl(volatile pg_atomic_uint64 *ptr, int64 add_) -{ - uint64 _t; - uint64 res; - - /* Like u32, but s/lwarx/ldarx/; s/stwcx/stdcx/ */ -#ifdef HAVE_I_CONSTRAINT__BUILTIN_CONSTANT_P - if (__builtin_constant_p(add_) && - add_ <= PG_INT16_MAX && add_ >= PG_INT16_MIN) - __asm__ __volatile__( - " sync \n" - " ldarx %1,0,%4,1 \n" - " addi %0,%1,%3 \n" - " stdcx. %0,0,%4 \n" - " bne $-12 \n" /* branch to ldarx */ - " lwsync \n" -: "=&r"(_t), "=&b"(res), "+m"(ptr->value) -: "i"(add_), "r"(&ptr->value) -: "memory", "cc"); - else -#endif - __asm__ __volatile__( - " sync \n" - " ldarx %1,0,%4,1 \n" - " add %0,%1,%3 \n" - " stdcx. %0,0,%4 \n" - " bne $-12 \n" /* branch to ldarx */ - " lwsync \n" -: "=&r"(_t), "=&r"(res), "+m"(ptr->value) -: "r"(add_), "r"(&ptr->value) -: "memory", "cc"); - - return res; -} - -#endif /* PG_HAVE_ATOMIC_U64_SUPPORT */ - -/* per architecture manual doubleword accesses have single copy atomicity */ -#define PG_HAVE_8BYTE_SINGLE_COPY_ATOMICITY diff --git a/src/include/port/atomics/arch-x86.h b/src/include/port/atomics/arch-x86.h deleted file mode 100644 index 231831dcf59..00000000000 --- a/src/include/port/atomics/arch-x86.h +++ /dev/null @@ -1,189 +0,0 @@ -/*------------------------------------------------------------------------- - * - * arch-x86.h - * Atomic operations considerations specific to intel x86 - * - * Note that we actually require a 486 upwards because the 386 doesn't have - * support for xadd and cmpxchg. Given that the 386 isn't supported anywhere - * anymore that's not much of a restriction luckily. - * - * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group - * Portions Copyright (c) 1994, Regents of the University of California - * - * NOTES: - * - * src/include/port/atomics/arch-x86.h - * - *------------------------------------------------------------------------- - */ - -/* - * Both 32 and 64 bit x86 do not allow loads to be reordered with other loads, - * or stores to be reordered with other stores, but a load can be performed - * before a subsequent store. - * - * Technically, some x86-ish chips support uncached memory access and/or - * special instructions that are weakly ordered. In those cases we'd need - * the read and write barriers to be lfence and sfence. But since we don't - * do those things, a compiler barrier should be enough. - * - * "lock; addl" has worked for longer than "mfence". It's also rumored to be - * faster in many scenarios. - */ - -#if defined(__GNUC__) || defined(__INTEL_COMPILER) -#if defined(__i386__) -#define pg_memory_barrier_impl() \ - __asm__ __volatile__ ("lock; addl $0,0(%%esp)" : : : "memory", "cc") -#elif defined(__x86_64__) -#define pg_memory_barrier_impl() \ - __asm__ __volatile__ ("lock; addl $0,0(%%rsp)" : : : "memory", "cc") -#endif -#endif /* defined(__GNUC__) || defined(__INTEL_COMPILER) */ - -#define pg_read_barrier_impl() pg_compiler_barrier_impl() -#define pg_write_barrier_impl() pg_compiler_barrier_impl() - -/* - * Provide implementation for atomics using inline assembly on x86 gcc. It's - * nice to support older gcc's and the compare/exchange implementation here is - * actually more efficient than the * __sync variant. - */ -#if defined(__GNUC__) || defined(__INTEL_COMPILER) - -#define PG_HAVE_ATOMIC_FLAG_SUPPORT -typedef struct pg_atomic_flag -{ - volatile char value; -} pg_atomic_flag; - -#define PG_HAVE_ATOMIC_U32_SUPPORT -typedef struct pg_atomic_uint32 -{ - volatile uint32 value; -} pg_atomic_uint32; - -/* - * It's too complicated to write inline asm for 64bit types on 32bit and the - * 486 can't do it anyway. - */ -#ifdef __x86_64__ -#define PG_HAVE_ATOMIC_U64_SUPPORT -typedef struct pg_atomic_uint64 -{ - /* alignment guaranteed due to being on a 64bit platform */ - volatile uint64 value; -} pg_atomic_uint64; -#endif /* __x86_64__ */ - -#define PG_HAVE_ATOMIC_TEST_SET_FLAG -static inline bool -pg_atomic_test_set_flag_impl(volatile pg_atomic_flag *ptr) -{ - char _res = 1; - - __asm__ __volatile__( - " lock \n" - " xchgb %0,%1 \n" -: "+q"(_res), "+m"(ptr->value) -: -: "memory"); - return _res == 0; -} - -#define PG_HAVE_ATOMIC_CLEAR_FLAG -static inline void -pg_atomic_clear_flag_impl(volatile pg_atomic_flag *ptr) -{ - /* - * On a TSO architecture like x86 it's sufficient to use a compiler - * barrier to achieve release semantics. - */ - __asm__ __volatile__("" ::: "memory"); - ptr->value = 0; -} - -#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32 -static inline bool -pg_atomic_compare_exchange_u32_impl(volatile pg_atomic_uint32 *ptr, - uint32 *expected, uint32 newval) -{ - char ret; - - /* - * Perform cmpxchg and use the zero flag which it implicitly sets when - * equal to measure the success. - */ - __asm__ __volatile__( - " lock \n" - " cmpxchgl %4,%5 \n" - " setz %2 \n" -: "=a" (*expected), "=m"(ptr->value), "=q" (ret) -: "a" (*expected), "r" (newval), "m"(ptr->value) -: "memory", "cc"); - return (bool) ret; -} - -#define PG_HAVE_ATOMIC_FETCH_ADD_U32 -static inline uint32 -pg_atomic_fetch_add_u32_impl(volatile pg_atomic_uint32 *ptr, int32 add_) -{ - uint32 res; - __asm__ __volatile__( - " lock \n" - " xaddl %0,%1 \n" -: "=q"(res), "=m"(ptr->value) -: "0" (add_), "m"(ptr->value) -: "memory", "cc"); - return res; -} - -#ifdef __x86_64__ - -#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64 -static inline bool -pg_atomic_compare_exchange_u64_impl(volatile pg_atomic_uint64 *ptr, - uint64 *expected, uint64 newval) -{ - char ret; - - AssertPointerAlignment(expected, 8); - - /* - * Perform cmpxchg and use the zero flag which it implicitly sets when - * equal to measure the success. - */ - __asm__ __volatile__( - " lock \n" - " cmpxchgq %4,%5 \n" - " setz %2 \n" -: "=a" (*expected), "=m"(ptr->value), "=q" (ret) -: "a" (*expected), "r" (newval), "m"(ptr->value) -: "memory", "cc"); - return (bool) ret; -} - -#define PG_HAVE_ATOMIC_FETCH_ADD_U64 -static inline uint64 -pg_atomic_fetch_add_u64_impl(volatile pg_atomic_uint64 *ptr, int64 add_) -{ - uint64 res; - __asm__ __volatile__( - " lock \n" - " xaddq %0,%1 \n" -: "=q"(res), "=m"(ptr->value) -: "0" (add_), "m"(ptr->value) -: "memory", "cc"); - return res; -} - -#endif /* __x86_64__ */ - -#endif /* defined(__GNUC__) || defined(__INTEL_COMPILER) */ - -/* - * 8 byte reads / writes have single-copy atomicity on all x86-64 cpus. - */ -#if defined(__x86_64__) -#define PG_HAVE_8BYTE_SINGLE_COPY_ATOMICITY -#endif /* 8 byte single-copy atomicity */ diff --git a/src/include/port/atomics/fallback.h b/src/include/port/atomics/fallback.h deleted file mode 100644 index f37225189fc..00000000000 --- a/src/include/port/atomics/fallback.h +++ /dev/null @@ -1,42 +0,0 @@ -/*------------------------------------------------------------------------- - * - * fallback.h - * Fallback for platforms without 64 bit atomics support. Slower - * than native atomics support, but not unusably slow. - * - * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group - * Portions Copyright (c) 1994, Regents of the University of California - * - * src/include/port/atomics/fallback.h - * - *------------------------------------------------------------------------- - */ - -/* intentionally no include guards, should only be included by atomics.h */ -#ifndef INSIDE_ATOMICS_H -# error "should be included via atomics.h" -#endif - - -#if !defined(PG_HAVE_ATOMIC_U64_SUPPORT) - -#define PG_HAVE_ATOMIC_U64_SIMULATION - -#define PG_HAVE_ATOMIC_U64_SUPPORT -typedef struct pg_atomic_uint64 -{ - int sema; - volatile uint64 value; -} pg_atomic_uint64; - -#define PG_HAVE_ATOMIC_INIT_U64 -extern void pg_atomic_init_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 val_); - -#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64 -extern bool pg_atomic_compare_exchange_u64_impl(volatile pg_atomic_uint64 *ptr, - uint64 *expected, uint64 newval); - -#define PG_HAVE_ATOMIC_FETCH_ADD_U64 -extern uint64 pg_atomic_fetch_add_u64_impl(volatile pg_atomic_uint64 *ptr, int64 add_); - -#endif /* PG_HAVE_ATOMIC_U64_SUPPORT */ diff --git a/src/include/port/atomics/generic-gcc.h b/src/include/port/atomics/generic-gcc.h deleted file mode 100644 index 5bfce82f687..00000000000 --- a/src/include/port/atomics/generic-gcc.h +++ /dev/null @@ -1,326 +0,0 @@ -/*------------------------------------------------------------------------- - * - * generic-gcc.h - * Atomic operations, implemented using gcc (or compatible) intrinsics. - * - * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group - * Portions Copyright (c) 1994, Regents of the University of California - * - * NOTES: - * - * Documentation: - * * Legacy __sync Built-in Functions for Atomic Memory Access - * https://gcc.gnu.org/onlinedocs/gcc-4.8.2/gcc/_005f_005fsync-Builtins.html - * * Built-in functions for memory model aware atomic operations - * https://gcc.gnu.org/onlinedocs/gcc-4.8.2/gcc/_005f_005fatomic-Builtins.html - * - * src/include/port/atomics/generic-gcc.h - * - *------------------------------------------------------------------------- - */ - -/* intentionally no include guards, should only be included by atomics.h */ -#ifndef INSIDE_ATOMICS_H -#error "should be included via atomics.h" -#endif - -/* - * An empty asm block should be a sufficient compiler barrier. - */ -#define pg_compiler_barrier_impl() __asm__ __volatile__("" ::: "memory") - -/* - * If we're on GCC, we should be able to get a memory barrier - * out of this compiler built-in. But we prefer to rely on platform specific - * definitions where possible, and use this only as a fallback. - */ -#if !defined(pg_memory_barrier_impl) -# if defined(HAVE_GCC__ATOMIC_INT32_CAS) -# define pg_memory_barrier_impl() __atomic_thread_fence(__ATOMIC_SEQ_CST) -# elif defined(__GNUC__) -# define pg_memory_barrier_impl() __sync_synchronize() -# endif -#endif /* !defined(pg_memory_barrier_impl) */ - -#if !defined(pg_read_barrier_impl) && defined(HAVE_GCC__ATOMIC_INT32_CAS) -/* acquire semantics include read barrier semantics */ -# define pg_read_barrier_impl() do \ -{ \ - pg_compiler_barrier_impl(); \ - __atomic_thread_fence(__ATOMIC_ACQUIRE); \ -} while (0) -#endif - -#if !defined(pg_write_barrier_impl) && defined(HAVE_GCC__ATOMIC_INT32_CAS) -/* release semantics include write barrier semantics */ -# define pg_write_barrier_impl() do \ -{ \ - pg_compiler_barrier_impl(); \ - __atomic_thread_fence(__ATOMIC_RELEASE); \ -} while (0) -#endif - - -/* generic gcc based atomic flag implementation */ -#if !defined(PG_HAVE_ATOMIC_FLAG_SUPPORT) \ - && (defined(HAVE_GCC__SYNC_INT32_TAS) || defined(HAVE_GCC__SYNC_CHAR_TAS)) - -#define PG_HAVE_ATOMIC_FLAG_SUPPORT -typedef struct pg_atomic_flag -{ - /* - * If we have a choice, use int-width TAS, because that is more efficient - * and/or more reliably implemented on most non-Intel platforms. (Note - * that this code isn't used on x86[_64]; see arch-x86.h for that.) - */ -#ifdef HAVE_GCC__SYNC_INT32_TAS - volatile int value; -#else - volatile char value; -#endif -} pg_atomic_flag; - -#endif /* !ATOMIC_FLAG_SUPPORT && SYNC_INT32_TAS */ - -/* generic gcc based atomic uint32 implementation */ -#if !defined(PG_HAVE_ATOMIC_U32_SUPPORT) \ - && (defined(HAVE_GCC__ATOMIC_INT32_CAS) || defined(HAVE_GCC__SYNC_INT32_CAS)) - -#define PG_HAVE_ATOMIC_U32_SUPPORT -typedef struct pg_atomic_uint32 -{ - volatile uint32 value; -} pg_atomic_uint32; - -#endif /* defined(HAVE_GCC__ATOMIC_INT32_CAS) || defined(HAVE_GCC__SYNC_INT32_CAS) */ - -/* generic gcc based atomic uint64 implementation */ -#if !defined(PG_HAVE_ATOMIC_U64_SUPPORT) \ - && !defined(PG_DISABLE_64_BIT_ATOMICS) \ - && (defined(HAVE_GCC__ATOMIC_INT64_CAS) || defined(HAVE_GCC__SYNC_INT64_CAS)) - -#define PG_HAVE_ATOMIC_U64_SUPPORT -typedef struct pg_atomic_uint64 -{ - alignas(8) volatile uint64 value; -} pg_atomic_uint64; - -#endif /* defined(HAVE_GCC__ATOMIC_INT64_CAS) || defined(HAVE_GCC__SYNC_INT64_CAS) */ - -#ifdef PG_HAVE_ATOMIC_FLAG_SUPPORT - -#if defined(HAVE_GCC__SYNC_CHAR_TAS) || defined(HAVE_GCC__SYNC_INT32_TAS) - -#ifndef PG_HAVE_ATOMIC_TEST_SET_FLAG -#define PG_HAVE_ATOMIC_TEST_SET_FLAG -static inline bool -pg_atomic_test_set_flag_impl(volatile pg_atomic_flag *ptr) -{ - /* NB: only an acquire barrier, not a full one */ - /* some platform only support a 1 here */ - return __sync_lock_test_and_set(&ptr->value, 1) == 0; -} -#endif - -#endif /* defined(HAVE_GCC__SYNC_*_TAS) */ - -#ifndef PG_HAVE_ATOMIC_UNLOCKED_TEST_FLAG -#define PG_HAVE_ATOMIC_UNLOCKED_TEST_FLAG -static inline bool -pg_atomic_unlocked_test_flag_impl(volatile pg_atomic_flag *ptr) -{ - return ptr->value == 0; -} -#endif - -#ifndef PG_HAVE_ATOMIC_CLEAR_FLAG -#define PG_HAVE_ATOMIC_CLEAR_FLAG -static inline void -pg_atomic_clear_flag_impl(volatile pg_atomic_flag *ptr) -{ - __sync_lock_release(&ptr->value); -} -#endif - -#ifndef PG_HAVE_ATOMIC_INIT_FLAG -#define PG_HAVE_ATOMIC_INIT_FLAG -static inline void -pg_atomic_init_flag_impl(volatile pg_atomic_flag *ptr) -{ - pg_atomic_clear_flag_impl(ptr); -} -#endif - -#endif /* defined(PG_HAVE_ATOMIC_FLAG_SUPPORT) */ - -/* prefer __atomic, it has a better API */ -#if !defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32) && defined(HAVE_GCC__ATOMIC_INT32_CAS) -#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32 -static inline bool -pg_atomic_compare_exchange_u32_impl(volatile pg_atomic_uint32 *ptr, - uint32 *expected, uint32 newval) -{ - /* FIXME: we can probably use a lower consistency model */ - return __atomic_compare_exchange_n(&ptr->value, expected, newval, false, - __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); -} -#endif - -#if !defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32) && defined(HAVE_GCC__SYNC_INT32_CAS) -#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32 -static inline bool -pg_atomic_compare_exchange_u32_impl(volatile pg_atomic_uint32 *ptr, - uint32 *expected, uint32 newval) -{ - bool ret; - uint32 current; - current = __sync_val_compare_and_swap(&ptr->value, *expected, newval); - ret = current == *expected; - *expected = current; - return ret; -} -#endif - -/* - * __sync_lock_test_and_set() only supports setting the value to 1 on some - * platforms, so we only provide an __atomic implementation for - * pg_atomic_exchange. - * - * We assume the availability of 32-bit __atomic_compare_exchange_n() implies - * the availability of 32-bit __atomic_exchange_n(). - */ -#if !defined(PG_HAVE_ATOMIC_EXCHANGE_U32) && defined(HAVE_GCC__ATOMIC_INT32_CAS) -#define PG_HAVE_ATOMIC_EXCHANGE_U32 -static inline uint32 -pg_atomic_exchange_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 newval) -{ - return __atomic_exchange_n(&ptr->value, newval, __ATOMIC_SEQ_CST); -} -#endif - -/* if we have 32-bit __sync_val_compare_and_swap, assume we have these too: */ - -#if !defined(PG_HAVE_ATOMIC_FETCH_ADD_U32) && defined(HAVE_GCC__SYNC_INT32_CAS) -#define PG_HAVE_ATOMIC_FETCH_ADD_U32 -static inline uint32 -pg_atomic_fetch_add_u32_impl(volatile pg_atomic_uint32 *ptr, int32 add_) -{ - return __sync_fetch_and_add(&ptr->value, add_); -} -#endif - -#if !defined(PG_HAVE_ATOMIC_FETCH_SUB_U32) && defined(HAVE_GCC__SYNC_INT32_CAS) -#define PG_HAVE_ATOMIC_FETCH_SUB_U32 -static inline uint32 -pg_atomic_fetch_sub_u32_impl(volatile pg_atomic_uint32 *ptr, int32 sub_) -{ - return __sync_fetch_and_sub(&ptr->value, sub_); -} -#endif - -#if !defined(PG_HAVE_ATOMIC_FETCH_AND_U32) && defined(HAVE_GCC__SYNC_INT32_CAS) -#define PG_HAVE_ATOMIC_FETCH_AND_U32 -static inline uint32 -pg_atomic_fetch_and_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 and_) -{ - return __sync_fetch_and_and(&ptr->value, and_); -} -#endif - -#if !defined(PG_HAVE_ATOMIC_FETCH_OR_U32) && defined(HAVE_GCC__SYNC_INT32_CAS) -#define PG_HAVE_ATOMIC_FETCH_OR_U32 -static inline uint32 -pg_atomic_fetch_or_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 or_) -{ - return __sync_fetch_and_or(&ptr->value, or_); -} -#endif - - -#if !defined(PG_DISABLE_64_BIT_ATOMICS) - -#if !defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64) && defined(HAVE_GCC__ATOMIC_INT64_CAS) -#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64 -static inline bool -pg_atomic_compare_exchange_u64_impl(volatile pg_atomic_uint64 *ptr, - uint64 *expected, uint64 newval) -{ - AssertPointerAlignment(expected, 8); - return __atomic_compare_exchange_n(&ptr->value, expected, newval, false, - __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); -} -#endif - -#if !defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64) && defined(HAVE_GCC__SYNC_INT64_CAS) -#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64 -static inline bool -pg_atomic_compare_exchange_u64_impl(volatile pg_atomic_uint64 *ptr, - uint64 *expected, uint64 newval) -{ - bool ret; - uint64 current; - - AssertPointerAlignment(expected, 8); - current = __sync_val_compare_and_swap(&ptr->value, *expected, newval); - ret = current == *expected; - *expected = current; - return ret; -} -#endif - -/* - * __sync_lock_test_and_set() only supports setting the value to 1 on some - * platforms, so we only provide an __atomic implementation for - * pg_atomic_exchange. - * - * We assume the availability of 64-bit __atomic_compare_exchange_n() implies - * the availability of 64-bit __atomic_exchange_n(). - */ -#if !defined(PG_HAVE_ATOMIC_EXCHANGE_U64) && defined(HAVE_GCC__ATOMIC_INT64_CAS) -#define PG_HAVE_ATOMIC_EXCHANGE_U64 -static inline uint64 -pg_atomic_exchange_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 newval) -{ - return __atomic_exchange_n(&ptr->value, newval, __ATOMIC_SEQ_CST); -} -#endif - -/* if we have 64-bit __sync_val_compare_and_swap, assume we have these too: */ - -#if !defined(PG_HAVE_ATOMIC_FETCH_ADD_U64) && defined(HAVE_GCC__SYNC_INT64_CAS) -#define PG_HAVE_ATOMIC_FETCH_ADD_U64 -static inline uint64 -pg_atomic_fetch_add_u64_impl(volatile pg_atomic_uint64 *ptr, int64 add_) -{ - return __sync_fetch_and_add(&ptr->value, add_); -} -#endif - -#if !defined(PG_HAVE_ATOMIC_FETCH_SUB_U64) && defined(HAVE_GCC__SYNC_INT64_CAS) -#define PG_HAVE_ATOMIC_FETCH_SUB_U64 -static inline uint64 -pg_atomic_fetch_sub_u64_impl(volatile pg_atomic_uint64 *ptr, int64 sub_) -{ - return __sync_fetch_and_sub(&ptr->value, sub_); -} -#endif - -#if !defined(PG_HAVE_ATOMIC_FETCH_AND_U64) && defined(HAVE_GCC__SYNC_INT64_CAS) -#define PG_HAVE_ATOMIC_FETCH_AND_U64 -static inline uint64 -pg_atomic_fetch_and_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 and_) -{ - return __sync_fetch_and_and(&ptr->value, and_); -} -#endif - -#if !defined(PG_HAVE_ATOMIC_FETCH_OR_U64) && defined(HAVE_GCC__SYNC_INT64_CAS) -#define PG_HAVE_ATOMIC_FETCH_OR_U64 -static inline uint64 -pg_atomic_fetch_or_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 or_) -{ - return __sync_fetch_and_or(&ptr->value, or_); -} -#endif - -#endif /* !defined(PG_DISABLE_64_BIT_ATOMICS) */ diff --git a/src/include/port/atomics/generic-msvc.h b/src/include/port/atomics/generic-msvc.h deleted file mode 100644 index 6d09cd0e043..00000000000 --- a/src/include/port/atomics/generic-msvc.h +++ /dev/null @@ -1,115 +0,0 @@ -/*------------------------------------------------------------------------- - * - * generic-msvc.h - * Atomic operations support when using MSVC - * - * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group - * Portions Copyright (c) 1994, Regents of the University of California - * - * NOTES: - * - * Documentation: - * * Interlocked Variable Access - * http://msdn.microsoft.com/en-us/library/ms684122%28VS.85%29.aspx - * - * src/include/port/atomics/generic-msvc.h - * - *------------------------------------------------------------------------- - */ -#include - -/* intentionally no include guards, should only be included by atomics.h */ -#ifndef INSIDE_ATOMICS_H -#error "should be included via atomics.h" -#endif - -#pragma intrinsic(_ReadWriteBarrier) -#define pg_compiler_barrier_impl() _ReadWriteBarrier() - -#ifndef pg_memory_barrier_impl -#define pg_memory_barrier_impl() MemoryBarrier() -#endif - -#define PG_HAVE_ATOMIC_U32_SUPPORT -typedef struct pg_atomic_uint32 -{ - volatile uint32 value; -} pg_atomic_uint32; - -#define PG_HAVE_ATOMIC_U64_SUPPORT -typedef struct pg_atomic_uint64 -{ - alignas(8) volatile uint64 value; -} pg_atomic_uint64; - - -#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32 -static inline bool -pg_atomic_compare_exchange_u32_impl(volatile pg_atomic_uint32 *ptr, - uint32 *expected, uint32 newval) -{ - bool ret; - uint32 current; - current = InterlockedCompareExchange(&ptr->value, newval, *expected); - ret = current == *expected; - *expected = current; - return ret; -} - -#define PG_HAVE_ATOMIC_EXCHANGE_U32 -static inline uint32 -pg_atomic_exchange_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 newval) -{ - return InterlockedExchange(&ptr->value, newval); -} - -#define PG_HAVE_ATOMIC_FETCH_ADD_U32 -static inline uint32 -pg_atomic_fetch_add_u32_impl(volatile pg_atomic_uint32 *ptr, int32 add_) -{ - return InterlockedExchangeAdd(&ptr->value, add_); -} - -/* - * The non-intrinsics versions are only available in vista upwards, so use the - * intrinsic version. Only supported on >486, but we require XP as a minimum - * baseline, which doesn't support the 486, so we don't need to add checks for - * that case. - */ -#pragma intrinsic(_InterlockedCompareExchange64) - -#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64 -static inline bool -pg_atomic_compare_exchange_u64_impl(volatile pg_atomic_uint64 *ptr, - uint64 *expected, uint64 newval) -{ - bool ret; - uint64 current; - current = _InterlockedCompareExchange64(&ptr->value, newval, *expected); - ret = current == *expected; - *expected = current; - return ret; -} - -/* Only implemented on 64bit builds */ -#ifdef _WIN64 - -#pragma intrinsic(_InterlockedExchange64) - -#define PG_HAVE_ATOMIC_EXCHANGE_U64 -static inline uint64 -pg_atomic_exchange_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 newval) -{ - return _InterlockedExchange64(&ptr->value, newval); -} - -#pragma intrinsic(_InterlockedExchangeAdd64) - -#define PG_HAVE_ATOMIC_FETCH_ADD_U64 -static inline uint64 -pg_atomic_fetch_add_u64_impl(volatile pg_atomic_uint64 *ptr, int64 add_) -{ - return _InterlockedExchangeAdd64(&ptr->value, add_); -} - -#endif /* _WIN64 */ diff --git a/src/include/port/atomics/generic.h b/src/include/port/atomics/generic.h deleted file mode 100644 index daa772e9a6d..00000000000 --- a/src/include/port/atomics/generic.h +++ /dev/null @@ -1,430 +0,0 @@ -/*------------------------------------------------------------------------- - * - * generic.h - * Implement higher level operations based on some lower level atomic - * operations. - * - * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group - * Portions Copyright (c) 1994, Regents of the University of California - * - * src/include/port/atomics/generic.h - * - *------------------------------------------------------------------------- - */ - -/* intentionally no include guards, should only be included by atomics.h */ -#ifndef INSIDE_ATOMICS_H -# error "should be included via atomics.h" -#endif - -/* - * If read or write barriers are undefined, we upgrade them to full memory - * barriers. - */ -#if !defined(pg_read_barrier_impl) -# define pg_read_barrier_impl pg_memory_barrier_impl -#endif -#if !defined(pg_write_barrier_impl) -# define pg_write_barrier_impl pg_memory_barrier_impl -#endif - -/* provide fallback */ -#if !defined(PG_HAVE_ATOMIC_FLAG_SUPPORT) && defined(PG_HAVE_ATOMIC_U32_SUPPORT) -#define PG_HAVE_ATOMIC_FLAG_SUPPORT -typedef pg_atomic_uint32 pg_atomic_flag; -#endif - -#ifndef PG_HAVE_ATOMIC_READ_U32 -#define PG_HAVE_ATOMIC_READ_U32 -static inline uint32 -pg_atomic_read_u32_impl(volatile pg_atomic_uint32 *ptr) -{ - return ptr->value; -} -#endif - -#ifndef PG_HAVE_ATOMIC_WRITE_U32 -#define PG_HAVE_ATOMIC_WRITE_U32 -static inline void -pg_atomic_write_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 val) -{ - ptr->value = val; -} -#endif - -#ifndef PG_HAVE_ATOMIC_UNLOCKED_WRITE_U32 -#define PG_HAVE_ATOMIC_UNLOCKED_WRITE_U32 -static inline void -pg_atomic_unlocked_write_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 val) -{ - ptr->value = val; -} -#endif - -/* - * provide fallback for test_and_set using atomic_exchange if available - */ -#if !defined(PG_HAVE_ATOMIC_TEST_SET_FLAG) && defined(PG_HAVE_ATOMIC_EXCHANGE_U32) - -#define PG_HAVE_ATOMIC_INIT_FLAG -static inline void -pg_atomic_init_flag_impl(volatile pg_atomic_flag *ptr) -{ - pg_atomic_write_u32_impl(ptr, 0); -} - -#define PG_HAVE_ATOMIC_TEST_SET_FLAG -static inline bool -pg_atomic_test_set_flag_impl(volatile pg_atomic_flag *ptr) -{ - return pg_atomic_exchange_u32_impl(ptr, 1) == 0; -} - -#define PG_HAVE_ATOMIC_UNLOCKED_TEST_FLAG -static inline bool -pg_atomic_unlocked_test_flag_impl(volatile pg_atomic_flag *ptr) -{ - return pg_atomic_read_u32_impl(ptr) == 0; -} - - -#define PG_HAVE_ATOMIC_CLEAR_FLAG -static inline void -pg_atomic_clear_flag_impl(volatile pg_atomic_flag *ptr) -{ - /* XXX: release semantics suffice? */ - pg_memory_barrier_impl(); - pg_atomic_write_u32_impl(ptr, 0); -} - -/* - * provide fallback for test_and_set using atomic_compare_exchange if - * available. - */ -#elif !defined(PG_HAVE_ATOMIC_TEST_SET_FLAG) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32) - -#define PG_HAVE_ATOMIC_INIT_FLAG -static inline void -pg_atomic_init_flag_impl(volatile pg_atomic_flag *ptr) -{ - pg_atomic_write_u32_impl(ptr, 0); -} - -#define PG_HAVE_ATOMIC_TEST_SET_FLAG -static inline bool -pg_atomic_test_set_flag_impl(volatile pg_atomic_flag *ptr) -{ - uint32 value = 0; - return pg_atomic_compare_exchange_u32_impl(ptr, &value, 1); -} - -#define PG_HAVE_ATOMIC_UNLOCKED_TEST_FLAG -static inline bool -pg_atomic_unlocked_test_flag_impl(volatile pg_atomic_flag *ptr) -{ - return pg_atomic_read_u32_impl(ptr) == 0; -} - -#define PG_HAVE_ATOMIC_CLEAR_FLAG -static inline void -pg_atomic_clear_flag_impl(volatile pg_atomic_flag *ptr) -{ - /* XXX: release semantics suffice? */ - pg_memory_barrier_impl(); - pg_atomic_write_u32_impl(ptr, 0); -} - -#elif !defined(PG_HAVE_ATOMIC_TEST_SET_FLAG) -# error "No pg_atomic_test_and_set provided" -#endif /* !defined(PG_HAVE_ATOMIC_TEST_SET_FLAG) */ - - -#ifndef PG_HAVE_ATOMIC_INIT_U32 -#define PG_HAVE_ATOMIC_INIT_U32 -static inline void -pg_atomic_init_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 val_) -{ - ptr->value = val_; -} -#endif - -#if !defined(PG_HAVE_ATOMIC_EXCHANGE_U32) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32) -#define PG_HAVE_ATOMIC_EXCHANGE_U32 -static inline uint32 -pg_atomic_exchange_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 xchg_) -{ - uint32 old; - old = ptr->value; /* ok if read is not atomic */ - while (!pg_atomic_compare_exchange_u32_impl(ptr, &old, xchg_)) - /* skip */; - return old; -} -#endif - -#if !defined(PG_HAVE_ATOMIC_FETCH_ADD_U32) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32) -#define PG_HAVE_ATOMIC_FETCH_ADD_U32 -static inline uint32 -pg_atomic_fetch_add_u32_impl(volatile pg_atomic_uint32 *ptr, int32 add_) -{ - uint32 old; - old = ptr->value; /* ok if read is not atomic */ - while (!pg_atomic_compare_exchange_u32_impl(ptr, &old, old + add_)) - /* skip */; - return old; -} -#endif - -#if !defined(PG_HAVE_ATOMIC_FETCH_SUB_U32) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32) -#define PG_HAVE_ATOMIC_FETCH_SUB_U32 -static inline uint32 -pg_atomic_fetch_sub_u32_impl(volatile pg_atomic_uint32 *ptr, int32 sub_) -{ - return pg_atomic_fetch_add_u32_impl(ptr, -sub_); -} -#endif - -#if !defined(PG_HAVE_ATOMIC_FETCH_AND_U32) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32) -#define PG_HAVE_ATOMIC_FETCH_AND_U32 -static inline uint32 -pg_atomic_fetch_and_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 and_) -{ - uint32 old; - old = ptr->value; /* ok if read is not atomic */ - while (!pg_atomic_compare_exchange_u32_impl(ptr, &old, old & and_)) - /* skip */; - return old; -} -#endif - -#if !defined(PG_HAVE_ATOMIC_FETCH_OR_U32) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32) -#define PG_HAVE_ATOMIC_FETCH_OR_U32 -static inline uint32 -pg_atomic_fetch_or_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 or_) -{ - uint32 old; - old = ptr->value; /* ok if read is not atomic */ - while (!pg_atomic_compare_exchange_u32_impl(ptr, &old, old | or_)) - /* skip */; - return old; -} -#endif - -#if !defined(PG_HAVE_ATOMIC_ADD_FETCH_U32) && defined(PG_HAVE_ATOMIC_FETCH_ADD_U32) -#define PG_HAVE_ATOMIC_ADD_FETCH_U32 -static inline uint32 -pg_atomic_add_fetch_u32_impl(volatile pg_atomic_uint32 *ptr, int32 add_) -{ - return pg_atomic_fetch_add_u32_impl(ptr, add_) + add_; -} -#endif - -#if !defined(PG_HAVE_ATOMIC_SUB_FETCH_U32) && defined(PG_HAVE_ATOMIC_FETCH_SUB_U32) -#define PG_HAVE_ATOMIC_SUB_FETCH_U32 -static inline uint32 -pg_atomic_sub_fetch_u32_impl(volatile pg_atomic_uint32 *ptr, int32 sub_) -{ - return pg_atomic_fetch_sub_u32_impl(ptr, sub_) - sub_; -} -#endif - -#if !defined(PG_HAVE_ATOMIC_READ_MEMBARRIER_U32) && defined(PG_HAVE_ATOMIC_FETCH_ADD_U32) -#define PG_HAVE_ATOMIC_READ_MEMBARRIER_U32 -static inline uint32 -pg_atomic_read_membarrier_u32_impl(volatile pg_atomic_uint32 *ptr) -{ - return pg_atomic_fetch_add_u32_impl(ptr, 0); -} -#endif - -#if !defined(PG_HAVE_ATOMIC_WRITE_MEMBARRIER_U32) && defined(PG_HAVE_ATOMIC_EXCHANGE_U32) -#define PG_HAVE_ATOMIC_WRITE_MEMBARRIER_U32 -static inline void -pg_atomic_write_membarrier_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 val) -{ - (void) pg_atomic_exchange_u32_impl(ptr, val); -} -#endif - -#if !defined(PG_HAVE_ATOMIC_EXCHANGE_U64) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64) -#define PG_HAVE_ATOMIC_EXCHANGE_U64 -static inline uint64 -pg_atomic_exchange_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 xchg_) -{ - uint64 old; - old = ptr->value; /* ok if read is not atomic */ - while (!pg_atomic_compare_exchange_u64_impl(ptr, &old, xchg_)) - /* skip */; - return old; -} -#endif - -#ifndef PG_HAVE_ATOMIC_WRITE_U64 -#define PG_HAVE_ATOMIC_WRITE_U64 - -#if defined(PG_HAVE_8BYTE_SINGLE_COPY_ATOMICITY) && \ - !defined(PG_HAVE_ATOMIC_U64_SIMULATION) - -static inline void -pg_atomic_write_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 val) -{ - /* - * On this platform aligned 64bit writes are guaranteed to be atomic, - * except if using the fallback implementation, where can't guarantee the - * required alignment. - */ - AssertPointerAlignment(ptr, 8); - ptr->value = val; -} - -#else - -static inline void -pg_atomic_write_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 val) -{ - /* - * 64 bit writes aren't safe on all platforms. In the generic - * implementation implement them as an atomic exchange. - */ - pg_atomic_exchange_u64_impl(ptr, val); -} - -#endif /* PG_HAVE_8BYTE_SINGLE_COPY_ATOMICITY && !PG_HAVE_ATOMIC_U64_SIMULATION */ -#endif /* PG_HAVE_ATOMIC_WRITE_U64 */ - -#ifndef PG_HAVE_ATOMIC_UNLOCKED_WRITE_U64 -#define PG_HAVE_ATOMIC_UNLOCKED_WRITE_U64 -static inline void -pg_atomic_unlocked_write_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 val) -{ - ptr->value = val; -} -#endif - -#ifndef PG_HAVE_ATOMIC_READ_U64 -#define PG_HAVE_ATOMIC_READ_U64 - -#if defined(PG_HAVE_8BYTE_SINGLE_COPY_ATOMICITY) && \ - !defined(PG_HAVE_ATOMIC_U64_SIMULATION) - -static inline uint64 -pg_atomic_read_u64_impl(volatile pg_atomic_uint64 *ptr) -{ - /* - * On this platform aligned 64-bit reads are guaranteed to be atomic. - */ - AssertPointerAlignment(ptr, 8); - return ptr->value; -} - -#else - -static inline uint64 -pg_atomic_read_u64_impl(volatile pg_atomic_uint64 *ptr) -{ - uint64 old = 0; - - /* - * 64-bit reads aren't atomic on all platforms. In the generic - * implementation implement them as a compare/exchange with 0. That'll - * fail or succeed, but always return the old value. Possibly might store - * a 0, but only if the previous value also was a 0 - i.e. harmless. - */ - pg_atomic_compare_exchange_u64_impl(ptr, &old, 0); - - return old; -} -#endif /* PG_HAVE_8BYTE_SINGLE_COPY_ATOMICITY && !PG_HAVE_ATOMIC_U64_SIMULATION */ -#endif /* PG_HAVE_ATOMIC_READ_U64 */ - -#ifndef PG_HAVE_ATOMIC_INIT_U64 -#define PG_HAVE_ATOMIC_INIT_U64 -static inline void -pg_atomic_init_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 val_) -{ - ptr->value = val_; -} -#endif - -#if !defined(PG_HAVE_ATOMIC_FETCH_ADD_U64) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64) -#define PG_HAVE_ATOMIC_FETCH_ADD_U64 -static inline uint64 -pg_atomic_fetch_add_u64_impl(volatile pg_atomic_uint64 *ptr, int64 add_) -{ - uint64 old; - old = ptr->value; /* ok if read is not atomic */ - while (!pg_atomic_compare_exchange_u64_impl(ptr, &old, old + add_)) - /* skip */; - return old; -} -#endif - -#if !defined(PG_HAVE_ATOMIC_FETCH_SUB_U64) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64) -#define PG_HAVE_ATOMIC_FETCH_SUB_U64 -static inline uint64 -pg_atomic_fetch_sub_u64_impl(volatile pg_atomic_uint64 *ptr, int64 sub_) -{ - return pg_atomic_fetch_add_u64_impl(ptr, -sub_); -} -#endif - -#if !defined(PG_HAVE_ATOMIC_FETCH_AND_U64) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64) -#define PG_HAVE_ATOMIC_FETCH_AND_U64 -static inline uint64 -pg_atomic_fetch_and_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 and_) -{ - uint64 old; - old = ptr->value; /* ok if read is not atomic */ - while (!pg_atomic_compare_exchange_u64_impl(ptr, &old, old & and_)) - /* skip */; - return old; -} -#endif - -#if !defined(PG_HAVE_ATOMIC_FETCH_OR_U64) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64) -#define PG_HAVE_ATOMIC_FETCH_OR_U64 -static inline uint64 -pg_atomic_fetch_or_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 or_) -{ - uint64 old; - old = ptr->value; /* ok if read is not atomic */ - while (!pg_atomic_compare_exchange_u64_impl(ptr, &old, old | or_)) - /* skip */; - return old; -} -#endif - -#if !defined(PG_HAVE_ATOMIC_ADD_FETCH_U64) && defined(PG_HAVE_ATOMIC_FETCH_ADD_U64) -#define PG_HAVE_ATOMIC_ADD_FETCH_U64 -static inline uint64 -pg_atomic_add_fetch_u64_impl(volatile pg_atomic_uint64 *ptr, int64 add_) -{ - return pg_atomic_fetch_add_u64_impl(ptr, add_) + add_; -} -#endif - -#if !defined(PG_HAVE_ATOMIC_SUB_FETCH_U64) && defined(PG_HAVE_ATOMIC_FETCH_SUB_U64) -#define PG_HAVE_ATOMIC_SUB_FETCH_U64 -static inline uint64 -pg_atomic_sub_fetch_u64_impl(volatile pg_atomic_uint64 *ptr, int64 sub_) -{ - return pg_atomic_fetch_sub_u64_impl(ptr, sub_) - sub_; -} -#endif - -#if !defined(PG_HAVE_ATOMIC_READ_MEMBARRIER_U64) && defined(PG_HAVE_ATOMIC_FETCH_ADD_U64) -#define PG_HAVE_ATOMIC_READ_MEMBARRIER_U64 -static inline uint64 -pg_atomic_read_membarrier_u64_impl(volatile pg_atomic_uint64 *ptr) -{ - return pg_atomic_fetch_add_u64_impl(ptr, 0); -} -#endif - -#if !defined(PG_HAVE_ATOMIC_WRITE_MEMBARRIER_U64) && defined(PG_HAVE_ATOMIC_EXCHANGE_U64) -#define PG_HAVE_ATOMIC_WRITE_MEMBARRIER_U64 -static inline void -pg_atomic_write_membarrier_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 val) -{ - (void) pg_atomic_exchange_u64_impl(ptr, val); -} -#endif diff --git a/src/include/port/atomics/stdatomic_impl.h b/src/include/port/atomics/stdatomic_impl.h index 0bda02d16f4..7519a096e07 100644 --- a/src/include/port/atomics/stdatomic_impl.h +++ b/src/include/port/atomics/stdatomic_impl.h @@ -4,11 +4,9 @@ * Atomic operations implementation using C11 stdatomic.h * * This file provides PostgreSQL atomic operations using the C11 standard - * . It is only included when USE_STDATOMIC_H is defined. - * - * The traditional platform-specific implementation (arch-*.h, generic-*.h, - * fallback.h) remains available and is used when stdatomic.h is not - * available or when explicitly requested via -Duse_stdatomic=no. + * . As of this commit it is the sole atomics implementation; + * the former platform-specific paths (arch-*.h, generic-*.h, fallback.h) + * have been removed and USE_STDATOMIC_H is always defined. * * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California @@ -21,12 +19,6 @@ #ifndef STDATOMIC_IMPL_H #define STDATOMIC_IMPL_H -/* - * Only include this file when USE_STDATOMIC_H is defined at build time. - * This ensures the traditional implementation remains available as fallback. - */ -#ifdef USE_STDATOMIC_H - /* * C++ Compatibility * @@ -584,6 +576,4 @@ pg_atomic_write_membarrier_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 val) atomic_store_explicit(ptr, val, memory_order_seq_cst); } -#endif /* USE_STDATOMIC_H */ - #endif /* STDATOMIC_IMPL_H */ diff --git a/src/include/storage/s_lock.h b/src/include/storage/s_lock.h deleted file mode 100644 index 96fcaaac01e..00000000000 --- a/src/include/storage/s_lock.h +++ /dev/null @@ -1,753 +0,0 @@ -/*------------------------------------------------------------------------- - * - * s_lock.h - * Implementation of spinlocks. - * - * NOTE: none of the macros in this file are intended to be called directly. - * Call them through the macros in spin.h. - * - * The following hardware-dependent macros must be provided for each - * supported platform: - * - * void S_INIT_LOCK(slock_t *lock) - * Initialize a spinlock (to the unlocked state). - * - * int S_LOCK(slock_t *lock) - * Acquire a spinlock, waiting if necessary. - * Time out and abort() if unable to acquire the lock in a - * "reasonable" amount of time --- typically ~ 1 minute. - * Should return number of "delays"; see s_lock.c - * - * void S_UNLOCK(slock_t *lock) - * Unlock a previously acquired lock. - * - * void SPIN_DELAY(void) - * Delay operation to occur inside spinlock wait loop. - * - * Note to implementors: there are default implementations for all these - * macros at the bottom of the file. Check if your platform can use - * these or needs to override them. - * - * Usually, S_LOCK() is implemented in terms of even lower-level macros - * TAS() and TAS_SPIN(): - * - * int TAS(slock_t *lock) - * Atomic test-and-set instruction. Attempt to acquire the lock, - * but do *not* wait. Returns 0 if successful, nonzero if unable - * to acquire the lock. - * - * int TAS_SPIN(slock_t *lock) - * Like TAS(), but this version is used when waiting for a lock - * previously found to be contended. By default, this is the - * same as TAS(), but on some architectures it's better to poll a - * contended lock using an unlocked instruction and retry the - * atomic test-and-set only when it appears free. - * - * TAS() and TAS_SPIN() are NOT part of the API, and should never be called - * directly. - * - * CAUTION: on some platforms TAS() and/or TAS_SPIN() may sometimes report - * failure to acquire a lock even when the lock is not locked. For example, - * on Alpha TAS() will "fail" if interrupted. Therefore a retry loop must - * always be used, even if you are certain the lock is free. - * - * It is the responsibility of these macros to make sure that the compiler - * does not re-order accesses to shared memory to precede the actual lock - * acquisition, or follow the lock release. Prior to PostgreSQL 9.5, this - * was the caller's responsibility, which meant that callers had to use - * volatile-qualified pointers to refer to both the spinlock itself and the - * shared data being accessed within the spinlocked critical section. This - * was notationally awkward, easy to forget (and thus error-prone), and - * prevented some useful compiler optimizations. For these reasons, we - * now require that the macros themselves prevent compiler re-ordering, - * so that the caller doesn't need to take special precautions. - * - * On platforms with weak memory ordering, the TAS(), TAS_SPIN(), and - * S_UNLOCK() macros must further include hardware-level memory fence - * instructions to prevent similar re-ordering at the hardware level. - * TAS() and TAS_SPIN() must guarantee that loads and stores issued after - * the macro are not executed until the lock has been obtained. Conversely, - * S_UNLOCK() must guarantee that loads and stores issued before the macro - * have been executed before the lock is released. - * - * On most supported platforms, TAS() uses a tas() function written - * in assembly language to execute a hardware atomic-test-and-set - * instruction. Equivalent OS-supplied mutex routines could be used too. - * - * - * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group - * Portions Copyright (c) 1994, Regents of the University of California - * - * src/include/storage/s_lock.h - * - *------------------------------------------------------------------------- - */ -#ifndef S_LOCK_H -#define S_LOCK_H - -#ifdef FRONTEND -#error "s_lock.h may not be included from frontend code" -#endif - -#if defined(__GNUC__) || defined(__INTEL_COMPILER) -/************************************************************************* - * All the gcc inlines - * Gcc consistently defines the CPU as __cpu__. - * Other compilers use __cpu or __cpu__ so we test for both in those cases. - */ - -/*---------- - * Standard gcc asm format (assuming "volatile slock_t *lock"): - - __asm__ __volatile__( - " instruction \n" - " instruction \n" - " instruction \n" -: "=r"(_res), "+m"(*lock) // return register, in/out lock value -: "r"(lock) // lock pointer, in input register -: "memory", "cc"); // show clobbered registers here - - * The output-operands list (after first colon) should always include - * "+m"(*lock), whether or not the asm code actually refers to this - * operand directly. This ensures that gcc believes the value in the - * lock variable is used and set by the asm code. Also, the clobbers - * list (after third colon) should always include "memory"; this prevents - * gcc from thinking it can cache the values of shared-memory fields - * across the asm code. Add "cc" if your asm code changes the condition - * code register, and also list any temp registers the code uses. - * - * If you need branch target labels within the asm block, include "%=" - * in the label names to make them distinct across multiple asm blocks - * within a source file. - *---------- - */ - - -#ifdef __i386__ /* 32-bit i386 */ -#define HAS_TEST_AND_SET - -typedef unsigned char slock_t; - -#define TAS(lock) tas(lock) - -static inline int -tas(volatile slock_t *lock) -{ - slock_t _res = 1; - - /* - * Use a non-locking test before asserting the bus lock. Note that the - * extra test appears to be a small loss on some x86 platforms and a small - * win on others; it's by no means clear that we should keep it. - * - * When this was last tested, we didn't have separate TAS() and TAS_SPIN() - * macros. Nowadays it probably would be better to do a non-locking test - * in TAS_SPIN() but not in TAS(), like on x86_64, but no-one's done the - * testing to verify that. Without some empirical evidence, better to - * leave it alone. - */ - __asm__ __volatile__( - " cmpb $0,%1 \n" - " jne TAS%=_out \n" - " lock \n" - " xchgb %0,%1 \n" - "TAS%=_out: \n" -: "+q"(_res), "+m"(*lock) -: /* no inputs */ -: "memory", "cc"); - return (int) _res; -} - -#define SPIN_DELAY() spin_delay() - -static inline void -spin_delay(void) -{ - /* - * This sequence is equivalent to the PAUSE instruction ("rep" is - * ignored by old IA32 processors if the following instruction is - * not a string operation); the IA-32 Architecture Software - * Developer's Manual, Vol. 3, Section 7.7.2 describes why using - * PAUSE in the inner loop of a spin lock is necessary for good - * performance: - * - * The PAUSE instruction improves the performance of IA-32 - * processors supporting Hyper-Threading Technology when - * executing spin-wait loops and other routines where one - * thread is accessing a shared lock or semaphore in a tight - * polling loop. When executing a spin-wait loop, the - * processor can suffer a severe performance penalty when - * exiting the loop because it detects a possible memory order - * violation and flushes the core processor's pipeline. The - * PAUSE instruction provides a hint to the processor that the - * code sequence is a spin-wait loop. The processor uses this - * hint to avoid the memory order violation and prevent the - * pipeline flush. In addition, the PAUSE instruction - * de-pipelines the spin-wait loop to prevent it from - * consuming execution resources excessively. - */ - __asm__ __volatile__( - " rep; nop \n"); -} - -#endif /* __i386__ */ - - -#ifdef __x86_64__ /* AMD Opteron, Intel EM64T */ -#define HAS_TEST_AND_SET - -typedef unsigned char slock_t; - -#define TAS(lock) tas(lock) - -/* - * On Intel EM64T, it's a win to use a non-locking test before the xchg proper, - * but only when spinning. - * - * See also Implementing Scalable Atomic Locks for Multi-Core Intel(tm) EM64T - * and IA32, by Michael Chynoweth and Mary R. Lee. As of this writing, it is - * available at: - * http://software.intel.com/en-us/articles/implementing-scalable-atomic-locks-for-multi-core-intel-em64t-and-ia32-architectures - */ -#define TAS_SPIN(lock) (*(lock) ? 1 : TAS(lock)) - -static inline int -tas(volatile slock_t *lock) -{ - slock_t _res = 1; - - __asm__ __volatile__( - " lock \n" - " xchgb %0,%1 \n" -: "+q"(_res), "+m"(*lock) -: /* no inputs */ -: "memory", "cc"); - return (int) _res; -} - -#define SPIN_DELAY() spin_delay() - -static inline void -spin_delay(void) -{ - /* - * Adding a PAUSE in the spin delay loop is demonstrably a no-op on - * Opteron, but it may be of some use on EM64T, so we keep it. - */ - __asm__ __volatile__( - " rep; nop \n"); -} - -#endif /* __x86_64__ */ - - -/* - * On ARM and ARM64, we use __sync_lock_test_and_set(int *, int) if available. - * - * We use the int-width variant of the builtin because it works on more chips - * than other widths. - */ -#if defined(__arm__) || defined(__aarch64__) -#ifdef HAVE_GCC__SYNC_INT32_TAS -#define HAS_TEST_AND_SET - -#define TAS(lock) tas(lock) - -typedef int slock_t; - -static inline int -tas(volatile slock_t *lock) -{ - return __sync_lock_test_and_set(lock, 1); -} - -#define S_UNLOCK(lock) __sync_lock_release(lock) - -#if defined(__aarch64__) - -/* - * On ARM64, it's a win to use a non-locking test before the TAS proper. It - * may be a win on 32-bit ARM, too, but nobody's tested it yet. - */ -#define TAS_SPIN(lock) (*(lock) ? 1 : TAS(lock)) - -#define SPIN_DELAY() spin_delay() - -static inline void -spin_delay(void) -{ - /* - * Using an ISB instruction to delay in spinlock loops appears beneficial - * on high-core-count ARM64 processors. It seems mostly a wash for smaller - * gear, and ISB doesn't exist at all on pre-v7 ARM chips. - */ - __asm__ __volatile__( - " isb; \n"); -} - -#endif /* __aarch64__ */ -#endif /* HAVE_GCC__SYNC_INT32_TAS */ -#endif /* __arm__ || __aarch64__ */ - - -/* S/390 and S/390x Linux (32- and 64-bit zSeries) */ -#if defined(__s390__) || defined(__s390x__) -#define HAS_TEST_AND_SET - -typedef unsigned int slock_t; - -#define TAS(lock) tas(lock) - -static inline int -tas(volatile slock_t *lock) -{ - int _res = 0; - - __asm__ __volatile__( - " cs %0,%3,0(%2) \n" -: "+d"(_res), "+m"(*lock) -: "a"(lock), "d"(1) -: "memory", "cc"); - return _res; -} - -#endif /* __s390__ || __s390x__ */ - - -#if defined(__sparc__) /* Sparc */ -/* - * Solaris has always run sparc processors in TSO (total store) mode, but - * linux didn't use to and the *BSDs still don't. So, be careful about - * acquire/release semantics. The CPU will treat superfluous members as - * NOPs, so it's just code space. - */ -#define HAS_TEST_AND_SET - -typedef unsigned char slock_t; - -#define TAS(lock) tas(lock) - -static inline int -tas(volatile slock_t *lock) -{ - slock_t _res; - - /* - * "cas" would be better than "ldstub", but it is only present on - * sparcv8plus and later, while some platforms still support sparcv7 or - * sparcv8. Also, "cas" requires that the system be running in TSO mode. - */ - __asm__ __volatile__( - " ldstub [%2], %0 \n" -: "=r"(_res), "+m"(*lock) -: "r"(lock) -: "memory"); -#if defined(__sparcv7) || defined(__sparc_v7__) - /* - * No stbar or membar available, luckily no actually produced hardware - * requires a barrier. - */ -#elif defined(__sparcv8) || defined(__sparc_v8__) - /* stbar is available (and required for both PSO, RMO), membar isn't */ - __asm__ __volatile__ ("stbar \n":::"memory"); -#else - /* - * #LoadStore (RMO) | #LoadLoad (RMO) together are the appropriate acquire - * barrier for sparcv8+ upwards. - */ - __asm__ __volatile__ ("membar #LoadStore | #LoadLoad \n":::"memory"); -#endif - return (int) _res; -} - -#if defined(__sparcv7) || defined(__sparc_v7__) -/* - * No stbar or membar available, luckily no actually produced hardware - * requires a barrier. We fall through to the default gcc definition of - * S_UNLOCK in this case. - */ -#elif defined(__sparcv8) || defined(__sparc_v8__) -/* stbar is available (and required for both PSO, RMO), membar isn't */ -#define S_UNLOCK(lock) \ -do \ -{ \ - __asm__ __volatile__ ("stbar \n":::"memory"); \ - *((volatile slock_t *) (lock)) = 0; \ -} while (0) -#else -/* - * #LoadStore (RMO) | #StoreStore (RMO, PSO) together are the appropriate - * release barrier for sparcv8+ upwards. - */ -#define S_UNLOCK(lock) \ -do \ -{ \ - __asm__ __volatile__ ("membar #LoadStore | #StoreStore \n":::"memory"); \ - *((volatile slock_t *) (lock)) = 0; \ -} while (0) -#endif - -#endif /* __sparc__ */ - - -/* PowerPC */ -#if defined(__powerpc__) || defined(__powerpc64__) -#define HAS_TEST_AND_SET - -typedef unsigned int slock_t; - -#define TAS(lock) tas(lock) - -/* On PPC, it's a win to use a non-locking test before the lwarx */ -#define TAS_SPIN(lock) (*(lock) ? 1 : TAS(lock)) - -/* - * The second operand of addi can hold a constant zero or a register number, - * hence constraint "=&b" to avoid allocating r0. "b" stands for "address - * base register"; most operands having this register-or-zero property are - * address bases, e.g. the second operand of lwax. - * - * NOTE: per the Enhanced PowerPC Architecture manual, v1.0 dated 7-May-2002, - * an isync is a sufficient synchronization barrier after a lwarx/stwcx loop. - * But if the spinlock is in ordinary memory, we can use lwsync instead for - * better performance. - */ -static inline int -tas(volatile slock_t *lock) -{ - slock_t _t; - int _res; - - __asm__ __volatile__( -" lwarx %0,0,%3,1 \n" -" cmpwi %0,0 \n" -" bne TAS%=_fail \n" -" addi %0,%0,1 \n" -" stwcx. %0,0,%3 \n" -" beq TAS%=_ok \n" -"TAS%=_fail: \n" -" li %1,1 \n" -" b TAS%=_out \n" -"TAS%=_ok: \n" -" lwsync \n" -" li %1,0 \n" -"TAS%=_out: \n" -: "=&b"(_t), "=r"(_res), "+m"(*lock) -: "r"(lock) -: "memory", "cc"); - return _res; -} - -/* - * PowerPC S_UNLOCK is almost standard but requires a "sync" instruction. - * But we can use lwsync instead for better performance. - */ -#define S_UNLOCK(lock) \ -do \ -{ \ - __asm__ __volatile__ (" lwsync \n" ::: "memory"); \ - *((volatile slock_t *) (lock)) = 0; \ -} while (0) - -#endif /* powerpc */ - - -#if defined(__mips__) && !defined(__sgi) /* non-SGI MIPS */ -#define HAS_TEST_AND_SET - -typedef unsigned int slock_t; - -#define TAS(lock) tas(lock) - -/* - * Original MIPS-I processors lacked the LL/SC instructions, but if we are - * so unfortunate as to be running on one of those, we expect that the kernel - * will handle the illegal-instruction traps and emulate them for us. On - * anything newer (and really, MIPS-I is extinct) LL/SC is the only sane - * choice because any other synchronization method must involve a kernel - * call. Unfortunately, many toolchains still default to MIPS-I as the - * codegen target; if the symbol __mips shows that that's the case, we - * have to force the assembler to accept LL/SC. - * - * R10000 and up processors require a separate SYNC, which has the same - * issues as LL/SC. - */ -#if __mips < 2 -#define MIPS_SET_MIPS2 " .set mips2 \n" -#else -#define MIPS_SET_MIPS2 -#endif - -static inline int -tas(volatile slock_t *lock) -{ - volatile slock_t *_l = lock; - int _res; - int _tmp; - - __asm__ __volatile__( - " .set push \n" - MIPS_SET_MIPS2 - " .set noreorder \n" - " .set nomacro \n" - " ll %0, %2 \n" - " or %1, %0, 1 \n" - " sc %1, %2 \n" - " xori %1, 1 \n" - " or %0, %0, %1 \n" - " sync \n" - " .set pop " -: "=&r" (_res), "=&r" (_tmp), "+R" (*_l) -: /* no inputs */ -: "memory"); - return _res; -} - -/* MIPS S_UNLOCK is almost standard but requires a "sync" instruction */ -#define S_UNLOCK(lock) \ -do \ -{ \ - __asm__ __volatile__( \ - " .set push \n" \ - MIPS_SET_MIPS2 \ - " .set noreorder \n" \ - " .set nomacro \n" \ - " sync \n" \ - " .set pop " \ -: /* no outputs */ \ -: /* no inputs */ \ -: "memory"); \ - *((volatile slock_t *) (lock)) = 0; \ -} while (0) - -#endif /* __mips__ && !__sgi */ - - - -/* - * If we have no platform-specific knowledge, but we found that the compiler - * provides __sync_lock_test_and_set(), use that. Prefer the int-width - * version over the char-width version if we have both, on the rather dubious - * grounds that that's known to be more likely to work in the ARM ecosystem. - * (But we dealt with ARM above.) - */ -#if !defined(HAS_TEST_AND_SET) - -#if defined(HAVE_GCC__SYNC_INT32_TAS) -#define HAS_TEST_AND_SET - -#define TAS(lock) tas(lock) - -typedef int slock_t; - -static inline int -tas(volatile slock_t *lock) -{ - return __sync_lock_test_and_set(lock, 1); -} - -#define S_UNLOCK(lock) __sync_lock_release(lock) - -#elif defined(HAVE_GCC__SYNC_CHAR_TAS) -#define HAS_TEST_AND_SET - -#define TAS(lock) tas(lock) - -typedef char slock_t; - -static inline int -tas(volatile slock_t *lock) -{ - return __sync_lock_test_and_set(lock, 1); -} - -#define S_UNLOCK(lock) __sync_lock_release(lock) - -#endif /* HAVE_GCC__SYNC_INT32_TAS */ - -#endif /* !defined(HAS_TEST_AND_SET) */ - - -/* - * Default implementation of S_UNLOCK() for gcc/icc. - * - * Note that this implementation is unsafe for any platform that can reorder - * a memory access (either load or store) after a following store. That - * happens not to be possible on x86 and most legacy architectures (some are - * single-processor!), but many modern systems have weaker memory ordering. - * Those that do must define their own version of S_UNLOCK() rather than - * relying on this one. - */ -#if !defined(S_UNLOCK) -#define S_UNLOCK(lock) \ - do { __asm__ __volatile__("" : : : "memory"); *(lock) = 0; } while (0) -#endif - -#endif /* defined(__GNUC__) || defined(__INTEL_COMPILER) */ - - -/* - * --------------------------------------------------------------------- - * Platforms that use non-gcc inline assembly: - * --------------------------------------------------------------------- - */ - -#if !defined(HAS_TEST_AND_SET) /* We didn't trigger above, let's try here */ - -#ifdef _MSC_VER -typedef LONG slock_t; - -#define HAS_TEST_AND_SET -#define TAS(lock) (InterlockedCompareExchange(lock, 1, 0)) - -#define SPIN_DELAY() spin_delay() - -#ifdef __aarch64__ -static __forceinline void -spin_delay(void) -{ - /* - * Research indicates ISB is better than __yield() on AArch64. See - * https://postgr.es/m/1c2a29b8-5b1e-44f7-a871-71ec5fefc120%40app.fastmail.com. - */ - __isb(_ARM64_BARRIER_SY); -} -#elif defined(_WIN64) -static __forceinline void -spin_delay(void) -{ - /* - * If using Visual C++ on Win64, inline assembly is unavailable. - * Use a _mm_pause intrinsic instead of rep nop. - */ - _mm_pause(); -} -#else -static __forceinline void -spin_delay(void) -{ - /* See comment for gcc code. Same code, MASM syntax */ - __asm rep nop; -} -#endif - -#include - -#ifdef __aarch64__ - -/* _ReadWriteBarrier() is insufficient on non-TSO architectures. */ -#pragma intrinsic(_InterlockedExchange) -#define S_UNLOCK(lock) _InterlockedExchange(lock, 0) - -#else - -#pragma intrinsic(_ReadWriteBarrier) -#define S_UNLOCK(lock) \ - do { _ReadWriteBarrier(); (*(lock)) = 0; } while (0) - -#endif -#endif - - -#endif /* !defined(HAS_TEST_AND_SET) */ - - -/* Blow up if we didn't have any way to do spinlocks */ -#ifndef HAS_TEST_AND_SET -#error PostgreSQL does not have spinlock support on this platform. Please report this to pgsql-bugs@lists.postgresql.org. -#endif - - -/* - * Default Definitions - override these above as needed. - */ - -#if !defined(S_LOCK) -#define S_LOCK(lock) \ - (TAS(lock) ? s_lock((lock), __FILE__, __LINE__, __func__) : 0) -#endif /* S_LOCK */ - -#if !defined(S_UNLOCK) -/* - * Our default implementation of S_UNLOCK is essentially *(lock) = 0. This - * is unsafe if the platform can reorder a memory access (either load or - * store) after a following store; platforms where this is possible must - * define their own S_UNLOCK. But CPU reordering is not the only concern: - * if we simply defined S_UNLOCK() as an inline macro, the compiler might - * reorder instructions from inside the critical section to occur after the - * lock release. Since the compiler probably can't know what the external - * function s_unlock is doing, putting the same logic there should be adequate. - * A sufficiently-smart globally optimizing compiler could break that - * assumption, though, and the cost of a function call for every spinlock - * release may hurt performance significantly, so we use this implementation - * only for platforms where we don't know of a suitable intrinsic. For the - * most part, those are relatively obscure platform/compiler combinations to - * which the PostgreSQL project does not have access. - */ -#define USE_DEFAULT_S_UNLOCK -extern void s_unlock(volatile slock_t *lock); -#define S_UNLOCK(lock) s_unlock(lock) -#endif /* S_UNLOCK */ - -#if !defined(S_INIT_LOCK) -#define S_INIT_LOCK(lock) S_UNLOCK(lock) -#endif /* S_INIT_LOCK */ - -#if !defined(SPIN_DELAY) -#define SPIN_DELAY() ((void) 0) -#endif /* SPIN_DELAY */ - -#if !defined(TAS) -extern int tas(volatile slock_t *lock); /* in port/.../tas.s, or - * s_lock.c */ - -#define TAS(lock) tas(lock) -#endif /* TAS */ - -#if !defined(TAS_SPIN) -#define TAS_SPIN(lock) TAS(lock) -#endif /* TAS_SPIN */ - - -/* - * Platform-independent out-of-line support routines - */ -extern int s_lock(volatile slock_t *lock, const char *file, int line, const char *func); - -/* Support for dynamic adjustment of spins_per_delay */ -#define DEFAULT_SPINS_PER_DELAY 100 - -extern void set_spins_per_delay(int shared_spins_per_delay); -extern int update_spins_per_delay(int shared_spins_per_delay); - -/* - * Support for spin delay which is useful in various places where - * spinlock-like procedures take place. - */ -typedef struct -{ - int spins; - int delays; - int cur_delay; - const char *file; - int line; - const char *func; -} SpinDelayStatus; - -static inline void -init_spin_delay(SpinDelayStatus *status, - const char *file, int line, const char *func) -{ - status->spins = 0; - status->delays = 0; - status->cur_delay = 0; - status->file = file; - status->line = line; - status->func = func; -} - -#define init_local_spin_delay(status) init_spin_delay(status, __FILE__, __LINE__, __func__) -extern void perform_spin_delay(SpinDelayStatus *status); -extern void finish_spin_delay(SpinDelayStatus *status); - -#endif /* S_LOCK_H */ diff --git a/src/include/storage/spin.h b/src/include/storage/spin.h index 28121482224..71cd69dc694 100644 --- a/src/include/storage/spin.h +++ b/src/include/storage/spin.h @@ -28,10 +28,9 @@ * for a CHECK_FOR_INTERRUPTS() to occur while holding a spinlock, and so * it is not necessary to do HOLD/RESUME_INTERRUPTS() in these functions. * - * These functions are implemented in terms of hardware-dependent macros - * supplied by s_lock.h. There is not currently any extra functionality - * added by this header, but there has been in the past and may someday - * be again. + * These functions are implemented on top of the C11 atomic + * operations in port/atomics.h; the contended-acquire slow path lives in + * s_lock() in s_lock.c. * * * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group @@ -44,19 +43,17 @@ #ifndef SPIN_H #define SPIN_H -#ifdef USE_STDATOMIC_H - /* * Atomics-based spinlocks. * - * When stdatomic.h is available, spinlocks are implemented on top of a plain - * 32-bit atomic rather than platform-specific TAS assembly. We deliberately - * do NOT use pg_atomic_flag: that type uses 1==unlocked so it can offer a - * relaxed "unlocked test", whereas a spinlock must be usable when its memory - * has merely been zeroed (much shared-memory state is set up with + * Spinlocks are implemented directly on top of a plain 32-bit atomic rather + * than platform-specific TAS assembly. We deliberately do NOT use + * pg_atomic_flag: that type uses 1==unlocked so it can offer a relaxed + * "unlocked test", whereas a spinlock must be usable when its memory has + * merely been zeroed (a great deal of shared-memory state is set up with * memset(...,0,...) and then relies on embedded spinlocks reading as free). * So slock_t uses the traditional convention: 0 == unlocked, 1 == locked, - * making a zero-initialized slock_t a valid, free lock. + * which makes a zero-initialized slock_t a valid, free lock. */ #include "port/atomics.h" @@ -112,32 +109,4 @@ SpinLockRelease(volatile slock_t *lock) pg_atomic_write_u32(lock, 0); } -#else /* !USE_STDATOMIC_H */ - -/* - * Traditional spinlock implementation using platform-specific TAS assembly. - * This branch is kept byte-for-byte equivalent to the pre-stdatomic spin.h. - */ -#include "storage/s_lock.h" - -static inline void -SpinLockInit(volatile slock_t *lock) -{ - S_INIT_LOCK(lock); -} - -static inline void -SpinLockAcquire(volatile slock_t *lock) -{ - S_LOCK(lock); -} - -static inline void -SpinLockRelease(volatile slock_t *lock) -{ - S_UNLOCK(lock); -} - -#endif /* USE_STDATOMIC_H */ - #endif /* SPIN_H */ diff --git a/src/test/regress/regress.c b/src/test/regress/regress.c index df6931a0cf5..c0e993fd453 100644 --- a/src/test/regress/regress.c +++ b/src/test/regress/regress.c @@ -650,11 +650,8 @@ test_spinlock(void) * Basic tests for spinlocks, as well as the underlying operations. * * We embed the spinlock in a struct with other members to test that the - * spinlock operations don't perform too wide writes. - * - * Note: slock_t is pg_atomic_uint32 (4 bytes) in the stdatomic path vs - * potentially smaller (e.g., unsigned char) in the traditional path. This - * test verifies no over-wide writes regardless of slock_t size. + * spinlock operations don't perform too wide writes. slock_t is a + * pg_atomic_uint32 (4 bytes); this test verifies no over-wide writes. */ { struct test_lock_struct @@ -672,40 +669,9 @@ test_spinlock(void) SpinLockAcquire(&struct_w_lock.lock); SpinLockRelease(&struct_w_lock.lock); -#ifndef USE_STDATOMIC_H - /* test basic operations via underlying S_* API */ - S_INIT_LOCK(&struct_w_lock.lock); - S_LOCK(&struct_w_lock.lock); - S_UNLOCK(&struct_w_lock.lock); -#endif - /* and that "contended" acquisition works */ s_lock(&struct_w_lock.lock, "testfile", 17, "testfunc"); -#ifdef USE_STDATOMIC_H SpinLockRelease(&struct_w_lock.lock); -#else - S_UNLOCK(&struct_w_lock.lock); -#endif - -#ifndef USE_STDATOMIC_H - /* - * Check, using TAS directly, that a single spin cycle doesn't block - * when acquiring an already acquired lock. - */ -#ifdef TAS - S_LOCK(&struct_w_lock.lock); - - if (!TAS(&struct_w_lock.lock)) - elog(ERROR, "acquired already held spinlock"); - -#ifdef TAS_SPIN - if (!TAS_SPIN(&struct_w_lock.lock)) - elog(ERROR, "acquired already held spinlock"); -#endif /* defined(TAS_SPIN) */ - - S_UNLOCK(&struct_w_lock.lock); -#endif /* defined(TAS) */ -#endif /* !USE_STDATOMIC_H */ /* * Verify that after all of this the non-lock contents are still diff --git a/src/tools/pginclude/headerscheck b/src/tools/pginclude/headerscheck index 785d6f867ad..378100acf2c 100755 --- a/src/tools/pginclude/headerscheck +++ b/src/tools/pginclude/headerscheck @@ -107,16 +107,6 @@ do test "$f" = src/include/port/win32ntdll.h && continue test "$f" = src/port/pthread-win32.h && continue - # Likewise, these files are platform-specific, and the one - # relevant to our platform will be included by atomics.h. - test "$f" = src/include/port/atomics/arch-arm.h && continue - test "$f" = src/include/port/atomics/arch-ppc.h && continue - test "$f" = src/include/port/atomics/arch-x86.h && continue - test "$f" = src/include/port/atomics/fallback.h && continue - test "$f" = src/include/port/atomics/generic.h && continue - test "$f" = src/include/port/atomics/generic-gcc.h && continue - test "$f" = src/include/port/atomics/generic-msvc.h && continue - # sepgsql.h depends on headers that aren't there on most platforms. test "$f" = contrib/sepgsql/sepgsql.h && continue -- 2.50.1