From 97fa7ee59fcbd76d096f9504c2b513ef9741d32a Mon Sep 17 00:00:00 2001 From: Greg Burd Date: Sat, 11 Jul 2026 18:42:50 -0400 Subject: [PATCH v8 3/3] [NOT FOR MERGE... yet.] Remove traditional atomics; use C11 stdatomic.h exclusively This commit is included to show the intended long-term destination of the series, and to let reviewers and the buildfarm exercise the stdatomic-only end state -- but it is deliberately NOT proposed for commit yet. The plan is to ship the earlier patches (which add C11 as a *selectable* implementation alongside the traditional platform-specific code), let the stdatomic path prove itself in the field across the buildfarm and at least one release, and only THEN remove the traditional implementation via a commit like this one. Removing the battle-tested arch-*/generic-* code on day one is not being asked for here; keeping both paths until stdatomic has earned trust is. Everything below describes what this (eventual) removal commit does. This collapses the dual-path atomics into a single stdatomic-only implementation, mirroring the destination state of the series. Rather than selecting between C11 and the platform-specific implementations at build time, stdatomic.h is now the sole backend for PostgreSQL's atomic operations and USE_STDATOMIC_H is effectively always-on. The platform-specific implementation files are deleted outright: - src/include/port/atomics/arch-arm.h - src/include/port/atomics/arch-ppc.h - src/include/port/atomics/arch-x86.h - src/include/port/atomics/generic.h - src/include/port/atomics/generic-gcc.h - src/include/port/atomics/generic-msvc.h - src/include/port/atomics/fallback.h atomics.h drops its #ifdef USE_STDATOMIC_H / #else conditional and the now-dead includes of the arch-*/generic-*/fallback headers, keeping only the stdatomic_impl.h include and the common public-API layer of static inline pg_atomic_* wrappers. storage/spin.h and s_lock.c lose their traditional-spinlock (#else) branches, so only the pg_atomic_flag-based spinlock path remains; the s_lock.h includes that served only that path are removed. Detection is retained but now hard-errors when a working C11 stdatomic.h is unavailable, in both meson.build and configure.ac, instead of silently falling back. The only remaining hand-written platform hint is the CPU spin-delay instruction in src/include/port/spin_delay.h, which stdatomic.h does not cover; it is intentionally left in place along with spin_delay_status.h and stdatomic_impl.h. This depends on the prior commits in the series (build-time detection, stdatomic_impl.h, conditional wire-up, documentation, and the test_atomics module). Co-authored-by: Greg Burd Co-authored-by: Thomas Munro (cherry picked from commit 252a8199c2ad96fb89e745e432b2bdf105604542) (cherry picked from commit dab4b758cc6e483b84c3b0f143db48589aa608e7) (cherry picked from commit 71f185eb4707414831f8051721b40fc30bb0deb2) (cherry picked from commit 974e9bc70f42809b4e1c9093bca9dcfb42154c5d) (cherry picked from commit ed2bf84be23dab6dbb7b0c8c0710a54d191e9ae6) (cherry picked from commit 962764d081160c37acbeeb8b4e07f4a1fb8a589f) (cherry picked from commit 1b5a4458d6b4d073dadbaf5e31fbfb332dc96405) --- configure | 80 +-- configure.ac | 79 +-- meson.build | 45 +- meson_options.txt | 5 - src/backend/storage/lmgr/.gitignore | 1 - src/backend/storage/lmgr/Makefile | 9 - src/backend/storage/lmgr/s_lock.c | 125 ---- src/include/port/atomics.h | 183 +----- src/include/port/atomics/arch-arm.h | 32 -- src/include/port/atomics/arch-ppc.h | 256 --------- src/include/port/atomics/arch-x86.h | 189 ------ src/include/port/atomics/fallback.h | 42 -- src/include/port/atomics/generic-gcc.h | 326 ----------- src/include/port/atomics/generic-msvc.h | 115 ---- src/include/port/atomics/generic.h | 430 -------------- src/include/storage/s_lock.h | 734 ------------------------ src/include/storage/spin.h | 26 - src/test/regress/regress.c | 39 +- src/tools/pginclude/headerscheck | 10 - 19 files changed, 92 insertions(+), 2634 deletions(-) delete mode 100644 src/include/port/atomics/arch-arm.h delete mode 100644 src/include/port/atomics/arch-ppc.h delete mode 100644 src/include/port/atomics/arch-x86.h delete mode 100644 src/include/port/atomics/fallback.h delete mode 100644 src/include/port/atomics/generic-gcc.h delete mode 100644 src/include/port/atomics/generic-msvc.h delete mode 100644 src/include/port/atomics/generic.h delete mode 100644 src/include/storage/s_lock.h diff --git a/configure b/configure index 94946b045cd..8bc6601f3ad 100755 --- a/configure +++ b/configure @@ -760,7 +760,6 @@ LLVM_LIBS CLANG LLVM_CONFIG AWK -with_stdatomic with_llvm have_cxx ac_ct_CXX @@ -859,7 +858,6 @@ with_segsize with_segsize_blocks with_wal_blocksize with_llvm -with_stdatomic enable_depend enable_cassert with_icu @@ -1579,8 +1577,6 @@ Optional Packages: --with-wal-blocksize=BLOCKSIZE set WAL block size in kB [8] --with-llvm build with LLVM based JIT support - --with-stdatomic[=yes/no/auto] - use C11 stdatomic.h for atomic operations [auto] --without-icu build without ICU support --with-tcl build Tcl modules (PL/Tcl) --with-tclconfig=DIR tclConfig.sh is in DIR @@ -4926,22 +4922,9 @@ fi # # C11 stdatomic.h # -# Accepts --with-stdatomic={yes,no,auto}. "auto" (the default) uses -# stdatomic.h if a compile test succeeds; "yes" requires it; "no" forces the -# traditional platform-specific atomics. The actual detection runs later, -# after the compiler and 64-bit integer handling have been configured. - -# Check whether --with-stdatomic was given. -if test "${with_stdatomic+set}" = set; then : - withval=$with_stdatomic; case $withval in - yes | no | auto) ;; - *) as_fn_error $? "invalid argument to --with-stdatomic; use yes, no, or auto" "$LINENO" 5 ;; - esac -else - with_stdatomic=no -fi - - +# stdatomic.h is the sole atomics implementation. The actual detection runs +# later, after the compiler and 64-bit integer handling have been configured; +# a missing or broken header is a hard error there. for ac_prog in gawk mawk nawk awk do # Extract the first word of "$ac_prog", so it can be a program name with args. @@ -17725,32 +17708,30 @@ $as_echo "#define HAVE_GCC__ATOMIC_INT64_CAS 1" >>confdefs.h fi -# Decide whether to use C11 stdatomic.h (see --with-stdatomic above). The -# probe mirrors the one used by the meson build: it must compile 32-bit -# fetch_add, a seq_cst thread fence, and a 64-bit compare-exchange. -if test "$with_stdatomic" != no; then - # Use a link test, not a compile test: 64-bit atomic_compare_exchange_strong() - # can lower to a libatomic call (__atomic_compare_exchange_8) on some 32-bit - # or weakly-aligned targets. A compile-only check would pass there and then - # fail at final link. Retry with -latomic if the bare link fails. - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for working C11 stdatomic.h" >&5 +# Require a working C11 stdatomic.h: it is the sole atomics implementation, +# so a missing or broken header is a hard error. Use a link test, not a +# compile test: 64-bit atomic_compare_exchange_strong() can lower to a +# libatomic call (__atomic_compare_exchange_8) on some 32-bit or +# weakly-aligned targets, where a compile-only check would pass and then fail +# at final link. Retry with -latomic if the bare link fails. +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for working C11 stdatomic.h" >&5 $as_echo_n "checking for working C11 stdatomic.h... " >&6; } if ${pgac_cv_stdatomic+:} false; then : $as_echo_n "(cached) " >&6 else pgac_cv_stdatomic=no - cat confdefs.h - <<_ACEOF >conftest.$ac_ext + cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include int main () { _Atomic int x = 0; - atomic_fetch_add(&x, 1); - atomic_thread_fence(memory_order_seq_cst); - _Atomic(unsigned long long) y = 0; - unsigned long long expected = 0; - atomic_compare_exchange_strong(&y, &expected, 42); + atomic_fetch_add(&x, 1); + atomic_thread_fence(memory_order_seq_cst); + _Atomic(unsigned long long) y = 0; + unsigned long long expected = 0; + atomic_compare_exchange_strong(&y, &expected, 42); ; return 0; @@ -17760,19 +17741,19 @@ if ac_fn_c_try_link "$LINENO"; then : pgac_cv_stdatomic=yes else pgac_save_LIBS=$LIBS - LIBS="$LIBS -latomic" - cat confdefs.h - <<_ACEOF >conftest.$ac_ext + LIBS="$LIBS -latomic" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ #include int main () { _Atomic int x = 0; - atomic_fetch_add(&x, 1); - atomic_thread_fence(memory_order_seq_cst); - _Atomic(unsigned long long) y = 0; - unsigned long long expected = 0; - atomic_compare_exchange_strong(&y, &expected, 42); + atomic_fetch_add(&x, 1); + atomic_thread_fence(memory_order_seq_cst); + _Atomic(unsigned long long) y = 0; + unsigned long long expected = 0; + atomic_compare_exchange_strong(&y, &expected, 42); ; return 0; @@ -17785,28 +17766,23 @@ else fi rm -f core conftest.err conftest.$ac_objext \ conftest$ac_exeext conftest.$ac_ext - LIBS=$pgac_save_LIBS + LIBS=$pgac_save_LIBS fi rm -f core conftest.err conftest.$ac_objext \ conftest$ac_exeext conftest.$ac_ext fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $pgac_cv_stdatomic" >&5 $as_echo "$pgac_cv_stdatomic" >&6; } -else - pgac_cv_stdatomic=no -fi -if test "$with_stdatomic" = yes && test "$pgac_cv_stdatomic" = no; then - as_fn_error $? "--with-stdatomic=yes was given, but a working C11 stdatomic.h could not be found" "$LINENO" 5 +if test "$pgac_cv_stdatomic" = no; then + as_fn_error $? "a working C11 stdatomic.h is required (it is the only atomics implementation)" "$LINENO" 5 fi -if test "$pgac_cv_stdatomic" != no; then $as_echo "#define USE_STDATOMIC_H 1" >>confdefs.h - if test "$pgac_cv_stdatomic" = 'yes, with -latomic'; then - LIBS="$LIBS -latomic" - fi +if test "$pgac_cv_stdatomic" = 'yes, with -latomic'; then + LIBS="$LIBS -latomic" fi diff --git a/configure.ac b/configure.ac index 8d3c657c069..2b274bf0e9f 100644 --- a/configure.ac +++ b/configure.ac @@ -447,19 +447,9 @@ AC_SUBST(with_llvm) # # C11 stdatomic.h # -# Accepts --with-stdatomic={yes,no,auto}. "auto" (the default) uses -# stdatomic.h if a compile test succeeds; "yes" requires it; "no" forces the -# traditional platform-specific atomics. The actual detection runs later, -# after the compiler and 64-bit integer handling have been configured. -AC_ARG_WITH(stdatomic, - [AS_HELP_STRING([--with-stdatomic@<:@=yes/no/auto@:>@], - [use C11 stdatomic.h for atomic operations @<:@auto@:>@])], - [case $withval in - yes | no | auto) ;; - *) AC_MSG_ERROR([invalid argument to --with-stdatomic; use yes, no, or auto]) ;; - esac], - [with_stdatomic=no]) -AC_SUBST(with_stdatomic) +# stdatomic.h is the sole atomics implementation. The actual detection runs +# later, after the compiler and 64-bit integer handling have been configured; +# a missing or broken header is a hard error there. dnl must use AS_IF here, else AC_REQUIRES inside PGAC_LLVM_SUPPORT malfunctions AS_IF([test "$with_llvm" = yes], [ PGAC_LLVM_SUPPORT() @@ -2129,16 +2119,25 @@ PGAC_HAVE_GCC__ATOMIC_INT32_CAS PGAC_HAVE_GCC__ATOMIC_INT64_CAS -# Decide whether to use C11 stdatomic.h (see --with-stdatomic above). The -# probe mirrors the one used by the meson build: it must compile 32-bit -# fetch_add, a seq_cst thread fence, and a 64-bit compare-exchange. -if test "$with_stdatomic" != no; then - # Use a link test, not a compile test: 64-bit atomic_compare_exchange_strong() - # can lower to a libatomic call (__atomic_compare_exchange_8) on some 32-bit - # or weakly-aligned targets. A compile-only check would pass there and then - # fail at final link. Retry with -latomic if the bare link fails. - AC_CACHE_CHECK([for working C11 stdatomic.h], [pgac_cv_stdatomic], - [pgac_cv_stdatomic=no +# Require a working C11 stdatomic.h: it is the sole atomics implementation, +# so a missing or broken header is a hard error. Use a link test, not a +# compile test: 64-bit atomic_compare_exchange_strong() can lower to a +# libatomic call (__atomic_compare_exchange_8) on some 32-bit or +# weakly-aligned targets, where a compile-only check would pass and then fail +# at final link. Retry with -latomic if the bare link fails. +AC_CACHE_CHECK([for working C11 stdatomic.h], [pgac_cv_stdatomic], +[pgac_cv_stdatomic=no + AC_LINK_IFELSE([AC_LANG_PROGRAM([#include ], + [[_Atomic int x = 0; + atomic_fetch_add(&x, 1); + atomic_thread_fence(memory_order_seq_cst); + _Atomic(unsigned long long) y = 0; + unsigned long long expected = 0; + atomic_compare_exchange_strong(&y, &expected, 42); + ]])], + [pgac_cv_stdatomic=yes], + [pgac_save_LIBS=$LIBS + LIBS="$LIBS -latomic" AC_LINK_IFELSE([AC_LANG_PROGRAM([#include ], [[_Atomic int x = 0; atomic_fetch_add(&x, 1); @@ -2147,34 +2146,18 @@ if test "$with_stdatomic" != no; then unsigned long long expected = 0; atomic_compare_exchange_strong(&y, &expected, 42); ]])], - [pgac_cv_stdatomic=yes], - [pgac_save_LIBS=$LIBS - LIBS="$LIBS -latomic" - AC_LINK_IFELSE([AC_LANG_PROGRAM([#include ], - [[_Atomic int x = 0; - atomic_fetch_add(&x, 1); - atomic_thread_fence(memory_order_seq_cst); - _Atomic(unsigned long long) y = 0; - unsigned long long expected = 0; - atomic_compare_exchange_strong(&y, &expected, 42); - ]])], - [pgac_cv_stdatomic='yes, with -latomic'], - [pgac_cv_stdatomic=no]) - LIBS=$pgac_save_LIBS])]) -else - pgac_cv_stdatomic=no -fi + [pgac_cv_stdatomic='yes, with -latomic'], + [pgac_cv_stdatomic=no]) + LIBS=$pgac_save_LIBS])]) -if test "$with_stdatomic" = yes && test "$pgac_cv_stdatomic" = no; then - AC_MSG_ERROR([--with-stdatomic=yes was given, but a working C11 stdatomic.h could not be found]) +if test "$pgac_cv_stdatomic" = no; then + AC_MSG_ERROR([a working C11 stdatomic.h is required (it is the only atomics implementation)]) fi -if test "$pgac_cv_stdatomic" != no; then - AC_DEFINE([USE_STDATOMIC_H], 1, - [Define to 1 to use C11 stdatomic.h for atomic operations. (--with-stdatomic)]) - if test "$pgac_cv_stdatomic" = 'yes, with -latomic'; then - LIBS="$LIBS -latomic" - fi +AC_DEFINE([USE_STDATOMIC_H], 1, + [Define to 1 to use C11 stdatomic.h for atomic operations.]) +if test "$pgac_cv_stdatomic" = 'yes, with -latomic'; then + LIBS="$LIBS -latomic" fi diff --git a/meson.build b/meson.build index 97ec150814c..e56aa587fae 100644 --- a/meson.build +++ b/meson.build @@ -692,7 +692,6 @@ int main(void) { ''' has_stdatomic = false -use_stdatomic = get_option('use_stdatomic') atomic_link_args = [] # The stdatomic probe below is a *link* test, not just a compile test: 64-bit @@ -744,36 +743,20 @@ else endif endif -# Process the use_stdatomic option -if use_stdatomic == 'yes' - if not has_stdatomic - error('stdatomic.h was requested with -Duse_stdatomic=yes but is not available or not working') - endif - cdata.set('USE_STDATOMIC_H', 1) - message('Using C11 stdatomic.h for atomic operations (forced by -Duse_stdatomic=yes)') - if cc.get_id() == 'msvc' - # MSVC needs C11 mode plus the experimental atomics switch for . - cflags += ['/std:c11', '/experimental:c11atomics'] - endif - if atomic_link_args.length() > 0 - os_deps += cc.find_library('atomic') - endif -elif use_stdatomic == 'no' - message('Using traditional platform-specific atomics (forced by -Duse_stdatomic=no)') - # Do not set USE_STDATOMIC_H -elif use_stdatomic == 'auto' - if has_stdatomic - cdata.set('USE_STDATOMIC_H', 1) - message('Using C11 stdatomic.h for atomic operations (auto-detected)') - if cc.get_id() == 'msvc' - cflags += ['/std:c11', '/experimental:c11atomics'] - endif - if atomic_link_args.length() > 0 - os_deps += cc.find_library('atomic') - endif - else - message('Using traditional platform-specific atomics (stdatomic.h not available)') - endif +# stdatomic.h is the sole atomics implementation, so a missing or broken +# header is a hard error. +if not has_stdatomic + error('a working C11 stdatomic.h is required (it is the only atomics implementation)') +endif +cdata.set('USE_STDATOMIC_H', 1) +message('Using C11 stdatomic.h for atomic operations') +if atomic_link_args.length() > 0 + # libatomic is needed for atomics the compiler cannot inline on this target. + os_deps += cc.find_library('atomic') +endif +if cc.get_id() == 'msvc' + # MSVC needs C11 mode plus the experimental atomics switch for . + cflags += ['/std:c11', '/experimental:c11atomics'] endif postgres_inc = [include_directories(postgres_inc_d)] diff --git a/meson_options.txt b/meson_options.txt index f8c5ccc14e1..6a793f3e479 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -52,11 +52,6 @@ option('PG_TEST_EXTRA', type: 'string', value: '', option('PG_GIT_REVISION', type: 'string', value: 'HEAD', description: 'git revision to be packaged by pgdist target') -option('use_stdatomic', type: 'combo', - choices: ['auto', 'yes', 'no'], - value: 'no', - description: 'Use C11 stdatomic.h for atomic operations (auto, yes, no)') - # Compilation options diff --git a/src/backend/storage/lmgr/.gitignore b/src/backend/storage/lmgr/.gitignore index 8e5b734f152..209c8be7223 100644 --- a/src/backend/storage/lmgr/.gitignore +++ b/src/backend/storage/lmgr/.gitignore @@ -1,2 +1 @@ /lwlocknames.h -/s_lock_test diff --git a/src/backend/storage/lmgr/Makefile b/src/backend/storage/lmgr/Makefile index a5fbc24ddad..a60e70ef0e1 100644 --- a/src/backend/storage/lmgr/Makefile +++ b/src/backend/storage/lmgr/Makefile @@ -24,17 +24,8 @@ OBJS = \ include $(top_srcdir)/src/backend/common.mk -s_lock_test: s_lock.c $(top_builddir)/src/common/libpgcommon.a $(top_builddir)/src/port/libpgport.a - $(CC) $(CPPFLAGS) $(CFLAGS) -DS_LOCK_TEST=1 $(srcdir)/s_lock.c \ - -L $(top_builddir)/src/common -lpgcommon \ - -L $(top_builddir)/src/port -lpgport -lm -o s_lock_test - lwlocknames.h: ../../../include/storage/lwlocklist.h ../../utils/activity/wait_event_names.txt generate-lwlocknames.pl $(PERL) $(srcdir)/generate-lwlocknames.pl $^ -check: s_lock_test - ./s_lock_test - clean: - rm -f s_lock_test rm -f lwlocknames.h diff --git a/src/backend/storage/lmgr/s_lock.c b/src/backend/storage/lmgr/s_lock.c index 52fdf9fd69a..8caaa562ced 100644 --- a/src/backend/storage/lmgr/s_lock.c +++ b/src/backend/storage/lmgr/s_lock.c @@ -51,13 +51,9 @@ #include #include "common/pg_prng.h" -#ifdef USE_STDATOMIC_H #include "port/atomics.h" #include "port/spin_delay.h" #include "storage/spin.h" -#else -#include "storage/s_lock.h" -#endif #include "utils/wait_event.h" #define MIN_SPINS_PER_DELAY 10 @@ -66,17 +62,6 @@ #define MIN_DELAY_USEC 1000L #define MAX_DELAY_USEC 1000000L -#ifndef USE_STDATOMIC_H -#ifdef S_LOCK_TEST -/* - * These are needed by pgstat_report_wait_start in the standalone compile of - * s_lock_test. - */ -static uint32 local_my_wait_event_info; -uint32 *my_wait_event_info = &local_my_wait_event_info; -#endif -#endif /* !USE_STDATOMIC_H */ - static int spins_per_delay = DEFAULT_SPINS_PER_DELAY; @@ -88,22 +73,10 @@ s_lock_stuck(const char *file, int line, const char *func) { if (!func) func = "(unknown)"; -#if !defined(USE_STDATOMIC_H) && defined(S_LOCK_TEST) - fprintf(stderr, - "\nStuck spinlock detected at %s, %s:%d.\n", - func, file, line); - exit(1); -#else elog(PANIC, "stuck spinlock detected at %s, %s:%d", func, file, line); -#endif } -/* - * USE_DEFAULT_S_LOCK is defined by s_lock.h, which the stdatomic path does - * not include; spin.h calls s_lock() unconditionally, so define it for both. - */ -#if defined(USE_STDATOMIC_H) || defined(USE_DEFAULT_S_LOCK) /* * s_lock(lock) - platform-independent portion of waiting for a spinlock. */ @@ -114,7 +87,6 @@ s_lock(volatile slock_t *lock, const char *file, int line, const char *func) init_spin_delay(&delayStatus, file, line, func); -#ifdef USE_STDATOMIC_H for (;;) { bool try_to_set; @@ -134,28 +106,11 @@ s_lock(volatile slock_t *lock, const char *file, int line, const char *func) break; perform_spin_delay(&delayStatus); } -#else /* !USE_STDATOMIC_H */ - while (TAS_SPIN(lock)) - { - perform_spin_delay(&delayStatus); - } -#endif /* USE_STDATOMIC_H */ finish_spin_delay(&delayStatus); return delayStatus.delays; } -#endif - -#ifndef USE_STDATOMIC_H -#ifdef USE_DEFAULT_S_UNLOCK -void -s_unlock(volatile slock_t *lock) -{ - *lock = 0; -} -#endif -#endif /* !USE_STDATOMIC_H */ /* * Wait while spinning on a contended spinlock. @@ -164,11 +119,7 @@ void perform_spin_delay(SpinDelayStatus *status) { /* CPU-specific delay each time through the loop */ -#ifdef USE_STDATOMIC_H pg_spin_delay(); -#else - SPIN_DELAY(); -#endif /* Block the process every spins_per_delay tries */ if (++(status->spins) >= spins_per_delay) @@ -191,11 +142,6 @@ perform_spin_delay(SpinDelayStatus *status) pg_usleep(status->cur_delay); pgstat_report_wait_end(); -#if !defined(USE_STDATOMIC_H) && defined(S_LOCK_TEST) - fprintf(stdout, "*"); - fflush(stdout); -#endif - /* increase delay by a random fraction between 1X and 2X */ status->cur_delay += (int) (status->cur_delay * pg_prng_double(&pg_global_prng_state) + 0.5); @@ -271,74 +217,3 @@ update_spins_per_delay(int shared_spins_per_delay) */ return (shared_spins_per_delay * 15 + spins_per_delay) / 16; } - - -/*****************************************************************************/ -#ifndef USE_STDATOMIC_H -#if defined(S_LOCK_TEST) - -/* - * test program for verifying a port's spinlock support. - */ - -struct test_lock_struct -{ - char pad1; - slock_t lock; - char pad2; -}; - -volatile struct test_lock_struct test_lock; - -int -main() -{ - pg_prng_seed(&pg_global_prng_state, (uint64) time(NULL)); - - test_lock.pad1 = test_lock.pad2 = 0x44; - - S_INIT_LOCK(&test_lock.lock); - - if (test_lock.pad1 != 0x44 || test_lock.pad2 != 0x44) - { - printf("S_LOCK_TEST: failed, declared datatype is wrong size\n"); - return 1; - } - - S_LOCK(&test_lock.lock); - - if (test_lock.pad1 != 0x44 || test_lock.pad2 != 0x44) - { - printf("S_LOCK_TEST: failed, declared datatype is wrong size\n"); - return 1; - } - - S_UNLOCK(&test_lock.lock); - - if (test_lock.pad1 != 0x44 || test_lock.pad2 != 0x44) - { - printf("S_LOCK_TEST: failed, declared datatype is wrong size\n"); - return 1; - } - - S_LOCK(&test_lock.lock); - - if (test_lock.pad1 != 0x44 || test_lock.pad2 != 0x44) - { - printf("S_LOCK_TEST: failed, declared datatype is wrong size\n"); - return 1; - } - - printf("S_LOCK_TEST: this will print %d stars and then\n", NUM_DELAYS); - printf(" exit with a 'stuck spinlock' message\n"); - printf(" if S_LOCK() and TAS() are working.\n"); - fflush(stdout); - - S_LOCK(&test_lock.lock); - - printf("S_LOCK_TEST: failed, lock not locked\n"); - return 1; -} - -#endif /* S_LOCK_TEST */ -#endif /* !USE_STDATOMIC_H */ diff --git a/src/include/port/atomics.h b/src/include/port/atomics.h index 2058dbb9238..0377a65a4af 100644 --- a/src/include/port/atomics.h +++ b/src/include/port/atomics.h @@ -7,35 +7,17 @@ * atomically and dealing with cache coherency. Used to implement locking * facilities and lockless algorithms/data structures. * - * IMPLEMENTATION SELECTION: + * IMPLEMENTATION: * - * PostgreSQL's atomic operations can be implemented using either: + * PostgreSQL's atomic operations are implemented on top of C11 + * (see stdatomic_impl.h). This standard header supplies all + * atomic types and operations, so no architecture-specific assembly or + * compiler intrinsics are required here. A working C11 stdatomic.h is a + * hard build requirement (USE_STDATOMIC_H is always defined). * - * 1. C11 stdatomic.h (when USE_STDATOMIC_H is defined) - * - Uses standard C11 for all atomic operations - * - Better portability and compiler optimizations - * - Requires MSVC 2022+ or GCC 4.9+/Clang 3.1+ - * - * 2. Traditional platform-specific implementations (when USE_STDATOMIC_H is not defined) - * - Uses battle-tested PostgreSQL implementations - * - Architecture-specific: arch-x86.h, arch-arm.h, arch-ppc.h - * - Compiler intrinsics: generic-gcc.h, generic-msvc.h - * - Fallback implementations: generic.h, fallback.h - * - * Both implementations provide identical public API and semantics. - * Selection is made at build time via -Duse_stdatomic=auto/yes/no. - * - * PORTING NOTES: - * - * To bring up postgres on a platform/compiler at the very least - * implementations for the following operations should be provided: - * * pg_compiler_barrier(), pg_write_barrier(), pg_read_barrier() - * * pg_atomic_compare_exchange_u32(), pg_atomic_fetch_add_u32() - * * pg_atomic_test_set_flag(), pg_atomic_init_flag(), pg_atomic_clear_flag() - * * PG_HAVE_8BYTE_SINGLE_COPY_ATOMICITY should be defined if appropriate. - * - * For new platforms, prefer using stdatomic.h if available, as it reduces - * maintenance burden and leverages compiler-provided implementations. + * The CPU spin-delay instruction, which stdatomic.h does not cover, is in + * port/spin_delay.h. The memory-ordering and flag-width choices are + * documented in port/atomics/stdatomic_impl.h. * * Use higher level functionality (lwlocks, spinlocks, heavyweight locks) * whenever possible. Writing correct code using these facilities is hard. @@ -53,155 +35,26 @@ #ifndef ATOMICS_H #define ATOMICS_H -/* - * Frontend code restriction - * - * Traditionally, atomics.h could not be included from frontend code because - * the platform-specific implementations often relied on backend-only features. - * - * With the stdatomic.h implementation, this restriction is no longer necessary - * since stdatomic.h is a standard header. However, we keep the restriction for - * the traditional implementation path to maintain compatibility. - */ -#ifdef FRONTEND -#ifndef USE_STDATOMIC_H -#error "atomics.h may not be included from frontend code (use -Duse_stdatomic=yes if atomics are needed)" -#endif -#endif - -#define INSIDE_ATOMICS_H - /* * The public API and the _impl signatures keep the historical volatile * qualifier on the atomic pointer, because many callers reach atomic fields - * through volatile-qualified struct pointers. Under C11 the _Atomic types - * carry their own ordering guarantees, so the volatile is redundant for the - * atomic accesses themselves and is accepted (and effectively ignored for the - * atomic operation) by the supported compilers, including MSVC. + * through volatile-qualified struct pointers. Passing a volatile-qualified + * _Atomic pointer to the C11 generic operations is well defined: the access + * remains fully atomic, and the qualifier is retained purely for source + * compatibility. */ #include /* - * PostgreSQL atomics can be implemented using either C11 stdatomic.h - * or platform-specific implementations. The choice is made at build - * configuration time via the USE_STDATOMIC_H preprocessor definition. - * - * When USE_STDATOMIC_H is defined, we use C11 for all - * atomic operations. This provides better portability and potentially - * better compiler optimizations. - * - * When USE_STDATOMIC_H is not defined (default initially), we use - * traditional platform-specific implementations (arch-*.h, generic-*.h) - * that have been battle-tested in PostgreSQL for many years. - * - * Both paths provide identical public API and semantics. - */ - -#ifdef USE_STDATOMIC_H - -/* - * C11 stdatomic.h implementation path - * - * This uses the standard C11 header for atomic operations. - * The type definitions and implementations are in stdatomic_impl.h. + * All atomic types and _impl operations come from C11 , wrapped + * in stdatomic_impl.h. */ - #include "port/atomics/stdatomic_impl.h" /* - * The public API is provided by static inline functions in the common code - * section below. Those functions call _impl functions, which are provided by - * stdatomic_impl.h for this path. - */ - -#else /* !USE_STDATOMIC_H */ - -/* - * Traditional platform-specific implementation path - * - * This is the original PostgreSQL atomics implementation using - * architecture-specific headers and compiler intrinsics. - * - * First a set of architecture specific files is included. - * - * These files can provide the full set of atomics or can do pretty much - * nothing if all the compilers commonly used on these platforms provide - * usable generics. - * - * Don't add an inline assembly of the actual atomic operations if all the - * common implementations of your platform provide intrinsics. Intrinsics are - * much easier to understand and potentially support more architectures. - * - * It will often make sense to define memory barrier semantics here, since - * e.g. generic compiler intrinsics for x86 memory barriers can't know that - * postgres doesn't need x86 read/write barriers do anything more than a - * compiler barrier. - * - */ -#if defined(__arm__) || defined(__aarch64__) -#include "port/atomics/arch-arm.h" -#elif defined(__i386__) || defined(__x86_64__) -#include "port/atomics/arch-x86.h" -#elif defined(__powerpc__) || defined(__powerpc64__) -#include "port/atomics/arch-ppc.h" -#endif - -/* - * Compiler specific, but architecture independent implementations. - * - * Provide architecture independent implementations of the atomic - * facilities. At the very least compiler barriers should be provided, but a - * full implementation of - * * pg_compiler_barrier(), pg_write_barrier(), pg_read_barrier() - * * pg_atomic_compare_exchange_u32(), pg_atomic_fetch_add_u32() - * using compiler intrinsics are a good idea. - */ -/* - * gcc or compatible, including clang and icc. - */ -#if defined(__GNUC__) || defined(__INTEL_COMPILER) -#include "port/atomics/generic-gcc.h" -#elif defined(_MSC_VER) -#include "port/atomics/generic-msvc.h" -#else -/* Unknown compiler. */ -#endif - -/* Fail if we couldn't find implementations of required facilities. */ -#if !defined(PG_HAVE_ATOMIC_U32_SUPPORT) -#error "could not find an implementation of pg_atomic_uint32" -#endif -#if !defined(pg_compiler_barrier_impl) -#error "could not find an implementation of pg_compiler_barrier" -#endif -#if !defined(pg_memory_barrier_impl) -#error "could not find an implementation of pg_memory_barrier_impl" -#endif - - -/* - * Provide a spinlock-based implementation of the 64 bit variants, if - * necessary. - */ -#include "port/atomics/fallback.h" - -/* - * Provide additional operations using supported infrastructure. These are - * expected to be efficient if the underlying atomic operations are efficient. - */ -#include "port/atomics/generic.h" - -#endif /* USE_STDATOMIC_H */ - -/* - * Common code for both stdatomic.h and traditional implementations. - * - * The following definitions provide the public API that works identically - * regardless of which implementation is used. The static inline functions - * below call _impl functions which are provided by either stdatomic_impl.h - * (when USE_STDATOMIC_H is defined) or the traditional implementation files - * (generic.h, etc.) otherwise. + * The public API below is provided by static inline functions that call the + * _impl functions supplied by stdatomic_impl.h. */ /* @@ -698,6 +551,4 @@ pg_atomic_monotonic_advance_u64(volatile pg_atomic_uint64 *ptr, uint64 target) return currval; } -#undef INSIDE_ATOMICS_H - #endif /* ATOMICS_H */ diff --git a/src/include/port/atomics/arch-arm.h b/src/include/port/atomics/arch-arm.h deleted file mode 100644 index 90280c7b751..00000000000 --- a/src/include/port/atomics/arch-arm.h +++ /dev/null @@ -1,32 +0,0 @@ -/*------------------------------------------------------------------------- - * - * arch-arm.h - * Atomic operations considerations specific to ARM - * - * Portions Copyright (c) 2013-2026, PostgreSQL Global Development Group - * - * NOTES: - * - * src/include/port/atomics/arch-arm.h - * - *------------------------------------------------------------------------- - */ - -/* intentionally no include guards, should only be included by atomics.h */ -#ifndef INSIDE_ATOMICS_H -#error "should be included via atomics.h" -#endif - -/* - * 64 bit atomics on ARM32 are implemented using kernel fallbacks and thus - * might be slow, so disable entirely. On ARM64 that problem doesn't exist. - */ -#if !defined(__aarch64__) -#define PG_DISABLE_64_BIT_ATOMICS -#else -/* - * Architecture Reference Manual for ARMv8 states aligned read/write to/from - * general purpose register is atomic. - */ -#define PG_HAVE_8BYTE_SINGLE_COPY_ATOMICITY -#endif /* __aarch64__ */ diff --git a/src/include/port/atomics/arch-ppc.h b/src/include/port/atomics/arch-ppc.h deleted file mode 100644 index 65ced2c6d3a..00000000000 --- a/src/include/port/atomics/arch-ppc.h +++ /dev/null @@ -1,256 +0,0 @@ -/*------------------------------------------------------------------------- - * - * arch-ppc.h - * Atomic operations considerations specific to PowerPC - * - * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group - * Portions Copyright (c) 1994, Regents of the University of California - * - * NOTES: - * - * src/include/port/atomics/arch-ppc.h - * - *------------------------------------------------------------------------- - */ - -#if defined(__GNUC__) - -/* - * lwsync orders loads with respect to each other, and similarly with stores. - * But a load can be performed before a subsequent store, so sync must be used - * for a full memory barrier. - */ -#define pg_memory_barrier_impl() __asm__ __volatile__ ("sync" : : : "memory") -#define pg_read_barrier_impl() __asm__ __volatile__ ("lwsync" : : : "memory") -#define pg_write_barrier_impl() __asm__ __volatile__ ("lwsync" : : : "memory") -#endif - -#define PG_HAVE_ATOMIC_U32_SUPPORT -typedef struct pg_atomic_uint32 -{ - volatile uint32 value; -} pg_atomic_uint32; - -/* 64bit atomics are only supported in 64bit mode */ -#if SIZEOF_VOID_P >= 8 -#define PG_HAVE_ATOMIC_U64_SUPPORT -typedef struct pg_atomic_uint64 -{ - alignas(8) volatile uint64 value; -} pg_atomic_uint64; - -#endif - -/* - * This mimics gcc __atomic_compare_exchange_n(..., __ATOMIC_SEQ_CST), but - * code generation differs at the end. __atomic_compare_exchange_n(): - * 100: isync - * 104: mfcr r3 - * 108: rlwinm r3,r3,3,31,31 - * 10c: bne 120 <.eb+0x10> - * 110: clrldi r3,r3,63 - * 114: addi r1,r1,112 - * 118: blr - * 11c: nop - * 120: clrldi r3,r3,63 - * 124: stw r9,0(r4) - * 128: addi r1,r1,112 - * 12c: blr - * - * This: - * f0: isync - * f4: mfcr r9 - * f8: rldicl. r3,r9,35,63 - * fc: bne 104 <.eb> - * 100: stw r10,0(r4) - * 104: addi r1,r1,112 - * 108: blr - * - * This implementation may or may not have materially different performance. - * It's not exploiting the fact that cr0 still holds the relevant comparison - * bits, set during the __asm__. One could fix that by moving more code into - * the __asm__. (That would remove the freedom to eliminate dead stores when - * the caller ignores "expected", but few callers do.) - * - * Recognizing constant "newval" would be superfluous, because there's no - * immediate-operand version of stwcx. - */ -#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32 -static inline bool -pg_atomic_compare_exchange_u32_impl(volatile pg_atomic_uint32 *ptr, - uint32 *expected, uint32 newval) -{ - uint32 found; - uint32 condition_register; - bool ret; - -#ifdef HAVE_I_CONSTRAINT__BUILTIN_CONSTANT_P - if (__builtin_constant_p(*expected) && - (int32) *expected <= PG_INT16_MAX && - (int32) *expected >= PG_INT16_MIN) - __asm__ __volatile__( - " sync \n" - " lwarx %0,0,%5,1 \n" - " cmpwi %0,%3 \n" - " bne $+12 \n" /* branch to lwsync */ - " stwcx. %4,0,%5 \n" - " bne $-16 \n" /* branch to lwarx */ - " lwsync \n" - " mfcr %1 \n" -: "=&r"(found), "=r"(condition_register), "+m"(ptr->value) -: "i"(*expected), "r"(newval), "r"(&ptr->value) -: "memory", "cc"); - else -#endif - __asm__ __volatile__( - " sync \n" - " lwarx %0,0,%5,1 \n" - " cmpw %0,%3 \n" - " bne $+12 \n" /* branch to lwsync */ - " stwcx. %4,0,%5 \n" - " bne $-16 \n" /* branch to lwarx */ - " lwsync \n" - " mfcr %1 \n" -: "=&r"(found), "=r"(condition_register), "+m"(ptr->value) -: "r"(*expected), "r"(newval), "r"(&ptr->value) -: "memory", "cc"); - - ret = (condition_register >> 29) & 1; /* test eq bit of cr0 */ - if (!ret) - *expected = found; - return ret; -} - -/* - * This mirrors gcc __sync_fetch_and_add(). - * - * Like tas(), use constraint "=&b" to avoid allocating r0. - */ -#define PG_HAVE_ATOMIC_FETCH_ADD_U32 -static inline uint32 -pg_atomic_fetch_add_u32_impl(volatile pg_atomic_uint32 *ptr, int32 add_) -{ - uint32 _t; - uint32 res; - -#ifdef HAVE_I_CONSTRAINT__BUILTIN_CONSTANT_P - if (__builtin_constant_p(add_) && - add_ <= PG_INT16_MAX && add_ >= PG_INT16_MIN) - __asm__ __volatile__( - " sync \n" - " lwarx %1,0,%4,1 \n" - " addi %0,%1,%3 \n" - " stwcx. %0,0,%4 \n" - " bne $-12 \n" /* branch to lwarx */ - " lwsync \n" -: "=&r"(_t), "=&b"(res), "+m"(ptr->value) -: "i"(add_), "r"(&ptr->value) -: "memory", "cc"); - else -#endif - __asm__ __volatile__( - " sync \n" - " lwarx %1,0,%4,1 \n" - " add %0,%1,%3 \n" - " stwcx. %0,0,%4 \n" - " bne $-12 \n" /* branch to lwarx */ - " lwsync \n" -: "=&r"(_t), "=&r"(res), "+m"(ptr->value) -: "r"(add_), "r"(&ptr->value) -: "memory", "cc"); - - return res; -} - -#ifdef PG_HAVE_ATOMIC_U64_SUPPORT - -#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64 -static inline bool -pg_atomic_compare_exchange_u64_impl(volatile pg_atomic_uint64 *ptr, - uint64 *expected, uint64 newval) -{ - uint64 found; - uint32 condition_register; - bool ret; - - AssertPointerAlignment(expected, 8); - - /* Like u32, but s/lwarx/ldarx/; s/stwcx/stdcx/; s/cmpw/cmpd/ */ -#ifdef HAVE_I_CONSTRAINT__BUILTIN_CONSTANT_P - if (__builtin_constant_p(*expected) && - (int64) *expected <= PG_INT16_MAX && - (int64) *expected >= PG_INT16_MIN) - __asm__ __volatile__( - " sync \n" - " ldarx %0,0,%5,1 \n" - " cmpdi %0,%3 \n" - " bne $+12 \n" /* branch to lwsync */ - " stdcx. %4,0,%5 \n" - " bne $-16 \n" /* branch to ldarx */ - " lwsync \n" - " mfcr %1 \n" -: "=&r"(found), "=r"(condition_register), "+m"(ptr->value) -: "i"(*expected), "r"(newval), "r"(&ptr->value) -: "memory", "cc"); - else -#endif - __asm__ __volatile__( - " sync \n" - " ldarx %0,0,%5,1 \n" - " cmpd %0,%3 \n" - " bne $+12 \n" /* branch to lwsync */ - " stdcx. %4,0,%5 \n" - " bne $-16 \n" /* branch to ldarx */ - " lwsync \n" - " mfcr %1 \n" -: "=&r"(found), "=r"(condition_register), "+m"(ptr->value) -: "r"(*expected), "r"(newval), "r"(&ptr->value) -: "memory", "cc"); - - ret = (condition_register >> 29) & 1; /* test eq bit of cr0 */ - if (!ret) - *expected = found; - return ret; -} - -#define PG_HAVE_ATOMIC_FETCH_ADD_U64 -static inline uint64 -pg_atomic_fetch_add_u64_impl(volatile pg_atomic_uint64 *ptr, int64 add_) -{ - uint64 _t; - uint64 res; - - /* Like u32, but s/lwarx/ldarx/; s/stwcx/stdcx/ */ -#ifdef HAVE_I_CONSTRAINT__BUILTIN_CONSTANT_P - if (__builtin_constant_p(add_) && - add_ <= PG_INT16_MAX && add_ >= PG_INT16_MIN) - __asm__ __volatile__( - " sync \n" - " ldarx %1,0,%4,1 \n" - " addi %0,%1,%3 \n" - " stdcx. %0,0,%4 \n" - " bne $-12 \n" /* branch to ldarx */ - " lwsync \n" -: "=&r"(_t), "=&b"(res), "+m"(ptr->value) -: "i"(add_), "r"(&ptr->value) -: "memory", "cc"); - else -#endif - __asm__ __volatile__( - " sync \n" - " ldarx %1,0,%4,1 \n" - " add %0,%1,%3 \n" - " stdcx. %0,0,%4 \n" - " bne $-12 \n" /* branch to ldarx */ - " lwsync \n" -: "=&r"(_t), "=&r"(res), "+m"(ptr->value) -: "r"(add_), "r"(&ptr->value) -: "memory", "cc"); - - return res; -} - -#endif /* PG_HAVE_ATOMIC_U64_SUPPORT */ - -/* per architecture manual doubleword accesses have single copy atomicity */ -#define PG_HAVE_8BYTE_SINGLE_COPY_ATOMICITY diff --git a/src/include/port/atomics/arch-x86.h b/src/include/port/atomics/arch-x86.h deleted file mode 100644 index 231831dcf59..00000000000 --- a/src/include/port/atomics/arch-x86.h +++ /dev/null @@ -1,189 +0,0 @@ -/*------------------------------------------------------------------------- - * - * arch-x86.h - * Atomic operations considerations specific to intel x86 - * - * Note that we actually require a 486 upwards because the 386 doesn't have - * support for xadd and cmpxchg. Given that the 386 isn't supported anywhere - * anymore that's not much of a restriction luckily. - * - * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group - * Portions Copyright (c) 1994, Regents of the University of California - * - * NOTES: - * - * src/include/port/atomics/arch-x86.h - * - *------------------------------------------------------------------------- - */ - -/* - * Both 32 and 64 bit x86 do not allow loads to be reordered with other loads, - * or stores to be reordered with other stores, but a load can be performed - * before a subsequent store. - * - * Technically, some x86-ish chips support uncached memory access and/or - * special instructions that are weakly ordered. In those cases we'd need - * the read and write barriers to be lfence and sfence. But since we don't - * do those things, a compiler barrier should be enough. - * - * "lock; addl" has worked for longer than "mfence". It's also rumored to be - * faster in many scenarios. - */ - -#if defined(__GNUC__) || defined(__INTEL_COMPILER) -#if defined(__i386__) -#define pg_memory_barrier_impl() \ - __asm__ __volatile__ ("lock; addl $0,0(%%esp)" : : : "memory", "cc") -#elif defined(__x86_64__) -#define pg_memory_barrier_impl() \ - __asm__ __volatile__ ("lock; addl $0,0(%%rsp)" : : : "memory", "cc") -#endif -#endif /* defined(__GNUC__) || defined(__INTEL_COMPILER) */ - -#define pg_read_barrier_impl() pg_compiler_barrier_impl() -#define pg_write_barrier_impl() pg_compiler_barrier_impl() - -/* - * Provide implementation for atomics using inline assembly on x86 gcc. It's - * nice to support older gcc's and the compare/exchange implementation here is - * actually more efficient than the * __sync variant. - */ -#if defined(__GNUC__) || defined(__INTEL_COMPILER) - -#define PG_HAVE_ATOMIC_FLAG_SUPPORT -typedef struct pg_atomic_flag -{ - volatile char value; -} pg_atomic_flag; - -#define PG_HAVE_ATOMIC_U32_SUPPORT -typedef struct pg_atomic_uint32 -{ - volatile uint32 value; -} pg_atomic_uint32; - -/* - * It's too complicated to write inline asm for 64bit types on 32bit and the - * 486 can't do it anyway. - */ -#ifdef __x86_64__ -#define PG_HAVE_ATOMIC_U64_SUPPORT -typedef struct pg_atomic_uint64 -{ - /* alignment guaranteed due to being on a 64bit platform */ - volatile uint64 value; -} pg_atomic_uint64; -#endif /* __x86_64__ */ - -#define PG_HAVE_ATOMIC_TEST_SET_FLAG -static inline bool -pg_atomic_test_set_flag_impl(volatile pg_atomic_flag *ptr) -{ - char _res = 1; - - __asm__ __volatile__( - " lock \n" - " xchgb %0,%1 \n" -: "+q"(_res), "+m"(ptr->value) -: -: "memory"); - return _res == 0; -} - -#define PG_HAVE_ATOMIC_CLEAR_FLAG -static inline void -pg_atomic_clear_flag_impl(volatile pg_atomic_flag *ptr) -{ - /* - * On a TSO architecture like x86 it's sufficient to use a compiler - * barrier to achieve release semantics. - */ - __asm__ __volatile__("" ::: "memory"); - ptr->value = 0; -} - -#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32 -static inline bool -pg_atomic_compare_exchange_u32_impl(volatile pg_atomic_uint32 *ptr, - uint32 *expected, uint32 newval) -{ - char ret; - - /* - * Perform cmpxchg and use the zero flag which it implicitly sets when - * equal to measure the success. - */ - __asm__ __volatile__( - " lock \n" - " cmpxchgl %4,%5 \n" - " setz %2 \n" -: "=a" (*expected), "=m"(ptr->value), "=q" (ret) -: "a" (*expected), "r" (newval), "m"(ptr->value) -: "memory", "cc"); - return (bool) ret; -} - -#define PG_HAVE_ATOMIC_FETCH_ADD_U32 -static inline uint32 -pg_atomic_fetch_add_u32_impl(volatile pg_atomic_uint32 *ptr, int32 add_) -{ - uint32 res; - __asm__ __volatile__( - " lock \n" - " xaddl %0,%1 \n" -: "=q"(res), "=m"(ptr->value) -: "0" (add_), "m"(ptr->value) -: "memory", "cc"); - return res; -} - -#ifdef __x86_64__ - -#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64 -static inline bool -pg_atomic_compare_exchange_u64_impl(volatile pg_atomic_uint64 *ptr, - uint64 *expected, uint64 newval) -{ - char ret; - - AssertPointerAlignment(expected, 8); - - /* - * Perform cmpxchg and use the zero flag which it implicitly sets when - * equal to measure the success. - */ - __asm__ __volatile__( - " lock \n" - " cmpxchgq %4,%5 \n" - " setz %2 \n" -: "=a" (*expected), "=m"(ptr->value), "=q" (ret) -: "a" (*expected), "r" (newval), "m"(ptr->value) -: "memory", "cc"); - return (bool) ret; -} - -#define PG_HAVE_ATOMIC_FETCH_ADD_U64 -static inline uint64 -pg_atomic_fetch_add_u64_impl(volatile pg_atomic_uint64 *ptr, int64 add_) -{ - uint64 res; - __asm__ __volatile__( - " lock \n" - " xaddq %0,%1 \n" -: "=q"(res), "=m"(ptr->value) -: "0" (add_), "m"(ptr->value) -: "memory", "cc"); - return res; -} - -#endif /* __x86_64__ */ - -#endif /* defined(__GNUC__) || defined(__INTEL_COMPILER) */ - -/* - * 8 byte reads / writes have single-copy atomicity on all x86-64 cpus. - */ -#if defined(__x86_64__) -#define PG_HAVE_8BYTE_SINGLE_COPY_ATOMICITY -#endif /* 8 byte single-copy atomicity */ diff --git a/src/include/port/atomics/fallback.h b/src/include/port/atomics/fallback.h deleted file mode 100644 index f37225189fc..00000000000 --- a/src/include/port/atomics/fallback.h +++ /dev/null @@ -1,42 +0,0 @@ -/*------------------------------------------------------------------------- - * - * fallback.h - * Fallback for platforms without 64 bit atomics support. Slower - * than native atomics support, but not unusably slow. - * - * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group - * Portions Copyright (c) 1994, Regents of the University of California - * - * src/include/port/atomics/fallback.h - * - *------------------------------------------------------------------------- - */ - -/* intentionally no include guards, should only be included by atomics.h */ -#ifndef INSIDE_ATOMICS_H -# error "should be included via atomics.h" -#endif - - -#if !defined(PG_HAVE_ATOMIC_U64_SUPPORT) - -#define PG_HAVE_ATOMIC_U64_SIMULATION - -#define PG_HAVE_ATOMIC_U64_SUPPORT -typedef struct pg_atomic_uint64 -{ - int sema; - volatile uint64 value; -} pg_atomic_uint64; - -#define PG_HAVE_ATOMIC_INIT_U64 -extern void pg_atomic_init_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 val_); - -#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64 -extern bool pg_atomic_compare_exchange_u64_impl(volatile pg_atomic_uint64 *ptr, - uint64 *expected, uint64 newval); - -#define PG_HAVE_ATOMIC_FETCH_ADD_U64 -extern uint64 pg_atomic_fetch_add_u64_impl(volatile pg_atomic_uint64 *ptr, int64 add_); - -#endif /* PG_HAVE_ATOMIC_U64_SUPPORT */ diff --git a/src/include/port/atomics/generic-gcc.h b/src/include/port/atomics/generic-gcc.h deleted file mode 100644 index 5bfce82f687..00000000000 --- a/src/include/port/atomics/generic-gcc.h +++ /dev/null @@ -1,326 +0,0 @@ -/*------------------------------------------------------------------------- - * - * generic-gcc.h - * Atomic operations, implemented using gcc (or compatible) intrinsics. - * - * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group - * Portions Copyright (c) 1994, Regents of the University of California - * - * NOTES: - * - * Documentation: - * * Legacy __sync Built-in Functions for Atomic Memory Access - * https://gcc.gnu.org/onlinedocs/gcc-4.8.2/gcc/_005f_005fsync-Builtins.html - * * Built-in functions for memory model aware atomic operations - * https://gcc.gnu.org/onlinedocs/gcc-4.8.2/gcc/_005f_005fatomic-Builtins.html - * - * src/include/port/atomics/generic-gcc.h - * - *------------------------------------------------------------------------- - */ - -/* intentionally no include guards, should only be included by atomics.h */ -#ifndef INSIDE_ATOMICS_H -#error "should be included via atomics.h" -#endif - -/* - * An empty asm block should be a sufficient compiler barrier. - */ -#define pg_compiler_barrier_impl() __asm__ __volatile__("" ::: "memory") - -/* - * If we're on GCC, we should be able to get a memory barrier - * out of this compiler built-in. But we prefer to rely on platform specific - * definitions where possible, and use this only as a fallback. - */ -#if !defined(pg_memory_barrier_impl) -# if defined(HAVE_GCC__ATOMIC_INT32_CAS) -# define pg_memory_barrier_impl() __atomic_thread_fence(__ATOMIC_SEQ_CST) -# elif defined(__GNUC__) -# define pg_memory_barrier_impl() __sync_synchronize() -# endif -#endif /* !defined(pg_memory_barrier_impl) */ - -#if !defined(pg_read_barrier_impl) && defined(HAVE_GCC__ATOMIC_INT32_CAS) -/* acquire semantics include read barrier semantics */ -# define pg_read_barrier_impl() do \ -{ \ - pg_compiler_barrier_impl(); \ - __atomic_thread_fence(__ATOMIC_ACQUIRE); \ -} while (0) -#endif - -#if !defined(pg_write_barrier_impl) && defined(HAVE_GCC__ATOMIC_INT32_CAS) -/* release semantics include write barrier semantics */ -# define pg_write_barrier_impl() do \ -{ \ - pg_compiler_barrier_impl(); \ - __atomic_thread_fence(__ATOMIC_RELEASE); \ -} while (0) -#endif - - -/* generic gcc based atomic flag implementation */ -#if !defined(PG_HAVE_ATOMIC_FLAG_SUPPORT) \ - && (defined(HAVE_GCC__SYNC_INT32_TAS) || defined(HAVE_GCC__SYNC_CHAR_TAS)) - -#define PG_HAVE_ATOMIC_FLAG_SUPPORT -typedef struct pg_atomic_flag -{ - /* - * If we have a choice, use int-width TAS, because that is more efficient - * and/or more reliably implemented on most non-Intel platforms. (Note - * that this code isn't used on x86[_64]; see arch-x86.h for that.) - */ -#ifdef HAVE_GCC__SYNC_INT32_TAS - volatile int value; -#else - volatile char value; -#endif -} pg_atomic_flag; - -#endif /* !ATOMIC_FLAG_SUPPORT && SYNC_INT32_TAS */ - -/* generic gcc based atomic uint32 implementation */ -#if !defined(PG_HAVE_ATOMIC_U32_SUPPORT) \ - && (defined(HAVE_GCC__ATOMIC_INT32_CAS) || defined(HAVE_GCC__SYNC_INT32_CAS)) - -#define PG_HAVE_ATOMIC_U32_SUPPORT -typedef struct pg_atomic_uint32 -{ - volatile uint32 value; -} pg_atomic_uint32; - -#endif /* defined(HAVE_GCC__ATOMIC_INT32_CAS) || defined(HAVE_GCC__SYNC_INT32_CAS) */ - -/* generic gcc based atomic uint64 implementation */ -#if !defined(PG_HAVE_ATOMIC_U64_SUPPORT) \ - && !defined(PG_DISABLE_64_BIT_ATOMICS) \ - && (defined(HAVE_GCC__ATOMIC_INT64_CAS) || defined(HAVE_GCC__SYNC_INT64_CAS)) - -#define PG_HAVE_ATOMIC_U64_SUPPORT -typedef struct pg_atomic_uint64 -{ - alignas(8) volatile uint64 value; -} pg_atomic_uint64; - -#endif /* defined(HAVE_GCC__ATOMIC_INT64_CAS) || defined(HAVE_GCC__SYNC_INT64_CAS) */ - -#ifdef PG_HAVE_ATOMIC_FLAG_SUPPORT - -#if defined(HAVE_GCC__SYNC_CHAR_TAS) || defined(HAVE_GCC__SYNC_INT32_TAS) - -#ifndef PG_HAVE_ATOMIC_TEST_SET_FLAG -#define PG_HAVE_ATOMIC_TEST_SET_FLAG -static inline bool -pg_atomic_test_set_flag_impl(volatile pg_atomic_flag *ptr) -{ - /* NB: only an acquire barrier, not a full one */ - /* some platform only support a 1 here */ - return __sync_lock_test_and_set(&ptr->value, 1) == 0; -} -#endif - -#endif /* defined(HAVE_GCC__SYNC_*_TAS) */ - -#ifndef PG_HAVE_ATOMIC_UNLOCKED_TEST_FLAG -#define PG_HAVE_ATOMIC_UNLOCKED_TEST_FLAG -static inline bool -pg_atomic_unlocked_test_flag_impl(volatile pg_atomic_flag *ptr) -{ - return ptr->value == 0; -} -#endif - -#ifndef PG_HAVE_ATOMIC_CLEAR_FLAG -#define PG_HAVE_ATOMIC_CLEAR_FLAG -static inline void -pg_atomic_clear_flag_impl(volatile pg_atomic_flag *ptr) -{ - __sync_lock_release(&ptr->value); -} -#endif - -#ifndef PG_HAVE_ATOMIC_INIT_FLAG -#define PG_HAVE_ATOMIC_INIT_FLAG -static inline void -pg_atomic_init_flag_impl(volatile pg_atomic_flag *ptr) -{ - pg_atomic_clear_flag_impl(ptr); -} -#endif - -#endif /* defined(PG_HAVE_ATOMIC_FLAG_SUPPORT) */ - -/* prefer __atomic, it has a better API */ -#if !defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32) && defined(HAVE_GCC__ATOMIC_INT32_CAS) -#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32 -static inline bool -pg_atomic_compare_exchange_u32_impl(volatile pg_atomic_uint32 *ptr, - uint32 *expected, uint32 newval) -{ - /* FIXME: we can probably use a lower consistency model */ - return __atomic_compare_exchange_n(&ptr->value, expected, newval, false, - __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); -} -#endif - -#if !defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32) && defined(HAVE_GCC__SYNC_INT32_CAS) -#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32 -static inline bool -pg_atomic_compare_exchange_u32_impl(volatile pg_atomic_uint32 *ptr, - uint32 *expected, uint32 newval) -{ - bool ret; - uint32 current; - current = __sync_val_compare_and_swap(&ptr->value, *expected, newval); - ret = current == *expected; - *expected = current; - return ret; -} -#endif - -/* - * __sync_lock_test_and_set() only supports setting the value to 1 on some - * platforms, so we only provide an __atomic implementation for - * pg_atomic_exchange. - * - * We assume the availability of 32-bit __atomic_compare_exchange_n() implies - * the availability of 32-bit __atomic_exchange_n(). - */ -#if !defined(PG_HAVE_ATOMIC_EXCHANGE_U32) && defined(HAVE_GCC__ATOMIC_INT32_CAS) -#define PG_HAVE_ATOMIC_EXCHANGE_U32 -static inline uint32 -pg_atomic_exchange_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 newval) -{ - return __atomic_exchange_n(&ptr->value, newval, __ATOMIC_SEQ_CST); -} -#endif - -/* if we have 32-bit __sync_val_compare_and_swap, assume we have these too: */ - -#if !defined(PG_HAVE_ATOMIC_FETCH_ADD_U32) && defined(HAVE_GCC__SYNC_INT32_CAS) -#define PG_HAVE_ATOMIC_FETCH_ADD_U32 -static inline uint32 -pg_atomic_fetch_add_u32_impl(volatile pg_atomic_uint32 *ptr, int32 add_) -{ - return __sync_fetch_and_add(&ptr->value, add_); -} -#endif - -#if !defined(PG_HAVE_ATOMIC_FETCH_SUB_U32) && defined(HAVE_GCC__SYNC_INT32_CAS) -#define PG_HAVE_ATOMIC_FETCH_SUB_U32 -static inline uint32 -pg_atomic_fetch_sub_u32_impl(volatile pg_atomic_uint32 *ptr, int32 sub_) -{ - return __sync_fetch_and_sub(&ptr->value, sub_); -} -#endif - -#if !defined(PG_HAVE_ATOMIC_FETCH_AND_U32) && defined(HAVE_GCC__SYNC_INT32_CAS) -#define PG_HAVE_ATOMIC_FETCH_AND_U32 -static inline uint32 -pg_atomic_fetch_and_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 and_) -{ - return __sync_fetch_and_and(&ptr->value, and_); -} -#endif - -#if !defined(PG_HAVE_ATOMIC_FETCH_OR_U32) && defined(HAVE_GCC__SYNC_INT32_CAS) -#define PG_HAVE_ATOMIC_FETCH_OR_U32 -static inline uint32 -pg_atomic_fetch_or_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 or_) -{ - return __sync_fetch_and_or(&ptr->value, or_); -} -#endif - - -#if !defined(PG_DISABLE_64_BIT_ATOMICS) - -#if !defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64) && defined(HAVE_GCC__ATOMIC_INT64_CAS) -#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64 -static inline bool -pg_atomic_compare_exchange_u64_impl(volatile pg_atomic_uint64 *ptr, - uint64 *expected, uint64 newval) -{ - AssertPointerAlignment(expected, 8); - return __atomic_compare_exchange_n(&ptr->value, expected, newval, false, - __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); -} -#endif - -#if !defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64) && defined(HAVE_GCC__SYNC_INT64_CAS) -#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64 -static inline bool -pg_atomic_compare_exchange_u64_impl(volatile pg_atomic_uint64 *ptr, - uint64 *expected, uint64 newval) -{ - bool ret; - uint64 current; - - AssertPointerAlignment(expected, 8); - current = __sync_val_compare_and_swap(&ptr->value, *expected, newval); - ret = current == *expected; - *expected = current; - return ret; -} -#endif - -/* - * __sync_lock_test_and_set() only supports setting the value to 1 on some - * platforms, so we only provide an __atomic implementation for - * pg_atomic_exchange. - * - * We assume the availability of 64-bit __atomic_compare_exchange_n() implies - * the availability of 64-bit __atomic_exchange_n(). - */ -#if !defined(PG_HAVE_ATOMIC_EXCHANGE_U64) && defined(HAVE_GCC__ATOMIC_INT64_CAS) -#define PG_HAVE_ATOMIC_EXCHANGE_U64 -static inline uint64 -pg_atomic_exchange_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 newval) -{ - return __atomic_exchange_n(&ptr->value, newval, __ATOMIC_SEQ_CST); -} -#endif - -/* if we have 64-bit __sync_val_compare_and_swap, assume we have these too: */ - -#if !defined(PG_HAVE_ATOMIC_FETCH_ADD_U64) && defined(HAVE_GCC__SYNC_INT64_CAS) -#define PG_HAVE_ATOMIC_FETCH_ADD_U64 -static inline uint64 -pg_atomic_fetch_add_u64_impl(volatile pg_atomic_uint64 *ptr, int64 add_) -{ - return __sync_fetch_and_add(&ptr->value, add_); -} -#endif - -#if !defined(PG_HAVE_ATOMIC_FETCH_SUB_U64) && defined(HAVE_GCC__SYNC_INT64_CAS) -#define PG_HAVE_ATOMIC_FETCH_SUB_U64 -static inline uint64 -pg_atomic_fetch_sub_u64_impl(volatile pg_atomic_uint64 *ptr, int64 sub_) -{ - return __sync_fetch_and_sub(&ptr->value, sub_); -} -#endif - -#if !defined(PG_HAVE_ATOMIC_FETCH_AND_U64) && defined(HAVE_GCC__SYNC_INT64_CAS) -#define PG_HAVE_ATOMIC_FETCH_AND_U64 -static inline uint64 -pg_atomic_fetch_and_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 and_) -{ - return __sync_fetch_and_and(&ptr->value, and_); -} -#endif - -#if !defined(PG_HAVE_ATOMIC_FETCH_OR_U64) && defined(HAVE_GCC__SYNC_INT64_CAS) -#define PG_HAVE_ATOMIC_FETCH_OR_U64 -static inline uint64 -pg_atomic_fetch_or_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 or_) -{ - return __sync_fetch_and_or(&ptr->value, or_); -} -#endif - -#endif /* !defined(PG_DISABLE_64_BIT_ATOMICS) */ diff --git a/src/include/port/atomics/generic-msvc.h b/src/include/port/atomics/generic-msvc.h deleted file mode 100644 index 6d09cd0e043..00000000000 --- a/src/include/port/atomics/generic-msvc.h +++ /dev/null @@ -1,115 +0,0 @@ -/*------------------------------------------------------------------------- - * - * generic-msvc.h - * Atomic operations support when using MSVC - * - * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group - * Portions Copyright (c) 1994, Regents of the University of California - * - * NOTES: - * - * Documentation: - * * Interlocked Variable Access - * http://msdn.microsoft.com/en-us/library/ms684122%28VS.85%29.aspx - * - * src/include/port/atomics/generic-msvc.h - * - *------------------------------------------------------------------------- - */ -#include - -/* intentionally no include guards, should only be included by atomics.h */ -#ifndef INSIDE_ATOMICS_H -#error "should be included via atomics.h" -#endif - -#pragma intrinsic(_ReadWriteBarrier) -#define pg_compiler_barrier_impl() _ReadWriteBarrier() - -#ifndef pg_memory_barrier_impl -#define pg_memory_barrier_impl() MemoryBarrier() -#endif - -#define PG_HAVE_ATOMIC_U32_SUPPORT -typedef struct pg_atomic_uint32 -{ - volatile uint32 value; -} pg_atomic_uint32; - -#define PG_HAVE_ATOMIC_U64_SUPPORT -typedef struct pg_atomic_uint64 -{ - alignas(8) volatile uint64 value; -} pg_atomic_uint64; - - -#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32 -static inline bool -pg_atomic_compare_exchange_u32_impl(volatile pg_atomic_uint32 *ptr, - uint32 *expected, uint32 newval) -{ - bool ret; - uint32 current; - current = InterlockedCompareExchange(&ptr->value, newval, *expected); - ret = current == *expected; - *expected = current; - return ret; -} - -#define PG_HAVE_ATOMIC_EXCHANGE_U32 -static inline uint32 -pg_atomic_exchange_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 newval) -{ - return InterlockedExchange(&ptr->value, newval); -} - -#define PG_HAVE_ATOMIC_FETCH_ADD_U32 -static inline uint32 -pg_atomic_fetch_add_u32_impl(volatile pg_atomic_uint32 *ptr, int32 add_) -{ - return InterlockedExchangeAdd(&ptr->value, add_); -} - -/* - * The non-intrinsics versions are only available in vista upwards, so use the - * intrinsic version. Only supported on >486, but we require XP as a minimum - * baseline, which doesn't support the 486, so we don't need to add checks for - * that case. - */ -#pragma intrinsic(_InterlockedCompareExchange64) - -#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64 -static inline bool -pg_atomic_compare_exchange_u64_impl(volatile pg_atomic_uint64 *ptr, - uint64 *expected, uint64 newval) -{ - bool ret; - uint64 current; - current = _InterlockedCompareExchange64(&ptr->value, newval, *expected); - ret = current == *expected; - *expected = current; - return ret; -} - -/* Only implemented on 64bit builds */ -#ifdef _WIN64 - -#pragma intrinsic(_InterlockedExchange64) - -#define PG_HAVE_ATOMIC_EXCHANGE_U64 -static inline uint64 -pg_atomic_exchange_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 newval) -{ - return _InterlockedExchange64(&ptr->value, newval); -} - -#pragma intrinsic(_InterlockedExchangeAdd64) - -#define PG_HAVE_ATOMIC_FETCH_ADD_U64 -static inline uint64 -pg_atomic_fetch_add_u64_impl(volatile pg_atomic_uint64 *ptr, int64 add_) -{ - return _InterlockedExchangeAdd64(&ptr->value, add_); -} - -#endif /* _WIN64 */ diff --git a/src/include/port/atomics/generic.h b/src/include/port/atomics/generic.h deleted file mode 100644 index daa772e9a6d..00000000000 --- a/src/include/port/atomics/generic.h +++ /dev/null @@ -1,430 +0,0 @@ -/*------------------------------------------------------------------------- - * - * generic.h - * Implement higher level operations based on some lower level atomic - * operations. - * - * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group - * Portions Copyright (c) 1994, Regents of the University of California - * - * src/include/port/atomics/generic.h - * - *------------------------------------------------------------------------- - */ - -/* intentionally no include guards, should only be included by atomics.h */ -#ifndef INSIDE_ATOMICS_H -# error "should be included via atomics.h" -#endif - -/* - * If read or write barriers are undefined, we upgrade them to full memory - * barriers. - */ -#if !defined(pg_read_barrier_impl) -# define pg_read_barrier_impl pg_memory_barrier_impl -#endif -#if !defined(pg_write_barrier_impl) -# define pg_write_barrier_impl pg_memory_barrier_impl -#endif - -/* provide fallback */ -#if !defined(PG_HAVE_ATOMIC_FLAG_SUPPORT) && defined(PG_HAVE_ATOMIC_U32_SUPPORT) -#define PG_HAVE_ATOMIC_FLAG_SUPPORT -typedef pg_atomic_uint32 pg_atomic_flag; -#endif - -#ifndef PG_HAVE_ATOMIC_READ_U32 -#define PG_HAVE_ATOMIC_READ_U32 -static inline uint32 -pg_atomic_read_u32_impl(volatile pg_atomic_uint32 *ptr) -{ - return ptr->value; -} -#endif - -#ifndef PG_HAVE_ATOMIC_WRITE_U32 -#define PG_HAVE_ATOMIC_WRITE_U32 -static inline void -pg_atomic_write_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 val) -{ - ptr->value = val; -} -#endif - -#ifndef PG_HAVE_ATOMIC_UNLOCKED_WRITE_U32 -#define PG_HAVE_ATOMIC_UNLOCKED_WRITE_U32 -static inline void -pg_atomic_unlocked_write_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 val) -{ - ptr->value = val; -} -#endif - -/* - * provide fallback for test_and_set using atomic_exchange if available - */ -#if !defined(PG_HAVE_ATOMIC_TEST_SET_FLAG) && defined(PG_HAVE_ATOMIC_EXCHANGE_U32) - -#define PG_HAVE_ATOMIC_INIT_FLAG -static inline void -pg_atomic_init_flag_impl(volatile pg_atomic_flag *ptr) -{ - pg_atomic_write_u32_impl(ptr, 0); -} - -#define PG_HAVE_ATOMIC_TEST_SET_FLAG -static inline bool -pg_atomic_test_set_flag_impl(volatile pg_atomic_flag *ptr) -{ - return pg_atomic_exchange_u32_impl(ptr, 1) == 0; -} - -#define PG_HAVE_ATOMIC_UNLOCKED_TEST_FLAG -static inline bool -pg_atomic_unlocked_test_flag_impl(volatile pg_atomic_flag *ptr) -{ - return pg_atomic_read_u32_impl(ptr) == 0; -} - - -#define PG_HAVE_ATOMIC_CLEAR_FLAG -static inline void -pg_atomic_clear_flag_impl(volatile pg_atomic_flag *ptr) -{ - /* XXX: release semantics suffice? */ - pg_memory_barrier_impl(); - pg_atomic_write_u32_impl(ptr, 0); -} - -/* - * provide fallback for test_and_set using atomic_compare_exchange if - * available. - */ -#elif !defined(PG_HAVE_ATOMIC_TEST_SET_FLAG) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32) - -#define PG_HAVE_ATOMIC_INIT_FLAG -static inline void -pg_atomic_init_flag_impl(volatile pg_atomic_flag *ptr) -{ - pg_atomic_write_u32_impl(ptr, 0); -} - -#define PG_HAVE_ATOMIC_TEST_SET_FLAG -static inline bool -pg_atomic_test_set_flag_impl(volatile pg_atomic_flag *ptr) -{ - uint32 value = 0; - return pg_atomic_compare_exchange_u32_impl(ptr, &value, 1); -} - -#define PG_HAVE_ATOMIC_UNLOCKED_TEST_FLAG -static inline bool -pg_atomic_unlocked_test_flag_impl(volatile pg_atomic_flag *ptr) -{ - return pg_atomic_read_u32_impl(ptr) == 0; -} - -#define PG_HAVE_ATOMIC_CLEAR_FLAG -static inline void -pg_atomic_clear_flag_impl(volatile pg_atomic_flag *ptr) -{ - /* XXX: release semantics suffice? */ - pg_memory_barrier_impl(); - pg_atomic_write_u32_impl(ptr, 0); -} - -#elif !defined(PG_HAVE_ATOMIC_TEST_SET_FLAG) -# error "No pg_atomic_test_and_set provided" -#endif /* !defined(PG_HAVE_ATOMIC_TEST_SET_FLAG) */ - - -#ifndef PG_HAVE_ATOMIC_INIT_U32 -#define PG_HAVE_ATOMIC_INIT_U32 -static inline void -pg_atomic_init_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 val_) -{ - ptr->value = val_; -} -#endif - -#if !defined(PG_HAVE_ATOMIC_EXCHANGE_U32) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32) -#define PG_HAVE_ATOMIC_EXCHANGE_U32 -static inline uint32 -pg_atomic_exchange_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 xchg_) -{ - uint32 old; - old = ptr->value; /* ok if read is not atomic */ - while (!pg_atomic_compare_exchange_u32_impl(ptr, &old, xchg_)) - /* skip */; - return old; -} -#endif - -#if !defined(PG_HAVE_ATOMIC_FETCH_ADD_U32) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32) -#define PG_HAVE_ATOMIC_FETCH_ADD_U32 -static inline uint32 -pg_atomic_fetch_add_u32_impl(volatile pg_atomic_uint32 *ptr, int32 add_) -{ - uint32 old; - old = ptr->value; /* ok if read is not atomic */ - while (!pg_atomic_compare_exchange_u32_impl(ptr, &old, old + add_)) - /* skip */; - return old; -} -#endif - -#if !defined(PG_HAVE_ATOMIC_FETCH_SUB_U32) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32) -#define PG_HAVE_ATOMIC_FETCH_SUB_U32 -static inline uint32 -pg_atomic_fetch_sub_u32_impl(volatile pg_atomic_uint32 *ptr, int32 sub_) -{ - return pg_atomic_fetch_add_u32_impl(ptr, -sub_); -} -#endif - -#if !defined(PG_HAVE_ATOMIC_FETCH_AND_U32) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32) -#define PG_HAVE_ATOMIC_FETCH_AND_U32 -static inline uint32 -pg_atomic_fetch_and_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 and_) -{ - uint32 old; - old = ptr->value; /* ok if read is not atomic */ - while (!pg_atomic_compare_exchange_u32_impl(ptr, &old, old & and_)) - /* skip */; - return old; -} -#endif - -#if !defined(PG_HAVE_ATOMIC_FETCH_OR_U32) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U32) -#define PG_HAVE_ATOMIC_FETCH_OR_U32 -static inline uint32 -pg_atomic_fetch_or_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 or_) -{ - uint32 old; - old = ptr->value; /* ok if read is not atomic */ - while (!pg_atomic_compare_exchange_u32_impl(ptr, &old, old | or_)) - /* skip */; - return old; -} -#endif - -#if !defined(PG_HAVE_ATOMIC_ADD_FETCH_U32) && defined(PG_HAVE_ATOMIC_FETCH_ADD_U32) -#define PG_HAVE_ATOMIC_ADD_FETCH_U32 -static inline uint32 -pg_atomic_add_fetch_u32_impl(volatile pg_atomic_uint32 *ptr, int32 add_) -{ - return pg_atomic_fetch_add_u32_impl(ptr, add_) + add_; -} -#endif - -#if !defined(PG_HAVE_ATOMIC_SUB_FETCH_U32) && defined(PG_HAVE_ATOMIC_FETCH_SUB_U32) -#define PG_HAVE_ATOMIC_SUB_FETCH_U32 -static inline uint32 -pg_atomic_sub_fetch_u32_impl(volatile pg_atomic_uint32 *ptr, int32 sub_) -{ - return pg_atomic_fetch_sub_u32_impl(ptr, sub_) - sub_; -} -#endif - -#if !defined(PG_HAVE_ATOMIC_READ_MEMBARRIER_U32) && defined(PG_HAVE_ATOMIC_FETCH_ADD_U32) -#define PG_HAVE_ATOMIC_READ_MEMBARRIER_U32 -static inline uint32 -pg_atomic_read_membarrier_u32_impl(volatile pg_atomic_uint32 *ptr) -{ - return pg_atomic_fetch_add_u32_impl(ptr, 0); -} -#endif - -#if !defined(PG_HAVE_ATOMIC_WRITE_MEMBARRIER_U32) && defined(PG_HAVE_ATOMIC_EXCHANGE_U32) -#define PG_HAVE_ATOMIC_WRITE_MEMBARRIER_U32 -static inline void -pg_atomic_write_membarrier_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 val) -{ - (void) pg_atomic_exchange_u32_impl(ptr, val); -} -#endif - -#if !defined(PG_HAVE_ATOMIC_EXCHANGE_U64) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64) -#define PG_HAVE_ATOMIC_EXCHANGE_U64 -static inline uint64 -pg_atomic_exchange_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 xchg_) -{ - uint64 old; - old = ptr->value; /* ok if read is not atomic */ - while (!pg_atomic_compare_exchange_u64_impl(ptr, &old, xchg_)) - /* skip */; - return old; -} -#endif - -#ifndef PG_HAVE_ATOMIC_WRITE_U64 -#define PG_HAVE_ATOMIC_WRITE_U64 - -#if defined(PG_HAVE_8BYTE_SINGLE_COPY_ATOMICITY) && \ - !defined(PG_HAVE_ATOMIC_U64_SIMULATION) - -static inline void -pg_atomic_write_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 val) -{ - /* - * On this platform aligned 64bit writes are guaranteed to be atomic, - * except if using the fallback implementation, where can't guarantee the - * required alignment. - */ - AssertPointerAlignment(ptr, 8); - ptr->value = val; -} - -#else - -static inline void -pg_atomic_write_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 val) -{ - /* - * 64 bit writes aren't safe on all platforms. In the generic - * implementation implement them as an atomic exchange. - */ - pg_atomic_exchange_u64_impl(ptr, val); -} - -#endif /* PG_HAVE_8BYTE_SINGLE_COPY_ATOMICITY && !PG_HAVE_ATOMIC_U64_SIMULATION */ -#endif /* PG_HAVE_ATOMIC_WRITE_U64 */ - -#ifndef PG_HAVE_ATOMIC_UNLOCKED_WRITE_U64 -#define PG_HAVE_ATOMIC_UNLOCKED_WRITE_U64 -static inline void -pg_atomic_unlocked_write_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 val) -{ - ptr->value = val; -} -#endif - -#ifndef PG_HAVE_ATOMIC_READ_U64 -#define PG_HAVE_ATOMIC_READ_U64 - -#if defined(PG_HAVE_8BYTE_SINGLE_COPY_ATOMICITY) && \ - !defined(PG_HAVE_ATOMIC_U64_SIMULATION) - -static inline uint64 -pg_atomic_read_u64_impl(volatile pg_atomic_uint64 *ptr) -{ - /* - * On this platform aligned 64-bit reads are guaranteed to be atomic. - */ - AssertPointerAlignment(ptr, 8); - return ptr->value; -} - -#else - -static inline uint64 -pg_atomic_read_u64_impl(volatile pg_atomic_uint64 *ptr) -{ - uint64 old = 0; - - /* - * 64-bit reads aren't atomic on all platforms. In the generic - * implementation implement them as a compare/exchange with 0. That'll - * fail or succeed, but always return the old value. Possibly might store - * a 0, but only if the previous value also was a 0 - i.e. harmless. - */ - pg_atomic_compare_exchange_u64_impl(ptr, &old, 0); - - return old; -} -#endif /* PG_HAVE_8BYTE_SINGLE_COPY_ATOMICITY && !PG_HAVE_ATOMIC_U64_SIMULATION */ -#endif /* PG_HAVE_ATOMIC_READ_U64 */ - -#ifndef PG_HAVE_ATOMIC_INIT_U64 -#define PG_HAVE_ATOMIC_INIT_U64 -static inline void -pg_atomic_init_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 val_) -{ - ptr->value = val_; -} -#endif - -#if !defined(PG_HAVE_ATOMIC_FETCH_ADD_U64) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64) -#define PG_HAVE_ATOMIC_FETCH_ADD_U64 -static inline uint64 -pg_atomic_fetch_add_u64_impl(volatile pg_atomic_uint64 *ptr, int64 add_) -{ - uint64 old; - old = ptr->value; /* ok if read is not atomic */ - while (!pg_atomic_compare_exchange_u64_impl(ptr, &old, old + add_)) - /* skip */; - return old; -} -#endif - -#if !defined(PG_HAVE_ATOMIC_FETCH_SUB_U64) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64) -#define PG_HAVE_ATOMIC_FETCH_SUB_U64 -static inline uint64 -pg_atomic_fetch_sub_u64_impl(volatile pg_atomic_uint64 *ptr, int64 sub_) -{ - return pg_atomic_fetch_add_u64_impl(ptr, -sub_); -} -#endif - -#if !defined(PG_HAVE_ATOMIC_FETCH_AND_U64) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64) -#define PG_HAVE_ATOMIC_FETCH_AND_U64 -static inline uint64 -pg_atomic_fetch_and_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 and_) -{ - uint64 old; - old = ptr->value; /* ok if read is not atomic */ - while (!pg_atomic_compare_exchange_u64_impl(ptr, &old, old & and_)) - /* skip */; - return old; -} -#endif - -#if !defined(PG_HAVE_ATOMIC_FETCH_OR_U64) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64) -#define PG_HAVE_ATOMIC_FETCH_OR_U64 -static inline uint64 -pg_atomic_fetch_or_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 or_) -{ - uint64 old; - old = ptr->value; /* ok if read is not atomic */ - while (!pg_atomic_compare_exchange_u64_impl(ptr, &old, old | or_)) - /* skip */; - return old; -} -#endif - -#if !defined(PG_HAVE_ATOMIC_ADD_FETCH_U64) && defined(PG_HAVE_ATOMIC_FETCH_ADD_U64) -#define PG_HAVE_ATOMIC_ADD_FETCH_U64 -static inline uint64 -pg_atomic_add_fetch_u64_impl(volatile pg_atomic_uint64 *ptr, int64 add_) -{ - return pg_atomic_fetch_add_u64_impl(ptr, add_) + add_; -} -#endif - -#if !defined(PG_HAVE_ATOMIC_SUB_FETCH_U64) && defined(PG_HAVE_ATOMIC_FETCH_SUB_U64) -#define PG_HAVE_ATOMIC_SUB_FETCH_U64 -static inline uint64 -pg_atomic_sub_fetch_u64_impl(volatile pg_atomic_uint64 *ptr, int64 sub_) -{ - return pg_atomic_fetch_sub_u64_impl(ptr, sub_) - sub_; -} -#endif - -#if !defined(PG_HAVE_ATOMIC_READ_MEMBARRIER_U64) && defined(PG_HAVE_ATOMIC_FETCH_ADD_U64) -#define PG_HAVE_ATOMIC_READ_MEMBARRIER_U64 -static inline uint64 -pg_atomic_read_membarrier_u64_impl(volatile pg_atomic_uint64 *ptr) -{ - return pg_atomic_fetch_add_u64_impl(ptr, 0); -} -#endif - -#if !defined(PG_HAVE_ATOMIC_WRITE_MEMBARRIER_U64) && defined(PG_HAVE_ATOMIC_EXCHANGE_U64) -#define PG_HAVE_ATOMIC_WRITE_MEMBARRIER_U64 -static inline void -pg_atomic_write_membarrier_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 val) -{ - (void) pg_atomic_exchange_u64_impl(ptr, val); -} -#endif diff --git a/src/include/storage/s_lock.h b/src/include/storage/s_lock.h deleted file mode 100644 index 7e049dfac18..00000000000 --- a/src/include/storage/s_lock.h +++ /dev/null @@ -1,734 +0,0 @@ -/*------------------------------------------------------------------------- - * - * s_lock.h - * Implementation of spinlocks. - * - * NOTE: none of the macros in this file are intended to be called directly. - * Call them through the macros in spin.h. - * - * The following hardware-dependent macros must be provided for each - * supported platform: - * - * void S_INIT_LOCK(slock_t *lock) - * Initialize a spinlock (to the unlocked state). - * - * int S_LOCK(slock_t *lock) - * Acquire a spinlock, waiting if necessary. - * Time out and abort() if unable to acquire the lock in a - * "reasonable" amount of time --- typically ~ 1 minute. - * Should return number of "delays"; see s_lock.c - * - * void S_UNLOCK(slock_t *lock) - * Unlock a previously acquired lock. - * - * void SPIN_DELAY(void) - * Delay operation to occur inside spinlock wait loop. - * - * Note to implementors: there are default implementations for all these - * macros at the bottom of the file. Check if your platform can use - * these or needs to override them. - * - * Usually, S_LOCK() is implemented in terms of even lower-level macros - * TAS() and TAS_SPIN(): - * - * int TAS(slock_t *lock) - * Atomic test-and-set instruction. Attempt to acquire the lock, - * but do *not* wait. Returns 0 if successful, nonzero if unable - * to acquire the lock. - * - * int TAS_SPIN(slock_t *lock) - * Like TAS(), but this version is used when waiting for a lock - * previously found to be contended. By default, this is the - * same as TAS(), but on some architectures it's better to poll a - * contended lock using an unlocked instruction and retry the - * atomic test-and-set only when it appears free. - * - * TAS() and TAS_SPIN() are NOT part of the API, and should never be called - * directly. A platform must provide either S_LOCK() or a TAS() for the - * default S_LOCK() to be built on. Currently, all supported platforms do - * the latter, so that is probably the best place to start if adding a new - * one. - * - * It is the responsibility of these macros to make sure that the compiler - * does not re-order accesses to shared memory to precede the actual lock - * acquisition, or follow the lock release. - * - * On platforms with weak memory ordering, the TAS(), TAS_SPIN(), and - * S_UNLOCK() macros must further include hardware-level memory fence - * instructions to prevent similar re-ordering at the hardware level. - * TAS() and TAS_SPIN() must guarantee that loads and stores issued after - * the macro are not executed until the lock has been obtained. Conversely, - * S_UNLOCK() must guarantee that loads and stores issued before the macro - * have been executed before the lock is released. - * - * On most supported platforms, TAS() uses a tas() function written - * in assembly language to execute a hardware atomic-test-and-set - * instruction. Equivalent compiler intrinsics are another popular option. - * - * - * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group - * Portions Copyright (c) 1994, Regents of the University of California - * - * src/include/storage/s_lock.h - * - *------------------------------------------------------------------------- - */ -#ifndef S_LOCK_H -#define S_LOCK_H - -#ifdef FRONTEND -#error "s_lock.h may not be included from frontend code" -#endif - -#if defined(__GNUC__) || defined(__INTEL_COMPILER) -/************************************************************************* - * All the gcc inlines - * Gcc consistently defines the CPU as __cpu__. - * Other compilers use __cpu or __cpu__ so we test for both in those cases. - */ - -/*---------- - * Standard gcc asm format (assuming "volatile slock_t *lock"): - - __asm__ __volatile__( - " instruction \n" - " instruction \n" - " instruction \n" -: "=r"(_res), "+m"(*lock) // return register, in/out lock value -: "r"(lock) // lock pointer, in input register -: "memory", "cc"); // show clobbered registers here - - * The output-operands list (after first colon) should always include - * "+m"(*lock), whether or not the asm code actually refers to this - * operand directly. This ensures that gcc believes the value in the - * lock variable is used and set by the asm code. Also, the clobbers - * list (after third colon) should always include "memory"; this prevents - * gcc from thinking it can cache the values of shared-memory fields - * across the asm code. Add "cc" if your asm code changes the condition - * code register, and also list any temp registers the code uses. - * - * If you need branch target labels within the asm block, include "%=" - * in the label names to make them distinct across multiple asm blocks - * within a source file. - *---------- - */ - - -#ifdef __i386__ /* 32-bit i386 */ - -typedef unsigned char slock_t; - -#define TAS(lock) tas(lock) - -static inline int -tas(volatile slock_t *lock) -{ - slock_t _res = 1; - - /* - * Use a non-locking test before asserting the bus lock. Note that the - * extra test appears to be a small loss on some x86 platforms and a small - * win on others; it's by no means clear that we should keep it. - * - * When this was last tested, we didn't have separate TAS() and TAS_SPIN() - * macros. Nowadays it probably would be better to do a non-locking test - * in TAS_SPIN() but not in TAS(), like on x86_64, but no-one's done the - * testing to verify that. Without some empirical evidence, better to - * leave it alone. - */ - __asm__ __volatile__( - " cmpb $0,%1 \n" - " jne TAS%=_out \n" - " lock \n" - " xchgb %0,%1 \n" - "TAS%=_out: \n" -: "+q"(_res), "+m"(*lock) -: /* no inputs */ -: "memory", "cc"); - return (int) _res; -} - -#define SPIN_DELAY() spin_delay() - -static inline void -spin_delay(void) -{ - /* - * This sequence is equivalent to the PAUSE instruction ("rep" is - * ignored by old IA32 processors if the following instruction is - * not a string operation); the IA-32 Architecture Software - * Developer's Manual, Vol. 3, Section 7.7.2 describes why using - * PAUSE in the inner loop of a spin lock is necessary for good - * performance: - * - * The PAUSE instruction improves the performance of IA-32 - * processors supporting Hyper-Threading Technology when - * executing spin-wait loops and other routines where one - * thread is accessing a shared lock or semaphore in a tight - * polling loop. When executing a spin-wait loop, the - * processor can suffer a severe performance penalty when - * exiting the loop because it detects a possible memory order - * violation and flushes the core processor's pipeline. The - * PAUSE instruction provides a hint to the processor that the - * code sequence is a spin-wait loop. The processor uses this - * hint to avoid the memory order violation and prevent the - * pipeline flush. In addition, the PAUSE instruction - * de-pipelines the spin-wait loop to prevent it from - * consuming execution resources excessively. - */ - __asm__ __volatile__( - " rep; nop \n"); -} - -#endif /* __i386__ */ - - -#ifdef __x86_64__ /* AMD Opteron, Intel EM64T */ - -typedef unsigned char slock_t; - -#define TAS(lock) tas(lock) - -/* - * On Intel EM64T, it's a win to use a non-locking test before the xchg proper, - * but only when spinning. - * - * See also Implementing Scalable Atomic Locks for Multi-Core Intel(tm) EM64T - * and IA32, by Michael Chynoweth and Mary R. Lee. As of this writing, it is - * available at: - * http://software.intel.com/en-us/articles/implementing-scalable-atomic-locks-for-multi-core-intel-em64t-and-ia32-architectures - */ -#define TAS_SPIN(lock) (*(lock) ? 1 : TAS(lock)) - -static inline int -tas(volatile slock_t *lock) -{ - slock_t _res = 1; - - __asm__ __volatile__( - " lock \n" - " xchgb %0,%1 \n" -: "+q"(_res), "+m"(*lock) -: /* no inputs */ -: "memory", "cc"); - return (int) _res; -} - -#define SPIN_DELAY() spin_delay() - -static inline void -spin_delay(void) -{ - /* - * Adding a PAUSE in the spin delay loop is demonstrably a no-op on - * Opteron, but it may be of some use on EM64T, so we keep it. - */ - __asm__ __volatile__( - " rep; nop \n"); -} - -#endif /* __x86_64__ */ - - -/* - * On ARM and ARM64, we use __sync_lock_test_and_set(int *, int) if available. - * - * We use the int-width variant of the builtin because it works on more chips - * than other widths. - */ -#if defined(__arm__) || defined(__aarch64__) -#ifdef HAVE_GCC__SYNC_INT32_TAS - -#define TAS(lock) tas(lock) - -typedef int slock_t; - -static inline int -tas(volatile slock_t *lock) -{ - return __sync_lock_test_and_set(lock, 1); -} - -#define S_UNLOCK(lock) __sync_lock_release(lock) - -#if defined(__aarch64__) - -/* - * On ARM64, it's a win to use a non-locking test before the TAS proper. It - * may be a win on 32-bit ARM, too, but nobody's tested it yet. - */ -#define TAS_SPIN(lock) (*(lock) ? 1 : TAS(lock)) - -#define SPIN_DELAY() spin_delay() - -static inline void -spin_delay(void) -{ - /* - * Using an ISB instruction to delay in spinlock loops appears beneficial - * on high-core-count ARM64 processors. It seems mostly a wash for smaller - * gear, and ISB doesn't exist at all on pre-v7 ARM chips. - */ - __asm__ __volatile__( - " isb; \n"); -} - -#endif /* __aarch64__ */ -#endif /* HAVE_GCC__SYNC_INT32_TAS */ -#endif /* __arm__ || __aarch64__ */ - - -/* S/390 and S/390x Linux (32- and 64-bit zSeries) */ -#if defined(__s390__) || defined(__s390x__) - -typedef unsigned int slock_t; - -#define TAS(lock) tas(lock) - -static inline int -tas(volatile slock_t *lock) -{ - int _res = 0; - - __asm__ __volatile__( - " cs %0,%3,0(%2) \n" -: "+d"(_res), "+m"(*lock) -: "a"(lock), "d"(1) -: "memory", "cc"); - return _res; -} - -#endif /* __s390__ || __s390x__ */ - - -#if defined(__sparc__) /* Sparc */ -/* - * Solaris has always run sparc processors in TSO (total store) mode, but - * linux didn't use to and the *BSDs still don't. So, be careful about - * acquire/release semantics. The CPU will treat superfluous members as - * NOPs, so it's just code space. - */ - -typedef unsigned char slock_t; - -#define TAS(lock) tas(lock) - -static inline int -tas(volatile slock_t *lock) -{ - slock_t _res; - - /* - * "cas" would be better than "ldstub", but it is only present on - * sparcv8plus and later, while some platforms still support sparcv7 or - * sparcv8. Also, "cas" requires that the system be running in TSO mode. - */ - __asm__ __volatile__( - " ldstub [%2], %0 \n" -: "=r"(_res), "+m"(*lock) -: "r"(lock) -: "memory"); -#if defined(__sparcv7) || defined(__sparc_v7__) - /* - * No stbar or membar available, luckily no actually produced hardware - * requires a barrier. - */ -#elif defined(__sparcv8) || defined(__sparc_v8__) - /* stbar is available (and required for both PSO, RMO), membar isn't */ - __asm__ __volatile__ ("stbar \n":::"memory"); -#else - /* - * #LoadStore (RMO) | #LoadLoad (RMO) together are the appropriate acquire - * barrier for sparcv8+ upwards. - */ - __asm__ __volatile__ ("membar #LoadStore | #LoadLoad \n":::"memory"); -#endif - return (int) _res; -} - -#if defined(__sparcv7) || defined(__sparc_v7__) -/* - * No stbar or membar available, luckily no actually produced hardware - * requires a barrier. We fall through to the default gcc definition of - * S_UNLOCK in this case. - */ -#elif defined(__sparcv8) || defined(__sparc_v8__) -/* stbar is available (and required for both PSO, RMO), membar isn't */ -#define S_UNLOCK(lock) \ -do \ -{ \ - __asm__ __volatile__ ("stbar \n":::"memory"); \ - *((volatile slock_t *) (lock)) = 0; \ -} while (0) -#else -/* - * #LoadStore (RMO) | #StoreStore (RMO, PSO) together are the appropriate - * release barrier for sparcv8+ upwards. - */ -#define S_UNLOCK(lock) \ -do \ -{ \ - __asm__ __volatile__ ("membar #LoadStore | #StoreStore \n":::"memory"); \ - *((volatile slock_t *) (lock)) = 0; \ -} while (0) -#endif - -#endif /* __sparc__ */ - - -/* PowerPC */ -#if defined(__powerpc__) || defined(__powerpc64__) - -typedef unsigned int slock_t; - -#define TAS(lock) tas(lock) - -/* On PPC, it's a win to use a non-locking test before the lwarx */ -#define TAS_SPIN(lock) (*(lock) ? 1 : TAS(lock)) - -/* - * The second operand of addi can hold a constant zero or a register number, - * hence constraint "=&b" to avoid allocating r0. "b" stands for "address - * base register"; most operands having this register-or-zero property are - * address bases, e.g. the second operand of lwax. - * - * NOTE: per the Enhanced PowerPC Architecture manual, v1.0 dated 7-May-2002, - * an isync is a sufficient synchronization barrier after a lwarx/stwcx loop. - * But if the spinlock is in ordinary memory, we can use lwsync instead for - * better performance. - */ -static inline int -tas(volatile slock_t *lock) -{ - slock_t _t; - int _res; - - __asm__ __volatile__( -" lwarx %0,0,%3,1 \n" -" cmpwi %0,0 \n" -" bne TAS%=_fail \n" -" addi %0,%0,1 \n" -" stwcx. %0,0,%3 \n" -" beq TAS%=_ok \n" -"TAS%=_fail: \n" -" li %1,1 \n" -" b TAS%=_out \n" -"TAS%=_ok: \n" -" lwsync \n" -" li %1,0 \n" -"TAS%=_out: \n" -: "=&b"(_t), "=r"(_res), "+m"(*lock) -: "r"(lock) -: "memory", "cc"); - return _res; -} - -/* - * PowerPC S_UNLOCK is almost standard but requires a "sync" instruction. - * But we can use lwsync instead for better performance. - */ -#define S_UNLOCK(lock) \ -do \ -{ \ - __asm__ __volatile__ (" lwsync \n" ::: "memory"); \ - *((volatile slock_t *) (lock)) = 0; \ -} while (0) - -#endif /* powerpc */ - - -#if defined(__mips__) && !defined(__sgi) /* non-SGI MIPS */ - -typedef unsigned int slock_t; - -#define TAS(lock) tas(lock) - -/* - * Original MIPS-I processors lacked the LL/SC instructions, but if we are - * so unfortunate as to be running on one of those, we expect that the kernel - * will handle the illegal-instruction traps and emulate them for us. On - * anything newer (and really, MIPS-I is extinct) LL/SC is the only sane - * choice because any other synchronization method must involve a kernel - * call. Unfortunately, many toolchains still default to MIPS-I as the - * codegen target; if the symbol __mips shows that that's the case, we - * have to force the assembler to accept LL/SC. - * - * R10000 and up processors require a separate SYNC, which has the same - * issues as LL/SC. - */ -#if __mips < 2 -#define MIPS_SET_MIPS2 " .set mips2 \n" -#else -#define MIPS_SET_MIPS2 -#endif - -static inline int -tas(volatile slock_t *lock) -{ - volatile slock_t *_l = lock; - int _res; - int _tmp; - - __asm__ __volatile__( - " .set push \n" - MIPS_SET_MIPS2 - " .set noreorder \n" - " .set nomacro \n" - " ll %0, %2 \n" - " or %1, %0, 1 \n" - " sc %1, %2 \n" - " xori %1, 1 \n" - " or %0, %0, %1 \n" - " sync \n" - " .set pop " -: "=&r" (_res), "=&r" (_tmp), "+R" (*_l) -: /* no inputs */ -: "memory"); - return _res; -} - -/* MIPS S_UNLOCK is almost standard but requires a "sync" instruction */ -#define S_UNLOCK(lock) \ -do \ -{ \ - __asm__ __volatile__( \ - " .set push \n" \ - MIPS_SET_MIPS2 \ - " .set noreorder \n" \ - " .set nomacro \n" \ - " sync \n" \ - " .set pop " \ -: /* no outputs */ \ -: /* no inputs */ \ -: "memory"); \ - *((volatile slock_t *) (lock)) = 0; \ -} while (0) - -#endif /* __mips__ && !__sgi */ - - - -/* - * If we have no platform-specific knowledge, but we found that the compiler - * provides __sync_lock_test_and_set(), use that. Prefer the int-width - * version over the char-width version if we have both, on the rather dubious - * grounds that that's known to be more likely to work in the ARM ecosystem. - * (But we dealt with ARM above.) - */ -#if !defined(TAS) - -#if defined(HAVE_GCC__SYNC_INT32_TAS) - -#define TAS(lock) tas(lock) - -typedef int slock_t; - -static inline int -tas(volatile slock_t *lock) -{ - return __sync_lock_test_and_set(lock, 1); -} - -#define S_UNLOCK(lock) __sync_lock_release(lock) - -#elif defined(HAVE_GCC__SYNC_CHAR_TAS) - -#define TAS(lock) tas(lock) - -typedef char slock_t; - -static inline int -tas(volatile slock_t *lock) -{ - return __sync_lock_test_and_set(lock, 1); -} - -#define S_UNLOCK(lock) __sync_lock_release(lock) - -#endif /* HAVE_GCC__SYNC_INT32_TAS */ - -#endif /* !defined(TAS) */ - - -/* - * Default implementation of S_UNLOCK() for gcc/icc. - * - * Note that this implementation is unsafe for any platform that can reorder - * a memory access (either load or store) after a following store. That - * happens not to be possible on x86 and most legacy architectures (some are - * single-processor!), but many modern systems have weaker memory ordering. - * Those that do must define their own version of S_UNLOCK() rather than - * relying on this one. - */ -#if !defined(S_UNLOCK) -#define S_UNLOCK(lock) \ - do { __asm__ __volatile__("" : : : "memory"); *(lock) = 0; } while (0) -#endif - -#endif /* defined(__GNUC__) || defined(__INTEL_COMPILER) */ - - -/* - * --------------------------------------------------------------------- - * Platforms that use non-gcc inline assembly: - * --------------------------------------------------------------------- - */ - -#if !defined(TAS) /* We didn't trigger above, let's try here */ - -#ifdef _MSC_VER -typedef LONG slock_t; - -#define TAS(lock) (InterlockedCompareExchange(lock, 1, 0)) - -#define SPIN_DELAY() spin_delay() - -#ifdef __aarch64__ -static __forceinline void -spin_delay(void) -{ - /* - * Research indicates ISB is better than __yield() on AArch64. See - * https://postgr.es/m/1c2a29b8-5b1e-44f7-a871-71ec5fefc120%40app.fastmail.com. - */ - __isb(_ARM64_BARRIER_SY); -} -#elif defined(_WIN64) -static __forceinline void -spin_delay(void) -{ - /* - * If using Visual C++ on Win64, inline assembly is unavailable. - * Use a _mm_pause intrinsic instead of rep nop. - */ - _mm_pause(); -} -#else -static __forceinline void -spin_delay(void) -{ - /* See comment for gcc code. Same code, MASM syntax */ - __asm rep nop; -} -#endif - -#include - -#ifdef __aarch64__ - -/* _ReadWriteBarrier() is insufficient on non-TSO architectures. */ -#pragma intrinsic(_InterlockedExchange) -#define S_UNLOCK(lock) _InterlockedExchange(lock, 0) - -#else - -#pragma intrinsic(_ReadWriteBarrier) -#define S_UNLOCK(lock) \ - do { _ReadWriteBarrier(); (*(lock)) = 0; } while (0) - -#endif -#endif - - -#endif /* !defined(TAS) */ - - -/* - * Default Definitions - override these above as needed. - */ - -/* - * Make sure S_LOCK is defined, either explicitly for the platform or via a TAS - * macro for the platform. - */ -#if !defined(S_LOCK) -#ifdef TAS -#define USE_DEFAULT_S_LOCK -extern int s_lock(volatile slock_t *lock, const char *file, int line, const char *func); -#define S_LOCK(lock) \ - (TAS(lock) ? s_lock((lock), __FILE__, __LINE__, __func__) : 0) -#else -#error PostgreSQL does not have spinlock support on this platform. Please report this to pgsql-bugs@lists.postgresql.org. -#endif /* TAS */ -#endif /* S_LOCK */ - -#if !defined(S_UNLOCK) -/* - * Our default implementation of S_UNLOCK is essentially *(lock) = 0. This - * is unsafe if the platform can reorder a memory access (either load or - * store) after a following store; platforms where this is possible must - * define their own S_UNLOCK. But CPU reordering is not the only concern: - * if we simply defined S_UNLOCK() as an inline macro, the compiler might - * reorder instructions from inside the critical section to occur after the - * lock release. Since the compiler probably can't know what the external - * function s_unlock is doing, putting the same logic there should be adequate. - * A sufficiently-smart globally optimizing compiler could break that - * assumption, though, and the cost of a function call for every spinlock - * release may hurt performance significantly, so we use this implementation - * only for platforms where we don't know of a suitable intrinsic. For the - * most part, those are relatively obscure platform/compiler combinations to - * which the PostgreSQL project does not have access. - */ -#define USE_DEFAULT_S_UNLOCK -extern void s_unlock(volatile slock_t *lock); -#define S_UNLOCK(lock) s_unlock(lock) -#endif /* S_UNLOCK */ - -#if !defined(S_INIT_LOCK) -#define S_INIT_LOCK(lock) S_UNLOCK(lock) -#endif /* S_INIT_LOCK */ - -#if !defined(SPIN_DELAY) -#define SPIN_DELAY() ((void) 0) -#endif /* SPIN_DELAY */ - -/* - * TAS_SPIN is only needed by the default S_LOCK's helper function (s_lock()), - * so we only provide a default when there is a TAS to base it on. - */ -#if !defined(TAS_SPIN) && defined(TAS) -#define TAS_SPIN(lock) TAS(lock) -#endif /* ! TAS_SPIN && TAS */ - - -/* - * Platform-independent out-of-line support routines - */ - -/* Support for dynamic adjustment of spins_per_delay */ -#define DEFAULT_SPINS_PER_DELAY 100 - -extern void set_spins_per_delay(int shared_spins_per_delay); -extern int update_spins_per_delay(int shared_spins_per_delay); - -/* - * Support for spin delay which is useful in various places where - * spinlock-like procedures take place. - */ -typedef struct -{ - int spins; - int delays; - int cur_delay; - const char *file; - int line; - const char *func; -} SpinDelayStatus; - -static inline void -init_spin_delay(SpinDelayStatus *status, - const char *file, int line, const char *func) -{ - status->spins = 0; - status->delays = 0; - status->cur_delay = 0; - status->file = file; - status->line = line; - status->func = func; -} - -#define init_local_spin_delay(status) init_spin_delay(status, __FILE__, __LINE__, __func__) -extern void perform_spin_delay(SpinDelayStatus *status); -extern void finish_spin_delay(SpinDelayStatus *status); - -#endif /* S_LOCK_H */ diff --git a/src/include/storage/spin.h b/src/include/storage/spin.h index 56ef38dffb0..5e10355032d 100644 --- a/src/include/storage/spin.h +++ b/src/include/storage/spin.h @@ -55,8 +55,6 @@ * So slock_t uses the traditional convention: 0 == unlocked, 1 == locked, * which makes a zero-initialized slock_t a valid, free lock. */ -#ifdef USE_STDATOMIC_H - #include "port/atomics.h" typedef pg_atomic_uint32 slock_t; @@ -107,29 +105,5 @@ SpinLockRelease(volatile slock_t *lock) atomic_store_explicit(lock, 0, memory_order_release); } -#else /* !USE_STDATOMIC_H */ - -/* Traditional implementation: unchanged from master. */ -#include "storage/s_lock.h" - -static inline void -SpinLockInit(volatile slock_t *lock) -{ - S_INIT_LOCK(lock); -} - -static inline void -SpinLockAcquire(volatile slock_t *lock) -{ - S_LOCK(lock); -} - -static inline void -SpinLockRelease(volatile slock_t *lock) -{ - S_UNLOCK(lock); -} - -#endif /* USE_STDATOMIC_H */ #endif /* SPIN_H */ diff --git a/src/test/regress/regress.c b/src/test/regress/regress.c index 2fd3b260cbd..ba6afe2bc79 100644 --- a/src/test/regress/regress.c +++ b/src/test/regress/regress.c @@ -651,11 +651,8 @@ test_spinlock(void) * Basic tests for spinlocks, as well as the underlying operations. * * We embed the spinlock in a struct with other members to test that the - * spinlock operations don't perform too wide writes. - * - * Note: slock_t is pg_atomic_uint32 (4 bytes) in the stdatomic path vs - * potentially smaller (e.g., unsigned char) in the traditional path. This - * test verifies no over-wide writes regardless of slock_t size. + * spinlock operations don't perform too wide writes. (slock_t is a + * pg_atomic_uint32, i.e. 4 bytes.) */ { struct test_lock_struct @@ -673,41 +670,9 @@ test_spinlock(void) SpinLockAcquire(&struct_w_lock.lock); SpinLockRelease(&struct_w_lock.lock); -#ifndef USE_STDATOMIC_H - /* test basic operations via underlying S_* API */ - S_INIT_LOCK(&struct_w_lock.lock); - S_LOCK(&struct_w_lock.lock); - S_UNLOCK(&struct_w_lock.lock); -#endif - /* and that "contended" acquisition works */ -#ifdef USE_STDATOMIC_H s_lock(&struct_w_lock.lock, "testfile", 17, "testfunc"); SpinLockRelease(&struct_w_lock.lock); -#else - S_LOCK(&struct_w_lock.lock); - S_UNLOCK(&struct_w_lock.lock); -#endif - -#ifndef USE_STDATOMIC_H - /* - * Check, using TAS directly, that a single spin cycle doesn't block - * when acquiring an already acquired lock. - */ -#ifdef TAS - S_LOCK(&struct_w_lock.lock); - - if (!TAS(&struct_w_lock.lock)) - elog(ERROR, "acquired already held spinlock"); - -#ifdef TAS_SPIN - if (!TAS_SPIN(&struct_w_lock.lock)) - elog(ERROR, "acquired already held spinlock"); -#endif /* defined(TAS_SPIN) */ - - S_UNLOCK(&struct_w_lock.lock); -#endif /* defined(TAS) */ -#endif /* !USE_STDATOMIC_H */ /* * Verify that after all of this the non-lock contents are still diff --git a/src/tools/pginclude/headerscheck b/src/tools/pginclude/headerscheck index 785d6f867ad..378100acf2c 100755 --- a/src/tools/pginclude/headerscheck +++ b/src/tools/pginclude/headerscheck @@ -107,16 +107,6 @@ do test "$f" = src/include/port/win32ntdll.h && continue test "$f" = src/port/pthread-win32.h && continue - # Likewise, these files are platform-specific, and the one - # relevant to our platform will be included by atomics.h. - test "$f" = src/include/port/atomics/arch-arm.h && continue - test "$f" = src/include/port/atomics/arch-ppc.h && continue - test "$f" = src/include/port/atomics/arch-x86.h && continue - test "$f" = src/include/port/atomics/fallback.h && continue - test "$f" = src/include/port/atomics/generic.h && continue - test "$f" = src/include/port/atomics/generic-gcc.h && continue - test "$f" = src/include/port/atomics/generic-msvc.h && continue - # sepgsql.h depends on headers that aren't there on most platforms. test "$f" = contrib/sepgsql/sepgsql.h && continue -- 2.50.1