From 3359d5d5dd828bc0317ca28474ebe5dd931d44e8 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Wed, 4 Jul 2018 22:13:58 +0200 Subject: [PATCH 1/2] Add some int128 atomics support --- config/c-compiler.m4 | 31 +++++++++ configure | 65 ++++++++++++++++++ configure.in | 2 + src/backend/port/atomics.c | 51 +++++++++++++++ src/include/pg_config.h.in | 8 +++ src/include/port/atomics.h | 49 ++++++++++++++ src/include/port/atomics/fallback.h | 30 +++++++++ src/include/port/atomics/generic-gcc.h | 46 +++++++++++++ src/include/port/atomics/generic.h | 91 ++++++++++++++++++++++++++ 9 files changed, 373 insertions(+) diff --git a/config/c-compiler.m4 b/config/c-compiler.m4 index 9731a517de..a4aecd3e61 100644 --- a/config/c-compiler.m4 +++ b/config/c-compiler.m4 @@ -606,6 +606,21 @@ if test x"$pgac_cv_gcc_sync_int64_cas" = x"yes"; then AC_DEFINE(HAVE_GCC__SYNC_INT64_CAS, 1, [Define to 1 if you have __sync_val_compare_and_swap(int64 *, int64, int64).]) fi])# PGAC_HAVE_GCC__SYNC_INT64_CAS +# PGAC_HAVE_GCC__SYNC_INT128_CAS +# ------------------------- +# Check if the C compiler understands __sync_compare_and_swap() for 128bit +# types, and define HAVE_GCC__SYNC_INT128_CAS if so. +AC_DEFUN([PGAC_HAVE_GCC__SYNC_INT128_CAS], +[AC_CACHE_CHECK(for builtin __sync int128 atomic operations, pgac_cv_gcc_sync_int128_cas, +[AC_LINK_IFELSE([AC_LANG_PROGRAM([], + [PG_INT128_TYPE lock = 0; + __sync_val_compare_and_swap(&lock, 0, (PG_INT128_TYPE) 37);])], + [pgac_cv_gcc_sync_int128_cas="yes"], + [pgac_cv_gcc_sync_int128_cas="no"])]) +if test x"$pgac_cv_gcc_sync_int128_cas" = x"yes"; then + AC_DEFINE(HAVE_GCC__SYNC_INT128_CAS, 1, [Define to 1 if you have __sync_val_compare_and_swap(int128 *, int128, int128).]) +fi])# PGAC_HAVE_GCC__SYNC_INT128_CAS + # PGAC_HAVE_GCC__ATOMIC_INT32_CAS # ------------------------- # Check if the C compiler understands __atomic_compare_exchange_n() for 32bit @@ -638,6 +653,22 @@ if test x"$pgac_cv_gcc_atomic_int64_cas" = x"yes"; then AC_DEFINE(HAVE_GCC__ATOMIC_INT64_CAS, 1, [Define to 1 if you have __atomic_compare_exchange_n(int64 *, int64 *, int64).]) fi])# PGAC_HAVE_GCC__ATOMIC_INT64_CAS +# PGAC_HAVE_GCC__ATOMIC_INT128_CAS +# ------------------------- +# Check if the C compiler understands __atomic_compare_exchange_n() for 128bit +# types, and define HAVE_GCC__ATOMIC_INT128_CAS if so. +AC_DEFUN([PGAC_HAVE_GCC__ATOMIC_INT128_CAS], +[AC_CACHE_CHECK(for builtin __atomic int128 atomic operations, pgac_cv_gcc_atomic_int128_cas, +[AC_LINK_IFELSE([AC_LANG_PROGRAM([], + [PG_INT128_TYPE val = 0; + PG_INT128_TYPE expect = 0; + __atomic_compare_exchange_n(&val, &expect, 37, 0, __ATOMIC_SEQ_CST, __ATOMIC_RELAXED);])], + [pgac_cv_gcc_atomic_int128_cas="yes"], + [pgac_cv_gcc_atomic_int128_cas="no"])]) +if test x"$pgac_cv_gcc_atomic_int128_cas" = x"yes"; then + AC_DEFINE(HAVE_GCC__ATOMIC_INT128_CAS, 1, [Define to 1 if you have __atomic_compare_exchange_n(int128 *, int128 *, int128).]) +fi])# PGAC_HAVE_GCC__ATOMIC_INT128_CAS + # PGAC_SSE42_CRC32_INTRINSICS # ----------------------- # Check if the compiler supports the x86 CRC instructions added in SSE 4.2, diff --git a/configure b/configure index 1bc465b942..e7bd1edc6a 100755 --- a/configure +++ b/configure @@ -17046,6 +17046,38 @@ if test x"$pgac_cv_gcc_sync_int64_cas" = x"yes"; then $as_echo "#define HAVE_GCC__SYNC_INT64_CAS 1" >>confdefs.h +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for builtin __sync int128 atomic operations" >&5 +$as_echo_n "checking for builtin __sync int128 atomic operations... " >&6; } +if ${pgac_cv_gcc_sync_int128_cas+:} false; then : + $as_echo_n "(cached) " >&6 +else + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main () +{ +PG_INT128_TYPE lock = 0; + __sync_val_compare_and_swap(&lock, 0, (PG_INT128_TYPE) 37); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + pgac_cv_gcc_sync_int128_cas="yes" +else + pgac_cv_gcc_sync_int128_cas="no" +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $pgac_cv_gcc_sync_int128_cas" >&5 +$as_echo "$pgac_cv_gcc_sync_int128_cas" >&6; } +if test x"$pgac_cv_gcc_sync_int128_cas" = x"yes"; then + +$as_echo "#define HAVE_GCC__SYNC_INT128_CAS 1" >>confdefs.h + fi { $as_echo "$as_me:${as_lineno-$LINENO}: checking for builtin __atomic int32 atomic operations" >&5 $as_echo_n "checking for builtin __atomic int32 atomic operations... " >&6; } @@ -17112,6 +17144,39 @@ if test x"$pgac_cv_gcc_atomic_int64_cas" = x"yes"; then $as_echo "#define HAVE_GCC__ATOMIC_INT64_CAS 1" >>confdefs.h +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for builtin __atomic int128 atomic operations" >&5 +$as_echo_n "checking for builtin __atomic int128 atomic operations... " >&6; } +if ${pgac_cv_gcc_atomic_int128_cas+:} false; then : + $as_echo_n "(cached) " >&6 +else + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main () +{ +PG_INT128_TYPE val = 0; + PG_INT128_TYPE expect = 0; + __atomic_compare_exchange_n(&val, &expect, 37, 0, __ATOMIC_SEQ_CST, __ATOMIC_RELAXED); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + pgac_cv_gcc_atomic_int128_cas="yes" +else + pgac_cv_gcc_atomic_int128_cas="no" +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $pgac_cv_gcc_atomic_int128_cas" >&5 +$as_echo "$pgac_cv_gcc_atomic_int128_cas" >&6; } +if test x"$pgac_cv_gcc_atomic_int128_cas" = x"yes"; then + +$as_echo "#define HAVE_GCC__ATOMIC_INT128_CAS 1" >>confdefs.h + fi diff --git a/configure.in b/configure.in index a6b3b88cfa..24a841daa8 100644 --- a/configure.in +++ b/configure.in @@ -1946,8 +1946,10 @@ PGAC_HAVE_GCC__SYNC_CHAR_TAS PGAC_HAVE_GCC__SYNC_INT32_TAS PGAC_HAVE_GCC__SYNC_INT32_CAS PGAC_HAVE_GCC__SYNC_INT64_CAS +PGAC_HAVE_GCC__SYNC_INT128_CAS PGAC_HAVE_GCC__ATOMIC_INT32_CAS PGAC_HAVE_GCC__ATOMIC_INT64_CAS +PGAC_HAVE_GCC__ATOMIC_INT128_CAS # Check for x86 cpuid instruction diff --git a/src/backend/port/atomics.c b/src/backend/port/atomics.c index caa84bf2b6..ee73d8ea0a 100644 --- a/src/backend/port/atomics.c +++ b/src/backend/port/atomics.c @@ -237,3 +237,54 @@ pg_atomic_fetch_add_u64_impl(volatile pg_atomic_uint64 *ptr, int64 add_) } #endif /* PG_HAVE_ATOMIC_U64_SIMULATION */ + + +#ifdef PG_HAVE_ATOMIC_U128_SIMULATION + +void +pg_atomic_init_u128_impl(volatile pg_atomic_uint128 *ptr, uint128 val_) +{ + StaticAssertStmt(sizeof(ptr->sema) >= sizeof(slock_t), + "size mismatch of atomic_uint128 vs slock_t"); + + /* + * If we're using semaphore based atomic flags, be careful about nested + * usage of atomics while a spinlock is held. + */ +#ifndef HAVE_SPINLOCKS + s_init_lock_sema((slock_t *) &ptr->sema, true); +#else + SpinLockInit((slock_t *) &ptr->sema); +#endif + ptr->value = val_; +} + +bool +pg_atomic_compare_exchange_u128_impl(volatile pg_atomic_uint128 *ptr, + uint128 *expected, uint128 newval) +{ + bool ret; + + /* + * Do atomic op under a spinlock. It might look like we could just skip + * the cmpxchg if the lock isn't available, but that'd just emulate a + * 'weak' compare and swap. I.e. one that allows spurious failures. Since + * several algorithms rely on a strong variant and that is efficiently + * implementable on most major architectures let's emulate it here as + * well. + */ + SpinLockAcquire((slock_t *) &ptr->sema); + + /* perform compare/exchange logic */ + ret = ptr->value == *expected; + *expected = ptr->value; + if (ret) + ptr->value = newval; + + /* and release lock */ + SpinLockRelease((slock_t *) &ptr->sema); + + return ret; +} + +#endif /* PG_HAVE_ATOMIC_U128_SIMULATION */ diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index dcb25bb723..9a745cdf6c 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -225,6 +225,10 @@ /* Define to 1 if your compiler understands __FUNCTION__. */ #undef HAVE_FUNCNAME__FUNCTION +/* Define to 1 if you have __atomic_compare_exchange_n(int128 *, int128 *, + int128). */ +#undef HAVE_GCC__ATOMIC_INT128_CAS + /* Define to 1 if you have __atomic_compare_exchange_n(int *, int *, int). */ #undef HAVE_GCC__ATOMIC_INT32_CAS @@ -235,6 +239,10 @@ /* Define to 1 if you have __sync_lock_test_and_set(char *) and friends. */ #undef HAVE_GCC__SYNC_CHAR_TAS +/* Define to 1 if you have __sync_val_compare_and_swap(int128 *, int128, + int128). */ +#undef HAVE_GCC__SYNC_INT128_CAS + /* Define to 1 if you have __sync_val_compare_and_swap(int *, int, int). */ #undef HAVE_GCC__SYNC_INT32_CAS diff --git a/src/include/port/atomics.h b/src/include/port/atomics.h index 0470391675..6cb2fddb70 100644 --- a/src/include/port/atomics.h +++ b/src/include/port/atomics.h @@ -522,6 +522,55 @@ pg_atomic_sub_fetch_u64(volatile pg_atomic_uint64 *ptr, int64 sub_) return pg_atomic_sub_fetch_u64_impl(ptr, sub_); } +/* ---- + * The 128 bit operations have the same semantics as their 32bit counterparts + * if they are available. Check the corresponding 32bit function for + * documentation. + * ---- + */ +static inline void +pg_atomic_init_u128(volatile pg_atomic_uint128 *ptr, uint128 val) +{ + /* + * Can't necessarily enforce alignment - and don't need it - when using + * the spinlock based fallback implementation. Therefore only assert when + * not using it. + */ +#ifndef PG_HAVE_ATOMIC_U128_SIMULATION + AssertPointerAlignment(ptr, 16); +#endif + pg_atomic_init_u128_impl(ptr, val); +} + +static inline uint128 +pg_atomic_read_u128(volatile pg_atomic_uint128 *ptr) +{ +#ifndef PG_HAVE_ATOMIC_U128_SIMULATION + AssertPointerAlignment(ptr, 16); +#endif + return pg_atomic_read_u128_impl(ptr); +} + +static inline void +pg_atomic_write_u128(volatile pg_atomic_uint128 *ptr, uint128 val) +{ +#ifndef PG_HAVE_ATOMIC_U128_SIMULATION + AssertPointerAlignment(ptr, 16); +#endif + pg_atomic_write_u128_impl(ptr, val); +} + +static inline bool +pg_atomic_compare_exchange_u128(volatile pg_atomic_uint128 *ptr, + uint128 *expected, uint128 newval) +{ +#ifndef PG_HAVE_ATOMIC_U128_SIMULATION + AssertPointerAlignment(ptr, 16); + AssertPointerAlignment(expected, 16); +#endif + return pg_atomic_compare_exchange_u128_impl(ptr, expected, newval); +} + #undef INSIDE_ATOMICS_H #endif /* ATOMICS_H */ diff --git a/src/include/port/atomics/fallback.h b/src/include/port/atomics/fallback.h index 88a967ad5b..9d0158e6da 100644 --- a/src/include/port/atomics/fallback.h +++ b/src/include/port/atomics/fallback.h @@ -121,6 +121,24 @@ typedef struct pg_atomic_uint64 #endif /* PG_HAVE_ATOMIC_U64_SUPPORT */ +#if !defined(PG_HAVE_ATOMIC_U128_SUPPORT) + +#define PG_HAVE_ATOMIC_U128_SIMULATION + +#define PG_HAVE_ATOMIC_U128_SUPPORT +typedef struct pg_atomic_uint128 +{ + /* Check pg_atomic_flag's definition above for an explanation */ +#if defined(__hppa) || defined(__hppa__) /* HP PA-RISC, GCC and HP compilers */ + int sema[4]; +#else + int sema; +#endif + volatile uint128 value; +} pg_atomic_uint128; + +#endif /* PG_HAVE_ATOMIC_U128_SUPPORT */ + #ifdef PG_HAVE_ATOMIC_FLAG_SIMULATION #define PG_HAVE_ATOMIC_INIT_FLAG @@ -168,3 +186,15 @@ extern bool pg_atomic_compare_exchange_u64_impl(volatile pg_atomic_uint64 *ptr, extern uint64 pg_atomic_fetch_add_u64_impl(volatile pg_atomic_uint64 *ptr, int64 add_); #endif /* PG_HAVE_ATOMIC_U64_SIMULATION */ + + +#ifdef PG_HAVE_ATOMIC_U128_SIMULATION + +#define PG_HAVE_ATOMIC_INIT_U128 +extern void pg_atomic_init_u128_impl(volatile pg_atomic_uint128 *ptr, uint128 val_); + +#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U128 +extern bool pg_atomic_compare_exchange_u128_impl(volatile pg_atomic_uint128 *ptr, + uint128 *expected, uint128 newval); + +#endif /* PG_HAVE_ATOMIC_U128_SIMULATION */ diff --git a/src/include/port/atomics/generic-gcc.h b/src/include/port/atomics/generic-gcc.h index 3a1dc88377..b0818dd644 100644 --- a/src/include/port/atomics/generic-gcc.h +++ b/src/include/port/atomics/generic-gcc.h @@ -102,6 +102,20 @@ typedef struct pg_atomic_uint64 #endif /* defined(HAVE_GCC__ATOMIC_INT64_CAS) || defined(HAVE_GCC__SYNC_INT64_CAS) */ +/* generic gcc based atomic uint128 implementation */ +#if !defined(PG_HAVE_ATOMIC_U128_SUPPORT) \ + && !defined(PG_DISABLE_128_BIT_ATOMICS) \ + && (defined(HAVE_GCC__ATOMIC_INT128_CAS) || defined(HAVE_GCC__SYNC_INT128_CAS)) + +#define PG_HAVE_ATOMIC_U128_SUPPORT + +typedef struct pg_atomic_uint128 +{ + volatile uint128 value pg_attribute_aligned(8); +} pg_atomic_uint128; + +#endif /* defined(HAVE_GCC__ATOMIC_INT128_CAS) || defined(HAVE_GCC__SYNC_INT128_CAS) */ + #ifdef PG_HAVE_ATOMIC_FLAG_SUPPORT #if defined(HAVE_GCC__SYNC_CHAR_TAS) || defined(HAVE_GCC__SYNC_INT32_TAS) @@ -283,4 +297,36 @@ pg_atomic_fetch_or_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 or_) #endif /* !defined(PG_DISABLE_64_BIT_ATOMICS) */ + +#if !defined(PG_DISABLE_128_BIT_ATOMICS) + +#if !defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U128) && defined(HAVE_GCC__ATOMIC_INT128_CAS) +#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U128 +static inline bool +pg_atomic_compare_exchange_u128_impl(volatile pg_atomic_uint128 *ptr, + uint128 *expected, uint128 newval) +{ + return __atomic_compare_exchange_n(&ptr->value, expected, newval, false, + __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); +} +#endif + +#if !defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U128) && defined(HAVE_GCC__SYNC_INT128_CAS) +#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U128 +static inline bool +pg_atomic_compare_exchange_u128_impl(volatile pg_atomic_uint128 *ptr, + uint128 *expected, uint128 newval) +{ + bool ret; + uint128 current; + current = __sync_val_compare_and_swap(&ptr->value, *expected, newval); + ret = current == *expected; + *expected = current; + return ret; +} +#endif + +#endif /* !defined(PG_DISABLE_128_BIT_ATOMICS) */ + + #endif /* defined(HAVE_ATOMICS) */ diff --git a/src/include/port/atomics/generic.h b/src/include/port/atomics/generic.h index ea11698a35..0d3ea2e3aa 100644 --- a/src/include/port/atomics/generic.h +++ b/src/include/port/atomics/generic.h @@ -399,3 +399,94 @@ pg_atomic_sub_fetch_u64_impl(volatile pg_atomic_uint64 *ptr, int64 sub_) return pg_atomic_fetch_sub_u64_impl(ptr, sub_) - sub_; } #endif + +#if !defined(PG_HAVE_ATOMIC_EXCHANGE_U128) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U128) +#define PG_HAVE_ATOMIC_EXCHANGE_U128 +static inline uint128 +pg_atomic_exchange_u128_impl(volatile pg_atomic_uint128 *ptr, uint128 xchg_) +{ + uint128 old; + old = ptr->value; /* ok if read is not atomic */ + while (!pg_atomic_compare_exchange_u128_impl(ptr, &old, xchg_)) + /* skip */; + return old; +} +#endif + +#ifndef PG_HAVE_ATOMIC_WRITE_U128 +#define PG_HAVE_ATOMIC_WRITE_U128 + +#if defined(PG_HAVE_16BYTE_SINGLE_COPY_ATOMICITY) && \ + !defined(PG_HAVE_ATOMIC_U128_SIMULATION) + +static inline void +pg_atomic_write_u128_impl(volatile pg_atomic_uint128 *ptr, uint128 val) +{ + /* + * On this platform aligned 128bit writes are guaranteed to be atomic, + * except if using the fallback implementation, where can't guarantee the + * required alignment. + */ + AssertPointerAlignment(ptr, 16); + ptr->value = val; +} + +#else + +static inline void +pg_atomic_write_u128_impl(volatile pg_atomic_uint128 *ptr, uint128 val) +{ + /* + * 128 bit writes aren't safe on all platforms. In the generic + * implementation implement them as an atomic exchange. + */ + pg_atomic_exchange_u128_impl(ptr, val); +} + +#endif /* PG_HAVE_16BYTE_SINGLE_COPY_ATOMICITY && !PG_HAVE_ATOMIC_U128_SIMULATION */ +#endif /* PG_HAVE_ATOMIC_WRITE_U128 */ + +#ifndef PG_HAVE_ATOMIC_READ_U128 +#define PG_HAVE_ATOMIC_READ_U128 + +#if defined(PG_HAVE_16BYTE_SINGLE_COPY_ATOMICITY) && \ + !defined(PG_HAVE_ATOMIC_U128_SIMULATION) + +static inline uint128 +pg_atomic_read_u128_impl(volatile pg_atomic_uint128 *ptr) +{ + /* + * On this platform aligned 128-bit reads are guaranteed to be atomic. + */ + AssertPointerAlignment(ptr, 16); + return ptr->value; +} + +#else + +static inline uint128 +pg_atomic_read_u128_impl(volatile pg_atomic_uint128 *ptr) +{ + uint128 old = 0; + + /* + * 128-bit reads aren't atomic on all platforms. In the generic + * implementation implement them as a compare/exchange with 0. That'll + * fail or succeed, but always return the old value. Possibly might store + * a 0, but only if the previous value also was a 0 - i.e. harmless. + */ + pg_atomic_compare_exchange_u128_impl(ptr, &old, 0); + + return old; +} +#endif /* PG_HAVE_16BYTE_SINGLE_COPY_ATOMICITY && !PG_HAVE_ATOMIC_U128_SIMULATION */ +#endif /* PG_HAVE_ATOMIC_READ_U64 */ + +#ifndef PG_HAVE_ATOMIC_INIT_U128 +#define PG_HAVE_ATOMIC_INIT_U128 +static inline void +pg_atomic_init_u128_impl(volatile pg_atomic_uint128 *ptr, uint128 val_) +{ + pg_atomic_write_u128_impl(ptr, val_); +} +#endif base-commit: 1486f7f981c0052988891677d4e734b14317816c -- 2.18.0