/* * Copyright 2024-present MongoDB, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ /* Note: this file was derived from libbson's bson-atomic.h */ #ifndef PHONGO_ATOMIC_H #define PHONGO_ATOMIC_H #include "bson/bson.h" #ifdef _MSC_VER #include #endif enum phongo_memory_order { phongo_memory_order_seq_cst, phongo_memory_order_acquire, phongo_memory_order_release, phongo_memory_order_relaxed, phongo_memory_order_acq_rel, phongo_memory_order_consume, }; #if defined(_M_ARM) /* MSVC memorder atomics are only avail on ARM */ #define MSVC_MEMORDER_SUFFIX(X) X #else #define MSVC_MEMORDER_SUFFIX(X) #endif #if defined(USE_LEGACY_GCC_ATOMICS) || (!defined(__clang__) && __GNUC__ == 4) || defined(__xlC__) #define PHONGO_USE_LEGACY_GCC_ATOMICS #else #undef PHONGO_USE_LEGACY_GCC_ATOMICS #endif /* Not all GCC-like compilers support the current __atomic built-ins. Older * GCC (pre-5) used different built-ins named with the __sync prefix. When * compiling with such older GCC versions, it is necessary to use the applicable * functions, which requires redefining BSON_IF_GNU_LIKE and defining the * additional PHONGO_IF_GNU_LEGACY_ATOMICS macro here. */ #ifdef PHONGO_USE_LEGACY_GCC_ATOMICS #undef BSON_IF_GNU_LIKE #define BSON_IF_GNU_LIKE(...) #define BSON_IF_MSVC(...) #define PHONGO_IF_GNU_LEGACY_ATOMICS(...) __VA_ARGS__ #else #define PHONGO_IF_GNU_LEGACY_ATOMICS(...) #endif /* CDRIVER-4229 zSeries with gcc 4.8.4 produces illegal instructions for int and * int32 atomic intrinsics. */ #if defined(__s390__) || defined(__s390x__) || defined(__zarch__) #define PHONGO_EMULATE_INT32 #endif /* clang-format off */ #define DEF_ATOMIC_OP(MSVC_Intrinsic, GNU_Intrinsic, GNU_Legacy_Intrinsic, Order, ...) \ do { \ switch (Order) { \ case phongo_memory_order_acq_rel: \ BSON_IF_MSVC(return MSVC_Intrinsic(__VA_ARGS__);) \ BSON_IF_GNU_LIKE(return GNU_Intrinsic(__VA_ARGS__, __ATOMIC_ACQ_REL);) \ PHONGO_IF_GNU_LEGACY_ATOMICS(return GNU_Legacy_Intrinsic(__VA_ARGS__);) \ case phongo_memory_order_seq_cst: \ BSON_IF_MSVC(return MSVC_Intrinsic(__VA_ARGS__);) \ BSON_IF_GNU_LIKE(return GNU_Intrinsic(__VA_ARGS__, __ATOMIC_SEQ_CST);) \ PHONGO_IF_GNU_LEGACY_ATOMICS(return GNU_Legacy_Intrinsic(__VA_ARGS__);) \ case phongo_memory_order_acquire: \ BSON_IF_MSVC(return BSON_CONCAT(MSVC_Intrinsic, MSVC_MEMORDER_SUFFIX(_acq))(__VA_ARGS__);) \ BSON_IF_GNU_LIKE(return GNU_Intrinsic(__VA_ARGS__, __ATOMIC_ACQUIRE);) \ PHONGO_IF_GNU_LEGACY_ATOMICS(return GNU_Legacy_Intrinsic(__VA_ARGS__);) \ case phongo_memory_order_consume: \ BSON_IF_MSVC(return BSON_CONCAT(MSVC_Intrinsic, MSVC_MEMORDER_SUFFIX(_acq))(__VA_ARGS__);) \ BSON_IF_GNU_LIKE(return GNU_Intrinsic(__VA_ARGS__, __ATOMIC_CONSUME);) \ PHONGO_IF_GNU_LEGACY_ATOMICS(return GNU_Legacy_Intrinsic(__VA_ARGS__);) \ case phongo_memory_order_release: \ BSON_IF_MSVC(return BSON_CONCAT(MSVC_Intrinsic, MSVC_MEMORDER_SUFFIX(_rel))(__VA_ARGS__);) \ BSON_IF_GNU_LIKE(return GNU_Intrinsic(__VA_ARGS__, __ATOMIC_RELEASE);) \ PHONGO_IF_GNU_LEGACY_ATOMICS(return GNU_Legacy_Intrinsic(__VA_ARGS__);) \ case phongo_memory_order_relaxed: \ BSON_IF_MSVC(return BSON_CONCAT(MSVC_Intrinsic, MSVC_MEMORDER_SUFFIX(_nf))(__VA_ARGS__);) \ BSON_IF_GNU_LIKE(return GNU_Intrinsic(__VA_ARGS__, __ATOMIC_RELAXED);) \ PHONGO_IF_GNU_LEGACY_ATOMICS(return GNU_Legacy_Intrinsic(__VA_ARGS__);) \ default: \ BSON_UNREACHABLE("Invalid phongo_memory_order value"); \ } \ } while (0) #define DEF_ATOMIC_CMPEXCH_STRONG(VCSuffix1, VCSuffix2, GNU_MemOrder, Ptr, ExpectActualVar, NewValue) \ do { \ BSON_IF_MSVC(ExpectActualVar = BSON_CONCAT3(_InterlockedCompareExchange, VCSuffix1, VCSuffix2)( \ Ptr, NewValue, ExpectActualVar);) \ BSON_IF_GNU_LIKE((void) __atomic_compare_exchange_n(Ptr, \ &ExpectActualVar, \ NewValue, \ false, /* Not weak */ \ GNU_MemOrder, \ GNU_MemOrder);) \ PHONGO_IF_GNU_LEGACY_ATOMICS(__typeof__(ExpectActualVar) _val; \ _val = __sync_val_compare_and_swap(Ptr, ExpectActualVar, NewValue); \ ExpectActualVar = _val;) \ } while (0) #define DEF_ATOMIC_CMPEXCH_WEAK(VCSuffix1, VCSuffix2, GNU_MemOrder, Ptr, ExpectActualVar, NewValue) \ do { \ BSON_IF_MSVC(ExpectActualVar = BSON_CONCAT3(_InterlockedCompareExchange, VCSuffix1, VCSuffix2)( \ Ptr, NewValue, ExpectActualVar);) \ BSON_IF_GNU_LIKE((void) __atomic_compare_exchange_n(Ptr, \ &ExpectActualVar, \ NewValue, \ true, /* Yes weak */ \ GNU_MemOrder, \ GNU_MemOrder);) \ PHONGO_IF_GNU_LEGACY_ATOMICS(__typeof__(ExpectActualVar) _val; \ _val = __sync_val_compare_and_swap(Ptr, ExpectActualVar, NewValue); \ ExpectActualVar = _val;) \ } while (0) #define DECL_ATOMIC_INTEGRAL(NamePart, Type, VCIntrinSuffix) \ static BSON_INLINE Type phongo_atomic_##NamePart##_fetch_add( \ Type volatile* a, Type addend, enum phongo_memory_order ord) \ { \ DEF_ATOMIC_OP(BSON_CONCAT(_InterlockedExchangeAdd, VCIntrinSuffix), \ __atomic_fetch_add, \ __sync_fetch_and_add, \ ord, \ a, \ addend); \ } \ \ static BSON_INLINE Type phongo_atomic_##NamePart##_fetch_sub( \ Type volatile* a, Type subtrahend, enum phongo_memory_order ord) \ { \ /* MSVC doesn't have a subtract intrinsic, so just reuse addition */ \ BSON_IF_MSVC(return phongo_atomic_##NamePart##_fetch_add(a, -subtrahend, ord);) \ BSON_IF_GNU_LIKE(DEF_ATOMIC_OP(~, __atomic_fetch_sub, ~, ord, a, subtrahend);) \ PHONGO_IF_GNU_LEGACY_ATOMICS(DEF_ATOMIC_OP(~, ~, __sync_fetch_and_sub, ord, a, subtrahend);) \ } \ \ static BSON_INLINE Type phongo_atomic_##NamePart##_fetch(Type volatile const* a, enum phongo_memory_order order) \ { \ /* MSVC doesn't have a load intrinsic, so just add zero */ \ BSON_IF_MSVC(return phongo_atomic_##NamePart##_fetch_add((Type volatile*) a, 0, order);) \ /* GNU doesn't want RELEASE order for the fetch operation, so we can't \ * just use DEF_ATOMIC_OP. */ \ BSON_IF_GNU_LIKE(switch (order) { \ case phongo_memory_order_release: /* Fall back to seqcst */ \ case phongo_memory_order_acq_rel: /* Fall back to seqcst */ \ case phongo_memory_order_seq_cst: \ return __atomic_load_n(a, __ATOMIC_SEQ_CST); \ case phongo_memory_order_acquire: \ return __atomic_load_n(a, __ATOMIC_ACQUIRE); \ case phongo_memory_order_consume: \ return __atomic_load_n(a, __ATOMIC_CONSUME); \ case phongo_memory_order_relaxed: \ return __atomic_load_n(a, __ATOMIC_RELAXED); \ default: \ BSON_UNREACHABLE("Invalid phongo_memory_order value"); \ }) \ PHONGO_IF_GNU_LEGACY_ATOMICS({ \ BSON_UNUSED(order); \ __sync_synchronize(); \ return *a; \ }) \ } \ \ static BSON_INLINE Type phongo_atomic_##NamePart##_exchange( \ Type volatile* a, Type value, enum phongo_memory_order ord) \ { \ BSON_IF_MSVC(DEF_ATOMIC_OP(BSON_CONCAT(_InterlockedExchange, VCIntrinSuffix), ~, ~, ord, a, value);) \ /* GNU doesn't want CONSUME order for the exchange operation, so we \ * cannot use DEF_ATOMIC_OP. */ \ BSON_IF_GNU_LIKE(switch (ord) { \ case phongo_memory_order_acq_rel: \ return __atomic_exchange_n(a, value, __ATOMIC_ACQ_REL); \ case phongo_memory_order_release: \ return __atomic_exchange_n(a, value, __ATOMIC_RELEASE); \ case phongo_memory_order_seq_cst: \ return __atomic_exchange_n(a, value, __ATOMIC_SEQ_CST); \ case phongo_memory_order_consume: /* Fall back to acquire */ \ case phongo_memory_order_acquire: \ return __atomic_exchange_n(a, value, __ATOMIC_ACQUIRE); \ case phongo_memory_order_relaxed: \ return __atomic_exchange_n(a, value, __ATOMIC_RELAXED); \ default: \ BSON_UNREACHABLE("Invalid phongo_memory_order value"); \ }) \ PHONGO_IF_GNU_LEGACY_ATOMICS(BSON_UNUSED(ord); return __sync_val_compare_and_swap(a, *a, value);) \ } \ \ static BSON_INLINE Type phongo_atomic_##NamePart##_compare_exchange_strong( \ Type volatile* a, Type expect, Type new_value, enum phongo_memory_order ord) \ { \ Type actual = expect; \ switch (ord) { \ case phongo_memory_order_release: \ case phongo_memory_order_acq_rel: \ case phongo_memory_order_seq_cst: \ DEF_ATOMIC_CMPEXCH_STRONG(VCIntrinSuffix, , __ATOMIC_SEQ_CST, a, actual, new_value); \ break; \ case phongo_memory_order_acquire: \ DEF_ATOMIC_CMPEXCH_STRONG( \ VCIntrinSuffix, MSVC_MEMORDER_SUFFIX(_acq), __ATOMIC_ACQUIRE, a, actual, new_value); \ break; \ case phongo_memory_order_consume: \ DEF_ATOMIC_CMPEXCH_STRONG( \ VCIntrinSuffix, MSVC_MEMORDER_SUFFIX(_acq), __ATOMIC_CONSUME, a, actual, new_value); \ break; \ case phongo_memory_order_relaxed: \ DEF_ATOMIC_CMPEXCH_STRONG( \ VCIntrinSuffix, MSVC_MEMORDER_SUFFIX(_nf), __ATOMIC_RELAXED, a, actual, new_value); \ break; \ default: \ BSON_UNREACHABLE("Invalid phongo_memory_order value"); \ } \ return actual; \ } \ \ static BSON_INLINE Type phongo_atomic_##NamePart##_compare_exchange_weak( \ Type volatile* a, Type expect, Type new_value, enum phongo_memory_order ord) \ { \ Type actual = expect; \ switch (ord) { \ case phongo_memory_order_release: \ case phongo_memory_order_acq_rel: \ case phongo_memory_order_seq_cst: \ DEF_ATOMIC_CMPEXCH_WEAK(VCIntrinSuffix, , __ATOMIC_SEQ_CST, a, actual, new_value); \ break; \ case phongo_memory_order_acquire: \ DEF_ATOMIC_CMPEXCH_WEAK( \ VCIntrinSuffix, MSVC_MEMORDER_SUFFIX(_acq), __ATOMIC_ACQUIRE, a, actual, new_value); \ break; \ case phongo_memory_order_consume: \ DEF_ATOMIC_CMPEXCH_WEAK( \ VCIntrinSuffix, MSVC_MEMORDER_SUFFIX(_acq), __ATOMIC_CONSUME, a, actual, new_value); \ break; \ case phongo_memory_order_relaxed: \ DEF_ATOMIC_CMPEXCH_WEAK(VCIntrinSuffix, MSVC_MEMORDER_SUFFIX(_nf), __ATOMIC_RELAXED, a, actual, new_value); \ break; \ default: \ BSON_UNREACHABLE("Invalid phongo_memory_order value"); \ } \ return actual; \ } /* clang-format on */ #define DECL_ATOMIC_STDINT(Name, VCSuffix) DECL_ATOMIC_INTEGRAL(Name, Name##_t, VCSuffix) #if defined(_MSC_VER) || defined(PHONGO_USE_LEGACY_GCC_ATOMICS) /* MSVC and GCC require built-in types (not typedefs) for their atomic * intrinsics. */ #if defined(_MSC_VER) #define DECL_ATOMIC_INTEGRAL_INT8 char #define DECL_ATOMIC_INTEGRAL_INT32 long #else #define DECL_ATOMIC_INTEGRAL_INT8 signed char #define DECL_ATOMIC_INTEGRAL_INT32 int #endif DECL_ATOMIC_INTEGRAL(int8, DECL_ATOMIC_INTEGRAL_INT8, 8) #if !defined(PHONGO_EMULATE_INT32) DECL_ATOMIC_INTEGRAL(int32, DECL_ATOMIC_INTEGRAL_INT32, ) #endif #else /* Other compilers that we support provide generic intrinsics */ DECL_ATOMIC_STDINT(int8, 8) #if !defined(PHONGO_EMULATE_INT32) DECL_ATOMIC_STDINT(int32, ) #endif #endif #ifndef DECL_ATOMIC_INTEGRAL_INT32 #define DECL_ATOMIC_INTEGRAL_INT32 int32_t #endif int32_t _phongo_emul_atomic_int32_fetch_add(int32_t volatile* val, int32_t v, enum phongo_memory_order); int32_t _phongo_emul_atomic_int32_exchange(int32_t volatile* val, int32_t v, enum phongo_memory_order); int32_t _phongo_emul_atomic_int32_compare_exchange_strong( int32_t volatile* val, int32_t expect_value, int32_t new_value, enum phongo_memory_order); int32_t _phongo_emul_atomic_int32_compare_exchange_weak( int32_t volatile* val, int32_t expect_value, int32_t new_value, enum phongo_memory_order); #if defined(PHONGO_EMULATE_INT32) static BSON_INLINE int32_t phongo_atomic_int32_fetch(const int32_t volatile* val, enum phongo_memory_order order) { return _phongo_emul_atomic_int32_fetch_add((int32_t volatile*) val, 0, order); } static BSON_INLINE int32_t phongo_atomic_int32_fetch_add(int32_t volatile* val, int32_t v, enum phongo_memory_order order) { return _phongo_emul_atomic_int32_fetch_add(val, v, order); } static BSON_INLINE int32_t phongo_atomic_int32_fetch_sub(int32_t volatile* val, int32_t v, enum phongo_memory_order order) { return _phongo_emul_atomic_int32_fetch_add(val, -v, order); } static BSON_INLINE int32_t phongo_atomic_int32_exchange(int32_t volatile* val, int32_t v, enum phongo_memory_order order) { return _phongo_emul_atomic_int32_exchange(val, v, order); } static BSON_INLINE int32_t phongo_atomic_int32_compare_exchange_strong( int32_t volatile* val, int32_t expect_value, int32_t new_value, enum phongo_memory_order order) { return _phongo_emul_atomic_int32_compare_exchange_strong(val, expect_value, new_value, order); } static BSON_INLINE int32_t phongo_atomic_int32_compare_exchange_weak( int32_t volatile* val, int32_t expect_value, int32_t new_value, enum phongo_memory_order order) { return _phongo_emul_atomic_int32_compare_exchange_weak(val, expect_value, new_value, order); } #endif /* PHONGO_EMULATE_INT32 */ #undef DECL_ATOMIC_STDINT #undef DECL_ATOMIC_INTEGRAL #undef DEF_ATOMIC_OP #undef DEF_ATOMIC_CMPEXCH_STRONG #undef DEF_ATOMIC_CMPEXCH_WEAK #undef MSVC_MEMORDER_SUFFIX /* TODO: Redefine BSON_IF_MSVC is necessary (see: CDRIVER-5813) */ #ifdef PHONGO_USE_LEGACY_GCC_ATOMICS #undef BSON_IF_GNU_LIKE #define BSON_IF_GNU_LIKE(...) __VA_ARGS__ #endif #undef PHONGO_IF_GNU_LEGACY_ATOMICS #undef PHONGO_USE_LEGACY_GCC_ATOMICS #undef PHONGO_EMULATE_INT32 #endif /* PHONGO_ATOMIC_H */