1 #ifndef JEMALLOC_INTERNAL_ATOMIC_GCC_ATOMIC_H 2 #define JEMALLOC_INTERNAL_ATOMIC_GCC_ATOMIC_H 3 4 #include "jemalloc/internal/assert.h" 5 6 #define ATOMIC_INIT(...) {__VA_ARGS__} 7 8 typedef enum { 9 atomic_memory_order_relaxed, 10 atomic_memory_order_acquire, 11 atomic_memory_order_release, 12 atomic_memory_order_acq_rel, 13 atomic_memory_order_seq_cst 14 } atomic_memory_order_t; 15 16 ATOMIC_INLINE int 17 atomic_enum_to_builtin(atomic_memory_order_t mo) { 18 switch (mo) { 19 case atomic_memory_order_relaxed: 20 return __ATOMIC_RELAXED; 21 case atomic_memory_order_acquire: 22 return __ATOMIC_ACQUIRE; 23 case atomic_memory_order_release: 24 return __ATOMIC_RELEASE; 25 case atomic_memory_order_acq_rel: 26 return __ATOMIC_ACQ_REL; 27 case atomic_memory_order_seq_cst: 28 return __ATOMIC_SEQ_CST; 29 } 30 /* Can't happen; the switch is exhaustive. */ 31 not_reached(); 32 } 33 34 ATOMIC_INLINE void 35 atomic_fence(atomic_memory_order_t mo) { 36 __atomic_thread_fence(atomic_enum_to_builtin(mo)); 37 } 38 39 #define JEMALLOC_GENERATE_ATOMICS(type, short_type, lg_size) \ 40 typedef struct { \ 41 type repr; \ 42 } atomic_##short_type##_t; \ 43 \ 44 ATOMIC_INLINE type \ 45 atomic_load_##short_type(const atomic_##short_type##_t *a, \ 46 atomic_memory_order_t mo) { \ 47 type result; \ 48 __atomic_load(&a->repr, &result, atomic_enum_to_builtin(mo)); \ 49 return result; \ 50 } \ 51 \ 52 ATOMIC_INLINE void \ 53 atomic_store_##short_type(atomic_##short_type##_t *a, type val, \ 54 atomic_memory_order_t mo) { \ 55 __atomic_store(&a->repr, &val, atomic_enum_to_builtin(mo)); \ 56 } \ 57 \ 58 ATOMIC_INLINE type \ 59 atomic_exchange_##short_type(atomic_##short_type##_t *a, type val, \ 60 atomic_memory_order_t mo) { \ 61 type result; \ 62 __atomic_exchange(&a->repr, &val, &result, \ 63 atomic_enum_to_builtin(mo)); \ 64 return result; \ 65 } \ 66 \ 67 ATOMIC_INLINE bool \ 68 atomic_compare_exchange_weak_##short_type(atomic_##short_type##_t *a, \ 69 type *expected, type desired, atomic_memory_order_t success_mo, \ 70 atomic_memory_order_t failure_mo) { \ 71 return __atomic_compare_exchange(&a->repr, expected, &desired, \ 72 true, atomic_enum_to_builtin(success_mo), \ 73 atomic_enum_to_builtin(failure_mo)); \ 74 } \ 75 \ 76 ATOMIC_INLINE bool \ 77 atomic_compare_exchange_strong_##short_type(atomic_##short_type##_t *a, \ 78 type *expected, type desired, atomic_memory_order_t success_mo, \ 79 atomic_memory_order_t failure_mo) { \ 80 return __atomic_compare_exchange(&a->repr, expected, &desired, \ 81 false, \ 82 atomic_enum_to_builtin(success_mo), \ 83 atomic_enum_to_builtin(failure_mo)); \ 84 } 85 86 87 #define JEMALLOC_GENERATE_INT_ATOMICS(type, short_type, lg_size) \ 88 JEMALLOC_GENERATE_ATOMICS(type, short_type, /* unused */ lg_size) \ 89 \ 90 ATOMIC_INLINE type \ 91 atomic_fetch_add_##short_type(atomic_##short_type##_t *a, type val, \ 92 atomic_memory_order_t mo) { \ 93 return __atomic_fetch_add(&a->repr, val, \ 94 atomic_enum_to_builtin(mo)); \ 95 } \ 96 \ 97 ATOMIC_INLINE type \ 98 atomic_fetch_sub_##short_type(atomic_##short_type##_t *a, type val, \ 99 atomic_memory_order_t mo) { \ 100 return __atomic_fetch_sub(&a->repr, val, \ 101 atomic_enum_to_builtin(mo)); \ 102 } \ 103 \ 104 ATOMIC_INLINE type \ 105 atomic_fetch_and_##short_type(atomic_##short_type##_t *a, type val, \ 106 atomic_memory_order_t mo) { \ 107 return __atomic_fetch_and(&a->repr, val, \ 108 atomic_enum_to_builtin(mo)); \ 109 } \ 110 \ 111 ATOMIC_INLINE type \ 112 atomic_fetch_or_##short_type(atomic_##short_type##_t *a, type val, \ 113 atomic_memory_order_t mo) { \ 114 return __atomic_fetch_or(&a->repr, val, \ 115 atomic_enum_to_builtin(mo)); \ 116 } \ 117 \ 118 ATOMIC_INLINE type \ 119 atomic_fetch_xor_##short_type(atomic_##short_type##_t *a, type val, \ 120 atomic_memory_order_t mo) { \ 121 return __atomic_fetch_xor(&a->repr, val, \ 122 atomic_enum_to_builtin(mo)); \ 123 } 124 125 #endif /* JEMALLOC_INTERNAL_ATOMIC_GCC_ATOMIC_H */ 126