1 #ifndef JEMALLOC_INTERNAL_MUTEX_H 2 #define JEMALLOC_INTERNAL_MUTEX_H 3 4 #include "jemalloc/internal/atomic.h" 5 #include "jemalloc/internal/mutex_prof.h" 6 #include "jemalloc/internal/tsd.h" 7 #include "jemalloc/internal/witness.h" 8 9 typedef enum { 10 /* Can only acquire one mutex of a given witness rank at a time. */ 11 malloc_mutex_rank_exclusive, 12 /* 13 * Can acquire multiple mutexes of the same witness rank, but in 14 * address-ascending order only. 15 */ 16 malloc_mutex_address_ordered 17 } malloc_mutex_lock_order_t; 18 19 typedef struct malloc_mutex_s malloc_mutex_t; 20 struct malloc_mutex_s { 21 union { 22 struct { 23 /* 24 * prof_data is defined first to reduce cacheline 25 * bouncing: the data is not touched by the mutex holder 26 * during unlocking, while might be modified by 27 * contenders. Having it before the mutex itself could 28 * avoid prefetching a modified cacheline (for the 29 * unlocking thread). 30 */ 31 mutex_prof_data_t prof_data; 32 #ifdef _WIN32 33 # if _WIN32_WINNT >= 0x0600 34 SRWLOCK lock; 35 # else 36 CRITICAL_SECTION lock; 37 # endif 38 #elif (defined(JEMALLOC_OS_UNFAIR_LOCK)) 39 os_unfair_lock lock; 40 #elif (defined(JEMALLOC_OSSPIN)) 41 OSSpinLock lock; 42 #elif (defined(JEMALLOC_MUTEX_INIT_CB)) 43 pthread_mutex_t lock; 44 malloc_mutex_t *postponed_next; 45 #else 46 pthread_mutex_t lock; 47 #endif 48 }; 49 /* 50 * We only touch witness when configured w/ debug. However we 51 * keep the field in a union when !debug so that we don't have 52 * to pollute the code base with #ifdefs, while avoid paying the 53 * memory cost. 54 */ 55 #if !defined(JEMALLOC_DEBUG) 56 witness_t witness; 57 malloc_mutex_lock_order_t lock_order; 58 #endif 59 }; 60 61 #if defined(JEMALLOC_DEBUG) 62 witness_t witness; 63 malloc_mutex_lock_order_t lock_order; 64 #define LOCK_ORDER_INITIALIZER(field, a) field = a, 65 #else 66 #define LOCK_ORDER_INITIALIZER(field, a) 67 #endif 68 }; 69 70 /* 71 * Based on benchmark results, a fixed spin with this amount of retries works 72 * well for our critical sections. 73 */ 74 #define MALLOC_MUTEX_MAX_SPIN 250 75 76 #ifdef _WIN32 77 # if _WIN32_WINNT >= 0x0600 78 # define MALLOC_MUTEX_LOCK(m) AcquireSRWLockExclusive(&(m)->lock) 79 # define MALLOC_MUTEX_UNLOCK(m) ReleaseSRWLockExclusive(&(m)->lock) 80 # define MALLOC_MUTEX_TRYLOCK(m) (!TryAcquireSRWLockExclusive(&(m)->lock)) 81 # else 82 # define MALLOC_MUTEX_LOCK(m) EnterCriticalSection(&(m)->lock) 83 # define MALLOC_MUTEX_UNLOCK(m) LeaveCriticalSection(&(m)->lock) 84 # define MALLOC_MUTEX_TRYLOCK(m) (!TryEnterCriticalSection(&(m)->lock)) 85 # endif 86 #elif (defined(JEMALLOC_OS_UNFAIR_LOCK)) 87 # define MALLOC_MUTEX_LOCK(m) os_unfair_lock_lock(&(m)->lock) 88 # define MALLOC_MUTEX_UNLOCK(m) os_unfair_lock_unlock(&(m)->lock) 89 # define MALLOC_MUTEX_TRYLOCK(m) (!os_unfair_lock_trylock(&(m)->lock)) 90 #elif (defined(JEMALLOC_OSSPIN)) 91 # define MALLOC_MUTEX_LOCK(m) OSSpinLockLock(&(m)->lock) 92 # define MALLOC_MUTEX_UNLOCK(m) OSSpinLockUnlock(&(m)->lock) 93 # define MALLOC_MUTEX_TRYLOCK(m) (!OSSpinLockTry(&(m)->lock)) 94 #else 95 # define MALLOC_MUTEX_LOCK(m) pthread_mutex_lock(&(m)->lock) 96 # define MALLOC_MUTEX_UNLOCK(m) pthread_mutex_unlock(&(m)->lock) 97 # define MALLOC_MUTEX_TRYLOCK(m) (pthread_mutex_trylock(&(m)->lock) != 0) 98 #endif 99 100 #ifdef _WIN32 101 # define MALLOC_MUTEX_INITIALIZER 102 #elif (defined(JEMALLOC_OS_UNFAIR_LOCK)) 103 # define MALLOC_MUTEX_INITIALIZER {{{ \ 104 .prof_data = MUTEX_PROF_DATA_INITIALIZER, \ 105 .lock = OS_UNFAIR_LOCK_INIT, \ 106 }}, \ 107 WITNESS_INITIALIZER(.witness, "mutex", WITNESS_RANK_OMIT) \ 108 LOCK_ORDER_INITIALIZER(.lock_order, malloc_mutex_rank_exclusive)} 109 #elif (defined(JEMALLOC_OSSPIN)) 110 # define MALLOC_MUTEX_INITIALIZER {{{ \ 111 .prof_data = MUTEX_PROF_DATA_INITIALIZER, \ 112 .lock = 0, \ 113 }}, \ 114 WITNESS_INITIALIZER(.witness, "mutex", WITNESS_RANK_OMIT) \ 115 LOCK_ORDER_INITIALIZER(.lock_order, malloc_mutex_rank_exclusive)} 116 #elif (defined(JEMALLOC_MUTEX_INIT_CB)) 117 # define MALLOC_MUTEX_INITIALIZER {{{ \ 118 .prof_data = MUTEX_PROF_DATA_INITIALIZER, \ 119 .lock = PTHREAD_MUTEX_INITIALIZER, \ 120 .postponed_next = NULL, \ 121 }}, \ 122 WITNESS_INITIALIZER(.witness, "mutex", WITNESS_RANK_OMIT) \ 123 LOCK_ORDER_INITIALIZER(.lock_order, malloc_mutex_rank_exclusive)} 124 #else 125 # define MALLOC_MUTEX_TYPE PTHREAD_MUTEX_DEFAULT 126 # define MALLOC_MUTEX_INITIALIZER {{{ \ 127 .prof_data = MUTEX_PROF_DATA_INITIALIZER, \ 128 .lock = PTHREAD_MUTEX_INITIALIZER, \ 129 }}, \ 130 WITNESS_INITIALIZER(.witness, "mutex", WITNESS_RANK_OMIT) \ 131 LOCK_ORDER_INITIALIZER(.lock_order, malloc_mutex_rank_exclusive)} 132 #endif 133 134 #ifdef JEMALLOC_LAZY_LOCK 135 extern bool isthreaded; 136 #else 137 # undef isthreaded /* Undo private_namespace.h definition. */ 138 # define isthreaded true 139 #endif 140 141 bool malloc_mutex_init(malloc_mutex_t *mutex, const char *name, 142 witness_rank_t rank, malloc_mutex_lock_order_t lock_order); 143 void malloc_mutex_prefork(tsdn_t *tsdn, malloc_mutex_t *mutex); 144 void malloc_mutex_postfork_parent(tsdn_t *tsdn, malloc_mutex_t *mutex); 145 void malloc_mutex_postfork_child(tsdn_t *tsdn, malloc_mutex_t *mutex); 146 bool malloc_mutex_boot(void); 147 void malloc_mutex_prof_data_reset(tsdn_t *tsdn, malloc_mutex_t *mutex); 148 149 void malloc_mutex_lock_slow(malloc_mutex_t *mutex); 150 151 static inline void 152 malloc_mutex_lock_final(malloc_mutex_t *mutex) { 153 MALLOC_MUTEX_LOCK(mutex); 154 } 155 156 static inline bool 157 malloc_mutex_trylock_final(malloc_mutex_t *mutex) { 158 return MALLOC_MUTEX_TRYLOCK(mutex); 159 } 160 161 static inline void 162 mutex_owner_stats_update(tsdn_t *tsdn, malloc_mutex_t *mutex) { 163 if (config_stats) { 164 mutex_prof_data_t *data = &mutex->prof_data; 165 data->n_lock_ops++; 166 if (data->prev_owner != tsdn) { 167 data->prev_owner = tsdn; 168 data->n_owner_switches++; 169 } 170 } 171 } 172 173 /* Trylock: return false if the lock is successfully acquired. */ 174 static inline bool 175 malloc_mutex_trylock(tsdn_t *tsdn, malloc_mutex_t *mutex) { 176 witness_assert_not_owner(tsdn_witness_tsdp_get(tsdn), &mutex->witness); 177 if (isthreaded) { 178 if (malloc_mutex_trylock_final(mutex)) { 179 return true; 180 } 181 mutex_owner_stats_update(tsdn, mutex); 182 } 183 witness_lock(tsdn_witness_tsdp_get(tsdn), &mutex->witness); 184 185 return false; 186 } 187 188 /* Aggregate lock prof data. */ 189 static inline void 190 malloc_mutex_prof_merge(mutex_prof_data_t *sum, mutex_prof_data_t *data) { 191 nstime_add(&sum->tot_wait_time, &data->tot_wait_time); 192 if (nstime_compare(&sum->max_wait_time, &data->max_wait_time) < 0) { 193 nstime_copy(&sum->max_wait_time, &data->max_wait_time); 194 } 195 196 sum->n_wait_times += data->n_wait_times; 197 sum->n_spin_acquired += data->n_spin_acquired; 198 199 if (sum->max_n_thds < data->max_n_thds) { 200 sum->max_n_thds = data->max_n_thds; 201 } 202 uint32_t cur_n_waiting_thds = atomic_load_u32(&sum->n_waiting_thds, 203 ATOMIC_RELAXED); 204 uint32_t new_n_waiting_thds = cur_n_waiting_thds + atomic_load_u32( 205 &data->n_waiting_thds, ATOMIC_RELAXED); 206 atomic_store_u32(&sum->n_waiting_thds, new_n_waiting_thds, 207 ATOMIC_RELAXED); 208 sum->n_owner_switches += data->n_owner_switches; 209 sum->n_lock_ops += data->n_lock_ops; 210 } 211 212 static inline void 213 malloc_mutex_lock(tsdn_t *tsdn, malloc_mutex_t *mutex) { 214 witness_assert_not_owner(tsdn_witness_tsdp_get(tsdn), &mutex->witness); 215 if (isthreaded) { 216 if (malloc_mutex_trylock_final(mutex)) { 217 malloc_mutex_lock_slow(mutex); 218 } 219 mutex_owner_stats_update(tsdn, mutex); 220 } 221 witness_lock(tsdn_witness_tsdp_get(tsdn), &mutex->witness); 222 } 223 224 static inline void 225 malloc_mutex_unlock(tsdn_t *tsdn, malloc_mutex_t *mutex) { 226 witness_unlock(tsdn_witness_tsdp_get(tsdn), &mutex->witness); 227 if (isthreaded) { 228 MALLOC_MUTEX_UNLOCK(mutex); 229 } 230 } 231 232 static inline void 233 malloc_mutex_assert_owner(tsdn_t *tsdn, malloc_mutex_t *mutex) { 234 witness_assert_owner(tsdn_witness_tsdp_get(tsdn), &mutex->witness); 235 } 236 237 static inline void 238 malloc_mutex_assert_not_owner(tsdn_t *tsdn, malloc_mutex_t *mutex) { 239 witness_assert_not_owner(tsdn_witness_tsdp_get(tsdn), &mutex->witness); 240 } 241 242 /* Copy the prof data from mutex for processing. */ 243 static inline void 244 malloc_mutex_prof_read(tsdn_t *tsdn, mutex_prof_data_t *data, 245 malloc_mutex_t *mutex) { 246 mutex_prof_data_t *source = &mutex->prof_data; 247 /* Can only read holding the mutex. */ 248 malloc_mutex_assert_owner(tsdn, mutex); 249 250 /* 251 * Not *really* allowed (we shouldn't be doing non-atomic loads of 252 * atomic data), but the mutex protection makes this safe, and writing 253 * a member-for-member copy is tedious for this situation. 254 */ 255 *data = *source; 256 /* n_wait_thds is not reported (modified w/o locking). */ 257 atomic_store_u32(&data->n_waiting_thds, 0, ATOMIC_RELAXED); 258 } 259 260 #endif /* JEMALLOC_INTERNAL_MUTEX_H */ 261