1 #define JEMALLOC_TSD_C_ 2 #include "jemalloc/internal/jemalloc_preamble.h" 3 #include "jemalloc/internal/jemalloc_internal_includes.h" 4 5 #include "jemalloc/internal/assert.h" 6 #include "jemalloc/internal/mutex.h" 7 #include "jemalloc/internal/rtree.h" 8 9 /******************************************************************************/ 10 /* Data. */ 11 12 static unsigned ncleanups; 13 static malloc_tsd_cleanup_t cleanups[MALLOC_TSD_CLEANUPS_MAX]; 14 15 #ifdef JEMALLOC_MALLOC_THREAD_CLEANUP 16 __thread tsd_t JEMALLOC_TLS_MODEL tsd_tls = TSD_INITIALIZER; 17 __thread bool JEMALLOC_TLS_MODEL tsd_initialized = false; 18 bool tsd_booted = false; 19 #elif (defined(JEMALLOC_TLS)) 20 __thread tsd_t JEMALLOC_TLS_MODEL tsd_tls = TSD_INITIALIZER; 21 pthread_key_t tsd_tsd; 22 bool tsd_booted = false; 23 #elif (defined(_WIN32)) 24 DWORD tsd_tsd; 25 tsd_wrapper_t tsd_boot_wrapper = {false, TSD_INITIALIZER}; 26 bool tsd_booted = false; 27 #else 28 29 /* 30 * This contains a mutex, but it's pretty convenient to allow the mutex code to 31 * have a dependency on tsd. So we define the struct here, and only refer to it 32 * by pointer in the header. 33 */ 34 struct tsd_init_head_s { 35 ql_head(tsd_init_block_t) blocks; 36 malloc_mutex_t lock; 37 }; 38 39 pthread_key_t tsd_tsd; 40 tsd_init_head_t tsd_init_head = { 41 ql_head_initializer(blocks), 42 #ifndef __lint__ 43 // XXX: broken lint 44 MALLOC_MUTEX_INITIALIZER 45 #endif 46 }; 47 tsd_wrapper_t tsd_boot_wrapper = { 48 false, 49 TSD_INITIALIZER 50 }; 51 bool tsd_booted = false; 52 #endif 53 54 55 /******************************************************************************/ 56 57 void 58 tsd_slow_update(tsd_t *tsd) { 59 if (tsd_nominal(tsd)) { 60 if (malloc_slow || !tsd_tcache_enabled_get(tsd) || 61 tsd_reentrancy_level_get(tsd) > 0) { 62 tsd->state = tsd_state_nominal_slow; 63 } else { 64 tsd->state = tsd_state_nominal; 65 } 66 } 67 } 68 69 static bool 70 tsd_data_init(tsd_t *tsd) { 71 /* 72 * We initialize the rtree context first (before the tcache), since the 73 * tcache initialization depends on it. 74 */ 75 rtree_ctx_data_init(tsd_rtree_ctxp_get_unsafe(tsd)); 76 77 /* 78 * A nondeterministic seed based on the address of tsd reduces 79 * the likelihood of lockstep non-uniform cache index 80 * utilization among identical concurrent processes, but at the 81 * cost of test repeatability. For debug builds, instead use a 82 * deterministic seed. 83 */ 84 *tsd_offset_statep_get(tsd) = config_debug ? 0 : 85 (uint64_t)(uintptr_t)tsd; 86 87 return tsd_tcache_enabled_data_init(tsd); 88 } 89 90 static void 91 assert_tsd_data_cleanup_done(tsd_t *tsd) { 92 assert(!tsd_nominal(tsd)); 93 assert(*tsd_arenap_get_unsafe(tsd) == NULL); 94 assert(*tsd_iarenap_get_unsafe(tsd) == NULL); 95 assert(*tsd_arenas_tdata_bypassp_get_unsafe(tsd) == true); 96 assert(*tsd_arenas_tdatap_get_unsafe(tsd) == NULL); 97 assert(*tsd_tcache_enabledp_get_unsafe(tsd) == false); 98 assert(*tsd_prof_tdatap_get_unsafe(tsd) == NULL); 99 } 100 101 static bool 102 tsd_data_init_nocleanup(tsd_t *tsd) { 103 assert(tsd->state == tsd_state_reincarnated || 104 tsd->state == tsd_state_minimal_initialized); 105 /* 106 * During reincarnation, there is no guarantee that the cleanup function 107 * will be called (deallocation may happen after all tsd destructors). 108 * We set up tsd in a way that no cleanup is needed. 109 */ 110 rtree_ctx_data_init(tsd_rtree_ctxp_get_unsafe(tsd)); 111 *tsd_arenas_tdata_bypassp_get(tsd) = true; 112 *tsd_tcache_enabledp_get_unsafe(tsd) = false; 113 *tsd_reentrancy_levelp_get(tsd) = 1; 114 assert_tsd_data_cleanup_done(tsd); 115 116 return false; 117 } 118 119 tsd_t * 120 tsd_fetch_slow(tsd_t *tsd, bool minimal) { 121 assert(!tsd_fast(tsd)); 122 123 if (tsd->state == tsd_state_nominal_slow) { 124 /* On slow path but no work needed. */ 125 assert(malloc_slow || !tsd_tcache_enabled_get(tsd) || 126 tsd_reentrancy_level_get(tsd) > 0 || 127 *tsd_arenas_tdata_bypassp_get(tsd)); 128 } else if (tsd->state == tsd_state_uninitialized) { 129 if (!minimal) { 130 tsd->state = tsd_state_nominal; 131 tsd_slow_update(tsd); 132 /* Trigger cleanup handler registration. */ 133 tsd_set(tsd); 134 tsd_data_init(tsd); 135 } else { 136 tsd->state = tsd_state_minimal_initialized; 137 tsd_set(tsd); 138 tsd_data_init_nocleanup(tsd); 139 } 140 } else if (tsd->state == tsd_state_minimal_initialized) { 141 if (!minimal) { 142 /* Switch to fully initialized. */ 143 tsd->state = tsd_state_nominal; 144 assert(*tsd_reentrancy_levelp_get(tsd) >= 1); 145 (*tsd_reentrancy_levelp_get(tsd))--; 146 tsd_slow_update(tsd); 147 tsd_data_init(tsd); 148 } else { 149 assert_tsd_data_cleanup_done(tsd); 150 } 151 } else if (tsd->state == tsd_state_purgatory) { 152 tsd->state = tsd_state_reincarnated; 153 tsd_set(tsd); 154 tsd_data_init_nocleanup(tsd); 155 } else { 156 assert(tsd->state == tsd_state_reincarnated); 157 } 158 159 return tsd; 160 } 161 162 void * 163 malloc_tsd_malloc(size_t size) { 164 return a0malloc(CACHELINE_CEILING(size)); 165 } 166 167 void 168 malloc_tsd_dalloc(void *wrapper) { 169 a0dalloc(wrapper); 170 } 171 172 __BEGIN_DECLS 173 void _malloc_thread_cleanup(void); 174 __END_DECLS 175 176 #if defined(JEMALLOC_MALLOC_THREAD_CLEANUP) || defined(_WIN32) 177 #ifndef _WIN32 178 JEMALLOC_EXPORT 179 #endif 180 void 181 _malloc_thread_cleanup(void) { 182 bool pending[MALLOC_TSD_CLEANUPS_MAX], again; 183 unsigned i; 184 185 for (i = 0; i < ncleanups; i++) { 186 pending[i] = true; 187 } 188 189 do { 190 again = false; 191 for (i = 0; i < ncleanups; i++) { 192 if (pending[i]) { 193 pending[i] = cleanups[i](); 194 if (pending[i]) { 195 again = true; 196 } 197 } 198 } 199 } while (again); 200 } 201 #endif 202 203 void 204 malloc_tsd_cleanup_register(bool (*f)(void)) { 205 assert(ncleanups < MALLOC_TSD_CLEANUPS_MAX); 206 cleanups[ncleanups] = f; 207 ncleanups++; 208 } 209 210 static void 211 tsd_do_data_cleanup(tsd_t *tsd) { 212 prof_tdata_cleanup(tsd); 213 iarena_cleanup(tsd); 214 arena_cleanup(tsd); 215 arenas_tdata_cleanup(tsd); 216 tcache_cleanup(tsd); 217 witnesses_cleanup(tsd_witness_tsdp_get_unsafe(tsd)); 218 } 219 220 void 221 tsd_cleanup(void *arg) { 222 tsd_t *tsd = (tsd_t *)arg; 223 224 switch (tsd->state) { 225 case tsd_state_uninitialized: 226 /* Do nothing. */ 227 break; 228 case tsd_state_minimal_initialized: 229 /* This implies the thread only did free() in its life time. */ 230 /* Fall through. */ 231 case tsd_state_reincarnated: 232 /* 233 * Reincarnated means another destructor deallocated memory 234 * after the destructor was called. Cleanup isn't required but 235 * is still called for testing and completeness. 236 */ 237 assert_tsd_data_cleanup_done(tsd); 238 /* Fall through. */ 239 case tsd_state_nominal: 240 case tsd_state_nominal_slow: 241 tsd_do_data_cleanup(tsd); 242 tsd->state = tsd_state_purgatory; 243 tsd_set(tsd); 244 break; 245 case tsd_state_purgatory: 246 /* 247 * The previous time this destructor was called, we set the 248 * state to tsd_state_purgatory so that other destructors 249 * wouldn't cause re-creation of the tsd. This time, do 250 * nothing, and do not request another callback. 251 */ 252 break; 253 default: 254 not_reached(); 255 } 256 #ifdef JEMALLOC_JET 257 test_callback_t test_callback = *tsd_test_callbackp_get_unsafe(tsd); 258 int *data = tsd_test_datap_get_unsafe(tsd); 259 if (test_callback != NULL) { 260 test_callback(data); 261 } 262 #endif 263 } 264 265 tsd_t * 266 malloc_tsd_boot0(void) { 267 tsd_t *tsd; 268 269 ncleanups = 0; 270 if (tsd_boot0()) { 271 return NULL; 272 } 273 tsd = tsd_fetch(); 274 *tsd_arenas_tdata_bypassp_get(tsd) = true; 275 return tsd; 276 } 277 278 void 279 malloc_tsd_boot1(void) { 280 tsd_boot1(); 281 tsd_t *tsd = tsd_fetch(); 282 /* malloc_slow has been set properly. Update tsd_slow. */ 283 tsd_slow_update(tsd); 284 *tsd_arenas_tdata_bypassp_get(tsd) = false; 285 } 286 287 #ifdef _WIN32 288 static BOOL WINAPI 289 _tls_callback(HINSTANCE hinstDLL, DWORD fdwReason, LPVOID lpvReserved) { 290 switch (fdwReason) { 291 #ifdef JEMALLOC_LAZY_LOCK 292 case DLL_THREAD_ATTACH: 293 isthreaded = true; 294 break; 295 #endif 296 case DLL_THREAD_DETACH: 297 _malloc_thread_cleanup(); 298 break; 299 default: 300 break; 301 } 302 return true; 303 } 304 305 /* 306 * We need to be able to say "read" here (in the "pragma section"), but have 307 * hooked "read". We won't read for the rest of the file, so we can get away 308 * with unhooking. 309 */ 310 #ifdef read 311 # undef read 312 #endif 313 314 #ifdef _MSC_VER 315 # ifdef _M_IX86 316 # pragma comment(linker, "/INCLUDE:__tls_used") 317 # pragma comment(linker, "/INCLUDE:_tls_callback") 318 # else 319 # pragma comment(linker, "/INCLUDE:_tls_used") 320 # pragma comment(linker, "/INCLUDE:tls_callback") 321 # endif 322 # pragma section(".CRT$XLY",long,read) 323 #endif 324 JEMALLOC_SECTION(".CRT$XLY") JEMALLOC_ATTR(used) 325 BOOL (WINAPI *const tls_callback)(HINSTANCE hinstDLL, 326 DWORD fdwReason, LPVOID lpvReserved) = _tls_callback; 327 #endif 328 329 #if (!defined(JEMALLOC_MALLOC_THREAD_CLEANUP) && !defined(JEMALLOC_TLS) && \ 330 !defined(_WIN32)) 331 void * 332 tsd_init_check_recursion(tsd_init_head_t *head, tsd_init_block_t *block) { 333 pthread_t self = pthread_self(); 334 tsd_init_block_t *iter; 335 336 /* Check whether this thread has already inserted into the list. */ 337 malloc_mutex_lock(TSDN_NULL, &head->lock); 338 ql_foreach(iter, &head->blocks, link) { 339 if (iter->thread == self) { 340 malloc_mutex_unlock(TSDN_NULL, &head->lock); 341 return iter->data; 342 } 343 } 344 /* Insert block into list. */ 345 ql_elm_new(block, link); 346 block->thread = self; 347 ql_tail_insert(&head->blocks, block, link); 348 malloc_mutex_unlock(TSDN_NULL, &head->lock); 349 return NULL; 350 } 351 352 void 353 tsd_init_finish(tsd_init_head_t *head, tsd_init_block_t *block) { 354 malloc_mutex_lock(TSDN_NULL, &head->lock); 355 ql_remove(&head->blocks, block, link); 356 malloc_mutex_unlock(TSDN_NULL, &head->lock); 357 } 358 #endif 359