1 //===-- tsan_interceptors_mac.cpp -----------------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file is a part of ThreadSanitizer (TSan), a race detector. 10 // 11 // Mac-specific interceptors. 12 //===----------------------------------------------------------------------===// 13 14 #include "sanitizer_common/sanitizer_platform.h" 15 #if SANITIZER_APPLE 16 17 # include <errno.h> 18 # include <libkern/OSAtomic.h> 19 # include <objc/objc-sync.h> 20 # include <os/lock.h> 21 # include <sys/ucontext.h> 22 23 # include "interception/interception.h" 24 # include "sanitizer_common/sanitizer_addrhashmap.h" 25 # include "tsan_interceptors.h" 26 # include "tsan_interface.h" 27 # include "tsan_interface_ann.h" 28 29 # if defined(__has_include) && __has_include(<xpc/xpc.h>) 30 # include <xpc/xpc.h> 31 # endif // #if defined(__has_include) && __has_include(<xpc/xpc.h>) 32 33 typedef long long_t; 34 35 extern "C" { 36 int getcontext(ucontext_t *ucp) __attribute__((returns_twice)); 37 int setcontext(const ucontext_t *ucp); 38 } 39 40 namespace __tsan { 41 42 // The non-barrier versions of OSAtomic* functions are semantically mo_relaxed, 43 // but the two variants (e.g. OSAtomicAdd32 and OSAtomicAdd32Barrier) are 44 // actually aliases of each other, and we cannot have different interceptors for 45 // them, because they're actually the same function. Thus, we have to stay 46 // conservative and treat the non-barrier versions as mo_acq_rel. 47 static constexpr morder kMacOrderBarrier = mo_acq_rel; 48 static constexpr morder kMacOrderNonBarrier = mo_acq_rel; 49 static constexpr morder kMacFailureOrder = mo_relaxed; 50 51 # define OSATOMIC_INTERCEPTOR(return_t, t, tsan_t, f, tsan_atomic_f, mo) \ 52 TSAN_INTERCEPTOR(return_t, f, t x, volatile t *ptr) { \ 53 SCOPED_TSAN_INTERCEPTOR(f, x, ptr); \ 54 return tsan_atomic_f((volatile tsan_t *)ptr, x, mo); \ 55 } 56 57 # define OSATOMIC_INTERCEPTOR_PLUS_X(return_t, t, tsan_t, f, tsan_atomic_f, \ 58 mo) \ 59 TSAN_INTERCEPTOR(return_t, f, t x, volatile t *ptr) { \ 60 SCOPED_TSAN_INTERCEPTOR(f, x, ptr); \ 61 return tsan_atomic_f((volatile tsan_t *)ptr, x, mo) + x; \ 62 } 63 64 # define OSATOMIC_INTERCEPTOR_PLUS_1(return_t, t, tsan_t, f, tsan_atomic_f, \ 65 mo) \ 66 TSAN_INTERCEPTOR(return_t, f, volatile t *ptr) { \ 67 SCOPED_TSAN_INTERCEPTOR(f, ptr); \ 68 return tsan_atomic_f((volatile tsan_t *)ptr, 1, mo) + 1; \ 69 } 70 71 # define OSATOMIC_INTERCEPTOR_MINUS_1(return_t, t, tsan_t, f, tsan_atomic_f, \ 72 mo) \ 73 TSAN_INTERCEPTOR(return_t, f, volatile t *ptr) { \ 74 SCOPED_TSAN_INTERCEPTOR(f, ptr); \ 75 return tsan_atomic_f((volatile tsan_t *)ptr, 1, mo) - 1; \ 76 } 77 78 # define OSATOMIC_INTERCEPTORS_ARITHMETIC(f, tsan_atomic_f, m) \ 79 m(int32_t, int32_t, a32, f##32, __tsan_atomic32_##tsan_atomic_f, \ 80 kMacOrderNonBarrier) \ 81 m(int32_t, int32_t, a32, f##32##Barrier, \ 82 __tsan_atomic32_##tsan_atomic_f, kMacOrderBarrier) \ 83 m(int64_t, int64_t, a64, f##64, __tsan_atomic64_##tsan_atomic_f, \ 84 kMacOrderNonBarrier) \ 85 m(int64_t, int64_t, a64, f##64##Barrier, \ 86 __tsan_atomic64_##tsan_atomic_f, kMacOrderBarrier) 87 88 # define OSATOMIC_INTERCEPTORS_BITWISE(f, tsan_atomic_f, m, m_orig) \ 89 m(int32_t, uint32_t, a32, f##32, __tsan_atomic32_##tsan_atomic_f, \ 90 kMacOrderNonBarrier) \ 91 m(int32_t, uint32_t, a32, f##32##Barrier, \ 92 __tsan_atomic32_##tsan_atomic_f, kMacOrderBarrier) \ 93 m_orig(int32_t, uint32_t, a32, f##32##Orig, \ 94 __tsan_atomic32_##tsan_atomic_f, kMacOrderNonBarrier) \ 95 m_orig(int32_t, uint32_t, a32, f##32##OrigBarrier, \ 96 __tsan_atomic32_##tsan_atomic_f, kMacOrderBarrier) 97 98 # pragma clang diagnostic push // OSAtomic* deprecation 99 # pragma clang diagnostic ignored "-Wdeprecated-declarations" 100 OSATOMIC_INTERCEPTORS_ARITHMETIC(OSAtomicAdd, fetch_add, 101 OSATOMIC_INTERCEPTOR_PLUS_X) 102 OSATOMIC_INTERCEPTORS_ARITHMETIC(OSAtomicIncrement, fetch_add, 103 OSATOMIC_INTERCEPTOR_PLUS_1) 104 OSATOMIC_INTERCEPTORS_ARITHMETIC(OSAtomicDecrement, fetch_sub, 105 OSATOMIC_INTERCEPTOR_MINUS_1) 106 OSATOMIC_INTERCEPTORS_BITWISE(OSAtomicOr, fetch_or, OSATOMIC_INTERCEPTOR_PLUS_X, 107 OSATOMIC_INTERCEPTOR) 108 OSATOMIC_INTERCEPTORS_BITWISE(OSAtomicAnd, fetch_and, 109 OSATOMIC_INTERCEPTOR_PLUS_X, OSATOMIC_INTERCEPTOR) 110 OSATOMIC_INTERCEPTORS_BITWISE(OSAtomicXor, fetch_xor, 111 OSATOMIC_INTERCEPTOR_PLUS_X, OSATOMIC_INTERCEPTOR) 112 # pragma clang diagnostic pop // OSAtomic* deprecation 113 114 # define OSATOMIC_INTERCEPTORS_CAS(f, tsan_atomic_f, tsan_t, t) \ 115 TSAN_INTERCEPTOR(bool, f, t old_value, t new_value, t volatile *ptr) { \ 116 SCOPED_TSAN_INTERCEPTOR(f, old_value, new_value, ptr); \ 117 return tsan_atomic_f##_compare_exchange_strong( \ 118 (volatile tsan_t *)ptr, (tsan_t *)&old_value, (tsan_t)new_value, \ 119 kMacOrderNonBarrier, kMacFailureOrder); \ 120 } \ 121 \ 122 TSAN_INTERCEPTOR(bool, f##Barrier, t old_value, t new_value, \ 123 t volatile *ptr) { \ 124 SCOPED_TSAN_INTERCEPTOR(f##Barrier, old_value, new_value, ptr); \ 125 return tsan_atomic_f##_compare_exchange_strong( \ 126 (volatile tsan_t *)ptr, (tsan_t *)&old_value, (tsan_t)new_value, \ 127 kMacOrderBarrier, kMacFailureOrder); \ 128 } 129 130 # pragma clang diagnostic push // OSAtomicCompareAndSwap* deprecation 131 # pragma clang diagnostic ignored "-Wdeprecated-declarations" 132 OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwapInt, __tsan_atomic32, a32, int) 133 OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwapLong, __tsan_atomic64, a64, 134 long_t) 135 OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwapPtr, __tsan_atomic64, a64, 136 void *) 137 OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwap32, __tsan_atomic32, a32, 138 int32_t) 139 OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwap64, __tsan_atomic64, a64, 140 int64_t) 141 # pragma clang diagnostic pop // OSAtomicCompareAndSwap* deprecation 142 143 # define OSATOMIC_INTERCEPTOR_BITOP(f, op, clear, mo) \ 144 TSAN_INTERCEPTOR(bool, f, uint32_t n, volatile void *ptr) { \ 145 SCOPED_TSAN_INTERCEPTOR(f, n, ptr); \ 146 volatile char *byte_ptr = ((volatile char *)ptr) + (n >> 3); \ 147 char bit = 0x80u >> (n & 7); \ 148 char mask = clear ? ~bit : bit; \ 149 char orig_byte = op((volatile a8 *)byte_ptr, mask, mo); \ 150 return orig_byte & bit; \ 151 } 152 153 # define OSATOMIC_INTERCEPTORS_BITOP(f, op, clear) \ 154 OSATOMIC_INTERCEPTOR_BITOP(f, op, clear, kMacOrderNonBarrier) \ 155 OSATOMIC_INTERCEPTOR_BITOP(f##Barrier, op, clear, kMacOrderBarrier) 156 157 # pragma clang diagnostic push // OSAtomicTestAnd* deprecation 158 # pragma clang diagnostic ignored "-Wdeprecated-declarations" 159 OSATOMIC_INTERCEPTORS_BITOP(OSAtomicTestAndSet, __tsan_atomic8_fetch_or, false) 160 OSATOMIC_INTERCEPTORS_BITOP(OSAtomicTestAndClear, __tsan_atomic8_fetch_and, 161 true) 162 # pragma clang diagnostic pop // OSAtomicTestAnd* deprecation 163 164 TSAN_INTERCEPTOR(void, OSAtomicEnqueue, OSQueueHead *list, void *item, 165 size_t offset) { 166 SCOPED_TSAN_INTERCEPTOR(OSAtomicEnqueue, list, item, offset); 167 __tsan_release(item); 168 REAL(OSAtomicEnqueue)(list, item, offset); 169 } 170 171 TSAN_INTERCEPTOR(void *, OSAtomicDequeue, OSQueueHead *list, size_t offset) { 172 SCOPED_TSAN_INTERCEPTOR(OSAtomicDequeue, list, offset); 173 void *item = REAL(OSAtomicDequeue)(list, offset); 174 if (item) 175 __tsan_acquire(item); 176 return item; 177 } 178 179 // OSAtomicFifoEnqueue and OSAtomicFifoDequeue are only on OS X. 180 # if !SANITIZER_IOS 181 182 TSAN_INTERCEPTOR(void, OSAtomicFifoEnqueue, OSFifoQueueHead *list, void *item, 183 size_t offset) { 184 SCOPED_TSAN_INTERCEPTOR(OSAtomicFifoEnqueue, list, item, offset); 185 __tsan_release(item); 186 REAL(OSAtomicFifoEnqueue)(list, item, offset); 187 } 188 189 TSAN_INTERCEPTOR(void *, OSAtomicFifoDequeue, OSFifoQueueHead *list, 190 size_t offset) { 191 SCOPED_TSAN_INTERCEPTOR(OSAtomicFifoDequeue, list, offset); 192 void *item = REAL(OSAtomicFifoDequeue)(list, offset); 193 if (item) 194 __tsan_acquire(item); 195 return item; 196 } 197 198 # endif 199 200 // If `OSSPINLOCK_USE_INLINED=1` is set, then SDK headers don't declare these 201 // as functions, but macros that call non-deprecated APIs. Undefine these 202 // macros so they don't interfere with the interceptor machinery. 203 # undef OSSpinLockLock 204 # undef OSSpinLockTry 205 # undef OSSpinLockUnlock 206 207 # pragma clang diagnostic push // OSSpinLock* deprecation 208 # pragma clang diagnostic ignored "-Wdeprecated-declarations" 209 210 TSAN_INTERCEPTOR(void, OSSpinLockLock, volatile OSSpinLock *lock) { 211 CHECK(!cur_thread()->is_dead); 212 if (!cur_thread()->is_inited) { 213 return REAL(OSSpinLockLock)(lock); 214 } 215 SCOPED_TSAN_INTERCEPTOR(OSSpinLockLock, lock); 216 REAL(OSSpinLockLock)(lock); 217 Acquire(thr, pc, (uptr)lock); 218 } 219 220 TSAN_INTERCEPTOR(bool, OSSpinLockTry, volatile OSSpinLock *lock) { 221 CHECK(!cur_thread()->is_dead); 222 if (!cur_thread()->is_inited) { 223 return REAL(OSSpinLockTry)(lock); 224 } 225 SCOPED_TSAN_INTERCEPTOR(OSSpinLockTry, lock); 226 bool result = REAL(OSSpinLockTry)(lock); 227 if (result) 228 Acquire(thr, pc, (uptr)lock); 229 return result; 230 } 231 232 TSAN_INTERCEPTOR(void, OSSpinLockUnlock, volatile OSSpinLock *lock) { 233 CHECK(!cur_thread()->is_dead); 234 if (!cur_thread()->is_inited) { 235 return REAL(OSSpinLockUnlock)(lock); 236 } 237 SCOPED_TSAN_INTERCEPTOR(OSSpinLockUnlock, lock); 238 Release(thr, pc, (uptr)lock); 239 REAL(OSSpinLockUnlock)(lock); 240 } 241 # pragma clang diagnostic pop // OSSpinLock* deprecation 242 243 TSAN_INTERCEPTOR(void, os_lock_lock, void *lock) { 244 CHECK(!cur_thread()->is_dead); 245 if (!cur_thread()->is_inited) { 246 return REAL(os_lock_lock)(lock); 247 } 248 SCOPED_TSAN_INTERCEPTOR(os_lock_lock, lock); 249 REAL(os_lock_lock)(lock); 250 Acquire(thr, pc, (uptr)lock); 251 } 252 253 TSAN_INTERCEPTOR(bool, os_lock_trylock, void *lock) { 254 CHECK(!cur_thread()->is_dead); 255 if (!cur_thread()->is_inited) { 256 return REAL(os_lock_trylock)(lock); 257 } 258 SCOPED_TSAN_INTERCEPTOR(os_lock_trylock, lock); 259 bool result = REAL(os_lock_trylock)(lock); 260 if (result) 261 Acquire(thr, pc, (uptr)lock); 262 return result; 263 } 264 265 TSAN_INTERCEPTOR(void, os_lock_unlock, void *lock) { 266 CHECK(!cur_thread()->is_dead); 267 if (!cur_thread()->is_inited) { 268 return REAL(os_lock_unlock)(lock); 269 } 270 SCOPED_TSAN_INTERCEPTOR(os_lock_unlock, lock); 271 Release(thr, pc, (uptr)lock); 272 REAL(os_lock_unlock)(lock); 273 } 274 275 TSAN_INTERCEPTOR(void, os_unfair_lock_lock, os_unfair_lock_t lock) { 276 if (!cur_thread()->is_inited || cur_thread()->is_dead) { 277 return REAL(os_unfair_lock_lock)(lock); 278 } 279 SCOPED_TSAN_INTERCEPTOR(os_unfair_lock_lock, lock); 280 REAL(os_unfair_lock_lock)(lock); 281 Acquire(thr, pc, (uptr)lock); 282 } 283 284 TSAN_INTERCEPTOR(void, os_unfair_lock_lock_with_options, os_unfair_lock_t lock, 285 u32 options) { 286 if (!cur_thread()->is_inited || cur_thread()->is_dead) { 287 return REAL(os_unfair_lock_lock_with_options)(lock, options); 288 } 289 SCOPED_TSAN_INTERCEPTOR(os_unfair_lock_lock_with_options, lock, options); 290 REAL(os_unfair_lock_lock_with_options)(lock, options); 291 Acquire(thr, pc, (uptr)lock); 292 } 293 294 TSAN_INTERCEPTOR(bool, os_unfair_lock_trylock, os_unfair_lock_t lock) { 295 if (!cur_thread()->is_inited || cur_thread()->is_dead) { 296 return REAL(os_unfair_lock_trylock)(lock); 297 } 298 SCOPED_TSAN_INTERCEPTOR(os_unfair_lock_trylock, lock); 299 bool result = REAL(os_unfair_lock_trylock)(lock); 300 if (result) 301 Acquire(thr, pc, (uptr)lock); 302 return result; 303 } 304 305 TSAN_INTERCEPTOR(void, os_unfair_lock_unlock, os_unfair_lock_t lock) { 306 if (!cur_thread()->is_inited || cur_thread()->is_dead) { 307 return REAL(os_unfair_lock_unlock)(lock); 308 } 309 SCOPED_TSAN_INTERCEPTOR(os_unfair_lock_unlock, lock); 310 Release(thr, pc, (uptr)lock); 311 REAL(os_unfair_lock_unlock)(lock); 312 } 313 314 # if defined(__has_include) && __has_include(<xpc/xpc.h>) 315 316 TSAN_INTERCEPTOR(void, xpc_connection_set_event_handler, 317 xpc_connection_t connection, xpc_handler_t handler) { 318 SCOPED_TSAN_INTERCEPTOR(xpc_connection_set_event_handler, connection, 319 handler); 320 Release(thr, pc, (uptr)connection); 321 xpc_handler_t new_handler = ^(xpc_object_t object) { 322 { 323 SCOPED_INTERCEPTOR_RAW(xpc_connection_set_event_handler); 324 Acquire(thr, pc, (uptr)connection); 325 } 326 handler(object); 327 }; 328 REAL(xpc_connection_set_event_handler)(connection, new_handler); 329 } 330 331 TSAN_INTERCEPTOR(void, xpc_connection_send_barrier, xpc_connection_t connection, 332 dispatch_block_t barrier) { 333 SCOPED_TSAN_INTERCEPTOR(xpc_connection_send_barrier, connection, barrier); 334 Release(thr, pc, (uptr)connection); 335 dispatch_block_t new_barrier = ^() { 336 { 337 SCOPED_INTERCEPTOR_RAW(xpc_connection_send_barrier); 338 Acquire(thr, pc, (uptr)connection); 339 } 340 barrier(); 341 }; 342 REAL(xpc_connection_send_barrier)(connection, new_barrier); 343 } 344 345 TSAN_INTERCEPTOR(void, xpc_connection_send_message_with_reply, 346 xpc_connection_t connection, xpc_object_t message, 347 dispatch_queue_t replyq, xpc_handler_t handler) { 348 SCOPED_TSAN_INTERCEPTOR(xpc_connection_send_message_with_reply, connection, 349 message, replyq, handler); 350 Release(thr, pc, (uptr)connection); 351 xpc_handler_t new_handler = ^(xpc_object_t object) { 352 { 353 SCOPED_INTERCEPTOR_RAW(xpc_connection_send_message_with_reply); 354 Acquire(thr, pc, (uptr)connection); 355 } 356 handler(object); 357 }; 358 REAL(xpc_connection_send_message_with_reply) 359 (connection, message, replyq, new_handler); 360 } 361 362 TSAN_INTERCEPTOR(void, xpc_connection_cancel, xpc_connection_t connection) { 363 SCOPED_TSAN_INTERCEPTOR(xpc_connection_cancel, connection); 364 Release(thr, pc, (uptr)connection); 365 REAL(xpc_connection_cancel)(connection); 366 } 367 368 # endif // #if defined(__has_include) && __has_include(<xpc/xpc.h>) 369 370 // Determines whether the Obj-C object pointer is a tagged pointer. Tagged 371 // pointers encode the object data directly in their pointer bits and do not 372 // have an associated memory allocation. The Obj-C runtime uses tagged pointers 373 // to transparently optimize small objects. 374 static bool IsTaggedObjCPointer(id obj) { 375 const uptr kPossibleTaggedBits = 0x8000000000000001ull; 376 return ((uptr)obj & kPossibleTaggedBits) != 0; 377 } 378 379 // Returns an address which can be used to inform TSan about synchronization 380 // points (MutexLock/Unlock). The TSan infrastructure expects this to be a valid 381 // address in the process space. We do a small allocation here to obtain a 382 // stable address (the array backing the hash map can change). The memory is 383 // never free'd (leaked) and allocation and locking are slow, but this code only 384 // runs for @synchronized with tagged pointers, which is very rare. 385 static uptr GetOrCreateSyncAddress(uptr addr, ThreadState *thr, uptr pc) { 386 typedef AddrHashMap<uptr, 5> Map; 387 static Map Addresses; 388 Map::Handle h(&Addresses, addr); 389 if (h.created()) { 390 ThreadIgnoreBegin(thr, pc); 391 *h = (uptr)user_alloc(thr, pc, /*size=*/1); 392 ThreadIgnoreEnd(thr); 393 } 394 return *h; 395 } 396 397 // Returns an address on which we can synchronize given an Obj-C object pointer. 398 // For normal object pointers, this is just the address of the object in memory. 399 // Tagged pointers are not backed by an actual memory allocation, so we need to 400 // synthesize a valid address. 401 static uptr SyncAddressForObjCObject(id obj, ThreadState *thr, uptr pc) { 402 if (IsTaggedObjCPointer(obj)) 403 return GetOrCreateSyncAddress((uptr)obj, thr, pc); 404 return (uptr)obj; 405 } 406 407 TSAN_INTERCEPTOR(int, objc_sync_enter, id obj) { 408 SCOPED_TSAN_INTERCEPTOR(objc_sync_enter, obj); 409 if (!obj) 410 return REAL(objc_sync_enter)(obj); 411 uptr addr = SyncAddressForObjCObject(obj, thr, pc); 412 MutexPreLock(thr, pc, addr, MutexFlagWriteReentrant); 413 int result = REAL(objc_sync_enter)(obj); 414 CHECK_EQ(result, OBJC_SYNC_SUCCESS); 415 MutexPostLock(thr, pc, addr, MutexFlagWriteReentrant); 416 return result; 417 } 418 419 TSAN_INTERCEPTOR(int, objc_sync_exit, id obj) { 420 SCOPED_TSAN_INTERCEPTOR(objc_sync_exit, obj); 421 if (!obj) 422 return REAL(objc_sync_exit)(obj); 423 uptr addr = SyncAddressForObjCObject(obj, thr, pc); 424 MutexUnlock(thr, pc, addr); 425 int result = REAL(objc_sync_exit)(obj); 426 if (result != OBJC_SYNC_SUCCESS) 427 MutexInvalidAccess(thr, pc, addr); 428 return result; 429 } 430 431 TSAN_INTERCEPTOR(int, swapcontext, ucontext_t *oucp, const ucontext_t *ucp) { 432 { 433 SCOPED_INTERCEPTOR_RAW(swapcontext, oucp, ucp); 434 } 435 // Because of swapcontext() semantics we have no option but to copy its 436 // implementation here 437 if (!oucp || !ucp) { 438 errno = EINVAL; 439 return -1; 440 } 441 ThreadState *thr = cur_thread(); 442 const int UCF_SWAPPED = 0x80000000; 443 oucp->uc_onstack &= ~UCF_SWAPPED; 444 thr->ignore_interceptors++; 445 int ret = getcontext(oucp); 446 if (!(oucp->uc_onstack & UCF_SWAPPED)) { 447 thr->ignore_interceptors--; 448 if (!ret) { 449 oucp->uc_onstack |= UCF_SWAPPED; 450 ret = setcontext(ucp); 451 } 452 } 453 return ret; 454 } 455 456 // On macOS, libc++ is always linked dynamically, so intercepting works the 457 // usual way. 458 # define STDCXX_INTERCEPTOR TSAN_INTERCEPTOR 459 460 namespace { 461 struct fake_shared_weak_count { 462 volatile a64 shared_owners; 463 volatile a64 shared_weak_owners; 464 virtual void _unused_0x0() = 0; 465 virtual void _unused_0x8() = 0; 466 virtual void on_zero_shared() = 0; 467 virtual void _unused_0x18() = 0; 468 virtual void on_zero_shared_weak() = 0; 469 virtual ~fake_shared_weak_count() = 0; // suppress -Wnon-virtual-dtor 470 }; 471 } // namespace 472 473 // The following code adds libc++ interceptors for: 474 // void __shared_weak_count::__release_shared() _NOEXCEPT; 475 // bool __shared_count::__release_shared() _NOEXCEPT; 476 // Shared and weak pointers in C++ maintain reference counts via atomics in 477 // libc++.dylib, which are TSan-invisible, and this leads to false positives in 478 // destructor code. These interceptors re-implements the whole functions so that 479 // the mo_acq_rel semantics of the atomic decrement are visible. 480 // 481 // Unfortunately, the interceptors cannot simply Acquire/Release some sync 482 // object and call the original function, because it would have a race between 483 // the sync and the destruction of the object. Calling both under a lock will 484 // not work because the destructor can invoke this interceptor again (and even 485 // in a different thread, so recursive locks don't help). 486 487 STDCXX_INTERCEPTOR(void, _ZNSt3__119__shared_weak_count16__release_sharedEv, 488 fake_shared_weak_count *o) { 489 if (!flags()->shared_ptr_interceptor) 490 return REAL(_ZNSt3__119__shared_weak_count16__release_sharedEv)(o); 491 492 SCOPED_TSAN_INTERCEPTOR(_ZNSt3__119__shared_weak_count16__release_sharedEv, 493 o); 494 if (__tsan_atomic64_fetch_add(&o->shared_owners, -1, mo_release) == 0) { 495 Acquire(thr, pc, (uptr)&o->shared_owners); 496 o->on_zero_shared(); 497 if (__tsan_atomic64_fetch_add(&o->shared_weak_owners, -1, mo_release) == 498 0) { 499 Acquire(thr, pc, (uptr)&o->shared_weak_owners); 500 o->on_zero_shared_weak(); 501 } 502 } 503 } 504 505 STDCXX_INTERCEPTOR(bool, _ZNSt3__114__shared_count16__release_sharedEv, 506 fake_shared_weak_count *o) { 507 if (!flags()->shared_ptr_interceptor) 508 return REAL(_ZNSt3__114__shared_count16__release_sharedEv)(o); 509 510 SCOPED_TSAN_INTERCEPTOR(_ZNSt3__114__shared_count16__release_sharedEv, o); 511 if (__tsan_atomic64_fetch_add(&o->shared_owners, -1, mo_release) == 0) { 512 Acquire(thr, pc, (uptr)&o->shared_owners); 513 o->on_zero_shared(); 514 return true; 515 } 516 return false; 517 } 518 519 namespace { 520 struct call_once_callback_args { 521 void (*orig_func)(void *arg); 522 void *orig_arg; 523 void *flag; 524 }; 525 526 void call_once_callback_wrapper(void *arg) { 527 call_once_callback_args *new_args = (call_once_callback_args *)arg; 528 new_args->orig_func(new_args->orig_arg); 529 __tsan_release(new_args->flag); 530 } 531 } // namespace 532 533 // This adds a libc++ interceptor for: 534 // void __call_once(volatile unsigned long&, void*, void(*)(void*)); 535 // C++11 call_once is implemented via an internal function __call_once which is 536 // inside libc++.dylib, and the atomic release store inside it is thus 537 // TSan-invisible. To avoid false positives, this interceptor wraps the callback 538 // function and performs an explicit Release after the user code has run. 539 STDCXX_INTERCEPTOR(void, _ZNSt3__111__call_onceERVmPvPFvS2_E, void *flag, 540 void *arg, void (*func)(void *arg)) { 541 call_once_callback_args new_args = {func, arg, flag}; 542 REAL(_ZNSt3__111__call_onceERVmPvPFvS2_E)(flag, &new_args, 543 call_once_callback_wrapper); 544 } 545 546 } // namespace __tsan 547 548 #endif // SANITIZER_APPLE 549