1 //===-- tsan_interceptors_mac.cc ------------------------------------------===// 2 // 3 // This file is distributed under the University of Illinois Open Source 4 // License. See LICENSE.TXT for details. 5 // 6 //===----------------------------------------------------------------------===// 7 // 8 // This file is a part of ThreadSanitizer (TSan), a race detector. 9 // 10 // Mac-specific interceptors. 11 //===----------------------------------------------------------------------===// 12 13 #include "sanitizer_common/sanitizer_platform.h" 14 #if SANITIZER_MAC 15 16 #include "interception/interception.h" 17 #include "tsan_interceptors.h" 18 #include "tsan_interface.h" 19 #include "tsan_interface_ann.h" 20 21 #include <libkern/OSAtomic.h> 22 23 #if defined(__has_include) && __has_include(<xpc/xpc.h>) 24 #include <xpc/xpc.h> 25 #endif // #if defined(__has_include) && __has_include(<xpc/xpc.h>) 26 27 typedef long long_t; // NOLINT 28 29 namespace __tsan { 30 31 // The non-barrier versions of OSAtomic* functions are semantically mo_relaxed, 32 // but the two variants (e.g. OSAtomicAdd32 and OSAtomicAdd32Barrier) are 33 // actually aliases of each other, and we cannot have different interceptors for 34 // them, because they're actually the same function. Thus, we have to stay 35 // conservative and treat the non-barrier versions as mo_acq_rel. 36 static const morder kMacOrderBarrier = mo_acq_rel; 37 static const morder kMacOrderNonBarrier = mo_acq_rel; 38 39 #define OSATOMIC_INTERCEPTOR(return_t, t, tsan_t, f, tsan_atomic_f, mo) \ 40 TSAN_INTERCEPTOR(return_t, f, t x, volatile t *ptr) { \ 41 SCOPED_TSAN_INTERCEPTOR(f, x, ptr); \ 42 return tsan_atomic_f((volatile tsan_t *)ptr, x, mo); \ 43 } 44 45 #define OSATOMIC_INTERCEPTOR_PLUS_X(return_t, t, tsan_t, f, tsan_atomic_f, mo) \ 46 TSAN_INTERCEPTOR(return_t, f, t x, volatile t *ptr) { \ 47 SCOPED_TSAN_INTERCEPTOR(f, x, ptr); \ 48 return tsan_atomic_f((volatile tsan_t *)ptr, x, mo) + x; \ 49 } 50 51 #define OSATOMIC_INTERCEPTOR_PLUS_1(return_t, t, tsan_t, f, tsan_atomic_f, mo) \ 52 TSAN_INTERCEPTOR(return_t, f, volatile t *ptr) { \ 53 SCOPED_TSAN_INTERCEPTOR(f, ptr); \ 54 return tsan_atomic_f((volatile tsan_t *)ptr, 1, mo) + 1; \ 55 } 56 57 #define OSATOMIC_INTERCEPTOR_MINUS_1(return_t, t, tsan_t, f, tsan_atomic_f, \ 58 mo) \ 59 TSAN_INTERCEPTOR(return_t, f, volatile t *ptr) { \ 60 SCOPED_TSAN_INTERCEPTOR(f, ptr); \ 61 return tsan_atomic_f((volatile tsan_t *)ptr, 1, mo) - 1; \ 62 } 63 64 #define OSATOMIC_INTERCEPTORS_ARITHMETIC(f, tsan_atomic_f, m) \ 65 m(int32_t, int32_t, a32, f##32, __tsan_atomic32_##tsan_atomic_f, \ 66 kMacOrderNonBarrier) \ 67 m(int32_t, int32_t, a32, f##32##Barrier, __tsan_atomic32_##tsan_atomic_f, \ 68 kMacOrderBarrier) \ 69 m(int64_t, int64_t, a64, f##64, __tsan_atomic64_##tsan_atomic_f, \ 70 kMacOrderNonBarrier) \ 71 m(int64_t, int64_t, a64, f##64##Barrier, __tsan_atomic64_##tsan_atomic_f, \ 72 kMacOrderBarrier) 73 74 #define OSATOMIC_INTERCEPTORS_BITWISE(f, tsan_atomic_f, m, m_orig) \ 75 m(int32_t, uint32_t, a32, f##32, __tsan_atomic32_##tsan_atomic_f, \ 76 kMacOrderNonBarrier) \ 77 m(int32_t, uint32_t, a32, f##32##Barrier, __tsan_atomic32_##tsan_atomic_f, \ 78 kMacOrderBarrier) \ 79 m_orig(int32_t, uint32_t, a32, f##32##Orig, __tsan_atomic32_##tsan_atomic_f, \ 80 kMacOrderNonBarrier) \ 81 m_orig(int32_t, uint32_t, a32, f##32##OrigBarrier, \ 82 __tsan_atomic32_##tsan_atomic_f, kMacOrderBarrier) 83 84 OSATOMIC_INTERCEPTORS_ARITHMETIC(OSAtomicAdd, fetch_add, 85 OSATOMIC_INTERCEPTOR_PLUS_X) 86 OSATOMIC_INTERCEPTORS_ARITHMETIC(OSAtomicIncrement, fetch_add, 87 OSATOMIC_INTERCEPTOR_PLUS_1) 88 OSATOMIC_INTERCEPTORS_ARITHMETIC(OSAtomicDecrement, fetch_sub, 89 OSATOMIC_INTERCEPTOR_MINUS_1) 90 OSATOMIC_INTERCEPTORS_BITWISE(OSAtomicOr, fetch_or, OSATOMIC_INTERCEPTOR_PLUS_X, 91 OSATOMIC_INTERCEPTOR) 92 OSATOMIC_INTERCEPTORS_BITWISE(OSAtomicAnd, fetch_and, 93 OSATOMIC_INTERCEPTOR_PLUS_X, OSATOMIC_INTERCEPTOR) 94 OSATOMIC_INTERCEPTORS_BITWISE(OSAtomicXor, fetch_xor, 95 OSATOMIC_INTERCEPTOR_PLUS_X, OSATOMIC_INTERCEPTOR) 96 97 #define OSATOMIC_INTERCEPTORS_CAS(f, tsan_atomic_f, tsan_t, t) \ 98 TSAN_INTERCEPTOR(bool, f, t old_value, t new_value, t volatile *ptr) { \ 99 SCOPED_TSAN_INTERCEPTOR(f, old_value, new_value, ptr); \ 100 return tsan_atomic_f##_compare_exchange_strong( \ 101 (volatile tsan_t *)ptr, (tsan_t *)&old_value, (tsan_t)new_value, \ 102 kMacOrderNonBarrier, kMacOrderNonBarrier); \ 103 } \ 104 \ 105 TSAN_INTERCEPTOR(bool, f##Barrier, t old_value, t new_value, \ 106 t volatile *ptr) { \ 107 SCOPED_TSAN_INTERCEPTOR(f##Barrier, old_value, new_value, ptr); \ 108 return tsan_atomic_f##_compare_exchange_strong( \ 109 (volatile tsan_t *)ptr, (tsan_t *)&old_value, (tsan_t)new_value, \ 110 kMacOrderBarrier, kMacOrderNonBarrier); \ 111 } 112 113 OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwapInt, __tsan_atomic32, a32, int) 114 OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwapLong, __tsan_atomic64, a64, 115 long_t) 116 OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwapPtr, __tsan_atomic64, a64, 117 void *) 118 OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwap32, __tsan_atomic32, a32, 119 int32_t) 120 OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwap64, __tsan_atomic64, a64, 121 int64_t) 122 123 #define OSATOMIC_INTERCEPTOR_BITOP(f, op, clear, mo) \ 124 TSAN_INTERCEPTOR(bool, f, uint32_t n, volatile void *ptr) { \ 125 SCOPED_TSAN_INTERCEPTOR(f, n, ptr); \ 126 volatile char *byte_ptr = ((volatile char *)ptr) + (n >> 3); \ 127 char bit = 0x80u >> (n & 7); \ 128 char mask = clear ? ~bit : bit; \ 129 char orig_byte = op((volatile a8 *)byte_ptr, mask, mo); \ 130 return orig_byte & bit; \ 131 } 132 133 #define OSATOMIC_INTERCEPTORS_BITOP(f, op, clear) \ 134 OSATOMIC_INTERCEPTOR_BITOP(f, op, clear, kMacOrderNonBarrier) \ 135 OSATOMIC_INTERCEPTOR_BITOP(f##Barrier, op, clear, kMacOrderBarrier) 136 137 OSATOMIC_INTERCEPTORS_BITOP(OSAtomicTestAndSet, __tsan_atomic8_fetch_or, false) 138 OSATOMIC_INTERCEPTORS_BITOP(OSAtomicTestAndClear, __tsan_atomic8_fetch_and, 139 true) 140 141 TSAN_INTERCEPTOR(void, OSAtomicEnqueue, OSQueueHead *list, void *item, 142 size_t offset) { 143 SCOPED_TSAN_INTERCEPTOR(OSAtomicEnqueue, list, item, offset); 144 __tsan_release(item); 145 REAL(OSAtomicEnqueue)(list, item, offset); 146 } 147 148 TSAN_INTERCEPTOR(void *, OSAtomicDequeue, OSQueueHead *list, size_t offset) { 149 SCOPED_TSAN_INTERCEPTOR(OSAtomicDequeue, list, offset); 150 void *item = REAL(OSAtomicDequeue)(list, offset); 151 if (item) __tsan_acquire(item); 152 return item; 153 } 154 155 // OSAtomicFifoEnqueue and OSAtomicFifoDequeue are only on OS X. 156 #if !SANITIZER_IOS 157 158 TSAN_INTERCEPTOR(void, OSAtomicFifoEnqueue, OSFifoQueueHead *list, void *item, 159 size_t offset) { 160 SCOPED_TSAN_INTERCEPTOR(OSAtomicFifoEnqueue, list, item, offset); 161 __tsan_release(item); 162 REAL(OSAtomicFifoEnqueue)(list, item, offset); 163 } 164 165 TSAN_INTERCEPTOR(void *, OSAtomicFifoDequeue, OSFifoQueueHead *list, 166 size_t offset) { 167 SCOPED_TSAN_INTERCEPTOR(OSAtomicFifoDequeue, list, offset); 168 void *item = REAL(OSAtomicFifoDequeue)(list, offset); 169 if (item) __tsan_acquire(item); 170 return item; 171 } 172 173 #endif 174 175 TSAN_INTERCEPTOR(void, OSSpinLockLock, volatile OSSpinLock *lock) { 176 CHECK(!cur_thread()->is_dead); 177 if (!cur_thread()->is_inited) { 178 return REAL(OSSpinLockLock)(lock); 179 } 180 SCOPED_TSAN_INTERCEPTOR(OSSpinLockLock, lock); 181 REAL(OSSpinLockLock)(lock); 182 Acquire(thr, pc, (uptr)lock); 183 } 184 185 TSAN_INTERCEPTOR(bool, OSSpinLockTry, volatile OSSpinLock *lock) { 186 CHECK(!cur_thread()->is_dead); 187 if (!cur_thread()->is_inited) { 188 return REAL(OSSpinLockTry)(lock); 189 } 190 SCOPED_TSAN_INTERCEPTOR(OSSpinLockTry, lock); 191 bool result = REAL(OSSpinLockTry)(lock); 192 if (result) 193 Acquire(thr, pc, (uptr)lock); 194 return result; 195 } 196 197 TSAN_INTERCEPTOR(void, OSSpinLockUnlock, volatile OSSpinLock *lock) { 198 CHECK(!cur_thread()->is_dead); 199 if (!cur_thread()->is_inited) { 200 return REAL(OSSpinLockUnlock)(lock); 201 } 202 SCOPED_TSAN_INTERCEPTOR(OSSpinLockUnlock, lock); 203 Release(thr, pc, (uptr)lock); 204 REAL(OSSpinLockUnlock)(lock); 205 } 206 207 TSAN_INTERCEPTOR(void, os_lock_lock, void *lock) { 208 CHECK(!cur_thread()->is_dead); 209 if (!cur_thread()->is_inited) { 210 return REAL(os_lock_lock)(lock); 211 } 212 SCOPED_TSAN_INTERCEPTOR(os_lock_lock, lock); 213 REAL(os_lock_lock)(lock); 214 Acquire(thr, pc, (uptr)lock); 215 } 216 217 TSAN_INTERCEPTOR(bool, os_lock_trylock, void *lock) { 218 CHECK(!cur_thread()->is_dead); 219 if (!cur_thread()->is_inited) { 220 return REAL(os_lock_trylock)(lock); 221 } 222 SCOPED_TSAN_INTERCEPTOR(os_lock_trylock, lock); 223 bool result = REAL(os_lock_trylock)(lock); 224 if (result) 225 Acquire(thr, pc, (uptr)lock); 226 return result; 227 } 228 229 TSAN_INTERCEPTOR(void, os_lock_unlock, void *lock) { 230 CHECK(!cur_thread()->is_dead); 231 if (!cur_thread()->is_inited) { 232 return REAL(os_lock_unlock)(lock); 233 } 234 SCOPED_TSAN_INTERCEPTOR(os_lock_unlock, lock); 235 Release(thr, pc, (uptr)lock); 236 REAL(os_lock_unlock)(lock); 237 } 238 239 #if defined(__has_include) && __has_include(<xpc/xpc.h>) 240 241 TSAN_INTERCEPTOR(void, xpc_connection_set_event_handler, 242 xpc_connection_t connection, xpc_handler_t handler) { 243 SCOPED_TSAN_INTERCEPTOR(xpc_connection_set_event_handler, connection, 244 handler); 245 Release(thr, pc, (uptr)connection); 246 xpc_handler_t new_handler = ^(xpc_object_t object) { 247 { 248 SCOPED_INTERCEPTOR_RAW(xpc_connection_set_event_handler); 249 Acquire(thr, pc, (uptr)connection); 250 } 251 handler(object); 252 }; 253 REAL(xpc_connection_set_event_handler)(connection, new_handler); 254 } 255 256 TSAN_INTERCEPTOR(void, xpc_connection_send_barrier, xpc_connection_t connection, 257 dispatch_block_t barrier) { 258 SCOPED_TSAN_INTERCEPTOR(xpc_connection_send_barrier, connection, barrier); 259 Release(thr, pc, (uptr)connection); 260 dispatch_block_t new_barrier = ^() { 261 { 262 SCOPED_INTERCEPTOR_RAW(xpc_connection_send_barrier); 263 Acquire(thr, pc, (uptr)connection); 264 } 265 barrier(); 266 }; 267 REAL(xpc_connection_send_barrier)(connection, new_barrier); 268 } 269 270 TSAN_INTERCEPTOR(void, xpc_connection_send_message_with_reply, 271 xpc_connection_t connection, xpc_object_t message, 272 dispatch_queue_t replyq, xpc_handler_t handler) { 273 SCOPED_TSAN_INTERCEPTOR(xpc_connection_send_message_with_reply, connection, 274 message, replyq, handler); 275 Release(thr, pc, (uptr)connection); 276 xpc_handler_t new_handler = ^(xpc_object_t object) { 277 { 278 SCOPED_INTERCEPTOR_RAW(xpc_connection_send_message_with_reply); 279 Acquire(thr, pc, (uptr)connection); 280 } 281 handler(object); 282 }; 283 REAL(xpc_connection_send_message_with_reply) 284 (connection, message, replyq, new_handler); 285 } 286 287 TSAN_INTERCEPTOR(void, xpc_connection_cancel, xpc_connection_t connection) { 288 SCOPED_TSAN_INTERCEPTOR(xpc_connection_cancel, connection); 289 Release(thr, pc, (uptr)connection); 290 REAL(xpc_connection_cancel)(connection); 291 } 292 293 #endif // #if defined(__has_include) && __has_include(<xpc/xpc.h>) 294 295 // Is the Obj-C object a tagged pointer (i.e. isn't really a valid pointer and 296 // contains data in the pointers bits instead)? 297 static bool IsTaggedObjCPointer(void *obj) { 298 const uptr kPossibleTaggedBits = 0x8000000000000001ull; 299 return ((uptr)obj & kPossibleTaggedBits) != 0; 300 } 301 302 // Return an address on which we can synchronize (Acquire and Release) for a 303 // Obj-C tagged pointer (which is not a valid pointer). Ideally should be a 304 // derived address from 'obj', but for now just return the same global address. 305 // TODO(kubamracek): Return different address for different pointers. 306 static uptr SyncAddressForTaggedPointer(void *obj) { 307 (void)obj; 308 static u64 addr; 309 return (uptr)&addr; 310 } 311 312 // Address on which we can synchronize for an Objective-C object. Supports 313 // tagged pointers. 314 static uptr SyncAddressForObjCObject(void *obj) { 315 if (IsTaggedObjCPointer(obj)) return SyncAddressForTaggedPointer(obj); 316 return (uptr)obj; 317 } 318 319 TSAN_INTERCEPTOR(int, objc_sync_enter, void *obj) { 320 SCOPED_TSAN_INTERCEPTOR(objc_sync_enter, obj); 321 int result = REAL(objc_sync_enter)(obj); 322 if (obj) Acquire(thr, pc, SyncAddressForObjCObject(obj)); 323 return result; 324 } 325 326 TSAN_INTERCEPTOR(int, objc_sync_exit, void *obj) { 327 SCOPED_TSAN_INTERCEPTOR(objc_sync_enter, obj); 328 if (obj) Release(thr, pc, SyncAddressForObjCObject(obj)); 329 return REAL(objc_sync_exit)(obj); 330 } 331 332 // On macOS, libc++ is always linked dynamically, so intercepting works the 333 // usual way. 334 #define STDCXX_INTERCEPTOR TSAN_INTERCEPTOR 335 336 namespace { 337 struct fake_shared_weak_count { 338 volatile a64 shared_owners; 339 volatile a64 shared_weak_owners; 340 virtual void _unused_0x0() = 0; 341 virtual void _unused_0x8() = 0; 342 virtual void on_zero_shared() = 0; 343 virtual void _unused_0x18() = 0; 344 virtual void on_zero_shared_weak() = 0; 345 }; 346 } // namespace 347 348 // The following code adds libc++ interceptors for: 349 // void __shared_weak_count::__release_shared() _NOEXCEPT; 350 // bool __shared_count::__release_shared() _NOEXCEPT; 351 // Shared and weak pointers in C++ maintain reference counts via atomics in 352 // libc++.dylib, which are TSan-invisible, and this leads to false positives in 353 // destructor code. These interceptors re-implements the whole functions so that 354 // the mo_acq_rel semantics of the atomic decrement are visible. 355 // 356 // Unfortunately, the interceptors cannot simply Acquire/Release some sync 357 // object and call the original function, because it would have a race between 358 // the sync and the destruction of the object. Calling both under a lock will 359 // not work because the destructor can invoke this interceptor again (and even 360 // in a different thread, so recursive locks don't help). 361 362 STDCXX_INTERCEPTOR(void, _ZNSt3__119__shared_weak_count16__release_sharedEv, 363 fake_shared_weak_count *o) { 364 if (!flags()->shared_ptr_interceptor) 365 return REAL(_ZNSt3__119__shared_weak_count16__release_sharedEv)(o); 366 367 SCOPED_TSAN_INTERCEPTOR(_ZNSt3__119__shared_weak_count16__release_sharedEv, 368 o); 369 if (__tsan_atomic64_fetch_add(&o->shared_owners, -1, mo_release) == 0) { 370 Acquire(thr, pc, (uptr)&o->shared_owners); 371 o->on_zero_shared(); 372 if (__tsan_atomic64_fetch_add(&o->shared_weak_owners, -1, mo_release) == 373 0) { 374 Acquire(thr, pc, (uptr)&o->shared_weak_owners); 375 o->on_zero_shared_weak(); 376 } 377 } 378 } 379 380 STDCXX_INTERCEPTOR(bool, _ZNSt3__114__shared_count16__release_sharedEv, 381 fake_shared_weak_count *o) { 382 if (!flags()->shared_ptr_interceptor) 383 return REAL(_ZNSt3__114__shared_count16__release_sharedEv)(o); 384 385 SCOPED_TSAN_INTERCEPTOR(_ZNSt3__114__shared_count16__release_sharedEv, o); 386 if (__tsan_atomic64_fetch_add(&o->shared_owners, -1, mo_release) == 0) { 387 Acquire(thr, pc, (uptr)&o->shared_owners); 388 o->on_zero_shared(); 389 return true; 390 } 391 return false; 392 } 393 394 namespace { 395 struct call_once_callback_args { 396 void (*orig_func)(void *arg); 397 void *orig_arg; 398 void *flag; 399 }; 400 401 void call_once_callback_wrapper(void *arg) { 402 call_once_callback_args *new_args = (call_once_callback_args *)arg; 403 new_args->orig_func(new_args->orig_arg); 404 __tsan_release(new_args->flag); 405 } 406 } // namespace 407 408 // This adds a libc++ interceptor for: 409 // void __call_once(volatile unsigned long&, void*, void(*)(void*)); 410 // C++11 call_once is implemented via an internal function __call_once which is 411 // inside libc++.dylib, and the atomic release store inside it is thus 412 // TSan-invisible. To avoid false positives, this interceptor wraps the callback 413 // function and performs an explicit Release after the user code has run. 414 STDCXX_INTERCEPTOR(void, _ZNSt3__111__call_onceERVmPvPFvS2_E, void *flag, 415 void *arg, void (*func)(void *arg)) { 416 call_once_callback_args new_args = {func, arg, flag}; 417 REAL(_ZNSt3__111__call_onceERVmPvPFvS2_E)(flag, &new_args, 418 call_once_callback_wrapper); 419 } 420 421 } // namespace __tsan 422 423 #endif // SANITIZER_MAC 424