1 //===-- hwasan_thread_list.h ------------------------------------*- C++ -*-===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file is a part of HWAddressSanitizer. 10 // 11 //===----------------------------------------------------------------------===// 12 13 // HwasanThreadList is a registry for live threads, as well as an allocator for 14 // HwasanThread objects and their stack history ring buffers. There are 15 // constraints on memory layout of the shadow region and CompactRingBuffer that 16 // are part of the ABI contract between compiler-rt and llvm. 17 // 18 // * Start of the shadow memory region is aligned to 2**kShadowBaseAlignment. 19 // * All stack ring buffers are located within (2**kShadowBaseAlignment) 20 // sized region below and adjacent to the shadow region. 21 // * Each ring buffer has a size of (2**N)*4096 where N is in [0, 7), and is 22 // aligned to twice its size. The value of N can be different for each buffer. 23 // 24 // These constrains guarantee that, given an address A of any element of the 25 // ring buffer, 26 // A_next = (A + sizeof(uptr)) & ~((1 << (N + 13)) - 1) 27 // is the address of the next element of that ring buffer (with wrap-around). 28 // And, with K = kShadowBaseAlignment, 29 // S = (A | ((1 << K) - 1)) + 1 30 // (align up to kShadowBaseAlignment) is the start of the shadow region. 31 // 32 // These calculations are used in compiler instrumentation to update the ring 33 // buffer and obtain the base address of shadow using only two inputs: address 34 // of the current element of the ring buffer, and N (i.e. size of the ring 35 // buffer). Since the value of N is very limited, we pack both inputs into a 36 // single thread-local word as 37 // (1 << (N + 56)) | A 38 // See the implementation of class CompactRingBuffer, which is what is stored in 39 // said thread-local word. 40 // 41 // Note the unusual way of aligning up the address of the shadow: 42 // (A | ((1 << K) - 1)) + 1 43 // It is only correct if A is not already equal to the shadow base address, but 44 // it saves 2 instructions on AArch64. 45 46 #include "hwasan.h" 47 #include "hwasan_allocator.h" 48 #include "hwasan_flags.h" 49 #include "hwasan_thread.h" 50 #include "sanitizer_common/sanitizer_thread_arg_retval.h" 51 52 namespace __hwasan { 53 54 static uptr RingBufferSize() { 55 uptr desired_bytes = flags()->stack_history_size * sizeof(uptr); 56 // FIXME: increase the limit to 8 once this bug is fixed: 57 // https://bugs.llvm.org/show_bug.cgi?id=39030 58 // Note that we *cannot* do that on Android, as the runtime will indefinitely 59 // have to support code that is compiled with ashr, which only works with 60 // shifts up to 6. 61 for (int shift = 0; shift < 7; ++shift) { 62 uptr size = 4096 * (1ULL << shift); 63 if (size >= desired_bytes) 64 return size; 65 } 66 Printf("stack history size too large: %d\n", flags()->stack_history_size); 67 CHECK(0); 68 return 0; 69 } 70 71 struct ThreadStats { 72 uptr n_live_threads; 73 uptr total_stack_size; 74 }; 75 76 class SANITIZER_MUTEX HwasanThreadList { 77 public: 78 HwasanThreadList(uptr storage, uptr size) 79 : free_space_(storage), free_space_end_(storage + size) { 80 // [storage, storage + size) is used as a vector of 81 // thread_alloc_size_-sized, ring_buffer_size_*2-aligned elements. 82 // Each element contains 83 // * a ring buffer at offset 0, 84 // * a Thread object at offset ring_buffer_size_. 85 ring_buffer_size_ = RingBufferSize(); 86 thread_alloc_size_ = 87 RoundUpTo(ring_buffer_size_ + sizeof(Thread), ring_buffer_size_ * 2); 88 } 89 90 Thread *CreateCurrentThread(const Thread::InitState *state = nullptr) 91 SANITIZER_EXCLUDES(free_list_mutex_, live_list_mutex_) { 92 Thread *t = nullptr; 93 { 94 SpinMutexLock l(&free_list_mutex_); 95 if (!free_list_.empty()) { 96 t = free_list_.back(); 97 free_list_.pop_back(); 98 } 99 } 100 if (t) { 101 uptr start = (uptr)t - ring_buffer_size_; 102 internal_memset((void *)start, 0, ring_buffer_size_ + sizeof(Thread)); 103 } else { 104 t = AllocThread(); 105 } 106 { 107 SpinMutexLock l(&live_list_mutex_); 108 live_list_.push_back(t); 109 } 110 t->Init((uptr)t - ring_buffer_size_, ring_buffer_size_, state); 111 AddThreadStats(t); 112 return t; 113 } 114 115 void DontNeedThread(Thread *t) { 116 uptr start = (uptr)t - ring_buffer_size_; 117 ReleaseMemoryPagesToOS(start, start + thread_alloc_size_); 118 } 119 120 void RemoveThreadFromLiveList(Thread *t) 121 SANITIZER_EXCLUDES(live_list_mutex_) { 122 SpinMutexLock l(&live_list_mutex_); 123 for (Thread *&t2 : live_list_) 124 if (t2 == t) { 125 // To remove t2, copy the last element of the list in t2's position, and 126 // pop_back(). This works even if t2 is itself the last element. 127 t2 = live_list_.back(); 128 live_list_.pop_back(); 129 return; 130 } 131 CHECK(0 && "thread not found in live list"); 132 } 133 134 void ReleaseThread(Thread *t) SANITIZER_EXCLUDES(free_list_mutex_) { 135 RemoveThreadStats(t); 136 RemoveThreadFromLiveList(t); 137 t->Destroy(); 138 DontNeedThread(t); 139 SpinMutexLock l(&free_list_mutex_); 140 free_list_.push_back(t); 141 } 142 143 Thread *GetThreadByBufferAddress(uptr p) { 144 return (Thread *)(RoundDownTo(p, ring_buffer_size_ * 2) + 145 ring_buffer_size_); 146 } 147 148 uptr MemoryUsedPerThread() { 149 uptr res = sizeof(Thread) + ring_buffer_size_; 150 if (auto sz = flags()->heap_history_size) 151 res += HeapAllocationsRingBuffer::SizeInBytes(sz); 152 return res; 153 } 154 155 template <class CB> 156 void VisitAllLiveThreads(CB cb) SANITIZER_EXCLUDES(live_list_mutex_) { 157 SpinMutexLock l(&live_list_mutex_); 158 for (Thread *t : live_list_) cb(t); 159 } 160 161 template <class CB> 162 Thread *FindThreadLocked(CB cb) SANITIZER_CHECK_LOCKED(live_list_mutex_) { 163 CheckLocked(); 164 for (Thread *t : live_list_) 165 if (cb(t)) 166 return t; 167 return nullptr; 168 } 169 170 void AddThreadStats(Thread *t) SANITIZER_EXCLUDES(stats_mutex_) { 171 SpinMutexLock l(&stats_mutex_); 172 stats_.n_live_threads++; 173 stats_.total_stack_size += t->stack_size(); 174 } 175 176 void RemoveThreadStats(Thread *t) SANITIZER_EXCLUDES(stats_mutex_) { 177 SpinMutexLock l(&stats_mutex_); 178 stats_.n_live_threads--; 179 stats_.total_stack_size -= t->stack_size(); 180 } 181 182 ThreadStats GetThreadStats() SANITIZER_EXCLUDES(stats_mutex_) { 183 SpinMutexLock l(&stats_mutex_); 184 return stats_; 185 } 186 187 uptr GetRingBufferSize() const { return ring_buffer_size_; } 188 189 void Lock() SANITIZER_ACQUIRE(live_list_mutex_) { live_list_mutex_.Lock(); } 190 void CheckLocked() const SANITIZER_CHECK_LOCKED(live_list_mutex_) { 191 live_list_mutex_.CheckLocked(); 192 } 193 void Unlock() SANITIZER_RELEASE(live_list_mutex_) { 194 live_list_mutex_.Unlock(); 195 } 196 197 private: 198 Thread *AllocThread() { 199 SpinMutexLock l(&free_space_mutex_); 200 uptr align = ring_buffer_size_ * 2; 201 CHECK(IsAligned(free_space_, align)); 202 Thread *t = (Thread *)(free_space_ + ring_buffer_size_); 203 free_space_ += thread_alloc_size_; 204 CHECK_LE(free_space_, free_space_end_); 205 return t; 206 } 207 208 SpinMutex free_space_mutex_; 209 uptr free_space_; 210 uptr free_space_end_; 211 uptr ring_buffer_size_; 212 uptr thread_alloc_size_; 213 214 SpinMutex free_list_mutex_; 215 InternalMmapVector<Thread *> free_list_ 216 SANITIZER_GUARDED_BY(free_list_mutex_); 217 SpinMutex live_list_mutex_; 218 InternalMmapVector<Thread *> live_list_ 219 SANITIZER_GUARDED_BY(live_list_mutex_); 220 221 SpinMutex stats_mutex_; 222 ThreadStats stats_ SANITIZER_GUARDED_BY(stats_mutex_); 223 }; 224 225 void InitThreadList(uptr storage, uptr size); 226 HwasanThreadList &hwasanThreadList(); 227 ThreadArgRetval &hwasanThreadArgRetval(); 228 229 } // namespace __hwasan 230