1 //===-- sanitizer_stackdepot.cpp ------------------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file is shared between AddressSanitizer and ThreadSanitizer 10 // run-time libraries. 11 //===----------------------------------------------------------------------===// 12 13 #include "sanitizer_stackdepot.h" 14 15 #include "sanitizer_common.h" 16 #include "sanitizer_hash.h" 17 #include "sanitizer_stack_store.h" 18 #include "sanitizer_stackdepotbase.h" 19 20 namespace __sanitizer { 21 22 struct StackDepotNode { 23 using hash_type = u64; 24 hash_type stack_hash; 25 u32 link; 26 StackStore::Id store_id; 27 28 static const u32 kTabSizeLog = SANITIZER_ANDROID ? 16 : 20; 29 30 typedef StackTrace args_type; 31 bool eq(hash_type hash, const args_type &args) const { 32 return hash == stack_hash; 33 } 34 static uptr allocated(); 35 static hash_type hash(const args_type &args) { 36 MurMur2Hash64Builder H(args.size * sizeof(uptr)); 37 for (uptr i = 0; i < args.size; i++) H.add(args.trace[i]); 38 H.add(args.tag); 39 return H.get(); 40 } 41 static bool is_valid(const args_type &args) { 42 return args.size > 0 && args.trace; 43 } 44 void store(u32 id, const args_type &args, hash_type hash); 45 args_type load(u32 id) const; 46 static StackDepotHandle get_handle(u32 id); 47 48 typedef StackDepotHandle handle_type; 49 }; 50 51 static StackStore stackStore; 52 53 // FIXME(dvyukov): this single reserved bit is used in TSan. 54 typedef StackDepotBase<StackDepotNode, 1, StackDepotNode::kTabSizeLog> 55 StackDepot; 56 static StackDepot theDepot; 57 // Keep mutable data out of frequently access nodes to improve caching 58 // efficiency. 59 static TwoLevelMap<atomic_uint32_t, StackDepot::kNodesSize1, 60 StackDepot::kNodesSize2> 61 useCounts; 62 63 int StackDepotHandle::use_count() const { 64 return atomic_load_relaxed(&useCounts[id_]); 65 } 66 67 void StackDepotHandle::inc_use_count_unsafe() { 68 atomic_fetch_add(&useCounts[id_], 1, memory_order_relaxed); 69 } 70 71 uptr StackDepotNode::allocated() { 72 return stackStore.Allocated() + useCounts.MemoryUsage(); 73 } 74 75 static void CompressStackStore() { 76 u64 start = MonotonicNanoTime(); 77 uptr diff = stackStore.Pack(static_cast<StackStore::Compression>( 78 common_flags()->compress_stack_depot)); 79 if (!diff) 80 return; 81 u64 finish = MonotonicNanoTime(); 82 uptr total_before = theDepot.GetStats().allocated + diff; 83 VPrintf(1, "%s: StackDepot released %zu KiB out of %zu KiB in %llu ms\n", 84 SanitizerToolName, diff >> 10, total_before >> 10, 85 (finish - start) / 1000000); 86 } 87 88 void StackDepotNode::store(u32 id, const args_type &args, hash_type hash) { 89 stack_hash = hash; 90 uptr pack = 0; 91 store_id = stackStore.Store(args, &pack); 92 if (pack && common_flags()->compress_stack_depot) 93 CompressStackStore(); 94 } 95 96 StackDepotNode::args_type StackDepotNode::load(u32 id) const { 97 if (!store_id) 98 return {}; 99 return stackStore.Load(store_id); 100 } 101 102 StackDepotStats StackDepotGetStats() { return theDepot.GetStats(); } 103 104 u32 StackDepotPut(StackTrace stack) { return theDepot.Put(stack); } 105 106 StackDepotHandle StackDepotPut_WithHandle(StackTrace stack) { 107 return StackDepotNode::get_handle(theDepot.Put(stack)); 108 } 109 110 StackTrace StackDepotGet(u32 id) { 111 return theDepot.Get(id); 112 } 113 114 void StackDepotLockAll() { 115 theDepot.LockAll(); 116 stackStore.LockAll(); 117 } 118 119 void StackDepotUnlockAll() { 120 stackStore.UnlockAll(); 121 theDepot.UnlockAll(); 122 } 123 124 void StackDepotPrintAll() { 125 #if !SANITIZER_GO 126 theDepot.PrintAll(); 127 #endif 128 } 129 130 StackDepotHandle StackDepotNode::get_handle(u32 id) { 131 return StackDepotHandle(&theDepot.nodes[id], id); 132 } 133 134 void StackDepotTestOnlyUnmap() { 135 theDepot.TestOnlyUnmap(); 136 stackStore.TestOnlyUnmap(); 137 } 138 139 } // namespace __sanitizer 140