10b57cec5SDimitry Andric //===-- hwasan.cpp --------------------------------------------------------===// 20b57cec5SDimitry Andric // 30b57cec5SDimitry Andric // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 40b57cec5SDimitry Andric // See https://llvm.org/LICENSE.txt for license information. 50b57cec5SDimitry Andric // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 60b57cec5SDimitry Andric // 70b57cec5SDimitry Andric //===----------------------------------------------------------------------===// 80b57cec5SDimitry Andric // 90b57cec5SDimitry Andric // This file is a part of HWAddressSanitizer. 100b57cec5SDimitry Andric // 110b57cec5SDimitry Andric // HWAddressSanitizer runtime. 120b57cec5SDimitry Andric //===----------------------------------------------------------------------===// 130b57cec5SDimitry Andric 140b57cec5SDimitry Andric #include "hwasan.h" 155ffd83dbSDimitry Andric 160b57cec5SDimitry Andric #include "hwasan_checks.h" 170b57cec5SDimitry Andric #include "hwasan_dynamic_shadow.h" 185ffd83dbSDimitry Andric #include "hwasan_globals.h" 19349cc55cSDimitry Andric #include "hwasan_mapping.h" 200b57cec5SDimitry Andric #include "hwasan_poisoning.h" 210b57cec5SDimitry Andric #include "hwasan_report.h" 220b57cec5SDimitry Andric #include "hwasan_thread.h" 230b57cec5SDimitry Andric #include "hwasan_thread_list.h" 240b57cec5SDimitry Andric #include "sanitizer_common/sanitizer_atomic.h" 250b57cec5SDimitry Andric #include "sanitizer_common/sanitizer_common.h" 260b57cec5SDimitry Andric #include "sanitizer_common/sanitizer_flag_parser.h" 270b57cec5SDimitry Andric #include "sanitizer_common/sanitizer_flags.h" 2881ad6265SDimitry Andric #include "sanitizer_common/sanitizer_interface_internal.h" 290b57cec5SDimitry Andric #include "sanitizer_common/sanitizer_libc.h" 300b57cec5SDimitry Andric #include "sanitizer_common/sanitizer_procmaps.h" 310b57cec5SDimitry Andric #include "sanitizer_common/sanitizer_stackdepot.h" 320b57cec5SDimitry Andric #include "sanitizer_common/sanitizer_stacktrace.h" 330b57cec5SDimitry Andric #include "sanitizer_common/sanitizer_symbolizer.h" 340b57cec5SDimitry Andric #include "ubsan/ubsan_flags.h" 350b57cec5SDimitry Andric #include "ubsan/ubsan_init.h" 360b57cec5SDimitry Andric 370b57cec5SDimitry Andric // ACHTUNG! No system header includes in this file. 380b57cec5SDimitry Andric 390b57cec5SDimitry Andric using namespace __sanitizer; 400b57cec5SDimitry Andric 410b57cec5SDimitry Andric namespace __hwasan { 420b57cec5SDimitry Andric 430b57cec5SDimitry Andric static Flags hwasan_flags; 440b57cec5SDimitry Andric 450b57cec5SDimitry Andric Flags *flags() { 460b57cec5SDimitry Andric return &hwasan_flags; 470b57cec5SDimitry Andric } 480b57cec5SDimitry Andric 490b57cec5SDimitry Andric int hwasan_inited = 0; 500b57cec5SDimitry Andric int hwasan_instrumentation_inited = 0; 510b57cec5SDimitry Andric bool hwasan_init_is_running; 520b57cec5SDimitry Andric 530b57cec5SDimitry Andric int hwasan_report_count = 0; 540b57cec5SDimitry Andric 55fe6060f1SDimitry Andric uptr kLowShadowStart; 56fe6060f1SDimitry Andric uptr kLowShadowEnd; 57fe6060f1SDimitry Andric uptr kHighShadowStart; 58fe6060f1SDimitry Andric uptr kHighShadowEnd; 59fe6060f1SDimitry Andric 600b57cec5SDimitry Andric void Flags::SetDefaults() { 610b57cec5SDimitry Andric #define HWASAN_FLAG(Type, Name, DefaultValue, Description) Name = DefaultValue; 620b57cec5SDimitry Andric #include "hwasan_flags.inc" 630b57cec5SDimitry Andric #undef HWASAN_FLAG 640b57cec5SDimitry Andric } 650b57cec5SDimitry Andric 660b57cec5SDimitry Andric static void RegisterHwasanFlags(FlagParser *parser, Flags *f) { 670b57cec5SDimitry Andric #define HWASAN_FLAG(Type, Name, DefaultValue, Description) \ 680b57cec5SDimitry Andric RegisterFlag(parser, #Name, Description, &f->Name); 690b57cec5SDimitry Andric #include "hwasan_flags.inc" 700b57cec5SDimitry Andric #undef HWASAN_FLAG 710b57cec5SDimitry Andric } 720b57cec5SDimitry Andric 730b57cec5SDimitry Andric static void InitializeFlags() { 740b57cec5SDimitry Andric SetCommonFlagsDefaults(); 750b57cec5SDimitry Andric { 760b57cec5SDimitry Andric CommonFlags cf; 770b57cec5SDimitry Andric cf.CopyFrom(*common_flags()); 780b57cec5SDimitry Andric cf.external_symbolizer_path = GetEnv("HWASAN_SYMBOLIZER_PATH"); 790b57cec5SDimitry Andric cf.malloc_context_size = 20; 800b57cec5SDimitry Andric cf.handle_ioctl = true; 810b57cec5SDimitry Andric // FIXME: test and enable. 820b57cec5SDimitry Andric cf.check_printf = false; 830b57cec5SDimitry Andric cf.intercept_tls_get_addr = true; 840b57cec5SDimitry Andric cf.exitcode = 99; 850b57cec5SDimitry Andric // 8 shadow pages ~512kB, small enough to cover common stack sizes. 860b57cec5SDimitry Andric cf.clear_shadow_mmap_threshold = 4096 * (SANITIZER_ANDROID ? 2 : 8); 870b57cec5SDimitry Andric // Sigtrap is used in error reporting. 880b57cec5SDimitry Andric cf.handle_sigtrap = kHandleSignalExclusive; 895f757f3fSDimitry Andric // For now only tested on Linux and Fuchsia. Other plantforms can be turned 905f757f3fSDimitry Andric // on as they become ready. 915f757f3fSDimitry Andric constexpr bool can_detect_leaks = 925f757f3fSDimitry Andric (SANITIZER_LINUX && !SANITIZER_ANDROID) || SANITIZER_FUCHSIA; 935f757f3fSDimitry Andric cf.detect_leaks = cf.detect_leaks && can_detect_leaks; 940b57cec5SDimitry Andric 950b57cec5SDimitry Andric #if SANITIZER_ANDROID 960b57cec5SDimitry Andric // Let platform handle other signals. It is better at reporting them then we 970b57cec5SDimitry Andric // are. 980b57cec5SDimitry Andric cf.handle_segv = kHandleSignalNo; 990b57cec5SDimitry Andric cf.handle_sigbus = kHandleSignalNo; 1000b57cec5SDimitry Andric cf.handle_abort = kHandleSignalNo; 1010b57cec5SDimitry Andric cf.handle_sigill = kHandleSignalNo; 1020b57cec5SDimitry Andric cf.handle_sigfpe = kHandleSignalNo; 1030b57cec5SDimitry Andric #endif 1040b57cec5SDimitry Andric OverrideCommonFlags(cf); 1050b57cec5SDimitry Andric } 1060b57cec5SDimitry Andric 1070b57cec5SDimitry Andric Flags *f = flags(); 1080b57cec5SDimitry Andric f->SetDefaults(); 1090b57cec5SDimitry Andric 1100b57cec5SDimitry Andric FlagParser parser; 1110b57cec5SDimitry Andric RegisterHwasanFlags(&parser, f); 1120b57cec5SDimitry Andric RegisterCommonFlags(&parser); 1130b57cec5SDimitry Andric 114bdd1243dSDimitry Andric #if CAN_SANITIZE_LEAKS 115bdd1243dSDimitry Andric __lsan::Flags *lf = __lsan::flags(); 116bdd1243dSDimitry Andric lf->SetDefaults(); 117bdd1243dSDimitry Andric 118bdd1243dSDimitry Andric FlagParser lsan_parser; 119bdd1243dSDimitry Andric __lsan::RegisterLsanFlags(&lsan_parser, lf); 120bdd1243dSDimitry Andric RegisterCommonFlags(&lsan_parser); 121bdd1243dSDimitry Andric #endif 122bdd1243dSDimitry Andric 1230b57cec5SDimitry Andric #if HWASAN_CONTAINS_UBSAN 1240b57cec5SDimitry Andric __ubsan::Flags *uf = __ubsan::flags(); 1250b57cec5SDimitry Andric uf->SetDefaults(); 1260b57cec5SDimitry Andric 1270b57cec5SDimitry Andric FlagParser ubsan_parser; 1280b57cec5SDimitry Andric __ubsan::RegisterUbsanFlags(&ubsan_parser, uf); 1290b57cec5SDimitry Andric RegisterCommonFlags(&ubsan_parser); 1300b57cec5SDimitry Andric #endif 1310b57cec5SDimitry Andric 1320b57cec5SDimitry Andric // Override from user-specified string. 1330b57cec5SDimitry Andric if (__hwasan_default_options) 1340b57cec5SDimitry Andric parser.ParseString(__hwasan_default_options()); 13506c3fb27SDimitry Andric #if CAN_SANITIZE_LEAKS 13606c3fb27SDimitry Andric lsan_parser.ParseString(__lsan_default_options()); 13706c3fb27SDimitry Andric #endif 1380b57cec5SDimitry Andric #if HWASAN_CONTAINS_UBSAN 139e8d8bef9SDimitry Andric const char *ubsan_default_options = __ubsan_default_options(); 1400b57cec5SDimitry Andric ubsan_parser.ParseString(ubsan_default_options); 1410b57cec5SDimitry Andric #endif 1420b57cec5SDimitry Andric 1430b57cec5SDimitry Andric parser.ParseStringFromEnv("HWASAN_OPTIONS"); 144bdd1243dSDimitry Andric #if CAN_SANITIZE_LEAKS 145bdd1243dSDimitry Andric lsan_parser.ParseStringFromEnv("LSAN_OPTIONS"); 146bdd1243dSDimitry Andric #endif 1470b57cec5SDimitry Andric #if HWASAN_CONTAINS_UBSAN 1480b57cec5SDimitry Andric ubsan_parser.ParseStringFromEnv("UBSAN_OPTIONS"); 1490b57cec5SDimitry Andric #endif 1500b57cec5SDimitry Andric 1510b57cec5SDimitry Andric InitializeCommonFlags(); 1520b57cec5SDimitry Andric 1530b57cec5SDimitry Andric if (Verbosity()) ReportUnrecognizedFlags(); 1540b57cec5SDimitry Andric 1550b57cec5SDimitry Andric if (common_flags()->help) parser.PrintFlagDescriptions(); 156bdd1243dSDimitry Andric // Flag validation: 157bdd1243dSDimitry Andric if (!CAN_SANITIZE_LEAKS && common_flags()->detect_leaks) { 158bdd1243dSDimitry Andric Report("%s: detect_leaks is not supported on this platform.\n", 159bdd1243dSDimitry Andric SanitizerToolName); 160bdd1243dSDimitry Andric Die(); 161bdd1243dSDimitry Andric } 1620b57cec5SDimitry Andric } 1630b57cec5SDimitry Andric 164fe6060f1SDimitry Andric static void CheckUnwind() { 165fe6060f1SDimitry Andric GET_FATAL_STACK_TRACE_PC_BP(StackTrace::GetCurrentPc(), GET_CURRENT_FRAME()); 166fe6060f1SDimitry Andric stack.Print(); 1670b57cec5SDimitry Andric } 1680b57cec5SDimitry Andric 1690b57cec5SDimitry Andric static void HwasanFormatMemoryUsage(InternalScopedString &s) { 1700b57cec5SDimitry Andric HwasanThreadList &thread_list = hwasanThreadList(); 1710b57cec5SDimitry Andric auto thread_stats = thread_list.GetThreadStats(); 172349cc55cSDimitry Andric auto sds = StackDepotGetStats(); 1730b57cec5SDimitry Andric AllocatorStatCounters asc; 1740b57cec5SDimitry Andric GetAllocatorStats(asc); 1755f757f3fSDimitry Andric s.AppendF( 1760b57cec5SDimitry Andric "HWASAN pid: %d rss: %zd threads: %zd stacks: %zd" 1770b57cec5SDimitry Andric " thr_aux: %zd stack_depot: %zd uniq_stacks: %zd" 1780b57cec5SDimitry Andric " heap: %zd", 1790b57cec5SDimitry Andric internal_getpid(), GetRSS(), thread_stats.n_live_threads, 1800b57cec5SDimitry Andric thread_stats.total_stack_size, 1810b57cec5SDimitry Andric thread_stats.n_live_threads * thread_list.MemoryUsedPerThread(), 182349cc55cSDimitry Andric sds.allocated, sds.n_uniq_ids, asc[AllocatorStatMapped]); 1830b57cec5SDimitry Andric } 1840b57cec5SDimitry Andric 1850b57cec5SDimitry Andric #if SANITIZER_ANDROID 186fe6060f1SDimitry Andric static constexpr uptr kMemoryUsageBufferSize = 4096; 187fe6060f1SDimitry Andric 1880b57cec5SDimitry Andric static char *memory_usage_buffer = nullptr; 1890b57cec5SDimitry Andric 1900b57cec5SDimitry Andric static void InitMemoryUsage() { 1910b57cec5SDimitry Andric memory_usage_buffer = 1920b57cec5SDimitry Andric (char *)MmapOrDie(kMemoryUsageBufferSize, "memory usage string"); 1930b57cec5SDimitry Andric CHECK(memory_usage_buffer); 1940b57cec5SDimitry Andric memory_usage_buffer[0] = '\0'; 1950b57cec5SDimitry Andric DecorateMapping((uptr)memory_usage_buffer, kMemoryUsageBufferSize, 1960b57cec5SDimitry Andric memory_usage_buffer); 1970b57cec5SDimitry Andric } 1980b57cec5SDimitry Andric 1990b57cec5SDimitry Andric void UpdateMemoryUsage() { 2000b57cec5SDimitry Andric if (!flags()->export_memory_stats) 2010b57cec5SDimitry Andric return; 2020b57cec5SDimitry Andric if (!memory_usage_buffer) 2030b57cec5SDimitry Andric InitMemoryUsage(); 204fe6060f1SDimitry Andric InternalScopedString s; 2050b57cec5SDimitry Andric HwasanFormatMemoryUsage(s); 2060b57cec5SDimitry Andric internal_strncpy(memory_usage_buffer, s.data(), kMemoryUsageBufferSize - 1); 2070b57cec5SDimitry Andric memory_usage_buffer[kMemoryUsageBufferSize - 1] = '\0'; 2080b57cec5SDimitry Andric } 2090b57cec5SDimitry Andric #else 2100b57cec5SDimitry Andric void UpdateMemoryUsage() {} 2110b57cec5SDimitry Andric #endif 2120b57cec5SDimitry Andric 213fe6060f1SDimitry Andric void HwasanAtExit() { 214fe6060f1SDimitry Andric if (common_flags()->print_module_map) 215fe6060f1SDimitry Andric DumpProcessMap(); 216fe6060f1SDimitry Andric if (flags()->print_stats && (flags()->atexit || hwasan_report_count > 0)) 217fe6060f1SDimitry Andric ReportStats(); 218fe6060f1SDimitry Andric if (hwasan_report_count > 0) { 219fe6060f1SDimitry Andric // ReportAtExitStatistics(); 220fe6060f1SDimitry Andric if (common_flags()->exitcode) 221fe6060f1SDimitry Andric internal__exit(common_flags()->exitcode); 222fe6060f1SDimitry Andric } 223fe6060f1SDimitry Andric } 224fe6060f1SDimitry Andric 225fe6060f1SDimitry Andric void HandleTagMismatch(AccessInfo ai, uptr pc, uptr frame, void *uc, 226fe6060f1SDimitry Andric uptr *registers_frame) { 227fe6060f1SDimitry Andric InternalMmapVector<BufferedStackTrace> stack_buffer(1); 228fe6060f1SDimitry Andric BufferedStackTrace *stack = stack_buffer.data(); 229fe6060f1SDimitry Andric stack->Reset(); 230fe6060f1SDimitry Andric stack->Unwind(pc, frame, uc, common_flags()->fast_unwind_on_fatal); 231fe6060f1SDimitry Andric 232fe6060f1SDimitry Andric // The second stack frame contains the failure __hwasan_check function, as 233fe6060f1SDimitry Andric // we have a stack frame for the registers saved in __hwasan_tag_mismatch that 234fe6060f1SDimitry Andric // we wish to ignore. This (currently) only occurs on AArch64, as x64 235fe6060f1SDimitry Andric // implementations use SIGTRAP to implement the failure, and thus do not go 236fe6060f1SDimitry Andric // through the stack saver. 237fe6060f1SDimitry Andric if (registers_frame && stack->trace && stack->size > 0) { 238fe6060f1SDimitry Andric stack->trace++; 239fe6060f1SDimitry Andric stack->size--; 240fe6060f1SDimitry Andric } 241fe6060f1SDimitry Andric 242fe6060f1SDimitry Andric bool fatal = flags()->halt_on_error || !ai.recover; 243fe6060f1SDimitry Andric ReportTagMismatch(stack, ai.addr, ai.size, ai.is_store, fatal, 244fe6060f1SDimitry Andric registers_frame); 245fe6060f1SDimitry Andric } 246fe6060f1SDimitry Andric 247bdd1243dSDimitry Andric void HwasanTagMismatch(uptr addr, uptr pc, uptr frame, uptr access_info, 248bdd1243dSDimitry Andric uptr *registers_frame, size_t outsize) { 249fe6060f1SDimitry Andric __hwasan::AccessInfo ai; 250fe6060f1SDimitry Andric ai.is_store = access_info & 0x10; 251fe6060f1SDimitry Andric ai.is_load = !ai.is_store; 252fe6060f1SDimitry Andric ai.recover = access_info & 0x20; 253fe6060f1SDimitry Andric ai.addr = addr; 254fe6060f1SDimitry Andric if ((access_info & 0xf) == 0xf) 255fe6060f1SDimitry Andric ai.size = outsize; 256fe6060f1SDimitry Andric else 257fe6060f1SDimitry Andric ai.size = 1 << (access_info & 0xf); 258fe6060f1SDimitry Andric 259bdd1243dSDimitry Andric HandleTagMismatch(ai, pc, frame, nullptr, registers_frame); 260fe6060f1SDimitry Andric } 261fe6060f1SDimitry Andric 262fe6060f1SDimitry Andric Thread *GetCurrentThread() { 263fe6060f1SDimitry Andric uptr *ThreadLongPtr = GetCurrentThreadLongPtr(); 264fe6060f1SDimitry Andric if (UNLIKELY(*ThreadLongPtr == 0)) 265fe6060f1SDimitry Andric return nullptr; 266fe6060f1SDimitry Andric auto *R = (StackAllocationsRingBuffer *)ThreadLongPtr; 267fe6060f1SDimitry Andric return hwasanThreadList().GetThreadByBufferAddress((uptr)R->Next()); 268fe6060f1SDimitry Andric } 269fe6060f1SDimitry Andric 2700b57cec5SDimitry Andric } // namespace __hwasan 2710b57cec5SDimitry Andric 27268d75effSDimitry Andric using namespace __hwasan; 27368d75effSDimitry Andric 2740b57cec5SDimitry Andric void __sanitizer::BufferedStackTrace::UnwindImpl( 2750b57cec5SDimitry Andric uptr pc, uptr bp, void *context, bool request_fast, u32 max_depth) { 2760b57cec5SDimitry Andric Thread *t = GetCurrentThread(); 2770b57cec5SDimitry Andric if (!t) { 2785ffd83dbSDimitry Andric // The thread is still being created, or has already been destroyed. 2790b57cec5SDimitry Andric size = 0; 2800b57cec5SDimitry Andric return; 2810b57cec5SDimitry Andric } 2825ffd83dbSDimitry Andric Unwind(max_depth, pc, bp, context, t->stack_top(), t->stack_bottom(), 2835ffd83dbSDimitry Andric request_fast); 2840b57cec5SDimitry Andric } 2850b57cec5SDimitry Andric 2865ffd83dbSDimitry Andric static bool InitializeSingleGlobal(const hwasan_global &global) { 2875ffd83dbSDimitry Andric uptr full_granule_size = RoundDownTo(global.size(), 16); 2885ffd83dbSDimitry Andric TagMemoryAligned(global.addr(), full_granule_size, global.tag()); 2895ffd83dbSDimitry Andric if (global.size() % 16) 2905ffd83dbSDimitry Andric TagMemoryAligned(global.addr() + full_granule_size, 16, global.size() % 16); 2915ffd83dbSDimitry Andric return false; 29268d75effSDimitry Andric } 29368d75effSDimitry Andric 29468d75effSDimitry Andric static void InitLoadedGlobals() { 29506c3fb27SDimitry Andric // Fuchsia's libc provides a hook (__sanitizer_module_loaded) that runs on 29606c3fb27SDimitry Andric // the startup path which calls into __hwasan_library_loaded on all 29706c3fb27SDimitry Andric // initially loaded modules, so explicitly registering the globals here 29806c3fb27SDimitry Andric // isn't needed. 29906c3fb27SDimitry Andric if constexpr (!SANITIZER_FUCHSIA) { 30068d75effSDimitry Andric dl_iterate_phdr( 3015ffd83dbSDimitry Andric [](dl_phdr_info *info, size_t /* size */, void * /* data */) -> int { 3025ffd83dbSDimitry Andric for (const hwasan_global &global : HwasanGlobalsFor( 3035ffd83dbSDimitry Andric info->dlpi_addr, info->dlpi_phdr, info->dlpi_phnum)) 3045ffd83dbSDimitry Andric InitializeSingleGlobal(global); 30568d75effSDimitry Andric return 0; 30668d75effSDimitry Andric }, 30768d75effSDimitry Andric nullptr); 30868d75effSDimitry Andric } 30906c3fb27SDimitry Andric } 31068d75effSDimitry Andric 31168d75effSDimitry Andric // Prepare to run instrumented code on the main thread. 31268d75effSDimitry Andric static void InitInstrumentation() { 31368d75effSDimitry Andric if (hwasan_instrumentation_inited) return; 31468d75effSDimitry Andric 315fe6060f1SDimitry Andric InitializeOsSupport(); 31668d75effSDimitry Andric 31768d75effSDimitry Andric if (!InitShadow()) { 31868d75effSDimitry Andric Printf("FATAL: HWAddressSanitizer cannot mmap the shadow memory.\n"); 31968d75effSDimitry Andric DumpProcessMap(); 32068d75effSDimitry Andric Die(); 32168d75effSDimitry Andric } 32268d75effSDimitry Andric 32368d75effSDimitry Andric InitThreads(); 32468d75effSDimitry Andric 32568d75effSDimitry Andric hwasan_instrumentation_inited = 1; 32668d75effSDimitry Andric } 32768d75effSDimitry Andric 32868d75effSDimitry Andric // Interface. 3290b57cec5SDimitry Andric 3300b57cec5SDimitry Andric uptr __hwasan_shadow_memory_dynamic_address; // Global interface symbol. 3310b57cec5SDimitry Andric 3320b57cec5SDimitry Andric // This function was used by the old frame descriptor mechanism. We keep it 3330b57cec5SDimitry Andric // around to avoid breaking ABI. 3340b57cec5SDimitry Andric void __hwasan_init_frames(uptr beg, uptr end) {} 3350b57cec5SDimitry Andric 3360b57cec5SDimitry Andric void __hwasan_init_static() { 3370b57cec5SDimitry Andric InitShadowGOT(); 3380b57cec5SDimitry Andric InitInstrumentation(); 33968d75effSDimitry Andric 34068d75effSDimitry Andric // In the non-static code path we call dl_iterate_phdr here. But at this point 34168d75effSDimitry Andric // libc might not have been initialized enough for dl_iterate_phdr to work. 34268d75effSDimitry Andric // Fortunately, since this is a statically linked executable we can use the 34368d75effSDimitry Andric // linker-defined symbol __ehdr_start to find the only relevant set of phdrs. 34468d75effSDimitry Andric extern ElfW(Ehdr) __ehdr_start; 3455ffd83dbSDimitry Andric for (const hwasan_global &global : HwasanGlobalsFor( 3465ffd83dbSDimitry Andric /* base */ 0, 34768d75effSDimitry Andric reinterpret_cast<const ElfW(Phdr) *>( 3485ffd83dbSDimitry Andric reinterpret_cast<const char *>(&__ehdr_start) + 3495ffd83dbSDimitry Andric __ehdr_start.e_phoff), 3505ffd83dbSDimitry Andric __ehdr_start.e_phnum)) 3515ffd83dbSDimitry Andric InitializeSingleGlobal(global); 3520b57cec5SDimitry Andric } 3530b57cec5SDimitry Andric 354349cc55cSDimitry Andric __attribute__((constructor(0))) void __hwasan_init() { 3550b57cec5SDimitry Andric CHECK(!hwasan_init_is_running); 3560b57cec5SDimitry Andric if (hwasan_inited) return; 3570b57cec5SDimitry Andric hwasan_init_is_running = 1; 3580b57cec5SDimitry Andric SanitizerToolName = "HWAddressSanitizer"; 3590b57cec5SDimitry Andric 3600b57cec5SDimitry Andric InitTlsSize(); 3610b57cec5SDimitry Andric 3620b57cec5SDimitry Andric CacheBinaryName(); 3630b57cec5SDimitry Andric InitializeFlags(); 3640b57cec5SDimitry Andric 3650b57cec5SDimitry Andric // Install tool-specific callbacks in sanitizer_common. 366fe6060f1SDimitry Andric SetCheckUnwindCallback(CheckUnwind); 3670b57cec5SDimitry Andric 3680b57cec5SDimitry Andric __sanitizer_set_report_path(common_flags()->log_path); 3690b57cec5SDimitry Andric 3700b57cec5SDimitry Andric AndroidTestTlsSlot(); 3710b57cec5SDimitry Andric 3720b57cec5SDimitry Andric DisableCoreDumperIfNecessary(); 3730b57cec5SDimitry Andric 3740b57cec5SDimitry Andric InitInstrumentation(); 37568d75effSDimitry Andric InitLoadedGlobals(); 3760b57cec5SDimitry Andric 3770b57cec5SDimitry Andric // Needs to be called here because flags()->random_tags might not have been 3780b57cec5SDimitry Andric // initialized when InitInstrumentation() was called. 379349cc55cSDimitry Andric GetCurrentThread()->EnsureRandomStateInited(); 3800b57cec5SDimitry Andric 3810b57cec5SDimitry Andric SetPrintfAndReportCallback(AppendToErrorMessageBuffer); 3820b57cec5SDimitry Andric // This may call libc -> needs initialized shadow. 3830b57cec5SDimitry Andric AndroidLogInit(); 3840b57cec5SDimitry Andric 3850b57cec5SDimitry Andric InitializeInterceptors(); 3860b57cec5SDimitry Andric InstallDeadlySignalHandlers(HwasanOnDeadlySignal); 3870b57cec5SDimitry Andric InstallAtExitHandler(); // Needs __cxa_atexit interceptor. 3880b57cec5SDimitry Andric 3890b57cec5SDimitry Andric InitializeCoverage(common_flags()->coverage, common_flags()->coverage_dir); 3900b57cec5SDimitry Andric 3910b57cec5SDimitry Andric HwasanTSDInit(); 3920b57cec5SDimitry Andric HwasanTSDThreadInit(); 3930b57cec5SDimitry Andric 3940b57cec5SDimitry Andric HwasanAllocatorInit(); 395349cc55cSDimitry Andric HwasanInstallAtForkHandler(); 3960b57cec5SDimitry Andric 397bdd1243dSDimitry Andric if (CAN_SANITIZE_LEAKS) { 398bdd1243dSDimitry Andric __lsan::InitCommonLsan(); 399bdd1243dSDimitry Andric InstallAtExitCheckLeaks(); 400bdd1243dSDimitry Andric } 401bdd1243dSDimitry Andric 4020b57cec5SDimitry Andric #if HWASAN_CONTAINS_UBSAN 4030b57cec5SDimitry Andric __ubsan::InitAsPlugin(); 4040b57cec5SDimitry Andric #endif 4050b57cec5SDimitry Andric 40606c3fb27SDimitry Andric if (CAN_SANITIZE_LEAKS && common_flags()->detect_leaks) { 407bdd1243dSDimitry Andric __lsan::ScopedInterceptorDisabler disabler; 408bdd1243dSDimitry Andric Symbolizer::LateInitialize(); 409bdd1243dSDimitry Andric } 410bdd1243dSDimitry Andric 4110b57cec5SDimitry Andric VPrintf(1, "HWAddressSanitizer init done\n"); 4120b57cec5SDimitry Andric 4130b57cec5SDimitry Andric hwasan_init_is_running = 0; 4140b57cec5SDimitry Andric hwasan_inited = 1; 4150b57cec5SDimitry Andric } 4160b57cec5SDimitry Andric 41768d75effSDimitry Andric void __hwasan_library_loaded(ElfW(Addr) base, const ElfW(Phdr) * phdr, 41868d75effSDimitry Andric ElfW(Half) phnum) { 4195ffd83dbSDimitry Andric for (const hwasan_global &global : HwasanGlobalsFor(base, phdr, phnum)) 4205ffd83dbSDimitry Andric InitializeSingleGlobal(global); 42168d75effSDimitry Andric } 42268d75effSDimitry Andric 42368d75effSDimitry Andric void __hwasan_library_unloaded(ElfW(Addr) base, const ElfW(Phdr) * phdr, 42468d75effSDimitry Andric ElfW(Half) phnum) { 42568d75effSDimitry Andric for (; phnum != 0; ++phdr, --phnum) 42668d75effSDimitry Andric if (phdr->p_type == PT_LOAD) 42768d75effSDimitry Andric TagMemory(base + phdr->p_vaddr, phdr->p_memsz, 0); 42868d75effSDimitry Andric } 42968d75effSDimitry Andric 4300b57cec5SDimitry Andric void __hwasan_print_shadow(const void *p, uptr sz) { 4310b57cec5SDimitry Andric uptr ptr_raw = UntagAddr(reinterpret_cast<uptr>(p)); 4320b57cec5SDimitry Andric uptr shadow_first = MemToShadow(ptr_raw); 4330b57cec5SDimitry Andric uptr shadow_last = MemToShadow(ptr_raw + sz - 1); 4340b57cec5SDimitry Andric Printf("HWASan shadow map for %zx .. %zx (pointer tag %x)\n", ptr_raw, 4350b57cec5SDimitry Andric ptr_raw + sz, GetTagFromPointer((uptr)p)); 436349cc55cSDimitry Andric for (uptr s = shadow_first; s <= shadow_last; ++s) { 437349cc55cSDimitry Andric tag_t mem_tag = *reinterpret_cast<tag_t *>(s); 438349cc55cSDimitry Andric uptr granule_addr = ShadowToMem(s); 439349cc55cSDimitry Andric if (mem_tag && mem_tag < kShadowAlignment) 440349cc55cSDimitry Andric Printf(" %zx: %02x(%02x)\n", granule_addr, mem_tag, 441349cc55cSDimitry Andric *reinterpret_cast<tag_t *>(granule_addr + kShadowAlignment - 1)); 442349cc55cSDimitry Andric else 443349cc55cSDimitry Andric Printf(" %zx: %02x\n", granule_addr, mem_tag); 444349cc55cSDimitry Andric } 4450b57cec5SDimitry Andric } 4460b57cec5SDimitry Andric 4470b57cec5SDimitry Andric sptr __hwasan_test_shadow(const void *p, uptr sz) { 4480b57cec5SDimitry Andric if (sz == 0) 4490b57cec5SDimitry Andric return -1; 45006c3fb27SDimitry Andric uptr ptr = reinterpret_cast<uptr>(p); 45106c3fb27SDimitry Andric tag_t ptr_tag = GetTagFromPointer(ptr); 45206c3fb27SDimitry Andric uptr ptr_raw = UntagAddr(ptr); 4530b57cec5SDimitry Andric uptr shadow_first = MemToShadow(ptr_raw); 45406c3fb27SDimitry Andric uptr shadow_last = MemToShadow(ptr_raw + sz); 45506c3fb27SDimitry Andric for (uptr s = shadow_first; s < shadow_last; ++s) { 45606c3fb27SDimitry Andric if (UNLIKELY(*(tag_t *)s != ptr_tag)) { 45706c3fb27SDimitry Andric uptr short_size = 45806c3fb27SDimitry Andric ShortTagSize(*(tag_t *)s, AddTagToPointer(ShadowToMem(s), ptr_tag)); 45906c3fb27SDimitry Andric sptr offset = ShadowToMem(s) - ptr_raw + short_size; 4600b57cec5SDimitry Andric return offset < 0 ? 0 : offset; 4610b57cec5SDimitry Andric } 46206c3fb27SDimitry Andric } 46306c3fb27SDimitry Andric 46406c3fb27SDimitry Andric uptr end = ptr + sz; 46506c3fb27SDimitry Andric uptr tail_sz = end & (kShadowAlignment - 1); 46606c3fb27SDimitry Andric if (!tail_sz) 4670b57cec5SDimitry Andric return -1; 46806c3fb27SDimitry Andric 46906c3fb27SDimitry Andric uptr short_size = 47006c3fb27SDimitry Andric ShortTagSize(*(tag_t *)shadow_last, end & ~(kShadowAlignment - 1)); 47106c3fb27SDimitry Andric if (LIKELY(tail_sz <= short_size)) 47206c3fb27SDimitry Andric return -1; 47306c3fb27SDimitry Andric 47406c3fb27SDimitry Andric sptr offset = sz - tail_sz + short_size; 47506c3fb27SDimitry Andric return offset < 0 ? 0 : offset; 4760b57cec5SDimitry Andric } 4770b57cec5SDimitry Andric 4780b57cec5SDimitry Andric u16 __sanitizer_unaligned_load16(const uu16 *p) { 4790b57cec5SDimitry Andric return *p; 4800b57cec5SDimitry Andric } 4810b57cec5SDimitry Andric u32 __sanitizer_unaligned_load32(const uu32 *p) { 4820b57cec5SDimitry Andric return *p; 4830b57cec5SDimitry Andric } 4840b57cec5SDimitry Andric u64 __sanitizer_unaligned_load64(const uu64 *p) { 4850b57cec5SDimitry Andric return *p; 4860b57cec5SDimitry Andric } 4870b57cec5SDimitry Andric void __sanitizer_unaligned_store16(uu16 *p, u16 x) { 4880b57cec5SDimitry Andric *p = x; 4890b57cec5SDimitry Andric } 4900b57cec5SDimitry Andric void __sanitizer_unaligned_store32(uu32 *p, u32 x) { 4910b57cec5SDimitry Andric *p = x; 4920b57cec5SDimitry Andric } 4930b57cec5SDimitry Andric void __sanitizer_unaligned_store64(uu64 *p, u64 x) { 4940b57cec5SDimitry Andric *p = x; 4950b57cec5SDimitry Andric } 4960b57cec5SDimitry Andric 4970b57cec5SDimitry Andric void __hwasan_loadN(uptr p, uptr sz) { 4980b57cec5SDimitry Andric CheckAddressSized<ErrorAction::Abort, AccessType::Load>(p, sz); 4990b57cec5SDimitry Andric } 5000b57cec5SDimitry Andric void __hwasan_load1(uptr p) { 5010b57cec5SDimitry Andric CheckAddress<ErrorAction::Abort, AccessType::Load, 0>(p); 5020b57cec5SDimitry Andric } 5030b57cec5SDimitry Andric void __hwasan_load2(uptr p) { 5040b57cec5SDimitry Andric CheckAddress<ErrorAction::Abort, AccessType::Load, 1>(p); 5050b57cec5SDimitry Andric } 5060b57cec5SDimitry Andric void __hwasan_load4(uptr p) { 5070b57cec5SDimitry Andric CheckAddress<ErrorAction::Abort, AccessType::Load, 2>(p); 5080b57cec5SDimitry Andric } 5090b57cec5SDimitry Andric void __hwasan_load8(uptr p) { 5100b57cec5SDimitry Andric CheckAddress<ErrorAction::Abort, AccessType::Load, 3>(p); 5110b57cec5SDimitry Andric } 5120b57cec5SDimitry Andric void __hwasan_load16(uptr p) { 5130b57cec5SDimitry Andric CheckAddress<ErrorAction::Abort, AccessType::Load, 4>(p); 5140b57cec5SDimitry Andric } 5150b57cec5SDimitry Andric 5160b57cec5SDimitry Andric void __hwasan_loadN_noabort(uptr p, uptr sz) { 5170b57cec5SDimitry Andric CheckAddressSized<ErrorAction::Recover, AccessType::Load>(p, sz); 5180b57cec5SDimitry Andric } 5190b57cec5SDimitry Andric void __hwasan_load1_noabort(uptr p) { 5200b57cec5SDimitry Andric CheckAddress<ErrorAction::Recover, AccessType::Load, 0>(p); 5210b57cec5SDimitry Andric } 5220b57cec5SDimitry Andric void __hwasan_load2_noabort(uptr p) { 5230b57cec5SDimitry Andric CheckAddress<ErrorAction::Recover, AccessType::Load, 1>(p); 5240b57cec5SDimitry Andric } 5250b57cec5SDimitry Andric void __hwasan_load4_noabort(uptr p) { 5260b57cec5SDimitry Andric CheckAddress<ErrorAction::Recover, AccessType::Load, 2>(p); 5270b57cec5SDimitry Andric } 5280b57cec5SDimitry Andric void __hwasan_load8_noabort(uptr p) { 5290b57cec5SDimitry Andric CheckAddress<ErrorAction::Recover, AccessType::Load, 3>(p); 5300b57cec5SDimitry Andric } 5310b57cec5SDimitry Andric void __hwasan_load16_noabort(uptr p) { 5320b57cec5SDimitry Andric CheckAddress<ErrorAction::Recover, AccessType::Load, 4>(p); 5330b57cec5SDimitry Andric } 5340b57cec5SDimitry Andric 53506c3fb27SDimitry Andric void __hwasan_loadN_match_all(uptr p, uptr sz, u8 match_all_tag) { 53606c3fb27SDimitry Andric if (GetTagFromPointer(p) != match_all_tag) 53706c3fb27SDimitry Andric CheckAddressSized<ErrorAction::Abort, AccessType::Load>(p, sz); 53806c3fb27SDimitry Andric } 53906c3fb27SDimitry Andric void __hwasan_load1_match_all(uptr p, u8 match_all_tag) { 54006c3fb27SDimitry Andric if (GetTagFromPointer(p) != match_all_tag) 54106c3fb27SDimitry Andric CheckAddress<ErrorAction::Abort, AccessType::Load, 0>(p); 54206c3fb27SDimitry Andric } 54306c3fb27SDimitry Andric void __hwasan_load2_match_all(uptr p, u8 match_all_tag) { 54406c3fb27SDimitry Andric if (GetTagFromPointer(p) != match_all_tag) 54506c3fb27SDimitry Andric CheckAddress<ErrorAction::Abort, AccessType::Load, 1>(p); 54606c3fb27SDimitry Andric } 54706c3fb27SDimitry Andric void __hwasan_load4_match_all(uptr p, u8 match_all_tag) { 54806c3fb27SDimitry Andric if (GetTagFromPointer(p) != match_all_tag) 54906c3fb27SDimitry Andric CheckAddress<ErrorAction::Abort, AccessType::Load, 2>(p); 55006c3fb27SDimitry Andric } 55106c3fb27SDimitry Andric void __hwasan_load8_match_all(uptr p, u8 match_all_tag) { 55206c3fb27SDimitry Andric if (GetTagFromPointer(p) != match_all_tag) 55306c3fb27SDimitry Andric CheckAddress<ErrorAction::Abort, AccessType::Load, 3>(p); 55406c3fb27SDimitry Andric } 55506c3fb27SDimitry Andric void __hwasan_load16_match_all(uptr p, u8 match_all_tag) { 55606c3fb27SDimitry Andric if (GetTagFromPointer(p) != match_all_tag) 55706c3fb27SDimitry Andric CheckAddress<ErrorAction::Abort, AccessType::Load, 4>(p); 55806c3fb27SDimitry Andric } 55906c3fb27SDimitry Andric 56006c3fb27SDimitry Andric void __hwasan_loadN_match_all_noabort(uptr p, uptr sz, u8 match_all_tag) { 56106c3fb27SDimitry Andric if (GetTagFromPointer(p) != match_all_tag) 56206c3fb27SDimitry Andric CheckAddressSized<ErrorAction::Recover, AccessType::Load>(p, sz); 56306c3fb27SDimitry Andric } 56406c3fb27SDimitry Andric void __hwasan_load1_match_all_noabort(uptr p, u8 match_all_tag) { 56506c3fb27SDimitry Andric if (GetTagFromPointer(p) != match_all_tag) 56606c3fb27SDimitry Andric CheckAddress<ErrorAction::Recover, AccessType::Load, 0>(p); 56706c3fb27SDimitry Andric } 56806c3fb27SDimitry Andric void __hwasan_load2_match_all_noabort(uptr p, u8 match_all_tag) { 56906c3fb27SDimitry Andric if (GetTagFromPointer(p) != match_all_tag) 57006c3fb27SDimitry Andric CheckAddress<ErrorAction::Recover, AccessType::Load, 1>(p); 57106c3fb27SDimitry Andric } 57206c3fb27SDimitry Andric void __hwasan_load4_match_all_noabort(uptr p, u8 match_all_tag) { 57306c3fb27SDimitry Andric if (GetTagFromPointer(p) != match_all_tag) 57406c3fb27SDimitry Andric CheckAddress<ErrorAction::Recover, AccessType::Load, 2>(p); 57506c3fb27SDimitry Andric } 57606c3fb27SDimitry Andric void __hwasan_load8_match_all_noabort(uptr p, u8 match_all_tag) { 57706c3fb27SDimitry Andric if (GetTagFromPointer(p) != match_all_tag) 57806c3fb27SDimitry Andric CheckAddress<ErrorAction::Recover, AccessType::Load, 3>(p); 57906c3fb27SDimitry Andric } 58006c3fb27SDimitry Andric void __hwasan_load16_match_all_noabort(uptr p, u8 match_all_tag) { 58106c3fb27SDimitry Andric if (GetTagFromPointer(p) != match_all_tag) 58206c3fb27SDimitry Andric CheckAddress<ErrorAction::Recover, AccessType::Load, 4>(p); 58306c3fb27SDimitry Andric } 58406c3fb27SDimitry Andric 5850b57cec5SDimitry Andric void __hwasan_storeN(uptr p, uptr sz) { 5860b57cec5SDimitry Andric CheckAddressSized<ErrorAction::Abort, AccessType::Store>(p, sz); 5870b57cec5SDimitry Andric } 5880b57cec5SDimitry Andric void __hwasan_store1(uptr p) { 5890b57cec5SDimitry Andric CheckAddress<ErrorAction::Abort, AccessType::Store, 0>(p); 5900b57cec5SDimitry Andric } 5910b57cec5SDimitry Andric void __hwasan_store2(uptr p) { 5920b57cec5SDimitry Andric CheckAddress<ErrorAction::Abort, AccessType::Store, 1>(p); 5930b57cec5SDimitry Andric } 5940b57cec5SDimitry Andric void __hwasan_store4(uptr p) { 5950b57cec5SDimitry Andric CheckAddress<ErrorAction::Abort, AccessType::Store, 2>(p); 5960b57cec5SDimitry Andric } 5970b57cec5SDimitry Andric void __hwasan_store8(uptr p) { 5980b57cec5SDimitry Andric CheckAddress<ErrorAction::Abort, AccessType::Store, 3>(p); 5990b57cec5SDimitry Andric } 6000b57cec5SDimitry Andric void __hwasan_store16(uptr p) { 6010b57cec5SDimitry Andric CheckAddress<ErrorAction::Abort, AccessType::Store, 4>(p); 6020b57cec5SDimitry Andric } 6030b57cec5SDimitry Andric 6040b57cec5SDimitry Andric void __hwasan_storeN_noabort(uptr p, uptr sz) { 6050b57cec5SDimitry Andric CheckAddressSized<ErrorAction::Recover, AccessType::Store>(p, sz); 6060b57cec5SDimitry Andric } 6070b57cec5SDimitry Andric void __hwasan_store1_noabort(uptr p) { 6080b57cec5SDimitry Andric CheckAddress<ErrorAction::Recover, AccessType::Store, 0>(p); 6090b57cec5SDimitry Andric } 6100b57cec5SDimitry Andric void __hwasan_store2_noabort(uptr p) { 6110b57cec5SDimitry Andric CheckAddress<ErrorAction::Recover, AccessType::Store, 1>(p); 6120b57cec5SDimitry Andric } 6130b57cec5SDimitry Andric void __hwasan_store4_noabort(uptr p) { 6140b57cec5SDimitry Andric CheckAddress<ErrorAction::Recover, AccessType::Store, 2>(p); 6150b57cec5SDimitry Andric } 6160b57cec5SDimitry Andric void __hwasan_store8_noabort(uptr p) { 6170b57cec5SDimitry Andric CheckAddress<ErrorAction::Recover, AccessType::Store, 3>(p); 6180b57cec5SDimitry Andric } 6190b57cec5SDimitry Andric void __hwasan_store16_noabort(uptr p) { 6200b57cec5SDimitry Andric CheckAddress<ErrorAction::Recover, AccessType::Store, 4>(p); 6210b57cec5SDimitry Andric } 6220b57cec5SDimitry Andric 62306c3fb27SDimitry Andric void __hwasan_storeN_match_all(uptr p, uptr sz, u8 match_all_tag) { 62406c3fb27SDimitry Andric if (GetTagFromPointer(p) != match_all_tag) 62506c3fb27SDimitry Andric CheckAddressSized<ErrorAction::Abort, AccessType::Store>(p, sz); 62606c3fb27SDimitry Andric } 62706c3fb27SDimitry Andric void __hwasan_store1_match_all(uptr p, u8 match_all_tag) { 62806c3fb27SDimitry Andric if (GetTagFromPointer(p) != match_all_tag) 62906c3fb27SDimitry Andric CheckAddress<ErrorAction::Abort, AccessType::Store, 0>(p); 63006c3fb27SDimitry Andric } 63106c3fb27SDimitry Andric void __hwasan_store2_match_all(uptr p, u8 match_all_tag) { 63206c3fb27SDimitry Andric if (GetTagFromPointer(p) != match_all_tag) 63306c3fb27SDimitry Andric CheckAddress<ErrorAction::Abort, AccessType::Store, 1>(p); 63406c3fb27SDimitry Andric } 63506c3fb27SDimitry Andric void __hwasan_store4_match_all(uptr p, u8 match_all_tag) { 63606c3fb27SDimitry Andric if (GetTagFromPointer(p) != match_all_tag) 63706c3fb27SDimitry Andric CheckAddress<ErrorAction::Abort, AccessType::Store, 2>(p); 63806c3fb27SDimitry Andric } 63906c3fb27SDimitry Andric void __hwasan_store8_match_all(uptr p, u8 match_all_tag) { 64006c3fb27SDimitry Andric if (GetTagFromPointer(p) != match_all_tag) 64106c3fb27SDimitry Andric CheckAddress<ErrorAction::Abort, AccessType::Store, 3>(p); 64206c3fb27SDimitry Andric } 64306c3fb27SDimitry Andric void __hwasan_store16_match_all(uptr p, u8 match_all_tag) { 64406c3fb27SDimitry Andric if (GetTagFromPointer(p) != match_all_tag) 64506c3fb27SDimitry Andric CheckAddress<ErrorAction::Abort, AccessType::Store, 4>(p); 64606c3fb27SDimitry Andric } 64706c3fb27SDimitry Andric 64806c3fb27SDimitry Andric void __hwasan_storeN_match_all_noabort(uptr p, uptr sz, u8 match_all_tag) { 64906c3fb27SDimitry Andric if (GetTagFromPointer(p) != match_all_tag) 65006c3fb27SDimitry Andric CheckAddressSized<ErrorAction::Recover, AccessType::Store>(p, sz); 65106c3fb27SDimitry Andric } 65206c3fb27SDimitry Andric void __hwasan_store1_match_all_noabort(uptr p, u8 match_all_tag) { 65306c3fb27SDimitry Andric if (GetTagFromPointer(p) != match_all_tag) 65406c3fb27SDimitry Andric CheckAddress<ErrorAction::Recover, AccessType::Store, 0>(p); 65506c3fb27SDimitry Andric } 65606c3fb27SDimitry Andric void __hwasan_store2_match_all_noabort(uptr p, u8 match_all_tag) { 65706c3fb27SDimitry Andric if (GetTagFromPointer(p) != match_all_tag) 65806c3fb27SDimitry Andric CheckAddress<ErrorAction::Recover, AccessType::Store, 1>(p); 65906c3fb27SDimitry Andric } 66006c3fb27SDimitry Andric void __hwasan_store4_match_all_noabort(uptr p, u8 match_all_tag) { 66106c3fb27SDimitry Andric if (GetTagFromPointer(p) != match_all_tag) 66206c3fb27SDimitry Andric CheckAddress<ErrorAction::Recover, AccessType::Store, 2>(p); 66306c3fb27SDimitry Andric } 66406c3fb27SDimitry Andric void __hwasan_store8_match_all_noabort(uptr p, u8 match_all_tag) { 66506c3fb27SDimitry Andric if (GetTagFromPointer(p) != match_all_tag) 66606c3fb27SDimitry Andric CheckAddress<ErrorAction::Recover, AccessType::Store, 3>(p); 66706c3fb27SDimitry Andric } 66806c3fb27SDimitry Andric void __hwasan_store16_match_all_noabort(uptr p, u8 match_all_tag) { 66906c3fb27SDimitry Andric if (GetTagFromPointer(p) != match_all_tag) 67006c3fb27SDimitry Andric CheckAddress<ErrorAction::Recover, AccessType::Store, 4>(p); 67106c3fb27SDimitry Andric } 67206c3fb27SDimitry Andric 6730b57cec5SDimitry Andric void __hwasan_tag_memory(uptr p, u8 tag, uptr sz) { 67406c3fb27SDimitry Andric TagMemoryAligned(UntagAddr(p), sz, tag); 6750b57cec5SDimitry Andric } 6760b57cec5SDimitry Andric 6770b57cec5SDimitry Andric uptr __hwasan_tag_pointer(uptr p, u8 tag) { 6780b57cec5SDimitry Andric return AddTagToPointer(p, tag); 6790b57cec5SDimitry Andric } 6800b57cec5SDimitry Andric 6815f757f3fSDimitry Andric u8 __hwasan_get_tag_from_pointer(uptr p) { return GetTagFromPointer(p); } 6825f757f3fSDimitry Andric 6830b57cec5SDimitry Andric void __hwasan_handle_longjmp(const void *sp_dst) { 6840b57cec5SDimitry Andric uptr dst = (uptr)sp_dst; 6850b57cec5SDimitry Andric // HWASan does not support tagged SP. 68606c3fb27SDimitry Andric CHECK_EQ(GetTagFromPointer(dst), 0); 6870b57cec5SDimitry Andric 6880b57cec5SDimitry Andric uptr sp = (uptr)__builtin_frame_address(0); 6890b57cec5SDimitry Andric static const uptr kMaxExpectedCleanupSize = 64 << 20; // 64M 6900b57cec5SDimitry Andric if (dst < sp || dst - sp > kMaxExpectedCleanupSize) { 6910b57cec5SDimitry Andric Report( 6920b57cec5SDimitry Andric "WARNING: HWASan is ignoring requested __hwasan_handle_longjmp: " 6930b57cec5SDimitry Andric "stack top: %p; target %p; distance: %p (%zd)\n" 6940b57cec5SDimitry Andric "False positive error reports may follow\n", 695*0fca6ea1SDimitry Andric (void *)sp, (void *)dst, dst - sp, dst - sp); 6960b57cec5SDimitry Andric return; 6970b57cec5SDimitry Andric } 6980b57cec5SDimitry Andric TagMemory(sp, dst - sp, 0); 6990b57cec5SDimitry Andric } 7000b57cec5SDimitry Andric 7010b57cec5SDimitry Andric void __hwasan_handle_vfork(const void *sp_dst) { 7020b57cec5SDimitry Andric uptr sp = (uptr)sp_dst; 7030b57cec5SDimitry Andric Thread *t = GetCurrentThread(); 7040b57cec5SDimitry Andric CHECK(t); 7050b57cec5SDimitry Andric uptr top = t->stack_top(); 7060b57cec5SDimitry Andric uptr bottom = t->stack_bottom(); 7070b57cec5SDimitry Andric if (top == 0 || bottom == 0 || sp < bottom || sp >= top) { 7080b57cec5SDimitry Andric Report( 7090b57cec5SDimitry Andric "WARNING: HWASan is ignoring requested __hwasan_handle_vfork: " 7100b57cec5SDimitry Andric "stack top: %zx; current %zx; bottom: %zx \n" 7110b57cec5SDimitry Andric "False positive error reports may follow\n", 7120b57cec5SDimitry Andric top, sp, bottom); 7130b57cec5SDimitry Andric return; 7140b57cec5SDimitry Andric } 7150b57cec5SDimitry Andric TagMemory(bottom, sp - bottom, 0); 7160b57cec5SDimitry Andric } 7170b57cec5SDimitry Andric 7180b57cec5SDimitry Andric extern "C" void *__hwasan_extra_spill_area() { 7190b57cec5SDimitry Andric Thread *t = GetCurrentThread(); 7200b57cec5SDimitry Andric return &t->vfork_spill(); 7210b57cec5SDimitry Andric } 7220b57cec5SDimitry Andric 7230b57cec5SDimitry Andric void __hwasan_print_memory_usage() { 724fe6060f1SDimitry Andric InternalScopedString s; 7250b57cec5SDimitry Andric HwasanFormatMemoryUsage(s); 7260b57cec5SDimitry Andric Printf("%s\n", s.data()); 7270b57cec5SDimitry Andric } 7280b57cec5SDimitry Andric 729fe6060f1SDimitry Andric static const u8 kFallbackTag = 0xBB & kTagMask; 7300b57cec5SDimitry Andric 7310b57cec5SDimitry Andric u8 __hwasan_generate_tag() { 7320b57cec5SDimitry Andric Thread *t = GetCurrentThread(); 7330b57cec5SDimitry Andric if (!t) return kFallbackTag; 7340b57cec5SDimitry Andric return t->GenerateRandomTag(); 7350b57cec5SDimitry Andric } 7360b57cec5SDimitry Andric 737753f127fSDimitry Andric void __hwasan_add_frame_record(u64 frame_record_info) { 738753f127fSDimitry Andric Thread *t = GetCurrentThread(); 739753f127fSDimitry Andric if (t) 740753f127fSDimitry Andric t->stack_allocations()->push(frame_record_info); 741753f127fSDimitry Andric } 742753f127fSDimitry Andric 7430b57cec5SDimitry Andric #if !SANITIZER_SUPPORTS_WEAK_HOOKS 7440b57cec5SDimitry Andric extern "C" { 7450b57cec5SDimitry Andric SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE 7460b57cec5SDimitry Andric const char* __hwasan_default_options() { return ""; } 7470b57cec5SDimitry Andric } // extern "C" 7480b57cec5SDimitry Andric #endif 7490b57cec5SDimitry Andric 7500b57cec5SDimitry Andric extern "C" { 7510b57cec5SDimitry Andric SANITIZER_INTERFACE_ATTRIBUTE 7520b57cec5SDimitry Andric void __sanitizer_print_stack_trace() { 7530b57cec5SDimitry Andric GET_FATAL_STACK_TRACE_PC_BP(StackTrace::GetCurrentPc(), GET_CURRENT_FRAME()); 7540b57cec5SDimitry Andric stack.Print(); 7550b57cec5SDimitry Andric } 756fe6060f1SDimitry Andric 757fe6060f1SDimitry Andric // Entry point for interoperability between __hwasan_tag_mismatch (ASM) and the 758fe6060f1SDimitry Andric // rest of the mismatch handling code (C++). 759fe6060f1SDimitry Andric void __hwasan_tag_mismatch4(uptr addr, uptr access_info, uptr *registers_frame, 760fe6060f1SDimitry Andric size_t outsize) { 761bdd1243dSDimitry Andric __hwasan::HwasanTagMismatch(addr, (uptr)__builtin_return_address(0), 762bdd1243dSDimitry Andric (uptr)__builtin_frame_address(0), access_info, 763bdd1243dSDimitry Andric registers_frame, outsize); 764fe6060f1SDimitry Andric } 765fe6060f1SDimitry Andric 7660b57cec5SDimitry Andric } // extern "C" 767