1 //===-- memtag.h ------------------------------------------------*- C++ -*-===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #ifndef SCUDO_MEMTAG_H_ 10 #define SCUDO_MEMTAG_H_ 11 12 #include "internal_defs.h" 13 14 #if SCUDO_CAN_USE_MTE 15 #include <sys/auxv.h> 16 #include <sys/prctl.h> 17 #endif 18 19 namespace scudo { 20 21 #if (__clang_major__ >= 12 && defined(__aarch64__) && !defined(__ILP32__)) || \ 22 defined(SCUDO_FUZZ) 23 24 // We assume that Top-Byte Ignore is enabled if the architecture supports memory 25 // tagging. Not all operating systems enable TBI, so we only claim architectural 26 // support for memory tagging if the operating system enables TBI. 27 // HWASan uses the top byte for its own purpose and Scudo should not touch it. 28 #if SCUDO_CAN_USE_MTE && !defined(SCUDO_DISABLE_TBI) && \ 29 !__has_feature(hwaddress_sanitizer) 30 inline constexpr bool archSupportsMemoryTagging() { return true; } 31 #else 32 inline constexpr bool archSupportsMemoryTagging() { return false; } 33 #endif 34 35 inline constexpr uptr archMemoryTagGranuleSize() { return 16; } 36 37 inline uptr untagPointer(uptr Ptr) { return Ptr & ((1ULL << 56) - 1); } 38 39 inline uint8_t extractTag(uptr Ptr) { return (Ptr >> 56) & 0xf; } 40 41 #else 42 43 inline constexpr bool archSupportsMemoryTagging() { return false; } 44 45 inline NORETURN uptr archMemoryTagGranuleSize() { 46 UNREACHABLE("memory tagging not supported"); 47 } 48 49 inline NORETURN uptr untagPointer(uptr Ptr) { 50 (void)Ptr; 51 UNREACHABLE("memory tagging not supported"); 52 } 53 54 inline NORETURN uint8_t extractTag(uptr Ptr) { 55 (void)Ptr; 56 UNREACHABLE("memory tagging not supported"); 57 } 58 59 #endif 60 61 #if __clang_major__ >= 12 && defined(__aarch64__) && !defined(__ILP32__) 62 63 #if SCUDO_CAN_USE_MTE 64 65 inline bool systemSupportsMemoryTagging() { 66 #ifndef HWCAP2_MTE 67 #define HWCAP2_MTE (1 << 18) 68 #endif 69 return getauxval(AT_HWCAP2) & HWCAP2_MTE; 70 } 71 72 inline bool systemDetectsMemoryTagFaultsTestOnly() { 73 #ifndef PR_SET_TAGGED_ADDR_CTRL 74 #define PR_SET_TAGGED_ADDR_CTRL 54 75 #endif 76 #ifndef PR_GET_TAGGED_ADDR_CTRL 77 #define PR_GET_TAGGED_ADDR_CTRL 56 78 #endif 79 #ifndef PR_TAGGED_ADDR_ENABLE 80 #define PR_TAGGED_ADDR_ENABLE (1UL << 0) 81 #endif 82 #ifndef PR_MTE_TCF_SHIFT 83 #define PR_MTE_TCF_SHIFT 1 84 #endif 85 #ifndef PR_MTE_TAG_SHIFT 86 #define PR_MTE_TAG_SHIFT 3 87 #endif 88 #ifndef PR_MTE_TCF_NONE 89 #define PR_MTE_TCF_NONE (0UL << PR_MTE_TCF_SHIFT) 90 #endif 91 #ifndef PR_MTE_TCF_SYNC 92 #define PR_MTE_TCF_SYNC (1UL << PR_MTE_TCF_SHIFT) 93 #endif 94 #ifndef PR_MTE_TCF_MASK 95 #define PR_MTE_TCF_MASK (3UL << PR_MTE_TCF_SHIFT) 96 #endif 97 int res = prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0); 98 if (res == -1) 99 return false; 100 return (static_cast<unsigned long>(res) & PR_MTE_TCF_MASK) != PR_MTE_TCF_NONE; 101 } 102 103 inline void enableSystemMemoryTaggingTestOnly() { 104 prctl(PR_SET_TAGGED_ADDR_CTRL, 105 PR_TAGGED_ADDR_ENABLE | PR_MTE_TCF_SYNC | (0xfffe << PR_MTE_TAG_SHIFT), 106 0, 0, 0); 107 } 108 109 #else // !SCUDO_CAN_USE_MTE 110 111 inline bool systemSupportsMemoryTagging() { return false; } 112 113 inline NORETURN bool systemDetectsMemoryTagFaultsTestOnly() { 114 UNREACHABLE("memory tagging not supported"); 115 } 116 117 inline NORETURN void enableSystemMemoryTaggingTestOnly() { 118 UNREACHABLE("memory tagging not supported"); 119 } 120 121 #endif // SCUDO_CAN_USE_MTE 122 123 class ScopedDisableMemoryTagChecks { 124 uptr PrevTCO; 125 bool active; 126 127 public: 128 ScopedDisableMemoryTagChecks(bool cond = true) : active(cond) { 129 if (!active) 130 return; 131 __asm__ __volatile__( 132 R"( 133 .arch_extension memtag 134 mrs %0, tco 135 msr tco, #1 136 )" 137 : "=r"(PrevTCO)); 138 } 139 140 ~ScopedDisableMemoryTagChecks() { 141 if (!active) 142 return; 143 __asm__ __volatile__( 144 R"( 145 .arch_extension memtag 146 msr tco, %0 147 )" 148 : 149 : "r"(PrevTCO)); 150 } 151 }; 152 153 inline uptr selectRandomTag(uptr Ptr, uptr ExcludeMask) { 154 ExcludeMask |= 1; // Always exclude Tag 0. 155 uptr TaggedPtr; 156 __asm__ __volatile__( 157 R"( 158 .arch_extension memtag 159 irg %[TaggedPtr], %[Ptr], %[ExcludeMask] 160 )" 161 : [TaggedPtr] "=r"(TaggedPtr) 162 : [Ptr] "r"(Ptr), [ExcludeMask] "r"(ExcludeMask)); 163 return TaggedPtr; 164 } 165 166 inline uptr addFixedTag(uptr Ptr, uptr Tag) { 167 DCHECK_LT(Tag, 16); 168 DCHECK_EQ(untagPointer(Ptr), Ptr); 169 return Ptr | (Tag << 56); 170 } 171 172 inline uptr storeTags(uptr Begin, uptr End) { 173 DCHECK_EQ(0, Begin % 16); 174 uptr LineSize, Next, Tmp; 175 __asm__ __volatile__( 176 R"( 177 .arch_extension memtag 178 179 // Compute the cache line size in bytes (DCZID_EL0 stores it as the log2 180 // of the number of 4-byte words) and bail out to the slow path if DCZID_EL0 181 // indicates that the DC instructions are unavailable. 182 DCZID .req %[Tmp] 183 mrs DCZID, dczid_el0 184 tbnz DCZID, #4, 3f 185 and DCZID, DCZID, #15 186 mov %[LineSize], #4 187 lsl %[LineSize], %[LineSize], DCZID 188 .unreq DCZID 189 190 // Our main loop doesn't handle the case where we don't need to perform any 191 // DC GZVA operations. If the size of our tagged region is less than 192 // twice the cache line size, bail out to the slow path since it's not 193 // guaranteed that we'll be able to do a DC GZVA. 194 Size .req %[Tmp] 195 sub Size, %[End], %[Cur] 196 cmp Size, %[LineSize], lsl #1 197 b.lt 3f 198 .unreq Size 199 200 LineMask .req %[Tmp] 201 sub LineMask, %[LineSize], #1 202 203 // STZG until the start of the next cache line. 204 orr %[Next], %[Cur], LineMask 205 1: 206 stzg %[Cur], [%[Cur]], #16 207 cmp %[Cur], %[Next] 208 b.lt 1b 209 210 // DC GZVA cache lines until we have no more full cache lines. 211 bic %[Next], %[End], LineMask 212 .unreq LineMask 213 2: 214 dc gzva, %[Cur] 215 add %[Cur], %[Cur], %[LineSize] 216 cmp %[Cur], %[Next] 217 b.lt 2b 218 219 // STZG until the end of the tagged region. This loop is also used to handle 220 // slow path cases. 221 3: 222 cmp %[Cur], %[End] 223 b.ge 4f 224 stzg %[Cur], [%[Cur]], #16 225 b 3b 226 227 4: 228 )" 229 : [Cur] "+&r"(Begin), [LineSize] "=&r"(LineSize), [Next] "=&r"(Next), 230 [Tmp] "=&r"(Tmp) 231 : [End] "r"(End) 232 : "memory"); 233 DCHECK_EQ(0, Begin % 16); 234 return Begin; 235 } 236 237 inline void storeTag(uptr Ptr) { 238 DCHECK_EQ(0, Ptr % 16); 239 __asm__ __volatile__(R"( 240 .arch_extension memtag 241 stg %0, [%0] 242 )" 243 : 244 : "r"(Ptr) 245 : "memory"); 246 } 247 248 inline uptr loadTag(uptr Ptr) { 249 DCHECK_EQ(0, Ptr % 16); 250 uptr TaggedPtr = Ptr; 251 __asm__ __volatile__( 252 R"( 253 .arch_extension memtag 254 ldg %0, [%0] 255 )" 256 : "+r"(TaggedPtr) 257 : 258 : "memory"); 259 return TaggedPtr; 260 } 261 262 #else 263 264 inline NORETURN bool systemSupportsMemoryTagging() { 265 UNREACHABLE("memory tagging not supported"); 266 } 267 268 inline NORETURN bool systemDetectsMemoryTagFaultsTestOnly() { 269 UNREACHABLE("memory tagging not supported"); 270 } 271 272 inline NORETURN void enableSystemMemoryTaggingTestOnly() { 273 UNREACHABLE("memory tagging not supported"); 274 } 275 276 struct ScopedDisableMemoryTagChecks { 277 ScopedDisableMemoryTagChecks(UNUSED bool cond = true) {} 278 }; 279 280 inline NORETURN uptr selectRandomTag(uptr Ptr, uptr ExcludeMask) { 281 (void)Ptr; 282 (void)ExcludeMask; 283 UNREACHABLE("memory tagging not supported"); 284 } 285 286 inline NORETURN uptr addFixedTag(uptr Ptr, uptr Tag) { 287 (void)Ptr; 288 (void)Tag; 289 UNREACHABLE("memory tagging not supported"); 290 } 291 292 inline NORETURN uptr storeTags(uptr Begin, uptr End) { 293 (void)Begin; 294 (void)End; 295 UNREACHABLE("memory tagging not supported"); 296 } 297 298 inline NORETURN void storeTag(uptr Ptr) { 299 (void)Ptr; 300 UNREACHABLE("memory tagging not supported"); 301 } 302 303 inline NORETURN uptr loadTag(uptr Ptr) { 304 (void)Ptr; 305 UNREACHABLE("memory tagging not supported"); 306 } 307 308 #endif 309 310 #pragma GCC diagnostic push 311 #pragma GCC diagnostic ignored "-Wmissing-noreturn" 312 inline void setRandomTag(void *Ptr, uptr Size, uptr ExcludeMask, 313 uptr *TaggedBegin, uptr *TaggedEnd) { 314 *TaggedBegin = selectRandomTag(reinterpret_cast<uptr>(Ptr), ExcludeMask); 315 *TaggedEnd = storeTags(*TaggedBegin, *TaggedBegin + Size); 316 } 317 #pragma GCC diagnostic pop 318 319 inline void *untagPointer(void *Ptr) { 320 return reinterpret_cast<void *>(untagPointer(reinterpret_cast<uptr>(Ptr))); 321 } 322 323 inline void *loadTag(void *Ptr) { 324 return reinterpret_cast<void *>(loadTag(reinterpret_cast<uptr>(Ptr))); 325 } 326 327 inline void *addFixedTag(void *Ptr, uptr Tag) { 328 return reinterpret_cast<void *>( 329 addFixedTag(reinterpret_cast<uptr>(Ptr), Tag)); 330 } 331 332 template <typename Config> 333 inline constexpr bool allocatorSupportsMemoryTagging() { 334 return archSupportsMemoryTagging() && Config::getMaySupportMemoryTagging() && 335 (1 << SCUDO_MIN_ALIGNMENT_LOG) >= archMemoryTagGranuleSize(); 336 } 337 338 } // namespace scudo 339 340 #endif 341