1 //===-- tsan_platform.h -----------------------------------------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file is a part of ThreadSanitizer (TSan), a race detector.
10 //
11 // Platform-specific code.
12 //===----------------------------------------------------------------------===//
13
14 #ifndef TSAN_PLATFORM_H
15 #define TSAN_PLATFORM_H
16
17 #if !defined(__LP64__) && !defined(_WIN64)
18 # error "Only 64-bit is supported"
19 #endif
20
21 #include "tsan_defs.h"
22 #include "tsan_trace.h"
23
24 namespace __tsan {
25
26 enum {
27 // App memory is not mapped onto shadow memory range.
28 kBrokenMapping = 1 << 0,
29 // Mapping app memory and back does not produce the same address,
30 // this can lead to wrong addresses in reports and potentially
31 // other bad consequences.
32 kBrokenReverseMapping = 1 << 1,
33 // Mapping is non-linear for linear user range.
34 // This is bad and can lead to unpredictable memory corruptions, etc
35 // because range access functions assume linearity.
36 kBrokenLinearity = 1 << 2,
37 };
38
39 /*
40 C/C++ on linux/x86_64 and freebsd/x86_64
41 0000 0000 1000 - 0080 0000 0000: main binary and/or MAP_32BIT mappings (512GB)
42 0040 0000 0000 - 0100 0000 0000: -
43 0100 0000 0000 - 2000 0000 0000: shadow
44 2000 0000 0000 - 3000 0000 0000: -
45 3000 0000 0000 - 4000 0000 0000: metainfo (memory blocks and sync objects)
46 4000 0000 0000 - 5500 0000 0000: -
47 5500 0000 0000 - 5680 0000 0000: pie binaries without ASLR or on 4.1+ kernels
48 5680 0000 0000 - 6000 0000 0000: -
49 6000 0000 0000 - 6200 0000 0000: traces
50 6200 0000 0000 - 7d00 0000 0000: -
51 7b00 0000 0000 - 7c00 0000 0000: heap
52 7c00 0000 0000 - 7e80 0000 0000: -
53 7e80 0000 0000 - 8000 0000 0000: modules and main thread stack
54
55 C/C++ on netbsd/amd64 can reuse the same mapping:
56 * The address space starts from 0x1000 (option with 0x0) and ends with
57 0x7f7ffffff000.
58 * LoAppMem-kHeapMemEnd can be reused as it is.
59 * No VDSO support.
60 * No MidAppMem region.
61 * No additional HeapMem region.
62 * HiAppMem contains the stack, loader, shared libraries and heap.
63 * Stack on NetBSD/amd64 has prereserved 128MB.
64 * Heap grows downwards (top-down).
65 * ASLR must be disabled per-process or globally.
66 */
67 struct Mapping48AddressSpace {
68 static const uptr kMetaShadowBeg = 0x300000000000ull;
69 static const uptr kMetaShadowEnd = 0x340000000000ull;
70 static const uptr kTraceMemBeg = 0x600000000000ull;
71 static const uptr kTraceMemEnd = 0x620000000000ull;
72 static const uptr kShadowBeg = 0x010000000000ull;
73 static const uptr kShadowEnd = 0x200000000000ull;
74 static const uptr kHeapMemBeg = 0x7b0000000000ull;
75 static const uptr kHeapMemEnd = 0x7c0000000000ull;
76 static const uptr kLoAppMemBeg = 0x000000001000ull;
77 static const uptr kLoAppMemEnd = 0x008000000000ull;
78 static const uptr kMidAppMemBeg = 0x550000000000ull;
79 static const uptr kMidAppMemEnd = 0x568000000000ull;
80 static const uptr kHiAppMemBeg = 0x7e8000000000ull;
81 static const uptr kHiAppMemEnd = 0x800000000000ull;
82 static const uptr kShadowMsk = 0x780000000000ull;
83 static const uptr kShadowXor = 0x040000000000ull;
84 static const uptr kShadowAdd = 0x000000000000ull;
85 static const uptr kVdsoBeg = 0xf000000000000000ull;
86 };
87
88 /*
89 C/C++ on linux/mips64 (40-bit VMA)
90 0000 0000 00 - 0100 0000 00: - (4 GB)
91 0100 0000 00 - 0200 0000 00: main binary (4 GB)
92 0200 0000 00 - 2000 0000 00: - (120 GB)
93 2000 0000 00 - 4000 0000 00: shadow (128 GB)
94 4000 0000 00 - 5000 0000 00: metainfo (memory blocks and sync objects) (64 GB)
95 5000 0000 00 - aa00 0000 00: - (360 GB)
96 aa00 0000 00 - ab00 0000 00: main binary (PIE) (4 GB)
97 ab00 0000 00 - b000 0000 00: - (20 GB)
98 b000 0000 00 - b200 0000 00: traces (8 GB)
99 b200 0000 00 - fe00 0000 00: - (304 GB)
100 fe00 0000 00 - ff00 0000 00: heap (4 GB)
101 ff00 0000 00 - ff80 0000 00: - (2 GB)
102 ff80 0000 00 - ffff ffff ff: modules and main thread stack (<2 GB)
103 */
104 struct MappingMips64_40 {
105 static const uptr kMetaShadowBeg = 0x4000000000ull;
106 static const uptr kMetaShadowEnd = 0x5000000000ull;
107 static const uptr kTraceMemBeg = 0xb000000000ull;
108 static const uptr kTraceMemEnd = 0xb200000000ull;
109 static const uptr kShadowBeg = 0x2000000000ull;
110 static const uptr kShadowEnd = 0x4000000000ull;
111 static const uptr kHeapMemBeg = 0xfe00000000ull;
112 static const uptr kHeapMemEnd = 0xff00000000ull;
113 static const uptr kLoAppMemBeg = 0x0100000000ull;
114 static const uptr kLoAppMemEnd = 0x0200000000ull;
115 static const uptr kMidAppMemBeg = 0xaa00000000ull;
116 static const uptr kMidAppMemEnd = 0xab00000000ull;
117 static const uptr kHiAppMemBeg = 0xff80000000ull;
118 static const uptr kHiAppMemEnd = 0xffffffffffull;
119 static const uptr kShadowMsk = 0xf800000000ull;
120 static const uptr kShadowXor = 0x0800000000ull;
121 static const uptr kShadowAdd = 0x0000000000ull;
122 static const uptr kVdsoBeg = 0xfffff00000ull;
123 };
124
125 /*
126 C/C++ on Darwin/iOS/ARM64 (36-bit VMA, 64 GB VM)
127 0000 0000 00 - 0100 0000 00: - (4 GB)
128 0100 0000 00 - 0200 0000 00: main binary, modules, thread stacks (4 GB)
129 0200 0000 00 - 0300 0000 00: heap (4 GB)
130 0300 0000 00 - 0400 0000 00: - (4 GB)
131 0400 0000 00 - 0c00 0000 00: shadow memory (32 GB)
132 0c00 0000 00 - 0d00 0000 00: - (4 GB)
133 0d00 0000 00 - 0e00 0000 00: metainfo (4 GB)
134 0e00 0000 00 - 0f00 0000 00: - (4 GB)
135 0f00 0000 00 - 0fc0 0000 00: traces (3 GB)
136 0fc0 0000 00 - 1000 0000 00: -
137 */
138 struct MappingAppleAarch64 {
139 static const uptr kLoAppMemBeg = 0x0100000000ull;
140 static const uptr kLoAppMemEnd = 0x0200000000ull;
141 static const uptr kHeapMemBeg = 0x0200000000ull;
142 static const uptr kHeapMemEnd = 0x0300000000ull;
143 static const uptr kShadowBeg = 0x0400000000ull;
144 static const uptr kShadowEnd = 0x0c00000000ull;
145 static const uptr kMetaShadowBeg = 0x0d00000000ull;
146 static const uptr kMetaShadowEnd = 0x0e00000000ull;
147 static const uptr kTraceMemBeg = 0x0f00000000ull;
148 static const uptr kTraceMemEnd = 0x0fc0000000ull;
149 static const uptr kHiAppMemBeg = 0x0fc0000000ull;
150 static const uptr kHiAppMemEnd = 0x0fc0000000ull;
151 static const uptr kShadowMsk = 0x0ull;
152 static const uptr kShadowXor = 0x0ull;
153 static const uptr kShadowAdd = 0x0ull;
154 static const uptr kVdsoBeg = 0x7000000000000000ull;
155 static const uptr kMidAppMemBeg = 0;
156 static const uptr kMidAppMemEnd = 0;
157 };
158
159 /*
160 C/C++ on linux/aarch64 (39-bit VMA)
161 0000 0010 00 - 0100 0000 00: main binary
162 0100 0000 00 - 0800 0000 00: -
163 0800 0000 00 - 2000 0000 00: shadow memory
164 2000 0000 00 - 3100 0000 00: -
165 3100 0000 00 - 3400 0000 00: metainfo
166 3400 0000 00 - 5500 0000 00: -
167 5500 0000 00 - 5600 0000 00: main binary (PIE)
168 5600 0000 00 - 6000 0000 00: -
169 6000 0000 00 - 6200 0000 00: traces
170 6200 0000 00 - 7d00 0000 00: -
171 7c00 0000 00 - 7d00 0000 00: heap
172 7d00 0000 00 - 7fff ffff ff: modules and main thread stack
173 */
174 struct MappingAarch64_39 {
175 static const uptr kLoAppMemBeg = 0x0000001000ull;
176 static const uptr kLoAppMemEnd = 0x0100000000ull;
177 static const uptr kShadowBeg = 0x0800000000ull;
178 static const uptr kShadowEnd = 0x2000000000ull;
179 static const uptr kMetaShadowBeg = 0x3100000000ull;
180 static const uptr kMetaShadowEnd = 0x3400000000ull;
181 static const uptr kMidAppMemBeg = 0x5500000000ull;
182 static const uptr kMidAppMemEnd = 0x5600000000ull;
183 static const uptr kTraceMemBeg = 0x6000000000ull;
184 static const uptr kTraceMemEnd = 0x6200000000ull;
185 static const uptr kHeapMemBeg = 0x7c00000000ull;
186 static const uptr kHeapMemEnd = 0x7d00000000ull;
187 static const uptr kHiAppMemBeg = 0x7e00000000ull;
188 static const uptr kHiAppMemEnd = 0x7fffffffffull;
189 static const uptr kShadowMsk = 0x7800000000ull;
190 static const uptr kShadowXor = 0x0200000000ull;
191 static const uptr kShadowAdd = 0x0000000000ull;
192 static const uptr kVdsoBeg = 0x7f00000000ull;
193 };
194
195 /*
196 C/C++ on linux/aarch64 (42-bit VMA)
197 00000 0010 00 - 01000 0000 00: main binary
198 01000 0000 00 - 10000 0000 00: -
199 10000 0000 00 - 20000 0000 00: shadow memory
200 20000 0000 00 - 26000 0000 00: -
201 26000 0000 00 - 28000 0000 00: metainfo
202 28000 0000 00 - 2aa00 0000 00: -
203 2aa00 0000 00 - 2ab00 0000 00: main binary (PIE)
204 2ab00 0000 00 - 36200 0000 00: -
205 36200 0000 00 - 36240 0000 00: traces
206 36240 0000 00 - 3e000 0000 00: -
207 3e000 0000 00 - 3f000 0000 00: heap
208 3f000 0000 00 - 3ffff ffff ff: modules and main thread stack
209 */
210 struct MappingAarch64_42 {
211 static const uptr kBroken = kBrokenReverseMapping;
212 static const uptr kLoAppMemBeg = 0x00000001000ull;
213 static const uptr kLoAppMemEnd = 0x01000000000ull;
214 static const uptr kShadowBeg = 0x10000000000ull;
215 static const uptr kShadowEnd = 0x20000000000ull;
216 static const uptr kMetaShadowBeg = 0x26000000000ull;
217 static const uptr kMetaShadowEnd = 0x28000000000ull;
218 static const uptr kMidAppMemBeg = 0x2aa00000000ull;
219 static const uptr kMidAppMemEnd = 0x2ab00000000ull;
220 static const uptr kTraceMemBeg = 0x36200000000ull;
221 static const uptr kTraceMemEnd = 0x36400000000ull;
222 static const uptr kHeapMemBeg = 0x3e000000000ull;
223 static const uptr kHeapMemEnd = 0x3f000000000ull;
224 static const uptr kHiAppMemBeg = 0x3f000000000ull;
225 static const uptr kHiAppMemEnd = 0x3ffffffffffull;
226 static const uptr kShadowMsk = 0x3c000000000ull;
227 static const uptr kShadowXor = 0x04000000000ull;
228 static const uptr kShadowAdd = 0x00000000000ull;
229 static const uptr kVdsoBeg = 0x37f00000000ull;
230 };
231
232 struct MappingAarch64_48 {
233 static const uptr kLoAppMemBeg = 0x0000000001000ull;
234 static const uptr kLoAppMemEnd = 0x0000200000000ull;
235 static const uptr kShadowBeg = 0x0002000000000ull;
236 static const uptr kShadowEnd = 0x0004000000000ull;
237 static const uptr kMetaShadowBeg = 0x0005000000000ull;
238 static const uptr kMetaShadowEnd = 0x0006000000000ull;
239 static const uptr kMidAppMemBeg = 0x0aaaa00000000ull;
240 static const uptr kMidAppMemEnd = 0x0aaaf00000000ull;
241 static const uptr kTraceMemBeg = 0x0f06000000000ull;
242 static const uptr kTraceMemEnd = 0x0f06200000000ull;
243 static const uptr kHeapMemBeg = 0x0ffff00000000ull;
244 static const uptr kHeapMemEnd = 0x0ffff00000000ull;
245 static const uptr kHiAppMemBeg = 0x0ffff00000000ull;
246 static const uptr kHiAppMemEnd = 0x1000000000000ull;
247 static const uptr kShadowMsk = 0x0fff800000000ull;
248 static const uptr kShadowXor = 0x0000800000000ull;
249 static const uptr kShadowAdd = 0x0000000000000ull;
250 static const uptr kVdsoBeg = 0xffff000000000ull;
251 };
252
253 /*
254 C/C++ on linux/powerpc64 (44-bit VMA)
255 0000 0000 0100 - 0001 0000 0000: main binary
256 0001 0000 0000 - 0001 0000 0000: -
257 0001 0000 0000 - 0b00 0000 0000: shadow
258 0b00 0000 0000 - 0b00 0000 0000: -
259 0b00 0000 0000 - 0d00 0000 0000: metainfo (memory blocks and sync objects)
260 0d00 0000 0000 - 0d00 0000 0000: -
261 0d00 0000 0000 - 0f00 0000 0000: traces
262 0f00 0000 0000 - 0f00 0000 0000: -
263 0f00 0000 0000 - 0f50 0000 0000: heap
264 0f50 0000 0000 - 0f60 0000 0000: -
265 0f60 0000 0000 - 1000 0000 0000: modules and main thread stack
266 */
267 struct MappingPPC64_44 {
268 static const uptr kBroken =
269 kBrokenMapping | kBrokenReverseMapping | kBrokenLinearity;
270 static const uptr kMetaShadowBeg = 0x0b0000000000ull;
271 static const uptr kMetaShadowEnd = 0x0d0000000000ull;
272 static const uptr kTraceMemBeg = 0x0d0000000000ull;
273 static const uptr kTraceMemEnd = 0x0f0000000000ull;
274 static const uptr kShadowBeg = 0x000100000000ull;
275 static const uptr kShadowEnd = 0x0b0000000000ull;
276 static const uptr kLoAppMemBeg = 0x000000000100ull;
277 static const uptr kLoAppMemEnd = 0x000100000000ull;
278 static const uptr kHeapMemBeg = 0x0f0000000000ull;
279 static const uptr kHeapMemEnd = 0x0f5000000000ull;
280 static const uptr kHiAppMemBeg = 0x0f6000000000ull;
281 static const uptr kHiAppMemEnd = 0x100000000000ull; // 44 bits
282 static const uptr kShadowMsk = 0x0f0000000000ull;
283 static const uptr kShadowXor = 0x002100000000ull;
284 static const uptr kShadowAdd = 0x000000000000ull;
285 static const uptr kVdsoBeg = 0x3c0000000000000ull;
286 static const uptr kMidAppMemBeg = 0;
287 static const uptr kMidAppMemEnd = 0;
288 };
289
290 /*
291 C/C++ on linux/powerpc64 (46-bit VMA)
292 0000 0000 1000 - 0100 0000 0000: main binary
293 0100 0000 0000 - 0200 0000 0000: -
294 0100 0000 0000 - 1000 0000 0000: shadow
295 1000 0000 0000 - 1000 0000 0000: -
296 1000 0000 0000 - 2000 0000 0000: metainfo (memory blocks and sync objects)
297 2000 0000 0000 - 2000 0000 0000: -
298 2000 0000 0000 - 2200 0000 0000: traces
299 2200 0000 0000 - 3d00 0000 0000: -
300 3d00 0000 0000 - 3e00 0000 0000: heap
301 3e00 0000 0000 - 3e80 0000 0000: -
302 3e80 0000 0000 - 4000 0000 0000: modules and main thread stack
303 */
304 struct MappingPPC64_46 {
305 static const uptr kMetaShadowBeg = 0x100000000000ull;
306 static const uptr kMetaShadowEnd = 0x200000000000ull;
307 static const uptr kTraceMemBeg = 0x200000000000ull;
308 static const uptr kTraceMemEnd = 0x220000000000ull;
309 static const uptr kShadowBeg = 0x010000000000ull;
310 static const uptr kShadowEnd = 0x100000000000ull;
311 static const uptr kHeapMemBeg = 0x3d0000000000ull;
312 static const uptr kHeapMemEnd = 0x3e0000000000ull;
313 static const uptr kLoAppMemBeg = 0x000000001000ull;
314 static const uptr kLoAppMemEnd = 0x010000000000ull;
315 static const uptr kHiAppMemBeg = 0x3e8000000000ull;
316 static const uptr kHiAppMemEnd = 0x400000000000ull; // 46 bits
317 static const uptr kShadowMsk = 0x3c0000000000ull;
318 static const uptr kShadowXor = 0x020000000000ull;
319 static const uptr kShadowAdd = 0x000000000000ull;
320 static const uptr kVdsoBeg = 0x7800000000000000ull;
321 static const uptr kMidAppMemBeg = 0;
322 static const uptr kMidAppMemEnd = 0;
323 };
324
325 /*
326 C/C++ on linux/powerpc64 (47-bit VMA)
327 0000 0000 1000 - 0100 0000 0000: main binary
328 0100 0000 0000 - 0200 0000 0000: -
329 0100 0000 0000 - 1000 0000 0000: shadow
330 1000 0000 0000 - 1000 0000 0000: -
331 1000 0000 0000 - 2000 0000 0000: metainfo (memory blocks and sync objects)
332 2000 0000 0000 - 2000 0000 0000: -
333 2000 0000 0000 - 2200 0000 0000: traces
334 2200 0000 0000 - 7d00 0000 0000: -
335 7d00 0000 0000 - 7e00 0000 0000: heap
336 7e00 0000 0000 - 7e80 0000 0000: -
337 7e80 0000 0000 - 8000 0000 0000: modules and main thread stack
338 */
339 struct MappingPPC64_47 {
340 static const uptr kMetaShadowBeg = 0x100000000000ull;
341 static const uptr kMetaShadowEnd = 0x200000000000ull;
342 static const uptr kTraceMemBeg = 0x200000000000ull;
343 static const uptr kTraceMemEnd = 0x220000000000ull;
344 static const uptr kShadowBeg = 0x010000000000ull;
345 static const uptr kShadowEnd = 0x100000000000ull;
346 static const uptr kHeapMemBeg = 0x7d0000000000ull;
347 static const uptr kHeapMemEnd = 0x7e0000000000ull;
348 static const uptr kLoAppMemBeg = 0x000000001000ull;
349 static const uptr kLoAppMemEnd = 0x010000000000ull;
350 static const uptr kHiAppMemBeg = 0x7e8000000000ull;
351 static const uptr kHiAppMemEnd = 0x800000000000ull; // 47 bits
352 static const uptr kShadowMsk = 0x7c0000000000ull;
353 static const uptr kShadowXor = 0x020000000000ull;
354 static const uptr kShadowAdd = 0x000000000000ull;
355 static const uptr kVdsoBeg = 0x7800000000000000ull;
356 static const uptr kMidAppMemBeg = 0;
357 static const uptr kMidAppMemEnd = 0;
358 };
359
360 /*
361 C/C++ on linux/s390x
362 While the kernel provides a 64-bit address space, we have to restrict ourselves
363 to 48 bits due to how e.g. SyncVar::GetId() works.
364 0000 0000 1000 - 0e00 0000 0000: binary, modules, stacks - 14 TiB
365 0e00 0000 0000 - 4000 0000 0000: -
366 4000 0000 0000 - 8000 0000 0000: shadow - 64TiB (4 * app)
367 8000 0000 0000 - 9000 0000 0000: -
368 9000 0000 0000 - 9800 0000 0000: metainfo - 8TiB (0.5 * app)
369 9800 0000 0000 - a000 0000 0000: -
370 a000 0000 0000 - b000 0000 0000: traces - 16TiB (max history * 128k threads)
371 b000 0000 0000 - be00 0000 0000: -
372 be00 0000 0000 - c000 0000 0000: heap - 2TiB (max supported by the allocator)
373 */
374 struct MappingS390x {
375 static const uptr kMetaShadowBeg = 0x900000000000ull;
376 static const uptr kMetaShadowEnd = 0x980000000000ull;
377 static const uptr kTraceMemBeg = 0xa00000000000ull;
378 static const uptr kTraceMemEnd = 0xb00000000000ull;
379 static const uptr kShadowBeg = 0x400000000000ull;
380 static const uptr kShadowEnd = 0x800000000000ull;
381 static const uptr kHeapMemBeg = 0xbe0000000000ull;
382 static const uptr kHeapMemEnd = 0xc00000000000ull;
383 static const uptr kLoAppMemBeg = 0x000000001000ull;
384 static const uptr kLoAppMemEnd = 0x0e0000000000ull;
385 static const uptr kHiAppMemBeg = 0xc00000004000ull;
386 static const uptr kHiAppMemEnd = 0xc00000004000ull;
387 static const uptr kShadowMsk = 0xb00000000000ull;
388 static const uptr kShadowXor = 0x100000000000ull;
389 static const uptr kShadowAdd = 0x000000000000ull;
390 static const uptr kVdsoBeg = 0xfffffffff000ull;
391 static const uptr kMidAppMemBeg = 0;
392 static const uptr kMidAppMemEnd = 0;
393 };
394
395 /* Go on linux, darwin and freebsd on x86_64
396 0000 0000 1000 - 0000 1000 0000: executable
397 0000 1000 0000 - 00c0 0000 0000: -
398 00c0 0000 0000 - 00e0 0000 0000: heap
399 00e0 0000 0000 - 2000 0000 0000: -
400 2000 0000 0000 - 2380 0000 0000: shadow
401 2380 0000 0000 - 3000 0000 0000: -
402 3000 0000 0000 - 4000 0000 0000: metainfo (memory blocks and sync objects)
403 4000 0000 0000 - 6000 0000 0000: -
404 6000 0000 0000 - 6200 0000 0000: traces
405 6200 0000 0000 - 8000 0000 0000: -
406 */
407
408 struct MappingGo48 {
409 static const uptr kMetaShadowBeg = 0x300000000000ull;
410 static const uptr kMetaShadowEnd = 0x400000000000ull;
411 static const uptr kTraceMemBeg = 0x600000000000ull;
412 static const uptr kTraceMemEnd = 0x620000000000ull;
413 static const uptr kShadowBeg = 0x200000000000ull;
414 static const uptr kShadowEnd = 0x238000000000ull;
415 static const uptr kLoAppMemBeg = 0x000000001000ull;
416 static const uptr kLoAppMemEnd = 0x00e000000000ull;
417 static const uptr kMidAppMemBeg = 0;
418 static const uptr kMidAppMemEnd = 0;
419 static const uptr kHiAppMemBeg = 0;
420 static const uptr kHiAppMemEnd = 0;
421 static const uptr kHeapMemBeg = 0;
422 static const uptr kHeapMemEnd = 0;
423 static const uptr kVdsoBeg = 0;
424 static const uptr kShadowMsk = 0;
425 static const uptr kShadowXor = 0;
426 static const uptr kShadowAdd = 0x200000000000ull;
427 };
428
429 /* Go on windows
430 0000 0000 1000 - 0000 1000 0000: executable
431 0000 1000 0000 - 00f8 0000 0000: -
432 00c0 0000 0000 - 00e0 0000 0000: heap
433 00e0 0000 0000 - 0100 0000 0000: -
434 0100 0000 0000 - 0500 0000 0000: shadow
435 0500 0000 0000 - 0700 0000 0000: traces
436 0700 0000 0000 - 0770 0000 0000: metainfo (memory blocks and sync objects)
437 07d0 0000 0000 - 8000 0000 0000: -
438 */
439
440 struct MappingGoWindows {
441 static const uptr kMetaShadowBeg = 0x070000000000ull;
442 static const uptr kMetaShadowEnd = 0x077000000000ull;
443 static const uptr kTraceMemBeg = 0x050000000000ull;
444 static const uptr kTraceMemEnd = 0x070000000000ull;
445 static const uptr kShadowBeg = 0x010000000000ull;
446 static const uptr kShadowEnd = 0x050000000000ull;
447 static const uptr kLoAppMemBeg = 0x000000001000ull;
448 static const uptr kLoAppMemEnd = 0x00e000000000ull;
449 static const uptr kMidAppMemBeg = 0;
450 static const uptr kMidAppMemEnd = 0;
451 static const uptr kHiAppMemBeg = 0;
452 static const uptr kHiAppMemEnd = 0;
453 static const uptr kHeapMemBeg = 0;
454 static const uptr kHeapMemEnd = 0;
455 static const uptr kVdsoBeg = 0;
456 static const uptr kShadowMsk = 0;
457 static const uptr kShadowXor = 0;
458 static const uptr kShadowAdd = 0x010000000000ull;
459 };
460
461 /* Go on linux/powerpc64 (46-bit VMA)
462 0000 0000 1000 - 0000 1000 0000: executable
463 0000 1000 0000 - 00c0 0000 0000: -
464 00c0 0000 0000 - 00e0 0000 0000: heap
465 00e0 0000 0000 - 2000 0000 0000: -
466 2000 0000 0000 - 2380 0000 0000: shadow
467 2380 0000 0000 - 2400 0000 0000: -
468 2400 0000 0000 - 3400 0000 0000: metainfo (memory blocks and sync objects)
469 3400 0000 0000 - 3600 0000 0000: -
470 3600 0000 0000 - 3800 0000 0000: traces
471 3800 0000 0000 - 4000 0000 0000: -
472 */
473
474 struct MappingGoPPC64_46 {
475 static const uptr kMetaShadowBeg = 0x240000000000ull;
476 static const uptr kMetaShadowEnd = 0x340000000000ull;
477 static const uptr kTraceMemBeg = 0x360000000000ull;
478 static const uptr kTraceMemEnd = 0x380000000000ull;
479 static const uptr kShadowBeg = 0x200000000000ull;
480 static const uptr kShadowEnd = 0x238000000000ull;
481 static const uptr kLoAppMemBeg = 0x000000001000ull;
482 static const uptr kLoAppMemEnd = 0x00e000000000ull;
483 static const uptr kMidAppMemBeg = 0;
484 static const uptr kMidAppMemEnd = 0;
485 static const uptr kHiAppMemBeg = 0;
486 static const uptr kHiAppMemEnd = 0;
487 static const uptr kHeapMemBeg = 0;
488 static const uptr kHeapMemEnd = 0;
489 static const uptr kVdsoBeg = 0;
490 static const uptr kShadowMsk = 0;
491 static const uptr kShadowXor = 0;
492 static const uptr kShadowAdd = 0x200000000000ull;
493 };
494
495 /* Go on linux/powerpc64 (47-bit VMA)
496 0000 0000 1000 - 0000 1000 0000: executable
497 0000 1000 0000 - 00c0 0000 0000: -
498 00c0 0000 0000 - 00e0 0000 0000: heap
499 00e0 0000 0000 - 2000 0000 0000: -
500 2000 0000 0000 - 3000 0000 0000: shadow
501 3000 0000 0000 - 3000 0000 0000: -
502 3000 0000 0000 - 4000 0000 0000: metainfo (memory blocks and sync objects)
503 4000 0000 0000 - 6000 0000 0000: -
504 6000 0000 0000 - 6200 0000 0000: traces
505 6200 0000 0000 - 8000 0000 0000: -
506 */
507
508 struct MappingGoPPC64_47 {
509 static const uptr kMetaShadowBeg = 0x300000000000ull;
510 static const uptr kMetaShadowEnd = 0x400000000000ull;
511 static const uptr kTraceMemBeg = 0x600000000000ull;
512 static const uptr kTraceMemEnd = 0x620000000000ull;
513 static const uptr kShadowBeg = 0x200000000000ull;
514 static const uptr kShadowEnd = 0x300000000000ull;
515 static const uptr kLoAppMemBeg = 0x000000001000ull;
516 static const uptr kLoAppMemEnd = 0x00e000000000ull;
517 static const uptr kMidAppMemBeg = 0;
518 static const uptr kMidAppMemEnd = 0;
519 static const uptr kHiAppMemBeg = 0;
520 static const uptr kHiAppMemEnd = 0;
521 static const uptr kHeapMemBeg = 0;
522 static const uptr kHeapMemEnd = 0;
523 static const uptr kVdsoBeg = 0;
524 static const uptr kShadowMsk = 0;
525 static const uptr kShadowXor = 0;
526 static const uptr kShadowAdd = 0x200000000000ull;
527 };
528
529 /* Go on linux/aarch64 (48-bit VMA) and darwin/aarch64 (47-bit VMA)
530 0000 0000 1000 - 0000 1000 0000: executable
531 0000 1000 0000 - 00c0 0000 0000: -
532 00c0 0000 0000 - 00e0 0000 0000: heap
533 00e0 0000 0000 - 2000 0000 0000: -
534 2000 0000 0000 - 3000 0000 0000: shadow
535 3000 0000 0000 - 3000 0000 0000: -
536 3000 0000 0000 - 4000 0000 0000: metainfo (memory blocks and sync objects)
537 4000 0000 0000 - 6000 0000 0000: -
538 6000 0000 0000 - 6200 0000 0000: traces
539 6200 0000 0000 - 8000 0000 0000: -
540 */
541 struct MappingGoAarch64 {
542 static const uptr kMetaShadowBeg = 0x300000000000ull;
543 static const uptr kMetaShadowEnd = 0x400000000000ull;
544 static const uptr kTraceMemBeg = 0x600000000000ull;
545 static const uptr kTraceMemEnd = 0x620000000000ull;
546 static const uptr kShadowBeg = 0x200000000000ull;
547 static const uptr kShadowEnd = 0x300000000000ull;
548 static const uptr kLoAppMemBeg = 0x000000001000ull;
549 static const uptr kLoAppMemEnd = 0x00e000000000ull;
550 static const uptr kMidAppMemBeg = 0;
551 static const uptr kMidAppMemEnd = 0;
552 static const uptr kHiAppMemBeg = 0;
553 static const uptr kHiAppMemEnd = 0;
554 static const uptr kHeapMemBeg = 0;
555 static const uptr kHeapMemEnd = 0;
556 static const uptr kVdsoBeg = 0;
557 static const uptr kShadowMsk = 0;
558 static const uptr kShadowXor = 0;
559 static const uptr kShadowAdd = 0x200000000000ull;
560 };
561
562 /*
563 Go on linux/mips64 (47-bit VMA)
564 0000 0000 1000 - 0000 1000 0000: executable
565 0000 1000 0000 - 00c0 0000 0000: -
566 00c0 0000 0000 - 00e0 0000 0000: heap
567 00e0 0000 0000 - 2000 0000 0000: -
568 2000 0000 0000 - 3000 0000 0000: shadow
569 3000 0000 0000 - 3000 0000 0000: -
570 3000 0000 0000 - 4000 0000 0000: metainfo (memory blocks and sync objects)
571 4000 0000 0000 - 6000 0000 0000: -
572 6000 0000 0000 - 6200 0000 0000: traces
573 6200 0000 0000 - 8000 0000 0000: -
574 */
575 struct MappingGoMips64_47 {
576 static const uptr kMetaShadowBeg = 0x300000000000ull;
577 static const uptr kMetaShadowEnd = 0x400000000000ull;
578 static const uptr kTraceMemBeg = 0x600000000000ull;
579 static const uptr kTraceMemEnd = 0x620000000000ull;
580 static const uptr kShadowBeg = 0x200000000000ull;
581 static const uptr kShadowEnd = 0x300000000000ull;
582 static const uptr kLoAppMemBeg = 0x000000001000ull;
583 static const uptr kLoAppMemEnd = 0x00e000000000ull;
584 static const uptr kMidAppMemBeg = 0;
585 static const uptr kMidAppMemEnd = 0;
586 static const uptr kHiAppMemBeg = 0;
587 static const uptr kHiAppMemEnd = 0;
588 static const uptr kHeapMemBeg = 0;
589 static const uptr kHeapMemEnd = 0;
590 static const uptr kVdsoBeg = 0;
591 static const uptr kShadowMsk = 0;
592 static const uptr kShadowXor = 0;
593 static const uptr kShadowAdd = 0x200000000000ull;
594 };
595
596 /*
597 Go on linux/s390x
598 0000 0000 1000 - 1000 0000 0000: executable and heap - 16 TiB
599 1000 0000 0000 - 4000 0000 0000: -
600 4000 0000 0000 - 8000 0000 0000: shadow - 64TiB (4 * app)
601 8000 0000 0000 - 9000 0000 0000: -
602 9000 0000 0000 - 9800 0000 0000: metainfo - 8TiB (0.5 * app)
603 9800 0000 0000 - a000 0000 0000: -
604 a000 0000 0000 - b000 0000 0000: traces - 16TiB (max history * 128k threads)
605 */
606 struct MappingGoS390x {
607 static const uptr kMetaShadowBeg = 0x900000000000ull;
608 static const uptr kMetaShadowEnd = 0x980000000000ull;
609 static const uptr kTraceMemBeg = 0xa00000000000ull;
610 static const uptr kTraceMemEnd = 0xb00000000000ull;
611 static const uptr kShadowBeg = 0x400000000000ull;
612 static const uptr kShadowEnd = 0x800000000000ull;
613 static const uptr kLoAppMemBeg = 0x000000001000ull;
614 static const uptr kLoAppMemEnd = 0x100000000000ull;
615 static const uptr kMidAppMemBeg = 0;
616 static const uptr kMidAppMemEnd = 0;
617 static const uptr kHiAppMemBeg = 0;
618 static const uptr kHiAppMemEnd = 0;
619 static const uptr kHeapMemBeg = 0;
620 static const uptr kHeapMemEnd = 0;
621 static const uptr kVdsoBeg = 0;
622 static const uptr kShadowMsk = 0;
623 static const uptr kShadowXor = 0;
624 static const uptr kShadowAdd = 0x400000000000ull;
625 };
626
627 extern uptr vmaSize;
628
629 template <typename Func, typename Arg>
SelectMapping(Arg arg)630 ALWAYS_INLINE auto SelectMapping(Arg arg) {
631 #if SANITIZER_GO
632 # if defined(__powerpc64__)
633 switch (vmaSize) {
634 case 46:
635 return Func::template Apply<MappingGoPPC64_46>(arg);
636 case 47:
637 return Func::template Apply<MappingGoPPC64_47>(arg);
638 }
639 # elif defined(__mips64)
640 return Func::template Apply<MappingGoMips64_47>(arg);
641 # elif defined(__s390x__)
642 return Func::template Apply<MappingGoS390x>(arg);
643 # elif defined(__aarch64__)
644 return Func::template Apply<MappingGoAarch64>(arg);
645 # elif SANITIZER_WINDOWS
646 return Func::template Apply<MappingGoWindows>(arg);
647 # else
648 return Func::template Apply<MappingGo48>(arg);
649 # endif
650 #else // SANITIZER_GO
651 # if defined(__x86_64__) || SANITIZER_IOSSIM || SANITIZER_MAC && !SANITIZER_IOS
652 return Func::template Apply<Mapping48AddressSpace>(arg);
653 # elif defined(__aarch64__) && defined(__APPLE__)
654 return Func::template Apply<MappingAppleAarch64>(arg);
655 # elif defined(__aarch64__) && !defined(__APPLE__)
656 switch (vmaSize) {
657 case 39:
658 return Func::template Apply<MappingAarch64_39>(arg);
659 case 42:
660 return Func::template Apply<MappingAarch64_42>(arg);
661 case 48:
662 return Func::template Apply<MappingAarch64_48>(arg);
663 }
664 # elif defined(__powerpc64__)
665 switch (vmaSize) {
666 case 44:
667 return Func::template Apply<MappingPPC64_44>(arg);
668 case 46:
669 return Func::template Apply<MappingPPC64_46>(arg);
670 case 47:
671 return Func::template Apply<MappingPPC64_47>(arg);
672 }
673 # elif defined(__mips64)
674 return Func::template Apply<MappingMips64_40>(arg);
675 # elif defined(__s390x__)
676 return Func::template Apply<MappingS390x>(arg);
677 # else
678 # error "unsupported platform"
679 # endif
680 #endif
681 Die();
682 }
683
684 template <typename Func>
ForEachMapping()685 void ForEachMapping() {
686 Func::template Apply<Mapping48AddressSpace>();
687 Func::template Apply<MappingMips64_40>();
688 Func::template Apply<MappingAppleAarch64>();
689 Func::template Apply<MappingAarch64_39>();
690 Func::template Apply<MappingAarch64_42>();
691 Func::template Apply<MappingAarch64_48>();
692 Func::template Apply<MappingPPC64_44>();
693 Func::template Apply<MappingPPC64_46>();
694 Func::template Apply<MappingPPC64_47>();
695 Func::template Apply<MappingS390x>();
696 Func::template Apply<MappingGo48>();
697 Func::template Apply<MappingGoWindows>();
698 Func::template Apply<MappingGoPPC64_46>();
699 Func::template Apply<MappingGoPPC64_47>();
700 Func::template Apply<MappingGoAarch64>();
701 Func::template Apply<MappingGoMips64_47>();
702 Func::template Apply<MappingGoS390x>();
703 }
704
705 enum MappingType {
706 kLoAppMemBeg,
707 kLoAppMemEnd,
708 kHiAppMemBeg,
709 kHiAppMemEnd,
710 kMidAppMemBeg,
711 kMidAppMemEnd,
712 kHeapMemBeg,
713 kHeapMemEnd,
714 kShadowBeg,
715 kShadowEnd,
716 kMetaShadowBeg,
717 kMetaShadowEnd,
718 kTraceMemBeg,
719 kTraceMemEnd,
720 kVdsoBeg,
721 };
722
723 struct MappingField {
724 template <typename Mapping>
ApplyMappingField725 static uptr Apply(MappingType type) {
726 switch (type) {
727 case kLoAppMemBeg:
728 return Mapping::kLoAppMemBeg;
729 case kLoAppMemEnd:
730 return Mapping::kLoAppMemEnd;
731 case kMidAppMemBeg:
732 return Mapping::kMidAppMemBeg;
733 case kMidAppMemEnd:
734 return Mapping::kMidAppMemEnd;
735 case kHiAppMemBeg:
736 return Mapping::kHiAppMemBeg;
737 case kHiAppMemEnd:
738 return Mapping::kHiAppMemEnd;
739 case kHeapMemBeg:
740 return Mapping::kHeapMemBeg;
741 case kHeapMemEnd:
742 return Mapping::kHeapMemEnd;
743 case kVdsoBeg:
744 return Mapping::kVdsoBeg;
745 case kShadowBeg:
746 return Mapping::kShadowBeg;
747 case kShadowEnd:
748 return Mapping::kShadowEnd;
749 case kMetaShadowBeg:
750 return Mapping::kMetaShadowBeg;
751 case kMetaShadowEnd:
752 return Mapping::kMetaShadowEnd;
753 case kTraceMemBeg:
754 return Mapping::kTraceMemBeg;
755 case kTraceMemEnd:
756 return Mapping::kTraceMemEnd;
757 }
758 Die();
759 }
760 };
761
762 ALWAYS_INLINE
LoAppMemBeg(void)763 uptr LoAppMemBeg(void) { return SelectMapping<MappingField>(kLoAppMemBeg); }
764 ALWAYS_INLINE
LoAppMemEnd(void)765 uptr LoAppMemEnd(void) { return SelectMapping<MappingField>(kLoAppMemEnd); }
766
767 ALWAYS_INLINE
MidAppMemBeg(void)768 uptr MidAppMemBeg(void) { return SelectMapping<MappingField>(kMidAppMemBeg); }
769 ALWAYS_INLINE
MidAppMemEnd(void)770 uptr MidAppMemEnd(void) { return SelectMapping<MappingField>(kMidAppMemEnd); }
771
772 ALWAYS_INLINE
HeapMemBeg(void)773 uptr HeapMemBeg(void) { return SelectMapping<MappingField>(kHeapMemBeg); }
774 ALWAYS_INLINE
HeapMemEnd(void)775 uptr HeapMemEnd(void) { return SelectMapping<MappingField>(kHeapMemEnd); }
776
777 ALWAYS_INLINE
HiAppMemBeg(void)778 uptr HiAppMemBeg(void) { return SelectMapping<MappingField>(kHiAppMemBeg); }
779 ALWAYS_INLINE
HiAppMemEnd(void)780 uptr HiAppMemEnd(void) { return SelectMapping<MappingField>(kHiAppMemEnd); }
781
782 ALWAYS_INLINE
VdsoBeg(void)783 uptr VdsoBeg(void) { return SelectMapping<MappingField>(kVdsoBeg); }
784
785 ALWAYS_INLINE
ShadowBeg(void)786 uptr ShadowBeg(void) { return SelectMapping<MappingField>(kShadowBeg); }
787 ALWAYS_INLINE
ShadowEnd(void)788 uptr ShadowEnd(void) { return SelectMapping<MappingField>(kShadowEnd); }
789
790 ALWAYS_INLINE
MetaShadowBeg(void)791 uptr MetaShadowBeg(void) { return SelectMapping<MappingField>(kMetaShadowBeg); }
792 ALWAYS_INLINE
MetaShadowEnd(void)793 uptr MetaShadowEnd(void) { return SelectMapping<MappingField>(kMetaShadowEnd); }
794
795 ALWAYS_INLINE
TraceMemBeg(void)796 uptr TraceMemBeg(void) { return SelectMapping<MappingField>(kTraceMemBeg); }
797 ALWAYS_INLINE
TraceMemEnd(void)798 uptr TraceMemEnd(void) { return SelectMapping<MappingField>(kTraceMemEnd); }
799
800 struct IsAppMemImpl {
801 template <typename Mapping>
ApplyIsAppMemImpl802 static bool Apply(uptr mem) {
803 return (mem >= Mapping::kHeapMemBeg && mem < Mapping::kHeapMemEnd) ||
804 (mem >= Mapping::kMidAppMemBeg && mem < Mapping::kMidAppMemEnd) ||
805 (mem >= Mapping::kLoAppMemBeg && mem < Mapping::kLoAppMemEnd) ||
806 (mem >= Mapping::kHiAppMemBeg && mem < Mapping::kHiAppMemEnd);
807 }
808 };
809
810 ALWAYS_INLINE
IsAppMem(uptr mem)811 bool IsAppMem(uptr mem) { return SelectMapping<IsAppMemImpl>(mem); }
812
813 struct IsShadowMemImpl {
814 template <typename Mapping>
ApplyIsShadowMemImpl815 static bool Apply(uptr mem) {
816 return mem >= Mapping::kShadowBeg && mem <= Mapping::kShadowEnd;
817 }
818 };
819
820 ALWAYS_INLINE
IsShadowMem(RawShadow * p)821 bool IsShadowMem(RawShadow *p) {
822 return SelectMapping<IsShadowMemImpl>(reinterpret_cast<uptr>(p));
823 }
824
825 struct IsMetaMemImpl {
826 template <typename Mapping>
ApplyIsMetaMemImpl827 static bool Apply(uptr mem) {
828 return mem >= Mapping::kMetaShadowBeg && mem <= Mapping::kMetaShadowEnd;
829 }
830 };
831
832 ALWAYS_INLINE
IsMetaMem(const u32 * p)833 bool IsMetaMem(const u32 *p) {
834 return SelectMapping<IsMetaMemImpl>(reinterpret_cast<uptr>(p));
835 }
836
837 struct MemToShadowImpl {
838 template <typename Mapping>
ApplyMemToShadowImpl839 static uptr Apply(uptr x) {
840 DCHECK(IsAppMemImpl::Apply<Mapping>(x));
841 return (((x) & ~(Mapping::kShadowMsk | (kShadowCell - 1))) ^
842 Mapping::kShadowXor) *
843 kShadowMultiplier +
844 Mapping::kShadowAdd;
845 }
846 };
847
848 ALWAYS_INLINE
MemToShadow(uptr x)849 RawShadow *MemToShadow(uptr x) {
850 return reinterpret_cast<RawShadow *>(SelectMapping<MemToShadowImpl>(x));
851 }
852
853 struct MemToMetaImpl {
854 template <typename Mapping>
ApplyMemToMetaImpl855 static u32 *Apply(uptr x) {
856 DCHECK(IsAppMemImpl::Apply<Mapping>(x));
857 return (u32 *)(((((x) & ~(Mapping::kShadowMsk | (kMetaShadowCell - 1)))) /
858 kMetaShadowCell * kMetaShadowSize) |
859 Mapping::kMetaShadowBeg);
860 }
861 };
862
863 ALWAYS_INLINE
MemToMeta(uptr x)864 u32 *MemToMeta(uptr x) { return SelectMapping<MemToMetaImpl>(x); }
865
866 struct ShadowToMemImpl {
867 template <typename Mapping>
ApplyShadowToMemImpl868 static uptr Apply(uptr sp) {
869 if (!IsShadowMemImpl::Apply<Mapping>(sp))
870 return 0;
871 // The shadow mapping is non-linear and we've lost some bits, so we don't
872 // have an easy way to restore the original app address. But the mapping is
873 // a bijection, so we try to restore the address as belonging to
874 // low/mid/high range consecutively and see if shadow->app->shadow mapping
875 // gives us the same address.
876 uptr p =
877 ((sp - Mapping::kShadowAdd) / kShadowMultiplier) ^ Mapping::kShadowXor;
878 if (p >= Mapping::kLoAppMemBeg && p < Mapping::kLoAppMemEnd &&
879 MemToShadowImpl::Apply<Mapping>(p) == sp)
880 return p;
881 if (Mapping::kMidAppMemBeg) {
882 uptr p_mid = p + (Mapping::kMidAppMemBeg & Mapping::kShadowMsk);
883 if (p_mid >= Mapping::kMidAppMemBeg && p_mid < Mapping::kMidAppMemEnd &&
884 MemToShadowImpl::Apply<Mapping>(p_mid) == sp)
885 return p_mid;
886 }
887 return p | Mapping::kShadowMsk;
888 }
889 };
890
891 ALWAYS_INLINE
ShadowToMem(RawShadow * s)892 uptr ShadowToMem(RawShadow *s) {
893 return SelectMapping<ShadowToMemImpl>(reinterpret_cast<uptr>(s));
894 }
895
896 // Compresses addr to kCompressedAddrBits stored in least significant bits.
CompressAddr(uptr addr)897 ALWAYS_INLINE uptr CompressAddr(uptr addr) {
898 return addr & ((1ull << kCompressedAddrBits) - 1);
899 }
900
901 struct RestoreAddrImpl {
902 typedef uptr Result;
903 template <typename Mapping>
ApplyRestoreAddrImpl904 static Result Apply(uptr addr) {
905 // To restore the address we go over all app memory ranges and check if top
906 // 3 bits of the compressed addr match that of the app range. If yes, we
907 // assume that the compressed address come from that range and restore the
908 // missing top bits to match the app range address.
909 const uptr ranges[] = {
910 Mapping::kLoAppMemBeg, Mapping::kLoAppMemEnd, Mapping::kMidAppMemBeg,
911 Mapping::kMidAppMemEnd, Mapping::kHiAppMemBeg, Mapping::kHiAppMemEnd,
912 Mapping::kHeapMemBeg, Mapping::kHeapMemEnd,
913 };
914 const uptr indicator = 0x0e0000000000ull;
915 const uptr ind_lsb = 1ull << LeastSignificantSetBitIndex(indicator);
916 for (uptr i = 0; i < ARRAY_SIZE(ranges); i += 2) {
917 uptr beg = ranges[i];
918 uptr end = ranges[i + 1];
919 if (beg == end)
920 continue;
921 for (uptr p = beg; p < end; p = RoundDown(p + ind_lsb, ind_lsb)) {
922 if ((addr & indicator) == (p & indicator))
923 return addr | (p & ~(ind_lsb - 1));
924 }
925 }
926 Printf("ThreadSanitizer: failed to restore address 0x%zx\n", addr);
927 Die();
928 }
929 };
930
931 // Restores compressed addr from kCompressedAddrBits to full representation.
932 // This is called only during reporting and is not performance-critical.
RestoreAddr(uptr addr)933 inline uptr RestoreAddr(uptr addr) {
934 return SelectMapping<RestoreAddrImpl>(addr);
935 }
936
937 // The additional page is to catch shadow stack overflow as paging fault.
938 // Windows wants 64K alignment for mmaps.
939 const uptr kTotalTraceSize = (kTraceSize * sizeof(Event) + sizeof(Trace)
940 + (64 << 10) + (64 << 10) - 1) & ~((64 << 10) - 1);
941
942 struct GetThreadTraceImpl {
943 template <typename Mapping>
ApplyGetThreadTraceImpl944 static uptr Apply(uptr tid) {
945 uptr p = Mapping::kTraceMemBeg + tid * kTotalTraceSize;
946 DCHECK_LT(p, Mapping::kTraceMemEnd);
947 return p;
948 }
949 };
950
951 ALWAYS_INLINE
GetThreadTrace(int tid)952 uptr GetThreadTrace(int tid) { return SelectMapping<GetThreadTraceImpl>(tid); }
953
954 struct GetThreadTraceHeaderImpl {
955 template <typename Mapping>
ApplyGetThreadTraceHeaderImpl956 static uptr Apply(uptr tid) {
957 uptr p = Mapping::kTraceMemBeg + tid * kTotalTraceSize +
958 kTraceSize * sizeof(Event);
959 DCHECK_LT(p, Mapping::kTraceMemEnd);
960 return p;
961 }
962 };
963
964 ALWAYS_INLINE
GetThreadTraceHeader(int tid)965 uptr GetThreadTraceHeader(int tid) {
966 return SelectMapping<GetThreadTraceHeaderImpl>(tid);
967 }
968
969 void InitializePlatform();
970 void InitializePlatformEarly();
971 void CheckAndProtect();
972 void InitializeShadowMemoryPlatform();
973 void FlushShadowMemory();
974 void WriteMemoryProfile(char *buf, uptr buf_size, u64 uptime_ns);
975 int ExtractResolvFDs(void *state, int *fds, int nfd);
976 int ExtractRecvmsgFDs(void *msg, int *fds, int nfd);
977 uptr ExtractLongJmpSp(uptr *env);
978 void ImitateTlsWrite(ThreadState *thr, uptr tls_addr, uptr tls_size);
979
980 int call_pthread_cancel_with_cleanup(int (*fn)(void *arg),
981 void (*cleanup)(void *arg), void *arg);
982
983 void DestroyThreadState();
984 void PlatformCleanUpThreadState(ThreadState *thr);
985
986 } // namespace __tsan
987
988 #endif // TSAN_PLATFORM_H
989