1 /* $NetBSD: subr_csan.c,v 1.9 2020/06/30 16:20:02 maxv Exp $ */ 2 3 /* 4 * Copyright (c) 2019 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Maxime Villard. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 #include <sys/cdefs.h> 33 __KERNEL_RCSID(0, "$NetBSD: subr_csan.c,v 1.9 2020/06/30 16:20:02 maxv Exp $"); 34 35 #include <sys/param.h> 36 #include <sys/device.h> 37 #include <sys/kernel.h> 38 #include <sys/param.h> 39 #include <sys/conf.h> 40 #include <sys/systm.h> 41 #include <sys/types.h> 42 #include <sys/csan.h> 43 #include <sys/cpu.h> 44 45 #ifdef KCSAN_PANIC 46 #define REPORT panic 47 #else 48 #define REPORT printf 49 #endif 50 51 typedef struct { 52 uintptr_t addr; 53 uint32_t size; 54 bool write:1; 55 bool atomic:1; 56 uintptr_t pc; 57 } csan_cell_t; 58 59 typedef struct { 60 bool inited; 61 uint32_t cnt; 62 csan_cell_t cell; 63 } csan_cpu_t; 64 65 static csan_cpu_t kcsan_cpus[MAXCPUS]; 66 static bool kcsan_enabled __read_mostly; 67 68 #define __RET_ADDR (uintptr_t)__builtin_return_address(0) 69 70 #define KCSAN_NACCESSES 1024 71 #define KCSAN_DELAY 10 /* 10 microseconds */ 72 73 /* -------------------------------------------------------------------------- */ 74 75 /* The MD code. */ 76 #include <machine/csan.h> 77 78 /* -------------------------------------------------------------------------- */ 79 80 void 81 kcsan_init(void) 82 { 83 kcsan_enabled = true; 84 } 85 86 void 87 kcsan_cpu_init(struct cpu_info *ci) 88 { 89 kcsan_cpus[cpu_index(ci)].inited = true; 90 } 91 92 /* -------------------------------------------------------------------------- */ 93 94 static inline void 95 kcsan_report(csan_cell_t *new, cpuid_t newcpu, csan_cell_t *old, cpuid_t oldcpu) 96 { 97 const char *newsym, *oldsym; 98 99 if (ksyms_getname(NULL, &newsym, (vaddr_t)new->pc, KSYMS_PROC) != 0) { 100 newsym = "Unknown"; 101 } 102 if (ksyms_getname(NULL, &oldsym, (vaddr_t)old->pc, KSYMS_PROC) != 0) { 103 oldsym = "Unknown"; 104 } 105 REPORT("CSan: Racy Access " 106 "[Cpu%lu %s%s Addr=%p Size=%u PC=%p<%s>] " 107 "[Cpu%lu %s%s Addr=%p Size=%u PC=%p<%s>]\n", 108 newcpu, 109 (new->atomic ? "Atomic " : ""), (new->write ? "Write" : "Read"), 110 (void *)new->addr, new->size, (void *)new->pc, newsym, 111 oldcpu, 112 (old->atomic ? "Atomic " : ""), (old->write ? "Write" : "Read"), 113 (void *)old->addr, old->size, (void *)old->pc, oldsym); 114 kcsan_md_unwind(); 115 } 116 117 static inline bool 118 kcsan_access_is_atomic(csan_cell_t *new, csan_cell_t *old) 119 { 120 if (new->write && !new->atomic) 121 return false; 122 if (old->write && !old->atomic) 123 return false; 124 return true; 125 } 126 127 static inline void 128 kcsan_access(uintptr_t addr, size_t size, bool write, bool atomic, uintptr_t pc) 129 { 130 csan_cell_t old, new; 131 csan_cpu_t *cpu; 132 uint64_t intr; 133 size_t i; 134 135 if (__predict_false(!kcsan_enabled)) 136 return; 137 if (__predict_false(kcsan_md_unsupported((vaddr_t)addr))) 138 return; 139 140 new.addr = addr; 141 new.size = size; 142 new.write = write; 143 new.atomic = atomic; 144 new.pc = pc; 145 146 for (i = 0; i < ncpu; i++) { 147 __builtin_memcpy(&old, &kcsan_cpus[i].cell, sizeof(old)); 148 149 if (old.addr + old.size <= new.addr) 150 continue; 151 if (new.addr + new.size <= old.addr) 152 continue; 153 if (__predict_true(!old.write && !new.write)) 154 continue; 155 if (__predict_true(kcsan_access_is_atomic(&new, &old))) 156 continue; 157 158 kcsan_report(&new, cpu_number(), &old, i); 159 break; 160 } 161 162 if (__predict_false(!kcsan_md_is_avail())) 163 return; 164 165 kcsan_md_disable_intrs(&intr); 166 167 cpu = &kcsan_cpus[cpu_number()]; 168 if (__predict_false(!cpu->inited)) 169 goto out; 170 cpu->cnt = (cpu->cnt + 1) % KCSAN_NACCESSES; 171 if (__predict_true(cpu->cnt != 0)) 172 goto out; 173 174 __builtin_memcpy(&cpu->cell, &new, sizeof(new)); 175 kcsan_md_delay(KCSAN_DELAY); 176 __builtin_memset(&cpu->cell, 0, sizeof(new)); 177 178 out: 179 kcsan_md_enable_intrs(&intr); 180 } 181 182 #define CSAN_READ(size) \ 183 void __tsan_read##size(uintptr_t); \ 184 void __tsan_read##size(uintptr_t addr) \ 185 { \ 186 kcsan_access(addr, size, false, false, __RET_ADDR); \ 187 } 188 189 CSAN_READ(1) 190 CSAN_READ(2) 191 CSAN_READ(4) 192 CSAN_READ(8) 193 CSAN_READ(16) 194 195 #define CSAN_WRITE(size) \ 196 void __tsan_write##size(uintptr_t); \ 197 void __tsan_write##size(uintptr_t addr) \ 198 { \ 199 kcsan_access(addr, size, true, false, __RET_ADDR); \ 200 } 201 202 CSAN_WRITE(1) 203 CSAN_WRITE(2) 204 CSAN_WRITE(4) 205 CSAN_WRITE(8) 206 CSAN_WRITE(16) 207 208 void __tsan_read_range(uintptr_t, size_t); 209 void __tsan_write_range(uintptr_t, size_t); 210 211 void 212 __tsan_read_range(uintptr_t addr, size_t size) 213 { 214 kcsan_access(addr, size, false, false, __RET_ADDR); 215 } 216 217 void 218 __tsan_write_range(uintptr_t addr, size_t size) 219 { 220 kcsan_access(addr, size, true, false, __RET_ADDR); 221 } 222 223 void __tsan_init(void); 224 void __tsan_func_entry(void *); 225 void __tsan_func_exit(void); 226 227 void 228 __tsan_init(void) 229 { 230 } 231 232 void 233 __tsan_func_entry(void *call_pc) 234 { 235 } 236 237 void 238 __tsan_func_exit(void) 239 { 240 } 241 242 /* -------------------------------------------------------------------------- */ 243 244 void * 245 kcsan_memcpy(void *dst, const void *src, size_t len) 246 { 247 kcsan_access((uintptr_t)src, len, false, false, __RET_ADDR); 248 kcsan_access((uintptr_t)dst, len, true, false, __RET_ADDR); 249 return __builtin_memcpy(dst, src, len); 250 } 251 252 int 253 kcsan_memcmp(const void *b1, const void *b2, size_t len) 254 { 255 kcsan_access((uintptr_t)b1, len, false, false, __RET_ADDR); 256 kcsan_access((uintptr_t)b2, len, false, false, __RET_ADDR); 257 return __builtin_memcmp(b1, b2, len); 258 } 259 260 void * 261 kcsan_memset(void *b, int c, size_t len) 262 { 263 kcsan_access((uintptr_t)b, len, true, false, __RET_ADDR); 264 return __builtin_memset(b, c, len); 265 } 266 267 void * 268 kcsan_memmove(void *dst, const void *src, size_t len) 269 { 270 kcsan_access((uintptr_t)src, len, false, false, __RET_ADDR); 271 kcsan_access((uintptr_t)dst, len, true, false, __RET_ADDR); 272 return __builtin_memmove(dst, src, len); 273 } 274 275 char * 276 kcsan_strcpy(char *dst, const char *src) 277 { 278 char *save = dst; 279 280 while (1) { 281 kcsan_access((uintptr_t)src, 1, false, false, __RET_ADDR); 282 kcsan_access((uintptr_t)dst, 1, true, false, __RET_ADDR); 283 *dst = *src; 284 if (*src == '\0') 285 break; 286 src++, dst++; 287 } 288 289 return save; 290 } 291 292 int 293 kcsan_strcmp(const char *s1, const char *s2) 294 { 295 while (1) { 296 kcsan_access((uintptr_t)s1, 1, false, false, __RET_ADDR); 297 kcsan_access((uintptr_t)s2, 1, false, false, __RET_ADDR); 298 if (*s1 != *s2) 299 break; 300 if (*s1 == '\0') 301 return 0; 302 s1++, s2++; 303 } 304 305 return (*(const unsigned char *)s1 - *(const unsigned char *)s2); 306 } 307 308 size_t 309 kcsan_strlen(const char *str) 310 { 311 const char *s; 312 313 s = str; 314 while (1) { 315 kcsan_access((uintptr_t)s, 1, false, false, __RET_ADDR); 316 if (*s == '\0') 317 break; 318 s++; 319 } 320 321 return (s - str); 322 } 323 324 #undef kcopy 325 #undef copyinstr 326 #undef copyoutstr 327 #undef copyin 328 #undef copyout 329 330 int kcsan_kcopy(const void *, void *, size_t); 331 int kcsan_copyinstr(const void *, void *, size_t, size_t *); 332 int kcsan_copyoutstr(const void *, void *, size_t, size_t *); 333 int kcsan_copyin(const void *, void *, size_t); 334 int kcsan_copyout(const void *, void *, size_t); 335 int kcopy(const void *, void *, size_t); 336 int copyinstr(const void *, void *, size_t, size_t *); 337 int copyoutstr(const void *, void *, size_t, size_t *); 338 int copyin(const void *, void *, size_t); 339 int copyout(const void *, void *, size_t); 340 341 int 342 kcsan_kcopy(const void *src, void *dst, size_t len) 343 { 344 kcsan_access((uintptr_t)src, len, false, false, __RET_ADDR); 345 kcsan_access((uintptr_t)dst, len, true, false, __RET_ADDR); 346 return kcopy(src, dst, len); 347 } 348 349 int 350 kcsan_copyin(const void *uaddr, void *kaddr, size_t len) 351 { 352 kcsan_access((uintptr_t)kaddr, len, true, false, __RET_ADDR); 353 return copyin(uaddr, kaddr, len); 354 } 355 356 int 357 kcsan_copyout(const void *kaddr, void *uaddr, size_t len) 358 { 359 kcsan_access((uintptr_t)kaddr, len, false, false, __RET_ADDR); 360 return copyout(kaddr, uaddr, len); 361 } 362 363 int 364 kcsan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done) 365 { 366 kcsan_access((uintptr_t)kaddr, len, true, false, __RET_ADDR); 367 return copyinstr(uaddr, kaddr, len, done); 368 } 369 370 int 371 kcsan_copyoutstr(const void *kaddr, void *uaddr, size_t len, size_t *done) 372 { 373 kcsan_access((uintptr_t)kaddr, len, false, false, __RET_ADDR); 374 return copyoutstr(kaddr, uaddr, len, done); 375 } 376 377 /* -------------------------------------------------------------------------- */ 378 379 #undef atomic_add_32 380 #undef atomic_add_int 381 #undef atomic_add_long 382 #undef atomic_add_ptr 383 #undef atomic_add_64 384 #undef atomic_add_32_nv 385 #undef atomic_add_int_nv 386 #undef atomic_add_long_nv 387 #undef atomic_add_ptr_nv 388 #undef atomic_add_64_nv 389 #undef atomic_and_32 390 #undef atomic_and_uint 391 #undef atomic_and_ulong 392 #undef atomic_and_64 393 #undef atomic_and_32_nv 394 #undef atomic_and_uint_nv 395 #undef atomic_and_ulong_nv 396 #undef atomic_and_64_nv 397 #undef atomic_or_32 398 #undef atomic_or_uint 399 #undef atomic_or_ulong 400 #undef atomic_or_64 401 #undef atomic_or_32_nv 402 #undef atomic_or_uint_nv 403 #undef atomic_or_ulong_nv 404 #undef atomic_or_64_nv 405 #undef atomic_cas_32 406 #undef atomic_cas_uint 407 #undef atomic_cas_ulong 408 #undef atomic_cas_ptr 409 #undef atomic_cas_64 410 #undef atomic_cas_32_ni 411 #undef atomic_cas_uint_ni 412 #undef atomic_cas_ulong_ni 413 #undef atomic_cas_ptr_ni 414 #undef atomic_cas_64_ni 415 #undef atomic_swap_32 416 #undef atomic_swap_uint 417 #undef atomic_swap_ulong 418 #undef atomic_swap_ptr 419 #undef atomic_swap_64 420 #undef atomic_dec_32 421 #undef atomic_dec_uint 422 #undef atomic_dec_ulong 423 #undef atomic_dec_ptr 424 #undef atomic_dec_64 425 #undef atomic_dec_32_nv 426 #undef atomic_dec_uint_nv 427 #undef atomic_dec_ulong_nv 428 #undef atomic_dec_ptr_nv 429 #undef atomic_dec_64_nv 430 #undef atomic_inc_32 431 #undef atomic_inc_uint 432 #undef atomic_inc_ulong 433 #undef atomic_inc_ptr 434 #undef atomic_inc_64 435 #undef atomic_inc_32_nv 436 #undef atomic_inc_uint_nv 437 #undef atomic_inc_ulong_nv 438 #undef atomic_inc_ptr_nv 439 #undef atomic_inc_64_nv 440 441 #define CSAN_ATOMIC_FUNC_ADD(name, tret, targ1, targ2) \ 442 void atomic_add_##name(volatile targ1 *, targ2); \ 443 void kcsan_atomic_add_##name(volatile targ1 *, targ2); \ 444 void kcsan_atomic_add_##name(volatile targ1 *ptr, targ2 val) \ 445 { \ 446 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ 447 __RET_ADDR); \ 448 atomic_add_##name(ptr, val); \ 449 } \ 450 tret atomic_add_##name##_nv(volatile targ1 *, targ2); \ 451 tret kcsan_atomic_add_##name##_nv(volatile targ1 *, targ2); \ 452 tret kcsan_atomic_add_##name##_nv(volatile targ1 *ptr, targ2 val) \ 453 { \ 454 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ 455 __RET_ADDR); \ 456 return atomic_add_##name##_nv(ptr, val); \ 457 } 458 459 #define CSAN_ATOMIC_FUNC_AND(name, tret, targ1, targ2) \ 460 void atomic_and_##name(volatile targ1 *, targ2); \ 461 void kcsan_atomic_and_##name(volatile targ1 *, targ2); \ 462 void kcsan_atomic_and_##name(volatile targ1 *ptr, targ2 val) \ 463 { \ 464 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ 465 __RET_ADDR); \ 466 atomic_and_##name(ptr, val); \ 467 } \ 468 tret atomic_and_##name##_nv(volatile targ1 *, targ2); \ 469 tret kcsan_atomic_and_##name##_nv(volatile targ1 *, targ2); \ 470 tret kcsan_atomic_and_##name##_nv(volatile targ1 *ptr, targ2 val) \ 471 { \ 472 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ 473 __RET_ADDR); \ 474 return atomic_and_##name##_nv(ptr, val); \ 475 } 476 477 #define CSAN_ATOMIC_FUNC_OR(name, tret, targ1, targ2) \ 478 void atomic_or_##name(volatile targ1 *, targ2); \ 479 void kcsan_atomic_or_##name(volatile targ1 *, targ2); \ 480 void kcsan_atomic_or_##name(volatile targ1 *ptr, targ2 val) \ 481 { \ 482 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ 483 __RET_ADDR); \ 484 atomic_or_##name(ptr, val); \ 485 } \ 486 tret atomic_or_##name##_nv(volatile targ1 *, targ2); \ 487 tret kcsan_atomic_or_##name##_nv(volatile targ1 *, targ2); \ 488 tret kcsan_atomic_or_##name##_nv(volatile targ1 *ptr, targ2 val) \ 489 { \ 490 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ 491 __RET_ADDR); \ 492 return atomic_or_##name##_nv(ptr, val); \ 493 } 494 495 #define CSAN_ATOMIC_FUNC_CAS(name, tret, targ1, targ2) \ 496 tret atomic_cas_##name(volatile targ1 *, targ2, targ2); \ 497 tret kcsan_atomic_cas_##name(volatile targ1 *, targ2, targ2); \ 498 tret kcsan_atomic_cas_##name(volatile targ1 *ptr, targ2 exp, targ2 new) \ 499 { \ 500 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ 501 __RET_ADDR); \ 502 return atomic_cas_##name(ptr, exp, new); \ 503 } \ 504 tret atomic_cas_##name##_ni(volatile targ1 *, targ2, targ2); \ 505 tret kcsan_atomic_cas_##name##_ni(volatile targ1 *, targ2, targ2); \ 506 tret kcsan_atomic_cas_##name##_ni(volatile targ1 *ptr, targ2 exp, targ2 new) \ 507 { \ 508 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ 509 __RET_ADDR); \ 510 return atomic_cas_##name##_ni(ptr, exp, new); \ 511 } 512 513 #define CSAN_ATOMIC_FUNC_SWAP(name, tret, targ1, targ2) \ 514 tret atomic_swap_##name(volatile targ1 *, targ2); \ 515 tret kcsan_atomic_swap_##name(volatile targ1 *, targ2); \ 516 tret kcsan_atomic_swap_##name(volatile targ1 *ptr, targ2 val) \ 517 { \ 518 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ 519 __RET_ADDR); \ 520 return atomic_swap_##name(ptr, val); \ 521 } 522 523 #define CSAN_ATOMIC_FUNC_DEC(name, tret, targ1) \ 524 void atomic_dec_##name(volatile targ1 *); \ 525 void kcsan_atomic_dec_##name(volatile targ1 *); \ 526 void kcsan_atomic_dec_##name(volatile targ1 *ptr) \ 527 { \ 528 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ 529 __RET_ADDR); \ 530 atomic_dec_##name(ptr); \ 531 } \ 532 tret atomic_dec_##name##_nv(volatile targ1 *); \ 533 tret kcsan_atomic_dec_##name##_nv(volatile targ1 *); \ 534 tret kcsan_atomic_dec_##name##_nv(volatile targ1 *ptr) \ 535 { \ 536 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ 537 __RET_ADDR); \ 538 return atomic_dec_##name##_nv(ptr); \ 539 } 540 541 #define CSAN_ATOMIC_FUNC_INC(name, tret, targ1) \ 542 void atomic_inc_##name(volatile targ1 *); \ 543 void kcsan_atomic_inc_##name(volatile targ1 *); \ 544 void kcsan_atomic_inc_##name(volatile targ1 *ptr) \ 545 { \ 546 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ 547 __RET_ADDR); \ 548 atomic_inc_##name(ptr); \ 549 } \ 550 tret atomic_inc_##name##_nv(volatile targ1 *); \ 551 tret kcsan_atomic_inc_##name##_nv(volatile targ1 *); \ 552 tret kcsan_atomic_inc_##name##_nv(volatile targ1 *ptr) \ 553 { \ 554 kcsan_access((uintptr_t)ptr, sizeof(tret), true, true, \ 555 __RET_ADDR); \ 556 return atomic_inc_##name##_nv(ptr); \ 557 } 558 559 CSAN_ATOMIC_FUNC_ADD(32, uint32_t, uint32_t, int32_t); 560 CSAN_ATOMIC_FUNC_ADD(64, uint64_t, uint64_t, int64_t); 561 CSAN_ATOMIC_FUNC_ADD(int, unsigned int, unsigned int, int); 562 CSAN_ATOMIC_FUNC_ADD(long, unsigned long, unsigned long, long); 563 CSAN_ATOMIC_FUNC_ADD(ptr, void *, void, ssize_t); 564 565 CSAN_ATOMIC_FUNC_AND(32, uint32_t, uint32_t, uint32_t); 566 CSAN_ATOMIC_FUNC_AND(64, uint64_t, uint64_t, uint64_t); 567 CSAN_ATOMIC_FUNC_AND(uint, unsigned int, unsigned int, unsigned int); 568 CSAN_ATOMIC_FUNC_AND(ulong, unsigned long, unsigned long, unsigned long); 569 570 CSAN_ATOMIC_FUNC_OR(32, uint32_t, uint32_t, uint32_t); 571 CSAN_ATOMIC_FUNC_OR(64, uint64_t, uint64_t, uint64_t); 572 CSAN_ATOMIC_FUNC_OR(uint, unsigned int, unsigned int, unsigned int); 573 CSAN_ATOMIC_FUNC_OR(ulong, unsigned long, unsigned long, unsigned long); 574 575 CSAN_ATOMIC_FUNC_CAS(32, uint32_t, uint32_t, uint32_t); 576 CSAN_ATOMIC_FUNC_CAS(64, uint64_t, uint64_t, uint64_t); 577 CSAN_ATOMIC_FUNC_CAS(uint, unsigned int, unsigned int, unsigned int); 578 CSAN_ATOMIC_FUNC_CAS(ulong, unsigned long, unsigned long, unsigned long); 579 CSAN_ATOMIC_FUNC_CAS(ptr, void *, void, void *); 580 581 CSAN_ATOMIC_FUNC_SWAP(32, uint32_t, uint32_t, uint32_t); 582 CSAN_ATOMIC_FUNC_SWAP(64, uint64_t, uint64_t, uint64_t); 583 CSAN_ATOMIC_FUNC_SWAP(uint, unsigned int, unsigned int, unsigned int); 584 CSAN_ATOMIC_FUNC_SWAP(ulong, unsigned long, unsigned long, unsigned long); 585 CSAN_ATOMIC_FUNC_SWAP(ptr, void *, void, void *); 586 587 CSAN_ATOMIC_FUNC_DEC(32, uint32_t, uint32_t) 588 CSAN_ATOMIC_FUNC_DEC(64, uint64_t, uint64_t) 589 CSAN_ATOMIC_FUNC_DEC(uint, unsigned int, unsigned int); 590 CSAN_ATOMIC_FUNC_DEC(ulong, unsigned long, unsigned long); 591 CSAN_ATOMIC_FUNC_DEC(ptr, void *, void); 592 593 CSAN_ATOMIC_FUNC_INC(32, uint32_t, uint32_t) 594 CSAN_ATOMIC_FUNC_INC(64, uint64_t, uint64_t) 595 CSAN_ATOMIC_FUNC_INC(uint, unsigned int, unsigned int); 596 CSAN_ATOMIC_FUNC_INC(ulong, unsigned long, unsigned long); 597 CSAN_ATOMIC_FUNC_INC(ptr, void *, void); 598 599 void 600 kcsan_atomic_load(const volatile void *p, void *v, int size) 601 { 602 kcsan_access((uintptr_t)p, size, false, true, __RET_ADDR); 603 switch (size) { 604 case 1: *(uint8_t *)v = *(const volatile uint8_t *)p; break; 605 case 2: *(uint16_t *)v = *(const volatile uint16_t *)p; break; 606 case 4: *(uint32_t *)v = *(const volatile uint32_t *)p; break; 607 case 8: *(uint64_t *)v = *(const volatile uint64_t *)p; break; 608 } 609 } 610 611 void 612 kcsan_atomic_store(volatile void *p, const void *v, int size) 613 { 614 kcsan_access((uintptr_t)p, size, true, true, __RET_ADDR); 615 switch (size) { 616 case 1: *(volatile uint8_t *)p = *(const uint8_t *)v; break; 617 case 2: *(volatile uint16_t *)p = *(const uint16_t *)v; break; 618 case 4: *(volatile uint32_t *)p = *(const uint32_t *)v; break; 619 case 8: *(volatile uint64_t *)p = *(const uint64_t *)v; break; 620 } 621 } 622 623 /* -------------------------------------------------------------------------- */ 624 625 #include <sys/bus.h> 626 627 #undef bus_space_read_multi_1 628 #undef bus_space_read_multi_2 629 #undef bus_space_read_multi_4 630 #undef bus_space_read_multi_8 631 #undef bus_space_read_multi_stream_1 632 #undef bus_space_read_multi_stream_2 633 #undef bus_space_read_multi_stream_4 634 #undef bus_space_read_multi_stream_8 635 #undef bus_space_read_region_1 636 #undef bus_space_read_region_2 637 #undef bus_space_read_region_4 638 #undef bus_space_read_region_8 639 #undef bus_space_read_region_stream_1 640 #undef bus_space_read_region_stream_2 641 #undef bus_space_read_region_stream_4 642 #undef bus_space_read_region_stream_8 643 #undef bus_space_write_multi_1 644 #undef bus_space_write_multi_2 645 #undef bus_space_write_multi_4 646 #undef bus_space_write_multi_8 647 #undef bus_space_write_multi_stream_1 648 #undef bus_space_write_multi_stream_2 649 #undef bus_space_write_multi_stream_4 650 #undef bus_space_write_multi_stream_8 651 #undef bus_space_write_region_1 652 #undef bus_space_write_region_2 653 #undef bus_space_write_region_4 654 #undef bus_space_write_region_8 655 #undef bus_space_write_region_stream_1 656 #undef bus_space_write_region_stream_2 657 #undef bus_space_write_region_stream_4 658 #undef bus_space_write_region_stream_8 659 660 #define CSAN_BUS_READ_FUNC(bytes, bits) \ 661 void bus_space_read_multi_##bytes(bus_space_tag_t, bus_space_handle_t, \ 662 bus_size_t, uint##bits##_t *, bus_size_t); \ 663 void kcsan_bus_space_read_multi_##bytes(bus_space_tag_t, \ 664 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 665 void kcsan_bus_space_read_multi_##bytes(bus_space_tag_t tag, \ 666 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \ 667 bus_size_t count) \ 668 { \ 669 kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \ 670 false, false, __RET_ADDR); \ 671 bus_space_read_multi_##bytes(tag, hnd, size, buf, count); \ 672 } \ 673 void bus_space_read_multi_stream_##bytes(bus_space_tag_t, \ 674 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 675 void kcsan_bus_space_read_multi_stream_##bytes(bus_space_tag_t, \ 676 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 677 void kcsan_bus_space_read_multi_stream_##bytes(bus_space_tag_t tag, \ 678 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \ 679 bus_size_t count) \ 680 { \ 681 kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \ 682 false, false, __RET_ADDR); \ 683 bus_space_read_multi_stream_##bytes(tag, hnd, size, buf, count);\ 684 } \ 685 void bus_space_read_region_##bytes(bus_space_tag_t, bus_space_handle_t, \ 686 bus_size_t, uint##bits##_t *, bus_size_t); \ 687 void kcsan_bus_space_read_region_##bytes(bus_space_tag_t, \ 688 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 689 void kcsan_bus_space_read_region_##bytes(bus_space_tag_t tag, \ 690 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \ 691 bus_size_t count) \ 692 { \ 693 kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \ 694 false, false, __RET_ADDR); \ 695 bus_space_read_region_##bytes(tag, hnd, size, buf, count); \ 696 } \ 697 void bus_space_read_region_stream_##bytes(bus_space_tag_t, \ 698 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 699 void kcsan_bus_space_read_region_stream_##bytes(bus_space_tag_t, \ 700 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 701 void kcsan_bus_space_read_region_stream_##bytes(bus_space_tag_t tag, \ 702 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \ 703 bus_size_t count) \ 704 { \ 705 kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \ 706 false, false, __RET_ADDR); \ 707 bus_space_read_region_stream_##bytes(tag, hnd, size, buf, count);\ 708 } 709 710 #define CSAN_BUS_WRITE_FUNC(bytes, bits) \ 711 void bus_space_write_multi_##bytes(bus_space_tag_t, bus_space_handle_t, \ 712 bus_size_t, const uint##bits##_t *, bus_size_t); \ 713 void kcsan_bus_space_write_multi_##bytes(bus_space_tag_t, \ 714 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 715 void kcsan_bus_space_write_multi_##bytes(bus_space_tag_t tag, \ 716 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \ 717 bus_size_t count) \ 718 { \ 719 kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \ 720 true, false, __RET_ADDR); \ 721 bus_space_write_multi_##bytes(tag, hnd, size, buf, count); \ 722 } \ 723 void bus_space_write_multi_stream_##bytes(bus_space_tag_t, \ 724 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 725 void kcsan_bus_space_write_multi_stream_##bytes(bus_space_tag_t, \ 726 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 727 void kcsan_bus_space_write_multi_stream_##bytes(bus_space_tag_t tag, \ 728 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \ 729 bus_size_t count) \ 730 { \ 731 kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \ 732 true, false, __RET_ADDR); \ 733 bus_space_write_multi_stream_##bytes(tag, hnd, size, buf, count);\ 734 } \ 735 void bus_space_write_region_##bytes(bus_space_tag_t, bus_space_handle_t,\ 736 bus_size_t, const uint##bits##_t *, bus_size_t); \ 737 void kcsan_bus_space_write_region_##bytes(bus_space_tag_t, \ 738 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 739 void kcsan_bus_space_write_region_##bytes(bus_space_tag_t tag, \ 740 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \ 741 bus_size_t count) \ 742 { \ 743 kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \ 744 true, false, __RET_ADDR); \ 745 bus_space_write_region_##bytes(tag, hnd, size, buf, count); \ 746 } \ 747 void bus_space_write_region_stream_##bytes(bus_space_tag_t, \ 748 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 749 void kcsan_bus_space_write_region_stream_##bytes(bus_space_tag_t, \ 750 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 751 void kcsan_bus_space_write_region_stream_##bytes(bus_space_tag_t tag, \ 752 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \ 753 bus_size_t count) \ 754 { \ 755 kcsan_access((uintptr_t)buf, sizeof(uint##bits##_t) * count, \ 756 true, false, __RET_ADDR); \ 757 bus_space_write_region_stream_##bytes(tag, hnd, size, buf, count);\ 758 } 759 760 CSAN_BUS_READ_FUNC(1, 8) 761 CSAN_BUS_READ_FUNC(2, 16) 762 CSAN_BUS_READ_FUNC(4, 32) 763 CSAN_BUS_READ_FUNC(8, 64) 764 765 CSAN_BUS_WRITE_FUNC(1, 8) 766 CSAN_BUS_WRITE_FUNC(2, 16) 767 CSAN_BUS_WRITE_FUNC(4, 32) 768 CSAN_BUS_WRITE_FUNC(8, 64) 769