1 /* $NetBSD: subr_asan.c,v 1.26 2020/09/10 14:10:46 maxv Exp $ */ 2 3 /* 4 * Copyright (c) 2018-2020 Maxime Villard, m00nbsd.net 5 * All rights reserved. 6 * 7 * This code is part of the KASAN subsystem of the NetBSD kernel. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 20 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 21 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 22 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 23 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 25 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 */ 30 31 #include <sys/cdefs.h> 32 __KERNEL_RCSID(0, "$NetBSD: subr_asan.c,v 1.26 2020/09/10 14:10:46 maxv Exp $"); 33 34 #include <sys/param.h> 35 #include <sys/device.h> 36 #include <sys/kernel.h> 37 #include <sys/param.h> 38 #include <sys/conf.h> 39 #include <sys/systm.h> 40 #include <sys/types.h> 41 #include <sys/asan.h> 42 43 #include <uvm/uvm_extern.h> 44 45 #ifdef KASAN_PANIC 46 #define REPORT panic 47 #else 48 #define REPORT printf 49 #endif 50 51 /* ASAN constants. Part of the compiler ABI. */ 52 #define KASAN_SHADOW_SCALE_SIZE (1UL << KASAN_SHADOW_SCALE_SHIFT) 53 #define KASAN_SHADOW_MASK (KASAN_SHADOW_SCALE_SIZE - 1) 54 #define KASAN_ALLOCA_SCALE_SIZE 32 55 56 /* The MD code. */ 57 #include <machine/asan.h> 58 59 /* ASAN ABI version. */ 60 #if defined(__clang__) && (__clang_major__ - 0 >= 6) 61 #define ASAN_ABI_VERSION 8 62 #elif __GNUC_PREREQ__(7, 1) && !defined(__clang__) 63 #define ASAN_ABI_VERSION 8 64 #elif __GNUC_PREREQ__(6, 1) && !defined(__clang__) 65 #define ASAN_ABI_VERSION 6 66 #else 67 #error "Unsupported compiler version" 68 #endif 69 70 #define __RET_ADDR (unsigned long)__builtin_return_address(0) 71 72 /* Global variable descriptor. Part of the compiler ABI. */ 73 struct __asan_global_source_location { 74 const char *filename; 75 int line_no; 76 int column_no; 77 }; 78 struct __asan_global { 79 const void *beg; /* address of the global variable */ 80 size_t size; /* size of the global variable */ 81 size_t size_with_redzone; /* size with the redzone */ 82 const void *name; /* name of the variable */ 83 const void *module_name; /* name of the module where the var is declared */ 84 unsigned long has_dynamic_init; /* the var has dyn initializer (c++) */ 85 struct __asan_global_source_location *location; 86 #if ASAN_ABI_VERSION >= 7 87 uintptr_t odr_indicator; /* the address of the ODR indicator symbol */ 88 #endif 89 }; 90 91 static bool kasan_enabled __read_mostly = false; 92 93 /* -------------------------------------------------------------------------- */ 94 95 void 96 kasan_shadow_map(void *addr, size_t size) 97 { 98 size_t sz, npages, i; 99 vaddr_t sva, eva; 100 101 KASSERT((vaddr_t)addr % KASAN_SHADOW_SCALE_SIZE == 0); 102 103 sz = roundup(size, KASAN_SHADOW_SCALE_SIZE) / KASAN_SHADOW_SCALE_SIZE; 104 105 sva = (vaddr_t)kasan_md_addr_to_shad(addr); 106 eva = (vaddr_t)kasan_md_addr_to_shad(addr) + sz; 107 108 sva = rounddown(sva, PAGE_SIZE); 109 eva = roundup(eva, PAGE_SIZE); 110 111 npages = (eva - sva) / PAGE_SIZE; 112 113 KASSERT(sva >= KASAN_MD_SHADOW_START && eva < KASAN_MD_SHADOW_END); 114 115 for (i = 0; i < npages; i++) { 116 kasan_md_shadow_map_page(sva + i * PAGE_SIZE); 117 } 118 } 119 120 static void 121 kasan_ctors(void) 122 { 123 extern Elf_Addr __CTOR_LIST__, __CTOR_END__; 124 size_t nentries, i; 125 Elf_Addr *ptr; 126 127 nentries = ((size_t)&__CTOR_END__ - (size_t)&__CTOR_LIST__) / 128 sizeof(uintptr_t); 129 130 ptr = &__CTOR_LIST__; 131 for (i = 0; i < nentries; i++) { 132 void (*func)(void); 133 134 func = (void *)(*ptr); 135 (*func)(); 136 137 ptr++; 138 } 139 } 140 141 void 142 kasan_early_init(void *stack) 143 { 144 kasan_md_early_init(stack); 145 } 146 147 void 148 kasan_init(void) 149 { 150 /* MD initialization. */ 151 kasan_md_init(); 152 153 /* Now officially enabled. */ 154 kasan_enabled = true; 155 156 /* Call the ASAN constructors. */ 157 kasan_ctors(); 158 } 159 160 static inline const char * 161 kasan_code_name(uint8_t code) 162 { 163 switch (code) { 164 case KASAN_GENERIC_REDZONE: 165 return "GenericRedZone"; 166 case KASAN_MALLOC_REDZONE: 167 return "MallocRedZone"; 168 case KASAN_KMEM_REDZONE: 169 return "KmemRedZone"; 170 case KASAN_POOL_REDZONE: 171 return "PoolRedZone"; 172 case KASAN_POOL_FREED: 173 return "PoolUseAfterFree"; 174 case 1 ... 7: 175 return "RedZonePartial"; 176 case KASAN_STACK_LEFT: 177 return "StackLeft"; 178 case KASAN_STACK_MID: 179 return "StackMiddle"; 180 case KASAN_STACK_RIGHT: 181 return "StackRight"; 182 case KASAN_USE_AFTER_RET: 183 return "UseAfterRet"; 184 case KASAN_USE_AFTER_SCOPE: 185 return "UseAfterScope"; 186 default: 187 return "Unknown"; 188 } 189 } 190 191 static void 192 kasan_report(unsigned long addr, size_t size, bool write, unsigned long pc, 193 uint8_t code) 194 { 195 REPORT("ASan: Unauthorized Access In %p: Addr %p [%zu byte%s, %s," 196 " %s]\n", 197 (void *)pc, (void *)addr, size, (size > 1 ? "s" : ""), 198 (write ? "write" : "read"), kasan_code_name(code)); 199 kasan_md_unwind(); 200 } 201 202 static __always_inline void 203 kasan_shadow_1byte_markvalid(unsigned long addr) 204 { 205 int8_t *byte = kasan_md_addr_to_shad((void *)addr); 206 int8_t last = (addr & KASAN_SHADOW_MASK) + 1; 207 208 *byte = last; 209 } 210 211 static __always_inline void 212 kasan_shadow_Nbyte_markvalid(const void *addr, size_t size) 213 { 214 size_t i; 215 216 for (i = 0; i < size; i++) { 217 kasan_shadow_1byte_markvalid((unsigned long)addr+i); 218 } 219 } 220 221 static __always_inline void 222 kasan_shadow_Nbyte_fill(const void *addr, size_t size, uint8_t code) 223 { 224 void *shad; 225 226 if (__predict_false(size == 0)) 227 return; 228 if (__predict_false(kasan_md_unsupported((vaddr_t)addr))) 229 return; 230 231 KASSERT((vaddr_t)addr % KASAN_SHADOW_SCALE_SIZE == 0); 232 KASSERT(size % KASAN_SHADOW_SCALE_SIZE == 0); 233 234 shad = (void *)kasan_md_addr_to_shad(addr); 235 size = size >> KASAN_SHADOW_SCALE_SHIFT; 236 237 __builtin_memset(shad, code, size); 238 } 239 240 void 241 kasan_add_redzone(size_t *size) 242 { 243 *size = roundup(*size, KASAN_SHADOW_SCALE_SIZE); 244 *size += KASAN_SHADOW_SCALE_SIZE; 245 } 246 247 void 248 kasan_softint(struct lwp *l) 249 { 250 const void *stk = (const void *)uvm_lwp_getuarea(l); 251 252 kasan_shadow_Nbyte_fill(stk, USPACE, 0); 253 } 254 255 /* 256 * In an area of size 'sz_with_redz', mark the 'size' first bytes as valid, 257 * and the rest as invalid. There are generally two use cases: 258 * 259 * o kasan_mark(addr, origsize, size, code), with origsize < size. This marks 260 * the redzone at the end of the buffer as invalid. 261 * 262 * o kasan_mark(addr, size, size, 0). This marks the entire buffer as valid. 263 */ 264 void 265 kasan_mark(const void *addr, size_t size, size_t sz_with_redz, uint8_t code) 266 { 267 size_t i, n, redz; 268 int8_t *shad; 269 270 KASSERT((vaddr_t)addr % KASAN_SHADOW_SCALE_SIZE == 0); 271 redz = sz_with_redz - roundup(size, KASAN_SHADOW_SCALE_SIZE); 272 KASSERT(redz % KASAN_SHADOW_SCALE_SIZE == 0); 273 shad = kasan_md_addr_to_shad(addr); 274 275 /* Chunks of 8 bytes, valid. */ 276 n = size / KASAN_SHADOW_SCALE_SIZE; 277 for (i = 0; i < n; i++) { 278 *shad++ = 0; 279 } 280 281 /* Possibly one chunk, mid. */ 282 if ((size & KASAN_SHADOW_MASK) != 0) { 283 *shad++ = (size & KASAN_SHADOW_MASK); 284 } 285 286 /* Chunks of 8 bytes, invalid. */ 287 n = redz / KASAN_SHADOW_SCALE_SIZE; 288 for (i = 0; i < n; i++) { 289 *shad++ = code; 290 } 291 } 292 293 /* -------------------------------------------------------------------------- */ 294 295 #define ADDR_CROSSES_SCALE_BOUNDARY(addr, size) \ 296 (addr >> KASAN_SHADOW_SCALE_SHIFT) != \ 297 ((addr + size - 1) >> KASAN_SHADOW_SCALE_SHIFT) 298 299 static __always_inline bool 300 kasan_shadow_1byte_isvalid(unsigned long addr, uint8_t *code) 301 { 302 int8_t *byte = kasan_md_addr_to_shad((void *)addr); 303 int8_t last = (addr & KASAN_SHADOW_MASK) + 1; 304 305 if (__predict_true(*byte == 0 || last <= *byte)) { 306 return true; 307 } 308 *code = *byte; 309 return false; 310 } 311 312 static __always_inline bool 313 kasan_shadow_2byte_isvalid(unsigned long addr, uint8_t *code) 314 { 315 int8_t *byte, last; 316 317 if (ADDR_CROSSES_SCALE_BOUNDARY(addr, 2)) { 318 return (kasan_shadow_1byte_isvalid(addr, code) && 319 kasan_shadow_1byte_isvalid(addr+1, code)); 320 } 321 322 byte = kasan_md_addr_to_shad((void *)addr); 323 last = ((addr + 1) & KASAN_SHADOW_MASK) + 1; 324 325 if (__predict_true(*byte == 0 || last <= *byte)) { 326 return true; 327 } 328 *code = *byte; 329 return false; 330 } 331 332 static __always_inline bool 333 kasan_shadow_4byte_isvalid(unsigned long addr, uint8_t *code) 334 { 335 int8_t *byte, last; 336 337 if (ADDR_CROSSES_SCALE_BOUNDARY(addr, 4)) { 338 return (kasan_shadow_2byte_isvalid(addr, code) && 339 kasan_shadow_2byte_isvalid(addr+2, code)); 340 } 341 342 byte = kasan_md_addr_to_shad((void *)addr); 343 last = ((addr + 3) & KASAN_SHADOW_MASK) + 1; 344 345 if (__predict_true(*byte == 0 || last <= *byte)) { 346 return true; 347 } 348 *code = *byte; 349 return false; 350 } 351 352 static __always_inline bool 353 kasan_shadow_8byte_isvalid(unsigned long addr, uint8_t *code) 354 { 355 int8_t *byte, last; 356 357 if (ADDR_CROSSES_SCALE_BOUNDARY(addr, 8)) { 358 return (kasan_shadow_4byte_isvalid(addr, code) && 359 kasan_shadow_4byte_isvalid(addr+4, code)); 360 } 361 362 byte = kasan_md_addr_to_shad((void *)addr); 363 last = ((addr + 7) & KASAN_SHADOW_MASK) + 1; 364 365 if (__predict_true(*byte == 0 || last <= *byte)) { 366 return true; 367 } 368 *code = *byte; 369 return false; 370 } 371 372 static __always_inline bool 373 kasan_shadow_Nbyte_isvalid(unsigned long addr, size_t size, uint8_t *code) 374 { 375 size_t i; 376 377 for (i = 0; i < size; i++) { 378 if (!kasan_shadow_1byte_isvalid(addr+i, code)) 379 return false; 380 } 381 382 return true; 383 } 384 385 static __always_inline void 386 kasan_shadow_check(unsigned long addr, size_t size, bool write, 387 unsigned long retaddr) 388 { 389 uint8_t code; 390 bool valid; 391 392 if (__predict_false(!kasan_enabled)) 393 return; 394 if (__predict_false(size == 0)) 395 return; 396 if (__predict_false(kasan_md_unsupported(addr))) 397 return; 398 399 if (__builtin_constant_p(size)) { 400 switch (size) { 401 case 1: 402 valid = kasan_shadow_1byte_isvalid(addr, &code); 403 break; 404 case 2: 405 valid = kasan_shadow_2byte_isvalid(addr, &code); 406 break; 407 case 4: 408 valid = kasan_shadow_4byte_isvalid(addr, &code); 409 break; 410 case 8: 411 valid = kasan_shadow_8byte_isvalid(addr, &code); 412 break; 413 default: 414 valid = kasan_shadow_Nbyte_isvalid(addr, size, &code); 415 break; 416 } 417 } else { 418 valid = kasan_shadow_Nbyte_isvalid(addr, size, &code); 419 } 420 421 if (__predict_false(!valid)) { 422 kasan_report(addr, size, write, retaddr, code); 423 } 424 } 425 426 /* -------------------------------------------------------------------------- */ 427 428 void * 429 kasan_memcpy(void *dst, const void *src, size_t len) 430 { 431 kasan_shadow_check((unsigned long)src, len, false, __RET_ADDR); 432 kasan_shadow_check((unsigned long)dst, len, true, __RET_ADDR); 433 return __builtin_memcpy(dst, src, len); 434 } 435 436 int 437 kasan_memcmp(const void *b1, const void *b2, size_t len) 438 { 439 kasan_shadow_check((unsigned long)b1, len, false, __RET_ADDR); 440 kasan_shadow_check((unsigned long)b2, len, false, __RET_ADDR); 441 return __builtin_memcmp(b1, b2, len); 442 } 443 444 void * 445 kasan_memset(void *b, int c, size_t len) 446 { 447 kasan_shadow_check((unsigned long)b, len, true, __RET_ADDR); 448 return __builtin_memset(b, c, len); 449 } 450 451 void * 452 kasan_memmove(void *dst, const void *src, size_t len) 453 { 454 kasan_shadow_check((unsigned long)src, len, false, __RET_ADDR); 455 kasan_shadow_check((unsigned long)dst, len, true, __RET_ADDR); 456 return __builtin_memmove(dst, src, len); 457 } 458 459 char * 460 kasan_strcpy(char *dst, const char *src) 461 { 462 char *save = dst; 463 464 while (1) { 465 kasan_shadow_check((unsigned long)src, 1, false, __RET_ADDR); 466 kasan_shadow_check((unsigned long)dst, 1, true, __RET_ADDR); 467 *dst = *src; 468 if (*src == '\0') 469 break; 470 src++, dst++; 471 } 472 473 return save; 474 } 475 476 int 477 kasan_strcmp(const char *s1, const char *s2) 478 { 479 while (1) { 480 kasan_shadow_check((unsigned long)s1, 1, false, __RET_ADDR); 481 kasan_shadow_check((unsigned long)s2, 1, false, __RET_ADDR); 482 if (*s1 != *s2) 483 break; 484 if (*s1 == '\0') 485 return 0; 486 s1++, s2++; 487 } 488 489 return (*(const unsigned char *)s1 - *(const unsigned char *)s2); 490 } 491 492 size_t 493 kasan_strlen(const char *str) 494 { 495 const char *s; 496 497 s = str; 498 while (1) { 499 kasan_shadow_check((unsigned long)s, 1, false, __RET_ADDR); 500 if (*s == '\0') 501 break; 502 s++; 503 } 504 505 return (s - str); 506 } 507 508 char * 509 kasan_strcat(char *dst, const char *src) 510 { 511 size_t ldst, lsrc; 512 513 ldst = __builtin_strlen(dst); 514 lsrc = __builtin_strlen(src); 515 kasan_shadow_check((unsigned long)dst, ldst + lsrc + 1, true, 516 __RET_ADDR); 517 kasan_shadow_check((unsigned long)src, lsrc + 1, false, 518 __RET_ADDR); 519 520 return __builtin_strcat(dst, src); 521 } 522 523 char * 524 kasan_strchr(const char *s, int c) 525 { 526 kasan_shadow_check((unsigned long)s, __builtin_strlen(s) + 1, false, 527 __RET_ADDR); 528 return __builtin_strchr(s, c); 529 } 530 531 char * 532 kasan_strrchr(const char *s, int c) 533 { 534 kasan_shadow_check((unsigned long)s, __builtin_strlen(s) + 1, false, 535 __RET_ADDR); 536 return __builtin_strrchr(s, c); 537 } 538 539 #undef kcopy 540 #undef copyinstr 541 #undef copyoutstr 542 #undef copyin 543 544 int kasan_kcopy(const void *, void *, size_t); 545 int kasan_copyinstr(const void *, void *, size_t, size_t *); 546 int kasan_copyoutstr(const void *, void *, size_t, size_t *); 547 int kasan_copyin(const void *, void *, size_t); 548 int kcopy(const void *, void *, size_t); 549 int copyinstr(const void *, void *, size_t, size_t *); 550 int copyoutstr(const void *, void *, size_t, size_t *); 551 int copyin(const void *, void *, size_t); 552 553 int 554 kasan_kcopy(const void *src, void *dst, size_t len) 555 { 556 kasan_shadow_check((unsigned long)src, len, false, __RET_ADDR); 557 kasan_shadow_check((unsigned long)dst, len, true, __RET_ADDR); 558 return kcopy(src, dst, len); 559 } 560 561 int 562 kasan_copyin(const void *uaddr, void *kaddr, size_t len) 563 { 564 kasan_shadow_check((unsigned long)kaddr, len, true, __RET_ADDR); 565 return copyin(uaddr, kaddr, len); 566 } 567 568 int 569 kasan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done) 570 { 571 kasan_shadow_check((unsigned long)kaddr, len, true, __RET_ADDR); 572 return copyinstr(uaddr, kaddr, len, done); 573 } 574 575 int 576 kasan_copyoutstr(const void *kaddr, void *uaddr, size_t len, size_t *done) 577 { 578 kasan_shadow_check((unsigned long)kaddr, len, false, __RET_ADDR); 579 return copyoutstr(kaddr, uaddr, len, done); 580 } 581 582 /* -------------------------------------------------------------------------- */ 583 584 #undef _ucas_32 585 #undef _ucas_32_mp 586 #undef _ucas_64 587 #undef _ucas_64_mp 588 #undef _ufetch_8 589 #undef _ufetch_16 590 #undef _ufetch_32 591 #undef _ufetch_64 592 593 int _ucas_32(volatile uint32_t *, uint32_t, uint32_t, uint32_t *); 594 int kasan__ucas_32(volatile uint32_t *, uint32_t, uint32_t, uint32_t *); 595 int 596 kasan__ucas_32(volatile uint32_t *uaddr, uint32_t old, uint32_t new, 597 uint32_t *ret) 598 { 599 kasan_shadow_check((unsigned long)ret, sizeof(*ret), true, 600 __RET_ADDR); 601 return _ucas_32(uaddr, old, new, ret); 602 } 603 604 #ifdef __HAVE_UCAS_MP 605 int _ucas_32_mp(volatile uint32_t *, uint32_t, uint32_t, uint32_t *); 606 int kasan__ucas_32_mp(volatile uint32_t *, uint32_t, uint32_t, uint32_t *); 607 int 608 kasan__ucas_32_mp(volatile uint32_t *uaddr, uint32_t old, uint32_t new, 609 uint32_t *ret) 610 { 611 kasan_shadow_check((unsigned long)ret, sizeof(*ret), true, 612 __RET_ADDR); 613 return _ucas_32_mp(uaddr, old, new, ret); 614 } 615 #endif 616 617 #ifdef _LP64 618 int _ucas_64(volatile uint64_t *, uint64_t, uint64_t, uint64_t *); 619 int kasan__ucas_64(volatile uint64_t *, uint64_t, uint64_t, uint64_t *); 620 int 621 kasan__ucas_64(volatile uint64_t *uaddr, uint64_t old, uint64_t new, 622 uint64_t *ret) 623 { 624 kasan_shadow_check((unsigned long)ret, sizeof(*ret), true, 625 __RET_ADDR); 626 return _ucas_64(uaddr, old, new, ret); 627 } 628 629 #ifdef __HAVE_UCAS_MP 630 int _ucas_64_mp(volatile uint64_t *, uint64_t, uint64_t, uint64_t *); 631 int kasan__ucas_64_mp(volatile uint64_t *, uint64_t, uint64_t, uint64_t *); 632 int 633 kasan__ucas_64_mp(volatile uint64_t *uaddr, uint64_t old, uint64_t new, 634 uint64_t *ret) 635 { 636 kasan_shadow_check((unsigned long)ret, sizeof(*ret), true, 637 __RET_ADDR); 638 return _ucas_64_mp(uaddr, old, new, ret); 639 } 640 #endif 641 #endif 642 643 int _ufetch_8(const uint8_t *, uint8_t *); 644 int kasan__ufetch_8(const uint8_t *, uint8_t *); 645 int 646 kasan__ufetch_8(const uint8_t *uaddr, uint8_t *valp) 647 { 648 kasan_shadow_check((unsigned long)valp, sizeof(*valp), true, 649 __RET_ADDR); 650 return _ufetch_8(uaddr, valp); 651 } 652 653 int _ufetch_16(const uint16_t *, uint16_t *); 654 int kasan__ufetch_16(const uint16_t *, uint16_t *); 655 int 656 kasan__ufetch_16(const uint16_t *uaddr, uint16_t *valp) 657 { 658 kasan_shadow_check((unsigned long)valp, sizeof(*valp), true, 659 __RET_ADDR); 660 return _ufetch_16(uaddr, valp); 661 } 662 663 int _ufetch_32(const uint32_t *, uint32_t *); 664 int kasan__ufetch_32(const uint32_t *, uint32_t *); 665 int 666 kasan__ufetch_32(const uint32_t *uaddr, uint32_t *valp) 667 { 668 kasan_shadow_check((unsigned long)valp, sizeof(*valp), true, 669 __RET_ADDR); 670 return _ufetch_32(uaddr, valp); 671 } 672 673 #ifdef _LP64 674 int _ufetch_64(const uint64_t *, uint64_t *); 675 int kasan__ufetch_64(const uint64_t *, uint64_t *); 676 int 677 kasan__ufetch_64(const uint64_t *uaddr, uint64_t *valp) 678 { 679 kasan_shadow_check((unsigned long)valp, sizeof(*valp), true, 680 __RET_ADDR); 681 return _ufetch_64(uaddr, valp); 682 } 683 #endif 684 685 /* -------------------------------------------------------------------------- */ 686 687 #undef atomic_add_32 688 #undef atomic_add_int 689 #undef atomic_add_long 690 #undef atomic_add_ptr 691 #undef atomic_add_64 692 #undef atomic_add_32_nv 693 #undef atomic_add_int_nv 694 #undef atomic_add_long_nv 695 #undef atomic_add_ptr_nv 696 #undef atomic_add_64_nv 697 #undef atomic_and_32 698 #undef atomic_and_uint 699 #undef atomic_and_ulong 700 #undef atomic_and_64 701 #undef atomic_and_32_nv 702 #undef atomic_and_uint_nv 703 #undef atomic_and_ulong_nv 704 #undef atomic_and_64_nv 705 #undef atomic_or_32 706 #undef atomic_or_uint 707 #undef atomic_or_ulong 708 #undef atomic_or_64 709 #undef atomic_or_32_nv 710 #undef atomic_or_uint_nv 711 #undef atomic_or_ulong_nv 712 #undef atomic_or_64_nv 713 #undef atomic_cas_32 714 #undef atomic_cas_uint 715 #undef atomic_cas_ulong 716 #undef atomic_cas_ptr 717 #undef atomic_cas_64 718 #undef atomic_cas_32_ni 719 #undef atomic_cas_uint_ni 720 #undef atomic_cas_ulong_ni 721 #undef atomic_cas_ptr_ni 722 #undef atomic_cas_64_ni 723 #undef atomic_swap_32 724 #undef atomic_swap_uint 725 #undef atomic_swap_ulong 726 #undef atomic_swap_ptr 727 #undef atomic_swap_64 728 #undef atomic_dec_32 729 #undef atomic_dec_uint 730 #undef atomic_dec_ulong 731 #undef atomic_dec_ptr 732 #undef atomic_dec_64 733 #undef atomic_dec_32_nv 734 #undef atomic_dec_uint_nv 735 #undef atomic_dec_ulong_nv 736 #undef atomic_dec_ptr_nv 737 #undef atomic_dec_64_nv 738 #undef atomic_inc_32 739 #undef atomic_inc_uint 740 #undef atomic_inc_ulong 741 #undef atomic_inc_ptr 742 #undef atomic_inc_64 743 #undef atomic_inc_32_nv 744 #undef atomic_inc_uint_nv 745 #undef atomic_inc_ulong_nv 746 #undef atomic_inc_ptr_nv 747 #undef atomic_inc_64_nv 748 749 #define ASAN_ATOMIC_FUNC_ADD(name, tret, targ1, targ2) \ 750 void atomic_add_##name(volatile targ1 *, targ2); \ 751 void kasan_atomic_add_##name(volatile targ1 *, targ2); \ 752 void kasan_atomic_add_##name(volatile targ1 *ptr, targ2 val) \ 753 { \ 754 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \ 755 __RET_ADDR); \ 756 atomic_add_##name(ptr, val); \ 757 } \ 758 tret atomic_add_##name##_nv(volatile targ1 *, targ2); \ 759 tret kasan_atomic_add_##name##_nv(volatile targ1 *, targ2); \ 760 tret kasan_atomic_add_##name##_nv(volatile targ1 *ptr, targ2 val) \ 761 { \ 762 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \ 763 __RET_ADDR); \ 764 return atomic_add_##name##_nv(ptr, val); \ 765 } 766 767 #define ASAN_ATOMIC_FUNC_AND(name, tret, targ1, targ2) \ 768 void atomic_and_##name(volatile targ1 *, targ2); \ 769 void kasan_atomic_and_##name(volatile targ1 *, targ2); \ 770 void kasan_atomic_and_##name(volatile targ1 *ptr, targ2 val) \ 771 { \ 772 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \ 773 __RET_ADDR); \ 774 atomic_and_##name(ptr, val); \ 775 } \ 776 tret atomic_and_##name##_nv(volatile targ1 *, targ2); \ 777 tret kasan_atomic_and_##name##_nv(volatile targ1 *, targ2); \ 778 tret kasan_atomic_and_##name##_nv(volatile targ1 *ptr, targ2 val) \ 779 { \ 780 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \ 781 __RET_ADDR); \ 782 return atomic_and_##name##_nv(ptr, val); \ 783 } 784 785 #define ASAN_ATOMIC_FUNC_OR(name, tret, targ1, targ2) \ 786 void atomic_or_##name(volatile targ1 *, targ2); \ 787 void kasan_atomic_or_##name(volatile targ1 *, targ2); \ 788 void kasan_atomic_or_##name(volatile targ1 *ptr, targ2 val) \ 789 { \ 790 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \ 791 __RET_ADDR); \ 792 atomic_or_##name(ptr, val); \ 793 } \ 794 tret atomic_or_##name##_nv(volatile targ1 *, targ2); \ 795 tret kasan_atomic_or_##name##_nv(volatile targ1 *, targ2); \ 796 tret kasan_atomic_or_##name##_nv(volatile targ1 *ptr, targ2 val) \ 797 { \ 798 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \ 799 __RET_ADDR); \ 800 return atomic_or_##name##_nv(ptr, val); \ 801 } 802 803 #define ASAN_ATOMIC_FUNC_CAS(name, tret, targ1, targ2) \ 804 tret atomic_cas_##name(volatile targ1 *, targ2, targ2); \ 805 tret kasan_atomic_cas_##name(volatile targ1 *, targ2, targ2); \ 806 tret kasan_atomic_cas_##name(volatile targ1 *ptr, targ2 exp, targ2 new) \ 807 { \ 808 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \ 809 __RET_ADDR); \ 810 return atomic_cas_##name(ptr, exp, new); \ 811 } \ 812 tret atomic_cas_##name##_ni(volatile targ1 *, targ2, targ2); \ 813 tret kasan_atomic_cas_##name##_ni(volatile targ1 *, targ2, targ2); \ 814 tret kasan_atomic_cas_##name##_ni(volatile targ1 *ptr, targ2 exp, targ2 new) \ 815 { \ 816 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \ 817 __RET_ADDR); \ 818 return atomic_cas_##name##_ni(ptr, exp, new); \ 819 } 820 821 #define ASAN_ATOMIC_FUNC_SWAP(name, tret, targ1, targ2) \ 822 tret atomic_swap_##name(volatile targ1 *, targ2); \ 823 tret kasan_atomic_swap_##name(volatile targ1 *, targ2); \ 824 tret kasan_atomic_swap_##name(volatile targ1 *ptr, targ2 val) \ 825 { \ 826 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \ 827 __RET_ADDR); \ 828 return atomic_swap_##name(ptr, val); \ 829 } 830 831 #define ASAN_ATOMIC_FUNC_DEC(name, tret, targ1) \ 832 void atomic_dec_##name(volatile targ1 *); \ 833 void kasan_atomic_dec_##name(volatile targ1 *); \ 834 void kasan_atomic_dec_##name(volatile targ1 *ptr) \ 835 { \ 836 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \ 837 __RET_ADDR); \ 838 atomic_dec_##name(ptr); \ 839 } \ 840 tret atomic_dec_##name##_nv(volatile targ1 *); \ 841 tret kasan_atomic_dec_##name##_nv(volatile targ1 *); \ 842 tret kasan_atomic_dec_##name##_nv(volatile targ1 *ptr) \ 843 { \ 844 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \ 845 __RET_ADDR); \ 846 return atomic_dec_##name##_nv(ptr); \ 847 } 848 849 #define ASAN_ATOMIC_FUNC_INC(name, tret, targ1) \ 850 void atomic_inc_##name(volatile targ1 *); \ 851 void kasan_atomic_inc_##name(volatile targ1 *); \ 852 void kasan_atomic_inc_##name(volatile targ1 *ptr) \ 853 { \ 854 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \ 855 __RET_ADDR); \ 856 atomic_inc_##name(ptr); \ 857 } \ 858 tret atomic_inc_##name##_nv(volatile targ1 *); \ 859 tret kasan_atomic_inc_##name##_nv(volatile targ1 *); \ 860 tret kasan_atomic_inc_##name##_nv(volatile targ1 *ptr) \ 861 { \ 862 kasan_shadow_check((uintptr_t)ptr, sizeof(tret), true, \ 863 __RET_ADDR); \ 864 return atomic_inc_##name##_nv(ptr); \ 865 } 866 867 ASAN_ATOMIC_FUNC_ADD(32, uint32_t, uint32_t, int32_t); 868 ASAN_ATOMIC_FUNC_ADD(64, uint64_t, uint64_t, int64_t); 869 ASAN_ATOMIC_FUNC_ADD(int, unsigned int, unsigned int, int); 870 ASAN_ATOMIC_FUNC_ADD(long, unsigned long, unsigned long, long); 871 ASAN_ATOMIC_FUNC_ADD(ptr, void *, void, ssize_t); 872 873 ASAN_ATOMIC_FUNC_AND(32, uint32_t, uint32_t, uint32_t); 874 ASAN_ATOMIC_FUNC_AND(64, uint64_t, uint64_t, uint64_t); 875 ASAN_ATOMIC_FUNC_AND(uint, unsigned int, unsigned int, unsigned int); 876 ASAN_ATOMIC_FUNC_AND(ulong, unsigned long, unsigned long, unsigned long); 877 878 ASAN_ATOMIC_FUNC_OR(32, uint32_t, uint32_t, uint32_t); 879 ASAN_ATOMIC_FUNC_OR(64, uint64_t, uint64_t, uint64_t); 880 ASAN_ATOMIC_FUNC_OR(uint, unsigned int, unsigned int, unsigned int); 881 ASAN_ATOMIC_FUNC_OR(ulong, unsigned long, unsigned long, unsigned long); 882 883 ASAN_ATOMIC_FUNC_CAS(32, uint32_t, uint32_t, uint32_t); 884 ASAN_ATOMIC_FUNC_CAS(64, uint64_t, uint64_t, uint64_t); 885 ASAN_ATOMIC_FUNC_CAS(uint, unsigned int, unsigned int, unsigned int); 886 ASAN_ATOMIC_FUNC_CAS(ulong, unsigned long, unsigned long, unsigned long); 887 ASAN_ATOMIC_FUNC_CAS(ptr, void *, void, void *); 888 889 ASAN_ATOMIC_FUNC_SWAP(32, uint32_t, uint32_t, uint32_t); 890 ASAN_ATOMIC_FUNC_SWAP(64, uint64_t, uint64_t, uint64_t); 891 ASAN_ATOMIC_FUNC_SWAP(uint, unsigned int, unsigned int, unsigned int); 892 ASAN_ATOMIC_FUNC_SWAP(ulong, unsigned long, unsigned long, unsigned long); 893 ASAN_ATOMIC_FUNC_SWAP(ptr, void *, void, void *); 894 895 ASAN_ATOMIC_FUNC_DEC(32, uint32_t, uint32_t) 896 ASAN_ATOMIC_FUNC_DEC(64, uint64_t, uint64_t) 897 ASAN_ATOMIC_FUNC_DEC(uint, unsigned int, unsigned int); 898 ASAN_ATOMIC_FUNC_DEC(ulong, unsigned long, unsigned long); 899 ASAN_ATOMIC_FUNC_DEC(ptr, void *, void); 900 901 ASAN_ATOMIC_FUNC_INC(32, uint32_t, uint32_t) 902 ASAN_ATOMIC_FUNC_INC(64, uint64_t, uint64_t) 903 ASAN_ATOMIC_FUNC_INC(uint, unsigned int, unsigned int); 904 ASAN_ATOMIC_FUNC_INC(ulong, unsigned long, unsigned long); 905 ASAN_ATOMIC_FUNC_INC(ptr, void *, void); 906 907 /* -------------------------------------------------------------------------- */ 908 909 #ifdef __HAVE_KASAN_INSTR_BUS 910 911 #include <sys/bus.h> 912 913 #undef bus_space_read_multi_1 914 #undef bus_space_read_multi_2 915 #undef bus_space_read_multi_4 916 #undef bus_space_read_multi_8 917 #undef bus_space_read_multi_stream_1 918 #undef bus_space_read_multi_stream_2 919 #undef bus_space_read_multi_stream_4 920 #undef bus_space_read_multi_stream_8 921 #undef bus_space_read_region_1 922 #undef bus_space_read_region_2 923 #undef bus_space_read_region_4 924 #undef bus_space_read_region_8 925 #undef bus_space_read_region_stream_1 926 #undef bus_space_read_region_stream_2 927 #undef bus_space_read_region_stream_4 928 #undef bus_space_read_region_stream_8 929 #undef bus_space_write_multi_1 930 #undef bus_space_write_multi_2 931 #undef bus_space_write_multi_4 932 #undef bus_space_write_multi_8 933 #undef bus_space_write_multi_stream_1 934 #undef bus_space_write_multi_stream_2 935 #undef bus_space_write_multi_stream_4 936 #undef bus_space_write_multi_stream_8 937 #undef bus_space_write_region_1 938 #undef bus_space_write_region_2 939 #undef bus_space_write_region_4 940 #undef bus_space_write_region_8 941 #undef bus_space_write_region_stream_1 942 #undef bus_space_write_region_stream_2 943 #undef bus_space_write_region_stream_4 944 #undef bus_space_write_region_stream_8 945 946 #define ASAN_BUS_READ_FUNC(bytes, bits) \ 947 void bus_space_read_multi_##bytes(bus_space_tag_t, bus_space_handle_t, \ 948 bus_size_t, uint##bits##_t *, bus_size_t); \ 949 void kasan_bus_space_read_multi_##bytes(bus_space_tag_t, \ 950 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 951 void kasan_bus_space_read_multi_##bytes(bus_space_tag_t tag, \ 952 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \ 953 bus_size_t count) \ 954 { \ 955 kasan_shadow_check((uintptr_t)buf, \ 956 sizeof(uint##bits##_t) * count, false, __RET_ADDR); \ 957 bus_space_read_multi_##bytes(tag, hnd, size, buf, count); \ 958 } \ 959 void bus_space_read_multi_stream_##bytes(bus_space_tag_t, \ 960 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 961 void kasan_bus_space_read_multi_stream_##bytes(bus_space_tag_t, \ 962 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 963 void kasan_bus_space_read_multi_stream_##bytes(bus_space_tag_t tag, \ 964 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \ 965 bus_size_t count) \ 966 { \ 967 kasan_shadow_check((uintptr_t)buf, \ 968 sizeof(uint##bits##_t) * count, false, __RET_ADDR); \ 969 bus_space_read_multi_stream_##bytes(tag, hnd, size, buf, count);\ 970 } \ 971 void bus_space_read_region_##bytes(bus_space_tag_t, bus_space_handle_t, \ 972 bus_size_t, uint##bits##_t *, bus_size_t); \ 973 void kasan_bus_space_read_region_##bytes(bus_space_tag_t, \ 974 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 975 void kasan_bus_space_read_region_##bytes(bus_space_tag_t tag, \ 976 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \ 977 bus_size_t count) \ 978 { \ 979 kasan_shadow_check((uintptr_t)buf, \ 980 sizeof(uint##bits##_t) * count, false, __RET_ADDR); \ 981 bus_space_read_region_##bytes(tag, hnd, size, buf, count); \ 982 } \ 983 void bus_space_read_region_stream_##bytes(bus_space_tag_t, \ 984 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 985 void kasan_bus_space_read_region_stream_##bytes(bus_space_tag_t, \ 986 bus_space_handle_t, bus_size_t, uint##bits##_t *, bus_size_t); \ 987 void kasan_bus_space_read_region_stream_##bytes(bus_space_tag_t tag, \ 988 bus_space_handle_t hnd, bus_size_t size, uint##bits##_t *buf, \ 989 bus_size_t count) \ 990 { \ 991 kasan_shadow_check((uintptr_t)buf, \ 992 sizeof(uint##bits##_t) * count, false, __RET_ADDR); \ 993 bus_space_read_region_stream_##bytes(tag, hnd, size, buf, count);\ 994 } 995 996 #define ASAN_BUS_WRITE_FUNC(bytes, bits) \ 997 void bus_space_write_multi_##bytes(bus_space_tag_t, bus_space_handle_t, \ 998 bus_size_t, const uint##bits##_t *, bus_size_t); \ 999 void kasan_bus_space_write_multi_##bytes(bus_space_tag_t, \ 1000 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 1001 void kasan_bus_space_write_multi_##bytes(bus_space_tag_t tag, \ 1002 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \ 1003 bus_size_t count) \ 1004 { \ 1005 kasan_shadow_check((uintptr_t)buf, \ 1006 sizeof(uint##bits##_t) * count, true, __RET_ADDR); \ 1007 bus_space_write_multi_##bytes(tag, hnd, size, buf, count); \ 1008 } \ 1009 void bus_space_write_multi_stream_##bytes(bus_space_tag_t, \ 1010 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 1011 void kasan_bus_space_write_multi_stream_##bytes(bus_space_tag_t, \ 1012 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 1013 void kasan_bus_space_write_multi_stream_##bytes(bus_space_tag_t tag, \ 1014 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \ 1015 bus_size_t count) \ 1016 { \ 1017 kasan_shadow_check((uintptr_t)buf, \ 1018 sizeof(uint##bits##_t) * count, true, __RET_ADDR); \ 1019 bus_space_write_multi_stream_##bytes(tag, hnd, size, buf, count);\ 1020 } \ 1021 void bus_space_write_region_##bytes(bus_space_tag_t, bus_space_handle_t,\ 1022 bus_size_t, const uint##bits##_t *, bus_size_t); \ 1023 void kasan_bus_space_write_region_##bytes(bus_space_tag_t, \ 1024 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 1025 void kasan_bus_space_write_region_##bytes(bus_space_tag_t tag, \ 1026 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \ 1027 bus_size_t count) \ 1028 { \ 1029 kasan_shadow_check((uintptr_t)buf, \ 1030 sizeof(uint##bits##_t) * count, true, __RET_ADDR); \ 1031 bus_space_write_region_##bytes(tag, hnd, size, buf, count); \ 1032 } \ 1033 void bus_space_write_region_stream_##bytes(bus_space_tag_t, \ 1034 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 1035 void kasan_bus_space_write_region_stream_##bytes(bus_space_tag_t, \ 1036 bus_space_handle_t, bus_size_t, const uint##bits##_t *, bus_size_t);\ 1037 void kasan_bus_space_write_region_stream_##bytes(bus_space_tag_t tag, \ 1038 bus_space_handle_t hnd, bus_size_t size, const uint##bits##_t *buf, \ 1039 bus_size_t count) \ 1040 { \ 1041 kasan_shadow_check((uintptr_t)buf, \ 1042 sizeof(uint##bits##_t) * count, true, __RET_ADDR); \ 1043 bus_space_write_region_stream_##bytes(tag, hnd, size, buf, count);\ 1044 } 1045 1046 ASAN_BUS_READ_FUNC(1, 8) 1047 ASAN_BUS_READ_FUNC(2, 16) 1048 ASAN_BUS_READ_FUNC(4, 32) 1049 ASAN_BUS_READ_FUNC(8, 64) 1050 1051 ASAN_BUS_WRITE_FUNC(1, 8) 1052 ASAN_BUS_WRITE_FUNC(2, 16) 1053 ASAN_BUS_WRITE_FUNC(4, 32) 1054 ASAN_BUS_WRITE_FUNC(8, 64) 1055 1056 #endif /* __HAVE_KASAN_INSTR_BUS */ 1057 1058 /* -------------------------------------------------------------------------- */ 1059 1060 #include <sys/mbuf.h> 1061 1062 static void 1063 kasan_dma_sync_linear(uint8_t *buf, bus_addr_t offset, bus_size_t len, 1064 bool write, uintptr_t pc) 1065 { 1066 kasan_shadow_check((uintptr_t)(buf + offset), len, write, pc); 1067 } 1068 1069 static void 1070 kasan_dma_sync_mbuf(struct mbuf *m, bus_addr_t offset, bus_size_t len, 1071 bool write, uintptr_t pc) 1072 { 1073 bus_addr_t minlen; 1074 1075 for (; m != NULL && len != 0; m = m->m_next) { 1076 kasan_shadow_check((uintptr_t)m, sizeof(*m), false, pc); 1077 1078 if (offset >= m->m_len) { 1079 offset -= m->m_len; 1080 continue; 1081 } 1082 1083 minlen = MIN(len, m->m_len - offset); 1084 kasan_shadow_check((uintptr_t)(mtod(m, char *) + offset), 1085 minlen, write, pc); 1086 1087 offset = 0; 1088 len -= minlen; 1089 } 1090 } 1091 1092 static void 1093 kasan_dma_sync_uio(struct uio *uio, bus_addr_t offset, bus_size_t len, 1094 bool write, uintptr_t pc) 1095 { 1096 bus_size_t minlen, resid; 1097 struct iovec *iov; 1098 int i; 1099 1100 kasan_shadow_check((uintptr_t)uio, sizeof(struct uio), false, pc); 1101 1102 if (!VMSPACE_IS_KERNEL_P(uio->uio_vmspace)) 1103 return; 1104 1105 resid = uio->uio_resid; 1106 iov = uio->uio_iov; 1107 1108 for (i = 0; i < uio->uio_iovcnt && resid != 0; i++) { 1109 kasan_shadow_check((uintptr_t)&iov[i], sizeof(iov[i]), 1110 false, pc); 1111 minlen = MIN(resid, iov[i].iov_len); 1112 kasan_shadow_check((uintptr_t)iov[i].iov_base, minlen, 1113 write, pc); 1114 resid -= minlen; 1115 } 1116 } 1117 1118 void 1119 kasan_dma_sync(bus_dmamap_t map, bus_addr_t offset, bus_size_t len, int ops) 1120 { 1121 bool write = (ops & (BUS_DMASYNC_PREWRITE|BUS_DMASYNC_POSTWRITE)) != 0; 1122 1123 switch (map->dm_buftype) { 1124 case KASAN_DMA_LINEAR: 1125 kasan_dma_sync_linear(map->dm_buf, offset, len, write, 1126 __RET_ADDR); 1127 break; 1128 case KASAN_DMA_MBUF: 1129 kasan_dma_sync_mbuf(map->dm_buf, offset, len, write, 1130 __RET_ADDR); 1131 break; 1132 case KASAN_DMA_UIO: 1133 kasan_dma_sync_uio(map->dm_buf, offset, len, write, 1134 __RET_ADDR); 1135 break; 1136 case KASAN_DMA_RAW: 1137 break; 1138 default: 1139 panic("%s: impossible", __func__); 1140 } 1141 } 1142 1143 void 1144 kasan_dma_load(bus_dmamap_t map, void *buf, bus_size_t buflen, int type) 1145 { 1146 map->dm_buf = buf; 1147 map->dm_buflen = buflen; 1148 map->dm_buftype = type; 1149 } 1150 1151 /* -------------------------------------------------------------------------- */ 1152 1153 void __asan_register_globals(struct __asan_global *, size_t); 1154 void __asan_unregister_globals(struct __asan_global *, size_t); 1155 1156 void 1157 __asan_register_globals(struct __asan_global *globals, size_t n) 1158 { 1159 size_t i; 1160 1161 for (i = 0; i < n; i++) { 1162 kasan_mark(globals[i].beg, globals[i].size, 1163 globals[i].size_with_redzone, KASAN_GENERIC_REDZONE); 1164 } 1165 } 1166 1167 void 1168 __asan_unregister_globals(struct __asan_global *globals, size_t n) 1169 { 1170 /* never called */ 1171 } 1172 1173 #define ASAN_LOAD_STORE(size) \ 1174 void __asan_load##size(unsigned long); \ 1175 void __asan_load##size(unsigned long addr) \ 1176 { \ 1177 kasan_shadow_check(addr, size, false, __RET_ADDR);\ 1178 } \ 1179 void __asan_load##size##_noabort(unsigned long); \ 1180 void __asan_load##size##_noabort(unsigned long addr) \ 1181 { \ 1182 kasan_shadow_check(addr, size, false, __RET_ADDR);\ 1183 } \ 1184 void __asan_store##size(unsigned long); \ 1185 void __asan_store##size(unsigned long addr) \ 1186 { \ 1187 kasan_shadow_check(addr, size, true, __RET_ADDR);\ 1188 } \ 1189 void __asan_store##size##_noabort(unsigned long); \ 1190 void __asan_store##size##_noabort(unsigned long addr) \ 1191 { \ 1192 kasan_shadow_check(addr, size, true, __RET_ADDR);\ 1193 } 1194 1195 ASAN_LOAD_STORE(1); 1196 ASAN_LOAD_STORE(2); 1197 ASAN_LOAD_STORE(4); 1198 ASAN_LOAD_STORE(8); 1199 ASAN_LOAD_STORE(16); 1200 1201 void __asan_loadN(unsigned long, size_t); 1202 void __asan_loadN_noabort(unsigned long, size_t); 1203 void __asan_storeN(unsigned long, size_t); 1204 void __asan_storeN_noabort(unsigned long, size_t); 1205 void __asan_handle_no_return(void); 1206 1207 void 1208 __asan_loadN(unsigned long addr, size_t size) 1209 { 1210 kasan_shadow_check(addr, size, false, __RET_ADDR); 1211 } 1212 1213 void 1214 __asan_loadN_noabort(unsigned long addr, size_t size) 1215 { 1216 kasan_shadow_check(addr, size, false, __RET_ADDR); 1217 } 1218 1219 void 1220 __asan_storeN(unsigned long addr, size_t size) 1221 { 1222 kasan_shadow_check(addr, size, true, __RET_ADDR); 1223 } 1224 1225 void 1226 __asan_storeN_noabort(unsigned long addr, size_t size) 1227 { 1228 kasan_shadow_check(addr, size, true, __RET_ADDR); 1229 } 1230 1231 void 1232 __asan_handle_no_return(void) 1233 { 1234 /* nothing */ 1235 } 1236 1237 #define ASAN_SET_SHADOW(byte) \ 1238 void __asan_set_shadow_##byte(void *, size_t); \ 1239 void __asan_set_shadow_##byte(void *addr, size_t size) \ 1240 { \ 1241 __builtin_memset((void *)addr, 0x##byte, size); \ 1242 } 1243 1244 ASAN_SET_SHADOW(00); 1245 ASAN_SET_SHADOW(f1); 1246 ASAN_SET_SHADOW(f2); 1247 ASAN_SET_SHADOW(f3); 1248 ASAN_SET_SHADOW(f5); 1249 ASAN_SET_SHADOW(f8); 1250 1251 void __asan_poison_stack_memory(const void *, size_t); 1252 void __asan_unpoison_stack_memory(const void *, size_t); 1253 1254 void 1255 __asan_poison_stack_memory(const void *addr, size_t size) 1256 { 1257 size = roundup(size, KASAN_SHADOW_SCALE_SIZE); 1258 kasan_shadow_Nbyte_fill(addr, size, KASAN_USE_AFTER_SCOPE); 1259 } 1260 1261 void 1262 __asan_unpoison_stack_memory(const void *addr, size_t size) 1263 { 1264 kasan_shadow_Nbyte_markvalid(addr, size); 1265 } 1266 1267 void __asan_alloca_poison(const void *, size_t); 1268 void __asan_allocas_unpoison(const void *, const void *); 1269 1270 void __asan_alloca_poison(const void *addr, size_t size) 1271 { 1272 const void *l, *r; 1273 1274 KASSERT((vaddr_t)addr % KASAN_ALLOCA_SCALE_SIZE == 0); 1275 1276 l = (const uint8_t *)addr - KASAN_ALLOCA_SCALE_SIZE; 1277 r = (const uint8_t *)addr + roundup(size, KASAN_ALLOCA_SCALE_SIZE); 1278 1279 kasan_shadow_Nbyte_fill(l, KASAN_ALLOCA_SCALE_SIZE, KASAN_STACK_LEFT); 1280 kasan_mark(addr, size, roundup(size, KASAN_ALLOCA_SCALE_SIZE), 1281 KASAN_STACK_MID); 1282 kasan_shadow_Nbyte_fill(r, KASAN_ALLOCA_SCALE_SIZE, KASAN_STACK_RIGHT); 1283 } 1284 1285 void __asan_allocas_unpoison(const void *stkbegin, const void *stkend) 1286 { 1287 size_t size; 1288 1289 if (__predict_false(!stkbegin)) 1290 return; 1291 if (__predict_false((uintptr_t)stkbegin > (uintptr_t)stkend)) 1292 return; 1293 size = (uintptr_t)stkend - (uintptr_t)stkbegin; 1294 1295 kasan_shadow_Nbyte_fill(stkbegin, size, 0); 1296 } 1297