1 /* $NetBSD: subr_kcov.c,v 1.8 2019/05/26 05:41:45 kamil Exp $ */ 2 3 /* 4 * Copyright (c) 2019 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Siddharth Muralee. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 #include <sys/cdefs.h> 33 34 #include <sys/module.h> 35 #include <sys/param.h> 36 #include <sys/systm.h> 37 #include <sys/kernel.h> 38 39 #include <sys/conf.h> 40 #include <sys/condvar.h> 41 #include <sys/file.h> 42 #include <sys/filedesc.h> 43 #include <sys/kmem.h> 44 #include <sys/mman.h> 45 #include <sys/mutex.h> 46 #include <sys/queue.h> 47 48 #include <uvm/uvm_extern.h> 49 #include <sys/kcov.h> 50 51 #define KCOV_BUF_MAX_ENTRIES (256 << 10) 52 53 #define KCOV_CMP_CONST 1 54 #define KCOV_CMP_SIZE(x) ((x) << 1) 55 56 static dev_type_open(kcov_open); 57 58 const struct cdevsw kcov_cdevsw = { 59 .d_open = kcov_open, 60 .d_close = noclose, 61 .d_read = noread, 62 .d_write = nowrite, 63 .d_ioctl = noioctl, 64 .d_stop = nostop, 65 .d_tty = notty, 66 .d_poll = nopoll, 67 .d_mmap = nommap, 68 .d_kqfilter = nokqfilter, 69 .d_discard = nodiscard, 70 .d_flag = D_OTHER | D_MPSAFE 71 }; 72 73 static int kcov_fops_ioctl(file_t *, u_long, void *); 74 static int kcov_fops_close(file_t *); 75 static int kcov_fops_mmap(file_t *, off_t *, size_t, int, int *, int *, 76 struct uvm_object **, int *); 77 78 const struct fileops kcov_fileops = { 79 .fo_read = fbadop_read, 80 .fo_write = fbadop_write, 81 .fo_ioctl = kcov_fops_ioctl, 82 .fo_fcntl = fnullop_fcntl, 83 .fo_poll = fnullop_poll, 84 .fo_stat = fbadop_stat, 85 .fo_close = kcov_fops_close, 86 .fo_kqfilter = fnullop_kqfilter, 87 .fo_restart = fnullop_restart, 88 .fo_mmap = kcov_fops_mmap, 89 }; 90 91 /* 92 * The KCOV descriptors (KD) are allocated during open(), and are associated 93 * with a file descriptor. 94 * 95 * An LWP can 'enable' a KD. When this happens, this LWP becomes the owner of 96 * the KD, and no LWP can 'disable' this KD except the owner. 97 * 98 * A KD is freed when its file descriptor is closed _iff_ the KD is not active 99 * on an LWP. If it is, we ask the LWP to free it when it exits. 100 * 101 * The buffers mmapped are in a dedicated uobj, therefore there is no risk 102 * that the kernel frees a buffer still mmapped in a process: the uobj 103 * refcount will be non-zero, so the backing is not freed until an munmap 104 * occurs on said process. 105 */ 106 107 typedef struct kcov_desc { 108 kmutex_t lock; 109 kcov_int_t *buf; 110 struct uvm_object *uobj; 111 size_t bufnent; 112 size_t bufsize; 113 int mode; 114 bool enabled; 115 bool lwpfree; 116 } kcov_t; 117 118 static specificdata_key_t kcov_lwp_key; 119 120 static void 121 kcov_lock(kcov_t *kd) 122 { 123 124 mutex_enter(&kd->lock); 125 } 126 127 static void 128 kcov_unlock(kcov_t *kd) 129 { 130 131 mutex_exit(&kd->lock); 132 } 133 134 static void 135 kcov_free(kcov_t *kd) 136 { 137 138 KASSERT(kd != NULL); 139 if (kd->buf != NULL) { 140 uvm_deallocate(kernel_map, (vaddr_t)kd->buf, kd->bufsize); 141 } 142 mutex_destroy(&kd->lock); 143 kmem_free(kd, sizeof(*kd)); 144 } 145 146 static void 147 kcov_lwp_free(void *arg) 148 { 149 kcov_t *kd = (kcov_t *)arg; 150 151 if (kd == NULL) { 152 return; 153 } 154 kcov_lock(kd); 155 kd->enabled = false; 156 kcov_unlock(kd); 157 if (kd->lwpfree) { 158 kcov_free(kd); 159 } 160 } 161 162 static int 163 kcov_allocbuf(kcov_t *kd, uint64_t nent) 164 { 165 size_t size; 166 int error; 167 168 if (nent < 2 || nent > KCOV_BUF_MAX_ENTRIES) 169 return EINVAL; 170 if (kd->buf != NULL) 171 return EEXIST; 172 173 size = roundup(nent * KCOV_ENTRY_SIZE, PAGE_SIZE); 174 kd->bufnent = nent - 1; 175 kd->bufsize = size; 176 kd->uobj = uao_create(kd->bufsize, 0); 177 178 /* Map the uobj into the kernel address space, as wired. */ 179 kd->buf = NULL; 180 error = uvm_map(kernel_map, (vaddr_t *)&kd->buf, kd->bufsize, kd->uobj, 181 0, 0, UVM_MAPFLAG(UVM_PROT_RW, UVM_PROT_RW, UVM_INH_SHARE, 182 UVM_ADV_RANDOM, 0)); 183 if (error) { 184 uao_detach(kd->uobj); 185 return error; 186 } 187 error = uvm_map_pageable(kernel_map, (vaddr_t)kd->buf, 188 (vaddr_t)kd->buf + size, false, 0); 189 if (error) { 190 uvm_deallocate(kernel_map, (vaddr_t)kd->buf, size); 191 return error; 192 } 193 194 return 0; 195 } 196 197 /* -------------------------------------------------------------------------- */ 198 199 static int 200 kcov_open(dev_t dev, int flag, int mode, struct lwp *l) 201 { 202 struct file *fp; 203 int error, fd; 204 kcov_t *kd; 205 206 error = fd_allocfile(&fp, &fd); 207 if (error) 208 return error; 209 210 kd = kmem_zalloc(sizeof(*kd), KM_SLEEP); 211 mutex_init(&kd->lock, MUTEX_DEFAULT, IPL_NONE); 212 213 return fd_clone(fp, fd, flag, &kcov_fileops, kd); 214 } 215 216 static int 217 kcov_fops_close(file_t *fp) 218 { 219 kcov_t *kd = fp->f_data; 220 221 kcov_lock(kd); 222 if (kd->enabled) { 223 kd->lwpfree = true; 224 kcov_unlock(kd); 225 } else { 226 kcov_unlock(kd); 227 kcov_free(kd); 228 } 229 fp->f_data = NULL; 230 231 return 0; 232 } 233 234 static int 235 kcov_fops_ioctl(file_t *fp, u_long cmd, void *addr) 236 { 237 int error = 0; 238 int mode; 239 kcov_t *kd; 240 241 kd = fp->f_data; 242 if (kd == NULL) 243 return ENXIO; 244 kcov_lock(kd); 245 246 switch (cmd) { 247 case KCOV_IOC_SETBUFSIZE: 248 if (kd->enabled) { 249 error = EBUSY; 250 break; 251 } 252 error = kcov_allocbuf(kd, *((uint64_t *)addr)); 253 break; 254 case KCOV_IOC_ENABLE: 255 if (kd->enabled) { 256 error = EBUSY; 257 break; 258 } 259 if (lwp_getspecific(kcov_lwp_key) != NULL) { 260 error = EBUSY; 261 break; 262 } 263 if (kd->buf == NULL) { 264 error = ENOBUFS; 265 break; 266 } 267 268 mode = *((int *)addr); 269 switch (mode) { 270 case KCOV_MODE_NONE: 271 case KCOV_MODE_TRACE_PC: 272 case KCOV_MODE_TRACE_CMP: 273 kd->mode = mode; 274 break; 275 default: 276 error = EINVAL; 277 } 278 if (error) 279 break; 280 281 lwp_setspecific(kcov_lwp_key, kd); 282 kd->enabled = true; 283 break; 284 case KCOV_IOC_DISABLE: 285 if (!kd->enabled) { 286 error = ENOENT; 287 break; 288 } 289 if (lwp_getspecific(kcov_lwp_key) != kd) { 290 error = ENOENT; 291 break; 292 } 293 lwp_setspecific(kcov_lwp_key, NULL); 294 kd->enabled = false; 295 break; 296 default: 297 error = EINVAL; 298 } 299 300 kcov_unlock(kd); 301 return error; 302 } 303 304 static int 305 kcov_fops_mmap(file_t *fp, off_t *offp, size_t size, int prot, int *flagsp, 306 int *advicep, struct uvm_object **uobjp, int *maxprotp) 307 { 308 off_t off = *offp; 309 kcov_t *kd; 310 int error = 0; 311 312 if (prot & PROT_EXEC) 313 return EACCES; 314 if (off < 0) 315 return EINVAL; 316 if (size > KCOV_BUF_MAX_ENTRIES * KCOV_ENTRY_SIZE) 317 return EINVAL; 318 if (off > KCOV_BUF_MAX_ENTRIES * KCOV_ENTRY_SIZE) 319 return EINVAL; 320 321 kd = fp->f_data; 322 if (kd == NULL) 323 return ENXIO; 324 kcov_lock(kd); 325 326 if ((size + off) > kd->bufsize) { 327 error = ENOMEM; 328 goto out; 329 } 330 331 uao_reference(kd->uobj); 332 333 *uobjp = kd->uobj; 334 *maxprotp = prot; 335 *advicep = UVM_ADV_RANDOM; 336 337 out: 338 kcov_unlock(kd); 339 return error; 340 } 341 342 static inline bool 343 in_interrupt(void) 344 { 345 return curcpu()->ci_idepth >= 0; 346 } 347 348 void __sanitizer_cov_trace_pc(void); 349 350 void 351 __sanitizer_cov_trace_pc(void) 352 { 353 extern int cold; 354 uint64_t idx; 355 kcov_t *kd; 356 357 if (__predict_false(cold)) { 358 /* Do not trace during boot. */ 359 return; 360 } 361 362 if (in_interrupt()) { 363 /* Do not trace in interrupts. */ 364 return; 365 } 366 367 kd = lwp_getspecific(kcov_lwp_key); 368 if (__predict_true(kd == NULL)) { 369 /* Not traced. */ 370 return; 371 } 372 373 if (!kd->enabled) { 374 /* Tracing not enabled */ 375 return; 376 } 377 378 if (kd->mode != KCOV_MODE_TRACE_PC) { 379 /* PC tracing mode not enabled */ 380 return; 381 } 382 383 idx = kd->buf[0]; 384 if (idx < kd->bufnent) { 385 kd->buf[idx+1] = 386 (intptr_t)__builtin_return_address(0); 387 kd->buf[0] = idx + 1; 388 } 389 } 390 391 static void 392 trace_cmp(uint64_t type, uint64_t arg1, uint64_t arg2, intptr_t pc) 393 { 394 extern int cold; 395 uint64_t idx; 396 kcov_t *kd; 397 398 if (__predict_false(cold)) { 399 /* Do not trace during boot. */ 400 return; 401 } 402 403 if (in_interrupt()) { 404 /* Do not trace in interrupts. */ 405 return; 406 } 407 408 kd = lwp_getspecific(kcov_lwp_key); 409 if (__predict_true(kd == NULL)) { 410 /* Not traced. */ 411 return; 412 } 413 414 if (!kd->enabled) { 415 /* Tracing not enabled */ 416 return; 417 } 418 419 if (kd->mode != KCOV_MODE_TRACE_CMP) { 420 /* CMP tracing mode not enabled */ 421 return; 422 } 423 424 idx = kd->buf[0]; 425 if ((idx * 4 + 4) <= kd->bufnent) { 426 kd->buf[idx * 4 + 1] = type; 427 kd->buf[idx * 4 + 2] = arg1; 428 kd->buf[idx * 4 + 3] = arg2; 429 kd->buf[idx * 4 + 4] = pc; 430 kd->buf[0] = idx + 1; 431 } 432 } 433 434 void __sanitizer_cov_trace_cmp1(uint8_t arg1, uint8_t arg2); 435 436 void 437 __sanitizer_cov_trace_cmp1(uint8_t arg1, uint8_t arg2) 438 { 439 440 trace_cmp(KCOV_CMP_SIZE(0), arg1, arg2, 441 (intptr_t)__builtin_return_address(0)); 442 } 443 444 void __sanitizer_cov_trace_cmp2(uint16_t arg1, uint16_t arg2); 445 446 void 447 __sanitizer_cov_trace_cmp2(uint16_t arg1, uint16_t arg2) 448 { 449 450 trace_cmp(KCOV_CMP_SIZE(1), arg1, arg2, 451 (intptr_t)__builtin_return_address(0)); 452 } 453 454 void __sanitizer_cov_trace_cmp4(uint32_t arg1, uint32_t arg2); 455 456 void 457 __sanitizer_cov_trace_cmp4(uint32_t arg1, uint32_t arg2) 458 { 459 460 trace_cmp(KCOV_CMP_SIZE(2), arg1, arg2, 461 (intptr_t)__builtin_return_address(0)); 462 } 463 464 void __sanitizer_cov_trace_cmp8(uint64_t arg1, uint64_t arg2); 465 466 void 467 __sanitizer_cov_trace_cmp8(uint64_t arg1, uint64_t arg2) 468 { 469 470 trace_cmp(KCOV_CMP_SIZE(3), arg1, arg2, 471 (intptr_t)__builtin_return_address(0)); 472 } 473 474 void __sanitizer_cov_trace_const_cmp1(uint8_t arg1, uint8_t arg2); 475 476 void 477 __sanitizer_cov_trace_const_cmp1(uint8_t arg1, uint8_t arg2) 478 { 479 480 trace_cmp(KCOV_CMP_SIZE(0) | KCOV_CMP_CONST, arg1, arg2, 481 (intptr_t)__builtin_return_address(0)); 482 } 483 484 void __sanitizer_cov_trace_const_cmp2(uint16_t arg1, uint16_t arg2); 485 486 void 487 __sanitizer_cov_trace_const_cmp2(uint16_t arg1, uint16_t arg2) 488 { 489 490 trace_cmp(KCOV_CMP_SIZE(1) | KCOV_CMP_CONST, arg1, arg2, 491 (intptr_t)__builtin_return_address(0)); 492 } 493 494 void __sanitizer_cov_trace_const_cmp4(uint32_t arg1, uint32_t arg2); 495 496 void 497 __sanitizer_cov_trace_const_cmp4(uint32_t arg1, uint32_t arg2) 498 { 499 500 trace_cmp(KCOV_CMP_SIZE(2) | KCOV_CMP_CONST, arg1, arg2, 501 (intptr_t)__builtin_return_address(0)); 502 } 503 504 void __sanitizer_cov_trace_const_cmp8(uint64_t arg1, uint64_t arg2); 505 506 void 507 __sanitizer_cov_trace_const_cmp8(uint64_t arg1, uint64_t arg2) 508 { 509 510 trace_cmp(KCOV_CMP_SIZE(3) | KCOV_CMP_CONST, arg1, arg2, 511 (intptr_t)__builtin_return_address(0)); 512 } 513 514 void __sanitizer_cov_trace_switch(uint64_t val, uint64_t *cases); 515 516 void 517 __sanitizer_cov_trace_switch(uint64_t val, uint64_t *cases) 518 { 519 uint64_t i, nbits, ncases, type; 520 intptr_t pc; 521 522 pc = (intptr_t)__builtin_return_address(0); 523 ncases = cases[0]; 524 nbits = cases[1]; 525 526 switch (nbits) { 527 case 8: 528 type = KCOV_CMP_SIZE(0); 529 break; 530 case 16: 531 type = KCOV_CMP_SIZE(1); 532 break; 533 case 32: 534 type = KCOV_CMP_SIZE(2); 535 break; 536 case 64: 537 type = KCOV_CMP_SIZE(3); 538 break; 539 default: 540 return; 541 } 542 type |= KCOV_CMP_CONST; 543 544 for (i = 0; i < ncases; i++) 545 trace_cmp(type, cases[i + 2], val, pc); 546 } 547 548 /* -------------------------------------------------------------------------- */ 549 550 MODULE(MODULE_CLASS_MISC, kcov, NULL); 551 552 static void 553 kcov_init(void) 554 { 555 556 lwp_specific_key_create(&kcov_lwp_key, kcov_lwp_free); 557 } 558 559 static int 560 kcov_modcmd(modcmd_t cmd, void *arg) 561 { 562 563 switch (cmd) { 564 case MODULE_CMD_INIT: 565 kcov_init(); 566 return 0; 567 case MODULE_CMD_FINI: 568 return EINVAL; 569 default: 570 return ENOTTY; 571 } 572 } 573