1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 * 21 * Portions Copyright 2006-2008 John Birrell jb@freebsd.org 22 * 23 * $FreeBSD: head/sys/cddl/dev/sdt/sdt.c 297771 2016-04-10 01:24:27Z markj $ 24 * 25 */ 26 27 /* 28 * This file contains a reimplementation of the statically-defined tracing (SDT) 29 * framework for DTrace. Probes and SDT providers are defined using the macros 30 * in sys/sdt.h, which append all the needed structures to linker sets. When 31 * this module is loaded, it iterates over all of the loaded modules and 32 * registers probes and providers with the DTrace framework based on the 33 * contents of these linker sets. 34 * 35 * A list of SDT providers is maintained here since a provider may span multiple 36 * modules. When a kernel module is unloaded, a provider defined in that module 37 * is unregistered only if no other modules refer to it. The DTrace framework is 38 * responsible for destroying individual probes when a kernel module is 39 * unloaded; in particular, probes may not span multiple kernel modules. 40 */ 41 #include <sys/cdefs.h> 42 __KERNEL_RCSID(0, "$NetBSD: sdt.c,v 1.24 2022/08/31 12:34:04 riastradh Exp $"); 43 44 #include <sys/cdefs.h> 45 #include <sys/proc.h> 46 #include <sys/param.h> 47 #include <sys/systm.h> 48 49 #include <sys/conf.h> 50 #ifdef __FreeBSD__ 51 #include <sys/eventhandler.h> 52 #endif 53 #include <sys/kernel.h> 54 #include <sys/syslimits.h> 55 #ifdef __NetBSD__ 56 #include <sys/kobj.h> 57 #endif 58 #ifdef __FreeBSD__ 59 #include <sys/linker.h> 60 #include <sys/linker_set.h> 61 #endif 62 #include <sys/lock.h> 63 #ifdef __FreeBSD__ 64 #include <sys/lockstat.h> 65 #endif 66 #include <sys/malloc.h> 67 #include <sys/kmem.h> 68 #include <sys/module.h> 69 #include <sys/mutex.h> 70 #include <sys/queue.h> 71 #include <sys/sdt.h> 72 73 #include <sys/dtrace.h> 74 #include <sys/dtrace_bsd.h> 75 76 /* DTrace methods. */ 77 static void sdt_getargdesc(void *, dtrace_id_t, void *, dtrace_argdesc_t *); 78 static void sdt_provide_probes(void *, dtrace_probedesc_t *); 79 static void sdt_destroy(void *, dtrace_id_t, void *); 80 static int sdt_enable(void *, dtrace_id_t, void *); 81 static void sdt_disable(void *, dtrace_id_t, void *); 82 83 static void sdt_load(void); 84 static int sdt_unload(void); 85 static void sdt_create_provider(struct sdt_provider *); 86 static void sdt_create_probe(struct sdt_probe *); 87 #ifdef __FreeBSD__ 88 static void sdt_kld_load(void *, struct linker_file *); 89 static void sdt_kld_unload_try(void *, struct linker_file *, int *); 90 #endif 91 92 MALLOC_DECLARE(M_SDT); 93 MALLOC_DEFINE(M_SDT, "SDT", "DTrace SDT providers"); 94 #define SDT_KASSERT(cond, msg) KASSERT(cond) 95 96 static dtrace_pattr_t sdt_attr = { 97 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON }, 98 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN }, 99 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_ISA }, 100 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON }, 101 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_ISA }, 102 }; 103 104 static dtrace_pops_t sdt_pops = { 105 sdt_provide_probes, 106 NULL, 107 sdt_enable, 108 sdt_disable, 109 NULL, 110 NULL, 111 sdt_getargdesc, 112 NULL, 113 NULL, 114 sdt_destroy, 115 }; 116 117 #ifdef __NetBSD__ 118 struct linker_file { 119 struct module lf_mod; 120 }; 121 122 static int 123 sdt_open(dev_t dev, int flags, int mode, struct lwp *l) 124 { 125 return 0; 126 } 127 128 static const struct cdevsw sdt_cdevsw = { 129 .d_open = sdt_open, 130 .d_close = noclose, 131 .d_read = noread, 132 .d_write = nowrite, 133 .d_ioctl = noioctl, 134 .d_stop = nostop, 135 .d_tty = notty, 136 .d_poll = nopoll, 137 .d_mmap = nommap, 138 .d_kqfilter = nokqfilter, 139 .d_discard = nodiscard, 140 .d_flag = D_OTHER 141 }; 142 #endif 143 144 static TAILQ_HEAD(, sdt_provider) sdt_prov_list; 145 146 #ifdef __FreeBSD__ 147 static eventhandler_tag sdt_kld_load_tag; 148 static eventhandler_tag sdt_kld_unload_try_tag; 149 #endif 150 151 #ifdef __NetBSD__ 152 static char * 153 strdup(const char *s, const struct malloc_type *m) 154 { 155 size_t l = strlen(s) + 1; 156 char *d = malloc(l, m, M_WAITOK); 157 memcpy(d, s, l); 158 return d; 159 } 160 #endif 161 162 static void 163 sdt_create_provider(struct sdt_provider *prov) 164 { 165 struct sdt_provider *curr, *newprov; 166 167 TAILQ_FOREACH(curr, &sdt_prov_list, prov_entry) 168 if (strcmp(prov->name, curr->name) == 0) { 169 /* The provider has already been defined. */ 170 curr->sdt_refs++; 171 return; 172 } 173 174 /* 175 * Make a copy of prov so that we don't lose fields if its module is 176 * unloaded but the provider isn't destroyed. This could happen with 177 * a provider that spans multiple modules. 178 */ 179 newprov = malloc(sizeof(*newprov), M_SDT, M_WAITOK | M_ZERO); 180 newprov->name = strdup(prov->name, M_SDT); 181 prov->sdt_refs = newprov->sdt_refs = 1; 182 183 TAILQ_INSERT_TAIL(&sdt_prov_list, newprov, prov_entry); 184 185 (void)dtrace_register(newprov->name, &sdt_attr, DTRACE_PRIV_USER, NULL, 186 &sdt_pops, NULL, (dtrace_provider_id_t *)&newprov->id); 187 prov->id = newprov->id; 188 } 189 190 static void 191 sdt_create_probe(struct sdt_probe *probe) 192 { 193 struct sdt_provider *prov; 194 char mod[DTRACE_MODNAMELEN]; 195 char func[DTRACE_FUNCNAMELEN]; 196 char name[DTRACE_NAMELEN]; 197 const char *from; 198 char *to; 199 size_t len; 200 201 if (probe->version != (int)sizeof(*probe)) { 202 printf("ignoring probe %p, version %u expected %u\n", 203 probe, probe->version, (int)sizeof(*probe)); 204 return; 205 } 206 207 TAILQ_FOREACH(prov, &sdt_prov_list, prov_entry) 208 if (strcmp(prov->name, probe->prov->name) == 0) 209 break; 210 211 SDT_KASSERT(prov != NULL, ("probe defined without a provider")); 212 213 /* If no module name was specified, use the module filename. */ 214 if (*probe->mod == 0) { 215 #ifdef __NetBSD__ 216 const char *modname = (probe->sdtp_lf == NULL ? "netbsd" : 217 module_name(&probe->sdtp_lf->lf_mod)); 218 strlcpy(mod, modname, sizeof(mod)); 219 #endif 220 #ifdef __FreeBSD__ 221 len = strlcpy(mod, probe->sdtp_lf->filename, sizeof(mod)); 222 if (len > 3 && strcmp(mod + len - 3, ".ko") == 0) 223 mod[len - 3] = '\0'; 224 #endif 225 } else 226 strlcpy(mod, probe->mod, sizeof(mod)); 227 228 /* 229 * Unfortunately this is necessary because the Solaris DTrace 230 * code mixes consts and non-consts with casts to override 231 * the incompatibilies. On FreeBSD, we use strict warnings 232 * in the C compiler, so we have to respect const vs non-const. 233 */ 234 strlcpy(func, probe->func, sizeof(func)); 235 if (func[0] == '\0') 236 strcpy(func, "none"); 237 238 from = probe->name; 239 to = name; 240 for (len = 0; len < (sizeof(name) - 1) && *from != '\0'; 241 len++, from++, to++) { 242 if (from[0] == '_' && from[1] == '_') { 243 *to = '-'; 244 from++; 245 } else 246 *to = *from; 247 } 248 *to = '\0'; 249 250 if (dtrace_probe_lookup(prov->id, mod, func, name) != DTRACE_IDNONE) 251 return; 252 253 (void)dtrace_probe_create(prov->id, mod, func, name, 1, probe); 254 } 255 256 /* 257 * Probes are created through the SDT module load/unload hook, so this function 258 * has nothing to do. It only exists because the DTrace provider framework 259 * requires one of provide_probes and provide_module to be defined. 260 */ 261 static void 262 sdt_provide_probes(void *arg, dtrace_probedesc_t *desc) 263 { 264 } 265 266 static int 267 sdt_enable(void *arg __unused, dtrace_id_t id, void *parg) 268 { 269 struct sdt_probe *probe = parg; 270 271 probe->id = id; 272 #ifdef __NetBSD__ 273 if (probe->sdtp_lf) 274 module_hold(&probe->sdtp_lf->lf_mod); 275 #endif 276 #ifdef __FreeBSD__ 277 probe->sdtp_lf->nenabled++; 278 if (strcmp(probe->prov->name, "lockstat") == 0) 279 lockstat_enabled++; 280 #endif 281 return 1; 282 } 283 284 static void 285 sdt_disable(void *arg __unused, dtrace_id_t id, void *parg) 286 { 287 struct sdt_probe *probe = parg; 288 289 #ifdef __NetBSD__ 290 if (probe->sdtp_lf) 291 module_rele(&probe->sdtp_lf->lf_mod); 292 #endif 293 #ifdef __FreeBSD__ 294 SDT_KASSERT(probe->sdtp_lf->nenabled > 0, ("no probes enabled")); 295 if (strcmp(probe->prov->name, "lockstat") == 0) 296 lockstat_enabled--; 297 probe->sdtp_lf->nenabled--; 298 #endif 299 probe->id = 0; 300 } 301 302 static void 303 sdt_getargdesc(void *arg, dtrace_id_t id, void *parg, dtrace_argdesc_t *desc) 304 { 305 struct sdt_argtype *argtype; 306 struct sdt_probe *probe = parg; 307 308 if (desc->dtargd_ndx >= probe->n_args) { 309 desc->dtargd_ndx = DTRACE_ARGNONE; 310 return; 311 } 312 313 TAILQ_FOREACH(argtype, &probe->argtype_list, argtype_entry) { 314 if (desc->dtargd_ndx == argtype->ndx) { 315 desc->dtargd_mapping = desc->dtargd_ndx; 316 if (argtype->type == NULL) { 317 desc->dtargd_native[0] = '\0'; 318 desc->dtargd_xlate[0] = '\0'; 319 continue; 320 } 321 strlcpy(desc->dtargd_native, argtype->type, 322 sizeof(desc->dtargd_native)); 323 if (argtype->xtype != NULL) 324 strlcpy(desc->dtargd_xlate, argtype->xtype, 325 sizeof(desc->dtargd_xlate)); 326 } 327 } 328 } 329 330 static void 331 sdt_destroy(void *arg, dtrace_id_t id, void *parg) 332 { 333 } 334 335 #ifdef __FreeBSD__ 336 /* 337 * Called from the kernel linker when a module is loaded, before 338 * dtrace_module_loaded() is called. This is done so that it's possible to 339 * register new providers when modules are loaded. The DTrace framework 340 * explicitly disallows calling into the framework from the provide_module 341 * provider method, so we cannot do this there. 342 */ 343 static void 344 sdt_kld_load(void *arg __unused, struct linker_file *lf) 345 { 346 struct sdt_provider **prov, **begin, **end; 347 struct sdt_probe **probe, **p_begin, **p_end; 348 struct sdt_argtype **argtype, **a_begin, **a_end; 349 350 if (linker_file_lookup_set(lf, "sdt_providers_set", &begin, &end, 351 NULL) == 0) { 352 for (prov = begin; prov < end; prov++) 353 sdt_create_provider(*prov); 354 } 355 356 if (linker_file_lookup_set(lf, "sdt_probes_set", &p_begin, &p_end, 357 NULL) == 0) { 358 for (probe = p_begin; probe < p_end; probe++) { 359 (*probe)->sdtp_lf = lf; 360 sdt_create_probe(*probe); 361 TAILQ_INIT(&(*probe)->argtype_list); 362 } 363 } 364 365 if (linker_file_lookup_set(lf, "sdt_argtypes_set", &a_begin, &a_end, 366 NULL) == 0) { 367 for (argtype = a_begin; argtype < a_end; argtype++) { 368 (*argtype)->probe->n_args++; 369 TAILQ_INSERT_TAIL(&(*argtype)->probe->argtype_list, 370 *argtype, argtype_entry); 371 } 372 } 373 } 374 375 static void 376 sdt_kld_unload_try(void *arg __unused, struct linker_file *lf, int *error) 377 { 378 struct sdt_provider *prov, **curr, **begin, **end, *tmp; 379 380 if (*error != 0) 381 /* We already have an error, so don't do anything. */ 382 return; 383 else if (linker_file_lookup_set(lf, "sdt_providers_set", &begin, &end, 384 NULL)) 385 /* No DTrace providers are declared in this file. */ 386 return; 387 388 /* 389 * Go through all the providers declared in this linker file and 390 * unregister any that aren't declared in another loaded file. 391 */ 392 for (curr = begin; curr < end; curr++) { 393 TAILQ_FOREACH_SAFE(prov, &sdt_prov_list, prov_entry, tmp) { 394 if (strcmp(prov->name, (*curr)->name) != 0) 395 continue; 396 397 if (prov->sdt_refs == 1) { 398 if (dtrace_unregister(prov->id) != 0) { 399 *error = 1; 400 return; 401 } 402 TAILQ_REMOVE(&sdt_prov_list, prov, prov_entry); 403 free(prov->name, M_SDT); 404 free(prov, M_SDT); 405 } else 406 prov->sdt_refs--; 407 break; 408 } 409 } 410 } 411 412 static int 413 sdt_linker_file_cb(linker_file_t lf, void *arg __unused) 414 { 415 416 sdt_kld_load(NULL, lf); 417 418 return (0); 419 } 420 #endif 421 422 #ifdef __NetBSD__ 423 /* 424 * weak symbols don't work in kernel modules; link set end symbols are 425 * weak by default, so we kill that. 426 */ 427 #undef __weak 428 #define __weak 429 __link_set_decl(sdt_providers_set, struct sdt_provider); 430 __link_set_decl(sdt_probes_set, struct sdt_probe); 431 __link_set_decl(sdt_argtypes_set, struct sdt_argtype); 432 433 static struct module_callbacks *sdt_link_set_callbacks; 434 435 static void 436 sdt_link_set_load_provider(struct sdt_provider *const *provider) 437 { 438 439 sdt_create_provider(*provider); 440 } 441 442 static void 443 sdt_link_set_unload_provider(struct sdt_provider *const *curr) 444 { 445 struct sdt_provider *prov, *tmp; 446 447 /* 448 * Go through all the providers declared in this linker file and 449 * unregister any that aren't declared in another loaded file. 450 */ 451 TAILQ_FOREACH_SAFE(prov, &sdt_prov_list, prov_entry, tmp) { 452 if (strcmp(prov->name, (*curr)->name) != 0) 453 continue; 454 455 if (prov->sdt_refs == 1) { 456 if (dtrace_unregister(prov->id) != 0) { 457 return; 458 } 459 TAILQ_REMOVE(&sdt_prov_list, prov, prov_entry); 460 free(__UNCONST(prov->name), M_SDT); 461 free(prov, M_SDT); 462 } else 463 prov->sdt_refs--; 464 break; 465 } 466 } 467 468 static void 469 sdt_link_set_load_probe(struct sdt_probe *const *probe, struct module *mod) 470 { 471 struct linker_file *lf = mod == NULL ? NULL : 472 container_of(mod, struct linker_file, lf_mod); 473 474 (*probe)->sdtp_lf = lf; 475 sdt_create_probe(*probe); 476 TAILQ_INIT(&(*probe)->argtype_list); 477 } 478 479 static void 480 sdt_link_set_load_argtype(struct sdt_argtype *const *argtype) 481 { 482 483 (*argtype)->probe->n_args++; 484 TAILQ_INSERT_TAIL(&(*argtype)->probe->argtype_list, 485 *argtype, argtype_entry); 486 } 487 488 static void 489 sdt_link_set_load_module(struct module *mod) 490 { 491 struct sdt_provider * const *provider; 492 struct sdt_probe * const *probe; 493 struct sdt_argtype * const *argtype; 494 void *p; 495 size_t size, n; 496 497 /* 498 * Skip builtin modules -- they are handled separately with 499 * __link_set_foreach. 500 */ 501 if (module_source(mod) == MODULE_SOURCE_KERNEL) 502 return; 503 504 if (kobj_find_section(mod->mod_kobj, "link_set_sdt_providers_set", 505 &p, &size) == 0) { 506 n = size/sizeof(*provider); 507 for (provider = p; n --> 0; provider++) 508 sdt_link_set_load_provider(provider); 509 } 510 511 if (kobj_find_section(mod->mod_kobj, "link_set_sdt_probes_set", 512 &p, &size) == 0) { 513 n = size/sizeof(*probe); 514 for (probe = p; n --> 0; probe++) 515 sdt_link_set_load_probe(probe, mod); 516 } 517 518 if (kobj_find_section(mod->mod_kobj, "link_set_sdt_argtypes_set", 519 &p, &size) == 0) { 520 n = size/sizeof(*argtype); 521 for (argtype = p; n --> 0; argtype++) 522 sdt_link_set_load_argtype(argtype); 523 } 524 } 525 526 static void 527 sdt_link_set_unload_module(struct module *mod) 528 { 529 struct sdt_provider *const *curr; 530 void *p; 531 size_t size, n; 532 533 /* 534 * Skip builtin modules -- they are handled separately with 535 * __link_set_foreach. 536 */ 537 if (module_source(mod) == MODULE_SOURCE_KERNEL) 538 return; 539 540 if (kobj_find_section(mod->mod_kobj, "link_set_sdt_providers_set", 541 &p, &size) == 0) { 542 n = size/sizeof(*curr); 543 for (curr = p; n --> 0; curr++) 544 sdt_link_set_unload_provider(curr); 545 } 546 } 547 548 static void 549 sdt_link_set_load(void) 550 { 551 struct sdt_provider * const *provider; 552 struct sdt_probe * const *probe; 553 struct sdt_argtype * const *argtype; 554 555 __link_set_foreach(provider, sdt_providers_set) { 556 sdt_link_set_load_provider(provider); 557 } 558 __link_set_foreach(probe, sdt_probes_set) { 559 sdt_link_set_load_probe(probe, NULL); 560 } 561 __link_set_foreach(argtype, sdt_argtypes_set) { 562 sdt_link_set_load_argtype(argtype); 563 } 564 565 sdt_link_set_callbacks = module_register_callbacks( 566 sdt_link_set_load_module, sdt_link_set_unload_module); 567 } 568 569 static void 570 sdt_link_set_unload(void) 571 { 572 struct sdt_provider *const *curr; 573 574 module_unregister_callbacks(sdt_link_set_callbacks); 575 sdt_link_set_callbacks = NULL; 576 577 __link_set_foreach(curr, sdt_providers_set) { 578 sdt_link_set_unload_provider(curr); 579 } 580 } 581 #endif 582 583 584 static void 585 sdt_load(void) 586 { 587 TAILQ_INIT(&sdt_prov_list); 588 589 #ifdef __FreeBSD__ 590 sdt_probe_func = dtrace_probe; 591 592 sdt_kld_load_tag = EVENTHANDLER_REGISTER(kld_load, sdt_kld_load, NULL, 593 EVENTHANDLER_PRI_ANY); 594 sdt_kld_unload_try_tag = EVENTHANDLER_REGISTER(kld_unload_try, 595 sdt_kld_unload_try, NULL, EVENTHANDLER_PRI_ANY); 596 597 /* Pick up probes from the kernel and already-loaded linker files. */ 598 linker_file_foreach(sdt_linker_file_cb, NULL); 599 #endif 600 #ifdef __NetBSD__ 601 sdt_init(dtrace_probe); 602 sdt_link_set_load(); 603 #endif 604 } 605 606 static int 607 sdt_unload(void) 608 { 609 struct sdt_provider *prov, *tmp; 610 int ret; 611 612 #ifdef __FreeBSD__ 613 EVENTHANDLER_DEREGISTER(kld_load, sdt_kld_load_tag); 614 EVENTHANDLER_DEREGISTER(kld_unload_try, sdt_kld_unload_try_tag); 615 616 sdt_probe_func = sdt_probe_stub; 617 #endif 618 619 #ifdef __NetBSD__ 620 sdt_exit(); 621 sdt_link_set_unload(); 622 #endif 623 TAILQ_FOREACH_SAFE(prov, &sdt_prov_list, prov_entry, tmp) { 624 ret = dtrace_unregister(prov->id); 625 if (ret != 0) 626 return (ret); 627 TAILQ_REMOVE(&sdt_prov_list, prov, prov_entry); 628 free(__UNCONST(prov->name), M_SDT); 629 free(prov, M_SDT); 630 } 631 632 return (0); 633 } 634 635 #ifdef __FreeBSD__ 636 static int 637 sdt_modevent(module_t mod __unused, int type, void *data __unused) 638 { 639 640 switch (type) { 641 case MOD_LOAD: 642 case MOD_UNLOAD: 643 case MOD_SHUTDOWN: 644 return (0); 645 default: 646 return (EOPNOTSUPP); 647 } 648 } 649 650 SYSINIT(sdt_load, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY, sdt_load, NULL); 651 SYSUNINIT(sdt_unload, SI_SUB_DTRACE_PROVIDER, SI_ORDER_ANY, sdt_unload, NULL); 652 653 DEV_MODULE(sdt, sdt_modevent, NULL); 654 MODULE_VERSION(sdt, 1); 655 MODULE_DEPEND(sdt, dtrace, 1, 1, 1); 656 #endif 657 658 #ifdef __NetBSD__ 659 static int 660 dtrace_sdt_modcmd(modcmd_t cmd, void *data) 661 { 662 int bmajor = -1, cmajor = 353; 663 int error; 664 665 switch (cmd) { 666 case MODULE_CMD_INIT: 667 sdt_load(); 668 return devsw_attach("sdt", NULL, &bmajor, 669 &sdt_cdevsw, &cmajor); 670 case MODULE_CMD_FINI: 671 error = sdt_unload(); 672 if (error != 0) 673 return error; 674 devsw_detach(NULL, &sdt_cdevsw); 675 return 0; 676 case MODULE_CMD_AUTOUNLOAD: 677 return EBUSY; 678 default: 679 error = EOPNOTSUPP; 680 break; 681 } 682 return error; 683 } 684 685 MODULE(MODULE_CLASS_MISC, dtrace_sdt, "dtrace"); 686 #endif 687