1 /* $NetBSD: subr_kobj.c,v 1.77 2023/04/17 08:14:51 skrll Exp $ */ 2 3 /* 4 * Copyright (c) 2008 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software developed for The NetBSD Foundation 8 * by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /* 33 * Copyright (c) 1998-2000 Doug Rabson 34 * Copyright (c) 2004 Peter Wemm 35 * All rights reserved. 36 * 37 * Redistribution and use in source and binary forms, with or without 38 * modification, are permitted provided that the following conditions 39 * are met: 40 * 1. Redistributions of source code must retain the above copyright 41 * notice, this list of conditions and the following disclaimer. 42 * 2. Redistributions in binary form must reproduce the above copyright 43 * notice, this list of conditions and the following disclaimer in the 44 * documentation and/or other materials provided with the distribution. 45 * 46 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 47 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 48 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 49 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 50 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 51 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 52 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 53 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 54 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 55 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 56 * SUCH DAMAGE. 57 */ 58 59 /* 60 * Kernel loader for ELF objects. 61 * 62 * TODO: adjust kmem_alloc() calls to avoid needless fragmentation. 63 */ 64 65 #include <sys/cdefs.h> 66 __KERNEL_RCSID(0, "$NetBSD: subr_kobj.c,v 1.77 2023/04/17 08:14:51 skrll Exp $"); 67 68 #ifdef _KERNEL_OPT 69 #include "opt_modular.h" 70 #endif 71 72 #include <sys/kobj_impl.h> 73 74 #ifdef MODULAR 75 76 #include <sys/param.h> 77 78 #include <sys/kernel.h> 79 #include <sys/kmem.h> 80 #include <sys/ksyms.h> 81 #include <sys/module.h> 82 #include <sys/proc.h> 83 84 #include <uvm/uvm_extern.h> 85 86 #define kobj_error(_kobj, ...) \ 87 kobj_out(__func__, __LINE__, _kobj, __VA_ARGS__) 88 89 static int kobj_relocate(kobj_t, bool); 90 static int kobj_checksyms(kobj_t, bool); 91 static void kobj_out(const char *, int, kobj_t, const char *, ...) 92 __printflike(4, 5); 93 static void kobj_jettison(kobj_t); 94 static void kobj_free(kobj_t, void *, size_t); 95 static void kobj_close(kobj_t); 96 static int kobj_read_mem(kobj_t, void **, size_t, off_t, bool); 97 static void kobj_close_mem(kobj_t); 98 99 /* 100 * kobj_load_mem: 101 * 102 * Load an object already resident in memory. If size is not -1, 103 * the complete size of the object is known. 104 */ 105 int 106 kobj_load_mem(kobj_t *kop, const char *name, void *base, ssize_t size) 107 { 108 kobj_t ko; 109 110 ko = kmem_zalloc(sizeof(*ko), KM_SLEEP); 111 ko->ko_type = KT_MEMORY; 112 kobj_setname(ko, name); 113 ko->ko_source = base; 114 ko->ko_memsize = size; 115 ko->ko_read = kobj_read_mem; 116 ko->ko_close = kobj_close_mem; 117 118 *kop = ko; 119 return kobj_load(ko); 120 } 121 122 /* 123 * kobj_close: 124 * 125 * Close an open ELF object. 126 */ 127 static void 128 kobj_close(kobj_t ko) 129 { 130 131 if (ko->ko_source == NULL) { 132 return; 133 } 134 135 ko->ko_close(ko); 136 ko->ko_source = NULL; 137 } 138 139 static void 140 kobj_close_mem(kobj_t ko) 141 { 142 143 return; 144 } 145 146 /* 147 * kobj_load: 148 * 149 * Load an ELF object and prepare to link into the running kernel 150 * image. 151 */ 152 int 153 kobj_load(kobj_t ko) 154 { 155 Elf_Ehdr *hdr; 156 Elf_Shdr *shdr; 157 Elf_Sym *es; 158 vaddr_t map_text_base; 159 vaddr_t map_data_base; 160 vaddr_t map_rodata_base; 161 size_t map_text_size; 162 size_t map_data_size; 163 size_t map_rodata_size; 164 int error; 165 int symtabindex; 166 int symstrindex; 167 int nsym; 168 int pb, rl, ra; 169 int alignmask; 170 int i, j; 171 void *addr; 172 173 KASSERT(ko->ko_type != KT_UNSET); 174 KASSERT(ko->ko_source != NULL); 175 176 shdr = NULL; 177 error = 0; 178 hdr = NULL; 179 180 /* 181 * Read the elf header from the file. 182 */ 183 error = ko->ko_read(ko, (void **)&hdr, sizeof(*hdr), 0, true); 184 if (error != 0) { 185 kobj_error(ko, "read failed %d", error); 186 goto out; 187 } 188 if (memcmp(hdr->e_ident, ELFMAG, SELFMAG) != 0) { 189 kobj_error(ko, "not an ELF object"); 190 error = ENOEXEC; 191 goto out; 192 } 193 194 if (hdr->e_ident[EI_VERSION] != EV_CURRENT || 195 hdr->e_version != EV_CURRENT) { 196 kobj_error(ko, "unsupported file version %d", 197 hdr->e_ident[EI_VERSION]); 198 error = ENOEXEC; 199 goto out; 200 } 201 if (hdr->e_type != ET_REL) { 202 kobj_error(ko, "unsupported file type %d", hdr->e_type); 203 error = ENOEXEC; 204 goto out; 205 } 206 switch (hdr->e_machine) { 207 #if ELFSIZE == 32 208 ELF32_MACHDEP_ID_CASES 209 #elif ELFSIZE == 64 210 ELF64_MACHDEP_ID_CASES 211 #else 212 #error not defined 213 #endif 214 default: 215 kobj_error(ko, "unsupported machine %d", hdr->e_machine); 216 error = ENOEXEC; 217 goto out; 218 } 219 220 ko->ko_nprogtab = 0; 221 ko->ko_shdr = 0; 222 ko->ko_nrel = 0; 223 ko->ko_nrela = 0; 224 225 /* 226 * Allocate and read in the section header. 227 */ 228 if (hdr->e_shnum == 0 || hdr->e_shnum > ELF_MAXSHNUM || 229 hdr->e_shoff == 0 || hdr->e_shentsize != sizeof(Elf_Shdr)) { 230 kobj_error(ko, "bad sizes"); 231 error = ENOEXEC; 232 goto out; 233 } 234 ko->ko_shdrsz = hdr->e_shnum * sizeof(Elf_Shdr); 235 error = ko->ko_read(ko, (void **)&shdr, ko->ko_shdrsz, hdr->e_shoff, 236 true); 237 if (error != 0) { 238 kobj_error(ko, "read failed %d", error); 239 goto out; 240 } 241 ko->ko_shdr = shdr; 242 243 /* 244 * Scan the section header for information and table sizing. 245 */ 246 nsym = 0; 247 symtabindex = symstrindex = -1; 248 for (i = 0; i < hdr->e_shnum; i++) { 249 switch (shdr[i].sh_type) { 250 case SHT_PROGBITS: 251 case SHT_NOBITS: 252 ko->ko_nprogtab++; 253 break; 254 case SHT_SYMTAB: 255 nsym++; 256 symtabindex = i; 257 symstrindex = shdr[i].sh_link; 258 break; 259 case SHT_REL: 260 if (shdr[shdr[i].sh_info].sh_type != SHT_PROGBITS) 261 continue; 262 ko->ko_nrel++; 263 break; 264 case SHT_RELA: 265 if (shdr[shdr[i].sh_info].sh_type != SHT_PROGBITS) 266 continue; 267 ko->ko_nrela++; 268 break; 269 case SHT_STRTAB: 270 break; 271 } 272 } 273 if (ko->ko_nprogtab == 0) { 274 kobj_error(ko, "file has no contents"); 275 error = ENOEXEC; 276 goto out; 277 } 278 if (nsym != 1) { 279 /* Only allow one symbol table for now */ 280 kobj_error(ko, "file has no valid symbol table"); 281 error = ENOEXEC; 282 goto out; 283 } 284 KASSERT(symtabindex != -1); 285 KASSERT(symstrindex != -1); 286 287 if (symstrindex == SHN_UNDEF || symstrindex >= hdr->e_shnum || 288 shdr[symstrindex].sh_type != SHT_STRTAB) { 289 kobj_error(ko, "file has invalid symbol strings"); 290 error = ENOEXEC; 291 goto out; 292 } 293 294 /* 295 * Allocate space for tracking the load chunks. 296 */ 297 if (ko->ko_nprogtab != 0) { 298 ko->ko_progtab = kmem_zalloc(ko->ko_nprogtab * 299 sizeof(*ko->ko_progtab), KM_SLEEP); 300 if (ko->ko_progtab == NULL) { 301 error = ENOMEM; 302 kobj_error(ko, "out of memory"); 303 goto out; 304 } 305 } 306 if (ko->ko_nrel != 0) { 307 ko->ko_reltab = kmem_zalloc(ko->ko_nrel * 308 sizeof(*ko->ko_reltab), KM_SLEEP); 309 if (ko->ko_reltab == NULL) { 310 error = ENOMEM; 311 kobj_error(ko, "out of memory"); 312 goto out; 313 } 314 } 315 if (ko->ko_nrela != 0) { 316 ko->ko_relatab = kmem_zalloc(ko->ko_nrela * 317 sizeof(*ko->ko_relatab), KM_SLEEP); 318 if (ko->ko_relatab == NULL) { 319 error = ENOMEM; 320 kobj_error(ko, "out of memory"); 321 goto out; 322 } 323 } 324 325 /* 326 * Allocate space for and load the symbol table. 327 */ 328 ko->ko_symcnt = shdr[symtabindex].sh_size / sizeof(Elf_Sym); 329 if (ko->ko_symcnt == 0) { 330 kobj_error(ko, "no symbol table"); 331 error = ENOEXEC; 332 goto out; 333 } 334 error = ko->ko_read(ko, (void **)&ko->ko_symtab, 335 ko->ko_symcnt * sizeof(Elf_Sym), 336 shdr[symtabindex].sh_offset, true); 337 if (error != 0) { 338 kobj_error(ko, "read failed %d", error); 339 goto out; 340 } 341 342 /* 343 * Allocate space for and load the symbol strings. 344 */ 345 ko->ko_strtabsz = shdr[symstrindex].sh_size; 346 if (ko->ko_strtabsz == 0) { 347 kobj_error(ko, "no symbol strings"); 348 error = ENOEXEC; 349 goto out; 350 } 351 error = ko->ko_read(ko, (void *)&ko->ko_strtab, ko->ko_strtabsz, 352 shdr[symstrindex].sh_offset, true); 353 if (error != 0) { 354 kobj_error(ko, "read failed %d", error); 355 goto out; 356 } 357 358 /* 359 * Adjust module symbol namespace, if necessary (e.g. with rump) 360 */ 361 error = kobj_renamespace(ko->ko_symtab, ko->ko_symcnt, 362 &ko->ko_strtab, &ko->ko_strtabsz); 363 if (error != 0) { 364 kobj_error(ko, "renamespace failed %d", error); 365 goto out; 366 } 367 368 /* 369 * Do we have a string table for the section names? 370 */ 371 if (hdr->e_shstrndx != SHN_UNDEF) { 372 if (hdr->e_shstrndx >= hdr->e_shnum) { 373 kobj_error(ko, "bad shstrndx"); 374 error = ENOEXEC; 375 goto out; 376 } 377 if (shdr[hdr->e_shstrndx].sh_size != 0 && 378 shdr[hdr->e_shstrndx].sh_type == SHT_STRTAB) { 379 ko->ko_shstrtabsz = shdr[hdr->e_shstrndx].sh_size; 380 error = ko->ko_read(ko, (void **)&ko->ko_shstrtab, 381 shdr[hdr->e_shstrndx].sh_size, 382 shdr[hdr->e_shstrndx].sh_offset, true); 383 if (error != 0) { 384 kobj_error(ko, "read failed %d", error); 385 goto out; 386 } 387 } 388 } 389 390 /* 391 * Size up code/data(progbits) and bss(nobits). 392 */ 393 alignmask = 0; 394 map_text_size = 0; 395 map_data_size = 0; 396 map_rodata_size = 0; 397 for (i = 0; i < hdr->e_shnum; i++) { 398 if (shdr[i].sh_type != SHT_PROGBITS && 399 shdr[i].sh_type != SHT_NOBITS) 400 continue; 401 alignmask = shdr[i].sh_addralign - 1; 402 if ((shdr[i].sh_flags & SHF_EXECINSTR)) { 403 map_text_size += alignmask; 404 map_text_size &= ~alignmask; 405 map_text_size += shdr[i].sh_size; 406 } else if (!(shdr[i].sh_flags & SHF_WRITE)) { 407 map_rodata_size += alignmask; 408 map_rodata_size &= ~alignmask; 409 map_rodata_size += shdr[i].sh_size; 410 } else { 411 map_data_size += alignmask; 412 map_data_size &= ~alignmask; 413 map_data_size += shdr[i].sh_size; 414 } 415 } 416 417 if (map_text_size == 0) { 418 kobj_error(ko, "no text"); 419 error = ENOEXEC; 420 goto out; 421 } 422 423 if (map_data_size != 0) { 424 map_data_base = uvm_km_alloc(module_map, round_page(map_data_size), 425 0, UVM_KMF_WIRED); 426 if (map_data_base == 0) { 427 kobj_error(ko, "out of memory"); 428 error = ENOMEM; 429 goto out; 430 } 431 ko->ko_data_address = map_data_base; 432 ko->ko_data_size = map_data_size; 433 } else { 434 map_data_base = 0; 435 ko->ko_data_address = 0; 436 ko->ko_data_size = 0; 437 } 438 439 if (map_rodata_size != 0) { 440 map_rodata_base = uvm_km_alloc(module_map, round_page(map_rodata_size), 441 0, UVM_KMF_WIRED); 442 if (map_rodata_base == 0) { 443 kobj_error(ko, "out of memory"); 444 error = ENOMEM; 445 goto out; 446 } 447 ko->ko_rodata_address = map_rodata_base; 448 ko->ko_rodata_size = map_rodata_size; 449 } else { 450 map_rodata_base = 0; 451 ko->ko_rodata_address = 0; 452 ko->ko_rodata_size = 0; 453 } 454 455 map_text_base = uvm_km_alloc(module_map, round_page(map_text_size), 456 0, UVM_KMF_WIRED | UVM_KMF_EXEC); 457 if (map_text_base == 0) { 458 kobj_error(ko, "out of memory"); 459 error = ENOMEM; 460 goto out; 461 } 462 ko->ko_text_address = map_text_base; 463 ko->ko_text_size = map_text_size; 464 465 /* 466 * Now load code/data(progbits), zero bss(nobits), allocate space 467 * for and load relocs 468 */ 469 pb = 0; 470 rl = 0; 471 ra = 0; 472 alignmask = 0; 473 for (i = 0; i < hdr->e_shnum; i++) { 474 switch (shdr[i].sh_type) { 475 case SHT_PROGBITS: 476 case SHT_NOBITS: 477 alignmask = shdr[i].sh_addralign - 1; 478 if ((shdr[i].sh_flags & SHF_EXECINSTR)) { 479 map_text_base += alignmask; 480 map_text_base &= ~alignmask; 481 addr = (void *)map_text_base; 482 map_text_base += shdr[i].sh_size; 483 } else if (!(shdr[i].sh_flags & SHF_WRITE)) { 484 map_rodata_base += alignmask; 485 map_rodata_base &= ~alignmask; 486 addr = (void *)map_rodata_base; 487 map_rodata_base += shdr[i].sh_size; 488 } else { 489 map_data_base += alignmask; 490 map_data_base &= ~alignmask; 491 addr = (void *)map_data_base; 492 map_data_base += shdr[i].sh_size; 493 } 494 495 ko->ko_progtab[pb].addr = addr; 496 if (shdr[i].sh_type == SHT_PROGBITS) { 497 ko->ko_progtab[pb].name = "<<PROGBITS>>"; 498 error = ko->ko_read(ko, &addr, 499 shdr[i].sh_size, shdr[i].sh_offset, false); 500 if (error != 0) { 501 kobj_error(ko, "read failed %d", error); 502 goto out; 503 } 504 } else { /* SHT_NOBITS */ 505 ko->ko_progtab[pb].name = "<<NOBITS>>"; 506 memset(addr, 0, shdr[i].sh_size); 507 } 508 509 ko->ko_progtab[pb].size = shdr[i].sh_size; 510 ko->ko_progtab[pb].sec = i; 511 if (ko->ko_shstrtab != NULL && shdr[i].sh_name != 0) { 512 ko->ko_progtab[pb].name = 513 ko->ko_shstrtab + shdr[i].sh_name; 514 } 515 516 /* Update all symbol values with the offset. */ 517 for (j = 0; j < ko->ko_symcnt; j++) { 518 es = &ko->ko_symtab[j]; 519 if (es->st_shndx != i) { 520 continue; 521 } 522 es->st_value += (Elf_Addr)addr; 523 } 524 pb++; 525 break; 526 case SHT_REL: 527 if (shdr[shdr[i].sh_info].sh_type != SHT_PROGBITS) 528 break; 529 ko->ko_reltab[rl].size = shdr[i].sh_size; 530 ko->ko_reltab[rl].size -= 531 shdr[i].sh_size % sizeof(Elf_Rel); 532 if (ko->ko_reltab[rl].size != 0) { 533 ko->ko_reltab[rl].nrel = 534 shdr[i].sh_size / sizeof(Elf_Rel); 535 ko->ko_reltab[rl].sec = shdr[i].sh_info; 536 error = ko->ko_read(ko, 537 (void **)&ko->ko_reltab[rl].rel, 538 ko->ko_reltab[rl].size, 539 shdr[i].sh_offset, true); 540 if (error != 0) { 541 kobj_error(ko, "read failed %d", 542 error); 543 goto out; 544 } 545 } 546 rl++; 547 break; 548 case SHT_RELA: 549 if (shdr[shdr[i].sh_info].sh_type != SHT_PROGBITS) 550 break; 551 ko->ko_relatab[ra].size = shdr[i].sh_size; 552 ko->ko_relatab[ra].size -= 553 shdr[i].sh_size % sizeof(Elf_Rela); 554 if (ko->ko_relatab[ra].size != 0) { 555 ko->ko_relatab[ra].nrela = 556 shdr[i].sh_size / sizeof(Elf_Rela); 557 ko->ko_relatab[ra].sec = shdr[i].sh_info; 558 error = ko->ko_read(ko, 559 (void **)&ko->ko_relatab[ra].rela, 560 shdr[i].sh_size, 561 shdr[i].sh_offset, true); 562 if (error != 0) { 563 kobj_error(ko, "read failed %d", error); 564 goto out; 565 } 566 } 567 ra++; 568 break; 569 default: 570 break; 571 } 572 } 573 if (pb != ko->ko_nprogtab) { 574 panic("%s:%d: %s: lost progbits", __func__, __LINE__, 575 ko->ko_name); 576 } 577 if (rl != ko->ko_nrel) { 578 panic("%s:%d: %s: lost rel", __func__, __LINE__, 579 ko->ko_name); 580 } 581 if (ra != ko->ko_nrela) { 582 panic("%s:%d: %s: lost rela", __func__, __LINE__, 583 ko->ko_name); 584 } 585 if (map_text_base != ko->ko_text_address + map_text_size) { 586 panic("%s:%d: %s: map_text_base 0x%lx != address %lx " 587 "+ map_text_size %ld (0x%lx)\n", 588 __func__, __LINE__, ko->ko_name, (long)map_text_base, 589 (long)ko->ko_text_address, (long)map_text_size, 590 (long)ko->ko_text_address + map_text_size); 591 } 592 if (map_data_base != ko->ko_data_address + map_data_size) { 593 panic("%s:%d: %s: map_data_base 0x%lx != address %lx " 594 "+ map_data_size %ld (0x%lx)\n", 595 __func__, __LINE__, ko->ko_name, (long)map_data_base, 596 (long)ko->ko_data_address, (long)map_data_size, 597 (long)ko->ko_data_address + map_data_size); 598 } 599 if (map_rodata_base != ko->ko_rodata_address + map_rodata_size) { 600 panic("%s:%d: %s: map_rodata_base 0x%lx != address %lx " 601 "+ map_rodata_size %ld (0x%lx)\n", 602 __func__, __LINE__, ko->ko_name, (long)map_rodata_base, 603 (long)ko->ko_rodata_address, (long)map_rodata_size, 604 (long)ko->ko_rodata_address + map_rodata_size); 605 } 606 607 /* 608 * Perform local relocations only. Relocations relating to global 609 * symbols will be done by kobj_affix(). 610 */ 611 error = kobj_checksyms(ko, false); 612 if (error) 613 goto out; 614 615 error = kobj_relocate(ko, true); 616 if (error) 617 goto out; 618 out: 619 if (hdr != NULL) { 620 kobj_free(ko, hdr, sizeof(*hdr)); 621 } 622 kobj_close(ko); 623 if (error != 0) { 624 kobj_unload(ko); 625 } 626 627 return error; 628 } 629 630 static void 631 kobj_unload_notify(kobj_t ko, vaddr_t addr, size_t size, const char *note) 632 { 633 if (addr == 0) 634 return; 635 636 int error = kobj_machdep(ko, (void *)addr, size, false); 637 if (error) 638 kobj_error(ko, "machine dependent deinit failed (%s) %d", 639 note, error); 640 } 641 642 #define KOBJ_SEGMENT_NOTIFY(ko, what) \ 643 kobj_unload_notify(ko, (ko)->ko_ ## what ## _address, \ 644 (ko)->ko_ ## what ## _size, # what); 645 646 #define KOBJ_SEGMENT_FREE(ko, what) \ 647 do \ 648 if ((ko)->ko_ ## what ## _address != 0) \ 649 uvm_km_free(module_map, (ko)->ko_ ## what ## _address, \ 650 round_page((ko)->ko_ ## what ## _size), UVM_KMF_WIRED); \ 651 while (/*CONSTCOND*/ 0) 652 653 /* 654 * kobj_unload: 655 * 656 * Unload an object previously loaded by kobj_load(). 657 */ 658 void 659 kobj_unload(kobj_t ko) 660 { 661 kobj_close(ko); 662 kobj_jettison(ko); 663 664 665 /* 666 * Notify MD code that a module has been unloaded. 667 */ 668 if (ko->ko_loaded) { 669 KOBJ_SEGMENT_NOTIFY(ko, text); 670 KOBJ_SEGMENT_NOTIFY(ko, data); 671 KOBJ_SEGMENT_NOTIFY(ko, rodata); 672 } 673 674 KOBJ_SEGMENT_FREE(ko, text); 675 KOBJ_SEGMENT_FREE(ko, data); 676 KOBJ_SEGMENT_FREE(ko, rodata); 677 678 if (ko->ko_ksyms == true) { 679 ksyms_modunload(ko->ko_name); 680 } 681 if (ko->ko_symtab != NULL) { 682 kobj_free(ko, ko->ko_symtab, ko->ko_symcnt * sizeof(Elf_Sym)); 683 } 684 if (ko->ko_strtab != NULL) { 685 kobj_free(ko, ko->ko_strtab, ko->ko_strtabsz); 686 } 687 if (ko->ko_progtab != NULL) { 688 kobj_free(ko, ko->ko_progtab, ko->ko_nprogtab * 689 sizeof(*ko->ko_progtab)); 690 ko->ko_progtab = NULL; 691 } 692 if (ko->ko_shstrtab) { 693 kobj_free(ko, ko->ko_shstrtab, ko->ko_shstrtabsz); 694 ko->ko_shstrtab = NULL; 695 } 696 697 kmem_free(ko, sizeof(*ko)); 698 } 699 700 /* 701 * kobj_stat: 702 * 703 * Return size and load address of an object. 704 */ 705 int 706 kobj_stat(kobj_t ko, vaddr_t *address, size_t *size) 707 { 708 709 if (address != NULL) { 710 *address = ko->ko_text_address; 711 } 712 if (size != NULL) { 713 *size = ko->ko_text_size; 714 } 715 return 0; 716 } 717 718 /* 719 * kobj_affix: 720 * 721 * Set an object's name and perform global relocs. May only be 722 * called after the module and any requisite modules are loaded. 723 */ 724 int 725 kobj_affix(kobj_t ko, const char *name) 726 { 727 int error; 728 729 KASSERT(ko->ko_ksyms == false); 730 KASSERT(ko->ko_loaded == false); 731 732 kobj_setname(ko, name); 733 734 /* Cache addresses of undefined symbols. */ 735 error = kobj_checksyms(ko, true); 736 if (error) 737 goto out; 738 739 /* Now do global relocations. */ 740 error = kobj_relocate(ko, false); 741 if (error) 742 goto out; 743 744 /* 745 * Now that we know the name, register the symbol table. 746 * Do after global relocations because ksyms will pack 747 * the table. 748 */ 749 ksyms_modload(ko->ko_name, ko->ko_symtab, 750 ko->ko_symcnt * sizeof(Elf_Sym), ko->ko_strtab, ko->ko_strtabsz); 751 ko->ko_ksyms = true; 752 753 /* Jettison unneeded memory post-link. */ 754 kobj_jettison(ko); 755 756 /* 757 * Notify MD code that a module has been loaded. 758 * 759 * Most architectures use this opportunity to flush their caches. 760 */ 761 if (ko->ko_text_address != 0) { 762 error = kobj_machdep(ko, (void *)ko->ko_text_address, 763 ko->ko_text_size, true); 764 if (error) { 765 kobj_error(ko, "machine dependent init failed (text)" 766 " %d", error); 767 goto out; 768 } 769 } 770 771 if (ko->ko_data_address != 0) { 772 error = kobj_machdep(ko, (void *)ko->ko_data_address, 773 ko->ko_data_size, true); 774 if (error) { 775 kobj_error(ko, "machine dependent init failed (data)" 776 " %d", error); 777 goto out; 778 } 779 } 780 781 if (ko->ko_rodata_address != 0) { 782 error = kobj_machdep(ko, (void *)ko->ko_rodata_address, 783 ko->ko_rodata_size, true); 784 if (error) { 785 kobj_error(ko, "machine dependent init failed (rodata)" 786 " %d", error); 787 goto out; 788 } 789 } 790 791 ko->ko_loaded = true; 792 793 /* Change the memory protections, when needed. */ 794 if (ko->ko_text_address != 0) { 795 uvm_km_protect(module_map, ko->ko_text_address, 796 ko->ko_text_size, VM_PROT_READ|VM_PROT_EXECUTE); 797 } 798 if (ko->ko_rodata_address != 0) { 799 uvm_km_protect(module_map, ko->ko_rodata_address, 800 ko->ko_rodata_size, VM_PROT_READ); 801 } 802 803 /* Success! */ 804 error = 0; 805 806 out: if (error) { 807 /* If there was an error, destroy the whole object. */ 808 kobj_unload(ko); 809 } 810 return error; 811 } 812 813 /* 814 * kobj_find_section: 815 * 816 * Given a section name, search the loaded object and return 817 * virtual address if present and loaded. 818 */ 819 int 820 kobj_find_section(kobj_t ko, const char *name, void **addr, size_t *size) 821 { 822 int i; 823 824 KASSERT(ko->ko_progtab != NULL); 825 826 for (i = 0; i < ko->ko_nprogtab; i++) { 827 if (strcmp(ko->ko_progtab[i].name, name) == 0) { 828 if (addr != NULL) { 829 *addr = ko->ko_progtab[i].addr; 830 } 831 if (size != NULL) { 832 *size = ko->ko_progtab[i].size; 833 } 834 return 0; 835 } 836 } 837 838 return ENOENT; 839 } 840 841 /* 842 * kobj_jettison: 843 * 844 * Release object data not needed after performing relocations. 845 */ 846 static void 847 kobj_jettison(kobj_t ko) 848 { 849 int i; 850 851 if (ko->ko_reltab != NULL) { 852 for (i = 0; i < ko->ko_nrel; i++) { 853 if (ko->ko_reltab[i].rel) { 854 kobj_free(ko, ko->ko_reltab[i].rel, 855 ko->ko_reltab[i].size); 856 } 857 } 858 kobj_free(ko, ko->ko_reltab, ko->ko_nrel * 859 sizeof(*ko->ko_reltab)); 860 ko->ko_reltab = NULL; 861 ko->ko_nrel = 0; 862 } 863 if (ko->ko_relatab != NULL) { 864 for (i = 0; i < ko->ko_nrela; i++) { 865 if (ko->ko_relatab[i].rela) { 866 kobj_free(ko, ko->ko_relatab[i].rela, 867 ko->ko_relatab[i].size); 868 } 869 } 870 kobj_free(ko, ko->ko_relatab, ko->ko_nrela * 871 sizeof(*ko->ko_relatab)); 872 ko->ko_relatab = NULL; 873 ko->ko_nrela = 0; 874 } 875 if (ko->ko_shdr != NULL) { 876 kobj_free(ko, ko->ko_shdr, ko->ko_shdrsz); 877 ko->ko_shdr = NULL; 878 } 879 } 880 881 /* 882 * kobj_sym_lookup: 883 * 884 * Symbol lookup function to be used when the symbol index 885 * is known (ie during relocation). 886 */ 887 int 888 kobj_sym_lookup(kobj_t ko, uintptr_t symidx, Elf_Addr *val) 889 { 890 const Elf_Sym *sym; 891 const char *symbol; 892 893 sym = ko->ko_symtab + symidx; 894 895 if (symidx == SHN_ABS || symidx == 0) { 896 *val = (uintptr_t)sym->st_value; 897 return 0; 898 } else if (symidx >= ko->ko_symcnt) { 899 /* 900 * Don't even try to lookup the symbol if the index is 901 * bogus. 902 */ 903 kobj_error(ko, "symbol index %ju out of range", 904 (uintmax_t)symidx); 905 return EINVAL; 906 } 907 908 /* Quick answer if there is a definition included. */ 909 if (sym->st_shndx != SHN_UNDEF) { 910 *val = (uintptr_t)sym->st_value; 911 return 0; 912 } 913 914 /* If we get here, then it is undefined and needs a lookup. */ 915 switch (ELF_ST_BIND(sym->st_info)) { 916 case STB_LOCAL: 917 /* Local, but undefined? huh? */ 918 kobj_error(ko, "local symbol @%ju undefined", 919 (uintmax_t)symidx); 920 return EINVAL; 921 922 case STB_GLOBAL: 923 /* Relative to Data or Function name */ 924 symbol = ko->ko_strtab + sym->st_name; 925 926 /* Force a lookup failure if the symbol name is bogus. */ 927 if (*symbol == 0) { 928 kobj_error(ko, "bad symbol @%ju name", 929 (uintmax_t)symidx); 930 return EINVAL; 931 } 932 if (sym->st_value == 0) { 933 kobj_error(ko, "%s @%ju: bad value", symbol, 934 (uintmax_t)symidx); 935 return EINVAL; 936 } 937 938 *val = (uintptr_t)sym->st_value; 939 return 0; 940 941 case STB_WEAK: 942 kobj_error(ko, "weak symbol @%ju not supported", 943 (uintmax_t)symidx); 944 return EINVAL; 945 946 default: 947 kobj_error(ko, "bad binding %#x for symbol @%ju", 948 ELF_ST_BIND(sym->st_info), (uintmax_t)symidx); 949 return EINVAL; 950 } 951 } 952 953 /* 954 * kobj_findbase: 955 * 956 * Return base address of the given section. 957 */ 958 static uintptr_t 959 kobj_findbase(kobj_t ko, int sec) 960 { 961 int i; 962 963 for (i = 0; i < ko->ko_nprogtab; i++) { 964 if (sec == ko->ko_progtab[i].sec) { 965 return (uintptr_t)ko->ko_progtab[i].addr; 966 } 967 } 968 return 0; 969 } 970 971 /* 972 * kobj_checksyms: 973 * 974 * Scan symbol table for duplicates or resolve references to 975 * external symbols. 976 */ 977 static int 978 kobj_checksyms(kobj_t ko, bool undefined) 979 { 980 unsigned long rval; 981 Elf_Sym *sym, *ksym, *ms; 982 const char *name; 983 int error; 984 985 error = 0; 986 987 for (ms = (sym = ko->ko_symtab) + ko->ko_symcnt; sym < ms; sym++) { 988 /* Check validity of the symbol. */ 989 if (ELF_ST_BIND(sym->st_info) != STB_GLOBAL || 990 sym->st_name == 0) 991 continue; 992 if (undefined != (sym->st_shndx == SHN_UNDEF)) { 993 continue; 994 } 995 996 /* 997 * Look it up. Don't need to lock, as it is known that 998 * the symbol tables aren't going to change (we hold 999 * module_lock). 1000 */ 1001 name = ko->ko_strtab + sym->st_name; 1002 if (ksyms_getval_unlocked(NULL, name, &ksym, &rval, 1003 KSYMS_EXTERN) != 0) { 1004 if (undefined) { 1005 kobj_error(ko, "symbol `%s' not found", 1006 name); 1007 error = ENOEXEC; 1008 } 1009 continue; 1010 } 1011 1012 /* Save values of undefined globals. */ 1013 if (undefined) { 1014 if (ksym->st_shndx == SHN_ABS) { 1015 sym->st_shndx = SHN_ABS; 1016 } 1017 sym->st_value = (Elf_Addr)rval; 1018 continue; 1019 } 1020 1021 /* Check (and complain) about differing values. */ 1022 if (sym->st_value == rval) { 1023 continue; 1024 } 1025 if (strcmp(name, "_bss_start") == 0 || 1026 strcmp(name, "__bss_start") == 0 || 1027 strcmp(name, "_bss_end__") == 0 || 1028 strcmp(name, "__bss_end__") == 0 || 1029 strcmp(name, "_edata") == 0 || 1030 strcmp(name, "_end") == 0 || 1031 strcmp(name, "__end") == 0 || 1032 strcmp(name, "__end__") == 0 || 1033 strncmp(name, "__start_link_set_", 17) == 0 || 1034 strncmp(name, "__stop_link_set_", 16) == 0) { 1035 continue; 1036 } 1037 kobj_error(ko, "global symbol `%s' redefined", 1038 name); 1039 error = ENOEXEC; 1040 } 1041 1042 return error; 1043 } 1044 1045 /* 1046 * kobj_relocate: 1047 * 1048 * Resolve relocations for the loaded object. 1049 */ 1050 static int 1051 kobj_relocate(kobj_t ko, bool local) 1052 { 1053 const Elf_Rel *rellim; 1054 const Elf_Rel *rel; 1055 const Elf_Rela *relalim; 1056 const Elf_Rela *rela; 1057 const Elf_Sym *sym; 1058 uintptr_t base; 1059 int i, error; 1060 uintptr_t symidx; 1061 1062 /* 1063 * Perform relocations without addend if there are any. 1064 */ 1065 for (i = 0; i < ko->ko_nrel; i++) { 1066 rel = ko->ko_reltab[i].rel; 1067 if (rel == NULL) { 1068 continue; 1069 } 1070 rellim = rel + ko->ko_reltab[i].nrel; 1071 base = kobj_findbase(ko, ko->ko_reltab[i].sec); 1072 if (base == 0) { 1073 panic("%s:%d: %s: lost base for e_reltab[%d] sec %d", 1074 __func__, __LINE__, ko->ko_name, i, 1075 ko->ko_reltab[i].sec); 1076 } 1077 for (; rel < rellim; rel++) { 1078 symidx = ELF_R_SYM(rel->r_info); 1079 if (symidx >= ko->ko_symcnt) { 1080 continue; 1081 } 1082 sym = ko->ko_symtab + symidx; 1083 if (local != (ELF_ST_BIND(sym->st_info) == STB_LOCAL)) { 1084 continue; 1085 } 1086 error = kobj_reloc(ko, base, rel, false, local); 1087 if (error != 0) { 1088 kobj_error(ko, "unresolved rel relocation " 1089 "@%#jx type=%d symidx=%d", 1090 (intmax_t)rel->r_offset, 1091 (int)ELF_R_TYPE(rel->r_info), 1092 (int)ELF_R_SYM(rel->r_info)); 1093 return ENOEXEC; 1094 } 1095 } 1096 } 1097 1098 /* 1099 * Perform relocations with addend if there are any. 1100 */ 1101 for (i = 0; i < ko->ko_nrela; i++) { 1102 rela = ko->ko_relatab[i].rela; 1103 if (rela == NULL) { 1104 continue; 1105 } 1106 relalim = rela + ko->ko_relatab[i].nrela; 1107 base = kobj_findbase(ko, ko->ko_relatab[i].sec); 1108 if (base == 0) { 1109 panic("%s:%d: %s: lost base for e_relatab[%d] sec %d", 1110 __func__, __LINE__, ko->ko_name, i, 1111 ko->ko_relatab[i].sec); 1112 } 1113 for (; rela < relalim; rela++) { 1114 symidx = ELF_R_SYM(rela->r_info); 1115 if (symidx >= ko->ko_symcnt) { 1116 continue; 1117 } 1118 sym = ko->ko_symtab + symidx; 1119 if (local != (ELF_ST_BIND(sym->st_info) == STB_LOCAL)) { 1120 continue; 1121 } 1122 error = kobj_reloc(ko, base, rela, true, local); 1123 if (error != 0) { 1124 kobj_error(ko, "unresolved rela relocation " 1125 "@%#jx type=%d symidx=%d", 1126 (intmax_t)rela->r_offset, 1127 (int)ELF_R_TYPE(rela->r_info), 1128 (int)ELF_R_SYM(rela->r_info)); 1129 return ENOEXEC; 1130 } 1131 } 1132 } 1133 1134 return 0; 1135 } 1136 1137 /* 1138 * kobj_out: 1139 * 1140 * Utility function: log an error. 1141 */ 1142 static void 1143 kobj_out(const char *fname, int lnum, kobj_t ko, const char *fmt, ...) 1144 { 1145 va_list ap; 1146 1147 printf("%s, %d: [%s]: linker error: ", fname, lnum, ko->ko_name); 1148 va_start(ap, fmt); 1149 vprintf(fmt, ap); 1150 va_end(ap); 1151 printf("\n"); 1152 } 1153 1154 static int 1155 kobj_read_mem(kobj_t ko, void **basep, size_t size, off_t off, 1156 bool allocate) 1157 { 1158 void *base = *basep; 1159 int error = 0; 1160 1161 KASSERT(ko->ko_source != NULL); 1162 1163 if (off < 0) { 1164 kobj_error(ko, "negative offset %lld", 1165 (unsigned long long)off); 1166 error = EINVAL; 1167 base = NULL; 1168 goto out; 1169 } else if (ko->ko_memsize != -1 && 1170 (size > ko->ko_memsize || off > ko->ko_memsize - size)) { 1171 kobj_error(ko, "preloaded object short"); 1172 error = EINVAL; 1173 base = NULL; 1174 goto out; 1175 } 1176 1177 if (allocate) 1178 base = kmem_alloc(size, KM_SLEEP); 1179 1180 /* Copy the section */ 1181 memcpy(base, (uint8_t *)ko->ko_source + off, size); 1182 1183 out: if (allocate) 1184 *basep = base; 1185 return error; 1186 } 1187 1188 /* 1189 * kobj_free: 1190 * 1191 * Utility function: free memory if it was allocated from the heap. 1192 */ 1193 static void 1194 kobj_free(kobj_t ko, void *base, size_t size) 1195 { 1196 1197 kmem_free(base, size); 1198 } 1199 1200 void 1201 kobj_setname(kobj_t ko, const char *name) 1202 { 1203 const char *d = name, *dots = ""; 1204 size_t len, dlen; 1205 1206 for (char *s = module_base; *d == *s; d++, s++) 1207 continue; 1208 1209 if (d == name) 1210 name = ""; 1211 else 1212 name = "%M"; 1213 dlen = strlen(d); 1214 len = dlen + strlen(name); 1215 if (len >= sizeof(ko->ko_name)) { 1216 len = (len - sizeof(ko->ko_name)) + 5; /* dots + NUL */ 1217 if (dlen >= len) { 1218 d += len; 1219 dots = "/..."; 1220 } 1221 } 1222 snprintf(ko->ko_name, sizeof(ko->ko_name), "%s%s%s", name, dots, d); 1223 } 1224 1225 #else /* MODULAR */ 1226 1227 int 1228 kobj_load_mem(kobj_t *kop, const char *name, void *base, ssize_t size) 1229 { 1230 1231 return ENOSYS; 1232 } 1233 1234 void 1235 kobj_unload(kobj_t ko) 1236 { 1237 1238 panic("not modular"); 1239 } 1240 1241 int 1242 kobj_stat(kobj_t ko, vaddr_t *base, size_t *size) 1243 { 1244 1245 return ENOSYS; 1246 } 1247 1248 int 1249 kobj_affix(kobj_t ko, const char *name) 1250 { 1251 1252 panic("not modular"); 1253 } 1254 1255 int 1256 kobj_find_section(kobj_t ko, const char *name, void **addr, size_t *size) 1257 { 1258 1259 panic("not modular"); 1260 } 1261 1262 void 1263 kobj_setname(kobj_t ko, const char *name) 1264 { 1265 1266 panic("not modular"); 1267 } 1268 1269 #endif /* MODULAR */ 1270