1 /* $NetBSD: subr_kobj.c,v 1.66 2018/06/23 14:22:30 jakllsch Exp $ */ 2 3 /* 4 * Copyright (c) 2008 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software developed for The NetBSD Foundation 8 * by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /* 33 * Copyright (c) 1998-2000 Doug Rabson 34 * Copyright (c) 2004 Peter Wemm 35 * All rights reserved. 36 * 37 * Redistribution and use in source and binary forms, with or without 38 * modification, are permitted provided that the following conditions 39 * are met: 40 * 1. Redistributions of source code must retain the above copyright 41 * notice, this list of conditions and the following disclaimer. 42 * 2. Redistributions in binary form must reproduce the above copyright 43 * notice, this list of conditions and the following disclaimer in the 44 * documentation and/or other materials provided with the distribution. 45 * 46 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 47 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 48 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 49 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 50 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 51 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 52 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 53 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 54 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 55 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 56 * SUCH DAMAGE. 57 */ 58 59 /* 60 * Kernel loader for ELF objects. 61 * 62 * TODO: adjust kmem_alloc() calls to avoid needless fragmentation. 63 */ 64 65 #include <sys/cdefs.h> 66 __KERNEL_RCSID(0, "$NetBSD: subr_kobj.c,v 1.66 2018/06/23 14:22:30 jakllsch Exp $"); 67 68 #ifdef _KERNEL_OPT 69 #include "opt_modular.h" 70 #endif 71 72 #include <sys/kobj_impl.h> 73 74 #ifdef MODULAR 75 76 #include <sys/param.h> 77 #include <sys/kernel.h> 78 #include <sys/kmem.h> 79 #include <sys/proc.h> 80 #include <sys/ksyms.h> 81 #include <sys/module.h> 82 83 #include <uvm/uvm_extern.h> 84 85 #define kobj_error(_kobj, ...) \ 86 kobj_out(__func__, __LINE__, _kobj, __VA_ARGS__) 87 88 static int kobj_relocate(kobj_t, bool); 89 static int kobj_checksyms(kobj_t, bool); 90 static void kobj_out(const char *, int, kobj_t, const char *, ...) 91 __printflike(4, 5); 92 static void kobj_jettison(kobj_t); 93 static void kobj_free(kobj_t, void *, size_t); 94 static void kobj_close(kobj_t); 95 static int kobj_read_mem(kobj_t, void **, size_t, off_t, bool); 96 static void kobj_close_mem(kobj_t); 97 98 extern struct vm_map *module_map; 99 100 /* 101 * kobj_load_mem: 102 * 103 * Load an object already resident in memory. If size is not -1, 104 * the complete size of the object is known. 105 */ 106 int 107 kobj_load_mem(kobj_t *kop, const char *name, void *base, ssize_t size) 108 { 109 kobj_t ko; 110 111 ko = kmem_zalloc(sizeof(*ko), KM_SLEEP); 112 ko->ko_type = KT_MEMORY; 113 kobj_setname(ko, name); 114 ko->ko_source = base; 115 ko->ko_memsize = size; 116 ko->ko_read = kobj_read_mem; 117 ko->ko_close = kobj_close_mem; 118 119 *kop = ko; 120 return kobj_load(ko); 121 } 122 123 /* 124 * kobj_close: 125 * 126 * Close an open ELF object. 127 */ 128 static void 129 kobj_close(kobj_t ko) 130 { 131 132 if (ko->ko_source == NULL) { 133 return; 134 } 135 136 ko->ko_close(ko); 137 ko->ko_source = NULL; 138 } 139 140 static void 141 kobj_close_mem(kobj_t ko) 142 { 143 144 return; 145 } 146 147 /* 148 * kobj_load: 149 * 150 * Load an ELF object and prepare to link into the running kernel 151 * image. 152 */ 153 int 154 kobj_load(kobj_t ko) 155 { 156 Elf_Ehdr *hdr; 157 Elf_Shdr *shdr; 158 Elf_Sym *es; 159 vaddr_t map_text_base; 160 vaddr_t map_data_base; 161 vaddr_t map_rodata_base; 162 size_t map_text_size; 163 size_t map_data_size; 164 size_t map_rodata_size; 165 int error; 166 int symtabindex; 167 int symstrindex; 168 int nsym; 169 int pb, rl, ra; 170 int alignmask; 171 int i, j; 172 void *addr; 173 174 KASSERT(ko->ko_type != KT_UNSET); 175 KASSERT(ko->ko_source != NULL); 176 177 shdr = NULL; 178 error = 0; 179 hdr = NULL; 180 181 /* 182 * Read the elf header from the file. 183 */ 184 error = ko->ko_read(ko, (void **)&hdr, sizeof(*hdr), 0, true); 185 if (error != 0) { 186 kobj_error(ko, "read failed %d", error); 187 goto out; 188 } 189 if (memcmp(hdr->e_ident, ELFMAG, SELFMAG) != 0) { 190 kobj_error(ko, "not an ELF object"); 191 error = ENOEXEC; 192 goto out; 193 } 194 195 if (hdr->e_ident[EI_VERSION] != EV_CURRENT || 196 hdr->e_version != EV_CURRENT) { 197 kobj_error(ko, "unsupported file version %d", 198 hdr->e_ident[EI_VERSION]); 199 error = ENOEXEC; 200 goto out; 201 } 202 if (hdr->e_type != ET_REL) { 203 kobj_error(ko, "unsupported file type %d", hdr->e_type); 204 error = ENOEXEC; 205 goto out; 206 } 207 switch (hdr->e_machine) { 208 #if ELFSIZE == 32 209 ELF32_MACHDEP_ID_CASES 210 #elif ELFSIZE == 64 211 ELF64_MACHDEP_ID_CASES 212 #else 213 #error not defined 214 #endif 215 default: 216 kobj_error(ko, "unsupported machine %d", hdr->e_machine); 217 error = ENOEXEC; 218 goto out; 219 } 220 221 ko->ko_nprogtab = 0; 222 ko->ko_shdr = 0; 223 ko->ko_nrel = 0; 224 ko->ko_nrela = 0; 225 226 /* 227 * Allocate and read in the section header. 228 */ 229 if (hdr->e_shnum == 0 || hdr->e_shnum > ELF_MAXSHNUM || 230 hdr->e_shoff == 0 || hdr->e_shentsize != sizeof(Elf_Shdr)) { 231 kobj_error(ko, "bad sizes"); 232 error = ENOEXEC; 233 goto out; 234 } 235 ko->ko_shdrsz = hdr->e_shnum * sizeof(Elf_Shdr); 236 error = ko->ko_read(ko, (void **)&shdr, ko->ko_shdrsz, hdr->e_shoff, 237 true); 238 if (error != 0) { 239 kobj_error(ko, "read failed %d", error); 240 goto out; 241 } 242 ko->ko_shdr = shdr; 243 244 /* 245 * Scan the section header for information and table sizing. 246 */ 247 nsym = 0; 248 symtabindex = symstrindex = -1; 249 for (i = 0; i < hdr->e_shnum; i++) { 250 switch (shdr[i].sh_type) { 251 case SHT_PROGBITS: 252 case SHT_NOBITS: 253 ko->ko_nprogtab++; 254 break; 255 case SHT_SYMTAB: 256 nsym++; 257 symtabindex = i; 258 symstrindex = shdr[i].sh_link; 259 break; 260 case SHT_REL: 261 if (shdr[shdr[i].sh_info].sh_type != SHT_PROGBITS) 262 continue; 263 ko->ko_nrel++; 264 break; 265 case SHT_RELA: 266 if (shdr[shdr[i].sh_info].sh_type != SHT_PROGBITS) 267 continue; 268 ko->ko_nrela++; 269 break; 270 case SHT_STRTAB: 271 break; 272 } 273 } 274 if (ko->ko_nprogtab == 0) { 275 kobj_error(ko, "file has no contents"); 276 error = ENOEXEC; 277 goto out; 278 } 279 if (nsym != 1) { 280 /* Only allow one symbol table for now */ 281 kobj_error(ko, "file has no valid symbol table"); 282 error = ENOEXEC; 283 goto out; 284 } 285 KASSERT(symtabindex != -1); 286 KASSERT(symstrindex != -1); 287 288 if (symstrindex == SHN_UNDEF || symstrindex >= hdr->e_shnum || 289 shdr[symstrindex].sh_type != SHT_STRTAB) { 290 kobj_error(ko, "file has invalid symbol strings"); 291 error = ENOEXEC; 292 goto out; 293 } 294 295 /* 296 * Allocate space for tracking the load chunks. 297 */ 298 if (ko->ko_nprogtab != 0) { 299 ko->ko_progtab = kmem_zalloc(ko->ko_nprogtab * 300 sizeof(*ko->ko_progtab), KM_SLEEP); 301 if (ko->ko_progtab == NULL) { 302 error = ENOMEM; 303 kobj_error(ko, "out of memory"); 304 goto out; 305 } 306 } 307 if (ko->ko_nrel != 0) { 308 ko->ko_reltab = kmem_zalloc(ko->ko_nrel * 309 sizeof(*ko->ko_reltab), KM_SLEEP); 310 if (ko->ko_reltab == NULL) { 311 error = ENOMEM; 312 kobj_error(ko, "out of memory"); 313 goto out; 314 } 315 } 316 if (ko->ko_nrela != 0) { 317 ko->ko_relatab = kmem_zalloc(ko->ko_nrela * 318 sizeof(*ko->ko_relatab), KM_SLEEP); 319 if (ko->ko_relatab == NULL) { 320 error = ENOMEM; 321 kobj_error(ko, "out of memory"); 322 goto out; 323 } 324 } 325 326 /* 327 * Allocate space for and load the symbol table. 328 */ 329 ko->ko_symcnt = shdr[symtabindex].sh_size / sizeof(Elf_Sym); 330 if (ko->ko_symcnt == 0) { 331 kobj_error(ko, "no symbol table"); 332 error = ENOEXEC; 333 goto out; 334 } 335 error = ko->ko_read(ko, (void **)&ko->ko_symtab, 336 ko->ko_symcnt * sizeof(Elf_Sym), 337 shdr[symtabindex].sh_offset, true); 338 if (error != 0) { 339 kobj_error(ko, "read failed %d", error); 340 goto out; 341 } 342 343 /* 344 * Allocate space for and load the symbol strings. 345 */ 346 ko->ko_strtabsz = shdr[symstrindex].sh_size; 347 if (ko->ko_strtabsz == 0) { 348 kobj_error(ko, "no symbol strings"); 349 error = ENOEXEC; 350 goto out; 351 } 352 error = ko->ko_read(ko, (void *)&ko->ko_strtab, ko->ko_strtabsz, 353 shdr[symstrindex].sh_offset, true); 354 if (error != 0) { 355 kobj_error(ko, "read failed %d", error); 356 goto out; 357 } 358 359 /* 360 * Adjust module symbol namespace, if necessary (e.g. with rump) 361 */ 362 error = kobj_renamespace(ko->ko_symtab, ko->ko_symcnt, 363 &ko->ko_strtab, &ko->ko_strtabsz); 364 if (error != 0) { 365 kobj_error(ko, "renamespace failed %d", error); 366 goto out; 367 } 368 369 /* 370 * Do we have a string table for the section names? 371 */ 372 if (hdr->e_shstrndx != SHN_UNDEF) { 373 if (hdr->e_shstrndx >= hdr->e_shnum) { 374 kobj_error(ko, "bad shstrndx"); 375 error = ENOEXEC; 376 goto out; 377 } 378 if (shdr[hdr->e_shstrndx].sh_size != 0 && 379 shdr[hdr->e_shstrndx].sh_type == SHT_STRTAB) { 380 ko->ko_shstrtabsz = shdr[hdr->e_shstrndx].sh_size; 381 error = ko->ko_read(ko, (void **)&ko->ko_shstrtab, 382 shdr[hdr->e_shstrndx].sh_size, 383 shdr[hdr->e_shstrndx].sh_offset, true); 384 if (error != 0) { 385 kobj_error(ko, "read failed %d", error); 386 goto out; 387 } 388 } 389 } 390 391 /* 392 * Size up code/data(progbits) and bss(nobits). 393 */ 394 alignmask = 0; 395 map_text_size = 0; 396 map_data_size = 0; 397 map_rodata_size = 0; 398 for (i = 0; i < hdr->e_shnum; i++) { 399 if (shdr[i].sh_type != SHT_PROGBITS && 400 shdr[i].sh_type != SHT_NOBITS) 401 continue; 402 alignmask = shdr[i].sh_addralign - 1; 403 if ((shdr[i].sh_flags & SHF_EXECINSTR)) { 404 map_text_size += alignmask; 405 map_text_size &= ~alignmask; 406 map_text_size += shdr[i].sh_size; 407 } else if (!(shdr[i].sh_flags & SHF_WRITE)) { 408 map_rodata_size += alignmask; 409 map_rodata_size &= ~alignmask; 410 map_rodata_size += shdr[i].sh_size; 411 } else { 412 map_data_size += alignmask; 413 map_data_size &= ~alignmask; 414 map_data_size += shdr[i].sh_size; 415 } 416 } 417 418 if (map_text_size == 0) { 419 kobj_error(ko, "no text"); 420 error = ENOEXEC; 421 goto out; 422 } 423 424 if (map_data_size != 0) { 425 map_data_base = uvm_km_alloc(module_map, round_page(map_data_size), 426 0, UVM_KMF_WIRED); 427 if (map_data_base == 0) { 428 kobj_error(ko, "out of memory"); 429 error = ENOMEM; 430 goto out; 431 } 432 ko->ko_data_address = map_data_base; 433 ko->ko_data_size = map_data_size; 434 } else { 435 map_data_base = 0; 436 ko->ko_data_address = 0; 437 ko->ko_data_size = 0; 438 } 439 440 if (map_rodata_size != 0) { 441 map_rodata_base = uvm_km_alloc(module_map, round_page(map_rodata_size), 442 0, UVM_KMF_WIRED); 443 if (map_rodata_base == 0) { 444 kobj_error(ko, "out of memory"); 445 error = ENOMEM; 446 goto out; 447 } 448 ko->ko_rodata_address = map_rodata_base; 449 ko->ko_rodata_size = map_rodata_size; 450 } else { 451 map_rodata_base = 0; 452 ko->ko_rodata_address = 0; 453 ko->ko_rodata_size = 0; 454 } 455 456 map_text_base = uvm_km_alloc(module_map, round_page(map_text_size), 457 0, UVM_KMF_WIRED | UVM_KMF_EXEC); 458 if (map_text_base == 0) { 459 kobj_error(ko, "out of memory"); 460 error = ENOMEM; 461 goto out; 462 } 463 ko->ko_text_address = map_text_base; 464 ko->ko_text_size = map_text_size; 465 466 /* 467 * Now load code/data(progbits), zero bss(nobits), allocate space 468 * for and load relocs 469 */ 470 pb = 0; 471 rl = 0; 472 ra = 0; 473 alignmask = 0; 474 for (i = 0; i < hdr->e_shnum; i++) { 475 switch (shdr[i].sh_type) { 476 case SHT_PROGBITS: 477 case SHT_NOBITS: 478 alignmask = shdr[i].sh_addralign - 1; 479 if ((shdr[i].sh_flags & SHF_EXECINSTR)) { 480 map_text_base += alignmask; 481 map_text_base &= ~alignmask; 482 addr = (void *)map_text_base; 483 map_text_base += shdr[i].sh_size; 484 } else if (!(shdr[i].sh_flags & SHF_WRITE)) { 485 map_rodata_base += alignmask; 486 map_rodata_base &= ~alignmask; 487 addr = (void *)map_rodata_base; 488 map_rodata_base += shdr[i].sh_size; 489 } else { 490 map_data_base += alignmask; 491 map_data_base &= ~alignmask; 492 addr = (void *)map_data_base; 493 map_data_base += shdr[i].sh_size; 494 } 495 496 ko->ko_progtab[pb].addr = addr; 497 if (shdr[i].sh_type == SHT_PROGBITS) { 498 ko->ko_progtab[pb].name = "<<PROGBITS>>"; 499 error = ko->ko_read(ko, &addr, 500 shdr[i].sh_size, shdr[i].sh_offset, false); 501 if (error != 0) { 502 kobj_error(ko, "read failed %d", error); 503 goto out; 504 } 505 } else { /* SHT_NOBITS */ 506 ko->ko_progtab[pb].name = "<<NOBITS>>"; 507 memset(addr, 0, shdr[i].sh_size); 508 } 509 510 ko->ko_progtab[pb].size = shdr[i].sh_size; 511 ko->ko_progtab[pb].sec = i; 512 if (ko->ko_shstrtab != NULL && shdr[i].sh_name != 0) { 513 ko->ko_progtab[pb].name = 514 ko->ko_shstrtab + shdr[i].sh_name; 515 } 516 517 /* Update all symbol values with the offset. */ 518 for (j = 0; j < ko->ko_symcnt; j++) { 519 es = &ko->ko_symtab[j]; 520 if (es->st_shndx != i) { 521 continue; 522 } 523 es->st_value += (Elf_Addr)addr; 524 } 525 pb++; 526 break; 527 case SHT_REL: 528 if (shdr[shdr[i].sh_info].sh_type != SHT_PROGBITS) 529 break; 530 ko->ko_reltab[rl].size = shdr[i].sh_size; 531 ko->ko_reltab[rl].size -= 532 shdr[i].sh_size % sizeof(Elf_Rel); 533 if (ko->ko_reltab[rl].size != 0) { 534 ko->ko_reltab[rl].nrel = 535 shdr[i].sh_size / sizeof(Elf_Rel); 536 ko->ko_reltab[rl].sec = shdr[i].sh_info; 537 error = ko->ko_read(ko, 538 (void **)&ko->ko_reltab[rl].rel, 539 ko->ko_reltab[rl].size, 540 shdr[i].sh_offset, true); 541 if (error != 0) { 542 kobj_error(ko, "read failed %d", 543 error); 544 goto out; 545 } 546 } 547 rl++; 548 break; 549 case SHT_RELA: 550 if (shdr[shdr[i].sh_info].sh_type != SHT_PROGBITS) 551 break; 552 ko->ko_relatab[ra].size = shdr[i].sh_size; 553 ko->ko_relatab[ra].size -= 554 shdr[i].sh_size % sizeof(Elf_Rela); 555 if (ko->ko_relatab[ra].size != 0) { 556 ko->ko_relatab[ra].nrela = 557 shdr[i].sh_size / sizeof(Elf_Rela); 558 ko->ko_relatab[ra].sec = shdr[i].sh_info; 559 error = ko->ko_read(ko, 560 (void **)&ko->ko_relatab[ra].rela, 561 shdr[i].sh_size, 562 shdr[i].sh_offset, true); 563 if (error != 0) { 564 kobj_error(ko, "read failed %d", error); 565 goto out; 566 } 567 } 568 ra++; 569 break; 570 default: 571 break; 572 } 573 } 574 if (pb != ko->ko_nprogtab) { 575 panic("%s:%d: %s: lost progbits", __func__, __LINE__, 576 ko->ko_name); 577 } 578 if (rl != ko->ko_nrel) { 579 panic("%s:%d: %s: lost rel", __func__, __LINE__, 580 ko->ko_name); 581 } 582 if (ra != ko->ko_nrela) { 583 panic("%s:%d: %s: lost rela", __func__, __LINE__, 584 ko->ko_name); 585 } 586 if (map_text_base != ko->ko_text_address + map_text_size) { 587 panic("%s:%d: %s: map_text_base 0x%lx != address %lx " 588 "+ map_text_size %ld (0x%lx)\n", 589 __func__, __LINE__, ko->ko_name, (long)map_text_base, 590 (long)ko->ko_text_address, (long)map_text_size, 591 (long)ko->ko_text_address + map_text_size); 592 } 593 if (map_data_base != ko->ko_data_address + map_data_size) { 594 panic("%s:%d: %s: map_data_base 0x%lx != address %lx " 595 "+ map_data_size %ld (0x%lx)\n", 596 __func__, __LINE__, ko->ko_name, (long)map_data_base, 597 (long)ko->ko_data_address, (long)map_data_size, 598 (long)ko->ko_data_address + map_data_size); 599 } 600 if (map_rodata_base != ko->ko_rodata_address + map_rodata_size) { 601 panic("%s:%d: %s: map_rodata_base 0x%lx != address %lx " 602 "+ map_rodata_size %ld (0x%lx)\n", 603 __func__, __LINE__, ko->ko_name, (long)map_rodata_base, 604 (long)ko->ko_rodata_address, (long)map_rodata_size, 605 (long)ko->ko_rodata_address + map_rodata_size); 606 } 607 608 /* 609 * Perform local relocations only. Relocations relating to global 610 * symbols will be done by kobj_affix(). 611 */ 612 error = kobj_checksyms(ko, false); 613 if (error == 0) { 614 error = kobj_relocate(ko, true); 615 } 616 out: 617 if (hdr != NULL) { 618 kobj_free(ko, hdr, sizeof(*hdr)); 619 } 620 kobj_close(ko); 621 if (error != 0) { 622 kobj_unload(ko); 623 } 624 625 return error; 626 } 627 628 static void 629 kobj_unload_notify(kobj_t ko, vaddr_t addr, size_t size, const char *note) 630 { 631 if (addr == 0) 632 return; 633 634 int error = kobj_machdep(ko, (void *)addr, size, false); 635 if (error) 636 kobj_error(ko, "machine dependent deinit failed (%s) %d", 637 note, error); 638 } 639 640 #define KOBJ_SEGMENT_NOTIFY(ko, what) \ 641 kobj_unload_notify(ko, (ko)->ko_ ## what ## _address, \ 642 (ko)->ko_ ## what ## _size, # what); 643 644 #define KOBJ_SEGMENT_FREE(ko, what) \ 645 do \ 646 if ((ko)->ko_ ## what ## _address != 0) \ 647 uvm_km_free(module_map, (ko)->ko_ ## what ## _address, \ 648 round_page((ko)->ko_ ## what ## _size), UVM_KMF_WIRED); \ 649 while (/*CONSTCOND*/ 0) 650 651 /* 652 * kobj_unload: 653 * 654 * Unload an object previously loaded by kobj_load(). 655 */ 656 void 657 kobj_unload(kobj_t ko) 658 { 659 kobj_close(ko); 660 kobj_jettison(ko); 661 662 663 /* 664 * Notify MD code that a module has been unloaded. 665 */ 666 if (ko->ko_loaded) { 667 KOBJ_SEGMENT_NOTIFY(ko, text); 668 KOBJ_SEGMENT_NOTIFY(ko, data); 669 KOBJ_SEGMENT_NOTIFY(ko, rodata); 670 } 671 672 KOBJ_SEGMENT_FREE(ko, text); 673 KOBJ_SEGMENT_FREE(ko, data); 674 KOBJ_SEGMENT_FREE(ko, rodata); 675 676 if (ko->ko_ksyms == true) { 677 ksyms_modunload(ko->ko_name); 678 } 679 if (ko->ko_symtab != NULL) { 680 kobj_free(ko, ko->ko_symtab, ko->ko_symcnt * sizeof(Elf_Sym)); 681 } 682 if (ko->ko_strtab != NULL) { 683 kobj_free(ko, ko->ko_strtab, ko->ko_strtabsz); 684 } 685 if (ko->ko_progtab != NULL) { 686 kobj_free(ko, ko->ko_progtab, ko->ko_nprogtab * 687 sizeof(*ko->ko_progtab)); 688 ko->ko_progtab = NULL; 689 } 690 if (ko->ko_shstrtab) { 691 kobj_free(ko, ko->ko_shstrtab, ko->ko_shstrtabsz); 692 ko->ko_shstrtab = NULL; 693 } 694 695 kmem_free(ko, sizeof(*ko)); 696 } 697 698 /* 699 * kobj_stat: 700 * 701 * Return size and load address of an object. 702 */ 703 int 704 kobj_stat(kobj_t ko, vaddr_t *address, size_t *size) 705 { 706 707 if (address != NULL) { 708 *address = ko->ko_text_address; 709 } 710 if (size != NULL) { 711 *size = ko->ko_text_size; 712 } 713 return 0; 714 } 715 716 /* 717 * kobj_affix: 718 * 719 * Set an object's name and perform global relocs. May only be 720 * called after the module and any requisite modules are loaded. 721 */ 722 int 723 kobj_affix(kobj_t ko, const char *name) 724 { 725 int error; 726 727 KASSERT(ko->ko_ksyms == false); 728 KASSERT(ko->ko_loaded == false); 729 730 kobj_setname(ko, name); 731 732 /* Cache addresses of undefined symbols. */ 733 error = kobj_checksyms(ko, true); 734 735 /* Now do global relocations. */ 736 if (error == 0) 737 error = kobj_relocate(ko, false); 738 739 /* 740 * Now that we know the name, register the symbol table. 741 * Do after global relocations because ksyms will pack 742 * the table. 743 */ 744 if (error == 0) { 745 ksyms_modload(ko->ko_name, ko->ko_symtab, ko->ko_symcnt * 746 sizeof(Elf_Sym), ko->ko_strtab, ko->ko_strtabsz); 747 ko->ko_ksyms = true; 748 } 749 750 /* Jettison unneeded memory post-link. */ 751 kobj_jettison(ko); 752 753 /* 754 * Notify MD code that a module has been loaded. 755 * 756 * Most architectures use this opportunity to flush their caches. 757 */ 758 if (error == 0 && ko->ko_text_address != 0) { 759 error = kobj_machdep(ko, (void *)ko->ko_text_address, 760 ko->ko_text_size, true); 761 if (error != 0) 762 kobj_error(ko, "machine dependent init failed (text)" 763 " %d", error); 764 } 765 766 if (error == 0 && ko->ko_data_address != 0) { 767 error = kobj_machdep(ko, (void *)ko->ko_data_address, 768 ko->ko_data_size, true); 769 if (error != 0) 770 kobj_error(ko, "machine dependent init failed (data)" 771 " %d", error); 772 } 773 774 if (error == 0 && ko->ko_rodata_address != 0) { 775 error = kobj_machdep(ko, (void *)ko->ko_rodata_address, 776 ko->ko_rodata_size, true); 777 if (error != 0) 778 kobj_error(ko, "machine dependent init failed (rodata)" 779 " %d", error); 780 } 781 782 if (error == 0) { 783 ko->ko_loaded = true; 784 785 /* Change the memory protections, when needed. */ 786 if (ko->ko_text_address != 0) { 787 uvm_km_protect(module_map, ko->ko_text_address, 788 ko->ko_text_size, VM_PROT_READ|VM_PROT_EXECUTE); 789 } 790 if (ko->ko_rodata_address != 0) { 791 uvm_km_protect(module_map, ko->ko_rodata_address, 792 ko->ko_rodata_size, VM_PROT_READ); 793 } 794 } else { 795 /* If there was an error, destroy the whole object. */ 796 kobj_unload(ko); 797 } 798 799 return error; 800 } 801 802 /* 803 * kobj_find_section: 804 * 805 * Given a section name, search the loaded object and return 806 * virtual address if present and loaded. 807 */ 808 int 809 kobj_find_section(kobj_t ko, const char *name, void **addr, size_t *size) 810 { 811 int i; 812 813 KASSERT(ko->ko_progtab != NULL); 814 815 for (i = 0; i < ko->ko_nprogtab; i++) { 816 if (strcmp(ko->ko_progtab[i].name, name) == 0) { 817 if (addr != NULL) { 818 *addr = ko->ko_progtab[i].addr; 819 } 820 if (size != NULL) { 821 *size = ko->ko_progtab[i].size; 822 } 823 return 0; 824 } 825 } 826 827 return ENOENT; 828 } 829 830 /* 831 * kobj_jettison: 832 * 833 * Release object data not needed after performing relocations. 834 */ 835 static void 836 kobj_jettison(kobj_t ko) 837 { 838 int i; 839 840 if (ko->ko_reltab != NULL) { 841 for (i = 0; i < ko->ko_nrel; i++) { 842 if (ko->ko_reltab[i].rel) { 843 kobj_free(ko, ko->ko_reltab[i].rel, 844 ko->ko_reltab[i].size); 845 } 846 } 847 kobj_free(ko, ko->ko_reltab, ko->ko_nrel * 848 sizeof(*ko->ko_reltab)); 849 ko->ko_reltab = NULL; 850 ko->ko_nrel = 0; 851 } 852 if (ko->ko_relatab != NULL) { 853 for (i = 0; i < ko->ko_nrela; i++) { 854 if (ko->ko_relatab[i].rela) { 855 kobj_free(ko, ko->ko_relatab[i].rela, 856 ko->ko_relatab[i].size); 857 } 858 } 859 kobj_free(ko, ko->ko_relatab, ko->ko_nrela * 860 sizeof(*ko->ko_relatab)); 861 ko->ko_relatab = NULL; 862 ko->ko_nrela = 0; 863 } 864 if (ko->ko_shdr != NULL) { 865 kobj_free(ko, ko->ko_shdr, ko->ko_shdrsz); 866 ko->ko_shdr = NULL; 867 } 868 } 869 870 /* 871 * kobj_sym_lookup: 872 * 873 * Symbol lookup function to be used when the symbol index 874 * is known (ie during relocation). 875 */ 876 int 877 kobj_sym_lookup(kobj_t ko, uintptr_t symidx, Elf_Addr *val) 878 { 879 const Elf_Sym *sym; 880 const char *symbol; 881 882 sym = ko->ko_symtab + symidx; 883 884 if (symidx == SHN_ABS) { 885 *val = (uintptr_t)sym->st_value; 886 return 0; 887 } else if (symidx >= ko->ko_symcnt) { 888 /* 889 * Don't even try to lookup the symbol if the index is 890 * bogus. 891 */ 892 kobj_error(ko, "symbol index out of range"); 893 return EINVAL; 894 } 895 896 /* Quick answer if there is a definition included. */ 897 if (sym->st_shndx != SHN_UNDEF) { 898 *val = (uintptr_t)sym->st_value; 899 return 0; 900 } 901 902 /* If we get here, then it is undefined and needs a lookup. */ 903 switch (ELF_ST_BIND(sym->st_info)) { 904 case STB_LOCAL: 905 /* Local, but undefined? huh? */ 906 kobj_error(ko, "local symbol undefined"); 907 return EINVAL; 908 909 case STB_GLOBAL: 910 /* Relative to Data or Function name */ 911 symbol = ko->ko_strtab + sym->st_name; 912 913 /* Force a lookup failure if the symbol name is bogus. */ 914 if (*symbol == 0) { 915 kobj_error(ko, "bad symbol name"); 916 return EINVAL; 917 } 918 if (sym->st_value == 0) { 919 kobj_error(ko, "bad value"); 920 return EINVAL; 921 } 922 923 *val = (uintptr_t)sym->st_value; 924 return 0; 925 926 case STB_WEAK: 927 kobj_error(ko, "weak symbols not supported"); 928 return EINVAL; 929 930 default: 931 return EINVAL; 932 } 933 } 934 935 /* 936 * kobj_findbase: 937 * 938 * Return base address of the given section. 939 */ 940 static uintptr_t 941 kobj_findbase(kobj_t ko, int sec) 942 { 943 int i; 944 945 for (i = 0; i < ko->ko_nprogtab; i++) { 946 if (sec == ko->ko_progtab[i].sec) { 947 return (uintptr_t)ko->ko_progtab[i].addr; 948 } 949 } 950 return 0; 951 } 952 953 /* 954 * kobj_checksyms: 955 * 956 * Scan symbol table for duplicates or resolve references to 957 * exernal symbols. 958 */ 959 static int 960 kobj_checksyms(kobj_t ko, bool undefined) 961 { 962 unsigned long rval; 963 Elf_Sym *sym, *ksym, *ms; 964 const char *name; 965 int error; 966 967 error = 0; 968 969 for (ms = (sym = ko->ko_symtab) + ko->ko_symcnt; sym < ms; sym++) { 970 /* Check validity of the symbol. */ 971 if (ELF_ST_BIND(sym->st_info) != STB_GLOBAL || 972 sym->st_name == 0) 973 continue; 974 if (undefined != (sym->st_shndx == SHN_UNDEF)) { 975 continue; 976 } 977 978 /* 979 * Look it up. Don't need to lock, as it is known that 980 * the symbol tables aren't going to change (we hold 981 * module_lock). 982 */ 983 name = ko->ko_strtab + sym->st_name; 984 if (ksyms_getval_unlocked(NULL, name, &ksym, &rval, 985 KSYMS_EXTERN) != 0) { 986 if (undefined) { 987 kobj_error(ko, "symbol `%s' not found", 988 name); 989 error = ENOEXEC; 990 } 991 continue; 992 } 993 994 /* Save values of undefined globals. */ 995 if (undefined) { 996 if (ksym->st_shndx == SHN_ABS) { 997 sym->st_shndx = SHN_ABS; 998 } 999 sym->st_value = (Elf_Addr)rval; 1000 continue; 1001 } 1002 1003 /* Check (and complain) about differing values. */ 1004 if (sym->st_value == rval) { 1005 continue; 1006 } 1007 if (strcmp(name, "_bss_start") == 0 || 1008 strcmp(name, "__bss_start") == 0 || 1009 strcmp(name, "_bss_end__") == 0 || 1010 strcmp(name, "__bss_end__") == 0 || 1011 strcmp(name, "_edata") == 0 || 1012 strcmp(name, "_end") == 0 || 1013 strcmp(name, "__end") == 0 || 1014 strcmp(name, "__end__") == 0 || 1015 strncmp(name, "__start_link_set_", 17) == 0 || 1016 strncmp(name, "__stop_link_set_", 16) == 0) { 1017 continue; 1018 } 1019 kobj_error(ko, "global symbol `%s' redefined", 1020 name); 1021 error = ENOEXEC; 1022 } 1023 1024 return error; 1025 } 1026 1027 /* 1028 * kobj_relocate: 1029 * 1030 * Resolve relocations for the loaded object. 1031 */ 1032 static int 1033 kobj_relocate(kobj_t ko, bool local) 1034 { 1035 const Elf_Rel *rellim; 1036 const Elf_Rel *rel; 1037 const Elf_Rela *relalim; 1038 const Elf_Rela *rela; 1039 const Elf_Sym *sym; 1040 uintptr_t base; 1041 int i, error; 1042 uintptr_t symidx; 1043 1044 /* 1045 * Perform relocations without addend if there are any. 1046 */ 1047 for (i = 0; i < ko->ko_nrel; i++) { 1048 rel = ko->ko_reltab[i].rel; 1049 if (rel == NULL) { 1050 continue; 1051 } 1052 rellim = rel + ko->ko_reltab[i].nrel; 1053 base = kobj_findbase(ko, ko->ko_reltab[i].sec); 1054 if (base == 0) { 1055 panic("%s:%d: %s: lost base for e_reltab[%d] sec %d", 1056 __func__, __LINE__, ko->ko_name, i, 1057 ko->ko_reltab[i].sec); 1058 } 1059 for (; rel < rellim; rel++) { 1060 symidx = ELF_R_SYM(rel->r_info); 1061 if (symidx >= ko->ko_symcnt) { 1062 continue; 1063 } 1064 sym = ko->ko_symtab + symidx; 1065 if (local != (ELF_ST_BIND(sym->st_info) == STB_LOCAL)) { 1066 continue; 1067 } 1068 error = kobj_reloc(ko, base, rel, false, local); 1069 if (error != 0) { 1070 return ENOENT; 1071 } 1072 } 1073 } 1074 1075 /* 1076 * Perform relocations with addend if there are any. 1077 */ 1078 for (i = 0; i < ko->ko_nrela; i++) { 1079 rela = ko->ko_relatab[i].rela; 1080 if (rela == NULL) { 1081 continue; 1082 } 1083 relalim = rela + ko->ko_relatab[i].nrela; 1084 base = kobj_findbase(ko, ko->ko_relatab[i].sec); 1085 if (base == 0) { 1086 panic("%s:%d: %s: lost base for e_relatab[%d] sec %d", 1087 __func__, __LINE__, ko->ko_name, i, 1088 ko->ko_relatab[i].sec); 1089 } 1090 for (; rela < relalim; rela++) { 1091 symidx = ELF_R_SYM(rela->r_info); 1092 if (symidx >= ko->ko_symcnt) { 1093 continue; 1094 } 1095 sym = ko->ko_symtab + symidx; 1096 if (local != (ELF_ST_BIND(sym->st_info) == STB_LOCAL)) { 1097 continue; 1098 } 1099 error = kobj_reloc(ko, base, rela, true, local); 1100 if (error != 0) { 1101 return ENOENT; 1102 } 1103 } 1104 } 1105 1106 return 0; 1107 } 1108 1109 /* 1110 * kobj_out: 1111 * 1112 * Utility function: log an error. 1113 */ 1114 static void 1115 kobj_out(const char *fname, int lnum, kobj_t ko, const char *fmt, ...) 1116 { 1117 va_list ap; 1118 1119 printf("%s, %d: [%s]: linker error: ", fname, lnum, ko->ko_name); 1120 va_start(ap, fmt); 1121 vprintf(fmt, ap); 1122 va_end(ap); 1123 printf("\n"); 1124 } 1125 1126 static int 1127 kobj_read_mem(kobj_t ko, void **basep, size_t size, off_t off, 1128 bool allocate) 1129 { 1130 void *base = *basep; 1131 int error; 1132 1133 KASSERT(ko->ko_source != NULL); 1134 1135 if (ko->ko_memsize != -1 && off + size > ko->ko_memsize) { 1136 kobj_error(ko, "preloaded object short"); 1137 error = EINVAL; 1138 base = NULL; 1139 } else if (allocate) { 1140 base = kmem_alloc(size, KM_SLEEP); 1141 error = 0; 1142 } else { 1143 error = 0; 1144 } 1145 1146 if (error == 0) { 1147 /* Copy the section */ 1148 memcpy(base, (uint8_t *)ko->ko_source + off, size); 1149 } 1150 1151 if (allocate && error != 0) { 1152 kmem_free(base, size); 1153 base = NULL; 1154 } 1155 1156 if (allocate) 1157 *basep = base; 1158 1159 return error; 1160 } 1161 1162 /* 1163 * kobj_free: 1164 * 1165 * Utility function: free memory if it was allocated from the heap. 1166 */ 1167 static void 1168 kobj_free(kobj_t ko, void *base, size_t size) 1169 { 1170 1171 kmem_free(base, size); 1172 } 1173 1174 extern char module_base[]; 1175 1176 void 1177 kobj_setname(kobj_t ko, const char *name) 1178 { 1179 const char *d = name, *dots = ""; 1180 size_t len, dlen; 1181 1182 for (char *s = module_base; *d == *s; d++, s++) 1183 continue; 1184 1185 if (d == name) 1186 name = ""; 1187 else 1188 name = "%M"; 1189 dlen = strlen(d); 1190 len = dlen + strlen(name); 1191 if (len >= sizeof(ko->ko_name)) { 1192 len = (len - sizeof(ko->ko_name)) + 5; /* dots + NUL */ 1193 if (dlen >= len) { 1194 d += len; 1195 dots = "/..."; 1196 } 1197 } 1198 snprintf(ko->ko_name, sizeof(ko->ko_name), "%s%s%s", name, dots, d); 1199 } 1200 1201 #else /* MODULAR */ 1202 1203 int 1204 kobj_load_mem(kobj_t *kop, const char *name, void *base, ssize_t size) 1205 { 1206 1207 return ENOSYS; 1208 } 1209 1210 void 1211 kobj_unload(kobj_t ko) 1212 { 1213 1214 panic("not modular"); 1215 } 1216 1217 int 1218 kobj_stat(kobj_t ko, vaddr_t *base, size_t *size) 1219 { 1220 1221 return ENOSYS; 1222 } 1223 1224 int 1225 kobj_affix(kobj_t ko, const char *name) 1226 { 1227 1228 panic("not modular"); 1229 } 1230 1231 int 1232 kobj_find_section(kobj_t ko, const char *name, void **addr, size_t *size) 1233 { 1234 1235 panic("not modular"); 1236 } 1237 1238 void 1239 kobj_setname(kobj_t ko, const char *name) 1240 { 1241 1242 panic("not modular"); 1243 } 1244 1245 #endif /* MODULAR */ 1246