1 /* $NetBSD: uvm_map.c,v 1.88 2001/01/14 02:10:01 thorpej Exp $ */ 2 3 /* 4 * Copyright (c) 1997 Charles D. Cranor and Washington University. 5 * Copyright (c) 1991, 1993, The Regents of the University of California. 6 * 7 * All rights reserved. 8 * 9 * This code is derived from software contributed to Berkeley by 10 * The Mach Operating System project at Carnegie-Mellon University. 11 * 12 * Redistribution and use in source and binary forms, with or without 13 * modification, are permitted provided that the following conditions 14 * are met: 15 * 1. Redistributions of source code must retain the above copyright 16 * notice, this list of conditions and the following disclaimer. 17 * 2. Redistributions in binary form must reproduce the above copyright 18 * notice, this list of conditions and the following disclaimer in the 19 * documentation and/or other materials provided with the distribution. 20 * 3. All advertising materials mentioning features or use of this software 21 * must display the following acknowledgement: 22 * This product includes software developed by Charles D. Cranor, 23 * Washington University, the University of California, Berkeley and 24 * its contributors. 25 * 4. Neither the name of the University nor the names of its contributors 26 * may be used to endorse or promote products derived from this software 27 * without specific prior written permission. 28 * 29 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 30 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 31 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 32 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 33 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 34 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 35 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 36 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 37 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 38 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 39 * SUCH DAMAGE. 40 * 41 * @(#)vm_map.c 8.3 (Berkeley) 1/12/94 42 * from: Id: uvm_map.c,v 1.1.2.27 1998/02/07 01:16:54 chs Exp 43 * 44 * 45 * Copyright (c) 1987, 1990 Carnegie-Mellon University. 46 * All rights reserved. 47 * 48 * Permission to use, copy, modify and distribute this software and 49 * its documentation is hereby granted, provided that both the copyright 50 * notice and this permission notice appear in all copies of the 51 * software, derivative works or modified versions, and any portions 52 * thereof, and that both notices appear in supporting documentation. 53 * 54 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" 55 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND 56 * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. 57 * 58 * Carnegie Mellon requests users of this software to return to 59 * 60 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU 61 * School of Computer Science 62 * Carnegie Mellon University 63 * Pittsburgh PA 15213-3890 64 * 65 * any improvements or extensions that they make and grant Carnegie the 66 * rights to redistribute these changes. 67 */ 68 69 #include "opt_ddb.h" 70 #include "opt_uvmhist.h" 71 #include "opt_sysv.h" 72 73 /* 74 * uvm_map.c: uvm map operations 75 */ 76 77 #include <sys/param.h> 78 #include <sys/systm.h> 79 #include <sys/mman.h> 80 #include <sys/proc.h> 81 #include <sys/malloc.h> 82 #include <sys/pool.h> 83 84 #ifdef SYSVSHM 85 #include <sys/shm.h> 86 #endif 87 88 #define UVM_MAP 89 #include <uvm/uvm.h> 90 91 #ifdef DDB 92 #include <uvm/uvm_ddb.h> 93 #endif 94 95 96 struct uvm_cnt uvm_map_call, map_backmerge, map_forwmerge; 97 struct uvm_cnt uvm_mlk_call, uvm_mlk_hint; 98 const char vmmapbsy[] = "vmmapbsy"; 99 100 /* 101 * pool for vmspace structures. 102 */ 103 104 struct pool uvm_vmspace_pool; 105 106 /* 107 * pool for dynamically-allocated map entries. 108 */ 109 110 struct pool uvm_map_entry_pool; 111 112 #ifdef PMAP_GROWKERNEL 113 /* 114 * This global represents the end of the kernel virtual address 115 * space. If we want to exceed this, we must grow the kernel 116 * virtual address space dynamically. 117 * 118 * Note, this variable is locked by kernel_map's lock. 119 */ 120 vaddr_t uvm_maxkaddr; 121 #endif 122 123 /* 124 * macros 125 */ 126 127 /* 128 * uvm_map_entry_link: insert entry into a map 129 * 130 * => map must be locked 131 */ 132 #define uvm_map_entry_link(map, after_where, entry) do { \ 133 (map)->nentries++; \ 134 (entry)->prev = (after_where); \ 135 (entry)->next = (after_where)->next; \ 136 (entry)->prev->next = (entry); \ 137 (entry)->next->prev = (entry); \ 138 } while (0) 139 140 /* 141 * uvm_map_entry_unlink: remove entry from a map 142 * 143 * => map must be locked 144 */ 145 #define uvm_map_entry_unlink(map, entry) do { \ 146 (map)->nentries--; \ 147 (entry)->next->prev = (entry)->prev; \ 148 (entry)->prev->next = (entry)->next; \ 149 } while (0) 150 151 /* 152 * SAVE_HINT: saves the specified entry as the hint for future lookups. 153 * 154 * => map need not be locked (protected by hint_lock). 155 */ 156 #define SAVE_HINT(map,check,value) do { \ 157 simple_lock(&(map)->hint_lock); \ 158 if ((map)->hint == (check)) \ 159 (map)->hint = (value); \ 160 simple_unlock(&(map)->hint_lock); \ 161 } while (0) 162 163 /* 164 * VM_MAP_RANGE_CHECK: check and correct range 165 * 166 * => map must at least be read locked 167 */ 168 169 #define VM_MAP_RANGE_CHECK(map, start, end) do { \ 170 if (start < vm_map_min(map)) \ 171 start = vm_map_min(map); \ 172 if (end > vm_map_max(map)) \ 173 end = vm_map_max(map); \ 174 if (start > end) \ 175 start = end; \ 176 } while (0) 177 178 /* 179 * local prototypes 180 */ 181 182 static vm_map_entry_t uvm_mapent_alloc __P((vm_map_t)); 183 static void uvm_mapent_copy __P((vm_map_entry_t,vm_map_entry_t)); 184 static void uvm_mapent_free __P((vm_map_entry_t)); 185 static void uvm_map_entry_unwire __P((vm_map_t, vm_map_entry_t)); 186 static void uvm_map_reference_amap __P((vm_map_entry_t, int)); 187 static void uvm_map_unreference_amap __P((vm_map_entry_t, int)); 188 189 /* 190 * local inlines 191 */ 192 193 /* 194 * uvm_mapent_alloc: allocate a map entry 195 * 196 * => XXX: static pool for kernel map? 197 */ 198 199 static __inline vm_map_entry_t 200 uvm_mapent_alloc(map) 201 vm_map_t map; 202 { 203 vm_map_entry_t me; 204 int s; 205 UVMHIST_FUNC("uvm_mapent_alloc"); 206 UVMHIST_CALLED(maphist); 207 208 if ((map->flags & VM_MAP_INTRSAFE) == 0 && 209 map != kernel_map && kernel_map != NULL /* XXX */) { 210 me = pool_get(&uvm_map_entry_pool, PR_WAITOK); 211 me->flags = 0; 212 /* me can't be null, wait ok */ 213 } else { 214 s = splvm(); /* protect kentry_free list with splvm */ 215 simple_lock(&uvm.kentry_lock); 216 me = uvm.kentry_free; 217 if (me) uvm.kentry_free = me->next; 218 simple_unlock(&uvm.kentry_lock); 219 splx(s); 220 if (!me) 221 panic("mapent_alloc: out of static map entries, check MAX_KMAPENT"); 222 me->flags = UVM_MAP_STATIC; 223 } 224 225 UVMHIST_LOG(maphist, "<- new entry=0x%x [kentry=%d]", 226 me, ((map->flags & VM_MAP_INTRSAFE) != 0 || map == kernel_map) 227 ? TRUE : FALSE, 0, 0); 228 return(me); 229 } 230 231 /* 232 * uvm_mapent_free: free map entry 233 * 234 * => XXX: static pool for kernel map? 235 */ 236 237 static __inline void 238 uvm_mapent_free(me) 239 vm_map_entry_t me; 240 { 241 int s; 242 UVMHIST_FUNC("uvm_mapent_free"); 243 UVMHIST_CALLED(maphist); 244 UVMHIST_LOG(maphist,"<- freeing map entry=0x%x [flags=%d]", 245 me, me->flags, 0, 0); 246 if ((me->flags & UVM_MAP_STATIC) == 0) { 247 pool_put(&uvm_map_entry_pool, me); 248 } else { 249 s = splvm(); /* protect kentry_free list with splvm */ 250 simple_lock(&uvm.kentry_lock); 251 me->next = uvm.kentry_free; 252 uvm.kentry_free = me; 253 simple_unlock(&uvm.kentry_lock); 254 splx(s); 255 } 256 } 257 258 /* 259 * uvm_mapent_copy: copy a map entry, preserving flags 260 */ 261 262 static __inline void 263 uvm_mapent_copy(src, dst) 264 vm_map_entry_t src; 265 vm_map_entry_t dst; 266 { 267 268 memcpy(dst, src, ((char *)&src->uvm_map_entry_stop_copy) - ((char*)src)); 269 } 270 271 /* 272 * uvm_map_entry_unwire: unwire a map entry 273 * 274 * => map should be locked by caller 275 */ 276 277 static __inline void 278 uvm_map_entry_unwire(map, entry) 279 vm_map_t map; 280 vm_map_entry_t entry; 281 { 282 283 entry->wired_count = 0; 284 uvm_fault_unwire_locked(map, entry->start, entry->end); 285 } 286 287 288 /* 289 * wrapper for calling amap_ref() 290 */ 291 static __inline void 292 uvm_map_reference_amap(entry, flags) 293 vm_map_entry_t entry; 294 int flags; 295 { 296 amap_ref(entry->aref.ar_amap, entry->aref.ar_pageoff, 297 (entry->end - entry->start) >> PAGE_SHIFT, flags); 298 } 299 300 301 /* 302 * wrapper for calling amap_unref() 303 */ 304 static __inline void 305 uvm_map_unreference_amap(entry, flags) 306 vm_map_entry_t entry; 307 int flags; 308 { 309 amap_unref(entry->aref.ar_amap, entry->aref.ar_pageoff, 310 (entry->end - entry->start) >> PAGE_SHIFT, flags); 311 } 312 313 314 /* 315 * uvm_map_init: init mapping system at boot time. note that we allocate 316 * and init the static pool of vm_map_entry_t's for the kernel here. 317 */ 318 319 void 320 uvm_map_init() 321 { 322 static struct vm_map_entry kernel_map_entry[MAX_KMAPENT]; 323 #if defined(UVMHIST) 324 static struct uvm_history_ent maphistbuf[100]; 325 static struct uvm_history_ent pdhistbuf[100]; 326 #endif 327 int lcv; 328 329 /* 330 * first, init logging system. 331 */ 332 333 UVMHIST_FUNC("uvm_map_init"); 334 UVMHIST_INIT_STATIC(maphist, maphistbuf); 335 UVMHIST_INIT_STATIC(pdhist, pdhistbuf); 336 UVMHIST_CALLED(maphist); 337 UVMHIST_LOG(maphist,"<starting uvm map system>", 0, 0, 0, 0); 338 UVMCNT_INIT(uvm_map_call, UVMCNT_CNT, 0, 339 "# uvm_map() successful calls", 0); 340 UVMCNT_INIT(map_backmerge, UVMCNT_CNT, 0, "# uvm_map() back merges", 0); 341 UVMCNT_INIT(map_forwmerge, UVMCNT_CNT, 0, "# uvm_map() missed forward", 342 0); 343 UVMCNT_INIT(uvm_mlk_call, UVMCNT_CNT, 0, "# map lookup calls", 0); 344 UVMCNT_INIT(uvm_mlk_hint, UVMCNT_CNT, 0, "# map lookup hint hits", 0); 345 346 /* 347 * now set up static pool of kernel map entrys ... 348 */ 349 350 simple_lock_init(&uvm.kentry_lock); 351 uvm.kentry_free = NULL; 352 for (lcv = 0 ; lcv < MAX_KMAPENT ; lcv++) { 353 kernel_map_entry[lcv].next = uvm.kentry_free; 354 uvm.kentry_free = &kernel_map_entry[lcv]; 355 } 356 357 /* 358 * initialize the map-related pools. 359 */ 360 pool_init(&uvm_vmspace_pool, sizeof(struct vmspace), 361 0, 0, 0, "vmsppl", 0, 362 pool_page_alloc_nointr, pool_page_free_nointr, M_VMMAP); 363 pool_init(&uvm_map_entry_pool, sizeof(struct vm_map_entry), 364 0, 0, 0, "vmmpepl", 0, 365 pool_page_alloc_nointr, pool_page_free_nointr, M_VMMAP); 366 } 367 368 /* 369 * clippers 370 */ 371 372 /* 373 * uvm_map_clip_start: ensure that the entry begins at or after 374 * the starting address, if it doesn't we split the entry. 375 * 376 * => caller should use UVM_MAP_CLIP_START macro rather than calling 377 * this directly 378 * => map must be locked by caller 379 */ 380 381 void uvm_map_clip_start(map, entry, start) 382 vm_map_t map; 383 vm_map_entry_t entry; 384 vaddr_t start; 385 { 386 vm_map_entry_t new_entry; 387 vaddr_t new_adj; 388 389 /* uvm_map_simplify_entry(map, entry); */ /* XXX */ 390 391 /* 392 * Split off the front portion. note that we must insert the new 393 * entry BEFORE this one, so that this entry has the specified 394 * starting address. 395 */ 396 397 new_entry = uvm_mapent_alloc(map); 398 uvm_mapent_copy(entry, new_entry); /* entry -> new_entry */ 399 400 new_entry->end = start; 401 new_adj = start - new_entry->start; 402 if (entry->object.uvm_obj) 403 entry->offset += new_adj; /* shift start over */ 404 entry->start = start; 405 406 if (new_entry->aref.ar_amap) { 407 amap_splitref(&new_entry->aref, &entry->aref, new_adj); 408 } 409 410 uvm_map_entry_link(map, entry->prev, new_entry); 411 412 if (UVM_ET_ISSUBMAP(entry)) { 413 /* ... unlikely to happen, but play it safe */ 414 uvm_map_reference(new_entry->object.sub_map); 415 } else { 416 if (UVM_ET_ISOBJ(entry) && 417 entry->object.uvm_obj->pgops && 418 entry->object.uvm_obj->pgops->pgo_reference) 419 entry->object.uvm_obj->pgops->pgo_reference( 420 entry->object.uvm_obj); 421 } 422 } 423 424 /* 425 * uvm_map_clip_end: ensure that the entry ends at or before 426 * the ending address, if it does't we split the reference 427 * 428 * => caller should use UVM_MAP_CLIP_END macro rather than calling 429 * this directly 430 * => map must be locked by caller 431 */ 432 433 void 434 uvm_map_clip_end(map, entry, end) 435 vm_map_t map; 436 vm_map_entry_t entry; 437 vaddr_t end; 438 { 439 vm_map_entry_t new_entry; 440 vaddr_t new_adj; /* #bytes we move start forward */ 441 442 /* 443 * Create a new entry and insert it 444 * AFTER the specified entry 445 */ 446 447 new_entry = uvm_mapent_alloc(map); 448 uvm_mapent_copy(entry, new_entry); /* entry -> new_entry */ 449 450 new_entry->start = entry->end = end; 451 new_adj = end - entry->start; 452 if (new_entry->object.uvm_obj) 453 new_entry->offset += new_adj; 454 455 if (entry->aref.ar_amap) 456 amap_splitref(&entry->aref, &new_entry->aref, new_adj); 457 458 uvm_map_entry_link(map, entry, new_entry); 459 460 if (UVM_ET_ISSUBMAP(entry)) { 461 /* ... unlikely to happen, but play it safe */ 462 uvm_map_reference(new_entry->object.sub_map); 463 } else { 464 if (UVM_ET_ISOBJ(entry) && 465 entry->object.uvm_obj->pgops && 466 entry->object.uvm_obj->pgops->pgo_reference) 467 entry->object.uvm_obj->pgops->pgo_reference( 468 entry->object.uvm_obj); 469 } 470 } 471 472 473 /* 474 * M A P - m a i n e n t r y p o i n t 475 */ 476 /* 477 * uvm_map: establish a valid mapping in a map 478 * 479 * => assume startp is page aligned. 480 * => assume size is a multiple of PAGE_SIZE. 481 * => assume sys_mmap provides enough of a "hint" to have us skip 482 * over text/data/bss area. 483 * => map must be unlocked (we will lock it) 484 * => <uobj,uoffset> value meanings (4 cases): 485 * [1] <NULL,uoffset> == uoffset is a hint for PMAP_PREFER 486 * [2] <NULL,UVM_UNKNOWN_OFFSET> == don't PMAP_PREFER 487 * [3] <uobj,uoffset> == normal mapping 488 * [4] <uobj,UVM_UNKNOWN_OFFSET> == uvm_map finds offset based on VA 489 * 490 * case [4] is for kernel mappings where we don't know the offset until 491 * we've found a virtual address. note that kernel object offsets are 492 * always relative to vm_map_min(kernel_map). 493 * 494 * => if `align' is non-zero, we try to align the virtual address to 495 * the specified alignment. this is only a hint; if we can't 496 * do it, the address will be unaligned. this is provided as 497 * a mechanism for large pages. 498 * 499 * => XXXCDC: need way to map in external amap? 500 */ 501 502 int 503 uvm_map(map, startp, size, uobj, uoffset, align, flags) 504 vm_map_t map; 505 vaddr_t *startp; /* IN/OUT */ 506 vsize_t size; 507 struct uvm_object *uobj; 508 voff_t uoffset; 509 vsize_t align; 510 uvm_flag_t flags; 511 { 512 vm_map_entry_t prev_entry, new_entry; 513 vm_prot_t prot = UVM_PROTECTION(flags), maxprot = 514 UVM_MAXPROTECTION(flags); 515 vm_inherit_t inherit = UVM_INHERIT(flags); 516 int advice = UVM_ADVICE(flags); 517 UVMHIST_FUNC("uvm_map"); 518 UVMHIST_CALLED(maphist); 519 520 UVMHIST_LOG(maphist, "(map=0x%x, *startp=0x%x, size=%d, flags=0x%x)", 521 map, *startp, size, flags); 522 UVMHIST_LOG(maphist, " uobj/offset 0x%x/%d", uobj, uoffset,0,0); 523 524 /* 525 * step 0: sanity check of protection code 526 */ 527 528 if ((prot & maxprot) != prot) { 529 UVMHIST_LOG(maphist, "<- prot. failure: prot=0x%x, max=0x%x", 530 prot, maxprot,0,0); 531 return(KERN_PROTECTION_FAILURE); 532 } 533 534 /* 535 * step 1: figure out where to put new VM range 536 */ 537 538 if (vm_map_lock_try(map) == FALSE) { 539 if (flags & UVM_FLAG_TRYLOCK) 540 return(KERN_FAILURE); 541 vm_map_lock(map); /* could sleep here */ 542 } 543 if ((prev_entry = uvm_map_findspace(map, *startp, size, startp, 544 uobj, uoffset, align, flags)) == NULL) { 545 UVMHIST_LOG(maphist,"<- uvm_map_findspace failed!",0,0,0,0); 546 vm_map_unlock(map); 547 return (KERN_NO_SPACE); 548 } 549 550 #ifdef PMAP_GROWKERNEL 551 { 552 /* 553 * If the kernel pmap can't map the requested space, 554 * then allocate more resources for it. 555 */ 556 if (map == kernel_map && uvm_maxkaddr < (*startp + size)) 557 uvm_maxkaddr = pmap_growkernel(*startp + size); 558 } 559 #endif 560 561 UVMCNT_INCR(uvm_map_call); 562 563 /* 564 * if uobj is null, then uoffset is either a VAC hint for PMAP_PREFER 565 * [typically from uvm_map_reserve] or it is UVM_UNKNOWN_OFFSET. in 566 * either case we want to zero it before storing it in the map entry 567 * (because it looks strange and confusing when debugging...) 568 * 569 * if uobj is not null 570 * if uoffset is not UVM_UNKNOWN_OFFSET then we have a normal mapping 571 * and we do not need to change uoffset. 572 * if uoffset is UVM_UNKNOWN_OFFSET then we need to find the offset 573 * now (based on the starting address of the map). this case is 574 * for kernel object mappings where we don't know the offset until 575 * the virtual address is found (with uvm_map_findspace). the 576 * offset is the distance we are from the start of the map. 577 */ 578 579 if (uobj == NULL) { 580 uoffset = 0; 581 } else { 582 if (uoffset == UVM_UNKNOWN_OFFSET) { 583 KASSERT(UVM_OBJ_IS_KERN_OBJECT(uobj)); 584 uoffset = *startp - vm_map_min(kernel_map); 585 } 586 } 587 588 /* 589 * step 2: try and insert in map by extending previous entry, if 590 * possible 591 * XXX: we don't try and pull back the next entry. might be useful 592 * for a stack, but we are currently allocating our stack in advance. 593 */ 594 595 if ((flags & UVM_FLAG_NOMERGE) == 0 && 596 prev_entry->end == *startp && prev_entry != &map->header && 597 prev_entry->object.uvm_obj == uobj) { 598 599 if (uobj && prev_entry->offset + 600 (prev_entry->end - prev_entry->start) != uoffset) 601 goto step3; 602 603 if (UVM_ET_ISSUBMAP(prev_entry)) 604 goto step3; 605 606 if (prev_entry->protection != prot || 607 prev_entry->max_protection != maxprot) 608 goto step3; 609 610 if (prev_entry->inheritance != inherit || 611 prev_entry->advice != advice) 612 goto step3; 613 614 /* wiring status must match (new area is unwired) */ 615 if (VM_MAPENT_ISWIRED(prev_entry)) 616 goto step3; 617 618 /* 619 * can't extend a shared amap. note: no need to lock amap to 620 * look at refs since we don't care about its exact value. 621 * if it is one (i.e. we have only reference) it will stay there 622 */ 623 624 if (prev_entry->aref.ar_amap && 625 amap_refs(prev_entry->aref.ar_amap) != 1) { 626 goto step3; 627 } 628 629 /* got it! */ 630 631 UVMCNT_INCR(map_backmerge); 632 UVMHIST_LOG(maphist," starting back merge", 0, 0, 0, 0); 633 634 /* 635 * drop our reference to uobj since we are extending a reference 636 * that we already have (the ref count can not drop to zero). 637 */ 638 if (uobj && uobj->pgops->pgo_detach) 639 uobj->pgops->pgo_detach(uobj); 640 641 if (prev_entry->aref.ar_amap) { 642 amap_extend(prev_entry, size); 643 } 644 645 prev_entry->end += size; 646 map->size += size; 647 648 UVMHIST_LOG(maphist,"<- done (via backmerge)!", 0, 0, 0, 0); 649 vm_map_unlock(map); 650 return (KERN_SUCCESS); 651 652 } 653 step3: 654 UVMHIST_LOG(maphist," allocating new map entry", 0, 0, 0, 0); 655 656 /* 657 * check for possible forward merge (which we don't do) and count 658 * the number of times we missed a *possible* chance to merge more 659 */ 660 661 if ((flags & UVM_FLAG_NOMERGE) == 0 && 662 prev_entry->next != &map->header && 663 prev_entry->next->start == (*startp + size)) 664 UVMCNT_INCR(map_forwmerge); 665 666 /* 667 * step 3: allocate new entry and link it in 668 */ 669 670 new_entry = uvm_mapent_alloc(map); 671 new_entry->start = *startp; 672 new_entry->end = new_entry->start + size; 673 new_entry->object.uvm_obj = uobj; 674 new_entry->offset = uoffset; 675 676 if (uobj) 677 new_entry->etype = UVM_ET_OBJ; 678 else 679 new_entry->etype = 0; 680 681 if (flags & UVM_FLAG_COPYONW) { 682 new_entry->etype |= UVM_ET_COPYONWRITE; 683 if ((flags & UVM_FLAG_OVERLAY) == 0) 684 new_entry->etype |= UVM_ET_NEEDSCOPY; 685 } 686 687 new_entry->protection = prot; 688 new_entry->max_protection = maxprot; 689 new_entry->inheritance = inherit; 690 new_entry->wired_count = 0; 691 new_entry->advice = advice; 692 if (flags & UVM_FLAG_OVERLAY) { 693 /* 694 * to_add: for BSS we overallocate a little since we 695 * are likely to extend 696 */ 697 vaddr_t to_add = (flags & UVM_FLAG_AMAPPAD) ? 698 UVM_AMAP_CHUNK << PAGE_SHIFT : 0; 699 struct vm_amap *amap = amap_alloc(size, to_add, M_WAITOK); 700 new_entry->aref.ar_pageoff = 0; 701 new_entry->aref.ar_amap = amap; 702 } else { 703 new_entry->aref.ar_pageoff = 0; 704 new_entry->aref.ar_amap = NULL; 705 } 706 707 uvm_map_entry_link(map, prev_entry, new_entry); 708 709 map->size += size; 710 711 /* 712 * Update the free space hint 713 */ 714 715 if ((map->first_free == prev_entry) && 716 (prev_entry->end >= new_entry->start)) 717 map->first_free = new_entry; 718 719 UVMHIST_LOG(maphist,"<- done!", 0, 0, 0, 0); 720 vm_map_unlock(map); 721 return(KERN_SUCCESS); 722 } 723 724 /* 725 * uvm_map_lookup_entry: find map entry at or before an address 726 * 727 * => map must at least be read-locked by caller 728 * => entry is returned in "entry" 729 * => return value is true if address is in the returned entry 730 */ 731 732 boolean_t 733 uvm_map_lookup_entry(map, address, entry) 734 vm_map_t map; 735 vaddr_t address; 736 vm_map_entry_t *entry; /* OUT */ 737 { 738 vm_map_entry_t cur; 739 vm_map_entry_t last; 740 UVMHIST_FUNC("uvm_map_lookup_entry"); 741 UVMHIST_CALLED(maphist); 742 743 UVMHIST_LOG(maphist,"(map=0x%x,addr=0x%x,ent=0x%x)", 744 map, address, entry, 0); 745 746 /* 747 * start looking either from the head of the 748 * list, or from the hint. 749 */ 750 751 simple_lock(&map->hint_lock); 752 cur = map->hint; 753 simple_unlock(&map->hint_lock); 754 755 if (cur == &map->header) 756 cur = cur->next; 757 758 UVMCNT_INCR(uvm_mlk_call); 759 if (address >= cur->start) { 760 /* 761 * go from hint to end of list. 762 * 763 * but first, make a quick check to see if 764 * we are already looking at the entry we 765 * want (which is usually the case). 766 * note also that we don't need to save the hint 767 * here... it is the same hint (unless we are 768 * at the header, in which case the hint didn't 769 * buy us anything anyway). 770 */ 771 last = &map->header; 772 if ((cur != last) && (cur->end > address)) { 773 UVMCNT_INCR(uvm_mlk_hint); 774 *entry = cur; 775 UVMHIST_LOG(maphist,"<- got it via hint (0x%x)", 776 cur, 0, 0, 0); 777 return (TRUE); 778 } 779 } else { 780 /* 781 * go from start to hint, *inclusively* 782 */ 783 last = cur->next; 784 cur = map->header.next; 785 } 786 787 /* 788 * search linearly 789 */ 790 791 while (cur != last) { 792 if (cur->end > address) { 793 if (address >= cur->start) { 794 /* 795 * save this lookup for future 796 * hints, and return 797 */ 798 799 *entry = cur; 800 SAVE_HINT(map, map->hint, cur); 801 UVMHIST_LOG(maphist,"<- search got it (0x%x)", 802 cur, 0, 0, 0); 803 return (TRUE); 804 } 805 break; 806 } 807 cur = cur->next; 808 } 809 *entry = cur->prev; 810 SAVE_HINT(map, map->hint, *entry); 811 UVMHIST_LOG(maphist,"<- failed!",0,0,0,0); 812 return (FALSE); 813 } 814 815 /* 816 * uvm_map_findspace: find "length" sized space in "map". 817 * 818 * => "hint" is a hint about where we want it, unless FINDSPACE_FIXED is 819 * set (in which case we insist on using "hint"). 820 * => "result" is VA returned 821 * => uobj/uoffset are to be used to handle VAC alignment, if required 822 * => if `align' is non-zero, we attempt to align to that value. 823 * => caller must at least have read-locked map 824 * => returns NULL on failure, or pointer to prev. map entry if success 825 * => note this is a cross between the old vm_map_findspace and vm_map_find 826 */ 827 828 vm_map_entry_t 829 uvm_map_findspace(map, hint, length, result, uobj, uoffset, align, flags) 830 vm_map_t map; 831 vaddr_t hint; 832 vsize_t length; 833 vaddr_t *result; /* OUT */ 834 struct uvm_object *uobj; 835 voff_t uoffset; 836 vsize_t align; 837 int flags; 838 { 839 vm_map_entry_t entry, next, tmp; 840 vaddr_t end, orig_hint; 841 UVMHIST_FUNC("uvm_map_findspace"); 842 UVMHIST_CALLED(maphist); 843 844 UVMHIST_LOG(maphist, "(map=0x%x, hint=0x%x, len=%d, flags=0x%x)", 845 map, hint, length, flags); 846 KASSERT((align & (align - 1)) == 0); 847 KASSERT((flags & UVM_FLAG_FIXED) == 0 || align == 0); 848 849 /* 850 * remember the original hint. if we are aligning, then we 851 * may have to try again with no alignment constraint if 852 * we fail the first time. 853 */ 854 855 orig_hint = hint; 856 if (hint < map->min_offset) { /* check ranges ... */ 857 if (flags & UVM_FLAG_FIXED) { 858 UVMHIST_LOG(maphist,"<- VA below map range",0,0,0,0); 859 return(NULL); 860 } 861 hint = map->min_offset; 862 } 863 if (hint > map->max_offset) { 864 UVMHIST_LOG(maphist,"<- VA 0x%x > range [0x%x->0x%x]", 865 hint, map->min_offset, map->max_offset, 0); 866 return(NULL); 867 } 868 869 /* 870 * Look for the first possible address; if there's already 871 * something at this address, we have to start after it. 872 */ 873 874 if ((flags & UVM_FLAG_FIXED) == 0 && hint == map->min_offset) { 875 if ((entry = map->first_free) != &map->header) 876 hint = entry->end; 877 } else { 878 if (uvm_map_lookup_entry(map, hint, &tmp)) { 879 /* "hint" address already in use ... */ 880 if (flags & UVM_FLAG_FIXED) { 881 UVMHIST_LOG(maphist,"<- fixed & VA in use", 882 0, 0, 0, 0); 883 return(NULL); 884 } 885 hint = tmp->end; 886 } 887 entry = tmp; 888 } 889 890 /* 891 * Look through the rest of the map, trying to fit a new region in 892 * the gap between existing regions, or after the very last region. 893 * note: entry->end = base VA of current gap, 894 * next->start = VA of end of current gap 895 */ 896 for (;; hint = (entry = next)->end) { 897 /* 898 * Find the end of the proposed new region. Be sure we didn't 899 * go beyond the end of the map, or wrap around the address; 900 * if so, we lose. Otherwise, if this is the last entry, or 901 * if the proposed new region fits before the next entry, we 902 * win. 903 */ 904 905 #ifdef PMAP_PREFER 906 /* 907 * push hint forward as needed to avoid VAC alias problems. 908 * we only do this if a valid offset is specified. 909 */ 910 if ((flags & UVM_FLAG_FIXED) == 0 && 911 uoffset != UVM_UNKNOWN_OFFSET) 912 PMAP_PREFER(uoffset, &hint); 913 #endif 914 if (align != 0) { 915 if ((hint & (align - 1)) != 0) 916 hint = roundup(hint, align); 917 /* 918 * XXX Should we PMAP_PREFER() here again? 919 */ 920 } 921 end = hint + length; 922 if (end > map->max_offset || end < hint) { 923 UVMHIST_LOG(maphist,"<- failed (off end)", 0,0,0,0); 924 if (align != 0) { 925 UVMHIST_LOG(maphist, 926 "calling recursively, no align", 927 0,0,0,0); 928 return (uvm_map_findspace(map, orig_hint, 929 length, result, uobj, uoffset, 0, flags)); 930 } 931 return (NULL); 932 } 933 next = entry->next; 934 if (next == &map->header || next->start >= end) 935 break; 936 if (flags & UVM_FLAG_FIXED) { 937 UVMHIST_LOG(maphist,"<- fixed mapping failed", 0,0,0,0); 938 return(NULL); /* only one shot at it ... */ 939 } 940 } 941 SAVE_HINT(map, map->hint, entry); 942 *result = hint; 943 UVMHIST_LOG(maphist,"<- got it! (result=0x%x)", hint, 0,0,0); 944 return (entry); 945 } 946 947 /* 948 * U N M A P - m a i n h e l p e r f u n c t i o n s 949 */ 950 951 /* 952 * uvm_unmap_remove: remove mappings from a vm_map (from "start" up to "stop") 953 * 954 * => caller must check alignment and size 955 * => map must be locked by caller 956 * => we return a list of map entries that we've remove from the map 957 * in "entry_list" 958 */ 959 960 int 961 uvm_unmap_remove(map, start, end, entry_list) 962 vm_map_t map; 963 vaddr_t start,end; 964 vm_map_entry_t *entry_list; /* OUT */ 965 { 966 vm_map_entry_t entry, first_entry, next; 967 vaddr_t len; 968 UVMHIST_FUNC("uvm_unmap_remove"); 969 UVMHIST_CALLED(maphist); 970 971 UVMHIST_LOG(maphist,"(map=0x%x, start=0x%x, end=0x%x)", 972 map, start, end, 0); 973 974 VM_MAP_RANGE_CHECK(map, start, end); 975 976 /* 977 * find first entry 978 */ 979 if (uvm_map_lookup_entry(map, start, &first_entry) == TRUE) { 980 /* clip and go... */ 981 entry = first_entry; 982 UVM_MAP_CLIP_START(map, entry, start); 983 /* critical! prevents stale hint */ 984 SAVE_HINT(map, entry, entry->prev); 985 986 } else { 987 entry = first_entry->next; 988 } 989 990 /* 991 * Save the free space hint 992 */ 993 994 if (map->first_free->start >= start) 995 map->first_free = entry->prev; 996 997 /* 998 * note: we now re-use first_entry for a different task. we remove 999 * a number of map entries from the map and save them in a linked 1000 * list headed by "first_entry". once we remove them from the map 1001 * the caller should unlock the map and drop the references to the 1002 * backing objects [c.f. uvm_unmap_detach]. the object is to 1003 * seperate unmapping from reference dropping. why? 1004 * [1] the map has to be locked for unmapping 1005 * [2] the map need not be locked for reference dropping 1006 * [3] dropping references may trigger pager I/O, and if we hit 1007 * a pager that does synchronous I/O we may have to wait for it. 1008 * [4] we would like all waiting for I/O to occur with maps unlocked 1009 * so that we don't block other threads. 1010 */ 1011 first_entry = NULL; 1012 *entry_list = NULL; /* to be safe */ 1013 1014 /* 1015 * break up the area into map entry sized regions and unmap. note 1016 * that all mappings have to be removed before we can even consider 1017 * dropping references to amaps or VM objects (otherwise we could end 1018 * up with a mapping to a page on the free list which would be very bad) 1019 */ 1020 1021 while ((entry != &map->header) && (entry->start < end)) { 1022 1023 UVM_MAP_CLIP_END(map, entry, end); 1024 next = entry->next; 1025 len = entry->end - entry->start; 1026 1027 /* 1028 * unwire before removing addresses from the pmap; otherwise 1029 * unwiring will put the entries back into the pmap (XXX). 1030 */ 1031 1032 if (VM_MAPENT_ISWIRED(entry)) 1033 uvm_map_entry_unwire(map, entry); 1034 1035 /* 1036 * special case: handle mappings to anonymous kernel objects. 1037 * we want to free these pages right away... 1038 */ 1039 if (UVM_ET_ISOBJ(entry) && 1040 UVM_OBJ_IS_KERN_OBJECT(entry->object.uvm_obj)) { 1041 KASSERT(vm_map_pmap(map) == pmap_kernel()); 1042 1043 /* 1044 * note: kernel object mappings are currently used in 1045 * two ways: 1046 * [1] "normal" mappings of pages in the kernel object 1047 * [2] uvm_km_valloc'd allocations in which we 1048 * pmap_enter in some non-kernel-object page 1049 * (e.g. vmapbuf). 1050 * 1051 * for case [1], we need to remove the mapping from 1052 * the pmap and then remove the page from the kernel 1053 * object (because, once pages in a kernel object are 1054 * unmapped they are no longer needed, unlike, say, 1055 * a vnode where you might want the data to persist 1056 * until flushed out of a queue). 1057 * 1058 * for case [2], we need to remove the mapping from 1059 * the pmap. there shouldn't be any pages at the 1060 * specified offset in the kernel object [but it 1061 * doesn't hurt to call uvm_km_pgremove just to be 1062 * safe?] 1063 * 1064 * uvm_km_pgremove currently does the following: 1065 * for pages in the kernel object in range: 1066 * - drops the swap slot 1067 * - uvm_pagefree the page 1068 * 1069 * note there is version of uvm_km_pgremove() that 1070 * is used for "intrsafe" objects. 1071 */ 1072 1073 /* 1074 * remove mappings from pmap and drop the pages 1075 * from the object. offsets are always relative 1076 * to vm_map_min(kernel_map). 1077 */ 1078 if (UVM_OBJ_IS_INTRSAFE_OBJECT(entry->object.uvm_obj)) { 1079 pmap_kremove(entry->start, len); 1080 uvm_km_pgremove_intrsafe(entry->object.uvm_obj, 1081 entry->start - vm_map_min(kernel_map), 1082 entry->end - vm_map_min(kernel_map)); 1083 } else { 1084 pmap_remove(pmap_kernel(), entry->start, 1085 entry->start + len); 1086 uvm_km_pgremove(entry->object.uvm_obj, 1087 entry->start - vm_map_min(kernel_map), 1088 entry->end - vm_map_min(kernel_map)); 1089 } 1090 1091 /* 1092 * null out kernel_object reference, we've just 1093 * dropped it 1094 */ 1095 entry->etype &= ~UVM_ET_OBJ; 1096 entry->object.uvm_obj = NULL; /* to be safe */ 1097 1098 } else { 1099 /* 1100 * remove mappings the standard way. 1101 */ 1102 pmap_remove(map->pmap, entry->start, entry->end); 1103 } 1104 1105 /* 1106 * remove entry from map and put it on our list of entries 1107 * that we've nuked. then go do next entry. 1108 */ 1109 UVMHIST_LOG(maphist, " removed map entry 0x%x", entry, 0, 0,0); 1110 1111 /* critical! prevents stale hint */ 1112 SAVE_HINT(map, entry, entry->prev); 1113 1114 uvm_map_entry_unlink(map, entry); 1115 map->size -= len; 1116 entry->next = first_entry; 1117 first_entry = entry; 1118 entry = next; /* next entry, please */ 1119 } 1120 1121 /* 1122 * now we've cleaned up the map and are ready for the caller to drop 1123 * references to the mapped objects. 1124 */ 1125 1126 *entry_list = first_entry; 1127 UVMHIST_LOG(maphist,"<- done!", 0, 0, 0, 0); 1128 return(KERN_SUCCESS); 1129 } 1130 1131 /* 1132 * uvm_unmap_detach: drop references in a chain of map entries 1133 * 1134 * => we will free the map entries as we traverse the list. 1135 */ 1136 1137 void 1138 uvm_unmap_detach(first_entry, flags) 1139 vm_map_entry_t first_entry; 1140 int flags; 1141 { 1142 vm_map_entry_t next_entry; 1143 UVMHIST_FUNC("uvm_unmap_detach"); UVMHIST_CALLED(maphist); 1144 1145 while (first_entry) { 1146 KASSERT(!VM_MAPENT_ISWIRED(first_entry)); 1147 UVMHIST_LOG(maphist, 1148 " detach 0x%x: amap=0x%x, obj=0x%x, submap?=%d", 1149 first_entry, first_entry->aref.ar_amap, 1150 first_entry->object.uvm_obj, 1151 UVM_ET_ISSUBMAP(first_entry)); 1152 1153 /* 1154 * drop reference to amap, if we've got one 1155 */ 1156 1157 if (first_entry->aref.ar_amap) 1158 uvm_map_unreference_amap(first_entry, flags); 1159 1160 /* 1161 * drop reference to our backing object, if we've got one 1162 */ 1163 1164 if (UVM_ET_ISSUBMAP(first_entry)) { 1165 /* ... unlikely to happen, but play it safe */ 1166 uvm_map_deallocate(first_entry->object.sub_map); 1167 } else { 1168 if (UVM_ET_ISOBJ(first_entry) && 1169 first_entry->object.uvm_obj->pgops->pgo_detach) 1170 first_entry->object.uvm_obj->pgops-> 1171 pgo_detach(first_entry->object.uvm_obj); 1172 } 1173 1174 next_entry = first_entry->next; 1175 uvm_mapent_free(first_entry); 1176 first_entry = next_entry; 1177 } 1178 UVMHIST_LOG(maphist, "<- done", 0,0,0,0); 1179 } 1180 1181 /* 1182 * E X T R A C T I O N F U N C T I O N S 1183 */ 1184 1185 /* 1186 * uvm_map_reserve: reserve space in a vm_map for future use. 1187 * 1188 * => we reserve space in a map by putting a dummy map entry in the 1189 * map (dummy means obj=NULL, amap=NULL, prot=VM_PROT_NONE) 1190 * => map should be unlocked (we will write lock it) 1191 * => we return true if we were able to reserve space 1192 * => XXXCDC: should be inline? 1193 */ 1194 1195 int 1196 uvm_map_reserve(map, size, offset, align, raddr) 1197 vm_map_t map; 1198 vsize_t size; 1199 vaddr_t offset; /* hint for pmap_prefer */ 1200 vsize_t align; /* alignment hint */ 1201 vaddr_t *raddr; /* IN:hint, OUT: reserved VA */ 1202 { 1203 UVMHIST_FUNC("uvm_map_reserve"); UVMHIST_CALLED(maphist); 1204 1205 UVMHIST_LOG(maphist, "(map=0x%x, size=0x%x, offset=0x%x,addr=0x%x)", 1206 map,size,offset,raddr); 1207 1208 size = round_page(size); 1209 if (*raddr < vm_map_min(map)) 1210 *raddr = vm_map_min(map); /* hint */ 1211 1212 /* 1213 * reserve some virtual space. 1214 */ 1215 1216 if (uvm_map(map, raddr, size, NULL, offset, 0, 1217 UVM_MAPFLAG(UVM_PROT_NONE, UVM_PROT_NONE, UVM_INH_NONE, 1218 UVM_ADV_RANDOM, UVM_FLAG_NOMERGE)) != KERN_SUCCESS) { 1219 UVMHIST_LOG(maphist, "<- done (no VM)", 0,0,0,0); 1220 return (FALSE); 1221 } 1222 1223 UVMHIST_LOG(maphist, "<- done (*raddr=0x%x)", *raddr,0,0,0); 1224 return (TRUE); 1225 } 1226 1227 /* 1228 * uvm_map_replace: replace a reserved (blank) area of memory with 1229 * real mappings. 1230 * 1231 * => caller must WRITE-LOCK the map 1232 * => we return TRUE if replacement was a success 1233 * => we expect the newents chain to have nnewents entrys on it and 1234 * we expect newents->prev to point to the last entry on the list 1235 * => note newents is allowed to be NULL 1236 */ 1237 1238 int 1239 uvm_map_replace(map, start, end, newents, nnewents) 1240 struct vm_map *map; 1241 vaddr_t start, end; 1242 vm_map_entry_t newents; 1243 int nnewents; 1244 { 1245 vm_map_entry_t oldent, last; 1246 1247 /* 1248 * first find the blank map entry at the specified address 1249 */ 1250 1251 if (!uvm_map_lookup_entry(map, start, &oldent)) { 1252 return(FALSE); 1253 } 1254 1255 /* 1256 * check to make sure we have a proper blank entry 1257 */ 1258 1259 if (oldent->start != start || oldent->end != end || 1260 oldent->object.uvm_obj != NULL || oldent->aref.ar_amap != NULL) { 1261 return (FALSE); 1262 } 1263 1264 #ifdef DIAGNOSTIC 1265 /* 1266 * sanity check the newents chain 1267 */ 1268 { 1269 vm_map_entry_t tmpent = newents; 1270 int nent = 0; 1271 vaddr_t cur = start; 1272 1273 while (tmpent) { 1274 nent++; 1275 if (tmpent->start < cur) 1276 panic("uvm_map_replace1"); 1277 if (tmpent->start > tmpent->end || tmpent->end > end) { 1278 printf("tmpent->start=0x%lx, tmpent->end=0x%lx, end=0x%lx\n", 1279 tmpent->start, tmpent->end, end); 1280 panic("uvm_map_replace2"); 1281 } 1282 cur = tmpent->end; 1283 if (tmpent->next) { 1284 if (tmpent->next->prev != tmpent) 1285 panic("uvm_map_replace3"); 1286 } else { 1287 if (newents->prev != tmpent) 1288 panic("uvm_map_replace4"); 1289 } 1290 tmpent = tmpent->next; 1291 } 1292 if (nent != nnewents) 1293 panic("uvm_map_replace5"); 1294 } 1295 #endif 1296 1297 /* 1298 * map entry is a valid blank! replace it. (this does all the 1299 * work of map entry link/unlink...). 1300 */ 1301 1302 if (newents) { 1303 1304 last = newents->prev; /* we expect this */ 1305 1306 /* critical: flush stale hints out of map */ 1307 SAVE_HINT(map, map->hint, newents); 1308 if (map->first_free == oldent) 1309 map->first_free = last; 1310 1311 last->next = oldent->next; 1312 last->next->prev = last; 1313 newents->prev = oldent->prev; 1314 newents->prev->next = newents; 1315 map->nentries = map->nentries + (nnewents - 1); 1316 1317 } else { 1318 1319 /* critical: flush stale hints out of map */ 1320 SAVE_HINT(map, map->hint, oldent->prev); 1321 if (map->first_free == oldent) 1322 map->first_free = oldent->prev; 1323 1324 /* NULL list of new entries: just remove the old one */ 1325 uvm_map_entry_unlink(map, oldent); 1326 } 1327 1328 1329 /* 1330 * now we can free the old blank entry, unlock the map and return. 1331 */ 1332 1333 uvm_mapent_free(oldent); 1334 return(TRUE); 1335 } 1336 1337 /* 1338 * uvm_map_extract: extract a mapping from a map and put it somewhere 1339 * (maybe removing the old mapping) 1340 * 1341 * => maps should be unlocked (we will write lock them) 1342 * => returns 0 on success, error code otherwise 1343 * => start must be page aligned 1344 * => len must be page sized 1345 * => flags: 1346 * UVM_EXTRACT_REMOVE: remove mappings from srcmap 1347 * UVM_EXTRACT_CONTIG: abort if unmapped area (advisory only) 1348 * UVM_EXTRACT_QREF: for a temporary extraction do quick obj refs 1349 * UVM_EXTRACT_FIXPROT: set prot to maxprot as we go 1350 * >>>NOTE: if you set REMOVE, you are not allowed to use CONTIG or QREF!<<< 1351 * >>>NOTE: QREF's must be unmapped via the QREF path, thus should only 1352 * be used from within the kernel in a kernel level map <<< 1353 */ 1354 1355 int 1356 uvm_map_extract(srcmap, start, len, dstmap, dstaddrp, flags) 1357 vm_map_t srcmap, dstmap; 1358 vaddr_t start, *dstaddrp; 1359 vsize_t len; 1360 int flags; 1361 { 1362 vaddr_t dstaddr, end, newend, oldoffset, fudge, orig_fudge, 1363 oldstart; 1364 vm_map_entry_t chain, endchain, entry, orig_entry, newentry, deadentry; 1365 vm_map_entry_t oldentry; 1366 vsize_t elen; 1367 int nchain, error, copy_ok; 1368 UVMHIST_FUNC("uvm_map_extract"); UVMHIST_CALLED(maphist); 1369 1370 UVMHIST_LOG(maphist,"(srcmap=0x%x,start=0x%x, len=0x%x", srcmap, start, 1371 len,0); 1372 UVMHIST_LOG(maphist," ...,dstmap=0x%x, flags=0x%x)", dstmap,flags,0,0); 1373 1374 /* 1375 * step 0: sanity check: start must be on a page boundary, length 1376 * must be page sized. can't ask for CONTIG/QREF if you asked for 1377 * REMOVE. 1378 */ 1379 1380 KASSERT((start & PAGE_MASK) == 0 && (len & PAGE_MASK) == 0); 1381 KASSERT((flags & UVM_EXTRACT_REMOVE) == 0 || 1382 (flags & (UVM_EXTRACT_CONTIG|UVM_EXTRACT_QREF)) == 0); 1383 1384 /* 1385 * step 1: reserve space in the target map for the extracted area 1386 */ 1387 1388 dstaddr = vm_map_min(dstmap); 1389 if (uvm_map_reserve(dstmap, len, start, 0, &dstaddr) == FALSE) 1390 return(ENOMEM); 1391 *dstaddrp = dstaddr; /* pass address back to caller */ 1392 UVMHIST_LOG(maphist, " dstaddr=0x%x", dstaddr,0,0,0); 1393 1394 /* 1395 * step 2: setup for the extraction process loop by init'ing the 1396 * map entry chain, locking src map, and looking up the first useful 1397 * entry in the map. 1398 */ 1399 1400 end = start + len; 1401 newend = dstaddr + len; 1402 chain = endchain = NULL; 1403 nchain = 0; 1404 vm_map_lock(srcmap); 1405 1406 if (uvm_map_lookup_entry(srcmap, start, &entry)) { 1407 1408 /* "start" is within an entry */ 1409 if (flags & UVM_EXTRACT_QREF) { 1410 1411 /* 1412 * for quick references we don't clip the entry, so 1413 * the entry may map space "before" the starting 1414 * virtual address... this is the "fudge" factor 1415 * (which can be non-zero only the first time 1416 * through the "while" loop in step 3). 1417 */ 1418 1419 fudge = start - entry->start; 1420 } else { 1421 1422 /* 1423 * normal reference: we clip the map to fit (thus 1424 * fudge is zero) 1425 */ 1426 1427 UVM_MAP_CLIP_START(srcmap, entry, start); 1428 SAVE_HINT(srcmap, srcmap->hint, entry->prev); 1429 fudge = 0; 1430 } 1431 } else { 1432 1433 /* "start" is not within an entry ... skip to next entry */ 1434 if (flags & UVM_EXTRACT_CONTIG) { 1435 error = EINVAL; 1436 goto bad; /* definite hole here ... */ 1437 } 1438 1439 entry = entry->next; 1440 fudge = 0; 1441 } 1442 1443 /* save values from srcmap for step 6 */ 1444 orig_entry = entry; 1445 orig_fudge = fudge; 1446 1447 /* 1448 * step 3: now start looping through the map entries, extracting 1449 * as we go. 1450 */ 1451 1452 while (entry->start < end && entry != &srcmap->header) { 1453 1454 /* if we are not doing a quick reference, clip it */ 1455 if ((flags & UVM_EXTRACT_QREF) == 0) 1456 UVM_MAP_CLIP_END(srcmap, entry, end); 1457 1458 /* clear needs_copy (allow chunking) */ 1459 if (UVM_ET_ISNEEDSCOPY(entry)) { 1460 if (fudge) 1461 oldstart = entry->start; 1462 else 1463 oldstart = 0; /* XXX: gcc */ 1464 amap_copy(srcmap, entry, M_NOWAIT, TRUE, start, end); 1465 if (UVM_ET_ISNEEDSCOPY(entry)) { /* failed? */ 1466 error = ENOMEM; 1467 goto bad; 1468 } 1469 1470 /* amap_copy could clip (during chunk)! update fudge */ 1471 if (fudge) { 1472 fudge = fudge - (entry->start - oldstart); 1473 orig_fudge = fudge; 1474 } 1475 } 1476 1477 /* calculate the offset of this from "start" */ 1478 oldoffset = (entry->start + fudge) - start; 1479 1480 /* allocate a new map entry */ 1481 newentry = uvm_mapent_alloc(dstmap); 1482 if (newentry == NULL) { 1483 error = ENOMEM; 1484 goto bad; 1485 } 1486 1487 /* set up new map entry */ 1488 newentry->next = NULL; 1489 newentry->prev = endchain; 1490 newentry->start = dstaddr + oldoffset; 1491 newentry->end = 1492 newentry->start + (entry->end - (entry->start + fudge)); 1493 if (newentry->end > newend || newentry->end < newentry->start) 1494 newentry->end = newend; 1495 newentry->object.uvm_obj = entry->object.uvm_obj; 1496 if (newentry->object.uvm_obj) { 1497 if (newentry->object.uvm_obj->pgops->pgo_reference) 1498 newentry->object.uvm_obj->pgops-> 1499 pgo_reference(newentry->object.uvm_obj); 1500 newentry->offset = entry->offset + fudge; 1501 } else { 1502 newentry->offset = 0; 1503 } 1504 newentry->etype = entry->etype; 1505 newentry->protection = (flags & UVM_EXTRACT_FIXPROT) ? 1506 entry->max_protection : entry->protection; 1507 newentry->max_protection = entry->max_protection; 1508 newentry->inheritance = entry->inheritance; 1509 newentry->wired_count = 0; 1510 newentry->aref.ar_amap = entry->aref.ar_amap; 1511 if (newentry->aref.ar_amap) { 1512 newentry->aref.ar_pageoff = 1513 entry->aref.ar_pageoff + (fudge >> PAGE_SHIFT); 1514 uvm_map_reference_amap(newentry, AMAP_SHARED | 1515 ((flags & UVM_EXTRACT_QREF) ? AMAP_REFALL : 0)); 1516 } else { 1517 newentry->aref.ar_pageoff = 0; 1518 } 1519 newentry->advice = entry->advice; 1520 1521 /* now link it on the chain */ 1522 nchain++; 1523 if (endchain == NULL) { 1524 chain = endchain = newentry; 1525 } else { 1526 endchain->next = newentry; 1527 endchain = newentry; 1528 } 1529 1530 /* end of 'while' loop! */ 1531 if ((flags & UVM_EXTRACT_CONTIG) && entry->end < end && 1532 (entry->next == &srcmap->header || 1533 entry->next->start != entry->end)) { 1534 error = EINVAL; 1535 goto bad; 1536 } 1537 entry = entry->next; 1538 fudge = 0; 1539 } 1540 1541 /* 1542 * step 4: close off chain (in format expected by uvm_map_replace) 1543 */ 1544 1545 if (chain) 1546 chain->prev = endchain; 1547 1548 /* 1549 * step 5: attempt to lock the dest map so we can pmap_copy. 1550 * note usage of copy_ok: 1551 * 1 => dstmap locked, pmap_copy ok, and we "replace" here (step 5) 1552 * 0 => dstmap unlocked, NO pmap_copy, and we will "replace" in step 7 1553 */ 1554 1555 if (srcmap == dstmap || vm_map_lock_try(dstmap) == TRUE) { 1556 copy_ok = 1; 1557 if (!uvm_map_replace(dstmap, dstaddr, dstaddr+len, chain, 1558 nchain)) { 1559 if (srcmap != dstmap) 1560 vm_map_unlock(dstmap); 1561 error = EIO; 1562 goto bad; 1563 } 1564 } else { 1565 copy_ok = 0; 1566 /* replace defered until step 7 */ 1567 } 1568 1569 /* 1570 * step 6: traverse the srcmap a second time to do the following: 1571 * - if we got a lock on the dstmap do pmap_copy 1572 * - if UVM_EXTRACT_REMOVE remove the entries 1573 * we make use of orig_entry and orig_fudge (saved in step 2) 1574 */ 1575 1576 if (copy_ok || (flags & UVM_EXTRACT_REMOVE)) { 1577 1578 /* purge possible stale hints from srcmap */ 1579 if (flags & UVM_EXTRACT_REMOVE) { 1580 SAVE_HINT(srcmap, srcmap->hint, orig_entry->prev); 1581 if (srcmap->first_free->start >= start) 1582 srcmap->first_free = orig_entry->prev; 1583 } 1584 1585 entry = orig_entry; 1586 fudge = orig_fudge; 1587 deadentry = NULL; /* for UVM_EXTRACT_REMOVE */ 1588 1589 while (entry->start < end && entry != &srcmap->header) { 1590 if (copy_ok) { 1591 oldoffset = (entry->start + fudge) - start; 1592 elen = min(end, entry->end) - 1593 (entry->start + fudge); 1594 pmap_copy(dstmap->pmap, srcmap->pmap, 1595 dstaddr + oldoffset, elen, 1596 entry->start + fudge); 1597 } 1598 1599 /* we advance "entry" in the following if statement */ 1600 if (flags & UVM_EXTRACT_REMOVE) { 1601 pmap_remove(srcmap->pmap, entry->start, 1602 entry->end); 1603 oldentry = entry; /* save entry */ 1604 entry = entry->next; /* advance */ 1605 uvm_map_entry_unlink(srcmap, oldentry); 1606 /* add to dead list */ 1607 oldentry->next = deadentry; 1608 deadentry = oldentry; 1609 } else { 1610 entry = entry->next; /* advance */ 1611 } 1612 1613 /* end of 'while' loop */ 1614 fudge = 0; 1615 } 1616 1617 /* 1618 * unlock dstmap. we will dispose of deadentry in 1619 * step 7 if needed 1620 */ 1621 1622 if (copy_ok && srcmap != dstmap) 1623 vm_map_unlock(dstmap); 1624 1625 } 1626 else 1627 deadentry = NULL; /* XXX: gcc */ 1628 1629 /* 1630 * step 7: we are done with the source map, unlock. if copy_ok 1631 * is 0 then we have not replaced the dummy mapping in dstmap yet 1632 * and we need to do so now. 1633 */ 1634 1635 vm_map_unlock(srcmap); 1636 if ((flags & UVM_EXTRACT_REMOVE) && deadentry) 1637 uvm_unmap_detach(deadentry, 0); /* dispose of old entries */ 1638 1639 /* now do the replacement if we didn't do it in step 5 */ 1640 if (copy_ok == 0) { 1641 vm_map_lock(dstmap); 1642 error = uvm_map_replace(dstmap, dstaddr, dstaddr+len, chain, 1643 nchain); 1644 vm_map_unlock(dstmap); 1645 1646 if (error == FALSE) { 1647 error = EIO; 1648 goto bad2; 1649 } 1650 } 1651 return(0); 1652 1653 /* 1654 * bad: failure recovery 1655 */ 1656 bad: 1657 vm_map_unlock(srcmap); 1658 bad2: /* src already unlocked */ 1659 if (chain) 1660 uvm_unmap_detach(chain, 1661 (flags & UVM_EXTRACT_QREF) ? AMAP_REFALL : 0); 1662 uvm_unmap(dstmap, dstaddr, dstaddr+len); /* ??? */ 1663 return(error); 1664 } 1665 1666 /* end of extraction functions */ 1667 1668 /* 1669 * uvm_map_submap: punch down part of a map into a submap 1670 * 1671 * => only the kernel_map is allowed to be submapped 1672 * => the purpose of submapping is to break up the locking granularity 1673 * of a larger map 1674 * => the range specified must have been mapped previously with a uvm_map() 1675 * call [with uobj==NULL] to create a blank map entry in the main map. 1676 * [And it had better still be blank!] 1677 * => maps which contain submaps should never be copied or forked. 1678 * => to remove a submap, use uvm_unmap() on the main map 1679 * and then uvm_map_deallocate() the submap. 1680 * => main map must be unlocked. 1681 * => submap must have been init'd and have a zero reference count. 1682 * [need not be locked as we don't actually reference it] 1683 */ 1684 1685 int 1686 uvm_map_submap(map, start, end, submap) 1687 vm_map_t map, submap; 1688 vaddr_t start, end; 1689 { 1690 vm_map_entry_t entry; 1691 int result; 1692 1693 vm_map_lock(map); 1694 VM_MAP_RANGE_CHECK(map, start, end); 1695 1696 if (uvm_map_lookup_entry(map, start, &entry)) { 1697 UVM_MAP_CLIP_START(map, entry, start); 1698 UVM_MAP_CLIP_END(map, entry, end); /* to be safe */ 1699 } 1700 else { 1701 entry = NULL; 1702 } 1703 1704 if (entry != NULL && 1705 entry->start == start && entry->end == end && 1706 entry->object.uvm_obj == NULL && entry->aref.ar_amap == NULL && 1707 !UVM_ET_ISCOPYONWRITE(entry) && !UVM_ET_ISNEEDSCOPY(entry)) { 1708 entry->etype |= UVM_ET_SUBMAP; 1709 entry->object.sub_map = submap; 1710 entry->offset = 0; 1711 uvm_map_reference(submap); 1712 result = KERN_SUCCESS; 1713 } else { 1714 result = KERN_INVALID_ARGUMENT; 1715 } 1716 vm_map_unlock(map); 1717 return(result); 1718 } 1719 1720 1721 /* 1722 * uvm_map_protect: change map protection 1723 * 1724 * => set_max means set max_protection. 1725 * => map must be unlocked. 1726 */ 1727 1728 #define MASK(entry) (UVM_ET_ISCOPYONWRITE(entry) ? \ 1729 ~VM_PROT_WRITE : VM_PROT_ALL) 1730 #define max(a,b) ((a) > (b) ? (a) : (b)) 1731 1732 int 1733 uvm_map_protect(map, start, end, new_prot, set_max) 1734 vm_map_t map; 1735 vaddr_t start, end; 1736 vm_prot_t new_prot; 1737 boolean_t set_max; 1738 { 1739 vm_map_entry_t current, entry; 1740 int rv = KERN_SUCCESS; 1741 UVMHIST_FUNC("uvm_map_protect"); UVMHIST_CALLED(maphist); 1742 UVMHIST_LOG(maphist,"(map=0x%x,start=0x%x,end=0x%x,new_prot=0x%x)", 1743 map, start, end, new_prot); 1744 1745 vm_map_lock(map); 1746 1747 VM_MAP_RANGE_CHECK(map, start, end); 1748 1749 if (uvm_map_lookup_entry(map, start, &entry)) { 1750 UVM_MAP_CLIP_START(map, entry, start); 1751 } else { 1752 entry = entry->next; 1753 } 1754 1755 /* 1756 * make a first pass to check for protection violations. 1757 */ 1758 1759 current = entry; 1760 while ((current != &map->header) && (current->start < end)) { 1761 if (UVM_ET_ISSUBMAP(current)) { 1762 rv = KERN_INVALID_ARGUMENT; 1763 goto out; 1764 } 1765 if ((new_prot & current->max_protection) != new_prot) { 1766 rv = KERN_PROTECTION_FAILURE; 1767 goto out; 1768 } 1769 current = current->next; 1770 } 1771 1772 /* go back and fix up protections (no need to clip this time). */ 1773 1774 current = entry; 1775 1776 while ((current != &map->header) && (current->start < end)) { 1777 vm_prot_t old_prot; 1778 1779 UVM_MAP_CLIP_END(map, current, end); 1780 1781 old_prot = current->protection; 1782 if (set_max) 1783 current->protection = 1784 (current->max_protection = new_prot) & old_prot; 1785 else 1786 current->protection = new_prot; 1787 1788 /* 1789 * update physical map if necessary. worry about copy-on-write 1790 * here -- CHECK THIS XXX 1791 */ 1792 1793 if (current->protection != old_prot) { 1794 /* update pmap! */ 1795 pmap_protect(map->pmap, current->start, current->end, 1796 current->protection & MASK(entry)); 1797 } 1798 1799 /* 1800 * If the map is configured to lock any future mappings, 1801 * wire this entry now if the old protection was VM_PROT_NONE 1802 * and the new protection is not VM_PROT_NONE. 1803 */ 1804 1805 if ((map->flags & VM_MAP_WIREFUTURE) != 0 && 1806 VM_MAPENT_ISWIRED(entry) == 0 && 1807 old_prot == VM_PROT_NONE && 1808 new_prot != VM_PROT_NONE) { 1809 if (uvm_map_pageable(map, entry->start, 1810 entry->end, FALSE, 1811 UVM_LK_ENTER|UVM_LK_EXIT) != KERN_SUCCESS) { 1812 /* 1813 * If locking the entry fails, remember the 1814 * error if it's the first one. Note we 1815 * still continue setting the protection in 1816 * the map, but will return the resource 1817 * shortage condition regardless. 1818 * 1819 * XXX Ignore what the actual error is, 1820 * XXX just call it a resource shortage 1821 * XXX so that it doesn't get confused 1822 * XXX what uvm_map_protect() itself would 1823 * XXX normally return. 1824 */ 1825 rv = KERN_RESOURCE_SHORTAGE; 1826 } 1827 } 1828 1829 current = current->next; 1830 } 1831 1832 out: 1833 vm_map_unlock(map); 1834 UVMHIST_LOG(maphist, "<- done, rv=%d",rv,0,0,0); 1835 return (rv); 1836 } 1837 1838 #undef max 1839 #undef MASK 1840 1841 /* 1842 * uvm_map_inherit: set inheritance code for range of addrs in map. 1843 * 1844 * => map must be unlocked 1845 * => note that the inherit code is used during a "fork". see fork 1846 * code for details. 1847 */ 1848 1849 int 1850 uvm_map_inherit(map, start, end, new_inheritance) 1851 vm_map_t map; 1852 vaddr_t start; 1853 vaddr_t end; 1854 vm_inherit_t new_inheritance; 1855 { 1856 vm_map_entry_t entry, temp_entry; 1857 UVMHIST_FUNC("uvm_map_inherit"); UVMHIST_CALLED(maphist); 1858 UVMHIST_LOG(maphist,"(map=0x%x,start=0x%x,end=0x%x,new_inh=0x%x)", 1859 map, start, end, new_inheritance); 1860 1861 switch (new_inheritance) { 1862 case MAP_INHERIT_NONE: 1863 case MAP_INHERIT_COPY: 1864 case MAP_INHERIT_SHARE: 1865 break; 1866 default: 1867 UVMHIST_LOG(maphist,"<- done (INVALID ARG)",0,0,0,0); 1868 return (KERN_INVALID_ARGUMENT); 1869 } 1870 1871 vm_map_lock(map); 1872 1873 VM_MAP_RANGE_CHECK(map, start, end); 1874 1875 if (uvm_map_lookup_entry(map, start, &temp_entry)) { 1876 entry = temp_entry; 1877 UVM_MAP_CLIP_START(map, entry, start); 1878 } else { 1879 entry = temp_entry->next; 1880 } 1881 while ((entry != &map->header) && (entry->start < end)) { 1882 UVM_MAP_CLIP_END(map, entry, end); 1883 entry->inheritance = new_inheritance; 1884 entry = entry->next; 1885 } 1886 1887 vm_map_unlock(map); 1888 UVMHIST_LOG(maphist,"<- done (OK)",0,0,0,0); 1889 return(KERN_SUCCESS); 1890 } 1891 1892 /* 1893 * uvm_map_advice: set advice code for range of addrs in map. 1894 * 1895 * => map must be unlocked 1896 */ 1897 1898 int 1899 uvm_map_advice(map, start, end, new_advice) 1900 vm_map_t map; 1901 vaddr_t start; 1902 vaddr_t end; 1903 int new_advice; 1904 { 1905 vm_map_entry_t entry, temp_entry; 1906 UVMHIST_FUNC("uvm_map_advice"); UVMHIST_CALLED(maphist); 1907 UVMHIST_LOG(maphist,"(map=0x%x,start=0x%x,end=0x%x,new_adv=0x%x)", 1908 map, start, end, new_advice); 1909 1910 vm_map_lock(map); 1911 VM_MAP_RANGE_CHECK(map, start, end); 1912 if (uvm_map_lookup_entry(map, start, &temp_entry)) { 1913 entry = temp_entry; 1914 UVM_MAP_CLIP_START(map, entry, start); 1915 } else { 1916 entry = temp_entry->next; 1917 } 1918 1919 /* 1920 * XXXJRT: disallow holes? 1921 */ 1922 1923 while ((entry != &map->header) && (entry->start < end)) { 1924 UVM_MAP_CLIP_END(map, entry, end); 1925 1926 switch (new_advice) { 1927 case MADV_NORMAL: 1928 case MADV_RANDOM: 1929 case MADV_SEQUENTIAL: 1930 /* nothing special here */ 1931 break; 1932 1933 default: 1934 vm_map_unlock(map); 1935 UVMHIST_LOG(maphist,"<- done (INVALID ARG)",0,0,0,0); 1936 return (KERN_INVALID_ARGUMENT); 1937 } 1938 entry->advice = new_advice; 1939 entry = entry->next; 1940 } 1941 1942 vm_map_unlock(map); 1943 UVMHIST_LOG(maphist,"<- done (OK)",0,0,0,0); 1944 return (KERN_SUCCESS); 1945 } 1946 1947 /* 1948 * uvm_map_pageable: sets the pageability of a range in a map. 1949 * 1950 * => wires map entries. should not be used for transient page locking. 1951 * for that, use uvm_fault_wire()/uvm_fault_unwire() (see uvm_vslock()). 1952 * => regions sepcified as not pageable require lock-down (wired) memory 1953 * and page tables. 1954 * => map must never be read-locked 1955 * => if islocked is TRUE, map is already write-locked 1956 * => we always unlock the map, since we must downgrade to a read-lock 1957 * to call uvm_fault_wire() 1958 * => XXXCDC: check this and try and clean it up. 1959 */ 1960 1961 int 1962 uvm_map_pageable(map, start, end, new_pageable, lockflags) 1963 vm_map_t map; 1964 vaddr_t start, end; 1965 boolean_t new_pageable; 1966 int lockflags; 1967 { 1968 vm_map_entry_t entry, start_entry, failed_entry; 1969 int rv; 1970 #ifdef DIAGNOSTIC 1971 u_int timestamp_save; 1972 #endif 1973 UVMHIST_FUNC("uvm_map_pageable"); UVMHIST_CALLED(maphist); 1974 UVMHIST_LOG(maphist,"(map=0x%x,start=0x%x,end=0x%x,new_pageable=0x%x)", 1975 map, start, end, new_pageable); 1976 KASSERT(map->flags & VM_MAP_PAGEABLE); 1977 1978 if ((lockflags & UVM_LK_ENTER) == 0) 1979 vm_map_lock(map); 1980 1981 VM_MAP_RANGE_CHECK(map, start, end); 1982 1983 /* 1984 * only one pageability change may take place at one time, since 1985 * uvm_fault_wire assumes it will be called only once for each 1986 * wiring/unwiring. therefore, we have to make sure we're actually 1987 * changing the pageability for the entire region. we do so before 1988 * making any changes. 1989 */ 1990 1991 if (uvm_map_lookup_entry(map, start, &start_entry) == FALSE) { 1992 if ((lockflags & UVM_LK_EXIT) == 0) 1993 vm_map_unlock(map); 1994 1995 UVMHIST_LOG(maphist,"<- done (INVALID ARG)",0,0,0,0); 1996 return (KERN_INVALID_ADDRESS); 1997 } 1998 entry = start_entry; 1999 2000 /* 2001 * handle wiring and unwiring seperately. 2002 */ 2003 2004 if (new_pageable) { /* unwire */ 2005 UVM_MAP_CLIP_START(map, entry, start); 2006 2007 /* 2008 * unwiring. first ensure that the range to be unwired is 2009 * really wired down and that there are no holes. 2010 */ 2011 2012 while ((entry != &map->header) && (entry->start < end)) { 2013 if (entry->wired_count == 0 || 2014 (entry->end < end && 2015 (entry->next == &map->header || 2016 entry->next->start > entry->end))) { 2017 if ((lockflags & UVM_LK_EXIT) == 0) 2018 vm_map_unlock(map); 2019 UVMHIST_LOG(maphist, 2020 "<- done (INVALID UNWIRE ARG)",0,0,0,0); 2021 return (KERN_INVALID_ARGUMENT); 2022 } 2023 entry = entry->next; 2024 } 2025 2026 /* 2027 * POSIX 1003.1b - a single munlock call unlocks a region, 2028 * regardless of the number of mlock calls made on that 2029 * region. 2030 */ 2031 2032 entry = start_entry; 2033 while ((entry != &map->header) && (entry->start < end)) { 2034 UVM_MAP_CLIP_END(map, entry, end); 2035 if (VM_MAPENT_ISWIRED(entry)) 2036 uvm_map_entry_unwire(map, entry); 2037 entry = entry->next; 2038 } 2039 if ((lockflags & UVM_LK_EXIT) == 0) 2040 vm_map_unlock(map); 2041 UVMHIST_LOG(maphist,"<- done (OK UNWIRE)",0,0,0,0); 2042 return(KERN_SUCCESS); 2043 } 2044 2045 /* 2046 * wire case: in two passes [XXXCDC: ugly block of code here] 2047 * 2048 * 1: holding the write lock, we create any anonymous maps that need 2049 * to be created. then we clip each map entry to the region to 2050 * be wired and increment its wiring count. 2051 * 2052 * 2: we downgrade to a read lock, and call uvm_fault_wire to fault 2053 * in the pages for any newly wired area (wired_count == 1). 2054 * 2055 * downgrading to a read lock for uvm_fault_wire avoids a possible 2056 * deadlock with another thread that may have faulted on one of 2057 * the pages to be wired (it would mark the page busy, blocking 2058 * us, then in turn block on the map lock that we hold). because 2059 * of problems in the recursive lock package, we cannot upgrade 2060 * to a write lock in vm_map_lookup. thus, any actions that 2061 * require the write lock must be done beforehand. because we 2062 * keep the read lock on the map, the copy-on-write status of the 2063 * entries we modify here cannot change. 2064 */ 2065 2066 while ((entry != &map->header) && (entry->start < end)) { 2067 if (VM_MAPENT_ISWIRED(entry) == 0) { /* not already wired? */ 2068 2069 /* 2070 * perform actions of vm_map_lookup that need the 2071 * write lock on the map: create an anonymous map 2072 * for a copy-on-write region, or an anonymous map 2073 * for a zero-fill region. (XXXCDC: submap case 2074 * ok?) 2075 */ 2076 2077 if (!UVM_ET_ISSUBMAP(entry)) { /* not submap */ 2078 if (UVM_ET_ISNEEDSCOPY(entry) && 2079 ((entry->protection & VM_PROT_WRITE) || 2080 (entry->object.uvm_obj == NULL))) { 2081 amap_copy(map, entry, M_WAITOK, TRUE, 2082 start, end); 2083 /* XXXCDC: wait OK? */ 2084 } 2085 } 2086 } 2087 UVM_MAP_CLIP_START(map, entry, start); 2088 UVM_MAP_CLIP_END(map, entry, end); 2089 entry->wired_count++; 2090 2091 /* 2092 * Check for holes 2093 */ 2094 2095 if (entry->protection == VM_PROT_NONE || 2096 (entry->end < end && 2097 (entry->next == &map->header || 2098 entry->next->start > entry->end))) { 2099 2100 /* 2101 * found one. amap creation actions do not need to 2102 * be undone, but the wired counts need to be restored. 2103 */ 2104 2105 while (entry != &map->header && entry->end > start) { 2106 entry->wired_count--; 2107 entry = entry->prev; 2108 } 2109 if ((lockflags & UVM_LK_EXIT) == 0) 2110 vm_map_unlock(map); 2111 UVMHIST_LOG(maphist,"<- done (INVALID WIRE)",0,0,0,0); 2112 return (KERN_INVALID_ARGUMENT); 2113 } 2114 entry = entry->next; 2115 } 2116 2117 /* 2118 * Pass 2. 2119 */ 2120 2121 #ifdef DIAGNOSTIC 2122 timestamp_save = map->timestamp; 2123 #endif 2124 vm_map_busy(map); 2125 vm_map_downgrade(map); 2126 2127 rv = 0; 2128 entry = start_entry; 2129 while (entry != &map->header && entry->start < end) { 2130 if (entry->wired_count == 1) { 2131 rv = uvm_fault_wire(map, entry->start, entry->end, 2132 entry->protection); 2133 if (rv) { 2134 /* 2135 * wiring failed. break out of the loop. 2136 * we'll clean up the map below, once we 2137 * have a write lock again. 2138 */ 2139 break; 2140 } 2141 } 2142 entry = entry->next; 2143 } 2144 2145 if (rv) { /* failed? */ 2146 2147 /* 2148 * Get back to an exclusive (write) lock. 2149 */ 2150 2151 vm_map_upgrade(map); 2152 vm_map_unbusy(map); 2153 2154 #ifdef DIAGNOSTIC 2155 if (timestamp_save != map->timestamp) 2156 panic("uvm_map_pageable: stale map"); 2157 #endif 2158 2159 /* 2160 * first drop the wiring count on all the entries 2161 * which haven't actually been wired yet. 2162 */ 2163 2164 failed_entry = entry; 2165 while (entry != &map->header && entry->start < end) { 2166 entry->wired_count--; 2167 entry = entry->next; 2168 } 2169 2170 /* 2171 * now, unwire all the entries that were successfully 2172 * wired above. 2173 */ 2174 2175 entry = start_entry; 2176 while (entry != failed_entry) { 2177 entry->wired_count--; 2178 if (VM_MAPENT_ISWIRED(entry) == 0) 2179 uvm_map_entry_unwire(map, entry); 2180 entry = entry->next; 2181 } 2182 if ((lockflags & UVM_LK_EXIT) == 0) 2183 vm_map_unlock(map); 2184 UVMHIST_LOG(maphist, "<- done (RV=%d)", rv,0,0,0); 2185 return(rv); 2186 } 2187 2188 /* We are holding a read lock here. */ 2189 if ((lockflags & UVM_LK_EXIT) == 0) { 2190 vm_map_unbusy(map); 2191 vm_map_unlock_read(map); 2192 } else { 2193 2194 /* 2195 * Get back to an exclusive (write) lock. 2196 */ 2197 2198 vm_map_upgrade(map); 2199 vm_map_unbusy(map); 2200 } 2201 2202 UVMHIST_LOG(maphist,"<- done (OK WIRE)",0,0,0,0); 2203 return(KERN_SUCCESS); 2204 } 2205 2206 /* 2207 * uvm_map_pageable_all: special case of uvm_map_pageable - affects 2208 * all mapped regions. 2209 * 2210 * => map must not be locked. 2211 * => if no flags are specified, all regions are unwired. 2212 * => XXXJRT: has some of the same problems as uvm_map_pageable() above. 2213 */ 2214 2215 int 2216 uvm_map_pageable_all(map, flags, limit) 2217 vm_map_t map; 2218 int flags; 2219 vsize_t limit; 2220 { 2221 vm_map_entry_t entry, failed_entry; 2222 vsize_t size; 2223 int rv; 2224 #ifdef DIAGNOSTIC 2225 u_int timestamp_save; 2226 #endif 2227 UVMHIST_FUNC("uvm_map_pageable_all"); UVMHIST_CALLED(maphist); 2228 UVMHIST_LOG(maphist,"(map=0x%x,flags=0x%x)", map, flags, 0, 0); 2229 2230 KASSERT(map->flags & VM_MAP_PAGEABLE); 2231 2232 vm_map_lock(map); 2233 2234 /* 2235 * handle wiring and unwiring separately. 2236 */ 2237 2238 if (flags == 0) { /* unwire */ 2239 /* 2240 * POSIX 1003.1b -- munlockall unlocks all regions, 2241 * regardless of how many times mlockall has been called. 2242 */ 2243 for (entry = map->header.next; entry != &map->header; 2244 entry = entry->next) { 2245 if (VM_MAPENT_ISWIRED(entry)) 2246 uvm_map_entry_unwire(map, entry); 2247 } 2248 vm_map_modflags(map, 0, VM_MAP_WIREFUTURE); 2249 vm_map_unlock(map); 2250 UVMHIST_LOG(maphist,"<- done (OK UNWIRE)",0,0,0,0); 2251 return (KERN_SUCCESS); 2252 2253 /* 2254 * end of unwire case! 2255 */ 2256 } 2257 2258 if (flags & MCL_FUTURE) { 2259 /* 2260 * must wire all future mappings; remember this. 2261 */ 2262 vm_map_modflags(map, VM_MAP_WIREFUTURE, 0); 2263 } 2264 2265 if ((flags & MCL_CURRENT) == 0) { 2266 /* 2267 * no more work to do! 2268 */ 2269 UVMHIST_LOG(maphist,"<- done (OK no wire)",0,0,0,0); 2270 vm_map_unlock(map); 2271 return (KERN_SUCCESS); 2272 } 2273 2274 /* 2275 * wire case: in three passes [XXXCDC: ugly block of code here] 2276 * 2277 * 1: holding the write lock, count all pages mapped by non-wired 2278 * entries. if this would cause us to go over our limit, we fail. 2279 * 2280 * 2: still holding the write lock, we create any anonymous maps that 2281 * need to be created. then we increment its wiring count. 2282 * 2283 * 3: we downgrade to a read lock, and call uvm_fault_wire to fault 2284 * in the pages for any newly wired area (wired_count == 1). 2285 * 2286 * downgrading to a read lock for uvm_fault_wire avoids a possible 2287 * deadlock with another thread that may have faulted on one of 2288 * the pages to be wired (it would mark the page busy, blocking 2289 * us, then in turn block on the map lock that we hold). because 2290 * of problems in the recursive lock package, we cannot upgrade 2291 * to a write lock in vm_map_lookup. thus, any actions that 2292 * require the write lock must be done beforehand. because we 2293 * keep the read lock on the map, the copy-on-write status of the 2294 * entries we modify here cannot change. 2295 */ 2296 2297 for (size = 0, entry = map->header.next; entry != &map->header; 2298 entry = entry->next) { 2299 if (entry->protection != VM_PROT_NONE && 2300 VM_MAPENT_ISWIRED(entry) == 0) { /* not already wired? */ 2301 size += entry->end - entry->start; 2302 } 2303 } 2304 2305 if (atop(size) + uvmexp.wired > uvmexp.wiredmax) { 2306 vm_map_unlock(map); 2307 return (KERN_NO_SPACE); /* XXX overloaded */ 2308 } 2309 2310 /* XXX non-pmap_wired_count case must be handled by caller */ 2311 #ifdef pmap_wired_count 2312 if (limit != 0 && 2313 (size + ptoa(pmap_wired_count(vm_map_pmap(map))) > limit)) { 2314 vm_map_unlock(map); 2315 return (KERN_NO_SPACE); /* XXX overloaded */ 2316 } 2317 #endif 2318 2319 /* 2320 * Pass 2. 2321 */ 2322 2323 for (entry = map->header.next; entry != &map->header; 2324 entry = entry->next) { 2325 if (entry->protection == VM_PROT_NONE) 2326 continue; 2327 if (VM_MAPENT_ISWIRED(entry) == 0) { /* not already wired? */ 2328 /* 2329 * perform actions of vm_map_lookup that need the 2330 * write lock on the map: create an anonymous map 2331 * for a copy-on-write region, or an anonymous map 2332 * for a zero-fill region. (XXXCDC: submap case 2333 * ok?) 2334 */ 2335 if (!UVM_ET_ISSUBMAP(entry)) { /* not submap */ 2336 if (UVM_ET_ISNEEDSCOPY(entry) && 2337 ((entry->protection & VM_PROT_WRITE) || 2338 (entry->object.uvm_obj == NULL))) { 2339 amap_copy(map, entry, M_WAITOK, TRUE, 2340 entry->start, entry->end); 2341 /* XXXCDC: wait OK? */ 2342 } 2343 } 2344 } 2345 entry->wired_count++; 2346 } 2347 2348 /* 2349 * Pass 3. 2350 */ 2351 2352 #ifdef DIAGNOSTIC 2353 timestamp_save = map->timestamp; 2354 #endif 2355 vm_map_busy(map); 2356 vm_map_downgrade(map); 2357 2358 rv = KERN_SUCCESS; 2359 for (entry = map->header.next; entry != &map->header; 2360 entry = entry->next) { 2361 if (entry->wired_count == 1) { 2362 rv = uvm_fault_wire(map, entry->start, entry->end, 2363 entry->protection); 2364 if (rv) { 2365 /* 2366 * wiring failed. break out of the loop. 2367 * we'll clean up the map below, once we 2368 * have a write lock again. 2369 */ 2370 break; 2371 } 2372 } 2373 } 2374 2375 if (rv) { /* failed? */ 2376 /* 2377 * Get back an exclusive (write) lock. 2378 */ 2379 vm_map_upgrade(map); 2380 vm_map_unbusy(map); 2381 2382 #ifdef DIAGNOSTIC 2383 if (timestamp_save != map->timestamp) 2384 panic("uvm_map_pageable_all: stale map"); 2385 #endif 2386 2387 /* 2388 * first drop the wiring count on all the entries 2389 * which haven't actually been wired yet. 2390 * 2391 * Skip VM_PROT_NONE entries like we did above. 2392 */ 2393 failed_entry = entry; 2394 for (/* nothing */; entry != &map->header; 2395 entry = entry->next) { 2396 if (entry->protection == VM_PROT_NONE) 2397 continue; 2398 entry->wired_count--; 2399 } 2400 2401 /* 2402 * now, unwire all the entries that were successfully 2403 * wired above. 2404 * 2405 * Skip VM_PROT_NONE entries like we did above. 2406 */ 2407 for (entry = map->header.next; entry != failed_entry; 2408 entry = entry->next) { 2409 if (entry->protection == VM_PROT_NONE) 2410 continue; 2411 entry->wired_count--; 2412 if (VM_MAPENT_ISWIRED(entry)) 2413 uvm_map_entry_unwire(map, entry); 2414 } 2415 vm_map_unlock(map); 2416 UVMHIST_LOG(maphist,"<- done (RV=%d)", rv,0,0,0); 2417 return (rv); 2418 } 2419 2420 /* We are holding a read lock here. */ 2421 vm_map_unbusy(map); 2422 vm_map_unlock_read(map); 2423 2424 UVMHIST_LOG(maphist,"<- done (OK WIRE)",0,0,0,0); 2425 return (KERN_SUCCESS); 2426 } 2427 2428 /* 2429 * uvm_map_clean: clean out a map range 2430 * 2431 * => valid flags: 2432 * if (flags & PGO_CLEANIT): dirty pages are cleaned first 2433 * if (flags & PGO_SYNCIO): dirty pages are written synchronously 2434 * if (flags & PGO_DEACTIVATE): any cached pages are deactivated after clean 2435 * if (flags & PGO_FREE): any cached pages are freed after clean 2436 * => returns an error if any part of the specified range isn't mapped 2437 * => never a need to flush amap layer since the anonymous memory has 2438 * no permanent home, but may deactivate pages there 2439 * => called from sys_msync() and sys_madvise() 2440 * => caller must not write-lock map (read OK). 2441 * => we may sleep while cleaning if SYNCIO [with map read-locked] 2442 */ 2443 2444 int amap_clean_works = 1; /* XXX for now, just in case... */ 2445 2446 int 2447 uvm_map_clean(map, start, end, flags) 2448 vm_map_t map; 2449 vaddr_t start, end; 2450 int flags; 2451 { 2452 vm_map_entry_t current, entry; 2453 struct uvm_object *uobj; 2454 struct vm_amap *amap; 2455 struct vm_anon *anon; 2456 struct vm_page *pg; 2457 vaddr_t offset; 2458 vsize_t size; 2459 int rv, error, refs; 2460 UVMHIST_FUNC("uvm_map_clean"); UVMHIST_CALLED(maphist); 2461 2462 UVMHIST_LOG(maphist,"(map=0x%x,start=0x%x,end=0x%x,flags=0x%x)", 2463 map, start, end, flags); 2464 KASSERT((flags & (PGO_FREE|PGO_DEACTIVATE)) != 2465 (PGO_FREE|PGO_DEACTIVATE)); 2466 2467 vm_map_lock_read(map); 2468 VM_MAP_RANGE_CHECK(map, start, end); 2469 if (uvm_map_lookup_entry(map, start, &entry) == FALSE) { 2470 vm_map_unlock_read(map); 2471 return(KERN_INVALID_ADDRESS); 2472 } 2473 2474 /* 2475 * Make a first pass to check for holes. 2476 */ 2477 2478 for (current = entry; current->start < end; current = current->next) { 2479 if (UVM_ET_ISSUBMAP(current)) { 2480 vm_map_unlock_read(map); 2481 return (KERN_INVALID_ARGUMENT); 2482 } 2483 if (end > current->end && (current->next == &map->header || 2484 current->end != current->next->start)) { 2485 vm_map_unlock_read(map); 2486 return (KERN_INVALID_ADDRESS); 2487 } 2488 } 2489 2490 error = KERN_SUCCESS; 2491 2492 for (current = entry; current->start < end; current = current->next) { 2493 amap = current->aref.ar_amap; /* top layer */ 2494 uobj = current->object.uvm_obj; /* bottom layer */ 2495 KASSERT(start >= current->start); 2496 2497 /* 2498 * No amap cleaning necessary if: 2499 * 2500 * (1) There's no amap. 2501 * 2502 * (2) We're not deactivating or freeing pages. 2503 */ 2504 2505 if (amap == NULL || 2506 (flags & (PGO_DEACTIVATE|PGO_FREE)) == 0) 2507 goto flush_object; 2508 2509 /* XXX for now, just in case... */ 2510 if (amap_clean_works == 0) 2511 goto flush_object; 2512 2513 amap_lock(amap); 2514 2515 offset = start - current->start; 2516 size = (end <= current->end ? end : current->end) - 2517 start; 2518 2519 for (/* nothing */; size != 0; size -= PAGE_SIZE, 2520 offset += PAGE_SIZE) { 2521 anon = amap_lookup(¤t->aref, offset); 2522 if (anon == NULL) 2523 continue; 2524 2525 simple_lock(&anon->an_lock); 2526 2527 pg = anon->u.an_page; 2528 if (pg == NULL) { 2529 simple_unlock(&anon->an_lock); 2530 continue; 2531 } 2532 2533 switch (flags & (PGO_CLEANIT|PGO_FREE|PGO_DEACTIVATE)) { 2534 2535 /* 2536 * XXX In these first 3 cases, we always just 2537 * XXX deactivate the page. We may want to 2538 * XXX handle the different cases more 2539 * XXX specifically, in the future. 2540 */ 2541 2542 case PGO_CLEANIT|PGO_FREE: 2543 case PGO_CLEANIT|PGO_DEACTIVATE: 2544 case PGO_DEACTIVATE: 2545 deactivate_it: 2546 /* skip the page if it's loaned or wired */ 2547 if (pg->loan_count != 0 || 2548 pg->wire_count != 0) { 2549 simple_unlock(&anon->an_lock); 2550 continue; 2551 } 2552 2553 uvm_lock_pageq(); 2554 2555 /* 2556 * skip the page if it's not actually owned 2557 * by the anon (may simply be loaned to the 2558 * anon). 2559 */ 2560 2561 if ((pg->pqflags & PQ_ANON) == 0) { 2562 KASSERT(pg->uobject == NULL); 2563 uvm_unlock_pageq(); 2564 simple_unlock(&anon->an_lock); 2565 continue; 2566 } 2567 KASSERT(pg->uanon == anon); 2568 2569 /* zap all mappings for the page. */ 2570 pmap_page_protect(pg, VM_PROT_NONE); 2571 2572 /* ...and deactivate the page. */ 2573 uvm_pagedeactivate(pg); 2574 2575 uvm_unlock_pageq(); 2576 simple_unlock(&anon->an_lock); 2577 continue; 2578 2579 case PGO_FREE: 2580 2581 /* 2582 * If there are multiple references to 2583 * the amap, just deactivate the page. 2584 */ 2585 2586 if (amap_refs(amap) > 1) 2587 goto deactivate_it; 2588 2589 /* XXX skip the page if it's wired */ 2590 if (pg->wire_count != 0) { 2591 simple_unlock(&anon->an_lock); 2592 continue; 2593 } 2594 amap_unadd(¤t->aref, offset); 2595 refs = --anon->an_ref; 2596 simple_unlock(&anon->an_lock); 2597 if (refs == 0) 2598 uvm_anfree(anon); 2599 continue; 2600 2601 default: 2602 panic("uvm_map_clean: wierd flags"); 2603 } 2604 #ifdef DIAGNOSTIC 2605 panic("uvm_map_clean: unreachable code"); 2606 #endif 2607 } 2608 2609 amap_unlock(amap); 2610 2611 flush_object: 2612 /* 2613 * flush pages if we've got a valid backing object. 2614 */ 2615 2616 offset = current->offset + (start - current->start); 2617 size = (end <= current->end ? end : current->end) - start; 2618 2619 if (uobj != NULL) { 2620 simple_lock(&uobj->vmobjlock); 2621 rv = uobj->pgops->pgo_flush(uobj, offset, 2622 offset + size, flags); 2623 simple_unlock(&uobj->vmobjlock); 2624 2625 if (rv == FALSE) 2626 error = KERN_FAILURE; 2627 } 2628 start += size; 2629 } 2630 vm_map_unlock_read(map); 2631 return (error); 2632 } 2633 2634 2635 /* 2636 * uvm_map_checkprot: check protection in map 2637 * 2638 * => must allow specified protection in a fully allocated region. 2639 * => map must be read or write locked by caller. 2640 */ 2641 2642 boolean_t 2643 uvm_map_checkprot(map, start, end, protection) 2644 vm_map_t map; 2645 vaddr_t start, end; 2646 vm_prot_t protection; 2647 { 2648 vm_map_entry_t entry; 2649 vm_map_entry_t tmp_entry; 2650 2651 if (!uvm_map_lookup_entry(map, start, &tmp_entry)) { 2652 return(FALSE); 2653 } 2654 entry = tmp_entry; 2655 while (start < end) { 2656 if (entry == &map->header) { 2657 return(FALSE); 2658 } 2659 2660 /* 2661 * no holes allowed 2662 */ 2663 2664 if (start < entry->start) { 2665 return(FALSE); 2666 } 2667 2668 /* 2669 * check protection associated with entry 2670 */ 2671 2672 if ((entry->protection & protection) != protection) { 2673 return(FALSE); 2674 } 2675 2676 /* go to next entry */ 2677 2678 start = entry->end; 2679 entry = entry->next; 2680 } 2681 return(TRUE); 2682 } 2683 2684 /* 2685 * uvmspace_alloc: allocate a vmspace structure. 2686 * 2687 * - structure includes vm_map and pmap 2688 * - XXX: no locking on this structure 2689 * - refcnt set to 1, rest must be init'd by caller 2690 */ 2691 struct vmspace * 2692 uvmspace_alloc(min, max, pageable) 2693 vaddr_t min, max; 2694 int pageable; 2695 { 2696 struct vmspace *vm; 2697 UVMHIST_FUNC("uvmspace_alloc"); UVMHIST_CALLED(maphist); 2698 2699 vm = pool_get(&uvm_vmspace_pool, PR_WAITOK); 2700 uvmspace_init(vm, NULL, min, max, pageable); 2701 UVMHIST_LOG(maphist,"<- done (vm=0x%x)", vm,0,0,0); 2702 return (vm); 2703 } 2704 2705 /* 2706 * uvmspace_init: initialize a vmspace structure. 2707 * 2708 * - XXX: no locking on this structure 2709 * - refcnt set to 1, rest must me init'd by caller 2710 */ 2711 void 2712 uvmspace_init(vm, pmap, min, max, pageable) 2713 struct vmspace *vm; 2714 struct pmap *pmap; 2715 vaddr_t min, max; 2716 boolean_t pageable; 2717 { 2718 UVMHIST_FUNC("uvmspace_init"); UVMHIST_CALLED(maphist); 2719 2720 memset(vm, 0, sizeof(*vm)); 2721 2722 uvm_map_setup(&vm->vm_map, min, max, pageable ? VM_MAP_PAGEABLE : 0); 2723 2724 if (pmap) 2725 pmap_reference(pmap); 2726 else 2727 pmap = pmap_create(); 2728 vm->vm_map.pmap = pmap; 2729 2730 vm->vm_refcnt = 1; 2731 UVMHIST_LOG(maphist,"<- done",0,0,0,0); 2732 } 2733 2734 /* 2735 * uvmspace_share: share a vmspace between two proceses 2736 * 2737 * - XXX: no locking on vmspace 2738 * - used for vfork, threads(?) 2739 */ 2740 2741 void 2742 uvmspace_share(p1, p2) 2743 struct proc *p1, *p2; 2744 { 2745 p2->p_vmspace = p1->p_vmspace; 2746 p1->p_vmspace->vm_refcnt++; 2747 } 2748 2749 /* 2750 * uvmspace_unshare: ensure that process "p" has its own, unshared, vmspace 2751 * 2752 * - XXX: no locking on vmspace 2753 */ 2754 2755 void 2756 uvmspace_unshare(p) 2757 struct proc *p; 2758 { 2759 struct vmspace *nvm, *ovm = p->p_vmspace; 2760 2761 if (ovm->vm_refcnt == 1) 2762 /* nothing to do: vmspace isn't shared in the first place */ 2763 return; 2764 2765 /* make a new vmspace, still holding old one */ 2766 nvm = uvmspace_fork(ovm); 2767 2768 pmap_deactivate(p); /* unbind old vmspace */ 2769 p->p_vmspace = nvm; 2770 pmap_activate(p); /* switch to new vmspace */ 2771 2772 uvmspace_free(ovm); /* drop reference to old vmspace */ 2773 } 2774 2775 /* 2776 * uvmspace_exec: the process wants to exec a new program 2777 * 2778 * - XXX: no locking on vmspace 2779 */ 2780 2781 void 2782 uvmspace_exec(p) 2783 struct proc *p; 2784 { 2785 struct vmspace *nvm, *ovm = p->p_vmspace; 2786 vm_map_t map = &ovm->vm_map; 2787 2788 #ifdef __sparc__ 2789 /* XXX cgd 960926: the sparc #ifdef should be a MD hook */ 2790 kill_user_windows(p); /* before stack addresses go away */ 2791 #endif 2792 2793 /* 2794 * see if more than one process is using this vmspace... 2795 */ 2796 2797 if (ovm->vm_refcnt == 1) { 2798 2799 /* 2800 * if p is the only process using its vmspace then we can safely 2801 * recycle that vmspace for the program that is being exec'd. 2802 */ 2803 2804 #ifdef SYSVSHM 2805 /* 2806 * SYSV SHM semantics require us to kill all segments on an exec 2807 */ 2808 if (ovm->vm_shm) 2809 shmexit(ovm); 2810 #endif 2811 2812 /* 2813 * POSIX 1003.1b -- "lock future mappings" is revoked 2814 * when a process execs another program image. 2815 */ 2816 vm_map_lock(map); 2817 vm_map_modflags(map, 0, VM_MAP_WIREFUTURE); 2818 vm_map_unlock(map); 2819 2820 /* 2821 * now unmap the old program 2822 */ 2823 uvm_unmap(map, VM_MIN_ADDRESS, VM_MAXUSER_ADDRESS); 2824 2825 } else { 2826 2827 /* 2828 * p's vmspace is being shared, so we can't reuse it for p since 2829 * it is still being used for others. allocate a new vmspace 2830 * for p 2831 */ 2832 nvm = uvmspace_alloc(map->min_offset, map->max_offset, 2833 (map->flags & VM_MAP_PAGEABLE) ? TRUE : FALSE); 2834 2835 /* 2836 * install new vmspace and drop our ref to the old one. 2837 */ 2838 2839 pmap_deactivate(p); 2840 p->p_vmspace = nvm; 2841 pmap_activate(p); 2842 2843 uvmspace_free(ovm); 2844 } 2845 } 2846 2847 /* 2848 * uvmspace_free: free a vmspace data structure 2849 * 2850 * - XXX: no locking on vmspace 2851 */ 2852 2853 void 2854 uvmspace_free(vm) 2855 struct vmspace *vm; 2856 { 2857 vm_map_entry_t dead_entries; 2858 UVMHIST_FUNC("uvmspace_free"); UVMHIST_CALLED(maphist); 2859 2860 UVMHIST_LOG(maphist,"(vm=0x%x) ref=%d", vm, vm->vm_refcnt,0,0); 2861 if (--vm->vm_refcnt == 0) { 2862 /* 2863 * lock the map, to wait out all other references to it. delete 2864 * all of the mappings and pages they hold, then call the pmap 2865 * module to reclaim anything left. 2866 */ 2867 vm_map_lock(&vm->vm_map); 2868 if (vm->vm_map.nentries) { 2869 (void)uvm_unmap_remove(&vm->vm_map, 2870 vm->vm_map.min_offset, vm->vm_map.max_offset, 2871 &dead_entries); 2872 if (dead_entries != NULL) 2873 uvm_unmap_detach(dead_entries, 0); 2874 } 2875 pmap_destroy(vm->vm_map.pmap); 2876 vm->vm_map.pmap = NULL; 2877 pool_put(&uvm_vmspace_pool, vm); 2878 } 2879 UVMHIST_LOG(maphist,"<- done", 0,0,0,0); 2880 } 2881 2882 /* 2883 * F O R K - m a i n e n t r y p o i n t 2884 */ 2885 /* 2886 * uvmspace_fork: fork a process' main map 2887 * 2888 * => create a new vmspace for child process from parent. 2889 * => parent's map must not be locked. 2890 */ 2891 2892 struct vmspace * 2893 uvmspace_fork(vm1) 2894 struct vmspace *vm1; 2895 { 2896 struct vmspace *vm2; 2897 vm_map_t old_map = &vm1->vm_map; 2898 vm_map_t new_map; 2899 vm_map_entry_t old_entry; 2900 vm_map_entry_t new_entry; 2901 pmap_t new_pmap; 2902 boolean_t protect_child; 2903 UVMHIST_FUNC("uvmspace_fork"); UVMHIST_CALLED(maphist); 2904 2905 vm_map_lock(old_map); 2906 2907 vm2 = uvmspace_alloc(old_map->min_offset, old_map->max_offset, 2908 (old_map->flags & VM_MAP_PAGEABLE) ? TRUE : FALSE); 2909 memcpy(&vm2->vm_startcopy, &vm1->vm_startcopy, 2910 (caddr_t) (vm1 + 1) - (caddr_t) &vm1->vm_startcopy); 2911 new_map = &vm2->vm_map; /* XXX */ 2912 new_pmap = new_map->pmap; 2913 2914 old_entry = old_map->header.next; 2915 2916 /* 2917 * go entry-by-entry 2918 */ 2919 2920 while (old_entry != &old_map->header) { 2921 2922 /* 2923 * first, some sanity checks on the old entry 2924 */ 2925 if (UVM_ET_ISSUBMAP(old_entry)) 2926 panic("fork: encountered a submap during fork (illegal)"); 2927 2928 if (!UVM_ET_ISCOPYONWRITE(old_entry) && 2929 UVM_ET_ISNEEDSCOPY(old_entry)) 2930 panic("fork: non-copy_on_write map entry marked needs_copy (illegal)"); 2931 2932 2933 switch (old_entry->inheritance) { 2934 case MAP_INHERIT_NONE: 2935 /* 2936 * drop the mapping 2937 */ 2938 break; 2939 2940 case MAP_INHERIT_SHARE: 2941 /* 2942 * share the mapping: this means we want the old and 2943 * new entries to share amaps and backing objects. 2944 */ 2945 2946 /* 2947 * if the old_entry needs a new amap (due to prev fork) 2948 * then we need to allocate it now so that we have 2949 * something we own to share with the new_entry. [in 2950 * other words, we need to clear needs_copy] 2951 */ 2952 2953 if (UVM_ET_ISNEEDSCOPY(old_entry)) { 2954 /* get our own amap, clears needs_copy */ 2955 amap_copy(old_map, old_entry, M_WAITOK, FALSE, 2956 0, 0); 2957 /* XXXCDC: WAITOK??? */ 2958 } 2959 2960 new_entry = uvm_mapent_alloc(new_map); 2961 /* old_entry -> new_entry */ 2962 uvm_mapent_copy(old_entry, new_entry); 2963 2964 /* new pmap has nothing wired in it */ 2965 new_entry->wired_count = 0; 2966 2967 /* 2968 * gain reference to object backing the map (can't 2969 * be a submap, already checked this case). 2970 */ 2971 if (new_entry->aref.ar_amap) 2972 /* share reference */ 2973 uvm_map_reference_amap(new_entry, AMAP_SHARED); 2974 2975 if (new_entry->object.uvm_obj && 2976 new_entry->object.uvm_obj->pgops->pgo_reference) 2977 new_entry->object.uvm_obj-> 2978 pgops->pgo_reference( 2979 new_entry->object.uvm_obj); 2980 2981 /* insert entry at end of new_map's entry list */ 2982 uvm_map_entry_link(new_map, new_map->header.prev, 2983 new_entry); 2984 2985 /* 2986 * pmap_copy the mappings: this routine is optional 2987 * but if it is there it will reduce the number of 2988 * page faults in the new proc. 2989 */ 2990 2991 pmap_copy(new_pmap, old_map->pmap, new_entry->start, 2992 (old_entry->end - old_entry->start), 2993 old_entry->start); 2994 2995 break; 2996 2997 case MAP_INHERIT_COPY: 2998 2999 /* 3000 * copy-on-write the mapping (using mmap's 3001 * MAP_PRIVATE semantics) 3002 * 3003 * allocate new_entry, adjust reference counts. 3004 * (note that new references are read-only). 3005 */ 3006 3007 new_entry = uvm_mapent_alloc(new_map); 3008 /* old_entry -> new_entry */ 3009 uvm_mapent_copy(old_entry, new_entry); 3010 3011 if (new_entry->aref.ar_amap) 3012 uvm_map_reference_amap(new_entry, 0); 3013 3014 if (new_entry->object.uvm_obj && 3015 new_entry->object.uvm_obj->pgops->pgo_reference) 3016 new_entry->object.uvm_obj->pgops->pgo_reference 3017 (new_entry->object.uvm_obj); 3018 3019 /* new pmap has nothing wired in it */ 3020 new_entry->wired_count = 0; 3021 3022 new_entry->etype |= 3023 (UVM_ET_COPYONWRITE|UVM_ET_NEEDSCOPY); 3024 uvm_map_entry_link(new_map, new_map->header.prev, 3025 new_entry); 3026 3027 /* 3028 * the new entry will need an amap. it will either 3029 * need to be copied from the old entry or created 3030 * from scratch (if the old entry does not have an 3031 * amap). can we defer this process until later 3032 * (by setting "needs_copy") or do we need to copy 3033 * the amap now? 3034 * 3035 * we must copy the amap now if any of the following 3036 * conditions hold: 3037 * 1. the old entry has an amap and that amap is 3038 * being shared. this means that the old (parent) 3039 * process is sharing the amap with another 3040 * process. if we do not clear needs_copy here 3041 * we will end up in a situation where both the 3042 * parent and child process are refering to the 3043 * same amap with "needs_copy" set. if the 3044 * parent write-faults, the fault routine will 3045 * clear "needs_copy" in the parent by allocating 3046 * a new amap. this is wrong because the 3047 * parent is supposed to be sharing the old amap 3048 * and the new amap will break that. 3049 * 3050 * 2. if the old entry has an amap and a non-zero 3051 * wire count then we are going to have to call 3052 * amap_cow_now to avoid page faults in the 3053 * parent process. since amap_cow_now requires 3054 * "needs_copy" to be clear we might as well 3055 * clear it here as well. 3056 * 3057 */ 3058 3059 if (old_entry->aref.ar_amap != NULL) { 3060 3061 if ((amap_flags(old_entry->aref.ar_amap) & 3062 AMAP_SHARED) != 0 || 3063 VM_MAPENT_ISWIRED(old_entry)) { 3064 3065 amap_copy(new_map, new_entry, M_WAITOK, FALSE, 3066 0, 0); 3067 /* XXXCDC: M_WAITOK ... ok? */ 3068 } 3069 } 3070 3071 /* 3072 * if the parent's entry is wired down, then the 3073 * parent process does not want page faults on 3074 * access to that memory. this means that we 3075 * cannot do copy-on-write because we can't write 3076 * protect the old entry. in this case we 3077 * resolve all copy-on-write faults now, using 3078 * amap_cow_now. note that we have already 3079 * allocated any needed amap (above). 3080 */ 3081 3082 if (VM_MAPENT_ISWIRED(old_entry)) { 3083 3084 /* 3085 * resolve all copy-on-write faults now 3086 * (note that there is nothing to do if 3087 * the old mapping does not have an amap). 3088 * XXX: is it worthwhile to bother with pmap_copy 3089 * in this case? 3090 */ 3091 if (old_entry->aref.ar_amap) 3092 amap_cow_now(new_map, new_entry); 3093 3094 } else { 3095 3096 /* 3097 * setup mappings to trigger copy-on-write faults 3098 * we must write-protect the parent if it has 3099 * an amap and it is not already "needs_copy"... 3100 * if it is already "needs_copy" then the parent 3101 * has already been write-protected by a previous 3102 * fork operation. 3103 * 3104 * if we do not write-protect the parent, then 3105 * we must be sure to write-protect the child 3106 * after the pmap_copy() operation. 3107 * 3108 * XXX: pmap_copy should have some way of telling 3109 * us that it didn't do anything so we can avoid 3110 * calling pmap_protect needlessly. 3111 */ 3112 3113 if (old_entry->aref.ar_amap) { 3114 3115 if (!UVM_ET_ISNEEDSCOPY(old_entry)) { 3116 if (old_entry->max_protection & VM_PROT_WRITE) { 3117 pmap_protect(old_map->pmap, 3118 old_entry->start, 3119 old_entry->end, 3120 old_entry->protection & 3121 ~VM_PROT_WRITE); 3122 } 3123 old_entry->etype |= UVM_ET_NEEDSCOPY; 3124 } 3125 3126 /* 3127 * parent must now be write-protected 3128 */ 3129 protect_child = FALSE; 3130 } else { 3131 3132 /* 3133 * we only need to protect the child if the 3134 * parent has write access. 3135 */ 3136 if (old_entry->max_protection & VM_PROT_WRITE) 3137 protect_child = TRUE; 3138 else 3139 protect_child = FALSE; 3140 3141 } 3142 3143 /* 3144 * copy the mappings 3145 * XXX: need a way to tell if this does anything 3146 */ 3147 3148 pmap_copy(new_pmap, old_map->pmap, 3149 new_entry->start, 3150 (old_entry->end - old_entry->start), 3151 old_entry->start); 3152 3153 /* 3154 * protect the child's mappings if necessary 3155 */ 3156 if (protect_child) { 3157 pmap_protect(new_pmap, new_entry->start, 3158 new_entry->end, 3159 new_entry->protection & 3160 ~VM_PROT_WRITE); 3161 } 3162 3163 } 3164 break; 3165 } /* end of switch statement */ 3166 old_entry = old_entry->next; 3167 } 3168 3169 new_map->size = old_map->size; 3170 vm_map_unlock(old_map); 3171 3172 #ifdef SYSVSHM 3173 if (vm1->vm_shm) 3174 shmfork(vm1, vm2); 3175 #endif 3176 3177 #ifdef PMAP_FORK 3178 pmap_fork(vm1->vm_map.pmap, vm2->vm_map.pmap); 3179 #endif 3180 3181 UVMHIST_LOG(maphist,"<- done",0,0,0,0); 3182 return(vm2); 3183 } 3184 3185 3186 #if defined(DDB) 3187 3188 /* 3189 * DDB hooks 3190 */ 3191 3192 /* 3193 * uvm_map_printit: actually prints the map 3194 */ 3195 3196 void 3197 uvm_map_printit(map, full, pr) 3198 vm_map_t map; 3199 boolean_t full; 3200 void (*pr) __P((const char *, ...)); 3201 { 3202 vm_map_entry_t entry; 3203 3204 (*pr)("MAP %p: [0x%lx->0x%lx]\n", map, map->min_offset,map->max_offset); 3205 (*pr)("\t#ent=%d, sz=%d, ref=%d, version=%d, flags=0x%x\n", 3206 map->nentries, map->size, map->ref_count, map->timestamp, 3207 map->flags); 3208 #ifdef pmap_resident_count 3209 (*pr)("\tpmap=%p(resident=%d)\n", map->pmap, 3210 pmap_resident_count(map->pmap)); 3211 #else 3212 /* XXXCDC: this should be required ... */ 3213 (*pr)("\tpmap=%p(resident=<<NOT SUPPORTED!!!>>)\n", map->pmap); 3214 #endif 3215 if (!full) 3216 return; 3217 for (entry = map->header.next; entry != &map->header; 3218 entry = entry->next) { 3219 (*pr)(" - %p: 0x%lx->0x%lx: obj=%p/0x%llx, amap=%p/%d\n", 3220 entry, entry->start, entry->end, entry->object.uvm_obj, 3221 (long long)entry->offset, entry->aref.ar_amap, 3222 entry->aref.ar_pageoff); 3223 (*pr)( 3224 "\tsubmap=%c, cow=%c, nc=%c, prot(max)=%d/%d, inh=%d, " 3225 "wc=%d, adv=%d\n", 3226 (entry->etype & UVM_ET_SUBMAP) ? 'T' : 'F', 3227 (entry->etype & UVM_ET_COPYONWRITE) ? 'T' : 'F', 3228 (entry->etype & UVM_ET_NEEDSCOPY) ? 'T' : 'F', 3229 entry->protection, entry->max_protection, 3230 entry->inheritance, entry->wired_count, entry->advice); 3231 } 3232 } 3233 3234 /* 3235 * uvm_object_printit: actually prints the object 3236 */ 3237 3238 void 3239 uvm_object_printit(uobj, full, pr) 3240 struct uvm_object *uobj; 3241 boolean_t full; 3242 void (*pr) __P((const char *, ...)); 3243 { 3244 struct vm_page *pg; 3245 int cnt = 0; 3246 3247 (*pr)("OBJECT %p: locked=%d, pgops=%p, npages=%d, ", 3248 uobj, uobj->vmobjlock.lock_data, uobj->pgops, uobj->uo_npages); 3249 if (UVM_OBJ_IS_KERN_OBJECT(uobj)) 3250 (*pr)("refs=<SYSTEM>\n"); 3251 else 3252 (*pr)("refs=%d\n", uobj->uo_refs); 3253 3254 if (!full) { 3255 return; 3256 } 3257 (*pr)(" PAGES <pg,offset>:\n "); 3258 for (pg = TAILQ_FIRST(&uobj->memq); 3259 pg != NULL; 3260 pg = TAILQ_NEXT(pg, listq), cnt++) { 3261 (*pr)("<%p,0x%llx> ", pg, (long long)pg->offset); 3262 if ((cnt % 3) == 2) { 3263 (*pr)("\n "); 3264 } 3265 } 3266 if ((cnt % 3) != 2) { 3267 (*pr)("\n"); 3268 } 3269 } 3270 3271 /* 3272 * uvm_page_printit: actually print the page 3273 */ 3274 3275 static const char page_flagbits[] = 3276 "\20\1BUSY\2WANTED\3TABLED\4CLEAN\5CLEANCHK\6RELEASED\7FAKE\10RDONLY" 3277 "\11ZERO\15PAGER1"; 3278 static const char page_pqflagbits[] = 3279 "\20\1FREE\2INACTIVE\3ACTIVE\4LAUNDRY\5ANON\6AOBJ"; 3280 3281 void 3282 uvm_page_printit(pg, full, pr) 3283 struct vm_page *pg; 3284 boolean_t full; 3285 void (*pr) __P((const char *, ...)); 3286 { 3287 struct vm_page *tpg; 3288 struct uvm_object *uobj; 3289 struct pglist *pgl; 3290 char pgbuf[128]; 3291 char pqbuf[128]; 3292 3293 (*pr)("PAGE %p:\n", pg); 3294 bitmask_snprintf(pg->flags, page_flagbits, pgbuf, sizeof(pgbuf)); 3295 bitmask_snprintf(pg->pqflags, page_pqflagbits, pqbuf, sizeof(pqbuf)); 3296 (*pr)(" flags=%s, pqflags=%s, vers=%d, wire_count=%d, pa=0x%lx\n", 3297 pgbuf, pqbuf, pg->version, pg->wire_count, (long)pg->phys_addr); 3298 (*pr)(" uobject=%p, uanon=%p, offset=0x%llx loan_count=%d\n", 3299 pg->uobject, pg->uanon, (long long)pg->offset, pg->loan_count); 3300 #if defined(UVM_PAGE_TRKOWN) 3301 if (pg->flags & PG_BUSY) 3302 (*pr)(" owning process = %d, tag=%s\n", 3303 pg->owner, pg->owner_tag); 3304 else 3305 (*pr)(" page not busy, no owner\n"); 3306 #else 3307 (*pr)(" [page ownership tracking disabled]\n"); 3308 #endif 3309 3310 if (!full) 3311 return; 3312 3313 /* cross-verify object/anon */ 3314 if ((pg->pqflags & PQ_FREE) == 0) { 3315 if (pg->pqflags & PQ_ANON) { 3316 if (pg->uanon == NULL || pg->uanon->u.an_page != pg) 3317 (*pr)(" >>> ANON DOES NOT POINT HERE <<< (%p)\n", 3318 (pg->uanon) ? pg->uanon->u.an_page : NULL); 3319 else 3320 (*pr)(" anon backpointer is OK\n"); 3321 } else { 3322 uobj = pg->uobject; 3323 if (uobj) { 3324 (*pr)(" checking object list\n"); 3325 TAILQ_FOREACH(tpg, &uobj->memq, listq) { 3326 if (tpg == pg) { 3327 break; 3328 } 3329 } 3330 if (tpg) 3331 (*pr)(" page found on object list\n"); 3332 else 3333 (*pr)(" >>> PAGE NOT FOUND ON OBJECT LIST! <<<\n"); 3334 } 3335 } 3336 } 3337 3338 /* cross-verify page queue */ 3339 if (pg->pqflags & PQ_FREE) { 3340 int fl = uvm_page_lookup_freelist(pg); 3341 pgl = &uvm.page_free[fl].pgfl_queues[((pg)->flags & PG_ZERO) ? 3342 PGFL_ZEROS : PGFL_UNKNOWN]; 3343 } else if (pg->pqflags & PQ_INACTIVE) { 3344 pgl = (pg->pqflags & PQ_SWAPBACKED) ? 3345 &uvm.page_inactive_swp : &uvm.page_inactive_obj; 3346 } else if (pg->pqflags & PQ_ACTIVE) { 3347 pgl = &uvm.page_active; 3348 } else { 3349 pgl = NULL; 3350 } 3351 3352 if (pgl) { 3353 (*pr)(" checking pageq list\n"); 3354 TAILQ_FOREACH(tpg, pgl, pageq) { 3355 if (tpg == pg) { 3356 break; 3357 } 3358 } 3359 if (tpg) 3360 (*pr)(" page found on pageq list\n"); 3361 else 3362 (*pr)(" >>> PAGE NOT FOUND ON PAGEQ LIST! <<<\n"); 3363 } 3364 } 3365 #endif 3366