1 /* $NetBSD: uvm_amap.c,v 1.83 2007/12/08 15:46:31 ad Exp $ */ 2 3 /* 4 * 5 * Copyright (c) 1997 Charles D. Cranor and Washington University. 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 3. All advertising materials mentioning features or use of this software 17 * must display the following acknowledgement: 18 * This product includes software developed by Charles D. Cranor and 19 * Washington University. 20 * 4. The name of the author may not be used to endorse or promote products 21 * derived from this software without specific prior written permission. 22 * 23 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 24 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 25 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 26 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 27 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 28 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 29 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 30 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 31 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 32 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 */ 34 35 /* 36 * uvm_amap.c: amap operations 37 */ 38 39 /* 40 * this file contains functions that perform operations on amaps. see 41 * uvm_amap.h for a brief explanation of the role of amaps in uvm. 42 */ 43 44 #include <sys/cdefs.h> 45 __KERNEL_RCSID(0, "$NetBSD: uvm_amap.c,v 1.83 2007/12/08 15:46:31 ad Exp $"); 46 47 #include "opt_uvmhist.h" 48 49 #include <sys/param.h> 50 #include <sys/systm.h> 51 #include <sys/proc.h> 52 #include <sys/kernel.h> 53 #include <sys/kmem.h> 54 #include <sys/pool.h> 55 56 #include <uvm/uvm.h> 57 #include <uvm/uvm_swap.h> 58 59 /* 60 * cache for allocation of vm_map structures. note that in order to 61 * avoid an endless loop, the amap cache's allocator cannot allocate 62 * memory from an amap (it currently goes through the kernel uobj, so 63 * we are ok). 64 */ 65 static struct pool_cache uvm_amap_cache; 66 static kmutex_t amap_list_lock; 67 static LIST_HEAD(, vm_amap) amap_list; 68 69 /* 70 * local functions 71 */ 72 73 static inline void 74 amap_list_insert(struct vm_amap *amap) 75 { 76 77 mutex_enter(&amap_list_lock); 78 LIST_INSERT_HEAD(&amap_list, amap, am_list); 79 mutex_exit(&amap_list_lock); 80 } 81 82 static inline void 83 amap_list_remove(struct vm_amap *amap) 84 { 85 86 mutex_enter(&amap_list_lock); 87 LIST_REMOVE(amap, am_list); 88 mutex_exit(&amap_list_lock); 89 } 90 91 static int 92 amap_roundup_slots(int slots) 93 { 94 95 return kmem_roundup_size(slots * sizeof(int)) / sizeof(int); 96 } 97 98 #ifdef UVM_AMAP_PPREF 99 /* 100 * what is ppref? ppref is an _optional_ amap feature which is used 101 * to keep track of reference counts on a per-page basis. it is enabled 102 * when UVM_AMAP_PPREF is defined. 103 * 104 * when enabled, an array of ints is allocated for the pprefs. this 105 * array is allocated only when a partial reference is added to the 106 * map (either by unmapping part of the amap, or gaining a reference 107 * to only a part of an amap). if the malloc of the array fails 108 * (M_NOWAIT), then we set the array pointer to PPREF_NONE to indicate 109 * that we tried to do ppref's but couldn't alloc the array so just 110 * give up (after all, this is an optional feature!). 111 * 112 * the array is divided into page sized "chunks." for chunks of length 1, 113 * the chunk reference count plus one is stored in that chunk's slot. 114 * for chunks of length > 1 the first slot contains (the reference count 115 * plus one) * -1. [the negative value indicates that the length is 116 * greater than one.] the second slot of the chunk contains the length 117 * of the chunk. here is an example: 118 * 119 * actual REFS: 2 2 2 2 3 1 1 0 0 0 4 4 0 1 1 1 120 * ppref: -3 4 x x 4 -2 2 -1 3 x -5 2 1 -2 3 x 121 * <----------><-><----><-------><----><-><-------> 122 * (x = don't care) 123 * 124 * this allows us to allow one int to contain the ref count for the whole 125 * chunk. note that the "plus one" part is needed because a reference 126 * count of zero is neither positive or negative (need a way to tell 127 * if we've got one zero or a bunch of them). 128 * 129 * here are some in-line functions to help us. 130 */ 131 132 /* 133 * pp_getreflen: get the reference and length for a specific offset 134 * 135 * => ppref's amap must be locked 136 */ 137 static inline void 138 pp_getreflen(int *ppref, int offset, int *refp, int *lenp) 139 { 140 141 if (ppref[offset] > 0) { /* chunk size must be 1 */ 142 *refp = ppref[offset] - 1; /* don't forget to adjust */ 143 *lenp = 1; 144 } else { 145 *refp = (ppref[offset] * -1) - 1; 146 *lenp = ppref[offset+1]; 147 } 148 } 149 150 /* 151 * pp_setreflen: set the reference and length for a specific offset 152 * 153 * => ppref's amap must be locked 154 */ 155 static inline void 156 pp_setreflen(int *ppref, int offset, int ref, int len) 157 { 158 if (len == 0) 159 return; 160 if (len == 1) { 161 ppref[offset] = ref + 1; 162 } else { 163 ppref[offset] = (ref + 1) * -1; 164 ppref[offset+1] = len; 165 } 166 } 167 #endif /* UVM_AMAP_PPREF */ 168 169 /* 170 * amap_alloc1: internal function that allocates an amap, but does not 171 * init the overlay. 172 * 173 * => lock on returned amap is init'd 174 */ 175 static inline struct vm_amap * 176 amap_alloc1(int slots, int padslots, int waitf) 177 { 178 struct vm_amap *amap; 179 int totalslots; 180 km_flag_t kmflags; 181 182 amap = pool_cache_get(&uvm_amap_cache, 183 ((waitf & UVM_FLAG_NOWAIT) != 0) ? PR_NOWAIT : PR_WAITOK); 184 if (amap == NULL) 185 return(NULL); 186 187 kmflags = ((waitf & UVM_FLAG_NOWAIT) != 0) ? KM_NOSLEEP : KM_SLEEP; 188 totalslots = amap_roundup_slots(slots + padslots); 189 mutex_init(&amap->am_l, MUTEX_DEFAULT, IPL_NONE); 190 amap->am_ref = 1; 191 amap->am_flags = 0; 192 #ifdef UVM_AMAP_PPREF 193 amap->am_ppref = NULL; 194 #endif 195 amap->am_maxslot = totalslots; 196 amap->am_nslot = slots; 197 amap->am_nused = 0; 198 199 amap->am_slots = kmem_alloc(totalslots * sizeof(int), kmflags); 200 if (amap->am_slots == NULL) 201 goto fail1; 202 203 amap->am_bckptr = kmem_alloc(totalslots * sizeof(int), kmflags); 204 if (amap->am_bckptr == NULL) 205 goto fail2; 206 207 amap->am_anon = kmem_alloc(totalslots * sizeof(struct vm_anon *), 208 kmflags); 209 if (amap->am_anon == NULL) 210 goto fail3; 211 212 return(amap); 213 214 fail3: 215 kmem_free(amap->am_bckptr, totalslots * sizeof(int)); 216 fail2: 217 kmem_free(amap->am_slots, totalslots * sizeof(int)); 218 fail1: 219 mutex_destroy(&amap->am_l); 220 pool_cache_put(&uvm_amap_cache, amap); 221 222 /* 223 * XXX hack to tell the pagedaemon how many pages we need, 224 * since we can need more than it would normally free. 225 */ 226 if ((waitf & UVM_FLAG_NOWAIT) != 0) { 227 extern int uvm_extrapages; 228 uvm_extrapages += ((sizeof(int) * 2 + 229 sizeof(struct vm_anon *)) * 230 totalslots) >> PAGE_SHIFT; 231 } 232 return (NULL); 233 } 234 235 /* 236 * amap_alloc: allocate an amap to manage "sz" bytes of anonymous VM 237 * 238 * => caller should ensure sz is a multiple of PAGE_SIZE 239 * => reference count to new amap is set to one 240 * => new amap is returned unlocked 241 */ 242 243 struct vm_amap * 244 amap_alloc(vaddr_t sz, vaddr_t padsz, int waitf) 245 { 246 struct vm_amap *amap; 247 int slots, padslots; 248 UVMHIST_FUNC("amap_alloc"); UVMHIST_CALLED(maphist); 249 250 AMAP_B2SLOT(slots, sz); 251 AMAP_B2SLOT(padslots, padsz); 252 253 amap = amap_alloc1(slots, padslots, waitf); 254 if (amap) { 255 memset(amap->am_anon, 0, 256 amap->am_maxslot * sizeof(struct vm_anon *)); 257 amap_list_insert(amap); 258 } 259 260 UVMHIST_LOG(maphist,"<- done, amap = 0x%x, sz=%d", amap, sz, 0, 0); 261 return(amap); 262 } 263 264 /* 265 * uvm_amap_init: initialize the amap system. 266 */ 267 void 268 uvm_amap_init(void) 269 { 270 271 mutex_init(&amap_list_lock, MUTEX_DEFAULT, IPL_NONE); 272 273 pool_cache_bootstrap(&uvm_amap_cache, sizeof(struct vm_amap), 0, 0, 0, 274 "amappl", NULL, IPL_NONE, NULL, NULL, NULL); 275 } 276 277 /* 278 * amap_free: free an amap 279 * 280 * => the amap must be unlocked 281 * => the amap should have a zero reference count and be empty 282 */ 283 void 284 amap_free(struct vm_amap *amap) 285 { 286 int slots; 287 288 UVMHIST_FUNC("amap_free"); UVMHIST_CALLED(maphist); 289 290 KASSERT(amap->am_ref == 0 && amap->am_nused == 0); 291 KASSERT((amap->am_flags & AMAP_SWAPOFF) == 0); 292 KASSERT(!mutex_owned(&amap->am_l)); 293 slots = amap->am_maxslot; 294 kmem_free(amap->am_slots, slots * sizeof(*amap->am_slots)); 295 kmem_free(amap->am_bckptr, slots * sizeof(*amap->am_bckptr)); 296 kmem_free(amap->am_anon, slots * sizeof(*amap->am_anon)); 297 #ifdef UVM_AMAP_PPREF 298 if (amap->am_ppref && amap->am_ppref != PPREF_NONE) 299 kmem_free(amap->am_ppref, slots * sizeof(*amap->am_ppref)); 300 #endif 301 mutex_destroy(&amap->am_l); 302 pool_cache_put(&uvm_amap_cache, amap); 303 UVMHIST_LOG(maphist,"<- done, freed amap = 0x%x", amap, 0, 0, 0); 304 } 305 306 /* 307 * amap_extend: extend the size of an amap (if needed) 308 * 309 * => called from uvm_map when we want to extend an amap to cover 310 * a new mapping (rather than allocate a new one) 311 * => amap should be unlocked (we will lock it) 312 * => to safely extend an amap it should have a reference count of 313 * one (thus it can't be shared) 314 */ 315 int 316 amap_extend(struct vm_map_entry *entry, vsize_t addsize, int flags) 317 { 318 struct vm_amap *amap = entry->aref.ar_amap; 319 int slotoff = entry->aref.ar_pageoff; 320 int slotmapped, slotadd, slotneed, slotadded, slotalloc; 321 int slotadj, slotspace; 322 int oldnslots; 323 #ifdef UVM_AMAP_PPREF 324 int *newppref, *oldppref; 325 #endif 326 int i, *newsl, *newbck, *oldsl, *oldbck; 327 struct vm_anon **newover, **oldover; 328 const km_flag_t kmflags = 329 (flags & AMAP_EXTEND_NOWAIT) ? KM_NOSLEEP : KM_SLEEP; 330 331 UVMHIST_FUNC("amap_extend"); UVMHIST_CALLED(maphist); 332 333 UVMHIST_LOG(maphist, " (entry=0x%x, addsize=0x%x, flags=0x%x)", 334 entry, addsize, flags, 0); 335 336 /* 337 * first, determine how many slots we need in the amap. don't 338 * forget that ar_pageoff could be non-zero: this means that 339 * there are some unused slots before us in the amap. 340 */ 341 342 amap_lock(amap); 343 KASSERT(amap_refs(amap) == 1); /* amap can't be shared */ 344 AMAP_B2SLOT(slotmapped, entry->end - entry->start); /* slots mapped */ 345 AMAP_B2SLOT(slotadd, addsize); /* slots to add */ 346 if (flags & AMAP_EXTEND_FORWARDS) { 347 slotneed = slotoff + slotmapped + slotadd; 348 slotadj = 0; 349 slotspace = 0; 350 } 351 else { 352 slotneed = slotadd + slotmapped; 353 slotadj = slotadd - slotoff; 354 slotspace = amap->am_maxslot - slotmapped; 355 } 356 357 /* 358 * case 1: we already have enough slots in the map and thus 359 * only need to bump the reference counts on the slots we are 360 * adding. 361 */ 362 363 if (flags & AMAP_EXTEND_FORWARDS) { 364 if (amap->am_nslot >= slotneed) { 365 #ifdef UVM_AMAP_PPREF 366 if (amap->am_ppref && amap->am_ppref != PPREF_NONE) { 367 amap_pp_adjref(amap, slotoff + slotmapped, 368 slotadd, 1); 369 } 370 #endif 371 amap_unlock(amap); 372 UVMHIST_LOG(maphist, 373 "<- done (case 1f), amap = 0x%x, sltneed=%d", 374 amap, slotneed, 0, 0); 375 return 0; 376 } 377 } else { 378 if (slotadj <= 0) { 379 slotoff -= slotadd; 380 entry->aref.ar_pageoff = slotoff; 381 #ifdef UVM_AMAP_PPREF 382 if (amap->am_ppref && amap->am_ppref != PPREF_NONE) { 383 amap_pp_adjref(amap, slotoff, slotadd, 1); 384 } 385 #endif 386 amap_unlock(amap); 387 UVMHIST_LOG(maphist, 388 "<- done (case 1b), amap = 0x%x, sltneed=%d", 389 amap, slotneed, 0, 0); 390 return 0; 391 } 392 } 393 394 /* 395 * case 2: we pre-allocated slots for use and we just need to 396 * bump nslot up to take account for these slots. 397 */ 398 399 if (amap->am_maxslot >= slotneed) { 400 if (flags & AMAP_EXTEND_FORWARDS) { 401 #ifdef UVM_AMAP_PPREF 402 if (amap->am_ppref && amap->am_ppref != PPREF_NONE) { 403 if ((slotoff + slotmapped) < amap->am_nslot) 404 amap_pp_adjref(amap, 405 slotoff + slotmapped, 406 (amap->am_nslot - 407 (slotoff + slotmapped)), 1); 408 pp_setreflen(amap->am_ppref, amap->am_nslot, 1, 409 slotneed - amap->am_nslot); 410 } 411 #endif 412 amap->am_nslot = slotneed; 413 amap_unlock(amap); 414 415 /* 416 * no need to zero am_anon since that was done at 417 * alloc time and we never shrink an allocation. 418 */ 419 420 UVMHIST_LOG(maphist,"<- done (case 2f), amap = 0x%x, " 421 "slotneed=%d", amap, slotneed, 0, 0); 422 return 0; 423 } else { 424 #ifdef UVM_AMAP_PPREF 425 if (amap->am_ppref && amap->am_ppref != PPREF_NONE) { 426 /* 427 * Slide up the ref counts on the pages that 428 * are actually in use. 429 */ 430 memmove(amap->am_ppref + slotspace, 431 amap->am_ppref + slotoff, 432 slotmapped * sizeof(int)); 433 /* 434 * Mark the (adjusted) gap at the front as 435 * referenced/not referenced. 436 */ 437 pp_setreflen(amap->am_ppref, 438 0, 0, slotspace - slotadd); 439 pp_setreflen(amap->am_ppref, 440 slotspace - slotadd, 1, slotadd); 441 } 442 #endif 443 444 /* 445 * Slide the anon pointers up and clear out 446 * the space we just made. 447 */ 448 memmove(amap->am_anon + slotspace, 449 amap->am_anon + slotoff, 450 slotmapped * sizeof(struct vm_anon*)); 451 memset(amap->am_anon + slotoff, 0, 452 (slotspace - slotoff) * sizeof(struct vm_anon *)); 453 454 /* 455 * Slide the backpointers up, but don't bother 456 * wiping out the old slots. 457 */ 458 memmove(amap->am_bckptr + slotspace, 459 amap->am_bckptr + slotoff, 460 slotmapped * sizeof(int)); 461 462 /* 463 * Adjust all the useful active slot numbers. 464 */ 465 for (i = 0; i < amap->am_nused; i++) 466 amap->am_slots[i] += (slotspace - slotoff); 467 468 /* 469 * We just filled all the empty space in the 470 * front of the amap by activating a few new 471 * slots. 472 */ 473 amap->am_nslot = amap->am_maxslot; 474 entry->aref.ar_pageoff = slotspace - slotadd; 475 amap_unlock(amap); 476 477 UVMHIST_LOG(maphist,"<- done (case 2b), amap = 0x%x, " 478 "slotneed=%d", amap, slotneed, 0, 0); 479 return 0; 480 } 481 } 482 483 /* 484 * case 3: we need to malloc a new amap and copy all the amap 485 * data over from old amap to the new one. 486 * 487 * note that the use of a kernel realloc() probably would not 488 * help here, since we wish to abort cleanly if one of the 489 * three (or four) mallocs fails. 490 */ 491 492 amap_unlock(amap); /* unlock in case we sleep in malloc */ 493 494 if (slotneed >= UVM_AMAP_LARGE) { 495 return E2BIG; 496 } 497 498 slotalloc = amap_roundup_slots(slotneed); 499 #ifdef UVM_AMAP_PPREF 500 newppref = NULL; 501 if (amap->am_ppref && amap->am_ppref != PPREF_NONE) 502 newppref = kmem_alloc(slotalloc * sizeof(*newppref), kmflags); 503 #endif 504 newsl = kmem_alloc(slotalloc * sizeof(*newsl), kmflags); 505 newbck = kmem_alloc(slotalloc * sizeof(*newbck), kmflags); 506 newover = kmem_alloc(slotalloc * sizeof(*newover), kmflags); 507 if (newsl == NULL || newbck == NULL || newover == NULL) { 508 #ifdef UVM_AMAP_PPREF 509 if (newppref != NULL) { 510 kmem_free(newppref, slotalloc * sizeof(*newppref)); 511 } 512 #endif 513 if (newsl != NULL) { 514 kmem_free(newsl, slotalloc * sizeof(*newsl)); 515 } 516 if (newbck != NULL) { 517 kmem_free(newbck, slotalloc * sizeof(*newbck)); 518 } 519 if (newover != NULL) { 520 kmem_free(newover, slotalloc * sizeof(*newover)); 521 } 522 return ENOMEM; 523 } 524 amap_lock(amap); 525 KASSERT(amap->am_maxslot < slotneed); 526 527 /* 528 * now copy everything over to new malloc'd areas... 529 */ 530 531 slotadded = slotalloc - amap->am_nslot; 532 if (!(flags & AMAP_EXTEND_FORWARDS)) 533 slotspace = slotalloc - slotmapped; 534 535 /* do am_slots */ 536 oldsl = amap->am_slots; 537 if (flags & AMAP_EXTEND_FORWARDS) 538 memcpy(newsl, oldsl, sizeof(int) * amap->am_nused); 539 else 540 for (i = 0; i < amap->am_nused; i++) 541 newsl[i] = oldsl[i] + slotspace - slotoff; 542 amap->am_slots = newsl; 543 544 /* do am_anon */ 545 oldover = amap->am_anon; 546 if (flags & AMAP_EXTEND_FORWARDS) { 547 memcpy(newover, oldover, 548 sizeof(struct vm_anon *) * amap->am_nslot); 549 memset(newover + amap->am_nslot, 0, 550 sizeof(struct vm_anon *) * slotadded); 551 } else { 552 memcpy(newover + slotspace, oldover + slotoff, 553 sizeof(struct vm_anon *) * slotmapped); 554 memset(newover, 0, 555 sizeof(struct vm_anon *) * slotspace); 556 } 557 amap->am_anon = newover; 558 559 /* do am_bckptr */ 560 oldbck = amap->am_bckptr; 561 if (flags & AMAP_EXTEND_FORWARDS) 562 memcpy(newbck, oldbck, sizeof(int) * amap->am_nslot); 563 else 564 memcpy(newbck + slotspace, oldbck + slotoff, 565 sizeof(int) * slotmapped); 566 amap->am_bckptr = newbck; 567 568 #ifdef UVM_AMAP_PPREF 569 /* do ppref */ 570 oldppref = amap->am_ppref; 571 if (newppref) { 572 if (flags & AMAP_EXTEND_FORWARDS) { 573 memcpy(newppref, oldppref, 574 sizeof(int) * amap->am_nslot); 575 memset(newppref + amap->am_nslot, 0, 576 sizeof(int) * slotadded); 577 } else { 578 memcpy(newppref + slotspace, oldppref + slotoff, 579 sizeof(int) * slotmapped); 580 } 581 amap->am_ppref = newppref; 582 if ((flags & AMAP_EXTEND_FORWARDS) && 583 (slotoff + slotmapped) < amap->am_nslot) 584 amap_pp_adjref(amap, slotoff + slotmapped, 585 (amap->am_nslot - (slotoff + slotmapped)), 1); 586 if (flags & AMAP_EXTEND_FORWARDS) 587 pp_setreflen(newppref, amap->am_nslot, 1, 588 slotneed - amap->am_nslot); 589 else { 590 pp_setreflen(newppref, 0, 0, 591 slotalloc - slotneed); 592 pp_setreflen(newppref, slotalloc - slotneed, 1, 593 slotneed - slotmapped); 594 } 595 } else { 596 if (amap->am_ppref) 597 amap->am_ppref = PPREF_NONE; 598 } 599 #endif 600 601 /* update master values */ 602 if (flags & AMAP_EXTEND_FORWARDS) 603 amap->am_nslot = slotneed; 604 else { 605 entry->aref.ar_pageoff = slotspace - slotadd; 606 amap->am_nslot = slotalloc; 607 } 608 oldnslots = amap->am_maxslot; 609 amap->am_maxslot = slotalloc; 610 611 amap_unlock(amap); 612 kmem_free(oldsl, oldnslots * sizeof(*oldsl)); 613 kmem_free(oldbck, oldnslots * sizeof(*oldbck)); 614 kmem_free(oldover, oldnslots * sizeof(*oldover)); 615 #ifdef UVM_AMAP_PPREF 616 if (oldppref && oldppref != PPREF_NONE) 617 kmem_free(oldppref, oldnslots * sizeof(*oldppref)); 618 #endif 619 UVMHIST_LOG(maphist,"<- done (case 3), amap = 0x%x, slotneed=%d", 620 amap, slotneed, 0, 0); 621 return 0; 622 } 623 624 /* 625 * amap_share_protect: change protection of anons in a shared amap 626 * 627 * for shared amaps, given the current data structure layout, it is 628 * not possible for us to directly locate all maps referencing the 629 * shared anon (to change the protection). in order to protect data 630 * in shared maps we use pmap_page_protect(). [this is useful for IPC 631 * mechanisms like map entry passing that may want to write-protect 632 * all mappings of a shared amap.] we traverse am_anon or am_slots 633 * depending on the current state of the amap. 634 * 635 * => entry's map and amap must be locked by the caller 636 */ 637 void 638 amap_share_protect(struct vm_map_entry *entry, vm_prot_t prot) 639 { 640 struct vm_amap *amap = entry->aref.ar_amap; 641 int slots, lcv, slot, stop; 642 643 KASSERT(mutex_owned(&amap->am_l)); 644 645 AMAP_B2SLOT(slots, (entry->end - entry->start)); 646 stop = entry->aref.ar_pageoff + slots; 647 648 if (slots < amap->am_nused) { 649 /* cheaper to traverse am_anon */ 650 for (lcv = entry->aref.ar_pageoff ; lcv < stop ; lcv++) { 651 if (amap->am_anon[lcv] == NULL) 652 continue; 653 if (amap->am_anon[lcv]->an_page != NULL) 654 pmap_page_protect(amap->am_anon[lcv]->an_page, 655 prot); 656 } 657 return; 658 } 659 660 /* cheaper to traverse am_slots */ 661 for (lcv = 0 ; lcv < amap->am_nused ; lcv++) { 662 slot = amap->am_slots[lcv]; 663 if (slot < entry->aref.ar_pageoff || slot >= stop) 664 continue; 665 if (amap->am_anon[slot]->an_page != NULL) 666 pmap_page_protect(amap->am_anon[slot]->an_page, prot); 667 } 668 } 669 670 /* 671 * amap_wipeout: wipeout all anon's in an amap; then free the amap! 672 * 673 * => called from amap_unref when the final reference to an amap is 674 * discarded (i.e. when reference count drops to 0) 675 * => the amap should be locked (by the caller) 676 */ 677 678 void 679 amap_wipeout(struct vm_amap *amap) 680 { 681 int lcv, slot; 682 struct vm_anon *anon; 683 UVMHIST_FUNC("amap_wipeout"); UVMHIST_CALLED(maphist); 684 UVMHIST_LOG(maphist,"(amap=0x%x)", amap, 0,0,0); 685 686 KASSERT(amap->am_ref == 0); 687 688 if (__predict_false((amap->am_flags & AMAP_SWAPOFF) != 0)) { 689 /* 690 * amap_swap_off will call us again. 691 */ 692 amap_unlock(amap); 693 return; 694 } 695 amap_list_remove(amap); 696 amap_unlock(amap); 697 698 for (lcv = 0 ; lcv < amap->am_nused ; lcv++) { 699 int refs; 700 701 slot = amap->am_slots[lcv]; 702 anon = amap->am_anon[slot]; 703 704 if (anon == NULL || anon->an_ref == 0) 705 panic("amap_wipeout: corrupt amap"); 706 707 simple_lock(&anon->an_lock); 708 UVMHIST_LOG(maphist," processing anon 0x%x, ref=%d", anon, 709 anon->an_ref, 0, 0); 710 refs = --anon->an_ref; 711 simple_unlock(&anon->an_lock); 712 if (refs == 0) { 713 714 /* 715 * we had the last reference to a vm_anon. free it. 716 */ 717 718 uvm_anfree(anon); 719 } 720 721 if (curlwp->l_cpu->ci_schedstate.spc_flags & SPCF_SHOULDYIELD) 722 preempt(); 723 } 724 725 /* 726 * now we free the map 727 */ 728 729 amap->am_nused = 0; 730 amap_free(amap); /* will unlock and free amap */ 731 UVMHIST_LOG(maphist,"<- done!", 0,0,0,0); 732 } 733 734 /* 735 * amap_copy: ensure that a map entry's "needs_copy" flag is false 736 * by copying the amap if necessary. 737 * 738 * => an entry with a null amap pointer will get a new (blank) one. 739 * => the map that the map entry belongs to must be locked by caller. 740 * => the amap currently attached to "entry" (if any) must be unlocked. 741 * => if canchunk is true, then we may clip the entry into a chunk 742 * => "startva" and "endva" are used only if canchunk is true. they are 743 * used to limit chunking (e.g. if you have a large space that you 744 * know you are going to need to allocate amaps for, there is no point 745 * in allowing that to be chunked) 746 */ 747 748 void 749 amap_copy(struct vm_map *map, struct vm_map_entry *entry, int flags, 750 vaddr_t startva, vaddr_t endva) 751 { 752 struct vm_amap *amap, *srcamap; 753 int slots, lcv; 754 vaddr_t chunksize; 755 const int waitf = (flags & AMAP_COPY_NOWAIT) ? UVM_FLAG_NOWAIT : 0; 756 const bool canchunk = (flags & AMAP_COPY_NOCHUNK) == 0; 757 UVMHIST_FUNC("amap_copy"); UVMHIST_CALLED(maphist); 758 UVMHIST_LOG(maphist, " (map=%p, entry=%p, flags=%d)", 759 map, entry, flags, 0); 760 761 KASSERT(map != kernel_map); /* we use nointr pool */ 762 763 /* 764 * is there a map to copy? if not, create one from scratch. 765 */ 766 767 if (entry->aref.ar_amap == NULL) { 768 769 /* 770 * check to see if we have a large amap that we can 771 * chunk. we align startva/endva to chunk-sized 772 * boundaries and then clip to them. 773 */ 774 775 if (canchunk && atop(entry->end - entry->start) >= 776 UVM_AMAP_LARGE) { 777 /* convert slots to bytes */ 778 chunksize = UVM_AMAP_CHUNK << PAGE_SHIFT; 779 startva = (startva / chunksize) * chunksize; 780 endva = roundup(endva, chunksize); 781 UVMHIST_LOG(maphist, " chunk amap ==> clip 0x%x->0x%x" 782 "to 0x%x->0x%x", entry->start, entry->end, startva, 783 endva); 784 UVM_MAP_CLIP_START(map, entry, startva, NULL); 785 /* watch out for endva wrap-around! */ 786 if (endva >= startva) 787 UVM_MAP_CLIP_END(map, entry, endva, NULL); 788 } 789 790 if ((flags & AMAP_COPY_NOMERGE) == 0 && 791 uvm_mapent_trymerge(map, entry, UVM_MERGE_COPYING)) { 792 return; 793 } 794 795 UVMHIST_LOG(maphist, "<- done [creating new amap 0x%x->0x%x]", 796 entry->start, entry->end, 0, 0); 797 entry->aref.ar_pageoff = 0; 798 entry->aref.ar_amap = amap_alloc(entry->end - entry->start, 0, 799 waitf); 800 if (entry->aref.ar_amap != NULL) 801 entry->etype &= ~UVM_ET_NEEDSCOPY; 802 return; 803 } 804 805 /* 806 * first check and see if we are the only map entry 807 * referencing the amap we currently have. if so, then we can 808 * just take it over rather than copying it. note that we are 809 * reading am_ref with the amap unlocked... the value can only 810 * be one if we have the only reference to the amap (via our 811 * locked map). if we are greater than one we fall through to 812 * the next case (where we double check the value). 813 */ 814 815 if (entry->aref.ar_amap->am_ref == 1) { 816 entry->etype &= ~UVM_ET_NEEDSCOPY; 817 UVMHIST_LOG(maphist, "<- done [ref cnt = 1, took it over]", 818 0, 0, 0, 0); 819 return; 820 } 821 822 /* 823 * looks like we need to copy the map. 824 */ 825 826 UVMHIST_LOG(maphist," amap=%p, ref=%d, must copy it", 827 entry->aref.ar_amap, entry->aref.ar_amap->am_ref, 0, 0); 828 AMAP_B2SLOT(slots, entry->end - entry->start); 829 amap = amap_alloc1(slots, 0, waitf); 830 if (amap == NULL) { 831 UVMHIST_LOG(maphist, " amap_alloc1 failed", 0,0,0,0); 832 return; 833 } 834 srcamap = entry->aref.ar_amap; 835 amap_lock(srcamap); 836 837 /* 838 * need to double check reference count now that we've got the 839 * src amap locked down. the reference count could have 840 * changed while we were in malloc. if the reference count 841 * dropped down to one we take over the old map rather than 842 * copying the amap. 843 */ 844 845 if (srcamap->am_ref == 1) { /* take it over? */ 846 entry->etype &= ~UVM_ET_NEEDSCOPY; 847 amap->am_ref--; /* drop final reference to map */ 848 amap_free(amap); /* dispose of new (unused) amap */ 849 amap_unlock(srcamap); 850 return; 851 } 852 853 /* 854 * we must copy it now. 855 */ 856 857 UVMHIST_LOG(maphist, " copying amap now",0, 0, 0, 0); 858 for (lcv = 0 ; lcv < slots; lcv++) { 859 amap->am_anon[lcv] = 860 srcamap->am_anon[entry->aref.ar_pageoff + lcv]; 861 if (amap->am_anon[lcv] == NULL) 862 continue; 863 simple_lock(&amap->am_anon[lcv]->an_lock); 864 amap->am_anon[lcv]->an_ref++; 865 simple_unlock(&amap->am_anon[lcv]->an_lock); 866 amap->am_bckptr[lcv] = amap->am_nused; 867 amap->am_slots[amap->am_nused] = lcv; 868 amap->am_nused++; 869 } 870 memset(&amap->am_anon[lcv], 0, 871 (amap->am_maxslot - lcv) * sizeof(struct vm_anon *)); 872 873 /* 874 * drop our reference to the old amap (srcamap) and unlock. 875 * we know that the reference count on srcamap is greater than 876 * one (we checked above), so there is no way we could drop 877 * the count to zero. [and no need to worry about freeing it] 878 */ 879 880 srcamap->am_ref--; 881 if (srcamap->am_ref == 1 && (srcamap->am_flags & AMAP_SHARED) != 0) 882 srcamap->am_flags &= ~AMAP_SHARED; /* clear shared flag */ 883 #ifdef UVM_AMAP_PPREF 884 if (srcamap->am_ppref && srcamap->am_ppref != PPREF_NONE) { 885 amap_pp_adjref(srcamap, entry->aref.ar_pageoff, 886 (entry->end - entry->start) >> PAGE_SHIFT, -1); 887 } 888 #endif 889 890 amap_unlock(srcamap); 891 892 amap_list_insert(amap); 893 894 /* 895 * install new amap. 896 */ 897 898 entry->aref.ar_pageoff = 0; 899 entry->aref.ar_amap = amap; 900 entry->etype &= ~UVM_ET_NEEDSCOPY; 901 UVMHIST_LOG(maphist, "<- done",0, 0, 0, 0); 902 } 903 904 /* 905 * amap_cow_now: resolve all copy-on-write faults in an amap now for fork(2) 906 * 907 * called during fork(2) when the parent process has a wired map 908 * entry. in that case we want to avoid write-protecting pages 909 * in the parent's map (e.g. like what you'd do for a COW page) 910 * so we resolve the COW here. 911 * 912 * => assume parent's entry was wired, thus all pages are resident. 913 * => assume pages that are loaned out (loan_count) are already mapped 914 * read-only in all maps, and thus no need for us to worry about them 915 * => assume both parent and child vm_map's are locked 916 * => caller passes child's map/entry in to us 917 * => if we run out of memory we will unlock the amap and sleep _with_ the 918 * parent and child vm_map's locked(!). we have to do this since 919 * we are in the middle of a fork(2) and we can't let the parent 920 * map change until we are done copying all the map entrys. 921 * => XXXCDC: out of memory should cause fork to fail, but there is 922 * currently no easy way to do this (needs fix) 923 * => page queues must be unlocked (we may lock them) 924 */ 925 926 void 927 amap_cow_now(struct vm_map *map, struct vm_map_entry *entry) 928 { 929 struct vm_amap *amap = entry->aref.ar_amap; 930 int lcv, slot; 931 struct vm_anon *anon, *nanon; 932 struct vm_page *pg, *npg; 933 934 /* 935 * note that if we unlock the amap then we must ReStart the "lcv" for 936 * loop because some other process could reorder the anon's in the 937 * am_anon[] array on us while the lock is dropped. 938 */ 939 940 ReStart: 941 amap_lock(amap); 942 for (lcv = 0 ; lcv < amap->am_nused ; lcv++) { 943 944 /* 945 * get the page 946 */ 947 948 slot = amap->am_slots[lcv]; 949 anon = amap->am_anon[slot]; 950 simple_lock(&anon->an_lock); 951 952 /* 953 * If the anon has only one ref, we must have already copied it. 954 * This can happen if we needed to sleep waiting for memory 955 * in a previous run through this loop. The new page might 956 * even have been paged out, since the new page is not wired. 957 */ 958 959 if (anon->an_ref == 1) { 960 KASSERT(anon->an_page != NULL || anon->an_swslot != 0); 961 simple_unlock(&anon->an_lock); 962 continue; 963 } 964 965 /* 966 * The old page must be resident since the parent is wired. 967 */ 968 969 pg = anon->an_page; 970 KASSERT(pg != NULL); 971 KASSERT(pg->wire_count > 0); 972 973 /* 974 * If the page is loaned then it must already be mapped 975 * read-only and we don't need to copy it. 976 */ 977 978 if (pg->loan_count != 0) { 979 simple_unlock(&anon->an_lock); 980 continue; 981 } 982 KASSERT(pg->uanon == anon && pg->uobject == NULL); 983 984 /* 985 * if the page is busy then we have to unlock, wait for 986 * it and then restart. 987 */ 988 989 if (pg->flags & PG_BUSY) { 990 pg->flags |= PG_WANTED; 991 amap_unlock(amap); 992 UVM_UNLOCK_AND_WAIT(pg, &anon->an_lock, false, 993 "cownow", 0); 994 goto ReStart; 995 } 996 997 /* 998 * ok, time to do a copy-on-write to a new anon 999 */ 1000 1001 nanon = uvm_analloc(); 1002 if (nanon) { 1003 npg = uvm_pagealloc(NULL, 0, nanon, 0); 1004 } else 1005 npg = NULL; /* XXX: quiet gcc warning */ 1006 if (nanon == NULL || npg == NULL) { 1007 1008 /* 1009 * XXXCDC: we should cause fork to fail, but we can't. 1010 */ 1011 1012 if (nanon) { 1013 nanon->an_ref--; 1014 simple_unlock(&nanon->an_lock); 1015 uvm_anfree(nanon); 1016 } 1017 simple_unlock(&anon->an_lock); 1018 amap_unlock(amap); 1019 uvm_wait("cownowpage"); 1020 goto ReStart; 1021 } 1022 1023 /* 1024 * got it... now we can copy the data and replace anon 1025 * with our new one... 1026 */ 1027 1028 uvm_pagecopy(pg, npg); /* old -> new */ 1029 anon->an_ref--; /* can't drop to zero */ 1030 amap->am_anon[slot] = nanon; /* replace */ 1031 1032 /* 1033 * drop PG_BUSY on new page ... since we have had its owner 1034 * locked the whole time it can't be PG_RELEASED or PG_WANTED. 1035 */ 1036 1037 uvm_lock_pageq(); 1038 uvm_pageactivate(npg); 1039 uvm_unlock_pageq(); 1040 npg->flags &= ~(PG_BUSY|PG_FAKE); 1041 UVM_PAGE_OWN(npg, NULL); 1042 simple_unlock(&nanon->an_lock); 1043 simple_unlock(&anon->an_lock); 1044 } 1045 amap_unlock(amap); 1046 } 1047 1048 /* 1049 * amap_splitref: split a single reference into two separate references 1050 * 1051 * => called from uvm_map's clip routines 1052 * => origref's map should be locked 1053 * => origref->ar_amap should be unlocked (we will lock) 1054 */ 1055 void 1056 amap_splitref(struct vm_aref *origref, struct vm_aref *splitref, vaddr_t offset) 1057 { 1058 int leftslots; 1059 struct vm_amap *amap; 1060 1061 KASSERT(splitref->ar_amap == origref->ar_amap); 1062 AMAP_B2SLOT(leftslots, offset); 1063 if (leftslots == 0) 1064 panic("amap_splitref: split at zero offset"); 1065 1066 amap = origref->ar_amap; 1067 amap_lock(amap); 1068 1069 /* 1070 * now: amap is locked and we have a valid am_mapped array. 1071 */ 1072 1073 if (amap->am_nslot - origref->ar_pageoff - leftslots <= 0) 1074 panic("amap_splitref: map size check failed"); 1075 1076 #ifdef UVM_AMAP_PPREF 1077 /* 1078 * establish ppref before we add a duplicate reference to the amap 1079 */ 1080 if (amap->am_ppref == NULL) 1081 amap_pp_establish(amap, origref->ar_pageoff); 1082 #endif 1083 1084 amap->am_ref++; /* not a share reference */ 1085 splitref->ar_pageoff = origref->ar_pageoff + leftslots; 1086 1087 amap_unlock(amap); 1088 } 1089 1090 #ifdef UVM_AMAP_PPREF 1091 1092 /* 1093 * amap_pp_establish: add a ppref array to an amap, if possible 1094 * 1095 * => amap locked by caller 1096 */ 1097 void 1098 amap_pp_establish(struct vm_amap *amap, vaddr_t offset) 1099 { 1100 1101 amap->am_ppref = kmem_alloc(amap->am_maxslot * sizeof(*amap->am_ppref), 1102 KM_NOSLEEP); 1103 1104 /* 1105 * if we fail then we just won't use ppref for this amap 1106 */ 1107 1108 if (amap->am_ppref == NULL) { 1109 amap->am_ppref = PPREF_NONE; /* not using it */ 1110 return; 1111 } 1112 memset(amap->am_ppref, 0, sizeof(int) * amap->am_maxslot); 1113 pp_setreflen(amap->am_ppref, 0, 0, offset); 1114 pp_setreflen(amap->am_ppref, offset, amap->am_ref, 1115 amap->am_nslot - offset); 1116 return; 1117 } 1118 1119 /* 1120 * amap_pp_adjref: adjust reference count to a part of an amap using the 1121 * per-page reference count array. 1122 * 1123 * => map and amap locked by caller 1124 * => caller must check that ppref != PPREF_NONE before calling 1125 */ 1126 void 1127 amap_pp_adjref(struct vm_amap *amap, int curslot, vsize_t slotlen, int adjval) 1128 { 1129 int stopslot, *ppref, lcv, prevlcv; 1130 int ref, len, prevref, prevlen; 1131 1132 stopslot = curslot + slotlen; 1133 ppref = amap->am_ppref; 1134 prevlcv = 0; 1135 1136 /* 1137 * first advance to the correct place in the ppref array, 1138 * fragment if needed. 1139 */ 1140 1141 for (lcv = 0 ; lcv < curslot ; lcv += len) { 1142 pp_getreflen(ppref, lcv, &ref, &len); 1143 if (lcv + len > curslot) { /* goes past start? */ 1144 pp_setreflen(ppref, lcv, ref, curslot - lcv); 1145 pp_setreflen(ppref, curslot, ref, len - (curslot -lcv)); 1146 len = curslot - lcv; /* new length of entry @ lcv */ 1147 } 1148 prevlcv = lcv; 1149 } 1150 if (lcv != 0) 1151 pp_getreflen(ppref, prevlcv, &prevref, &prevlen); 1152 else { 1153 /* Ensure that the "prevref == ref" test below always 1154 * fails, since we're starting from the beginning of 1155 * the ppref array; that is, there is no previous 1156 * chunk. 1157 */ 1158 prevref = -1; 1159 prevlen = 0; 1160 } 1161 1162 /* 1163 * now adjust reference counts in range. merge the first 1164 * changed entry with the last unchanged entry if possible. 1165 */ 1166 1167 if (lcv != curslot) 1168 panic("amap_pp_adjref: overshot target"); 1169 1170 for (/* lcv already set */; lcv < stopslot ; lcv += len) { 1171 pp_getreflen(ppref, lcv, &ref, &len); 1172 if (lcv + len > stopslot) { /* goes past end? */ 1173 pp_setreflen(ppref, lcv, ref, stopslot - lcv); 1174 pp_setreflen(ppref, stopslot, ref, 1175 len - (stopslot - lcv)); 1176 len = stopslot - lcv; 1177 } 1178 ref += adjval; 1179 if (ref < 0) 1180 panic("amap_pp_adjref: negative reference count"); 1181 if (lcv == prevlcv + prevlen && ref == prevref) { 1182 pp_setreflen(ppref, prevlcv, ref, prevlen + len); 1183 } else { 1184 pp_setreflen(ppref, lcv, ref, len); 1185 } 1186 if (ref == 0) 1187 amap_wiperange(amap, lcv, len); 1188 } 1189 1190 } 1191 1192 /* 1193 * amap_wiperange: wipe out a range of an amap 1194 * [different from amap_wipeout because the amap is kept intact] 1195 * 1196 * => both map and amap must be locked by caller. 1197 */ 1198 void 1199 amap_wiperange(struct vm_amap *amap, int slotoff, int slots) 1200 { 1201 int byanon, lcv, stop, curslot, ptr, slotend; 1202 struct vm_anon *anon; 1203 1204 /* 1205 * we can either traverse the amap by am_anon or by am_slots depending 1206 * on which is cheaper. decide now. 1207 */ 1208 1209 if (slots < amap->am_nused) { 1210 byanon = true; 1211 lcv = slotoff; 1212 stop = slotoff + slots; 1213 slotend = 0; 1214 } else { 1215 byanon = false; 1216 lcv = 0; 1217 stop = amap->am_nused; 1218 slotend = slotoff + slots; 1219 } 1220 1221 while (lcv < stop) { 1222 int refs; 1223 1224 if (byanon) { 1225 curslot = lcv++; /* lcv advances here */ 1226 if (amap->am_anon[curslot] == NULL) 1227 continue; 1228 } else { 1229 curslot = amap->am_slots[lcv]; 1230 if (curslot < slotoff || curslot >= slotend) { 1231 lcv++; /* lcv advances here */ 1232 continue; 1233 } 1234 stop--; /* drop stop, since anon will be removed */ 1235 } 1236 anon = amap->am_anon[curslot]; 1237 1238 /* 1239 * remove it from the amap 1240 */ 1241 1242 amap->am_anon[curslot] = NULL; 1243 ptr = amap->am_bckptr[curslot]; 1244 if (ptr != (amap->am_nused - 1)) { 1245 amap->am_slots[ptr] = 1246 amap->am_slots[amap->am_nused - 1]; 1247 amap->am_bckptr[amap->am_slots[ptr]] = 1248 ptr; /* back ptr. */ 1249 } 1250 amap->am_nused--; 1251 1252 /* 1253 * drop anon reference count 1254 */ 1255 1256 simple_lock(&anon->an_lock); 1257 refs = --anon->an_ref; 1258 simple_unlock(&anon->an_lock); 1259 if (refs == 0) { 1260 1261 /* 1262 * we just eliminated the last reference to an anon. 1263 * free it. 1264 */ 1265 1266 uvm_anfree(anon); 1267 } 1268 } 1269 } 1270 1271 #endif 1272 1273 #if defined(VMSWAP) 1274 1275 /* 1276 * amap_swap_off: pagein anonymous pages in amaps and drop swap slots. 1277 * 1278 * => called with swap_syscall_lock held. 1279 * => note that we don't always traverse all anons. 1280 * eg. amaps being wiped out, released anons. 1281 * => return true if failed. 1282 */ 1283 1284 bool 1285 amap_swap_off(int startslot, int endslot) 1286 { 1287 struct vm_amap *am; 1288 struct vm_amap *am_next; 1289 struct vm_amap marker_prev; 1290 struct vm_amap marker_next; 1291 struct lwp *l = curlwp; 1292 bool rv = false; 1293 1294 #if defined(DIAGNOSTIC) 1295 memset(&marker_prev, 0, sizeof(marker_prev)); 1296 memset(&marker_next, 0, sizeof(marker_next)); 1297 #endif /* defined(DIAGNOSTIC) */ 1298 1299 uvm_lwp_hold(l); 1300 mutex_enter(&amap_list_lock); 1301 for (am = LIST_FIRST(&amap_list); am != NULL && !rv; am = am_next) { 1302 int i; 1303 1304 LIST_INSERT_BEFORE(am, &marker_prev, am_list); 1305 LIST_INSERT_AFTER(am, &marker_next, am_list); 1306 1307 if (!amap_lock_try(am)) { 1308 mutex_exit(&amap_list_lock); 1309 preempt(); 1310 mutex_enter(&amap_list_lock); 1311 am_next = LIST_NEXT(&marker_prev, am_list); 1312 if (am_next == &marker_next) { 1313 am_next = LIST_NEXT(am_next, am_list); 1314 } else { 1315 KASSERT(LIST_NEXT(am_next, am_list) == 1316 &marker_next); 1317 } 1318 LIST_REMOVE(&marker_prev, am_list); 1319 LIST_REMOVE(&marker_next, am_list); 1320 continue; 1321 } 1322 1323 mutex_exit(&amap_list_lock); 1324 1325 if (am->am_nused <= 0) { 1326 amap_unlock(am); 1327 goto next; 1328 } 1329 1330 for (i = 0; i < am->am_nused; i++) { 1331 int slot; 1332 int swslot; 1333 struct vm_anon *anon; 1334 1335 slot = am->am_slots[i]; 1336 anon = am->am_anon[slot]; 1337 simple_lock(&anon->an_lock); 1338 1339 swslot = anon->an_swslot; 1340 if (swslot < startslot || endslot <= swslot) { 1341 simple_unlock(&anon->an_lock); 1342 continue; 1343 } 1344 1345 am->am_flags |= AMAP_SWAPOFF; 1346 amap_unlock(am); 1347 1348 rv = uvm_anon_pagein(anon); 1349 1350 amap_lock(am); 1351 am->am_flags &= ~AMAP_SWAPOFF; 1352 if (amap_refs(am) == 0) { 1353 amap_wipeout(am); 1354 am = NULL; 1355 break; 1356 } 1357 if (rv) { 1358 break; 1359 } 1360 i = 0; 1361 } 1362 1363 if (am) { 1364 amap_unlock(am); 1365 } 1366 1367 next: 1368 mutex_enter(&amap_list_lock); 1369 KASSERT(LIST_NEXT(&marker_prev, am_list) == &marker_next || 1370 LIST_NEXT(LIST_NEXT(&marker_prev, am_list), am_list) == 1371 &marker_next); 1372 am_next = LIST_NEXT(&marker_next, am_list); 1373 LIST_REMOVE(&marker_prev, am_list); 1374 LIST_REMOVE(&marker_next, am_list); 1375 } 1376 mutex_exit(&amap_list_lock); 1377 uvm_lwp_rele(l); 1378 1379 return rv; 1380 } 1381 1382 #endif /* defined(VMSWAP) */ 1383 1384 /* 1385 * amap_lookup: look up a page in an amap 1386 * 1387 * => amap should be locked by caller. 1388 */ 1389 struct vm_anon * 1390 amap_lookup(struct vm_aref *aref, vaddr_t offset) 1391 { 1392 int slot; 1393 struct vm_amap *amap = aref->ar_amap; 1394 UVMHIST_FUNC("amap_lookup"); UVMHIST_CALLED(maphist); 1395 KASSERT(mutex_owned(&amap->am_l)); 1396 1397 AMAP_B2SLOT(slot, offset); 1398 slot += aref->ar_pageoff; 1399 1400 if (slot >= amap->am_nslot) 1401 panic("amap_lookup: offset out of range"); 1402 1403 UVMHIST_LOG(maphist, "<- done (amap=0x%x, offset=0x%x, result=0x%x)", 1404 amap, offset, amap->am_anon[slot], 0); 1405 return(amap->am_anon[slot]); 1406 } 1407 1408 /* 1409 * amap_lookups: look up a range of pages in an amap 1410 * 1411 * => amap should be locked by caller. 1412 * => XXXCDC: this interface is biased toward array-based amaps. fix. 1413 */ 1414 void 1415 amap_lookups(struct vm_aref *aref, vaddr_t offset, struct vm_anon **anons, 1416 int npages) 1417 { 1418 int slot; 1419 struct vm_amap *amap = aref->ar_amap; 1420 UVMHIST_FUNC("amap_lookups"); UVMHIST_CALLED(maphist); 1421 KASSERT(mutex_owned(&amap->am_l)); 1422 1423 AMAP_B2SLOT(slot, offset); 1424 slot += aref->ar_pageoff; 1425 1426 UVMHIST_LOG(maphist, " slot=%d, npages=%d, nslot=%d", slot, npages, 1427 amap->am_nslot, 0); 1428 1429 if ((slot + (npages - 1)) >= amap->am_nslot) 1430 panic("amap_lookups: offset out of range"); 1431 1432 memcpy(anons, &amap->am_anon[slot], npages * sizeof(struct vm_anon *)); 1433 1434 UVMHIST_LOG(maphist, "<- done", 0, 0, 0, 0); 1435 return; 1436 } 1437 1438 /* 1439 * amap_add: add (or replace) a page to an amap 1440 * 1441 * => caller must lock amap. 1442 * => if (replace) caller must lock anon because we might have to call 1443 * pmap_page_protect on the anon's page. 1444 */ 1445 void 1446 amap_add(struct vm_aref *aref, vaddr_t offset, struct vm_anon *anon, 1447 bool replace) 1448 { 1449 int slot; 1450 struct vm_amap *amap = aref->ar_amap; 1451 UVMHIST_FUNC("amap_add"); UVMHIST_CALLED(maphist); 1452 KASSERT(mutex_owned(&amap->am_l)); 1453 1454 AMAP_B2SLOT(slot, offset); 1455 slot += aref->ar_pageoff; 1456 1457 if (slot >= amap->am_nslot) 1458 panic("amap_add: offset out of range"); 1459 1460 if (replace) { 1461 1462 if (amap->am_anon[slot] == NULL) 1463 panic("amap_add: replacing null anon"); 1464 if (amap->am_anon[slot]->an_page != NULL && 1465 (amap->am_flags & AMAP_SHARED) != 0) { 1466 pmap_page_protect(amap->am_anon[slot]->an_page, 1467 VM_PROT_NONE); 1468 /* 1469 * XXX: suppose page is supposed to be wired somewhere? 1470 */ 1471 } 1472 } else { /* !replace */ 1473 if (amap->am_anon[slot] != NULL) 1474 panic("amap_add: slot in use"); 1475 1476 amap->am_bckptr[slot] = amap->am_nused; 1477 amap->am_slots[amap->am_nused] = slot; 1478 amap->am_nused++; 1479 } 1480 amap->am_anon[slot] = anon; 1481 UVMHIST_LOG(maphist, 1482 "<- done (amap=0x%x, offset=0x%x, anon=0x%x, rep=%d)", 1483 amap, offset, anon, replace); 1484 } 1485 1486 /* 1487 * amap_unadd: remove a page from an amap 1488 * 1489 * => caller must lock amap 1490 */ 1491 void 1492 amap_unadd(struct vm_aref *aref, vaddr_t offset) 1493 { 1494 int ptr, slot; 1495 struct vm_amap *amap = aref->ar_amap; 1496 UVMHIST_FUNC("amap_unadd"); UVMHIST_CALLED(maphist); 1497 KASSERT(mutex_owned(&amap->am_l)); 1498 1499 AMAP_B2SLOT(slot, offset); 1500 slot += aref->ar_pageoff; 1501 1502 if (slot >= amap->am_nslot) 1503 panic("amap_unadd: offset out of range"); 1504 1505 if (amap->am_anon[slot] == NULL) 1506 panic("amap_unadd: nothing there"); 1507 1508 amap->am_anon[slot] = NULL; 1509 ptr = amap->am_bckptr[slot]; 1510 1511 if (ptr != (amap->am_nused - 1)) { /* swap to keep slots contig? */ 1512 amap->am_slots[ptr] = amap->am_slots[amap->am_nused - 1]; 1513 amap->am_bckptr[amap->am_slots[ptr]] = ptr; /* back link */ 1514 } 1515 amap->am_nused--; 1516 UVMHIST_LOG(maphist, "<- done (amap=0x%x, slot=0x%x)", amap, slot,0, 0); 1517 } 1518 1519 /* 1520 * amap_ref: gain a reference to an amap 1521 * 1522 * => amap must not be locked (we will lock) 1523 * => "offset" and "len" are in units of pages 1524 * => called at fork time to gain the child's reference 1525 */ 1526 void 1527 amap_ref(struct vm_amap *amap, vaddr_t offset, vsize_t len, int flags) 1528 { 1529 UVMHIST_FUNC("amap_ref"); UVMHIST_CALLED(maphist); 1530 1531 amap_lock(amap); 1532 if (flags & AMAP_SHARED) 1533 amap->am_flags |= AMAP_SHARED; 1534 #ifdef UVM_AMAP_PPREF 1535 if (amap->am_ppref == NULL && (flags & AMAP_REFALL) == 0 && 1536 len != amap->am_nslot) 1537 amap_pp_establish(amap, offset); 1538 #endif 1539 amap->am_ref++; 1540 #ifdef UVM_AMAP_PPREF 1541 if (amap->am_ppref && amap->am_ppref != PPREF_NONE) { 1542 if (flags & AMAP_REFALL) 1543 amap_pp_adjref(amap, 0, amap->am_nslot, 1); 1544 else 1545 amap_pp_adjref(amap, offset, len, 1); 1546 } 1547 #endif 1548 amap_unlock(amap); 1549 UVMHIST_LOG(maphist,"<- done! amap=0x%x", amap, 0, 0, 0); 1550 } 1551 1552 /* 1553 * amap_unref: remove a reference to an amap 1554 * 1555 * => caller must remove all pmap-level references to this amap before 1556 * dropping the reference 1557 * => called from uvm_unmap_detach [only] ... note that entry is no 1558 * longer part of a map and thus has no need for locking 1559 * => amap must be unlocked (we will lock it). 1560 */ 1561 void 1562 amap_unref(struct vm_amap *amap, vaddr_t offset, vsize_t len, bool all) 1563 { 1564 UVMHIST_FUNC("amap_unref"); UVMHIST_CALLED(maphist); 1565 1566 /* 1567 * lock it 1568 */ 1569 amap_lock(amap); 1570 UVMHIST_LOG(maphist," amap=0x%x refs=%d, nused=%d", 1571 amap, amap->am_ref, amap->am_nused, 0); 1572 1573 KASSERT(amap_refs(amap) > 0); 1574 1575 /* 1576 * if we are the last reference, free the amap and return. 1577 */ 1578 1579 amap->am_ref--; 1580 1581 if (amap_refs(amap) == 0) { 1582 amap_wipeout(amap); /* drops final ref and frees */ 1583 UVMHIST_LOG(maphist,"<- done (was last ref)!", 0, 0, 0, 0); 1584 return; /* no need to unlock */ 1585 } 1586 1587 /* 1588 * otherwise just drop the reference count(s) 1589 */ 1590 1591 if (amap_refs(amap) == 1 && (amap->am_flags & AMAP_SHARED) != 0) 1592 amap->am_flags &= ~AMAP_SHARED; /* clear shared flag */ 1593 #ifdef UVM_AMAP_PPREF 1594 if (amap->am_ppref == NULL && all == 0 && len != amap->am_nslot) 1595 amap_pp_establish(amap, offset); 1596 if (amap->am_ppref && amap->am_ppref != PPREF_NONE) { 1597 if (all) 1598 amap_pp_adjref(amap, 0, amap->am_nslot, -1); 1599 else 1600 amap_pp_adjref(amap, offset, len, -1); 1601 } 1602 #endif 1603 amap_unlock(amap); 1604 1605 UVMHIST_LOG(maphist,"<- done!", 0, 0, 0, 0); 1606 } 1607 1608