1 /* $NetBSD: uvm_amap.c,v 1.101 2011/07/05 13:47:24 yamt Exp $ */ 2 3 /* 4 * Copyright (c) 1997 Charles D. Cranor and Washington University. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28 /* 29 * uvm_amap.c: amap operations 30 */ 31 32 /* 33 * this file contains functions that perform operations on amaps. see 34 * uvm_amap.h for a brief explanation of the role of amaps in uvm. 35 */ 36 37 #include <sys/cdefs.h> 38 __KERNEL_RCSID(0, "$NetBSD: uvm_amap.c,v 1.101 2011/07/05 13:47:24 yamt Exp $"); 39 40 #include "opt_uvmhist.h" 41 42 #include <sys/param.h> 43 #include <sys/systm.h> 44 #include <sys/kernel.h> 45 #include <sys/kmem.h> 46 #include <sys/pool.h> 47 #include <sys/atomic.h> 48 49 #include <uvm/uvm.h> 50 #include <uvm/uvm_swap.h> 51 52 /* 53 * cache for allocation of vm_map structures. note that in order to 54 * avoid an endless loop, the amap cache's allocator cannot allocate 55 * memory from an amap (it currently goes through the kernel uobj, so 56 * we are ok). 57 */ 58 static struct pool_cache uvm_amap_cache; 59 static kmutex_t amap_list_lock; 60 static LIST_HEAD(, vm_amap) amap_list; 61 62 /* 63 * local functions 64 */ 65 66 static inline void 67 amap_list_insert(struct vm_amap *amap) 68 { 69 70 mutex_enter(&amap_list_lock); 71 LIST_INSERT_HEAD(&amap_list, amap, am_list); 72 mutex_exit(&amap_list_lock); 73 } 74 75 static inline void 76 amap_list_remove(struct vm_amap *amap) 77 { 78 79 mutex_enter(&amap_list_lock); 80 LIST_REMOVE(amap, am_list); 81 mutex_exit(&amap_list_lock); 82 } 83 84 static int 85 amap_roundup_slots(int slots) 86 { 87 88 return kmem_roundup_size(slots * sizeof(int)) / sizeof(int); 89 } 90 91 #ifdef UVM_AMAP_PPREF 92 /* 93 * what is ppref? ppref is an _optional_ amap feature which is used 94 * to keep track of reference counts on a per-page basis. it is enabled 95 * when UVM_AMAP_PPREF is defined. 96 * 97 * when enabled, an array of ints is allocated for the pprefs. this 98 * array is allocated only when a partial reference is added to the 99 * map (either by unmapping part of the amap, or gaining a reference 100 * to only a part of an amap). if the allocation of the array fails 101 * (KM_NOSLEEP), then we set the array pointer to PPREF_NONE to indicate 102 * that we tried to do ppref's but couldn't alloc the array so just 103 * give up (after all, this is an optional feature!). 104 * 105 * the array is divided into page sized "chunks." for chunks of length 1, 106 * the chunk reference count plus one is stored in that chunk's slot. 107 * for chunks of length > 1 the first slot contains (the reference count 108 * plus one) * -1. [the negative value indicates that the length is 109 * greater than one.] the second slot of the chunk contains the length 110 * of the chunk. here is an example: 111 * 112 * actual REFS: 2 2 2 2 3 1 1 0 0 0 4 4 0 1 1 1 113 * ppref: -3 4 x x 4 -2 2 -1 3 x -5 2 1 -2 3 x 114 * <----------><-><----><-------><----><-><-------> 115 * (x = don't care) 116 * 117 * this allows us to allow one int to contain the ref count for the whole 118 * chunk. note that the "plus one" part is needed because a reference 119 * count of zero is neither positive or negative (need a way to tell 120 * if we've got one zero or a bunch of them). 121 * 122 * here are some in-line functions to help us. 123 */ 124 125 /* 126 * pp_getreflen: get the reference and length for a specific offset 127 * 128 * => ppref's amap must be locked 129 */ 130 static inline void 131 pp_getreflen(int *ppref, int offset, int *refp, int *lenp) 132 { 133 134 if (ppref[offset] > 0) { /* chunk size must be 1 */ 135 *refp = ppref[offset] - 1; /* don't forget to adjust */ 136 *lenp = 1; 137 } else { 138 *refp = (ppref[offset] * -1) - 1; 139 *lenp = ppref[offset+1]; 140 } 141 } 142 143 /* 144 * pp_setreflen: set the reference and length for a specific offset 145 * 146 * => ppref's amap must be locked 147 */ 148 static inline void 149 pp_setreflen(int *ppref, int offset, int ref, int len) 150 { 151 if (len == 0) 152 return; 153 if (len == 1) { 154 ppref[offset] = ref + 1; 155 } else { 156 ppref[offset] = (ref + 1) * -1; 157 ppref[offset+1] = len; 158 } 159 } 160 #endif /* UVM_AMAP_PPREF */ 161 162 /* 163 * amap_alloc1: allocate an amap, but do not initialise the overlay. 164 * 165 * => Note: lock is not set. 166 */ 167 static struct vm_amap * 168 amap_alloc1(int slots, int padslots, int flags) 169 { 170 const bool nowait = (flags & UVM_FLAG_NOWAIT) != 0; 171 const km_flag_t kmflags = nowait ? KM_NOSLEEP : KM_SLEEP; 172 struct vm_amap *amap; 173 int totalslots; 174 175 amap = pool_cache_get(&uvm_amap_cache, nowait ? PR_NOWAIT : PR_WAITOK); 176 if (amap == NULL) { 177 return NULL; 178 } 179 totalslots = amap_roundup_slots(slots + padslots); 180 amap->am_lock = NULL; 181 amap->am_ref = 1; 182 amap->am_flags = 0; 183 #ifdef UVM_AMAP_PPREF 184 amap->am_ppref = NULL; 185 #endif 186 amap->am_maxslot = totalslots; 187 amap->am_nslot = slots; 188 amap->am_nused = 0; 189 190 /* 191 * Note: since allocations are likely big, we expect to reduce the 192 * memory fragmentation by allocating them in separate blocks. 193 */ 194 amap->am_slots = kmem_alloc(totalslots * sizeof(int), kmflags); 195 if (amap->am_slots == NULL) 196 goto fail1; 197 198 amap->am_bckptr = kmem_alloc(totalslots * sizeof(int), kmflags); 199 if (amap->am_bckptr == NULL) 200 goto fail2; 201 202 amap->am_anon = kmem_alloc(totalslots * sizeof(struct vm_anon *), 203 kmflags); 204 if (amap->am_anon == NULL) 205 goto fail3; 206 207 return amap; 208 209 fail3: 210 kmem_free(amap->am_bckptr, totalslots * sizeof(int)); 211 fail2: 212 kmem_free(amap->am_slots, totalslots * sizeof(int)); 213 fail1: 214 pool_cache_put(&uvm_amap_cache, amap); 215 216 /* 217 * XXX hack to tell the pagedaemon how many pages we need, 218 * since we can need more than it would normally free. 219 */ 220 if (nowait) { 221 extern u_int uvm_extrapages; 222 atomic_add_int(&uvm_extrapages, 223 ((sizeof(int) * 2 + sizeof(struct vm_anon *)) * 224 totalslots) >> PAGE_SHIFT); 225 } 226 return NULL; 227 } 228 229 /* 230 * amap_alloc: allocate an amap to manage "sz" bytes of anonymous VM 231 * 232 * => caller should ensure sz is a multiple of PAGE_SIZE 233 * => reference count to new amap is set to one 234 * => new amap is returned unlocked 235 */ 236 237 struct vm_amap * 238 amap_alloc(vaddr_t sz, vaddr_t padsz, int waitf) 239 { 240 struct vm_amap *amap; 241 int slots, padslots; 242 UVMHIST_FUNC("amap_alloc"); UVMHIST_CALLED(maphist); 243 244 AMAP_B2SLOT(slots, sz); 245 AMAP_B2SLOT(padslots, padsz); 246 247 amap = amap_alloc1(slots, padslots, waitf); 248 if (amap) { 249 memset(amap->am_anon, 0, 250 amap->am_maxslot * sizeof(struct vm_anon *)); 251 amap->am_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NONE); 252 amap_list_insert(amap); 253 } 254 255 UVMHIST_LOG(maphist,"<- done, amap = 0x%x, sz=%d", amap, sz, 0, 0); 256 return(amap); 257 } 258 259 /* 260 * uvm_amap_init: initialize the amap system. 261 */ 262 void 263 uvm_amap_init(void) 264 { 265 266 mutex_init(&amap_list_lock, MUTEX_DEFAULT, IPL_NONE); 267 268 pool_cache_bootstrap(&uvm_amap_cache, sizeof(struct vm_amap), 0, 0, 0, 269 "amappl", NULL, IPL_NONE, NULL, NULL, NULL); 270 } 271 272 /* 273 * amap_free: free an amap 274 * 275 * => the amap must be unlocked 276 * => the amap should have a zero reference count and be empty 277 */ 278 void 279 amap_free(struct vm_amap *amap) 280 { 281 int slots; 282 283 UVMHIST_FUNC("amap_free"); UVMHIST_CALLED(maphist); 284 285 KASSERT(amap->am_ref == 0 && amap->am_nused == 0); 286 KASSERT((amap->am_flags & AMAP_SWAPOFF) == 0); 287 if (amap->am_lock != NULL) { 288 KASSERT(!mutex_owned(amap->am_lock)); 289 mutex_obj_free(amap->am_lock); 290 } 291 slots = amap->am_maxslot; 292 kmem_free(amap->am_slots, slots * sizeof(*amap->am_slots)); 293 kmem_free(amap->am_bckptr, slots * sizeof(*amap->am_bckptr)); 294 kmem_free(amap->am_anon, slots * sizeof(*amap->am_anon)); 295 #ifdef UVM_AMAP_PPREF 296 if (amap->am_ppref && amap->am_ppref != PPREF_NONE) 297 kmem_free(amap->am_ppref, slots * sizeof(*amap->am_ppref)); 298 #endif 299 pool_cache_put(&uvm_amap_cache, amap); 300 UVMHIST_LOG(maphist,"<- done, freed amap = 0x%x", amap, 0, 0, 0); 301 } 302 303 /* 304 * amap_extend: extend the size of an amap (if needed) 305 * 306 * => called from uvm_map when we want to extend an amap to cover 307 * a new mapping (rather than allocate a new one) 308 * => amap should be unlocked (we will lock it) 309 * => to safely extend an amap it should have a reference count of 310 * one (thus it can't be shared) 311 */ 312 int 313 amap_extend(struct vm_map_entry *entry, vsize_t addsize, int flags) 314 { 315 struct vm_amap *amap = entry->aref.ar_amap; 316 int slotoff = entry->aref.ar_pageoff; 317 int slotmapped, slotadd, slotneed, slotadded, slotalloc; 318 int slotadj, slotspace; 319 int oldnslots; 320 #ifdef UVM_AMAP_PPREF 321 int *newppref, *oldppref; 322 #endif 323 int i, *newsl, *newbck, *oldsl, *oldbck; 324 struct vm_anon **newover, **oldover, *tofree; 325 const km_flag_t kmflags = 326 (flags & AMAP_EXTEND_NOWAIT) ? KM_NOSLEEP : KM_SLEEP; 327 328 UVMHIST_FUNC("amap_extend"); UVMHIST_CALLED(maphist); 329 330 UVMHIST_LOG(maphist, " (entry=0x%x, addsize=0x%x, flags=0x%x)", 331 entry, addsize, flags, 0); 332 333 /* 334 * first, determine how many slots we need in the amap. don't 335 * forget that ar_pageoff could be non-zero: this means that 336 * there are some unused slots before us in the amap. 337 */ 338 339 amap_lock(amap); 340 KASSERT(amap_refs(amap) == 1); /* amap can't be shared */ 341 AMAP_B2SLOT(slotmapped, entry->end - entry->start); /* slots mapped */ 342 AMAP_B2SLOT(slotadd, addsize); /* slots to add */ 343 if (flags & AMAP_EXTEND_FORWARDS) { 344 slotneed = slotoff + slotmapped + slotadd; 345 slotadj = 0; 346 slotspace = 0; 347 } 348 else { 349 slotneed = slotadd + slotmapped; 350 slotadj = slotadd - slotoff; 351 slotspace = amap->am_maxslot - slotmapped; 352 } 353 tofree = NULL; 354 355 /* 356 * case 1: we already have enough slots in the map and thus 357 * only need to bump the reference counts on the slots we are 358 * adding. 359 */ 360 361 if (flags & AMAP_EXTEND_FORWARDS) { 362 if (amap->am_nslot >= slotneed) { 363 #ifdef UVM_AMAP_PPREF 364 if (amap->am_ppref && amap->am_ppref != PPREF_NONE) { 365 amap_pp_adjref(amap, slotoff + slotmapped, 366 slotadd, 1, &tofree); 367 } 368 uvm_anfree(tofree); 369 #endif 370 amap_unlock(amap); 371 UVMHIST_LOG(maphist, 372 "<- done (case 1f), amap = 0x%x, sltneed=%d", 373 amap, slotneed, 0, 0); 374 return 0; 375 } 376 } else { 377 if (slotadj <= 0) { 378 slotoff -= slotadd; 379 entry->aref.ar_pageoff = slotoff; 380 #ifdef UVM_AMAP_PPREF 381 if (amap->am_ppref && amap->am_ppref != PPREF_NONE) { 382 amap_pp_adjref(amap, slotoff, slotadd, 1, 383 &tofree); 384 } 385 uvm_anfree(tofree); 386 #endif 387 amap_unlock(amap); 388 UVMHIST_LOG(maphist, 389 "<- done (case 1b), amap = 0x%x, sltneed=%d", 390 amap, slotneed, 0, 0); 391 return 0; 392 } 393 } 394 395 /* 396 * case 2: we pre-allocated slots for use and we just need to 397 * bump nslot up to take account for these slots. 398 */ 399 400 if (amap->am_maxslot >= slotneed) { 401 if (flags & AMAP_EXTEND_FORWARDS) { 402 #ifdef UVM_AMAP_PPREF 403 if (amap->am_ppref && amap->am_ppref != PPREF_NONE) { 404 if ((slotoff + slotmapped) < amap->am_nslot) 405 amap_pp_adjref(amap, 406 slotoff + slotmapped, 407 (amap->am_nslot - 408 (slotoff + slotmapped)), 1, 409 &tofree); 410 pp_setreflen(amap->am_ppref, amap->am_nslot, 1, 411 slotneed - amap->am_nslot); 412 } 413 #endif 414 amap->am_nslot = slotneed; 415 uvm_anfree(tofree); 416 amap_unlock(amap); 417 418 /* 419 * no need to zero am_anon since that was done at 420 * alloc time and we never shrink an allocation. 421 */ 422 423 UVMHIST_LOG(maphist,"<- done (case 2f), amap = 0x%x, " 424 "slotneed=%d", amap, slotneed, 0, 0); 425 return 0; 426 } else { 427 #ifdef UVM_AMAP_PPREF 428 if (amap->am_ppref && amap->am_ppref != PPREF_NONE) { 429 /* 430 * Slide up the ref counts on the pages that 431 * are actually in use. 432 */ 433 memmove(amap->am_ppref + slotspace, 434 amap->am_ppref + slotoff, 435 slotmapped * sizeof(int)); 436 /* 437 * Mark the (adjusted) gap at the front as 438 * referenced/not referenced. 439 */ 440 pp_setreflen(amap->am_ppref, 441 0, 0, slotspace - slotadd); 442 pp_setreflen(amap->am_ppref, 443 slotspace - slotadd, 1, slotadd); 444 } 445 #endif 446 447 /* 448 * Slide the anon pointers up and clear out 449 * the space we just made. 450 */ 451 memmove(amap->am_anon + slotspace, 452 amap->am_anon + slotoff, 453 slotmapped * sizeof(struct vm_anon*)); 454 memset(amap->am_anon + slotoff, 0, 455 (slotspace - slotoff) * sizeof(struct vm_anon *)); 456 457 /* 458 * Slide the backpointers up, but don't bother 459 * wiping out the old slots. 460 */ 461 memmove(amap->am_bckptr + slotspace, 462 amap->am_bckptr + slotoff, 463 slotmapped * sizeof(int)); 464 465 /* 466 * Adjust all the useful active slot numbers. 467 */ 468 for (i = 0; i < amap->am_nused; i++) 469 amap->am_slots[i] += (slotspace - slotoff); 470 471 /* 472 * We just filled all the empty space in the 473 * front of the amap by activating a few new 474 * slots. 475 */ 476 amap->am_nslot = amap->am_maxslot; 477 entry->aref.ar_pageoff = slotspace - slotadd; 478 amap_unlock(amap); 479 480 UVMHIST_LOG(maphist,"<- done (case 2b), amap = 0x%x, " 481 "slotneed=%d", amap, slotneed, 0, 0); 482 return 0; 483 } 484 } 485 486 /* 487 * Case 3: we need to allocate a new amap and copy all the amap 488 * data over from old amap to the new one. Drop the lock before 489 * performing allocation. 490 * 491 * Note: since allocations are likely big, we expect to reduce the 492 * memory fragmentation by allocating them in separate blocks. 493 */ 494 495 amap_unlock(amap); 496 497 if (slotneed >= UVM_AMAP_LARGE) { 498 return E2BIG; 499 } 500 501 slotalloc = amap_roundup_slots(slotneed); 502 #ifdef UVM_AMAP_PPREF 503 newppref = NULL; 504 if (amap->am_ppref && amap->am_ppref != PPREF_NONE) { 505 /* Will be handled later if fails. */ 506 newppref = kmem_alloc(slotalloc * sizeof(*newppref), kmflags); 507 } 508 #endif 509 newsl = kmem_alloc(slotalloc * sizeof(*newsl), kmflags); 510 newbck = kmem_alloc(slotalloc * sizeof(*newbck), kmflags); 511 newover = kmem_alloc(slotalloc * sizeof(*newover), kmflags); 512 if (newsl == NULL || newbck == NULL || newover == NULL) { 513 #ifdef UVM_AMAP_PPREF 514 if (newppref != NULL) { 515 kmem_free(newppref, slotalloc * sizeof(*newppref)); 516 } 517 #endif 518 if (newsl != NULL) { 519 kmem_free(newsl, slotalloc * sizeof(*newsl)); 520 } 521 if (newbck != NULL) { 522 kmem_free(newbck, slotalloc * sizeof(*newbck)); 523 } 524 if (newover != NULL) { 525 kmem_free(newover, slotalloc * sizeof(*newover)); 526 } 527 return ENOMEM; 528 } 529 amap_lock(amap); 530 KASSERT(amap->am_maxslot < slotneed); 531 532 /* 533 * Copy everything over to new allocated areas. 534 */ 535 536 slotadded = slotalloc - amap->am_nslot; 537 if (!(flags & AMAP_EXTEND_FORWARDS)) 538 slotspace = slotalloc - slotmapped; 539 540 /* do am_slots */ 541 oldsl = amap->am_slots; 542 if (flags & AMAP_EXTEND_FORWARDS) 543 memcpy(newsl, oldsl, sizeof(int) * amap->am_nused); 544 else 545 for (i = 0; i < amap->am_nused; i++) 546 newsl[i] = oldsl[i] + slotspace - slotoff; 547 amap->am_slots = newsl; 548 549 /* do am_anon */ 550 oldover = amap->am_anon; 551 if (flags & AMAP_EXTEND_FORWARDS) { 552 memcpy(newover, oldover, 553 sizeof(struct vm_anon *) * amap->am_nslot); 554 memset(newover + amap->am_nslot, 0, 555 sizeof(struct vm_anon *) * slotadded); 556 } else { 557 memcpy(newover + slotspace, oldover + slotoff, 558 sizeof(struct vm_anon *) * slotmapped); 559 memset(newover, 0, 560 sizeof(struct vm_anon *) * slotspace); 561 } 562 amap->am_anon = newover; 563 564 /* do am_bckptr */ 565 oldbck = amap->am_bckptr; 566 if (flags & AMAP_EXTEND_FORWARDS) 567 memcpy(newbck, oldbck, sizeof(int) * amap->am_nslot); 568 else 569 memcpy(newbck + slotspace, oldbck + slotoff, 570 sizeof(int) * slotmapped); 571 amap->am_bckptr = newbck; 572 573 #ifdef UVM_AMAP_PPREF 574 /* do ppref */ 575 oldppref = amap->am_ppref; 576 if (newppref) { 577 if (flags & AMAP_EXTEND_FORWARDS) { 578 memcpy(newppref, oldppref, 579 sizeof(int) * amap->am_nslot); 580 memset(newppref + amap->am_nslot, 0, 581 sizeof(int) * slotadded); 582 } else { 583 memcpy(newppref + slotspace, oldppref + slotoff, 584 sizeof(int) * slotmapped); 585 } 586 amap->am_ppref = newppref; 587 if ((flags & AMAP_EXTEND_FORWARDS) && 588 (slotoff + slotmapped) < amap->am_nslot) 589 amap_pp_adjref(amap, slotoff + slotmapped, 590 (amap->am_nslot - (slotoff + slotmapped)), 1, 591 &tofree); 592 if (flags & AMAP_EXTEND_FORWARDS) 593 pp_setreflen(newppref, amap->am_nslot, 1, 594 slotneed - amap->am_nslot); 595 else { 596 pp_setreflen(newppref, 0, 0, 597 slotalloc - slotneed); 598 pp_setreflen(newppref, slotalloc - slotneed, 1, 599 slotneed - slotmapped); 600 } 601 } else { 602 if (amap->am_ppref) 603 amap->am_ppref = PPREF_NONE; 604 } 605 #endif 606 607 /* update master values */ 608 if (flags & AMAP_EXTEND_FORWARDS) 609 amap->am_nslot = slotneed; 610 else { 611 entry->aref.ar_pageoff = slotspace - slotadd; 612 amap->am_nslot = slotalloc; 613 } 614 oldnslots = amap->am_maxslot; 615 amap->am_maxslot = slotalloc; 616 617 uvm_anfree(tofree); 618 amap_unlock(amap); 619 kmem_free(oldsl, oldnslots * sizeof(*oldsl)); 620 kmem_free(oldbck, oldnslots * sizeof(*oldbck)); 621 kmem_free(oldover, oldnslots * sizeof(*oldover)); 622 #ifdef UVM_AMAP_PPREF 623 if (oldppref && oldppref != PPREF_NONE) 624 kmem_free(oldppref, oldnslots * sizeof(*oldppref)); 625 #endif 626 UVMHIST_LOG(maphist,"<- done (case 3), amap = 0x%x, slotneed=%d", 627 amap, slotneed, 0, 0); 628 return 0; 629 } 630 631 /* 632 * amap_share_protect: change protection of anons in a shared amap 633 * 634 * for shared amaps, given the current data structure layout, it is 635 * not possible for us to directly locate all maps referencing the 636 * shared anon (to change the protection). in order to protect data 637 * in shared maps we use pmap_page_protect(). [this is useful for IPC 638 * mechanisms like map entry passing that may want to write-protect 639 * all mappings of a shared amap.] we traverse am_anon or am_slots 640 * depending on the current state of the amap. 641 * 642 * => entry's map and amap must be locked by the caller 643 */ 644 void 645 amap_share_protect(struct vm_map_entry *entry, vm_prot_t prot) 646 { 647 struct vm_amap *amap = entry->aref.ar_amap; 648 u_int slots, lcv, slot, stop; 649 struct vm_anon *anon; 650 651 KASSERT(mutex_owned(amap->am_lock)); 652 653 AMAP_B2SLOT(slots, (entry->end - entry->start)); 654 stop = entry->aref.ar_pageoff + slots; 655 656 if (slots < amap->am_nused) { 657 /* 658 * Cheaper to traverse am_anon. 659 */ 660 for (lcv = entry->aref.ar_pageoff ; lcv < stop ; lcv++) { 661 anon = amap->am_anon[lcv]; 662 if (anon == NULL) { 663 continue; 664 } 665 if (anon->an_page) { 666 pmap_page_protect(anon->an_page, prot); 667 } 668 } 669 return; 670 } 671 672 /* 673 * Cheaper to traverse am_slots. 674 */ 675 for (lcv = 0 ; lcv < amap->am_nused ; lcv++) { 676 slot = amap->am_slots[lcv]; 677 if (slot < entry->aref.ar_pageoff || slot >= stop) { 678 continue; 679 } 680 anon = amap->am_anon[slot]; 681 if (anon->an_page) { 682 pmap_page_protect(anon->an_page, prot); 683 } 684 } 685 } 686 687 /* 688 * amap_wipeout: wipeout all anon's in an amap; then free the amap! 689 * 690 * => Called from amap_unref(), when reference count drops to zero. 691 * => amap must be locked. 692 */ 693 694 void 695 amap_wipeout(struct vm_amap *amap) 696 { 697 u_int lcv; 698 699 UVMHIST_FUNC("amap_wipeout"); UVMHIST_CALLED(maphist); 700 UVMHIST_LOG(maphist,"(amap=0x%x)", amap, 0,0,0); 701 702 KASSERT(mutex_owned(amap->am_lock)); 703 KASSERT(amap->am_ref == 0); 704 705 if (__predict_false(amap->am_flags & AMAP_SWAPOFF)) { 706 /* 707 * Note: amap_swap_off() will call us again. 708 */ 709 amap_unlock(amap); 710 return; 711 } 712 amap_list_remove(amap); 713 714 for (lcv = 0 ; lcv < amap->am_nused ; lcv++) { 715 struct vm_anon *anon; 716 u_int slot; 717 718 slot = amap->am_slots[lcv]; 719 anon = amap->am_anon[slot]; 720 KASSERT(anon != NULL && anon->an_ref != 0); 721 722 KASSERT(anon->an_lock == amap->am_lock); 723 UVMHIST_LOG(maphist," processing anon 0x%x, ref=%d", anon, 724 anon->an_ref, 0, 0); 725 726 /* 727 * Drop the reference, and free the anon, if it is last. 728 */ 729 730 if (--anon->an_ref == 0) { 731 uvm_anfree(anon); 732 } 733 if (curlwp->l_cpu->ci_schedstate.spc_flags & SPCF_SHOULDYIELD) { 734 preempt(); 735 } 736 } 737 738 /* 739 * Finally, destroy the amap. 740 */ 741 742 amap->am_nused = 0; 743 amap_unlock(amap); 744 amap_free(amap); 745 UVMHIST_LOG(maphist,"<- done!", 0,0,0,0); 746 } 747 748 /* 749 * amap_copy: ensure that a map entry's "needs_copy" flag is false 750 * by copying the amap if necessary. 751 * 752 * => an entry with a null amap pointer will get a new (blank) one. 753 * => the map that the map entry belongs to must be locked by caller. 754 * => the amap currently attached to "entry" (if any) must be unlocked. 755 * => if canchunk is true, then we may clip the entry into a chunk 756 * => "startva" and "endva" are used only if canchunk is true. they are 757 * used to limit chunking (e.g. if you have a large space that you 758 * know you are going to need to allocate amaps for, there is no point 759 * in allowing that to be chunked) 760 */ 761 762 void 763 amap_copy(struct vm_map *map, struct vm_map_entry *entry, int flags, 764 vaddr_t startva, vaddr_t endva) 765 { 766 const int waitf = (flags & AMAP_COPY_NOWAIT) ? UVM_FLAG_NOWAIT : 0; 767 struct vm_amap *amap, *srcamap; 768 struct vm_anon *tofree; 769 u_int slots, lcv; 770 vsize_t len; 771 772 UVMHIST_FUNC("amap_copy"); UVMHIST_CALLED(maphist); 773 UVMHIST_LOG(maphist, " (map=%p, entry=%p, flags=%d)", 774 map, entry, flags, 0); 775 776 KASSERT(map != kernel_map); /* we use nointr pool */ 777 778 srcamap = entry->aref.ar_amap; 779 len = entry->end - entry->start; 780 781 /* 782 * Is there an amap to copy? If not, create one. 783 */ 784 785 if (srcamap == NULL) { 786 const bool canchunk = (flags & AMAP_COPY_NOCHUNK) == 0; 787 788 /* 789 * Check to see if we have a large amap that we can 790 * chunk. We align startva/endva to chunk-sized 791 * boundaries and then clip to them. 792 */ 793 794 if (canchunk && atop(len) >= UVM_AMAP_LARGE) { 795 vsize_t chunksize; 796 797 /* Convert slots to bytes. */ 798 chunksize = UVM_AMAP_CHUNK << PAGE_SHIFT; 799 startva = (startva / chunksize) * chunksize; 800 endva = roundup(endva, chunksize); 801 UVMHIST_LOG(maphist, " chunk amap ==> clip 0x%x->0x%x" 802 "to 0x%x->0x%x", entry->start, entry->end, startva, 803 endva); 804 UVM_MAP_CLIP_START(map, entry, startva, NULL); 805 806 /* Watch out for endva wrap-around! */ 807 if (endva >= startva) { 808 UVM_MAP_CLIP_END(map, entry, endva, NULL); 809 } 810 } 811 812 if ((flags & AMAP_COPY_NOMERGE) == 0 && 813 uvm_mapent_trymerge(map, entry, UVM_MERGE_COPYING)) { 814 return; 815 } 816 817 UVMHIST_LOG(maphist, "<- done [creating new amap 0x%x->0x%x]", 818 entry->start, entry->end, 0, 0); 819 820 /* 821 * Allocate an initialised amap and install it. 822 * Note: we must update the length after clipping. 823 */ 824 len = entry->end - entry->start; 825 entry->aref.ar_pageoff = 0; 826 entry->aref.ar_amap = amap_alloc(len, 0, waitf); 827 if (entry->aref.ar_amap != NULL) { 828 entry->etype &= ~UVM_ET_NEEDSCOPY; 829 } 830 return; 831 } 832 833 /* 834 * First check and see if we are the only map entry referencing 835 * he amap we currently have. If so, then just take it over instead 836 * of copying it. Note that we are reading am_ref without lock held 837 * as the value value can only be one if we have the only reference 838 * to the amap (via our locked map). If the value is greater than 839 * one, then allocate amap and re-check the value. 840 */ 841 842 if (srcamap->am_ref == 1) { 843 entry->etype &= ~UVM_ET_NEEDSCOPY; 844 UVMHIST_LOG(maphist, "<- done [ref cnt = 1, took it over]", 845 0, 0, 0, 0); 846 return; 847 } 848 849 UVMHIST_LOG(maphist," amap=%p, ref=%d, must copy it", 850 srcamap, srcamap->am_ref, 0, 0); 851 852 /* 853 * Allocate a new amap (note: not initialised, no lock set, etc). 854 */ 855 856 AMAP_B2SLOT(slots, len); 857 amap = amap_alloc1(slots, 0, waitf); 858 if (amap == NULL) { 859 UVMHIST_LOG(maphist, " amap_alloc1 failed", 0,0,0,0); 860 return; 861 } 862 863 amap_lock(srcamap); 864 865 /* 866 * Re-check the reference count with the lock held. If it has 867 * dropped to one - we can take over the existing map. 868 */ 869 870 if (srcamap->am_ref == 1) { 871 /* Just take over the existing amap. */ 872 entry->etype &= ~UVM_ET_NEEDSCOPY; 873 amap_unlock(srcamap); 874 /* Destroy the new (unused) amap. */ 875 amap->am_ref--; 876 amap_free(amap); 877 return; 878 } 879 880 /* 881 * Copy the slots. Zero the padded part. 882 */ 883 884 UVMHIST_LOG(maphist, " copying amap now",0, 0, 0, 0); 885 for (lcv = 0 ; lcv < slots; lcv++) { 886 amap->am_anon[lcv] = 887 srcamap->am_anon[entry->aref.ar_pageoff + lcv]; 888 if (amap->am_anon[lcv] == NULL) 889 continue; 890 KASSERT(amap->am_anon[lcv]->an_lock == srcamap->am_lock); 891 amap->am_anon[lcv]->an_ref++; 892 amap->am_bckptr[lcv] = amap->am_nused; 893 amap->am_slots[amap->am_nused] = lcv; 894 amap->am_nused++; 895 } 896 memset(&amap->am_anon[lcv], 0, 897 (amap->am_maxslot - lcv) * sizeof(struct vm_anon *)); 898 899 /* 900 * Drop our reference to the old amap (srcamap) and unlock. 901 * Since the reference count on srcamap is greater than one, 902 * (we checked above), it cannot drop to zero while it is locked. 903 */ 904 905 srcamap->am_ref--; 906 KASSERT(srcamap->am_ref > 0); 907 908 if (srcamap->am_ref == 1 && (srcamap->am_flags & AMAP_SHARED) != 0) { 909 srcamap->am_flags &= ~AMAP_SHARED; 910 } 911 tofree = NULL; 912 #ifdef UVM_AMAP_PPREF 913 if (srcamap->am_ppref && srcamap->am_ppref != PPREF_NONE) { 914 amap_pp_adjref(srcamap, entry->aref.ar_pageoff, 915 len >> PAGE_SHIFT, -1, &tofree); 916 } 917 #endif 918 uvm_anfree(tofree); 919 920 /* 921 * If we referenced any anons, then share the source amap's lock. 922 * Otherwise, we have nothing in common, so allocate a new one. 923 */ 924 925 KASSERT(amap->am_lock == NULL); 926 if (amap->am_nused != 0) { 927 amap->am_lock = srcamap->am_lock; 928 mutex_obj_hold(amap->am_lock); 929 } 930 amap_unlock(srcamap); 931 if (amap->am_lock == NULL) 932 amap->am_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NONE); 933 amap_list_insert(amap); 934 935 /* 936 * Install new amap. 937 */ 938 939 entry->aref.ar_pageoff = 0; 940 entry->aref.ar_amap = amap; 941 entry->etype &= ~UVM_ET_NEEDSCOPY; 942 UVMHIST_LOG(maphist, "<- done",0, 0, 0, 0); 943 } 944 945 /* 946 * amap_cow_now: resolve all copy-on-write faults in an amap now for fork(2) 947 * 948 * called during fork(2) when the parent process has a wired map 949 * entry. in that case we want to avoid write-protecting pages 950 * in the parent's map (e.g. like what you'd do for a COW page) 951 * so we resolve the COW here. 952 * 953 * => assume parent's entry was wired, thus all pages are resident. 954 * => assume pages that are loaned out (loan_count) are already mapped 955 * read-only in all maps, and thus no need for us to worry about them 956 * => assume both parent and child vm_map's are locked 957 * => caller passes child's map/entry in to us 958 * => if we run out of memory we will unlock the amap and sleep _with_ the 959 * parent and child vm_map's locked(!). we have to do this since 960 * we are in the middle of a fork(2) and we can't let the parent 961 * map change until we are done copying all the map entrys. 962 * => XXXCDC: out of memory should cause fork to fail, but there is 963 * currently no easy way to do this (needs fix) 964 * => page queues must be unlocked (we may lock them) 965 */ 966 967 void 968 amap_cow_now(struct vm_map *map, struct vm_map_entry *entry) 969 { 970 struct vm_amap *amap = entry->aref.ar_amap; 971 struct vm_anon *anon, *nanon; 972 struct vm_page *pg, *npg; 973 u_int lcv, slot; 974 975 /* 976 * note that if we unlock the amap then we must ReStart the "lcv" for 977 * loop because some other process could reorder the anon's in the 978 * am_anon[] array on us while the lock is dropped. 979 */ 980 981 ReStart: 982 amap_lock(amap); 983 for (lcv = 0 ; lcv < amap->am_nused ; lcv++) { 984 slot = amap->am_slots[lcv]; 985 anon = amap->am_anon[slot]; 986 KASSERT(anon->an_lock == amap->am_lock); 987 988 /* 989 * If anon has only one reference - we must have already 990 * copied it. This can happen if we needed to sleep waiting 991 * for memory in a previous run through this loop. The new 992 * page might even have been paged out, since is not wired. 993 */ 994 995 if (anon->an_ref == 1) { 996 KASSERT(anon->an_page != NULL || anon->an_swslot != 0); 997 continue; 998 } 999 1000 /* 1001 * The old page must be resident since the parent is wired. 1002 */ 1003 1004 pg = anon->an_page; 1005 KASSERT(pg != NULL); 1006 KASSERT(pg->wire_count > 0); 1007 1008 /* 1009 * If the page is loaned then it must already be mapped 1010 * read-only and we don't need to copy it. 1011 */ 1012 1013 if (pg->loan_count != 0) { 1014 continue; 1015 } 1016 KASSERT(pg->uanon == anon && pg->uobject == NULL); 1017 1018 /* 1019 * If the page is busy, then we have to unlock, wait for 1020 * it and then restart. 1021 */ 1022 1023 if (pg->flags & PG_BUSY) { 1024 pg->flags |= PG_WANTED; 1025 UVM_UNLOCK_AND_WAIT(pg, amap->am_lock, false, 1026 "cownow", 0); 1027 goto ReStart; 1028 } 1029 1030 /* 1031 * Perform a copy-on-write. 1032 * First - get a new anon and a page. 1033 */ 1034 1035 nanon = uvm_analloc(); 1036 if (nanon) { 1037 npg = uvm_pagealloc(NULL, 0, nanon, 0); 1038 } else { 1039 npg = NULL; 1040 } 1041 if (nanon == NULL || npg == NULL) { 1042 1043 /* 1044 * XXXCDC: we should cause fork to fail, but we can't. 1045 */ 1046 1047 if (nanon) { 1048 nanon->an_ref--; 1049 KASSERT(nanon->an_ref == 0); 1050 uvm_anfree(nanon); 1051 } 1052 amap_unlock(amap); 1053 uvm_wait("cownowpage"); 1054 goto ReStart; 1055 } 1056 1057 /* 1058 * Copy the data and replace anon with the new one. 1059 * Also, setup its lock (share the with amap's lock). 1060 */ 1061 1062 nanon->an_lock = amap->am_lock; 1063 uvm_pagecopy(pg, npg); 1064 anon->an_ref--; 1065 KASSERT(anon->an_ref > 0); 1066 amap->am_anon[slot] = nanon; 1067 1068 /* 1069 * Drop PG_BUSY on new page. Since its owner was locked all 1070 * this time - it cannot be PG_RELEASED or PG_WANTED. 1071 */ 1072 1073 mutex_enter(&uvm_pageqlock); 1074 uvm_pageactivate(npg); 1075 mutex_exit(&uvm_pageqlock); 1076 npg->flags &= ~(PG_BUSY|PG_FAKE); 1077 UVM_PAGE_OWN(npg, NULL); 1078 } 1079 amap_unlock(amap); 1080 } 1081 1082 /* 1083 * amap_splitref: split a single reference into two separate references 1084 * 1085 * => called from uvm_map's clip routines 1086 * => origref's map should be locked 1087 * => origref->ar_amap should be unlocked (we will lock) 1088 */ 1089 void 1090 amap_splitref(struct vm_aref *origref, struct vm_aref *splitref, vaddr_t offset) 1091 { 1092 struct vm_amap *amap = origref->ar_amap; 1093 u_int leftslots; 1094 1095 KASSERT(splitref->ar_amap == origref->ar_amap); 1096 AMAP_B2SLOT(leftslots, offset); 1097 KASSERT(leftslots != 0); 1098 1099 amap_lock(amap); 1100 KASSERT(amap->am_nslot - origref->ar_pageoff - leftslots > 0); 1101 1102 #ifdef UVM_AMAP_PPREF 1103 /* Establish ppref before we add a duplicate reference to the amap. */ 1104 if (amap->am_ppref == NULL) { 1105 amap_pp_establish(amap, origref->ar_pageoff); 1106 } 1107 #endif 1108 /* Note: not a share reference. */ 1109 amap->am_ref++; 1110 splitref->ar_pageoff = origref->ar_pageoff + leftslots; 1111 amap_unlock(amap); 1112 } 1113 1114 #ifdef UVM_AMAP_PPREF 1115 1116 /* 1117 * amap_pp_establish: add a ppref array to an amap, if possible. 1118 * 1119 * => amap should be locked by caller. 1120 */ 1121 void 1122 amap_pp_establish(struct vm_amap *amap, vaddr_t offset) 1123 { 1124 const size_t sz = amap->am_maxslot * sizeof(*amap->am_ppref); 1125 1126 KASSERT(mutex_owned(amap->am_lock)); 1127 1128 amap->am_ppref = kmem_zalloc(sz, KM_NOSLEEP); 1129 if (amap->am_ppref == NULL) { 1130 /* Failure - just do not use ppref. */ 1131 amap->am_ppref = PPREF_NONE; 1132 return; 1133 } 1134 pp_setreflen(amap->am_ppref, 0, 0, offset); 1135 pp_setreflen(amap->am_ppref, offset, amap->am_ref, 1136 amap->am_nslot - offset); 1137 } 1138 1139 /* 1140 * amap_pp_adjref: adjust reference count to a part of an amap using the 1141 * per-page reference count array. 1142 * 1143 * => caller must check that ppref != PPREF_NONE before calling. 1144 * => map and amap must be locked. 1145 */ 1146 void 1147 amap_pp_adjref(struct vm_amap *amap, int curslot, vsize_t slotlen, int adjval, 1148 struct vm_anon **tofree) 1149 { 1150 int stopslot, *ppref, lcv, prevlcv; 1151 int ref, len, prevref, prevlen; 1152 1153 KASSERT(mutex_owned(amap->am_lock)); 1154 1155 stopslot = curslot + slotlen; 1156 ppref = amap->am_ppref; 1157 prevlcv = 0; 1158 1159 /* 1160 * Advance to the correct place in the array, fragment if needed. 1161 */ 1162 1163 for (lcv = 0 ; lcv < curslot ; lcv += len) { 1164 pp_getreflen(ppref, lcv, &ref, &len); 1165 if (lcv + len > curslot) { /* goes past start? */ 1166 pp_setreflen(ppref, lcv, ref, curslot - lcv); 1167 pp_setreflen(ppref, curslot, ref, len - (curslot -lcv)); 1168 len = curslot - lcv; /* new length of entry @ lcv */ 1169 } 1170 prevlcv = lcv; 1171 } 1172 if (lcv == 0) { 1173 /* 1174 * Ensure that the "prevref == ref" test below always 1175 * fails, since we are starting from the beginning of 1176 * the ppref array; that is, there is no previous chunk. 1177 */ 1178 prevref = -1; 1179 prevlen = 0; 1180 } else { 1181 pp_getreflen(ppref, prevlcv, &prevref, &prevlen); 1182 } 1183 1184 /* 1185 * Now adjust reference counts in range. Merge the first 1186 * changed entry with the last unchanged entry if possible. 1187 */ 1188 KASSERT(lcv == curslot); 1189 for (/* lcv already set */; lcv < stopslot ; lcv += len) { 1190 pp_getreflen(ppref, lcv, &ref, &len); 1191 if (lcv + len > stopslot) { /* goes past end? */ 1192 pp_setreflen(ppref, lcv, ref, stopslot - lcv); 1193 pp_setreflen(ppref, stopslot, ref, 1194 len - (stopslot - lcv)); 1195 len = stopslot - lcv; 1196 } 1197 ref += adjval; 1198 KASSERT(ref >= 0); 1199 if (lcv == prevlcv + prevlen && ref == prevref) { 1200 pp_setreflen(ppref, prevlcv, ref, prevlen + len); 1201 } else { 1202 pp_setreflen(ppref, lcv, ref, len); 1203 } 1204 if (ref == 0) { 1205 amap_wiperange(amap, lcv, len, tofree); 1206 } 1207 } 1208 } 1209 1210 /* 1211 * amap_wiperange: wipe out a range of an amap. 1212 * Note: different from amap_wipeout because the amap is kept intact. 1213 * 1214 * => Both map and amap must be locked by caller. 1215 */ 1216 void 1217 amap_wiperange(struct vm_amap *amap, int slotoff, int slots, 1218 struct vm_anon **tofree) 1219 { 1220 u_int lcv, stop, slotend; 1221 bool byanon; 1222 1223 KASSERT(mutex_owned(amap->am_lock)); 1224 1225 /* 1226 * We can either traverse the amap by am_anon or by am_slots. 1227 * Determine which way is less expensive. 1228 */ 1229 1230 if (slots < amap->am_nused) { 1231 byanon = true; 1232 lcv = slotoff; 1233 stop = slotoff + slots; 1234 slotend = 0; 1235 } else { 1236 byanon = false; 1237 lcv = 0; 1238 stop = amap->am_nused; 1239 slotend = slotoff + slots; 1240 } 1241 1242 while (lcv < stop) { 1243 struct vm_anon *anon; 1244 u_int curslot, ptr, last; 1245 1246 if (byanon) { 1247 curslot = lcv++; /* lcv advances here */ 1248 if (amap->am_anon[curslot] == NULL) 1249 continue; 1250 } else { 1251 curslot = amap->am_slots[lcv]; 1252 if (curslot < slotoff || curslot >= slotend) { 1253 lcv++; /* lcv advances here */ 1254 continue; 1255 } 1256 stop--; /* drop stop, since anon will be removed */ 1257 } 1258 anon = amap->am_anon[curslot]; 1259 KASSERT(anon->an_lock == amap->am_lock); 1260 1261 /* 1262 * Remove anon from the amap. 1263 */ 1264 1265 amap->am_anon[curslot] = NULL; 1266 ptr = amap->am_bckptr[curslot]; 1267 last = amap->am_nused - 1; 1268 if (ptr != last) { 1269 amap->am_slots[ptr] = amap->am_slots[last]; 1270 amap->am_bckptr[amap->am_slots[ptr]] = ptr; 1271 } 1272 amap->am_nused--; 1273 1274 /* 1275 * Drop its reference count. 1276 */ 1277 1278 KASSERT(anon->an_lock == amap->am_lock); 1279 if (--anon->an_ref == 0) { 1280 /* 1281 * Eliminated the last reference to an anon - defer 1282 * freeing as uvm_anfree() can unlock the amap. 1283 */ 1284 anon->an_link = *tofree; 1285 *tofree = anon; 1286 } 1287 } 1288 } 1289 1290 #endif 1291 1292 #if defined(VMSWAP) 1293 1294 /* 1295 * amap_swap_off: pagein anonymous pages in amaps and drop swap slots. 1296 * 1297 * => called with swap_syscall_lock held. 1298 * => note that we don't always traverse all anons. 1299 * eg. amaps being wiped out, released anons. 1300 * => return true if failed. 1301 */ 1302 1303 bool 1304 amap_swap_off(int startslot, int endslot) 1305 { 1306 struct vm_amap *am; 1307 struct vm_amap *am_next; 1308 struct vm_amap marker_prev; 1309 struct vm_amap marker_next; 1310 bool rv = false; 1311 1312 #if defined(DIAGNOSTIC) 1313 memset(&marker_prev, 0, sizeof(marker_prev)); 1314 memset(&marker_next, 0, sizeof(marker_next)); 1315 #endif /* defined(DIAGNOSTIC) */ 1316 1317 mutex_enter(&amap_list_lock); 1318 for (am = LIST_FIRST(&amap_list); am != NULL && !rv; am = am_next) { 1319 int i; 1320 1321 LIST_INSERT_BEFORE(am, &marker_prev, am_list); 1322 LIST_INSERT_AFTER(am, &marker_next, am_list); 1323 1324 if (!amap_lock_try(am)) { 1325 mutex_exit(&amap_list_lock); 1326 preempt(); 1327 mutex_enter(&amap_list_lock); 1328 am_next = LIST_NEXT(&marker_prev, am_list); 1329 if (am_next == &marker_next) { 1330 am_next = LIST_NEXT(am_next, am_list); 1331 } else { 1332 KASSERT(LIST_NEXT(am_next, am_list) == 1333 &marker_next); 1334 } 1335 LIST_REMOVE(&marker_prev, am_list); 1336 LIST_REMOVE(&marker_next, am_list); 1337 continue; 1338 } 1339 1340 mutex_exit(&amap_list_lock); 1341 1342 if (am->am_nused <= 0) { 1343 amap_unlock(am); 1344 goto next; 1345 } 1346 1347 for (i = 0; i < am->am_nused; i++) { 1348 int slot; 1349 int swslot; 1350 struct vm_anon *anon; 1351 1352 slot = am->am_slots[i]; 1353 anon = am->am_anon[slot]; 1354 KASSERT(anon->an_lock == am->am_lock); 1355 1356 swslot = anon->an_swslot; 1357 if (swslot < startslot || endslot <= swslot) { 1358 continue; 1359 } 1360 1361 am->am_flags |= AMAP_SWAPOFF; 1362 1363 rv = uvm_anon_pagein(am, anon); 1364 amap_lock(am); 1365 1366 am->am_flags &= ~AMAP_SWAPOFF; 1367 if (amap_refs(am) == 0) { 1368 amap_wipeout(am); 1369 am = NULL; 1370 break; 1371 } 1372 if (rv) { 1373 break; 1374 } 1375 i = 0; 1376 } 1377 1378 if (am) { 1379 amap_unlock(am); 1380 } 1381 1382 next: 1383 mutex_enter(&amap_list_lock); 1384 KASSERT(LIST_NEXT(&marker_prev, am_list) == &marker_next || 1385 LIST_NEXT(LIST_NEXT(&marker_prev, am_list), am_list) == 1386 &marker_next); 1387 am_next = LIST_NEXT(&marker_next, am_list); 1388 LIST_REMOVE(&marker_prev, am_list); 1389 LIST_REMOVE(&marker_next, am_list); 1390 } 1391 mutex_exit(&amap_list_lock); 1392 1393 return rv; 1394 } 1395 1396 #endif /* defined(VMSWAP) */ 1397 1398 /* 1399 * amap_lookup: look up a page in an amap. 1400 * 1401 * => amap should be locked by caller. 1402 */ 1403 struct vm_anon * 1404 amap_lookup(struct vm_aref *aref, vaddr_t offset) 1405 { 1406 struct vm_amap *amap = aref->ar_amap; 1407 struct vm_anon *an; 1408 u_int slot; 1409 1410 UVMHIST_FUNC("amap_lookup"); UVMHIST_CALLED(maphist); 1411 KASSERT(mutex_owned(amap->am_lock)); 1412 1413 AMAP_B2SLOT(slot, offset); 1414 slot += aref->ar_pageoff; 1415 an = amap->am_anon[slot]; 1416 1417 UVMHIST_LOG(maphist, "<- done (amap=0x%x, offset=0x%x, result=0x%x)", 1418 amap, offset, an, 0); 1419 1420 KASSERT(slot < amap->am_nslot); 1421 KASSERT(an == NULL || an->an_ref != 0); 1422 KASSERT(an == NULL || an->an_lock == amap->am_lock); 1423 return an; 1424 } 1425 1426 /* 1427 * amap_lookups: look up a range of pages in an amap. 1428 * 1429 * => amap should be locked by caller. 1430 */ 1431 void 1432 amap_lookups(struct vm_aref *aref, vaddr_t offset, struct vm_anon **anons, 1433 int npages) 1434 { 1435 struct vm_amap *amap = aref->ar_amap; 1436 u_int slot; 1437 1438 UVMHIST_FUNC("amap_lookups"); UVMHIST_CALLED(maphist); 1439 KASSERT(mutex_owned(amap->am_lock)); 1440 1441 AMAP_B2SLOT(slot, offset); 1442 slot += aref->ar_pageoff; 1443 1444 UVMHIST_LOG(maphist, " slot=%u, npages=%d, nslot=%d", 1445 slot, npages, amap->am_nslot, 0); 1446 1447 KASSERT((slot + (npages - 1)) < amap->am_nslot); 1448 memcpy(anons, &amap->am_anon[slot], npages * sizeof(struct vm_anon *)); 1449 1450 #if defined(DIAGNOSTIC) 1451 for (int i = 0; i < npages; i++) { 1452 struct vm_anon * const an = anons[i]; 1453 if (an == NULL) { 1454 continue; 1455 } 1456 KASSERT(an->an_ref != 0); 1457 KASSERT(an->an_lock == amap->am_lock); 1458 } 1459 #endif 1460 UVMHIST_LOG(maphist, "<- done", 0, 0, 0, 0); 1461 } 1462 1463 /* 1464 * amap_add: add (or replace) a page to an amap. 1465 * 1466 * => amap should be locked by caller. 1467 * => anon must have the lock associated with this amap. 1468 */ 1469 void 1470 amap_add(struct vm_aref *aref, vaddr_t offset, struct vm_anon *anon, 1471 bool replace) 1472 { 1473 struct vm_amap *amap = aref->ar_amap; 1474 u_int slot; 1475 1476 UVMHIST_FUNC("amap_add"); UVMHIST_CALLED(maphist); 1477 KASSERT(mutex_owned(amap->am_lock)); 1478 KASSERT(anon->an_lock == amap->am_lock); 1479 1480 AMAP_B2SLOT(slot, offset); 1481 slot += aref->ar_pageoff; 1482 KASSERT(slot < amap->am_nslot); 1483 1484 if (replace) { 1485 struct vm_anon *oanon = amap->am_anon[slot]; 1486 1487 KASSERT(oanon != NULL); 1488 if (oanon->an_page && (amap->am_flags & AMAP_SHARED) != 0) { 1489 pmap_page_protect(oanon->an_page, VM_PROT_NONE); 1490 /* 1491 * XXX: suppose page is supposed to be wired somewhere? 1492 */ 1493 } 1494 } else { 1495 KASSERT(amap->am_anon[slot] == NULL); 1496 amap->am_bckptr[slot] = amap->am_nused; 1497 amap->am_slots[amap->am_nused] = slot; 1498 amap->am_nused++; 1499 } 1500 amap->am_anon[slot] = anon; 1501 UVMHIST_LOG(maphist, 1502 "<- done (amap=0x%x, offset=0x%x, anon=0x%x, rep=%d)", 1503 amap, offset, anon, replace); 1504 } 1505 1506 /* 1507 * amap_unadd: remove a page from an amap. 1508 * 1509 * => amap should be locked by caller. 1510 */ 1511 void 1512 amap_unadd(struct vm_aref *aref, vaddr_t offset) 1513 { 1514 struct vm_amap *amap = aref->ar_amap; 1515 u_int slot, ptr, last; 1516 1517 UVMHIST_FUNC("amap_unadd"); UVMHIST_CALLED(maphist); 1518 KASSERT(mutex_owned(amap->am_lock)); 1519 1520 AMAP_B2SLOT(slot, offset); 1521 slot += aref->ar_pageoff; 1522 KASSERT(slot < amap->am_nslot); 1523 KASSERT(amap->am_anon[slot] != NULL); 1524 KASSERT(amap->am_anon[slot]->an_lock == amap->am_lock); 1525 1526 amap->am_anon[slot] = NULL; 1527 ptr = amap->am_bckptr[slot]; 1528 1529 last = amap->am_nused - 1; 1530 if (ptr != last) { 1531 /* Move the last entry to keep the slots contiguous. */ 1532 amap->am_slots[ptr] = amap->am_slots[last]; 1533 amap->am_bckptr[amap->am_slots[ptr]] = ptr; 1534 } 1535 amap->am_nused--; 1536 UVMHIST_LOG(maphist, "<- done (amap=0x%x, slot=0x%x)", amap, slot,0, 0); 1537 } 1538 1539 /* 1540 * amap_adjref_anons: adjust the reference count(s) on anons of the amap. 1541 */ 1542 static void 1543 amap_adjref_anons(struct vm_amap *amap, vaddr_t offset, vsize_t len, 1544 int refv, bool all) 1545 { 1546 #ifdef UVM_AMAP_PPREF 1547 KASSERT(mutex_owned(amap->am_lock)); 1548 1549 if (amap->am_ppref == NULL && !all && len != amap->am_nslot) { 1550 amap_pp_establish(amap, offset); 1551 } 1552 if (amap->am_ppref && amap->am_ppref != PPREF_NONE) { 1553 struct vm_anon *tofree = NULL; 1554 1555 if (all) { 1556 amap_pp_adjref(amap, 0, amap->am_nslot, refv, &tofree); 1557 } else { 1558 amap_pp_adjref(amap, offset, len, refv, &tofree); 1559 } 1560 uvm_anfree(tofree); 1561 } 1562 #endif 1563 } 1564 1565 /* 1566 * amap_ref: gain a reference to an amap. 1567 * 1568 * => amap must not be locked (we will lock). 1569 * => "offset" and "len" are in units of pages. 1570 * => Called at fork time to gain the child's reference. 1571 */ 1572 void 1573 amap_ref(struct vm_amap *amap, vaddr_t offset, vsize_t len, int flags) 1574 { 1575 UVMHIST_FUNC("amap_ref"); UVMHIST_CALLED(maphist); 1576 1577 amap_lock(amap); 1578 if (flags & AMAP_SHARED) { 1579 amap->am_flags |= AMAP_SHARED; 1580 } 1581 amap->am_ref++; 1582 amap_adjref_anons(amap, offset, len, 1, (flags & AMAP_REFALL) != 0); 1583 amap_unlock(amap); 1584 1585 UVMHIST_LOG(maphist,"<- done! amap=0x%x", amap, 0, 0, 0); 1586 } 1587 1588 /* 1589 * amap_unref: remove a reference to an amap. 1590 * 1591 * => All pmap-level references to this amap must be already removed. 1592 * => Called from uvm_unmap_detach(); entry is already removed from the map. 1593 * => We will lock amap, so it must be unlocked. 1594 */ 1595 void 1596 amap_unref(struct vm_amap *amap, vaddr_t offset, vsize_t len, bool all) 1597 { 1598 UVMHIST_FUNC("amap_unref"); UVMHIST_CALLED(maphist); 1599 1600 amap_lock(amap); 1601 1602 UVMHIST_LOG(maphist," amap=0x%x refs=%d, nused=%d", 1603 amap, amap->am_ref, amap->am_nused, 0); 1604 KASSERT(amap->am_ref > 0); 1605 1606 if (--amap->am_ref == 0) { 1607 /* 1608 * If the last reference - wipeout and destroy the amap. 1609 */ 1610 amap_wipeout(amap); 1611 UVMHIST_LOG(maphist,"<- done (was last ref)!", 0, 0, 0, 0); 1612 return; 1613 } 1614 1615 /* 1616 * Otherwise, drop the reference count(s) on anons. 1617 */ 1618 1619 if (amap->am_ref == 1 && (amap->am_flags & AMAP_SHARED) != 0) { 1620 amap->am_flags &= ~AMAP_SHARED; 1621 } 1622 amap_adjref_anons(amap, offset, len, -1, all); 1623 amap_unlock(amap); 1624 1625 UVMHIST_LOG(maphist,"<- done!", 0, 0, 0, 0); 1626 } 1627