1 /* $NetBSD: pmap.c,v 1.67 2022/09/15 06:44:18 skrll Exp $ */ 2 3 /*- 4 * Copyright (c) 1998, 2001 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility, 9 * NASA Ames Research Center and by Chris G. Demetriou. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 22 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 23 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 24 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 * POSSIBILITY OF SUCH DAMAGE. 31 */ 32 33 /* 34 * Copyright (c) 1992, 1993 35 * The Regents of the University of California. All rights reserved. 36 * 37 * This code is derived from software contributed to Berkeley by 38 * the Systems Programming Group of the University of Utah Computer 39 * Science Department and Ralph Campbell. 40 * 41 * Redistribution and use in source and binary forms, with or without 42 * modification, are permitted provided that the following conditions 43 * are met: 44 * 1. Redistributions of source code must retain the above copyright 45 * notice, this list of conditions and the following disclaimer. 46 * 2. Redistributions in binary form must reproduce the above copyright 47 * notice, this list of conditions and the following disclaimer in the 48 * documentation and/or other materials provided with the distribution. 49 * 3. Neither the name of the University nor the names of its contributors 50 * may be used to endorse or promote products derived from this software 51 * without specific prior written permission. 52 * 53 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 54 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 55 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 56 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 57 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 58 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 59 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 60 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 61 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 62 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 63 * SUCH DAMAGE. 64 * 65 * @(#)pmap.c 8.4 (Berkeley) 1/26/94 66 */ 67 68 #include <sys/cdefs.h> 69 70 __KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.67 2022/09/15 06:44:18 skrll Exp $"); 71 72 /* 73 * Manages physical address maps. 74 * 75 * In addition to hardware address maps, this 76 * module is called upon to provide software-use-only 77 * maps which may or may not be stored in the same 78 * form as hardware maps. These pseudo-maps are 79 * used to store intermediate results from copy 80 * operations to and from address spaces. 81 * 82 * Since the information managed by this module is 83 * also stored by the logical address mapping module, 84 * this module may throw away valid virtual-to-physical 85 * mappings at almost any time. However, invalidations 86 * of virtual-to-physical mappings must be done as 87 * requested. 88 * 89 * In order to cope with hardware architectures which 90 * make virtual-to-physical map invalidates expensive, 91 * this module may delay invalidate or reduced protection 92 * operations until such time as they are actually 93 * necessary. This module is given full information as 94 * to which processors are currently using which maps, 95 * and to when physical maps must be made correct. 96 */ 97 98 #include "opt_modular.h" 99 #include "opt_multiprocessor.h" 100 #include "opt_sysv.h" 101 102 #define __PMAP_PRIVATE 103 104 #include <sys/param.h> 105 106 #include <sys/asan.h> 107 #include <sys/atomic.h> 108 #include <sys/buf.h> 109 #include <sys/cpu.h> 110 #include <sys/mutex.h> 111 #include <sys/pool.h> 112 113 #include <uvm/uvm.h> 114 #include <uvm/uvm_physseg.h> 115 #include <uvm/pmap/pmap_pvt.h> 116 117 #if defined(MULTIPROCESSOR) && defined(PMAP_VIRTUAL_CACHE_ALIASES) \ 118 && !defined(PMAP_NO_PV_UNCACHED) 119 #error PMAP_VIRTUAL_CACHE_ALIASES with MULTIPROCESSOR requires \ 120 PMAP_NO_PV_UNCACHED to be defined 121 #endif 122 123 #if defined(PMAP_PV_TRACK_ONLY_STUBS) 124 #undef __HAVE_PMAP_PV_TRACK 125 #endif 126 127 PMAP_COUNTER(remove_kernel_calls, "remove kernel calls"); 128 PMAP_COUNTER(remove_kernel_pages, "kernel pages unmapped"); 129 PMAP_COUNTER(remove_user_calls, "remove user calls"); 130 PMAP_COUNTER(remove_user_pages, "user pages unmapped"); 131 PMAP_COUNTER(remove_flushes, "remove cache flushes"); 132 PMAP_COUNTER(remove_tlb_ops, "remove tlb ops"); 133 PMAP_COUNTER(remove_pvfirst, "remove pv first"); 134 PMAP_COUNTER(remove_pvsearch, "remove pv search"); 135 136 PMAP_COUNTER(prefer_requests, "prefer requests"); 137 PMAP_COUNTER(prefer_adjustments, "prefer adjustments"); 138 139 PMAP_COUNTER(idlezeroed_pages, "pages idle zeroed"); 140 141 PMAP_COUNTER(kenter_pa, "kernel fast mapped pages"); 142 PMAP_COUNTER(kenter_pa_bad, "kernel fast mapped pages (bad color)"); 143 PMAP_COUNTER(kenter_pa_unmanaged, "kernel fast mapped unmanaged pages"); 144 PMAP_COUNTER(kremove_pages, "kernel fast unmapped pages"); 145 146 PMAP_COUNTER(page_cache_evictions, "pages changed to uncacheable"); 147 PMAP_COUNTER(page_cache_restorations, "pages changed to cacheable"); 148 149 PMAP_COUNTER(kernel_mappings_bad, "kernel pages mapped (bad color)"); 150 PMAP_COUNTER(user_mappings_bad, "user pages mapped (bad color)"); 151 PMAP_COUNTER(kernel_mappings, "kernel pages mapped"); 152 PMAP_COUNTER(user_mappings, "user pages mapped"); 153 PMAP_COUNTER(user_mappings_changed, "user mapping changed"); 154 PMAP_COUNTER(kernel_mappings_changed, "kernel mapping changed"); 155 PMAP_COUNTER(uncached_mappings, "uncached pages mapped"); 156 PMAP_COUNTER(unmanaged_mappings, "unmanaged pages mapped"); 157 PMAP_COUNTER(pvtracked_mappings, "pv-tracked unmanaged pages mapped"); 158 PMAP_COUNTER(managed_mappings, "managed pages mapped"); 159 PMAP_COUNTER(mappings, "pages mapped"); 160 PMAP_COUNTER(remappings, "pages remapped"); 161 PMAP_COUNTER(unmappings, "pages unmapped"); 162 PMAP_COUNTER(primary_mappings, "page initial mappings"); 163 PMAP_COUNTER(primary_unmappings, "page final unmappings"); 164 PMAP_COUNTER(tlb_hit, "page mapping"); 165 166 PMAP_COUNTER(exec_mappings, "exec pages mapped"); 167 PMAP_COUNTER(exec_synced_mappings, "exec pages synced"); 168 PMAP_COUNTER(exec_synced_remove, "exec pages synced (PR)"); 169 PMAP_COUNTER(exec_synced_clear_modify, "exec pages synced (CM)"); 170 PMAP_COUNTER(exec_synced_page_protect, "exec pages synced (PP)"); 171 PMAP_COUNTER(exec_synced_protect, "exec pages synced (P)"); 172 PMAP_COUNTER(exec_uncached_page_protect, "exec pages uncached (PP)"); 173 PMAP_COUNTER(exec_uncached_clear_modify, "exec pages uncached (CM)"); 174 PMAP_COUNTER(exec_uncached_zero_page, "exec pages uncached (ZP)"); 175 PMAP_COUNTER(exec_uncached_copy_page, "exec pages uncached (CP)"); 176 PMAP_COUNTER(exec_uncached_remove, "exec pages uncached (PR)"); 177 178 PMAP_COUNTER(create, "creates"); 179 PMAP_COUNTER(reference, "references"); 180 PMAP_COUNTER(dereference, "dereferences"); 181 PMAP_COUNTER(destroy, "destroyed"); 182 PMAP_COUNTER(activate, "activations"); 183 PMAP_COUNTER(deactivate, "deactivations"); 184 PMAP_COUNTER(update, "updates"); 185 #ifdef MULTIPROCESSOR 186 PMAP_COUNTER(shootdown_ipis, "shootdown IPIs"); 187 #endif 188 PMAP_COUNTER(unwire, "unwires"); 189 PMAP_COUNTER(copy, "copies"); 190 PMAP_COUNTER(clear_modify, "clear_modifies"); 191 PMAP_COUNTER(protect, "protects"); 192 PMAP_COUNTER(page_protect, "page_protects"); 193 194 #define PMAP_ASID_RESERVED 0 195 CTASSERT(PMAP_ASID_RESERVED == 0); 196 197 #ifndef PMAP_SEGTAB_ALIGN 198 #define PMAP_SEGTAB_ALIGN /* nothing */ 199 #endif 200 #ifdef _LP64 201 pmap_segtab_t pmap_kstart_segtab PMAP_SEGTAB_ALIGN; /* first mid-level segtab for kernel */ 202 #endif 203 pmap_segtab_t pmap_kern_segtab PMAP_SEGTAB_ALIGN = { /* top level segtab for kernel */ 204 #ifdef _LP64 205 .seg_seg[(VM_MIN_KERNEL_ADDRESS & XSEGOFSET) >> SEGSHIFT] = &pmap_kstart_segtab, 206 #endif 207 }; 208 209 struct pmap_kernel kernel_pmap_store = { 210 .kernel_pmap = { 211 .pm_count = 1, 212 .pm_segtab = &pmap_kern_segtab, 213 .pm_minaddr = VM_MIN_KERNEL_ADDRESS, 214 .pm_maxaddr = VM_MAX_KERNEL_ADDRESS, 215 }, 216 }; 217 218 struct pmap * const kernel_pmap_ptr = &kernel_pmap_store.kernel_pmap; 219 220 /* The current top of kernel VM - gets updated by pmap_growkernel */ 221 vaddr_t pmap_curmaxkvaddr; 222 223 struct pmap_limits pmap_limits = { /* VA and PA limits */ 224 .virtual_start = VM_MIN_KERNEL_ADDRESS, 225 .virtual_end = VM_MAX_KERNEL_ADDRESS, 226 }; 227 228 #ifdef UVMHIST 229 static struct kern_history_ent pmapexechistbuf[10000]; 230 static struct kern_history_ent pmaphistbuf[10000]; 231 static struct kern_history_ent pmapsegtabhistbuf[1000]; 232 UVMHIST_DEFINE(pmapexechist) = UVMHIST_INITIALIZER(pmapexechist, pmapexechistbuf); 233 UVMHIST_DEFINE(pmaphist) = UVMHIST_INITIALIZER(pmaphist, pmaphistbuf); 234 UVMHIST_DEFINE(pmapsegtabhist) = UVMHIST_INITIALIZER(pmapsegtabhist, pmapsegtabhistbuf); 235 #endif 236 237 /* 238 * The pools from which pmap structures and sub-structures are allocated. 239 */ 240 struct pool pmap_pmap_pool; 241 struct pool pmap_pv_pool; 242 243 #ifndef PMAP_PV_LOWAT 244 #define PMAP_PV_LOWAT 16 245 #endif 246 int pmap_pv_lowat = PMAP_PV_LOWAT; 247 248 bool pmap_initialized = false; 249 #define PMAP_PAGE_COLOROK_P(a, b) \ 250 ((((int)(a) ^ (int)(b)) & pmap_page_colormask) == 0) 251 u_int pmap_page_colormask; 252 253 #define PAGE_IS_MANAGED(pa) (pmap_initialized && uvm_pageismanaged(pa)) 254 255 #define PMAP_IS_ACTIVE(pm) \ 256 ((pm) == pmap_kernel() || \ 257 (pm) == curlwp->l_proc->p_vmspace->vm_map.pmap) 258 259 /* Forward function declarations */ 260 void pmap_page_remove(struct vm_page_md *); 261 static void pmap_pvlist_check(struct vm_page_md *); 262 void pmap_remove_pv(pmap_t, vaddr_t, struct vm_page *, bool); 263 void pmap_enter_pv(pmap_t, vaddr_t, paddr_t, struct vm_page_md *, pt_entry_t *, u_int); 264 265 /* 266 * PV table management functions. 267 */ 268 void *pmap_pv_page_alloc(struct pool *, int); 269 void pmap_pv_page_free(struct pool *, void *); 270 271 struct pool_allocator pmap_pv_page_allocator = { 272 pmap_pv_page_alloc, pmap_pv_page_free, 0, 273 }; 274 275 #define pmap_pv_alloc() pool_get(&pmap_pv_pool, PR_NOWAIT) 276 #define pmap_pv_free(pv) pool_put(&pmap_pv_pool, (pv)) 277 278 #ifndef PMAP_NEED_TLB_MISS_LOCK 279 280 #if defined(PMAP_MD_NEED_TLB_MISS_LOCK) || defined(DEBUG) 281 #define PMAP_NEED_TLB_MISS_LOCK 282 #endif /* PMAP_MD_NEED_TLB_MISS_LOCK || DEBUG */ 283 284 #endif /* PMAP_NEED_TLB_MISS_LOCK */ 285 286 #ifdef PMAP_NEED_TLB_MISS_LOCK 287 288 #ifdef PMAP_MD_NEED_TLB_MISS_LOCK 289 #define pmap_tlb_miss_lock_init() __nothing /* MD code deals with this */ 290 #define pmap_tlb_miss_lock_enter() pmap_md_tlb_miss_lock_enter() 291 #define pmap_tlb_miss_lock_exit() pmap_md_tlb_miss_lock_exit() 292 #else 293 kmutex_t pmap_tlb_miss_lock __cacheline_aligned; 294 295 static void 296 pmap_tlb_miss_lock_init(void) 297 { 298 mutex_init(&pmap_tlb_miss_lock, MUTEX_SPIN, IPL_HIGH); 299 } 300 301 static inline void 302 pmap_tlb_miss_lock_enter(void) 303 { 304 mutex_spin_enter(&pmap_tlb_miss_lock); 305 } 306 307 static inline void 308 pmap_tlb_miss_lock_exit(void) 309 { 310 mutex_spin_exit(&pmap_tlb_miss_lock); 311 } 312 #endif /* PMAP_MD_NEED_TLB_MISS_LOCK */ 313 314 #else 315 316 #define pmap_tlb_miss_lock_init() __nothing 317 #define pmap_tlb_miss_lock_enter() __nothing 318 #define pmap_tlb_miss_lock_exit() __nothing 319 320 #endif /* PMAP_NEED_TLB_MISS_LOCK */ 321 322 #ifndef MULTIPROCESSOR 323 kmutex_t pmap_pvlist_mutex __cacheline_aligned; 324 #endif 325 326 /* 327 * Debug functions. 328 */ 329 330 #ifdef DEBUG 331 static inline void 332 pmap_asid_check(pmap_t pm, const char *func) 333 { 334 if (!PMAP_IS_ACTIVE(pm)) 335 return; 336 337 struct pmap_asid_info * const pai = PMAP_PAI(pm, cpu_tlb_info(curcpu())); 338 tlb_asid_t asid = tlb_get_asid(); 339 if (asid != pai->pai_asid) 340 panic("%s: inconsistency for active TLB update: %u <-> %u", 341 func, asid, pai->pai_asid); 342 } 343 #endif 344 345 static void 346 pmap_addr_range_check(pmap_t pmap, vaddr_t sva, vaddr_t eva, const char *func) 347 { 348 #ifdef DEBUG 349 if (pmap == pmap_kernel()) { 350 if (sva < VM_MIN_KERNEL_ADDRESS) 351 panic("%s: kva %#"PRIxVADDR" not in range", 352 func, sva); 353 if (eva >= pmap_limits.virtual_end) 354 panic("%s: kva %#"PRIxVADDR" not in range", 355 func, eva); 356 } else { 357 if (eva > VM_MAXUSER_ADDRESS) 358 panic("%s: uva %#"PRIxVADDR" not in range", 359 func, eva); 360 pmap_asid_check(pmap, func); 361 } 362 #endif 363 } 364 365 /* 366 * Misc. functions. 367 */ 368 369 bool 370 pmap_page_clear_attributes(struct vm_page_md *mdpg, u_int clear_attributes) 371 { 372 volatile unsigned long * const attrp = &mdpg->mdpg_attrs; 373 #ifdef MULTIPROCESSOR 374 for (;;) { 375 u_int old_attr = *attrp; 376 if ((old_attr & clear_attributes) == 0) 377 return false; 378 u_int new_attr = old_attr & ~clear_attributes; 379 if (old_attr == atomic_cas_ulong(attrp, old_attr, new_attr)) 380 return true; 381 } 382 #else 383 unsigned long old_attr = *attrp; 384 if ((old_attr & clear_attributes) == 0) 385 return false; 386 *attrp &= ~clear_attributes; 387 return true; 388 #endif 389 } 390 391 void 392 pmap_page_set_attributes(struct vm_page_md *mdpg, u_int set_attributes) 393 { 394 #ifdef MULTIPROCESSOR 395 atomic_or_ulong(&mdpg->mdpg_attrs, set_attributes); 396 #else 397 mdpg->mdpg_attrs |= set_attributes; 398 #endif 399 } 400 401 static void 402 pmap_page_syncicache(struct vm_page *pg) 403 { 404 UVMHIST_FUNC(__func__); 405 UVMHIST_CALLED(pmaphist); 406 #ifndef MULTIPROCESSOR 407 struct pmap * const curpmap = curlwp->l_proc->p_vmspace->vm_map.pmap; 408 #endif 409 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 410 pv_entry_t pv = &mdpg->mdpg_first; 411 kcpuset_t *onproc; 412 #ifdef MULTIPROCESSOR 413 kcpuset_create(&onproc, true); 414 KASSERT(onproc != NULL); 415 #else 416 onproc = NULL; 417 #endif 418 VM_PAGEMD_PVLIST_READLOCK(mdpg); 419 pmap_pvlist_check(mdpg); 420 421 UVMHIST_LOG(pmaphist, "pv %#jx pv_pmap %#jx", (uintptr_t)pv, 422 (uintptr_t)pv->pv_pmap, 0, 0); 423 424 if (pv->pv_pmap != NULL) { 425 for (; pv != NULL; pv = pv->pv_next) { 426 #ifdef MULTIPROCESSOR 427 UVMHIST_LOG(pmaphist, "pv %#jx pv_pmap %#jx", 428 (uintptr_t)pv, (uintptr_t)pv->pv_pmap, 0, 0); 429 kcpuset_merge(onproc, pv->pv_pmap->pm_onproc); 430 if (kcpuset_match(onproc, kcpuset_running)) { 431 break; 432 } 433 #else 434 if (pv->pv_pmap == curpmap) { 435 onproc = curcpu()->ci_data.cpu_kcpuset; 436 break; 437 } 438 #endif 439 } 440 } 441 pmap_pvlist_check(mdpg); 442 VM_PAGEMD_PVLIST_UNLOCK(mdpg); 443 kpreempt_disable(); 444 pmap_md_page_syncicache(mdpg, onproc); 445 kpreempt_enable(); 446 #ifdef MULTIPROCESSOR 447 kcpuset_destroy(onproc); 448 #endif 449 } 450 451 /* 452 * Define the initial bounds of the kernel virtual address space. 453 */ 454 void 455 pmap_virtual_space(vaddr_t *vstartp, vaddr_t *vendp) 456 { 457 458 *vstartp = pmap_limits.virtual_start; 459 *vendp = pmap_limits.virtual_end; 460 } 461 462 vaddr_t 463 pmap_growkernel(vaddr_t maxkvaddr) 464 { 465 UVMHIST_FUNC(__func__); 466 UVMHIST_CALLARGS(pmaphist, "maxkvaddr=%#jx (%#jx)", maxkvaddr, 467 pmap_curmaxkvaddr, 0, 0); 468 469 vaddr_t virtual_end = pmap_curmaxkvaddr; 470 maxkvaddr = pmap_round_seg(maxkvaddr) - 1; 471 472 /* 473 * Don't exceed VM_MAX_KERNEL_ADDRESS! 474 */ 475 if (maxkvaddr == 0 || maxkvaddr > VM_MAX_KERNEL_ADDRESS) 476 maxkvaddr = VM_MAX_KERNEL_ADDRESS; 477 478 /* 479 * Reserve PTEs for the new KVA space. 480 */ 481 for (; virtual_end < maxkvaddr; virtual_end += NBSEG) { 482 pmap_pte_reserve(pmap_kernel(), virtual_end, 0); 483 } 484 485 kasan_shadow_map((void *)pmap_curmaxkvaddr, 486 (size_t)(virtual_end - pmap_curmaxkvaddr)); 487 488 /* 489 * Update new end. 490 */ 491 pmap_curmaxkvaddr = virtual_end; 492 493 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 494 495 return virtual_end; 496 } 497 498 /* 499 * Bootstrap memory allocator (alternative to vm_bootstrap_steal_memory()). 500 * This function allows for early dynamic memory allocation until the virtual 501 * memory system has been bootstrapped. After that point, either kmem_alloc 502 * or malloc should be used. This function works by stealing pages from the 503 * (to be) managed page pool, then implicitly mapping the pages (by using 504 * their direct mapped addresses) and zeroing them. 505 * 506 * It may be used once the physical memory segments have been pre-loaded 507 * into the vm_physmem[] array. Early memory allocation MUST use this 508 * interface! This cannot be used after vm_page_startup(), and will 509 * generate a panic if tried. 510 * 511 * Note that this memory will never be freed, and in essence it is wired 512 * down. 513 * 514 * We must adjust *vstartp and/or *vendp iff we use address space 515 * from the kernel virtual address range defined by pmap_virtual_space(). 516 */ 517 vaddr_t 518 pmap_steal_memory(vsize_t size, vaddr_t *vstartp, vaddr_t *vendp) 519 { 520 size_t npgs; 521 paddr_t pa; 522 vaddr_t va; 523 524 uvm_physseg_t maybe_bank = UVM_PHYSSEG_TYPE_INVALID; 525 526 size = round_page(size); 527 npgs = atop(size); 528 529 aprint_debug("%s: need %zu pages\n", __func__, npgs); 530 531 for (uvm_physseg_t bank = uvm_physseg_get_first(); 532 uvm_physseg_valid_p(bank); 533 bank = uvm_physseg_get_next(bank)) { 534 535 if (uvm.page_init_done == true) 536 panic("pmap_steal_memory: called _after_ bootstrap"); 537 538 aprint_debug("%s: seg %"PRIxPHYSSEG": %#"PRIxPADDR" %#"PRIxPADDR" %#"PRIxPADDR" %#"PRIxPADDR"\n", 539 __func__, bank, 540 uvm_physseg_get_avail_start(bank), uvm_physseg_get_start(bank), 541 uvm_physseg_get_avail_end(bank), uvm_physseg_get_end(bank)); 542 543 if (uvm_physseg_get_avail_start(bank) != uvm_physseg_get_start(bank) 544 || uvm_physseg_get_avail_start(bank) >= uvm_physseg_get_avail_end(bank)) { 545 aprint_debug("%s: seg %"PRIxPHYSSEG": bad start\n", __func__, bank); 546 continue; 547 } 548 549 if (uvm_physseg_get_avail_end(bank) - uvm_physseg_get_avail_start(bank) < npgs) { 550 aprint_debug("%s: seg %"PRIxPHYSSEG": too small for %zu pages\n", 551 __func__, bank, npgs); 552 continue; 553 } 554 555 if (!pmap_md_ok_to_steal_p(bank, npgs)) { 556 continue; 557 } 558 559 /* 560 * Always try to allocate from the segment with the least 561 * amount of space left. 562 */ 563 #define VM_PHYSMEM_SPACE(b) ((uvm_physseg_get_avail_end(b)) - (uvm_physseg_get_avail_start(b))) 564 if (uvm_physseg_valid_p(maybe_bank) == false 565 || VM_PHYSMEM_SPACE(bank) < VM_PHYSMEM_SPACE(maybe_bank)) { 566 maybe_bank = bank; 567 } 568 } 569 570 if (uvm_physseg_valid_p(maybe_bank)) { 571 const uvm_physseg_t bank = maybe_bank; 572 573 /* 574 * There are enough pages here; steal them! 575 */ 576 pa = ptoa(uvm_physseg_get_start(bank)); 577 uvm_physseg_unplug(atop(pa), npgs); 578 579 aprint_debug("%s: seg %"PRIxPHYSSEG": %zu pages stolen (%#"PRIxPADDR" left)\n", 580 __func__, bank, npgs, VM_PHYSMEM_SPACE(bank)); 581 582 va = pmap_md_map_poolpage(pa, size); 583 memset((void *)va, 0, size); 584 return va; 585 } 586 587 /* 588 * If we got here, there was no memory left. 589 */ 590 panic("pmap_steal_memory: no memory to steal %zu pages", npgs); 591 } 592 593 /* 594 * Bootstrap the system enough to run with virtual memory. 595 * (Common routine called by machine-dependent bootstrap code.) 596 */ 597 void 598 pmap_bootstrap_common(void) 599 { 600 pmap_tlb_miss_lock_init(); 601 } 602 603 /* 604 * Initialize the pmap module. 605 * Called by vm_init, to initialize any structures that the pmap 606 * system needs to map virtual memory. 607 */ 608 void 609 pmap_init(void) 610 { 611 UVMHIST_LINK_STATIC(pmapexechist); 612 UVMHIST_LINK_STATIC(pmaphist); 613 UVMHIST_LINK_STATIC(pmapsegtabhist); 614 615 UVMHIST_FUNC(__func__); 616 UVMHIST_CALLED(pmaphist); 617 618 /* 619 * Initialize the segtab lock. 620 */ 621 mutex_init(&pmap_segtab_lock, MUTEX_DEFAULT, IPL_HIGH); 622 623 /* 624 * Set a low water mark on the pv_entry pool, so that we are 625 * more likely to have these around even in extreme memory 626 * starvation. 627 */ 628 pool_setlowat(&pmap_pv_pool, pmap_pv_lowat); 629 630 /* 631 * Set the page colormask but allow pmap_md_init to override it. 632 */ 633 pmap_page_colormask = ptoa(uvmexp.colormask); 634 635 pmap_md_init(); 636 637 /* 638 * Now it is safe to enable pv entry recording. 639 */ 640 pmap_initialized = true; 641 } 642 643 /* 644 * Create and return a physical map. 645 * 646 * If the size specified for the map 647 * is zero, the map is an actual physical 648 * map, and may be referenced by the 649 * hardware. 650 * 651 * If the size specified is non-zero, 652 * the map will be used in software only, and 653 * is bounded by that size. 654 */ 655 pmap_t 656 pmap_create(void) 657 { 658 UVMHIST_FUNC(__func__); 659 UVMHIST_CALLED(pmaphist); 660 PMAP_COUNT(create); 661 662 pmap_t pmap = pool_get(&pmap_pmap_pool, PR_WAITOK); 663 memset(pmap, 0, PMAP_SIZE); 664 665 KASSERT(pmap->pm_pai[0].pai_link.le_prev == NULL); 666 667 pmap->pm_count = 1; 668 pmap->pm_minaddr = VM_MIN_ADDRESS; 669 pmap->pm_maxaddr = VM_MAXUSER_ADDRESS; 670 671 pmap_segtab_init(pmap); 672 673 #ifdef MULTIPROCESSOR 674 kcpuset_create(&pmap->pm_active, true); 675 kcpuset_create(&pmap->pm_onproc, true); 676 KASSERT(pmap->pm_active != NULL); 677 KASSERT(pmap->pm_onproc != NULL); 678 #endif 679 680 UVMHIST_LOG(pmaphist, " <-- done (pmap=%#jx)", (uintptr_t)pmap, 681 0, 0, 0); 682 683 return pmap; 684 } 685 686 /* 687 * Retire the given physical map from service. 688 * Should only be called if the map contains 689 * no valid mappings. 690 */ 691 void 692 pmap_destroy(pmap_t pmap) 693 { 694 UVMHIST_FUNC(__func__); 695 UVMHIST_CALLARGS(pmaphist, "(pmap=%#jx)", (uintptr_t)pmap, 0, 0, 0); 696 697 membar_release(); 698 if (atomic_dec_uint_nv(&pmap->pm_count) > 0) { 699 PMAP_COUNT(dereference); 700 UVMHIST_LOG(pmaphist, " <-- done (deref)", 0, 0, 0, 0); 701 return; 702 } 703 membar_acquire(); 704 705 PMAP_COUNT(destroy); 706 KASSERT(pmap->pm_count == 0); 707 kpreempt_disable(); 708 pmap_tlb_miss_lock_enter(); 709 pmap_tlb_asid_release_all(pmap); 710 pmap_segtab_destroy(pmap, NULL, 0); 711 pmap_tlb_miss_lock_exit(); 712 713 #ifdef MULTIPROCESSOR 714 kcpuset_destroy(pmap->pm_active); 715 kcpuset_destroy(pmap->pm_onproc); 716 pmap->pm_active = NULL; 717 pmap->pm_onproc = NULL; 718 #endif 719 720 pool_put(&pmap_pmap_pool, pmap); 721 kpreempt_enable(); 722 723 UVMHIST_LOG(pmaphist, " <-- done (freed)", 0, 0, 0, 0); 724 } 725 726 /* 727 * Add a reference to the specified pmap. 728 */ 729 void 730 pmap_reference(pmap_t pmap) 731 { 732 UVMHIST_FUNC(__func__); 733 UVMHIST_CALLARGS(pmaphist, "(pmap=%#jx)", (uintptr_t)pmap, 0, 0, 0); 734 PMAP_COUNT(reference); 735 736 if (pmap != NULL) { 737 atomic_inc_uint(&pmap->pm_count); 738 } 739 740 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 741 } 742 743 /* 744 * Make a new pmap (vmspace) active for the given process. 745 */ 746 void 747 pmap_activate(struct lwp *l) 748 { 749 pmap_t pmap = l->l_proc->p_vmspace->vm_map.pmap; 750 751 UVMHIST_FUNC(__func__); 752 UVMHIST_CALLARGS(pmaphist, "(l=%#jx pmap=%#jx)", (uintptr_t)l, 753 (uintptr_t)pmap, 0, 0); 754 PMAP_COUNT(activate); 755 756 kpreempt_disable(); 757 pmap_tlb_miss_lock_enter(); 758 pmap_tlb_asid_acquire(pmap, l); 759 pmap_segtab_activate(pmap, l); 760 pmap_tlb_miss_lock_exit(); 761 kpreempt_enable(); 762 763 UVMHIST_LOG(pmaphist, " <-- done (%ju:%ju)", l->l_proc->p_pid, 764 l->l_lid, 0, 0); 765 } 766 767 /* 768 * Remove this page from all physical maps in which it resides. 769 * Reflects back modify bits to the pager. 770 */ 771 void 772 pmap_page_remove(struct vm_page_md *mdpg) 773 { 774 kpreempt_disable(); 775 VM_PAGEMD_PVLIST_LOCK(mdpg); 776 pmap_pvlist_check(mdpg); 777 778 struct vm_page * const pg = 779 VM_PAGEMD_VMPAGE_P(mdpg) ? VM_MD_TO_PAGE(mdpg) : NULL; 780 781 UVMHIST_FUNC(__func__); 782 if (pg) { 783 UVMHIST_CALLARGS(pmaphist, "mdpg %#jx pg %#jx (pa %#jx): " 784 "execpage cleared", (uintptr_t)mdpg, (uintptr_t)pg, 785 VM_PAGE_TO_PHYS(pg), 0); 786 } else { 787 UVMHIST_CALLARGS(pmaphist, "mdpg %#jx", (uintptr_t)mdpg, 0, 788 0, 0); 789 } 790 791 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 792 pmap_page_clear_attributes(mdpg, VM_PAGEMD_EXECPAGE|VM_PAGEMD_UNCACHED); 793 #else 794 pmap_page_clear_attributes(mdpg, VM_PAGEMD_EXECPAGE); 795 #endif 796 PMAP_COUNT(exec_uncached_remove); 797 798 pv_entry_t pv = &mdpg->mdpg_first; 799 if (pv->pv_pmap == NULL) { 800 VM_PAGEMD_PVLIST_UNLOCK(mdpg); 801 kpreempt_enable(); 802 UVMHIST_LOG(pmaphist, " <-- done (empty)", 0, 0, 0, 0); 803 return; 804 } 805 806 pv_entry_t npv; 807 pv_entry_t pvp = NULL; 808 809 for (; pv != NULL; pv = npv) { 810 npv = pv->pv_next; 811 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 812 if (PV_ISKENTER_P(pv)) { 813 UVMHIST_LOG(pmaphist, " pv %#jx pmap %#jx va %#jx" 814 " skip", (uintptr_t)pv, (uintptr_t)pv->pv_pmap, 815 pv->pv_va, 0); 816 817 KASSERT(pv->pv_pmap == pmap_kernel()); 818 819 /* Assume no more - it'll get fixed if there are */ 820 pv->pv_next = NULL; 821 822 /* 823 * pvp is non-null when we already have a PV_KENTER 824 * pv in pvh_first; otherwise we haven't seen a 825 * PV_KENTER pv and we need to copy this one to 826 * pvh_first 827 */ 828 if (pvp) { 829 /* 830 * The previous PV_KENTER pv needs to point to 831 * this PV_KENTER pv 832 */ 833 pvp->pv_next = pv; 834 } else { 835 pv_entry_t fpv = &mdpg->mdpg_first; 836 *fpv = *pv; 837 KASSERT(fpv->pv_pmap == pmap_kernel()); 838 } 839 pvp = pv; 840 continue; 841 } 842 #endif 843 const pmap_t pmap = pv->pv_pmap; 844 vaddr_t va = trunc_page(pv->pv_va); 845 pt_entry_t * const ptep = pmap_pte_lookup(pmap, va); 846 KASSERTMSG(ptep != NULL, "%#"PRIxVADDR " %#"PRIxVADDR, va, 847 pmap_limits.virtual_end); 848 pt_entry_t pte = *ptep; 849 UVMHIST_LOG(pmaphist, " pv %#jx pmap %#jx va %#jx" 850 " pte %#jx", (uintptr_t)pv, (uintptr_t)pmap, va, 851 pte_value(pte)); 852 if (!pte_valid_p(pte)) 853 continue; 854 const bool is_kernel_pmap_p = (pmap == pmap_kernel()); 855 if (is_kernel_pmap_p) { 856 PMAP_COUNT(remove_kernel_pages); 857 } else { 858 PMAP_COUNT(remove_user_pages); 859 } 860 if (pte_wired_p(pte)) 861 pmap->pm_stats.wired_count--; 862 pmap->pm_stats.resident_count--; 863 864 pmap_tlb_miss_lock_enter(); 865 const pt_entry_t npte = pte_nv_entry(is_kernel_pmap_p); 866 pte_set(ptep, npte); 867 if (__predict_true(!(pmap->pm_flags & PMAP_DEFERRED_ACTIVATE))) { 868 /* 869 * Flush the TLB for the given address. 870 */ 871 pmap_tlb_invalidate_addr(pmap, va); 872 } 873 pmap_tlb_miss_lock_exit(); 874 875 /* 876 * non-null means this is a non-pvh_first pv, so we should 877 * free it. 878 */ 879 if (pvp) { 880 KASSERT(pvp->pv_pmap == pmap_kernel()); 881 KASSERT(pvp->pv_next == NULL); 882 pmap_pv_free(pv); 883 } else { 884 pv->pv_pmap = NULL; 885 pv->pv_next = NULL; 886 } 887 } 888 889 pmap_pvlist_check(mdpg); 890 VM_PAGEMD_PVLIST_UNLOCK(mdpg); 891 kpreempt_enable(); 892 893 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 894 } 895 896 #ifdef __HAVE_PMAP_PV_TRACK 897 /* 898 * pmap_pv_protect: change protection of an unmanaged pv-tracked page from 899 * all pmaps that map it 900 */ 901 void 902 pmap_pv_protect(paddr_t pa, vm_prot_t prot) 903 { 904 905 /* the only case is remove at the moment */ 906 KASSERT(prot == VM_PROT_NONE); 907 struct pmap_page *pp; 908 909 pp = pmap_pv_tracked(pa); 910 if (pp == NULL) 911 panic("pmap_pv_protect: page not pv-tracked: 0x%"PRIxPADDR, 912 pa); 913 914 struct vm_page_md *mdpg = PMAP_PAGE_TO_MD(pp); 915 pmap_page_remove(mdpg); 916 } 917 #endif 918 919 /* 920 * Make a previously active pmap (vmspace) inactive. 921 */ 922 void 923 pmap_deactivate(struct lwp *l) 924 { 925 pmap_t pmap = l->l_proc->p_vmspace->vm_map.pmap; 926 927 UVMHIST_FUNC(__func__); 928 UVMHIST_CALLARGS(pmaphist, "(l=%#jx pmap=%#jx)", (uintptr_t)l, 929 (uintptr_t)pmap, 0, 0); 930 PMAP_COUNT(deactivate); 931 932 kpreempt_disable(); 933 KASSERT(l == curlwp || l->l_cpu == curlwp->l_cpu); 934 pmap_tlb_miss_lock_enter(); 935 pmap_tlb_asid_deactivate(pmap); 936 pmap_segtab_deactivate(pmap); 937 pmap_tlb_miss_lock_exit(); 938 kpreempt_enable(); 939 940 UVMHIST_LOG(pmaphist, " <-- done (%ju:%ju)", l->l_proc->p_pid, 941 l->l_lid, 0, 0); 942 } 943 944 void 945 pmap_update(struct pmap *pmap) 946 { 947 UVMHIST_FUNC(__func__); 948 UVMHIST_CALLARGS(pmaphist, "(pmap=%#jx)", (uintptr_t)pmap, 0, 0, 0); 949 PMAP_COUNT(update); 950 951 kpreempt_disable(); 952 #if defined(MULTIPROCESSOR) && defined(PMAP_TLB_NEED_SHOOTDOWN) 953 u_int pending = atomic_swap_uint(&pmap->pm_shootdown_pending, 0); 954 if (pending && pmap_tlb_shootdown_bystanders(pmap)) 955 PMAP_COUNT(shootdown_ipis); 956 #endif 957 pmap_tlb_miss_lock_enter(); 958 #if defined(DEBUG) && !defined(MULTIPROCESSOR) 959 pmap_tlb_check(pmap, pmap_md_tlb_check_entry); 960 #endif /* DEBUG */ 961 962 /* 963 * If pmap_remove_all was called, we deactivated ourselves and nuked 964 * our ASID. Now we have to reactivate ourselves. 965 */ 966 if (__predict_false(pmap->pm_flags & PMAP_DEFERRED_ACTIVATE)) { 967 pmap->pm_flags ^= PMAP_DEFERRED_ACTIVATE; 968 pmap_tlb_asid_acquire(pmap, curlwp); 969 pmap_segtab_activate(pmap, curlwp); 970 } 971 pmap_tlb_miss_lock_exit(); 972 kpreempt_enable(); 973 974 UVMHIST_LOG(pmaphist, " <-- done (kernel=%jd)", 975 (pmap == pmap_kernel() ? 1 : 0), 0, 0, 0); 976 } 977 978 /* 979 * Remove the given range of addresses from the specified map. 980 * 981 * It is assumed that the start and end are properly 982 * rounded to the page size. 983 */ 984 985 static bool 986 pmap_pte_remove(pmap_t pmap, vaddr_t sva, vaddr_t eva, pt_entry_t *ptep, 987 uintptr_t flags) 988 { 989 const pt_entry_t npte = flags; 990 const bool is_kernel_pmap_p = (pmap == pmap_kernel()); 991 992 UVMHIST_FUNC(__func__); 993 UVMHIST_CALLARGS(pmaphist, "(pmap=%#jx kernel=%jd va=%#jx..%#jx)", 994 (uintptr_t)pmap, (is_kernel_pmap_p ? 1 : 0), sva, eva); 995 UVMHIST_LOG(pmaphist, "ptep=%#jx, flags(npte)=%#jx)", 996 (uintptr_t)ptep, flags, 0, 0); 997 998 KASSERT(kpreempt_disabled()); 999 1000 for (; sva < eva; sva += NBPG, ptep++) { 1001 const pt_entry_t pte = *ptep; 1002 if (!pte_valid_p(pte)) 1003 continue; 1004 if (is_kernel_pmap_p) { 1005 PMAP_COUNT(remove_kernel_pages); 1006 } else { 1007 PMAP_COUNT(remove_user_pages); 1008 } 1009 if (pte_wired_p(pte)) 1010 pmap->pm_stats.wired_count--; 1011 pmap->pm_stats.resident_count--; 1012 struct vm_page * const pg = PHYS_TO_VM_PAGE(pte_to_paddr(pte)); 1013 if (__predict_true(pg != NULL)) { 1014 pmap_remove_pv(pmap, sva, pg, pte_modified_p(pte)); 1015 } 1016 pmap_tlb_miss_lock_enter(); 1017 pte_set(ptep, npte); 1018 if (__predict_true(!(pmap->pm_flags & PMAP_DEFERRED_ACTIVATE))) { 1019 1020 /* 1021 * Flush the TLB for the given address. 1022 */ 1023 pmap_tlb_invalidate_addr(pmap, sva); 1024 } 1025 pmap_tlb_miss_lock_exit(); 1026 } 1027 1028 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 1029 1030 return false; 1031 } 1032 1033 void 1034 pmap_remove(pmap_t pmap, vaddr_t sva, vaddr_t eva) 1035 { 1036 const bool is_kernel_pmap_p = (pmap == pmap_kernel()); 1037 const pt_entry_t npte = pte_nv_entry(is_kernel_pmap_p); 1038 1039 UVMHIST_FUNC(__func__); 1040 UVMHIST_CALLARGS(pmaphist, "(pmap=%#jx, va=%#jx..%#jx)", 1041 (uintptr_t)pmap, sva, eva, 0); 1042 1043 if (is_kernel_pmap_p) { 1044 PMAP_COUNT(remove_kernel_calls); 1045 } else { 1046 PMAP_COUNT(remove_user_calls); 1047 } 1048 #ifdef PMAP_FAULTINFO 1049 curpcb->pcb_faultinfo.pfi_faultaddr = 0; 1050 curpcb->pcb_faultinfo.pfi_repeats = 0; 1051 curpcb->pcb_faultinfo.pfi_faultptep = NULL; 1052 #endif 1053 kpreempt_disable(); 1054 pmap_addr_range_check(pmap, sva, eva, __func__); 1055 pmap_pte_process(pmap, sva, eva, pmap_pte_remove, npte); 1056 kpreempt_enable(); 1057 1058 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 1059 } 1060 1061 /* 1062 * pmap_page_protect: 1063 * 1064 * Lower the permission for all mappings to a given page. 1065 */ 1066 void 1067 pmap_page_protect(struct vm_page *pg, vm_prot_t prot) 1068 { 1069 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 1070 pv_entry_t pv; 1071 vaddr_t va; 1072 1073 UVMHIST_FUNC(__func__); 1074 UVMHIST_CALLARGS(pmaphist, "(pg=%#jx (pa %#jx) prot=%#jx)", 1075 (uintptr_t)pg, VM_PAGE_TO_PHYS(pg), prot, 0); 1076 PMAP_COUNT(page_protect); 1077 1078 switch (prot) { 1079 case VM_PROT_READ|VM_PROT_WRITE: 1080 case VM_PROT_ALL: 1081 break; 1082 1083 /* copy_on_write */ 1084 case VM_PROT_READ: 1085 case VM_PROT_READ|VM_PROT_EXECUTE: 1086 pv = &mdpg->mdpg_first; 1087 kpreempt_disable(); 1088 VM_PAGEMD_PVLIST_READLOCK(mdpg); 1089 pmap_pvlist_check(mdpg); 1090 /* 1091 * Loop over all current mappings setting/clearing as 1092 * appropriate. 1093 */ 1094 if (pv->pv_pmap != NULL) { 1095 while (pv != NULL) { 1096 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 1097 if (PV_ISKENTER_P(pv)) { 1098 pv = pv->pv_next; 1099 continue; 1100 } 1101 #endif 1102 const pmap_t pmap = pv->pv_pmap; 1103 va = trunc_page(pv->pv_va); 1104 const uintptr_t gen = 1105 VM_PAGEMD_PVLIST_UNLOCK(mdpg); 1106 pmap_protect(pmap, va, va + PAGE_SIZE, prot); 1107 KASSERT(pv->pv_pmap == pmap); 1108 pmap_update(pmap); 1109 if (gen != VM_PAGEMD_PVLIST_READLOCK(mdpg)) { 1110 pv = &mdpg->mdpg_first; 1111 } else { 1112 pv = pv->pv_next; 1113 } 1114 pmap_pvlist_check(mdpg); 1115 } 1116 } 1117 pmap_pvlist_check(mdpg); 1118 VM_PAGEMD_PVLIST_UNLOCK(mdpg); 1119 kpreempt_enable(); 1120 break; 1121 1122 /* remove_all */ 1123 default: 1124 pmap_page_remove(mdpg); 1125 } 1126 1127 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 1128 } 1129 1130 static bool 1131 pmap_pte_protect(pmap_t pmap, vaddr_t sva, vaddr_t eva, pt_entry_t *ptep, 1132 uintptr_t flags) 1133 { 1134 const vm_prot_t prot = (flags & VM_PROT_ALL); 1135 1136 UVMHIST_FUNC(__func__); 1137 UVMHIST_CALLARGS(pmaphist, "(pmap=%#jx kernel=%jd va=%#jx..%#jx)", 1138 (uintptr_t)pmap, (pmap == pmap_kernel() ? 1 : 0), sva, eva); 1139 UVMHIST_LOG(pmaphist, "ptep=%#jx, flags(npte)=%#jx)", 1140 (uintptr_t)ptep, flags, 0, 0); 1141 1142 KASSERT(kpreempt_disabled()); 1143 /* 1144 * Change protection on every valid mapping within this segment. 1145 */ 1146 for (; sva < eva; sva += NBPG, ptep++) { 1147 pt_entry_t pte = *ptep; 1148 if (!pte_valid_p(pte)) 1149 continue; 1150 struct vm_page * const pg = PHYS_TO_VM_PAGE(pte_to_paddr(pte)); 1151 if (pg != NULL && pte_modified_p(pte)) { 1152 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 1153 if (VM_PAGEMD_EXECPAGE_P(mdpg)) { 1154 KASSERT(!VM_PAGEMD_PVLIST_EMPTY_P(mdpg)); 1155 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 1156 if (VM_PAGEMD_CACHED_P(mdpg)) { 1157 #endif 1158 UVMHIST_LOG(pmapexechist, 1159 "pg %#jx (pa %#jx): " 1160 "syncicached performed", 1161 (uintptr_t)pg, VM_PAGE_TO_PHYS(pg), 1162 0, 0); 1163 pmap_page_syncicache(pg); 1164 PMAP_COUNT(exec_synced_protect); 1165 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 1166 } 1167 #endif 1168 } 1169 } 1170 pte = pte_prot_downgrade(pte, prot); 1171 if (*ptep != pte) { 1172 pmap_tlb_miss_lock_enter(); 1173 pte_set(ptep, pte); 1174 /* 1175 * Update the TLB if needed. 1176 */ 1177 pmap_tlb_update_addr(pmap, sva, pte, PMAP_TLB_NEED_IPI); 1178 pmap_tlb_miss_lock_exit(); 1179 } 1180 } 1181 1182 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 1183 1184 return false; 1185 } 1186 1187 /* 1188 * Set the physical protection on the 1189 * specified range of this map as requested. 1190 */ 1191 void 1192 pmap_protect(pmap_t pmap, vaddr_t sva, vaddr_t eva, vm_prot_t prot) 1193 { 1194 UVMHIST_FUNC(__func__); 1195 UVMHIST_CALLARGS(pmaphist, "(pmap=%#jx, va=%#jx..%#jx, prot=%ju)", 1196 (uintptr_t)pmap, sva, eva, prot); 1197 PMAP_COUNT(protect); 1198 1199 if ((prot & VM_PROT_READ) == VM_PROT_NONE) { 1200 pmap_remove(pmap, sva, eva); 1201 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 1202 return; 1203 } 1204 1205 /* 1206 * Change protection on every valid mapping within this segment. 1207 */ 1208 kpreempt_disable(); 1209 pmap_addr_range_check(pmap, sva, eva, __func__); 1210 pmap_pte_process(pmap, sva, eva, pmap_pte_protect, prot); 1211 kpreempt_enable(); 1212 1213 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 1214 } 1215 1216 #if defined(PMAP_VIRTUAL_CACHE_ALIASES) && !defined(PMAP_NO_PV_UNCACHED) 1217 /* 1218 * pmap_page_cache: 1219 * 1220 * Change all mappings of a managed page to cached/uncached. 1221 */ 1222 void 1223 pmap_page_cache(struct vm_page_md *mdpg, bool cached) 1224 { 1225 #ifdef UVMHIST 1226 const bool vmpage_p = VM_PAGEMD_VMPAGE_P(mdpg); 1227 struct vm_page * const pg = vmpage_p ? VM_MD_TO_PAGE(mdpg) : NULL; 1228 #endif 1229 1230 UVMHIST_FUNC(__func__); 1231 UVMHIST_CALLARGS(pmaphist, "(mdpg=%#jx (pa %#jx) cached=%jd vmpage %jd)", 1232 (uintptr_t)mdpg, pg ? VM_PAGE_TO_PHYS(pg) : 0, cached, vmpage_p); 1233 1234 KASSERT(kpreempt_disabled()); 1235 KASSERT(VM_PAGEMD_PVLIST_LOCKED_P(mdpg)); 1236 1237 if (cached) { 1238 pmap_page_clear_attributes(mdpg, VM_PAGEMD_UNCACHED); 1239 PMAP_COUNT(page_cache_restorations); 1240 } else { 1241 pmap_page_set_attributes(mdpg, VM_PAGEMD_UNCACHED); 1242 PMAP_COUNT(page_cache_evictions); 1243 } 1244 1245 for (pv_entry_t pv = &mdpg->mdpg_first; pv != NULL; pv = pv->pv_next) { 1246 pmap_t pmap = pv->pv_pmap; 1247 vaddr_t va = trunc_page(pv->pv_va); 1248 1249 KASSERT(pmap != NULL); 1250 KASSERT(pmap != pmap_kernel() || !pmap_md_direct_mapped_vaddr_p(va)); 1251 pt_entry_t * const ptep = pmap_pte_lookup(pmap, va); 1252 if (ptep == NULL) 1253 continue; 1254 pt_entry_t pte = *ptep; 1255 if (pte_valid_p(pte)) { 1256 pte = pte_cached_change(pte, cached); 1257 pmap_tlb_miss_lock_enter(); 1258 pte_set(ptep, pte); 1259 pmap_tlb_update_addr(pmap, va, pte, PMAP_TLB_NEED_IPI); 1260 pmap_tlb_miss_lock_exit(); 1261 } 1262 } 1263 1264 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 1265 } 1266 #endif /* PMAP_VIRTUAL_CACHE_ALIASES && !PMAP_NO_PV_UNCACHED */ 1267 1268 /* 1269 * Insert the given physical page (p) at 1270 * the specified virtual address (v) in the 1271 * target physical map with the protection requested. 1272 * 1273 * If specified, the page will be wired down, meaning 1274 * that the related pte can not be reclaimed. 1275 * 1276 * NB: This is the only routine which MAY NOT lazy-evaluate 1277 * or lose information. That is, this routine must actually 1278 * insert this page into the given map NOW. 1279 */ 1280 int 1281 pmap_enter(pmap_t pmap, vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags) 1282 { 1283 const bool wired = (flags & PMAP_WIRED) != 0; 1284 const bool is_kernel_pmap_p = (pmap == pmap_kernel()); 1285 u_int update_flags = (flags & VM_PROT_ALL) != 0 ? PMAP_TLB_INSERT : 0; 1286 #ifdef UVMHIST 1287 struct kern_history * const histp = 1288 ((prot & VM_PROT_EXECUTE) ? &pmapexechist : &pmaphist); 1289 #endif 1290 1291 UVMHIST_FUNC(__func__); 1292 UVMHIST_CALLARGS(*histp, "(pmap=%#jx, va=%#jx, pa=%#jx", 1293 (uintptr_t)pmap, va, pa, 0); 1294 UVMHIST_LOG(*histp, "prot=%#jx flags=%#jx)", prot, flags, 0, 0); 1295 1296 const bool good_color = PMAP_PAGE_COLOROK_P(pa, va); 1297 if (is_kernel_pmap_p) { 1298 PMAP_COUNT(kernel_mappings); 1299 if (!good_color) 1300 PMAP_COUNT(kernel_mappings_bad); 1301 } else { 1302 PMAP_COUNT(user_mappings); 1303 if (!good_color) 1304 PMAP_COUNT(user_mappings_bad); 1305 } 1306 pmap_addr_range_check(pmap, va, va, __func__); 1307 1308 KASSERTMSG(prot & VM_PROT_READ, "no READ (%#x) in prot %#x", 1309 VM_PROT_READ, prot); 1310 1311 struct vm_page * const pg = PHYS_TO_VM_PAGE(pa); 1312 struct vm_page_md * const mdpg = (pg ? VM_PAGE_TO_MD(pg) : NULL); 1313 1314 struct vm_page_md *mdpp = NULL; 1315 #ifdef __HAVE_PMAP_PV_TRACK 1316 struct pmap_page *pp = pmap_pv_tracked(pa); 1317 mdpp = pp ? PMAP_PAGE_TO_MD(pp) : NULL; 1318 #endif 1319 1320 if (mdpg) { 1321 /* Set page referenced/modified status based on flags */ 1322 if (flags & VM_PROT_WRITE) { 1323 pmap_page_set_attributes(mdpg, VM_PAGEMD_MODIFIED|VM_PAGEMD_REFERENCED); 1324 } else if (flags & VM_PROT_ALL) { 1325 pmap_page_set_attributes(mdpg, VM_PAGEMD_REFERENCED); 1326 } 1327 1328 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 1329 if (!VM_PAGEMD_CACHED_P(mdpg)) { 1330 flags |= PMAP_NOCACHE; 1331 PMAP_COUNT(uncached_mappings); 1332 } 1333 #endif 1334 1335 PMAP_COUNT(managed_mappings); 1336 } else if (mdpp) { 1337 #ifdef __HAVE_PMAP_PV_TRACK 1338 pmap_page_set_attributes(mdpg, VM_PAGEMD_REFERENCED); 1339 1340 PMAP_COUNT(pvtracked_mappings); 1341 #endif 1342 } else { 1343 /* 1344 * Assumption: if it is not part of our managed memory 1345 * then it must be device memory which may be volatile. 1346 */ 1347 if ((flags & PMAP_CACHE_MASK) == 0) 1348 flags |= PMAP_NOCACHE; 1349 PMAP_COUNT(unmanaged_mappings); 1350 } 1351 1352 KASSERTMSG(mdpg == NULL || mdpp == NULL, "mdpg %p mdpp %p", mdpg, mdpp); 1353 1354 struct vm_page_md *md = (mdpg != NULL) ? mdpg : mdpp; 1355 pt_entry_t npte = pte_make_enter(pa, md, prot, flags, 1356 is_kernel_pmap_p); 1357 1358 kpreempt_disable(); 1359 1360 pt_entry_t * const ptep = pmap_pte_reserve(pmap, va, flags); 1361 if (__predict_false(ptep == NULL)) { 1362 kpreempt_enable(); 1363 UVMHIST_LOG(*histp, " <-- ENOMEM", 0, 0, 0, 0); 1364 return ENOMEM; 1365 } 1366 const pt_entry_t opte = *ptep; 1367 const bool resident = pte_valid_p(opte); 1368 bool remap = false; 1369 if (resident) { 1370 if (pte_to_paddr(opte) != pa) { 1371 KASSERT(!is_kernel_pmap_p); 1372 const pt_entry_t rpte = pte_nv_entry(false); 1373 1374 pmap_addr_range_check(pmap, va, va + NBPG, __func__); 1375 pmap_pte_process(pmap, va, va + NBPG, pmap_pte_remove, 1376 rpte); 1377 PMAP_COUNT(user_mappings_changed); 1378 remap = true; 1379 } 1380 update_flags |= PMAP_TLB_NEED_IPI; 1381 } 1382 1383 if (!resident || remap) { 1384 pmap->pm_stats.resident_count++; 1385 } 1386 1387 /* Done after case that may sleep/return. */ 1388 if (md) 1389 pmap_enter_pv(pmap, va, pa, md, &npte, 0); 1390 1391 /* 1392 * Now validate mapping with desired protection/wiring. 1393 */ 1394 if (wired) { 1395 pmap->pm_stats.wired_count++; 1396 npte = pte_wire_entry(npte); 1397 } 1398 1399 UVMHIST_LOG(*histp, "new pte %#jx (pa %#jx)", 1400 pte_value(npte), pa, 0, 0); 1401 1402 KASSERT(pte_valid_p(npte)); 1403 1404 pmap_tlb_miss_lock_enter(); 1405 pte_set(ptep, npte); 1406 pmap_tlb_update_addr(pmap, va, npte, update_flags); 1407 pmap_tlb_miss_lock_exit(); 1408 kpreempt_enable(); 1409 1410 if (pg != NULL && (prot == (VM_PROT_READ | VM_PROT_EXECUTE))) { 1411 KASSERT(mdpg != NULL); 1412 PMAP_COUNT(exec_mappings); 1413 if (!VM_PAGEMD_EXECPAGE_P(mdpg) && pte_cached_p(npte)) { 1414 if (!pte_deferred_exec_p(npte)) { 1415 UVMHIST_LOG(*histp, "va=%#jx pg %#jx: " 1416 "immediate syncicache", 1417 va, (uintptr_t)pg, 0, 0); 1418 pmap_page_syncicache(pg); 1419 pmap_page_set_attributes(mdpg, 1420 VM_PAGEMD_EXECPAGE); 1421 PMAP_COUNT(exec_synced_mappings); 1422 } else { 1423 UVMHIST_LOG(*histp, "va=%#jx pg %#jx: defer " 1424 "syncicache: pte %#jx", 1425 va, (uintptr_t)pg, npte, 0); 1426 } 1427 } else { 1428 UVMHIST_LOG(*histp, 1429 "va=%#jx pg %#jx: no syncicache cached %jd", 1430 va, (uintptr_t)pg, pte_cached_p(npte), 0); 1431 } 1432 } else if (pg != NULL && (prot & VM_PROT_EXECUTE)) { 1433 KASSERT(mdpg != NULL); 1434 KASSERT(prot & VM_PROT_WRITE); 1435 PMAP_COUNT(exec_mappings); 1436 pmap_page_syncicache(pg); 1437 pmap_page_clear_attributes(mdpg, VM_PAGEMD_EXECPAGE); 1438 UVMHIST_LOG(*histp, 1439 "va=%#jx pg %#jx: immediate syncicache (writeable)", 1440 va, (uintptr_t)pg, 0, 0); 1441 } 1442 1443 UVMHIST_LOG(*histp, " <-- 0 (OK)", 0, 0, 0, 0); 1444 return 0; 1445 } 1446 1447 void 1448 pmap_kenter_pa(vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags) 1449 { 1450 pmap_t pmap = pmap_kernel(); 1451 struct vm_page * const pg = PHYS_TO_VM_PAGE(pa); 1452 struct vm_page_md * const mdpg = (pg ? VM_PAGE_TO_MD(pg) : NULL); 1453 1454 UVMHIST_FUNC(__func__); 1455 UVMHIST_CALLARGS(pmaphist, "(va=%#jx pa=%#jx prot=%ju, flags=%#jx)", 1456 va, pa, prot, flags); 1457 PMAP_COUNT(kenter_pa); 1458 1459 if (mdpg == NULL) { 1460 PMAP_COUNT(kenter_pa_unmanaged); 1461 if ((flags & PMAP_CACHE_MASK) == 0) 1462 flags |= PMAP_NOCACHE; 1463 } else { 1464 if ((flags & PMAP_NOCACHE) == 0 && !PMAP_PAGE_COLOROK_P(pa, va)) 1465 PMAP_COUNT(kenter_pa_bad); 1466 } 1467 1468 pt_entry_t npte = pte_make_kenter_pa(pa, mdpg, prot, flags); 1469 kpreempt_disable(); 1470 pt_entry_t * const ptep = pmap_pte_lookup(pmap, va); 1471 KASSERTMSG(ptep != NULL, "%#"PRIxVADDR " %#"PRIxVADDR, va, 1472 pmap_limits.virtual_end); 1473 KASSERT(!pte_valid_p(*ptep)); 1474 1475 /* 1476 * No need to track non-managed pages or PMAP_KMPAGEs pages for aliases 1477 */ 1478 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 1479 if (pg != NULL && (flags & PMAP_KMPAGE) == 0 1480 && pmap_md_virtual_cache_aliasing_p()) { 1481 pmap_enter_pv(pmap, va, pa, mdpg, &npte, PV_KENTER); 1482 } 1483 #endif 1484 1485 /* 1486 * We have the option to force this mapping into the TLB but we 1487 * don't. Instead let the next reference to the page do it. 1488 */ 1489 pmap_tlb_miss_lock_enter(); 1490 pte_set(ptep, npte); 1491 pmap_tlb_update_addr(pmap_kernel(), va, npte, 0); 1492 pmap_tlb_miss_lock_exit(); 1493 kpreempt_enable(); 1494 #if DEBUG > 1 1495 for (u_int i = 0; i < PAGE_SIZE / sizeof(long); i++) { 1496 if (((long *)va)[i] != ((long *)pa)[i]) 1497 panic("%s: contents (%lx) of va %#"PRIxVADDR 1498 " != contents (%lx) of pa %#"PRIxPADDR, __func__, 1499 ((long *)va)[i], va, ((long *)pa)[i], pa); 1500 } 1501 #endif 1502 1503 UVMHIST_LOG(pmaphist, " <-- done (ptep=%#jx)", (uintptr_t)ptep, 0, 0, 1504 0); 1505 } 1506 1507 /* 1508 * Remove the given range of addresses from the kernel map. 1509 * 1510 * It is assumed that the start and end are properly 1511 * rounded to the page size. 1512 */ 1513 1514 static bool 1515 pmap_pte_kremove(pmap_t pmap, vaddr_t sva, vaddr_t eva, pt_entry_t *ptep, 1516 uintptr_t flags) 1517 { 1518 const pt_entry_t new_pte = pte_nv_entry(true); 1519 1520 UVMHIST_FUNC(__func__); 1521 UVMHIST_CALLARGS(pmaphist, "(pmap=%#jx, sva=%#jx eva=%#jx ptep=%#jx)", 1522 (uintptr_t)pmap, sva, eva, (uintptr_t)ptep); 1523 1524 KASSERT(kpreempt_disabled()); 1525 1526 for (; sva < eva; sva += NBPG, ptep++) { 1527 pt_entry_t pte = *ptep; 1528 if (!pte_valid_p(pte)) 1529 continue; 1530 1531 PMAP_COUNT(kremove_pages); 1532 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 1533 struct vm_page * const pg = PHYS_TO_VM_PAGE(pte_to_paddr(pte)); 1534 if (pg != NULL && pmap_md_virtual_cache_aliasing_p()) { 1535 pmap_remove_pv(pmap, sva, pg, !pte_readonly_p(pte)); 1536 } 1537 #endif 1538 1539 pmap_tlb_miss_lock_enter(); 1540 pte_set(ptep, new_pte); 1541 pmap_tlb_invalidate_addr(pmap, sva); 1542 pmap_tlb_miss_lock_exit(); 1543 } 1544 1545 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 1546 1547 return false; 1548 } 1549 1550 void 1551 pmap_kremove(vaddr_t va, vsize_t len) 1552 { 1553 const vaddr_t sva = trunc_page(va); 1554 const vaddr_t eva = round_page(va + len); 1555 1556 UVMHIST_FUNC(__func__); 1557 UVMHIST_CALLARGS(pmaphist, "(va=%#jx len=%#jx)", va, len, 0, 0); 1558 1559 kpreempt_disable(); 1560 pmap_pte_process(pmap_kernel(), sva, eva, pmap_pte_kremove, 0); 1561 kpreempt_enable(); 1562 1563 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 1564 } 1565 1566 bool 1567 pmap_remove_all(struct pmap *pmap) 1568 { 1569 UVMHIST_FUNC(__func__); 1570 UVMHIST_CALLARGS(pmaphist, "(pm=%#jx)", (uintptr_t)pmap, 0, 0, 0); 1571 1572 KASSERT(pmap != pmap_kernel()); 1573 1574 kpreempt_disable(); 1575 /* 1576 * Free all of our ASIDs which means we can skip doing all the 1577 * tlb_invalidate_addrs(). 1578 */ 1579 pmap_tlb_miss_lock_enter(); 1580 #ifdef MULTIPROCESSOR 1581 // This should be the last CPU with this pmap onproc 1582 KASSERT(!kcpuset_isotherset(pmap->pm_onproc, cpu_index(curcpu()))); 1583 if (kcpuset_isset(pmap->pm_onproc, cpu_index(curcpu()))) 1584 #endif 1585 pmap_tlb_asid_deactivate(pmap); 1586 #ifdef MULTIPROCESSOR 1587 KASSERT(kcpuset_iszero(pmap->pm_onproc)); 1588 #endif 1589 pmap_tlb_asid_release_all(pmap); 1590 pmap_tlb_miss_lock_exit(); 1591 pmap->pm_flags |= PMAP_DEFERRED_ACTIVATE; 1592 1593 #ifdef PMAP_FAULTINFO 1594 curpcb->pcb_faultinfo.pfi_faultaddr = 0; 1595 curpcb->pcb_faultinfo.pfi_repeats = 0; 1596 curpcb->pcb_faultinfo.pfi_faultptep = NULL; 1597 #endif 1598 kpreempt_enable(); 1599 1600 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 1601 return false; 1602 } 1603 1604 /* 1605 * Routine: pmap_unwire 1606 * Function: Clear the wired attribute for a map/virtual-address 1607 * pair. 1608 * In/out conditions: 1609 * The mapping must already exist in the pmap. 1610 */ 1611 void 1612 pmap_unwire(pmap_t pmap, vaddr_t va) 1613 { 1614 UVMHIST_FUNC(__func__); 1615 UVMHIST_CALLARGS(pmaphist, "(pmap=%#jx, va=%#jx)", (uintptr_t)pmap, va, 1616 0, 0); 1617 PMAP_COUNT(unwire); 1618 1619 /* 1620 * Don't need to flush the TLB since PG_WIRED is only in software. 1621 */ 1622 kpreempt_disable(); 1623 pmap_addr_range_check(pmap, va, va, __func__); 1624 pt_entry_t * const ptep = pmap_pte_lookup(pmap, va); 1625 KASSERTMSG(ptep != NULL, "pmap %p va %#"PRIxVADDR" invalid STE", 1626 pmap, va); 1627 pt_entry_t pte = *ptep; 1628 KASSERTMSG(pte_valid_p(pte), 1629 "pmap %p va %#" PRIxVADDR " invalid PTE %#" PRIxPTE " @ %p", 1630 pmap, va, pte_value(pte), ptep); 1631 1632 if (pte_wired_p(pte)) { 1633 pmap_tlb_miss_lock_enter(); 1634 pte_set(ptep, pte_unwire_entry(pte)); 1635 pmap_tlb_miss_lock_exit(); 1636 pmap->pm_stats.wired_count--; 1637 } 1638 #ifdef DIAGNOSTIC 1639 else { 1640 printf("%s: wiring for pmap %p va %#"PRIxVADDR" unchanged!\n", 1641 __func__, pmap, va); 1642 } 1643 #endif 1644 kpreempt_enable(); 1645 1646 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 1647 } 1648 1649 /* 1650 * Routine: pmap_extract 1651 * Function: 1652 * Extract the physical page address associated 1653 * with the given map/virtual_address pair. 1654 */ 1655 bool 1656 pmap_extract(pmap_t pmap, vaddr_t va, paddr_t *pap) 1657 { 1658 paddr_t pa; 1659 1660 if (pmap == pmap_kernel()) { 1661 if (pmap_md_direct_mapped_vaddr_p(va)) { 1662 pa = pmap_md_direct_mapped_vaddr_to_paddr(va); 1663 goto done; 1664 } 1665 if (pmap_md_io_vaddr_p(va)) 1666 panic("pmap_extract: io address %#"PRIxVADDR"", va); 1667 1668 if (va >= pmap_limits.virtual_end) 1669 panic("%s: illegal kernel mapped address %#"PRIxVADDR, 1670 __func__, va); 1671 } 1672 kpreempt_disable(); 1673 const pt_entry_t * const ptep = pmap_pte_lookup(pmap, va); 1674 if (ptep == NULL || !pte_valid_p(*ptep)) { 1675 kpreempt_enable(); 1676 return false; 1677 } 1678 pa = pte_to_paddr(*ptep) | (va & PGOFSET); 1679 kpreempt_enable(); 1680 done: 1681 if (pap != NULL) { 1682 *pap = pa; 1683 } 1684 return true; 1685 } 1686 1687 /* 1688 * Copy the range specified by src_addr/len 1689 * from the source map to the range dst_addr/len 1690 * in the destination map. 1691 * 1692 * This routine is only advisory and need not do anything. 1693 */ 1694 void 1695 pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vaddr_t dst_addr, vsize_t len, 1696 vaddr_t src_addr) 1697 { 1698 UVMHIST_FUNC(__func__); 1699 UVMHIST_CALLED(pmaphist); 1700 PMAP_COUNT(copy); 1701 } 1702 1703 /* 1704 * pmap_clear_reference: 1705 * 1706 * Clear the reference bit on the specified physical page. 1707 */ 1708 bool 1709 pmap_clear_reference(struct vm_page *pg) 1710 { 1711 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 1712 1713 UVMHIST_FUNC(__func__); 1714 UVMHIST_CALLARGS(pmaphist, "(pg=%#jx (pa %#jx))", 1715 (uintptr_t)pg, VM_PAGE_TO_PHYS(pg), 0,0); 1716 1717 bool rv = pmap_page_clear_attributes(mdpg, VM_PAGEMD_REFERENCED); 1718 1719 UVMHIST_LOG(pmaphist, " <-- wasref %ju", rv, 0, 0, 0); 1720 1721 return rv; 1722 } 1723 1724 /* 1725 * pmap_is_referenced: 1726 * 1727 * Return whether or not the specified physical page is referenced 1728 * by any physical maps. 1729 */ 1730 bool 1731 pmap_is_referenced(struct vm_page *pg) 1732 { 1733 return VM_PAGEMD_REFERENCED_P(VM_PAGE_TO_MD(pg)); 1734 } 1735 1736 /* 1737 * Clear the modify bits on the specified physical page. 1738 */ 1739 bool 1740 pmap_clear_modify(struct vm_page *pg) 1741 { 1742 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 1743 pv_entry_t pv = &mdpg->mdpg_first; 1744 pv_entry_t pv_next; 1745 1746 UVMHIST_FUNC(__func__); 1747 UVMHIST_CALLARGS(pmaphist, "(pg=%#jx (%#jx))", 1748 (uintptr_t)pg, VM_PAGE_TO_PHYS(pg), 0,0); 1749 PMAP_COUNT(clear_modify); 1750 1751 if (VM_PAGEMD_EXECPAGE_P(mdpg)) { 1752 if (pv->pv_pmap == NULL) { 1753 UVMHIST_LOG(pmapexechist, 1754 "pg %#jx (pa %#jx): execpage cleared", 1755 (uintptr_t)pg, VM_PAGE_TO_PHYS(pg), 0, 0); 1756 pmap_page_clear_attributes(mdpg, VM_PAGEMD_EXECPAGE); 1757 PMAP_COUNT(exec_uncached_clear_modify); 1758 } else { 1759 UVMHIST_LOG(pmapexechist, 1760 "pg %#jx (pa %#jx): syncicache performed", 1761 (uintptr_t)pg, VM_PAGE_TO_PHYS(pg), 0, 0); 1762 pmap_page_syncicache(pg); 1763 PMAP_COUNT(exec_synced_clear_modify); 1764 } 1765 } 1766 if (!pmap_page_clear_attributes(mdpg, VM_PAGEMD_MODIFIED)) { 1767 UVMHIST_LOG(pmaphist, " <-- false", 0, 0, 0, 0); 1768 return false; 1769 } 1770 if (pv->pv_pmap == NULL) { 1771 UVMHIST_LOG(pmaphist, " <-- true (no mappings)", 0, 0, 0, 0); 1772 return true; 1773 } 1774 1775 /* 1776 * remove write access from any pages that are dirty 1777 * so we can tell if they are written to again later. 1778 * flush the VAC first if there is one. 1779 */ 1780 kpreempt_disable(); 1781 VM_PAGEMD_PVLIST_READLOCK(mdpg); 1782 pmap_pvlist_check(mdpg); 1783 for (; pv != NULL; pv = pv_next) { 1784 pmap_t pmap = pv->pv_pmap; 1785 vaddr_t va = trunc_page(pv->pv_va); 1786 1787 pv_next = pv->pv_next; 1788 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 1789 if (PV_ISKENTER_P(pv)) 1790 continue; 1791 #endif 1792 pt_entry_t * const ptep = pmap_pte_lookup(pmap, va); 1793 KASSERT(ptep); 1794 pt_entry_t pte = pte_prot_nowrite(*ptep); 1795 if (*ptep == pte) { 1796 continue; 1797 } 1798 KASSERT(pte_valid_p(pte)); 1799 const uintptr_t gen = VM_PAGEMD_PVLIST_UNLOCK(mdpg); 1800 pmap_tlb_miss_lock_enter(); 1801 pte_set(ptep, pte); 1802 pmap_tlb_invalidate_addr(pmap, va); 1803 pmap_tlb_miss_lock_exit(); 1804 pmap_update(pmap); 1805 if (__predict_false(gen != VM_PAGEMD_PVLIST_READLOCK(mdpg))) { 1806 /* 1807 * The list changed! So restart from the beginning. 1808 */ 1809 pv_next = &mdpg->mdpg_first; 1810 pmap_pvlist_check(mdpg); 1811 } 1812 } 1813 pmap_pvlist_check(mdpg); 1814 VM_PAGEMD_PVLIST_UNLOCK(mdpg); 1815 kpreempt_enable(); 1816 1817 UVMHIST_LOG(pmaphist, " <-- true (mappings changed)", 0, 0, 0, 0); 1818 return true; 1819 } 1820 1821 /* 1822 * pmap_is_modified: 1823 * 1824 * Return whether or not the specified physical page is modified 1825 * by any physical maps. 1826 */ 1827 bool 1828 pmap_is_modified(struct vm_page *pg) 1829 { 1830 return VM_PAGEMD_MODIFIED_P(VM_PAGE_TO_MD(pg)); 1831 } 1832 1833 /* 1834 * pmap_set_modified: 1835 * 1836 * Sets the page modified reference bit for the specified page. 1837 */ 1838 void 1839 pmap_set_modified(paddr_t pa) 1840 { 1841 struct vm_page * const pg = PHYS_TO_VM_PAGE(pa); 1842 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 1843 pmap_page_set_attributes(mdpg, VM_PAGEMD_MODIFIED|VM_PAGEMD_REFERENCED); 1844 } 1845 1846 /******************** pv_entry management ********************/ 1847 1848 static void 1849 pmap_pvlist_check(struct vm_page_md *mdpg) 1850 { 1851 #ifdef DEBUG 1852 pv_entry_t pv = &mdpg->mdpg_first; 1853 if (pv->pv_pmap != NULL) { 1854 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 1855 const u_int colormask = uvmexp.colormask; 1856 u_int colors = 0; 1857 #endif 1858 for (; pv != NULL; pv = pv->pv_next) { 1859 KASSERT(pv->pv_pmap != pmap_kernel() || !pmap_md_direct_mapped_vaddr_p(pv->pv_va)); 1860 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 1861 colors |= __BIT(atop(pv->pv_va) & colormask); 1862 #endif 1863 } 1864 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 1865 // Assert that if there is more than 1 color mapped, that the 1866 // page is uncached. 1867 KASSERTMSG(!pmap_md_virtual_cache_aliasing_p() 1868 || colors == 0 || (colors & (colors-1)) == 0 1869 || VM_PAGEMD_UNCACHED_P(mdpg), "colors=%#x uncached=%u", 1870 colors, VM_PAGEMD_UNCACHED_P(mdpg)); 1871 #endif 1872 } else { 1873 KASSERT(pv->pv_next == NULL); 1874 } 1875 #endif /* DEBUG */ 1876 } 1877 1878 /* 1879 * Enter the pmap and virtual address into the 1880 * physical to virtual map table. 1881 */ 1882 void 1883 pmap_enter_pv(pmap_t pmap, vaddr_t va, paddr_t pa, struct vm_page_md *mdpg, 1884 pt_entry_t *nptep, u_int flags) 1885 { 1886 pv_entry_t pv, npv, apv; 1887 #ifdef UVMHIST 1888 bool first = false; 1889 struct vm_page *pg = VM_PAGEMD_VMPAGE_P(mdpg) ? VM_MD_TO_PAGE(mdpg) : 1890 NULL; 1891 #endif 1892 1893 UVMHIST_FUNC(__func__); 1894 UVMHIST_CALLARGS(pmaphist, "(pmap=%#jx va=%#jx pg=%#jx (%#jx)", 1895 (uintptr_t)pmap, va, (uintptr_t)pg, pa); 1896 UVMHIST_LOG(pmaphist, "nptep=%#jx (%#jx))", 1897 (uintptr_t)nptep, pte_value(*nptep), 0, 0); 1898 1899 KASSERT(kpreempt_disabled()); 1900 KASSERT(pmap != pmap_kernel() || !pmap_md_direct_mapped_vaddr_p(va)); 1901 KASSERTMSG(pmap != pmap_kernel() || !pmap_md_io_vaddr_p(va), 1902 "va %#"PRIxVADDR, va); 1903 1904 apv = NULL; 1905 VM_PAGEMD_PVLIST_LOCK(mdpg); 1906 again: 1907 pv = &mdpg->mdpg_first; 1908 pmap_pvlist_check(mdpg); 1909 if (pv->pv_pmap == NULL) { 1910 KASSERT(pv->pv_next == NULL); 1911 /* 1912 * No entries yet, use header as the first entry 1913 */ 1914 PMAP_COUNT(primary_mappings); 1915 PMAP_COUNT(mappings); 1916 #ifdef UVMHIST 1917 first = true; 1918 #endif 1919 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 1920 KASSERT(VM_PAGEMD_CACHED_P(mdpg)); 1921 // If the new mapping has an incompatible color the last 1922 // mapping of this page, clean the page before using it. 1923 if (!PMAP_PAGE_COLOROK_P(va, pv->pv_va)) { 1924 pmap_md_vca_clean(mdpg, PMAP_WBINV); 1925 } 1926 #endif 1927 pv->pv_pmap = pmap; 1928 pv->pv_va = va | flags; 1929 } else { 1930 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 1931 if (pmap_md_vca_add(mdpg, va, nptep)) { 1932 goto again; 1933 } 1934 #endif 1935 1936 /* 1937 * There is at least one other VA mapping this page. 1938 * Place this entry after the header. 1939 * 1940 * Note: the entry may already be in the table if 1941 * we are only changing the protection bits. 1942 */ 1943 1944 for (npv = pv; npv; npv = npv->pv_next) { 1945 if (pmap == npv->pv_pmap 1946 && va == trunc_page(npv->pv_va)) { 1947 #ifdef PARANOIADIAG 1948 pt_entry_t *ptep = pmap_pte_lookup(pmap, va); 1949 pt_entry_t pte = (ptep != NULL) ? *ptep : 0; 1950 if (!pte_valid_p(pte) || pte_to_paddr(pte) != pa) 1951 printf("%s: found va %#"PRIxVADDR 1952 " pa %#"PRIxPADDR 1953 " in pv_table but != %#"PRIxPTE"\n", 1954 __func__, va, pa, pte_value(pte)); 1955 #endif 1956 PMAP_COUNT(remappings); 1957 VM_PAGEMD_PVLIST_UNLOCK(mdpg); 1958 if (__predict_false(apv != NULL)) 1959 pmap_pv_free(apv); 1960 1961 UVMHIST_LOG(pmaphist, 1962 " <-- done pv=%#jx (reused)", 1963 (uintptr_t)pv, 0, 0, 0); 1964 return; 1965 } 1966 } 1967 if (__predict_true(apv == NULL)) { 1968 /* 1969 * To allocate a PV, we have to release the PVLIST lock 1970 * so get the page generation. We allocate the PV, and 1971 * then reacquire the lock. 1972 */ 1973 pmap_pvlist_check(mdpg); 1974 const uintptr_t gen = VM_PAGEMD_PVLIST_UNLOCK(mdpg); 1975 1976 apv = (pv_entry_t)pmap_pv_alloc(); 1977 if (apv == NULL) 1978 panic("pmap_enter_pv: pmap_pv_alloc() failed"); 1979 1980 /* 1981 * If the generation has changed, then someone else 1982 * tinkered with this page so we should start over. 1983 */ 1984 if (gen != VM_PAGEMD_PVLIST_LOCK(mdpg)) 1985 goto again; 1986 } 1987 npv = apv; 1988 apv = NULL; 1989 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 1990 /* 1991 * If need to deal with virtual cache aliases, keep mappings 1992 * in the kernel pmap at the head of the list. This allows 1993 * the VCA code to easily use them for cache operations if 1994 * present. 1995 */ 1996 pmap_t kpmap = pmap_kernel(); 1997 if (pmap != kpmap) { 1998 while (pv->pv_pmap == kpmap && pv->pv_next != NULL) { 1999 pv = pv->pv_next; 2000 } 2001 } 2002 #endif 2003 npv->pv_va = va | flags; 2004 npv->pv_pmap = pmap; 2005 npv->pv_next = pv->pv_next; 2006 pv->pv_next = npv; 2007 PMAP_COUNT(mappings); 2008 } 2009 pmap_pvlist_check(mdpg); 2010 VM_PAGEMD_PVLIST_UNLOCK(mdpg); 2011 if (__predict_false(apv != NULL)) 2012 pmap_pv_free(apv); 2013 2014 UVMHIST_LOG(pmaphist, " <-- done pv=%#jx (first %ju)", (uintptr_t)pv, 2015 first, 0, 0); 2016 } 2017 2018 /* 2019 * Remove a physical to virtual address translation. 2020 * If cache was inhibited on this page, and there are no more cache 2021 * conflicts, restore caching. 2022 * Flush the cache if the last page is removed (should always be cached 2023 * at this point). 2024 */ 2025 void 2026 pmap_remove_pv(pmap_t pmap, vaddr_t va, struct vm_page *pg, bool dirty) 2027 { 2028 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 2029 pv_entry_t pv, npv; 2030 bool last; 2031 2032 UVMHIST_FUNC(__func__); 2033 UVMHIST_CALLARGS(pmaphist, "(pmap=%#jx, va=%#jx, pg=%#jx (pa %#jx)", 2034 (uintptr_t)pmap, va, (uintptr_t)pg, VM_PAGE_TO_PHYS(pg)); 2035 UVMHIST_LOG(pmaphist, "dirty=%ju)", dirty, 0, 0, 0); 2036 2037 KASSERT(kpreempt_disabled()); 2038 KASSERT((va & PAGE_MASK) == 0); 2039 pv = &mdpg->mdpg_first; 2040 2041 VM_PAGEMD_PVLIST_LOCK(mdpg); 2042 pmap_pvlist_check(mdpg); 2043 2044 /* 2045 * If it is the first entry on the list, it is actually 2046 * in the header and we must copy the following entry up 2047 * to the header. Otherwise we must search the list for 2048 * the entry. In either case we free the now unused entry. 2049 */ 2050 2051 last = false; 2052 if (pmap == pv->pv_pmap && va == trunc_page(pv->pv_va)) { 2053 npv = pv->pv_next; 2054 if (npv) { 2055 *pv = *npv; 2056 KASSERT(pv->pv_pmap != NULL); 2057 } else { 2058 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 2059 pmap_page_clear_attributes(mdpg, VM_PAGEMD_UNCACHED); 2060 #endif 2061 pv->pv_pmap = NULL; 2062 last = true; /* Last mapping removed */ 2063 } 2064 PMAP_COUNT(remove_pvfirst); 2065 } else { 2066 for (npv = pv->pv_next; npv; pv = npv, npv = npv->pv_next) { 2067 PMAP_COUNT(remove_pvsearch); 2068 if (pmap == npv->pv_pmap && va == trunc_page(npv->pv_va)) 2069 break; 2070 } 2071 if (npv) { 2072 pv->pv_next = npv->pv_next; 2073 } 2074 } 2075 2076 pmap_pvlist_check(mdpg); 2077 VM_PAGEMD_PVLIST_UNLOCK(mdpg); 2078 2079 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 2080 pmap_md_vca_remove(pg, va, dirty, last); 2081 #endif 2082 2083 /* 2084 * Free the pv_entry if needed. 2085 */ 2086 if (npv) 2087 pmap_pv_free(npv); 2088 if (VM_PAGEMD_EXECPAGE_P(mdpg) && dirty) { 2089 if (last) { 2090 /* 2091 * If this was the page's last mapping, we no longer 2092 * care about its execness. 2093 */ 2094 UVMHIST_LOG(pmapexechist, 2095 "pg %#jx (pa %#jx)last %ju: execpage cleared", 2096 (uintptr_t)pg, VM_PAGE_TO_PHYS(pg), last, 0); 2097 pmap_page_clear_attributes(mdpg, VM_PAGEMD_EXECPAGE); 2098 PMAP_COUNT(exec_uncached_remove); 2099 } else { 2100 /* 2101 * Someone still has it mapped as an executable page 2102 * so we must sync it. 2103 */ 2104 UVMHIST_LOG(pmapexechist, 2105 "pg %#jx (pa %#jx) last %ju: performed syncicache", 2106 (uintptr_t)pg, VM_PAGE_TO_PHYS(pg), last, 0); 2107 pmap_page_syncicache(pg); 2108 PMAP_COUNT(exec_synced_remove); 2109 } 2110 } 2111 2112 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 2113 } 2114 2115 #if defined(MULTIPROCESSOR) 2116 struct pmap_pvlist_info { 2117 kmutex_t *pli_locks[PAGE_SIZE / 32]; 2118 volatile u_int pli_lock_refs[PAGE_SIZE / 32]; 2119 volatile u_int pli_lock_index; 2120 u_int pli_lock_mask; 2121 } pmap_pvlist_info; 2122 2123 void 2124 pmap_pvlist_lock_init(size_t cache_line_size) 2125 { 2126 struct pmap_pvlist_info * const pli = &pmap_pvlist_info; 2127 const vaddr_t lock_page = uvm_pageboot_alloc(PAGE_SIZE); 2128 vaddr_t lock_va = lock_page; 2129 if (sizeof(kmutex_t) > cache_line_size) { 2130 cache_line_size = roundup2(sizeof(kmutex_t), cache_line_size); 2131 } 2132 const size_t nlocks = PAGE_SIZE / cache_line_size; 2133 KASSERT((nlocks & (nlocks - 1)) == 0); 2134 /* 2135 * Now divide the page into a number of mutexes, one per cacheline. 2136 */ 2137 for (size_t i = 0; i < nlocks; lock_va += cache_line_size, i++) { 2138 kmutex_t * const lock = (kmutex_t *)lock_va; 2139 mutex_init(lock, MUTEX_DEFAULT, IPL_HIGH); 2140 pli->pli_locks[i] = lock; 2141 } 2142 pli->pli_lock_mask = nlocks - 1; 2143 } 2144 2145 kmutex_t * 2146 pmap_pvlist_lock_addr(struct vm_page_md *mdpg) 2147 { 2148 struct pmap_pvlist_info * const pli = &pmap_pvlist_info; 2149 kmutex_t *lock = mdpg->mdpg_lock; 2150 2151 /* 2152 * Allocate a lock on an as-needed basis. This will hopefully give us 2153 * semi-random distribution not based on page color. 2154 */ 2155 if (__predict_false(lock == NULL)) { 2156 size_t locknum = atomic_add_int_nv(&pli->pli_lock_index, 37); 2157 size_t lockid = locknum & pli->pli_lock_mask; 2158 kmutex_t * const new_lock = pli->pli_locks[lockid]; 2159 /* 2160 * Set the lock. If some other thread already did, just use 2161 * the one they assigned. 2162 */ 2163 lock = atomic_cas_ptr(&mdpg->mdpg_lock, NULL, new_lock); 2164 if (lock == NULL) { 2165 lock = new_lock; 2166 atomic_inc_uint(&pli->pli_lock_refs[lockid]); 2167 } 2168 } 2169 2170 /* 2171 * Now finally provide the lock. 2172 */ 2173 return lock; 2174 } 2175 #else /* !MULTIPROCESSOR */ 2176 void 2177 pmap_pvlist_lock_init(size_t cache_line_size) 2178 { 2179 mutex_init(&pmap_pvlist_mutex, MUTEX_DEFAULT, IPL_HIGH); 2180 } 2181 2182 #ifdef MODULAR 2183 kmutex_t * 2184 pmap_pvlist_lock_addr(struct vm_page_md *mdpg) 2185 { 2186 /* 2187 * We just use a global lock. 2188 */ 2189 if (__predict_false(mdpg->mdpg_lock == NULL)) { 2190 mdpg->mdpg_lock = &pmap_pvlist_mutex; 2191 } 2192 2193 /* 2194 * Now finally provide the lock. 2195 */ 2196 return mdpg->mdpg_lock; 2197 } 2198 #endif /* MODULAR */ 2199 #endif /* !MULTIPROCESSOR */ 2200 2201 /* 2202 * pmap_pv_page_alloc: 2203 * 2204 * Allocate a page for the pv_entry pool. 2205 */ 2206 void * 2207 pmap_pv_page_alloc(struct pool *pp, int flags) 2208 { 2209 struct vm_page * const pg = PMAP_ALLOC_POOLPAGE(UVM_PGA_USERESERVE); 2210 if (pg == NULL) 2211 return NULL; 2212 2213 return (void *)pmap_map_poolpage(VM_PAGE_TO_PHYS(pg)); 2214 } 2215 2216 /* 2217 * pmap_pv_page_free: 2218 * 2219 * Free a pv_entry pool page. 2220 */ 2221 void 2222 pmap_pv_page_free(struct pool *pp, void *v) 2223 { 2224 vaddr_t va = (vaddr_t)v; 2225 2226 KASSERT(pmap_md_direct_mapped_vaddr_p(va)); 2227 const paddr_t pa = pmap_md_direct_mapped_vaddr_to_paddr(va); 2228 struct vm_page * const pg = PHYS_TO_VM_PAGE(pa); 2229 KASSERT(pg != NULL); 2230 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 2231 kpreempt_disable(); 2232 pmap_md_vca_remove(pg, va, true, true); 2233 kpreempt_enable(); 2234 #endif 2235 pmap_page_clear_attributes(VM_PAGE_TO_MD(pg), VM_PAGEMD_POOLPAGE); 2236 KASSERT(!VM_PAGEMD_EXECPAGE_P(VM_PAGE_TO_MD(pg))); 2237 uvm_pagefree(pg); 2238 } 2239 2240 #ifdef PMAP_PREFER 2241 /* 2242 * Find first virtual address >= *vap that doesn't cause 2243 * a cache alias conflict. 2244 */ 2245 void 2246 pmap_prefer(vaddr_t foff, vaddr_t *vap, vsize_t sz, int td) 2247 { 2248 vsize_t prefer_mask = ptoa(uvmexp.colormask); 2249 2250 PMAP_COUNT(prefer_requests); 2251 2252 prefer_mask |= pmap_md_cache_prefer_mask(); 2253 2254 if (prefer_mask) { 2255 vaddr_t va = *vap; 2256 vsize_t d = (foff - va) & prefer_mask; 2257 if (d) { 2258 if (td) 2259 *vap = trunc_page(va - ((-d) & prefer_mask)); 2260 else 2261 *vap = round_page(va + d); 2262 PMAP_COUNT(prefer_adjustments); 2263 } 2264 } 2265 } 2266 #endif /* PMAP_PREFER */ 2267 2268 #ifdef PMAP_MAP_POOLPAGE 2269 vaddr_t 2270 pmap_map_poolpage(paddr_t pa) 2271 { 2272 struct vm_page * const pg = PHYS_TO_VM_PAGE(pa); 2273 KASSERT(pg); 2274 2275 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 2276 KASSERT(!VM_PAGEMD_EXECPAGE_P(mdpg)); 2277 2278 pmap_page_set_attributes(mdpg, VM_PAGEMD_POOLPAGE); 2279 2280 return pmap_md_map_poolpage(pa, NBPG); 2281 } 2282 2283 paddr_t 2284 pmap_unmap_poolpage(vaddr_t va) 2285 { 2286 KASSERT(pmap_md_direct_mapped_vaddr_p(va)); 2287 paddr_t pa = pmap_md_direct_mapped_vaddr_to_paddr(va); 2288 2289 struct vm_page * const pg = PHYS_TO_VM_PAGE(pa); 2290 KASSERT(pg != NULL); 2291 KASSERT(!VM_PAGEMD_EXECPAGE_P(VM_PAGE_TO_MD(pg))); 2292 2293 pmap_page_clear_attributes(VM_PAGE_TO_MD(pg), VM_PAGEMD_POOLPAGE); 2294 pmap_md_unmap_poolpage(va, NBPG); 2295 2296 return pa; 2297 } 2298 #endif /* PMAP_MAP_POOLPAGE */ 2299