1 /* $NetBSD: pmap.c,v 1.53 2020/08/11 06:09:44 skrll Exp $ */ 2 3 /*- 4 * Copyright (c) 1998, 2001 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility, 9 * NASA Ames Research Center and by Chris G. Demetriou. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 22 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 23 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 24 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 * POSSIBILITY OF SUCH DAMAGE. 31 */ 32 33 /* 34 * Copyright (c) 1992, 1993 35 * The Regents of the University of California. All rights reserved. 36 * 37 * This code is derived from software contributed to Berkeley by 38 * the Systems Programming Group of the University of Utah Computer 39 * Science Department and Ralph Campbell. 40 * 41 * Redistribution and use in source and binary forms, with or without 42 * modification, are permitted provided that the following conditions 43 * are met: 44 * 1. Redistributions of source code must retain the above copyright 45 * notice, this list of conditions and the following disclaimer. 46 * 2. Redistributions in binary form must reproduce the above copyright 47 * notice, this list of conditions and the following disclaimer in the 48 * documentation and/or other materials provided with the distribution. 49 * 3. Neither the name of the University nor the names of its contributors 50 * may be used to endorse or promote products derived from this software 51 * without specific prior written permission. 52 * 53 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 54 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 55 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 56 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 57 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 58 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 59 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 60 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 61 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 62 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 63 * SUCH DAMAGE. 64 * 65 * @(#)pmap.c 8.4 (Berkeley) 1/26/94 66 */ 67 68 #include <sys/cdefs.h> 69 70 __KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.53 2020/08/11 06:09:44 skrll Exp $"); 71 72 /* 73 * Manages physical address maps. 74 * 75 * In addition to hardware address maps, this 76 * module is called upon to provide software-use-only 77 * maps which may or may not be stored in the same 78 * form as hardware maps. These pseudo-maps are 79 * used to store intermediate results from copy 80 * operations to and from address spaces. 81 * 82 * Since the information managed by this module is 83 * also stored by the logical address mapping module, 84 * this module may throw away valid virtual-to-physical 85 * mappings at almost any time. However, invalidations 86 * of virtual-to-physical mappings must be done as 87 * requested. 88 * 89 * In order to cope with hardware architectures which 90 * make virtual-to-physical map invalidates expensive, 91 * this module may delay invalidate or reduced protection 92 * operations until such time as they are actually 93 * necessary. This module is given full information as 94 * to which processors are currently using which maps, 95 * and to when physical maps must be made correct. 96 */ 97 98 #include "opt_modular.h" 99 #include "opt_multiprocessor.h" 100 #include "opt_sysv.h" 101 102 #define __PMAP_PRIVATE 103 104 #include <sys/param.h> 105 106 #include <sys/atomic.h> 107 #include <sys/buf.h> 108 #include <sys/cpu.h> 109 #include <sys/mutex.h> 110 #include <sys/pool.h> 111 112 #include <uvm/uvm.h> 113 #include <uvm/uvm_physseg.h> 114 115 #if defined(MULTIPROCESSOR) && defined(PMAP_VIRTUAL_CACHE_ALIASES) \ 116 && !defined(PMAP_NO_PV_UNCACHED) 117 #error PMAP_VIRTUAL_CACHE_ALIASES with MULTIPROCESSOR requires \ 118 PMAP_NO_PV_UNCACHED to be defined 119 #endif 120 121 PMAP_COUNTER(remove_kernel_calls, "remove kernel calls"); 122 PMAP_COUNTER(remove_kernel_pages, "kernel pages unmapped"); 123 PMAP_COUNTER(remove_user_calls, "remove user calls"); 124 PMAP_COUNTER(remove_user_pages, "user pages unmapped"); 125 PMAP_COUNTER(remove_flushes, "remove cache flushes"); 126 PMAP_COUNTER(remove_tlb_ops, "remove tlb ops"); 127 PMAP_COUNTER(remove_pvfirst, "remove pv first"); 128 PMAP_COUNTER(remove_pvsearch, "remove pv search"); 129 130 PMAP_COUNTER(prefer_requests, "prefer requests"); 131 PMAP_COUNTER(prefer_adjustments, "prefer adjustments"); 132 133 PMAP_COUNTER(idlezeroed_pages, "pages idle zeroed"); 134 135 PMAP_COUNTER(kenter_pa, "kernel fast mapped pages"); 136 PMAP_COUNTER(kenter_pa_bad, "kernel fast mapped pages (bad color)"); 137 PMAP_COUNTER(kenter_pa_unmanaged, "kernel fast mapped unmanaged pages"); 138 PMAP_COUNTER(kremove_pages, "kernel fast unmapped pages"); 139 140 PMAP_COUNTER(page_cache_evictions, "pages changed to uncacheable"); 141 PMAP_COUNTER(page_cache_restorations, "pages changed to cacheable"); 142 143 PMAP_COUNTER(kernel_mappings_bad, "kernel pages mapped (bad color)"); 144 PMAP_COUNTER(user_mappings_bad, "user pages mapped (bad color)"); 145 PMAP_COUNTER(kernel_mappings, "kernel pages mapped"); 146 PMAP_COUNTER(user_mappings, "user pages mapped"); 147 PMAP_COUNTER(user_mappings_changed, "user mapping changed"); 148 PMAP_COUNTER(kernel_mappings_changed, "kernel mapping changed"); 149 PMAP_COUNTER(uncached_mappings, "uncached pages mapped"); 150 PMAP_COUNTER(unmanaged_mappings, "unmanaged pages mapped"); 151 PMAP_COUNTER(managed_mappings, "managed pages mapped"); 152 PMAP_COUNTER(mappings, "pages mapped"); 153 PMAP_COUNTER(remappings, "pages remapped"); 154 PMAP_COUNTER(unmappings, "pages unmapped"); 155 PMAP_COUNTER(primary_mappings, "page initial mappings"); 156 PMAP_COUNTER(primary_unmappings, "page final unmappings"); 157 PMAP_COUNTER(tlb_hit, "page mapping"); 158 159 PMAP_COUNTER(exec_mappings, "exec pages mapped"); 160 PMAP_COUNTER(exec_synced_mappings, "exec pages synced"); 161 PMAP_COUNTER(exec_synced_remove, "exec pages synced (PR)"); 162 PMAP_COUNTER(exec_synced_clear_modify, "exec pages synced (CM)"); 163 PMAP_COUNTER(exec_synced_page_protect, "exec pages synced (PP)"); 164 PMAP_COUNTER(exec_synced_protect, "exec pages synced (P)"); 165 PMAP_COUNTER(exec_uncached_page_protect, "exec pages uncached (PP)"); 166 PMAP_COUNTER(exec_uncached_clear_modify, "exec pages uncached (CM)"); 167 PMAP_COUNTER(exec_uncached_zero_page, "exec pages uncached (ZP)"); 168 PMAP_COUNTER(exec_uncached_copy_page, "exec pages uncached (CP)"); 169 PMAP_COUNTER(exec_uncached_remove, "exec pages uncached (PR)"); 170 171 PMAP_COUNTER(create, "creates"); 172 PMAP_COUNTER(reference, "references"); 173 PMAP_COUNTER(dereference, "dereferences"); 174 PMAP_COUNTER(destroy, "destroyed"); 175 PMAP_COUNTER(activate, "activations"); 176 PMAP_COUNTER(deactivate, "deactivations"); 177 PMAP_COUNTER(update, "updates"); 178 #ifdef MULTIPROCESSOR 179 PMAP_COUNTER(shootdown_ipis, "shootdown IPIs"); 180 #endif 181 PMAP_COUNTER(unwire, "unwires"); 182 PMAP_COUNTER(copy, "copies"); 183 PMAP_COUNTER(clear_modify, "clear_modifies"); 184 PMAP_COUNTER(protect, "protects"); 185 PMAP_COUNTER(page_protect, "page_protects"); 186 187 #define PMAP_ASID_RESERVED 0 188 CTASSERT(PMAP_ASID_RESERVED == 0); 189 190 #ifndef PMAP_SEGTAB_ALIGN 191 #define PMAP_SEGTAB_ALIGN /* nothing */ 192 #endif 193 #ifdef _LP64 194 pmap_segtab_t pmap_kstart_segtab PMAP_SEGTAB_ALIGN; /* first mid-level segtab for kernel */ 195 #endif 196 pmap_segtab_t pmap_kern_segtab PMAP_SEGTAB_ALIGN = { /* top level segtab for kernel */ 197 #ifdef _LP64 198 .seg_seg[(VM_MIN_KERNEL_ADDRESS & XSEGOFSET) >> SEGSHIFT] = &pmap_kstart_segtab, 199 #endif 200 }; 201 202 struct pmap_kernel kernel_pmap_store = { 203 .kernel_pmap = { 204 .pm_count = 1, 205 .pm_segtab = &pmap_kern_segtab, 206 .pm_minaddr = VM_MIN_KERNEL_ADDRESS, 207 .pm_maxaddr = VM_MAX_KERNEL_ADDRESS, 208 }, 209 }; 210 211 struct pmap * const kernel_pmap_ptr = &kernel_pmap_store.kernel_pmap; 212 213 struct pmap_limits pmap_limits = { /* VA and PA limits */ 214 .virtual_start = VM_MIN_KERNEL_ADDRESS, 215 }; 216 217 #ifdef UVMHIST 218 static struct kern_history_ent pmapexechistbuf[10000]; 219 static struct kern_history_ent pmaphistbuf[10000]; 220 UVMHIST_DEFINE(pmapexechist); 221 UVMHIST_DEFINE(pmaphist); 222 #endif 223 224 /* 225 * The pools from which pmap structures and sub-structures are allocated. 226 */ 227 struct pool pmap_pmap_pool; 228 struct pool pmap_pv_pool; 229 230 #ifndef PMAP_PV_LOWAT 231 #define PMAP_PV_LOWAT 16 232 #endif 233 int pmap_pv_lowat = PMAP_PV_LOWAT; 234 235 bool pmap_initialized = false; 236 #define PMAP_PAGE_COLOROK_P(a, b) \ 237 ((((int)(a) ^ (int)(b)) & pmap_page_colormask) == 0) 238 u_int pmap_page_colormask; 239 240 #define PAGE_IS_MANAGED(pa) (pmap_initialized && uvm_pageismanaged(pa)) 241 242 #define PMAP_IS_ACTIVE(pm) \ 243 ((pm) == pmap_kernel() || \ 244 (pm) == curlwp->l_proc->p_vmspace->vm_map.pmap) 245 246 /* Forward function declarations */ 247 void pmap_page_remove(struct vm_page *); 248 static void pmap_pvlist_check(struct vm_page_md *); 249 void pmap_remove_pv(pmap_t, vaddr_t, struct vm_page *, bool); 250 void pmap_enter_pv(pmap_t, vaddr_t, struct vm_page *, pt_entry_t *, u_int); 251 252 /* 253 * PV table management functions. 254 */ 255 void *pmap_pv_page_alloc(struct pool *, int); 256 void pmap_pv_page_free(struct pool *, void *); 257 258 struct pool_allocator pmap_pv_page_allocator = { 259 pmap_pv_page_alloc, pmap_pv_page_free, 0, 260 }; 261 262 #define pmap_pv_alloc() pool_get(&pmap_pv_pool, PR_NOWAIT) 263 #define pmap_pv_free(pv) pool_put(&pmap_pv_pool, (pv)) 264 265 #ifndef PMAP_NEED_TLB_MISS_LOCK 266 267 #if defined(PMAP_MD_NEED_TLB_MISS_LOCK) || defined(DEBUG) 268 #define PMAP_NEED_TLB_MISS_LOCK 269 #endif /* PMAP_MD_NEED_TLB_MISS_LOCK || DEBUG */ 270 271 #endif /* PMAP_NEED_TLB_MISS_LOCK */ 272 273 #ifdef PMAP_NEED_TLB_MISS_LOCK 274 275 #ifdef PMAP_MD_NEED_TLB_MISS_LOCK 276 #define pmap_tlb_miss_lock_init() __nothing /* MD code deals with this */ 277 #define pmap_tlb_miss_lock_enter() pmap_md_tlb_miss_lock_enter() 278 #define pmap_tlb_miss_lock_exit() pmap_md_tlb_miss_lock_exit() 279 #else 280 kmutex_t pmap_tlb_miss_lock __cacheline_aligned; 281 282 static void 283 pmap_tlb_miss_lock_init(void) 284 { 285 mutex_init(&pmap_tlb_miss_lock, MUTEX_SPIN, IPL_HIGH); 286 } 287 288 static inline void 289 pmap_tlb_miss_lock_enter(void) 290 { 291 mutex_spin_enter(&pmap_tlb_miss_lock); 292 } 293 294 static inline void 295 pmap_tlb_miss_lock_exit(void) 296 { 297 mutex_spin_exit(&pmap_tlb_miss_lock); 298 } 299 #endif /* PMAP_MD_NEED_TLB_MISS_LOCK */ 300 301 #else 302 303 #define pmap_tlb_miss_lock_init() __nothing 304 #define pmap_tlb_miss_lock_enter() __nothing 305 #define pmap_tlb_miss_lock_exit() __nothing 306 307 #endif /* PMAP_NEED_TLB_MISS_LOCK */ 308 309 #ifndef MULTIPROCESSOR 310 kmutex_t pmap_pvlist_mutex __cacheline_aligned; 311 #endif 312 313 /* 314 * Debug functions. 315 */ 316 317 #ifdef DEBUG 318 static inline void 319 pmap_asid_check(pmap_t pm, const char *func) 320 { 321 if (!PMAP_IS_ACTIVE(pm)) 322 return; 323 324 struct pmap_asid_info * const pai = PMAP_PAI(pm, cpu_tlb_info(curcpu())); 325 tlb_asid_t asid = tlb_get_asid(); 326 if (asid != pai->pai_asid) 327 panic("%s: inconsistency for active TLB update: %u <-> %u", 328 func, asid, pai->pai_asid); 329 } 330 #endif 331 332 static void 333 pmap_addr_range_check(pmap_t pmap, vaddr_t sva, vaddr_t eva, const char *func) 334 { 335 #ifdef DEBUG 336 if (pmap == pmap_kernel()) { 337 if (sva < VM_MIN_KERNEL_ADDRESS) 338 panic("%s: kva %#"PRIxVADDR" not in range", 339 func, sva); 340 if (eva >= pmap_limits.virtual_end) 341 panic("%s: kva %#"PRIxVADDR" not in range", 342 func, eva); 343 } else { 344 if (eva > VM_MAXUSER_ADDRESS) 345 panic("%s: uva %#"PRIxVADDR" not in range", 346 func, eva); 347 pmap_asid_check(pmap, func); 348 } 349 #endif 350 } 351 352 /* 353 * Misc. functions. 354 */ 355 356 bool 357 pmap_page_clear_attributes(struct vm_page_md *mdpg, u_int clear_attributes) 358 { 359 volatile unsigned long * const attrp = &mdpg->mdpg_attrs; 360 #ifdef MULTIPROCESSOR 361 for (;;) { 362 u_int old_attr = *attrp; 363 if ((old_attr & clear_attributes) == 0) 364 return false; 365 u_int new_attr = old_attr & ~clear_attributes; 366 if (old_attr == atomic_cas_ulong(attrp, old_attr, new_attr)) 367 return true; 368 } 369 #else 370 unsigned long old_attr = *attrp; 371 if ((old_attr & clear_attributes) == 0) 372 return false; 373 *attrp &= ~clear_attributes; 374 return true; 375 #endif 376 } 377 378 void 379 pmap_page_set_attributes(struct vm_page_md *mdpg, u_int set_attributes) 380 { 381 #ifdef MULTIPROCESSOR 382 atomic_or_ulong(&mdpg->mdpg_attrs, set_attributes); 383 #else 384 mdpg->mdpg_attrs |= set_attributes; 385 #endif 386 } 387 388 static void 389 pmap_page_syncicache(struct vm_page *pg) 390 { 391 UVMHIST_FUNC(__func__); 392 UVMHIST_CALLED(pmaphist); 393 #ifndef MULTIPROCESSOR 394 struct pmap * const curpmap = curlwp->l_proc->p_vmspace->vm_map.pmap; 395 #endif 396 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 397 pv_entry_t pv = &mdpg->mdpg_first; 398 kcpuset_t *onproc; 399 #ifdef MULTIPROCESSOR 400 kcpuset_create(&onproc, true); 401 KASSERT(onproc != NULL); 402 #else 403 onproc = NULL; 404 #endif 405 VM_PAGEMD_PVLIST_READLOCK(mdpg); 406 pmap_pvlist_check(mdpg); 407 408 UVMHIST_LOG(pmaphist, "pv %jx pv_pmap %jx\n", (uintptr_t)pv, 409 (uintptr_t)pv->pv_pmap, 0, 0); 410 411 if (pv->pv_pmap != NULL) { 412 for (; pv != NULL; pv = pv->pv_next) { 413 #ifdef MULTIPROCESSOR 414 UVMHIST_LOG(pmaphist, "pv %jx pv_pmap %jx\n", 415 (uintptr_t)pv, (uintptr_t)pv->pv_pmap, 0, 0); 416 kcpuset_merge(onproc, pv->pv_pmap->pm_onproc); 417 if (kcpuset_match(onproc, kcpuset_running)) { 418 break; 419 } 420 #else 421 if (pv->pv_pmap == curpmap) { 422 onproc = curcpu()->ci_data.cpu_kcpuset; 423 break; 424 } 425 #endif 426 } 427 } 428 pmap_pvlist_check(mdpg); 429 VM_PAGEMD_PVLIST_UNLOCK(mdpg); 430 kpreempt_disable(); 431 pmap_md_page_syncicache(pg, onproc); 432 kpreempt_enable(); 433 #ifdef MULTIPROCESSOR 434 kcpuset_destroy(onproc); 435 #endif 436 } 437 438 /* 439 * Define the initial bounds of the kernel virtual address space. 440 */ 441 void 442 pmap_virtual_space(vaddr_t *vstartp, vaddr_t *vendp) 443 { 444 445 *vstartp = pmap_limits.virtual_start; 446 *vendp = pmap_limits.virtual_end; 447 } 448 449 vaddr_t 450 pmap_growkernel(vaddr_t maxkvaddr) 451 { 452 vaddr_t virtual_end = pmap_limits.virtual_end; 453 maxkvaddr = pmap_round_seg(maxkvaddr) - 1; 454 455 /* 456 * Reserve PTEs for the new KVA space. 457 */ 458 for (; virtual_end < maxkvaddr; virtual_end += NBSEG) { 459 pmap_pte_reserve(pmap_kernel(), virtual_end, 0); 460 } 461 462 /* 463 * Don't exceed VM_MAX_KERNEL_ADDRESS! 464 */ 465 if (virtual_end == 0 || virtual_end > VM_MAX_KERNEL_ADDRESS) 466 virtual_end = VM_MAX_KERNEL_ADDRESS; 467 468 /* 469 * Update new end. 470 */ 471 pmap_limits.virtual_end = virtual_end; 472 return virtual_end; 473 } 474 475 /* 476 * Bootstrap memory allocator (alternative to vm_bootstrap_steal_memory()). 477 * This function allows for early dynamic memory allocation until the virtual 478 * memory system has been bootstrapped. After that point, either kmem_alloc 479 * or malloc should be used. This function works by stealing pages from the 480 * (to be) managed page pool, then implicitly mapping the pages (by using 481 * their direct mapped addresses) and zeroing them. 482 * 483 * It may be used once the physical memory segments have been pre-loaded 484 * into the vm_physmem[] array. Early memory allocation MUST use this 485 * interface! This cannot be used after vm_page_startup(), and will 486 * generate a panic if tried. 487 * 488 * Note that this memory will never be freed, and in essence it is wired 489 * down. 490 * 491 * We must adjust *vstartp and/or *vendp iff we use address space 492 * from the kernel virtual address range defined by pmap_virtual_space(). 493 */ 494 vaddr_t 495 pmap_steal_memory(vsize_t size, vaddr_t *vstartp, vaddr_t *vendp) 496 { 497 size_t npgs; 498 paddr_t pa; 499 vaddr_t va; 500 501 uvm_physseg_t maybe_bank = UVM_PHYSSEG_TYPE_INVALID; 502 503 size = round_page(size); 504 npgs = atop(size); 505 506 aprint_debug("%s: need %zu pages\n", __func__, npgs); 507 508 for (uvm_physseg_t bank = uvm_physseg_get_first(); 509 uvm_physseg_valid_p(bank); 510 bank = uvm_physseg_get_next(bank)) { 511 512 if (uvm.page_init_done == true) 513 panic("pmap_steal_memory: called _after_ bootstrap"); 514 515 aprint_debug("%s: seg %"PRIxPHYSSEG": %#"PRIxPADDR" %#"PRIxPADDR" %#"PRIxPADDR" %#"PRIxPADDR"\n", 516 __func__, bank, 517 uvm_physseg_get_avail_start(bank), uvm_physseg_get_start(bank), 518 uvm_physseg_get_avail_end(bank), uvm_physseg_get_end(bank)); 519 520 if (uvm_physseg_get_avail_start(bank) != uvm_physseg_get_start(bank) 521 || uvm_physseg_get_avail_start(bank) >= uvm_physseg_get_avail_end(bank)) { 522 aprint_debug("%s: seg %"PRIxPHYSSEG": bad start\n", __func__, bank); 523 continue; 524 } 525 526 if (uvm_physseg_get_avail_end(bank) - uvm_physseg_get_avail_start(bank) < npgs) { 527 aprint_debug("%s: seg %"PRIxPHYSSEG": too small for %zu pages\n", 528 __func__, bank, npgs); 529 continue; 530 } 531 532 if (!pmap_md_ok_to_steal_p(bank, npgs)) { 533 continue; 534 } 535 536 /* 537 * Always try to allocate from the segment with the least 538 * amount of space left. 539 */ 540 #define VM_PHYSMEM_SPACE(b) ((uvm_physseg_get_avail_end(b)) - (uvm_physseg_get_avail_start(b))) 541 if (uvm_physseg_valid_p(maybe_bank) == false 542 || VM_PHYSMEM_SPACE(bank) < VM_PHYSMEM_SPACE(maybe_bank)) { 543 maybe_bank = bank; 544 } 545 } 546 547 if (uvm_physseg_valid_p(maybe_bank)) { 548 const uvm_physseg_t bank = maybe_bank; 549 550 /* 551 * There are enough pages here; steal them! 552 */ 553 pa = ptoa(uvm_physseg_get_start(bank)); 554 uvm_physseg_unplug(atop(pa), npgs); 555 556 aprint_debug("%s: seg %"PRIxPHYSSEG": %zu pages stolen (%#"PRIxPADDR" left)\n", 557 __func__, bank, npgs, VM_PHYSMEM_SPACE(bank)); 558 559 va = pmap_md_map_poolpage(pa, size); 560 memset((void *)va, 0, size); 561 return va; 562 } 563 564 /* 565 * If we got here, there was no memory left. 566 */ 567 panic("pmap_steal_memory: no memory to steal %zu pages", npgs); 568 } 569 570 /* 571 * Bootstrap the system enough to run with virtual memory. 572 * (Common routine called by machine-dependent bootstrap code.) 573 */ 574 void 575 pmap_bootstrap_common(void) 576 { 577 pmap_tlb_miss_lock_init(); 578 } 579 580 /* 581 * Initialize the pmap module. 582 * Called by vm_init, to initialize any structures that the pmap 583 * system needs to map virtual memory. 584 */ 585 void 586 pmap_init(void) 587 { 588 UVMHIST_INIT_STATIC(pmapexechist, pmapexechistbuf); 589 UVMHIST_INIT_STATIC(pmaphist, pmaphistbuf); 590 591 UVMHIST_FUNC(__func__); 592 UVMHIST_CALLED(pmaphist); 593 594 /* 595 * Initialize the segtab lock. 596 */ 597 mutex_init(&pmap_segtab_lock, MUTEX_DEFAULT, IPL_HIGH); 598 599 /* 600 * Set a low water mark on the pv_entry pool, so that we are 601 * more likely to have these around even in extreme memory 602 * starvation. 603 */ 604 pool_setlowat(&pmap_pv_pool, pmap_pv_lowat); 605 606 /* 607 * Set the page colormask but allow pmap_md_init to override it. 608 */ 609 pmap_page_colormask = ptoa(uvmexp.colormask); 610 611 pmap_md_init(); 612 613 /* 614 * Now it is safe to enable pv entry recording. 615 */ 616 pmap_initialized = true; 617 } 618 619 /* 620 * Create and return a physical map. 621 * 622 * If the size specified for the map 623 * is zero, the map is an actual physical 624 * map, and may be referenced by the 625 * hardware. 626 * 627 * If the size specified is non-zero, 628 * the map will be used in software only, and 629 * is bounded by that size. 630 */ 631 pmap_t 632 pmap_create(void) 633 { 634 UVMHIST_FUNC(__func__); 635 UVMHIST_CALLED(pmaphist); 636 PMAP_COUNT(create); 637 638 pmap_t pmap = pool_get(&pmap_pmap_pool, PR_WAITOK); 639 memset(pmap, 0, PMAP_SIZE); 640 641 KASSERT(pmap->pm_pai[0].pai_link.le_prev == NULL); 642 643 pmap->pm_count = 1; 644 pmap->pm_minaddr = VM_MIN_ADDRESS; 645 pmap->pm_maxaddr = VM_MAXUSER_ADDRESS; 646 647 pmap_segtab_init(pmap); 648 649 #ifdef MULTIPROCESSOR 650 kcpuset_create(&pmap->pm_active, true); 651 kcpuset_create(&pmap->pm_onproc, true); 652 KASSERT(pmap->pm_active != NULL); 653 KASSERT(pmap->pm_onproc != NULL); 654 #endif 655 656 UVMHIST_LOG(pmaphist, " <-- done (pmap=%#jx)", (uintptr_t)pmap, 657 0, 0, 0); 658 659 return pmap; 660 } 661 662 /* 663 * Retire the given physical map from service. 664 * Should only be called if the map contains 665 * no valid mappings. 666 */ 667 void 668 pmap_destroy(pmap_t pmap) 669 { 670 UVMHIST_FUNC(__func__); 671 UVMHIST_CALLARGS(pmaphist, "(pmap=%#jx)", (uintptr_t)pmap, 0, 0, 0); 672 673 if (atomic_dec_uint_nv(&pmap->pm_count) > 0) { 674 PMAP_COUNT(dereference); 675 UVMHIST_LOG(pmaphist, " <-- done (deref)", 0, 0, 0, 0); 676 return; 677 } 678 679 PMAP_COUNT(destroy); 680 KASSERT(pmap->pm_count == 0); 681 kpreempt_disable(); 682 pmap_tlb_miss_lock_enter(); 683 pmap_tlb_asid_release_all(pmap); 684 pmap_segtab_destroy(pmap, NULL, 0); 685 pmap_tlb_miss_lock_exit(); 686 687 #ifdef MULTIPROCESSOR 688 kcpuset_destroy(pmap->pm_active); 689 kcpuset_destroy(pmap->pm_onproc); 690 pmap->pm_active = NULL; 691 pmap->pm_onproc = NULL; 692 #endif 693 694 pool_put(&pmap_pmap_pool, pmap); 695 kpreempt_enable(); 696 697 UVMHIST_LOG(pmaphist, " <-- done (freed)", 0, 0, 0, 0); 698 } 699 700 /* 701 * Add a reference to the specified pmap. 702 */ 703 void 704 pmap_reference(pmap_t pmap) 705 { 706 UVMHIST_FUNC(__func__); 707 UVMHIST_CALLARGS(pmaphist, "(pmap=%#jx)", (uintptr_t)pmap, 0, 0, 0); 708 PMAP_COUNT(reference); 709 710 if (pmap != NULL) { 711 atomic_inc_uint(&pmap->pm_count); 712 } 713 714 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 715 } 716 717 /* 718 * Make a new pmap (vmspace) active for the given process. 719 */ 720 void 721 pmap_activate(struct lwp *l) 722 { 723 pmap_t pmap = l->l_proc->p_vmspace->vm_map.pmap; 724 725 UVMHIST_FUNC(__func__); 726 UVMHIST_CALLARGS(pmaphist, "(l=%#jx pmap=%#jx)", (uintptr_t)l, 727 (uintptr_t)pmap, 0, 0); 728 PMAP_COUNT(activate); 729 730 kpreempt_disable(); 731 pmap_tlb_miss_lock_enter(); 732 pmap_tlb_asid_acquire(pmap, l); 733 pmap_segtab_activate(pmap, l); 734 pmap_tlb_miss_lock_exit(); 735 kpreempt_enable(); 736 737 UVMHIST_LOG(pmaphist, " <-- done (%ju:%ju)", l->l_proc->p_pid, 738 l->l_lid, 0, 0); 739 } 740 741 /* 742 * Remove this page from all physical maps in which it resides. 743 * Reflects back modify bits to the pager. 744 */ 745 void 746 pmap_page_remove(struct vm_page *pg) 747 { 748 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 749 750 kpreempt_disable(); 751 VM_PAGEMD_PVLIST_LOCK(mdpg); 752 pmap_pvlist_check(mdpg); 753 754 UVMHIST_FUNC(__func__); 755 UVMHIST_CALLARGS(pmapexechist, "pg %#jx (pa %#jx) [page removed]: " 756 "execpage cleared", (uintptr_t)pg, VM_PAGE_TO_PHYS(pg), 0, 0); 757 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 758 pmap_page_clear_attributes(mdpg, VM_PAGEMD_EXECPAGE|VM_PAGEMD_UNCACHED); 759 #else 760 pmap_page_clear_attributes(mdpg, VM_PAGEMD_EXECPAGE); 761 #endif 762 PMAP_COUNT(exec_uncached_remove); 763 764 pv_entry_t pv = &mdpg->mdpg_first; 765 if (pv->pv_pmap == NULL) { 766 VM_PAGEMD_PVLIST_UNLOCK(mdpg); 767 kpreempt_enable(); 768 UVMHIST_LOG(pmaphist, " <-- done (empty)", 0, 0, 0, 0); 769 return; 770 } 771 772 pv_entry_t npv; 773 pv_entry_t pvp = NULL; 774 775 for (; pv != NULL; pv = npv) { 776 npv = pv->pv_next; 777 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 778 if (PV_ISKENTER_P(pv)) { 779 UVMHIST_LOG(pmaphist, " pv %#jx pmap %#jx va %jx" 780 " skip", (uintptr_t)pv, (uintptr_t)pv->pv_pmap, 781 pv->pv_va, 0); 782 783 KASSERT(pv->pv_pmap == pmap_kernel()); 784 785 /* Assume no more - it'll get fixed if there are */ 786 pv->pv_next = NULL; 787 788 /* 789 * pvp is non-null when we already have a PV_KENTER 790 * pv in pvh_first; otherwise we haven't seen a 791 * PV_KENTER pv and we need to copy this one to 792 * pvh_first 793 */ 794 if (pvp) { 795 /* 796 * The previous PV_KENTER pv needs to point to 797 * this PV_KENTER pv 798 */ 799 pvp->pv_next = pv; 800 } else { 801 pv_entry_t fpv = &mdpg->mdpg_first; 802 *fpv = *pv; 803 KASSERT(fpv->pv_pmap == pmap_kernel()); 804 } 805 pvp = pv; 806 continue; 807 } 808 #endif 809 const pmap_t pmap = pv->pv_pmap; 810 vaddr_t va = trunc_page(pv->pv_va); 811 pt_entry_t * const ptep = pmap_pte_lookup(pmap, va); 812 KASSERTMSG(ptep != NULL, "%#"PRIxVADDR " %#"PRIxVADDR, va, 813 pmap_limits.virtual_end); 814 pt_entry_t pte = *ptep; 815 UVMHIST_LOG(pmaphist, " pv %#jx pmap %#jx va %jx" 816 " pte %jx", (uintptr_t)pv, (uintptr_t)pmap, va, 817 pte_value(pte)); 818 if (!pte_valid_p(pte)) 819 continue; 820 const bool is_kernel_pmap_p = (pmap == pmap_kernel()); 821 if (is_kernel_pmap_p) { 822 PMAP_COUNT(remove_kernel_pages); 823 } else { 824 PMAP_COUNT(remove_user_pages); 825 } 826 if (pte_wired_p(pte)) 827 pmap->pm_stats.wired_count--; 828 pmap->pm_stats.resident_count--; 829 830 pmap_tlb_miss_lock_enter(); 831 const pt_entry_t npte = pte_nv_entry(is_kernel_pmap_p); 832 pte_set(ptep, npte); 833 if (__predict_true(!(pmap->pm_flags & PMAP_DEFERRED_ACTIVATE))) { 834 /* 835 * Flush the TLB for the given address. 836 */ 837 pmap_tlb_invalidate_addr(pmap, va); 838 } 839 pmap_tlb_miss_lock_exit(); 840 841 /* 842 * non-null means this is a non-pvh_first pv, so we should 843 * free it. 844 */ 845 if (pvp) { 846 KASSERT(pvp->pv_pmap == pmap_kernel()); 847 KASSERT(pvp->pv_next == NULL); 848 pmap_pv_free(pv); 849 } else { 850 pv->pv_pmap = NULL; 851 pv->pv_next = NULL; 852 } 853 } 854 855 pmap_pvlist_check(mdpg); 856 VM_PAGEMD_PVLIST_UNLOCK(mdpg); 857 kpreempt_enable(); 858 859 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 860 } 861 862 863 /* 864 * Make a previously active pmap (vmspace) inactive. 865 */ 866 void 867 pmap_deactivate(struct lwp *l) 868 { 869 pmap_t pmap = l->l_proc->p_vmspace->vm_map.pmap; 870 871 UVMHIST_FUNC(__func__); 872 UVMHIST_CALLARGS(pmaphist, "(l=%#jx pmap=%#jx)", (uintptr_t)l, 873 (uintptr_t)pmap, 0, 0); 874 PMAP_COUNT(deactivate); 875 876 kpreempt_disable(); 877 KASSERT(l == curlwp || l->l_cpu == curlwp->l_cpu); 878 pmap_tlb_miss_lock_enter(); 879 pmap_tlb_asid_deactivate(pmap); 880 pmap_segtab_deactivate(pmap); 881 pmap_tlb_miss_lock_exit(); 882 kpreempt_enable(); 883 884 UVMHIST_LOG(pmaphist, " <-- done (%ju:%ju)", l->l_proc->p_pid, 885 l->l_lid, 0, 0); 886 } 887 888 void 889 pmap_update(struct pmap *pmap) 890 { 891 UVMHIST_FUNC(__func__); 892 UVMHIST_CALLARGS(pmaphist, "(pmap=%#jx)", (uintptr_t)pmap, 0, 0, 0); 893 PMAP_COUNT(update); 894 895 kpreempt_disable(); 896 #if defined(MULTIPROCESSOR) && defined(PMAP_TLB_NEED_SHOOTDOWN) 897 u_int pending = atomic_swap_uint(&pmap->pm_shootdown_pending, 0); 898 if (pending && pmap_tlb_shootdown_bystanders(pmap)) 899 PMAP_COUNT(shootdown_ipis); 900 #endif 901 pmap_tlb_miss_lock_enter(); 902 #if defined(DEBUG) && !defined(MULTIPROCESSOR) 903 pmap_tlb_check(pmap, pmap_md_tlb_check_entry); 904 #endif /* DEBUG */ 905 906 /* 907 * If pmap_remove_all was called, we deactivated ourselves and nuked 908 * our ASID. Now we have to reactivate ourselves. 909 */ 910 if (__predict_false(pmap->pm_flags & PMAP_DEFERRED_ACTIVATE)) { 911 pmap->pm_flags ^= PMAP_DEFERRED_ACTIVATE; 912 pmap_tlb_asid_acquire(pmap, curlwp); 913 pmap_segtab_activate(pmap, curlwp); 914 } 915 pmap_tlb_miss_lock_exit(); 916 kpreempt_enable(); 917 918 UVMHIST_LOG(pmaphist, " <-- done (kernel=%#jx)", 919 (pmap == pmap_kernel() ? 1 : 0), 0, 0, 0); 920 } 921 922 /* 923 * Remove the given range of addresses from the specified map. 924 * 925 * It is assumed that the start and end are properly 926 * rounded to the page size. 927 */ 928 929 static bool 930 pmap_pte_remove(pmap_t pmap, vaddr_t sva, vaddr_t eva, pt_entry_t *ptep, 931 uintptr_t flags) 932 { 933 const pt_entry_t npte = flags; 934 const bool is_kernel_pmap_p = (pmap == pmap_kernel()); 935 936 UVMHIST_FUNC(__func__); 937 UVMHIST_CALLARGS(pmaphist, "(pmap=%#jx kernel=%jx va=%#jx..%#jx)", 938 (uintptr_t)pmap, (pmap == pmap_kernel() ? 1 : 0), sva, eva); 939 UVMHIST_LOG(pmaphist, "ptep=%#jx, flags(npte)=%#jx)", 940 (uintptr_t)ptep, flags, 0, 0); 941 942 KASSERT(kpreempt_disabled()); 943 944 for (; sva < eva; sva += NBPG, ptep++) { 945 const pt_entry_t pte = *ptep; 946 if (!pte_valid_p(pte)) 947 continue; 948 if (is_kernel_pmap_p) { 949 PMAP_COUNT(remove_kernel_pages); 950 } else { 951 PMAP_COUNT(remove_user_pages); 952 } 953 if (pte_wired_p(pte)) 954 pmap->pm_stats.wired_count--; 955 pmap->pm_stats.resident_count--; 956 struct vm_page * const pg = PHYS_TO_VM_PAGE(pte_to_paddr(pte)); 957 if (__predict_true(pg != NULL)) { 958 pmap_remove_pv(pmap, sva, pg, pte_modified_p(pte)); 959 } 960 pmap_tlb_miss_lock_enter(); 961 pte_set(ptep, npte); 962 if (__predict_true(!(pmap->pm_flags & PMAP_DEFERRED_ACTIVATE))) { 963 964 /* 965 * Flush the TLB for the given address. 966 */ 967 pmap_tlb_invalidate_addr(pmap, sva); 968 } 969 pmap_tlb_miss_lock_exit(); 970 } 971 972 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 973 974 return false; 975 } 976 977 void 978 pmap_remove(pmap_t pmap, vaddr_t sva, vaddr_t eva) 979 { 980 const bool is_kernel_pmap_p = (pmap == pmap_kernel()); 981 const pt_entry_t npte = pte_nv_entry(is_kernel_pmap_p); 982 983 UVMHIST_FUNC(__func__); 984 UVMHIST_CALLARGS(pmaphist, "(pmap=%#jx, va=%#jx..%#jx)", 985 (uintptr_t)pmap, sva, eva, 0); 986 987 if (is_kernel_pmap_p) { 988 PMAP_COUNT(remove_kernel_calls); 989 } else { 990 PMAP_COUNT(remove_user_calls); 991 } 992 #ifdef PMAP_FAULTINFO 993 curpcb->pcb_faultinfo.pfi_faultaddr = 0; 994 curpcb->pcb_faultinfo.pfi_repeats = 0; 995 curpcb->pcb_faultinfo.pfi_faultpte = NULL; 996 #endif 997 kpreempt_disable(); 998 pmap_addr_range_check(pmap, sva, eva, __func__); 999 pmap_pte_process(pmap, sva, eva, pmap_pte_remove, npte); 1000 kpreempt_enable(); 1001 1002 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 1003 } 1004 1005 /* 1006 * pmap_page_protect: 1007 * 1008 * Lower the permission for all mappings to a given page. 1009 */ 1010 void 1011 pmap_page_protect(struct vm_page *pg, vm_prot_t prot) 1012 { 1013 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 1014 pv_entry_t pv; 1015 vaddr_t va; 1016 1017 UVMHIST_FUNC(__func__); 1018 UVMHIST_CALLARGS(pmaphist, "(pg=%#jx (pa %#jx) prot=%#jx)", 1019 (uintptr_t)pg, VM_PAGE_TO_PHYS(pg), prot, 0); 1020 PMAP_COUNT(page_protect); 1021 1022 switch (prot) { 1023 case VM_PROT_READ|VM_PROT_WRITE: 1024 case VM_PROT_ALL: 1025 break; 1026 1027 /* copy_on_write */ 1028 case VM_PROT_READ: 1029 case VM_PROT_READ|VM_PROT_EXECUTE: 1030 pv = &mdpg->mdpg_first; 1031 kpreempt_disable(); 1032 VM_PAGEMD_PVLIST_READLOCK(mdpg); 1033 pmap_pvlist_check(mdpg); 1034 /* 1035 * Loop over all current mappings setting/clearing as 1036 * appropriate. 1037 */ 1038 if (pv->pv_pmap != NULL) { 1039 while (pv != NULL) { 1040 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 1041 if (PV_ISKENTER_P(pv)) { 1042 pv = pv->pv_next; 1043 continue; 1044 } 1045 #endif 1046 const pmap_t pmap = pv->pv_pmap; 1047 va = trunc_page(pv->pv_va); 1048 const uintptr_t gen = 1049 VM_PAGEMD_PVLIST_UNLOCK(mdpg); 1050 pmap_protect(pmap, va, va + PAGE_SIZE, prot); 1051 KASSERT(pv->pv_pmap == pmap); 1052 pmap_update(pmap); 1053 if (gen != VM_PAGEMD_PVLIST_READLOCK(mdpg)) { 1054 pv = &mdpg->mdpg_first; 1055 } else { 1056 pv = pv->pv_next; 1057 } 1058 pmap_pvlist_check(mdpg); 1059 } 1060 } 1061 pmap_pvlist_check(mdpg); 1062 VM_PAGEMD_PVLIST_UNLOCK(mdpg); 1063 kpreempt_enable(); 1064 break; 1065 1066 /* remove_all */ 1067 default: 1068 pmap_page_remove(pg); 1069 } 1070 1071 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 1072 } 1073 1074 static bool 1075 pmap_pte_protect(pmap_t pmap, vaddr_t sva, vaddr_t eva, pt_entry_t *ptep, 1076 uintptr_t flags) 1077 { 1078 const vm_prot_t prot = (flags & VM_PROT_ALL); 1079 1080 UVMHIST_FUNC(__func__); 1081 UVMHIST_CALLARGS(pmaphist, "(pmap=%#jx kernel=%jx va=%#jx..%#jx)", 1082 (uintptr_t)pmap, (pmap == pmap_kernel() ? 1 : 0), sva, eva); 1083 UVMHIST_LOG(pmaphist, "ptep=%#jx, flags(npte)=%#jx)", 1084 (uintptr_t)ptep, flags, 0, 0); 1085 1086 KASSERT(kpreempt_disabled()); 1087 /* 1088 * Change protection on every valid mapping within this segment. 1089 */ 1090 for (; sva < eva; sva += NBPG, ptep++) { 1091 pt_entry_t pte = *ptep; 1092 if (!pte_valid_p(pte)) 1093 continue; 1094 struct vm_page * const pg = PHYS_TO_VM_PAGE(pte_to_paddr(pte)); 1095 if (pg != NULL && pte_modified_p(pte)) { 1096 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 1097 if (VM_PAGEMD_EXECPAGE_P(mdpg)) { 1098 KASSERT(!VM_PAGEMD_PVLIST_EMPTY_P(mdpg)); 1099 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 1100 if (VM_PAGEMD_CACHED_P(mdpg)) { 1101 #endif 1102 UVMHIST_LOG(pmapexechist, 1103 "pg %#jx (pa %#jx): " 1104 "syncicached performed", 1105 (uintptr_t)pg, VM_PAGE_TO_PHYS(pg), 1106 0, 0); 1107 pmap_page_syncicache(pg); 1108 PMAP_COUNT(exec_synced_protect); 1109 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 1110 } 1111 #endif 1112 } 1113 } 1114 pte = pte_prot_downgrade(pte, prot); 1115 if (*ptep != pte) { 1116 pmap_tlb_miss_lock_enter(); 1117 pte_set(ptep, pte); 1118 /* 1119 * Update the TLB if needed. 1120 */ 1121 pmap_tlb_update_addr(pmap, sva, pte, PMAP_TLB_NEED_IPI); 1122 pmap_tlb_miss_lock_exit(); 1123 } 1124 } 1125 1126 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 1127 1128 return false; 1129 } 1130 1131 /* 1132 * Set the physical protection on the 1133 * specified range of this map as requested. 1134 */ 1135 void 1136 pmap_protect(pmap_t pmap, vaddr_t sva, vaddr_t eva, vm_prot_t prot) 1137 { 1138 UVMHIST_FUNC(__func__); 1139 UVMHIST_CALLARGS(pmaphist, "(pmap=%#jx, va=%#jx..%#jx, prot=%ju)", 1140 (uintptr_t)pmap, sva, eva, prot); 1141 PMAP_COUNT(protect); 1142 1143 if ((prot & VM_PROT_READ) == VM_PROT_NONE) { 1144 pmap_remove(pmap, sva, eva); 1145 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 1146 return; 1147 } 1148 1149 /* 1150 * Change protection on every valid mapping within this segment. 1151 */ 1152 kpreempt_disable(); 1153 pmap_addr_range_check(pmap, sva, eva, __func__); 1154 pmap_pte_process(pmap, sva, eva, pmap_pte_protect, prot); 1155 kpreempt_enable(); 1156 1157 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 1158 } 1159 1160 #if defined(PMAP_VIRTUAL_CACHE_ALIASES) && !defined(PMAP_NO_PV_UNCACHED) 1161 /* 1162 * pmap_page_cache: 1163 * 1164 * Change all mappings of a managed page to cached/uncached. 1165 */ 1166 void 1167 pmap_page_cache(struct vm_page *pg, bool cached) 1168 { 1169 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 1170 1171 UVMHIST_FUNC(__func__); 1172 UVMHIST_CALLARGS(pmaphist, "(pg=%#jx (pa %#jx) cached=%jd)", 1173 (uintptr_t)pg, VM_PAGE_TO_PHYS(pg), cached, 0); 1174 1175 KASSERT(kpreempt_disabled()); 1176 KASSERT(VM_PAGEMD_PVLIST_LOCKED_P(mdpg)); 1177 1178 if (cached) { 1179 pmap_page_clear_attributes(mdpg, VM_PAGEMD_UNCACHED); 1180 PMAP_COUNT(page_cache_restorations); 1181 } else { 1182 pmap_page_set_attributes(mdpg, VM_PAGEMD_UNCACHED); 1183 PMAP_COUNT(page_cache_evictions); 1184 } 1185 1186 for (pv_entry_t pv = &mdpg->mdpg_first; pv != NULL; pv = pv->pv_next) { 1187 pmap_t pmap = pv->pv_pmap; 1188 vaddr_t va = trunc_page(pv->pv_va); 1189 1190 KASSERT(pmap != NULL); 1191 KASSERT(pmap != pmap_kernel() || !pmap_md_direct_mapped_vaddr_p(va)); 1192 pt_entry_t * const ptep = pmap_pte_lookup(pmap, va); 1193 if (ptep == NULL) 1194 continue; 1195 pt_entry_t pte = *ptep; 1196 if (pte_valid_p(pte)) { 1197 pte = pte_cached_change(pte, cached); 1198 pmap_tlb_miss_lock_enter(); 1199 pte_set(ptep, pte); 1200 pmap_tlb_update_addr(pmap, va, pte, PMAP_TLB_NEED_IPI); 1201 pmap_tlb_miss_lock_exit(); 1202 } 1203 } 1204 1205 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 1206 } 1207 #endif /* PMAP_VIRTUAL_CACHE_ALIASES && !PMAP_NO_PV_UNCACHED */ 1208 1209 /* 1210 * Insert the given physical page (p) at 1211 * the specified virtual address (v) in the 1212 * target physical map with the protection requested. 1213 * 1214 * If specified, the page will be wired down, meaning 1215 * that the related pte can not be reclaimed. 1216 * 1217 * NB: This is the only routine which MAY NOT lazy-evaluate 1218 * or lose information. That is, this routine must actually 1219 * insert this page into the given map NOW. 1220 */ 1221 int 1222 pmap_enter(pmap_t pmap, vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags) 1223 { 1224 const bool wired = (flags & PMAP_WIRED) != 0; 1225 const bool is_kernel_pmap_p = (pmap == pmap_kernel()); 1226 u_int update_flags = (flags & VM_PROT_ALL) != 0 ? PMAP_TLB_INSERT : 0; 1227 #ifdef UVMHIST 1228 struct kern_history * const histp = 1229 ((prot & VM_PROT_EXECUTE) ? &pmapexechist : &pmaphist); 1230 #endif 1231 1232 UVMHIST_FUNC(__func__); 1233 UVMHIST_CALLARGS(*histp, "(pmap=%#jx, va=%#jx, pa=%#jx", 1234 (uintptr_t)pmap, va, pa, 0); 1235 UVMHIST_LOG(*histp, "prot=%#jx flags=%#jx)", prot, flags, 0, 0); 1236 1237 const bool good_color = PMAP_PAGE_COLOROK_P(pa, va); 1238 if (is_kernel_pmap_p) { 1239 PMAP_COUNT(kernel_mappings); 1240 if (!good_color) 1241 PMAP_COUNT(kernel_mappings_bad); 1242 } else { 1243 PMAP_COUNT(user_mappings); 1244 if (!good_color) 1245 PMAP_COUNT(user_mappings_bad); 1246 } 1247 pmap_addr_range_check(pmap, va, va, __func__); 1248 1249 KASSERTMSG(prot & VM_PROT_READ, "no READ (%#x) in prot %#x", 1250 VM_PROT_READ, prot); 1251 1252 struct vm_page * const pg = PHYS_TO_VM_PAGE(pa); 1253 struct vm_page_md * const mdpg = (pg ? VM_PAGE_TO_MD(pg) : NULL); 1254 1255 if (pg) { 1256 /* Set page referenced/modified status based on flags */ 1257 if (flags & VM_PROT_WRITE) { 1258 pmap_page_set_attributes(mdpg, VM_PAGEMD_MODIFIED|VM_PAGEMD_REFERENCED); 1259 } else if (flags & VM_PROT_ALL) { 1260 pmap_page_set_attributes(mdpg, VM_PAGEMD_REFERENCED); 1261 } 1262 1263 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 1264 if (!VM_PAGEMD_CACHED_P(mdpg)) { 1265 flags |= PMAP_NOCACHE; 1266 PMAP_COUNT(uncached_mappings); 1267 } 1268 #endif 1269 1270 PMAP_COUNT(managed_mappings); 1271 } else { 1272 /* 1273 * Assumption: if it is not part of our managed memory 1274 * then it must be device memory which may be volatile. 1275 */ 1276 if ((flags & PMAP_CACHE_MASK) == 0) 1277 flags |= PMAP_NOCACHE; 1278 PMAP_COUNT(unmanaged_mappings); 1279 } 1280 1281 pt_entry_t npte = pte_make_enter(pa, mdpg, prot, flags, 1282 is_kernel_pmap_p); 1283 1284 kpreempt_disable(); 1285 1286 pt_entry_t * const ptep = pmap_pte_reserve(pmap, va, flags); 1287 if (__predict_false(ptep == NULL)) { 1288 kpreempt_enable(); 1289 UVMHIST_LOG(*histp, " <-- ENOMEM", 0, 0, 0, 0); 1290 return ENOMEM; 1291 } 1292 const pt_entry_t opte = *ptep; 1293 const bool resident = pte_valid_p(opte); 1294 bool remap = false; 1295 if (resident) { 1296 if (pte_to_paddr(opte) != pa) { 1297 KASSERT(!is_kernel_pmap_p); 1298 const pt_entry_t rpte = pte_nv_entry(false); 1299 1300 pmap_addr_range_check(pmap, va, va + NBPG, __func__); 1301 pmap_pte_process(pmap, va, va + NBPG, pmap_pte_remove, 1302 rpte); 1303 PMAP_COUNT(user_mappings_changed); 1304 remap = true; 1305 } 1306 update_flags |= PMAP_TLB_NEED_IPI; 1307 } 1308 1309 if (!resident || remap) { 1310 pmap->pm_stats.resident_count++; 1311 } 1312 1313 /* Done after case that may sleep/return. */ 1314 if (pg) 1315 pmap_enter_pv(pmap, va, pg, &npte, 0); 1316 1317 /* 1318 * Now validate mapping with desired protection/wiring. 1319 */ 1320 if (wired) { 1321 pmap->pm_stats.wired_count++; 1322 npte = pte_wire_entry(npte); 1323 } 1324 1325 UVMHIST_LOG(*histp, "new pte %#jx (pa %#jx)", 1326 pte_value(npte), pa, 0, 0); 1327 1328 KASSERT(pte_valid_p(npte)); 1329 1330 pmap_tlb_miss_lock_enter(); 1331 pte_set(ptep, npte); 1332 pmap_tlb_update_addr(pmap, va, npte, update_flags); 1333 pmap_tlb_miss_lock_exit(); 1334 kpreempt_enable(); 1335 1336 if (pg != NULL && (prot == (VM_PROT_READ | VM_PROT_EXECUTE))) { 1337 KASSERT(mdpg != NULL); 1338 PMAP_COUNT(exec_mappings); 1339 if (!VM_PAGEMD_EXECPAGE_P(mdpg) && pte_cached_p(npte)) { 1340 if (!pte_deferred_exec_p(npte)) { 1341 UVMHIST_LOG(*histp, "va=%#jx pg %#jx: " 1342 "immediate syncicache", 1343 va, (uintptr_t)pg, 0, 0); 1344 pmap_page_syncicache(pg); 1345 pmap_page_set_attributes(mdpg, 1346 VM_PAGEMD_EXECPAGE); 1347 PMAP_COUNT(exec_synced_mappings); 1348 } else { 1349 UVMHIST_LOG(*histp, "va=%#jx pg %#jx: defer " 1350 "syncicache: pte %#jx", 1351 va, (uintptr_t)pg, npte, 0); 1352 } 1353 } else { 1354 UVMHIST_LOG(*histp, 1355 "va=%#jx pg %#jx: no syncicache cached %jd", 1356 va, (uintptr_t)pg, pte_cached_p(npte), 0); 1357 } 1358 } else if (pg != NULL && (prot & VM_PROT_EXECUTE)) { 1359 KASSERT(mdpg != NULL); 1360 KASSERT(prot & VM_PROT_WRITE); 1361 PMAP_COUNT(exec_mappings); 1362 pmap_page_syncicache(pg); 1363 pmap_page_clear_attributes(mdpg, VM_PAGEMD_EXECPAGE); 1364 UVMHIST_LOG(*histp, 1365 "va=%#jx pg %#jx: immediate syncicache (writeable)", 1366 va, (uintptr_t)pg, 0, 0); 1367 } 1368 1369 UVMHIST_LOG(*histp, " <-- 0 (OK)", 0, 0, 0, 0); 1370 return 0; 1371 } 1372 1373 void 1374 pmap_kenter_pa(vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags) 1375 { 1376 pmap_t pmap = pmap_kernel(); 1377 struct vm_page * const pg = PHYS_TO_VM_PAGE(pa); 1378 struct vm_page_md * const mdpg = (pg ? VM_PAGE_TO_MD(pg) : NULL); 1379 1380 UVMHIST_FUNC(__func__); 1381 UVMHIST_CALLARGS(pmaphist, "(va=%#jx pa=%#jx prot=%ju, flags=%#jx)", 1382 va, pa, prot, flags); 1383 PMAP_COUNT(kenter_pa); 1384 1385 if (mdpg == NULL) { 1386 PMAP_COUNT(kenter_pa_unmanaged); 1387 if ((flags & PMAP_CACHE_MASK) == 0) 1388 flags |= PMAP_NOCACHE; 1389 } else { 1390 if ((flags & PMAP_NOCACHE) == 0 && !PMAP_PAGE_COLOROK_P(pa, va)) 1391 PMAP_COUNT(kenter_pa_bad); 1392 } 1393 1394 pt_entry_t npte = pte_make_kenter_pa(pa, mdpg, prot, flags); 1395 kpreempt_disable(); 1396 pt_entry_t * const ptep = pmap_pte_lookup(pmap, va); 1397 KASSERTMSG(ptep != NULL, "%#"PRIxVADDR " %#"PRIxVADDR, va, 1398 pmap_limits.virtual_end); 1399 KASSERT(!pte_valid_p(*ptep)); 1400 1401 /* 1402 * No need to track non-managed pages or PMAP_KMPAGEs pages for aliases 1403 */ 1404 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 1405 if (pg != NULL && (flags & PMAP_KMPAGE) == 0 1406 && pmap_md_virtual_cache_aliasing_p()) { 1407 pmap_enter_pv(pmap, va, pg, &npte, PV_KENTER); 1408 } 1409 #endif 1410 1411 /* 1412 * We have the option to force this mapping into the TLB but we 1413 * don't. Instead let the next reference to the page do it. 1414 */ 1415 pmap_tlb_miss_lock_enter(); 1416 pte_set(ptep, npte); 1417 pmap_tlb_update_addr(pmap_kernel(), va, npte, 0); 1418 pmap_tlb_miss_lock_exit(); 1419 kpreempt_enable(); 1420 #if DEBUG > 1 1421 for (u_int i = 0; i < PAGE_SIZE / sizeof(long); i++) { 1422 if (((long *)va)[i] != ((long *)pa)[i]) 1423 panic("%s: contents (%lx) of va %#"PRIxVADDR 1424 " != contents (%lx) of pa %#"PRIxPADDR, __func__, 1425 ((long *)va)[i], va, ((long *)pa)[i], pa); 1426 } 1427 #endif 1428 1429 UVMHIST_LOG(pmaphist, " <-- done (ptep=%#jx)", (uintptr_t)ptep, 0, 0, 1430 0); 1431 } 1432 1433 /* 1434 * Remove the given range of addresses from the kernel map. 1435 * 1436 * It is assumed that the start and end are properly 1437 * rounded to the page size. 1438 */ 1439 1440 static bool 1441 pmap_pte_kremove(pmap_t pmap, vaddr_t sva, vaddr_t eva, pt_entry_t *ptep, 1442 uintptr_t flags) 1443 { 1444 const pt_entry_t new_pte = pte_nv_entry(true); 1445 1446 UVMHIST_FUNC(__func__); 1447 UVMHIST_CALLARGS(pmaphist, "(pmap=%#jx, sva=%#jx eva=%#jx ptep=%#jx)", 1448 (uintptr_t)pmap, sva, eva, (uintptr_t)ptep); 1449 1450 KASSERT(kpreempt_disabled()); 1451 1452 for (; sva < eva; sva += NBPG, ptep++) { 1453 pt_entry_t pte = *ptep; 1454 if (!pte_valid_p(pte)) 1455 continue; 1456 1457 PMAP_COUNT(kremove_pages); 1458 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 1459 struct vm_page * const pg = PHYS_TO_VM_PAGE(pte_to_paddr(pte)); 1460 if (pg != NULL && pmap_md_virtual_cache_aliasing_p()) { 1461 pmap_remove_pv(pmap, sva, pg, !pte_readonly_p(pte)); 1462 } 1463 #endif 1464 1465 pmap_tlb_miss_lock_enter(); 1466 pte_set(ptep, new_pte); 1467 pmap_tlb_invalidate_addr(pmap, sva); 1468 pmap_tlb_miss_lock_exit(); 1469 } 1470 1471 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 1472 1473 return false; 1474 } 1475 1476 void 1477 pmap_kremove(vaddr_t va, vsize_t len) 1478 { 1479 const vaddr_t sva = trunc_page(va); 1480 const vaddr_t eva = round_page(va + len); 1481 1482 UVMHIST_FUNC(__func__); 1483 UVMHIST_CALLARGS(pmaphist, "(va=%#jx len=%#jx)", va, len, 0, 0); 1484 1485 kpreempt_disable(); 1486 pmap_pte_process(pmap_kernel(), sva, eva, pmap_pte_kremove, 0); 1487 kpreempt_enable(); 1488 1489 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 1490 } 1491 1492 bool 1493 pmap_remove_all(struct pmap *pmap) 1494 { 1495 UVMHIST_FUNC(__func__); 1496 UVMHIST_CALLARGS(pmaphist, "(pm=%#jx)", (uintptr_t)pmap, 0, 0, 0); 1497 1498 KASSERT(pmap != pmap_kernel()); 1499 1500 kpreempt_disable(); 1501 /* 1502 * Free all of our ASIDs which means we can skip doing all the 1503 * tlb_invalidate_addrs(). 1504 */ 1505 pmap_tlb_miss_lock_enter(); 1506 #ifdef MULTIPROCESSOR 1507 // This should be the last CPU with this pmap onproc 1508 KASSERT(!kcpuset_isotherset(pmap->pm_onproc, cpu_index(curcpu()))); 1509 if (kcpuset_isset(pmap->pm_onproc, cpu_index(curcpu()))) 1510 #endif 1511 pmap_tlb_asid_deactivate(pmap); 1512 #ifdef MULTIPROCESSOR 1513 KASSERT(kcpuset_iszero(pmap->pm_onproc)); 1514 #endif 1515 pmap_tlb_asid_release_all(pmap); 1516 pmap_tlb_miss_lock_exit(); 1517 pmap->pm_flags |= PMAP_DEFERRED_ACTIVATE; 1518 1519 #ifdef PMAP_FAULTINFO 1520 curpcb->pcb_faultinfo.pfi_faultaddr = 0; 1521 curpcb->pcb_faultinfo.pfi_repeats = 0; 1522 curpcb->pcb_faultinfo.pfi_faultpte = NULL; 1523 #endif 1524 kpreempt_enable(); 1525 1526 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 1527 return false; 1528 } 1529 1530 /* 1531 * Routine: pmap_unwire 1532 * Function: Clear the wired attribute for a map/virtual-address 1533 * pair. 1534 * In/out conditions: 1535 * The mapping must already exist in the pmap. 1536 */ 1537 void 1538 pmap_unwire(pmap_t pmap, vaddr_t va) 1539 { 1540 UVMHIST_FUNC(__func__); 1541 UVMHIST_CALLARGS(pmaphist, "(pmap=%#jx, va=%#jx)", (uintptr_t)pmap, va, 1542 0, 0); 1543 PMAP_COUNT(unwire); 1544 1545 /* 1546 * Don't need to flush the TLB since PG_WIRED is only in software. 1547 */ 1548 kpreempt_disable(); 1549 pmap_addr_range_check(pmap, va, va, __func__); 1550 pt_entry_t * const ptep = pmap_pte_lookup(pmap, va); 1551 KASSERTMSG(ptep != NULL, "pmap %p va %#"PRIxVADDR" invalid STE", 1552 pmap, va); 1553 pt_entry_t pte = *ptep; 1554 KASSERTMSG(pte_valid_p(pte), 1555 "pmap %p va %#"PRIxVADDR" invalid PTE %#"PRIxPTE" @ %p", 1556 pmap, va, pte_value(pte), ptep); 1557 1558 if (pte_wired_p(pte)) { 1559 pmap_tlb_miss_lock_enter(); 1560 pte_set(ptep, pte_unwire_entry(pte)); 1561 pmap_tlb_miss_lock_exit(); 1562 pmap->pm_stats.wired_count--; 1563 } 1564 #ifdef DIAGNOSTIC 1565 else { 1566 printf("%s: wiring for pmap %p va %#"PRIxVADDR" unchanged!\n", 1567 __func__, pmap, va); 1568 } 1569 #endif 1570 kpreempt_enable(); 1571 1572 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 1573 } 1574 1575 /* 1576 * Routine: pmap_extract 1577 * Function: 1578 * Extract the physical page address associated 1579 * with the given map/virtual_address pair. 1580 */ 1581 bool 1582 pmap_extract(pmap_t pmap, vaddr_t va, paddr_t *pap) 1583 { 1584 paddr_t pa; 1585 1586 if (pmap == pmap_kernel()) { 1587 if (pmap_md_direct_mapped_vaddr_p(va)) { 1588 pa = pmap_md_direct_mapped_vaddr_to_paddr(va); 1589 goto done; 1590 } 1591 if (pmap_md_io_vaddr_p(va)) 1592 panic("pmap_extract: io address %#"PRIxVADDR"", va); 1593 1594 if (va >= pmap_limits.virtual_end) 1595 panic("%s: illegal kernel mapped address %#"PRIxVADDR, 1596 __func__, va); 1597 } 1598 kpreempt_disable(); 1599 const pt_entry_t * const ptep = pmap_pte_lookup(pmap, va); 1600 if (ptep == NULL || !pte_valid_p(*ptep)) { 1601 kpreempt_enable(); 1602 return false; 1603 } 1604 pa = pte_to_paddr(*ptep) | (va & PGOFSET); 1605 kpreempt_enable(); 1606 done: 1607 if (pap != NULL) { 1608 *pap = pa; 1609 } 1610 return true; 1611 } 1612 1613 /* 1614 * Copy the range specified by src_addr/len 1615 * from the source map to the range dst_addr/len 1616 * in the destination map. 1617 * 1618 * This routine is only advisory and need not do anything. 1619 */ 1620 void 1621 pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vaddr_t dst_addr, vsize_t len, 1622 vaddr_t src_addr) 1623 { 1624 UVMHIST_FUNC(__func__); 1625 UVMHIST_CALLED(pmaphist); 1626 PMAP_COUNT(copy); 1627 } 1628 1629 /* 1630 * pmap_clear_reference: 1631 * 1632 * Clear the reference bit on the specified physical page. 1633 */ 1634 bool 1635 pmap_clear_reference(struct vm_page *pg) 1636 { 1637 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 1638 1639 UVMHIST_FUNC(__func__); 1640 UVMHIST_CALLARGS(pmaphist, "(pg=%#jx (pa %#jx))", 1641 (uintptr_t)pg, VM_PAGE_TO_PHYS(pg), 0,0); 1642 1643 bool rv = pmap_page_clear_attributes(mdpg, VM_PAGEMD_REFERENCED); 1644 1645 UVMHIST_LOG(pmaphist, " <-- wasref %ju", rv, 0, 0, 0); 1646 1647 return rv; 1648 } 1649 1650 /* 1651 * pmap_is_referenced: 1652 * 1653 * Return whether or not the specified physical page is referenced 1654 * by any physical maps. 1655 */ 1656 bool 1657 pmap_is_referenced(struct vm_page *pg) 1658 { 1659 return VM_PAGEMD_REFERENCED_P(VM_PAGE_TO_MD(pg)); 1660 } 1661 1662 /* 1663 * Clear the modify bits on the specified physical page. 1664 */ 1665 bool 1666 pmap_clear_modify(struct vm_page *pg) 1667 { 1668 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 1669 pv_entry_t pv = &mdpg->mdpg_first; 1670 pv_entry_t pv_next; 1671 1672 UVMHIST_FUNC(__func__); 1673 UVMHIST_CALLARGS(pmaphist, "(pg=%#jx (%#jx))", 1674 (uintptr_t)pg, VM_PAGE_TO_PHYS(pg), 0,0); 1675 PMAP_COUNT(clear_modify); 1676 1677 if (VM_PAGEMD_EXECPAGE_P(mdpg)) { 1678 if (pv->pv_pmap == NULL) { 1679 UVMHIST_LOG(pmapexechist, 1680 "pg %#jx (pa %#jx): execpage cleared", 1681 (uintptr_t)pg, VM_PAGE_TO_PHYS(pg), 0, 0); 1682 pmap_page_clear_attributes(mdpg, VM_PAGEMD_EXECPAGE); 1683 PMAP_COUNT(exec_uncached_clear_modify); 1684 } else { 1685 UVMHIST_LOG(pmapexechist, 1686 "pg %#jx (pa %#jx): syncicache performed", 1687 (uintptr_t)pg, VM_PAGE_TO_PHYS(pg), 0, 0); 1688 pmap_page_syncicache(pg); 1689 PMAP_COUNT(exec_synced_clear_modify); 1690 } 1691 } 1692 if (!pmap_page_clear_attributes(mdpg, VM_PAGEMD_MODIFIED)) { 1693 UVMHIST_LOG(pmaphist, " <-- false", 0, 0, 0, 0); 1694 return false; 1695 } 1696 if (pv->pv_pmap == NULL) { 1697 UVMHIST_LOG(pmaphist, " <-- true (no mappings)", 0, 0, 0, 0); 1698 return true; 1699 } 1700 1701 /* 1702 * remove write access from any pages that are dirty 1703 * so we can tell if they are written to again later. 1704 * flush the VAC first if there is one. 1705 */ 1706 kpreempt_disable(); 1707 VM_PAGEMD_PVLIST_READLOCK(mdpg); 1708 pmap_pvlist_check(mdpg); 1709 for (; pv != NULL; pv = pv_next) { 1710 pmap_t pmap = pv->pv_pmap; 1711 vaddr_t va = trunc_page(pv->pv_va); 1712 1713 pv_next = pv->pv_next; 1714 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 1715 if (PV_ISKENTER_P(pv)) 1716 continue; 1717 #endif 1718 pt_entry_t * const ptep = pmap_pte_lookup(pmap, va); 1719 KASSERT(ptep); 1720 pt_entry_t pte = pte_prot_nowrite(*ptep); 1721 if (*ptep == pte) { 1722 continue; 1723 } 1724 KASSERT(pte_valid_p(pte)); 1725 const uintptr_t gen = VM_PAGEMD_PVLIST_UNLOCK(mdpg); 1726 pmap_tlb_miss_lock_enter(); 1727 pte_set(ptep, pte); 1728 pmap_tlb_invalidate_addr(pmap, va); 1729 pmap_tlb_miss_lock_exit(); 1730 pmap_update(pmap); 1731 if (__predict_false(gen != VM_PAGEMD_PVLIST_READLOCK(mdpg))) { 1732 /* 1733 * The list changed! So restart from the beginning. 1734 */ 1735 pv_next = &mdpg->mdpg_first; 1736 pmap_pvlist_check(mdpg); 1737 } 1738 } 1739 pmap_pvlist_check(mdpg); 1740 VM_PAGEMD_PVLIST_UNLOCK(mdpg); 1741 kpreempt_enable(); 1742 1743 UVMHIST_LOG(pmaphist, " <-- true (mappings changed)", 0, 0, 0, 0); 1744 return true; 1745 } 1746 1747 /* 1748 * pmap_is_modified: 1749 * 1750 * Return whether or not the specified physical page is modified 1751 * by any physical maps. 1752 */ 1753 bool 1754 pmap_is_modified(struct vm_page *pg) 1755 { 1756 return VM_PAGEMD_MODIFIED_P(VM_PAGE_TO_MD(pg)); 1757 } 1758 1759 /* 1760 * pmap_set_modified: 1761 * 1762 * Sets the page modified reference bit for the specified page. 1763 */ 1764 void 1765 pmap_set_modified(paddr_t pa) 1766 { 1767 struct vm_page * const pg = PHYS_TO_VM_PAGE(pa); 1768 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 1769 pmap_page_set_attributes(mdpg, VM_PAGEMD_MODIFIED|VM_PAGEMD_REFERENCED); 1770 } 1771 1772 /******************** pv_entry management ********************/ 1773 1774 static void 1775 pmap_pvlist_check(struct vm_page_md *mdpg) 1776 { 1777 #ifdef DEBUG 1778 pv_entry_t pv = &mdpg->mdpg_first; 1779 if (pv->pv_pmap != NULL) { 1780 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 1781 const u_int colormask = uvmexp.colormask; 1782 u_int colors = 0; 1783 #endif 1784 for (; pv != NULL; pv = pv->pv_next) { 1785 KASSERT(pv->pv_pmap != pmap_kernel() || !pmap_md_direct_mapped_vaddr_p(pv->pv_va)); 1786 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 1787 colors |= __BIT(atop(pv->pv_va) & colormask); 1788 #endif 1789 } 1790 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 1791 // Assert that if there is more than 1 color mapped, that the 1792 // page is uncached. 1793 KASSERTMSG(!pmap_md_virtual_cache_aliasing_p() 1794 || colors == 0 || (colors & (colors-1)) == 0 1795 || VM_PAGEMD_UNCACHED_P(mdpg), "colors=%#x uncached=%u", 1796 colors, VM_PAGEMD_UNCACHED_P(mdpg)); 1797 #endif 1798 } else { 1799 KASSERT(pv->pv_next == NULL); 1800 } 1801 #endif /* DEBUG */ 1802 } 1803 1804 /* 1805 * Enter the pmap and virtual address into the 1806 * physical to virtual map table. 1807 */ 1808 void 1809 pmap_enter_pv(pmap_t pmap, vaddr_t va, struct vm_page *pg, pt_entry_t *nptep, 1810 u_int flags) 1811 { 1812 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 1813 pv_entry_t pv, npv, apv; 1814 #ifdef UVMHIST 1815 bool first = false; 1816 #endif 1817 1818 UVMHIST_FUNC(__func__); 1819 UVMHIST_CALLARGS(pmaphist, "(pmap=%#jx va=%#jx pg=%#jx (%#jx)", 1820 (uintptr_t)pmap, va, (uintptr_t)pg, VM_PAGE_TO_PHYS(pg)); 1821 UVMHIST_LOG(pmaphist, "nptep=%#jx (%#jx))", 1822 (uintptr_t)nptep, pte_value(*nptep), 0, 0); 1823 1824 KASSERT(kpreempt_disabled()); 1825 KASSERT(pmap != pmap_kernel() || !pmap_md_direct_mapped_vaddr_p(va)); 1826 KASSERTMSG(pmap != pmap_kernel() || !pmap_md_io_vaddr_p(va), 1827 "va %#"PRIxVADDR, va); 1828 1829 apv = NULL; 1830 VM_PAGEMD_PVLIST_LOCK(mdpg); 1831 again: 1832 pv = &mdpg->mdpg_first; 1833 pmap_pvlist_check(mdpg); 1834 if (pv->pv_pmap == NULL) { 1835 KASSERT(pv->pv_next == NULL); 1836 /* 1837 * No entries yet, use header as the first entry 1838 */ 1839 PMAP_COUNT(primary_mappings); 1840 PMAP_COUNT(mappings); 1841 #ifdef UVMHIST 1842 first = true; 1843 #endif 1844 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 1845 KASSERT(VM_PAGEMD_CACHED_P(mdpg)); 1846 // If the new mapping has an incompatible color the last 1847 // mapping of this page, clean the page before using it. 1848 if (!PMAP_PAGE_COLOROK_P(va, pv->pv_va)) { 1849 pmap_md_vca_clean(pg, PMAP_WBINV); 1850 } 1851 #endif 1852 pv->pv_pmap = pmap; 1853 pv->pv_va = va | flags; 1854 } else { 1855 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 1856 if (pmap_md_vca_add(pg, va, nptep)) { 1857 goto again; 1858 } 1859 #endif 1860 1861 /* 1862 * There is at least one other VA mapping this page. 1863 * Place this entry after the header. 1864 * 1865 * Note: the entry may already be in the table if 1866 * we are only changing the protection bits. 1867 */ 1868 1869 #ifdef PARANOIADIAG 1870 const paddr_t pa = VM_PAGE_TO_PHYS(pg); 1871 #endif 1872 for (npv = pv; npv; npv = npv->pv_next) { 1873 if (pmap == npv->pv_pmap 1874 && va == trunc_page(npv->pv_va)) { 1875 #ifdef PARANOIADIAG 1876 pt_entry_t *ptep = pmap_pte_lookup(pmap, va); 1877 pt_entry_t pte = (ptep != NULL) ? *ptep : 0; 1878 if (!pte_valid_p(pte) || pte_to_paddr(pte) != pa) 1879 printf("%s: found va %#"PRIxVADDR 1880 " pa %#"PRIxPADDR 1881 " in pv_table but != %#"PRIxPTE"\n", 1882 __func__, va, pa, pte_value(pte)); 1883 #endif 1884 PMAP_COUNT(remappings); 1885 VM_PAGEMD_PVLIST_UNLOCK(mdpg); 1886 if (__predict_false(apv != NULL)) 1887 pmap_pv_free(apv); 1888 1889 UVMHIST_LOG(pmaphist, 1890 " <-- done pv=%#jx (reused)", 1891 (uintptr_t)pv, 0, 0, 0); 1892 return; 1893 } 1894 } 1895 if (__predict_true(apv == NULL)) { 1896 /* 1897 * To allocate a PV, we have to release the PVLIST lock 1898 * so get the page generation. We allocate the PV, and 1899 * then reacquire the lock. 1900 */ 1901 pmap_pvlist_check(mdpg); 1902 const uintptr_t gen = VM_PAGEMD_PVLIST_UNLOCK(mdpg); 1903 1904 apv = (pv_entry_t)pmap_pv_alloc(); 1905 if (apv == NULL) 1906 panic("pmap_enter_pv: pmap_pv_alloc() failed"); 1907 1908 /* 1909 * If the generation has changed, then someone else 1910 * tinkered with this page so we should start over. 1911 */ 1912 if (gen != VM_PAGEMD_PVLIST_LOCK(mdpg)) 1913 goto again; 1914 } 1915 npv = apv; 1916 apv = NULL; 1917 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 1918 /* 1919 * If need to deal with virtual cache aliases, keep mappings 1920 * in the kernel pmap at the head of the list. This allows 1921 * the VCA code to easily use them for cache operations if 1922 * present. 1923 */ 1924 pmap_t kpmap = pmap_kernel(); 1925 if (pmap != kpmap) { 1926 while (pv->pv_pmap == kpmap && pv->pv_next != NULL) { 1927 pv = pv->pv_next; 1928 } 1929 } 1930 #endif 1931 npv->pv_va = va | flags; 1932 npv->pv_pmap = pmap; 1933 npv->pv_next = pv->pv_next; 1934 pv->pv_next = npv; 1935 PMAP_COUNT(mappings); 1936 } 1937 pmap_pvlist_check(mdpg); 1938 VM_PAGEMD_PVLIST_UNLOCK(mdpg); 1939 if (__predict_false(apv != NULL)) 1940 pmap_pv_free(apv); 1941 1942 UVMHIST_LOG(pmaphist, " <-- done pv=%#jx (first %ju)", (uintptr_t)pv, 1943 first, 0, 0); 1944 } 1945 1946 /* 1947 * Remove a physical to virtual address translation. 1948 * If cache was inhibited on this page, and there are no more cache 1949 * conflicts, restore caching. 1950 * Flush the cache if the last page is removed (should always be cached 1951 * at this point). 1952 */ 1953 void 1954 pmap_remove_pv(pmap_t pmap, vaddr_t va, struct vm_page *pg, bool dirty) 1955 { 1956 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 1957 pv_entry_t pv, npv; 1958 bool last; 1959 1960 UVMHIST_FUNC(__func__); 1961 UVMHIST_CALLARGS(pmaphist, "(pmap=%#jx, va=%#jx, pg=%#jx (pa %#jx)", 1962 (uintptr_t)pmap, va, (uintptr_t)pg, VM_PAGE_TO_PHYS(pg)); 1963 UVMHIST_LOG(pmaphist, "dirty=%ju)", dirty, 0, 0, 0); 1964 1965 KASSERT(kpreempt_disabled()); 1966 KASSERT((va & PAGE_MASK) == 0); 1967 pv = &mdpg->mdpg_first; 1968 1969 VM_PAGEMD_PVLIST_LOCK(mdpg); 1970 pmap_pvlist_check(mdpg); 1971 1972 /* 1973 * If it is the first entry on the list, it is actually 1974 * in the header and we must copy the following entry up 1975 * to the header. Otherwise we must search the list for 1976 * the entry. In either case we free the now unused entry. 1977 */ 1978 1979 last = false; 1980 if (pmap == pv->pv_pmap && va == trunc_page(pv->pv_va)) { 1981 npv = pv->pv_next; 1982 if (npv) { 1983 *pv = *npv; 1984 KASSERT(pv->pv_pmap != NULL); 1985 } else { 1986 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 1987 pmap_page_clear_attributes(mdpg, VM_PAGEMD_UNCACHED); 1988 #endif 1989 pv->pv_pmap = NULL; 1990 last = true; /* Last mapping removed */ 1991 } 1992 PMAP_COUNT(remove_pvfirst); 1993 } else { 1994 for (npv = pv->pv_next; npv; pv = npv, npv = npv->pv_next) { 1995 PMAP_COUNT(remove_pvsearch); 1996 if (pmap == npv->pv_pmap && va == trunc_page(npv->pv_va)) 1997 break; 1998 } 1999 if (npv) { 2000 pv->pv_next = npv->pv_next; 2001 } 2002 } 2003 2004 pmap_pvlist_check(mdpg); 2005 VM_PAGEMD_PVLIST_UNLOCK(mdpg); 2006 2007 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 2008 pmap_md_vca_remove(pg, va, dirty, last); 2009 #endif 2010 2011 /* 2012 * Free the pv_entry if needed. 2013 */ 2014 if (npv) 2015 pmap_pv_free(npv); 2016 if (VM_PAGEMD_EXECPAGE_P(mdpg) && dirty) { 2017 if (last) { 2018 /* 2019 * If this was the page's last mapping, we no longer 2020 * care about its execness. 2021 */ 2022 UVMHIST_LOG(pmapexechist, 2023 "pg %#jx (pa %#jx)last %ju: execpage cleared", 2024 (uintptr_t)pg, VM_PAGE_TO_PHYS(pg), last, 0); 2025 pmap_page_clear_attributes(mdpg, VM_PAGEMD_EXECPAGE); 2026 PMAP_COUNT(exec_uncached_remove); 2027 } else { 2028 /* 2029 * Someone still has it mapped as an executable page 2030 * so we must sync it. 2031 */ 2032 UVMHIST_LOG(pmapexechist, 2033 "pg %#jx (pa %#jx) last %ju: performed syncicache", 2034 (uintptr_t)pg, VM_PAGE_TO_PHYS(pg), last, 0); 2035 pmap_page_syncicache(pg); 2036 PMAP_COUNT(exec_synced_remove); 2037 } 2038 } 2039 2040 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 2041 } 2042 2043 #if defined(MULTIPROCESSOR) 2044 struct pmap_pvlist_info { 2045 kmutex_t *pli_locks[PAGE_SIZE / 32]; 2046 volatile u_int pli_lock_refs[PAGE_SIZE / 32]; 2047 volatile u_int pli_lock_index; 2048 u_int pli_lock_mask; 2049 } pmap_pvlist_info; 2050 2051 void 2052 pmap_pvlist_lock_init(size_t cache_line_size) 2053 { 2054 struct pmap_pvlist_info * const pli = &pmap_pvlist_info; 2055 const vaddr_t lock_page = uvm_pageboot_alloc(PAGE_SIZE); 2056 vaddr_t lock_va = lock_page; 2057 if (sizeof(kmutex_t) > cache_line_size) { 2058 cache_line_size = roundup2(sizeof(kmutex_t), cache_line_size); 2059 } 2060 const size_t nlocks = PAGE_SIZE / cache_line_size; 2061 KASSERT((nlocks & (nlocks - 1)) == 0); 2062 /* 2063 * Now divide the page into a number of mutexes, one per cacheline. 2064 */ 2065 for (size_t i = 0; i < nlocks; lock_va += cache_line_size, i++) { 2066 kmutex_t * const lock = (kmutex_t *)lock_va; 2067 mutex_init(lock, MUTEX_DEFAULT, IPL_HIGH); 2068 pli->pli_locks[i] = lock; 2069 } 2070 pli->pli_lock_mask = nlocks - 1; 2071 } 2072 2073 kmutex_t * 2074 pmap_pvlist_lock_addr(struct vm_page_md *mdpg) 2075 { 2076 struct pmap_pvlist_info * const pli = &pmap_pvlist_info; 2077 kmutex_t *lock = mdpg->mdpg_lock; 2078 2079 /* 2080 * Allocate a lock on an as-needed basis. This will hopefully give us 2081 * semi-random distribution not based on page color. 2082 */ 2083 if (__predict_false(lock == NULL)) { 2084 size_t locknum = atomic_add_int_nv(&pli->pli_lock_index, 37); 2085 size_t lockid = locknum & pli->pli_lock_mask; 2086 kmutex_t * const new_lock = pli->pli_locks[lockid]; 2087 /* 2088 * Set the lock. If some other thread already did, just use 2089 * the one they assigned. 2090 */ 2091 lock = atomic_cas_ptr(&mdpg->mdpg_lock, NULL, new_lock); 2092 if (lock == NULL) { 2093 lock = new_lock; 2094 atomic_inc_uint(&pli->pli_lock_refs[lockid]); 2095 } 2096 } 2097 2098 /* 2099 * Now finally provide the lock. 2100 */ 2101 return lock; 2102 } 2103 #else /* !MULTIPROCESSOR */ 2104 void 2105 pmap_pvlist_lock_init(size_t cache_line_size) 2106 { 2107 mutex_init(&pmap_pvlist_mutex, MUTEX_DEFAULT, IPL_HIGH); 2108 } 2109 2110 #ifdef MODULAR 2111 kmutex_t * 2112 pmap_pvlist_lock_addr(struct vm_page_md *mdpg) 2113 { 2114 /* 2115 * We just use a global lock. 2116 */ 2117 if (__predict_false(mdpg->mdpg_lock == NULL)) { 2118 mdpg->mdpg_lock = &pmap_pvlist_mutex; 2119 } 2120 2121 /* 2122 * Now finally provide the lock. 2123 */ 2124 return mdpg->mdpg_lock; 2125 } 2126 #endif /* MODULAR */ 2127 #endif /* !MULTIPROCESSOR */ 2128 2129 /* 2130 * pmap_pv_page_alloc: 2131 * 2132 * Allocate a page for the pv_entry pool. 2133 */ 2134 void * 2135 pmap_pv_page_alloc(struct pool *pp, int flags) 2136 { 2137 struct vm_page * const pg = PMAP_ALLOC_POOLPAGE(UVM_PGA_USERESERVE); 2138 if (pg == NULL) 2139 return NULL; 2140 2141 return (void *)pmap_map_poolpage(VM_PAGE_TO_PHYS(pg)); 2142 } 2143 2144 /* 2145 * pmap_pv_page_free: 2146 * 2147 * Free a pv_entry pool page. 2148 */ 2149 void 2150 pmap_pv_page_free(struct pool *pp, void *v) 2151 { 2152 vaddr_t va = (vaddr_t)v; 2153 2154 KASSERT(pmap_md_direct_mapped_vaddr_p(va)); 2155 const paddr_t pa = pmap_md_direct_mapped_vaddr_to_paddr(va); 2156 struct vm_page * const pg = PHYS_TO_VM_PAGE(pa); 2157 KASSERT(pg != NULL); 2158 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 2159 kpreempt_disable(); 2160 pmap_md_vca_remove(pg, va, true, true); 2161 kpreempt_enable(); 2162 #endif 2163 pmap_page_clear_attributes(VM_PAGE_TO_MD(pg), VM_PAGEMD_POOLPAGE); 2164 KASSERT(!VM_PAGEMD_EXECPAGE_P(VM_PAGE_TO_MD(pg))); 2165 uvm_pagefree(pg); 2166 } 2167 2168 #ifdef PMAP_PREFER 2169 /* 2170 * Find first virtual address >= *vap that doesn't cause 2171 * a cache alias conflict. 2172 */ 2173 void 2174 pmap_prefer(vaddr_t foff, vaddr_t *vap, vsize_t sz, int td) 2175 { 2176 vsize_t prefer_mask = ptoa(uvmexp.colormask); 2177 2178 PMAP_COUNT(prefer_requests); 2179 2180 prefer_mask |= pmap_md_cache_prefer_mask(); 2181 2182 if (prefer_mask) { 2183 vaddr_t va = *vap; 2184 vsize_t d = (foff - va) & prefer_mask; 2185 if (d) { 2186 if (td) 2187 *vap = trunc_page(va - ((-d) & prefer_mask)); 2188 else 2189 *vap = round_page(va + d); 2190 PMAP_COUNT(prefer_adjustments); 2191 } 2192 } 2193 } 2194 #endif /* PMAP_PREFER */ 2195 2196 #ifdef PMAP_MAP_POOLPAGE 2197 vaddr_t 2198 pmap_map_poolpage(paddr_t pa) 2199 { 2200 struct vm_page * const pg = PHYS_TO_VM_PAGE(pa); 2201 KASSERT(pg); 2202 2203 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 2204 KASSERT(!VM_PAGEMD_EXECPAGE_P(mdpg)); 2205 2206 pmap_page_set_attributes(mdpg, VM_PAGEMD_POOLPAGE); 2207 2208 return pmap_md_map_poolpage(pa, NBPG); 2209 } 2210 2211 paddr_t 2212 pmap_unmap_poolpage(vaddr_t va) 2213 { 2214 KASSERT(pmap_md_direct_mapped_vaddr_p(va)); 2215 paddr_t pa = pmap_md_direct_mapped_vaddr_to_paddr(va); 2216 2217 struct vm_page * const pg = PHYS_TO_VM_PAGE(pa); 2218 KASSERT(pg != NULL); 2219 KASSERT(!VM_PAGEMD_EXECPAGE_P(VM_PAGE_TO_MD(pg))); 2220 2221 pmap_page_clear_attributes(VM_PAGE_TO_MD(pg), VM_PAGEMD_POOLPAGE); 2222 pmap_md_unmap_poolpage(va, NBPG); 2223 2224 return pa; 2225 } 2226 #endif /* PMAP_MAP_POOLPAGE */ 2227