1 /* $NetBSD: pmap.c,v 1.45 2019/12/18 10:55:50 skrll Exp $ */ 2 3 /*- 4 * Copyright (c) 1998, 2001 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility, 9 * NASA Ames Research Center and by Chris G. Demetriou. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 22 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 23 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 24 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 * POSSIBILITY OF SUCH DAMAGE. 31 */ 32 33 /* 34 * Copyright (c) 1992, 1993 35 * The Regents of the University of California. All rights reserved. 36 * 37 * This code is derived from software contributed to Berkeley by 38 * the Systems Programming Group of the University of Utah Computer 39 * Science Department and Ralph Campbell. 40 * 41 * Redistribution and use in source and binary forms, with or without 42 * modification, are permitted provided that the following conditions 43 * are met: 44 * 1. Redistributions of source code must retain the above copyright 45 * notice, this list of conditions and the following disclaimer. 46 * 2. Redistributions in binary form must reproduce the above copyright 47 * notice, this list of conditions and the following disclaimer in the 48 * documentation and/or other materials provided with the distribution. 49 * 3. Neither the name of the University nor the names of its contributors 50 * may be used to endorse or promote products derived from this software 51 * without specific prior written permission. 52 * 53 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 54 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 55 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 56 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 57 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 58 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 59 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 60 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 61 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 62 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 63 * SUCH DAMAGE. 64 * 65 * @(#)pmap.c 8.4 (Berkeley) 1/26/94 66 */ 67 68 #include <sys/cdefs.h> 69 70 __KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.45 2019/12/18 10:55:50 skrll Exp $"); 71 72 /* 73 * Manages physical address maps. 74 * 75 * In addition to hardware address maps, this 76 * module is called upon to provide software-use-only 77 * maps which may or may not be stored in the same 78 * form as hardware maps. These pseudo-maps are 79 * used to store intermediate results from copy 80 * operations to and from address spaces. 81 * 82 * Since the information managed by this module is 83 * also stored by the logical address mapping module, 84 * this module may throw away valid virtual-to-physical 85 * mappings at almost any time. However, invalidations 86 * of virtual-to-physical mappings must be done as 87 * requested. 88 * 89 * In order to cope with hardware architectures which 90 * make virtual-to-physical map invalidates expensive, 91 * this module may delay invalidate or reduced protection 92 * operations until such time as they are actually 93 * necessary. This module is given full information as 94 * to which processors are currently using which maps, 95 * and to when physical maps must be made correct. 96 */ 97 98 #include "opt_modular.h" 99 #include "opt_multiprocessor.h" 100 #include "opt_sysv.h" 101 102 #define __PMAP_PRIVATE 103 104 #include <sys/param.h> 105 106 #include <sys/atomic.h> 107 #include <sys/buf.h> 108 #include <sys/cpu.h> 109 #include <sys/mutex.h> 110 #include <sys/pool.h> 111 112 #include <uvm/uvm.h> 113 #include <uvm/uvm_physseg.h> 114 115 #if defined(MULTIPROCESSOR) && defined(PMAP_VIRTUAL_CACHE_ALIASES) \ 116 && !defined(PMAP_NO_PV_UNCACHED) 117 #error PMAP_VIRTUAL_CACHE_ALIASES with MULTIPROCESSOR requires \ 118 PMAP_NO_PV_UNCACHED to be defined 119 #endif 120 121 PMAP_COUNTER(remove_kernel_calls, "remove kernel calls"); 122 PMAP_COUNTER(remove_kernel_pages, "kernel pages unmapped"); 123 PMAP_COUNTER(remove_user_calls, "remove user calls"); 124 PMAP_COUNTER(remove_user_pages, "user pages unmapped"); 125 PMAP_COUNTER(remove_flushes, "remove cache flushes"); 126 PMAP_COUNTER(remove_tlb_ops, "remove tlb ops"); 127 PMAP_COUNTER(remove_pvfirst, "remove pv first"); 128 PMAP_COUNTER(remove_pvsearch, "remove pv search"); 129 130 PMAP_COUNTER(prefer_requests, "prefer requests"); 131 PMAP_COUNTER(prefer_adjustments, "prefer adjustments"); 132 133 PMAP_COUNTER(idlezeroed_pages, "pages idle zeroed"); 134 135 PMAP_COUNTER(kenter_pa, "kernel fast mapped pages"); 136 PMAP_COUNTER(kenter_pa_bad, "kernel fast mapped pages (bad color)"); 137 PMAP_COUNTER(kenter_pa_unmanaged, "kernel fast mapped unmanaged pages"); 138 PMAP_COUNTER(kremove_pages, "kernel fast unmapped pages"); 139 140 PMAP_COUNTER(page_cache_evictions, "pages changed to uncacheable"); 141 PMAP_COUNTER(page_cache_restorations, "pages changed to cacheable"); 142 143 PMAP_COUNTER(kernel_mappings_bad, "kernel pages mapped (bad color)"); 144 PMAP_COUNTER(user_mappings_bad, "user pages mapped (bad color)"); 145 PMAP_COUNTER(kernel_mappings, "kernel pages mapped"); 146 PMAP_COUNTER(user_mappings, "user pages mapped"); 147 PMAP_COUNTER(user_mappings_changed, "user mapping changed"); 148 PMAP_COUNTER(kernel_mappings_changed, "kernel mapping changed"); 149 PMAP_COUNTER(uncached_mappings, "uncached pages mapped"); 150 PMAP_COUNTER(unmanaged_mappings, "unmanaged pages mapped"); 151 PMAP_COUNTER(managed_mappings, "managed pages mapped"); 152 PMAP_COUNTER(mappings, "pages mapped"); 153 PMAP_COUNTER(remappings, "pages remapped"); 154 PMAP_COUNTER(unmappings, "pages unmapped"); 155 PMAP_COUNTER(primary_mappings, "page initial mappings"); 156 PMAP_COUNTER(primary_unmappings, "page final unmappings"); 157 PMAP_COUNTER(tlb_hit, "page mapping"); 158 159 PMAP_COUNTER(exec_mappings, "exec pages mapped"); 160 PMAP_COUNTER(exec_synced_mappings, "exec pages synced"); 161 PMAP_COUNTER(exec_synced_remove, "exec pages synced (PR)"); 162 PMAP_COUNTER(exec_synced_clear_modify, "exec pages synced (CM)"); 163 PMAP_COUNTER(exec_synced_page_protect, "exec pages synced (PP)"); 164 PMAP_COUNTER(exec_synced_protect, "exec pages synced (P)"); 165 PMAP_COUNTER(exec_uncached_page_protect, "exec pages uncached (PP)"); 166 PMAP_COUNTER(exec_uncached_clear_modify, "exec pages uncached (CM)"); 167 PMAP_COUNTER(exec_uncached_zero_page, "exec pages uncached (ZP)"); 168 PMAP_COUNTER(exec_uncached_copy_page, "exec pages uncached (CP)"); 169 PMAP_COUNTER(exec_uncached_remove, "exec pages uncached (PR)"); 170 171 PMAP_COUNTER(create, "creates"); 172 PMAP_COUNTER(reference, "references"); 173 PMAP_COUNTER(dereference, "dereferences"); 174 PMAP_COUNTER(destroy, "destroyed"); 175 PMAP_COUNTER(activate, "activations"); 176 PMAP_COUNTER(deactivate, "deactivations"); 177 PMAP_COUNTER(update, "updates"); 178 #ifdef MULTIPROCESSOR 179 PMAP_COUNTER(shootdown_ipis, "shootdown IPIs"); 180 #endif 181 PMAP_COUNTER(unwire, "unwires"); 182 PMAP_COUNTER(copy, "copies"); 183 PMAP_COUNTER(clear_modify, "clear_modifies"); 184 PMAP_COUNTER(protect, "protects"); 185 PMAP_COUNTER(page_protect, "page_protects"); 186 187 #define PMAP_ASID_RESERVED 0 188 CTASSERT(PMAP_ASID_RESERVED == 0); 189 190 #ifndef PMAP_SEGTAB_ALIGN 191 #define PMAP_SEGTAB_ALIGN /* nothing */ 192 #endif 193 #ifdef _LP64 194 pmap_segtab_t pmap_kstart_segtab PMAP_SEGTAB_ALIGN; /* first mid-level segtab for kernel */ 195 #endif 196 pmap_segtab_t pmap_kern_segtab PMAP_SEGTAB_ALIGN = { /* top level segtab for kernel */ 197 #ifdef _LP64 198 .seg_seg[(VM_MIN_KERNEL_ADDRESS & XSEGOFSET) >> SEGSHIFT] = &pmap_kstart_segtab, 199 #endif 200 }; 201 202 struct pmap_kernel kernel_pmap_store = { 203 .kernel_pmap = { 204 .pm_count = 1, 205 .pm_segtab = &pmap_kern_segtab, 206 .pm_minaddr = VM_MIN_KERNEL_ADDRESS, 207 .pm_maxaddr = VM_MAX_KERNEL_ADDRESS, 208 }, 209 }; 210 211 struct pmap * const kernel_pmap_ptr = &kernel_pmap_store.kernel_pmap; 212 213 struct pmap_limits pmap_limits = { /* VA and PA limits */ 214 .virtual_start = VM_MIN_KERNEL_ADDRESS, 215 }; 216 217 #ifdef UVMHIST 218 static struct kern_history_ent pmapexechistbuf[10000]; 219 static struct kern_history_ent pmaphistbuf[10000]; 220 UVMHIST_DEFINE(pmapexechist); 221 UVMHIST_DEFINE(pmaphist); 222 #endif 223 224 /* 225 * The pools from which pmap structures and sub-structures are allocated. 226 */ 227 struct pool pmap_pmap_pool; 228 struct pool pmap_pv_pool; 229 230 #ifndef PMAP_PV_LOWAT 231 #define PMAP_PV_LOWAT 16 232 #endif 233 int pmap_pv_lowat = PMAP_PV_LOWAT; 234 235 bool pmap_initialized = false; 236 #define PMAP_PAGE_COLOROK_P(a, b) \ 237 ((((int)(a) ^ (int)(b)) & pmap_page_colormask) == 0) 238 u_int pmap_page_colormask; 239 240 #define PAGE_IS_MANAGED(pa) (pmap_initialized && uvm_pageismanaged(pa)) 241 242 #define PMAP_IS_ACTIVE(pm) \ 243 ((pm) == pmap_kernel() || \ 244 (pm) == curlwp->l_proc->p_vmspace->vm_map.pmap) 245 246 /* Forward function declarations */ 247 void pmap_page_remove(struct vm_page *); 248 static void pmap_pvlist_check(struct vm_page_md *); 249 void pmap_remove_pv(pmap_t, vaddr_t, struct vm_page *, bool); 250 void pmap_enter_pv(pmap_t, vaddr_t, struct vm_page *, pt_entry_t *, u_int); 251 252 /* 253 * PV table management functions. 254 */ 255 void *pmap_pv_page_alloc(struct pool *, int); 256 void pmap_pv_page_free(struct pool *, void *); 257 258 struct pool_allocator pmap_pv_page_allocator = { 259 pmap_pv_page_alloc, pmap_pv_page_free, 0, 260 }; 261 262 #define pmap_pv_alloc() pool_get(&pmap_pv_pool, PR_NOWAIT) 263 #define pmap_pv_free(pv) pool_put(&pmap_pv_pool, (pv)) 264 265 #if !defined(MULTIPROCESSOR) || !defined(PMAP_MD_NEED_TLB_MISS_LOCK) 266 #define pmap_md_tlb_miss_lock_enter() do { } while(/*CONSTCOND*/0) 267 #define pmap_md_tlb_miss_lock_exit() do { } while(/*CONSTCOND*/0) 268 #endif /* !MULTIPROCESSOR || !PMAP_MD_NEED_TLB_MISS_LOCK */ 269 270 #ifndef MULTIPROCESSOR 271 kmutex_t pmap_pvlist_mutex __cacheline_aligned; 272 #endif 273 274 /* 275 * Debug functions. 276 */ 277 278 #ifdef DEBUG 279 static inline void 280 pmap_asid_check(pmap_t pm, const char *func) 281 { 282 if (!PMAP_IS_ACTIVE(pm)) 283 return; 284 285 struct pmap_asid_info * const pai = PMAP_PAI(pm, cpu_tlb_info(curcpu())); 286 tlb_asid_t asid = tlb_get_asid(); 287 if (asid != pai->pai_asid) 288 panic("%s: inconsistency for active TLB update: %u <-> %u", 289 func, asid, pai->pai_asid); 290 } 291 #endif 292 293 static void 294 pmap_addr_range_check(pmap_t pmap, vaddr_t sva, vaddr_t eva, const char *func) 295 { 296 #ifdef DEBUG 297 if (pmap == pmap_kernel()) { 298 if (sva < VM_MIN_KERNEL_ADDRESS) 299 panic("%s: kva %#"PRIxVADDR" not in range", 300 func, sva); 301 if (eva >= pmap_limits.virtual_end) 302 panic("%s: kva %#"PRIxVADDR" not in range", 303 func, eva); 304 } else { 305 if (eva > VM_MAXUSER_ADDRESS) 306 panic("%s: uva %#"PRIxVADDR" not in range", 307 func, eva); 308 pmap_asid_check(pmap, func); 309 } 310 #endif 311 } 312 313 /* 314 * Misc. functions. 315 */ 316 317 bool 318 pmap_page_clear_attributes(struct vm_page_md *mdpg, u_int clear_attributes) 319 { 320 volatile unsigned long * const attrp = &mdpg->mdpg_attrs; 321 #ifdef MULTIPROCESSOR 322 for (;;) { 323 u_int old_attr = *attrp; 324 if ((old_attr & clear_attributes) == 0) 325 return false; 326 u_int new_attr = old_attr & ~clear_attributes; 327 if (old_attr == atomic_cas_ulong(attrp, old_attr, new_attr)) 328 return true; 329 } 330 #else 331 unsigned long old_attr = *attrp; 332 if ((old_attr & clear_attributes) == 0) 333 return false; 334 *attrp &= ~clear_attributes; 335 return true; 336 #endif 337 } 338 339 void 340 pmap_page_set_attributes(struct vm_page_md *mdpg, u_int set_attributes) 341 { 342 #ifdef MULTIPROCESSOR 343 atomic_or_ulong(&mdpg->mdpg_attrs, set_attributes); 344 #else 345 mdpg->mdpg_attrs |= set_attributes; 346 #endif 347 } 348 349 static void 350 pmap_page_syncicache(struct vm_page *pg) 351 { 352 #ifndef MULTIPROCESSOR 353 struct pmap * const curpmap = curlwp->l_proc->p_vmspace->vm_map.pmap; 354 #endif 355 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 356 pv_entry_t pv = &mdpg->mdpg_first; 357 kcpuset_t *onproc; 358 #ifdef MULTIPROCESSOR 359 kcpuset_create(&onproc, true); 360 KASSERT(onproc != NULL); 361 #else 362 onproc = NULL; 363 #endif 364 VM_PAGEMD_PVLIST_READLOCK(mdpg); 365 pmap_pvlist_check(mdpg); 366 367 if (pv->pv_pmap != NULL) { 368 for (; pv != NULL; pv = pv->pv_next) { 369 #ifdef MULTIPROCESSOR 370 kcpuset_merge(onproc, pv->pv_pmap->pm_onproc); 371 if (kcpuset_match(onproc, kcpuset_running)) { 372 break; 373 } 374 #else 375 if (pv->pv_pmap == curpmap) { 376 onproc = curcpu()->ci_data.cpu_kcpuset; 377 break; 378 } 379 #endif 380 } 381 } 382 pmap_pvlist_check(mdpg); 383 VM_PAGEMD_PVLIST_UNLOCK(mdpg); 384 kpreempt_disable(); 385 pmap_md_page_syncicache(pg, onproc); 386 kpreempt_enable(); 387 #ifdef MULTIPROCESSOR 388 kcpuset_destroy(onproc); 389 #endif 390 } 391 392 /* 393 * Define the initial bounds of the kernel virtual address space. 394 */ 395 void 396 pmap_virtual_space(vaddr_t *vstartp, vaddr_t *vendp) 397 { 398 399 *vstartp = pmap_limits.virtual_start; 400 *vendp = pmap_limits.virtual_end; 401 } 402 403 vaddr_t 404 pmap_growkernel(vaddr_t maxkvaddr) 405 { 406 vaddr_t virtual_end = pmap_limits.virtual_end; 407 maxkvaddr = pmap_round_seg(maxkvaddr) - 1; 408 409 /* 410 * Reserve PTEs for the new KVA space. 411 */ 412 for (; virtual_end < maxkvaddr; virtual_end += NBSEG) { 413 pmap_pte_reserve(pmap_kernel(), virtual_end, 0); 414 } 415 416 /* 417 * Don't exceed VM_MAX_KERNEL_ADDRESS! 418 */ 419 if (virtual_end == 0 || virtual_end > VM_MAX_KERNEL_ADDRESS) 420 virtual_end = VM_MAX_KERNEL_ADDRESS; 421 422 /* 423 * Update new end. 424 */ 425 pmap_limits.virtual_end = virtual_end; 426 return virtual_end; 427 } 428 429 /* 430 * Bootstrap memory allocator (alternative to vm_bootstrap_steal_memory()). 431 * This function allows for early dynamic memory allocation until the virtual 432 * memory system has been bootstrapped. After that point, either kmem_alloc 433 * or malloc should be used. This function works by stealing pages from the 434 * (to be) managed page pool, then implicitly mapping the pages (by using 435 * their direct mapped addresses) and zeroing them. 436 * 437 * It may be used once the physical memory segments have been pre-loaded 438 * into the vm_physmem[] array. Early memory allocation MUST use this 439 * interface! This cannot be used after vm_page_startup(), and will 440 * generate a panic if tried. 441 * 442 * Note that this memory will never be freed, and in essence it is wired 443 * down. 444 * 445 * We must adjust *vstartp and/or *vendp iff we use address space 446 * from the kernel virtual address range defined by pmap_virtual_space(). 447 */ 448 vaddr_t 449 pmap_steal_memory(vsize_t size, vaddr_t *vstartp, vaddr_t *vendp) 450 { 451 size_t npgs; 452 paddr_t pa; 453 vaddr_t va; 454 455 uvm_physseg_t maybe_bank = UVM_PHYSSEG_TYPE_INVALID; 456 457 size = round_page(size); 458 npgs = atop(size); 459 460 aprint_debug("%s: need %zu pages\n", __func__, npgs); 461 462 for (uvm_physseg_t bank = uvm_physseg_get_first(); 463 uvm_physseg_valid_p(bank); 464 bank = uvm_physseg_get_next(bank)) { 465 466 if (uvm.page_init_done == true) 467 panic("pmap_steal_memory: called _after_ bootstrap"); 468 469 aprint_debug("%s: seg %"PRIxPHYSSEG": %#"PRIxPADDR" %#"PRIxPADDR" %#"PRIxPADDR" %#"PRIxPADDR"\n", 470 __func__, bank, 471 uvm_physseg_get_avail_start(bank), uvm_physseg_get_start(bank), 472 uvm_physseg_get_avail_end(bank), uvm_physseg_get_end(bank)); 473 474 if (uvm_physseg_get_avail_start(bank) != uvm_physseg_get_start(bank) 475 || uvm_physseg_get_avail_start(bank) >= uvm_physseg_get_avail_end(bank)) { 476 aprint_debug("%s: seg %"PRIxPHYSSEG": bad start\n", __func__, bank); 477 continue; 478 } 479 480 if (uvm_physseg_get_avail_end(bank) - uvm_physseg_get_avail_start(bank) < npgs) { 481 aprint_debug("%s: seg %"PRIxPHYSSEG": too small for %zu pages\n", 482 __func__, bank, npgs); 483 continue; 484 } 485 486 if (!pmap_md_ok_to_steal_p(bank, npgs)) { 487 continue; 488 } 489 490 /* 491 * Always try to allocate from the segment with the least 492 * amount of space left. 493 */ 494 #define VM_PHYSMEM_SPACE(b) ((uvm_physseg_get_avail_end(b)) - (uvm_physseg_get_avail_start(b))) 495 if (uvm_physseg_valid_p(maybe_bank) == false 496 || VM_PHYSMEM_SPACE(bank) < VM_PHYSMEM_SPACE(maybe_bank)) { 497 maybe_bank = bank; 498 } 499 } 500 501 if (uvm_physseg_valid_p(maybe_bank)) { 502 const uvm_physseg_t bank = maybe_bank; 503 504 /* 505 * There are enough pages here; steal them! 506 */ 507 pa = ptoa(uvm_physseg_get_start(bank)); 508 uvm_physseg_unplug(atop(pa), npgs); 509 510 aprint_debug("%s: seg %"PRIxPHYSSEG": %zu pages stolen (%#"PRIxPADDR" left)\n", 511 __func__, bank, npgs, VM_PHYSMEM_SPACE(bank)); 512 513 va = pmap_md_map_poolpage(pa, size); 514 memset((void *)va, 0, size); 515 return va; 516 } 517 518 /* 519 * If we got here, there was no memory left. 520 */ 521 panic("pmap_steal_memory: no memory to steal %zu pages", npgs); 522 } 523 524 /* 525 * Initialize the pmap module. 526 * Called by vm_init, to initialize any structures that the pmap 527 * system needs to map virtual memory. 528 */ 529 void 530 pmap_init(void) 531 { 532 UVMHIST_INIT_STATIC(pmapexechist, pmapexechistbuf); 533 UVMHIST_INIT_STATIC(pmaphist, pmaphistbuf); 534 535 UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist); 536 537 /* 538 * Initialize the segtab lock. 539 */ 540 mutex_init(&pmap_segtab_lock, MUTEX_DEFAULT, IPL_HIGH); 541 542 /* 543 * Set a low water mark on the pv_entry pool, so that we are 544 * more likely to have these around even in extreme memory 545 * starvation. 546 */ 547 pool_setlowat(&pmap_pv_pool, pmap_pv_lowat); 548 549 /* 550 * Set the page colormask but allow pmap_md_init to override it. 551 */ 552 pmap_page_colormask = ptoa(uvmexp.colormask); 553 554 pmap_md_init(); 555 556 /* 557 * Now it is safe to enable pv entry recording. 558 */ 559 pmap_initialized = true; 560 } 561 562 /* 563 * Create and return a physical map. 564 * 565 * If the size specified for the map 566 * is zero, the map is an actual physical 567 * map, and may be referenced by the 568 * hardware. 569 * 570 * If the size specified is non-zero, 571 * the map will be used in software only, and 572 * is bounded by that size. 573 */ 574 pmap_t 575 pmap_create(void) 576 { 577 UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist); 578 PMAP_COUNT(create); 579 580 pmap_t pmap = pool_get(&pmap_pmap_pool, PR_WAITOK); 581 memset(pmap, 0, PMAP_SIZE); 582 583 KASSERT(pmap->pm_pai[0].pai_link.le_prev == NULL); 584 585 pmap->pm_count = 1; 586 pmap->pm_minaddr = VM_MIN_ADDRESS; 587 pmap->pm_maxaddr = VM_MAXUSER_ADDRESS; 588 589 pmap_segtab_init(pmap); 590 591 #ifdef MULTIPROCESSOR 592 kcpuset_create(&pmap->pm_active, true); 593 kcpuset_create(&pmap->pm_onproc, true); 594 KASSERT(pmap->pm_active != NULL); 595 KASSERT(pmap->pm_onproc != NULL); 596 #endif 597 598 UVMHIST_LOG(pmaphist, " <-- done (pmap=%#jx)", (uintptr_t)pmap, 599 0, 0, 0); 600 601 return pmap; 602 } 603 604 /* 605 * Retire the given physical map from service. 606 * Should only be called if the map contains 607 * no valid mappings. 608 */ 609 void 610 pmap_destroy(pmap_t pmap) 611 { 612 UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist); 613 UVMHIST_LOG(pmaphist, "(pmap=%#jx)", (uintptr_t)pmap, 0, 0, 0); 614 615 if (atomic_dec_uint_nv(&pmap->pm_count) > 0) { 616 PMAP_COUNT(dereference); 617 UVMHIST_LOG(pmaphist, " <-- done (deref)", 0, 0, 0, 0); 618 return; 619 } 620 621 PMAP_COUNT(destroy); 622 KASSERT(pmap->pm_count == 0); 623 kpreempt_disable(); 624 pmap_md_tlb_miss_lock_enter(); 625 pmap_tlb_asid_release_all(pmap); 626 pmap_segtab_destroy(pmap, NULL, 0); 627 pmap_md_tlb_miss_lock_exit(); 628 629 #ifdef MULTIPROCESSOR 630 kcpuset_destroy(pmap->pm_active); 631 kcpuset_destroy(pmap->pm_onproc); 632 pmap->pm_active = NULL; 633 pmap->pm_onproc = NULL; 634 #endif 635 636 pool_put(&pmap_pmap_pool, pmap); 637 kpreempt_enable(); 638 639 UVMHIST_LOG(pmaphist, " <-- done (freed)", 0, 0, 0, 0); 640 } 641 642 /* 643 * Add a reference to the specified pmap. 644 */ 645 void 646 pmap_reference(pmap_t pmap) 647 { 648 UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist); 649 UVMHIST_LOG(pmaphist, "(pmap=%#jx)", (uintptr_t)pmap, 0, 0, 0); 650 PMAP_COUNT(reference); 651 652 if (pmap != NULL) { 653 atomic_inc_uint(&pmap->pm_count); 654 } 655 656 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 657 } 658 659 /* 660 * Make a new pmap (vmspace) active for the given process. 661 */ 662 void 663 pmap_activate(struct lwp *l) 664 { 665 pmap_t pmap = l->l_proc->p_vmspace->vm_map.pmap; 666 667 UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist); 668 UVMHIST_LOG(pmaphist, "(l=%#jx pmap=%#jx)", (uintptr_t)l, 669 (uintptr_t)pmap, 0, 0); 670 PMAP_COUNT(activate); 671 672 kpreempt_disable(); 673 pmap_md_tlb_miss_lock_enter(); 674 pmap_tlb_asid_acquire(pmap, l); 675 if (l == curlwp) { 676 pmap_segtab_activate(pmap, l); 677 } 678 pmap_md_tlb_miss_lock_exit(); 679 kpreempt_enable(); 680 681 UVMHIST_LOG(pmaphist, " <-- done (%ju:%ju)", l->l_proc->p_pid, 682 l->l_lid, 0, 0); 683 } 684 685 /* 686 * Remove this page from all physical maps in which it resides. 687 * Reflects back modify bits to the pager. 688 */ 689 void 690 pmap_page_remove(struct vm_page *pg) 691 { 692 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 693 694 kpreempt_disable(); 695 VM_PAGEMD_PVLIST_LOCK(mdpg); 696 pmap_pvlist_check(mdpg); 697 698 UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist); 699 700 UVMHIST_LOG(pmapexechist, "pg %#jx (pa %#jx) [page removed]: " 701 "execpage cleared", (uintptr_t)pg, VM_PAGE_TO_PHYS(pg), 0, 0); 702 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 703 pmap_page_clear_attributes(mdpg, VM_PAGEMD_EXECPAGE|VM_PAGEMD_UNCACHED); 704 #else 705 pmap_page_clear_attributes(mdpg, VM_PAGEMD_EXECPAGE); 706 #endif 707 PMAP_COUNT(exec_uncached_remove); 708 709 pv_entry_t pv = &mdpg->mdpg_first; 710 if (pv->pv_pmap == NULL) { 711 VM_PAGEMD_PVLIST_UNLOCK(mdpg); 712 kpreempt_enable(); 713 UVMHIST_LOG(pmaphist, " <-- done (empty)", 0, 0, 0, 0); 714 return; 715 } 716 717 pv_entry_t npv; 718 pv_entry_t pvp = NULL; 719 720 for (; pv != NULL; pv = npv) { 721 npv = pv->pv_next; 722 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 723 if (PV_ISKENTER_P(pv)) { 724 UVMHIST_LOG(pmaphist, " pv %#jx pmap %#jx va %jx" 725 " skip", (uintptr_t)pv, (uintptr_t)pv->pv_pmap, 726 pv->pv_va, 0); 727 728 KASSERT(pv->pv_pmap == pmap_kernel()); 729 730 /* Assume no more - it'll get fixed if there are */ 731 pv->pv_next = NULL; 732 733 /* 734 * pvp is non-null when we already have a PV_KENTER 735 * pv in pvh_first; otherwise we haven't seen a 736 * PV_KENTER pv and we need to copy this one to 737 * pvh_first 738 */ 739 if (pvp) { 740 /* 741 * The previous PV_KENTER pv needs to point to 742 * this PV_KENTER pv 743 */ 744 pvp->pv_next = pv; 745 } else { 746 pv_entry_t fpv = &mdpg->mdpg_first; 747 *fpv = *pv; 748 KASSERT(fpv->pv_pmap == pmap_kernel()); 749 } 750 pvp = pv; 751 continue; 752 } 753 #endif 754 const pmap_t pmap = pv->pv_pmap; 755 vaddr_t va = trunc_page(pv->pv_va); 756 pt_entry_t * const ptep = pmap_pte_lookup(pmap, va); 757 KASSERTMSG(ptep != NULL, "%#"PRIxVADDR " %#"PRIxVADDR, va, 758 pmap_limits.virtual_end); 759 pt_entry_t pte = *ptep; 760 UVMHIST_LOG(pmaphist, " pv %#jx pmap %#jx va %jx" 761 " pte %jx", (uintptr_t)pv, (uintptr_t)pmap, va, 762 pte_value(pte)); 763 if (!pte_valid_p(pte)) 764 continue; 765 const bool is_kernel_pmap_p = (pmap == pmap_kernel()); 766 if (is_kernel_pmap_p) { 767 PMAP_COUNT(remove_kernel_pages); 768 } else { 769 PMAP_COUNT(remove_user_pages); 770 } 771 if (pte_wired_p(pte)) 772 pmap->pm_stats.wired_count--; 773 pmap->pm_stats.resident_count--; 774 775 pmap_md_tlb_miss_lock_enter(); 776 const pt_entry_t npte = pte_nv_entry(is_kernel_pmap_p); 777 pte_set(ptep, npte); 778 if (__predict_true(!(pmap->pm_flags & PMAP_DEFERRED_ACTIVATE))) { 779 /* 780 * Flush the TLB for the given address. 781 */ 782 pmap_tlb_invalidate_addr(pmap, va); 783 } 784 pmap_md_tlb_miss_lock_exit(); 785 786 /* 787 * non-null means this is a non-pvh_first pv, so we should 788 * free it. 789 */ 790 if (pvp) { 791 KASSERT(pvp->pv_pmap == pmap_kernel()); 792 KASSERT(pvp->pv_next == NULL); 793 pmap_pv_free(pv); 794 } else { 795 pv->pv_pmap = NULL; 796 pv->pv_next = NULL; 797 } 798 } 799 800 pmap_pvlist_check(mdpg); 801 VM_PAGEMD_PVLIST_UNLOCK(mdpg); 802 kpreempt_enable(); 803 804 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 805 } 806 807 808 /* 809 * Make a previously active pmap (vmspace) inactive. 810 */ 811 void 812 pmap_deactivate(struct lwp *l) 813 { 814 pmap_t pmap = l->l_proc->p_vmspace->vm_map.pmap; 815 816 UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist); 817 UVMHIST_LOG(pmaphist, "(l=%#jx pmap=%#jx)", (uintptr_t)l, 818 (uintptr_t)pmap, 0, 0); 819 PMAP_COUNT(deactivate); 820 821 kpreempt_disable(); 822 KASSERT(l == curlwp || l->l_cpu == curlwp->l_cpu); 823 pmap_md_tlb_miss_lock_enter(); 824 curcpu()->ci_pmap_user_segtab = PMAP_INVALID_SEGTAB_ADDRESS; 825 #ifdef _LP64 826 curcpu()->ci_pmap_user_seg0tab = NULL; 827 #endif 828 pmap_tlb_asid_deactivate(pmap); 829 pmap_md_tlb_miss_lock_exit(); 830 kpreempt_enable(); 831 832 UVMHIST_LOG(pmaphist, " <-- done (%ju:%ju)", l->l_proc->p_pid, 833 l->l_lid, 0, 0); 834 } 835 836 void 837 pmap_update(struct pmap *pmap) 838 { 839 UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist); 840 UVMHIST_LOG(pmaphist, "(pmap=%#jx)", (uintptr_t)pmap, 0, 0, 0); 841 PMAP_COUNT(update); 842 843 kpreempt_disable(); 844 #if defined(MULTIPROCESSOR) && defined(PMAP_TLB_NEED_SHOOTDOWN) 845 u_int pending = atomic_swap_uint(&pmap->pm_shootdown_pending, 0); 846 if (pending && pmap_tlb_shootdown_bystanders(pmap)) 847 PMAP_COUNT(shootdown_ipis); 848 #endif 849 pmap_md_tlb_miss_lock_enter(); 850 #if defined(DEBUG) && !defined(MULTIPROCESSOR) 851 pmap_tlb_check(pmap, pmap_md_tlb_check_entry); 852 #endif /* DEBUG */ 853 854 /* 855 * If pmap_remove_all was called, we deactivated ourselves and nuked 856 * our ASID. Now we have to reactivate ourselves. 857 */ 858 if (__predict_false(pmap->pm_flags & PMAP_DEFERRED_ACTIVATE)) { 859 pmap->pm_flags ^= PMAP_DEFERRED_ACTIVATE; 860 pmap_tlb_asid_acquire(pmap, curlwp); 861 pmap_segtab_activate(pmap, curlwp); 862 } 863 pmap_md_tlb_miss_lock_exit(); 864 kpreempt_enable(); 865 866 UVMHIST_LOG(pmaphist, " <-- done (kernel=%#jx)", 867 (pmap == pmap_kernel() ? 1 : 0), 0, 0, 0); 868 } 869 870 /* 871 * Remove the given range of addresses from the specified map. 872 * 873 * It is assumed that the start and end are properly 874 * rounded to the page size. 875 */ 876 877 static bool 878 pmap_pte_remove(pmap_t pmap, vaddr_t sva, vaddr_t eva, pt_entry_t *ptep, 879 uintptr_t flags) 880 { 881 const pt_entry_t npte = flags; 882 const bool is_kernel_pmap_p = (pmap == pmap_kernel()); 883 884 UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist); 885 UVMHIST_LOG(pmaphist, "(pmap=%#jx kernel=%c va=%#jx..%#jx)", 886 (uintptr_t)pmap, (is_kernel_pmap_p ? 1 : 0), sva, eva); 887 UVMHIST_LOG(pmaphist, "ptep=%#jx, flags(npte)=%#jx", 888 (uintptr_t)ptep, flags, 0, 0); 889 890 KASSERT(kpreempt_disabled()); 891 892 for (; sva < eva; sva += NBPG, ptep++) { 893 const pt_entry_t pte = *ptep; 894 if (!pte_valid_p(pte)) 895 continue; 896 if (is_kernel_pmap_p) { 897 PMAP_COUNT(remove_kernel_pages); 898 } else { 899 PMAP_COUNT(remove_user_pages); 900 } 901 if (pte_wired_p(pte)) 902 pmap->pm_stats.wired_count--; 903 pmap->pm_stats.resident_count--; 904 struct vm_page * const pg = PHYS_TO_VM_PAGE(pte_to_paddr(pte)); 905 if (__predict_true(pg != NULL)) { 906 pmap_remove_pv(pmap, sva, pg, pte_modified_p(pte)); 907 } 908 pmap_md_tlb_miss_lock_enter(); 909 pte_set(ptep, npte); 910 if (__predict_true(!(pmap->pm_flags & PMAP_DEFERRED_ACTIVATE))) { 911 912 /* 913 * Flush the TLB for the given address. 914 */ 915 pmap_tlb_invalidate_addr(pmap, sva); 916 } 917 pmap_md_tlb_miss_lock_exit(); 918 } 919 920 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 921 922 return false; 923 } 924 925 void 926 pmap_remove(pmap_t pmap, vaddr_t sva, vaddr_t eva) 927 { 928 const bool is_kernel_pmap_p = (pmap == pmap_kernel()); 929 const pt_entry_t npte = pte_nv_entry(is_kernel_pmap_p); 930 931 UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist); 932 UVMHIST_LOG(pmaphist, "(pmap=%#jx, va=%#jx..%#jx)", 933 (uintptr_t)pmap, sva, eva, 0); 934 935 if (is_kernel_pmap_p) { 936 PMAP_COUNT(remove_kernel_calls); 937 } else { 938 PMAP_COUNT(remove_user_calls); 939 } 940 #ifdef PMAP_FAULTINFO 941 curpcb->pcb_faultinfo.pfi_faultaddr = 0; 942 curpcb->pcb_faultinfo.pfi_repeats = 0; 943 curpcb->pcb_faultinfo.pfi_faultpte = NULL; 944 #endif 945 kpreempt_disable(); 946 pmap_addr_range_check(pmap, sva, eva, __func__); 947 pmap_pte_process(pmap, sva, eva, pmap_pte_remove, npte); 948 kpreempt_enable(); 949 950 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 951 } 952 953 /* 954 * pmap_page_protect: 955 * 956 * Lower the permission for all mappings to a given page. 957 */ 958 void 959 pmap_page_protect(struct vm_page *pg, vm_prot_t prot) 960 { 961 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 962 pv_entry_t pv; 963 vaddr_t va; 964 965 UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist); 966 UVMHIST_LOG(pmaphist, "(pg=%#jx (pa %#jx) prot=%#jx)", 967 (uintptr_t)pg, VM_PAGE_TO_PHYS(pg), prot, 0); 968 PMAP_COUNT(page_protect); 969 970 switch (prot) { 971 case VM_PROT_READ|VM_PROT_WRITE: 972 case VM_PROT_ALL: 973 break; 974 975 /* copy_on_write */ 976 case VM_PROT_READ: 977 case VM_PROT_READ|VM_PROT_EXECUTE: 978 pv = &mdpg->mdpg_first; 979 kpreempt_disable(); 980 VM_PAGEMD_PVLIST_READLOCK(mdpg); 981 pmap_pvlist_check(mdpg); 982 /* 983 * Loop over all current mappings setting/clearing as 984 * appropriate. 985 */ 986 if (pv->pv_pmap != NULL) { 987 while (pv != NULL) { 988 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 989 if (PV_ISKENTER_P(pv)) { 990 pv = pv->pv_next; 991 continue; 992 } 993 #endif 994 const pmap_t pmap = pv->pv_pmap; 995 va = trunc_page(pv->pv_va); 996 const uintptr_t gen = 997 VM_PAGEMD_PVLIST_UNLOCK(mdpg); 998 pmap_protect(pmap, va, va + PAGE_SIZE, prot); 999 KASSERT(pv->pv_pmap == pmap); 1000 pmap_update(pmap); 1001 if (gen != VM_PAGEMD_PVLIST_READLOCK(mdpg)) { 1002 pv = &mdpg->mdpg_first; 1003 } else { 1004 pv = pv->pv_next; 1005 } 1006 pmap_pvlist_check(mdpg); 1007 } 1008 } 1009 pmap_pvlist_check(mdpg); 1010 VM_PAGEMD_PVLIST_UNLOCK(mdpg); 1011 kpreempt_enable(); 1012 break; 1013 1014 /* remove_all */ 1015 default: 1016 pmap_page_remove(pg); 1017 } 1018 1019 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 1020 } 1021 1022 static bool 1023 pmap_pte_protect(pmap_t pmap, vaddr_t sva, vaddr_t eva, pt_entry_t *ptep, 1024 uintptr_t flags) 1025 { 1026 const vm_prot_t prot = (flags & VM_PROT_ALL); 1027 1028 UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist); 1029 UVMHIST_LOG(pmaphist, "(pmap=%#jx kernel=%jx va=%#jx..%#jx)", 1030 (uintptr_t)pmap, (pmap == pmap_kernel() ? 1 : 0), sva, eva); 1031 UVMHIST_LOG(pmaphist, "ptep=%#jx, flags(npte)=%#jx)", 1032 (uintptr_t)ptep, flags, 0, 0); 1033 1034 KASSERT(kpreempt_disabled()); 1035 /* 1036 * Change protection on every valid mapping within this segment. 1037 */ 1038 for (; sva < eva; sva += NBPG, ptep++) { 1039 pt_entry_t pte = *ptep; 1040 if (!pte_valid_p(pte)) 1041 continue; 1042 struct vm_page * const pg = PHYS_TO_VM_PAGE(pte_to_paddr(pte)); 1043 if (pg != NULL && pte_modified_p(pte)) { 1044 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 1045 if (VM_PAGEMD_EXECPAGE_P(mdpg)) { 1046 KASSERT(!VM_PAGEMD_PVLIST_EMPTY_P(mdpg)); 1047 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 1048 if (VM_PAGEMD_CACHED_P(mdpg)) { 1049 #endif 1050 UVMHIST_LOG(pmapexechist, 1051 "pg %#jx (pa %#jx): " 1052 "syncicached performed", 1053 (uintptr_t)pg, VM_PAGE_TO_PHYS(pg), 1054 0, 0); 1055 pmap_page_syncicache(pg); 1056 PMAP_COUNT(exec_synced_protect); 1057 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 1058 } 1059 #endif 1060 } 1061 } 1062 pte = pte_prot_downgrade(pte, prot); 1063 if (*ptep != pte) { 1064 pmap_md_tlb_miss_lock_enter(); 1065 pte_set(ptep, pte); 1066 /* 1067 * Update the TLB if needed. 1068 */ 1069 pmap_tlb_update_addr(pmap, sva, pte, PMAP_TLB_NEED_IPI); 1070 pmap_md_tlb_miss_lock_exit(); 1071 } 1072 } 1073 1074 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 1075 1076 return false; 1077 } 1078 1079 /* 1080 * Set the physical protection on the 1081 * specified range of this map as requested. 1082 */ 1083 void 1084 pmap_protect(pmap_t pmap, vaddr_t sva, vaddr_t eva, vm_prot_t prot) 1085 { 1086 UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist); 1087 UVMHIST_LOG(pmaphist, "(pmap=%#jx, va=%#jx..%#jx, prot=%ju)", 1088 (uintptr_t)pmap, sva, eva, prot); 1089 PMAP_COUNT(protect); 1090 1091 if ((prot & VM_PROT_READ) == VM_PROT_NONE) { 1092 pmap_remove(pmap, sva, eva); 1093 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 1094 return; 1095 } 1096 1097 /* 1098 * Change protection on every valid mapping within this segment. 1099 */ 1100 kpreempt_disable(); 1101 pmap_addr_range_check(pmap, sva, eva, __func__); 1102 pmap_pte_process(pmap, sva, eva, pmap_pte_protect, prot); 1103 kpreempt_enable(); 1104 1105 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 1106 } 1107 1108 #if defined(PMAP_VIRTUAL_CACHE_ALIASES) && !defined(PMAP_NO_PV_UNCACHED) 1109 /* 1110 * pmap_page_cache: 1111 * 1112 * Change all mappings of a managed page to cached/uncached. 1113 */ 1114 void 1115 pmap_page_cache(struct vm_page *pg, bool cached) 1116 { 1117 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 1118 1119 UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist); 1120 UVMHIST_LOG(pmaphist, "(pg=%#jx (pa %#jx) cached=%jd)", 1121 (uintptr_t)pg, VM_PAGE_TO_PHYS(pg), cached, 0); 1122 1123 KASSERT(kpreempt_disabled()); 1124 KASSERT(VM_PAGEMD_PVLIST_LOCKED_P(mdpg)); 1125 1126 if (cached) { 1127 pmap_page_clear_attributes(mdpg, VM_PAGEMD_UNCACHED); 1128 PMAP_COUNT(page_cache_restorations); 1129 } else { 1130 pmap_page_set_attributes(mdpg, VM_PAGEMD_UNCACHED); 1131 PMAP_COUNT(page_cache_evictions); 1132 } 1133 1134 for (pv_entry_t pv = &mdpg->mdpg_first; pv != NULL; pv = pv->pv_next) { 1135 pmap_t pmap = pv->pv_pmap; 1136 vaddr_t va = trunc_page(pv->pv_va); 1137 1138 KASSERT(pmap != NULL); 1139 KASSERT(pmap != pmap_kernel() || !pmap_md_direct_mapped_vaddr_p(va)); 1140 pt_entry_t * const ptep = pmap_pte_lookup(pmap, va); 1141 if (ptep == NULL) 1142 continue; 1143 pt_entry_t pte = *ptep; 1144 if (pte_valid_p(pte)) { 1145 pte = pte_cached_change(pte, cached); 1146 pmap_md_tlb_miss_lock_enter(); 1147 pte_set(ptep, pte); 1148 pmap_tlb_update_addr(pmap, va, pte, PMAP_TLB_NEED_IPI); 1149 pmap_md_tlb_miss_lock_exit(); 1150 } 1151 } 1152 1153 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 1154 } 1155 #endif /* PMAP_VIRTUAL_CACHE_ALIASES && !PMAP_NO_PV_UNCACHED */ 1156 1157 /* 1158 * Insert the given physical page (p) at 1159 * the specified virtual address (v) in the 1160 * target physical map with the protection requested. 1161 * 1162 * If specified, the page will be wired down, meaning 1163 * that the related pte can not be reclaimed. 1164 * 1165 * NB: This is the only routine which MAY NOT lazy-evaluate 1166 * or lose information. That is, this routine must actually 1167 * insert this page into the given map NOW. 1168 */ 1169 int 1170 pmap_enter(pmap_t pmap, vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags) 1171 { 1172 const bool wired = (flags & PMAP_WIRED) != 0; 1173 const bool is_kernel_pmap_p = (pmap == pmap_kernel()); 1174 u_int update_flags = (flags & VM_PROT_ALL) != 0 ? PMAP_TLB_INSERT : 0; 1175 #ifdef UVMHIST 1176 struct kern_history * const histp = 1177 ((prot & VM_PROT_EXECUTE) ? &pmapexechist : &pmaphist); 1178 #endif 1179 1180 UVMHIST_FUNC(__func__); UVMHIST_CALLED(*histp); 1181 UVMHIST_LOG(*histp, "(pmap=%#jx, va=%#jx, pa=%#jx", 1182 (uintptr_t)pmap, va, pa, 0); 1183 UVMHIST_LOG(*histp, "prot=%#jx flags=%#jx)", prot, flags, 0, 0); 1184 1185 const bool good_color = PMAP_PAGE_COLOROK_P(pa, va); 1186 if (is_kernel_pmap_p) { 1187 PMAP_COUNT(kernel_mappings); 1188 if (!good_color) 1189 PMAP_COUNT(kernel_mappings_bad); 1190 } else { 1191 PMAP_COUNT(user_mappings); 1192 if (!good_color) 1193 PMAP_COUNT(user_mappings_bad); 1194 } 1195 pmap_addr_range_check(pmap, va, va, __func__); 1196 1197 KASSERTMSG(prot & VM_PROT_READ, "no READ (%#x) in prot %#x", 1198 VM_PROT_READ, prot); 1199 1200 struct vm_page * const pg = PHYS_TO_VM_PAGE(pa); 1201 struct vm_page_md * const mdpg = (pg ? VM_PAGE_TO_MD(pg) : NULL); 1202 1203 if (pg) { 1204 /* Set page referenced/modified status based on flags */ 1205 if (flags & VM_PROT_WRITE) { 1206 pmap_page_set_attributes(mdpg, VM_PAGEMD_MODIFIED|VM_PAGEMD_REFERENCED); 1207 } else if (flags & VM_PROT_ALL) { 1208 pmap_page_set_attributes(mdpg, VM_PAGEMD_REFERENCED); 1209 } 1210 1211 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 1212 if (!VM_PAGEMD_CACHED_P(mdpg)) { 1213 flags |= PMAP_NOCACHE; 1214 PMAP_COUNT(uncached_mappings); 1215 } 1216 #endif 1217 1218 PMAP_COUNT(managed_mappings); 1219 } else { 1220 /* 1221 * Assumption: if it is not part of our managed memory 1222 * then it must be device memory which may be volatile. 1223 */ 1224 if ((flags & PMAP_CACHE_MASK) == 0) 1225 flags |= PMAP_NOCACHE; 1226 PMAP_COUNT(unmanaged_mappings); 1227 } 1228 1229 pt_entry_t npte = pte_make_enter(pa, mdpg, prot, flags, 1230 is_kernel_pmap_p); 1231 1232 kpreempt_disable(); 1233 1234 pt_entry_t * const ptep = pmap_pte_reserve(pmap, va, flags); 1235 if (__predict_false(ptep == NULL)) { 1236 kpreempt_enable(); 1237 UVMHIST_LOG(*histp, " <-- ENOMEM", 0, 0, 0, 0); 1238 return ENOMEM; 1239 } 1240 const pt_entry_t opte = *ptep; 1241 const bool resident = pte_valid_p(opte); 1242 bool remap = false; 1243 if (resident) { 1244 if (pte_to_paddr(opte) != pa) { 1245 KASSERT(!is_kernel_pmap_p); 1246 const pt_entry_t rpte = pte_nv_entry(false); 1247 1248 pmap_addr_range_check(pmap, va, va + NBPG, __func__); 1249 pmap_pte_process(pmap, va, va + NBPG, pmap_pte_remove, 1250 rpte); 1251 PMAP_COUNT(user_mappings_changed); 1252 remap = true; 1253 } 1254 update_flags |= PMAP_TLB_NEED_IPI; 1255 } 1256 1257 if (!resident || remap) { 1258 pmap->pm_stats.resident_count++; 1259 } 1260 1261 /* Done after case that may sleep/return. */ 1262 if (pg) 1263 pmap_enter_pv(pmap, va, pg, &npte, 0); 1264 1265 /* 1266 * Now validate mapping with desired protection/wiring. 1267 * Assume uniform modified and referenced status for all 1268 * MIPS pages in a MACH page. 1269 */ 1270 if (wired) { 1271 pmap->pm_stats.wired_count++; 1272 npte = pte_wire_entry(npte); 1273 } 1274 1275 UVMHIST_LOG(*histp, "new pte %#jx (pa %#jx)", 1276 pte_value(npte), pa, 0, 0); 1277 1278 KASSERT(pte_valid_p(npte)); 1279 1280 pmap_md_tlb_miss_lock_enter(); 1281 pte_set(ptep, npte); 1282 pmap_tlb_update_addr(pmap, va, npte, update_flags); 1283 pmap_md_tlb_miss_lock_exit(); 1284 kpreempt_enable(); 1285 1286 if (pg != NULL && (prot == (VM_PROT_READ | VM_PROT_EXECUTE))) { 1287 KASSERT(mdpg != NULL); 1288 PMAP_COUNT(exec_mappings); 1289 if (!VM_PAGEMD_EXECPAGE_P(mdpg) && pte_cached_p(npte)) { 1290 if (!pte_deferred_exec_p(npte)) { 1291 UVMHIST_LOG(*histp, "va=%#jx pg %#jx: " 1292 "immediate syncicache", 1293 va, (uintptr_t)pg, 0, 0); 1294 pmap_page_syncicache(pg); 1295 pmap_page_set_attributes(mdpg, 1296 VM_PAGEMD_EXECPAGE); 1297 PMAP_COUNT(exec_synced_mappings); 1298 } else { 1299 UVMHIST_LOG(*histp, "va=%#jx pg %#jx: defer " 1300 "syncicache: pte %#jx", 1301 va, (uintptr_t)pg, npte, 0); 1302 } 1303 } else { 1304 UVMHIST_LOG(*histp, 1305 "va=%#jx pg %#jx: no syncicache cached %jd", 1306 va, (uintptr_t)pg, pte_cached_p(npte), 0); 1307 } 1308 } else if (pg != NULL && (prot & VM_PROT_EXECUTE)) { 1309 KASSERT(mdpg != NULL); 1310 KASSERT(prot & VM_PROT_WRITE); 1311 PMAP_COUNT(exec_mappings); 1312 pmap_page_syncicache(pg); 1313 pmap_page_clear_attributes(mdpg, VM_PAGEMD_EXECPAGE); 1314 UVMHIST_LOG(*histp, 1315 "va=%#jx pg %#jx: immediate syncicache (writeable)", 1316 va, (uintptr_t)pg, 0, 0); 1317 } 1318 1319 UVMHIST_LOG(*histp, " <-- 0 (OK)", 0, 0, 0, 0); 1320 return 0; 1321 } 1322 1323 void 1324 pmap_kenter_pa(vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags) 1325 { 1326 pmap_t pmap = pmap_kernel(); 1327 struct vm_page * const pg = PHYS_TO_VM_PAGE(pa); 1328 struct vm_page_md * const mdpg = (pg ? VM_PAGE_TO_MD(pg) : NULL); 1329 1330 UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist); 1331 UVMHIST_LOG(pmaphist, "(va=%#jx pa=%#jx prot=%ju, flags=%#jx)", 1332 va, pa, prot, flags); 1333 PMAP_COUNT(kenter_pa); 1334 1335 if (mdpg == NULL) { 1336 PMAP_COUNT(kenter_pa_unmanaged); 1337 if ((flags & PMAP_CACHE_MASK) == 0) 1338 flags |= PMAP_NOCACHE; 1339 } else { 1340 if ((flags & PMAP_NOCACHE) == 0 && !PMAP_PAGE_COLOROK_P(pa, va)) 1341 PMAP_COUNT(kenter_pa_bad); 1342 } 1343 1344 pt_entry_t npte = pte_make_kenter_pa(pa, mdpg, prot, flags); 1345 kpreempt_disable(); 1346 pt_entry_t * const ptep = pmap_pte_lookup(pmap, va); 1347 KASSERTMSG(ptep != NULL, "%#"PRIxVADDR " %#"PRIxVADDR, va, 1348 pmap_limits.virtual_end); 1349 KASSERT(!pte_valid_p(*ptep)); 1350 1351 /* 1352 * No need to track non-managed pages or PMAP_KMPAGEs pages for aliases 1353 */ 1354 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 1355 if (pg != NULL && (flags & PMAP_KMPAGE) == 0 1356 && pmap_md_virtual_cache_aliasing_p()) { 1357 pmap_enter_pv(pmap, va, pg, &npte, PV_KENTER); 1358 } 1359 #endif 1360 1361 /* 1362 * We have the option to force this mapping into the TLB but we 1363 * don't. Instead let the next reference to the page do it. 1364 */ 1365 pmap_md_tlb_miss_lock_enter(); 1366 pte_set(ptep, npte); 1367 pmap_tlb_update_addr(pmap_kernel(), va, npte, 0); 1368 pmap_md_tlb_miss_lock_exit(); 1369 kpreempt_enable(); 1370 #if DEBUG > 1 1371 for (u_int i = 0; i < PAGE_SIZE / sizeof(long); i++) { 1372 if (((long *)va)[i] != ((long *)pa)[i]) 1373 panic("%s: contents (%lx) of va %#"PRIxVADDR 1374 " != contents (%lx) of pa %#"PRIxPADDR, __func__, 1375 ((long *)va)[i], va, ((long *)pa)[i], pa); 1376 } 1377 #endif 1378 1379 UVMHIST_LOG(pmaphist, " <-- done (ptep=%#jx)", (uintptr_t)ptep, 0, 0, 1380 0); 1381 } 1382 1383 /* 1384 * Remove the given range of addresses from the kernel map. 1385 * 1386 * It is assumed that the start and end are properly 1387 * rounded to the page size. 1388 */ 1389 1390 static bool 1391 pmap_pte_kremove(pmap_t pmap, vaddr_t sva, vaddr_t eva, pt_entry_t *ptep, 1392 uintptr_t flags) 1393 { 1394 const pt_entry_t new_pte = pte_nv_entry(true); 1395 1396 UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist); 1397 UVMHIST_LOG(pmaphist, 1398 "(pmap=%#jx, sva=%#jx eva=%#jx ptep=%#jx)", 1399 (uintptr_t)pmap, sva, eva, (uintptr_t)ptep); 1400 1401 KASSERT(kpreempt_disabled()); 1402 1403 for (; sva < eva; sva += NBPG, ptep++) { 1404 pt_entry_t pte = *ptep; 1405 if (!pte_valid_p(pte)) 1406 continue; 1407 1408 PMAP_COUNT(kremove_pages); 1409 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 1410 struct vm_page * const pg = PHYS_TO_VM_PAGE(pte_to_paddr(pte)); 1411 if (pg != NULL && pmap_md_virtual_cache_aliasing_p()) { 1412 pmap_remove_pv(pmap, sva, pg, !pte_readonly_p(pte)); 1413 } 1414 #endif 1415 1416 pmap_md_tlb_miss_lock_enter(); 1417 pte_set(ptep, new_pte); 1418 pmap_tlb_invalidate_addr(pmap, sva); 1419 pmap_md_tlb_miss_lock_exit(); 1420 } 1421 1422 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 1423 1424 return false; 1425 } 1426 1427 void 1428 pmap_kremove(vaddr_t va, vsize_t len) 1429 { 1430 const vaddr_t sva = trunc_page(va); 1431 const vaddr_t eva = round_page(va + len); 1432 1433 UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist); 1434 UVMHIST_LOG(pmaphist, "(va=%#jx len=%#jx)", va, len, 0, 0); 1435 1436 kpreempt_disable(); 1437 pmap_pte_process(pmap_kernel(), sva, eva, pmap_pte_kremove, 0); 1438 kpreempt_enable(); 1439 1440 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 1441 } 1442 1443 void 1444 pmap_remove_all(struct pmap *pmap) 1445 { 1446 UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist); 1447 UVMHIST_LOG(pmaphist, "(pm=%#jx)", (uintptr_t)pmap, 0, 0, 0); 1448 1449 KASSERT(pmap != pmap_kernel()); 1450 1451 kpreempt_disable(); 1452 /* 1453 * Free all of our ASIDs which means we can skip doing all the 1454 * tlb_invalidate_addrs(). 1455 */ 1456 pmap_md_tlb_miss_lock_enter(); 1457 #ifdef MULTIPROCESSOR 1458 // This should be the last CPU with this pmap onproc 1459 KASSERT(!kcpuset_isotherset(pmap->pm_onproc, cpu_index(curcpu()))); 1460 if (kcpuset_isset(pmap->pm_onproc, cpu_index(curcpu()))) 1461 #endif 1462 pmap_tlb_asid_deactivate(pmap); 1463 #ifdef MULTIPROCESSOR 1464 KASSERT(kcpuset_iszero(pmap->pm_onproc)); 1465 #endif 1466 pmap_tlb_asid_release_all(pmap); 1467 pmap_md_tlb_miss_lock_exit(); 1468 pmap->pm_flags |= PMAP_DEFERRED_ACTIVATE; 1469 1470 #ifdef PMAP_FAULTINFO 1471 curpcb->pcb_faultinfo.pfi_faultaddr = 0; 1472 curpcb->pcb_faultinfo.pfi_repeats = 0; 1473 curpcb->pcb_faultinfo.pfi_faultpte = NULL; 1474 #endif 1475 kpreempt_enable(); 1476 1477 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 1478 } 1479 1480 /* 1481 * Routine: pmap_unwire 1482 * Function: Clear the wired attribute for a map/virtual-address 1483 * pair. 1484 * In/out conditions: 1485 * The mapping must already exist in the pmap. 1486 */ 1487 void 1488 pmap_unwire(pmap_t pmap, vaddr_t va) 1489 { 1490 UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist); 1491 UVMHIST_LOG(pmaphist, "(pmap=%#jx, va=%#jx)", (uintptr_t)pmap, va, 1492 0, 0); 1493 PMAP_COUNT(unwire); 1494 1495 /* 1496 * Don't need to flush the TLB since PG_WIRED is only in software. 1497 */ 1498 kpreempt_disable(); 1499 pmap_addr_range_check(pmap, va, va, __func__); 1500 pt_entry_t * const ptep = pmap_pte_lookup(pmap, va); 1501 KASSERTMSG(ptep != NULL, "pmap %p va %#"PRIxVADDR" invalid STE", 1502 pmap, va); 1503 pt_entry_t pte = *ptep; 1504 KASSERTMSG(pte_valid_p(pte), 1505 "pmap %p va %#"PRIxVADDR" invalid PTE %#"PRIxPTE" @ %p", 1506 pmap, va, pte_value(pte), ptep); 1507 1508 if (pte_wired_p(pte)) { 1509 pmap_md_tlb_miss_lock_enter(); 1510 pte_set(ptep, pte_unwire_entry(pte)); 1511 pmap_md_tlb_miss_lock_exit(); 1512 pmap->pm_stats.wired_count--; 1513 } 1514 #ifdef DIAGNOSTIC 1515 else { 1516 printf("%s: wiring for pmap %p va %#"PRIxVADDR" unchanged!\n", 1517 __func__, pmap, va); 1518 } 1519 #endif 1520 kpreempt_enable(); 1521 1522 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 1523 } 1524 1525 /* 1526 * Routine: pmap_extract 1527 * Function: 1528 * Extract the physical page address associated 1529 * with the given map/virtual_address pair. 1530 */ 1531 bool 1532 pmap_extract(pmap_t pmap, vaddr_t va, paddr_t *pap) 1533 { 1534 paddr_t pa; 1535 1536 if (pmap == pmap_kernel()) { 1537 if (pmap_md_direct_mapped_vaddr_p(va)) { 1538 pa = pmap_md_direct_mapped_vaddr_to_paddr(va); 1539 goto done; 1540 } 1541 if (pmap_md_io_vaddr_p(va)) 1542 panic("pmap_extract: io address %#"PRIxVADDR"", va); 1543 1544 if (va >= pmap_limits.virtual_end) 1545 panic("%s: illegal kernel mapped address %#"PRIxVADDR, 1546 __func__, va); 1547 } 1548 kpreempt_disable(); 1549 const pt_entry_t * const ptep = pmap_pte_lookup(pmap, va); 1550 if (ptep == NULL || !pte_valid_p(*ptep)) { 1551 kpreempt_enable(); 1552 return false; 1553 } 1554 pa = pte_to_paddr(*ptep) | (va & PGOFSET); 1555 kpreempt_enable(); 1556 done: 1557 if (pap != NULL) { 1558 *pap = pa; 1559 } 1560 return true; 1561 } 1562 1563 /* 1564 * Copy the range specified by src_addr/len 1565 * from the source map to the range dst_addr/len 1566 * in the destination map. 1567 * 1568 * This routine is only advisory and need not do anything. 1569 */ 1570 void 1571 pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vaddr_t dst_addr, vsize_t len, 1572 vaddr_t src_addr) 1573 { 1574 UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist); 1575 PMAP_COUNT(copy); 1576 } 1577 1578 /* 1579 * pmap_clear_reference: 1580 * 1581 * Clear the reference bit on the specified physical page. 1582 */ 1583 bool 1584 pmap_clear_reference(struct vm_page *pg) 1585 { 1586 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 1587 1588 UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist); 1589 UVMHIST_LOG(pmaphist, "(pg=%#jx (pa %#jx))", 1590 (uintptr_t)pg, VM_PAGE_TO_PHYS(pg), 0,0); 1591 1592 bool rv = pmap_page_clear_attributes(mdpg, VM_PAGEMD_REFERENCED); 1593 1594 UVMHIST_LOG(pmaphist, " <-- wasref %ju", rv, 0, 0, 0); 1595 1596 return rv; 1597 } 1598 1599 /* 1600 * pmap_is_referenced: 1601 * 1602 * Return whether or not the specified physical page is referenced 1603 * by any physical maps. 1604 */ 1605 bool 1606 pmap_is_referenced(struct vm_page *pg) 1607 { 1608 return VM_PAGEMD_REFERENCED_P(VM_PAGE_TO_MD(pg)); 1609 } 1610 1611 /* 1612 * Clear the modify bits on the specified physical page. 1613 */ 1614 bool 1615 pmap_clear_modify(struct vm_page *pg) 1616 { 1617 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 1618 pv_entry_t pv = &mdpg->mdpg_first; 1619 pv_entry_t pv_next; 1620 1621 UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist); 1622 UVMHIST_LOG(pmaphist, "(pg=%#jx (%#jx))", 1623 (uintptr_t)pg, VM_PAGE_TO_PHYS(pg), 0,0); 1624 PMAP_COUNT(clear_modify); 1625 1626 if (VM_PAGEMD_EXECPAGE_P(mdpg)) { 1627 if (pv->pv_pmap == NULL) { 1628 UVMHIST_LOG(pmapexechist, 1629 "pg %#jx (pa %#jx): execpage cleared", 1630 (uintptr_t)pg, VM_PAGE_TO_PHYS(pg), 0, 0); 1631 pmap_page_clear_attributes(mdpg, VM_PAGEMD_EXECPAGE); 1632 PMAP_COUNT(exec_uncached_clear_modify); 1633 } else { 1634 UVMHIST_LOG(pmapexechist, 1635 "pg %#jx (pa %#jx): syncicache performed", 1636 (uintptr_t)pg, VM_PAGE_TO_PHYS(pg), 0, 0); 1637 pmap_page_syncicache(pg); 1638 PMAP_COUNT(exec_synced_clear_modify); 1639 } 1640 } 1641 if (!pmap_page_clear_attributes(mdpg, VM_PAGEMD_MODIFIED)) { 1642 UVMHIST_LOG(pmaphist, " <-- false", 0, 0, 0, 0); 1643 return false; 1644 } 1645 if (pv->pv_pmap == NULL) { 1646 UVMHIST_LOG(pmaphist, " <-- true (no mappings)", 0, 0, 0, 0); 1647 return true; 1648 } 1649 1650 /* 1651 * remove write access from any pages that are dirty 1652 * so we can tell if they are written to again later. 1653 * flush the VAC first if there is one. 1654 */ 1655 kpreempt_disable(); 1656 VM_PAGEMD_PVLIST_READLOCK(mdpg); 1657 pmap_pvlist_check(mdpg); 1658 for (; pv != NULL; pv = pv_next) { 1659 pmap_t pmap = pv->pv_pmap; 1660 vaddr_t va = trunc_page(pv->pv_va); 1661 1662 pv_next = pv->pv_next; 1663 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 1664 if (PV_ISKENTER_P(pv)) 1665 continue; 1666 #endif 1667 pt_entry_t * const ptep = pmap_pte_lookup(pmap, va); 1668 KASSERT(ptep); 1669 pt_entry_t pte = pte_prot_nowrite(*ptep); 1670 if (*ptep == pte) { 1671 continue; 1672 } 1673 KASSERT(pte_valid_p(pte)); 1674 const uintptr_t gen = VM_PAGEMD_PVLIST_UNLOCK(mdpg); 1675 pmap_md_tlb_miss_lock_enter(); 1676 pte_set(ptep, pte); 1677 pmap_tlb_invalidate_addr(pmap, va); 1678 pmap_md_tlb_miss_lock_exit(); 1679 pmap_update(pmap); 1680 if (__predict_false(gen != VM_PAGEMD_PVLIST_READLOCK(mdpg))) { 1681 /* 1682 * The list changed! So restart from the beginning. 1683 */ 1684 pv_next = &mdpg->mdpg_first; 1685 pmap_pvlist_check(mdpg); 1686 } 1687 } 1688 pmap_pvlist_check(mdpg); 1689 VM_PAGEMD_PVLIST_UNLOCK(mdpg); 1690 kpreempt_enable(); 1691 1692 UVMHIST_LOG(pmaphist, " <-- true (mappings changed)", 0, 0, 0, 0); 1693 return true; 1694 } 1695 1696 /* 1697 * pmap_is_modified: 1698 * 1699 * Return whether or not the specified physical page is modified 1700 * by any physical maps. 1701 */ 1702 bool 1703 pmap_is_modified(struct vm_page *pg) 1704 { 1705 return VM_PAGEMD_MODIFIED_P(VM_PAGE_TO_MD(pg)); 1706 } 1707 1708 /* 1709 * pmap_set_modified: 1710 * 1711 * Sets the page modified reference bit for the specified page. 1712 */ 1713 void 1714 pmap_set_modified(paddr_t pa) 1715 { 1716 struct vm_page * const pg = PHYS_TO_VM_PAGE(pa); 1717 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 1718 pmap_page_set_attributes(mdpg, VM_PAGEMD_MODIFIED|VM_PAGEMD_REFERENCED); 1719 } 1720 1721 /******************** pv_entry management ********************/ 1722 1723 static void 1724 pmap_pvlist_check(struct vm_page_md *mdpg) 1725 { 1726 #ifdef DEBUG 1727 pv_entry_t pv = &mdpg->mdpg_first; 1728 if (pv->pv_pmap != NULL) { 1729 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 1730 const u_int colormask = uvmexp.colormask; 1731 u_int colors = 0; 1732 #endif 1733 for (; pv != NULL; pv = pv->pv_next) { 1734 KASSERT(pv->pv_pmap != pmap_kernel() || !pmap_md_direct_mapped_vaddr_p(pv->pv_va)); 1735 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 1736 colors |= __BIT(atop(pv->pv_va) & colormask); 1737 #endif 1738 } 1739 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 1740 // Assert that if there is more than 1 color mapped, that the 1741 // page is uncached. 1742 KASSERTMSG(!pmap_md_virtual_cache_aliasing_p() 1743 || colors == 0 || (colors & (colors-1)) == 0 1744 || VM_PAGEMD_UNCACHED_P(mdpg), "colors=%#x uncached=%u", 1745 colors, VM_PAGEMD_UNCACHED_P(mdpg)); 1746 #endif 1747 } else { 1748 KASSERT(pv->pv_next == NULL); 1749 } 1750 #endif /* DEBUG */ 1751 } 1752 1753 /* 1754 * Enter the pmap and virtual address into the 1755 * physical to virtual map table. 1756 */ 1757 void 1758 pmap_enter_pv(pmap_t pmap, vaddr_t va, struct vm_page *pg, pt_entry_t *nptep, 1759 u_int flags) 1760 { 1761 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 1762 pv_entry_t pv, npv, apv; 1763 #ifdef UVMHIST 1764 bool first = false; 1765 #endif 1766 1767 UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist); 1768 UVMHIST_LOG(pmaphist, 1769 "(pmap=%#jx va=%#jx pg=%#jx (%#jx)", 1770 (uintptr_t)pmap, va, (uintptr_t)pg, VM_PAGE_TO_PHYS(pg)); 1771 UVMHIST_LOG(pmaphist, "nptep=%#jx (%#jx))", 1772 (uintptr_t)nptep, pte_value(*nptep), 0, 0); 1773 1774 KASSERT(kpreempt_disabled()); 1775 KASSERT(pmap != pmap_kernel() || !pmap_md_direct_mapped_vaddr_p(va)); 1776 KASSERTMSG(pmap != pmap_kernel() || !pmap_md_io_vaddr_p(va), 1777 "va %#"PRIxVADDR, va); 1778 1779 apv = NULL; 1780 VM_PAGEMD_PVLIST_LOCK(mdpg); 1781 again: 1782 pv = &mdpg->mdpg_first; 1783 pmap_pvlist_check(mdpg); 1784 if (pv->pv_pmap == NULL) { 1785 KASSERT(pv->pv_next == NULL); 1786 /* 1787 * No entries yet, use header as the first entry 1788 */ 1789 PMAP_COUNT(primary_mappings); 1790 PMAP_COUNT(mappings); 1791 #ifdef UVMHIST 1792 first = true; 1793 #endif 1794 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 1795 KASSERT(VM_PAGEMD_CACHED_P(mdpg)); 1796 // If the new mapping has an incompatible color the last 1797 // mapping of this page, clean the page before using it. 1798 if (!PMAP_PAGE_COLOROK_P(va, pv->pv_va)) { 1799 pmap_md_vca_clean(pg, PMAP_WBINV); 1800 } 1801 #endif 1802 pv->pv_pmap = pmap; 1803 pv->pv_va = va | flags; 1804 } else { 1805 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 1806 if (pmap_md_vca_add(pg, va, nptep)) { 1807 goto again; 1808 } 1809 #endif 1810 1811 /* 1812 * There is at least one other VA mapping this page. 1813 * Place this entry after the header. 1814 * 1815 * Note: the entry may already be in the table if 1816 * we are only changing the protection bits. 1817 */ 1818 1819 #ifdef PARANOIADIAG 1820 const paddr_t pa = VM_PAGE_TO_PHYS(pg); 1821 #endif 1822 for (npv = pv; npv; npv = npv->pv_next) { 1823 if (pmap == npv->pv_pmap 1824 && va == trunc_page(npv->pv_va)) { 1825 #ifdef PARANOIADIAG 1826 pt_entry_t *ptep = pmap_pte_lookup(pmap, va); 1827 pt_entry_t pte = (ptep != NULL) ? *ptep : 0; 1828 if (!pte_valid_p(pte) || pte_to_paddr(pte) != pa) 1829 printf("%s: found va %#"PRIxVADDR 1830 " pa %#"PRIxPADDR 1831 " in pv_table but != %#"PRIxPTE"\n", 1832 __func__, va, pa, pte_value(pte)); 1833 #endif 1834 PMAP_COUNT(remappings); 1835 VM_PAGEMD_PVLIST_UNLOCK(mdpg); 1836 if (__predict_false(apv != NULL)) 1837 pmap_pv_free(apv); 1838 1839 UVMHIST_LOG(pmaphist, 1840 " <-- done pv=%#jx (reused)", 1841 (uintptr_t)pv, 0, 0, 0); 1842 return; 1843 } 1844 } 1845 if (__predict_true(apv == NULL)) { 1846 /* 1847 * To allocate a PV, we have to release the PVLIST lock 1848 * so get the page generation. We allocate the PV, and 1849 * then reacquire the lock. 1850 */ 1851 pmap_pvlist_check(mdpg); 1852 const uintptr_t gen = VM_PAGEMD_PVLIST_UNLOCK(mdpg); 1853 1854 apv = (pv_entry_t)pmap_pv_alloc(); 1855 if (apv == NULL) 1856 panic("pmap_enter_pv: pmap_pv_alloc() failed"); 1857 1858 /* 1859 * If the generation has changed, then someone else 1860 * tinkered with this page so we should start over. 1861 */ 1862 if (gen != VM_PAGEMD_PVLIST_LOCK(mdpg)) 1863 goto again; 1864 } 1865 npv = apv; 1866 apv = NULL; 1867 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 1868 /* 1869 * If need to deal with virtual cache aliases, keep mappings 1870 * in the kernel pmap at the head of the list. This allows 1871 * the VCA code to easily use them for cache operations if 1872 * present. 1873 */ 1874 pmap_t kpmap = pmap_kernel(); 1875 if (pmap != kpmap) { 1876 while (pv->pv_pmap == kpmap && pv->pv_next != NULL) { 1877 pv = pv->pv_next; 1878 } 1879 } 1880 #endif 1881 npv->pv_va = va | flags; 1882 npv->pv_pmap = pmap; 1883 npv->pv_next = pv->pv_next; 1884 pv->pv_next = npv; 1885 PMAP_COUNT(mappings); 1886 } 1887 pmap_pvlist_check(mdpg); 1888 VM_PAGEMD_PVLIST_UNLOCK(mdpg); 1889 if (__predict_false(apv != NULL)) 1890 pmap_pv_free(apv); 1891 1892 UVMHIST_LOG(pmaphist, " <-- done pv=%#jx (first %ju)", (uintptr_t)pv, 1893 first, 0, 0); 1894 } 1895 1896 /* 1897 * Remove a physical to virtual address translation. 1898 * If cache was inhibited on this page, and there are no more cache 1899 * conflicts, restore caching. 1900 * Flush the cache if the last page is removed (should always be cached 1901 * at this point). 1902 */ 1903 void 1904 pmap_remove_pv(pmap_t pmap, vaddr_t va, struct vm_page *pg, bool dirty) 1905 { 1906 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 1907 pv_entry_t pv, npv; 1908 bool last; 1909 1910 UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist); 1911 UVMHIST_LOG(pmaphist, 1912 "(pmap=%#jx, va=%#jx, pg=%#jx (pa %#jx)", 1913 (uintptr_t)pmap, va, (uintptr_t)pg, VM_PAGE_TO_PHYS(pg)); 1914 UVMHIST_LOG(pmaphist, "dirty=%ju)", dirty, 0, 0, 0); 1915 1916 KASSERT(kpreempt_disabled()); 1917 KASSERT((va & PAGE_MASK) == 0); 1918 pv = &mdpg->mdpg_first; 1919 1920 VM_PAGEMD_PVLIST_LOCK(mdpg); 1921 pmap_pvlist_check(mdpg); 1922 1923 /* 1924 * If it is the first entry on the list, it is actually 1925 * in the header and we must copy the following entry up 1926 * to the header. Otherwise we must search the list for 1927 * the entry. In either case we free the now unused entry. 1928 */ 1929 1930 last = false; 1931 if (pmap == pv->pv_pmap && va == trunc_page(pv->pv_va)) { 1932 npv = pv->pv_next; 1933 if (npv) { 1934 *pv = *npv; 1935 KASSERT(pv->pv_pmap != NULL); 1936 } else { 1937 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 1938 pmap_page_clear_attributes(mdpg, VM_PAGEMD_UNCACHED); 1939 #endif 1940 pv->pv_pmap = NULL; 1941 last = true; /* Last mapping removed */ 1942 } 1943 PMAP_COUNT(remove_pvfirst); 1944 } else { 1945 for (npv = pv->pv_next; npv; pv = npv, npv = npv->pv_next) { 1946 PMAP_COUNT(remove_pvsearch); 1947 if (pmap == npv->pv_pmap && va == trunc_page(npv->pv_va)) 1948 break; 1949 } 1950 if (npv) { 1951 pv->pv_next = npv->pv_next; 1952 } 1953 } 1954 1955 pmap_pvlist_check(mdpg); 1956 VM_PAGEMD_PVLIST_UNLOCK(mdpg); 1957 1958 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 1959 pmap_md_vca_remove(pg, va, dirty, last); 1960 #endif 1961 1962 /* 1963 * Free the pv_entry if needed. 1964 */ 1965 if (npv) 1966 pmap_pv_free(npv); 1967 if (VM_PAGEMD_EXECPAGE_P(mdpg) && dirty) { 1968 if (last) { 1969 /* 1970 * If this was the page's last mapping, we no longer 1971 * care about its execness. 1972 */ 1973 UVMHIST_LOG(pmapexechist, 1974 "pg %#jx (pa %#jx)last %ju: execpage cleared", 1975 (uintptr_t)pg, VM_PAGE_TO_PHYS(pg), last, 0); 1976 pmap_page_clear_attributes(mdpg, VM_PAGEMD_EXECPAGE); 1977 PMAP_COUNT(exec_uncached_remove); 1978 } else { 1979 /* 1980 * Someone still has it mapped as an executable page 1981 * so we must sync it. 1982 */ 1983 UVMHIST_LOG(pmapexechist, 1984 "pg %#jx (pa %#jx) last %ju: performed syncicache", 1985 (uintptr_t)pg, VM_PAGE_TO_PHYS(pg), last, 0); 1986 pmap_page_syncicache(pg); 1987 PMAP_COUNT(exec_synced_remove); 1988 } 1989 } 1990 1991 UVMHIST_LOG(pmaphist, " <-- done", 0, 0, 0, 0); 1992 } 1993 1994 #if defined(MULTIPROCESSOR) 1995 struct pmap_pvlist_info { 1996 kmutex_t *pli_locks[PAGE_SIZE / 32]; 1997 volatile u_int pli_lock_refs[PAGE_SIZE / 32]; 1998 volatile u_int pli_lock_index; 1999 u_int pli_lock_mask; 2000 } pmap_pvlist_info; 2001 2002 void 2003 pmap_pvlist_lock_init(size_t cache_line_size) 2004 { 2005 struct pmap_pvlist_info * const pli = &pmap_pvlist_info; 2006 const vaddr_t lock_page = uvm_pageboot_alloc(PAGE_SIZE); 2007 vaddr_t lock_va = lock_page; 2008 if (sizeof(kmutex_t) > cache_line_size) { 2009 cache_line_size = roundup2(sizeof(kmutex_t), cache_line_size); 2010 } 2011 const size_t nlocks = PAGE_SIZE / cache_line_size; 2012 KASSERT((nlocks & (nlocks - 1)) == 0); 2013 /* 2014 * Now divide the page into a number of mutexes, one per cacheline. 2015 */ 2016 for (size_t i = 0; i < nlocks; lock_va += cache_line_size, i++) { 2017 kmutex_t * const lock = (kmutex_t *)lock_va; 2018 mutex_init(lock, MUTEX_DEFAULT, IPL_HIGH); 2019 pli->pli_locks[i] = lock; 2020 } 2021 pli->pli_lock_mask = nlocks - 1; 2022 } 2023 2024 kmutex_t * 2025 pmap_pvlist_lock_addr(struct vm_page_md *mdpg) 2026 { 2027 struct pmap_pvlist_info * const pli = &pmap_pvlist_info; 2028 kmutex_t *lock = mdpg->mdpg_lock; 2029 2030 /* 2031 * Allocate a lock on an as-needed basis. This will hopefully give us 2032 * semi-random distribution not based on page color. 2033 */ 2034 if (__predict_false(lock == NULL)) { 2035 size_t locknum = atomic_add_int_nv(&pli->pli_lock_index, 37); 2036 size_t lockid = locknum & pli->pli_lock_mask; 2037 kmutex_t * const new_lock = pli->pli_locks[lockid]; 2038 /* 2039 * Set the lock. If some other thread already did, just use 2040 * the one they assigned. 2041 */ 2042 lock = atomic_cas_ptr(&mdpg->mdpg_lock, NULL, new_lock); 2043 if (lock == NULL) { 2044 lock = new_lock; 2045 atomic_inc_uint(&pli->pli_lock_refs[lockid]); 2046 } 2047 } 2048 2049 /* 2050 * Now finally provide the lock. 2051 */ 2052 return lock; 2053 } 2054 #else /* !MULTIPROCESSOR */ 2055 void 2056 pmap_pvlist_lock_init(size_t cache_line_size) 2057 { 2058 mutex_init(&pmap_pvlist_mutex, MUTEX_DEFAULT, IPL_HIGH); 2059 } 2060 2061 #ifdef MODULAR 2062 kmutex_t * 2063 pmap_pvlist_lock_addr(struct vm_page_md *mdpg) 2064 { 2065 /* 2066 * We just use a global lock. 2067 */ 2068 if (__predict_false(mdpg->mdpg_lock == NULL)) { 2069 mdpg->mdpg_lock = &pmap_pvlist_mutex; 2070 } 2071 2072 /* 2073 * Now finally provide the lock. 2074 */ 2075 return mdpg->mdpg_lock; 2076 } 2077 #endif /* MODULAR */ 2078 #endif /* !MULTIPROCESSOR */ 2079 2080 /* 2081 * pmap_pv_page_alloc: 2082 * 2083 * Allocate a page for the pv_entry pool. 2084 */ 2085 void * 2086 pmap_pv_page_alloc(struct pool *pp, int flags) 2087 { 2088 struct vm_page * const pg = PMAP_ALLOC_POOLPAGE(UVM_PGA_USERESERVE); 2089 if (pg == NULL) 2090 return NULL; 2091 2092 return (void *)pmap_map_poolpage(VM_PAGE_TO_PHYS(pg)); 2093 } 2094 2095 /* 2096 * pmap_pv_page_free: 2097 * 2098 * Free a pv_entry pool page. 2099 */ 2100 void 2101 pmap_pv_page_free(struct pool *pp, void *v) 2102 { 2103 vaddr_t va = (vaddr_t)v; 2104 2105 KASSERT(pmap_md_direct_mapped_vaddr_p(va)); 2106 const paddr_t pa = pmap_md_direct_mapped_vaddr_to_paddr(va); 2107 struct vm_page * const pg = PHYS_TO_VM_PAGE(pa); 2108 KASSERT(pg != NULL); 2109 #ifdef PMAP_VIRTUAL_CACHE_ALIASES 2110 kpreempt_disable(); 2111 pmap_md_vca_remove(pg, va, true, true); 2112 kpreempt_enable(); 2113 #endif 2114 pmap_page_clear_attributes(VM_PAGE_TO_MD(pg), VM_PAGEMD_POOLPAGE); 2115 KASSERT(!VM_PAGEMD_EXECPAGE_P(VM_PAGE_TO_MD(pg))); 2116 uvm_pagefree(pg); 2117 } 2118 2119 #ifdef PMAP_PREFER 2120 /* 2121 * Find first virtual address >= *vap that doesn't cause 2122 * a cache alias conflict. 2123 */ 2124 void 2125 pmap_prefer(vaddr_t foff, vaddr_t *vap, vsize_t sz, int td) 2126 { 2127 vsize_t prefer_mask = ptoa(uvmexp.colormask); 2128 2129 PMAP_COUNT(prefer_requests); 2130 2131 prefer_mask |= pmap_md_cache_prefer_mask(); 2132 2133 if (prefer_mask) { 2134 vaddr_t va = *vap; 2135 vsize_t d = (foff - va) & prefer_mask; 2136 if (d) { 2137 if (td) 2138 *vap = trunc_page(va - ((-d) & prefer_mask)); 2139 else 2140 *vap = round_page(va + d); 2141 PMAP_COUNT(prefer_adjustments); 2142 } 2143 } 2144 } 2145 #endif /* PMAP_PREFER */ 2146 2147 #ifdef PMAP_MAP_POOLPAGE 2148 vaddr_t 2149 pmap_map_poolpage(paddr_t pa) 2150 { 2151 struct vm_page * const pg = PHYS_TO_VM_PAGE(pa); 2152 KASSERT(pg); 2153 2154 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 2155 KASSERT(!VM_PAGEMD_EXECPAGE_P(mdpg)); 2156 2157 pmap_page_set_attributes(mdpg, VM_PAGEMD_POOLPAGE); 2158 2159 return pmap_md_map_poolpage(pa, NBPG); 2160 } 2161 2162 paddr_t 2163 pmap_unmap_poolpage(vaddr_t va) 2164 { 2165 KASSERT(pmap_md_direct_mapped_vaddr_p(va)); 2166 paddr_t pa = pmap_md_direct_mapped_vaddr_to_paddr(va); 2167 2168 struct vm_page * const pg = PHYS_TO_VM_PAGE(pa); 2169 KASSERT(pg != NULL); 2170 KASSERT(!VM_PAGEMD_EXECPAGE_P(VM_PAGE_TO_MD(pg))); 2171 2172 pmap_page_clear_attributes(VM_PAGE_TO_MD(pg), VM_PAGEMD_POOLPAGE); 2173 pmap_md_unmap_poolpage(va, NBPG); 2174 2175 return pa; 2176 } 2177 #endif /* PMAP_MAP_POOLPAGE */ 2178