1 /* $NetBSD: pmap.c,v 1.4 2014/02/25 15:20:29 martin Exp $ */ 2 3 /*- 4 * Copyright (c) 1998, 2001 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility, 9 * NASA Ames Research Center and by Chris G. Demetriou. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 22 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 23 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 24 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 * POSSIBILITY OF SUCH DAMAGE. 31 */ 32 33 /* 34 * Copyright (c) 1992, 1993 35 * The Regents of the University of California. All rights reserved. 36 * 37 * This code is derived from software contributed to Berkeley by 38 * the Systems Programming Group of the University of Utah Computer 39 * Science Department and Ralph Campbell. 40 * 41 * Redistribution and use in source and binary forms, with or without 42 * modification, are permitted provided that the following conditions 43 * are met: 44 * 1. Redistributions of source code must retain the above copyright 45 * notice, this list of conditions and the following disclaimer. 46 * 2. Redistributions in binary form must reproduce the above copyright 47 * notice, this list of conditions and the following disclaimer in the 48 * documentation and/or other materials provided with the distribution. 49 * 3. Neither the name of the University nor the names of its contributors 50 * may be used to endorse or promote products derived from this software 51 * without specific prior written permission. 52 * 53 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 54 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 55 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 56 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 57 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 58 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 59 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 60 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 61 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 62 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 63 * SUCH DAMAGE. 64 * 65 * @(#)pmap.c 8.4 (Berkeley) 1/26/94 66 */ 67 68 #include <sys/cdefs.h> 69 70 __KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.4 2014/02/25 15:20:29 martin Exp $"); 71 72 /* 73 * Manages physical address maps. 74 * 75 * In addition to hardware address maps, this 76 * module is called upon to provide software-use-only 77 * maps which may or may not be stored in the same 78 * form as hardware maps. These pseudo-maps are 79 * used to store intermediate results from copy 80 * operations to and from address spaces. 81 * 82 * Since the information managed by this module is 83 * also stored by the logical address mapping module, 84 * this module may throw away valid virtual-to-physical 85 * mappings at almost any time. However, invalidations 86 * of virtual-to-physical mappings must be done as 87 * requested. 88 * 89 * In order to cope with hardware architectures which 90 * make virtual-to-physical map invalidates expensive, 91 * this module may delay invalidate or reduced protection 92 * operations until such time as they are actually 93 * necessary. This module is given full information as 94 * to which processors are currently using which maps, 95 * and to when physical maps must be made correct. 96 */ 97 98 #include "opt_modular.h" 99 #include "opt_multiprocessor.h" 100 #include "opt_sysv.h" 101 102 #define __PMAP_PRIVATE 103 104 #include <sys/param.h> 105 #include <sys/systm.h> 106 #include <sys/proc.h> 107 #include <sys/buf.h> 108 #include <sys/pool.h> 109 #include <sys/atomic.h> 110 #include <sys/mutex.h> 111 #include <sys/atomic.h> 112 #ifdef SYSVSHM 113 #include <sys/shm.h> 114 #endif 115 #include <sys/socketvar.h> /* XXX: for sock_loan_thresh */ 116 117 #include <uvm/uvm.h> 118 119 #define PMAP_COUNT(name) (pmap_evcnt_##name.ev_count++ + 0) 120 #define PMAP_COUNTER(name, desc) \ 121 static struct evcnt pmap_evcnt_##name = \ 122 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap", desc); \ 123 EVCNT_ATTACH_STATIC(pmap_evcnt_##name) 124 125 PMAP_COUNTER(remove_kernel_calls, "remove kernel calls"); 126 PMAP_COUNTER(remove_kernel_pages, "kernel pages unmapped"); 127 PMAP_COUNTER(remove_user_calls, "remove user calls"); 128 PMAP_COUNTER(remove_user_pages, "user pages unmapped"); 129 PMAP_COUNTER(remove_flushes, "remove cache flushes"); 130 PMAP_COUNTER(remove_tlb_ops, "remove tlb ops"); 131 PMAP_COUNTER(remove_pvfirst, "remove pv first"); 132 PMAP_COUNTER(remove_pvsearch, "remove pv search"); 133 134 PMAP_COUNTER(prefer_requests, "prefer requests"); 135 PMAP_COUNTER(prefer_adjustments, "prefer adjustments"); 136 137 PMAP_COUNTER(idlezeroed_pages, "pages idle zeroed"); 138 PMAP_COUNTER(zeroed_pages, "pages zeroed"); 139 PMAP_COUNTER(copied_pages, "pages copied"); 140 141 PMAP_COUNTER(kenter_pa, "kernel fast mapped pages"); 142 PMAP_COUNTER(kenter_pa_bad, "kernel fast mapped pages (bad color)"); 143 PMAP_COUNTER(kenter_pa_unmanaged, "kernel fast mapped unmanaged pages"); 144 PMAP_COUNTER(kremove_pages, "kernel fast unmapped pages"); 145 146 PMAP_COUNTER(page_cache_evictions, "pages changed to uncacheable"); 147 PMAP_COUNTER(page_cache_restorations, "pages changed to cacheable"); 148 149 PMAP_COUNTER(kernel_mappings_bad, "kernel pages mapped (bad color)"); 150 PMAP_COUNTER(user_mappings_bad, "user pages mapped (bad color)"); 151 PMAP_COUNTER(kernel_mappings, "kernel pages mapped"); 152 PMAP_COUNTER(user_mappings, "user pages mapped"); 153 PMAP_COUNTER(user_mappings_changed, "user mapping changed"); 154 PMAP_COUNTER(kernel_mappings_changed, "kernel mapping changed"); 155 PMAP_COUNTER(uncached_mappings, "uncached pages mapped"); 156 PMAP_COUNTER(unmanaged_mappings, "unmanaged pages mapped"); 157 PMAP_COUNTER(managed_mappings, "managed pages mapped"); 158 PMAP_COUNTER(mappings, "pages mapped"); 159 PMAP_COUNTER(remappings, "pages remapped"); 160 PMAP_COUNTER(unmappings, "pages unmapped"); 161 PMAP_COUNTER(primary_mappings, "page initial mappings"); 162 PMAP_COUNTER(primary_unmappings, "page final unmappings"); 163 PMAP_COUNTER(tlb_hit, "page mapping"); 164 165 PMAP_COUNTER(exec_mappings, "exec pages mapped"); 166 PMAP_COUNTER(exec_synced_mappings, "exec pages synced"); 167 PMAP_COUNTER(exec_synced_remove, "exec pages synced (PR)"); 168 PMAP_COUNTER(exec_synced_clear_modify, "exec pages synced (CM)"); 169 PMAP_COUNTER(exec_synced_page_protect, "exec pages synced (PP)"); 170 PMAP_COUNTER(exec_synced_protect, "exec pages synced (P)"); 171 PMAP_COUNTER(exec_uncached_page_protect, "exec pages uncached (PP)"); 172 PMAP_COUNTER(exec_uncached_clear_modify, "exec pages uncached (CM)"); 173 PMAP_COUNTER(exec_uncached_zero_page, "exec pages uncached (ZP)"); 174 PMAP_COUNTER(exec_uncached_copy_page, "exec pages uncached (CP)"); 175 PMAP_COUNTER(exec_uncached_remove, "exec pages uncached (PR)"); 176 177 PMAP_COUNTER(create, "creates"); 178 PMAP_COUNTER(reference, "references"); 179 PMAP_COUNTER(dereference, "dereferences"); 180 PMAP_COUNTER(destroy, "destroyed"); 181 PMAP_COUNTER(activate, "activations"); 182 PMAP_COUNTER(deactivate, "deactivations"); 183 PMAP_COUNTER(update, "updates"); 184 #ifdef MULTIPROCESSOR 185 PMAP_COUNTER(shootdown_ipis, "shootdown IPIs"); 186 #endif 187 PMAP_COUNTER(unwire, "unwires"); 188 PMAP_COUNTER(copy, "copies"); 189 PMAP_COUNTER(clear_modify, "clear_modifies"); 190 PMAP_COUNTER(protect, "protects"); 191 PMAP_COUNTER(page_protect, "page_protects"); 192 193 #define PMAP_ASID_RESERVED 0 194 CTASSERT(PMAP_ASID_RESERVED == 0); 195 196 /* 197 * Initialize the kernel pmap. 198 */ 199 #ifdef MULTIPROCESSOR 200 #define PMAP_SIZE offsetof(struct pmap, pm_pai[MAXCPUS]) 201 #else 202 #define PMAP_SIZE sizeof(struct pmap) 203 kmutex_t pmap_pvlist_mutex __aligned(COHERENCY_UNIT); 204 #endif 205 206 struct pmap_kernel kernel_pmap_store = { 207 .kernel_pmap = { 208 .pm_count = 1, 209 .pm_segtab = PMAP_INVALID_SEGTAB_ADDRESS, 210 .pm_minaddr = VM_MIN_KERNEL_ADDRESS, 211 .pm_maxaddr = VM_MAX_KERNEL_ADDRESS, 212 }, 213 }; 214 215 struct pmap * const kernel_pmap_ptr = &kernel_pmap_store.kernel_pmap; 216 217 struct pmap_limits pmap_limits; 218 219 #ifdef UVMHIST 220 static struct kern_history_ent pmapexechistbuf[10000]; 221 static struct kern_history_ent pmaphistbuf[10000]; 222 #endif 223 224 /* 225 * The pools from which pmap structures and sub-structures are allocated. 226 */ 227 struct pool pmap_pmap_pool; 228 struct pool pmap_pv_pool; 229 230 #ifndef PMAP_PV_LOWAT 231 #define PMAP_PV_LOWAT 16 232 #endif 233 int pmap_pv_lowat = PMAP_PV_LOWAT; 234 235 bool pmap_initialized = false; 236 #define PMAP_PAGE_COLOROK_P(a, b) \ 237 ((((int)(a) ^ (int)(b)) & pmap_page_colormask) == 0) 238 u_int pmap_page_colormask; 239 240 #define PAGE_IS_MANAGED(pa) \ 241 (pmap_initialized == true && vm_physseg_find(atop(pa), NULL) != -1) 242 243 #define PMAP_IS_ACTIVE(pm) \ 244 ((pm) == pmap_kernel() || \ 245 (pm) == curlwp->l_proc->p_vmspace->vm_map.pmap) 246 247 /* Forward function declarations */ 248 void pmap_remove_pv(pmap_t, vaddr_t, struct vm_page *, bool); 249 void pmap_enter_pv(pmap_t, vaddr_t, struct vm_page *, u_int *); 250 251 /* 252 * PV table management functions. 253 */ 254 void *pmap_pv_page_alloc(struct pool *, int); 255 void pmap_pv_page_free(struct pool *, void *); 256 257 struct pool_allocator pmap_pv_page_allocator = { 258 pmap_pv_page_alloc, pmap_pv_page_free, 0, 259 }; 260 261 #define pmap_pv_alloc() pool_get(&pmap_pv_pool, PR_NOWAIT) 262 #define pmap_pv_free(pv) pool_put(&pmap_pv_pool, (pv)) 263 264 /* 265 * Misc. functions. 266 */ 267 268 bool 269 pmap_page_clear_attributes(struct vm_page_md *mdpg, u_int clear_attributes) 270 { 271 volatile u_int * const attrp = &mdpg->mdpg_attrs; 272 #ifdef MULTIPROCESSOR 273 for (;;) { 274 u_int old_attr = *attrp; 275 if ((old_attr & clear_attributes) == 0) 276 return false; 277 u_int new_attr = old_attr & ~clear_attributes; 278 if (old_attr == atomic_cas_uint(attrp, old_attr, new_attr)) 279 return true; 280 } 281 #else 282 u_int old_attr = *attrp; 283 if ((old_attr & clear_attributes) == 0) 284 return false; 285 *attrp &= ~clear_attributes; 286 return true; 287 #endif 288 } 289 290 void 291 pmap_page_set_attributes(struct vm_page_md *mdpg, u_int set_attributes) 292 { 293 #ifdef MULTIPROCESSOR 294 atomic_or_uint(&mdpg->mdpg_attrs, set_attributes); 295 #else 296 mdpg->mdpg_attrs |= set_attributes; 297 #endif 298 } 299 300 static void 301 pmap_page_syncicache(struct vm_page *pg) 302 { 303 #ifndef MULTIPROCESSOR 304 struct pmap * const curpmap = curcpu()->ci_curpm; 305 #endif 306 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 307 pv_entry_t pv = &mdpg->mdpg_first; 308 kcpuset_t *onproc; 309 #ifdef MULTIPROCESSOR 310 kcpuset_create(&onproc, true); 311 #else 312 onproc = NULL; 313 #endif 314 (void)VM_PAGEMD_PVLIST_LOCK(mdpg, false); 315 316 if (pv->pv_pmap != NULL) { 317 for (; pv != NULL; pv = pv->pv_next) { 318 #ifdef MULTIPROCESSOR 319 kcpuset_merge(onproc, pv->pv_pmap->pm_onproc); 320 if (kcpuset_match(onproc, kcpuset_running)) { 321 break; 322 } 323 #else 324 if (pv->pv_pmap == curpmap) { 325 onproc = curcpu()->ci_data.cpu_kcpuset; 326 break; 327 } 328 #endif 329 } 330 } 331 VM_PAGEMD_PVLIST_UNLOCK(mdpg); 332 kpreempt_disable(); 333 pmap_md_page_syncicache(pg, onproc); 334 #ifdef MULTIPROCESSOR 335 kcpuset_destroy(onproc); 336 #endif 337 kpreempt_enable(); 338 } 339 340 /* 341 * Define the initial bounds of the kernel virtual address space. 342 */ 343 void 344 pmap_virtual_space(vaddr_t *vstartp, vaddr_t *vendp) 345 { 346 347 *vstartp = VM_MIN_KERNEL_ADDRESS; 348 *vendp = VM_MAX_KERNEL_ADDRESS; 349 } 350 351 vaddr_t 352 pmap_growkernel(vaddr_t maxkvaddr) 353 { 354 vaddr_t virtual_end = pmap_limits.virtual_end; 355 maxkvaddr = pmap_round_seg(maxkvaddr) - 1; 356 357 /* 358 * Reserve PTEs for the new KVA space. 359 */ 360 for (; virtual_end < maxkvaddr; virtual_end += NBSEG) { 361 pmap_pte_reserve(pmap_kernel(), virtual_end, 0); 362 } 363 364 /* 365 * Don't exceed VM_MAX_KERNEL_ADDRESS! 366 */ 367 if (virtual_end == 0 || virtual_end > VM_MAX_KERNEL_ADDRESS) 368 virtual_end = VM_MAX_KERNEL_ADDRESS; 369 370 /* 371 * Update new end. 372 */ 373 pmap_limits.virtual_end = virtual_end; 374 return virtual_end; 375 } 376 377 /* 378 * Bootstrap memory allocator (alternative to vm_bootstrap_steal_memory()). 379 * This function allows for early dynamic memory allocation until the virtual 380 * memory system has been bootstrapped. After that point, either kmem_alloc 381 * or malloc should be used. This function works by stealing pages from the 382 * (to be) managed page pool, then implicitly mapping the pages (by using 383 * their k0seg addresses) and zeroing them. 384 * 385 * It may be used once the physical memory segments have been pre-loaded 386 * into the vm_physmem[] array. Early memory allocation MUST use this 387 * interface! This cannot be used after vm_page_startup(), and will 388 * generate a panic if tried. 389 * 390 * Note that this memory will never be freed, and in essence it is wired 391 * down. 392 * 393 * We must adjust *vstartp and/or *vendp iff we use address space 394 * from the kernel virtual address range defined by pmap_virtual_space(). 395 */ 396 vaddr_t 397 pmap_steal_memory(vsize_t size, vaddr_t *vstartp, vaddr_t *vendp) 398 { 399 u_int npgs; 400 paddr_t pa; 401 vaddr_t va; 402 403 size = round_page(size); 404 npgs = atop(size); 405 406 for (u_int bank = 0; bank < vm_nphysseg; bank++) { 407 struct vm_physseg * const seg = VM_PHYSMEM_PTR(bank); 408 if (uvm.page_init_done == true) 409 panic("pmap_steal_memory: called _after_ bootstrap"); 410 411 if (seg->avail_start != seg->start || 412 seg->avail_start >= seg->avail_end) 413 continue; 414 415 if ((seg->avail_end - seg->avail_start) < npgs) 416 continue; 417 418 /* 419 * There are enough pages here; steal them! 420 */ 421 pa = ptoa(seg->avail_start); 422 seg->avail_start += npgs; 423 seg->start += npgs; 424 425 /* 426 * Have we used up this segment? 427 */ 428 if (seg->avail_start == seg->end) { 429 if (vm_nphysseg == 1) 430 panic("pmap_steal_memory: out of memory!"); 431 432 /* Remove this segment from the list. */ 433 vm_nphysseg--; 434 if (bank < vm_nphysseg) 435 memmove(seg, seg+1, 436 sizeof(*seg) * (vm_nphysseg - bank)); 437 } 438 439 va = pmap_md_map_poolpage(pa, size); 440 memset((void *)va, 0, size); 441 return va; 442 } 443 444 /* 445 * If we got here, there was no memory left. 446 */ 447 panic("pmap_steal_memory: no memory to steal"); 448 } 449 450 /* 451 * Initialize the pmap module. 452 * Called by vm_init, to initialize any structures that the pmap 453 * system needs to map virtual memory. 454 */ 455 void 456 pmap_init(void) 457 { 458 UVMHIST_INIT_STATIC(pmapexechist, pmapexechistbuf); 459 UVMHIST_INIT_STATIC(pmaphist, pmaphistbuf); 460 461 UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist); 462 463 /* 464 * Initialize the segtab lock. 465 */ 466 mutex_init(&pmap_segtab_lock, MUTEX_DEFAULT, IPL_HIGH); 467 468 /* 469 * Set a low water mark on the pv_entry pool, so that we are 470 * more likely to have these around even in extreme memory 471 * starvation. 472 */ 473 pool_setlowat(&pmap_pv_pool, pmap_pv_lowat); 474 475 pmap_md_init(); 476 477 /* 478 * Now it is safe to enable pv entry recording. 479 */ 480 pmap_initialized = true; 481 } 482 483 /* 484 * Create and return a physical map. 485 * 486 * If the size specified for the map 487 * is zero, the map is an actual physical 488 * map, and may be referenced by the 489 * hardware. 490 * 491 * If the size specified is non-zero, 492 * the map will be used in software only, and 493 * is bounded by that size. 494 */ 495 pmap_t 496 pmap_create(void) 497 { 498 pmap_t pmap; 499 500 UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist); 501 PMAP_COUNT(create); 502 503 pmap = pool_get(&pmap_pmap_pool, PR_WAITOK); 504 memset(pmap, 0, PMAP_SIZE); 505 506 KASSERT(pmap->pm_pai[0].pai_link.le_prev == NULL); 507 508 pmap->pm_count = 1; 509 pmap->pm_minaddr = VM_MIN_ADDRESS; 510 pmap->pm_maxaddr = VM_MAXUSER_ADDRESS; 511 512 pmap_segtab_init(pmap); 513 514 UVMHIST_LOG(pmaphist, "<- pmap %p", pmap,0,0,0); 515 return pmap; 516 } 517 518 /* 519 * Retire the given physical map from service. 520 * Should only be called if the map contains 521 * no valid mappings. 522 */ 523 void 524 pmap_destroy(pmap_t pmap) 525 { 526 UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist); 527 UVMHIST_LOG(pmaphist, "(pmap=%p)", pmap, 0,0,0); 528 529 if (atomic_dec_uint_nv(&pmap->pm_count) > 0) { 530 PMAP_COUNT(dereference); 531 return; 532 } 533 534 KASSERT(pmap->pm_count == 0); 535 PMAP_COUNT(destroy); 536 kpreempt_disable(); 537 pmap_tlb_asid_release_all(pmap); 538 pmap_segtab_destroy(pmap, NULL, 0); 539 540 pool_put(&pmap_pmap_pool, pmap); 541 kpreempt_enable(); 542 543 UVMHIST_LOG(pmaphist, "<- done", 0,0,0,0); 544 } 545 546 /* 547 * Add a reference to the specified pmap. 548 */ 549 void 550 pmap_reference(pmap_t pmap) 551 { 552 553 UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist); 554 UVMHIST_LOG(pmaphist, "(pmap=%p)", pmap, 0,0,0); 555 PMAP_COUNT(reference); 556 557 if (pmap != NULL) { 558 atomic_inc_uint(&pmap->pm_count); 559 } 560 561 UVMHIST_LOG(pmaphist, "<- done", 0,0,0,0); 562 } 563 564 /* 565 * Make a new pmap (vmspace) active for the given process. 566 */ 567 void 568 pmap_activate(struct lwp *l) 569 { 570 pmap_t pmap = l->l_proc->p_vmspace->vm_map.pmap; 571 572 UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist); 573 UVMHIST_LOG(pmaphist, "(l=%p (pmap=%p))", l, pmap, 0,0); 574 PMAP_COUNT(activate); 575 576 kpreempt_disable(); 577 pmap_tlb_asid_acquire(pmap, l); 578 if (l == curlwp) { 579 pmap_segtab_activate(pmap, l); 580 } 581 kpreempt_enable(); 582 583 UVMHIST_LOG(pmaphist, "<- done", 0,0,0,0); 584 } 585 586 /* 587 * Make a previously active pmap (vmspace) inactive. 588 */ 589 void 590 pmap_deactivate(struct lwp *l) 591 { 592 pmap_t pmap = l->l_proc->p_vmspace->vm_map.pmap; 593 594 UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist); 595 UVMHIST_LOG(pmaphist, "(l=%p (pmap=%p))", l, pmap, 0,0); 596 PMAP_COUNT(deactivate); 597 598 kpreempt_disable(); 599 curcpu()->ci_pmap_user_segtab = PMAP_INVALID_SEGTAB_ADDRESS; 600 pmap_tlb_asid_deactivate(pmap); 601 kpreempt_enable(); 602 603 UVMHIST_LOG(pmaphist, "<- done", 0,0,0,0); 604 } 605 606 void 607 pmap_update(struct pmap *pmap) 608 { 609 610 UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist); 611 UVMHIST_LOG(pmaphist, "(pmap=%p)", pmap, 0,0,0); 612 PMAP_COUNT(update); 613 614 kpreempt_disable(); 615 #if defined(MULTIPROCESSOR) && defined(PMAP_NEED_TLB_SHOOTDOWN) 616 u_int pending = atomic_swap_uint(&pmap->pm_shootdown_pending, 0); 617 if (pending && pmap_tlb_shootdown_bystanders(pmap)) 618 PMAP_COUNT(shootdown_ipis); 619 #endif 620 #ifdef DEBUG 621 pmap_tlb_check(pmap, pmap_md_tlb_check_entry); 622 #endif /* DEBUG */ 623 624 /* 625 * If pmap_remove_all was called, we deactivated ourselves and nuked 626 * our ASID. Now we have to reactivate ourselves. 627 */ 628 if (__predict_false(pmap->pm_flags & PMAP_DEFERRED_ACTIVATE)) { 629 pmap->pm_flags ^= PMAP_DEFERRED_ACTIVATE; 630 pmap_tlb_asid_acquire(pmap, curlwp); 631 pmap_segtab_activate(pmap, curlwp); 632 } 633 kpreempt_enable(); 634 635 UVMHIST_LOG(pmaphist, "<- done", 0,0,0,0); 636 } 637 638 /* 639 * Remove the given range of addresses from the specified map. 640 * 641 * It is assumed that the start and end are properly 642 * rounded to the page size. 643 */ 644 645 static bool 646 pmap_pte_remove(pmap_t pmap, vaddr_t sva, vaddr_t eva, pt_entry_t *ptep, 647 uintptr_t flags) 648 { 649 const pt_entry_t npte = flags; 650 const bool is_kernel_pmap_p = (pmap == pmap_kernel()); 651 652 UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist); 653 UVMHIST_LOG(pmaphist, "(pmap=%p %sva=%"PRIxVADDR"..%"PRIxVADDR, 654 pmap, (is_kernel_pmap_p ? "(kernel) " : ""), sva, eva); 655 UVMHIST_LOG(pmaphist, "ptep=%p, flags(npte)=%#"PRIxPTR")", 656 ptep, flags, 0, 0); 657 658 KASSERT(kpreempt_disabled()); 659 660 for (; sva < eva; sva += NBPG, ptep++) { 661 pt_entry_t pt_entry = *ptep; 662 if (!pte_valid_p(pt_entry)) 663 continue; 664 if (is_kernel_pmap_p) 665 PMAP_COUNT(remove_kernel_calls); 666 else 667 PMAP_COUNT(remove_user_pages); 668 if (pte_wired_p(pt_entry)) 669 pmap->pm_stats.wired_count--; 670 pmap->pm_stats.resident_count--; 671 struct vm_page *pg = PHYS_TO_VM_PAGE(pte_to_paddr(pt_entry)); 672 if (__predict_true(pg != NULL)) { 673 pmap_remove_pv(pmap, sva, pg, 674 pte_modified_p(pt_entry)); 675 } 676 *ptep = npte; 677 /* 678 * Flush the TLB for the given address. 679 */ 680 pmap_tlb_invalidate_addr(pmap, sva); 681 } 682 return false; 683 } 684 685 void 686 pmap_remove(pmap_t pmap, vaddr_t sva, vaddr_t eva) 687 { 688 const bool is_kernel_pmap_p = (pmap == pmap_kernel()); 689 const pt_entry_t npte = pte_nv_entry(is_kernel_pmap_p); 690 691 UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist); 692 UVMHIST_LOG(pmaphist, "(pmap=%p, va=%#"PRIxVADDR"..%#"PRIxVADDR")", 693 pmap, sva, eva, 0); 694 695 if (is_kernel_pmap_p) 696 PMAP_COUNT(remove_kernel_calls); 697 else 698 PMAP_COUNT(remove_user_calls); 699 #ifdef PARANOIADIAG 700 if (sva < pm->pm_minaddr || eva > pm->pm_maxaddr) 701 panic("%s: va range %#"PRIxVADDR"-%#"PRIxVADDR" not in range", 702 __func__, sva, eva - 1); 703 if (PMAP_IS_ACTIVE(pmap)) { 704 struct pmap_asid_info * const pai = PMAP_PAI(pmap, curcpu()); 705 uint32_t asid = tlb_get_asid(); 706 if (asid != pai->pai_asid) { 707 panic("%s: inconsistency for active TLB flush" 708 ": %d <-> %d", __func__, asid, pai->pai_asid); 709 } 710 } 711 #endif 712 kpreempt_disable(); 713 pmap_pte_process(pmap, sva, eva, pmap_pte_remove, npte); 714 kpreempt_enable(); 715 716 UVMHIST_LOG(pmaphist, "<- done", 0,0,0,0); 717 } 718 719 /* 720 * pmap_page_protect: 721 * 722 * Lower the permission for all mappings to a given page. 723 */ 724 void 725 pmap_page_protect(struct vm_page *pg, vm_prot_t prot) 726 { 727 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 728 pv_entry_t pv; 729 vaddr_t va; 730 731 UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist); 732 UVMHIST_LOG(pmaphist, "(pg=%p (pa %#"PRIxPADDR") prot=%#x)", 733 pg, VM_PAGE_TO_PHYS(pg), prot, 0); 734 PMAP_COUNT(page_protect); 735 736 switch (prot) { 737 case VM_PROT_READ|VM_PROT_WRITE: 738 case VM_PROT_ALL: 739 break; 740 741 /* copy_on_write */ 742 case VM_PROT_READ: 743 case VM_PROT_READ|VM_PROT_EXECUTE: 744 (void)VM_PAGEMD_PVLIST_LOCK(mdpg, false); 745 pv = &mdpg->mdpg_first; 746 /* 747 * Loop over all current mappings setting/clearing as appropriate. 748 */ 749 if (pv->pv_pmap != NULL) { 750 while (pv != NULL) { 751 const pmap_t pmap = pv->pv_pmap; 752 const uint16_t gen = VM_PAGEMD_PVLIST_GEN(mdpg); 753 va = pv->pv_va; 754 VM_PAGEMD_PVLIST_UNLOCK(mdpg); 755 pmap_protect(pmap, va, va + PAGE_SIZE, prot); 756 KASSERT(pv->pv_pmap == pmap); 757 pmap_update(pmap); 758 if (gen != VM_PAGEMD_PVLIST_LOCK(mdpg, false)) { 759 pv = &mdpg->mdpg_first; 760 } else { 761 pv = pv->pv_next; 762 } 763 } 764 } 765 VM_PAGEMD_PVLIST_UNLOCK(mdpg); 766 break; 767 768 /* remove_all */ 769 default: 770 /* 771 * Do this first so that for each unmapping, pmap_remove_pv 772 * won't try to sync the icache. 773 */ 774 if (pmap_page_clear_attributes(mdpg, VM_PAGEMD_EXECPAGE)) { 775 UVMHIST_LOG(pmapexechist, "pg %p (pa %#"PRIxPADDR 776 "): execpage cleared", pg, VM_PAGE_TO_PHYS(pg),0,0); 777 PMAP_COUNT(exec_uncached_page_protect); 778 } 779 (void)VM_PAGEMD_PVLIST_LOCK(mdpg, false); 780 pv = &mdpg->mdpg_first; 781 while (pv->pv_pmap != NULL) { 782 const pmap_t pmap = pv->pv_pmap; 783 va = pv->pv_va; 784 VM_PAGEMD_PVLIST_UNLOCK(mdpg); 785 pmap_remove(pmap, va, va + PAGE_SIZE); 786 pmap_update(pmap); 787 (void)VM_PAGEMD_PVLIST_LOCK(mdpg, false); 788 } 789 VM_PAGEMD_PVLIST_UNLOCK(mdpg); 790 } 791 792 UVMHIST_LOG(pmaphist, "<- done", 0,0,0,0); 793 } 794 795 static bool 796 pmap_pte_protect(pmap_t pmap, vaddr_t sva, vaddr_t eva, pt_entry_t *ptep, 797 uintptr_t flags) 798 { 799 const vm_prot_t prot = (flags & VM_PROT_ALL); 800 801 UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist); 802 UVMHIST_LOG(pmaphist, "(pmap=%p %sva=%"PRIxVADDR"..%"PRIxVADDR, 803 pmap, (pmap == pmap_kernel() ? "(kernel) " : ""), sva, eva); 804 UVMHIST_LOG(pmaphist, "ptep=%p, flags(npte)=%#"PRIxPTR")", 805 ptep, flags, 0, 0); 806 807 KASSERT(kpreempt_disabled()); 808 /* 809 * Change protection on every valid mapping within this segment. 810 */ 811 for (; sva < eva; sva += NBPG, ptep++) { 812 pt_entry_t pt_entry = *ptep; 813 if (!pte_valid_p(pt_entry)) 814 continue; 815 struct vm_page * const pg = 816 PHYS_TO_VM_PAGE(pte_to_paddr(pt_entry)); 817 if (pg != NULL && pte_modified_p(pt_entry)) { 818 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 819 pmap_md_vca_clean(pg, sva, PMAP_WBINV); 820 if (VM_PAGEMD_EXECPAGE_P(mdpg)) { 821 KASSERT(mdpg->mdpg_first.pv_pmap != NULL); 822 if (pte_cached_p(pt_entry)) { 823 UVMHIST_LOG(pmapexechist, 824 "pg %p (pa %#"PRIxPADDR"): %s", 825 pg, VM_PAGE_TO_PHYS(pg), 826 "syncicached performed", 0); 827 pmap_page_syncicache(pg); 828 PMAP_COUNT(exec_synced_protect); 829 } 830 } 831 } 832 pt_entry = pte_prot_downgrade(pt_entry, prot); 833 if (*ptep != pt_entry) { 834 *ptep = pt_entry; 835 /* 836 * Update the TLB if needed. 837 */ 838 pmap_tlb_update_addr(pmap, sva, pt_entry, 839 PMAP_TLB_NEED_IPI); 840 } 841 } 842 return false; 843 } 844 845 /* 846 * Set the physical protection on the 847 * specified range of this map as requested. 848 */ 849 void 850 pmap_protect(pmap_t pmap, vaddr_t sva, vaddr_t eva, vm_prot_t prot) 851 { 852 853 UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist); 854 UVMHIST_LOG(pmaphist, 855 " pmap=%p, va=%#"PRIxVADDR"..%#"PRIxVADDR" port=%#x)", 856 pmap, sva, eva, prot); 857 PMAP_COUNT(protect); 858 859 if ((prot & VM_PROT_READ) == VM_PROT_NONE) { 860 pmap_remove(pmap, sva, eva); 861 UVMHIST_LOG(pmaphist, "<- done", 0,0,0,0); 862 return; 863 } 864 865 #ifdef PARANOIADIAG 866 if (sva < pm->pm_minaddr || eva > pm->pm_maxaddr) 867 panic("%s: va range %#"PRIxVADDR"-%#"PRIxVADDR" not in range", 868 __func__, sva, eva - 1); 869 if (PMAP_IS_ACTIVE(pmap)) { 870 struct pmap_asid_info * const pai = PMAP_PAI(pmap, curcpu()); 871 uint32_t asid = tlb_get_asid(); 872 if (asid != pai->pai_asid) { 873 panic("%s: inconsistency for active TLB update" 874 ": %d <-> %d", __func__, asid, pai->pai_asid); 875 } 876 } 877 #endif 878 879 /* 880 * Change protection on every valid mapping within this segment. 881 */ 882 kpreempt_disable(); 883 pmap_pte_process(pmap, sva, eva, pmap_pte_protect, prot); 884 kpreempt_enable(); 885 886 UVMHIST_LOG(pmaphist, "<- done", 0,0,0,0); 887 } 888 889 #if defined(__PMAP_VIRTUAL_CACHE_ALIASES) 890 /* 891 * pmap_page_cache: 892 * 893 * Change all mappings of a managed page to cached/uncached. 894 */ 895 static void 896 pmap_page_cache(struct vm_page *pg, bool cached) 897 { 898 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 899 UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist); 900 UVMHIST_LOG(pmaphist, "(pg=%p (pa %#"PRIxPADDR") cached=%s)", 901 pg, VM_PAGE_TO_PHYS(pg), cached ? "true" : "false", 0); 902 KASSERT(kpreempt_disabled()); 903 904 if (cached) { 905 pmap_page_clear_attributes(mdpg, VM_PAGEMD_UNCACHED); 906 PMAP_COUNT(page_cache_restorations); 907 } else { 908 pmap_page_set_attributes(mdpg, VM_PAGEMD_UNCACHED); 909 PMAP_COUNT(page_cache_evictions); 910 } 911 912 KASSERT(VM_PAGEMD_PVLIST_LOCKED_P(mdpg)); 913 KASSERT(kpreempt_disabled()); 914 for (pv_entry_t pv = &mdpg->mdpg_first; 915 pv != NULL; 916 pv = pv->pv_next) { 917 pmap_t pmap = pv->pv_pmap; 918 vaddr_t va = pv->pv_va; 919 920 KASSERT(pmap != NULL); 921 KASSERT(pmap != pmap_kernel() || !pmap_md_direct_mapped_vaddr_p(va)); 922 pt_entry_t * const ptep = pmap_pte_lookup(pmap, va); 923 if (ptep == NULL) 924 continue; 925 pt_entry_t pt_entry = *ptep; 926 if (pte_valid_p(pt_entry)) { 927 pt_entry = pte_cached_change(pt_entry, cached); 928 *ptep = pt_entry; 929 pmap_tlb_update_addr(pmap, va, pt_entry, 930 PMAP_TLB_NEED_IPI); 931 } 932 } 933 UVMHIST_LOG(pmaphist, "<- done", 0,0,0,0); 934 } 935 #endif /* __PMAP_VIRTUAL_CACHE_ALIASES */ 936 937 /* 938 * Insert the given physical page (p) at 939 * the specified virtual address (v) in the 940 * target physical map with the protection requested. 941 * 942 * If specified, the page will be wired down, meaning 943 * that the related pte can not be reclaimed. 944 * 945 * NB: This is the only routine which MAY NOT lazy-evaluate 946 * or lose information. That is, this routine must actually 947 * insert this page into the given map NOW. 948 */ 949 int 950 pmap_enter(pmap_t pmap, vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags) 951 { 952 pt_entry_t npte; 953 const bool wired = (flags & PMAP_WIRED) != 0; 954 const bool is_kernel_pmap_p = (pmap == pmap_kernel()); 955 #ifdef UVMHIST 956 struct kern_history * const histp = 957 ((prot & VM_PROT_EXECUTE) ? &pmapexechist : &pmaphist); 958 #endif 959 960 UVMHIST_FUNC(__func__); 961 #define VM_PROT_STRING(prot) \ 962 &"\0 (R)\0 (W)\0 (RW)\0 (X)\0 (RX)\0 (WX)\0 (RWX)\0"[UVM_PROTECTION(prot)*6] 963 UVMHIST_CALLED(*histp); 964 UVMHIST_LOG(*histp, "(pmap=%p, va=%#"PRIxVADDR", pa=%#"PRIxPADDR, 965 pmap, va, pa, 0); 966 UVMHIST_LOG(*histp, "prot=%#x%s flags=%#x%s)", 967 prot, VM_PROT_STRING(prot), flags, VM_PROT_STRING(flags)); 968 969 const bool good_color = PMAP_PAGE_COLOROK_P(pa, va); 970 if (is_kernel_pmap_p) { 971 PMAP_COUNT(kernel_mappings); 972 if (!good_color) 973 PMAP_COUNT(kernel_mappings_bad); 974 } else { 975 PMAP_COUNT(user_mappings); 976 if (!good_color) 977 PMAP_COUNT(user_mappings_bad); 978 } 979 #if defined(DEBUG) || defined(DIAGNOSTIC) || defined(PARANOIADIAG) 980 if (va < pmap->pm_minaddr || va >= pmap->pm_maxaddr) 981 panic("%s: %s %#"PRIxVADDR" too big", 982 __func__, is_kernel_pmap_p ? "kva" : "uva", va); 983 #endif 984 985 KASSERTMSG(prot & VM_PROT_READ, 986 "%s: no READ (%#x) in prot %#x", __func__, VM_PROT_READ, prot); 987 988 struct vm_page * const pg = PHYS_TO_VM_PAGE(pa); 989 struct vm_page_md *mdpg; 990 991 if (pg) { 992 mdpg = VM_PAGE_TO_MD(pg); 993 /* Set page referenced/modified status based on flags */ 994 if (flags & VM_PROT_WRITE) 995 pmap_page_set_attributes(mdpg, VM_PAGEMD_MODIFIED|VM_PAGEMD_REFERENCED); 996 else if (flags & VM_PROT_ALL) 997 pmap_page_set_attributes(mdpg, VM_PAGEMD_REFERENCED); 998 999 #ifdef __PMAP_VIRTUAL_CACHE_ALIASES 1000 if (!VM_PAGEMD_CACHED(pg)) 1001 flags |= PMAP_NOCACHE; 1002 #endif 1003 1004 PMAP_COUNT(managed_mappings); 1005 } else { 1006 /* 1007 * Assumption: if it is not part of our managed memory 1008 * then it must be device memory which may be volatile. 1009 */ 1010 mdpg = NULL; 1011 flags |= PMAP_NOCACHE; 1012 PMAP_COUNT(unmanaged_mappings); 1013 } 1014 1015 npte = pte_make_enter(pa, mdpg, prot, flags, is_kernel_pmap_p); 1016 1017 kpreempt_disable(); 1018 pt_entry_t * const ptep = pmap_pte_reserve(pmap, va, flags); 1019 if (__predict_false(ptep == NULL)) { 1020 kpreempt_enable(); 1021 UVMHIST_LOG(*histp, "<- ENOMEM", 0,0,0,0); 1022 return ENOMEM; 1023 } 1024 pt_entry_t opte = *ptep; 1025 1026 /* Done after case that may sleep/return. */ 1027 if (pg) 1028 pmap_enter_pv(pmap, va, pg, &npte); 1029 1030 /* 1031 * Now validate mapping with desired protection/wiring. 1032 * Assume uniform modified and referenced status for all 1033 * MIPS pages in a MACH page. 1034 */ 1035 if (wired) { 1036 pmap->pm_stats.wired_count++; 1037 npte = pte_wire_entry(npte); 1038 } 1039 1040 UVMHIST_LOG(*histp, "new pte %#x (pa %#"PRIxPADDR")", npte, pa, 0,0); 1041 1042 if (pte_valid_p(opte) && pte_to_paddr(opte) != pa) { 1043 pmap_remove(pmap, va, va + NBPG); 1044 PMAP_COUNT(user_mappings_changed); 1045 } 1046 1047 KASSERT(pte_valid_p(npte)); 1048 bool resident = pte_valid_p(opte); 1049 if (!resident) 1050 pmap->pm_stats.resident_count++; 1051 *ptep = npte; 1052 1053 pmap_tlb_update_addr(pmap, va, npte, 1054 ((flags & VM_PROT_ALL) ? PMAP_TLB_INSERT : 0) 1055 | (resident ? PMAP_TLB_NEED_IPI : 0)); 1056 kpreempt_enable(); 1057 1058 if (pg != NULL && (prot == (VM_PROT_READ | VM_PROT_EXECUTE))) { 1059 KASSERT(mdpg != NULL); 1060 PMAP_COUNT(exec_mappings); 1061 if (!VM_PAGEMD_EXECPAGE_P(mdpg) && pte_cached_p(npte)) { 1062 if (!pte_deferred_exec_p(npte)) { 1063 UVMHIST_LOG(*histp, 1064 "va=%#"PRIxVADDR" pg %p: %s syncicache%s", 1065 va, pg, "immediate", ""); 1066 pmap_page_syncicache(pg); 1067 pmap_page_set_attributes(mdpg, 1068 VM_PAGEMD_EXECPAGE); 1069 PMAP_COUNT(exec_synced_mappings); 1070 } else { 1071 UVMHIST_LOG(*histp, "va=%#"PRIxVADDR 1072 " pg %p: %s syncicache: pte %#x", 1073 va, pg, "defer", npte); 1074 } 1075 } else { 1076 UVMHIST_LOG(*histp, 1077 "va=%#"PRIxVADDR" pg %p: %s syncicache%s", 1078 va, pg, "no", 1079 (pte_cached_p(npte) 1080 ? " (already exec)" 1081 : " (uncached)")); 1082 } 1083 } else if (pg != NULL && (prot & VM_PROT_EXECUTE)) { 1084 KASSERT(mdpg != NULL); 1085 KASSERT(prot & VM_PROT_WRITE); 1086 PMAP_COUNT(exec_mappings); 1087 pmap_page_syncicache(pg); 1088 pmap_page_clear_attributes(mdpg, VM_PAGEMD_EXECPAGE); 1089 UVMHIST_LOG(pmapexechist, 1090 "va=%#"PRIxVADDR" pg %p: %s syncicache%s", 1091 va, pg, "immediate", " (writeable)"); 1092 } 1093 1094 if (prot & VM_PROT_EXECUTE) { 1095 UVMHIST_LOG(pmapexechist, "<- 0 (OK)", 0,0,0,0); 1096 } else { 1097 UVMHIST_LOG(pmaphist, "<- 0 (OK)", 0,0,0,0); 1098 } 1099 return 0; 1100 } 1101 1102 void 1103 pmap_kenter_pa(vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags) 1104 { 1105 struct vm_page * const pg = PHYS_TO_VM_PAGE(pa); 1106 struct vm_page_md *mdpg; 1107 1108 UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist); 1109 UVMHIST_LOG(pmaphist, "(va=%#"PRIxVADDR" pa=%#"PRIxPADDR 1110 ", prot=%#x, flags=%#x)", va, pa, prot, flags); 1111 PMAP_COUNT(kenter_pa); 1112 1113 if (pg == NULL) { 1114 mdpg = NULL; 1115 PMAP_COUNT(kenter_pa_unmanaged); 1116 flags |= PMAP_NOCACHE; 1117 } else { 1118 mdpg = VM_PAGE_TO_MD(pg); 1119 } 1120 1121 if ((flags & PMAP_NOCACHE) == 0 && !PMAP_PAGE_COLOROK_P(pa, va)) 1122 PMAP_COUNT(kenter_pa_bad); 1123 1124 const pt_entry_t npte = pte_make_kenter_pa(pa, mdpg, prot, flags); 1125 kpreempt_disable(); 1126 pt_entry_t * const ptep = pmap_pte_reserve(pmap_kernel(), va, 0); 1127 KASSERT(ptep != NULL); 1128 KASSERT(!pte_valid_p(*ptep)); 1129 *ptep = npte; 1130 /* 1131 * We have the option to force this mapping into the TLB but we 1132 * don't. Instead let the next reference to the page do it. 1133 */ 1134 pmap_tlb_update_addr(pmap_kernel(), va, npte, 0); 1135 kpreempt_enable(); 1136 #if DEBUG > 1 1137 for (u_int i = 0; i < PAGE_SIZE / sizeof(long); i++) { 1138 if (((long *)va)[i] != ((long *)pa)[i]) 1139 panic("%s: contents (%lx) of va %#"PRIxVADDR 1140 " != contents (%lx) of pa %#"PRIxPADDR, __func__, 1141 ((long *)va)[i], va, ((long *)pa)[i], pa); 1142 } 1143 #endif 1144 UVMHIST_LOG(pmaphist, "<- done", 0,0,0,0); 1145 } 1146 1147 static bool 1148 pmap_pte_kremove(pmap_t pmap, vaddr_t sva, vaddr_t eva, pt_entry_t *ptep, 1149 uintptr_t flags) 1150 { 1151 const pt_entry_t new_pt_entry = pte_nv_entry(true); 1152 1153 KASSERT(kpreempt_disabled()); 1154 1155 /* 1156 * Set every pt on every valid mapping within this segment. 1157 */ 1158 for (; sva < eva; sva += NBPG, ptep++) { 1159 pt_entry_t pt_entry = *ptep; 1160 if (!pte_valid_p(pt_entry)) { 1161 continue; 1162 } 1163 1164 PMAP_COUNT(kremove_pages); 1165 struct vm_page * const pg = 1166 PHYS_TO_VM_PAGE(pte_to_paddr(pt_entry)); 1167 if (pg != NULL) 1168 pmap_md_vca_clean(pg, sva, PMAP_WBINV); 1169 1170 *ptep = new_pt_entry; 1171 pmap_tlb_invalidate_addr(pmap_kernel(), sva); 1172 } 1173 1174 return false; 1175 } 1176 1177 void 1178 pmap_kremove(vaddr_t va, vsize_t len) 1179 { 1180 const vaddr_t sva = trunc_page(va); 1181 const vaddr_t eva = round_page(va + len); 1182 1183 UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist); 1184 UVMHIST_LOG(pmaphist, "(va=%#"PRIxVADDR" len=%#"PRIxVSIZE")", 1185 va, len, 0,0); 1186 1187 kpreempt_disable(); 1188 pmap_pte_process(pmap_kernel(), sva, eva, pmap_pte_kremove, 0); 1189 kpreempt_enable(); 1190 1191 UVMHIST_LOG(pmaphist, "<- done", 0,0,0,0); 1192 } 1193 1194 void 1195 pmap_remove_all(struct pmap *pmap) 1196 { 1197 KASSERT(pmap != pmap_kernel()); 1198 1199 kpreempt_disable(); 1200 /* 1201 * Free all of our ASIDs which means we can skip doing all the 1202 * tlb_invalidate_addrs(). 1203 */ 1204 pmap_tlb_asid_deactivate(pmap); 1205 pmap_tlb_asid_release_all(pmap); 1206 pmap->pm_flags |= PMAP_DEFERRED_ACTIVATE; 1207 1208 kpreempt_enable(); 1209 } 1210 1211 /* 1212 * Routine: pmap_unwire 1213 * Function: Clear the wired attribute for a map/virtual-address 1214 * pair. 1215 * In/out conditions: 1216 * The mapping must already exist in the pmap. 1217 */ 1218 void 1219 pmap_unwire(pmap_t pmap, vaddr_t va) 1220 { 1221 1222 UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist); 1223 UVMHIST_LOG(pmaphist, "(pmap=%p va=%#"PRIxVADDR")", pmap, va, 0,0); 1224 PMAP_COUNT(unwire); 1225 1226 /* 1227 * Don't need to flush the TLB since PG_WIRED is only in software. 1228 */ 1229 #ifdef PARANOIADIAG 1230 if (va < pmap->pm_minaddr || pmap->pm_maxaddr <= va) 1231 panic("pmap_unwire"); 1232 #endif 1233 kpreempt_disable(); 1234 pt_entry_t * const ptep = pmap_pte_lookup(pmap, va); 1235 pt_entry_t pt_entry = *ptep; 1236 #ifdef DIAGNOSTIC 1237 if (ptep == NULL) 1238 panic("%s: pmap %p va %#"PRIxVADDR" invalid STE", 1239 __func__, pmap, va); 1240 #endif 1241 1242 #ifdef DIAGNOSTIC 1243 if (!pte_valid_p(pt_entry)) 1244 panic("pmap_unwire: pmap %p va %#"PRIxVADDR" invalid PTE", 1245 pmap, va); 1246 #endif 1247 1248 if (pte_wired_p(pt_entry)) { 1249 *ptep = pte_unwire_entry(*ptep); 1250 pmap->pm_stats.wired_count--; 1251 } 1252 #ifdef DIAGNOSTIC 1253 else { 1254 printf("%s: wiring for pmap %p va %#"PRIxVADDR" unchanged!\n", 1255 __func__, pmap, va); 1256 } 1257 #endif 1258 kpreempt_enable(); 1259 } 1260 1261 /* 1262 * Routine: pmap_extract 1263 * Function: 1264 * Extract the physical page address associated 1265 * with the given map/virtual_address pair. 1266 */ 1267 bool 1268 pmap_extract(pmap_t pmap, vaddr_t va, paddr_t *pap) 1269 { 1270 paddr_t pa; 1271 1272 //UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist); 1273 //UVMHIST_LOG(pmaphist, "(pmap=%p va=%#"PRIxVADDR")", pmap, va, 0,0); 1274 if (pmap == pmap_kernel()) { 1275 if (pmap_md_direct_mapped_vaddr_p(va)) { 1276 pa = pmap_md_direct_mapped_vaddr_to_paddr(va); 1277 goto done; 1278 } 1279 if (pmap_md_io_vaddr_p(va)) 1280 panic("pmap_extract: io address %#"PRIxVADDR"", va); 1281 } 1282 kpreempt_disable(); 1283 pt_entry_t * const ptep = pmap_pte_lookup(pmap, va); 1284 if (ptep == NULL) { 1285 //UVMHIST_LOG(pmaphist, "<- false (not in segmap)", 0,0,0,0); 1286 kpreempt_enable(); 1287 return false; 1288 } 1289 if (!pte_valid_p(*ptep)) { 1290 //UVMHIST_LOG(pmaphist, "<- false (PTE not valid)", 0,0,0,0); 1291 kpreempt_enable(); 1292 return false; 1293 } 1294 pa = pte_to_paddr(*ptep) | (va & PGOFSET); 1295 kpreempt_enable(); 1296 done: 1297 if (pap != NULL) { 1298 *pap = pa; 1299 } 1300 //UVMHIST_LOG(pmaphist, "<- true (pa %#"PRIxPADDR")", pa, 0,0,0); 1301 return true; 1302 } 1303 1304 /* 1305 * Copy the range specified by src_addr/len 1306 * from the source map to the range dst_addr/len 1307 * in the destination map. 1308 * 1309 * This routine is only advisory and need not do anything. 1310 */ 1311 void 1312 pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vaddr_t dst_addr, vsize_t len, 1313 vaddr_t src_addr) 1314 { 1315 1316 UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist); 1317 PMAP_COUNT(copy); 1318 } 1319 1320 /* 1321 * pmap_clear_reference: 1322 * 1323 * Clear the reference bit on the specified physical page. 1324 */ 1325 bool 1326 pmap_clear_reference(struct vm_page *pg) 1327 { 1328 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 1329 1330 UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist); 1331 UVMHIST_LOG(pmaphist, "(pg=%p (pa %#"PRIxPADDR"))", 1332 pg, VM_PAGE_TO_PHYS(pg), 0,0); 1333 1334 bool rv = pmap_page_clear_attributes(mdpg, VM_PAGEMD_REFERENCED); 1335 1336 UVMHIST_LOG(pmaphist, "<- %s", rv ? "true" : "false", 0,0,0); 1337 1338 return rv; 1339 } 1340 1341 /* 1342 * pmap_is_referenced: 1343 * 1344 * Return whether or not the specified physical page is referenced 1345 * by any physical maps. 1346 */ 1347 bool 1348 pmap_is_referenced(struct vm_page *pg) 1349 { 1350 1351 return VM_PAGEMD_REFERENCED_P(VM_PAGE_TO_MD(pg)); 1352 } 1353 1354 /* 1355 * Clear the modify bits on the specified physical page. 1356 */ 1357 bool 1358 pmap_clear_modify(struct vm_page *pg) 1359 { 1360 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 1361 pv_entry_t pv = &mdpg->mdpg_first; 1362 pv_entry_t pv_next; 1363 uint16_t gen; 1364 1365 UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist); 1366 UVMHIST_LOG(pmaphist, "(pg=%p (%#"PRIxPADDR"))", 1367 pg, VM_PAGE_TO_PHYS(pg), 0,0); 1368 PMAP_COUNT(clear_modify); 1369 1370 if (VM_PAGEMD_EXECPAGE_P(mdpg)) { 1371 if (pv->pv_pmap == NULL) { 1372 UVMHIST_LOG(pmapexechist, 1373 "pg %p (pa %#"PRIxPADDR"): %s", 1374 pg, VM_PAGE_TO_PHYS(pg), "execpage cleared", 0); 1375 pmap_page_clear_attributes(mdpg, VM_PAGEMD_EXECPAGE); 1376 PMAP_COUNT(exec_uncached_clear_modify); 1377 } else { 1378 UVMHIST_LOG(pmapexechist, 1379 "pg %p (pa %#"PRIxPADDR"): %s", 1380 pg, VM_PAGE_TO_PHYS(pg), "syncicache performed", 0); 1381 pmap_page_syncicache(pg); 1382 PMAP_COUNT(exec_synced_clear_modify); 1383 } 1384 } 1385 if (!pmap_page_clear_attributes(mdpg, VM_PAGEMD_MODIFIED)) { 1386 UVMHIST_LOG(pmaphist, "<- false", 0,0,0,0); 1387 return false; 1388 } 1389 if (pv->pv_pmap == NULL) { 1390 UVMHIST_LOG(pmaphist, "<- true (no mappings)", 0,0,0,0); 1391 return true; 1392 } 1393 1394 /* 1395 * remove write access from any pages that are dirty 1396 * so we can tell if they are written to again later. 1397 * flush the VAC first if there is one. 1398 */ 1399 kpreempt_disable(); 1400 gen = VM_PAGEMD_PVLIST_LOCK(mdpg, false); 1401 for (; pv != NULL; pv = pv_next) { 1402 pmap_t pmap = pv->pv_pmap; 1403 vaddr_t va = pv->pv_va; 1404 pt_entry_t * const ptep = pmap_pte_lookup(pmap, va); 1405 KASSERT(ptep); 1406 pv_next = pv->pv_next; 1407 pt_entry_t pt_entry = pte_prot_nowrite(*ptep); 1408 if (*ptep == pt_entry) { 1409 continue; 1410 } 1411 pmap_md_vca_clean(pg, va, PMAP_WBINV); 1412 *ptep = pt_entry; 1413 VM_PAGEMD_PVLIST_UNLOCK(mdpg); 1414 pmap_tlb_invalidate_addr(pmap, va); 1415 pmap_update(pmap); 1416 if (__predict_false(gen != VM_PAGEMD_PVLIST_LOCK(mdpg, false))) { 1417 /* 1418 * The list changed! So restart from the beginning. 1419 */ 1420 pv_next = &mdpg->mdpg_first; 1421 } 1422 } 1423 VM_PAGEMD_PVLIST_UNLOCK(mdpg); 1424 kpreempt_enable(); 1425 1426 UVMHIST_LOG(pmaphist, "<- true (mappings changed)", 0,0,0,0); 1427 return true; 1428 } 1429 1430 /* 1431 * pmap_is_modified: 1432 * 1433 * Return whether or not the specified physical page is modified 1434 * by any physical maps. 1435 */ 1436 bool 1437 pmap_is_modified(struct vm_page *pg) 1438 { 1439 1440 return VM_PAGEMD_MODIFIED_P(VM_PAGE_TO_MD(pg)); 1441 } 1442 1443 /* 1444 * pmap_set_modified: 1445 * 1446 * Sets the page modified reference bit for the specified page. 1447 */ 1448 void 1449 pmap_set_modified(paddr_t pa) 1450 { 1451 struct vm_page * const pg = PHYS_TO_VM_PAGE(pa); 1452 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 1453 pmap_page_set_attributes(mdpg, VM_PAGEMD_MODIFIED|VM_PAGEMD_REFERENCED); 1454 } 1455 1456 /******************** pv_entry management ********************/ 1457 1458 static void 1459 pmap_check_pvlist(struct vm_page *pg) 1460 { 1461 #ifdef PARANOIADIAG 1462 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 1463 pt_entry_t pv = &mdpg->mdpg_first; 1464 if (pv->pv_pmap != NULL) { 1465 for (; pv != NULL; pv = pv->pv_next) { 1466 KASSERT(!pmap_md_direct_mapped_vaddr_p(pv->pv_va)); 1467 } 1468 } 1469 #endif /* PARANOIADIAG */ 1470 } 1471 1472 /* 1473 * Enter the pmap and virtual address into the 1474 * physical to virtual map table. 1475 */ 1476 void 1477 pmap_enter_pv(pmap_t pmap, vaddr_t va, struct vm_page *pg, u_int *npte) 1478 { 1479 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 1480 pv_entry_t pv, npv, apv; 1481 int16_t gen; 1482 bool first __unused = false; 1483 1484 UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist); 1485 UVMHIST_LOG(pmaphist, 1486 "(pmap=%p va=%#"PRIxVADDR" pg=%p (%#"PRIxPADDR")", 1487 pmap, va, pg, VM_PAGE_TO_PHYS(pg)); 1488 UVMHIST_LOG(pmaphist, "nptep=%p (%#x))", npte, *npte, 0, 0); 1489 1490 KASSERT(kpreempt_disabled()); 1491 KASSERT(pmap != pmap_kernel() || !pmap_md_direct_mapped_vaddr_p(va)); 1492 1493 apv = NULL; 1494 pv = &mdpg->mdpg_first; 1495 gen = VM_PAGEMD_PVLIST_LOCK(mdpg, true); 1496 pmap_check_pvlist(pg); 1497 again: 1498 if (pv->pv_pmap == NULL) { 1499 KASSERT(pv->pv_next == NULL); 1500 /* 1501 * No entries yet, use header as the first entry 1502 */ 1503 PMAP_COUNT(primary_mappings); 1504 PMAP_COUNT(mappings); 1505 first = true; 1506 #ifdef __PMAP_VIRTUAL_CACHE_ALIASES 1507 pmap_page_clear_attributes(pg, VM_PAGEMD_UNCACHED); 1508 #endif 1509 pv->pv_pmap = pmap; 1510 pv->pv_va = va; 1511 } else { 1512 if (pmap_md_vca_add(pg, va, npte)) 1513 goto again; 1514 1515 /* 1516 * There is at least one other VA mapping this page. 1517 * Place this entry after the header. 1518 * 1519 * Note: the entry may already be in the table if 1520 * we are only changing the protection bits. 1521 */ 1522 1523 #ifdef PARANOIADIAG 1524 const paddr_t pa = VM_PAGE_TO_PHYS(pg); 1525 #endif 1526 for (npv = pv; npv; npv = npv->pv_next) { 1527 if (pmap == npv->pv_pmap && va == npv->pv_va) { 1528 #ifdef PARANOIADIAG 1529 pt_entry_t *ptep = pmap_pte_lookup(pmap, va); 1530 pt_entry_t pt_entry = (ptep ? *ptep : 0); 1531 if (!pte_valid_p(pt_entry) 1532 || pte_to_paddr(pt_entry) != pa) 1533 printf( 1534 "pmap_enter_pv: found va %#"PRIxVADDR" pa %#"PRIxPADDR" in pv_table but != %x\n", 1535 va, pa, pt_entry); 1536 #endif 1537 PMAP_COUNT(remappings); 1538 VM_PAGEMD_PVLIST_UNLOCK(mdpg); 1539 if (__predict_false(apv != NULL)) 1540 pmap_pv_free(apv); 1541 return; 1542 } 1543 } 1544 if (__predict_true(apv == NULL)) { 1545 /* 1546 * To allocate a PV, we have to release the PVLIST lock 1547 * so get the page generation. We allocate the PV, and 1548 * then reacquire the lock. 1549 */ 1550 VM_PAGEMD_PVLIST_UNLOCK(mdpg); 1551 1552 apv = (pv_entry_t)pmap_pv_alloc(); 1553 if (apv == NULL) 1554 panic("pmap_enter_pv: pmap_pv_alloc() failed"); 1555 1556 /* 1557 * If the generation has changed, then someone else 1558 * tinkered with this page so we should 1559 * start over. 1560 */ 1561 uint16_t oldgen = gen; 1562 gen = VM_PAGEMD_PVLIST_LOCK(mdpg, true); 1563 if (gen != oldgen) 1564 goto again; 1565 } 1566 npv = apv; 1567 apv = NULL; 1568 npv->pv_va = va; 1569 npv->pv_pmap = pmap; 1570 npv->pv_next = pv->pv_next; 1571 pv->pv_next = npv; 1572 PMAP_COUNT(mappings); 1573 } 1574 pmap_check_pvlist(pg); 1575 VM_PAGEMD_PVLIST_UNLOCK(mdpg); 1576 if (__predict_false(apv != NULL)) 1577 pmap_pv_free(apv); 1578 1579 UVMHIST_LOG(pmaphist, "<- done pv=%p%s", 1580 pv, first ? " (first pv)" : "",0,0); 1581 } 1582 1583 /* 1584 * Remove a physical to virtual address translation. 1585 * If cache was inhibited on this page, and there are no more cache 1586 * conflicts, restore caching. 1587 * Flush the cache if the last page is removed (should always be cached 1588 * at this point). 1589 */ 1590 void 1591 pmap_remove_pv(pmap_t pmap, vaddr_t va, struct vm_page *pg, bool dirty) 1592 { 1593 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 1594 pv_entry_t pv, npv; 1595 bool last; 1596 1597 UVMHIST_FUNC(__func__); UVMHIST_CALLED(pmaphist); 1598 UVMHIST_LOG(pmaphist, 1599 "(pmap=%p va=%#"PRIxVADDR" pg=%p (pa %#"PRIxPADDR")\n", 1600 pmap, va, pg, VM_PAGE_TO_PHYS(pg)); 1601 UVMHIST_LOG(pmaphist, "dirty=%s)", dirty ? "true" : "false", 0,0,0); 1602 1603 KASSERT(kpreempt_disabled()); 1604 pv = &mdpg->mdpg_first; 1605 1606 (void)VM_PAGEMD_PVLIST_LOCK(mdpg, true); 1607 pmap_check_pvlist(pg); 1608 1609 /* 1610 * If it is the first entry on the list, it is actually 1611 * in the header and we must copy the following entry up 1612 * to the header. Otherwise we must search the list for 1613 * the entry. In either case we free the now unused entry. 1614 */ 1615 1616 last = false; 1617 if (pmap == pv->pv_pmap && va == pv->pv_va) { 1618 npv = pv->pv_next; 1619 if (npv) { 1620 *pv = *npv; 1621 KASSERT(pv->pv_pmap != NULL); 1622 } else { 1623 #ifdef __PMAP_VIRTUAL_CACHE_ALIASES 1624 pmap_page_clear_attributes(pg, VM_PAGEMD_UNCACHED); 1625 #endif 1626 pv->pv_pmap = NULL; 1627 last = true; /* Last mapping removed */ 1628 } 1629 PMAP_COUNT(remove_pvfirst); 1630 } else { 1631 for (npv = pv->pv_next; npv; pv = npv, npv = npv->pv_next) { 1632 PMAP_COUNT(remove_pvsearch); 1633 if (pmap == npv->pv_pmap && va == npv->pv_va) 1634 break; 1635 } 1636 if (npv) { 1637 pv->pv_next = npv->pv_next; 1638 } 1639 } 1640 pmap_md_vca_remove(pg, va); 1641 1642 pmap_check_pvlist(pg); 1643 VM_PAGEMD_PVLIST_UNLOCK(mdpg); 1644 1645 /* 1646 * Free the pv_entry if needed. 1647 */ 1648 if (npv) 1649 pmap_pv_free(npv); 1650 if (VM_PAGEMD_EXECPAGE_P(mdpg) && dirty) { 1651 if (last) { 1652 /* 1653 * If this was the page's last mapping, we no longer 1654 * care about its execness. 1655 */ 1656 UVMHIST_LOG(pmapexechist, 1657 "pg %p (pa %#"PRIxPADDR")%s: %s", 1658 pg, VM_PAGE_TO_PHYS(pg), 1659 last ? " [last mapping]" : "", 1660 "execpage cleared"); 1661 pmap_page_clear_attributes(mdpg, VM_PAGEMD_EXECPAGE); 1662 PMAP_COUNT(exec_uncached_remove); 1663 } else { 1664 /* 1665 * Someone still has it mapped as an executable page 1666 * so we must sync it. 1667 */ 1668 UVMHIST_LOG(pmapexechist, 1669 "pg %p (pa %#"PRIxPADDR")%s: %s", 1670 pg, VM_PAGE_TO_PHYS(pg), 1671 last ? " [last mapping]" : "", 1672 "performed syncicache"); 1673 pmap_page_syncicache(pg); 1674 PMAP_COUNT(exec_synced_remove); 1675 } 1676 } 1677 UVMHIST_LOG(pmaphist, "<- done", 0,0,0,0); 1678 } 1679 1680 #if defined(MULTIPROCESSOR) 1681 struct pmap_pvlist_info { 1682 kmutex_t *pli_locks[PAGE_SIZE / 32]; 1683 volatile u_int pli_lock_refs[PAGE_SIZE / 32]; 1684 volatile u_int pli_lock_index; 1685 u_int pli_lock_mask; 1686 } pmap_pvlist_info; 1687 1688 void 1689 pmap_pvlist_lock_init(size_t cache_line_size) 1690 { 1691 struct pmap_pvlist_info * const pli = &pmap_pvlist_info; 1692 const vaddr_t lock_page = uvm_pageboot_alloc(PAGE_SIZE); 1693 vaddr_t lock_va = lock_page; 1694 if (sizeof(kmutex_t) > cache_line_size) { 1695 cache_line_size = roundup2(sizeof(kmutex_t), cache_line_size); 1696 } 1697 const size_t nlocks = PAGE_SIZE / cache_line_size; 1698 KASSERT((nlocks & (nlocks - 1)) == 0); 1699 /* 1700 * Now divide the page into a number of mutexes, one per cacheline. 1701 */ 1702 for (size_t i = 0; i < nlocks; lock_va += cache_line_size, i++) { 1703 kmutex_t * const lock = (kmutex_t *)lock_va; 1704 mutex_init(lock, MUTEX_DEFAULT, IPL_VM); 1705 pli->pli_locks[i] = lock; 1706 } 1707 pli->pli_lock_mask = nlocks - 1; 1708 } 1709 1710 uint16_t 1711 pmap_pvlist_lock(struct vm_page_md *mdpg, bool list_change) 1712 { 1713 struct pmap_pvlist_info * const pli = &pmap_pvlist_info; 1714 kmutex_t *lock = mdpg->mdpg_lock; 1715 int16_t gen; 1716 1717 /* 1718 * Allocate a lock on an as-needed basis. This will hopefully give us 1719 * semi-random distribution not based on page color. 1720 */ 1721 if (__predict_false(lock == NULL)) { 1722 size_t locknum = atomic_add_int_nv(&pli->pli_lock_index, 37); 1723 size_t lockid = locknum & pli->pli_lock_mask; 1724 kmutex_t * const new_lock = pli->pli_locks[lockid]; 1725 /* 1726 * Set the lock. If some other thread already did, just use 1727 * the one they assigned. 1728 */ 1729 lock = atomic_cas_ptr(&mdpg->mdpg_lock, NULL, new_lock); 1730 if (lock == NULL) { 1731 lock = new_lock; 1732 atomic_inc_uint(&pli->pli_lock_refs[lockid]); 1733 } 1734 } 1735 1736 /* 1737 * Now finally lock the pvlists. 1738 */ 1739 mutex_spin_enter(lock); 1740 1741 /* 1742 * If the locker will be changing the list, increment the high 16 bits 1743 * of attrs so we use that as a generation number. 1744 */ 1745 gen = VM_PAGEMD_PVLIST_GEN(mdpg); /* get old value */ 1746 if (list_change) 1747 atomic_add_int(&mdpg->mdpg_attrs, 0x10000); 1748 1749 /* 1750 * Return the generation number. 1751 */ 1752 return gen; 1753 } 1754 #else /* !MULTIPROCESSOR */ 1755 void 1756 pmap_pvlist_lock_init(size_t cache_line_size) 1757 { 1758 mutex_init(&pmap_pvlist_mutex, MUTEX_DEFAULT, IPL_VM); 1759 } 1760 1761 #ifdef MODULAR 1762 uint16_t 1763 pmap_pvlist_lock(struct vm_page_md *mdpg, bool list_change) 1764 { 1765 /* 1766 * We just use a global lock. 1767 */ 1768 if (__predict_false(mdpg->mdpg_lock == NULL)) { 1769 mdpg->mdpg_lock = &pmap_pvlist_mutex; 1770 } 1771 1772 /* 1773 * Now finally lock the pvlists. 1774 */ 1775 mutex_spin_enter(mdpg->mdpg_lock); 1776 1777 return 0; 1778 } 1779 #endif /* MODULAR */ 1780 #endif /* !MULTIPROCESSOR */ 1781 1782 /* 1783 * pmap_pv_page_alloc: 1784 * 1785 * Allocate a page for the pv_entry pool. 1786 */ 1787 void * 1788 pmap_pv_page_alloc(struct pool *pp, int flags) 1789 { 1790 struct vm_page *pg = PMAP_ALLOC_POOLPAGE(UVM_PGA_USERESERVE); 1791 if (pg == NULL) 1792 return NULL; 1793 1794 return (void *)pmap_map_poolpage(VM_PAGE_TO_PHYS(pg)); 1795 } 1796 1797 /* 1798 * pmap_pv_page_free: 1799 * 1800 * Free a pv_entry pool page. 1801 */ 1802 void 1803 pmap_pv_page_free(struct pool *pp, void *v) 1804 { 1805 vaddr_t va = (vaddr_t)v; 1806 1807 KASSERT(pmap_md_direct_mapped_vaddr_p(va)); 1808 const paddr_t pa = pmap_md_direct_mapped_vaddr_to_paddr(va); 1809 struct vm_page * const pg = PHYS_TO_VM_PAGE(pa); 1810 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 1811 pmap_md_vca_remove(pg, va); 1812 pmap_page_clear_attributes(mdpg, VM_PAGEMD_POOLPAGE); 1813 uvm_pagefree(pg); 1814 } 1815 1816 #ifdef PMAP_PREFER 1817 /* 1818 * Find first virtual address >= *vap that doesn't cause 1819 * a cache alias conflict. 1820 */ 1821 void 1822 pmap_prefer(vaddr_t foff, vaddr_t *vap, vsize_t sz, int td) 1823 { 1824 vaddr_t va; 1825 vsize_t d; 1826 vsize_t prefer_mask = ptoa(uvmexp.colormask); 1827 1828 PMAP_COUNT(prefer_requests); 1829 1830 prefer_mask |= pmap_md_cache_prefer_mask(); 1831 1832 if (prefer_mask) { 1833 va = *vap; 1834 1835 d = foff - va; 1836 d &= prefer_mask; 1837 if (d) { 1838 if (td) 1839 *vap = trunc_page(va -((-d) & prefer_mask)); 1840 else 1841 *vap = round_page(va + d); 1842 PMAP_COUNT(prefer_adjustments); 1843 } 1844 } 1845 } 1846 #endif /* PMAP_PREFER */ 1847 1848 #ifdef PMAP_MAP_POOLPAGE 1849 vaddr_t 1850 pmap_map_poolpage(paddr_t pa) 1851 { 1852 1853 struct vm_page * const pg = PHYS_TO_VM_PAGE(pa); 1854 KASSERT(pg); 1855 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 1856 pmap_page_set_attributes(mdpg, VM_PAGEMD_POOLPAGE); 1857 1858 const vaddr_t va = pmap_md_map_poolpage(pa, NBPG); 1859 pmap_md_vca_add(pg, va, NULL); 1860 return va; 1861 } 1862 1863 paddr_t 1864 pmap_unmap_poolpage(vaddr_t va) 1865 { 1866 1867 KASSERT(pmap_md_direct_mapped_vaddr_p(va)); 1868 paddr_t pa = pmap_md_direct_mapped_vaddr_to_paddr(va); 1869 1870 struct vm_page * const pg = PHYS_TO_VM_PAGE(pa); 1871 KASSERT(pg); 1872 struct vm_page_md * const mdpg = VM_PAGE_TO_MD(pg); 1873 pmap_page_clear_attributes(mdpg, VM_PAGEMD_POOLPAGE); 1874 pmap_md_unmap_poolpage(va, NBPG); 1875 pmap_md_vca_remove(pg, va); 1876 1877 return pa; 1878 } 1879 #endif /* PMAP_MAP_POOLPAGE */ 1880