Lines Matching +full:virtual +full:- +full:wire +full:- +full:mode
1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
88 #include <powerpc/pseries/phyp-hvcall.h>
94 #define PPC_BITLSHIFT(bit) (sizeof(long)*NBBY - 1 - (bit))
115 #define NLS_MASK ((1UL<<5)-1)
117 #define RPTE_MASK (RPTE_ENTRIES-1)
120 #define NLB_MASK (((1UL<<52)-1) << 8)
129 #define POWER9_TLB_SETS_RADIX 128 /* # sets in POWER9 TLB Radix mode */
185 #define TLBIE_RIC_INVALIDATE_SEQ 0x3 /* HPT - only:
367 return (&pmap->pm_pml1[pmap_pml1e_index(va)]);
609 static int pmap_change_attr_locked(vm_offset_t va, vm_size_t size, int mode, bool flush);
646 #define RADIX_PGD_INDEX_SHIFT (RADIX_PGD_SIZE_SHIFT-3)
726 * bits 52 - 31 = 21, 0b10101
728 * bits 0 - 3 of rts -> bits 6 - 8 unsigned long
729 * bits 4 - 5 of rts -> bits 62 - 63 of unsigned long
799 * In HV mode, we init AMOR (Authority Mask Override Register) so that
823 mtspr(SPR_PID, pmap->pm_pid);
834 return (-1);
848 * NB: should only be used for adding directories - not for direct mappings
872 pmap->pm_stats.resident_count += count;
880 KASSERT(pmap->pm_stats.resident_count >= count,
882 pmap->pm_stats.resident_count, count));
883 pmap->pm_stats.resident_count -= count;
930 return (superpages_enabled && (pmap->pm_flags & PMAP_PDE_SUPERPAGE) != 0);
976 printf("%s(%p, %#lx, %#x) (%#lx) -- invalidate all\n",
1040 if ((m->oflags & VPO_UNMANAGED) != 0)
1044 rv = !TAILQ_EMPTY(&m->md.pv_list) ||
1045 ((m->flags & PG_FICTITIOUS) == 0 &&
1046 !TAILQ_EMPTY(&pa_to_pvh(VM_PAGE_TO_PHYS(m))->pv_list));
1053 * caching mode.
1080 radix_tlbie_invlpg_user_4k(pmap->pm_pid, start);
1091 radix_tlbie_invlpg_user_2m(pmap->pm_pid, start);
1102 radix_tlbie_invlpwc_user(pmap->pm_pid);
1109 if (((start - end) >> PAGE_SHIFT) > 8) {
1121 radix_tlbie_invlpg_user_4k(pmap->pm_pid, start);
1135 radix_tlbie_flush_user(pmap->pm_pid);
1156 pmap_invalidate_range(pmap, va, va + L3_PAGE_SIZE - 1);
1170 #define PV_PMAP(pv) (pv_to_chunk(pv)->pc_pmap)
1173 #define PC_FREE1 ((1ul << (_NPCPV % 64)) - 1)
1204 TAILQ_FOREACH(pc, &pmap->pm_pvchunk, pc_list) {
1206 bit_count((bitstr_t *)pc->pc_map, 0,
1207 sizeof(pc->pc_map) * NBBY, &free);
1209 free = popcnt_pc_map_pq(pc->pc_map);
1227 dump_add_page(m->phys_addr);
1228 pc = (void *)PHYS_TO_DMAP(m->phys_addr);
1229 pc->pc_pmap = pmap;
1230 pc->pc_map[0] = PC_FREE0;
1231 pc->pc_map[1] = PC_FREE1;
1232 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);
1239 * re-count the number of available entries.
1252 * First find and then remove the pv entry for the specified pmap and virtual
1262 TAILQ_FOREACH(pv, &pvh->pv_list, pv_link) {
1269 MPASS(pv->pv_va != 0);
1271 if (pmap == PV_PMAP(pv) && va == pv->pv_va) {
1272 TAILQ_REMOVE(&pvh->pv_list, pv, pv_link);
1273 pvh->pv_gen++;
1311 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_link);
1313 m->md.pv_gen++;
1314 /* Instantiate the remaining NPTEPG - 1 pv entries. */
1315 PV_STAT(atomic_add_long(&pv_entry_allocs, NPTEPG - 1));
1316 va_last = va + L3_PAGE_SIZE - PAGE_SIZE;
1318 pc = TAILQ_FIRST(&pmap->pm_pvchunk);
1319 KASSERT(pc->pc_map[0] != 0 || pc->pc_map[1] != 0
1322 while (pc->pc_map[field]) {
1323 bit = cnttzd(pc->pc_map[field]);
1324 pc->pc_map[field] &= ~(1ul << bit);
1325 pv = &pc->pc_pventry[field * 64 + bit];
1327 pv->pv_va = va;
1329 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
1331 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_link);
1333 m->md.pv_gen++;
1338 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
1339 TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list);
1342 if (pc->pc_map[0] == 0 && pc->pc_map[1] == 0) {
1343 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
1344 TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list);
1346 PV_STAT(atomic_add_long(&pv_entry_count, NPTEPG - 1));
1347 PV_STAT(atomic_subtract_int(&pv_entry_spare, NPTEPG - 1));
1369 * allocate per-page pv entries until repromotion occurs, thereby
1405 next_pmap = pc->pc_pmap;
1440 pc->pc_pmap != next_pmap)
1448 * Destroy every non-wired, 4 KB page mapping in the chunk.
1452 for (inuse = ~pc->pc_map[field] & pc_freemask[field];
1455 pv = &pc->pc_pventry[field * 64 + bit];
1456 va = pv->pv_va;
1470 TAILQ_REMOVE(&m->md.pv_list, pv, pv_link);
1472 m->md.pv_gen++;
1473 if (TAILQ_EMPTY(&m->md.pv_list) &&
1474 (m->flags & PG_FICTITIOUS) == 0) {
1476 if (TAILQ_EMPTY(&pvh->pv_list)) {
1481 pc->pc_map[field] |= 1UL << bit;
1495 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
1496 if (pc->pc_map[0] == PC_FREE0 && pc->pc_map[1] == PC_FREE1) {
1502 dump_drop_page(m_pc->phys_addr);
1507 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);
1524 MPASS(pc->pc_pmap != NULL);
1532 active_reclaims--;
1539 m_pc->ref_count = 1;
1563 idx = pv - &pc->pc_pventry[0];
1566 pc->pc_map[field] |= 1ul << bit;
1567 if (pc->pc_map[0] != PC_FREE0 || pc->pc_map[1] != PC_FREE1) {
1569 if (__predict_false(pc != TAILQ_FIRST(&pmap->pm_pvchunk))) {
1570 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
1571 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);
1575 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
1592 dump_drop_page(m->phys_addr);
1616 pc = TAILQ_FIRST(&pmap->pm_pvchunk);
1619 if (pc->pc_map[field]) {
1620 bit = cnttzd(pc->pc_map[field]);
1625 pv = &pc->pc_pventry[field * 64 + bit];
1626 pc->pc_map[field] &= ~(1ul << bit);
1628 if (pc->pc_map[0] == 0 && pc->pc_map[1] == 0) {
1629 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
1630 TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc,
1652 dump_add_page(m->phys_addr);
1653 pc = (void *)PHYS_TO_DMAP(m->phys_addr);
1654 pc->pc_pmap = pmap;
1655 pc->pc_map[0] = PC_FREE0 & ~1ul; /* preallocated bit 0 */
1656 pc->pc_map[1] = PC_FREE1;
1660 pv = &pc->pc_pventry[0];
1661 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);
1663 PV_STAT(atomic_add_int(&pv_entry_spare, _NPCPV - 1));
1696 pv = pmap_pvh_remove(&m->md, pmap, va);
1699 TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_link);
1700 pvh->pv_gen++;
1701 /* Free the remaining NPTEPG - 1 pv entries. */
1702 va_last = va + L3_PAGE_SIZE - PAGE_SIZE;
1706 pmap_pvh_free(&m->md, pmap, va);
1712 * First find and then destroy the pv entry for the specified pmap and virtual
1739 pv->pv_va = va;
1741 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_link);
1742 m->md.pv_gen++;
1763 KASSERT(found, ("%#lx-%#lx outside of initial phys_avail array",
1788 printf("%s %lx -> %lx\n", __func__, start, end);
1798 end - start >= L2_PAGE_SIZE) {
1808 end - start >= L3_PAGE_SIZE) {
1847 vm_radix_init(&kernel_pmap->pm_radix);
1854 MPASS((l1phys & (RADIX_PGD_SIZE-1)) == 0);
1857 kernel_pmap->pm_pml1 = (pml1_entry_t *)PHYS_TO_DMAP(l1phys);
1880 printf("phys_avail[%d]=%08lx - phys_avail[%d]=%08lx\n",
1888 printf("kernel_pmap pml1 %p\n", kernel_pmap->pm_pml1);
1893 * are required for promotion of the corresponding kernel virtual
1953 (hwphyssz - physsz);
2016 phys_avail_count -= rm_pavail;
2024 printf("phys_avail[%d]=%08lx - phys_avail[%d]=%08lx\n",
2029 /* XXX assume we're running non-virtualized and
2061 * mode.
2096 * Allocate virtual address space for the message buffer.
2102 * Allocate virtual address space for the dynamic percpu area.
2126 ptcr = parttab_phys | (PARTTAB_SIZE_SHIFT-12);
2162 pagetab = RTS_SIZE | DMAP_TO_PHYS((vm_offset_t)kernel_pmap->pm_pml1) | \
2184 isa3_proctab->proctab0 =
2185 htobe64(RTS_SIZE | DMAP_TO_PHYS((vm_offset_t)kernel_pmap->pm_pml1) |
2189 mmu_radix_proctab_register(proctab0pa, PROCTAB_SIZE_SHIFT - 12);
2200 proctab0pa, 0, PROCTAB_SIZE_SHIFT - 12);
2210 isa3_proctab, kernel_pmap->pm_pml1);
2213 kernel_pmap->pm_pid = isa3_base_pid;
2283 va -= PAGE_SIZE;
2337 * Routines used in machine-dependent code
2386 ptcr = parttab_phys | (PARTTAB_SIZE_SHIFT-12);
2436 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
2446 if ((m->a.flags & PGA_WRITEABLE) == 0)
2448 pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy :
2453 TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_link, next_pv) {
2456 pvh_gen = pvh->pv_gen;
2460 if (pvh_gen != pvh->pv_gen) {
2465 va = pv->pv_va;
2476 va += VM_PAGE_TO_PHYS(m) - (oldl3e &
2489 TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) {
2492 md_gen = m->md.pv_gen;
2493 pvh_gen = pvh->pv_gen;
2497 if (pvh_gen != pvh->pv_gen || md_gen != m->md.pv_gen) {
2502 l3e = pmap_pml3e(pmap, pv->pv_va);
2505 pte = pmap_l3e_to_pte(l3e, pv->pv_va);
2508 pmap_invalidate_page(pmap, pv->pv_va);
2592 dst_pdpg->ref_count--;
2598 KASSERT(srcmpte->ref_count > 0,
2611 * we only virtual copy managed pages
2615 dstmpte->pindex == pmap_l3e_pindex(addr))
2616 dstmpte->ref_count++;
2641 * mapped, paging-structure
2653 if (dstmpte->ref_count >= srcmpte->ref_count)
2698 cnt = min(xfersize, PAGE_SIZE - a_pg_offset);
2703 cnt = min(cnt, PAGE_SIZE - b_pg_offset);
2710 xfersize -= cnt;
2760 pa = (newpde & (PG_PS_FRAME | PG_A | PG_V)) + L3_PAGE_SIZE - PAGE_SIZE;
2761 for (pte = firstpte + NPTEPG - 1; pte > firstpte; pte--) {
2786 pa -= PAGE_SIZE;
2798 KASSERT(mpte->pindex == pmap_l3e_pindex(va),
2845 KASSERT((m->oflags & VPO_UNMANAGED) != 0 || !VA_IS_CLEANMAP(va),
2847 if ((m->oflags & VPO_UNMANAGED) == 0)
2871 newpte |= pmap_cache_bits(m->md.mdpg_cache_attrs);
2877 if ((m->oflags & VPO_UNMANAGED) != 0) {
2886 /* Assert the required virtual and physical alignment. */
2888 KASSERT(m->psind > 0, ("pmap_enter: m->psind < psind"));
2905 mpte->ref_count++;
2930 * Is the specified virtual address already mapped?
2935 printf("cow fault pmap_enter(%p, %#lx, %p, %#x, %x, %d) --"
2937 pmap, va, m, prot, flags, psind, pmap->pm_pid,
2938 curproc->p_pid, curproc->p_comm, origpte);
2940 pmap_pte_walk(pmap->pm_pml1, va);
2951 pmap->pm_stats.wired_count++;
2953 pmap->pm_stats.wired_count--;
2959 mpte->ref_count--;
2960 KASSERT(mpte->ref_count > 0,
3019 pv = pmap_pvh_remove(&om->md, pmap, va);
3032 if ((om->a.flags & PGA_WRITEABLE) != 0 &&
3033 TAILQ_EMPTY(&om->md.pv_list) &&
3034 ((om->flags & PG_FICTITIOUS) != 0 ||
3035 TAILQ_EMPTY(&pa_to_pvh(opa)->pv_list)))
3045 printf("pmap_enter(%p, %#lx, %p, %#x, %x, %d) -- asid=%lu curpid=%d name=%s\n",
3047 pmap->pm_pid, curproc->p_pid,
3048 curproc->p_comm);
3056 pmap->pm_stats.wired_count++;
3066 pv->pv_va = va;
3074 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_link);
3075 m->md.pv_gen++;
3119 if ((mpte == NULL || mpte->ref_count == NPTEPG) &&
3121 (m->flags & PG_FICTITIOUS) == 0 &&
3152 * Although "va" is not mapped, paging-
3163 * Tries to create a read- and/or execute-only 2MB page mapping. Returns true
3165 * without sleeping, (2) a mapping already exists at the specified virtual
3176 newpde = VM_PAGE_TO_PHYS(m) | pmap_cache_bits(m->md.mdpg_cache_attrs) |
3178 if ((m->oflags & VPO_UNMANAGED) == 0)
3195 * a mapping already exists at the specified virtual address. Returns
3225 KASSERT(pdpg->ref_count > 1,
3226 ("pmap_enter_pde: pdpg's wire count is too low"));
3228 pdpg->ref_count--;
3261 KASSERT(be64toh(*l3e) == 0, ("pmap_enter_pde: non-zero pde %p",
3275 uwptpg->pindex = pmap_l3e_pindex(va);
3283 uwptpg->ref_count = NPTEPG;
3299 uwptpg->ref_count = 1;
3317 pmap->pm_stats.wired_count += L3_PAGE_SIZE / PAGE_SIZE;
3343 VM_OBJECT_ASSERT_LOCKED(m_start->object);
3349 psize = atop(end - start);
3354 while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) {
3357 m->psind == 1 && mmu_radix_ps_enabled(pmap) &&
3359 m = &m[L3_PAGE_SIZE / PAGE_SIZE - 1];
3382 (m->oflags & VPO_UNMANAGED) != 0,
3398 if (mpte && (mpte->pindex == ptepindex)) {
3399 mpte->ref_count++;
3416 mpte->ref_count++;
3435 mpte->ref_count--;
3444 if ((m->oflags & VPO_UNMANAGED) == 0 &&
3450 * Although "va" is not mapped, paging-
3468 pa = VM_PAGE_TO_PHYS(m) | pmap_cache_bits(m->md.mdpg_cache_attrs);
3473 if ((m->oflags & VPO_UNMANAGED) == 0)
3576 if (addr - 1 >= vm_map_max(kernel_map))
3586 nkpg->pindex = kernel_vm_end >> L2_PAGE_SIZE_SHIFT;
3594 if (kernel_vm_end - 1 >= vm_map_max(kernel_map)) {
3605 nkpg->pindex = pmap_l3e_pindex(kernel_vm_end);
3610 if (kernel_vm_end - 1 >= vm_map_max(kernel_map)) {
3631 0, (vm_paddr_t)-1, RADIX_PGD_SIZE, L1_PAGE_SIZE,
3654 for (int j = page_count-1; j >= 0; j--) {
3694 mpte->pindex = pmap_l3e_pindex(VM_MIN_KERNEL_ADDRESS) + i;
3695 mpte->phys_addr = KPTphys + (i << PAGE_SHIFT);
3696 MPASS(PHYS_TO_VM_PAGE(mpte->phys_addr) == mpte);
3698 mpte->ref_count = 1;
3730 pv_npg = howmany(vm_phys_segs[vm_phys_nsegs - 1].end, L3_PAGE_SIZE);
3768 TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) {
3771 md_gen = m->md.pv_gen;
3775 if (md_gen != m->md.pv_gen) {
3780 pte = pmap_pte(pmap, pv->pv_va);
3791 if ((m->flags & PG_FICTITIOUS) == 0) {
3793 TAILQ_FOREACH(pv, &pvh->pv_list, pv_link) {
3796 md_gen = m->md.pv_gen;
3797 pvh_gen = pvh->pv_gen;
3801 if (md_gen != m->md.pv_gen ||
3802 pvh_gen != pvh->pv_gen) {
3807 pte = pmap_pml3e(pmap, pv->pv_va);
3834 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
3868 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
3908 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
3914 pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy : pa_to_pvh(pa);
3918 if ((pvf = TAILQ_FIRST(&pvh->pv_list)) == NULL)
3926 pvh_gen = pvh->pv_gen;
3930 if (pvh_gen != pvh->pv_gen) {
3935 l3e = pmap_pml3e(pmap, pv->pv_va);
3950 * physical page number, the virtual superpage number,
3964 if ((((pa >> PAGE_SHIFT) ^ (pv->pv_va >> L3_PAGE_SIZE_SHIFT) ^
3965 (uintptr_t)pmap) & (NPTEPG - 1)) == 0 &&
3968 pmap_invalidate_page(pmap, pv->pv_va);
3979 TAILQ_REMOVE(&pvh->pv_list, pv, pv_link);
3980 TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_link);
3981 pvh->pv_gen++;
3985 } while ((pv = TAILQ_FIRST(&pvh->pv_list)) != pvf);
3987 if ((pvf = TAILQ_FIRST(&m->md.pv_list)) == NULL)
3995 pvh_gen = pvh->pv_gen;
3996 md_gen = m->md.pv_gen;
4000 if (pvh_gen != pvh->pv_gen || md_gen != m->md.pv_gen) {
4005 l3e = pmap_pml3e(pmap, pv->pv_va);
4009 pte = pmap_l3e_to_pte(l3e, pv->pv_va);
4014 pmap_invalidate_page(pmap, pv->pv_va);
4020 TAILQ_REMOVE(&m->md.pv_list, pv, pv_link);
4021 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_link);
4022 m->md.pv_gen++;
4024 } while ((pv = TAILQ_FIRST(&m->md.pv_list)) != pvf && cleared +
4054 KASSERT(object->type == OBJT_DEVICE || object->type == OBJT_SG,
4055 ("pmap_object_init_pt: non-device object"));
4063 KASSERT(p->valid == VM_PAGE_BITS_ALL,
4065 ma = p->md.mdpg_cache_attrs;
4083 KASSERT(p->valid == VM_PAGE_BITS_ALL,
4086 ma != p->md.mdpg_cache_attrs)
4115 pdpg->ref_count--;
4116 KASSERT(pdpg->ref_count > 0,
4136 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
4142 TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) {
4151 if (!rv && loops < 16 && (m->flags & PG_FICTITIOUS) == 0) {
4153 TAILQ_FOREACH(pv, &pvh->pv_list, pv_link) {
4172 TAILQ_INIT(&m->md.pv_list);
4173 m->md.mdpg_cache_attrs = VM_MEMATTR_DEFAULT;
4186 if ((m->oflags & VPO_UNMANAGED) != 0)
4193 TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) {
4196 md_gen = m->md.pv_gen;
4200 if (md_gen != m->md.pv_gen) {
4205 pte = pmap_pte(pmap, pv->pv_va);
4210 if ((m->flags & PG_FICTITIOUS) == 0) {
4212 TAILQ_FOREACH(pv, &pvh->pv_list, pv_link) {
4215 md_gen = m->md.pv_gen;
4216 pvh_gen = pvh->pv_gen;
4220 if (md_gen != m->md.pv_gen ||
4221 pvh_gen != pvh->pv_gen) {
4226 pte = pmap_pml3e(pmap, pv->pv_va);
4253 pmap->pm_pml1 = uma_zalloc(zone_radix_pgd, M_WAITOK);
4256 pagezero((vm_offset_t)pmap->pm_pml1 + j * PAGE_SIZE);
4257 vm_radix_init(&pmap->pm_radix);
4258 TAILQ_INIT(&pmap->pm_pvchunk);
4259 bzero(&pmap->pm_stats, sizeof pmap->pm_stats);
4260 pmap->pm_flags = PMAP_PDE_SUPERPAGE;
4263 pmap->pm_pid = pid;
4264 l1pa = DMAP_TO_PHYS((vm_offset_t)pmap->pm_pml1);
4305 m->pindex = ptepindex;
4316 /* Wire up a new PDPE page */
4317 pml1index = ptepindex - (NUPDE + NUPDPE);
4318 l1e = &pmap->pm_pml1[pml1index];
4328 /* Wire up a new l2e page */
4329 pdpindex = ptepindex - NUPDE;
4332 l1e = &pmap->pm_pml1[pml1index];
4344 pdppg->ref_count++;
4360 /* Wire up a new PTE page */
4365 l1e = &pmap->pm_pml1[pml1index];
4390 pdpg->ref_count++;
4417 pdpg->ref_count++;
4466 m->ref_count++;
4485 pmap->pm_pml1 = kernel_pmap->pm_pml1;
4486 pmap->pm_pid = kernel_pmap->pm_pid;
4488 vm_radix_init(&pmap->pm_radix);
4489 TAILQ_INIT(&pmap->pm_pvchunk);
4490 bzero(&pmap->pm_stats, sizeof pmap->pm_stats);
4491 kernel_pmap->pm_flags =
4492 pmap->pm_flags = PMAP_PDE_SUPERPAGE;
4563 printf("pmap_protect(%p, %#lx, %#lx, %x) - asid: %lu\n",
4564 pmap, sva, eva, prot, pmap->pm_pid);
4652 printf("%#lx %#lx -> %#lx\n",
4685 * the page table every time - but go for correctness for
4690 cache_bits = pmap_cache_bits(m->md.mdpg_cache_attrs);
4740 m->flags |= PG_ZERO;
4742 m->flags &= ~PG_ZERO;
4749 * for mapping a distinct range of virtual addresses. The pmap's collection is
4750 * ordered by this virtual address range.
4757 return (vm_radix_insert(&pmap->pm_radix, mpte));
4761 * Removes the page table page mapping the specified virtual address from the
4764 * specified virtual address.
4771 return (vm_radix_remove(&pmap->pm_radix, pmap_l3e_pindex(va)));
4775 * Decrements a page table page's wire count, which is used to record the
4776 * number of valid page table entries within the page. If the wire count
4784 --m->ref_count;
4785 if (m->ref_count == 0) {
4800 if (m->pindex >= NUPDE + NUPDPE) {
4805 } else if (m->pindex >= NUPDE) {
4817 if (m->pindex < NUPDE) {
4824 else if (m->pindex >= NUPDE && m->pindex < (NUPDE + NUPDPE)) {
4841 * conditionally free the page, and manage the hold/wire counts.
4861 KASSERT(pmap->pm_stats.resident_count == 0,
4863 pmap->pm_stats.resident_count));
4864 KASSERT(vm_radix_is_empty(&pmap->pm_radix),
4868 isa3_proctab[pmap->pm_pid].proctab0 = 0;
4869 uma_zfree(zone_radix_pgd, pmap->pm_pml1);
4870 vmem_free(asid_arena, pmap->pm_pid, 1);
4891 pv->pv_va = va;
4895 TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_link);
4896 pvh->pv_gen++;
4973 mpte->pindex = pmap_l3e_pindex(va);
4987 if (mpte->ref_count == 1) {
4988 mpte->ref_count = NPTEPG;
5012 reserve_pv_entries(pmap, NPTEPG - 1, lockp);
5081 pmap->pm_stats.wired_count -= (L3_PAGE_SIZE / PAGE_SIZE);
5094 if (TAILQ_EMPTY(&m->md.pv_list) &&
5095 TAILQ_EMPTY(&pvh->pv_list))
5105 KASSERT(mpte->ref_count == NPTEPG,
5106 ("pmap_remove_l3e: pte page wire count error"));
5107 mpte->ref_count = 0;
5128 pmap->pm_stats.wired_count -= 1;
5137 pmap_pvh_free(&m->md, pmap, va);
5138 if (TAILQ_EMPTY(&m->md.pv_list) &&
5139 (m->flags & PG_FICTITIOUS) == 0) {
5141 if (TAILQ_EMPTY(&pvh->pv_list))
5232 if (pmap->pm_stats.resident_count == 0)
5259 if (pmap->pm_stats.resident_count == 0)
5347 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
5351 pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy :
5355 while ((pv = TAILQ_FIRST(&pvh->pv_list)) != NULL) {
5358 pvh_gen = pvh->pv_gen;
5362 if (pvh_gen != pvh->pv_gen) {
5368 va = pv->pv_va;
5373 while ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) {
5376 pvh_gen = pvh->pv_gen;
5377 md_gen = m->md.pv_gen;
5381 if (pvh_gen != pvh->pv_gen || md_gen != m->md.pv_gen) {
5388 l3e = pmap_pml3e(pmap, pv->pv_va);
5391 pte = pmap_l3e_to_pte(l3e, pv->pv_va);
5394 pmap->pm_stats.wired_count--;
5403 pmap_unuse_pt(pmap, pv->pv_va, be64toh(*l3e), &free);
5404 pmap_invalidate_page(pmap, pv->pv_va);
5405 TAILQ_REMOVE(&m->md.pv_list, pv, pv_link);
5406 m->md.pv_gen++;
5416 * Destroy all managed, non-wired mappings in the given user-space
5432 * non-wired mappings, it can delay and batch the invalidation of TLB
5468 KASSERT(pmap->pm_pid == mfspr(SPR_PID),
5469 ("non-current asid %lu - expected %lu", pmap->pm_pid,
5476 TAILQ_FOREACH_SAFE(pc, &pmap->pm_pvchunk, pc_list, npc) {
5482 inuse = ~pc->pc_map[field] & pc_freemask[field];
5487 pv = &pc->pc_pventry[idx];
5490 pte = pmap_pml2e(pmap, pv->pv_va);
5492 pte = pmap_l2e_to_l3e(pte, pv->pv_va);
5499 pte = &pte[pmap_pte_index(pv->pv_va)];
5517 pv->pv_va, tpte);
5534 KASSERT(m->phys_addr == pa,
5536 m, (uintmax_t)m->phys_addr,
5539 KASSERT((m->flags & PG_FICTITIOUS) != 0 ||
5560 pc->pc_map[field] |= bitmask;
5564 TAILQ_REMOVE(&pvh->pv_list, pv, pv_link);
5565 pvh->pv_gen++;
5566 if (TAILQ_EMPTY(&pvh->pv_list)) {
5568 if ((mt->a.flags & PGA_WRITEABLE) != 0 &&
5569 TAILQ_EMPTY(&mt->md.pv_list))
5572 mpte = pmap_remove_pt_page(pmap, pv->pv_va);
5575 KASSERT(mpte->ref_count == NPTEPG,
5576 ("pmap_remove_pages: pte page wire count error"));
5577 mpte->ref_count = 0;
5586 TAILQ_REMOVE(&m->md.pv_list, pv, pv_link);
5587 m->md.pv_gen++;
5588 if ((m->a.flags & PGA_WRITEABLE) != 0 &&
5589 TAILQ_EMPTY(&m->md.pv_list) &&
5590 (m->flags & PG_FICTITIOUS) == 0) {
5592 if (TAILQ_EMPTY(&pvh->pv_list))
5596 pmap_unuse_pt(pmap, pv->pv_va, ptel3e, &free);
5606 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
5629 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
5636 pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy :
5640 TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_link, next_pv) {
5643 pvh_gen = pvh->pv_gen;
5647 if (pvh_gen != pvh->pv_gen) {
5653 l3e = pmap_pml3e(pmap, pv->pv_va);
5655 (void)pmap_demote_l3e_locked(pmap, l3e, pv->pv_va, &lock);
5661 TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) {
5664 pvh_gen = pvh->pv_gen;
5665 md_gen = m->md.pv_gen;
5669 if (pvh_gen != pvh->pv_gen ||
5670 md_gen != m->md.pv_gen) {
5676 l3e = pmap_pml3e(pmap, pv->pv_va);
5680 pte = pmap_l3e_to_pte(l3e, pv->pv_va);
5689 pmap_invalidate_page(pmap, pv->pv_va);
5752 pmap->pm_stats.wired_count -= L3_PAGE_SIZE /
5775 pmap->pm_stats.wired_count--;
5856 pmap = vmspace_pmap(td->td_proc->p_vmspace);
5858 if (pmap->pm_pid > isa3_base_pid &&
5859 curpid != pmap->pm_pid) {
5866 * Increase the starting virtual address of the given mapping if a
5880 if (object != NULL && (object->flags & OBJ_COLORED) != 0)
5881 offset += ptoa(object->pg_color);
5883 if (size - ((L3_PAGE_SIZE - superpage_offset) & L3_PAGE_MASK) < L3_PAGE_SIZE ||
5909 panic("%s: Couldn't alloc kernel virtual memory", __func__);
5913 size -= PAGE_SIZE;
5936 m->md.mdpg_cache_attrs = ma;
5943 if ((m->flags & PG_FICTITIOUS) == 0 &&
5945 PAGE_SIZE, m->md.mdpg_cache_attrs))
5978 pm = &curthread->td_proc->p_vmspace->vm_pmap;
5982 sync_sz = PAGE_SIZE - (va & PAGE_MASK);
5989 sz -= sync_sz;
5999 * The cache mode bits are all in the low 32-bits of the
6000 * PTE, so we can just spin on updating the low 32-bits.
6030 pdpg->pindex = va >> L2_PAGE_SIZE_SHIFT;
6053 * Flush PWC --- XXX revisit
6073 /* Big-endian PTE on stack */
6194 cpu_flush_dcache((void *)sva, eva - sva);
6199 vm_memattr_t mode)
6203 CTR4(KTR_PMAP, "%s(%#x, %#zx, %d)", __func__, va, size, mode);
6205 error = pmap_change_attr_locked(va, size, mode, true);
6211 pmap_change_attr_locked(vm_offset_t va, vm_size_t size, int mode, bool flush)
6227 * Only supported on kernel virtual addresses, including the direct
6233 cache_bits = pmap_cache_bits(mode);
6306 * cache mode if required.
6329 pa_end - pa_start, mode, flush);
6359 pa_end - pa_start, mode, flush);
6387 pa_end - pa_start, mode, flush);
6402 pa_end1 - pa_start, mode, flush);
6420 * attempting to back the vm_pages with domain-local memory.
6441 pa = vm_phys_early_alloc(0, end - start);
6443 start = mmu_radix_map(&start, pa, end - start, VM_MEMATTR_DEFAULT);
6447 pfn = first_page + (va - start) / sizeof(struct vm_page);
6509 db_printf("page %p(%lx)\n", m, m->phys_addr);
6511 TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) {
6513 db_printf("va: %#016lx ", pv->pv_va);
6517 db_printf("asid: %lu\n", pmap->pm_pid);
6518 pmap_pte_walk(pmap->pm_pml1, pv->pv_va);
6537 pmap = vmspace_pmap(kdb_thread->td_proc->p_vmspace);
6539 pmap = vmspace_pmap(curthread->td_proc->p_vmspace);
6541 pmap_pte_walk(pmap->pm_pml1, va);