Lines Matching +full:4 +full:kb +full:- +full:page
1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
88 #include <powerpc/pseries/phyp-hvcall.h>
94 #define PPC_BITLSHIFT(bit) (sizeof(long)*NBBY - 1 - (bit))
115 #define NLS_MASK ((1UL<<5)-1)
117 #define RPTE_MASK (RPTE_ENTRIES-1)
120 #define NLB_MASK (((1UL<<52)-1) << 8)
166 #define TLBIEL_INVAL_PAGE 0x000 /* invalidate a single page */
185 #define TLBIE_RIC_INVALIDATE_SEQ 0x3 /* HPT - only:
367 return (&pmap->pm_pml1[pmap_pml1e_index(va)]);
405 "Number of kernel page table pages allocated on bootup");
627 #define PROCTAB_SIZE_SHIFT (isa3_pid_bits + 4)
646 #define RADIX_PGD_INDEX_SHIFT (RADIX_PGD_SIZE_SHIFT-3)
726 * bits 52 - 31 = 21, 0b10101
728 * bits 0 - 3 of rts -> bits 6 - 8 unsigned long
729 * bits 4 - 5 of rts -> bits 62 - 63 of unsigned long
758 * Flush the first set of the TLB, and the entire Page Walk Cache
823 mtspr(SPR_PID, pmap->pm_pid);
834 return (-1);
848 * NB: should only be used for adding directories - not for direct mappings
861 * Promotion to a 2MB (PDE) page mapping requires that the corresponding 4KB
862 * (PTE) page mappings have identical settings for the following fields:
872 pmap->pm_stats.resident_count += count;
880 KASSERT(pmap->pm_stats.resident_count >= count,
882 pmap->pm_stats.resident_count, count));
883 pmap->pm_stats.resident_count -= count;
930 return (superpages_enabled && (pmap->pm_flags & PMAP_PDE_SUPERPAGE) != 0);
976 printf("%s(%p, %#lx, %#x) (%#lx) -- invalidate all\n",
1031 * Returns true if the given page is mapped individually or as part of
1040 if ((m->oflags & VPO_UNMANAGED) != 0)
1044 rv = !TAILQ_EMPTY(&m->md.pv_list) ||
1045 ((m->flags & PG_FICTITIOUS) == 0 &&
1046 !TAILQ_EMPTY(&pa_to_pvh(VM_PAGE_TO_PHYS(m))->pv_list));
1080 radix_tlbie_invlpg_user_4k(pmap->pm_pid, start);
1091 radix_tlbie_invlpg_user_2m(pmap->pm_pid, start);
1102 radix_tlbie_invlpwc_user(pmap->pm_pid);
1109 if (((start - end) >> PAGE_SHIFT) > 8) {
1121 radix_tlbie_invlpg_user_4k(pmap->pm_pid, start);
1135 radix_tlbie_flush_user(pmap->pm_pid);
1144 * When the PDE has PG_PROMOTED set, the 2MB page mapping was created
1145 * by a promotion that did not invalidate the 512 4KB page mappings
1147 * may hold both 4KB and 2MB page mappings for the address range [va,
1150 * 4KB page mappings for the address range [va, va + L3_PAGE_SIZE), and so a
1151 * single INVLPG suffices to invalidate the 2MB page mapping from the
1156 pmap_invalidate_range(pmap, va, va + L3_PAGE_SIZE - 1);
1170 #define PV_PMAP(pv) (pv_to_chunk(pv)->pc_pmap)
1173 #define PC_FREE1 ((1ul << (_NPCPV % 64)) - 1)
1204 TAILQ_FOREACH(pc, &pmap->pm_pvchunk, pc_list) {
1206 bit_count((bitstr_t *)pc->pc_map, 0,
1207 sizeof(pc->pc_map) * NBBY, &free);
1209 free = popcnt_pc_map_pq(pc->pc_map);
1227 dump_add_page(m->phys_addr);
1228 pc = (void *)PHYS_TO_DMAP(m->phys_addr);
1229 pc->pc_pmap = pmap;
1230 pc->pc_map[0] = PC_FREE0;
1231 pc->pc_map[1] = PC_FREE1;
1232 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);
1239 * re-count the number of available entries.
1254 * otherwise. This operation can be performed on pv lists for either 4KB or
1255 * 2MB page mappings.
1262 TAILQ_FOREACH(pv, &pvh->pv_list, pv_link) {
1269 MPASS(pv->pv_va != 0);
1271 if (pmap == PV_PMAP(pv) && va == pv->pv_va) {
1272 TAILQ_REMOVE(&pvh->pv_list, pv, pv_link);
1273 pvh->pv_gen++;
1281 * After demotion from a 2MB page mapping to 512 4KB page mappings,
1282 * destroy the pv entry for the 2MB page mapping and reinstantiate the pv
1283 * entries for each of the 4KB page mappings.
1303 * page's pv list. Once this transfer begins, the pv list lock
1311 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_link);
1313 m->md.pv_gen++;
1314 /* Instantiate the remaining NPTEPG - 1 pv entries. */
1315 PV_STAT(atomic_add_long(&pv_entry_allocs, NPTEPG - 1));
1316 va_last = va + L3_PAGE_SIZE - PAGE_SIZE;
1318 pc = TAILQ_FIRST(&pmap->pm_pvchunk);
1319 KASSERT(pc->pc_map[0] != 0 || pc->pc_map[1] != 0
1322 while (pc->pc_map[field]) {
1323 bit = cnttzd(pc->pc_map[field]);
1324 pc->pc_map[field] &= ~(1ul << bit);
1325 pv = &pc->pc_pventry[field * 64 + bit];
1327 pv->pv_va = va;
1329 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
1330 ("pmap_pv_demote_pde: page %p is not managed", m));
1331 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_link);
1333 m->md.pv_gen++;
1338 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
1339 TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list);
1342 if (pc->pc_map[0] == 0 && pc->pc_map[1] == 0) {
1343 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
1344 TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list);
1346 PV_STAT(atomic_add_long(&pv_entry_count, NPTEPG - 1));
1347 PV_STAT(atomic_subtract_int(&pv_entry_spare, NPTEPG - 1));
1369 * allocate per-page pv entries until repromotion occurs, thereby
1405 next_pmap = pc->pc_pmap;
1440 pc->pc_pmap != next_pmap)
1448 * Destroy every non-wired, 4 KB page mapping in the chunk.
1452 for (inuse = ~pc->pc_map[field] & pc_freemask[field];
1455 pv = &pc->pc_pventry[field * 64 + bit];
1456 va = pv->pv_va;
1470 TAILQ_REMOVE(&m->md.pv_list, pv, pv_link);
1472 m->md.pv_gen++;
1473 if (TAILQ_EMPTY(&m->md.pv_list) &&
1474 (m->flags & PG_FICTITIOUS) == 0) {
1476 if (TAILQ_EMPTY(&pvh->pv_list)) {
1481 pc->pc_map[field] |= 1UL << bit;
1490 /* Every freed mapping is for a 4 KB page. */
1495 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
1496 if (pc->pc_map[0] == PC_FREE0 && pc->pc_map[1] == PC_FREE1) {
1502 dump_drop_page(m_pc->phys_addr);
1507 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);
1524 MPASS(pc->pc_pmap != NULL);
1532 active_reclaims--;
1538 /* Recycle a freed page table page. */
1539 m_pc->ref_count = 1;
1563 idx = pv - &pc->pc_pventry[0];
1566 pc->pc_map[field] |= 1ul << bit;
1567 if (pc->pc_map[0] != PC_FREE0 || pc->pc_map[1] != PC_FREE1) {
1569 if (__predict_false(pc != TAILQ_FIRST(&pmap->pm_pvchunk))) {
1570 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
1571 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);
1575 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
1592 dump_drop_page(m->phys_addr);
1616 pc = TAILQ_FIRST(&pmap->pm_pvchunk);
1619 if (pc->pc_map[field]) {
1620 bit = cnttzd(pc->pc_map[field]);
1625 pv = &pc->pc_pventry[field * 64 + bit];
1626 pc->pc_map[field] &= ~(1ul << bit);
1628 if (pc->pc_map[0] == 0 && pc->pc_map[1] == 0) {
1629 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
1630 TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc,
1652 dump_add_page(m->phys_addr);
1653 pc = (void *)PHYS_TO_DMAP(m->phys_addr);
1654 pc->pc_pmap = pmap;
1655 pc->pc_map[0] = PC_FREE0 & ~1ul; /* preallocated bit 0 */
1656 pc->pc_map[1] = PC_FREE1;
1660 pv = &pc->pc_pventry[0];
1661 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list);
1663 PV_STAT(atomic_add_int(&pv_entry_spare, _NPCPV - 1));
1670 * After promotion from 512 4KB page mappings to a single 2MB page mapping,
1671 * replace the many pv entries for the 4KB page mappings by a single pv entry
1672 * for the 2MB page mapping.
1688 * Transfer the first page's pv entry for this mapping to the 2mpage's
1696 pv = pmap_pvh_remove(&m->md, pmap, va);
1699 TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_link);
1700 pvh->pv_gen++;
1701 /* Free the remaining NPTEPG - 1 pv entries. */
1702 va_last = va + L3_PAGE_SIZE - PAGE_SIZE;
1706 pmap_pvh_free(&m->md, pmap, va);
1713 * address. This operation can be performed on pv lists for either 4KB or 2MB
1714 * page mappings.
1727 * Conditionally create the PV entry for a 4KB page mapping if the required
1739 pv->pv_va = va;
1741 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_link);
1742 m->md.pv_gen++;
1763 KASSERT(found, ("%#lx-%#lx outside of initial phys_avail array",
1774 vm_paddr_t page;
1776 page = allocpages(1);
1777 pagezero(PHYS_TO_DMAP(page));
1778 return (page);
1785 vm_paddr_t page;
1788 printf("%s %lx -> %lx\n", __func__, start, end);
1793 page = alloc_pt_page();
1794 pde_store(pte, page);
1798 end - start >= L2_PAGE_SIZE) {
1802 page = alloc_pt_page();
1803 pde_store(pte, page);
1808 end - start >= L3_PAGE_SIZE) {
1812 page = alloc_pt_page();
1813 pde_store(pte, page);
1847 vm_radix_init(&kernel_pmap->pm_radix);
1854 MPASS((l1phys & (RADIX_PGD_SIZE-1)) == 0);
1857 kernel_pmap->pm_pml1 = (pml1_entry_t *)PHYS_TO_DMAP(l1phys);
1862 * Create page tables for first 128MB of KVA
1873 * the kernel page table pages need to be preserved in
1880 printf("phys_avail[%d]=%08lx - phys_avail[%d]=%08lx\n",
1888 printf("kernel_pmap pml1 %p\n", kernel_pmap->pm_pml1);
1891 * preallocated kernel page table pages so that vm_page structures
1953 (hwphyssz - physsz);
2016 phys_avail_count -= rm_pavail;
2024 printf("phys_avail[%d]=%08lx - phys_avail[%d]=%08lx\n",
2029 /* XXX assume we're running non-virtualized and
2080 * Allocate a kernel stack with a guard page for thread0 and map it
2081 * into the kernel page map.
2112 * Reserve some special page table entries/VA space for temporary
2126 ptcr = parttab_phys | (PARTTAB_SIZE_SHIFT-12);
2162 pagetab = RTS_SIZE | DMAP_TO_PHYS((vm_offset_t)kernel_pmap->pm_pml1) | \
2184 isa3_proctab->proctab0 =
2185 htobe64(RTS_SIZE | DMAP_TO_PHYS((vm_offset_t)kernel_pmap->pm_pml1) |
2189 mmu_radix_proctab_register(proctab0pa, PROCTAB_SIZE_SHIFT - 12);
2200 proctab0pa, 0, PROCTAB_SIZE_SHIFT - 12);
2210 isa3_proctab, kernel_pmap->pm_pml1);
2213 kernel_pmap->pm_pid = isa3_base_pid;
2265 * The large page mapping was destroyed.
2271 * Unless the page mappings are wired, remove the
2272 * mapping to a single page so that a subsequent
2273 * access may repromote. Choosing the last page
2276 * underlying page table page is fully populated, this
2277 * removal never frees a page table page.
2283 va -= PAGE_SIZE;
2309 * can be avoided by making the page
2337 * Routines used in machine-dependent code
2386 ptcr = parttab_phys | (PARTTAB_SIZE_SHIFT-12);
2399 "2MB page mapping counters");
2403 &pmap_l3e_demotions, "2MB page demotions");
2407 &pmap_l3e_mappings, "2MB page mappings");
2411 &pmap_l3e_p_failures, "2MB page promotion failures");
2415 &pmap_l3e_promotions, "2MB page promotions");
2418 "1GB page mapping counters");
2422 &pmap_l2e_demotions, "1GB page demotions");
2436 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
2437 ("pmap_clear_modify: page %p is not managed", m));
2442 * If the page is not PGA_WRITEABLE, then no PTEs can have PG_M set.
2443 * If the object containing the page is locked and the page is not
2446 if ((m->a.flags & PGA_WRITEABLE) == 0)
2448 pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy :
2453 TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_link, next_pv) {
2456 pvh_gen = pvh->pv_gen;
2460 if (pvh_gen != pvh->pv_gen) {
2465 va = pv->pv_va;
2473 * single page so that a subsequent
2476 va += VM_PAGE_TO_PHYS(m) - (oldl3e &
2489 TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) {
2492 md_gen = m->md.pv_gen;
2493 pvh_gen = pvh->pv_gen;
2497 if (pvh_gen != pvh->pv_gen || md_gen != m->md.pv_gen) {
2502 l3e = pmap_pml3e(pmap, pv->pv_va);
2504 " a 2mpage in page %p's pv list", m));
2505 pte = pmap_l3e_to_pte(l3e, pv->pv_va);
2508 pmap_invalidate_page(pmap, pv->pv_va);
2592 dst_pdpg->ref_count--;
2598 KASSERT(srcmpte->ref_count > 0,
2599 ("pmap_copy: source page table page is unused"));
2615 dstmpte->pindex == pmap_l3e_pindex(addr))
2616 dstmpte->ref_count++;
2641 * mapped, paging-structure
2644 * the freed page table pages.
2653 if (dstmpte->ref_count >= srcmpte->ref_count)
2698 cnt = min(xfersize, PAGE_SIZE - a_pg_offset);
2703 cnt = min(cnt, PAGE_SIZE - b_pg_offset);
2710 xfersize -= cnt;
2716 * Tries to promote the 512, contiguous 4KB page mappings that are within a
2717 * single page table page (PTP) to a single 2MB page mapping. For promotion
2718 * to occur, two conditions must be met: (1) the 4KB page mappings must map
2719 * aligned, contiguous physical memory and (2) the 4KB page mappings must have
2734 * either invalid, unused, or does not map the first 4KB physical page
2735 * within a 2MB page.
2757 * PTE maps an unexpected 4KB physical page or does not have identical
2760 pa = (newpde & (PG_PS_FRAME | PG_A | PG_V)) + L3_PAGE_SIZE - PAGE_SIZE;
2761 for (pte = firstpte + NPTEPG - 1; pte > firstpte; pte--) {
2786 pa -= PAGE_SIZE;
2790 * Save the page table page in its current state until the PDE
2797 ("pmap_promote_l3e: page table page is out of range"));
2798 KASSERT(mpte->pindex == pmap_l3e_pindex(va),
2799 ("pmap_promote_l3e: page table page's pindex is wrong"));
2845 KASSERT((m->oflags & VPO_UNMANAGED) != 0 || !VA_IS_CLEANMAP(va),
2847 if ((m->oflags & VPO_UNMANAGED) == 0)
2871 newpte |= pmap_cache_bits(m->md.mdpg_cache_attrs);
2874 * the page is unmanaged. We do not want to take a fault
2877 if ((m->oflags & VPO_UNMANAGED) != 0) {
2888 KASSERT(m->psind > 0, ("pmap_enter: m->psind < psind"));
2895 * In the case that a page table page is not
2905 mpte->ref_count++;
2909 * Here if the pte page isn't mapped, or if it has been
2924 panic("pmap_enter: invalid page directory va=%#lx", va);
2935 printf("cow fault pmap_enter(%p, %#lx, %p, %#x, %x, %d) --"
2937 pmap, va, m, prot, flags, psind, pmap->pm_pid,
2938 curproc->p_pid, curproc->p_comm, origpte);
2940 pmap_pte_walk(pmap->pm_pml1, va);
2947 * are valid mappings in them. Hence, if a user page is wired,
2948 * the PT page will be also.
2951 pmap->pm_stats.wired_count++;
2953 pmap->pm_stats.wired_count--;
2956 * Remove the extra PT page reference.
2959 mpte->ref_count--;
2960 KASSERT(mpte->ref_count > 0,
2961 ("pmap_enter: missing reference to page table page,"
2966 * Has the physical page changed?
2994 * The physical page has changed. Temporarily invalidate
3019 pv = pmap_pvh_remove(&om->md, pmap, va);
3032 if ((om->a.flags & PGA_WRITEABLE) != 0 &&
3033 TAILQ_EMPTY(&om->md.pv_list) &&
3034 ((om->flags & PG_FICTITIOUS) != 0 ||
3035 TAILQ_EMPTY(&pa_to_pvh(opa)->pv_list)))
3045 printf("pmap_enter(%p, %#lx, %p, %#x, %x, %d) -- asid=%lu curpid=%d name=%s\n",
3047 pmap->pm_pid, curproc->p_pid,
3048 curproc->p_comm);
3056 pmap->pm_stats.wired_count++;
3066 pv->pv_va = va;
3074 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_link);
3075 m->md.pv_gen++;
3116 * If both the page table page and the reservation are fully
3119 if ((mpte == NULL || mpte->ref_count == NPTEPG) &&
3121 (m->flags & PG_FICTITIOUS) == 0 &&
3141 * Release a page table page reference after a failed attempt to create a
3152 * Although "va" is not mapped, paging-
3154 * entries that refer to the freed page table
3163 * Tries to create a read- and/or execute-only 2MB page mapping. Returns true
3164 * if successful. Returns false if (1) a page table page cannot be allocated
3176 newpde = VM_PAGE_TO_PHYS(m) | pmap_cache_bits(m->md.mdpg_cache_attrs) |
3178 if ((m->oflags & VPO_UNMANAGED) == 0)
3192 * Tries to create the specified 2MB page mapping. Returns KERN_SUCCESS if
3196 * KERN_RESOURCE_SHORTAGE if PMAP_ENTER_NOSLEEP was specified and a page table
3197 * page allocation failed. Returns KERN_RESOURCE_SHORTAGE if
3225 KASSERT(pdpg->ref_count > 1,
3228 pdpg->ref_count--;
3237 * The reference to the PD page that was acquired by
3240 * a reserved PT page could be freed.
3261 KASSERT(be64toh(*l3e) == 0, ("pmap_enter_pde: non-zero pde %p",
3275 uwptpg->pindex = pmap_l3e_pindex(va);
3283 uwptpg->ref_count = NPTEPG;
3296 ("removed pt page %p, expected %p", mt,
3299 uwptpg->ref_count = 1;
3317 pmap->pm_stats.wired_count += L3_PAGE_SIZE / PAGE_SIZE;
3322 * be any lingering 4KB page mappings in the TLB.)
3343 VM_OBJECT_ASSERT_LOCKED(m_start->object);
3349 psize = atop(end - start);
3354 while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) {
3357 m->psind == 1 && mmu_radix_ps_enabled(pmap) &&
3359 m = &m[L3_PAGE_SIZE / PAGE_SIZE - 1];
3382 (m->oflags & VPO_UNMANAGED) != 0,
3387 * In the case that a page table page is not
3395 * Calculate pagetable page index
3398 if (mpte && (mpte->pindex == ptepindex)) {
3399 mpte->ref_count++;
3402 * Get the page directory entry
3407 * If the page table page is mapped, we just increment
3409 * attempt to allocate a page table page. If this
3416 mpte->ref_count++;
3435 mpte->ref_count--;
3444 if ((m->oflags & VPO_UNMANAGED) == 0 &&
3450 * Although "va" is not mapped, paging-
3452 * entries that refer to the freed page table
3468 pa = VM_PAGE_TO_PHYS(m) | pmap_cache_bits(m->md.mdpg_cache_attrs);
3473 if ((m->oflags & VPO_UNMANAGED) == 0)
3519 * because the page table page is preserved by the
3576 if (addr - 1 >= vm_map_max(kernel_map))
3586 nkpg->pindex = kernel_vm_end >> L2_PAGE_SIZE_SHIFT;
3594 if (kernel_vm_end - 1 >= vm_map_max(kernel_map)) {
3605 nkpg->pindex = pmap_l3e_pindex(kernel_vm_end);
3610 if (kernel_vm_end - 1 >= vm_map_max(kernel_map)) {
3618 static MALLOC_DEFINE(M_RADIX_PGD, "radix_pgd", "radix page table root directory");
3631 0, (vm_paddr_t)-1, RADIX_PGD_SIZE, L1_PAGE_SIZE,
3654 for (int j = page_count-1; j >= 0; j--) {
3670 /* L1TF, reserve page @0 unconditionally */
3684 * Initialize the vm page array entries for the kernel pmap's
3685 * page table pages.
3692 ("pmap_init: page table page is out of range size: %lu",
3694 mpte->pindex = pmap_l3e_pindex(VM_MIN_KERNEL_ADDRESS) + i;
3695 mpte->phys_addr = KPTphys + (i << PAGE_SHIFT);
3696 MPASS(PHYS_TO_VM_PAGE(mpte->phys_addr) == mpte);
3698 mpte->ref_count = 1;
3707 * Are large page mappings enabled?
3730 pv_npg = howmany(vm_phys_segs[vm_phys_nsegs - 1].end, L3_PAGE_SIZE);
3768 TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) {
3771 md_gen = m->md.pv_gen;
3775 if (md_gen != m->md.pv_gen) {
3780 pte = pmap_pte(pmap, pv->pv_va);
3791 if ((m->flags & PG_FICTITIOUS) == 0) {
3793 TAILQ_FOREACH(pv, &pvh->pv_list, pv_link) {
3796 md_gen = m->md.pv_gen;
3797 pvh_gen = pvh->pv_gen;
3801 if (md_gen != m->md.pv_gen ||
3802 pvh_gen != pvh->pv_gen) {
3807 pte = pmap_pml3e(pmap, pv->pv_va);
3827 * Return whether or not the specified physical page was modified
3834 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
3835 ("pmap_is_modified: page %p is not managed", m));
3839 * If the page is not busied then this check is racy.
3868 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
3869 ("pmap_is_referenced: page %p is not managed", m));
3877 * Return a count of reference bits for a page, clearing those bits.
3882 * As an optimization, update the page's dirty field if a modified bit is
3908 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
3909 ("pmap_ts_referenced: page %p is not managed", m));
3914 pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy : pa_to_pvh(pa);
3918 if ((pvf = TAILQ_FIRST(&pvh->pv_list)) == NULL)
3926 pvh_gen = pvh->pv_gen;
3930 if (pvh_gen != pvh->pv_gen) {
3935 l3e = pmap_pml3e(pmap, pv->pv_va);
3939 * Although "oldpde" is mapping a 2MB page, because
3940 * this function is called at a 4KB page granularity,
3941 * we only update the 4KB page under test.
3947 * Since this reference bit is shared by 512 4KB
3950 * physical page number, the virtual superpage number,
3951 * and the pmap address to select one 4KB page out of
3955 * same 4KB page for every 2MB page mapping.
3959 * subsequent page fault on a demoted wired mapping,
3962 * its reference bit won't affect page replacement.
3964 if ((((pa >> PAGE_SHIFT) ^ (pv->pv_va >> L3_PAGE_SIZE_SHIFT) ^
3965 (uintptr_t)pmap) & (NPTEPG - 1)) == 0 &&
3968 pmap_invalidate_page(pmap, pv->pv_va);
3971 ("inconsistent pv lock %p %p for page %p",
3979 TAILQ_REMOVE(&pvh->pv_list, pv, pv_link);
3980 TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_link);
3981 pvh->pv_gen++;
3985 } while ((pv = TAILQ_FIRST(&pvh->pv_list)) != pvf);
3987 if ((pvf = TAILQ_FIRST(&m->md.pv_list)) == NULL)
3995 pvh_gen = pvh->pv_gen;
3996 md_gen = m->md.pv_gen;
4000 if (pvh_gen != pvh->pv_gen || md_gen != m->md.pv_gen) {
4005 l3e = pmap_pml3e(pmap, pv->pv_va);
4007 ("pmap_ts_referenced: found a 2mpage in page %p's pv list",
4009 pte = pmap_l3e_to_pte(l3e, pv->pv_va);
4014 pmap_invalidate_page(pmap, pv->pv_va);
4020 TAILQ_REMOVE(&m->md.pv_list, pv, pv_link);
4021 TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_link);
4022 m->md.pv_gen++;
4024 } while ((pv = TAILQ_FIRST(&m->md.pv_list)) != pvf && cleared +
4054 KASSERT(object->type == OBJT_DEVICE || object->type == OBJT_SG,
4055 ("pmap_object_init_pt: non-device object"));
4063 KASSERT(p->valid == VM_PAGE_BITS_ALL,
4064 ("pmap_object_init_pt: invalid page %p", p));
4065 ma = p->md.mdpg_cache_attrs;
4068 * Abort the mapping if the first page is not physically
4069 * aligned to a 2MB page boundary.
4076 * Skip the first page. Abort the mapping if the rest of
4083 KASSERT(p->valid == VM_PAGE_BITS_ALL,
4084 ("pmap_object_init_pt: invalid page %p", p));
4086 ma != p->md.mdpg_cache_attrs)
4098 * optimization. If a page directory page
4115 pdpg->ref_count--;
4116 KASSERT(pdpg->ref_count > 0,
4118 "to page directory page, va: 0x%lx", addr));
4136 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
4137 ("pmap_page_exists_quick: page %p is not managed", m));
4142 TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) {
4151 if (!rv && loops < 16 && (m->flags & PG_FICTITIOUS) == 0) {
4153 TAILQ_FOREACH(pv, &pvh->pv_list, pv_link) {
4172 TAILQ_INIT(&m->md.pv_list);
4173 m->md.mdpg_cache_attrs = VM_MEMATTR_DEFAULT;
4186 if ((m->oflags & VPO_UNMANAGED) != 0)
4193 TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) {
4196 md_gen = m->md.pv_gen;
4200 if (md_gen != m->md.pv_gen) {
4205 pte = pmap_pte(pmap, pv->pv_va);
4210 if ((m->flags & PG_FICTITIOUS) == 0) {
4212 TAILQ_FOREACH(pv, &pvh->pv_list, pv_link) {
4215 md_gen = m->md.pv_gen;
4216 pvh_gen = pvh->pv_gen;
4220 if (md_gen != m->md.pv_gen ||
4221 pvh_gen != pvh->pv_gen) {
4226 pte = pmap_pml3e(pmap, pv->pv_va);
4251 * allocate the page directory page
4253 pmap->pm_pml1 = uma_zalloc(zone_radix_pgd, M_WAITOK);
4256 pagezero((vm_offset_t)pmap->pm_pml1 + j * PAGE_SIZE);
4257 vm_radix_init(&pmap->pm_radix);
4258 TAILQ_INIT(&pmap->pm_pvchunk);
4259 bzero(&pmap->pm_stats, sizeof pmap->pm_stats);
4260 pmap->pm_flags = PMAP_PDE_SUPERPAGE;
4263 pmap->pm_pid = pid;
4264 l1pa = DMAP_TO_PHYS((vm_offset_t)pmap->pm_pml1);
4272 * This routine is called if the desired page table page does not exist.
4274 * If page table page allocation fails, this routine may sleep before
4277 * Note: If a page allocation fails at page table level two or three,
4290 * Allocate a page table page.
4300 * Indicate the need to retry. While waiting, the page table
4301 * page may have been allocated.
4305 m->pindex = ptepindex;
4308 * Map the pagetable page into the process address space, if
4316 /* Wire up a new PDPE page */
4317 pml1index = ptepindex - (NUPDE + NUPDPE);
4318 l1e = &pmap->pm_pml1[pml1index];
4328 /* Wire up a new l2e page */
4329 pdpindex = ptepindex - NUPDE;
4332 l1e = &pmap->pm_pml1[pml1index];
4342 /* Add reference to l2e page */
4344 pdppg->ref_count++;
4348 /* Now find the pdp page */
4360 /* Wire up a new PTE page */
4365 l1e = &pmap->pm_pml1[pml1index];
4388 /* Add reference to the pd page */
4390 pdpg->ref_count++;
4395 /* Now we know where the page directory page is */
4415 /* Add a reference to the pd page. */
4417 pdpg->ref_count++;
4419 /* Allocate a pd page. */
4437 * Calculate pagetable page index
4442 * Get the page directory entry
4447 * This supports switching from a 2MB page to a
4448 * normal 4K page.
4453 * Invalidation of the 2MB page mapping may have caused
4454 * the deallocation of the underlying PD page.
4461 * If the page table page is mapped, we just increment the
4466 m->ref_count++;
4469 * Here if the pte page isn't mapped, or if it has been
4485 pmap->pm_pml1 = kernel_pmap->pm_pml1;
4486 pmap->pm_pid = kernel_pmap->pm_pid;
4488 vm_radix_init(&pmap->pm_radix);
4489 TAILQ_INIT(&pmap->pm_pvchunk);
4490 bzero(&pmap->pm_stats, sizeof pmap->pm_stats);
4491 kernel_pmap->pm_flags =
4492 pmap->pm_flags = PMAP_PDE_SUPERPAGE;
4528 * lingering 4KB page mappings from the TLB.
4563 printf("pmap_protect(%p, %#lx, %#lx, %x) - asid: %lu\n",
4564 pmap, sva, eva, prot, pmap->pm_pid);
4600 * Check for large page.
4604 * Are we protecting the entire large page? If not,
4613 * The large page mapping was destroyed.
4652 printf("%#lx %#lx -> %#lx\n",
4685 * the page table every time - but go for correctness for
4690 cache_bits = pmap_cache_bits(m->md.mdpg_cache_attrs);
4728 * Page table page management routines.....
4731 * Schedule the specified unused page table page to be freed. Specifically,
4732 * add the page to the specified list of pages that will be released to the
4740 m->flags |= PG_ZERO;
4742 m->flags &= ~PG_ZERO;
4747 * Inserts the specified page table page into the specified pmap's collection
4748 * of idle page table pages. Each of a pmap's page table pages is responsible
4757 return (vm_radix_insert(&pmap->pm_radix, mpte));
4761 * Removes the page table page mapping the specified virtual address from the
4762 * specified pmap's collection of idle page table pages, and returns it.
4763 * Otherwise, returns NULL if there is no page table page corresponding to the
4771 return (vm_radix_remove(&pmap->pm_radix, pmap_l3e_pindex(va)));
4775 * Decrements a page table page's wire count, which is used to record the
4776 * number of valid page table entries within the page. If the wire count
4777 * drops to zero, then the page table page is unmapped. Returns true if the
4778 * page table page was unmapped and false otherwise.
4784 --m->ref_count;
4785 if (m->ref_count == 0) {
4798 * unmap the page table page
4800 if (m->pindex >= NUPDE + NUPDPE) {
4801 /* PDP page */
4805 } else if (m->pindex >= NUPDE) {
4806 /* PD page */
4811 /* PTE page */
4817 if (m->pindex < NUPDE) {
4824 else if (m->pindex >= NUPDE && m->pindex < (NUPDE + NUPDPE)) {
4833 * Put page on a list so that it is released after
4840 * After removing a page table entry, this routine is used to
4841 * conditionally free the page, and manage the hold/wire counts.
4861 KASSERT(pmap->pm_stats.resident_count == 0,
4863 pmap->pm_stats.resident_count));
4864 KASSERT(vm_radix_is_empty(&pmap->pm_radix),
4865 ("pmap_release: pmap has reserved page table page(s)"));
4868 isa3_proctab[pmap->pm_pid].proctab0 = 0;
4869 uma_zfree(zone_radix_pgd, pmap->pm_pml1);
4870 vmem_free(asid_arena, pmap->pm_pid, 1);
4874 * Create the PV entry for a 2MB page mapping. Always returns true unless the
4891 pv->pv_va = va;
4895 TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_link);
4896 pvh->pv_gen++;
4901 * Fills a page table page with mappings to consecutive physical pages.
4946 ("pmap_demote_l3e: page table page for a wired mapping"
4950 * Invalidate the 2MB page mapping and return "failure" if the
4952 * page table page fails. If the 2MB page mapping belongs to
4954 * the page allocation request specifies the highest possible
4956 * normal. Page table pages are preallocated for every other
4973 mpte->pindex = pmap_l3e_pindex(va);
4985 * If the page table page is new, initialize it.
4987 if (mpte->ref_count == 1) {
4988 mpte->ref_count = NPTEPG;
4997 * If the mapping has changed attributes, update the page table
5009 * PV entry for the 2MB page mapping that is being demoted.
5012 reserve_pv_entries(pmap, NPTEPG - 1, lockp);
5048 panic("pmap_remove_kernel_pde: Missing pt page.");
5053 * Initialize the page table page.
5081 pmap->pm_stats.wired_count -= (L3_PAGE_SIZE / PAGE_SIZE);
5094 if (TAILQ_EMPTY(&m->md.pv_list) &&
5095 TAILQ_EMPTY(&pvh->pv_list))
5105 KASSERT(mpte->ref_count == NPTEPG,
5106 ("pmap_remove_l3e: pte page wire count error"));
5107 mpte->ref_count = 0;
5115 * pmap_remove_pte: do the things to unmap a page in a process
5128 pmap->pm_stats.wired_count -= 1;
5137 pmap_pvh_free(&m->md, pmap, va);
5138 if (TAILQ_EMPTY(&m->md.pv_list) &&
5139 (m->flags & PG_FICTITIOUS) == 0) {
5141 if (TAILQ_EMPTY(&pvh->pv_list))
5149 * Remove a single page from a process address space
5178 * Removes the specified range of addresses from the page table page.
5232 if (pmap->pm_stats.resident_count == 0)
5245 * special handling of removing one page. a very
5259 if (pmap->pm_stats.resident_count == 0)
5278 * Calculate index for next page table.
5294 * Check for large page.
5298 * Are we removing the entire large page? If not,
5307 /* The large page mapping was destroyed. */
5315 * by the current page table page, or to the end of the
5347 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
5348 ("pmap_remove_all: page %p is not managed", m));
5351 pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy :
5355 while ((pv = TAILQ_FIRST(&pvh->pv_list)) != NULL) {
5358 pvh_gen = pvh->pv_gen;
5362 if (pvh_gen != pvh->pv_gen) {
5368 va = pv->pv_va;
5373 while ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) {
5376 pvh_gen = pvh->pv_gen;
5377 md_gen = m->md.pv_gen;
5381 if (pvh_gen != pvh->pv_gen || md_gen != m->md.pv_gen) {
5388 l3e = pmap_pml3e(pmap, pv->pv_va);
5390 " a 2mpage in page %p's pv list", m));
5391 pte = pmap_l3e_to_pte(l3e, pv->pv_va);
5394 pmap->pm_stats.wired_count--;
5403 pmap_unuse_pt(pmap, pv->pv_va, be64toh(*l3e), &free);
5404 pmap_invalidate_page(pmap, pv->pv_va);
5405 TAILQ_REMOVE(&m->md.pv_list, pv, pv_link);
5406 m->md.pv_gen++;
5416 * Destroy all managed, non-wired mappings in the given user-space
5425 * entries, rather than searching the page table. Second, it doesn't
5426 * have to test and clear the page table entries atomically, because
5428 * particular, a page table entry's dirty bit won't change state once
5432 * non-wired mappings, it can delay and batch the invalidation of TLB
5468 KASSERT(pmap->pm_pid == mfspr(SPR_PID),
5469 ("non-current asid %lu - expected %lu", pmap->pm_pid,
5476 TAILQ_FOREACH_SAFE(pc, &pmap->pm_pvchunk, pc_list, npc) {
5482 inuse = ~pc->pc_map[field] & pc_freemask[field];
5487 pv = &pc->pc_pventry[idx];
5490 pte = pmap_pml2e(pmap, pv->pv_va);
5492 pte = pmap_l2e_to_l3e(pte, pv->pv_va);
5499 pte = &pte[pmap_pte_index(pv->pv_va)];
5509 * regular page could be mistaken for
5517 pv->pv_va, tpte);
5534 KASSERT(m->phys_addr == pa,
5536 m, (uintmax_t)m->phys_addr,
5539 KASSERT((m->flags & PG_FICTITIOUS) != 0 ||
5560 pc->pc_map[field] |= bitmask;
5564 TAILQ_REMOVE(&pvh->pv_list, pv, pv_link);
5565 pvh->pv_gen++;
5566 if (TAILQ_EMPTY(&pvh->pv_list)) {
5568 if ((mt->a.flags & PGA_WRITEABLE) != 0 &&
5569 TAILQ_EMPTY(&mt->md.pv_list))
5572 mpte = pmap_remove_pt_page(pmap, pv->pv_va);
5575 KASSERT(mpte->ref_count == NPTEPG,
5576 ("pmap_remove_pages: pte page wire count error"));
5577 mpte->ref_count = 0;
5586 TAILQ_REMOVE(&m->md.pv_list, pv, pv_link);
5587 m->md.pv_gen++;
5588 if ((m->a.flags & PGA_WRITEABLE) != 0 &&
5589 TAILQ_EMPTY(&m->md.pv_list) &&
5590 (m->flags & PG_FICTITIOUS) == 0) {
5592 if (TAILQ_EMPTY(&pvh->pv_list))
5596 pmap_unuse_pt(pmap, pv->pv_va, ptel3e, &free);
5606 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list);
5629 KASSERT((m->oflags & VPO_UNMANAGED) == 0,
5630 ("pmap_remove_write: page %p is not managed", m));
5636 pvh = (m->flags & PG_FICTITIOUS) != 0 ? &pv_dummy :
5640 TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_link, next_pv) {
5643 pvh_gen = pvh->pv_gen;
5647 if (pvh_gen != pvh->pv_gen) {
5653 l3e = pmap_pml3e(pmap, pv->pv_va);
5655 (void)pmap_demote_l3e_locked(pmap, l3e, pv->pv_va, &lock);
5657 ("inconsistent pv lock %p %p for page %p",
5661 TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) {
5664 pvh_gen = pvh->pv_gen;
5665 md_gen = m->md.pv_gen;
5669 if (pvh_gen != pvh->pv_gen ||
5670 md_gen != m->md.pv_gen) {
5676 l3e = pmap_pml3e(pmap, pv->pv_va);
5678 ("pmap_remove_write: found a 2mpage in page %p's pv list",
5680 pte = pmap_l3e_to_pte(l3e, pv->pv_va);
5689 pmap_invalidate_page(pmap, pv->pv_va);
5703 * The wired attribute of the page table entry is not a hardware
5747 * Are we unwiring the entire large page? If not,
5752 pmap->pm_stats.wired_count -= L3_PAGE_SIZE /
5775 pmap->pm_stats.wired_count--;
5817 /* Compute the physical address of the 4KB page. */
5856 pmap = vmspace_pmap(td->td_proc->p_vmspace);
5858 if (pmap->pm_pid > isa3_base_pid &&
5859 curpid != pmap->pm_pid) {
5880 if (object != NULL && (object->flags & OBJ_COLORED) != 0)
5881 offset += ptoa(object->pg_color);
5883 if (size - ((L3_PAGE_SIZE - superpage_offset) & L3_PAGE_MASK) < L3_PAGE_SIZE ||
5913 size -= PAGE_SIZE;
5936 m->md.mdpg_cache_attrs = ma;
5939 * If "m" is a normal page, update its direct mapping. This update
5943 if ((m->flags & PG_FICTITIOUS) == 0 &&
5945 PAGE_SIZE, m->md.mdpg_cache_attrs))
5978 pm = &curthread->td_proc->p_vmspace->vm_pmap;
5982 sync_sz = PAGE_SIZE - (va & PAGE_MASK);
5989 sz -= sync_sz;
5999 * The cache mode bits are all in the low 32-bits of the
6000 * PTE, so we can just spin on updating the low 32-bits.
6010 * Tries to demote a 1GB page mapping.
6030 pdpg->pindex = va >> L2_PAGE_SIZE_SHIFT;
6040 * Initialize the page directory page.
6053 * Flush PWC --- XXX revisit
6073 /* Big-endian PTE on stack */
6084 * because the page table page is preserved by the
6104 * Assume the page is cache inhibited and access is guarded unless
6194 cpu_flush_dcache((void *)sva, eva - sva);
6238 * into 4KB pages if required.
6246 * If the current 1GB page already has the required
6247 * memory type, then we need not demote this page. Just
6248 * increment tmpva to the next 1GB page frame.
6256 * If the current offset aligns with a 1GB page frame
6258 * we need not break down this page into 2MB pages.
6275 * If the current 2MB page already has the required
6276 * memory type, then we need not demote this page. Just
6277 * increment tmpva to the next 2MB page frame.
6285 * If the current offset aligns with a 2MB page frame
6287 * we need not break down this page into 4KB pages.
6329 pa_end - pa_start, mode, flush);
6359 pa_end - pa_start, mode, flush);
6387 pa_end - pa_start, mode, flush);
6402 pa_end1 - pa_start, mode, flush);
6420 * attempting to back the vm_pages with domain-local memory.
6441 pa = vm_phys_early_alloc(0, end - start);
6443 start = mmu_radix_map(&start, pa, end - start, VM_MEMATTR_DEFAULT);
6447 pfn = first_page + (va - start) / sizeof(struct vm_page);
6509 db_printf("page %p(%lx)\n", m, m->phys_addr);
6511 TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) {
6513 db_printf("va: %#016lx ", pv->pv_va);
6517 db_printf("asid: %lu\n", pmap->pm_pid);
6518 pmap_pte_walk(pmap->pm_pml1, pv->pv_va);
6537 pmap = vmspace_pmap(kdb_thread->td_proc->p_vmspace);
6539 pmap = vmspace_pmap(curthread->td_proc->p_vmspace);
6541 pmap_pte_walk(pmap->pm_pml1, va);