xref: /openbsd-src/sys/arch/arm64/arm64/pmap.c (revision 6f6231dc4f0a07fe7b8a4824090617757514f823)

/* $OpenBSD: pmap.c,v 1.107 2025/01/25 12:29:35 kettenis Exp $ */
/*
 * Copyright (c) 2008-2009,2014-2016 Dale Rahn <drahn@dalerahn.com>
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */

#include <sys/param.h>
#include <sys/systm.h>
#include <sys/atomic.h>
#include <sys/pool.h>
#include <sys/proc.h>

#include <uvm/uvm.h>

#include <machine/cpufunc.h>
#include <machine/pmap.h>

#include <machine/db_machdep.h>
#include <ddb/db_extern.h>
#include <ddb/db_output.h>

void pmap_setttb(struct proc *p);
void pmap_allocate_asid(pmap_t);
void pmap_free_asid(pmap_t pm);

/* We run userland code with ASIDs that have the low bit set. */
#define ASID_USER	1

static inline void
ttlb_flush(pmap_t pm, vaddr_t va)
{
	vaddr_t resva;

	resva = ((va >> PAGE_SHIFT) & ((1ULL << 44) - 1));
	if (pm == pmap_kernel()) {
		cpu_tlb_flush_all_asid(resva);
	} else {
		resva |= (uint64_t)pm->pm_asid << 48;
		cpu_tlb_flush_asid(resva);
		resva |= (uint64_t)ASID_USER << 48;
		cpu_tlb_flush_asid(resva);
	}
}

struct pmap kernel_pmap_;
struct pmap pmap_tramp;

LIST_HEAD(pted_pv_head, pte_desc);

struct pte_desc {
	LIST_ENTRY(pte_desc) pted_pv_list;
	uint64_t pted_pte;
	pmap_t pted_pmap;
	vaddr_t pted_va;
};

struct pmapvp0 {
	uint64_t l0[VP_IDX0_CNT];
	struct pmapvp1 *vp[VP_IDX0_CNT];
};

struct pmapvp1 {
	uint64_t l1[VP_IDX1_CNT];
	struct pmapvp2 *vp[VP_IDX1_CNT];
};

struct pmapvp2 {
	uint64_t l2[VP_IDX2_CNT];
	struct pmapvp3 *vp[VP_IDX2_CNT];
};

struct pmapvp3 {
	uint64_t l3[VP_IDX3_CNT];
	struct pte_desc *vp[VP_IDX3_CNT];
};
CTASSERT(sizeof(struct pmapvp0) == sizeof(struct pmapvp1));
CTASSERT(sizeof(struct pmapvp0) == sizeof(struct pmapvp2));
CTASSERT(sizeof(struct pmapvp0) == sizeof(struct pmapvp3));

void	pmap_vp_destroy(pmap_t pm);

/* Allocator for VP pool. */
void	*pmap_vp_page_alloc(struct pool *, int, int *);
void	pmap_vp_page_free(struct pool *, void *);

struct pool_allocator pmap_vp_allocator = {
	pmap_vp_page_alloc, pmap_vp_page_free, sizeof(struct pmapvp0)
};

void	pmap_remove_pted(pmap_t, struct pte_desc *);
void	pmap_kremove_pg(vaddr_t);
void	pmap_set_l1(struct pmap *, uint64_t, struct pmapvp1 *);
void	pmap_set_l2(struct pmap *, uint64_t, struct pmapvp1 *,
	    struct pmapvp2 *);
void	pmap_set_l3(struct pmap *, uint64_t, struct pmapvp2 *,
	    struct pmapvp3 *);

void	pmap_fill_pte(pmap_t, vaddr_t, paddr_t, struct pte_desc *,
	    vm_prot_t, int, int);
void	pmap_icache_sync_page(struct pmap *, paddr_t);
void	pmap_pte_insert(struct pte_desc *);
void	pmap_pte_remove(struct pte_desc *, int);
void	pmap_pte_update(struct pte_desc *, uint64_t *);
void	pmap_release(pmap_t);
paddr_t	pmap_steal_avail(size_t, int, void **);
void	pmap_remove_avail(paddr_t, paddr_t);
vaddr_t	pmap_map_stolen(vaddr_t);

vaddr_t vmmap;
vaddr_t zero_page;
vaddr_t copy_src_page;
vaddr_t copy_dst_page;

struct pool pmap_pmap_pool;
struct pool pmap_pted_pool;
struct pool pmap_vp_pool;

int pmap_initialized = 0;

struct mem_region {
	vaddr_t start;
	vsize_t size;
};

struct mem_region pmap_avail_regions[10];
struct mem_region pmap_allocated_regions[10];
struct mem_region *pmap_avail = &pmap_avail_regions[0];
struct mem_region *pmap_allocated = &pmap_allocated_regions[0];
int pmap_cnt_avail, pmap_cnt_allocated;
uint64_t pmap_avail_kvo;

static inline void
pmap_lock(struct pmap *pmap)
{
	if (pmap != pmap_kernel())
		mtx_enter(&pmap->pm_mtx);
}

static inline void
pmap_unlock(struct pmap *pmap)
{
	if (pmap != pmap_kernel())
		mtx_leave(&pmap->pm_mtx);
}

#define PMAP_ASSERT_LOCKED(pmap) 			\
	if ((pmap) != pmap_kernel()) 			\
		MUTEX_ASSERT_LOCKED(&(pmap)->pm_mtx);

/* virtual to physical helpers */
static inline int
VP_IDX0(vaddr_t va)
{
	return (va >> VP_IDX0_POS) & VP_IDX0_MASK;
}

static inline int
VP_IDX1(vaddr_t va)
{
	return (va >> VP_IDX1_POS) & VP_IDX1_MASK;
}

static inline int
VP_IDX2(vaddr_t va)
{
	return (va >> VP_IDX2_POS) & VP_IDX2_MASK;
}

static inline int
VP_IDX3(vaddr_t va)
{
	return (va >> VP_IDX3_POS) & VP_IDX3_MASK;
}

const uint64_t ap_bits_user[8] = {
	[PROT_NONE]				= 0,
	[PROT_READ]				= ATTR_PXN|ATTR_UXN|ATTR_AF|ATTR_AP(3),
	[PROT_WRITE]				= ATTR_PXN|ATTR_UXN|ATTR_AF|ATTR_AP(1),
	[PROT_WRITE|PROT_READ]			= ATTR_PXN|ATTR_UXN|ATTR_AF|ATTR_AP(1),
	[PROT_EXEC]				= ATTR_PXN|ATTR_AF|ATTR_AP(2),
	[PROT_EXEC|PROT_READ]			= ATTR_PXN|ATTR_AF|ATTR_AP(3),
	[PROT_EXEC|PROT_WRITE]			= ATTR_PXN|ATTR_AF|ATTR_AP(1),
	[PROT_EXEC|PROT_WRITE|PROT_READ]	= ATTR_PXN|ATTR_AF|ATTR_AP(1),
};

const uint64_t ap_bits_kern[8] = {
	[PROT_NONE]				= 0,
	[PROT_READ]				= ATTR_PXN|ATTR_UXN|ATTR_AF|ATTR_AP(2),
	[PROT_WRITE]				= ATTR_PXN|ATTR_UXN|ATTR_AF|ATTR_AP(0),
	[PROT_WRITE|PROT_READ]			= ATTR_PXN|ATTR_UXN|ATTR_AF|ATTR_AP(0),
	[PROT_EXEC]				= ATTR_UXN|ATTR_AF|ATTR_AP(2),
	[PROT_EXEC|PROT_READ]			= ATTR_UXN|ATTR_AF|ATTR_AP(2),
	[PROT_EXEC|PROT_WRITE]			= ATTR_UXN|ATTR_AF|ATTR_AP(0),
	[PROT_EXEC|PROT_WRITE|PROT_READ]	= ATTR_UXN|ATTR_AF|ATTR_AP(0),
};

/*
 * We allocate ASIDs in pairs.  The first ASID is used to run the
 * kernel and has both userland and the full kernel mapped.  The
 * second ASID is used for running userland and has only the
 * trampoline page mapped in addition to userland.
 */

#define PMAP_MAX_NASID	(1 << 16)
#define PMAP_ASID_MASK	(PMAP_MAX_NASID - 1)
int pmap_nasid = (1 << 8);

uint32_t pmap_asid[PMAP_MAX_NASID / 32];
unsigned long pmap_asid_gen = PMAP_MAX_NASID;
struct mutex pmap_asid_mtx = MUTEX_INITIALIZER(IPL_HIGH);

int
pmap_find_asid(pmap_t pm)
{
	uint32_t bits;
	int asid, bit;
	int retry;

	MUTEX_ASSERT_LOCKED(&pmap_asid_mtx);

	/* Attempt to re-use the old ASID. */
	asid = pm->pm_asid & PMAP_ASID_MASK;
	bit = asid & (32 - 1);
	bits = pmap_asid[asid / 32];
	if ((bits & (3U << bit)) == 0)
		return asid;

	/* Attempt to obtain a random ASID. */
	for (retry = 5; retry > 0; retry--) {
		asid = arc4random() & (pmap_nasid - 2);
		bit = (asid & (32 - 1));
		bits = pmap_asid[asid / 32];
		if ((bits & (3U << bit)) == 0)
			return asid;
	}

	/* Do a linear search if that fails. */
	for (asid = 0; asid < pmap_nasid; asid += 32) {
		bits = pmap_asid[asid / 32];
		if (bits == ~0)
			continue;
		for (bit = 0; bit < 32; bit += 2) {
			if ((bits & (3U << bit)) == 0)
				return asid + bit;
		}
	}

	return -1;
}

int
pmap_rollover_asid(pmap_t pm)
{
	struct cpu_info *ci;
	CPU_INFO_ITERATOR cii;
	unsigned long gen;
	int asid, bit;

	MUTEX_ASSERT_LOCKED(&pmap_asid_mtx);

	/* Start a new generation.  Mark ASID 0 as in-use again. */
	gen = atomic_add_long_nv(&pmap_asid_gen, PMAP_MAX_NASID);
	memset(pmap_asid, 0, (pmap_nasid / 32) * sizeof(uint32_t));
	pmap_asid[0] |= (3U << 0);

	/*
	 * Carry over all the ASIDs that are currently active into the
	 * new generation and reserve them.
	 * CPUs in cpu_switchto() will spin in pmap_setttb() waiting for
	 * the mutex. In that case an old ASID will be carried over but
	 * that is not problematic.
	 */
	CPU_INFO_FOREACH(cii, ci) {
		asid = ci->ci_curpm->pm_asid & PMAP_ASID_MASK;
		ci->ci_curpm->pm_asid = asid | gen;
		bit = (asid & (32 - 1));
		pmap_asid[asid / 32] |= (3U << bit);
	}

	/* Flush the TLBs on all CPUs. */
	cpu_tlb_flush();

	if ((pm->pm_asid & ~PMAP_ASID_MASK) == gen)
		return pm->pm_asid & PMAP_ASID_MASK;

	return pmap_find_asid(pm);
}

void
pmap_allocate_asid(pmap_t pm)
{
	int asid, bit;

	mtx_enter(&pmap_asid_mtx);
	asid = pmap_find_asid(pm);
	if (asid == -1) {
		/*
		 * We have no free ASIDs.  Do a rollover to clear all
		 * inactive ASIDs and pick a fresh one.
		 */
		asid = pmap_rollover_asid(pm);
	}
	KASSERT(asid > 0 && asid < pmap_nasid);
	bit = asid & (32 - 1);
	pmap_asid[asid / 32] |= (3U << bit);
	pm->pm_asid = asid | pmap_asid_gen;
	mtx_leave(&pmap_asid_mtx);
}

void
pmap_free_asid(pmap_t pm)
{
	int asid, bit;

	KASSERT(pm != curcpu()->ci_curpm);
	cpu_tlb_flush_asid_all((uint64_t)pm->pm_asid << 48);
	cpu_tlb_flush_asid_all((uint64_t)(pm->pm_asid | ASID_USER) << 48);

	mtx_enter(&pmap_asid_mtx);
	if ((pm->pm_asid & ~PMAP_ASID_MASK) == pmap_asid_gen) {
		asid = pm->pm_asid & PMAP_ASID_MASK;
		bit = (asid & (32 - 1));
		pmap_asid[asid / 32] &= ~(3U << bit);
	}
	mtx_leave(&pmap_asid_mtx);
}

/*
 * This is used for pmap_kernel() mappings, they are not to be removed
 * from the vp table because they were statically initialized at the
 * initial pmap initialization. This is so that memory allocation
 * is not necessary in the pmap_kernel() mappings.
 * Otherwise bad race conditions can appear.
 */
struct pte_desc *
pmap_vp_lookup(pmap_t pm, vaddr_t va, uint64_t **pl3entry)
{
	struct pmapvp1 *vp1;
	struct pmapvp2 *vp2;
	struct pmapvp3 *vp3;
	struct pte_desc *pted;

	if (pm->have_4_level_pt) {
		if (pm->pm_vp.l0 == NULL) {
			return NULL;
		}
		vp1 = pm->pm_vp.l0->vp[VP_IDX0(va)];
	} else {
		vp1 = pm->pm_vp.l1;
	}
	if (vp1 == NULL) {
		return NULL;
	}

	vp2 = vp1->vp[VP_IDX1(va)];
	if (vp2 == NULL) {
		return NULL;
	}

	vp3 = vp2->vp[VP_IDX2(va)];
	if (vp3 == NULL) {
		return NULL;
	}

	pted = vp3->vp[VP_IDX3(va)];
	if (pl3entry != NULL)
		*pl3entry = &(vp3->l3[VP_IDX3(va)]);

	return pted;
}

/*
 * Create a V -> P mapping for the given pmap and virtual address
 * with reference to the pte descriptor that is used to map the page.
 * This code should track allocations of vp table allocations
 * so they can be freed efficiently.
 *
 * XXX it may be possible to save some bits of count in the
 * upper address bits of the pa or the pte entry.
 * However that does make populating the other bits more tricky.
 * each level has 512 entries, so that mean 9 bits to store
 * stash 3 bits each in the first 3 entries?
 */
int
pmap_vp_enter(pmap_t pm, vaddr_t va, struct pte_desc *pted, int flags)
{
	struct pmapvp1 *vp1;
	struct pmapvp2 *vp2;
	struct pmapvp3 *vp3;

	PMAP_ASSERT_LOCKED(pm);

	if (pm->have_4_level_pt) {
		vp1 = pm->pm_vp.l0->vp[VP_IDX0(va)];
		if (vp1 == NULL) {
			vp1 = pool_get(&pmap_vp_pool, PR_NOWAIT | PR_ZERO);
			if (vp1 == NULL) {
				if ((flags & PMAP_CANFAIL) == 0)
					panic("%s: unable to allocate L1",
					    __func__);
				return ENOMEM;
			}
			pmap_set_l1(pm, va, vp1);
		}
	} else {
		vp1 = pm->pm_vp.l1;
	}

	vp2 = vp1->vp[VP_IDX1(va)];
	if (vp2 == NULL) {
		vp2 = pool_get(&pmap_vp_pool, PR_NOWAIT | PR_ZERO);
		if (vp2 == NULL) {
			if ((flags & PMAP_CANFAIL) == 0)
				panic("%s: unable to allocate L2", __func__);
			return ENOMEM;
		}
		pmap_set_l2(pm, va, vp1, vp2);
	}

	vp3 = vp2->vp[VP_IDX2(va)];
	if (vp3 == NULL) {
		vp3 = pool_get(&pmap_vp_pool, PR_NOWAIT | PR_ZERO);
		if (vp3 == NULL) {
			if ((flags & PMAP_CANFAIL) == 0)
				panic("%s: unable to allocate L3", __func__);
			return ENOMEM;
		}
		pmap_set_l3(pm, va, vp2, vp3);
	}

	vp3->vp[VP_IDX3(va)] = pted;
	return 0;
}

void
pmap_vp_populate(pmap_t pm, vaddr_t va)
{
	struct pte_desc *pted;
	struct pmapvp1 *vp1;
	struct pmapvp2 *vp2;
	struct pmapvp3 *vp3;
	void *vp;

	pted = pool_get(&pmap_pted_pool, PR_WAITOK | PR_ZERO);
	vp = pool_get(&pmap_vp_pool, PR_WAITOK | PR_ZERO);

	pmap_lock(pm);

	if (pm->have_4_level_pt) {
		vp1 = pm->pm_vp.l0->vp[VP_IDX0(va)];
		if (vp1 == NULL) {
			vp1 = vp; vp = NULL;
			pmap_set_l1(pm, va, vp1);
		}
	} else {
		vp1 = pm->pm_vp.l1;
	}

	if (vp == NULL) {
		pmap_unlock(pm);
		vp = pool_get(&pmap_vp_pool, PR_WAITOK | PR_ZERO);
		pmap_lock(pm);
	}

	vp2 = vp1->vp[VP_IDX1(va)];
	if (vp2 == NULL) {
		vp2 = vp; vp = NULL;
		pmap_set_l2(pm, va, vp1, vp2);
	}

	if (vp == NULL) {
		pmap_unlock(pm);
		vp = pool_get(&pmap_vp_pool, PR_WAITOK | PR_ZERO);
		pmap_lock(pm);
	}

	vp3 = vp2->vp[VP_IDX2(va)];
	if (vp3 == NULL) {
		vp3 = vp; vp = NULL;
		pmap_set_l3(pm, va, vp2, vp3);
	}

	if (vp3->vp[VP_IDX3(va)] == NULL) {
		vp3->vp[VP_IDX3(va)] = pted;
		pted = NULL;
	}

	pmap_unlock(pm);

	if (vp)
		pool_put(&pmap_vp_pool, vp);
	if (pted)
		pool_put(&pmap_pted_pool, pted);
}

void *
pmap_vp_page_alloc(struct pool *pp, int flags, int *slowdown)
{
	struct kmem_dyn_mode kd = KMEM_DYN_INITIALIZER;

	kd.kd_waitok = ISSET(flags, PR_WAITOK);
	kd.kd_trylock = ISSET(flags, PR_NOWAIT);
	kd.kd_slowdown = slowdown;

	return km_alloc(pp->pr_pgsize, &kv_any, &kp_dirty, &kd);
}

void
pmap_vp_page_free(struct pool *pp, void *v)
{
	km_free(v, pp->pr_pgsize, &kv_any, &kp_dirty);
}

static inline u_int32_t
PTED_MANAGED(struct pte_desc *pted)
{
	return (pted->pted_va & PTED_VA_MANAGED_M);
}

static inline u_int32_t
PTED_WIRED(struct pte_desc *pted)
{
	return (pted->pted_va & PTED_VA_WIRED_M);
}

static inline u_int32_t
PTED_VALID(struct pte_desc *pted)
{
	return (pted->pted_pte != 0);
}

/*
 * PV entries -
 * manipulate the physical to virtual translations for the entire system.
 *
 * QUESTION: should all mapped memory be stored in PV tables? Or
 * is it alright to only store "ram" memory. Currently device mappings
 * are not stored.
 * It makes sense to pre-allocate mappings for all of "ram" memory, since
 * it is likely that it will be mapped at some point, but would it also
 * make sense to use a tree/table like is use for pmap to store device
 * mappings?
 * Further notes: It seems that the PV table is only used for pmap_protect
 * and other paging related operations. Given this, it is not necessary
 * to store any pmap_kernel() entries in PV tables and does not make
 * sense to store device mappings in PV either.
 *
 * Note: unlike other powerpc pmap designs, the array is only an array
 * of pointers. Since the same structure is used for holding information
 * in the VP table, the PV table, and for kernel mappings, the wired entries.
 * Allocate one data structure to hold all of the info, instead of replicating
 * it multiple times.
 *
 * One issue of making this a single data structure is that two pointers are
 * wasted for every page which does not map ram (device mappings), this
 * should be a low percentage of mapped pages in the system, so should not
 * have too noticeable unnecessary ram consumption.
 */

void
pmap_enter_pv(struct pte_desc *pted, struct vm_page *pg)
{
	/*
	 * XXX does this test mean that some pages try to be managed,
	 * but this is called too soon?
	 */
	if (__predict_false(!pmap_initialized))
		return;

	mtx_enter(&pg->mdpage.pv_mtx);
	LIST_INSERT_HEAD(&(pg->mdpage.pv_list), pted, pted_pv_list);
	pted->pted_va |= PTED_VA_MANAGED_M;
	mtx_leave(&pg->mdpage.pv_mtx);
}

void
pmap_remove_pv(struct pte_desc *pted)
{
	struct vm_page *pg = PHYS_TO_VM_PAGE(pted->pted_pte & PTE_RPGN);

	mtx_enter(&pg->mdpage.pv_mtx);
	LIST_REMOVE(pted, pted_pv_list);
	mtx_leave(&pg->mdpage.pv_mtx);
}

int
pmap_enter(pmap_t pm, vaddr_t va, paddr_t pa, vm_prot_t prot, int flags)
{
	struct pte_desc *pted;
	struct vm_page *pg;
	int error;
	int cache = PMAP_CACHE_WB;

	if (pa & PMAP_NOCACHE)
		cache = PMAP_CACHE_CI;
	if (pa & PMAP_DEVICE)
		cache = PMAP_CACHE_DEV_NGNRNE;
	pg = PHYS_TO_VM_PAGE(pa);

	pmap_lock(pm);
	pted = pmap_vp_lookup(pm, va, NULL);
	if (pted && PTED_VALID(pted)) {
		pmap_remove_pted(pm, pted);
		/* we lost our pted if it was user */
		if (pm != pmap_kernel())
			pted = pmap_vp_lookup(pm, va, NULL);
	}

	pm->pm_stats.resident_count++;

	/* Do not have pted for this, get one and put it in VP */
	if (pted == NULL) {
		pted = pool_get(&pmap_pted_pool, PR_NOWAIT | PR_ZERO);
		if (pted == NULL) {
			if ((flags & PMAP_CANFAIL) == 0)
				panic("%s: failed to allocate pted", __func__);
			error = ENOMEM;
			goto out;
		}
		if (pmap_vp_enter(pm, va, pted, flags)) {
			if ((flags & PMAP_CANFAIL) == 0)
				panic("%s: failed to allocate L2/L3", __func__);
			error = ENOMEM;
			pool_put(&pmap_pted_pool, pted);
			goto out;
		}
	}

	/*
	 * If it should be enabled _right now_, we can skip doing ref/mod
	 * emulation. Any access includes reference, modified only by write.
	 */
	if (pg != NULL &&
	    ((flags & PROT_MASK) || (pg->pg_flags & PG_PMAP_REF))) {
		atomic_setbits_int(&pg->pg_flags, PG_PMAP_REF);
		if ((prot & PROT_WRITE) && (flags & PROT_WRITE)) {
			atomic_setbits_int(&pg->pg_flags, PG_PMAP_MOD);
			atomic_clearbits_int(&pg->pg_flags, PG_PMAP_EXE);
		}
	}

	pmap_fill_pte(pm, va, pa, pted, prot, flags, cache);

	if (pg != NULL) {
		pmap_enter_pv(pted, pg); /* only managed mem */
	}

	if (pg != NULL && (flags & PROT_EXEC)) {
		if ((pg->pg_flags & PG_PMAP_EXE) == 0)
			pmap_icache_sync_page(pm, pa);
		atomic_setbits_int(&pg->pg_flags, PG_PMAP_EXE);
	}

	/*
	 * Insert into table, if this mapping said it needed to be mapped
	 * now.
	 */
	if (flags & (PROT_READ|PROT_WRITE|PROT_EXEC|PMAP_WIRED)) {
		pmap_pte_insert(pted);
		ttlb_flush(pm, va & ~PAGE_MASK);
	}

	error = 0;
out:
	pmap_unlock(pm);
	return error;
}

void
pmap_populate(pmap_t pm, vaddr_t va)
{
	pmap_vp_populate(pm, va);
}

/*
 * Remove the given range of mapping entries.
 */
void
pmap_remove(pmap_t pm, vaddr_t sva, vaddr_t eva)
{
	struct pte_desc *pted;
	vaddr_t va;

	pmap_lock(pm);
	for (va = sva; va < eva; va += PAGE_SIZE) {
		pted = pmap_vp_lookup(pm, va, NULL);

		if (pted == NULL)
			continue;

		if (PTED_WIRED(pted)) {
			pm->pm_stats.wired_count--;
			pted->pted_va &= ~PTED_VA_WIRED_M;
		}

		if (PTED_VALID(pted))
			pmap_remove_pted(pm, pted);
	}
	pmap_unlock(pm);
}

/*
 * remove a single mapping, notice that this code is O(1)
 */
void
pmap_remove_pted(pmap_t pm, struct pte_desc *pted)
{
	pm->pm_stats.resident_count--;

	if (PTED_WIRED(pted)) {
		pm->pm_stats.wired_count--;
		pted->pted_va &= ~PTED_VA_WIRED_M;
	}

	pmap_pte_remove(pted, pm != pmap_kernel());
	ttlb_flush(pm, pted->pted_va & ~PAGE_MASK);

	if (pted->pted_va & PTED_VA_EXEC_M) {
		pted->pted_va &= ~PTED_VA_EXEC_M;
	}

	if (PTED_MANAGED(pted))
		pmap_remove_pv(pted);

	pted->pted_pte = 0;
	pted->pted_va = 0;

	if (pm != pmap_kernel())
		pool_put(&pmap_pted_pool, pted);
}


/*
 * Enter a kernel mapping for the given page.
 * kernel mappings have a larger set of prerequisites than normal mappings.
 *
 * 1. no memory should be allocated to create a kernel mapping.
 * 2. a vp mapping should already exist, even if invalid. (see 1)
 * 3. all vp tree mappings should already exist (see 1)
 *
 */
void
_pmap_kenter_pa(vaddr_t va, paddr_t pa, vm_prot_t prot, int flags, int cache)
{
	pmap_t pm = pmap_kernel();
	struct pte_desc *pted;
	struct vm_page *pg;

	pted = pmap_vp_lookup(pm, va, NULL);
	if (pted == NULL) {
		panic("pted not preallocated in pmap_kernel() va %lx pa %lx",
		    va, pa);
	}

	if (pted && PTED_VALID(pted))
		pmap_kremove_pg(va); /* pted is reused */

	pm->pm_stats.resident_count++;

	flags |= PMAP_WIRED; /* kernel mappings are always wired. */
	/* Calculate PTE */
	pmap_fill_pte(pm, va, pa, pted, prot, flags, cache);

	/* Insert into table */
	pmap_pte_insert(pted);
	ttlb_flush(pm, va & ~PAGE_MASK);

	pg = PHYS_TO_VM_PAGE(pted->pted_pte & PTE_RPGN);
	if (pg && (cache == PMAP_CACHE_CI || cache == PMAP_CACHE_DEV_NGNRNE))
		cpu_idcache_wbinv_range(va & ~PAGE_MASK, PAGE_SIZE);
}

void
pmap_kenter_pa(vaddr_t va, paddr_t pa, vm_prot_t prot)
{
	_pmap_kenter_pa(va, pa, prot, prot,
	    (pa & PMAP_NOCACHE) ? PMAP_CACHE_CI : PMAP_CACHE_WB);
}

void
pmap_kenter_cache(vaddr_t va, paddr_t pa, vm_prot_t prot, int cacheable)
{
	_pmap_kenter_pa(va, pa, prot, prot, cacheable);
}

/*
 * remove kernel (pmap_kernel()) mapping, one page
 */
void
pmap_kremove_pg(vaddr_t va)
{
	pmap_t pm = pmap_kernel();
	struct pte_desc *pted;
	int s;

	pted = pmap_vp_lookup(pm, va, NULL);
	if (pted == NULL)
		return;

	if (!PTED_VALID(pted))
		return; /* not mapped */

	s = splvm();

	pm->pm_stats.resident_count--;

	pmap_pte_remove(pted, 0);
	ttlb_flush(pm, pted->pted_va & ~PAGE_MASK);

	if (pted->pted_va & PTED_VA_EXEC_M)
		pted->pted_va &= ~PTED_VA_EXEC_M;

	if (PTED_MANAGED(pted))
		pmap_remove_pv(pted);

	if (PTED_WIRED(pted))
		pm->pm_stats.wired_count--;

	/* invalidate pted; */
	pted->pted_pte = 0;
	pted->pted_va = 0;

	splx(s);
}

/*
 * remove kernel (pmap_kernel()) mappings
 */
void
pmap_kremove(vaddr_t va, vsize_t len)
{
	for (len >>= PAGE_SHIFT; len >0; len--, va += PAGE_SIZE)
		pmap_kremove_pg(va);
}

void
pmap_fill_pte(pmap_t pm, vaddr_t va, paddr_t pa, struct pte_desc *pted,
    vm_prot_t prot, int flags, int cache)
{
	pted->pted_va = va;
	pted->pted_pmap = pm;

	switch (cache) {
	case PMAP_CACHE_WB:
		break;
	case PMAP_CACHE_WT:
		break;
	case PMAP_CACHE_CI:
		break;
	case PMAP_CACHE_DEV_NGNRNE:
		break;
	case PMAP_CACHE_DEV_NGNRE:
		break;
	default:
		panic("%s: invalid cache mode", __func__);
	}
	pted->pted_va |= cache;

	pted->pted_va |= prot & (PROT_READ|PROT_WRITE|PROT_EXEC);

	if (flags & PMAP_WIRED) {
		pted->pted_va |= PTED_VA_WIRED_M;
		pm->pm_stats.wired_count++;
	}

	pted->pted_pte = pa & PTE_RPGN;
	pted->pted_pte |= flags & (PROT_READ|PROT_WRITE|PROT_EXEC);
}

/*
 * Fill the given physical page with zeros.
 */
void
pmap_zero_page(struct vm_page *pg)
{
	paddr_t pa = VM_PAGE_TO_PHYS(pg);
	vaddr_t va = zero_page + cpu_number() * PAGE_SIZE;

	KASSERT(curcpu()->ci_idepth == 0);

	pmap_kenter_pa(va, pa, PROT_READ|PROT_WRITE);
	pagezero_cache(va);
	pmap_kremove_pg(va);
}

/*
 * Copy the given physical page.
 */
void
pmap_copy_page(struct vm_page *srcpg, struct vm_page *dstpg)
{
	paddr_t srcpa = VM_PAGE_TO_PHYS(srcpg);
	paddr_t dstpa = VM_PAGE_TO_PHYS(dstpg);
	vaddr_t srcva = copy_src_page + cpu_number() * PAGE_SIZE;
	vaddr_t dstva = copy_dst_page + cpu_number() * PAGE_SIZE;
	int s;

	/*
	 * XXX The buffer flipper (incorrectly?) uses pmap_copy_page()
	 * (from uvm_pagerealloc_multi()) from interrupt context!
	 */
	s = splbio();
	pmap_kenter_pa(srcva, srcpa, PROT_READ);
	pmap_kenter_pa(dstva, dstpa, PROT_READ|PROT_WRITE);
	memcpy((void *)dstva, (void *)srcva, PAGE_SIZE);
	pmap_kremove_pg(srcva);
	pmap_kremove_pg(dstva);
	splx(s);
}

void
pmap_pinit(pmap_t pm)
{
	vaddr_t l0va;

	/* Allocate a full L0/L1 table. */
	if (pm->have_4_level_pt) {
		while (pm->pm_vp.l0 == NULL) {
			pm->pm_vp.l0 = pool_get(&pmap_vp_pool,
			    PR_WAITOK | PR_ZERO);
		}
		l0va = (vaddr_t)pm->pm_vp.l0->l0; /* top level is l0 */
	} else {
		while (pm->pm_vp.l1 == NULL) {

			pm->pm_vp.l1 = pool_get(&pmap_vp_pool,
			    PR_WAITOK | PR_ZERO);
		}
		l0va = (vaddr_t)pm->pm_vp.l1->l1; /* top level is l1 */

	}

	pmap_extract(pmap_kernel(), l0va, (paddr_t *)&pm->pm_pt0pa);

	pmap_reference(pm);
}

int pmap_vp_poolcache = 0; /* force vp poolcache to allocate late */

/*
 * Create and return a physical map.
 */
pmap_t
pmap_create(void)
{
	pmap_t pmap;

	pmap = pool_get(&pmap_pmap_pool, PR_WAITOK | PR_ZERO);

	mtx_init(&pmap->pm_mtx, IPL_VM);

	pmap_pinit(pmap);
	if (pmap_vp_poolcache == 0) {
		pool_setlowat(&pmap_vp_pool, 20);
		pmap_vp_poolcache = 20;
	}
	return (pmap);
}

/*
 * Add a reference to a given pmap.
 */
void
pmap_reference(pmap_t pm)
{
	atomic_inc_int(&pm->pm_refs);
}

/*
 * Retire the given pmap from service.
 * Should only be called if the map contains no valid mappings.
 */
void
pmap_destroy(pmap_t pm)
{
	int refs;

	refs = atomic_dec_int_nv(&pm->pm_refs);
	if (refs > 0)
		return;

	/*
	 * reference count is zero, free pmap resources and free pmap.
	 */
	pmap_release(pm);
	pmap_free_asid(pm);
	pool_put(&pmap_pmap_pool, pm);
}

/*
 * Release any resources held by the given physical map.
 * Called when a pmap initialized by pmap_pinit is being released.
 */
void
pmap_release(pmap_t pm)
{
	pmap_vp_destroy(pm);
}

void
pmap_vp_destroy_l2_l3(pmap_t pm, struct pmapvp1 *vp1)
{
	struct pmapvp2 *vp2;
	struct pmapvp3 *vp3;
	struct pte_desc *pted;
	int j, k, l;

	for (j = 0; j < VP_IDX1_CNT; j++) {
		vp2 = vp1->vp[j];
		if (vp2 == NULL)
			continue;
		vp1->vp[j] = NULL;

		for (k = 0; k < VP_IDX2_CNT; k++) {
			vp3 = vp2->vp[k];
			if (vp3 == NULL)
				continue;
			vp2->vp[k] = NULL;

			for (l = 0; l < VP_IDX3_CNT; l++) {
				pted = vp3->vp[l];
				if (pted == NULL)
					continue;
				vp3->vp[l] = NULL;

				pool_put(&pmap_pted_pool, pted);
			}
			pool_put(&pmap_vp_pool, vp3);
		}
		pool_put(&pmap_vp_pool, vp2);
	}
}

void
pmap_vp_destroy(pmap_t pm)
{
	struct pmapvp0 *vp0;
	struct pmapvp1 *vp1;
	int i;

	/*
	 * XXX Is there a better way to share this code between 3 and
	 * 4 level tables?  Split the lower levels into a different
	 * function?
	 */
	if (!pm->have_4_level_pt) {
		pmap_vp_destroy_l2_l3(pm, pm->pm_vp.l1);
		pool_put(&pmap_vp_pool, pm->pm_vp.l1);
		pm->pm_vp.l1 = NULL;
		return;
	}

	vp0 = pm->pm_vp.l0;
	for (i = 0; i < VP_IDX0_CNT; i++) {
		vp1 = vp0->vp[i];
		if (vp1 == NULL)
			continue;
		vp0->vp[i] = NULL;

		pmap_vp_destroy_l2_l3(pm, vp1);
		pool_put(&pmap_vp_pool, vp1);
	}
	pool_put(&pmap_vp_pool, vp0);
	pm->pm_vp.l0 = NULL;
}

vaddr_t virtual_avail;
int	pmap_virtual_space_called;

static inline uint64_t
VP_Lx(paddr_t pa)
{
	/*
	 * This function takes the pa address given and manipulates it
	 * into the form that should be inserted into the VM table.
	 */
	return pa | Lx_TYPE_PT;
}

/*
 * In pmap_bootstrap() we allocate the page tables for the first GB
 * of the kernel address space.
 */
vaddr_t pmap_maxkvaddr = VM_MIN_KERNEL_ADDRESS + 1024 * 1024 * 1024;

/*
 * Allocator for growing the kernel page tables.  We use a dedicated
 * submap to make sure we have the space to map them as we are called
 * when address space is tight!
 */

struct vm_map *pmap_kvp_map;

const struct kmem_va_mode kv_kvp = {
	.kv_map = &pmap_kvp_map,
	.kv_wait = 0
};

void *
pmap_kvp_alloc(void)
{
	void *kvp;

	if (!uvm.page_init_done && !pmap_virtual_space_called) {
		paddr_t pa[2];
		vaddr_t va;

		if (!uvm_page_physget(&pa[0]) || !uvm_page_physget(&pa[1]))
			panic("%s: out of memory", __func__);

		va = virtual_avail;
		virtual_avail += 2 * PAGE_SIZE;
		KASSERT(virtual_avail <= pmap_maxkvaddr);
		kvp = (void *)va;

		pmap_kenter_pa(va, pa[0], PROT_READ|PROT_WRITE);
		pmap_kenter_pa(va + PAGE_SIZE, pa[1], PROT_READ|PROT_WRITE);
		pagezero_cache(va);
		pagezero_cache(va + PAGE_SIZE);
	} else {
		kvp = km_alloc(sizeof(struct pmapvp0), &kv_kvp, &kp_zero,
		    &kd_nowait);
	}

	return kvp;
}

struct pte_desc *
pmap_kpted_alloc(void)
{
	static struct pte_desc *pted;
	static int npted;

	if (npted == 0) {
		if (!uvm.page_init_done && !pmap_virtual_space_called) {
			paddr_t pa;
			vaddr_t va;

			if (!uvm_page_physget(&pa))
				panic("%s: out of memory", __func__);

			va = virtual_avail;
			virtual_avail += PAGE_SIZE;
			KASSERT(virtual_avail <= pmap_maxkvaddr);
			pted = (struct pte_desc *)va;

			pmap_kenter_pa(va, pa, PROT_READ|PROT_WRITE);
			pagezero_cache(va);
		} else {
			pted = km_alloc(PAGE_SIZE, &kv_kvp, &kp_zero,
			    &kd_nowait);
			if (pted == NULL)
				return NULL;
		}

		npted = PAGE_SIZE / sizeof(struct pte_desc);
	}

	npted--;
	return pted++;
}

vaddr_t
pmap_growkernel(vaddr_t maxkvaddr)
{
	struct pmapvp1 *vp1 = pmap_kernel()->pm_vp.l1;
	struct pmapvp2 *vp2;
	struct pmapvp3 *vp3;
	struct pte_desc *pted;
	paddr_t pa;
	int lb_idx2, ub_idx2;
	int i, j, k;
	int s;

	if (maxkvaddr <= pmap_maxkvaddr)
		return pmap_maxkvaddr;

	/*
	 * Not strictly necessary, but we use an interrupt-safe map
	 * and uvm asserts that we're at IPL_VM.
	 */
	s = splvm();

	for (i = VP_IDX1(pmap_maxkvaddr); i <= VP_IDX1(maxkvaddr - 1); i++) {
		vp2 = vp1->vp[i];
		if (vp2 == NULL) {
			vp2 = pmap_kvp_alloc();
			if (vp2 == NULL)
				goto fail;
			pmap_extract(pmap_kernel(), (vaddr_t)vp2, &pa);
			vp1->vp[i] = vp2;
			vp1->l1[i] = VP_Lx(pa);
		}

		if (i == VP_IDX1(pmap_maxkvaddr)) {
			lb_idx2 = VP_IDX2(pmap_maxkvaddr);
		} else {
			lb_idx2 = 0;
		}

		if (i == VP_IDX1(maxkvaddr - 1)) {
			ub_idx2 = VP_IDX2(maxkvaddr - 1);
		} else {
			ub_idx2 = VP_IDX2_CNT - 1;
		}

		for (j = lb_idx2; j <= ub_idx2; j++) {
			vp3 = vp2->vp[j];
			if (vp3 == NULL) {
				vp3 = pmap_kvp_alloc();
				if (vp3 == NULL)
					goto fail;
				pmap_extract(pmap_kernel(), (vaddr_t)vp3, &pa);
				vp2->vp[j] = vp3;
				vp2->l2[j] = VP_Lx(pa);
			}

			for (k = 0; k <= VP_IDX3_CNT - 1; k++) {
				if (vp3->vp[k] == NULL) {
					pted = pmap_kpted_alloc();
					if (pted == NULL)
						goto fail;
					vp3->vp[k] = pted;
					pmap_maxkvaddr += PAGE_SIZE;
				}
			}
		}
	}
	KASSERT(pmap_maxkvaddr >= maxkvaddr);

fail:
	splx(s);

	return pmap_maxkvaddr;
}

void pmap_setup_avail(uint64_t ram_start, uint64_t ram_end, uint64_t kvo);

/*
 * Initialize pmap setup.
 * ALL of the code which deals with avail needs rewritten as an actual
 * memory allocation.
 */
CTASSERT(sizeof(struct pmapvp0) == 2 * PAGE_SIZE);

int mappings_allocated = 0;
int pted_allocated = 0;

extern char __text_start[], _etext[];
extern char __rodata_start[], _erodata[];

vaddr_t
pmap_bootstrap(long kvo, paddr_t lpt1, long kernelstart, long kernelend,
    long ram_start, long ram_end)
{
	void  *va;
	paddr_t pa, pt1pa;
	struct pmapvp1 *vp1;
	struct pmapvp2 *vp2;
	struct pmapvp3 *vp3;
	struct pte_desc *pted;
	vaddr_t vstart;
	uint64_t id_aa64mmfr0;
	int i, j, k;
	int lb_idx2, ub_idx2;

	pmap_setup_avail(ram_start, ram_end, kvo);

	/*
	 * in theory we could start with just the memory in the
	 * kernel, however this could 'allocate' the bootloader and
	 * bootstrap vm table, which we may need to preserve until
	 * later.
	 */
	printf("removing %lx-%lx\n", ram_start, kernelstart+kvo);
	pmap_remove_avail(ram_start, kernelstart+kvo);
	printf("removing %lx-%lx\n", kernelstart+kvo, kernelend+kvo);
	pmap_remove_avail(kernelstart+kvo, kernelend+kvo);

	/*
	 * KERNEL IS ASSUMED TO BE 39 bits (or less), start from L1,
	 * not L0 ALSO kernel mappings may not cover enough ram to
	 * bootstrap so all accesses initializing tables must be done
	 * via physical pointers
	 */

	pt1pa = pmap_steal_avail(2 * sizeof(struct pmapvp1), Lx_TABLE_ALIGN,
	    &va);
	vp1 = (struct pmapvp1 *)pt1pa;
	pmap_kernel()->pm_vp.l1 = (struct pmapvp1 *)va;
	pmap_kernel()->pm_privileged = 1;
	pmap_kernel()->pm_guarded = ATTR_GP;
	pmap_kernel()->pm_asid = 0;

	mtx_init(&pmap_tramp.pm_mtx, IPL_VM);
	pmap_tramp.pm_vp.l1 = (struct pmapvp1 *)va + 1;
	pmap_tramp.pm_privileged = 1;
	pmap_tramp.pm_guarded = ATTR_GP;
	pmap_tramp.pm_asid = 0;

	/* Mark ASID 0 as in-use. */
	pmap_asid[0] |= (3U << 0);

	/* allocate Lx entries */
	for (i = VP_IDX1(VM_MIN_KERNEL_ADDRESS);
	    i <= VP_IDX1(pmap_maxkvaddr - 1);
	    i++) {
		mappings_allocated++;
		pa = pmap_steal_avail(sizeof(struct pmapvp2), Lx_TABLE_ALIGN,
		    &va);
		vp2 = (struct pmapvp2 *)pa; /* indexed physically */
		vp1->vp[i] = va;
		vp1->l1[i] = VP_Lx(pa);

		if (i == VP_IDX1(VM_MIN_KERNEL_ADDRESS)) {
			lb_idx2 = VP_IDX2(VM_MIN_KERNEL_ADDRESS);
		} else {
			lb_idx2 = 0;
		}
		if (i == VP_IDX1(pmap_maxkvaddr - 1)) {
			ub_idx2 = VP_IDX2(pmap_maxkvaddr - 1);
		} else {
			ub_idx2 = VP_IDX2_CNT - 1;
		}
		for (j = lb_idx2; j <= ub_idx2; j++) {
			mappings_allocated++;
			pa = pmap_steal_avail(sizeof(struct pmapvp3),
			    Lx_TABLE_ALIGN, &va);
			vp3 = (struct pmapvp3 *)pa; /* indexed physically */
			vp2->vp[j] = va;
			vp2->l2[j] = VP_Lx(pa);

		}
	}
	/* allocate Lx entries */
	for (i = VP_IDX1(VM_MIN_KERNEL_ADDRESS);
	    i <= VP_IDX1(pmap_maxkvaddr - 1);
	    i++) {
		/* access must be performed physical */
		vp2 = (void *)((long)vp1->vp[i] + kvo);

		if (i == VP_IDX1(VM_MIN_KERNEL_ADDRESS)) {
			lb_idx2 = VP_IDX2(VM_MIN_KERNEL_ADDRESS);
		} else {
			lb_idx2 = 0;
		}
		if (i == VP_IDX1(pmap_maxkvaddr - 1)) {
			ub_idx2 = VP_IDX2(pmap_maxkvaddr - 1);
		} else {
			ub_idx2 = VP_IDX2_CNT - 1;
		}
		for (j = lb_idx2; j <= ub_idx2; j++) {
			/* access must be performed physical */
			vp3 = (void *)((long)vp2->vp[j] + kvo);

			for (k = 0; k <= VP_IDX3_CNT - 1; k++) {
				pted_allocated++;
				pa = pmap_steal_avail(sizeof(struct pte_desc),
				    4, &va);
				pted = va;
				vp3->vp[k] = pted;
			}
		}
	}

	pa = pmap_steal_avail(Lx_TABLE_ALIGN, Lx_TABLE_ALIGN, &va);
	memset((void *)pa, 0, Lx_TABLE_ALIGN);
	pmap_kernel()->pm_pt0pa = pa;

	pmap_avail_fixup();

	/*
	 * At this point we are still running on the bootstrap page
	 * tables however all memory for the final page tables is
	 * 'allocated' and should now be mapped.  This means we are
	 * able to use the virtual addressing to enter the final
	 * mappings into the new mapping tables.
	 */
	vstart = pmap_map_stolen(kernelstart);

	void (switch_mmu_kernel)(long);
	void (*switch_mmu_kernel_table)(long) =
	    (void *)((long)&switch_mmu_kernel + kvo);
	switch_mmu_kernel_table(pt1pa);

	printf("all mapped\n");

	curcpu()->ci_curpm = pmap_kernel();

	id_aa64mmfr0 = READ_SPECIALREG(id_aa64mmfr0_el1);
	if (ID_AA64MMFR0_ASID_BITS(id_aa64mmfr0) == ID_AA64MMFR0_ASID_BITS_16)
		pmap_nasid = (1 << 16);

	vmmap = vstart;
	vstart += PAGE_SIZE;

	return vstart;
}

void
pmap_set_l1(struct pmap *pm, uint64_t va, struct pmapvp1 *l1_va)
{
	uint64_t pg_entry;
	paddr_t l1_pa;
	int idx0;

	if (pmap_extract(pmap_kernel(), (vaddr_t)l1_va, &l1_pa) == 0)
		panic("unable to find vp pa mapping %p", l1_va);

	if (l1_pa & (Lx_TABLE_ALIGN-1))
		panic("misaligned L2 table");

	pg_entry = VP_Lx(l1_pa);

	idx0 = VP_IDX0(va);
	pm->pm_vp.l0->vp[idx0] = l1_va;
	pm->pm_vp.l0->l0[idx0] = pg_entry;
}

void
pmap_set_l2(struct pmap *pm, uint64_t va, struct pmapvp1 *vp1,
    struct pmapvp2 *l2_va)
{
	uint64_t pg_entry;
	paddr_t l2_pa;
	int idx1;

	if (pmap_extract(pmap_kernel(), (vaddr_t)l2_va, &l2_pa) == 0)
		panic("unable to find vp pa mapping %p", l2_va);

	if (l2_pa & (Lx_TABLE_ALIGN-1))
		panic("misaligned L2 table");

	pg_entry = VP_Lx(l2_pa);

	idx1 = VP_IDX1(va);
	vp1->vp[idx1] = l2_va;
	vp1->l1[idx1] = pg_entry;
}

void
pmap_set_l3(struct pmap *pm, uint64_t va, struct pmapvp2 *vp2,
    struct pmapvp3 *l3_va)
{
	uint64_t pg_entry;
	paddr_t l3_pa;
	int idx2;

	if (pmap_extract(pmap_kernel(), (vaddr_t)l3_va, &l3_pa) == 0)
		panic("unable to find vp pa mapping %p", l3_va);

	if (l3_pa & (Lx_TABLE_ALIGN-1))
		panic("misaligned L2 table");

	pg_entry = VP_Lx(l3_pa);

	idx2 = VP_IDX2(va);
	vp2->vp[idx2] = l3_va;
	vp2->l2[idx2] = pg_entry;
}

/*
 * activate a pmap entry
 */
void
pmap_activate(struct proc *p)
{
	pmap_t pm = p->p_vmspace->vm_map.pmap;

	if (p == curproc && pm != curcpu()->ci_curpm)
		pmap_setttb(p);
}

/*
 * deactivate a pmap entry
 */
void
pmap_deactivate(struct proc *p)
{
}

/*
 * Get the physical page address for the given pmap/virtual address.
 */
int
pmap_extract(pmap_t pm, vaddr_t va, paddr_t *pap)
{
	struct pte_desc *pted;

	pmap_lock(pm);
	pted = pmap_vp_lookup(pm, va, NULL);
	if (!pted || !PTED_VALID(pted)) {
		pmap_unlock(pm);
		return 0;
	}
	if (pap != NULL)
		*pap = (pted->pted_pte & PTE_RPGN) | (va & PAGE_MASK);
	pmap_unlock(pm);

	return 1;
}

void
pmap_page_ro(pmap_t pm, vaddr_t va, vm_prot_t prot)
{
	struct pte_desc *pted;
	uint64_t *pl3;

	/* Every VA needs a pted, even unmanaged ones. */
	pted = pmap_vp_lookup(pm, va, &pl3);
	if (!pted || !PTED_VALID(pted)) {
		return;
	}

	pted->pted_va &= ~PROT_WRITE;
	pted->pted_pte &= ~PROT_WRITE;
	if ((prot & PROT_READ) == 0) {
		pted->pted_va &= ~PROT_READ;
		pted->pted_pte &= ~PROT_READ;
	}
	if ((prot & PROT_EXEC) == 0) {
		pted->pted_va &= ~PROT_EXEC;
		pted->pted_pte &= ~PROT_EXEC;
	}
	pmap_pte_update(pted, pl3);
	ttlb_flush(pm, pted->pted_va & ~PAGE_MASK);
}

#ifdef DDB
void
pmap_page_rw(pmap_t pm, vaddr_t va)
{
	struct pte_desc *pted;
	uint64_t *pl3;

	/* Every VA needs a pted, even unmanaged ones. */
	pted = pmap_vp_lookup(pm, va, &pl3);
	if (!pted || !PTED_VALID(pted)) {
		return;
	}

	pted->pted_va |= PROT_WRITE;
	pted->pted_pte |= PROT_WRITE;
	pmap_pte_update(pted, pl3);
	ttlb_flush(pm, pted->pted_va & ~PAGE_MASK);
}
#endif /* DDB */

/*
 * Lower the protection on the specified physical page.
 */
void
pmap_page_protect(struct vm_page *pg, vm_prot_t prot)
{
	struct pte_desc *pted;
	struct pmap *pm;

	if (prot != PROT_NONE) {
		mtx_enter(&pg->mdpage.pv_mtx);
		LIST_FOREACH(pted, &(pg->mdpage.pv_list), pted_pv_list) {
			pmap_page_ro(pted->pted_pmap, pted->pted_va, prot);
		}
		mtx_leave(&pg->mdpage.pv_mtx);
		return;
	}

	mtx_enter(&pg->mdpage.pv_mtx);
	while ((pted = LIST_FIRST(&(pg->mdpage.pv_list))) != NULL) {
		pmap_reference(pted->pted_pmap);
		pm = pted->pted_pmap;
		mtx_leave(&pg->mdpage.pv_mtx);

		pmap_lock(pm);

		/*
		 * We dropped the pvlist lock before grabbing the pmap
		 * lock to avoid lock ordering problems.  This means
		 * we have to check the pvlist again since somebody
		 * else might have modified it.  All we care about is
		 * that the pvlist entry matches the pmap we just
		 * locked.  If it doesn't, unlock the pmap and try
		 * again.
		 */
		mtx_enter(&pg->mdpage.pv_mtx);
		pted = LIST_FIRST(&(pg->mdpage.pv_list));
		if (pted == NULL || pted->pted_pmap != pm) {
			mtx_leave(&pg->mdpage.pv_mtx);
			pmap_unlock(pm);
			pmap_destroy(pm);
			mtx_enter(&pg->mdpage.pv_mtx);
			continue;
		}
		mtx_leave(&pg->mdpage.pv_mtx);

		pmap_remove_pted(pm, pted);
		pmap_unlock(pm);
		pmap_destroy(pm);

		mtx_enter(&pg->mdpage.pv_mtx);
	}
	/* page is being reclaimed, sync icache next use */
	atomic_clearbits_int(&pg->pg_flags, PG_PMAP_EXE);
	mtx_leave(&pg->mdpage.pv_mtx);
}

void
pmap_protect(pmap_t pm, vaddr_t sva, vaddr_t eva, vm_prot_t prot)
{
	if (prot & (PROT_READ | PROT_EXEC)) {
		pmap_lock(pm);
		while (sva < eva) {
			pmap_page_ro(pm, sva, prot);
			sva += PAGE_SIZE;
		}
		pmap_unlock(pm);
		return;
	}
	pmap_remove(pm, sva, eva);
}

void
pmap_init(void)
{
	uint64_t tcr;

	/*
	 * Now that we are in virtual address space we don't need
	 * the identity mapping in TTBR0 and can set the TCR to a
	 * more useful value.
	 */
	WRITE_SPECIALREG(ttbr0_el1, pmap_kernel()->pm_pt0pa);
	__asm volatile("isb");
	tcr = READ_SPECIALREG(tcr_el1);
	tcr &= ~TCR_T0SZ(0x3f);
	tcr |= TCR_T0SZ(64 - USER_SPACE_BITS);
	tcr |= TCR_A1;
	WRITE_SPECIALREG(tcr_el1, tcr);
	cpu_tlb_flush();

	pool_init(&pmap_pmap_pool, sizeof(struct pmap), 0, IPL_NONE, 0,
	    "pmap", NULL);
	pool_setlowat(&pmap_pmap_pool, 2);
	pool_init(&pmap_pted_pool, sizeof(struct pte_desc), 0, IPL_VM, 0,
	    "pted", NULL);
	pool_setlowat(&pmap_pted_pool, 20);
	pool_init(&pmap_vp_pool, sizeof(struct pmapvp0), PAGE_SIZE, IPL_VM, 0,
	    "vp", &pmap_vp_allocator);
	/* pool_setlowat(&pmap_vp_pool, 20); */

	pmap_initialized = 1;
}

void
pmap_proc_iflush(struct process *pr, vaddr_t va, vsize_t len)
{
	struct pmap *pm = vm_map_pmap(&pr->ps_vmspace->vm_map);
	vaddr_t kva = zero_page + cpu_number() * PAGE_SIZE;
	paddr_t pa;
	vsize_t clen;
	vsize_t off;

	/*
	 * If we're called for the current process, we can simply
	 * flush the data cache to the point of unification and
	 * invalidate the instruction cache.
	 */
	if (pr == curproc->p_p) {
		cpu_icache_sync_range(va, len);
		return;
	}

	/*
	 * Flush and invalidate through an aliased mapping.  This
	 * assumes the instruction cache is PIPT.  That is only true
	 * for some of the hardware we run on.
	 */
	while (len > 0) {
		/* add one to always round up to the next page */
		clen = round_page(va + 1) - va;
		if (clen > len)
			clen = len;

		off = va - trunc_page(va);
		if (pmap_extract(pm, trunc_page(va), &pa)) {
			pmap_kenter_pa(kva, pa, PROT_READ|PROT_WRITE);
			cpu_icache_sync_range(kva + off, clen);
			pmap_kremove_pg(kva);
		}

		len -= clen;
		va += clen;
	}
}

void
pmap_icache_sync_page(struct pmap *pm, paddr_t pa)
{
	vaddr_t kva = zero_page + cpu_number() * PAGE_SIZE;

	pmap_kenter_pa(kva, pa, PROT_READ|PROT_WRITE);
	cpu_icache_sync_range(kva, PAGE_SIZE);
	pmap_kremove_pg(kva);
}

void
pmap_pte_insert(struct pte_desc *pted)
{
	pmap_t pm = pted->pted_pmap;
	uint64_t *pl3;

	if (pmap_vp_lookup(pm, pted->pted_va, &pl3) == NULL) {
		panic("%s: have a pted, but missing a vp"
		    " for %lx va pmap %p", __func__, pted->pted_va, pm);
	}

	pmap_pte_update(pted, pl3);
}

void
pmap_pte_update(struct pte_desc *pted, uint64_t *pl3)
{
	uint64_t pte, access_bits;
	pmap_t pm = pted->pted_pmap;
	uint64_t attr = ATTR_nG;

	/* see mair in locore.S */
	switch (pted->pted_va & PMAP_CACHE_BITS) {
	case PMAP_CACHE_WB:
		/* inner and outer writeback */
		attr |= ATTR_IDX(PTE_ATTR_WB);
		attr |= ATTR_SH(SH_INNER);
		break;
	case PMAP_CACHE_WT:
		 /* inner and outer writethrough */
		attr |= ATTR_IDX(PTE_ATTR_WT);
		attr |= ATTR_SH(SH_INNER);
		break;
	case PMAP_CACHE_CI:
		attr |= ATTR_IDX(PTE_ATTR_CI);
		attr |= ATTR_SH(SH_INNER);
		break;
	case PMAP_CACHE_DEV_NGNRNE:
		attr |= ATTR_IDX(PTE_ATTR_DEV_NGNRNE);
		attr |= ATTR_SH(SH_INNER);
		break;
	case PMAP_CACHE_DEV_NGNRE:
		attr |= ATTR_IDX(PTE_ATTR_DEV_NGNRE);
		attr |= ATTR_SH(SH_INNER);
		break;
	default:
		panic("%s: invalid cache mode", __func__);
	}

	if (pm->pm_privileged)
		access_bits = ap_bits_kern[pted->pted_pte & PROT_MASK];
	else
		access_bits = ap_bits_user[pted->pted_pte & PROT_MASK];

#ifndef SMALL_KERNEL
	access_bits |= pm->pm_guarded;
#endif

	pte = (pted->pted_pte & PTE_RPGN) | attr | access_bits | L3_P;
	*pl3 = access_bits ? pte : 0;
}

void
pmap_pte_remove(struct pte_desc *pted, int remove_pted)
{
	struct pmapvp1 *vp1;
	struct pmapvp2 *vp2;
	struct pmapvp3 *vp3;
	pmap_t pm = pted->pted_pmap;

	if (pm->have_4_level_pt)
		vp1 = pm->pm_vp.l0->vp[VP_IDX0(pted->pted_va)];
	else
		vp1 = pm->pm_vp.l1;
	if (vp1 == NULL) {
		panic("have a pted, but missing the l1 for %lx va pmap %p",
		    pted->pted_va, pm);
	}
	vp2 = vp1->vp[VP_IDX1(pted->pted_va)];
	if (vp2 == NULL) {
		panic("have a pted, but missing the l2 for %lx va pmap %p",
		    pted->pted_va, pm);
	}
	vp3 = vp2->vp[VP_IDX2(pted->pted_va)];
	if (vp3 == NULL) {
		panic("have a pted, but missing the l3 for %lx va pmap %p",
		    pted->pted_va, pm);
	}
	vp3->l3[VP_IDX3(pted->pted_va)] = 0;
	if (remove_pted)
		vp3->vp[VP_IDX3(pted->pted_va)] = NULL;
}

/*
 * This function exists to do software referenced/modified emulation.
 * Its purpose is to tell the caller that a fault was generated either
 * for this emulation, or to tell the caller that it's a legit fault.
 */
int
pmap_fault_fixup(pmap_t pm, vaddr_t va, vm_prot_t ftype)
{
	struct pte_desc *pted;
	struct vm_page *pg;
	paddr_t pa;
	uint64_t *pl3 = NULL;
	int retcode = 0;

	pmap_lock(pm);

	/* Every VA needs a pted, even unmanaged ones. */
	pted = pmap_vp_lookup(pm, va, &pl3);
	if (!pted || !PTED_VALID(pted))
		goto done;

	/* There has to be a PA for the VA, get it. */
	pa = (pted->pted_pte & PTE_RPGN);

	/* If it's unmanaged, it must not fault. */
	pg = PHYS_TO_VM_PAGE(pa);
	if (pg == NULL)
		goto done;

	/*
	 * Check the fault types to find out if we were doing
	 * any mod/ref emulation and fixup the PTE if we were.
	 */
	if ((ftype & PROT_WRITE) && /* fault caused by a write */
	    !(pted->pted_pte & PROT_WRITE) && /* and write is disabled now */
	    (pted->pted_va & PROT_WRITE)) { /* but is supposedly allowed */

		/*
		 * Page modified emulation. A write always includes
		 * a reference.  This means that we can enable read and
		 * exec as well, akin to the page reference emulation.
		 */
		atomic_setbits_int(&pg->pg_flags, PG_PMAP_MOD|PG_PMAP_REF);
		atomic_clearbits_int(&pg->pg_flags, PG_PMAP_EXE);

		/* Thus, enable read, write and exec. */
		pted->pted_pte |=
		    (pted->pted_va & (PROT_READ|PROT_WRITE|PROT_EXEC));
	} else if ((ftype & PROT_EXEC) && /* fault caused by an exec */
	    !(pted->pted_pte & PROT_EXEC) && /* and exec is disabled now */
	    (pted->pted_va & PROT_EXEC)) { /* but is supposedly allowed */

		/*
		 * Exec always includes a reference. Since we now know
		 * the page has been accessed, we can enable read as well
		 * if UVM allows it.
		 */
		atomic_setbits_int(&pg->pg_flags, PG_PMAP_REF);

		/* Thus, enable read and exec. */
		pted->pted_pte |= (pted->pted_va & (PROT_READ|PROT_EXEC));
	} else if ((ftype & PROT_READ) && /* fault caused by a read */
	    !(pted->pted_pte & PROT_READ) && /* and read is disabled now */
	    (pted->pted_va & PROT_READ)) { /* but is supposedly allowed */

		/*
		 * Page referenced emulation. Since we now know the page
		 * has been accessed, we can enable exec as well if UVM
		 * allows it.
		 */
		atomic_setbits_int(&pg->pg_flags, PG_PMAP_REF);

		/* Thus, enable read and exec. */
		pted->pted_pte |= (pted->pted_va & (PROT_READ|PROT_EXEC));
	} else {
		/* didn't catch it, so probably broken */
		goto done;
	}

	/*
	 * If this is a page that can be executed, make sure to invalidate
	 * the instruction cache if the page has been modified or not used
	 * yet.
	 */
	if (pted->pted_va & PROT_EXEC) {
		if ((pg->pg_flags & PG_PMAP_EXE) == 0)
			pmap_icache_sync_page(pm, pa);
		atomic_setbits_int(&pg->pg_flags, PG_PMAP_EXE);
	}

	/* We actually made a change, so flush it and sync. */
	pmap_pte_update(pted, pl3);
	ttlb_flush(pm, va & ~PAGE_MASK);

	retcode = 1;
done:
	pmap_unlock(pm);
	return retcode;
}

void
pmap_postinit(void)
{
	extern char trampoline_vectors[];
	extern char trampoline_vectors_end[];
	paddr_t pa;
	vaddr_t minaddr, maxaddr;
	u_long npteds, npages;

	memset(pmap_tramp.pm_vp.l1, 0, sizeof(struct pmapvp1));
	pmap_extract(pmap_kernel(), (vaddr_t)trampoline_vectors, &pa);
	minaddr = (vaddr_t)trampoline_vectors;
	maxaddr = (vaddr_t)trampoline_vectors_end;
	while (minaddr < maxaddr) {
		pmap_enter(&pmap_tramp, minaddr, pa,
		    PROT_READ | PROT_EXEC, PROT_READ | PROT_EXEC | PMAP_WIRED);
		minaddr += PAGE_SIZE;
		pa += PAGE_SIZE;
	}

	/*
	 * Reserve enough virtual address space to grow the kernel
	 * page tables.  We need a descriptor for each page as well as
	 * an extra page for level 1/2/3 page tables for management.
	 * To simplify the code, we always allocate full tables at
	 * level 3, so take that into account.
	 */
	npteds = (VM_MAX_KERNEL_ADDRESS - pmap_maxkvaddr + 1) / PAGE_SIZE;
	npteds = roundup(npteds, VP_IDX3_CNT);
	npages = howmany(npteds, PAGE_SIZE / (sizeof(struct pte_desc)));
	npages += 2 * howmany(npteds, VP_IDX3_CNT);
	npages += 2 * howmany(npteds, VP_IDX3_CNT * VP_IDX2_CNT);
	npages += 2 * howmany(npteds, VP_IDX3_CNT * VP_IDX2_CNT * VP_IDX1_CNT);

	/*
	 * Use an interrupt safe map such that we don't recurse into
	 * uvm_map() to allocate map entries.
	 */
	minaddr = vm_map_min(kernel_map);
	pmap_kvp_map = uvm_km_suballoc(kernel_map, &minaddr, &maxaddr,
	    npages * PAGE_SIZE, VM_MAP_INTRSAFE, FALSE, NULL);
}

void
pmap_init_percpu(void)
{
	pool_cache_init(&pmap_pted_pool);
	pool_cache_init(&pmap_vp_pool);
}

void
pmap_update(pmap_t pm)
{
}

int
pmap_is_referenced(struct vm_page *pg)
{
	return ((pg->pg_flags & PG_PMAP_REF) != 0);
}

int
pmap_is_modified(struct vm_page *pg)
{
	return ((pg->pg_flags & PG_PMAP_MOD) != 0);
}

int
pmap_clear_modify(struct vm_page *pg)
{
	struct pte_desc *pted;

	atomic_clearbits_int(&pg->pg_flags, PG_PMAP_MOD);

	mtx_enter(&pg->mdpage.pv_mtx);
	LIST_FOREACH(pted, &(pg->mdpage.pv_list), pted_pv_list) {
		pted->pted_pte &= ~PROT_WRITE;
		pmap_pte_insert(pted);
		ttlb_flush(pted->pted_pmap, pted->pted_va & ~PAGE_MASK);
	}
	mtx_leave(&pg->mdpage.pv_mtx);

	return 0;
}

/*
 * When this turns off read permissions it also disables write permissions
 * so that mod is correctly tracked after clear_ref; FAULT_READ; FAULT_WRITE;
 */
int
pmap_clear_reference(struct vm_page *pg)
{
	struct pte_desc *pted;

	atomic_clearbits_int(&pg->pg_flags, PG_PMAP_REF);

	mtx_enter(&pg->mdpage.pv_mtx);
	LIST_FOREACH(pted, &(pg->mdpage.pv_list), pted_pv_list) {
		pted->pted_pte &= ~PROT_MASK;
		pmap_pte_insert(pted);
		ttlb_flush(pted->pted_pmap, pted->pted_va & ~PAGE_MASK);
	}
	mtx_leave(&pg->mdpage.pv_mtx);

	return 0;
}

void
pmap_unwire(pmap_t pm, vaddr_t va)
{
	struct pte_desc *pted;

	pmap_lock(pm);
	pted = pmap_vp_lookup(pm, va, NULL);
	if (pted != NULL && PTED_WIRED(pted)) {
		pm->pm_stats.wired_count--;
		pted->pted_va &= ~PTED_VA_WIRED_M;
	}
	pmap_unlock(pm);
}

void
pmap_remove_holes(struct vmspace *vm)
{
	/* NOOP */
}

void
pmap_virtual_space(vaddr_t *start, vaddr_t *end)
{
	*start = virtual_avail;
	*end = VM_MAX_KERNEL_ADDRESS;

	/* Prevent further KVA stealing. */
	pmap_virtual_space_called = 1;
}

void
pmap_setup_avail(uint64_t ram_start, uint64_t ram_end, uint64_t kvo)
{
	/* This makes several assumptions
	 * 1) kernel will be located 'low' in memory
	 * 2) memory will not start at VM_MIN_KERNEL_ADDRESS
	 * 3) several MB of memory starting just after the kernel will
	 *    be premapped at the kernel address in the bootstrap mappings
	 * 4) kvo will be the 64 bit number to add to the ram address to
	 *    obtain the kernel virtual mapping of the ram. KVO == PA -> VA
	 * 5) it is generally assumed that these translations will occur with
	 *    large granularity, at minimum the translation will be page
	 *    aligned, if not 'section' or greater.
	 */

	pmap_avail_kvo = kvo;
	pmap_avail[0].start = ram_start;
	pmap_avail[0].size = ram_end-ram_start;

	/* XXX - multiple sections */
	physmem = atop(pmap_avail[0].size);

	pmap_cnt_avail = 1;

	pmap_avail_fixup();
}

void
pmap_avail_fixup(void)
{
	struct mem_region *mp;
	vaddr_t align;
	vaddr_t end;

	mp = pmap_avail;
	while(mp->size !=0) {
		align = round_page(mp->start);
		if (mp->start != align) {
			pmap_remove_avail(mp->start, align);
			mp = pmap_avail;
			continue;
		}
		end = mp->start+mp->size;
		align = trunc_page(end);
		if (end != align) {
			pmap_remove_avail(align, end);
			mp = pmap_avail;
			continue;
		}
		mp++;
	}
}

/* remove a given region from avail memory */
void
pmap_remove_avail(paddr_t base, paddr_t end)
{
	struct mem_region *mp;
	int i;
	long mpend;

	/* remove given region from available */
	for (mp = pmap_avail; mp->size; mp++) {
		/*
		 * Check if this region holds all of the region
		 */
		mpend = mp->start + mp->size;
		if (base > mpend) {
			continue;
		}
		if (base <= mp->start) {
			if (end <= mp->start)
				break; /* region not present -??? */

			if (end >= mpend) {
				/* covers whole region */
				/* shorten */
				for (i = mp - pmap_avail;
				    i < pmap_cnt_avail;
				    i++) {
					pmap_avail[i] = pmap_avail[i+1];
				}
				pmap_cnt_avail--;
				pmap_avail[pmap_cnt_avail].size = 0;
			} else {
				mp->start = end;
				mp->size = mpend - end;
			}
		} else {
			/* start after the beginning */
			if (end >= mpend) {
				/* just truncate */
				mp->size = base - mp->start;
			} else {
				/* split */
				for (i = pmap_cnt_avail;
				    i > (mp - pmap_avail);
				    i--) {
					pmap_avail[i] = pmap_avail[i - 1];
				}
				pmap_cnt_avail++;
				mp->size = base - mp->start;
				mp++;
				mp->start = end;
				mp->size = mpend - end;
			}
		}
	}
	for (mp = pmap_allocated; mp->size != 0; mp++) {
		if (base < mp->start) {
			if (end == mp->start) {
				mp->start = base;
				mp->size += end - base;
				break;
			}
			/* lengthen */
			for (i = pmap_cnt_allocated; i > (mp - pmap_allocated);
			    i--) {
				pmap_allocated[i] = pmap_allocated[i - 1];
			}
			pmap_cnt_allocated++;
			mp->start = base;
			mp->size = end - base;
			return;
		}
		if (base == (mp->start + mp->size)) {
			mp->size += end - base;
			return;
		}
	}
	if (mp->size == 0) {
		mp->start = base;
		mp->size  = end - base;
		pmap_cnt_allocated++;
	}
}

/* XXX - this zeros pages via their physical address */
paddr_t
pmap_steal_avail(size_t size, int align, void **kva)
{
	struct mem_region *mp;
	long start;
	long remsize;

	for (mp = pmap_avail; mp->size; mp++) {
		if (mp->size > size) {
			start = (mp->start + (align -1)) & ~(align -1);
			remsize = mp->size - (start - mp->start);
			if (remsize >= 0) {
				pmap_remove_avail(start, start+size);
				if (kva != NULL){
					*kva = (void *)(start - pmap_avail_kvo);
				}
				bzero((void*)(start), size);
				return start;
			}
		}
	}
	panic ("unable to allocate region with size %lx align %x",
	    size, align);
}

vaddr_t
pmap_map_stolen(vaddr_t kernel_start)
{
	struct mem_region *mp;
	paddr_t pa;
	vaddr_t va;
	uint64_t e;

	for (mp = pmap_allocated; mp->size; mp++) {
		for (e = 0; e < mp->size; e += PAGE_SIZE) {
			int prot = PROT_READ | PROT_WRITE;

			pa = mp->start + e;
			va = pa - pmap_avail_kvo;

			if (va < VM_MIN_KERNEL_ADDRESS ||
			    va >= VM_MAX_KERNEL_ADDRESS)
				continue;

			if (va >= (vaddr_t)__text_start &&
			    va < (vaddr_t)_etext)
				prot = PROT_READ | PROT_EXEC;
			else if (va >= (vaddr_t)__rodata_start &&
			    va < (vaddr_t)_erodata)
				prot = PROT_READ;

			pmap_kenter_cache(va, pa, prot, PMAP_CACHE_WB);
		}
	}

	return va + PAGE_SIZE;
}

void
pmap_physload_avail(void)
{
	struct mem_region *mp;
	uint64_t start, end;

	for (mp = pmap_avail; mp->size; mp++) {
		if (mp->size < PAGE_SIZE) {
			printf(" skipped - too small\n");
			continue;
		}
		start = mp->start;
		if (start & PAGE_MASK) {
			start = PAGE_SIZE + (start & PMAP_PA_MASK);
		}
		end = mp->start + mp->size;
		if (end & PAGE_MASK) {
			end = (end & PMAP_PA_MASK);
		}
		uvm_page_physload(atop(start), atop(end),
		    atop(start), atop(end), 0);

	}
}

void
pmap_show_mapping(uint64_t va)
{
	struct pmapvp1 *vp1;
	struct pmapvp2 *vp2;
	struct pmapvp3 *vp3;
	struct pte_desc *pted;
	struct pmap *pm;
	uint64_t ttbr0, tcr;

	printf("showing mapping of %llx\n", va);

	if (va & 1ULL << 63)
		pm = pmap_kernel();
	else
		pm = curproc->p_vmspace->vm_map.pmap;

	if (pm->have_4_level_pt) {
		printf("  vp0 = %p off %x\n",  pm->pm_vp.l0, VP_IDX0(va)*8);
		vp1 = pm->pm_vp.l0->vp[VP_IDX0(va)];
		if (vp1 == NULL)
			return;
	} else {
		vp1 = pm->pm_vp.l1;
	}

	__asm volatile ("mrs     %x0, ttbr0_el1" : "=r"(ttbr0));
	__asm volatile ("mrs     %x0, tcr_el1" : "=r"(tcr));
	printf("  ttbr0 %llx %llx tcr %llx\n", ttbr0, pm->pm_pt0pa, tcr);
	printf("  vp1 = %p\n", vp1);

	vp2 = vp1->vp[VP_IDX1(va)];
	printf("  vp2 = %p lp2 = %llx idx1 off %x\n",
		vp2, vp1->l1[VP_IDX1(va)], VP_IDX1(va)*8);
	if (vp2 == NULL)
		return;

	vp3 = vp2->vp[VP_IDX2(va)];
	printf("  vp3 = %p lp3 = %llx idx2 off %x\n",
		vp3, vp2->l2[VP_IDX2(va)], VP_IDX2(va)*8);
	if (vp3 == NULL)
		return;

	pted = vp3->vp[VP_IDX3(va)];
	printf("  pted = %p lp3 = %llx idx3 off  %x\n",
		pted, vp3->l3[VP_IDX3(va)], VP_IDX3(va)*8);
}

__attribute__((target("+pauth")))
void
pmap_setpauthkeys(struct pmap *pm)
{
	if (ID_AA64ISAR1_APA(cpu_id_aa64isar1) >= ID_AA64ISAR1_APA_PAC ||
	    ID_AA64ISAR1_API(cpu_id_aa64isar1) >= ID_AA64ISAR1_API_PAC) {
		__asm volatile ("msr apiakeylo_el1, %0"
		    :: "r"(pm->pm_apiakey[0]));
		__asm volatile ("msr apiakeyhi_el1, %0"
		    :: "r"(pm->pm_apiakey[1]));
		__asm volatile ("msr apdakeylo_el1, %0"
		    :: "r"(pm->pm_apdakey[0]));
		__asm volatile ("msr apdakeyhi_el1, %0"
		    :: "r"(pm->pm_apdakey[1]));
		__asm volatile ("msr apibkeylo_el1, %0"
		    :: "r"(pm->pm_apibkey[0]));
		__asm volatile ("msr apibkeyhi_el1, %0"
		    :: "r"(pm->pm_apibkey[1]));
		__asm volatile ("msr apdbkeylo_el1, %0"
		    :: "r"(pm->pm_apdbkey[0]));
		__asm volatile ("msr apdbkeyhi_el1, %0"
		    :: "r"(pm->pm_apdbkey[1]));
	}

	if (ID_AA64ISAR1_GPA(cpu_id_aa64isar1) >= ID_AA64ISAR1_GPA_IMPL ||
	    ID_AA64ISAR1_GPI(cpu_id_aa64isar1) >= ID_AA64ISAR1_GPI_IMPL) {
		__asm volatile ("msr apgakeylo_el1, %0"
		    :: "r"(pm->pm_apgakey[0]));
		__asm volatile ("msr apgakeyhi_el1, %0"
		    :: "r"(pm->pm_apgakey[1]));
	}
}

void
pmap_setttb(struct proc *p)
{
	struct cpu_info *ci = curcpu();
	pmap_t pm = p->p_vmspace->vm_map.pmap;

	/*
	 * If the generation of the ASID for the new pmap doesn't
	 * match the current generation, allocate a new ASID.
	 */
	if (pm != pmap_kernel() &&
	    (pm->pm_asid & ~PMAP_ASID_MASK) != READ_ONCE(pmap_asid_gen))
		pmap_allocate_asid(pm);

	if (pm != pmap_kernel())
		pmap_setpauthkeys(pm);

	WRITE_SPECIALREG(ttbr0_el1, pmap_kernel()->pm_pt0pa);
	__asm volatile("isb");
	cpu_setttb(pm->pm_asid, pm->pm_pt0pa);
	ci->ci_curpm = pm;
	ci->ci_flush_bp();
}