xref: /freebsd-src/sys/x86/iommu/iommu_utils.c (revision d97838b7c2a605932b6edf36f87abe7ccce74314)

/*-
 * SPDX-License-Identifier: BSD-2-Clause
 *
 * Copyright (c) 2013, 2014, 2024 The FreeBSD Foundation
 *
 * This software was developed by Konstantin Belousov <kib@FreeBSD.org>
 * under sponsorship from the FreeBSD Foundation.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 */

#include "opt_acpi.h"
#if defined(__amd64__)
#define	DEV_APIC
#else
#include "opt_apic.h"
#endif
#include "opt_ddb.h"

#include <sys/systm.h>
#include <sys/bus.h>
#include <sys/kernel.h>
#include <sys/lock.h>
#include <sys/malloc.h>
#include <sys/memdesc.h>
#include <sys/mutex.h>
#include <sys/sf_buf.h>
#include <sys/sysctl.h>
#include <sys/proc.h>
#include <sys/sched.h>
#include <sys/rman.h>
#include <sys/rwlock.h>
#include <sys/taskqueue.h>
#include <sys/tree.h>
#include <vm/vm.h>
#include <vm/vm_extern.h>
#include <vm/vm_kern.h>
#include <vm/vm_map.h>
#include <vm/vm_object.h>
#include <vm/vm_page.h>
#include <dev/pci/pcireg.h>
#include <dev/pci/pcivar.h>
#include <machine/atomic.h>
#include <machine/bus.h>
#include <machine/cpu.h>
#include <x86/include/busdma_impl.h>
#include <dev/iommu/busdma_iommu.h>
#include <dev/iommu/iommu.h>
#include <x86/iommu/x86_iommu.h>
#include <x86/iommu/iommu_intrmap.h>
#ifdef DEV_APIC
#include "pcib_if.h"
#include <machine/intr_machdep.h>
#include <x86/apicreg.h>
#include <x86/apicvar.h>
#endif

vm_page_t
iommu_pgalloc(vm_object_t obj, vm_pindex_t idx, int flags)
{
	vm_page_t m;
	int zeroed, aflags;

	zeroed = (flags & IOMMU_PGF_ZERO) != 0 ? VM_ALLOC_ZERO : 0;
	aflags = zeroed | VM_ALLOC_NOBUSY | VM_ALLOC_SYSTEM | VM_ALLOC_NODUMP |
	    ((flags & IOMMU_PGF_WAITOK) != 0 ? VM_ALLOC_WAITFAIL :
	    VM_ALLOC_NOWAIT);
	for (;;) {
		if ((flags & IOMMU_PGF_OBJL) == 0)
			VM_OBJECT_WLOCK(obj);
		m = vm_page_lookup(obj, idx);
		if ((flags & IOMMU_PGF_NOALLOC) != 0 || m != NULL) {
			if ((flags & IOMMU_PGF_OBJL) == 0)
				VM_OBJECT_WUNLOCK(obj);
			break;
		}
		m = vm_page_alloc_contig(obj, idx, aflags, 1, 0,
		    iommu_high, PAGE_SIZE, 0, VM_MEMATTR_DEFAULT);
		if ((flags & IOMMU_PGF_OBJL) == 0)
			VM_OBJECT_WUNLOCK(obj);
		if (m != NULL) {
			if (zeroed && (m->flags & PG_ZERO) == 0)
				pmap_zero_page(m);
			atomic_add_int(&iommu_tbl_pagecnt, 1);
			break;
		}
		if ((flags & IOMMU_PGF_WAITOK) == 0)
			break;
	}
	return (m);
}

void
iommu_pgfree(vm_object_t obj, vm_pindex_t idx, int flags,
    struct iommu_map_entry *entry)
{
	vm_page_t m;

	if ((flags & IOMMU_PGF_OBJL) == 0)
		VM_OBJECT_WLOCK(obj);
	m = vm_page_grab(obj, idx, VM_ALLOC_NOCREAT);
	if (m != NULL) {
		if (entry == NULL) {
			vm_page_free(m);
			atomic_subtract_int(&iommu_tbl_pagecnt, 1);
		} else {
			vm_page_remove_xbusy(m);	/* keep page busy */
			SLIST_INSERT_HEAD(&entry->pgtbl_free, m, plinks.s.ss);
		}
	}
	if ((flags & IOMMU_PGF_OBJL) == 0)
		VM_OBJECT_WUNLOCK(obj);
}

void *
iommu_map_pgtbl(vm_object_t obj, vm_pindex_t idx, int flags,
    struct sf_buf **sf)
{
	vm_page_t m;
	bool allocated;

	if ((flags & IOMMU_PGF_OBJL) == 0)
		VM_OBJECT_WLOCK(obj);
	m = vm_page_lookup(obj, idx);
	if (m == NULL && (flags & IOMMU_PGF_ALLOC) != 0) {
		m = iommu_pgalloc(obj, idx, flags | IOMMU_PGF_OBJL);
		allocated = true;
	} else
		allocated = false;
	if (m == NULL) {
		if ((flags & IOMMU_PGF_OBJL) == 0)
			VM_OBJECT_WUNLOCK(obj);
		return (NULL);
	}
	/* Sleepable allocations cannot fail. */
	if ((flags & IOMMU_PGF_WAITOK) != 0)
		VM_OBJECT_WUNLOCK(obj);
	sched_pin();
	*sf = sf_buf_alloc(m, SFB_CPUPRIVATE | ((flags & IOMMU_PGF_WAITOK)
	    == 0 ? SFB_NOWAIT : 0));
	if (*sf == NULL) {
		sched_unpin();
		if (allocated) {
			VM_OBJECT_ASSERT_WLOCKED(obj);
			iommu_pgfree(obj, m->pindex, flags | IOMMU_PGF_OBJL,
			    NULL);
		}
		if ((flags & IOMMU_PGF_OBJL) == 0)
			VM_OBJECT_WUNLOCK(obj);
		return (NULL);
	}
	if ((flags & (IOMMU_PGF_WAITOK | IOMMU_PGF_OBJL)) ==
	    (IOMMU_PGF_WAITOK | IOMMU_PGF_OBJL))
		VM_OBJECT_WLOCK(obj);
	else if ((flags & (IOMMU_PGF_WAITOK | IOMMU_PGF_OBJL)) == 0)
		VM_OBJECT_WUNLOCK(obj);
	return ((void *)sf_buf_kva(*sf));
}

void
iommu_unmap_pgtbl(struct sf_buf *sf)
{

	sf_buf_free(sf);
	sched_unpin();
}

iommu_haddr_t iommu_high;
int iommu_tbl_pagecnt;

SYSCTL_NODE(_hw_iommu, OID_AUTO, dmar, CTLFLAG_RD | CTLFLAG_MPSAFE,
    NULL, "");
SYSCTL_INT(_hw_iommu, OID_AUTO, tbl_pagecnt, CTLFLAG_RD,
    &iommu_tbl_pagecnt, 0,
    "Count of pages used for IOMMU pagetables");

int iommu_qi_batch_coalesce = 100;
SYSCTL_INT(_hw_iommu, OID_AUTO, batch_coalesce, CTLFLAG_RWTUN,
    &iommu_qi_batch_coalesce, 0,
    "Number of qi batches between interrupt");

static struct iommu_unit *
x86_no_iommu_find(device_t dev, bool verbose)
{
	return (NULL);
}

static int
x86_no_iommu_alloc_msi_intr(device_t src, u_int *cookies, u_int count)
{
	return (EOPNOTSUPP);
}

static int
x86_no_iommu_map_msi_intr(device_t src, u_int cpu, u_int vector,
    u_int cookie, uint64_t *addr, uint32_t *data)
{
	return (EOPNOTSUPP);
}

static int
x86_no_iommu_unmap_msi_intr(device_t src, u_int cookie)
{
	return (0);
}

static int
x86_no_iommu_map_ioapic_intr(u_int ioapic_id, u_int cpu, u_int vector,
    bool edge, bool activehi, int irq, u_int *cookie, uint32_t *hi,
    uint32_t *lo)
{
	return (EOPNOTSUPP);
}

static int
x86_no_iommu_unmap_ioapic_intr(u_int ioapic_id, u_int *cookie)
{
	return (0);
}

static struct x86_iommu x86_no_iommu = {
	.find = x86_no_iommu_find,
	.alloc_msi_intr = x86_no_iommu_alloc_msi_intr,
	.map_msi_intr = x86_no_iommu_map_msi_intr,
	.unmap_msi_intr = x86_no_iommu_unmap_msi_intr,
	.map_ioapic_intr = x86_no_iommu_map_ioapic_intr,
	.unmap_ioapic_intr = x86_no_iommu_unmap_ioapic_intr,
};

static struct x86_iommu *x86_iommu = &x86_no_iommu;

void
set_x86_iommu(struct x86_iommu *x)
{
	MPASS(x86_iommu == &x86_no_iommu);
	x86_iommu = x;
}

struct x86_iommu *
get_x86_iommu(void)
{
	return (x86_iommu);
}

void
iommu_domain_unload_entry(struct iommu_map_entry *entry, bool free,
    bool cansleep)
{
	x86_iommu->domain_unload_entry(entry, free, cansleep);
}

void
iommu_domain_unload(struct iommu_domain *iodom,
    struct iommu_map_entries_tailq *entries, bool cansleep)
{
	x86_iommu->domain_unload(iodom, entries, cansleep);
}

struct iommu_ctx *
iommu_get_ctx(struct iommu_unit *iommu, device_t dev, uint16_t rid,
    bool id_mapped, bool rmrr_init)
{
	return (x86_iommu->get_ctx(iommu, dev, rid, id_mapped, rmrr_init));
}

void
iommu_free_ctx_locked(struct iommu_unit *iommu, struct iommu_ctx *context)
{
	x86_iommu->free_ctx_locked(iommu, context);
}

struct iommu_unit *
iommu_find(device_t dev, bool verbose)
{
	return (x86_iommu->find(dev, verbose));
}

int
iommu_alloc_msi_intr(device_t src, u_int *cookies, u_int count)
{
	return (x86_iommu->alloc_msi_intr(src, cookies, count));
}

int
iommu_map_msi_intr(device_t src, u_int cpu, u_int vector, u_int cookie,
    uint64_t *addr, uint32_t *data)
{
	return (x86_iommu->map_msi_intr(src, cpu, vector, cookie,
	    addr, data));
}

int
iommu_unmap_msi_intr(device_t src, u_int cookie)
{
	return (x86_iommu->unmap_msi_intr(src, cookie));
}

int
iommu_map_ioapic_intr(u_int ioapic_id, u_int cpu, u_int vector, bool edge,
    bool activehi, int irq, u_int *cookie, uint32_t *hi, uint32_t *lo)
{
	return (x86_iommu->map_ioapic_intr(ioapic_id, cpu, vector, edge,
	    activehi, irq, cookie, hi, lo));
}

int
iommu_unmap_ioapic_intr(u_int ioapic_id, u_int *cookie)
{
	return (x86_iommu->unmap_ioapic_intr(ioapic_id, cookie));
}

void
iommu_unit_pre_instantiate_ctx(struct iommu_unit *unit)
{
	x86_iommu->unit_pre_instantiate_ctx(unit);
}

#define	IOMMU2X86C(iommu)	(x86_iommu->get_x86_common(iommu))

static bool
iommu_qi_seq_processed(struct iommu_unit *unit,
    const struct iommu_qi_genseq *pseq)
{
	struct x86_unit_common *x86c;
	u_int gen;

	x86c = IOMMU2X86C(unit);
	gen = x86c->inv_waitd_gen;
	return (pseq->gen < gen || (pseq->gen == gen && pseq->seq <=
	    atomic_load_64(&x86c->inv_waitd_seq_hw)));
}

void
iommu_qi_emit_wait_seq(struct iommu_unit *unit, struct iommu_qi_genseq *pseq,
    bool emit_wait)
{
	struct x86_unit_common *x86c;
	struct iommu_qi_genseq gsec;
	uint32_t seq;

	KASSERT(pseq != NULL, ("wait descriptor with no place for seq"));
	IOMMU_ASSERT_LOCKED(unit);
	x86c = IOMMU2X86C(unit);

	if (x86c->inv_waitd_seq == 0xffffffff) {
		gsec.gen = x86c->inv_waitd_gen;
		gsec.seq = x86c->inv_waitd_seq;
		x86_iommu->qi_ensure(unit, 1);
		x86_iommu->qi_emit_wait_descr(unit, gsec.seq, false,
		    true, false);
		x86_iommu->qi_advance_tail(unit);
		while (!iommu_qi_seq_processed(unit, &gsec))
			cpu_spinwait();
		x86c->inv_waitd_gen++;
		x86c->inv_waitd_seq = 1;
	}
	seq = x86c->inv_waitd_seq++;
	pseq->gen = x86c->inv_waitd_gen;
	pseq->seq = seq;
	if (emit_wait) {
		x86_iommu->qi_ensure(unit, 1);
		x86_iommu->qi_emit_wait_descr(unit, seq, true, true, false);
	}
}

/*
 * To avoid missed wakeups, callers must increment the unit's waiters count
 * before advancing the tail past the wait descriptor.
 */
void
iommu_qi_wait_for_seq(struct iommu_unit *unit, const struct iommu_qi_genseq *
    gseq, bool nowait)
{
	struct x86_unit_common *x86c;

	IOMMU_ASSERT_LOCKED(unit);
	x86c = IOMMU2X86C(unit);

	KASSERT(x86c->inv_seq_waiters > 0, ("%s: no waiters", __func__));
	while (!iommu_qi_seq_processed(unit, gseq)) {
		if (cold || nowait) {
			cpu_spinwait();
		} else {
			msleep(&x86c->inv_seq_waiters, &unit->lock, 0,
			    "dmarse", hz);
		}
	}
	x86c->inv_seq_waiters--;
}

/*
 * The caller must not be using the entry's dmamap_link field.
 */
void
iommu_qi_invalidate_locked(struct iommu_domain *domain,
    struct iommu_map_entry *entry, bool emit_wait)
{
	struct iommu_unit *unit;
	struct x86_unit_common *x86c;

	unit = domain->iommu;
	x86c = IOMMU2X86C(unit);
	IOMMU_ASSERT_LOCKED(unit);

	x86_iommu->qi_invalidate_emit(domain, entry->start, entry->end -
	    entry->start, &entry->gseq, emit_wait);

	/*
	 * To avoid a data race in dmar_qi_task(), the entry's gseq must be
	 * initialized before the entry is added to the TLB flush list, and the
	 * entry must be added to that list before the tail is advanced.  More
	 * precisely, the tail must not be advanced past the wait descriptor
	 * that will generate the interrupt that schedules dmar_qi_task() for
	 * execution before the entry is added to the list.  While an earlier
	 * call to dmar_qi_ensure() might have advanced the tail, it will not
	 * advance it past the wait descriptor.
	 *
	 * See the definition of struct dmar_unit for more information on
	 * synchronization.
	 */
	entry->tlb_flush_next = NULL;
	atomic_store_rel_ptr((uintptr_t *)&x86c->tlb_flush_tail->
	    tlb_flush_next, (uintptr_t)entry);
	x86c->tlb_flush_tail = entry;

	x86_iommu->qi_advance_tail(unit);
}

void
iommu_qi_invalidate_sync(struct iommu_domain *domain, iommu_gaddr_t base,
    iommu_gaddr_t size, bool cansleep)
{
	struct iommu_unit *unit;
	struct iommu_qi_genseq gseq;

	unit = domain->iommu;
	IOMMU_LOCK(unit);
	x86_iommu->qi_invalidate_emit(domain, base, size, &gseq, true);

	/*
	 * To avoid a missed wakeup in iommu_qi_task(), the unit's
	 * waiters count must be incremented before the tail is
	 * advanced.
	 */
	IOMMU2X86C(unit)->inv_seq_waiters++;

	x86_iommu->qi_advance_tail(unit);
	iommu_qi_wait_for_seq(unit, &gseq, !cansleep);
	IOMMU_UNLOCK(unit);
}

void
iommu_qi_drain_tlb_flush(struct iommu_unit *unit)
{
	struct x86_unit_common *x86c;
	struct iommu_map_entry *entry, *head;

	x86c = IOMMU2X86C(unit);
	for (head = x86c->tlb_flush_head;; head = entry) {
		entry = (struct iommu_map_entry *)
		    atomic_load_acq_ptr((uintptr_t *)&head->tlb_flush_next);
		if (entry == NULL ||
		    !iommu_qi_seq_processed(unit, &entry->gseq))
			break;
		x86c->tlb_flush_head = entry;
		iommu_gas_free_entry(head);
		if ((entry->flags & IOMMU_MAP_ENTRY_RMRR) != 0)
			iommu_gas_free_region(entry);
		else
			iommu_gas_free_space(entry);
	}
}

void
iommu_qi_common_init(struct iommu_unit *unit, task_fn_t qi_task)
{
	struct x86_unit_common *x86c;
	u_int qi_sz;

	x86c = IOMMU2X86C(unit);

	x86c->tlb_flush_head = x86c->tlb_flush_tail =
            iommu_gas_alloc_entry(NULL, 0);
	TASK_INIT(&x86c->qi_task, 0, qi_task, unit);
	x86c->qi_taskqueue = taskqueue_create_fast("iommuqf", M_WAITOK,
	    taskqueue_thread_enqueue, &x86c->qi_taskqueue);
	taskqueue_start_threads(&x86c->qi_taskqueue, 1, PI_AV,
	    "iommu%d qi taskq", unit->unit);

	x86c->inv_waitd_gen = 0;
	x86c->inv_waitd_seq = 1;

	qi_sz = 3;
	TUNABLE_INT_FETCH("hw.iommu.qi_size", &qi_sz);
	if (qi_sz > x86c->qi_buf_maxsz)
		qi_sz = x86c->qi_buf_maxsz;
	x86c->inv_queue_size = (1ULL << qi_sz) * PAGE_SIZE;
	/* Reserve one descriptor to prevent wraparound. */
	x86c->inv_queue_avail = x86c->inv_queue_size -
	    x86c->qi_cmd_sz;

	/*
	 * The invalidation queue reads by DMARs/AMDIOMMUs are always
	 * coherent.
	 */
	x86c->inv_queue = kmem_alloc_contig(x86c->inv_queue_size,
	    M_WAITOK | M_ZERO, 0, iommu_high, PAGE_SIZE, 0,
	    VM_MEMATTR_DEFAULT);
	x86c->inv_waitd_seq_hw_phys = pmap_kextract(
	    (vm_offset_t)&x86c->inv_waitd_seq_hw);
}

void
iommu_qi_common_fini(struct iommu_unit *unit, void (*disable_qi)(
    struct iommu_unit *))
{
	struct x86_unit_common *x86c;
	struct iommu_qi_genseq gseq;

	x86c = IOMMU2X86C(unit);

	taskqueue_drain(x86c->qi_taskqueue, &x86c->qi_task);
	taskqueue_free(x86c->qi_taskqueue);
	x86c->qi_taskqueue = NULL;

	IOMMU_LOCK(unit);
	/* quisce */
	x86_iommu->qi_ensure(unit, 1);
	iommu_qi_emit_wait_seq(unit, &gseq, true);
	/* See iommu_qi_invalidate_locked(). */
	x86c->inv_seq_waiters++;
	x86_iommu->qi_advance_tail(unit);
	iommu_qi_wait_for_seq(unit, &gseq, false);
	/* only after the quisce, disable queue */
	disable_qi(unit);
	KASSERT(x86c->inv_seq_waiters == 0,
	    ("iommu%d: waiters on disabled queue", unit->unit));
	IOMMU_UNLOCK(unit);

	kmem_free(x86c->inv_queue, x86c->inv_queue_size);
	x86c->inv_queue = NULL;
	x86c->inv_queue_size = 0;
}

int
iommu_alloc_irq(struct iommu_unit *unit, int idx)
{
	device_t dev, pcib;
	struct iommu_msi_data *dmd;
	uint64_t msi_addr;
	uint32_t msi_data;
	int error;

	MPASS(idx >= 0 || idx < IOMMU_MAX_MSI);

	dev = unit->dev;
	dmd = &IOMMU2X86C(unit)->intrs[idx];
	pcib = device_get_parent(device_get_parent(dev)); /* Really not pcib */
	error = PCIB_ALLOC_MSIX(pcib, dev, &dmd->irq);
	if (error != 0) {
		device_printf(dev, "cannot allocate %s interrupt, %d\n",
		    dmd->name, error);
		goto err1;
	}
	error = bus_set_resource(dev, SYS_RES_IRQ, dmd->irq_rid,
	    dmd->irq, 1);
	if (error != 0) {
		device_printf(dev, "cannot set %s interrupt resource, %d\n",
		    dmd->name, error);
		goto err2;
	}
	dmd->irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ,
	    &dmd->irq_rid, RF_ACTIVE);
	if (dmd->irq_res == NULL) {
		device_printf(dev,
		    "cannot allocate resource for %s interrupt\n", dmd->name);
		error = ENXIO;
		goto err3;
	}
	error = bus_setup_intr(dev, dmd->irq_res, INTR_TYPE_MISC,
	    dmd->handler, NULL, unit, &dmd->intr_handle);
	if (error != 0) {
		device_printf(dev, "cannot setup %s interrupt, %d\n",
		    dmd->name, error);
		goto err4;
	}
	bus_describe_intr(dev, dmd->irq_res, dmd->intr_handle, "%s", dmd->name);
	error = PCIB_MAP_MSI(pcib, dev, dmd->irq, &msi_addr, &msi_data);
	if (error != 0) {
		device_printf(dev, "cannot map %s interrupt, %d\n",
		    dmd->name, error);
		goto err5;
	}

	dmd->msi_data = msi_data;
	dmd->msi_addr = msi_addr;

	return (0);

err5:
	bus_teardown_intr(dev, dmd->irq_res, dmd->intr_handle);
err4:
	bus_release_resource(dev, SYS_RES_IRQ, dmd->irq_rid, dmd->irq_res);
err3:
	bus_delete_resource(dev, SYS_RES_IRQ, dmd->irq_rid);
err2:
	PCIB_RELEASE_MSIX(pcib, dev, dmd->irq);
	dmd->irq = -1;
err1:
	return (error);
}

void
iommu_release_intr(struct iommu_unit *unit, int idx)
{
	device_t dev;
	struct iommu_msi_data *dmd;

	MPASS(idx >= 0 || idx < IOMMU_MAX_MSI);

	dmd = &IOMMU2X86C(unit)->intrs[idx];
	if (dmd->handler == NULL || dmd->irq == -1)
		return;
	dev = unit->dev;

	bus_teardown_intr(dev, dmd->irq_res, dmd->intr_handle);
	bus_release_resource(dev, SYS_RES_IRQ, dmd->irq_rid, dmd->irq_res);
	bus_delete_resource(dev, SYS_RES_IRQ, dmd->irq_rid);
	PCIB_RELEASE_MSIX(device_get_parent(device_get_parent(dev)),
	    dev, dmd->irq);
	dmd->irq = -1;
}

void
iommu_device_tag_init(struct iommu_ctx *ctx, device_t dev)
{
	bus_addr_t maxaddr;

	maxaddr = MIN(ctx->domain->end, BUS_SPACE_MAXADDR);
	ctx->tag->common.impl = &bus_dma_iommu_impl;
	ctx->tag->common.boundary = 0;
	ctx->tag->common.lowaddr = maxaddr;
	ctx->tag->common.highaddr = maxaddr;
	ctx->tag->common.maxsize = maxaddr;
	ctx->tag->common.nsegments = BUS_SPACE_UNRESTRICTED;
	ctx->tag->common.maxsegsz = maxaddr;
	ctx->tag->ctx = ctx;
	ctx->tag->owner = dev;
}

void
iommu_domain_free_entry(struct iommu_map_entry *entry, bool free)
{
	if ((entry->flags & IOMMU_MAP_ENTRY_RMRR) != 0)
		iommu_gas_free_region(entry);
	else
		iommu_gas_free_space(entry);
	if (free)
		iommu_gas_free_entry(entry);
	else
		entry->flags = 0;
}

/*
 * Index of the pte for the guest address base in the page table at
 * the level lvl.
 */
int
pglvl_pgtbl_pte_off(int pglvl, iommu_gaddr_t base, int lvl)
{

	base >>= IOMMU_PAGE_SHIFT + (pglvl - lvl - 1) *
	    IOMMU_NPTEPGSHIFT;
	return (base & IOMMU_PTEMASK);
}

/*
 * Returns the page index of the page table page in the page table
 * object, which maps the given address base at the page table level
 * lvl.
 */
vm_pindex_t
pglvl_pgtbl_get_pindex(int pglvl, iommu_gaddr_t base, int lvl)
{
	vm_pindex_t idx, pidx;
	int i;

	KASSERT(lvl >= 0 && lvl < pglvl,
	    ("wrong lvl %d %d", pglvl, lvl));

	for (pidx = idx = 0, i = 0; i < lvl; i++, pidx = idx) {
		idx = pglvl_pgtbl_pte_off(pglvl, base, i) +
		    pidx * IOMMU_NPTEPG + 1;
	}
	return (idx);
}

/*
 * Calculate the total amount of page table pages needed to map the
 * whole bus address space on the context with the selected agaw.
 */
vm_pindex_t
pglvl_max_pages(int pglvl)
{
	vm_pindex_t res;
	int i;

	for (res = 0, i = pglvl; i > 0; i--) {
		res *= IOMMU_NPTEPG;
		res++;
	}
	return (res);
}

iommu_gaddr_t
pglvl_page_size(int total_pglvl, int lvl)
{
	int rlvl;
	static const iommu_gaddr_t pg_sz[] = {
		(iommu_gaddr_t)IOMMU_PAGE_SIZE,
		(iommu_gaddr_t)IOMMU_PAGE_SIZE << IOMMU_NPTEPGSHIFT,
		(iommu_gaddr_t)IOMMU_PAGE_SIZE << (2 * IOMMU_NPTEPGSHIFT),
		(iommu_gaddr_t)IOMMU_PAGE_SIZE << (3 * IOMMU_NPTEPGSHIFT),
		(iommu_gaddr_t)IOMMU_PAGE_SIZE << (4 * IOMMU_NPTEPGSHIFT),
		(iommu_gaddr_t)IOMMU_PAGE_SIZE << (5 * IOMMU_NPTEPGSHIFT),
		(iommu_gaddr_t)IOMMU_PAGE_SIZE << (6 * IOMMU_NPTEPGSHIFT),
	};

	KASSERT(lvl >= 0 && lvl < total_pglvl,
	    ("total %d lvl %d", total_pglvl, lvl));
	rlvl = total_pglvl - lvl - 1;
	KASSERT(rlvl < nitems(pg_sz), ("sizeof pg_sz lvl %d", lvl));
	return (pg_sz[rlvl]);
}

void
iommu_device_set_iommu_prop(device_t dev, device_t iommu)
{
	device_t iommu_dev;
	int error;

	bus_topo_lock();
	error = device_get_prop(dev, DEV_PROP_NAME_IOMMU, (void **)&iommu_dev);
	if (error == ENOENT)
		device_set_prop(dev, DEV_PROP_NAME_IOMMU, iommu, NULL, NULL);
	bus_topo_unlock();
}

#ifdef DDB
#include <ddb/ddb.h>
#include <ddb/db_lex.h>

void
iommu_db_print_domain_entry(const struct iommu_map_entry *entry)
{
	struct iommu_map_entry *l, *r;

	db_printf(
	    "    start %jx end %jx first %jx last %jx free_down %jx flags %x ",
	    entry->start, entry->end, entry->first, entry->last,
	    entry->free_down, entry->flags);
	db_printf("left ");
	l = RB_LEFT(entry, rb_entry);
	if (l == NULL)
		db_printf("NULL ");
	else
		db_printf("%jx ", l->start);
	db_printf("right ");
	r = RB_RIGHT(entry, rb_entry);
	if (r == NULL)
		db_printf("NULL");
	else
		db_printf("%jx", r->start);
	db_printf("\n");
}

void
iommu_db_print_ctx(struct iommu_ctx *ctx)
{
	db_printf(
	    "    @%p pci%d:%d:%d refs %d flags %#x loads %lu unloads %lu\n",
	    ctx, pci_get_bus(ctx->tag->owner),
	    pci_get_slot(ctx->tag->owner),
	    pci_get_function(ctx->tag->owner), ctx->refs,
	    ctx->flags, ctx->loads, ctx->unloads);
}

void
iommu_db_domain_print_contexts(struct iommu_domain *iodom)
{
	struct iommu_ctx *ctx;

	if (LIST_EMPTY(&iodom->contexts))
		return;

	db_printf("  Contexts:\n");
	LIST_FOREACH(ctx, &iodom->contexts, link)
		iommu_db_print_ctx(ctx);
}

void
iommu_db_domain_print_mappings(struct iommu_domain *iodom)
{
	struct iommu_map_entry *entry;

	db_printf("    mapped:\n");
	RB_FOREACH(entry, iommu_gas_entries_tree, &iodom->rb_root) {
		iommu_db_print_domain_entry(entry);
		if (db_pager_quit)
			break;
	}
	if (db_pager_quit)
		return;
	db_printf("    unloading:\n");
	TAILQ_FOREACH(entry, &iodom->unload_entries, dmamap_link) {
		iommu_db_print_domain_entry(entry);
		if (db_pager_quit)
			break;
	}
}

#endif