xref: /onnv-gate/usr/src/uts/i86pc/io/immu_dvma.c (revision 13050:515b1e9bea30)

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Portions Copyright (c) 2010, Oracle and/or its affiliates.
 * All rights reserved.
 */
/*
 * Copyright (c) 2009, Intel Corporation.
 * All rights reserved.
 */

/*
 * DVMA code
 * This file contains Intel IOMMU code that deals with DVMA
 * i.e. DMA remapping.
 */

#include <sys/sysmacros.h>
#include <sys/pcie.h>
#include <sys/pci_cfgspace.h>
#include <vm/hat_i86.h>
#include <sys/memlist.h>
#include <sys/acpi/acpi.h>
#include <sys/acpica.h>
#include <sys/modhash.h>
#include <sys/immu.h>
#include <sys/x86_archext.h>
#include <sys/archsystm.h>

#undef	TEST

/*
 * Macros based on PCI spec
 */
#define	IMMU_PCI_REV2CLASS(r)   ((r) >> 8)  /* classcode from revid */
#define	IMMU_PCI_CLASS2BASE(c)  ((c) >> 16) /* baseclass from classcode */
#define	IMMU_PCI_CLASS2SUB(c)   (((c) >> 8) & 0xff); /* classcode */

#define	IMMU_CONTIG_PADDR(d, p) \
	((d).dck_paddr && ((d).dck_paddr + IMMU_PAGESIZE) == (p))

typedef struct dvma_arg {
	immu_t *dva_immu;
	dev_info_t *dva_rdip;
	dev_info_t *dva_ddip;
	domain_t *dva_domain;
	int dva_level;
	immu_flags_t dva_flags;
	list_t *dva_list;
	int dva_error;
} dvma_arg_t;

static domain_t *domain_create(immu_t *immu, dev_info_t *ddip,
    dev_info_t *rdip, immu_flags_t immu_flags);
static immu_devi_t *create_immu_devi(dev_info_t *rdip, int bus,
    int dev, int func, immu_flags_t immu_flags);
static void destroy_immu_devi(immu_devi_t *immu_devi);
static boolean_t dvma_map(domain_t *domain, uint64_t sdvma,
    uint64_t nvpages, immu_dcookie_t *dcookies, int dcount, dev_info_t *rdip,
    immu_flags_t immu_flags);

/* Extern globals */
extern struct memlist  *phys_install;

/*
 * iommulib interface functions.
 */
static int immu_probe(iommulib_handle_t unitp, dev_info_t *dip);
static int immu_allochdl(iommulib_handle_t handle,
    dev_info_t *dip, dev_info_t *rdip, ddi_dma_attr_t *attr,
    int (*waitfp)(caddr_t), caddr_t arg, ddi_dma_handle_t *dma_handlep);
static int immu_freehdl(iommulib_handle_t handle,
    dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t dma_handle);
static int immu_bindhdl(iommulib_handle_t handle, dev_info_t *dip,
    dev_info_t *rdip, ddi_dma_handle_t dma_handle, struct ddi_dma_req *dma_req,
    ddi_dma_cookie_t *cookiep, uint_t *ccountp);
static int immu_unbindhdl(iommulib_handle_t handle,
    dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t dma_handle);
static int immu_sync(iommulib_handle_t handle, dev_info_t *dip,
    dev_info_t *rdip, ddi_dma_handle_t dma_handle, off_t off, size_t len,
    uint_t cachefl);
static int immu_win(iommulib_handle_t handle, dev_info_t *dip,
    dev_info_t *rdip, ddi_dma_handle_t dma_handle, uint_t win,
    off_t *offp, size_t *lenp, ddi_dma_cookie_t *cookiep, uint_t *ccountp);
static int immu_mapobject(iommulib_handle_t handle, dev_info_t *dip,
    dev_info_t *rdip, ddi_dma_handle_t dma_handle,
    struct ddi_dma_req *dmareq, ddi_dma_obj_t *dmao);
static int immu_unmapobject(iommulib_handle_t handle, dev_info_t *dip,
    dev_info_t *rdip, ddi_dma_handle_t dma_handle, ddi_dma_obj_t *dmao);
static int immu_map(iommulib_handle_t handle, dev_info_t *dip,
    dev_info_t *rdip, struct ddi_dma_req *dmareq,
    ddi_dma_handle_t *dma_handle);
static int immu_mctl(iommulib_handle_t handle, dev_info_t *dip,
    dev_info_t *rdip, ddi_dma_handle_t dma_handle,
    enum ddi_dma_ctlops request, off_t *offp, size_t *lenp,
    caddr_t *objpp, uint_t cachefl);

/* static Globals */

/*
 * Used to setup DMA objects (memory regions)
 * for DMA reads by IOMMU units
 */
static ddi_dma_attr_t immu_dma_attr = {
	DMA_ATTR_V0,
	0U,
	0xffffffffffffffffULL,
	0xffffffffU,
	MMU_PAGESIZE, /* MMU page aligned */
	0x1,
	0x1,
	0xffffffffU,
	0xffffffffffffffffULL,
	1,
	4,
	0
};

static ddi_device_acc_attr_t immu_acc_attr = {
	DDI_DEVICE_ATTR_V0,
	DDI_NEVERSWAP_ACC,
	DDI_STRICTORDER_ACC
};

struct iommulib_ops immulib_ops = {
	IOMMU_OPS_VERSION,
	INTEL_IOMMU,
	"Intel IOMMU",
	NULL,
	immu_probe,
	immu_allochdl,
	immu_freehdl,
	immu_bindhdl,
	immu_unbindhdl,
	immu_sync,
	immu_win,
	immu_mapobject,
	immu_unmapobject,
	immu_map,
	immu_mctl
};

/*
 * Fake physical address range used to set up initial prealloc mappings.
 * This memory is never actually accessed. It is mapped read-only,
 * and is overwritten as soon as the first DMA bind operation is
 * performed. Since 0 is a special case, just start at the 2nd
 * physical page.
 */

static immu_dcookie_t immu_precookie = { MMU_PAGESIZE, IMMU_NPREPTES };

/* globals private to this file */
static kmutex_t immu_domain_lock;
static list_t immu_unity_domain_list;
static list_t immu_xlate_domain_list;

/* structure used to store idx into each level of the page tables */
typedef struct xlate {
	int xlt_level;
	uint_t xlt_idx;
	pgtable_t *xlt_pgtable;
} xlate_t;

/* 0 is reserved by Vt-d spec. Solaris reserves 1 */
#define	IMMU_UNITY_DID   1

static mod_hash_t *bdf_domain_hash;

int immu_use_alh;
int immu_use_tm;

static domain_t *
bdf_domain_lookup(immu_devi_t *immu_devi)
{
	domain_t *domain;
	int16_t seg = immu_devi->imd_seg;
	int16_t bus = immu_devi->imd_bus;
	int16_t devfunc = immu_devi->imd_devfunc;
	uintptr_t bdf = (seg << 16 | bus << 8 | devfunc);

	if (seg < 0 || bus < 0 || devfunc < 0) {
		return (NULL);
	}

	domain = NULL;
	if (mod_hash_find(bdf_domain_hash,
	    (void *)bdf, (void *)&domain) == 0) {
		ASSERT(domain);
		ASSERT(domain->dom_did > 0);
		return (domain);
	} else {
		return (NULL);
	}
}

static void
bdf_domain_insert(immu_devi_t *immu_devi, domain_t *domain)
{
	int16_t seg = immu_devi->imd_seg;
	int16_t bus = immu_devi->imd_bus;
	int16_t devfunc = immu_devi->imd_devfunc;
	uintptr_t bdf = (seg << 16 | bus << 8 | devfunc);

	if (seg < 0 || bus < 0 || devfunc < 0) {
		return;
	}

	(void) mod_hash_insert(bdf_domain_hash, (void *)bdf, (void *)domain);
}

static int
match_lpc(dev_info_t *pdip, void *arg)
{
	immu_devi_t *immu_devi;
	dvma_arg_t *dvap = (dvma_arg_t *)arg;

	if (list_is_empty(dvap->dva_list)) {
		return (DDI_WALK_TERMINATE);
	}

	immu_devi = list_head(dvap->dva_list);
	for (; immu_devi; immu_devi = list_next(dvap->dva_list,
	    immu_devi)) {
		if (immu_devi->imd_dip == pdip) {
			dvap->dva_ddip = pdip;
			dvap->dva_error = DDI_SUCCESS;
			return (DDI_WALK_TERMINATE);
		}
	}

	return (DDI_WALK_CONTINUE);
}

static void
immu_devi_set_spclist(dev_info_t *dip, immu_t *immu)
{
	list_t *spclist = NULL;
	immu_devi_t *immu_devi;

	immu_devi = IMMU_DEVI(dip);
	if (immu_devi->imd_display == B_TRUE) {
		spclist = &(immu->immu_dvma_gfx_list);
	} else if (immu_devi->imd_lpc == B_TRUE) {
		spclist = &(immu->immu_dvma_lpc_list);
	}

	if (spclist) {
		mutex_enter(&(immu->immu_lock));
		list_insert_head(spclist, immu_devi);
		mutex_exit(&(immu->immu_lock));
	}
}

/*
 * Set the immu_devi struct in the immu_devi field of a devinfo node
 */
int
immu_devi_set(dev_info_t *dip, immu_flags_t immu_flags)
{
	int bus, dev, func;
	immu_devi_t *new_imd;
	immu_devi_t *immu_devi;

	immu_devi = immu_devi_get(dip);
	if (immu_devi != NULL) {
		return (DDI_SUCCESS);
	}

	bus = dev = func = -1;

	/*
	 * Assume a new immu_devi struct is needed
	 */
	if (!DEVI_IS_PCI(dip) || acpica_get_bdf(dip, &bus, &dev, &func) != 0) {
		/*
		 * No BDF. Set bus = -1 to indicate this.
		 * We still need to create a immu_devi struct
		 * though
		 */
		bus = -1;
		dev = 0;
		func = 0;
	}

	new_imd = create_immu_devi(dip, bus, dev, func, immu_flags);
	if (new_imd  == NULL) {
		ddi_err(DER_WARN, dip, "Failed to create immu_devi "
		    "structure");
		return (DDI_FAILURE);
	}

	/*
	 * Check if some other thread allocated a immu_devi while we
	 * didn't own the lock.
	 */
	mutex_enter(&(DEVI(dip)->devi_lock));
	if (IMMU_DEVI(dip) == NULL) {
		IMMU_DEVI_SET(dip, new_imd);
	} else {
		destroy_immu_devi(new_imd);
	}
	mutex_exit(&(DEVI(dip)->devi_lock));

	return (DDI_SUCCESS);
}

static dev_info_t *
get_lpc_devinfo(immu_t *immu, dev_info_t *rdip, immu_flags_t immu_flags)
{
	dvma_arg_t dvarg = {0};
	dvarg.dva_list = &(immu->immu_dvma_lpc_list);
	dvarg.dva_rdip = rdip;
	dvarg.dva_error = DDI_FAILURE;

	if (immu_walk_ancestor(rdip, NULL, match_lpc,
	    &dvarg, NULL, immu_flags) != DDI_SUCCESS) {
		ddi_err(DER_MODE, rdip, "Could not walk ancestors to "
		    "find lpc_devinfo for ISA device");
		return (NULL);
	}

	if (dvarg.dva_error != DDI_SUCCESS || dvarg.dva_ddip == NULL) {
		ddi_err(DER_MODE, rdip, "Could not find lpc_devinfo for "
		    "ISA device");
		return (NULL);
	}

	return (dvarg.dva_ddip);
}

static dev_info_t *
get_gfx_devinfo(dev_info_t *rdip)
{
	immu_t *immu;
	immu_devi_t *immu_devi;
	list_t *list_gfx;

	/*
	 * The GFX device may not be on the same iommu unit as "agpgart"
	 * so search globally
	 */
	immu_devi = NULL;
	immu = list_head(&immu_list);
	for (; immu; immu = list_next(&immu_list, immu)) {
		list_gfx = &(immu->immu_dvma_gfx_list);
		if (!list_is_empty(list_gfx)) {
			immu_devi = list_head(list_gfx);
			break;
		}
	}

	if (immu_devi == NULL) {
		ddi_err(DER_WARN, rdip, "iommu: No GFX device. "
		    "Cannot redirect agpgart");
		return (NULL);
	}

	ddi_err(DER_LOG, rdip, "iommu: GFX redirect to %s",
	    ddi_node_name(immu_devi->imd_dip));

	return (immu_devi->imd_dip);
}

static immu_flags_t
dma_to_immu_flags(struct ddi_dma_req *dmareq)
{
	immu_flags_t flags = 0;

	if (dmareq->dmar_fp == DDI_DMA_SLEEP) {
		flags |= IMMU_FLAGS_SLEEP;
	} else {
		flags |= IMMU_FLAGS_NOSLEEP;
	}

#ifdef BUGGY_DRIVERS

	flags |= (IMMU_FLAGS_READ | IMMU_FLAGS_WRITE);

#else
	/*
	 * Read and write flags need to be reversed.
	 * DMA_READ means read from device and write
	 * to memory. So DMA read means DVMA write.
	 */
	if (dmareq->dmar_flags & DDI_DMA_READ)
		flags |= IMMU_FLAGS_WRITE;

	if (dmareq->dmar_flags & DDI_DMA_WRITE)
		flags |= IMMU_FLAGS_READ;

	/*
	 * Some buggy drivers specify neither READ or WRITE
	 * For such drivers set both read and write permissions
	 */
	if ((dmareq->dmar_flags & (DDI_DMA_READ | DDI_DMA_WRITE)) == 0) {
		flags |= (IMMU_FLAGS_READ | IMMU_FLAGS_WRITE);
	}
#endif

	return (flags);
}

/*ARGSUSED*/
int
pgtable_ctor(void *buf, void *arg, int kmflag)
{
	size_t actual_size = 0;
	pgtable_t *pgtable;
	int (*dmafp)(caddr_t);
	caddr_t vaddr;
	void *next;
	uint_t flags;
	immu_t *immu = arg;

	pgtable = (pgtable_t *)buf;

	dmafp = (kmflag & KM_NOSLEEP) ? DDI_DMA_DONTWAIT : DDI_DMA_SLEEP;

	next = kmem_zalloc(IMMU_PAGESIZE, kmflag);
	if (next == NULL) {
		return (-1);
	}

	if (ddi_dma_alloc_handle(root_devinfo, &immu_dma_attr,
	    dmafp, NULL, &pgtable->hwpg_dmahdl) != DDI_SUCCESS) {
		kmem_free(next, IMMU_PAGESIZE);
		return (-1);
	}

	flags = DDI_DMA_CONSISTENT;
	if (!immu->immu_dvma_coherent)
		flags |= IOMEM_DATA_UC_WR_COMBINE;

	if (ddi_dma_mem_alloc(pgtable->hwpg_dmahdl, IMMU_PAGESIZE,
	    &immu_acc_attr, flags,
	    dmafp, NULL, &vaddr, &actual_size,
	    &pgtable->hwpg_memhdl) != DDI_SUCCESS) {
		ddi_dma_free_handle(&pgtable->hwpg_dmahdl);
		kmem_free(next, IMMU_PAGESIZE);
		return (-1);
	}

	/*
	 * Memory allocation failure. Maybe a temporary condition
	 * so return error rather than panic, so we can try again
	 */
	if (actual_size < IMMU_PAGESIZE) {
		ddi_dma_mem_free(&pgtable->hwpg_memhdl);
		ddi_dma_free_handle(&pgtable->hwpg_dmahdl);
		kmem_free(next, IMMU_PAGESIZE);
		return (-1);
	}

	pgtable->hwpg_paddr = pfn_to_pa(hat_getpfnum(kas.a_hat, vaddr));
	pgtable->hwpg_vaddr = vaddr;
	pgtable->swpg_next_array = next;

	rw_init(&(pgtable->swpg_rwlock), NULL, RW_DEFAULT, NULL);

	return (0);
}

/*ARGSUSED*/
void
pgtable_dtor(void *buf, void *arg)
{
	pgtable_t *pgtable;

	pgtable = (pgtable_t *)buf;

	/* destroy will panic if lock is held. */
	rw_destroy(&(pgtable->swpg_rwlock));

	ddi_dma_mem_free(&pgtable->hwpg_memhdl);
	ddi_dma_free_handle(&pgtable->hwpg_dmahdl);
	kmem_free(pgtable->swpg_next_array, IMMU_PAGESIZE);
}

/*
 * pgtable_alloc()
 *	alloc a IOMMU pgtable structure.
 *	This same struct is used for root and context tables as well.
 *	This routine allocs the f/ollowing:
 *	- a pgtable_t struct
 *	- a HW page which holds PTEs/entries which is accesssed by HW
 *        so we set up DMA for this page
 *	- a SW page which is only for our bookeeping
 *        (for example to  hold pointers to the next level pgtable).
 *        So a simple kmem_alloc suffices
 */
static pgtable_t *
pgtable_alloc(immu_t *immu, immu_flags_t immu_flags)
{
	pgtable_t *pgtable;
	int kmflags;

	kmflags = (immu_flags & IMMU_FLAGS_NOSLEEP) ? KM_NOSLEEP : KM_SLEEP;

	pgtable = kmem_cache_alloc(immu->immu_pgtable_cache, kmflags);
	if (pgtable == NULL) {
		return (NULL);
	}
	return (pgtable);
}

static void
pgtable_zero(pgtable_t *pgtable)
{
	bzero(pgtable->hwpg_vaddr, IMMU_PAGESIZE);
	bzero(pgtable->swpg_next_array, IMMU_PAGESIZE);
}

static void
pgtable_free(immu_t *immu, pgtable_t *pgtable)
{
	kmem_cache_free(immu->immu_pgtable_cache, pgtable);
}

/*
 * Function to identify a display device from the PCI class code
 */
static boolean_t
device_is_display(uint_t classcode)
{
	static uint_t disp_classes[] = {
		0x000100,
		0x030000,
		0x030001
	};
	int i, nclasses = sizeof (disp_classes) / sizeof (uint_t);

	for (i = 0; i < nclasses; i++) {
		if (classcode == disp_classes[i])
			return (B_TRUE);
	}
	return (B_FALSE);
}

/*
 * Function that determines if device is PCIEX and/or PCIEX bridge
 */
static boolean_t
device_is_pciex(
	uchar_t bus, uchar_t dev, uchar_t func, boolean_t *is_pcib)
{
	ushort_t cap;
	ushort_t capsp;
	ushort_t cap_count = PCI_CAP_MAX_PTR;
	ushort_t status;
	boolean_t is_pciex = B_FALSE;

	*is_pcib = B_FALSE;

	status = pci_getw_func(bus, dev, func, PCI_CONF_STAT);
	if (!(status & PCI_STAT_CAP))
		return (B_FALSE);

	capsp = pci_getb_func(bus, dev, func, PCI_CONF_CAP_PTR);
	while (cap_count-- && capsp >= PCI_CAP_PTR_OFF) {
		capsp &= PCI_CAP_PTR_MASK;
		cap = pci_getb_func(bus, dev, func, capsp);

		if (cap == PCI_CAP_ID_PCI_E) {
			status = pci_getw_func(bus, dev, func, capsp + 2);
			/*
			 * See section 7.8.2 of PCI-Express Base Spec v1.0a
			 * for Device/Port Type.
			 * PCIE_PCIECAP_DEV_TYPE_PCIE2PCI implies that the
			 * device is a PCIE2PCI bridge
			 */
			*is_pcib =
			    ((status & PCIE_PCIECAP_DEV_TYPE_MASK) ==
			    PCIE_PCIECAP_DEV_TYPE_PCIE2PCI) ? B_TRUE : B_FALSE;
			is_pciex = B_TRUE;
		}

		capsp = (*pci_getb_func)(bus, dev, func,
		    capsp + PCI_CAP_NEXT_PTR);
	}

	return (is_pciex);
}

static boolean_t
device_use_premap(uint_t classcode)
{
	if (IMMU_PCI_CLASS2BASE(classcode) == PCI_CLASS_NET)
		return (B_TRUE);
	return (B_FALSE);
}


/*
 * immu_dvma_get_immu()
 *   get the immu unit structure for a dev_info node
 */
immu_t *
immu_dvma_get_immu(dev_info_t *dip, immu_flags_t immu_flags)
{
	immu_devi_t *immu_devi;
	immu_t *immu;

	/*
	 * check if immu unit was already found earlier.
	 * If yes, then it will be stashed in immu_devi struct.
	 */
	immu_devi = immu_devi_get(dip);
	if (immu_devi == NULL) {
		if (immu_devi_set(dip, immu_flags) != DDI_SUCCESS) {
			/*
			 * May fail because of low memory. Return error rather
			 * than panic as we want driver to rey again later
			 */
			ddi_err(DER_PANIC, dip, "immu_dvma_get_immu: "
			    "No immu_devi structure");
			/*NOTREACHED*/
		}
		immu_devi = immu_devi_get(dip);
	}

	mutex_enter(&(DEVI(dip)->devi_lock));
	if (immu_devi->imd_immu) {
		immu = immu_devi->imd_immu;
		mutex_exit(&(DEVI(dip)->devi_lock));
		return (immu);
	}
	mutex_exit(&(DEVI(dip)->devi_lock));

	immu = immu_dmar_get_immu(dip);
	if (immu == NULL) {
		ddi_err(DER_PANIC, dip, "immu_dvma_get_immu: "
		    "Cannot find immu_t for device");
		/*NOTREACHED*/
	}

	/*
	 * Check if some other thread found immu
	 * while lock was not held
	 */
	immu_devi = immu_devi_get(dip);
	/* immu_devi should be present as we found it earlier */
	if (immu_devi == NULL) {
		ddi_err(DER_PANIC, dip,
		    "immu_dvma_get_immu: No immu_devi structure");
		/*NOTREACHED*/
	}

	mutex_enter(&(DEVI(dip)->devi_lock));
	if (immu_devi->imd_immu == NULL) {
		/* nobody else set it, so we should do it */
		immu_devi->imd_immu = immu;
		immu_devi_set_spclist(dip, immu);
	} else {
		/*
		 * if some other thread got immu before
		 * us, it should get the same results
		 */
		if (immu_devi->imd_immu != immu) {
			ddi_err(DER_PANIC, dip, "Multiple "
			    "immu units found for device. Expected (%p), "
			    "actual (%p)", (void *)immu,
			    (void *)immu_devi->imd_immu);
			mutex_exit(&(DEVI(dip)->devi_lock));
			/*NOTREACHED*/
		}
	}
	mutex_exit(&(DEVI(dip)->devi_lock));

	return (immu);
}


/* ############################# IMMU_DEVI code ############################ */

/*
 * Allocate a immu_devi structure and initialize it
 */
static immu_devi_t *
create_immu_devi(dev_info_t *rdip, int bus, int dev, int func,
    immu_flags_t immu_flags)
{
	uchar_t baseclass, subclass;
	uint_t classcode, revclass;
	immu_devi_t *immu_devi;
	boolean_t pciex = B_FALSE;
	int kmflags;
	boolean_t is_pcib = B_FALSE;

	/* bus ==  -1 indicate non-PCI device (no BDF) */
	ASSERT(bus == -1 || bus >= 0);
	ASSERT(dev >= 0);
	ASSERT(func >= 0);

	kmflags = (immu_flags & IMMU_FLAGS_NOSLEEP) ? KM_NOSLEEP : KM_SLEEP;
	immu_devi = kmem_zalloc(sizeof (immu_devi_t), kmflags);
	if (immu_devi == NULL) {
		ddi_err(DER_WARN, rdip, "Failed to allocate memory for "
		    "Intel IOMMU immu_devi structure");
		return (NULL);
	}
	immu_devi->imd_dip = rdip;
	immu_devi->imd_seg = 0; /* Currently seg can only be 0 */
	immu_devi->imd_bus = bus;
	immu_devi->imd_pcib_type = IMMU_PCIB_BAD;

	if (bus == -1) {
		immu_devi->imd_pcib_type = IMMU_PCIB_NOBDF;
		return (immu_devi);
	}

	immu_devi->imd_devfunc = IMMU_PCI_DEVFUNC(dev, func);
	immu_devi->imd_sec = 0;
	immu_devi->imd_sub = 0;

	revclass = pci_getl_func(bus, dev, func, PCI_CONF_REVID);

	classcode = IMMU_PCI_REV2CLASS(revclass);
	baseclass = IMMU_PCI_CLASS2BASE(classcode);
	subclass = IMMU_PCI_CLASS2SUB(classcode);

	if (baseclass == PCI_CLASS_BRIDGE && subclass == PCI_BRIDGE_PCI) {

		immu_devi->imd_sec = pci_getb_func(bus, dev, func,
		    PCI_BCNF_SECBUS);
		immu_devi->imd_sub = pci_getb_func(bus, dev, func,
		    PCI_BCNF_SUBBUS);

		pciex = device_is_pciex(bus, dev, func, &is_pcib);
		if (pciex  == B_TRUE && is_pcib == B_TRUE) {
			immu_devi->imd_pcib_type = IMMU_PCIB_PCIE_PCI;
		} else if (pciex == B_TRUE) {
			immu_devi->imd_pcib_type = IMMU_PCIB_PCIE_PCIE;
		} else {
			immu_devi->imd_pcib_type = IMMU_PCIB_PCI_PCI;
		}
	} else {
		immu_devi->imd_pcib_type = IMMU_PCIB_ENDPOINT;
	}

	/* check for certain special devices */
	immu_devi->imd_display = device_is_display(classcode);
	immu_devi->imd_lpc = ((baseclass == PCI_CLASS_BRIDGE) &&
	    (subclass == PCI_BRIDGE_ISA)) ? B_TRUE : B_FALSE;
	immu_devi->imd_use_premap = device_use_premap(classcode);

	immu_devi->imd_domain = NULL;

	immu_devi->imd_dvma_flags = immu_global_dvma_flags;

	return (immu_devi);
}

static void
destroy_immu_devi(immu_devi_t *immu_devi)
{
	kmem_free(immu_devi, sizeof (immu_devi_t));
}

static domain_t *
immu_devi_domain(dev_info_t *rdip, dev_info_t **ddipp)
{
	immu_devi_t *immu_devi;
	domain_t *domain;
	dev_info_t *ddip;

	*ddipp = NULL;

	immu_devi = immu_devi_get(rdip);
	if (immu_devi == NULL) {
		return (NULL);
	}

	mutex_enter(&(DEVI(rdip)->devi_lock));
	domain = immu_devi->imd_domain;
	ddip = immu_devi->imd_ddip;
	mutex_exit(&(DEVI(rdip)->devi_lock));

	if (domain)
		*ddipp = ddip;

	return (domain);

}

/* ############################# END IMMU_DEVI code ######################## */
/* ############################# DOMAIN code ############################### */

/*
 * This routine always succeeds
 */
static int
did_alloc(immu_t *immu, dev_info_t *rdip,
    dev_info_t *ddip, immu_flags_t immu_flags)
{
	int did;

	did = (uintptr_t)vmem_alloc(immu->immu_did_arena, 1,
	    (immu_flags & IMMU_FLAGS_NOSLEEP) ? VM_NOSLEEP : VM_SLEEP);

	if (did == 0) {
		ddi_err(DER_WARN, rdip, "device domain-id alloc error"
		    " domain-device: %s%d. immu unit is %s. Using "
		    "unity domain with domain-id (%d)",
		    ddi_driver_name(ddip), ddi_get_instance(ddip),
		    immu->immu_name, immu->immu_unity_domain->dom_did);
		did = immu->immu_unity_domain->dom_did;
	}

	return (did);
}

static int
get_branch_domain(dev_info_t *pdip, void *arg)
{
	immu_devi_t *immu_devi;
	domain_t *domain;
	dev_info_t *ddip;
	immu_t *immu;
	dvma_arg_t *dvp = (dvma_arg_t *)arg;

	/*
	 * The field dvp->dva_rdip is a work-in-progress
	 * and gets updated as we walk up the ancestor
	 * tree. The final ddip is set only when we reach
	 * the top of the tree. So the dvp->dva_ddip field cannot
	 * be relied on until we reach the top of the field.
	 */

	/* immu_devi may not be set. */
	immu_devi = immu_devi_get(pdip);
	if (immu_devi == NULL) {
		if (immu_devi_set(pdip, dvp->dva_flags) != DDI_SUCCESS) {
			dvp->dva_error = DDI_FAILURE;
			return (DDI_WALK_TERMINATE);
		}
	}

	immu_devi = immu_devi_get(pdip);
	immu = immu_devi->imd_immu;
	if (immu == NULL)
		immu = immu_dvma_get_immu(pdip, dvp->dva_flags);

	/*
	 * If we encounter a PCIE_PCIE bridge *ANCESTOR* we need to
	 * terminate the walk (since the device under the PCIE bridge
	 * is a PCIE device and has an independent entry in the
	 * root/context table)
	 */
	if (dvp->dva_rdip != pdip &&
	    immu_devi->imd_pcib_type == IMMU_PCIB_PCIE_PCIE) {
		return (DDI_WALK_TERMINATE);
	}

	/*
	 * In order to be a domain-dim, it must be a PCI device i.e.
	 * must have valid BDF. This also eliminates the root complex.
	 */
	if (immu_devi->imd_pcib_type != IMMU_PCIB_BAD &&
	    immu_devi->imd_pcib_type != IMMU_PCIB_NOBDF) {
		ASSERT(immu_devi->imd_bus >= 0);
		ASSERT(immu_devi->imd_devfunc >= 0);
		dvp->dva_ddip = pdip;
	}

	if (immu_devi->imd_display == B_TRUE ||
	    (dvp->dva_flags & IMMU_FLAGS_UNITY)) {
		dvp->dva_domain = immu->immu_unity_domain;
		/* continue walking to find ddip */
		return (DDI_WALK_CONTINUE);
	}

	mutex_enter(&(DEVI(pdip)->devi_lock));
	domain = immu_devi->imd_domain;
	ddip = immu_devi->imd_ddip;
	mutex_exit(&(DEVI(pdip)->devi_lock));

	if (domain && ddip) {
		/* if domain is set, it must be the same */
		if (dvp->dva_domain) {
			ASSERT(domain == dvp->dva_domain);
		}
		dvp->dva_domain = domain;
		dvp->dva_ddip = ddip;
		return (DDI_WALK_TERMINATE);
	}

	/* Domain may already be set, continue walking so that ddip gets set */
	if (dvp->dva_domain) {
		return (DDI_WALK_CONTINUE);
	}

	/* domain is not set in either immu_devi or dvp */
	domain = bdf_domain_lookup(immu_devi);
	if (domain == NULL) {
		return (DDI_WALK_CONTINUE);
	}

	/* ok, the BDF hash had a domain for this BDF. */

	/* Grab lock again to check if something else set immu_devi fields */
	mutex_enter(&(DEVI(pdip)->devi_lock));
	if (immu_devi->imd_domain != NULL) {
		dvp->dva_domain = domain;
	} else {
		dvp->dva_domain = domain;
	}
	mutex_exit(&(DEVI(pdip)->devi_lock));

	/*
	 * walk upwards until the topmost PCI bridge is found
	 */
	return (DDI_WALK_CONTINUE);

}

static void
map_unity_domain(domain_t *domain)
{
	struct memlist *mp;
	uint64_t start;
	uint64_t npages;
	immu_dcookie_t dcookies[1] = {0};
	int dcount = 0;

	/*
	 * UNITY arenas are a mirror of the physical memory
	 * installed on the system.
	 */

#ifdef BUGGY_DRIVERS
	/*
	 * Dont skip page0. Some broken HW/FW access it.
	 */
	dcookies[0].dck_paddr = 0;
	dcookies[0].dck_npages = 1;
	dcount = 1;
	(void) dvma_map(domain, 0, 1, dcookies, dcount, NULL,
	    IMMU_FLAGS_READ | IMMU_FLAGS_WRITE | IMMU_FLAGS_PAGE1);
#endif

	memlist_read_lock();

	mp = phys_install;

	if (mp->ml_address == 0) {
		/* since we already mapped page1 above */
		start = IMMU_PAGESIZE;
	} else {
		start = mp->ml_address;
	}
	npages = mp->ml_size/IMMU_PAGESIZE + 1;

	dcookies[0].dck_paddr = start;
	dcookies[0].dck_npages = npages;
	dcount = 1;
	(void) dvma_map(domain, start, npages, dcookies,
	    dcount, NULL, IMMU_FLAGS_READ | IMMU_FLAGS_WRITE);

	ddi_err(DER_LOG, domain->dom_dip, "iommu: mapping PHYS span [0x%" PRIx64
	    " - 0x%" PRIx64 "]", start, start + mp->ml_size);

	mp = mp->ml_next;
	while (mp) {
		ddi_err(DER_LOG, domain->dom_dip,
		    "iommu: mapping PHYS span [0x%" PRIx64 " - 0x%" PRIx64 "]",
		    mp->ml_address, mp->ml_address + mp->ml_size);

		start = mp->ml_address;
		npages = mp->ml_size/IMMU_PAGESIZE + 1;

		dcookies[0].dck_paddr = start;
		dcookies[0].dck_npages = npages;
		dcount = 1;
		(void) dvma_map(domain, start, npages,
		    dcookies, dcount, NULL, IMMU_FLAGS_READ | IMMU_FLAGS_WRITE);
		mp = mp->ml_next;
	}

	mp = bios_rsvd;
	while (mp) {
		ddi_err(DER_LOG, domain->dom_dip,
		    "iommu: mapping PHYS span [0x%" PRIx64 " - 0x%" PRIx64 "]",
		    mp->ml_address, mp->ml_address + mp->ml_size);

		start = mp->ml_address;
		npages = mp->ml_size/IMMU_PAGESIZE + 1;

		dcookies[0].dck_paddr = start;
		dcookies[0].dck_npages = npages;
		dcount = 1;
		(void) dvma_map(domain, start, npages,
		    dcookies, dcount, NULL, IMMU_FLAGS_READ | IMMU_FLAGS_WRITE);

		mp = mp->ml_next;
	}

	memlist_read_unlock();
}

/*
 * create_xlate_arena()
 * 	Create the dvma arena for a domain with translation
 *	mapping
 */
static void
create_xlate_arena(immu_t *immu, domain_t *domain,
    dev_info_t *rdip, immu_flags_t immu_flags)
{
	char *arena_name;
	struct memlist *mp;
	int vmem_flags;
	uint64_t start;
	uint_t mgaw;
	uint64_t size;
	uint64_t maxaddr;
	void *vmem_ret;

	arena_name = domain->dom_dvma_arena_name;

	/* Note, don't do sizeof (arena_name) - it is just a pointer */
	(void) snprintf(arena_name,
	    sizeof (domain->dom_dvma_arena_name),
	    "%s-domain-%d-xlate-DVMA-arena", immu->immu_name,
	    domain->dom_did);

	vmem_flags = (immu_flags & IMMU_FLAGS_NOSLEEP) ? VM_NOSLEEP : VM_SLEEP;

	/* Restrict mgaddr (max guest addr) to MGAW */
	mgaw = IMMU_CAP_MGAW(immu->immu_regs_cap);

	/*
	 * To ensure we avoid ioapic and PCI MMIO ranges we just
	 * use the physical memory address range of the system as the
	 * range
	 */
	maxaddr = ((uint64_t)1 << mgaw);

	memlist_read_lock();

	mp = phys_install;

	if (mp->ml_address == 0)
		start = MMU_PAGESIZE;
	else
		start = mp->ml_address;

	if (start + mp->ml_size > maxaddr)
		size = maxaddr - start;
	else
		size = mp->ml_size;

	ddi_err(DER_VERB, rdip,
	    "iommu: %s: Creating dvma vmem arena [0x%" PRIx64
	    " - 0x%" PRIx64 "]", arena_name, start, start + size);

	/*
	 * We always allocate in quanta of IMMU_PAGESIZE
	 */
	domain->dom_dvma_arena = vmem_create(arena_name,
	    (void *)(uintptr_t)start,	/* start addr */
	    size,			/* size */
	    IMMU_PAGESIZE,		/* quantum */
	    NULL,			/* afunc */
	    NULL,			/* ffunc */
	    NULL,			/* source */
	    0,				/* qcache_max */
	    vmem_flags);

	if (domain->dom_dvma_arena == NULL) {
		ddi_err(DER_PANIC, rdip,
		    "Failed to allocate DVMA arena(%s) "
		    "for domain ID (%d)", arena_name, domain->dom_did);
		/*NOTREACHED*/
	}

	mp = mp->ml_next;
	while (mp) {

		if (mp->ml_address == 0)
			start = MMU_PAGESIZE;
		else
			start = mp->ml_address;

		if (start + mp->ml_size > maxaddr)
			size = maxaddr - start;
		else
			size = mp->ml_size;

		ddi_err(DER_VERB, rdip,
		    "iommu: %s: Adding dvma vmem span [0x%" PRIx64
		    " - 0x%" PRIx64 "]", arena_name, start,
		    start + size);

		vmem_ret = vmem_add(domain->dom_dvma_arena,
		    (void *)(uintptr_t)start, size,  vmem_flags);

		if (vmem_ret == NULL) {
			ddi_err(DER_PANIC, rdip,
			    "Failed to allocate DVMA arena(%s) "
			    "for domain ID (%d)",
			    arena_name, domain->dom_did);
			/*NOTREACHED*/
		}
		mp = mp->ml_next;
	}
	memlist_read_unlock();
}

/* ################################### DOMAIN CODE ######################### */

/*
 * Set the domain and domain-dip for a dip
 */
static void
set_domain(
	dev_info_t *dip,
	dev_info_t *ddip,
	domain_t *domain)
{
	immu_devi_t *immu_devi;
	domain_t *fdomain;
	dev_info_t *fddip;

	immu_devi = immu_devi_get(dip);

	mutex_enter(&(DEVI(dip)->devi_lock));
	fddip = immu_devi->imd_ddip;
	fdomain = immu_devi->imd_domain;

	if (fddip) {
		ASSERT(fddip == ddip);
	} else {
		immu_devi->imd_ddip = ddip;
	}

	if (fdomain) {
		ASSERT(fdomain == domain);
	} else {
		immu_devi->imd_domain = domain;
	}
	mutex_exit(&(DEVI(dip)->devi_lock));
}

/*
 * device_domain()
 * 	Get domain for a device. The domain may be global in which case it
 *	is shared between all IOMMU units. Due to potential AGAW differences
 *      between IOMMU units, such global domains *have to be* UNITY mapping
 *      domains. Alternatively, the domain may be local to a IOMMU unit.
 *	Local domains may be shared or immu_devi, although the
 *      scope of sharing
 *	is restricted to devices controlled by the IOMMU unit to
 *      which the domain
 *	belongs. If shared, they (currently) have to be UNITY domains. If
 *      immu_devi a domain may be either UNITY or translation (XLATE) domain.
 */
static domain_t *
device_domain(dev_info_t *rdip, dev_info_t **ddipp, immu_flags_t immu_flags)
{
	dev_info_t *ddip; /* topmost dip in domain i.e. domain owner */
	immu_t *immu;
	domain_t *domain;
	dvma_arg_t dvarg = {0};
	int level;

	*ddipp = NULL;

	/*
	 * Check if the domain is already set. This is usually true
	 * if this is not the first DVMA transaction.
	 */
	ddip = NULL;
	domain = immu_devi_domain(rdip, &ddip);
	if (domain) {
		*ddipp = ddip;
		return (domain);
	}

	immu = immu_dvma_get_immu(rdip, immu_flags);
	if (immu == NULL) {
		/*
		 * possible that there is no IOMMU unit for this device
		 * - BIOS bugs are one example.
		 */
		ddi_err(DER_WARN, rdip, "No iommu unit found for device");
		return (NULL);
	}

	immu_flags |= immu_devi_get(rdip)->imd_dvma_flags;

	dvarg.dva_rdip = rdip;
	dvarg.dva_ddip = NULL;
	dvarg.dva_domain = NULL;
	dvarg.dva_flags = immu_flags;
	level = 0;
	if (immu_walk_ancestor(rdip, NULL, get_branch_domain,
	    &dvarg, &level, immu_flags) != DDI_SUCCESS) {
		/*
		 * maybe low memory. return error,
		 * so driver tries again later
		 */
		return (NULL);
	}

	/* should have walked at least 1 dip (i.e. edip) */
	ASSERT(level > 0);

	ddip = dvarg.dva_ddip;	/* must be present */
	domain = dvarg.dva_domain;	/* may be NULL */

	/*
	 * We may find the domain during our ancestor walk on any one of our
	 * ancestor dips, If the domain is found then the domain-dip
	 * (i.e. ddip) will also be found in the same immu_devi struct.
	 * The domain-dip is the highest ancestor dip which shares the
	 * same domain with edip.
	 * The domain may or may not be found, but the domain dip must
	 * be found.
	 */
	if (ddip == NULL) {
		ddi_err(DER_MODE, rdip, "Cannot find domain dip for device.");
		return (NULL);
	}

	/*
	 * Did we find a domain ?
	 */
	if (domain) {
		goto found;
	}

	/* nope, so allocate */
	domain = domain_create(immu, ddip, rdip, immu_flags);
	if (domain == NULL) {
		return (NULL);
	}

	/*FALLTHROUGH*/
found:
	/*
	 * We know *domain *is* the right domain, so panic if
	 * another domain is set for either the request-dip or
	 * effective dip.
	 */
	set_domain(ddip, ddip, domain);
	set_domain(rdip, ddip, domain);

	*ddipp = ddip;
	return (domain);
}

static void
create_unity_domain(immu_t *immu)
{
	domain_t *domain;

	/* domain created during boot and always use sleep flag */
	domain = kmem_zalloc(sizeof (domain_t), KM_SLEEP);

	rw_init(&(domain->dom_pgtable_rwlock), NULL, RW_DEFAULT, NULL);

	domain->dom_did = IMMU_UNITY_DID;
	domain->dom_maptype = IMMU_MAPTYPE_UNITY;

	domain->dom_immu = immu;
	immu->immu_unity_domain = domain;

	/*
	 * Setup the domain's initial page table
	 * should never fail.
	 */
	domain->dom_pgtable_root = pgtable_alloc(immu, IMMU_FLAGS_SLEEP);
	pgtable_zero(domain->dom_pgtable_root);

	/*
	 * Only map all physical memory in to the unity domain
	 * if passthrough is not supported. If it is supported,
	 * passthrough is set in the context entry instead.
	 */
	if (!IMMU_ECAP_GET_PT(immu->immu_regs_excap))
		map_unity_domain(domain);


	/*
	 * put it on the system-wide UNITY domain list
	 */
	mutex_enter(&(immu_domain_lock));
	list_insert_tail(&immu_unity_domain_list, domain);
	mutex_exit(&(immu_domain_lock));
}

/*
 * ddip is the domain-dip - the topmost dip in a domain
 * rdip is the requesting-dip - the device which is
 * requesting DVMA setup
 * if domain is a non-shared domain rdip == ddip
 */
static domain_t *
domain_create(immu_t *immu, dev_info_t *ddip, dev_info_t *rdip,
    immu_flags_t immu_flags)
{
	int kmflags;
	domain_t *domain;
	char mod_hash_name[128];
	immu_devi_t *immu_devi;
	int did;
	immu_dcookie_t dcookies[1] = {0};
	int dcount = 0;

	immu_devi = immu_devi_get(rdip);

	/*
	 * First allocate a domainid.
	 * This routine will never fail, since if we run out
	 * of domains the unity domain will be allocated.
	 */
	did = did_alloc(immu, rdip, ddip, immu_flags);
	if (did == IMMU_UNITY_DID) {
		/* domain overflow */
		ASSERT(immu->immu_unity_domain);
		return (immu->immu_unity_domain);
	}

	kmflags = (immu_flags & IMMU_FLAGS_NOSLEEP) ? KM_NOSLEEP : KM_SLEEP;
	domain = kmem_zalloc(sizeof (domain_t), kmflags);
	if (domain == NULL) {
		ddi_err(DER_PANIC, rdip, "Failed to alloc DVMA domain "
		    "structure for device. IOMMU unit: %s", immu->immu_name);
		/*NOTREACHED*/
	}

	rw_init(&(domain->dom_pgtable_rwlock), NULL, RW_DEFAULT, NULL);

	(void) snprintf(mod_hash_name, sizeof (mod_hash_name),
	    "immu%s-domain%d-pava-hash", immu->immu_name, did);

	domain->dom_did = did;
	domain->dom_immu = immu;
	domain->dom_maptype = IMMU_MAPTYPE_XLATE;
	domain->dom_dip = ddip;

	/*
	 * Create xlate DVMA arena for this domain.
	 */
	create_xlate_arena(immu, domain, rdip, immu_flags);

	/*
	 * Setup the domain's initial page table
	 */
	domain->dom_pgtable_root = pgtable_alloc(immu, immu_flags);
	if (domain->dom_pgtable_root == NULL) {
		ddi_err(DER_PANIC, rdip, "Failed to alloc root "
		    "pgtable for domain (%d). IOMMU unit: %s",
		    domain->dom_did, immu->immu_name);
		/*NOTREACHED*/
	}
	pgtable_zero(domain->dom_pgtable_root);

	/*
	 * Since this is a immu unit-specific domain, put it on
	 * the per-immu domain list.
	 */
	mutex_enter(&(immu->immu_lock));
	list_insert_head(&immu->immu_domain_list, domain);
	mutex_exit(&(immu->immu_lock));

	/*
	 * Also put it on the system-wide xlate domain list
	 */
	mutex_enter(&(immu_domain_lock));
	list_insert_head(&immu_xlate_domain_list, domain);
	mutex_exit(&(immu_domain_lock));

	bdf_domain_insert(immu_devi, domain);

#ifdef BUGGY_DRIVERS
	/*
	 * Map page0. Some broken HW/FW access it.
	 */
	dcookies[0].dck_paddr = 0;
	dcookies[0].dck_npages = 1;
	dcount = 1;
	(void) dvma_map(domain, 0, 1, dcookies, dcount, NULL,
	    IMMU_FLAGS_READ | IMMU_FLAGS_WRITE | IMMU_FLAGS_PAGE1);
#endif
	return (domain);
}

/*
 * Create domainid arena.
 * Domainid 0 is reserved by Vt-d spec and cannot be used by
 * system software.
 * Domainid 1 is reserved by solaris and used for *all* of the following:
 *	as the "uninitialized" domain - For devices not yet controlled
 *	by Solaris
 *	as the "unity" domain - For devices that will always belong
 *	to the unity domain
 *	as the "overflow" domain - Used for any new device after we
 *	run out of domains
 * All of the above domains map into a single domain with
 * domainid 1 and UNITY DVMA mapping
 * Each IMMU unity has its own unity/uninit/overflow domain
 */
static void
did_init(immu_t *immu)
{
	(void) snprintf(immu->immu_did_arena_name,
	    sizeof (immu->immu_did_arena_name),
	    "%s_domainid_arena", immu->immu_name);

	ddi_err(DER_VERB, immu->immu_dip, "creating domainid arena %s",
	    immu->immu_did_arena_name);

	immu->immu_did_arena = vmem_create(
	    immu->immu_did_arena_name,
	    (void *)(uintptr_t)(IMMU_UNITY_DID + 1),   /* start addr */
	    immu->immu_max_domains - IMMU_UNITY_DID,
	    1,				/* quantum */
	    NULL,			/* afunc */
	    NULL,			/* ffunc */
	    NULL,			/* source */
	    0,				/* qcache_max */
	    VM_SLEEP);

	/* Even with SLEEP flag, vmem_create() can fail */
	if (immu->immu_did_arena == NULL) {
		ddi_err(DER_PANIC, NULL, "%s: Failed to create Intel "
		    "IOMMU domainid allocator: %s", immu->immu_name,
		    immu->immu_did_arena_name);
	}
}

/* #########################  CONTEXT CODE ################################# */

static void
context_set(immu_t *immu, domain_t *domain, pgtable_t *root_table,
    int bus, int devfunc)
{
	pgtable_t *context;
	pgtable_t *pgtable_root;
	hw_rce_t *hw_rent;
	hw_rce_t *hw_cent;
	hw_rce_t *ctxp;
	int sid;
	krw_t rwtype;
	boolean_t fill_root;
	boolean_t fill_ctx;

	pgtable_root = domain->dom_pgtable_root;

	ctxp = (hw_rce_t *)(root_table->swpg_next_array);
	context = *(pgtable_t **)(ctxp + bus);
	hw_rent = (hw_rce_t *)(root_table->hwpg_vaddr) + bus;

	fill_root = B_FALSE;
	fill_ctx = B_FALSE;

	/* Check the most common case first with reader lock */
	rw_enter(&(immu->immu_ctx_rwlock), RW_READER);
	rwtype = RW_READER;
again:
	if (ROOT_GET_P(hw_rent)) {
		hw_cent = (hw_rce_t *)(context->hwpg_vaddr) + devfunc;
		if (CONT_GET_AVAIL(hw_cent) == IMMU_CONT_INITED) {
			rw_exit(&(immu->immu_ctx_rwlock));
			return;
		} else {
			fill_ctx = B_TRUE;
		}
	} else {
		fill_root = B_TRUE;
		fill_ctx = B_TRUE;
	}

	if (rwtype == RW_READER &&
	    rw_tryupgrade(&(immu->immu_ctx_rwlock)) == 0) {
		rw_exit(&(immu->immu_ctx_rwlock));
		rw_enter(&(immu->immu_ctx_rwlock), RW_WRITER);
		rwtype = RW_WRITER;
		goto again;
	}
	rwtype = RW_WRITER;

	if (fill_root == B_TRUE) {
		ROOT_SET_CONT(hw_rent, context->hwpg_paddr);
		ROOT_SET_P(hw_rent);
		immu_regs_cpu_flush(immu, (caddr_t)hw_rent, sizeof (hw_rce_t));
	}

	if (fill_ctx == B_TRUE) {
		hw_cent = (hw_rce_t *)(context->hwpg_vaddr) + devfunc;
		/* need to disable context entry before reprogramming it */
		bzero(hw_cent, sizeof (hw_rce_t));

		/* flush caches */
		immu_regs_cpu_flush(immu, (caddr_t)hw_cent, sizeof (hw_rce_t));

		sid = ((bus << 8) | devfunc);
		immu_flush_context_fsi(immu, 0, sid, domain->dom_did,
		    &immu->immu_ctx_inv_wait);

		CONT_SET_AVAIL(hw_cent, IMMU_CONT_INITED);
		CONT_SET_DID(hw_cent, domain->dom_did);
		CONT_SET_AW(hw_cent, immu->immu_dvma_agaw);
		CONT_SET_ASR(hw_cent, pgtable_root->hwpg_paddr);
		if (domain->dom_did == IMMU_UNITY_DID &&
		    IMMU_ECAP_GET_PT(immu->immu_regs_excap))
			CONT_SET_TTYPE(hw_cent, TTYPE_PASSTHRU);
		else
			/*LINTED*/
			CONT_SET_TTYPE(hw_cent, TTYPE_XLATE_ONLY);
		CONT_SET_P(hw_cent);
		if (IMMU_ECAP_GET_CH(immu->immu_regs_excap)) {
			CONT_SET_EH(hw_cent);
			if (immu_use_alh)
				CONT_SET_ALH(hw_cent);
		}
		immu_regs_cpu_flush(immu, (caddr_t)hw_cent, sizeof (hw_rce_t));
	}
	rw_exit(&(immu->immu_ctx_rwlock));
}

static pgtable_t *
context_create(immu_t *immu)
{
	int	bus;
	int	devfunc;
	pgtable_t *root_table;
	pgtable_t *context;
	pgtable_t *pgtable_root;
	hw_rce_t *ctxp;
	hw_rce_t *hw_rent;
	hw_rce_t *hw_cent;

	/* Allocate a zeroed root table (4K 256b entries) */
	root_table = pgtable_alloc(immu, IMMU_FLAGS_SLEEP);
	pgtable_zero(root_table);

	/*
	 * Setup context tables for all possible root table entries.
	 * Start out with unity domains for all entries.
	 */
	ctxp = (hw_rce_t *)(root_table->swpg_next_array);
	hw_rent = (hw_rce_t *)(root_table->hwpg_vaddr);
	for (bus = 0; bus < IMMU_ROOT_NUM; bus++, ctxp++, hw_rent++) {
		context = pgtable_alloc(immu, IMMU_FLAGS_SLEEP);
		pgtable_zero(context);
		ROOT_SET_P(hw_rent);
		ROOT_SET_CONT(hw_rent, context->hwpg_paddr);
		hw_cent = (hw_rce_t *)(context->hwpg_vaddr);
		for (devfunc = 0; devfunc < IMMU_CONT_NUM;
		    devfunc++, hw_cent++) {
			pgtable_root =
			    immu->immu_unity_domain->dom_pgtable_root;
			CONT_SET_DID(hw_cent,
			    immu->immu_unity_domain->dom_did);
			CONT_SET_AW(hw_cent, immu->immu_dvma_agaw);
			CONT_SET_ASR(hw_cent, pgtable_root->hwpg_paddr);
			if (IMMU_ECAP_GET_PT(immu->immu_regs_excap))
				CONT_SET_TTYPE(hw_cent, TTYPE_PASSTHRU);
			else
				/*LINTED*/
				CONT_SET_TTYPE(hw_cent, TTYPE_XLATE_ONLY);
			CONT_SET_AVAIL(hw_cent, IMMU_CONT_UNINITED);
			CONT_SET_P(hw_cent);
		}
		immu_regs_cpu_flush(immu, context->hwpg_vaddr, IMMU_PAGESIZE);
		*((pgtable_t **)ctxp) = context;
	}

	return (root_table);
}

/*
 * Called during rootnex attach, so no locks needed
 */
static void
context_init(immu_t *immu)
{
	rw_init(&(immu->immu_ctx_rwlock), NULL, RW_DEFAULT, NULL);

	immu_init_inv_wait(&immu->immu_ctx_inv_wait, "ctxglobal", B_TRUE);

	immu_regs_wbf_flush(immu);

	immu->immu_ctx_root = context_create(immu);

	immu_regs_set_root_table(immu);

	rw_enter(&(immu->immu_ctx_rwlock), RW_WRITER);
	immu_flush_context_gbl(immu, &immu->immu_ctx_inv_wait);
	immu_flush_iotlb_gbl(immu, &immu->immu_ctx_inv_wait);
	rw_exit(&(immu->immu_ctx_rwlock));
}


/*
 * Find top pcib
 */
static int
find_top_pcib(dev_info_t *dip, void *arg)
{
	immu_devi_t *immu_devi;
	dev_info_t **pcibdipp = (dev_info_t **)arg;

	immu_devi = immu_devi_get(dip);

	if (immu_devi->imd_pcib_type == IMMU_PCIB_PCI_PCI) {
		*pcibdipp = dip;
	}

	return (DDI_WALK_CONTINUE);
}

static int
immu_context_update(immu_t *immu, domain_t *domain, dev_info_t *ddip,
    dev_info_t *rdip, immu_flags_t immu_flags)
{
	immu_devi_t *r_immu_devi;
	immu_devi_t *d_immu_devi;
	int r_bus;
	int d_bus;
	int r_devfunc;
	int d_devfunc;
	immu_pcib_t d_pcib_type;
	dev_info_t *pcibdip;

	if (ddip == NULL || rdip == NULL ||
	    ddip == root_devinfo || rdip == root_devinfo) {
		ddi_err(DER_MODE, rdip, "immu_contexts_update: domain-dip or "
		    "request-dip are NULL or are root devinfo");
		return (DDI_FAILURE);
	}

	/*
	 * We need to set the context fields
	 * based on what type of device rdip and ddip are.
	 * To do that we need the immu_devi field.
	 * Set the immu_devi field (if not already set)
	 */
	if (immu_devi_set(ddip, immu_flags) == DDI_FAILURE) {
		ddi_err(DER_MODE, rdip,
		    "immu_context_update: failed to set immu_devi for ddip");
		return (DDI_FAILURE);
	}

	if (immu_devi_set(rdip, immu_flags) == DDI_FAILURE) {
		ddi_err(DER_MODE, rdip,
		    "immu_context_update: failed to set immu_devi for rdip");
		return (DDI_FAILURE);
	}

	d_immu_devi = immu_devi_get(ddip);
	r_immu_devi = immu_devi_get(rdip);

	d_bus = d_immu_devi->imd_bus;
	d_devfunc = d_immu_devi->imd_devfunc;
	d_pcib_type = d_immu_devi->imd_pcib_type;
	r_bus = r_immu_devi->imd_bus;
	r_devfunc = r_immu_devi->imd_devfunc;

	if (rdip == ddip) {
		/* rdip is a PCIE device. set context for it only */
		context_set(immu, domain, immu->immu_ctx_root, r_bus,
		    r_devfunc);
#ifdef BUGGY_DRIVERS
	} else if (r_immu_devi == d_immu_devi) {
#ifdef TEST
		ddi_err(DER_WARN, rdip, "Driver bug: Devices 0x%lx and "
		    "0x%lx are identical", rdip, ddip);
#endif
		/* rdip is a PCIE device. set context for it only */
		context_set(immu, domain, immu->immu_ctx_root, r_bus,
		    r_devfunc);
#endif
	} else if (d_pcib_type == IMMU_PCIB_PCIE_PCI) {
		/*
		 * ddip is a PCIE_PCI bridge. Set context for ddip's
		 * secondary bus. If rdip is on ddip's secondary
		 * bus, set context for rdip. Else, set context
		 * for rdip's PCI bridge on ddip's secondary bus.
		 */
		context_set(immu, domain, immu->immu_ctx_root,
		    d_immu_devi->imd_sec, 0);
		if (d_immu_devi->imd_sec == r_bus) {
			context_set(immu, domain, immu->immu_ctx_root,
			    r_bus, r_devfunc);
		} else {
			pcibdip = NULL;
			if (immu_walk_ancestor(rdip, ddip, find_top_pcib,
			    &pcibdip, NULL, immu_flags) == DDI_SUCCESS &&
			    pcibdip != NULL) {
				r_immu_devi = immu_devi_get(pcibdip);
				r_bus = r_immu_devi->imd_bus;
				r_devfunc = r_immu_devi->imd_devfunc;
				context_set(immu, domain, immu->immu_ctx_root,
				    r_bus, r_devfunc);
			} else {
				ddi_err(DER_PANIC, rdip, "Failed to find PCI "
				    " bridge for PCI device");
				/*NOTREACHED*/
			}
		}
	} else if (d_pcib_type == IMMU_PCIB_PCI_PCI) {
		context_set(immu, domain, immu->immu_ctx_root, d_bus,
		    d_devfunc);
	} else if (d_pcib_type == IMMU_PCIB_ENDPOINT) {
		/*
		 * ddip is a PCIE device which has a non-PCI device under it
		 * i.e. it is a PCI-nonPCI bridge. Example: pciicde-ata
		 */
		context_set(immu, domain, immu->immu_ctx_root, d_bus,
		    d_devfunc);
	} else {
		ddi_err(DER_PANIC, rdip, "unknown device type. Cannot "
		    "set iommu context.");
		/*NOTREACHED*/
	}

	/* XXX do we need a membar_producer() here */
	return (DDI_SUCCESS);
}

/* ##################### END CONTEXT CODE ################################## */
/* ##################### MAPPING CODE ################################## */


#ifdef DEBUG
static boolean_t
PDTE_check(immu_t *immu, hw_pdte_t pdte, pgtable_t *next, paddr_t paddr,
    dev_info_t *rdip, immu_flags_t immu_flags)
{
	/* The PDTE must be set i.e. present bit is set */
	if (!PDTE_P(pdte)) {
		ddi_err(DER_MODE, rdip, "No present flag");
		return (B_FALSE);
	}

	/*
	 * Just assert to check most significant system software field
	 * (PDTE_SW4) as it is same as present bit and we
	 * checked that above
	 */
	ASSERT(PDTE_SW4(pdte));

	/*
	 * TM field should be clear if not reserved.
	 * non-leaf is always reserved
	 */
	if (next == NULL && immu->immu_TM_reserved == B_FALSE) {
		if (PDTE_TM(pdte)) {
			ddi_err(DER_MODE, rdip, "TM flag set");
			return (B_FALSE);
		}
	}

	/*
	 * The SW3 field is not used and must be clear
	 */
	if (PDTE_SW3(pdte)) {
		ddi_err(DER_MODE, rdip, "SW3 set");
		return (B_FALSE);
	}

	/*
	 * PFN (for PTE) or next level pgtable-paddr (for PDE) must be set
	 */
	if (next == NULL) {
		ASSERT(paddr % IMMU_PAGESIZE == 0);
		if (PDTE_PADDR(pdte) != paddr) {
			ddi_err(DER_MODE, rdip,
			    "PTE paddr mismatch: %lx != %lx",
			    PDTE_PADDR(pdte), paddr);
			return (B_FALSE);
		}
	} else {
		if (PDTE_PADDR(pdte) != next->hwpg_paddr) {
			ddi_err(DER_MODE, rdip,
			    "PDE paddr mismatch: %lx != %lx",
			    PDTE_PADDR(pdte), next->hwpg_paddr);
			return (B_FALSE);
		}
	}

	/*
	 * SNP field should be clear if not reserved.
	 * non-leaf is always reserved
	 */
	if (next == NULL && immu->immu_SNP_reserved == B_FALSE) {
		if (PDTE_SNP(pdte)) {
			ddi_err(DER_MODE, rdip, "SNP set");
			return (B_FALSE);
		}
	}

	/* second field available for system software should be clear */
	if (PDTE_SW2(pdte)) {
		ddi_err(DER_MODE, rdip, "SW2 set");
		return (B_FALSE);
	}

	/* Super pages field should be clear */
	if (PDTE_SP(pdte)) {
		ddi_err(DER_MODE, rdip, "SP set");
		return (B_FALSE);
	}

	/*
	 * least significant field available for
	 * system software should be clear
	 */
	if (PDTE_SW1(pdte)) {
		ddi_err(DER_MODE, rdip, "SW1 set");
		return (B_FALSE);
	}

	if ((immu_flags & IMMU_FLAGS_READ) && !PDTE_READ(pdte)) {
		ddi_err(DER_MODE, rdip, "READ not set");
		return (B_FALSE);
	}

	if ((immu_flags & IMMU_FLAGS_WRITE) && !PDTE_WRITE(pdte)) {
		ddi_err(DER_MODE, rdip, "WRITE not set");
		return (B_FALSE);
	}

	return (B_TRUE);
}
#endif

/*ARGSUSED*/
static void
PTE_clear_all(immu_t *immu, domain_t *domain, xlate_t *xlate,
    uint64_t *dvma_ptr, uint64_t *npages_ptr, dev_info_t *rdip)
{
	uint64_t npages;
	uint64_t dvma;
	pgtable_t *pgtable;
	hw_pdte_t *hwp;
	hw_pdte_t *shwp;
	int idx;

	pgtable = xlate->xlt_pgtable;
	idx = xlate->xlt_idx;

	dvma = *dvma_ptr;
	npages = *npages_ptr;

	/*
	 * since a caller gets a unique dvma for a physical address,
	 * no other concurrent thread will be writing to the same
	 * PTE even if it has the same paddr. So no locks needed.
	 */
	shwp = (hw_pdte_t *)(pgtable->hwpg_vaddr) + idx;

	hwp = shwp;
	for (; npages > 0 && idx <= IMMU_PGTABLE_MAXIDX; idx++, hwp++) {
		PDTE_CLEAR_P(*hwp);
		dvma += IMMU_PAGESIZE;
		npages--;
	}

	*dvma_ptr = dvma;
	*npages_ptr = npages;

	xlate->xlt_idx = idx;
}

static void
xlate_setup(uint64_t dvma, xlate_t *xlate, int nlevels)
{
	int level;
	uint64_t offbits;

	/*
	 * Skip the first 12 bits which is the offset into
	 * 4K PFN (phys page frame based on IMMU_PAGESIZE)
	 */
	offbits = dvma >> IMMU_PAGESHIFT;

	/* skip to level 1 i.e. leaf PTE */
	for (level = 1, xlate++; level <= nlevels; level++, xlate++) {
		xlate->xlt_level = level;
		xlate->xlt_idx = (offbits & IMMU_PGTABLE_LEVEL_MASK);
		ASSERT(xlate->xlt_idx <= IMMU_PGTABLE_MAXIDX);
		xlate->xlt_pgtable = NULL;
		offbits >>= IMMU_PGTABLE_LEVEL_STRIDE;
	}
}

/*
 * Read the pgtables
 */
static boolean_t
PDE_lookup(domain_t *domain, xlate_t *xlate, int nlevels)
{
	pgtable_t *pgtable;
	pgtable_t *next;
	uint_t idx;

	/* start with highest level pgtable i.e. root */
	xlate += nlevels;

	if (xlate->xlt_pgtable == NULL) {
		xlate->xlt_pgtable = domain->dom_pgtable_root;
	}

	for (; xlate->xlt_level > 1; xlate--) {
		idx = xlate->xlt_idx;
		pgtable = xlate->xlt_pgtable;

		if ((xlate - 1)->xlt_pgtable) {
			continue;
		}

		/* Lock the pgtable in read mode */
		rw_enter(&(pgtable->swpg_rwlock), RW_READER);

		/*
		 * since we are unmapping, the pgtable should
		 * already point to a leafier pgtable.
		 */
		next = *(pgtable->swpg_next_array + idx);
		(xlate - 1)->xlt_pgtable = next;
		rw_exit(&(pgtable->swpg_rwlock));
		if (next == NULL)
			return (B_FALSE);
	}

	return (B_TRUE);
}

static void
immu_fault_walk(void *arg, void *base, size_t len)
{
	uint64_t dvma, start;

	dvma = *(uint64_t *)arg;
	start = (uint64_t)(uintptr_t)base;

	if (dvma >= start && dvma < (start + len)) {
		ddi_err(DER_WARN, NULL,
		    "faulting DVMA address is in vmem arena "
		    "(%" PRIx64 "-%" PRIx64 ")",
		    start, start + len);
		*(uint64_t *)arg = ~0ULL;
	}
}

void
immu_print_fault_info(uint_t sid, uint64_t dvma)
{
	int nlevels;
	xlate_t xlate[IMMU_PGTABLE_MAX_LEVELS + 1] = {0};
	xlate_t *xlatep;
	hw_pdte_t pte;
	domain_t *domain;
	immu_t *immu;
	uint64_t dvma_arg;

	if (mod_hash_find(bdf_domain_hash,
	    (void *)(uintptr_t)sid, (void *)&domain) != 0) {
		ddi_err(DER_WARN, NULL,
		    "no domain for faulting SID %08x", sid);
		return;
	}

	immu = domain->dom_immu;

	dvma_arg = dvma;
	vmem_walk(domain->dom_dvma_arena, VMEM_ALLOC, immu_fault_walk,
	    (void *)&dvma_arg);
	if (dvma_arg != ~0ULL)
		ddi_err(DER_WARN, domain->dom_dip,
		    "faulting DVMA address is not in vmem arena");

	nlevels = immu->immu_dvma_nlevels;
	xlate_setup(dvma, xlate, nlevels);

	if (!PDE_lookup(domain, xlate, nlevels)) {
		ddi_err(DER_WARN, domain->dom_dip,
		    "pte not found in domid %d for faulting addr %" PRIx64,
		    domain->dom_did, dvma);
		return;
	}

	xlatep = &xlate[1];
	pte = *((hw_pdte_t *)
	    (xlatep->xlt_pgtable->hwpg_vaddr) + xlatep->xlt_idx);

	ddi_err(DER_WARN, domain->dom_dip,
	    "domid %d pte: %" PRIx64 "(paddr %" PRIx64 ")", domain->dom_did,
	    (unsigned long long)pte, (unsigned long long)PDTE_PADDR(pte));
}

/*ARGSUSED*/
static void
PTE_set_one(immu_t *immu, hw_pdte_t *hwp, paddr_t paddr,
    dev_info_t *rdip, immu_flags_t immu_flags)
{
	hw_pdte_t pte;

#ifndef DEBUG
	pte = immu->immu_ptemask;
	PDTE_SET_PADDR(pte, paddr);
#else
	pte = *hwp;

	if (PDTE_P(pte)) {
		if (PDTE_PADDR(pte) != paddr) {
			ddi_err(DER_MODE, rdip, "PTE paddr %lx != paddr %lx",
			    PDTE_PADDR(pte), paddr);
		}
#ifdef BUGGY_DRIVERS
		return;
#else
		goto out;
#endif
	}

	/* clear TM field if not reserved */
	if (immu->immu_TM_reserved == B_FALSE) {
		PDTE_CLEAR_TM(pte);
	}

	/* Clear 3rd field for system software  - not used */
	PDTE_CLEAR_SW3(pte);

	/* Set paddr */
	ASSERT(paddr % IMMU_PAGESIZE == 0);
	PDTE_CLEAR_PADDR(pte);
	PDTE_SET_PADDR(pte, paddr);

	/*  clear SNP field if not reserved. */
	if (immu->immu_SNP_reserved == B_FALSE) {
		PDTE_CLEAR_SNP(pte);
	}

	/* Clear SW2 field available for software */
	PDTE_CLEAR_SW2(pte);


	/* SP is don't care for PTEs. Clear it for cleanliness */
	PDTE_CLEAR_SP(pte);

	/* Clear SW1 field available for software */
	PDTE_CLEAR_SW1(pte);

	/*
	 * Now that we are done writing the PTE
	 * set the "present" flag. Note this present
	 * flag is a bit in the PDE/PTE that the
	 * spec says is available for system software.
	 * This is an implementation detail of Solaris
	 * bare-metal Intel IOMMU.
	 * The present field in a PDE/PTE is not defined
	 * by the Vt-d spec
	 */

	PDTE_SET_P(pte);

	pte |= immu->immu_ptemask;

out:
#endif /* DEBUG */
#ifdef BUGGY_DRIVERS
	PDTE_SET_READ(pte);
	PDTE_SET_WRITE(pte);
#else
	if (immu_flags & IMMU_FLAGS_READ)
		PDTE_SET_READ(pte);
	if (immu_flags & IMMU_FLAGS_WRITE)
		PDTE_SET_WRITE(pte);
#endif /* BUGGY_DRIVERS */

	*hwp = pte;
}

/*ARGSUSED*/
static void
PTE_set_all(immu_t *immu, domain_t *domain, xlate_t *xlate,
    uint64_t *dvma_ptr, uint64_t *nvpages_ptr, immu_dcookie_t *dcookies,
    int dcount, dev_info_t *rdip, immu_flags_t immu_flags)
{
	paddr_t paddr;
	uint64_t nvpages;
	uint64_t nppages;
	uint64_t dvma;
	pgtable_t *pgtable;
	hw_pdte_t *hwp;
	hw_pdte_t *shwp;
	int idx, nset;
	int j;

	pgtable = xlate->xlt_pgtable;
	idx = xlate->xlt_idx;

	dvma = *dvma_ptr;
	nvpages = *nvpages_ptr;

	/*
	 * since a caller gets a unique dvma for a physical address,
	 * no other concurrent thread will be writing to the same
	 * PTE even if it has the same paddr. So no locks needed.
	 */
	shwp = (hw_pdte_t *)(pgtable->hwpg_vaddr) + idx;

	hwp = shwp;
	for (j = dcount - 1; j >= 0; j--) {
		if (nvpages <= dcookies[j].dck_npages)
			break;
		nvpages -= dcookies[j].dck_npages;
	}

	nppages = nvpages;
	paddr = dcookies[j].dck_paddr +
	    (dcookies[j].dck_npages - nppages) * IMMU_PAGESIZE;

	nvpages = *nvpages_ptr;
	nset = 0;
	for (; nvpages > 0 && idx <= IMMU_PGTABLE_MAXIDX; idx++, hwp++) {
		PTE_set_one(immu, hwp, paddr, rdip, immu_flags);
		nset++;

		ASSERT(PDTE_check(immu, *hwp, NULL, paddr, rdip, immu_flags)
		    == B_TRUE);
		nppages--;
		nvpages--;
		paddr += IMMU_PAGESIZE;
		dvma += IMMU_PAGESIZE;

		if (nppages == 0) {
			j++;
		}

		if (j == dcount)
			break;

		if (nppages == 0) {
			nppages = dcookies[j].dck_npages;
			paddr = dcookies[j].dck_paddr;
		}
	}

	if (nvpages) {
		*dvma_ptr = dvma;
		*nvpages_ptr = nvpages;
	} else {
		*dvma_ptr = 0;
		*nvpages_ptr = 0;
	}

	xlate->xlt_idx = idx;
}

/*ARGSUSED*/
static void
PDE_set_one(immu_t *immu, hw_pdte_t *hwp, pgtable_t *next,
    dev_info_t *rdip, immu_flags_t immu_flags)
{
	hw_pdte_t pde;

	pde = *hwp;

	/* if PDE is already set, make sure it is correct */
	if (PDTE_P(pde)) {
		ASSERT(PDTE_PADDR(pde) == next->hwpg_paddr);
#ifdef BUGGY_DRIVERS
		return;
#else
		goto out;
#endif
	}

	/* Dont touch SW4, it is the present bit */

	/* don't touch TM field it is reserved for PDEs */

	/* 3rd field available for system software is not used */
	PDTE_CLEAR_SW3(pde);

	/* Set next level pgtable-paddr for PDE */
	PDTE_CLEAR_PADDR(pde);
	PDTE_SET_PADDR(pde, next->hwpg_paddr);

	/* don't touch SNP field it is reserved for PDEs */

	/* Clear second field available for system software */
	PDTE_CLEAR_SW2(pde);

	/* No super pages for PDEs */
	PDTE_CLEAR_SP(pde);

	/* Clear SW1 for software */
	PDTE_CLEAR_SW1(pde);

	/*
	 * Now that we are done writing the PDE
	 * set the "present" flag. Note this present
	 * flag is a bit in the PDE/PTE that the
	 * spec says is available for system software.
	 * This is an implementation detail of Solaris
	 * base-metal Intel IOMMU.
	 * The present field in a PDE/PTE is not defined
	 * by the Vt-d spec
	 */

out:
#ifdef  BUGGY_DRIVERS
	PDTE_SET_READ(pde);
	PDTE_SET_WRITE(pde);
#else
	if (immu_flags & IMMU_FLAGS_READ)
		PDTE_SET_READ(pde);
	if (immu_flags & IMMU_FLAGS_WRITE)
		PDTE_SET_WRITE(pde);
#endif

	PDTE_SET_P(pde);

	*hwp = pde;
}

/*
 * Used to set PDEs
 */
static boolean_t
PDE_set_all(immu_t *immu, domain_t *domain, xlate_t *xlate, int nlevels,
    dev_info_t *rdip, immu_flags_t immu_flags)
{
	pgtable_t *pgtable;
	pgtable_t *new;
	pgtable_t *next;
	hw_pdte_t *hwp;
	int level;
	uint_t idx;
	krw_t rwtype;
	boolean_t set = B_FALSE;

	/* start with highest level pgtable i.e. root */
	xlate += nlevels;

	new = NULL;
	xlate->xlt_pgtable = domain->dom_pgtable_root;
	for (level = nlevels; level > 1; level--, xlate--) {
		idx = xlate->xlt_idx;
		pgtable = xlate->xlt_pgtable;

		/* Lock the pgtable in READ mode first */
		rw_enter(&(pgtable->swpg_rwlock), RW_READER);
		rwtype = RW_READER;
again:
		hwp = (hw_pdte_t *)(pgtable->hwpg_vaddr) + idx;
		next = (pgtable->swpg_next_array)[idx];

		/*
		 * check if leafier level already has a pgtable
		 * if yes, verify
		 */
		if (next == NULL) {
			if (new == NULL) {

				IMMU_DPROBE2(immu__pdp__alloc, dev_info_t *,
				    rdip, int, level);

				new = pgtable_alloc(immu, immu_flags);
				if (new == NULL) {
					ddi_err(DER_PANIC, rdip,
					    "pgtable alloc err");
				}
				pgtable_zero(new);
			}

			/* Change to a write lock */
			if (rwtype == RW_READER &&
			    rw_tryupgrade(&(pgtable->swpg_rwlock)) == 0) {
				rw_exit(&(pgtable->swpg_rwlock));
				rw_enter(&(pgtable->swpg_rwlock), RW_WRITER);
				rwtype = RW_WRITER;
				goto again;
			}
			rwtype = RW_WRITER;
			next = new;
			(pgtable->swpg_next_array)[idx] = next;
			new = NULL;
			PDE_set_one(immu, hwp, next, rdip, immu_flags);
			set = B_TRUE;
			rw_downgrade(&(pgtable->swpg_rwlock));
			rwtype = RW_READER;
		}
#ifndef  BUGGY_DRIVERS
		else {
			hw_pdte_t pde = *hwp;

			/*
			 * If buggy driver we already set permission
			 * READ+WRITE so nothing to do for that case
			 * XXX Check that read writer perms change before
			 * actually setting perms. Also need to hold lock
			 */
			if (immu_flags & IMMU_FLAGS_READ)
				PDTE_SET_READ(pde);
			if (immu_flags & IMMU_FLAGS_WRITE)
				PDTE_SET_WRITE(pde);

			*hwp = pde;
		}
#endif

		ASSERT(PDTE_check(immu, *hwp, next, 0, rdip, immu_flags)
		    == B_TRUE);

		(xlate - 1)->xlt_pgtable = next;
		rw_exit(&(pgtable->swpg_rwlock));
	}

	if (new) {
		pgtable_free(immu, new);
	}

	return (set);
}

/*
 * dvma_map()
 *     map a contiguous range of DVMA pages
 *
 *     immu: IOMMU unit for which we are generating DVMA cookies
 *   domain: domain
 *    sdvma: Starting dvma
 *   spaddr: Starting paddr
 *   npages: Number of pages
 *     rdip: requesting device
 *     immu_flags: flags
 */
static boolean_t
dvma_map(domain_t *domain, uint64_t sdvma, uint64_t snvpages,
    immu_dcookie_t *dcookies, int dcount, dev_info_t *rdip,
    immu_flags_t immu_flags)
{
	uint64_t dvma;
	uint64_t n;
	immu_t *immu = domain->dom_immu;
	int nlevels = immu->immu_dvma_nlevels;
	xlate_t xlate[IMMU_PGTABLE_MAX_LEVELS + 1] = {0};
	boolean_t pde_set = B_FALSE;

	n = snvpages;
	dvma = sdvma;

	while (n > 0) {
		xlate_setup(dvma, xlate, nlevels);

		/* Lookup or allocate PGDIRs and PGTABLEs if necessary */
		if (PDE_set_all(immu, domain, xlate, nlevels, rdip, immu_flags)
		    == B_TRUE) {
			pde_set = B_TRUE;
		}

		/* set all matching ptes that fit into this leaf pgtable */
		PTE_set_all(immu, domain, &xlate[1], &dvma, &n, dcookies,
		    dcount, rdip, immu_flags);
	}

	return (pde_set);
}

/*
 * dvma_unmap()
 *   unmap a range of DVMAs
 *
 * immu: IOMMU unit state
 * domain: domain for requesting device
 * ddip: domain-dip
 * dvma: starting DVMA
 * npages: Number of IMMU pages to be unmapped
 * rdip: requesting device
 */
static void
dvma_unmap(domain_t *domain, uint64_t sdvma, uint64_t snpages,
    dev_info_t *rdip)
{
	immu_t *immu = domain->dom_immu;
	int nlevels = immu->immu_dvma_nlevels;
	xlate_t xlate[IMMU_PGTABLE_MAX_LEVELS + 1] = {0};
	uint64_t n;
	uint64_t dvma;

	dvma = sdvma;
	n = snpages;

	while (n > 0) {
		/* setup the xlate array */
		xlate_setup(dvma, xlate, nlevels);

		/* just lookup existing pgtables. Should never fail */
		if (!PDE_lookup(domain, xlate, nlevels))
			ddi_err(DER_PANIC, rdip,
			    "PTE not found for addr %" PRIx64,
			    (unsigned long long)dvma);

		/* clear all matching ptes that fit into this leaf pgtable */
		PTE_clear_all(immu, domain, &xlate[1], &dvma, &n, rdip);
	}

	/* No need to flush IOTLB after unmap */
}

static uint64_t
dvma_alloc(domain_t *domain, ddi_dma_attr_t *dma_attr, uint_t npages, int kmf)
{
	uint64_t dvma;
	size_t xsize, align;
	uint64_t minaddr, maxaddr;

	/* parameters */
	xsize = npages * IMMU_PAGESIZE;
	align = MAX((size_t)(dma_attr->dma_attr_align), IMMU_PAGESIZE);
	minaddr = dma_attr->dma_attr_addr_lo;
	maxaddr = dma_attr->dma_attr_addr_hi + 1;

	/* handle the rollover cases */
	if (maxaddr < dma_attr->dma_attr_addr_hi) {
		maxaddr = dma_attr->dma_attr_addr_hi;
	}

	/*
	 * allocate from vmem arena.
	 */
	dvma = (uint64_t)(uintptr_t)vmem_xalloc(domain->dom_dvma_arena,
	    xsize, align, 0, 0, (void *)(uintptr_t)minaddr,
	    (void *)(uintptr_t)maxaddr, kmf);

	return (dvma);
}

static void
dvma_prealloc(dev_info_t *rdip, immu_hdl_priv_t *ihp, ddi_dma_attr_t *dma_attr)
{
	int nlevels;
	xlate_t xlate[IMMU_PGTABLE_MAX_LEVELS + 1] = {0}, *xlp;
	uint64_t dvma, n;
	size_t xsize, align;
	uint64_t minaddr, maxaddr, dmamax;
	int on, npte, pindex;
	hw_pdte_t *shwp;
	immu_t *immu;
	domain_t *domain;

	/* parameters */
	domain = IMMU_DEVI(rdip)->imd_domain;
	immu = domain->dom_immu;
	nlevels = immu->immu_dvma_nlevels;
	xsize = IMMU_NPREPTES * IMMU_PAGESIZE;
	align = MAX((size_t)(dma_attr->dma_attr_align), IMMU_PAGESIZE);
	minaddr = dma_attr->dma_attr_addr_lo;
	if (dma_attr->dma_attr_flags & _DDI_DMA_BOUNCE_ON_SEG)
		dmamax = dma_attr->dma_attr_seg;
	else
		dmamax = dma_attr->dma_attr_addr_hi;
	maxaddr = dmamax + 1;

	if (maxaddr < dmamax)
		maxaddr = dmamax;

	dvma = (uint64_t)(uintptr_t)vmem_xalloc(domain->dom_dvma_arena,
	    xsize, align, 0, dma_attr->dma_attr_seg + 1,
	    (void *)(uintptr_t)minaddr, (void *)(uintptr_t)maxaddr, VM_NOSLEEP);

	ihp->ihp_predvma = dvma;
	ihp->ihp_npremapped = 0;
	if (dvma == 0)
		return;

	n = IMMU_NPREPTES;
	pindex = 0;

	/*
	 * Set up a mapping at address 0, just so that all PDPs get allocated
	 * now. Although this initial mapping should never be used,
	 * explicitly set it to read-only, just to be safe.
	 */
	while (n > 0) {
		xlate_setup(dvma, xlate, nlevels);

		(void) PDE_set_all(immu, domain, xlate, nlevels, rdip,
		    IMMU_FLAGS_READ | IMMU_FLAGS_WRITE);

		xlp = &xlate[1];
		shwp = (hw_pdte_t *)(xlp->xlt_pgtable->hwpg_vaddr)
		    + xlp->xlt_idx;
		on = n;

		PTE_set_all(immu, domain, xlp, &dvma, &n, &immu_precookie,
		    1, rdip, IMMU_FLAGS_READ);

		npte = on - n;

		while (npte > 0) {
			ihp->ihp_preptes[pindex++] = shwp;
#ifdef BUGGY_DRIVERS
			PDTE_CLEAR_WRITE(*shwp);
#endif
			shwp++;
			npte--;
		}
	}
}

static void
dvma_prefree(dev_info_t *rdip, immu_hdl_priv_t *ihp)
{
	domain_t *domain;

	domain = IMMU_DEVI(rdip)->imd_domain;

	if (ihp->ihp_predvma != 0) {
		dvma_unmap(domain, ihp->ihp_predvma, IMMU_NPREPTES, rdip);
		vmem_free(domain->dom_dvma_arena,
		    (void *)(uintptr_t)ihp->ihp_predvma,
		    IMMU_NPREPTES * IMMU_PAGESIZE);
	}
}

static void
dvma_free(domain_t *domain, uint64_t dvma, uint64_t npages)
{
	uint64_t size = npages * IMMU_PAGESIZE;

	if (domain->dom_maptype != IMMU_MAPTYPE_XLATE)
		return;

	vmem_free(domain->dom_dvma_arena, (void *)(uintptr_t)dvma, size);
}

static int
immu_map_dvmaseg(dev_info_t *rdip, ddi_dma_handle_t handle,
    immu_hdl_priv_t *ihp, struct ddi_dma_req *dmareq,
    ddi_dma_obj_t *dma_out)
{
	domain_t *domain;
	immu_t *immu;
	immu_flags_t immu_flags;
	ddi_dma_atyp_t buftype;
	ddi_dma_obj_t *dmar_object;
	ddi_dma_attr_t *attrp;
	uint64_t offset, paddr, dvma, sdvma, rwmask;
	size_t npages, npgalloc;
	uint_t psize, size, pcnt, dmax;
	page_t **pparray;
	caddr_t vaddr;
	page_t *page;
	struct as *vas;
	immu_dcookie_t *dcookies;
	int pde_set;

	domain = IMMU_DEVI(rdip)->imd_domain;
	immu = domain->dom_immu;
	immu_flags = dma_to_immu_flags(dmareq);

	attrp = &((ddi_dma_impl_t *)handle)->dmai_attr;

	dmar_object = &dmareq->dmar_object;
	pparray = dmar_object->dmao_obj.virt_obj.v_priv;
	vaddr = dmar_object->dmao_obj.virt_obj.v_addr;
	buftype = dmar_object->dmao_type;
	size = dmar_object->dmao_size;

	IMMU_DPROBE3(immu__map__dvma, dev_info_t *, rdip, ddi_dma_atyp_t,
	    buftype, uint_t, size);

	dcookies = &ihp->ihp_dcookies[0];

	pcnt = dmax = 0;

	/* retrieve paddr, psize, offset from dmareq */
	if (buftype == DMA_OTYP_PAGES) {
		page = dmar_object->dmao_obj.pp_obj.pp_pp;
		offset =  dmar_object->dmao_obj.pp_obj.pp_offset &
		    MMU_PAGEOFFSET;
		paddr = pfn_to_pa(page->p_pagenum) + offset;
		psize = MIN((MMU_PAGESIZE - offset), size);
		page = page->p_next;
		vas = dmar_object->dmao_obj.virt_obj.v_as;
	} else {
		if (vas == NULL) {
			vas = &kas;
		}
		offset = (uintptr_t)vaddr & MMU_PAGEOFFSET;
		if (pparray != NULL) {
			paddr = pfn_to_pa(pparray[pcnt]->p_pagenum) + offset;
			psize = MIN((MMU_PAGESIZE - offset), size);
			pcnt++;
		} else {
			paddr = pfn_to_pa(hat_getpfnum(vas->a_hat,
			    vaddr)) + offset;
			psize = MIN(size, (MMU_PAGESIZE - offset));
			vaddr += psize;
		}
	}

	npgalloc = IMMU_BTOPR(size + offset);

	if (npgalloc <= IMMU_NPREPTES && ihp->ihp_predvma != 0) {
#ifdef BUGGY_DRIVERS
		rwmask = PDTE_MASK_R | PDTE_MASK_W | immu->immu_ptemask;
#else
		rwmask = immu->immu_ptemask;
		if (immu_flags & IMMU_FLAGS_READ)
			rwmask |= PDTE_MASK_R;
		if (immu_flags & IMMU_FLAGS_WRITE)
			rwmask |= PDTE_MASK_W;
#endif
#ifdef DEBUG
		rwmask |= PDTE_MASK_P;
#endif
		sdvma = ihp->ihp_predvma;
		ihp->ihp_npremapped = npgalloc;
		*ihp->ihp_preptes[0] =
		    PDTE_PADDR(paddr & ~MMU_PAGEOFFSET) | rwmask;
	} else {
		ihp->ihp_npremapped = 0;
		sdvma = dvma_alloc(domain, attrp, npgalloc,
		    dmareq->dmar_fp == DDI_DMA_SLEEP ? VM_SLEEP : VM_NOSLEEP);
		if (sdvma == 0)
			return (DDI_DMA_NORESOURCES);

		dcookies[0].dck_paddr = (paddr & ~MMU_PAGEOFFSET);
		dcookies[0].dck_npages = 1;
	}

	IMMU_DPROBE3(immu__dvma__alloc, dev_info_t *, rdip, uint64_t, npgalloc,
	    uint64_t, sdvma);

	dvma = sdvma;
	pde_set = 0;
	npages = 1;
	size -= psize;
	while (size > 0) {
		/* get the size for this page (i.e. partial or full page) */
		psize = MIN(size, MMU_PAGESIZE);
		if (buftype == DMA_OTYP_PAGES) {
			/* get the paddr from the page_t */
			paddr = pfn_to_pa(page->p_pagenum);
			page = page->p_next;
		} else if (pparray != NULL) {
			/* index into the array of page_t's to get the paddr */
			paddr = pfn_to_pa(pparray[pcnt]->p_pagenum);
			pcnt++;
		} else {
			/* call into the VM to get the paddr */
			paddr = pfn_to_pa(hat_getpfnum(vas->a_hat, vaddr));
			vaddr += psize;
		}

		npages++;

		if (ihp->ihp_npremapped > 0) {
			*ihp->ihp_preptes[npages - 1] =
			    PDTE_PADDR(paddr) | rwmask;
		} else if (IMMU_CONTIG_PADDR(dcookies[dmax], paddr)) {
			dcookies[dmax].dck_npages++;
		} else {
			/* No, we need a new dcookie */
			if (dmax == (IMMU_NDCK - 1)) {
				/*
				 * Ran out of dcookies. Map them now.
				 */
				if (dvma_map(domain, dvma,
				    npages, dcookies, dmax + 1, rdip,
				    immu_flags))
					pde_set++;

				IMMU_DPROBE4(immu__dvmamap__early,
				    dev_info_t *, rdip, uint64_t, dvma,
				    uint_t, npages, uint_t, dmax+1);

				dvma += (npages << IMMU_PAGESHIFT);
				npages = 0;
				dmax = 0;
			} else
				dmax++;
			dcookies[dmax].dck_paddr = paddr;
			dcookies[dmax].dck_npages = 1;
		}
		size -= psize;
	}

	/*
	 * Finish up, mapping all, or all of the remaining,
	 * physical memory ranges.
	 */
	if (ihp->ihp_npremapped == 0 && npages > 0) {
		IMMU_DPROBE4(immu__dvmamap__late, dev_info_t *, rdip, \
		    uint64_t, dvma, uint_t, npages, uint_t, dmax+1);

		if (dvma_map(domain, dvma, npages, dcookies,
		    dmax + 1, rdip, immu_flags))
			pde_set++;
	}

	/* Invalidate the IOTLB */
	immu_flush_iotlb_psi(immu, domain->dom_did, sdvma, npgalloc,
	    pde_set > 0 ? TLB_IVA_WHOLE : TLB_IVA_LEAF,
	    &ihp->ihp_inv_wait);

	ihp->ihp_ndvseg = 1;
	ihp->ihp_dvseg[0].dvs_start = sdvma;
	ihp->ihp_dvseg[0].dvs_len = dmar_object->dmao_size;

	dma_out->dmao_size = dmar_object->dmao_size;
	dma_out->dmao_obj.dvma_obj.dv_off = offset & IMMU_PAGEOFFSET;
	dma_out->dmao_obj.dvma_obj.dv_nseg = 1;
	dma_out->dmao_obj.dvma_obj.dv_seg = &ihp->ihp_dvseg[0];
	dma_out->dmao_type = DMA_OTYP_DVADDR;

	return (DDI_DMA_MAPPED);
}

static int
immu_unmap_dvmaseg(dev_info_t *rdip, ddi_dma_obj_t *dmao)
{
	uint64_t dvma, npages;
	domain_t *domain;
	struct dvmaseg *dvs;

	domain = IMMU_DEVI(rdip)->imd_domain;
	dvs = dmao->dmao_obj.dvma_obj.dv_seg;

	dvma = dvs[0].dvs_start;
	npages = IMMU_BTOPR(dvs[0].dvs_len + dmao->dmao_obj.dvma_obj.dv_off);

#ifdef DEBUG
	/* Unmap only in DEBUG mode */
	dvma_unmap(domain, dvma, npages, rdip);
#endif
	dvma_free(domain, dvma, npages);

	IMMU_DPROBE3(immu__dvma__free, dev_info_t *, rdip, uint_t, npages,
	    uint64_t, dvma);

#ifdef DEBUG
	/*
	 * In the DEBUG case, the unmap was actually done,
	 * but an IOTLB flush was not done. So, an explicit
	 * write back flush is needed.
	 */
	immu_regs_wbf_flush(domain->dom_immu);
#endif

	return (DDI_SUCCESS);
}

/* ############################# Functions exported ######################## */

/*
 * setup the DVMA subsystem
 * this code runs only for the first IOMMU unit
 */
void
immu_dvma_setup(list_t *listp)
{
	immu_t *immu;
	uint_t kval;
	size_t nchains;

	/* locks */
	mutex_init(&immu_domain_lock, NULL, MUTEX_DEFAULT, NULL);

	/* Create lists */
	list_create(&immu_unity_domain_list, sizeof (domain_t),
	    offsetof(domain_t, dom_maptype_node));
	list_create(&immu_xlate_domain_list, sizeof (domain_t),
	    offsetof(domain_t, dom_maptype_node));

	/* Setup BDF domain hash */
	nchains = 0xff;
	kval = mod_hash_iddata_gen(nchains);

	bdf_domain_hash = mod_hash_create_extended("BDF-DOMAIN_HASH",
	    nchains, mod_hash_null_keydtor, mod_hash_null_valdtor,
	    mod_hash_byid, (void *)(uintptr_t)kval, mod_hash_idkey_cmp,
	    KM_NOSLEEP);

	immu = list_head(listp);
	for (; immu; immu = list_next(listp, immu)) {
		create_unity_domain(immu);
		did_init(immu);
		context_init(immu);
		immu->immu_dvma_setup = B_TRUE;
	}
}

/*
 * Startup up one DVMA unit
 */
void
immu_dvma_startup(immu_t *immu)
{
	if (immu_gfxdvma_enable == B_FALSE &&
	    immu->immu_dvma_gfx_only == B_TRUE) {
		return;
	}

	/*
	 * DVMA will start once IOMMU is "running"
	 */
	immu->immu_dvma_running = B_TRUE;
}

/*
 * immu_dvma_physmem_update()
 *       called when the installed memory on a
 *       system increases, to expand domain DVMA
 *       for domains with UNITY mapping
 */
void
immu_dvma_physmem_update(uint64_t addr, uint64_t size)
{
	uint64_t start;
	uint64_t npages;
	int dcount;
	immu_dcookie_t dcookies[1] = {0};
	domain_t *domain;

	/*
	 * Just walk the system-wide list of domains with
	 * UNITY mapping. Both the list of *all* domains
	 * and *UNITY* domains is protected by the same
	 * single lock
	 */
	mutex_enter(&immu_domain_lock);
	domain = list_head(&immu_unity_domain_list);
	for (; domain; domain = list_next(&immu_unity_domain_list, domain)) {
		/*
		 * Nothing to do if the IOMMU supports passthrough.
		 */
		if (IMMU_ECAP_GET_PT(domain->dom_immu->immu_regs_excap))
			continue;

		/* There is no vmem_arena for unity domains. Just map it */
		ddi_err(DER_LOG, domain->dom_dip,
		    "iommu: unity-domain: Adding map "
		    "[0x%" PRIx64 " - 0x%" PRIx64 "]", addr, addr + size);

		start = IMMU_ROUNDOWN(addr);
		npages = (IMMU_ROUNDUP(size) / IMMU_PAGESIZE) + 1;

		dcookies[0].dck_paddr = start;
		dcookies[0].dck_npages = npages;
		dcount = 1;
		(void) dvma_map(domain, start, npages,
		    dcookies, dcount, NULL, IMMU_FLAGS_READ | IMMU_FLAGS_WRITE);

	}
	mutex_exit(&immu_domain_lock);
}

int
immu_dvma_device_setup(dev_info_t *rdip, immu_flags_t immu_flags)
{
	dev_info_t *ddip, *odip;
	immu_t *immu;
	domain_t *domain;

	odip = rdip;

	immu = immu_dvma_get_immu(rdip, immu_flags);
	if (immu == NULL) {
		/*
		 * possible that there is no IOMMU unit for this device
		 * - BIOS bugs are one example.
		 */
		ddi_err(DER_WARN, rdip, "No iommu unit found for device");
		return (DDI_DMA_NORESOURCES);
	}

	/*
	 * redirect isa devices attached under lpc to lpc dip
	 */
	if (strcmp(ddi_node_name(ddi_get_parent(rdip)), "isa") == 0) {
		rdip = get_lpc_devinfo(immu, rdip, immu_flags);
		if (rdip == NULL) {
			ddi_err(DER_PANIC, rdip, "iommu redirect failed");
			/*NOTREACHED*/
		}
	}

	/* Reset immu, as redirection can change IMMU */
	immu = NULL;

	/*
	 * for gart, redirect to the real graphic devinfo
	 */
	if (strcmp(ddi_node_name(rdip), "agpgart") == 0) {
		rdip = get_gfx_devinfo(rdip);
		if (rdip == NULL) {
			ddi_err(DER_PANIC, rdip, "iommu redirect failed");
			/*NOTREACHED*/
		}
	}

	/*
	 * Setup DVMA domain for the device. This does
	 * work only the first time we do DVMA for a
	 * device.
	 */
	ddip = NULL;
	domain = device_domain(rdip, &ddip, immu_flags);
	if (domain == NULL) {
		ddi_err(DER_MODE, rdip, "Intel IOMMU setup failed for device");
		return (DDI_DMA_NORESOURCES);
	}

	immu = domain->dom_immu;

	/*
	 * If a domain is found, we must also have a domain dip
	 * which is the topmost ancestor dip of rdip that shares
	 * the same domain with rdip.
	 */
	if (domain->dom_did == 0 || ddip == NULL) {
		ddi_err(DER_MODE, rdip, "domain did 0(%d) or ddip NULL(%p)",
		    domain->dom_did, ddip);
		return (DDI_DMA_NORESOURCES);
	}

	if (odip != rdip)
		set_domain(odip, ddip, domain);

	/*
	 * Update the root and context entries
	 */
	if (immu_context_update(immu, domain, ddip, rdip, immu_flags)
	    != DDI_SUCCESS) {
		ddi_err(DER_MODE, rdip, "DVMA map: context update failed");
		return (DDI_DMA_NORESOURCES);
	}

	return (DDI_SUCCESS);
}

int
immu_map_memrange(dev_info_t *rdip, memrng_t *mrng)
{
	immu_dcookie_t dcookies[1] = {0};
	boolean_t pde_set;
	immu_t *immu;
	domain_t *domain;
	immu_inv_wait_t iw;

	dcookies[0].dck_paddr = mrng->mrng_start;
	dcookies[0].dck_npages = mrng->mrng_npages;

	domain = IMMU_DEVI(rdip)->imd_domain;
	immu = domain->dom_immu;

	pde_set = dvma_map(domain, mrng->mrng_start,
	    mrng->mrng_npages, dcookies, 1, rdip,
	    IMMU_FLAGS_READ | IMMU_FLAGS_WRITE);

	immu_init_inv_wait(&iw, "memrange", B_TRUE);

	immu_flush_iotlb_psi(immu, domain->dom_did, mrng->mrng_start,
	    mrng->mrng_npages, pde_set == B_TRUE ?
	    TLB_IVA_WHOLE : TLB_IVA_LEAF, &iw);

	return (DDI_SUCCESS);
}

immu_devi_t *
immu_devi_get(dev_info_t *rdip)
{
	immu_devi_t *immu_devi;
	volatile uintptr_t *vptr = (uintptr_t *)&(DEVI(rdip)->devi_iommu);

	/* Just want atomic reads. No need for lock */
	immu_devi = (immu_devi_t *)(uintptr_t)atomic_or_64_nv((uint64_t *)vptr,
	    0);
	return (immu_devi);
}

/*ARGSUSED*/
int
immu_hdl_priv_ctor(void *buf, void *arg, int kmf)
{
	immu_hdl_priv_t *ihp;

	ihp = buf;
	immu_init_inv_wait(&ihp->ihp_inv_wait, "dmahandle", B_FALSE);

	return (0);
}

/*
 * iommulib interface functions
 */
static int
immu_probe(iommulib_handle_t handle, dev_info_t *dip)
{
	immu_devi_t *immu_devi;
	int ret;

	if (!immu_enable)
		return (DDI_FAILURE);

	/*
	 * Make sure the device has all the IOMMU structures
	 * initialized. If this device goes through an IOMMU
	 * unit (e.g. this probe function returns success),
	 * this will be called at most N times, with N being
	 * the number of IOMMUs in the system.
	 *
	 * After that, when iommulib_nex_open succeeds,
	 * we can always assume that this device has all
	 * the structures initialized. IOMMU_USED(dip) will
	 * be true. There is no need to find the controlling
	 * IOMMU/domain again.
	 */
	ret = immu_dvma_device_setup(dip, IMMU_FLAGS_NOSLEEP);
	if (ret != DDI_SUCCESS)
		return (ret);

	immu_devi = IMMU_DEVI(dip);

	/*
	 * For unity domains, there is no need to call in to
	 * the IOMMU code.
	 */
	if (immu_devi->imd_domain->dom_did == IMMU_UNITY_DID)
		return (DDI_FAILURE);

	if (immu_devi->imd_immu->immu_dip == iommulib_iommu_getdip(handle))
		return (DDI_SUCCESS);

	return (DDI_FAILURE);
}

/*ARGSUSED*/
static int
immu_allochdl(iommulib_handle_t handle,
    dev_info_t *dip, dev_info_t *rdip, ddi_dma_attr_t *attr,
    int (*waitfp)(caddr_t), caddr_t arg, ddi_dma_handle_t *dma_handlep)
{
	int ret;
	immu_hdl_priv_t *ihp;
	immu_t *immu;

	ret = iommulib_iommu_dma_allochdl(dip, rdip, attr, waitfp,
	    arg, dma_handlep);
	if (ret == DDI_SUCCESS) {
		immu = IMMU_DEVI(rdip)->imd_immu;

		ihp = kmem_cache_alloc(immu->immu_hdl_cache,
		    waitfp == DDI_DMA_SLEEP ? KM_SLEEP : KM_NOSLEEP);
		if (ihp == NULL) {
			(void) iommulib_iommu_dma_freehdl(dip, rdip,
			    *dma_handlep);
			return (DDI_DMA_NORESOURCES);
		}

		if (IMMU_DEVI(rdip)->imd_use_premap)
			dvma_prealloc(rdip, ihp, attr);
		else {
			ihp->ihp_npremapped = 0;
			ihp->ihp_predvma = 0;
		}
		ret = iommulib_iommu_dmahdl_setprivate(dip, rdip, *dma_handlep,
		    ihp);
	}
	return (ret);
}

/*ARGSUSED*/
static int
immu_freehdl(iommulib_handle_t handle,
    dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t dma_handle)
{
	immu_hdl_priv_t *ihp;

	ihp = iommulib_iommu_dmahdl_getprivate(dip, rdip, dma_handle);
	if (ihp != NULL) {
		if (IMMU_DEVI(rdip)->imd_use_premap)
			dvma_prefree(rdip, ihp);
		kmem_cache_free(IMMU_DEVI(rdip)->imd_immu->immu_hdl_cache, ihp);
	}

	return (iommulib_iommu_dma_freehdl(dip, rdip, dma_handle));
}


/*ARGSUSED*/
static int
immu_bindhdl(iommulib_handle_t handle, dev_info_t *dip,
    dev_info_t *rdip, ddi_dma_handle_t dma_handle,
    struct ddi_dma_req *dma_req, ddi_dma_cookie_t *cookiep,
    uint_t *ccountp)
{
	int ret;
	immu_hdl_priv_t *ihp;

	ret = iommulib_iommu_dma_bindhdl(dip, rdip, dma_handle,
	    dma_req, cookiep, ccountp);

	if (ret == DDI_DMA_MAPPED) {
		ihp = iommulib_iommu_dmahdl_getprivate(dip, rdip, dma_handle);
		immu_flush_wait(IMMU_DEVI(rdip)->imd_immu, &ihp->ihp_inv_wait);
	}

	return (ret);
}

/*ARGSUSED*/
static int
immu_unbindhdl(iommulib_handle_t handle,
    dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t dma_handle)
{
	return (iommulib_iommu_dma_unbindhdl(dip, rdip, dma_handle));
}

/*ARGSUSED*/
static int
immu_sync(iommulib_handle_t handle, dev_info_t *dip,
    dev_info_t *rdip, ddi_dma_handle_t dma_handle, off_t off,
    size_t len, uint_t cachefl)
{
	return (iommulib_iommu_dma_sync(dip, rdip, dma_handle, off, len,
	    cachefl));
}

/*ARGSUSED*/
static int
immu_win(iommulib_handle_t handle, dev_info_t *dip,
    dev_info_t *rdip, ddi_dma_handle_t dma_handle, uint_t win,
    off_t *offp, size_t *lenp, ddi_dma_cookie_t *cookiep,
    uint_t *ccountp)
{
	return (iommulib_iommu_dma_win(dip, rdip, dma_handle, win, offp,
	    lenp, cookiep, ccountp));
}

/*ARGSUSED*/
static int
immu_mapobject(iommulib_handle_t handle, dev_info_t *dip,
    dev_info_t *rdip, ddi_dma_handle_t dma_handle,
    struct ddi_dma_req *dmareq, ddi_dma_obj_t *dmao)
{
	immu_hdl_priv_t *ihp;

	ihp = iommulib_iommu_dmahdl_getprivate(dip, rdip, dma_handle);

	return (immu_map_dvmaseg(rdip, dma_handle, ihp, dmareq, dmao));
}

/*ARGSUSED*/
static int
immu_unmapobject(iommulib_handle_t handle, dev_info_t *dip,
    dev_info_t *rdip, ddi_dma_handle_t dma_handle, ddi_dma_obj_t *dmao)
{
	immu_hdl_priv_t *ihp;

	ihp = iommulib_iommu_dmahdl_getprivate(dip, rdip, dma_handle);
	if (ihp->ihp_npremapped > 0)
		return (DDI_SUCCESS);
	return (immu_unmap_dvmaseg(rdip, dmao));
}

/*ARGSUSED*/
static int
immu_map(iommulib_handle_t handle, dev_info_t *dip,
    dev_info_t *rdip, struct ddi_dma_req *dmareq,
    ddi_dma_handle_t *dma_handle)
{
	ASSERT(0);
	return (DDI_FAILURE);
}

/*ARGSUSED*/
static int
immu_mctl(iommulib_handle_t handle, dev_info_t *dip,
    dev_info_t *rdip, ddi_dma_handle_t dma_handle,
    enum ddi_dma_ctlops request, off_t *offp, size_t *lenp,
    caddr_t *objpp, uint_t cachefl)
{
	ASSERT(0);
	return (DDI_FAILURE);
}