x86/pci/msipic.c

/*	$NetBSD: msipic.c,v 1.27 2022/05/24 14:00:23 bouyer Exp $	*/

/*
 * Copyright (c) 2015 Internet Initiative Japan Inc.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: msipic.c,v 1.27 2022/05/24 14:00:23 bouyer Exp $");

#include "opt_intrdebug.h"

#include <sys/types.h>
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/errno.h>
#include <sys/kmem.h>
#include <sys/mutex.h>
#include <sys/bitops.h>

#include <dev/pci/pcivar.h>

#include <machine/i82489reg.h>
#include <machine/i82489var.h>
#include <machine/i82093reg.h>
#include <machine/i82093var.h>
#include <machine/pic.h>
#include <machine/lock.h>

#include <x86/pci/msipic.h>

#ifdef INTRDEBUG
#define MSIPICDEBUG
#endif

#ifdef MSIPICDEBUG
#define DPRINTF(msg) printf msg
#else
#define DPRINTF(msg)
#endif

#define BUS_SPACE_WRITE_FLUSH(pc, tag) (void)bus_space_read_4(pc, tag, 0)

#define MSIPICNAMEBUF 16

/*
 * A Pseudo pic for single MSI/MSI-X device.
 * The pic and MSI/MSI-X device are distinbuished by "devid". The "devid"
 * is managed by below "dev_seqs".
 */
struct msipic {
	struct msipic_pci_info mp_i;

	int mp_devid; /* The device id for the MSI/MSI-X device. */

	char mp_pic_name[MSIPICNAMEBUF]; /* The MSI/MSI-X device's name. */

	struct pci_attach_args mp_pa;
	bus_space_tag_t mp_bstag;
	bus_space_handle_t mp_bshandle;
	bus_size_t mp_bssize;
	struct pic *mp_pic;

	LIST_ENTRY(msipic) mp_list;
};

static kmutex_t msipic_list_lock;

static LIST_HEAD(, msipic) msipic_list =
	LIST_HEAD_INITIALIZER(msipic_list);

/*
 * This struct managements "devid" to use the same "devid" for the device
 * re-attached. If the device's bus number and device number and function
 * number are equal, it is assumed re-attached.
 */
struct dev_last_used_seq {
	bool ds_using;
	int ds_bus;
	int ds_dev;
	int ds_fun;
};
/* The number of MSI/MSI-X devices supported by system. */
#define NUM_MSI_DEVS 256
/* Record devids to use the same devid when the device is re-attached. */
static struct dev_last_used_seq dev_seqs[NUM_MSI_DEVS];

static int msipic_allocate_common_msi_devid(const struct pci_attach_args *);
static void msipic_release_common_msi_devid(int);

static struct pic *msipic_find_msi_pic_locked(int);
static struct pic *msipic_construct_common_msi_pic(const struct pci_attach_args *,
						   const struct pic *);
static void msipic_destruct_common_msi_pic(struct pic *);

static void msi_set_msictl_enablebit(struct pic *, int, int);
static void msi_hwmask(struct pic *, int);
static void msi_hwunmask(struct pic *, int);
static void msi_addroute(struct pic *, struct cpu_info *, int, int, int);
static void msi_delroute(struct pic *, struct cpu_info *, int, int, int);

static void msix_set_vecctl_mask(struct pic *, int, int);
static void msix_hwmask(struct pic *, int);
static void msix_hwunmask(struct pic *, int);
static void msix_addroute(struct pic *, struct cpu_info *, int, int, int);
static void msix_delroute(struct pic *, struct cpu_info *, int, int, int);

/*
 * Return new "devid" for the device attached first.
 * Return the same "devid" for the device re-attached after dettached once.
 * Return -1 if the number of attached MSI/MSI-X devices is over NUM_MSI_DEVS.
 */
static int
msipic_allocate_common_msi_devid(const struct pci_attach_args *pa)
{
	pci_chipset_tag_t pc;
	pcitag_t tag;
	int bus, dev, fun, i;

	KASSERT(mutex_owned(&msipic_list_lock));

	pc = pa->pa_pc;
	tag = pa->pa_tag;
	pci_decompose_tag(pc, tag, &bus, &dev, &fun);

	/* if the device was once attached, use same devid */
	for (i = 0; i < NUM_MSI_DEVS; i++) {
		/* skip host bridge */
		if (dev_seqs[i].ds_bus == 0
		    && dev_seqs[i].ds_dev == 0
		    && dev_seqs[i].ds_fun == 0)
			break;

		if (dev_seqs[i].ds_bus == bus
		    && dev_seqs[i].ds_dev == dev
		    && dev_seqs[i].ds_fun == fun) {
			dev_seqs[i].ds_using = true;
			return i;
		}
	}

	for (i = 0; i < NUM_MSI_DEVS; i++) {
		if (dev_seqs[i].ds_using == 0) {
			dev_seqs[i].ds_using = true;
			dev_seqs[i].ds_bus = bus;
			dev_seqs[i].ds_dev = dev;
			dev_seqs[i].ds_fun = fun;
			return i;
		}
	}

	DPRINTF(("too many MSI devices.\n"));
	return -1;
}

/*
 * Set the "devid" unused, but keep reserving the "devid" to reuse when
 * the device is re-attached.
 */
static void
msipic_release_common_msi_devid(int devid)
{

	KASSERT(mutex_owned(&msipic_list_lock));

	if (devid < 0 || NUM_MSI_DEVS <= devid) {
		DPRINTF(("%s: invalid devid.\n", __func__));
		return;
	}

	dev_seqs[devid].ds_using = false;
	/* Keep ds_* to reuse the same devid for the same device. */
}

static struct pic *
msipic_find_msi_pic_locked(int devid)
{
	struct msipic *mpp;

	KASSERT(mutex_owned(&msipic_list_lock));

	LIST_FOREACH(mpp, &msipic_list, mp_list) {
		if (mpp->mp_devid == devid)
			return mpp->mp_pic;
	}
	return NULL;
}

/*
 * Return the msi_pic whose device is already registered.
 * If the device is not registered yet, return NULL.
 */
struct pic *
msipic_find_msi_pic(int devid)
{
	struct pic *msipic;

	mutex_enter(&msipic_list_lock);
	msipic = msipic_find_msi_pic_locked(devid);
	mutex_exit(&msipic_list_lock);

	return msipic;
}

/*
 * A common construct process of MSI and MSI-X.
 */
static struct pic *
msipic_construct_common_msi_pic(const struct pci_attach_args *pa,
    const struct pic *pic_tmpl)
{
	struct pic *pic;
	struct msipic *msipic;
	int devid;

	pic = kmem_alloc(sizeof(*pic), KM_SLEEP);
	msipic = kmem_zalloc(sizeof(*msipic), KM_SLEEP);

	mutex_enter(&msipic_list_lock);

	devid = msipic_allocate_common_msi_devid(pa);
	if (devid == -1) {
		mutex_exit(&msipic_list_lock);
		kmem_free(pic, sizeof(*pic));
		kmem_free(msipic, sizeof(*msipic));
		return NULL;
	}

	memcpy(pic, pic_tmpl, sizeof(*pic));
	pic->pic_edge_stubs
	    = x2apic_mode ? x2apic_edge_stubs : ioapic_edge_stubs;
	pic->pic_msipic = msipic;
	msipic->mp_pic = pic;
	pci_decompose_tag(pa->pa_pc, pa->pa_tag,
	    &msipic->mp_i.mp_bus, &msipic->mp_i.mp_dev, &msipic->mp_i.mp_fun);
	memcpy(&msipic->mp_pa, pa, sizeof(msipic->mp_pa));
	msipic->mp_devid = devid;
	/*
	 * pci_msi{,x}_alloc() must be called only once in the device driver.
	 */
	KASSERT(msipic_find_msi_pic_locked(msipic->mp_devid) == NULL);

	LIST_INSERT_HEAD(&msipic_list, msipic, mp_list);

	mutex_exit(&msipic_list_lock);

	return pic;
}

static void
msipic_destruct_common_msi_pic(struct pic *msi_pic)
{
	struct msipic *msipic;

	if (msi_pic == NULL)
		return;

	msipic = msi_pic->pic_msipic;
	mutex_enter(&msipic_list_lock);
	LIST_REMOVE(msipic, mp_list);
	msipic_release_common_msi_devid(msipic->mp_devid);
	mutex_exit(&msipic_list_lock);

	if (msipic->mp_i.mp_xen_pirq != NULL) {
		KASSERT(msipic->mp_i.mp_veccnt > 0);
#ifdef DIAGNOSTIC
		for (int i = 0; i < msipic->mp_i.mp_veccnt; i++) {
			KASSERT(msipic->mp_i.mp_xen_pirq[i] == 0);
		}
#endif
		kmem_free(msipic->mp_i.mp_xen_pirq,
	            sizeof(*msipic->mp_i.mp_xen_pirq) * msipic->mp_i.mp_veccnt);
	}
	kmem_free(msipic, sizeof(*msipic));
	kmem_free(msi_pic, sizeof(*msi_pic));
}

/*
 * The pic is MSI/MSI-X pic or not.
 */
bool
msipic_is_msi_pic(struct pic *pic)
{

	return (pic->pic_msipic != NULL);
}

/*
 * Return the MSI/MSI-X devid which is unique for each devices.
 */
int
msipic_get_devid(struct pic *pic)
{

	KASSERT(msipic_is_msi_pic(pic));

	return pic->pic_msipic->mp_devid;
}

/*
 * Return the PCI bus/dev/func info for the device.
 */
const struct msipic_pci_info *
msipic_get_pci_info(struct pic *pic)
{
	KASSERT(msipic_is_msi_pic(pic));

	return &pic->pic_msipic->mp_i;
}

#define MSI_MSICTL_ENABLE 1
#define MSI_MSICTL_DISABLE 0
static void
msi_set_msictl_enablebit(struct pic *pic, int msi_vec, int flag)
{
	pci_chipset_tag_t pc;
	struct pci_attach_args *pa;
	pcitag_t tag;
	pcireg_t ctl;
	int off, err __diagused;

	pc = NULL;
	pa = &pic->pic_msipic->mp_pa;
	tag = pa->pa_tag;
	err = pci_get_capability(pc, tag, PCI_CAP_MSI, &off, NULL);
	KASSERT(err != 0);

	/*
	 * MSI can establish only one vector at once.
	 * So, use whole device mask bit instead of a vector mask bit.
	 */
	ctl = pci_conf_read(pc, tag, off + PCI_MSI_CTL);
	if (flag == MSI_MSICTL_ENABLE)
		ctl |= PCI_MSI_CTL_MSI_ENABLE;
	else
		ctl &= ~PCI_MSI_CTL_MSI_ENABLE;

#ifdef XENPV
	pci_conf_write16(pc, tag, off + PCI_MSI_CTL + 2, ctl >> 16);
#else
	pci_conf_write(pc, tag, off, ctl);
#endif
}

static void
msi_hwmask(struct pic *pic, int msi_vec)
{

	msi_set_msictl_enablebit(pic, msi_vec, MSI_MSICTL_DISABLE);
}

/*
 * Do not use pic->hwunmask() immediately after pic->delroute().
 * It is required to use pic->addroute() before pic->hwunmask().
 */
static void
msi_hwunmask(struct pic *pic, int msi_vec)
{

	msi_set_msictl_enablebit(pic, msi_vec, MSI_MSICTL_ENABLE);
}

static void
msi_addroute(struct pic *pic, struct cpu_info *ci,
	     int unused, int idt_vec, int type)
{
	pci_chipset_tag_t pc;
	struct pci_attach_args *pa;
	pcitag_t tag;
#ifndef XENPV
	pcireg_t addr, data;
#endif
	pcireg_t ctl;
	int off, err __diagused;

	pc = NULL;
	pa = &pic->pic_msipic->mp_pa;
	tag = pa->pa_tag;
	err = pci_get_capability(pc, tag, PCI_CAP_MSI, &off, NULL);
	KASSERT(err != 0);

	ctl = pci_conf_read(pc, tag, off + PCI_MSI_CTL);
#ifndef XENPV
	/*
	 * See Intel 64 and IA-32 Architectures Software Developer's Manual
	 * Volume 3 10.11 Message Signalled Interrupts.
	 */
	/*
	 * "cpuid" for MSI address is local APIC ID. In NetBSD, the ID is
	 * the same as ci->ci_cpuid.
	 */
	addr = LAPIC_MSIADDR_BASE | __SHIFTIN(ci->ci_cpuid,
	    LAPIC_MSIADDR_DSTID_MASK);
	/* If trigger mode is edge, it don't care level for trigger mode. */
	data = __SHIFTIN(idt_vec, LAPIC_VECTOR_MASK)
		| LAPIC_TRIGMODE_EDGE | LAPIC_DLMODE_FIXED;

	/*
	 * The size of the message data register is 16bit if the extended
	 * message data is not implemented. If it's 16bit and the per-vector
	 * masking is not capable, the location of the upper 16bit is out of
	 * the MSI capability structure's range. The PCI spec says the upper
	 * 16bit is driven to 0 if the message data register is 16bit. It's the
	 * spec, so it's OK just to write it regardless of the value of the
	 * upper 16bit.
	 */
	if (ctl & PCI_MSI_CTL_64BIT_ADDR) {
		pci_conf_write(pc, tag, off + PCI_MSI_MADDR64_LO, addr);
		pci_conf_write(pc, tag, off + PCI_MSI_MADDR64_HI, 0);
		pci_conf_write(pc, tag, off + PCI_MSI_MDATA64, data);
	} else {
		pci_conf_write(pc, tag, off + PCI_MSI_MADDR, addr);
		pci_conf_write(pc, tag, off + PCI_MSI_MDATA, data);
	}
#endif /* !XENPV */
	ctl |= PCI_MSI_CTL_MSI_ENABLE;
#ifdef XENPV
	pci_conf_write16(pc, tag, off + PCI_MSI_CTL + 2, ctl >> 16);
#else
	pci_conf_write(pc, tag, off + PCI_MSI_CTL, ctl);
#endif
}

/*
 * Do not use pic->hwunmask() immediately after pic->delroute().
 * It is required to use pic->addroute() before pic->hwunmask().
 */
static void
msi_delroute(struct pic *pic, struct cpu_info *ci,
	     int msi_vec, int idt_vec, int type)
{

	msi_hwmask(pic, msi_vec);
}

/*
 * Template for MSI pic.
 * .pic_msipic is set later in construct_msi_pic().
 */
static const struct pic msi_pic_tmpl = {
	.pic_type = PIC_MSI,
	.pic_vecbase = 0,
	.pic_apicid = 0,
	.pic_lock = __SIMPLELOCK_UNLOCKED, /* not used for msi_pic */
	.pic_hwmask = msi_hwmask,
	.pic_hwunmask = msi_hwunmask,
	.pic_addroute = msi_addroute,
	.pic_delroute = msi_delroute,
	.pic_intr_get_devname = x86_intr_get_devname,
	.pic_intr_get_assigned = x86_intr_get_assigned,
	.pic_intr_get_count = x86_intr_get_count,
};

/*
 * Create pseudo pic for a MSI device.
 */
struct pic *
msipic_construct_msi_pic(const struct pci_attach_args *pa)
{
	struct pic *msi_pic;
	char pic_name_buf[MSIPICNAMEBUF];

	msi_pic = msipic_construct_common_msi_pic(pa, &msi_pic_tmpl);
	if (msi_pic == NULL) {
		DPRINTF(("cannot allocate MSI pic.\n"));
		return NULL;
	}

	memset(pic_name_buf, 0, MSIPICNAMEBUF);
	snprintf(pic_name_buf, MSIPICNAMEBUF, "msi%d",
	    msi_pic->pic_msipic->mp_devid);
	strncpy(msi_pic->pic_msipic->mp_pic_name, pic_name_buf,
	    MSIPICNAMEBUF - 1);
	msi_pic->pic_name = msi_pic->pic_msipic->mp_pic_name;

	return msi_pic;
}

/*
 * Delete pseudo pic for a MSI device.
 */
void
msipic_destruct_msi_pic(struct pic *msi_pic)
{

	msipic_destruct_common_msi_pic(msi_pic);
}

#define MSIX_VECCTL_HWMASK 1
#define MSIX_VECCTL_HWUNMASK 0
static void
msix_set_vecctl_mask(struct pic *pic, int msix_vec, int flag)
{
	bus_space_tag_t bstag;
	bus_space_handle_t bshandle;
	uint64_t entry_base;
	uint32_t vecctl;

	if (msix_vec < 0) {
		DPRINTF(("%s: invalid MSI-X table index, devid=%d vecid=%d",
			__func__, msipic_get_devid(pic), msix_vec));
		return;
	}

	entry_base = PCI_MSIX_TABLE_ENTRY_SIZE * msix_vec;

	bstag = pic->pic_msipic->mp_bstag;
	bshandle = pic->pic_msipic->mp_bshandle;
	vecctl = bus_space_read_4(bstag, bshandle,
	    entry_base + PCI_MSIX_TABLE_ENTRY_VECTCTL);
	if (flag == MSIX_VECCTL_HWMASK)
		vecctl |= PCI_MSIX_VECTCTL_MASK;
	else
		vecctl &= ~PCI_MSIX_VECTCTL_MASK;

	bus_space_write_4(bstag, bshandle,
	    entry_base + PCI_MSIX_TABLE_ENTRY_VECTCTL, vecctl);
	BUS_SPACE_WRITE_FLUSH(bstag, bshandle);
}

static void
msix_hwmask(struct pic *pic, int msix_vec)
{

	msix_set_vecctl_mask(pic, msix_vec, MSIX_VECCTL_HWMASK);
}

/*
 * Do not use pic->hwunmask() immediately after pic->delroute().
 * It is required to use pic->addroute() before pic->hwunmask().
 */
static void
msix_hwunmask(struct pic *pic, int msix_vec)
{

	msix_set_vecctl_mask(pic, msix_vec, MSIX_VECCTL_HWUNMASK);
}

static void
msix_addroute(struct pic *pic, struct cpu_info *ci,
	     int msix_vec, int idt_vec, int type)
{
	pci_chipset_tag_t pc;
	struct pci_attach_args *pa;
	pcitag_t tag;
#ifndef XENPV
	bus_space_tag_t bstag;
	bus_space_handle_t bshandle;
	uint64_t entry_base;
	pcireg_t addr, data;
#endif
	pcireg_t ctl;
	int off, err __diagused;

	if (msix_vec < 0) {
		DPRINTF(("%s: invalid MSI-X table index, devid=%d vecid=%d",
			__func__, msipic_get_devid(pic), msix_vec));
		return;
	}

	pa = &pic->pic_msipic->mp_pa;
	pc = pa->pa_pc;
	tag = pa->pa_tag;
	err = pci_get_capability(pc, tag, PCI_CAP_MSIX, &off, NULL);
	KASSERT(err != 0);

#ifndef XENPV
	/* Disable MSI-X before writing MSI-X table */
	ctl = pci_conf_read(pc, tag, off + PCI_MSIX_CTL);
	ctl &= ~PCI_MSIX_CTL_ENABLE;
	pci_conf_write(pc, tag, off + PCI_MSIX_CTL, ctl);

	bstag = pic->pic_msipic->mp_bstag;
	bshandle = pic->pic_msipic->mp_bshandle;
	entry_base = PCI_MSIX_TABLE_ENTRY_SIZE * msix_vec;

	/*
	 * See Intel 64 and IA-32 Architectures Software Developer's Manual
	 * Volume 3 10.11 Message Signalled Interrupts.
	 */
	/*
	 * "cpuid" for MSI-X address is local APIC ID. In NetBSD, the ID is
	 * the same as ci->ci_cpuid.
	 */
	addr = LAPIC_MSIADDR_BASE | __SHIFTIN(ci->ci_cpuid,
	    LAPIC_MSIADDR_DSTID_MASK);
	/* If trigger mode is edge, it don't care level for trigger mode. */
	data = __SHIFTIN(idt_vec, LAPIC_VECTOR_MASK)
		| LAPIC_TRIGMODE_EDGE | LAPIC_DLMODE_FIXED;

	bus_space_write_4(bstag, bshandle,
	    entry_base + PCI_MSIX_TABLE_ENTRY_ADDR_LO, addr);
	bus_space_write_4(bstag, bshandle,
	    entry_base + PCI_MSIX_TABLE_ENTRY_ADDR_HI, 0);
	bus_space_write_4(bstag, bshandle,
	    entry_base + PCI_MSIX_TABLE_ENTRY_DATA, data);
	BUS_SPACE_WRITE_FLUSH(bstag, bshandle);
#endif /* !XENPV */

	ctl = pci_conf_read(pc, tag, off + PCI_MSIX_CTL);
	if (ctl & PCI_MSIX_CTL_FUNCMASK) {
		ctl &= ~PCI_MSIX_CTL_FUNCMASK;
	}
	ctl |= PCI_MSIX_CTL_ENABLE;
#ifdef XENPV
	pci_conf_write16(pc, tag, off + PCI_MSIX_CTL + 2, ctl >> 16);
#else
	pci_conf_write(pc, tag, off + PCI_MSIX_CTL, ctl);
#endif
}

/*
 * Do not use pic->hwunmask() immediately after pic->delroute().
 * It is required to use pic->addroute() before pic->hwunmask().
 */
static void
msix_delroute(struct pic *pic, struct cpu_info *ci,
	     int msix_vec, int vec, int type)
{

	msix_hwmask(pic, msix_vec);
}

/*
 * Template for MSI-X pic.
 * .pic_msipic is set later in construct_msix_pic().
 */
static const struct pic msix_pic_tmpl = {
	.pic_type = PIC_MSIX,
	.pic_vecbase = 0,
	.pic_apicid = 0,
	.pic_lock = __SIMPLELOCK_UNLOCKED, /* not used for msix_pic */
	.pic_hwmask = msix_hwmask,
	.pic_hwunmask = msix_hwunmask,
	.pic_addroute = msix_addroute,
	.pic_delroute = msix_delroute,
	.pic_intr_get_devname = x86_intr_get_devname,
	.pic_intr_get_assigned = x86_intr_get_assigned,
	.pic_intr_get_count = x86_intr_get_count,
};

struct pic *
msipic_construct_msix_pic(const struct pci_attach_args *pa)
{
	struct pic *msix_pic;
	pci_chipset_tag_t pc;
	pcitag_t tag;
	pcireg_t tbl;
	bus_space_tag_t bstag;
	bus_space_handle_t bshandle;
	bus_size_t bssize;
	size_t table_size;
	uint32_t table_offset;
	u_int memtype;
	bus_addr_t memaddr;
	int flags;
	int bir, bar, err, off, table_nentry;
	char pic_name_buf[MSIPICNAMEBUF];

	table_nentry = pci_msix_count(pa->pa_pc, pa->pa_tag);
	if (table_nentry == 0) {
		DPRINTF(("MSI-X table entry is 0.\n"));
		return NULL;
	}

	pc = pa->pa_pc;
	tag = pa->pa_tag;
	if (pci_get_capability(pc, tag, PCI_CAP_MSIX, &off, NULL) == 0) {
		DPRINTF(("%s: no msix capability", __func__));
		return NULL;
	}

	msix_pic = msipic_construct_common_msi_pic(pa, &msix_pic_tmpl);
	if (msix_pic == NULL) {
		DPRINTF(("cannot allocate MSI-X pic.\n"));
		return NULL;
	}

	memset(pic_name_buf, 0, MSIPICNAMEBUF);
	snprintf(pic_name_buf, MSIPICNAMEBUF, "msix%d",
	    msix_pic->pic_msipic->mp_devid);
	strncpy(msix_pic->pic_msipic->mp_pic_name, pic_name_buf,
	    MSIPICNAMEBUF - 1);
	msix_pic->pic_name = msix_pic->pic_msipic->mp_pic_name;

	tbl = pci_conf_read(pc, tag, off + PCI_MSIX_TBLOFFSET);
	table_offset = tbl & PCI_MSIX_TBLOFFSET_MASK;
	bir = tbl & PCI_MSIX_TBLBIR_MASK;
	switch (bir) {
	case 0:
		bar = PCI_BAR0;
		break;
	case 1:
		bar = PCI_BAR1;
		break;
	case 2:
		bar = PCI_BAR2;
		break;
	case 3:
		bar = PCI_BAR3;
		break;
	case 4:
		bar = PCI_BAR4;
		break;
	case 5:
		bar = PCI_BAR5;
		break;
	default:
		aprint_error("detect an illegal device! "
		    "The device use reserved BIR values.\n");
		msipic_destruct_common_msi_pic(msix_pic);
		return NULL;
	}
	memtype = pci_mapreg_type(pc, tag, bar);
	/*
	 * PCI_MSIX_TABLE_ENTRY_SIZE consists below
	 *     - Vector Control (32bit)
	 *     - Message Data (32bit)
	 *     - Message Upper Address (32bit)
	 *     - Message Lower Address (32bit)
	 */
	table_size = table_nentry * PCI_MSIX_TABLE_ENTRY_SIZE;
#if 0
	err = pci_mapreg_submap(pa, bar, memtype, BUS_SPACE_MAP_LINEAR,
	    roundup(table_size, PAGE_SIZE), table_offset,
	    &bstag, &bshandle, NULL, &bssize);
#else
	/*
	 * Workaround for PCI prefetchable bit. Some chips (e.g. Intel 82599)
	 * report SERR and MSI-X doesn't work. This problem might not be the
	 * driver's bug but our PCI common part or VMs' bug. Until we find a
	 * real reason, we ignore the prefetchable bit.
	 */
	if (pci_mapreg_info(pa->pa_pc, pa->pa_tag, bar, memtype,
		&memaddr, NULL, &flags) != 0) {
		DPRINTF(("cannot get a map info.\n"));
		msipic_destruct_common_msi_pic(msix_pic);
		return NULL;
	}
	if ((flags & BUS_SPACE_MAP_PREFETCHABLE) != 0) {
		DPRINTF(( "clear prefetchable bit\n"));
		flags &= ~BUS_SPACE_MAP_PREFETCHABLE;
	}
	bssize = roundup(table_size, PAGE_SIZE);
	err = _x86_memio_map(pa->pa_memt, memaddr + table_offset, bssize, flags,
	    &bshandle);
	bstag = pa->pa_memt;
#endif
	if (err) {
		DPRINTF(("cannot map msix table.\n"));
		msipic_destruct_common_msi_pic(msix_pic);
		return NULL;
	}
	msix_pic->pic_msipic->mp_bstag = bstag;
	msix_pic->pic_msipic->mp_bshandle = bshandle;
	msix_pic->pic_msipic->mp_bssize = bssize;
	msix_pic->pic_msipic->mp_i.mp_table_base = memaddr;

	return msix_pic;
}

/*
 * Delete pseudo pic for a MSI-X device.
 */
void
msipic_destruct_msix_pic(struct pic *msix_pic)
{
	struct msipic *msipic;

	KASSERT(msipic_is_msi_pic(msix_pic));
	KASSERT(msix_pic->pic_type == PIC_MSIX);

	msipic = msix_pic->pic_msipic;
	_x86_memio_unmap(msipic->mp_bstag, msipic->mp_bshandle,
	    msipic->mp_bssize, NULL);

	msipic_destruct_common_msi_pic(msix_pic);
}

/*
 * Set the number of MSI vectors for pseudo MSI pic.
 */
int
msipic_set_msi_vectors(struct pic *msi_pic, pci_intr_handle_t *pihs,
    int count)
{

	KASSERT(msipic_is_msi_pic(msi_pic));

	if (msi_pic->pic_type == PIC_MSI) {
		pci_chipset_tag_t pc;
		struct pci_attach_args *pa;
		pcitag_t tag;
		int off, err __diagused;
		pcireg_t ctl;

		pc = NULL;
		pa = &msi_pic->pic_msipic->mp_pa;
		tag = pa->pa_tag;
		err = pci_get_capability(pc, tag, PCI_CAP_MSI, &off, NULL);
		KASSERT(err != 0);

		ctl = pci_conf_read(pc, tag, off + PCI_MSI_CTL);
		ctl &= ~PCI_MSI_CTL_MME_MASK;
		ctl |= __SHIFTIN(ilog2(count), PCI_MSI_CTL_MME_MASK);
		pci_conf_write(pc, tag, off + PCI_MSI_CTL, ctl);
	}

	msi_pic->pic_msipic->mp_i.mp_veccnt = count;
#ifdef XENPV
	msi_pic->pic_msipic->mp_i.mp_xen_pirq =
	    kmem_zalloc(sizeof(*msi_pic->pic_msipic->mp_i.mp_xen_pirq) * count,
	    KM_SLEEP);
#endif
	return 0;
}

/*
 * Initialize the system to use MSI/MSI-X.
 */
void
msipic_init(void)
{

	mutex_init(&msipic_list_lock, MUTEX_DEFAULT, IPL_NONE);
}