xref: /netbsd-src/sys/dev/pci/nvme_pci.c (revision 94acf541710557f8da9281dce936542d67dcaf9e)
1*94acf541Spgoyette /*	$NetBSD: nvme_pci.c,v 1.37 2022/08/15 18:06:04 pgoyette Exp $	*/
28b5163f0Snonaka /*	$OpenBSD: nvme_pci.c,v 1.3 2016/04/14 11:18:32 dlg Exp $ */
38b5163f0Snonaka 
48b5163f0Snonaka /*
58b5163f0Snonaka  * Copyright (c) 2014 David Gwynne <dlg@openbsd.org>
68b5163f0Snonaka  *
78b5163f0Snonaka  * Permission to use, copy, modify, and distribute this software for any
88b5163f0Snonaka  * purpose with or without fee is hereby granted, provided that the above
98b5163f0Snonaka  * copyright notice and this permission notice appear in all copies.
108b5163f0Snonaka  *
118b5163f0Snonaka  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
128b5163f0Snonaka  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
138b5163f0Snonaka  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
148b5163f0Snonaka  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
158b5163f0Snonaka  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
168b5163f0Snonaka  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
178b5163f0Snonaka  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
188b5163f0Snonaka  */
198b5163f0Snonaka 
208b5163f0Snonaka /*-
218b5163f0Snonaka  * Copyright (C) 2016 NONAKA Kimihiro <nonaka@netbsd.org>
228b5163f0Snonaka  * All rights reserved.
238b5163f0Snonaka  *
248b5163f0Snonaka  * Redistribution and use in source and binary forms, with or without
258b5163f0Snonaka  * modification, are permitted provided that the following conditions
268b5163f0Snonaka  * are met:
278b5163f0Snonaka  * 1. Redistributions of source code must retain the above copyright
288b5163f0Snonaka  *    notice, this list of conditions and the following disclaimer.
298b5163f0Snonaka  * 2. Redistributions in binary form must reproduce the above copyright
308b5163f0Snonaka  *    notice, this list of conditions and the following disclaimer in the
318b5163f0Snonaka  *    documentation and/or other materials provided with the distribution.
328b5163f0Snonaka  *
338b5163f0Snonaka  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
348b5163f0Snonaka  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
358b5163f0Snonaka  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
368b5163f0Snonaka  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
378b5163f0Snonaka  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
388b5163f0Snonaka  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
398b5163f0Snonaka  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
408b5163f0Snonaka  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
418b5163f0Snonaka  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
428b5163f0Snonaka  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
438b5163f0Snonaka  */
448b5163f0Snonaka 
458b5163f0Snonaka #include <sys/cdefs.h>
46*94acf541Spgoyette __KERNEL_RCSID(0, "$NetBSD: nvme_pci.c,v 1.37 2022/08/15 18:06:04 pgoyette Exp $");
478b5163f0Snonaka 
488b5163f0Snonaka #include <sys/param.h>
498b5163f0Snonaka #include <sys/systm.h>
508b5163f0Snonaka #include <sys/kernel.h>
518b5163f0Snonaka #include <sys/device.h>
528b5163f0Snonaka #include <sys/bitops.h>
538b5163f0Snonaka #include <sys/bus.h>
548b5163f0Snonaka #include <sys/cpu.h>
558b5163f0Snonaka #include <sys/interrupt.h>
568b5163f0Snonaka #include <sys/kmem.h>
578b5163f0Snonaka #include <sys/pmf.h>
588eedc00aSjdolecek #include <sys/module.h>
598b5163f0Snonaka 
608b5163f0Snonaka #include <dev/pci/pcireg.h>
618b5163f0Snonaka #include <dev/pci/pcivar.h>
6245a08be1Snonaka #include <dev/pci/pcidevs.h>
638b5163f0Snonaka 
648b5163f0Snonaka #include <dev/ic/nvmereg.h>
658b5163f0Snonaka #include <dev/ic/nvmevar.h>
668b5163f0Snonaka 
678b5163f0Snonaka int nvme_pci_force_intx = 0;
68ef172b9fSjdolecek int nvme_pci_mpsafe = 1;
698b5163f0Snonaka int nvme_pci_mq = 1;		/* INTx: ioq=1, MSI/MSI-X: ioq=ncpu */
708b5163f0Snonaka 
718b5163f0Snonaka #define NVME_PCI_BAR		0x10
728b5163f0Snonaka 
738b5163f0Snonaka struct nvme_pci_softc {
748b5163f0Snonaka 	struct nvme_softc	psc_nvme;
758b5163f0Snonaka 
768b5163f0Snonaka 	pci_chipset_tag_t	psc_pc;
778b5163f0Snonaka 	pci_intr_handle_t	*psc_intrs;
788b5163f0Snonaka 	int			psc_nintrs;
798b5163f0Snonaka };
808b5163f0Snonaka 
818b5163f0Snonaka static int	nvme_pci_match(device_t, cfdata_t, void *);
828b5163f0Snonaka static void	nvme_pci_attach(device_t, device_t, void *);
838b5163f0Snonaka static int	nvme_pci_detach(device_t, int);
84916bdfa5Spgoyette static int	nvme_pci_rescan(device_t, const char *, const int *);
85a6e80066Sriastradh static bool	nvme_pci_suspend(device_t, const pmf_qual_t *);
86a6e80066Sriastradh static bool	nvme_pci_resume(device_t, const pmf_qual_t *);
878b5163f0Snonaka 
888b5163f0Snonaka CFATTACH_DECL3_NEW(nvme_pci, sizeof(struct nvme_pci_softc),
89916bdfa5Spgoyette     nvme_pci_match, nvme_pci_attach, nvme_pci_detach, NULL, nvme_pci_rescan,
908b5163f0Snonaka     nvme_childdet, DVF_DETACH_SHUTDOWN);
918b5163f0Snonaka 
928b5163f0Snonaka static int	nvme_pci_intr_establish(struct nvme_softc *,
938b5163f0Snonaka 		    uint16_t, struct nvme_queue *);
948b5163f0Snonaka static int	nvme_pci_intr_disestablish(struct nvme_softc *, uint16_t);
958b5163f0Snonaka static int	nvme_pci_setup_intr(struct pci_attach_args *,
968b5163f0Snonaka 		    struct nvme_pci_softc *);
978b5163f0Snonaka 
9845a08be1Snonaka static const struct nvme_pci_quirk {
9945a08be1Snonaka 	pci_vendor_id_t		vendor;
10045a08be1Snonaka 	pci_product_id_t	product;
10145a08be1Snonaka 	uint32_t		quirks;
10245a08be1Snonaka } nvme_pci_quirks[] = {
10345a08be1Snonaka 	{ PCI_VENDOR_HGST, PCI_PRODUCT_HGST_SN100,
10445a08be1Snonaka 	    NVME_QUIRK_DELAY_B4_CHK_RDY },
10545a08be1Snonaka 	{ PCI_VENDOR_HGST, PCI_PRODUCT_HGST_SN200,
10645a08be1Snonaka 	    NVME_QUIRK_DELAY_B4_CHK_RDY },
10745a08be1Snonaka 	{ PCI_VENDOR_BEIJING_MEMBLAZE, PCI_PRODUCT_BEIJING_MEMBLAZE_PBLAZE4,
10845a08be1Snonaka 	    NVME_QUIRK_DELAY_B4_CHK_RDY },
10945a08be1Snonaka 	{ PCI_VENDOR_SAMSUNGELEC3, PCI_PRODUCT_SAMSUNGELEC3_172X,
11045a08be1Snonaka 	    NVME_QUIRK_DELAY_B4_CHK_RDY },
11145a08be1Snonaka 	{ PCI_VENDOR_SAMSUNGELEC3, PCI_PRODUCT_SAMSUNGELEC3_172XAB,
11245a08be1Snonaka 	    NVME_QUIRK_DELAY_B4_CHK_RDY },
113b5fd1af3Sjdolecek 	{ PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_DC_P4500_SSD,
114b5fd1af3Sjdolecek 	    NVME_QUIRK_NOMSI },
11545a08be1Snonaka };
11645a08be1Snonaka 
11745a08be1Snonaka static const struct nvme_pci_quirk *
nvme_pci_lookup_quirk(struct pci_attach_args * pa)11845a08be1Snonaka nvme_pci_lookup_quirk(struct pci_attach_args *pa)
11945a08be1Snonaka {
12045a08be1Snonaka 	const struct nvme_pci_quirk *q;
12145a08be1Snonaka 	int i;
12245a08be1Snonaka 
12345a08be1Snonaka 	for (i = 0; i < __arraycount(nvme_pci_quirks); i++) {
12445a08be1Snonaka 		q = &nvme_pci_quirks[i];
12545a08be1Snonaka 
12645a08be1Snonaka 		if (PCI_VENDOR(pa->pa_id) == q->vendor &&
12745a08be1Snonaka 		    PCI_PRODUCT(pa->pa_id) == q->product)
12845a08be1Snonaka 			return q;
12945a08be1Snonaka 	}
13045a08be1Snonaka 	return NULL;
13145a08be1Snonaka }
13245a08be1Snonaka 
1338b5163f0Snonaka static int
nvme_pci_match(device_t parent,cfdata_t match,void * aux)1348b5163f0Snonaka nvme_pci_match(device_t parent, cfdata_t match, void *aux)
1358b5163f0Snonaka {
1368b5163f0Snonaka 	struct pci_attach_args *pa = aux;
1378b5163f0Snonaka 
1388b5163f0Snonaka 	if (PCI_CLASS(pa->pa_class) == PCI_CLASS_MASS_STORAGE &&
1398b5163f0Snonaka 	    PCI_SUBCLASS(pa->pa_class) == PCI_SUBCLASS_MASS_STORAGE_NVM &&
140a19ed151Sskrll 	    PCI_INTERFACE(pa->pa_class) == PCI_INTERFACE_NVM_NVME_IO)
1418b5163f0Snonaka 		return 1;
1428b5163f0Snonaka 
1438b5163f0Snonaka 	return 0;
1448b5163f0Snonaka }
1458b5163f0Snonaka 
1468b5163f0Snonaka static void
nvme_pci_attach(device_t parent,device_t self,void * aux)1478b5163f0Snonaka nvme_pci_attach(device_t parent, device_t self, void *aux)
1488b5163f0Snonaka {
1498b5163f0Snonaka 	struct nvme_pci_softc *psc = device_private(self);
1508b5163f0Snonaka 	struct nvme_softc *sc = &psc->psc_nvme;
1518b5163f0Snonaka 	struct pci_attach_args *pa = aux;
15245a08be1Snonaka 	const struct nvme_pci_quirk *quirk;
1532cf9f7e5Sjdolecek 	pcireg_t memtype, reg;
1548b5163f0Snonaka 	bus_addr_t memaddr;
155c1a81aa6Sjdolecek 	int flags, error;
156c1a81aa6Sjdolecek 	int msixoff;
1578b5163f0Snonaka 
1588b5163f0Snonaka 	sc->sc_dev = self;
1598b5163f0Snonaka 	psc->psc_pc = pa->pa_pc;
1608b5163f0Snonaka 	if (pci_dma64_available(pa))
1618b5163f0Snonaka 		sc->sc_dmat = pa->pa_dmat64;
1628b5163f0Snonaka 	else
1638b5163f0Snonaka 		sc->sc_dmat = pa->pa_dmat;
1648b5163f0Snonaka 
165b5fd1af3Sjdolecek 	quirk = nvme_pci_lookup_quirk(pa);
166b5fd1af3Sjdolecek 	if (quirk != NULL)
167b5fd1af3Sjdolecek 		sc->sc_quirks = quirk->quirks;
168b5fd1af3Sjdolecek 
1698b5163f0Snonaka 	pci_aprint_devinfo(pa, NULL);
1708b5163f0Snonaka 
1718b5163f0Snonaka 	/* Map registers */
1728b5163f0Snonaka 	memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, NVME_PCI_BAR);
1738b5163f0Snonaka 	if (PCI_MAPREG_TYPE(memtype) != PCI_MAPREG_TYPE_MEM) {
1748b5163f0Snonaka 		aprint_error_dev(self, "invalid type (type=0x%x)\n", memtype);
1758b5163f0Snonaka 		return;
1768b5163f0Snonaka 	}
17709af7763Smsaitoh 	reg = pci_conf_read(pa->pa_pc, pa->pa_tag, PCI_COMMAND_STATUS_REG);
17809af7763Smsaitoh 	if (((reg & PCI_COMMAND_MASTER_ENABLE) == 0) ||
17909af7763Smsaitoh 	    ((reg & PCI_COMMAND_MEM_ENABLE) == 0)) {
18009af7763Smsaitoh 		/*
18109af7763Smsaitoh 		 * Enable address decoding for memory range in case BIOS or
18209af7763Smsaitoh 		 * UEFI didn't set it.
18309af7763Smsaitoh 		 */
18409af7763Smsaitoh 		reg |= PCI_COMMAND_MASTER_ENABLE | PCI_COMMAND_MEM_ENABLE;
18509af7763Smsaitoh         	pci_conf_write(pa->pa_pc, pa->pa_tag, PCI_COMMAND_STATUS_REG,
18609af7763Smsaitoh 		    reg);
18709af7763Smsaitoh 	}
18809af7763Smsaitoh 
1898b5163f0Snonaka 	sc->sc_iot = pa->pa_memt;
1907dae1339Smsaitoh 	error = pci_mapreg_info(pa->pa_pc, pa->pa_tag, NVME_PCI_BAR,
1918b5163f0Snonaka 	    memtype, &memaddr, &sc->sc_ios, &flags);
1928b5163f0Snonaka 	if (error) {
1938b5163f0Snonaka 		aprint_error_dev(self, "can't get map info\n");
1948b5163f0Snonaka 		return;
1958b5163f0Snonaka 	}
196c1a81aa6Sjdolecek 
1978b5163f0Snonaka 	if (pci_get_capability(pa->pa_pc, pa->pa_tag, PCI_CAP_MSIX, &msixoff,
1988b5163f0Snonaka 	    NULL)) {
1998b5163f0Snonaka 		pcireg_t msixtbl;
2008b5163f0Snonaka 		uint32_t table_offset;
2018b5163f0Snonaka 		int bir;
2028b5163f0Snonaka 
2038b5163f0Snonaka 		msixtbl = pci_conf_read(pa->pa_pc, pa->pa_tag,
2048b5163f0Snonaka 		    msixoff + PCI_MSIX_TBLOFFSET);
2058b5163f0Snonaka 		table_offset = msixtbl & PCI_MSIX_TBLOFFSET_MASK;
206f500185eSmsaitoh 		bir = msixtbl & PCI_MSIX_TBLBIR_MASK;
2077dae1339Smsaitoh 		if (bir == PCI_MAPREG_NUM(NVME_PCI_BAR)) {
2088b5163f0Snonaka 			sc->sc_ios = table_offset;
2098b5163f0Snonaka 		}
2108b5163f0Snonaka 	}
211c1a81aa6Sjdolecek 
2128b5163f0Snonaka 	error = bus_space_map(sc->sc_iot, memaddr, sc->sc_ios, flags,
2138b5163f0Snonaka 	    &sc->sc_ioh);
2148b5163f0Snonaka 	if (error != 0) {
2158b5163f0Snonaka 		aprint_error_dev(self, "can't map mem space (error=%d)\n",
2168b5163f0Snonaka 		    error);
2178b5163f0Snonaka 		return;
2188b5163f0Snonaka 	}
2198b5163f0Snonaka 
2208b5163f0Snonaka 	/* Establish interrupts */
2218b5163f0Snonaka 	if (nvme_pci_setup_intr(pa, psc) != 0) {
2228b5163f0Snonaka 		aprint_error_dev(self, "unable to allocate interrupt\n");
2238b5163f0Snonaka 		goto unmap;
2248b5163f0Snonaka 	}
2258b5163f0Snonaka 	sc->sc_intr_establish = nvme_pci_intr_establish;
2268b5163f0Snonaka 	sc->sc_intr_disestablish = nvme_pci_intr_disestablish;
2278b5163f0Snonaka 
2287a7d1b8dSjdolecek 	sc->sc_ih = kmem_zalloc(sizeof(*sc->sc_ih) * psc->psc_nintrs, KM_SLEEP);
229ef172b9fSjdolecek 	sc->sc_softih = kmem_zalloc(
230ef172b9fSjdolecek 	    sizeof(*sc->sc_softih) * psc->psc_nintrs, KM_SLEEP);
23145a08be1Snonaka 
2328b5163f0Snonaka 	if (nvme_attach(sc) != 0) {
2338b5163f0Snonaka 		/* error printed by nvme_attach() */
234ef172b9fSjdolecek 		goto softintr_free;
2358b5163f0Snonaka 	}
2368b5163f0Snonaka 
237a6e80066Sriastradh 	if (!pmf_device_register(self, nvme_pci_suspend, nvme_pci_resume))
2388b5163f0Snonaka 		aprint_error_dev(self, "couldn't establish power handler\n");
2398b5163f0Snonaka 
2408b5163f0Snonaka 	SET(sc->sc_flags, NVME_F_ATTACHED);
2418b5163f0Snonaka 	return;
2428b5163f0Snonaka 
243ef172b9fSjdolecek softintr_free:
244684dc5aaSjdolecek 	kmem_free(sc->sc_softih, sizeof(*sc->sc_softih) * psc->psc_nintrs);
2457a7d1b8dSjdolecek 	kmem_free(sc->sc_ih, sizeof(*sc->sc_ih) * psc->psc_nintrs);
2468b5163f0Snonaka 	sc->sc_nq = 0;
2478b5163f0Snonaka 	pci_intr_release(pa->pa_pc, psc->psc_intrs, psc->psc_nintrs);
2488b5163f0Snonaka 	psc->psc_nintrs = 0;
2498b5163f0Snonaka unmap:
2508b5163f0Snonaka 	bus_space_unmap(sc->sc_iot, sc->sc_ioh, sc->sc_ios);
2518b5163f0Snonaka 	sc->sc_ios = 0;
2528b5163f0Snonaka }
2538b5163f0Snonaka 
2548b5163f0Snonaka static int
nvme_pci_rescan(device_t self,const char * attr,const int * flags)255916bdfa5Spgoyette nvme_pci_rescan(device_t self, const char *attr, const int *flags)
256916bdfa5Spgoyette {
257916bdfa5Spgoyette 
258916bdfa5Spgoyette 	return nvme_rescan(self, attr, flags);
259916bdfa5Spgoyette }
260916bdfa5Spgoyette 
261a6e80066Sriastradh static bool
nvme_pci_suspend(device_t self,const pmf_qual_t * qual)262a6e80066Sriastradh nvme_pci_suspend(device_t self, const pmf_qual_t *qual)
263a6e80066Sriastradh {
264a6e80066Sriastradh 	struct nvme_pci_softc *psc = device_private(self);
265a6e80066Sriastradh 	struct nvme_softc *sc = &psc->psc_nvme;
266a6e80066Sriastradh 	int error;
267a6e80066Sriastradh 
268a6e80066Sriastradh 	error = nvme_suspend(sc);
269a6e80066Sriastradh 	if (error)
270a6e80066Sriastradh 		return false;
271a6e80066Sriastradh 
272a6e80066Sriastradh 	return true;
273a6e80066Sriastradh }
274a6e80066Sriastradh 
275a6e80066Sriastradh static bool
nvme_pci_resume(device_t self,const pmf_qual_t * qual)276a6e80066Sriastradh nvme_pci_resume(device_t self, const pmf_qual_t *qual)
277a6e80066Sriastradh {
278a6e80066Sriastradh 	struct nvme_pci_softc *psc = device_private(self);
279a6e80066Sriastradh 	struct nvme_softc *sc = &psc->psc_nvme;
280a6e80066Sriastradh 	int error;
281a6e80066Sriastradh 
282a6e80066Sriastradh 	error = nvme_resume(sc);
283a6e80066Sriastradh 	if (error)
284a6e80066Sriastradh 		return false;
285a6e80066Sriastradh 
286a6e80066Sriastradh 	return true;
287a6e80066Sriastradh }
288a6e80066Sriastradh 
289916bdfa5Spgoyette static int
nvme_pci_detach(device_t self,int flags)2908b5163f0Snonaka nvme_pci_detach(device_t self, int flags)
2918b5163f0Snonaka {
2928b5163f0Snonaka 	struct nvme_pci_softc *psc = device_private(self);
2938b5163f0Snonaka 	struct nvme_softc *sc = &psc->psc_nvme;
2947a7d1b8dSjdolecek 	int error;
2958b5163f0Snonaka 
2968b5163f0Snonaka 	if (!ISSET(sc->sc_flags, NVME_F_ATTACHED))
2978b5163f0Snonaka 		return 0;
2988b5163f0Snonaka 
2998b5163f0Snonaka 	error = nvme_detach(sc, flags);
3008b5163f0Snonaka 	if (error)
3018b5163f0Snonaka 		return error;
3028b5163f0Snonaka 
303684dc5aaSjdolecek 	kmem_free(sc->sc_softih, sizeof(*sc->sc_softih) * psc->psc_nintrs);
304ef172b9fSjdolecek 	sc->sc_softih = NULL;
305684dc5aaSjdolecek 
3067a7d1b8dSjdolecek 	kmem_free(sc->sc_ih, sizeof(*sc->sc_ih) * psc->psc_nintrs);
3078b5163f0Snonaka 	pci_intr_release(psc->psc_pc, psc->psc_intrs, psc->psc_nintrs);
3088b5163f0Snonaka 	bus_space_unmap(sc->sc_iot, sc->sc_ioh, sc->sc_ios);
3098b5163f0Snonaka 	return 0;
3108b5163f0Snonaka }
3118b5163f0Snonaka 
3128b5163f0Snonaka static int
nvme_pci_intr_establish(struct nvme_softc * sc,uint16_t qid,struct nvme_queue * q)3138b5163f0Snonaka nvme_pci_intr_establish(struct nvme_softc *sc, uint16_t qid,
3148b5163f0Snonaka     struct nvme_queue *q)
3158b5163f0Snonaka {
3168b5163f0Snonaka 	struct nvme_pci_softc *psc = (struct nvme_pci_softc *)sc;
3178b5163f0Snonaka 	char intr_xname[INTRDEVNAMEBUF];
3188b5163f0Snonaka 	char intrbuf[PCI_INTRSTR_LEN];
3198b5163f0Snonaka 	const char *intrstr = NULL;
3208b5163f0Snonaka 	int (*ih_func)(void *);
321684dc5aaSjdolecek 	void (*ih_func_soft)(void *);
3228b5163f0Snonaka 	void *ih_arg;
3238b5163f0Snonaka 	int error;
3248b5163f0Snonaka 
325c1a81aa6Sjdolecek 	KASSERT(sc->sc_use_mq || qid == NVME_ADMIN_Q);
3268b5163f0Snonaka 	KASSERT(sc->sc_ih[qid] == NULL);
3278b5163f0Snonaka 
3288b5163f0Snonaka 	if (nvme_pci_mpsafe) {
3298b5163f0Snonaka 		pci_intr_setattr(psc->psc_pc, &psc->psc_intrs[qid],
3308b5163f0Snonaka 		    PCI_INTR_MPSAFE, true);
3318b5163f0Snonaka 	}
332c1a81aa6Sjdolecek 
3338b5163f0Snonaka 	if (!sc->sc_use_mq) {
3348b5163f0Snonaka 		snprintf(intr_xname, sizeof(intr_xname), "%s",
3358b5163f0Snonaka 		    device_xname(sc->sc_dev));
3368b5163f0Snonaka 		ih_arg = sc;
3378b5163f0Snonaka 		ih_func = nvme_intr;
338684dc5aaSjdolecek 		ih_func_soft = nvme_softintr_intx;
33904f2bb21Sknakahara 	} else {
340c1a81aa6Sjdolecek 		if (qid == NVME_ADMIN_Q) {
3418b5163f0Snonaka 			snprintf(intr_xname, sizeof(intr_xname), "%s adminq",
3428b5163f0Snonaka 			    device_xname(sc->sc_dev));
3438b5163f0Snonaka 		} else {
3448b5163f0Snonaka 			snprintf(intr_xname, sizeof(intr_xname), "%s ioq%d",
3458b5163f0Snonaka 			    device_xname(sc->sc_dev), qid);
3468b5163f0Snonaka 		}
3478b5163f0Snonaka 		ih_arg = q;
348ef172b9fSjdolecek 		ih_func = nvme_intr_msi;
349684dc5aaSjdolecek 		ih_func_soft = nvme_softintr_msi;
3508b5163f0Snonaka 	}
351ef172b9fSjdolecek 
352ef172b9fSjdolecek 	/* establish hardware interrupt */
3538b5163f0Snonaka 	sc->sc_ih[qid] = pci_intr_establish_xname(psc->psc_pc,
3548b5163f0Snonaka 	    psc->psc_intrs[qid], IPL_BIO, ih_func, ih_arg, intr_xname);
3558b5163f0Snonaka 	if (sc->sc_ih[qid] == NULL) {
3568b5163f0Snonaka 		aprint_error_dev(sc->sc_dev,
3578b5163f0Snonaka 		    "unable to establish %s interrupt\n", intr_xname);
3588b5163f0Snonaka 		return 1;
3598b5163f0Snonaka 	}
360ef172b9fSjdolecek 
361684dc5aaSjdolecek 	/* establish also the software interrupt */
362ef172b9fSjdolecek 	sc->sc_softih[qid] = softint_establish(
363ef172b9fSjdolecek 	    SOFTINT_BIO|(nvme_pci_mpsafe ? SOFTINT_MPSAFE : 0),
364684dc5aaSjdolecek 	    ih_func_soft, q);
365ef172b9fSjdolecek 	if (sc->sc_softih[qid] == NULL) {
366ef172b9fSjdolecek 		pci_intr_disestablish(psc->psc_pc, sc->sc_ih[qid]);
367ef172b9fSjdolecek 		sc->sc_ih[qid] = NULL;
368ef172b9fSjdolecek 
369ef172b9fSjdolecek 		aprint_error_dev(sc->sc_dev,
370ef172b9fSjdolecek 		    "unable to establish %s soft interrupt\n",
371ef172b9fSjdolecek 		    intr_xname);
372ef172b9fSjdolecek 		return 1;
373ef172b9fSjdolecek 	}
374ef172b9fSjdolecek 
3758b5163f0Snonaka 	intrstr = pci_intr_string(psc->psc_pc, psc->psc_intrs[qid], intrbuf,
3768b5163f0Snonaka 	    sizeof(intrbuf));
3778b5163f0Snonaka 	if (!sc->sc_use_mq) {
3788b5163f0Snonaka 		aprint_normal_dev(sc->sc_dev, "interrupting at %s\n", intrstr);
37904f2bb21Sknakahara 	} else if (qid == NVME_ADMIN_Q) {
3808b5163f0Snonaka 		aprint_normal_dev(sc->sc_dev,
3818b5163f0Snonaka 		    "for admin queue interrupting at %s\n", intrstr);
3828b5163f0Snonaka 	} else if (!nvme_pci_mpsafe) {
3838b5163f0Snonaka 		aprint_normal_dev(sc->sc_dev,
3848b5163f0Snonaka 		    "for io queue %d interrupting at %s\n", qid, intrstr);
3858b5163f0Snonaka 	} else {
386c1a81aa6Sjdolecek 		kcpuset_t *affinity;
387c1a81aa6Sjdolecek 		cpuid_t affinity_to;
388c1a81aa6Sjdolecek 
3898b5163f0Snonaka 		kcpuset_create(&affinity, true);
3908b5163f0Snonaka 		affinity_to = (qid - 1) % ncpu;
3918b5163f0Snonaka 		kcpuset_set(affinity, affinity_to);
3928b5163f0Snonaka 		error = interrupt_distribute(sc->sc_ih[qid], affinity, NULL);
3938b5163f0Snonaka 		kcpuset_destroy(affinity);
3948b5163f0Snonaka 		aprint_normal_dev(sc->sc_dev,
3958b5163f0Snonaka 		    "for io queue %d interrupting at %s", qid, intrstr);
3968b5163f0Snonaka 		if (error == 0)
3978b5163f0Snonaka 			aprint_normal(" affinity to cpu%lu", affinity_to);
3988b5163f0Snonaka 		aprint_normal("\n");
3998b5163f0Snonaka 	}
4008b5163f0Snonaka 	return 0;
4018b5163f0Snonaka }
4028b5163f0Snonaka 
4038b5163f0Snonaka static int
nvme_pci_intr_disestablish(struct nvme_softc * sc,uint16_t qid)4048b5163f0Snonaka nvme_pci_intr_disestablish(struct nvme_softc *sc, uint16_t qid)
4058b5163f0Snonaka {
4068b5163f0Snonaka 	struct nvme_pci_softc *psc = (struct nvme_pci_softc *)sc;
4078b5163f0Snonaka 
408ef172b9fSjdolecek 	KASSERT(sc->sc_use_mq || qid == NVME_ADMIN_Q);
4098b5163f0Snonaka 	KASSERT(sc->sc_ih[qid] != NULL);
4108b5163f0Snonaka 
411ef172b9fSjdolecek 	if (sc->sc_softih) {
412ef172b9fSjdolecek 		softint_disestablish(sc->sc_softih[qid]);
413ef172b9fSjdolecek 		sc->sc_softih[qid] = NULL;
414ef172b9fSjdolecek 	}
415ef172b9fSjdolecek 
4168b5163f0Snonaka 	pci_intr_disestablish(psc->psc_pc, sc->sc_ih[qid]);
4178b5163f0Snonaka 	sc->sc_ih[qid] = NULL;
4188b5163f0Snonaka 
4198b5163f0Snonaka 	return 0;
4208b5163f0Snonaka }
4218b5163f0Snonaka 
4228b5163f0Snonaka static int
nvme_pci_setup_intr(struct pci_attach_args * pa,struct nvme_pci_softc * psc)4238b5163f0Snonaka nvme_pci_setup_intr(struct pci_attach_args *pa, struct nvme_pci_softc *psc)
4248b5163f0Snonaka {
4258b5163f0Snonaka 	struct nvme_softc *sc = &psc->psc_nvme;
4268b5163f0Snonaka 	int error;
4272ca07679Sjdolecek 	int counts[PCI_INTR_TYPE_SIZE];
428c1a81aa6Sjdolecek 	pci_intr_handle_t *ihps;
4292ca07679Sjdolecek 	int intr_type;
4308b5163f0Snonaka 
4312ca07679Sjdolecek 	memset(counts, 0, sizeof(counts));
4322ca07679Sjdolecek 
4332ca07679Sjdolecek 	if (nvme_pci_force_intx)
4347f49d656Sjdolecek 		goto setup_intx;
4358b5163f0Snonaka 
4368b5163f0Snonaka 	/* MSI-X */
437d1579b2dSriastradh 	counts[PCI_INTR_TYPE_MSIX] = uimin(pci_msix_count(pa->pa_pc, pa->pa_tag),
4388b5163f0Snonaka 	    ncpu + 1);
439411f2dd2Sjdolecek 	if (counts[PCI_INTR_TYPE_MSIX] < 1) {
4408b5163f0Snonaka 		counts[PCI_INTR_TYPE_MSIX] = 0;
4418b5163f0Snonaka 	} else if (!nvme_pci_mq || !nvme_pci_mpsafe) {
442baffc23fSjdolecek 		if (counts[PCI_INTR_TYPE_MSIX] > 2)
4438b5163f0Snonaka 			counts[PCI_INTR_TYPE_MSIX] = 2;	/* adminq + 1 ioq */
4448b5163f0Snonaka 	}
4458b5163f0Snonaka 
4468b5163f0Snonaka 	/* MSI */
447b5fd1af3Sjdolecek 	if (sc->sc_quirks & NVME_QUIRK_NOMSI)
4487f49d656Sjdolecek 		goto setup_intx;
4498b5163f0Snonaka 	counts[PCI_INTR_TYPE_MSI] = pci_msi_count(pa->pa_pc, pa->pa_tag);
4508b5163f0Snonaka 	if (counts[PCI_INTR_TYPE_MSI] > 0) {
4518b5163f0Snonaka 		while (counts[PCI_INTR_TYPE_MSI] > ncpu + 1) {
4528b5163f0Snonaka 			if (counts[PCI_INTR_TYPE_MSI] / 2 <= ncpu + 1)
4538b5163f0Snonaka 				break;
4548b5163f0Snonaka 			counts[PCI_INTR_TYPE_MSI] /= 2;
4558b5163f0Snonaka 		}
4568b5163f0Snonaka 	}
4578b5163f0Snonaka 	if (counts[PCI_INTR_TYPE_MSI] < 1) {
4588b5163f0Snonaka 		counts[PCI_INTR_TYPE_MSI] = 0;
4598b5163f0Snonaka 	} else if (!nvme_pci_mq || !nvme_pci_mpsafe) {
4608b5163f0Snonaka 		if (counts[PCI_INTR_TYPE_MSI] > 2)
4618b5163f0Snonaka 			counts[PCI_INTR_TYPE_MSI] = 2;	/* adminq + 1 ioq */
4628b5163f0Snonaka 	}
4638b5163f0Snonaka 
4647f49d656Sjdolecek setup_intx:
4658b5163f0Snonaka 	/* INTx */
4668b5163f0Snonaka 	counts[PCI_INTR_TYPE_INTX] = 1;
4678b5163f0Snonaka 
4682ca07679Sjdolecek 	error = pci_intr_alloc(pa, &ihps, counts, PCI_INTR_TYPE_MSIX);
4692ca07679Sjdolecek 	if (error)
4708b5163f0Snonaka 		return error;
4718b5163f0Snonaka 
472bf83a4a7Sknakahara 	intr_type = pci_intr_type(pa->pa_pc, ihps[0]);
4738b5163f0Snonaka 
4748b5163f0Snonaka 	psc->psc_intrs = ihps;
4752ca07679Sjdolecek 	psc->psc_nintrs = counts[intr_type];
4768b5163f0Snonaka 	if (intr_type == PCI_INTR_TYPE_MSI) {
4772ca07679Sjdolecek 		if (counts[intr_type] > ncpu + 1)
4782ca07679Sjdolecek 			counts[intr_type] = ncpu + 1;
4798b5163f0Snonaka 	}
4802ca07679Sjdolecek 	sc->sc_use_mq = counts[intr_type] > 1;
4812ca07679Sjdolecek 	sc->sc_nq = sc->sc_use_mq ? counts[intr_type] - 1 : 1;
482c1a81aa6Sjdolecek 
4838b5163f0Snonaka 	return 0;
4848b5163f0Snonaka }
4858eedc00aSjdolecek 
486ffcbcf35Spgoyette MODULE(MODULE_CLASS_DRIVER, nvme, "pci,dk_subr");
4878eedc00aSjdolecek 
4888eedc00aSjdolecek #ifdef _MODULE
4898eedc00aSjdolecek #include "ioconf.c"
4908eedc00aSjdolecek #endif
4918eedc00aSjdolecek 
4928eedc00aSjdolecek static int
nvme_modcmd(modcmd_t cmd,void * opaque)4938eedc00aSjdolecek nvme_modcmd(modcmd_t cmd, void *opaque)
4948eedc00aSjdolecek {
4958eedc00aSjdolecek #ifdef _MODULE
4968eedc00aSjdolecek 	devmajor_t cmajor, bmajor;
4976ade779aSjdolecek 	extern const struct cdevsw nvme_cdevsw;
498e1b16a68Spgoyette 	static bool devsw_ok;
4998eedc00aSjdolecek #endif
5008eedc00aSjdolecek 	int error = 0;
5018eedc00aSjdolecek 
502916bdfa5Spgoyette #ifdef _MODULE
5038eedc00aSjdolecek 	switch (cmd) {
5048eedc00aSjdolecek 	case MODULE_CMD_INIT:
505d383284eSpgoyette 		bmajor = cmajor = NODEVMAJOR;
5066ade779aSjdolecek 		error = devsw_attach(nvme_cd.cd_name, NULL, &bmajor,
5076ade779aSjdolecek 		    &nvme_cdevsw, &cmajor);
5086ade779aSjdolecek 		if (error) {
5094420eaf1Spgoyette 			aprint_error("%s: unable to register devsw, err %d\n",
5104420eaf1Spgoyette 			    nvme_cd.cd_name, error);
5116ade779aSjdolecek 			/* do not abort, just /dev/nvme* will not work */
5126ade779aSjdolecek 		}
513d383284eSpgoyette 		else
514d383284eSpgoyette 			devsw_ok = true;
515d383284eSpgoyette 
51697f8debdSpgoyette 		error = config_init_component(cfdriver_ioconf_nvme_pci,
51797f8debdSpgoyette 		    cfattach_ioconf_nvme_pci, cfdata_ioconf_nvme_pci);
51897f8debdSpgoyette 		if (error) {
519e1b16a68Spgoyette 			if (devsw_ok) {
52097f8debdSpgoyette 				devsw_detach(NULL, &nvme_cdevsw);
521e1b16a68Spgoyette 				devsw_ok = false;
522e1b16a68Spgoyette 			}
52397f8debdSpgoyette 			break;
52497f8debdSpgoyette 		}
525916bdfa5Spgoyette 		break;
5268eedc00aSjdolecek 	case MODULE_CMD_FINI:
5278eedc00aSjdolecek 		error = config_fini_component(cfdriver_ioconf_nvme_pci,
5288eedc00aSjdolecek 		    cfattach_ioconf_nvme_pci, cfdata_ioconf_nvme_pci);
529e1b16a68Spgoyette 		if (devsw_ok) {
53097f8debdSpgoyette 			devsw_detach(NULL, &nvme_cdevsw);
531e1b16a68Spgoyette 			devsw_ok = false;
532e1b16a68Spgoyette 		}
533916bdfa5Spgoyette 		break;
534916bdfa5Spgoyette 	default:
535916bdfa5Spgoyette 		break;
536916bdfa5Spgoyette 	}
5378eedc00aSjdolecek #endif
5388eedc00aSjdolecek 	return error;
5398eedc00aSjdolecek }
540