1*ea87a8bfSkettenis /* $OpenBSD: aplns.c,v 1.15 2022/11/11 11:45:10 kettenis Exp $ */
21c31dbaaSdlg /*
31c31dbaaSdlg * Copyright (c) 2014, 2021 David Gwynne <dlg@openbsd.org>
41c31dbaaSdlg *
51c31dbaaSdlg * Permission to use, copy, modify, and distribute this software for any
61c31dbaaSdlg * purpose with or without fee is hereby granted, provided that the above
71c31dbaaSdlg * copyright notice and this permission notice appear in all copies.
81c31dbaaSdlg *
91c31dbaaSdlg * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
101c31dbaaSdlg * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
111c31dbaaSdlg * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
121c31dbaaSdlg * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
131c31dbaaSdlg * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
141c31dbaaSdlg * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
151c31dbaaSdlg * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
161c31dbaaSdlg */
171c31dbaaSdlg
181c31dbaaSdlg #include <sys/param.h>
191c31dbaaSdlg #include <sys/systm.h>
201c31dbaaSdlg #include <sys/buf.h>
211c31dbaaSdlg #include <sys/kernel.h>
221c31dbaaSdlg #include <sys/malloc.h>
231c31dbaaSdlg #include <sys/device.h>
241c31dbaaSdlg #include <sys/timeout.h>
251c31dbaaSdlg #include <sys/queue.h>
261c31dbaaSdlg #include <sys/mutex.h>
271c31dbaaSdlg #include <sys/pool.h>
281c31dbaaSdlg
291c31dbaaSdlg #include <machine/bus.h>
301c31dbaaSdlg #include <machine/fdt.h>
311c31dbaaSdlg
321c31dbaaSdlg #include <dev/ofw/openfirm.h>
331c31dbaaSdlg #include <dev/ofw/ofw_misc.h>
349a869755Skettenis #include <dev/ofw/ofw_power.h>
3519c18994Skettenis #include <dev/ofw/ofw_clock.h>
361c31dbaaSdlg #include <dev/ofw/fdt.h>
371c31dbaaSdlg
381c31dbaaSdlg #include <scsi/scsi_all.h>
391c31dbaaSdlg #include <scsi/scsiconf.h>
401c31dbaaSdlg
411c31dbaaSdlg #include <dev/ic/nvmereg.h>
421c31dbaaSdlg #include <dev/ic/nvmevar.h>
431c31dbaaSdlg
440b0ff99eSkettenis #include <arm64/dev/rtkit.h>
450b0ff99eSkettenis
460b0ff99eSkettenis #define ANS_CPU_CTRL 0x0044
470b0ff99eSkettenis #define ANS_CPU_CTRL_RUN (1 << 4)
480b0ff99eSkettenis
490b0ff99eSkettenis #define ANS_MAX_PEND_CMDS_CTRL 0x01210
500b0ff99eSkettenis #define ANS_MAX_QUEUE_DEPTH 64
510b0ff99eSkettenis #define ANS_BOOT_STATUS 0x01300
520b0ff99eSkettenis #define ANS_BOOT_STATUS_OK 0xde71ce55
531c31dbaaSdlg #define ANS_MODESEL_REG 0x01304
540b0ff99eSkettenis #define ANS_UNKNOWN_CTRL 0x24008
550b0ff99eSkettenis #define ANS_PRP_NULL_CHECK (1 << 11)
560b0ff99eSkettenis #define ANS_LINEAR_SQ_CTRL 0x24908
570b0ff99eSkettenis #define ANS_LINEAR_SQ_CTRL_EN (1 << 0)
581c31dbaaSdlg #define ANS_LINEAR_ASQ_DB 0x2490c
591c31dbaaSdlg #define ANS_LINEAR_IOSQ_DB 0x24910
601c31dbaaSdlg
611c31dbaaSdlg #define ANS_NVMMU_NUM 0x28100
621c31dbaaSdlg #define ANS_NVMMU_BASE_ASQ 0x28108
631c31dbaaSdlg #define ANS_NVMMU_BASE_IOSQ 0x28110
641c31dbaaSdlg #define ANS_NVMMU_TCB_INVAL 0x28118
651c31dbaaSdlg #define ANS_NVMMU_TCB_STAT 0x28120
661c31dbaaSdlg
671c31dbaaSdlg #define ANS_NVMMU_TCB_SIZE 0x4000
681c31dbaaSdlg #define ANS_NVMMU_TCB_PITCH 0x80
691c31dbaaSdlg
701c31dbaaSdlg struct ans_nvmmu_tcb {
711c31dbaaSdlg uint8_t tcb_opcode;
721c31dbaaSdlg uint8_t tcb_flags;
731c31dbaaSdlg #define ANS_NVMMU_TCB_WRITE (1 << 0)
741c31dbaaSdlg #define ANS_NVMMU_TCB_READ (1 << 1)
751c31dbaaSdlg uint8_t tcb_cid;
761c31dbaaSdlg uint8_t tcb_pad0[1];
771c31dbaaSdlg
781c31dbaaSdlg uint32_t tcb_prpl_len;
791c31dbaaSdlg uint8_t tcb_pad1[16];
801c31dbaaSdlg
811c31dbaaSdlg uint64_t tcb_prp[2];
821c31dbaaSdlg };
831c31dbaaSdlg
841c31dbaaSdlg int aplns_match(struct device *, void *, void *);
851c31dbaaSdlg void aplns_attach(struct device *, struct device *, void *);
861c31dbaaSdlg
87471aeecfSnaddy const struct cfattach aplns_ca = {
8819c18994Skettenis sizeof(struct device), aplns_match, aplns_attach
891c31dbaaSdlg };
901c31dbaaSdlg
911c31dbaaSdlg struct cfdriver aplns_cd = {
921c31dbaaSdlg NULL, "aplns", DV_DULL
931c31dbaaSdlg };
941c31dbaaSdlg
952ec5c635Skettenis int nvme_ans_sart_map(void *, bus_addr_t, bus_size_t);
96*ea87a8bfSkettenis int nvme_ans_sart_unmap(void *, bus_addr_t, bus_size_t);
972ec5c635Skettenis
981c31dbaaSdlg int
aplns_match(struct device * parent,void * match,void * aux)991c31dbaaSdlg aplns_match(struct device *parent, void *match, void *aux)
1001c31dbaaSdlg {
1011c31dbaaSdlg struct fdt_attach_args *faa = aux;
1021c31dbaaSdlg
103c9b8d2f6Skettenis return (OF_is_compatible(faa->fa_node, "apple,nvme-m1") ||
104c9b8d2f6Skettenis OF_is_compatible(faa->fa_node, "apple,nvme-ans2"));
1051c31dbaaSdlg }
1061c31dbaaSdlg
1071c31dbaaSdlg void
aplns_attach(struct device * parent,struct device * self,void * aux)1081c31dbaaSdlg aplns_attach(struct device *parent, struct device *self, void *aux)
1091c31dbaaSdlg {
1101c31dbaaSdlg struct fdt_attach_args *faa = aux;
1111c31dbaaSdlg
1121c31dbaaSdlg printf("\n");
1131c31dbaaSdlg
1141c31dbaaSdlg config_found(self, faa, NULL);
1151c31dbaaSdlg }
1161c31dbaaSdlg
1171c31dbaaSdlg struct nvme_ans_softc {
1181c31dbaaSdlg struct nvme_softc asc_nvme;
1190b0ff99eSkettenis bus_space_tag_t asc_iot;
1200b0ff99eSkettenis bus_space_handle_t asc_ioh;
12119c18994Skettenis int asc_node;
1221c31dbaaSdlg
1232ec5c635Skettenis uint32_t asc_sart;
1242ec5c635Skettenis struct rtkit asc_rtkit;
1252ec5c635Skettenis struct rtkit_state *asc_rtkit_state;
1261c31dbaaSdlg struct nvme_dmamem *asc_nvmmu;
1271c31dbaaSdlg };
1281c31dbaaSdlg
1291c31dbaaSdlg int nvme_ans_match(struct device *, void *, void *);
1301c31dbaaSdlg void nvme_ans_attach(struct device *, struct device *, void *);
13119c18994Skettenis int nvme_ans_activate(struct device *, int act);
1321c31dbaaSdlg
133471aeecfSnaddy const struct cfattach nvme_ans_ca = {
13419c18994Skettenis sizeof(struct nvme_ans_softc), nvme_ans_match, nvme_ans_attach, NULL,
13519c18994Skettenis nvme_ans_activate
1361c31dbaaSdlg };
1371c31dbaaSdlg
13819c18994Skettenis int nvme_ans_init(struct nvme_ans_softc *sc);
13919c18994Skettenis void nvme_ans_shutdown(struct nvme_ans_softc *sc);
1401c31dbaaSdlg void nvme_ans_enable(struct nvme_softc *);
1411c31dbaaSdlg
1421c31dbaaSdlg int nvme_ans_q_alloc(struct nvme_softc *,
1431c31dbaaSdlg struct nvme_queue *);
1441c31dbaaSdlg void nvme_ans_q_free(struct nvme_softc *,
1451c31dbaaSdlg struct nvme_queue *);
1461c31dbaaSdlg
1471c31dbaaSdlg uint32_t nvme_ans_sq_enter(struct nvme_softc *,
1481c31dbaaSdlg struct nvme_queue *, struct nvme_ccb *);
1491c31dbaaSdlg void nvme_ans_sq_leave(struct nvme_softc *,
1501c31dbaaSdlg struct nvme_queue *, struct nvme_ccb *);
1511c31dbaaSdlg
15201dd4d92Sdlg void nvme_ans_cq_done(struct nvme_softc *,
15301dd4d92Sdlg struct nvme_queue *, struct nvme_ccb *);
15401dd4d92Sdlg
1551c31dbaaSdlg static const struct nvme_ops nvme_ans_ops = {
1561c31dbaaSdlg .op_enable = nvme_ans_enable,
1571c31dbaaSdlg
1581c31dbaaSdlg .op_q_alloc = nvme_ans_q_alloc,
1591c31dbaaSdlg .op_q_free = nvme_ans_q_free,
1601c31dbaaSdlg
1611c31dbaaSdlg .op_sq_enter = nvme_ans_sq_enter,
1621c31dbaaSdlg .op_sq_leave = nvme_ans_sq_leave,
1631c31dbaaSdlg .op_sq_enter_locked = nvme_ans_sq_enter,
1641c31dbaaSdlg .op_sq_leave_locked = nvme_ans_sq_leave,
16501dd4d92Sdlg
16601dd4d92Sdlg .op_cq_done = nvme_ans_cq_done,
1671c31dbaaSdlg };
1681c31dbaaSdlg
1691c31dbaaSdlg int
nvme_ans_match(struct device * parent,void * match,void * aux)1701c31dbaaSdlg nvme_ans_match(struct device *parent, void *match, void *aux)
1711c31dbaaSdlg {
1721c31dbaaSdlg struct fdt_attach_args *faa = aux;
1731c31dbaaSdlg
174bcf979e4Skettenis return (OF_is_compatible(faa->fa_node, "apple,nvme-m1") ||
175bcf979e4Skettenis OF_is_compatible(faa->fa_node, "apple,nvme-ans2"));
1761c31dbaaSdlg }
1771c31dbaaSdlg
1781c31dbaaSdlg void
nvme_ans_attach(struct device * parent,struct device * self,void * aux)1791c31dbaaSdlg nvme_ans_attach(struct device *parent, struct device *self, void *aux)
1801c31dbaaSdlg {
1811c31dbaaSdlg struct nvme_ans_softc *asc = (struct nvme_ans_softc *)self;
1821c31dbaaSdlg struct nvme_softc *sc = &asc->asc_nvme;
1831c31dbaaSdlg struct fdt_attach_args *faa = aux;
1841c31dbaaSdlg
1850b0ff99eSkettenis if (faa->fa_nreg < 2) {
1860b0ff99eSkettenis printf(": no registers\n");
1870b0ff99eSkettenis return;
1880b0ff99eSkettenis }
1897317d3e7Sdlg
1900b0ff99eSkettenis sc->sc_iot = faa->fa_iot;
1910b0ff99eSkettenis if (bus_space_map(sc->sc_iot, faa->fa_reg[0].addr,
1920b0ff99eSkettenis faa->fa_reg[0].size, 0, &sc->sc_ioh) != 0) {
1930b0ff99eSkettenis printf(": can't map registers\n");
1940b0ff99eSkettenis return;
1950b0ff99eSkettenis }
1960b0ff99eSkettenis
1970b0ff99eSkettenis asc->asc_iot = faa->fa_iot;
1980b0ff99eSkettenis if (bus_space_map(asc->asc_iot, faa->fa_reg[1].addr,
1990b0ff99eSkettenis faa->fa_reg[1].size, 0, &asc->asc_ioh)) {
2000b0ff99eSkettenis bus_space_unmap(sc->sc_iot, sc->sc_ioh, sc->sc_ios);
2010b0ff99eSkettenis printf(": can't map registers\n");
2021c31dbaaSdlg return;
2031c31dbaaSdlg }
2041c31dbaaSdlg
2051c31dbaaSdlg sc->sc_ih = fdt_intr_establish(faa->fa_node, IPL_BIO,
2061c31dbaaSdlg nvme_intr, sc, sc->sc_dev.dv_xname);
2071c31dbaaSdlg if (sc->sc_ih == NULL) {
2080b0ff99eSkettenis printf(": can't establish interrupt\n");
2091c31dbaaSdlg goto unmap;
2101c31dbaaSdlg }
2111c31dbaaSdlg
21219c18994Skettenis asc->asc_node = faa->fa_node;
2132ec5c635Skettenis asc->asc_sart = OF_getpropint(faa->fa_node, "apple,sart", 0);
2142ec5c635Skettenis asc->asc_rtkit.rk_cookie = asc;
2152ec5c635Skettenis asc->asc_rtkit.rk_dmat = faa->fa_dmat;
2162ec5c635Skettenis asc->asc_rtkit.rk_map = nvme_ans_sart_map;
217*ea87a8bfSkettenis asc->asc_rtkit.rk_unmap = nvme_ans_sart_unmap;
2182ec5c635Skettenis
21919c18994Skettenis asc->asc_rtkit_state =
2203b6109e0Skettenis rtkit_init(faa->fa_node, NULL, 0, &asc->asc_rtkit);
2212ec5c635Skettenis if (asc->asc_rtkit_state == NULL) {
2220b0ff99eSkettenis printf(": can't map mailbox channel\n");
2230b0ff99eSkettenis goto disestablish;
2240b0ff99eSkettenis }
2250b0ff99eSkettenis
22619c18994Skettenis if (nvme_ans_init(asc)) {
2270b0ff99eSkettenis printf(": firmware not ready\n");
2280b0ff99eSkettenis goto disestablish;
2290b0ff99eSkettenis }
2300b0ff99eSkettenis
2310b0ff99eSkettenis printf(": ");
2320b0ff99eSkettenis
2331c31dbaaSdlg sc->sc_dmat = faa->fa_dmat;
2341c31dbaaSdlg sc->sc_ios = faa->fa_reg[0].size;
2351c31dbaaSdlg sc->sc_ops = &nvme_ans_ops;
236e9317b4bSkettenis sc->sc_openings = 1;
2371c31dbaaSdlg
2381c31dbaaSdlg if (nvme_attach(sc) != 0) {
2391c31dbaaSdlg /* error printed by nvme_attach() */
2401c31dbaaSdlg goto disestablish;
2411c31dbaaSdlg }
2421c31dbaaSdlg
2431c31dbaaSdlg return;
2441c31dbaaSdlg
2451c31dbaaSdlg disestablish:
2461c31dbaaSdlg fdt_intr_disestablish(sc->sc_ih);
2471c31dbaaSdlg sc->sc_ih = NULL;
2481c31dbaaSdlg
2491c31dbaaSdlg unmap:
2500b0ff99eSkettenis bus_space_unmap(asc->asc_iot, asc->asc_ioh, faa->fa_reg[1].size);
2510b0ff99eSkettenis bus_space_unmap(sc->sc_iot, sc->sc_ioh, faa->fa_reg[0].size);
2521c31dbaaSdlg sc->sc_ios = 0;
2531c31dbaaSdlg }
2541c31dbaaSdlg
2551c31dbaaSdlg int
nvme_ans_activate(struct device * self,int act)25619c18994Skettenis nvme_ans_activate(struct device *self, int act)
25719c18994Skettenis {
25819c18994Skettenis struct nvme_ans_softc *asc = (struct nvme_ans_softc *)self;
25919c18994Skettenis struct nvme_softc *sc = &asc->asc_nvme;
26019c18994Skettenis int rv;
26119c18994Skettenis
26219c18994Skettenis switch (act) {
26319c18994Skettenis case DVACT_POWERDOWN:
26419c18994Skettenis rv = nvme_activate(&asc->asc_nvme, act);
26519c18994Skettenis nvme_ans_shutdown(asc);
26619c18994Skettenis break;
26719c18994Skettenis case DVACT_RESUME:
26819c18994Skettenis rv = nvme_ans_init(asc);
26919c18994Skettenis if (rv) {
27019c18994Skettenis printf("%s: firmware not ready\n", DEVNAME(sc));
27119c18994Skettenis goto fail;
27219c18994Skettenis }
27319c18994Skettenis rv = nvme_activate(&asc->asc_nvme, act);
27419c18994Skettenis break;
27519c18994Skettenis default:
27619c18994Skettenis rv = nvme_activate(&asc->asc_nvme, act);
27719c18994Skettenis break;
27819c18994Skettenis }
27919c18994Skettenis
28019c18994Skettenis fail:
28119c18994Skettenis return rv;
28219c18994Skettenis }
28319c18994Skettenis
28419c18994Skettenis int
nvme_ans_init(struct nvme_ans_softc * asc)28519c18994Skettenis nvme_ans_init(struct nvme_ans_softc *asc)
28619c18994Skettenis {
28719c18994Skettenis struct nvme_softc *sc = &asc->asc_nvme;
28819c18994Skettenis uint32_t ctrl, status;
28919c18994Skettenis
29019c18994Skettenis power_domain_enable_all(asc->asc_node);
29119c18994Skettenis
29219c18994Skettenis ctrl = bus_space_read_4(asc->asc_iot, asc->asc_ioh, ANS_CPU_CTRL);
29319c18994Skettenis bus_space_write_4(asc->asc_iot, asc->asc_ioh, ANS_CPU_CTRL,
29419c18994Skettenis ctrl | ANS_CPU_CTRL_RUN);
29519c18994Skettenis
29619c18994Skettenis status = bus_space_read_4(sc->sc_iot, sc->sc_ioh, ANS_BOOT_STATUS);
29719c18994Skettenis if (status != ANS_BOOT_STATUS_OK)
29819c18994Skettenis rtkit_boot(asc->asc_rtkit_state);
29919c18994Skettenis
30019c18994Skettenis status = bus_space_read_4(sc->sc_iot, sc->sc_ioh, ANS_BOOT_STATUS);
30119c18994Skettenis if (status != ANS_BOOT_STATUS_OK)
30219c18994Skettenis return ENXIO;
30319c18994Skettenis
30419c18994Skettenis bus_space_write_4(sc->sc_iot, sc->sc_ioh, ANS_LINEAR_SQ_CTRL,
30519c18994Skettenis ANS_LINEAR_SQ_CTRL_EN);
30619c18994Skettenis bus_space_write_4(sc->sc_iot, sc->sc_ioh, ANS_MAX_PEND_CMDS_CTRL,
30719c18994Skettenis (ANS_MAX_QUEUE_DEPTH << 16) | ANS_MAX_QUEUE_DEPTH);
30819c18994Skettenis
30919c18994Skettenis ctrl = bus_space_read_4(sc->sc_iot, sc->sc_ioh, ANS_UNKNOWN_CTRL);
31019c18994Skettenis bus_space_write_4(sc->sc_iot, sc->sc_ioh, ANS_UNKNOWN_CTRL,
31119c18994Skettenis ctrl & ~ANS_PRP_NULL_CHECK);
31219c18994Skettenis
31319c18994Skettenis return 0;
31419c18994Skettenis }
31519c18994Skettenis
31619c18994Skettenis void
nvme_ans_shutdown(struct nvme_ans_softc * asc)31719c18994Skettenis nvme_ans_shutdown(struct nvme_ans_softc *asc)
31819c18994Skettenis {
31919c18994Skettenis uint32_t ctrl;
32019c18994Skettenis
32119c18994Skettenis rtkit_shutdown(asc->asc_rtkit_state);
32219c18994Skettenis
32319c18994Skettenis ctrl = bus_space_read_4(asc->asc_iot, asc->asc_ioh, ANS_CPU_CTRL);
32419c18994Skettenis bus_space_write_4(asc->asc_iot, asc->asc_ioh, ANS_CPU_CTRL,
32519c18994Skettenis ctrl & ~ANS_CPU_CTRL_RUN);
32619c18994Skettenis
32719c18994Skettenis reset_assert_all(asc->asc_node);
32819c18994Skettenis reset_deassert_all(asc->asc_node);
32919c18994Skettenis
33019c18994Skettenis power_domain_disable_all(asc->asc_node);
33119c18994Skettenis }
33219c18994Skettenis
33319c18994Skettenis int
nvme_ans_sart_map(void * cookie,bus_addr_t addr,bus_size_t size)3342ec5c635Skettenis nvme_ans_sart_map(void *cookie, bus_addr_t addr, bus_size_t size)
3352ec5c635Skettenis {
3362ec5c635Skettenis struct nvme_ans_softc *asc = cookie;
3372ec5c635Skettenis
3382ec5c635Skettenis return aplsart_map(asc->asc_sart, addr, size);
3392ec5c635Skettenis }
3402ec5c635Skettenis
3412ec5c635Skettenis int
nvme_ans_sart_unmap(void * cookie,bus_addr_t addr,bus_size_t size)342*ea87a8bfSkettenis nvme_ans_sart_unmap(void *cookie, bus_addr_t addr, bus_size_t size)
343*ea87a8bfSkettenis {
344*ea87a8bfSkettenis struct nvme_ans_softc *asc = cookie;
345*ea87a8bfSkettenis
346*ea87a8bfSkettenis return aplsart_unmap(asc->asc_sart, addr, size);
347*ea87a8bfSkettenis }
348*ea87a8bfSkettenis
349*ea87a8bfSkettenis int
nvme_ans_q_alloc(struct nvme_softc * sc,struct nvme_queue * q)3501c31dbaaSdlg nvme_ans_q_alloc(struct nvme_softc *sc,
3511c31dbaaSdlg struct nvme_queue *q)
3521c31dbaaSdlg {
353bfce4b2dSkettenis bus_size_t db, base;
3541c31dbaaSdlg
3551c31dbaaSdlg KASSERT(q->q_entries <= (ANS_NVMMU_TCB_SIZE / ANS_NVMMU_TCB_PITCH));
3561c31dbaaSdlg
3571c31dbaaSdlg q->q_nvmmu_dmamem = nvme_dmamem_alloc(sc, ANS_NVMMU_TCB_SIZE);
3581c31dbaaSdlg if (q->q_nvmmu_dmamem == NULL)
3591c31dbaaSdlg return (-1);
3601c31dbaaSdlg
3611c31dbaaSdlg memset(NVME_DMA_KVA(q->q_nvmmu_dmamem),
3621c31dbaaSdlg 0, NVME_DMA_LEN(q->q_nvmmu_dmamem));
3631c31dbaaSdlg
3641c31dbaaSdlg switch (q->q_id) {
3651c31dbaaSdlg case NVME_IO_Q:
366bfce4b2dSkettenis db = ANS_LINEAR_IOSQ_DB;
367bfce4b2dSkettenis base = ANS_NVMMU_BASE_IOSQ;
3681c31dbaaSdlg break;
3691c31dbaaSdlg case NVME_ADMIN_Q:
3701c31dbaaSdlg db = ANS_LINEAR_ASQ_DB;
3711c31dbaaSdlg base = ANS_NVMMU_BASE_ASQ;
3721c31dbaaSdlg break;
3731c31dbaaSdlg default:
3741c31dbaaSdlg panic("unsupported queue id %u", q->q_id);
3751c31dbaaSdlg /* NOTREACHED */
3761c31dbaaSdlg }
3771c31dbaaSdlg
3781c31dbaaSdlg q->q_sqtdbl = db;
3791c31dbaaSdlg
3801c31dbaaSdlg nvme_dmamem_sync(sc, q->q_nvmmu_dmamem, BUS_DMASYNC_PREWRITE);
3811c31dbaaSdlg nvme_write8(sc, base, NVME_DMA_DVA(q->q_nvmmu_dmamem));
3821c31dbaaSdlg
3831c31dbaaSdlg return (0);
3841c31dbaaSdlg }
3851c31dbaaSdlg
3861c31dbaaSdlg void
nvme_ans_enable(struct nvme_softc * sc)3871c31dbaaSdlg nvme_ans_enable(struct nvme_softc *sc)
3881c31dbaaSdlg {
3891c31dbaaSdlg nvme_write4(sc, ANS_NVMMU_NUM,
3901c31dbaaSdlg (ANS_NVMMU_TCB_SIZE / ANS_NVMMU_TCB_PITCH) - 1);
3911c31dbaaSdlg nvme_write4(sc, ANS_MODESEL_REG, 0);
3921c31dbaaSdlg }
3931c31dbaaSdlg
3941c31dbaaSdlg void
nvme_ans_q_free(struct nvme_softc * sc,struct nvme_queue * q)3951c31dbaaSdlg nvme_ans_q_free(struct nvme_softc *sc,
3961c31dbaaSdlg struct nvme_queue *q)
3971c31dbaaSdlg {
3981c31dbaaSdlg nvme_dmamem_sync(sc, q->q_nvmmu_dmamem, BUS_DMASYNC_POSTWRITE);
3991c31dbaaSdlg nvme_dmamem_free(sc, q->q_nvmmu_dmamem);
4001c31dbaaSdlg }
4011c31dbaaSdlg
4021c31dbaaSdlg uint32_t
nvme_ans_sq_enter(struct nvme_softc * sc,struct nvme_queue * q,struct nvme_ccb * ccb)4031c31dbaaSdlg nvme_ans_sq_enter(struct nvme_softc *sc,
4041c31dbaaSdlg struct nvme_queue *q, struct nvme_ccb *ccb)
4051c31dbaaSdlg {
4061c31dbaaSdlg return (ccb->ccb_id);
4071c31dbaaSdlg }
4081c31dbaaSdlg
4091c31dbaaSdlg static inline struct ans_nvmmu_tcb *
nvme_ans_tcb(struct nvme_queue * q,unsigned int qid)4101c31dbaaSdlg nvme_ans_tcb(struct nvme_queue *q, unsigned int qid)
4111c31dbaaSdlg {
4121c31dbaaSdlg caddr_t ptr = NVME_DMA_KVA(q->q_nvmmu_dmamem);
4131c31dbaaSdlg ptr += qid * ANS_NVMMU_TCB_PITCH;
4141c31dbaaSdlg return ((struct ans_nvmmu_tcb *)ptr);
4151c31dbaaSdlg }
4161c31dbaaSdlg
4171c31dbaaSdlg void
nvme_ans_sq_leave(struct nvme_softc * sc,struct nvme_queue * q,struct nvme_ccb * ccb)4181c31dbaaSdlg nvme_ans_sq_leave(struct nvme_softc *sc,
4191c31dbaaSdlg struct nvme_queue *q, struct nvme_ccb *ccb)
4201c31dbaaSdlg {
4211c31dbaaSdlg unsigned int id = ccb->ccb_id;
4221c31dbaaSdlg struct nvme_sqe_io *sqe;
4231c31dbaaSdlg struct ans_nvmmu_tcb *tcb = nvme_ans_tcb(q, id);
4241c31dbaaSdlg
4251c31dbaaSdlg sqe = NVME_DMA_KVA(q->q_sq_dmamem);
4261c31dbaaSdlg sqe += id;
4271c31dbaaSdlg
4281c31dbaaSdlg bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(q->q_nvmmu_dmamem),
4291c31dbaaSdlg ANS_NVMMU_TCB_PITCH * id, sizeof(*tcb), BUS_DMASYNC_POSTWRITE);
4301c31dbaaSdlg
4311c31dbaaSdlg memset(tcb, 0, sizeof(*tcb));
4321c31dbaaSdlg tcb->tcb_opcode = sqe->opcode;
4331c31dbaaSdlg tcb->tcb_flags = ANS_NVMMU_TCB_WRITE | ANS_NVMMU_TCB_READ;
4341c31dbaaSdlg tcb->tcb_cid = id;
4351c31dbaaSdlg tcb->tcb_prpl_len = sqe->nlb;
4361c31dbaaSdlg tcb->tcb_prp[0] = sqe->entry.prp[0];
4371c31dbaaSdlg tcb->tcb_prp[1] = sqe->entry.prp[1];
4381c31dbaaSdlg
4391c31dbaaSdlg bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(q->q_nvmmu_dmamem),
4401c31dbaaSdlg ANS_NVMMU_TCB_PITCH * id, sizeof(*tcb), BUS_DMASYNC_PREWRITE);
4411c31dbaaSdlg
4421c31dbaaSdlg nvme_write4(sc, q->q_sqtdbl, id);
4431c31dbaaSdlg }
4441c31dbaaSdlg
4451c31dbaaSdlg void
nvme_ans_cq_done(struct nvme_softc * sc,struct nvme_queue * q,struct nvme_ccb * ccb)4461c31dbaaSdlg nvme_ans_cq_done(struct nvme_softc *sc,
4471c31dbaaSdlg struct nvme_queue *q, struct nvme_ccb *ccb)
4481c31dbaaSdlg {
4491c31dbaaSdlg unsigned int id = ccb->ccb_id;
4501c31dbaaSdlg struct ans_nvmmu_tcb *tcb = nvme_ans_tcb(q, id);
4511c31dbaaSdlg uint32_t stat;
4521c31dbaaSdlg
4531c31dbaaSdlg bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(q->q_nvmmu_dmamem),
4541c31dbaaSdlg ANS_NVMMU_TCB_PITCH * id, sizeof(*tcb), BUS_DMASYNC_POSTWRITE);
4551c31dbaaSdlg memset(tcb, 0, sizeof(*tcb));
4561c31dbaaSdlg bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(q->q_nvmmu_dmamem),
4571c31dbaaSdlg ANS_NVMMU_TCB_PITCH * id, sizeof(*tcb), BUS_DMASYNC_PREWRITE);
4581c31dbaaSdlg
4591c31dbaaSdlg nvme_write4(sc, ANS_NVMMU_TCB_INVAL, id);
4601c31dbaaSdlg stat = nvme_read4(sc, ANS_NVMMU_TCB_STAT);
4611c31dbaaSdlg if (stat != 0) {
4621c31dbaaSdlg printf("%s: nvmmu tcp stat is non-zero: 0x%08x\n",
4631c31dbaaSdlg DEVNAME(sc), stat);
4641c31dbaaSdlg }
4651c31dbaaSdlg }
466