1 /* $OpenBSD: aplns.c,v 1.11 2022/04/06 18:59:26 naddy Exp $ */ 2 /* 3 * Copyright (c) 2014, 2021 David Gwynne <dlg@openbsd.org> 4 * 5 * Permission to use, copy, modify, and distribute this software for any 6 * purpose with or without fee is hereby granted, provided that the above 7 * copyright notice and this permission notice appear in all copies. 8 * 9 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 10 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 11 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 12 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 13 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 14 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 15 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 16 */ 17 18 #include <sys/param.h> 19 #include <sys/systm.h> 20 #include <sys/buf.h> 21 #include <sys/kernel.h> 22 #include <sys/malloc.h> 23 #include <sys/device.h> 24 #include <sys/timeout.h> 25 #include <sys/queue.h> 26 #include <sys/mutex.h> 27 #include <sys/pool.h> 28 29 #include <machine/bus.h> 30 #include <machine/fdt.h> 31 32 #include <dev/ofw/openfirm.h> 33 #include <dev/ofw/ofw_misc.h> 34 #include <dev/ofw/ofw_power.h> 35 #include <dev/ofw/fdt.h> 36 37 #include <scsi/scsi_all.h> 38 #include <scsi/scsiconf.h> 39 40 #include <dev/ic/nvmereg.h> 41 #include <dev/ic/nvmevar.h> 42 43 #include <arm64/dev/rtkit.h> 44 45 #define ANS_CPU_CTRL 0x0044 46 #define ANS_CPU_CTRL_RUN (1 << 4) 47 48 #define ANS_MAX_PEND_CMDS_CTRL 0x01210 49 #define ANS_MAX_QUEUE_DEPTH 64 50 #define ANS_BOOT_STATUS 0x01300 51 #define ANS_BOOT_STATUS_OK 0xde71ce55 52 #define ANS_MODESEL_REG 0x01304 53 #define ANS_UNKNOWN_CTRL 0x24008 54 #define ANS_PRP_NULL_CHECK (1 << 11) 55 #define ANS_LINEAR_SQ_CTRL 0x24908 56 #define ANS_LINEAR_SQ_CTRL_EN (1 << 0) 57 #define ANS_LINEAR_ASQ_DB 0x2490c 58 #define ANS_LINEAR_IOSQ_DB 0x24910 59 60 #define ANS_NVMMU_NUM 0x28100 61 #define ANS_NVMMU_BASE_ASQ 0x28108 62 #define ANS_NVMMU_BASE_IOSQ 0x28110 63 #define ANS_NVMMU_TCB_INVAL 0x28118 64 #define ANS_NVMMU_TCB_STAT 0x28120 65 66 #define ANS_NVMMU_TCB_SIZE 0x4000 67 #define ANS_NVMMU_TCB_PITCH 0x80 68 69 struct ans_nvmmu_tcb { 70 uint8_t tcb_opcode; 71 uint8_t tcb_flags; 72 #define ANS_NVMMU_TCB_WRITE (1 << 0) 73 #define ANS_NVMMU_TCB_READ (1 << 1) 74 uint8_t tcb_cid; 75 uint8_t tcb_pad0[1]; 76 77 uint32_t tcb_prpl_len; 78 uint8_t tcb_pad1[16]; 79 80 uint64_t tcb_prp[2]; 81 }; 82 83 int aplns_match(struct device *, void *, void *); 84 void aplns_attach(struct device *, struct device *, void *); 85 86 const struct cfattach aplns_ca = { 87 sizeof(struct device), 88 aplns_match, 89 aplns_attach 90 }; 91 92 struct cfdriver aplns_cd = { 93 NULL, "aplns", DV_DULL 94 }; 95 96 int 97 aplns_match(struct device *parent, void *match, void *aux) 98 { 99 struct fdt_attach_args *faa = aux; 100 101 return (OF_is_compatible(faa->fa_node, "apple,nvme-m1") || 102 OF_is_compatible(faa->fa_node, "apple,nvme-ans2")); 103 } 104 105 void 106 aplns_attach(struct device *parent, struct device *self, void *aux) 107 { 108 struct fdt_attach_args *faa = aux; 109 110 printf("\n"); 111 112 config_found(self, faa, NULL); 113 } 114 115 struct nvme_ans_softc { 116 struct nvme_softc asc_nvme; 117 bus_space_tag_t asc_iot; 118 bus_space_handle_t asc_ioh; 119 120 struct rtkit_state *asc_rtkit; 121 struct nvme_dmamem *asc_nvmmu; 122 }; 123 124 int nvme_ans_match(struct device *, void *, void *); 125 void nvme_ans_attach(struct device *, struct device *, void *); 126 127 const struct cfattach nvme_ans_ca = { 128 sizeof(struct nvme_ans_softc), 129 nvme_ans_match, 130 nvme_ans_attach, 131 }; 132 133 void nvme_ans_enable(struct nvme_softc *); 134 135 int nvme_ans_q_alloc(struct nvme_softc *, 136 struct nvme_queue *); 137 void nvme_ans_q_free(struct nvme_softc *, 138 struct nvme_queue *); 139 140 uint32_t nvme_ans_sq_enter(struct nvme_softc *, 141 struct nvme_queue *, struct nvme_ccb *); 142 void nvme_ans_sq_leave(struct nvme_softc *, 143 struct nvme_queue *, struct nvme_ccb *); 144 145 void nvme_ans_cq_done(struct nvme_softc *, 146 struct nvme_queue *, struct nvme_ccb *); 147 148 static const struct nvme_ops nvme_ans_ops = { 149 .op_enable = nvme_ans_enable, 150 151 .op_q_alloc = nvme_ans_q_alloc, 152 .op_q_free = nvme_ans_q_free, 153 154 .op_sq_enter = nvme_ans_sq_enter, 155 .op_sq_leave = nvme_ans_sq_leave, 156 .op_sq_enter_locked = nvme_ans_sq_enter, 157 .op_sq_leave_locked = nvme_ans_sq_leave, 158 159 .op_cq_done = nvme_ans_cq_done, 160 }; 161 162 int 163 nvme_ans_match(struct device *parent, void *match, void *aux) 164 { 165 struct fdt_attach_args *faa = aux; 166 167 return (OF_is_compatible(faa->fa_node, "apple,nvme-m1") || 168 OF_is_compatible(faa->fa_node, "apple,nvme-ans2")); 169 } 170 171 void 172 nvme_ans_attach(struct device *parent, struct device *self, void *aux) 173 { 174 struct nvme_ans_softc *asc = (struct nvme_ans_softc *)self; 175 struct nvme_softc *sc = &asc->asc_nvme; 176 struct fdt_attach_args *faa = aux; 177 uint32_t ctrl, status; 178 179 if (faa->fa_nreg < 2) { 180 printf(": no registers\n"); 181 return; 182 } 183 184 sc->sc_iot = faa->fa_iot; 185 if (bus_space_map(sc->sc_iot, faa->fa_reg[0].addr, 186 faa->fa_reg[0].size, 0, &sc->sc_ioh) != 0) { 187 printf(": can't map registers\n"); 188 return; 189 } 190 191 asc->asc_iot = faa->fa_iot; 192 if (bus_space_map(asc->asc_iot, faa->fa_reg[1].addr, 193 faa->fa_reg[1].size, 0, &asc->asc_ioh)) { 194 bus_space_unmap(sc->sc_iot, sc->sc_ioh, sc->sc_ios); 195 printf(": can't map registers\n"); 196 return; 197 } 198 199 power_domain_enable(faa->fa_node); 200 201 sc->sc_ih = fdt_intr_establish(faa->fa_node, IPL_BIO, 202 nvme_intr, sc, sc->sc_dev.dv_xname); 203 if (sc->sc_ih == NULL) { 204 printf(": can't establish interrupt\n"); 205 goto unmap; 206 } 207 208 asc->asc_rtkit = rtkit_init(faa->fa_node, NULL); 209 if (asc->asc_rtkit == NULL) { 210 printf(": can't map mailbox channel\n"); 211 goto disestablish; 212 } 213 214 ctrl = bus_space_read_4(asc->asc_iot, asc->asc_ioh, ANS_CPU_CTRL); 215 bus_space_write_4(asc->asc_iot, asc->asc_ioh, ANS_CPU_CTRL, 216 ctrl | ANS_CPU_CTRL_RUN); 217 218 status = bus_space_read_4(sc->sc_iot, sc->sc_ioh, ANS_BOOT_STATUS); 219 if (status != ANS_BOOT_STATUS_OK) 220 rtkit_boot(asc->asc_rtkit); 221 222 status = bus_space_read_4(sc->sc_iot, sc->sc_ioh, ANS_BOOT_STATUS); 223 if (status != ANS_BOOT_STATUS_OK) { 224 printf(": firmware not ready\n"); 225 goto disestablish; 226 } 227 228 bus_space_write_4(sc->sc_iot, sc->sc_ioh, ANS_LINEAR_SQ_CTRL, 229 ANS_LINEAR_SQ_CTRL_EN); 230 bus_space_write_4(sc->sc_iot, sc->sc_ioh, ANS_MAX_PEND_CMDS_CTRL, 231 (ANS_MAX_QUEUE_DEPTH << 16) | ANS_MAX_QUEUE_DEPTH); 232 233 ctrl = bus_space_read_4(sc->sc_iot, sc->sc_ioh, ANS_UNKNOWN_CTRL); 234 bus_space_write_4(sc->sc_iot, sc->sc_ioh, ANS_UNKNOWN_CTRL, 235 ctrl & ~ANS_PRP_NULL_CHECK); 236 237 printf(": "); 238 239 sc->sc_dmat = faa->fa_dmat; 240 sc->sc_ios = faa->fa_reg[0].size; 241 sc->sc_ops = &nvme_ans_ops; 242 sc->sc_openings = 1; 243 244 if (nvme_attach(sc) != 0) { 245 /* error printed by nvme_attach() */ 246 goto disestablish; 247 } 248 249 return; 250 251 disestablish: 252 fdt_intr_disestablish(sc->sc_ih); 253 sc->sc_ih = NULL; 254 255 unmap: 256 bus_space_unmap(asc->asc_iot, asc->asc_ioh, faa->fa_reg[1].size); 257 bus_space_unmap(sc->sc_iot, sc->sc_ioh, faa->fa_reg[0].size); 258 sc->sc_ios = 0; 259 } 260 261 int 262 nvme_ans_q_alloc(struct nvme_softc *sc, 263 struct nvme_queue *q) 264 { 265 bus_size_t db, base; 266 267 KASSERT(q->q_entries <= (ANS_NVMMU_TCB_SIZE / ANS_NVMMU_TCB_PITCH)); 268 269 q->q_nvmmu_dmamem = nvme_dmamem_alloc(sc, ANS_NVMMU_TCB_SIZE); 270 if (q->q_nvmmu_dmamem == NULL) 271 return (-1); 272 273 memset(NVME_DMA_KVA(q->q_nvmmu_dmamem), 274 0, NVME_DMA_LEN(q->q_nvmmu_dmamem)); 275 276 switch (q->q_id) { 277 case NVME_IO_Q: 278 db = ANS_LINEAR_IOSQ_DB; 279 base = ANS_NVMMU_BASE_IOSQ; 280 break; 281 case NVME_ADMIN_Q: 282 db = ANS_LINEAR_ASQ_DB; 283 base = ANS_NVMMU_BASE_ASQ; 284 break; 285 default: 286 panic("unsupported queue id %u", q->q_id); 287 /* NOTREACHED */ 288 } 289 290 q->q_sqtdbl = db; 291 292 nvme_dmamem_sync(sc, q->q_nvmmu_dmamem, BUS_DMASYNC_PREWRITE); 293 nvme_write8(sc, base, NVME_DMA_DVA(q->q_nvmmu_dmamem)); 294 295 return (0); 296 } 297 298 void 299 nvme_ans_enable(struct nvme_softc *sc) 300 { 301 nvme_write4(sc, ANS_NVMMU_NUM, 302 (ANS_NVMMU_TCB_SIZE / ANS_NVMMU_TCB_PITCH) - 1); 303 nvme_write4(sc, ANS_MODESEL_REG, 0); 304 } 305 306 void 307 nvme_ans_q_free(struct nvme_softc *sc, 308 struct nvme_queue *q) 309 { 310 nvme_dmamem_sync(sc, q->q_nvmmu_dmamem, BUS_DMASYNC_POSTWRITE); 311 nvme_dmamem_free(sc, q->q_nvmmu_dmamem); 312 } 313 314 uint32_t 315 nvme_ans_sq_enter(struct nvme_softc *sc, 316 struct nvme_queue *q, struct nvme_ccb *ccb) 317 { 318 return (ccb->ccb_id); 319 } 320 321 static inline struct ans_nvmmu_tcb * 322 nvme_ans_tcb(struct nvme_queue *q, unsigned int qid) 323 { 324 caddr_t ptr = NVME_DMA_KVA(q->q_nvmmu_dmamem); 325 ptr += qid * ANS_NVMMU_TCB_PITCH; 326 return ((struct ans_nvmmu_tcb *)ptr); 327 } 328 329 void 330 nvme_ans_sq_leave(struct nvme_softc *sc, 331 struct nvme_queue *q, struct nvme_ccb *ccb) 332 { 333 unsigned int id = ccb->ccb_id; 334 struct nvme_sqe_io *sqe; 335 struct ans_nvmmu_tcb *tcb = nvme_ans_tcb(q, id); 336 337 sqe = NVME_DMA_KVA(q->q_sq_dmamem); 338 sqe += id; 339 340 bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(q->q_nvmmu_dmamem), 341 ANS_NVMMU_TCB_PITCH * id, sizeof(*tcb), BUS_DMASYNC_POSTWRITE); 342 343 memset(tcb, 0, sizeof(*tcb)); 344 tcb->tcb_opcode = sqe->opcode; 345 tcb->tcb_flags = ANS_NVMMU_TCB_WRITE | ANS_NVMMU_TCB_READ; 346 tcb->tcb_cid = id; 347 tcb->tcb_prpl_len = sqe->nlb; 348 tcb->tcb_prp[0] = sqe->entry.prp[0]; 349 tcb->tcb_prp[1] = sqe->entry.prp[1]; 350 351 bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(q->q_nvmmu_dmamem), 352 ANS_NVMMU_TCB_PITCH * id, sizeof(*tcb), BUS_DMASYNC_PREWRITE); 353 354 nvme_write4(sc, q->q_sqtdbl, id); 355 } 356 357 void 358 nvme_ans_cq_done(struct nvme_softc *sc, 359 struct nvme_queue *q, struct nvme_ccb *ccb) 360 { 361 unsigned int id = ccb->ccb_id; 362 struct ans_nvmmu_tcb *tcb = nvme_ans_tcb(q, id); 363 uint32_t stat; 364 365 bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(q->q_nvmmu_dmamem), 366 ANS_NVMMU_TCB_PITCH * id, sizeof(*tcb), BUS_DMASYNC_POSTWRITE); 367 memset(tcb, 0, sizeof(*tcb)); 368 bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(q->q_nvmmu_dmamem), 369 ANS_NVMMU_TCB_PITCH * id, sizeof(*tcb), BUS_DMASYNC_PREWRITE); 370 371 nvme_write4(sc, ANS_NVMMU_TCB_INVAL, id); 372 stat = nvme_read4(sc, ANS_NVMMU_TCB_STAT); 373 if (stat != 0) { 374 printf("%s: nvmmu tcp stat is non-zero: 0x%08x\n", 375 DEVNAME(sc), stat); 376 } 377 } 378