1 /* $OpenBSD: nvme.c,v 1.50 2016/05/20 11:11:05 dlg Exp $ */ 2 3 /* 4 * Copyright (c) 2014 David Gwynne <dlg@openbsd.org> 5 * 6 * Permission to use, copy, modify, and distribute this software for any 7 * purpose with or without fee is hereby granted, provided that the above 8 * copyright notice and this permission notice appear in all copies. 9 * 10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 17 */ 18 19 #include <sys/param.h> 20 #include <sys/systm.h> 21 #include <sys/buf.h> 22 #include <sys/kernel.h> 23 #include <sys/malloc.h> 24 #include <sys/device.h> 25 #include <sys/queue.h> 26 #include <sys/mutex.h> 27 #include <sys/pool.h> 28 29 #include <machine/bus.h> 30 31 #include <scsi/scsi_all.h> 32 #include <scsi/scsi_disk.h> 33 #include <scsi/scsiconf.h> 34 35 #include <dev/ic/nvmereg.h> 36 #include <dev/ic/nvmevar.h> 37 38 struct cfdriver nvme_cd = { 39 NULL, 40 "nvme", 41 DV_DULL 42 }; 43 44 int nvme_ready(struct nvme_softc *, u_int32_t); 45 int nvme_enable(struct nvme_softc *, u_int); 46 int nvme_disable(struct nvme_softc *); 47 int nvme_shutdown(struct nvme_softc *); 48 49 void nvme_version(struct nvme_softc *, u_int32_t); 50 void nvme_dumpregs(struct nvme_softc *); 51 int nvme_identify(struct nvme_softc *, u_int); 52 void nvme_fill_identify(struct nvme_softc *, struct nvme_ccb *, void *); 53 54 int nvme_ccbs_alloc(struct nvme_softc *, u_int); 55 void nvme_ccbs_free(struct nvme_softc *); 56 57 void * nvme_ccb_get(void *); 58 void nvme_ccb_put(void *, void *); 59 60 int nvme_poll(struct nvme_softc *, struct nvme_queue *, struct nvme_ccb *, 61 void (*)(struct nvme_softc *, struct nvme_ccb *, void *)); 62 void nvme_poll_fill(struct nvme_softc *, struct nvme_ccb *, void *); 63 void nvme_poll_done(struct nvme_softc *, struct nvme_ccb *, 64 struct nvme_cqe *); 65 void nvme_sqe_fill(struct nvme_softc *, struct nvme_ccb *, void *); 66 void nvme_empty_done(struct nvme_softc *, struct nvme_ccb *, 67 struct nvme_cqe *); 68 69 struct nvme_queue * 70 nvme_q_alloc(struct nvme_softc *, u_int16_t, u_int, u_int); 71 int nvme_q_create(struct nvme_softc *, struct nvme_queue *); 72 int nvme_q_delete(struct nvme_softc *, struct nvme_queue *); 73 void nvme_q_submit(struct nvme_softc *, 74 struct nvme_queue *, struct nvme_ccb *, 75 void (*)(struct nvme_softc *, struct nvme_ccb *, void *)); 76 int nvme_q_complete(struct nvme_softc *, struct nvme_queue *); 77 void nvme_q_free(struct nvme_softc *, struct nvme_queue *); 78 79 struct nvme_dmamem * 80 nvme_dmamem_alloc(struct nvme_softc *, size_t); 81 void nvme_dmamem_free(struct nvme_softc *, struct nvme_dmamem *); 82 void nvme_dmamem_sync(struct nvme_softc *, struct nvme_dmamem *, int); 83 84 void nvme_scsi_cmd(struct scsi_xfer *); 85 int nvme_scsi_probe(struct scsi_link *); 86 void nvme_scsi_free(struct scsi_link *); 87 88 struct scsi_adapter nvme_switch = { 89 nvme_scsi_cmd, /* cmd */ 90 scsi_minphys, /* minphys */ 91 nvme_scsi_probe, /* dev probe */ 92 nvme_scsi_free, /* dev free */ 93 NULL, /* ioctl */ 94 }; 95 96 void nvme_scsi_io(struct scsi_xfer *, int); 97 void nvme_scsi_io_fill(struct nvme_softc *, struct nvme_ccb *, void *); 98 void nvme_scsi_io_done(struct nvme_softc *, struct nvme_ccb *, 99 struct nvme_cqe *); 100 101 void nvme_scsi_sync(struct scsi_xfer *); 102 void nvme_scsi_sync_fill(struct nvme_softc *, struct nvme_ccb *, void *); 103 void nvme_scsi_sync_done(struct nvme_softc *, struct nvme_ccb *, 104 struct nvme_cqe *); 105 106 void nvme_scsi_inq(struct scsi_xfer *); 107 void nvme_scsi_inquiry(struct scsi_xfer *); 108 void nvme_scsi_capacity16(struct scsi_xfer *); 109 void nvme_scsi_capacity(struct scsi_xfer *); 110 111 #define nvme_read4(_s, _r) \ 112 bus_space_read_4((_s)->sc_iot, (_s)->sc_ioh, (_r)) 113 #define nvme_write4(_s, _r, _v) \ 114 bus_space_write_4((_s)->sc_iot, (_s)->sc_ioh, (_r), (_v)) 115 #ifdef __LP64__ 116 #define nvme_read8(_s, _r) \ 117 bus_space_read_8((_s)->sc_iot, (_s)->sc_ioh, (_r)) 118 #define nvme_write8(_s, _r, _v) \ 119 bus_space_write_8((_s)->sc_iot, (_s)->sc_ioh, (_r), (_v)) 120 #else /* __LP64__ */ 121 static inline u_int64_t 122 nvme_read8(struct nvme_softc *sc, bus_size_t r) 123 { 124 u_int64_t v; 125 u_int32_t *a = (u_int32_t *)&v; 126 127 #if _BYTE_ORDER == _LITTLE_ENDIAN 128 a[0] = nvme_read4(sc, r); 129 a[1] = nvme_read4(sc, r + 4); 130 #else /* _BYTE_ORDER == _LITTLE_ENDIAN */ 131 a[1] = nvme_read4(sc, r); 132 a[0] = nvme_read4(sc, r + 4); 133 #endif 134 135 return (v); 136 } 137 138 static inline void 139 nvme_write8(struct nvme_softc *sc, bus_size_t r, u_int64_t v) 140 { 141 u_int32_t *a = (u_int32_t *)&v; 142 143 #if _BYTE_ORDER == _LITTLE_ENDIAN 144 nvme_write4(sc, r, a[0]); 145 nvme_write4(sc, r + 4, a[1]); 146 #else /* _BYTE_ORDER == _LITTLE_ENDIAN */ 147 nvme_write4(sc, r, a[1]); 148 nvme_write4(sc, r + 4, a[0]); 149 #endif 150 } 151 #endif /* __LP64__ */ 152 #define nvme_barrier(_s, _r, _l, _f) \ 153 bus_space_barrier((_s)->sc_iot, (_s)->sc_ioh, (_r), (_l), (_f)) 154 155 void 156 nvme_version(struct nvme_softc *sc, u_int32_t version) 157 { 158 const char *v = NULL; 159 160 switch (version) { 161 case NVME_VS_1_0: 162 v = "1.0"; 163 break; 164 case NVME_VS_1_1: 165 v = "1.1"; 166 break; 167 case NVME_VS_1_2: 168 v = "1.2"; 169 break; 170 default: 171 printf(", unknown version 0x%08x", version); 172 return; 173 } 174 175 printf(", NVMe %s", v); 176 } 177 178 void 179 nvme_dumpregs(struct nvme_softc *sc) 180 { 181 u_int64_t r8; 182 u_int32_t r4; 183 184 r8 = nvme_read8(sc, NVME_CAP); 185 printf("%s: cap 0x%016llx\n", DEVNAME(sc), nvme_read8(sc, NVME_CAP)); 186 printf("%s: mpsmax %u (%u)\n", DEVNAME(sc), 187 (u_int)NVME_CAP_MPSMAX(r8), (1 << NVME_CAP_MPSMAX(r8))); 188 printf("%s: mpsmin %u (%u)\n", DEVNAME(sc), 189 (u_int)NVME_CAP_MPSMIN(r8), (1 << NVME_CAP_MPSMIN(r8))); 190 printf("%s: css %llu\n", DEVNAME(sc), NVME_CAP_CSS(r8)); 191 printf("%s: nssrs %llu\n", DEVNAME(sc), NVME_CAP_NSSRS(r8)); 192 printf("%s: dstrd %u\n", DEVNAME(sc), NVME_CAP_DSTRD(r8)); 193 printf("%s: to %llu msec\n", DEVNAME(sc), NVME_CAP_TO(r8)); 194 printf("%s: ams %llu\n", DEVNAME(sc), NVME_CAP_AMS(r8)); 195 printf("%s: cqr %llu\n", DEVNAME(sc), NVME_CAP_CQR(r8)); 196 printf("%s: mqes %llu\n", DEVNAME(sc), NVME_CAP_MQES(r8)); 197 198 printf("%s: vs 0x%04x\n", DEVNAME(sc), nvme_read4(sc, NVME_VS)); 199 200 r4 = nvme_read4(sc, NVME_CC); 201 printf("%s: cc 0x%04x\n", DEVNAME(sc), r4); 202 printf("%s: iocqes %u\n", DEVNAME(sc), NVME_CC_IOCQES_R(r4)); 203 printf("%s: iosqes %u\n", DEVNAME(sc), NVME_CC_IOSQES_R(r4)); 204 printf("%s: shn %u\n", DEVNAME(sc), NVME_CC_SHN_R(r4)); 205 printf("%s: ams %u\n", DEVNAME(sc), NVME_CC_AMS_R(r4)); 206 printf("%s: mps %u\n", DEVNAME(sc), NVME_CC_MPS_R(r4)); 207 printf("%s: css %u\n", DEVNAME(sc), NVME_CC_CSS_R(r4)); 208 printf("%s: en %u\n", DEVNAME(sc), ISSET(r4, NVME_CC_EN)); 209 210 printf("%s: csts 0x%08x\n", DEVNAME(sc), nvme_read4(sc, NVME_CSTS)); 211 printf("%s: aqa 0x%08x\n", DEVNAME(sc), nvme_read4(sc, NVME_AQA)); 212 printf("%s: asq 0x%016llx\n", DEVNAME(sc), nvme_read8(sc, NVME_ASQ)); 213 printf("%s: acq 0x%016llx\n", DEVNAME(sc), nvme_read8(sc, NVME_ACQ)); 214 } 215 216 int 217 nvme_ready(struct nvme_softc *sc, u_int32_t rdy) 218 { 219 u_int i = 0; 220 221 while ((nvme_read4(sc, NVME_CSTS) & NVME_CSTS_RDY) != rdy) { 222 if (i++ > sc->sc_rdy_to) 223 return (1); 224 225 delay(1000); 226 nvme_barrier(sc, NVME_CSTS, 4, BUS_SPACE_BARRIER_READ); 227 } 228 229 return (0); 230 } 231 232 int 233 nvme_enable(struct nvme_softc *sc, u_int mps) 234 { 235 u_int32_t cc; 236 237 cc = nvme_read4(sc, NVME_CC); 238 if (ISSET(cc, NVME_CC_EN)) 239 return (nvme_ready(sc, NVME_CSTS_RDY)); 240 241 nvme_write4(sc, NVME_AQA, NVME_AQA_ACQS(sc->sc_admin_q->q_entries) | 242 NVME_AQA_ASQS(sc->sc_admin_q->q_entries)); 243 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE); 244 245 nvme_write8(sc, NVME_ASQ, NVME_DMA_DVA(sc->sc_admin_q->q_sq_dmamem)); 246 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE); 247 nvme_write8(sc, NVME_ACQ, NVME_DMA_DVA(sc->sc_admin_q->q_cq_dmamem)); 248 nvme_barrier(sc, 0, sc->sc_ios, BUS_SPACE_BARRIER_WRITE); 249 250 CLR(cc, NVME_CC_IOCQES_MASK | NVME_CC_IOSQES_MASK | NVME_CC_SHN_MASK | 251 NVME_CC_AMS_MASK | NVME_CC_MPS_MASK | NVME_CC_CSS_MASK); 252 SET(cc, NVME_CC_IOSQES(ffs(64) - 1) | NVME_CC_IOCQES(ffs(16) - 1)); 253 SET(cc, NVME_CC_SHN(NVME_CC_SHN_NONE)); 254 SET(cc, NVME_CC_CSS(NVME_CC_CSS_NVM)); 255 SET(cc, NVME_CC_AMS(NVME_CC_AMS_RR)); 256 SET(cc, NVME_CC_MPS(mps)); 257 SET(cc, NVME_CC_EN); 258 259 nvme_write4(sc, NVME_CC, cc); 260 nvme_barrier(sc, 0, sc->sc_ios, 261 BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE); 262 263 return (nvme_ready(sc, NVME_CSTS_RDY)); 264 } 265 266 int 267 nvme_disable(struct nvme_softc *sc) 268 { 269 u_int32_t cc, csts; 270 271 cc = nvme_read4(sc, NVME_CC); 272 if (ISSET(cc, NVME_CC_EN)) { 273 csts = nvme_read4(sc, NVME_CSTS); 274 if (!ISSET(csts, NVME_CSTS_CFS) && 275 nvme_ready(sc, NVME_CSTS_RDY) != 0) 276 return (1); 277 } 278 279 CLR(cc, NVME_CC_EN); 280 281 nvme_write4(sc, NVME_CC, cc); 282 nvme_barrier(sc, 0, sc->sc_ios, 283 BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE); 284 285 return (nvme_ready(sc, 0)); 286 } 287 288 int 289 nvme_attach(struct nvme_softc *sc) 290 { 291 struct scsibus_attach_args saa; 292 u_int64_t cap; 293 u_int32_t reg; 294 u_int dstrd; 295 u_int mps = PAGE_SHIFT; 296 297 mtx_init(&sc->sc_ccb_mtx, IPL_BIO); 298 SIMPLEQ_INIT(&sc->sc_ccb_list); 299 scsi_iopool_init(&sc->sc_iopool, sc, nvme_ccb_get, nvme_ccb_put); 300 301 reg = nvme_read4(sc, NVME_VS); 302 if (reg == 0xffffffff) { 303 printf(", invalid mapping\n"); 304 return (1); 305 } 306 307 nvme_version(sc, reg); 308 printf("\n"); 309 310 cap = nvme_read8(sc, NVME_CAP); 311 dstrd = NVME_CAP_DSTRD(cap); 312 if (NVME_CAP_MPSMIN(cap) > PAGE_SHIFT) { 313 printf("%s: NVMe minimum page size %u " 314 "is greater than CPU page size %u\n", DEVNAME(sc), 315 1 << NVME_CAP_MPSMIN(cap), 1 << PAGE_SHIFT); 316 return (1); 317 } 318 if (NVME_CAP_MPSMAX(cap) < mps) 319 mps = NVME_CAP_MPSMAX(cap); 320 321 sc->sc_rdy_to = NVME_CAP_TO(cap); 322 sc->sc_mps = 1 << mps; 323 sc->sc_mdts = MAXPHYS; 324 sc->sc_max_sgl = 2; 325 326 if (nvme_disable(sc) != 0) { 327 printf("%s: unable to disable controller\n", DEVNAME(sc)); 328 return (1); 329 } 330 331 sc->sc_admin_q = nvme_q_alloc(sc, NVME_ADMIN_Q, 128, dstrd); 332 if (sc->sc_admin_q == NULL) { 333 printf("%s: unable to allocate admin queue\n", DEVNAME(sc)); 334 return (1); 335 } 336 337 if (nvme_ccbs_alloc(sc, 16) != 0) { 338 printf("%s: unable to allocate initial ccbs\n", DEVNAME(sc)); 339 goto free_admin_q; 340 } 341 342 if (nvme_enable(sc, mps) != 0) { 343 printf("%s: unable to enable controller\n", DEVNAME(sc)); 344 goto free_ccbs; 345 } 346 347 if (nvme_identify(sc, NVME_CAP_MPSMIN(cap)) != 0) { 348 printf("%s: unable to identify controller\n", DEVNAME(sc)); 349 goto disable; 350 } 351 352 /* we know how big things are now */ 353 sc->sc_max_sgl = sc->sc_mdts / sc->sc_mps; 354 355 nvme_ccbs_free(sc); 356 if (nvme_ccbs_alloc(sc, 64) != 0) { 357 printf("%s: unable to allocate ccbs\n", DEVNAME(sc)); 358 goto free_admin_q; 359 } 360 361 sc->sc_q = nvme_q_alloc(sc, 1, 128, dstrd); 362 if (sc->sc_q == NULL) { 363 printf("%s: unable to allocate io q\n", DEVNAME(sc)); 364 goto disable; 365 } 366 367 if (nvme_q_create(sc, sc->sc_q) != 0) { 368 printf("%s: unable to create io q\n", DEVNAME(sc)); 369 goto free_q; 370 } 371 372 nvme_write4(sc, NVME_INTMC, 1); 373 374 sc->sc_namespaces = mallocarray(sc->sc_nn, sizeof(*sc->sc_namespaces), 375 M_DEVBUF, M_WAITOK|M_ZERO); 376 377 sc->sc_link.adapter = &nvme_switch; 378 sc->sc_link.adapter_softc = sc; 379 sc->sc_link.adapter_buswidth = sc->sc_nn; 380 sc->sc_link.luns = 1; 381 sc->sc_link.adapter_target = sc->sc_nn; 382 sc->sc_link.openings = 64; 383 sc->sc_link.pool = &sc->sc_iopool; 384 385 memset(&saa, 0, sizeof(saa)); 386 saa.saa_sc_link = &sc->sc_link; 387 388 sc->sc_scsibus = (struct scsibus_softc *)config_found(&sc->sc_dev, 389 &saa, scsiprint); 390 391 return (0); 392 393 free_q: 394 nvme_q_free(sc, sc->sc_q); 395 disable: 396 nvme_disable(sc); 397 free_ccbs: 398 nvme_ccbs_free(sc); 399 free_admin_q: 400 nvme_q_free(sc, sc->sc_admin_q); 401 402 return (1); 403 } 404 405 int 406 nvme_scsi_probe(struct scsi_link *link) 407 { 408 struct nvme_softc *sc = link->adapter_softc; 409 struct nvme_sqe sqe; 410 struct nvm_identify_namespace *identify; 411 struct nvme_dmamem *mem; 412 struct nvme_ccb *ccb; 413 int rv; 414 415 ccb = scsi_io_get(&sc->sc_iopool, 0); 416 KASSERT(ccb != NULL); 417 418 mem = nvme_dmamem_alloc(sc, sizeof(*identify)); 419 if (mem == NULL) 420 return (ENOMEM); 421 422 memset(&sqe, 0, sizeof(sqe)); 423 sqe.opcode = NVM_ADMIN_IDENTIFY; 424 htolem32(&sqe.nsid, link->target + 1); 425 htolem64(&sqe.entry.prp[0], NVME_DMA_DVA(mem)); 426 htolem32(&sqe.cdw10, 0); 427 428 ccb->ccb_done = nvme_empty_done; 429 ccb->ccb_cookie = &sqe; 430 431 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_PREREAD); 432 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill); 433 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_POSTREAD); 434 435 scsi_io_put(&sc->sc_iopool, ccb); 436 437 if (rv != 0) { 438 rv = EIO; 439 goto done; 440 } 441 442 /* commit */ 443 444 identify = malloc(sizeof(*identify), M_DEVBUF, M_WAITOK|M_ZERO); 445 memcpy(identify, NVME_DMA_KVA(mem), sizeof(*identify)); 446 447 sc->sc_namespaces[link->target].ident = identify; 448 449 done: 450 nvme_dmamem_free(sc, mem); 451 452 return (rv); 453 } 454 455 int 456 nvme_shutdown(struct nvme_softc *sc) 457 { 458 u_int32_t cc, csts; 459 int i; 460 461 nvme_write4(sc, NVME_INTMC, 0); 462 463 if (nvme_q_delete(sc, sc->sc_q) != 0) { 464 printf("%s: unable to delete q, disabling\n", DEVNAME(sc)); 465 goto disable; 466 } 467 468 cc = nvme_read4(sc, NVME_CC); 469 CLR(cc, NVME_CC_SHN_MASK); 470 SET(cc, NVME_CC_SHN(NVME_CC_SHN_NORMAL)); 471 nvme_write4(sc, NVME_CC, cc); 472 473 for (i = 0; i < 4000; i++) { 474 nvme_barrier(sc, 0, sc->sc_ios, 475 BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE); 476 csts = nvme_read4(sc, NVME_CSTS); 477 if ((csts & NVME_CSTS_SHST_MASK) == NVME_CSTS_SHST_DONE) 478 return (0); 479 480 delay(1000); 481 } 482 483 printf("%s: unable to shudown, disabling\n", DEVNAME(sc)); 484 485 disable: 486 nvme_disable(sc); 487 return (0); 488 } 489 490 int 491 nvme_activate(struct nvme_softc *sc, int act) 492 { 493 int rv; 494 495 switch (act) { 496 case DVACT_POWERDOWN: 497 rv = config_activate_children(&sc->sc_dev, act); 498 nvme_shutdown(sc); 499 break; 500 default: 501 rv = config_activate_children(&sc->sc_dev, act); 502 break; 503 } 504 505 return (rv); 506 } 507 508 void 509 nvme_scsi_cmd(struct scsi_xfer *xs) 510 { 511 switch (xs->cmd->opcode) { 512 case READ_COMMAND: 513 case READ_BIG: 514 case READ_12: 515 case READ_16: 516 nvme_scsi_io(xs, SCSI_DATA_IN); 517 return; 518 case WRITE_COMMAND: 519 case WRITE_BIG: 520 case WRITE_12: 521 case WRITE_16: 522 nvme_scsi_io(xs, SCSI_DATA_OUT); 523 return; 524 525 case SYNCHRONIZE_CACHE: 526 nvme_scsi_sync(xs); 527 return; 528 529 case INQUIRY: 530 nvme_scsi_inq(xs); 531 return; 532 case READ_CAPACITY_16: 533 nvme_scsi_capacity16(xs); 534 return; 535 case READ_CAPACITY: 536 nvme_scsi_capacity(xs); 537 return; 538 539 case TEST_UNIT_READY: 540 case PREVENT_ALLOW: 541 case START_STOP: 542 xs->error = XS_NOERROR; 543 scsi_done(xs); 544 return; 545 546 default: 547 break; 548 } 549 550 xs->error = XS_DRIVER_STUFFUP; 551 scsi_done(xs); 552 } 553 554 void 555 nvme_scsi_io(struct scsi_xfer *xs, int dir) 556 { 557 struct scsi_link *link = xs->sc_link; 558 struct nvme_softc *sc = link->adapter_softc; 559 struct nvme_ccb *ccb = xs->io; 560 bus_dmamap_t dmap = ccb->ccb_dmamap; 561 int i; 562 563 if ((xs->flags & (SCSI_DATA_IN|SCSI_DATA_OUT)) != dir) 564 goto stuffup; 565 566 ccb->ccb_done = nvme_scsi_io_done; 567 ccb->ccb_cookie = xs; 568 569 if (bus_dmamap_load(sc->sc_dmat, dmap, 570 xs->data, xs->datalen, NULL, ISSET(xs->flags, SCSI_NOSLEEP) ? 571 BUS_DMA_NOWAIT : BUS_DMA_WAITOK) != 0) 572 goto stuffup; 573 574 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 575 ISSET(xs->flags, SCSI_DATA_IN) ? 576 BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE); 577 578 if (dmap->dm_nsegs > 2) { 579 for (i = 1; i < dmap->dm_nsegs; i++) { 580 htolem64(&ccb->ccb_prpl[i - 1], 581 dmap->dm_segs[i].ds_addr); 582 } 583 bus_dmamap_sync(sc->sc_dmat, 584 NVME_DMA_MAP(sc->sc_ccb_prpls), 585 ccb->ccb_prpl_off, 586 sizeof(*ccb->ccb_prpl) * dmap->dm_nsegs - 1, 587 BUS_DMASYNC_PREWRITE); 588 } 589 590 if (ISSET(xs->flags, SCSI_POLL)) { 591 nvme_poll(sc, sc->sc_q, ccb, nvme_scsi_io_fill); 592 return; 593 } 594 595 nvme_q_submit(sc, sc->sc_q, ccb, nvme_scsi_io_fill); 596 return; 597 598 stuffup: 599 xs->error = XS_DRIVER_STUFFUP; 600 scsi_done(xs); 601 } 602 603 void 604 nvme_scsi_io_fill(struct nvme_softc *sc, struct nvme_ccb *ccb, void *slot) 605 { 606 struct nvme_sqe_io *sqe = slot; 607 struct scsi_xfer *xs = ccb->ccb_cookie; 608 struct scsi_link *link = xs->sc_link; 609 bus_dmamap_t dmap = ccb->ccb_dmamap; 610 u_int64_t lba; 611 u_int32_t blocks; 612 613 scsi_cmd_rw_decode(xs->cmd, &lba, &blocks); 614 615 sqe->opcode = ISSET(xs->flags, SCSI_DATA_IN) ? 616 NVM_CMD_READ : NVM_CMD_WRITE; 617 htolem32(&sqe->nsid, link->target + 1); 618 619 htolem64(&sqe->entry.prp[0], dmap->dm_segs[0].ds_addr); 620 switch (dmap->dm_nsegs) { 621 case 1: 622 break; 623 case 2: 624 htolem64(&sqe->entry.prp[1], dmap->dm_segs[1].ds_addr); 625 break; 626 default: 627 /* the prp list is already set up and synced */ 628 htolem64(&sqe->entry.prp[1], ccb->ccb_prpl_dva); 629 break; 630 } 631 632 htolem64(&sqe->slba, lba); 633 htolem16(&sqe->nlb, blocks - 1); 634 } 635 636 void 637 nvme_scsi_io_done(struct nvme_softc *sc, struct nvme_ccb *ccb, 638 struct nvme_cqe *cqe) 639 { 640 struct scsi_xfer *xs = ccb->ccb_cookie; 641 bus_dmamap_t dmap = ccb->ccb_dmamap; 642 u_int16_t flags; 643 644 if (dmap->dm_nsegs > 2) { 645 bus_dmamap_sync(sc->sc_dmat, 646 NVME_DMA_MAP(sc->sc_ccb_prpls), 647 ccb->ccb_prpl_off, 648 sizeof(*ccb->ccb_prpl) * dmap->dm_nsegs - 1, 649 BUS_DMASYNC_POSTWRITE); 650 } 651 652 bus_dmamap_sync(sc->sc_dmat, dmap, 0, dmap->dm_mapsize, 653 ISSET(xs->flags, SCSI_DATA_IN) ? 654 BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE); 655 656 bus_dmamap_unload(sc->sc_dmat, dmap); 657 658 flags = lemtoh16(&cqe->flags); 659 660 xs->error = (NVME_CQE_SC(flags) == NVME_CQE_SC_SUCCESS) ? 661 XS_NOERROR : XS_DRIVER_STUFFUP; 662 xs->status = SCSI_OK; 663 xs->resid = 0; 664 scsi_done(xs); 665 } 666 667 void 668 nvme_scsi_sync(struct scsi_xfer *xs) 669 { 670 struct scsi_link *link = xs->sc_link; 671 struct nvme_softc *sc = link->adapter_softc; 672 struct nvme_ccb *ccb = xs->io; 673 674 ccb->ccb_done = nvme_scsi_sync_done; 675 ccb->ccb_cookie = xs; 676 677 if (ISSET(xs->flags, SCSI_POLL)) { 678 nvme_poll(sc, sc->sc_q, ccb, nvme_scsi_sync_fill); 679 return; 680 } 681 682 nvme_q_submit(sc, sc->sc_q, ccb, nvme_scsi_sync_fill); 683 } 684 685 void 686 nvme_scsi_sync_fill(struct nvme_softc *sc, struct nvme_ccb *ccb, void *slot) 687 { 688 struct nvme_sqe *sqe = slot; 689 struct scsi_xfer *xs = ccb->ccb_cookie; 690 struct scsi_link *link = xs->sc_link; 691 692 sqe->opcode = NVM_CMD_FLUSH; 693 htolem32(&sqe->nsid, link->target + 1); 694 } 695 696 void 697 nvme_scsi_sync_done(struct nvme_softc *sc, struct nvme_ccb *ccb, 698 struct nvme_cqe *cqe) 699 { 700 struct scsi_xfer *xs = ccb->ccb_cookie; 701 u_int16_t flags; 702 703 flags = lemtoh16(&cqe->flags); 704 705 xs->error = (NVME_CQE_SC(flags) == NVME_CQE_SC_SUCCESS) ? 706 XS_NOERROR : XS_DRIVER_STUFFUP; 707 xs->status = SCSI_OK; 708 xs->resid = 0; 709 scsi_done(xs); 710 } 711 712 void 713 nvme_scsi_inq(struct scsi_xfer *xs) 714 { 715 struct scsi_inquiry *inq = (struct scsi_inquiry *)xs->cmd; 716 717 if (!ISSET(inq->flags, SI_EVPD)) { 718 nvme_scsi_inquiry(xs); 719 return; 720 } 721 722 switch (inq->pagecode) { 723 default: 724 /* printf("%s: %d\n", __func__, inq->pagecode); */ 725 break; 726 } 727 728 xs->error = XS_DRIVER_STUFFUP; 729 scsi_done(xs); 730 } 731 732 void 733 nvme_scsi_inquiry(struct scsi_xfer *xs) 734 { 735 struct scsi_inquiry_data inq; 736 struct scsi_link *link = xs->sc_link; 737 struct nvme_softc *sc = link->adapter_softc; 738 struct nvm_identify_namespace *ns; 739 740 ns = sc->sc_namespaces[link->target].ident; 741 742 memset(&inq, 0, sizeof(inq)); 743 744 inq.device = T_DIRECT; 745 inq.version = 0x06; /* SPC-4 */ 746 inq.response_format = 2; 747 inq.additional_length = 32; 748 inq.flags |= SID_CmdQue; 749 memcpy(inq.vendor, "NVMe ", sizeof(inq.vendor)); 750 memcpy(inq.product, sc->sc_identify.mn, sizeof(inq.product)); 751 memcpy(inq.revision, sc->sc_identify.fr, sizeof(inq.revision)); 752 753 memcpy(xs->data, &inq, MIN(sizeof(inq), xs->datalen)); 754 755 xs->error = XS_NOERROR; 756 scsi_done(xs); 757 } 758 759 void 760 nvme_scsi_capacity16(struct scsi_xfer *xs) 761 { 762 struct scsi_read_cap_data_16 rcd; 763 struct scsi_link *link = xs->sc_link; 764 struct nvme_softc *sc = link->adapter_softc; 765 struct nvm_identify_namespace *ns; 766 struct nvm_namespace_format *f; 767 u_int64_t nsze; 768 u_int16_t tpe = READ_CAP_16_TPE; 769 770 ns = sc->sc_namespaces[link->target].ident; 771 772 if (xs->cmdlen != sizeof(struct scsi_read_capacity_16)) { 773 xs->error = XS_DRIVER_STUFFUP; 774 scsi_done(xs); 775 return; 776 } 777 778 nsze = lemtoh64(&ns->nsze); 779 f = &ns->lbaf[NVME_ID_NS_FLBAS(ns->flbas)]; 780 781 memset(&rcd, 0, sizeof(rcd)); 782 _lto8b(nsze, rcd.addr); 783 _lto4b(1 << f->lbads, rcd.length); 784 _lto2b(tpe, rcd.lowest_aligned); 785 786 memcpy(xs->data, &rcd, MIN(sizeof(rcd), xs->datalen)); 787 788 xs->error = XS_NOERROR; 789 scsi_done(xs); 790 } 791 792 void 793 nvme_scsi_capacity(struct scsi_xfer *xs) 794 { 795 struct scsi_read_cap_data rcd; 796 struct scsi_link *link = xs->sc_link; 797 struct nvme_softc *sc = link->adapter_softc; 798 struct nvm_identify_namespace *ns; 799 struct nvm_namespace_format *f; 800 u_int64_t nsze; 801 802 ns = sc->sc_namespaces[link->target].ident; 803 804 if (xs->cmdlen != sizeof(struct scsi_read_capacity)) { 805 xs->error = XS_DRIVER_STUFFUP; 806 scsi_done(xs); 807 return; 808 } 809 810 nsze = lemtoh64(&ns->nsze); 811 if (nsze > 0xffffffff) 812 nsze = 0xffffffff; 813 814 f = &ns->lbaf[NVME_ID_NS_FLBAS(ns->flbas)]; 815 816 memset(&rcd, 0, sizeof(rcd)); 817 _lto4b(nsze, rcd.addr); 818 _lto4b(1 << f->lbads, rcd.length); 819 820 memcpy(xs->data, &rcd, MIN(sizeof(rcd), xs->datalen)); 821 822 xs->error = XS_NOERROR; 823 scsi_done(xs); 824 } 825 826 void 827 nvme_scsi_free(struct scsi_link *link) 828 { 829 struct nvme_softc *sc = link->adapter_softc; 830 struct nvm_identify_namespace *identify; 831 832 identify = sc->sc_namespaces[link->target].ident; 833 sc->sc_namespaces[link->target].ident = NULL; 834 835 free(identify, M_DEVBUF, sizeof(*identify)); 836 } 837 838 void 839 nvme_q_submit(struct nvme_softc *sc, struct nvme_queue *q, struct nvme_ccb *ccb, 840 void (*fill)(struct nvme_softc *, struct nvme_ccb *, void *)) 841 { 842 struct nvme_sqe *sqe = NVME_DMA_KVA(q->q_sq_dmamem); 843 u_int32_t tail; 844 845 mtx_enter(&q->q_sq_mtx); 846 tail = q->q_sq_tail; 847 if (++q->q_sq_tail >= q->q_entries) 848 q->q_sq_tail = 0; 849 850 sqe += tail; 851 852 bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(q->q_sq_dmamem), 853 sizeof(*sqe) * tail, sizeof(*sqe), BUS_DMASYNC_POSTWRITE); 854 memset(sqe, 0, sizeof(*sqe)); 855 (*fill)(sc, ccb, sqe); 856 sqe->cid = ccb->ccb_id; 857 bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(q->q_sq_dmamem), 858 sizeof(*sqe) * tail, sizeof(*sqe), BUS_DMASYNC_PREWRITE); 859 860 nvme_write4(sc, q->q_sqtdbl, q->q_sq_tail); 861 mtx_leave(&q->q_sq_mtx); 862 } 863 864 struct nvme_poll_state { 865 struct nvme_sqe s; 866 struct nvme_cqe c; 867 }; 868 869 int 870 nvme_poll(struct nvme_softc *sc, struct nvme_queue *q, struct nvme_ccb *ccb, 871 void (*fill)(struct nvme_softc *, struct nvme_ccb *, void *)) 872 { 873 struct nvme_poll_state state; 874 void (*done)(struct nvme_softc *, struct nvme_ccb *, struct nvme_cqe *); 875 void *cookie; 876 u_int16_t flags; 877 878 memset(&state, 0, sizeof(state)); 879 (*fill)(sc, ccb, &state.s); 880 881 done = ccb->ccb_done; 882 cookie = ccb->ccb_cookie; 883 884 ccb->ccb_done = nvme_poll_done; 885 ccb->ccb_cookie = &state; 886 887 nvme_q_submit(sc, q, ccb, nvme_poll_fill); 888 while (!ISSET(state.c.flags, htole16(NVME_CQE_PHASE))) { 889 if (nvme_q_complete(sc, q) == 0) 890 delay(10); 891 892 /* XXX no timeout? */ 893 } 894 895 ccb->ccb_cookie = cookie; 896 done(sc, ccb, &state.c); 897 898 flags = lemtoh16(&state.c.flags); 899 900 return (flags & ~NVME_CQE_PHASE); 901 } 902 903 void 904 nvme_poll_fill(struct nvme_softc *sc, struct nvme_ccb *ccb, void *slot) 905 { 906 struct nvme_sqe *sqe = slot; 907 struct nvme_poll_state *state = ccb->ccb_cookie; 908 909 *sqe = state->s; 910 } 911 912 void 913 nvme_poll_done(struct nvme_softc *sc, struct nvme_ccb *ccb, 914 struct nvme_cqe *cqe) 915 { 916 struct nvme_poll_state *state = ccb->ccb_cookie; 917 918 SET(cqe->flags, htole16(NVME_CQE_PHASE)); 919 state->c = *cqe; 920 } 921 922 void 923 nvme_sqe_fill(struct nvme_softc *sc, struct nvme_ccb *ccb, void *slot) 924 { 925 struct nvme_sqe *src = ccb->ccb_cookie; 926 struct nvme_sqe *dst = slot; 927 928 *dst = *src; 929 } 930 931 void 932 nvme_empty_done(struct nvme_softc *sc, struct nvme_ccb *ccb, 933 struct nvme_cqe *cqe) 934 { 935 } 936 937 int 938 nvme_q_complete(struct nvme_softc *sc, struct nvme_queue *q) 939 { 940 struct nvme_ccb *ccb; 941 struct nvme_cqe *ring = NVME_DMA_KVA(q->q_cq_dmamem), *cqe; 942 u_int32_t head; 943 u_int16_t flags; 944 int rv = 0; 945 946 if (!mtx_enter_try(&q->q_cq_mtx)) 947 return (-1); 948 949 head = q->q_cq_head; 950 951 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_POSTREAD); 952 for (;;) { 953 cqe = &ring[head]; 954 flags = lemtoh16(&cqe->flags); 955 if ((flags & NVME_CQE_PHASE) != q->q_cq_phase) 956 break; 957 958 ccb = &sc->sc_ccbs[cqe->cid]; 959 ccb->ccb_done(sc, ccb, cqe); 960 961 if (++head >= q->q_entries) { 962 head = 0; 963 q->q_cq_phase ^= NVME_CQE_PHASE; 964 } 965 966 rv = 1; 967 } 968 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_PREREAD); 969 970 if (rv) 971 nvme_write4(sc, q->q_cqhdbl, q->q_cq_head = head); 972 mtx_leave(&q->q_cq_mtx); 973 974 return (rv); 975 } 976 977 int 978 nvme_identify(struct nvme_softc *sc, u_int mps) 979 { 980 char sn[41], mn[81], fr[17]; 981 struct nvm_identify_controller *identify; 982 struct nvme_dmamem *mem; 983 struct nvme_ccb *ccb; 984 u_int mdts; 985 int rv = 1; 986 987 ccb = nvme_ccb_get(sc); 988 if (ccb == NULL) 989 panic("nvme_identify: nvme_ccb_get returned NULL"); 990 991 mem = nvme_dmamem_alloc(sc, sizeof(*identify)); 992 if (mem == NULL) 993 return (1); 994 995 ccb->ccb_done = nvme_empty_done; 996 ccb->ccb_cookie = mem; 997 998 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_PREREAD); 999 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_fill_identify); 1000 nvme_dmamem_sync(sc, mem, BUS_DMASYNC_POSTREAD); 1001 1002 nvme_ccb_put(sc, ccb); 1003 1004 if (rv != 0) 1005 goto done; 1006 1007 identify = NVME_DMA_KVA(mem); 1008 1009 scsi_strvis(sn, identify->sn, sizeof(identify->sn)); 1010 scsi_strvis(mn, identify->mn, sizeof(identify->mn)); 1011 scsi_strvis(fr, identify->fr, sizeof(identify->fr)); 1012 1013 printf("%s: %s, firmware %s, serial %s\n", DEVNAME(sc), mn, fr, sn); 1014 1015 if (identify->mdts > 0) { 1016 mdts = (1 << identify->mdts) * (1 << mps); 1017 if (mdts < sc->sc_mdts) 1018 sc->sc_mdts = mdts; 1019 } 1020 1021 sc->sc_nn = lemtoh32(&identify->nn); 1022 1023 memcpy(&sc->sc_identify, identify, sizeof(sc->sc_identify)); 1024 1025 done: 1026 nvme_dmamem_free(sc, mem); 1027 1028 return (rv); 1029 } 1030 1031 int 1032 nvme_q_create(struct nvme_softc *sc, struct nvme_queue *q) 1033 { 1034 struct nvme_sqe_q sqe; 1035 struct nvme_ccb *ccb; 1036 int rv; 1037 1038 ccb = scsi_io_get(&sc->sc_iopool, 0); 1039 KASSERT(ccb != NULL); 1040 1041 ccb->ccb_done = nvme_empty_done; 1042 ccb->ccb_cookie = &sqe; 1043 1044 memset(&sqe, 0, sizeof(sqe)); 1045 sqe.opcode = NVM_ADMIN_ADD_IOCQ; 1046 htolem64(&sqe.prp1, NVME_DMA_DVA(q->q_cq_dmamem)); 1047 htolem16(&sqe.qsize, q->q_entries - 1); 1048 htolem16(&sqe.qid, q->q_id); 1049 sqe.qflags = NVM_SQE_CQ_IEN | NVM_SQE_Q_PC; 1050 1051 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill); 1052 if (rv != 0) 1053 goto fail; 1054 1055 ccb->ccb_done = nvme_empty_done; 1056 ccb->ccb_cookie = &sqe; 1057 1058 memset(&sqe, 0, sizeof(sqe)); 1059 sqe.opcode = NVM_ADMIN_ADD_IOSQ; 1060 htolem64(&sqe.prp1, NVME_DMA_DVA(q->q_sq_dmamem)); 1061 htolem16(&sqe.qsize, q->q_entries - 1); 1062 htolem16(&sqe.qid, q->q_id); 1063 htolem16(&sqe.cqid, q->q_id); 1064 sqe.qflags = NVM_SQE_Q_PC; 1065 1066 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill); 1067 if (rv != 0) 1068 goto fail; 1069 1070 fail: 1071 scsi_io_put(&sc->sc_iopool, ccb); 1072 return (rv); 1073 } 1074 1075 int 1076 nvme_q_delete(struct nvme_softc *sc, struct nvme_queue *q) 1077 { 1078 struct nvme_sqe_q sqe; 1079 struct nvme_ccb *ccb; 1080 int rv; 1081 1082 ccb = scsi_io_get(&sc->sc_iopool, 0); 1083 KASSERT(ccb != NULL); 1084 1085 ccb->ccb_done = nvme_empty_done; 1086 ccb->ccb_cookie = &sqe; 1087 1088 memset(&sqe, 0, sizeof(sqe)); 1089 sqe.opcode = NVM_ADMIN_DEL_IOSQ; 1090 htolem16(&sqe.qid, q->q_id); 1091 1092 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill); 1093 if (rv != 0) 1094 goto fail; 1095 1096 ccb->ccb_done = nvme_empty_done; 1097 ccb->ccb_cookie = &sqe; 1098 1099 memset(&sqe, 0, sizeof(sqe)); 1100 sqe.opcode = NVM_ADMIN_DEL_IOCQ; 1101 htolem64(&sqe.prp1, NVME_DMA_DVA(q->q_sq_dmamem)); 1102 htolem16(&sqe.qid, q->q_id); 1103 1104 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill); 1105 if (rv != 0) 1106 goto fail; 1107 1108 fail: 1109 scsi_io_put(&sc->sc_iopool, ccb); 1110 return (rv); 1111 1112 } 1113 1114 void 1115 nvme_fill_identify(struct nvme_softc *sc, struct nvme_ccb *ccb, void *slot) 1116 { 1117 struct nvme_sqe *sqe = slot; 1118 struct nvme_dmamem *mem = ccb->ccb_cookie; 1119 1120 sqe->opcode = NVM_ADMIN_IDENTIFY; 1121 htolem64(&sqe->entry.prp[0], NVME_DMA_DVA(mem)); 1122 htolem32(&sqe->cdw10, 1); 1123 } 1124 1125 int 1126 nvme_ccbs_alloc(struct nvme_softc *sc, u_int nccbs) 1127 { 1128 struct nvme_ccb *ccb; 1129 bus_addr_t off; 1130 u_int64_t *prpl; 1131 u_int i; 1132 1133 sc->sc_ccbs = mallocarray(nccbs, sizeof(*ccb), M_DEVBUF, 1134 M_WAITOK | M_CANFAIL); 1135 if (sc->sc_ccbs == NULL) 1136 return (1); 1137 1138 sc->sc_ccb_prpls = nvme_dmamem_alloc(sc, 1139 sizeof(*prpl) * sc->sc_max_sgl * nccbs); 1140 1141 prpl = NVME_DMA_KVA(sc->sc_ccb_prpls); 1142 off = 0; 1143 1144 for (i = 0; i < nccbs; i++) { 1145 ccb = &sc->sc_ccbs[i]; 1146 1147 if (bus_dmamap_create(sc->sc_dmat, sc->sc_mdts, 1148 sc->sc_max_sgl + 1 /* we get a free prp in the sqe */, 1149 sc->sc_mps, sc->sc_mps, BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, 1150 &ccb->ccb_dmamap) != 0) 1151 goto free_maps; 1152 1153 ccb->ccb_id = i; 1154 ccb->ccb_prpl = prpl; 1155 ccb->ccb_prpl_off = off; 1156 ccb->ccb_prpl_dva = NVME_DMA_DVA(sc->sc_ccb_prpls) + off; 1157 1158 SIMPLEQ_INSERT_TAIL(&sc->sc_ccb_list, ccb, ccb_entry); 1159 1160 prpl += sc->sc_max_sgl; 1161 off += sizeof(*prpl) * sc->sc_max_sgl; 1162 } 1163 1164 return (0); 1165 1166 free_maps: 1167 nvme_ccbs_free(sc); 1168 return (1); 1169 } 1170 1171 void * 1172 nvme_ccb_get(void *cookie) 1173 { 1174 struct nvme_softc *sc = cookie; 1175 struct nvme_ccb *ccb; 1176 1177 mtx_enter(&sc->sc_ccb_mtx); 1178 ccb = SIMPLEQ_FIRST(&sc->sc_ccb_list); 1179 if (ccb != NULL) 1180 SIMPLEQ_REMOVE_HEAD(&sc->sc_ccb_list, ccb_entry); 1181 mtx_leave(&sc->sc_ccb_mtx); 1182 1183 return (ccb); 1184 } 1185 1186 void 1187 nvme_ccb_put(void *cookie, void *io) 1188 { 1189 struct nvme_softc *sc = cookie; 1190 struct nvme_ccb *ccb = io; 1191 1192 mtx_enter(&sc->sc_ccb_mtx); 1193 SIMPLEQ_INSERT_HEAD(&sc->sc_ccb_list, ccb, ccb_entry); 1194 mtx_leave(&sc->sc_ccb_mtx); 1195 } 1196 1197 void 1198 nvme_ccbs_free(struct nvme_softc *sc) 1199 { 1200 struct nvme_ccb *ccb; 1201 1202 while ((ccb = SIMPLEQ_FIRST(&sc->sc_ccb_list)) != NULL) { 1203 SIMPLEQ_REMOVE_HEAD(&sc->sc_ccb_list, ccb_entry); 1204 bus_dmamap_destroy(sc->sc_dmat, ccb->ccb_dmamap); 1205 } 1206 1207 nvme_dmamem_free(sc, sc->sc_ccb_prpls); 1208 free(sc->sc_ccbs, M_DEVBUF, 0); 1209 } 1210 1211 struct nvme_queue * 1212 nvme_q_alloc(struct nvme_softc *sc, u_int16_t id, u_int entries, u_int dstrd) 1213 { 1214 struct nvme_queue *q; 1215 1216 q = malloc(sizeof(*q), M_DEVBUF, M_WAITOK | M_CANFAIL); 1217 if (q == NULL) 1218 return (NULL); 1219 1220 q->q_sq_dmamem = nvme_dmamem_alloc(sc, 1221 sizeof(struct nvme_sqe) * entries); 1222 if (q->q_sq_dmamem == NULL) 1223 goto free; 1224 1225 q->q_cq_dmamem = nvme_dmamem_alloc(sc, 1226 sizeof(struct nvme_cqe) * entries); 1227 if (q->q_cq_dmamem == NULL) 1228 goto free_sq; 1229 1230 memset(NVME_DMA_KVA(q->q_sq_dmamem), 0, NVME_DMA_LEN(q->q_sq_dmamem)); 1231 memset(NVME_DMA_KVA(q->q_cq_dmamem), 0, NVME_DMA_LEN(q->q_cq_dmamem)); 1232 1233 mtx_init(&q->q_sq_mtx, IPL_BIO); 1234 mtx_init(&q->q_cq_mtx, IPL_BIO); 1235 q->q_sqtdbl = NVME_SQTDBL(id, dstrd); 1236 q->q_cqhdbl = NVME_CQHDBL(id, dstrd); 1237 q->q_id = id; 1238 q->q_entries = entries; 1239 q->q_sq_tail = 0; 1240 q->q_cq_head = 0; 1241 q->q_cq_phase = NVME_CQE_PHASE; 1242 1243 nvme_dmamem_sync(sc, q->q_sq_dmamem, BUS_DMASYNC_PREWRITE); 1244 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_PREREAD); 1245 1246 return (q); 1247 1248 free_sq: 1249 nvme_dmamem_free(sc, q->q_sq_dmamem); 1250 free: 1251 free(q, M_DEVBUF, 0); 1252 1253 return (NULL); 1254 } 1255 1256 void 1257 nvme_q_free(struct nvme_softc *sc, struct nvme_queue *q) 1258 { 1259 nvme_dmamem_sync(sc, q->q_cq_dmamem, BUS_DMASYNC_POSTREAD); 1260 nvme_dmamem_sync(sc, q->q_sq_dmamem, BUS_DMASYNC_POSTWRITE); 1261 nvme_dmamem_free(sc, q->q_cq_dmamem); 1262 nvme_dmamem_free(sc, q->q_sq_dmamem); 1263 free(q, M_DEVBUF, 0); 1264 } 1265 1266 int 1267 nvme_intr(void *xsc) 1268 { 1269 struct nvme_softc *sc = xsc; 1270 int rv = 0; 1271 1272 if (nvme_q_complete(sc, sc->sc_q)) 1273 rv = 1; 1274 if (nvme_q_complete(sc, sc->sc_admin_q)) 1275 rv = 1; 1276 1277 return (rv); 1278 } 1279 1280 struct nvme_dmamem * 1281 nvme_dmamem_alloc(struct nvme_softc *sc, size_t size) 1282 { 1283 struct nvme_dmamem *ndm; 1284 int nsegs; 1285 1286 ndm = malloc(sizeof(*ndm), M_DEVBUF, M_WAITOK | M_ZERO); 1287 if (ndm == NULL) 1288 return (NULL); 1289 1290 ndm->ndm_size = size; 1291 1292 if (bus_dmamap_create(sc->sc_dmat, size, 1, size, 0, 1293 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &ndm->ndm_map) != 0) 1294 goto ndmfree; 1295 1296 if (bus_dmamem_alloc(sc->sc_dmat, size, sc->sc_mps, 0, &ndm->ndm_seg, 1297 1, &nsegs, BUS_DMA_WAITOK | BUS_DMA_ZERO) != 0) 1298 goto destroy; 1299 1300 if (bus_dmamem_map(sc->sc_dmat, &ndm->ndm_seg, nsegs, size, 1301 &ndm->ndm_kva, BUS_DMA_WAITOK) != 0) 1302 goto free; 1303 1304 if (bus_dmamap_load(sc->sc_dmat, ndm->ndm_map, ndm->ndm_kva, size, 1305 NULL, BUS_DMA_WAITOK) != 0) 1306 goto unmap; 1307 1308 return (ndm); 1309 1310 unmap: 1311 bus_dmamem_unmap(sc->sc_dmat, ndm->ndm_kva, size); 1312 free: 1313 bus_dmamem_free(sc->sc_dmat, &ndm->ndm_seg, 1); 1314 destroy: 1315 bus_dmamap_destroy(sc->sc_dmat, ndm->ndm_map); 1316 ndmfree: 1317 free(ndm, M_DEVBUF, 0); 1318 1319 return (NULL); 1320 } 1321 1322 void 1323 nvme_dmamem_sync(struct nvme_softc *sc, struct nvme_dmamem *mem, int ops) 1324 { 1325 bus_dmamap_sync(sc->sc_dmat, NVME_DMA_MAP(mem), 1326 0, NVME_DMA_LEN(mem), ops); 1327 } 1328 1329 void 1330 nvme_dmamem_free(struct nvme_softc *sc, struct nvme_dmamem *ndm) 1331 { 1332 bus_dmamap_unload(sc->sc_dmat, ndm->ndm_map); 1333 bus_dmamem_unmap(sc->sc_dmat, ndm->ndm_kva, ndm->ndm_size); 1334 bus_dmamem_free(sc->sc_dmat, &ndm->ndm_seg, 1); 1335 bus_dmamap_destroy(sc->sc_dmat, ndm->ndm_map); 1336 free(ndm, M_DEVBUF, 0); 1337 } 1338 1339