Lines Matching refs:ccb
703 struct nvme_ccb *ccb; in nvme_ns_identify() local
715 ccb = nvme_ccb_get(sc->sc_admin_q, false); in nvme_ns_identify()
716 KASSERT(ccb != NULL); /* it's a bug if we don't have spare ccb here */ in nvme_ns_identify()
720 nvme_ccb_put(sc->sc_admin_q, ccb); in nvme_ns_identify()
730 ccb->ccb_done = nvme_empty_done; in nvme_ns_identify()
731 ccb->ccb_cookie = &sqe; in nvme_ns_identify()
734 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_IDENT); in nvme_ns_identify()
737 nvme_ccb_put(sc->sc_admin_q, ccb); in nvme_ns_identify()
766 struct nvme_ccb *ccb; in nvme_ns_dobio() local
770 ccb = nvme_ccb_get_bio(sc, bp, &q); in nvme_ns_dobio()
771 if (ccb == NULL) in nvme_ns_dobio()
774 ccb->ccb_done = nvme_ns_io_done; in nvme_ns_dobio()
775 ccb->ccb_cookie = cookie; in nvme_ns_dobio()
778 ccb->nnc_nsid = nsid; in nvme_ns_dobio()
779 ccb->nnc_flags = flags; in nvme_ns_dobio()
780 ccb->nnc_buf = bp; in nvme_ns_dobio()
781 ccb->nnc_datasize = datasize; in nvme_ns_dobio()
782 ccb->nnc_secsize = secsize; in nvme_ns_dobio()
783 ccb->nnc_blkno = blkno; in nvme_ns_dobio()
784 ccb->nnc_done = nnc_done; in nvme_ns_dobio()
786 dmap = ccb->ccb_dmamap; in nvme_ns_dobio()
794 nvme_ccb_put(q, ccb); in nvme_ns_dobio()
804 htolem64(&ccb->ccb_prpl[i - 1], in nvme_ns_dobio()
809 ccb->ccb_prpl_off, in nvme_ns_dobio()
810 sizeof(*ccb->ccb_prpl) * (dmap->dm_nsegs - 1), in nvme_ns_dobio()
815 if (nvme_poll(sc, q, ccb, nvme_ns_io_fill, NVME_TIMO_PT) != 0) in nvme_ns_dobio()
820 nvme_q_submit(sc, q, ccb, nvme_ns_io_fill); in nvme_ns_dobio()
825 nvme_ns_io_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) in nvme_ns_io_fill() argument
828 bus_dmamap_t dmap = ccb->ccb_dmamap; in nvme_ns_io_fill()
830 sqe->opcode = ISSET(ccb->nnc_flags, NVME_NS_CTX_F_READ) ? in nvme_ns_io_fill()
832 htolem32(&sqe->nsid, ccb->nnc_nsid); in nvme_ns_io_fill()
843 htolem64(&sqe->entry.prp[1], ccb->ccb_prpl_dva); in nvme_ns_io_fill()
847 htolem64(&sqe->slba, ccb->nnc_blkno); in nvme_ns_io_fill()
849 if (ISSET(ccb->nnc_flags, NVME_NS_CTX_F_FUA)) in nvme_ns_io_fill()
853 KASSERT((ccb->nnc_datasize % ccb->nnc_secsize) == 0); in nvme_ns_io_fill()
854 htolem16(&sqe->nlb, (ccb->nnc_datasize / ccb->nnc_secsize) - 1); in nvme_ns_io_fill()
858 nvme_ns_io_done(struct nvme_queue *q, struct nvme_ccb *ccb, in nvme_ns_io_done() argument
862 bus_dmamap_t dmap = ccb->ccb_dmamap; in nvme_ns_io_done()
863 void *nnc_cookie = ccb->ccb_cookie; in nvme_ns_io_done()
864 nvme_nnc_done nnc_done = ccb->nnc_done; in nvme_ns_io_done()
865 struct buf *bp = ccb->nnc_buf; in nvme_ns_io_done()
870 ccb->ccb_prpl_off, in nvme_ns_io_done()
871 sizeof(*ccb->ccb_prpl) * (dmap->dm_nsegs - 1), in nvme_ns_io_done()
876 ISSET(ccb->nnc_flags, NVME_NS_CTX_F_READ) ? in nvme_ns_io_done()
880 nvme_ccb_put(q, ccb); in nvme_ns_io_done()
908 struct nvme_ccb *ccb; in nvme_ns_sync() local
916 ccb = nvme_ccb_get(q, true); in nvme_ns_sync()
917 KASSERT(ccb != NULL); in nvme_ns_sync()
919 ccb->ccb_done = nvme_ns_sync_done; in nvme_ns_sync()
920 ccb->ccb_cookie = &result; in nvme_ns_sync()
923 ccb->nnc_nsid = nsid; in nvme_ns_sync()
924 ccb->nnc_flags = flags; in nvme_ns_sync()
925 ccb->nnc_done = NULL; in nvme_ns_sync()
928 if (nvme_poll(sc, q, ccb, nvme_ns_sync_fill, NVME_TIMO_SY) != 0) in nvme_ns_sync()
933 nvme_q_submit(sc, q, ccb, nvme_ns_sync_fill); in nvme_ns_sync()
943 nvme_ns_sync_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) in nvme_ns_sync_fill() argument
948 htolem32(&sqe->nsid, ccb->nnc_nsid); in nvme_ns_sync_fill()
952 nvme_ns_sync_done(struct nvme_queue *q, struct nvme_ccb *ccb, in nvme_ns_sync_done() argument
955 int *result = ccb->ccb_cookie; in nvme_ns_sync_done()
963 nvme_ccb_put(q, ccb); in nvme_ns_sync_done()
980 struct nvme_ccb *ccb; in nvme_admin_getcache() local
990 ccb = nvme_ccb_get(q, true); in nvme_admin_getcache()
991 KASSERT(ccb != NULL); in nvme_admin_getcache()
993 ccb->ccb_done = nvme_getcache_done; in nvme_admin_getcache()
994 ccb->ccb_cookie = &result; in nvme_admin_getcache()
997 ccb->nnc_flags = 0; in nvme_admin_getcache()
998 ccb->nnc_done = NULL; in nvme_admin_getcache()
1000 nvme_q_submit(sc, q, ccb, nvme_getcache_fill); in nvme_admin_getcache()
1016 nvme_getcache_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) in nvme_getcache_fill() argument
1026 nvme_getcache_done(struct nvme_queue *q, struct nvme_ccb *ccb, in nvme_getcache_done() argument
1029 int *addr = ccb->ccb_cookie; in nvme_getcache_done()
1065 nvme_ccb_put(q, ccb); in nvme_getcache_done()
1082 nvme_setcache_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) in nvme_setcache_fill() argument
1085 struct nvme_setcache_state *st = ccb->ccb_cookie; in nvme_setcache_fill()
1094 nvme_setcache_done(struct nvme_queue *q, struct nvme_ccb *ccb, in nvme_setcache_done() argument
1097 struct nvme_setcache_state *st = ccb->ccb_cookie; in nvme_setcache_done()
1106 nvme_ccb_put(q, ccb); in nvme_setcache_done()
1115 struct nvme_ccb *ccb; in nvme_admin_setcache() local
1130 ccb = nvme_ccb_get(q, true); in nvme_admin_setcache()
1131 KASSERT(ccb != NULL); in nvme_admin_setcache()
1136 ccb->ccb_done = nvme_setcache_done; in nvme_admin_setcache()
1137 ccb->ccb_cookie = &st; in nvme_admin_setcache()
1140 ccb->nnc_flags = 0; in nvme_admin_setcache()
1141 ccb->nnc_done = NULL; in nvme_admin_setcache()
1143 nvme_q_submit(sc, q, ccb, nvme_setcache_fill); in nvme_admin_setcache()
1178 nvme_pt_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) in nvme_pt_fill() argument
1182 struct nvme_pt_state *state = ccb->ccb_cookie; in nvme_pt_fill()
1184 bus_dmamap_t dmap = ccb->ccb_dmamap; in nvme_pt_fill()
1200 htolem64(&ccb->ccb_prpl[i - 1], in nvme_pt_fill()
1205 ccb->ccb_prpl_off, in nvme_pt_fill()
1206 sizeof(*ccb->ccb_prpl) * (dmap->dm_nsegs - 1), in nvme_pt_fill()
1208 htolem64(&sqe->entry.prp[1], ccb->ccb_prpl_dva); in nvme_pt_fill()
1222 nvme_pt_done(struct nvme_queue *q, struct nvme_ccb *ccb, struct nvme_cqe *cqe) in nvme_pt_done() argument
1225 struct nvme_pt_state *state = ccb->ccb_cookie; in nvme_pt_done()
1227 bus_dmamap_t dmap = ccb->ccb_dmamap; in nvme_pt_done()
1233 ccb->ccb_prpl_off, in nvme_pt_done()
1234 sizeof(*ccb->ccb_prpl) * (dmap->dm_nsegs - 1), in nvme_pt_done()
1248 nvme_ccb_put(q, ccb); in nvme_pt_done()
1264 struct nvme_ccb *ccb; in nvme_command_passthrough() local
1275 ccb = nvme_ccb_get(q, true); in nvme_command_passthrough()
1276 KASSERT(ccb != NULL); in nvme_command_passthrough()
1286 error = bus_dmamap_load(sc->sc_dmat, ccb->ccb_dmamap, buf, in nvme_command_passthrough()
1292 bus_dmamap_sync(sc->sc_dmat, ccb->ccb_dmamap, in nvme_command_passthrough()
1293 0, ccb->ccb_dmamap->dm_mapsize, in nvme_command_passthrough()
1301 ccb->ccb_done = nvme_pt_done; in nvme_command_passthrough()
1302 ccb->ccb_cookie = &state; in nvme_command_passthrough()
1306 nvme_q_submit(sc, q, ccb, nvme_pt_fill); in nvme_command_passthrough()
1326 struct nvme_queue *q, struct nvme_ccb *ccb) in nvme_op_sq_enter() argument
1330 return nvme_op_sq_enter_locked(sc, q, ccb); in nvme_op_sq_enter()
1335 struct nvme_queue *q, struct nvme_ccb *ccb) in nvme_op_sq_enter_locked() argument
1342 struct nvme_queue *q, struct nvme_ccb *ccb) in nvme_op_sq_leave_locked() argument
1355 struct nvme_queue *q, struct nvme_ccb *ccb) in nvme_op_sq_leave() argument
1357 nvme_op_sq_leave_locked(sc, q, ccb); in nvme_op_sq_leave()
1363 nvme_q_submit(struct nvme_softc *sc, struct nvme_queue *q, struct nvme_ccb *ccb, in nvme_q_submit() argument
1369 tail = sc->sc_ops->op_sq_enter(sc, q, ccb); in nvme_q_submit()
1376 (*fill)(q, ccb, sqe); in nvme_q_submit()
1377 htolem16(&sqe->cid, ccb->ccb_id); in nvme_q_submit()
1381 sc->sc_ops->op_sq_leave(sc, q, ccb); in nvme_q_submit()
1392 nvme_poll(struct nvme_softc *sc, struct nvme_queue *q, struct nvme_ccb *ccb, in nvme_poll() argument
1402 (*fill)(q, ccb, &state.s); in nvme_poll()
1404 state.done = ccb->ccb_done; in nvme_poll()
1405 state.cookie = ccb->ccb_cookie; in nvme_poll()
1407 ccb->ccb_done = nvme_poll_done; in nvme_poll()
1408 ccb->ccb_cookie = &state; in nvme_poll()
1410 nvme_q_submit(sc, q, ccb, nvme_poll_fill); in nvme_poll()
1435 ccb->ccb_done(q, ccb, &cqe); in nvme_poll()
1442 nvme_poll_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) in nvme_poll_fill() argument
1445 struct nvme_poll_state *state = ccb->ccb_cookie; in nvme_poll_fill()
1451 nvme_poll_done(struct nvme_queue *q, struct nvme_ccb *ccb, in nvme_poll_done() argument
1454 struct nvme_poll_state *state = ccb->ccb_cookie; in nvme_poll_done()
1459 ccb->ccb_cookie = state->cookie; in nvme_poll_done()
1460 state->done(q, ccb, &state->c); in nvme_poll_done()
1464 nvme_sqe_fill(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) in nvme_sqe_fill() argument
1466 struct nvme_sqe *src = ccb->ccb_cookie; in nvme_sqe_fill()
1473 nvme_empty_done(struct nvme_queue *q, struct nvme_ccb *ccb, in nvme_empty_done() argument
1480 struct nvme_queue *q, struct nvme_ccb *ccb) in nvme_op_cq_done() argument
1488 struct nvme_ccb *ccb; in nvme_q_complete() local
1510 ccb = &q->q_ccbs[lemtoh16(&cqe->cid)]; in nvme_q_complete()
1523 if ((intptr_t)ccb->ccb_cookie == NVME_CCB_FREE) { in nvme_q_complete()
1532 sc->sc_ops->op_cq_done(sc, q, ccb); in nvme_q_complete()
1541 ccb->ccb_done(q, ccb, cqe); in nvme_q_complete()
1580 struct nvme_ccb *ccb; in nvme_identify() local
1584 ccb = nvme_ccb_get(sc->sc_admin_q, false); in nvme_identify()
1585 KASSERT(ccb != NULL); /* it's a bug if we don't have spare ccb here */ in nvme_identify()
1591 ccb->ccb_done = nvme_empty_done; in nvme_identify()
1592 ccb->ccb_cookie = mem; in nvme_identify()
1595 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_fill_identify, in nvme_identify()
1599 nvme_ccb_put(sc->sc_admin_q, ccb); in nvme_identify()
1640 struct nvme_ccb *ccb; in nvme_q_create() local
1646 ccb = nvme_ccb_get(sc->sc_admin_q, false); in nvme_q_create()
1647 KASSERT(ccb != NULL); in nvme_q_create()
1649 ccb->ccb_done = nvme_empty_done; in nvme_q_create()
1650 ccb->ccb_cookie = &sqe; in nvme_q_create()
1661 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP); in nvme_q_create()
1665 ccb->ccb_done = nvme_empty_done; in nvme_q_create()
1666 ccb->ccb_cookie = &sqe; in nvme_q_create()
1676 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP); in nvme_q_create()
1680 nvme_ccb_put(sc->sc_admin_q, ccb); in nvme_q_create()
1687 nvme_ccb_put(sc->sc_admin_q, ccb); in nvme_q_create()
1695 struct nvme_ccb *ccb; in nvme_q_delete() local
1698 ccb = nvme_ccb_get(sc->sc_admin_q, false); in nvme_q_delete()
1699 KASSERT(ccb != NULL); in nvme_q_delete()
1701 ccb->ccb_done = nvme_empty_done; in nvme_q_delete()
1702 ccb->ccb_cookie = &sqe; in nvme_q_delete()
1708 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP); in nvme_q_delete()
1712 ccb->ccb_done = nvme_empty_done; in nvme_q_delete()
1713 ccb->ccb_cookie = &sqe; in nvme_q_delete()
1719 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_sqe_fill, NVME_TIMO_QOP); in nvme_q_delete()
1724 nvme_ccb_put(sc->sc_admin_q, ccb); in nvme_q_delete()
1735 nvme_fill_identify(struct nvme_queue *q, struct nvme_ccb *ccb, void *slot) in nvme_fill_identify() argument
1738 struct nvme_dmamem *mem = ccb->ccb_cookie; in nvme_fill_identify()
1751 struct nvme_ccb *ccb; in nvme_set_number_of_queues() local
1754 ccb = nvme_ccb_get(sc->sc_admin_q, false); in nvme_set_number_of_queues()
1755 KASSERT(ccb != NULL); /* it's a bug if we don't have spare ccb here */ in nvme_set_number_of_queues()
1766 ccb->ccb_done = nvme_pt_done; in nvme_set_number_of_queues()
1767 ccb->ccb_cookie = &state; in nvme_set_number_of_queues()
1769 rv = nvme_poll(sc, sc->sc_admin_q, ccb, nvme_pt_fill, NVME_TIMO_QOP); in nvme_set_number_of_queues()
1786 struct nvme_ccb *ccb; in nvme_ccbs_alloc() local
1796 q->q_ccbs = kmem_alloc(sizeof(*ccb) * nccbs, KM_SLEEP); in nvme_ccbs_alloc()
1806 ccb = &q->q_ccbs[i]; in nvme_ccbs_alloc()
1811 &ccb->ccb_dmamap) != 0) in nvme_ccbs_alloc()
1814 ccb->ccb_id = i; in nvme_ccbs_alloc()
1815 ccb->ccb_prpl = prpl; in nvme_ccbs_alloc()
1816 ccb->ccb_prpl_off = off; in nvme_ccbs_alloc()
1817 ccb->ccb_prpl_dva = NVME_DMA_DVA(q->q_ccb_prpls) + off; in nvme_ccbs_alloc()
1819 SIMPLEQ_INSERT_TAIL(&q->q_ccb_list, ccb, ccb_entry); in nvme_ccbs_alloc()
1835 struct nvme_ccb *ccb = NULL; in nvme_ccb_get() local
1839 ccb = SIMPLEQ_FIRST(&q->q_ccb_list); in nvme_ccb_get()
1840 if (ccb != NULL) { in nvme_ccb_get()
1843 ccb->ccb_cookie = NULL; in nvme_ccb_get()
1854 return ccb; in nvme_ccb_get()
1870 struct nvme_ccb *ccb; in nvme_ccb_get_bio() local
1873 ccb = SIMPLEQ_FIRST(&q->q_ccb_list); in nvme_ccb_get_bio()
1874 if (ccb != NULL) { in nvme_ccb_get_bio()
1877 ccb->ccb_cookie = NULL; in nvme_ccb_get_bio()
1882 if (ccb != NULL) { in nvme_ccb_get_bio()
1884 return ccb; in nvme_ccb_get_bio()
1892 nvme_ccb_put(struct nvme_queue *q, struct nvme_ccb *ccb) in nvme_ccb_put() argument
1897 ccb->ccb_cookie = (void *)NVME_CCB_FREE; in nvme_ccb_put()
1899 SIMPLEQ_INSERT_HEAD(&q->q_ccb_list, ccb, ccb_entry); in nvme_ccb_put()
1914 struct nvme_ccb *ccb; in nvme_ccbs_free() local
1917 while ((ccb = SIMPLEQ_FIRST(&q->q_ccb_list)) != NULL) { in nvme_ccbs_free()
1924 bus_dmamap_destroy(sc->sc_dmat, ccb->ccb_dmamap); in nvme_ccbs_free()
1930 kmem_free(q->q_ccbs, sizeof(*ccb) * q->q_nccbs); in nvme_ccbs_free()