Lines Matching defs:req

58 int nvmf_passthru_admin_cmd_for_ctrlr(struct spdk_nvmf_request *req, struct spdk_nvmf_ctrlr *ctrlr);
59 static int nvmf_passthru_admin_cmd(struct spdk_nvmf_request *req);
240 struct spdk_nvmf_request *req = ctx;
241 struct spdk_nvmf_qpair *qpair = req->qpair;
243 struct spdk_nvmf_fabric_connect_rsp *rsp = &req->rsp->connect_rsp;
255 spdk_nvmf_request_complete(req);
271 spdk_nvmf_request_complete(req);
277 struct spdk_nvmf_request *req)
279 struct spdk_nvmf_fabric_connect_rsp *rsp = &req->rsp->connect_rsp;
287 spdk_nvmf_request_complete(req);
301 spdk_nvmf_request_complete(req);
305 qpair->connect_req = req;
308 req->poller = SPDK_POLLER_REGISTER(_retry_qid_check, qpair,
322 spdk_thread_send_msg(qpair->group->thread, nvmf_ctrlr_send_connect_rsp, req);
329 struct spdk_nvmf_request *req = qpair->connect_req;
330 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr;
332 spdk_poller_unregister(&req->poller);
334 nvmf_ctrlr_add_qpair(qpair, ctrlr, req);
341 struct spdk_nvmf_request *req = ctx;
342 struct spdk_nvmf_qpair *qpair = req->qpair;
348 nvmf_ctrlr_add_qpair(qpair, ctrlr, req);
354 struct spdk_nvmf_request *req = ctx;
355 struct spdk_nvmf_qpair *qpair = req->qpair;
356 struct spdk_nvmf_fabric_connect_rsp *rsp = &req->rsp->connect_rsp;
365 spdk_nvmf_request_complete(req);
369 spdk_thread_send_msg(ctrlr->thread, _nvmf_ctrlr_add_admin_qpair, req);
436 struct spdk_nvmf_request *req,
441 struct spdk_nvmf_transport *transport = req->qpair->transport;
458 spdk_thread_get_id(req->qpair->group->thread));
463 ctrlr->thread = req->qpair->group->thread;
582 if (spdk_nvmf_qpair_get_listen_trid(req->qpair, &listen_trid) != 0) {
594 nvmf_qpair_set_ctrlr(req->qpair, ctrlr);
595 spdk_thread_send_msg(subsystem->thread, _nvmf_subsystem_add_ctrlr, req);
654 struct spdk_nvmf_request *req = ctx;
655 struct spdk_nvmf_fabric_connect_rsp *rsp = &req->rsp->connect_rsp;
656 struct spdk_nvmf_qpair *qpair = req->qpair;
663 SPDK_DTRACE_PROBE4_TICKS(nvmf_ctrlr_add_io_qpair, ctrlr, req->qpair, req->qpair->qid,
729 nvmf_ctrlr_add_qpair(qpair, ctrlr, req);
732 spdk_nvmf_request_complete(req);
738 struct spdk_nvmf_request *req = ctx;
739 struct spdk_nvmf_fabric_connect_rsp *rsp = &req->rsp->connect_rsp;
742 struct spdk_nvmf_qpair *qpair = req->qpair;
752 assert(req->iovcnt == 1);
754 data = req->iov[0].iov_base;
766 spdk_nvmf_request_complete(req);
774 spdk_nvmf_request_complete(req);
780 if (spdk_nvmf_qpair_get_listen_trid(req->qpair, &listen_trid) != 0) {
783 spdk_nvmf_request_complete(req);
791 spdk_nvmf_request_complete(req);
811 spdk_nvmf_request_complete(req);
815 spdk_thread_send_msg(admin_qpair_group->thread, nvmf_ctrlr_add_io_qpair, req);
845 _nvmf_ctrlr_connect(struct spdk_nvmf_request *req)
847 struct spdk_nvmf_fabric_connect_data *data = req->iov[0].iov_base;
848 struct spdk_nvmf_fabric_connect_cmd *cmd = &req->cmd->connect_cmd;
849 struct spdk_nvmf_fabric_connect_rsp *rsp = &req->rsp->connect_rsp;
850 struct spdk_nvmf_qpair *qpair = req->qpair;
936 ctrlr = nvmf_ctrlr_create(subsystem, req, cmd, data);
945 spdk_thread_send_msg(subsystem->thread, _nvmf_ctrlr_add_io_qpair, req);
951 nvmf_subsystem_pg_from_connect_cmd(struct spdk_nvmf_request *req)
957 assert(nvmf_request_is_fabric_connect(req));
958 assert(req->qpair->ctrlr == NULL);
959 assert(req->iovcnt == 1);
961 data = req->iov[0].iov_base;
962 tgt = req->qpair->transport->tgt;
969 return &req->qpair->group->sgroups[subsystem->id];
973 spdk_nvmf_ctrlr_connect(struct spdk_nvmf_request *req)
975 struct spdk_nvmf_fabric_connect_rsp *rsp = &req->rsp->connect_rsp;
977 struct spdk_nvmf_qpair *qpair = req->qpair;
980 if (req->iovcnt > 1) {
981 SPDK_ERRLOG("Connect command invalid iovcnt: %d\n", req->iovcnt);
987 sgroup = nvmf_subsystem_pg_from_connect_cmd(req);
995 TAILQ_INSERT_TAIL(&qpair->outstanding, req, link);
997 status = _nvmf_ctrlr_connect(req);
1001 _nvmf_request_complete(req);
1008 nvmf_ctrlr_cmd_connect(struct spdk_nvmf_request *req)
1010 struct spdk_nvmf_fabric_connect_data *data = req->iov[0].iov_base;
1011 struct spdk_nvmf_fabric_connect_rsp *rsp = &req->rsp->connect_rsp;
1012 struct spdk_nvmf_transport *transport = req->qpair->transport;
1015 if (req->length < sizeof(struct spdk_nvmf_fabric_connect_data)) {
1016 SPDK_ERRLOG("Connect command data length 0x%x too small\n", req->length);
1021 if (req->iovcnt > 1) {
1022 SPDK_ERRLOG("Connect command invalid iovcnt: %d\n", req->iovcnt);
1044 sgroup = nvmf_subsystem_pg_from_connect_cmd(req);
1047 TAILQ_REMOVE(&req->qpair->outstanding, req, link);
1048 TAILQ_INSERT_TAIL(&sgroup->queued, req, link);
1060 if (!nvmf_qpair_access_allowed(req->qpair, subsystem, data->hostnqn)) {
1066 return _nvmf_ctrlr_connect(req);
1478 nvmf_property_get(struct spdk_nvmf_request *req)
1480 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr;
1481 struct spdk_nvmf_fabric_prop_get_cmd *cmd = &req->cmd->prop_get_cmd;
1482 struct spdk_nvmf_fabric_prop_get_rsp *response = &req->rsp->prop_get_rsp;
1537 nvmf_property_set(struct spdk_nvmf_request *req)
1539 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr;
1540 struct spdk_nvmf_fabric_prop_set_cmd *cmd = &req->cmd->prop_set_cmd;
1541 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl;
1606 nvmf_ctrlr_set_features_arbitration(struct spdk_nvmf_request *req)
1608 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr;
1609 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd;
1620 nvmf_ctrlr_set_features_power_management(struct spdk_nvmf_request *req)
1622 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr;
1623 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd;
1624 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl;
1671 nvmf_ctrlr_set_features_temperature_threshold(struct spdk_nvmf_request *req)
1673 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd;
1674 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl;
1689 nvmf_ctrlr_get_features_temperature_threshold(struct spdk_nvmf_request *req)
1691 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd;
1692 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl;
1709 nvmf_ctrlr_get_features_interrupt_vector_configuration(struct spdk_nvmf_request *req)
1711 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr;
1712 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd;
1713 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl;
1726 nvmf_ctrlr_set_features_error_recovery(struct spdk_nvmf_request *req)
1728 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr;
1729 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd;
1730 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl;
1752 nvmf_ctrlr_set_features_volatile_write_cache(struct spdk_nvmf_request *req)
1754 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr;
1755 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd;
1768 nvmf_ctrlr_set_features_write_atomicity(struct spdk_nvmf_request *req)
1770 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr;
1771 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd;
1782 nvmf_ctrlr_set_features_host_identifier(struct spdk_nvmf_request *req)
1784 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl;
1792 nvmf_ctrlr_get_features_host_identifier(struct spdk_nvmf_request *req)
1794 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr;
1795 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd;
1796 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl;
1808 if (req->iovcnt < 1 || req->length < sizeof(ctrlr->hostid)) {
1814 spdk_iov_xfer_init(&ix, req->iov, req->iovcnt);
1821 nvmf_ctrlr_get_features_reservation_notification_mask(struct spdk_nvmf_request *req)
1823 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr;
1824 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd;
1825 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl;
1848 nvmf_ctrlr_set_features_reservation_notification_mask(struct spdk_nvmf_request *req)
1850 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr;
1852 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd;
1853 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl;
1878 nvmf_ctrlr_get_features_reservation_persistence(struct spdk_nvmf_request *req)
1880 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr;
1881 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd;
1882 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl;
1904 nvmf_ctrlr_set_features_reservation_persistence(struct spdk_nvmf_request *req)
1906 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr;
1907 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd;
1908 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl;
1940 nvmf_ctrlr_get_features_host_behavior_support(struct spdk_nvmf_request *req)
1942 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr;
1943 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl;
1949 if (req->iovcnt < 1 || req->length < sizeof(struct spdk_nvme_host_behavior)) {
1959 spdk_iov_xfer_init(&ix, req->iov, req->iovcnt);
1966 nvmf_ctrlr_set_features_host_behavior_support(struct spdk_nvmf_request *req)
1968 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr;
1969 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl;
1973 if (req->iovcnt != 1) {
1974 SPDK_ERRLOG("Host Behavior Support invalid iovcnt: %d\n", req->iovcnt);
1979 if (req->iov[0].iov_len != sizeof(struct spdk_nvme_host_behavior)) {
1980 SPDK_ERRLOG("Host Behavior Support invalid iov_len: %zd\n", req->iov[0].iov_len);
1986 host_behavior = (struct spdk_nvme_host_behavior *)req->iov[0].iov_base;
2011 nvmf_ctrlr_set_features_keep_alive_timer(struct spdk_nvmf_request *req)
2013 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr;
2014 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd;
2015 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl;
2054 nvmf_ctrlr_set_features_number_of_queues(struct spdk_nvmf_request *req)
2056 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr;
2057 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd;
2058 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl;
2062 req->cmd->nvme_cmd.cdw11);
2154 struct spdk_nvmf_request *req = ctrlr->aer_req[i];
2155 data_local.aer_cids[i] = req->cmd->nvme_cmd.cid;
2202 nvmf_ctrlr_set_features_async_event_configuration(struct spdk_nvmf_request *req)
2204 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr;
2205 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd;
2216 nvmf_ctrlr_async_event_request(struct spdk_nvmf_request *req)
2218 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr;
2219 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl;
2240 ctrlr->aer_req[ctrlr->nr_aer_reqs++] = req;
2638 nvmf_ctrlr_get_log_page(struct spdk_nvmf_request *req)
2640 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr;
2642 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd;
2643 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl;
2649 if (req->iovcnt < 1) {
2668 if (len > req->length) {
2670 len, req->length);
2683 if (spdk_nvmf_qpair_get_listen_trid(req->qpair, &cmd_source_trid)) {
2689 nvmf_get_discovery_log_page(subsystem->tgt, ctrlr->hostnqn, req->iov, req->iovcnt,
2709 nvmf_get_error_log_page(ctrlr, req->iov, req->iovcnt, offset, len, rae);
2715 nvmf_get_firmware_slot_log_page(req->iov, req->iovcnt, offset, len);
2720 nvmf_get_ana_log_page(ctrlr, req->iov, req->iovcnt, offset, len, rae, rgo);
2726 nvmf_get_cmds_and_effects_log_page(ctrlr, req->iov, req->iovcnt, offset, len);
2729 nvmf_get_changed_ns_list_log_page(ctrlr, req->iov, req->iovcnt, offset, len, rae);
2732 nvmf_get_reservation_notification_log_page(ctrlr, req->iov, req->iovcnt, offset, len, rae);
2833 identify_ns_passthru_cb(struct spdk_nvmf_request *req)
2835 struct spdk_nvme_cmd *cmd = spdk_nvmf_request_get_cmd(req);
2836 struct spdk_nvmf_ctrlr *ctrlr = spdk_nvmf_request_get_ctrlr(req);
2837 struct spdk_nvme_cpl *rsp = spdk_nvmf_request_get_response(req);
2843 datalen = spdk_nvmf_request_copy_to_buf(req, &nvme_nsdata,
2845 nvmf_ctrlr_identify_ns(ctrlr, cmd, rsp, &nvmf_nsdata, req->orig_nsid);
2868 spdk_nvmf_request_copy_from_buf(req, &nvmf_nsdata, datalen);
2872 spdk_nvmf_ctrlr_identify_ns_ext(struct spdk_nvmf_request *req)
2874 struct spdk_nvme_cmd *cmd = spdk_nvmf_request_get_cmd(req);
2875 struct spdk_nvmf_ctrlr *ctrlr = spdk_nvmf_request_get_ctrlr(req);
2877 struct spdk_nvme_cpl *rsp = spdk_nvmf_request_get_response(req);
2887 rc = spdk_nvmf_request_get_bdev(cmd->nsid, req, &bdev, &desc, &ch);
2893 spdk_iov_xfer_init(&ix, req->iov, req->iovcnt);
2900 req->orig_nsid = ns->nsid;
2903 return spdk_nvmf_bdev_ctrlr_nvme_passthru_admin(bdev, desc, ch, req, identify_ns_passthru_cb);
3350 nvmf_ctrlr_identify(struct spdk_nvmf_request *req)
3353 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr;
3354 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd;
3355 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl;
3361 if (req->iovcnt < 1 || req->length < SPDK_NVME_IDENTIFY_BUFLEN) {
3380 spdk_iov_xfer_init(&ix, req->iov, req->iovcnt);
3388 return spdk_nvmf_ctrlr_identify_ns_ext(req);
3397 tmpbuf, req->length);
3400 ret = spdk_nvmf_ns_identify_iocs_specific(ctrlr, cmd, rsp, (void *)&tmpbuf, req->length);
3403 ret = spdk_nvmf_ctrlr_identify_iocs_specific(ctrlr, cmd, rsp, (void *)&tmpbuf, req->length);
3406 ret = nvmf_ctrlr_identify_iocs(ctrlr, cmd, rsp, (void *)&tmpbuf, req->length);
3429 struct spdk_nvmf_request *req;
3441 req = ctrlr->aer_req[i];
3445 /* Move the last req to the aborting position for making aer_reqs
3453 req->rsp->nvme_cpl.status.sct = SPDK_NVME_SCT_GENERIC;
3454 req->rsp->nvme_cpl.status.sc = SPDK_NVME_SC_ABORTED_BY_REQUEST;
3455 _nvmf_request_complete(req);
3466 struct spdk_nvmf_request *req, *tmp;
3468 TAILQ_FOREACH_SAFE(req, &qpair->outstanding, link, tmp) {
3469 if (req->zcopy_phase == NVMF_ZCOPY_PHASE_EXECUTE) {
3476 req->rsp->nvme_cpl.status.sct = SPDK_NVME_SCT_GENERIC;
3477 req->rsp->nvme_cpl.status.sc = SPDK_NVME_SC_ABORTED_BY_REQUEST;
3478 nvmf_transport_req_free(req);
3484 nvmf_qpair_abort_request(struct spdk_nvmf_qpair *qpair, struct spdk_nvmf_request *req)
3486 uint16_t cid = req->cmd->nvme_cmd.cdw10_bits.abort.cid;
3491 req->rsp->nvme_cpl.cdw0 &= ~1U; /* Command successfully aborted */
3493 spdk_nvmf_request_complete(req);
3497 nvmf_transport_qpair_abort_request(qpair, req);
3503 struct spdk_nvmf_request *req = spdk_io_channel_iter_get_ctx(i);
3509 _nvmf_request_complete(req);
3516 struct spdk_nvmf_request *req = spdk_io_channel_iter_get_ctx(i);
3519 uint16_t sqid = req->cmd->nvme_cmd.cdw10_bits.abort.sqid;
3523 if (qpair->ctrlr == req->qpair->ctrlr && qpair->qid == sqid) {
3526 nvmf_qpair_abort_request(qpair, req);
3538 nvmf_ctrlr_abort(struct spdk_nvmf_request *req)
3540 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl;
3547 spdk_for_each_channel(req->qpair->ctrlr->subsys->tgt,
3549 req,
3557 nvmf_ctrlr_abort_request(struct spdk_nvmf_request *req)
3559 struct spdk_nvmf_request *req_to_abort = req->req_to_abort;
3569 return g_nvmf_custom_admin_cmd_hdlrs[SPDK_NVME_OPC_ABORT].hdlr(req);
3578 return spdk_nvmf_bdev_ctrlr_abort_cmd(bdev, desc, ch, req, req_to_abort);
3582 get_features_generic(struct spdk_nvmf_request *req, uint32_t cdw0)
3584 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl;
3609 nvmf_ctrlr_get_features(struct spdk_nvmf_request *req)
3612 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr;
3613 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd;
3614 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl;
3625 return get_features_generic(req, ctrlr->feat.keep_alive_timer.raw);
3627 return get_features_generic(req, ctrlr->feat.async_event_configuration.raw);
3660 return get_features_generic(req, ctrlr->feat.arbitration.raw);
3662 return get_features_generic(req, ctrlr->feat.power_management.raw);
3664 return nvmf_ctrlr_get_features_temperature_threshold(req);
3666 return get_features_generic(req, ctrlr->feat.error_recovery.raw);
3668 return get_features_generic(req, ctrlr->feat.volatile_write_cache.raw);
3670 return get_features_generic(req, ctrlr->feat.number_of_queues.raw);
3672 return get_features_generic(req, ctrlr->feat.interrupt_coalescing.raw);
3674 return nvmf_ctrlr_get_features_interrupt_vector_configuration(req);
3676 return get_features_generic(req, ctrlr->feat.write_atomicity.raw);
3678 return get_features_generic(req, ctrlr->feat.async_event_configuration.raw);
3680 return get_features_generic(req, ctrlr->feat.keep_alive_timer.raw);
3682 return nvmf_ctrlr_get_features_host_identifier(req);
3684 return nvmf_ctrlr_get_features_reservation_notification_mask(req);
3686 return nvmf_ctrlr_get_features_reservation_persistence(req);
3688 return nvmf_ctrlr_get_features_host_behavior_support(req);
3697 nvmf_ctrlr_set_features(struct spdk_nvmf_request *req)
3700 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr;
3701 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd;
3702 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl;
3723 return nvmf_ctrlr_set_features_keep_alive_timer(req);
3725 return nvmf_ctrlr_set_features_async_event_configuration(req);
3767 return nvmf_ctrlr_set_features_arbitration(req);
3769 return nvmf_ctrlr_set_features_power_management(req);
3771 return nvmf_ctrlr_set_features_temperature_threshold(req);
3773 return nvmf_ctrlr_set_features_error_recovery(req);
3775 return nvmf_ctrlr_set_features_volatile_write_cache(req);
3777 return nvmf_ctrlr_set_features_number_of_queues(req);
3783 return nvmf_ctrlr_set_features_write_atomicity(req);
3785 return nvmf_ctrlr_set_features_async_event_configuration(req);
3787 return nvmf_ctrlr_set_features_keep_alive_timer(req);
3789 return nvmf_ctrlr_set_features_host_identifier(req);
3791 return nvmf_ctrlr_set_features_reservation_notification_mask(req);
3793 return nvmf_ctrlr_set_features_reservation_persistence(req);
3795 return nvmf_ctrlr_set_features_host_behavior_support(req);
3804 nvmf_ctrlr_keep_alive(struct spdk_nvmf_request *req)
3806 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr;
3847 nvmf_ctrlr_process_admin_cmd(struct spdk_nvmf_request *req)
3849 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr;
3850 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd;
3851 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl;
3861 sgroup = &req->qpair->group->sgroups[ctrlr->subsys->id];
3885 if (req->iovcnt && spdk_nvme_opc_get_data_transfer(cmd->opc) == SPDK_NVME_DATA_CONTROLLER_TO_HOST) {
3886 spdk_iov_memset(req->iov, req->iovcnt, 0);
3906 rc = g_nvmf_custom_admin_cmd_hdlrs[cmd->opc].hdlr(req);
3917 return nvmf_passthru_admin_cmd(req);
3922 return nvmf_ctrlr_get_log_page(req);
3924 return nvmf_ctrlr_identify(req);
3926 return nvmf_ctrlr_abort(req);
3928 return nvmf_ctrlr_get_features(req);
3930 return nvmf_ctrlr_set_features(req);
3932 return nvmf_ctrlr_async_event_request(req);
3934 return nvmf_ctrlr_keep_alive(req);
3955 nvmf_ctrlr_process_fabrics_cmd(struct spdk_nvmf_request *req)
3957 struct spdk_nvmf_qpair *qpair = req->qpair;
3960 cap_hdr = &req->cmd->nvmf_cmd;
3965 return nvmf_ctrlr_cmd_connect(req);
3973 return nvmf_property_set(req);
3975 return nvmf_property_get(req);
3978 return nvmf_auth_request_exec(req);
3982 req->rsp->nvme_cpl.status.sct = SPDK_NVME_SCT_GENERIC;
3983 req->rsp->nvme_cpl.status.sc = SPDK_NVME_SC_INVALID_OPCODE;
3994 return nvmf_auth_request_exec(req);
3997 req->rsp->nvme_cpl.status.sct = SPDK_NVME_SCT_GENERIC;
3998 req->rsp->nvme_cpl.status.sc = SPDK_NVME_SC_INVALID_OPCODE;
4023 struct spdk_nvmf_request *req;
4037 req = ctrlr->aer_req[--ctrlr->nr_aer_reqs];
4038 rsp = &req->rsp->nvme_cpl;
4042 _nvmf_request_complete(req);
4179 struct spdk_nvmf_request *req;
4189 req = ctrlr->aer_req[i];
4191 req->rsp->nvme_cpl.status.sct = SPDK_NVME_SCT_GENERIC;
4192 req->rsp->nvme_cpl.status.sc = SPDK_NVME_SC_ABORTED_BY_REQUEST;
4193 _nvmf_request_complete(req);
4288 struct spdk_nvmf_request *req)
4290 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd;
4357 req->rsp->nvme_cpl.status.sct = SPDK_NVME_SCT_GENERIC;
4358 req->rsp->nvme_cpl.status.sc = status;
4367 nvmf_ctrlr_process_io_fused_cmd(struct spdk_nvmf_request *req, struct spdk_bdev *bdev,
4370 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd;
4371 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl;
4372 struct spdk_nvmf_request *first_fused_req = req->qpair->first_fused_req;
4382 /* abort req->qpair->first_fused_request and continue with new fused command */
4393 req->qpair->first_fused_req = req;
4407 /* abort req->qpair->first_fused_request and fail current command */
4414 req->qpair->first_fused_req = NULL;
4419 req->first_fused_req = first_fused_req;
4420 req->first_fused = true;
4421 req->qpair->first_fused_req = NULL;
4429 rc = nvmf_bdev_ctrlr_compare_and_write_cmd(bdev, desc, ch, req->first_fused_req, req);
4440 req->first_fused_req = NULL;
4441 req->first_fused = false;
4449 nvmf_ctrlr_use_zcopy(struct spdk_nvmf_request *req)
4451 struct spdk_nvmf_transport *transport = req->qpair->transport;
4454 assert(req->zcopy_phase == NVMF_ZCOPY_PHASE_NONE);
4460 if (nvmf_qpair_is_admin_queue(req->qpair)) {
4465 if ((req->cmd->nvme_cmd.opc != SPDK_NVME_OPC_WRITE) &&
4466 (req->cmd->nvme_cmd.opc != SPDK_NVME_OPC_READ)) {
4471 if (req->cmd->nvme_cmd.fuse != SPDK_NVME_CMD_FUSE_NONE) {
4476 ns = nvmf_ctrlr_get_ns(req->qpair->ctrlr, req->cmd->nvme_cmd.nsid);
4481 req->zcopy_phase = NVMF_ZCOPY_PHASE_INIT;
4486 spdk_nvmf_request_zcopy_start(struct spdk_nvmf_request *req)
4488 assert(req->zcopy_phase == NVMF_ZCOPY_PHASE_INIT);
4491 req->iovcnt = NVMF_REQ_MAX_BUFFERS;
4493 spdk_nvmf_request_exec(req);
4497 spdk_nvmf_request_zcopy_end(struct spdk_nvmf_request *req, bool commit)
4499 assert(req->zcopy_phase == NVMF_ZCOPY_PHASE_EXECUTE);
4500 req->zcopy_phase = NVMF_ZCOPY_PHASE_END_PENDING;
4502 nvmf_bdev_ctrlr_zcopy_end(req, commit);
4506 nvmf_ctrlr_process_io_cmd(struct spdk_nvmf_request *req)
4513 struct spdk_nvmf_qpair *qpair = req->qpair;
4516 struct spdk_nvme_cmd *cmd = &req->cmd->nvme_cmd;
4517 struct spdk_nvme_cpl *response = &req->rsp->nvme_cpl;
4552 SPDK_DTRACE_PROBE3_TICKS(nvmf_request_io_exec_path, req,
4560 if (nvmf_ns_reservation_request_check(ns_info, ctrlr, req)) {
4571 return nvmf_ctrlr_process_io_fused_cmd(req, bdev, desc, ch);
4577 req->qpair->first_fused_req->cmd->nvmf_cmd.opcode);
4588 req->cmd->nvme_cmd.nsid = ns->passthru_nsid;
4590 return nvmf_bdev_ctrlr_nvme_passthru_io(bdev, desc, ch, req);
4593 if (spdk_nvmf_request_using_zcopy(req)) {
4594 assert(req->zcopy_phase == NVMF_ZCOPY_PHASE_INIT);
4595 return nvmf_bdev_ctrlr_zcopy_start(bdev, desc, ch, req);
4599 return nvmf_bdev_ctrlr_read_cmd(bdev, desc, ch, req);
4601 return nvmf_bdev_ctrlr_write_cmd(bdev, desc, ch, req);
4603 return nvmf_bdev_ctrlr_flush_cmd(bdev, desc, ch, req);
4608 return nvmf_bdev_ctrlr_compare_cmd(bdev, desc, ch, req);
4613 return nvmf_bdev_ctrlr_write_zeroes_cmd(bdev, desc, ch, req);
4618 return nvmf_bdev_ctrlr_dsm_cmd(bdev, desc, ch, req);
4626 spdk_thread_send_msg(ctrlr->subsys->thread, nvmf_ns_reservation_request, req);
4632 return nvmf_bdev_ctrlr_copy_cmd(bdev, desc, ch, req);
4638 req->cmd->nvme_cmd.nsid = ns->passthru_nsid;
4640 return nvmf_bdev_ctrlr_nvme_passthru_io(bdev, desc, ch, req);
4664 spdk_nvmf_request_free(struct spdk_nvmf_request *req)
4666 struct spdk_nvmf_qpair *qpair = req->qpair;
4668 TAILQ_REMOVE(&qpair->outstanding, req, link);
4669 if (spdk_unlikely(nvmf_transport_req_free(req))) {
4681 struct spdk_nvmf_request *req = ctx;
4682 struct spdk_nvme_cpl *rsp = &req->rsp->nvme_cpl;
4693 rsp->cid = req->cmd->nvme_cmd.cid;
4694 nsid = req->cmd->nvme_cmd.nsid;
4695 opcode = req->cmd->nvmf_cmd.opcode;
4697 qpair = req->qpair;
4701 is_aer = req->cmd->nvme_cmd.opc == SPDK_NVME_OPC_ASYNC_EVENT_REQUEST;
4716 } else if (spdk_unlikely(nvmf_request_is_fabric_connect(req))) {
4717 sgroup = nvmf_subsystem_pg_from_connect_cmd(req);
4724 switch (req->zcopy_phase) {
4726 TAILQ_REMOVE(&qpair->outstanding, req, link);
4730 req->zcopy_phase = NVMF_ZCOPY_PHASE_INIT_FAILED;
4731 TAILQ_REMOVE(&qpair->outstanding, req, link);
4733 req->zcopy_phase = NVMF_ZCOPY_PHASE_EXECUTE;
4739 TAILQ_REMOVE(&qpair->outstanding, req, link);
4740 req->zcopy_phase = NVMF_ZCOPY_PHASE_COMPLETE;
4743 SPDK_ERRLOG("Invalid ZCOPY phase %u\n", req->zcopy_phase);
4747 if (spdk_unlikely(nvmf_transport_req_complete(req))) {
4758 if (req->zcopy_phase == NVMF_ZCOPY_PHASE_NONE ||
4759 req->zcopy_phase == NVMF_ZCOPY_PHASE_COMPLETE ||
4760 req->zcopy_phase == NVMF_ZCOPY_PHASE_INIT_FAILED) {
4797 spdk_nvmf_request_complete(struct spdk_nvmf_request *req)
4799 struct spdk_nvmf_qpair *qpair = req->qpair;
4801 spdk_thread_exec_msg(qpair->group->thread, _nvmf_request_complete, req);
4807 nvmf_check_subsystem_active(struct spdk_nvmf_request *req)
4809 struct spdk_nvmf_qpair *qpair = req->qpair;
4817 } else if (spdk_unlikely(nvmf_request_is_fabric_connect(req))) {
4818 sgroup = nvmf_subsystem_pg_from_connect_cmd(req);
4826 if (spdk_unlikely(req->cmd->nvmf_cmd.opcode == SPDK_NVME_OPC_FABRIC ||
4830 TAILQ_INSERT_TAIL(&sgroup->queued, req, link);
4835 nsid = req->cmd->nvme_cmd.nsid;
4839 req->rsp->nvme_cpl.status.sct = SPDK_NVME_SCT_GENERIC;
4840 req->rsp->nvme_cpl.status.sc = SPDK_NVME_SC_INVALID_NAMESPACE_OR_FORMAT;
4841 req->rsp->nvme_cpl.status.dnr = 1;
4842 TAILQ_INSERT_TAIL(&qpair->outstanding, req, link);
4843 _nvmf_request_complete(req);
4853 req->rsp->nvme_cpl.status.sct = SPDK_NVME_SCT_GENERIC;
4854 req->rsp->nvme_cpl.status.sc = SPDK_NVME_SC_INVALID_NAMESPACE_OR_FORMAT;
4855 req->rsp->nvme_cpl.status.dnr = 1;
4856 TAILQ_INSERT_TAIL(&qpair->outstanding, req, link);
4858 _nvmf_request_complete(req);
4864 TAILQ_INSERT_TAIL(&sgroup->queued, req, link);
4875 nvmf_check_qpair_active(struct spdk_nvmf_request *req)
4877 struct spdk_nvmf_qpair *qpair = req->qpair;
4889 if (req->cmd->nvmf_cmd.opcode != SPDK_NVME_OPC_FABRIC) {
4891 req->cmd->nvmf_cmd.opcode, qpair->qid);
4894 if (req->cmd->nvmf_cmd.fctype != SPDK_NVMF_FABRIC_COMMAND_CONNECT) {
4896 req->cmd->nvmf_cmd.fctype, qpair->qid);
4903 if (req->cmd->nvmf_cmd.opcode != SPDK_NVME_OPC_FABRIC) {
4905 req->cmd->nvmf_cmd.opcode, qpair->qid);
4908 if (req->cmd->nvmf_cmd.fctype != SPDK_NVMF_FABRIC_COMMAND_AUTHENTICATION_SEND &&
4909 req->cmd->nvmf_cmd.fctype != SPDK_NVMF_FABRIC_COMMAND_AUTHENTICATION_RECV) {
4911 req->cmd->nvmf_cmd.fctype, qpair->qid);
4917 req->cmd->nvmf_cmd.opcode, qpair->qid, qpair->state);
4921 req->rsp->nvme_cpl.status.sct = sct;
4922 req->rsp->nvme_cpl.status.sc = sc;
4923 TAILQ_INSERT_TAIL(&qpair->outstanding, req, link);
4924 _nvmf_request_complete(req);
4930 spdk_nvmf_request_exec(struct spdk_nvmf_request *req)
4932 struct spdk_nvmf_qpair *qpair = req->qpair;
4935 if (spdk_unlikely(!nvmf_check_subsystem_active(req))) {
4938 if (spdk_unlikely(!nvmf_check_qpair_active(req))) {
4943 spdk_nvme_print_command(qpair->qid, &req->cmd->nvme_cmd);
4947 TAILQ_INSERT_TAIL(&qpair->outstanding, req, link);
4949 if (spdk_unlikely(req->cmd->nvmf_cmd.opcode == SPDK_NVME_OPC_FABRIC)) {
4950 status = nvmf_ctrlr_process_fabrics_cmd(req);
4952 status = nvmf_ctrlr_process_admin_cmd(req);
4954 status = nvmf_ctrlr_process_io_cmd(req);
4958 _nvmf_request_complete(req);
4993 spdk_nvmf_request_get_dif_ctx(struct spdk_nvmf_request *req, struct spdk_dif_ctx *dif_ctx)
4995 struct spdk_nvmf_qpair *qpair = req->qpair;
5006 if (spdk_unlikely(req->cmd->nvmf_cmd.opcode == SPDK_NVME_OPC_FABRIC)) {
5014 return nvmf_ctrlr_get_dif_ctx(ctrlr, &req->cmd->nvme_cmd, dif_ctx);
5024 nvmf_passthru_admin_cmd_for_bdev_nsid(struct spdk_nvmf_request *req, uint32_t bdev_nsid)
5031 struct spdk_nvme_cpl *response = spdk_nvmf_request_get_response(req);
5034 rc = spdk_nvmf_request_get_bdev(bdev_nsid, req, &bdev, &desc, &ch);
5041 ctrlr = req->qpair->ctrlr;
5045 req->cmd->nvme_cmd.nsid = ns->passthru_nsid;
5048 return spdk_nvmf_bdev_ctrlr_nvme_passthru_admin(bdev, desc, ch, req, NULL);
5052 nvmf_passthru_admin_cmd(struct spdk_nvmf_request *req)
5054 struct spdk_nvme_cmd *cmd = spdk_nvmf_request_get_cmd(req);
5063 return nvmf_passthru_admin_cmd_for_bdev_nsid(req, bdev_nsid);
5067 nvmf_passthru_admin_cmd_for_ctrlr(struct spdk_nvmf_request *req, struct spdk_nvmf_ctrlr *ctrlr)
5069 struct spdk_nvme_cpl *response = spdk_nvmf_request_get_response(req);
5080 return nvmf_passthru_admin_cmd_for_bdev_nsid(req, ns->nsid);
5091 spdk_nvmf_request_get_bdev(uint32_t nsid, struct spdk_nvmf_request *req,
5094 struct spdk_nvmf_ctrlr *ctrlr = req->qpair->ctrlr;
5096 struct spdk_nvmf_poll_group *group = req->qpair->group;
5117 struct spdk_nvmf_ctrlr *spdk_nvmf_request_get_ctrlr(struct spdk_nvmf_request *req)
5119 return req->qpair->ctrlr;
5122 struct spdk_nvme_cmd *spdk_nvmf_request_get_cmd(struct spdk_nvmf_request *req)
5124 return &req->cmd->nvme_cmd;
5127 struct spdk_nvme_cpl *spdk_nvmf_request_get_response(struct spdk_nvmf_request *req)
5129 return &req->rsp->nvme_cpl;
5132 struct spdk_nvmf_subsystem *spdk_nvmf_request_get_subsystem(struct spdk_nvmf_request *req)
5134 return req->qpair->ctrlr->subsys;
5138 spdk_nvmf_request_copy_from_buf(struct spdk_nvmf_request *req,
5143 spdk_iov_xfer_init(&ix, req->iov, req->iovcnt);
5148 spdk_nvmf_request_copy_to_buf(struct spdk_nvmf_request *req,
5153 spdk_iov_xfer_init(&ix, req->iov, req->iovcnt);
5168 struct spdk_nvmf_request *spdk_nvmf_request_get_req_to_abort(struct spdk_nvmf_request *req)
5170 return req->req_to_abort;