Lines Matching refs:qpair
24 static void nvme_pcie_fail_request_bad_vtophys(struct spdk_nvme_qpair *qpair,
39 nvme_pcie_qpair_reset(struct spdk_nvme_qpair *qpair)
41 struct nvme_pcie_qpair *pqpair = nvme_pcie_qpair(qpair);
63 nvme_pcie_qpair_get_fd(struct spdk_nvme_qpair *qpair, struct spdk_event_handler_opts *opts)
65 struct spdk_nvme_ctrlr *ctrlr = qpair->ctrlr;
74 return spdk_pci_device_get_interrupt_efd_by_index(devhandle, qpair->id);
84 return spdk_pci_device_get_interrupt_efd_by_index(devhandle, qpair->id);
123 nvme_pcie_qpair_construct(struct spdk_nvme_qpair *qpair,
126 struct spdk_nvme_ctrlr *ctrlr = qpair->ctrlr;
128 struct nvme_pcie_qpair *pqpair = nvme_pcie_qpair(qpair);
166 if (nvme_qpair_is_admin_queue(&pqpair->qpair)) {
229 pqpair->sq_tdbl = pctrlr->doorbell_base + (2 * qpair->id + 0) * pctrlr->doorbell_stride_u32;
230 pqpair->cq_hdbl = pctrlr->doorbell_base + (2 * qpair->id + 1) * pctrlr->doorbell_stride_u32;
247 pqpair->qpair.queue_depth = 0;
255 nvme_pcie_qpair_reset(qpair);
275 ctrlr->adminq = &pqpair->qpair;
278 0, /* qpair ID */
290 SPDK_ERRLOG("Failed to allocate admin qpair statistics\n");
301 nvme_pcie_qpair_insert_pending_admin_request(struct spdk_nvme_qpair *qpair,
304 struct spdk_nvme_ctrlr *ctrlr = qpair->ctrlr;
312 assert(nvme_qpair_is_admin_queue(qpair));
332 nvme_pcie_qpair_complete_pending_admin_request(struct spdk_nvme_qpair *qpair)
334 struct spdk_nvme_ctrlr *ctrlr = qpair->ctrlr;
343 assert(nvme_qpair_is_admin_queue(qpair));
357 nvme_complete_request(req->cb_fn, req->cb_arg, qpair, req, &req->cpl);
423 nvme_pcie_ctrlr_cmd_delete_io_cq(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair,
436 cmd->cdw10_bits.delete_io_q.qid = qpair->id;
442 nvme_pcie_ctrlr_cmd_delete_io_sq(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair,
455 cmd->cdw10_bits.delete_io_q.qid = qpair->id;
463 struct spdk_nvme_qpair *qpair = arg;
464 struct nvme_pcie_qpair *pqpair = nvme_pcie_qpair(qpair);
476 struct spdk_nvme_qpair *qpair = arg;
477 struct nvme_pcie_qpair *pqpair = nvme_pcie_qpair(qpair);
478 struct spdk_nvme_ctrlr *ctrlr = qpair->ctrlr;
483 /* This qpair was deleted by the application while the
485 * to free the qpair resources until this outstanding
489 nvme_pcie_qpair_destroy(qpair);
495 rc = nvme_pcie_ctrlr_cmd_delete_io_cq(qpair->ctrlr, qpair, nvme_completion_sq_error_delete_cq_cb,
496 qpair);
505 pqpair->shadow_doorbell.sq_tdbl = ctrlr->shadow_doorbell + (2 * qpair->id + 0) *
507 pqpair->shadow_doorbell.cq_hdbl = ctrlr->shadow_doorbell + (2 * qpair->id + 1) *
509 pqpair->shadow_doorbell.sq_eventidx = ctrlr->eventidx + (2 * qpair->id + 0) *
511 pqpair->shadow_doorbell.cq_eventidx = ctrlr->eventidx + (2 * qpair->id + 1) *
517 nvme_pcie_qpair_reset(qpair);
524 struct spdk_nvme_qpair *qpair = arg;
525 struct nvme_pcie_qpair *pqpair = nvme_pcie_qpair(qpair);
529 /* This qpair was deleted by the application while the
531 * to free the qpair resources until this outstanding
535 nvme_pcie_qpair_destroy(qpair);
545 rc = nvme_pcie_ctrlr_cmd_create_io_sq(qpair->ctrlr, qpair, nvme_completion_create_sq_cb, qpair);
549 rc = nvme_pcie_ctrlr_cmd_delete_io_cq(qpair->ctrlr, qpair, nvme_completion_sq_error_delete_cq_cb,
550 qpair);
561 _nvme_pcie_ctrlr_create_io_qpair(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair,
564 struct nvme_pcie_qpair *pqpair = nvme_pcie_qpair(qpair);
568 if (qpair->poll_group) {
569 struct nvme_pcie_poll_group *group = SPDK_CONTAINEROF(qpair->poll_group,
578 SPDK_ERRLOG("Failed to allocate qpair statistics\n");
579 nvme_qpair_set_state(qpair, NVME_QPAIR_DISCONNECTED);
585 rc = nvme_pcie_ctrlr_cmd_create_io_cq(ctrlr, qpair, nvme_completion_create_cq_cb, qpair);
589 nvme_qpair_set_state(qpair, NVME_QPAIR_DISCONNECTED);
597 nvme_pcie_ctrlr_connect_qpair(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair)
601 if (!nvme_qpair_is_admin_queue(qpair)) {
602 rc = _nvme_pcie_ctrlr_create_io_qpair(ctrlr, qpair, qpair->id);
604 nvme_qpair_set_state(qpair, NVME_QPAIR_CONNECTED);
611 nvme_pcie_ctrlr_disconnect_qpair(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair)
613 if (!nvme_qpair_is_admin_queue(qpair) || !ctrlr->is_disconnecting) {
614 nvme_transport_ctrlr_disconnect_qpair_done(qpair);
616 /* If this function is called for the admin qpair via spdk_nvme_ctrlr_reset()
659 nvme_pcie_qpair_submit_tracker(struct spdk_nvme_qpair *qpair, struct nvme_tracker *tr)
662 struct nvme_pcie_qpair *pqpair = nvme_pcie_qpair(qpair);
663 struct spdk_nvme_ctrlr *ctrlr = qpair->ctrlr;
668 spdk_trace_record(TRACE_NVME_PCIE_SUBMIT, qpair->id, 0, (uintptr_t)req, req->cb_arg,
671 pqpair->qpair.queue_depth);
678 qpair->last_fuse = req->cmd.fuse;
700 nvme_pcie_qpair_ring_sq_doorbell(qpair);
705 nvme_pcie_qpair_complete_tracker(struct spdk_nvme_qpair *qpair, struct nvme_tracker *tr,
708 struct nvme_pcie_qpair *pqpair = nvme_pcie_qpair(qpair);
715 spdk_trace_record(TRACE_NVME_PCIE_COMPLETE, qpair->id, 0, (uintptr_t)req, req->cb_arg,
716 (uint32_t)req->cmd.cid, (uint32_t)cpl->status_raw, pqpair->qpair.queue_depth);
723 print_error = error && print_on_error && !qpair->ctrlr->opts.disable_error_logging;
726 spdk_nvme_qpair_print_command(qpair, &req->cmd);
730 spdk_nvme_qpair_print_completion(qpair, cpl);
737 nvme_pcie_qpair_submit_tracker(qpair, tr);
740 pqpair->qpair.queue_depth--;
743 if (nvme_qpair_is_admin_queue(qpair) && req->pid != getpid()) {
744 nvme_pcie_qpair_insert_pending_admin_request(qpair, req, cpl);
746 nvme_complete_request(tr->cb_fn, tr->cb_arg, qpair, req, cpl);
756 nvme_pcie_qpair_manual_complete_tracker(struct spdk_nvme_qpair *qpair,
763 cpl.sqid = qpair->id;
768 nvme_pcie_qpair_complete_tracker(qpair, tr, &cpl, print_on_error);
772 nvme_pcie_qpair_abort_trackers(struct spdk_nvme_qpair *qpair, uint32_t dnr)
774 struct nvme_pcie_qpair *pqpair = nvme_pcie_qpair(qpair);
781 if (!qpair->ctrlr->opts.disable_error_logging) {
784 nvme_pcie_qpair_manual_complete_tracker(qpair, tr, SPDK_NVME_SCT_GENERIC,
794 nvme_pcie_admin_qpair_abort_aers(struct spdk_nvme_qpair *qpair)
796 struct nvme_pcie_qpair *pqpair = nvme_pcie_qpair(qpair);
803 nvme_pcie_qpair_manual_complete_tracker(qpair, tr,
814 nvme_pcie_admin_qpair_destroy(struct spdk_nvme_qpair *qpair)
816 nvme_pcie_admin_qpair_abort_aers(qpair);
820 nvme_pcie_qpair_abort_reqs(struct spdk_nvme_qpair *qpair, uint32_t dnr)
822 nvme_pcie_qpair_abort_trackers(qpair, dnr);
826 nvme_pcie_qpair_check_timeout(struct spdk_nvme_qpair *qpair)
830 struct nvme_pcie_qpair *pqpair = nvme_pcie_qpair(qpair);
831 struct spdk_nvme_ctrlr *ctrlr = qpair->ctrlr;
839 if (nvme_qpair_is_admin_queue(qpair)) {
842 active_proc = qpair->active_proc;
865 nvme_pcie_qpair_process_completions(struct spdk_nvme_qpair *qpair, uint32_t max_completions)
867 struct nvme_pcie_qpair *pqpair = nvme_pcie_qpair(qpair);
871 struct spdk_nvme_ctrlr *ctrlr = qpair->ctrlr;
881 if (spdk_unlikely(nvme_qpair_get_state(qpair) == NVME_QPAIR_CONNECTING)) {
885 * completion for this qpair. So check for that condition
886 * here and then update the qpair's state to CONNECTED, since
887 * we can only set the qpair state from the qpair's thread.
890 nvme_qpair_set_state(qpair, NVME_QPAIR_CONNECTED);
892 nvme_qpair_set_state(qpair, NVME_QPAIR_DISCONNECTED);
899 nvme_qpair_set_state(qpair, NVME_QPAIR_DISCONNECTED);
906 if (spdk_unlikely(nvme_qpair_is_admin_queue(qpair))) {
962 * as part of putting the req back on the qpair's free list.
965 nvme_pcie_qpair_complete_tracker(qpair, tr, cpl, true);
968 spdk_nvme_qpair_print_completion(qpair, cpl);
979 nvme_pcie_qpair_ring_cq_doorbell(qpair);
986 nvme_pcie_qpair_ring_sq_doorbell(qpair);
995 nvme_pcie_qpair_check_timeout(qpair);
999 * process the admin qpair disconnection.
1001 if (spdk_unlikely(nvme_qpair_is_admin_queue(qpair))) {
1002 nvme_pcie_qpair_complete_pending_admin_request(qpair);
1004 if (nvme_qpair_get_state(qpair) == NVME_QPAIR_DISCONNECTING) {
1005 rc = nvme_ctrlr_disable_poll(qpair->ctrlr);
1007 nvme_transport_ctrlr_disconnect_qpair_done(qpair);
1020 nvme_pcie_fail_request_bad_vtophys(qpair, tr);
1030 nvme_pcie_qpair_destroy(struct spdk_nvme_qpair *qpair)
1032 struct nvme_pcie_qpair *pqpair = nvme_pcie_qpair(qpair);
1034 if (nvme_qpair_is_admin_queue(qpair)) {
1035 nvme_pcie_admin_qpair_destroy(qpair);
1053 nvme_qpair_deinit(qpair);
1055 if (!pqpair->shared_stats && (!qpair->active_proc ||
1056 qpair->active_proc == nvme_ctrlr_get_current_process(qpair->ctrlr))) {
1057 if (qpair->id) {
1060 /* statistics of admin qpair are allocates from huge pages because
1061 * admin qpair is shared for multi-process */
1077 struct spdk_nvme_qpair *qpair;
1091 qpair = &pqpair->qpair;
1093 rc = nvme_qpair_init(qpair, qid, ctrlr, opts->qprio, opts->io_queue_requests, opts->async_mode);
1095 nvme_pcie_qpair_destroy(qpair);
1099 rc = nvme_pcie_qpair_construct(qpair, opts);
1102 nvme_pcie_qpair_destroy(qpair);
1106 return qpair;
1110 nvme_pcie_ctrlr_delete_io_qpair(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair)
1112 struct nvme_pcie_qpair *pqpair = nvme_pcie_qpair(qpair);
1123 if (nvme_qpair_get_state(qpair) == NVME_QPAIR_CONNECTING) {
1129 /* If attempting to delete a qpair that's still being connected, we have to wait until it's
1147 rc = nvme_pcie_ctrlr_cmd_delete_io_sq(ctrlr, qpair, nvme_completion_poll_cb, status);
1163 if (qpair->active_proc == nvme_ctrlr_get_current_process(ctrlr)) {
1164 nvme_pcie_qpair_process_completions(qpair, 0);
1169 rc = nvme_pcie_ctrlr_cmd_delete_io_cq(ctrlr, qpair, nvme_completion_poll_cb, status);
1191 if (qpair->no_deletion_notification_needed == 0) {
1193 nvme_pcie_qpair_abort_trackers(qpair, 1);
1197 nvme_pcie_qpair_destroy(qpair);
1203 nvme_pcie_fail_request_bad_vtophys(struct spdk_nvme_qpair *qpair, struct nvme_tracker *tr)
1205 if (!qpair->in_completion_context) {
1206 struct nvme_pcie_qpair *pqpair = nvme_pcie_qpair(qpair);
1218 nvme_pcie_qpair_manual_complete_tracker(qpair, tr, SPDK_NVME_SCT_GENERIC,
1302 nvme_pcie_qpair_build_request_invalid(struct spdk_nvme_qpair *qpair,
1306 nvme_pcie_fail_request_bad_vtophys(qpair, tr);
1314 nvme_pcie_qpair_build_contig_request(struct spdk_nvme_qpair *qpair, struct nvme_request *req,
1320 rc = nvme_pcie_prp_list_append(qpair->ctrlr, tr, &prp_index,
1322 req->payload_size, qpair->ctrlr->page_size);
1324 nvme_pcie_fail_request_bad_vtophys(qpair, tr);
1339 nvme_pcie_qpair_build_contig_hw_sgl_request(struct spdk_nvme_qpair *qpair, struct nvme_request *req,
1362 nvme_pcie_fail_request_bad_vtophys(qpair, tr);
1368 nvme_pcie_fail_request_bad_vtophys(qpair, tr);
1373 phys_addr = nvme_pcie_vtophys(qpair->ctrlr, virt_addr, &mapping_length);
1375 nvme_pcie_fail_request_bad_vtophys(qpair, tr);
1420 nvme_pcie_qpair_build_hw_sgl_request(struct spdk_nvme_qpair *qpair, struct nvme_request *req,
1429 struct nvme_pcie_qpair *pqpair = nvme_pcie_qpair(qpair);
1450 nvme_pcie_fail_request_bad_vtophys(qpair, tr);
1499 phys_addr = nvme_pcie_vtophys(qpair->ctrlr, virt_addr, &mapping_length);
1548 nvme_pcie_fail_request_bad_vtophys(qpair, tr);
1556 nvme_pcie_qpair_build_prps_sgl_request(struct spdk_nvme_qpair *qpair, struct nvme_request *req,
1563 uint32_t page_size = qpair->ctrlr->page_size;
1577 nvme_pcie_fail_request_bad_vtophys(qpair, tr);
1592 rc = nvme_pcie_prp_list_append(qpair->ctrlr, tr, &prp_index, virt_addr, length, page_size);
1594 nvme_pcie_fail_request_bad_vtophys(qpair, tr);
1624 nvme_pcie_qpair_build_metadata(struct spdk_nvme_qpair *qpair, struct nvme_tracker *tr,
1643 tr->meta_sgl.address = nvme_pcie_vtophys(qpair->ctrlr, md_payload, &mapping_length);
1652 req->cmd.mptr = nvme_pcie_vtophys(qpair->ctrlr, md_payload, &mapping_length);
1662 nvme_pcie_fail_request_bad_vtophys(qpair, tr);
1667 nvme_pcie_qpair_submit_request(struct spdk_nvme_qpair *qpair, struct nvme_request *req)
1671 struct spdk_nvme_ctrlr *ctrlr = qpair->ctrlr;
1672 struct nvme_pcie_qpair *pqpair = nvme_pcie_qpair(qpair);
1678 if (spdk_unlikely(nvme_qpair_is_admin_queue(qpair))) {
1694 pqpair->qpair.queue_depth++;
1708 !nvme_qpair_is_admin_queue(qpair);
1710 !nvme_qpair_is_admin_queue(qpair);
1728 rc = g_nvme_pcie_build_req_table[payload_type][sgl_supported](qpair, req, tr, dword_aligned);
1735 rc = nvme_pcie_qpair_build_metadata(qpair, tr, sgl_supported, mptr_sgl_supported, dword_aligned);
1743 nvme_pcie_qpair_submit_tracker(qpair, tr);
1746 if (spdk_unlikely(nvme_qpair_is_admin_queue(qpair))) {
1767 nvme_pcie_poll_group_connect_qpair(struct spdk_nvme_qpair *qpair)
1773 nvme_pcie_poll_group_disconnect_qpair(struct spdk_nvme_qpair *qpair)
1780 struct spdk_nvme_qpair *qpair)
1787 struct spdk_nvme_qpair *qpair)
1789 struct nvme_pcie_qpair *pqpair = nvme_pcie_qpair(qpair);
1799 struct spdk_nvme_qpair *qpair, *tmp_qpair;
1803 STAILQ_FOREACH_SAFE(qpair, &tgroup->disconnected_qpairs, poll_group_stailq, tmp_qpair) {
1804 disconnected_qpair_cb(qpair, tgroup->group->ctx);
1807 STAILQ_FOREACH_SAFE(qpair, &tgroup->connected_qpairs, poll_group_stailq, tmp_qpair) {
1808 local_completions = spdk_nvme_qpair_process_completions(qpair, completions_per_qpair);
1810 disconnected_qpair_cb(qpair, tgroup->group->ctx);
1824 struct spdk_nvme_qpair *qpair, *tmp_qpair;
1826 STAILQ_FOREACH_SAFE(qpair, &tgroup->disconnected_qpairs, poll_group_stailq, tmp_qpair) {
1827 disconnected_qpair_cb(qpair, tgroup->group->ctx);