1a15f7c96SJohn Baldwin /*- 2a15f7c96SJohn Baldwin * SPDX-License-Identifier: BSD-2-Clause 3a15f7c96SJohn Baldwin * 4a15f7c96SJohn Baldwin * Copyright (c) 2023-2024 Chelsio Communications, Inc. 5a15f7c96SJohn Baldwin * Written by: John Baldwin <jhb@FreeBSD.org> 6a15f7c96SJohn Baldwin */ 7a15f7c96SJohn Baldwin 8a15f7c96SJohn Baldwin #include <sys/types.h> 9a15f7c96SJohn Baldwin #include <sys/_bitset.h> 10a15f7c96SJohn Baldwin #include <sys/bitset.h> 11a15f7c96SJohn Baldwin #include <sys/lock.h> 12a15f7c96SJohn Baldwin #include <sys/mutex.h> 13a15f7c96SJohn Baldwin 14a15f7c96SJohn Baldwin #include <dev/nvmf/nvmf_transport.h> 15a15f7c96SJohn Baldwin #include <dev/nvmf/controller/nvmft_var.h> 16a15f7c96SJohn Baldwin 17a15f7c96SJohn Baldwin /* 18a15f7c96SJohn Baldwin * A bitmask of command ID values. This is used to detect duplicate 19a15f7c96SJohn Baldwin * commands with the same ID. 20a15f7c96SJohn Baldwin */ 21a15f7c96SJohn Baldwin #define NUM_CIDS (UINT16_MAX + 1) 22a15f7c96SJohn Baldwin BITSET_DEFINE(cidset, NUM_CIDS); 23a15f7c96SJohn Baldwin 24a15f7c96SJohn Baldwin struct nvmft_qpair { 25a15f7c96SJohn Baldwin struct nvmft_controller *ctrlr; 26a15f7c96SJohn Baldwin struct nvmf_qpair *qp; 27a15f7c96SJohn Baldwin struct cidset *cids; 28a15f7c96SJohn Baldwin 29a15f7c96SJohn Baldwin bool admin; 30a15f7c96SJohn Baldwin bool sq_flow_control; 31a15f7c96SJohn Baldwin uint16_t qid; 32a15f7c96SJohn Baldwin u_int qsize; 33a15f7c96SJohn Baldwin uint16_t sqhd; 34a15f7c96SJohn Baldwin volatile u_int qp_refs; /* Internal references on 'qp'. */ 35a15f7c96SJohn Baldwin 361b3fa1acSJohn Baldwin struct task datamove_task; 371b3fa1acSJohn Baldwin STAILQ_HEAD(, ctl_io_hdr) datamove_queue; 381b3fa1acSJohn Baldwin 39a15f7c96SJohn Baldwin struct mtx lock; 40a15f7c96SJohn Baldwin 41a15f7c96SJohn Baldwin char name[16]; 42a15f7c96SJohn Baldwin }; 43a15f7c96SJohn Baldwin 44a15f7c96SJohn Baldwin static int _nvmft_send_generic_error(struct nvmft_qpair *qp, 45a15f7c96SJohn Baldwin struct nvmf_capsule *nc, uint8_t sc_status); 461b3fa1acSJohn Baldwin static void nvmft_datamove_task(void *context, int pending); 47a15f7c96SJohn Baldwin 48a15f7c96SJohn Baldwin static void 49a15f7c96SJohn Baldwin nvmft_qpair_error(void *arg, int error) 50a15f7c96SJohn Baldwin { 51a15f7c96SJohn Baldwin struct nvmft_qpair *qp = arg; 52a15f7c96SJohn Baldwin struct nvmft_controller *ctrlr = qp->ctrlr; 53a15f7c96SJohn Baldwin 54a15f7c96SJohn Baldwin /* 55a15f7c96SJohn Baldwin * XXX: The Linux TCP initiator sends a RST immediately after 56a15f7c96SJohn Baldwin * the FIN, so treat ECONNRESET as plain EOF to avoid spurious 57a15f7c96SJohn Baldwin * errors on shutdown. 58a15f7c96SJohn Baldwin */ 59a15f7c96SJohn Baldwin if (error == ECONNRESET) 60a15f7c96SJohn Baldwin error = 0; 61a15f7c96SJohn Baldwin 62a15f7c96SJohn Baldwin if (error != 0) 63a15f7c96SJohn Baldwin nvmft_printf(ctrlr, "error %d on %s\n", error, qp->name); 64a15f7c96SJohn Baldwin nvmft_controller_error(ctrlr, qp, error); 65a15f7c96SJohn Baldwin } 66a15f7c96SJohn Baldwin 67a15f7c96SJohn Baldwin static void 68a15f7c96SJohn Baldwin nvmft_receive_capsule(void *arg, struct nvmf_capsule *nc) 69a15f7c96SJohn Baldwin { 70a15f7c96SJohn Baldwin struct nvmft_qpair *qp = arg; 71a15f7c96SJohn Baldwin struct nvmft_controller *ctrlr = qp->ctrlr; 72a15f7c96SJohn Baldwin const struct nvme_command *cmd; 73a15f7c96SJohn Baldwin uint8_t sc_status; 74a15f7c96SJohn Baldwin 75a15f7c96SJohn Baldwin cmd = nvmf_capsule_sqe(nc); 76a15f7c96SJohn Baldwin if (ctrlr == NULL) { 77a15f7c96SJohn Baldwin printf("NVMFT: %s received CID %u opcode %u on newborn queue\n", 78a15f7c96SJohn Baldwin qp->name, le16toh(cmd->cid), cmd->opc); 79a15f7c96SJohn Baldwin nvmf_free_capsule(nc); 80a15f7c96SJohn Baldwin return; 81a15f7c96SJohn Baldwin } 82a15f7c96SJohn Baldwin 83a15f7c96SJohn Baldwin sc_status = nvmf_validate_command_capsule(nc); 84a15f7c96SJohn Baldwin if (sc_status != NVME_SC_SUCCESS) { 85a15f7c96SJohn Baldwin _nvmft_send_generic_error(qp, nc, sc_status); 86a15f7c96SJohn Baldwin nvmf_free_capsule(nc); 87a15f7c96SJohn Baldwin return; 88a15f7c96SJohn Baldwin } 89a15f7c96SJohn Baldwin 90a15f7c96SJohn Baldwin /* Don't bother byte-swapping CID. */ 91a15f7c96SJohn Baldwin if (BIT_TEST_SET_ATOMIC(NUM_CIDS, cmd->cid, qp->cids)) { 92a15f7c96SJohn Baldwin _nvmft_send_generic_error(qp, nc, NVME_SC_COMMAND_ID_CONFLICT); 93a15f7c96SJohn Baldwin nvmf_free_capsule(nc); 94a15f7c96SJohn Baldwin return; 95a15f7c96SJohn Baldwin } 96a15f7c96SJohn Baldwin 97a15f7c96SJohn Baldwin if (qp->admin) 98a15f7c96SJohn Baldwin nvmft_handle_admin_command(ctrlr, nc); 99a15f7c96SJohn Baldwin else 100a15f7c96SJohn Baldwin nvmft_handle_io_command(qp, qp->qid, nc); 101a15f7c96SJohn Baldwin } 102a15f7c96SJohn Baldwin 103a15f7c96SJohn Baldwin struct nvmft_qpair * 104*365b89e8SJohn Baldwin nvmft_qpair_init(enum nvmf_trtype trtype, const nvlist_t *params, uint16_t qid, 105a15f7c96SJohn Baldwin const char *name) 106a15f7c96SJohn Baldwin { 107a15f7c96SJohn Baldwin struct nvmft_qpair *qp; 108a15f7c96SJohn Baldwin 109a15f7c96SJohn Baldwin qp = malloc(sizeof(*qp), M_NVMFT, M_WAITOK | M_ZERO); 110*365b89e8SJohn Baldwin qp->admin = nvlist_get_bool(params, "admin"); 111*365b89e8SJohn Baldwin qp->sq_flow_control = nvlist_get_bool(params, "sq_flow_control"); 112*365b89e8SJohn Baldwin qp->qsize = nvlist_get_number(params, "qsize"); 113a15f7c96SJohn Baldwin qp->qid = qid; 114*365b89e8SJohn Baldwin qp->sqhd = nvlist_get_number(params, "sqhd"); 115a15f7c96SJohn Baldwin strlcpy(qp->name, name, sizeof(qp->name)); 116a15f7c96SJohn Baldwin mtx_init(&qp->lock, "nvmft qp", NULL, MTX_DEF); 117a15f7c96SJohn Baldwin qp->cids = BITSET_ALLOC(NUM_CIDS, M_NVMFT, M_WAITOK | M_ZERO); 1181b3fa1acSJohn Baldwin STAILQ_INIT(&qp->datamove_queue); 1191b3fa1acSJohn Baldwin TASK_INIT(&qp->datamove_task, 0, nvmft_datamove_task, qp); 120a15f7c96SJohn Baldwin 121*365b89e8SJohn Baldwin qp->qp = nvmf_allocate_qpair(trtype, true, params, nvmft_qpair_error, 122a15f7c96SJohn Baldwin qp, nvmft_receive_capsule, qp); 123a15f7c96SJohn Baldwin if (qp->qp == NULL) { 124a15f7c96SJohn Baldwin mtx_destroy(&qp->lock); 125a15f7c96SJohn Baldwin free(qp->cids, M_NVMFT); 126a15f7c96SJohn Baldwin free(qp, M_NVMFT); 127a15f7c96SJohn Baldwin return (NULL); 128a15f7c96SJohn Baldwin } 129a15f7c96SJohn Baldwin 130a15f7c96SJohn Baldwin refcount_init(&qp->qp_refs, 1); 131a15f7c96SJohn Baldwin return (qp); 132a15f7c96SJohn Baldwin } 133a15f7c96SJohn Baldwin 134a15f7c96SJohn Baldwin void 135a15f7c96SJohn Baldwin nvmft_qpair_shutdown(struct nvmft_qpair *qp) 136a15f7c96SJohn Baldwin { 1371b3fa1acSJohn Baldwin STAILQ_HEAD(, ctl_io_hdr) datamove_queue; 138a15f7c96SJohn Baldwin struct nvmf_qpair *nq; 1391b3fa1acSJohn Baldwin union ctl_io *io; 140a15f7c96SJohn Baldwin 1411b3fa1acSJohn Baldwin STAILQ_INIT(&datamove_queue); 142a15f7c96SJohn Baldwin mtx_lock(&qp->lock); 143a15f7c96SJohn Baldwin nq = qp->qp; 144a15f7c96SJohn Baldwin qp->qp = NULL; 1451b3fa1acSJohn Baldwin STAILQ_CONCAT(&datamove_queue, &qp->datamove_queue); 146a15f7c96SJohn Baldwin mtx_unlock(&qp->lock); 147a15f7c96SJohn Baldwin if (nq != NULL && refcount_release(&qp->qp_refs)) 148a15f7c96SJohn Baldwin nvmf_free_qpair(nq); 1491b3fa1acSJohn Baldwin 1501b3fa1acSJohn Baldwin while (!STAILQ_EMPTY(&datamove_queue)) { 1511b3fa1acSJohn Baldwin io = (union ctl_io *)STAILQ_FIRST(&datamove_queue); 1521b3fa1acSJohn Baldwin STAILQ_REMOVE_HEAD(&datamove_queue, links); 1531b3fa1acSJohn Baldwin nvmft_abort_datamove(io); 1541b3fa1acSJohn Baldwin } 1551b3fa1acSJohn Baldwin nvmft_drain_task(&qp->datamove_task); 156a15f7c96SJohn Baldwin } 157a15f7c96SJohn Baldwin 158a15f7c96SJohn Baldwin void 159a15f7c96SJohn Baldwin nvmft_qpair_destroy(struct nvmft_qpair *qp) 160a15f7c96SJohn Baldwin { 161a15f7c96SJohn Baldwin nvmft_qpair_shutdown(qp); 162a15f7c96SJohn Baldwin mtx_destroy(&qp->lock); 163a15f7c96SJohn Baldwin free(qp->cids, M_NVMFT); 164a15f7c96SJohn Baldwin free(qp, M_NVMFT); 165a15f7c96SJohn Baldwin } 166a15f7c96SJohn Baldwin 167a15f7c96SJohn Baldwin struct nvmft_controller * 168a15f7c96SJohn Baldwin nvmft_qpair_ctrlr(struct nvmft_qpair *qp) 169a15f7c96SJohn Baldwin { 170a15f7c96SJohn Baldwin return (qp->ctrlr); 171a15f7c96SJohn Baldwin } 172a15f7c96SJohn Baldwin 173a15f7c96SJohn Baldwin uint16_t 174a15f7c96SJohn Baldwin nvmft_qpair_id(struct nvmft_qpair *qp) 175a15f7c96SJohn Baldwin { 176a15f7c96SJohn Baldwin return (qp->qid); 177a15f7c96SJohn Baldwin } 178a15f7c96SJohn Baldwin 179a15f7c96SJohn Baldwin const char * 180a15f7c96SJohn Baldwin nvmft_qpair_name(struct nvmft_qpair *qp) 181a15f7c96SJohn Baldwin { 182a15f7c96SJohn Baldwin return (qp->name); 183a15f7c96SJohn Baldwin } 184a15f7c96SJohn Baldwin 185a15f7c96SJohn Baldwin static int 186a15f7c96SJohn Baldwin _nvmft_send_response(struct nvmft_qpair *qp, const void *cqe) 187a15f7c96SJohn Baldwin { 188a15f7c96SJohn Baldwin struct nvme_completion cpl; 189a15f7c96SJohn Baldwin struct nvmf_qpair *nq; 190a15f7c96SJohn Baldwin struct nvmf_capsule *rc; 191a15f7c96SJohn Baldwin int error; 192a15f7c96SJohn Baldwin 193a15f7c96SJohn Baldwin memcpy(&cpl, cqe, sizeof(cpl)); 194a15f7c96SJohn Baldwin mtx_lock(&qp->lock); 195a15f7c96SJohn Baldwin nq = qp->qp; 196a15f7c96SJohn Baldwin if (nq == NULL) { 197a15f7c96SJohn Baldwin mtx_unlock(&qp->lock); 198a15f7c96SJohn Baldwin return (ENOTCONN); 199a15f7c96SJohn Baldwin } 200a15f7c96SJohn Baldwin refcount_acquire(&qp->qp_refs); 201a15f7c96SJohn Baldwin 202a15f7c96SJohn Baldwin /* Set SQHD. */ 203a15f7c96SJohn Baldwin if (qp->sq_flow_control) { 204a15f7c96SJohn Baldwin qp->sqhd = (qp->sqhd + 1) % qp->qsize; 205a15f7c96SJohn Baldwin cpl.sqhd = htole16(qp->sqhd); 206a15f7c96SJohn Baldwin } else 207a15f7c96SJohn Baldwin cpl.sqhd = 0; 208a15f7c96SJohn Baldwin mtx_unlock(&qp->lock); 209a15f7c96SJohn Baldwin 210a15f7c96SJohn Baldwin rc = nvmf_allocate_response(nq, &cpl, M_WAITOK); 211a15f7c96SJohn Baldwin error = nvmf_transmit_capsule(rc); 212a15f7c96SJohn Baldwin nvmf_free_capsule(rc); 213a15f7c96SJohn Baldwin 214a15f7c96SJohn Baldwin if (refcount_release(&qp->qp_refs)) 215a15f7c96SJohn Baldwin nvmf_free_qpair(nq); 216a15f7c96SJohn Baldwin return (error); 217a15f7c96SJohn Baldwin } 218a15f7c96SJohn Baldwin 219a15f7c96SJohn Baldwin void 220a15f7c96SJohn Baldwin nvmft_command_completed(struct nvmft_qpair *qp, struct nvmf_capsule *nc) 221a15f7c96SJohn Baldwin { 222a15f7c96SJohn Baldwin const struct nvme_command *cmd = nvmf_capsule_sqe(nc); 223a15f7c96SJohn Baldwin 224a15f7c96SJohn Baldwin /* Don't bother byte-swapping CID. */ 225a15f7c96SJohn Baldwin KASSERT(BIT_ISSET(NUM_CIDS, cmd->cid, qp->cids), 226a15f7c96SJohn Baldwin ("%s: CID %u not busy", __func__, cmd->cid)); 227a15f7c96SJohn Baldwin 228a15f7c96SJohn Baldwin BIT_CLR_ATOMIC(NUM_CIDS, cmd->cid, qp->cids); 229a15f7c96SJohn Baldwin } 230a15f7c96SJohn Baldwin 231a15f7c96SJohn Baldwin int 232a15f7c96SJohn Baldwin nvmft_send_response(struct nvmft_qpair *qp, const void *cqe) 233a15f7c96SJohn Baldwin { 234a15f7c96SJohn Baldwin const struct nvme_completion *cpl = cqe; 235a15f7c96SJohn Baldwin 236a15f7c96SJohn Baldwin /* Don't bother byte-swapping CID. */ 237a15f7c96SJohn Baldwin KASSERT(BIT_ISSET(NUM_CIDS, cpl->cid, qp->cids), 238a15f7c96SJohn Baldwin ("%s: CID %u not busy", __func__, cpl->cid)); 239a15f7c96SJohn Baldwin 240a15f7c96SJohn Baldwin BIT_CLR_ATOMIC(NUM_CIDS, cpl->cid, qp->cids); 241a15f7c96SJohn Baldwin return (_nvmft_send_response(qp, cqe)); 242a15f7c96SJohn Baldwin } 243a15f7c96SJohn Baldwin 244a15f7c96SJohn Baldwin void 245a15f7c96SJohn Baldwin nvmft_init_cqe(void *cqe, struct nvmf_capsule *nc, uint16_t status) 246a15f7c96SJohn Baldwin { 247a15f7c96SJohn Baldwin struct nvme_completion *cpl = cqe; 248a15f7c96SJohn Baldwin const struct nvme_command *cmd = nvmf_capsule_sqe(nc); 249a15f7c96SJohn Baldwin 250a15f7c96SJohn Baldwin memset(cpl, 0, sizeof(*cpl)); 251a15f7c96SJohn Baldwin cpl->cid = cmd->cid; 252a15f7c96SJohn Baldwin cpl->status = htole16(status); 253a15f7c96SJohn Baldwin } 254a15f7c96SJohn Baldwin 255a15f7c96SJohn Baldwin int 256a15f7c96SJohn Baldwin nvmft_send_error(struct nvmft_qpair *qp, struct nvmf_capsule *nc, 257a15f7c96SJohn Baldwin uint8_t sc_type, uint8_t sc_status) 258a15f7c96SJohn Baldwin { 259a15f7c96SJohn Baldwin struct nvme_completion cpl; 260a15f7c96SJohn Baldwin uint16_t status; 261a15f7c96SJohn Baldwin 262a15f7c96SJohn Baldwin status = NVMEF(NVME_STATUS_SCT, sc_type) | 263a15f7c96SJohn Baldwin NVMEF(NVME_STATUS_SC, sc_status); 264a15f7c96SJohn Baldwin nvmft_init_cqe(&cpl, nc, status); 265a15f7c96SJohn Baldwin return (nvmft_send_response(qp, &cpl)); 266a15f7c96SJohn Baldwin } 267a15f7c96SJohn Baldwin 268a15f7c96SJohn Baldwin int 269a15f7c96SJohn Baldwin nvmft_send_generic_error(struct nvmft_qpair *qp, struct nvmf_capsule *nc, 270a15f7c96SJohn Baldwin uint8_t sc_status) 271a15f7c96SJohn Baldwin { 272a15f7c96SJohn Baldwin return (nvmft_send_error(qp, nc, NVME_SCT_GENERIC, sc_status)); 273a15f7c96SJohn Baldwin } 274a15f7c96SJohn Baldwin 275a15f7c96SJohn Baldwin /* 276a15f7c96SJohn Baldwin * This version doesn't clear CID in qp->cids and is used for errors 277a15f7c96SJohn Baldwin * before the CID is validated. 278a15f7c96SJohn Baldwin */ 279a15f7c96SJohn Baldwin static int 280a15f7c96SJohn Baldwin _nvmft_send_generic_error(struct nvmft_qpair *qp, struct nvmf_capsule *nc, 281a15f7c96SJohn Baldwin uint8_t sc_status) 282a15f7c96SJohn Baldwin { 283a15f7c96SJohn Baldwin struct nvme_completion cpl; 284a15f7c96SJohn Baldwin uint16_t status; 285a15f7c96SJohn Baldwin 286a15f7c96SJohn Baldwin status = NVMEF(NVME_STATUS_SCT, NVME_SCT_GENERIC) | 287a15f7c96SJohn Baldwin NVMEF(NVME_STATUS_SC, sc_status); 288a15f7c96SJohn Baldwin nvmft_init_cqe(&cpl, nc, status); 289a15f7c96SJohn Baldwin return (_nvmft_send_response(qp, &cpl)); 290a15f7c96SJohn Baldwin } 291a15f7c96SJohn Baldwin 292a15f7c96SJohn Baldwin int 293a15f7c96SJohn Baldwin nvmft_send_success(struct nvmft_qpair *qp, struct nvmf_capsule *nc) 294a15f7c96SJohn Baldwin { 295a15f7c96SJohn Baldwin return (nvmft_send_generic_error(qp, nc, NVME_SC_SUCCESS)); 296a15f7c96SJohn Baldwin } 297a15f7c96SJohn Baldwin 298a15f7c96SJohn Baldwin static void 299a15f7c96SJohn Baldwin nvmft_init_connect_rsp(struct nvmf_fabric_connect_rsp *rsp, 300a15f7c96SJohn Baldwin const struct nvmf_fabric_connect_cmd *cmd, uint16_t status) 301a15f7c96SJohn Baldwin { 302a15f7c96SJohn Baldwin memset(rsp, 0, sizeof(*rsp)); 303a15f7c96SJohn Baldwin rsp->cid = cmd->cid; 304a15f7c96SJohn Baldwin rsp->status = htole16(status); 305a15f7c96SJohn Baldwin } 306a15f7c96SJohn Baldwin 307a15f7c96SJohn Baldwin static int 308a15f7c96SJohn Baldwin nvmft_send_connect_response(struct nvmft_qpair *qp, 309a15f7c96SJohn Baldwin const struct nvmf_fabric_connect_rsp *rsp) 310a15f7c96SJohn Baldwin { 311a15f7c96SJohn Baldwin struct nvmf_capsule *rc; 312a15f7c96SJohn Baldwin struct nvmf_qpair *nq; 313a15f7c96SJohn Baldwin int error; 314a15f7c96SJohn Baldwin 315a15f7c96SJohn Baldwin mtx_lock(&qp->lock); 316a15f7c96SJohn Baldwin nq = qp->qp; 317a15f7c96SJohn Baldwin if (nq == NULL) { 318a15f7c96SJohn Baldwin mtx_unlock(&qp->lock); 319a15f7c96SJohn Baldwin return (ENOTCONN); 320a15f7c96SJohn Baldwin } 321a15f7c96SJohn Baldwin refcount_acquire(&qp->qp_refs); 322a15f7c96SJohn Baldwin mtx_unlock(&qp->lock); 323a15f7c96SJohn Baldwin 324a15f7c96SJohn Baldwin rc = nvmf_allocate_response(qp->qp, rsp, M_WAITOK); 325a15f7c96SJohn Baldwin error = nvmf_transmit_capsule(rc); 326a15f7c96SJohn Baldwin nvmf_free_capsule(rc); 327a15f7c96SJohn Baldwin 328a15f7c96SJohn Baldwin if (refcount_release(&qp->qp_refs)) 329a15f7c96SJohn Baldwin nvmf_free_qpair(nq); 330a15f7c96SJohn Baldwin return (error); 331a15f7c96SJohn Baldwin } 332a15f7c96SJohn Baldwin 333a15f7c96SJohn Baldwin void 334a15f7c96SJohn Baldwin nvmft_connect_error(struct nvmft_qpair *qp, 335a15f7c96SJohn Baldwin const struct nvmf_fabric_connect_cmd *cmd, uint8_t sc_type, 336a15f7c96SJohn Baldwin uint8_t sc_status) 337a15f7c96SJohn Baldwin { 338a15f7c96SJohn Baldwin struct nvmf_fabric_connect_rsp rsp; 339a15f7c96SJohn Baldwin uint16_t status; 340a15f7c96SJohn Baldwin 341a15f7c96SJohn Baldwin status = NVMEF(NVME_STATUS_SCT, sc_type) | 342a15f7c96SJohn Baldwin NVMEF(NVME_STATUS_SC, sc_status); 343a15f7c96SJohn Baldwin nvmft_init_connect_rsp(&rsp, cmd, status); 344a15f7c96SJohn Baldwin nvmft_send_connect_response(qp, &rsp); 345a15f7c96SJohn Baldwin } 346a15f7c96SJohn Baldwin 347a15f7c96SJohn Baldwin void 348a15f7c96SJohn Baldwin nvmft_connect_invalid_parameters(struct nvmft_qpair *qp, 349a15f7c96SJohn Baldwin const struct nvmf_fabric_connect_cmd *cmd, bool data, uint16_t offset) 350a15f7c96SJohn Baldwin { 351a15f7c96SJohn Baldwin struct nvmf_fabric_connect_rsp rsp; 352a15f7c96SJohn Baldwin 353a15f7c96SJohn Baldwin nvmft_init_connect_rsp(&rsp, cmd, 354a15f7c96SJohn Baldwin NVMEF(NVME_STATUS_SCT, NVME_SCT_COMMAND_SPECIFIC) | 355a15f7c96SJohn Baldwin NVMEF(NVME_STATUS_SC, NVMF_FABRIC_SC_INVALID_PARAM)); 356a15f7c96SJohn Baldwin rsp.status_code_specific.invalid.ipo = htole16(offset); 357a15f7c96SJohn Baldwin rsp.status_code_specific.invalid.iattr = data ? 1 : 0; 358a15f7c96SJohn Baldwin nvmft_send_connect_response(qp, &rsp); 359a15f7c96SJohn Baldwin } 360a15f7c96SJohn Baldwin 361a15f7c96SJohn Baldwin int 362a15f7c96SJohn Baldwin nvmft_finish_accept(struct nvmft_qpair *qp, 363a15f7c96SJohn Baldwin const struct nvmf_fabric_connect_cmd *cmd, struct nvmft_controller *ctrlr) 364a15f7c96SJohn Baldwin { 365a15f7c96SJohn Baldwin struct nvmf_fabric_connect_rsp rsp; 366a15f7c96SJohn Baldwin 367a15f7c96SJohn Baldwin qp->ctrlr = ctrlr; 368a15f7c96SJohn Baldwin nvmft_init_connect_rsp(&rsp, cmd, 0); 369a15f7c96SJohn Baldwin if (qp->sq_flow_control) 370a15f7c96SJohn Baldwin rsp.sqhd = htole16(qp->sqhd); 371a15f7c96SJohn Baldwin else 372a15f7c96SJohn Baldwin rsp.sqhd = htole16(0xffff); 373a15f7c96SJohn Baldwin rsp.status_code_specific.success.cntlid = htole16(ctrlr->cntlid); 374a15f7c96SJohn Baldwin return (nvmft_send_connect_response(qp, &rsp)); 375a15f7c96SJohn Baldwin } 3761b3fa1acSJohn Baldwin 3771b3fa1acSJohn Baldwin void 3781b3fa1acSJohn Baldwin nvmft_qpair_datamove(struct nvmft_qpair *qp, union ctl_io *io) 3791b3fa1acSJohn Baldwin { 3801b3fa1acSJohn Baldwin bool enqueue_task; 3811b3fa1acSJohn Baldwin 3821b3fa1acSJohn Baldwin mtx_lock(&qp->lock); 3831b3fa1acSJohn Baldwin if (qp->qp == NULL) { 3841b3fa1acSJohn Baldwin mtx_unlock(&qp->lock); 3851b3fa1acSJohn Baldwin nvmft_abort_datamove(io); 3861b3fa1acSJohn Baldwin return; 3871b3fa1acSJohn Baldwin } 3881b3fa1acSJohn Baldwin enqueue_task = STAILQ_EMPTY(&qp->datamove_queue); 3891b3fa1acSJohn Baldwin STAILQ_INSERT_TAIL(&qp->datamove_queue, &io->io_hdr, links); 3901b3fa1acSJohn Baldwin mtx_unlock(&qp->lock); 3911b3fa1acSJohn Baldwin if (enqueue_task) 3921b3fa1acSJohn Baldwin nvmft_enqueue_task(&qp->datamove_task); 3931b3fa1acSJohn Baldwin } 3941b3fa1acSJohn Baldwin 3951b3fa1acSJohn Baldwin static void 3961b3fa1acSJohn Baldwin nvmft_datamove_task(void *context, int pending __unused) 3971b3fa1acSJohn Baldwin { 3981b3fa1acSJohn Baldwin struct nvmft_qpair *qp = context; 3991b3fa1acSJohn Baldwin union ctl_io *io; 4001b3fa1acSJohn Baldwin bool abort; 4011b3fa1acSJohn Baldwin 4021b3fa1acSJohn Baldwin mtx_lock(&qp->lock); 4031b3fa1acSJohn Baldwin while (!STAILQ_EMPTY(&qp->datamove_queue)) { 4041b3fa1acSJohn Baldwin io = (union ctl_io *)STAILQ_FIRST(&qp->datamove_queue); 4051b3fa1acSJohn Baldwin STAILQ_REMOVE_HEAD(&qp->datamove_queue, links); 4061b3fa1acSJohn Baldwin abort = (qp->qp == NULL); 4071b3fa1acSJohn Baldwin mtx_unlock(&qp->lock); 4081b3fa1acSJohn Baldwin if (abort) 4091b3fa1acSJohn Baldwin nvmft_abort_datamove(io); 4101b3fa1acSJohn Baldwin else 4111b3fa1acSJohn Baldwin nvmft_handle_datamove(io); 4121b3fa1acSJohn Baldwin mtx_lock(&qp->lock); 4131b3fa1acSJohn Baldwin } 4141b3fa1acSJohn Baldwin mtx_unlock(&qp->lock); 4151b3fa1acSJohn Baldwin } 416