1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) Intel Corporation. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include "nvme_internal.h" 35 #include "spdk/nvme_ocssd.h" 36 37 static int nvme_qpair_resubmit_request(struct spdk_nvme_qpair *qpair, struct nvme_request *req); 38 39 struct nvme_string { 40 uint16_t value; 41 const char *str; 42 }; 43 44 static const struct nvme_string admin_opcode[] = { 45 { SPDK_NVME_OPC_DELETE_IO_SQ, "DELETE IO SQ" }, 46 { SPDK_NVME_OPC_CREATE_IO_SQ, "CREATE IO SQ" }, 47 { SPDK_NVME_OPC_GET_LOG_PAGE, "GET LOG PAGE" }, 48 { SPDK_NVME_OPC_DELETE_IO_CQ, "DELETE IO CQ" }, 49 { SPDK_NVME_OPC_CREATE_IO_CQ, "CREATE IO CQ" }, 50 { SPDK_NVME_OPC_IDENTIFY, "IDENTIFY" }, 51 { SPDK_NVME_OPC_ABORT, "ABORT" }, 52 { SPDK_NVME_OPC_SET_FEATURES, "SET FEATURES" }, 53 { SPDK_NVME_OPC_GET_FEATURES, "GET FEATURES" }, 54 { SPDK_NVME_OPC_ASYNC_EVENT_REQUEST, "ASYNC EVENT REQUEST" }, 55 { SPDK_NVME_OPC_NS_MANAGEMENT, "NAMESPACE MANAGEMENT" }, 56 { SPDK_NVME_OPC_FIRMWARE_COMMIT, "FIRMWARE COMMIT" }, 57 { SPDK_NVME_OPC_FIRMWARE_IMAGE_DOWNLOAD, "FIRMWARE IMAGE DOWNLOAD" }, 58 { SPDK_NVME_OPC_DEVICE_SELF_TEST, "DEVICE SELF-TEST" }, 59 { SPDK_NVME_OPC_NS_ATTACHMENT, "NAMESPACE ATTACHMENT" }, 60 { SPDK_NVME_OPC_KEEP_ALIVE, "KEEP ALIVE" }, 61 { SPDK_NVME_OPC_DIRECTIVE_SEND, "DIRECTIVE SEND" }, 62 { SPDK_NVME_OPC_DIRECTIVE_RECEIVE, "DIRECTIVE RECEIVE" }, 63 { SPDK_NVME_OPC_VIRTUALIZATION_MANAGEMENT, "VIRTUALIZATION MANAGEMENT" }, 64 { SPDK_NVME_OPC_NVME_MI_SEND, "NVME-MI SEND" }, 65 { SPDK_NVME_OPC_NVME_MI_RECEIVE, "NVME-MI RECEIVE" }, 66 { SPDK_NVME_OPC_DOORBELL_BUFFER_CONFIG, "DOORBELL BUFFER CONFIG" }, 67 { SPDK_NVME_OPC_FORMAT_NVM, "FORMAT NVM" }, 68 { SPDK_NVME_OPC_SECURITY_SEND, "SECURITY SEND" }, 69 { SPDK_NVME_OPC_SECURITY_RECEIVE, "SECURITY RECEIVE" }, 70 { SPDK_NVME_OPC_SANITIZE, "SANITIZE" }, 71 { SPDK_OCSSD_OPC_GEOMETRY, "OCSSD / GEOMETRY" }, 72 { 0xFFFF, "ADMIN COMMAND" } 73 }; 74 75 static const struct nvme_string io_opcode[] = { 76 { SPDK_NVME_OPC_FLUSH, "FLUSH" }, 77 { SPDK_NVME_OPC_WRITE, "WRITE" }, 78 { SPDK_NVME_OPC_READ, "READ" }, 79 { SPDK_NVME_OPC_WRITE_UNCORRECTABLE, "WRITE UNCORRECTABLE" }, 80 { SPDK_NVME_OPC_COMPARE, "COMPARE" }, 81 { SPDK_NVME_OPC_WRITE_ZEROES, "WRITE ZEROES" }, 82 { SPDK_NVME_OPC_DATASET_MANAGEMENT, "DATASET MANAGEMENT" }, 83 { SPDK_NVME_OPC_RESERVATION_REGISTER, "RESERVATION REGISTER" }, 84 { SPDK_NVME_OPC_RESERVATION_REPORT, "RESERVATION REPORT" }, 85 { SPDK_NVME_OPC_RESERVATION_ACQUIRE, "RESERVATION ACQUIRE" }, 86 { SPDK_NVME_OPC_RESERVATION_RELEASE, "RESERVATION RELEASE" }, 87 { SPDK_OCSSD_OPC_VECTOR_RESET, "OCSSD / VECTOR RESET" }, 88 { SPDK_OCSSD_OPC_VECTOR_WRITE, "OCSSD / VECTOR WRITE" }, 89 { SPDK_OCSSD_OPC_VECTOR_READ, "OCSSD / VECTOR READ" }, 90 { SPDK_OCSSD_OPC_VECTOR_COPY, "OCSSD / VECTOR COPY" }, 91 { 0xFFFF, "IO COMMAND" } 92 }; 93 94 static const char * 95 nvme_get_string(const struct nvme_string *strings, uint16_t value) 96 { 97 const struct nvme_string *entry; 98 99 entry = strings; 100 101 while (entry->value != 0xFFFF) { 102 if (entry->value == value) { 103 return entry->str; 104 } 105 entry++; 106 } 107 return entry->str; 108 } 109 110 static void 111 nvme_admin_qpair_print_command(struct spdk_nvme_qpair *qpair, 112 struct spdk_nvme_cmd *cmd) 113 { 114 115 SPDK_NOTICELOG("%s (%02x) sqid:%d cid:%d nsid:%x " 116 "cdw10:%08x cdw11:%08x\n", 117 nvme_get_string(admin_opcode, cmd->opc), cmd->opc, qpair->id, cmd->cid, 118 cmd->nsid, cmd->cdw10, cmd->cdw11); 119 } 120 121 static void 122 nvme_io_qpair_print_command(struct spdk_nvme_qpair *qpair, 123 struct spdk_nvme_cmd *cmd) 124 { 125 assert(qpair != NULL); 126 assert(cmd != NULL); 127 switch ((int)cmd->opc) { 128 case SPDK_NVME_OPC_WRITE: 129 case SPDK_NVME_OPC_READ: 130 case SPDK_NVME_OPC_WRITE_UNCORRECTABLE: 131 case SPDK_NVME_OPC_COMPARE: 132 SPDK_NOTICELOG("%s sqid:%d cid:%d nsid:%d " 133 "lba:%llu len:%d\n", 134 nvme_get_string(io_opcode, cmd->opc), qpair->id, cmd->cid, 135 cmd->nsid, 136 ((unsigned long long)cmd->cdw11 << 32) + cmd->cdw10, 137 (cmd->cdw12 & 0xFFFF) + 1); 138 break; 139 case SPDK_NVME_OPC_FLUSH: 140 case SPDK_NVME_OPC_DATASET_MANAGEMENT: 141 SPDK_NOTICELOG("%s sqid:%d cid:%d nsid:%d\n", 142 nvme_get_string(io_opcode, cmd->opc), qpair->id, cmd->cid, 143 cmd->nsid); 144 break; 145 default: 146 SPDK_NOTICELOG("%s (%02x) sqid:%d cid:%d nsid:%d\n", 147 nvme_get_string(io_opcode, cmd->opc), cmd->opc, qpair->id, 148 cmd->cid, cmd->nsid); 149 break; 150 } 151 } 152 153 void 154 spdk_nvme_qpair_print_command(struct spdk_nvme_qpair *qpair, struct spdk_nvme_cmd *cmd) 155 { 156 assert(qpair != NULL); 157 assert(cmd != NULL); 158 159 if (nvme_qpair_is_admin_queue(qpair)) { 160 nvme_admin_qpair_print_command(qpair, cmd); 161 } else { 162 nvme_io_qpair_print_command(qpair, cmd); 163 } 164 } 165 166 static const struct nvme_string generic_status[] = { 167 { SPDK_NVME_SC_SUCCESS, "SUCCESS" }, 168 { SPDK_NVME_SC_INVALID_OPCODE, "INVALID OPCODE" }, 169 { SPDK_NVME_SC_INVALID_FIELD, "INVALID FIELD" }, 170 { SPDK_NVME_SC_COMMAND_ID_CONFLICT, "COMMAND ID CONFLICT" }, 171 { SPDK_NVME_SC_DATA_TRANSFER_ERROR, "DATA TRANSFER ERROR" }, 172 { SPDK_NVME_SC_ABORTED_POWER_LOSS, "ABORTED - POWER LOSS" }, 173 { SPDK_NVME_SC_INTERNAL_DEVICE_ERROR, "INTERNAL DEVICE ERROR" }, 174 { SPDK_NVME_SC_ABORTED_BY_REQUEST, "ABORTED - BY REQUEST" }, 175 { SPDK_NVME_SC_ABORTED_SQ_DELETION, "ABORTED - SQ DELETION" }, 176 { SPDK_NVME_SC_ABORTED_FAILED_FUSED, "ABORTED - FAILED FUSED" }, 177 { SPDK_NVME_SC_ABORTED_MISSING_FUSED, "ABORTED - MISSING FUSED" }, 178 { SPDK_NVME_SC_INVALID_NAMESPACE_OR_FORMAT, "INVALID NAMESPACE OR FORMAT" }, 179 { SPDK_NVME_SC_COMMAND_SEQUENCE_ERROR, "COMMAND SEQUENCE ERROR" }, 180 { SPDK_NVME_SC_INVALID_SGL_SEG_DESCRIPTOR, "INVALID SGL SEGMENT DESCRIPTOR" }, 181 { SPDK_NVME_SC_INVALID_NUM_SGL_DESCIRPTORS, "INVALID NUMBER OF SGL DESCRIPTORS" }, 182 { SPDK_NVME_SC_DATA_SGL_LENGTH_INVALID, "DATA SGL LENGTH INVALID" }, 183 { SPDK_NVME_SC_METADATA_SGL_LENGTH_INVALID, "METADATA SGL LENGTH INVALID" }, 184 { SPDK_NVME_SC_SGL_DESCRIPTOR_TYPE_INVALID, "SGL DESCRIPTOR TYPE INVALID" }, 185 { SPDK_NVME_SC_INVALID_CONTROLLER_MEM_BUF, "INVALID CONTROLLER MEMORY BUFFER" }, 186 { SPDK_NVME_SC_INVALID_PRP_OFFSET, "INVALID PRP OFFSET" }, 187 { SPDK_NVME_SC_ATOMIC_WRITE_UNIT_EXCEEDED, "ATOMIC WRITE UNIT EXCEEDED" }, 188 { SPDK_NVME_SC_OPERATION_DENIED, "OPERATION DENIED" }, 189 { SPDK_NVME_SC_INVALID_SGL_OFFSET, "INVALID SGL OFFSET" }, 190 { SPDK_NVME_SC_HOSTID_INCONSISTENT_FORMAT, "HOSTID INCONSISTENT FORMAT" }, 191 { SPDK_NVME_SC_KEEP_ALIVE_EXPIRED, "KEEP ALIVE EXPIRED" }, 192 { SPDK_NVME_SC_KEEP_ALIVE_INVALID, "KEEP ALIVE INVALID" }, 193 { SPDK_NVME_SC_ABORTED_PREEMPT, "ABORTED - PREEMPT AND ABORT" }, 194 { SPDK_NVME_SC_SANITIZE_FAILED, "SANITIZE FAILED" }, 195 { SPDK_NVME_SC_SANITIZE_IN_PROGRESS, "SANITIZE IN PROGRESS" }, 196 { SPDK_NVME_SC_SGL_DATA_BLOCK_GRANULARITY_INVALID, "DATA BLOCK GRANULARITY INVALID" }, 197 { SPDK_NVME_SC_COMMAND_INVALID_IN_CMB, "COMMAND NOT SUPPORTED FOR QUEUE IN CMB" }, 198 { SPDK_NVME_SC_LBA_OUT_OF_RANGE, "LBA OUT OF RANGE" }, 199 { SPDK_NVME_SC_CAPACITY_EXCEEDED, "CAPACITY EXCEEDED" }, 200 { SPDK_NVME_SC_NAMESPACE_NOT_READY, "NAMESPACE NOT READY" }, 201 { SPDK_NVME_SC_RESERVATION_CONFLICT, "RESERVATION CONFLICT" }, 202 { SPDK_NVME_SC_FORMAT_IN_PROGRESS, "FORMAT IN PROGRESS" }, 203 { 0xFFFF, "GENERIC" } 204 }; 205 206 static const struct nvme_string command_specific_status[] = { 207 { SPDK_NVME_SC_COMPLETION_QUEUE_INVALID, "INVALID COMPLETION QUEUE" }, 208 { SPDK_NVME_SC_INVALID_QUEUE_IDENTIFIER, "INVALID QUEUE IDENTIFIER" }, 209 { SPDK_NVME_SC_INVALID_QUEUE_SIZE, "INVALID QUEUE SIZE" }, 210 { SPDK_NVME_SC_ABORT_COMMAND_LIMIT_EXCEEDED, "ABORT CMD LIMIT EXCEEDED" }, 211 { SPDK_NVME_SC_ASYNC_EVENT_REQUEST_LIMIT_EXCEEDED, "ASYNC LIMIT EXCEEDED" }, 212 { SPDK_NVME_SC_INVALID_FIRMWARE_SLOT, "INVALID FIRMWARE SLOT" }, 213 { SPDK_NVME_SC_INVALID_FIRMWARE_IMAGE, "INVALID FIRMWARE IMAGE" }, 214 { SPDK_NVME_SC_INVALID_INTERRUPT_VECTOR, "INVALID INTERRUPT VECTOR" }, 215 { SPDK_NVME_SC_INVALID_LOG_PAGE, "INVALID LOG PAGE" }, 216 { SPDK_NVME_SC_INVALID_FORMAT, "INVALID FORMAT" }, 217 { SPDK_NVME_SC_FIRMWARE_REQ_CONVENTIONAL_RESET, "FIRMWARE REQUIRES CONVENTIONAL RESET" }, 218 { SPDK_NVME_SC_INVALID_QUEUE_DELETION, "INVALID QUEUE DELETION" }, 219 { SPDK_NVME_SC_FEATURE_ID_NOT_SAVEABLE, "FEATURE ID NOT SAVEABLE" }, 220 { SPDK_NVME_SC_FEATURE_NOT_CHANGEABLE, "FEATURE NOT CHANGEABLE" }, 221 { SPDK_NVME_SC_FEATURE_NOT_NAMESPACE_SPECIFIC, "FEATURE NOT NAMESPACE SPECIFIC" }, 222 { SPDK_NVME_SC_FIRMWARE_REQ_NVM_RESET, "FIRMWARE REQUIRES NVM RESET" }, 223 { SPDK_NVME_SC_FIRMWARE_REQ_RESET, "FIRMWARE REQUIRES RESET" }, 224 { SPDK_NVME_SC_FIRMWARE_REQ_MAX_TIME_VIOLATION, "FIRMWARE REQUIRES MAX TIME VIOLATION" }, 225 { SPDK_NVME_SC_FIRMWARE_ACTIVATION_PROHIBITED, "FIRMWARE ACTIVATION PROHIBITED" }, 226 { SPDK_NVME_SC_OVERLAPPING_RANGE, "OVERLAPPING RANGE" }, 227 { SPDK_NVME_SC_NAMESPACE_INSUFFICIENT_CAPACITY, "NAMESPACE INSUFFICIENT CAPACITY" }, 228 { SPDK_NVME_SC_NAMESPACE_ID_UNAVAILABLE, "NAMESPACE ID UNAVAILABLE" }, 229 { SPDK_NVME_SC_NAMESPACE_ALREADY_ATTACHED, "NAMESPACE ALREADY ATTACHED" }, 230 { SPDK_NVME_SC_NAMESPACE_IS_PRIVATE, "NAMESPACE IS PRIVATE" }, 231 { SPDK_NVME_SC_NAMESPACE_NOT_ATTACHED, "NAMESPACE NOT ATTACHED" }, 232 { SPDK_NVME_SC_THINPROVISIONING_NOT_SUPPORTED, "THINPROVISIONING NOT SUPPORTED" }, 233 { SPDK_NVME_SC_CONTROLLER_LIST_INVALID, "CONTROLLER LIST INVALID" }, 234 { SPDK_NVME_SC_DEVICE_SELF_TEST_IN_PROGRESS, "DEVICE SELF-TEST IN PROGRESS" }, 235 { SPDK_NVME_SC_BOOT_PARTITION_WRITE_PROHIBITED, "BOOT PARTITION WRITE PROHIBITED" }, 236 { SPDK_NVME_SC_INVALID_CTRLR_ID, "INVALID CONTROLLER ID" }, 237 { SPDK_NVME_SC_INVALID_SECONDARY_CTRLR_STATE, "INVALID SECONDARY CONTROLLER STATE" }, 238 { SPDK_NVME_SC_INVALID_NUM_CTRLR_RESOURCES, "INVALID NUMBER OF CONTROLLER RESOURCES" }, 239 { SPDK_NVME_SC_INVALID_RESOURCE_ID, "INVALID RESOURCE IDENTIFIER" }, 240 { SPDK_NVME_SC_CONFLICTING_ATTRIBUTES, "CONFLICTING ATTRIBUTES" }, 241 { SPDK_NVME_SC_INVALID_PROTECTION_INFO, "INVALID PROTECTION INFO" }, 242 { SPDK_NVME_SC_ATTEMPTED_WRITE_TO_RO_RANGE, "WRITE TO RO RANGE" }, 243 { 0xFFFF, "COMMAND SPECIFIC" } 244 }; 245 246 static const struct nvme_string media_error_status[] = { 247 { SPDK_NVME_SC_WRITE_FAULTS, "WRITE FAULTS" }, 248 { SPDK_NVME_SC_UNRECOVERED_READ_ERROR, "UNRECOVERED READ ERROR" }, 249 { SPDK_NVME_SC_GUARD_CHECK_ERROR, "GUARD CHECK ERROR" }, 250 { SPDK_NVME_SC_APPLICATION_TAG_CHECK_ERROR, "APPLICATION TAG CHECK ERROR" }, 251 { SPDK_NVME_SC_REFERENCE_TAG_CHECK_ERROR, "REFERENCE TAG CHECK ERROR" }, 252 { SPDK_NVME_SC_COMPARE_FAILURE, "COMPARE FAILURE" }, 253 { SPDK_NVME_SC_ACCESS_DENIED, "ACCESS DENIED" }, 254 { SPDK_NVME_SC_DEALLOCATED_OR_UNWRITTEN_BLOCK, "DEALLOCATED OR UNWRITTEN BLOCK" }, 255 { SPDK_OCSSD_SC_OFFLINE_CHUNK, "RESET OFFLINE CHUNK" }, 256 { SPDK_OCSSD_SC_INVALID_RESET, "INVALID RESET" }, 257 { SPDK_OCSSD_SC_WRITE_FAIL_WRITE_NEXT_UNIT, "WRITE FAIL WRITE NEXT UNIT" }, 258 { SPDK_OCSSD_SC_WRITE_FAIL_CHUNK_EARLY_CLOSE, "WRITE FAIL CHUNK EARLY CLOSE" }, 259 { SPDK_OCSSD_SC_OUT_OF_ORDER_WRITE, "OUT OF ORDER WRITE" }, 260 { SPDK_OCSSD_SC_READ_HIGH_ECC, "READ HIGH ECC" }, 261 { 0xFFFF, "MEDIA ERROR" } 262 }; 263 264 static const struct nvme_string path_status[] = { 265 { SPDK_NVME_SC_INTERNAL_PATH_ERROR, "INTERNAL PATH ERROR" }, 266 { SPDK_NVME_SC_CONTROLLER_PATH_ERROR, "CONTROLLER PATH ERROR" }, 267 { SPDK_NVME_SC_HOST_PATH_ERROR, "HOST PATH ERROR" }, 268 { SPDK_NVME_SC_ABORTED_BY_HOST, "ABORTED BY HOST" }, 269 { 0xFFFF, "PATH ERROR" } 270 }; 271 272 const char * 273 spdk_nvme_cpl_get_status_string(const struct spdk_nvme_status *status) 274 { 275 const struct nvme_string *entry; 276 277 switch (status->sct) { 278 case SPDK_NVME_SCT_GENERIC: 279 entry = generic_status; 280 break; 281 case SPDK_NVME_SCT_COMMAND_SPECIFIC: 282 entry = command_specific_status; 283 break; 284 case SPDK_NVME_SCT_MEDIA_ERROR: 285 entry = media_error_status; 286 break; 287 case SPDK_NVME_SCT_PATH: 288 entry = path_status; 289 break; 290 case SPDK_NVME_SCT_VENDOR_SPECIFIC: 291 return "VENDOR SPECIFIC"; 292 default: 293 return "RESERVED"; 294 } 295 296 return nvme_get_string(entry, status->sc); 297 } 298 299 void 300 spdk_nvme_qpair_print_completion(struct spdk_nvme_qpair *qpair, 301 struct spdk_nvme_cpl *cpl) 302 { 303 SPDK_NOTICELOG("%s (%02x/%02x) sqid:%d cid:%d cdw0:%x sqhd:%04x p:%x m:%x dnr:%x\n", 304 spdk_nvme_cpl_get_status_string(&cpl->status), 305 cpl->status.sct, cpl->status.sc, cpl->sqid, cpl->cid, cpl->cdw0, 306 cpl->sqhd, cpl->status.p, cpl->status.m, cpl->status.dnr); 307 } 308 309 bool 310 nvme_completion_is_retry(const struct spdk_nvme_cpl *cpl) 311 { 312 /* 313 * TODO: spec is not clear how commands that are aborted due 314 * to TLER will be marked. So for now, it seems 315 * NAMESPACE_NOT_READY is the only case where we should 316 * look at the DNR bit. 317 */ 318 switch ((int)cpl->status.sct) { 319 case SPDK_NVME_SCT_GENERIC: 320 switch ((int)cpl->status.sc) { 321 case SPDK_NVME_SC_NAMESPACE_NOT_READY: 322 case SPDK_NVME_SC_FORMAT_IN_PROGRESS: 323 if (cpl->status.dnr) { 324 return false; 325 } else { 326 return true; 327 } 328 case SPDK_NVME_SC_INVALID_OPCODE: 329 case SPDK_NVME_SC_INVALID_FIELD: 330 case SPDK_NVME_SC_COMMAND_ID_CONFLICT: 331 case SPDK_NVME_SC_DATA_TRANSFER_ERROR: 332 case SPDK_NVME_SC_ABORTED_POWER_LOSS: 333 case SPDK_NVME_SC_INTERNAL_DEVICE_ERROR: 334 case SPDK_NVME_SC_ABORTED_BY_REQUEST: 335 case SPDK_NVME_SC_ABORTED_SQ_DELETION: 336 case SPDK_NVME_SC_ABORTED_FAILED_FUSED: 337 case SPDK_NVME_SC_ABORTED_MISSING_FUSED: 338 case SPDK_NVME_SC_INVALID_NAMESPACE_OR_FORMAT: 339 case SPDK_NVME_SC_COMMAND_SEQUENCE_ERROR: 340 case SPDK_NVME_SC_LBA_OUT_OF_RANGE: 341 case SPDK_NVME_SC_CAPACITY_EXCEEDED: 342 default: 343 return false; 344 } 345 case SPDK_NVME_SCT_PATH: 346 /* 347 * Per NVMe TP 4028 (Path and Transport Error Enhancements), retries should be 348 * based on the setting of the DNR bit for Internal Path Error 349 */ 350 switch ((int)cpl->status.sc) { 351 case SPDK_NVME_SC_INTERNAL_PATH_ERROR: 352 return !cpl->status.dnr; 353 default: 354 return false; 355 } 356 case SPDK_NVME_SCT_COMMAND_SPECIFIC: 357 case SPDK_NVME_SCT_MEDIA_ERROR: 358 case SPDK_NVME_SCT_VENDOR_SPECIFIC: 359 default: 360 return false; 361 } 362 } 363 364 static void 365 nvme_qpair_manual_complete_request(struct spdk_nvme_qpair *qpair, 366 struct nvme_request *req, uint32_t sct, uint32_t sc, 367 uint32_t dnr, bool print_on_error) 368 { 369 struct spdk_nvme_cpl cpl; 370 bool error; 371 372 memset(&cpl, 0, sizeof(cpl)); 373 cpl.sqid = qpair->id; 374 cpl.status.sct = sct; 375 cpl.status.sc = sc; 376 cpl.status.dnr = dnr; 377 378 error = spdk_nvme_cpl_is_error(&cpl); 379 380 if (error && print_on_error && !qpair->ctrlr->opts.disable_error_logging) { 381 SPDK_NOTICELOG("Command completed manually:\n"); 382 spdk_nvme_qpair_print_command(qpair, &req->cmd); 383 spdk_nvme_qpair_print_completion(qpair, &cpl); 384 } 385 386 nvme_complete_request(req->cb_fn, req->cb_arg, qpair, req, &cpl); 387 nvme_free_request(req); 388 } 389 390 static void 391 nvme_qpair_abort_queued_reqs(struct spdk_nvme_qpair *qpair, uint32_t dnr) 392 { 393 struct nvme_request *req; 394 395 while (!STAILQ_EMPTY(&qpair->queued_req)) { 396 req = STAILQ_FIRST(&qpair->queued_req); 397 STAILQ_REMOVE_HEAD(&qpair->queued_req, stailq); 398 if (!qpair->ctrlr->opts.disable_error_logging) { 399 SPDK_ERRLOG("aborting queued i/o\n"); 400 } 401 nvme_qpair_manual_complete_request(qpair, req, SPDK_NVME_SCT_GENERIC, 402 SPDK_NVME_SC_ABORTED_BY_REQUEST, dnr, true); 403 } 404 } 405 406 static inline bool 407 nvme_qpair_check_enabled(struct spdk_nvme_qpair *qpair) 408 { 409 struct nvme_request *req; 410 411 /* 412 * Either during initial connect or reset, the qpair should follow the given state machine. 413 * QPAIR_DISABLED->QPAIR_CONNECTING->QPAIR_CONNECTED->QPAIR_ENABLING->QPAIR_ENABLED. In the 414 * reset case, once the qpair is properly connected, we need to abort any outstanding requests 415 * from the old transport connection and encourage the application to retry them. We also need 416 * to submit any queued requests that built up while we were in the connected or enabling state. 417 */ 418 if (nvme_qpair_get_state(qpair) == NVME_QPAIR_CONNECTED && !qpair->ctrlr->is_resetting) { 419 nvme_qpair_set_state(qpair, NVME_QPAIR_ENABLING); 420 /* 421 * PCIe is special, for fabrics transports, we can abort requests before disconnect during reset 422 * but we have historically not disconnected pcie qpairs during reset so we have to abort requests 423 * here. 424 */ 425 if (qpair->ctrlr->trid.trtype == SPDK_NVME_TRANSPORT_PCIE) { 426 nvme_qpair_abort_reqs(qpair, 0); 427 } 428 nvme_qpair_set_state(qpair, NVME_QPAIR_ENABLED); 429 while (!STAILQ_EMPTY(&qpair->queued_req)) { 430 req = STAILQ_FIRST(&qpair->queued_req); 431 STAILQ_REMOVE_HEAD(&qpair->queued_req, stailq); 432 if (nvme_qpair_resubmit_request(qpair, req)) { 433 break; 434 } 435 } 436 } 437 438 /* 439 * When doing a reset, we must disconnect the qpair on the proper core. 440 * Note, reset is the only case where we set the failure reason without 441 * setting the qpair state since reset is done at the generic layer on the 442 * controller thread and we can't disconnect I/O qpairs from the controller 443 * thread. 444 */ 445 if (qpair->transport_failure_reason != SPDK_NVME_QPAIR_FAILURE_NONE && 446 nvme_qpair_get_state(qpair) == NVME_QPAIR_ENABLED) { 447 /* Don't disconnect PCIe qpairs. They are a special case for reset. */ 448 if (qpair->ctrlr->trid.trtype != SPDK_NVME_TRANSPORT_PCIE) { 449 nvme_ctrlr_disconnect_qpair(qpair); 450 } 451 return false; 452 } 453 454 return nvme_qpair_get_state(qpair) == NVME_QPAIR_ENABLED; 455 } 456 457 void 458 nvme_qpair_resubmit_requests(struct spdk_nvme_qpair *qpair, uint32_t num_requests) 459 { 460 uint32_t i; 461 int resubmit_rc; 462 struct nvme_request *req; 463 464 for (i = 0; i < num_requests; i++) { 465 if (qpair->ctrlr->is_resetting) { 466 break; 467 } 468 if ((req = STAILQ_FIRST(&qpair->queued_req)) == NULL) { 469 break; 470 } 471 STAILQ_REMOVE_HEAD(&qpair->queued_req, stailq); 472 resubmit_rc = nvme_qpair_resubmit_request(qpair, req); 473 if (spdk_unlikely(resubmit_rc != 0)) { 474 SPDK_ERRLOG("Unable to resubmit as many requests as we completed.\n"); 475 break; 476 } 477 } 478 } 479 480 int32_t 481 spdk_nvme_qpair_process_completions(struct spdk_nvme_qpair *qpair, uint32_t max_completions) 482 { 483 int32_t ret; 484 struct nvme_request *req, *tmp; 485 486 if (spdk_unlikely(qpair->ctrlr->is_failed)) { 487 if (qpair->ctrlr->is_removed) { 488 nvme_qpair_set_state(qpair, NVME_QPAIR_DESTROYING); 489 nvme_qpair_abort_reqs(qpair, 1 /* Do not retry */); 490 } 491 return -ENXIO; 492 } 493 494 if (spdk_unlikely(!nvme_qpair_check_enabled(qpair) && 495 !(nvme_qpair_get_state(qpair) == NVME_QPAIR_CONNECTING))) { 496 /* 497 * qpair is not enabled, likely because a controller reset is 498 * in progress. 499 */ 500 return -ENXIO; 501 } 502 503 /* error injection for those queued error requests */ 504 if (spdk_unlikely(!STAILQ_EMPTY(&qpair->err_req_head))) { 505 STAILQ_FOREACH_SAFE(req, &qpair->err_req_head, stailq, tmp) { 506 if (spdk_get_ticks() - req->submit_tick > req->timeout_tsc) { 507 STAILQ_REMOVE(&qpair->err_req_head, req, nvme_request, stailq); 508 nvme_qpair_manual_complete_request(qpair, req, 509 req->cpl.status.sct, 510 req->cpl.status.sc, 0, true); 511 } 512 } 513 } 514 515 qpair->in_completion_context = 1; 516 ret = nvme_transport_qpair_process_completions(qpair, max_completions); 517 if (ret < 0) { 518 SPDK_ERRLOG("CQ error, abort requests after transport retry counter exceeded\n"); 519 if (nvme_qpair_is_admin_queue(qpair)) { 520 nvme_ctrlr_fail(qpair->ctrlr, false); 521 } 522 } 523 qpair->in_completion_context = 0; 524 if (qpair->delete_after_completion_context) { 525 /* 526 * A request to delete this qpair was made in the context of this completion 527 * routine - so it is safe to delete it now. 528 */ 529 spdk_nvme_ctrlr_free_io_qpair(qpair); 530 return ret; 531 } 532 533 /* 534 * At this point, ret must represent the number of completions we reaped. 535 * submit as many queued requests as we completed. 536 */ 537 nvme_qpair_resubmit_requests(qpair, ret); 538 539 return ret; 540 } 541 542 spdk_nvme_qp_failure_reason 543 spdk_nvme_qpair_get_failure_reason(struct spdk_nvme_qpair *qpair) 544 { 545 return qpair->transport_failure_reason; 546 } 547 548 int 549 nvme_qpair_init(struct spdk_nvme_qpair *qpair, uint16_t id, 550 struct spdk_nvme_ctrlr *ctrlr, 551 enum spdk_nvme_qprio qprio, 552 uint32_t num_requests) 553 { 554 size_t req_size_padded; 555 uint32_t i; 556 557 qpair->id = id; 558 qpair->qprio = qprio; 559 560 qpair->in_completion_context = 0; 561 qpair->delete_after_completion_context = 0; 562 qpair->no_deletion_notification_needed = 0; 563 564 qpair->ctrlr = ctrlr; 565 qpair->trtype = ctrlr->trid.trtype; 566 567 STAILQ_INIT(&qpair->free_req); 568 STAILQ_INIT(&qpair->queued_req); 569 TAILQ_INIT(&qpair->err_cmd_head); 570 STAILQ_INIT(&qpair->err_req_head); 571 572 req_size_padded = (sizeof(struct nvme_request) + 63) & ~(size_t)63; 573 574 qpair->req_buf = spdk_zmalloc(req_size_padded * num_requests, 64, NULL, 575 SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_SHARE); 576 if (qpair->req_buf == NULL) { 577 SPDK_ERRLOG("no memory to allocate qpair(cntlid:0x%x sqid:%d) req_buf with %d request\n", 578 ctrlr->cntlid, qpair->id, num_requests); 579 return -ENOMEM; 580 } 581 582 for (i = 0; i < num_requests; i++) { 583 struct nvme_request *req = qpair->req_buf + i * req_size_padded; 584 585 req->qpair = qpair; 586 STAILQ_INSERT_HEAD(&qpair->free_req, req, stailq); 587 } 588 589 return 0; 590 } 591 592 void 593 nvme_qpair_complete_error_reqs(struct spdk_nvme_qpair *qpair) 594 { 595 struct nvme_request *req; 596 597 while (!STAILQ_EMPTY(&qpair->err_req_head)) { 598 req = STAILQ_FIRST(&qpair->err_req_head); 599 STAILQ_REMOVE_HEAD(&qpair->err_req_head, stailq); 600 nvme_qpair_manual_complete_request(qpair, req, 601 req->cpl.status.sct, 602 req->cpl.status.sc, 0, true); 603 } 604 } 605 606 void 607 nvme_qpair_deinit(struct spdk_nvme_qpair *qpair) 608 { 609 struct nvme_error_cmd *cmd, *entry; 610 611 nvme_qpair_abort_queued_reqs(qpair, 1); 612 nvme_qpair_complete_error_reqs(qpair); 613 614 TAILQ_FOREACH_SAFE(cmd, &qpair->err_cmd_head, link, entry) { 615 TAILQ_REMOVE(&qpair->err_cmd_head, cmd, link); 616 spdk_free(cmd); 617 } 618 619 spdk_free(qpair->req_buf); 620 } 621 622 static inline int 623 _nvme_qpair_submit_request(struct spdk_nvme_qpair *qpair, struct nvme_request *req) 624 { 625 int rc = 0; 626 struct nvme_request *child_req, *tmp; 627 struct nvme_error_cmd *cmd; 628 struct spdk_nvme_ctrlr *ctrlr = qpair->ctrlr; 629 bool child_req_failed = false; 630 631 nvme_qpair_check_enabled(qpair); 632 633 if (req->num_children) { 634 /* 635 * This is a split (parent) request. Submit all of the children but not the parent 636 * request itself, since the parent is the original unsplit request. 637 */ 638 TAILQ_FOREACH_SAFE(child_req, &req->children, child_tailq, tmp) { 639 if (spdk_likely(!child_req_failed)) { 640 rc = nvme_qpair_submit_request(qpair, child_req); 641 if (spdk_unlikely(rc != 0)) { 642 child_req_failed = true; 643 } 644 } else { /* free remaining child_reqs since one child_req fails */ 645 nvme_request_remove_child(req, child_req); 646 nvme_request_free_children(child_req); 647 nvme_free_request(child_req); 648 } 649 } 650 651 if (spdk_unlikely(child_req_failed)) { 652 /* part of children requests have been submitted, 653 * return success for this case. 654 */ 655 if (req->num_children) { 656 return 0; 657 } 658 goto error; 659 } 660 661 return rc; 662 } 663 664 /* queue those requests which matches with opcode in err_cmd list */ 665 if (spdk_unlikely(!TAILQ_EMPTY(&qpair->err_cmd_head))) { 666 TAILQ_FOREACH(cmd, &qpair->err_cmd_head, link) { 667 if (!cmd->do_not_submit) { 668 continue; 669 } 670 671 if ((cmd->opc == req->cmd.opc) && cmd->err_count) { 672 /* add to error request list and set cpl */ 673 req->timeout_tsc = cmd->timeout_tsc; 674 req->submit_tick = spdk_get_ticks(); 675 req->cpl.status.sct = cmd->status.sct; 676 req->cpl.status.sc = cmd->status.sc; 677 STAILQ_INSERT_TAIL(&qpair->err_req_head, req, stailq); 678 cmd->err_count--; 679 return 0; 680 } 681 } 682 } 683 684 if (spdk_unlikely(ctrlr->is_failed)) { 685 rc = -ENXIO; 686 goto error; 687 } 688 689 /* assign submit_tick before submitting req to specific transport */ 690 if (spdk_unlikely(ctrlr->timeout_enabled)) { 691 if (req->submit_tick == 0) { /* req submitted for the first time */ 692 req->submit_tick = spdk_get_ticks(); 693 req->timed_out = false; 694 } 695 } else { 696 req->submit_tick = 0; 697 } 698 699 /* Allow two cases: 700 * 1. NVMe qpair is enabled. 701 * 2. Always allow fabrics commands through - these get 702 * the controller out of reset state. 703 */ 704 if (spdk_likely(nvme_qpair_get_state(qpair) == NVME_QPAIR_ENABLED) || 705 (req->cmd.opc == SPDK_NVME_OPC_FABRIC && 706 nvme_qpair_get_state(qpair) == NVME_QPAIR_CONNECTING)) { 707 rc = nvme_transport_qpair_submit_request(qpair, req); 708 } else { 709 /* The controller is being reset - queue this request and 710 * submit it later when the reset is completed. 711 */ 712 return -EAGAIN; 713 } 714 715 if (spdk_likely(rc == 0)) { 716 req->queued = false; 717 return 0; 718 } 719 720 if (rc == -EAGAIN) { 721 return -EAGAIN; 722 } 723 724 error: 725 if (req->parent != NULL) { 726 nvme_request_remove_child(req->parent, req); 727 } 728 729 /* The request is from queued_req list we should trigger the callback from caller */ 730 if (spdk_unlikely(req->queued)) { 731 nvme_qpair_manual_complete_request(qpair, req, SPDK_NVME_SCT_GENERIC, 732 SPDK_NVME_SC_INTERNAL_DEVICE_ERROR, true, true); 733 return rc; 734 } 735 736 nvme_free_request(req); 737 738 return rc; 739 } 740 741 int 742 nvme_qpair_submit_request(struct spdk_nvme_qpair *qpair, struct nvme_request *req) 743 { 744 int rc; 745 746 /* This prevents us from entering an infinite loop when freeing queued I/O in disconnect. */ 747 if (spdk_unlikely(nvme_qpair_get_state(qpair) == NVME_QPAIR_DISCONNECTING || 748 nvme_qpair_get_state(qpair) == NVME_QPAIR_DESTROYING)) { 749 return -ENXIO; 750 } 751 752 if (spdk_unlikely(!STAILQ_EMPTY(&qpair->queued_req) && req->num_children == 0)) { 753 /* 754 * requests that have no children should be sent to the transport after all 755 * currently queued requests. Requests with chilren will be split and go back 756 * through this path. 757 */ 758 STAILQ_INSERT_TAIL(&qpair->queued_req, req, stailq); 759 req->queued = true; 760 return 0; 761 } 762 763 rc = _nvme_qpair_submit_request(qpair, req); 764 if (rc == -EAGAIN) { 765 STAILQ_INSERT_TAIL(&qpair->queued_req, req, stailq); 766 req->queued = true; 767 rc = 0; 768 } 769 770 return rc; 771 } 772 773 static int 774 nvme_qpair_resubmit_request(struct spdk_nvme_qpair *qpair, struct nvme_request *req) 775 { 776 int rc; 777 778 /* 779 * We should never have a request with children on the queue. 780 * This is necessary to preserve the 1:1 relationship between 781 * completions and resubmissions. 782 */ 783 assert(req->num_children == 0); 784 assert(req->queued); 785 rc = _nvme_qpair_submit_request(qpair, req); 786 if (spdk_unlikely(rc == -EAGAIN)) { 787 STAILQ_INSERT_HEAD(&qpair->queued_req, req, stailq); 788 } 789 790 return rc; 791 } 792 793 void 794 nvme_qpair_abort_reqs(struct spdk_nvme_qpair *qpair, uint32_t dnr) 795 { 796 nvme_qpair_complete_error_reqs(qpair); 797 nvme_qpair_abort_queued_reqs(qpair, dnr); 798 nvme_transport_qpair_abort_reqs(qpair, dnr); 799 } 800 801 int 802 spdk_nvme_qpair_add_cmd_error_injection(struct spdk_nvme_ctrlr *ctrlr, 803 struct spdk_nvme_qpair *qpair, 804 uint8_t opc, bool do_not_submit, 805 uint64_t timeout_in_us, 806 uint32_t err_count, 807 uint8_t sct, uint8_t sc) 808 { 809 struct nvme_error_cmd *entry, *cmd = NULL; 810 811 if (qpair == NULL) { 812 qpair = ctrlr->adminq; 813 } 814 815 TAILQ_FOREACH(entry, &qpair->err_cmd_head, link) { 816 if (entry->opc == opc) { 817 cmd = entry; 818 break; 819 } 820 } 821 822 if (cmd == NULL) { 823 cmd = spdk_zmalloc(sizeof(*cmd), 64, NULL, SPDK_ENV_LCORE_ID_ANY, SPDK_MALLOC_DMA); 824 if (!cmd) { 825 return -ENOMEM; 826 } 827 TAILQ_INSERT_TAIL(&qpair->err_cmd_head, cmd, link); 828 } 829 830 cmd->do_not_submit = do_not_submit; 831 cmd->err_count = err_count; 832 cmd->timeout_tsc = timeout_in_us * spdk_get_ticks_hz() / 1000000ULL; 833 cmd->opc = opc; 834 cmd->status.sct = sct; 835 cmd->status.sc = sc; 836 837 return 0; 838 } 839 840 void 841 spdk_nvme_qpair_remove_cmd_error_injection(struct spdk_nvme_ctrlr *ctrlr, 842 struct spdk_nvme_qpair *qpair, 843 uint8_t opc) 844 { 845 struct nvme_error_cmd *cmd, *entry; 846 847 if (qpair == NULL) { 848 qpair = ctrlr->adminq; 849 } 850 851 TAILQ_FOREACH_SAFE(cmd, &qpair->err_cmd_head, link, entry) { 852 if (cmd->opc == opc) { 853 TAILQ_REMOVE(&qpair->err_cmd_head, cmd, link); 854 spdk_free(cmd); 855 return; 856 } 857 } 858 859 return; 860 } 861