1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2015 Intel Corporation. All rights reserved. 3 * Copyright (c) 2019-2021 Mellanox Technologies LTD. All rights reserved. 4 * Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 5 */ 6 7 #include "spdk/stdinc.h" 8 9 #include "nvme_internal.h" 10 #include "nvme_io_msg.h" 11 12 #include "spdk/env.h" 13 #include "spdk/string.h" 14 #include "spdk/endian.h" 15 16 struct nvme_active_ns_ctx; 17 18 static int nvme_ctrlr_construct_and_submit_aer(struct spdk_nvme_ctrlr *ctrlr, 19 struct nvme_async_event_request *aer); 20 static void nvme_ctrlr_identify_active_ns_async(struct nvme_active_ns_ctx *ctx); 21 static int nvme_ctrlr_identify_ns_async(struct spdk_nvme_ns *ns); 22 static int nvme_ctrlr_identify_ns_iocs_specific_async(struct spdk_nvme_ns *ns); 23 static int nvme_ctrlr_identify_id_desc_async(struct spdk_nvme_ns *ns); 24 static void nvme_ctrlr_init_cap(struct spdk_nvme_ctrlr *ctrlr); 25 static void nvme_ctrlr_set_state(struct spdk_nvme_ctrlr *ctrlr, enum nvme_ctrlr_state state, 26 uint64_t timeout_in_ms); 27 28 static int 29 nvme_ns_cmp(struct spdk_nvme_ns *ns1, struct spdk_nvme_ns *ns2) 30 { 31 if (ns1->id < ns2->id) { 32 return -1; 33 } else if (ns1->id > ns2->id) { 34 return 1; 35 } else { 36 return 0; 37 } 38 } 39 40 RB_GENERATE_STATIC(nvme_ns_tree, spdk_nvme_ns, node, nvme_ns_cmp); 41 42 #define CTRLR_STRING(ctrlr) \ 43 ((ctrlr->trid.trtype == SPDK_NVME_TRANSPORT_TCP || ctrlr->trid.trtype == SPDK_NVME_TRANSPORT_RDMA) ? \ 44 ctrlr->trid.subnqn : ctrlr->trid.traddr) 45 46 #define NVME_CTRLR_ERRLOG(ctrlr, format, ...) \ 47 SPDK_ERRLOG("[%s] " format, CTRLR_STRING(ctrlr), ##__VA_ARGS__); 48 49 #define NVME_CTRLR_WARNLOG(ctrlr, format, ...) \ 50 SPDK_WARNLOG("[%s] " format, CTRLR_STRING(ctrlr), ##__VA_ARGS__); 51 52 #define NVME_CTRLR_NOTICELOG(ctrlr, format, ...) \ 53 SPDK_NOTICELOG("[%s] " format, CTRLR_STRING(ctrlr), ##__VA_ARGS__); 54 55 #define NVME_CTRLR_INFOLOG(ctrlr, format, ...) \ 56 SPDK_INFOLOG(nvme, "[%s] " format, CTRLR_STRING(ctrlr), ##__VA_ARGS__); 57 58 #ifdef DEBUG 59 #define NVME_CTRLR_DEBUGLOG(ctrlr, format, ...) \ 60 SPDK_DEBUGLOG(nvme, "[%s] " format, CTRLR_STRING(ctrlr), ##__VA_ARGS__); 61 #else 62 #define NVME_CTRLR_DEBUGLOG(ctrlr, ...) do { } while (0) 63 #endif 64 65 #define nvme_ctrlr_get_reg_async(ctrlr, reg, sz, cb_fn, cb_arg) \ 66 nvme_transport_ctrlr_get_reg_ ## sz ## _async(ctrlr, \ 67 offsetof(struct spdk_nvme_registers, reg), cb_fn, cb_arg) 68 69 #define nvme_ctrlr_set_reg_async(ctrlr, reg, sz, val, cb_fn, cb_arg) \ 70 nvme_transport_ctrlr_set_reg_ ## sz ## _async(ctrlr, \ 71 offsetof(struct spdk_nvme_registers, reg), val, cb_fn, cb_arg) 72 73 #define nvme_ctrlr_get_cc_async(ctrlr, cb_fn, cb_arg) \ 74 nvme_ctrlr_get_reg_async(ctrlr, cc, 4, cb_fn, cb_arg) 75 76 #define nvme_ctrlr_get_csts_async(ctrlr, cb_fn, cb_arg) \ 77 nvme_ctrlr_get_reg_async(ctrlr, csts, 4, cb_fn, cb_arg) 78 79 #define nvme_ctrlr_get_cap_async(ctrlr, cb_fn, cb_arg) \ 80 nvme_ctrlr_get_reg_async(ctrlr, cap, 8, cb_fn, cb_arg) 81 82 #define nvme_ctrlr_get_vs_async(ctrlr, cb_fn, cb_arg) \ 83 nvme_ctrlr_get_reg_async(ctrlr, vs, 4, cb_fn, cb_arg) 84 85 #define nvme_ctrlr_set_cc_async(ctrlr, value, cb_fn, cb_arg) \ 86 nvme_ctrlr_set_reg_async(ctrlr, cc, 4, value, cb_fn, cb_arg) 87 88 static int 89 nvme_ctrlr_get_cc(struct spdk_nvme_ctrlr *ctrlr, union spdk_nvme_cc_register *cc) 90 { 91 return nvme_transport_ctrlr_get_reg_4(ctrlr, offsetof(struct spdk_nvme_registers, cc.raw), 92 &cc->raw); 93 } 94 95 static int 96 nvme_ctrlr_get_csts(struct spdk_nvme_ctrlr *ctrlr, union spdk_nvme_csts_register *csts) 97 { 98 return nvme_transport_ctrlr_get_reg_4(ctrlr, offsetof(struct spdk_nvme_registers, csts.raw), 99 &csts->raw); 100 } 101 102 int 103 nvme_ctrlr_get_cap(struct spdk_nvme_ctrlr *ctrlr, union spdk_nvme_cap_register *cap) 104 { 105 return nvme_transport_ctrlr_get_reg_8(ctrlr, offsetof(struct spdk_nvme_registers, cap.raw), 106 &cap->raw); 107 } 108 109 int 110 nvme_ctrlr_get_vs(struct spdk_nvme_ctrlr *ctrlr, union spdk_nvme_vs_register *vs) 111 { 112 return nvme_transport_ctrlr_get_reg_4(ctrlr, offsetof(struct spdk_nvme_registers, vs.raw), 113 &vs->raw); 114 } 115 116 int 117 nvme_ctrlr_get_cmbsz(struct spdk_nvme_ctrlr *ctrlr, union spdk_nvme_cmbsz_register *cmbsz) 118 { 119 return nvme_transport_ctrlr_get_reg_4(ctrlr, offsetof(struct spdk_nvme_registers, cmbsz.raw), 120 &cmbsz->raw); 121 } 122 123 int 124 nvme_ctrlr_get_pmrcap(struct spdk_nvme_ctrlr *ctrlr, union spdk_nvme_pmrcap_register *pmrcap) 125 { 126 return nvme_transport_ctrlr_get_reg_4(ctrlr, offsetof(struct spdk_nvme_registers, pmrcap.raw), 127 &pmrcap->raw); 128 } 129 130 int 131 nvme_ctrlr_get_bpinfo(struct spdk_nvme_ctrlr *ctrlr, union spdk_nvme_bpinfo_register *bpinfo) 132 { 133 return nvme_transport_ctrlr_get_reg_4(ctrlr, offsetof(struct spdk_nvme_registers, bpinfo.raw), 134 &bpinfo->raw); 135 } 136 137 int 138 nvme_ctrlr_set_bprsel(struct spdk_nvme_ctrlr *ctrlr, union spdk_nvme_bprsel_register *bprsel) 139 { 140 return nvme_transport_ctrlr_set_reg_4(ctrlr, offsetof(struct spdk_nvme_registers, bprsel.raw), 141 bprsel->raw); 142 } 143 144 int 145 nvme_ctrlr_set_bpmbl(struct spdk_nvme_ctrlr *ctrlr, uint64_t bpmbl_value) 146 { 147 return nvme_transport_ctrlr_set_reg_8(ctrlr, offsetof(struct spdk_nvme_registers, bpmbl), 148 bpmbl_value); 149 } 150 151 static int 152 nvme_ctrlr_set_nssr(struct spdk_nvme_ctrlr *ctrlr, uint32_t nssr_value) 153 { 154 return nvme_transport_ctrlr_set_reg_4(ctrlr, offsetof(struct spdk_nvme_registers, nssr), 155 nssr_value); 156 } 157 158 bool 159 nvme_ctrlr_multi_iocs_enabled(struct spdk_nvme_ctrlr *ctrlr) 160 { 161 return ctrlr->cap.bits.css & SPDK_NVME_CAP_CSS_IOCS && 162 ctrlr->opts.command_set == SPDK_NVME_CC_CSS_IOCS; 163 } 164 165 /* When the field in spdk_nvme_ctrlr_opts are changed and you change this function, please 166 * also update the nvme_ctrl_opts_init function in nvme_ctrlr.c 167 */ 168 void 169 spdk_nvme_ctrlr_get_default_ctrlr_opts(struct spdk_nvme_ctrlr_opts *opts, size_t opts_size) 170 { 171 char host_id_str[SPDK_UUID_STRING_LEN]; 172 173 assert(opts); 174 175 opts->opts_size = opts_size; 176 177 #define FIELD_OK(field) \ 178 offsetof(struct spdk_nvme_ctrlr_opts, field) + sizeof(opts->field) <= opts_size 179 180 #define SET_FIELD(field, value) \ 181 if (offsetof(struct spdk_nvme_ctrlr_opts, field) + sizeof(opts->field) <= opts_size) { \ 182 opts->field = value; \ 183 } \ 184 185 SET_FIELD(num_io_queues, DEFAULT_MAX_IO_QUEUES); 186 SET_FIELD(use_cmb_sqs, false); 187 SET_FIELD(no_shn_notification, false); 188 SET_FIELD(arb_mechanism, SPDK_NVME_CC_AMS_RR); 189 SET_FIELD(arbitration_burst, 0); 190 SET_FIELD(low_priority_weight, 0); 191 SET_FIELD(medium_priority_weight, 0); 192 SET_FIELD(high_priority_weight, 0); 193 SET_FIELD(keep_alive_timeout_ms, MIN_KEEP_ALIVE_TIMEOUT_IN_MS); 194 SET_FIELD(transport_retry_count, SPDK_NVME_DEFAULT_RETRY_COUNT); 195 SET_FIELD(io_queue_size, DEFAULT_IO_QUEUE_SIZE); 196 197 if (nvme_driver_init() == 0) { 198 if (FIELD_OK(hostnqn)) { 199 spdk_uuid_fmt_lower(host_id_str, sizeof(host_id_str), 200 &g_spdk_nvme_driver->default_extended_host_id); 201 snprintf(opts->hostnqn, sizeof(opts->hostnqn), 202 "nqn.2014-08.org.nvmexpress:uuid:%s", host_id_str); 203 } 204 205 if (FIELD_OK(extended_host_id)) { 206 memcpy(opts->extended_host_id, &g_spdk_nvme_driver->default_extended_host_id, 207 sizeof(opts->extended_host_id)); 208 } 209 210 } 211 212 SET_FIELD(io_queue_requests, DEFAULT_IO_QUEUE_REQUESTS); 213 214 if (FIELD_OK(src_addr)) { 215 memset(opts->src_addr, 0, sizeof(opts->src_addr)); 216 } 217 218 if (FIELD_OK(src_svcid)) { 219 memset(opts->src_svcid, 0, sizeof(opts->src_svcid)); 220 } 221 222 if (FIELD_OK(host_id)) { 223 memset(opts->host_id, 0, sizeof(opts->host_id)); 224 } 225 226 SET_FIELD(command_set, CHAR_BIT); 227 SET_FIELD(admin_timeout_ms, NVME_MAX_ADMIN_TIMEOUT_IN_SECS * 1000); 228 SET_FIELD(header_digest, false); 229 SET_FIELD(data_digest, false); 230 SET_FIELD(disable_error_logging, false); 231 SET_FIELD(transport_ack_timeout, SPDK_NVME_DEFAULT_TRANSPORT_ACK_TIMEOUT); 232 SET_FIELD(admin_queue_size, DEFAULT_ADMIN_QUEUE_SIZE); 233 SET_FIELD(fabrics_connect_timeout_us, NVME_FABRIC_CONNECT_COMMAND_TIMEOUT); 234 SET_FIELD(disable_read_ana_log_page, false); 235 SET_FIELD(disable_read_changed_ns_list_log_page, false); 236 237 if (FIELD_OK(psk)) { 238 memset(opts->psk, 0, sizeof(opts->psk)); 239 } 240 241 #undef FIELD_OK 242 #undef SET_FIELD 243 } 244 245 const struct spdk_nvme_ctrlr_opts * 246 spdk_nvme_ctrlr_get_opts(struct spdk_nvme_ctrlr *ctrlr) 247 { 248 return &ctrlr->opts; 249 } 250 251 /** 252 * This function will be called when the process allocates the IO qpair. 253 * Note: the ctrlr_lock must be held when calling this function. 254 */ 255 static void 256 nvme_ctrlr_proc_add_io_qpair(struct spdk_nvme_qpair *qpair) 257 { 258 struct spdk_nvme_ctrlr_process *active_proc; 259 struct spdk_nvme_ctrlr *ctrlr = qpair->ctrlr; 260 261 active_proc = nvme_ctrlr_get_current_process(ctrlr); 262 if (active_proc) { 263 TAILQ_INSERT_TAIL(&active_proc->allocated_io_qpairs, qpair, per_process_tailq); 264 qpair->active_proc = active_proc; 265 } 266 } 267 268 /** 269 * This function will be called when the process frees the IO qpair. 270 * Note: the ctrlr_lock must be held when calling this function. 271 */ 272 static void 273 nvme_ctrlr_proc_remove_io_qpair(struct spdk_nvme_qpair *qpair) 274 { 275 struct spdk_nvme_ctrlr_process *active_proc; 276 struct spdk_nvme_ctrlr *ctrlr = qpair->ctrlr; 277 struct spdk_nvme_qpair *active_qpair, *tmp_qpair; 278 279 active_proc = nvme_ctrlr_get_current_process(ctrlr); 280 if (!active_proc) { 281 return; 282 } 283 284 TAILQ_FOREACH_SAFE(active_qpair, &active_proc->allocated_io_qpairs, 285 per_process_tailq, tmp_qpair) { 286 if (active_qpair == qpair) { 287 TAILQ_REMOVE(&active_proc->allocated_io_qpairs, 288 active_qpair, per_process_tailq); 289 290 break; 291 } 292 } 293 } 294 295 void 296 spdk_nvme_ctrlr_get_default_io_qpair_opts(struct spdk_nvme_ctrlr *ctrlr, 297 struct spdk_nvme_io_qpair_opts *opts, 298 size_t opts_size) 299 { 300 assert(ctrlr); 301 302 assert(opts); 303 304 memset(opts, 0, opts_size); 305 306 #define FIELD_OK(field) \ 307 offsetof(struct spdk_nvme_io_qpair_opts, field) + sizeof(opts->field) <= opts_size 308 309 if (FIELD_OK(qprio)) { 310 opts->qprio = SPDK_NVME_QPRIO_URGENT; 311 } 312 313 if (FIELD_OK(io_queue_size)) { 314 opts->io_queue_size = ctrlr->opts.io_queue_size; 315 } 316 317 if (FIELD_OK(io_queue_requests)) { 318 opts->io_queue_requests = ctrlr->opts.io_queue_requests; 319 } 320 321 if (FIELD_OK(delay_cmd_submit)) { 322 opts->delay_cmd_submit = false; 323 } 324 325 if (FIELD_OK(sq.vaddr)) { 326 opts->sq.vaddr = NULL; 327 } 328 329 if (FIELD_OK(sq.paddr)) { 330 opts->sq.paddr = 0; 331 } 332 333 if (FIELD_OK(sq.buffer_size)) { 334 opts->sq.buffer_size = 0; 335 } 336 337 if (FIELD_OK(cq.vaddr)) { 338 opts->cq.vaddr = NULL; 339 } 340 341 if (FIELD_OK(cq.paddr)) { 342 opts->cq.paddr = 0; 343 } 344 345 if (FIELD_OK(cq.buffer_size)) { 346 opts->cq.buffer_size = 0; 347 } 348 349 if (FIELD_OK(create_only)) { 350 opts->create_only = false; 351 } 352 353 if (FIELD_OK(async_mode)) { 354 opts->async_mode = false; 355 } 356 357 #undef FIELD_OK 358 } 359 360 static struct spdk_nvme_qpair * 361 nvme_ctrlr_create_io_qpair(struct spdk_nvme_ctrlr *ctrlr, 362 const struct spdk_nvme_io_qpair_opts *opts) 363 { 364 int32_t qid; 365 struct spdk_nvme_qpair *qpair; 366 union spdk_nvme_cc_register cc; 367 368 if (!ctrlr) { 369 return NULL; 370 } 371 372 nvme_robust_mutex_lock(&ctrlr->ctrlr_lock); 373 cc.raw = ctrlr->process_init_cc.raw; 374 375 if (opts->qprio & ~SPDK_NVME_CREATE_IO_SQ_QPRIO_MASK) { 376 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 377 return NULL; 378 } 379 380 /* 381 * Only value SPDK_NVME_QPRIO_URGENT(0) is valid for the 382 * default round robin arbitration method. 383 */ 384 if ((cc.bits.ams == SPDK_NVME_CC_AMS_RR) && (opts->qprio != SPDK_NVME_QPRIO_URGENT)) { 385 NVME_CTRLR_ERRLOG(ctrlr, "invalid queue priority for default round robin arbitration method\n"); 386 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 387 return NULL; 388 } 389 390 qid = spdk_nvme_ctrlr_alloc_qid(ctrlr); 391 if (qid < 0) { 392 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 393 return NULL; 394 } 395 396 qpair = nvme_transport_ctrlr_create_io_qpair(ctrlr, qid, opts); 397 if (qpair == NULL) { 398 NVME_CTRLR_ERRLOG(ctrlr, "nvme_transport_ctrlr_create_io_qpair() failed\n"); 399 spdk_nvme_ctrlr_free_qid(ctrlr, qid); 400 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 401 return NULL; 402 } 403 404 TAILQ_INSERT_TAIL(&ctrlr->active_io_qpairs, qpair, tailq); 405 406 nvme_ctrlr_proc_add_io_qpair(qpair); 407 408 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 409 410 return qpair; 411 } 412 413 int 414 spdk_nvme_ctrlr_connect_io_qpair(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair) 415 { 416 int rc; 417 418 if (nvme_qpair_get_state(qpair) != NVME_QPAIR_DISCONNECTED) { 419 return -EISCONN; 420 } 421 422 nvme_robust_mutex_lock(&ctrlr->ctrlr_lock); 423 rc = nvme_transport_ctrlr_connect_qpair(ctrlr, qpair); 424 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 425 426 if (ctrlr->quirks & NVME_QUIRK_DELAY_AFTER_QUEUE_ALLOC) { 427 spdk_delay_us(100); 428 } 429 430 return rc; 431 } 432 433 void 434 spdk_nvme_ctrlr_disconnect_io_qpair(struct spdk_nvme_qpair *qpair) 435 { 436 struct spdk_nvme_ctrlr *ctrlr = qpair->ctrlr; 437 438 nvme_robust_mutex_lock(&ctrlr->ctrlr_lock); 439 nvme_transport_ctrlr_disconnect_qpair(ctrlr, qpair); 440 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 441 } 442 443 struct spdk_nvme_qpair * 444 spdk_nvme_ctrlr_alloc_io_qpair(struct spdk_nvme_ctrlr *ctrlr, 445 const struct spdk_nvme_io_qpair_opts *user_opts, 446 size_t opts_size) 447 { 448 449 struct spdk_nvme_qpair *qpair = NULL; 450 struct spdk_nvme_io_qpair_opts opts; 451 int rc; 452 453 nvme_robust_mutex_lock(&ctrlr->ctrlr_lock); 454 455 if (spdk_unlikely(ctrlr->state != NVME_CTRLR_STATE_READY)) { 456 /* When controller is resetting or initializing, free_io_qids is deleted or not created yet. 457 * We can't create IO qpair in that case */ 458 goto unlock; 459 } 460 461 /* 462 * Get the default options, then overwrite them with the user-provided options 463 * up to opts_size. 464 * 465 * This allows for extensions of the opts structure without breaking 466 * ABI compatibility. 467 */ 468 spdk_nvme_ctrlr_get_default_io_qpair_opts(ctrlr, &opts, sizeof(opts)); 469 if (user_opts) { 470 memcpy(&opts, user_opts, spdk_min(sizeof(opts), opts_size)); 471 472 /* If user passes buffers, make sure they're big enough for the requested queue size */ 473 if (opts.sq.vaddr) { 474 if (opts.sq.buffer_size < (opts.io_queue_size * sizeof(struct spdk_nvme_cmd))) { 475 NVME_CTRLR_ERRLOG(ctrlr, "sq buffer size %" PRIx64 " is too small for sq size %zx\n", 476 opts.sq.buffer_size, (opts.io_queue_size * sizeof(struct spdk_nvme_cmd))); 477 goto unlock; 478 } 479 } 480 if (opts.cq.vaddr) { 481 if (opts.cq.buffer_size < (opts.io_queue_size * sizeof(struct spdk_nvme_cpl))) { 482 NVME_CTRLR_ERRLOG(ctrlr, "cq buffer size %" PRIx64 " is too small for cq size %zx\n", 483 opts.cq.buffer_size, (opts.io_queue_size * sizeof(struct spdk_nvme_cpl))); 484 goto unlock; 485 } 486 } 487 } 488 489 qpair = nvme_ctrlr_create_io_qpair(ctrlr, &opts); 490 491 if (qpair == NULL || opts.create_only == true) { 492 goto unlock; 493 } 494 495 rc = spdk_nvme_ctrlr_connect_io_qpair(ctrlr, qpair); 496 if (rc != 0) { 497 NVME_CTRLR_ERRLOG(ctrlr, "nvme_transport_ctrlr_connect_io_qpair() failed\n"); 498 nvme_ctrlr_proc_remove_io_qpair(qpair); 499 TAILQ_REMOVE(&ctrlr->active_io_qpairs, qpair, tailq); 500 spdk_bit_array_set(ctrlr->free_io_qids, qpair->id); 501 nvme_transport_ctrlr_delete_io_qpair(ctrlr, qpair); 502 qpair = NULL; 503 goto unlock; 504 } 505 506 unlock: 507 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 508 509 return qpair; 510 } 511 512 int 513 spdk_nvme_ctrlr_reconnect_io_qpair(struct spdk_nvme_qpair *qpair) 514 { 515 struct spdk_nvme_ctrlr *ctrlr; 516 enum nvme_qpair_state qpair_state; 517 int rc; 518 519 assert(qpair != NULL); 520 assert(nvme_qpair_is_admin_queue(qpair) == false); 521 assert(qpair->ctrlr != NULL); 522 523 ctrlr = qpair->ctrlr; 524 nvme_robust_mutex_lock(&ctrlr->ctrlr_lock); 525 qpair_state = nvme_qpair_get_state(qpair); 526 527 if (ctrlr->is_removed) { 528 rc = -ENODEV; 529 goto out; 530 } 531 532 if (ctrlr->is_resetting || qpair_state == NVME_QPAIR_DISCONNECTING) { 533 rc = -EAGAIN; 534 goto out; 535 } 536 537 if (ctrlr->is_failed || qpair_state == NVME_QPAIR_DESTROYING) { 538 rc = -ENXIO; 539 goto out; 540 } 541 542 if (qpair_state != NVME_QPAIR_DISCONNECTED) { 543 rc = 0; 544 goto out; 545 } 546 547 rc = nvme_transport_ctrlr_connect_qpair(ctrlr, qpair); 548 if (rc) { 549 rc = -EAGAIN; 550 goto out; 551 } 552 553 out: 554 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 555 return rc; 556 } 557 558 spdk_nvme_qp_failure_reason 559 spdk_nvme_ctrlr_get_admin_qp_failure_reason(struct spdk_nvme_ctrlr *ctrlr) 560 { 561 return ctrlr->adminq->transport_failure_reason; 562 } 563 564 /* 565 * This internal function will attempt to take the controller 566 * lock before calling disconnect on a controller qpair. 567 * Functions already holding the controller lock should 568 * call nvme_transport_ctrlr_disconnect_qpair directly. 569 */ 570 void 571 nvme_ctrlr_disconnect_qpair(struct spdk_nvme_qpair *qpair) 572 { 573 struct spdk_nvme_ctrlr *ctrlr = qpair->ctrlr; 574 575 assert(ctrlr != NULL); 576 nvme_robust_mutex_lock(&ctrlr->ctrlr_lock); 577 nvme_transport_ctrlr_disconnect_qpair(ctrlr, qpair); 578 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 579 } 580 581 int 582 spdk_nvme_ctrlr_free_io_qpair(struct spdk_nvme_qpair *qpair) 583 { 584 struct spdk_nvme_ctrlr *ctrlr; 585 586 if (qpair == NULL) { 587 return 0; 588 } 589 590 ctrlr = qpair->ctrlr; 591 592 if (qpair->in_completion_context) { 593 /* 594 * There are many cases where it is convenient to delete an io qpair in the context 595 * of that qpair's completion routine. To handle this properly, set a flag here 596 * so that the completion routine will perform an actual delete after the context 597 * unwinds. 598 */ 599 qpair->delete_after_completion_context = 1; 600 return 0; 601 } 602 603 qpair->destroy_in_progress = 1; 604 605 nvme_transport_ctrlr_disconnect_qpair(ctrlr, qpair); 606 607 if (qpair->poll_group && (qpair->active_proc == nvme_ctrlr_get_current_process(ctrlr))) { 608 spdk_nvme_poll_group_remove(qpair->poll_group->group, qpair); 609 } 610 611 /* Do not retry. */ 612 nvme_qpair_set_state(qpair, NVME_QPAIR_DESTROYING); 613 614 /* In the multi-process case, a process may call this function on a foreign 615 * I/O qpair (i.e. one that this process did not create) when that qpairs process 616 * exits unexpectedly. In that case, we must not try to abort any reqs associated 617 * with that qpair, since the callbacks will also be foreign to this process. 618 */ 619 if (qpair->active_proc == nvme_ctrlr_get_current_process(ctrlr)) { 620 nvme_qpair_abort_all_queued_reqs(qpair); 621 } 622 623 nvme_robust_mutex_lock(&ctrlr->ctrlr_lock); 624 625 nvme_ctrlr_proc_remove_io_qpair(qpair); 626 627 TAILQ_REMOVE(&ctrlr->active_io_qpairs, qpair, tailq); 628 spdk_nvme_ctrlr_free_qid(ctrlr, qpair->id); 629 630 nvme_transport_ctrlr_delete_io_qpair(ctrlr, qpair); 631 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 632 return 0; 633 } 634 635 static void 636 nvme_ctrlr_construct_intel_support_log_page_list(struct spdk_nvme_ctrlr *ctrlr, 637 struct spdk_nvme_intel_log_page_directory *log_page_directory) 638 { 639 if (log_page_directory == NULL) { 640 return; 641 } 642 643 assert(ctrlr->cdata.vid == SPDK_PCI_VID_INTEL); 644 645 ctrlr->log_page_supported[SPDK_NVME_INTEL_LOG_PAGE_DIRECTORY] = true; 646 647 if (log_page_directory->read_latency_log_len || 648 (ctrlr->quirks & NVME_INTEL_QUIRK_READ_LATENCY)) { 649 ctrlr->log_page_supported[SPDK_NVME_INTEL_LOG_READ_CMD_LATENCY] = true; 650 } 651 if (log_page_directory->write_latency_log_len || 652 (ctrlr->quirks & NVME_INTEL_QUIRK_WRITE_LATENCY)) { 653 ctrlr->log_page_supported[SPDK_NVME_INTEL_LOG_WRITE_CMD_LATENCY] = true; 654 } 655 if (log_page_directory->temperature_statistics_log_len) { 656 ctrlr->log_page_supported[SPDK_NVME_INTEL_LOG_TEMPERATURE] = true; 657 } 658 if (log_page_directory->smart_log_len) { 659 ctrlr->log_page_supported[SPDK_NVME_INTEL_LOG_SMART] = true; 660 } 661 if (log_page_directory->marketing_description_log_len) { 662 ctrlr->log_page_supported[SPDK_NVME_INTEL_MARKETING_DESCRIPTION] = true; 663 } 664 } 665 666 struct intel_log_pages_ctx { 667 struct spdk_nvme_intel_log_page_directory log_page_directory; 668 struct spdk_nvme_ctrlr *ctrlr; 669 }; 670 671 static void 672 nvme_ctrlr_set_intel_support_log_pages_done(void *arg, const struct spdk_nvme_cpl *cpl) 673 { 674 struct intel_log_pages_ctx *ctx = arg; 675 struct spdk_nvme_ctrlr *ctrlr = ctx->ctrlr; 676 677 if (!spdk_nvme_cpl_is_error(cpl)) { 678 nvme_ctrlr_construct_intel_support_log_page_list(ctrlr, &ctx->log_page_directory); 679 } 680 681 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_SUPPORTED_FEATURES, 682 ctrlr->opts.admin_timeout_ms); 683 free(ctx); 684 } 685 686 static int 687 nvme_ctrlr_set_intel_support_log_pages(struct spdk_nvme_ctrlr *ctrlr) 688 { 689 int rc = 0; 690 struct intel_log_pages_ctx *ctx; 691 692 ctx = calloc(1, sizeof(*ctx)); 693 if (!ctx) { 694 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_SUPPORTED_FEATURES, 695 ctrlr->opts.admin_timeout_ms); 696 return 0; 697 } 698 699 ctx->ctrlr = ctrlr; 700 701 rc = spdk_nvme_ctrlr_cmd_get_log_page(ctrlr, SPDK_NVME_INTEL_LOG_PAGE_DIRECTORY, 702 SPDK_NVME_GLOBAL_NS_TAG, &ctx->log_page_directory, 703 sizeof(struct spdk_nvme_intel_log_page_directory), 704 0, nvme_ctrlr_set_intel_support_log_pages_done, ctx); 705 if (rc != 0) { 706 free(ctx); 707 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_SUPPORTED_FEATURES, 708 ctrlr->opts.admin_timeout_ms); 709 return 0; 710 } 711 712 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_WAIT_FOR_SUPPORTED_INTEL_LOG_PAGES, 713 ctrlr->opts.admin_timeout_ms); 714 715 return 0; 716 } 717 718 static int 719 nvme_ctrlr_alloc_ana_log_page(struct spdk_nvme_ctrlr *ctrlr) 720 { 721 uint32_t ana_log_page_size; 722 723 ana_log_page_size = sizeof(struct spdk_nvme_ana_page) + ctrlr->cdata.nanagrpid * 724 sizeof(struct spdk_nvme_ana_group_descriptor) + ctrlr->active_ns_count * 725 sizeof(uint32_t); 726 727 /* Number of active namespaces may have changed. 728 * Check if ANA log page fits into existing buffer. 729 */ 730 if (ana_log_page_size > ctrlr->ana_log_page_size) { 731 void *new_buffer; 732 733 if (ctrlr->ana_log_page) { 734 new_buffer = realloc(ctrlr->ana_log_page, ana_log_page_size); 735 } else { 736 new_buffer = calloc(1, ana_log_page_size); 737 } 738 739 if (!new_buffer) { 740 NVME_CTRLR_ERRLOG(ctrlr, "could not allocate ANA log page buffer, size %u\n", 741 ana_log_page_size); 742 return -ENXIO; 743 } 744 745 ctrlr->ana_log_page = new_buffer; 746 if (ctrlr->copied_ana_desc) { 747 new_buffer = realloc(ctrlr->copied_ana_desc, ana_log_page_size); 748 } else { 749 new_buffer = calloc(1, ana_log_page_size); 750 } 751 752 if (!new_buffer) { 753 NVME_CTRLR_ERRLOG(ctrlr, "could not allocate a buffer to parse ANA descriptor, size %u\n", 754 ana_log_page_size); 755 return -ENOMEM; 756 } 757 758 ctrlr->copied_ana_desc = new_buffer; 759 ctrlr->ana_log_page_size = ana_log_page_size; 760 } 761 762 return 0; 763 } 764 765 static int 766 nvme_ctrlr_update_ana_log_page(struct spdk_nvme_ctrlr *ctrlr) 767 { 768 struct nvme_completion_poll_status *status; 769 int rc; 770 771 rc = nvme_ctrlr_alloc_ana_log_page(ctrlr); 772 if (rc != 0) { 773 return rc; 774 } 775 776 status = calloc(1, sizeof(*status)); 777 if (status == NULL) { 778 NVME_CTRLR_ERRLOG(ctrlr, "Failed to allocate status tracker\n"); 779 return -ENOMEM; 780 } 781 782 rc = spdk_nvme_ctrlr_cmd_get_log_page(ctrlr, SPDK_NVME_LOG_ASYMMETRIC_NAMESPACE_ACCESS, 783 SPDK_NVME_GLOBAL_NS_TAG, ctrlr->ana_log_page, 784 ctrlr->ana_log_page_size, 0, 785 nvme_completion_poll_cb, status); 786 if (rc != 0) { 787 free(status); 788 return rc; 789 } 790 791 if (nvme_wait_for_completion_robust_lock_timeout(ctrlr->adminq, status, &ctrlr->ctrlr_lock, 792 ctrlr->opts.admin_timeout_ms * 1000)) { 793 if (!status->timed_out) { 794 free(status); 795 } 796 return -EIO; 797 } 798 799 free(status); 800 return 0; 801 } 802 803 static int 804 nvme_ctrlr_update_ns_ana_states(const struct spdk_nvme_ana_group_descriptor *desc, 805 void *cb_arg) 806 { 807 struct spdk_nvme_ctrlr *ctrlr = cb_arg; 808 struct spdk_nvme_ns *ns; 809 uint32_t i, nsid; 810 811 for (i = 0; i < desc->num_of_nsid; i++) { 812 nsid = desc->nsid[i]; 813 if (nsid == 0 || nsid > ctrlr->cdata.nn) { 814 continue; 815 } 816 817 ns = spdk_nvme_ctrlr_get_ns(ctrlr, nsid); 818 assert(ns != NULL); 819 820 ns->ana_group_id = desc->ana_group_id; 821 ns->ana_state = desc->ana_state; 822 } 823 824 return 0; 825 } 826 827 int 828 nvme_ctrlr_parse_ana_log_page(struct spdk_nvme_ctrlr *ctrlr, 829 spdk_nvme_parse_ana_log_page_cb cb_fn, void *cb_arg) 830 { 831 struct spdk_nvme_ana_group_descriptor *copied_desc; 832 uint8_t *orig_desc; 833 uint32_t i, desc_size, copy_len; 834 int rc = 0; 835 836 if (ctrlr->ana_log_page == NULL) { 837 return -EINVAL; 838 } 839 840 copied_desc = ctrlr->copied_ana_desc; 841 842 orig_desc = (uint8_t *)ctrlr->ana_log_page + sizeof(struct spdk_nvme_ana_page); 843 copy_len = ctrlr->ana_log_page_size - sizeof(struct spdk_nvme_ana_page); 844 845 for (i = 0; i < ctrlr->ana_log_page->num_ana_group_desc; i++) { 846 memcpy(copied_desc, orig_desc, copy_len); 847 848 rc = cb_fn(copied_desc, cb_arg); 849 if (rc != 0) { 850 break; 851 } 852 853 desc_size = sizeof(struct spdk_nvme_ana_group_descriptor) + 854 copied_desc->num_of_nsid * sizeof(uint32_t); 855 orig_desc += desc_size; 856 copy_len -= desc_size; 857 } 858 859 return rc; 860 } 861 862 static int 863 nvme_ctrlr_set_supported_log_pages(struct spdk_nvme_ctrlr *ctrlr) 864 { 865 int rc = 0; 866 867 memset(ctrlr->log_page_supported, 0, sizeof(ctrlr->log_page_supported)); 868 /* Mandatory pages */ 869 ctrlr->log_page_supported[SPDK_NVME_LOG_ERROR] = true; 870 ctrlr->log_page_supported[SPDK_NVME_LOG_HEALTH_INFORMATION] = true; 871 ctrlr->log_page_supported[SPDK_NVME_LOG_FIRMWARE_SLOT] = true; 872 if (ctrlr->cdata.lpa.celp) { 873 ctrlr->log_page_supported[SPDK_NVME_LOG_COMMAND_EFFECTS_LOG] = true; 874 } 875 876 if (ctrlr->cdata.cmic.ana_reporting) { 877 ctrlr->log_page_supported[SPDK_NVME_LOG_ASYMMETRIC_NAMESPACE_ACCESS] = true; 878 if (!ctrlr->opts.disable_read_ana_log_page) { 879 rc = nvme_ctrlr_update_ana_log_page(ctrlr); 880 if (rc == 0) { 881 nvme_ctrlr_parse_ana_log_page(ctrlr, nvme_ctrlr_update_ns_ana_states, 882 ctrlr); 883 } 884 } 885 } 886 887 if (ctrlr->cdata.ctratt.fdps) { 888 ctrlr->log_page_supported[SPDK_NVME_LOG_FDP_CONFIGURATIONS] = true; 889 ctrlr->log_page_supported[SPDK_NVME_LOG_RECLAIM_UNIT_HANDLE_USAGE] = true; 890 ctrlr->log_page_supported[SPDK_NVME_LOG_FDP_STATISTICS] = true; 891 ctrlr->log_page_supported[SPDK_NVME_LOG_FDP_EVENTS] = true; 892 } 893 894 if (ctrlr->cdata.vid == SPDK_PCI_VID_INTEL && 895 ctrlr->trid.trtype == SPDK_NVME_TRANSPORT_PCIE && 896 !(ctrlr->quirks & NVME_INTEL_QUIRK_NO_LOG_PAGES)) { 897 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_SUPPORTED_INTEL_LOG_PAGES, 898 ctrlr->opts.admin_timeout_ms); 899 900 } else { 901 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_SUPPORTED_FEATURES, 902 ctrlr->opts.admin_timeout_ms); 903 904 } 905 906 return rc; 907 } 908 909 static void 910 nvme_ctrlr_set_intel_supported_features(struct spdk_nvme_ctrlr *ctrlr) 911 { 912 ctrlr->feature_supported[SPDK_NVME_INTEL_FEAT_MAX_LBA] = true; 913 ctrlr->feature_supported[SPDK_NVME_INTEL_FEAT_NATIVE_MAX_LBA] = true; 914 ctrlr->feature_supported[SPDK_NVME_INTEL_FEAT_POWER_GOVERNOR_SETTING] = true; 915 ctrlr->feature_supported[SPDK_NVME_INTEL_FEAT_SMBUS_ADDRESS] = true; 916 ctrlr->feature_supported[SPDK_NVME_INTEL_FEAT_LED_PATTERN] = true; 917 ctrlr->feature_supported[SPDK_NVME_INTEL_FEAT_RESET_TIMED_WORKLOAD_COUNTERS] = true; 918 ctrlr->feature_supported[SPDK_NVME_INTEL_FEAT_LATENCY_TRACKING] = true; 919 } 920 921 static void 922 nvme_ctrlr_set_arbitration_feature(struct spdk_nvme_ctrlr *ctrlr) 923 { 924 uint32_t cdw11; 925 struct nvme_completion_poll_status *status; 926 927 if (ctrlr->opts.arbitration_burst == 0) { 928 return; 929 } 930 931 if (ctrlr->opts.arbitration_burst > 7) { 932 NVME_CTRLR_WARNLOG(ctrlr, "Valid arbitration burst values is from 0-7\n"); 933 return; 934 } 935 936 status = calloc(1, sizeof(*status)); 937 if (!status) { 938 NVME_CTRLR_ERRLOG(ctrlr, "Failed to allocate status tracker\n"); 939 return; 940 } 941 942 cdw11 = ctrlr->opts.arbitration_burst; 943 944 if (spdk_nvme_ctrlr_get_flags(ctrlr) & SPDK_NVME_CTRLR_WRR_SUPPORTED) { 945 cdw11 |= (uint32_t)ctrlr->opts.low_priority_weight << 8; 946 cdw11 |= (uint32_t)ctrlr->opts.medium_priority_weight << 16; 947 cdw11 |= (uint32_t)ctrlr->opts.high_priority_weight << 24; 948 } 949 950 if (spdk_nvme_ctrlr_cmd_set_feature(ctrlr, SPDK_NVME_FEAT_ARBITRATION, 951 cdw11, 0, NULL, 0, 952 nvme_completion_poll_cb, status) < 0) { 953 NVME_CTRLR_ERRLOG(ctrlr, "Set arbitration feature failed\n"); 954 free(status); 955 return; 956 } 957 958 if (nvme_wait_for_completion_timeout(ctrlr->adminq, status, 959 ctrlr->opts.admin_timeout_ms * 1000)) { 960 NVME_CTRLR_ERRLOG(ctrlr, "Timeout to set arbitration feature\n"); 961 } 962 963 if (!status->timed_out) { 964 free(status); 965 } 966 } 967 968 static void 969 nvme_ctrlr_set_supported_features(struct spdk_nvme_ctrlr *ctrlr) 970 { 971 memset(ctrlr->feature_supported, 0, sizeof(ctrlr->feature_supported)); 972 /* Mandatory features */ 973 ctrlr->feature_supported[SPDK_NVME_FEAT_ARBITRATION] = true; 974 ctrlr->feature_supported[SPDK_NVME_FEAT_POWER_MANAGEMENT] = true; 975 ctrlr->feature_supported[SPDK_NVME_FEAT_TEMPERATURE_THRESHOLD] = true; 976 ctrlr->feature_supported[SPDK_NVME_FEAT_ERROR_RECOVERY] = true; 977 ctrlr->feature_supported[SPDK_NVME_FEAT_NUMBER_OF_QUEUES] = true; 978 ctrlr->feature_supported[SPDK_NVME_FEAT_INTERRUPT_COALESCING] = true; 979 ctrlr->feature_supported[SPDK_NVME_FEAT_INTERRUPT_VECTOR_CONFIGURATION] = true; 980 ctrlr->feature_supported[SPDK_NVME_FEAT_WRITE_ATOMICITY] = true; 981 ctrlr->feature_supported[SPDK_NVME_FEAT_ASYNC_EVENT_CONFIGURATION] = true; 982 /* Optional features */ 983 if (ctrlr->cdata.vwc.present) { 984 ctrlr->feature_supported[SPDK_NVME_FEAT_VOLATILE_WRITE_CACHE] = true; 985 } 986 if (ctrlr->cdata.apsta.supported) { 987 ctrlr->feature_supported[SPDK_NVME_FEAT_AUTONOMOUS_POWER_STATE_TRANSITION] = true; 988 } 989 if (ctrlr->cdata.hmpre) { 990 ctrlr->feature_supported[SPDK_NVME_FEAT_HOST_MEM_BUFFER] = true; 991 } 992 if (ctrlr->cdata.vid == SPDK_PCI_VID_INTEL) { 993 nvme_ctrlr_set_intel_supported_features(ctrlr); 994 } 995 996 nvme_ctrlr_set_arbitration_feature(ctrlr); 997 } 998 999 bool 1000 spdk_nvme_ctrlr_is_failed(struct spdk_nvme_ctrlr *ctrlr) 1001 { 1002 return ctrlr->is_failed; 1003 } 1004 1005 void 1006 nvme_ctrlr_fail(struct spdk_nvme_ctrlr *ctrlr, bool hot_remove) 1007 { 1008 /* 1009 * Set the flag here and leave the work failure of qpairs to 1010 * spdk_nvme_qpair_process_completions(). 1011 */ 1012 if (hot_remove) { 1013 ctrlr->is_removed = true; 1014 } 1015 1016 if (ctrlr->is_failed) { 1017 NVME_CTRLR_NOTICELOG(ctrlr, "already in failed state\n"); 1018 return; 1019 } 1020 1021 if (ctrlr->is_disconnecting) { 1022 NVME_CTRLR_DEBUGLOG(ctrlr, "already disconnecting\n"); 1023 return; 1024 } 1025 1026 ctrlr->is_failed = true; 1027 nvme_transport_ctrlr_disconnect_qpair(ctrlr, ctrlr->adminq); 1028 NVME_CTRLR_ERRLOG(ctrlr, "in failed state.\n"); 1029 } 1030 1031 /** 1032 * This public API function will try to take the controller lock. 1033 * Any private functions being called from a thread already holding 1034 * the ctrlr lock should call nvme_ctrlr_fail directly. 1035 */ 1036 void 1037 spdk_nvme_ctrlr_fail(struct spdk_nvme_ctrlr *ctrlr) 1038 { 1039 nvme_robust_mutex_lock(&ctrlr->ctrlr_lock); 1040 nvme_ctrlr_fail(ctrlr, false); 1041 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 1042 } 1043 1044 static void 1045 nvme_ctrlr_shutdown_set_cc_done(void *_ctx, uint64_t value, const struct spdk_nvme_cpl *cpl) 1046 { 1047 struct nvme_ctrlr_detach_ctx *ctx = _ctx; 1048 struct spdk_nvme_ctrlr *ctrlr = ctx->ctrlr; 1049 1050 if (spdk_nvme_cpl_is_error(cpl)) { 1051 NVME_CTRLR_ERRLOG(ctrlr, "Failed to write CC.SHN\n"); 1052 ctx->shutdown_complete = true; 1053 return; 1054 } 1055 1056 if (ctrlr->opts.no_shn_notification) { 1057 ctx->shutdown_complete = true; 1058 return; 1059 } 1060 1061 /* 1062 * The NVMe specification defines RTD3E to be the time between 1063 * setting SHN = 1 until the controller will set SHST = 10b. 1064 * If the device doesn't report RTD3 entry latency, or if it 1065 * reports RTD3 entry latency less than 10 seconds, pick 1066 * 10 seconds as a reasonable amount of time to 1067 * wait before proceeding. 1068 */ 1069 NVME_CTRLR_DEBUGLOG(ctrlr, "RTD3E = %" PRIu32 " us\n", ctrlr->cdata.rtd3e); 1070 ctx->shutdown_timeout_ms = SPDK_CEIL_DIV(ctrlr->cdata.rtd3e, 1000); 1071 ctx->shutdown_timeout_ms = spdk_max(ctx->shutdown_timeout_ms, 10000); 1072 NVME_CTRLR_DEBUGLOG(ctrlr, "shutdown timeout = %" PRIu32 " ms\n", ctx->shutdown_timeout_ms); 1073 1074 ctx->shutdown_start_tsc = spdk_get_ticks(); 1075 ctx->state = NVME_CTRLR_DETACH_CHECK_CSTS; 1076 } 1077 1078 static void 1079 nvme_ctrlr_shutdown_get_cc_done(void *_ctx, uint64_t value, const struct spdk_nvme_cpl *cpl) 1080 { 1081 struct nvme_ctrlr_detach_ctx *ctx = _ctx; 1082 struct spdk_nvme_ctrlr *ctrlr = ctx->ctrlr; 1083 union spdk_nvme_cc_register cc; 1084 int rc; 1085 1086 if (spdk_nvme_cpl_is_error(cpl)) { 1087 NVME_CTRLR_ERRLOG(ctrlr, "Failed to read the CC register\n"); 1088 ctx->shutdown_complete = true; 1089 return; 1090 } 1091 1092 assert(value <= UINT32_MAX); 1093 cc.raw = (uint32_t)value; 1094 1095 if (ctrlr->opts.no_shn_notification) { 1096 NVME_CTRLR_INFOLOG(ctrlr, "Disable SSD without shutdown notification\n"); 1097 if (cc.bits.en == 0) { 1098 ctx->shutdown_complete = true; 1099 return; 1100 } 1101 1102 cc.bits.en = 0; 1103 } else { 1104 cc.bits.shn = SPDK_NVME_SHN_NORMAL; 1105 } 1106 1107 rc = nvme_ctrlr_set_cc_async(ctrlr, cc.raw, nvme_ctrlr_shutdown_set_cc_done, ctx); 1108 if (rc != 0) { 1109 NVME_CTRLR_ERRLOG(ctrlr, "Failed to write CC.SHN\n"); 1110 ctx->shutdown_complete = true; 1111 } 1112 } 1113 1114 static void 1115 nvme_ctrlr_shutdown_async(struct spdk_nvme_ctrlr *ctrlr, 1116 struct nvme_ctrlr_detach_ctx *ctx) 1117 { 1118 int rc; 1119 1120 if (ctrlr->is_removed) { 1121 ctx->shutdown_complete = true; 1122 return; 1123 } 1124 1125 if (ctrlr->adminq == NULL || 1126 ctrlr->adminq->transport_failure_reason != SPDK_NVME_QPAIR_FAILURE_NONE) { 1127 NVME_CTRLR_INFOLOG(ctrlr, "Adminq is not connected.\n"); 1128 ctx->shutdown_complete = true; 1129 return; 1130 } 1131 1132 ctx->state = NVME_CTRLR_DETACH_SET_CC; 1133 rc = nvme_ctrlr_get_cc_async(ctrlr, nvme_ctrlr_shutdown_get_cc_done, ctx); 1134 if (rc != 0) { 1135 NVME_CTRLR_ERRLOG(ctrlr, "Failed to read the CC register\n"); 1136 ctx->shutdown_complete = true; 1137 } 1138 } 1139 1140 static void 1141 nvme_ctrlr_shutdown_get_csts_done(void *_ctx, uint64_t value, const struct spdk_nvme_cpl *cpl) 1142 { 1143 struct nvme_ctrlr_detach_ctx *ctx = _ctx; 1144 1145 if (spdk_nvme_cpl_is_error(cpl)) { 1146 NVME_CTRLR_ERRLOG(ctx->ctrlr, "Failed to read the CSTS register\n"); 1147 ctx->shutdown_complete = true; 1148 return; 1149 } 1150 1151 assert(value <= UINT32_MAX); 1152 ctx->csts.raw = (uint32_t)value; 1153 ctx->state = NVME_CTRLR_DETACH_GET_CSTS_DONE; 1154 } 1155 1156 static int 1157 nvme_ctrlr_shutdown_poll_async(struct spdk_nvme_ctrlr *ctrlr, 1158 struct nvme_ctrlr_detach_ctx *ctx) 1159 { 1160 union spdk_nvme_csts_register csts; 1161 uint32_t ms_waited; 1162 1163 switch (ctx->state) { 1164 case NVME_CTRLR_DETACH_SET_CC: 1165 case NVME_CTRLR_DETACH_GET_CSTS: 1166 /* We're still waiting for the register operation to complete */ 1167 spdk_nvme_qpair_process_completions(ctrlr->adminq, 0); 1168 return -EAGAIN; 1169 1170 case NVME_CTRLR_DETACH_CHECK_CSTS: 1171 ctx->state = NVME_CTRLR_DETACH_GET_CSTS; 1172 if (nvme_ctrlr_get_csts_async(ctrlr, nvme_ctrlr_shutdown_get_csts_done, ctx)) { 1173 NVME_CTRLR_ERRLOG(ctrlr, "Failed to read the CSTS register\n"); 1174 return -EIO; 1175 } 1176 return -EAGAIN; 1177 1178 case NVME_CTRLR_DETACH_GET_CSTS_DONE: 1179 ctx->state = NVME_CTRLR_DETACH_CHECK_CSTS; 1180 break; 1181 1182 default: 1183 assert(0 && "Should never happen"); 1184 return -EINVAL; 1185 } 1186 1187 ms_waited = (spdk_get_ticks() - ctx->shutdown_start_tsc) * 1000 / spdk_get_ticks_hz(); 1188 csts.raw = ctx->csts.raw; 1189 1190 if (csts.bits.shst == SPDK_NVME_SHST_COMPLETE) { 1191 NVME_CTRLR_DEBUGLOG(ctrlr, "shutdown complete in %u milliseconds\n", ms_waited); 1192 return 0; 1193 } 1194 1195 if (ms_waited < ctx->shutdown_timeout_ms) { 1196 return -EAGAIN; 1197 } 1198 1199 NVME_CTRLR_ERRLOG(ctrlr, "did not shutdown within %u milliseconds\n", 1200 ctx->shutdown_timeout_ms); 1201 if (ctrlr->quirks & NVME_QUIRK_SHST_COMPLETE) { 1202 NVME_CTRLR_ERRLOG(ctrlr, "likely due to shutdown handling in the VMWare emulated NVMe SSD\n"); 1203 } 1204 1205 return 0; 1206 } 1207 1208 static inline uint64_t 1209 nvme_ctrlr_get_ready_timeout(struct spdk_nvme_ctrlr *ctrlr) 1210 { 1211 return ctrlr->cap.bits.to * 500; 1212 } 1213 1214 static void 1215 nvme_ctrlr_set_cc_en_done(void *ctx, uint64_t value, const struct spdk_nvme_cpl *cpl) 1216 { 1217 struct spdk_nvme_ctrlr *ctrlr = ctx; 1218 1219 if (spdk_nvme_cpl_is_error(cpl)) { 1220 NVME_CTRLR_ERRLOG(ctrlr, "Failed to set the CC register\n"); 1221 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 1222 return; 1223 } 1224 1225 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ENABLE_WAIT_FOR_READY_1, 1226 nvme_ctrlr_get_ready_timeout(ctrlr)); 1227 } 1228 1229 static int 1230 nvme_ctrlr_enable(struct spdk_nvme_ctrlr *ctrlr) 1231 { 1232 union spdk_nvme_cc_register cc; 1233 int rc; 1234 1235 rc = nvme_transport_ctrlr_enable(ctrlr); 1236 if (rc != 0) { 1237 NVME_CTRLR_ERRLOG(ctrlr, "transport ctrlr_enable failed\n"); 1238 return rc; 1239 } 1240 1241 cc.raw = ctrlr->process_init_cc.raw; 1242 if (cc.bits.en != 0) { 1243 NVME_CTRLR_ERRLOG(ctrlr, "called with CC.EN = 1\n"); 1244 return -EINVAL; 1245 } 1246 1247 cc.bits.en = 1; 1248 cc.bits.css = 0; 1249 cc.bits.shn = 0; 1250 cc.bits.iosqes = 6; /* SQ entry size == 64 == 2^6 */ 1251 cc.bits.iocqes = 4; /* CQ entry size == 16 == 2^4 */ 1252 1253 /* Page size is 2 ^ (12 + mps). */ 1254 cc.bits.mps = spdk_u32log2(ctrlr->page_size) - 12; 1255 1256 /* 1257 * Since NVMe 1.0, a controller should have at least one bit set in CAP.CSS. 1258 * A controller that does not have any bit set in CAP.CSS is not spec compliant. 1259 * Try to support such a controller regardless. 1260 */ 1261 if (ctrlr->cap.bits.css == 0) { 1262 NVME_CTRLR_INFOLOG(ctrlr, "Drive reports no command sets supported. Assuming NVM is supported.\n"); 1263 ctrlr->cap.bits.css = SPDK_NVME_CAP_CSS_NVM; 1264 } 1265 1266 /* 1267 * If the user did not explicitly request a command set, or supplied a value larger than 1268 * what can be saved in CC.CSS, use the most reasonable default. 1269 */ 1270 if (ctrlr->opts.command_set >= CHAR_BIT) { 1271 if (ctrlr->cap.bits.css & SPDK_NVME_CAP_CSS_IOCS) { 1272 ctrlr->opts.command_set = SPDK_NVME_CC_CSS_IOCS; 1273 } else if (ctrlr->cap.bits.css & SPDK_NVME_CAP_CSS_NVM) { 1274 ctrlr->opts.command_set = SPDK_NVME_CC_CSS_NVM; 1275 } else if (ctrlr->cap.bits.css & SPDK_NVME_CAP_CSS_NOIO) { 1276 ctrlr->opts.command_set = SPDK_NVME_CC_CSS_NOIO; 1277 } else { 1278 /* Invalid supported bits detected, falling back to NVM. */ 1279 ctrlr->opts.command_set = SPDK_NVME_CC_CSS_NVM; 1280 } 1281 } 1282 1283 /* Verify that the selected command set is supported by the controller. */ 1284 if (!(ctrlr->cap.bits.css & (1u << ctrlr->opts.command_set))) { 1285 NVME_CTRLR_DEBUGLOG(ctrlr, "Requested I/O command set %u but supported mask is 0x%x\n", 1286 ctrlr->opts.command_set, ctrlr->cap.bits.css); 1287 NVME_CTRLR_DEBUGLOG(ctrlr, "Falling back to NVM. Assuming NVM is supported.\n"); 1288 ctrlr->opts.command_set = SPDK_NVME_CC_CSS_NVM; 1289 } 1290 1291 cc.bits.css = ctrlr->opts.command_set; 1292 1293 switch (ctrlr->opts.arb_mechanism) { 1294 case SPDK_NVME_CC_AMS_RR: 1295 break; 1296 case SPDK_NVME_CC_AMS_WRR: 1297 if (SPDK_NVME_CAP_AMS_WRR & ctrlr->cap.bits.ams) { 1298 break; 1299 } 1300 return -EINVAL; 1301 case SPDK_NVME_CC_AMS_VS: 1302 if (SPDK_NVME_CAP_AMS_VS & ctrlr->cap.bits.ams) { 1303 break; 1304 } 1305 return -EINVAL; 1306 default: 1307 return -EINVAL; 1308 } 1309 1310 cc.bits.ams = ctrlr->opts.arb_mechanism; 1311 ctrlr->process_init_cc.raw = cc.raw; 1312 1313 if (nvme_ctrlr_set_cc_async(ctrlr, cc.raw, nvme_ctrlr_set_cc_en_done, ctrlr)) { 1314 NVME_CTRLR_ERRLOG(ctrlr, "set_cc() failed\n"); 1315 return -EIO; 1316 } 1317 1318 return 0; 1319 } 1320 1321 static const char * 1322 nvme_ctrlr_state_string(enum nvme_ctrlr_state state) 1323 { 1324 switch (state) { 1325 case NVME_CTRLR_STATE_INIT_DELAY: 1326 return "delay init"; 1327 case NVME_CTRLR_STATE_CONNECT_ADMINQ: 1328 return "connect adminq"; 1329 case NVME_CTRLR_STATE_WAIT_FOR_CONNECT_ADMINQ: 1330 return "wait for connect adminq"; 1331 case NVME_CTRLR_STATE_READ_VS: 1332 return "read vs"; 1333 case NVME_CTRLR_STATE_READ_VS_WAIT_FOR_VS: 1334 return "read vs wait for vs"; 1335 case NVME_CTRLR_STATE_READ_CAP: 1336 return "read cap"; 1337 case NVME_CTRLR_STATE_READ_CAP_WAIT_FOR_CAP: 1338 return "read cap wait for cap"; 1339 case NVME_CTRLR_STATE_CHECK_EN: 1340 return "check en"; 1341 case NVME_CTRLR_STATE_CHECK_EN_WAIT_FOR_CC: 1342 return "check en wait for cc"; 1343 case NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_1: 1344 return "disable and wait for CSTS.RDY = 1"; 1345 case NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_1_WAIT_FOR_CSTS: 1346 return "disable and wait for CSTS.RDY = 1 reg"; 1347 case NVME_CTRLR_STATE_SET_EN_0: 1348 return "set CC.EN = 0"; 1349 case NVME_CTRLR_STATE_SET_EN_0_WAIT_FOR_CC: 1350 return "set CC.EN = 0 wait for cc"; 1351 case NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0: 1352 return "disable and wait for CSTS.RDY = 0"; 1353 case NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0_WAIT_FOR_CSTS: 1354 return "disable and wait for CSTS.RDY = 0 reg"; 1355 case NVME_CTRLR_STATE_DISABLED: 1356 return "controller is disabled"; 1357 case NVME_CTRLR_STATE_ENABLE: 1358 return "enable controller by writing CC.EN = 1"; 1359 case NVME_CTRLR_STATE_ENABLE_WAIT_FOR_CC: 1360 return "enable controller by writing CC.EN = 1 reg"; 1361 case NVME_CTRLR_STATE_ENABLE_WAIT_FOR_READY_1: 1362 return "wait for CSTS.RDY = 1"; 1363 case NVME_CTRLR_STATE_ENABLE_WAIT_FOR_READY_1_WAIT_FOR_CSTS: 1364 return "wait for CSTS.RDY = 1 reg"; 1365 case NVME_CTRLR_STATE_RESET_ADMIN_QUEUE: 1366 return "reset admin queue"; 1367 case NVME_CTRLR_STATE_IDENTIFY: 1368 return "identify controller"; 1369 case NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY: 1370 return "wait for identify controller"; 1371 case NVME_CTRLR_STATE_CONFIGURE_AER: 1372 return "configure AER"; 1373 case NVME_CTRLR_STATE_WAIT_FOR_CONFIGURE_AER: 1374 return "wait for configure aer"; 1375 case NVME_CTRLR_STATE_SET_KEEP_ALIVE_TIMEOUT: 1376 return "set keep alive timeout"; 1377 case NVME_CTRLR_STATE_WAIT_FOR_KEEP_ALIVE_TIMEOUT: 1378 return "wait for set keep alive timeout"; 1379 case NVME_CTRLR_STATE_IDENTIFY_IOCS_SPECIFIC: 1380 return "identify controller iocs specific"; 1381 case NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_IOCS_SPECIFIC: 1382 return "wait for identify controller iocs specific"; 1383 case NVME_CTRLR_STATE_GET_ZNS_CMD_EFFECTS_LOG: 1384 return "get zns cmd and effects log page"; 1385 case NVME_CTRLR_STATE_WAIT_FOR_GET_ZNS_CMD_EFFECTS_LOG: 1386 return "wait for get zns cmd and effects log page"; 1387 case NVME_CTRLR_STATE_SET_NUM_QUEUES: 1388 return "set number of queues"; 1389 case NVME_CTRLR_STATE_WAIT_FOR_SET_NUM_QUEUES: 1390 return "wait for set number of queues"; 1391 case NVME_CTRLR_STATE_IDENTIFY_ACTIVE_NS: 1392 return "identify active ns"; 1393 case NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_ACTIVE_NS: 1394 return "wait for identify active ns"; 1395 case NVME_CTRLR_STATE_IDENTIFY_NS: 1396 return "identify ns"; 1397 case NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_NS: 1398 return "wait for identify ns"; 1399 case NVME_CTRLR_STATE_IDENTIFY_ID_DESCS: 1400 return "identify namespace id descriptors"; 1401 case NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_ID_DESCS: 1402 return "wait for identify namespace id descriptors"; 1403 case NVME_CTRLR_STATE_IDENTIFY_NS_IOCS_SPECIFIC: 1404 return "identify ns iocs specific"; 1405 case NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_NS_IOCS_SPECIFIC: 1406 return "wait for identify ns iocs specific"; 1407 case NVME_CTRLR_STATE_SET_SUPPORTED_LOG_PAGES: 1408 return "set supported log pages"; 1409 case NVME_CTRLR_STATE_SET_SUPPORTED_INTEL_LOG_PAGES: 1410 return "set supported INTEL log pages"; 1411 case NVME_CTRLR_STATE_WAIT_FOR_SUPPORTED_INTEL_LOG_PAGES: 1412 return "wait for supported INTEL log pages"; 1413 case NVME_CTRLR_STATE_SET_SUPPORTED_FEATURES: 1414 return "set supported features"; 1415 case NVME_CTRLR_STATE_SET_DB_BUF_CFG: 1416 return "set doorbell buffer config"; 1417 case NVME_CTRLR_STATE_WAIT_FOR_DB_BUF_CFG: 1418 return "wait for doorbell buffer config"; 1419 case NVME_CTRLR_STATE_SET_HOST_ID: 1420 return "set host ID"; 1421 case NVME_CTRLR_STATE_WAIT_FOR_HOST_ID: 1422 return "wait for set host ID"; 1423 case NVME_CTRLR_STATE_TRANSPORT_READY: 1424 return "transport ready"; 1425 case NVME_CTRLR_STATE_READY: 1426 return "ready"; 1427 case NVME_CTRLR_STATE_ERROR: 1428 return "error"; 1429 case NVME_CTRLR_STATE_DISCONNECTED: 1430 return "disconnected"; 1431 } 1432 return "unknown"; 1433 }; 1434 1435 static void 1436 _nvme_ctrlr_set_state(struct spdk_nvme_ctrlr *ctrlr, enum nvme_ctrlr_state state, 1437 uint64_t timeout_in_ms, bool quiet) 1438 { 1439 uint64_t ticks_per_ms, timeout_in_ticks, now_ticks; 1440 1441 ctrlr->state = state; 1442 if (timeout_in_ms == NVME_TIMEOUT_KEEP_EXISTING) { 1443 if (!quiet) { 1444 NVME_CTRLR_DEBUGLOG(ctrlr, "setting state to %s (keeping existing timeout)\n", 1445 nvme_ctrlr_state_string(ctrlr->state)); 1446 } 1447 return; 1448 } 1449 1450 if (timeout_in_ms == NVME_TIMEOUT_INFINITE) { 1451 goto inf; 1452 } 1453 1454 ticks_per_ms = spdk_get_ticks_hz() / 1000; 1455 if (timeout_in_ms > UINT64_MAX / ticks_per_ms) { 1456 NVME_CTRLR_ERRLOG(ctrlr, 1457 "Specified timeout would cause integer overflow. Defaulting to no timeout.\n"); 1458 goto inf; 1459 } 1460 1461 now_ticks = spdk_get_ticks(); 1462 timeout_in_ticks = timeout_in_ms * ticks_per_ms; 1463 if (timeout_in_ticks > UINT64_MAX - now_ticks) { 1464 NVME_CTRLR_ERRLOG(ctrlr, 1465 "Specified timeout would cause integer overflow. Defaulting to no timeout.\n"); 1466 goto inf; 1467 } 1468 1469 ctrlr->state_timeout_tsc = timeout_in_ticks + now_ticks; 1470 if (!quiet) { 1471 NVME_CTRLR_DEBUGLOG(ctrlr, "setting state to %s (timeout %" PRIu64 " ms)\n", 1472 nvme_ctrlr_state_string(ctrlr->state), timeout_in_ms); 1473 } 1474 return; 1475 inf: 1476 if (!quiet) { 1477 NVME_CTRLR_DEBUGLOG(ctrlr, "setting state to %s (no timeout)\n", 1478 nvme_ctrlr_state_string(ctrlr->state)); 1479 } 1480 ctrlr->state_timeout_tsc = NVME_TIMEOUT_INFINITE; 1481 } 1482 1483 static void 1484 nvme_ctrlr_set_state(struct spdk_nvme_ctrlr *ctrlr, enum nvme_ctrlr_state state, 1485 uint64_t timeout_in_ms) 1486 { 1487 _nvme_ctrlr_set_state(ctrlr, state, timeout_in_ms, false); 1488 } 1489 1490 static void 1491 nvme_ctrlr_set_state_quiet(struct spdk_nvme_ctrlr *ctrlr, enum nvme_ctrlr_state state, 1492 uint64_t timeout_in_ms) 1493 { 1494 _nvme_ctrlr_set_state(ctrlr, state, timeout_in_ms, true); 1495 } 1496 1497 static void 1498 nvme_ctrlr_free_zns_specific_data(struct spdk_nvme_ctrlr *ctrlr) 1499 { 1500 spdk_free(ctrlr->cdata_zns); 1501 ctrlr->cdata_zns = NULL; 1502 } 1503 1504 static void 1505 nvme_ctrlr_free_iocs_specific_data(struct spdk_nvme_ctrlr *ctrlr) 1506 { 1507 nvme_ctrlr_free_zns_specific_data(ctrlr); 1508 } 1509 1510 static void 1511 nvme_ctrlr_free_doorbell_buffer(struct spdk_nvme_ctrlr *ctrlr) 1512 { 1513 if (ctrlr->shadow_doorbell) { 1514 spdk_free(ctrlr->shadow_doorbell); 1515 ctrlr->shadow_doorbell = NULL; 1516 } 1517 1518 if (ctrlr->eventidx) { 1519 spdk_free(ctrlr->eventidx); 1520 ctrlr->eventidx = NULL; 1521 } 1522 } 1523 1524 static void 1525 nvme_ctrlr_set_doorbell_buffer_config_done(void *arg, const struct spdk_nvme_cpl *cpl) 1526 { 1527 struct spdk_nvme_ctrlr *ctrlr = (struct spdk_nvme_ctrlr *)arg; 1528 1529 if (spdk_nvme_cpl_is_error(cpl)) { 1530 NVME_CTRLR_WARNLOG(ctrlr, "Doorbell buffer config failed\n"); 1531 } else { 1532 NVME_CTRLR_INFOLOG(ctrlr, "Doorbell buffer config enabled\n"); 1533 } 1534 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_HOST_ID, 1535 ctrlr->opts.admin_timeout_ms); 1536 } 1537 1538 static int 1539 nvme_ctrlr_set_doorbell_buffer_config(struct spdk_nvme_ctrlr *ctrlr) 1540 { 1541 int rc = 0; 1542 uint64_t prp1, prp2, len; 1543 1544 if (!ctrlr->cdata.oacs.doorbell_buffer_config) { 1545 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_HOST_ID, 1546 ctrlr->opts.admin_timeout_ms); 1547 return 0; 1548 } 1549 1550 if (ctrlr->trid.trtype != SPDK_NVME_TRANSPORT_PCIE) { 1551 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_HOST_ID, 1552 ctrlr->opts.admin_timeout_ms); 1553 return 0; 1554 } 1555 1556 /* only 1 page size for doorbell buffer */ 1557 ctrlr->shadow_doorbell = spdk_zmalloc(ctrlr->page_size, ctrlr->page_size, 1558 NULL, SPDK_ENV_LCORE_ID_ANY, 1559 SPDK_MALLOC_DMA | SPDK_MALLOC_SHARE); 1560 if (ctrlr->shadow_doorbell == NULL) { 1561 rc = -ENOMEM; 1562 goto error; 1563 } 1564 1565 len = ctrlr->page_size; 1566 prp1 = spdk_vtophys(ctrlr->shadow_doorbell, &len); 1567 if (prp1 == SPDK_VTOPHYS_ERROR || len != ctrlr->page_size) { 1568 rc = -EFAULT; 1569 goto error; 1570 } 1571 1572 ctrlr->eventidx = spdk_zmalloc(ctrlr->page_size, ctrlr->page_size, 1573 NULL, SPDK_ENV_LCORE_ID_ANY, 1574 SPDK_MALLOC_DMA | SPDK_MALLOC_SHARE); 1575 if (ctrlr->eventidx == NULL) { 1576 rc = -ENOMEM; 1577 goto error; 1578 } 1579 1580 len = ctrlr->page_size; 1581 prp2 = spdk_vtophys(ctrlr->eventidx, &len); 1582 if (prp2 == SPDK_VTOPHYS_ERROR || len != ctrlr->page_size) { 1583 rc = -EFAULT; 1584 goto error; 1585 } 1586 1587 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_WAIT_FOR_DB_BUF_CFG, 1588 ctrlr->opts.admin_timeout_ms); 1589 1590 rc = nvme_ctrlr_cmd_doorbell_buffer_config(ctrlr, prp1, prp2, 1591 nvme_ctrlr_set_doorbell_buffer_config_done, ctrlr); 1592 if (rc != 0) { 1593 goto error; 1594 } 1595 1596 return 0; 1597 1598 error: 1599 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 1600 nvme_ctrlr_free_doorbell_buffer(ctrlr); 1601 return rc; 1602 } 1603 1604 void 1605 nvme_ctrlr_abort_queued_aborts(struct spdk_nvme_ctrlr *ctrlr) 1606 { 1607 struct nvme_request *req, *tmp; 1608 struct spdk_nvme_cpl cpl = {}; 1609 1610 cpl.status.sc = SPDK_NVME_SC_ABORTED_SQ_DELETION; 1611 cpl.status.sct = SPDK_NVME_SCT_GENERIC; 1612 1613 STAILQ_FOREACH_SAFE(req, &ctrlr->queued_aborts, stailq, tmp) { 1614 STAILQ_REMOVE_HEAD(&ctrlr->queued_aborts, stailq); 1615 ctrlr->outstanding_aborts++; 1616 1617 nvme_complete_request(req->cb_fn, req->cb_arg, req->qpair, req, &cpl); 1618 } 1619 } 1620 1621 static int 1622 nvme_ctrlr_disconnect(struct spdk_nvme_ctrlr *ctrlr) 1623 { 1624 if (ctrlr->is_resetting || ctrlr->is_removed) { 1625 /* 1626 * Controller is already resetting or has been removed. Return 1627 * immediately since there is no need to kick off another 1628 * reset in these cases. 1629 */ 1630 return ctrlr->is_resetting ? -EBUSY : -ENXIO; 1631 } 1632 1633 ctrlr->is_resetting = true; 1634 ctrlr->is_failed = false; 1635 ctrlr->is_disconnecting = true; 1636 ctrlr->prepare_for_reset = true; 1637 1638 NVME_CTRLR_NOTICELOG(ctrlr, "resetting controller\n"); 1639 1640 /* Disable keep-alive, it'll be re-enabled as part of the init process */ 1641 ctrlr->keep_alive_interval_ticks = 0; 1642 1643 /* Abort all of the queued abort requests */ 1644 nvme_ctrlr_abort_queued_aborts(ctrlr); 1645 1646 nvme_transport_admin_qpair_abort_aers(ctrlr->adminq); 1647 1648 ctrlr->adminq->transport_failure_reason = SPDK_NVME_QPAIR_FAILURE_LOCAL; 1649 nvme_transport_ctrlr_disconnect_qpair(ctrlr, ctrlr->adminq); 1650 1651 return 0; 1652 } 1653 1654 static void 1655 nvme_ctrlr_disconnect_done(struct spdk_nvme_ctrlr *ctrlr) 1656 { 1657 assert(ctrlr->is_failed == false); 1658 ctrlr->is_disconnecting = false; 1659 1660 /* Doorbell buffer config is invalid during reset */ 1661 nvme_ctrlr_free_doorbell_buffer(ctrlr); 1662 1663 /* I/O Command Set Specific Identify Controller data is invalidated during reset */ 1664 nvme_ctrlr_free_iocs_specific_data(ctrlr); 1665 1666 spdk_bit_array_free(&ctrlr->free_io_qids); 1667 1668 /* Set the state back to DISCONNECTED to cause a full hardware reset. */ 1669 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_DISCONNECTED, NVME_TIMEOUT_INFINITE); 1670 } 1671 1672 int 1673 spdk_nvme_ctrlr_disconnect(struct spdk_nvme_ctrlr *ctrlr) 1674 { 1675 int rc; 1676 1677 nvme_robust_mutex_lock(&ctrlr->ctrlr_lock); 1678 rc = nvme_ctrlr_disconnect(ctrlr); 1679 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 1680 1681 return rc; 1682 } 1683 1684 void 1685 spdk_nvme_ctrlr_reconnect_async(struct spdk_nvme_ctrlr *ctrlr) 1686 { 1687 nvme_robust_mutex_lock(&ctrlr->ctrlr_lock); 1688 1689 ctrlr->prepare_for_reset = false; 1690 1691 /* Set the state back to INIT to cause a full hardware reset. */ 1692 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_INIT, NVME_TIMEOUT_INFINITE); 1693 1694 /* Return without releasing ctrlr_lock. ctrlr_lock will be released when 1695 * spdk_nvme_ctrlr_reset_poll_async() returns 0. 1696 */ 1697 } 1698 1699 int 1700 nvme_ctrlr_reinitialize_io_qpair(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair) 1701 { 1702 bool async; 1703 int rc; 1704 1705 if (nvme_ctrlr_get_current_process(ctrlr) != qpair->active_proc || 1706 spdk_nvme_ctrlr_is_fabrics(ctrlr) || nvme_qpair_is_admin_queue(qpair)) { 1707 assert(false); 1708 return -EINVAL; 1709 } 1710 1711 /* Force a synchronous connect. */ 1712 async = qpair->async; 1713 qpair->async = false; 1714 rc = nvme_transport_ctrlr_connect_qpair(ctrlr, qpair); 1715 qpair->async = async; 1716 1717 if (rc != 0) { 1718 qpair->transport_failure_reason = SPDK_NVME_QPAIR_FAILURE_LOCAL; 1719 } 1720 1721 return rc; 1722 } 1723 1724 /** 1725 * This function will be called when the controller is being reinitialized. 1726 * Note: the ctrlr_lock must be held when calling this function. 1727 */ 1728 int 1729 spdk_nvme_ctrlr_reconnect_poll_async(struct spdk_nvme_ctrlr *ctrlr) 1730 { 1731 struct spdk_nvme_ns *ns, *tmp_ns; 1732 struct spdk_nvme_qpair *qpair; 1733 int rc = 0, rc_tmp = 0; 1734 1735 if (nvme_ctrlr_process_init(ctrlr) != 0) { 1736 NVME_CTRLR_ERRLOG(ctrlr, "controller reinitialization failed\n"); 1737 rc = -1; 1738 } 1739 if (ctrlr->state != NVME_CTRLR_STATE_READY && rc != -1) { 1740 return -EAGAIN; 1741 } 1742 1743 /* 1744 * For non-fabrics controllers, the memory locations of the transport qpair 1745 * don't change when the controller is reset. They simply need to be 1746 * re-enabled with admin commands to the controller. For fabric 1747 * controllers we need to disconnect and reconnect the qpair on its 1748 * own thread outside of the context of the reset. 1749 */ 1750 if (rc == 0 && !spdk_nvme_ctrlr_is_fabrics(ctrlr)) { 1751 /* Reinitialize qpairs */ 1752 TAILQ_FOREACH(qpair, &ctrlr->active_io_qpairs, tailq) { 1753 /* Always clear the qid bit here, even for a foreign qpair. We need 1754 * to make sure another process doesn't get the chance to grab that 1755 * qid. 1756 */ 1757 assert(spdk_bit_array_get(ctrlr->free_io_qids, qpair->id)); 1758 spdk_bit_array_clear(ctrlr->free_io_qids, qpair->id); 1759 if (nvme_ctrlr_get_current_process(ctrlr) != qpair->active_proc) { 1760 /* 1761 * We cannot reinitialize a foreign qpair. The qpair's owning 1762 * process will take care of it. Set failure reason to FAILURE_RESET 1763 * to ensure that happens. 1764 */ 1765 qpair->transport_failure_reason = SPDK_NVME_QPAIR_FAILURE_RESET; 1766 continue; 1767 } 1768 rc_tmp = nvme_ctrlr_reinitialize_io_qpair(ctrlr, qpair); 1769 if (rc_tmp != 0) { 1770 rc = rc_tmp; 1771 } 1772 } 1773 } 1774 1775 /* 1776 * Take this opportunity to remove inactive namespaces. During a reset namespace 1777 * handles can be invalidated. 1778 */ 1779 RB_FOREACH_SAFE(ns, nvme_ns_tree, &ctrlr->ns, tmp_ns) { 1780 if (!ns->active) { 1781 RB_REMOVE(nvme_ns_tree, &ctrlr->ns, ns); 1782 spdk_free(ns); 1783 } 1784 } 1785 1786 if (rc) { 1787 nvme_ctrlr_fail(ctrlr, false); 1788 } 1789 ctrlr->is_resetting = false; 1790 1791 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 1792 1793 if (!ctrlr->cdata.oaes.ns_attribute_notices) { 1794 /* 1795 * If controller doesn't support ns_attribute_notices and 1796 * namespace attributes change (e.g. number of namespaces) 1797 * we need to update system handling device reset. 1798 */ 1799 nvme_io_msg_ctrlr_update(ctrlr); 1800 } 1801 1802 return rc; 1803 } 1804 1805 /* 1806 * For PCIe transport, spdk_nvme_ctrlr_disconnect() will do a Controller Level Reset 1807 * (Change CC.EN from 1 to 0) as a operation to disconnect the admin qpair. 1808 * The following two functions are added to do a Controller Level Reset. They have 1809 * to be called under the nvme controller's lock. 1810 */ 1811 void 1812 nvme_ctrlr_disable(struct spdk_nvme_ctrlr *ctrlr) 1813 { 1814 assert(ctrlr->is_disconnecting == true); 1815 1816 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_CHECK_EN, NVME_TIMEOUT_INFINITE); 1817 } 1818 1819 int 1820 nvme_ctrlr_disable_poll(struct spdk_nvme_ctrlr *ctrlr) 1821 { 1822 int rc = 0; 1823 1824 if (nvme_ctrlr_process_init(ctrlr) != 0) { 1825 NVME_CTRLR_ERRLOG(ctrlr, "failed to disable controller\n"); 1826 rc = -1; 1827 } 1828 1829 if (ctrlr->state != NVME_CTRLR_STATE_DISABLED && rc != -1) { 1830 return -EAGAIN; 1831 } 1832 1833 return rc; 1834 } 1835 1836 static void 1837 nvme_ctrlr_fail_io_qpairs(struct spdk_nvme_ctrlr *ctrlr) 1838 { 1839 struct spdk_nvme_qpair *qpair; 1840 1841 TAILQ_FOREACH(qpair, &ctrlr->active_io_qpairs, tailq) { 1842 qpair->transport_failure_reason = SPDK_NVME_QPAIR_FAILURE_LOCAL; 1843 } 1844 } 1845 1846 int 1847 spdk_nvme_ctrlr_reset(struct spdk_nvme_ctrlr *ctrlr) 1848 { 1849 int rc; 1850 1851 nvme_robust_mutex_lock(&ctrlr->ctrlr_lock); 1852 1853 rc = nvme_ctrlr_disconnect(ctrlr); 1854 if (rc == 0) { 1855 nvme_ctrlr_fail_io_qpairs(ctrlr); 1856 } 1857 1858 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 1859 1860 if (rc != 0) { 1861 if (rc == -EBUSY) { 1862 rc = 0; 1863 } 1864 return rc; 1865 } 1866 1867 while (1) { 1868 rc = spdk_nvme_ctrlr_process_admin_completions(ctrlr); 1869 if (rc == -ENXIO) { 1870 break; 1871 } 1872 } 1873 1874 spdk_nvme_ctrlr_reconnect_async(ctrlr); 1875 1876 while (true) { 1877 rc = spdk_nvme_ctrlr_reconnect_poll_async(ctrlr); 1878 if (rc != -EAGAIN) { 1879 break; 1880 } 1881 } 1882 1883 return rc; 1884 } 1885 1886 int 1887 spdk_nvme_ctrlr_reset_subsystem(struct spdk_nvme_ctrlr *ctrlr) 1888 { 1889 union spdk_nvme_cap_register cap; 1890 int rc = 0; 1891 1892 cap = spdk_nvme_ctrlr_get_regs_cap(ctrlr); 1893 if (cap.bits.nssrs == 0) { 1894 NVME_CTRLR_WARNLOG(ctrlr, "subsystem reset is not supported\n"); 1895 return -ENOTSUP; 1896 } 1897 1898 NVME_CTRLR_NOTICELOG(ctrlr, "resetting subsystem\n"); 1899 nvme_robust_mutex_lock(&ctrlr->ctrlr_lock); 1900 ctrlr->is_resetting = true; 1901 rc = nvme_ctrlr_set_nssr(ctrlr, SPDK_NVME_NSSR_VALUE); 1902 ctrlr->is_resetting = false; 1903 1904 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 1905 /* 1906 * No more cleanup at this point like in the ctrlr reset. A subsystem reset will cause 1907 * a hot remove for PCIe transport. The hot remove handling does all the necessary ctrlr cleanup. 1908 */ 1909 return rc; 1910 } 1911 1912 int 1913 spdk_nvme_ctrlr_set_trid(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_transport_id *trid) 1914 { 1915 int rc = 0; 1916 1917 nvme_robust_mutex_lock(&ctrlr->ctrlr_lock); 1918 1919 if (ctrlr->is_failed == false) { 1920 rc = -EPERM; 1921 goto out; 1922 } 1923 1924 if (trid->trtype != ctrlr->trid.trtype) { 1925 rc = -EINVAL; 1926 goto out; 1927 } 1928 1929 if (strncmp(trid->subnqn, ctrlr->trid.subnqn, SPDK_NVMF_NQN_MAX_LEN)) { 1930 rc = -EINVAL; 1931 goto out; 1932 } 1933 1934 ctrlr->trid = *trid; 1935 1936 out: 1937 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 1938 return rc; 1939 } 1940 1941 void 1942 spdk_nvme_ctrlr_set_remove_cb(struct spdk_nvme_ctrlr *ctrlr, 1943 spdk_nvme_remove_cb remove_cb, void *remove_ctx) 1944 { 1945 if (!spdk_process_is_primary()) { 1946 return; 1947 } 1948 1949 nvme_robust_mutex_lock(&ctrlr->ctrlr_lock); 1950 ctrlr->remove_cb = remove_cb; 1951 ctrlr->cb_ctx = remove_ctx; 1952 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 1953 } 1954 1955 static void 1956 nvme_ctrlr_identify_done(void *arg, const struct spdk_nvme_cpl *cpl) 1957 { 1958 struct spdk_nvme_ctrlr *ctrlr = (struct spdk_nvme_ctrlr *)arg; 1959 1960 if (spdk_nvme_cpl_is_error(cpl)) { 1961 NVME_CTRLR_ERRLOG(ctrlr, "nvme_identify_controller failed!\n"); 1962 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 1963 return; 1964 } 1965 1966 /* 1967 * Use MDTS to ensure our default max_xfer_size doesn't exceed what the 1968 * controller supports. 1969 */ 1970 ctrlr->max_xfer_size = nvme_transport_ctrlr_get_max_xfer_size(ctrlr); 1971 NVME_CTRLR_DEBUGLOG(ctrlr, "transport max_xfer_size %u\n", ctrlr->max_xfer_size); 1972 if (ctrlr->cdata.mdts > 0) { 1973 ctrlr->max_xfer_size = spdk_min(ctrlr->max_xfer_size, 1974 ctrlr->min_page_size * (1 << ctrlr->cdata.mdts)); 1975 NVME_CTRLR_DEBUGLOG(ctrlr, "MDTS max_xfer_size %u\n", ctrlr->max_xfer_size); 1976 } 1977 1978 NVME_CTRLR_DEBUGLOG(ctrlr, "CNTLID 0x%04" PRIx16 "\n", ctrlr->cdata.cntlid); 1979 if (ctrlr->trid.trtype == SPDK_NVME_TRANSPORT_PCIE) { 1980 ctrlr->cntlid = ctrlr->cdata.cntlid; 1981 } else { 1982 /* 1983 * Fabrics controllers should already have CNTLID from the Connect command. 1984 * 1985 * If CNTLID from Connect doesn't match CNTLID in the Identify Controller data, 1986 * trust the one from Connect. 1987 */ 1988 if (ctrlr->cntlid != ctrlr->cdata.cntlid) { 1989 NVME_CTRLR_DEBUGLOG(ctrlr, "Identify CNTLID 0x%04" PRIx16 " != Connect CNTLID 0x%04" PRIx16 "\n", 1990 ctrlr->cdata.cntlid, ctrlr->cntlid); 1991 } 1992 } 1993 1994 if (ctrlr->cdata.sgls.supported && !(ctrlr->quirks & NVME_QUIRK_NOT_USE_SGL)) { 1995 assert(ctrlr->cdata.sgls.supported != 0x3); 1996 ctrlr->flags |= SPDK_NVME_CTRLR_SGL_SUPPORTED; 1997 if (ctrlr->cdata.sgls.supported == 0x2) { 1998 ctrlr->flags |= SPDK_NVME_CTRLR_SGL_REQUIRES_DWORD_ALIGNMENT; 1999 } 2000 2001 ctrlr->max_sges = nvme_transport_ctrlr_get_max_sges(ctrlr); 2002 NVME_CTRLR_DEBUGLOG(ctrlr, "transport max_sges %u\n", ctrlr->max_sges); 2003 } 2004 2005 if (ctrlr->cdata.sgls.metadata_address && !(ctrlr->quirks & NVME_QUIRK_NOT_USE_SGL)) { 2006 ctrlr->flags |= SPDK_NVME_CTRLR_MPTR_SGL_SUPPORTED; 2007 } 2008 2009 if (ctrlr->cdata.oacs.security && !(ctrlr->quirks & NVME_QUIRK_OACS_SECURITY)) { 2010 ctrlr->flags |= SPDK_NVME_CTRLR_SECURITY_SEND_RECV_SUPPORTED; 2011 } 2012 2013 if (ctrlr->cdata.oacs.directives) { 2014 ctrlr->flags |= SPDK_NVME_CTRLR_DIRECTIVES_SUPPORTED; 2015 } 2016 2017 NVME_CTRLR_DEBUGLOG(ctrlr, "fuses compare and write: %d\n", 2018 ctrlr->cdata.fuses.compare_and_write); 2019 if (ctrlr->cdata.fuses.compare_and_write) { 2020 ctrlr->flags |= SPDK_NVME_CTRLR_COMPARE_AND_WRITE_SUPPORTED; 2021 } 2022 2023 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_CONFIGURE_AER, 2024 ctrlr->opts.admin_timeout_ms); 2025 } 2026 2027 static int 2028 nvme_ctrlr_identify(struct spdk_nvme_ctrlr *ctrlr) 2029 { 2030 int rc; 2031 2032 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY, 2033 ctrlr->opts.admin_timeout_ms); 2034 2035 rc = nvme_ctrlr_cmd_identify(ctrlr, SPDK_NVME_IDENTIFY_CTRLR, 0, 0, 0, 2036 &ctrlr->cdata, sizeof(ctrlr->cdata), 2037 nvme_ctrlr_identify_done, ctrlr); 2038 if (rc != 0) { 2039 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 2040 return rc; 2041 } 2042 2043 return 0; 2044 } 2045 2046 static void 2047 nvme_ctrlr_get_zns_cmd_and_effects_log_done(void *arg, const struct spdk_nvme_cpl *cpl) 2048 { 2049 struct spdk_nvme_cmds_and_effect_log_page *log_page; 2050 struct spdk_nvme_ctrlr *ctrlr = arg; 2051 2052 if (spdk_nvme_cpl_is_error(cpl)) { 2053 NVME_CTRLR_ERRLOG(ctrlr, "nvme_ctrlr_get_zns_cmd_and_effects_log failed!\n"); 2054 spdk_free(ctrlr->tmp_ptr); 2055 ctrlr->tmp_ptr = NULL; 2056 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 2057 return; 2058 } 2059 2060 log_page = ctrlr->tmp_ptr; 2061 2062 if (log_page->io_cmds_supported[SPDK_NVME_OPC_ZONE_APPEND].csupp) { 2063 ctrlr->flags |= SPDK_NVME_CTRLR_ZONE_APPEND_SUPPORTED; 2064 } 2065 spdk_free(ctrlr->tmp_ptr); 2066 ctrlr->tmp_ptr = NULL; 2067 2068 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_NUM_QUEUES, ctrlr->opts.admin_timeout_ms); 2069 } 2070 2071 static int 2072 nvme_ctrlr_get_zns_cmd_and_effects_log(struct spdk_nvme_ctrlr *ctrlr) 2073 { 2074 int rc; 2075 2076 assert(!ctrlr->tmp_ptr); 2077 ctrlr->tmp_ptr = spdk_zmalloc(sizeof(struct spdk_nvme_cmds_and_effect_log_page), 64, NULL, 2078 SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_SHARE | SPDK_MALLOC_DMA); 2079 if (!ctrlr->tmp_ptr) { 2080 rc = -ENOMEM; 2081 goto error; 2082 } 2083 2084 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_WAIT_FOR_GET_ZNS_CMD_EFFECTS_LOG, 2085 ctrlr->opts.admin_timeout_ms); 2086 2087 rc = spdk_nvme_ctrlr_cmd_get_log_page_ext(ctrlr, SPDK_NVME_LOG_COMMAND_EFFECTS_LOG, 2088 0, ctrlr->tmp_ptr, sizeof(struct spdk_nvme_cmds_and_effect_log_page), 2089 0, 0, 0, SPDK_NVME_CSI_ZNS << 24, 2090 nvme_ctrlr_get_zns_cmd_and_effects_log_done, ctrlr); 2091 if (rc != 0) { 2092 goto error; 2093 } 2094 2095 return 0; 2096 2097 error: 2098 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 2099 spdk_free(ctrlr->tmp_ptr); 2100 ctrlr->tmp_ptr = NULL; 2101 return rc; 2102 } 2103 2104 static void 2105 nvme_ctrlr_identify_zns_specific_done(void *arg, const struct spdk_nvme_cpl *cpl) 2106 { 2107 struct spdk_nvme_ctrlr *ctrlr = (struct spdk_nvme_ctrlr *)arg; 2108 2109 if (spdk_nvme_cpl_is_error(cpl)) { 2110 /* no need to print an error, the controller simply does not support ZNS */ 2111 nvme_ctrlr_free_zns_specific_data(ctrlr); 2112 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_NUM_QUEUES, 2113 ctrlr->opts.admin_timeout_ms); 2114 return; 2115 } 2116 2117 /* A zero zasl value means use mdts */ 2118 if (ctrlr->cdata_zns->zasl) { 2119 uint32_t max_append = ctrlr->min_page_size * (1 << ctrlr->cdata_zns->zasl); 2120 ctrlr->max_zone_append_size = spdk_min(ctrlr->max_xfer_size, max_append); 2121 } else { 2122 ctrlr->max_zone_append_size = ctrlr->max_xfer_size; 2123 } 2124 2125 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_GET_ZNS_CMD_EFFECTS_LOG, 2126 ctrlr->opts.admin_timeout_ms); 2127 } 2128 2129 /** 2130 * This function will try to fetch the I/O Command Specific Controller data structure for 2131 * each I/O Command Set supported by SPDK. 2132 * 2133 * If an I/O Command Set is not supported by the controller, "Invalid Field in Command" 2134 * will be returned. Since we are fetching in a exploratively way, getting an error back 2135 * from the controller should not be treated as fatal. 2136 * 2137 * I/O Command Sets not supported by SPDK will be skipped (e.g. Key Value Command Set). 2138 * 2139 * I/O Command Sets without a IOCS specific data structure (i.e. a zero-filled IOCS specific 2140 * data structure) will be skipped (e.g. NVM Command Set, Key Value Command Set). 2141 */ 2142 static int 2143 nvme_ctrlr_identify_iocs_specific(struct spdk_nvme_ctrlr *ctrlr) 2144 { 2145 int rc; 2146 2147 if (!nvme_ctrlr_multi_iocs_enabled(ctrlr)) { 2148 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_NUM_QUEUES, 2149 ctrlr->opts.admin_timeout_ms); 2150 return 0; 2151 } 2152 2153 /* 2154 * Since SPDK currently only needs to fetch a single Command Set, keep the code here, 2155 * instead of creating multiple NVME_CTRLR_STATE_IDENTIFY_IOCS_SPECIFIC substates, 2156 * which would require additional functions and complexity for no good reason. 2157 */ 2158 assert(!ctrlr->cdata_zns); 2159 ctrlr->cdata_zns = spdk_zmalloc(sizeof(*ctrlr->cdata_zns), 64, NULL, SPDK_ENV_SOCKET_ID_ANY, 2160 SPDK_MALLOC_SHARE | SPDK_MALLOC_DMA); 2161 if (!ctrlr->cdata_zns) { 2162 rc = -ENOMEM; 2163 goto error; 2164 } 2165 2166 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_IOCS_SPECIFIC, 2167 ctrlr->opts.admin_timeout_ms); 2168 2169 rc = nvme_ctrlr_cmd_identify(ctrlr, SPDK_NVME_IDENTIFY_CTRLR_IOCS, 0, 0, SPDK_NVME_CSI_ZNS, 2170 ctrlr->cdata_zns, sizeof(*ctrlr->cdata_zns), 2171 nvme_ctrlr_identify_zns_specific_done, ctrlr); 2172 if (rc != 0) { 2173 goto error; 2174 } 2175 2176 return 0; 2177 2178 error: 2179 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 2180 nvme_ctrlr_free_zns_specific_data(ctrlr); 2181 return rc; 2182 } 2183 2184 enum nvme_active_ns_state { 2185 NVME_ACTIVE_NS_STATE_IDLE, 2186 NVME_ACTIVE_NS_STATE_PROCESSING, 2187 NVME_ACTIVE_NS_STATE_DONE, 2188 NVME_ACTIVE_NS_STATE_ERROR 2189 }; 2190 2191 typedef void (*nvme_active_ns_ctx_deleter)(struct nvme_active_ns_ctx *); 2192 2193 struct nvme_active_ns_ctx { 2194 struct spdk_nvme_ctrlr *ctrlr; 2195 uint32_t page_count; 2196 uint32_t next_nsid; 2197 uint32_t *new_ns_list; 2198 nvme_active_ns_ctx_deleter deleter; 2199 2200 enum nvme_active_ns_state state; 2201 }; 2202 2203 static struct nvme_active_ns_ctx * 2204 nvme_active_ns_ctx_create(struct spdk_nvme_ctrlr *ctrlr, nvme_active_ns_ctx_deleter deleter) 2205 { 2206 struct nvme_active_ns_ctx *ctx; 2207 uint32_t *new_ns_list = NULL; 2208 2209 ctx = calloc(1, sizeof(*ctx)); 2210 if (!ctx) { 2211 NVME_CTRLR_ERRLOG(ctrlr, "Failed to allocate nvme_active_ns_ctx!\n"); 2212 return NULL; 2213 } 2214 2215 new_ns_list = spdk_zmalloc(sizeof(struct spdk_nvme_ns_list), ctrlr->page_size, 2216 NULL, SPDK_ENV_LCORE_ID_ANY, SPDK_MALLOC_SHARE); 2217 if (!new_ns_list) { 2218 NVME_CTRLR_ERRLOG(ctrlr, "Failed to allocate active_ns_list!\n"); 2219 free(ctx); 2220 return NULL; 2221 } 2222 2223 ctx->page_count = 1; 2224 ctx->new_ns_list = new_ns_list; 2225 ctx->ctrlr = ctrlr; 2226 ctx->deleter = deleter; 2227 2228 return ctx; 2229 } 2230 2231 static void 2232 nvme_active_ns_ctx_destroy(struct nvme_active_ns_ctx *ctx) 2233 { 2234 spdk_free(ctx->new_ns_list); 2235 free(ctx); 2236 } 2237 2238 static int 2239 nvme_ctrlr_destruct_namespace(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid) 2240 { 2241 struct spdk_nvme_ns tmp, *ns; 2242 2243 assert(ctrlr != NULL); 2244 2245 tmp.id = nsid; 2246 ns = RB_FIND(nvme_ns_tree, &ctrlr->ns, &tmp); 2247 if (ns == NULL) { 2248 return -EINVAL; 2249 } 2250 2251 nvme_ns_destruct(ns); 2252 ns->active = false; 2253 2254 return 0; 2255 } 2256 2257 static int 2258 nvme_ctrlr_construct_namespace(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid) 2259 { 2260 struct spdk_nvme_ns *ns; 2261 2262 if (nsid < 1 || nsid > ctrlr->cdata.nn) { 2263 return -EINVAL; 2264 } 2265 2266 /* Namespaces are constructed on demand, so simply request it. */ 2267 ns = spdk_nvme_ctrlr_get_ns(ctrlr, nsid); 2268 if (ns == NULL) { 2269 return -ENOMEM; 2270 } 2271 2272 ns->active = true; 2273 2274 return 0; 2275 } 2276 2277 static void 2278 nvme_ctrlr_identify_active_ns_swap(struct spdk_nvme_ctrlr *ctrlr, uint32_t *new_ns_list, 2279 size_t max_entries) 2280 { 2281 uint32_t active_ns_count = 0; 2282 size_t i; 2283 uint32_t nsid; 2284 struct spdk_nvme_ns *ns, *tmp_ns; 2285 int rc; 2286 2287 /* First, remove namespaces that no longer exist */ 2288 RB_FOREACH_SAFE(ns, nvme_ns_tree, &ctrlr->ns, tmp_ns) { 2289 nsid = new_ns_list[0]; 2290 active_ns_count = 0; 2291 while (nsid != 0) { 2292 if (nsid == ns->id) { 2293 break; 2294 } 2295 2296 nsid = new_ns_list[active_ns_count++]; 2297 } 2298 2299 if (nsid != ns->id) { 2300 /* Did not find this namespace id in the new list. */ 2301 NVME_CTRLR_DEBUGLOG(ctrlr, "Namespace %u was removed\n", ns->id); 2302 nvme_ctrlr_destruct_namespace(ctrlr, ns->id); 2303 } 2304 } 2305 2306 /* Next, add new namespaces */ 2307 active_ns_count = 0; 2308 for (i = 0; i < max_entries; i++) { 2309 nsid = new_ns_list[active_ns_count]; 2310 2311 if (nsid == 0) { 2312 break; 2313 } 2314 2315 /* If the namespace already exists, this will not construct it a second time. */ 2316 rc = nvme_ctrlr_construct_namespace(ctrlr, nsid); 2317 if (rc != 0) { 2318 /* We can't easily handle a failure here. But just move on. */ 2319 assert(false); 2320 NVME_CTRLR_DEBUGLOG(ctrlr, "Failed to allocate a namespace object.\n"); 2321 continue; 2322 } 2323 2324 active_ns_count++; 2325 } 2326 2327 ctrlr->active_ns_count = active_ns_count; 2328 } 2329 2330 static void 2331 nvme_ctrlr_identify_active_ns_async_done(void *arg, const struct spdk_nvme_cpl *cpl) 2332 { 2333 struct nvme_active_ns_ctx *ctx = arg; 2334 uint32_t *new_ns_list = NULL; 2335 2336 if (spdk_nvme_cpl_is_error(cpl)) { 2337 ctx->state = NVME_ACTIVE_NS_STATE_ERROR; 2338 goto out; 2339 } 2340 2341 ctx->next_nsid = ctx->new_ns_list[1024 * ctx->page_count - 1]; 2342 if (ctx->next_nsid == 0) { 2343 ctx->state = NVME_ACTIVE_NS_STATE_DONE; 2344 goto out; 2345 } 2346 2347 ctx->page_count++; 2348 new_ns_list = spdk_realloc(ctx->new_ns_list, 2349 ctx->page_count * sizeof(struct spdk_nvme_ns_list), 2350 ctx->ctrlr->page_size); 2351 if (!new_ns_list) { 2352 SPDK_ERRLOG("Failed to reallocate active_ns_list!\n"); 2353 ctx->state = NVME_ACTIVE_NS_STATE_ERROR; 2354 goto out; 2355 } 2356 2357 ctx->new_ns_list = new_ns_list; 2358 nvme_ctrlr_identify_active_ns_async(ctx); 2359 return; 2360 2361 out: 2362 if (ctx->deleter) { 2363 ctx->deleter(ctx); 2364 } 2365 } 2366 2367 static void 2368 nvme_ctrlr_identify_active_ns_async(struct nvme_active_ns_ctx *ctx) 2369 { 2370 struct spdk_nvme_ctrlr *ctrlr = ctx->ctrlr; 2371 uint32_t i; 2372 int rc; 2373 2374 if (ctrlr->cdata.nn == 0) { 2375 ctx->state = NVME_ACTIVE_NS_STATE_DONE; 2376 goto out; 2377 } 2378 2379 assert(ctx->new_ns_list != NULL); 2380 2381 /* 2382 * If controller doesn't support active ns list CNS 0x02 dummy up 2383 * an active ns list, i.e. all namespaces report as active 2384 */ 2385 if (ctrlr->vs.raw < SPDK_NVME_VERSION(1, 1, 0) || ctrlr->quirks & NVME_QUIRK_IDENTIFY_CNS) { 2386 uint32_t *new_ns_list; 2387 2388 /* 2389 * Active NS list must always end with zero element. 2390 * So, we allocate for cdata.nn+1. 2391 */ 2392 ctx->page_count = spdk_divide_round_up(ctrlr->cdata.nn + 1, 2393 sizeof(struct spdk_nvme_ns_list) / sizeof(new_ns_list[0])); 2394 new_ns_list = spdk_realloc(ctx->new_ns_list, 2395 ctx->page_count * sizeof(struct spdk_nvme_ns_list), 2396 ctx->ctrlr->page_size); 2397 if (!new_ns_list) { 2398 SPDK_ERRLOG("Failed to reallocate active_ns_list!\n"); 2399 ctx->state = NVME_ACTIVE_NS_STATE_ERROR; 2400 goto out; 2401 } 2402 2403 ctx->new_ns_list = new_ns_list; 2404 ctx->new_ns_list[ctrlr->cdata.nn] = 0; 2405 for (i = 0; i < ctrlr->cdata.nn; i++) { 2406 ctx->new_ns_list[i] = i + 1; 2407 } 2408 2409 ctx->state = NVME_ACTIVE_NS_STATE_DONE; 2410 goto out; 2411 } 2412 2413 ctx->state = NVME_ACTIVE_NS_STATE_PROCESSING; 2414 rc = nvme_ctrlr_cmd_identify(ctrlr, SPDK_NVME_IDENTIFY_ACTIVE_NS_LIST, 0, ctx->next_nsid, 0, 2415 &ctx->new_ns_list[1024 * (ctx->page_count - 1)], sizeof(struct spdk_nvme_ns_list), 2416 nvme_ctrlr_identify_active_ns_async_done, ctx); 2417 if (rc != 0) { 2418 ctx->state = NVME_ACTIVE_NS_STATE_ERROR; 2419 goto out; 2420 } 2421 2422 return; 2423 2424 out: 2425 if (ctx->deleter) { 2426 ctx->deleter(ctx); 2427 } 2428 } 2429 2430 static void 2431 _nvme_active_ns_ctx_deleter(struct nvme_active_ns_ctx *ctx) 2432 { 2433 struct spdk_nvme_ctrlr *ctrlr = ctx->ctrlr; 2434 struct spdk_nvme_ns *ns; 2435 2436 if (ctx->state == NVME_ACTIVE_NS_STATE_ERROR) { 2437 nvme_active_ns_ctx_destroy(ctx); 2438 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 2439 return; 2440 } 2441 2442 assert(ctx->state == NVME_ACTIVE_NS_STATE_DONE); 2443 2444 RB_FOREACH(ns, nvme_ns_tree, &ctrlr->ns) { 2445 nvme_ns_free_iocs_specific_data(ns); 2446 } 2447 2448 nvme_ctrlr_identify_active_ns_swap(ctrlr, ctx->new_ns_list, ctx->page_count * 1024); 2449 nvme_active_ns_ctx_destroy(ctx); 2450 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_IDENTIFY_NS, ctrlr->opts.admin_timeout_ms); 2451 } 2452 2453 static void 2454 _nvme_ctrlr_identify_active_ns(struct spdk_nvme_ctrlr *ctrlr) 2455 { 2456 struct nvme_active_ns_ctx *ctx; 2457 2458 ctx = nvme_active_ns_ctx_create(ctrlr, _nvme_active_ns_ctx_deleter); 2459 if (!ctx) { 2460 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 2461 return; 2462 } 2463 2464 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_ACTIVE_NS, 2465 ctrlr->opts.admin_timeout_ms); 2466 nvme_ctrlr_identify_active_ns_async(ctx); 2467 } 2468 2469 int 2470 nvme_ctrlr_identify_active_ns(struct spdk_nvme_ctrlr *ctrlr) 2471 { 2472 struct nvme_active_ns_ctx *ctx; 2473 int rc; 2474 2475 ctx = nvme_active_ns_ctx_create(ctrlr, NULL); 2476 if (!ctx) { 2477 return -ENOMEM; 2478 } 2479 2480 nvme_ctrlr_identify_active_ns_async(ctx); 2481 while (ctx->state == NVME_ACTIVE_NS_STATE_PROCESSING) { 2482 rc = spdk_nvme_qpair_process_completions(ctrlr->adminq, 0); 2483 if (rc < 0) { 2484 ctx->state = NVME_ACTIVE_NS_STATE_ERROR; 2485 break; 2486 } 2487 } 2488 2489 if (ctx->state == NVME_ACTIVE_NS_STATE_ERROR) { 2490 nvme_active_ns_ctx_destroy(ctx); 2491 return -ENXIO; 2492 } 2493 2494 assert(ctx->state == NVME_ACTIVE_NS_STATE_DONE); 2495 nvme_ctrlr_identify_active_ns_swap(ctrlr, ctx->new_ns_list, ctx->page_count * 1024); 2496 nvme_active_ns_ctx_destroy(ctx); 2497 2498 return 0; 2499 } 2500 2501 static void 2502 nvme_ctrlr_identify_ns_async_done(void *arg, const struct spdk_nvme_cpl *cpl) 2503 { 2504 struct spdk_nvme_ns *ns = (struct spdk_nvme_ns *)arg; 2505 struct spdk_nvme_ctrlr *ctrlr = ns->ctrlr; 2506 uint32_t nsid; 2507 int rc; 2508 2509 if (spdk_nvme_cpl_is_error(cpl)) { 2510 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 2511 return; 2512 } 2513 2514 nvme_ns_set_identify_data(ns); 2515 2516 /* move on to the next active NS */ 2517 nsid = spdk_nvme_ctrlr_get_next_active_ns(ctrlr, ns->id); 2518 ns = spdk_nvme_ctrlr_get_ns(ctrlr, nsid); 2519 if (ns == NULL) { 2520 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_IDENTIFY_ID_DESCS, 2521 ctrlr->opts.admin_timeout_ms); 2522 return; 2523 } 2524 ns->ctrlr = ctrlr; 2525 ns->id = nsid; 2526 2527 rc = nvme_ctrlr_identify_ns_async(ns); 2528 if (rc) { 2529 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 2530 } 2531 } 2532 2533 static int 2534 nvme_ctrlr_identify_ns_async(struct spdk_nvme_ns *ns) 2535 { 2536 struct spdk_nvme_ctrlr *ctrlr = ns->ctrlr; 2537 struct spdk_nvme_ns_data *nsdata; 2538 2539 nsdata = &ns->nsdata; 2540 2541 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_NS, 2542 ctrlr->opts.admin_timeout_ms); 2543 return nvme_ctrlr_cmd_identify(ns->ctrlr, SPDK_NVME_IDENTIFY_NS, 0, ns->id, 0, 2544 nsdata, sizeof(*nsdata), 2545 nvme_ctrlr_identify_ns_async_done, ns); 2546 } 2547 2548 static int 2549 nvme_ctrlr_identify_namespaces(struct spdk_nvme_ctrlr *ctrlr) 2550 { 2551 uint32_t nsid; 2552 struct spdk_nvme_ns *ns; 2553 int rc; 2554 2555 nsid = spdk_nvme_ctrlr_get_first_active_ns(ctrlr); 2556 ns = spdk_nvme_ctrlr_get_ns(ctrlr, nsid); 2557 if (ns == NULL) { 2558 /* No active NS, move on to the next state */ 2559 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_IDENTIFY_ID_DESCS, 2560 ctrlr->opts.admin_timeout_ms); 2561 return 0; 2562 } 2563 2564 ns->ctrlr = ctrlr; 2565 ns->id = nsid; 2566 2567 rc = nvme_ctrlr_identify_ns_async(ns); 2568 if (rc) { 2569 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 2570 } 2571 2572 return rc; 2573 } 2574 2575 static int 2576 nvme_ctrlr_identify_namespaces_iocs_specific_next(struct spdk_nvme_ctrlr *ctrlr, uint32_t prev_nsid) 2577 { 2578 uint32_t nsid; 2579 struct spdk_nvme_ns *ns; 2580 int rc; 2581 2582 if (!prev_nsid) { 2583 nsid = spdk_nvme_ctrlr_get_first_active_ns(ctrlr); 2584 } else { 2585 /* move on to the next active NS */ 2586 nsid = spdk_nvme_ctrlr_get_next_active_ns(ctrlr, prev_nsid); 2587 } 2588 2589 ns = spdk_nvme_ctrlr_get_ns(ctrlr, nsid); 2590 if (ns == NULL) { 2591 /* No first/next active NS, move on to the next state */ 2592 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_SUPPORTED_LOG_PAGES, 2593 ctrlr->opts.admin_timeout_ms); 2594 return 0; 2595 } 2596 2597 /* loop until we find a ns which has (supported) iocs specific data */ 2598 while (!nvme_ns_has_supported_iocs_specific_data(ns)) { 2599 nsid = spdk_nvme_ctrlr_get_next_active_ns(ctrlr, ns->id); 2600 ns = spdk_nvme_ctrlr_get_ns(ctrlr, nsid); 2601 if (ns == NULL) { 2602 /* no namespace with (supported) iocs specific data found */ 2603 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_SUPPORTED_LOG_PAGES, 2604 ctrlr->opts.admin_timeout_ms); 2605 return 0; 2606 } 2607 } 2608 2609 rc = nvme_ctrlr_identify_ns_iocs_specific_async(ns); 2610 if (rc) { 2611 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 2612 } 2613 2614 return rc; 2615 } 2616 2617 static void 2618 nvme_ctrlr_identify_ns_zns_specific_async_done(void *arg, const struct spdk_nvme_cpl *cpl) 2619 { 2620 struct spdk_nvme_ns *ns = (struct spdk_nvme_ns *)arg; 2621 struct spdk_nvme_ctrlr *ctrlr = ns->ctrlr; 2622 2623 if (spdk_nvme_cpl_is_error(cpl)) { 2624 nvme_ns_free_zns_specific_data(ns); 2625 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 2626 return; 2627 } 2628 2629 nvme_ctrlr_identify_namespaces_iocs_specific_next(ctrlr, ns->id); 2630 } 2631 2632 static int 2633 nvme_ctrlr_identify_ns_iocs_specific_async(struct spdk_nvme_ns *ns) 2634 { 2635 struct spdk_nvme_ctrlr *ctrlr = ns->ctrlr; 2636 int rc; 2637 2638 switch (ns->csi) { 2639 case SPDK_NVME_CSI_ZNS: 2640 break; 2641 default: 2642 /* 2643 * This switch must handle all cases for which 2644 * nvme_ns_has_supported_iocs_specific_data() returns true, 2645 * other cases should never happen. 2646 */ 2647 assert(0); 2648 } 2649 2650 assert(!ns->nsdata_zns); 2651 ns->nsdata_zns = spdk_zmalloc(sizeof(*ns->nsdata_zns), 64, NULL, SPDK_ENV_SOCKET_ID_ANY, 2652 SPDK_MALLOC_SHARE); 2653 if (!ns->nsdata_zns) { 2654 return -ENOMEM; 2655 } 2656 2657 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_NS_IOCS_SPECIFIC, 2658 ctrlr->opts.admin_timeout_ms); 2659 rc = nvme_ctrlr_cmd_identify(ns->ctrlr, SPDK_NVME_IDENTIFY_NS_IOCS, 0, ns->id, ns->csi, 2660 ns->nsdata_zns, sizeof(*ns->nsdata_zns), 2661 nvme_ctrlr_identify_ns_zns_specific_async_done, ns); 2662 if (rc) { 2663 nvme_ns_free_zns_specific_data(ns); 2664 } 2665 2666 return rc; 2667 } 2668 2669 static int 2670 nvme_ctrlr_identify_namespaces_iocs_specific(struct spdk_nvme_ctrlr *ctrlr) 2671 { 2672 if (!nvme_ctrlr_multi_iocs_enabled(ctrlr)) { 2673 /* Multi IOCS not supported/enabled, move on to the next state */ 2674 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_SUPPORTED_LOG_PAGES, 2675 ctrlr->opts.admin_timeout_ms); 2676 return 0; 2677 } 2678 2679 return nvme_ctrlr_identify_namespaces_iocs_specific_next(ctrlr, 0); 2680 } 2681 2682 static void 2683 nvme_ctrlr_identify_id_desc_async_done(void *arg, const struct spdk_nvme_cpl *cpl) 2684 { 2685 struct spdk_nvme_ns *ns = (struct spdk_nvme_ns *)arg; 2686 struct spdk_nvme_ctrlr *ctrlr = ns->ctrlr; 2687 uint32_t nsid; 2688 int rc; 2689 2690 if (spdk_nvme_cpl_is_error(cpl)) { 2691 /* 2692 * Many controllers claim to be compatible with NVMe 1.3, however, 2693 * they do not implement NS ID Desc List. Therefore, instead of setting 2694 * the state to NVME_CTRLR_STATE_ERROR, silently ignore the completion 2695 * error and move on to the next state. 2696 * 2697 * The proper way is to create a new quirk for controllers that violate 2698 * the NVMe 1.3 spec by not supporting NS ID Desc List. 2699 * (Re-using the NVME_QUIRK_IDENTIFY_CNS quirk is not possible, since 2700 * it is too generic and was added in order to handle controllers that 2701 * violate the NVMe 1.1 spec by not supporting ACTIVE LIST). 2702 */ 2703 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_IDENTIFY_NS_IOCS_SPECIFIC, 2704 ctrlr->opts.admin_timeout_ms); 2705 return; 2706 } 2707 2708 nvme_ns_set_id_desc_list_data(ns); 2709 2710 /* move on to the next active NS */ 2711 nsid = spdk_nvme_ctrlr_get_next_active_ns(ctrlr, ns->id); 2712 ns = spdk_nvme_ctrlr_get_ns(ctrlr, nsid); 2713 if (ns == NULL) { 2714 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_IDENTIFY_NS_IOCS_SPECIFIC, 2715 ctrlr->opts.admin_timeout_ms); 2716 return; 2717 } 2718 2719 rc = nvme_ctrlr_identify_id_desc_async(ns); 2720 if (rc) { 2721 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 2722 } 2723 } 2724 2725 static int 2726 nvme_ctrlr_identify_id_desc_async(struct spdk_nvme_ns *ns) 2727 { 2728 struct spdk_nvme_ctrlr *ctrlr = ns->ctrlr; 2729 2730 memset(ns->id_desc_list, 0, sizeof(ns->id_desc_list)); 2731 2732 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_ID_DESCS, 2733 ctrlr->opts.admin_timeout_ms); 2734 return nvme_ctrlr_cmd_identify(ns->ctrlr, SPDK_NVME_IDENTIFY_NS_ID_DESCRIPTOR_LIST, 2735 0, ns->id, 0, ns->id_desc_list, sizeof(ns->id_desc_list), 2736 nvme_ctrlr_identify_id_desc_async_done, ns); 2737 } 2738 2739 static int 2740 nvme_ctrlr_identify_id_desc_namespaces(struct spdk_nvme_ctrlr *ctrlr) 2741 { 2742 uint32_t nsid; 2743 struct spdk_nvme_ns *ns; 2744 int rc; 2745 2746 if ((ctrlr->vs.raw < SPDK_NVME_VERSION(1, 3, 0) && 2747 !(ctrlr->cap.bits.css & SPDK_NVME_CAP_CSS_IOCS)) || 2748 (ctrlr->quirks & NVME_QUIRK_IDENTIFY_CNS)) { 2749 NVME_CTRLR_DEBUGLOG(ctrlr, "Version < 1.3; not attempting to retrieve NS ID Descriptor List\n"); 2750 /* NS ID Desc List not supported, move on to the next state */ 2751 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_IDENTIFY_NS_IOCS_SPECIFIC, 2752 ctrlr->opts.admin_timeout_ms); 2753 return 0; 2754 } 2755 2756 nsid = spdk_nvme_ctrlr_get_first_active_ns(ctrlr); 2757 ns = spdk_nvme_ctrlr_get_ns(ctrlr, nsid); 2758 if (ns == NULL) { 2759 /* No active NS, move on to the next state */ 2760 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_IDENTIFY_NS_IOCS_SPECIFIC, 2761 ctrlr->opts.admin_timeout_ms); 2762 return 0; 2763 } 2764 2765 rc = nvme_ctrlr_identify_id_desc_async(ns); 2766 if (rc) { 2767 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 2768 } 2769 2770 return rc; 2771 } 2772 2773 static void 2774 nvme_ctrlr_update_nvmf_ioccsz(struct spdk_nvme_ctrlr *ctrlr) 2775 { 2776 if (spdk_nvme_ctrlr_is_fabrics(ctrlr)) { 2777 if (ctrlr->cdata.nvmf_specific.ioccsz < 4) { 2778 NVME_CTRLR_ERRLOG(ctrlr, "Incorrect IOCCSZ %u, the minimum value should be 4\n", 2779 ctrlr->cdata.nvmf_specific.ioccsz); 2780 ctrlr->cdata.nvmf_specific.ioccsz = 4; 2781 assert(0); 2782 } 2783 ctrlr->ioccsz_bytes = ctrlr->cdata.nvmf_specific.ioccsz * 16 - sizeof(struct spdk_nvme_cmd); 2784 ctrlr->icdoff = ctrlr->cdata.nvmf_specific.icdoff; 2785 } 2786 } 2787 2788 static void 2789 nvme_ctrlr_set_num_queues_done(void *arg, const struct spdk_nvme_cpl *cpl) 2790 { 2791 uint32_t cq_allocated, sq_allocated, min_allocated, i; 2792 struct spdk_nvme_ctrlr *ctrlr = (struct spdk_nvme_ctrlr *)arg; 2793 2794 if (spdk_nvme_cpl_is_error(cpl)) { 2795 NVME_CTRLR_ERRLOG(ctrlr, "Set Features - Number of Queues failed!\n"); 2796 ctrlr->opts.num_io_queues = 0; 2797 } else { 2798 /* 2799 * Data in cdw0 is 0-based. 2800 * Lower 16-bits indicate number of submission queues allocated. 2801 * Upper 16-bits indicate number of completion queues allocated. 2802 */ 2803 sq_allocated = (cpl->cdw0 & 0xFFFF) + 1; 2804 cq_allocated = (cpl->cdw0 >> 16) + 1; 2805 2806 /* 2807 * For 1:1 queue mapping, set number of allocated queues to be minimum of 2808 * submission and completion queues. 2809 */ 2810 min_allocated = spdk_min(sq_allocated, cq_allocated); 2811 2812 /* Set number of queues to be minimum of requested and actually allocated. */ 2813 ctrlr->opts.num_io_queues = spdk_min(min_allocated, ctrlr->opts.num_io_queues); 2814 } 2815 2816 ctrlr->free_io_qids = spdk_bit_array_create(ctrlr->opts.num_io_queues + 1); 2817 if (ctrlr->free_io_qids == NULL) { 2818 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 2819 return; 2820 } 2821 2822 /* Initialize list of free I/O queue IDs. QID 0 is the admin queue (implicitly allocated). */ 2823 for (i = 1; i <= ctrlr->opts.num_io_queues; i++) { 2824 spdk_nvme_ctrlr_free_qid(ctrlr, i); 2825 } 2826 2827 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_IDENTIFY_ACTIVE_NS, 2828 ctrlr->opts.admin_timeout_ms); 2829 } 2830 2831 static int 2832 nvme_ctrlr_set_num_queues(struct spdk_nvme_ctrlr *ctrlr) 2833 { 2834 int rc; 2835 2836 if (ctrlr->opts.num_io_queues > SPDK_NVME_MAX_IO_QUEUES) { 2837 NVME_CTRLR_NOTICELOG(ctrlr, "Limiting requested num_io_queues %u to max %d\n", 2838 ctrlr->opts.num_io_queues, SPDK_NVME_MAX_IO_QUEUES); 2839 ctrlr->opts.num_io_queues = SPDK_NVME_MAX_IO_QUEUES; 2840 } else if (ctrlr->opts.num_io_queues < 1) { 2841 NVME_CTRLR_NOTICELOG(ctrlr, "Requested num_io_queues 0, increasing to 1\n"); 2842 ctrlr->opts.num_io_queues = 1; 2843 } 2844 2845 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_WAIT_FOR_SET_NUM_QUEUES, 2846 ctrlr->opts.admin_timeout_ms); 2847 2848 rc = nvme_ctrlr_cmd_set_num_queues(ctrlr, ctrlr->opts.num_io_queues, 2849 nvme_ctrlr_set_num_queues_done, ctrlr); 2850 if (rc != 0) { 2851 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 2852 return rc; 2853 } 2854 2855 return 0; 2856 } 2857 2858 static void 2859 nvme_ctrlr_set_keep_alive_timeout_done(void *arg, const struct spdk_nvme_cpl *cpl) 2860 { 2861 uint32_t keep_alive_interval_us; 2862 struct spdk_nvme_ctrlr *ctrlr = (struct spdk_nvme_ctrlr *)arg; 2863 2864 if (spdk_nvme_cpl_is_error(cpl)) { 2865 if ((cpl->status.sct == SPDK_NVME_SCT_GENERIC) && 2866 (cpl->status.sc == SPDK_NVME_SC_INVALID_FIELD)) { 2867 NVME_CTRLR_DEBUGLOG(ctrlr, "Keep alive timeout Get Feature is not supported\n"); 2868 } else { 2869 NVME_CTRLR_ERRLOG(ctrlr, "Keep alive timeout Get Feature failed: SC %x SCT %x\n", 2870 cpl->status.sc, cpl->status.sct); 2871 ctrlr->opts.keep_alive_timeout_ms = 0; 2872 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 2873 return; 2874 } 2875 } else { 2876 if (ctrlr->opts.keep_alive_timeout_ms != cpl->cdw0) { 2877 NVME_CTRLR_DEBUGLOG(ctrlr, "Controller adjusted keep alive timeout to %u ms\n", 2878 cpl->cdw0); 2879 } 2880 2881 ctrlr->opts.keep_alive_timeout_ms = cpl->cdw0; 2882 } 2883 2884 if (ctrlr->opts.keep_alive_timeout_ms == 0) { 2885 ctrlr->keep_alive_interval_ticks = 0; 2886 } else { 2887 keep_alive_interval_us = ctrlr->opts.keep_alive_timeout_ms * 1000 / 2; 2888 2889 NVME_CTRLR_DEBUGLOG(ctrlr, "Sending keep alive every %u us\n", keep_alive_interval_us); 2890 2891 ctrlr->keep_alive_interval_ticks = (keep_alive_interval_us * spdk_get_ticks_hz()) / 2892 UINT64_C(1000000); 2893 2894 /* Schedule the first Keep Alive to be sent as soon as possible. */ 2895 ctrlr->next_keep_alive_tick = spdk_get_ticks(); 2896 } 2897 2898 if (spdk_nvme_ctrlr_is_discovery(ctrlr)) { 2899 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_READY, NVME_TIMEOUT_INFINITE); 2900 } else { 2901 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_IDENTIFY_IOCS_SPECIFIC, 2902 ctrlr->opts.admin_timeout_ms); 2903 } 2904 } 2905 2906 static int 2907 nvme_ctrlr_set_keep_alive_timeout(struct spdk_nvme_ctrlr *ctrlr) 2908 { 2909 int rc; 2910 2911 if (ctrlr->opts.keep_alive_timeout_ms == 0) { 2912 if (spdk_nvme_ctrlr_is_discovery(ctrlr)) { 2913 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_READY, NVME_TIMEOUT_INFINITE); 2914 } else { 2915 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_IDENTIFY_IOCS_SPECIFIC, 2916 ctrlr->opts.admin_timeout_ms); 2917 } 2918 return 0; 2919 } 2920 2921 /* Note: Discovery controller identify data does not populate KAS according to spec. */ 2922 if (!spdk_nvme_ctrlr_is_discovery(ctrlr) && ctrlr->cdata.kas == 0) { 2923 NVME_CTRLR_DEBUGLOG(ctrlr, "Controller KAS is 0 - not enabling Keep Alive\n"); 2924 ctrlr->opts.keep_alive_timeout_ms = 0; 2925 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_IDENTIFY_IOCS_SPECIFIC, 2926 ctrlr->opts.admin_timeout_ms); 2927 return 0; 2928 } 2929 2930 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_WAIT_FOR_KEEP_ALIVE_TIMEOUT, 2931 ctrlr->opts.admin_timeout_ms); 2932 2933 /* Retrieve actual keep alive timeout, since the controller may have adjusted it. */ 2934 rc = spdk_nvme_ctrlr_cmd_get_feature(ctrlr, SPDK_NVME_FEAT_KEEP_ALIVE_TIMER, 0, NULL, 0, 2935 nvme_ctrlr_set_keep_alive_timeout_done, ctrlr); 2936 if (rc != 0) { 2937 NVME_CTRLR_ERRLOG(ctrlr, "Keep alive timeout Get Feature failed: %d\n", rc); 2938 ctrlr->opts.keep_alive_timeout_ms = 0; 2939 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 2940 return rc; 2941 } 2942 2943 return 0; 2944 } 2945 2946 static void 2947 nvme_ctrlr_set_host_id_done(void *arg, const struct spdk_nvme_cpl *cpl) 2948 { 2949 struct spdk_nvme_ctrlr *ctrlr = (struct spdk_nvme_ctrlr *)arg; 2950 2951 if (spdk_nvme_cpl_is_error(cpl)) { 2952 /* 2953 * Treat Set Features - Host ID failure as non-fatal, since the Host ID feature 2954 * is optional. 2955 */ 2956 NVME_CTRLR_WARNLOG(ctrlr, "Set Features - Host ID failed: SC 0x%x SCT 0x%x\n", 2957 cpl->status.sc, cpl->status.sct); 2958 } else { 2959 NVME_CTRLR_DEBUGLOG(ctrlr, "Set Features - Host ID was successful\n"); 2960 } 2961 2962 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_TRANSPORT_READY, ctrlr->opts.admin_timeout_ms); 2963 } 2964 2965 static int 2966 nvme_ctrlr_set_host_id(struct spdk_nvme_ctrlr *ctrlr) 2967 { 2968 uint8_t *host_id; 2969 uint32_t host_id_size; 2970 int rc; 2971 2972 if (ctrlr->trid.trtype != SPDK_NVME_TRANSPORT_PCIE) { 2973 /* 2974 * NVMe-oF sends the host ID during Connect and doesn't allow 2975 * Set Features - Host Identifier after Connect, so we don't need to do anything here. 2976 */ 2977 NVME_CTRLR_DEBUGLOG(ctrlr, "NVMe-oF transport - not sending Set Features - Host ID\n"); 2978 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_TRANSPORT_READY, ctrlr->opts.admin_timeout_ms); 2979 return 0; 2980 } 2981 2982 if (ctrlr->cdata.ctratt.host_id_exhid_supported) { 2983 NVME_CTRLR_DEBUGLOG(ctrlr, "Using 128-bit extended host identifier\n"); 2984 host_id = ctrlr->opts.extended_host_id; 2985 host_id_size = sizeof(ctrlr->opts.extended_host_id); 2986 } else { 2987 NVME_CTRLR_DEBUGLOG(ctrlr, "Using 64-bit host identifier\n"); 2988 host_id = ctrlr->opts.host_id; 2989 host_id_size = sizeof(ctrlr->opts.host_id); 2990 } 2991 2992 /* If the user specified an all-zeroes host identifier, don't send the command. */ 2993 if (spdk_mem_all_zero(host_id, host_id_size)) { 2994 NVME_CTRLR_DEBUGLOG(ctrlr, "User did not specify host ID - not sending Set Features - Host ID\n"); 2995 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_TRANSPORT_READY, ctrlr->opts.admin_timeout_ms); 2996 return 0; 2997 } 2998 2999 SPDK_LOGDUMP(nvme, "host_id", host_id, host_id_size); 3000 3001 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_WAIT_FOR_HOST_ID, 3002 ctrlr->opts.admin_timeout_ms); 3003 3004 rc = nvme_ctrlr_cmd_set_host_id(ctrlr, host_id, host_id_size, nvme_ctrlr_set_host_id_done, ctrlr); 3005 if (rc != 0) { 3006 NVME_CTRLR_ERRLOG(ctrlr, "Set Features - Host ID failed: %d\n", rc); 3007 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 3008 return rc; 3009 } 3010 3011 return 0; 3012 } 3013 3014 void 3015 nvme_ctrlr_update_namespaces(struct spdk_nvme_ctrlr *ctrlr) 3016 { 3017 uint32_t nsid; 3018 struct spdk_nvme_ns *ns; 3019 3020 for (nsid = spdk_nvme_ctrlr_get_first_active_ns(ctrlr); 3021 nsid != 0; nsid = spdk_nvme_ctrlr_get_next_active_ns(ctrlr, nsid)) { 3022 ns = spdk_nvme_ctrlr_get_ns(ctrlr, nsid); 3023 nvme_ns_construct(ns, nsid, ctrlr); 3024 } 3025 } 3026 3027 static int 3028 nvme_ctrlr_clear_changed_ns_log(struct spdk_nvme_ctrlr *ctrlr) 3029 { 3030 struct nvme_completion_poll_status *status; 3031 int rc = -ENOMEM; 3032 char *buffer = NULL; 3033 uint32_t nsid; 3034 size_t buf_size = (SPDK_NVME_MAX_CHANGED_NAMESPACES * sizeof(uint32_t)); 3035 3036 if (ctrlr->opts.disable_read_changed_ns_list_log_page) { 3037 return 0; 3038 } 3039 3040 buffer = spdk_dma_zmalloc(buf_size, 4096, NULL); 3041 if (!buffer) { 3042 NVME_CTRLR_ERRLOG(ctrlr, "Failed to allocate buffer for getting " 3043 "changed ns log.\n"); 3044 return rc; 3045 } 3046 3047 status = calloc(1, sizeof(*status)); 3048 if (!status) { 3049 NVME_CTRLR_ERRLOG(ctrlr, "Failed to allocate status tracker\n"); 3050 goto free_buffer; 3051 } 3052 3053 rc = spdk_nvme_ctrlr_cmd_get_log_page(ctrlr, 3054 SPDK_NVME_LOG_CHANGED_NS_LIST, 3055 SPDK_NVME_GLOBAL_NS_TAG, 3056 buffer, buf_size, 0, 3057 nvme_completion_poll_cb, status); 3058 3059 if (rc) { 3060 NVME_CTRLR_ERRLOG(ctrlr, "spdk_nvme_ctrlr_cmd_get_log_page() failed: rc=%d\n", rc); 3061 free(status); 3062 goto free_buffer; 3063 } 3064 3065 rc = nvme_wait_for_completion_timeout(ctrlr->adminq, status, 3066 ctrlr->opts.admin_timeout_ms * 1000); 3067 if (!status->timed_out) { 3068 free(status); 3069 } 3070 3071 if (rc) { 3072 NVME_CTRLR_ERRLOG(ctrlr, "wait for spdk_nvme_ctrlr_cmd_get_log_page failed: rc=%d\n", rc); 3073 goto free_buffer; 3074 } 3075 3076 /* only check the case of overflow. */ 3077 nsid = from_le32(buffer); 3078 if (nsid == 0xffffffffu) { 3079 NVME_CTRLR_WARNLOG(ctrlr, "changed ns log overflowed.\n"); 3080 } 3081 3082 free_buffer: 3083 spdk_dma_free(buffer); 3084 return rc; 3085 } 3086 3087 void 3088 nvme_ctrlr_process_async_event(struct spdk_nvme_ctrlr *ctrlr, 3089 const struct spdk_nvme_cpl *cpl) 3090 { 3091 union spdk_nvme_async_event_completion event; 3092 struct spdk_nvme_ctrlr_process *active_proc; 3093 int rc; 3094 3095 event.raw = cpl->cdw0; 3096 3097 if ((event.bits.async_event_type == SPDK_NVME_ASYNC_EVENT_TYPE_NOTICE) && 3098 (event.bits.async_event_info == SPDK_NVME_ASYNC_EVENT_NS_ATTR_CHANGED)) { 3099 nvme_ctrlr_clear_changed_ns_log(ctrlr); 3100 3101 rc = nvme_ctrlr_identify_active_ns(ctrlr); 3102 if (rc) { 3103 return; 3104 } 3105 nvme_ctrlr_update_namespaces(ctrlr); 3106 nvme_io_msg_ctrlr_update(ctrlr); 3107 } 3108 3109 if ((event.bits.async_event_type == SPDK_NVME_ASYNC_EVENT_TYPE_NOTICE) && 3110 (event.bits.async_event_info == SPDK_NVME_ASYNC_EVENT_ANA_CHANGE)) { 3111 if (!ctrlr->opts.disable_read_ana_log_page) { 3112 rc = nvme_ctrlr_update_ana_log_page(ctrlr); 3113 if (rc) { 3114 return; 3115 } 3116 nvme_ctrlr_parse_ana_log_page(ctrlr, nvme_ctrlr_update_ns_ana_states, 3117 ctrlr); 3118 } 3119 } 3120 3121 active_proc = nvme_ctrlr_get_current_process(ctrlr); 3122 if (active_proc && active_proc->aer_cb_fn) { 3123 active_proc->aer_cb_fn(active_proc->aer_cb_arg, cpl); 3124 } 3125 } 3126 3127 static void 3128 nvme_ctrlr_queue_async_event(struct spdk_nvme_ctrlr *ctrlr, 3129 const struct spdk_nvme_cpl *cpl) 3130 { 3131 struct spdk_nvme_ctrlr_aer_completion_list *nvme_event; 3132 struct spdk_nvme_ctrlr_process *proc; 3133 3134 /* Add async event to each process objects event list */ 3135 TAILQ_FOREACH(proc, &ctrlr->active_procs, tailq) { 3136 /* Must be shared memory so other processes can access */ 3137 nvme_event = spdk_zmalloc(sizeof(*nvme_event), 0, NULL, SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_SHARE); 3138 if (!nvme_event) { 3139 NVME_CTRLR_ERRLOG(ctrlr, "Alloc nvme event failed, ignore the event\n"); 3140 return; 3141 } 3142 nvme_event->cpl = *cpl; 3143 3144 STAILQ_INSERT_TAIL(&proc->async_events, nvme_event, link); 3145 } 3146 } 3147 3148 void 3149 nvme_ctrlr_complete_queued_async_events(struct spdk_nvme_ctrlr *ctrlr) 3150 { 3151 struct spdk_nvme_ctrlr_aer_completion_list *nvme_event, *nvme_event_tmp; 3152 struct spdk_nvme_ctrlr_process *active_proc; 3153 3154 active_proc = nvme_ctrlr_get_current_process(ctrlr); 3155 3156 STAILQ_FOREACH_SAFE(nvme_event, &active_proc->async_events, link, nvme_event_tmp) { 3157 STAILQ_REMOVE(&active_proc->async_events, nvme_event, 3158 spdk_nvme_ctrlr_aer_completion_list, link); 3159 nvme_ctrlr_process_async_event(ctrlr, &nvme_event->cpl); 3160 spdk_free(nvme_event); 3161 3162 } 3163 } 3164 3165 static void 3166 nvme_ctrlr_async_event_cb(void *arg, const struct spdk_nvme_cpl *cpl) 3167 { 3168 struct nvme_async_event_request *aer = arg; 3169 struct spdk_nvme_ctrlr *ctrlr = aer->ctrlr; 3170 3171 if (cpl->status.sct == SPDK_NVME_SCT_GENERIC && 3172 cpl->status.sc == SPDK_NVME_SC_ABORTED_SQ_DELETION) { 3173 /* 3174 * This is simulated when controller is being shut down, to 3175 * effectively abort outstanding asynchronous event requests 3176 * and make sure all memory is freed. Do not repost the 3177 * request in this case. 3178 */ 3179 return; 3180 } 3181 3182 if (cpl->status.sct == SPDK_NVME_SCT_COMMAND_SPECIFIC && 3183 cpl->status.sc == SPDK_NVME_SC_ASYNC_EVENT_REQUEST_LIMIT_EXCEEDED) { 3184 /* 3185 * SPDK will only send as many AERs as the device says it supports, 3186 * so this status code indicates an out-of-spec device. Do not repost 3187 * the request in this case. 3188 */ 3189 NVME_CTRLR_ERRLOG(ctrlr, "Controller appears out-of-spec for asynchronous event request\n" 3190 "handling. Do not repost this AER.\n"); 3191 return; 3192 } 3193 3194 /* Add the events to the list */ 3195 nvme_ctrlr_queue_async_event(ctrlr, cpl); 3196 3197 /* If the ctrlr was removed or in the destruct state, we should not send aer again */ 3198 if (ctrlr->is_removed || ctrlr->is_destructed) { 3199 return; 3200 } 3201 3202 /* 3203 * Repost another asynchronous event request to replace the one 3204 * that just completed. 3205 */ 3206 if (nvme_ctrlr_construct_and_submit_aer(ctrlr, aer)) { 3207 /* 3208 * We can't do anything to recover from a failure here, 3209 * so just print a warning message and leave the AER unsubmitted. 3210 */ 3211 NVME_CTRLR_ERRLOG(ctrlr, "resubmitting AER failed!\n"); 3212 } 3213 } 3214 3215 static int 3216 nvme_ctrlr_construct_and_submit_aer(struct spdk_nvme_ctrlr *ctrlr, 3217 struct nvme_async_event_request *aer) 3218 { 3219 struct nvme_request *req; 3220 3221 aer->ctrlr = ctrlr; 3222 req = nvme_allocate_request_null(ctrlr->adminq, nvme_ctrlr_async_event_cb, aer); 3223 aer->req = req; 3224 if (req == NULL) { 3225 return -1; 3226 } 3227 3228 req->cmd.opc = SPDK_NVME_OPC_ASYNC_EVENT_REQUEST; 3229 return nvme_ctrlr_submit_admin_request(ctrlr, req); 3230 } 3231 3232 static void 3233 nvme_ctrlr_configure_aer_done(void *arg, const struct spdk_nvme_cpl *cpl) 3234 { 3235 struct nvme_async_event_request *aer; 3236 int rc; 3237 uint32_t i; 3238 struct spdk_nvme_ctrlr *ctrlr = (struct spdk_nvme_ctrlr *)arg; 3239 3240 if (spdk_nvme_cpl_is_error(cpl)) { 3241 NVME_CTRLR_NOTICELOG(ctrlr, "nvme_ctrlr_configure_aer failed!\n"); 3242 ctrlr->num_aers = 0; 3243 } else { 3244 /* aerl is a zero-based value, so we need to add 1 here. */ 3245 ctrlr->num_aers = spdk_min(NVME_MAX_ASYNC_EVENTS, (ctrlr->cdata.aerl + 1)); 3246 } 3247 3248 for (i = 0; i < ctrlr->num_aers; i++) { 3249 aer = &ctrlr->aer[i]; 3250 rc = nvme_ctrlr_construct_and_submit_aer(ctrlr, aer); 3251 if (rc) { 3252 NVME_CTRLR_ERRLOG(ctrlr, "nvme_ctrlr_construct_and_submit_aer failed!\n"); 3253 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 3254 return; 3255 } 3256 } 3257 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_KEEP_ALIVE_TIMEOUT, ctrlr->opts.admin_timeout_ms); 3258 } 3259 3260 static int 3261 nvme_ctrlr_configure_aer(struct spdk_nvme_ctrlr *ctrlr) 3262 { 3263 union spdk_nvme_feat_async_event_configuration config; 3264 int rc; 3265 3266 config.raw = 0; 3267 3268 if (spdk_nvme_ctrlr_is_discovery(ctrlr)) { 3269 config.bits.discovery_log_change_notice = 1; 3270 } else { 3271 config.bits.crit_warn.bits.available_spare = 1; 3272 config.bits.crit_warn.bits.temperature = 1; 3273 config.bits.crit_warn.bits.device_reliability = 1; 3274 config.bits.crit_warn.bits.read_only = 1; 3275 config.bits.crit_warn.bits.volatile_memory_backup = 1; 3276 3277 if (ctrlr->vs.raw >= SPDK_NVME_VERSION(1, 2, 0)) { 3278 if (ctrlr->cdata.oaes.ns_attribute_notices) { 3279 config.bits.ns_attr_notice = 1; 3280 } 3281 if (ctrlr->cdata.oaes.fw_activation_notices) { 3282 config.bits.fw_activation_notice = 1; 3283 } 3284 if (ctrlr->cdata.oaes.ana_change_notices) { 3285 config.bits.ana_change_notice = 1; 3286 } 3287 } 3288 if (ctrlr->vs.raw >= SPDK_NVME_VERSION(1, 3, 0) && ctrlr->cdata.lpa.telemetry) { 3289 config.bits.telemetry_log_notice = 1; 3290 } 3291 } 3292 3293 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_WAIT_FOR_CONFIGURE_AER, 3294 ctrlr->opts.admin_timeout_ms); 3295 3296 rc = nvme_ctrlr_cmd_set_async_event_config(ctrlr, config, 3297 nvme_ctrlr_configure_aer_done, 3298 ctrlr); 3299 if (rc != 0) { 3300 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 3301 return rc; 3302 } 3303 3304 return 0; 3305 } 3306 3307 struct spdk_nvme_ctrlr_process * 3308 nvme_ctrlr_get_process(struct spdk_nvme_ctrlr *ctrlr, pid_t pid) 3309 { 3310 struct spdk_nvme_ctrlr_process *active_proc; 3311 3312 TAILQ_FOREACH(active_proc, &ctrlr->active_procs, tailq) { 3313 if (active_proc->pid == pid) { 3314 return active_proc; 3315 } 3316 } 3317 3318 return NULL; 3319 } 3320 3321 struct spdk_nvme_ctrlr_process * 3322 nvme_ctrlr_get_current_process(struct spdk_nvme_ctrlr *ctrlr) 3323 { 3324 return nvme_ctrlr_get_process(ctrlr, getpid()); 3325 } 3326 3327 /** 3328 * This function will be called when a process is using the controller. 3329 * 1. For the primary process, it is called when constructing the controller. 3330 * 2. For the secondary process, it is called at probing the controller. 3331 * Note: will check whether the process is already added for the same process. 3332 */ 3333 int 3334 nvme_ctrlr_add_process(struct spdk_nvme_ctrlr *ctrlr, void *devhandle) 3335 { 3336 struct spdk_nvme_ctrlr_process *ctrlr_proc; 3337 pid_t pid = getpid(); 3338 3339 /* Check whether the process is already added or not */ 3340 if (nvme_ctrlr_get_process(ctrlr, pid)) { 3341 return 0; 3342 } 3343 3344 /* Initialize the per process properties for this ctrlr */ 3345 ctrlr_proc = spdk_zmalloc(sizeof(struct spdk_nvme_ctrlr_process), 3346 64, NULL, SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_SHARE); 3347 if (ctrlr_proc == NULL) { 3348 NVME_CTRLR_ERRLOG(ctrlr, "failed to allocate memory to track the process props\n"); 3349 3350 return -1; 3351 } 3352 3353 ctrlr_proc->is_primary = spdk_process_is_primary(); 3354 ctrlr_proc->pid = pid; 3355 STAILQ_INIT(&ctrlr_proc->active_reqs); 3356 ctrlr_proc->devhandle = devhandle; 3357 ctrlr_proc->ref = 0; 3358 TAILQ_INIT(&ctrlr_proc->allocated_io_qpairs); 3359 STAILQ_INIT(&ctrlr_proc->async_events); 3360 3361 TAILQ_INSERT_TAIL(&ctrlr->active_procs, ctrlr_proc, tailq); 3362 3363 return 0; 3364 } 3365 3366 /** 3367 * This function will be called when the process detaches the controller. 3368 * Note: the ctrlr_lock must be held when calling this function. 3369 */ 3370 static void 3371 nvme_ctrlr_remove_process(struct spdk_nvme_ctrlr *ctrlr, 3372 struct spdk_nvme_ctrlr_process *proc) 3373 { 3374 struct spdk_nvme_qpair *qpair, *tmp_qpair; 3375 3376 assert(STAILQ_EMPTY(&proc->active_reqs)); 3377 3378 TAILQ_FOREACH_SAFE(qpair, &proc->allocated_io_qpairs, per_process_tailq, tmp_qpair) { 3379 spdk_nvme_ctrlr_free_io_qpair(qpair); 3380 } 3381 3382 TAILQ_REMOVE(&ctrlr->active_procs, proc, tailq); 3383 3384 if (ctrlr->trid.trtype == SPDK_NVME_TRANSPORT_PCIE) { 3385 spdk_pci_device_detach(proc->devhandle); 3386 } 3387 3388 spdk_free(proc); 3389 } 3390 3391 /** 3392 * This function will be called when the process exited unexpectedly 3393 * in order to free any incomplete nvme request, allocated IO qpairs 3394 * and allocated memory. 3395 * Note: the ctrlr_lock must be held when calling this function. 3396 */ 3397 static void 3398 nvme_ctrlr_cleanup_process(struct spdk_nvme_ctrlr_process *proc) 3399 { 3400 struct nvme_request *req, *tmp_req; 3401 struct spdk_nvme_qpair *qpair, *tmp_qpair; 3402 struct spdk_nvme_ctrlr_aer_completion_list *event; 3403 3404 STAILQ_FOREACH_SAFE(req, &proc->active_reqs, stailq, tmp_req) { 3405 STAILQ_REMOVE(&proc->active_reqs, req, nvme_request, stailq); 3406 3407 assert(req->pid == proc->pid); 3408 nvme_cleanup_user_req(req); 3409 nvme_free_request(req); 3410 } 3411 3412 /* Remove async event from each process objects event list */ 3413 while (!STAILQ_EMPTY(&proc->async_events)) { 3414 event = STAILQ_FIRST(&proc->async_events); 3415 STAILQ_REMOVE_HEAD(&proc->async_events, link); 3416 spdk_free(event); 3417 } 3418 3419 TAILQ_FOREACH_SAFE(qpair, &proc->allocated_io_qpairs, per_process_tailq, tmp_qpair) { 3420 TAILQ_REMOVE(&proc->allocated_io_qpairs, qpair, per_process_tailq); 3421 3422 /* 3423 * The process may have been killed while some qpairs were in their 3424 * completion context. Clear that flag here to allow these IO 3425 * qpairs to be deleted. 3426 */ 3427 qpair->in_completion_context = 0; 3428 3429 qpair->no_deletion_notification_needed = 1; 3430 3431 spdk_nvme_ctrlr_free_io_qpair(qpair); 3432 } 3433 3434 spdk_free(proc); 3435 } 3436 3437 /** 3438 * This function will be called when destructing the controller. 3439 * 1. There is no more admin request on this controller. 3440 * 2. Clean up any left resource allocation when its associated process is gone. 3441 */ 3442 void 3443 nvme_ctrlr_free_processes(struct spdk_nvme_ctrlr *ctrlr) 3444 { 3445 struct spdk_nvme_ctrlr_process *active_proc, *tmp; 3446 3447 /* Free all the processes' properties and make sure no pending admin IOs */ 3448 TAILQ_FOREACH_SAFE(active_proc, &ctrlr->active_procs, tailq, tmp) { 3449 TAILQ_REMOVE(&ctrlr->active_procs, active_proc, tailq); 3450 3451 assert(STAILQ_EMPTY(&active_proc->active_reqs)); 3452 3453 spdk_free(active_proc); 3454 } 3455 } 3456 3457 /** 3458 * This function will be called when any other process attaches or 3459 * detaches the controller in order to cleanup those unexpectedly 3460 * terminated processes. 3461 * Note: the ctrlr_lock must be held when calling this function. 3462 */ 3463 static int 3464 nvme_ctrlr_remove_inactive_proc(struct spdk_nvme_ctrlr *ctrlr) 3465 { 3466 struct spdk_nvme_ctrlr_process *active_proc, *tmp; 3467 int active_proc_count = 0; 3468 3469 TAILQ_FOREACH_SAFE(active_proc, &ctrlr->active_procs, tailq, tmp) { 3470 if ((kill(active_proc->pid, 0) == -1) && (errno == ESRCH)) { 3471 NVME_CTRLR_ERRLOG(ctrlr, "process %d terminated unexpected\n", active_proc->pid); 3472 3473 TAILQ_REMOVE(&ctrlr->active_procs, active_proc, tailq); 3474 3475 nvme_ctrlr_cleanup_process(active_proc); 3476 } else { 3477 active_proc_count++; 3478 } 3479 } 3480 3481 return active_proc_count; 3482 } 3483 3484 void 3485 nvme_ctrlr_proc_get_ref(struct spdk_nvme_ctrlr *ctrlr) 3486 { 3487 struct spdk_nvme_ctrlr_process *active_proc; 3488 3489 nvme_robust_mutex_lock(&ctrlr->ctrlr_lock); 3490 3491 nvme_ctrlr_remove_inactive_proc(ctrlr); 3492 3493 active_proc = nvme_ctrlr_get_current_process(ctrlr); 3494 if (active_proc) { 3495 active_proc->ref++; 3496 } 3497 3498 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 3499 } 3500 3501 void 3502 nvme_ctrlr_proc_put_ref(struct spdk_nvme_ctrlr *ctrlr) 3503 { 3504 struct spdk_nvme_ctrlr_process *active_proc; 3505 int proc_count; 3506 3507 nvme_robust_mutex_lock(&ctrlr->ctrlr_lock); 3508 3509 proc_count = nvme_ctrlr_remove_inactive_proc(ctrlr); 3510 3511 active_proc = nvme_ctrlr_get_current_process(ctrlr); 3512 if (active_proc) { 3513 active_proc->ref--; 3514 assert(active_proc->ref >= 0); 3515 3516 /* 3517 * The last active process will be removed at the end of 3518 * the destruction of the controller. 3519 */ 3520 if (active_proc->ref == 0 && proc_count != 1) { 3521 nvme_ctrlr_remove_process(ctrlr, active_proc); 3522 } 3523 } 3524 3525 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 3526 } 3527 3528 int 3529 nvme_ctrlr_get_ref_count(struct spdk_nvme_ctrlr *ctrlr) 3530 { 3531 struct spdk_nvme_ctrlr_process *active_proc; 3532 int ref = 0; 3533 3534 nvme_robust_mutex_lock(&ctrlr->ctrlr_lock); 3535 3536 nvme_ctrlr_remove_inactive_proc(ctrlr); 3537 3538 TAILQ_FOREACH(active_proc, &ctrlr->active_procs, tailq) { 3539 ref += active_proc->ref; 3540 } 3541 3542 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 3543 3544 return ref; 3545 } 3546 3547 /** 3548 * Get the PCI device handle which is only visible to its associated process. 3549 */ 3550 struct spdk_pci_device * 3551 nvme_ctrlr_proc_get_devhandle(struct spdk_nvme_ctrlr *ctrlr) 3552 { 3553 struct spdk_nvme_ctrlr_process *active_proc; 3554 struct spdk_pci_device *devhandle = NULL; 3555 3556 nvme_robust_mutex_lock(&ctrlr->ctrlr_lock); 3557 3558 active_proc = nvme_ctrlr_get_current_process(ctrlr); 3559 if (active_proc) { 3560 devhandle = active_proc->devhandle; 3561 } 3562 3563 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 3564 3565 return devhandle; 3566 } 3567 3568 static void 3569 nvme_ctrlr_process_init_vs_done(void *ctx, uint64_t value, const struct spdk_nvme_cpl *cpl) 3570 { 3571 struct spdk_nvme_ctrlr *ctrlr = ctx; 3572 3573 if (spdk_nvme_cpl_is_error(cpl)) { 3574 NVME_CTRLR_ERRLOG(ctrlr, "Failed to read the VS register\n"); 3575 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 3576 return; 3577 } 3578 3579 assert(value <= UINT32_MAX); 3580 ctrlr->vs.raw = (uint32_t)value; 3581 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_READ_CAP, NVME_TIMEOUT_INFINITE); 3582 } 3583 3584 static void 3585 nvme_ctrlr_process_init_cap_done(void *ctx, uint64_t value, const struct spdk_nvme_cpl *cpl) 3586 { 3587 struct spdk_nvme_ctrlr *ctrlr = ctx; 3588 3589 if (spdk_nvme_cpl_is_error(cpl)) { 3590 NVME_CTRLR_ERRLOG(ctrlr, "Failed to read the CAP register\n"); 3591 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 3592 return; 3593 } 3594 3595 ctrlr->cap.raw = value; 3596 nvme_ctrlr_init_cap(ctrlr); 3597 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_CHECK_EN, NVME_TIMEOUT_INFINITE); 3598 } 3599 3600 static void 3601 nvme_ctrlr_process_init_check_en(void *ctx, uint64_t value, const struct spdk_nvme_cpl *cpl) 3602 { 3603 struct spdk_nvme_ctrlr *ctrlr = ctx; 3604 enum nvme_ctrlr_state state; 3605 3606 if (spdk_nvme_cpl_is_error(cpl)) { 3607 NVME_CTRLR_ERRLOG(ctrlr, "Failed to read the CC register\n"); 3608 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 3609 return; 3610 } 3611 3612 assert(value <= UINT32_MAX); 3613 ctrlr->process_init_cc.raw = (uint32_t)value; 3614 3615 if (ctrlr->process_init_cc.bits.en) { 3616 NVME_CTRLR_DEBUGLOG(ctrlr, "CC.EN = 1\n"); 3617 state = NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_1; 3618 } else { 3619 state = NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0; 3620 } 3621 3622 nvme_ctrlr_set_state(ctrlr, state, nvme_ctrlr_get_ready_timeout(ctrlr)); 3623 } 3624 3625 static void 3626 nvme_ctrlr_process_init_set_en_0(void *ctx, uint64_t value, const struct spdk_nvme_cpl *cpl) 3627 { 3628 struct spdk_nvme_ctrlr *ctrlr = ctx; 3629 3630 if (spdk_nvme_cpl_is_error(cpl)) { 3631 NVME_CTRLR_ERRLOG(ctrlr, "Failed to write the CC register\n"); 3632 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 3633 return; 3634 } 3635 3636 /* 3637 * Wait 2.5 seconds before accessing PCI registers. 3638 * Not using sleep() to avoid blocking other controller's initialization. 3639 */ 3640 if (ctrlr->quirks & NVME_QUIRK_DELAY_BEFORE_CHK_RDY) { 3641 NVME_CTRLR_DEBUGLOG(ctrlr, "Applying quirk: delay 2.5 seconds before reading registers\n"); 3642 ctrlr->sleep_timeout_tsc = spdk_get_ticks() + (2500 * spdk_get_ticks_hz() / 1000); 3643 } 3644 3645 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0, 3646 nvme_ctrlr_get_ready_timeout(ctrlr)); 3647 } 3648 3649 static void 3650 nvme_ctrlr_process_init_set_en_0_read_cc(void *ctx, uint64_t value, const struct spdk_nvme_cpl *cpl) 3651 { 3652 struct spdk_nvme_ctrlr *ctrlr = ctx; 3653 union spdk_nvme_cc_register cc; 3654 int rc; 3655 3656 if (spdk_nvme_cpl_is_error(cpl)) { 3657 NVME_CTRLR_ERRLOG(ctrlr, "Failed to read the CC register\n"); 3658 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 3659 return; 3660 } 3661 3662 assert(value <= UINT32_MAX); 3663 cc.raw = (uint32_t)value; 3664 cc.bits.en = 0; 3665 ctrlr->process_init_cc.raw = cc.raw; 3666 3667 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_EN_0_WAIT_FOR_CC, 3668 nvme_ctrlr_get_ready_timeout(ctrlr)); 3669 3670 rc = nvme_ctrlr_set_cc_async(ctrlr, cc.raw, nvme_ctrlr_process_init_set_en_0, ctrlr); 3671 if (rc != 0) { 3672 NVME_CTRLR_ERRLOG(ctrlr, "set_cc() failed\n"); 3673 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 3674 } 3675 } 3676 3677 static void 3678 nvme_ctrlr_process_init_wait_for_ready_1(void *ctx, uint64_t value, const struct spdk_nvme_cpl *cpl) 3679 { 3680 struct spdk_nvme_ctrlr *ctrlr = ctx; 3681 union spdk_nvme_csts_register csts; 3682 3683 if (spdk_nvme_cpl_is_error(cpl)) { 3684 /* While a device is resetting, it may be unable to service MMIO reads 3685 * temporarily. Allow for this case. 3686 */ 3687 if (!ctrlr->is_failed && ctrlr->state_timeout_tsc != NVME_TIMEOUT_INFINITE) { 3688 NVME_CTRLR_DEBUGLOG(ctrlr, "Failed to read the CSTS register\n"); 3689 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_1, 3690 NVME_TIMEOUT_KEEP_EXISTING); 3691 } else { 3692 NVME_CTRLR_ERRLOG(ctrlr, "Failed to read the CSTS register\n"); 3693 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 3694 } 3695 3696 return; 3697 } 3698 3699 assert(value <= UINT32_MAX); 3700 csts.raw = (uint32_t)value; 3701 if (csts.bits.rdy == 1 || csts.bits.cfs == 1) { 3702 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_EN_0, 3703 nvme_ctrlr_get_ready_timeout(ctrlr)); 3704 } else { 3705 NVME_CTRLR_DEBUGLOG(ctrlr, "CC.EN = 1 && CSTS.RDY = 0 - waiting for reset to complete\n"); 3706 nvme_ctrlr_set_state_quiet(ctrlr, NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_1, 3707 NVME_TIMEOUT_KEEP_EXISTING); 3708 } 3709 } 3710 3711 static void 3712 nvme_ctrlr_process_init_wait_for_ready_0(void *ctx, uint64_t value, const struct spdk_nvme_cpl *cpl) 3713 { 3714 struct spdk_nvme_ctrlr *ctrlr = ctx; 3715 union spdk_nvme_csts_register csts; 3716 3717 if (spdk_nvme_cpl_is_error(cpl)) { 3718 /* While a device is resetting, it may be unable to service MMIO reads 3719 * temporarily. Allow for this case. 3720 */ 3721 if (!ctrlr->is_failed && ctrlr->state_timeout_tsc != NVME_TIMEOUT_INFINITE) { 3722 NVME_CTRLR_DEBUGLOG(ctrlr, "Failed to read the CSTS register\n"); 3723 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0, 3724 NVME_TIMEOUT_KEEP_EXISTING); 3725 } else { 3726 NVME_CTRLR_ERRLOG(ctrlr, "Failed to read the CSTS register\n"); 3727 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 3728 } 3729 3730 return; 3731 } 3732 3733 assert(value <= UINT32_MAX); 3734 csts.raw = (uint32_t)value; 3735 if (csts.bits.rdy == 0) { 3736 NVME_CTRLR_DEBUGLOG(ctrlr, "CC.EN = 0 && CSTS.RDY = 0\n"); 3737 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_DISABLED, 3738 nvme_ctrlr_get_ready_timeout(ctrlr)); 3739 } else { 3740 nvme_ctrlr_set_state_quiet(ctrlr, NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0, 3741 NVME_TIMEOUT_KEEP_EXISTING); 3742 } 3743 } 3744 3745 static void 3746 nvme_ctrlr_process_init_enable_wait_for_ready_1(void *ctx, uint64_t value, 3747 const struct spdk_nvme_cpl *cpl) 3748 { 3749 struct spdk_nvme_ctrlr *ctrlr = ctx; 3750 union spdk_nvme_csts_register csts; 3751 3752 if (spdk_nvme_cpl_is_error(cpl)) { 3753 /* While a device is resetting, it may be unable to service MMIO reads 3754 * temporarily. Allow for this case. 3755 */ 3756 if (!ctrlr->is_failed && ctrlr->state_timeout_tsc != NVME_TIMEOUT_INFINITE) { 3757 NVME_CTRLR_DEBUGLOG(ctrlr, "Failed to read the CSTS register\n"); 3758 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ENABLE_WAIT_FOR_READY_1, 3759 NVME_TIMEOUT_KEEP_EXISTING); 3760 } else { 3761 NVME_CTRLR_ERRLOG(ctrlr, "Failed to read the CSTS register\n"); 3762 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 3763 } 3764 3765 return; 3766 } 3767 3768 assert(value <= UINT32_MAX); 3769 csts.raw = value; 3770 if (csts.bits.rdy == 1) { 3771 NVME_CTRLR_DEBUGLOG(ctrlr, "CC.EN = 1 && CSTS.RDY = 1 - controller is ready\n"); 3772 /* 3773 * The controller has been enabled. 3774 * Perform the rest of initialization serially. 3775 */ 3776 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_RESET_ADMIN_QUEUE, 3777 ctrlr->opts.admin_timeout_ms); 3778 } else { 3779 nvme_ctrlr_set_state_quiet(ctrlr, NVME_CTRLR_STATE_ENABLE_WAIT_FOR_READY_1, 3780 NVME_TIMEOUT_KEEP_EXISTING); 3781 } 3782 } 3783 3784 /** 3785 * This function will be called repeatedly during initialization until the controller is ready. 3786 */ 3787 int 3788 nvme_ctrlr_process_init(struct spdk_nvme_ctrlr *ctrlr) 3789 { 3790 uint32_t ready_timeout_in_ms; 3791 uint64_t ticks; 3792 int rc = 0; 3793 3794 ticks = spdk_get_ticks(); 3795 3796 /* 3797 * May need to avoid accessing any register on the target controller 3798 * for a while. Return early without touching the FSM. 3799 * Check sleep_timeout_tsc > 0 for unit test. 3800 */ 3801 if ((ctrlr->sleep_timeout_tsc > 0) && 3802 (ticks <= ctrlr->sleep_timeout_tsc)) { 3803 return 0; 3804 } 3805 ctrlr->sleep_timeout_tsc = 0; 3806 3807 ready_timeout_in_ms = nvme_ctrlr_get_ready_timeout(ctrlr); 3808 3809 /* 3810 * Check if the current initialization step is done or has timed out. 3811 */ 3812 switch (ctrlr->state) { 3813 case NVME_CTRLR_STATE_INIT_DELAY: 3814 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_INIT, ready_timeout_in_ms); 3815 if (ctrlr->quirks & NVME_QUIRK_DELAY_BEFORE_INIT) { 3816 /* 3817 * Controller may need some delay before it's enabled. 3818 * 3819 * This is a workaround for an issue where the PCIe-attached NVMe controller 3820 * is not ready after VFIO reset. We delay the initialization rather than the 3821 * enabling itself, because this is required only for the very first enabling 3822 * - directly after a VFIO reset. 3823 */ 3824 NVME_CTRLR_DEBUGLOG(ctrlr, "Adding 2 second delay before initializing the controller\n"); 3825 ctrlr->sleep_timeout_tsc = ticks + (2000 * spdk_get_ticks_hz() / 1000); 3826 } 3827 break; 3828 3829 case NVME_CTRLR_STATE_DISCONNECTED: 3830 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_INIT, NVME_TIMEOUT_INFINITE); 3831 break; 3832 3833 case NVME_CTRLR_STATE_CONNECT_ADMINQ: /* synonymous with NVME_CTRLR_STATE_INIT and NVME_CTRLR_STATE_DISCONNECTED */ 3834 rc = nvme_transport_ctrlr_connect_qpair(ctrlr, ctrlr->adminq); 3835 if (rc == 0) { 3836 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_WAIT_FOR_CONNECT_ADMINQ, 3837 NVME_TIMEOUT_INFINITE); 3838 } else { 3839 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 3840 } 3841 break; 3842 3843 case NVME_CTRLR_STATE_WAIT_FOR_CONNECT_ADMINQ: 3844 spdk_nvme_qpair_process_completions(ctrlr->adminq, 0); 3845 3846 switch (nvme_qpair_get_state(ctrlr->adminq)) { 3847 case NVME_QPAIR_CONNECTING: 3848 break; 3849 case NVME_QPAIR_CONNECTED: 3850 nvme_qpair_set_state(ctrlr->adminq, NVME_QPAIR_ENABLED); 3851 /* Fall through */ 3852 case NVME_QPAIR_ENABLED: 3853 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_READ_VS, 3854 NVME_TIMEOUT_INFINITE); 3855 /* Abort any queued requests that were sent while the adminq was connecting 3856 * to avoid stalling the init process during a reset, as requests don't get 3857 * resubmitted while the controller is resetting and subsequent commands 3858 * would get queued too. 3859 */ 3860 nvme_qpair_abort_queued_reqs(ctrlr->adminq); 3861 break; 3862 case NVME_QPAIR_DISCONNECTING: 3863 assert(ctrlr->adminq->async == true); 3864 break; 3865 case NVME_QPAIR_DISCONNECTED: 3866 /* fallthrough */ 3867 default: 3868 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 3869 break; 3870 } 3871 3872 break; 3873 3874 case NVME_CTRLR_STATE_READ_VS: 3875 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_READ_VS_WAIT_FOR_VS, NVME_TIMEOUT_INFINITE); 3876 rc = nvme_ctrlr_get_vs_async(ctrlr, nvme_ctrlr_process_init_vs_done, ctrlr); 3877 break; 3878 3879 case NVME_CTRLR_STATE_READ_CAP: 3880 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_READ_CAP_WAIT_FOR_CAP, NVME_TIMEOUT_INFINITE); 3881 rc = nvme_ctrlr_get_cap_async(ctrlr, nvme_ctrlr_process_init_cap_done, ctrlr); 3882 break; 3883 3884 case NVME_CTRLR_STATE_CHECK_EN: 3885 /* Begin the hardware initialization by making sure the controller is disabled. */ 3886 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_CHECK_EN_WAIT_FOR_CC, ready_timeout_in_ms); 3887 rc = nvme_ctrlr_get_cc_async(ctrlr, nvme_ctrlr_process_init_check_en, ctrlr); 3888 break; 3889 3890 case NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_1: 3891 /* 3892 * Controller is currently enabled. We need to disable it to cause a reset. 3893 * 3894 * If CC.EN = 1 && CSTS.RDY = 0, the controller is in the process of becoming ready. 3895 * Wait for the ready bit to be 1 before disabling the controller. 3896 */ 3897 nvme_ctrlr_set_state_quiet(ctrlr, NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_1_WAIT_FOR_CSTS, 3898 NVME_TIMEOUT_KEEP_EXISTING); 3899 rc = nvme_ctrlr_get_csts_async(ctrlr, nvme_ctrlr_process_init_wait_for_ready_1, ctrlr); 3900 break; 3901 3902 case NVME_CTRLR_STATE_SET_EN_0: 3903 NVME_CTRLR_DEBUGLOG(ctrlr, "Setting CC.EN = 0\n"); 3904 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_EN_0_WAIT_FOR_CC, ready_timeout_in_ms); 3905 rc = nvme_ctrlr_get_cc_async(ctrlr, nvme_ctrlr_process_init_set_en_0_read_cc, ctrlr); 3906 break; 3907 3908 case NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0: 3909 nvme_ctrlr_set_state_quiet(ctrlr, NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0_WAIT_FOR_CSTS, 3910 NVME_TIMEOUT_KEEP_EXISTING); 3911 rc = nvme_ctrlr_get_csts_async(ctrlr, nvme_ctrlr_process_init_wait_for_ready_0, ctrlr); 3912 break; 3913 3914 case NVME_CTRLR_STATE_DISABLED: 3915 if (ctrlr->is_disconnecting) { 3916 NVME_CTRLR_DEBUGLOG(ctrlr, "Ctrlr was disabled.\n"); 3917 } else { 3918 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ENABLE, ready_timeout_in_ms); 3919 3920 /* 3921 * Delay 100us before setting CC.EN = 1. Some NVMe SSDs miss CC.EN getting 3922 * set to 1 if it is too soon after CSTS.RDY is reported as 0. 3923 */ 3924 spdk_delay_us(100); 3925 } 3926 break; 3927 3928 case NVME_CTRLR_STATE_ENABLE: 3929 NVME_CTRLR_DEBUGLOG(ctrlr, "Setting CC.EN = 1\n"); 3930 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ENABLE_WAIT_FOR_CC, ready_timeout_in_ms); 3931 rc = nvme_ctrlr_enable(ctrlr); 3932 if (rc) { 3933 NVME_CTRLR_ERRLOG(ctrlr, "Ctrlr enable failed with error: %d", rc); 3934 } 3935 return rc; 3936 3937 case NVME_CTRLR_STATE_ENABLE_WAIT_FOR_READY_1: 3938 nvme_ctrlr_set_state_quiet(ctrlr, NVME_CTRLR_STATE_ENABLE_WAIT_FOR_READY_1_WAIT_FOR_CSTS, 3939 NVME_TIMEOUT_KEEP_EXISTING); 3940 rc = nvme_ctrlr_get_csts_async(ctrlr, nvme_ctrlr_process_init_enable_wait_for_ready_1, 3941 ctrlr); 3942 break; 3943 3944 case NVME_CTRLR_STATE_RESET_ADMIN_QUEUE: 3945 nvme_transport_qpair_reset(ctrlr->adminq); 3946 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_IDENTIFY, NVME_TIMEOUT_INFINITE); 3947 break; 3948 3949 case NVME_CTRLR_STATE_IDENTIFY: 3950 rc = nvme_ctrlr_identify(ctrlr); 3951 break; 3952 3953 case NVME_CTRLR_STATE_CONFIGURE_AER: 3954 rc = nvme_ctrlr_configure_aer(ctrlr); 3955 break; 3956 3957 case NVME_CTRLR_STATE_SET_KEEP_ALIVE_TIMEOUT: 3958 rc = nvme_ctrlr_set_keep_alive_timeout(ctrlr); 3959 break; 3960 3961 case NVME_CTRLR_STATE_IDENTIFY_IOCS_SPECIFIC: 3962 rc = nvme_ctrlr_identify_iocs_specific(ctrlr); 3963 break; 3964 3965 case NVME_CTRLR_STATE_GET_ZNS_CMD_EFFECTS_LOG: 3966 rc = nvme_ctrlr_get_zns_cmd_and_effects_log(ctrlr); 3967 break; 3968 3969 case NVME_CTRLR_STATE_SET_NUM_QUEUES: 3970 nvme_ctrlr_update_nvmf_ioccsz(ctrlr); 3971 rc = nvme_ctrlr_set_num_queues(ctrlr); 3972 break; 3973 3974 case NVME_CTRLR_STATE_IDENTIFY_ACTIVE_NS: 3975 _nvme_ctrlr_identify_active_ns(ctrlr); 3976 break; 3977 3978 case NVME_CTRLR_STATE_IDENTIFY_NS: 3979 rc = nvme_ctrlr_identify_namespaces(ctrlr); 3980 break; 3981 3982 case NVME_CTRLR_STATE_IDENTIFY_ID_DESCS: 3983 rc = nvme_ctrlr_identify_id_desc_namespaces(ctrlr); 3984 break; 3985 3986 case NVME_CTRLR_STATE_IDENTIFY_NS_IOCS_SPECIFIC: 3987 rc = nvme_ctrlr_identify_namespaces_iocs_specific(ctrlr); 3988 break; 3989 3990 case NVME_CTRLR_STATE_SET_SUPPORTED_LOG_PAGES: 3991 rc = nvme_ctrlr_set_supported_log_pages(ctrlr); 3992 break; 3993 3994 case NVME_CTRLR_STATE_SET_SUPPORTED_INTEL_LOG_PAGES: 3995 rc = nvme_ctrlr_set_intel_support_log_pages(ctrlr); 3996 break; 3997 3998 case NVME_CTRLR_STATE_SET_SUPPORTED_FEATURES: 3999 nvme_ctrlr_set_supported_features(ctrlr); 4000 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_DB_BUF_CFG, 4001 ctrlr->opts.admin_timeout_ms); 4002 break; 4003 4004 case NVME_CTRLR_STATE_SET_DB_BUF_CFG: 4005 rc = nvme_ctrlr_set_doorbell_buffer_config(ctrlr); 4006 break; 4007 4008 case NVME_CTRLR_STATE_SET_HOST_ID: 4009 rc = nvme_ctrlr_set_host_id(ctrlr); 4010 break; 4011 4012 case NVME_CTRLR_STATE_TRANSPORT_READY: 4013 rc = nvme_transport_ctrlr_ready(ctrlr); 4014 if (rc) { 4015 NVME_CTRLR_ERRLOG(ctrlr, "Transport controller ready step failed: rc %d\n", rc); 4016 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 4017 } else { 4018 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_READY, NVME_TIMEOUT_INFINITE); 4019 } 4020 break; 4021 4022 case NVME_CTRLR_STATE_READY: 4023 NVME_CTRLR_DEBUGLOG(ctrlr, "Ctrlr already in ready state\n"); 4024 return 0; 4025 4026 case NVME_CTRLR_STATE_ERROR: 4027 NVME_CTRLR_ERRLOG(ctrlr, "Ctrlr is in error state\n"); 4028 return -1; 4029 4030 case NVME_CTRLR_STATE_READ_VS_WAIT_FOR_VS: 4031 case NVME_CTRLR_STATE_READ_CAP_WAIT_FOR_CAP: 4032 case NVME_CTRLR_STATE_CHECK_EN_WAIT_FOR_CC: 4033 case NVME_CTRLR_STATE_SET_EN_0_WAIT_FOR_CC: 4034 case NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_1_WAIT_FOR_CSTS: 4035 case NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0_WAIT_FOR_CSTS: 4036 case NVME_CTRLR_STATE_ENABLE_WAIT_FOR_CC: 4037 case NVME_CTRLR_STATE_ENABLE_WAIT_FOR_READY_1_WAIT_FOR_CSTS: 4038 case NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY: 4039 case NVME_CTRLR_STATE_WAIT_FOR_CONFIGURE_AER: 4040 case NVME_CTRLR_STATE_WAIT_FOR_KEEP_ALIVE_TIMEOUT: 4041 case NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_IOCS_SPECIFIC: 4042 case NVME_CTRLR_STATE_WAIT_FOR_GET_ZNS_CMD_EFFECTS_LOG: 4043 case NVME_CTRLR_STATE_WAIT_FOR_SET_NUM_QUEUES: 4044 case NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_ACTIVE_NS: 4045 case NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_NS: 4046 case NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_ID_DESCS: 4047 case NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_NS_IOCS_SPECIFIC: 4048 case NVME_CTRLR_STATE_WAIT_FOR_SUPPORTED_INTEL_LOG_PAGES: 4049 case NVME_CTRLR_STATE_WAIT_FOR_DB_BUF_CFG: 4050 case NVME_CTRLR_STATE_WAIT_FOR_HOST_ID: 4051 /* 4052 * nvme_ctrlr_process_init() may be called from the completion context 4053 * for the admin qpair. Avoid recursive calls for this case. 4054 */ 4055 if (!ctrlr->adminq->in_completion_context) { 4056 spdk_nvme_qpair_process_completions(ctrlr->adminq, 0); 4057 } 4058 break; 4059 4060 default: 4061 assert(0); 4062 return -1; 4063 } 4064 4065 if (rc) { 4066 NVME_CTRLR_ERRLOG(ctrlr, "Ctrlr operation failed with error: %d, ctrlr state: %d (%s)\n", 4067 rc, ctrlr->state, nvme_ctrlr_state_string(ctrlr->state)); 4068 } 4069 4070 /* Note: we use the ticks captured when we entered this function. 4071 * This covers environments where the SPDK process gets swapped out after 4072 * we tried to advance the state but before we check the timeout here. 4073 * It is not normal for this to happen, but harmless to handle it in this 4074 * way. 4075 */ 4076 if (ctrlr->state_timeout_tsc != NVME_TIMEOUT_INFINITE && 4077 ticks > ctrlr->state_timeout_tsc) { 4078 NVME_CTRLR_ERRLOG(ctrlr, "Initialization timed out in state %d (%s)\n", 4079 ctrlr->state, nvme_ctrlr_state_string(ctrlr->state)); 4080 return -1; 4081 } 4082 4083 return rc; 4084 } 4085 4086 int 4087 nvme_robust_mutex_init_recursive_shared(pthread_mutex_t *mtx) 4088 { 4089 pthread_mutexattr_t attr; 4090 int rc = 0; 4091 4092 if (pthread_mutexattr_init(&attr)) { 4093 return -1; 4094 } 4095 if (pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE) || 4096 #ifndef __FreeBSD__ 4097 pthread_mutexattr_setrobust(&attr, PTHREAD_MUTEX_ROBUST) || 4098 pthread_mutexattr_setpshared(&attr, PTHREAD_PROCESS_SHARED) || 4099 #endif 4100 pthread_mutex_init(mtx, &attr)) { 4101 rc = -1; 4102 } 4103 pthread_mutexattr_destroy(&attr); 4104 return rc; 4105 } 4106 4107 int 4108 nvme_ctrlr_construct(struct spdk_nvme_ctrlr *ctrlr) 4109 { 4110 int rc; 4111 4112 if (ctrlr->trid.trtype == SPDK_NVME_TRANSPORT_PCIE) { 4113 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_INIT_DELAY, NVME_TIMEOUT_INFINITE); 4114 } else { 4115 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_INIT, NVME_TIMEOUT_INFINITE); 4116 } 4117 4118 if (ctrlr->opts.admin_queue_size > SPDK_NVME_ADMIN_QUEUE_MAX_ENTRIES) { 4119 NVME_CTRLR_ERRLOG(ctrlr, "admin_queue_size %u exceeds max defined by NVMe spec, use max value\n", 4120 ctrlr->opts.admin_queue_size); 4121 ctrlr->opts.admin_queue_size = SPDK_NVME_ADMIN_QUEUE_MAX_ENTRIES; 4122 } 4123 4124 if (ctrlr->quirks & NVME_QUIRK_MINIMUM_ADMIN_QUEUE_SIZE && 4125 (ctrlr->opts.admin_queue_size % SPDK_NVME_ADMIN_QUEUE_QUIRK_ENTRIES_MULTIPLE) != 0) { 4126 NVME_CTRLR_ERRLOG(ctrlr, 4127 "admin_queue_size %u is invalid for this NVMe device, adjust to next multiple\n", 4128 ctrlr->opts.admin_queue_size); 4129 ctrlr->opts.admin_queue_size = SPDK_ALIGN_CEIL(ctrlr->opts.admin_queue_size, 4130 SPDK_NVME_ADMIN_QUEUE_QUIRK_ENTRIES_MULTIPLE); 4131 } 4132 4133 if (ctrlr->opts.admin_queue_size < SPDK_NVME_ADMIN_QUEUE_MIN_ENTRIES) { 4134 NVME_CTRLR_ERRLOG(ctrlr, 4135 "admin_queue_size %u is less than minimum defined by NVMe spec, use min value\n", 4136 ctrlr->opts.admin_queue_size); 4137 ctrlr->opts.admin_queue_size = SPDK_NVME_ADMIN_QUEUE_MIN_ENTRIES; 4138 } 4139 4140 ctrlr->flags = 0; 4141 ctrlr->free_io_qids = NULL; 4142 ctrlr->is_resetting = false; 4143 ctrlr->is_failed = false; 4144 ctrlr->is_destructed = false; 4145 4146 TAILQ_INIT(&ctrlr->active_io_qpairs); 4147 STAILQ_INIT(&ctrlr->queued_aborts); 4148 ctrlr->outstanding_aborts = 0; 4149 4150 ctrlr->ana_log_page = NULL; 4151 ctrlr->ana_log_page_size = 0; 4152 4153 rc = nvme_robust_mutex_init_recursive_shared(&ctrlr->ctrlr_lock); 4154 if (rc != 0) { 4155 return rc; 4156 } 4157 4158 TAILQ_INIT(&ctrlr->active_procs); 4159 STAILQ_INIT(&ctrlr->register_operations); 4160 4161 RB_INIT(&ctrlr->ns); 4162 4163 return rc; 4164 } 4165 4166 static void 4167 nvme_ctrlr_init_cap(struct spdk_nvme_ctrlr *ctrlr) 4168 { 4169 if (ctrlr->cap.bits.ams & SPDK_NVME_CAP_AMS_WRR) { 4170 ctrlr->flags |= SPDK_NVME_CTRLR_WRR_SUPPORTED; 4171 } 4172 4173 ctrlr->min_page_size = 1u << (12 + ctrlr->cap.bits.mpsmin); 4174 4175 /* For now, always select page_size == min_page_size. */ 4176 ctrlr->page_size = ctrlr->min_page_size; 4177 4178 ctrlr->opts.io_queue_size = spdk_max(ctrlr->opts.io_queue_size, SPDK_NVME_IO_QUEUE_MIN_ENTRIES); 4179 ctrlr->opts.io_queue_size = spdk_min(ctrlr->opts.io_queue_size, MAX_IO_QUEUE_ENTRIES); 4180 if (ctrlr->quirks & NVME_QUIRK_MINIMUM_IO_QUEUE_SIZE && 4181 ctrlr->opts.io_queue_size == DEFAULT_IO_QUEUE_SIZE) { 4182 /* If the user specifically set an IO queue size different than the 4183 * default, use that value. Otherwise overwrite with the quirked value. 4184 * This allows this quirk to be overridden when necessary. 4185 * However, cap.mqes still needs to be respected. 4186 */ 4187 ctrlr->opts.io_queue_size = DEFAULT_IO_QUEUE_SIZE_FOR_QUIRK; 4188 } 4189 ctrlr->opts.io_queue_size = spdk_min(ctrlr->opts.io_queue_size, ctrlr->cap.bits.mqes + 1u); 4190 4191 ctrlr->opts.io_queue_requests = spdk_max(ctrlr->opts.io_queue_requests, ctrlr->opts.io_queue_size); 4192 } 4193 4194 void 4195 nvme_ctrlr_destruct_finish(struct spdk_nvme_ctrlr *ctrlr) 4196 { 4197 pthread_mutex_destroy(&ctrlr->ctrlr_lock); 4198 } 4199 4200 void 4201 nvme_ctrlr_destruct_async(struct spdk_nvme_ctrlr *ctrlr, 4202 struct nvme_ctrlr_detach_ctx *ctx) 4203 { 4204 struct spdk_nvme_qpair *qpair, *tmp; 4205 4206 NVME_CTRLR_DEBUGLOG(ctrlr, "Prepare to destruct SSD\n"); 4207 4208 ctrlr->prepare_for_reset = false; 4209 ctrlr->is_destructed = true; 4210 4211 spdk_nvme_qpair_process_completions(ctrlr->adminq, 0); 4212 4213 nvme_ctrlr_abort_queued_aborts(ctrlr); 4214 nvme_transport_admin_qpair_abort_aers(ctrlr->adminq); 4215 4216 TAILQ_FOREACH_SAFE(qpair, &ctrlr->active_io_qpairs, tailq, tmp) { 4217 spdk_nvme_ctrlr_free_io_qpair(qpair); 4218 } 4219 4220 nvme_ctrlr_free_doorbell_buffer(ctrlr); 4221 nvme_ctrlr_free_iocs_specific_data(ctrlr); 4222 4223 nvme_ctrlr_shutdown_async(ctrlr, ctx); 4224 } 4225 4226 int 4227 nvme_ctrlr_destruct_poll_async(struct spdk_nvme_ctrlr *ctrlr, 4228 struct nvme_ctrlr_detach_ctx *ctx) 4229 { 4230 struct spdk_nvme_ns *ns, *tmp_ns; 4231 int rc = 0; 4232 4233 if (!ctx->shutdown_complete) { 4234 rc = nvme_ctrlr_shutdown_poll_async(ctrlr, ctx); 4235 if (rc == -EAGAIN) { 4236 return -EAGAIN; 4237 } 4238 /* Destruct ctrlr forcefully for any other error. */ 4239 } 4240 4241 if (ctx->cb_fn) { 4242 ctx->cb_fn(ctrlr); 4243 } 4244 4245 nvme_transport_ctrlr_disconnect_qpair(ctrlr, ctrlr->adminq); 4246 4247 RB_FOREACH_SAFE(ns, nvme_ns_tree, &ctrlr->ns, tmp_ns) { 4248 nvme_ctrlr_destruct_namespace(ctrlr, ns->id); 4249 RB_REMOVE(nvme_ns_tree, &ctrlr->ns, ns); 4250 spdk_free(ns); 4251 } 4252 4253 ctrlr->active_ns_count = 0; 4254 4255 spdk_bit_array_free(&ctrlr->free_io_qids); 4256 4257 free(ctrlr->ana_log_page); 4258 free(ctrlr->copied_ana_desc); 4259 ctrlr->ana_log_page = NULL; 4260 ctrlr->copied_ana_desc = NULL; 4261 ctrlr->ana_log_page_size = 0; 4262 4263 nvme_transport_ctrlr_destruct(ctrlr); 4264 4265 return rc; 4266 } 4267 4268 void 4269 nvme_ctrlr_destruct(struct spdk_nvme_ctrlr *ctrlr) 4270 { 4271 struct nvme_ctrlr_detach_ctx ctx = { .ctrlr = ctrlr }; 4272 int rc; 4273 4274 nvme_ctrlr_destruct_async(ctrlr, &ctx); 4275 4276 while (1) { 4277 rc = nvme_ctrlr_destruct_poll_async(ctrlr, &ctx); 4278 if (rc != -EAGAIN) { 4279 break; 4280 } 4281 nvme_delay(1000); 4282 } 4283 } 4284 4285 int 4286 nvme_ctrlr_submit_admin_request(struct spdk_nvme_ctrlr *ctrlr, 4287 struct nvme_request *req) 4288 { 4289 return nvme_qpair_submit_request(ctrlr->adminq, req); 4290 } 4291 4292 static void 4293 nvme_keep_alive_completion(void *cb_ctx, const struct spdk_nvme_cpl *cpl) 4294 { 4295 /* Do nothing */ 4296 } 4297 4298 /* 4299 * Check if we need to send a Keep Alive command. 4300 * Caller must hold ctrlr->ctrlr_lock. 4301 */ 4302 static int 4303 nvme_ctrlr_keep_alive(struct spdk_nvme_ctrlr *ctrlr) 4304 { 4305 uint64_t now; 4306 struct nvme_request *req; 4307 struct spdk_nvme_cmd *cmd; 4308 int rc = 0; 4309 4310 now = spdk_get_ticks(); 4311 if (now < ctrlr->next_keep_alive_tick) { 4312 return rc; 4313 } 4314 4315 req = nvme_allocate_request_null(ctrlr->adminq, nvme_keep_alive_completion, NULL); 4316 if (req == NULL) { 4317 return rc; 4318 } 4319 4320 cmd = &req->cmd; 4321 cmd->opc = SPDK_NVME_OPC_KEEP_ALIVE; 4322 4323 rc = nvme_ctrlr_submit_admin_request(ctrlr, req); 4324 if (rc != 0) { 4325 NVME_CTRLR_ERRLOG(ctrlr, "Submitting Keep Alive failed\n"); 4326 rc = -ENXIO; 4327 } 4328 4329 ctrlr->next_keep_alive_tick = now + ctrlr->keep_alive_interval_ticks; 4330 return rc; 4331 } 4332 4333 int32_t 4334 spdk_nvme_ctrlr_process_admin_completions(struct spdk_nvme_ctrlr *ctrlr) 4335 { 4336 int32_t num_completions; 4337 int32_t rc; 4338 struct spdk_nvme_ctrlr_process *active_proc; 4339 4340 nvme_robust_mutex_lock(&ctrlr->ctrlr_lock); 4341 4342 if (ctrlr->keep_alive_interval_ticks) { 4343 rc = nvme_ctrlr_keep_alive(ctrlr); 4344 if (rc) { 4345 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 4346 return rc; 4347 } 4348 } 4349 4350 rc = nvme_io_msg_process(ctrlr); 4351 if (rc < 0) { 4352 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 4353 return rc; 4354 } 4355 num_completions = rc; 4356 4357 rc = spdk_nvme_qpair_process_completions(ctrlr->adminq, 0); 4358 4359 /* Each process has an async list, complete the ones for this process object */ 4360 active_proc = nvme_ctrlr_get_current_process(ctrlr); 4361 if (active_proc) { 4362 nvme_ctrlr_complete_queued_async_events(ctrlr); 4363 } 4364 4365 if (rc == -ENXIO && ctrlr->is_disconnecting) { 4366 nvme_ctrlr_disconnect_done(ctrlr); 4367 } 4368 4369 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 4370 4371 if (rc < 0) { 4372 num_completions = rc; 4373 } else { 4374 num_completions += rc; 4375 } 4376 4377 return num_completions; 4378 } 4379 4380 const struct spdk_nvme_ctrlr_data * 4381 spdk_nvme_ctrlr_get_data(struct spdk_nvme_ctrlr *ctrlr) 4382 { 4383 return &ctrlr->cdata; 4384 } 4385 4386 union spdk_nvme_csts_register spdk_nvme_ctrlr_get_regs_csts(struct spdk_nvme_ctrlr *ctrlr) 4387 { 4388 union spdk_nvme_csts_register csts; 4389 4390 if (nvme_ctrlr_get_csts(ctrlr, &csts)) { 4391 csts.raw = SPDK_NVME_INVALID_REGISTER_VALUE; 4392 } 4393 return csts; 4394 } 4395 4396 union spdk_nvme_cc_register spdk_nvme_ctrlr_get_regs_cc(struct spdk_nvme_ctrlr *ctrlr) 4397 { 4398 union spdk_nvme_cc_register cc; 4399 4400 if (nvme_ctrlr_get_cc(ctrlr, &cc)) { 4401 cc.raw = SPDK_NVME_INVALID_REGISTER_VALUE; 4402 } 4403 return cc; 4404 } 4405 4406 union spdk_nvme_cap_register spdk_nvme_ctrlr_get_regs_cap(struct spdk_nvme_ctrlr *ctrlr) 4407 { 4408 return ctrlr->cap; 4409 } 4410 4411 union spdk_nvme_vs_register spdk_nvme_ctrlr_get_regs_vs(struct spdk_nvme_ctrlr *ctrlr) 4412 { 4413 return ctrlr->vs; 4414 } 4415 4416 union spdk_nvme_cmbsz_register spdk_nvme_ctrlr_get_regs_cmbsz(struct spdk_nvme_ctrlr *ctrlr) 4417 { 4418 union spdk_nvme_cmbsz_register cmbsz; 4419 4420 if (nvme_ctrlr_get_cmbsz(ctrlr, &cmbsz)) { 4421 cmbsz.raw = 0; 4422 } 4423 4424 return cmbsz; 4425 } 4426 4427 union spdk_nvme_pmrcap_register spdk_nvme_ctrlr_get_regs_pmrcap(struct spdk_nvme_ctrlr *ctrlr) 4428 { 4429 union spdk_nvme_pmrcap_register pmrcap; 4430 4431 if (nvme_ctrlr_get_pmrcap(ctrlr, &pmrcap)) { 4432 pmrcap.raw = 0; 4433 } 4434 4435 return pmrcap; 4436 } 4437 4438 union spdk_nvme_bpinfo_register spdk_nvme_ctrlr_get_regs_bpinfo(struct spdk_nvme_ctrlr *ctrlr) 4439 { 4440 union spdk_nvme_bpinfo_register bpinfo; 4441 4442 if (nvme_ctrlr_get_bpinfo(ctrlr, &bpinfo)) { 4443 bpinfo.raw = 0; 4444 } 4445 4446 return bpinfo; 4447 } 4448 4449 uint64_t 4450 spdk_nvme_ctrlr_get_pmrsz(struct spdk_nvme_ctrlr *ctrlr) 4451 { 4452 return ctrlr->pmr_size; 4453 } 4454 4455 uint32_t 4456 spdk_nvme_ctrlr_get_num_ns(struct spdk_nvme_ctrlr *ctrlr) 4457 { 4458 return ctrlr->cdata.nn; 4459 } 4460 4461 bool 4462 spdk_nvme_ctrlr_is_active_ns(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid) 4463 { 4464 struct spdk_nvme_ns tmp, *ns; 4465 4466 tmp.id = nsid; 4467 ns = RB_FIND(nvme_ns_tree, &ctrlr->ns, &tmp); 4468 4469 if (ns != NULL) { 4470 return ns->active; 4471 } 4472 4473 return false; 4474 } 4475 4476 uint32_t 4477 spdk_nvme_ctrlr_get_first_active_ns(struct spdk_nvme_ctrlr *ctrlr) 4478 { 4479 struct spdk_nvme_ns *ns; 4480 4481 ns = RB_MIN(nvme_ns_tree, &ctrlr->ns); 4482 if (ns == NULL) { 4483 return 0; 4484 } 4485 4486 while (ns != NULL) { 4487 if (ns->active) { 4488 return ns->id; 4489 } 4490 4491 ns = RB_NEXT(nvme_ns_tree, &ctrlr->ns, ns); 4492 } 4493 4494 return 0; 4495 } 4496 4497 uint32_t 4498 spdk_nvme_ctrlr_get_next_active_ns(struct spdk_nvme_ctrlr *ctrlr, uint32_t prev_nsid) 4499 { 4500 struct spdk_nvme_ns tmp, *ns; 4501 4502 tmp.id = prev_nsid; 4503 ns = RB_FIND(nvme_ns_tree, &ctrlr->ns, &tmp); 4504 if (ns == NULL) { 4505 return 0; 4506 } 4507 4508 ns = RB_NEXT(nvme_ns_tree, &ctrlr->ns, ns); 4509 while (ns != NULL) { 4510 if (ns->active) { 4511 return ns->id; 4512 } 4513 4514 ns = RB_NEXT(nvme_ns_tree, &ctrlr->ns, ns); 4515 } 4516 4517 return 0; 4518 } 4519 4520 struct spdk_nvme_ns * 4521 spdk_nvme_ctrlr_get_ns(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid) 4522 { 4523 struct spdk_nvme_ns tmp; 4524 struct spdk_nvme_ns *ns; 4525 4526 if (nsid < 1 || nsid > ctrlr->cdata.nn) { 4527 return NULL; 4528 } 4529 4530 nvme_robust_mutex_lock(&ctrlr->ctrlr_lock); 4531 4532 tmp.id = nsid; 4533 ns = RB_FIND(nvme_ns_tree, &ctrlr->ns, &tmp); 4534 4535 if (ns == NULL) { 4536 ns = spdk_zmalloc(sizeof(struct spdk_nvme_ns), 64, NULL, SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_SHARE); 4537 if (ns == NULL) { 4538 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 4539 return NULL; 4540 } 4541 4542 NVME_CTRLR_DEBUGLOG(ctrlr, "Namespace %u was added\n", nsid); 4543 ns->id = nsid; 4544 RB_INSERT(nvme_ns_tree, &ctrlr->ns, ns); 4545 } 4546 4547 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 4548 4549 return ns; 4550 } 4551 4552 struct spdk_pci_device * 4553 spdk_nvme_ctrlr_get_pci_device(struct spdk_nvme_ctrlr *ctrlr) 4554 { 4555 if (ctrlr == NULL) { 4556 return NULL; 4557 } 4558 4559 if (ctrlr->trid.trtype != SPDK_NVME_TRANSPORT_PCIE) { 4560 return NULL; 4561 } 4562 4563 return nvme_ctrlr_proc_get_devhandle(ctrlr); 4564 } 4565 4566 uint32_t 4567 spdk_nvme_ctrlr_get_max_xfer_size(const struct spdk_nvme_ctrlr *ctrlr) 4568 { 4569 return ctrlr->max_xfer_size; 4570 } 4571 4572 void 4573 spdk_nvme_ctrlr_register_aer_callback(struct spdk_nvme_ctrlr *ctrlr, 4574 spdk_nvme_aer_cb aer_cb_fn, 4575 void *aer_cb_arg) 4576 { 4577 struct spdk_nvme_ctrlr_process *active_proc; 4578 4579 nvme_robust_mutex_lock(&ctrlr->ctrlr_lock); 4580 4581 active_proc = nvme_ctrlr_get_current_process(ctrlr); 4582 if (active_proc) { 4583 active_proc->aer_cb_fn = aer_cb_fn; 4584 active_proc->aer_cb_arg = aer_cb_arg; 4585 } 4586 4587 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 4588 } 4589 4590 void 4591 spdk_nvme_ctrlr_disable_read_changed_ns_list_log_page(struct spdk_nvme_ctrlr *ctrlr) 4592 { 4593 ctrlr->opts.disable_read_changed_ns_list_log_page = true; 4594 } 4595 4596 void 4597 spdk_nvme_ctrlr_register_timeout_callback(struct spdk_nvme_ctrlr *ctrlr, 4598 uint64_t timeout_io_us, uint64_t timeout_admin_us, 4599 spdk_nvme_timeout_cb cb_fn, void *cb_arg) 4600 { 4601 struct spdk_nvme_ctrlr_process *active_proc; 4602 4603 nvme_robust_mutex_lock(&ctrlr->ctrlr_lock); 4604 4605 active_proc = nvme_ctrlr_get_current_process(ctrlr); 4606 if (active_proc) { 4607 active_proc->timeout_io_ticks = timeout_io_us * spdk_get_ticks_hz() / 1000000ULL; 4608 active_proc->timeout_admin_ticks = timeout_admin_us * spdk_get_ticks_hz() / 1000000ULL; 4609 active_proc->timeout_cb_fn = cb_fn; 4610 active_proc->timeout_cb_arg = cb_arg; 4611 } 4612 4613 ctrlr->timeout_enabled = true; 4614 4615 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 4616 } 4617 4618 bool 4619 spdk_nvme_ctrlr_is_log_page_supported(struct spdk_nvme_ctrlr *ctrlr, uint8_t log_page) 4620 { 4621 /* No bounds check necessary, since log_page is uint8_t and log_page_supported has 256 entries */ 4622 SPDK_STATIC_ASSERT(sizeof(ctrlr->log_page_supported) == 256, "log_page_supported size mismatch"); 4623 return ctrlr->log_page_supported[log_page]; 4624 } 4625 4626 bool 4627 spdk_nvme_ctrlr_is_feature_supported(struct spdk_nvme_ctrlr *ctrlr, uint8_t feature_code) 4628 { 4629 /* No bounds check necessary, since feature_code is uint8_t and feature_supported has 256 entries */ 4630 SPDK_STATIC_ASSERT(sizeof(ctrlr->feature_supported) == 256, "feature_supported size mismatch"); 4631 return ctrlr->feature_supported[feature_code]; 4632 } 4633 4634 int 4635 spdk_nvme_ctrlr_attach_ns(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid, 4636 struct spdk_nvme_ctrlr_list *payload) 4637 { 4638 struct nvme_completion_poll_status *status; 4639 struct spdk_nvme_ns *ns; 4640 int res; 4641 4642 if (nsid == 0) { 4643 return -EINVAL; 4644 } 4645 4646 status = calloc(1, sizeof(*status)); 4647 if (!status) { 4648 NVME_CTRLR_ERRLOG(ctrlr, "Failed to allocate status tracker\n"); 4649 return -ENOMEM; 4650 } 4651 4652 res = nvme_ctrlr_cmd_attach_ns(ctrlr, nsid, payload, 4653 nvme_completion_poll_cb, status); 4654 if (res) { 4655 free(status); 4656 return res; 4657 } 4658 if (nvme_wait_for_completion_robust_lock(ctrlr->adminq, status, &ctrlr->ctrlr_lock)) { 4659 NVME_CTRLR_ERRLOG(ctrlr, "spdk_nvme_ctrlr_attach_ns failed!\n"); 4660 if (!status->timed_out) { 4661 free(status); 4662 } 4663 return -ENXIO; 4664 } 4665 free(status); 4666 4667 res = nvme_ctrlr_identify_active_ns(ctrlr); 4668 if (res) { 4669 return res; 4670 } 4671 4672 ns = spdk_nvme_ctrlr_get_ns(ctrlr, nsid); 4673 if (ns == NULL) { 4674 NVME_CTRLR_ERRLOG(ctrlr, "spdk_nvme_ctrlr_get_ns failed!\n"); 4675 return -ENXIO; 4676 } 4677 4678 return nvme_ns_construct(ns, nsid, ctrlr); 4679 } 4680 4681 int 4682 spdk_nvme_ctrlr_detach_ns(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid, 4683 struct spdk_nvme_ctrlr_list *payload) 4684 { 4685 struct nvme_completion_poll_status *status; 4686 int res; 4687 4688 if (nsid == 0) { 4689 return -EINVAL; 4690 } 4691 4692 status = calloc(1, sizeof(*status)); 4693 if (!status) { 4694 NVME_CTRLR_ERRLOG(ctrlr, "Failed to allocate status tracker\n"); 4695 return -ENOMEM; 4696 } 4697 4698 res = nvme_ctrlr_cmd_detach_ns(ctrlr, nsid, payload, 4699 nvme_completion_poll_cb, status); 4700 if (res) { 4701 free(status); 4702 return res; 4703 } 4704 if (nvme_wait_for_completion_robust_lock(ctrlr->adminq, status, &ctrlr->ctrlr_lock)) { 4705 NVME_CTRLR_ERRLOG(ctrlr, "spdk_nvme_ctrlr_detach_ns failed!\n"); 4706 if (!status->timed_out) { 4707 free(status); 4708 } 4709 return -ENXIO; 4710 } 4711 free(status); 4712 4713 return nvme_ctrlr_identify_active_ns(ctrlr); 4714 } 4715 4716 uint32_t 4717 spdk_nvme_ctrlr_create_ns(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_ns_data *payload) 4718 { 4719 struct nvme_completion_poll_status *status; 4720 int res; 4721 uint32_t nsid; 4722 4723 status = calloc(1, sizeof(*status)); 4724 if (!status) { 4725 NVME_CTRLR_ERRLOG(ctrlr, "Failed to allocate status tracker\n"); 4726 return 0; 4727 } 4728 4729 res = nvme_ctrlr_cmd_create_ns(ctrlr, payload, nvme_completion_poll_cb, status); 4730 if (res) { 4731 free(status); 4732 return 0; 4733 } 4734 if (nvme_wait_for_completion_robust_lock(ctrlr->adminq, status, &ctrlr->ctrlr_lock)) { 4735 NVME_CTRLR_ERRLOG(ctrlr, "spdk_nvme_ctrlr_create_ns failed!\n"); 4736 if (!status->timed_out) { 4737 free(status); 4738 } 4739 return 0; 4740 } 4741 4742 nsid = status->cpl.cdw0; 4743 free(status); 4744 4745 assert(nsid > 0); 4746 4747 /* Return the namespace ID that was created */ 4748 return nsid; 4749 } 4750 4751 int 4752 spdk_nvme_ctrlr_delete_ns(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid) 4753 { 4754 struct nvme_completion_poll_status *status; 4755 int res; 4756 4757 if (nsid == 0) { 4758 return -EINVAL; 4759 } 4760 4761 status = calloc(1, sizeof(*status)); 4762 if (!status) { 4763 NVME_CTRLR_ERRLOG(ctrlr, "Failed to allocate status tracker\n"); 4764 return -ENOMEM; 4765 } 4766 4767 res = nvme_ctrlr_cmd_delete_ns(ctrlr, nsid, nvme_completion_poll_cb, status); 4768 if (res) { 4769 free(status); 4770 return res; 4771 } 4772 if (nvme_wait_for_completion_robust_lock(ctrlr->adminq, status, &ctrlr->ctrlr_lock)) { 4773 NVME_CTRLR_ERRLOG(ctrlr, "spdk_nvme_ctrlr_delete_ns failed!\n"); 4774 if (!status->timed_out) { 4775 free(status); 4776 } 4777 return -ENXIO; 4778 } 4779 free(status); 4780 4781 return nvme_ctrlr_identify_active_ns(ctrlr); 4782 } 4783 4784 int 4785 spdk_nvme_ctrlr_format(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid, 4786 struct spdk_nvme_format *format) 4787 { 4788 struct nvme_completion_poll_status *status; 4789 int res; 4790 4791 status = calloc(1, sizeof(*status)); 4792 if (!status) { 4793 NVME_CTRLR_ERRLOG(ctrlr, "Failed to allocate status tracker\n"); 4794 return -ENOMEM; 4795 } 4796 4797 res = nvme_ctrlr_cmd_format(ctrlr, nsid, format, nvme_completion_poll_cb, 4798 status); 4799 if (res) { 4800 free(status); 4801 return res; 4802 } 4803 if (nvme_wait_for_completion_robust_lock(ctrlr->adminq, status, &ctrlr->ctrlr_lock)) { 4804 NVME_CTRLR_ERRLOG(ctrlr, "spdk_nvme_ctrlr_format failed!\n"); 4805 if (!status->timed_out) { 4806 free(status); 4807 } 4808 return -ENXIO; 4809 } 4810 free(status); 4811 4812 return spdk_nvme_ctrlr_reset(ctrlr); 4813 } 4814 4815 int 4816 spdk_nvme_ctrlr_update_firmware(struct spdk_nvme_ctrlr *ctrlr, void *payload, uint32_t size, 4817 int slot, enum spdk_nvme_fw_commit_action commit_action, struct spdk_nvme_status *completion_status) 4818 { 4819 struct spdk_nvme_fw_commit fw_commit; 4820 struct nvme_completion_poll_status *status; 4821 int res; 4822 unsigned int size_remaining; 4823 unsigned int offset; 4824 unsigned int transfer; 4825 uint8_t *p; 4826 4827 if (!completion_status) { 4828 return -EINVAL; 4829 } 4830 memset(completion_status, 0, sizeof(struct spdk_nvme_status)); 4831 if (size % 4) { 4832 NVME_CTRLR_ERRLOG(ctrlr, "spdk_nvme_ctrlr_update_firmware invalid size!\n"); 4833 return -1; 4834 } 4835 4836 /* Current support only for SPDK_NVME_FW_COMMIT_REPLACE_IMG 4837 * and SPDK_NVME_FW_COMMIT_REPLACE_AND_ENABLE_IMG 4838 */ 4839 if ((commit_action != SPDK_NVME_FW_COMMIT_REPLACE_IMG) && 4840 (commit_action != SPDK_NVME_FW_COMMIT_REPLACE_AND_ENABLE_IMG)) { 4841 NVME_CTRLR_ERRLOG(ctrlr, "spdk_nvme_ctrlr_update_firmware invalid command!\n"); 4842 return -1; 4843 } 4844 4845 status = calloc(1, sizeof(*status)); 4846 if (!status) { 4847 NVME_CTRLR_ERRLOG(ctrlr, "Failed to allocate status tracker\n"); 4848 return -ENOMEM; 4849 } 4850 4851 /* Firmware download */ 4852 size_remaining = size; 4853 offset = 0; 4854 p = payload; 4855 4856 while (size_remaining > 0) { 4857 transfer = spdk_min(size_remaining, ctrlr->min_page_size); 4858 4859 memset(status, 0, sizeof(*status)); 4860 res = nvme_ctrlr_cmd_fw_image_download(ctrlr, transfer, offset, p, 4861 nvme_completion_poll_cb, 4862 status); 4863 if (res) { 4864 free(status); 4865 return res; 4866 } 4867 4868 if (nvme_wait_for_completion_robust_lock(ctrlr->adminq, status, &ctrlr->ctrlr_lock)) { 4869 NVME_CTRLR_ERRLOG(ctrlr, "spdk_nvme_ctrlr_fw_image_download failed!\n"); 4870 if (!status->timed_out) { 4871 free(status); 4872 } 4873 return -ENXIO; 4874 } 4875 p += transfer; 4876 offset += transfer; 4877 size_remaining -= transfer; 4878 } 4879 4880 /* Firmware commit */ 4881 memset(&fw_commit, 0, sizeof(struct spdk_nvme_fw_commit)); 4882 fw_commit.fs = slot; 4883 fw_commit.ca = commit_action; 4884 4885 memset(status, 0, sizeof(*status)); 4886 res = nvme_ctrlr_cmd_fw_commit(ctrlr, &fw_commit, nvme_completion_poll_cb, 4887 status); 4888 if (res) { 4889 free(status); 4890 return res; 4891 } 4892 4893 res = nvme_wait_for_completion_robust_lock(ctrlr->adminq, status, &ctrlr->ctrlr_lock); 4894 4895 memcpy(completion_status, &status->cpl.status, sizeof(struct spdk_nvme_status)); 4896 4897 if (!status->timed_out) { 4898 free(status); 4899 } 4900 4901 if (res) { 4902 if (completion_status->sct != SPDK_NVME_SCT_COMMAND_SPECIFIC || 4903 completion_status->sc != SPDK_NVME_SC_FIRMWARE_REQ_NVM_RESET) { 4904 if (completion_status->sct == SPDK_NVME_SCT_COMMAND_SPECIFIC && 4905 completion_status->sc == SPDK_NVME_SC_FIRMWARE_REQ_CONVENTIONAL_RESET) { 4906 NVME_CTRLR_NOTICELOG(ctrlr, 4907 "firmware activation requires conventional reset to be performed. !\n"); 4908 } else { 4909 NVME_CTRLR_ERRLOG(ctrlr, "nvme_ctrlr_cmd_fw_commit failed!\n"); 4910 } 4911 return -ENXIO; 4912 } 4913 } 4914 4915 return spdk_nvme_ctrlr_reset(ctrlr); 4916 } 4917 4918 int 4919 spdk_nvme_ctrlr_reserve_cmb(struct spdk_nvme_ctrlr *ctrlr) 4920 { 4921 int rc, size; 4922 union spdk_nvme_cmbsz_register cmbsz; 4923 4924 cmbsz = spdk_nvme_ctrlr_get_regs_cmbsz(ctrlr); 4925 4926 if (cmbsz.bits.rds == 0 || cmbsz.bits.wds == 0) { 4927 return -ENOTSUP; 4928 } 4929 4930 size = cmbsz.bits.sz * (0x1000 << (cmbsz.bits.szu * 4)); 4931 4932 nvme_robust_mutex_lock(&ctrlr->ctrlr_lock); 4933 rc = nvme_transport_ctrlr_reserve_cmb(ctrlr); 4934 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 4935 4936 if (rc < 0) { 4937 return rc; 4938 } 4939 4940 return size; 4941 } 4942 4943 void * 4944 spdk_nvme_ctrlr_map_cmb(struct spdk_nvme_ctrlr *ctrlr, size_t *size) 4945 { 4946 void *buf; 4947 4948 nvme_robust_mutex_lock(&ctrlr->ctrlr_lock); 4949 buf = nvme_transport_ctrlr_map_cmb(ctrlr, size); 4950 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 4951 4952 return buf; 4953 } 4954 4955 void 4956 spdk_nvme_ctrlr_unmap_cmb(struct spdk_nvme_ctrlr *ctrlr) 4957 { 4958 nvme_robust_mutex_lock(&ctrlr->ctrlr_lock); 4959 nvme_transport_ctrlr_unmap_cmb(ctrlr); 4960 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 4961 } 4962 4963 int 4964 spdk_nvme_ctrlr_enable_pmr(struct spdk_nvme_ctrlr *ctrlr) 4965 { 4966 int rc; 4967 4968 nvme_robust_mutex_lock(&ctrlr->ctrlr_lock); 4969 rc = nvme_transport_ctrlr_enable_pmr(ctrlr); 4970 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 4971 4972 return rc; 4973 } 4974 4975 int 4976 spdk_nvme_ctrlr_disable_pmr(struct spdk_nvme_ctrlr *ctrlr) 4977 { 4978 int rc; 4979 4980 nvme_robust_mutex_lock(&ctrlr->ctrlr_lock); 4981 rc = nvme_transport_ctrlr_disable_pmr(ctrlr); 4982 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 4983 4984 return rc; 4985 } 4986 4987 void * 4988 spdk_nvme_ctrlr_map_pmr(struct spdk_nvme_ctrlr *ctrlr, size_t *size) 4989 { 4990 void *buf; 4991 4992 nvme_robust_mutex_lock(&ctrlr->ctrlr_lock); 4993 buf = nvme_transport_ctrlr_map_pmr(ctrlr, size); 4994 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 4995 4996 return buf; 4997 } 4998 4999 int 5000 spdk_nvme_ctrlr_unmap_pmr(struct spdk_nvme_ctrlr *ctrlr) 5001 { 5002 int rc; 5003 5004 nvme_robust_mutex_lock(&ctrlr->ctrlr_lock); 5005 rc = nvme_transport_ctrlr_unmap_pmr(ctrlr); 5006 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 5007 5008 return rc; 5009 } 5010 5011 int 5012 spdk_nvme_ctrlr_read_boot_partition_start(struct spdk_nvme_ctrlr *ctrlr, void *payload, 5013 uint32_t bprsz, uint32_t bprof, uint32_t bpid) 5014 { 5015 union spdk_nvme_bprsel_register bprsel; 5016 union spdk_nvme_bpinfo_register bpinfo; 5017 uint64_t bpmbl, bpmb_size; 5018 5019 if (ctrlr->cap.bits.bps == 0) { 5020 return -ENOTSUP; 5021 } 5022 5023 if (nvme_ctrlr_get_bpinfo(ctrlr, &bpinfo)) { 5024 NVME_CTRLR_ERRLOG(ctrlr, "get bpinfo failed\n"); 5025 return -EIO; 5026 } 5027 5028 if (bpinfo.bits.brs == SPDK_NVME_BRS_READ_IN_PROGRESS) { 5029 NVME_CTRLR_ERRLOG(ctrlr, "Boot Partition read already initiated\n"); 5030 return -EALREADY; 5031 } 5032 5033 nvme_robust_mutex_lock(&ctrlr->ctrlr_lock); 5034 5035 bpmb_size = bprsz * 4096; 5036 bpmbl = spdk_vtophys(payload, &bpmb_size); 5037 if (bpmbl == SPDK_VTOPHYS_ERROR) { 5038 NVME_CTRLR_ERRLOG(ctrlr, "spdk_vtophys of bpmbl failed\n"); 5039 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 5040 return -EFAULT; 5041 } 5042 5043 if (bpmb_size != bprsz * 4096) { 5044 NVME_CTRLR_ERRLOG(ctrlr, "Boot Partition buffer is not physically contiguous\n"); 5045 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 5046 return -EFAULT; 5047 } 5048 5049 if (nvme_ctrlr_set_bpmbl(ctrlr, bpmbl)) { 5050 NVME_CTRLR_ERRLOG(ctrlr, "set_bpmbl() failed\n"); 5051 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 5052 return -EIO; 5053 } 5054 5055 bprsel.bits.bpid = bpid; 5056 bprsel.bits.bprof = bprof; 5057 bprsel.bits.bprsz = bprsz; 5058 5059 if (nvme_ctrlr_set_bprsel(ctrlr, &bprsel)) { 5060 NVME_CTRLR_ERRLOG(ctrlr, "set_bprsel() failed\n"); 5061 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 5062 return -EIO; 5063 } 5064 5065 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 5066 return 0; 5067 } 5068 5069 int 5070 spdk_nvme_ctrlr_read_boot_partition_poll(struct spdk_nvme_ctrlr *ctrlr) 5071 { 5072 int rc = 0; 5073 union spdk_nvme_bpinfo_register bpinfo; 5074 5075 if (nvme_ctrlr_get_bpinfo(ctrlr, &bpinfo)) { 5076 NVME_CTRLR_ERRLOG(ctrlr, "get bpinfo failed\n"); 5077 return -EIO; 5078 } 5079 5080 switch (bpinfo.bits.brs) { 5081 case SPDK_NVME_BRS_NO_READ: 5082 NVME_CTRLR_ERRLOG(ctrlr, "Boot Partition read not initiated\n"); 5083 rc = -EINVAL; 5084 break; 5085 case SPDK_NVME_BRS_READ_IN_PROGRESS: 5086 NVME_CTRLR_DEBUGLOG(ctrlr, "Boot Partition read in progress\n"); 5087 rc = -EAGAIN; 5088 break; 5089 case SPDK_NVME_BRS_READ_ERROR: 5090 NVME_CTRLR_ERRLOG(ctrlr, "Error completing Boot Partition read\n"); 5091 rc = -EIO; 5092 break; 5093 case SPDK_NVME_BRS_READ_SUCCESS: 5094 NVME_CTRLR_INFOLOG(ctrlr, "Boot Partition read completed successfully\n"); 5095 break; 5096 default: 5097 NVME_CTRLR_ERRLOG(ctrlr, "Invalid Boot Partition read status\n"); 5098 rc = -EINVAL; 5099 } 5100 5101 return rc; 5102 } 5103 5104 static void 5105 nvme_write_boot_partition_cb(void *arg, const struct spdk_nvme_cpl *cpl) 5106 { 5107 int res; 5108 struct spdk_nvme_ctrlr *ctrlr = arg; 5109 struct spdk_nvme_fw_commit fw_commit; 5110 struct spdk_nvme_cpl err_cpl = 5111 {.status = {.sct = SPDK_NVME_SCT_GENERIC, .sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR }}; 5112 5113 if (spdk_nvme_cpl_is_error(cpl)) { 5114 NVME_CTRLR_ERRLOG(ctrlr, "Write Boot Partition failed\n"); 5115 ctrlr->bp_write_cb_fn(ctrlr->bp_write_cb_arg, cpl); 5116 return; 5117 } 5118 5119 if (ctrlr->bp_ws == SPDK_NVME_BP_WS_DOWNLOADING) { 5120 NVME_CTRLR_DEBUGLOG(ctrlr, "Boot Partition Downloading at Offset %d Success\n", ctrlr->fw_offset); 5121 ctrlr->fw_payload = (uint8_t *)ctrlr->fw_payload + ctrlr->fw_transfer_size; 5122 ctrlr->fw_offset += ctrlr->fw_transfer_size; 5123 ctrlr->fw_size_remaining -= ctrlr->fw_transfer_size; 5124 ctrlr->fw_transfer_size = spdk_min(ctrlr->fw_size_remaining, ctrlr->min_page_size); 5125 res = nvme_ctrlr_cmd_fw_image_download(ctrlr, ctrlr->fw_transfer_size, ctrlr->fw_offset, 5126 ctrlr->fw_payload, nvme_write_boot_partition_cb, ctrlr); 5127 if (res) { 5128 NVME_CTRLR_ERRLOG(ctrlr, "nvme_ctrlr_cmd_fw_image_download failed!\n"); 5129 ctrlr->bp_write_cb_fn(ctrlr->bp_write_cb_arg, &err_cpl); 5130 return; 5131 } 5132 5133 if (ctrlr->fw_transfer_size < ctrlr->min_page_size) { 5134 ctrlr->bp_ws = SPDK_NVME_BP_WS_DOWNLOADED; 5135 } 5136 } else if (ctrlr->bp_ws == SPDK_NVME_BP_WS_DOWNLOADED) { 5137 NVME_CTRLR_DEBUGLOG(ctrlr, "Boot Partition Download Success\n"); 5138 memset(&fw_commit, 0, sizeof(struct spdk_nvme_fw_commit)); 5139 fw_commit.bpid = ctrlr->bpid; 5140 fw_commit.ca = SPDK_NVME_FW_COMMIT_REPLACE_BOOT_PARTITION; 5141 res = nvme_ctrlr_cmd_fw_commit(ctrlr, &fw_commit, 5142 nvme_write_boot_partition_cb, ctrlr); 5143 if (res) { 5144 NVME_CTRLR_ERRLOG(ctrlr, "nvme_ctrlr_cmd_fw_commit failed!\n"); 5145 NVME_CTRLR_ERRLOG(ctrlr, "commit action: %d\n", fw_commit.ca); 5146 ctrlr->bp_write_cb_fn(ctrlr->bp_write_cb_arg, &err_cpl); 5147 return; 5148 } 5149 5150 ctrlr->bp_ws = SPDK_NVME_BP_WS_REPLACE; 5151 } else if (ctrlr->bp_ws == SPDK_NVME_BP_WS_REPLACE) { 5152 NVME_CTRLR_DEBUGLOG(ctrlr, "Boot Partition Replacement Success\n"); 5153 memset(&fw_commit, 0, sizeof(struct spdk_nvme_fw_commit)); 5154 fw_commit.bpid = ctrlr->bpid; 5155 fw_commit.ca = SPDK_NVME_FW_COMMIT_ACTIVATE_BOOT_PARTITION; 5156 res = nvme_ctrlr_cmd_fw_commit(ctrlr, &fw_commit, 5157 nvme_write_boot_partition_cb, ctrlr); 5158 if (res) { 5159 NVME_CTRLR_ERRLOG(ctrlr, "nvme_ctrlr_cmd_fw_commit failed!\n"); 5160 NVME_CTRLR_ERRLOG(ctrlr, "commit action: %d\n", fw_commit.ca); 5161 ctrlr->bp_write_cb_fn(ctrlr->bp_write_cb_arg, &err_cpl); 5162 return; 5163 } 5164 5165 ctrlr->bp_ws = SPDK_NVME_BP_WS_ACTIVATE; 5166 } else if (ctrlr->bp_ws == SPDK_NVME_BP_WS_ACTIVATE) { 5167 NVME_CTRLR_DEBUGLOG(ctrlr, "Boot Partition Activation Success\n"); 5168 ctrlr->bp_write_cb_fn(ctrlr->bp_write_cb_arg, cpl); 5169 } else { 5170 NVME_CTRLR_ERRLOG(ctrlr, "Invalid Boot Partition write state\n"); 5171 ctrlr->bp_write_cb_fn(ctrlr->bp_write_cb_arg, &err_cpl); 5172 return; 5173 } 5174 } 5175 5176 int 5177 spdk_nvme_ctrlr_write_boot_partition(struct spdk_nvme_ctrlr *ctrlr, 5178 void *payload, uint32_t size, uint32_t bpid, 5179 spdk_nvme_cmd_cb cb_fn, void *cb_arg) 5180 { 5181 int res; 5182 5183 if (ctrlr->cap.bits.bps == 0) { 5184 return -ENOTSUP; 5185 } 5186 5187 ctrlr->bp_ws = SPDK_NVME_BP_WS_DOWNLOADING; 5188 ctrlr->bpid = bpid; 5189 ctrlr->bp_write_cb_fn = cb_fn; 5190 ctrlr->bp_write_cb_arg = cb_arg; 5191 ctrlr->fw_offset = 0; 5192 ctrlr->fw_size_remaining = size; 5193 ctrlr->fw_payload = payload; 5194 ctrlr->fw_transfer_size = spdk_min(ctrlr->fw_size_remaining, ctrlr->min_page_size); 5195 5196 res = nvme_ctrlr_cmd_fw_image_download(ctrlr, ctrlr->fw_transfer_size, ctrlr->fw_offset, 5197 ctrlr->fw_payload, nvme_write_boot_partition_cb, ctrlr); 5198 5199 return res; 5200 } 5201 5202 bool 5203 spdk_nvme_ctrlr_is_discovery(struct spdk_nvme_ctrlr *ctrlr) 5204 { 5205 assert(ctrlr); 5206 5207 return !strncmp(ctrlr->trid.subnqn, SPDK_NVMF_DISCOVERY_NQN, 5208 strlen(SPDK_NVMF_DISCOVERY_NQN)); 5209 } 5210 5211 bool 5212 spdk_nvme_ctrlr_is_fabrics(struct spdk_nvme_ctrlr *ctrlr) 5213 { 5214 assert(ctrlr); 5215 5216 return spdk_nvme_trtype_is_fabrics(ctrlr->trid.trtype); 5217 } 5218 5219 int 5220 spdk_nvme_ctrlr_security_receive(struct spdk_nvme_ctrlr *ctrlr, uint8_t secp, 5221 uint16_t spsp, uint8_t nssf, void *payload, size_t size) 5222 { 5223 struct nvme_completion_poll_status *status; 5224 int res; 5225 5226 status = calloc(1, sizeof(*status)); 5227 if (!status) { 5228 NVME_CTRLR_ERRLOG(ctrlr, "Failed to allocate status tracker\n"); 5229 return -ENOMEM; 5230 } 5231 5232 res = spdk_nvme_ctrlr_cmd_security_receive(ctrlr, secp, spsp, nssf, payload, size, 5233 nvme_completion_poll_cb, status); 5234 if (res) { 5235 free(status); 5236 return res; 5237 } 5238 if (nvme_wait_for_completion_robust_lock(ctrlr->adminq, status, &ctrlr->ctrlr_lock)) { 5239 NVME_CTRLR_ERRLOG(ctrlr, "spdk_nvme_ctrlr_cmd_security_receive failed!\n"); 5240 if (!status->timed_out) { 5241 free(status); 5242 } 5243 return -ENXIO; 5244 } 5245 free(status); 5246 5247 return 0; 5248 } 5249 5250 int 5251 spdk_nvme_ctrlr_security_send(struct spdk_nvme_ctrlr *ctrlr, uint8_t secp, 5252 uint16_t spsp, uint8_t nssf, void *payload, size_t size) 5253 { 5254 struct nvme_completion_poll_status *status; 5255 int res; 5256 5257 status = calloc(1, sizeof(*status)); 5258 if (!status) { 5259 NVME_CTRLR_ERRLOG(ctrlr, "Failed to allocate status tracker\n"); 5260 return -ENOMEM; 5261 } 5262 5263 res = spdk_nvme_ctrlr_cmd_security_send(ctrlr, secp, spsp, nssf, payload, size, 5264 nvme_completion_poll_cb, 5265 status); 5266 if (res) { 5267 free(status); 5268 return res; 5269 } 5270 if (nvme_wait_for_completion_robust_lock(ctrlr->adminq, status, &ctrlr->ctrlr_lock)) { 5271 NVME_CTRLR_ERRLOG(ctrlr, "spdk_nvme_ctrlr_cmd_security_send failed!\n"); 5272 if (!status->timed_out) { 5273 free(status); 5274 } 5275 return -ENXIO; 5276 } 5277 5278 free(status); 5279 5280 return 0; 5281 } 5282 5283 uint64_t 5284 spdk_nvme_ctrlr_get_flags(struct spdk_nvme_ctrlr *ctrlr) 5285 { 5286 return ctrlr->flags; 5287 } 5288 5289 const struct spdk_nvme_transport_id * 5290 spdk_nvme_ctrlr_get_transport_id(struct spdk_nvme_ctrlr *ctrlr) 5291 { 5292 return &ctrlr->trid; 5293 } 5294 5295 int32_t 5296 spdk_nvme_ctrlr_alloc_qid(struct spdk_nvme_ctrlr *ctrlr) 5297 { 5298 uint32_t qid; 5299 5300 assert(ctrlr->free_io_qids); 5301 nvme_robust_mutex_lock(&ctrlr->ctrlr_lock); 5302 qid = spdk_bit_array_find_first_set(ctrlr->free_io_qids, 1); 5303 if (qid > ctrlr->opts.num_io_queues) { 5304 NVME_CTRLR_ERRLOG(ctrlr, "No free I/O queue IDs\n"); 5305 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 5306 return -1; 5307 } 5308 5309 spdk_bit_array_clear(ctrlr->free_io_qids, qid); 5310 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 5311 return qid; 5312 } 5313 5314 void 5315 spdk_nvme_ctrlr_free_qid(struct spdk_nvme_ctrlr *ctrlr, uint16_t qid) 5316 { 5317 assert(qid <= ctrlr->opts.num_io_queues); 5318 5319 nvme_robust_mutex_lock(&ctrlr->ctrlr_lock); 5320 5321 if (spdk_likely(ctrlr->free_io_qids)) { 5322 spdk_bit_array_set(ctrlr->free_io_qids, qid); 5323 } 5324 5325 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 5326 } 5327 5328 int 5329 spdk_nvme_ctrlr_get_memory_domains(const struct spdk_nvme_ctrlr *ctrlr, 5330 struct spdk_memory_domain **domains, int array_size) 5331 { 5332 return nvme_transport_ctrlr_get_memory_domains(ctrlr, domains, array_size); 5333 } 5334