1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2015 Intel Corporation. All rights reserved. 3 * Copyright (c) 2019-2021 Mellanox Technologies LTD. All rights reserved. 4 * Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 5 */ 6 7 #include "spdk/stdinc.h" 8 9 #include "nvme_internal.h" 10 #include "nvme_io_msg.h" 11 12 #include "spdk/env.h" 13 #include "spdk/string.h" 14 #include "spdk/endian.h" 15 16 struct nvme_active_ns_ctx; 17 18 static int nvme_ctrlr_construct_and_submit_aer(struct spdk_nvme_ctrlr *ctrlr, 19 struct nvme_async_event_request *aer); 20 static void nvme_ctrlr_identify_active_ns_async(struct nvme_active_ns_ctx *ctx); 21 static int nvme_ctrlr_identify_ns_async(struct spdk_nvme_ns *ns); 22 static int nvme_ctrlr_identify_ns_iocs_specific_async(struct spdk_nvme_ns *ns); 23 static int nvme_ctrlr_identify_id_desc_async(struct spdk_nvme_ns *ns); 24 static void nvme_ctrlr_init_cap(struct spdk_nvme_ctrlr *ctrlr); 25 static void nvme_ctrlr_set_state(struct spdk_nvme_ctrlr *ctrlr, enum nvme_ctrlr_state state, 26 uint64_t timeout_in_ms); 27 28 static int 29 nvme_ns_cmp(struct spdk_nvme_ns *ns1, struct spdk_nvme_ns *ns2) 30 { 31 if (ns1->id < ns2->id) { 32 return -1; 33 } else if (ns1->id > ns2->id) { 34 return 1; 35 } else { 36 return 0; 37 } 38 } 39 40 RB_GENERATE_STATIC(nvme_ns_tree, spdk_nvme_ns, node, nvme_ns_cmp); 41 42 #define nvme_ctrlr_get_reg_async(ctrlr, reg, sz, cb_fn, cb_arg) \ 43 nvme_transport_ctrlr_get_reg_ ## sz ## _async(ctrlr, \ 44 offsetof(struct spdk_nvme_registers, reg), cb_fn, cb_arg) 45 46 #define nvme_ctrlr_set_reg_async(ctrlr, reg, sz, val, cb_fn, cb_arg) \ 47 nvme_transport_ctrlr_set_reg_ ## sz ## _async(ctrlr, \ 48 offsetof(struct spdk_nvme_registers, reg), val, cb_fn, cb_arg) 49 50 #define nvme_ctrlr_get_cc_async(ctrlr, cb_fn, cb_arg) \ 51 nvme_ctrlr_get_reg_async(ctrlr, cc, 4, cb_fn, cb_arg) 52 53 #define nvme_ctrlr_get_csts_async(ctrlr, cb_fn, cb_arg) \ 54 nvme_ctrlr_get_reg_async(ctrlr, csts, 4, cb_fn, cb_arg) 55 56 #define nvme_ctrlr_get_cap_async(ctrlr, cb_fn, cb_arg) \ 57 nvme_ctrlr_get_reg_async(ctrlr, cap, 8, cb_fn, cb_arg) 58 59 #define nvme_ctrlr_get_vs_async(ctrlr, cb_fn, cb_arg) \ 60 nvme_ctrlr_get_reg_async(ctrlr, vs, 4, cb_fn, cb_arg) 61 62 #define nvme_ctrlr_set_cc_async(ctrlr, value, cb_fn, cb_arg) \ 63 nvme_ctrlr_set_reg_async(ctrlr, cc, 4, value, cb_fn, cb_arg) 64 65 static int 66 nvme_ctrlr_get_cc(struct spdk_nvme_ctrlr *ctrlr, union spdk_nvme_cc_register *cc) 67 { 68 return nvme_transport_ctrlr_get_reg_4(ctrlr, offsetof(struct spdk_nvme_registers, cc.raw), 69 &cc->raw); 70 } 71 72 static int 73 nvme_ctrlr_get_csts(struct spdk_nvme_ctrlr *ctrlr, union spdk_nvme_csts_register *csts) 74 { 75 return nvme_transport_ctrlr_get_reg_4(ctrlr, offsetof(struct spdk_nvme_registers, csts.raw), 76 &csts->raw); 77 } 78 79 int 80 nvme_ctrlr_get_cap(struct spdk_nvme_ctrlr *ctrlr, union spdk_nvme_cap_register *cap) 81 { 82 return nvme_transport_ctrlr_get_reg_8(ctrlr, offsetof(struct spdk_nvme_registers, cap.raw), 83 &cap->raw); 84 } 85 86 int 87 nvme_ctrlr_get_vs(struct spdk_nvme_ctrlr *ctrlr, union spdk_nvme_vs_register *vs) 88 { 89 return nvme_transport_ctrlr_get_reg_4(ctrlr, offsetof(struct spdk_nvme_registers, vs.raw), 90 &vs->raw); 91 } 92 93 int 94 nvme_ctrlr_get_cmbsz(struct spdk_nvme_ctrlr *ctrlr, union spdk_nvme_cmbsz_register *cmbsz) 95 { 96 return nvme_transport_ctrlr_get_reg_4(ctrlr, offsetof(struct spdk_nvme_registers, cmbsz.raw), 97 &cmbsz->raw); 98 } 99 100 int 101 nvme_ctrlr_get_pmrcap(struct spdk_nvme_ctrlr *ctrlr, union spdk_nvme_pmrcap_register *pmrcap) 102 { 103 return nvme_transport_ctrlr_get_reg_4(ctrlr, offsetof(struct spdk_nvme_registers, pmrcap.raw), 104 &pmrcap->raw); 105 } 106 107 int 108 nvme_ctrlr_get_bpinfo(struct spdk_nvme_ctrlr *ctrlr, union spdk_nvme_bpinfo_register *bpinfo) 109 { 110 return nvme_transport_ctrlr_get_reg_4(ctrlr, offsetof(struct spdk_nvme_registers, bpinfo.raw), 111 &bpinfo->raw); 112 } 113 114 int 115 nvme_ctrlr_set_bprsel(struct spdk_nvme_ctrlr *ctrlr, union spdk_nvme_bprsel_register *bprsel) 116 { 117 return nvme_transport_ctrlr_set_reg_4(ctrlr, offsetof(struct spdk_nvme_registers, bprsel.raw), 118 bprsel->raw); 119 } 120 121 int 122 nvme_ctrlr_set_bpmbl(struct spdk_nvme_ctrlr *ctrlr, uint64_t bpmbl_value) 123 { 124 return nvme_transport_ctrlr_set_reg_8(ctrlr, offsetof(struct spdk_nvme_registers, bpmbl), 125 bpmbl_value); 126 } 127 128 static int 129 nvme_ctrlr_set_nssr(struct spdk_nvme_ctrlr *ctrlr, uint32_t nssr_value) 130 { 131 return nvme_transport_ctrlr_set_reg_4(ctrlr, offsetof(struct spdk_nvme_registers, nssr), 132 nssr_value); 133 } 134 135 bool 136 nvme_ctrlr_multi_iocs_enabled(struct spdk_nvme_ctrlr *ctrlr) 137 { 138 return ctrlr->cap.bits.css & SPDK_NVME_CAP_CSS_IOCS && 139 ctrlr->opts.command_set == SPDK_NVME_CC_CSS_IOCS; 140 } 141 142 /* When the field in spdk_nvme_ctrlr_opts are changed and you change this function, please 143 * also update the nvme_ctrl_opts_init function in nvme_ctrlr.c 144 */ 145 void 146 spdk_nvme_ctrlr_get_default_ctrlr_opts(struct spdk_nvme_ctrlr_opts *opts, size_t opts_size) 147 { 148 assert(opts); 149 150 opts->opts_size = opts_size; 151 152 #define FIELD_OK(field) \ 153 offsetof(struct spdk_nvme_ctrlr_opts, field) + sizeof(opts->field) <= opts_size 154 155 #define SET_FIELD(field, value) \ 156 if (offsetof(struct spdk_nvme_ctrlr_opts, field) + sizeof(opts->field) <= opts_size) { \ 157 opts->field = value; \ 158 } \ 159 160 SET_FIELD(num_io_queues, DEFAULT_MAX_IO_QUEUES); 161 SET_FIELD(use_cmb_sqs, false); 162 SET_FIELD(no_shn_notification, false); 163 SET_FIELD(enable_interrupts, false); 164 SET_FIELD(arb_mechanism, SPDK_NVME_CC_AMS_RR); 165 SET_FIELD(arbitration_burst, 0); 166 SET_FIELD(low_priority_weight, 0); 167 SET_FIELD(medium_priority_weight, 0); 168 SET_FIELD(high_priority_weight, 0); 169 SET_FIELD(keep_alive_timeout_ms, MIN_KEEP_ALIVE_TIMEOUT_IN_MS); 170 SET_FIELD(transport_retry_count, SPDK_NVME_DEFAULT_RETRY_COUNT); 171 SET_FIELD(io_queue_size, DEFAULT_IO_QUEUE_SIZE); 172 173 if (nvme_driver_init() == 0) { 174 if (FIELD_OK(hostnqn)) { 175 nvme_get_default_hostnqn(opts->hostnqn, sizeof(opts->hostnqn)); 176 } 177 178 if (FIELD_OK(extended_host_id)) { 179 memcpy(opts->extended_host_id, &g_spdk_nvme_driver->default_extended_host_id, 180 sizeof(opts->extended_host_id)); 181 } 182 183 } 184 185 SET_FIELD(io_queue_requests, DEFAULT_IO_QUEUE_REQUESTS); 186 187 if (FIELD_OK(src_addr)) { 188 memset(opts->src_addr, 0, sizeof(opts->src_addr)); 189 } 190 191 if (FIELD_OK(src_svcid)) { 192 memset(opts->src_svcid, 0, sizeof(opts->src_svcid)); 193 } 194 195 if (FIELD_OK(host_id)) { 196 memset(opts->host_id, 0, sizeof(opts->host_id)); 197 } 198 199 SET_FIELD(command_set, CHAR_BIT); 200 SET_FIELD(admin_timeout_ms, NVME_MAX_ADMIN_TIMEOUT_IN_SECS * 1000); 201 SET_FIELD(header_digest, false); 202 SET_FIELD(data_digest, false); 203 SET_FIELD(disable_error_logging, false); 204 SET_FIELD(transport_ack_timeout, SPDK_NVME_DEFAULT_TRANSPORT_ACK_TIMEOUT); 205 SET_FIELD(admin_queue_size, DEFAULT_ADMIN_QUEUE_SIZE); 206 SET_FIELD(fabrics_connect_timeout_us, NVME_FABRIC_CONNECT_COMMAND_TIMEOUT); 207 SET_FIELD(disable_read_ana_log_page, false); 208 SET_FIELD(disable_read_changed_ns_list_log_page, false); 209 SET_FIELD(tls_psk, NULL); 210 SET_FIELD(dhchap_key, NULL); 211 SET_FIELD(dhchap_ctrlr_key, NULL); 212 SET_FIELD(dhchap_digests, 213 SPDK_BIT(SPDK_NVMF_DHCHAP_HASH_SHA256) | 214 SPDK_BIT(SPDK_NVMF_DHCHAP_HASH_SHA384) | 215 SPDK_BIT(SPDK_NVMF_DHCHAP_HASH_SHA512)); 216 SET_FIELD(dhchap_dhgroups, 217 SPDK_BIT(SPDK_NVMF_DHCHAP_DHGROUP_NULL) | 218 SPDK_BIT(SPDK_NVMF_DHCHAP_DHGROUP_2048) | 219 SPDK_BIT(SPDK_NVMF_DHCHAP_DHGROUP_3072) | 220 SPDK_BIT(SPDK_NVMF_DHCHAP_DHGROUP_4096) | 221 SPDK_BIT(SPDK_NVMF_DHCHAP_DHGROUP_6144) | 222 SPDK_BIT(SPDK_NVMF_DHCHAP_DHGROUP_8192)); 223 #undef FIELD_OK 224 #undef SET_FIELD 225 } 226 227 const struct spdk_nvme_ctrlr_opts * 228 spdk_nvme_ctrlr_get_opts(struct spdk_nvme_ctrlr *ctrlr) 229 { 230 return &ctrlr->opts; 231 } 232 233 /** 234 * This function will be called when the process allocates the IO qpair. 235 * Note: the ctrlr_lock must be held when calling this function. 236 */ 237 static void 238 nvme_ctrlr_proc_add_io_qpair(struct spdk_nvme_qpair *qpair) 239 { 240 struct spdk_nvme_ctrlr_process *active_proc; 241 struct spdk_nvme_ctrlr *ctrlr = qpair->ctrlr; 242 243 active_proc = nvme_ctrlr_get_current_process(ctrlr); 244 if (active_proc) { 245 TAILQ_INSERT_TAIL(&active_proc->allocated_io_qpairs, qpair, per_process_tailq); 246 qpair->active_proc = active_proc; 247 } 248 } 249 250 /** 251 * This function will be called when the process frees the IO qpair. 252 * Note: the ctrlr_lock must be held when calling this function. 253 */ 254 static void 255 nvme_ctrlr_proc_remove_io_qpair(struct spdk_nvme_qpair *qpair) 256 { 257 struct spdk_nvme_ctrlr_process *active_proc; 258 struct spdk_nvme_ctrlr *ctrlr = qpair->ctrlr; 259 struct spdk_nvme_qpair *active_qpair, *tmp_qpair; 260 261 active_proc = nvme_ctrlr_get_current_process(ctrlr); 262 if (!active_proc) { 263 return; 264 } 265 266 TAILQ_FOREACH_SAFE(active_qpair, &active_proc->allocated_io_qpairs, 267 per_process_tailq, tmp_qpair) { 268 if (active_qpair == qpair) { 269 TAILQ_REMOVE(&active_proc->allocated_io_qpairs, 270 active_qpair, per_process_tailq); 271 272 break; 273 } 274 } 275 } 276 277 void 278 spdk_nvme_ctrlr_get_default_io_qpair_opts(struct spdk_nvme_ctrlr *ctrlr, 279 struct spdk_nvme_io_qpair_opts *opts, 280 size_t opts_size) 281 { 282 assert(ctrlr); 283 284 assert(opts); 285 286 memset(opts, 0, opts_size); 287 opts->opts_size = opts_size; 288 289 #define FIELD_OK(field) \ 290 offsetof(struct spdk_nvme_io_qpair_opts, field) + sizeof(opts->field) <= opts_size 291 292 #define SET_FIELD(field, value) \ 293 if (FIELD_OK(field)) { \ 294 opts->field = value; \ 295 } \ 296 297 SET_FIELD(qprio, SPDK_NVME_QPRIO_URGENT); 298 SET_FIELD(io_queue_size, ctrlr->opts.io_queue_size); 299 SET_FIELD(io_queue_requests, ctrlr->opts.io_queue_requests); 300 SET_FIELD(delay_cmd_submit, false); 301 SET_FIELD(sq.vaddr, NULL); 302 SET_FIELD(sq.paddr, 0); 303 SET_FIELD(sq.buffer_size, 0); 304 SET_FIELD(cq.vaddr, NULL); 305 SET_FIELD(cq.paddr, 0); 306 SET_FIELD(cq.buffer_size, 0); 307 SET_FIELD(create_only, false); 308 SET_FIELD(async_mode, false); 309 SET_FIELD(disable_pcie_sgl_merge, false); 310 311 #undef FIELD_OK 312 #undef SET_FIELD 313 } 314 315 static void 316 nvme_ctrlr_io_qpair_opts_copy(struct spdk_nvme_io_qpair_opts *dst, 317 const struct spdk_nvme_io_qpair_opts *src, size_t opts_size_src) 318 { 319 if (!opts_size_src) { 320 SPDK_ERRLOG("opts_size_src should not be zero value\n"); 321 assert(false); 322 } 323 324 #define FIELD_OK(field) \ 325 offsetof(struct spdk_nvme_io_qpair_opts, field) + sizeof(src->field) <= opts_size_src 326 327 #define SET_FIELD(field) \ 328 if (FIELD_OK(field)) { \ 329 dst->field = src->field; \ 330 } \ 331 332 SET_FIELD(qprio); 333 SET_FIELD(io_queue_size); 334 SET_FIELD(io_queue_requests); 335 SET_FIELD(delay_cmd_submit); 336 SET_FIELD(sq.vaddr); 337 SET_FIELD(sq.paddr); 338 SET_FIELD(sq.buffer_size); 339 SET_FIELD(cq.vaddr); 340 SET_FIELD(cq.paddr); 341 SET_FIELD(cq.buffer_size); 342 SET_FIELD(create_only); 343 SET_FIELD(async_mode); 344 SET_FIELD(disable_pcie_sgl_merge); 345 346 dst->opts_size = opts_size_src; 347 348 /* You should not remove this statement, but need to update the assert statement 349 * if you add a new field, and also add a corresponding SET_FIELD statement */ 350 SPDK_STATIC_ASSERT(sizeof(struct spdk_nvme_io_qpair_opts) == 80, "Incorrect size"); 351 352 #undef FIELD_OK 353 #undef SET_FIELD 354 } 355 356 static struct spdk_nvme_qpair * 357 nvme_ctrlr_create_io_qpair(struct spdk_nvme_ctrlr *ctrlr, 358 const struct spdk_nvme_io_qpair_opts *opts) 359 { 360 int32_t qid; 361 struct spdk_nvme_qpair *qpair; 362 union spdk_nvme_cc_register cc; 363 364 if (!ctrlr) { 365 return NULL; 366 } 367 368 nvme_ctrlr_lock(ctrlr); 369 cc.raw = ctrlr->process_init_cc.raw; 370 371 if (opts->qprio & ~SPDK_NVME_CREATE_IO_SQ_QPRIO_MASK) { 372 nvme_ctrlr_unlock(ctrlr); 373 return NULL; 374 } 375 376 /* 377 * Only value SPDK_NVME_QPRIO_URGENT(0) is valid for the 378 * default round robin arbitration method. 379 */ 380 if ((cc.bits.ams == SPDK_NVME_CC_AMS_RR) && (opts->qprio != SPDK_NVME_QPRIO_URGENT)) { 381 NVME_CTRLR_ERRLOG(ctrlr, "invalid queue priority for default round robin arbitration method\n"); 382 nvme_ctrlr_unlock(ctrlr); 383 return NULL; 384 } 385 386 qid = spdk_nvme_ctrlr_alloc_qid(ctrlr); 387 if (qid < 0) { 388 nvme_ctrlr_unlock(ctrlr); 389 return NULL; 390 } 391 392 qpair = nvme_transport_ctrlr_create_io_qpair(ctrlr, qid, opts); 393 if (qpair == NULL) { 394 NVME_CTRLR_ERRLOG(ctrlr, "nvme_transport_ctrlr_create_io_qpair() failed\n"); 395 spdk_nvme_ctrlr_free_qid(ctrlr, qid); 396 nvme_ctrlr_unlock(ctrlr); 397 return NULL; 398 } 399 400 TAILQ_INSERT_TAIL(&ctrlr->active_io_qpairs, qpair, tailq); 401 402 nvme_ctrlr_proc_add_io_qpair(qpair); 403 404 nvme_ctrlr_unlock(ctrlr); 405 406 return qpair; 407 } 408 409 int 410 spdk_nvme_ctrlr_connect_io_qpair(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair) 411 { 412 int rc; 413 414 if (nvme_qpair_get_state(qpair) != NVME_QPAIR_DISCONNECTED) { 415 return -EISCONN; 416 } 417 418 nvme_ctrlr_lock(ctrlr); 419 rc = nvme_transport_ctrlr_connect_qpair(ctrlr, qpair); 420 nvme_ctrlr_unlock(ctrlr); 421 422 if (ctrlr->quirks & NVME_QUIRK_DELAY_AFTER_QUEUE_ALLOC) { 423 spdk_delay_us(100); 424 } 425 426 return rc; 427 } 428 429 void 430 spdk_nvme_ctrlr_disconnect_io_qpair(struct spdk_nvme_qpair *qpair) 431 { 432 struct spdk_nvme_ctrlr *ctrlr = qpair->ctrlr; 433 434 nvme_ctrlr_lock(ctrlr); 435 nvme_transport_ctrlr_disconnect_qpair(ctrlr, qpair); 436 nvme_ctrlr_unlock(ctrlr); 437 } 438 439 int 440 spdk_nvme_ctrlr_get_admin_qp_fd(struct spdk_nvme_ctrlr *ctrlr, 441 struct spdk_event_handler_opts *opts) 442 { 443 return spdk_nvme_qpair_get_fd(ctrlr->adminq, opts); 444 } 445 446 struct spdk_nvme_qpair * 447 spdk_nvme_ctrlr_alloc_io_qpair(struct spdk_nvme_ctrlr *ctrlr, 448 const struct spdk_nvme_io_qpair_opts *user_opts, 449 size_t opts_size) 450 { 451 452 struct spdk_nvme_qpair *qpair = NULL; 453 struct spdk_nvme_io_qpair_opts opts; 454 int rc; 455 456 nvme_ctrlr_lock(ctrlr); 457 458 if (spdk_unlikely(ctrlr->state != NVME_CTRLR_STATE_READY)) { 459 /* When controller is resetting or initializing, free_io_qids is deleted or not created yet. 460 * We can't create IO qpair in that case */ 461 goto unlock; 462 } 463 464 /* 465 * Get the default options, then overwrite them with the user-provided options 466 * up to opts_size. 467 * 468 * This allows for extensions of the opts structure without breaking 469 * ABI compatibility. 470 */ 471 spdk_nvme_ctrlr_get_default_io_qpair_opts(ctrlr, &opts, sizeof(opts)); 472 if (user_opts) { 473 nvme_ctrlr_io_qpair_opts_copy(&opts, user_opts, spdk_min(opts.opts_size, opts_size)); 474 475 /* If user passes buffers, make sure they're big enough for the requested queue size */ 476 if (opts.sq.vaddr) { 477 if (opts.sq.buffer_size < (opts.io_queue_size * sizeof(struct spdk_nvme_cmd))) { 478 NVME_CTRLR_ERRLOG(ctrlr, "sq buffer size %" PRIx64 " is too small for sq size %zx\n", 479 opts.sq.buffer_size, (opts.io_queue_size * sizeof(struct spdk_nvme_cmd))); 480 goto unlock; 481 } 482 } 483 if (opts.cq.vaddr) { 484 if (opts.cq.buffer_size < (opts.io_queue_size * sizeof(struct spdk_nvme_cpl))) { 485 NVME_CTRLR_ERRLOG(ctrlr, "cq buffer size %" PRIx64 " is too small for cq size %zx\n", 486 opts.cq.buffer_size, (opts.io_queue_size * sizeof(struct spdk_nvme_cpl))); 487 goto unlock; 488 } 489 } 490 } 491 492 if (ctrlr->opts.enable_interrupts && opts.delay_cmd_submit) { 493 NVME_CTRLR_ERRLOG(ctrlr, "delay command submit cannot work with interrupts\n"); 494 goto unlock; 495 } 496 497 qpair = nvme_ctrlr_create_io_qpair(ctrlr, &opts); 498 499 if (qpair == NULL || opts.create_only == true) { 500 goto unlock; 501 } 502 503 rc = spdk_nvme_ctrlr_connect_io_qpair(ctrlr, qpair); 504 if (rc != 0) { 505 NVME_CTRLR_ERRLOG(ctrlr, "nvme_transport_ctrlr_connect_io_qpair() failed\n"); 506 nvme_ctrlr_proc_remove_io_qpair(qpair); 507 TAILQ_REMOVE(&ctrlr->active_io_qpairs, qpair, tailq); 508 spdk_bit_array_set(ctrlr->free_io_qids, qpair->id); 509 nvme_transport_ctrlr_delete_io_qpair(ctrlr, qpair); 510 qpair = NULL; 511 goto unlock; 512 } 513 514 unlock: 515 nvme_ctrlr_unlock(ctrlr); 516 517 return qpair; 518 } 519 520 int 521 spdk_nvme_ctrlr_reconnect_io_qpair(struct spdk_nvme_qpair *qpair) 522 { 523 struct spdk_nvme_ctrlr *ctrlr; 524 enum nvme_qpair_state qpair_state; 525 int rc; 526 527 assert(qpair != NULL); 528 assert(nvme_qpair_is_admin_queue(qpair) == false); 529 assert(qpair->ctrlr != NULL); 530 531 ctrlr = qpair->ctrlr; 532 nvme_ctrlr_lock(ctrlr); 533 qpair_state = nvme_qpair_get_state(qpair); 534 535 if (ctrlr->is_removed) { 536 rc = -ENODEV; 537 goto out; 538 } 539 540 if (ctrlr->is_resetting || qpair_state == NVME_QPAIR_DISCONNECTING) { 541 rc = -EAGAIN; 542 goto out; 543 } 544 545 if (ctrlr->is_failed || qpair_state == NVME_QPAIR_DESTROYING) { 546 rc = -ENXIO; 547 goto out; 548 } 549 550 if (qpair_state != NVME_QPAIR_DISCONNECTED) { 551 rc = 0; 552 goto out; 553 } 554 555 rc = nvme_transport_ctrlr_connect_qpair(ctrlr, qpair); 556 if (rc) { 557 rc = -EAGAIN; 558 goto out; 559 } 560 561 out: 562 nvme_ctrlr_unlock(ctrlr); 563 return rc; 564 } 565 566 spdk_nvme_qp_failure_reason 567 spdk_nvme_ctrlr_get_admin_qp_failure_reason(struct spdk_nvme_ctrlr *ctrlr) 568 { 569 return ctrlr->adminq->transport_failure_reason; 570 } 571 572 /* 573 * This internal function will attempt to take the controller 574 * lock before calling disconnect on a controller qpair. 575 * Functions already holding the controller lock should 576 * call nvme_transport_ctrlr_disconnect_qpair directly. 577 */ 578 void 579 nvme_ctrlr_disconnect_qpair(struct spdk_nvme_qpair *qpair) 580 { 581 struct spdk_nvme_ctrlr *ctrlr = qpair->ctrlr; 582 583 assert(ctrlr != NULL); 584 nvme_ctrlr_lock(ctrlr); 585 nvme_transport_ctrlr_disconnect_qpair(ctrlr, qpair); 586 nvme_ctrlr_unlock(ctrlr); 587 } 588 589 int 590 spdk_nvme_ctrlr_free_io_qpair(struct spdk_nvme_qpair *qpair) 591 { 592 struct spdk_nvme_ctrlr *ctrlr; 593 594 if (qpair == NULL) { 595 return 0; 596 } 597 598 ctrlr = qpair->ctrlr; 599 600 if (qpair->in_completion_context) { 601 /* 602 * There are many cases where it is convenient to delete an io qpair in the context 603 * of that qpair's completion routine. To handle this properly, set a flag here 604 * so that the completion routine will perform an actual delete after the context 605 * unwinds. 606 */ 607 qpair->delete_after_completion_context = 1; 608 return 0; 609 } 610 611 if (qpair->auth.cb_fn != NULL) { 612 qpair->auth.cb_fn(qpair->auth.cb_ctx, -ECANCELED); 613 qpair->auth.cb_fn = NULL; 614 } 615 616 qpair->destroy_in_progress = 1; 617 618 nvme_transport_ctrlr_disconnect_qpair(ctrlr, qpair); 619 620 if (qpair->poll_group && (qpair->active_proc == nvme_ctrlr_get_current_process(ctrlr))) { 621 spdk_nvme_poll_group_remove(qpair->poll_group->group, qpair); 622 } 623 624 /* Do not retry. */ 625 nvme_qpair_set_state(qpair, NVME_QPAIR_DESTROYING); 626 627 /* In the multi-process case, a process may call this function on a foreign 628 * I/O qpair (i.e. one that this process did not create) when that qpairs process 629 * exits unexpectedly. In that case, we must not try to abort any reqs associated 630 * with that qpair, since the callbacks will also be foreign to this process. 631 */ 632 if (qpair->active_proc == nvme_ctrlr_get_current_process(ctrlr)) { 633 nvme_qpair_abort_all_queued_reqs(qpair); 634 } 635 636 nvme_ctrlr_lock(ctrlr); 637 638 nvme_ctrlr_proc_remove_io_qpair(qpair); 639 640 TAILQ_REMOVE(&ctrlr->active_io_qpairs, qpair, tailq); 641 spdk_nvme_ctrlr_free_qid(ctrlr, qpair->id); 642 643 nvme_transport_ctrlr_delete_io_qpair(ctrlr, qpair); 644 nvme_ctrlr_unlock(ctrlr); 645 return 0; 646 } 647 648 static void 649 nvme_ctrlr_construct_intel_support_log_page_list(struct spdk_nvme_ctrlr *ctrlr, 650 struct spdk_nvme_intel_log_page_directory *log_page_directory) 651 { 652 if (log_page_directory == NULL) { 653 return; 654 } 655 656 assert(ctrlr->cdata.vid == SPDK_PCI_VID_INTEL); 657 658 ctrlr->log_page_supported[SPDK_NVME_INTEL_LOG_PAGE_DIRECTORY] = true; 659 660 if (log_page_directory->read_latency_log_len || 661 (ctrlr->quirks & NVME_INTEL_QUIRK_READ_LATENCY)) { 662 ctrlr->log_page_supported[SPDK_NVME_INTEL_LOG_READ_CMD_LATENCY] = true; 663 } 664 if (log_page_directory->write_latency_log_len || 665 (ctrlr->quirks & NVME_INTEL_QUIRK_WRITE_LATENCY)) { 666 ctrlr->log_page_supported[SPDK_NVME_INTEL_LOG_WRITE_CMD_LATENCY] = true; 667 } 668 if (log_page_directory->temperature_statistics_log_len) { 669 ctrlr->log_page_supported[SPDK_NVME_INTEL_LOG_TEMPERATURE] = true; 670 } 671 if (log_page_directory->smart_log_len) { 672 ctrlr->log_page_supported[SPDK_NVME_INTEL_LOG_SMART] = true; 673 } 674 if (log_page_directory->marketing_description_log_len) { 675 ctrlr->log_page_supported[SPDK_NVME_INTEL_MARKETING_DESCRIPTION] = true; 676 } 677 } 678 679 struct intel_log_pages_ctx { 680 struct spdk_nvme_intel_log_page_directory log_page_directory; 681 struct spdk_nvme_ctrlr *ctrlr; 682 }; 683 684 static void 685 nvme_ctrlr_set_intel_support_log_pages_done(void *arg, const struct spdk_nvme_cpl *cpl) 686 { 687 struct intel_log_pages_ctx *ctx = arg; 688 struct spdk_nvme_ctrlr *ctrlr = ctx->ctrlr; 689 690 if (!spdk_nvme_cpl_is_error(cpl)) { 691 nvme_ctrlr_construct_intel_support_log_page_list(ctrlr, &ctx->log_page_directory); 692 } 693 694 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_SUPPORTED_FEATURES, 695 ctrlr->opts.admin_timeout_ms); 696 free(ctx); 697 } 698 699 static int 700 nvme_ctrlr_set_intel_support_log_pages(struct spdk_nvme_ctrlr *ctrlr) 701 { 702 int rc = 0; 703 struct intel_log_pages_ctx *ctx; 704 705 ctx = calloc(1, sizeof(*ctx)); 706 if (!ctx) { 707 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_SUPPORTED_FEATURES, 708 ctrlr->opts.admin_timeout_ms); 709 return 0; 710 } 711 712 ctx->ctrlr = ctrlr; 713 714 rc = spdk_nvme_ctrlr_cmd_get_log_page(ctrlr, SPDK_NVME_INTEL_LOG_PAGE_DIRECTORY, 715 SPDK_NVME_GLOBAL_NS_TAG, &ctx->log_page_directory, 716 sizeof(struct spdk_nvme_intel_log_page_directory), 717 0, nvme_ctrlr_set_intel_support_log_pages_done, ctx); 718 if (rc != 0) { 719 free(ctx); 720 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_SUPPORTED_FEATURES, 721 ctrlr->opts.admin_timeout_ms); 722 return 0; 723 } 724 725 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_WAIT_FOR_SUPPORTED_INTEL_LOG_PAGES, 726 ctrlr->opts.admin_timeout_ms); 727 728 return 0; 729 } 730 731 static int 732 nvme_ctrlr_alloc_ana_log_page(struct spdk_nvme_ctrlr *ctrlr) 733 { 734 uint32_t ana_log_page_size; 735 736 ana_log_page_size = sizeof(struct spdk_nvme_ana_page) + ctrlr->cdata.nanagrpid * 737 sizeof(struct spdk_nvme_ana_group_descriptor) + ctrlr->active_ns_count * 738 sizeof(uint32_t); 739 740 /* Number of active namespaces may have changed. 741 * Check if ANA log page fits into existing buffer. 742 */ 743 if (ana_log_page_size > ctrlr->ana_log_page_size) { 744 void *new_buffer; 745 746 if (ctrlr->ana_log_page) { 747 new_buffer = realloc(ctrlr->ana_log_page, ana_log_page_size); 748 } else { 749 new_buffer = calloc(1, ana_log_page_size); 750 } 751 752 if (!new_buffer) { 753 NVME_CTRLR_ERRLOG(ctrlr, "could not allocate ANA log page buffer, size %u\n", 754 ana_log_page_size); 755 return -ENXIO; 756 } 757 758 ctrlr->ana_log_page = new_buffer; 759 if (ctrlr->copied_ana_desc) { 760 new_buffer = realloc(ctrlr->copied_ana_desc, ana_log_page_size); 761 } else { 762 new_buffer = calloc(1, ana_log_page_size); 763 } 764 765 if (!new_buffer) { 766 NVME_CTRLR_ERRLOG(ctrlr, "could not allocate a buffer to parse ANA descriptor, size %u\n", 767 ana_log_page_size); 768 return -ENOMEM; 769 } 770 771 ctrlr->copied_ana_desc = new_buffer; 772 ctrlr->ana_log_page_size = ana_log_page_size; 773 } 774 775 return 0; 776 } 777 778 static int 779 nvme_ctrlr_update_ana_log_page(struct spdk_nvme_ctrlr *ctrlr) 780 { 781 struct nvme_completion_poll_status *status; 782 int rc; 783 784 rc = nvme_ctrlr_alloc_ana_log_page(ctrlr); 785 if (rc != 0) { 786 return rc; 787 } 788 789 status = calloc(1, sizeof(*status)); 790 if (status == NULL) { 791 NVME_CTRLR_ERRLOG(ctrlr, "Failed to allocate status tracker\n"); 792 return -ENOMEM; 793 } 794 795 rc = spdk_nvme_ctrlr_cmd_get_log_page(ctrlr, SPDK_NVME_LOG_ASYMMETRIC_NAMESPACE_ACCESS, 796 SPDK_NVME_GLOBAL_NS_TAG, ctrlr->ana_log_page, 797 ctrlr->ana_log_page_size, 0, 798 nvme_completion_poll_cb, status); 799 if (rc != 0) { 800 free(status); 801 return rc; 802 } 803 804 if (nvme_wait_for_completion_robust_lock_timeout(ctrlr->adminq, status, &ctrlr->ctrlr_lock, 805 ctrlr->opts.admin_timeout_ms * 1000)) { 806 if (!status->timed_out) { 807 free(status); 808 } 809 return -EIO; 810 } 811 812 free(status); 813 return 0; 814 } 815 816 static int 817 nvme_ctrlr_update_ns_ana_states(const struct spdk_nvme_ana_group_descriptor *desc, 818 void *cb_arg) 819 { 820 struct spdk_nvme_ctrlr *ctrlr = cb_arg; 821 struct spdk_nvme_ns *ns; 822 uint32_t i, nsid; 823 824 for (i = 0; i < desc->num_of_nsid; i++) { 825 nsid = desc->nsid[i]; 826 if (nsid == 0 || nsid > ctrlr->cdata.nn) { 827 continue; 828 } 829 830 ns = spdk_nvme_ctrlr_get_ns(ctrlr, nsid); 831 assert(ns != NULL); 832 833 ns->ana_group_id = desc->ana_group_id; 834 ns->ana_state = desc->ana_state; 835 } 836 837 return 0; 838 } 839 840 int 841 nvme_ctrlr_parse_ana_log_page(struct spdk_nvme_ctrlr *ctrlr, 842 spdk_nvme_parse_ana_log_page_cb cb_fn, void *cb_arg) 843 { 844 struct spdk_nvme_ana_group_descriptor *copied_desc; 845 uint8_t *orig_desc; 846 uint32_t i, desc_size, copy_len; 847 int rc = 0; 848 849 if (ctrlr->ana_log_page == NULL) { 850 return -EINVAL; 851 } 852 853 copied_desc = ctrlr->copied_ana_desc; 854 855 orig_desc = (uint8_t *)ctrlr->ana_log_page + sizeof(struct spdk_nvme_ana_page); 856 copy_len = ctrlr->ana_log_page_size - sizeof(struct spdk_nvme_ana_page); 857 858 for (i = 0; i < ctrlr->ana_log_page->num_ana_group_desc; i++) { 859 memcpy(copied_desc, orig_desc, copy_len); 860 861 rc = cb_fn(copied_desc, cb_arg); 862 if (rc != 0) { 863 break; 864 } 865 866 desc_size = sizeof(struct spdk_nvme_ana_group_descriptor) + 867 copied_desc->num_of_nsid * sizeof(uint32_t); 868 orig_desc += desc_size; 869 copy_len -= desc_size; 870 } 871 872 return rc; 873 } 874 875 static int 876 nvme_ctrlr_set_supported_log_pages(struct spdk_nvme_ctrlr *ctrlr) 877 { 878 int rc = 0; 879 880 memset(ctrlr->log_page_supported, 0, sizeof(ctrlr->log_page_supported)); 881 /* Mandatory pages */ 882 ctrlr->log_page_supported[SPDK_NVME_LOG_ERROR] = true; 883 ctrlr->log_page_supported[SPDK_NVME_LOG_HEALTH_INFORMATION] = true; 884 ctrlr->log_page_supported[SPDK_NVME_LOG_FIRMWARE_SLOT] = true; 885 if (ctrlr->cdata.lpa.celp) { 886 ctrlr->log_page_supported[SPDK_NVME_LOG_COMMAND_EFFECTS_LOG] = true; 887 } 888 889 if (ctrlr->cdata.cmic.ana_reporting) { 890 ctrlr->log_page_supported[SPDK_NVME_LOG_ASYMMETRIC_NAMESPACE_ACCESS] = true; 891 if (!ctrlr->opts.disable_read_ana_log_page) { 892 rc = nvme_ctrlr_update_ana_log_page(ctrlr); 893 if (rc == 0) { 894 nvme_ctrlr_parse_ana_log_page(ctrlr, nvme_ctrlr_update_ns_ana_states, 895 ctrlr); 896 } 897 } 898 } 899 900 if (ctrlr->cdata.ctratt.bits.fdps) { 901 ctrlr->log_page_supported[SPDK_NVME_LOG_FDP_CONFIGURATIONS] = true; 902 ctrlr->log_page_supported[SPDK_NVME_LOG_RECLAIM_UNIT_HANDLE_USAGE] = true; 903 ctrlr->log_page_supported[SPDK_NVME_LOG_FDP_STATISTICS] = true; 904 ctrlr->log_page_supported[SPDK_NVME_LOG_FDP_EVENTS] = true; 905 } 906 907 if (ctrlr->cdata.vid == SPDK_PCI_VID_INTEL && 908 ctrlr->trid.trtype == SPDK_NVME_TRANSPORT_PCIE && 909 !(ctrlr->quirks & NVME_INTEL_QUIRK_NO_LOG_PAGES)) { 910 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_SUPPORTED_INTEL_LOG_PAGES, 911 ctrlr->opts.admin_timeout_ms); 912 913 } else { 914 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_SUPPORTED_FEATURES, 915 ctrlr->opts.admin_timeout_ms); 916 917 } 918 919 return rc; 920 } 921 922 static void 923 nvme_ctrlr_set_intel_supported_features(struct spdk_nvme_ctrlr *ctrlr) 924 { 925 ctrlr->feature_supported[SPDK_NVME_INTEL_FEAT_MAX_LBA] = true; 926 ctrlr->feature_supported[SPDK_NVME_INTEL_FEAT_NATIVE_MAX_LBA] = true; 927 ctrlr->feature_supported[SPDK_NVME_INTEL_FEAT_POWER_GOVERNOR_SETTING] = true; 928 ctrlr->feature_supported[SPDK_NVME_INTEL_FEAT_SMBUS_ADDRESS] = true; 929 ctrlr->feature_supported[SPDK_NVME_INTEL_FEAT_LED_PATTERN] = true; 930 ctrlr->feature_supported[SPDK_NVME_INTEL_FEAT_RESET_TIMED_WORKLOAD_COUNTERS] = true; 931 ctrlr->feature_supported[SPDK_NVME_INTEL_FEAT_LATENCY_TRACKING] = true; 932 } 933 934 static void 935 nvme_ctrlr_set_arbitration_feature(struct spdk_nvme_ctrlr *ctrlr) 936 { 937 uint32_t cdw11; 938 struct nvme_completion_poll_status *status; 939 940 if (ctrlr->opts.arbitration_burst == 0) { 941 return; 942 } 943 944 if (ctrlr->opts.arbitration_burst > 7) { 945 NVME_CTRLR_WARNLOG(ctrlr, "Valid arbitration burst values is from 0-7\n"); 946 return; 947 } 948 949 status = calloc(1, sizeof(*status)); 950 if (!status) { 951 NVME_CTRLR_ERRLOG(ctrlr, "Failed to allocate status tracker\n"); 952 return; 953 } 954 955 cdw11 = ctrlr->opts.arbitration_burst; 956 957 if (spdk_nvme_ctrlr_get_flags(ctrlr) & SPDK_NVME_CTRLR_WRR_SUPPORTED) { 958 cdw11 |= (uint32_t)ctrlr->opts.low_priority_weight << 8; 959 cdw11 |= (uint32_t)ctrlr->opts.medium_priority_weight << 16; 960 cdw11 |= (uint32_t)ctrlr->opts.high_priority_weight << 24; 961 } 962 963 if (spdk_nvme_ctrlr_cmd_set_feature(ctrlr, SPDK_NVME_FEAT_ARBITRATION, 964 cdw11, 0, NULL, 0, 965 nvme_completion_poll_cb, status) < 0) { 966 NVME_CTRLR_ERRLOG(ctrlr, "Set arbitration feature failed\n"); 967 free(status); 968 return; 969 } 970 971 if (nvme_wait_for_completion_timeout(ctrlr->adminq, status, 972 ctrlr->opts.admin_timeout_ms * 1000)) { 973 NVME_CTRLR_ERRLOG(ctrlr, "Timeout to set arbitration feature\n"); 974 } 975 976 if (!status->timed_out) { 977 free(status); 978 } 979 } 980 981 static void 982 nvme_ctrlr_set_supported_features(struct spdk_nvme_ctrlr *ctrlr) 983 { 984 memset(ctrlr->feature_supported, 0, sizeof(ctrlr->feature_supported)); 985 /* Mandatory features */ 986 ctrlr->feature_supported[SPDK_NVME_FEAT_ARBITRATION] = true; 987 ctrlr->feature_supported[SPDK_NVME_FEAT_POWER_MANAGEMENT] = true; 988 ctrlr->feature_supported[SPDK_NVME_FEAT_TEMPERATURE_THRESHOLD] = true; 989 ctrlr->feature_supported[SPDK_NVME_FEAT_ERROR_RECOVERY] = true; 990 ctrlr->feature_supported[SPDK_NVME_FEAT_NUMBER_OF_QUEUES] = true; 991 ctrlr->feature_supported[SPDK_NVME_FEAT_INTERRUPT_COALESCING] = true; 992 ctrlr->feature_supported[SPDK_NVME_FEAT_INTERRUPT_VECTOR_CONFIGURATION] = true; 993 ctrlr->feature_supported[SPDK_NVME_FEAT_WRITE_ATOMICITY] = true; 994 ctrlr->feature_supported[SPDK_NVME_FEAT_ASYNC_EVENT_CONFIGURATION] = true; 995 /* Optional features */ 996 if (ctrlr->cdata.vwc.present) { 997 ctrlr->feature_supported[SPDK_NVME_FEAT_VOLATILE_WRITE_CACHE] = true; 998 } 999 if (ctrlr->cdata.apsta.supported) { 1000 ctrlr->feature_supported[SPDK_NVME_FEAT_AUTONOMOUS_POWER_STATE_TRANSITION] = true; 1001 } 1002 if (ctrlr->cdata.hmpre) { 1003 ctrlr->feature_supported[SPDK_NVME_FEAT_HOST_MEM_BUFFER] = true; 1004 } 1005 if (ctrlr->cdata.vid == SPDK_PCI_VID_INTEL) { 1006 nvme_ctrlr_set_intel_supported_features(ctrlr); 1007 } 1008 1009 nvme_ctrlr_set_arbitration_feature(ctrlr); 1010 } 1011 1012 static void 1013 nvme_ctrlr_set_host_feature_done(void *arg, const struct spdk_nvme_cpl *cpl) 1014 { 1015 struct spdk_nvme_ctrlr *ctrlr = (struct spdk_nvme_ctrlr *)arg; 1016 1017 spdk_free(ctrlr->tmp_ptr); 1018 ctrlr->tmp_ptr = NULL; 1019 1020 if (spdk_nvme_cpl_is_error(cpl)) { 1021 NVME_CTRLR_ERRLOG(ctrlr, "Set host behavior support feature failed: SC %x SCT %x\n", 1022 cpl->status.sc, cpl->status.sct); 1023 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 1024 return; 1025 } 1026 1027 ctrlr->feature_supported[SPDK_NVME_FEAT_HOST_BEHAVIOR_SUPPORT] = true; 1028 1029 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_DB_BUF_CFG, 1030 ctrlr->opts.admin_timeout_ms); 1031 } 1032 1033 /* We do not want to do add synchronous operation anymore. 1034 * We set the Host Behavior Support feature asynchronousin in different states. 1035 */ 1036 static int 1037 nvme_ctrlr_set_host_feature(struct spdk_nvme_ctrlr *ctrlr) 1038 { 1039 struct spdk_nvme_host_behavior *host; 1040 int rc; 1041 1042 if (!ctrlr->cdata.ctratt.bits.elbas) { 1043 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_DB_BUF_CFG, 1044 ctrlr->opts.admin_timeout_ms); 1045 return 0; 1046 } 1047 1048 ctrlr->tmp_ptr = spdk_dma_zmalloc(sizeof(struct spdk_nvme_host_behavior), 4096, NULL); 1049 if (!ctrlr->tmp_ptr) { 1050 NVME_CTRLR_ERRLOG(ctrlr, "Failed to allocate host behavior support data\n"); 1051 rc = -ENOMEM; 1052 goto error; 1053 } 1054 1055 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_WAIT_FOR_SET_HOST_FEATURE, 1056 ctrlr->opts.admin_timeout_ms); 1057 1058 host = ctrlr->tmp_ptr; 1059 1060 host->lbafee = 1; 1061 1062 rc = spdk_nvme_ctrlr_cmd_set_feature(ctrlr, SPDK_NVME_FEAT_HOST_BEHAVIOR_SUPPORT, 1063 0, 0, host, sizeof(struct spdk_nvme_host_behavior), 1064 nvme_ctrlr_set_host_feature_done, ctrlr); 1065 if (rc != 0) { 1066 NVME_CTRLR_ERRLOG(ctrlr, "Set host behavior support feature failed: %d\n", rc); 1067 goto error; 1068 } 1069 1070 return 0; 1071 1072 error: 1073 spdk_free(ctrlr->tmp_ptr); 1074 ctrlr->tmp_ptr = NULL; 1075 1076 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 1077 return rc; 1078 } 1079 1080 bool 1081 spdk_nvme_ctrlr_is_failed(struct spdk_nvme_ctrlr *ctrlr) 1082 { 1083 return ctrlr->is_failed; 1084 } 1085 1086 void 1087 nvme_ctrlr_fail(struct spdk_nvme_ctrlr *ctrlr, bool hot_remove) 1088 { 1089 /* 1090 * Set the flag here and leave the work failure of qpairs to 1091 * spdk_nvme_qpair_process_completions(). 1092 */ 1093 if (hot_remove) { 1094 ctrlr->is_removed = true; 1095 } 1096 1097 if (ctrlr->is_failed) { 1098 NVME_CTRLR_NOTICELOG(ctrlr, "already in failed state\n"); 1099 return; 1100 } 1101 1102 if (ctrlr->is_disconnecting) { 1103 NVME_CTRLR_DEBUGLOG(ctrlr, "already disconnecting\n"); 1104 return; 1105 } 1106 1107 ctrlr->is_failed = true; 1108 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 1109 nvme_transport_ctrlr_disconnect_qpair(ctrlr, ctrlr->adminq); 1110 NVME_CTRLR_ERRLOG(ctrlr, "in failed state.\n"); 1111 } 1112 1113 /** 1114 * This public API function will try to take the controller lock. 1115 * Any private functions being called from a thread already holding 1116 * the ctrlr lock should call nvme_ctrlr_fail directly. 1117 */ 1118 void 1119 spdk_nvme_ctrlr_fail(struct spdk_nvme_ctrlr *ctrlr) 1120 { 1121 nvme_ctrlr_lock(ctrlr); 1122 nvme_ctrlr_fail(ctrlr, false); 1123 nvme_ctrlr_unlock(ctrlr); 1124 } 1125 1126 static void 1127 nvme_ctrlr_shutdown_set_cc_done(void *_ctx, uint64_t value, const struct spdk_nvme_cpl *cpl) 1128 { 1129 struct nvme_ctrlr_detach_ctx *ctx = _ctx; 1130 struct spdk_nvme_ctrlr *ctrlr = ctx->ctrlr; 1131 1132 if (spdk_nvme_cpl_is_error(cpl)) { 1133 NVME_CTRLR_ERRLOG(ctrlr, "Failed to write CC.SHN\n"); 1134 ctx->shutdown_complete = true; 1135 return; 1136 } 1137 1138 if (ctrlr->opts.no_shn_notification) { 1139 ctx->shutdown_complete = true; 1140 return; 1141 } 1142 1143 /* 1144 * The NVMe specification defines RTD3E to be the time between 1145 * setting SHN = 1 until the controller will set SHST = 10b. 1146 * If the device doesn't report RTD3 entry latency, or if it 1147 * reports RTD3 entry latency less than 10 seconds, pick 1148 * 10 seconds as a reasonable amount of time to 1149 * wait before proceeding. 1150 */ 1151 NVME_CTRLR_DEBUGLOG(ctrlr, "RTD3E = %" PRIu32 " us\n", ctrlr->cdata.rtd3e); 1152 ctx->shutdown_timeout_ms = SPDK_CEIL_DIV(ctrlr->cdata.rtd3e, 1000); 1153 ctx->shutdown_timeout_ms = spdk_max(ctx->shutdown_timeout_ms, 10000); 1154 NVME_CTRLR_DEBUGLOG(ctrlr, "shutdown timeout = %" PRIu32 " ms\n", ctx->shutdown_timeout_ms); 1155 1156 ctx->shutdown_start_tsc = spdk_get_ticks(); 1157 ctx->state = NVME_CTRLR_DETACH_CHECK_CSTS; 1158 } 1159 1160 static void 1161 nvme_ctrlr_shutdown_get_cc_done(void *_ctx, uint64_t value, const struct spdk_nvme_cpl *cpl) 1162 { 1163 struct nvme_ctrlr_detach_ctx *ctx = _ctx; 1164 struct spdk_nvme_ctrlr *ctrlr = ctx->ctrlr; 1165 union spdk_nvme_cc_register cc; 1166 int rc; 1167 1168 if (spdk_nvme_cpl_is_error(cpl)) { 1169 NVME_CTRLR_ERRLOG(ctrlr, "Failed to read the CC register\n"); 1170 ctx->shutdown_complete = true; 1171 return; 1172 } 1173 1174 assert(value <= UINT32_MAX); 1175 cc.raw = (uint32_t)value; 1176 1177 if (ctrlr->opts.no_shn_notification) { 1178 NVME_CTRLR_INFOLOG(ctrlr, "Disable SSD without shutdown notification\n"); 1179 if (cc.bits.en == 0) { 1180 ctx->shutdown_complete = true; 1181 return; 1182 } 1183 1184 cc.bits.en = 0; 1185 } else { 1186 cc.bits.shn = SPDK_NVME_SHN_NORMAL; 1187 } 1188 1189 rc = nvme_ctrlr_set_cc_async(ctrlr, cc.raw, nvme_ctrlr_shutdown_set_cc_done, ctx); 1190 if (rc != 0) { 1191 NVME_CTRLR_ERRLOG(ctrlr, "Failed to write CC.SHN\n"); 1192 ctx->shutdown_complete = true; 1193 } 1194 } 1195 1196 static void 1197 nvme_ctrlr_shutdown_async(struct spdk_nvme_ctrlr *ctrlr, 1198 struct nvme_ctrlr_detach_ctx *ctx) 1199 { 1200 int rc; 1201 1202 if (ctrlr->is_removed) { 1203 ctx->shutdown_complete = true; 1204 return; 1205 } 1206 1207 if (ctrlr->adminq == NULL || 1208 ctrlr->adminq->transport_failure_reason != SPDK_NVME_QPAIR_FAILURE_NONE) { 1209 NVME_CTRLR_INFOLOG(ctrlr, "Adminq is not connected.\n"); 1210 ctx->shutdown_complete = true; 1211 return; 1212 } 1213 1214 ctx->state = NVME_CTRLR_DETACH_SET_CC; 1215 rc = nvme_ctrlr_get_cc_async(ctrlr, nvme_ctrlr_shutdown_get_cc_done, ctx); 1216 if (rc != 0) { 1217 NVME_CTRLR_ERRLOG(ctrlr, "Failed to read the CC register\n"); 1218 ctx->shutdown_complete = true; 1219 } 1220 } 1221 1222 static void 1223 nvme_ctrlr_shutdown_get_csts_done(void *_ctx, uint64_t value, const struct spdk_nvme_cpl *cpl) 1224 { 1225 struct nvme_ctrlr_detach_ctx *ctx = _ctx; 1226 1227 if (spdk_nvme_cpl_is_error(cpl)) { 1228 NVME_CTRLR_ERRLOG(ctx->ctrlr, "Failed to read the CSTS register\n"); 1229 ctx->shutdown_complete = true; 1230 return; 1231 } 1232 1233 assert(value <= UINT32_MAX); 1234 ctx->csts.raw = (uint32_t)value; 1235 ctx->state = NVME_CTRLR_DETACH_GET_CSTS_DONE; 1236 } 1237 1238 static int 1239 nvme_ctrlr_shutdown_poll_async(struct spdk_nvme_ctrlr *ctrlr, 1240 struct nvme_ctrlr_detach_ctx *ctx) 1241 { 1242 union spdk_nvme_csts_register csts; 1243 uint32_t ms_waited; 1244 1245 switch (ctx->state) { 1246 case NVME_CTRLR_DETACH_SET_CC: 1247 case NVME_CTRLR_DETACH_GET_CSTS: 1248 /* We're still waiting for the register operation to complete */ 1249 spdk_nvme_qpair_process_completions(ctrlr->adminq, 0); 1250 return -EAGAIN; 1251 1252 case NVME_CTRLR_DETACH_CHECK_CSTS: 1253 ctx->state = NVME_CTRLR_DETACH_GET_CSTS; 1254 if (nvme_ctrlr_get_csts_async(ctrlr, nvme_ctrlr_shutdown_get_csts_done, ctx)) { 1255 NVME_CTRLR_ERRLOG(ctrlr, "Failed to read the CSTS register\n"); 1256 return -EIO; 1257 } 1258 return -EAGAIN; 1259 1260 case NVME_CTRLR_DETACH_GET_CSTS_DONE: 1261 ctx->state = NVME_CTRLR_DETACH_CHECK_CSTS; 1262 break; 1263 1264 default: 1265 assert(0 && "Should never happen"); 1266 return -EINVAL; 1267 } 1268 1269 ms_waited = (spdk_get_ticks() - ctx->shutdown_start_tsc) * 1000 / spdk_get_ticks_hz(); 1270 csts.raw = ctx->csts.raw; 1271 1272 if (csts.bits.shst == SPDK_NVME_SHST_COMPLETE) { 1273 NVME_CTRLR_DEBUGLOG(ctrlr, "shutdown complete in %u milliseconds\n", ms_waited); 1274 return 0; 1275 } 1276 1277 if (ms_waited < ctx->shutdown_timeout_ms) { 1278 return -EAGAIN; 1279 } 1280 1281 NVME_CTRLR_ERRLOG(ctrlr, "did not shutdown within %u milliseconds\n", 1282 ctx->shutdown_timeout_ms); 1283 if (ctrlr->quirks & NVME_QUIRK_SHST_COMPLETE) { 1284 NVME_CTRLR_ERRLOG(ctrlr, "likely due to shutdown handling in the VMWare emulated NVMe SSD\n"); 1285 } 1286 1287 return 0; 1288 } 1289 1290 static inline uint64_t 1291 nvme_ctrlr_get_ready_timeout(struct spdk_nvme_ctrlr *ctrlr) 1292 { 1293 return ctrlr->cap.bits.to * 500; 1294 } 1295 1296 static void 1297 nvme_ctrlr_set_cc_en_done(void *ctx, uint64_t value, const struct spdk_nvme_cpl *cpl) 1298 { 1299 struct spdk_nvme_ctrlr *ctrlr = ctx; 1300 1301 if (spdk_nvme_cpl_is_error(cpl)) { 1302 NVME_CTRLR_ERRLOG(ctrlr, "Failed to set the CC register\n"); 1303 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 1304 return; 1305 } 1306 1307 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ENABLE_WAIT_FOR_READY_1, 1308 nvme_ctrlr_get_ready_timeout(ctrlr)); 1309 } 1310 1311 static int 1312 nvme_ctrlr_enable(struct spdk_nvme_ctrlr *ctrlr) 1313 { 1314 union spdk_nvme_cc_register cc; 1315 int rc; 1316 1317 rc = nvme_transport_ctrlr_enable(ctrlr); 1318 if (rc != 0) { 1319 NVME_CTRLR_ERRLOG(ctrlr, "transport ctrlr_enable failed\n"); 1320 return rc; 1321 } 1322 1323 cc.raw = ctrlr->process_init_cc.raw; 1324 if (cc.bits.en != 0) { 1325 NVME_CTRLR_ERRLOG(ctrlr, "called with CC.EN = 1\n"); 1326 return -EINVAL; 1327 } 1328 1329 cc.bits.en = 1; 1330 cc.bits.css = 0; 1331 cc.bits.shn = 0; 1332 cc.bits.iosqes = 6; /* SQ entry size == 64 == 2^6 */ 1333 cc.bits.iocqes = 4; /* CQ entry size == 16 == 2^4 */ 1334 1335 /* Page size is 2 ^ (12 + mps). */ 1336 cc.bits.mps = spdk_u32log2(ctrlr->page_size) - 12; 1337 1338 /* 1339 * Since NVMe 1.0, a controller should have at least one bit set in CAP.CSS. 1340 * A controller that does not have any bit set in CAP.CSS is not spec compliant. 1341 * Try to support such a controller regardless. 1342 */ 1343 if (ctrlr->cap.bits.css == 0) { 1344 NVME_CTRLR_INFOLOG(ctrlr, "Drive reports no command sets supported. Assuming NVM is supported.\n"); 1345 ctrlr->cap.bits.css = SPDK_NVME_CAP_CSS_NVM; 1346 } 1347 1348 /* 1349 * If the user did not explicitly request a command set, or supplied a value larger than 1350 * what can be saved in CC.CSS, use the most reasonable default. 1351 */ 1352 if (ctrlr->opts.command_set >= CHAR_BIT) { 1353 if (ctrlr->cap.bits.css & SPDK_NVME_CAP_CSS_IOCS) { 1354 ctrlr->opts.command_set = SPDK_NVME_CC_CSS_IOCS; 1355 } else if (ctrlr->cap.bits.css & SPDK_NVME_CAP_CSS_NVM) { 1356 ctrlr->opts.command_set = SPDK_NVME_CC_CSS_NVM; 1357 } else if (ctrlr->cap.bits.css & SPDK_NVME_CAP_CSS_NOIO) { 1358 /* Technically we should respond with CC_CSS_NOIO in 1359 * this case, but we use NVM instead to work around 1360 * buggy targets and to match Linux driver behavior. 1361 */ 1362 ctrlr->opts.command_set = SPDK_NVME_CC_CSS_NVM; 1363 } else { 1364 /* Invalid supported bits detected, falling back to NVM. */ 1365 ctrlr->opts.command_set = SPDK_NVME_CC_CSS_NVM; 1366 } 1367 } 1368 1369 /* Verify that the selected command set is supported by the controller. */ 1370 if (!(ctrlr->cap.bits.css & (1u << ctrlr->opts.command_set))) { 1371 NVME_CTRLR_DEBUGLOG(ctrlr, "Requested I/O command set %u but supported mask is 0x%x\n", 1372 ctrlr->opts.command_set, ctrlr->cap.bits.css); 1373 NVME_CTRLR_DEBUGLOG(ctrlr, "Falling back to NVM. Assuming NVM is supported.\n"); 1374 ctrlr->opts.command_set = SPDK_NVME_CC_CSS_NVM; 1375 } 1376 1377 cc.bits.css = ctrlr->opts.command_set; 1378 1379 switch (ctrlr->opts.arb_mechanism) { 1380 case SPDK_NVME_CC_AMS_RR: 1381 break; 1382 case SPDK_NVME_CC_AMS_WRR: 1383 if (SPDK_NVME_CAP_AMS_WRR & ctrlr->cap.bits.ams) { 1384 break; 1385 } 1386 return -EINVAL; 1387 case SPDK_NVME_CC_AMS_VS: 1388 if (SPDK_NVME_CAP_AMS_VS & ctrlr->cap.bits.ams) { 1389 break; 1390 } 1391 return -EINVAL; 1392 default: 1393 return -EINVAL; 1394 } 1395 1396 cc.bits.ams = ctrlr->opts.arb_mechanism; 1397 ctrlr->process_init_cc.raw = cc.raw; 1398 1399 if (nvme_ctrlr_set_cc_async(ctrlr, cc.raw, nvme_ctrlr_set_cc_en_done, ctrlr)) { 1400 NVME_CTRLR_ERRLOG(ctrlr, "set_cc() failed\n"); 1401 return -EIO; 1402 } 1403 1404 return 0; 1405 } 1406 1407 static const char * 1408 nvme_ctrlr_state_string(enum nvme_ctrlr_state state) 1409 { 1410 switch (state) { 1411 case NVME_CTRLR_STATE_INIT_DELAY: 1412 return "delay init"; 1413 case NVME_CTRLR_STATE_CONNECT_ADMINQ: 1414 return "connect adminq"; 1415 case NVME_CTRLR_STATE_WAIT_FOR_CONNECT_ADMINQ: 1416 return "wait for connect adminq"; 1417 case NVME_CTRLR_STATE_READ_VS: 1418 return "read vs"; 1419 case NVME_CTRLR_STATE_READ_VS_WAIT_FOR_VS: 1420 return "read vs wait for vs"; 1421 case NVME_CTRLR_STATE_READ_CAP: 1422 return "read cap"; 1423 case NVME_CTRLR_STATE_READ_CAP_WAIT_FOR_CAP: 1424 return "read cap wait for cap"; 1425 case NVME_CTRLR_STATE_CHECK_EN: 1426 return "check en"; 1427 case NVME_CTRLR_STATE_CHECK_EN_WAIT_FOR_CC: 1428 return "check en wait for cc"; 1429 case NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_1: 1430 return "disable and wait for CSTS.RDY = 1"; 1431 case NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_1_WAIT_FOR_CSTS: 1432 return "disable and wait for CSTS.RDY = 1 reg"; 1433 case NVME_CTRLR_STATE_SET_EN_0: 1434 return "set CC.EN = 0"; 1435 case NVME_CTRLR_STATE_SET_EN_0_WAIT_FOR_CC: 1436 return "set CC.EN = 0 wait for cc"; 1437 case NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0: 1438 return "disable and wait for CSTS.RDY = 0"; 1439 case NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0_WAIT_FOR_CSTS: 1440 return "disable and wait for CSTS.RDY = 0 reg"; 1441 case NVME_CTRLR_STATE_DISABLED: 1442 return "controller is disabled"; 1443 case NVME_CTRLR_STATE_ENABLE: 1444 return "enable controller by writing CC.EN = 1"; 1445 case NVME_CTRLR_STATE_ENABLE_WAIT_FOR_CC: 1446 return "enable controller by writing CC.EN = 1 reg"; 1447 case NVME_CTRLR_STATE_ENABLE_WAIT_FOR_READY_1: 1448 return "wait for CSTS.RDY = 1"; 1449 case NVME_CTRLR_STATE_ENABLE_WAIT_FOR_READY_1_WAIT_FOR_CSTS: 1450 return "wait for CSTS.RDY = 1 reg"; 1451 case NVME_CTRLR_STATE_RESET_ADMIN_QUEUE: 1452 return "reset admin queue"; 1453 case NVME_CTRLR_STATE_IDENTIFY: 1454 return "identify controller"; 1455 case NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY: 1456 return "wait for identify controller"; 1457 case NVME_CTRLR_STATE_CONFIGURE_AER: 1458 return "configure AER"; 1459 case NVME_CTRLR_STATE_WAIT_FOR_CONFIGURE_AER: 1460 return "wait for configure aer"; 1461 case NVME_CTRLR_STATE_SET_KEEP_ALIVE_TIMEOUT: 1462 return "set keep alive timeout"; 1463 case NVME_CTRLR_STATE_WAIT_FOR_KEEP_ALIVE_TIMEOUT: 1464 return "wait for set keep alive timeout"; 1465 case NVME_CTRLR_STATE_IDENTIFY_IOCS_SPECIFIC: 1466 return "identify controller iocs specific"; 1467 case NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_IOCS_SPECIFIC: 1468 return "wait for identify controller iocs specific"; 1469 case NVME_CTRLR_STATE_GET_ZNS_CMD_EFFECTS_LOG: 1470 return "get zns cmd and effects log page"; 1471 case NVME_CTRLR_STATE_WAIT_FOR_GET_ZNS_CMD_EFFECTS_LOG: 1472 return "wait for get zns cmd and effects log page"; 1473 case NVME_CTRLR_STATE_SET_NUM_QUEUES: 1474 return "set number of queues"; 1475 case NVME_CTRLR_STATE_WAIT_FOR_SET_NUM_QUEUES: 1476 return "wait for set number of queues"; 1477 case NVME_CTRLR_STATE_IDENTIFY_ACTIVE_NS: 1478 return "identify active ns"; 1479 case NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_ACTIVE_NS: 1480 return "wait for identify active ns"; 1481 case NVME_CTRLR_STATE_IDENTIFY_NS: 1482 return "identify ns"; 1483 case NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_NS: 1484 return "wait for identify ns"; 1485 case NVME_CTRLR_STATE_IDENTIFY_ID_DESCS: 1486 return "identify namespace id descriptors"; 1487 case NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_ID_DESCS: 1488 return "wait for identify namespace id descriptors"; 1489 case NVME_CTRLR_STATE_IDENTIFY_NS_IOCS_SPECIFIC: 1490 return "identify ns iocs specific"; 1491 case NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_NS_IOCS_SPECIFIC: 1492 return "wait for identify ns iocs specific"; 1493 case NVME_CTRLR_STATE_SET_SUPPORTED_LOG_PAGES: 1494 return "set supported log pages"; 1495 case NVME_CTRLR_STATE_SET_SUPPORTED_INTEL_LOG_PAGES: 1496 return "set supported INTEL log pages"; 1497 case NVME_CTRLR_STATE_WAIT_FOR_SUPPORTED_INTEL_LOG_PAGES: 1498 return "wait for supported INTEL log pages"; 1499 case NVME_CTRLR_STATE_SET_SUPPORTED_FEATURES: 1500 return "set supported features"; 1501 case NVME_CTRLR_STATE_SET_HOST_FEATURE: 1502 return "set host behavior support feature"; 1503 case NVME_CTRLR_STATE_WAIT_FOR_SET_HOST_FEATURE: 1504 return "wait for set host behavior support feature"; 1505 case NVME_CTRLR_STATE_SET_DB_BUF_CFG: 1506 return "set doorbell buffer config"; 1507 case NVME_CTRLR_STATE_WAIT_FOR_DB_BUF_CFG: 1508 return "wait for doorbell buffer config"; 1509 case NVME_CTRLR_STATE_SET_HOST_ID: 1510 return "set host ID"; 1511 case NVME_CTRLR_STATE_WAIT_FOR_HOST_ID: 1512 return "wait for set host ID"; 1513 case NVME_CTRLR_STATE_TRANSPORT_READY: 1514 return "transport ready"; 1515 case NVME_CTRLR_STATE_READY: 1516 return "ready"; 1517 case NVME_CTRLR_STATE_ERROR: 1518 return "error"; 1519 case NVME_CTRLR_STATE_DISCONNECTED: 1520 return "disconnected"; 1521 } 1522 return "unknown"; 1523 }; 1524 1525 static void 1526 _nvme_ctrlr_set_state(struct spdk_nvme_ctrlr *ctrlr, enum nvme_ctrlr_state state, 1527 uint64_t timeout_in_ms, bool quiet) 1528 { 1529 uint64_t ticks_per_ms, timeout_in_ticks, now_ticks; 1530 1531 ctrlr->state = state; 1532 if (timeout_in_ms == NVME_TIMEOUT_KEEP_EXISTING) { 1533 if (!quiet) { 1534 NVME_CTRLR_DEBUGLOG(ctrlr, "setting state to %s (keeping existing timeout)\n", 1535 nvme_ctrlr_state_string(ctrlr->state)); 1536 } 1537 return; 1538 } 1539 1540 if (timeout_in_ms == NVME_TIMEOUT_INFINITE) { 1541 goto inf; 1542 } 1543 1544 ticks_per_ms = spdk_get_ticks_hz() / 1000; 1545 if (timeout_in_ms > UINT64_MAX / ticks_per_ms) { 1546 NVME_CTRLR_ERRLOG(ctrlr, 1547 "Specified timeout would cause integer overflow. Defaulting to no timeout.\n"); 1548 goto inf; 1549 } 1550 1551 now_ticks = spdk_get_ticks(); 1552 timeout_in_ticks = timeout_in_ms * ticks_per_ms; 1553 if (timeout_in_ticks > UINT64_MAX - now_ticks) { 1554 NVME_CTRLR_ERRLOG(ctrlr, 1555 "Specified timeout would cause integer overflow. Defaulting to no timeout.\n"); 1556 goto inf; 1557 } 1558 1559 ctrlr->state_timeout_tsc = timeout_in_ticks + now_ticks; 1560 if (!quiet) { 1561 NVME_CTRLR_DEBUGLOG(ctrlr, "setting state to %s (timeout %" PRIu64 " ms)\n", 1562 nvme_ctrlr_state_string(ctrlr->state), timeout_in_ms); 1563 } 1564 return; 1565 inf: 1566 if (!quiet) { 1567 NVME_CTRLR_DEBUGLOG(ctrlr, "setting state to %s (no timeout)\n", 1568 nvme_ctrlr_state_string(ctrlr->state)); 1569 } 1570 ctrlr->state_timeout_tsc = NVME_TIMEOUT_INFINITE; 1571 } 1572 1573 static void 1574 nvme_ctrlr_set_state(struct spdk_nvme_ctrlr *ctrlr, enum nvme_ctrlr_state state, 1575 uint64_t timeout_in_ms) 1576 { 1577 _nvme_ctrlr_set_state(ctrlr, state, timeout_in_ms, false); 1578 } 1579 1580 static void 1581 nvme_ctrlr_set_state_quiet(struct spdk_nvme_ctrlr *ctrlr, enum nvme_ctrlr_state state, 1582 uint64_t timeout_in_ms) 1583 { 1584 _nvme_ctrlr_set_state(ctrlr, state, timeout_in_ms, true); 1585 } 1586 1587 static void 1588 nvme_ctrlr_free_zns_specific_data(struct spdk_nvme_ctrlr *ctrlr) 1589 { 1590 spdk_free(ctrlr->cdata_zns); 1591 ctrlr->cdata_zns = NULL; 1592 } 1593 1594 static void 1595 nvme_ctrlr_free_iocs_specific_data(struct spdk_nvme_ctrlr *ctrlr) 1596 { 1597 nvme_ctrlr_free_zns_specific_data(ctrlr); 1598 } 1599 1600 static void 1601 nvme_ctrlr_free_doorbell_buffer(struct spdk_nvme_ctrlr *ctrlr) 1602 { 1603 if (ctrlr->shadow_doorbell) { 1604 spdk_free(ctrlr->shadow_doorbell); 1605 ctrlr->shadow_doorbell = NULL; 1606 } 1607 1608 if (ctrlr->eventidx) { 1609 spdk_free(ctrlr->eventidx); 1610 ctrlr->eventidx = NULL; 1611 } 1612 } 1613 1614 static void 1615 nvme_ctrlr_set_doorbell_buffer_config_done(void *arg, const struct spdk_nvme_cpl *cpl) 1616 { 1617 struct spdk_nvme_ctrlr *ctrlr = (struct spdk_nvme_ctrlr *)arg; 1618 1619 if (spdk_nvme_cpl_is_error(cpl)) { 1620 NVME_CTRLR_WARNLOG(ctrlr, "Doorbell buffer config failed\n"); 1621 } else { 1622 NVME_CTRLR_INFOLOG(ctrlr, "Doorbell buffer config enabled\n"); 1623 } 1624 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_HOST_ID, 1625 ctrlr->opts.admin_timeout_ms); 1626 } 1627 1628 static int 1629 nvme_ctrlr_set_doorbell_buffer_config(struct spdk_nvme_ctrlr *ctrlr) 1630 { 1631 int rc = 0; 1632 uint64_t prp1, prp2, len; 1633 1634 if (!ctrlr->cdata.oacs.doorbell_buffer_config) { 1635 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_HOST_ID, 1636 ctrlr->opts.admin_timeout_ms); 1637 return 0; 1638 } 1639 1640 if (ctrlr->trid.trtype != SPDK_NVME_TRANSPORT_PCIE) { 1641 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_HOST_ID, 1642 ctrlr->opts.admin_timeout_ms); 1643 return 0; 1644 } 1645 1646 /* only 1 page size for doorbell buffer */ 1647 ctrlr->shadow_doorbell = spdk_zmalloc(ctrlr->page_size, ctrlr->page_size, 1648 NULL, SPDK_ENV_LCORE_ID_ANY, 1649 SPDK_MALLOC_DMA | SPDK_MALLOC_SHARE); 1650 if (ctrlr->shadow_doorbell == NULL) { 1651 rc = -ENOMEM; 1652 goto error; 1653 } 1654 1655 len = ctrlr->page_size; 1656 prp1 = spdk_vtophys(ctrlr->shadow_doorbell, &len); 1657 if (prp1 == SPDK_VTOPHYS_ERROR || len != ctrlr->page_size) { 1658 rc = -EFAULT; 1659 goto error; 1660 } 1661 1662 ctrlr->eventidx = spdk_zmalloc(ctrlr->page_size, ctrlr->page_size, 1663 NULL, SPDK_ENV_LCORE_ID_ANY, 1664 SPDK_MALLOC_DMA | SPDK_MALLOC_SHARE); 1665 if (ctrlr->eventidx == NULL) { 1666 rc = -ENOMEM; 1667 goto error; 1668 } 1669 1670 len = ctrlr->page_size; 1671 prp2 = spdk_vtophys(ctrlr->eventidx, &len); 1672 if (prp2 == SPDK_VTOPHYS_ERROR || len != ctrlr->page_size) { 1673 rc = -EFAULT; 1674 goto error; 1675 } 1676 1677 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_WAIT_FOR_DB_BUF_CFG, 1678 ctrlr->opts.admin_timeout_ms); 1679 1680 rc = nvme_ctrlr_cmd_doorbell_buffer_config(ctrlr, prp1, prp2, 1681 nvme_ctrlr_set_doorbell_buffer_config_done, ctrlr); 1682 if (rc != 0) { 1683 goto error; 1684 } 1685 1686 return 0; 1687 1688 error: 1689 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 1690 nvme_ctrlr_free_doorbell_buffer(ctrlr); 1691 return rc; 1692 } 1693 1694 void 1695 nvme_ctrlr_abort_queued_aborts(struct spdk_nvme_ctrlr *ctrlr) 1696 { 1697 struct nvme_request *req, *tmp; 1698 struct spdk_nvme_cpl cpl = {}; 1699 1700 cpl.status.sc = SPDK_NVME_SC_ABORTED_SQ_DELETION; 1701 cpl.status.sct = SPDK_NVME_SCT_GENERIC; 1702 1703 STAILQ_FOREACH_SAFE(req, &ctrlr->queued_aborts, stailq, tmp) { 1704 STAILQ_REMOVE_HEAD(&ctrlr->queued_aborts, stailq); 1705 ctrlr->outstanding_aborts++; 1706 1707 nvme_complete_request(req->cb_fn, req->cb_arg, req->qpair, req, &cpl); 1708 } 1709 } 1710 1711 static int 1712 nvme_ctrlr_disconnect(struct spdk_nvme_ctrlr *ctrlr) 1713 { 1714 if (ctrlr->is_resetting || ctrlr->is_removed) { 1715 /* 1716 * Controller is already resetting or has been removed. Return 1717 * immediately since there is no need to kick off another 1718 * reset in these cases. 1719 */ 1720 return ctrlr->is_resetting ? -EBUSY : -ENXIO; 1721 } 1722 1723 ctrlr->is_resetting = true; 1724 ctrlr->is_failed = false; 1725 ctrlr->is_disconnecting = true; 1726 ctrlr->prepare_for_reset = true; 1727 1728 NVME_CTRLR_NOTICELOG(ctrlr, "resetting controller\n"); 1729 1730 /* Disable keep-alive, it'll be re-enabled as part of the init process */ 1731 ctrlr->keep_alive_interval_ticks = 0; 1732 1733 /* Abort all of the queued abort requests */ 1734 nvme_ctrlr_abort_queued_aborts(ctrlr); 1735 1736 nvme_transport_admin_qpair_abort_aers(ctrlr->adminq); 1737 1738 ctrlr->adminq->transport_failure_reason = SPDK_NVME_QPAIR_FAILURE_LOCAL; 1739 nvme_transport_ctrlr_disconnect_qpair(ctrlr, ctrlr->adminq); 1740 1741 return 0; 1742 } 1743 1744 static void 1745 nvme_ctrlr_disconnect_done(struct spdk_nvme_ctrlr *ctrlr) 1746 { 1747 assert(ctrlr->is_failed == false); 1748 ctrlr->is_disconnecting = false; 1749 1750 /* Doorbell buffer config is invalid during reset */ 1751 nvme_ctrlr_free_doorbell_buffer(ctrlr); 1752 1753 /* I/O Command Set Specific Identify Controller data is invalidated during reset */ 1754 nvme_ctrlr_free_iocs_specific_data(ctrlr); 1755 1756 spdk_bit_array_free(&ctrlr->free_io_qids); 1757 1758 /* Set the state back to DISCONNECTED to cause a full hardware reset. */ 1759 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_DISCONNECTED, NVME_TIMEOUT_INFINITE); 1760 } 1761 1762 int 1763 spdk_nvme_ctrlr_disconnect(struct spdk_nvme_ctrlr *ctrlr) 1764 { 1765 int rc; 1766 1767 nvme_ctrlr_lock(ctrlr); 1768 rc = nvme_ctrlr_disconnect(ctrlr); 1769 nvme_ctrlr_unlock(ctrlr); 1770 1771 return rc; 1772 } 1773 1774 void 1775 spdk_nvme_ctrlr_reconnect_async(struct spdk_nvme_ctrlr *ctrlr) 1776 { 1777 nvme_ctrlr_lock(ctrlr); 1778 1779 ctrlr->prepare_for_reset = false; 1780 1781 /* Set the state back to INIT to cause a full hardware reset. */ 1782 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_INIT, NVME_TIMEOUT_INFINITE); 1783 1784 /* Return without releasing ctrlr_lock. ctrlr_lock will be released when 1785 * spdk_nvme_ctrlr_reset_poll_async() returns 0. 1786 */ 1787 } 1788 1789 int 1790 nvme_ctrlr_reinitialize_io_qpair(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair) 1791 { 1792 bool async; 1793 int rc; 1794 1795 if (nvme_ctrlr_get_current_process(ctrlr) != qpair->active_proc || 1796 spdk_nvme_ctrlr_is_fabrics(ctrlr) || nvme_qpair_is_admin_queue(qpair)) { 1797 assert(false); 1798 return -EINVAL; 1799 } 1800 1801 /* Force a synchronous connect. */ 1802 async = qpair->async; 1803 qpair->async = false; 1804 rc = nvme_transport_ctrlr_connect_qpair(ctrlr, qpair); 1805 qpair->async = async; 1806 1807 if (rc != 0) { 1808 qpair->transport_failure_reason = SPDK_NVME_QPAIR_FAILURE_LOCAL; 1809 } 1810 1811 return rc; 1812 } 1813 1814 /** 1815 * This function will be called when the controller is being reinitialized. 1816 * Note: the ctrlr_lock must be held when calling this function. 1817 */ 1818 int 1819 spdk_nvme_ctrlr_reconnect_poll_async(struct spdk_nvme_ctrlr *ctrlr) 1820 { 1821 struct spdk_nvme_ns *ns, *tmp_ns; 1822 struct spdk_nvme_qpair *qpair; 1823 int rc = 0, rc_tmp = 0; 1824 1825 if (nvme_ctrlr_process_init(ctrlr) != 0) { 1826 NVME_CTRLR_ERRLOG(ctrlr, "controller reinitialization failed\n"); 1827 rc = -1; 1828 } 1829 if (ctrlr->state != NVME_CTRLR_STATE_READY && rc != -1) { 1830 return -EAGAIN; 1831 } 1832 1833 /* 1834 * For non-fabrics controllers, the memory locations of the transport qpair 1835 * don't change when the controller is reset. They simply need to be 1836 * re-enabled with admin commands to the controller. For fabric 1837 * controllers we need to disconnect and reconnect the qpair on its 1838 * own thread outside of the context of the reset. 1839 */ 1840 if (rc == 0 && !spdk_nvme_ctrlr_is_fabrics(ctrlr)) { 1841 /* Reinitialize qpairs */ 1842 TAILQ_FOREACH(qpair, &ctrlr->active_io_qpairs, tailq) { 1843 /* Always clear the qid bit here, even for a foreign qpair. We need 1844 * to make sure another process doesn't get the chance to grab that 1845 * qid. 1846 */ 1847 assert(spdk_bit_array_get(ctrlr->free_io_qids, qpair->id)); 1848 spdk_bit_array_clear(ctrlr->free_io_qids, qpair->id); 1849 if (nvme_ctrlr_get_current_process(ctrlr) != qpair->active_proc) { 1850 /* 1851 * We cannot reinitialize a foreign qpair. The qpair's owning 1852 * process will take care of it. Set failure reason to FAILURE_RESET 1853 * to ensure that happens. 1854 */ 1855 qpair->transport_failure_reason = SPDK_NVME_QPAIR_FAILURE_RESET; 1856 continue; 1857 } 1858 rc_tmp = nvme_ctrlr_reinitialize_io_qpair(ctrlr, qpair); 1859 if (rc_tmp != 0) { 1860 rc = rc_tmp; 1861 } 1862 } 1863 } 1864 1865 /* 1866 * Take this opportunity to remove inactive namespaces. During a reset namespace 1867 * handles can be invalidated. 1868 */ 1869 RB_FOREACH_SAFE(ns, nvme_ns_tree, &ctrlr->ns, tmp_ns) { 1870 if (!ns->active) { 1871 RB_REMOVE(nvme_ns_tree, &ctrlr->ns, ns); 1872 spdk_free(ns); 1873 } 1874 } 1875 1876 if (rc) { 1877 nvme_ctrlr_fail(ctrlr, false); 1878 } 1879 ctrlr->is_resetting = false; 1880 1881 nvme_ctrlr_unlock(ctrlr); 1882 1883 if (!ctrlr->cdata.oaes.ns_attribute_notices) { 1884 /* 1885 * If controller doesn't support ns_attribute_notices and 1886 * namespace attributes change (e.g. number of namespaces) 1887 * we need to update system handling device reset. 1888 */ 1889 nvme_io_msg_ctrlr_update(ctrlr); 1890 } 1891 1892 return rc; 1893 } 1894 1895 /* 1896 * For PCIe transport, spdk_nvme_ctrlr_disconnect() will do a Controller Level Reset 1897 * (Change CC.EN from 1 to 0) as a operation to disconnect the admin qpair. 1898 * The following two functions are added to do a Controller Level Reset. They have 1899 * to be called under the nvme controller's lock. 1900 */ 1901 void 1902 nvme_ctrlr_disable(struct spdk_nvme_ctrlr *ctrlr) 1903 { 1904 assert(ctrlr->is_disconnecting == true); 1905 1906 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_CHECK_EN, NVME_TIMEOUT_INFINITE); 1907 } 1908 1909 int 1910 nvme_ctrlr_disable_poll(struct spdk_nvme_ctrlr *ctrlr) 1911 { 1912 int rc = 0; 1913 1914 if (nvme_ctrlr_process_init(ctrlr) != 0) { 1915 NVME_CTRLR_ERRLOG(ctrlr, "failed to disable controller\n"); 1916 rc = -1; 1917 } 1918 1919 if (ctrlr->state != NVME_CTRLR_STATE_DISABLED && rc != -1) { 1920 return -EAGAIN; 1921 } 1922 1923 return rc; 1924 } 1925 1926 static void 1927 nvme_ctrlr_fail_io_qpairs(struct spdk_nvme_ctrlr *ctrlr) 1928 { 1929 struct spdk_nvme_qpair *qpair; 1930 1931 TAILQ_FOREACH(qpair, &ctrlr->active_io_qpairs, tailq) { 1932 qpair->transport_failure_reason = SPDK_NVME_QPAIR_FAILURE_LOCAL; 1933 } 1934 } 1935 1936 int 1937 spdk_nvme_ctrlr_reset(struct spdk_nvme_ctrlr *ctrlr) 1938 { 1939 int rc; 1940 1941 nvme_ctrlr_lock(ctrlr); 1942 1943 rc = nvme_ctrlr_disconnect(ctrlr); 1944 if (rc == 0) { 1945 nvme_ctrlr_fail_io_qpairs(ctrlr); 1946 } 1947 1948 nvme_ctrlr_unlock(ctrlr); 1949 1950 if (rc != 0) { 1951 if (rc == -EBUSY) { 1952 rc = 0; 1953 } 1954 return rc; 1955 } 1956 1957 while (1) { 1958 rc = spdk_nvme_ctrlr_process_admin_completions(ctrlr); 1959 if (rc == -ENXIO) { 1960 break; 1961 } 1962 } 1963 1964 spdk_nvme_ctrlr_reconnect_async(ctrlr); 1965 1966 while (true) { 1967 rc = spdk_nvme_ctrlr_reconnect_poll_async(ctrlr); 1968 if (rc != -EAGAIN) { 1969 break; 1970 } 1971 } 1972 1973 return rc; 1974 } 1975 1976 int 1977 spdk_nvme_ctrlr_reset_subsystem(struct spdk_nvme_ctrlr *ctrlr) 1978 { 1979 union spdk_nvme_cap_register cap; 1980 int rc = 0; 1981 1982 cap = spdk_nvme_ctrlr_get_regs_cap(ctrlr); 1983 if (cap.bits.nssrs == 0) { 1984 NVME_CTRLR_WARNLOG(ctrlr, "subsystem reset is not supported\n"); 1985 return -ENOTSUP; 1986 } 1987 1988 NVME_CTRLR_NOTICELOG(ctrlr, "resetting subsystem\n"); 1989 nvme_ctrlr_lock(ctrlr); 1990 ctrlr->is_resetting = true; 1991 rc = nvme_ctrlr_set_nssr(ctrlr, SPDK_NVME_NSSR_VALUE); 1992 ctrlr->is_resetting = false; 1993 1994 nvme_ctrlr_unlock(ctrlr); 1995 /* 1996 * No more cleanup at this point like in the ctrlr reset. A subsystem reset will cause 1997 * a hot remove for PCIe transport. The hot remove handling does all the necessary ctrlr cleanup. 1998 */ 1999 return rc; 2000 } 2001 2002 int 2003 spdk_nvme_ctrlr_set_trid(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_transport_id *trid) 2004 { 2005 int rc = 0; 2006 2007 nvme_ctrlr_lock(ctrlr); 2008 2009 if (ctrlr->is_failed == false) { 2010 rc = -EPERM; 2011 goto out; 2012 } 2013 2014 if (trid->trtype != ctrlr->trid.trtype) { 2015 rc = -EINVAL; 2016 goto out; 2017 } 2018 2019 if (strncmp(trid->subnqn, ctrlr->trid.subnqn, SPDK_NVMF_NQN_MAX_LEN)) { 2020 rc = -EINVAL; 2021 goto out; 2022 } 2023 2024 ctrlr->trid = *trid; 2025 2026 out: 2027 nvme_ctrlr_unlock(ctrlr); 2028 return rc; 2029 } 2030 2031 void 2032 spdk_nvme_ctrlr_set_remove_cb(struct spdk_nvme_ctrlr *ctrlr, 2033 spdk_nvme_remove_cb remove_cb, void *remove_ctx) 2034 { 2035 if (!spdk_process_is_primary()) { 2036 return; 2037 } 2038 2039 nvme_ctrlr_lock(ctrlr); 2040 ctrlr->remove_cb = remove_cb; 2041 ctrlr->cb_ctx = remove_ctx; 2042 nvme_ctrlr_unlock(ctrlr); 2043 } 2044 2045 int 2046 spdk_nvme_ctrlr_set_keys(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_ctrlr_key_opts *opts) 2047 { 2048 nvme_ctrlr_lock(ctrlr); 2049 if (SPDK_GET_FIELD(opts, dhchap_key, ctrlr->opts.dhchap_key) == NULL && 2050 SPDK_GET_FIELD(opts, dhchap_ctrlr_key, ctrlr->opts.dhchap_ctrlr_key) != NULL) { 2051 NVME_CTRLR_ERRLOG(ctrlr, "DH-HMAC-CHAP controller key requires host key to be set\n"); 2052 nvme_ctrlr_unlock(ctrlr); 2053 return -EINVAL; 2054 } 2055 2056 ctrlr->opts.dhchap_key = 2057 SPDK_GET_FIELD(opts, dhchap_key, ctrlr->opts.dhchap_key); 2058 ctrlr->opts.dhchap_ctrlr_key = 2059 SPDK_GET_FIELD(opts, dhchap_ctrlr_key, ctrlr->opts.dhchap_ctrlr_key); 2060 nvme_ctrlr_unlock(ctrlr); 2061 2062 return 0; 2063 } 2064 2065 static void 2066 nvme_ctrlr_identify_done(void *arg, const struct spdk_nvme_cpl *cpl) 2067 { 2068 struct spdk_nvme_ctrlr *ctrlr = (struct spdk_nvme_ctrlr *)arg; 2069 2070 if (spdk_nvme_cpl_is_error(cpl)) { 2071 NVME_CTRLR_ERRLOG(ctrlr, "nvme_identify_controller failed!\n"); 2072 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 2073 return; 2074 } 2075 2076 /* 2077 * Use MDTS to ensure our default max_xfer_size doesn't exceed what the 2078 * controller supports. 2079 */ 2080 ctrlr->max_xfer_size = nvme_transport_ctrlr_get_max_xfer_size(ctrlr); 2081 NVME_CTRLR_DEBUGLOG(ctrlr, "transport max_xfer_size %u\n", ctrlr->max_xfer_size); 2082 if (ctrlr->cdata.mdts > 0) { 2083 ctrlr->max_xfer_size = spdk_min(ctrlr->max_xfer_size, 2084 ctrlr->min_page_size * (1 << ctrlr->cdata.mdts)); 2085 NVME_CTRLR_DEBUGLOG(ctrlr, "MDTS max_xfer_size %u\n", ctrlr->max_xfer_size); 2086 } 2087 2088 NVME_CTRLR_DEBUGLOG(ctrlr, "CNTLID 0x%04" PRIx16 "\n", ctrlr->cdata.cntlid); 2089 if (ctrlr->trid.trtype == SPDK_NVME_TRANSPORT_PCIE) { 2090 ctrlr->cntlid = ctrlr->cdata.cntlid; 2091 } else { 2092 /* 2093 * Fabrics controllers should already have CNTLID from the Connect command. 2094 * 2095 * If CNTLID from Connect doesn't match CNTLID in the Identify Controller data, 2096 * trust the one from Connect. 2097 */ 2098 if (ctrlr->cntlid != ctrlr->cdata.cntlid) { 2099 NVME_CTRLR_DEBUGLOG(ctrlr, "Identify CNTLID 0x%04" PRIx16 " != Connect CNTLID 0x%04" PRIx16 "\n", 2100 ctrlr->cdata.cntlid, ctrlr->cntlid); 2101 } 2102 } 2103 2104 if (ctrlr->cdata.sgls.supported && !(ctrlr->quirks & NVME_QUIRK_NOT_USE_SGL)) { 2105 assert(ctrlr->cdata.sgls.supported != 0x3); 2106 ctrlr->flags |= SPDK_NVME_CTRLR_SGL_SUPPORTED; 2107 if (ctrlr->cdata.sgls.supported == 0x2) { 2108 ctrlr->flags |= SPDK_NVME_CTRLR_SGL_REQUIRES_DWORD_ALIGNMENT; 2109 } 2110 2111 ctrlr->max_sges = nvme_transport_ctrlr_get_max_sges(ctrlr); 2112 NVME_CTRLR_DEBUGLOG(ctrlr, "transport max_sges %u\n", ctrlr->max_sges); 2113 } 2114 2115 if (ctrlr->cdata.sgls.metadata_address && !(ctrlr->quirks & NVME_QUIRK_NOT_USE_SGL)) { 2116 ctrlr->flags |= SPDK_NVME_CTRLR_MPTR_SGL_SUPPORTED; 2117 } 2118 2119 if (ctrlr->cdata.oacs.security && !(ctrlr->quirks & NVME_QUIRK_OACS_SECURITY)) { 2120 ctrlr->flags |= SPDK_NVME_CTRLR_SECURITY_SEND_RECV_SUPPORTED; 2121 } 2122 2123 if (ctrlr->cdata.oacs.directives) { 2124 ctrlr->flags |= SPDK_NVME_CTRLR_DIRECTIVES_SUPPORTED; 2125 } 2126 2127 NVME_CTRLR_DEBUGLOG(ctrlr, "fuses compare and write: %d\n", 2128 ctrlr->cdata.fuses.compare_and_write); 2129 if (ctrlr->cdata.fuses.compare_and_write) { 2130 ctrlr->flags |= SPDK_NVME_CTRLR_COMPARE_AND_WRITE_SUPPORTED; 2131 } 2132 2133 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_CONFIGURE_AER, 2134 ctrlr->opts.admin_timeout_ms); 2135 } 2136 2137 static int 2138 nvme_ctrlr_identify(struct spdk_nvme_ctrlr *ctrlr) 2139 { 2140 int rc; 2141 2142 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY, 2143 ctrlr->opts.admin_timeout_ms); 2144 2145 rc = nvme_ctrlr_cmd_identify(ctrlr, SPDK_NVME_IDENTIFY_CTRLR, 0, 0, 0, 2146 &ctrlr->cdata, sizeof(ctrlr->cdata), 2147 nvme_ctrlr_identify_done, ctrlr); 2148 if (rc != 0) { 2149 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 2150 return rc; 2151 } 2152 2153 return 0; 2154 } 2155 2156 static void 2157 nvme_ctrlr_get_zns_cmd_and_effects_log_done(void *arg, const struct spdk_nvme_cpl *cpl) 2158 { 2159 struct spdk_nvme_cmds_and_effect_log_page *log_page; 2160 struct spdk_nvme_ctrlr *ctrlr = arg; 2161 2162 if (spdk_nvme_cpl_is_error(cpl)) { 2163 NVME_CTRLR_ERRLOG(ctrlr, "nvme_ctrlr_get_zns_cmd_and_effects_log failed!\n"); 2164 spdk_free(ctrlr->tmp_ptr); 2165 ctrlr->tmp_ptr = NULL; 2166 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 2167 return; 2168 } 2169 2170 log_page = ctrlr->tmp_ptr; 2171 2172 if (log_page->io_cmds_supported[SPDK_NVME_OPC_ZONE_APPEND].csupp) { 2173 ctrlr->flags |= SPDK_NVME_CTRLR_ZONE_APPEND_SUPPORTED; 2174 } 2175 spdk_free(ctrlr->tmp_ptr); 2176 ctrlr->tmp_ptr = NULL; 2177 2178 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_NUM_QUEUES, ctrlr->opts.admin_timeout_ms); 2179 } 2180 2181 static int 2182 nvme_ctrlr_get_zns_cmd_and_effects_log(struct spdk_nvme_ctrlr *ctrlr) 2183 { 2184 int rc; 2185 2186 assert(!ctrlr->tmp_ptr); 2187 ctrlr->tmp_ptr = spdk_zmalloc(sizeof(struct spdk_nvme_cmds_and_effect_log_page), 64, NULL, 2188 SPDK_ENV_NUMA_ID_ANY, SPDK_MALLOC_SHARE | SPDK_MALLOC_DMA); 2189 if (!ctrlr->tmp_ptr) { 2190 rc = -ENOMEM; 2191 goto error; 2192 } 2193 2194 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_WAIT_FOR_GET_ZNS_CMD_EFFECTS_LOG, 2195 ctrlr->opts.admin_timeout_ms); 2196 2197 rc = spdk_nvme_ctrlr_cmd_get_log_page_ext(ctrlr, SPDK_NVME_LOG_COMMAND_EFFECTS_LOG, 2198 0, ctrlr->tmp_ptr, sizeof(struct spdk_nvme_cmds_and_effect_log_page), 2199 0, 0, 0, SPDK_NVME_CSI_ZNS << 24, 2200 nvme_ctrlr_get_zns_cmd_and_effects_log_done, ctrlr); 2201 if (rc != 0) { 2202 goto error; 2203 } 2204 2205 return 0; 2206 2207 error: 2208 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 2209 spdk_free(ctrlr->tmp_ptr); 2210 ctrlr->tmp_ptr = NULL; 2211 return rc; 2212 } 2213 2214 static void 2215 nvme_ctrlr_identify_zns_specific_done(void *arg, const struct spdk_nvme_cpl *cpl) 2216 { 2217 struct spdk_nvme_ctrlr *ctrlr = (struct spdk_nvme_ctrlr *)arg; 2218 2219 if (spdk_nvme_cpl_is_error(cpl)) { 2220 /* no need to print an error, the controller simply does not support ZNS */ 2221 nvme_ctrlr_free_zns_specific_data(ctrlr); 2222 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_NUM_QUEUES, 2223 ctrlr->opts.admin_timeout_ms); 2224 return; 2225 } 2226 2227 /* A zero zasl value means use mdts */ 2228 if (ctrlr->cdata_zns->zasl) { 2229 uint32_t max_append = ctrlr->min_page_size * (1 << ctrlr->cdata_zns->zasl); 2230 ctrlr->max_zone_append_size = spdk_min(ctrlr->max_xfer_size, max_append); 2231 } else { 2232 ctrlr->max_zone_append_size = ctrlr->max_xfer_size; 2233 } 2234 2235 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_GET_ZNS_CMD_EFFECTS_LOG, 2236 ctrlr->opts.admin_timeout_ms); 2237 } 2238 2239 /** 2240 * This function will try to fetch the I/O Command Specific Controller data structure for 2241 * each I/O Command Set supported by SPDK. 2242 * 2243 * If an I/O Command Set is not supported by the controller, "Invalid Field in Command" 2244 * will be returned. Since we are fetching in a exploratively way, getting an error back 2245 * from the controller should not be treated as fatal. 2246 * 2247 * I/O Command Sets not supported by SPDK will be skipped (e.g. Key Value Command Set). 2248 * 2249 * I/O Command Sets without a IOCS specific data structure (i.e. a zero-filled IOCS specific 2250 * data structure) will be skipped (e.g. NVM Command Set, Key Value Command Set). 2251 */ 2252 static int 2253 nvme_ctrlr_identify_iocs_specific(struct spdk_nvme_ctrlr *ctrlr) 2254 { 2255 int rc; 2256 2257 if (!nvme_ctrlr_multi_iocs_enabled(ctrlr)) { 2258 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_NUM_QUEUES, 2259 ctrlr->opts.admin_timeout_ms); 2260 return 0; 2261 } 2262 2263 /* 2264 * Since SPDK currently only needs to fetch a single Command Set, keep the code here, 2265 * instead of creating multiple NVME_CTRLR_STATE_IDENTIFY_IOCS_SPECIFIC substates, 2266 * which would require additional functions and complexity for no good reason. 2267 */ 2268 assert(!ctrlr->cdata_zns); 2269 ctrlr->cdata_zns = spdk_zmalloc(sizeof(*ctrlr->cdata_zns), 64, NULL, SPDK_ENV_NUMA_ID_ANY, 2270 SPDK_MALLOC_SHARE | SPDK_MALLOC_DMA); 2271 if (!ctrlr->cdata_zns) { 2272 rc = -ENOMEM; 2273 goto error; 2274 } 2275 2276 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_IOCS_SPECIFIC, 2277 ctrlr->opts.admin_timeout_ms); 2278 2279 rc = nvme_ctrlr_cmd_identify(ctrlr, SPDK_NVME_IDENTIFY_CTRLR_IOCS, 0, 0, SPDK_NVME_CSI_ZNS, 2280 ctrlr->cdata_zns, sizeof(*ctrlr->cdata_zns), 2281 nvme_ctrlr_identify_zns_specific_done, ctrlr); 2282 if (rc != 0) { 2283 goto error; 2284 } 2285 2286 return 0; 2287 2288 error: 2289 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 2290 nvme_ctrlr_free_zns_specific_data(ctrlr); 2291 return rc; 2292 } 2293 2294 enum nvme_active_ns_state { 2295 NVME_ACTIVE_NS_STATE_IDLE, 2296 NVME_ACTIVE_NS_STATE_PROCESSING, 2297 NVME_ACTIVE_NS_STATE_DONE, 2298 NVME_ACTIVE_NS_STATE_ERROR 2299 }; 2300 2301 typedef void (*nvme_active_ns_ctx_deleter)(struct nvme_active_ns_ctx *); 2302 2303 struct nvme_active_ns_ctx { 2304 struct spdk_nvme_ctrlr *ctrlr; 2305 uint32_t page_count; 2306 uint32_t next_nsid; 2307 uint32_t *new_ns_list; 2308 nvme_active_ns_ctx_deleter deleter; 2309 2310 enum nvme_active_ns_state state; 2311 }; 2312 2313 static struct nvme_active_ns_ctx * 2314 nvme_active_ns_ctx_create(struct spdk_nvme_ctrlr *ctrlr, nvme_active_ns_ctx_deleter deleter) 2315 { 2316 struct nvme_active_ns_ctx *ctx; 2317 uint32_t *new_ns_list = NULL; 2318 2319 ctx = calloc(1, sizeof(*ctx)); 2320 if (!ctx) { 2321 NVME_CTRLR_ERRLOG(ctrlr, "Failed to allocate nvme_active_ns_ctx!\n"); 2322 return NULL; 2323 } 2324 2325 new_ns_list = spdk_zmalloc(sizeof(struct spdk_nvme_ns_list), ctrlr->page_size, 2326 NULL, SPDK_ENV_LCORE_ID_ANY, SPDK_MALLOC_SHARE); 2327 if (!new_ns_list) { 2328 NVME_CTRLR_ERRLOG(ctrlr, "Failed to allocate active_ns_list!\n"); 2329 free(ctx); 2330 return NULL; 2331 } 2332 2333 ctx->page_count = 1; 2334 ctx->new_ns_list = new_ns_list; 2335 ctx->ctrlr = ctrlr; 2336 ctx->deleter = deleter; 2337 2338 return ctx; 2339 } 2340 2341 static void 2342 nvme_active_ns_ctx_destroy(struct nvme_active_ns_ctx *ctx) 2343 { 2344 spdk_free(ctx->new_ns_list); 2345 free(ctx); 2346 } 2347 2348 static int 2349 nvme_ctrlr_destruct_namespace(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid) 2350 { 2351 struct spdk_nvme_ns tmp, *ns; 2352 2353 assert(ctrlr != NULL); 2354 2355 tmp.id = nsid; 2356 ns = RB_FIND(nvme_ns_tree, &ctrlr->ns, &tmp); 2357 if (ns == NULL) { 2358 return -EINVAL; 2359 } 2360 2361 nvme_ns_destruct(ns); 2362 ns->active = false; 2363 2364 return 0; 2365 } 2366 2367 static int 2368 nvme_ctrlr_construct_namespace(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid) 2369 { 2370 struct spdk_nvme_ns *ns; 2371 2372 if (nsid < 1 || nsid > ctrlr->cdata.nn) { 2373 return -EINVAL; 2374 } 2375 2376 /* Namespaces are constructed on demand, so simply request it. */ 2377 ns = spdk_nvme_ctrlr_get_ns(ctrlr, nsid); 2378 if (ns == NULL) { 2379 return -ENOMEM; 2380 } 2381 2382 ns->active = true; 2383 2384 return 0; 2385 } 2386 2387 static void 2388 nvme_ctrlr_identify_active_ns_swap(struct spdk_nvme_ctrlr *ctrlr, uint32_t *new_ns_list, 2389 size_t max_entries) 2390 { 2391 uint32_t active_ns_count = 0; 2392 size_t i; 2393 uint32_t nsid; 2394 struct spdk_nvme_ns *ns, *tmp_ns; 2395 int rc; 2396 2397 /* First, remove namespaces that no longer exist */ 2398 RB_FOREACH_SAFE(ns, nvme_ns_tree, &ctrlr->ns, tmp_ns) { 2399 nsid = new_ns_list[0]; 2400 active_ns_count = 0; 2401 while (nsid != 0) { 2402 if (nsid == ns->id) { 2403 break; 2404 } 2405 2406 nsid = new_ns_list[active_ns_count++]; 2407 } 2408 2409 if (nsid != ns->id) { 2410 /* Did not find this namespace id in the new list. */ 2411 NVME_CTRLR_DEBUGLOG(ctrlr, "Namespace %u was removed\n", ns->id); 2412 nvme_ctrlr_destruct_namespace(ctrlr, ns->id); 2413 } 2414 } 2415 2416 /* Next, add new namespaces */ 2417 active_ns_count = 0; 2418 for (i = 0; i < max_entries; i++) { 2419 nsid = new_ns_list[active_ns_count]; 2420 2421 if (nsid == 0) { 2422 break; 2423 } 2424 2425 /* If the namespace already exists, this will not construct it a second time. */ 2426 rc = nvme_ctrlr_construct_namespace(ctrlr, nsid); 2427 if (rc != 0) { 2428 /* We can't easily handle a failure here. But just move on. */ 2429 assert(false); 2430 NVME_CTRLR_DEBUGLOG(ctrlr, "Failed to allocate a namespace object.\n"); 2431 continue; 2432 } 2433 2434 active_ns_count++; 2435 } 2436 2437 ctrlr->active_ns_count = active_ns_count; 2438 } 2439 2440 static void 2441 nvme_ctrlr_identify_active_ns_async_done(void *arg, const struct spdk_nvme_cpl *cpl) 2442 { 2443 struct nvme_active_ns_ctx *ctx = arg; 2444 uint32_t *new_ns_list = NULL; 2445 2446 if (spdk_nvme_cpl_is_error(cpl)) { 2447 ctx->state = NVME_ACTIVE_NS_STATE_ERROR; 2448 goto out; 2449 } 2450 2451 ctx->next_nsid = ctx->new_ns_list[1024 * ctx->page_count - 1]; 2452 if (ctx->next_nsid == 0) { 2453 ctx->state = NVME_ACTIVE_NS_STATE_DONE; 2454 goto out; 2455 } 2456 2457 ctx->page_count++; 2458 new_ns_list = spdk_realloc(ctx->new_ns_list, 2459 ctx->page_count * sizeof(struct spdk_nvme_ns_list), 2460 ctx->ctrlr->page_size); 2461 if (!new_ns_list) { 2462 SPDK_ERRLOG("Failed to reallocate active_ns_list!\n"); 2463 ctx->state = NVME_ACTIVE_NS_STATE_ERROR; 2464 goto out; 2465 } 2466 2467 ctx->new_ns_list = new_ns_list; 2468 nvme_ctrlr_identify_active_ns_async(ctx); 2469 return; 2470 2471 out: 2472 if (ctx->deleter) { 2473 ctx->deleter(ctx); 2474 } 2475 } 2476 2477 static void 2478 nvme_ctrlr_identify_active_ns_async(struct nvme_active_ns_ctx *ctx) 2479 { 2480 struct spdk_nvme_ctrlr *ctrlr = ctx->ctrlr; 2481 uint32_t i; 2482 int rc; 2483 2484 if (ctrlr->cdata.nn == 0) { 2485 ctx->state = NVME_ACTIVE_NS_STATE_DONE; 2486 goto out; 2487 } 2488 2489 assert(ctx->new_ns_list != NULL); 2490 2491 /* 2492 * If controller doesn't support active ns list CNS 0x02 dummy up 2493 * an active ns list, i.e. all namespaces report as active 2494 */ 2495 if (ctrlr->vs.raw < SPDK_NVME_VERSION(1, 1, 0) || ctrlr->quirks & NVME_QUIRK_IDENTIFY_CNS) { 2496 uint32_t *new_ns_list; 2497 2498 /* 2499 * Active NS list must always end with zero element. 2500 * So, we allocate for cdata.nn+1. 2501 */ 2502 ctx->page_count = spdk_divide_round_up(ctrlr->cdata.nn + 1, 2503 sizeof(struct spdk_nvme_ns_list) / sizeof(new_ns_list[0])); 2504 new_ns_list = spdk_realloc(ctx->new_ns_list, 2505 ctx->page_count * sizeof(struct spdk_nvme_ns_list), 2506 ctx->ctrlr->page_size); 2507 if (!new_ns_list) { 2508 SPDK_ERRLOG("Failed to reallocate active_ns_list!\n"); 2509 ctx->state = NVME_ACTIVE_NS_STATE_ERROR; 2510 goto out; 2511 } 2512 2513 ctx->new_ns_list = new_ns_list; 2514 ctx->new_ns_list[ctrlr->cdata.nn] = 0; 2515 for (i = 0; i < ctrlr->cdata.nn; i++) { 2516 ctx->new_ns_list[i] = i + 1; 2517 } 2518 2519 ctx->state = NVME_ACTIVE_NS_STATE_DONE; 2520 goto out; 2521 } 2522 2523 ctx->state = NVME_ACTIVE_NS_STATE_PROCESSING; 2524 rc = nvme_ctrlr_cmd_identify(ctrlr, SPDK_NVME_IDENTIFY_ACTIVE_NS_LIST, 0, ctx->next_nsid, 0, 2525 &ctx->new_ns_list[1024 * (ctx->page_count - 1)], sizeof(struct spdk_nvme_ns_list), 2526 nvme_ctrlr_identify_active_ns_async_done, ctx); 2527 if (rc != 0) { 2528 ctx->state = NVME_ACTIVE_NS_STATE_ERROR; 2529 goto out; 2530 } 2531 2532 return; 2533 2534 out: 2535 if (ctx->deleter) { 2536 ctx->deleter(ctx); 2537 } 2538 } 2539 2540 static void 2541 _nvme_active_ns_ctx_deleter(struct nvme_active_ns_ctx *ctx) 2542 { 2543 struct spdk_nvme_ctrlr *ctrlr = ctx->ctrlr; 2544 struct spdk_nvme_ns *ns; 2545 2546 if (ctx->state == NVME_ACTIVE_NS_STATE_ERROR) { 2547 nvme_active_ns_ctx_destroy(ctx); 2548 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 2549 return; 2550 } 2551 2552 assert(ctx->state == NVME_ACTIVE_NS_STATE_DONE); 2553 2554 RB_FOREACH(ns, nvme_ns_tree, &ctrlr->ns) { 2555 nvme_ns_free_iocs_specific_data(ns); 2556 } 2557 2558 nvme_ctrlr_identify_active_ns_swap(ctrlr, ctx->new_ns_list, ctx->page_count * 1024); 2559 nvme_active_ns_ctx_destroy(ctx); 2560 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_IDENTIFY_NS, ctrlr->opts.admin_timeout_ms); 2561 } 2562 2563 static void 2564 _nvme_ctrlr_identify_active_ns(struct spdk_nvme_ctrlr *ctrlr) 2565 { 2566 struct nvme_active_ns_ctx *ctx; 2567 2568 ctx = nvme_active_ns_ctx_create(ctrlr, _nvme_active_ns_ctx_deleter); 2569 if (!ctx) { 2570 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 2571 return; 2572 } 2573 2574 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_ACTIVE_NS, 2575 ctrlr->opts.admin_timeout_ms); 2576 nvme_ctrlr_identify_active_ns_async(ctx); 2577 } 2578 2579 int 2580 nvme_ctrlr_identify_active_ns(struct spdk_nvme_ctrlr *ctrlr) 2581 { 2582 struct nvme_active_ns_ctx *ctx; 2583 int rc; 2584 2585 ctx = nvme_active_ns_ctx_create(ctrlr, NULL); 2586 if (!ctx) { 2587 return -ENOMEM; 2588 } 2589 2590 nvme_ctrlr_identify_active_ns_async(ctx); 2591 while (ctx->state == NVME_ACTIVE_NS_STATE_PROCESSING) { 2592 rc = spdk_nvme_qpair_process_completions(ctrlr->adminq, 0); 2593 if (rc < 0) { 2594 ctx->state = NVME_ACTIVE_NS_STATE_ERROR; 2595 break; 2596 } 2597 } 2598 2599 if (ctx->state == NVME_ACTIVE_NS_STATE_ERROR) { 2600 nvme_active_ns_ctx_destroy(ctx); 2601 return -ENXIO; 2602 } 2603 2604 assert(ctx->state == NVME_ACTIVE_NS_STATE_DONE); 2605 nvme_ctrlr_identify_active_ns_swap(ctrlr, ctx->new_ns_list, ctx->page_count * 1024); 2606 nvme_active_ns_ctx_destroy(ctx); 2607 2608 return 0; 2609 } 2610 2611 static void 2612 nvme_ctrlr_identify_ns_async_done(void *arg, const struct spdk_nvme_cpl *cpl) 2613 { 2614 struct spdk_nvme_ns *ns = (struct spdk_nvme_ns *)arg; 2615 struct spdk_nvme_ctrlr *ctrlr = ns->ctrlr; 2616 uint32_t nsid; 2617 int rc; 2618 2619 if (spdk_nvme_cpl_is_error(cpl)) { 2620 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 2621 return; 2622 } 2623 2624 nvme_ns_set_identify_data(ns); 2625 2626 /* move on to the next active NS */ 2627 nsid = spdk_nvme_ctrlr_get_next_active_ns(ctrlr, ns->id); 2628 ns = spdk_nvme_ctrlr_get_ns(ctrlr, nsid); 2629 if (ns == NULL) { 2630 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_IDENTIFY_ID_DESCS, 2631 ctrlr->opts.admin_timeout_ms); 2632 return; 2633 } 2634 ns->ctrlr = ctrlr; 2635 ns->id = nsid; 2636 2637 rc = nvme_ctrlr_identify_ns_async(ns); 2638 if (rc) { 2639 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 2640 } 2641 } 2642 2643 static int 2644 nvme_ctrlr_identify_ns_async(struct spdk_nvme_ns *ns) 2645 { 2646 struct spdk_nvme_ctrlr *ctrlr = ns->ctrlr; 2647 struct spdk_nvme_ns_data *nsdata; 2648 2649 nsdata = &ns->nsdata; 2650 2651 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_NS, 2652 ctrlr->opts.admin_timeout_ms); 2653 return nvme_ctrlr_cmd_identify(ns->ctrlr, SPDK_NVME_IDENTIFY_NS, 0, ns->id, 0, 2654 nsdata, sizeof(*nsdata), 2655 nvme_ctrlr_identify_ns_async_done, ns); 2656 } 2657 2658 static int 2659 nvme_ctrlr_identify_namespaces(struct spdk_nvme_ctrlr *ctrlr) 2660 { 2661 uint32_t nsid; 2662 struct spdk_nvme_ns *ns; 2663 int rc; 2664 2665 nsid = spdk_nvme_ctrlr_get_first_active_ns(ctrlr); 2666 ns = spdk_nvme_ctrlr_get_ns(ctrlr, nsid); 2667 if (ns == NULL) { 2668 /* No active NS, move on to the next state */ 2669 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_IDENTIFY_ID_DESCS, 2670 ctrlr->opts.admin_timeout_ms); 2671 return 0; 2672 } 2673 2674 ns->ctrlr = ctrlr; 2675 ns->id = nsid; 2676 2677 rc = nvme_ctrlr_identify_ns_async(ns); 2678 if (rc) { 2679 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 2680 } 2681 2682 return rc; 2683 } 2684 2685 static int 2686 nvme_ctrlr_identify_namespaces_iocs_specific_next(struct spdk_nvme_ctrlr *ctrlr, uint32_t prev_nsid) 2687 { 2688 uint32_t nsid; 2689 struct spdk_nvme_ns *ns; 2690 int rc; 2691 2692 if (!prev_nsid) { 2693 nsid = spdk_nvme_ctrlr_get_first_active_ns(ctrlr); 2694 } else { 2695 /* move on to the next active NS */ 2696 nsid = spdk_nvme_ctrlr_get_next_active_ns(ctrlr, prev_nsid); 2697 } 2698 2699 ns = spdk_nvme_ctrlr_get_ns(ctrlr, nsid); 2700 if (ns == NULL) { 2701 /* No first/next active NS, move on to the next state */ 2702 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_SUPPORTED_LOG_PAGES, 2703 ctrlr->opts.admin_timeout_ms); 2704 return 0; 2705 } 2706 2707 /* loop until we find a ns which has (supported) iocs specific data */ 2708 while (!nvme_ns_has_supported_iocs_specific_data(ns)) { 2709 nsid = spdk_nvme_ctrlr_get_next_active_ns(ctrlr, ns->id); 2710 ns = spdk_nvme_ctrlr_get_ns(ctrlr, nsid); 2711 if (ns == NULL) { 2712 /* no namespace with (supported) iocs specific data found */ 2713 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_SUPPORTED_LOG_PAGES, 2714 ctrlr->opts.admin_timeout_ms); 2715 return 0; 2716 } 2717 } 2718 2719 rc = nvme_ctrlr_identify_ns_iocs_specific_async(ns); 2720 if (rc) { 2721 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 2722 } 2723 2724 return rc; 2725 } 2726 2727 static void 2728 nvme_ctrlr_identify_ns_zns_specific_async_done(void *arg, const struct spdk_nvme_cpl *cpl) 2729 { 2730 struct spdk_nvme_ns *ns = (struct spdk_nvme_ns *)arg; 2731 struct spdk_nvme_ctrlr *ctrlr = ns->ctrlr; 2732 2733 if (spdk_nvme_cpl_is_error(cpl)) { 2734 nvme_ns_free_zns_specific_data(ns); 2735 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 2736 return; 2737 } 2738 2739 nvme_ctrlr_identify_namespaces_iocs_specific_next(ctrlr, ns->id); 2740 } 2741 2742 static int 2743 nvme_ctrlr_identify_ns_zns_specific_async(struct spdk_nvme_ns *ns) 2744 { 2745 struct spdk_nvme_ctrlr *ctrlr = ns->ctrlr; 2746 int rc; 2747 2748 assert(!ns->nsdata_zns); 2749 ns->nsdata_zns = spdk_zmalloc(sizeof(*ns->nsdata_zns), 64, NULL, SPDK_ENV_NUMA_ID_ANY, 2750 SPDK_MALLOC_SHARE); 2751 if (!ns->nsdata_zns) { 2752 return -ENOMEM; 2753 } 2754 2755 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_NS_IOCS_SPECIFIC, 2756 ctrlr->opts.admin_timeout_ms); 2757 rc = nvme_ctrlr_cmd_identify(ns->ctrlr, SPDK_NVME_IDENTIFY_NS_IOCS, 0, ns->id, ns->csi, 2758 ns->nsdata_zns, sizeof(*ns->nsdata_zns), 2759 nvme_ctrlr_identify_ns_zns_specific_async_done, ns); 2760 if (rc) { 2761 nvme_ns_free_zns_specific_data(ns); 2762 } 2763 2764 return rc; 2765 } 2766 2767 static void 2768 nvme_ctrlr_identify_ns_nvm_specific_async_done(void *arg, const struct spdk_nvme_cpl *cpl) 2769 { 2770 struct spdk_nvme_ns *ns = (struct spdk_nvme_ns *)arg; 2771 struct spdk_nvme_ctrlr *ctrlr = ns->ctrlr; 2772 2773 if (spdk_nvme_cpl_is_error(cpl)) { 2774 nvme_ns_free_nvm_specific_data(ns); 2775 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 2776 return; 2777 } 2778 2779 nvme_ctrlr_identify_namespaces_iocs_specific_next(ctrlr, ns->id); 2780 } 2781 2782 static int 2783 nvme_ctrlr_identify_ns_nvm_specific_async(struct spdk_nvme_ns *ns) 2784 { 2785 struct spdk_nvme_ctrlr *ctrlr = ns->ctrlr; 2786 int rc; 2787 2788 assert(!ns->nsdata_nvm); 2789 ns->nsdata_nvm = spdk_zmalloc(sizeof(*ns->nsdata_nvm), 64, NULL, SPDK_ENV_NUMA_ID_ANY, 2790 SPDK_MALLOC_SHARE); 2791 if (!ns->nsdata_nvm) { 2792 return -ENOMEM; 2793 } 2794 2795 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_NS_IOCS_SPECIFIC, 2796 ctrlr->opts.admin_timeout_ms); 2797 rc = nvme_ctrlr_cmd_identify(ns->ctrlr, SPDK_NVME_IDENTIFY_NS_IOCS, 0, ns->id, ns->csi, 2798 ns->nsdata_nvm, sizeof(*ns->nsdata_nvm), 2799 nvme_ctrlr_identify_ns_nvm_specific_async_done, ns); 2800 if (rc) { 2801 nvme_ns_free_nvm_specific_data(ns); 2802 } 2803 2804 return rc; 2805 } 2806 2807 static int 2808 nvme_ctrlr_identify_ns_iocs_specific_async(struct spdk_nvme_ns *ns) 2809 { 2810 switch (ns->csi) { 2811 case SPDK_NVME_CSI_ZNS: 2812 return nvme_ctrlr_identify_ns_zns_specific_async(ns); 2813 case SPDK_NVME_CSI_NVM: 2814 if (ns->ctrlr->cdata.ctratt.bits.elbas) { 2815 return nvme_ctrlr_identify_ns_nvm_specific_async(ns); 2816 } 2817 /* fallthrough */ 2818 default: 2819 /* 2820 * This switch must handle all cases for which 2821 * nvme_ns_has_supported_iocs_specific_data() returns true, 2822 * other cases should never happen. 2823 */ 2824 assert(0); 2825 } 2826 2827 return -EINVAL; 2828 } 2829 2830 static int 2831 nvme_ctrlr_identify_namespaces_iocs_specific(struct spdk_nvme_ctrlr *ctrlr) 2832 { 2833 if (!nvme_ctrlr_multi_iocs_enabled(ctrlr)) { 2834 /* Multi IOCS not supported/enabled, move on to the next state */ 2835 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_SUPPORTED_LOG_PAGES, 2836 ctrlr->opts.admin_timeout_ms); 2837 return 0; 2838 } 2839 2840 return nvme_ctrlr_identify_namespaces_iocs_specific_next(ctrlr, 0); 2841 } 2842 2843 static void 2844 nvme_ctrlr_identify_id_desc_async_done(void *arg, const struct spdk_nvme_cpl *cpl) 2845 { 2846 struct spdk_nvme_ns *ns = (struct spdk_nvme_ns *)arg; 2847 struct spdk_nvme_ctrlr *ctrlr = ns->ctrlr; 2848 uint32_t nsid; 2849 int rc; 2850 2851 if (spdk_nvme_cpl_is_error(cpl)) { 2852 /* 2853 * Many controllers claim to be compatible with NVMe 1.3, however, 2854 * they do not implement NS ID Desc List. Therefore, instead of setting 2855 * the state to NVME_CTRLR_STATE_ERROR, silently ignore the completion 2856 * error and move on to the next state. 2857 * 2858 * The proper way is to create a new quirk for controllers that violate 2859 * the NVMe 1.3 spec by not supporting NS ID Desc List. 2860 * (Re-using the NVME_QUIRK_IDENTIFY_CNS quirk is not possible, since 2861 * it is too generic and was added in order to handle controllers that 2862 * violate the NVMe 1.1 spec by not supporting ACTIVE LIST). 2863 */ 2864 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_IDENTIFY_NS_IOCS_SPECIFIC, 2865 ctrlr->opts.admin_timeout_ms); 2866 return; 2867 } 2868 2869 nvme_ns_set_id_desc_list_data(ns); 2870 2871 /* move on to the next active NS */ 2872 nsid = spdk_nvme_ctrlr_get_next_active_ns(ctrlr, ns->id); 2873 ns = spdk_nvme_ctrlr_get_ns(ctrlr, nsid); 2874 if (ns == NULL) { 2875 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_IDENTIFY_NS_IOCS_SPECIFIC, 2876 ctrlr->opts.admin_timeout_ms); 2877 return; 2878 } 2879 2880 rc = nvme_ctrlr_identify_id_desc_async(ns); 2881 if (rc) { 2882 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 2883 } 2884 } 2885 2886 static int 2887 nvme_ctrlr_identify_id_desc_async(struct spdk_nvme_ns *ns) 2888 { 2889 struct spdk_nvme_ctrlr *ctrlr = ns->ctrlr; 2890 2891 memset(ns->id_desc_list, 0, sizeof(ns->id_desc_list)); 2892 2893 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_ID_DESCS, 2894 ctrlr->opts.admin_timeout_ms); 2895 return nvme_ctrlr_cmd_identify(ns->ctrlr, SPDK_NVME_IDENTIFY_NS_ID_DESCRIPTOR_LIST, 2896 0, ns->id, 0, ns->id_desc_list, sizeof(ns->id_desc_list), 2897 nvme_ctrlr_identify_id_desc_async_done, ns); 2898 } 2899 2900 static int 2901 nvme_ctrlr_identify_id_desc_namespaces(struct spdk_nvme_ctrlr *ctrlr) 2902 { 2903 uint32_t nsid; 2904 struct spdk_nvme_ns *ns; 2905 int rc; 2906 2907 if ((ctrlr->vs.raw < SPDK_NVME_VERSION(1, 3, 0) && 2908 !(ctrlr->cap.bits.css & SPDK_NVME_CAP_CSS_IOCS)) || 2909 (ctrlr->quirks & NVME_QUIRK_IDENTIFY_CNS)) { 2910 NVME_CTRLR_DEBUGLOG(ctrlr, "Version < 1.3; not attempting to retrieve NS ID Descriptor List\n"); 2911 /* NS ID Desc List not supported, move on to the next state */ 2912 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_IDENTIFY_NS_IOCS_SPECIFIC, 2913 ctrlr->opts.admin_timeout_ms); 2914 return 0; 2915 } 2916 2917 nsid = spdk_nvme_ctrlr_get_first_active_ns(ctrlr); 2918 ns = spdk_nvme_ctrlr_get_ns(ctrlr, nsid); 2919 if (ns == NULL) { 2920 /* No active NS, move on to the next state */ 2921 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_IDENTIFY_NS_IOCS_SPECIFIC, 2922 ctrlr->opts.admin_timeout_ms); 2923 return 0; 2924 } 2925 2926 rc = nvme_ctrlr_identify_id_desc_async(ns); 2927 if (rc) { 2928 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 2929 } 2930 2931 return rc; 2932 } 2933 2934 static void 2935 nvme_ctrlr_update_nvmf_ioccsz(struct spdk_nvme_ctrlr *ctrlr) 2936 { 2937 if (spdk_nvme_ctrlr_is_fabrics(ctrlr)) { 2938 if (ctrlr->cdata.nvmf_specific.ioccsz < 4) { 2939 NVME_CTRLR_ERRLOG(ctrlr, "Incorrect IOCCSZ %u, the minimum value should be 4\n", 2940 ctrlr->cdata.nvmf_specific.ioccsz); 2941 ctrlr->cdata.nvmf_specific.ioccsz = 4; 2942 assert(0); 2943 } 2944 ctrlr->ioccsz_bytes = ctrlr->cdata.nvmf_specific.ioccsz * 16 - sizeof(struct spdk_nvme_cmd); 2945 ctrlr->icdoff = ctrlr->cdata.nvmf_specific.icdoff; 2946 } 2947 } 2948 2949 static void 2950 nvme_ctrlr_set_num_queues_done(void *arg, const struct spdk_nvme_cpl *cpl) 2951 { 2952 uint32_t cq_allocated, sq_allocated, min_allocated, i; 2953 struct spdk_nvme_ctrlr *ctrlr = (struct spdk_nvme_ctrlr *)arg; 2954 2955 if (spdk_nvme_cpl_is_error(cpl)) { 2956 NVME_CTRLR_ERRLOG(ctrlr, "Set Features - Number of Queues failed!\n"); 2957 ctrlr->opts.num_io_queues = 0; 2958 } else { 2959 /* 2960 * Data in cdw0 is 0-based. 2961 * Lower 16-bits indicate number of submission queues allocated. 2962 * Upper 16-bits indicate number of completion queues allocated. 2963 */ 2964 sq_allocated = (cpl->cdw0 & 0xFFFF) + 1; 2965 cq_allocated = (cpl->cdw0 >> 16) + 1; 2966 2967 /* 2968 * For 1:1 queue mapping, set number of allocated queues to be minimum of 2969 * submission and completion queues. 2970 */ 2971 min_allocated = spdk_min(sq_allocated, cq_allocated); 2972 2973 /* Set number of queues to be minimum of requested and actually allocated. */ 2974 ctrlr->opts.num_io_queues = spdk_min(min_allocated, ctrlr->opts.num_io_queues); 2975 2976 if (ctrlr->opts.enable_interrupts) { 2977 ctrlr->opts.num_io_queues = spdk_min(MAX_IO_QUEUES_WITH_INTERRUPTS, 2978 ctrlr->opts.num_io_queues); 2979 if (nvme_transport_ctrlr_enable_interrupts(ctrlr) < 0) { 2980 NVME_CTRLR_ERRLOG(ctrlr, "Failed to enable interrupts!\n"); 2981 ctrlr->opts.enable_interrupts = false; 2982 } 2983 } 2984 } 2985 2986 ctrlr->free_io_qids = spdk_bit_array_create(ctrlr->opts.num_io_queues + 1); 2987 if (ctrlr->free_io_qids == NULL) { 2988 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 2989 return; 2990 } 2991 2992 /* Initialize list of free I/O queue IDs. QID 0 is the admin queue (implicitly allocated). */ 2993 for (i = 1; i <= ctrlr->opts.num_io_queues; i++) { 2994 spdk_nvme_ctrlr_free_qid(ctrlr, i); 2995 } 2996 2997 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_IDENTIFY_ACTIVE_NS, 2998 ctrlr->opts.admin_timeout_ms); 2999 } 3000 3001 static int 3002 nvme_ctrlr_set_num_queues(struct spdk_nvme_ctrlr *ctrlr) 3003 { 3004 int rc; 3005 3006 if (ctrlr->opts.num_io_queues > SPDK_NVME_MAX_IO_QUEUES) { 3007 NVME_CTRLR_NOTICELOG(ctrlr, "Limiting requested num_io_queues %u to max %d\n", 3008 ctrlr->opts.num_io_queues, SPDK_NVME_MAX_IO_QUEUES); 3009 ctrlr->opts.num_io_queues = SPDK_NVME_MAX_IO_QUEUES; 3010 } else if (ctrlr->opts.num_io_queues < 1) { 3011 NVME_CTRLR_NOTICELOG(ctrlr, "Requested num_io_queues 0, increasing to 1\n"); 3012 ctrlr->opts.num_io_queues = 1; 3013 } 3014 3015 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_WAIT_FOR_SET_NUM_QUEUES, 3016 ctrlr->opts.admin_timeout_ms); 3017 3018 rc = nvme_ctrlr_cmd_set_num_queues(ctrlr, ctrlr->opts.num_io_queues, 3019 nvme_ctrlr_set_num_queues_done, ctrlr); 3020 if (rc != 0) { 3021 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 3022 return rc; 3023 } 3024 3025 return 0; 3026 } 3027 3028 static void 3029 nvme_ctrlr_set_keep_alive_timeout_done(void *arg, const struct spdk_nvme_cpl *cpl) 3030 { 3031 uint32_t keep_alive_interval_us; 3032 struct spdk_nvme_ctrlr *ctrlr = (struct spdk_nvme_ctrlr *)arg; 3033 3034 if (spdk_nvme_cpl_is_error(cpl)) { 3035 if ((cpl->status.sct == SPDK_NVME_SCT_GENERIC) && 3036 (cpl->status.sc == SPDK_NVME_SC_INVALID_FIELD)) { 3037 NVME_CTRLR_DEBUGLOG(ctrlr, "Keep alive timeout Get Feature is not supported\n"); 3038 } else { 3039 NVME_CTRLR_ERRLOG(ctrlr, "Keep alive timeout Get Feature failed: SC %x SCT %x\n", 3040 cpl->status.sc, cpl->status.sct); 3041 ctrlr->opts.keep_alive_timeout_ms = 0; 3042 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 3043 return; 3044 } 3045 } else { 3046 if (ctrlr->opts.keep_alive_timeout_ms != cpl->cdw0) { 3047 NVME_CTRLR_DEBUGLOG(ctrlr, "Controller adjusted keep alive timeout to %u ms\n", 3048 cpl->cdw0); 3049 } 3050 3051 ctrlr->opts.keep_alive_timeout_ms = cpl->cdw0; 3052 } 3053 3054 if (ctrlr->opts.keep_alive_timeout_ms == 0) { 3055 ctrlr->keep_alive_interval_ticks = 0; 3056 } else { 3057 keep_alive_interval_us = ctrlr->opts.keep_alive_timeout_ms * 1000 / 2; 3058 3059 NVME_CTRLR_DEBUGLOG(ctrlr, "Sending keep alive every %u us\n", keep_alive_interval_us); 3060 3061 ctrlr->keep_alive_interval_ticks = (keep_alive_interval_us * spdk_get_ticks_hz()) / 3062 UINT64_C(1000000); 3063 3064 /* Schedule the first Keep Alive to be sent as soon as possible. */ 3065 ctrlr->next_keep_alive_tick = spdk_get_ticks(); 3066 } 3067 3068 if (spdk_nvme_ctrlr_is_discovery(ctrlr)) { 3069 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_READY, NVME_TIMEOUT_INFINITE); 3070 } else { 3071 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_IDENTIFY_IOCS_SPECIFIC, 3072 ctrlr->opts.admin_timeout_ms); 3073 } 3074 } 3075 3076 static int 3077 nvme_ctrlr_set_keep_alive_timeout(struct spdk_nvme_ctrlr *ctrlr) 3078 { 3079 int rc; 3080 3081 if (ctrlr->opts.keep_alive_timeout_ms == 0) { 3082 if (spdk_nvme_ctrlr_is_discovery(ctrlr)) { 3083 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_READY, NVME_TIMEOUT_INFINITE); 3084 } else { 3085 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_IDENTIFY_IOCS_SPECIFIC, 3086 ctrlr->opts.admin_timeout_ms); 3087 } 3088 return 0; 3089 } 3090 3091 /* Note: Discovery controller identify data does not populate KAS according to spec. */ 3092 if (!spdk_nvme_ctrlr_is_discovery(ctrlr) && ctrlr->cdata.kas == 0) { 3093 NVME_CTRLR_DEBUGLOG(ctrlr, "Controller KAS is 0 - not enabling Keep Alive\n"); 3094 ctrlr->opts.keep_alive_timeout_ms = 0; 3095 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_IDENTIFY_IOCS_SPECIFIC, 3096 ctrlr->opts.admin_timeout_ms); 3097 return 0; 3098 } 3099 3100 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_WAIT_FOR_KEEP_ALIVE_TIMEOUT, 3101 ctrlr->opts.admin_timeout_ms); 3102 3103 /* Retrieve actual keep alive timeout, since the controller may have adjusted it. */ 3104 rc = spdk_nvme_ctrlr_cmd_get_feature(ctrlr, SPDK_NVME_FEAT_KEEP_ALIVE_TIMER, 0, NULL, 0, 3105 nvme_ctrlr_set_keep_alive_timeout_done, ctrlr); 3106 if (rc != 0) { 3107 NVME_CTRLR_ERRLOG(ctrlr, "Keep alive timeout Get Feature failed: %d\n", rc); 3108 ctrlr->opts.keep_alive_timeout_ms = 0; 3109 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 3110 return rc; 3111 } 3112 3113 return 0; 3114 } 3115 3116 static void 3117 nvme_ctrlr_set_host_id_done(void *arg, const struct spdk_nvme_cpl *cpl) 3118 { 3119 struct spdk_nvme_ctrlr *ctrlr = (struct spdk_nvme_ctrlr *)arg; 3120 3121 if (spdk_nvme_cpl_is_error(cpl)) { 3122 /* 3123 * Treat Set Features - Host ID failure as non-fatal, since the Host ID feature 3124 * is optional. 3125 */ 3126 NVME_CTRLR_WARNLOG(ctrlr, "Set Features - Host ID failed: SC 0x%x SCT 0x%x\n", 3127 cpl->status.sc, cpl->status.sct); 3128 } else { 3129 NVME_CTRLR_DEBUGLOG(ctrlr, "Set Features - Host ID was successful\n"); 3130 } 3131 3132 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_TRANSPORT_READY, ctrlr->opts.admin_timeout_ms); 3133 } 3134 3135 static int 3136 nvme_ctrlr_set_host_id(struct spdk_nvme_ctrlr *ctrlr) 3137 { 3138 uint8_t *host_id; 3139 uint32_t host_id_size; 3140 int rc; 3141 3142 if (ctrlr->trid.trtype != SPDK_NVME_TRANSPORT_PCIE) { 3143 /* 3144 * NVMe-oF sends the host ID during Connect and doesn't allow 3145 * Set Features - Host Identifier after Connect, so we don't need to do anything here. 3146 */ 3147 NVME_CTRLR_DEBUGLOG(ctrlr, "NVMe-oF transport - not sending Set Features - Host ID\n"); 3148 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_TRANSPORT_READY, ctrlr->opts.admin_timeout_ms); 3149 return 0; 3150 } 3151 3152 if (ctrlr->cdata.ctratt.bits.host_id_exhid_supported) { 3153 NVME_CTRLR_DEBUGLOG(ctrlr, "Using 128-bit extended host identifier\n"); 3154 host_id = ctrlr->opts.extended_host_id; 3155 host_id_size = sizeof(ctrlr->opts.extended_host_id); 3156 } else { 3157 NVME_CTRLR_DEBUGLOG(ctrlr, "Using 64-bit host identifier\n"); 3158 host_id = ctrlr->opts.host_id; 3159 host_id_size = sizeof(ctrlr->opts.host_id); 3160 } 3161 3162 /* If the user specified an all-zeroes host identifier, don't send the command. */ 3163 if (spdk_mem_all_zero(host_id, host_id_size)) { 3164 NVME_CTRLR_DEBUGLOG(ctrlr, "User did not specify host ID - not sending Set Features - Host ID\n"); 3165 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_TRANSPORT_READY, ctrlr->opts.admin_timeout_ms); 3166 return 0; 3167 } 3168 3169 SPDK_LOGDUMP(nvme, "host_id", host_id, host_id_size); 3170 3171 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_WAIT_FOR_HOST_ID, 3172 ctrlr->opts.admin_timeout_ms); 3173 3174 rc = nvme_ctrlr_cmd_set_host_id(ctrlr, host_id, host_id_size, nvme_ctrlr_set_host_id_done, ctrlr); 3175 if (rc != 0) { 3176 NVME_CTRLR_ERRLOG(ctrlr, "Set Features - Host ID failed: %d\n", rc); 3177 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 3178 return rc; 3179 } 3180 3181 return 0; 3182 } 3183 3184 void 3185 nvme_ctrlr_update_namespaces(struct spdk_nvme_ctrlr *ctrlr) 3186 { 3187 uint32_t nsid; 3188 struct spdk_nvme_ns *ns; 3189 3190 for (nsid = spdk_nvme_ctrlr_get_first_active_ns(ctrlr); 3191 nsid != 0; nsid = spdk_nvme_ctrlr_get_next_active_ns(ctrlr, nsid)) { 3192 ns = spdk_nvme_ctrlr_get_ns(ctrlr, nsid); 3193 nvme_ns_construct(ns, nsid, ctrlr); 3194 } 3195 } 3196 3197 static int 3198 nvme_ctrlr_clear_changed_ns_log(struct spdk_nvme_ctrlr *ctrlr) 3199 { 3200 struct nvme_completion_poll_status *status; 3201 int rc = -ENOMEM; 3202 char *buffer = NULL; 3203 uint32_t nsid; 3204 size_t buf_size = (SPDK_NVME_MAX_CHANGED_NAMESPACES * sizeof(uint32_t)); 3205 3206 if (ctrlr->opts.disable_read_changed_ns_list_log_page) { 3207 return 0; 3208 } 3209 3210 buffer = spdk_dma_zmalloc(buf_size, 4096, NULL); 3211 if (!buffer) { 3212 NVME_CTRLR_ERRLOG(ctrlr, "Failed to allocate buffer for getting " 3213 "changed ns log.\n"); 3214 return rc; 3215 } 3216 3217 status = calloc(1, sizeof(*status)); 3218 if (!status) { 3219 NVME_CTRLR_ERRLOG(ctrlr, "Failed to allocate status tracker\n"); 3220 goto free_buffer; 3221 } 3222 3223 rc = spdk_nvme_ctrlr_cmd_get_log_page(ctrlr, 3224 SPDK_NVME_LOG_CHANGED_NS_LIST, 3225 SPDK_NVME_GLOBAL_NS_TAG, 3226 buffer, buf_size, 0, 3227 nvme_completion_poll_cb, status); 3228 3229 if (rc) { 3230 NVME_CTRLR_ERRLOG(ctrlr, "spdk_nvme_ctrlr_cmd_get_log_page() failed: rc=%d\n", rc); 3231 free(status); 3232 goto free_buffer; 3233 } 3234 3235 rc = nvme_wait_for_completion_timeout(ctrlr->adminq, status, 3236 ctrlr->opts.admin_timeout_ms * 1000); 3237 if (!status->timed_out) { 3238 free(status); 3239 } 3240 3241 if (rc) { 3242 NVME_CTRLR_ERRLOG(ctrlr, "wait for spdk_nvme_ctrlr_cmd_get_log_page failed: rc=%d\n", rc); 3243 goto free_buffer; 3244 } 3245 3246 /* only check the case of overflow. */ 3247 nsid = from_le32(buffer); 3248 if (nsid == 0xffffffffu) { 3249 NVME_CTRLR_WARNLOG(ctrlr, "changed ns log overflowed.\n"); 3250 } 3251 3252 free_buffer: 3253 spdk_dma_free(buffer); 3254 return rc; 3255 } 3256 3257 static void 3258 nvme_ctrlr_process_async_event(struct spdk_nvme_ctrlr *ctrlr, 3259 const struct spdk_nvme_cpl *cpl) 3260 { 3261 union spdk_nvme_async_event_completion event; 3262 struct spdk_nvme_ctrlr_process *active_proc; 3263 int rc; 3264 3265 event.raw = cpl->cdw0; 3266 3267 if ((event.bits.async_event_type == SPDK_NVME_ASYNC_EVENT_TYPE_NOTICE) && 3268 (event.bits.async_event_info == SPDK_NVME_ASYNC_EVENT_NS_ATTR_CHANGED)) { 3269 nvme_ctrlr_clear_changed_ns_log(ctrlr); 3270 3271 rc = nvme_ctrlr_identify_active_ns(ctrlr); 3272 if (rc) { 3273 return; 3274 } 3275 nvme_ctrlr_update_namespaces(ctrlr); 3276 nvme_io_msg_ctrlr_update(ctrlr); 3277 } 3278 3279 if ((event.bits.async_event_type == SPDK_NVME_ASYNC_EVENT_TYPE_NOTICE) && 3280 (event.bits.async_event_info == SPDK_NVME_ASYNC_EVENT_ANA_CHANGE)) { 3281 if (!ctrlr->opts.disable_read_ana_log_page) { 3282 rc = nvme_ctrlr_update_ana_log_page(ctrlr); 3283 if (rc) { 3284 return; 3285 } 3286 nvme_ctrlr_parse_ana_log_page(ctrlr, nvme_ctrlr_update_ns_ana_states, 3287 ctrlr); 3288 } 3289 } 3290 3291 active_proc = nvme_ctrlr_get_current_process(ctrlr); 3292 if (active_proc && active_proc->aer_cb_fn) { 3293 active_proc->aer_cb_fn(active_proc->aer_cb_arg, cpl); 3294 } 3295 } 3296 3297 static void 3298 nvme_ctrlr_queue_async_event(struct spdk_nvme_ctrlr *ctrlr, 3299 const struct spdk_nvme_cpl *cpl) 3300 { 3301 struct spdk_nvme_ctrlr_aer_completion *nvme_event; 3302 struct spdk_nvme_ctrlr_process *proc; 3303 3304 /* Add async event to each process objects event list */ 3305 TAILQ_FOREACH(proc, &ctrlr->active_procs, tailq) { 3306 /* Must be shared memory so other processes can access */ 3307 nvme_event = spdk_zmalloc(sizeof(*nvme_event), 0, NULL, SPDK_ENV_NUMA_ID_ANY, SPDK_MALLOC_SHARE); 3308 if (!nvme_event) { 3309 NVME_CTRLR_ERRLOG(ctrlr, "Alloc nvme event failed, ignore the event\n"); 3310 return; 3311 } 3312 nvme_event->cpl = *cpl; 3313 3314 STAILQ_INSERT_TAIL(&proc->async_events, nvme_event, link); 3315 } 3316 } 3317 3318 static void 3319 nvme_ctrlr_complete_queued_async_events(struct spdk_nvme_ctrlr *ctrlr) 3320 { 3321 struct spdk_nvme_ctrlr_aer_completion *nvme_event, *nvme_event_tmp; 3322 struct spdk_nvme_ctrlr_process *active_proc; 3323 3324 active_proc = nvme_ctrlr_get_current_process(ctrlr); 3325 3326 STAILQ_FOREACH_SAFE(nvme_event, &active_proc->async_events, link, nvme_event_tmp) { 3327 STAILQ_REMOVE(&active_proc->async_events, nvme_event, 3328 spdk_nvme_ctrlr_aer_completion, link); 3329 nvme_ctrlr_process_async_event(ctrlr, &nvme_event->cpl); 3330 spdk_free(nvme_event); 3331 3332 } 3333 } 3334 3335 static void 3336 nvme_ctrlr_async_event_cb(void *arg, const struct spdk_nvme_cpl *cpl) 3337 { 3338 struct nvme_async_event_request *aer = arg; 3339 struct spdk_nvme_ctrlr *ctrlr = aer->ctrlr; 3340 3341 if (cpl->status.sct == SPDK_NVME_SCT_GENERIC && 3342 cpl->status.sc == SPDK_NVME_SC_ABORTED_SQ_DELETION) { 3343 /* 3344 * This is simulated when controller is being shut down, to 3345 * effectively abort outstanding asynchronous event requests 3346 * and make sure all memory is freed. Do not repost the 3347 * request in this case. 3348 */ 3349 return; 3350 } 3351 3352 if (cpl->status.sct == SPDK_NVME_SCT_COMMAND_SPECIFIC && 3353 cpl->status.sc == SPDK_NVME_SC_ASYNC_EVENT_REQUEST_LIMIT_EXCEEDED) { 3354 /* 3355 * SPDK will only send as many AERs as the device says it supports, 3356 * so this status code indicates an out-of-spec device. Do not repost 3357 * the request in this case. 3358 */ 3359 NVME_CTRLR_ERRLOG(ctrlr, "Controller appears out-of-spec for asynchronous event request\n" 3360 "handling. Do not repost this AER.\n"); 3361 return; 3362 } 3363 3364 /* Add the events to the list */ 3365 nvme_ctrlr_queue_async_event(ctrlr, cpl); 3366 3367 /* If the ctrlr was removed or in the destruct state, we should not send aer again */ 3368 if (ctrlr->is_removed || ctrlr->is_destructed) { 3369 return; 3370 } 3371 3372 /* 3373 * Repost another asynchronous event request to replace the one 3374 * that just completed. 3375 */ 3376 if (nvme_ctrlr_construct_and_submit_aer(ctrlr, aer)) { 3377 /* 3378 * We can't do anything to recover from a failure here, 3379 * so just print a warning message and leave the AER unsubmitted. 3380 */ 3381 NVME_CTRLR_ERRLOG(ctrlr, "resubmitting AER failed!\n"); 3382 } 3383 } 3384 3385 static int 3386 nvme_ctrlr_construct_and_submit_aer(struct spdk_nvme_ctrlr *ctrlr, 3387 struct nvme_async_event_request *aer) 3388 { 3389 struct nvme_request *req; 3390 3391 aer->ctrlr = ctrlr; 3392 req = nvme_allocate_request_null(ctrlr->adminq, nvme_ctrlr_async_event_cb, aer); 3393 aer->req = req; 3394 if (req == NULL) { 3395 return -1; 3396 } 3397 3398 req->cmd.opc = SPDK_NVME_OPC_ASYNC_EVENT_REQUEST; 3399 return nvme_ctrlr_submit_admin_request(ctrlr, req); 3400 } 3401 3402 static void 3403 nvme_ctrlr_configure_aer_done(void *arg, const struct spdk_nvme_cpl *cpl) 3404 { 3405 struct nvme_async_event_request *aer; 3406 int rc; 3407 uint32_t i; 3408 struct spdk_nvme_ctrlr *ctrlr = (struct spdk_nvme_ctrlr *)arg; 3409 3410 if (spdk_nvme_cpl_is_error(cpl)) { 3411 NVME_CTRLR_NOTICELOG(ctrlr, "nvme_ctrlr_configure_aer failed!\n"); 3412 ctrlr->num_aers = 0; 3413 } else { 3414 /* aerl is a zero-based value, so we need to add 1 here. */ 3415 ctrlr->num_aers = spdk_min(NVME_MAX_ASYNC_EVENTS, (ctrlr->cdata.aerl + 1)); 3416 } 3417 3418 for (i = 0; i < ctrlr->num_aers; i++) { 3419 aer = &ctrlr->aer[i]; 3420 rc = nvme_ctrlr_construct_and_submit_aer(ctrlr, aer); 3421 if (rc) { 3422 NVME_CTRLR_ERRLOG(ctrlr, "nvme_ctrlr_construct_and_submit_aer failed!\n"); 3423 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 3424 return; 3425 } 3426 } 3427 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_KEEP_ALIVE_TIMEOUT, ctrlr->opts.admin_timeout_ms); 3428 } 3429 3430 static int 3431 nvme_ctrlr_configure_aer(struct spdk_nvme_ctrlr *ctrlr) 3432 { 3433 union spdk_nvme_feat_async_event_configuration config; 3434 int rc; 3435 3436 config.raw = 0; 3437 3438 if (spdk_nvme_ctrlr_is_discovery(ctrlr)) { 3439 config.bits.discovery_log_change_notice = 1; 3440 } else { 3441 config.bits.crit_warn.bits.available_spare = 1; 3442 config.bits.crit_warn.bits.temperature = 1; 3443 config.bits.crit_warn.bits.device_reliability = 1; 3444 config.bits.crit_warn.bits.read_only = 1; 3445 config.bits.crit_warn.bits.volatile_memory_backup = 1; 3446 3447 if (ctrlr->vs.raw >= SPDK_NVME_VERSION(1, 2, 0)) { 3448 if (ctrlr->cdata.oaes.ns_attribute_notices) { 3449 config.bits.ns_attr_notice = 1; 3450 } 3451 if (ctrlr->cdata.oaes.fw_activation_notices) { 3452 config.bits.fw_activation_notice = 1; 3453 } 3454 if (ctrlr->cdata.oaes.ana_change_notices) { 3455 config.bits.ana_change_notice = 1; 3456 } 3457 } 3458 if (ctrlr->vs.raw >= SPDK_NVME_VERSION(1, 3, 0) && ctrlr->cdata.lpa.telemetry) { 3459 config.bits.telemetry_log_notice = 1; 3460 } 3461 } 3462 3463 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_WAIT_FOR_CONFIGURE_AER, 3464 ctrlr->opts.admin_timeout_ms); 3465 3466 rc = nvme_ctrlr_cmd_set_async_event_config(ctrlr, config, 3467 nvme_ctrlr_configure_aer_done, 3468 ctrlr); 3469 if (rc != 0) { 3470 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 3471 return rc; 3472 } 3473 3474 return 0; 3475 } 3476 3477 struct spdk_nvme_ctrlr_process * 3478 nvme_ctrlr_get_process(struct spdk_nvme_ctrlr *ctrlr, pid_t pid) 3479 { 3480 struct spdk_nvme_ctrlr_process *active_proc; 3481 3482 TAILQ_FOREACH(active_proc, &ctrlr->active_procs, tailq) { 3483 if (active_proc->pid == pid) { 3484 return active_proc; 3485 } 3486 } 3487 3488 return NULL; 3489 } 3490 3491 struct spdk_nvme_ctrlr_process * 3492 nvme_ctrlr_get_current_process(struct spdk_nvme_ctrlr *ctrlr) 3493 { 3494 return nvme_ctrlr_get_process(ctrlr, getpid()); 3495 } 3496 3497 /** 3498 * This function will be called when a process is using the controller. 3499 * 1. For the primary process, it is called when constructing the controller. 3500 * 2. For the secondary process, it is called at probing the controller. 3501 * Note: will check whether the process is already added for the same process. 3502 */ 3503 int 3504 nvme_ctrlr_add_process(struct spdk_nvme_ctrlr *ctrlr, void *devhandle) 3505 { 3506 struct spdk_nvme_ctrlr_process *ctrlr_proc; 3507 pid_t pid = getpid(); 3508 3509 /* Check whether the process is already added or not */ 3510 if (nvme_ctrlr_get_process(ctrlr, pid)) { 3511 return 0; 3512 } 3513 3514 /* Initialize the per process properties for this ctrlr */ 3515 ctrlr_proc = spdk_zmalloc(sizeof(struct spdk_nvme_ctrlr_process), 3516 64, NULL, SPDK_ENV_NUMA_ID_ANY, SPDK_MALLOC_SHARE); 3517 if (ctrlr_proc == NULL) { 3518 NVME_CTRLR_ERRLOG(ctrlr, "failed to allocate memory to track the process props\n"); 3519 3520 return -1; 3521 } 3522 3523 ctrlr_proc->is_primary = spdk_process_is_primary(); 3524 ctrlr_proc->pid = pid; 3525 STAILQ_INIT(&ctrlr_proc->active_reqs); 3526 ctrlr_proc->devhandle = devhandle; 3527 ctrlr_proc->ref = 0; 3528 TAILQ_INIT(&ctrlr_proc->allocated_io_qpairs); 3529 STAILQ_INIT(&ctrlr_proc->async_events); 3530 3531 TAILQ_INSERT_TAIL(&ctrlr->active_procs, ctrlr_proc, tailq); 3532 3533 return 0; 3534 } 3535 3536 /** 3537 * This function will be called when the process detaches the controller. 3538 * Note: the ctrlr_lock must be held when calling this function. 3539 */ 3540 static void 3541 nvme_ctrlr_remove_process(struct spdk_nvme_ctrlr *ctrlr, 3542 struct spdk_nvme_ctrlr_process *proc) 3543 { 3544 struct spdk_nvme_qpair *qpair, *tmp_qpair; 3545 3546 assert(STAILQ_EMPTY(&proc->active_reqs)); 3547 3548 TAILQ_FOREACH_SAFE(qpair, &proc->allocated_io_qpairs, per_process_tailq, tmp_qpair) { 3549 spdk_nvme_ctrlr_free_io_qpair(qpair); 3550 } 3551 3552 TAILQ_REMOVE(&ctrlr->active_procs, proc, tailq); 3553 3554 if (ctrlr->trid.trtype == SPDK_NVME_TRANSPORT_PCIE) { 3555 spdk_pci_device_detach(proc->devhandle); 3556 } 3557 3558 spdk_free(proc); 3559 } 3560 3561 /** 3562 * This function will be called when the process exited unexpectedly 3563 * in order to free any incomplete nvme request, allocated IO qpairs 3564 * and allocated memory. 3565 * Note: the ctrlr_lock must be held when calling this function. 3566 */ 3567 static void 3568 nvme_ctrlr_cleanup_process(struct spdk_nvme_ctrlr_process *proc) 3569 { 3570 struct nvme_request *req, *tmp_req; 3571 struct spdk_nvme_qpair *qpair, *tmp_qpair; 3572 struct spdk_nvme_ctrlr_aer_completion *event; 3573 3574 STAILQ_FOREACH_SAFE(req, &proc->active_reqs, stailq, tmp_req) { 3575 STAILQ_REMOVE(&proc->active_reqs, req, nvme_request, stailq); 3576 3577 assert(req->pid == proc->pid); 3578 nvme_cleanup_user_req(req); 3579 nvme_free_request(req); 3580 } 3581 3582 /* Remove async event from each process objects event list */ 3583 while (!STAILQ_EMPTY(&proc->async_events)) { 3584 event = STAILQ_FIRST(&proc->async_events); 3585 STAILQ_REMOVE_HEAD(&proc->async_events, link); 3586 spdk_free(event); 3587 } 3588 3589 TAILQ_FOREACH_SAFE(qpair, &proc->allocated_io_qpairs, per_process_tailq, tmp_qpair) { 3590 TAILQ_REMOVE(&proc->allocated_io_qpairs, qpair, per_process_tailq); 3591 3592 /* 3593 * The process may have been killed while some qpairs were in their 3594 * completion context. Clear that flag here to allow these IO 3595 * qpairs to be deleted. 3596 */ 3597 qpair->in_completion_context = 0; 3598 3599 qpair->no_deletion_notification_needed = 1; 3600 3601 spdk_nvme_ctrlr_free_io_qpair(qpair); 3602 } 3603 3604 spdk_free(proc); 3605 } 3606 3607 /** 3608 * This function will be called when destructing the controller. 3609 * 1. There is no more admin request on this controller. 3610 * 2. Clean up any left resource allocation when its associated process is gone. 3611 */ 3612 void 3613 nvme_ctrlr_free_processes(struct spdk_nvme_ctrlr *ctrlr) 3614 { 3615 struct spdk_nvme_ctrlr_process *active_proc, *tmp; 3616 3617 /* Free all the processes' properties and make sure no pending admin IOs */ 3618 TAILQ_FOREACH_SAFE(active_proc, &ctrlr->active_procs, tailq, tmp) { 3619 TAILQ_REMOVE(&ctrlr->active_procs, active_proc, tailq); 3620 3621 assert(STAILQ_EMPTY(&active_proc->active_reqs)); 3622 3623 spdk_free(active_proc); 3624 } 3625 } 3626 3627 /** 3628 * This function will be called when any other process attaches or 3629 * detaches the controller in order to cleanup those unexpectedly 3630 * terminated processes. 3631 * Note: the ctrlr_lock must be held when calling this function. 3632 */ 3633 static int 3634 nvme_ctrlr_remove_inactive_proc(struct spdk_nvme_ctrlr *ctrlr) 3635 { 3636 struct spdk_nvme_ctrlr_process *active_proc, *tmp; 3637 int active_proc_count = 0; 3638 3639 TAILQ_FOREACH_SAFE(active_proc, &ctrlr->active_procs, tailq, tmp) { 3640 if ((kill(active_proc->pid, 0) == -1) && (errno == ESRCH)) { 3641 NVME_CTRLR_ERRLOG(ctrlr, "process %d terminated unexpected\n", active_proc->pid); 3642 3643 TAILQ_REMOVE(&ctrlr->active_procs, active_proc, tailq); 3644 3645 nvme_ctrlr_cleanup_process(active_proc); 3646 } else { 3647 active_proc_count++; 3648 } 3649 } 3650 3651 return active_proc_count; 3652 } 3653 3654 void 3655 nvme_ctrlr_proc_get_ref(struct spdk_nvme_ctrlr *ctrlr) 3656 { 3657 struct spdk_nvme_ctrlr_process *active_proc; 3658 3659 nvme_ctrlr_lock(ctrlr); 3660 3661 nvme_ctrlr_remove_inactive_proc(ctrlr); 3662 3663 active_proc = nvme_ctrlr_get_current_process(ctrlr); 3664 if (active_proc) { 3665 active_proc->ref++; 3666 } 3667 3668 nvme_ctrlr_unlock(ctrlr); 3669 } 3670 3671 void 3672 nvme_ctrlr_proc_put_ref(struct spdk_nvme_ctrlr *ctrlr) 3673 { 3674 struct spdk_nvme_ctrlr_process *active_proc; 3675 int proc_count; 3676 3677 nvme_ctrlr_lock(ctrlr); 3678 3679 proc_count = nvme_ctrlr_remove_inactive_proc(ctrlr); 3680 3681 active_proc = nvme_ctrlr_get_current_process(ctrlr); 3682 if (active_proc) { 3683 active_proc->ref--; 3684 assert(active_proc->ref >= 0); 3685 3686 /* 3687 * The last active process will be removed at the end of 3688 * the destruction of the controller. 3689 */ 3690 if (active_proc->ref == 0 && proc_count != 1) { 3691 nvme_ctrlr_remove_process(ctrlr, active_proc); 3692 } 3693 } 3694 3695 nvme_ctrlr_unlock(ctrlr); 3696 } 3697 3698 int 3699 nvme_ctrlr_get_ref_count(struct spdk_nvme_ctrlr *ctrlr) 3700 { 3701 struct spdk_nvme_ctrlr_process *active_proc; 3702 int ref = 0; 3703 3704 nvme_ctrlr_lock(ctrlr); 3705 3706 nvme_ctrlr_remove_inactive_proc(ctrlr); 3707 3708 TAILQ_FOREACH(active_proc, &ctrlr->active_procs, tailq) { 3709 ref += active_proc->ref; 3710 } 3711 3712 nvme_ctrlr_unlock(ctrlr); 3713 3714 return ref; 3715 } 3716 3717 /** 3718 * Get the PCI device handle which is only visible to its associated process. 3719 */ 3720 struct spdk_pci_device * 3721 nvme_ctrlr_proc_get_devhandle(struct spdk_nvme_ctrlr *ctrlr) 3722 { 3723 struct spdk_nvme_ctrlr_process *active_proc; 3724 struct spdk_pci_device *devhandle = NULL; 3725 3726 nvme_ctrlr_lock(ctrlr); 3727 3728 active_proc = nvme_ctrlr_get_current_process(ctrlr); 3729 if (active_proc) { 3730 devhandle = active_proc->devhandle; 3731 } 3732 3733 nvme_ctrlr_unlock(ctrlr); 3734 3735 return devhandle; 3736 } 3737 3738 static void 3739 nvme_ctrlr_process_init_vs_done(void *ctx, uint64_t value, const struct spdk_nvme_cpl *cpl) 3740 { 3741 struct spdk_nvme_ctrlr *ctrlr = ctx; 3742 3743 if (spdk_nvme_cpl_is_error(cpl)) { 3744 NVME_CTRLR_ERRLOG(ctrlr, "Failed to read the VS register\n"); 3745 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 3746 return; 3747 } 3748 3749 assert(value <= UINT32_MAX); 3750 ctrlr->vs.raw = (uint32_t)value; 3751 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_READ_CAP, NVME_TIMEOUT_INFINITE); 3752 } 3753 3754 static void 3755 nvme_ctrlr_process_init_cap_done(void *ctx, uint64_t value, const struct spdk_nvme_cpl *cpl) 3756 { 3757 struct spdk_nvme_ctrlr *ctrlr = ctx; 3758 3759 if (spdk_nvme_cpl_is_error(cpl)) { 3760 NVME_CTRLR_ERRLOG(ctrlr, "Failed to read the CAP register\n"); 3761 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 3762 return; 3763 } 3764 3765 ctrlr->cap.raw = value; 3766 nvme_ctrlr_init_cap(ctrlr); 3767 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_CHECK_EN, NVME_TIMEOUT_INFINITE); 3768 } 3769 3770 static void 3771 nvme_ctrlr_process_init_check_en(void *ctx, uint64_t value, const struct spdk_nvme_cpl *cpl) 3772 { 3773 struct spdk_nvme_ctrlr *ctrlr = ctx; 3774 enum nvme_ctrlr_state state; 3775 3776 if (spdk_nvme_cpl_is_error(cpl)) { 3777 NVME_CTRLR_ERRLOG(ctrlr, "Failed to read the CC register\n"); 3778 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 3779 return; 3780 } 3781 3782 assert(value <= UINT32_MAX); 3783 ctrlr->process_init_cc.raw = (uint32_t)value; 3784 3785 if (ctrlr->process_init_cc.bits.en) { 3786 NVME_CTRLR_DEBUGLOG(ctrlr, "CC.EN = 1\n"); 3787 state = NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_1; 3788 } else { 3789 state = NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0; 3790 } 3791 3792 nvme_ctrlr_set_state(ctrlr, state, nvme_ctrlr_get_ready_timeout(ctrlr)); 3793 } 3794 3795 static void 3796 nvme_ctrlr_process_init_set_en_0(void *ctx, uint64_t value, const struct spdk_nvme_cpl *cpl) 3797 { 3798 struct spdk_nvme_ctrlr *ctrlr = ctx; 3799 3800 if (spdk_nvme_cpl_is_error(cpl)) { 3801 NVME_CTRLR_ERRLOG(ctrlr, "Failed to write the CC register\n"); 3802 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 3803 return; 3804 } 3805 3806 /* 3807 * Wait 2.5 seconds before accessing PCI registers. 3808 * Not using sleep() to avoid blocking other controller's initialization. 3809 */ 3810 if (ctrlr->quirks & NVME_QUIRK_DELAY_BEFORE_CHK_RDY) { 3811 NVME_CTRLR_DEBUGLOG(ctrlr, "Applying quirk: delay 2.5 seconds before reading registers\n"); 3812 ctrlr->sleep_timeout_tsc = spdk_get_ticks() + (2500 * spdk_get_ticks_hz() / 1000); 3813 } 3814 3815 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0, 3816 nvme_ctrlr_get_ready_timeout(ctrlr)); 3817 } 3818 3819 static void 3820 nvme_ctrlr_process_init_set_en_0_read_cc(void *ctx, uint64_t value, const struct spdk_nvme_cpl *cpl) 3821 { 3822 struct spdk_nvme_ctrlr *ctrlr = ctx; 3823 union spdk_nvme_cc_register cc; 3824 int rc; 3825 3826 if (spdk_nvme_cpl_is_error(cpl)) { 3827 NVME_CTRLR_ERRLOG(ctrlr, "Failed to read the CC register\n"); 3828 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 3829 return; 3830 } 3831 3832 assert(value <= UINT32_MAX); 3833 cc.raw = (uint32_t)value; 3834 cc.bits.en = 0; 3835 ctrlr->process_init_cc.raw = cc.raw; 3836 3837 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_EN_0_WAIT_FOR_CC, 3838 nvme_ctrlr_get_ready_timeout(ctrlr)); 3839 3840 rc = nvme_ctrlr_set_cc_async(ctrlr, cc.raw, nvme_ctrlr_process_init_set_en_0, ctrlr); 3841 if (rc != 0) { 3842 NVME_CTRLR_ERRLOG(ctrlr, "set_cc() failed\n"); 3843 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 3844 } 3845 } 3846 3847 static void 3848 nvme_ctrlr_process_init_wait_for_ready_1(void *ctx, uint64_t value, const struct spdk_nvme_cpl *cpl) 3849 { 3850 struct spdk_nvme_ctrlr *ctrlr = ctx; 3851 union spdk_nvme_csts_register csts; 3852 3853 if (spdk_nvme_cpl_is_error(cpl)) { 3854 /* While a device is resetting, it may be unable to service MMIO reads 3855 * temporarily. Allow for this case. 3856 */ 3857 if (!ctrlr->is_failed && ctrlr->state_timeout_tsc != NVME_TIMEOUT_INFINITE) { 3858 NVME_CTRLR_DEBUGLOG(ctrlr, "Failed to read the CSTS register\n"); 3859 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_1, 3860 NVME_TIMEOUT_KEEP_EXISTING); 3861 } else { 3862 NVME_CTRLR_ERRLOG(ctrlr, "Failed to read the CSTS register\n"); 3863 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 3864 } 3865 3866 return; 3867 } 3868 3869 assert(value <= UINT32_MAX); 3870 csts.raw = (uint32_t)value; 3871 if (csts.bits.rdy == 1 || csts.bits.cfs == 1) { 3872 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_EN_0, 3873 nvme_ctrlr_get_ready_timeout(ctrlr)); 3874 } else { 3875 NVME_CTRLR_DEBUGLOG(ctrlr, "CC.EN = 1 && CSTS.RDY = 0 - waiting for reset to complete\n"); 3876 nvme_ctrlr_set_state_quiet(ctrlr, NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_1, 3877 NVME_TIMEOUT_KEEP_EXISTING); 3878 } 3879 } 3880 3881 static void 3882 nvme_ctrlr_process_init_wait_for_ready_0(void *ctx, uint64_t value, const struct spdk_nvme_cpl *cpl) 3883 { 3884 struct spdk_nvme_ctrlr *ctrlr = ctx; 3885 union spdk_nvme_csts_register csts; 3886 3887 if (spdk_nvme_cpl_is_error(cpl)) { 3888 /* While a device is resetting, it may be unable to service MMIO reads 3889 * temporarily. Allow for this case. 3890 */ 3891 if (!ctrlr->is_failed && ctrlr->state_timeout_tsc != NVME_TIMEOUT_INFINITE) { 3892 NVME_CTRLR_DEBUGLOG(ctrlr, "Failed to read the CSTS register\n"); 3893 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0, 3894 NVME_TIMEOUT_KEEP_EXISTING); 3895 } else { 3896 NVME_CTRLR_ERRLOG(ctrlr, "Failed to read the CSTS register\n"); 3897 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 3898 } 3899 3900 return; 3901 } 3902 3903 assert(value <= UINT32_MAX); 3904 csts.raw = (uint32_t)value; 3905 if (csts.bits.rdy == 0) { 3906 NVME_CTRLR_DEBUGLOG(ctrlr, "CC.EN = 0 && CSTS.RDY = 0\n"); 3907 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_DISABLED, 3908 nvme_ctrlr_get_ready_timeout(ctrlr)); 3909 } else { 3910 nvme_ctrlr_set_state_quiet(ctrlr, NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0, 3911 NVME_TIMEOUT_KEEP_EXISTING); 3912 } 3913 } 3914 3915 static void 3916 nvme_ctrlr_process_init_enable_wait_for_ready_1(void *ctx, uint64_t value, 3917 const struct spdk_nvme_cpl *cpl) 3918 { 3919 struct spdk_nvme_ctrlr *ctrlr = ctx; 3920 union spdk_nvme_csts_register csts; 3921 3922 if (spdk_nvme_cpl_is_error(cpl)) { 3923 /* While a device is resetting, it may be unable to service MMIO reads 3924 * temporarily. Allow for this case. 3925 */ 3926 if (!ctrlr->is_failed && ctrlr->state_timeout_tsc != NVME_TIMEOUT_INFINITE) { 3927 NVME_CTRLR_DEBUGLOG(ctrlr, "Failed to read the CSTS register\n"); 3928 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ENABLE_WAIT_FOR_READY_1, 3929 NVME_TIMEOUT_KEEP_EXISTING); 3930 } else { 3931 NVME_CTRLR_ERRLOG(ctrlr, "Failed to read the CSTS register\n"); 3932 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 3933 } 3934 3935 return; 3936 } 3937 3938 assert(value <= UINT32_MAX); 3939 csts.raw = value; 3940 if (csts.bits.rdy == 1) { 3941 NVME_CTRLR_DEBUGLOG(ctrlr, "CC.EN = 1 && CSTS.RDY = 1 - controller is ready\n"); 3942 /* 3943 * The controller has been enabled. 3944 * Perform the rest of initialization serially. 3945 */ 3946 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_RESET_ADMIN_QUEUE, 3947 ctrlr->opts.admin_timeout_ms); 3948 } else { 3949 nvme_ctrlr_set_state_quiet(ctrlr, NVME_CTRLR_STATE_ENABLE_WAIT_FOR_READY_1, 3950 NVME_TIMEOUT_KEEP_EXISTING); 3951 } 3952 } 3953 3954 /** 3955 * This function will be called repeatedly during initialization until the controller is ready. 3956 */ 3957 int 3958 nvme_ctrlr_process_init(struct spdk_nvme_ctrlr *ctrlr) 3959 { 3960 uint32_t ready_timeout_in_ms; 3961 uint64_t ticks; 3962 int rc = 0; 3963 3964 ticks = spdk_get_ticks(); 3965 3966 /* 3967 * May need to avoid accessing any register on the target controller 3968 * for a while. Return early without touching the FSM. 3969 * Check sleep_timeout_tsc > 0 for unit test. 3970 */ 3971 if ((ctrlr->sleep_timeout_tsc > 0) && 3972 (ticks <= ctrlr->sleep_timeout_tsc)) { 3973 return 0; 3974 } 3975 ctrlr->sleep_timeout_tsc = 0; 3976 3977 ready_timeout_in_ms = nvme_ctrlr_get_ready_timeout(ctrlr); 3978 3979 /* 3980 * Check if the current initialization step is done or has timed out. 3981 */ 3982 switch (ctrlr->state) { 3983 case NVME_CTRLR_STATE_INIT_DELAY: 3984 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_INIT, ready_timeout_in_ms); 3985 if (ctrlr->quirks & NVME_QUIRK_DELAY_BEFORE_INIT) { 3986 /* 3987 * Controller may need some delay before it's enabled. 3988 * 3989 * This is a workaround for an issue where the PCIe-attached NVMe controller 3990 * is not ready after VFIO reset. We delay the initialization rather than the 3991 * enabling itself, because this is required only for the very first enabling 3992 * - directly after a VFIO reset. 3993 */ 3994 NVME_CTRLR_DEBUGLOG(ctrlr, "Adding 2 second delay before initializing the controller\n"); 3995 ctrlr->sleep_timeout_tsc = ticks + (2000 * spdk_get_ticks_hz() / 1000); 3996 } 3997 break; 3998 3999 case NVME_CTRLR_STATE_DISCONNECTED: 4000 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_INIT, NVME_TIMEOUT_INFINITE); 4001 break; 4002 4003 case NVME_CTRLR_STATE_CONNECT_ADMINQ: /* synonymous with NVME_CTRLR_STATE_INIT and NVME_CTRLR_STATE_DISCONNECTED */ 4004 rc = nvme_transport_ctrlr_connect_qpair(ctrlr, ctrlr->adminq); 4005 if (rc == 0) { 4006 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_WAIT_FOR_CONNECT_ADMINQ, 4007 NVME_TIMEOUT_INFINITE); 4008 } else { 4009 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 4010 } 4011 break; 4012 4013 case NVME_CTRLR_STATE_WAIT_FOR_CONNECT_ADMINQ: 4014 spdk_nvme_qpair_process_completions(ctrlr->adminq, 0); 4015 4016 switch (nvme_qpair_get_state(ctrlr->adminq)) { 4017 case NVME_QPAIR_CONNECTING: 4018 if (ctrlr->is_failed) { 4019 nvme_transport_ctrlr_disconnect_qpair(ctrlr, ctrlr->adminq); 4020 break; 4021 } 4022 4023 break; 4024 case NVME_QPAIR_CONNECTED: 4025 nvme_qpair_set_state(ctrlr->adminq, NVME_QPAIR_ENABLED); 4026 /* Fall through */ 4027 case NVME_QPAIR_ENABLED: 4028 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_READ_VS, 4029 NVME_TIMEOUT_INFINITE); 4030 /* Abort any queued requests that were sent while the adminq was connecting 4031 * to avoid stalling the init process during a reset, as requests don't get 4032 * resubmitted while the controller is resetting and subsequent commands 4033 * would get queued too. 4034 */ 4035 nvme_qpair_abort_queued_reqs(ctrlr->adminq); 4036 break; 4037 case NVME_QPAIR_DISCONNECTING: 4038 assert(ctrlr->adminq->async == true); 4039 break; 4040 case NVME_QPAIR_DISCONNECTED: 4041 /* fallthrough */ 4042 default: 4043 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 4044 break; 4045 } 4046 4047 break; 4048 4049 case NVME_CTRLR_STATE_READ_VS: 4050 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_READ_VS_WAIT_FOR_VS, NVME_TIMEOUT_INFINITE); 4051 rc = nvme_ctrlr_get_vs_async(ctrlr, nvme_ctrlr_process_init_vs_done, ctrlr); 4052 break; 4053 4054 case NVME_CTRLR_STATE_READ_CAP: 4055 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_READ_CAP_WAIT_FOR_CAP, NVME_TIMEOUT_INFINITE); 4056 rc = nvme_ctrlr_get_cap_async(ctrlr, nvme_ctrlr_process_init_cap_done, ctrlr); 4057 break; 4058 4059 case NVME_CTRLR_STATE_CHECK_EN: 4060 /* Begin the hardware initialization by making sure the controller is disabled. */ 4061 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_CHECK_EN_WAIT_FOR_CC, ready_timeout_in_ms); 4062 rc = nvme_ctrlr_get_cc_async(ctrlr, nvme_ctrlr_process_init_check_en, ctrlr); 4063 break; 4064 4065 case NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_1: 4066 /* 4067 * Controller is currently enabled. We need to disable it to cause a reset. 4068 * 4069 * If CC.EN = 1 && CSTS.RDY = 0, the controller is in the process of becoming ready. 4070 * Wait for the ready bit to be 1 before disabling the controller. 4071 */ 4072 nvme_ctrlr_set_state_quiet(ctrlr, NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_1_WAIT_FOR_CSTS, 4073 NVME_TIMEOUT_KEEP_EXISTING); 4074 rc = nvme_ctrlr_get_csts_async(ctrlr, nvme_ctrlr_process_init_wait_for_ready_1, ctrlr); 4075 break; 4076 4077 case NVME_CTRLR_STATE_SET_EN_0: 4078 NVME_CTRLR_DEBUGLOG(ctrlr, "Setting CC.EN = 0\n"); 4079 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_EN_0_WAIT_FOR_CC, ready_timeout_in_ms); 4080 rc = nvme_ctrlr_get_cc_async(ctrlr, nvme_ctrlr_process_init_set_en_0_read_cc, ctrlr); 4081 break; 4082 4083 case NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0: 4084 nvme_ctrlr_set_state_quiet(ctrlr, NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0_WAIT_FOR_CSTS, 4085 NVME_TIMEOUT_KEEP_EXISTING); 4086 rc = nvme_ctrlr_get_csts_async(ctrlr, nvme_ctrlr_process_init_wait_for_ready_0, ctrlr); 4087 break; 4088 4089 case NVME_CTRLR_STATE_DISABLED: 4090 if (ctrlr->is_disconnecting) { 4091 NVME_CTRLR_DEBUGLOG(ctrlr, "Ctrlr was disabled.\n"); 4092 } else { 4093 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ENABLE, ready_timeout_in_ms); 4094 4095 /* 4096 * Delay 100us before setting CC.EN = 1. Some NVMe SSDs miss CC.EN getting 4097 * set to 1 if it is too soon after CSTS.RDY is reported as 0. 4098 */ 4099 spdk_delay_us(100); 4100 } 4101 break; 4102 4103 case NVME_CTRLR_STATE_ENABLE: 4104 NVME_CTRLR_DEBUGLOG(ctrlr, "Setting CC.EN = 1\n"); 4105 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ENABLE_WAIT_FOR_CC, ready_timeout_in_ms); 4106 rc = nvme_ctrlr_enable(ctrlr); 4107 if (rc) { 4108 NVME_CTRLR_ERRLOG(ctrlr, "Ctrlr enable failed with error: %d", rc); 4109 } 4110 return rc; 4111 4112 case NVME_CTRLR_STATE_ENABLE_WAIT_FOR_READY_1: 4113 nvme_ctrlr_set_state_quiet(ctrlr, NVME_CTRLR_STATE_ENABLE_WAIT_FOR_READY_1_WAIT_FOR_CSTS, 4114 NVME_TIMEOUT_KEEP_EXISTING); 4115 rc = nvme_ctrlr_get_csts_async(ctrlr, nvme_ctrlr_process_init_enable_wait_for_ready_1, 4116 ctrlr); 4117 break; 4118 4119 case NVME_CTRLR_STATE_RESET_ADMIN_QUEUE: 4120 nvme_transport_qpair_reset(ctrlr->adminq); 4121 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_IDENTIFY, NVME_TIMEOUT_INFINITE); 4122 break; 4123 4124 case NVME_CTRLR_STATE_IDENTIFY: 4125 rc = nvme_ctrlr_identify(ctrlr); 4126 break; 4127 4128 case NVME_CTRLR_STATE_CONFIGURE_AER: 4129 rc = nvme_ctrlr_configure_aer(ctrlr); 4130 break; 4131 4132 case NVME_CTRLR_STATE_SET_KEEP_ALIVE_TIMEOUT: 4133 rc = nvme_ctrlr_set_keep_alive_timeout(ctrlr); 4134 break; 4135 4136 case NVME_CTRLR_STATE_IDENTIFY_IOCS_SPECIFIC: 4137 rc = nvme_ctrlr_identify_iocs_specific(ctrlr); 4138 break; 4139 4140 case NVME_CTRLR_STATE_GET_ZNS_CMD_EFFECTS_LOG: 4141 rc = nvme_ctrlr_get_zns_cmd_and_effects_log(ctrlr); 4142 break; 4143 4144 case NVME_CTRLR_STATE_SET_NUM_QUEUES: 4145 nvme_ctrlr_update_nvmf_ioccsz(ctrlr); 4146 rc = nvme_ctrlr_set_num_queues(ctrlr); 4147 break; 4148 4149 case NVME_CTRLR_STATE_IDENTIFY_ACTIVE_NS: 4150 _nvme_ctrlr_identify_active_ns(ctrlr); 4151 break; 4152 4153 case NVME_CTRLR_STATE_IDENTIFY_NS: 4154 rc = nvme_ctrlr_identify_namespaces(ctrlr); 4155 break; 4156 4157 case NVME_CTRLR_STATE_IDENTIFY_ID_DESCS: 4158 rc = nvme_ctrlr_identify_id_desc_namespaces(ctrlr); 4159 break; 4160 4161 case NVME_CTRLR_STATE_IDENTIFY_NS_IOCS_SPECIFIC: 4162 rc = nvme_ctrlr_identify_namespaces_iocs_specific(ctrlr); 4163 break; 4164 4165 case NVME_CTRLR_STATE_SET_SUPPORTED_LOG_PAGES: 4166 rc = nvme_ctrlr_set_supported_log_pages(ctrlr); 4167 break; 4168 4169 case NVME_CTRLR_STATE_SET_SUPPORTED_INTEL_LOG_PAGES: 4170 rc = nvme_ctrlr_set_intel_support_log_pages(ctrlr); 4171 break; 4172 4173 case NVME_CTRLR_STATE_SET_SUPPORTED_FEATURES: 4174 nvme_ctrlr_set_supported_features(ctrlr); 4175 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_HOST_FEATURE, 4176 ctrlr->opts.admin_timeout_ms); 4177 break; 4178 4179 case NVME_CTRLR_STATE_SET_HOST_FEATURE: 4180 rc = nvme_ctrlr_set_host_feature(ctrlr); 4181 break; 4182 4183 case NVME_CTRLR_STATE_SET_DB_BUF_CFG: 4184 rc = nvme_ctrlr_set_doorbell_buffer_config(ctrlr); 4185 break; 4186 4187 case NVME_CTRLR_STATE_SET_HOST_ID: 4188 rc = nvme_ctrlr_set_host_id(ctrlr); 4189 break; 4190 4191 case NVME_CTRLR_STATE_TRANSPORT_READY: 4192 rc = nvme_transport_ctrlr_ready(ctrlr); 4193 if (rc) { 4194 NVME_CTRLR_ERRLOG(ctrlr, "Transport controller ready step failed: rc %d\n", rc); 4195 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 4196 } else { 4197 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_READY, NVME_TIMEOUT_INFINITE); 4198 } 4199 break; 4200 4201 case NVME_CTRLR_STATE_READY: 4202 NVME_CTRLR_DEBUGLOG(ctrlr, "Ctrlr already in ready state\n"); 4203 return 0; 4204 4205 case NVME_CTRLR_STATE_ERROR: 4206 NVME_CTRLR_ERRLOG(ctrlr, "Ctrlr is in error state\n"); 4207 return -1; 4208 4209 case NVME_CTRLR_STATE_READ_VS_WAIT_FOR_VS: 4210 case NVME_CTRLR_STATE_READ_CAP_WAIT_FOR_CAP: 4211 case NVME_CTRLR_STATE_CHECK_EN_WAIT_FOR_CC: 4212 case NVME_CTRLR_STATE_SET_EN_0_WAIT_FOR_CC: 4213 case NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_1_WAIT_FOR_CSTS: 4214 case NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0_WAIT_FOR_CSTS: 4215 case NVME_CTRLR_STATE_ENABLE_WAIT_FOR_CC: 4216 case NVME_CTRLR_STATE_ENABLE_WAIT_FOR_READY_1_WAIT_FOR_CSTS: 4217 case NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY: 4218 case NVME_CTRLR_STATE_WAIT_FOR_CONFIGURE_AER: 4219 case NVME_CTRLR_STATE_WAIT_FOR_KEEP_ALIVE_TIMEOUT: 4220 case NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_IOCS_SPECIFIC: 4221 case NVME_CTRLR_STATE_WAIT_FOR_GET_ZNS_CMD_EFFECTS_LOG: 4222 case NVME_CTRLR_STATE_WAIT_FOR_SET_NUM_QUEUES: 4223 case NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_ACTIVE_NS: 4224 case NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_NS: 4225 case NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_ID_DESCS: 4226 case NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_NS_IOCS_SPECIFIC: 4227 case NVME_CTRLR_STATE_WAIT_FOR_SUPPORTED_INTEL_LOG_PAGES: 4228 case NVME_CTRLR_STATE_WAIT_FOR_SET_HOST_FEATURE: 4229 case NVME_CTRLR_STATE_WAIT_FOR_DB_BUF_CFG: 4230 case NVME_CTRLR_STATE_WAIT_FOR_HOST_ID: 4231 /* 4232 * nvme_ctrlr_process_init() may be called from the completion context 4233 * for the admin qpair. Avoid recursive calls for this case. 4234 */ 4235 if (!ctrlr->adminq->in_completion_context) { 4236 spdk_nvme_qpair_process_completions(ctrlr->adminq, 0); 4237 } 4238 break; 4239 4240 default: 4241 assert(0); 4242 return -1; 4243 } 4244 4245 if (rc) { 4246 NVME_CTRLR_ERRLOG(ctrlr, "Ctrlr operation failed with error: %d, ctrlr state: %d (%s)\n", 4247 rc, ctrlr->state, nvme_ctrlr_state_string(ctrlr->state)); 4248 } 4249 4250 /* Note: we use the ticks captured when we entered this function. 4251 * This covers environments where the SPDK process gets swapped out after 4252 * we tried to advance the state but before we check the timeout here. 4253 * It is not normal for this to happen, but harmless to handle it in this 4254 * way. 4255 */ 4256 if (ctrlr->state_timeout_tsc != NVME_TIMEOUT_INFINITE && 4257 ticks > ctrlr->state_timeout_tsc) { 4258 NVME_CTRLR_ERRLOG(ctrlr, "Initialization timed out in state %d (%s)\n", 4259 ctrlr->state, nvme_ctrlr_state_string(ctrlr->state)); 4260 return -1; 4261 } 4262 4263 return rc; 4264 } 4265 4266 int 4267 nvme_robust_mutex_init_recursive_shared(pthread_mutex_t *mtx) 4268 { 4269 pthread_mutexattr_t attr; 4270 int rc = 0; 4271 4272 if (pthread_mutexattr_init(&attr)) { 4273 return -1; 4274 } 4275 if (pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE) || 4276 #ifndef __FreeBSD__ 4277 pthread_mutexattr_setrobust(&attr, PTHREAD_MUTEX_ROBUST) || 4278 pthread_mutexattr_setpshared(&attr, PTHREAD_PROCESS_SHARED) || 4279 #endif 4280 pthread_mutex_init(mtx, &attr)) { 4281 rc = -1; 4282 } 4283 pthread_mutexattr_destroy(&attr); 4284 return rc; 4285 } 4286 4287 int 4288 nvme_ctrlr_construct(struct spdk_nvme_ctrlr *ctrlr) 4289 { 4290 int rc; 4291 4292 if (ctrlr->trid.trtype == SPDK_NVME_TRANSPORT_PCIE) { 4293 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_INIT_DELAY, NVME_TIMEOUT_INFINITE); 4294 } else { 4295 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_INIT, NVME_TIMEOUT_INFINITE); 4296 } 4297 4298 if (ctrlr->opts.admin_queue_size > SPDK_NVME_ADMIN_QUEUE_MAX_ENTRIES) { 4299 NVME_CTRLR_ERRLOG(ctrlr, "admin_queue_size %u exceeds max defined by NVMe spec, use max value\n", 4300 ctrlr->opts.admin_queue_size); 4301 ctrlr->opts.admin_queue_size = SPDK_NVME_ADMIN_QUEUE_MAX_ENTRIES; 4302 } 4303 4304 if (ctrlr->quirks & NVME_QUIRK_MINIMUM_ADMIN_QUEUE_SIZE && 4305 (ctrlr->opts.admin_queue_size % SPDK_NVME_ADMIN_QUEUE_QUIRK_ENTRIES_MULTIPLE) != 0) { 4306 NVME_CTRLR_ERRLOG(ctrlr, 4307 "admin_queue_size %u is invalid for this NVMe device, adjust to next multiple\n", 4308 ctrlr->opts.admin_queue_size); 4309 ctrlr->opts.admin_queue_size = SPDK_ALIGN_CEIL(ctrlr->opts.admin_queue_size, 4310 SPDK_NVME_ADMIN_QUEUE_QUIRK_ENTRIES_MULTIPLE); 4311 } 4312 4313 if (ctrlr->opts.admin_queue_size < SPDK_NVME_ADMIN_QUEUE_MIN_ENTRIES) { 4314 NVME_CTRLR_ERRLOG(ctrlr, 4315 "admin_queue_size %u is less than minimum defined by NVMe spec, use min value\n", 4316 ctrlr->opts.admin_queue_size); 4317 ctrlr->opts.admin_queue_size = SPDK_NVME_ADMIN_QUEUE_MIN_ENTRIES; 4318 } 4319 4320 ctrlr->flags = 0; 4321 ctrlr->free_io_qids = NULL; 4322 ctrlr->is_resetting = false; 4323 ctrlr->is_failed = false; 4324 ctrlr->is_destructed = false; 4325 4326 TAILQ_INIT(&ctrlr->active_io_qpairs); 4327 STAILQ_INIT(&ctrlr->queued_aborts); 4328 ctrlr->outstanding_aborts = 0; 4329 4330 ctrlr->ana_log_page = NULL; 4331 ctrlr->ana_log_page_size = 0; 4332 4333 rc = nvme_robust_mutex_init_recursive_shared(&ctrlr->ctrlr_lock); 4334 if (rc != 0) { 4335 return rc; 4336 } 4337 4338 TAILQ_INIT(&ctrlr->active_procs); 4339 STAILQ_INIT(&ctrlr->register_operations); 4340 4341 RB_INIT(&ctrlr->ns); 4342 4343 return rc; 4344 } 4345 4346 static void 4347 nvme_ctrlr_init_cap(struct spdk_nvme_ctrlr *ctrlr) 4348 { 4349 if (ctrlr->cap.bits.ams & SPDK_NVME_CAP_AMS_WRR) { 4350 ctrlr->flags |= SPDK_NVME_CTRLR_WRR_SUPPORTED; 4351 } 4352 4353 ctrlr->min_page_size = 1u << (12 + ctrlr->cap.bits.mpsmin); 4354 4355 /* For now, always select page_size == min_page_size. */ 4356 ctrlr->page_size = ctrlr->min_page_size; 4357 4358 ctrlr->opts.io_queue_size = spdk_max(ctrlr->opts.io_queue_size, SPDK_NVME_IO_QUEUE_MIN_ENTRIES); 4359 ctrlr->opts.io_queue_size = spdk_min(ctrlr->opts.io_queue_size, MAX_IO_QUEUE_ENTRIES); 4360 if (ctrlr->quirks & NVME_QUIRK_MINIMUM_IO_QUEUE_SIZE && 4361 ctrlr->opts.io_queue_size == DEFAULT_IO_QUEUE_SIZE) { 4362 /* If the user specifically set an IO queue size different than the 4363 * default, use that value. Otherwise overwrite with the quirked value. 4364 * This allows this quirk to be overridden when necessary. 4365 * However, cap.mqes still needs to be respected. 4366 */ 4367 ctrlr->opts.io_queue_size = DEFAULT_IO_QUEUE_SIZE_FOR_QUIRK; 4368 } 4369 ctrlr->opts.io_queue_size = spdk_min(ctrlr->opts.io_queue_size, ctrlr->cap.bits.mqes + 1u); 4370 4371 ctrlr->opts.io_queue_requests = spdk_max(ctrlr->opts.io_queue_requests, ctrlr->opts.io_queue_size); 4372 } 4373 4374 void 4375 nvme_ctrlr_destruct_finish(struct spdk_nvme_ctrlr *ctrlr) 4376 { 4377 int rc; 4378 4379 if (ctrlr->lock_depth > 0) { 4380 SPDK_ERRLOG("lock currently held (depth=%d)!\n", ctrlr->lock_depth); 4381 assert(false); 4382 } 4383 4384 rc = pthread_mutex_destroy(&ctrlr->ctrlr_lock); 4385 if (rc) { 4386 SPDK_ERRLOG("could not destroy ctrlr_lock: %s\n", spdk_strerror(rc)); 4387 assert(false); 4388 } 4389 4390 nvme_ctrlr_free_processes(ctrlr); 4391 } 4392 4393 void 4394 nvme_ctrlr_destruct_async(struct spdk_nvme_ctrlr *ctrlr, 4395 struct nvme_ctrlr_detach_ctx *ctx) 4396 { 4397 struct spdk_nvme_qpair *qpair, *tmp; 4398 4399 NVME_CTRLR_DEBUGLOG(ctrlr, "Prepare to destruct SSD\n"); 4400 4401 ctrlr->prepare_for_reset = false; 4402 ctrlr->is_destructed = true; 4403 4404 spdk_nvme_qpair_process_completions(ctrlr->adminq, 0); 4405 4406 nvme_ctrlr_abort_queued_aborts(ctrlr); 4407 nvme_transport_admin_qpair_abort_aers(ctrlr->adminq); 4408 4409 TAILQ_FOREACH_SAFE(qpair, &ctrlr->active_io_qpairs, tailq, tmp) { 4410 spdk_nvme_ctrlr_free_io_qpair(qpair); 4411 } 4412 4413 nvme_ctrlr_free_doorbell_buffer(ctrlr); 4414 nvme_ctrlr_free_iocs_specific_data(ctrlr); 4415 4416 nvme_ctrlr_shutdown_async(ctrlr, ctx); 4417 } 4418 4419 int 4420 nvme_ctrlr_destruct_poll_async(struct spdk_nvme_ctrlr *ctrlr, 4421 struct nvme_ctrlr_detach_ctx *ctx) 4422 { 4423 struct spdk_nvme_ns *ns, *tmp_ns; 4424 int rc = 0; 4425 4426 if (!ctx->shutdown_complete) { 4427 rc = nvme_ctrlr_shutdown_poll_async(ctrlr, ctx); 4428 if (rc == -EAGAIN) { 4429 return -EAGAIN; 4430 } 4431 /* Destruct ctrlr forcefully for any other error. */ 4432 } 4433 4434 if (ctx->cb_fn) { 4435 ctx->cb_fn(ctrlr); 4436 } 4437 4438 nvme_transport_ctrlr_disconnect_qpair(ctrlr, ctrlr->adminq); 4439 4440 RB_FOREACH_SAFE(ns, nvme_ns_tree, &ctrlr->ns, tmp_ns) { 4441 nvme_ctrlr_destruct_namespace(ctrlr, ns->id); 4442 RB_REMOVE(nvme_ns_tree, &ctrlr->ns, ns); 4443 spdk_free(ns); 4444 } 4445 4446 ctrlr->active_ns_count = 0; 4447 4448 spdk_bit_array_free(&ctrlr->free_io_qids); 4449 4450 free(ctrlr->ana_log_page); 4451 free(ctrlr->copied_ana_desc); 4452 ctrlr->ana_log_page = NULL; 4453 ctrlr->copied_ana_desc = NULL; 4454 ctrlr->ana_log_page_size = 0; 4455 4456 nvme_transport_ctrlr_destruct(ctrlr); 4457 4458 return rc; 4459 } 4460 4461 void 4462 nvme_ctrlr_destruct(struct spdk_nvme_ctrlr *ctrlr) 4463 { 4464 struct nvme_ctrlr_detach_ctx ctx = { .ctrlr = ctrlr }; 4465 int rc; 4466 4467 nvme_ctrlr_destruct_async(ctrlr, &ctx); 4468 4469 while (1) { 4470 rc = nvme_ctrlr_destruct_poll_async(ctrlr, &ctx); 4471 if (rc != -EAGAIN) { 4472 break; 4473 } 4474 nvme_delay(1000); 4475 } 4476 } 4477 4478 int 4479 nvme_ctrlr_submit_admin_request(struct spdk_nvme_ctrlr *ctrlr, 4480 struct nvme_request *req) 4481 { 4482 return nvme_qpair_submit_request(ctrlr->adminq, req); 4483 } 4484 4485 static void 4486 nvme_keep_alive_completion(void *cb_ctx, const struct spdk_nvme_cpl *cpl) 4487 { 4488 /* Do nothing */ 4489 } 4490 4491 /* 4492 * Check if we need to send a Keep Alive command. 4493 * Caller must hold ctrlr->ctrlr_lock. 4494 */ 4495 static int 4496 nvme_ctrlr_keep_alive(struct spdk_nvme_ctrlr *ctrlr) 4497 { 4498 uint64_t now; 4499 struct nvme_request *req; 4500 struct spdk_nvme_cmd *cmd; 4501 int rc = 0; 4502 4503 now = spdk_get_ticks(); 4504 if (now < ctrlr->next_keep_alive_tick) { 4505 return rc; 4506 } 4507 4508 req = nvme_allocate_request_null(ctrlr->adminq, nvme_keep_alive_completion, NULL); 4509 if (req == NULL) { 4510 return rc; 4511 } 4512 4513 cmd = &req->cmd; 4514 cmd->opc = SPDK_NVME_OPC_KEEP_ALIVE; 4515 4516 rc = nvme_ctrlr_submit_admin_request(ctrlr, req); 4517 if (rc != 0) { 4518 NVME_CTRLR_ERRLOG(ctrlr, "Submitting Keep Alive failed\n"); 4519 rc = -ENXIO; 4520 } 4521 4522 ctrlr->next_keep_alive_tick = now + ctrlr->keep_alive_interval_ticks; 4523 return rc; 4524 } 4525 4526 int32_t 4527 spdk_nvme_ctrlr_process_admin_completions(struct spdk_nvme_ctrlr *ctrlr) 4528 { 4529 int32_t num_completions; 4530 int32_t rc; 4531 struct spdk_nvme_ctrlr_process *active_proc; 4532 4533 nvme_ctrlr_lock(ctrlr); 4534 4535 if (ctrlr->keep_alive_interval_ticks) { 4536 rc = nvme_ctrlr_keep_alive(ctrlr); 4537 if (rc) { 4538 nvme_ctrlr_unlock(ctrlr); 4539 return rc; 4540 } 4541 } 4542 4543 rc = nvme_io_msg_process(ctrlr); 4544 if (rc < 0) { 4545 nvme_ctrlr_unlock(ctrlr); 4546 return rc; 4547 } 4548 num_completions = rc; 4549 4550 rc = spdk_nvme_qpair_process_completions(ctrlr->adminq, 0); 4551 4552 /* Each process has an async list, complete the ones for this process object */ 4553 active_proc = nvme_ctrlr_get_current_process(ctrlr); 4554 if (active_proc) { 4555 nvme_ctrlr_complete_queued_async_events(ctrlr); 4556 } 4557 4558 if (rc == -ENXIO && ctrlr->is_disconnecting) { 4559 nvme_ctrlr_disconnect_done(ctrlr); 4560 } 4561 4562 nvme_ctrlr_unlock(ctrlr); 4563 4564 if (rc < 0) { 4565 num_completions = rc; 4566 } else { 4567 num_completions += rc; 4568 } 4569 4570 return num_completions; 4571 } 4572 4573 const struct spdk_nvme_ctrlr_data * 4574 spdk_nvme_ctrlr_get_data(struct spdk_nvme_ctrlr *ctrlr) 4575 { 4576 return &ctrlr->cdata; 4577 } 4578 4579 union spdk_nvme_csts_register spdk_nvme_ctrlr_get_regs_csts(struct spdk_nvme_ctrlr *ctrlr) 4580 { 4581 union spdk_nvme_csts_register csts; 4582 4583 if (nvme_ctrlr_get_csts(ctrlr, &csts)) { 4584 csts.raw = SPDK_NVME_INVALID_REGISTER_VALUE; 4585 } 4586 return csts; 4587 } 4588 4589 union spdk_nvme_cc_register spdk_nvme_ctrlr_get_regs_cc(struct spdk_nvme_ctrlr *ctrlr) 4590 { 4591 union spdk_nvme_cc_register cc; 4592 4593 if (nvme_ctrlr_get_cc(ctrlr, &cc)) { 4594 cc.raw = SPDK_NVME_INVALID_REGISTER_VALUE; 4595 } 4596 return cc; 4597 } 4598 4599 union spdk_nvme_cap_register spdk_nvme_ctrlr_get_regs_cap(struct spdk_nvme_ctrlr *ctrlr) 4600 { 4601 return ctrlr->cap; 4602 } 4603 4604 union spdk_nvme_vs_register spdk_nvme_ctrlr_get_regs_vs(struct spdk_nvme_ctrlr *ctrlr) 4605 { 4606 return ctrlr->vs; 4607 } 4608 4609 union spdk_nvme_cmbsz_register spdk_nvme_ctrlr_get_regs_cmbsz(struct spdk_nvme_ctrlr *ctrlr) 4610 { 4611 union spdk_nvme_cmbsz_register cmbsz; 4612 4613 if (nvme_ctrlr_get_cmbsz(ctrlr, &cmbsz)) { 4614 cmbsz.raw = 0; 4615 } 4616 4617 return cmbsz; 4618 } 4619 4620 union spdk_nvme_pmrcap_register spdk_nvme_ctrlr_get_regs_pmrcap(struct spdk_nvme_ctrlr *ctrlr) 4621 { 4622 union spdk_nvme_pmrcap_register pmrcap; 4623 4624 if (nvme_ctrlr_get_pmrcap(ctrlr, &pmrcap)) { 4625 pmrcap.raw = 0; 4626 } 4627 4628 return pmrcap; 4629 } 4630 4631 union spdk_nvme_bpinfo_register spdk_nvme_ctrlr_get_regs_bpinfo(struct spdk_nvme_ctrlr *ctrlr) 4632 { 4633 union spdk_nvme_bpinfo_register bpinfo; 4634 4635 if (nvme_ctrlr_get_bpinfo(ctrlr, &bpinfo)) { 4636 bpinfo.raw = 0; 4637 } 4638 4639 return bpinfo; 4640 } 4641 4642 uint64_t 4643 spdk_nvme_ctrlr_get_pmrsz(struct spdk_nvme_ctrlr *ctrlr) 4644 { 4645 return ctrlr->pmr_size; 4646 } 4647 4648 uint32_t 4649 spdk_nvme_ctrlr_get_num_ns(struct spdk_nvme_ctrlr *ctrlr) 4650 { 4651 return ctrlr->cdata.nn; 4652 } 4653 4654 bool 4655 spdk_nvme_ctrlr_is_active_ns(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid) 4656 { 4657 struct spdk_nvme_ns tmp, *ns; 4658 4659 tmp.id = nsid; 4660 ns = RB_FIND(nvme_ns_tree, &ctrlr->ns, &tmp); 4661 4662 if (ns != NULL) { 4663 return ns->active; 4664 } 4665 4666 return false; 4667 } 4668 4669 uint32_t 4670 spdk_nvme_ctrlr_get_first_active_ns(struct spdk_nvme_ctrlr *ctrlr) 4671 { 4672 struct spdk_nvme_ns *ns; 4673 4674 ns = RB_MIN(nvme_ns_tree, &ctrlr->ns); 4675 if (ns == NULL) { 4676 return 0; 4677 } 4678 4679 while (ns != NULL) { 4680 if (ns->active) { 4681 return ns->id; 4682 } 4683 4684 ns = RB_NEXT(nvme_ns_tree, &ctrlr->ns, ns); 4685 } 4686 4687 return 0; 4688 } 4689 4690 uint32_t 4691 spdk_nvme_ctrlr_get_next_active_ns(struct spdk_nvme_ctrlr *ctrlr, uint32_t prev_nsid) 4692 { 4693 struct spdk_nvme_ns tmp, *ns; 4694 4695 tmp.id = prev_nsid; 4696 ns = RB_FIND(nvme_ns_tree, &ctrlr->ns, &tmp); 4697 if (ns == NULL) { 4698 return 0; 4699 } 4700 4701 ns = RB_NEXT(nvme_ns_tree, &ctrlr->ns, ns); 4702 while (ns != NULL) { 4703 if (ns->active) { 4704 return ns->id; 4705 } 4706 4707 ns = RB_NEXT(nvme_ns_tree, &ctrlr->ns, ns); 4708 } 4709 4710 return 0; 4711 } 4712 4713 struct spdk_nvme_ns * 4714 spdk_nvme_ctrlr_get_ns(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid) 4715 { 4716 struct spdk_nvme_ns tmp; 4717 struct spdk_nvme_ns *ns; 4718 4719 if (nsid < 1 || nsid > ctrlr->cdata.nn) { 4720 return NULL; 4721 } 4722 4723 nvme_ctrlr_lock(ctrlr); 4724 4725 tmp.id = nsid; 4726 ns = RB_FIND(nvme_ns_tree, &ctrlr->ns, &tmp); 4727 4728 if (ns == NULL) { 4729 ns = spdk_zmalloc(sizeof(struct spdk_nvme_ns), 64, NULL, SPDK_ENV_NUMA_ID_ANY, SPDK_MALLOC_SHARE); 4730 if (ns == NULL) { 4731 nvme_ctrlr_unlock(ctrlr); 4732 return NULL; 4733 } 4734 4735 NVME_CTRLR_DEBUGLOG(ctrlr, "Namespace %u was added\n", nsid); 4736 ns->id = nsid; 4737 RB_INSERT(nvme_ns_tree, &ctrlr->ns, ns); 4738 } 4739 4740 nvme_ctrlr_unlock(ctrlr); 4741 4742 return ns; 4743 } 4744 4745 struct spdk_pci_device * 4746 spdk_nvme_ctrlr_get_pci_device(struct spdk_nvme_ctrlr *ctrlr) 4747 { 4748 if (ctrlr == NULL) { 4749 return NULL; 4750 } 4751 4752 if (ctrlr->trid.trtype != SPDK_NVME_TRANSPORT_PCIE) { 4753 return NULL; 4754 } 4755 4756 return nvme_ctrlr_proc_get_devhandle(ctrlr); 4757 } 4758 4759 int32_t 4760 spdk_nvme_ctrlr_get_numa_id(struct spdk_nvme_ctrlr *ctrlr) 4761 { 4762 if (ctrlr->numa.id_valid) { 4763 return ctrlr->numa.id; 4764 } else { 4765 return SPDK_ENV_NUMA_ID_ANY; 4766 } 4767 } 4768 4769 uint16_t 4770 spdk_nvme_ctrlr_get_id(struct spdk_nvme_ctrlr *ctrlr) 4771 { 4772 return ctrlr->cntlid; 4773 } 4774 4775 uint32_t 4776 spdk_nvme_ctrlr_get_max_xfer_size(const struct spdk_nvme_ctrlr *ctrlr) 4777 { 4778 return ctrlr->max_xfer_size; 4779 } 4780 4781 uint16_t 4782 spdk_nvme_ctrlr_get_max_sges(const struct spdk_nvme_ctrlr *ctrlr) 4783 { 4784 if (ctrlr->flags & SPDK_NVME_CTRLR_SGL_SUPPORTED) { 4785 return ctrlr->max_sges; 4786 } else { 4787 return UINT16_MAX; 4788 } 4789 } 4790 4791 void 4792 spdk_nvme_ctrlr_register_aer_callback(struct spdk_nvme_ctrlr *ctrlr, 4793 spdk_nvme_aer_cb aer_cb_fn, 4794 void *aer_cb_arg) 4795 { 4796 struct spdk_nvme_ctrlr_process *active_proc; 4797 4798 nvme_ctrlr_lock(ctrlr); 4799 4800 active_proc = nvme_ctrlr_get_current_process(ctrlr); 4801 if (active_proc) { 4802 active_proc->aer_cb_fn = aer_cb_fn; 4803 active_proc->aer_cb_arg = aer_cb_arg; 4804 } 4805 4806 nvme_ctrlr_unlock(ctrlr); 4807 } 4808 4809 void 4810 spdk_nvme_ctrlr_disable_read_changed_ns_list_log_page(struct spdk_nvme_ctrlr *ctrlr) 4811 { 4812 ctrlr->opts.disable_read_changed_ns_list_log_page = true; 4813 } 4814 4815 void 4816 spdk_nvme_ctrlr_register_timeout_callback(struct spdk_nvme_ctrlr *ctrlr, 4817 uint64_t timeout_io_us, uint64_t timeout_admin_us, 4818 spdk_nvme_timeout_cb cb_fn, void *cb_arg) 4819 { 4820 struct spdk_nvme_ctrlr_process *active_proc; 4821 4822 nvme_ctrlr_lock(ctrlr); 4823 4824 active_proc = nvme_ctrlr_get_current_process(ctrlr); 4825 if (active_proc) { 4826 active_proc->timeout_io_ticks = timeout_io_us * spdk_get_ticks_hz() / 1000000ULL; 4827 active_proc->timeout_admin_ticks = timeout_admin_us * spdk_get_ticks_hz() / 1000000ULL; 4828 active_proc->timeout_cb_fn = cb_fn; 4829 active_proc->timeout_cb_arg = cb_arg; 4830 } 4831 4832 ctrlr->timeout_enabled = true; 4833 4834 nvme_ctrlr_unlock(ctrlr); 4835 } 4836 4837 bool 4838 spdk_nvme_ctrlr_is_log_page_supported(struct spdk_nvme_ctrlr *ctrlr, uint8_t log_page) 4839 { 4840 /* No bounds check necessary, since log_page is uint8_t and log_page_supported has 256 entries */ 4841 SPDK_STATIC_ASSERT(sizeof(ctrlr->log_page_supported) == 256, "log_page_supported size mismatch"); 4842 return ctrlr->log_page_supported[log_page]; 4843 } 4844 4845 bool 4846 spdk_nvme_ctrlr_is_feature_supported(struct spdk_nvme_ctrlr *ctrlr, uint8_t feature_code) 4847 { 4848 /* No bounds check necessary, since feature_code is uint8_t and feature_supported has 256 entries */ 4849 SPDK_STATIC_ASSERT(sizeof(ctrlr->feature_supported) == 256, "feature_supported size mismatch"); 4850 return ctrlr->feature_supported[feature_code]; 4851 } 4852 4853 int 4854 spdk_nvme_ctrlr_attach_ns(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid, 4855 struct spdk_nvme_ctrlr_list *payload) 4856 { 4857 struct nvme_completion_poll_status *status; 4858 struct spdk_nvme_ns *ns; 4859 int res; 4860 4861 if (nsid == 0) { 4862 return -EINVAL; 4863 } 4864 4865 status = calloc(1, sizeof(*status)); 4866 if (!status) { 4867 NVME_CTRLR_ERRLOG(ctrlr, "Failed to allocate status tracker\n"); 4868 return -ENOMEM; 4869 } 4870 4871 res = nvme_ctrlr_cmd_attach_ns(ctrlr, nsid, payload, 4872 nvme_completion_poll_cb, status); 4873 if (res) { 4874 free(status); 4875 return res; 4876 } 4877 if (nvme_wait_for_completion_robust_lock(ctrlr->adminq, status, &ctrlr->ctrlr_lock)) { 4878 NVME_CTRLR_ERRLOG(ctrlr, "spdk_nvme_ctrlr_attach_ns failed!\n"); 4879 if (!status->timed_out) { 4880 free(status); 4881 } 4882 return -ENXIO; 4883 } 4884 free(status); 4885 4886 res = nvme_ctrlr_identify_active_ns(ctrlr); 4887 if (res) { 4888 return res; 4889 } 4890 4891 ns = spdk_nvme_ctrlr_get_ns(ctrlr, nsid); 4892 if (ns == NULL) { 4893 NVME_CTRLR_ERRLOG(ctrlr, "spdk_nvme_ctrlr_get_ns failed!\n"); 4894 return -ENXIO; 4895 } 4896 4897 return nvme_ns_construct(ns, nsid, ctrlr); 4898 } 4899 4900 int 4901 spdk_nvme_ctrlr_detach_ns(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid, 4902 struct spdk_nvme_ctrlr_list *payload) 4903 { 4904 struct nvme_completion_poll_status *status; 4905 int res; 4906 4907 if (nsid == 0) { 4908 return -EINVAL; 4909 } 4910 4911 status = calloc(1, sizeof(*status)); 4912 if (!status) { 4913 NVME_CTRLR_ERRLOG(ctrlr, "Failed to allocate status tracker\n"); 4914 return -ENOMEM; 4915 } 4916 4917 res = nvme_ctrlr_cmd_detach_ns(ctrlr, nsid, payload, 4918 nvme_completion_poll_cb, status); 4919 if (res) { 4920 free(status); 4921 return res; 4922 } 4923 if (nvme_wait_for_completion_robust_lock(ctrlr->adminq, status, &ctrlr->ctrlr_lock)) { 4924 NVME_CTRLR_ERRLOG(ctrlr, "spdk_nvme_ctrlr_detach_ns failed!\n"); 4925 if (!status->timed_out) { 4926 free(status); 4927 } 4928 return -ENXIO; 4929 } 4930 free(status); 4931 4932 return nvme_ctrlr_identify_active_ns(ctrlr); 4933 } 4934 4935 uint32_t 4936 spdk_nvme_ctrlr_create_ns(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_ns_data *payload) 4937 { 4938 struct nvme_completion_poll_status *status; 4939 int res; 4940 uint32_t nsid; 4941 4942 status = calloc(1, sizeof(*status)); 4943 if (!status) { 4944 NVME_CTRLR_ERRLOG(ctrlr, "Failed to allocate status tracker\n"); 4945 return 0; 4946 } 4947 4948 res = nvme_ctrlr_cmd_create_ns(ctrlr, payload, nvme_completion_poll_cb, status); 4949 if (res) { 4950 free(status); 4951 return 0; 4952 } 4953 if (nvme_wait_for_completion_robust_lock(ctrlr->adminq, status, &ctrlr->ctrlr_lock)) { 4954 NVME_CTRLR_ERRLOG(ctrlr, "spdk_nvme_ctrlr_create_ns failed!\n"); 4955 if (!status->timed_out) { 4956 free(status); 4957 } 4958 return 0; 4959 } 4960 4961 nsid = status->cpl.cdw0; 4962 free(status); 4963 4964 assert(nsid > 0); 4965 4966 /* Return the namespace ID that was created */ 4967 return nsid; 4968 } 4969 4970 int 4971 spdk_nvme_ctrlr_delete_ns(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid) 4972 { 4973 struct nvme_completion_poll_status *status; 4974 int res; 4975 4976 if (nsid == 0) { 4977 return -EINVAL; 4978 } 4979 4980 status = calloc(1, sizeof(*status)); 4981 if (!status) { 4982 NVME_CTRLR_ERRLOG(ctrlr, "Failed to allocate status tracker\n"); 4983 return -ENOMEM; 4984 } 4985 4986 res = nvme_ctrlr_cmd_delete_ns(ctrlr, nsid, nvme_completion_poll_cb, status); 4987 if (res) { 4988 free(status); 4989 return res; 4990 } 4991 if (nvme_wait_for_completion_robust_lock(ctrlr->adminq, status, &ctrlr->ctrlr_lock)) { 4992 NVME_CTRLR_ERRLOG(ctrlr, "spdk_nvme_ctrlr_delete_ns failed!\n"); 4993 if (!status->timed_out) { 4994 free(status); 4995 } 4996 return -ENXIO; 4997 } 4998 free(status); 4999 5000 return nvme_ctrlr_identify_active_ns(ctrlr); 5001 } 5002 5003 int 5004 spdk_nvme_ctrlr_format(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid, 5005 struct spdk_nvme_format *format) 5006 { 5007 struct nvme_completion_poll_status *status; 5008 int res; 5009 5010 status = calloc(1, sizeof(*status)); 5011 if (!status) { 5012 NVME_CTRLR_ERRLOG(ctrlr, "Failed to allocate status tracker\n"); 5013 return -ENOMEM; 5014 } 5015 5016 res = nvme_ctrlr_cmd_format(ctrlr, nsid, format, nvme_completion_poll_cb, 5017 status); 5018 if (res) { 5019 free(status); 5020 return res; 5021 } 5022 if (nvme_wait_for_completion_robust_lock(ctrlr->adminq, status, &ctrlr->ctrlr_lock)) { 5023 NVME_CTRLR_ERRLOG(ctrlr, "spdk_nvme_ctrlr_format failed!\n"); 5024 if (!status->timed_out) { 5025 free(status); 5026 } 5027 return -ENXIO; 5028 } 5029 free(status); 5030 5031 return spdk_nvme_ctrlr_reset(ctrlr); 5032 } 5033 5034 int 5035 spdk_nvme_ctrlr_update_firmware(struct spdk_nvme_ctrlr *ctrlr, void *payload, uint32_t size, 5036 int slot, enum spdk_nvme_fw_commit_action commit_action, struct spdk_nvme_status *completion_status) 5037 { 5038 struct spdk_nvme_fw_commit fw_commit; 5039 struct nvme_completion_poll_status *status; 5040 int res; 5041 unsigned int size_remaining; 5042 unsigned int offset; 5043 unsigned int transfer; 5044 uint8_t *p; 5045 5046 if (!completion_status) { 5047 return -EINVAL; 5048 } 5049 memset(completion_status, 0, sizeof(struct spdk_nvme_status)); 5050 if (size % 4) { 5051 NVME_CTRLR_ERRLOG(ctrlr, "spdk_nvme_ctrlr_update_firmware invalid size!\n"); 5052 return -1; 5053 } 5054 5055 /* Current support only for SPDK_NVME_FW_COMMIT_REPLACE_IMG 5056 * and SPDK_NVME_FW_COMMIT_REPLACE_AND_ENABLE_IMG 5057 */ 5058 if ((commit_action != SPDK_NVME_FW_COMMIT_REPLACE_IMG) && 5059 (commit_action != SPDK_NVME_FW_COMMIT_REPLACE_AND_ENABLE_IMG)) { 5060 NVME_CTRLR_ERRLOG(ctrlr, "spdk_nvme_ctrlr_update_firmware invalid command!\n"); 5061 return -1; 5062 } 5063 5064 status = calloc(1, sizeof(*status)); 5065 if (!status) { 5066 NVME_CTRLR_ERRLOG(ctrlr, "Failed to allocate status tracker\n"); 5067 return -ENOMEM; 5068 } 5069 5070 /* Firmware download */ 5071 size_remaining = size; 5072 offset = 0; 5073 p = payload; 5074 5075 while (size_remaining > 0) { 5076 transfer = spdk_min(size_remaining, ctrlr->min_page_size); 5077 5078 memset(status, 0, sizeof(*status)); 5079 res = nvme_ctrlr_cmd_fw_image_download(ctrlr, transfer, offset, p, 5080 nvme_completion_poll_cb, 5081 status); 5082 if (res) { 5083 free(status); 5084 return res; 5085 } 5086 5087 if (nvme_wait_for_completion_robust_lock(ctrlr->adminq, status, &ctrlr->ctrlr_lock)) { 5088 NVME_CTRLR_ERRLOG(ctrlr, "spdk_nvme_ctrlr_fw_image_download failed!\n"); 5089 if (!status->timed_out) { 5090 free(status); 5091 } 5092 return -ENXIO; 5093 } 5094 p += transfer; 5095 offset += transfer; 5096 size_remaining -= transfer; 5097 } 5098 5099 /* Firmware commit */ 5100 memset(&fw_commit, 0, sizeof(struct spdk_nvme_fw_commit)); 5101 fw_commit.fs = slot; 5102 fw_commit.ca = commit_action; 5103 5104 memset(status, 0, sizeof(*status)); 5105 res = nvme_ctrlr_cmd_fw_commit(ctrlr, &fw_commit, nvme_completion_poll_cb, 5106 status); 5107 if (res) { 5108 free(status); 5109 return res; 5110 } 5111 5112 res = nvme_wait_for_completion_robust_lock(ctrlr->adminq, status, &ctrlr->ctrlr_lock); 5113 5114 memcpy(completion_status, &status->cpl.status, sizeof(struct spdk_nvme_status)); 5115 5116 if (!status->timed_out) { 5117 free(status); 5118 } 5119 5120 if (res) { 5121 if (completion_status->sct != SPDK_NVME_SCT_COMMAND_SPECIFIC || 5122 completion_status->sc != SPDK_NVME_SC_FIRMWARE_REQ_NVM_RESET) { 5123 if (completion_status->sct == SPDK_NVME_SCT_COMMAND_SPECIFIC && 5124 completion_status->sc == SPDK_NVME_SC_FIRMWARE_REQ_CONVENTIONAL_RESET) { 5125 NVME_CTRLR_NOTICELOG(ctrlr, 5126 "firmware activation requires conventional reset to be performed. !\n"); 5127 } else { 5128 NVME_CTRLR_ERRLOG(ctrlr, "nvme_ctrlr_cmd_fw_commit failed!\n"); 5129 } 5130 return -ENXIO; 5131 } 5132 } 5133 5134 return spdk_nvme_ctrlr_reset(ctrlr); 5135 } 5136 5137 int 5138 spdk_nvme_ctrlr_reserve_cmb(struct spdk_nvme_ctrlr *ctrlr) 5139 { 5140 int rc, size; 5141 union spdk_nvme_cmbsz_register cmbsz; 5142 5143 cmbsz = spdk_nvme_ctrlr_get_regs_cmbsz(ctrlr); 5144 5145 if (cmbsz.bits.rds == 0 || cmbsz.bits.wds == 0) { 5146 return -ENOTSUP; 5147 } 5148 5149 size = cmbsz.bits.sz * (0x1000 << (cmbsz.bits.szu * 4)); 5150 5151 nvme_ctrlr_lock(ctrlr); 5152 rc = nvme_transport_ctrlr_reserve_cmb(ctrlr); 5153 nvme_ctrlr_unlock(ctrlr); 5154 5155 if (rc < 0) { 5156 return rc; 5157 } 5158 5159 return size; 5160 } 5161 5162 void * 5163 spdk_nvme_ctrlr_map_cmb(struct spdk_nvme_ctrlr *ctrlr, size_t *size) 5164 { 5165 void *buf; 5166 5167 nvme_ctrlr_lock(ctrlr); 5168 buf = nvme_transport_ctrlr_map_cmb(ctrlr, size); 5169 nvme_ctrlr_unlock(ctrlr); 5170 5171 return buf; 5172 } 5173 5174 void 5175 spdk_nvme_ctrlr_unmap_cmb(struct spdk_nvme_ctrlr *ctrlr) 5176 { 5177 nvme_ctrlr_lock(ctrlr); 5178 nvme_transport_ctrlr_unmap_cmb(ctrlr); 5179 nvme_ctrlr_unlock(ctrlr); 5180 } 5181 5182 int 5183 spdk_nvme_ctrlr_enable_pmr(struct spdk_nvme_ctrlr *ctrlr) 5184 { 5185 int rc; 5186 5187 nvme_ctrlr_lock(ctrlr); 5188 rc = nvme_transport_ctrlr_enable_pmr(ctrlr); 5189 nvme_ctrlr_unlock(ctrlr); 5190 5191 return rc; 5192 } 5193 5194 int 5195 spdk_nvme_ctrlr_disable_pmr(struct spdk_nvme_ctrlr *ctrlr) 5196 { 5197 int rc; 5198 5199 nvme_ctrlr_lock(ctrlr); 5200 rc = nvme_transport_ctrlr_disable_pmr(ctrlr); 5201 nvme_ctrlr_unlock(ctrlr); 5202 5203 return rc; 5204 } 5205 5206 void * 5207 spdk_nvme_ctrlr_map_pmr(struct spdk_nvme_ctrlr *ctrlr, size_t *size) 5208 { 5209 void *buf; 5210 5211 nvme_ctrlr_lock(ctrlr); 5212 buf = nvme_transport_ctrlr_map_pmr(ctrlr, size); 5213 nvme_ctrlr_unlock(ctrlr); 5214 5215 return buf; 5216 } 5217 5218 int 5219 spdk_nvme_ctrlr_unmap_pmr(struct spdk_nvme_ctrlr *ctrlr) 5220 { 5221 int rc; 5222 5223 nvme_ctrlr_lock(ctrlr); 5224 rc = nvme_transport_ctrlr_unmap_pmr(ctrlr); 5225 nvme_ctrlr_unlock(ctrlr); 5226 5227 return rc; 5228 } 5229 5230 int 5231 spdk_nvme_ctrlr_read_boot_partition_start(struct spdk_nvme_ctrlr *ctrlr, void *payload, 5232 uint32_t bprsz, uint32_t bprof, uint32_t bpid) 5233 { 5234 union spdk_nvme_bprsel_register bprsel; 5235 union spdk_nvme_bpinfo_register bpinfo; 5236 uint64_t bpmbl, bpmb_size; 5237 5238 if (ctrlr->cap.bits.bps == 0) { 5239 return -ENOTSUP; 5240 } 5241 5242 if (nvme_ctrlr_get_bpinfo(ctrlr, &bpinfo)) { 5243 NVME_CTRLR_ERRLOG(ctrlr, "get bpinfo failed\n"); 5244 return -EIO; 5245 } 5246 5247 if (bpinfo.bits.brs == SPDK_NVME_BRS_READ_IN_PROGRESS) { 5248 NVME_CTRLR_ERRLOG(ctrlr, "Boot Partition read already initiated\n"); 5249 return -EALREADY; 5250 } 5251 5252 nvme_ctrlr_lock(ctrlr); 5253 5254 bpmb_size = bprsz * 4096; 5255 bpmbl = spdk_vtophys(payload, &bpmb_size); 5256 if (bpmbl == SPDK_VTOPHYS_ERROR) { 5257 NVME_CTRLR_ERRLOG(ctrlr, "spdk_vtophys of bpmbl failed\n"); 5258 nvme_ctrlr_unlock(ctrlr); 5259 return -EFAULT; 5260 } 5261 5262 if (bpmb_size != bprsz * 4096) { 5263 NVME_CTRLR_ERRLOG(ctrlr, "Boot Partition buffer is not physically contiguous\n"); 5264 nvme_ctrlr_unlock(ctrlr); 5265 return -EFAULT; 5266 } 5267 5268 if (nvme_ctrlr_set_bpmbl(ctrlr, bpmbl)) { 5269 NVME_CTRLR_ERRLOG(ctrlr, "set_bpmbl() failed\n"); 5270 nvme_ctrlr_unlock(ctrlr); 5271 return -EIO; 5272 } 5273 5274 bprsel.bits.bpid = bpid; 5275 bprsel.bits.bprof = bprof; 5276 bprsel.bits.bprsz = bprsz; 5277 5278 if (nvme_ctrlr_set_bprsel(ctrlr, &bprsel)) { 5279 NVME_CTRLR_ERRLOG(ctrlr, "set_bprsel() failed\n"); 5280 nvme_ctrlr_unlock(ctrlr); 5281 return -EIO; 5282 } 5283 5284 nvme_ctrlr_unlock(ctrlr); 5285 return 0; 5286 } 5287 5288 int 5289 spdk_nvme_ctrlr_read_boot_partition_poll(struct spdk_nvme_ctrlr *ctrlr) 5290 { 5291 int rc = 0; 5292 union spdk_nvme_bpinfo_register bpinfo; 5293 5294 if (nvme_ctrlr_get_bpinfo(ctrlr, &bpinfo)) { 5295 NVME_CTRLR_ERRLOG(ctrlr, "get bpinfo failed\n"); 5296 return -EIO; 5297 } 5298 5299 switch (bpinfo.bits.brs) { 5300 case SPDK_NVME_BRS_NO_READ: 5301 NVME_CTRLR_ERRLOG(ctrlr, "Boot Partition read not initiated\n"); 5302 rc = -EINVAL; 5303 break; 5304 case SPDK_NVME_BRS_READ_IN_PROGRESS: 5305 NVME_CTRLR_DEBUGLOG(ctrlr, "Boot Partition read in progress\n"); 5306 rc = -EAGAIN; 5307 break; 5308 case SPDK_NVME_BRS_READ_ERROR: 5309 NVME_CTRLR_ERRLOG(ctrlr, "Error completing Boot Partition read\n"); 5310 rc = -EIO; 5311 break; 5312 case SPDK_NVME_BRS_READ_SUCCESS: 5313 NVME_CTRLR_INFOLOG(ctrlr, "Boot Partition read completed successfully\n"); 5314 break; 5315 default: 5316 NVME_CTRLR_ERRLOG(ctrlr, "Invalid Boot Partition read status\n"); 5317 rc = -EINVAL; 5318 } 5319 5320 return rc; 5321 } 5322 5323 static void 5324 nvme_write_boot_partition_cb(void *arg, const struct spdk_nvme_cpl *cpl) 5325 { 5326 int res; 5327 struct spdk_nvme_ctrlr *ctrlr = arg; 5328 struct spdk_nvme_fw_commit fw_commit; 5329 struct spdk_nvme_cpl err_cpl = 5330 {.status = {.sct = SPDK_NVME_SCT_GENERIC, .sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR }}; 5331 5332 if (spdk_nvme_cpl_is_error(cpl)) { 5333 NVME_CTRLR_ERRLOG(ctrlr, "Write Boot Partition failed\n"); 5334 ctrlr->bp_write_cb_fn(ctrlr->bp_write_cb_arg, cpl); 5335 return; 5336 } 5337 5338 if (ctrlr->bp_ws == SPDK_NVME_BP_WS_DOWNLOADING) { 5339 NVME_CTRLR_DEBUGLOG(ctrlr, "Boot Partition Downloading at Offset %d Success\n", ctrlr->fw_offset); 5340 ctrlr->fw_payload = (uint8_t *)ctrlr->fw_payload + ctrlr->fw_transfer_size; 5341 ctrlr->fw_offset += ctrlr->fw_transfer_size; 5342 ctrlr->fw_size_remaining -= ctrlr->fw_transfer_size; 5343 ctrlr->fw_transfer_size = spdk_min(ctrlr->fw_size_remaining, ctrlr->min_page_size); 5344 res = nvme_ctrlr_cmd_fw_image_download(ctrlr, ctrlr->fw_transfer_size, ctrlr->fw_offset, 5345 ctrlr->fw_payload, nvme_write_boot_partition_cb, ctrlr); 5346 if (res) { 5347 NVME_CTRLR_ERRLOG(ctrlr, "nvme_ctrlr_cmd_fw_image_download failed!\n"); 5348 ctrlr->bp_write_cb_fn(ctrlr->bp_write_cb_arg, &err_cpl); 5349 return; 5350 } 5351 5352 if (ctrlr->fw_transfer_size < ctrlr->min_page_size) { 5353 ctrlr->bp_ws = SPDK_NVME_BP_WS_DOWNLOADED; 5354 } 5355 } else if (ctrlr->bp_ws == SPDK_NVME_BP_WS_DOWNLOADED) { 5356 NVME_CTRLR_DEBUGLOG(ctrlr, "Boot Partition Download Success\n"); 5357 memset(&fw_commit, 0, sizeof(struct spdk_nvme_fw_commit)); 5358 fw_commit.bpid = ctrlr->bpid; 5359 fw_commit.ca = SPDK_NVME_FW_COMMIT_REPLACE_BOOT_PARTITION; 5360 res = nvme_ctrlr_cmd_fw_commit(ctrlr, &fw_commit, 5361 nvme_write_boot_partition_cb, ctrlr); 5362 if (res) { 5363 NVME_CTRLR_ERRLOG(ctrlr, "nvme_ctrlr_cmd_fw_commit failed!\n"); 5364 NVME_CTRLR_ERRLOG(ctrlr, "commit action: %d\n", fw_commit.ca); 5365 ctrlr->bp_write_cb_fn(ctrlr->bp_write_cb_arg, &err_cpl); 5366 return; 5367 } 5368 5369 ctrlr->bp_ws = SPDK_NVME_BP_WS_REPLACE; 5370 } else if (ctrlr->bp_ws == SPDK_NVME_BP_WS_REPLACE) { 5371 NVME_CTRLR_DEBUGLOG(ctrlr, "Boot Partition Replacement Success\n"); 5372 memset(&fw_commit, 0, sizeof(struct spdk_nvme_fw_commit)); 5373 fw_commit.bpid = ctrlr->bpid; 5374 fw_commit.ca = SPDK_NVME_FW_COMMIT_ACTIVATE_BOOT_PARTITION; 5375 res = nvme_ctrlr_cmd_fw_commit(ctrlr, &fw_commit, 5376 nvme_write_boot_partition_cb, ctrlr); 5377 if (res) { 5378 NVME_CTRLR_ERRLOG(ctrlr, "nvme_ctrlr_cmd_fw_commit failed!\n"); 5379 NVME_CTRLR_ERRLOG(ctrlr, "commit action: %d\n", fw_commit.ca); 5380 ctrlr->bp_write_cb_fn(ctrlr->bp_write_cb_arg, &err_cpl); 5381 return; 5382 } 5383 5384 ctrlr->bp_ws = SPDK_NVME_BP_WS_ACTIVATE; 5385 } else if (ctrlr->bp_ws == SPDK_NVME_BP_WS_ACTIVATE) { 5386 NVME_CTRLR_DEBUGLOG(ctrlr, "Boot Partition Activation Success\n"); 5387 ctrlr->bp_write_cb_fn(ctrlr->bp_write_cb_arg, cpl); 5388 } else { 5389 NVME_CTRLR_ERRLOG(ctrlr, "Invalid Boot Partition write state\n"); 5390 ctrlr->bp_write_cb_fn(ctrlr->bp_write_cb_arg, &err_cpl); 5391 return; 5392 } 5393 } 5394 5395 int 5396 spdk_nvme_ctrlr_write_boot_partition(struct spdk_nvme_ctrlr *ctrlr, 5397 void *payload, uint32_t size, uint32_t bpid, 5398 spdk_nvme_cmd_cb cb_fn, void *cb_arg) 5399 { 5400 int res; 5401 5402 if (ctrlr->cap.bits.bps == 0) { 5403 return -ENOTSUP; 5404 } 5405 5406 ctrlr->bp_ws = SPDK_NVME_BP_WS_DOWNLOADING; 5407 ctrlr->bpid = bpid; 5408 ctrlr->bp_write_cb_fn = cb_fn; 5409 ctrlr->bp_write_cb_arg = cb_arg; 5410 ctrlr->fw_offset = 0; 5411 ctrlr->fw_size_remaining = size; 5412 ctrlr->fw_payload = payload; 5413 ctrlr->fw_transfer_size = spdk_min(ctrlr->fw_size_remaining, ctrlr->min_page_size); 5414 5415 res = nvme_ctrlr_cmd_fw_image_download(ctrlr, ctrlr->fw_transfer_size, ctrlr->fw_offset, 5416 ctrlr->fw_payload, nvme_write_boot_partition_cb, ctrlr); 5417 5418 return res; 5419 } 5420 5421 bool 5422 spdk_nvme_ctrlr_is_discovery(struct spdk_nvme_ctrlr *ctrlr) 5423 { 5424 assert(ctrlr); 5425 5426 return !strncmp(ctrlr->trid.subnqn, SPDK_NVMF_DISCOVERY_NQN, 5427 strlen(SPDK_NVMF_DISCOVERY_NQN)); 5428 } 5429 5430 bool 5431 spdk_nvme_ctrlr_is_fabrics(struct spdk_nvme_ctrlr *ctrlr) 5432 { 5433 assert(ctrlr); 5434 5435 return spdk_nvme_trtype_is_fabrics(ctrlr->trid.trtype); 5436 } 5437 5438 int 5439 spdk_nvme_ctrlr_security_receive(struct spdk_nvme_ctrlr *ctrlr, uint8_t secp, 5440 uint16_t spsp, uint8_t nssf, void *payload, size_t size) 5441 { 5442 struct nvme_completion_poll_status *status; 5443 int res; 5444 5445 status = calloc(1, sizeof(*status)); 5446 if (!status) { 5447 NVME_CTRLR_ERRLOG(ctrlr, "Failed to allocate status tracker\n"); 5448 return -ENOMEM; 5449 } 5450 5451 res = spdk_nvme_ctrlr_cmd_security_receive(ctrlr, secp, spsp, nssf, payload, size, 5452 nvme_completion_poll_cb, status); 5453 if (res) { 5454 free(status); 5455 return res; 5456 } 5457 if (nvme_wait_for_completion_robust_lock(ctrlr->adminq, status, &ctrlr->ctrlr_lock)) { 5458 NVME_CTRLR_ERRLOG(ctrlr, "spdk_nvme_ctrlr_cmd_security_receive failed!\n"); 5459 if (!status->timed_out) { 5460 free(status); 5461 } 5462 return -ENXIO; 5463 } 5464 free(status); 5465 5466 return 0; 5467 } 5468 5469 int 5470 spdk_nvme_ctrlr_security_send(struct spdk_nvme_ctrlr *ctrlr, uint8_t secp, 5471 uint16_t spsp, uint8_t nssf, void *payload, size_t size) 5472 { 5473 struct nvme_completion_poll_status *status; 5474 int res; 5475 5476 status = calloc(1, sizeof(*status)); 5477 if (!status) { 5478 NVME_CTRLR_ERRLOG(ctrlr, "Failed to allocate status tracker\n"); 5479 return -ENOMEM; 5480 } 5481 5482 res = spdk_nvme_ctrlr_cmd_security_send(ctrlr, secp, spsp, nssf, payload, size, 5483 nvme_completion_poll_cb, 5484 status); 5485 if (res) { 5486 free(status); 5487 return res; 5488 } 5489 if (nvme_wait_for_completion_robust_lock(ctrlr->adminq, status, &ctrlr->ctrlr_lock)) { 5490 NVME_CTRLR_ERRLOG(ctrlr, "spdk_nvme_ctrlr_cmd_security_send failed!\n"); 5491 if (!status->timed_out) { 5492 free(status); 5493 } 5494 return -ENXIO; 5495 } 5496 5497 free(status); 5498 5499 return 0; 5500 } 5501 5502 uint64_t 5503 spdk_nvme_ctrlr_get_flags(struct spdk_nvme_ctrlr *ctrlr) 5504 { 5505 return ctrlr->flags; 5506 } 5507 5508 const struct spdk_nvme_transport_id * 5509 spdk_nvme_ctrlr_get_transport_id(struct spdk_nvme_ctrlr *ctrlr) 5510 { 5511 return &ctrlr->trid; 5512 } 5513 5514 int32_t 5515 spdk_nvme_ctrlr_alloc_qid(struct spdk_nvme_ctrlr *ctrlr) 5516 { 5517 uint32_t qid; 5518 5519 assert(ctrlr->free_io_qids); 5520 nvme_ctrlr_lock(ctrlr); 5521 qid = spdk_bit_array_find_first_set(ctrlr->free_io_qids, 1); 5522 if (qid > ctrlr->opts.num_io_queues) { 5523 NVME_CTRLR_ERRLOG(ctrlr, "No free I/O queue IDs\n"); 5524 nvme_ctrlr_unlock(ctrlr); 5525 return -1; 5526 } 5527 5528 spdk_bit_array_clear(ctrlr->free_io_qids, qid); 5529 nvme_ctrlr_unlock(ctrlr); 5530 return qid; 5531 } 5532 5533 void 5534 spdk_nvme_ctrlr_free_qid(struct spdk_nvme_ctrlr *ctrlr, uint16_t qid) 5535 { 5536 assert(qid <= ctrlr->opts.num_io_queues); 5537 5538 nvme_ctrlr_lock(ctrlr); 5539 5540 if (spdk_likely(ctrlr->free_io_qids)) { 5541 spdk_bit_array_set(ctrlr->free_io_qids, qid); 5542 } 5543 5544 nvme_ctrlr_unlock(ctrlr); 5545 } 5546 5547 int 5548 spdk_nvme_ctrlr_get_memory_domains(const struct spdk_nvme_ctrlr *ctrlr, 5549 struct spdk_memory_domain **domains, int array_size) 5550 { 5551 return nvme_transport_ctrlr_get_memory_domains(ctrlr, domains, array_size); 5552 } 5553 5554 int 5555 spdk_nvme_ctrlr_authenticate(struct spdk_nvme_ctrlr *ctrlr, 5556 spdk_nvme_authenticate_cb cb_fn, void *cb_ctx) 5557 { 5558 return spdk_nvme_qpair_authenticate(ctrlr->adminq, cb_fn, cb_ctx); 5559 } 5560