1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2015 Intel Corporation. All rights reserved. 3 * Copyright (c) 2019-2021 Mellanox Technologies LTD. All rights reserved. 4 * Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 5 */ 6 7 #include "spdk/stdinc.h" 8 9 #include "nvme_internal.h" 10 #include "nvme_io_msg.h" 11 12 #include "spdk/env.h" 13 #include "spdk/string.h" 14 #include "spdk/endian.h" 15 16 struct nvme_active_ns_ctx; 17 18 static int nvme_ctrlr_construct_and_submit_aer(struct spdk_nvme_ctrlr *ctrlr, 19 struct nvme_async_event_request *aer); 20 static void nvme_ctrlr_identify_active_ns_async(struct nvme_active_ns_ctx *ctx); 21 static int nvme_ctrlr_identify_ns_async(struct spdk_nvme_ns *ns); 22 static int nvme_ctrlr_identify_ns_iocs_specific_async(struct spdk_nvme_ns *ns); 23 static int nvme_ctrlr_identify_id_desc_async(struct spdk_nvme_ns *ns); 24 static void nvme_ctrlr_init_cap(struct spdk_nvme_ctrlr *ctrlr); 25 static void nvme_ctrlr_set_state(struct spdk_nvme_ctrlr *ctrlr, enum nvme_ctrlr_state state, 26 uint64_t timeout_in_ms); 27 28 static int 29 nvme_ns_cmp(struct spdk_nvme_ns *ns1, struct spdk_nvme_ns *ns2) 30 { 31 if (ns1->id < ns2->id) { 32 return -1; 33 } else if (ns1->id > ns2->id) { 34 return 1; 35 } else { 36 return 0; 37 } 38 } 39 40 RB_GENERATE_STATIC(nvme_ns_tree, spdk_nvme_ns, node, nvme_ns_cmp); 41 42 #define CTRLR_STRING(ctrlr) \ 43 ((ctrlr->trid.trtype == SPDK_NVME_TRANSPORT_TCP || ctrlr->trid.trtype == SPDK_NVME_TRANSPORT_RDMA) ? \ 44 ctrlr->trid.subnqn : ctrlr->trid.traddr) 45 46 #define NVME_CTRLR_ERRLOG(ctrlr, format, ...) \ 47 SPDK_ERRLOG("[%s] " format, CTRLR_STRING(ctrlr), ##__VA_ARGS__); 48 49 #define NVME_CTRLR_WARNLOG(ctrlr, format, ...) \ 50 SPDK_WARNLOG("[%s] " format, CTRLR_STRING(ctrlr), ##__VA_ARGS__); 51 52 #define NVME_CTRLR_NOTICELOG(ctrlr, format, ...) \ 53 SPDK_NOTICELOG("[%s] " format, CTRLR_STRING(ctrlr), ##__VA_ARGS__); 54 55 #define NVME_CTRLR_INFOLOG(ctrlr, format, ...) \ 56 SPDK_INFOLOG(nvme, "[%s] " format, CTRLR_STRING(ctrlr), ##__VA_ARGS__); 57 58 #ifdef DEBUG 59 #define NVME_CTRLR_DEBUGLOG(ctrlr, format, ...) \ 60 SPDK_DEBUGLOG(nvme, "[%s] " format, CTRLR_STRING(ctrlr), ##__VA_ARGS__); 61 #else 62 #define NVME_CTRLR_DEBUGLOG(ctrlr, ...) do { } while (0) 63 #endif 64 65 #define nvme_ctrlr_get_reg_async(ctrlr, reg, sz, cb_fn, cb_arg) \ 66 nvme_transport_ctrlr_get_reg_ ## sz ## _async(ctrlr, \ 67 offsetof(struct spdk_nvme_registers, reg), cb_fn, cb_arg) 68 69 #define nvme_ctrlr_set_reg_async(ctrlr, reg, sz, val, cb_fn, cb_arg) \ 70 nvme_transport_ctrlr_set_reg_ ## sz ## _async(ctrlr, \ 71 offsetof(struct spdk_nvme_registers, reg), val, cb_fn, cb_arg) 72 73 #define nvme_ctrlr_get_cc_async(ctrlr, cb_fn, cb_arg) \ 74 nvme_ctrlr_get_reg_async(ctrlr, cc, 4, cb_fn, cb_arg) 75 76 #define nvme_ctrlr_get_csts_async(ctrlr, cb_fn, cb_arg) \ 77 nvme_ctrlr_get_reg_async(ctrlr, csts, 4, cb_fn, cb_arg) 78 79 #define nvme_ctrlr_get_cap_async(ctrlr, cb_fn, cb_arg) \ 80 nvme_ctrlr_get_reg_async(ctrlr, cap, 8, cb_fn, cb_arg) 81 82 #define nvme_ctrlr_get_vs_async(ctrlr, cb_fn, cb_arg) \ 83 nvme_ctrlr_get_reg_async(ctrlr, vs, 4, cb_fn, cb_arg) 84 85 #define nvme_ctrlr_set_cc_async(ctrlr, value, cb_fn, cb_arg) \ 86 nvme_ctrlr_set_reg_async(ctrlr, cc, 4, value, cb_fn, cb_arg) 87 88 static int 89 nvme_ctrlr_get_cc(struct spdk_nvme_ctrlr *ctrlr, union spdk_nvme_cc_register *cc) 90 { 91 return nvme_transport_ctrlr_get_reg_4(ctrlr, offsetof(struct spdk_nvme_registers, cc.raw), 92 &cc->raw); 93 } 94 95 static int 96 nvme_ctrlr_get_csts(struct spdk_nvme_ctrlr *ctrlr, union spdk_nvme_csts_register *csts) 97 { 98 return nvme_transport_ctrlr_get_reg_4(ctrlr, offsetof(struct spdk_nvme_registers, csts.raw), 99 &csts->raw); 100 } 101 102 int 103 nvme_ctrlr_get_cap(struct spdk_nvme_ctrlr *ctrlr, union spdk_nvme_cap_register *cap) 104 { 105 return nvme_transport_ctrlr_get_reg_8(ctrlr, offsetof(struct spdk_nvme_registers, cap.raw), 106 &cap->raw); 107 } 108 109 int 110 nvme_ctrlr_get_vs(struct spdk_nvme_ctrlr *ctrlr, union spdk_nvme_vs_register *vs) 111 { 112 return nvme_transport_ctrlr_get_reg_4(ctrlr, offsetof(struct spdk_nvme_registers, vs.raw), 113 &vs->raw); 114 } 115 116 int 117 nvme_ctrlr_get_cmbsz(struct spdk_nvme_ctrlr *ctrlr, union spdk_nvme_cmbsz_register *cmbsz) 118 { 119 return nvme_transport_ctrlr_get_reg_4(ctrlr, offsetof(struct spdk_nvme_registers, cmbsz.raw), 120 &cmbsz->raw); 121 } 122 123 int 124 nvme_ctrlr_get_pmrcap(struct spdk_nvme_ctrlr *ctrlr, union spdk_nvme_pmrcap_register *pmrcap) 125 { 126 return nvme_transport_ctrlr_get_reg_4(ctrlr, offsetof(struct spdk_nvme_registers, pmrcap.raw), 127 &pmrcap->raw); 128 } 129 130 int 131 nvme_ctrlr_get_bpinfo(struct spdk_nvme_ctrlr *ctrlr, union spdk_nvme_bpinfo_register *bpinfo) 132 { 133 return nvme_transport_ctrlr_get_reg_4(ctrlr, offsetof(struct spdk_nvme_registers, bpinfo.raw), 134 &bpinfo->raw); 135 } 136 137 int 138 nvme_ctrlr_set_bprsel(struct spdk_nvme_ctrlr *ctrlr, union spdk_nvme_bprsel_register *bprsel) 139 { 140 return nvme_transport_ctrlr_set_reg_4(ctrlr, offsetof(struct spdk_nvme_registers, bprsel.raw), 141 bprsel->raw); 142 } 143 144 int 145 nvme_ctrlr_set_bpmbl(struct spdk_nvme_ctrlr *ctrlr, uint64_t bpmbl_value) 146 { 147 return nvme_transport_ctrlr_set_reg_8(ctrlr, offsetof(struct spdk_nvme_registers, bpmbl), 148 bpmbl_value); 149 } 150 151 static int 152 nvme_ctrlr_set_nssr(struct spdk_nvme_ctrlr *ctrlr, uint32_t nssr_value) 153 { 154 return nvme_transport_ctrlr_set_reg_4(ctrlr, offsetof(struct spdk_nvme_registers, nssr), 155 nssr_value); 156 } 157 158 bool 159 nvme_ctrlr_multi_iocs_enabled(struct spdk_nvme_ctrlr *ctrlr) 160 { 161 return ctrlr->cap.bits.css & SPDK_NVME_CAP_CSS_IOCS && 162 ctrlr->opts.command_set == SPDK_NVME_CC_CSS_IOCS; 163 } 164 165 /* When the field in spdk_nvme_ctrlr_opts are changed and you change this function, please 166 * also update the nvme_ctrl_opts_init function in nvme_ctrlr.c 167 */ 168 void 169 spdk_nvme_ctrlr_get_default_ctrlr_opts(struct spdk_nvme_ctrlr_opts *opts, size_t opts_size) 170 { 171 char host_id_str[SPDK_UUID_STRING_LEN]; 172 173 assert(opts); 174 175 opts->opts_size = opts_size; 176 177 #define FIELD_OK(field) \ 178 offsetof(struct spdk_nvme_ctrlr_opts, field) + sizeof(opts->field) <= opts_size 179 180 #define SET_FIELD(field, value) \ 181 if (offsetof(struct spdk_nvme_ctrlr_opts, field) + sizeof(opts->field) <= opts_size) { \ 182 opts->field = value; \ 183 } \ 184 185 SET_FIELD(num_io_queues, DEFAULT_MAX_IO_QUEUES); 186 SET_FIELD(use_cmb_sqs, false); 187 SET_FIELD(no_shn_notification, false); 188 SET_FIELD(arb_mechanism, SPDK_NVME_CC_AMS_RR); 189 SET_FIELD(arbitration_burst, 0); 190 SET_FIELD(low_priority_weight, 0); 191 SET_FIELD(medium_priority_weight, 0); 192 SET_FIELD(high_priority_weight, 0); 193 SET_FIELD(keep_alive_timeout_ms, MIN_KEEP_ALIVE_TIMEOUT_IN_MS); 194 SET_FIELD(transport_retry_count, SPDK_NVME_DEFAULT_RETRY_COUNT); 195 SET_FIELD(io_queue_size, DEFAULT_IO_QUEUE_SIZE); 196 197 if (nvme_driver_init() == 0) { 198 if (FIELD_OK(hostnqn)) { 199 spdk_uuid_fmt_lower(host_id_str, sizeof(host_id_str), 200 &g_spdk_nvme_driver->default_extended_host_id); 201 snprintf(opts->hostnqn, sizeof(opts->hostnqn), 202 "nqn.2014-08.org.nvmexpress:uuid:%s", host_id_str); 203 } 204 205 if (FIELD_OK(extended_host_id)) { 206 memcpy(opts->extended_host_id, &g_spdk_nvme_driver->default_extended_host_id, 207 sizeof(opts->extended_host_id)); 208 } 209 210 } 211 212 SET_FIELD(io_queue_requests, DEFAULT_IO_QUEUE_REQUESTS); 213 214 if (FIELD_OK(src_addr)) { 215 memset(opts->src_addr, 0, sizeof(opts->src_addr)); 216 } 217 218 if (FIELD_OK(src_svcid)) { 219 memset(opts->src_svcid, 0, sizeof(opts->src_svcid)); 220 } 221 222 if (FIELD_OK(host_id)) { 223 memset(opts->host_id, 0, sizeof(opts->host_id)); 224 } 225 226 SET_FIELD(command_set, CHAR_BIT); 227 SET_FIELD(admin_timeout_ms, NVME_MAX_ADMIN_TIMEOUT_IN_SECS * 1000); 228 SET_FIELD(header_digest, false); 229 SET_FIELD(data_digest, false); 230 SET_FIELD(disable_error_logging, false); 231 SET_FIELD(transport_ack_timeout, SPDK_NVME_DEFAULT_TRANSPORT_ACK_TIMEOUT); 232 SET_FIELD(admin_queue_size, DEFAULT_ADMIN_QUEUE_SIZE); 233 SET_FIELD(fabrics_connect_timeout_us, NVME_FABRIC_CONNECT_COMMAND_TIMEOUT); 234 SET_FIELD(disable_read_ana_log_page, false); 235 SET_FIELD(disable_read_changed_ns_list_log_page, false); 236 SET_FIELD(tls_psk, NULL); 237 SET_FIELD(dhchap_key, NULL); 238 SET_FIELD(dhchap_digests, 239 SPDK_BIT(SPDK_NVMF_DHCHAP_HASH_SHA256) | 240 SPDK_BIT(SPDK_NVMF_DHCHAP_HASH_SHA384) | 241 SPDK_BIT(SPDK_NVMF_DHCHAP_HASH_SHA512)); 242 SET_FIELD(dhchap_dhgroups, 243 SPDK_BIT(SPDK_NVMF_DHCHAP_DHGROUP_NULL) | 244 SPDK_BIT(SPDK_NVMF_DHCHAP_DHGROUP_2048) | 245 SPDK_BIT(SPDK_NVMF_DHCHAP_DHGROUP_3072) | 246 SPDK_BIT(SPDK_NVMF_DHCHAP_DHGROUP_4096) | 247 SPDK_BIT(SPDK_NVMF_DHCHAP_DHGROUP_6144) | 248 SPDK_BIT(SPDK_NVMF_DHCHAP_DHGROUP_8192)); 249 250 if (FIELD_OK(psk)) { 251 memset(opts->psk, 0, sizeof(opts->psk)); 252 } 253 254 #undef FIELD_OK 255 #undef SET_FIELD 256 } 257 258 const struct spdk_nvme_ctrlr_opts * 259 spdk_nvme_ctrlr_get_opts(struct spdk_nvme_ctrlr *ctrlr) 260 { 261 return &ctrlr->opts; 262 } 263 264 /** 265 * This function will be called when the process allocates the IO qpair. 266 * Note: the ctrlr_lock must be held when calling this function. 267 */ 268 static void 269 nvme_ctrlr_proc_add_io_qpair(struct spdk_nvme_qpair *qpair) 270 { 271 struct spdk_nvme_ctrlr_process *active_proc; 272 struct spdk_nvme_ctrlr *ctrlr = qpair->ctrlr; 273 274 active_proc = nvme_ctrlr_get_current_process(ctrlr); 275 if (active_proc) { 276 TAILQ_INSERT_TAIL(&active_proc->allocated_io_qpairs, qpair, per_process_tailq); 277 qpair->active_proc = active_proc; 278 } 279 } 280 281 /** 282 * This function will be called when the process frees the IO qpair. 283 * Note: the ctrlr_lock must be held when calling this function. 284 */ 285 static void 286 nvme_ctrlr_proc_remove_io_qpair(struct spdk_nvme_qpair *qpair) 287 { 288 struct spdk_nvme_ctrlr_process *active_proc; 289 struct spdk_nvme_ctrlr *ctrlr = qpair->ctrlr; 290 struct spdk_nvme_qpair *active_qpair, *tmp_qpair; 291 292 active_proc = nvme_ctrlr_get_current_process(ctrlr); 293 if (!active_proc) { 294 return; 295 } 296 297 TAILQ_FOREACH_SAFE(active_qpair, &active_proc->allocated_io_qpairs, 298 per_process_tailq, tmp_qpair) { 299 if (active_qpair == qpair) { 300 TAILQ_REMOVE(&active_proc->allocated_io_qpairs, 301 active_qpair, per_process_tailq); 302 303 break; 304 } 305 } 306 } 307 308 void 309 spdk_nvme_ctrlr_get_default_io_qpair_opts(struct spdk_nvme_ctrlr *ctrlr, 310 struct spdk_nvme_io_qpair_opts *opts, 311 size_t opts_size) 312 { 313 assert(ctrlr); 314 315 assert(opts); 316 317 memset(opts, 0, opts_size); 318 319 #define FIELD_OK(field) \ 320 offsetof(struct spdk_nvme_io_qpair_opts, field) + sizeof(opts->field) <= opts_size 321 322 if (FIELD_OK(qprio)) { 323 opts->qprio = SPDK_NVME_QPRIO_URGENT; 324 } 325 326 if (FIELD_OK(io_queue_size)) { 327 opts->io_queue_size = ctrlr->opts.io_queue_size; 328 } 329 330 if (FIELD_OK(io_queue_requests)) { 331 opts->io_queue_requests = ctrlr->opts.io_queue_requests; 332 } 333 334 if (FIELD_OK(delay_cmd_submit)) { 335 opts->delay_cmd_submit = false; 336 } 337 338 if (FIELD_OK(sq.vaddr)) { 339 opts->sq.vaddr = NULL; 340 } 341 342 if (FIELD_OK(sq.paddr)) { 343 opts->sq.paddr = 0; 344 } 345 346 if (FIELD_OK(sq.buffer_size)) { 347 opts->sq.buffer_size = 0; 348 } 349 350 if (FIELD_OK(cq.vaddr)) { 351 opts->cq.vaddr = NULL; 352 } 353 354 if (FIELD_OK(cq.paddr)) { 355 opts->cq.paddr = 0; 356 } 357 358 if (FIELD_OK(cq.buffer_size)) { 359 opts->cq.buffer_size = 0; 360 } 361 362 if (FIELD_OK(create_only)) { 363 opts->create_only = false; 364 } 365 366 if (FIELD_OK(async_mode)) { 367 opts->async_mode = false; 368 } 369 370 #undef FIELD_OK 371 } 372 373 static struct spdk_nvme_qpair * 374 nvme_ctrlr_create_io_qpair(struct spdk_nvme_ctrlr *ctrlr, 375 const struct spdk_nvme_io_qpair_opts *opts) 376 { 377 int32_t qid; 378 struct spdk_nvme_qpair *qpair; 379 union spdk_nvme_cc_register cc; 380 381 if (!ctrlr) { 382 return NULL; 383 } 384 385 nvme_robust_mutex_lock(&ctrlr->ctrlr_lock); 386 cc.raw = ctrlr->process_init_cc.raw; 387 388 if (opts->qprio & ~SPDK_NVME_CREATE_IO_SQ_QPRIO_MASK) { 389 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 390 return NULL; 391 } 392 393 /* 394 * Only value SPDK_NVME_QPRIO_URGENT(0) is valid for the 395 * default round robin arbitration method. 396 */ 397 if ((cc.bits.ams == SPDK_NVME_CC_AMS_RR) && (opts->qprio != SPDK_NVME_QPRIO_URGENT)) { 398 NVME_CTRLR_ERRLOG(ctrlr, "invalid queue priority for default round robin arbitration method\n"); 399 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 400 return NULL; 401 } 402 403 qid = spdk_nvme_ctrlr_alloc_qid(ctrlr); 404 if (qid < 0) { 405 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 406 return NULL; 407 } 408 409 qpair = nvme_transport_ctrlr_create_io_qpair(ctrlr, qid, opts); 410 if (qpair == NULL) { 411 NVME_CTRLR_ERRLOG(ctrlr, "nvme_transport_ctrlr_create_io_qpair() failed\n"); 412 spdk_nvme_ctrlr_free_qid(ctrlr, qid); 413 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 414 return NULL; 415 } 416 417 TAILQ_INSERT_TAIL(&ctrlr->active_io_qpairs, qpair, tailq); 418 419 nvme_ctrlr_proc_add_io_qpair(qpair); 420 421 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 422 423 return qpair; 424 } 425 426 int 427 spdk_nvme_ctrlr_connect_io_qpair(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair) 428 { 429 int rc; 430 431 if (nvme_qpair_get_state(qpair) != NVME_QPAIR_DISCONNECTED) { 432 return -EISCONN; 433 } 434 435 nvme_robust_mutex_lock(&ctrlr->ctrlr_lock); 436 rc = nvme_transport_ctrlr_connect_qpair(ctrlr, qpair); 437 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 438 439 if (ctrlr->quirks & NVME_QUIRK_DELAY_AFTER_QUEUE_ALLOC) { 440 spdk_delay_us(100); 441 } 442 443 return rc; 444 } 445 446 void 447 spdk_nvme_ctrlr_disconnect_io_qpair(struct spdk_nvme_qpair *qpair) 448 { 449 struct spdk_nvme_ctrlr *ctrlr = qpair->ctrlr; 450 451 nvme_robust_mutex_lock(&ctrlr->ctrlr_lock); 452 nvme_transport_ctrlr_disconnect_qpair(ctrlr, qpair); 453 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 454 } 455 456 struct spdk_nvme_qpair * 457 spdk_nvme_ctrlr_alloc_io_qpair(struct spdk_nvme_ctrlr *ctrlr, 458 const struct spdk_nvme_io_qpair_opts *user_opts, 459 size_t opts_size) 460 { 461 462 struct spdk_nvme_qpair *qpair = NULL; 463 struct spdk_nvme_io_qpair_opts opts; 464 int rc; 465 466 nvme_robust_mutex_lock(&ctrlr->ctrlr_lock); 467 468 if (spdk_unlikely(ctrlr->state != NVME_CTRLR_STATE_READY)) { 469 /* When controller is resetting or initializing, free_io_qids is deleted or not created yet. 470 * We can't create IO qpair in that case */ 471 goto unlock; 472 } 473 474 /* 475 * Get the default options, then overwrite them with the user-provided options 476 * up to opts_size. 477 * 478 * This allows for extensions of the opts structure without breaking 479 * ABI compatibility. 480 */ 481 spdk_nvme_ctrlr_get_default_io_qpair_opts(ctrlr, &opts, sizeof(opts)); 482 if (user_opts) { 483 memcpy(&opts, user_opts, spdk_min(sizeof(opts), opts_size)); 484 485 /* If user passes buffers, make sure they're big enough for the requested queue size */ 486 if (opts.sq.vaddr) { 487 if (opts.sq.buffer_size < (opts.io_queue_size * sizeof(struct spdk_nvme_cmd))) { 488 NVME_CTRLR_ERRLOG(ctrlr, "sq buffer size %" PRIx64 " is too small for sq size %zx\n", 489 opts.sq.buffer_size, (opts.io_queue_size * sizeof(struct spdk_nvme_cmd))); 490 goto unlock; 491 } 492 } 493 if (opts.cq.vaddr) { 494 if (opts.cq.buffer_size < (opts.io_queue_size * sizeof(struct spdk_nvme_cpl))) { 495 NVME_CTRLR_ERRLOG(ctrlr, "cq buffer size %" PRIx64 " is too small for cq size %zx\n", 496 opts.cq.buffer_size, (opts.io_queue_size * sizeof(struct spdk_nvme_cpl))); 497 goto unlock; 498 } 499 } 500 } 501 502 qpair = nvme_ctrlr_create_io_qpair(ctrlr, &opts); 503 504 if (qpair == NULL || opts.create_only == true) { 505 goto unlock; 506 } 507 508 rc = spdk_nvme_ctrlr_connect_io_qpair(ctrlr, qpair); 509 if (rc != 0) { 510 NVME_CTRLR_ERRLOG(ctrlr, "nvme_transport_ctrlr_connect_io_qpair() failed\n"); 511 nvme_ctrlr_proc_remove_io_qpair(qpair); 512 TAILQ_REMOVE(&ctrlr->active_io_qpairs, qpair, tailq); 513 spdk_bit_array_set(ctrlr->free_io_qids, qpair->id); 514 nvme_transport_ctrlr_delete_io_qpair(ctrlr, qpair); 515 qpair = NULL; 516 goto unlock; 517 } 518 519 unlock: 520 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 521 522 return qpair; 523 } 524 525 int 526 spdk_nvme_ctrlr_reconnect_io_qpair(struct spdk_nvme_qpair *qpair) 527 { 528 struct spdk_nvme_ctrlr *ctrlr; 529 enum nvme_qpair_state qpair_state; 530 int rc; 531 532 assert(qpair != NULL); 533 assert(nvme_qpair_is_admin_queue(qpair) == false); 534 assert(qpair->ctrlr != NULL); 535 536 ctrlr = qpair->ctrlr; 537 nvme_robust_mutex_lock(&ctrlr->ctrlr_lock); 538 qpair_state = nvme_qpair_get_state(qpair); 539 540 if (ctrlr->is_removed) { 541 rc = -ENODEV; 542 goto out; 543 } 544 545 if (ctrlr->is_resetting || qpair_state == NVME_QPAIR_DISCONNECTING) { 546 rc = -EAGAIN; 547 goto out; 548 } 549 550 if (ctrlr->is_failed || qpair_state == NVME_QPAIR_DESTROYING) { 551 rc = -ENXIO; 552 goto out; 553 } 554 555 if (qpair_state != NVME_QPAIR_DISCONNECTED) { 556 rc = 0; 557 goto out; 558 } 559 560 rc = nvme_transport_ctrlr_connect_qpair(ctrlr, qpair); 561 if (rc) { 562 rc = -EAGAIN; 563 goto out; 564 } 565 566 out: 567 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 568 return rc; 569 } 570 571 spdk_nvme_qp_failure_reason 572 spdk_nvme_ctrlr_get_admin_qp_failure_reason(struct spdk_nvme_ctrlr *ctrlr) 573 { 574 return ctrlr->adminq->transport_failure_reason; 575 } 576 577 /* 578 * This internal function will attempt to take the controller 579 * lock before calling disconnect on a controller qpair. 580 * Functions already holding the controller lock should 581 * call nvme_transport_ctrlr_disconnect_qpair directly. 582 */ 583 void 584 nvme_ctrlr_disconnect_qpair(struct spdk_nvme_qpair *qpair) 585 { 586 struct spdk_nvme_ctrlr *ctrlr = qpair->ctrlr; 587 588 assert(ctrlr != NULL); 589 nvme_robust_mutex_lock(&ctrlr->ctrlr_lock); 590 nvme_transport_ctrlr_disconnect_qpair(ctrlr, qpair); 591 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 592 } 593 594 int 595 spdk_nvme_ctrlr_free_io_qpair(struct spdk_nvme_qpair *qpair) 596 { 597 struct spdk_nvme_ctrlr *ctrlr; 598 599 if (qpair == NULL) { 600 return 0; 601 } 602 603 ctrlr = qpair->ctrlr; 604 605 if (qpair->in_completion_context) { 606 /* 607 * There are many cases where it is convenient to delete an io qpair in the context 608 * of that qpair's completion routine. To handle this properly, set a flag here 609 * so that the completion routine will perform an actual delete after the context 610 * unwinds. 611 */ 612 qpair->delete_after_completion_context = 1; 613 return 0; 614 } 615 616 qpair->destroy_in_progress = 1; 617 618 nvme_transport_ctrlr_disconnect_qpair(ctrlr, qpair); 619 620 if (qpair->poll_group && (qpair->active_proc == nvme_ctrlr_get_current_process(ctrlr))) { 621 spdk_nvme_poll_group_remove(qpair->poll_group->group, qpair); 622 } 623 624 /* Do not retry. */ 625 nvme_qpair_set_state(qpair, NVME_QPAIR_DESTROYING); 626 627 /* In the multi-process case, a process may call this function on a foreign 628 * I/O qpair (i.e. one that this process did not create) when that qpairs process 629 * exits unexpectedly. In that case, we must not try to abort any reqs associated 630 * with that qpair, since the callbacks will also be foreign to this process. 631 */ 632 if (qpair->active_proc == nvme_ctrlr_get_current_process(ctrlr)) { 633 nvme_qpair_abort_all_queued_reqs(qpair); 634 } 635 636 nvme_robust_mutex_lock(&ctrlr->ctrlr_lock); 637 638 nvme_ctrlr_proc_remove_io_qpair(qpair); 639 640 TAILQ_REMOVE(&ctrlr->active_io_qpairs, qpair, tailq); 641 spdk_nvme_ctrlr_free_qid(ctrlr, qpair->id); 642 643 nvme_transport_ctrlr_delete_io_qpair(ctrlr, qpair); 644 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 645 return 0; 646 } 647 648 static void 649 nvme_ctrlr_construct_intel_support_log_page_list(struct spdk_nvme_ctrlr *ctrlr, 650 struct spdk_nvme_intel_log_page_directory *log_page_directory) 651 { 652 if (log_page_directory == NULL) { 653 return; 654 } 655 656 assert(ctrlr->cdata.vid == SPDK_PCI_VID_INTEL); 657 658 ctrlr->log_page_supported[SPDK_NVME_INTEL_LOG_PAGE_DIRECTORY] = true; 659 660 if (log_page_directory->read_latency_log_len || 661 (ctrlr->quirks & NVME_INTEL_QUIRK_READ_LATENCY)) { 662 ctrlr->log_page_supported[SPDK_NVME_INTEL_LOG_READ_CMD_LATENCY] = true; 663 } 664 if (log_page_directory->write_latency_log_len || 665 (ctrlr->quirks & NVME_INTEL_QUIRK_WRITE_LATENCY)) { 666 ctrlr->log_page_supported[SPDK_NVME_INTEL_LOG_WRITE_CMD_LATENCY] = true; 667 } 668 if (log_page_directory->temperature_statistics_log_len) { 669 ctrlr->log_page_supported[SPDK_NVME_INTEL_LOG_TEMPERATURE] = true; 670 } 671 if (log_page_directory->smart_log_len) { 672 ctrlr->log_page_supported[SPDK_NVME_INTEL_LOG_SMART] = true; 673 } 674 if (log_page_directory->marketing_description_log_len) { 675 ctrlr->log_page_supported[SPDK_NVME_INTEL_MARKETING_DESCRIPTION] = true; 676 } 677 } 678 679 struct intel_log_pages_ctx { 680 struct spdk_nvme_intel_log_page_directory log_page_directory; 681 struct spdk_nvme_ctrlr *ctrlr; 682 }; 683 684 static void 685 nvme_ctrlr_set_intel_support_log_pages_done(void *arg, const struct spdk_nvme_cpl *cpl) 686 { 687 struct intel_log_pages_ctx *ctx = arg; 688 struct spdk_nvme_ctrlr *ctrlr = ctx->ctrlr; 689 690 if (!spdk_nvme_cpl_is_error(cpl)) { 691 nvme_ctrlr_construct_intel_support_log_page_list(ctrlr, &ctx->log_page_directory); 692 } 693 694 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_SUPPORTED_FEATURES, 695 ctrlr->opts.admin_timeout_ms); 696 free(ctx); 697 } 698 699 static int 700 nvme_ctrlr_set_intel_support_log_pages(struct spdk_nvme_ctrlr *ctrlr) 701 { 702 int rc = 0; 703 struct intel_log_pages_ctx *ctx; 704 705 ctx = calloc(1, sizeof(*ctx)); 706 if (!ctx) { 707 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_SUPPORTED_FEATURES, 708 ctrlr->opts.admin_timeout_ms); 709 return 0; 710 } 711 712 ctx->ctrlr = ctrlr; 713 714 rc = spdk_nvme_ctrlr_cmd_get_log_page(ctrlr, SPDK_NVME_INTEL_LOG_PAGE_DIRECTORY, 715 SPDK_NVME_GLOBAL_NS_TAG, &ctx->log_page_directory, 716 sizeof(struct spdk_nvme_intel_log_page_directory), 717 0, nvme_ctrlr_set_intel_support_log_pages_done, ctx); 718 if (rc != 0) { 719 free(ctx); 720 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_SUPPORTED_FEATURES, 721 ctrlr->opts.admin_timeout_ms); 722 return 0; 723 } 724 725 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_WAIT_FOR_SUPPORTED_INTEL_LOG_PAGES, 726 ctrlr->opts.admin_timeout_ms); 727 728 return 0; 729 } 730 731 static int 732 nvme_ctrlr_alloc_ana_log_page(struct spdk_nvme_ctrlr *ctrlr) 733 { 734 uint32_t ana_log_page_size; 735 736 ana_log_page_size = sizeof(struct spdk_nvme_ana_page) + ctrlr->cdata.nanagrpid * 737 sizeof(struct spdk_nvme_ana_group_descriptor) + ctrlr->active_ns_count * 738 sizeof(uint32_t); 739 740 /* Number of active namespaces may have changed. 741 * Check if ANA log page fits into existing buffer. 742 */ 743 if (ana_log_page_size > ctrlr->ana_log_page_size) { 744 void *new_buffer; 745 746 if (ctrlr->ana_log_page) { 747 new_buffer = realloc(ctrlr->ana_log_page, ana_log_page_size); 748 } else { 749 new_buffer = calloc(1, ana_log_page_size); 750 } 751 752 if (!new_buffer) { 753 NVME_CTRLR_ERRLOG(ctrlr, "could not allocate ANA log page buffer, size %u\n", 754 ana_log_page_size); 755 return -ENXIO; 756 } 757 758 ctrlr->ana_log_page = new_buffer; 759 if (ctrlr->copied_ana_desc) { 760 new_buffer = realloc(ctrlr->copied_ana_desc, ana_log_page_size); 761 } else { 762 new_buffer = calloc(1, ana_log_page_size); 763 } 764 765 if (!new_buffer) { 766 NVME_CTRLR_ERRLOG(ctrlr, "could not allocate a buffer to parse ANA descriptor, size %u\n", 767 ana_log_page_size); 768 return -ENOMEM; 769 } 770 771 ctrlr->copied_ana_desc = new_buffer; 772 ctrlr->ana_log_page_size = ana_log_page_size; 773 } 774 775 return 0; 776 } 777 778 static int 779 nvme_ctrlr_update_ana_log_page(struct spdk_nvme_ctrlr *ctrlr) 780 { 781 struct nvme_completion_poll_status *status; 782 int rc; 783 784 rc = nvme_ctrlr_alloc_ana_log_page(ctrlr); 785 if (rc != 0) { 786 return rc; 787 } 788 789 status = calloc(1, sizeof(*status)); 790 if (status == NULL) { 791 NVME_CTRLR_ERRLOG(ctrlr, "Failed to allocate status tracker\n"); 792 return -ENOMEM; 793 } 794 795 rc = spdk_nvme_ctrlr_cmd_get_log_page(ctrlr, SPDK_NVME_LOG_ASYMMETRIC_NAMESPACE_ACCESS, 796 SPDK_NVME_GLOBAL_NS_TAG, ctrlr->ana_log_page, 797 ctrlr->ana_log_page_size, 0, 798 nvme_completion_poll_cb, status); 799 if (rc != 0) { 800 free(status); 801 return rc; 802 } 803 804 if (nvme_wait_for_completion_robust_lock_timeout(ctrlr->adminq, status, &ctrlr->ctrlr_lock, 805 ctrlr->opts.admin_timeout_ms * 1000)) { 806 if (!status->timed_out) { 807 free(status); 808 } 809 return -EIO; 810 } 811 812 free(status); 813 return 0; 814 } 815 816 static int 817 nvme_ctrlr_update_ns_ana_states(const struct spdk_nvme_ana_group_descriptor *desc, 818 void *cb_arg) 819 { 820 struct spdk_nvme_ctrlr *ctrlr = cb_arg; 821 struct spdk_nvme_ns *ns; 822 uint32_t i, nsid; 823 824 for (i = 0; i < desc->num_of_nsid; i++) { 825 nsid = desc->nsid[i]; 826 if (nsid == 0 || nsid > ctrlr->cdata.nn) { 827 continue; 828 } 829 830 ns = spdk_nvme_ctrlr_get_ns(ctrlr, nsid); 831 assert(ns != NULL); 832 833 ns->ana_group_id = desc->ana_group_id; 834 ns->ana_state = desc->ana_state; 835 } 836 837 return 0; 838 } 839 840 int 841 nvme_ctrlr_parse_ana_log_page(struct spdk_nvme_ctrlr *ctrlr, 842 spdk_nvme_parse_ana_log_page_cb cb_fn, void *cb_arg) 843 { 844 struct spdk_nvme_ana_group_descriptor *copied_desc; 845 uint8_t *orig_desc; 846 uint32_t i, desc_size, copy_len; 847 int rc = 0; 848 849 if (ctrlr->ana_log_page == NULL) { 850 return -EINVAL; 851 } 852 853 copied_desc = ctrlr->copied_ana_desc; 854 855 orig_desc = (uint8_t *)ctrlr->ana_log_page + sizeof(struct spdk_nvme_ana_page); 856 copy_len = ctrlr->ana_log_page_size - sizeof(struct spdk_nvme_ana_page); 857 858 for (i = 0; i < ctrlr->ana_log_page->num_ana_group_desc; i++) { 859 memcpy(copied_desc, orig_desc, copy_len); 860 861 rc = cb_fn(copied_desc, cb_arg); 862 if (rc != 0) { 863 break; 864 } 865 866 desc_size = sizeof(struct spdk_nvme_ana_group_descriptor) + 867 copied_desc->num_of_nsid * sizeof(uint32_t); 868 orig_desc += desc_size; 869 copy_len -= desc_size; 870 } 871 872 return rc; 873 } 874 875 static int 876 nvme_ctrlr_set_supported_log_pages(struct spdk_nvme_ctrlr *ctrlr) 877 { 878 int rc = 0; 879 880 memset(ctrlr->log_page_supported, 0, sizeof(ctrlr->log_page_supported)); 881 /* Mandatory pages */ 882 ctrlr->log_page_supported[SPDK_NVME_LOG_ERROR] = true; 883 ctrlr->log_page_supported[SPDK_NVME_LOG_HEALTH_INFORMATION] = true; 884 ctrlr->log_page_supported[SPDK_NVME_LOG_FIRMWARE_SLOT] = true; 885 if (ctrlr->cdata.lpa.celp) { 886 ctrlr->log_page_supported[SPDK_NVME_LOG_COMMAND_EFFECTS_LOG] = true; 887 } 888 889 if (ctrlr->cdata.cmic.ana_reporting) { 890 ctrlr->log_page_supported[SPDK_NVME_LOG_ASYMMETRIC_NAMESPACE_ACCESS] = true; 891 if (!ctrlr->opts.disable_read_ana_log_page) { 892 rc = nvme_ctrlr_update_ana_log_page(ctrlr); 893 if (rc == 0) { 894 nvme_ctrlr_parse_ana_log_page(ctrlr, nvme_ctrlr_update_ns_ana_states, 895 ctrlr); 896 } 897 } 898 } 899 900 if (ctrlr->cdata.ctratt.fdps) { 901 ctrlr->log_page_supported[SPDK_NVME_LOG_FDP_CONFIGURATIONS] = true; 902 ctrlr->log_page_supported[SPDK_NVME_LOG_RECLAIM_UNIT_HANDLE_USAGE] = true; 903 ctrlr->log_page_supported[SPDK_NVME_LOG_FDP_STATISTICS] = true; 904 ctrlr->log_page_supported[SPDK_NVME_LOG_FDP_EVENTS] = true; 905 } 906 907 if (ctrlr->cdata.vid == SPDK_PCI_VID_INTEL && 908 ctrlr->trid.trtype == SPDK_NVME_TRANSPORT_PCIE && 909 !(ctrlr->quirks & NVME_INTEL_QUIRK_NO_LOG_PAGES)) { 910 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_SUPPORTED_INTEL_LOG_PAGES, 911 ctrlr->opts.admin_timeout_ms); 912 913 } else { 914 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_SUPPORTED_FEATURES, 915 ctrlr->opts.admin_timeout_ms); 916 917 } 918 919 return rc; 920 } 921 922 static void 923 nvme_ctrlr_set_intel_supported_features(struct spdk_nvme_ctrlr *ctrlr) 924 { 925 ctrlr->feature_supported[SPDK_NVME_INTEL_FEAT_MAX_LBA] = true; 926 ctrlr->feature_supported[SPDK_NVME_INTEL_FEAT_NATIVE_MAX_LBA] = true; 927 ctrlr->feature_supported[SPDK_NVME_INTEL_FEAT_POWER_GOVERNOR_SETTING] = true; 928 ctrlr->feature_supported[SPDK_NVME_INTEL_FEAT_SMBUS_ADDRESS] = true; 929 ctrlr->feature_supported[SPDK_NVME_INTEL_FEAT_LED_PATTERN] = true; 930 ctrlr->feature_supported[SPDK_NVME_INTEL_FEAT_RESET_TIMED_WORKLOAD_COUNTERS] = true; 931 ctrlr->feature_supported[SPDK_NVME_INTEL_FEAT_LATENCY_TRACKING] = true; 932 } 933 934 static void 935 nvme_ctrlr_set_arbitration_feature(struct spdk_nvme_ctrlr *ctrlr) 936 { 937 uint32_t cdw11; 938 struct nvme_completion_poll_status *status; 939 940 if (ctrlr->opts.arbitration_burst == 0) { 941 return; 942 } 943 944 if (ctrlr->opts.arbitration_burst > 7) { 945 NVME_CTRLR_WARNLOG(ctrlr, "Valid arbitration burst values is from 0-7\n"); 946 return; 947 } 948 949 status = calloc(1, sizeof(*status)); 950 if (!status) { 951 NVME_CTRLR_ERRLOG(ctrlr, "Failed to allocate status tracker\n"); 952 return; 953 } 954 955 cdw11 = ctrlr->opts.arbitration_burst; 956 957 if (spdk_nvme_ctrlr_get_flags(ctrlr) & SPDK_NVME_CTRLR_WRR_SUPPORTED) { 958 cdw11 |= (uint32_t)ctrlr->opts.low_priority_weight << 8; 959 cdw11 |= (uint32_t)ctrlr->opts.medium_priority_weight << 16; 960 cdw11 |= (uint32_t)ctrlr->opts.high_priority_weight << 24; 961 } 962 963 if (spdk_nvme_ctrlr_cmd_set_feature(ctrlr, SPDK_NVME_FEAT_ARBITRATION, 964 cdw11, 0, NULL, 0, 965 nvme_completion_poll_cb, status) < 0) { 966 NVME_CTRLR_ERRLOG(ctrlr, "Set arbitration feature failed\n"); 967 free(status); 968 return; 969 } 970 971 if (nvme_wait_for_completion_timeout(ctrlr->adminq, status, 972 ctrlr->opts.admin_timeout_ms * 1000)) { 973 NVME_CTRLR_ERRLOG(ctrlr, "Timeout to set arbitration feature\n"); 974 } 975 976 if (!status->timed_out) { 977 free(status); 978 } 979 } 980 981 static void 982 nvme_ctrlr_set_supported_features(struct spdk_nvme_ctrlr *ctrlr) 983 { 984 memset(ctrlr->feature_supported, 0, sizeof(ctrlr->feature_supported)); 985 /* Mandatory features */ 986 ctrlr->feature_supported[SPDK_NVME_FEAT_ARBITRATION] = true; 987 ctrlr->feature_supported[SPDK_NVME_FEAT_POWER_MANAGEMENT] = true; 988 ctrlr->feature_supported[SPDK_NVME_FEAT_TEMPERATURE_THRESHOLD] = true; 989 ctrlr->feature_supported[SPDK_NVME_FEAT_ERROR_RECOVERY] = true; 990 ctrlr->feature_supported[SPDK_NVME_FEAT_NUMBER_OF_QUEUES] = true; 991 ctrlr->feature_supported[SPDK_NVME_FEAT_INTERRUPT_COALESCING] = true; 992 ctrlr->feature_supported[SPDK_NVME_FEAT_INTERRUPT_VECTOR_CONFIGURATION] = true; 993 ctrlr->feature_supported[SPDK_NVME_FEAT_WRITE_ATOMICITY] = true; 994 ctrlr->feature_supported[SPDK_NVME_FEAT_ASYNC_EVENT_CONFIGURATION] = true; 995 /* Optional features */ 996 if (ctrlr->cdata.vwc.present) { 997 ctrlr->feature_supported[SPDK_NVME_FEAT_VOLATILE_WRITE_CACHE] = true; 998 } 999 if (ctrlr->cdata.apsta.supported) { 1000 ctrlr->feature_supported[SPDK_NVME_FEAT_AUTONOMOUS_POWER_STATE_TRANSITION] = true; 1001 } 1002 if (ctrlr->cdata.hmpre) { 1003 ctrlr->feature_supported[SPDK_NVME_FEAT_HOST_MEM_BUFFER] = true; 1004 } 1005 if (ctrlr->cdata.vid == SPDK_PCI_VID_INTEL) { 1006 nvme_ctrlr_set_intel_supported_features(ctrlr); 1007 } 1008 1009 nvme_ctrlr_set_arbitration_feature(ctrlr); 1010 } 1011 1012 bool 1013 spdk_nvme_ctrlr_is_failed(struct spdk_nvme_ctrlr *ctrlr) 1014 { 1015 return ctrlr->is_failed; 1016 } 1017 1018 void 1019 nvme_ctrlr_fail(struct spdk_nvme_ctrlr *ctrlr, bool hot_remove) 1020 { 1021 /* 1022 * Set the flag here and leave the work failure of qpairs to 1023 * spdk_nvme_qpair_process_completions(). 1024 */ 1025 if (hot_remove) { 1026 ctrlr->is_removed = true; 1027 } 1028 1029 if (ctrlr->is_failed) { 1030 NVME_CTRLR_NOTICELOG(ctrlr, "already in failed state\n"); 1031 return; 1032 } 1033 1034 if (ctrlr->is_disconnecting) { 1035 NVME_CTRLR_DEBUGLOG(ctrlr, "already disconnecting\n"); 1036 return; 1037 } 1038 1039 ctrlr->is_failed = true; 1040 nvme_transport_ctrlr_disconnect_qpair(ctrlr, ctrlr->adminq); 1041 NVME_CTRLR_ERRLOG(ctrlr, "in failed state.\n"); 1042 } 1043 1044 /** 1045 * This public API function will try to take the controller lock. 1046 * Any private functions being called from a thread already holding 1047 * the ctrlr lock should call nvme_ctrlr_fail directly. 1048 */ 1049 void 1050 spdk_nvme_ctrlr_fail(struct spdk_nvme_ctrlr *ctrlr) 1051 { 1052 nvme_robust_mutex_lock(&ctrlr->ctrlr_lock); 1053 nvme_ctrlr_fail(ctrlr, false); 1054 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 1055 } 1056 1057 static void 1058 nvme_ctrlr_shutdown_set_cc_done(void *_ctx, uint64_t value, const struct spdk_nvme_cpl *cpl) 1059 { 1060 struct nvme_ctrlr_detach_ctx *ctx = _ctx; 1061 struct spdk_nvme_ctrlr *ctrlr = ctx->ctrlr; 1062 1063 if (spdk_nvme_cpl_is_error(cpl)) { 1064 NVME_CTRLR_ERRLOG(ctrlr, "Failed to write CC.SHN\n"); 1065 ctx->shutdown_complete = true; 1066 return; 1067 } 1068 1069 if (ctrlr->opts.no_shn_notification) { 1070 ctx->shutdown_complete = true; 1071 return; 1072 } 1073 1074 /* 1075 * The NVMe specification defines RTD3E to be the time between 1076 * setting SHN = 1 until the controller will set SHST = 10b. 1077 * If the device doesn't report RTD3 entry latency, or if it 1078 * reports RTD3 entry latency less than 10 seconds, pick 1079 * 10 seconds as a reasonable amount of time to 1080 * wait before proceeding. 1081 */ 1082 NVME_CTRLR_DEBUGLOG(ctrlr, "RTD3E = %" PRIu32 " us\n", ctrlr->cdata.rtd3e); 1083 ctx->shutdown_timeout_ms = SPDK_CEIL_DIV(ctrlr->cdata.rtd3e, 1000); 1084 ctx->shutdown_timeout_ms = spdk_max(ctx->shutdown_timeout_ms, 10000); 1085 NVME_CTRLR_DEBUGLOG(ctrlr, "shutdown timeout = %" PRIu32 " ms\n", ctx->shutdown_timeout_ms); 1086 1087 ctx->shutdown_start_tsc = spdk_get_ticks(); 1088 ctx->state = NVME_CTRLR_DETACH_CHECK_CSTS; 1089 } 1090 1091 static void 1092 nvme_ctrlr_shutdown_get_cc_done(void *_ctx, uint64_t value, const struct spdk_nvme_cpl *cpl) 1093 { 1094 struct nvme_ctrlr_detach_ctx *ctx = _ctx; 1095 struct spdk_nvme_ctrlr *ctrlr = ctx->ctrlr; 1096 union spdk_nvme_cc_register cc; 1097 int rc; 1098 1099 if (spdk_nvme_cpl_is_error(cpl)) { 1100 NVME_CTRLR_ERRLOG(ctrlr, "Failed to read the CC register\n"); 1101 ctx->shutdown_complete = true; 1102 return; 1103 } 1104 1105 assert(value <= UINT32_MAX); 1106 cc.raw = (uint32_t)value; 1107 1108 if (ctrlr->opts.no_shn_notification) { 1109 NVME_CTRLR_INFOLOG(ctrlr, "Disable SSD without shutdown notification\n"); 1110 if (cc.bits.en == 0) { 1111 ctx->shutdown_complete = true; 1112 return; 1113 } 1114 1115 cc.bits.en = 0; 1116 } else { 1117 cc.bits.shn = SPDK_NVME_SHN_NORMAL; 1118 } 1119 1120 rc = nvme_ctrlr_set_cc_async(ctrlr, cc.raw, nvme_ctrlr_shutdown_set_cc_done, ctx); 1121 if (rc != 0) { 1122 NVME_CTRLR_ERRLOG(ctrlr, "Failed to write CC.SHN\n"); 1123 ctx->shutdown_complete = true; 1124 } 1125 } 1126 1127 static void 1128 nvme_ctrlr_shutdown_async(struct spdk_nvme_ctrlr *ctrlr, 1129 struct nvme_ctrlr_detach_ctx *ctx) 1130 { 1131 int rc; 1132 1133 if (ctrlr->is_removed) { 1134 ctx->shutdown_complete = true; 1135 return; 1136 } 1137 1138 if (ctrlr->adminq == NULL || 1139 ctrlr->adminq->transport_failure_reason != SPDK_NVME_QPAIR_FAILURE_NONE) { 1140 NVME_CTRLR_INFOLOG(ctrlr, "Adminq is not connected.\n"); 1141 ctx->shutdown_complete = true; 1142 return; 1143 } 1144 1145 ctx->state = NVME_CTRLR_DETACH_SET_CC; 1146 rc = nvme_ctrlr_get_cc_async(ctrlr, nvme_ctrlr_shutdown_get_cc_done, ctx); 1147 if (rc != 0) { 1148 NVME_CTRLR_ERRLOG(ctrlr, "Failed to read the CC register\n"); 1149 ctx->shutdown_complete = true; 1150 } 1151 } 1152 1153 static void 1154 nvme_ctrlr_shutdown_get_csts_done(void *_ctx, uint64_t value, const struct spdk_nvme_cpl *cpl) 1155 { 1156 struct nvme_ctrlr_detach_ctx *ctx = _ctx; 1157 1158 if (spdk_nvme_cpl_is_error(cpl)) { 1159 NVME_CTRLR_ERRLOG(ctx->ctrlr, "Failed to read the CSTS register\n"); 1160 ctx->shutdown_complete = true; 1161 return; 1162 } 1163 1164 assert(value <= UINT32_MAX); 1165 ctx->csts.raw = (uint32_t)value; 1166 ctx->state = NVME_CTRLR_DETACH_GET_CSTS_DONE; 1167 } 1168 1169 static int 1170 nvme_ctrlr_shutdown_poll_async(struct spdk_nvme_ctrlr *ctrlr, 1171 struct nvme_ctrlr_detach_ctx *ctx) 1172 { 1173 union spdk_nvme_csts_register csts; 1174 uint32_t ms_waited; 1175 1176 switch (ctx->state) { 1177 case NVME_CTRLR_DETACH_SET_CC: 1178 case NVME_CTRLR_DETACH_GET_CSTS: 1179 /* We're still waiting for the register operation to complete */ 1180 spdk_nvme_qpair_process_completions(ctrlr->adminq, 0); 1181 return -EAGAIN; 1182 1183 case NVME_CTRLR_DETACH_CHECK_CSTS: 1184 ctx->state = NVME_CTRLR_DETACH_GET_CSTS; 1185 if (nvme_ctrlr_get_csts_async(ctrlr, nvme_ctrlr_shutdown_get_csts_done, ctx)) { 1186 NVME_CTRLR_ERRLOG(ctrlr, "Failed to read the CSTS register\n"); 1187 return -EIO; 1188 } 1189 return -EAGAIN; 1190 1191 case NVME_CTRLR_DETACH_GET_CSTS_DONE: 1192 ctx->state = NVME_CTRLR_DETACH_CHECK_CSTS; 1193 break; 1194 1195 default: 1196 assert(0 && "Should never happen"); 1197 return -EINVAL; 1198 } 1199 1200 ms_waited = (spdk_get_ticks() - ctx->shutdown_start_tsc) * 1000 / spdk_get_ticks_hz(); 1201 csts.raw = ctx->csts.raw; 1202 1203 if (csts.bits.shst == SPDK_NVME_SHST_COMPLETE) { 1204 NVME_CTRLR_DEBUGLOG(ctrlr, "shutdown complete in %u milliseconds\n", ms_waited); 1205 return 0; 1206 } 1207 1208 if (ms_waited < ctx->shutdown_timeout_ms) { 1209 return -EAGAIN; 1210 } 1211 1212 NVME_CTRLR_ERRLOG(ctrlr, "did not shutdown within %u milliseconds\n", 1213 ctx->shutdown_timeout_ms); 1214 if (ctrlr->quirks & NVME_QUIRK_SHST_COMPLETE) { 1215 NVME_CTRLR_ERRLOG(ctrlr, "likely due to shutdown handling in the VMWare emulated NVMe SSD\n"); 1216 } 1217 1218 return 0; 1219 } 1220 1221 static inline uint64_t 1222 nvme_ctrlr_get_ready_timeout(struct spdk_nvme_ctrlr *ctrlr) 1223 { 1224 return ctrlr->cap.bits.to * 500; 1225 } 1226 1227 static void 1228 nvme_ctrlr_set_cc_en_done(void *ctx, uint64_t value, const struct spdk_nvme_cpl *cpl) 1229 { 1230 struct spdk_nvme_ctrlr *ctrlr = ctx; 1231 1232 if (spdk_nvme_cpl_is_error(cpl)) { 1233 NVME_CTRLR_ERRLOG(ctrlr, "Failed to set the CC register\n"); 1234 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 1235 return; 1236 } 1237 1238 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ENABLE_WAIT_FOR_READY_1, 1239 nvme_ctrlr_get_ready_timeout(ctrlr)); 1240 } 1241 1242 static int 1243 nvme_ctrlr_enable(struct spdk_nvme_ctrlr *ctrlr) 1244 { 1245 union spdk_nvme_cc_register cc; 1246 int rc; 1247 1248 rc = nvme_transport_ctrlr_enable(ctrlr); 1249 if (rc != 0) { 1250 NVME_CTRLR_ERRLOG(ctrlr, "transport ctrlr_enable failed\n"); 1251 return rc; 1252 } 1253 1254 cc.raw = ctrlr->process_init_cc.raw; 1255 if (cc.bits.en != 0) { 1256 NVME_CTRLR_ERRLOG(ctrlr, "called with CC.EN = 1\n"); 1257 return -EINVAL; 1258 } 1259 1260 cc.bits.en = 1; 1261 cc.bits.css = 0; 1262 cc.bits.shn = 0; 1263 cc.bits.iosqes = 6; /* SQ entry size == 64 == 2^6 */ 1264 cc.bits.iocqes = 4; /* CQ entry size == 16 == 2^4 */ 1265 1266 /* Page size is 2 ^ (12 + mps). */ 1267 cc.bits.mps = spdk_u32log2(ctrlr->page_size) - 12; 1268 1269 /* 1270 * Since NVMe 1.0, a controller should have at least one bit set in CAP.CSS. 1271 * A controller that does not have any bit set in CAP.CSS is not spec compliant. 1272 * Try to support such a controller regardless. 1273 */ 1274 if (ctrlr->cap.bits.css == 0) { 1275 NVME_CTRLR_INFOLOG(ctrlr, "Drive reports no command sets supported. Assuming NVM is supported.\n"); 1276 ctrlr->cap.bits.css = SPDK_NVME_CAP_CSS_NVM; 1277 } 1278 1279 /* 1280 * If the user did not explicitly request a command set, or supplied a value larger than 1281 * what can be saved in CC.CSS, use the most reasonable default. 1282 */ 1283 if (ctrlr->opts.command_set >= CHAR_BIT) { 1284 if (ctrlr->cap.bits.css & SPDK_NVME_CAP_CSS_IOCS) { 1285 ctrlr->opts.command_set = SPDK_NVME_CC_CSS_IOCS; 1286 } else if (ctrlr->cap.bits.css & SPDK_NVME_CAP_CSS_NVM) { 1287 ctrlr->opts.command_set = SPDK_NVME_CC_CSS_NVM; 1288 } else if (ctrlr->cap.bits.css & SPDK_NVME_CAP_CSS_NOIO) { 1289 ctrlr->opts.command_set = SPDK_NVME_CC_CSS_NOIO; 1290 } else { 1291 /* Invalid supported bits detected, falling back to NVM. */ 1292 ctrlr->opts.command_set = SPDK_NVME_CC_CSS_NVM; 1293 } 1294 } 1295 1296 /* Verify that the selected command set is supported by the controller. */ 1297 if (!(ctrlr->cap.bits.css & (1u << ctrlr->opts.command_set))) { 1298 NVME_CTRLR_DEBUGLOG(ctrlr, "Requested I/O command set %u but supported mask is 0x%x\n", 1299 ctrlr->opts.command_set, ctrlr->cap.bits.css); 1300 NVME_CTRLR_DEBUGLOG(ctrlr, "Falling back to NVM. Assuming NVM is supported.\n"); 1301 ctrlr->opts.command_set = SPDK_NVME_CC_CSS_NVM; 1302 } 1303 1304 cc.bits.css = ctrlr->opts.command_set; 1305 1306 switch (ctrlr->opts.arb_mechanism) { 1307 case SPDK_NVME_CC_AMS_RR: 1308 break; 1309 case SPDK_NVME_CC_AMS_WRR: 1310 if (SPDK_NVME_CAP_AMS_WRR & ctrlr->cap.bits.ams) { 1311 break; 1312 } 1313 return -EINVAL; 1314 case SPDK_NVME_CC_AMS_VS: 1315 if (SPDK_NVME_CAP_AMS_VS & ctrlr->cap.bits.ams) { 1316 break; 1317 } 1318 return -EINVAL; 1319 default: 1320 return -EINVAL; 1321 } 1322 1323 cc.bits.ams = ctrlr->opts.arb_mechanism; 1324 ctrlr->process_init_cc.raw = cc.raw; 1325 1326 if (nvme_ctrlr_set_cc_async(ctrlr, cc.raw, nvme_ctrlr_set_cc_en_done, ctrlr)) { 1327 NVME_CTRLR_ERRLOG(ctrlr, "set_cc() failed\n"); 1328 return -EIO; 1329 } 1330 1331 return 0; 1332 } 1333 1334 static const char * 1335 nvme_ctrlr_state_string(enum nvme_ctrlr_state state) 1336 { 1337 switch (state) { 1338 case NVME_CTRLR_STATE_INIT_DELAY: 1339 return "delay init"; 1340 case NVME_CTRLR_STATE_CONNECT_ADMINQ: 1341 return "connect adminq"; 1342 case NVME_CTRLR_STATE_WAIT_FOR_CONNECT_ADMINQ: 1343 return "wait for connect adminq"; 1344 case NVME_CTRLR_STATE_READ_VS: 1345 return "read vs"; 1346 case NVME_CTRLR_STATE_READ_VS_WAIT_FOR_VS: 1347 return "read vs wait for vs"; 1348 case NVME_CTRLR_STATE_READ_CAP: 1349 return "read cap"; 1350 case NVME_CTRLR_STATE_READ_CAP_WAIT_FOR_CAP: 1351 return "read cap wait for cap"; 1352 case NVME_CTRLR_STATE_CHECK_EN: 1353 return "check en"; 1354 case NVME_CTRLR_STATE_CHECK_EN_WAIT_FOR_CC: 1355 return "check en wait for cc"; 1356 case NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_1: 1357 return "disable and wait for CSTS.RDY = 1"; 1358 case NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_1_WAIT_FOR_CSTS: 1359 return "disable and wait for CSTS.RDY = 1 reg"; 1360 case NVME_CTRLR_STATE_SET_EN_0: 1361 return "set CC.EN = 0"; 1362 case NVME_CTRLR_STATE_SET_EN_0_WAIT_FOR_CC: 1363 return "set CC.EN = 0 wait for cc"; 1364 case NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0: 1365 return "disable and wait for CSTS.RDY = 0"; 1366 case NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0_WAIT_FOR_CSTS: 1367 return "disable and wait for CSTS.RDY = 0 reg"; 1368 case NVME_CTRLR_STATE_DISABLED: 1369 return "controller is disabled"; 1370 case NVME_CTRLR_STATE_ENABLE: 1371 return "enable controller by writing CC.EN = 1"; 1372 case NVME_CTRLR_STATE_ENABLE_WAIT_FOR_CC: 1373 return "enable controller by writing CC.EN = 1 reg"; 1374 case NVME_CTRLR_STATE_ENABLE_WAIT_FOR_READY_1: 1375 return "wait for CSTS.RDY = 1"; 1376 case NVME_CTRLR_STATE_ENABLE_WAIT_FOR_READY_1_WAIT_FOR_CSTS: 1377 return "wait for CSTS.RDY = 1 reg"; 1378 case NVME_CTRLR_STATE_RESET_ADMIN_QUEUE: 1379 return "reset admin queue"; 1380 case NVME_CTRLR_STATE_IDENTIFY: 1381 return "identify controller"; 1382 case NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY: 1383 return "wait for identify controller"; 1384 case NVME_CTRLR_STATE_CONFIGURE_AER: 1385 return "configure AER"; 1386 case NVME_CTRLR_STATE_WAIT_FOR_CONFIGURE_AER: 1387 return "wait for configure aer"; 1388 case NVME_CTRLR_STATE_SET_KEEP_ALIVE_TIMEOUT: 1389 return "set keep alive timeout"; 1390 case NVME_CTRLR_STATE_WAIT_FOR_KEEP_ALIVE_TIMEOUT: 1391 return "wait for set keep alive timeout"; 1392 case NVME_CTRLR_STATE_IDENTIFY_IOCS_SPECIFIC: 1393 return "identify controller iocs specific"; 1394 case NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_IOCS_SPECIFIC: 1395 return "wait for identify controller iocs specific"; 1396 case NVME_CTRLR_STATE_GET_ZNS_CMD_EFFECTS_LOG: 1397 return "get zns cmd and effects log page"; 1398 case NVME_CTRLR_STATE_WAIT_FOR_GET_ZNS_CMD_EFFECTS_LOG: 1399 return "wait for get zns cmd and effects log page"; 1400 case NVME_CTRLR_STATE_SET_NUM_QUEUES: 1401 return "set number of queues"; 1402 case NVME_CTRLR_STATE_WAIT_FOR_SET_NUM_QUEUES: 1403 return "wait for set number of queues"; 1404 case NVME_CTRLR_STATE_IDENTIFY_ACTIVE_NS: 1405 return "identify active ns"; 1406 case NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_ACTIVE_NS: 1407 return "wait for identify active ns"; 1408 case NVME_CTRLR_STATE_IDENTIFY_NS: 1409 return "identify ns"; 1410 case NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_NS: 1411 return "wait for identify ns"; 1412 case NVME_CTRLR_STATE_IDENTIFY_ID_DESCS: 1413 return "identify namespace id descriptors"; 1414 case NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_ID_DESCS: 1415 return "wait for identify namespace id descriptors"; 1416 case NVME_CTRLR_STATE_IDENTIFY_NS_IOCS_SPECIFIC: 1417 return "identify ns iocs specific"; 1418 case NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_NS_IOCS_SPECIFIC: 1419 return "wait for identify ns iocs specific"; 1420 case NVME_CTRLR_STATE_SET_SUPPORTED_LOG_PAGES: 1421 return "set supported log pages"; 1422 case NVME_CTRLR_STATE_SET_SUPPORTED_INTEL_LOG_PAGES: 1423 return "set supported INTEL log pages"; 1424 case NVME_CTRLR_STATE_WAIT_FOR_SUPPORTED_INTEL_LOG_PAGES: 1425 return "wait for supported INTEL log pages"; 1426 case NVME_CTRLR_STATE_SET_SUPPORTED_FEATURES: 1427 return "set supported features"; 1428 case NVME_CTRLR_STATE_SET_DB_BUF_CFG: 1429 return "set doorbell buffer config"; 1430 case NVME_CTRLR_STATE_WAIT_FOR_DB_BUF_CFG: 1431 return "wait for doorbell buffer config"; 1432 case NVME_CTRLR_STATE_SET_HOST_ID: 1433 return "set host ID"; 1434 case NVME_CTRLR_STATE_WAIT_FOR_HOST_ID: 1435 return "wait for set host ID"; 1436 case NVME_CTRLR_STATE_TRANSPORT_READY: 1437 return "transport ready"; 1438 case NVME_CTRLR_STATE_READY: 1439 return "ready"; 1440 case NVME_CTRLR_STATE_ERROR: 1441 return "error"; 1442 case NVME_CTRLR_STATE_DISCONNECTED: 1443 return "disconnected"; 1444 } 1445 return "unknown"; 1446 }; 1447 1448 static void 1449 _nvme_ctrlr_set_state(struct spdk_nvme_ctrlr *ctrlr, enum nvme_ctrlr_state state, 1450 uint64_t timeout_in_ms, bool quiet) 1451 { 1452 uint64_t ticks_per_ms, timeout_in_ticks, now_ticks; 1453 1454 ctrlr->state = state; 1455 if (timeout_in_ms == NVME_TIMEOUT_KEEP_EXISTING) { 1456 if (!quiet) { 1457 NVME_CTRLR_DEBUGLOG(ctrlr, "setting state to %s (keeping existing timeout)\n", 1458 nvme_ctrlr_state_string(ctrlr->state)); 1459 } 1460 return; 1461 } 1462 1463 if (timeout_in_ms == NVME_TIMEOUT_INFINITE) { 1464 goto inf; 1465 } 1466 1467 ticks_per_ms = spdk_get_ticks_hz() / 1000; 1468 if (timeout_in_ms > UINT64_MAX / ticks_per_ms) { 1469 NVME_CTRLR_ERRLOG(ctrlr, 1470 "Specified timeout would cause integer overflow. Defaulting to no timeout.\n"); 1471 goto inf; 1472 } 1473 1474 now_ticks = spdk_get_ticks(); 1475 timeout_in_ticks = timeout_in_ms * ticks_per_ms; 1476 if (timeout_in_ticks > UINT64_MAX - now_ticks) { 1477 NVME_CTRLR_ERRLOG(ctrlr, 1478 "Specified timeout would cause integer overflow. Defaulting to no timeout.\n"); 1479 goto inf; 1480 } 1481 1482 ctrlr->state_timeout_tsc = timeout_in_ticks + now_ticks; 1483 if (!quiet) { 1484 NVME_CTRLR_DEBUGLOG(ctrlr, "setting state to %s (timeout %" PRIu64 " ms)\n", 1485 nvme_ctrlr_state_string(ctrlr->state), timeout_in_ms); 1486 } 1487 return; 1488 inf: 1489 if (!quiet) { 1490 NVME_CTRLR_DEBUGLOG(ctrlr, "setting state to %s (no timeout)\n", 1491 nvme_ctrlr_state_string(ctrlr->state)); 1492 } 1493 ctrlr->state_timeout_tsc = NVME_TIMEOUT_INFINITE; 1494 } 1495 1496 static void 1497 nvme_ctrlr_set_state(struct spdk_nvme_ctrlr *ctrlr, enum nvme_ctrlr_state state, 1498 uint64_t timeout_in_ms) 1499 { 1500 _nvme_ctrlr_set_state(ctrlr, state, timeout_in_ms, false); 1501 } 1502 1503 static void 1504 nvme_ctrlr_set_state_quiet(struct spdk_nvme_ctrlr *ctrlr, enum nvme_ctrlr_state state, 1505 uint64_t timeout_in_ms) 1506 { 1507 _nvme_ctrlr_set_state(ctrlr, state, timeout_in_ms, true); 1508 } 1509 1510 static void 1511 nvme_ctrlr_free_zns_specific_data(struct spdk_nvme_ctrlr *ctrlr) 1512 { 1513 spdk_free(ctrlr->cdata_zns); 1514 ctrlr->cdata_zns = NULL; 1515 } 1516 1517 static void 1518 nvme_ctrlr_free_iocs_specific_data(struct spdk_nvme_ctrlr *ctrlr) 1519 { 1520 nvme_ctrlr_free_zns_specific_data(ctrlr); 1521 } 1522 1523 static void 1524 nvme_ctrlr_free_doorbell_buffer(struct spdk_nvme_ctrlr *ctrlr) 1525 { 1526 if (ctrlr->shadow_doorbell) { 1527 spdk_free(ctrlr->shadow_doorbell); 1528 ctrlr->shadow_doorbell = NULL; 1529 } 1530 1531 if (ctrlr->eventidx) { 1532 spdk_free(ctrlr->eventidx); 1533 ctrlr->eventidx = NULL; 1534 } 1535 } 1536 1537 static void 1538 nvme_ctrlr_set_doorbell_buffer_config_done(void *arg, const struct spdk_nvme_cpl *cpl) 1539 { 1540 struct spdk_nvme_ctrlr *ctrlr = (struct spdk_nvme_ctrlr *)arg; 1541 1542 if (spdk_nvme_cpl_is_error(cpl)) { 1543 NVME_CTRLR_WARNLOG(ctrlr, "Doorbell buffer config failed\n"); 1544 } else { 1545 NVME_CTRLR_INFOLOG(ctrlr, "Doorbell buffer config enabled\n"); 1546 } 1547 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_HOST_ID, 1548 ctrlr->opts.admin_timeout_ms); 1549 } 1550 1551 static int 1552 nvme_ctrlr_set_doorbell_buffer_config(struct spdk_nvme_ctrlr *ctrlr) 1553 { 1554 int rc = 0; 1555 uint64_t prp1, prp2, len; 1556 1557 if (!ctrlr->cdata.oacs.doorbell_buffer_config) { 1558 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_HOST_ID, 1559 ctrlr->opts.admin_timeout_ms); 1560 return 0; 1561 } 1562 1563 if (ctrlr->trid.trtype != SPDK_NVME_TRANSPORT_PCIE) { 1564 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_HOST_ID, 1565 ctrlr->opts.admin_timeout_ms); 1566 return 0; 1567 } 1568 1569 /* only 1 page size for doorbell buffer */ 1570 ctrlr->shadow_doorbell = spdk_zmalloc(ctrlr->page_size, ctrlr->page_size, 1571 NULL, SPDK_ENV_LCORE_ID_ANY, 1572 SPDK_MALLOC_DMA | SPDK_MALLOC_SHARE); 1573 if (ctrlr->shadow_doorbell == NULL) { 1574 rc = -ENOMEM; 1575 goto error; 1576 } 1577 1578 len = ctrlr->page_size; 1579 prp1 = spdk_vtophys(ctrlr->shadow_doorbell, &len); 1580 if (prp1 == SPDK_VTOPHYS_ERROR || len != ctrlr->page_size) { 1581 rc = -EFAULT; 1582 goto error; 1583 } 1584 1585 ctrlr->eventidx = spdk_zmalloc(ctrlr->page_size, ctrlr->page_size, 1586 NULL, SPDK_ENV_LCORE_ID_ANY, 1587 SPDK_MALLOC_DMA | SPDK_MALLOC_SHARE); 1588 if (ctrlr->eventidx == NULL) { 1589 rc = -ENOMEM; 1590 goto error; 1591 } 1592 1593 len = ctrlr->page_size; 1594 prp2 = spdk_vtophys(ctrlr->eventidx, &len); 1595 if (prp2 == SPDK_VTOPHYS_ERROR || len != ctrlr->page_size) { 1596 rc = -EFAULT; 1597 goto error; 1598 } 1599 1600 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_WAIT_FOR_DB_BUF_CFG, 1601 ctrlr->opts.admin_timeout_ms); 1602 1603 rc = nvme_ctrlr_cmd_doorbell_buffer_config(ctrlr, prp1, prp2, 1604 nvme_ctrlr_set_doorbell_buffer_config_done, ctrlr); 1605 if (rc != 0) { 1606 goto error; 1607 } 1608 1609 return 0; 1610 1611 error: 1612 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 1613 nvme_ctrlr_free_doorbell_buffer(ctrlr); 1614 return rc; 1615 } 1616 1617 void 1618 nvme_ctrlr_abort_queued_aborts(struct spdk_nvme_ctrlr *ctrlr) 1619 { 1620 struct nvme_request *req, *tmp; 1621 struct spdk_nvme_cpl cpl = {}; 1622 1623 cpl.status.sc = SPDK_NVME_SC_ABORTED_SQ_DELETION; 1624 cpl.status.sct = SPDK_NVME_SCT_GENERIC; 1625 1626 STAILQ_FOREACH_SAFE(req, &ctrlr->queued_aborts, stailq, tmp) { 1627 STAILQ_REMOVE_HEAD(&ctrlr->queued_aborts, stailq); 1628 ctrlr->outstanding_aborts++; 1629 1630 nvme_complete_request(req->cb_fn, req->cb_arg, req->qpair, req, &cpl); 1631 } 1632 } 1633 1634 static int 1635 nvme_ctrlr_disconnect(struct spdk_nvme_ctrlr *ctrlr) 1636 { 1637 if (ctrlr->is_resetting || ctrlr->is_removed) { 1638 /* 1639 * Controller is already resetting or has been removed. Return 1640 * immediately since there is no need to kick off another 1641 * reset in these cases. 1642 */ 1643 return ctrlr->is_resetting ? -EBUSY : -ENXIO; 1644 } 1645 1646 ctrlr->is_resetting = true; 1647 ctrlr->is_failed = false; 1648 ctrlr->is_disconnecting = true; 1649 ctrlr->prepare_for_reset = true; 1650 1651 NVME_CTRLR_NOTICELOG(ctrlr, "resetting controller\n"); 1652 1653 /* Disable keep-alive, it'll be re-enabled as part of the init process */ 1654 ctrlr->keep_alive_interval_ticks = 0; 1655 1656 /* Abort all of the queued abort requests */ 1657 nvme_ctrlr_abort_queued_aborts(ctrlr); 1658 1659 nvme_transport_admin_qpair_abort_aers(ctrlr->adminq); 1660 1661 ctrlr->adminq->transport_failure_reason = SPDK_NVME_QPAIR_FAILURE_LOCAL; 1662 nvme_transport_ctrlr_disconnect_qpair(ctrlr, ctrlr->adminq); 1663 1664 return 0; 1665 } 1666 1667 static void 1668 nvme_ctrlr_disconnect_done(struct spdk_nvme_ctrlr *ctrlr) 1669 { 1670 assert(ctrlr->is_failed == false); 1671 ctrlr->is_disconnecting = false; 1672 1673 /* Doorbell buffer config is invalid during reset */ 1674 nvme_ctrlr_free_doorbell_buffer(ctrlr); 1675 1676 /* I/O Command Set Specific Identify Controller data is invalidated during reset */ 1677 nvme_ctrlr_free_iocs_specific_data(ctrlr); 1678 1679 spdk_bit_array_free(&ctrlr->free_io_qids); 1680 1681 /* Set the state back to DISCONNECTED to cause a full hardware reset. */ 1682 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_DISCONNECTED, NVME_TIMEOUT_INFINITE); 1683 } 1684 1685 int 1686 spdk_nvme_ctrlr_disconnect(struct spdk_nvme_ctrlr *ctrlr) 1687 { 1688 int rc; 1689 1690 nvme_robust_mutex_lock(&ctrlr->ctrlr_lock); 1691 rc = nvme_ctrlr_disconnect(ctrlr); 1692 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 1693 1694 return rc; 1695 } 1696 1697 void 1698 spdk_nvme_ctrlr_reconnect_async(struct spdk_nvme_ctrlr *ctrlr) 1699 { 1700 nvme_robust_mutex_lock(&ctrlr->ctrlr_lock); 1701 1702 ctrlr->prepare_for_reset = false; 1703 1704 /* Set the state back to INIT to cause a full hardware reset. */ 1705 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_INIT, NVME_TIMEOUT_INFINITE); 1706 1707 /* Return without releasing ctrlr_lock. ctrlr_lock will be released when 1708 * spdk_nvme_ctrlr_reset_poll_async() returns 0. 1709 */ 1710 } 1711 1712 int 1713 nvme_ctrlr_reinitialize_io_qpair(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair) 1714 { 1715 bool async; 1716 int rc; 1717 1718 if (nvme_ctrlr_get_current_process(ctrlr) != qpair->active_proc || 1719 spdk_nvme_ctrlr_is_fabrics(ctrlr) || nvme_qpair_is_admin_queue(qpair)) { 1720 assert(false); 1721 return -EINVAL; 1722 } 1723 1724 /* Force a synchronous connect. */ 1725 async = qpair->async; 1726 qpair->async = false; 1727 rc = nvme_transport_ctrlr_connect_qpair(ctrlr, qpair); 1728 qpair->async = async; 1729 1730 if (rc != 0) { 1731 qpair->transport_failure_reason = SPDK_NVME_QPAIR_FAILURE_LOCAL; 1732 } 1733 1734 return rc; 1735 } 1736 1737 /** 1738 * This function will be called when the controller is being reinitialized. 1739 * Note: the ctrlr_lock must be held when calling this function. 1740 */ 1741 int 1742 spdk_nvme_ctrlr_reconnect_poll_async(struct spdk_nvme_ctrlr *ctrlr) 1743 { 1744 struct spdk_nvme_ns *ns, *tmp_ns; 1745 struct spdk_nvme_qpair *qpair; 1746 int rc = 0, rc_tmp = 0; 1747 1748 if (nvme_ctrlr_process_init(ctrlr) != 0) { 1749 NVME_CTRLR_ERRLOG(ctrlr, "controller reinitialization failed\n"); 1750 rc = -1; 1751 } 1752 if (ctrlr->state != NVME_CTRLR_STATE_READY && rc != -1) { 1753 return -EAGAIN; 1754 } 1755 1756 /* 1757 * For non-fabrics controllers, the memory locations of the transport qpair 1758 * don't change when the controller is reset. They simply need to be 1759 * re-enabled with admin commands to the controller. For fabric 1760 * controllers we need to disconnect and reconnect the qpair on its 1761 * own thread outside of the context of the reset. 1762 */ 1763 if (rc == 0 && !spdk_nvme_ctrlr_is_fabrics(ctrlr)) { 1764 /* Reinitialize qpairs */ 1765 TAILQ_FOREACH(qpair, &ctrlr->active_io_qpairs, tailq) { 1766 /* Always clear the qid bit here, even for a foreign qpair. We need 1767 * to make sure another process doesn't get the chance to grab that 1768 * qid. 1769 */ 1770 assert(spdk_bit_array_get(ctrlr->free_io_qids, qpair->id)); 1771 spdk_bit_array_clear(ctrlr->free_io_qids, qpair->id); 1772 if (nvme_ctrlr_get_current_process(ctrlr) != qpair->active_proc) { 1773 /* 1774 * We cannot reinitialize a foreign qpair. The qpair's owning 1775 * process will take care of it. Set failure reason to FAILURE_RESET 1776 * to ensure that happens. 1777 */ 1778 qpair->transport_failure_reason = SPDK_NVME_QPAIR_FAILURE_RESET; 1779 continue; 1780 } 1781 rc_tmp = nvme_ctrlr_reinitialize_io_qpair(ctrlr, qpair); 1782 if (rc_tmp != 0) { 1783 rc = rc_tmp; 1784 } 1785 } 1786 } 1787 1788 /* 1789 * Take this opportunity to remove inactive namespaces. During a reset namespace 1790 * handles can be invalidated. 1791 */ 1792 RB_FOREACH_SAFE(ns, nvme_ns_tree, &ctrlr->ns, tmp_ns) { 1793 if (!ns->active) { 1794 RB_REMOVE(nvme_ns_tree, &ctrlr->ns, ns); 1795 spdk_free(ns); 1796 } 1797 } 1798 1799 if (rc) { 1800 nvme_ctrlr_fail(ctrlr, false); 1801 } 1802 ctrlr->is_resetting = false; 1803 1804 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 1805 1806 if (!ctrlr->cdata.oaes.ns_attribute_notices) { 1807 /* 1808 * If controller doesn't support ns_attribute_notices and 1809 * namespace attributes change (e.g. number of namespaces) 1810 * we need to update system handling device reset. 1811 */ 1812 nvme_io_msg_ctrlr_update(ctrlr); 1813 } 1814 1815 return rc; 1816 } 1817 1818 /* 1819 * For PCIe transport, spdk_nvme_ctrlr_disconnect() will do a Controller Level Reset 1820 * (Change CC.EN from 1 to 0) as a operation to disconnect the admin qpair. 1821 * The following two functions are added to do a Controller Level Reset. They have 1822 * to be called under the nvme controller's lock. 1823 */ 1824 void 1825 nvme_ctrlr_disable(struct spdk_nvme_ctrlr *ctrlr) 1826 { 1827 assert(ctrlr->is_disconnecting == true); 1828 1829 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_CHECK_EN, NVME_TIMEOUT_INFINITE); 1830 } 1831 1832 int 1833 nvme_ctrlr_disable_poll(struct spdk_nvme_ctrlr *ctrlr) 1834 { 1835 int rc = 0; 1836 1837 if (nvme_ctrlr_process_init(ctrlr) != 0) { 1838 NVME_CTRLR_ERRLOG(ctrlr, "failed to disable controller\n"); 1839 rc = -1; 1840 } 1841 1842 if (ctrlr->state != NVME_CTRLR_STATE_DISABLED && rc != -1) { 1843 return -EAGAIN; 1844 } 1845 1846 return rc; 1847 } 1848 1849 static void 1850 nvme_ctrlr_fail_io_qpairs(struct spdk_nvme_ctrlr *ctrlr) 1851 { 1852 struct spdk_nvme_qpair *qpair; 1853 1854 TAILQ_FOREACH(qpair, &ctrlr->active_io_qpairs, tailq) { 1855 qpair->transport_failure_reason = SPDK_NVME_QPAIR_FAILURE_LOCAL; 1856 } 1857 } 1858 1859 int 1860 spdk_nvme_ctrlr_reset(struct spdk_nvme_ctrlr *ctrlr) 1861 { 1862 int rc; 1863 1864 nvme_robust_mutex_lock(&ctrlr->ctrlr_lock); 1865 1866 rc = nvme_ctrlr_disconnect(ctrlr); 1867 if (rc == 0) { 1868 nvme_ctrlr_fail_io_qpairs(ctrlr); 1869 } 1870 1871 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 1872 1873 if (rc != 0) { 1874 if (rc == -EBUSY) { 1875 rc = 0; 1876 } 1877 return rc; 1878 } 1879 1880 while (1) { 1881 rc = spdk_nvme_ctrlr_process_admin_completions(ctrlr); 1882 if (rc == -ENXIO) { 1883 break; 1884 } 1885 } 1886 1887 spdk_nvme_ctrlr_reconnect_async(ctrlr); 1888 1889 while (true) { 1890 rc = spdk_nvme_ctrlr_reconnect_poll_async(ctrlr); 1891 if (rc != -EAGAIN) { 1892 break; 1893 } 1894 } 1895 1896 return rc; 1897 } 1898 1899 int 1900 spdk_nvme_ctrlr_reset_subsystem(struct spdk_nvme_ctrlr *ctrlr) 1901 { 1902 union spdk_nvme_cap_register cap; 1903 int rc = 0; 1904 1905 cap = spdk_nvme_ctrlr_get_regs_cap(ctrlr); 1906 if (cap.bits.nssrs == 0) { 1907 NVME_CTRLR_WARNLOG(ctrlr, "subsystem reset is not supported\n"); 1908 return -ENOTSUP; 1909 } 1910 1911 NVME_CTRLR_NOTICELOG(ctrlr, "resetting subsystem\n"); 1912 nvme_robust_mutex_lock(&ctrlr->ctrlr_lock); 1913 ctrlr->is_resetting = true; 1914 rc = nvme_ctrlr_set_nssr(ctrlr, SPDK_NVME_NSSR_VALUE); 1915 ctrlr->is_resetting = false; 1916 1917 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 1918 /* 1919 * No more cleanup at this point like in the ctrlr reset. A subsystem reset will cause 1920 * a hot remove for PCIe transport. The hot remove handling does all the necessary ctrlr cleanup. 1921 */ 1922 return rc; 1923 } 1924 1925 int 1926 spdk_nvme_ctrlr_set_trid(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_transport_id *trid) 1927 { 1928 int rc = 0; 1929 1930 nvme_robust_mutex_lock(&ctrlr->ctrlr_lock); 1931 1932 if (ctrlr->is_failed == false) { 1933 rc = -EPERM; 1934 goto out; 1935 } 1936 1937 if (trid->trtype != ctrlr->trid.trtype) { 1938 rc = -EINVAL; 1939 goto out; 1940 } 1941 1942 if (strncmp(trid->subnqn, ctrlr->trid.subnqn, SPDK_NVMF_NQN_MAX_LEN)) { 1943 rc = -EINVAL; 1944 goto out; 1945 } 1946 1947 ctrlr->trid = *trid; 1948 1949 out: 1950 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 1951 return rc; 1952 } 1953 1954 void 1955 spdk_nvme_ctrlr_set_remove_cb(struct spdk_nvme_ctrlr *ctrlr, 1956 spdk_nvme_remove_cb remove_cb, void *remove_ctx) 1957 { 1958 if (!spdk_process_is_primary()) { 1959 return; 1960 } 1961 1962 nvme_robust_mutex_lock(&ctrlr->ctrlr_lock); 1963 ctrlr->remove_cb = remove_cb; 1964 ctrlr->cb_ctx = remove_ctx; 1965 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 1966 } 1967 1968 static void 1969 nvme_ctrlr_identify_done(void *arg, const struct spdk_nvme_cpl *cpl) 1970 { 1971 struct spdk_nvme_ctrlr *ctrlr = (struct spdk_nvme_ctrlr *)arg; 1972 1973 if (spdk_nvme_cpl_is_error(cpl)) { 1974 NVME_CTRLR_ERRLOG(ctrlr, "nvme_identify_controller failed!\n"); 1975 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 1976 return; 1977 } 1978 1979 /* 1980 * Use MDTS to ensure our default max_xfer_size doesn't exceed what the 1981 * controller supports. 1982 */ 1983 ctrlr->max_xfer_size = nvme_transport_ctrlr_get_max_xfer_size(ctrlr); 1984 NVME_CTRLR_DEBUGLOG(ctrlr, "transport max_xfer_size %u\n", ctrlr->max_xfer_size); 1985 if (ctrlr->cdata.mdts > 0) { 1986 ctrlr->max_xfer_size = spdk_min(ctrlr->max_xfer_size, 1987 ctrlr->min_page_size * (1 << ctrlr->cdata.mdts)); 1988 NVME_CTRLR_DEBUGLOG(ctrlr, "MDTS max_xfer_size %u\n", ctrlr->max_xfer_size); 1989 } 1990 1991 NVME_CTRLR_DEBUGLOG(ctrlr, "CNTLID 0x%04" PRIx16 "\n", ctrlr->cdata.cntlid); 1992 if (ctrlr->trid.trtype == SPDK_NVME_TRANSPORT_PCIE) { 1993 ctrlr->cntlid = ctrlr->cdata.cntlid; 1994 } else { 1995 /* 1996 * Fabrics controllers should already have CNTLID from the Connect command. 1997 * 1998 * If CNTLID from Connect doesn't match CNTLID in the Identify Controller data, 1999 * trust the one from Connect. 2000 */ 2001 if (ctrlr->cntlid != ctrlr->cdata.cntlid) { 2002 NVME_CTRLR_DEBUGLOG(ctrlr, "Identify CNTLID 0x%04" PRIx16 " != Connect CNTLID 0x%04" PRIx16 "\n", 2003 ctrlr->cdata.cntlid, ctrlr->cntlid); 2004 } 2005 } 2006 2007 if (ctrlr->cdata.sgls.supported && !(ctrlr->quirks & NVME_QUIRK_NOT_USE_SGL)) { 2008 assert(ctrlr->cdata.sgls.supported != 0x3); 2009 ctrlr->flags |= SPDK_NVME_CTRLR_SGL_SUPPORTED; 2010 if (ctrlr->cdata.sgls.supported == 0x2) { 2011 ctrlr->flags |= SPDK_NVME_CTRLR_SGL_REQUIRES_DWORD_ALIGNMENT; 2012 } 2013 2014 ctrlr->max_sges = nvme_transport_ctrlr_get_max_sges(ctrlr); 2015 NVME_CTRLR_DEBUGLOG(ctrlr, "transport max_sges %u\n", ctrlr->max_sges); 2016 } 2017 2018 if (ctrlr->cdata.sgls.metadata_address && !(ctrlr->quirks & NVME_QUIRK_NOT_USE_SGL)) { 2019 ctrlr->flags |= SPDK_NVME_CTRLR_MPTR_SGL_SUPPORTED; 2020 } 2021 2022 if (ctrlr->cdata.oacs.security && !(ctrlr->quirks & NVME_QUIRK_OACS_SECURITY)) { 2023 ctrlr->flags |= SPDK_NVME_CTRLR_SECURITY_SEND_RECV_SUPPORTED; 2024 } 2025 2026 if (ctrlr->cdata.oacs.directives) { 2027 ctrlr->flags |= SPDK_NVME_CTRLR_DIRECTIVES_SUPPORTED; 2028 } 2029 2030 NVME_CTRLR_DEBUGLOG(ctrlr, "fuses compare and write: %d\n", 2031 ctrlr->cdata.fuses.compare_and_write); 2032 if (ctrlr->cdata.fuses.compare_and_write) { 2033 ctrlr->flags |= SPDK_NVME_CTRLR_COMPARE_AND_WRITE_SUPPORTED; 2034 } 2035 2036 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_CONFIGURE_AER, 2037 ctrlr->opts.admin_timeout_ms); 2038 } 2039 2040 static int 2041 nvme_ctrlr_identify(struct spdk_nvme_ctrlr *ctrlr) 2042 { 2043 int rc; 2044 2045 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY, 2046 ctrlr->opts.admin_timeout_ms); 2047 2048 rc = nvme_ctrlr_cmd_identify(ctrlr, SPDK_NVME_IDENTIFY_CTRLR, 0, 0, 0, 2049 &ctrlr->cdata, sizeof(ctrlr->cdata), 2050 nvme_ctrlr_identify_done, ctrlr); 2051 if (rc != 0) { 2052 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 2053 return rc; 2054 } 2055 2056 return 0; 2057 } 2058 2059 static void 2060 nvme_ctrlr_get_zns_cmd_and_effects_log_done(void *arg, const struct spdk_nvme_cpl *cpl) 2061 { 2062 struct spdk_nvme_cmds_and_effect_log_page *log_page; 2063 struct spdk_nvme_ctrlr *ctrlr = arg; 2064 2065 if (spdk_nvme_cpl_is_error(cpl)) { 2066 NVME_CTRLR_ERRLOG(ctrlr, "nvme_ctrlr_get_zns_cmd_and_effects_log failed!\n"); 2067 spdk_free(ctrlr->tmp_ptr); 2068 ctrlr->tmp_ptr = NULL; 2069 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 2070 return; 2071 } 2072 2073 log_page = ctrlr->tmp_ptr; 2074 2075 if (log_page->io_cmds_supported[SPDK_NVME_OPC_ZONE_APPEND].csupp) { 2076 ctrlr->flags |= SPDK_NVME_CTRLR_ZONE_APPEND_SUPPORTED; 2077 } 2078 spdk_free(ctrlr->tmp_ptr); 2079 ctrlr->tmp_ptr = NULL; 2080 2081 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_NUM_QUEUES, ctrlr->opts.admin_timeout_ms); 2082 } 2083 2084 static int 2085 nvme_ctrlr_get_zns_cmd_and_effects_log(struct spdk_nvme_ctrlr *ctrlr) 2086 { 2087 int rc; 2088 2089 assert(!ctrlr->tmp_ptr); 2090 ctrlr->tmp_ptr = spdk_zmalloc(sizeof(struct spdk_nvme_cmds_and_effect_log_page), 64, NULL, 2091 SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_SHARE | SPDK_MALLOC_DMA); 2092 if (!ctrlr->tmp_ptr) { 2093 rc = -ENOMEM; 2094 goto error; 2095 } 2096 2097 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_WAIT_FOR_GET_ZNS_CMD_EFFECTS_LOG, 2098 ctrlr->opts.admin_timeout_ms); 2099 2100 rc = spdk_nvme_ctrlr_cmd_get_log_page_ext(ctrlr, SPDK_NVME_LOG_COMMAND_EFFECTS_LOG, 2101 0, ctrlr->tmp_ptr, sizeof(struct spdk_nvme_cmds_and_effect_log_page), 2102 0, 0, 0, SPDK_NVME_CSI_ZNS << 24, 2103 nvme_ctrlr_get_zns_cmd_and_effects_log_done, ctrlr); 2104 if (rc != 0) { 2105 goto error; 2106 } 2107 2108 return 0; 2109 2110 error: 2111 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 2112 spdk_free(ctrlr->tmp_ptr); 2113 ctrlr->tmp_ptr = NULL; 2114 return rc; 2115 } 2116 2117 static void 2118 nvme_ctrlr_identify_zns_specific_done(void *arg, const struct spdk_nvme_cpl *cpl) 2119 { 2120 struct spdk_nvme_ctrlr *ctrlr = (struct spdk_nvme_ctrlr *)arg; 2121 2122 if (spdk_nvme_cpl_is_error(cpl)) { 2123 /* no need to print an error, the controller simply does not support ZNS */ 2124 nvme_ctrlr_free_zns_specific_data(ctrlr); 2125 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_NUM_QUEUES, 2126 ctrlr->opts.admin_timeout_ms); 2127 return; 2128 } 2129 2130 /* A zero zasl value means use mdts */ 2131 if (ctrlr->cdata_zns->zasl) { 2132 uint32_t max_append = ctrlr->min_page_size * (1 << ctrlr->cdata_zns->zasl); 2133 ctrlr->max_zone_append_size = spdk_min(ctrlr->max_xfer_size, max_append); 2134 } else { 2135 ctrlr->max_zone_append_size = ctrlr->max_xfer_size; 2136 } 2137 2138 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_GET_ZNS_CMD_EFFECTS_LOG, 2139 ctrlr->opts.admin_timeout_ms); 2140 } 2141 2142 /** 2143 * This function will try to fetch the I/O Command Specific Controller data structure for 2144 * each I/O Command Set supported by SPDK. 2145 * 2146 * If an I/O Command Set is not supported by the controller, "Invalid Field in Command" 2147 * will be returned. Since we are fetching in a exploratively way, getting an error back 2148 * from the controller should not be treated as fatal. 2149 * 2150 * I/O Command Sets not supported by SPDK will be skipped (e.g. Key Value Command Set). 2151 * 2152 * I/O Command Sets without a IOCS specific data structure (i.e. a zero-filled IOCS specific 2153 * data structure) will be skipped (e.g. NVM Command Set, Key Value Command Set). 2154 */ 2155 static int 2156 nvme_ctrlr_identify_iocs_specific(struct spdk_nvme_ctrlr *ctrlr) 2157 { 2158 int rc; 2159 2160 if (!nvme_ctrlr_multi_iocs_enabled(ctrlr)) { 2161 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_NUM_QUEUES, 2162 ctrlr->opts.admin_timeout_ms); 2163 return 0; 2164 } 2165 2166 /* 2167 * Since SPDK currently only needs to fetch a single Command Set, keep the code here, 2168 * instead of creating multiple NVME_CTRLR_STATE_IDENTIFY_IOCS_SPECIFIC substates, 2169 * which would require additional functions and complexity for no good reason. 2170 */ 2171 assert(!ctrlr->cdata_zns); 2172 ctrlr->cdata_zns = spdk_zmalloc(sizeof(*ctrlr->cdata_zns), 64, NULL, SPDK_ENV_SOCKET_ID_ANY, 2173 SPDK_MALLOC_SHARE | SPDK_MALLOC_DMA); 2174 if (!ctrlr->cdata_zns) { 2175 rc = -ENOMEM; 2176 goto error; 2177 } 2178 2179 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_IOCS_SPECIFIC, 2180 ctrlr->opts.admin_timeout_ms); 2181 2182 rc = nvme_ctrlr_cmd_identify(ctrlr, SPDK_NVME_IDENTIFY_CTRLR_IOCS, 0, 0, SPDK_NVME_CSI_ZNS, 2183 ctrlr->cdata_zns, sizeof(*ctrlr->cdata_zns), 2184 nvme_ctrlr_identify_zns_specific_done, ctrlr); 2185 if (rc != 0) { 2186 goto error; 2187 } 2188 2189 return 0; 2190 2191 error: 2192 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 2193 nvme_ctrlr_free_zns_specific_data(ctrlr); 2194 return rc; 2195 } 2196 2197 enum nvme_active_ns_state { 2198 NVME_ACTIVE_NS_STATE_IDLE, 2199 NVME_ACTIVE_NS_STATE_PROCESSING, 2200 NVME_ACTIVE_NS_STATE_DONE, 2201 NVME_ACTIVE_NS_STATE_ERROR 2202 }; 2203 2204 typedef void (*nvme_active_ns_ctx_deleter)(struct nvme_active_ns_ctx *); 2205 2206 struct nvme_active_ns_ctx { 2207 struct spdk_nvme_ctrlr *ctrlr; 2208 uint32_t page_count; 2209 uint32_t next_nsid; 2210 uint32_t *new_ns_list; 2211 nvme_active_ns_ctx_deleter deleter; 2212 2213 enum nvme_active_ns_state state; 2214 }; 2215 2216 static struct nvme_active_ns_ctx * 2217 nvme_active_ns_ctx_create(struct spdk_nvme_ctrlr *ctrlr, nvme_active_ns_ctx_deleter deleter) 2218 { 2219 struct nvme_active_ns_ctx *ctx; 2220 uint32_t *new_ns_list = NULL; 2221 2222 ctx = calloc(1, sizeof(*ctx)); 2223 if (!ctx) { 2224 NVME_CTRLR_ERRLOG(ctrlr, "Failed to allocate nvme_active_ns_ctx!\n"); 2225 return NULL; 2226 } 2227 2228 new_ns_list = spdk_zmalloc(sizeof(struct spdk_nvme_ns_list), ctrlr->page_size, 2229 NULL, SPDK_ENV_LCORE_ID_ANY, SPDK_MALLOC_SHARE); 2230 if (!new_ns_list) { 2231 NVME_CTRLR_ERRLOG(ctrlr, "Failed to allocate active_ns_list!\n"); 2232 free(ctx); 2233 return NULL; 2234 } 2235 2236 ctx->page_count = 1; 2237 ctx->new_ns_list = new_ns_list; 2238 ctx->ctrlr = ctrlr; 2239 ctx->deleter = deleter; 2240 2241 return ctx; 2242 } 2243 2244 static void 2245 nvme_active_ns_ctx_destroy(struct nvme_active_ns_ctx *ctx) 2246 { 2247 spdk_free(ctx->new_ns_list); 2248 free(ctx); 2249 } 2250 2251 static int 2252 nvme_ctrlr_destruct_namespace(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid) 2253 { 2254 struct spdk_nvme_ns tmp, *ns; 2255 2256 assert(ctrlr != NULL); 2257 2258 tmp.id = nsid; 2259 ns = RB_FIND(nvme_ns_tree, &ctrlr->ns, &tmp); 2260 if (ns == NULL) { 2261 return -EINVAL; 2262 } 2263 2264 nvme_ns_destruct(ns); 2265 ns->active = false; 2266 2267 return 0; 2268 } 2269 2270 static int 2271 nvme_ctrlr_construct_namespace(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid) 2272 { 2273 struct spdk_nvme_ns *ns; 2274 2275 if (nsid < 1 || nsid > ctrlr->cdata.nn) { 2276 return -EINVAL; 2277 } 2278 2279 /* Namespaces are constructed on demand, so simply request it. */ 2280 ns = spdk_nvme_ctrlr_get_ns(ctrlr, nsid); 2281 if (ns == NULL) { 2282 return -ENOMEM; 2283 } 2284 2285 ns->active = true; 2286 2287 return 0; 2288 } 2289 2290 static void 2291 nvme_ctrlr_identify_active_ns_swap(struct spdk_nvme_ctrlr *ctrlr, uint32_t *new_ns_list, 2292 size_t max_entries) 2293 { 2294 uint32_t active_ns_count = 0; 2295 size_t i; 2296 uint32_t nsid; 2297 struct spdk_nvme_ns *ns, *tmp_ns; 2298 int rc; 2299 2300 /* First, remove namespaces that no longer exist */ 2301 RB_FOREACH_SAFE(ns, nvme_ns_tree, &ctrlr->ns, tmp_ns) { 2302 nsid = new_ns_list[0]; 2303 active_ns_count = 0; 2304 while (nsid != 0) { 2305 if (nsid == ns->id) { 2306 break; 2307 } 2308 2309 nsid = new_ns_list[active_ns_count++]; 2310 } 2311 2312 if (nsid != ns->id) { 2313 /* Did not find this namespace id in the new list. */ 2314 NVME_CTRLR_DEBUGLOG(ctrlr, "Namespace %u was removed\n", ns->id); 2315 nvme_ctrlr_destruct_namespace(ctrlr, ns->id); 2316 } 2317 } 2318 2319 /* Next, add new namespaces */ 2320 active_ns_count = 0; 2321 for (i = 0; i < max_entries; i++) { 2322 nsid = new_ns_list[active_ns_count]; 2323 2324 if (nsid == 0) { 2325 break; 2326 } 2327 2328 /* If the namespace already exists, this will not construct it a second time. */ 2329 rc = nvme_ctrlr_construct_namespace(ctrlr, nsid); 2330 if (rc != 0) { 2331 /* We can't easily handle a failure here. But just move on. */ 2332 assert(false); 2333 NVME_CTRLR_DEBUGLOG(ctrlr, "Failed to allocate a namespace object.\n"); 2334 continue; 2335 } 2336 2337 active_ns_count++; 2338 } 2339 2340 ctrlr->active_ns_count = active_ns_count; 2341 } 2342 2343 static void 2344 nvme_ctrlr_identify_active_ns_async_done(void *arg, const struct spdk_nvme_cpl *cpl) 2345 { 2346 struct nvme_active_ns_ctx *ctx = arg; 2347 uint32_t *new_ns_list = NULL; 2348 2349 if (spdk_nvme_cpl_is_error(cpl)) { 2350 ctx->state = NVME_ACTIVE_NS_STATE_ERROR; 2351 goto out; 2352 } 2353 2354 ctx->next_nsid = ctx->new_ns_list[1024 * ctx->page_count - 1]; 2355 if (ctx->next_nsid == 0) { 2356 ctx->state = NVME_ACTIVE_NS_STATE_DONE; 2357 goto out; 2358 } 2359 2360 ctx->page_count++; 2361 new_ns_list = spdk_realloc(ctx->new_ns_list, 2362 ctx->page_count * sizeof(struct spdk_nvme_ns_list), 2363 ctx->ctrlr->page_size); 2364 if (!new_ns_list) { 2365 SPDK_ERRLOG("Failed to reallocate active_ns_list!\n"); 2366 ctx->state = NVME_ACTIVE_NS_STATE_ERROR; 2367 goto out; 2368 } 2369 2370 ctx->new_ns_list = new_ns_list; 2371 nvme_ctrlr_identify_active_ns_async(ctx); 2372 return; 2373 2374 out: 2375 if (ctx->deleter) { 2376 ctx->deleter(ctx); 2377 } 2378 } 2379 2380 static void 2381 nvme_ctrlr_identify_active_ns_async(struct nvme_active_ns_ctx *ctx) 2382 { 2383 struct spdk_nvme_ctrlr *ctrlr = ctx->ctrlr; 2384 uint32_t i; 2385 int rc; 2386 2387 if (ctrlr->cdata.nn == 0) { 2388 ctx->state = NVME_ACTIVE_NS_STATE_DONE; 2389 goto out; 2390 } 2391 2392 assert(ctx->new_ns_list != NULL); 2393 2394 /* 2395 * If controller doesn't support active ns list CNS 0x02 dummy up 2396 * an active ns list, i.e. all namespaces report as active 2397 */ 2398 if (ctrlr->vs.raw < SPDK_NVME_VERSION(1, 1, 0) || ctrlr->quirks & NVME_QUIRK_IDENTIFY_CNS) { 2399 uint32_t *new_ns_list; 2400 2401 /* 2402 * Active NS list must always end with zero element. 2403 * So, we allocate for cdata.nn+1. 2404 */ 2405 ctx->page_count = spdk_divide_round_up(ctrlr->cdata.nn + 1, 2406 sizeof(struct spdk_nvme_ns_list) / sizeof(new_ns_list[0])); 2407 new_ns_list = spdk_realloc(ctx->new_ns_list, 2408 ctx->page_count * sizeof(struct spdk_nvme_ns_list), 2409 ctx->ctrlr->page_size); 2410 if (!new_ns_list) { 2411 SPDK_ERRLOG("Failed to reallocate active_ns_list!\n"); 2412 ctx->state = NVME_ACTIVE_NS_STATE_ERROR; 2413 goto out; 2414 } 2415 2416 ctx->new_ns_list = new_ns_list; 2417 ctx->new_ns_list[ctrlr->cdata.nn] = 0; 2418 for (i = 0; i < ctrlr->cdata.nn; i++) { 2419 ctx->new_ns_list[i] = i + 1; 2420 } 2421 2422 ctx->state = NVME_ACTIVE_NS_STATE_DONE; 2423 goto out; 2424 } 2425 2426 ctx->state = NVME_ACTIVE_NS_STATE_PROCESSING; 2427 rc = nvme_ctrlr_cmd_identify(ctrlr, SPDK_NVME_IDENTIFY_ACTIVE_NS_LIST, 0, ctx->next_nsid, 0, 2428 &ctx->new_ns_list[1024 * (ctx->page_count - 1)], sizeof(struct spdk_nvme_ns_list), 2429 nvme_ctrlr_identify_active_ns_async_done, ctx); 2430 if (rc != 0) { 2431 ctx->state = NVME_ACTIVE_NS_STATE_ERROR; 2432 goto out; 2433 } 2434 2435 return; 2436 2437 out: 2438 if (ctx->deleter) { 2439 ctx->deleter(ctx); 2440 } 2441 } 2442 2443 static void 2444 _nvme_active_ns_ctx_deleter(struct nvme_active_ns_ctx *ctx) 2445 { 2446 struct spdk_nvme_ctrlr *ctrlr = ctx->ctrlr; 2447 struct spdk_nvme_ns *ns; 2448 2449 if (ctx->state == NVME_ACTIVE_NS_STATE_ERROR) { 2450 nvme_active_ns_ctx_destroy(ctx); 2451 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 2452 return; 2453 } 2454 2455 assert(ctx->state == NVME_ACTIVE_NS_STATE_DONE); 2456 2457 RB_FOREACH(ns, nvme_ns_tree, &ctrlr->ns) { 2458 nvme_ns_free_iocs_specific_data(ns); 2459 } 2460 2461 nvme_ctrlr_identify_active_ns_swap(ctrlr, ctx->new_ns_list, ctx->page_count * 1024); 2462 nvme_active_ns_ctx_destroy(ctx); 2463 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_IDENTIFY_NS, ctrlr->opts.admin_timeout_ms); 2464 } 2465 2466 static void 2467 _nvme_ctrlr_identify_active_ns(struct spdk_nvme_ctrlr *ctrlr) 2468 { 2469 struct nvme_active_ns_ctx *ctx; 2470 2471 ctx = nvme_active_ns_ctx_create(ctrlr, _nvme_active_ns_ctx_deleter); 2472 if (!ctx) { 2473 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 2474 return; 2475 } 2476 2477 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_ACTIVE_NS, 2478 ctrlr->opts.admin_timeout_ms); 2479 nvme_ctrlr_identify_active_ns_async(ctx); 2480 } 2481 2482 int 2483 nvme_ctrlr_identify_active_ns(struct spdk_nvme_ctrlr *ctrlr) 2484 { 2485 struct nvme_active_ns_ctx *ctx; 2486 int rc; 2487 2488 ctx = nvme_active_ns_ctx_create(ctrlr, NULL); 2489 if (!ctx) { 2490 return -ENOMEM; 2491 } 2492 2493 nvme_ctrlr_identify_active_ns_async(ctx); 2494 while (ctx->state == NVME_ACTIVE_NS_STATE_PROCESSING) { 2495 rc = spdk_nvme_qpair_process_completions(ctrlr->adminq, 0); 2496 if (rc < 0) { 2497 ctx->state = NVME_ACTIVE_NS_STATE_ERROR; 2498 break; 2499 } 2500 } 2501 2502 if (ctx->state == NVME_ACTIVE_NS_STATE_ERROR) { 2503 nvme_active_ns_ctx_destroy(ctx); 2504 return -ENXIO; 2505 } 2506 2507 assert(ctx->state == NVME_ACTIVE_NS_STATE_DONE); 2508 nvme_ctrlr_identify_active_ns_swap(ctrlr, ctx->new_ns_list, ctx->page_count * 1024); 2509 nvme_active_ns_ctx_destroy(ctx); 2510 2511 return 0; 2512 } 2513 2514 static void 2515 nvme_ctrlr_identify_ns_async_done(void *arg, const struct spdk_nvme_cpl *cpl) 2516 { 2517 struct spdk_nvme_ns *ns = (struct spdk_nvme_ns *)arg; 2518 struct spdk_nvme_ctrlr *ctrlr = ns->ctrlr; 2519 uint32_t nsid; 2520 int rc; 2521 2522 if (spdk_nvme_cpl_is_error(cpl)) { 2523 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 2524 return; 2525 } 2526 2527 nvme_ns_set_identify_data(ns); 2528 2529 /* move on to the next active NS */ 2530 nsid = spdk_nvme_ctrlr_get_next_active_ns(ctrlr, ns->id); 2531 ns = spdk_nvme_ctrlr_get_ns(ctrlr, nsid); 2532 if (ns == NULL) { 2533 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_IDENTIFY_ID_DESCS, 2534 ctrlr->opts.admin_timeout_ms); 2535 return; 2536 } 2537 ns->ctrlr = ctrlr; 2538 ns->id = nsid; 2539 2540 rc = nvme_ctrlr_identify_ns_async(ns); 2541 if (rc) { 2542 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 2543 } 2544 } 2545 2546 static int 2547 nvme_ctrlr_identify_ns_async(struct spdk_nvme_ns *ns) 2548 { 2549 struct spdk_nvme_ctrlr *ctrlr = ns->ctrlr; 2550 struct spdk_nvme_ns_data *nsdata; 2551 2552 nsdata = &ns->nsdata; 2553 2554 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_NS, 2555 ctrlr->opts.admin_timeout_ms); 2556 return nvme_ctrlr_cmd_identify(ns->ctrlr, SPDK_NVME_IDENTIFY_NS, 0, ns->id, 0, 2557 nsdata, sizeof(*nsdata), 2558 nvme_ctrlr_identify_ns_async_done, ns); 2559 } 2560 2561 static int 2562 nvme_ctrlr_identify_namespaces(struct spdk_nvme_ctrlr *ctrlr) 2563 { 2564 uint32_t nsid; 2565 struct spdk_nvme_ns *ns; 2566 int rc; 2567 2568 nsid = spdk_nvme_ctrlr_get_first_active_ns(ctrlr); 2569 ns = spdk_nvme_ctrlr_get_ns(ctrlr, nsid); 2570 if (ns == NULL) { 2571 /* No active NS, move on to the next state */ 2572 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_IDENTIFY_ID_DESCS, 2573 ctrlr->opts.admin_timeout_ms); 2574 return 0; 2575 } 2576 2577 ns->ctrlr = ctrlr; 2578 ns->id = nsid; 2579 2580 rc = nvme_ctrlr_identify_ns_async(ns); 2581 if (rc) { 2582 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 2583 } 2584 2585 return rc; 2586 } 2587 2588 static int 2589 nvme_ctrlr_identify_namespaces_iocs_specific_next(struct spdk_nvme_ctrlr *ctrlr, uint32_t prev_nsid) 2590 { 2591 uint32_t nsid; 2592 struct spdk_nvme_ns *ns; 2593 int rc; 2594 2595 if (!prev_nsid) { 2596 nsid = spdk_nvme_ctrlr_get_first_active_ns(ctrlr); 2597 } else { 2598 /* move on to the next active NS */ 2599 nsid = spdk_nvme_ctrlr_get_next_active_ns(ctrlr, prev_nsid); 2600 } 2601 2602 ns = spdk_nvme_ctrlr_get_ns(ctrlr, nsid); 2603 if (ns == NULL) { 2604 /* No first/next active NS, move on to the next state */ 2605 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_SUPPORTED_LOG_PAGES, 2606 ctrlr->opts.admin_timeout_ms); 2607 return 0; 2608 } 2609 2610 /* loop until we find a ns which has (supported) iocs specific data */ 2611 while (!nvme_ns_has_supported_iocs_specific_data(ns)) { 2612 nsid = spdk_nvme_ctrlr_get_next_active_ns(ctrlr, ns->id); 2613 ns = spdk_nvme_ctrlr_get_ns(ctrlr, nsid); 2614 if (ns == NULL) { 2615 /* no namespace with (supported) iocs specific data found */ 2616 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_SUPPORTED_LOG_PAGES, 2617 ctrlr->opts.admin_timeout_ms); 2618 return 0; 2619 } 2620 } 2621 2622 rc = nvme_ctrlr_identify_ns_iocs_specific_async(ns); 2623 if (rc) { 2624 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 2625 } 2626 2627 return rc; 2628 } 2629 2630 static void 2631 nvme_ctrlr_identify_ns_zns_specific_async_done(void *arg, const struct spdk_nvme_cpl *cpl) 2632 { 2633 struct spdk_nvme_ns *ns = (struct spdk_nvme_ns *)arg; 2634 struct spdk_nvme_ctrlr *ctrlr = ns->ctrlr; 2635 2636 if (spdk_nvme_cpl_is_error(cpl)) { 2637 nvme_ns_free_zns_specific_data(ns); 2638 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 2639 return; 2640 } 2641 2642 nvme_ctrlr_identify_namespaces_iocs_specific_next(ctrlr, ns->id); 2643 } 2644 2645 static int 2646 nvme_ctrlr_identify_ns_iocs_specific_async(struct spdk_nvme_ns *ns) 2647 { 2648 struct spdk_nvme_ctrlr *ctrlr = ns->ctrlr; 2649 int rc; 2650 2651 switch (ns->csi) { 2652 case SPDK_NVME_CSI_ZNS: 2653 break; 2654 default: 2655 /* 2656 * This switch must handle all cases for which 2657 * nvme_ns_has_supported_iocs_specific_data() returns true, 2658 * other cases should never happen. 2659 */ 2660 assert(0); 2661 } 2662 2663 assert(!ns->nsdata_zns); 2664 ns->nsdata_zns = spdk_zmalloc(sizeof(*ns->nsdata_zns), 64, NULL, SPDK_ENV_SOCKET_ID_ANY, 2665 SPDK_MALLOC_SHARE); 2666 if (!ns->nsdata_zns) { 2667 return -ENOMEM; 2668 } 2669 2670 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_NS_IOCS_SPECIFIC, 2671 ctrlr->opts.admin_timeout_ms); 2672 rc = nvme_ctrlr_cmd_identify(ns->ctrlr, SPDK_NVME_IDENTIFY_NS_IOCS, 0, ns->id, ns->csi, 2673 ns->nsdata_zns, sizeof(*ns->nsdata_zns), 2674 nvme_ctrlr_identify_ns_zns_specific_async_done, ns); 2675 if (rc) { 2676 nvme_ns_free_zns_specific_data(ns); 2677 } 2678 2679 return rc; 2680 } 2681 2682 static int 2683 nvme_ctrlr_identify_namespaces_iocs_specific(struct spdk_nvme_ctrlr *ctrlr) 2684 { 2685 if (!nvme_ctrlr_multi_iocs_enabled(ctrlr)) { 2686 /* Multi IOCS not supported/enabled, move on to the next state */ 2687 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_SUPPORTED_LOG_PAGES, 2688 ctrlr->opts.admin_timeout_ms); 2689 return 0; 2690 } 2691 2692 return nvme_ctrlr_identify_namespaces_iocs_specific_next(ctrlr, 0); 2693 } 2694 2695 static void 2696 nvme_ctrlr_identify_id_desc_async_done(void *arg, const struct spdk_nvme_cpl *cpl) 2697 { 2698 struct spdk_nvme_ns *ns = (struct spdk_nvme_ns *)arg; 2699 struct spdk_nvme_ctrlr *ctrlr = ns->ctrlr; 2700 uint32_t nsid; 2701 int rc; 2702 2703 if (spdk_nvme_cpl_is_error(cpl)) { 2704 /* 2705 * Many controllers claim to be compatible with NVMe 1.3, however, 2706 * they do not implement NS ID Desc List. Therefore, instead of setting 2707 * the state to NVME_CTRLR_STATE_ERROR, silently ignore the completion 2708 * error and move on to the next state. 2709 * 2710 * The proper way is to create a new quirk for controllers that violate 2711 * the NVMe 1.3 spec by not supporting NS ID Desc List. 2712 * (Re-using the NVME_QUIRK_IDENTIFY_CNS quirk is not possible, since 2713 * it is too generic and was added in order to handle controllers that 2714 * violate the NVMe 1.1 spec by not supporting ACTIVE LIST). 2715 */ 2716 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_IDENTIFY_NS_IOCS_SPECIFIC, 2717 ctrlr->opts.admin_timeout_ms); 2718 return; 2719 } 2720 2721 nvme_ns_set_id_desc_list_data(ns); 2722 2723 /* move on to the next active NS */ 2724 nsid = spdk_nvme_ctrlr_get_next_active_ns(ctrlr, ns->id); 2725 ns = spdk_nvme_ctrlr_get_ns(ctrlr, nsid); 2726 if (ns == NULL) { 2727 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_IDENTIFY_NS_IOCS_SPECIFIC, 2728 ctrlr->opts.admin_timeout_ms); 2729 return; 2730 } 2731 2732 rc = nvme_ctrlr_identify_id_desc_async(ns); 2733 if (rc) { 2734 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 2735 } 2736 } 2737 2738 static int 2739 nvme_ctrlr_identify_id_desc_async(struct spdk_nvme_ns *ns) 2740 { 2741 struct spdk_nvme_ctrlr *ctrlr = ns->ctrlr; 2742 2743 memset(ns->id_desc_list, 0, sizeof(ns->id_desc_list)); 2744 2745 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_ID_DESCS, 2746 ctrlr->opts.admin_timeout_ms); 2747 return nvme_ctrlr_cmd_identify(ns->ctrlr, SPDK_NVME_IDENTIFY_NS_ID_DESCRIPTOR_LIST, 2748 0, ns->id, 0, ns->id_desc_list, sizeof(ns->id_desc_list), 2749 nvme_ctrlr_identify_id_desc_async_done, ns); 2750 } 2751 2752 static int 2753 nvme_ctrlr_identify_id_desc_namespaces(struct spdk_nvme_ctrlr *ctrlr) 2754 { 2755 uint32_t nsid; 2756 struct spdk_nvme_ns *ns; 2757 int rc; 2758 2759 if ((ctrlr->vs.raw < SPDK_NVME_VERSION(1, 3, 0) && 2760 !(ctrlr->cap.bits.css & SPDK_NVME_CAP_CSS_IOCS)) || 2761 (ctrlr->quirks & NVME_QUIRK_IDENTIFY_CNS)) { 2762 NVME_CTRLR_DEBUGLOG(ctrlr, "Version < 1.3; not attempting to retrieve NS ID Descriptor List\n"); 2763 /* NS ID Desc List not supported, move on to the next state */ 2764 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_IDENTIFY_NS_IOCS_SPECIFIC, 2765 ctrlr->opts.admin_timeout_ms); 2766 return 0; 2767 } 2768 2769 nsid = spdk_nvme_ctrlr_get_first_active_ns(ctrlr); 2770 ns = spdk_nvme_ctrlr_get_ns(ctrlr, nsid); 2771 if (ns == NULL) { 2772 /* No active NS, move on to the next state */ 2773 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_IDENTIFY_NS_IOCS_SPECIFIC, 2774 ctrlr->opts.admin_timeout_ms); 2775 return 0; 2776 } 2777 2778 rc = nvme_ctrlr_identify_id_desc_async(ns); 2779 if (rc) { 2780 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 2781 } 2782 2783 return rc; 2784 } 2785 2786 static void 2787 nvme_ctrlr_update_nvmf_ioccsz(struct spdk_nvme_ctrlr *ctrlr) 2788 { 2789 if (spdk_nvme_ctrlr_is_fabrics(ctrlr)) { 2790 if (ctrlr->cdata.nvmf_specific.ioccsz < 4) { 2791 NVME_CTRLR_ERRLOG(ctrlr, "Incorrect IOCCSZ %u, the minimum value should be 4\n", 2792 ctrlr->cdata.nvmf_specific.ioccsz); 2793 ctrlr->cdata.nvmf_specific.ioccsz = 4; 2794 assert(0); 2795 } 2796 ctrlr->ioccsz_bytes = ctrlr->cdata.nvmf_specific.ioccsz * 16 - sizeof(struct spdk_nvme_cmd); 2797 ctrlr->icdoff = ctrlr->cdata.nvmf_specific.icdoff; 2798 } 2799 } 2800 2801 static void 2802 nvme_ctrlr_set_num_queues_done(void *arg, const struct spdk_nvme_cpl *cpl) 2803 { 2804 uint32_t cq_allocated, sq_allocated, min_allocated, i; 2805 struct spdk_nvme_ctrlr *ctrlr = (struct spdk_nvme_ctrlr *)arg; 2806 2807 if (spdk_nvme_cpl_is_error(cpl)) { 2808 NVME_CTRLR_ERRLOG(ctrlr, "Set Features - Number of Queues failed!\n"); 2809 ctrlr->opts.num_io_queues = 0; 2810 } else { 2811 /* 2812 * Data in cdw0 is 0-based. 2813 * Lower 16-bits indicate number of submission queues allocated. 2814 * Upper 16-bits indicate number of completion queues allocated. 2815 */ 2816 sq_allocated = (cpl->cdw0 & 0xFFFF) + 1; 2817 cq_allocated = (cpl->cdw0 >> 16) + 1; 2818 2819 /* 2820 * For 1:1 queue mapping, set number of allocated queues to be minimum of 2821 * submission and completion queues. 2822 */ 2823 min_allocated = spdk_min(sq_allocated, cq_allocated); 2824 2825 /* Set number of queues to be minimum of requested and actually allocated. */ 2826 ctrlr->opts.num_io_queues = spdk_min(min_allocated, ctrlr->opts.num_io_queues); 2827 } 2828 2829 ctrlr->free_io_qids = spdk_bit_array_create(ctrlr->opts.num_io_queues + 1); 2830 if (ctrlr->free_io_qids == NULL) { 2831 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 2832 return; 2833 } 2834 2835 /* Initialize list of free I/O queue IDs. QID 0 is the admin queue (implicitly allocated). */ 2836 for (i = 1; i <= ctrlr->opts.num_io_queues; i++) { 2837 spdk_nvme_ctrlr_free_qid(ctrlr, i); 2838 } 2839 2840 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_IDENTIFY_ACTIVE_NS, 2841 ctrlr->opts.admin_timeout_ms); 2842 } 2843 2844 static int 2845 nvme_ctrlr_set_num_queues(struct spdk_nvme_ctrlr *ctrlr) 2846 { 2847 int rc; 2848 2849 if (ctrlr->opts.num_io_queues > SPDK_NVME_MAX_IO_QUEUES) { 2850 NVME_CTRLR_NOTICELOG(ctrlr, "Limiting requested num_io_queues %u to max %d\n", 2851 ctrlr->opts.num_io_queues, SPDK_NVME_MAX_IO_QUEUES); 2852 ctrlr->opts.num_io_queues = SPDK_NVME_MAX_IO_QUEUES; 2853 } else if (ctrlr->opts.num_io_queues < 1) { 2854 NVME_CTRLR_NOTICELOG(ctrlr, "Requested num_io_queues 0, increasing to 1\n"); 2855 ctrlr->opts.num_io_queues = 1; 2856 } 2857 2858 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_WAIT_FOR_SET_NUM_QUEUES, 2859 ctrlr->opts.admin_timeout_ms); 2860 2861 rc = nvme_ctrlr_cmd_set_num_queues(ctrlr, ctrlr->opts.num_io_queues, 2862 nvme_ctrlr_set_num_queues_done, ctrlr); 2863 if (rc != 0) { 2864 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 2865 return rc; 2866 } 2867 2868 return 0; 2869 } 2870 2871 static void 2872 nvme_ctrlr_set_keep_alive_timeout_done(void *arg, const struct spdk_nvme_cpl *cpl) 2873 { 2874 uint32_t keep_alive_interval_us; 2875 struct spdk_nvme_ctrlr *ctrlr = (struct spdk_nvme_ctrlr *)arg; 2876 2877 if (spdk_nvme_cpl_is_error(cpl)) { 2878 if ((cpl->status.sct == SPDK_NVME_SCT_GENERIC) && 2879 (cpl->status.sc == SPDK_NVME_SC_INVALID_FIELD)) { 2880 NVME_CTRLR_DEBUGLOG(ctrlr, "Keep alive timeout Get Feature is not supported\n"); 2881 } else { 2882 NVME_CTRLR_ERRLOG(ctrlr, "Keep alive timeout Get Feature failed: SC %x SCT %x\n", 2883 cpl->status.sc, cpl->status.sct); 2884 ctrlr->opts.keep_alive_timeout_ms = 0; 2885 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 2886 return; 2887 } 2888 } else { 2889 if (ctrlr->opts.keep_alive_timeout_ms != cpl->cdw0) { 2890 NVME_CTRLR_DEBUGLOG(ctrlr, "Controller adjusted keep alive timeout to %u ms\n", 2891 cpl->cdw0); 2892 } 2893 2894 ctrlr->opts.keep_alive_timeout_ms = cpl->cdw0; 2895 } 2896 2897 if (ctrlr->opts.keep_alive_timeout_ms == 0) { 2898 ctrlr->keep_alive_interval_ticks = 0; 2899 } else { 2900 keep_alive_interval_us = ctrlr->opts.keep_alive_timeout_ms * 1000 / 2; 2901 2902 NVME_CTRLR_DEBUGLOG(ctrlr, "Sending keep alive every %u us\n", keep_alive_interval_us); 2903 2904 ctrlr->keep_alive_interval_ticks = (keep_alive_interval_us * spdk_get_ticks_hz()) / 2905 UINT64_C(1000000); 2906 2907 /* Schedule the first Keep Alive to be sent as soon as possible. */ 2908 ctrlr->next_keep_alive_tick = spdk_get_ticks(); 2909 } 2910 2911 if (spdk_nvme_ctrlr_is_discovery(ctrlr)) { 2912 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_READY, NVME_TIMEOUT_INFINITE); 2913 } else { 2914 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_IDENTIFY_IOCS_SPECIFIC, 2915 ctrlr->opts.admin_timeout_ms); 2916 } 2917 } 2918 2919 static int 2920 nvme_ctrlr_set_keep_alive_timeout(struct spdk_nvme_ctrlr *ctrlr) 2921 { 2922 int rc; 2923 2924 if (ctrlr->opts.keep_alive_timeout_ms == 0) { 2925 if (spdk_nvme_ctrlr_is_discovery(ctrlr)) { 2926 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_READY, NVME_TIMEOUT_INFINITE); 2927 } else { 2928 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_IDENTIFY_IOCS_SPECIFIC, 2929 ctrlr->opts.admin_timeout_ms); 2930 } 2931 return 0; 2932 } 2933 2934 /* Note: Discovery controller identify data does not populate KAS according to spec. */ 2935 if (!spdk_nvme_ctrlr_is_discovery(ctrlr) && ctrlr->cdata.kas == 0) { 2936 NVME_CTRLR_DEBUGLOG(ctrlr, "Controller KAS is 0 - not enabling Keep Alive\n"); 2937 ctrlr->opts.keep_alive_timeout_ms = 0; 2938 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_IDENTIFY_IOCS_SPECIFIC, 2939 ctrlr->opts.admin_timeout_ms); 2940 return 0; 2941 } 2942 2943 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_WAIT_FOR_KEEP_ALIVE_TIMEOUT, 2944 ctrlr->opts.admin_timeout_ms); 2945 2946 /* Retrieve actual keep alive timeout, since the controller may have adjusted it. */ 2947 rc = spdk_nvme_ctrlr_cmd_get_feature(ctrlr, SPDK_NVME_FEAT_KEEP_ALIVE_TIMER, 0, NULL, 0, 2948 nvme_ctrlr_set_keep_alive_timeout_done, ctrlr); 2949 if (rc != 0) { 2950 NVME_CTRLR_ERRLOG(ctrlr, "Keep alive timeout Get Feature failed: %d\n", rc); 2951 ctrlr->opts.keep_alive_timeout_ms = 0; 2952 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 2953 return rc; 2954 } 2955 2956 return 0; 2957 } 2958 2959 static void 2960 nvme_ctrlr_set_host_id_done(void *arg, const struct spdk_nvme_cpl *cpl) 2961 { 2962 struct spdk_nvme_ctrlr *ctrlr = (struct spdk_nvme_ctrlr *)arg; 2963 2964 if (spdk_nvme_cpl_is_error(cpl)) { 2965 /* 2966 * Treat Set Features - Host ID failure as non-fatal, since the Host ID feature 2967 * is optional. 2968 */ 2969 NVME_CTRLR_WARNLOG(ctrlr, "Set Features - Host ID failed: SC 0x%x SCT 0x%x\n", 2970 cpl->status.sc, cpl->status.sct); 2971 } else { 2972 NVME_CTRLR_DEBUGLOG(ctrlr, "Set Features - Host ID was successful\n"); 2973 } 2974 2975 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_TRANSPORT_READY, ctrlr->opts.admin_timeout_ms); 2976 } 2977 2978 static int 2979 nvme_ctrlr_set_host_id(struct spdk_nvme_ctrlr *ctrlr) 2980 { 2981 uint8_t *host_id; 2982 uint32_t host_id_size; 2983 int rc; 2984 2985 if (ctrlr->trid.trtype != SPDK_NVME_TRANSPORT_PCIE) { 2986 /* 2987 * NVMe-oF sends the host ID during Connect and doesn't allow 2988 * Set Features - Host Identifier after Connect, so we don't need to do anything here. 2989 */ 2990 NVME_CTRLR_DEBUGLOG(ctrlr, "NVMe-oF transport - not sending Set Features - Host ID\n"); 2991 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_TRANSPORT_READY, ctrlr->opts.admin_timeout_ms); 2992 return 0; 2993 } 2994 2995 if (ctrlr->cdata.ctratt.host_id_exhid_supported) { 2996 NVME_CTRLR_DEBUGLOG(ctrlr, "Using 128-bit extended host identifier\n"); 2997 host_id = ctrlr->opts.extended_host_id; 2998 host_id_size = sizeof(ctrlr->opts.extended_host_id); 2999 } else { 3000 NVME_CTRLR_DEBUGLOG(ctrlr, "Using 64-bit host identifier\n"); 3001 host_id = ctrlr->opts.host_id; 3002 host_id_size = sizeof(ctrlr->opts.host_id); 3003 } 3004 3005 /* If the user specified an all-zeroes host identifier, don't send the command. */ 3006 if (spdk_mem_all_zero(host_id, host_id_size)) { 3007 NVME_CTRLR_DEBUGLOG(ctrlr, "User did not specify host ID - not sending Set Features - Host ID\n"); 3008 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_TRANSPORT_READY, ctrlr->opts.admin_timeout_ms); 3009 return 0; 3010 } 3011 3012 SPDK_LOGDUMP(nvme, "host_id", host_id, host_id_size); 3013 3014 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_WAIT_FOR_HOST_ID, 3015 ctrlr->opts.admin_timeout_ms); 3016 3017 rc = nvme_ctrlr_cmd_set_host_id(ctrlr, host_id, host_id_size, nvme_ctrlr_set_host_id_done, ctrlr); 3018 if (rc != 0) { 3019 NVME_CTRLR_ERRLOG(ctrlr, "Set Features - Host ID failed: %d\n", rc); 3020 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 3021 return rc; 3022 } 3023 3024 return 0; 3025 } 3026 3027 void 3028 nvme_ctrlr_update_namespaces(struct spdk_nvme_ctrlr *ctrlr) 3029 { 3030 uint32_t nsid; 3031 struct spdk_nvme_ns *ns; 3032 3033 for (nsid = spdk_nvme_ctrlr_get_first_active_ns(ctrlr); 3034 nsid != 0; nsid = spdk_nvme_ctrlr_get_next_active_ns(ctrlr, nsid)) { 3035 ns = spdk_nvme_ctrlr_get_ns(ctrlr, nsid); 3036 nvme_ns_construct(ns, nsid, ctrlr); 3037 } 3038 } 3039 3040 static int 3041 nvme_ctrlr_clear_changed_ns_log(struct spdk_nvme_ctrlr *ctrlr) 3042 { 3043 struct nvme_completion_poll_status *status; 3044 int rc = -ENOMEM; 3045 char *buffer = NULL; 3046 uint32_t nsid; 3047 size_t buf_size = (SPDK_NVME_MAX_CHANGED_NAMESPACES * sizeof(uint32_t)); 3048 3049 if (ctrlr->opts.disable_read_changed_ns_list_log_page) { 3050 return 0; 3051 } 3052 3053 buffer = spdk_dma_zmalloc(buf_size, 4096, NULL); 3054 if (!buffer) { 3055 NVME_CTRLR_ERRLOG(ctrlr, "Failed to allocate buffer for getting " 3056 "changed ns log.\n"); 3057 return rc; 3058 } 3059 3060 status = calloc(1, sizeof(*status)); 3061 if (!status) { 3062 NVME_CTRLR_ERRLOG(ctrlr, "Failed to allocate status tracker\n"); 3063 goto free_buffer; 3064 } 3065 3066 rc = spdk_nvme_ctrlr_cmd_get_log_page(ctrlr, 3067 SPDK_NVME_LOG_CHANGED_NS_LIST, 3068 SPDK_NVME_GLOBAL_NS_TAG, 3069 buffer, buf_size, 0, 3070 nvme_completion_poll_cb, status); 3071 3072 if (rc) { 3073 NVME_CTRLR_ERRLOG(ctrlr, "spdk_nvme_ctrlr_cmd_get_log_page() failed: rc=%d\n", rc); 3074 free(status); 3075 goto free_buffer; 3076 } 3077 3078 rc = nvme_wait_for_completion_timeout(ctrlr->adminq, status, 3079 ctrlr->opts.admin_timeout_ms * 1000); 3080 if (!status->timed_out) { 3081 free(status); 3082 } 3083 3084 if (rc) { 3085 NVME_CTRLR_ERRLOG(ctrlr, "wait for spdk_nvme_ctrlr_cmd_get_log_page failed: rc=%d\n", rc); 3086 goto free_buffer; 3087 } 3088 3089 /* only check the case of overflow. */ 3090 nsid = from_le32(buffer); 3091 if (nsid == 0xffffffffu) { 3092 NVME_CTRLR_WARNLOG(ctrlr, "changed ns log overflowed.\n"); 3093 } 3094 3095 free_buffer: 3096 spdk_dma_free(buffer); 3097 return rc; 3098 } 3099 3100 void 3101 nvme_ctrlr_process_async_event(struct spdk_nvme_ctrlr *ctrlr, 3102 const struct spdk_nvme_cpl *cpl) 3103 { 3104 union spdk_nvme_async_event_completion event; 3105 struct spdk_nvme_ctrlr_process *active_proc; 3106 int rc; 3107 3108 event.raw = cpl->cdw0; 3109 3110 if ((event.bits.async_event_type == SPDK_NVME_ASYNC_EVENT_TYPE_NOTICE) && 3111 (event.bits.async_event_info == SPDK_NVME_ASYNC_EVENT_NS_ATTR_CHANGED)) { 3112 nvme_ctrlr_clear_changed_ns_log(ctrlr); 3113 3114 rc = nvme_ctrlr_identify_active_ns(ctrlr); 3115 if (rc) { 3116 return; 3117 } 3118 nvme_ctrlr_update_namespaces(ctrlr); 3119 nvme_io_msg_ctrlr_update(ctrlr); 3120 } 3121 3122 if ((event.bits.async_event_type == SPDK_NVME_ASYNC_EVENT_TYPE_NOTICE) && 3123 (event.bits.async_event_info == SPDK_NVME_ASYNC_EVENT_ANA_CHANGE)) { 3124 if (!ctrlr->opts.disable_read_ana_log_page) { 3125 rc = nvme_ctrlr_update_ana_log_page(ctrlr); 3126 if (rc) { 3127 return; 3128 } 3129 nvme_ctrlr_parse_ana_log_page(ctrlr, nvme_ctrlr_update_ns_ana_states, 3130 ctrlr); 3131 } 3132 } 3133 3134 active_proc = nvme_ctrlr_get_current_process(ctrlr); 3135 if (active_proc && active_proc->aer_cb_fn) { 3136 active_proc->aer_cb_fn(active_proc->aer_cb_arg, cpl); 3137 } 3138 } 3139 3140 static void 3141 nvme_ctrlr_queue_async_event(struct spdk_nvme_ctrlr *ctrlr, 3142 const struct spdk_nvme_cpl *cpl) 3143 { 3144 struct spdk_nvme_ctrlr_aer_completion_list *nvme_event; 3145 struct spdk_nvme_ctrlr_process *proc; 3146 3147 /* Add async event to each process objects event list */ 3148 TAILQ_FOREACH(proc, &ctrlr->active_procs, tailq) { 3149 /* Must be shared memory so other processes can access */ 3150 nvme_event = spdk_zmalloc(sizeof(*nvme_event), 0, NULL, SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_SHARE); 3151 if (!nvme_event) { 3152 NVME_CTRLR_ERRLOG(ctrlr, "Alloc nvme event failed, ignore the event\n"); 3153 return; 3154 } 3155 nvme_event->cpl = *cpl; 3156 3157 STAILQ_INSERT_TAIL(&proc->async_events, nvme_event, link); 3158 } 3159 } 3160 3161 void 3162 nvme_ctrlr_complete_queued_async_events(struct spdk_nvme_ctrlr *ctrlr) 3163 { 3164 struct spdk_nvme_ctrlr_aer_completion_list *nvme_event, *nvme_event_tmp; 3165 struct spdk_nvme_ctrlr_process *active_proc; 3166 3167 active_proc = nvme_ctrlr_get_current_process(ctrlr); 3168 3169 STAILQ_FOREACH_SAFE(nvme_event, &active_proc->async_events, link, nvme_event_tmp) { 3170 STAILQ_REMOVE(&active_proc->async_events, nvme_event, 3171 spdk_nvme_ctrlr_aer_completion_list, link); 3172 nvme_ctrlr_process_async_event(ctrlr, &nvme_event->cpl); 3173 spdk_free(nvme_event); 3174 3175 } 3176 } 3177 3178 static void 3179 nvme_ctrlr_async_event_cb(void *arg, const struct spdk_nvme_cpl *cpl) 3180 { 3181 struct nvme_async_event_request *aer = arg; 3182 struct spdk_nvme_ctrlr *ctrlr = aer->ctrlr; 3183 3184 if (cpl->status.sct == SPDK_NVME_SCT_GENERIC && 3185 cpl->status.sc == SPDK_NVME_SC_ABORTED_SQ_DELETION) { 3186 /* 3187 * This is simulated when controller is being shut down, to 3188 * effectively abort outstanding asynchronous event requests 3189 * and make sure all memory is freed. Do not repost the 3190 * request in this case. 3191 */ 3192 return; 3193 } 3194 3195 if (cpl->status.sct == SPDK_NVME_SCT_COMMAND_SPECIFIC && 3196 cpl->status.sc == SPDK_NVME_SC_ASYNC_EVENT_REQUEST_LIMIT_EXCEEDED) { 3197 /* 3198 * SPDK will only send as many AERs as the device says it supports, 3199 * so this status code indicates an out-of-spec device. Do not repost 3200 * the request in this case. 3201 */ 3202 NVME_CTRLR_ERRLOG(ctrlr, "Controller appears out-of-spec for asynchronous event request\n" 3203 "handling. Do not repost this AER.\n"); 3204 return; 3205 } 3206 3207 /* Add the events to the list */ 3208 nvme_ctrlr_queue_async_event(ctrlr, cpl); 3209 3210 /* If the ctrlr was removed or in the destruct state, we should not send aer again */ 3211 if (ctrlr->is_removed || ctrlr->is_destructed) { 3212 return; 3213 } 3214 3215 /* 3216 * Repost another asynchronous event request to replace the one 3217 * that just completed. 3218 */ 3219 if (nvme_ctrlr_construct_and_submit_aer(ctrlr, aer)) { 3220 /* 3221 * We can't do anything to recover from a failure here, 3222 * so just print a warning message and leave the AER unsubmitted. 3223 */ 3224 NVME_CTRLR_ERRLOG(ctrlr, "resubmitting AER failed!\n"); 3225 } 3226 } 3227 3228 static int 3229 nvme_ctrlr_construct_and_submit_aer(struct spdk_nvme_ctrlr *ctrlr, 3230 struct nvme_async_event_request *aer) 3231 { 3232 struct nvme_request *req; 3233 3234 aer->ctrlr = ctrlr; 3235 req = nvme_allocate_request_null(ctrlr->adminq, nvme_ctrlr_async_event_cb, aer); 3236 aer->req = req; 3237 if (req == NULL) { 3238 return -1; 3239 } 3240 3241 req->cmd.opc = SPDK_NVME_OPC_ASYNC_EVENT_REQUEST; 3242 return nvme_ctrlr_submit_admin_request(ctrlr, req); 3243 } 3244 3245 static void 3246 nvme_ctrlr_configure_aer_done(void *arg, const struct spdk_nvme_cpl *cpl) 3247 { 3248 struct nvme_async_event_request *aer; 3249 int rc; 3250 uint32_t i; 3251 struct spdk_nvme_ctrlr *ctrlr = (struct spdk_nvme_ctrlr *)arg; 3252 3253 if (spdk_nvme_cpl_is_error(cpl)) { 3254 NVME_CTRLR_NOTICELOG(ctrlr, "nvme_ctrlr_configure_aer failed!\n"); 3255 ctrlr->num_aers = 0; 3256 } else { 3257 /* aerl is a zero-based value, so we need to add 1 here. */ 3258 ctrlr->num_aers = spdk_min(NVME_MAX_ASYNC_EVENTS, (ctrlr->cdata.aerl + 1)); 3259 } 3260 3261 for (i = 0; i < ctrlr->num_aers; i++) { 3262 aer = &ctrlr->aer[i]; 3263 rc = nvme_ctrlr_construct_and_submit_aer(ctrlr, aer); 3264 if (rc) { 3265 NVME_CTRLR_ERRLOG(ctrlr, "nvme_ctrlr_construct_and_submit_aer failed!\n"); 3266 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 3267 return; 3268 } 3269 } 3270 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_KEEP_ALIVE_TIMEOUT, ctrlr->opts.admin_timeout_ms); 3271 } 3272 3273 static int 3274 nvme_ctrlr_configure_aer(struct spdk_nvme_ctrlr *ctrlr) 3275 { 3276 union spdk_nvme_feat_async_event_configuration config; 3277 int rc; 3278 3279 config.raw = 0; 3280 3281 if (spdk_nvme_ctrlr_is_discovery(ctrlr)) { 3282 config.bits.discovery_log_change_notice = 1; 3283 } else { 3284 config.bits.crit_warn.bits.available_spare = 1; 3285 config.bits.crit_warn.bits.temperature = 1; 3286 config.bits.crit_warn.bits.device_reliability = 1; 3287 config.bits.crit_warn.bits.read_only = 1; 3288 config.bits.crit_warn.bits.volatile_memory_backup = 1; 3289 3290 if (ctrlr->vs.raw >= SPDK_NVME_VERSION(1, 2, 0)) { 3291 if (ctrlr->cdata.oaes.ns_attribute_notices) { 3292 config.bits.ns_attr_notice = 1; 3293 } 3294 if (ctrlr->cdata.oaes.fw_activation_notices) { 3295 config.bits.fw_activation_notice = 1; 3296 } 3297 if (ctrlr->cdata.oaes.ana_change_notices) { 3298 config.bits.ana_change_notice = 1; 3299 } 3300 } 3301 if (ctrlr->vs.raw >= SPDK_NVME_VERSION(1, 3, 0) && ctrlr->cdata.lpa.telemetry) { 3302 config.bits.telemetry_log_notice = 1; 3303 } 3304 } 3305 3306 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_WAIT_FOR_CONFIGURE_AER, 3307 ctrlr->opts.admin_timeout_ms); 3308 3309 rc = nvme_ctrlr_cmd_set_async_event_config(ctrlr, config, 3310 nvme_ctrlr_configure_aer_done, 3311 ctrlr); 3312 if (rc != 0) { 3313 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 3314 return rc; 3315 } 3316 3317 return 0; 3318 } 3319 3320 struct spdk_nvme_ctrlr_process * 3321 nvme_ctrlr_get_process(struct spdk_nvme_ctrlr *ctrlr, pid_t pid) 3322 { 3323 struct spdk_nvme_ctrlr_process *active_proc; 3324 3325 TAILQ_FOREACH(active_proc, &ctrlr->active_procs, tailq) { 3326 if (active_proc->pid == pid) { 3327 return active_proc; 3328 } 3329 } 3330 3331 return NULL; 3332 } 3333 3334 struct spdk_nvme_ctrlr_process * 3335 nvme_ctrlr_get_current_process(struct spdk_nvme_ctrlr *ctrlr) 3336 { 3337 return nvme_ctrlr_get_process(ctrlr, getpid()); 3338 } 3339 3340 /** 3341 * This function will be called when a process is using the controller. 3342 * 1. For the primary process, it is called when constructing the controller. 3343 * 2. For the secondary process, it is called at probing the controller. 3344 * Note: will check whether the process is already added for the same process. 3345 */ 3346 int 3347 nvme_ctrlr_add_process(struct spdk_nvme_ctrlr *ctrlr, void *devhandle) 3348 { 3349 struct spdk_nvme_ctrlr_process *ctrlr_proc; 3350 pid_t pid = getpid(); 3351 3352 /* Check whether the process is already added or not */ 3353 if (nvme_ctrlr_get_process(ctrlr, pid)) { 3354 return 0; 3355 } 3356 3357 /* Initialize the per process properties for this ctrlr */ 3358 ctrlr_proc = spdk_zmalloc(sizeof(struct spdk_nvme_ctrlr_process), 3359 64, NULL, SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_SHARE); 3360 if (ctrlr_proc == NULL) { 3361 NVME_CTRLR_ERRLOG(ctrlr, "failed to allocate memory to track the process props\n"); 3362 3363 return -1; 3364 } 3365 3366 ctrlr_proc->is_primary = spdk_process_is_primary(); 3367 ctrlr_proc->pid = pid; 3368 STAILQ_INIT(&ctrlr_proc->active_reqs); 3369 ctrlr_proc->devhandle = devhandle; 3370 ctrlr_proc->ref = 0; 3371 TAILQ_INIT(&ctrlr_proc->allocated_io_qpairs); 3372 STAILQ_INIT(&ctrlr_proc->async_events); 3373 3374 TAILQ_INSERT_TAIL(&ctrlr->active_procs, ctrlr_proc, tailq); 3375 3376 return 0; 3377 } 3378 3379 /** 3380 * This function will be called when the process detaches the controller. 3381 * Note: the ctrlr_lock must be held when calling this function. 3382 */ 3383 static void 3384 nvme_ctrlr_remove_process(struct spdk_nvme_ctrlr *ctrlr, 3385 struct spdk_nvme_ctrlr_process *proc) 3386 { 3387 struct spdk_nvme_qpair *qpair, *tmp_qpair; 3388 3389 assert(STAILQ_EMPTY(&proc->active_reqs)); 3390 3391 TAILQ_FOREACH_SAFE(qpair, &proc->allocated_io_qpairs, per_process_tailq, tmp_qpair) { 3392 spdk_nvme_ctrlr_free_io_qpair(qpair); 3393 } 3394 3395 TAILQ_REMOVE(&ctrlr->active_procs, proc, tailq); 3396 3397 if (ctrlr->trid.trtype == SPDK_NVME_TRANSPORT_PCIE) { 3398 spdk_pci_device_detach(proc->devhandle); 3399 } 3400 3401 spdk_free(proc); 3402 } 3403 3404 /** 3405 * This function will be called when the process exited unexpectedly 3406 * in order to free any incomplete nvme request, allocated IO qpairs 3407 * and allocated memory. 3408 * Note: the ctrlr_lock must be held when calling this function. 3409 */ 3410 static void 3411 nvme_ctrlr_cleanup_process(struct spdk_nvme_ctrlr_process *proc) 3412 { 3413 struct nvme_request *req, *tmp_req; 3414 struct spdk_nvme_qpair *qpair, *tmp_qpair; 3415 struct spdk_nvme_ctrlr_aer_completion_list *event; 3416 3417 STAILQ_FOREACH_SAFE(req, &proc->active_reqs, stailq, tmp_req) { 3418 STAILQ_REMOVE(&proc->active_reqs, req, nvme_request, stailq); 3419 3420 assert(req->pid == proc->pid); 3421 nvme_cleanup_user_req(req); 3422 nvme_free_request(req); 3423 } 3424 3425 /* Remove async event from each process objects event list */ 3426 while (!STAILQ_EMPTY(&proc->async_events)) { 3427 event = STAILQ_FIRST(&proc->async_events); 3428 STAILQ_REMOVE_HEAD(&proc->async_events, link); 3429 spdk_free(event); 3430 } 3431 3432 TAILQ_FOREACH_SAFE(qpair, &proc->allocated_io_qpairs, per_process_tailq, tmp_qpair) { 3433 TAILQ_REMOVE(&proc->allocated_io_qpairs, qpair, per_process_tailq); 3434 3435 /* 3436 * The process may have been killed while some qpairs were in their 3437 * completion context. Clear that flag here to allow these IO 3438 * qpairs to be deleted. 3439 */ 3440 qpair->in_completion_context = 0; 3441 3442 qpair->no_deletion_notification_needed = 1; 3443 3444 spdk_nvme_ctrlr_free_io_qpair(qpair); 3445 } 3446 3447 spdk_free(proc); 3448 } 3449 3450 /** 3451 * This function will be called when destructing the controller. 3452 * 1. There is no more admin request on this controller. 3453 * 2. Clean up any left resource allocation when its associated process is gone. 3454 */ 3455 void 3456 nvme_ctrlr_free_processes(struct spdk_nvme_ctrlr *ctrlr) 3457 { 3458 struct spdk_nvme_ctrlr_process *active_proc, *tmp; 3459 3460 /* Free all the processes' properties and make sure no pending admin IOs */ 3461 TAILQ_FOREACH_SAFE(active_proc, &ctrlr->active_procs, tailq, tmp) { 3462 TAILQ_REMOVE(&ctrlr->active_procs, active_proc, tailq); 3463 3464 assert(STAILQ_EMPTY(&active_proc->active_reqs)); 3465 3466 spdk_free(active_proc); 3467 } 3468 } 3469 3470 /** 3471 * This function will be called when any other process attaches or 3472 * detaches the controller in order to cleanup those unexpectedly 3473 * terminated processes. 3474 * Note: the ctrlr_lock must be held when calling this function. 3475 */ 3476 static int 3477 nvme_ctrlr_remove_inactive_proc(struct spdk_nvme_ctrlr *ctrlr) 3478 { 3479 struct spdk_nvme_ctrlr_process *active_proc, *tmp; 3480 int active_proc_count = 0; 3481 3482 TAILQ_FOREACH_SAFE(active_proc, &ctrlr->active_procs, tailq, tmp) { 3483 if ((kill(active_proc->pid, 0) == -1) && (errno == ESRCH)) { 3484 NVME_CTRLR_ERRLOG(ctrlr, "process %d terminated unexpected\n", active_proc->pid); 3485 3486 TAILQ_REMOVE(&ctrlr->active_procs, active_proc, tailq); 3487 3488 nvme_ctrlr_cleanup_process(active_proc); 3489 } else { 3490 active_proc_count++; 3491 } 3492 } 3493 3494 return active_proc_count; 3495 } 3496 3497 void 3498 nvme_ctrlr_proc_get_ref(struct spdk_nvme_ctrlr *ctrlr) 3499 { 3500 struct spdk_nvme_ctrlr_process *active_proc; 3501 3502 nvme_robust_mutex_lock(&ctrlr->ctrlr_lock); 3503 3504 nvme_ctrlr_remove_inactive_proc(ctrlr); 3505 3506 active_proc = nvme_ctrlr_get_current_process(ctrlr); 3507 if (active_proc) { 3508 active_proc->ref++; 3509 } 3510 3511 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 3512 } 3513 3514 void 3515 nvme_ctrlr_proc_put_ref(struct spdk_nvme_ctrlr *ctrlr) 3516 { 3517 struct spdk_nvme_ctrlr_process *active_proc; 3518 int proc_count; 3519 3520 nvme_robust_mutex_lock(&ctrlr->ctrlr_lock); 3521 3522 proc_count = nvme_ctrlr_remove_inactive_proc(ctrlr); 3523 3524 active_proc = nvme_ctrlr_get_current_process(ctrlr); 3525 if (active_proc) { 3526 active_proc->ref--; 3527 assert(active_proc->ref >= 0); 3528 3529 /* 3530 * The last active process will be removed at the end of 3531 * the destruction of the controller. 3532 */ 3533 if (active_proc->ref == 0 && proc_count != 1) { 3534 nvme_ctrlr_remove_process(ctrlr, active_proc); 3535 } 3536 } 3537 3538 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 3539 } 3540 3541 int 3542 nvme_ctrlr_get_ref_count(struct spdk_nvme_ctrlr *ctrlr) 3543 { 3544 struct spdk_nvme_ctrlr_process *active_proc; 3545 int ref = 0; 3546 3547 nvme_robust_mutex_lock(&ctrlr->ctrlr_lock); 3548 3549 nvme_ctrlr_remove_inactive_proc(ctrlr); 3550 3551 TAILQ_FOREACH(active_proc, &ctrlr->active_procs, tailq) { 3552 ref += active_proc->ref; 3553 } 3554 3555 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 3556 3557 return ref; 3558 } 3559 3560 /** 3561 * Get the PCI device handle which is only visible to its associated process. 3562 */ 3563 struct spdk_pci_device * 3564 nvme_ctrlr_proc_get_devhandle(struct spdk_nvme_ctrlr *ctrlr) 3565 { 3566 struct spdk_nvme_ctrlr_process *active_proc; 3567 struct spdk_pci_device *devhandle = NULL; 3568 3569 nvme_robust_mutex_lock(&ctrlr->ctrlr_lock); 3570 3571 active_proc = nvme_ctrlr_get_current_process(ctrlr); 3572 if (active_proc) { 3573 devhandle = active_proc->devhandle; 3574 } 3575 3576 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 3577 3578 return devhandle; 3579 } 3580 3581 static void 3582 nvme_ctrlr_process_init_vs_done(void *ctx, uint64_t value, const struct spdk_nvme_cpl *cpl) 3583 { 3584 struct spdk_nvme_ctrlr *ctrlr = ctx; 3585 3586 if (spdk_nvme_cpl_is_error(cpl)) { 3587 NVME_CTRLR_ERRLOG(ctrlr, "Failed to read the VS register\n"); 3588 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 3589 return; 3590 } 3591 3592 assert(value <= UINT32_MAX); 3593 ctrlr->vs.raw = (uint32_t)value; 3594 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_READ_CAP, NVME_TIMEOUT_INFINITE); 3595 } 3596 3597 static void 3598 nvme_ctrlr_process_init_cap_done(void *ctx, uint64_t value, const struct spdk_nvme_cpl *cpl) 3599 { 3600 struct spdk_nvme_ctrlr *ctrlr = ctx; 3601 3602 if (spdk_nvme_cpl_is_error(cpl)) { 3603 NVME_CTRLR_ERRLOG(ctrlr, "Failed to read the CAP register\n"); 3604 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 3605 return; 3606 } 3607 3608 ctrlr->cap.raw = value; 3609 nvme_ctrlr_init_cap(ctrlr); 3610 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_CHECK_EN, NVME_TIMEOUT_INFINITE); 3611 } 3612 3613 static void 3614 nvme_ctrlr_process_init_check_en(void *ctx, uint64_t value, const struct spdk_nvme_cpl *cpl) 3615 { 3616 struct spdk_nvme_ctrlr *ctrlr = ctx; 3617 enum nvme_ctrlr_state state; 3618 3619 if (spdk_nvme_cpl_is_error(cpl)) { 3620 NVME_CTRLR_ERRLOG(ctrlr, "Failed to read the CC register\n"); 3621 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 3622 return; 3623 } 3624 3625 assert(value <= UINT32_MAX); 3626 ctrlr->process_init_cc.raw = (uint32_t)value; 3627 3628 if (ctrlr->process_init_cc.bits.en) { 3629 NVME_CTRLR_DEBUGLOG(ctrlr, "CC.EN = 1\n"); 3630 state = NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_1; 3631 } else { 3632 state = NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0; 3633 } 3634 3635 nvme_ctrlr_set_state(ctrlr, state, nvme_ctrlr_get_ready_timeout(ctrlr)); 3636 } 3637 3638 static void 3639 nvme_ctrlr_process_init_set_en_0(void *ctx, uint64_t value, const struct spdk_nvme_cpl *cpl) 3640 { 3641 struct spdk_nvme_ctrlr *ctrlr = ctx; 3642 3643 if (spdk_nvme_cpl_is_error(cpl)) { 3644 NVME_CTRLR_ERRLOG(ctrlr, "Failed to write the CC register\n"); 3645 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 3646 return; 3647 } 3648 3649 /* 3650 * Wait 2.5 seconds before accessing PCI registers. 3651 * Not using sleep() to avoid blocking other controller's initialization. 3652 */ 3653 if (ctrlr->quirks & NVME_QUIRK_DELAY_BEFORE_CHK_RDY) { 3654 NVME_CTRLR_DEBUGLOG(ctrlr, "Applying quirk: delay 2.5 seconds before reading registers\n"); 3655 ctrlr->sleep_timeout_tsc = spdk_get_ticks() + (2500 * spdk_get_ticks_hz() / 1000); 3656 } 3657 3658 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0, 3659 nvme_ctrlr_get_ready_timeout(ctrlr)); 3660 } 3661 3662 static void 3663 nvme_ctrlr_process_init_set_en_0_read_cc(void *ctx, uint64_t value, const struct spdk_nvme_cpl *cpl) 3664 { 3665 struct spdk_nvme_ctrlr *ctrlr = ctx; 3666 union spdk_nvme_cc_register cc; 3667 int rc; 3668 3669 if (spdk_nvme_cpl_is_error(cpl)) { 3670 NVME_CTRLR_ERRLOG(ctrlr, "Failed to read the CC register\n"); 3671 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 3672 return; 3673 } 3674 3675 assert(value <= UINT32_MAX); 3676 cc.raw = (uint32_t)value; 3677 cc.bits.en = 0; 3678 ctrlr->process_init_cc.raw = cc.raw; 3679 3680 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_EN_0_WAIT_FOR_CC, 3681 nvme_ctrlr_get_ready_timeout(ctrlr)); 3682 3683 rc = nvme_ctrlr_set_cc_async(ctrlr, cc.raw, nvme_ctrlr_process_init_set_en_0, ctrlr); 3684 if (rc != 0) { 3685 NVME_CTRLR_ERRLOG(ctrlr, "set_cc() failed\n"); 3686 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 3687 } 3688 } 3689 3690 static void 3691 nvme_ctrlr_process_init_wait_for_ready_1(void *ctx, uint64_t value, const struct spdk_nvme_cpl *cpl) 3692 { 3693 struct spdk_nvme_ctrlr *ctrlr = ctx; 3694 union spdk_nvme_csts_register csts; 3695 3696 if (spdk_nvme_cpl_is_error(cpl)) { 3697 /* While a device is resetting, it may be unable to service MMIO reads 3698 * temporarily. Allow for this case. 3699 */ 3700 if (!ctrlr->is_failed && ctrlr->state_timeout_tsc != NVME_TIMEOUT_INFINITE) { 3701 NVME_CTRLR_DEBUGLOG(ctrlr, "Failed to read the CSTS register\n"); 3702 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_1, 3703 NVME_TIMEOUT_KEEP_EXISTING); 3704 } else { 3705 NVME_CTRLR_ERRLOG(ctrlr, "Failed to read the CSTS register\n"); 3706 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 3707 } 3708 3709 return; 3710 } 3711 3712 assert(value <= UINT32_MAX); 3713 csts.raw = (uint32_t)value; 3714 if (csts.bits.rdy == 1 || csts.bits.cfs == 1) { 3715 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_EN_0, 3716 nvme_ctrlr_get_ready_timeout(ctrlr)); 3717 } else { 3718 NVME_CTRLR_DEBUGLOG(ctrlr, "CC.EN = 1 && CSTS.RDY = 0 - waiting for reset to complete\n"); 3719 nvme_ctrlr_set_state_quiet(ctrlr, NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_1, 3720 NVME_TIMEOUT_KEEP_EXISTING); 3721 } 3722 } 3723 3724 static void 3725 nvme_ctrlr_process_init_wait_for_ready_0(void *ctx, uint64_t value, const struct spdk_nvme_cpl *cpl) 3726 { 3727 struct spdk_nvme_ctrlr *ctrlr = ctx; 3728 union spdk_nvme_csts_register csts; 3729 3730 if (spdk_nvme_cpl_is_error(cpl)) { 3731 /* While a device is resetting, it may be unable to service MMIO reads 3732 * temporarily. Allow for this case. 3733 */ 3734 if (!ctrlr->is_failed && ctrlr->state_timeout_tsc != NVME_TIMEOUT_INFINITE) { 3735 NVME_CTRLR_DEBUGLOG(ctrlr, "Failed to read the CSTS register\n"); 3736 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0, 3737 NVME_TIMEOUT_KEEP_EXISTING); 3738 } else { 3739 NVME_CTRLR_ERRLOG(ctrlr, "Failed to read the CSTS register\n"); 3740 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 3741 } 3742 3743 return; 3744 } 3745 3746 assert(value <= UINT32_MAX); 3747 csts.raw = (uint32_t)value; 3748 if (csts.bits.rdy == 0) { 3749 NVME_CTRLR_DEBUGLOG(ctrlr, "CC.EN = 0 && CSTS.RDY = 0\n"); 3750 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_DISABLED, 3751 nvme_ctrlr_get_ready_timeout(ctrlr)); 3752 } else { 3753 nvme_ctrlr_set_state_quiet(ctrlr, NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0, 3754 NVME_TIMEOUT_KEEP_EXISTING); 3755 } 3756 } 3757 3758 static void 3759 nvme_ctrlr_process_init_enable_wait_for_ready_1(void *ctx, uint64_t value, 3760 const struct spdk_nvme_cpl *cpl) 3761 { 3762 struct spdk_nvme_ctrlr *ctrlr = ctx; 3763 union spdk_nvme_csts_register csts; 3764 3765 if (spdk_nvme_cpl_is_error(cpl)) { 3766 /* While a device is resetting, it may be unable to service MMIO reads 3767 * temporarily. Allow for this case. 3768 */ 3769 if (!ctrlr->is_failed && ctrlr->state_timeout_tsc != NVME_TIMEOUT_INFINITE) { 3770 NVME_CTRLR_DEBUGLOG(ctrlr, "Failed to read the CSTS register\n"); 3771 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ENABLE_WAIT_FOR_READY_1, 3772 NVME_TIMEOUT_KEEP_EXISTING); 3773 } else { 3774 NVME_CTRLR_ERRLOG(ctrlr, "Failed to read the CSTS register\n"); 3775 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 3776 } 3777 3778 return; 3779 } 3780 3781 assert(value <= UINT32_MAX); 3782 csts.raw = value; 3783 if (csts.bits.rdy == 1) { 3784 NVME_CTRLR_DEBUGLOG(ctrlr, "CC.EN = 1 && CSTS.RDY = 1 - controller is ready\n"); 3785 /* 3786 * The controller has been enabled. 3787 * Perform the rest of initialization serially. 3788 */ 3789 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_RESET_ADMIN_QUEUE, 3790 ctrlr->opts.admin_timeout_ms); 3791 } else { 3792 nvme_ctrlr_set_state_quiet(ctrlr, NVME_CTRLR_STATE_ENABLE_WAIT_FOR_READY_1, 3793 NVME_TIMEOUT_KEEP_EXISTING); 3794 } 3795 } 3796 3797 /** 3798 * This function will be called repeatedly during initialization until the controller is ready. 3799 */ 3800 int 3801 nvme_ctrlr_process_init(struct spdk_nvme_ctrlr *ctrlr) 3802 { 3803 uint32_t ready_timeout_in_ms; 3804 uint64_t ticks; 3805 int rc = 0; 3806 3807 ticks = spdk_get_ticks(); 3808 3809 /* 3810 * May need to avoid accessing any register on the target controller 3811 * for a while. Return early without touching the FSM. 3812 * Check sleep_timeout_tsc > 0 for unit test. 3813 */ 3814 if ((ctrlr->sleep_timeout_tsc > 0) && 3815 (ticks <= ctrlr->sleep_timeout_tsc)) { 3816 return 0; 3817 } 3818 ctrlr->sleep_timeout_tsc = 0; 3819 3820 ready_timeout_in_ms = nvme_ctrlr_get_ready_timeout(ctrlr); 3821 3822 /* 3823 * Check if the current initialization step is done or has timed out. 3824 */ 3825 switch (ctrlr->state) { 3826 case NVME_CTRLR_STATE_INIT_DELAY: 3827 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_INIT, ready_timeout_in_ms); 3828 if (ctrlr->quirks & NVME_QUIRK_DELAY_BEFORE_INIT) { 3829 /* 3830 * Controller may need some delay before it's enabled. 3831 * 3832 * This is a workaround for an issue where the PCIe-attached NVMe controller 3833 * is not ready after VFIO reset. We delay the initialization rather than the 3834 * enabling itself, because this is required only for the very first enabling 3835 * - directly after a VFIO reset. 3836 */ 3837 NVME_CTRLR_DEBUGLOG(ctrlr, "Adding 2 second delay before initializing the controller\n"); 3838 ctrlr->sleep_timeout_tsc = ticks + (2000 * spdk_get_ticks_hz() / 1000); 3839 } 3840 break; 3841 3842 case NVME_CTRLR_STATE_DISCONNECTED: 3843 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_INIT, NVME_TIMEOUT_INFINITE); 3844 break; 3845 3846 case NVME_CTRLR_STATE_CONNECT_ADMINQ: /* synonymous with NVME_CTRLR_STATE_INIT and NVME_CTRLR_STATE_DISCONNECTED */ 3847 rc = nvme_transport_ctrlr_connect_qpair(ctrlr, ctrlr->adminq); 3848 if (rc == 0) { 3849 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_WAIT_FOR_CONNECT_ADMINQ, 3850 NVME_TIMEOUT_INFINITE); 3851 } else { 3852 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 3853 } 3854 break; 3855 3856 case NVME_CTRLR_STATE_WAIT_FOR_CONNECT_ADMINQ: 3857 spdk_nvme_qpair_process_completions(ctrlr->adminq, 0); 3858 3859 switch (nvme_qpair_get_state(ctrlr->adminq)) { 3860 case NVME_QPAIR_CONNECTING: 3861 break; 3862 case NVME_QPAIR_CONNECTED: 3863 nvme_qpair_set_state(ctrlr->adminq, NVME_QPAIR_ENABLED); 3864 /* Fall through */ 3865 case NVME_QPAIR_ENABLED: 3866 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_READ_VS, 3867 NVME_TIMEOUT_INFINITE); 3868 /* Abort any queued requests that were sent while the adminq was connecting 3869 * to avoid stalling the init process during a reset, as requests don't get 3870 * resubmitted while the controller is resetting and subsequent commands 3871 * would get queued too. 3872 */ 3873 nvme_qpair_abort_queued_reqs(ctrlr->adminq); 3874 break; 3875 case NVME_QPAIR_DISCONNECTING: 3876 assert(ctrlr->adminq->async == true); 3877 break; 3878 case NVME_QPAIR_DISCONNECTED: 3879 /* fallthrough */ 3880 default: 3881 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 3882 break; 3883 } 3884 3885 break; 3886 3887 case NVME_CTRLR_STATE_READ_VS: 3888 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_READ_VS_WAIT_FOR_VS, NVME_TIMEOUT_INFINITE); 3889 rc = nvme_ctrlr_get_vs_async(ctrlr, nvme_ctrlr_process_init_vs_done, ctrlr); 3890 break; 3891 3892 case NVME_CTRLR_STATE_READ_CAP: 3893 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_READ_CAP_WAIT_FOR_CAP, NVME_TIMEOUT_INFINITE); 3894 rc = nvme_ctrlr_get_cap_async(ctrlr, nvme_ctrlr_process_init_cap_done, ctrlr); 3895 break; 3896 3897 case NVME_CTRLR_STATE_CHECK_EN: 3898 /* Begin the hardware initialization by making sure the controller is disabled. */ 3899 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_CHECK_EN_WAIT_FOR_CC, ready_timeout_in_ms); 3900 rc = nvme_ctrlr_get_cc_async(ctrlr, nvme_ctrlr_process_init_check_en, ctrlr); 3901 break; 3902 3903 case NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_1: 3904 /* 3905 * Controller is currently enabled. We need to disable it to cause a reset. 3906 * 3907 * If CC.EN = 1 && CSTS.RDY = 0, the controller is in the process of becoming ready. 3908 * Wait for the ready bit to be 1 before disabling the controller. 3909 */ 3910 nvme_ctrlr_set_state_quiet(ctrlr, NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_1_WAIT_FOR_CSTS, 3911 NVME_TIMEOUT_KEEP_EXISTING); 3912 rc = nvme_ctrlr_get_csts_async(ctrlr, nvme_ctrlr_process_init_wait_for_ready_1, ctrlr); 3913 break; 3914 3915 case NVME_CTRLR_STATE_SET_EN_0: 3916 NVME_CTRLR_DEBUGLOG(ctrlr, "Setting CC.EN = 0\n"); 3917 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_EN_0_WAIT_FOR_CC, ready_timeout_in_ms); 3918 rc = nvme_ctrlr_get_cc_async(ctrlr, nvme_ctrlr_process_init_set_en_0_read_cc, ctrlr); 3919 break; 3920 3921 case NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0: 3922 nvme_ctrlr_set_state_quiet(ctrlr, NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0_WAIT_FOR_CSTS, 3923 NVME_TIMEOUT_KEEP_EXISTING); 3924 rc = nvme_ctrlr_get_csts_async(ctrlr, nvme_ctrlr_process_init_wait_for_ready_0, ctrlr); 3925 break; 3926 3927 case NVME_CTRLR_STATE_DISABLED: 3928 if (ctrlr->is_disconnecting) { 3929 NVME_CTRLR_DEBUGLOG(ctrlr, "Ctrlr was disabled.\n"); 3930 } else { 3931 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ENABLE, ready_timeout_in_ms); 3932 3933 /* 3934 * Delay 100us before setting CC.EN = 1. Some NVMe SSDs miss CC.EN getting 3935 * set to 1 if it is too soon after CSTS.RDY is reported as 0. 3936 */ 3937 spdk_delay_us(100); 3938 } 3939 break; 3940 3941 case NVME_CTRLR_STATE_ENABLE: 3942 NVME_CTRLR_DEBUGLOG(ctrlr, "Setting CC.EN = 1\n"); 3943 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ENABLE_WAIT_FOR_CC, ready_timeout_in_ms); 3944 rc = nvme_ctrlr_enable(ctrlr); 3945 if (rc) { 3946 NVME_CTRLR_ERRLOG(ctrlr, "Ctrlr enable failed with error: %d", rc); 3947 } 3948 return rc; 3949 3950 case NVME_CTRLR_STATE_ENABLE_WAIT_FOR_READY_1: 3951 nvme_ctrlr_set_state_quiet(ctrlr, NVME_CTRLR_STATE_ENABLE_WAIT_FOR_READY_1_WAIT_FOR_CSTS, 3952 NVME_TIMEOUT_KEEP_EXISTING); 3953 rc = nvme_ctrlr_get_csts_async(ctrlr, nvme_ctrlr_process_init_enable_wait_for_ready_1, 3954 ctrlr); 3955 break; 3956 3957 case NVME_CTRLR_STATE_RESET_ADMIN_QUEUE: 3958 nvme_transport_qpair_reset(ctrlr->adminq); 3959 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_IDENTIFY, NVME_TIMEOUT_INFINITE); 3960 break; 3961 3962 case NVME_CTRLR_STATE_IDENTIFY: 3963 rc = nvme_ctrlr_identify(ctrlr); 3964 break; 3965 3966 case NVME_CTRLR_STATE_CONFIGURE_AER: 3967 rc = nvme_ctrlr_configure_aer(ctrlr); 3968 break; 3969 3970 case NVME_CTRLR_STATE_SET_KEEP_ALIVE_TIMEOUT: 3971 rc = nvme_ctrlr_set_keep_alive_timeout(ctrlr); 3972 break; 3973 3974 case NVME_CTRLR_STATE_IDENTIFY_IOCS_SPECIFIC: 3975 rc = nvme_ctrlr_identify_iocs_specific(ctrlr); 3976 break; 3977 3978 case NVME_CTRLR_STATE_GET_ZNS_CMD_EFFECTS_LOG: 3979 rc = nvme_ctrlr_get_zns_cmd_and_effects_log(ctrlr); 3980 break; 3981 3982 case NVME_CTRLR_STATE_SET_NUM_QUEUES: 3983 nvme_ctrlr_update_nvmf_ioccsz(ctrlr); 3984 rc = nvme_ctrlr_set_num_queues(ctrlr); 3985 break; 3986 3987 case NVME_CTRLR_STATE_IDENTIFY_ACTIVE_NS: 3988 _nvme_ctrlr_identify_active_ns(ctrlr); 3989 break; 3990 3991 case NVME_CTRLR_STATE_IDENTIFY_NS: 3992 rc = nvme_ctrlr_identify_namespaces(ctrlr); 3993 break; 3994 3995 case NVME_CTRLR_STATE_IDENTIFY_ID_DESCS: 3996 rc = nvme_ctrlr_identify_id_desc_namespaces(ctrlr); 3997 break; 3998 3999 case NVME_CTRLR_STATE_IDENTIFY_NS_IOCS_SPECIFIC: 4000 rc = nvme_ctrlr_identify_namespaces_iocs_specific(ctrlr); 4001 break; 4002 4003 case NVME_CTRLR_STATE_SET_SUPPORTED_LOG_PAGES: 4004 rc = nvme_ctrlr_set_supported_log_pages(ctrlr); 4005 break; 4006 4007 case NVME_CTRLR_STATE_SET_SUPPORTED_INTEL_LOG_PAGES: 4008 rc = nvme_ctrlr_set_intel_support_log_pages(ctrlr); 4009 break; 4010 4011 case NVME_CTRLR_STATE_SET_SUPPORTED_FEATURES: 4012 nvme_ctrlr_set_supported_features(ctrlr); 4013 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_SET_DB_BUF_CFG, 4014 ctrlr->opts.admin_timeout_ms); 4015 break; 4016 4017 case NVME_CTRLR_STATE_SET_DB_BUF_CFG: 4018 rc = nvme_ctrlr_set_doorbell_buffer_config(ctrlr); 4019 break; 4020 4021 case NVME_CTRLR_STATE_SET_HOST_ID: 4022 rc = nvme_ctrlr_set_host_id(ctrlr); 4023 break; 4024 4025 case NVME_CTRLR_STATE_TRANSPORT_READY: 4026 rc = nvme_transport_ctrlr_ready(ctrlr); 4027 if (rc) { 4028 NVME_CTRLR_ERRLOG(ctrlr, "Transport controller ready step failed: rc %d\n", rc); 4029 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_ERROR, NVME_TIMEOUT_INFINITE); 4030 } else { 4031 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_READY, NVME_TIMEOUT_INFINITE); 4032 } 4033 break; 4034 4035 case NVME_CTRLR_STATE_READY: 4036 NVME_CTRLR_DEBUGLOG(ctrlr, "Ctrlr already in ready state\n"); 4037 return 0; 4038 4039 case NVME_CTRLR_STATE_ERROR: 4040 NVME_CTRLR_ERRLOG(ctrlr, "Ctrlr is in error state\n"); 4041 return -1; 4042 4043 case NVME_CTRLR_STATE_READ_VS_WAIT_FOR_VS: 4044 case NVME_CTRLR_STATE_READ_CAP_WAIT_FOR_CAP: 4045 case NVME_CTRLR_STATE_CHECK_EN_WAIT_FOR_CC: 4046 case NVME_CTRLR_STATE_SET_EN_0_WAIT_FOR_CC: 4047 case NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_1_WAIT_FOR_CSTS: 4048 case NVME_CTRLR_STATE_DISABLE_WAIT_FOR_READY_0_WAIT_FOR_CSTS: 4049 case NVME_CTRLR_STATE_ENABLE_WAIT_FOR_CC: 4050 case NVME_CTRLR_STATE_ENABLE_WAIT_FOR_READY_1_WAIT_FOR_CSTS: 4051 case NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY: 4052 case NVME_CTRLR_STATE_WAIT_FOR_CONFIGURE_AER: 4053 case NVME_CTRLR_STATE_WAIT_FOR_KEEP_ALIVE_TIMEOUT: 4054 case NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_IOCS_SPECIFIC: 4055 case NVME_CTRLR_STATE_WAIT_FOR_GET_ZNS_CMD_EFFECTS_LOG: 4056 case NVME_CTRLR_STATE_WAIT_FOR_SET_NUM_QUEUES: 4057 case NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_ACTIVE_NS: 4058 case NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_NS: 4059 case NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_ID_DESCS: 4060 case NVME_CTRLR_STATE_WAIT_FOR_IDENTIFY_NS_IOCS_SPECIFIC: 4061 case NVME_CTRLR_STATE_WAIT_FOR_SUPPORTED_INTEL_LOG_PAGES: 4062 case NVME_CTRLR_STATE_WAIT_FOR_DB_BUF_CFG: 4063 case NVME_CTRLR_STATE_WAIT_FOR_HOST_ID: 4064 /* 4065 * nvme_ctrlr_process_init() may be called from the completion context 4066 * for the admin qpair. Avoid recursive calls for this case. 4067 */ 4068 if (!ctrlr->adminq->in_completion_context) { 4069 spdk_nvme_qpair_process_completions(ctrlr->adminq, 0); 4070 } 4071 break; 4072 4073 default: 4074 assert(0); 4075 return -1; 4076 } 4077 4078 if (rc) { 4079 NVME_CTRLR_ERRLOG(ctrlr, "Ctrlr operation failed with error: %d, ctrlr state: %d (%s)\n", 4080 rc, ctrlr->state, nvme_ctrlr_state_string(ctrlr->state)); 4081 } 4082 4083 /* Note: we use the ticks captured when we entered this function. 4084 * This covers environments where the SPDK process gets swapped out after 4085 * we tried to advance the state but before we check the timeout here. 4086 * It is not normal for this to happen, but harmless to handle it in this 4087 * way. 4088 */ 4089 if (ctrlr->state_timeout_tsc != NVME_TIMEOUT_INFINITE && 4090 ticks > ctrlr->state_timeout_tsc) { 4091 NVME_CTRLR_ERRLOG(ctrlr, "Initialization timed out in state %d (%s)\n", 4092 ctrlr->state, nvme_ctrlr_state_string(ctrlr->state)); 4093 return -1; 4094 } 4095 4096 return rc; 4097 } 4098 4099 int 4100 nvme_robust_mutex_init_recursive_shared(pthread_mutex_t *mtx) 4101 { 4102 pthread_mutexattr_t attr; 4103 int rc = 0; 4104 4105 if (pthread_mutexattr_init(&attr)) { 4106 return -1; 4107 } 4108 if (pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE) || 4109 #ifndef __FreeBSD__ 4110 pthread_mutexattr_setrobust(&attr, PTHREAD_MUTEX_ROBUST) || 4111 pthread_mutexattr_setpshared(&attr, PTHREAD_PROCESS_SHARED) || 4112 #endif 4113 pthread_mutex_init(mtx, &attr)) { 4114 rc = -1; 4115 } 4116 pthread_mutexattr_destroy(&attr); 4117 return rc; 4118 } 4119 4120 int 4121 nvme_ctrlr_construct(struct spdk_nvme_ctrlr *ctrlr) 4122 { 4123 int rc; 4124 4125 if (ctrlr->trid.trtype == SPDK_NVME_TRANSPORT_PCIE) { 4126 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_INIT_DELAY, NVME_TIMEOUT_INFINITE); 4127 } else { 4128 nvme_ctrlr_set_state(ctrlr, NVME_CTRLR_STATE_INIT, NVME_TIMEOUT_INFINITE); 4129 } 4130 4131 if (ctrlr->opts.admin_queue_size > SPDK_NVME_ADMIN_QUEUE_MAX_ENTRIES) { 4132 NVME_CTRLR_ERRLOG(ctrlr, "admin_queue_size %u exceeds max defined by NVMe spec, use max value\n", 4133 ctrlr->opts.admin_queue_size); 4134 ctrlr->opts.admin_queue_size = SPDK_NVME_ADMIN_QUEUE_MAX_ENTRIES; 4135 } 4136 4137 if (ctrlr->quirks & NVME_QUIRK_MINIMUM_ADMIN_QUEUE_SIZE && 4138 (ctrlr->opts.admin_queue_size % SPDK_NVME_ADMIN_QUEUE_QUIRK_ENTRIES_MULTIPLE) != 0) { 4139 NVME_CTRLR_ERRLOG(ctrlr, 4140 "admin_queue_size %u is invalid for this NVMe device, adjust to next multiple\n", 4141 ctrlr->opts.admin_queue_size); 4142 ctrlr->opts.admin_queue_size = SPDK_ALIGN_CEIL(ctrlr->opts.admin_queue_size, 4143 SPDK_NVME_ADMIN_QUEUE_QUIRK_ENTRIES_MULTIPLE); 4144 } 4145 4146 if (ctrlr->opts.admin_queue_size < SPDK_NVME_ADMIN_QUEUE_MIN_ENTRIES) { 4147 NVME_CTRLR_ERRLOG(ctrlr, 4148 "admin_queue_size %u is less than minimum defined by NVMe spec, use min value\n", 4149 ctrlr->opts.admin_queue_size); 4150 ctrlr->opts.admin_queue_size = SPDK_NVME_ADMIN_QUEUE_MIN_ENTRIES; 4151 } 4152 4153 ctrlr->flags = 0; 4154 ctrlr->free_io_qids = NULL; 4155 ctrlr->is_resetting = false; 4156 ctrlr->is_failed = false; 4157 ctrlr->is_destructed = false; 4158 4159 TAILQ_INIT(&ctrlr->active_io_qpairs); 4160 STAILQ_INIT(&ctrlr->queued_aborts); 4161 ctrlr->outstanding_aborts = 0; 4162 4163 ctrlr->ana_log_page = NULL; 4164 ctrlr->ana_log_page_size = 0; 4165 4166 rc = nvme_robust_mutex_init_recursive_shared(&ctrlr->ctrlr_lock); 4167 if (rc != 0) { 4168 return rc; 4169 } 4170 4171 TAILQ_INIT(&ctrlr->active_procs); 4172 STAILQ_INIT(&ctrlr->register_operations); 4173 4174 RB_INIT(&ctrlr->ns); 4175 4176 return rc; 4177 } 4178 4179 static void 4180 nvme_ctrlr_init_cap(struct spdk_nvme_ctrlr *ctrlr) 4181 { 4182 if (ctrlr->cap.bits.ams & SPDK_NVME_CAP_AMS_WRR) { 4183 ctrlr->flags |= SPDK_NVME_CTRLR_WRR_SUPPORTED; 4184 } 4185 4186 ctrlr->min_page_size = 1u << (12 + ctrlr->cap.bits.mpsmin); 4187 4188 /* For now, always select page_size == min_page_size. */ 4189 ctrlr->page_size = ctrlr->min_page_size; 4190 4191 ctrlr->opts.io_queue_size = spdk_max(ctrlr->opts.io_queue_size, SPDK_NVME_IO_QUEUE_MIN_ENTRIES); 4192 ctrlr->opts.io_queue_size = spdk_min(ctrlr->opts.io_queue_size, MAX_IO_QUEUE_ENTRIES); 4193 if (ctrlr->quirks & NVME_QUIRK_MINIMUM_IO_QUEUE_SIZE && 4194 ctrlr->opts.io_queue_size == DEFAULT_IO_QUEUE_SIZE) { 4195 /* If the user specifically set an IO queue size different than the 4196 * default, use that value. Otherwise overwrite with the quirked value. 4197 * This allows this quirk to be overridden when necessary. 4198 * However, cap.mqes still needs to be respected. 4199 */ 4200 ctrlr->opts.io_queue_size = DEFAULT_IO_QUEUE_SIZE_FOR_QUIRK; 4201 } 4202 ctrlr->opts.io_queue_size = spdk_min(ctrlr->opts.io_queue_size, ctrlr->cap.bits.mqes + 1u); 4203 4204 ctrlr->opts.io_queue_requests = spdk_max(ctrlr->opts.io_queue_requests, ctrlr->opts.io_queue_size); 4205 } 4206 4207 void 4208 nvme_ctrlr_destruct_finish(struct spdk_nvme_ctrlr *ctrlr) 4209 { 4210 pthread_mutex_destroy(&ctrlr->ctrlr_lock); 4211 4212 nvme_ctrlr_free_processes(ctrlr); 4213 } 4214 4215 void 4216 nvme_ctrlr_destruct_async(struct spdk_nvme_ctrlr *ctrlr, 4217 struct nvme_ctrlr_detach_ctx *ctx) 4218 { 4219 struct spdk_nvme_qpair *qpair, *tmp; 4220 4221 NVME_CTRLR_DEBUGLOG(ctrlr, "Prepare to destruct SSD\n"); 4222 4223 ctrlr->prepare_for_reset = false; 4224 ctrlr->is_destructed = true; 4225 4226 spdk_nvme_qpair_process_completions(ctrlr->adminq, 0); 4227 4228 nvme_ctrlr_abort_queued_aborts(ctrlr); 4229 nvme_transport_admin_qpair_abort_aers(ctrlr->adminq); 4230 4231 TAILQ_FOREACH_SAFE(qpair, &ctrlr->active_io_qpairs, tailq, tmp) { 4232 spdk_nvme_ctrlr_free_io_qpair(qpair); 4233 } 4234 4235 nvme_ctrlr_free_doorbell_buffer(ctrlr); 4236 nvme_ctrlr_free_iocs_specific_data(ctrlr); 4237 4238 nvme_ctrlr_shutdown_async(ctrlr, ctx); 4239 } 4240 4241 int 4242 nvme_ctrlr_destruct_poll_async(struct spdk_nvme_ctrlr *ctrlr, 4243 struct nvme_ctrlr_detach_ctx *ctx) 4244 { 4245 struct spdk_nvme_ns *ns, *tmp_ns; 4246 int rc = 0; 4247 4248 if (!ctx->shutdown_complete) { 4249 rc = nvme_ctrlr_shutdown_poll_async(ctrlr, ctx); 4250 if (rc == -EAGAIN) { 4251 return -EAGAIN; 4252 } 4253 /* Destruct ctrlr forcefully for any other error. */ 4254 } 4255 4256 if (ctx->cb_fn) { 4257 ctx->cb_fn(ctrlr); 4258 } 4259 4260 nvme_transport_ctrlr_disconnect_qpair(ctrlr, ctrlr->adminq); 4261 4262 RB_FOREACH_SAFE(ns, nvme_ns_tree, &ctrlr->ns, tmp_ns) { 4263 nvme_ctrlr_destruct_namespace(ctrlr, ns->id); 4264 RB_REMOVE(nvme_ns_tree, &ctrlr->ns, ns); 4265 spdk_free(ns); 4266 } 4267 4268 ctrlr->active_ns_count = 0; 4269 4270 spdk_bit_array_free(&ctrlr->free_io_qids); 4271 4272 free(ctrlr->ana_log_page); 4273 free(ctrlr->copied_ana_desc); 4274 ctrlr->ana_log_page = NULL; 4275 ctrlr->copied_ana_desc = NULL; 4276 ctrlr->ana_log_page_size = 0; 4277 4278 nvme_transport_ctrlr_destruct(ctrlr); 4279 4280 return rc; 4281 } 4282 4283 void 4284 nvme_ctrlr_destruct(struct spdk_nvme_ctrlr *ctrlr) 4285 { 4286 struct nvme_ctrlr_detach_ctx ctx = { .ctrlr = ctrlr }; 4287 int rc; 4288 4289 nvme_ctrlr_destruct_async(ctrlr, &ctx); 4290 4291 while (1) { 4292 rc = nvme_ctrlr_destruct_poll_async(ctrlr, &ctx); 4293 if (rc != -EAGAIN) { 4294 break; 4295 } 4296 nvme_delay(1000); 4297 } 4298 } 4299 4300 int 4301 nvme_ctrlr_submit_admin_request(struct spdk_nvme_ctrlr *ctrlr, 4302 struct nvme_request *req) 4303 { 4304 return nvme_qpair_submit_request(ctrlr->adminq, req); 4305 } 4306 4307 static void 4308 nvme_keep_alive_completion(void *cb_ctx, const struct spdk_nvme_cpl *cpl) 4309 { 4310 /* Do nothing */ 4311 } 4312 4313 /* 4314 * Check if we need to send a Keep Alive command. 4315 * Caller must hold ctrlr->ctrlr_lock. 4316 */ 4317 static int 4318 nvme_ctrlr_keep_alive(struct spdk_nvme_ctrlr *ctrlr) 4319 { 4320 uint64_t now; 4321 struct nvme_request *req; 4322 struct spdk_nvme_cmd *cmd; 4323 int rc = 0; 4324 4325 now = spdk_get_ticks(); 4326 if (now < ctrlr->next_keep_alive_tick) { 4327 return rc; 4328 } 4329 4330 req = nvme_allocate_request_null(ctrlr->adminq, nvme_keep_alive_completion, NULL); 4331 if (req == NULL) { 4332 return rc; 4333 } 4334 4335 cmd = &req->cmd; 4336 cmd->opc = SPDK_NVME_OPC_KEEP_ALIVE; 4337 4338 rc = nvme_ctrlr_submit_admin_request(ctrlr, req); 4339 if (rc != 0) { 4340 NVME_CTRLR_ERRLOG(ctrlr, "Submitting Keep Alive failed\n"); 4341 rc = -ENXIO; 4342 } 4343 4344 ctrlr->next_keep_alive_tick = now + ctrlr->keep_alive_interval_ticks; 4345 return rc; 4346 } 4347 4348 int32_t 4349 spdk_nvme_ctrlr_process_admin_completions(struct spdk_nvme_ctrlr *ctrlr) 4350 { 4351 int32_t num_completions; 4352 int32_t rc; 4353 struct spdk_nvme_ctrlr_process *active_proc; 4354 4355 nvme_robust_mutex_lock(&ctrlr->ctrlr_lock); 4356 4357 if (ctrlr->keep_alive_interval_ticks) { 4358 rc = nvme_ctrlr_keep_alive(ctrlr); 4359 if (rc) { 4360 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 4361 return rc; 4362 } 4363 } 4364 4365 rc = nvme_io_msg_process(ctrlr); 4366 if (rc < 0) { 4367 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 4368 return rc; 4369 } 4370 num_completions = rc; 4371 4372 rc = spdk_nvme_qpair_process_completions(ctrlr->adminq, 0); 4373 4374 /* Each process has an async list, complete the ones for this process object */ 4375 active_proc = nvme_ctrlr_get_current_process(ctrlr); 4376 if (active_proc) { 4377 nvme_ctrlr_complete_queued_async_events(ctrlr); 4378 } 4379 4380 if (rc == -ENXIO && ctrlr->is_disconnecting) { 4381 nvme_ctrlr_disconnect_done(ctrlr); 4382 } 4383 4384 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 4385 4386 if (rc < 0) { 4387 num_completions = rc; 4388 } else { 4389 num_completions += rc; 4390 } 4391 4392 return num_completions; 4393 } 4394 4395 const struct spdk_nvme_ctrlr_data * 4396 spdk_nvme_ctrlr_get_data(struct spdk_nvme_ctrlr *ctrlr) 4397 { 4398 return &ctrlr->cdata; 4399 } 4400 4401 union spdk_nvme_csts_register spdk_nvme_ctrlr_get_regs_csts(struct spdk_nvme_ctrlr *ctrlr) 4402 { 4403 union spdk_nvme_csts_register csts; 4404 4405 if (nvme_ctrlr_get_csts(ctrlr, &csts)) { 4406 csts.raw = SPDK_NVME_INVALID_REGISTER_VALUE; 4407 } 4408 return csts; 4409 } 4410 4411 union spdk_nvme_cc_register spdk_nvme_ctrlr_get_regs_cc(struct spdk_nvme_ctrlr *ctrlr) 4412 { 4413 union spdk_nvme_cc_register cc; 4414 4415 if (nvme_ctrlr_get_cc(ctrlr, &cc)) { 4416 cc.raw = SPDK_NVME_INVALID_REGISTER_VALUE; 4417 } 4418 return cc; 4419 } 4420 4421 union spdk_nvme_cap_register spdk_nvme_ctrlr_get_regs_cap(struct spdk_nvme_ctrlr *ctrlr) 4422 { 4423 return ctrlr->cap; 4424 } 4425 4426 union spdk_nvme_vs_register spdk_nvme_ctrlr_get_regs_vs(struct spdk_nvme_ctrlr *ctrlr) 4427 { 4428 return ctrlr->vs; 4429 } 4430 4431 union spdk_nvme_cmbsz_register spdk_nvme_ctrlr_get_regs_cmbsz(struct spdk_nvme_ctrlr *ctrlr) 4432 { 4433 union spdk_nvme_cmbsz_register cmbsz; 4434 4435 if (nvme_ctrlr_get_cmbsz(ctrlr, &cmbsz)) { 4436 cmbsz.raw = 0; 4437 } 4438 4439 return cmbsz; 4440 } 4441 4442 union spdk_nvme_pmrcap_register spdk_nvme_ctrlr_get_regs_pmrcap(struct spdk_nvme_ctrlr *ctrlr) 4443 { 4444 union spdk_nvme_pmrcap_register pmrcap; 4445 4446 if (nvme_ctrlr_get_pmrcap(ctrlr, &pmrcap)) { 4447 pmrcap.raw = 0; 4448 } 4449 4450 return pmrcap; 4451 } 4452 4453 union spdk_nvme_bpinfo_register spdk_nvme_ctrlr_get_regs_bpinfo(struct spdk_nvme_ctrlr *ctrlr) 4454 { 4455 union spdk_nvme_bpinfo_register bpinfo; 4456 4457 if (nvme_ctrlr_get_bpinfo(ctrlr, &bpinfo)) { 4458 bpinfo.raw = 0; 4459 } 4460 4461 return bpinfo; 4462 } 4463 4464 uint64_t 4465 spdk_nvme_ctrlr_get_pmrsz(struct spdk_nvme_ctrlr *ctrlr) 4466 { 4467 return ctrlr->pmr_size; 4468 } 4469 4470 uint32_t 4471 spdk_nvme_ctrlr_get_num_ns(struct spdk_nvme_ctrlr *ctrlr) 4472 { 4473 return ctrlr->cdata.nn; 4474 } 4475 4476 bool 4477 spdk_nvme_ctrlr_is_active_ns(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid) 4478 { 4479 struct spdk_nvme_ns tmp, *ns; 4480 4481 tmp.id = nsid; 4482 ns = RB_FIND(nvme_ns_tree, &ctrlr->ns, &tmp); 4483 4484 if (ns != NULL) { 4485 return ns->active; 4486 } 4487 4488 return false; 4489 } 4490 4491 uint32_t 4492 spdk_nvme_ctrlr_get_first_active_ns(struct spdk_nvme_ctrlr *ctrlr) 4493 { 4494 struct spdk_nvme_ns *ns; 4495 4496 ns = RB_MIN(nvme_ns_tree, &ctrlr->ns); 4497 if (ns == NULL) { 4498 return 0; 4499 } 4500 4501 while (ns != NULL) { 4502 if (ns->active) { 4503 return ns->id; 4504 } 4505 4506 ns = RB_NEXT(nvme_ns_tree, &ctrlr->ns, ns); 4507 } 4508 4509 return 0; 4510 } 4511 4512 uint32_t 4513 spdk_nvme_ctrlr_get_next_active_ns(struct spdk_nvme_ctrlr *ctrlr, uint32_t prev_nsid) 4514 { 4515 struct spdk_nvme_ns tmp, *ns; 4516 4517 tmp.id = prev_nsid; 4518 ns = RB_FIND(nvme_ns_tree, &ctrlr->ns, &tmp); 4519 if (ns == NULL) { 4520 return 0; 4521 } 4522 4523 ns = RB_NEXT(nvme_ns_tree, &ctrlr->ns, ns); 4524 while (ns != NULL) { 4525 if (ns->active) { 4526 return ns->id; 4527 } 4528 4529 ns = RB_NEXT(nvme_ns_tree, &ctrlr->ns, ns); 4530 } 4531 4532 return 0; 4533 } 4534 4535 struct spdk_nvme_ns * 4536 spdk_nvme_ctrlr_get_ns(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid) 4537 { 4538 struct spdk_nvme_ns tmp; 4539 struct spdk_nvme_ns *ns; 4540 4541 if (nsid < 1 || nsid > ctrlr->cdata.nn) { 4542 return NULL; 4543 } 4544 4545 nvme_robust_mutex_lock(&ctrlr->ctrlr_lock); 4546 4547 tmp.id = nsid; 4548 ns = RB_FIND(nvme_ns_tree, &ctrlr->ns, &tmp); 4549 4550 if (ns == NULL) { 4551 ns = spdk_zmalloc(sizeof(struct spdk_nvme_ns), 64, NULL, SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_SHARE); 4552 if (ns == NULL) { 4553 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 4554 return NULL; 4555 } 4556 4557 NVME_CTRLR_DEBUGLOG(ctrlr, "Namespace %u was added\n", nsid); 4558 ns->id = nsid; 4559 RB_INSERT(nvme_ns_tree, &ctrlr->ns, ns); 4560 } 4561 4562 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 4563 4564 return ns; 4565 } 4566 4567 struct spdk_pci_device * 4568 spdk_nvme_ctrlr_get_pci_device(struct spdk_nvme_ctrlr *ctrlr) 4569 { 4570 if (ctrlr == NULL) { 4571 return NULL; 4572 } 4573 4574 if (ctrlr->trid.trtype != SPDK_NVME_TRANSPORT_PCIE) { 4575 return NULL; 4576 } 4577 4578 return nvme_ctrlr_proc_get_devhandle(ctrlr); 4579 } 4580 4581 uint32_t 4582 spdk_nvme_ctrlr_get_max_xfer_size(const struct spdk_nvme_ctrlr *ctrlr) 4583 { 4584 return ctrlr->max_xfer_size; 4585 } 4586 4587 uint16_t 4588 spdk_nvme_ctrlr_get_max_sges(const struct spdk_nvme_ctrlr *ctrlr) 4589 { 4590 if (ctrlr->flags & SPDK_NVME_CTRLR_SGL_SUPPORTED) { 4591 return ctrlr->max_sges; 4592 } else { 4593 return UINT16_MAX; 4594 } 4595 } 4596 4597 void 4598 spdk_nvme_ctrlr_register_aer_callback(struct spdk_nvme_ctrlr *ctrlr, 4599 spdk_nvme_aer_cb aer_cb_fn, 4600 void *aer_cb_arg) 4601 { 4602 struct spdk_nvme_ctrlr_process *active_proc; 4603 4604 nvme_robust_mutex_lock(&ctrlr->ctrlr_lock); 4605 4606 active_proc = nvme_ctrlr_get_current_process(ctrlr); 4607 if (active_proc) { 4608 active_proc->aer_cb_fn = aer_cb_fn; 4609 active_proc->aer_cb_arg = aer_cb_arg; 4610 } 4611 4612 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 4613 } 4614 4615 void 4616 spdk_nvme_ctrlr_disable_read_changed_ns_list_log_page(struct spdk_nvme_ctrlr *ctrlr) 4617 { 4618 ctrlr->opts.disable_read_changed_ns_list_log_page = true; 4619 } 4620 4621 void 4622 spdk_nvme_ctrlr_register_timeout_callback(struct spdk_nvme_ctrlr *ctrlr, 4623 uint64_t timeout_io_us, uint64_t timeout_admin_us, 4624 spdk_nvme_timeout_cb cb_fn, void *cb_arg) 4625 { 4626 struct spdk_nvme_ctrlr_process *active_proc; 4627 4628 nvme_robust_mutex_lock(&ctrlr->ctrlr_lock); 4629 4630 active_proc = nvme_ctrlr_get_current_process(ctrlr); 4631 if (active_proc) { 4632 active_proc->timeout_io_ticks = timeout_io_us * spdk_get_ticks_hz() / 1000000ULL; 4633 active_proc->timeout_admin_ticks = timeout_admin_us * spdk_get_ticks_hz() / 1000000ULL; 4634 active_proc->timeout_cb_fn = cb_fn; 4635 active_proc->timeout_cb_arg = cb_arg; 4636 } 4637 4638 ctrlr->timeout_enabled = true; 4639 4640 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 4641 } 4642 4643 bool 4644 spdk_nvme_ctrlr_is_log_page_supported(struct spdk_nvme_ctrlr *ctrlr, uint8_t log_page) 4645 { 4646 /* No bounds check necessary, since log_page is uint8_t and log_page_supported has 256 entries */ 4647 SPDK_STATIC_ASSERT(sizeof(ctrlr->log_page_supported) == 256, "log_page_supported size mismatch"); 4648 return ctrlr->log_page_supported[log_page]; 4649 } 4650 4651 bool 4652 spdk_nvme_ctrlr_is_feature_supported(struct spdk_nvme_ctrlr *ctrlr, uint8_t feature_code) 4653 { 4654 /* No bounds check necessary, since feature_code is uint8_t and feature_supported has 256 entries */ 4655 SPDK_STATIC_ASSERT(sizeof(ctrlr->feature_supported) == 256, "feature_supported size mismatch"); 4656 return ctrlr->feature_supported[feature_code]; 4657 } 4658 4659 int 4660 spdk_nvme_ctrlr_attach_ns(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid, 4661 struct spdk_nvme_ctrlr_list *payload) 4662 { 4663 struct nvme_completion_poll_status *status; 4664 struct spdk_nvme_ns *ns; 4665 int res; 4666 4667 if (nsid == 0) { 4668 return -EINVAL; 4669 } 4670 4671 status = calloc(1, sizeof(*status)); 4672 if (!status) { 4673 NVME_CTRLR_ERRLOG(ctrlr, "Failed to allocate status tracker\n"); 4674 return -ENOMEM; 4675 } 4676 4677 res = nvme_ctrlr_cmd_attach_ns(ctrlr, nsid, payload, 4678 nvme_completion_poll_cb, status); 4679 if (res) { 4680 free(status); 4681 return res; 4682 } 4683 if (nvme_wait_for_completion_robust_lock(ctrlr->adminq, status, &ctrlr->ctrlr_lock)) { 4684 NVME_CTRLR_ERRLOG(ctrlr, "spdk_nvme_ctrlr_attach_ns failed!\n"); 4685 if (!status->timed_out) { 4686 free(status); 4687 } 4688 return -ENXIO; 4689 } 4690 free(status); 4691 4692 res = nvme_ctrlr_identify_active_ns(ctrlr); 4693 if (res) { 4694 return res; 4695 } 4696 4697 ns = spdk_nvme_ctrlr_get_ns(ctrlr, nsid); 4698 if (ns == NULL) { 4699 NVME_CTRLR_ERRLOG(ctrlr, "spdk_nvme_ctrlr_get_ns failed!\n"); 4700 return -ENXIO; 4701 } 4702 4703 return nvme_ns_construct(ns, nsid, ctrlr); 4704 } 4705 4706 int 4707 spdk_nvme_ctrlr_detach_ns(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid, 4708 struct spdk_nvme_ctrlr_list *payload) 4709 { 4710 struct nvme_completion_poll_status *status; 4711 int res; 4712 4713 if (nsid == 0) { 4714 return -EINVAL; 4715 } 4716 4717 status = calloc(1, sizeof(*status)); 4718 if (!status) { 4719 NVME_CTRLR_ERRLOG(ctrlr, "Failed to allocate status tracker\n"); 4720 return -ENOMEM; 4721 } 4722 4723 res = nvme_ctrlr_cmd_detach_ns(ctrlr, nsid, payload, 4724 nvme_completion_poll_cb, status); 4725 if (res) { 4726 free(status); 4727 return res; 4728 } 4729 if (nvme_wait_for_completion_robust_lock(ctrlr->adminq, status, &ctrlr->ctrlr_lock)) { 4730 NVME_CTRLR_ERRLOG(ctrlr, "spdk_nvme_ctrlr_detach_ns failed!\n"); 4731 if (!status->timed_out) { 4732 free(status); 4733 } 4734 return -ENXIO; 4735 } 4736 free(status); 4737 4738 return nvme_ctrlr_identify_active_ns(ctrlr); 4739 } 4740 4741 uint32_t 4742 spdk_nvme_ctrlr_create_ns(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_ns_data *payload) 4743 { 4744 struct nvme_completion_poll_status *status; 4745 int res; 4746 uint32_t nsid; 4747 4748 status = calloc(1, sizeof(*status)); 4749 if (!status) { 4750 NVME_CTRLR_ERRLOG(ctrlr, "Failed to allocate status tracker\n"); 4751 return 0; 4752 } 4753 4754 res = nvme_ctrlr_cmd_create_ns(ctrlr, payload, nvme_completion_poll_cb, status); 4755 if (res) { 4756 free(status); 4757 return 0; 4758 } 4759 if (nvme_wait_for_completion_robust_lock(ctrlr->adminq, status, &ctrlr->ctrlr_lock)) { 4760 NVME_CTRLR_ERRLOG(ctrlr, "spdk_nvme_ctrlr_create_ns failed!\n"); 4761 if (!status->timed_out) { 4762 free(status); 4763 } 4764 return 0; 4765 } 4766 4767 nsid = status->cpl.cdw0; 4768 free(status); 4769 4770 assert(nsid > 0); 4771 4772 /* Return the namespace ID that was created */ 4773 return nsid; 4774 } 4775 4776 int 4777 spdk_nvme_ctrlr_delete_ns(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid) 4778 { 4779 struct nvme_completion_poll_status *status; 4780 int res; 4781 4782 if (nsid == 0) { 4783 return -EINVAL; 4784 } 4785 4786 status = calloc(1, sizeof(*status)); 4787 if (!status) { 4788 NVME_CTRLR_ERRLOG(ctrlr, "Failed to allocate status tracker\n"); 4789 return -ENOMEM; 4790 } 4791 4792 res = nvme_ctrlr_cmd_delete_ns(ctrlr, nsid, nvme_completion_poll_cb, status); 4793 if (res) { 4794 free(status); 4795 return res; 4796 } 4797 if (nvme_wait_for_completion_robust_lock(ctrlr->adminq, status, &ctrlr->ctrlr_lock)) { 4798 NVME_CTRLR_ERRLOG(ctrlr, "spdk_nvme_ctrlr_delete_ns failed!\n"); 4799 if (!status->timed_out) { 4800 free(status); 4801 } 4802 return -ENXIO; 4803 } 4804 free(status); 4805 4806 return nvme_ctrlr_identify_active_ns(ctrlr); 4807 } 4808 4809 int 4810 spdk_nvme_ctrlr_format(struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid, 4811 struct spdk_nvme_format *format) 4812 { 4813 struct nvme_completion_poll_status *status; 4814 int res; 4815 4816 status = calloc(1, sizeof(*status)); 4817 if (!status) { 4818 NVME_CTRLR_ERRLOG(ctrlr, "Failed to allocate status tracker\n"); 4819 return -ENOMEM; 4820 } 4821 4822 res = nvme_ctrlr_cmd_format(ctrlr, nsid, format, nvme_completion_poll_cb, 4823 status); 4824 if (res) { 4825 free(status); 4826 return res; 4827 } 4828 if (nvme_wait_for_completion_robust_lock(ctrlr->adminq, status, &ctrlr->ctrlr_lock)) { 4829 NVME_CTRLR_ERRLOG(ctrlr, "spdk_nvme_ctrlr_format failed!\n"); 4830 if (!status->timed_out) { 4831 free(status); 4832 } 4833 return -ENXIO; 4834 } 4835 free(status); 4836 4837 return spdk_nvme_ctrlr_reset(ctrlr); 4838 } 4839 4840 int 4841 spdk_nvme_ctrlr_update_firmware(struct spdk_nvme_ctrlr *ctrlr, void *payload, uint32_t size, 4842 int slot, enum spdk_nvme_fw_commit_action commit_action, struct spdk_nvme_status *completion_status) 4843 { 4844 struct spdk_nvme_fw_commit fw_commit; 4845 struct nvme_completion_poll_status *status; 4846 int res; 4847 unsigned int size_remaining; 4848 unsigned int offset; 4849 unsigned int transfer; 4850 uint8_t *p; 4851 4852 if (!completion_status) { 4853 return -EINVAL; 4854 } 4855 memset(completion_status, 0, sizeof(struct spdk_nvme_status)); 4856 if (size % 4) { 4857 NVME_CTRLR_ERRLOG(ctrlr, "spdk_nvme_ctrlr_update_firmware invalid size!\n"); 4858 return -1; 4859 } 4860 4861 /* Current support only for SPDK_NVME_FW_COMMIT_REPLACE_IMG 4862 * and SPDK_NVME_FW_COMMIT_REPLACE_AND_ENABLE_IMG 4863 */ 4864 if ((commit_action != SPDK_NVME_FW_COMMIT_REPLACE_IMG) && 4865 (commit_action != SPDK_NVME_FW_COMMIT_REPLACE_AND_ENABLE_IMG)) { 4866 NVME_CTRLR_ERRLOG(ctrlr, "spdk_nvme_ctrlr_update_firmware invalid command!\n"); 4867 return -1; 4868 } 4869 4870 status = calloc(1, sizeof(*status)); 4871 if (!status) { 4872 NVME_CTRLR_ERRLOG(ctrlr, "Failed to allocate status tracker\n"); 4873 return -ENOMEM; 4874 } 4875 4876 /* Firmware download */ 4877 size_remaining = size; 4878 offset = 0; 4879 p = payload; 4880 4881 while (size_remaining > 0) { 4882 transfer = spdk_min(size_remaining, ctrlr->min_page_size); 4883 4884 memset(status, 0, sizeof(*status)); 4885 res = nvme_ctrlr_cmd_fw_image_download(ctrlr, transfer, offset, p, 4886 nvme_completion_poll_cb, 4887 status); 4888 if (res) { 4889 free(status); 4890 return res; 4891 } 4892 4893 if (nvme_wait_for_completion_robust_lock(ctrlr->adminq, status, &ctrlr->ctrlr_lock)) { 4894 NVME_CTRLR_ERRLOG(ctrlr, "spdk_nvme_ctrlr_fw_image_download failed!\n"); 4895 if (!status->timed_out) { 4896 free(status); 4897 } 4898 return -ENXIO; 4899 } 4900 p += transfer; 4901 offset += transfer; 4902 size_remaining -= transfer; 4903 } 4904 4905 /* Firmware commit */ 4906 memset(&fw_commit, 0, sizeof(struct spdk_nvme_fw_commit)); 4907 fw_commit.fs = slot; 4908 fw_commit.ca = commit_action; 4909 4910 memset(status, 0, sizeof(*status)); 4911 res = nvme_ctrlr_cmd_fw_commit(ctrlr, &fw_commit, nvme_completion_poll_cb, 4912 status); 4913 if (res) { 4914 free(status); 4915 return res; 4916 } 4917 4918 res = nvme_wait_for_completion_robust_lock(ctrlr->adminq, status, &ctrlr->ctrlr_lock); 4919 4920 memcpy(completion_status, &status->cpl.status, sizeof(struct spdk_nvme_status)); 4921 4922 if (!status->timed_out) { 4923 free(status); 4924 } 4925 4926 if (res) { 4927 if (completion_status->sct != SPDK_NVME_SCT_COMMAND_SPECIFIC || 4928 completion_status->sc != SPDK_NVME_SC_FIRMWARE_REQ_NVM_RESET) { 4929 if (completion_status->sct == SPDK_NVME_SCT_COMMAND_SPECIFIC && 4930 completion_status->sc == SPDK_NVME_SC_FIRMWARE_REQ_CONVENTIONAL_RESET) { 4931 NVME_CTRLR_NOTICELOG(ctrlr, 4932 "firmware activation requires conventional reset to be performed. !\n"); 4933 } else { 4934 NVME_CTRLR_ERRLOG(ctrlr, "nvme_ctrlr_cmd_fw_commit failed!\n"); 4935 } 4936 return -ENXIO; 4937 } 4938 } 4939 4940 return spdk_nvme_ctrlr_reset(ctrlr); 4941 } 4942 4943 int 4944 spdk_nvme_ctrlr_reserve_cmb(struct spdk_nvme_ctrlr *ctrlr) 4945 { 4946 int rc, size; 4947 union spdk_nvme_cmbsz_register cmbsz; 4948 4949 cmbsz = spdk_nvme_ctrlr_get_regs_cmbsz(ctrlr); 4950 4951 if (cmbsz.bits.rds == 0 || cmbsz.bits.wds == 0) { 4952 return -ENOTSUP; 4953 } 4954 4955 size = cmbsz.bits.sz * (0x1000 << (cmbsz.bits.szu * 4)); 4956 4957 nvme_robust_mutex_lock(&ctrlr->ctrlr_lock); 4958 rc = nvme_transport_ctrlr_reserve_cmb(ctrlr); 4959 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 4960 4961 if (rc < 0) { 4962 return rc; 4963 } 4964 4965 return size; 4966 } 4967 4968 void * 4969 spdk_nvme_ctrlr_map_cmb(struct spdk_nvme_ctrlr *ctrlr, size_t *size) 4970 { 4971 void *buf; 4972 4973 nvme_robust_mutex_lock(&ctrlr->ctrlr_lock); 4974 buf = nvme_transport_ctrlr_map_cmb(ctrlr, size); 4975 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 4976 4977 return buf; 4978 } 4979 4980 void 4981 spdk_nvme_ctrlr_unmap_cmb(struct spdk_nvme_ctrlr *ctrlr) 4982 { 4983 nvme_robust_mutex_lock(&ctrlr->ctrlr_lock); 4984 nvme_transport_ctrlr_unmap_cmb(ctrlr); 4985 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 4986 } 4987 4988 int 4989 spdk_nvme_ctrlr_enable_pmr(struct spdk_nvme_ctrlr *ctrlr) 4990 { 4991 int rc; 4992 4993 nvme_robust_mutex_lock(&ctrlr->ctrlr_lock); 4994 rc = nvme_transport_ctrlr_enable_pmr(ctrlr); 4995 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 4996 4997 return rc; 4998 } 4999 5000 int 5001 spdk_nvme_ctrlr_disable_pmr(struct spdk_nvme_ctrlr *ctrlr) 5002 { 5003 int rc; 5004 5005 nvme_robust_mutex_lock(&ctrlr->ctrlr_lock); 5006 rc = nvme_transport_ctrlr_disable_pmr(ctrlr); 5007 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 5008 5009 return rc; 5010 } 5011 5012 void * 5013 spdk_nvme_ctrlr_map_pmr(struct spdk_nvme_ctrlr *ctrlr, size_t *size) 5014 { 5015 void *buf; 5016 5017 nvme_robust_mutex_lock(&ctrlr->ctrlr_lock); 5018 buf = nvme_transport_ctrlr_map_pmr(ctrlr, size); 5019 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 5020 5021 return buf; 5022 } 5023 5024 int 5025 spdk_nvme_ctrlr_unmap_pmr(struct spdk_nvme_ctrlr *ctrlr) 5026 { 5027 int rc; 5028 5029 nvme_robust_mutex_lock(&ctrlr->ctrlr_lock); 5030 rc = nvme_transport_ctrlr_unmap_pmr(ctrlr); 5031 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 5032 5033 return rc; 5034 } 5035 5036 int 5037 spdk_nvme_ctrlr_read_boot_partition_start(struct spdk_nvme_ctrlr *ctrlr, void *payload, 5038 uint32_t bprsz, uint32_t bprof, uint32_t bpid) 5039 { 5040 union spdk_nvme_bprsel_register bprsel; 5041 union spdk_nvme_bpinfo_register bpinfo; 5042 uint64_t bpmbl, bpmb_size; 5043 5044 if (ctrlr->cap.bits.bps == 0) { 5045 return -ENOTSUP; 5046 } 5047 5048 if (nvme_ctrlr_get_bpinfo(ctrlr, &bpinfo)) { 5049 NVME_CTRLR_ERRLOG(ctrlr, "get bpinfo failed\n"); 5050 return -EIO; 5051 } 5052 5053 if (bpinfo.bits.brs == SPDK_NVME_BRS_READ_IN_PROGRESS) { 5054 NVME_CTRLR_ERRLOG(ctrlr, "Boot Partition read already initiated\n"); 5055 return -EALREADY; 5056 } 5057 5058 nvme_robust_mutex_lock(&ctrlr->ctrlr_lock); 5059 5060 bpmb_size = bprsz * 4096; 5061 bpmbl = spdk_vtophys(payload, &bpmb_size); 5062 if (bpmbl == SPDK_VTOPHYS_ERROR) { 5063 NVME_CTRLR_ERRLOG(ctrlr, "spdk_vtophys of bpmbl failed\n"); 5064 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 5065 return -EFAULT; 5066 } 5067 5068 if (bpmb_size != bprsz * 4096) { 5069 NVME_CTRLR_ERRLOG(ctrlr, "Boot Partition buffer is not physically contiguous\n"); 5070 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 5071 return -EFAULT; 5072 } 5073 5074 if (nvme_ctrlr_set_bpmbl(ctrlr, bpmbl)) { 5075 NVME_CTRLR_ERRLOG(ctrlr, "set_bpmbl() failed\n"); 5076 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 5077 return -EIO; 5078 } 5079 5080 bprsel.bits.bpid = bpid; 5081 bprsel.bits.bprof = bprof; 5082 bprsel.bits.bprsz = bprsz; 5083 5084 if (nvme_ctrlr_set_bprsel(ctrlr, &bprsel)) { 5085 NVME_CTRLR_ERRLOG(ctrlr, "set_bprsel() failed\n"); 5086 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 5087 return -EIO; 5088 } 5089 5090 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 5091 return 0; 5092 } 5093 5094 int 5095 spdk_nvme_ctrlr_read_boot_partition_poll(struct spdk_nvme_ctrlr *ctrlr) 5096 { 5097 int rc = 0; 5098 union spdk_nvme_bpinfo_register bpinfo; 5099 5100 if (nvme_ctrlr_get_bpinfo(ctrlr, &bpinfo)) { 5101 NVME_CTRLR_ERRLOG(ctrlr, "get bpinfo failed\n"); 5102 return -EIO; 5103 } 5104 5105 switch (bpinfo.bits.brs) { 5106 case SPDK_NVME_BRS_NO_READ: 5107 NVME_CTRLR_ERRLOG(ctrlr, "Boot Partition read not initiated\n"); 5108 rc = -EINVAL; 5109 break; 5110 case SPDK_NVME_BRS_READ_IN_PROGRESS: 5111 NVME_CTRLR_DEBUGLOG(ctrlr, "Boot Partition read in progress\n"); 5112 rc = -EAGAIN; 5113 break; 5114 case SPDK_NVME_BRS_READ_ERROR: 5115 NVME_CTRLR_ERRLOG(ctrlr, "Error completing Boot Partition read\n"); 5116 rc = -EIO; 5117 break; 5118 case SPDK_NVME_BRS_READ_SUCCESS: 5119 NVME_CTRLR_INFOLOG(ctrlr, "Boot Partition read completed successfully\n"); 5120 break; 5121 default: 5122 NVME_CTRLR_ERRLOG(ctrlr, "Invalid Boot Partition read status\n"); 5123 rc = -EINVAL; 5124 } 5125 5126 return rc; 5127 } 5128 5129 static void 5130 nvme_write_boot_partition_cb(void *arg, const struct spdk_nvme_cpl *cpl) 5131 { 5132 int res; 5133 struct spdk_nvme_ctrlr *ctrlr = arg; 5134 struct spdk_nvme_fw_commit fw_commit; 5135 struct spdk_nvme_cpl err_cpl = 5136 {.status = {.sct = SPDK_NVME_SCT_GENERIC, .sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR }}; 5137 5138 if (spdk_nvme_cpl_is_error(cpl)) { 5139 NVME_CTRLR_ERRLOG(ctrlr, "Write Boot Partition failed\n"); 5140 ctrlr->bp_write_cb_fn(ctrlr->bp_write_cb_arg, cpl); 5141 return; 5142 } 5143 5144 if (ctrlr->bp_ws == SPDK_NVME_BP_WS_DOWNLOADING) { 5145 NVME_CTRLR_DEBUGLOG(ctrlr, "Boot Partition Downloading at Offset %d Success\n", ctrlr->fw_offset); 5146 ctrlr->fw_payload = (uint8_t *)ctrlr->fw_payload + ctrlr->fw_transfer_size; 5147 ctrlr->fw_offset += ctrlr->fw_transfer_size; 5148 ctrlr->fw_size_remaining -= ctrlr->fw_transfer_size; 5149 ctrlr->fw_transfer_size = spdk_min(ctrlr->fw_size_remaining, ctrlr->min_page_size); 5150 res = nvme_ctrlr_cmd_fw_image_download(ctrlr, ctrlr->fw_transfer_size, ctrlr->fw_offset, 5151 ctrlr->fw_payload, nvme_write_boot_partition_cb, ctrlr); 5152 if (res) { 5153 NVME_CTRLR_ERRLOG(ctrlr, "nvme_ctrlr_cmd_fw_image_download failed!\n"); 5154 ctrlr->bp_write_cb_fn(ctrlr->bp_write_cb_arg, &err_cpl); 5155 return; 5156 } 5157 5158 if (ctrlr->fw_transfer_size < ctrlr->min_page_size) { 5159 ctrlr->bp_ws = SPDK_NVME_BP_WS_DOWNLOADED; 5160 } 5161 } else if (ctrlr->bp_ws == SPDK_NVME_BP_WS_DOWNLOADED) { 5162 NVME_CTRLR_DEBUGLOG(ctrlr, "Boot Partition Download Success\n"); 5163 memset(&fw_commit, 0, sizeof(struct spdk_nvme_fw_commit)); 5164 fw_commit.bpid = ctrlr->bpid; 5165 fw_commit.ca = SPDK_NVME_FW_COMMIT_REPLACE_BOOT_PARTITION; 5166 res = nvme_ctrlr_cmd_fw_commit(ctrlr, &fw_commit, 5167 nvme_write_boot_partition_cb, ctrlr); 5168 if (res) { 5169 NVME_CTRLR_ERRLOG(ctrlr, "nvme_ctrlr_cmd_fw_commit failed!\n"); 5170 NVME_CTRLR_ERRLOG(ctrlr, "commit action: %d\n", fw_commit.ca); 5171 ctrlr->bp_write_cb_fn(ctrlr->bp_write_cb_arg, &err_cpl); 5172 return; 5173 } 5174 5175 ctrlr->bp_ws = SPDK_NVME_BP_WS_REPLACE; 5176 } else if (ctrlr->bp_ws == SPDK_NVME_BP_WS_REPLACE) { 5177 NVME_CTRLR_DEBUGLOG(ctrlr, "Boot Partition Replacement Success\n"); 5178 memset(&fw_commit, 0, sizeof(struct spdk_nvme_fw_commit)); 5179 fw_commit.bpid = ctrlr->bpid; 5180 fw_commit.ca = SPDK_NVME_FW_COMMIT_ACTIVATE_BOOT_PARTITION; 5181 res = nvme_ctrlr_cmd_fw_commit(ctrlr, &fw_commit, 5182 nvme_write_boot_partition_cb, ctrlr); 5183 if (res) { 5184 NVME_CTRLR_ERRLOG(ctrlr, "nvme_ctrlr_cmd_fw_commit failed!\n"); 5185 NVME_CTRLR_ERRLOG(ctrlr, "commit action: %d\n", fw_commit.ca); 5186 ctrlr->bp_write_cb_fn(ctrlr->bp_write_cb_arg, &err_cpl); 5187 return; 5188 } 5189 5190 ctrlr->bp_ws = SPDK_NVME_BP_WS_ACTIVATE; 5191 } else if (ctrlr->bp_ws == SPDK_NVME_BP_WS_ACTIVATE) { 5192 NVME_CTRLR_DEBUGLOG(ctrlr, "Boot Partition Activation Success\n"); 5193 ctrlr->bp_write_cb_fn(ctrlr->bp_write_cb_arg, cpl); 5194 } else { 5195 NVME_CTRLR_ERRLOG(ctrlr, "Invalid Boot Partition write state\n"); 5196 ctrlr->bp_write_cb_fn(ctrlr->bp_write_cb_arg, &err_cpl); 5197 return; 5198 } 5199 } 5200 5201 int 5202 spdk_nvme_ctrlr_write_boot_partition(struct spdk_nvme_ctrlr *ctrlr, 5203 void *payload, uint32_t size, uint32_t bpid, 5204 spdk_nvme_cmd_cb cb_fn, void *cb_arg) 5205 { 5206 int res; 5207 5208 if (ctrlr->cap.bits.bps == 0) { 5209 return -ENOTSUP; 5210 } 5211 5212 ctrlr->bp_ws = SPDK_NVME_BP_WS_DOWNLOADING; 5213 ctrlr->bpid = bpid; 5214 ctrlr->bp_write_cb_fn = cb_fn; 5215 ctrlr->bp_write_cb_arg = cb_arg; 5216 ctrlr->fw_offset = 0; 5217 ctrlr->fw_size_remaining = size; 5218 ctrlr->fw_payload = payload; 5219 ctrlr->fw_transfer_size = spdk_min(ctrlr->fw_size_remaining, ctrlr->min_page_size); 5220 5221 res = nvme_ctrlr_cmd_fw_image_download(ctrlr, ctrlr->fw_transfer_size, ctrlr->fw_offset, 5222 ctrlr->fw_payload, nvme_write_boot_partition_cb, ctrlr); 5223 5224 return res; 5225 } 5226 5227 bool 5228 spdk_nvme_ctrlr_is_discovery(struct spdk_nvme_ctrlr *ctrlr) 5229 { 5230 assert(ctrlr); 5231 5232 return !strncmp(ctrlr->trid.subnqn, SPDK_NVMF_DISCOVERY_NQN, 5233 strlen(SPDK_NVMF_DISCOVERY_NQN)); 5234 } 5235 5236 bool 5237 spdk_nvme_ctrlr_is_fabrics(struct spdk_nvme_ctrlr *ctrlr) 5238 { 5239 assert(ctrlr); 5240 5241 return spdk_nvme_trtype_is_fabrics(ctrlr->trid.trtype); 5242 } 5243 5244 int 5245 spdk_nvme_ctrlr_security_receive(struct spdk_nvme_ctrlr *ctrlr, uint8_t secp, 5246 uint16_t spsp, uint8_t nssf, void *payload, size_t size) 5247 { 5248 struct nvme_completion_poll_status *status; 5249 int res; 5250 5251 status = calloc(1, sizeof(*status)); 5252 if (!status) { 5253 NVME_CTRLR_ERRLOG(ctrlr, "Failed to allocate status tracker\n"); 5254 return -ENOMEM; 5255 } 5256 5257 res = spdk_nvme_ctrlr_cmd_security_receive(ctrlr, secp, spsp, nssf, payload, size, 5258 nvme_completion_poll_cb, status); 5259 if (res) { 5260 free(status); 5261 return res; 5262 } 5263 if (nvme_wait_for_completion_robust_lock(ctrlr->adminq, status, &ctrlr->ctrlr_lock)) { 5264 NVME_CTRLR_ERRLOG(ctrlr, "spdk_nvme_ctrlr_cmd_security_receive failed!\n"); 5265 if (!status->timed_out) { 5266 free(status); 5267 } 5268 return -ENXIO; 5269 } 5270 free(status); 5271 5272 return 0; 5273 } 5274 5275 int 5276 spdk_nvme_ctrlr_security_send(struct spdk_nvme_ctrlr *ctrlr, uint8_t secp, 5277 uint16_t spsp, uint8_t nssf, void *payload, size_t size) 5278 { 5279 struct nvme_completion_poll_status *status; 5280 int res; 5281 5282 status = calloc(1, sizeof(*status)); 5283 if (!status) { 5284 NVME_CTRLR_ERRLOG(ctrlr, "Failed to allocate status tracker\n"); 5285 return -ENOMEM; 5286 } 5287 5288 res = spdk_nvme_ctrlr_cmd_security_send(ctrlr, secp, spsp, nssf, payload, size, 5289 nvme_completion_poll_cb, 5290 status); 5291 if (res) { 5292 free(status); 5293 return res; 5294 } 5295 if (nvme_wait_for_completion_robust_lock(ctrlr->adminq, status, &ctrlr->ctrlr_lock)) { 5296 NVME_CTRLR_ERRLOG(ctrlr, "spdk_nvme_ctrlr_cmd_security_send failed!\n"); 5297 if (!status->timed_out) { 5298 free(status); 5299 } 5300 return -ENXIO; 5301 } 5302 5303 free(status); 5304 5305 return 0; 5306 } 5307 5308 uint64_t 5309 spdk_nvme_ctrlr_get_flags(struct spdk_nvme_ctrlr *ctrlr) 5310 { 5311 return ctrlr->flags; 5312 } 5313 5314 const struct spdk_nvme_transport_id * 5315 spdk_nvme_ctrlr_get_transport_id(struct spdk_nvme_ctrlr *ctrlr) 5316 { 5317 return &ctrlr->trid; 5318 } 5319 5320 int32_t 5321 spdk_nvme_ctrlr_alloc_qid(struct spdk_nvme_ctrlr *ctrlr) 5322 { 5323 uint32_t qid; 5324 5325 assert(ctrlr->free_io_qids); 5326 nvme_robust_mutex_lock(&ctrlr->ctrlr_lock); 5327 qid = spdk_bit_array_find_first_set(ctrlr->free_io_qids, 1); 5328 if (qid > ctrlr->opts.num_io_queues) { 5329 NVME_CTRLR_ERRLOG(ctrlr, "No free I/O queue IDs\n"); 5330 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 5331 return -1; 5332 } 5333 5334 spdk_bit_array_clear(ctrlr->free_io_qids, qid); 5335 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 5336 return qid; 5337 } 5338 5339 void 5340 spdk_nvme_ctrlr_free_qid(struct spdk_nvme_ctrlr *ctrlr, uint16_t qid) 5341 { 5342 assert(qid <= ctrlr->opts.num_io_queues); 5343 5344 nvme_robust_mutex_lock(&ctrlr->ctrlr_lock); 5345 5346 if (spdk_likely(ctrlr->free_io_qids)) { 5347 spdk_bit_array_set(ctrlr->free_io_qids, qid); 5348 } 5349 5350 nvme_robust_mutex_unlock(&ctrlr->ctrlr_lock); 5351 } 5352 5353 int 5354 spdk_nvme_ctrlr_get_memory_domains(const struct spdk_nvme_ctrlr *ctrlr, 5355 struct spdk_memory_domain **domains, int array_size) 5356 { 5357 return nvme_transport_ctrlr_get_memory_domains(ctrlr, domains, array_size); 5358 } 5359