1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) Intel Corporation. All rights reserved. 5 * Copyright (c) 2020 Mellanox Technologies LTD. All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include "spdk/nvmf_spec.h" 35 #include "spdk/string.h" 36 #include "nvme_internal.h" 37 #include "nvme_io_msg.h" 38 #include "nvme_uevent.h" 39 40 #define SPDK_NVME_DRIVER_NAME "spdk_nvme_driver" 41 42 struct nvme_driver *g_spdk_nvme_driver; 43 pid_t g_spdk_nvme_pid; 44 45 /* gross timeout of 180 seconds in milliseconds */ 46 static int g_nvme_driver_timeout_ms = 3 * 60 * 1000; 47 48 /* Per-process attached controller list */ 49 static TAILQ_HEAD(, spdk_nvme_ctrlr) g_nvme_attached_ctrlrs = 50 TAILQ_HEAD_INITIALIZER(g_nvme_attached_ctrlrs); 51 52 /* Returns true if ctrlr should be stored on the multi-process shared_attached_ctrlrs list */ 53 static bool 54 nvme_ctrlr_shared(const struct spdk_nvme_ctrlr *ctrlr) 55 { 56 return ctrlr->trid.trtype == SPDK_NVME_TRANSPORT_PCIE; 57 } 58 59 void 60 nvme_ctrlr_connected(struct spdk_nvme_probe_ctx *probe_ctx, 61 struct spdk_nvme_ctrlr *ctrlr) 62 { 63 TAILQ_INSERT_TAIL(&probe_ctx->init_ctrlrs, ctrlr, tailq); 64 } 65 66 int 67 spdk_nvme_detach(struct spdk_nvme_ctrlr *ctrlr) 68 { 69 nvme_robust_mutex_lock(&g_spdk_nvme_driver->lock); 70 71 nvme_ctrlr_proc_put_ref(ctrlr); 72 73 if (nvme_ctrlr_get_ref_count(ctrlr) == 0) { 74 nvme_io_msg_ctrlr_detach(ctrlr); 75 if (nvme_ctrlr_shared(ctrlr)) { 76 TAILQ_REMOVE(&g_spdk_nvme_driver->shared_attached_ctrlrs, ctrlr, tailq); 77 } else { 78 TAILQ_REMOVE(&g_nvme_attached_ctrlrs, ctrlr, tailq); 79 } 80 nvme_ctrlr_destruct(ctrlr); 81 } 82 83 nvme_robust_mutex_unlock(&g_spdk_nvme_driver->lock); 84 return 0; 85 } 86 87 void 88 nvme_completion_poll_cb(void *arg, const struct spdk_nvme_cpl *cpl) 89 { 90 struct nvme_completion_poll_status *status = arg; 91 92 if (status->timed_out) { 93 /* There is no routine waiting for the completion of this request, free allocated memory */ 94 free(status); 95 return; 96 } 97 98 /* 99 * Copy status into the argument passed by the caller, so that 100 * the caller can check the status to determine if the 101 * the request passed or failed. 102 */ 103 memcpy(&status->cpl, cpl, sizeof(*cpl)); 104 status->done = true; 105 } 106 107 /** 108 * Poll qpair for completions until a command completes. 109 * 110 * \param qpair queue to poll 111 * \param status completion status. The user must fill this structure with zeroes before calling 112 * this function 113 * \param robust_mutex optional robust mutex to lock while polling qpair 114 * \param timeout_in_usecs optional timeout 115 * 116 * \return 0 if command completed without error, 117 * -EIO if command completed with error, 118 * -ECANCELED if command is not completed due to transport/device error or time expired 119 * 120 * The command to wait upon must be submitted with nvme_completion_poll_cb as the callback 121 * and status as the callback argument. 122 */ 123 int 124 nvme_wait_for_completion_robust_lock_timeout( 125 struct spdk_nvme_qpair *qpair, 126 struct nvme_completion_poll_status *status, 127 pthread_mutex_t *robust_mutex, 128 uint64_t timeout_in_usecs) 129 { 130 uint64_t timeout_tsc = 0; 131 int rc = 0; 132 133 if (timeout_in_usecs) { 134 timeout_tsc = spdk_get_ticks() + timeout_in_usecs * spdk_get_ticks_hz() / SPDK_SEC_TO_USEC; 135 } 136 137 while (status->done == false) { 138 if (robust_mutex) { 139 nvme_robust_mutex_lock(robust_mutex); 140 } 141 142 rc = spdk_nvme_qpair_process_completions(qpair, 0); 143 144 if (robust_mutex) { 145 nvme_robust_mutex_unlock(robust_mutex); 146 } 147 148 if (rc < 0) { 149 status->cpl.status.sct = SPDK_NVME_SCT_GENERIC; 150 status->cpl.status.sc = SPDK_NVME_SC_ABORTED_SQ_DELETION; 151 break; 152 } 153 if (timeout_tsc && spdk_get_ticks() > timeout_tsc) { 154 rc = -1; 155 break; 156 } 157 } 158 159 if (status->done == false) { 160 status->timed_out = true; 161 } 162 163 if (rc < 0) { 164 return -ECANCELED; 165 } 166 167 return spdk_nvme_cpl_is_error(&status->cpl) ? -EIO : 0; 168 } 169 170 /** 171 * Poll qpair for completions until a command completes. 172 * 173 * \param qpair queue to poll 174 * \param status completion status. The user must fill this structure with zeroes before calling 175 * this function 176 * \param robust_mutex optional robust mutex to lock while polling qpair 177 * 178 * \return 0 if command completed without error, 179 * -EIO if command completed with error, 180 * -ECANCELED if command is not completed due to transport/device error 181 * 182 * The command to wait upon must be submitted with nvme_completion_poll_cb as the callback 183 * and status as the callback argument. 184 */ 185 int 186 nvme_wait_for_completion_robust_lock( 187 struct spdk_nvme_qpair *qpair, 188 struct nvme_completion_poll_status *status, 189 pthread_mutex_t *robust_mutex) 190 { 191 return nvme_wait_for_completion_robust_lock_timeout(qpair, status, robust_mutex, 0); 192 } 193 194 int 195 nvme_wait_for_completion(struct spdk_nvme_qpair *qpair, 196 struct nvme_completion_poll_status *status) 197 { 198 return nvme_wait_for_completion_robust_lock_timeout(qpair, status, NULL, 0); 199 } 200 201 /** 202 * Poll qpair for completions until a command completes. 203 * 204 * \param qpair queue to poll 205 * \param status completion status. The user must fill this structure with zeroes before calling 206 * this function 207 * \param timeout_in_usecs optional timeout 208 * 209 * \return 0 if command completed without error, 210 * -EIO if command completed with error, 211 * -ECANCELED if command is not completed due to transport/device error or time expired 212 * 213 * The command to wait upon must be submitted with nvme_completion_poll_cb as the callback 214 * and status as the callback argument. 215 */ 216 int 217 nvme_wait_for_completion_timeout(struct spdk_nvme_qpair *qpair, 218 struct nvme_completion_poll_status *status, 219 uint64_t timeout_in_usecs) 220 { 221 return nvme_wait_for_completion_robust_lock_timeout(qpair, status, NULL, timeout_in_usecs); 222 } 223 224 static void 225 nvme_user_copy_cmd_complete(void *arg, const struct spdk_nvme_cpl *cpl) 226 { 227 struct nvme_request *req = arg; 228 enum spdk_nvme_data_transfer xfer; 229 230 if (req->user_buffer && req->payload_size) { 231 /* Copy back to the user buffer and free the contig buffer */ 232 assert(nvme_payload_type(&req->payload) == NVME_PAYLOAD_TYPE_CONTIG); 233 xfer = spdk_nvme_opc_get_data_transfer(req->cmd.opc); 234 if (xfer == SPDK_NVME_DATA_CONTROLLER_TO_HOST || 235 xfer == SPDK_NVME_DATA_BIDIRECTIONAL) { 236 assert(req->pid == getpid()); 237 memcpy(req->user_buffer, req->payload.contig_or_cb_arg, req->payload_size); 238 } 239 240 spdk_free(req->payload.contig_or_cb_arg); 241 } 242 243 /* Call the user's original callback now that the buffer has been copied */ 244 req->user_cb_fn(req->user_cb_arg, cpl); 245 } 246 247 /** 248 * Allocate a request as well as a DMA-capable buffer to copy to/from the user's buffer. 249 * 250 * This is intended for use in non-fast-path functions (admin commands, reservations, etc.) 251 * where the overhead of a copy is not a problem. 252 */ 253 struct nvme_request * 254 nvme_allocate_request_user_copy(struct spdk_nvme_qpair *qpair, 255 void *buffer, uint32_t payload_size, spdk_nvme_cmd_cb cb_fn, 256 void *cb_arg, bool host_to_controller) 257 { 258 struct nvme_request *req; 259 void *dma_buffer = NULL; 260 261 if (buffer && payload_size) { 262 dma_buffer = spdk_zmalloc(payload_size, 4096, NULL, 263 SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA); 264 if (!dma_buffer) { 265 return NULL; 266 } 267 268 if (host_to_controller) { 269 memcpy(dma_buffer, buffer, payload_size); 270 } 271 } 272 273 req = nvme_allocate_request_contig(qpair, dma_buffer, payload_size, nvme_user_copy_cmd_complete, 274 NULL); 275 if (!req) { 276 spdk_free(dma_buffer); 277 return NULL; 278 } 279 280 req->user_cb_fn = cb_fn; 281 req->user_cb_arg = cb_arg; 282 req->user_buffer = buffer; 283 req->cb_arg = req; 284 285 return req; 286 } 287 288 /** 289 * Check if a request has exceeded the controller timeout. 290 * 291 * \param req request to check for timeout. 292 * \param cid command ID for command submitted by req (will be passed to timeout_cb_fn) 293 * \param active_proc per-process data for the controller associated with req 294 * \param now_tick current time from spdk_get_ticks() 295 * \return 0 if requests submitted more recently than req should still be checked for timeouts, or 296 * 1 if requests newer than req need not be checked. 297 * 298 * The request's timeout callback will be called if needed; the caller is only responsible for 299 * calling this function on each outstanding request. 300 */ 301 int 302 nvme_request_check_timeout(struct nvme_request *req, uint16_t cid, 303 struct spdk_nvme_ctrlr_process *active_proc, 304 uint64_t now_tick) 305 { 306 struct spdk_nvme_qpair *qpair = req->qpair; 307 struct spdk_nvme_ctrlr *ctrlr = qpair->ctrlr; 308 309 assert(active_proc->timeout_cb_fn != NULL); 310 311 if (req->timed_out || req->submit_tick == 0) { 312 return 0; 313 } 314 315 if (req->pid != g_spdk_nvme_pid) { 316 return 0; 317 } 318 319 if (nvme_qpair_is_admin_queue(qpair) && 320 req->cmd.opc == SPDK_NVME_OPC_ASYNC_EVENT_REQUEST) { 321 return 0; 322 } 323 324 if (req->submit_tick + active_proc->timeout_ticks > now_tick) { 325 return 1; 326 } 327 328 req->timed_out = true; 329 330 /* 331 * We don't want to expose the admin queue to the user, 332 * so when we're timing out admin commands set the 333 * qpair to NULL. 334 */ 335 active_proc->timeout_cb_fn(active_proc->timeout_cb_arg, ctrlr, 336 nvme_qpair_is_admin_queue(qpair) ? NULL : qpair, 337 cid); 338 return 0; 339 } 340 341 int 342 nvme_robust_mutex_init_shared(pthread_mutex_t *mtx) 343 { 344 int rc = 0; 345 346 #ifdef __FreeBSD__ 347 pthread_mutex_init(mtx, NULL); 348 #else 349 pthread_mutexattr_t attr; 350 351 if (pthread_mutexattr_init(&attr)) { 352 return -1; 353 } 354 if (pthread_mutexattr_setpshared(&attr, PTHREAD_PROCESS_SHARED) || 355 pthread_mutexattr_setrobust(&attr, PTHREAD_MUTEX_ROBUST) || 356 pthread_mutex_init(mtx, &attr)) { 357 rc = -1; 358 } 359 pthread_mutexattr_destroy(&attr); 360 #endif 361 362 return rc; 363 } 364 365 int 366 nvme_driver_init(void) 367 { 368 static pthread_mutex_t g_init_mutex = PTHREAD_MUTEX_INITIALIZER; 369 int ret = 0; 370 /* Any socket ID */ 371 int socket_id = -1; 372 373 /* Use a special process-private mutex to ensure the global 374 * nvme driver object (g_spdk_nvme_driver) gets initialized by 375 * only one thread. Once that object is established and its 376 * mutex is initialized, we can unlock this mutex and use that 377 * one instead. 378 */ 379 pthread_mutex_lock(&g_init_mutex); 380 381 /* Each process needs its own pid. */ 382 g_spdk_nvme_pid = getpid(); 383 384 /* 385 * Only one thread from one process will do this driver init work. 386 * The primary process will reserve the shared memory and do the 387 * initialization. 388 * The secondary process will lookup the existing reserved memory. 389 */ 390 if (spdk_process_is_primary()) { 391 /* The unique named memzone already reserved. */ 392 if (g_spdk_nvme_driver != NULL) { 393 pthread_mutex_unlock(&g_init_mutex); 394 return 0; 395 } else { 396 g_spdk_nvme_driver = spdk_memzone_reserve(SPDK_NVME_DRIVER_NAME, 397 sizeof(struct nvme_driver), socket_id, 398 SPDK_MEMZONE_NO_IOVA_CONTIG); 399 } 400 401 if (g_spdk_nvme_driver == NULL) { 402 SPDK_ERRLOG("primary process failed to reserve memory\n"); 403 pthread_mutex_unlock(&g_init_mutex); 404 return -1; 405 } 406 } else { 407 g_spdk_nvme_driver = spdk_memzone_lookup(SPDK_NVME_DRIVER_NAME); 408 409 /* The unique named memzone already reserved by the primary process. */ 410 if (g_spdk_nvme_driver != NULL) { 411 int ms_waited = 0; 412 413 /* Wait the nvme driver to get initialized. */ 414 while ((g_spdk_nvme_driver->initialized == false) && 415 (ms_waited < g_nvme_driver_timeout_ms)) { 416 ms_waited++; 417 nvme_delay(1000); /* delay 1ms */ 418 } 419 if (g_spdk_nvme_driver->initialized == false) { 420 SPDK_ERRLOG("timeout waiting for primary process to init\n"); 421 pthread_mutex_unlock(&g_init_mutex); 422 return -1; 423 } 424 } else { 425 SPDK_ERRLOG("primary process is not started yet\n"); 426 pthread_mutex_unlock(&g_init_mutex); 427 return -1; 428 } 429 430 pthread_mutex_unlock(&g_init_mutex); 431 return 0; 432 } 433 434 /* 435 * At this moment, only one thread from the primary process will do 436 * the g_spdk_nvme_driver initialization 437 */ 438 assert(spdk_process_is_primary()); 439 440 ret = nvme_robust_mutex_init_shared(&g_spdk_nvme_driver->lock); 441 if (ret != 0) { 442 SPDK_ERRLOG("failed to initialize mutex\n"); 443 spdk_memzone_free(SPDK_NVME_DRIVER_NAME); 444 pthread_mutex_unlock(&g_init_mutex); 445 return ret; 446 } 447 448 /* The lock in the shared g_spdk_nvme_driver object is now ready to 449 * be used - so we can unlock the g_init_mutex here. 450 */ 451 pthread_mutex_unlock(&g_init_mutex); 452 nvme_robust_mutex_lock(&g_spdk_nvme_driver->lock); 453 454 g_spdk_nvme_driver->initialized = false; 455 g_spdk_nvme_driver->hotplug_fd = nvme_uevent_connect(); 456 if (g_spdk_nvme_driver->hotplug_fd < 0) { 457 SPDK_DEBUGLOG(SPDK_LOG_NVME, "Failed to open uevent netlink socket\n"); 458 } 459 460 TAILQ_INIT(&g_spdk_nvme_driver->shared_attached_ctrlrs); 461 462 spdk_uuid_generate(&g_spdk_nvme_driver->default_extended_host_id); 463 464 nvme_robust_mutex_unlock(&g_spdk_nvme_driver->lock); 465 466 return ret; 467 } 468 469 /* This function must only be called while holding g_spdk_nvme_driver->lock */ 470 int 471 nvme_ctrlr_probe(const struct spdk_nvme_transport_id *trid, 472 struct spdk_nvme_probe_ctx *probe_ctx, void *devhandle) 473 { 474 struct spdk_nvme_ctrlr *ctrlr; 475 struct spdk_nvme_ctrlr_opts opts; 476 477 assert(trid != NULL); 478 479 spdk_nvme_ctrlr_get_default_ctrlr_opts(&opts, sizeof(opts)); 480 481 if (!probe_ctx->probe_cb || probe_ctx->probe_cb(probe_ctx->cb_ctx, trid, &opts)) { 482 ctrlr = nvme_get_ctrlr_by_trid_unsafe(trid); 483 if (ctrlr) { 484 /* This ctrlr already exists. 485 * Increase the ref count before calling attach_cb() as the user may 486 * call nvme_detach() immediately. */ 487 nvme_ctrlr_proc_get_ref(ctrlr); 488 489 if (probe_ctx->attach_cb) { 490 nvme_robust_mutex_unlock(&g_spdk_nvme_driver->lock); 491 probe_ctx->attach_cb(probe_ctx->cb_ctx, &ctrlr->trid, ctrlr, &ctrlr->opts); 492 nvme_robust_mutex_lock(&g_spdk_nvme_driver->lock); 493 } 494 return 0; 495 } 496 497 ctrlr = nvme_transport_ctrlr_construct(trid, &opts, devhandle); 498 if (ctrlr == NULL) { 499 SPDK_ERRLOG("Failed to construct NVMe controller for SSD: %s\n", trid->traddr); 500 return -1; 501 } 502 ctrlr->remove_cb = probe_ctx->remove_cb; 503 ctrlr->cb_ctx = probe_ctx->cb_ctx; 504 505 if (ctrlr->quirks & NVME_QUIRK_MINIMUM_IO_QUEUE_SIZE && 506 ctrlr->opts.io_queue_size == DEFAULT_IO_QUEUE_SIZE) { 507 /* If the user specifically set an IO queue size different than the 508 * default, use that value. Otherwise overwrite with the quirked value. 509 * This allows this quirk to be overridden when necessary. 510 * However, cap.mqes still needs to be respected. 511 */ 512 ctrlr->opts.io_queue_size = spdk_min(DEFAULT_IO_QUEUE_SIZE_FOR_QUIRK, ctrlr->cap.bits.mqes + 1u); 513 } 514 515 nvme_qpair_set_state(ctrlr->adminq, NVME_QPAIR_ENABLED); 516 TAILQ_INSERT_TAIL(&probe_ctx->init_ctrlrs, ctrlr, tailq); 517 return 0; 518 } 519 520 return 1; 521 } 522 523 static int 524 nvme_ctrlr_poll_internal(struct spdk_nvme_ctrlr *ctrlr, 525 struct spdk_nvme_probe_ctx *probe_ctx) 526 { 527 int rc = 0; 528 529 rc = nvme_ctrlr_process_init(ctrlr); 530 531 if (rc) { 532 /* Controller failed to initialize. */ 533 TAILQ_REMOVE(&probe_ctx->init_ctrlrs, ctrlr, tailq); 534 SPDK_ERRLOG("Failed to initialize SSD: %s\n", ctrlr->trid.traddr); 535 nvme_ctrlr_fail(ctrlr, false); 536 nvme_ctrlr_destruct(ctrlr); 537 return rc; 538 } 539 540 if (ctrlr->state != NVME_CTRLR_STATE_READY) { 541 return 0; 542 } 543 544 STAILQ_INIT(&ctrlr->io_producers); 545 546 /* 547 * Controller has been initialized. 548 * Move it to the attached_ctrlrs list. 549 */ 550 TAILQ_REMOVE(&probe_ctx->init_ctrlrs, ctrlr, tailq); 551 552 nvme_robust_mutex_lock(&g_spdk_nvme_driver->lock); 553 if (nvme_ctrlr_shared(ctrlr)) { 554 TAILQ_INSERT_TAIL(&g_spdk_nvme_driver->shared_attached_ctrlrs, ctrlr, tailq); 555 } else { 556 TAILQ_INSERT_TAIL(&g_nvme_attached_ctrlrs, ctrlr, tailq); 557 } 558 559 /* 560 * Increase the ref count before calling attach_cb() as the user may 561 * call nvme_detach() immediately. 562 */ 563 nvme_ctrlr_proc_get_ref(ctrlr); 564 nvme_robust_mutex_unlock(&g_spdk_nvme_driver->lock); 565 566 if (probe_ctx->attach_cb) { 567 probe_ctx->attach_cb(probe_ctx->cb_ctx, &ctrlr->trid, ctrlr, &ctrlr->opts); 568 return 0; 569 } 570 571 return 0; 572 } 573 574 static int 575 nvme_init_controllers(struct spdk_nvme_probe_ctx *probe_ctx) 576 { 577 int rc = 0; 578 579 while (true) { 580 rc = spdk_nvme_probe_poll_async(probe_ctx); 581 if (rc != -EAGAIN) { 582 return rc; 583 } 584 } 585 586 return rc; 587 } 588 589 /* This function must not be called while holding g_spdk_nvme_driver->lock */ 590 static struct spdk_nvme_ctrlr * 591 nvme_get_ctrlr_by_trid(const struct spdk_nvme_transport_id *trid) 592 { 593 struct spdk_nvme_ctrlr *ctrlr; 594 595 nvme_robust_mutex_lock(&g_spdk_nvme_driver->lock); 596 ctrlr = nvme_get_ctrlr_by_trid_unsafe(trid); 597 nvme_robust_mutex_unlock(&g_spdk_nvme_driver->lock); 598 599 return ctrlr; 600 } 601 602 /* This function must be called while holding g_spdk_nvme_driver->lock */ 603 struct spdk_nvme_ctrlr * 604 nvme_get_ctrlr_by_trid_unsafe(const struct spdk_nvme_transport_id *trid) 605 { 606 struct spdk_nvme_ctrlr *ctrlr; 607 608 /* Search per-process list */ 609 TAILQ_FOREACH(ctrlr, &g_nvme_attached_ctrlrs, tailq) { 610 if (spdk_nvme_transport_id_compare(&ctrlr->trid, trid) == 0) { 611 return ctrlr; 612 } 613 } 614 615 /* Search multi-process shared list */ 616 TAILQ_FOREACH(ctrlr, &g_spdk_nvme_driver->shared_attached_ctrlrs, tailq) { 617 if (spdk_nvme_transport_id_compare(&ctrlr->trid, trid) == 0) { 618 return ctrlr; 619 } 620 } 621 622 return NULL; 623 } 624 625 /* This function must only be called while holding g_spdk_nvme_driver->lock */ 626 static int 627 nvme_probe_internal(struct spdk_nvme_probe_ctx *probe_ctx, 628 bool direct_connect) 629 { 630 int rc; 631 struct spdk_nvme_ctrlr *ctrlr, *ctrlr_tmp; 632 633 spdk_nvme_trid_populate_transport(&probe_ctx->trid, probe_ctx->trid.trtype); 634 if (!spdk_nvme_transport_available_by_name(probe_ctx->trid.trstring)) { 635 SPDK_ERRLOG("NVMe trtype %u not available\n", probe_ctx->trid.trtype); 636 return -1; 637 } 638 639 nvme_robust_mutex_lock(&g_spdk_nvme_driver->lock); 640 641 rc = nvme_transport_ctrlr_scan(probe_ctx, direct_connect); 642 if (rc != 0) { 643 SPDK_ERRLOG("NVMe ctrlr scan failed\n"); 644 TAILQ_FOREACH_SAFE(ctrlr, &probe_ctx->init_ctrlrs, tailq, ctrlr_tmp) { 645 TAILQ_REMOVE(&probe_ctx->init_ctrlrs, ctrlr, tailq); 646 nvme_transport_ctrlr_destruct(ctrlr); 647 } 648 nvme_robust_mutex_unlock(&g_spdk_nvme_driver->lock); 649 return -1; 650 } 651 652 /* 653 * Probe controllers on the shared_attached_ctrlrs list 654 */ 655 if (!spdk_process_is_primary() && (probe_ctx->trid.trtype == SPDK_NVME_TRANSPORT_PCIE)) { 656 TAILQ_FOREACH(ctrlr, &g_spdk_nvme_driver->shared_attached_ctrlrs, tailq) { 657 /* Do not attach other ctrlrs if user specify a valid trid */ 658 if ((strlen(probe_ctx->trid.traddr) != 0) && 659 (spdk_nvme_transport_id_compare(&probe_ctx->trid, &ctrlr->trid))) { 660 continue; 661 } 662 663 /* Do not attach if we failed to initialize it in this process */ 664 if (nvme_ctrlr_get_current_process(ctrlr) == NULL) { 665 continue; 666 } 667 668 nvme_ctrlr_proc_get_ref(ctrlr); 669 670 /* 671 * Unlock while calling attach_cb() so the user can call other functions 672 * that may take the driver lock, like nvme_detach(). 673 */ 674 if (probe_ctx->attach_cb) { 675 nvme_robust_mutex_unlock(&g_spdk_nvme_driver->lock); 676 probe_ctx->attach_cb(probe_ctx->cb_ctx, &ctrlr->trid, ctrlr, &ctrlr->opts); 677 nvme_robust_mutex_lock(&g_spdk_nvme_driver->lock); 678 } 679 } 680 } 681 682 nvme_robust_mutex_unlock(&g_spdk_nvme_driver->lock); 683 684 return 0; 685 } 686 687 static void 688 nvme_probe_ctx_init(struct spdk_nvme_probe_ctx *probe_ctx, 689 const struct spdk_nvme_transport_id *trid, 690 void *cb_ctx, 691 spdk_nvme_probe_cb probe_cb, 692 spdk_nvme_attach_cb attach_cb, 693 spdk_nvme_remove_cb remove_cb) 694 { 695 probe_ctx->trid = *trid; 696 probe_ctx->cb_ctx = cb_ctx; 697 probe_ctx->probe_cb = probe_cb; 698 probe_ctx->attach_cb = attach_cb; 699 probe_ctx->remove_cb = remove_cb; 700 TAILQ_INIT(&probe_ctx->init_ctrlrs); 701 } 702 703 int 704 spdk_nvme_probe(const struct spdk_nvme_transport_id *trid, void *cb_ctx, 705 spdk_nvme_probe_cb probe_cb, spdk_nvme_attach_cb attach_cb, 706 spdk_nvme_remove_cb remove_cb) 707 { 708 struct spdk_nvme_transport_id trid_pcie; 709 struct spdk_nvme_probe_ctx *probe_ctx; 710 711 if (trid == NULL) { 712 memset(&trid_pcie, 0, sizeof(trid_pcie)); 713 spdk_nvme_trid_populate_transport(&trid_pcie, SPDK_NVME_TRANSPORT_PCIE); 714 trid = &trid_pcie; 715 } 716 717 probe_ctx = spdk_nvme_probe_async(trid, cb_ctx, probe_cb, 718 attach_cb, remove_cb); 719 if (!probe_ctx) { 720 SPDK_ERRLOG("Create probe context failed\n"); 721 return -1; 722 } 723 724 /* 725 * Keep going even if one or more nvme_attach() calls failed, 726 * but maintain the value of rc to signal errors when we return. 727 */ 728 return nvme_init_controllers(probe_ctx); 729 } 730 731 static bool 732 nvme_connect_probe_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid, 733 struct spdk_nvme_ctrlr_opts *opts) 734 { 735 struct spdk_nvme_ctrlr_opts *requested_opts = cb_ctx; 736 737 assert(requested_opts); 738 memcpy(opts, requested_opts, sizeof(*opts)); 739 740 return true; 741 } 742 743 static void 744 nvme_ctrlr_opts_init(struct spdk_nvme_ctrlr_opts *opts, 745 const struct spdk_nvme_ctrlr_opts *opts_user, 746 size_t opts_size_user) 747 { 748 assert(opts); 749 assert(opts_user); 750 751 spdk_nvme_ctrlr_get_default_ctrlr_opts(opts, opts_size_user); 752 753 #define FIELD_OK(field) \ 754 offsetof(struct spdk_nvme_ctrlr_opts, field) + sizeof(opts->field) <= (opts->opts_size) 755 756 if (FIELD_OK(num_io_queues)) { 757 opts->num_io_queues = opts_user->num_io_queues; 758 } 759 760 if (FIELD_OK(use_cmb_sqs)) { 761 opts->use_cmb_sqs = opts_user->use_cmb_sqs; 762 } 763 764 if (FIELD_OK(no_shn_notification)) { 765 opts->no_shn_notification = opts_user->no_shn_notification; 766 } 767 768 if (FIELD_OK(arb_mechanism)) { 769 opts->arb_mechanism = opts_user->arb_mechanism; 770 } 771 772 if (FIELD_OK(arbitration_burst)) { 773 opts->arbitration_burst = opts_user->arbitration_burst; 774 } 775 776 if (FIELD_OK(low_priority_weight)) { 777 opts->low_priority_weight = opts_user->low_priority_weight; 778 } 779 780 if (FIELD_OK(medium_priority_weight)) { 781 opts->medium_priority_weight = opts_user->medium_priority_weight; 782 } 783 784 if (FIELD_OK(high_priority_weight)) { 785 opts->high_priority_weight = opts_user->high_priority_weight; 786 } 787 788 if (FIELD_OK(keep_alive_timeout_ms)) { 789 opts->keep_alive_timeout_ms = opts_user->keep_alive_timeout_ms; 790 } 791 792 if (FIELD_OK(transport_retry_count)) { 793 opts->transport_retry_count = opts_user->transport_retry_count; 794 } 795 796 if (FIELD_OK(io_queue_size)) { 797 opts->io_queue_size = opts_user->io_queue_size; 798 } 799 800 if (FIELD_OK(hostnqn)) { 801 memcpy(opts->hostnqn, opts_user->hostnqn, sizeof(opts_user->hostnqn)); 802 } 803 804 if (FIELD_OK(io_queue_requests)) { 805 opts->io_queue_requests = opts_user->io_queue_requests; 806 } 807 808 if (FIELD_OK(src_addr)) { 809 memcpy(opts->src_addr, opts_user->src_addr, sizeof(opts_user->src_addr)); 810 } 811 812 if (FIELD_OK(src_svcid)) { 813 memcpy(opts->src_svcid, opts_user->src_svcid, sizeof(opts_user->src_svcid)); 814 } 815 816 if (FIELD_OK(host_id)) { 817 memcpy(opts->host_id, opts_user->host_id, sizeof(opts_user->host_id)); 818 } 819 if (FIELD_OK(extended_host_id)) { 820 memcpy(opts->extended_host_id, opts_user->extended_host_id, 821 sizeof(opts_user->extended_host_id)); 822 } 823 824 if (FIELD_OK(command_set)) { 825 opts->command_set = opts_user->command_set; 826 } 827 828 if (FIELD_OK(admin_timeout_ms)) { 829 opts->admin_timeout_ms = opts_user->admin_timeout_ms; 830 } 831 832 if (FIELD_OK(header_digest)) { 833 opts->header_digest = opts_user->header_digest; 834 } 835 836 if (FIELD_OK(data_digest)) { 837 opts->data_digest = opts_user->data_digest; 838 } 839 840 if (FIELD_OK(disable_error_logging)) { 841 opts->disable_error_logging = opts_user->disable_error_logging; 842 } 843 844 if (FIELD_OK(transport_ack_timeout)) { 845 opts->transport_ack_timeout = opts_user->transport_ack_timeout; 846 } 847 848 if (FIELD_OK(admin_queue_size)) { 849 opts->admin_queue_size = opts_user->admin_queue_size; 850 } 851 #undef FIELD_OK 852 } 853 854 struct spdk_nvme_ctrlr * 855 spdk_nvme_connect(const struct spdk_nvme_transport_id *trid, 856 const struct spdk_nvme_ctrlr_opts *opts, size_t opts_size) 857 { 858 int rc; 859 struct spdk_nvme_ctrlr *ctrlr = NULL; 860 struct spdk_nvme_probe_ctx *probe_ctx; 861 struct spdk_nvme_ctrlr_opts *opts_local_p = NULL; 862 struct spdk_nvme_ctrlr_opts opts_local; 863 864 if (trid == NULL) { 865 SPDK_ERRLOG("No transport ID specified\n"); 866 return NULL; 867 } 868 869 if (opts) { 870 opts_local_p = &opts_local; 871 nvme_ctrlr_opts_init(opts_local_p, opts, opts_size); 872 } 873 874 probe_ctx = spdk_nvme_connect_async(trid, opts_local_p, NULL); 875 if (!probe_ctx) { 876 SPDK_ERRLOG("Create probe context failed\n"); 877 return NULL; 878 } 879 880 rc = nvme_init_controllers(probe_ctx); 881 if (rc != 0) { 882 return NULL; 883 } 884 885 ctrlr = nvme_get_ctrlr_by_trid(trid); 886 887 return ctrlr; 888 } 889 890 void 891 spdk_nvme_trid_populate_transport(struct spdk_nvme_transport_id *trid, 892 enum spdk_nvme_transport_type trtype) 893 { 894 const char *trstring = ""; 895 896 trid->trtype = trtype; 897 switch (trtype) { 898 case SPDK_NVME_TRANSPORT_FC: 899 trstring = SPDK_NVME_TRANSPORT_NAME_FC; 900 break; 901 case SPDK_NVME_TRANSPORT_PCIE: 902 trstring = SPDK_NVME_TRANSPORT_NAME_PCIE; 903 break; 904 case SPDK_NVME_TRANSPORT_RDMA: 905 trstring = SPDK_NVME_TRANSPORT_NAME_RDMA; 906 break; 907 case SPDK_NVME_TRANSPORT_TCP: 908 trstring = SPDK_NVME_TRANSPORT_NAME_TCP; 909 break; 910 case SPDK_NVME_TRANSPORT_CUSTOM: 911 trstring = SPDK_NVME_TRANSPORT_NAME_CUSTOM; 912 break; 913 default: 914 SPDK_ERRLOG("no available transports\n"); 915 assert(0); 916 return; 917 } 918 snprintf(trid->trstring, SPDK_NVMF_TRSTRING_MAX_LEN, "%s", trstring); 919 } 920 921 int 922 spdk_nvme_transport_id_populate_trstring(struct spdk_nvme_transport_id *trid, const char *trstring) 923 { 924 int len, i, rc; 925 926 if (trstring == NULL) { 927 return -EINVAL; 928 } 929 930 len = strnlen(trstring, SPDK_NVMF_TRSTRING_MAX_LEN); 931 if (len == SPDK_NVMF_TRSTRING_MAX_LEN) { 932 return -EINVAL; 933 } 934 935 rc = snprintf(trid->trstring, SPDK_NVMF_TRSTRING_MAX_LEN, "%s", trstring); 936 if (rc < 0) { 937 return rc; 938 } 939 940 /* cast official trstring to uppercase version of input. */ 941 for (i = 0; i < len; i++) { 942 trid->trstring[i] = toupper(trid->trstring[i]); 943 } 944 return 0; 945 } 946 947 int 948 spdk_nvme_transport_id_parse_trtype(enum spdk_nvme_transport_type *trtype, const char *str) 949 { 950 if (trtype == NULL || str == NULL) { 951 return -EINVAL; 952 } 953 954 if (strcasecmp(str, "PCIe") == 0) { 955 *trtype = SPDK_NVME_TRANSPORT_PCIE; 956 } else if (strcasecmp(str, "RDMA") == 0) { 957 *trtype = SPDK_NVME_TRANSPORT_RDMA; 958 } else if (strcasecmp(str, "FC") == 0) { 959 *trtype = SPDK_NVME_TRANSPORT_FC; 960 } else if (strcasecmp(str, "TCP") == 0) { 961 *trtype = SPDK_NVME_TRANSPORT_TCP; 962 } else { 963 *trtype = SPDK_NVME_TRANSPORT_CUSTOM; 964 } 965 return 0; 966 } 967 968 const char * 969 spdk_nvme_transport_id_trtype_str(enum spdk_nvme_transport_type trtype) 970 { 971 switch (trtype) { 972 case SPDK_NVME_TRANSPORT_PCIE: 973 return "PCIe"; 974 case SPDK_NVME_TRANSPORT_RDMA: 975 return "RDMA"; 976 case SPDK_NVME_TRANSPORT_FC: 977 return "FC"; 978 case SPDK_NVME_TRANSPORT_TCP: 979 return "TCP"; 980 case SPDK_NVME_TRANSPORT_CUSTOM: 981 return "CUSTOM"; 982 default: 983 return NULL; 984 } 985 } 986 987 int 988 spdk_nvme_transport_id_parse_adrfam(enum spdk_nvmf_adrfam *adrfam, const char *str) 989 { 990 if (adrfam == NULL || str == NULL) { 991 return -EINVAL; 992 } 993 994 if (strcasecmp(str, "IPv4") == 0) { 995 *adrfam = SPDK_NVMF_ADRFAM_IPV4; 996 } else if (strcasecmp(str, "IPv6") == 0) { 997 *adrfam = SPDK_NVMF_ADRFAM_IPV6; 998 } else if (strcasecmp(str, "IB") == 0) { 999 *adrfam = SPDK_NVMF_ADRFAM_IB; 1000 } else if (strcasecmp(str, "FC") == 0) { 1001 *adrfam = SPDK_NVMF_ADRFAM_FC; 1002 } else { 1003 return -ENOENT; 1004 } 1005 return 0; 1006 } 1007 1008 const char * 1009 spdk_nvme_transport_id_adrfam_str(enum spdk_nvmf_adrfam adrfam) 1010 { 1011 switch (adrfam) { 1012 case SPDK_NVMF_ADRFAM_IPV4: 1013 return "IPv4"; 1014 case SPDK_NVMF_ADRFAM_IPV6: 1015 return "IPv6"; 1016 case SPDK_NVMF_ADRFAM_IB: 1017 return "IB"; 1018 case SPDK_NVMF_ADRFAM_FC: 1019 return "FC"; 1020 default: 1021 return NULL; 1022 } 1023 } 1024 1025 static size_t 1026 parse_next_key(const char **str, char *key, char *val, size_t key_buf_size, size_t val_buf_size) 1027 { 1028 1029 const char *sep, *sep1; 1030 const char *whitespace = " \t\n"; 1031 size_t key_len, val_len; 1032 1033 *str += strspn(*str, whitespace); 1034 1035 sep = strchr(*str, ':'); 1036 if (!sep) { 1037 sep = strchr(*str, '='); 1038 if (!sep) { 1039 SPDK_ERRLOG("Key without ':' or '=' separator\n"); 1040 return 0; 1041 } 1042 } else { 1043 sep1 = strchr(*str, '='); 1044 if ((sep1 != NULL) && (sep1 < sep)) { 1045 sep = sep1; 1046 } 1047 } 1048 1049 key_len = sep - *str; 1050 if (key_len >= key_buf_size) { 1051 SPDK_ERRLOG("Key length %zu greater than maximum allowed %zu\n", 1052 key_len, key_buf_size - 1); 1053 return 0; 1054 } 1055 1056 memcpy(key, *str, key_len); 1057 key[key_len] = '\0'; 1058 1059 *str += key_len + 1; /* Skip key: */ 1060 val_len = strcspn(*str, whitespace); 1061 if (val_len == 0) { 1062 SPDK_ERRLOG("Key without value\n"); 1063 return 0; 1064 } 1065 1066 if (val_len >= val_buf_size) { 1067 SPDK_ERRLOG("Value length %zu greater than maximum allowed %zu\n", 1068 val_len, val_buf_size - 1); 1069 return 0; 1070 } 1071 1072 memcpy(val, *str, val_len); 1073 val[val_len] = '\0'; 1074 1075 *str += val_len; 1076 1077 return val_len; 1078 } 1079 1080 int 1081 spdk_nvme_transport_id_parse(struct spdk_nvme_transport_id *trid, const char *str) 1082 { 1083 size_t val_len; 1084 char key[32]; 1085 char val[1024]; 1086 1087 if (trid == NULL || str == NULL) { 1088 return -EINVAL; 1089 } 1090 1091 while (*str != '\0') { 1092 1093 val_len = parse_next_key(&str, key, val, sizeof(key), sizeof(val)); 1094 1095 if (val_len == 0) { 1096 SPDK_ERRLOG("Failed to parse transport ID\n"); 1097 return -EINVAL; 1098 } 1099 1100 if (strcasecmp(key, "trtype") == 0) { 1101 if (spdk_nvme_transport_id_populate_trstring(trid, val) != 0) { 1102 SPDK_ERRLOG("invalid transport '%s'\n", val); 1103 return -EINVAL; 1104 } 1105 if (spdk_nvme_transport_id_parse_trtype(&trid->trtype, val) != 0) { 1106 SPDK_ERRLOG("Unknown trtype '%s'\n", val); 1107 return -EINVAL; 1108 } 1109 } else if (strcasecmp(key, "adrfam") == 0) { 1110 if (spdk_nvme_transport_id_parse_adrfam(&trid->adrfam, val) != 0) { 1111 SPDK_ERRLOG("Unknown adrfam '%s'\n", val); 1112 return -EINVAL; 1113 } 1114 } else if (strcasecmp(key, "traddr") == 0) { 1115 if (val_len > SPDK_NVMF_TRADDR_MAX_LEN) { 1116 SPDK_ERRLOG("traddr length %zu greater than maximum allowed %u\n", 1117 val_len, SPDK_NVMF_TRADDR_MAX_LEN); 1118 return -EINVAL; 1119 } 1120 memcpy(trid->traddr, val, val_len + 1); 1121 } else if (strcasecmp(key, "trsvcid") == 0) { 1122 if (val_len > SPDK_NVMF_TRSVCID_MAX_LEN) { 1123 SPDK_ERRLOG("trsvcid length %zu greater than maximum allowed %u\n", 1124 val_len, SPDK_NVMF_TRSVCID_MAX_LEN); 1125 return -EINVAL; 1126 } 1127 memcpy(trid->trsvcid, val, val_len + 1); 1128 } else if (strcasecmp(key, "priority") == 0) { 1129 if (val_len > SPDK_NVMF_PRIORITY_MAX_LEN) { 1130 SPDK_ERRLOG("priority length %zu greater than maximum allowed %u\n", 1131 val_len, SPDK_NVMF_PRIORITY_MAX_LEN); 1132 return -EINVAL; 1133 } 1134 trid->priority = spdk_strtol(val, 10); 1135 } else if (strcasecmp(key, "subnqn") == 0) { 1136 if (val_len > SPDK_NVMF_NQN_MAX_LEN) { 1137 SPDK_ERRLOG("subnqn length %zu greater than maximum allowed %u\n", 1138 val_len, SPDK_NVMF_NQN_MAX_LEN); 1139 return -EINVAL; 1140 } 1141 memcpy(trid->subnqn, val, val_len + 1); 1142 } else if (strcasecmp(key, "hostaddr") == 0) { 1143 continue; 1144 } else if (strcasecmp(key, "hostsvcid") == 0) { 1145 continue; 1146 } else if (strcasecmp(key, "ns") == 0) { 1147 /* 1148 * Special case. The namespace id parameter may 1149 * optionally be passed in the transport id string 1150 * for an SPDK application (e.g. nvme/perf) 1151 * and additionally parsed therein to limit 1152 * targeting a specific namespace. For this 1153 * scenario, just silently ignore this key 1154 * rather than letting it default to logging 1155 * it as an invalid key. 1156 */ 1157 continue; 1158 } else if (strcasecmp(key, "alt_traddr") == 0) { 1159 /* 1160 * Used by applications for enabling transport ID failover. 1161 * Please see the case above for more information on custom parameters. 1162 */ 1163 continue; 1164 } else { 1165 SPDK_ERRLOG("Unknown transport ID key '%s'\n", key); 1166 } 1167 } 1168 1169 return 0; 1170 } 1171 1172 int 1173 spdk_nvme_host_id_parse(struct spdk_nvme_host_id *hostid, const char *str) 1174 { 1175 1176 size_t key_size = 32; 1177 size_t val_size = 1024; 1178 size_t val_len; 1179 char key[key_size]; 1180 char val[val_size]; 1181 1182 if (hostid == NULL || str == NULL) { 1183 return -EINVAL; 1184 } 1185 1186 while (*str != '\0') { 1187 1188 val_len = parse_next_key(&str, key, val, key_size, val_size); 1189 1190 if (val_len == 0) { 1191 SPDK_ERRLOG("Failed to parse host ID\n"); 1192 return val_len; 1193 } 1194 1195 /* Ignore the rest of the options from the transport ID. */ 1196 if (strcasecmp(key, "trtype") == 0) { 1197 continue; 1198 } else if (strcasecmp(key, "adrfam") == 0) { 1199 continue; 1200 } else if (strcasecmp(key, "traddr") == 0) { 1201 continue; 1202 } else if (strcasecmp(key, "trsvcid") == 0) { 1203 continue; 1204 } else if (strcasecmp(key, "subnqn") == 0) { 1205 continue; 1206 } else if (strcasecmp(key, "priority") == 0) { 1207 continue; 1208 } else if (strcasecmp(key, "ns") == 0) { 1209 continue; 1210 } else if (strcasecmp(key, "hostaddr") == 0) { 1211 if (val_len > SPDK_NVMF_TRADDR_MAX_LEN) { 1212 SPDK_ERRLOG("hostaddr length %zu greater than maximum allowed %u\n", 1213 val_len, SPDK_NVMF_TRADDR_MAX_LEN); 1214 return -EINVAL; 1215 } 1216 memcpy(hostid->hostaddr, val, val_len + 1); 1217 1218 } else if (strcasecmp(key, "hostsvcid") == 0) { 1219 if (val_len > SPDK_NVMF_TRSVCID_MAX_LEN) { 1220 SPDK_ERRLOG("trsvcid length %zu greater than maximum allowed %u\n", 1221 val_len, SPDK_NVMF_TRSVCID_MAX_LEN); 1222 return -EINVAL; 1223 } 1224 memcpy(hostid->hostsvcid, val, val_len + 1); 1225 } else { 1226 SPDK_ERRLOG("Unknown transport ID key '%s'\n", key); 1227 } 1228 } 1229 1230 return 0; 1231 } 1232 1233 static int 1234 cmp_int(int a, int b) 1235 { 1236 return a - b; 1237 } 1238 1239 int 1240 spdk_nvme_transport_id_compare(const struct spdk_nvme_transport_id *trid1, 1241 const struct spdk_nvme_transport_id *trid2) 1242 { 1243 int cmp; 1244 1245 if (trid1->trtype == SPDK_NVME_TRANSPORT_CUSTOM) { 1246 cmp = strcasecmp(trid1->trstring, trid2->trstring); 1247 } else { 1248 cmp = cmp_int(trid1->trtype, trid2->trtype); 1249 } 1250 1251 if (cmp) { 1252 return cmp; 1253 } 1254 1255 if (trid1->trtype == SPDK_NVME_TRANSPORT_PCIE) { 1256 struct spdk_pci_addr pci_addr1 = {}; 1257 struct spdk_pci_addr pci_addr2 = {}; 1258 1259 /* Normalize PCI addresses before comparing */ 1260 if (spdk_pci_addr_parse(&pci_addr1, trid1->traddr) < 0 || 1261 spdk_pci_addr_parse(&pci_addr2, trid2->traddr) < 0) { 1262 return -1; 1263 } 1264 1265 /* PCIe transport ID only uses trtype and traddr */ 1266 return spdk_pci_addr_compare(&pci_addr1, &pci_addr2); 1267 } 1268 1269 cmp = strcasecmp(trid1->traddr, trid2->traddr); 1270 if (cmp) { 1271 return cmp; 1272 } 1273 1274 cmp = cmp_int(trid1->adrfam, trid2->adrfam); 1275 if (cmp) { 1276 return cmp; 1277 } 1278 1279 cmp = strcasecmp(trid1->trsvcid, trid2->trsvcid); 1280 if (cmp) { 1281 return cmp; 1282 } 1283 1284 cmp = strcmp(trid1->subnqn, trid2->subnqn); 1285 if (cmp) { 1286 return cmp; 1287 } 1288 1289 return 0; 1290 } 1291 1292 int 1293 spdk_nvme_prchk_flags_parse(uint32_t *prchk_flags, const char *str) 1294 { 1295 size_t val_len; 1296 char key[32]; 1297 char val[1024]; 1298 1299 if (prchk_flags == NULL || str == NULL) { 1300 return -EINVAL; 1301 } 1302 1303 while (*str != '\0') { 1304 val_len = parse_next_key(&str, key, val, sizeof(key), sizeof(val)); 1305 1306 if (val_len == 0) { 1307 SPDK_ERRLOG("Failed to parse prchk\n"); 1308 return -EINVAL; 1309 } 1310 1311 if (strcasecmp(key, "prchk") == 0) { 1312 if (strcasestr(val, "reftag") != NULL) { 1313 *prchk_flags |= SPDK_NVME_IO_FLAGS_PRCHK_REFTAG; 1314 } 1315 if (strcasestr(val, "guard") != NULL) { 1316 *prchk_flags |= SPDK_NVME_IO_FLAGS_PRCHK_GUARD; 1317 } 1318 } else { 1319 SPDK_ERRLOG("Unknown key '%s'\n", key); 1320 return -EINVAL; 1321 } 1322 } 1323 1324 return 0; 1325 } 1326 1327 const char * 1328 spdk_nvme_prchk_flags_str(uint32_t prchk_flags) 1329 { 1330 if (prchk_flags & SPDK_NVME_IO_FLAGS_PRCHK_REFTAG) { 1331 if (prchk_flags & SPDK_NVME_IO_FLAGS_PRCHK_GUARD) { 1332 return "prchk:reftag|guard"; 1333 } else { 1334 return "prchk:reftag"; 1335 } 1336 } else { 1337 if (prchk_flags & SPDK_NVME_IO_FLAGS_PRCHK_GUARD) { 1338 return "prchk:guard"; 1339 } else { 1340 return NULL; 1341 } 1342 } 1343 } 1344 1345 struct spdk_nvme_probe_ctx * 1346 spdk_nvme_probe_async(const struct spdk_nvme_transport_id *trid, 1347 void *cb_ctx, 1348 spdk_nvme_probe_cb probe_cb, 1349 spdk_nvme_attach_cb attach_cb, 1350 spdk_nvme_remove_cb remove_cb) 1351 { 1352 int rc; 1353 struct spdk_nvme_probe_ctx *probe_ctx; 1354 1355 rc = nvme_driver_init(); 1356 if (rc != 0) { 1357 return NULL; 1358 } 1359 1360 probe_ctx = calloc(1, sizeof(*probe_ctx)); 1361 if (!probe_ctx) { 1362 return NULL; 1363 } 1364 1365 nvme_probe_ctx_init(probe_ctx, trid, cb_ctx, probe_cb, attach_cb, remove_cb); 1366 rc = nvme_probe_internal(probe_ctx, false); 1367 if (rc != 0) { 1368 free(probe_ctx); 1369 return NULL; 1370 } 1371 1372 return probe_ctx; 1373 } 1374 1375 int 1376 spdk_nvme_probe_poll_async(struct spdk_nvme_probe_ctx *probe_ctx) 1377 { 1378 int rc = 0; 1379 struct spdk_nvme_ctrlr *ctrlr, *ctrlr_tmp; 1380 1381 if (!spdk_process_is_primary() && probe_ctx->trid.trtype == SPDK_NVME_TRANSPORT_PCIE) { 1382 free(probe_ctx); 1383 return 0; 1384 } 1385 1386 TAILQ_FOREACH_SAFE(ctrlr, &probe_ctx->init_ctrlrs, tailq, ctrlr_tmp) { 1387 rc = nvme_ctrlr_poll_internal(ctrlr, probe_ctx); 1388 if (rc != 0) { 1389 rc = -EIO; 1390 break; 1391 } 1392 } 1393 1394 if (rc != 0 || TAILQ_EMPTY(&probe_ctx->init_ctrlrs)) { 1395 nvme_robust_mutex_lock(&g_spdk_nvme_driver->lock); 1396 g_spdk_nvme_driver->initialized = true; 1397 nvme_robust_mutex_unlock(&g_spdk_nvme_driver->lock); 1398 free(probe_ctx); 1399 return rc; 1400 } 1401 1402 return -EAGAIN; 1403 } 1404 1405 struct spdk_nvme_probe_ctx * 1406 spdk_nvme_connect_async(const struct spdk_nvme_transport_id *trid, 1407 const struct spdk_nvme_ctrlr_opts *opts, 1408 spdk_nvme_attach_cb attach_cb) 1409 { 1410 int rc; 1411 spdk_nvme_probe_cb probe_cb = NULL; 1412 struct spdk_nvme_probe_ctx *probe_ctx; 1413 1414 rc = nvme_driver_init(); 1415 if (rc != 0) { 1416 return NULL; 1417 } 1418 1419 probe_ctx = calloc(1, sizeof(*probe_ctx)); 1420 if (!probe_ctx) { 1421 return NULL; 1422 } 1423 1424 if (opts) { 1425 probe_cb = nvme_connect_probe_cb; 1426 } 1427 1428 nvme_probe_ctx_init(probe_ctx, trid, (void *)opts, probe_cb, attach_cb, NULL); 1429 rc = nvme_probe_internal(probe_ctx, true); 1430 if (rc != 0) { 1431 free(probe_ctx); 1432 return NULL; 1433 } 1434 1435 return probe_ctx; 1436 } 1437 1438 SPDK_LOG_REGISTER_COMPONENT("nvme", SPDK_LOG_NVME) 1439