1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2016 Intel Corporation. All rights reserved. 3 * Copyright (c) 2018-2019, 2021 Mellanox Technologies LTD. All rights reserved. 4 * Copyright (c) 2021, 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 5 */ 6 7 #include "spdk/stdinc.h" 8 9 #include "spdk/bdev.h" 10 #include "spdk/bit_array.h" 11 #include "spdk/thread.h" 12 #include "spdk/nvmf.h" 13 #include "spdk/endian.h" 14 #include "spdk/string.h" 15 #include "spdk/log.h" 16 #include "spdk_internal/usdt.h" 17 18 #include "nvmf_internal.h" 19 #include "transport.h" 20 21 SPDK_LOG_REGISTER_COMPONENT(nvmf) 22 23 #define SPDK_NVMF_DEFAULT_MAX_SUBSYSTEMS 1024 24 25 static TAILQ_HEAD(, spdk_nvmf_tgt) g_nvmf_tgts = TAILQ_HEAD_INITIALIZER(g_nvmf_tgts); 26 27 typedef void (*nvmf_qpair_disconnect_cpl)(void *ctx, int status); 28 29 /* supplied to a single call to nvmf_qpair_disconnect */ 30 struct nvmf_qpair_disconnect_ctx { 31 struct spdk_nvmf_qpair *qpair; 32 struct spdk_nvmf_ctrlr *ctrlr; 33 nvmf_qpair_disconnect_cb cb_fn; 34 struct spdk_thread *thread; 35 void *ctx; 36 uint16_t qid; 37 }; 38 39 /* 40 * There are several times when we need to iterate through the list of all qpairs and selectively delete them. 41 * In order to do this sequentially without overlap, we must provide a context to recover the next qpair from 42 * to enable calling nvmf_qpair_disconnect on the next desired qpair. 43 */ 44 struct nvmf_qpair_disconnect_many_ctx { 45 struct spdk_nvmf_subsystem *subsystem; 46 struct spdk_nvmf_poll_group *group; 47 spdk_nvmf_poll_group_mod_done cpl_fn; 48 void *cpl_ctx; 49 }; 50 51 static struct spdk_nvmf_referral * 52 nvmf_tgt_find_referral(struct spdk_nvmf_tgt *tgt, 53 const struct spdk_nvme_transport_id *trid) 54 { 55 struct spdk_nvmf_referral *referral; 56 57 TAILQ_FOREACH(referral, &tgt->referrals, link) { 58 if (spdk_nvme_transport_id_compare(&referral->trid, trid) == 0) { 59 return referral; 60 } 61 } 62 63 return NULL; 64 } 65 66 int 67 spdk_nvmf_tgt_add_referral(struct spdk_nvmf_tgt *tgt, 68 const struct spdk_nvmf_referral_opts *uopts) 69 { 70 struct spdk_nvmf_referral *referral; 71 struct spdk_nvmf_referral_opts opts = {}; 72 struct spdk_nvme_transport_id *trid = &opts.trid; 73 74 memcpy(&opts, uopts, spdk_min(uopts->size, sizeof(opts))); 75 if (trid->subnqn[0] == '\0') { 76 snprintf(trid->subnqn, sizeof(trid->subnqn), "%s", SPDK_NVMF_DISCOVERY_NQN); 77 } 78 79 if (!nvmf_nqn_is_valid(trid->subnqn)) { 80 SPDK_ERRLOG("Invalid subsystem NQN\n"); 81 return -EINVAL; 82 } 83 84 /* If the entry already exists, just ignore it. */ 85 if (nvmf_tgt_find_referral(tgt, trid)) { 86 return 0; 87 } 88 89 referral = calloc(1, sizeof(*referral)); 90 if (!referral) { 91 SPDK_ERRLOG("Failed to allocate memory for a referral\n"); 92 return -ENOMEM; 93 } 94 95 referral->entry.subtype = nvmf_nqn_is_discovery(trid->subnqn) ? 96 SPDK_NVMF_SUBTYPE_DISCOVERY : 97 SPDK_NVMF_SUBTYPE_NVME; 98 referral->entry.treq.secure_channel = opts.secure_channel ? 99 SPDK_NVMF_TREQ_SECURE_CHANNEL_REQUIRED : 100 SPDK_NVMF_TREQ_SECURE_CHANNEL_NOT_REQUIRED; 101 referral->entry.cntlid = 0xffff; 102 referral->entry.trtype = trid->trtype; 103 referral->entry.adrfam = trid->adrfam; 104 memcpy(&referral->trid, trid, sizeof(struct spdk_nvme_transport_id)); 105 spdk_strcpy_pad(referral->entry.subnqn, trid->subnqn, sizeof(trid->subnqn), '\0'); 106 spdk_strcpy_pad(referral->entry.trsvcid, trid->trsvcid, sizeof(referral->entry.trsvcid), ' '); 107 spdk_strcpy_pad(referral->entry.traddr, trid->traddr, sizeof(referral->entry.traddr), ' '); 108 109 TAILQ_INSERT_HEAD(&tgt->referrals, referral, link); 110 nvmf_update_discovery_log(tgt, NULL); 111 112 return 0; 113 } 114 115 int 116 spdk_nvmf_tgt_remove_referral(struct spdk_nvmf_tgt *tgt, 117 const struct spdk_nvmf_referral_opts *uopts) 118 { 119 struct spdk_nvmf_referral *referral; 120 struct spdk_nvmf_referral_opts opts = {}; 121 struct spdk_nvme_transport_id *trid = &opts.trid; 122 123 memcpy(&opts, uopts, spdk_min(uopts->size, sizeof(opts))); 124 if (trid->subnqn[0] == '\0') { 125 snprintf(trid->subnqn, sizeof(trid->subnqn), "%s", SPDK_NVMF_DISCOVERY_NQN); 126 } 127 128 referral = nvmf_tgt_find_referral(tgt, &opts.trid); 129 if (referral == NULL) { 130 return -ENOENT; 131 } 132 133 TAILQ_REMOVE(&tgt->referrals, referral, link); 134 nvmf_update_discovery_log(tgt, NULL); 135 136 free(referral); 137 138 return 0; 139 } 140 141 static void 142 nvmf_qpair_set_state(struct spdk_nvmf_qpair *qpair, 143 enum spdk_nvmf_qpair_state state) 144 { 145 assert(qpair != NULL); 146 assert(qpair->group->thread == spdk_get_thread()); 147 148 qpair->state = state; 149 } 150 151 static int 152 nvmf_poll_group_poll(void *ctx) 153 { 154 struct spdk_nvmf_poll_group *group = ctx; 155 int rc; 156 int count = 0; 157 struct spdk_nvmf_transport_poll_group *tgroup; 158 159 TAILQ_FOREACH(tgroup, &group->tgroups, link) { 160 rc = nvmf_transport_poll_group_poll(tgroup); 161 if (rc < 0) { 162 return SPDK_POLLER_BUSY; 163 } 164 count += rc; 165 } 166 167 return count > 0 ? SPDK_POLLER_BUSY : SPDK_POLLER_IDLE; 168 } 169 170 /* 171 * Reset and clean up the poll group (I/O channel code will actually free the 172 * group). 173 */ 174 static void 175 nvmf_tgt_cleanup_poll_group(struct spdk_nvmf_poll_group *group) 176 { 177 struct spdk_nvmf_transport_poll_group *tgroup, *tmp; 178 struct spdk_nvmf_subsystem_poll_group *sgroup; 179 uint32_t sid, nsid; 180 181 TAILQ_FOREACH_SAFE(tgroup, &group->tgroups, link, tmp) { 182 TAILQ_REMOVE(&group->tgroups, tgroup, link); 183 nvmf_transport_poll_group_destroy(tgroup); 184 } 185 186 for (sid = 0; sid < group->num_sgroups; sid++) { 187 sgroup = &group->sgroups[sid]; 188 189 assert(sgroup != NULL); 190 191 for (nsid = 0; nsid < sgroup->num_ns; nsid++) { 192 if (sgroup->ns_info[nsid].channel) { 193 spdk_put_io_channel(sgroup->ns_info[nsid].channel); 194 sgroup->ns_info[nsid].channel = NULL; 195 } 196 } 197 198 free(sgroup->ns_info); 199 } 200 201 free(group->sgroups); 202 203 spdk_poller_unregister(&group->poller); 204 205 if (group->destroy_cb_fn) { 206 group->destroy_cb_fn(group->destroy_cb_arg, 0); 207 } 208 } 209 210 /* 211 * Callback to unregister a poll group from the target, and clean up its state. 212 */ 213 static void 214 nvmf_tgt_destroy_poll_group(void *io_device, void *ctx_buf) 215 { 216 struct spdk_nvmf_tgt *tgt = io_device; 217 struct spdk_nvmf_poll_group *group = ctx_buf; 218 219 SPDK_DTRACE_PROBE1_TICKS(nvmf_destroy_poll_group, spdk_thread_get_id(group->thread)); 220 221 pthread_mutex_lock(&tgt->mutex); 222 TAILQ_REMOVE(&tgt->poll_groups, group, link); 223 tgt->num_poll_groups--; 224 pthread_mutex_unlock(&tgt->mutex); 225 226 assert(!(tgt->state == NVMF_TGT_PAUSING || tgt->state == NVMF_TGT_RESUMING)); 227 nvmf_tgt_cleanup_poll_group(group); 228 } 229 230 static int 231 nvmf_poll_group_add_transport(struct spdk_nvmf_poll_group *group, 232 struct spdk_nvmf_transport *transport) 233 { 234 struct spdk_nvmf_transport_poll_group *tgroup; 235 236 TAILQ_FOREACH(tgroup, &group->tgroups, link) { 237 if (tgroup->transport == transport) { 238 /* Transport already in the poll group */ 239 return 0; 240 } 241 } 242 243 tgroup = nvmf_transport_poll_group_create(transport, group); 244 if (!tgroup) { 245 SPDK_ERRLOG("Unable to create poll group for transport\n"); 246 return -1; 247 } 248 SPDK_DTRACE_PROBE2_TICKS(nvmf_transport_poll_group_create, transport, 249 spdk_thread_get_id(group->thread)); 250 251 tgroup->group = group; 252 TAILQ_INSERT_TAIL(&group->tgroups, tgroup, link); 253 254 return 0; 255 } 256 257 static int 258 nvmf_tgt_create_poll_group(void *io_device, void *ctx_buf) 259 { 260 struct spdk_nvmf_tgt *tgt = io_device; 261 struct spdk_nvmf_poll_group *group = ctx_buf; 262 struct spdk_nvmf_transport *transport; 263 struct spdk_nvmf_subsystem *subsystem; 264 struct spdk_thread *thread = spdk_get_thread(); 265 int rc; 266 267 group->tgt = tgt; 268 TAILQ_INIT(&group->tgroups); 269 TAILQ_INIT(&group->qpairs); 270 group->thread = thread; 271 pthread_mutex_init(&group->mutex, NULL); 272 273 group->poller = SPDK_POLLER_REGISTER(nvmf_poll_group_poll, group, 0); 274 275 SPDK_DTRACE_PROBE1_TICKS(nvmf_create_poll_group, spdk_thread_get_id(thread)); 276 277 TAILQ_FOREACH(transport, &tgt->transports, link) { 278 rc = nvmf_poll_group_add_transport(group, transport); 279 if (rc != 0) { 280 nvmf_tgt_cleanup_poll_group(group); 281 return rc; 282 } 283 } 284 285 group->num_sgroups = tgt->max_subsystems; 286 group->sgroups = calloc(tgt->max_subsystems, sizeof(struct spdk_nvmf_subsystem_poll_group)); 287 if (!group->sgroups) { 288 nvmf_tgt_cleanup_poll_group(group); 289 return -ENOMEM; 290 } 291 292 for (subsystem = spdk_nvmf_subsystem_get_first(tgt); 293 subsystem != NULL; 294 subsystem = spdk_nvmf_subsystem_get_next(subsystem)) { 295 if (nvmf_poll_group_add_subsystem(group, subsystem, NULL, NULL) != 0) { 296 nvmf_tgt_cleanup_poll_group(group); 297 return -1; 298 } 299 } 300 301 pthread_mutex_lock(&tgt->mutex); 302 tgt->num_poll_groups++; 303 TAILQ_INSERT_TAIL(&tgt->poll_groups, group, link); 304 pthread_mutex_unlock(&tgt->mutex); 305 306 return 0; 307 } 308 309 static void 310 _nvmf_tgt_disconnect_qpairs(void *ctx) 311 { 312 struct spdk_nvmf_qpair *qpair, *qpair_tmp; 313 struct nvmf_qpair_disconnect_many_ctx *qpair_ctx = ctx; 314 struct spdk_nvmf_poll_group *group = qpair_ctx->group; 315 struct spdk_io_channel *ch; 316 int rc; 317 318 TAILQ_FOREACH_SAFE(qpair, &group->qpairs, link, qpair_tmp) { 319 rc = spdk_nvmf_qpair_disconnect(qpair, NULL, NULL); 320 if (rc && rc != -EINPROGRESS) { 321 break; 322 } 323 } 324 325 if (TAILQ_EMPTY(&group->qpairs)) { 326 /* When the refcount from the channels reaches 0, nvmf_tgt_destroy_poll_group will be called. */ 327 ch = spdk_io_channel_from_ctx(group); 328 spdk_put_io_channel(ch); 329 free(qpair_ctx); 330 return; 331 } 332 333 /* Some qpairs are in process of being disconnected. Send a message and try to remove them again */ 334 spdk_thread_send_msg(spdk_get_thread(), _nvmf_tgt_disconnect_qpairs, ctx); 335 } 336 337 static void 338 nvmf_tgt_destroy_poll_group_qpairs(struct spdk_nvmf_poll_group *group) 339 { 340 struct nvmf_qpair_disconnect_many_ctx *ctx; 341 342 SPDK_DTRACE_PROBE1_TICKS(nvmf_destroy_poll_group_qpairs, spdk_thread_get_id(group->thread)); 343 344 ctx = calloc(1, sizeof(struct nvmf_qpair_disconnect_many_ctx)); 345 if (!ctx) { 346 SPDK_ERRLOG("Failed to allocate memory for destroy poll group ctx\n"); 347 return; 348 } 349 350 ctx->group = group; 351 _nvmf_tgt_disconnect_qpairs(ctx); 352 } 353 354 struct spdk_nvmf_tgt * 355 spdk_nvmf_tgt_create(struct spdk_nvmf_target_opts *opts) 356 { 357 struct spdk_nvmf_tgt *tgt, *tmp_tgt; 358 359 if (strnlen(opts->name, NVMF_TGT_NAME_MAX_LENGTH) == NVMF_TGT_NAME_MAX_LENGTH) { 360 SPDK_ERRLOG("Provided target name exceeds the max length of %u.\n", NVMF_TGT_NAME_MAX_LENGTH); 361 return NULL; 362 } 363 364 TAILQ_FOREACH(tmp_tgt, &g_nvmf_tgts, link) { 365 if (!strncmp(opts->name, tmp_tgt->name, NVMF_TGT_NAME_MAX_LENGTH)) { 366 SPDK_ERRLOG("Provided target name must be unique.\n"); 367 return NULL; 368 } 369 } 370 371 tgt = calloc(1, sizeof(*tgt)); 372 if (!tgt) { 373 return NULL; 374 } 375 376 snprintf(tgt->name, NVMF_TGT_NAME_MAX_LENGTH, "%s", opts->name); 377 378 if (!opts || !opts->max_subsystems) { 379 tgt->max_subsystems = SPDK_NVMF_DEFAULT_MAX_SUBSYSTEMS; 380 } else { 381 tgt->max_subsystems = opts->max_subsystems; 382 } 383 384 if (!opts) { 385 tgt->crdt[0] = 0; 386 tgt->crdt[1] = 0; 387 tgt->crdt[2] = 0; 388 } else { 389 tgt->crdt[0] = opts->crdt[0]; 390 tgt->crdt[1] = opts->crdt[1]; 391 tgt->crdt[2] = opts->crdt[2]; 392 } 393 394 if (!opts) { 395 tgt->discovery_filter = SPDK_NVMF_TGT_DISCOVERY_MATCH_ANY; 396 } else { 397 tgt->discovery_filter = opts->discovery_filter; 398 } 399 400 tgt->discovery_genctr = 0; 401 TAILQ_INIT(&tgt->transports); 402 TAILQ_INIT(&tgt->poll_groups); 403 TAILQ_INIT(&tgt->referrals); 404 tgt->num_poll_groups = 0; 405 406 tgt->subsystem_ids = spdk_bit_array_create(tgt->max_subsystems); 407 if (tgt->subsystem_ids == NULL) { 408 free(tgt); 409 return NULL; 410 } 411 412 RB_INIT(&tgt->subsystems); 413 414 pthread_mutex_init(&tgt->mutex, NULL); 415 416 spdk_io_device_register(tgt, 417 nvmf_tgt_create_poll_group, 418 nvmf_tgt_destroy_poll_group, 419 sizeof(struct spdk_nvmf_poll_group), 420 tgt->name); 421 422 tgt->state = NVMF_TGT_RUNNING; 423 424 TAILQ_INSERT_HEAD(&g_nvmf_tgts, tgt, link); 425 426 return tgt; 427 } 428 429 static void 430 _nvmf_tgt_destroy_next_transport(void *ctx) 431 { 432 struct spdk_nvmf_tgt *tgt = ctx; 433 struct spdk_nvmf_transport *transport; 434 435 if (!TAILQ_EMPTY(&tgt->transports)) { 436 transport = TAILQ_FIRST(&tgt->transports); 437 TAILQ_REMOVE(&tgt->transports, transport, link); 438 spdk_nvmf_transport_destroy(transport, _nvmf_tgt_destroy_next_transport, tgt); 439 } else { 440 spdk_nvmf_tgt_destroy_done_fn *destroy_cb_fn = tgt->destroy_cb_fn; 441 void *destroy_cb_arg = tgt->destroy_cb_arg; 442 443 pthread_mutex_destroy(&tgt->mutex); 444 free(tgt); 445 446 if (destroy_cb_fn) { 447 destroy_cb_fn(destroy_cb_arg, 0); 448 } 449 } 450 } 451 452 static void 453 nvmf_tgt_destroy_cb(void *io_device) 454 { 455 struct spdk_nvmf_tgt *tgt = io_device; 456 struct spdk_nvmf_subsystem *subsystem, *subsystem_next; 457 int rc; 458 struct spdk_nvmf_referral *referral; 459 460 while ((referral = TAILQ_FIRST(&tgt->referrals))) { 461 TAILQ_REMOVE(&tgt->referrals, referral, link); 462 free(referral); 463 } 464 465 /* We will be freeing subsystems in this loop, so we always need to get the next one 466 * ahead of time, since we can't call get_next() on a subsystem that's been freed. 467 */ 468 for (subsystem = spdk_nvmf_subsystem_get_first(tgt), 469 subsystem_next = spdk_nvmf_subsystem_get_next(subsystem); 470 subsystem != NULL; 471 subsystem = subsystem_next, 472 subsystem_next = spdk_nvmf_subsystem_get_next(subsystem_next)) { 473 nvmf_subsystem_remove_all_listeners(subsystem, true); 474 475 rc = spdk_nvmf_subsystem_destroy(subsystem, nvmf_tgt_destroy_cb, tgt); 476 if (rc) { 477 if (rc == -EINPROGRESS) { 478 /* If rc is -EINPROGRESS, nvmf_tgt_destroy_cb will be called again when subsystem #i 479 * is destroyed, nvmf_tgt_destroy_cb will continue to destroy other subsystems if any */ 480 return; 481 } else { 482 SPDK_ERRLOG("Failed to destroy subsystem %s, rc %d\n", subsystem->subnqn, rc); 483 } 484 } 485 } 486 spdk_bit_array_free(&tgt->subsystem_ids); 487 _nvmf_tgt_destroy_next_transport(tgt); 488 } 489 490 void 491 spdk_nvmf_tgt_destroy(struct spdk_nvmf_tgt *tgt, 492 spdk_nvmf_tgt_destroy_done_fn cb_fn, 493 void *cb_arg) 494 { 495 assert(!(tgt->state == NVMF_TGT_PAUSING || tgt->state == NVMF_TGT_RESUMING)); 496 497 tgt->destroy_cb_fn = cb_fn; 498 tgt->destroy_cb_arg = cb_arg; 499 500 TAILQ_REMOVE(&g_nvmf_tgts, tgt, link); 501 502 spdk_io_device_unregister(tgt, nvmf_tgt_destroy_cb); 503 } 504 505 const char * 506 spdk_nvmf_tgt_get_name(struct spdk_nvmf_tgt *tgt) 507 { 508 return tgt->name; 509 } 510 511 struct spdk_nvmf_tgt * 512 spdk_nvmf_get_tgt(const char *name) 513 { 514 struct spdk_nvmf_tgt *tgt; 515 uint32_t num_targets = 0; 516 517 TAILQ_FOREACH(tgt, &g_nvmf_tgts, link) { 518 if (name) { 519 if (!strncmp(tgt->name, name, NVMF_TGT_NAME_MAX_LENGTH)) { 520 return tgt; 521 } 522 } 523 num_targets++; 524 } 525 526 /* 527 * special case. If there is only one target and 528 * no name was specified, return the only available 529 * target. If there is more than one target, name must 530 * be specified. 531 */ 532 if (!name && num_targets == 1) { 533 return TAILQ_FIRST(&g_nvmf_tgts); 534 } 535 536 return NULL; 537 } 538 539 struct spdk_nvmf_tgt * 540 spdk_nvmf_get_first_tgt(void) 541 { 542 return TAILQ_FIRST(&g_nvmf_tgts); 543 } 544 545 struct spdk_nvmf_tgt * 546 spdk_nvmf_get_next_tgt(struct spdk_nvmf_tgt *prev) 547 { 548 return TAILQ_NEXT(prev, link); 549 } 550 551 static void 552 nvmf_write_nvme_subsystem_config(struct spdk_json_write_ctx *w, 553 struct spdk_nvmf_subsystem *subsystem) 554 { 555 struct spdk_nvmf_host *host; 556 struct spdk_nvmf_ns *ns; 557 struct spdk_nvmf_ns_opts ns_opts; 558 uint32_t max_namespaces; 559 struct spdk_nvmf_transport *transport; 560 561 assert(spdk_nvmf_subsystem_get_type(subsystem) == SPDK_NVMF_SUBTYPE_NVME); 562 563 /* { */ 564 spdk_json_write_object_begin(w); 565 spdk_json_write_named_string(w, "method", "nvmf_create_subsystem"); 566 567 /* "params" : { */ 568 spdk_json_write_named_object_begin(w, "params"); 569 spdk_json_write_named_string(w, "nqn", spdk_nvmf_subsystem_get_nqn(subsystem)); 570 spdk_json_write_named_bool(w, "allow_any_host", spdk_nvmf_subsystem_get_allow_any_host(subsystem)); 571 spdk_json_write_named_string(w, "serial_number", spdk_nvmf_subsystem_get_sn(subsystem)); 572 spdk_json_write_named_string(w, "model_number", spdk_nvmf_subsystem_get_mn(subsystem)); 573 574 max_namespaces = spdk_nvmf_subsystem_get_max_namespaces(subsystem); 575 if (max_namespaces != 0) { 576 spdk_json_write_named_uint32(w, "max_namespaces", max_namespaces); 577 } 578 579 spdk_json_write_named_uint32(w, "min_cntlid", spdk_nvmf_subsystem_get_min_cntlid(subsystem)); 580 spdk_json_write_named_uint32(w, "max_cntlid", spdk_nvmf_subsystem_get_max_cntlid(subsystem)); 581 spdk_json_write_named_bool(w, "ana_reporting", spdk_nvmf_subsystem_get_ana_reporting(subsystem)); 582 583 /* } "params" */ 584 spdk_json_write_object_end(w); 585 586 /* } */ 587 spdk_json_write_object_end(w); 588 589 for (host = spdk_nvmf_subsystem_get_first_host(subsystem); host != NULL; 590 host = spdk_nvmf_subsystem_get_next_host(subsystem, host)) { 591 592 spdk_json_write_object_begin(w); 593 spdk_json_write_named_string(w, "method", "nvmf_subsystem_add_host"); 594 595 /* "params" : { */ 596 spdk_json_write_named_object_begin(w, "params"); 597 598 spdk_json_write_named_string(w, "nqn", spdk_nvmf_subsystem_get_nqn(subsystem)); 599 spdk_json_write_named_string(w, "host", spdk_nvmf_host_get_nqn(host)); 600 601 TAILQ_FOREACH(transport, &subsystem->tgt->transports, link) { 602 if (transport->ops->subsystem_dump_host != NULL) { 603 transport->ops->subsystem_dump_host(transport, subsystem, host->nqn, w); 604 } 605 } 606 607 /* } "params" */ 608 spdk_json_write_object_end(w); 609 610 /* } */ 611 spdk_json_write_object_end(w); 612 } 613 614 for (ns = spdk_nvmf_subsystem_get_first_ns(subsystem); ns != NULL; 615 ns = spdk_nvmf_subsystem_get_next_ns(subsystem, ns)) { 616 spdk_nvmf_ns_get_opts(ns, &ns_opts, sizeof(ns_opts)); 617 618 spdk_json_write_object_begin(w); 619 spdk_json_write_named_string(w, "method", "nvmf_subsystem_add_ns"); 620 621 /* "params" : { */ 622 spdk_json_write_named_object_begin(w, "params"); 623 624 spdk_json_write_named_string(w, "nqn", spdk_nvmf_subsystem_get_nqn(subsystem)); 625 626 /* "namespace" : { */ 627 spdk_json_write_named_object_begin(w, "namespace"); 628 629 spdk_json_write_named_uint32(w, "nsid", spdk_nvmf_ns_get_id(ns)); 630 spdk_json_write_named_string(w, "bdev_name", spdk_bdev_get_name(spdk_nvmf_ns_get_bdev(ns))); 631 632 if (ns->ptpl_file != NULL) { 633 spdk_json_write_named_string(w, "ptpl_file", ns->ptpl_file); 634 } 635 636 if (!spdk_mem_all_zero(ns_opts.nguid, sizeof(ns_opts.nguid))) { 637 SPDK_STATIC_ASSERT(sizeof(ns_opts.nguid) == sizeof(uint64_t) * 2, "size mismatch"); 638 spdk_json_write_named_string_fmt(w, "nguid", "%016"PRIX64"%016"PRIX64, from_be64(&ns_opts.nguid[0]), 639 from_be64(&ns_opts.nguid[8])); 640 } 641 642 if (!spdk_mem_all_zero(ns_opts.eui64, sizeof(ns_opts.eui64))) { 643 SPDK_STATIC_ASSERT(sizeof(ns_opts.eui64) == sizeof(uint64_t), "size mismatch"); 644 spdk_json_write_named_string_fmt(w, "eui64", "%016"PRIX64, from_be64(&ns_opts.eui64)); 645 } 646 647 if (!spdk_uuid_is_null(&ns_opts.uuid)) { 648 spdk_json_write_named_uuid(w, "uuid", &ns_opts.uuid); 649 } 650 651 if (spdk_nvmf_subsystem_get_ana_reporting(subsystem)) { 652 spdk_json_write_named_uint32(w, "anagrpid", ns_opts.anagrpid); 653 } 654 655 spdk_json_write_named_bool(w, "no_auto_visible", !ns->always_visible); 656 657 /* "namespace" */ 658 spdk_json_write_object_end(w); 659 660 /* } "params" */ 661 spdk_json_write_object_end(w); 662 663 /* } */ 664 spdk_json_write_object_end(w); 665 } 666 } 667 668 static void 669 nvmf_write_subsystem_config_json(struct spdk_json_write_ctx *w, 670 struct spdk_nvmf_subsystem *subsystem) 671 { 672 struct spdk_nvmf_subsystem_listener *listener; 673 struct spdk_nvmf_transport *transport; 674 const struct spdk_nvme_transport_id *trid; 675 676 if (spdk_nvmf_subsystem_get_type(subsystem) == SPDK_NVMF_SUBTYPE_NVME) { 677 nvmf_write_nvme_subsystem_config(w, subsystem); 678 } 679 680 for (listener = spdk_nvmf_subsystem_get_first_listener(subsystem); listener != NULL; 681 listener = spdk_nvmf_subsystem_get_next_listener(subsystem, listener)) { 682 transport = listener->transport; 683 trid = spdk_nvmf_subsystem_listener_get_trid(listener); 684 685 spdk_json_write_object_begin(w); 686 spdk_json_write_named_string(w, "method", "nvmf_subsystem_add_listener"); 687 688 /* "params" : { */ 689 spdk_json_write_named_object_begin(w, "params"); 690 691 spdk_json_write_named_string(w, "nqn", spdk_nvmf_subsystem_get_nqn(subsystem)); 692 693 spdk_json_write_named_object_begin(w, "listen_address"); 694 nvmf_transport_listen_dump_trid(trid, w); 695 spdk_json_write_object_end(w); 696 if (transport->ops->listen_dump_opts) { 697 transport->ops->listen_dump_opts(transport, trid, w); 698 } 699 700 spdk_json_write_named_bool(w, "secure_channel", listener->opts.secure_channel); 701 702 /* } "params" */ 703 spdk_json_write_object_end(w); 704 705 /* } */ 706 spdk_json_write_object_end(w); 707 } 708 } 709 710 void 711 spdk_nvmf_tgt_write_config_json(struct spdk_json_write_ctx *w, struct spdk_nvmf_tgt *tgt) 712 { 713 struct spdk_nvmf_subsystem *subsystem; 714 struct spdk_nvmf_transport *transport; 715 struct spdk_nvmf_referral *referral; 716 717 spdk_json_write_object_begin(w); 718 spdk_json_write_named_string(w, "method", "nvmf_set_max_subsystems"); 719 720 spdk_json_write_named_object_begin(w, "params"); 721 spdk_json_write_named_uint32(w, "max_subsystems", tgt->max_subsystems); 722 spdk_json_write_object_end(w); 723 724 spdk_json_write_object_end(w); 725 726 spdk_json_write_object_begin(w); 727 spdk_json_write_named_string(w, "method", "nvmf_set_crdt"); 728 spdk_json_write_named_object_begin(w, "params"); 729 spdk_json_write_named_uint32(w, "crdt1", tgt->crdt[0]); 730 spdk_json_write_named_uint32(w, "crdt2", tgt->crdt[1]); 731 spdk_json_write_named_uint32(w, "crdt3", tgt->crdt[2]); 732 spdk_json_write_object_end(w); 733 spdk_json_write_object_end(w); 734 735 /* write transports */ 736 TAILQ_FOREACH(transport, &tgt->transports, link) { 737 spdk_json_write_object_begin(w); 738 spdk_json_write_named_string(w, "method", "nvmf_create_transport"); 739 nvmf_transport_dump_opts(transport, w, true); 740 spdk_json_write_object_end(w); 741 } 742 743 TAILQ_FOREACH(referral, &tgt->referrals, link) { 744 spdk_json_write_object_begin(w); 745 spdk_json_write_named_string(w, "method", "nvmf_discovery_add_referral"); 746 747 spdk_json_write_named_object_begin(w, "params"); 748 spdk_json_write_named_object_begin(w, "address"); 749 nvmf_transport_listen_dump_trid(&referral->trid, w); 750 spdk_json_write_object_end(w); 751 spdk_json_write_named_bool(w, "secure_channel", 752 referral->entry.treq.secure_channel == 753 SPDK_NVMF_TREQ_SECURE_CHANNEL_REQUIRED); 754 spdk_json_write_named_string(w, "subnqn", referral->trid.subnqn); 755 spdk_json_write_object_end(w); 756 757 spdk_json_write_object_end(w); 758 } 759 760 subsystem = spdk_nvmf_subsystem_get_first(tgt); 761 while (subsystem) { 762 nvmf_write_subsystem_config_json(w, subsystem); 763 subsystem = spdk_nvmf_subsystem_get_next(subsystem); 764 } 765 } 766 767 static void 768 nvmf_listen_opts_copy(struct spdk_nvmf_listen_opts *opts, 769 const struct spdk_nvmf_listen_opts *opts_src, size_t opts_size) 770 { 771 assert(opts); 772 assert(opts_src); 773 774 opts->opts_size = opts_size; 775 776 #define SET_FIELD(field) \ 777 if (offsetof(struct spdk_nvmf_listen_opts, field) + sizeof(opts->field) <= opts_size) { \ 778 opts->field = opts_src->field; \ 779 } \ 780 781 SET_FIELD(transport_specific); 782 SET_FIELD(secure_channel); 783 SET_FIELD(ana_state); 784 #undef SET_FIELD 785 786 /* Do not remove this statement, you should always update this statement when you adding a new field, 787 * and do not forget to add the SET_FIELD statement for your added field. */ 788 SPDK_STATIC_ASSERT(sizeof(struct spdk_nvmf_listen_opts) == 24, "Incorrect size"); 789 } 790 791 void 792 spdk_nvmf_listen_opts_init(struct spdk_nvmf_listen_opts *opts, size_t opts_size) 793 { 794 struct spdk_nvmf_listen_opts opts_local = {}; 795 796 /* local version of opts should have defaults set here */ 797 opts_local.ana_state = SPDK_NVME_ANA_OPTIMIZED_STATE; 798 nvmf_listen_opts_copy(opts, &opts_local, opts_size); 799 } 800 801 int 802 spdk_nvmf_tgt_listen_ext(struct spdk_nvmf_tgt *tgt, const struct spdk_nvme_transport_id *trid, 803 struct spdk_nvmf_listen_opts *opts) 804 { 805 struct spdk_nvmf_transport *transport; 806 int rc; 807 struct spdk_nvmf_listen_opts opts_local = {}; 808 809 if (!opts) { 810 SPDK_ERRLOG("opts should not be NULL\n"); 811 return -EINVAL; 812 } 813 814 if (!opts->opts_size) { 815 SPDK_ERRLOG("The opts_size in opts structure should not be zero\n"); 816 return -EINVAL; 817 } 818 819 transport = spdk_nvmf_tgt_get_transport(tgt, trid->trstring); 820 if (!transport) { 821 SPDK_ERRLOG("Unable to find %s transport. The transport must be created first also make sure it is properly registered.\n", 822 trid->trstring); 823 return -EINVAL; 824 } 825 826 nvmf_listen_opts_copy(&opts_local, opts, opts->opts_size); 827 rc = spdk_nvmf_transport_listen(transport, trid, &opts_local); 828 if (rc < 0) { 829 SPDK_ERRLOG("Unable to listen on address '%s'\n", trid->traddr); 830 } 831 832 return rc; 833 } 834 835 int 836 spdk_nvmf_tgt_stop_listen(struct spdk_nvmf_tgt *tgt, 837 struct spdk_nvme_transport_id *trid) 838 { 839 struct spdk_nvmf_transport *transport; 840 int rc; 841 842 transport = spdk_nvmf_tgt_get_transport(tgt, trid->trstring); 843 if (!transport) { 844 SPDK_ERRLOG("Unable to find %s transport. The transport must be created first also make sure it is properly registered.\n", 845 trid->trstring); 846 return -EINVAL; 847 } 848 849 rc = spdk_nvmf_transport_stop_listen(transport, trid); 850 if (rc < 0) { 851 SPDK_ERRLOG("Failed to stop listening on address '%s'\n", trid->traddr); 852 return rc; 853 } 854 return 0; 855 } 856 857 struct spdk_nvmf_tgt_add_transport_ctx { 858 struct spdk_nvmf_tgt *tgt; 859 struct spdk_nvmf_transport *transport; 860 spdk_nvmf_tgt_add_transport_done_fn cb_fn; 861 void *cb_arg; 862 int status; 863 }; 864 865 static void 866 _nvmf_tgt_remove_transport_done(struct spdk_io_channel_iter *i, int status) 867 { 868 struct spdk_nvmf_tgt_add_transport_ctx *ctx = spdk_io_channel_iter_get_ctx(i); 869 870 ctx->cb_fn(ctx->cb_arg, ctx->status); 871 free(ctx); 872 } 873 874 static void 875 _nvmf_tgt_remove_transport(struct spdk_io_channel_iter *i) 876 { 877 struct spdk_nvmf_tgt_add_transport_ctx *ctx = spdk_io_channel_iter_get_ctx(i); 878 struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i); 879 struct spdk_nvmf_poll_group *group = spdk_io_channel_get_ctx(ch); 880 struct spdk_nvmf_transport_poll_group *tgroup, *tmp; 881 882 TAILQ_FOREACH_SAFE(tgroup, &group->tgroups, link, tmp) { 883 if (tgroup->transport == ctx->transport) { 884 TAILQ_REMOVE(&group->tgroups, tgroup, link); 885 nvmf_transport_poll_group_destroy(tgroup); 886 } 887 } 888 889 spdk_for_each_channel_continue(i, 0); 890 } 891 892 static void 893 _nvmf_tgt_add_transport_done(struct spdk_io_channel_iter *i, int status) 894 { 895 struct spdk_nvmf_tgt_add_transport_ctx *ctx = spdk_io_channel_iter_get_ctx(i); 896 897 if (status) { 898 ctx->status = status; 899 spdk_for_each_channel(ctx->tgt, 900 _nvmf_tgt_remove_transport, 901 ctx, 902 _nvmf_tgt_remove_transport_done); 903 return; 904 } 905 906 ctx->transport->tgt = ctx->tgt; 907 TAILQ_INSERT_TAIL(&ctx->tgt->transports, ctx->transport, link); 908 ctx->cb_fn(ctx->cb_arg, status); 909 free(ctx); 910 } 911 912 static void 913 _nvmf_tgt_add_transport(struct spdk_io_channel_iter *i) 914 { 915 struct spdk_nvmf_tgt_add_transport_ctx *ctx = spdk_io_channel_iter_get_ctx(i); 916 struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i); 917 struct spdk_nvmf_poll_group *group = spdk_io_channel_get_ctx(ch); 918 int rc; 919 920 rc = nvmf_poll_group_add_transport(group, ctx->transport); 921 spdk_for_each_channel_continue(i, rc); 922 } 923 924 void 925 spdk_nvmf_tgt_add_transport(struct spdk_nvmf_tgt *tgt, 926 struct spdk_nvmf_transport *transport, 927 spdk_nvmf_tgt_add_transport_done_fn cb_fn, 928 void *cb_arg) 929 { 930 struct spdk_nvmf_tgt_add_transport_ctx *ctx; 931 932 SPDK_DTRACE_PROBE2_TICKS(nvmf_tgt_add_transport, transport, tgt->name); 933 934 if (spdk_nvmf_tgt_get_transport(tgt, transport->ops->name)) { 935 cb_fn(cb_arg, -EEXIST); 936 return; /* transport already created */ 937 } 938 939 ctx = calloc(1, sizeof(*ctx)); 940 if (!ctx) { 941 cb_fn(cb_arg, -ENOMEM); 942 return; 943 } 944 945 ctx->tgt = tgt; 946 ctx->transport = transport; 947 ctx->cb_fn = cb_fn; 948 ctx->cb_arg = cb_arg; 949 950 spdk_for_each_channel(tgt, 951 _nvmf_tgt_add_transport, 952 ctx, 953 _nvmf_tgt_add_transport_done); 954 } 955 956 struct nvmf_tgt_pause_ctx { 957 struct spdk_nvmf_tgt *tgt; 958 spdk_nvmf_tgt_pause_polling_cb_fn cb_fn; 959 void *cb_arg; 960 }; 961 962 static void 963 _nvmf_tgt_pause_polling_done(struct spdk_io_channel_iter *i, int status) 964 { 965 struct nvmf_tgt_pause_ctx *ctx = spdk_io_channel_iter_get_ctx(i); 966 967 ctx->tgt->state = NVMF_TGT_PAUSED; 968 969 ctx->cb_fn(ctx->cb_arg, status); 970 free(ctx); 971 } 972 973 static void 974 _nvmf_tgt_pause_polling(struct spdk_io_channel_iter *i) 975 { 976 struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i); 977 struct spdk_nvmf_poll_group *group = spdk_io_channel_get_ctx(ch); 978 979 spdk_poller_unregister(&group->poller); 980 981 spdk_for_each_channel_continue(i, 0); 982 } 983 984 int 985 spdk_nvmf_tgt_pause_polling(struct spdk_nvmf_tgt *tgt, spdk_nvmf_tgt_pause_polling_cb_fn cb_fn, 986 void *cb_arg) 987 { 988 struct nvmf_tgt_pause_ctx *ctx; 989 990 SPDK_DTRACE_PROBE2_TICKS(nvmf_tgt_pause_polling, tgt, tgt->name); 991 992 switch (tgt->state) { 993 case NVMF_TGT_PAUSING: 994 case NVMF_TGT_RESUMING: 995 return -EBUSY; 996 case NVMF_TGT_RUNNING: 997 break; 998 default: 999 return -EINVAL; 1000 } 1001 1002 ctx = calloc(1, sizeof(*ctx)); 1003 if (!ctx) { 1004 return -ENOMEM; 1005 } 1006 1007 1008 tgt->state = NVMF_TGT_PAUSING; 1009 1010 ctx->tgt = tgt; 1011 ctx->cb_fn = cb_fn; 1012 ctx->cb_arg = cb_arg; 1013 1014 spdk_for_each_channel(tgt, 1015 _nvmf_tgt_pause_polling, 1016 ctx, 1017 _nvmf_tgt_pause_polling_done); 1018 return 0; 1019 } 1020 1021 static void 1022 _nvmf_tgt_resume_polling_done(struct spdk_io_channel_iter *i, int status) 1023 { 1024 struct nvmf_tgt_pause_ctx *ctx = spdk_io_channel_iter_get_ctx(i); 1025 1026 ctx->tgt->state = NVMF_TGT_RUNNING; 1027 1028 ctx->cb_fn(ctx->cb_arg, status); 1029 free(ctx); 1030 } 1031 1032 static void 1033 _nvmf_tgt_resume_polling(struct spdk_io_channel_iter *i) 1034 { 1035 struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i); 1036 struct spdk_nvmf_poll_group *group = spdk_io_channel_get_ctx(ch); 1037 1038 assert(group->poller == NULL); 1039 group->poller = SPDK_POLLER_REGISTER(nvmf_poll_group_poll, group, 0); 1040 1041 spdk_for_each_channel_continue(i, 0); 1042 } 1043 1044 int 1045 spdk_nvmf_tgt_resume_polling(struct spdk_nvmf_tgt *tgt, spdk_nvmf_tgt_resume_polling_cb_fn cb_fn, 1046 void *cb_arg) 1047 { 1048 struct nvmf_tgt_pause_ctx *ctx; 1049 1050 SPDK_DTRACE_PROBE2_TICKS(nvmf_tgt_resume_polling, tgt, tgt->name); 1051 1052 switch (tgt->state) { 1053 case NVMF_TGT_PAUSING: 1054 case NVMF_TGT_RESUMING: 1055 return -EBUSY; 1056 case NVMF_TGT_PAUSED: 1057 break; 1058 default: 1059 return -EINVAL; 1060 } 1061 1062 ctx = calloc(1, sizeof(*ctx)); 1063 if (!ctx) { 1064 return -ENOMEM; 1065 } 1066 1067 tgt->state = NVMF_TGT_RESUMING; 1068 1069 ctx->tgt = tgt; 1070 ctx->cb_fn = cb_fn; 1071 ctx->cb_arg = cb_arg; 1072 1073 spdk_for_each_channel(tgt, 1074 _nvmf_tgt_resume_polling, 1075 ctx, 1076 _nvmf_tgt_resume_polling_done); 1077 return 0; 1078 } 1079 1080 struct spdk_nvmf_subsystem * 1081 spdk_nvmf_tgt_find_subsystem(struct spdk_nvmf_tgt *tgt, const char *subnqn) 1082 { 1083 struct spdk_nvmf_subsystem subsystem; 1084 1085 if (!subnqn) { 1086 return NULL; 1087 } 1088 1089 /* Ensure that subnqn is null terminated */ 1090 if (!memchr(subnqn, '\0', SPDK_NVMF_NQN_MAX_LEN + 1)) { 1091 SPDK_ERRLOG("Connect SUBNQN is not null terminated\n"); 1092 return NULL; 1093 } 1094 1095 snprintf(subsystem.subnqn, sizeof(subsystem.subnqn), "%s", subnqn); 1096 return RB_FIND(subsystem_tree, &tgt->subsystems, &subsystem); 1097 } 1098 1099 struct spdk_nvmf_transport * 1100 spdk_nvmf_tgt_get_transport(struct spdk_nvmf_tgt *tgt, const char *transport_name) 1101 { 1102 struct spdk_nvmf_transport *transport; 1103 1104 TAILQ_FOREACH(transport, &tgt->transports, link) { 1105 if (!strncasecmp(transport->ops->name, transport_name, SPDK_NVMF_TRSTRING_MAX_LEN)) { 1106 return transport; 1107 } 1108 } 1109 return NULL; 1110 } 1111 1112 struct nvmf_new_qpair_ctx { 1113 struct spdk_nvmf_qpair *qpair; 1114 struct spdk_nvmf_poll_group *group; 1115 }; 1116 1117 static void 1118 _nvmf_poll_group_add(void *_ctx) 1119 { 1120 struct nvmf_new_qpair_ctx *ctx = _ctx; 1121 struct spdk_nvmf_qpair *qpair = ctx->qpair; 1122 struct spdk_nvmf_poll_group *group = ctx->group; 1123 1124 free(_ctx); 1125 1126 if (spdk_nvmf_poll_group_add(group, qpair) != 0) { 1127 SPDK_ERRLOG("Unable to add the qpair to a poll group.\n"); 1128 spdk_nvmf_qpair_disconnect(qpair, NULL, NULL); 1129 } 1130 } 1131 1132 void 1133 spdk_nvmf_tgt_new_qpair(struct spdk_nvmf_tgt *tgt, struct spdk_nvmf_qpair *qpair) 1134 { 1135 struct spdk_nvmf_poll_group *group; 1136 struct nvmf_new_qpair_ctx *ctx; 1137 1138 group = spdk_nvmf_get_optimal_poll_group(qpair); 1139 if (group == NULL) { 1140 if (tgt->next_poll_group == NULL) { 1141 tgt->next_poll_group = TAILQ_FIRST(&tgt->poll_groups); 1142 if (tgt->next_poll_group == NULL) { 1143 SPDK_ERRLOG("No poll groups exist.\n"); 1144 spdk_nvmf_qpair_disconnect(qpair, NULL, NULL); 1145 return; 1146 } 1147 } 1148 group = tgt->next_poll_group; 1149 tgt->next_poll_group = TAILQ_NEXT(group, link); 1150 } 1151 1152 ctx = calloc(1, sizeof(*ctx)); 1153 if (!ctx) { 1154 SPDK_ERRLOG("Unable to send message to poll group.\n"); 1155 spdk_nvmf_qpair_disconnect(qpair, NULL, NULL); 1156 return; 1157 } 1158 1159 ctx->qpair = qpair; 1160 ctx->group = group; 1161 1162 pthread_mutex_lock(&group->mutex); 1163 group->current_unassociated_qpairs++; 1164 pthread_mutex_unlock(&group->mutex); 1165 1166 spdk_thread_send_msg(group->thread, _nvmf_poll_group_add, ctx); 1167 } 1168 1169 struct spdk_nvmf_poll_group * 1170 spdk_nvmf_poll_group_create(struct spdk_nvmf_tgt *tgt) 1171 { 1172 struct spdk_io_channel *ch; 1173 1174 ch = spdk_get_io_channel(tgt); 1175 if (!ch) { 1176 SPDK_ERRLOG("Unable to get I/O channel for target\n"); 1177 return NULL; 1178 } 1179 1180 return spdk_io_channel_get_ctx(ch); 1181 } 1182 1183 void 1184 spdk_nvmf_poll_group_destroy(struct spdk_nvmf_poll_group *group, 1185 spdk_nvmf_poll_group_destroy_done_fn cb_fn, 1186 void *cb_arg) 1187 { 1188 assert(group->destroy_cb_fn == NULL); 1189 group->destroy_cb_fn = cb_fn; 1190 group->destroy_cb_arg = cb_arg; 1191 1192 /* This function will put the io_channel associated with this poll group */ 1193 nvmf_tgt_destroy_poll_group_qpairs(group); 1194 } 1195 1196 int 1197 spdk_nvmf_poll_group_add(struct spdk_nvmf_poll_group *group, 1198 struct spdk_nvmf_qpair *qpair) 1199 { 1200 int rc = -1; 1201 struct spdk_nvmf_transport_poll_group *tgroup; 1202 1203 TAILQ_INIT(&qpair->outstanding); 1204 qpair->group = group; 1205 qpair->ctrlr = NULL; 1206 qpair->disconnect_started = false; 1207 1208 TAILQ_FOREACH(tgroup, &group->tgroups, link) { 1209 if (tgroup->transport == qpair->transport) { 1210 rc = nvmf_transport_poll_group_add(tgroup, qpair); 1211 break; 1212 } 1213 } 1214 1215 /* We add the qpair to the group only it is successfully added into the tgroup */ 1216 if (rc == 0) { 1217 SPDK_DTRACE_PROBE2_TICKS(nvmf_poll_group_add_qpair, qpair, spdk_thread_get_id(group->thread)); 1218 TAILQ_INSERT_TAIL(&group->qpairs, qpair, link); 1219 nvmf_qpair_set_state(qpair, SPDK_NVMF_QPAIR_ACTIVE); 1220 } 1221 1222 return rc; 1223 } 1224 1225 static void 1226 _nvmf_ctrlr_destruct(void *ctx) 1227 { 1228 struct spdk_nvmf_ctrlr *ctrlr = ctx; 1229 1230 nvmf_ctrlr_destruct(ctrlr); 1231 } 1232 1233 static void 1234 _nvmf_ctrlr_free_from_qpair(void *ctx) 1235 { 1236 struct nvmf_qpair_disconnect_ctx *qpair_ctx = ctx; 1237 struct spdk_nvmf_ctrlr *ctrlr = qpair_ctx->ctrlr; 1238 uint32_t count; 1239 1240 spdk_bit_array_clear(ctrlr->qpair_mask, qpair_ctx->qid); 1241 count = spdk_bit_array_count_set(ctrlr->qpair_mask); 1242 if (count == 0) { 1243 assert(!ctrlr->in_destruct); 1244 SPDK_DEBUGLOG(nvmf, "Last qpair %u, destroy ctrlr 0x%hx\n", qpair_ctx->qid, ctrlr->cntlid); 1245 ctrlr->in_destruct = true; 1246 spdk_thread_send_msg(ctrlr->subsys->thread, _nvmf_ctrlr_destruct, ctrlr); 1247 } 1248 free(qpair_ctx); 1249 } 1250 1251 static void 1252 _nvmf_transport_qpair_fini_complete(void *cb_ctx) 1253 { 1254 struct nvmf_qpair_disconnect_ctx *qpair_ctx = cb_ctx; 1255 struct spdk_nvmf_ctrlr *ctrlr; 1256 /* Store cb args since cb_ctx can be freed in _nvmf_ctrlr_free_from_qpair */ 1257 nvmf_qpair_disconnect_cb cb_fn = qpair_ctx->cb_fn; 1258 void *cb_arg = qpair_ctx->ctx; 1259 struct spdk_thread *cb_thread = qpair_ctx->thread; 1260 1261 ctrlr = qpair_ctx->ctrlr; 1262 SPDK_DEBUGLOG(nvmf, "Finish destroying qid %u\n", qpair_ctx->qid); 1263 1264 if (ctrlr) { 1265 if (qpair_ctx->qid == 0) { 1266 /* Admin qpair is removed, so set the pointer to NULL. 1267 * This operation is safe since we are on ctrlr thread now, admin qpair's thread is the same 1268 * as controller's thread */ 1269 assert(ctrlr->thread == spdk_get_thread()); 1270 ctrlr->admin_qpair = NULL; 1271 } 1272 /* Free qpair id from controller's bit mask and destroy the controller if it is the last qpair */ 1273 if (ctrlr->thread) { 1274 spdk_thread_send_msg(ctrlr->thread, _nvmf_ctrlr_free_from_qpair, qpair_ctx); 1275 } else { 1276 _nvmf_ctrlr_free_from_qpair(qpair_ctx); 1277 } 1278 } else { 1279 free(qpair_ctx); 1280 } 1281 1282 if (cb_fn) { 1283 spdk_thread_send_msg(cb_thread, cb_fn, cb_arg); 1284 } 1285 } 1286 1287 void 1288 spdk_nvmf_poll_group_remove(struct spdk_nvmf_qpair *qpair) 1289 { 1290 struct spdk_nvmf_transport_poll_group *tgroup; 1291 int rc; 1292 1293 SPDK_DTRACE_PROBE2_TICKS(nvmf_poll_group_remove_qpair, qpair, 1294 spdk_thread_get_id(qpair->group->thread)); 1295 nvmf_qpair_set_state(qpair, SPDK_NVMF_QPAIR_ERROR); 1296 1297 /* Find the tgroup and remove the qpair from the tgroup */ 1298 TAILQ_FOREACH(tgroup, &qpair->group->tgroups, link) { 1299 if (tgroup->transport == qpair->transport) { 1300 rc = nvmf_transport_poll_group_remove(tgroup, qpair); 1301 if (rc && (rc != ENOTSUP)) { 1302 SPDK_ERRLOG("Cannot remove qpair=%p from transport group=%p\n", 1303 qpair, tgroup); 1304 } 1305 break; 1306 } 1307 } 1308 1309 TAILQ_REMOVE(&qpair->group->qpairs, qpair, link); 1310 qpair->group = NULL; 1311 } 1312 1313 static void 1314 _nvmf_qpair_sgroup_req_clean(struct spdk_nvmf_subsystem_poll_group *sgroup, 1315 const struct spdk_nvmf_qpair *qpair) 1316 { 1317 struct spdk_nvmf_request *req, *tmp; 1318 TAILQ_FOREACH_SAFE(req, &sgroup->queued, link, tmp) { 1319 if (req->qpair == qpair) { 1320 TAILQ_REMOVE(&sgroup->queued, req, link); 1321 if (nvmf_transport_req_free(req)) { 1322 SPDK_ERRLOG("Transport request free error!\n"); 1323 } 1324 } 1325 } 1326 } 1327 1328 static void 1329 _nvmf_qpair_destroy(void *ctx, int status) 1330 { 1331 struct nvmf_qpair_disconnect_ctx *qpair_ctx = ctx; 1332 struct spdk_nvmf_qpair *qpair = qpair_ctx->qpair; 1333 struct spdk_nvmf_ctrlr *ctrlr = qpair->ctrlr; 1334 struct spdk_nvmf_subsystem_poll_group *sgroup; 1335 uint32_t sid; 1336 1337 assert(qpair->state == SPDK_NVMF_QPAIR_DEACTIVATING); 1338 qpair_ctx->qid = qpair->qid; 1339 1340 if (qpair->connect_received) { 1341 if (0 == qpair->qid) { 1342 assert(qpair->group->stat.current_admin_qpairs > 0); 1343 qpair->group->stat.current_admin_qpairs--; 1344 } else { 1345 assert(qpair->group->stat.current_io_qpairs > 0); 1346 qpair->group->stat.current_io_qpairs--; 1347 } 1348 } else { 1349 pthread_mutex_lock(&qpair->group->mutex); 1350 qpair->group->current_unassociated_qpairs--; 1351 pthread_mutex_unlock(&qpair->group->mutex); 1352 } 1353 1354 if (ctrlr) { 1355 sgroup = &qpair->group->sgroups[ctrlr->subsys->id]; 1356 _nvmf_qpair_sgroup_req_clean(sgroup, qpair); 1357 } else { 1358 for (sid = 0; sid < qpair->group->num_sgroups; sid++) { 1359 sgroup = &qpair->group->sgroups[sid]; 1360 assert(sgroup != NULL); 1361 _nvmf_qpair_sgroup_req_clean(sgroup, qpair); 1362 } 1363 } 1364 1365 qpair_ctx->ctrlr = ctrlr; 1366 spdk_nvmf_poll_group_remove(qpair); 1367 nvmf_transport_qpair_fini(qpair, _nvmf_transport_qpair_fini_complete, qpair_ctx); 1368 } 1369 1370 static void 1371 _nvmf_qpair_disconnect_msg(void *ctx) 1372 { 1373 struct nvmf_qpair_disconnect_ctx *qpair_ctx = ctx; 1374 1375 spdk_nvmf_qpair_disconnect(qpair_ctx->qpair, qpair_ctx->cb_fn, qpair_ctx->ctx); 1376 free(ctx); 1377 } 1378 1379 SPDK_LOG_DEPRECATION_REGISTER(spdk_nvmf_qpair_disconnect, "cb_fn and ctx are deprecated", "v24.05", 1380 0); 1381 1382 int 1383 spdk_nvmf_qpair_disconnect(struct spdk_nvmf_qpair *qpair, nvmf_qpair_disconnect_cb cb_fn, void *ctx) 1384 { 1385 struct spdk_nvmf_poll_group *group = qpair->group; 1386 struct nvmf_qpair_disconnect_ctx *qpair_ctx; 1387 1388 if (__atomic_test_and_set(&qpair->disconnect_started, __ATOMIC_RELAXED)) { 1389 return -EINPROGRESS; 1390 } 1391 1392 if (cb_fn || ctx) { 1393 SPDK_LOG_DEPRECATED(spdk_nvmf_qpair_disconnect); 1394 } 1395 1396 /* If we get a qpair in the uninitialized state, we can just destroy it immediately */ 1397 if (qpair->state == SPDK_NVMF_QPAIR_UNINITIALIZED) { 1398 nvmf_transport_qpair_fini(qpair, NULL, NULL); 1399 if (cb_fn) { 1400 cb_fn(ctx); 1401 } 1402 return 0; 1403 } 1404 1405 assert(group != NULL); 1406 if (spdk_get_thread() != group->thread) { 1407 /* clear the atomic so we can set it on the next call on the proper thread. */ 1408 __atomic_clear(&qpair->disconnect_started, __ATOMIC_RELAXED); 1409 qpair_ctx = calloc(1, sizeof(struct nvmf_qpair_disconnect_ctx)); 1410 if (!qpair_ctx) { 1411 SPDK_ERRLOG("Unable to allocate context for nvmf_qpair_disconnect\n"); 1412 return -ENOMEM; 1413 } 1414 qpair_ctx->qpair = qpair; 1415 qpair_ctx->cb_fn = cb_fn; 1416 qpair_ctx->thread = group->thread; 1417 qpair_ctx->ctx = ctx; 1418 spdk_thread_send_msg(group->thread, _nvmf_qpair_disconnect_msg, qpair_ctx); 1419 return 0; 1420 } 1421 1422 SPDK_DTRACE_PROBE2_TICKS(nvmf_qpair_disconnect, qpair, spdk_thread_get_id(group->thread)); 1423 assert(qpair->state == SPDK_NVMF_QPAIR_ACTIVE); 1424 nvmf_qpair_set_state(qpair, SPDK_NVMF_QPAIR_DEACTIVATING); 1425 1426 qpair_ctx = calloc(1, sizeof(struct nvmf_qpair_disconnect_ctx)); 1427 if (!qpair_ctx) { 1428 SPDK_ERRLOG("Unable to allocate context for nvmf_qpair_disconnect\n"); 1429 return -ENOMEM; 1430 } 1431 1432 qpair_ctx->qpair = qpair; 1433 qpair_ctx->cb_fn = cb_fn; 1434 qpair_ctx->thread = group->thread; 1435 qpair_ctx->ctx = ctx; 1436 1437 /* Check for outstanding I/O */ 1438 if (!TAILQ_EMPTY(&qpair->outstanding)) { 1439 SPDK_DTRACE_PROBE2_TICKS(nvmf_poll_group_drain_qpair, qpair, spdk_thread_get_id(group->thread)); 1440 qpair->state_cb = _nvmf_qpair_destroy; 1441 qpair->state_cb_arg = qpair_ctx; 1442 nvmf_qpair_abort_pending_zcopy_reqs(qpair); 1443 nvmf_qpair_free_aer(qpair); 1444 return 0; 1445 } 1446 1447 _nvmf_qpair_destroy(qpair_ctx, 0); 1448 1449 return 0; 1450 } 1451 1452 int 1453 spdk_nvmf_qpair_get_peer_trid(struct spdk_nvmf_qpair *qpair, 1454 struct spdk_nvme_transport_id *trid) 1455 { 1456 memset(trid, 0, sizeof(*trid)); 1457 return nvmf_transport_qpair_get_peer_trid(qpair, trid); 1458 } 1459 1460 int 1461 spdk_nvmf_qpair_get_local_trid(struct spdk_nvmf_qpair *qpair, 1462 struct spdk_nvme_transport_id *trid) 1463 { 1464 memset(trid, 0, sizeof(*trid)); 1465 return nvmf_transport_qpair_get_local_trid(qpair, trid); 1466 } 1467 1468 int 1469 spdk_nvmf_qpair_get_listen_trid(struct spdk_nvmf_qpair *qpair, 1470 struct spdk_nvme_transport_id *trid) 1471 { 1472 memset(trid, 0, sizeof(*trid)); 1473 return nvmf_transport_qpair_get_listen_trid(qpair, trid); 1474 } 1475 1476 static int 1477 poll_group_update_subsystem(struct spdk_nvmf_poll_group *group, 1478 struct spdk_nvmf_subsystem *subsystem) 1479 { 1480 struct spdk_nvmf_subsystem_poll_group *sgroup; 1481 uint32_t new_num_ns, old_num_ns; 1482 uint32_t i, j; 1483 struct spdk_nvmf_ns *ns; 1484 struct spdk_nvmf_registrant *reg, *tmp; 1485 struct spdk_io_channel *ch; 1486 struct spdk_nvmf_subsystem_pg_ns_info *ns_info; 1487 struct spdk_nvmf_ctrlr *ctrlr; 1488 bool ns_changed; 1489 1490 /* Make sure our poll group has memory for this subsystem allocated */ 1491 if (subsystem->id >= group->num_sgroups) { 1492 return -ENOMEM; 1493 } 1494 1495 sgroup = &group->sgroups[subsystem->id]; 1496 1497 /* Make sure the array of namespace information is the correct size */ 1498 new_num_ns = subsystem->max_nsid; 1499 old_num_ns = sgroup->num_ns; 1500 1501 ns_changed = false; 1502 1503 if (old_num_ns == 0) { 1504 if (new_num_ns > 0) { 1505 /* First allocation */ 1506 sgroup->ns_info = calloc(new_num_ns, sizeof(struct spdk_nvmf_subsystem_pg_ns_info)); 1507 if (!sgroup->ns_info) { 1508 return -ENOMEM; 1509 } 1510 } 1511 } else if (new_num_ns > old_num_ns) { 1512 void *buf; 1513 1514 /* Make the array larger */ 1515 buf = realloc(sgroup->ns_info, new_num_ns * sizeof(struct spdk_nvmf_subsystem_pg_ns_info)); 1516 if (!buf) { 1517 return -ENOMEM; 1518 } 1519 1520 sgroup->ns_info = buf; 1521 1522 /* Null out the new namespace information slots */ 1523 for (i = old_num_ns; i < new_num_ns; i++) { 1524 memset(&sgroup->ns_info[i], 0, sizeof(struct spdk_nvmf_subsystem_pg_ns_info)); 1525 } 1526 } else if (new_num_ns < old_num_ns) { 1527 void *buf; 1528 1529 /* Free the extra I/O channels */ 1530 for (i = new_num_ns; i < old_num_ns; i++) { 1531 ns_info = &sgroup->ns_info[i]; 1532 1533 if (ns_info->channel) { 1534 spdk_put_io_channel(ns_info->channel); 1535 ns_info->channel = NULL; 1536 } 1537 } 1538 1539 /* Make the array smaller */ 1540 if (new_num_ns > 0) { 1541 buf = realloc(sgroup->ns_info, new_num_ns * sizeof(struct spdk_nvmf_subsystem_pg_ns_info)); 1542 if (!buf) { 1543 return -ENOMEM; 1544 } 1545 sgroup->ns_info = buf; 1546 } else { 1547 free(sgroup->ns_info); 1548 sgroup->ns_info = NULL; 1549 } 1550 } 1551 1552 sgroup->num_ns = new_num_ns; 1553 1554 /* Detect bdevs that were added or removed */ 1555 for (i = 0; i < sgroup->num_ns; i++) { 1556 ns = subsystem->ns[i]; 1557 ns_info = &sgroup->ns_info[i]; 1558 ch = ns_info->channel; 1559 1560 if (ns == NULL && ch == NULL) { 1561 /* Both NULL. Leave empty */ 1562 } else if (ns == NULL && ch != NULL) { 1563 /* There was a channel here, but the namespace is gone. */ 1564 ns_changed = true; 1565 spdk_put_io_channel(ch); 1566 ns_info->channel = NULL; 1567 } else if (ns != NULL && ch == NULL) { 1568 /* A namespace appeared but there is no channel yet */ 1569 ns_changed = true; 1570 ch = spdk_bdev_get_io_channel(ns->desc); 1571 if (ch == NULL) { 1572 SPDK_ERRLOG("Could not allocate I/O channel.\n"); 1573 return -ENOMEM; 1574 } 1575 ns_info->channel = ch; 1576 } else if (spdk_uuid_compare(&ns_info->uuid, spdk_bdev_get_uuid(ns->bdev)) != 0) { 1577 /* A namespace was here before, but was replaced by a new one. */ 1578 ns_changed = true; 1579 spdk_put_io_channel(ns_info->channel); 1580 memset(ns_info, 0, sizeof(*ns_info)); 1581 1582 ch = spdk_bdev_get_io_channel(ns->desc); 1583 if (ch == NULL) { 1584 SPDK_ERRLOG("Could not allocate I/O channel.\n"); 1585 return -ENOMEM; 1586 } 1587 ns_info->channel = ch; 1588 } else if (ns_info->num_blocks != spdk_bdev_get_num_blocks(ns->bdev)) { 1589 /* Namespace is still there but size has changed */ 1590 SPDK_DEBUGLOG(nvmf, "Namespace resized: subsystem_id %u," 1591 " nsid %u, pg %p, old %" PRIu64 ", new %" PRIu64 "\n", 1592 subsystem->id, 1593 ns->nsid, 1594 group, 1595 ns_info->num_blocks, 1596 spdk_bdev_get_num_blocks(ns->bdev)); 1597 ns_changed = true; 1598 } 1599 1600 if (ns == NULL) { 1601 memset(ns_info, 0, sizeof(*ns_info)); 1602 } else { 1603 ns_info->uuid = *spdk_bdev_get_uuid(ns->bdev); 1604 ns_info->num_blocks = spdk_bdev_get_num_blocks(ns->bdev); 1605 ns_info->crkey = ns->crkey; 1606 ns_info->rtype = ns->rtype; 1607 if (ns->holder) { 1608 ns_info->holder_id = ns->holder->hostid; 1609 } 1610 1611 memset(&ns_info->reg_hostid, 0, SPDK_NVMF_MAX_NUM_REGISTRANTS * sizeof(struct spdk_uuid)); 1612 j = 0; 1613 TAILQ_FOREACH_SAFE(reg, &ns->registrants, link, tmp) { 1614 if (j >= SPDK_NVMF_MAX_NUM_REGISTRANTS) { 1615 SPDK_ERRLOG("Maximum %u registrants can support.\n", SPDK_NVMF_MAX_NUM_REGISTRANTS); 1616 return -EINVAL; 1617 } 1618 ns_info->reg_hostid[j++] = reg->hostid; 1619 } 1620 } 1621 } 1622 1623 if (ns_changed) { 1624 TAILQ_FOREACH(ctrlr, &subsystem->ctrlrs, link) { 1625 if (ctrlr->thread != spdk_get_thread()) { 1626 continue; 1627 } 1628 /* It is possible that a ctrlr was added but the admin_qpair hasn't been 1629 * assigned yet. 1630 */ 1631 if (!ctrlr->admin_qpair) { 1632 continue; 1633 } 1634 if (ctrlr->admin_qpair->group == group) { 1635 nvmf_ctrlr_async_event_ns_notice(ctrlr); 1636 nvmf_ctrlr_async_event_ana_change_notice(ctrlr); 1637 } 1638 } 1639 } 1640 1641 return 0; 1642 } 1643 1644 int 1645 nvmf_poll_group_update_subsystem(struct spdk_nvmf_poll_group *group, 1646 struct spdk_nvmf_subsystem *subsystem) 1647 { 1648 return poll_group_update_subsystem(group, subsystem); 1649 } 1650 1651 int 1652 nvmf_poll_group_add_subsystem(struct spdk_nvmf_poll_group *group, 1653 struct spdk_nvmf_subsystem *subsystem, 1654 spdk_nvmf_poll_group_mod_done cb_fn, void *cb_arg) 1655 { 1656 int rc = 0; 1657 struct spdk_nvmf_subsystem_poll_group *sgroup = &group->sgroups[subsystem->id]; 1658 uint32_t i; 1659 1660 TAILQ_INIT(&sgroup->queued); 1661 1662 rc = poll_group_update_subsystem(group, subsystem); 1663 if (rc) { 1664 nvmf_poll_group_remove_subsystem(group, subsystem, NULL, NULL); 1665 goto fini; 1666 } 1667 1668 sgroup->state = SPDK_NVMF_SUBSYSTEM_ACTIVE; 1669 1670 for (i = 0; i < sgroup->num_ns; i++) { 1671 sgroup->ns_info[i].state = SPDK_NVMF_SUBSYSTEM_ACTIVE; 1672 } 1673 1674 fini: 1675 if (cb_fn) { 1676 cb_fn(cb_arg, rc); 1677 } 1678 1679 SPDK_DTRACE_PROBE2_TICKS(nvmf_poll_group_add_subsystem, spdk_thread_get_id(group->thread), 1680 subsystem->subnqn); 1681 1682 return rc; 1683 } 1684 1685 static void 1686 _nvmf_poll_group_remove_subsystem_cb(void *ctx, int status) 1687 { 1688 struct nvmf_qpair_disconnect_many_ctx *qpair_ctx = ctx; 1689 struct spdk_nvmf_subsystem *subsystem; 1690 struct spdk_nvmf_poll_group *group; 1691 struct spdk_nvmf_subsystem_poll_group *sgroup; 1692 spdk_nvmf_poll_group_mod_done cpl_fn = NULL; 1693 void *cpl_ctx = NULL; 1694 uint32_t nsid; 1695 1696 group = qpair_ctx->group; 1697 subsystem = qpair_ctx->subsystem; 1698 cpl_fn = qpair_ctx->cpl_fn; 1699 cpl_ctx = qpair_ctx->cpl_ctx; 1700 sgroup = &group->sgroups[subsystem->id]; 1701 1702 if (status) { 1703 goto fini; 1704 } 1705 1706 for (nsid = 0; nsid < sgroup->num_ns; nsid++) { 1707 if (sgroup->ns_info[nsid].channel) { 1708 spdk_put_io_channel(sgroup->ns_info[nsid].channel); 1709 sgroup->ns_info[nsid].channel = NULL; 1710 } 1711 } 1712 1713 sgroup->num_ns = 0; 1714 free(sgroup->ns_info); 1715 sgroup->ns_info = NULL; 1716 fini: 1717 free(qpair_ctx); 1718 if (cpl_fn) { 1719 cpl_fn(cpl_ctx, status); 1720 } 1721 } 1722 1723 static void nvmf_poll_group_remove_subsystem_msg(void *ctx); 1724 1725 static void 1726 nvmf_poll_group_remove_subsystem_msg(void *ctx) 1727 { 1728 struct spdk_nvmf_qpair *qpair, *qpair_tmp; 1729 struct spdk_nvmf_subsystem *subsystem; 1730 struct spdk_nvmf_poll_group *group; 1731 struct nvmf_qpair_disconnect_many_ctx *qpair_ctx = ctx; 1732 bool qpairs_found = false; 1733 int rc = 0; 1734 1735 group = qpair_ctx->group; 1736 subsystem = qpair_ctx->subsystem; 1737 1738 TAILQ_FOREACH_SAFE(qpair, &group->qpairs, link, qpair_tmp) { 1739 if ((qpair->ctrlr != NULL) && (qpair->ctrlr->subsys == subsystem)) { 1740 qpairs_found = true; 1741 rc = spdk_nvmf_qpair_disconnect(qpair, NULL, NULL); 1742 if (rc && rc != -EINPROGRESS) { 1743 break; 1744 } 1745 } 1746 } 1747 1748 if (!qpairs_found) { 1749 _nvmf_poll_group_remove_subsystem_cb(ctx, 0); 1750 return; 1751 } 1752 1753 /* Some qpairs are in process of being disconnected. Send a message and try to remove them again */ 1754 spdk_thread_send_msg(spdk_get_thread(), nvmf_poll_group_remove_subsystem_msg, ctx); 1755 } 1756 1757 void 1758 nvmf_poll_group_remove_subsystem(struct spdk_nvmf_poll_group *group, 1759 struct spdk_nvmf_subsystem *subsystem, 1760 spdk_nvmf_poll_group_mod_done cb_fn, void *cb_arg) 1761 { 1762 struct spdk_nvmf_subsystem_poll_group *sgroup; 1763 struct nvmf_qpair_disconnect_many_ctx *ctx; 1764 uint32_t i; 1765 1766 SPDK_DTRACE_PROBE3_TICKS(nvmf_poll_group_remove_subsystem, group, spdk_thread_get_id(group->thread), 1767 subsystem->subnqn); 1768 1769 ctx = calloc(1, sizeof(struct nvmf_qpair_disconnect_many_ctx)); 1770 if (!ctx) { 1771 SPDK_ERRLOG("Unable to allocate memory for context to remove poll subsystem\n"); 1772 if (cb_fn) { 1773 cb_fn(cb_arg, -1); 1774 } 1775 return; 1776 } 1777 1778 ctx->group = group; 1779 ctx->subsystem = subsystem; 1780 ctx->cpl_fn = cb_fn; 1781 ctx->cpl_ctx = cb_arg; 1782 1783 sgroup = &group->sgroups[subsystem->id]; 1784 sgroup->state = SPDK_NVMF_SUBSYSTEM_INACTIVE; 1785 1786 for (i = 0; i < sgroup->num_ns; i++) { 1787 sgroup->ns_info[i].state = SPDK_NVMF_SUBSYSTEM_INACTIVE; 1788 } 1789 1790 nvmf_poll_group_remove_subsystem_msg(ctx); 1791 } 1792 1793 void 1794 nvmf_poll_group_pause_subsystem(struct spdk_nvmf_poll_group *group, 1795 struct spdk_nvmf_subsystem *subsystem, 1796 uint32_t nsid, 1797 spdk_nvmf_poll_group_mod_done cb_fn, void *cb_arg) 1798 { 1799 struct spdk_nvmf_subsystem_poll_group *sgroup; 1800 struct spdk_nvmf_subsystem_pg_ns_info *ns_info = NULL; 1801 int rc = 0; 1802 uint32_t i; 1803 1804 if (subsystem->id >= group->num_sgroups) { 1805 rc = -1; 1806 goto fini; 1807 } 1808 1809 sgroup = &group->sgroups[subsystem->id]; 1810 if (sgroup->state == SPDK_NVMF_SUBSYSTEM_PAUSED) { 1811 goto fini; 1812 } 1813 sgroup->state = SPDK_NVMF_SUBSYSTEM_PAUSING; 1814 1815 if (nsid == SPDK_NVME_GLOBAL_NS_TAG) { 1816 for (i = 0; i < sgroup->num_ns; i++) { 1817 ns_info = &sgroup->ns_info[i]; 1818 ns_info->state = SPDK_NVMF_SUBSYSTEM_PAUSING; 1819 } 1820 } else { 1821 /* NOTE: This implicitly also checks for 0, since 0 - 1 wraps around to UINT32_MAX. */ 1822 if (nsid - 1 < sgroup->num_ns) { 1823 ns_info = &sgroup->ns_info[nsid - 1]; 1824 ns_info->state = SPDK_NVMF_SUBSYSTEM_PAUSING; 1825 } 1826 } 1827 1828 if (sgroup->mgmt_io_outstanding > 0) { 1829 assert(sgroup->cb_fn == NULL); 1830 sgroup->cb_fn = cb_fn; 1831 assert(sgroup->cb_arg == NULL); 1832 sgroup->cb_arg = cb_arg; 1833 return; 1834 } 1835 1836 if (nsid == SPDK_NVME_GLOBAL_NS_TAG) { 1837 for (i = 0; i < sgroup->num_ns; i++) { 1838 ns_info = &sgroup->ns_info[i]; 1839 1840 if (ns_info->io_outstanding > 0) { 1841 assert(sgroup->cb_fn == NULL); 1842 sgroup->cb_fn = cb_fn; 1843 assert(sgroup->cb_arg == NULL); 1844 sgroup->cb_arg = cb_arg; 1845 return; 1846 } 1847 } 1848 } else { 1849 if (ns_info != NULL && ns_info->io_outstanding > 0) { 1850 assert(sgroup->cb_fn == NULL); 1851 sgroup->cb_fn = cb_fn; 1852 assert(sgroup->cb_arg == NULL); 1853 sgroup->cb_arg = cb_arg; 1854 return; 1855 } 1856 } 1857 1858 assert(sgroup->mgmt_io_outstanding == 0); 1859 sgroup->state = SPDK_NVMF_SUBSYSTEM_PAUSED; 1860 fini: 1861 if (cb_fn) { 1862 cb_fn(cb_arg, rc); 1863 } 1864 } 1865 1866 void 1867 nvmf_poll_group_resume_subsystem(struct spdk_nvmf_poll_group *group, 1868 struct spdk_nvmf_subsystem *subsystem, 1869 spdk_nvmf_poll_group_mod_done cb_fn, void *cb_arg) 1870 { 1871 struct spdk_nvmf_request *req, *tmp; 1872 struct spdk_nvmf_subsystem_poll_group *sgroup; 1873 int rc = 0; 1874 uint32_t i; 1875 1876 if (subsystem->id >= group->num_sgroups) { 1877 rc = -1; 1878 goto fini; 1879 } 1880 1881 sgroup = &group->sgroups[subsystem->id]; 1882 1883 if (sgroup->state == SPDK_NVMF_SUBSYSTEM_ACTIVE) { 1884 goto fini; 1885 } 1886 1887 rc = poll_group_update_subsystem(group, subsystem); 1888 if (rc) { 1889 goto fini; 1890 } 1891 1892 for (i = 0; i < sgroup->num_ns; i++) { 1893 sgroup->ns_info[i].state = SPDK_NVMF_SUBSYSTEM_ACTIVE; 1894 } 1895 1896 sgroup->state = SPDK_NVMF_SUBSYSTEM_ACTIVE; 1897 1898 /* Release all queued requests */ 1899 TAILQ_FOREACH_SAFE(req, &sgroup->queued, link, tmp) { 1900 TAILQ_REMOVE(&sgroup->queued, req, link); 1901 if (spdk_nvmf_request_using_zcopy(req)) { 1902 spdk_nvmf_request_zcopy_start(req); 1903 } else { 1904 spdk_nvmf_request_exec(req); 1905 } 1906 1907 } 1908 fini: 1909 if (cb_fn) { 1910 cb_fn(cb_arg, rc); 1911 } 1912 } 1913 1914 1915 struct spdk_nvmf_poll_group * 1916 spdk_nvmf_get_optimal_poll_group(struct spdk_nvmf_qpair *qpair) 1917 { 1918 struct spdk_nvmf_transport_poll_group *tgroup; 1919 1920 tgroup = nvmf_transport_get_optimal_poll_group(qpair->transport, qpair); 1921 1922 if (tgroup == NULL) { 1923 return NULL; 1924 } 1925 1926 return tgroup->group; 1927 } 1928 1929 void 1930 spdk_nvmf_poll_group_dump_stat(struct spdk_nvmf_poll_group *group, struct spdk_json_write_ctx *w) 1931 { 1932 struct spdk_nvmf_transport_poll_group *tgroup; 1933 1934 spdk_json_write_object_begin(w); 1935 1936 spdk_json_write_named_string(w, "name", spdk_thread_get_name(spdk_get_thread())); 1937 spdk_json_write_named_uint32(w, "admin_qpairs", group->stat.admin_qpairs); 1938 spdk_json_write_named_uint32(w, "io_qpairs", group->stat.io_qpairs); 1939 spdk_json_write_named_uint32(w, "current_admin_qpairs", group->stat.current_admin_qpairs); 1940 spdk_json_write_named_uint32(w, "current_io_qpairs", group->stat.current_io_qpairs); 1941 spdk_json_write_named_uint64(w, "pending_bdev_io", group->stat.pending_bdev_io); 1942 spdk_json_write_named_uint64(w, "completed_nvme_io", group->stat.completed_nvme_io); 1943 1944 spdk_json_write_named_array_begin(w, "transports"); 1945 1946 TAILQ_FOREACH(tgroup, &group->tgroups, link) { 1947 spdk_json_write_object_begin(w); 1948 /* 1949 * The trtype field intentionally contains a transport name as this is more informative. 1950 * The field has not been renamed for backward compatibility. 1951 */ 1952 spdk_json_write_named_string(w, "trtype", spdk_nvmf_get_transport_name(tgroup->transport)); 1953 1954 if (tgroup->transport->ops->poll_group_dump_stat) { 1955 tgroup->transport->ops->poll_group_dump_stat(tgroup, w); 1956 } 1957 1958 spdk_json_write_object_end(w); 1959 } 1960 1961 spdk_json_write_array_end(w); 1962 spdk_json_write_object_end(w); 1963 } 1964