1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2016 Intel Corporation. All rights reserved. 3 * Copyright (c) 2018-2019, 2021 Mellanox Technologies LTD. All rights reserved. 4 * Copyright (c) 2021, 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 5 */ 6 7 #include "spdk/stdinc.h" 8 9 #include "spdk/bdev.h" 10 #include "spdk/bit_array.h" 11 #include "spdk/thread.h" 12 #include "spdk/nvmf.h" 13 #include "spdk/endian.h" 14 #include "spdk/string.h" 15 #include "spdk/log.h" 16 #include "spdk_internal/usdt.h" 17 18 #include "nvmf_internal.h" 19 #include "transport.h" 20 21 SPDK_LOG_REGISTER_COMPONENT(nvmf) 22 23 #define SPDK_NVMF_DEFAULT_MAX_SUBSYSTEMS 1024 24 25 static TAILQ_HEAD(, spdk_nvmf_tgt) g_nvmf_tgts = TAILQ_HEAD_INITIALIZER(g_nvmf_tgts); 26 27 typedef void (*nvmf_qpair_disconnect_cpl)(void *ctx, int status); 28 29 /* supplied to a single call to nvmf_qpair_disconnect */ 30 struct nvmf_qpair_disconnect_ctx { 31 struct spdk_nvmf_qpair *qpair; 32 struct spdk_nvmf_ctrlr *ctrlr; 33 uint16_t qid; 34 }; 35 36 /* 37 * There are several times when we need to iterate through the list of all qpairs and selectively delete them. 38 * In order to do this sequentially without overlap, we must provide a context to recover the next qpair from 39 * to enable calling nvmf_qpair_disconnect on the next desired qpair. 40 */ 41 struct nvmf_qpair_disconnect_many_ctx { 42 struct spdk_nvmf_subsystem *subsystem; 43 struct spdk_nvmf_poll_group *group; 44 spdk_nvmf_poll_group_mod_done cpl_fn; 45 void *cpl_ctx; 46 }; 47 48 static struct spdk_nvmf_referral * 49 nvmf_tgt_find_referral(struct spdk_nvmf_tgt *tgt, 50 const struct spdk_nvme_transport_id *trid) 51 { 52 struct spdk_nvmf_referral *referral; 53 54 TAILQ_FOREACH(referral, &tgt->referrals, link) { 55 if (spdk_nvme_transport_id_compare(&referral->trid, trid) == 0) { 56 return referral; 57 } 58 } 59 60 return NULL; 61 } 62 63 int 64 spdk_nvmf_tgt_add_referral(struct spdk_nvmf_tgt *tgt, 65 const struct spdk_nvmf_referral_opts *uopts) 66 { 67 struct spdk_nvmf_referral *referral; 68 struct spdk_nvmf_referral_opts opts = {}; 69 struct spdk_nvme_transport_id *trid = &opts.trid; 70 71 memcpy(&opts, uopts, spdk_min(uopts->size, sizeof(opts))); 72 if (trid->subnqn[0] == '\0') { 73 snprintf(trid->subnqn, sizeof(trid->subnqn), "%s", SPDK_NVMF_DISCOVERY_NQN); 74 } 75 76 if (!nvmf_nqn_is_valid(trid->subnqn)) { 77 SPDK_ERRLOG("Invalid subsystem NQN\n"); 78 return -EINVAL; 79 } 80 81 /* If the entry already exists, just ignore it. */ 82 if (nvmf_tgt_find_referral(tgt, trid)) { 83 return 0; 84 } 85 86 referral = calloc(1, sizeof(*referral)); 87 if (!referral) { 88 SPDK_ERRLOG("Failed to allocate memory for a referral\n"); 89 return -ENOMEM; 90 } 91 92 referral->entry.subtype = nvmf_nqn_is_discovery(trid->subnqn) ? 93 SPDK_NVMF_SUBTYPE_DISCOVERY : 94 SPDK_NVMF_SUBTYPE_NVME; 95 referral->entry.treq.secure_channel = opts.secure_channel ? 96 SPDK_NVMF_TREQ_SECURE_CHANNEL_REQUIRED : 97 SPDK_NVMF_TREQ_SECURE_CHANNEL_NOT_REQUIRED; 98 referral->entry.cntlid = 0xffff; 99 referral->entry.trtype = trid->trtype; 100 referral->entry.adrfam = trid->adrfam; 101 memcpy(&referral->trid, trid, sizeof(struct spdk_nvme_transport_id)); 102 spdk_strcpy_pad(referral->entry.subnqn, trid->subnqn, sizeof(trid->subnqn), '\0'); 103 spdk_strcpy_pad(referral->entry.trsvcid, trid->trsvcid, sizeof(referral->entry.trsvcid), ' '); 104 spdk_strcpy_pad(referral->entry.traddr, trid->traddr, sizeof(referral->entry.traddr), ' '); 105 106 TAILQ_INSERT_HEAD(&tgt->referrals, referral, link); 107 nvmf_update_discovery_log(tgt, NULL); 108 109 return 0; 110 } 111 112 int 113 spdk_nvmf_tgt_remove_referral(struct spdk_nvmf_tgt *tgt, 114 const struct spdk_nvmf_referral_opts *uopts) 115 { 116 struct spdk_nvmf_referral *referral; 117 struct spdk_nvmf_referral_opts opts = {}; 118 struct spdk_nvme_transport_id *trid = &opts.trid; 119 120 memcpy(&opts, uopts, spdk_min(uopts->size, sizeof(opts))); 121 if (trid->subnqn[0] == '\0') { 122 snprintf(trid->subnqn, sizeof(trid->subnqn), "%s", SPDK_NVMF_DISCOVERY_NQN); 123 } 124 125 referral = nvmf_tgt_find_referral(tgt, &opts.trid); 126 if (referral == NULL) { 127 return -ENOENT; 128 } 129 130 TAILQ_REMOVE(&tgt->referrals, referral, link); 131 nvmf_update_discovery_log(tgt, NULL); 132 133 free(referral); 134 135 return 0; 136 } 137 138 void 139 nvmf_qpair_set_state(struct spdk_nvmf_qpair *qpair, 140 enum spdk_nvmf_qpair_state state) 141 { 142 assert(qpair != NULL); 143 assert(qpair->group->thread == spdk_get_thread()); 144 145 qpair->state = state; 146 } 147 148 static int 149 nvmf_poll_group_poll(void *ctx) 150 { 151 struct spdk_nvmf_poll_group *group = ctx; 152 int rc; 153 int count = 0; 154 struct spdk_nvmf_transport_poll_group *tgroup; 155 156 TAILQ_FOREACH(tgroup, &group->tgroups, link) { 157 rc = nvmf_transport_poll_group_poll(tgroup); 158 if (rc < 0) { 159 return SPDK_POLLER_BUSY; 160 } 161 count += rc; 162 } 163 164 return count > 0 ? SPDK_POLLER_BUSY : SPDK_POLLER_IDLE; 165 } 166 167 /* 168 * Reset and clean up the poll group (I/O channel code will actually free the 169 * group). 170 */ 171 static void 172 nvmf_tgt_cleanup_poll_group(struct spdk_nvmf_poll_group *group) 173 { 174 struct spdk_nvmf_transport_poll_group *tgroup, *tmp; 175 struct spdk_nvmf_subsystem_poll_group *sgroup; 176 uint32_t sid, nsid; 177 178 TAILQ_FOREACH_SAFE(tgroup, &group->tgroups, link, tmp) { 179 TAILQ_REMOVE(&group->tgroups, tgroup, link); 180 nvmf_transport_poll_group_destroy(tgroup); 181 } 182 183 for (sid = 0; sid < group->num_sgroups; sid++) { 184 sgroup = &group->sgroups[sid]; 185 186 assert(sgroup != NULL); 187 188 for (nsid = 0; nsid < sgroup->num_ns; nsid++) { 189 if (sgroup->ns_info[nsid].channel) { 190 spdk_put_io_channel(sgroup->ns_info[nsid].channel); 191 sgroup->ns_info[nsid].channel = NULL; 192 } 193 } 194 195 free(sgroup->ns_info); 196 } 197 198 free(group->sgroups); 199 200 spdk_poller_unregister(&group->poller); 201 202 if (group->destroy_cb_fn) { 203 group->destroy_cb_fn(group->destroy_cb_arg, 0); 204 } 205 } 206 207 /* 208 * Callback to unregister a poll group from the target, and clean up its state. 209 */ 210 static void 211 nvmf_tgt_destroy_poll_group(void *io_device, void *ctx_buf) 212 { 213 struct spdk_nvmf_tgt *tgt = io_device; 214 struct spdk_nvmf_poll_group *group = ctx_buf; 215 216 SPDK_DTRACE_PROBE1_TICKS(nvmf_destroy_poll_group, spdk_thread_get_id(group->thread)); 217 218 pthread_mutex_lock(&tgt->mutex); 219 TAILQ_REMOVE(&tgt->poll_groups, group, link); 220 tgt->num_poll_groups--; 221 pthread_mutex_unlock(&tgt->mutex); 222 223 assert(!(tgt->state == NVMF_TGT_PAUSING || tgt->state == NVMF_TGT_RESUMING)); 224 nvmf_tgt_cleanup_poll_group(group); 225 } 226 227 static int 228 nvmf_poll_group_add_transport(struct spdk_nvmf_poll_group *group, 229 struct spdk_nvmf_transport *transport) 230 { 231 struct spdk_nvmf_transport_poll_group *tgroup = nvmf_get_transport_poll_group(group, transport); 232 233 if (tgroup != NULL) { 234 /* Transport already in the poll group */ 235 return 0; 236 } 237 238 tgroup = nvmf_transport_poll_group_create(transport, group); 239 if (!tgroup) { 240 SPDK_ERRLOG("Unable to create poll group for transport\n"); 241 return -1; 242 } 243 SPDK_DTRACE_PROBE2_TICKS(nvmf_transport_poll_group_create, transport, 244 spdk_thread_get_id(group->thread)); 245 246 tgroup->group = group; 247 TAILQ_INSERT_TAIL(&group->tgroups, tgroup, link); 248 249 return 0; 250 } 251 252 static int 253 nvmf_tgt_create_poll_group(void *io_device, void *ctx_buf) 254 { 255 struct spdk_nvmf_tgt *tgt = io_device; 256 struct spdk_nvmf_poll_group *group = ctx_buf; 257 struct spdk_nvmf_transport *transport; 258 struct spdk_nvmf_subsystem *subsystem; 259 struct spdk_thread *thread = spdk_get_thread(); 260 uint32_t i; 261 int rc; 262 263 group->tgt = tgt; 264 TAILQ_INIT(&group->tgroups); 265 TAILQ_INIT(&group->qpairs); 266 group->thread = thread; 267 pthread_mutex_init(&group->mutex, NULL); 268 269 group->poller = SPDK_POLLER_REGISTER(nvmf_poll_group_poll, group, 0); 270 271 SPDK_DTRACE_PROBE1_TICKS(nvmf_create_poll_group, spdk_thread_get_id(thread)); 272 273 TAILQ_FOREACH(transport, &tgt->transports, link) { 274 rc = nvmf_poll_group_add_transport(group, transport); 275 if (rc != 0) { 276 nvmf_tgt_cleanup_poll_group(group); 277 return rc; 278 } 279 } 280 281 group->num_sgroups = tgt->max_subsystems; 282 group->sgroups = calloc(tgt->max_subsystems, sizeof(struct spdk_nvmf_subsystem_poll_group)); 283 if (!group->sgroups) { 284 nvmf_tgt_cleanup_poll_group(group); 285 return -ENOMEM; 286 } 287 288 for (i = 0; i < tgt->max_subsystems; i++) { 289 TAILQ_INIT(&group->sgroups[i].queued); 290 } 291 292 for (subsystem = spdk_nvmf_subsystem_get_first(tgt); 293 subsystem != NULL; 294 subsystem = spdk_nvmf_subsystem_get_next(subsystem)) { 295 if (nvmf_poll_group_add_subsystem(group, subsystem, NULL, NULL) != 0) { 296 nvmf_tgt_cleanup_poll_group(group); 297 return -1; 298 } 299 } 300 301 pthread_mutex_lock(&tgt->mutex); 302 tgt->num_poll_groups++; 303 TAILQ_INSERT_TAIL(&tgt->poll_groups, group, link); 304 pthread_mutex_unlock(&tgt->mutex); 305 306 return 0; 307 } 308 309 static void 310 _nvmf_tgt_disconnect_qpairs(void *ctx) 311 { 312 struct spdk_nvmf_qpair *qpair, *qpair_tmp; 313 struct nvmf_qpair_disconnect_many_ctx *qpair_ctx = ctx; 314 struct spdk_nvmf_poll_group *group = qpair_ctx->group; 315 struct spdk_io_channel *ch; 316 int rc; 317 318 TAILQ_FOREACH_SAFE(qpair, &group->qpairs, link, qpair_tmp) { 319 rc = spdk_nvmf_qpair_disconnect(qpair); 320 if (rc && rc != -EINPROGRESS) { 321 break; 322 } 323 } 324 325 if (TAILQ_EMPTY(&group->qpairs)) { 326 /* When the refcount from the channels reaches 0, nvmf_tgt_destroy_poll_group will be called. */ 327 ch = spdk_io_channel_from_ctx(group); 328 spdk_put_io_channel(ch); 329 free(qpair_ctx); 330 return; 331 } 332 333 /* Some qpairs are in process of being disconnected. Send a message and try to remove them again */ 334 spdk_thread_send_msg(spdk_get_thread(), _nvmf_tgt_disconnect_qpairs, ctx); 335 } 336 337 static void 338 nvmf_tgt_destroy_poll_group_qpairs(struct spdk_nvmf_poll_group *group) 339 { 340 struct nvmf_qpair_disconnect_many_ctx *ctx; 341 342 SPDK_DTRACE_PROBE1_TICKS(nvmf_destroy_poll_group_qpairs, spdk_thread_get_id(group->thread)); 343 344 ctx = calloc(1, sizeof(struct nvmf_qpair_disconnect_many_ctx)); 345 if (!ctx) { 346 SPDK_ERRLOG("Failed to allocate memory for destroy poll group ctx\n"); 347 return; 348 } 349 350 ctx->group = group; 351 _nvmf_tgt_disconnect_qpairs(ctx); 352 } 353 354 struct spdk_nvmf_tgt * 355 spdk_nvmf_tgt_create(struct spdk_nvmf_target_opts *_opts) 356 { 357 struct spdk_nvmf_tgt *tgt, *tmp_tgt; 358 struct spdk_nvmf_target_opts opts = { 359 .max_subsystems = SPDK_NVMF_DEFAULT_MAX_SUBSYSTEMS, 360 .discovery_filter = SPDK_NVMF_TGT_DISCOVERY_MATCH_ANY, 361 }; 362 363 memcpy(&opts, _opts, _opts->size); 364 if (strnlen(opts.name, NVMF_TGT_NAME_MAX_LENGTH) == NVMF_TGT_NAME_MAX_LENGTH) { 365 SPDK_ERRLOG("Provided target name exceeds the max length of %u.\n", NVMF_TGT_NAME_MAX_LENGTH); 366 return NULL; 367 } 368 369 TAILQ_FOREACH(tmp_tgt, &g_nvmf_tgts, link) { 370 if (!strncmp(opts.name, tmp_tgt->name, NVMF_TGT_NAME_MAX_LENGTH)) { 371 SPDK_ERRLOG("Provided target name must be unique.\n"); 372 return NULL; 373 } 374 } 375 376 tgt = calloc(1, sizeof(*tgt)); 377 if (!tgt) { 378 return NULL; 379 } 380 381 snprintf(tgt->name, NVMF_TGT_NAME_MAX_LENGTH, "%s", opts.name); 382 383 if (!opts.max_subsystems) { 384 tgt->max_subsystems = SPDK_NVMF_DEFAULT_MAX_SUBSYSTEMS; 385 } else { 386 tgt->max_subsystems = opts.max_subsystems; 387 } 388 389 tgt->crdt[0] = opts.crdt[0]; 390 tgt->crdt[1] = opts.crdt[1]; 391 tgt->crdt[2] = opts.crdt[2]; 392 tgt->discovery_filter = opts.discovery_filter; 393 tgt->discovery_genctr = 0; 394 tgt->dhchap_digests = opts.dhchap_digests; 395 tgt->dhchap_dhgroups = opts.dhchap_dhgroups; 396 TAILQ_INIT(&tgt->transports); 397 TAILQ_INIT(&tgt->poll_groups); 398 TAILQ_INIT(&tgt->referrals); 399 tgt->num_poll_groups = 0; 400 401 tgt->subsystem_ids = spdk_bit_array_create(tgt->max_subsystems); 402 if (tgt->subsystem_ids == NULL) { 403 free(tgt); 404 return NULL; 405 } 406 407 RB_INIT(&tgt->subsystems); 408 409 pthread_mutex_init(&tgt->mutex, NULL); 410 411 spdk_io_device_register(tgt, 412 nvmf_tgt_create_poll_group, 413 nvmf_tgt_destroy_poll_group, 414 sizeof(struct spdk_nvmf_poll_group), 415 tgt->name); 416 417 tgt->state = NVMF_TGT_RUNNING; 418 419 TAILQ_INSERT_HEAD(&g_nvmf_tgts, tgt, link); 420 421 return tgt; 422 } 423 424 static void 425 _nvmf_tgt_destroy_next_transport(void *ctx) 426 { 427 struct spdk_nvmf_tgt *tgt = ctx; 428 struct spdk_nvmf_transport *transport; 429 430 if (!TAILQ_EMPTY(&tgt->transports)) { 431 transport = TAILQ_FIRST(&tgt->transports); 432 TAILQ_REMOVE(&tgt->transports, transport, link); 433 spdk_nvmf_transport_destroy(transport, _nvmf_tgt_destroy_next_transport, tgt); 434 } else { 435 spdk_nvmf_tgt_destroy_done_fn *destroy_cb_fn = tgt->destroy_cb_fn; 436 void *destroy_cb_arg = tgt->destroy_cb_arg; 437 438 pthread_mutex_destroy(&tgt->mutex); 439 free(tgt); 440 441 if (destroy_cb_fn) { 442 destroy_cb_fn(destroy_cb_arg, 0); 443 } 444 } 445 } 446 447 static void 448 nvmf_tgt_destroy_cb(void *io_device) 449 { 450 struct spdk_nvmf_tgt *tgt = io_device; 451 struct spdk_nvmf_subsystem *subsystem, *subsystem_next; 452 int rc; 453 struct spdk_nvmf_referral *referral; 454 455 while ((referral = TAILQ_FIRST(&tgt->referrals))) { 456 TAILQ_REMOVE(&tgt->referrals, referral, link); 457 free(referral); 458 } 459 460 /* We will be freeing subsystems in this loop, so we always need to get the next one 461 * ahead of time, since we can't call get_next() on a subsystem that's been freed. 462 */ 463 for (subsystem = spdk_nvmf_subsystem_get_first(tgt), 464 subsystem_next = spdk_nvmf_subsystem_get_next(subsystem); 465 subsystem != NULL; 466 subsystem = subsystem_next, 467 subsystem_next = spdk_nvmf_subsystem_get_next(subsystem_next)) { 468 nvmf_subsystem_remove_all_listeners(subsystem, true); 469 470 rc = spdk_nvmf_subsystem_destroy(subsystem, nvmf_tgt_destroy_cb, tgt); 471 if (rc) { 472 if (rc == -EINPROGRESS) { 473 /* If rc is -EINPROGRESS, nvmf_tgt_destroy_cb will be called again when subsystem #i 474 * is destroyed, nvmf_tgt_destroy_cb will continue to destroy other subsystems if any */ 475 return; 476 } else { 477 SPDK_ERRLOG("Failed to destroy subsystem %s, rc %d\n", subsystem->subnqn, rc); 478 } 479 } 480 } 481 spdk_bit_array_free(&tgt->subsystem_ids); 482 _nvmf_tgt_destroy_next_transport(tgt); 483 } 484 485 void 486 spdk_nvmf_tgt_destroy(struct spdk_nvmf_tgt *tgt, 487 spdk_nvmf_tgt_destroy_done_fn cb_fn, 488 void *cb_arg) 489 { 490 assert(!(tgt->state == NVMF_TGT_PAUSING || tgt->state == NVMF_TGT_RESUMING)); 491 492 tgt->destroy_cb_fn = cb_fn; 493 tgt->destroy_cb_arg = cb_arg; 494 495 TAILQ_REMOVE(&g_nvmf_tgts, tgt, link); 496 497 spdk_io_device_unregister(tgt, nvmf_tgt_destroy_cb); 498 } 499 500 const char * 501 spdk_nvmf_tgt_get_name(struct spdk_nvmf_tgt *tgt) 502 { 503 return tgt->name; 504 } 505 506 struct spdk_nvmf_tgt * 507 spdk_nvmf_get_tgt(const char *name) 508 { 509 struct spdk_nvmf_tgt *tgt; 510 uint32_t num_targets = 0; 511 512 TAILQ_FOREACH(tgt, &g_nvmf_tgts, link) { 513 if (name) { 514 if (!strncmp(tgt->name, name, NVMF_TGT_NAME_MAX_LENGTH)) { 515 return tgt; 516 } 517 } 518 num_targets++; 519 } 520 521 /* 522 * special case. If there is only one target and 523 * no name was specified, return the only available 524 * target. If there is more than one target, name must 525 * be specified. 526 */ 527 if (!name && num_targets == 1) { 528 return TAILQ_FIRST(&g_nvmf_tgts); 529 } 530 531 return NULL; 532 } 533 534 struct spdk_nvmf_tgt * 535 spdk_nvmf_get_first_tgt(void) 536 { 537 return TAILQ_FIRST(&g_nvmf_tgts); 538 } 539 540 struct spdk_nvmf_tgt * 541 spdk_nvmf_get_next_tgt(struct spdk_nvmf_tgt *prev) 542 { 543 return TAILQ_NEXT(prev, link); 544 } 545 546 static void 547 nvmf_write_nvme_subsystem_config(struct spdk_json_write_ctx *w, 548 struct spdk_nvmf_subsystem *subsystem) 549 { 550 struct spdk_nvmf_host *host; 551 struct spdk_nvmf_ns *ns; 552 struct spdk_nvmf_ns_opts ns_opts; 553 uint32_t max_namespaces; 554 struct spdk_nvmf_transport *transport; 555 556 assert(spdk_nvmf_subsystem_get_type(subsystem) == SPDK_NVMF_SUBTYPE_NVME); 557 558 /* { */ 559 spdk_json_write_object_begin(w); 560 spdk_json_write_named_string(w, "method", "nvmf_create_subsystem"); 561 562 /* "params" : { */ 563 spdk_json_write_named_object_begin(w, "params"); 564 spdk_json_write_named_string(w, "nqn", spdk_nvmf_subsystem_get_nqn(subsystem)); 565 spdk_json_write_named_bool(w, "allow_any_host", spdk_nvmf_subsystem_get_allow_any_host(subsystem)); 566 spdk_json_write_named_string(w, "serial_number", spdk_nvmf_subsystem_get_sn(subsystem)); 567 spdk_json_write_named_string(w, "model_number", spdk_nvmf_subsystem_get_mn(subsystem)); 568 569 max_namespaces = spdk_nvmf_subsystem_get_max_namespaces(subsystem); 570 if (max_namespaces != 0) { 571 spdk_json_write_named_uint32(w, "max_namespaces", max_namespaces); 572 } 573 574 spdk_json_write_named_uint32(w, "min_cntlid", spdk_nvmf_subsystem_get_min_cntlid(subsystem)); 575 spdk_json_write_named_uint32(w, "max_cntlid", spdk_nvmf_subsystem_get_max_cntlid(subsystem)); 576 spdk_json_write_named_bool(w, "ana_reporting", spdk_nvmf_subsystem_get_ana_reporting(subsystem)); 577 578 /* } "params" */ 579 spdk_json_write_object_end(w); 580 581 /* } */ 582 spdk_json_write_object_end(w); 583 584 for (host = spdk_nvmf_subsystem_get_first_host(subsystem); host != NULL; 585 host = spdk_nvmf_subsystem_get_next_host(subsystem, host)) { 586 587 spdk_json_write_object_begin(w); 588 spdk_json_write_named_string(w, "method", "nvmf_subsystem_add_host"); 589 590 /* "params" : { */ 591 spdk_json_write_named_object_begin(w, "params"); 592 593 spdk_json_write_named_string(w, "nqn", spdk_nvmf_subsystem_get_nqn(subsystem)); 594 spdk_json_write_named_string(w, "host", spdk_nvmf_host_get_nqn(host)); 595 if (host->dhchap_key != NULL) { 596 spdk_json_write_named_string(w, "dhchap_key", 597 spdk_key_get_name(host->dhchap_key)); 598 } 599 if (host->dhchap_ctrlr_key != NULL) { 600 spdk_json_write_named_string(w, "dhchap_ctrlr_key", 601 spdk_key_get_name(host->dhchap_ctrlr_key)); 602 } 603 TAILQ_FOREACH(transport, &subsystem->tgt->transports, link) { 604 if (transport->ops->subsystem_dump_host != NULL) { 605 transport->ops->subsystem_dump_host(transport, subsystem, host->nqn, w); 606 } 607 } 608 609 /* } "params" */ 610 spdk_json_write_object_end(w); 611 612 /* } */ 613 spdk_json_write_object_end(w); 614 } 615 616 for (ns = spdk_nvmf_subsystem_get_first_ns(subsystem); ns != NULL; 617 ns = spdk_nvmf_subsystem_get_next_ns(subsystem, ns)) { 618 spdk_nvmf_ns_get_opts(ns, &ns_opts, sizeof(ns_opts)); 619 620 spdk_json_write_object_begin(w); 621 spdk_json_write_named_string(w, "method", "nvmf_subsystem_add_ns"); 622 623 /* "params" : { */ 624 spdk_json_write_named_object_begin(w, "params"); 625 626 spdk_json_write_named_string(w, "nqn", spdk_nvmf_subsystem_get_nqn(subsystem)); 627 628 /* "namespace" : { */ 629 spdk_json_write_named_object_begin(w, "namespace"); 630 631 spdk_json_write_named_uint32(w, "nsid", spdk_nvmf_ns_get_id(ns)); 632 spdk_json_write_named_string(w, "bdev_name", spdk_bdev_get_name(spdk_nvmf_ns_get_bdev(ns))); 633 634 if (ns->ptpl_file != NULL) { 635 spdk_json_write_named_string(w, "ptpl_file", ns->ptpl_file); 636 } 637 638 if (!spdk_mem_all_zero(ns_opts.nguid, sizeof(ns_opts.nguid))) { 639 SPDK_STATIC_ASSERT(sizeof(ns_opts.nguid) == sizeof(uint64_t) * 2, "size mismatch"); 640 spdk_json_write_named_string_fmt(w, "nguid", "%016"PRIX64"%016"PRIX64, from_be64(&ns_opts.nguid[0]), 641 from_be64(&ns_opts.nguid[8])); 642 } 643 644 if (!spdk_mem_all_zero(ns_opts.eui64, sizeof(ns_opts.eui64))) { 645 SPDK_STATIC_ASSERT(sizeof(ns_opts.eui64) == sizeof(uint64_t), "size mismatch"); 646 spdk_json_write_named_string_fmt(w, "eui64", "%016"PRIX64, from_be64(&ns_opts.eui64)); 647 } 648 649 if (!spdk_uuid_is_null(&ns_opts.uuid)) { 650 spdk_json_write_named_uuid(w, "uuid", &ns_opts.uuid); 651 } 652 653 if (spdk_nvmf_subsystem_get_ana_reporting(subsystem)) { 654 spdk_json_write_named_uint32(w, "anagrpid", ns_opts.anagrpid); 655 } 656 657 spdk_json_write_named_bool(w, "no_auto_visible", !ns->always_visible); 658 659 /* "namespace" */ 660 spdk_json_write_object_end(w); 661 662 /* } "params" */ 663 spdk_json_write_object_end(w); 664 665 /* } */ 666 spdk_json_write_object_end(w); 667 668 TAILQ_FOREACH(host, &ns->hosts, link) { 669 spdk_json_write_object_begin(w); 670 spdk_json_write_named_string(w, "method", "nvmf_ns_add_host"); 671 spdk_json_write_named_object_begin(w, "params"); 672 spdk_json_write_named_string(w, "nqn", spdk_nvmf_subsystem_get_nqn(subsystem)); 673 spdk_json_write_named_uint32(w, "nsid", spdk_nvmf_ns_get_id(ns)); 674 spdk_json_write_named_string(w, "host", spdk_nvmf_host_get_nqn(host)); 675 spdk_json_write_object_end(w); 676 spdk_json_write_object_end(w); 677 } 678 } 679 } 680 681 static void 682 nvmf_write_subsystem_config_json(struct spdk_json_write_ctx *w, 683 struct spdk_nvmf_subsystem *subsystem) 684 { 685 struct spdk_nvmf_subsystem_listener *listener; 686 struct spdk_nvmf_transport *transport; 687 const struct spdk_nvme_transport_id *trid; 688 689 if (spdk_nvmf_subsystem_get_type(subsystem) == SPDK_NVMF_SUBTYPE_NVME) { 690 nvmf_write_nvme_subsystem_config(w, subsystem); 691 } 692 693 for (listener = spdk_nvmf_subsystem_get_first_listener(subsystem); listener != NULL; 694 listener = spdk_nvmf_subsystem_get_next_listener(subsystem, listener)) { 695 transport = listener->transport; 696 trid = spdk_nvmf_subsystem_listener_get_trid(listener); 697 698 spdk_json_write_object_begin(w); 699 spdk_json_write_named_string(w, "method", "nvmf_subsystem_add_listener"); 700 701 /* "params" : { */ 702 spdk_json_write_named_object_begin(w, "params"); 703 704 spdk_json_write_named_string(w, "nqn", spdk_nvmf_subsystem_get_nqn(subsystem)); 705 706 spdk_json_write_named_object_begin(w, "listen_address"); 707 nvmf_transport_listen_dump_trid(trid, w); 708 spdk_json_write_object_end(w); 709 if (transport->ops->listen_dump_opts) { 710 transport->ops->listen_dump_opts(transport, trid, w); 711 } 712 713 spdk_json_write_named_bool(w, "secure_channel", listener->opts.secure_channel); 714 715 if (listener->opts.sock_impl) { 716 spdk_json_write_named_string(w, "sock_impl", listener->opts.sock_impl); 717 } 718 719 /* } "params" */ 720 spdk_json_write_object_end(w); 721 722 /* } */ 723 spdk_json_write_object_end(w); 724 } 725 } 726 727 void 728 spdk_nvmf_tgt_write_config_json(struct spdk_json_write_ctx *w, struct spdk_nvmf_tgt *tgt) 729 { 730 struct spdk_nvmf_subsystem *subsystem; 731 struct spdk_nvmf_transport *transport; 732 struct spdk_nvmf_referral *referral; 733 734 spdk_json_write_object_begin(w); 735 spdk_json_write_named_string(w, "method", "nvmf_set_max_subsystems"); 736 737 spdk_json_write_named_object_begin(w, "params"); 738 spdk_json_write_named_uint32(w, "max_subsystems", tgt->max_subsystems); 739 spdk_json_write_object_end(w); 740 741 spdk_json_write_object_end(w); 742 743 spdk_json_write_object_begin(w); 744 spdk_json_write_named_string(w, "method", "nvmf_set_crdt"); 745 spdk_json_write_named_object_begin(w, "params"); 746 spdk_json_write_named_uint32(w, "crdt1", tgt->crdt[0]); 747 spdk_json_write_named_uint32(w, "crdt2", tgt->crdt[1]); 748 spdk_json_write_named_uint32(w, "crdt3", tgt->crdt[2]); 749 spdk_json_write_object_end(w); 750 spdk_json_write_object_end(w); 751 752 /* write transports */ 753 TAILQ_FOREACH(transport, &tgt->transports, link) { 754 spdk_json_write_object_begin(w); 755 spdk_json_write_named_string(w, "method", "nvmf_create_transport"); 756 nvmf_transport_dump_opts(transport, w, true); 757 spdk_json_write_object_end(w); 758 } 759 760 TAILQ_FOREACH(referral, &tgt->referrals, link) { 761 spdk_json_write_object_begin(w); 762 spdk_json_write_named_string(w, "method", "nvmf_discovery_add_referral"); 763 764 spdk_json_write_named_object_begin(w, "params"); 765 spdk_json_write_named_object_begin(w, "address"); 766 nvmf_transport_listen_dump_trid(&referral->trid, w); 767 spdk_json_write_object_end(w); 768 spdk_json_write_named_bool(w, "secure_channel", 769 referral->entry.treq.secure_channel == 770 SPDK_NVMF_TREQ_SECURE_CHANNEL_REQUIRED); 771 spdk_json_write_named_string(w, "subnqn", referral->trid.subnqn); 772 spdk_json_write_object_end(w); 773 774 spdk_json_write_object_end(w); 775 } 776 777 subsystem = spdk_nvmf_subsystem_get_first(tgt); 778 while (subsystem) { 779 nvmf_write_subsystem_config_json(w, subsystem); 780 subsystem = spdk_nvmf_subsystem_get_next(subsystem); 781 } 782 } 783 784 static void 785 nvmf_listen_opts_copy(struct spdk_nvmf_listen_opts *opts, 786 const struct spdk_nvmf_listen_opts *opts_src, size_t opts_size) 787 { 788 assert(opts); 789 assert(opts_src); 790 791 opts->opts_size = opts_size; 792 793 #define SET_FIELD(field) \ 794 if (offsetof(struct spdk_nvmf_listen_opts, field) + sizeof(opts->field) <= opts_size) { \ 795 opts->field = opts_src->field; \ 796 } \ 797 798 SET_FIELD(transport_specific); 799 SET_FIELD(secure_channel); 800 SET_FIELD(ana_state); 801 SET_FIELD(sock_impl); 802 #undef SET_FIELD 803 804 /* Do not remove this statement, you should always update this statement when you adding a new field, 805 * and do not forget to add the SET_FIELD statement for your added field. */ 806 SPDK_STATIC_ASSERT(sizeof(struct spdk_nvmf_listen_opts) == 32, "Incorrect size"); 807 } 808 809 void 810 spdk_nvmf_listen_opts_init(struct spdk_nvmf_listen_opts *opts, size_t opts_size) 811 { 812 struct spdk_nvmf_listen_opts opts_local = {}; 813 814 /* local version of opts should have defaults set here */ 815 opts_local.ana_state = SPDK_NVME_ANA_OPTIMIZED_STATE; 816 nvmf_listen_opts_copy(opts, &opts_local, opts_size); 817 } 818 819 int 820 spdk_nvmf_tgt_listen_ext(struct spdk_nvmf_tgt *tgt, const struct spdk_nvme_transport_id *trid, 821 struct spdk_nvmf_listen_opts *opts) 822 { 823 struct spdk_nvmf_transport *transport; 824 int rc; 825 struct spdk_nvmf_listen_opts opts_local = {}; 826 827 if (!opts) { 828 SPDK_ERRLOG("opts should not be NULL\n"); 829 return -EINVAL; 830 } 831 832 if (!opts->opts_size) { 833 SPDK_ERRLOG("The opts_size in opts structure should not be zero\n"); 834 return -EINVAL; 835 } 836 837 transport = spdk_nvmf_tgt_get_transport(tgt, trid->trstring); 838 if (!transport) { 839 SPDK_ERRLOG("Unable to find %s transport. The transport must be created first also make sure it is properly registered.\n", 840 trid->trstring); 841 return -EINVAL; 842 } 843 844 nvmf_listen_opts_copy(&opts_local, opts, opts->opts_size); 845 rc = spdk_nvmf_transport_listen(transport, trid, &opts_local); 846 if (rc < 0) { 847 SPDK_ERRLOG("Unable to listen on address '%s'\n", trid->traddr); 848 } 849 850 return rc; 851 } 852 853 int 854 spdk_nvmf_tgt_stop_listen(struct spdk_nvmf_tgt *tgt, 855 struct spdk_nvme_transport_id *trid) 856 { 857 struct spdk_nvmf_transport *transport; 858 int rc; 859 860 transport = spdk_nvmf_tgt_get_transport(tgt, trid->trstring); 861 if (!transport) { 862 SPDK_ERRLOG("Unable to find %s transport. The transport must be created first also make sure it is properly registered.\n", 863 trid->trstring); 864 return -EINVAL; 865 } 866 867 rc = spdk_nvmf_transport_stop_listen(transport, trid); 868 if (rc < 0) { 869 SPDK_ERRLOG("Failed to stop listening on address '%s'\n", trid->traddr); 870 return rc; 871 } 872 return 0; 873 } 874 875 struct spdk_nvmf_tgt_add_transport_ctx { 876 struct spdk_nvmf_tgt *tgt; 877 struct spdk_nvmf_transport *transport; 878 spdk_nvmf_tgt_add_transport_done_fn cb_fn; 879 void *cb_arg; 880 int status; 881 }; 882 883 static void 884 _nvmf_tgt_remove_transport_done(struct spdk_io_channel_iter *i, int status) 885 { 886 struct spdk_nvmf_tgt_add_transport_ctx *ctx = spdk_io_channel_iter_get_ctx(i); 887 888 ctx->cb_fn(ctx->cb_arg, ctx->status); 889 free(ctx); 890 } 891 892 static void 893 _nvmf_tgt_remove_transport(struct spdk_io_channel_iter *i) 894 { 895 struct spdk_nvmf_tgt_add_transport_ctx *ctx = spdk_io_channel_iter_get_ctx(i); 896 struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i); 897 struct spdk_nvmf_poll_group *group = spdk_io_channel_get_ctx(ch); 898 struct spdk_nvmf_transport_poll_group *tgroup, *tmp; 899 900 TAILQ_FOREACH_SAFE(tgroup, &group->tgroups, link, tmp) { 901 if (tgroup->transport == ctx->transport) { 902 TAILQ_REMOVE(&group->tgroups, tgroup, link); 903 nvmf_transport_poll_group_destroy(tgroup); 904 } 905 } 906 907 spdk_for_each_channel_continue(i, 0); 908 } 909 910 static void 911 _nvmf_tgt_add_transport_done(struct spdk_io_channel_iter *i, int status) 912 { 913 struct spdk_nvmf_tgt_add_transport_ctx *ctx = spdk_io_channel_iter_get_ctx(i); 914 915 if (status) { 916 ctx->status = status; 917 spdk_for_each_channel(ctx->tgt, 918 _nvmf_tgt_remove_transport, 919 ctx, 920 _nvmf_tgt_remove_transport_done); 921 return; 922 } 923 924 ctx->transport->tgt = ctx->tgt; 925 TAILQ_INSERT_TAIL(&ctx->tgt->transports, ctx->transport, link); 926 ctx->cb_fn(ctx->cb_arg, status); 927 free(ctx); 928 } 929 930 static void 931 _nvmf_tgt_add_transport(struct spdk_io_channel_iter *i) 932 { 933 struct spdk_nvmf_tgt_add_transport_ctx *ctx = spdk_io_channel_iter_get_ctx(i); 934 struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i); 935 struct spdk_nvmf_poll_group *group = spdk_io_channel_get_ctx(ch); 936 int rc; 937 938 rc = nvmf_poll_group_add_transport(group, ctx->transport); 939 spdk_for_each_channel_continue(i, rc); 940 } 941 942 void 943 spdk_nvmf_tgt_add_transport(struct spdk_nvmf_tgt *tgt, 944 struct spdk_nvmf_transport *transport, 945 spdk_nvmf_tgt_add_transport_done_fn cb_fn, 946 void *cb_arg) 947 { 948 struct spdk_nvmf_tgt_add_transport_ctx *ctx; 949 950 SPDK_DTRACE_PROBE2_TICKS(nvmf_tgt_add_transport, transport, tgt->name); 951 952 if (spdk_nvmf_tgt_get_transport(tgt, transport->ops->name)) { 953 cb_fn(cb_arg, -EEXIST); 954 return; /* transport already created */ 955 } 956 957 ctx = calloc(1, sizeof(*ctx)); 958 if (!ctx) { 959 cb_fn(cb_arg, -ENOMEM); 960 return; 961 } 962 963 ctx->tgt = tgt; 964 ctx->transport = transport; 965 ctx->cb_fn = cb_fn; 966 ctx->cb_arg = cb_arg; 967 968 spdk_for_each_channel(tgt, 969 _nvmf_tgt_add_transport, 970 ctx, 971 _nvmf_tgt_add_transport_done); 972 } 973 974 struct nvmf_tgt_pause_ctx { 975 struct spdk_nvmf_tgt *tgt; 976 spdk_nvmf_tgt_pause_polling_cb_fn cb_fn; 977 void *cb_arg; 978 }; 979 980 static void 981 _nvmf_tgt_pause_polling_done(struct spdk_io_channel_iter *i, int status) 982 { 983 struct nvmf_tgt_pause_ctx *ctx = spdk_io_channel_iter_get_ctx(i); 984 985 ctx->tgt->state = NVMF_TGT_PAUSED; 986 987 ctx->cb_fn(ctx->cb_arg, status); 988 free(ctx); 989 } 990 991 static void 992 _nvmf_tgt_pause_polling(struct spdk_io_channel_iter *i) 993 { 994 struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i); 995 struct spdk_nvmf_poll_group *group = spdk_io_channel_get_ctx(ch); 996 997 spdk_poller_unregister(&group->poller); 998 999 spdk_for_each_channel_continue(i, 0); 1000 } 1001 1002 int 1003 spdk_nvmf_tgt_pause_polling(struct spdk_nvmf_tgt *tgt, spdk_nvmf_tgt_pause_polling_cb_fn cb_fn, 1004 void *cb_arg) 1005 { 1006 struct nvmf_tgt_pause_ctx *ctx; 1007 1008 SPDK_DTRACE_PROBE2_TICKS(nvmf_tgt_pause_polling, tgt, tgt->name); 1009 1010 switch (tgt->state) { 1011 case NVMF_TGT_PAUSING: 1012 case NVMF_TGT_RESUMING: 1013 return -EBUSY; 1014 case NVMF_TGT_RUNNING: 1015 break; 1016 default: 1017 return -EINVAL; 1018 } 1019 1020 ctx = calloc(1, sizeof(*ctx)); 1021 if (!ctx) { 1022 return -ENOMEM; 1023 } 1024 1025 1026 tgt->state = NVMF_TGT_PAUSING; 1027 1028 ctx->tgt = tgt; 1029 ctx->cb_fn = cb_fn; 1030 ctx->cb_arg = cb_arg; 1031 1032 spdk_for_each_channel(tgt, 1033 _nvmf_tgt_pause_polling, 1034 ctx, 1035 _nvmf_tgt_pause_polling_done); 1036 return 0; 1037 } 1038 1039 static void 1040 _nvmf_tgt_resume_polling_done(struct spdk_io_channel_iter *i, int status) 1041 { 1042 struct nvmf_tgt_pause_ctx *ctx = spdk_io_channel_iter_get_ctx(i); 1043 1044 ctx->tgt->state = NVMF_TGT_RUNNING; 1045 1046 ctx->cb_fn(ctx->cb_arg, status); 1047 free(ctx); 1048 } 1049 1050 static void 1051 _nvmf_tgt_resume_polling(struct spdk_io_channel_iter *i) 1052 { 1053 struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i); 1054 struct spdk_nvmf_poll_group *group = spdk_io_channel_get_ctx(ch); 1055 1056 assert(group->poller == NULL); 1057 group->poller = SPDK_POLLER_REGISTER(nvmf_poll_group_poll, group, 0); 1058 1059 spdk_for_each_channel_continue(i, 0); 1060 } 1061 1062 int 1063 spdk_nvmf_tgt_resume_polling(struct spdk_nvmf_tgt *tgt, spdk_nvmf_tgt_resume_polling_cb_fn cb_fn, 1064 void *cb_arg) 1065 { 1066 struct nvmf_tgt_pause_ctx *ctx; 1067 1068 SPDK_DTRACE_PROBE2_TICKS(nvmf_tgt_resume_polling, tgt, tgt->name); 1069 1070 switch (tgt->state) { 1071 case NVMF_TGT_PAUSING: 1072 case NVMF_TGT_RESUMING: 1073 return -EBUSY; 1074 case NVMF_TGT_PAUSED: 1075 break; 1076 default: 1077 return -EINVAL; 1078 } 1079 1080 ctx = calloc(1, sizeof(*ctx)); 1081 if (!ctx) { 1082 return -ENOMEM; 1083 } 1084 1085 tgt->state = NVMF_TGT_RESUMING; 1086 1087 ctx->tgt = tgt; 1088 ctx->cb_fn = cb_fn; 1089 ctx->cb_arg = cb_arg; 1090 1091 spdk_for_each_channel(tgt, 1092 _nvmf_tgt_resume_polling, 1093 ctx, 1094 _nvmf_tgt_resume_polling_done); 1095 return 0; 1096 } 1097 1098 struct spdk_nvmf_subsystem * 1099 spdk_nvmf_tgt_find_subsystem(struct spdk_nvmf_tgt *tgt, const char *subnqn) 1100 { 1101 struct spdk_nvmf_subsystem subsystem; 1102 1103 if (!subnqn) { 1104 return NULL; 1105 } 1106 1107 /* Ensure that subnqn is null terminated */ 1108 if (!memchr(subnqn, '\0', SPDK_NVMF_NQN_MAX_LEN + 1)) { 1109 SPDK_ERRLOG("Connect SUBNQN is not null terminated\n"); 1110 return NULL; 1111 } 1112 1113 snprintf(subsystem.subnqn, sizeof(subsystem.subnqn), "%s", subnqn); 1114 return RB_FIND(subsystem_tree, &tgt->subsystems, &subsystem); 1115 } 1116 1117 struct spdk_nvmf_transport * 1118 spdk_nvmf_tgt_get_transport(struct spdk_nvmf_tgt *tgt, const char *transport_name) 1119 { 1120 struct spdk_nvmf_transport *transport; 1121 1122 TAILQ_FOREACH(transport, &tgt->transports, link) { 1123 if (!strncasecmp(transport->ops->name, transport_name, SPDK_NVMF_TRSTRING_MAX_LEN)) { 1124 return transport; 1125 } 1126 } 1127 return NULL; 1128 } 1129 1130 struct nvmf_new_qpair_ctx { 1131 struct spdk_nvmf_qpair *qpair; 1132 struct spdk_nvmf_poll_group *group; 1133 }; 1134 1135 static void 1136 _nvmf_poll_group_add(void *_ctx) 1137 { 1138 struct nvmf_new_qpair_ctx *ctx = _ctx; 1139 struct spdk_nvmf_qpair *qpair = ctx->qpair; 1140 struct spdk_nvmf_poll_group *group = ctx->group; 1141 1142 free(_ctx); 1143 1144 if (spdk_nvmf_poll_group_add(group, qpair) != 0) { 1145 SPDK_ERRLOG("Unable to add the qpair to a poll group.\n"); 1146 spdk_nvmf_qpair_disconnect(qpair); 1147 } 1148 } 1149 1150 void 1151 spdk_nvmf_tgt_new_qpair(struct spdk_nvmf_tgt *tgt, struct spdk_nvmf_qpair *qpair) 1152 { 1153 struct spdk_nvmf_poll_group *group; 1154 struct nvmf_new_qpair_ctx *ctx; 1155 1156 group = spdk_nvmf_get_optimal_poll_group(qpair); 1157 if (group == NULL) { 1158 if (tgt->next_poll_group == NULL) { 1159 tgt->next_poll_group = TAILQ_FIRST(&tgt->poll_groups); 1160 if (tgt->next_poll_group == NULL) { 1161 SPDK_ERRLOG("No poll groups exist.\n"); 1162 spdk_nvmf_qpair_disconnect(qpair); 1163 return; 1164 } 1165 } 1166 group = tgt->next_poll_group; 1167 tgt->next_poll_group = TAILQ_NEXT(group, link); 1168 } 1169 1170 ctx = calloc(1, sizeof(*ctx)); 1171 if (!ctx) { 1172 SPDK_ERRLOG("Unable to send message to poll group.\n"); 1173 spdk_nvmf_qpair_disconnect(qpair); 1174 return; 1175 } 1176 1177 ctx->qpair = qpair; 1178 ctx->group = group; 1179 1180 pthread_mutex_lock(&group->mutex); 1181 group->current_unassociated_qpairs++; 1182 pthread_mutex_unlock(&group->mutex); 1183 1184 spdk_thread_send_msg(group->thread, _nvmf_poll_group_add, ctx); 1185 } 1186 1187 struct spdk_nvmf_poll_group * 1188 spdk_nvmf_poll_group_create(struct spdk_nvmf_tgt *tgt) 1189 { 1190 struct spdk_io_channel *ch; 1191 1192 ch = spdk_get_io_channel(tgt); 1193 if (!ch) { 1194 SPDK_ERRLOG("Unable to get I/O channel for target\n"); 1195 return NULL; 1196 } 1197 1198 return spdk_io_channel_get_ctx(ch); 1199 } 1200 1201 void 1202 spdk_nvmf_poll_group_destroy(struct spdk_nvmf_poll_group *group, 1203 spdk_nvmf_poll_group_destroy_done_fn cb_fn, 1204 void *cb_arg) 1205 { 1206 assert(group->destroy_cb_fn == NULL); 1207 group->destroy_cb_fn = cb_fn; 1208 group->destroy_cb_arg = cb_arg; 1209 1210 /* This function will put the io_channel associated with this poll group */ 1211 nvmf_tgt_destroy_poll_group_qpairs(group); 1212 } 1213 1214 int 1215 spdk_nvmf_poll_group_add(struct spdk_nvmf_poll_group *group, 1216 struct spdk_nvmf_qpair *qpair) 1217 { 1218 int rc; 1219 struct spdk_nvmf_transport_poll_group *tgroup; 1220 1221 TAILQ_INIT(&qpair->outstanding); 1222 qpair->group = group; 1223 qpair->ctrlr = NULL; 1224 qpair->disconnect_started = false; 1225 1226 tgroup = nvmf_get_transport_poll_group(group, qpair->transport); 1227 if (tgroup == NULL) { 1228 return -1; 1229 } 1230 1231 rc = nvmf_transport_poll_group_add(tgroup, qpair); 1232 1233 /* We add the qpair to the group only it is successfully added into the tgroup */ 1234 if (rc == 0) { 1235 SPDK_DTRACE_PROBE2_TICKS(nvmf_poll_group_add_qpair, qpair, spdk_thread_get_id(group->thread)); 1236 TAILQ_INSERT_TAIL(&group->qpairs, qpair, link); 1237 nvmf_qpair_set_state(qpair, SPDK_NVMF_QPAIR_CONNECTING); 1238 } 1239 1240 return rc; 1241 } 1242 1243 static void 1244 _nvmf_ctrlr_destruct(void *ctx) 1245 { 1246 struct spdk_nvmf_ctrlr *ctrlr = ctx; 1247 1248 nvmf_ctrlr_destruct(ctrlr); 1249 } 1250 1251 static void 1252 _nvmf_ctrlr_free_from_qpair(void *ctx) 1253 { 1254 struct nvmf_qpair_disconnect_ctx *qpair_ctx = ctx; 1255 struct spdk_nvmf_ctrlr *ctrlr = qpair_ctx->ctrlr; 1256 uint32_t count; 1257 1258 spdk_bit_array_clear(ctrlr->qpair_mask, qpair_ctx->qid); 1259 SPDK_DEBUGLOG(nvmf, "qpair_mask cleared, qid %u\n", qpair_ctx->qid); 1260 count = spdk_bit_array_count_set(ctrlr->qpair_mask); 1261 if (count == 0) { 1262 assert(!ctrlr->in_destruct); 1263 SPDK_DEBUGLOG(nvmf, "Last qpair %u, destroy ctrlr 0x%hx\n", qpair_ctx->qid, ctrlr->cntlid); 1264 ctrlr->in_destruct = true; 1265 spdk_thread_send_msg(ctrlr->subsys->thread, _nvmf_ctrlr_destruct, ctrlr); 1266 } 1267 free(qpair_ctx); 1268 } 1269 1270 static void 1271 _nvmf_transport_qpair_fini_complete(void *cb_ctx) 1272 { 1273 struct nvmf_qpair_disconnect_ctx *qpair_ctx = cb_ctx; 1274 struct spdk_nvmf_ctrlr *ctrlr; 1275 1276 ctrlr = qpair_ctx->ctrlr; 1277 SPDK_DEBUGLOG(nvmf, "Finish destroying qid %u\n", qpair_ctx->qid); 1278 1279 if (ctrlr) { 1280 if (qpair_ctx->qid == 0) { 1281 /* Admin qpair is removed, so set the pointer to NULL. 1282 * This operation is safe since we are on ctrlr thread now, admin qpair's thread is the same 1283 * as controller's thread */ 1284 assert(ctrlr->thread == spdk_get_thread()); 1285 ctrlr->admin_qpair = NULL; 1286 } 1287 /* Free qpair id from controller's bit mask and destroy the controller if it is the last qpair */ 1288 if (ctrlr->thread) { 1289 spdk_thread_send_msg(ctrlr->thread, _nvmf_ctrlr_free_from_qpair, qpair_ctx); 1290 } else { 1291 _nvmf_ctrlr_free_from_qpair(qpair_ctx); 1292 } 1293 } else { 1294 free(qpair_ctx); 1295 } 1296 } 1297 1298 void 1299 spdk_nvmf_poll_group_remove(struct spdk_nvmf_qpair *qpair) 1300 { 1301 struct spdk_nvmf_transport_poll_group *tgroup; 1302 int rc; 1303 1304 SPDK_DTRACE_PROBE2_TICKS(nvmf_poll_group_remove_qpair, qpair, 1305 spdk_thread_get_id(qpair->group->thread)); 1306 nvmf_qpair_set_state(qpair, SPDK_NVMF_QPAIR_ERROR); 1307 1308 /* Find the tgroup and remove the qpair from the tgroup */ 1309 tgroup = nvmf_get_transport_poll_group(qpair->group, qpair->transport); 1310 if (tgroup != NULL) { 1311 rc = nvmf_transport_poll_group_remove(tgroup, qpair); 1312 if (rc && (rc != ENOTSUP)) { 1313 SPDK_ERRLOG("Cannot remove qpair=%p from transport group=%p\n", 1314 qpair, tgroup); 1315 } 1316 } 1317 1318 TAILQ_REMOVE(&qpair->group->qpairs, qpair, link); 1319 qpair->group = NULL; 1320 } 1321 1322 static void 1323 _nvmf_qpair_sgroup_req_clean(struct spdk_nvmf_subsystem_poll_group *sgroup, 1324 const struct spdk_nvmf_qpair *qpair) 1325 { 1326 struct spdk_nvmf_request *req, *tmp; 1327 TAILQ_FOREACH_SAFE(req, &sgroup->queued, link, tmp) { 1328 if (req->qpair == qpair) { 1329 TAILQ_REMOVE(&sgroup->queued, req, link); 1330 if (nvmf_transport_req_free(req)) { 1331 SPDK_ERRLOG("Transport request free error!\n"); 1332 } 1333 } 1334 } 1335 } 1336 1337 static void 1338 _nvmf_qpair_destroy(void *ctx, int status) 1339 { 1340 struct nvmf_qpair_disconnect_ctx *qpair_ctx = ctx; 1341 struct spdk_nvmf_qpair *qpair = qpair_ctx->qpair; 1342 struct spdk_nvmf_ctrlr *ctrlr = qpair->ctrlr; 1343 struct spdk_nvmf_subsystem_poll_group *sgroup; 1344 uint32_t sid; 1345 1346 assert(qpair->state == SPDK_NVMF_QPAIR_DEACTIVATING); 1347 qpair_ctx->qid = qpair->qid; 1348 1349 if (qpair->connect_received) { 1350 if (0 == qpair->qid) { 1351 assert(qpair->group->stat.current_admin_qpairs > 0); 1352 qpair->group->stat.current_admin_qpairs--; 1353 } else { 1354 assert(qpair->group->stat.current_io_qpairs > 0); 1355 qpair->group->stat.current_io_qpairs--; 1356 } 1357 } else { 1358 pthread_mutex_lock(&qpair->group->mutex); 1359 qpair->group->current_unassociated_qpairs--; 1360 pthread_mutex_unlock(&qpair->group->mutex); 1361 } 1362 1363 if (ctrlr) { 1364 sgroup = &qpair->group->sgroups[ctrlr->subsys->id]; 1365 _nvmf_qpair_sgroup_req_clean(sgroup, qpair); 1366 } else { 1367 for (sid = 0; sid < qpair->group->num_sgroups; sid++) { 1368 sgroup = &qpair->group->sgroups[sid]; 1369 assert(sgroup != NULL); 1370 _nvmf_qpair_sgroup_req_clean(sgroup, qpair); 1371 } 1372 } 1373 1374 nvmf_qpair_auth_destroy(qpair); 1375 qpair_ctx->ctrlr = ctrlr; 1376 spdk_nvmf_poll_group_remove(qpair); 1377 nvmf_transport_qpair_fini(qpair, _nvmf_transport_qpair_fini_complete, qpair_ctx); 1378 } 1379 1380 static void 1381 _nvmf_qpair_disconnect_msg(void *ctx) 1382 { 1383 struct nvmf_qpair_disconnect_ctx *qpair_ctx = ctx; 1384 1385 spdk_nvmf_qpair_disconnect(qpair_ctx->qpair); 1386 free(ctx); 1387 } 1388 1389 int 1390 spdk_nvmf_qpair_disconnect(struct spdk_nvmf_qpair *qpair) 1391 { 1392 struct spdk_nvmf_poll_group *group = qpair->group; 1393 struct nvmf_qpair_disconnect_ctx *qpair_ctx; 1394 1395 if (__atomic_test_and_set(&qpair->disconnect_started, __ATOMIC_RELAXED)) { 1396 return -EINPROGRESS; 1397 } 1398 1399 /* If we get a qpair in the uninitialized state, we can just destroy it immediately */ 1400 if (qpair->state == SPDK_NVMF_QPAIR_UNINITIALIZED) { 1401 nvmf_transport_qpair_fini(qpair, NULL, NULL); 1402 return 0; 1403 } 1404 1405 assert(group != NULL); 1406 if (spdk_get_thread() != group->thread) { 1407 /* clear the atomic so we can set it on the next call on the proper thread. */ 1408 __atomic_clear(&qpair->disconnect_started, __ATOMIC_RELAXED); 1409 qpair_ctx = calloc(1, sizeof(struct nvmf_qpair_disconnect_ctx)); 1410 if (!qpair_ctx) { 1411 SPDK_ERRLOG("Unable to allocate context for nvmf_qpair_disconnect\n"); 1412 return -ENOMEM; 1413 } 1414 qpair_ctx->qpair = qpair; 1415 spdk_thread_send_msg(group->thread, _nvmf_qpair_disconnect_msg, qpair_ctx); 1416 return 0; 1417 } 1418 1419 SPDK_DTRACE_PROBE2_TICKS(nvmf_qpair_disconnect, qpair, spdk_thread_get_id(group->thread)); 1420 assert(spdk_nvmf_qpair_is_active(qpair)); 1421 nvmf_qpair_set_state(qpair, SPDK_NVMF_QPAIR_DEACTIVATING); 1422 1423 qpair_ctx = calloc(1, sizeof(struct nvmf_qpair_disconnect_ctx)); 1424 if (!qpair_ctx) { 1425 SPDK_ERRLOG("Unable to allocate context for nvmf_qpair_disconnect\n"); 1426 return -ENOMEM; 1427 } 1428 1429 qpair_ctx->qpair = qpair; 1430 1431 /* Check for outstanding I/O */ 1432 if (!TAILQ_EMPTY(&qpair->outstanding)) { 1433 SPDK_DTRACE_PROBE2_TICKS(nvmf_poll_group_drain_qpair, qpair, spdk_thread_get_id(group->thread)); 1434 qpair->state_cb = _nvmf_qpair_destroy; 1435 qpair->state_cb_arg = qpair_ctx; 1436 nvmf_qpair_abort_pending_zcopy_reqs(qpair); 1437 nvmf_qpair_free_aer(qpair); 1438 return 0; 1439 } 1440 1441 _nvmf_qpair_destroy(qpair_ctx, 0); 1442 1443 return 0; 1444 } 1445 1446 int 1447 spdk_nvmf_qpair_get_peer_trid(struct spdk_nvmf_qpair *qpair, 1448 struct spdk_nvme_transport_id *trid) 1449 { 1450 memset(trid, 0, sizeof(*trid)); 1451 return nvmf_transport_qpair_get_peer_trid(qpair, trid); 1452 } 1453 1454 int 1455 spdk_nvmf_qpair_get_local_trid(struct spdk_nvmf_qpair *qpair, 1456 struct spdk_nvme_transport_id *trid) 1457 { 1458 memset(trid, 0, sizeof(*trid)); 1459 return nvmf_transport_qpair_get_local_trid(qpair, trid); 1460 } 1461 1462 int 1463 spdk_nvmf_qpair_get_listen_trid(struct spdk_nvmf_qpair *qpair, 1464 struct spdk_nvme_transport_id *trid) 1465 { 1466 memset(trid, 0, sizeof(*trid)); 1467 return nvmf_transport_qpair_get_listen_trid(qpair, trid); 1468 } 1469 1470 static int 1471 poll_group_update_subsystem(struct spdk_nvmf_poll_group *group, 1472 struct spdk_nvmf_subsystem *subsystem) 1473 { 1474 struct spdk_nvmf_subsystem_poll_group *sgroup; 1475 uint32_t i, j; 1476 struct spdk_nvmf_ns *ns; 1477 struct spdk_nvmf_registrant *reg, *tmp; 1478 struct spdk_io_channel *ch; 1479 struct spdk_nvmf_subsystem_pg_ns_info *ns_info; 1480 struct spdk_nvmf_ctrlr *ctrlr; 1481 bool ns_changed; 1482 1483 /* Make sure our poll group has memory for this subsystem allocated */ 1484 if (subsystem->id >= group->num_sgroups) { 1485 return -ENOMEM; 1486 } 1487 1488 sgroup = &group->sgroups[subsystem->id]; 1489 1490 /* Make sure the array of namespace information is the correct size */ 1491 if (sgroup->num_ns == 0 && subsystem->max_nsid > 0) { 1492 /* First allocation */ 1493 sgroup->ns_info = calloc(subsystem->max_nsid, sizeof(struct spdk_nvmf_subsystem_pg_ns_info)); 1494 if (!sgroup->ns_info) { 1495 return -ENOMEM; 1496 } 1497 sgroup->num_ns = subsystem->max_nsid; 1498 } 1499 1500 ns_changed = false; 1501 1502 /* Detect bdevs that were added or removed */ 1503 for (i = 0; i < sgroup->num_ns; i++) { 1504 ns = subsystem->ns[i]; 1505 ns_info = &sgroup->ns_info[i]; 1506 ch = ns_info->channel; 1507 1508 if (ns == NULL && ch == NULL) { 1509 /* Both NULL. Leave empty */ 1510 } else if (ns == NULL && ch != NULL) { 1511 /* There was a channel here, but the namespace is gone. */ 1512 ns_changed = true; 1513 spdk_put_io_channel(ch); 1514 ns_info->channel = NULL; 1515 } else if (ns != NULL && ch == NULL) { 1516 /* A namespace appeared but there is no channel yet */ 1517 ns_changed = true; 1518 ch = spdk_bdev_get_io_channel(ns->desc); 1519 if (ch == NULL) { 1520 SPDK_ERRLOG("Could not allocate I/O channel.\n"); 1521 return -ENOMEM; 1522 } 1523 ns_info->channel = ch; 1524 } else if (spdk_uuid_compare(&ns_info->uuid, spdk_bdev_get_uuid(ns->bdev)) != 0) { 1525 /* A namespace was here before, but was replaced by a new one. */ 1526 ns_changed = true; 1527 spdk_put_io_channel(ns_info->channel); 1528 memset(ns_info, 0, sizeof(*ns_info)); 1529 1530 ch = spdk_bdev_get_io_channel(ns->desc); 1531 if (ch == NULL) { 1532 SPDK_ERRLOG("Could not allocate I/O channel.\n"); 1533 return -ENOMEM; 1534 } 1535 ns_info->channel = ch; 1536 } else if (ns_info->num_blocks != spdk_bdev_get_num_blocks(ns->bdev)) { 1537 /* Namespace is still there but size has changed */ 1538 SPDK_DEBUGLOG(nvmf, "Namespace resized: subsystem_id %u," 1539 " nsid %u, pg %p, old %" PRIu64 ", new %" PRIu64 "\n", 1540 subsystem->id, 1541 ns->nsid, 1542 group, 1543 ns_info->num_blocks, 1544 spdk_bdev_get_num_blocks(ns->bdev)); 1545 ns_changed = true; 1546 } 1547 1548 if (ns == NULL) { 1549 memset(ns_info, 0, sizeof(*ns_info)); 1550 } else { 1551 ns_info->uuid = *spdk_bdev_get_uuid(ns->bdev); 1552 ns_info->num_blocks = spdk_bdev_get_num_blocks(ns->bdev); 1553 ns_info->crkey = ns->crkey; 1554 ns_info->rtype = ns->rtype; 1555 if (ns->holder) { 1556 ns_info->holder_id = ns->holder->hostid; 1557 } 1558 1559 memset(&ns_info->reg_hostid, 0, SPDK_NVMF_MAX_NUM_REGISTRANTS * sizeof(struct spdk_uuid)); 1560 j = 0; 1561 TAILQ_FOREACH_SAFE(reg, &ns->registrants, link, tmp) { 1562 if (j >= SPDK_NVMF_MAX_NUM_REGISTRANTS) { 1563 SPDK_ERRLOG("Maximum %u registrants can support.\n", SPDK_NVMF_MAX_NUM_REGISTRANTS); 1564 return -EINVAL; 1565 } 1566 ns_info->reg_hostid[j++] = reg->hostid; 1567 } 1568 } 1569 } 1570 1571 if (ns_changed) { 1572 TAILQ_FOREACH(ctrlr, &subsystem->ctrlrs, link) { 1573 if (ctrlr->thread != spdk_get_thread()) { 1574 continue; 1575 } 1576 /* It is possible that a ctrlr was added but the admin_qpair hasn't been 1577 * assigned yet. 1578 */ 1579 if (!ctrlr->admin_qpair) { 1580 continue; 1581 } 1582 if (ctrlr->admin_qpair->group == group) { 1583 nvmf_ctrlr_async_event_ns_notice(ctrlr); 1584 nvmf_ctrlr_async_event_ana_change_notice(ctrlr); 1585 } 1586 } 1587 } 1588 1589 return 0; 1590 } 1591 1592 int 1593 nvmf_poll_group_update_subsystem(struct spdk_nvmf_poll_group *group, 1594 struct spdk_nvmf_subsystem *subsystem) 1595 { 1596 return poll_group_update_subsystem(group, subsystem); 1597 } 1598 1599 int 1600 nvmf_poll_group_add_subsystem(struct spdk_nvmf_poll_group *group, 1601 struct spdk_nvmf_subsystem *subsystem, 1602 spdk_nvmf_poll_group_mod_done cb_fn, void *cb_arg) 1603 { 1604 int rc = 0; 1605 struct spdk_nvmf_subsystem_poll_group *sgroup = &group->sgroups[subsystem->id]; 1606 struct spdk_nvmf_request *req, *tmp; 1607 uint32_t i; 1608 1609 if (!TAILQ_EMPTY(&sgroup->queued)) { 1610 SPDK_ERRLOG("sgroup->queued not empty when adding subsystem\n"); 1611 TAILQ_FOREACH_SAFE(req, &sgroup->queued, link, tmp) { 1612 TAILQ_REMOVE(&sgroup->queued, req, link); 1613 if (nvmf_transport_req_free(req)) { 1614 SPDK_ERRLOG("Transport request free error!\n"); 1615 } 1616 } 1617 assert(false); 1618 } 1619 1620 rc = poll_group_update_subsystem(group, subsystem); 1621 if (rc) { 1622 nvmf_poll_group_remove_subsystem(group, subsystem, NULL, NULL); 1623 goto fini; 1624 } 1625 1626 sgroup->state = SPDK_NVMF_SUBSYSTEM_ACTIVE; 1627 1628 for (i = 0; i < sgroup->num_ns; i++) { 1629 sgroup->ns_info[i].state = SPDK_NVMF_SUBSYSTEM_ACTIVE; 1630 } 1631 1632 fini: 1633 if (cb_fn) { 1634 cb_fn(cb_arg, rc); 1635 } 1636 1637 SPDK_DTRACE_PROBE2_TICKS(nvmf_poll_group_add_subsystem, spdk_thread_get_id(group->thread), 1638 subsystem->subnqn); 1639 1640 return rc; 1641 } 1642 1643 static void 1644 _nvmf_poll_group_remove_subsystem_cb(void *ctx, int status) 1645 { 1646 struct nvmf_qpair_disconnect_many_ctx *qpair_ctx = ctx; 1647 struct spdk_nvmf_subsystem *subsystem; 1648 struct spdk_nvmf_poll_group *group; 1649 struct spdk_nvmf_subsystem_poll_group *sgroup; 1650 spdk_nvmf_poll_group_mod_done cpl_fn = NULL; 1651 void *cpl_ctx = NULL; 1652 uint32_t nsid; 1653 1654 group = qpair_ctx->group; 1655 subsystem = qpair_ctx->subsystem; 1656 cpl_fn = qpair_ctx->cpl_fn; 1657 cpl_ctx = qpair_ctx->cpl_ctx; 1658 sgroup = &group->sgroups[subsystem->id]; 1659 1660 if (status) { 1661 goto fini; 1662 } 1663 1664 for (nsid = 0; nsid < sgroup->num_ns; nsid++) { 1665 if (sgroup->ns_info[nsid].channel) { 1666 spdk_put_io_channel(sgroup->ns_info[nsid].channel); 1667 sgroup->ns_info[nsid].channel = NULL; 1668 } 1669 } 1670 1671 sgroup->num_ns = 0; 1672 free(sgroup->ns_info); 1673 sgroup->ns_info = NULL; 1674 fini: 1675 free(qpair_ctx); 1676 if (cpl_fn) { 1677 cpl_fn(cpl_ctx, status); 1678 } 1679 } 1680 1681 static void nvmf_poll_group_remove_subsystem_msg(void *ctx); 1682 1683 static void 1684 nvmf_poll_group_remove_subsystem_msg(void *ctx) 1685 { 1686 struct spdk_nvmf_qpair *qpair, *qpair_tmp; 1687 struct spdk_nvmf_subsystem *subsystem; 1688 struct spdk_nvmf_poll_group *group; 1689 struct nvmf_qpair_disconnect_many_ctx *qpair_ctx = ctx; 1690 bool qpairs_found = false; 1691 int rc = 0; 1692 1693 group = qpair_ctx->group; 1694 subsystem = qpair_ctx->subsystem; 1695 1696 TAILQ_FOREACH_SAFE(qpair, &group->qpairs, link, qpair_tmp) { 1697 if ((qpair->ctrlr != NULL) && (qpair->ctrlr->subsys == subsystem)) { 1698 qpairs_found = true; 1699 rc = spdk_nvmf_qpair_disconnect(qpair); 1700 if (rc && rc != -EINPROGRESS) { 1701 break; 1702 } 1703 } 1704 } 1705 1706 if (!qpairs_found) { 1707 _nvmf_poll_group_remove_subsystem_cb(ctx, 0); 1708 return; 1709 } 1710 1711 /* Some qpairs are in process of being disconnected. Send a message and try to remove them again */ 1712 spdk_thread_send_msg(spdk_get_thread(), nvmf_poll_group_remove_subsystem_msg, ctx); 1713 } 1714 1715 void 1716 nvmf_poll_group_remove_subsystem(struct spdk_nvmf_poll_group *group, 1717 struct spdk_nvmf_subsystem *subsystem, 1718 spdk_nvmf_poll_group_mod_done cb_fn, void *cb_arg) 1719 { 1720 struct spdk_nvmf_subsystem_poll_group *sgroup; 1721 struct nvmf_qpair_disconnect_many_ctx *ctx; 1722 uint32_t i; 1723 1724 SPDK_DTRACE_PROBE3_TICKS(nvmf_poll_group_remove_subsystem, group, spdk_thread_get_id(group->thread), 1725 subsystem->subnqn); 1726 1727 ctx = calloc(1, sizeof(struct nvmf_qpair_disconnect_many_ctx)); 1728 if (!ctx) { 1729 SPDK_ERRLOG("Unable to allocate memory for context to remove poll subsystem\n"); 1730 if (cb_fn) { 1731 cb_fn(cb_arg, -1); 1732 } 1733 return; 1734 } 1735 1736 ctx->group = group; 1737 ctx->subsystem = subsystem; 1738 ctx->cpl_fn = cb_fn; 1739 ctx->cpl_ctx = cb_arg; 1740 1741 sgroup = &group->sgroups[subsystem->id]; 1742 sgroup->state = SPDK_NVMF_SUBSYSTEM_INACTIVE; 1743 1744 for (i = 0; i < sgroup->num_ns; i++) { 1745 sgroup->ns_info[i].state = SPDK_NVMF_SUBSYSTEM_INACTIVE; 1746 } 1747 1748 nvmf_poll_group_remove_subsystem_msg(ctx); 1749 } 1750 1751 void 1752 nvmf_poll_group_pause_subsystem(struct spdk_nvmf_poll_group *group, 1753 struct spdk_nvmf_subsystem *subsystem, 1754 uint32_t nsid, 1755 spdk_nvmf_poll_group_mod_done cb_fn, void *cb_arg) 1756 { 1757 struct spdk_nvmf_subsystem_poll_group *sgroup; 1758 struct spdk_nvmf_subsystem_pg_ns_info *ns_info = NULL; 1759 int rc = 0; 1760 uint32_t i; 1761 1762 if (subsystem->id >= group->num_sgroups) { 1763 rc = -1; 1764 goto fini; 1765 } 1766 1767 sgroup = &group->sgroups[subsystem->id]; 1768 if (sgroup->state == SPDK_NVMF_SUBSYSTEM_PAUSED) { 1769 goto fini; 1770 } 1771 sgroup->state = SPDK_NVMF_SUBSYSTEM_PAUSING; 1772 1773 if (nsid == SPDK_NVME_GLOBAL_NS_TAG) { 1774 for (i = 0; i < sgroup->num_ns; i++) { 1775 ns_info = &sgroup->ns_info[i]; 1776 ns_info->state = SPDK_NVMF_SUBSYSTEM_PAUSING; 1777 } 1778 } else { 1779 /* NOTE: This implicitly also checks for 0, since 0 - 1 wraps around to UINT32_MAX. */ 1780 if (nsid - 1 < sgroup->num_ns) { 1781 ns_info = &sgroup->ns_info[nsid - 1]; 1782 ns_info->state = SPDK_NVMF_SUBSYSTEM_PAUSING; 1783 } 1784 } 1785 1786 if (sgroup->mgmt_io_outstanding > 0) { 1787 assert(sgroup->cb_fn == NULL); 1788 sgroup->cb_fn = cb_fn; 1789 assert(sgroup->cb_arg == NULL); 1790 sgroup->cb_arg = cb_arg; 1791 return; 1792 } 1793 1794 if (nsid == SPDK_NVME_GLOBAL_NS_TAG) { 1795 for (i = 0; i < sgroup->num_ns; i++) { 1796 ns_info = &sgroup->ns_info[i]; 1797 1798 if (ns_info->io_outstanding > 0) { 1799 assert(sgroup->cb_fn == NULL); 1800 sgroup->cb_fn = cb_fn; 1801 assert(sgroup->cb_arg == NULL); 1802 sgroup->cb_arg = cb_arg; 1803 return; 1804 } 1805 } 1806 } else { 1807 if (ns_info != NULL && ns_info->io_outstanding > 0) { 1808 assert(sgroup->cb_fn == NULL); 1809 sgroup->cb_fn = cb_fn; 1810 assert(sgroup->cb_arg == NULL); 1811 sgroup->cb_arg = cb_arg; 1812 return; 1813 } 1814 } 1815 1816 assert(sgroup->mgmt_io_outstanding == 0); 1817 sgroup->state = SPDK_NVMF_SUBSYSTEM_PAUSED; 1818 fini: 1819 if (cb_fn) { 1820 cb_fn(cb_arg, rc); 1821 } 1822 } 1823 1824 void 1825 nvmf_poll_group_resume_subsystem(struct spdk_nvmf_poll_group *group, 1826 struct spdk_nvmf_subsystem *subsystem, 1827 spdk_nvmf_poll_group_mod_done cb_fn, void *cb_arg) 1828 { 1829 struct spdk_nvmf_request *req, *tmp; 1830 struct spdk_nvmf_subsystem_poll_group *sgroup; 1831 int rc = 0; 1832 uint32_t i; 1833 1834 if (subsystem->id >= group->num_sgroups) { 1835 rc = -1; 1836 goto fini; 1837 } 1838 1839 sgroup = &group->sgroups[subsystem->id]; 1840 1841 if (sgroup->state == SPDK_NVMF_SUBSYSTEM_ACTIVE) { 1842 goto fini; 1843 } 1844 1845 rc = poll_group_update_subsystem(group, subsystem); 1846 if (rc) { 1847 goto fini; 1848 } 1849 1850 for (i = 0; i < sgroup->num_ns; i++) { 1851 sgroup->ns_info[i].state = SPDK_NVMF_SUBSYSTEM_ACTIVE; 1852 } 1853 1854 sgroup->state = SPDK_NVMF_SUBSYSTEM_ACTIVE; 1855 1856 /* Release all queued requests */ 1857 TAILQ_FOREACH_SAFE(req, &sgroup->queued, link, tmp) { 1858 TAILQ_REMOVE(&sgroup->queued, req, link); 1859 if (spdk_nvmf_request_using_zcopy(req)) { 1860 spdk_nvmf_request_zcopy_start(req); 1861 } else { 1862 spdk_nvmf_request_exec(req); 1863 } 1864 1865 } 1866 fini: 1867 if (cb_fn) { 1868 cb_fn(cb_arg, rc); 1869 } 1870 } 1871 1872 1873 struct spdk_nvmf_poll_group * 1874 spdk_nvmf_get_optimal_poll_group(struct spdk_nvmf_qpair *qpair) 1875 { 1876 struct spdk_nvmf_transport_poll_group *tgroup; 1877 1878 tgroup = nvmf_transport_get_optimal_poll_group(qpair->transport, qpair); 1879 1880 if (tgroup == NULL) { 1881 return NULL; 1882 } 1883 1884 return tgroup->group; 1885 } 1886 1887 void 1888 spdk_nvmf_poll_group_dump_stat(struct spdk_nvmf_poll_group *group, struct spdk_json_write_ctx *w) 1889 { 1890 struct spdk_nvmf_transport_poll_group *tgroup; 1891 1892 spdk_json_write_object_begin(w); 1893 1894 spdk_json_write_named_string(w, "name", spdk_thread_get_name(spdk_get_thread())); 1895 spdk_json_write_named_uint32(w, "admin_qpairs", group->stat.admin_qpairs); 1896 spdk_json_write_named_uint32(w, "io_qpairs", group->stat.io_qpairs); 1897 spdk_json_write_named_uint32(w, "current_admin_qpairs", group->stat.current_admin_qpairs); 1898 spdk_json_write_named_uint32(w, "current_io_qpairs", group->stat.current_io_qpairs); 1899 spdk_json_write_named_uint64(w, "pending_bdev_io", group->stat.pending_bdev_io); 1900 spdk_json_write_named_uint64(w, "completed_nvme_io", group->stat.completed_nvme_io); 1901 1902 spdk_json_write_named_array_begin(w, "transports"); 1903 1904 TAILQ_FOREACH(tgroup, &group->tgroups, link) { 1905 spdk_json_write_object_begin(w); 1906 /* 1907 * The trtype field intentionally contains a transport name as this is more informative. 1908 * The field has not been renamed for backward compatibility. 1909 */ 1910 spdk_json_write_named_string(w, "trtype", spdk_nvmf_get_transport_name(tgroup->transport)); 1911 1912 if (tgroup->transport->ops->poll_group_dump_stat) { 1913 tgroup->transport->ops->poll_group_dump_stat(tgroup, w); 1914 } 1915 1916 spdk_json_write_object_end(w); 1917 } 1918 1919 spdk_json_write_array_end(w); 1920 spdk_json_write_object_end(w); 1921 } 1922