1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2016 Intel Corporation. All rights reserved. 3 * Copyright (c) 2018-2019, 2021 Mellanox Technologies LTD. All rights reserved. 4 * Copyright (c) 2021, 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 5 */ 6 7 #include "spdk/stdinc.h" 8 9 #include "spdk/bdev.h" 10 #include "spdk/bit_array.h" 11 #include "spdk/thread.h" 12 #include "spdk/nvmf.h" 13 #include "spdk/endian.h" 14 #include "spdk/string.h" 15 #include "spdk/log.h" 16 #include "spdk_internal/usdt.h" 17 18 #include "nvmf_internal.h" 19 #include "transport.h" 20 21 SPDK_LOG_REGISTER_COMPONENT(nvmf) 22 23 #define SPDK_NVMF_DEFAULT_MAX_SUBSYSTEMS 1024 24 25 static TAILQ_HEAD(, spdk_nvmf_tgt) g_nvmf_tgts = TAILQ_HEAD_INITIALIZER(g_nvmf_tgts); 26 27 typedef void (*nvmf_qpair_disconnect_cpl)(void *ctx, int status); 28 29 /* supplied to a single call to nvmf_qpair_disconnect */ 30 struct nvmf_qpair_disconnect_ctx { 31 struct spdk_nvmf_qpair *qpair; 32 struct spdk_nvmf_ctrlr *ctrlr; 33 uint16_t qid; 34 }; 35 36 /* 37 * There are several times when we need to iterate through the list of all qpairs and selectively delete them. 38 * In order to do this sequentially without overlap, we must provide a context to recover the next qpair from 39 * to enable calling nvmf_qpair_disconnect on the next desired qpair. 40 */ 41 struct nvmf_qpair_disconnect_many_ctx { 42 struct spdk_nvmf_subsystem *subsystem; 43 struct spdk_nvmf_poll_group *group; 44 spdk_nvmf_poll_group_mod_done cpl_fn; 45 void *cpl_ctx; 46 }; 47 48 static struct spdk_nvmf_referral * 49 nvmf_tgt_find_referral(struct spdk_nvmf_tgt *tgt, 50 const struct spdk_nvme_transport_id *trid) 51 { 52 struct spdk_nvmf_referral *referral; 53 54 TAILQ_FOREACH(referral, &tgt->referrals, link) { 55 if (spdk_nvme_transport_id_compare(&referral->trid, trid) == 0) { 56 return referral; 57 } 58 } 59 60 return NULL; 61 } 62 63 int 64 spdk_nvmf_tgt_add_referral(struct spdk_nvmf_tgt *tgt, 65 const struct spdk_nvmf_referral_opts *uopts) 66 { 67 struct spdk_nvmf_referral *referral; 68 struct spdk_nvmf_referral_opts opts = {}; 69 struct spdk_nvme_transport_id *trid = &opts.trid; 70 71 memcpy(&opts, uopts, spdk_min(uopts->size, sizeof(opts))); 72 if (trid->subnqn[0] == '\0') { 73 snprintf(trid->subnqn, sizeof(trid->subnqn), "%s", SPDK_NVMF_DISCOVERY_NQN); 74 } 75 76 if (!nvmf_nqn_is_valid(trid->subnqn)) { 77 SPDK_ERRLOG("Invalid subsystem NQN\n"); 78 return -EINVAL; 79 } 80 81 /* If the entry already exists, just ignore it. */ 82 if (nvmf_tgt_find_referral(tgt, trid)) { 83 return 0; 84 } 85 86 referral = calloc(1, sizeof(*referral)); 87 if (!referral) { 88 SPDK_ERRLOG("Failed to allocate memory for a referral\n"); 89 return -ENOMEM; 90 } 91 92 referral->entry.subtype = nvmf_nqn_is_discovery(trid->subnqn) ? 93 SPDK_NVMF_SUBTYPE_DISCOVERY : 94 SPDK_NVMF_SUBTYPE_NVME; 95 referral->entry.treq.secure_channel = opts.secure_channel ? 96 SPDK_NVMF_TREQ_SECURE_CHANNEL_REQUIRED : 97 SPDK_NVMF_TREQ_SECURE_CHANNEL_NOT_REQUIRED; 98 referral->entry.cntlid = 0xffff; 99 referral->entry.trtype = trid->trtype; 100 referral->entry.adrfam = trid->adrfam; 101 memcpy(&referral->trid, trid, sizeof(struct spdk_nvme_transport_id)); 102 spdk_strcpy_pad(referral->entry.subnqn, trid->subnqn, sizeof(trid->subnqn), '\0'); 103 spdk_strcpy_pad(referral->entry.trsvcid, trid->trsvcid, sizeof(referral->entry.trsvcid), ' '); 104 spdk_strcpy_pad(referral->entry.traddr, trid->traddr, sizeof(referral->entry.traddr), ' '); 105 106 TAILQ_INSERT_HEAD(&tgt->referrals, referral, link); 107 spdk_nvmf_send_discovery_log_notice(tgt, NULL); 108 109 return 0; 110 } 111 112 int 113 spdk_nvmf_tgt_remove_referral(struct spdk_nvmf_tgt *tgt, 114 const struct spdk_nvmf_referral_opts *uopts) 115 { 116 struct spdk_nvmf_referral *referral; 117 struct spdk_nvmf_referral_opts opts = {}; 118 struct spdk_nvme_transport_id *trid = &opts.trid; 119 120 memcpy(&opts, uopts, spdk_min(uopts->size, sizeof(opts))); 121 if (trid->subnqn[0] == '\0') { 122 snprintf(trid->subnqn, sizeof(trid->subnqn), "%s", SPDK_NVMF_DISCOVERY_NQN); 123 } 124 125 referral = nvmf_tgt_find_referral(tgt, &opts.trid); 126 if (referral == NULL) { 127 return -ENOENT; 128 } 129 130 TAILQ_REMOVE(&tgt->referrals, referral, link); 131 spdk_nvmf_send_discovery_log_notice(tgt, NULL); 132 133 free(referral); 134 135 return 0; 136 } 137 138 void 139 nvmf_qpair_set_state(struct spdk_nvmf_qpair *qpair, 140 enum spdk_nvmf_qpair_state state) 141 { 142 assert(qpair != NULL); 143 assert(qpair->group->thread == spdk_get_thread()); 144 145 qpair->state = state; 146 } 147 148 static int 149 nvmf_poll_group_poll(void *ctx) 150 { 151 struct spdk_nvmf_poll_group *group = ctx; 152 int rc; 153 int count = 0; 154 struct spdk_nvmf_transport_poll_group *tgroup; 155 156 TAILQ_FOREACH(tgroup, &group->tgroups, link) { 157 rc = nvmf_transport_poll_group_poll(tgroup); 158 if (rc < 0) { 159 return SPDK_POLLER_BUSY; 160 } 161 count += rc; 162 } 163 164 return count > 0 ? SPDK_POLLER_BUSY : SPDK_POLLER_IDLE; 165 } 166 167 /* 168 * Reset and clean up the poll group (I/O channel code will actually free the 169 * group). 170 */ 171 static void 172 nvmf_tgt_cleanup_poll_group(struct spdk_nvmf_poll_group *group) 173 { 174 struct spdk_nvmf_transport_poll_group *tgroup, *tmp; 175 struct spdk_nvmf_subsystem_poll_group *sgroup; 176 uint32_t sid, nsid; 177 178 TAILQ_FOREACH_SAFE(tgroup, &group->tgroups, link, tmp) { 179 TAILQ_REMOVE(&group->tgroups, tgroup, link); 180 nvmf_transport_poll_group_destroy(tgroup); 181 } 182 183 for (sid = 0; sid < group->num_sgroups; sid++) { 184 sgroup = &group->sgroups[sid]; 185 186 assert(sgroup != NULL); 187 188 for (nsid = 0; nsid < sgroup->num_ns; nsid++) { 189 if (sgroup->ns_info[nsid].channel) { 190 spdk_put_io_channel(sgroup->ns_info[nsid].channel); 191 sgroup->ns_info[nsid].channel = NULL; 192 } 193 } 194 195 free(sgroup->ns_info); 196 } 197 198 free(group->sgroups); 199 200 spdk_poller_unregister(&group->poller); 201 202 if (group->destroy_cb_fn) { 203 group->destroy_cb_fn(group->destroy_cb_arg, 0); 204 } 205 } 206 207 /* 208 * Callback to unregister a poll group from the target, and clean up its state. 209 */ 210 static void 211 nvmf_tgt_destroy_poll_group(void *io_device, void *ctx_buf) 212 { 213 struct spdk_nvmf_tgt *tgt = io_device; 214 struct spdk_nvmf_poll_group *group = ctx_buf; 215 216 SPDK_DTRACE_PROBE1_TICKS(nvmf_destroy_poll_group, spdk_thread_get_id(group->thread)); 217 218 pthread_mutex_lock(&tgt->mutex); 219 TAILQ_REMOVE(&tgt->poll_groups, group, link); 220 tgt->num_poll_groups--; 221 pthread_mutex_unlock(&tgt->mutex); 222 223 assert(!(tgt->state == NVMF_TGT_PAUSING || tgt->state == NVMF_TGT_RESUMING)); 224 nvmf_tgt_cleanup_poll_group(group); 225 } 226 227 static int 228 nvmf_poll_group_add_transport(struct spdk_nvmf_poll_group *group, 229 struct spdk_nvmf_transport *transport) 230 { 231 struct spdk_nvmf_transport_poll_group *tgroup = nvmf_get_transport_poll_group(group, transport); 232 233 if (tgroup != NULL) { 234 /* Transport already in the poll group */ 235 return 0; 236 } 237 238 tgroup = nvmf_transport_poll_group_create(transport, group); 239 if (!tgroup) { 240 SPDK_ERRLOG("Unable to create poll group for transport\n"); 241 return -1; 242 } 243 SPDK_DTRACE_PROBE2_TICKS(nvmf_transport_poll_group_create, transport, 244 spdk_thread_get_id(group->thread)); 245 246 tgroup->group = group; 247 TAILQ_INSERT_TAIL(&group->tgroups, tgroup, link); 248 249 return 0; 250 } 251 252 static int 253 nvmf_tgt_create_poll_group(void *io_device, void *ctx_buf) 254 { 255 struct spdk_nvmf_tgt *tgt = io_device; 256 struct spdk_nvmf_poll_group *group = ctx_buf; 257 struct spdk_nvmf_transport *transport; 258 struct spdk_nvmf_subsystem *subsystem; 259 struct spdk_thread *thread = spdk_get_thread(); 260 uint32_t i; 261 int rc; 262 263 group->tgt = tgt; 264 TAILQ_INIT(&group->tgroups); 265 TAILQ_INIT(&group->qpairs); 266 group->thread = thread; 267 pthread_mutex_init(&group->mutex, NULL); 268 269 group->poller = SPDK_POLLER_REGISTER(nvmf_poll_group_poll, group, 0); 270 spdk_poller_register_interrupt(group->poller, NULL, NULL); 271 272 SPDK_DTRACE_PROBE1_TICKS(nvmf_create_poll_group, spdk_thread_get_id(thread)); 273 274 TAILQ_FOREACH(transport, &tgt->transports, link) { 275 rc = nvmf_poll_group_add_transport(group, transport); 276 if (rc != 0) { 277 nvmf_tgt_cleanup_poll_group(group); 278 return rc; 279 } 280 } 281 282 group->num_sgroups = tgt->max_subsystems; 283 group->sgroups = calloc(tgt->max_subsystems, sizeof(struct spdk_nvmf_subsystem_poll_group)); 284 if (!group->sgroups) { 285 nvmf_tgt_cleanup_poll_group(group); 286 return -ENOMEM; 287 } 288 289 for (i = 0; i < tgt->max_subsystems; i++) { 290 TAILQ_INIT(&group->sgroups[i].queued); 291 } 292 293 for (subsystem = spdk_nvmf_subsystem_get_first(tgt); 294 subsystem != NULL; 295 subsystem = spdk_nvmf_subsystem_get_next(subsystem)) { 296 if (nvmf_poll_group_add_subsystem(group, subsystem, NULL, NULL) != 0) { 297 nvmf_tgt_cleanup_poll_group(group); 298 return -1; 299 } 300 } 301 302 pthread_mutex_lock(&tgt->mutex); 303 tgt->num_poll_groups++; 304 TAILQ_INSERT_TAIL(&tgt->poll_groups, group, link); 305 pthread_mutex_unlock(&tgt->mutex); 306 307 return 0; 308 } 309 310 static void 311 _nvmf_tgt_disconnect_qpairs(void *ctx) 312 { 313 struct spdk_nvmf_qpair *qpair, *qpair_tmp; 314 struct nvmf_qpair_disconnect_many_ctx *qpair_ctx = ctx; 315 struct spdk_nvmf_poll_group *group = qpair_ctx->group; 316 struct spdk_io_channel *ch; 317 int rc; 318 319 TAILQ_FOREACH_SAFE(qpair, &group->qpairs, link, qpair_tmp) { 320 rc = spdk_nvmf_qpair_disconnect(qpair); 321 if (rc && rc != -EINPROGRESS) { 322 break; 323 } 324 } 325 326 if (TAILQ_EMPTY(&group->qpairs)) { 327 /* When the refcount from the channels reaches 0, nvmf_tgt_destroy_poll_group will be called. */ 328 ch = spdk_io_channel_from_ctx(group); 329 spdk_put_io_channel(ch); 330 free(qpair_ctx); 331 return; 332 } 333 334 /* Some qpairs are in process of being disconnected. Send a message and try to remove them again */ 335 spdk_thread_send_msg(spdk_get_thread(), _nvmf_tgt_disconnect_qpairs, ctx); 336 } 337 338 static void 339 nvmf_tgt_destroy_poll_group_qpairs(struct spdk_nvmf_poll_group *group) 340 { 341 struct nvmf_qpair_disconnect_many_ctx *ctx; 342 343 SPDK_DTRACE_PROBE1_TICKS(nvmf_destroy_poll_group_qpairs, spdk_thread_get_id(group->thread)); 344 345 ctx = calloc(1, sizeof(struct nvmf_qpair_disconnect_many_ctx)); 346 if (!ctx) { 347 SPDK_ERRLOG("Failed to allocate memory for destroy poll group ctx\n"); 348 return; 349 } 350 351 ctx->group = group; 352 _nvmf_tgt_disconnect_qpairs(ctx); 353 } 354 355 struct spdk_nvmf_tgt * 356 spdk_nvmf_tgt_create(struct spdk_nvmf_target_opts *_opts) 357 { 358 struct spdk_nvmf_tgt *tgt, *tmp_tgt; 359 struct spdk_nvmf_target_opts opts = { 360 .max_subsystems = SPDK_NVMF_DEFAULT_MAX_SUBSYSTEMS, 361 .discovery_filter = SPDK_NVMF_TGT_DISCOVERY_MATCH_ANY, 362 }; 363 364 memcpy(&opts, _opts, _opts->size); 365 if (strnlen(opts.name, NVMF_TGT_NAME_MAX_LENGTH) == NVMF_TGT_NAME_MAX_LENGTH) { 366 SPDK_ERRLOG("Provided target name exceeds the max length of %u.\n", NVMF_TGT_NAME_MAX_LENGTH); 367 return NULL; 368 } 369 370 TAILQ_FOREACH(tmp_tgt, &g_nvmf_tgts, link) { 371 if (!strncmp(opts.name, tmp_tgt->name, NVMF_TGT_NAME_MAX_LENGTH)) { 372 SPDK_ERRLOG("Provided target name must be unique.\n"); 373 return NULL; 374 } 375 } 376 377 tgt = calloc(1, sizeof(*tgt)); 378 if (!tgt) { 379 return NULL; 380 } 381 382 snprintf(tgt->name, NVMF_TGT_NAME_MAX_LENGTH, "%s", opts.name); 383 384 if (!opts.max_subsystems) { 385 tgt->max_subsystems = SPDK_NVMF_DEFAULT_MAX_SUBSYSTEMS; 386 } else { 387 tgt->max_subsystems = opts.max_subsystems; 388 } 389 390 tgt->crdt[0] = opts.crdt[0]; 391 tgt->crdt[1] = opts.crdt[1]; 392 tgt->crdt[2] = opts.crdt[2]; 393 tgt->discovery_filter = opts.discovery_filter; 394 tgt->discovery_genctr = 0; 395 tgt->dhchap_digests = opts.dhchap_digests; 396 tgt->dhchap_dhgroups = opts.dhchap_dhgroups; 397 TAILQ_INIT(&tgt->transports); 398 TAILQ_INIT(&tgt->poll_groups); 399 TAILQ_INIT(&tgt->referrals); 400 tgt->num_poll_groups = 0; 401 402 tgt->subsystem_ids = spdk_bit_array_create(tgt->max_subsystems); 403 if (tgt->subsystem_ids == NULL) { 404 free(tgt); 405 return NULL; 406 } 407 408 RB_INIT(&tgt->subsystems); 409 410 pthread_mutex_init(&tgt->mutex, NULL); 411 412 spdk_io_device_register(tgt, 413 nvmf_tgt_create_poll_group, 414 nvmf_tgt_destroy_poll_group, 415 sizeof(struct spdk_nvmf_poll_group), 416 tgt->name); 417 418 tgt->state = NVMF_TGT_RUNNING; 419 420 TAILQ_INSERT_HEAD(&g_nvmf_tgts, tgt, link); 421 422 return tgt; 423 } 424 425 static void 426 _nvmf_tgt_destroy_next_transport(void *ctx) 427 { 428 struct spdk_nvmf_tgt *tgt = ctx; 429 struct spdk_nvmf_transport *transport; 430 431 if (!TAILQ_EMPTY(&tgt->transports)) { 432 transport = TAILQ_FIRST(&tgt->transports); 433 TAILQ_REMOVE(&tgt->transports, transport, link); 434 spdk_nvmf_transport_destroy(transport, _nvmf_tgt_destroy_next_transport, tgt); 435 } else { 436 spdk_nvmf_tgt_destroy_done_fn *destroy_cb_fn = tgt->destroy_cb_fn; 437 void *destroy_cb_arg = tgt->destroy_cb_arg; 438 439 pthread_mutex_destroy(&tgt->mutex); 440 free(tgt); 441 442 if (destroy_cb_fn) { 443 destroy_cb_fn(destroy_cb_arg, 0); 444 } 445 } 446 } 447 448 static void 449 nvmf_tgt_destroy_cb(void *io_device) 450 { 451 struct spdk_nvmf_tgt *tgt = io_device; 452 struct spdk_nvmf_subsystem *subsystem, *subsystem_next; 453 int rc; 454 struct spdk_nvmf_referral *referral; 455 456 while ((referral = TAILQ_FIRST(&tgt->referrals))) { 457 TAILQ_REMOVE(&tgt->referrals, referral, link); 458 free(referral); 459 } 460 461 nvmf_tgt_stop_mdns_prr(tgt); 462 463 /* We will be freeing subsystems in this loop, so we always need to get the next one 464 * ahead of time, since we can't call get_next() on a subsystem that's been freed. 465 */ 466 for (subsystem = spdk_nvmf_subsystem_get_first(tgt), 467 subsystem_next = spdk_nvmf_subsystem_get_next(subsystem); 468 subsystem != NULL; 469 subsystem = subsystem_next, 470 subsystem_next = spdk_nvmf_subsystem_get_next(subsystem_next)) { 471 nvmf_subsystem_remove_all_listeners(subsystem, true); 472 473 rc = spdk_nvmf_subsystem_destroy(subsystem, nvmf_tgt_destroy_cb, tgt); 474 if (rc) { 475 if (rc == -EINPROGRESS) { 476 /* If rc is -EINPROGRESS, nvmf_tgt_destroy_cb will be called again when subsystem #i 477 * is destroyed, nvmf_tgt_destroy_cb will continue to destroy other subsystems if any */ 478 return; 479 } else { 480 SPDK_ERRLOG("Failed to destroy subsystem %s, rc %d\n", subsystem->subnqn, rc); 481 } 482 } 483 } 484 spdk_bit_array_free(&tgt->subsystem_ids); 485 _nvmf_tgt_destroy_next_transport(tgt); 486 } 487 488 void 489 spdk_nvmf_tgt_destroy(struct spdk_nvmf_tgt *tgt, 490 spdk_nvmf_tgt_destroy_done_fn cb_fn, 491 void *cb_arg) 492 { 493 assert(!(tgt->state == NVMF_TGT_PAUSING || tgt->state == NVMF_TGT_RESUMING)); 494 495 tgt->destroy_cb_fn = cb_fn; 496 tgt->destroy_cb_arg = cb_arg; 497 498 TAILQ_REMOVE(&g_nvmf_tgts, tgt, link); 499 500 spdk_io_device_unregister(tgt, nvmf_tgt_destroy_cb); 501 } 502 503 const char * 504 spdk_nvmf_tgt_get_name(struct spdk_nvmf_tgt *tgt) 505 { 506 return tgt->name; 507 } 508 509 struct spdk_nvmf_tgt * 510 spdk_nvmf_get_tgt(const char *name) 511 { 512 struct spdk_nvmf_tgt *tgt; 513 uint32_t num_targets = 0; 514 515 TAILQ_FOREACH(tgt, &g_nvmf_tgts, link) { 516 if (name) { 517 if (!strncmp(tgt->name, name, NVMF_TGT_NAME_MAX_LENGTH)) { 518 return tgt; 519 } 520 } 521 num_targets++; 522 } 523 524 /* 525 * special case. If there is only one target and 526 * no name was specified, return the only available 527 * target. If there is more than one target, name must 528 * be specified. 529 */ 530 if (!name && num_targets == 1) { 531 return TAILQ_FIRST(&g_nvmf_tgts); 532 } 533 534 return NULL; 535 } 536 537 struct spdk_nvmf_tgt * 538 spdk_nvmf_get_first_tgt(void) 539 { 540 return TAILQ_FIRST(&g_nvmf_tgts); 541 } 542 543 struct spdk_nvmf_tgt * 544 spdk_nvmf_get_next_tgt(struct spdk_nvmf_tgt *prev) 545 { 546 return TAILQ_NEXT(prev, link); 547 } 548 549 static void 550 nvmf_write_nvme_subsystem_config(struct spdk_json_write_ctx *w, 551 struct spdk_nvmf_subsystem *subsystem) 552 { 553 struct spdk_nvmf_host *host; 554 struct spdk_nvmf_ns *ns; 555 struct spdk_nvmf_ns_opts ns_opts; 556 uint32_t max_namespaces; 557 struct spdk_nvmf_transport *transport; 558 559 assert(spdk_nvmf_subsystem_get_type(subsystem) == SPDK_NVMF_SUBTYPE_NVME); 560 561 /* { */ 562 spdk_json_write_object_begin(w); 563 spdk_json_write_named_string(w, "method", "nvmf_create_subsystem"); 564 565 /* "params" : { */ 566 spdk_json_write_named_object_begin(w, "params"); 567 spdk_json_write_named_string(w, "nqn", spdk_nvmf_subsystem_get_nqn(subsystem)); 568 spdk_json_write_named_bool(w, "allow_any_host", spdk_nvmf_subsystem_get_allow_any_host(subsystem)); 569 spdk_json_write_named_string(w, "serial_number", spdk_nvmf_subsystem_get_sn(subsystem)); 570 spdk_json_write_named_string(w, "model_number", spdk_nvmf_subsystem_get_mn(subsystem)); 571 572 max_namespaces = spdk_nvmf_subsystem_get_max_namespaces(subsystem); 573 if (max_namespaces != 0) { 574 spdk_json_write_named_uint32(w, "max_namespaces", max_namespaces); 575 } 576 577 spdk_json_write_named_uint32(w, "min_cntlid", spdk_nvmf_subsystem_get_min_cntlid(subsystem)); 578 spdk_json_write_named_uint32(w, "max_cntlid", spdk_nvmf_subsystem_get_max_cntlid(subsystem)); 579 spdk_json_write_named_bool(w, "ana_reporting", spdk_nvmf_subsystem_get_ana_reporting(subsystem)); 580 581 /* } "params" */ 582 spdk_json_write_object_end(w); 583 584 /* } */ 585 spdk_json_write_object_end(w); 586 587 for (host = spdk_nvmf_subsystem_get_first_host(subsystem); host != NULL; 588 host = spdk_nvmf_subsystem_get_next_host(subsystem, host)) { 589 590 spdk_json_write_object_begin(w); 591 spdk_json_write_named_string(w, "method", "nvmf_subsystem_add_host"); 592 593 /* "params" : { */ 594 spdk_json_write_named_object_begin(w, "params"); 595 596 spdk_json_write_named_string(w, "nqn", spdk_nvmf_subsystem_get_nqn(subsystem)); 597 spdk_json_write_named_string(w, "host", spdk_nvmf_host_get_nqn(host)); 598 if (host->dhchap_key != NULL) { 599 spdk_json_write_named_string(w, "dhchap_key", 600 spdk_key_get_name(host->dhchap_key)); 601 } 602 if (host->dhchap_ctrlr_key != NULL) { 603 spdk_json_write_named_string(w, "dhchap_ctrlr_key", 604 spdk_key_get_name(host->dhchap_ctrlr_key)); 605 } 606 TAILQ_FOREACH(transport, &subsystem->tgt->transports, link) { 607 if (transport->ops->subsystem_dump_host != NULL) { 608 transport->ops->subsystem_dump_host(transport, subsystem, host->nqn, w); 609 } 610 } 611 612 /* } "params" */ 613 spdk_json_write_object_end(w); 614 615 /* } */ 616 spdk_json_write_object_end(w); 617 } 618 619 for (ns = spdk_nvmf_subsystem_get_first_ns(subsystem); ns != NULL; 620 ns = spdk_nvmf_subsystem_get_next_ns(subsystem, ns)) { 621 spdk_nvmf_ns_get_opts(ns, &ns_opts, sizeof(ns_opts)); 622 623 spdk_json_write_object_begin(w); 624 spdk_json_write_named_string(w, "method", "nvmf_subsystem_add_ns"); 625 626 /* "params" : { */ 627 spdk_json_write_named_object_begin(w, "params"); 628 629 spdk_json_write_named_string(w, "nqn", spdk_nvmf_subsystem_get_nqn(subsystem)); 630 631 /* "namespace" : { */ 632 spdk_json_write_named_object_begin(w, "namespace"); 633 634 spdk_json_write_named_uint32(w, "nsid", spdk_nvmf_ns_get_id(ns)); 635 spdk_json_write_named_string(w, "bdev_name", spdk_bdev_get_name(spdk_nvmf_ns_get_bdev(ns))); 636 637 if (ns->ptpl_file != NULL) { 638 spdk_json_write_named_string(w, "ptpl_file", ns->ptpl_file); 639 } 640 641 if (!spdk_mem_all_zero(ns_opts.nguid, sizeof(ns_opts.nguid))) { 642 SPDK_STATIC_ASSERT(sizeof(ns_opts.nguid) == sizeof(uint64_t) * 2, "size mismatch"); 643 spdk_json_write_named_string_fmt(w, "nguid", "%016"PRIX64"%016"PRIX64, from_be64(&ns_opts.nguid[0]), 644 from_be64(&ns_opts.nguid[8])); 645 } 646 647 if (!spdk_mem_all_zero(ns_opts.eui64, sizeof(ns_opts.eui64))) { 648 SPDK_STATIC_ASSERT(sizeof(ns_opts.eui64) == sizeof(uint64_t), "size mismatch"); 649 spdk_json_write_named_string_fmt(w, "eui64", "%016"PRIX64, from_be64(&ns_opts.eui64)); 650 } 651 652 if (!spdk_uuid_is_null(&ns_opts.uuid)) { 653 spdk_json_write_named_uuid(w, "uuid", &ns_opts.uuid); 654 } 655 656 if (spdk_nvmf_subsystem_get_ana_reporting(subsystem)) { 657 spdk_json_write_named_uint32(w, "anagrpid", ns_opts.anagrpid); 658 } 659 660 spdk_json_write_named_bool(w, "no_auto_visible", !ns->always_visible); 661 662 /* "namespace" */ 663 spdk_json_write_object_end(w); 664 665 /* } "params" */ 666 spdk_json_write_object_end(w); 667 668 /* } */ 669 spdk_json_write_object_end(w); 670 671 TAILQ_FOREACH(host, &ns->hosts, link) { 672 spdk_json_write_object_begin(w); 673 spdk_json_write_named_string(w, "method", "nvmf_ns_add_host"); 674 spdk_json_write_named_object_begin(w, "params"); 675 spdk_json_write_named_string(w, "nqn", spdk_nvmf_subsystem_get_nqn(subsystem)); 676 spdk_json_write_named_uint32(w, "nsid", spdk_nvmf_ns_get_id(ns)); 677 spdk_json_write_named_string(w, "host", spdk_nvmf_host_get_nqn(host)); 678 spdk_json_write_object_end(w); 679 spdk_json_write_object_end(w); 680 } 681 } 682 } 683 684 static void 685 nvmf_write_subsystem_config_json(struct spdk_json_write_ctx *w, 686 struct spdk_nvmf_subsystem *subsystem) 687 { 688 struct spdk_nvmf_subsystem_listener *listener; 689 struct spdk_nvmf_transport *transport; 690 const struct spdk_nvme_transport_id *trid; 691 692 if (spdk_nvmf_subsystem_get_type(subsystem) == SPDK_NVMF_SUBTYPE_NVME) { 693 nvmf_write_nvme_subsystem_config(w, subsystem); 694 } 695 696 for (listener = spdk_nvmf_subsystem_get_first_listener(subsystem); listener != NULL; 697 listener = spdk_nvmf_subsystem_get_next_listener(subsystem, listener)) { 698 transport = listener->transport; 699 trid = spdk_nvmf_subsystem_listener_get_trid(listener); 700 701 spdk_json_write_object_begin(w); 702 spdk_json_write_named_string(w, "method", "nvmf_subsystem_add_listener"); 703 704 /* "params" : { */ 705 spdk_json_write_named_object_begin(w, "params"); 706 707 spdk_json_write_named_string(w, "nqn", spdk_nvmf_subsystem_get_nqn(subsystem)); 708 709 spdk_json_write_named_object_begin(w, "listen_address"); 710 nvmf_transport_listen_dump_trid(trid, w); 711 spdk_json_write_object_end(w); 712 if (transport->ops->listen_dump_opts) { 713 transport->ops->listen_dump_opts(transport, trid, w); 714 } 715 716 spdk_json_write_named_bool(w, "secure_channel", listener->opts.secure_channel); 717 718 if (listener->opts.sock_impl) { 719 spdk_json_write_named_string(w, "sock_impl", listener->opts.sock_impl); 720 } 721 722 /* } "params" */ 723 spdk_json_write_object_end(w); 724 725 /* } */ 726 spdk_json_write_object_end(w); 727 } 728 } 729 730 void 731 spdk_nvmf_tgt_write_config_json(struct spdk_json_write_ctx *w, struct spdk_nvmf_tgt *tgt) 732 { 733 struct spdk_nvmf_subsystem *subsystem; 734 struct spdk_nvmf_transport *transport; 735 struct spdk_nvmf_referral *referral; 736 737 spdk_json_write_object_begin(w); 738 spdk_json_write_named_string(w, "method", "nvmf_set_max_subsystems"); 739 740 spdk_json_write_named_object_begin(w, "params"); 741 spdk_json_write_named_uint32(w, "max_subsystems", tgt->max_subsystems); 742 spdk_json_write_object_end(w); 743 744 spdk_json_write_object_end(w); 745 746 spdk_json_write_object_begin(w); 747 spdk_json_write_named_string(w, "method", "nvmf_set_crdt"); 748 spdk_json_write_named_object_begin(w, "params"); 749 spdk_json_write_named_uint32(w, "crdt1", tgt->crdt[0]); 750 spdk_json_write_named_uint32(w, "crdt2", tgt->crdt[1]); 751 spdk_json_write_named_uint32(w, "crdt3", tgt->crdt[2]); 752 spdk_json_write_object_end(w); 753 spdk_json_write_object_end(w); 754 755 /* write transports */ 756 TAILQ_FOREACH(transport, &tgt->transports, link) { 757 spdk_json_write_object_begin(w); 758 spdk_json_write_named_string(w, "method", "nvmf_create_transport"); 759 nvmf_transport_dump_opts(transport, w, true); 760 spdk_json_write_object_end(w); 761 } 762 763 TAILQ_FOREACH(referral, &tgt->referrals, link) { 764 spdk_json_write_object_begin(w); 765 spdk_json_write_named_string(w, "method", "nvmf_discovery_add_referral"); 766 767 spdk_json_write_named_object_begin(w, "params"); 768 spdk_json_write_named_object_begin(w, "address"); 769 nvmf_transport_listen_dump_trid(&referral->trid, w); 770 spdk_json_write_object_end(w); 771 spdk_json_write_named_bool(w, "secure_channel", 772 referral->entry.treq.secure_channel == 773 SPDK_NVMF_TREQ_SECURE_CHANNEL_REQUIRED); 774 spdk_json_write_named_string(w, "subnqn", referral->trid.subnqn); 775 spdk_json_write_object_end(w); 776 777 spdk_json_write_object_end(w); 778 } 779 780 subsystem = spdk_nvmf_subsystem_get_first(tgt); 781 while (subsystem) { 782 nvmf_write_subsystem_config_json(w, subsystem); 783 subsystem = spdk_nvmf_subsystem_get_next(subsystem); 784 } 785 } 786 787 static void 788 nvmf_listen_opts_copy(struct spdk_nvmf_listen_opts *opts, 789 const struct spdk_nvmf_listen_opts *opts_src, size_t opts_size) 790 { 791 assert(opts); 792 assert(opts_src); 793 794 opts->opts_size = opts_size; 795 796 #define SET_FIELD(field) \ 797 if (offsetof(struct spdk_nvmf_listen_opts, field) + sizeof(opts->field) <= opts_size) { \ 798 opts->field = opts_src->field; \ 799 } \ 800 801 SET_FIELD(transport_specific); 802 SET_FIELD(secure_channel); 803 SET_FIELD(ana_state); 804 SET_FIELD(sock_impl); 805 #undef SET_FIELD 806 807 /* Do not remove this statement, you should always update this statement when you adding a new field, 808 * and do not forget to add the SET_FIELD statement for your added field. */ 809 SPDK_STATIC_ASSERT(sizeof(struct spdk_nvmf_listen_opts) == 32, "Incorrect size"); 810 } 811 812 void 813 spdk_nvmf_listen_opts_init(struct spdk_nvmf_listen_opts *opts, size_t opts_size) 814 { 815 struct spdk_nvmf_listen_opts opts_local = {}; 816 817 /* local version of opts should have defaults set here */ 818 opts_local.ana_state = SPDK_NVME_ANA_OPTIMIZED_STATE; 819 nvmf_listen_opts_copy(opts, &opts_local, opts_size); 820 } 821 822 int 823 spdk_nvmf_tgt_listen_ext(struct spdk_nvmf_tgt *tgt, const struct spdk_nvme_transport_id *trid, 824 struct spdk_nvmf_listen_opts *opts) 825 { 826 struct spdk_nvmf_transport *transport; 827 int rc; 828 struct spdk_nvmf_listen_opts opts_local = {}; 829 830 if (!opts) { 831 SPDK_ERRLOG("opts should not be NULL\n"); 832 return -EINVAL; 833 } 834 835 if (!opts->opts_size) { 836 SPDK_ERRLOG("The opts_size in opts structure should not be zero\n"); 837 return -EINVAL; 838 } 839 840 transport = spdk_nvmf_tgt_get_transport(tgt, trid->trstring); 841 if (!transport) { 842 SPDK_ERRLOG("Unable to find %s transport. The transport must be created first also make sure it is properly registered.\n", 843 trid->trstring); 844 return -EINVAL; 845 } 846 847 nvmf_listen_opts_copy(&opts_local, opts, opts->opts_size); 848 rc = spdk_nvmf_transport_listen(transport, trid, &opts_local); 849 if (rc < 0) { 850 SPDK_ERRLOG("Unable to listen on address '%s'\n", trid->traddr); 851 } 852 853 return rc; 854 } 855 856 int 857 spdk_nvmf_tgt_stop_listen(struct spdk_nvmf_tgt *tgt, 858 struct spdk_nvme_transport_id *trid) 859 { 860 struct spdk_nvmf_transport *transport; 861 int rc; 862 863 transport = spdk_nvmf_tgt_get_transport(tgt, trid->trstring); 864 if (!transport) { 865 SPDK_ERRLOG("Unable to find %s transport. The transport must be created first also make sure it is properly registered.\n", 866 trid->trstring); 867 return -EINVAL; 868 } 869 870 rc = spdk_nvmf_transport_stop_listen(transport, trid); 871 if (rc < 0) { 872 SPDK_ERRLOG("Failed to stop listening on address '%s'\n", trid->traddr); 873 return rc; 874 } 875 return 0; 876 } 877 878 struct spdk_nvmf_tgt_add_transport_ctx { 879 struct spdk_nvmf_tgt *tgt; 880 struct spdk_nvmf_transport *transport; 881 spdk_nvmf_tgt_add_transport_done_fn cb_fn; 882 void *cb_arg; 883 int status; 884 }; 885 886 static void 887 _nvmf_tgt_remove_transport_done(struct spdk_io_channel_iter *i, int status) 888 { 889 struct spdk_nvmf_tgt_add_transport_ctx *ctx = spdk_io_channel_iter_get_ctx(i); 890 891 ctx->cb_fn(ctx->cb_arg, ctx->status); 892 free(ctx); 893 } 894 895 static void 896 _nvmf_tgt_remove_transport(struct spdk_io_channel_iter *i) 897 { 898 struct spdk_nvmf_tgt_add_transport_ctx *ctx = spdk_io_channel_iter_get_ctx(i); 899 struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i); 900 struct spdk_nvmf_poll_group *group = spdk_io_channel_get_ctx(ch); 901 struct spdk_nvmf_transport_poll_group *tgroup, *tmp; 902 903 TAILQ_FOREACH_SAFE(tgroup, &group->tgroups, link, tmp) { 904 if (tgroup->transport == ctx->transport) { 905 TAILQ_REMOVE(&group->tgroups, tgroup, link); 906 nvmf_transport_poll_group_destroy(tgroup); 907 } 908 } 909 910 spdk_for_each_channel_continue(i, 0); 911 } 912 913 static void 914 _nvmf_tgt_add_transport_done(struct spdk_io_channel_iter *i, int status) 915 { 916 struct spdk_nvmf_tgt_add_transport_ctx *ctx = spdk_io_channel_iter_get_ctx(i); 917 918 if (status) { 919 ctx->status = status; 920 spdk_for_each_channel(ctx->tgt, 921 _nvmf_tgt_remove_transport, 922 ctx, 923 _nvmf_tgt_remove_transport_done); 924 return; 925 } 926 927 ctx->transport->tgt = ctx->tgt; 928 TAILQ_INSERT_TAIL(&ctx->tgt->transports, ctx->transport, link); 929 ctx->cb_fn(ctx->cb_arg, status); 930 free(ctx); 931 } 932 933 static void 934 _nvmf_tgt_add_transport(struct spdk_io_channel_iter *i) 935 { 936 struct spdk_nvmf_tgt_add_transport_ctx *ctx = spdk_io_channel_iter_get_ctx(i); 937 struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i); 938 struct spdk_nvmf_poll_group *group = spdk_io_channel_get_ctx(ch); 939 int rc; 940 941 rc = nvmf_poll_group_add_transport(group, ctx->transport); 942 spdk_for_each_channel_continue(i, rc); 943 } 944 945 void 946 spdk_nvmf_tgt_add_transport(struct spdk_nvmf_tgt *tgt, 947 struct spdk_nvmf_transport *transport, 948 spdk_nvmf_tgt_add_transport_done_fn cb_fn, 949 void *cb_arg) 950 { 951 struct spdk_nvmf_tgt_add_transport_ctx *ctx; 952 953 SPDK_DTRACE_PROBE2_TICKS(nvmf_tgt_add_transport, transport, tgt->name); 954 955 if (spdk_nvmf_tgt_get_transport(tgt, transport->ops->name)) { 956 cb_fn(cb_arg, -EEXIST); 957 return; /* transport already created */ 958 } 959 960 ctx = calloc(1, sizeof(*ctx)); 961 if (!ctx) { 962 cb_fn(cb_arg, -ENOMEM); 963 return; 964 } 965 966 ctx->tgt = tgt; 967 ctx->transport = transport; 968 ctx->cb_fn = cb_fn; 969 ctx->cb_arg = cb_arg; 970 971 spdk_for_each_channel(tgt, 972 _nvmf_tgt_add_transport, 973 ctx, 974 _nvmf_tgt_add_transport_done); 975 } 976 977 struct nvmf_tgt_pause_ctx { 978 struct spdk_nvmf_tgt *tgt; 979 spdk_nvmf_tgt_pause_polling_cb_fn cb_fn; 980 void *cb_arg; 981 }; 982 983 static void 984 _nvmf_tgt_pause_polling_done(struct spdk_io_channel_iter *i, int status) 985 { 986 struct nvmf_tgt_pause_ctx *ctx = spdk_io_channel_iter_get_ctx(i); 987 988 ctx->tgt->state = NVMF_TGT_PAUSED; 989 990 ctx->cb_fn(ctx->cb_arg, status); 991 free(ctx); 992 } 993 994 static void 995 _nvmf_tgt_pause_polling(struct spdk_io_channel_iter *i) 996 { 997 struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i); 998 struct spdk_nvmf_poll_group *group = spdk_io_channel_get_ctx(ch); 999 1000 spdk_poller_unregister(&group->poller); 1001 1002 spdk_for_each_channel_continue(i, 0); 1003 } 1004 1005 int 1006 spdk_nvmf_tgt_pause_polling(struct spdk_nvmf_tgt *tgt, spdk_nvmf_tgt_pause_polling_cb_fn cb_fn, 1007 void *cb_arg) 1008 { 1009 struct nvmf_tgt_pause_ctx *ctx; 1010 1011 SPDK_DTRACE_PROBE2_TICKS(nvmf_tgt_pause_polling, tgt, tgt->name); 1012 1013 switch (tgt->state) { 1014 case NVMF_TGT_PAUSING: 1015 case NVMF_TGT_RESUMING: 1016 return -EBUSY; 1017 case NVMF_TGT_RUNNING: 1018 break; 1019 default: 1020 return -EINVAL; 1021 } 1022 1023 ctx = calloc(1, sizeof(*ctx)); 1024 if (!ctx) { 1025 return -ENOMEM; 1026 } 1027 1028 1029 tgt->state = NVMF_TGT_PAUSING; 1030 1031 ctx->tgt = tgt; 1032 ctx->cb_fn = cb_fn; 1033 ctx->cb_arg = cb_arg; 1034 1035 spdk_for_each_channel(tgt, 1036 _nvmf_tgt_pause_polling, 1037 ctx, 1038 _nvmf_tgt_pause_polling_done); 1039 return 0; 1040 } 1041 1042 static void 1043 _nvmf_tgt_resume_polling_done(struct spdk_io_channel_iter *i, int status) 1044 { 1045 struct nvmf_tgt_pause_ctx *ctx = spdk_io_channel_iter_get_ctx(i); 1046 1047 ctx->tgt->state = NVMF_TGT_RUNNING; 1048 1049 ctx->cb_fn(ctx->cb_arg, status); 1050 free(ctx); 1051 } 1052 1053 static void 1054 _nvmf_tgt_resume_polling(struct spdk_io_channel_iter *i) 1055 { 1056 struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i); 1057 struct spdk_nvmf_poll_group *group = spdk_io_channel_get_ctx(ch); 1058 1059 assert(group->poller == NULL); 1060 group->poller = SPDK_POLLER_REGISTER(nvmf_poll_group_poll, group, 0); 1061 1062 spdk_for_each_channel_continue(i, 0); 1063 } 1064 1065 int 1066 spdk_nvmf_tgt_resume_polling(struct spdk_nvmf_tgt *tgt, spdk_nvmf_tgt_resume_polling_cb_fn cb_fn, 1067 void *cb_arg) 1068 { 1069 struct nvmf_tgt_pause_ctx *ctx; 1070 1071 SPDK_DTRACE_PROBE2_TICKS(nvmf_tgt_resume_polling, tgt, tgt->name); 1072 1073 switch (tgt->state) { 1074 case NVMF_TGT_PAUSING: 1075 case NVMF_TGT_RESUMING: 1076 return -EBUSY; 1077 case NVMF_TGT_PAUSED: 1078 break; 1079 default: 1080 return -EINVAL; 1081 } 1082 1083 ctx = calloc(1, sizeof(*ctx)); 1084 if (!ctx) { 1085 return -ENOMEM; 1086 } 1087 1088 tgt->state = NVMF_TGT_RESUMING; 1089 1090 ctx->tgt = tgt; 1091 ctx->cb_fn = cb_fn; 1092 ctx->cb_arg = cb_arg; 1093 1094 spdk_for_each_channel(tgt, 1095 _nvmf_tgt_resume_polling, 1096 ctx, 1097 _nvmf_tgt_resume_polling_done); 1098 return 0; 1099 } 1100 1101 struct spdk_nvmf_subsystem * 1102 spdk_nvmf_tgt_find_subsystem(struct spdk_nvmf_tgt *tgt, const char *subnqn) 1103 { 1104 struct spdk_nvmf_subsystem subsystem; 1105 1106 if (!subnqn) { 1107 return NULL; 1108 } 1109 1110 /* Ensure that subnqn is null terminated */ 1111 if (!memchr(subnqn, '\0', SPDK_NVMF_NQN_MAX_LEN + 1)) { 1112 SPDK_ERRLOG("Connect SUBNQN is not null terminated\n"); 1113 return NULL; 1114 } 1115 1116 snprintf(subsystem.subnqn, sizeof(subsystem.subnqn), "%s", subnqn); 1117 return RB_FIND(subsystem_tree, &tgt->subsystems, &subsystem); 1118 } 1119 1120 struct spdk_nvmf_transport * 1121 spdk_nvmf_tgt_get_transport(struct spdk_nvmf_tgt *tgt, const char *transport_name) 1122 { 1123 struct spdk_nvmf_transport *transport; 1124 1125 TAILQ_FOREACH(transport, &tgt->transports, link) { 1126 if (!strncasecmp(transport->ops->name, transport_name, SPDK_NVMF_TRSTRING_MAX_LEN)) { 1127 return transport; 1128 } 1129 } 1130 return NULL; 1131 } 1132 1133 struct nvmf_new_qpair_ctx { 1134 struct spdk_nvmf_qpair *qpair; 1135 struct spdk_nvmf_poll_group *group; 1136 }; 1137 1138 static void 1139 _nvmf_poll_group_add(void *_ctx) 1140 { 1141 struct nvmf_new_qpair_ctx *ctx = _ctx; 1142 struct spdk_nvmf_qpair *qpair = ctx->qpair; 1143 struct spdk_nvmf_poll_group *group = ctx->group; 1144 1145 free(_ctx); 1146 1147 if (spdk_nvmf_poll_group_add(group, qpair) != 0) { 1148 SPDK_ERRLOG("Unable to add the qpair to a poll group.\n"); 1149 spdk_nvmf_qpair_disconnect(qpair); 1150 } 1151 } 1152 1153 void 1154 spdk_nvmf_tgt_new_qpair(struct spdk_nvmf_tgt *tgt, struct spdk_nvmf_qpair *qpair) 1155 { 1156 struct spdk_nvmf_poll_group *group; 1157 struct nvmf_new_qpair_ctx *ctx; 1158 1159 group = spdk_nvmf_get_optimal_poll_group(qpair); 1160 if (group == NULL) { 1161 if (tgt->next_poll_group == NULL) { 1162 tgt->next_poll_group = TAILQ_FIRST(&tgt->poll_groups); 1163 if (tgt->next_poll_group == NULL) { 1164 SPDK_ERRLOG("No poll groups exist.\n"); 1165 spdk_nvmf_qpair_disconnect(qpair); 1166 return; 1167 } 1168 } 1169 group = tgt->next_poll_group; 1170 tgt->next_poll_group = TAILQ_NEXT(group, link); 1171 } 1172 1173 ctx = calloc(1, sizeof(*ctx)); 1174 if (!ctx) { 1175 SPDK_ERRLOG("Unable to send message to poll group.\n"); 1176 spdk_nvmf_qpair_disconnect(qpair); 1177 return; 1178 } 1179 1180 ctx->qpair = qpair; 1181 ctx->group = group; 1182 1183 pthread_mutex_lock(&group->mutex); 1184 group->current_unassociated_qpairs++; 1185 pthread_mutex_unlock(&group->mutex); 1186 1187 spdk_thread_send_msg(group->thread, _nvmf_poll_group_add, ctx); 1188 } 1189 1190 struct spdk_nvmf_poll_group * 1191 spdk_nvmf_poll_group_create(struct spdk_nvmf_tgt *tgt) 1192 { 1193 struct spdk_io_channel *ch; 1194 1195 ch = spdk_get_io_channel(tgt); 1196 if (!ch) { 1197 SPDK_ERRLOG("Unable to get I/O channel for target\n"); 1198 return NULL; 1199 } 1200 1201 return spdk_io_channel_get_ctx(ch); 1202 } 1203 1204 void 1205 spdk_nvmf_poll_group_destroy(struct spdk_nvmf_poll_group *group, 1206 spdk_nvmf_poll_group_destroy_done_fn cb_fn, 1207 void *cb_arg) 1208 { 1209 assert(group->destroy_cb_fn == NULL); 1210 group->destroy_cb_fn = cb_fn; 1211 group->destroy_cb_arg = cb_arg; 1212 1213 /* This function will put the io_channel associated with this poll group */ 1214 nvmf_tgt_destroy_poll_group_qpairs(group); 1215 } 1216 1217 int 1218 spdk_nvmf_poll_group_add(struct spdk_nvmf_poll_group *group, 1219 struct spdk_nvmf_qpair *qpair) 1220 { 1221 int rc; 1222 struct spdk_nvmf_transport_poll_group *tgroup; 1223 1224 TAILQ_INIT(&qpair->outstanding); 1225 qpair->group = group; 1226 qpair->ctrlr = NULL; 1227 qpair->disconnect_started = false; 1228 1229 tgroup = nvmf_get_transport_poll_group(group, qpair->transport); 1230 if (tgroup == NULL) { 1231 return -1; 1232 } 1233 1234 rc = nvmf_transport_poll_group_add(tgroup, qpair); 1235 1236 /* We add the qpair to the group only it is successfully added into the tgroup */ 1237 if (rc == 0) { 1238 SPDK_DTRACE_PROBE2_TICKS(nvmf_poll_group_add_qpair, qpair, spdk_thread_get_id(group->thread)); 1239 TAILQ_INSERT_TAIL(&group->qpairs, qpair, link); 1240 nvmf_qpair_set_state(qpair, SPDK_NVMF_QPAIR_CONNECTING); 1241 } 1242 1243 return rc; 1244 } 1245 1246 static void 1247 _nvmf_ctrlr_destruct(void *ctx) 1248 { 1249 struct spdk_nvmf_ctrlr *ctrlr = ctx; 1250 1251 nvmf_ctrlr_destruct(ctrlr); 1252 } 1253 1254 static void 1255 _nvmf_ctrlr_free_from_qpair(void *ctx) 1256 { 1257 struct nvmf_qpair_disconnect_ctx *qpair_ctx = ctx; 1258 struct spdk_nvmf_ctrlr *ctrlr = qpair_ctx->ctrlr; 1259 uint32_t count; 1260 1261 spdk_bit_array_clear(ctrlr->qpair_mask, qpair_ctx->qid); 1262 SPDK_DEBUGLOG(nvmf, "qpair_mask cleared, qid %u\n", qpair_ctx->qid); 1263 count = spdk_bit_array_count_set(ctrlr->qpair_mask); 1264 if (count == 0) { 1265 assert(!ctrlr->in_destruct); 1266 SPDK_DEBUGLOG(nvmf, "Last qpair %u, destroy ctrlr 0x%hx\n", qpair_ctx->qid, ctrlr->cntlid); 1267 ctrlr->in_destruct = true; 1268 spdk_thread_send_msg(ctrlr->subsys->thread, _nvmf_ctrlr_destruct, ctrlr); 1269 } 1270 free(qpair_ctx); 1271 } 1272 1273 static void 1274 _nvmf_transport_qpair_fini_complete(void *cb_ctx) 1275 { 1276 struct nvmf_qpair_disconnect_ctx *qpair_ctx = cb_ctx; 1277 struct spdk_nvmf_ctrlr *ctrlr; 1278 1279 ctrlr = qpair_ctx->ctrlr; 1280 SPDK_DEBUGLOG(nvmf, "Finish destroying qid %u\n", qpair_ctx->qid); 1281 1282 if (ctrlr) { 1283 if (qpair_ctx->qid == 0) { 1284 /* Admin qpair is removed, so set the pointer to NULL. 1285 * This operation is safe since we are on ctrlr thread now, admin qpair's thread is the same 1286 * as controller's thread */ 1287 assert(ctrlr->thread == spdk_get_thread()); 1288 ctrlr->admin_qpair = NULL; 1289 } 1290 /* Free qpair id from controller's bit mask and destroy the controller if it is the last qpair */ 1291 if (ctrlr->thread) { 1292 spdk_thread_send_msg(ctrlr->thread, _nvmf_ctrlr_free_from_qpair, qpair_ctx); 1293 } else { 1294 _nvmf_ctrlr_free_from_qpair(qpair_ctx); 1295 } 1296 } else { 1297 free(qpair_ctx); 1298 } 1299 } 1300 1301 void 1302 spdk_nvmf_poll_group_remove(struct spdk_nvmf_qpair *qpair) 1303 { 1304 struct spdk_nvmf_transport_poll_group *tgroup; 1305 int rc; 1306 1307 SPDK_DTRACE_PROBE2_TICKS(nvmf_poll_group_remove_qpair, qpair, 1308 spdk_thread_get_id(qpair->group->thread)); 1309 nvmf_qpair_set_state(qpair, SPDK_NVMF_QPAIR_ERROR); 1310 1311 /* Find the tgroup and remove the qpair from the tgroup */ 1312 tgroup = nvmf_get_transport_poll_group(qpair->group, qpair->transport); 1313 if (tgroup != NULL) { 1314 rc = nvmf_transport_poll_group_remove(tgroup, qpair); 1315 if (rc && (rc != ENOTSUP)) { 1316 SPDK_ERRLOG("Cannot remove qpair=%p from transport group=%p\n", 1317 qpair, tgroup); 1318 } 1319 } 1320 1321 TAILQ_REMOVE(&qpair->group->qpairs, qpair, link); 1322 qpair->group = NULL; 1323 } 1324 1325 static void 1326 _nvmf_qpair_sgroup_req_clean(struct spdk_nvmf_subsystem_poll_group *sgroup, 1327 const struct spdk_nvmf_qpair *qpair) 1328 { 1329 struct spdk_nvmf_request *req, *tmp; 1330 TAILQ_FOREACH_SAFE(req, &sgroup->queued, link, tmp) { 1331 if (req->qpair == qpair) { 1332 TAILQ_REMOVE(&sgroup->queued, req, link); 1333 if (nvmf_transport_req_free(req)) { 1334 SPDK_ERRLOG("Transport request free error!\n"); 1335 } 1336 } 1337 } 1338 } 1339 1340 static void 1341 _nvmf_qpair_destroy(void *ctx, int status) 1342 { 1343 struct nvmf_qpair_disconnect_ctx *qpair_ctx = ctx; 1344 struct spdk_nvmf_qpair *qpair = qpair_ctx->qpair; 1345 struct spdk_nvmf_ctrlr *ctrlr = qpair->ctrlr; 1346 struct spdk_nvmf_subsystem_poll_group *sgroup; 1347 uint32_t sid; 1348 1349 assert(qpair->state == SPDK_NVMF_QPAIR_DEACTIVATING); 1350 qpair_ctx->qid = qpair->qid; 1351 1352 if (qpair->connect_received) { 1353 if (0 == qpair->qid) { 1354 assert(qpair->group->stat.current_admin_qpairs > 0); 1355 qpair->group->stat.current_admin_qpairs--; 1356 } else { 1357 assert(qpair->group->stat.current_io_qpairs > 0); 1358 qpair->group->stat.current_io_qpairs--; 1359 } 1360 } else { 1361 pthread_mutex_lock(&qpair->group->mutex); 1362 qpair->group->current_unassociated_qpairs--; 1363 pthread_mutex_unlock(&qpair->group->mutex); 1364 } 1365 1366 if (ctrlr) { 1367 sgroup = &qpair->group->sgroups[ctrlr->subsys->id]; 1368 _nvmf_qpair_sgroup_req_clean(sgroup, qpair); 1369 } else { 1370 for (sid = 0; sid < qpair->group->num_sgroups; sid++) { 1371 sgroup = &qpair->group->sgroups[sid]; 1372 assert(sgroup != NULL); 1373 _nvmf_qpair_sgroup_req_clean(sgroup, qpair); 1374 } 1375 } 1376 1377 nvmf_qpair_auth_destroy(qpair); 1378 qpair_ctx->ctrlr = ctrlr; 1379 spdk_nvmf_poll_group_remove(qpair); 1380 nvmf_transport_qpair_fini(qpair, _nvmf_transport_qpair_fini_complete, qpair_ctx); 1381 } 1382 1383 static void 1384 _nvmf_qpair_disconnect_msg(void *ctx) 1385 { 1386 struct nvmf_qpair_disconnect_ctx *qpair_ctx = ctx; 1387 1388 spdk_nvmf_qpair_disconnect(qpair_ctx->qpair); 1389 free(ctx); 1390 } 1391 1392 int 1393 spdk_nvmf_qpair_disconnect(struct spdk_nvmf_qpair *qpair) 1394 { 1395 struct spdk_nvmf_poll_group *group = qpair->group; 1396 struct nvmf_qpair_disconnect_ctx *qpair_ctx; 1397 1398 if (__atomic_test_and_set(&qpair->disconnect_started, __ATOMIC_RELAXED)) { 1399 return -EINPROGRESS; 1400 } 1401 1402 /* If we get a qpair in the uninitialized state, we can just destroy it immediately */ 1403 if (qpair->state == SPDK_NVMF_QPAIR_UNINITIALIZED) { 1404 nvmf_transport_qpair_fini(qpair, NULL, NULL); 1405 return 0; 1406 } 1407 1408 assert(group != NULL); 1409 if (spdk_get_thread() != group->thread) { 1410 /* clear the atomic so we can set it on the next call on the proper thread. */ 1411 __atomic_clear(&qpair->disconnect_started, __ATOMIC_RELAXED); 1412 qpair_ctx = calloc(1, sizeof(struct nvmf_qpair_disconnect_ctx)); 1413 if (!qpair_ctx) { 1414 SPDK_ERRLOG("Unable to allocate context for nvmf_qpair_disconnect\n"); 1415 return -ENOMEM; 1416 } 1417 qpair_ctx->qpair = qpair; 1418 spdk_thread_send_msg(group->thread, _nvmf_qpair_disconnect_msg, qpair_ctx); 1419 return 0; 1420 } 1421 1422 SPDK_DTRACE_PROBE2_TICKS(nvmf_qpair_disconnect, qpair, spdk_thread_get_id(group->thread)); 1423 assert(spdk_nvmf_qpair_is_active(qpair)); 1424 nvmf_qpair_set_state(qpair, SPDK_NVMF_QPAIR_DEACTIVATING); 1425 1426 qpair_ctx = calloc(1, sizeof(struct nvmf_qpair_disconnect_ctx)); 1427 if (!qpair_ctx) { 1428 SPDK_ERRLOG("Unable to allocate context for nvmf_qpair_disconnect\n"); 1429 return -ENOMEM; 1430 } 1431 1432 qpair_ctx->qpair = qpair; 1433 1434 /* Check for outstanding I/O */ 1435 if (!TAILQ_EMPTY(&qpair->outstanding)) { 1436 SPDK_DTRACE_PROBE2_TICKS(nvmf_poll_group_drain_qpair, qpair, spdk_thread_get_id(group->thread)); 1437 qpair->state_cb = _nvmf_qpair_destroy; 1438 qpair->state_cb_arg = qpair_ctx; 1439 nvmf_qpair_abort_pending_zcopy_reqs(qpair); 1440 nvmf_qpair_free_aer(qpair); 1441 return 0; 1442 } 1443 1444 _nvmf_qpair_destroy(qpair_ctx, 0); 1445 1446 return 0; 1447 } 1448 1449 int 1450 spdk_nvmf_qpair_get_peer_trid(struct spdk_nvmf_qpair *qpair, 1451 struct spdk_nvme_transport_id *trid) 1452 { 1453 memset(trid, 0, sizeof(*trid)); 1454 return nvmf_transport_qpair_get_peer_trid(qpair, trid); 1455 } 1456 1457 int 1458 spdk_nvmf_qpair_get_local_trid(struct spdk_nvmf_qpair *qpair, 1459 struct spdk_nvme_transport_id *trid) 1460 { 1461 memset(trid, 0, sizeof(*trid)); 1462 return nvmf_transport_qpair_get_local_trid(qpair, trid); 1463 } 1464 1465 int 1466 spdk_nvmf_qpair_get_listen_trid(struct spdk_nvmf_qpair *qpair, 1467 struct spdk_nvme_transport_id *trid) 1468 { 1469 memset(trid, 0, sizeof(*trid)); 1470 return nvmf_transport_qpair_get_listen_trid(qpair, trid); 1471 } 1472 1473 static int 1474 poll_group_update_subsystem(struct spdk_nvmf_poll_group *group, 1475 struct spdk_nvmf_subsystem *subsystem) 1476 { 1477 struct spdk_nvmf_subsystem_poll_group *sgroup; 1478 uint32_t i, j; 1479 struct spdk_nvmf_ns *ns; 1480 struct spdk_nvmf_registrant *reg, *tmp; 1481 struct spdk_io_channel *ch; 1482 struct spdk_nvmf_subsystem_pg_ns_info *ns_info; 1483 struct spdk_nvmf_ctrlr *ctrlr; 1484 bool ns_changed; 1485 1486 /* Make sure our poll group has memory for this subsystem allocated */ 1487 if (subsystem->id >= group->num_sgroups) { 1488 return -ENOMEM; 1489 } 1490 1491 sgroup = &group->sgroups[subsystem->id]; 1492 1493 /* Make sure the array of namespace information is the correct size */ 1494 if (sgroup->num_ns == 0 && subsystem->max_nsid > 0) { 1495 /* First allocation */ 1496 sgroup->ns_info = calloc(subsystem->max_nsid, sizeof(struct spdk_nvmf_subsystem_pg_ns_info)); 1497 if (!sgroup->ns_info) { 1498 return -ENOMEM; 1499 } 1500 sgroup->num_ns = subsystem->max_nsid; 1501 } 1502 1503 ns_changed = false; 1504 1505 /* Detect bdevs that were added or removed */ 1506 for (i = 0; i < sgroup->num_ns; i++) { 1507 ns = subsystem->ns[i]; 1508 ns_info = &sgroup->ns_info[i]; 1509 ch = ns_info->channel; 1510 1511 if (ns == NULL && ch == NULL) { 1512 /* Both NULL. Leave empty */ 1513 } else if (ns == NULL && ch != NULL) { 1514 /* There was a channel here, but the namespace is gone. */ 1515 ns_changed = true; 1516 spdk_put_io_channel(ch); 1517 ns_info->channel = NULL; 1518 } else if (ns != NULL && ch == NULL) { 1519 /* A namespace appeared but there is no channel yet */ 1520 ns_changed = true; 1521 ch = spdk_bdev_get_io_channel(ns->desc); 1522 if (ch == NULL) { 1523 SPDK_ERRLOG("Could not allocate I/O channel.\n"); 1524 return -ENOMEM; 1525 } 1526 ns_info->channel = ch; 1527 } else if (spdk_uuid_compare(&ns_info->uuid, spdk_bdev_get_uuid(ns->bdev)) != 0) { 1528 /* A namespace was here before, but was replaced by a new one. */ 1529 ns_changed = true; 1530 spdk_put_io_channel(ns_info->channel); 1531 memset(ns_info, 0, sizeof(*ns_info)); 1532 1533 ch = spdk_bdev_get_io_channel(ns->desc); 1534 if (ch == NULL) { 1535 SPDK_ERRLOG("Could not allocate I/O channel.\n"); 1536 return -ENOMEM; 1537 } 1538 ns_info->channel = ch; 1539 } else if (ns_info->num_blocks != spdk_bdev_get_num_blocks(ns->bdev)) { 1540 /* Namespace is still there but size has changed */ 1541 SPDK_DEBUGLOG(nvmf, "Namespace resized: subsystem_id %u," 1542 " nsid %u, pg %p, old %" PRIu64 ", new %" PRIu64 "\n", 1543 subsystem->id, 1544 ns->nsid, 1545 group, 1546 ns_info->num_blocks, 1547 spdk_bdev_get_num_blocks(ns->bdev)); 1548 ns_changed = true; 1549 } else if (ns_info->anagrpid != ns->anagrpid) { 1550 /* Namespace is still there but ANA group ID has changed */ 1551 SPDK_DEBUGLOG(nvmf, "ANA group ID changed: subsystem_id %u," 1552 "nsid %u, pg %p, old %u, new %u\n", 1553 subsystem->id, 1554 ns->nsid, 1555 group, 1556 ns_info->anagrpid, 1557 ns->anagrpid); 1558 ns_changed = true; 1559 } 1560 1561 if (ns == NULL) { 1562 memset(ns_info, 0, sizeof(*ns_info)); 1563 } else { 1564 ns_info->uuid = *spdk_bdev_get_uuid(ns->bdev); 1565 ns_info->num_blocks = spdk_bdev_get_num_blocks(ns->bdev); 1566 ns_info->anagrpid = ns->anagrpid; 1567 ns_info->crkey = ns->crkey; 1568 ns_info->rtype = ns->rtype; 1569 if (ns->holder) { 1570 ns_info->holder_id = ns->holder->hostid; 1571 } 1572 1573 memset(&ns_info->reg_hostid, 0, SPDK_NVMF_MAX_NUM_REGISTRANTS * sizeof(struct spdk_uuid)); 1574 j = 0; 1575 TAILQ_FOREACH_SAFE(reg, &ns->registrants, link, tmp) { 1576 if (j >= SPDK_NVMF_MAX_NUM_REGISTRANTS) { 1577 SPDK_ERRLOG("Maximum %u registrants can support.\n", SPDK_NVMF_MAX_NUM_REGISTRANTS); 1578 return -EINVAL; 1579 } 1580 ns_info->reg_hostid[j++] = reg->hostid; 1581 } 1582 } 1583 } 1584 1585 if (ns_changed) { 1586 TAILQ_FOREACH(ctrlr, &subsystem->ctrlrs, link) { 1587 if (ctrlr->thread != spdk_get_thread()) { 1588 continue; 1589 } 1590 /* It is possible that a ctrlr was added but the admin_qpair hasn't been 1591 * assigned yet. 1592 */ 1593 if (!ctrlr->admin_qpair) { 1594 continue; 1595 } 1596 if (ctrlr->admin_qpair->group == group) { 1597 nvmf_ctrlr_async_event_ns_notice(ctrlr); 1598 nvmf_ctrlr_async_event_ana_change_notice(ctrlr); 1599 } 1600 } 1601 } 1602 1603 return 0; 1604 } 1605 1606 int 1607 nvmf_poll_group_update_subsystem(struct spdk_nvmf_poll_group *group, 1608 struct spdk_nvmf_subsystem *subsystem) 1609 { 1610 return poll_group_update_subsystem(group, subsystem); 1611 } 1612 1613 int 1614 nvmf_poll_group_add_subsystem(struct spdk_nvmf_poll_group *group, 1615 struct spdk_nvmf_subsystem *subsystem, 1616 spdk_nvmf_poll_group_mod_done cb_fn, void *cb_arg) 1617 { 1618 int rc = 0; 1619 struct spdk_nvmf_subsystem_poll_group *sgroup = &group->sgroups[subsystem->id]; 1620 struct spdk_nvmf_request *req, *tmp; 1621 uint32_t i; 1622 1623 if (!TAILQ_EMPTY(&sgroup->queued)) { 1624 SPDK_ERRLOG("sgroup->queued not empty when adding subsystem\n"); 1625 TAILQ_FOREACH_SAFE(req, &sgroup->queued, link, tmp) { 1626 TAILQ_REMOVE(&sgroup->queued, req, link); 1627 if (nvmf_transport_req_free(req)) { 1628 SPDK_ERRLOG("Transport request free error!\n"); 1629 } 1630 } 1631 } 1632 1633 rc = poll_group_update_subsystem(group, subsystem); 1634 if (rc) { 1635 nvmf_poll_group_remove_subsystem(group, subsystem, NULL, NULL); 1636 goto fini; 1637 } 1638 1639 sgroup->state = SPDK_NVMF_SUBSYSTEM_ACTIVE; 1640 1641 for (i = 0; i < sgroup->num_ns; i++) { 1642 sgroup->ns_info[i].state = SPDK_NVMF_SUBSYSTEM_ACTIVE; 1643 } 1644 1645 fini: 1646 if (cb_fn) { 1647 cb_fn(cb_arg, rc); 1648 } 1649 1650 SPDK_DTRACE_PROBE2_TICKS(nvmf_poll_group_add_subsystem, spdk_thread_get_id(group->thread), 1651 subsystem->subnqn); 1652 1653 return rc; 1654 } 1655 1656 static void 1657 _nvmf_poll_group_remove_subsystem_cb(void *ctx, int status) 1658 { 1659 struct nvmf_qpair_disconnect_many_ctx *qpair_ctx = ctx; 1660 struct spdk_nvmf_subsystem *subsystem; 1661 struct spdk_nvmf_poll_group *group; 1662 struct spdk_nvmf_subsystem_poll_group *sgroup; 1663 spdk_nvmf_poll_group_mod_done cpl_fn = NULL; 1664 void *cpl_ctx = NULL; 1665 uint32_t nsid; 1666 1667 group = qpair_ctx->group; 1668 subsystem = qpair_ctx->subsystem; 1669 cpl_fn = qpair_ctx->cpl_fn; 1670 cpl_ctx = qpair_ctx->cpl_ctx; 1671 sgroup = &group->sgroups[subsystem->id]; 1672 1673 if (status) { 1674 goto fini; 1675 } 1676 1677 for (nsid = 0; nsid < sgroup->num_ns; nsid++) { 1678 if (sgroup->ns_info[nsid].channel) { 1679 spdk_put_io_channel(sgroup->ns_info[nsid].channel); 1680 sgroup->ns_info[nsid].channel = NULL; 1681 } 1682 } 1683 1684 sgroup->num_ns = 0; 1685 free(sgroup->ns_info); 1686 sgroup->ns_info = NULL; 1687 fini: 1688 free(qpair_ctx); 1689 if (cpl_fn) { 1690 cpl_fn(cpl_ctx, status); 1691 } 1692 } 1693 1694 static void nvmf_poll_group_remove_subsystem_msg(void *ctx); 1695 1696 static void 1697 nvmf_poll_group_remove_subsystem_msg(void *ctx) 1698 { 1699 struct spdk_nvmf_qpair *qpair, *qpair_tmp; 1700 struct spdk_nvmf_subsystem *subsystem; 1701 struct spdk_nvmf_poll_group *group; 1702 struct nvmf_qpair_disconnect_many_ctx *qpair_ctx = ctx; 1703 bool qpairs_found = false; 1704 int rc = 0; 1705 1706 group = qpair_ctx->group; 1707 subsystem = qpair_ctx->subsystem; 1708 1709 TAILQ_FOREACH_SAFE(qpair, &group->qpairs, link, qpair_tmp) { 1710 if ((qpair->ctrlr != NULL) && (qpair->ctrlr->subsys == subsystem)) { 1711 qpairs_found = true; 1712 rc = spdk_nvmf_qpair_disconnect(qpair); 1713 if (rc && rc != -EINPROGRESS) { 1714 break; 1715 } 1716 } 1717 } 1718 1719 if (!qpairs_found) { 1720 _nvmf_poll_group_remove_subsystem_cb(ctx, 0); 1721 return; 1722 } 1723 1724 /* Some qpairs are in process of being disconnected. Send a message and try to remove them again */ 1725 spdk_thread_send_msg(spdk_get_thread(), nvmf_poll_group_remove_subsystem_msg, ctx); 1726 } 1727 1728 void 1729 nvmf_poll_group_remove_subsystem(struct spdk_nvmf_poll_group *group, 1730 struct spdk_nvmf_subsystem *subsystem, 1731 spdk_nvmf_poll_group_mod_done cb_fn, void *cb_arg) 1732 { 1733 struct spdk_nvmf_subsystem_poll_group *sgroup; 1734 struct nvmf_qpair_disconnect_many_ctx *ctx; 1735 uint32_t i; 1736 1737 SPDK_DTRACE_PROBE3_TICKS(nvmf_poll_group_remove_subsystem, group, spdk_thread_get_id(group->thread), 1738 subsystem->subnqn); 1739 1740 ctx = calloc(1, sizeof(struct nvmf_qpair_disconnect_many_ctx)); 1741 if (!ctx) { 1742 SPDK_ERRLOG("Unable to allocate memory for context to remove poll subsystem\n"); 1743 if (cb_fn) { 1744 cb_fn(cb_arg, -1); 1745 } 1746 return; 1747 } 1748 1749 ctx->group = group; 1750 ctx->subsystem = subsystem; 1751 ctx->cpl_fn = cb_fn; 1752 ctx->cpl_ctx = cb_arg; 1753 1754 sgroup = &group->sgroups[subsystem->id]; 1755 sgroup->state = SPDK_NVMF_SUBSYSTEM_INACTIVE; 1756 1757 for (i = 0; i < sgroup->num_ns; i++) { 1758 sgroup->ns_info[i].state = SPDK_NVMF_SUBSYSTEM_INACTIVE; 1759 } 1760 1761 nvmf_poll_group_remove_subsystem_msg(ctx); 1762 } 1763 1764 void 1765 nvmf_poll_group_pause_subsystem(struct spdk_nvmf_poll_group *group, 1766 struct spdk_nvmf_subsystem *subsystem, 1767 uint32_t nsid, 1768 spdk_nvmf_poll_group_mod_done cb_fn, void *cb_arg) 1769 { 1770 struct spdk_nvmf_subsystem_poll_group *sgroup; 1771 struct spdk_nvmf_subsystem_pg_ns_info *ns_info = NULL; 1772 int rc = 0; 1773 uint32_t i; 1774 1775 if (subsystem->id >= group->num_sgroups) { 1776 rc = -1; 1777 goto fini; 1778 } 1779 1780 sgroup = &group->sgroups[subsystem->id]; 1781 if (sgroup->state == SPDK_NVMF_SUBSYSTEM_PAUSED) { 1782 goto fini; 1783 } 1784 sgroup->state = SPDK_NVMF_SUBSYSTEM_PAUSING; 1785 1786 if (nsid == SPDK_NVME_GLOBAL_NS_TAG) { 1787 for (i = 0; i < sgroup->num_ns; i++) { 1788 ns_info = &sgroup->ns_info[i]; 1789 ns_info->state = SPDK_NVMF_SUBSYSTEM_PAUSING; 1790 } 1791 } else { 1792 /* NOTE: This implicitly also checks for 0, since 0 - 1 wraps around to UINT32_MAX. */ 1793 if (nsid - 1 < sgroup->num_ns) { 1794 ns_info = &sgroup->ns_info[nsid - 1]; 1795 ns_info->state = SPDK_NVMF_SUBSYSTEM_PAUSING; 1796 } 1797 } 1798 1799 if (sgroup->mgmt_io_outstanding > 0) { 1800 assert(sgroup->cb_fn == NULL); 1801 sgroup->cb_fn = cb_fn; 1802 assert(sgroup->cb_arg == NULL); 1803 sgroup->cb_arg = cb_arg; 1804 return; 1805 } 1806 1807 if (nsid == SPDK_NVME_GLOBAL_NS_TAG) { 1808 for (i = 0; i < sgroup->num_ns; i++) { 1809 ns_info = &sgroup->ns_info[i]; 1810 1811 if (ns_info->io_outstanding > 0) { 1812 assert(sgroup->cb_fn == NULL); 1813 sgroup->cb_fn = cb_fn; 1814 assert(sgroup->cb_arg == NULL); 1815 sgroup->cb_arg = cb_arg; 1816 return; 1817 } 1818 } 1819 } else { 1820 if (ns_info != NULL && ns_info->io_outstanding > 0) { 1821 assert(sgroup->cb_fn == NULL); 1822 sgroup->cb_fn = cb_fn; 1823 assert(sgroup->cb_arg == NULL); 1824 sgroup->cb_arg = cb_arg; 1825 return; 1826 } 1827 } 1828 1829 assert(sgroup->mgmt_io_outstanding == 0); 1830 sgroup->state = SPDK_NVMF_SUBSYSTEM_PAUSED; 1831 fini: 1832 if (cb_fn) { 1833 cb_fn(cb_arg, rc); 1834 } 1835 } 1836 1837 void 1838 nvmf_poll_group_resume_subsystem(struct spdk_nvmf_poll_group *group, 1839 struct spdk_nvmf_subsystem *subsystem, 1840 spdk_nvmf_poll_group_mod_done cb_fn, void *cb_arg) 1841 { 1842 struct spdk_nvmf_request *req, *tmp; 1843 struct spdk_nvmf_subsystem_poll_group *sgroup; 1844 int rc = 0; 1845 uint32_t i; 1846 1847 if (subsystem->id >= group->num_sgroups) { 1848 rc = -1; 1849 goto fini; 1850 } 1851 1852 sgroup = &group->sgroups[subsystem->id]; 1853 1854 if (sgroup->state == SPDK_NVMF_SUBSYSTEM_ACTIVE) { 1855 goto fini; 1856 } 1857 1858 rc = poll_group_update_subsystem(group, subsystem); 1859 if (rc) { 1860 goto fini; 1861 } 1862 1863 for (i = 0; i < sgroup->num_ns; i++) { 1864 sgroup->ns_info[i].state = SPDK_NVMF_SUBSYSTEM_ACTIVE; 1865 } 1866 1867 sgroup->state = SPDK_NVMF_SUBSYSTEM_ACTIVE; 1868 1869 /* Release all queued requests */ 1870 TAILQ_FOREACH_SAFE(req, &sgroup->queued, link, tmp) { 1871 TAILQ_REMOVE(&sgroup->queued, req, link); 1872 if (spdk_nvmf_request_using_zcopy(req)) { 1873 spdk_nvmf_request_zcopy_start(req); 1874 } else { 1875 spdk_nvmf_request_exec(req); 1876 } 1877 1878 } 1879 fini: 1880 if (cb_fn) { 1881 cb_fn(cb_arg, rc); 1882 } 1883 } 1884 1885 1886 struct spdk_nvmf_poll_group * 1887 spdk_nvmf_get_optimal_poll_group(struct spdk_nvmf_qpair *qpair) 1888 { 1889 struct spdk_nvmf_transport_poll_group *tgroup; 1890 1891 tgroup = nvmf_transport_get_optimal_poll_group(qpair->transport, qpair); 1892 1893 if (tgroup == NULL) { 1894 return NULL; 1895 } 1896 1897 return tgroup->group; 1898 } 1899 1900 void 1901 spdk_nvmf_poll_group_dump_stat(struct spdk_nvmf_poll_group *group, struct spdk_json_write_ctx *w) 1902 { 1903 struct spdk_nvmf_transport_poll_group *tgroup; 1904 1905 spdk_json_write_object_begin(w); 1906 1907 spdk_json_write_named_string(w, "name", spdk_thread_get_name(spdk_get_thread())); 1908 spdk_json_write_named_uint32(w, "admin_qpairs", group->stat.admin_qpairs); 1909 spdk_json_write_named_uint32(w, "io_qpairs", group->stat.io_qpairs); 1910 spdk_json_write_named_uint32(w, "current_admin_qpairs", group->stat.current_admin_qpairs); 1911 spdk_json_write_named_uint32(w, "current_io_qpairs", group->stat.current_io_qpairs); 1912 spdk_json_write_named_uint64(w, "pending_bdev_io", group->stat.pending_bdev_io); 1913 spdk_json_write_named_uint64(w, "completed_nvme_io", group->stat.completed_nvme_io); 1914 1915 spdk_json_write_named_array_begin(w, "transports"); 1916 1917 TAILQ_FOREACH(tgroup, &group->tgroups, link) { 1918 spdk_json_write_object_begin(w); 1919 /* 1920 * The trtype field intentionally contains a transport name as this is more informative. 1921 * The field has not been renamed for backward compatibility. 1922 */ 1923 spdk_json_write_named_string(w, "trtype", spdk_nvmf_get_transport_name(tgroup->transport)); 1924 1925 if (tgroup->transport->ops->poll_group_dump_stat) { 1926 tgroup->transport->ops->poll_group_dump_stat(tgroup, w); 1927 } 1928 1929 spdk_json_write_object_end(w); 1930 } 1931 1932 spdk_json_write_array_end(w); 1933 spdk_json_write_object_end(w); 1934 } 1935