1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) Intel Corporation. All rights reserved. 5 * Copyright (c) 2018-2019, 2021 Mellanox Technologies LTD. All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include "spdk/stdinc.h" 35 36 #include "spdk/bdev.h" 37 #include "spdk/bit_array.h" 38 #include "spdk/thread.h" 39 #include "spdk/nvmf.h" 40 #include "spdk/trace.h" 41 #include "spdk/endian.h" 42 #include "spdk/string.h" 43 44 #include "spdk/log.h" 45 46 #include "nvmf_internal.h" 47 #include "transport.h" 48 49 SPDK_LOG_REGISTER_COMPONENT(nvmf) 50 51 #define SPDK_NVMF_DEFAULT_MAX_SUBSYSTEMS 1024 52 #define SPDK_NVMF_DEFAULT_ACCEPT_POLL_RATE_US 10000 53 54 static TAILQ_HEAD(, spdk_nvmf_tgt) g_nvmf_tgts = TAILQ_HEAD_INITIALIZER(g_nvmf_tgts); 55 56 typedef void (*nvmf_qpair_disconnect_cpl)(void *ctx, int status); 57 static void nvmf_tgt_destroy_poll_group(void *io_device, void *ctx_buf); 58 59 /* supplied to a single call to nvmf_qpair_disconnect */ 60 struct nvmf_qpair_disconnect_ctx { 61 struct spdk_nvmf_qpair *qpair; 62 struct spdk_nvmf_ctrlr *ctrlr; 63 nvmf_qpair_disconnect_cb cb_fn; 64 struct spdk_thread *thread; 65 void *ctx; 66 uint16_t qid; 67 }; 68 69 /* 70 * There are several times when we need to iterate through the list of all qpairs and selectively delete them. 71 * In order to do this sequentially without overlap, we must provide a context to recover the next qpair from 72 * to enable calling nvmf_qpair_disconnect on the next desired qpair. 73 */ 74 struct nvmf_qpair_disconnect_many_ctx { 75 struct spdk_nvmf_subsystem *subsystem; 76 struct spdk_nvmf_poll_group *group; 77 spdk_nvmf_poll_group_mod_done cpl_fn; 78 void *cpl_ctx; 79 }; 80 81 static void 82 nvmf_qpair_set_state(struct spdk_nvmf_qpair *qpair, 83 enum spdk_nvmf_qpair_state state) 84 { 85 assert(qpair != NULL); 86 assert(qpair->group->thread == spdk_get_thread()); 87 88 qpair->state = state; 89 } 90 91 static int 92 nvmf_poll_group_poll(void *ctx) 93 { 94 struct spdk_nvmf_poll_group *group = ctx; 95 int rc; 96 int count = 0; 97 struct spdk_nvmf_transport_poll_group *tgroup; 98 99 TAILQ_FOREACH(tgroup, &group->tgroups, link) { 100 rc = nvmf_transport_poll_group_poll(tgroup); 101 if (rc < 0) { 102 return SPDK_POLLER_BUSY; 103 } 104 count += rc; 105 } 106 107 return count > 0 ? SPDK_POLLER_BUSY : SPDK_POLLER_IDLE; 108 } 109 110 static int 111 nvmf_tgt_create_poll_group(void *io_device, void *ctx_buf) 112 { 113 struct spdk_nvmf_tgt *tgt = io_device; 114 struct spdk_nvmf_poll_group *group = ctx_buf; 115 struct spdk_nvmf_transport *transport; 116 uint32_t sid; 117 118 TAILQ_INIT(&group->tgroups); 119 TAILQ_INIT(&group->qpairs); 120 121 TAILQ_FOREACH(transport, &tgt->transports, link) { 122 nvmf_poll_group_add_transport(group, transport); 123 } 124 125 group->num_sgroups = tgt->max_subsystems; 126 group->sgroups = calloc(tgt->max_subsystems, sizeof(struct spdk_nvmf_subsystem_poll_group)); 127 if (!group->sgroups) { 128 return -ENOMEM; 129 } 130 131 for (sid = 0; sid < tgt->max_subsystems; sid++) { 132 struct spdk_nvmf_subsystem *subsystem; 133 134 subsystem = tgt->subsystems[sid]; 135 if (!subsystem) { 136 continue; 137 } 138 139 if (nvmf_poll_group_add_subsystem(group, subsystem, NULL, NULL) != 0) { 140 nvmf_tgt_destroy_poll_group(io_device, ctx_buf); 141 return -1; 142 } 143 } 144 145 pthread_mutex_lock(&tgt->mutex); 146 TAILQ_INSERT_TAIL(&tgt->poll_groups, group, link); 147 pthread_mutex_unlock(&tgt->mutex); 148 149 group->poller = SPDK_POLLER_REGISTER(nvmf_poll_group_poll, group, 0); 150 group->thread = spdk_get_thread(); 151 152 return 0; 153 } 154 155 static void 156 nvmf_tgt_destroy_poll_group(void *io_device, void *ctx_buf) 157 { 158 struct spdk_nvmf_tgt *tgt = io_device; 159 struct spdk_nvmf_poll_group *group = ctx_buf; 160 struct spdk_nvmf_transport_poll_group *tgroup, *tmp; 161 struct spdk_nvmf_subsystem_poll_group *sgroup; 162 uint32_t sid, nsid; 163 164 pthread_mutex_lock(&tgt->mutex); 165 TAILQ_REMOVE(&tgt->poll_groups, group, link); 166 pthread_mutex_unlock(&tgt->mutex); 167 168 TAILQ_FOREACH_SAFE(tgroup, &group->tgroups, link, tmp) { 169 TAILQ_REMOVE(&group->tgroups, tgroup, link); 170 nvmf_transport_poll_group_destroy(tgroup); 171 } 172 173 for (sid = 0; sid < group->num_sgroups; sid++) { 174 sgroup = &group->sgroups[sid]; 175 176 for (nsid = 0; nsid < sgroup->num_ns; nsid++) { 177 if (sgroup->ns_info[nsid].channel) { 178 spdk_put_io_channel(sgroup->ns_info[nsid].channel); 179 sgroup->ns_info[nsid].channel = NULL; 180 } 181 } 182 183 free(sgroup->ns_info); 184 } 185 186 free(group->sgroups); 187 188 spdk_poller_unregister(&group->poller); 189 190 if (group->destroy_cb_fn) { 191 group->destroy_cb_fn(group->destroy_cb_arg, 0); 192 } 193 } 194 195 static void 196 _nvmf_tgt_disconnect_next_qpair(void *ctx) 197 { 198 struct spdk_nvmf_qpair *qpair; 199 struct nvmf_qpair_disconnect_many_ctx *qpair_ctx = ctx; 200 struct spdk_nvmf_poll_group *group = qpair_ctx->group; 201 struct spdk_io_channel *ch; 202 int rc = 0; 203 204 qpair = TAILQ_FIRST(&group->qpairs); 205 206 if (qpair) { 207 rc = spdk_nvmf_qpair_disconnect(qpair, _nvmf_tgt_disconnect_next_qpair, ctx); 208 } 209 210 if (!qpair || rc != 0) { 211 /* When the refcount from the channels reaches 0, nvmf_tgt_destroy_poll_group will be called. */ 212 ch = spdk_io_channel_from_ctx(group); 213 spdk_put_io_channel(ch); 214 free(qpair_ctx); 215 } 216 } 217 218 static void 219 nvmf_tgt_destroy_poll_group_qpairs(struct spdk_nvmf_poll_group *group) 220 { 221 struct nvmf_qpair_disconnect_many_ctx *ctx; 222 223 ctx = calloc(1, sizeof(struct nvmf_qpair_disconnect_many_ctx)); 224 225 if (!ctx) { 226 SPDK_ERRLOG("Failed to allocate memory for destroy poll group ctx\n"); 227 return; 228 } 229 230 ctx->group = group; 231 _nvmf_tgt_disconnect_next_qpair(ctx); 232 } 233 234 static int 235 nvmf_tgt_accept(void *ctx) 236 { 237 struct spdk_nvmf_tgt *tgt = ctx; 238 struct spdk_nvmf_transport *transport, *tmp; 239 int count = 0; 240 241 TAILQ_FOREACH_SAFE(transport, &tgt->transports, link, tmp) { 242 count += nvmf_transport_accept(transport); 243 } 244 245 return count; 246 } 247 248 struct spdk_nvmf_tgt * 249 spdk_nvmf_tgt_create(struct spdk_nvmf_target_opts *opts) 250 { 251 struct spdk_nvmf_tgt *tgt, *tmp_tgt; 252 uint32_t acceptor_poll_rate; 253 254 if (strnlen(opts->name, NVMF_TGT_NAME_MAX_LENGTH) == NVMF_TGT_NAME_MAX_LENGTH) { 255 SPDK_ERRLOG("Provided target name exceeds the max length of %u.\n", NVMF_TGT_NAME_MAX_LENGTH); 256 return NULL; 257 } 258 259 TAILQ_FOREACH(tmp_tgt, &g_nvmf_tgts, link) { 260 if (!strncmp(opts->name, tmp_tgt->name, NVMF_TGT_NAME_MAX_LENGTH)) { 261 SPDK_ERRLOG("Provided target name must be unique.\n"); 262 return NULL; 263 } 264 } 265 266 tgt = calloc(1, sizeof(*tgt)); 267 if (!tgt) { 268 return NULL; 269 } 270 271 snprintf(tgt->name, NVMF_TGT_NAME_MAX_LENGTH, "%s", opts->name); 272 273 if (!opts || !opts->max_subsystems) { 274 tgt->max_subsystems = SPDK_NVMF_DEFAULT_MAX_SUBSYSTEMS; 275 } else { 276 tgt->max_subsystems = opts->max_subsystems; 277 } 278 279 if (!opts || !opts->acceptor_poll_rate) { 280 acceptor_poll_rate = SPDK_NVMF_DEFAULT_ACCEPT_POLL_RATE_US; 281 } else { 282 acceptor_poll_rate = opts->acceptor_poll_rate; 283 } 284 285 tgt->discovery_genctr = 0; 286 TAILQ_INIT(&tgt->transports); 287 TAILQ_INIT(&tgt->poll_groups); 288 289 tgt->subsystems = calloc(tgt->max_subsystems, sizeof(struct spdk_nvmf_subsystem *)); 290 if (!tgt->subsystems) { 291 free(tgt); 292 return NULL; 293 } 294 295 pthread_mutex_init(&tgt->mutex, NULL); 296 297 tgt->accept_poller = SPDK_POLLER_REGISTER(nvmf_tgt_accept, tgt, acceptor_poll_rate); 298 if (!tgt->accept_poller) { 299 pthread_mutex_destroy(&tgt->mutex); 300 free(tgt->subsystems); 301 free(tgt); 302 return NULL; 303 } 304 305 spdk_io_device_register(tgt, 306 nvmf_tgt_create_poll_group, 307 nvmf_tgt_destroy_poll_group, 308 sizeof(struct spdk_nvmf_poll_group), 309 tgt->name); 310 311 TAILQ_INSERT_HEAD(&g_nvmf_tgts, tgt, link); 312 313 return tgt; 314 } 315 316 static void 317 _nvmf_tgt_destroy_next_transport(void *ctx) 318 { 319 struct spdk_nvmf_tgt *tgt = ctx; 320 struct spdk_nvmf_transport *transport; 321 322 if (!TAILQ_EMPTY(&tgt->transports)) { 323 transport = TAILQ_FIRST(&tgt->transports); 324 TAILQ_REMOVE(&tgt->transports, transport, link); 325 spdk_nvmf_transport_destroy(transport, _nvmf_tgt_destroy_next_transport, tgt); 326 } else { 327 spdk_nvmf_tgt_destroy_done_fn *destroy_cb_fn = tgt->destroy_cb_fn; 328 void *destroy_cb_arg = tgt->destroy_cb_arg; 329 330 pthread_mutex_destroy(&tgt->mutex); 331 free(tgt); 332 333 if (destroy_cb_fn) { 334 destroy_cb_fn(destroy_cb_arg, 0); 335 } 336 } 337 } 338 339 static void 340 nvmf_tgt_destroy_cb(void *io_device) 341 { 342 struct spdk_nvmf_tgt *tgt = io_device; 343 uint32_t i; 344 345 if (tgt->subsystems) { 346 for (i = 0; i < tgt->max_subsystems; i++) { 347 if (tgt->subsystems[i]) { 348 nvmf_subsystem_remove_all_listeners(tgt->subsystems[i], true); 349 spdk_nvmf_subsystem_destroy(tgt->subsystems[i]); 350 } 351 } 352 free(tgt->subsystems); 353 } 354 355 _nvmf_tgt_destroy_next_transport(tgt); 356 } 357 358 void 359 spdk_nvmf_tgt_destroy(struct spdk_nvmf_tgt *tgt, 360 spdk_nvmf_tgt_destroy_done_fn cb_fn, 361 void *cb_arg) 362 { 363 tgt->destroy_cb_fn = cb_fn; 364 tgt->destroy_cb_arg = cb_arg; 365 366 spdk_poller_unregister(&tgt->accept_poller); 367 368 TAILQ_REMOVE(&g_nvmf_tgts, tgt, link); 369 370 spdk_io_device_unregister(tgt, nvmf_tgt_destroy_cb); 371 } 372 373 const char * 374 spdk_nvmf_tgt_get_name(struct spdk_nvmf_tgt *tgt) 375 { 376 return tgt->name; 377 } 378 379 struct spdk_nvmf_tgt * 380 spdk_nvmf_get_tgt(const char *name) 381 { 382 struct spdk_nvmf_tgt *tgt; 383 uint32_t num_targets = 0; 384 385 TAILQ_FOREACH(tgt, &g_nvmf_tgts, link) { 386 if (name) { 387 if (!strncmp(tgt->name, name, NVMF_TGT_NAME_MAX_LENGTH)) { 388 return tgt; 389 } 390 } 391 num_targets++; 392 } 393 394 /* 395 * special case. If there is only one target and 396 * no name was specified, return the only available 397 * target. If there is more than one target, name must 398 * be specified. 399 */ 400 if (!name && num_targets == 1) { 401 return TAILQ_FIRST(&g_nvmf_tgts); 402 } 403 404 return NULL; 405 } 406 407 struct spdk_nvmf_tgt * 408 spdk_nvmf_get_first_tgt(void) 409 { 410 return TAILQ_FIRST(&g_nvmf_tgts); 411 } 412 413 struct spdk_nvmf_tgt * 414 spdk_nvmf_get_next_tgt(struct spdk_nvmf_tgt *prev) 415 { 416 return TAILQ_NEXT(prev, link); 417 } 418 419 static void 420 nvmf_write_subsystem_config_json(struct spdk_json_write_ctx *w, 421 struct spdk_nvmf_subsystem *subsystem) 422 { 423 struct spdk_nvmf_host *host; 424 struct spdk_nvmf_subsystem_listener *listener; 425 const struct spdk_nvme_transport_id *trid; 426 struct spdk_nvmf_ns *ns; 427 struct spdk_nvmf_ns_opts ns_opts; 428 uint32_t max_namespaces; 429 char uuid_str[SPDK_UUID_STRING_LEN]; 430 const char *adrfam; 431 432 if (spdk_nvmf_subsystem_get_type(subsystem) != SPDK_NVMF_SUBTYPE_NVME) { 433 return; 434 } 435 436 /* { */ 437 spdk_json_write_object_begin(w); 438 spdk_json_write_named_string(w, "method", "nvmf_create_subsystem"); 439 440 /* "params" : { */ 441 spdk_json_write_named_object_begin(w, "params"); 442 spdk_json_write_named_string(w, "nqn", spdk_nvmf_subsystem_get_nqn(subsystem)); 443 spdk_json_write_named_bool(w, "allow_any_host", spdk_nvmf_subsystem_get_allow_any_host(subsystem)); 444 spdk_json_write_named_string(w, "serial_number", spdk_nvmf_subsystem_get_sn(subsystem)); 445 spdk_json_write_named_string(w, "model_number", spdk_nvmf_subsystem_get_mn(subsystem)); 446 447 max_namespaces = spdk_nvmf_subsystem_get_max_namespaces(subsystem); 448 if (max_namespaces != 0) { 449 spdk_json_write_named_uint32(w, "max_namespaces", max_namespaces); 450 } 451 452 /* } "params" */ 453 spdk_json_write_object_end(w); 454 455 /* } */ 456 spdk_json_write_object_end(w); 457 458 for (listener = spdk_nvmf_subsystem_get_first_listener(subsystem); listener != NULL; 459 listener = spdk_nvmf_subsystem_get_next_listener(subsystem, listener)) { 460 trid = spdk_nvmf_subsystem_listener_get_trid(listener); 461 462 adrfam = spdk_nvme_transport_id_adrfam_str(trid->adrfam); 463 464 spdk_json_write_object_begin(w); 465 spdk_json_write_named_string(w, "method", "nvmf_subsystem_add_listener"); 466 467 /* "params" : { */ 468 spdk_json_write_named_object_begin(w, "params"); 469 470 spdk_json_write_named_string(w, "nqn", spdk_nvmf_subsystem_get_nqn(subsystem)); 471 472 /* "listen_address" : { */ 473 spdk_json_write_named_object_begin(w, "listen_address"); 474 475 spdk_json_write_named_string(w, "trtype", trid->trstring); 476 if (adrfam) { 477 spdk_json_write_named_string(w, "adrfam", adrfam); 478 } 479 480 spdk_json_write_named_string(w, "traddr", trid->traddr); 481 spdk_json_write_named_string(w, "trsvcid", trid->trsvcid); 482 /* } "listen_address" */ 483 spdk_json_write_object_end(w); 484 485 /* } "params" */ 486 spdk_json_write_object_end(w); 487 488 /* } */ 489 spdk_json_write_object_end(w); 490 } 491 492 for (host = spdk_nvmf_subsystem_get_first_host(subsystem); host != NULL; 493 host = spdk_nvmf_subsystem_get_next_host(subsystem, host)) { 494 495 spdk_json_write_object_begin(w); 496 spdk_json_write_named_string(w, "method", "nvmf_subsystem_add_host"); 497 498 /* "params" : { */ 499 spdk_json_write_named_object_begin(w, "params"); 500 501 spdk_json_write_named_string(w, "nqn", spdk_nvmf_subsystem_get_nqn(subsystem)); 502 spdk_json_write_named_string(w, "host", spdk_nvmf_host_get_nqn(host)); 503 504 /* } "params" */ 505 spdk_json_write_object_end(w); 506 507 /* } */ 508 spdk_json_write_object_end(w); 509 } 510 511 for (ns = spdk_nvmf_subsystem_get_first_ns(subsystem); ns != NULL; 512 ns = spdk_nvmf_subsystem_get_next_ns(subsystem, ns)) { 513 spdk_nvmf_ns_get_opts(ns, &ns_opts, sizeof(ns_opts)); 514 515 spdk_json_write_object_begin(w); 516 spdk_json_write_named_string(w, "method", "nvmf_subsystem_add_ns"); 517 518 /* "params" : { */ 519 spdk_json_write_named_object_begin(w, "params"); 520 521 spdk_json_write_named_string(w, "nqn", spdk_nvmf_subsystem_get_nqn(subsystem)); 522 523 /* "namespace" : { */ 524 spdk_json_write_named_object_begin(w, "namespace"); 525 526 spdk_json_write_named_uint32(w, "nsid", spdk_nvmf_ns_get_id(ns)); 527 spdk_json_write_named_string(w, "bdev_name", spdk_bdev_get_name(spdk_nvmf_ns_get_bdev(ns))); 528 529 if (!spdk_mem_all_zero(ns_opts.nguid, sizeof(ns_opts.nguid))) { 530 SPDK_STATIC_ASSERT(sizeof(ns_opts.nguid) == sizeof(uint64_t) * 2, "size mismatch"); 531 spdk_json_write_named_string_fmt(w, "nguid", "%016"PRIX64"%016"PRIX64, from_be64(&ns_opts.nguid[0]), 532 from_be64(&ns_opts.nguid[8])); 533 } 534 535 if (!spdk_mem_all_zero(ns_opts.eui64, sizeof(ns_opts.eui64))) { 536 SPDK_STATIC_ASSERT(sizeof(ns_opts.eui64) == sizeof(uint64_t), "size mismatch"); 537 spdk_json_write_named_string_fmt(w, "eui64", "%016"PRIX64, from_be64(&ns_opts.eui64)); 538 } 539 540 if (!spdk_mem_all_zero(&ns_opts.uuid, sizeof(ns_opts.uuid))) { 541 spdk_uuid_fmt_lower(uuid_str, sizeof(uuid_str), &ns_opts.uuid); 542 spdk_json_write_named_string(w, "uuid", uuid_str); 543 } 544 545 /* "namespace" */ 546 spdk_json_write_object_end(w); 547 548 /* } "params" */ 549 spdk_json_write_object_end(w); 550 551 /* } */ 552 spdk_json_write_object_end(w); 553 } 554 } 555 556 void 557 spdk_nvmf_tgt_write_config_json(struct spdk_json_write_ctx *w, struct spdk_nvmf_tgt *tgt) 558 { 559 struct spdk_nvmf_subsystem *subsystem; 560 struct spdk_nvmf_transport *transport; 561 562 spdk_json_write_object_begin(w); 563 spdk_json_write_named_string(w, "method", "nvmf_set_max_subsystems"); 564 565 spdk_json_write_named_object_begin(w, "params"); 566 spdk_json_write_named_uint32(w, "max_subsystems", tgt->max_subsystems); 567 spdk_json_write_object_end(w); 568 569 spdk_json_write_object_end(w); 570 571 /* write transports */ 572 TAILQ_FOREACH(transport, &tgt->transports, link) { 573 spdk_json_write_object_begin(w); 574 spdk_json_write_named_string(w, "method", "nvmf_create_transport"); 575 576 spdk_json_write_named_object_begin(w, "params"); 577 spdk_json_write_named_string(w, "trtype", transport->ops->name); 578 spdk_json_write_named_uint32(w, "max_queue_depth", transport->opts.max_queue_depth); 579 spdk_json_write_named_uint32(w, "max_io_qpairs_per_ctrlr", 580 transport->opts.max_qpairs_per_ctrlr - 1); 581 spdk_json_write_named_uint32(w, "in_capsule_data_size", transport->opts.in_capsule_data_size); 582 spdk_json_write_named_uint32(w, "max_io_size", transport->opts.max_io_size); 583 spdk_json_write_named_uint32(w, "io_unit_size", transport->opts.io_unit_size); 584 spdk_json_write_named_uint32(w, "max_aq_depth", transport->opts.max_aq_depth); 585 spdk_json_write_named_uint32(w, "num_shared_buffers", transport->opts.num_shared_buffers); 586 spdk_json_write_named_uint32(w, "buf_cache_size", transport->opts.buf_cache_size); 587 spdk_json_write_named_bool(w, "dif_insert_or_strip", transport->opts.dif_insert_or_strip); 588 if (transport->ops->dump_opts) { 589 transport->ops->dump_opts(transport, w); 590 } 591 spdk_json_write_named_uint32(w, "abort_timeout_sec", transport->opts.abort_timeout_sec); 592 spdk_json_write_object_end(w); 593 594 spdk_json_write_object_end(w); 595 } 596 597 subsystem = spdk_nvmf_subsystem_get_first(tgt); 598 while (subsystem) { 599 nvmf_write_subsystem_config_json(w, subsystem); 600 subsystem = spdk_nvmf_subsystem_get_next(subsystem); 601 } 602 } 603 604 static void 605 nvmf_listen_opts_copy(struct spdk_nvmf_listen_opts *opts, 606 const struct spdk_nvmf_listen_opts *opts_src, size_t opts_size) 607 { 608 assert(opts); 609 assert(opts_src); 610 611 opts->opts_size = opts_size; 612 613 #define SET_FIELD(field) \ 614 if (offsetof(struct spdk_nvmf_listen_opts, field) + sizeof(opts->field) <= opts_size) { \ 615 opts->field = opts_src->field; \ 616 } \ 617 618 SET_FIELD(transport_specific); 619 #undef SET_FIELD 620 621 /* Do not remove this statement, you should always update this statement when you adding a new field, 622 * and do not forget to add the SET_FIELD statement for your added field. */ 623 SPDK_STATIC_ASSERT(sizeof(struct spdk_nvmf_listen_opts) == 16, "Incorrect size"); 624 } 625 626 void 627 spdk_nvmf_listen_opts_init(struct spdk_nvmf_listen_opts *opts, size_t opts_size) 628 { 629 struct spdk_nvmf_listen_opts opts_local = {}; 630 631 /* local version of opts should have defaults set here */ 632 633 nvmf_listen_opts_copy(opts, &opts_local, opts_size); 634 } 635 636 int 637 spdk_nvmf_tgt_listen_ext(struct spdk_nvmf_tgt *tgt, const struct spdk_nvme_transport_id *trid, 638 struct spdk_nvmf_listen_opts *opts) 639 { 640 struct spdk_nvmf_transport *transport; 641 int rc; 642 struct spdk_nvmf_listen_opts opts_local = {}; 643 644 if (!opts) { 645 SPDK_ERRLOG("opts should not be NULL\n"); 646 return -EINVAL; 647 } 648 649 if (!opts->opts_size) { 650 SPDK_ERRLOG("The opts_size in opts structure should not be zero\n"); 651 return -EINVAL; 652 } 653 654 transport = spdk_nvmf_tgt_get_transport(tgt, trid->trstring); 655 if (!transport) { 656 SPDK_ERRLOG("Unable to find %s transport. The transport must be created first also make sure it is properly registered.\n", 657 trid->trstring); 658 return -EINVAL; 659 } 660 661 nvmf_listen_opts_copy(&opts_local, opts, opts->opts_size); 662 rc = spdk_nvmf_transport_listen(transport, trid, &opts_local); 663 if (rc < 0) { 664 SPDK_ERRLOG("Unable to listen on address '%s'\n", trid->traddr); 665 } 666 667 return rc; 668 } 669 670 int 671 spdk_nvmf_tgt_stop_listen(struct spdk_nvmf_tgt *tgt, 672 struct spdk_nvme_transport_id *trid) 673 { 674 struct spdk_nvmf_transport *transport; 675 int rc; 676 677 transport = spdk_nvmf_tgt_get_transport(tgt, trid->trstring); 678 if (!transport) { 679 SPDK_ERRLOG("Unable to find %s transport. The transport must be created first also make sure it is properly registered.\n", 680 trid->trstring); 681 return -EINVAL; 682 } 683 684 rc = spdk_nvmf_transport_stop_listen(transport, trid); 685 if (rc < 0) { 686 SPDK_ERRLOG("Failed to stop listening on address '%s'\n", trid->traddr); 687 return rc; 688 } 689 return 0; 690 } 691 692 struct spdk_nvmf_tgt_add_transport_ctx { 693 struct spdk_nvmf_tgt *tgt; 694 struct spdk_nvmf_transport *transport; 695 spdk_nvmf_tgt_add_transport_done_fn cb_fn; 696 void *cb_arg; 697 }; 698 699 static void 700 _nvmf_tgt_add_transport_done(struct spdk_io_channel_iter *i, int status) 701 { 702 struct spdk_nvmf_tgt_add_transport_ctx *ctx = spdk_io_channel_iter_get_ctx(i); 703 704 ctx->cb_fn(ctx->cb_arg, status); 705 706 free(ctx); 707 } 708 709 static void 710 _nvmf_tgt_add_transport(struct spdk_io_channel_iter *i) 711 { 712 struct spdk_nvmf_tgt_add_transport_ctx *ctx = spdk_io_channel_iter_get_ctx(i); 713 struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i); 714 struct spdk_nvmf_poll_group *group = spdk_io_channel_get_ctx(ch); 715 int rc; 716 717 rc = nvmf_poll_group_add_transport(group, ctx->transport); 718 spdk_for_each_channel_continue(i, rc); 719 } 720 721 void spdk_nvmf_tgt_add_transport(struct spdk_nvmf_tgt *tgt, 722 struct spdk_nvmf_transport *transport, 723 spdk_nvmf_tgt_add_transport_done_fn cb_fn, 724 void *cb_arg) 725 { 726 struct spdk_nvmf_tgt_add_transport_ctx *ctx; 727 728 if (spdk_nvmf_tgt_get_transport(tgt, transport->ops->name)) { 729 cb_fn(cb_arg, -EEXIST); 730 return; /* transport already created */ 731 } 732 733 transport->tgt = tgt; 734 TAILQ_INSERT_TAIL(&tgt->transports, transport, link); 735 736 ctx = calloc(1, sizeof(*ctx)); 737 if (!ctx) { 738 cb_fn(cb_arg, -ENOMEM); 739 return; 740 } 741 742 ctx->tgt = tgt; 743 ctx->transport = transport; 744 ctx->cb_fn = cb_fn; 745 ctx->cb_arg = cb_arg; 746 747 spdk_for_each_channel(tgt, 748 _nvmf_tgt_add_transport, 749 ctx, 750 _nvmf_tgt_add_transport_done); 751 } 752 753 struct spdk_nvmf_subsystem * 754 spdk_nvmf_tgt_find_subsystem(struct spdk_nvmf_tgt *tgt, const char *subnqn) 755 { 756 struct spdk_nvmf_subsystem *subsystem; 757 uint32_t sid; 758 759 if (!subnqn) { 760 return NULL; 761 } 762 763 /* Ensure that subnqn is null terminated */ 764 if (!memchr(subnqn, '\0', SPDK_NVMF_NQN_MAX_LEN + 1)) { 765 SPDK_ERRLOG("Connect SUBNQN is not null terminated\n"); 766 return NULL; 767 } 768 769 for (sid = 0; sid < tgt->max_subsystems; sid++) { 770 subsystem = tgt->subsystems[sid]; 771 if (subsystem == NULL) { 772 continue; 773 } 774 775 if (strcmp(subnqn, subsystem->subnqn) == 0) { 776 return subsystem; 777 } 778 } 779 780 return NULL; 781 } 782 783 struct spdk_nvmf_transport * 784 spdk_nvmf_tgt_get_transport(struct spdk_nvmf_tgt *tgt, const char *transport_name) 785 { 786 struct spdk_nvmf_transport *transport; 787 788 TAILQ_FOREACH(transport, &tgt->transports, link) { 789 if (!strncasecmp(transport->ops->name, transport_name, SPDK_NVMF_TRSTRING_MAX_LEN)) { 790 return transport; 791 } 792 } 793 return NULL; 794 } 795 796 struct nvmf_new_qpair_ctx { 797 struct spdk_nvmf_qpair *qpair; 798 struct spdk_nvmf_poll_group *group; 799 }; 800 801 static void 802 _nvmf_poll_group_add(void *_ctx) 803 { 804 struct nvmf_new_qpair_ctx *ctx = _ctx; 805 struct spdk_nvmf_qpair *qpair = ctx->qpair; 806 struct spdk_nvmf_poll_group *group = ctx->group; 807 808 free(_ctx); 809 810 if (spdk_nvmf_poll_group_add(group, qpair) != 0) { 811 SPDK_ERRLOG("Unable to add the qpair to a poll group.\n"); 812 spdk_nvmf_qpair_disconnect(qpair, NULL, NULL); 813 } 814 } 815 816 void 817 spdk_nvmf_tgt_new_qpair(struct spdk_nvmf_tgt *tgt, struct spdk_nvmf_qpair *qpair) 818 { 819 struct spdk_nvmf_poll_group *group; 820 struct nvmf_new_qpair_ctx *ctx; 821 822 group = spdk_nvmf_get_optimal_poll_group(qpair); 823 if (group == NULL) { 824 if (tgt->next_poll_group == NULL) { 825 tgt->next_poll_group = TAILQ_FIRST(&tgt->poll_groups); 826 if (tgt->next_poll_group == NULL) { 827 SPDK_ERRLOG("No poll groups exist.\n"); 828 spdk_nvmf_qpair_disconnect(qpair, NULL, NULL); 829 return; 830 } 831 } 832 group = tgt->next_poll_group; 833 tgt->next_poll_group = TAILQ_NEXT(group, link); 834 } 835 836 ctx = calloc(1, sizeof(*ctx)); 837 if (!ctx) { 838 SPDK_ERRLOG("Unable to send message to poll group.\n"); 839 spdk_nvmf_qpair_disconnect(qpair, NULL, NULL); 840 return; 841 } 842 843 ctx->qpair = qpair; 844 ctx->group = group; 845 846 spdk_thread_send_msg(group->thread, _nvmf_poll_group_add, ctx); 847 } 848 849 struct spdk_nvmf_poll_group * 850 spdk_nvmf_poll_group_create(struct spdk_nvmf_tgt *tgt) 851 { 852 struct spdk_io_channel *ch; 853 854 ch = spdk_get_io_channel(tgt); 855 if (!ch) { 856 SPDK_ERRLOG("Unable to get I/O channel for target\n"); 857 return NULL; 858 } 859 860 return spdk_io_channel_get_ctx(ch); 861 } 862 863 void 864 spdk_nvmf_poll_group_destroy(struct spdk_nvmf_poll_group *group, 865 spdk_nvmf_poll_group_destroy_done_fn cb_fn, 866 void *cb_arg) 867 { 868 assert(group->destroy_cb_fn == NULL); 869 group->destroy_cb_fn = cb_fn; 870 group->destroy_cb_arg = cb_arg; 871 872 /* This function will put the io_channel associated with this poll group */ 873 nvmf_tgt_destroy_poll_group_qpairs(group); 874 } 875 876 int 877 spdk_nvmf_poll_group_add(struct spdk_nvmf_poll_group *group, 878 struct spdk_nvmf_qpair *qpair) 879 { 880 int rc = -1; 881 struct spdk_nvmf_transport_poll_group *tgroup; 882 883 TAILQ_INIT(&qpair->outstanding); 884 qpair->group = group; 885 qpair->ctrlr = NULL; 886 qpair->disconnect_started = false; 887 888 TAILQ_FOREACH(tgroup, &group->tgroups, link) { 889 if (tgroup->transport == qpair->transport) { 890 rc = nvmf_transport_poll_group_add(tgroup, qpair); 891 break; 892 } 893 } 894 895 /* We add the qpair to the group only it is succesfully added into the tgroup */ 896 if (rc == 0) { 897 TAILQ_INSERT_TAIL(&group->qpairs, qpair, link); 898 nvmf_qpair_set_state(qpair, SPDK_NVMF_QPAIR_ACTIVE); 899 } 900 901 return rc; 902 } 903 904 static void 905 _nvmf_ctrlr_destruct(void *ctx) 906 { 907 struct spdk_nvmf_ctrlr *ctrlr = ctx; 908 909 nvmf_ctrlr_destruct(ctrlr); 910 } 911 912 static void 913 _nvmf_transport_qpair_fini_complete(void *cb_ctx) 914 { 915 struct nvmf_qpair_disconnect_ctx *qpair_ctx = cb_ctx; 916 917 if (qpair_ctx->cb_fn) { 918 spdk_thread_send_msg(qpair_ctx->thread, qpair_ctx->cb_fn, qpair_ctx->ctx); 919 } 920 free(qpair_ctx); 921 } 922 923 static void 924 _nvmf_transport_qpair_fini(void *ctx) 925 { 926 struct nvmf_qpair_disconnect_ctx *qpair_ctx = ctx; 927 928 nvmf_transport_qpair_fini(qpair_ctx->qpair, _nvmf_transport_qpair_fini_complete, qpair_ctx); 929 } 930 931 static void 932 _nvmf_ctrlr_free_from_qpair(void *ctx) 933 { 934 struct nvmf_qpair_disconnect_ctx *qpair_ctx = ctx; 935 struct spdk_nvmf_ctrlr *ctrlr = qpair_ctx->ctrlr; 936 uint32_t count; 937 938 spdk_bit_array_clear(ctrlr->qpair_mask, qpair_ctx->qid); 939 count = spdk_bit_array_count_set(ctrlr->qpair_mask); 940 if (count == 0) { 941 ctrlr->in_destruct = true; 942 spdk_thread_send_msg(ctrlr->subsys->thread, _nvmf_ctrlr_destruct, ctrlr); 943 } 944 945 spdk_thread_send_msg(qpair_ctx->thread, _nvmf_transport_qpair_fini, qpair_ctx); 946 } 947 948 void 949 spdk_nvmf_poll_group_remove(struct spdk_nvmf_qpair *qpair) 950 { 951 struct spdk_nvmf_ctrlr *ctrlr = qpair->ctrlr; 952 struct spdk_nvmf_transport_poll_group *tgroup; 953 struct spdk_nvmf_request *req, *tmp; 954 struct spdk_nvmf_subsystem_poll_group *sgroup; 955 int rc; 956 957 nvmf_qpair_set_state(qpair, SPDK_NVMF_QPAIR_ERROR); 958 959 /* Find the tgroup and remove the qpair from the tgroup */ 960 TAILQ_FOREACH(tgroup, &qpair->group->tgroups, link) { 961 if (tgroup->transport == qpair->transport) { 962 rc = nvmf_transport_poll_group_remove(tgroup, qpair); 963 if (rc && (rc != ENOTSUP)) { 964 SPDK_ERRLOG("Cannot remove qpair=%p from transport group=%p\n", 965 qpair, tgroup); 966 } 967 break; 968 } 969 } 970 971 if (ctrlr) { 972 sgroup = &qpair->group->sgroups[ctrlr->subsys->id]; 973 TAILQ_FOREACH_SAFE(req, &sgroup->queued, link, tmp) { 974 if (req->qpair == qpair) { 975 TAILQ_REMOVE(&sgroup->queued, req, link); 976 if (nvmf_transport_req_free(req)) { 977 SPDK_ERRLOG("Transport request free error!\n"); 978 } 979 } 980 } 981 } 982 983 TAILQ_REMOVE(&qpair->group->qpairs, qpair, link); 984 qpair->group = NULL; 985 } 986 987 static void 988 _nvmf_qpair_destroy(void *ctx, int status) 989 { 990 struct nvmf_qpair_disconnect_ctx *qpair_ctx = ctx; 991 struct spdk_nvmf_qpair *qpair = qpair_ctx->qpair; 992 struct spdk_nvmf_ctrlr *ctrlr = qpair->ctrlr; 993 994 assert(qpair->state == SPDK_NVMF_QPAIR_DEACTIVATING); 995 qpair_ctx->qid = qpair->qid; 996 997 spdk_nvmf_poll_group_remove(qpair); 998 999 if (!ctrlr || !ctrlr->thread) { 1000 nvmf_transport_qpair_fini(qpair, _nvmf_transport_qpair_fini_complete, qpair_ctx); 1001 return; 1002 } 1003 1004 qpair_ctx->ctrlr = ctrlr; 1005 spdk_thread_send_msg(ctrlr->thread, _nvmf_ctrlr_free_from_qpair, qpair_ctx); 1006 } 1007 1008 static void 1009 _nvmf_qpair_disconnect_msg(void *ctx) 1010 { 1011 struct nvmf_qpair_disconnect_ctx *qpair_ctx = ctx; 1012 1013 spdk_nvmf_qpair_disconnect(qpair_ctx->qpair, qpair_ctx->cb_fn, qpair_ctx->ctx); 1014 free(ctx); 1015 } 1016 1017 int 1018 spdk_nvmf_qpair_disconnect(struct spdk_nvmf_qpair *qpair, nvmf_qpair_disconnect_cb cb_fn, void *ctx) 1019 { 1020 struct nvmf_qpair_disconnect_ctx *qpair_ctx; 1021 1022 if (__atomic_test_and_set(&qpair->disconnect_started, __ATOMIC_RELAXED)) { 1023 if (cb_fn) { 1024 cb_fn(ctx); 1025 } 1026 return 0; 1027 } 1028 1029 /* If we get a qpair in the uninitialized state, we can just destroy it immediately */ 1030 if (qpair->state == SPDK_NVMF_QPAIR_UNINITIALIZED) { 1031 nvmf_transport_qpair_fini(qpair, NULL, NULL); 1032 if (cb_fn) { 1033 cb_fn(ctx); 1034 } 1035 return 0; 1036 } 1037 1038 assert(qpair->group != NULL); 1039 if (spdk_get_thread() != qpair->group->thread) { 1040 /* clear the atomic so we can set it on the next call on the proper thread. */ 1041 __atomic_clear(&qpair->disconnect_started, __ATOMIC_RELAXED); 1042 qpair_ctx = calloc(1, sizeof(struct nvmf_qpair_disconnect_ctx)); 1043 if (!qpair_ctx) { 1044 SPDK_ERRLOG("Unable to allocate context for nvmf_qpair_disconnect\n"); 1045 return -ENOMEM; 1046 } 1047 qpair_ctx->qpair = qpair; 1048 qpair_ctx->cb_fn = cb_fn; 1049 qpair_ctx->thread = qpair->group->thread; 1050 qpair_ctx->ctx = ctx; 1051 spdk_thread_send_msg(qpair->group->thread, _nvmf_qpair_disconnect_msg, qpair_ctx); 1052 return 0; 1053 } 1054 1055 assert(qpair->state == SPDK_NVMF_QPAIR_ACTIVE); 1056 nvmf_qpair_set_state(qpair, SPDK_NVMF_QPAIR_DEACTIVATING); 1057 1058 qpair_ctx = calloc(1, sizeof(struct nvmf_qpair_disconnect_ctx)); 1059 if (!qpair_ctx) { 1060 SPDK_ERRLOG("Unable to allocate context for nvmf_qpair_disconnect\n"); 1061 return -ENOMEM; 1062 } 1063 1064 qpair_ctx->qpair = qpair; 1065 qpair_ctx->cb_fn = cb_fn; 1066 qpair_ctx->thread = qpair->group->thread; 1067 qpair_ctx->ctx = ctx; 1068 1069 /* Check for outstanding I/O */ 1070 if (!TAILQ_EMPTY(&qpair->outstanding)) { 1071 qpair->state_cb = _nvmf_qpair_destroy; 1072 qpair->state_cb_arg = qpair_ctx; 1073 nvmf_qpair_free_aer(qpair); 1074 return 0; 1075 } 1076 1077 _nvmf_qpair_destroy(qpair_ctx, 0); 1078 1079 return 0; 1080 } 1081 1082 int 1083 spdk_nvmf_qpair_get_peer_trid(struct spdk_nvmf_qpair *qpair, 1084 struct spdk_nvme_transport_id *trid) 1085 { 1086 return nvmf_transport_qpair_get_peer_trid(qpair, trid); 1087 } 1088 1089 int 1090 spdk_nvmf_qpair_get_local_trid(struct spdk_nvmf_qpair *qpair, 1091 struct spdk_nvme_transport_id *trid) 1092 { 1093 return nvmf_transport_qpair_get_local_trid(qpair, trid); 1094 } 1095 1096 int 1097 spdk_nvmf_qpair_get_listen_trid(struct spdk_nvmf_qpair *qpair, 1098 struct spdk_nvme_transport_id *trid) 1099 { 1100 return nvmf_transport_qpair_get_listen_trid(qpair, trid); 1101 } 1102 1103 int 1104 nvmf_poll_group_add_transport(struct spdk_nvmf_poll_group *group, 1105 struct spdk_nvmf_transport *transport) 1106 { 1107 struct spdk_nvmf_transport_poll_group *tgroup; 1108 1109 TAILQ_FOREACH(tgroup, &group->tgroups, link) { 1110 if (tgroup->transport == transport) { 1111 /* Transport already in the poll group */ 1112 return 0; 1113 } 1114 } 1115 1116 tgroup = nvmf_transport_poll_group_create(transport); 1117 if (!tgroup) { 1118 SPDK_ERRLOG("Unable to create poll group for transport\n"); 1119 return -1; 1120 } 1121 1122 tgroup->group = group; 1123 TAILQ_INSERT_TAIL(&group->tgroups, tgroup, link); 1124 1125 return 0; 1126 } 1127 1128 static int 1129 poll_group_update_subsystem(struct spdk_nvmf_poll_group *group, 1130 struct spdk_nvmf_subsystem *subsystem) 1131 { 1132 struct spdk_nvmf_subsystem_poll_group *sgroup; 1133 uint32_t new_num_ns, old_num_ns; 1134 uint32_t i, j; 1135 struct spdk_nvmf_ns *ns; 1136 struct spdk_nvmf_registrant *reg, *tmp; 1137 struct spdk_io_channel *ch; 1138 struct spdk_nvmf_subsystem_pg_ns_info *ns_info; 1139 struct spdk_nvmf_ctrlr *ctrlr; 1140 bool ns_changed; 1141 1142 /* Make sure our poll group has memory for this subsystem allocated */ 1143 if (subsystem->id >= group->num_sgroups) { 1144 return -ENOMEM; 1145 } 1146 1147 sgroup = &group->sgroups[subsystem->id]; 1148 1149 /* Make sure the array of namespace information is the correct size */ 1150 new_num_ns = subsystem->max_nsid; 1151 old_num_ns = sgroup->num_ns; 1152 1153 ns_changed = false; 1154 1155 if (old_num_ns == 0) { 1156 if (new_num_ns > 0) { 1157 /* First allocation */ 1158 sgroup->ns_info = calloc(new_num_ns, sizeof(struct spdk_nvmf_subsystem_pg_ns_info)); 1159 if (!sgroup->ns_info) { 1160 return -ENOMEM; 1161 } 1162 } 1163 } else if (new_num_ns > old_num_ns) { 1164 void *buf; 1165 1166 /* Make the array larger */ 1167 buf = realloc(sgroup->ns_info, new_num_ns * sizeof(struct spdk_nvmf_subsystem_pg_ns_info)); 1168 if (!buf) { 1169 return -ENOMEM; 1170 } 1171 1172 sgroup->ns_info = buf; 1173 1174 /* Null out the new namespace information slots */ 1175 for (i = old_num_ns; i < new_num_ns; i++) { 1176 memset(&sgroup->ns_info[i], 0, sizeof(struct spdk_nvmf_subsystem_pg_ns_info)); 1177 } 1178 } else if (new_num_ns < old_num_ns) { 1179 void *buf; 1180 1181 /* Free the extra I/O channels */ 1182 for (i = new_num_ns; i < old_num_ns; i++) { 1183 ns_info = &sgroup->ns_info[i]; 1184 1185 if (ns_info->channel) { 1186 spdk_put_io_channel(ns_info->channel); 1187 ns_info->channel = NULL; 1188 } 1189 } 1190 1191 /* Make the array smaller */ 1192 if (new_num_ns > 0) { 1193 buf = realloc(sgroup->ns_info, new_num_ns * sizeof(struct spdk_nvmf_subsystem_pg_ns_info)); 1194 if (!buf) { 1195 return -ENOMEM; 1196 } 1197 sgroup->ns_info = buf; 1198 } else { 1199 free(sgroup->ns_info); 1200 sgroup->ns_info = NULL; 1201 } 1202 } 1203 1204 sgroup->num_ns = new_num_ns; 1205 1206 /* Detect bdevs that were added or removed */ 1207 for (i = 0; i < sgroup->num_ns; i++) { 1208 ns = subsystem->ns[i]; 1209 ns_info = &sgroup->ns_info[i]; 1210 ch = ns_info->channel; 1211 1212 if (ns == NULL && ch == NULL) { 1213 /* Both NULL. Leave empty */ 1214 } else if (ns == NULL && ch != NULL) { 1215 /* There was a channel here, but the namespace is gone. */ 1216 ns_changed = true; 1217 spdk_put_io_channel(ch); 1218 ns_info->channel = NULL; 1219 } else if (ns != NULL && ch == NULL) { 1220 /* A namespace appeared but there is no channel yet */ 1221 ns_changed = true; 1222 ch = spdk_bdev_get_io_channel(ns->desc); 1223 if (ch == NULL) { 1224 SPDK_ERRLOG("Could not allocate I/O channel.\n"); 1225 return -ENOMEM; 1226 } 1227 ns_info->channel = ch; 1228 } else if (spdk_uuid_compare(&ns_info->uuid, spdk_bdev_get_uuid(ns->bdev)) != 0) { 1229 /* A namespace was here before, but was replaced by a new one. */ 1230 ns_changed = true; 1231 spdk_put_io_channel(ns_info->channel); 1232 memset(ns_info, 0, sizeof(*ns_info)); 1233 1234 ch = spdk_bdev_get_io_channel(ns->desc); 1235 if (ch == NULL) { 1236 SPDK_ERRLOG("Could not allocate I/O channel.\n"); 1237 return -ENOMEM; 1238 } 1239 ns_info->channel = ch; 1240 } else if (ns_info->num_blocks != spdk_bdev_get_num_blocks(ns->bdev)) { 1241 /* Namespace is still there but size has changed */ 1242 SPDK_DEBUGLOG(nvmf, "Namespace resized: subsystem_id %u," 1243 " nsid %u, pg %p, old %" PRIu64 ", new %" PRIu64 "\n", 1244 subsystem->id, 1245 ns->nsid, 1246 group, 1247 ns_info->num_blocks, 1248 spdk_bdev_get_num_blocks(ns->bdev)); 1249 ns_changed = true; 1250 } 1251 1252 if (ns == NULL) { 1253 memset(ns_info, 0, sizeof(*ns_info)); 1254 } else { 1255 ns_info->uuid = *spdk_bdev_get_uuid(ns->bdev); 1256 ns_info->num_blocks = spdk_bdev_get_num_blocks(ns->bdev); 1257 ns_info->crkey = ns->crkey; 1258 ns_info->rtype = ns->rtype; 1259 if (ns->holder) { 1260 ns_info->holder_id = ns->holder->hostid; 1261 } 1262 1263 memset(&ns_info->reg_hostid, 0, SPDK_NVMF_MAX_NUM_REGISTRANTS * sizeof(struct spdk_uuid)); 1264 j = 0; 1265 TAILQ_FOREACH_SAFE(reg, &ns->registrants, link, tmp) { 1266 if (j >= SPDK_NVMF_MAX_NUM_REGISTRANTS) { 1267 SPDK_ERRLOG("Maximum %u registrants can support.\n", SPDK_NVMF_MAX_NUM_REGISTRANTS); 1268 return -EINVAL; 1269 } 1270 ns_info->reg_hostid[j++] = reg->hostid; 1271 } 1272 } 1273 } 1274 1275 if (ns_changed) { 1276 TAILQ_FOREACH(ctrlr, &subsystem->ctrlrs, link) { 1277 if (ctrlr->admin_qpair->group == group) { 1278 nvmf_ctrlr_async_event_ns_notice(ctrlr); 1279 nvmf_ctrlr_async_event_ana_change_notice(ctrlr); 1280 } 1281 } 1282 } 1283 1284 return 0; 1285 } 1286 1287 int 1288 nvmf_poll_group_update_subsystem(struct spdk_nvmf_poll_group *group, 1289 struct spdk_nvmf_subsystem *subsystem) 1290 { 1291 return poll_group_update_subsystem(group, subsystem); 1292 } 1293 1294 int 1295 nvmf_poll_group_add_subsystem(struct spdk_nvmf_poll_group *group, 1296 struct spdk_nvmf_subsystem *subsystem, 1297 spdk_nvmf_poll_group_mod_done cb_fn, void *cb_arg) 1298 { 1299 int rc = 0; 1300 struct spdk_nvmf_subsystem_poll_group *sgroup = &group->sgroups[subsystem->id]; 1301 uint32_t i; 1302 1303 TAILQ_INIT(&sgroup->queued); 1304 1305 rc = poll_group_update_subsystem(group, subsystem); 1306 if (rc) { 1307 nvmf_poll_group_remove_subsystem(group, subsystem, NULL, NULL); 1308 goto fini; 1309 } 1310 1311 sgroup->state = SPDK_NVMF_SUBSYSTEM_ACTIVE; 1312 1313 for (i = 0; i < sgroup->num_ns; i++) { 1314 sgroup->ns_info[i].state = SPDK_NVMF_SUBSYSTEM_ACTIVE; 1315 } 1316 1317 fini: 1318 if (cb_fn) { 1319 cb_fn(cb_arg, rc); 1320 } 1321 1322 return rc; 1323 } 1324 1325 static void 1326 _nvmf_poll_group_remove_subsystem_cb(void *ctx, int status) 1327 { 1328 struct nvmf_qpair_disconnect_many_ctx *qpair_ctx = ctx; 1329 struct spdk_nvmf_subsystem *subsystem; 1330 struct spdk_nvmf_poll_group *group; 1331 struct spdk_nvmf_subsystem_poll_group *sgroup; 1332 spdk_nvmf_poll_group_mod_done cpl_fn = NULL; 1333 void *cpl_ctx = NULL; 1334 uint32_t nsid; 1335 1336 group = qpair_ctx->group; 1337 subsystem = qpair_ctx->subsystem; 1338 cpl_fn = qpair_ctx->cpl_fn; 1339 cpl_ctx = qpair_ctx->cpl_ctx; 1340 sgroup = &group->sgroups[subsystem->id]; 1341 1342 if (status) { 1343 goto fini; 1344 } 1345 1346 for (nsid = 0; nsid < sgroup->num_ns; nsid++) { 1347 if (sgroup->ns_info[nsid].channel) { 1348 spdk_put_io_channel(sgroup->ns_info[nsid].channel); 1349 sgroup->ns_info[nsid].channel = NULL; 1350 } 1351 } 1352 1353 sgroup->num_ns = 0; 1354 free(sgroup->ns_info); 1355 sgroup->ns_info = NULL; 1356 fini: 1357 free(qpair_ctx); 1358 if (cpl_fn) { 1359 cpl_fn(cpl_ctx, status); 1360 } 1361 } 1362 1363 static void 1364 nvmf_poll_group_remove_subsystem_msg(void *ctx) 1365 { 1366 struct spdk_nvmf_qpair *qpair, *qpair_tmp; 1367 struct spdk_nvmf_subsystem *subsystem; 1368 struct spdk_nvmf_poll_group *group; 1369 struct nvmf_qpair_disconnect_many_ctx *qpair_ctx = ctx; 1370 int rc = 0; 1371 bool have_qpairs = false; 1372 1373 group = qpair_ctx->group; 1374 subsystem = qpair_ctx->subsystem; 1375 1376 TAILQ_FOREACH_SAFE(qpair, &group->qpairs, link, qpair_tmp) { 1377 if ((qpair->ctrlr != NULL) && (qpair->ctrlr->subsys == subsystem)) { 1378 /* Use another variable to check if there were any qpairs disconnected in this call since 1379 * we can loop over all qpairs and iterator will be NULL in the end */ 1380 have_qpairs = true; 1381 rc = spdk_nvmf_qpair_disconnect(qpair, NULL, NULL); 1382 if (rc) { 1383 break; 1384 } 1385 } 1386 } 1387 1388 if (!have_qpairs || rc) { 1389 _nvmf_poll_group_remove_subsystem_cb(ctx, rc); 1390 return; 1391 } 1392 1393 spdk_thread_send_msg(spdk_get_thread(), nvmf_poll_group_remove_subsystem_msg, ctx); 1394 } 1395 1396 void 1397 nvmf_poll_group_remove_subsystem(struct spdk_nvmf_poll_group *group, 1398 struct spdk_nvmf_subsystem *subsystem, 1399 spdk_nvmf_poll_group_mod_done cb_fn, void *cb_arg) 1400 { 1401 struct spdk_nvmf_subsystem_poll_group *sgroup; 1402 struct nvmf_qpair_disconnect_many_ctx *ctx; 1403 uint32_t i; 1404 1405 ctx = calloc(1, sizeof(struct nvmf_qpair_disconnect_many_ctx)); 1406 if (!ctx) { 1407 SPDK_ERRLOG("Unable to allocate memory for context to remove poll subsystem\n"); 1408 if (cb_fn) { 1409 cb_fn(cb_arg, -1); 1410 } 1411 return; 1412 } 1413 1414 ctx->group = group; 1415 ctx->subsystem = subsystem; 1416 ctx->cpl_fn = cb_fn; 1417 ctx->cpl_ctx = cb_arg; 1418 1419 sgroup = &group->sgroups[subsystem->id]; 1420 sgroup->state = SPDK_NVMF_SUBSYSTEM_INACTIVE; 1421 1422 for (i = 0; i < sgroup->num_ns; i++) { 1423 sgroup->ns_info[i].state = SPDK_NVMF_SUBSYSTEM_INACTIVE; 1424 } 1425 1426 nvmf_poll_group_remove_subsystem_msg(ctx); 1427 } 1428 1429 void 1430 nvmf_poll_group_pause_subsystem(struct spdk_nvmf_poll_group *group, 1431 struct spdk_nvmf_subsystem *subsystem, 1432 uint32_t nsid, 1433 spdk_nvmf_poll_group_mod_done cb_fn, void *cb_arg) 1434 { 1435 struct spdk_nvmf_subsystem_poll_group *sgroup; 1436 struct spdk_nvmf_subsystem_pg_ns_info *ns_info = NULL; 1437 int rc = 0; 1438 1439 if (subsystem->id >= group->num_sgroups) { 1440 rc = -1; 1441 goto fini; 1442 } 1443 1444 sgroup = &group->sgroups[subsystem->id]; 1445 if (sgroup == NULL) { 1446 rc = -1; 1447 goto fini; 1448 } 1449 1450 if (sgroup->state == SPDK_NVMF_SUBSYSTEM_PAUSED) { 1451 goto fini; 1452 } 1453 sgroup->state = SPDK_NVMF_SUBSYSTEM_PAUSING; 1454 1455 /* NOTE: This implicitly also checks for 0, since 0 - 1 wraps around to UINT32_MAX. */ 1456 if (nsid - 1 < sgroup->num_ns) { 1457 ns_info = &sgroup->ns_info[nsid - 1]; 1458 ns_info->state = SPDK_NVMF_SUBSYSTEM_PAUSING; 1459 } 1460 1461 if (sgroup->mgmt_io_outstanding > 0) { 1462 assert(sgroup->cb_fn == NULL); 1463 sgroup->cb_fn = cb_fn; 1464 assert(sgroup->cb_arg == NULL); 1465 sgroup->cb_arg = cb_arg; 1466 return; 1467 } 1468 1469 if (ns_info != NULL && ns_info->io_outstanding > 0) { 1470 assert(sgroup->cb_fn == NULL); 1471 sgroup->cb_fn = cb_fn; 1472 assert(sgroup->cb_arg == NULL); 1473 sgroup->cb_arg = cb_arg; 1474 return; 1475 } 1476 1477 assert(sgroup->mgmt_io_outstanding == 0); 1478 sgroup->state = SPDK_NVMF_SUBSYSTEM_PAUSED; 1479 fini: 1480 if (cb_fn) { 1481 cb_fn(cb_arg, rc); 1482 } 1483 } 1484 1485 void 1486 nvmf_poll_group_resume_subsystem(struct spdk_nvmf_poll_group *group, 1487 struct spdk_nvmf_subsystem *subsystem, 1488 spdk_nvmf_poll_group_mod_done cb_fn, void *cb_arg) 1489 { 1490 struct spdk_nvmf_request *req, *tmp; 1491 struct spdk_nvmf_subsystem_poll_group *sgroup; 1492 int rc = 0; 1493 uint32_t i; 1494 1495 if (subsystem->id >= group->num_sgroups) { 1496 rc = -1; 1497 goto fini; 1498 } 1499 1500 sgroup = &group->sgroups[subsystem->id]; 1501 1502 if (sgroup->state == SPDK_NVMF_SUBSYSTEM_ACTIVE) { 1503 goto fini; 1504 } 1505 1506 rc = poll_group_update_subsystem(group, subsystem); 1507 if (rc) { 1508 goto fini; 1509 } 1510 1511 for (i = 0; i < sgroup->num_ns; i++) { 1512 sgroup->ns_info[i].state = SPDK_NVMF_SUBSYSTEM_ACTIVE; 1513 } 1514 1515 sgroup->state = SPDK_NVMF_SUBSYSTEM_ACTIVE; 1516 1517 /* Release all queued requests */ 1518 TAILQ_FOREACH_SAFE(req, &sgroup->queued, link, tmp) { 1519 TAILQ_REMOVE(&sgroup->queued, req, link); 1520 spdk_nvmf_request_exec(req); 1521 } 1522 fini: 1523 if (cb_fn) { 1524 cb_fn(cb_arg, rc); 1525 } 1526 } 1527 1528 1529 struct spdk_nvmf_poll_group * 1530 spdk_nvmf_get_optimal_poll_group(struct spdk_nvmf_qpair *qpair) 1531 { 1532 struct spdk_nvmf_transport_poll_group *tgroup; 1533 1534 tgroup = nvmf_transport_get_optimal_poll_group(qpair->transport, qpair); 1535 1536 if (tgroup == NULL) { 1537 return NULL; 1538 } 1539 1540 return tgroup->group; 1541 } 1542 1543 int 1544 spdk_nvmf_poll_group_get_stat(struct spdk_nvmf_tgt *tgt, 1545 struct spdk_nvmf_poll_group_stat *stat) 1546 { 1547 struct spdk_io_channel *ch; 1548 struct spdk_nvmf_poll_group *group; 1549 1550 SPDK_ERRLOG("spdk_nvmf_poll_group_get_stat is deprecated and will be removed\n"); 1551 1552 if (tgt == NULL || stat == NULL) { 1553 return -EINVAL; 1554 } 1555 1556 ch = spdk_get_io_channel(tgt); 1557 group = spdk_io_channel_get_ctx(ch); 1558 *stat = group->stat; 1559 spdk_put_io_channel(ch); 1560 return 0; 1561 } 1562 1563 void 1564 spdk_nvmf_poll_group_dump_stat(struct spdk_nvmf_poll_group *group, struct spdk_json_write_ctx *w) 1565 { 1566 struct spdk_nvmf_transport_poll_group *tgroup; 1567 1568 spdk_json_write_object_begin(w); 1569 1570 spdk_json_write_named_string(w, "name", spdk_thread_get_name(spdk_get_thread())); 1571 spdk_json_write_named_uint32(w, "admin_qpairs", group->stat.admin_qpairs); 1572 spdk_json_write_named_uint32(w, "io_qpairs", group->stat.io_qpairs); 1573 spdk_json_write_named_uint64(w, "pending_bdev_io", group->stat.pending_bdev_io); 1574 1575 spdk_json_write_named_array_begin(w, "transports"); 1576 1577 TAILQ_FOREACH(tgroup, &group->tgroups, link) { 1578 spdk_json_write_object_begin(w); 1579 /* 1580 * The trtype field intentionally contains a transport name as this is more informative. 1581 * The field has not been renamed for backward compatibility. 1582 */ 1583 spdk_json_write_named_string(w, "trtype", spdk_nvmf_get_transport_name(tgroup->transport)); 1584 1585 if (tgroup->transport->ops->poll_group_dump_stat) { 1586 tgroup->transport->ops->poll_group_dump_stat(tgroup, w); 1587 } 1588 1589 spdk_json_write_object_end(w); 1590 } 1591 1592 spdk_json_write_array_end(w); 1593 spdk_json_write_object_end(w); 1594 } 1595