1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) Intel Corporation. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include "spdk/stdinc.h" 35 36 #include "spdk/bdev.h" 37 #include "spdk/bit_array.h" 38 #include "spdk/conf.h" 39 #include "spdk/thread.h" 40 #include "spdk/nvmf.h" 41 #include "spdk/trace.h" 42 #include "spdk/endian.h" 43 #include "spdk/string.h" 44 45 #include "spdk_internal/log.h" 46 47 #include "nvmf_internal.h" 48 #include "transport.h" 49 50 SPDK_LOG_REGISTER_COMPONENT("nvmf", SPDK_LOG_NVMF) 51 52 #define SPDK_NVMF_DEFAULT_MAX_QUEUE_DEPTH 128 53 #define SPDK_NVMF_DEFAULT_MAX_QPAIRS_PER_CTRLR 64 54 #define SPDK_NVMF_DEFAULT_IN_CAPSULE_DATA_SIZE 4096 55 #define SPDK_NVMF_DEFAULT_MAX_IO_SIZE 131072 56 #define SPDK_NVMF_DEFAULT_MAX_SUBSYSTEMS 1024 57 #define SPDK_NVMF_DEFAULT_IO_UNIT_SIZE 131072 58 59 typedef void (*nvmf_qpair_disconnect_cpl)(void *ctx, int status); 60 61 /* supplied to a single call to nvmf_qpair_disconnect */ 62 struct nvmf_qpair_disconnect_ctx { 63 struct spdk_nvmf_qpair *qpair; 64 struct spdk_nvmf_ctrlr *ctrlr; 65 nvmf_qpair_disconnect_cb cb_fn; 66 struct spdk_thread *thread; 67 void *ctx; 68 uint16_t qid; 69 }; 70 71 /* 72 * There are several times when we need to iterate through the list of all qpairs and selectively delete them. 73 * In order to do this sequentially without overlap, we must provide a context to recover the next qpair from 74 * to enable calling nvmf_qpair_disconnect on the next desired qpair. 75 */ 76 struct nvmf_qpair_disconnect_many_ctx { 77 struct spdk_nvmf_subsystem *subsystem; 78 struct spdk_nvmf_poll_group *group; 79 spdk_nvmf_poll_group_mod_done cpl_fn; 80 void *cpl_ctx; 81 }; 82 83 static void 84 spdk_nvmf_qpair_set_state(struct spdk_nvmf_qpair *qpair, 85 enum spdk_nvmf_qpair_state state) 86 { 87 assert(qpair != NULL); 88 assert(qpair->group->thread == spdk_get_thread()); 89 90 qpair->state = state; 91 } 92 93 void 94 spdk_nvmf_tgt_opts_init(struct spdk_nvmf_tgt_opts *opts) 95 { 96 opts->max_queue_depth = SPDK_NVMF_DEFAULT_MAX_QUEUE_DEPTH; 97 opts->max_qpairs_per_ctrlr = SPDK_NVMF_DEFAULT_MAX_QPAIRS_PER_CTRLR; 98 opts->in_capsule_data_size = SPDK_NVMF_DEFAULT_IN_CAPSULE_DATA_SIZE; 99 opts->max_io_size = SPDK_NVMF_DEFAULT_MAX_IO_SIZE; 100 opts->max_subsystems = SPDK_NVMF_DEFAULT_MAX_SUBSYSTEMS; 101 opts->io_unit_size = SPDK_NVMF_DEFAULT_IO_UNIT_SIZE; 102 } 103 104 static int 105 spdk_nvmf_poll_group_poll(void *ctx) 106 { 107 struct spdk_nvmf_poll_group *group = ctx; 108 int rc; 109 int count = 0; 110 struct spdk_nvmf_transport_poll_group *tgroup; 111 112 TAILQ_FOREACH(tgroup, &group->tgroups, link) { 113 rc = spdk_nvmf_transport_poll_group_poll(tgroup); 114 if (rc < 0) { 115 return -1; 116 } 117 count += rc; 118 } 119 120 return count; 121 } 122 123 static int 124 spdk_nvmf_tgt_create_poll_group(void *io_device, void *ctx_buf) 125 { 126 struct spdk_nvmf_tgt *tgt = io_device; 127 struct spdk_nvmf_poll_group *group = ctx_buf; 128 struct spdk_nvmf_transport *transport; 129 uint32_t sid; 130 131 TAILQ_INIT(&group->tgroups); 132 TAILQ_INIT(&group->qpairs); 133 134 TAILQ_FOREACH(transport, &tgt->transports, link) { 135 spdk_nvmf_poll_group_add_transport(group, transport); 136 } 137 138 group->num_sgroups = tgt->opts.max_subsystems; 139 group->sgroups = calloc(tgt->opts.max_subsystems, sizeof(struct spdk_nvmf_subsystem_poll_group)); 140 if (!group->sgroups) { 141 return -1; 142 } 143 144 for (sid = 0; sid < tgt->opts.max_subsystems; sid++) { 145 struct spdk_nvmf_subsystem *subsystem; 146 147 subsystem = tgt->subsystems[sid]; 148 if (!subsystem) { 149 continue; 150 } 151 152 spdk_nvmf_poll_group_add_subsystem(group, subsystem, NULL, NULL); 153 } 154 155 group->poller = spdk_poller_register(spdk_nvmf_poll_group_poll, group, 0); 156 group->thread = spdk_get_thread(); 157 158 return 0; 159 } 160 161 static void 162 spdk_nvmf_tgt_destroy_poll_group(void *io_device, void *ctx_buf) 163 { 164 struct spdk_nvmf_poll_group *group = ctx_buf; 165 struct spdk_nvmf_transport_poll_group *tgroup, *tmp; 166 struct spdk_nvmf_subsystem_poll_group *sgroup; 167 uint32_t sid, nsid; 168 169 TAILQ_FOREACH_SAFE(tgroup, &group->tgroups, link, tmp) { 170 TAILQ_REMOVE(&group->tgroups, tgroup, link); 171 spdk_nvmf_transport_poll_group_destroy(tgroup); 172 } 173 174 for (sid = 0; sid < group->num_sgroups; sid++) { 175 sgroup = &group->sgroups[sid]; 176 177 for (nsid = 0; nsid < sgroup->num_channels; nsid++) { 178 if (sgroup->channels[nsid]) { 179 spdk_put_io_channel(sgroup->channels[nsid]); 180 sgroup->channels[nsid] = NULL; 181 } 182 } 183 184 free(sgroup->channels); 185 } 186 187 free(group->sgroups); 188 } 189 190 static void 191 _nvmf_tgt_disconnect_next_qpair(void *ctx) 192 { 193 struct spdk_nvmf_qpair *qpair; 194 struct nvmf_qpair_disconnect_many_ctx *qpair_ctx = ctx; 195 struct spdk_nvmf_poll_group *group = qpair_ctx->group; 196 struct spdk_io_channel *ch; 197 int rc = 0; 198 199 qpair = TAILQ_FIRST(&group->qpairs); 200 201 if (qpair) { 202 rc = spdk_nvmf_qpair_disconnect(qpair, _nvmf_tgt_disconnect_next_qpair, ctx); 203 } 204 205 if (!qpair || rc != 0) { 206 /* When the refcount from the channels reaches 0, spdk_nvmf_tgt_destroy_poll_group will be called. */ 207 ch = spdk_io_channel_from_ctx(group); 208 spdk_put_io_channel(ch); 209 free(qpair_ctx); 210 } 211 } 212 213 static void 214 spdk_nvmf_tgt_destroy_poll_group_qpairs(struct spdk_nvmf_poll_group *group) 215 { 216 struct nvmf_qpair_disconnect_many_ctx *ctx; 217 218 ctx = calloc(1, sizeof(struct nvmf_qpair_disconnect_many_ctx)); 219 220 if (!ctx) { 221 SPDK_ERRLOG("Failed to allocate memory for destroy poll group ctx\n"); 222 return; 223 } 224 225 spdk_poller_unregister(&group->poller); 226 227 ctx->group = group; 228 _nvmf_tgt_disconnect_next_qpair(ctx); 229 } 230 231 struct spdk_nvmf_tgt * 232 spdk_nvmf_tgt_create(struct spdk_nvmf_tgt_opts *opts) 233 { 234 struct spdk_nvmf_tgt *tgt; 235 236 tgt = calloc(1, sizeof(*tgt)); 237 if (!tgt) { 238 return NULL; 239 } 240 241 if (!opts) { 242 spdk_nvmf_tgt_opts_init(&tgt->opts); 243 } else { 244 tgt->opts = *opts; 245 } 246 247 tgt->discovery_genctr = 0; 248 tgt->discovery_log_page = NULL; 249 tgt->discovery_log_page_size = 0; 250 TAILQ_INIT(&tgt->transports); 251 252 tgt->subsystems = calloc(tgt->opts.max_subsystems, sizeof(struct spdk_nvmf_subsystem *)); 253 if (!tgt->subsystems) { 254 free(tgt); 255 return NULL; 256 } 257 258 spdk_io_device_register(tgt, 259 spdk_nvmf_tgt_create_poll_group, 260 spdk_nvmf_tgt_destroy_poll_group, 261 sizeof(struct spdk_nvmf_poll_group), 262 "nvmf_tgt"); 263 264 return tgt; 265 } 266 267 static void 268 spdk_nvmf_tgt_destroy_cb(void *io_device) 269 { 270 struct spdk_nvmf_tgt *tgt = io_device; 271 struct spdk_nvmf_transport *transport, *transport_tmp; 272 spdk_nvmf_tgt_destroy_done_fn *destroy_cb_fn; 273 void *destroy_cb_arg; 274 uint32_t i; 275 276 if (tgt->discovery_log_page) { 277 free(tgt->discovery_log_page); 278 } 279 280 if (tgt->subsystems) { 281 for (i = 0; i < tgt->opts.max_subsystems; i++) { 282 if (tgt->subsystems[i]) { 283 spdk_nvmf_subsystem_destroy(tgt->subsystems[i]); 284 } 285 } 286 free(tgt->subsystems); 287 } 288 289 TAILQ_FOREACH_SAFE(transport, &tgt->transports, link, transport_tmp) { 290 TAILQ_REMOVE(&tgt->transports, transport, link); 291 spdk_nvmf_transport_destroy(transport); 292 } 293 294 destroy_cb_fn = tgt->destroy_cb_fn; 295 destroy_cb_arg = tgt->destroy_cb_arg; 296 297 free(tgt); 298 299 if (destroy_cb_fn) { 300 destroy_cb_fn(destroy_cb_arg, 0); 301 } 302 } 303 304 void 305 spdk_nvmf_tgt_destroy(struct spdk_nvmf_tgt *tgt, 306 spdk_nvmf_tgt_destroy_done_fn cb_fn, 307 void *cb_arg) 308 { 309 tgt->destroy_cb_fn = cb_fn; 310 tgt->destroy_cb_arg = cb_arg; 311 312 spdk_io_device_unregister(tgt, spdk_nvmf_tgt_destroy_cb); 313 } 314 315 static void 316 spdk_nvmf_write_subsystem_config_json(struct spdk_json_write_ctx *w, 317 struct spdk_nvmf_subsystem *subsystem) 318 { 319 struct spdk_nvmf_host *host; 320 struct spdk_nvmf_listener *listener; 321 const struct spdk_nvme_transport_id *trid; 322 struct spdk_nvmf_ns *ns; 323 struct spdk_nvmf_ns_opts ns_opts; 324 uint32_t max_namespaces; 325 char uuid_str[SPDK_UUID_STRING_LEN]; 326 const char *trtype; 327 const char *adrfam; 328 329 if (spdk_nvmf_subsystem_get_type(subsystem) != SPDK_NVMF_SUBTYPE_NVME) { 330 return; 331 } 332 333 /* { */ 334 spdk_json_write_object_begin(w); 335 spdk_json_write_named_string(w, "method", "nvmf_subsystem_create"); 336 337 /* "params" : { */ 338 spdk_json_write_named_object_begin(w, "params"); 339 spdk_json_write_named_string(w, "nqn", spdk_nvmf_subsystem_get_nqn(subsystem)); 340 spdk_json_write_named_bool(w, "allow_any_host", spdk_nvmf_subsystem_get_allow_any_host(subsystem)); 341 spdk_json_write_named_string(w, "serial_number", spdk_nvmf_subsystem_get_sn(subsystem)); 342 343 max_namespaces = spdk_nvmf_subsystem_get_max_namespaces(subsystem); 344 if (max_namespaces != 0) { 345 spdk_json_write_named_uint32(w, "max_namespaces", max_namespaces); 346 } 347 348 /* } "params" */ 349 spdk_json_write_object_end(w); 350 351 /* } */ 352 spdk_json_write_object_end(w); 353 354 for (listener = spdk_nvmf_subsystem_get_first_listener(subsystem); listener != NULL; 355 listener = spdk_nvmf_subsystem_get_next_listener(subsystem, listener)) { 356 trid = spdk_nvmf_listener_get_trid(listener); 357 358 trtype = spdk_nvme_transport_id_trtype_str(trid->trtype); 359 adrfam = spdk_nvme_transport_id_adrfam_str(trid->adrfam); 360 361 spdk_json_write_object_begin(w); 362 spdk_json_write_named_string(w, "method", "nvmf_subsystem_add_listener"); 363 364 /* "params" : { */ 365 spdk_json_write_named_object_begin(w, "params"); 366 367 spdk_json_write_named_string(w, "nqn", spdk_nvmf_subsystem_get_nqn(subsystem)); 368 369 /* "listen_address" : { */ 370 spdk_json_write_named_object_begin(w, "listen_address"); 371 372 spdk_json_write_named_string(w, "trtype", trtype); 373 if (adrfam) { 374 spdk_json_write_named_string(w, "adrfam", adrfam); 375 } 376 377 spdk_json_write_named_string(w, "traddr", trid->traddr); 378 spdk_json_write_named_string(w, "trsvcid", trid->trsvcid); 379 /* } "listen_address" */ 380 spdk_json_write_object_end(w); 381 382 /* } "params" */ 383 spdk_json_write_object_end(w); 384 385 /* } */ 386 spdk_json_write_object_end(w); 387 } 388 389 for (host = spdk_nvmf_subsystem_get_first_host(subsystem); host != NULL; 390 host = spdk_nvmf_subsystem_get_next_host(subsystem, host)) { 391 392 spdk_json_write_object_begin(w); 393 spdk_json_write_named_string(w, "method", "nvmf_subsystem_add_host"); 394 395 /* "params" : { */ 396 spdk_json_write_named_object_begin(w, "params"); 397 398 spdk_json_write_named_string(w, "nqn", spdk_nvmf_subsystem_get_nqn(subsystem)); 399 spdk_json_write_named_string(w, "host", spdk_nvmf_host_get_nqn(host)); 400 401 /* } "params" */ 402 spdk_json_write_object_end(w); 403 404 /* } */ 405 spdk_json_write_object_end(w); 406 } 407 408 for (ns = spdk_nvmf_subsystem_get_first_ns(subsystem); ns != NULL; 409 ns = spdk_nvmf_subsystem_get_next_ns(subsystem, ns)) { 410 spdk_nvmf_ns_get_opts(ns, &ns_opts, sizeof(ns_opts)); 411 412 spdk_json_write_object_begin(w); 413 spdk_json_write_named_string(w, "method", "nvmf_subsystem_add_ns"); 414 415 /* "params" : { */ 416 spdk_json_write_named_object_begin(w, "params"); 417 418 spdk_json_write_named_string(w, "nqn", spdk_nvmf_subsystem_get_nqn(subsystem)); 419 420 /* "namespace" : { */ 421 spdk_json_write_named_object_begin(w, "namespace"); 422 423 spdk_json_write_named_uint32(w, "nsid", spdk_nvmf_ns_get_id(ns)); 424 spdk_json_write_named_string(w, "bdev_name", spdk_bdev_get_name(spdk_nvmf_ns_get_bdev(ns))); 425 426 if (!spdk_mem_all_zero(ns_opts.nguid, sizeof(ns_opts.nguid))) { 427 SPDK_STATIC_ASSERT(sizeof(ns_opts.nguid) == sizeof(uint64_t) * 2, "size mismatch"); 428 spdk_json_write_named_string_fmt(w, "nguid", "%016"PRIX64"%016"PRIX64, from_be64(&ns_opts.nguid[0]), 429 from_be64(&ns_opts.nguid[8])); 430 } 431 432 if (!spdk_mem_all_zero(ns_opts.eui64, sizeof(ns_opts.eui64))) { 433 SPDK_STATIC_ASSERT(sizeof(ns_opts.eui64) == sizeof(uint64_t), "size mismatch"); 434 spdk_json_write_named_string_fmt(w, "eui64", "%016"PRIX64, from_be64(&ns_opts.eui64)); 435 } 436 437 if (!spdk_mem_all_zero(&ns_opts.uuid, sizeof(ns_opts.uuid))) { 438 spdk_uuid_fmt_lower(uuid_str, sizeof(uuid_str), &ns_opts.uuid); 439 spdk_json_write_named_string(w, "uuid", uuid_str); 440 } 441 442 /* "namespace" */ 443 spdk_json_write_object_end(w); 444 445 /* } "params" */ 446 spdk_json_write_object_end(w); 447 448 /* } */ 449 spdk_json_write_object_end(w); 450 } 451 } 452 453 void 454 spdk_nvmf_tgt_write_config_json(struct spdk_json_write_ctx *w, struct spdk_nvmf_tgt *tgt) 455 { 456 struct spdk_nvmf_subsystem *subsystem; 457 struct spdk_nvmf_transport *transport; 458 459 spdk_json_write_object_begin(w); 460 spdk_json_write_named_string(w, "method", "set_nvmf_target_options"); 461 462 spdk_json_write_named_object_begin(w, "params"); 463 spdk_json_write_named_uint32(w, "max_queue_depth", tgt->opts.max_queue_depth); 464 spdk_json_write_named_uint32(w, "max_qpairs_per_ctrlr", tgt->opts.max_qpairs_per_ctrlr); 465 spdk_json_write_named_uint32(w, "in_capsule_data_size", tgt->opts.in_capsule_data_size); 466 spdk_json_write_named_uint32(w, "max_io_size", tgt->opts.max_io_size); 467 spdk_json_write_named_uint32(w, "max_subsystems", tgt->opts.max_subsystems); 468 spdk_json_write_named_uint32(w, "io_unit_size", tgt->opts.io_unit_size); 469 spdk_json_write_object_end(w); 470 471 spdk_json_write_object_end(w); 472 473 /* write transports */ 474 TAILQ_FOREACH(transport, &tgt->transports, link) { 475 spdk_json_write_object_begin(w); 476 spdk_json_write_named_string(w, "method", "nvmf_create_transport"); 477 478 spdk_json_write_named_object_begin(w, "params"); 479 spdk_json_write_named_string(w, "trtype", spdk_nvme_transport_id_trtype_str(transport->ops->type)); 480 spdk_json_write_named_uint32(w, "max_queue_depth", transport->opts.max_queue_depth); 481 spdk_json_write_named_uint32(w, "max_qpairs_per_ctrlr", transport->opts.max_qpairs_per_ctrlr); 482 spdk_json_write_named_uint32(w, "in_capsule_data_size", transport->opts.in_capsule_data_size); 483 spdk_json_write_named_uint32(w, "max_io_size", transport->opts.max_io_size); 484 spdk_json_write_named_uint32(w, "io_unit_size", transport->opts.io_unit_size); 485 spdk_json_write_named_uint32(w, "max_aq_depth", transport->opts.max_aq_depth); 486 spdk_json_write_object_end(w); 487 488 spdk_json_write_object_end(w); 489 } 490 491 subsystem = spdk_nvmf_subsystem_get_first(tgt); 492 while (subsystem) { 493 spdk_nvmf_write_subsystem_config_json(w, subsystem); 494 subsystem = spdk_nvmf_subsystem_get_next(subsystem); 495 } 496 } 497 498 void 499 spdk_nvmf_tgt_listen(struct spdk_nvmf_tgt *tgt, 500 struct spdk_nvme_transport_id *trid, 501 spdk_nvmf_tgt_listen_done_fn cb_fn, 502 void *cb_arg) 503 { 504 struct spdk_nvmf_transport *transport; 505 int rc; 506 bool propagate = false; 507 508 transport = spdk_nvmf_tgt_get_transport(tgt, trid->trtype); 509 if (!transport) { 510 struct spdk_nvmf_transport_opts opts; 511 512 opts.max_queue_depth = tgt->opts.max_queue_depth; 513 opts.max_qpairs_per_ctrlr = tgt->opts.max_qpairs_per_ctrlr; 514 opts.in_capsule_data_size = tgt->opts.in_capsule_data_size; 515 opts.max_io_size = tgt->opts.max_io_size; 516 opts.io_unit_size = tgt->opts.io_unit_size; 517 /* use max_queue depth since tgt. opts. doesn't have max_aq_depth */ 518 opts.max_aq_depth = tgt->opts.max_queue_depth; 519 520 transport = spdk_nvmf_transport_create(trid->trtype, &opts); 521 if (!transport) { 522 SPDK_ERRLOG("Transport initialization failed\n"); 523 cb_fn(cb_arg, -EINVAL); 524 return; 525 } 526 527 propagate = true; 528 } 529 530 rc = spdk_nvmf_transport_listen(transport, trid); 531 if (rc < 0) { 532 SPDK_ERRLOG("Unable to listen on address '%s'\n", trid->traddr); 533 cb_fn(cb_arg, rc); 534 return; 535 } 536 537 tgt->discovery_genctr++; 538 539 if (propagate) { 540 spdk_nvmf_tgt_add_transport(tgt, transport, cb_fn, cb_arg); 541 } else { 542 cb_fn(cb_arg, 0); 543 } 544 } 545 546 struct spdk_nvmf_tgt_add_transport_ctx { 547 struct spdk_nvmf_tgt *tgt; 548 struct spdk_nvmf_transport *transport; 549 spdk_nvmf_tgt_add_transport_done_fn cb_fn; 550 void *cb_arg; 551 }; 552 553 static void 554 _spdk_nvmf_tgt_add_transport_done(struct spdk_io_channel_iter *i, int status) 555 { 556 struct spdk_nvmf_tgt_add_transport_ctx *ctx = spdk_io_channel_iter_get_ctx(i); 557 558 ctx->cb_fn(ctx->cb_arg, status); 559 560 free(ctx); 561 } 562 563 static void 564 _spdk_nvmf_tgt_add_transport(struct spdk_io_channel_iter *i) 565 { 566 struct spdk_nvmf_tgt_add_transport_ctx *ctx = spdk_io_channel_iter_get_ctx(i); 567 struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i); 568 struct spdk_nvmf_poll_group *group = spdk_io_channel_get_ctx(ch); 569 int rc; 570 571 rc = spdk_nvmf_poll_group_add_transport(group, ctx->transport); 572 spdk_for_each_channel_continue(i, rc); 573 } 574 575 void spdk_nvmf_tgt_add_transport(struct spdk_nvmf_tgt *tgt, 576 struct spdk_nvmf_transport *transport, 577 spdk_nvmf_tgt_add_transport_done_fn cb_fn, 578 void *cb_arg) 579 { 580 struct spdk_nvmf_tgt_add_transport_ctx *ctx; 581 582 if (spdk_nvmf_tgt_get_transport(tgt, transport->ops->type)) { 583 cb_fn(cb_arg, -EEXIST); 584 return; /* transport already created */ 585 } 586 587 transport->tgt = tgt; 588 TAILQ_INSERT_TAIL(&tgt->transports, transport, link); 589 590 ctx = calloc(1, sizeof(*ctx)); 591 if (!ctx) { 592 cb_fn(cb_arg, -ENOMEM); 593 return; 594 } 595 596 ctx->tgt = tgt; 597 ctx->transport = transport; 598 ctx->cb_fn = cb_fn; 599 ctx->cb_arg = cb_arg; 600 601 spdk_for_each_channel(tgt, 602 _spdk_nvmf_tgt_add_transport, 603 ctx, 604 _spdk_nvmf_tgt_add_transport_done); 605 } 606 607 struct spdk_nvmf_subsystem * 608 spdk_nvmf_tgt_find_subsystem(struct spdk_nvmf_tgt *tgt, const char *subnqn) 609 { 610 struct spdk_nvmf_subsystem *subsystem; 611 uint32_t sid; 612 613 if (!subnqn) { 614 return NULL; 615 } 616 617 for (sid = 0; sid < tgt->opts.max_subsystems; sid++) { 618 subsystem = tgt->subsystems[sid]; 619 if (subsystem == NULL) { 620 continue; 621 } 622 623 if (strcmp(subnqn, subsystem->subnqn) == 0) { 624 return subsystem; 625 } 626 } 627 628 return NULL; 629 } 630 631 struct spdk_nvmf_transport * 632 spdk_nvmf_tgt_get_transport(struct spdk_nvmf_tgt *tgt, enum spdk_nvme_transport_type type) 633 { 634 struct spdk_nvmf_transport *transport; 635 636 TAILQ_FOREACH(transport, &tgt->transports, link) { 637 if (transport->ops->type == type) { 638 return transport; 639 } 640 } 641 642 return NULL; 643 } 644 645 void 646 spdk_nvmf_tgt_accept(struct spdk_nvmf_tgt *tgt, new_qpair_fn cb_fn) 647 { 648 struct spdk_nvmf_transport *transport, *tmp; 649 650 TAILQ_FOREACH_SAFE(transport, &tgt->transports, link, tmp) { 651 spdk_nvmf_transport_accept(transport, cb_fn); 652 } 653 } 654 655 struct spdk_nvmf_poll_group * 656 spdk_nvmf_poll_group_create(struct spdk_nvmf_tgt *tgt) 657 { 658 struct spdk_io_channel *ch; 659 660 ch = spdk_get_io_channel(tgt); 661 if (!ch) { 662 SPDK_ERRLOG("Unable to get I/O channel for target\n"); 663 return NULL; 664 } 665 666 return spdk_io_channel_get_ctx(ch); 667 } 668 669 void 670 spdk_nvmf_poll_group_destroy(struct spdk_nvmf_poll_group *group) 671 { 672 /* This function will put the io_channel associated with this poll group */ 673 spdk_nvmf_tgt_destroy_poll_group_qpairs(group); 674 } 675 676 int 677 spdk_nvmf_poll_group_add(struct spdk_nvmf_poll_group *group, 678 struct spdk_nvmf_qpair *qpair) 679 { 680 int rc = -1; 681 struct spdk_nvmf_transport_poll_group *tgroup; 682 683 TAILQ_INIT(&qpair->outstanding); 684 qpair->group = group; 685 spdk_nvmf_qpair_set_state(qpair, SPDK_NVMF_QPAIR_ACTIVATING); 686 687 TAILQ_INSERT_TAIL(&group->qpairs, qpair, link); 688 689 TAILQ_FOREACH(tgroup, &group->tgroups, link) { 690 if (tgroup->transport == qpair->transport) { 691 rc = spdk_nvmf_transport_poll_group_add(tgroup, qpair); 692 break; 693 } 694 } 695 696 if (rc == 0) { 697 spdk_nvmf_qpair_set_state(qpair, SPDK_NVMF_QPAIR_ACTIVE); 698 } else { 699 spdk_nvmf_qpair_set_state(qpair, SPDK_NVMF_QPAIR_INACTIVE); 700 } 701 702 return rc; 703 } 704 705 static 706 void _nvmf_ctrlr_destruct(void *ctx) 707 { 708 struct spdk_nvmf_ctrlr *ctrlr = ctx; 709 710 spdk_nvmf_ctrlr_destruct(ctrlr); 711 } 712 713 static void 714 _spdk_nvmf_ctrlr_free_from_qpair(void *ctx) 715 { 716 struct nvmf_qpair_disconnect_ctx *qpair_ctx = ctx; 717 struct spdk_nvmf_ctrlr *ctrlr = qpair_ctx->ctrlr; 718 uint32_t count; 719 720 spdk_bit_array_clear(ctrlr->qpair_mask, qpair_ctx->qid); 721 count = spdk_bit_array_count_set(ctrlr->qpair_mask); 722 if (count == 0) { 723 spdk_bit_array_free(&ctrlr->qpair_mask); 724 725 spdk_thread_send_msg(ctrlr->subsys->thread, _nvmf_ctrlr_destruct, ctrlr); 726 } 727 728 if (qpair_ctx->cb_fn) { 729 spdk_thread_send_msg(qpair_ctx->thread, qpair_ctx->cb_fn, qpair_ctx->ctx); 730 } 731 free(qpair_ctx); 732 } 733 734 static void 735 _spdk_nvmf_qpair_destroy(void *ctx, int status) 736 { 737 struct nvmf_qpair_disconnect_ctx *qpair_ctx = ctx; 738 struct spdk_nvmf_qpair *qpair = qpair_ctx->qpair; 739 struct spdk_nvmf_ctrlr *ctrlr = qpair->ctrlr; 740 741 assert(qpair->state == SPDK_NVMF_QPAIR_DEACTIVATING); 742 spdk_nvmf_qpair_set_state(qpair, SPDK_NVMF_QPAIR_INACTIVE); 743 qpair_ctx->qid = qpair->qid; 744 745 TAILQ_REMOVE(&qpair->group->qpairs, qpair, link); 746 qpair->group = NULL; 747 748 spdk_nvmf_transport_qpair_fini(qpair); 749 750 if (!ctrlr || !ctrlr->thread) { 751 if (qpair_ctx->cb_fn) { 752 spdk_thread_send_msg(qpair_ctx->thread, qpair_ctx->cb_fn, qpair_ctx->ctx); 753 } 754 free(qpair_ctx); 755 return; 756 } 757 758 qpair_ctx->ctrlr = ctrlr; 759 spdk_thread_send_msg(ctrlr->thread, _spdk_nvmf_ctrlr_free_from_qpair, qpair_ctx); 760 761 } 762 763 int 764 spdk_nvmf_qpair_disconnect(struct spdk_nvmf_qpair *qpair, nvmf_qpair_disconnect_cb cb_fn, void *ctx) 765 { 766 struct nvmf_qpair_disconnect_ctx *qpair_ctx; 767 768 /* If we get a qpair in the uninitialized state, we can just destroy it immediately */ 769 if (qpair->state == SPDK_NVMF_QPAIR_UNINITIALIZED) { 770 spdk_nvmf_transport_qpair_fini(qpair); 771 if (cb_fn) { 772 cb_fn(ctx); 773 } 774 return 0; 775 } 776 777 /* The queue pair must be disconnected from the thread that owns it */ 778 assert(qpair->group->thread == spdk_get_thread()); 779 780 if (qpair->state == SPDK_NVMF_QPAIR_DEACTIVATING || 781 qpair->state == SPDK_NVMF_QPAIR_INACTIVE) { 782 /* This can occur if the connection is killed by the target, 783 * which results in a notification that the connection 784 * died. Send a message to defer the processing of this 785 * callback. This allows the stack to unwind in the case 786 * where a bunch of connections are disconnected in 787 * a loop. */ 788 if (cb_fn) { 789 spdk_thread_send_msg(qpair->group->thread, cb_fn, ctx); 790 } 791 return 0; 792 } 793 794 assert(qpair->state == SPDK_NVMF_QPAIR_ACTIVE); 795 spdk_nvmf_qpair_set_state(qpair, SPDK_NVMF_QPAIR_DEACTIVATING); 796 797 qpair_ctx = calloc(1, sizeof(struct nvmf_qpair_disconnect_ctx)); 798 if (!qpair_ctx) { 799 SPDK_ERRLOG("Unable to allocate context for nvmf_qpair_disconnect\n"); 800 return -ENOMEM; 801 } 802 803 qpair_ctx->qpair = qpair; 804 qpair_ctx->cb_fn = cb_fn; 805 qpair_ctx->thread = qpair->group->thread; 806 qpair_ctx->ctx = ctx; 807 808 /* Check for outstanding I/O */ 809 if (!TAILQ_EMPTY(&qpair->outstanding)) { 810 qpair->state_cb = _spdk_nvmf_qpair_destroy; 811 qpair->state_cb_arg = qpair_ctx; 812 spdk_nvmf_qpair_free_aer(qpair); 813 return 0; 814 } 815 816 _spdk_nvmf_qpair_destroy(qpair_ctx, 0); 817 818 return 0; 819 } 820 821 int 822 spdk_nvmf_qpair_get_peer_trid(struct spdk_nvmf_qpair *qpair, 823 struct spdk_nvme_transport_id *trid) 824 { 825 return spdk_nvmf_transport_qpair_get_peer_trid(qpair, trid); 826 } 827 828 int 829 spdk_nvmf_qpair_get_local_trid(struct spdk_nvmf_qpair *qpair, 830 struct spdk_nvme_transport_id *trid) 831 { 832 return spdk_nvmf_transport_qpair_get_local_trid(qpair, trid); 833 } 834 835 int 836 spdk_nvmf_qpair_get_listen_trid(struct spdk_nvmf_qpair *qpair, 837 struct spdk_nvme_transport_id *trid) 838 { 839 return spdk_nvmf_transport_qpair_get_listen_trid(qpair, trid); 840 } 841 842 int 843 spdk_nvmf_poll_group_add_transport(struct spdk_nvmf_poll_group *group, 844 struct spdk_nvmf_transport *transport) 845 { 846 struct spdk_nvmf_transport_poll_group *tgroup; 847 848 TAILQ_FOREACH(tgroup, &group->tgroups, link) { 849 if (tgroup->transport == transport) { 850 /* Transport already in the poll group */ 851 return 0; 852 } 853 } 854 855 tgroup = spdk_nvmf_transport_poll_group_create(transport); 856 if (!tgroup) { 857 SPDK_ERRLOG("Unable to create poll group for transport\n"); 858 return -1; 859 } 860 861 TAILQ_INSERT_TAIL(&group->tgroups, tgroup, link); 862 863 return 0; 864 } 865 866 static int 867 poll_group_update_subsystem(struct spdk_nvmf_poll_group *group, 868 struct spdk_nvmf_subsystem *subsystem) 869 { 870 struct spdk_nvmf_subsystem_poll_group *sgroup; 871 uint32_t new_num_channels, old_num_channels; 872 uint32_t i; 873 struct spdk_nvmf_ns *ns; 874 875 /* Make sure our poll group has memory for this subsystem allocated */ 876 if (subsystem->id >= group->num_sgroups) { 877 return -ENOMEM; 878 } 879 880 sgroup = &group->sgroups[subsystem->id]; 881 882 /* Make sure the array of channels is the correct size */ 883 new_num_channels = subsystem->max_nsid; 884 old_num_channels = sgroup->num_channels; 885 886 if (old_num_channels == 0) { 887 if (new_num_channels > 0) { 888 /* First allocation */ 889 sgroup->channels = calloc(new_num_channels, sizeof(sgroup->channels[0])); 890 if (!sgroup->channels) { 891 return -ENOMEM; 892 } 893 } 894 } else if (new_num_channels > old_num_channels) { 895 void *buf; 896 897 /* Make the array larger */ 898 buf = realloc(sgroup->channels, new_num_channels * sizeof(sgroup->channels[0])); 899 if (!buf) { 900 return -ENOMEM; 901 } 902 903 sgroup->channels = buf; 904 905 /* Null out the new channels slots */ 906 for (i = old_num_channels; i < new_num_channels; i++) { 907 sgroup->channels[i] = NULL; 908 } 909 } else if (new_num_channels < old_num_channels) { 910 void *buf; 911 912 /* Free the extra I/O channels */ 913 for (i = new_num_channels; i < old_num_channels; i++) { 914 if (sgroup->channels[i]) { 915 spdk_put_io_channel(sgroup->channels[i]); 916 sgroup->channels[i] = NULL; 917 } 918 } 919 920 /* Make the array smaller */ 921 if (new_num_channels > 0) { 922 buf = realloc(sgroup->channels, new_num_channels * sizeof(sgroup->channels[0])); 923 if (!buf) { 924 return -ENOMEM; 925 } 926 sgroup->channels = buf; 927 } else { 928 free(sgroup->channels); 929 sgroup->channels = NULL; 930 } 931 } 932 933 sgroup->num_channels = new_num_channels; 934 935 /* Detect bdevs that were added or removed */ 936 for (i = 0; i < sgroup->num_channels; i++) { 937 ns = subsystem->ns[i]; 938 if (ns == NULL && sgroup->channels[i] == NULL) { 939 /* Both NULL. Leave empty */ 940 } else if (ns == NULL && sgroup->channels[i] != NULL) { 941 /* There was a channel here, but the namespace is gone. */ 942 spdk_put_io_channel(sgroup->channels[i]); 943 sgroup->channels[i] = NULL; 944 } else if (ns != NULL && sgroup->channels[i] == NULL) { 945 /* A namespace appeared but there is no channel yet */ 946 sgroup->channels[i] = spdk_bdev_get_io_channel(ns->desc); 947 if (sgroup->channels[i] == NULL) { 948 SPDK_ERRLOG("Could not allocate I/O channel.\n"); 949 return -ENOMEM; 950 } 951 } else { 952 /* A namespace was present before and didn't change. */ 953 } 954 } 955 956 return 0; 957 } 958 959 int 960 spdk_nvmf_poll_group_update_subsystem(struct spdk_nvmf_poll_group *group, 961 struct spdk_nvmf_subsystem *subsystem) 962 { 963 return poll_group_update_subsystem(group, subsystem); 964 } 965 966 void 967 spdk_nvmf_poll_group_add_subsystem(struct spdk_nvmf_poll_group *group, 968 struct spdk_nvmf_subsystem *subsystem, 969 spdk_nvmf_poll_group_mod_done cb_fn, void *cb_arg) 970 { 971 int rc = 0; 972 struct spdk_nvmf_subsystem_poll_group *sgroup = &group->sgroups[subsystem->id]; 973 974 TAILQ_INIT(&sgroup->queued); 975 976 rc = poll_group_update_subsystem(group, subsystem); 977 if (rc) { 978 sgroup->state = SPDK_NVMF_SUBSYSTEM_INACTIVE; 979 goto fini; 980 } 981 982 sgroup->state = SPDK_NVMF_SUBSYSTEM_ACTIVE; 983 fini: 984 if (cb_fn) { 985 cb_fn(cb_arg, rc); 986 } 987 } 988 989 static void 990 _nvmf_poll_group_remove_subsystem_cb(void *ctx, int status) 991 { 992 struct nvmf_qpair_disconnect_many_ctx *qpair_ctx = ctx; 993 struct spdk_nvmf_subsystem *subsystem; 994 struct spdk_nvmf_poll_group *group; 995 struct spdk_nvmf_subsystem_poll_group *sgroup; 996 spdk_nvmf_poll_group_mod_done cpl_fn = NULL; 997 void *cpl_ctx = NULL; 998 uint32_t nsid; 999 1000 group = qpair_ctx->group; 1001 subsystem = qpair_ctx->subsystem; 1002 cpl_fn = qpair_ctx->cpl_fn; 1003 cpl_ctx = qpair_ctx->cpl_ctx; 1004 sgroup = &group->sgroups[subsystem->id]; 1005 1006 if (status) { 1007 goto fini; 1008 } 1009 1010 for (nsid = 0; nsid < sgroup->num_channels; nsid++) { 1011 if (sgroup->channels[nsid]) { 1012 spdk_put_io_channel(sgroup->channels[nsid]); 1013 sgroup->channels[nsid] = NULL; 1014 } 1015 } 1016 1017 sgroup->num_channels = 0; 1018 free(sgroup->channels); 1019 sgroup->channels = NULL; 1020 fini: 1021 free(qpair_ctx); 1022 if (cpl_fn) { 1023 cpl_fn(cpl_ctx, status); 1024 } 1025 } 1026 1027 static void 1028 _nvmf_subsystem_disconnect_next_qpair(void *ctx) 1029 { 1030 struct spdk_nvmf_qpair *qpair; 1031 struct nvmf_qpair_disconnect_many_ctx *qpair_ctx = ctx; 1032 struct spdk_nvmf_subsystem *subsystem; 1033 struct spdk_nvmf_poll_group *group; 1034 int rc = 0; 1035 1036 group = qpair_ctx->group; 1037 subsystem = qpair_ctx->subsystem; 1038 1039 TAILQ_FOREACH(qpair, &group->qpairs, link) { 1040 if (qpair->ctrlr->subsys == subsystem) { 1041 break; 1042 } 1043 } 1044 1045 if (qpair) { 1046 rc = spdk_nvmf_qpair_disconnect(qpair, _nvmf_subsystem_disconnect_next_qpair, qpair_ctx); 1047 } 1048 1049 if (!qpair || rc != 0) { 1050 _nvmf_poll_group_remove_subsystem_cb(ctx, rc); 1051 } 1052 return; 1053 } 1054 1055 void 1056 spdk_nvmf_poll_group_remove_subsystem(struct spdk_nvmf_poll_group *group, 1057 struct spdk_nvmf_subsystem *subsystem, 1058 spdk_nvmf_poll_group_mod_done cb_fn, void *cb_arg) 1059 { 1060 struct spdk_nvmf_qpair *qpair; 1061 struct spdk_nvmf_subsystem_poll_group *sgroup; 1062 struct nvmf_qpair_disconnect_many_ctx *ctx; 1063 int rc = 0; 1064 1065 ctx = calloc(1, sizeof(struct nvmf_qpair_disconnect_many_ctx)); 1066 1067 if (!ctx) { 1068 SPDK_ERRLOG("Unable to allocate memory for context to remove poll subsystem\n"); 1069 goto fini; 1070 } 1071 1072 ctx->group = group; 1073 ctx->subsystem = subsystem; 1074 ctx->cpl_fn = cb_fn; 1075 ctx->cpl_ctx = cb_arg; 1076 1077 sgroup = &group->sgroups[subsystem->id]; 1078 sgroup->state = SPDK_NVMF_SUBSYSTEM_INACTIVE; 1079 1080 TAILQ_FOREACH(qpair, &group->qpairs, link) { 1081 if (qpair->ctrlr->subsys == subsystem) { 1082 break; 1083 } 1084 } 1085 1086 if (qpair) { 1087 rc = spdk_nvmf_qpair_disconnect(qpair, _nvmf_subsystem_disconnect_next_qpair, ctx); 1088 } else { 1089 /* call the callback immediately. It will handle any channel iteration */ 1090 _nvmf_poll_group_remove_subsystem_cb(ctx, 0); 1091 } 1092 1093 if (rc != 0) { 1094 free(ctx); 1095 goto fini; 1096 } 1097 1098 return; 1099 fini: 1100 if (cb_fn) { 1101 cb_fn(cb_arg, rc); 1102 } 1103 } 1104 1105 void 1106 spdk_nvmf_poll_group_pause_subsystem(struct spdk_nvmf_poll_group *group, 1107 struct spdk_nvmf_subsystem *subsystem, 1108 spdk_nvmf_poll_group_mod_done cb_fn, void *cb_arg) 1109 { 1110 struct spdk_nvmf_subsystem_poll_group *sgroup; 1111 int rc = 0; 1112 1113 if (subsystem->id >= group->num_sgroups) { 1114 rc = -1; 1115 goto fini; 1116 } 1117 1118 sgroup = &group->sgroups[subsystem->id]; 1119 if (sgroup == NULL) { 1120 rc = -1; 1121 goto fini; 1122 } 1123 1124 assert(sgroup->state == SPDK_NVMF_SUBSYSTEM_ACTIVE); 1125 /* TODO: This currently does not quiesce I/O */ 1126 sgroup->state = SPDK_NVMF_SUBSYSTEM_PAUSED; 1127 fini: 1128 if (cb_fn) { 1129 cb_fn(cb_arg, rc); 1130 } 1131 } 1132 1133 void 1134 spdk_nvmf_poll_group_resume_subsystem(struct spdk_nvmf_poll_group *group, 1135 struct spdk_nvmf_subsystem *subsystem, 1136 spdk_nvmf_poll_group_mod_done cb_fn, void *cb_arg) 1137 { 1138 struct spdk_nvmf_request *req, *tmp; 1139 struct spdk_nvmf_subsystem_poll_group *sgroup; 1140 int rc = 0; 1141 1142 if (subsystem->id >= group->num_sgroups) { 1143 rc = -1; 1144 goto fini; 1145 } 1146 1147 sgroup = &group->sgroups[subsystem->id]; 1148 1149 assert(sgroup->state == SPDK_NVMF_SUBSYSTEM_PAUSED); 1150 1151 rc = poll_group_update_subsystem(group, subsystem); 1152 if (rc) { 1153 goto fini; 1154 } 1155 1156 sgroup->state = SPDK_NVMF_SUBSYSTEM_ACTIVE; 1157 1158 /* Release all queued requests */ 1159 TAILQ_FOREACH_SAFE(req, &sgroup->queued, link, tmp) { 1160 TAILQ_REMOVE(&sgroup->queued, req, link); 1161 spdk_nvmf_request_exec(req); 1162 } 1163 fini: 1164 if (cb_fn) { 1165 cb_fn(cb_arg, rc); 1166 } 1167 } 1168