1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2020 Intel Corporation. 3 * All rights reserved. 4 * Copyright 2023 Solidigm All Rights Reserved 5 */ 6 7 #include "spdk/stdinc.h" 8 #include "spdk/bdev.h" 9 #include "spdk/env.h" 10 #include "spdk/thread.h" 11 #include "spdk/json.h" 12 #include "spdk/string.h" 13 #include "spdk/likely.h" 14 #include "spdk/util.h" 15 #include "spdk/string.h" 16 #include "spdk/ftl.h" 17 #include "spdk/log.h" 18 19 #include "bdev_ftl.h" 20 21 struct ftl_bdev { 22 struct spdk_bdev bdev; 23 struct spdk_ftl_dev *dev; 24 ftl_bdev_init_fn init_cb; 25 void *init_arg; 26 int rc; 27 struct spdk_bdev_desc *base_bdev_desc; 28 struct spdk_bdev_desc *cache_bdev_desc; 29 }; 30 31 struct bdev_ftl_action { 32 struct spdk_bdev_desc *ftl_bdev_desc; 33 struct ftl_bdev *ftl_bdev_dev; 34 spdk_ftl_fn cb_fn; 35 void *cb_arg; 36 int rc; 37 size_t ctx_size; 38 char ctx[0]; 39 }; 40 41 struct ftl_deferred_init { 42 struct spdk_ftl_conf conf; 43 44 LIST_ENTRY(ftl_deferred_init) entry; 45 }; 46 47 static LIST_HEAD(, ftl_deferred_init) g_deferred_init = LIST_HEAD_INITIALIZER(g_deferred_init); 48 49 static int bdev_ftl_initialize(void); 50 static void bdev_ftl_finish(void); 51 static void bdev_ftl_examine(struct spdk_bdev *bdev); 52 53 static void bdev_ftl_action_finish_cb(void *cb_arg, int status); 54 static struct bdev_ftl_action *bdev_ftl_action_start(const char *bdev_name, 55 size_t ctx_size, spdk_ftl_fn cb_fn, void *cb_arg); 56 static void bdev_ftl_action_finish(struct bdev_ftl_action *action); 57 58 static int 59 bdev_ftl_get_ctx_size(void) 60 { 61 return spdk_ftl_io_size(); 62 } 63 64 static struct spdk_bdev_module g_ftl_if = { 65 .name = "ftl", 66 .module_init = bdev_ftl_initialize, 67 .module_fini = bdev_ftl_finish, 68 .examine_disk = bdev_ftl_examine, 69 .get_ctx_size = bdev_ftl_get_ctx_size, 70 }; 71 72 SPDK_BDEV_MODULE_REGISTER(ftl, &g_ftl_if) 73 74 static void 75 bdev_ftl_free(struct ftl_bdev *ftl_bdev) 76 { 77 spdk_bdev_close(ftl_bdev->base_bdev_desc); 78 spdk_bdev_close(ftl_bdev->cache_bdev_desc); 79 free(ftl_bdev->bdev.name); 80 free(ftl_bdev); 81 } 82 83 static void 84 bdev_ftl_dev_free_cb(void *ctx, int status) 85 { 86 struct ftl_bdev *ftl_bdev = ctx; 87 88 spdk_bdev_destruct_done(&ftl_bdev->bdev, status); 89 bdev_ftl_free(ftl_bdev); 90 } 91 92 static int 93 bdev_ftl_destruct(void *ctx) 94 { 95 struct ftl_bdev *ftl_bdev = ctx; 96 97 spdk_ftl_dev_free(ftl_bdev->dev, bdev_ftl_dev_free_cb, ftl_bdev); 98 99 /* return 1 to indicate that the destruction is asynchronous */ 100 return 1; 101 } 102 103 static void 104 bdev_ftl_cb(void *arg, int rc) 105 { 106 struct spdk_bdev_io *bdev_io = arg; 107 enum spdk_bdev_io_status status; 108 109 switch (rc) { 110 case 0: 111 status = SPDK_BDEV_IO_STATUS_SUCCESS; 112 break; 113 case -EAGAIN: 114 case -ENOMEM: 115 status = SPDK_BDEV_IO_STATUS_NOMEM; 116 break; 117 default: 118 status = SPDK_BDEV_IO_STATUS_FAILED; 119 break; 120 } 121 122 spdk_bdev_io_complete(bdev_io, status); 123 } 124 125 static void 126 bdev_ftl_get_buf_cb(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io, 127 bool success) 128 { 129 struct ftl_bdev *ftl_bdev; 130 int rc; 131 132 ftl_bdev = bdev_io->bdev->ctxt; 133 134 if (!success) { 135 spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED); 136 return; 137 } 138 139 rc = spdk_ftl_readv(ftl_bdev->dev, (struct ftl_io *)bdev_io->driver_ctx, 140 ch, 141 bdev_io->u.bdev.offset_blocks, 142 bdev_io->u.bdev.num_blocks, 143 bdev_io->u.bdev.iovs, bdev_io->u.bdev.iovcnt, bdev_ftl_cb, bdev_io); 144 145 if (spdk_unlikely(rc != 0)) { 146 bdev_ftl_cb(bdev_io, rc); 147 } 148 } 149 150 static int 151 _bdev_ftl_submit_request(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io) 152 { 153 struct ftl_bdev *ftl_bdev = (struct ftl_bdev *)bdev_io->bdev->ctxt; 154 155 switch (bdev_io->type) { 156 case SPDK_BDEV_IO_TYPE_READ: 157 spdk_bdev_io_get_buf(bdev_io, bdev_ftl_get_buf_cb, 158 bdev_io->u.bdev.num_blocks * bdev_io->bdev->blocklen); 159 return 0; 160 161 case SPDK_BDEV_IO_TYPE_WRITE: 162 return spdk_ftl_writev(ftl_bdev->dev, (struct ftl_io *)bdev_io->driver_ctx, 163 ch, bdev_io->u.bdev.offset_blocks, 164 bdev_io->u.bdev.num_blocks, bdev_io->u.bdev.iovs, 165 bdev_io->u.bdev.iovcnt, bdev_ftl_cb, bdev_io); 166 167 case SPDK_BDEV_IO_TYPE_UNMAP: 168 return spdk_ftl_unmap(ftl_bdev->dev, (struct ftl_io *)bdev_io->driver_ctx, 169 ch, bdev_io->u.bdev.offset_blocks, 170 bdev_io->u.bdev.num_blocks, bdev_ftl_cb, bdev_io); 171 case SPDK_BDEV_IO_TYPE_FLUSH: 172 spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_SUCCESS); 173 return 0; 174 default: 175 return -ENOTSUP; 176 } 177 } 178 179 static void 180 bdev_ftl_submit_request(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io) 181 { 182 int rc = _bdev_ftl_submit_request(ch, bdev_io); 183 184 if (spdk_unlikely(rc != 0)) { 185 bdev_ftl_cb(bdev_io, rc); 186 } 187 } 188 189 static bool 190 bdev_ftl_io_type_supported(void *ctx, enum spdk_bdev_io_type io_type) 191 { 192 switch (io_type) { 193 case SPDK_BDEV_IO_TYPE_READ: 194 case SPDK_BDEV_IO_TYPE_WRITE: 195 case SPDK_BDEV_IO_TYPE_FLUSH: 196 case SPDK_BDEV_IO_TYPE_UNMAP: 197 return true; 198 default: 199 return false; 200 } 201 } 202 203 static struct spdk_io_channel * 204 bdev_ftl_get_io_channel(void *ctx) 205 { 206 struct ftl_bdev *ftl_bdev = ctx; 207 208 return spdk_ftl_get_io_channel(ftl_bdev->dev); 209 } 210 211 static void 212 bdev_ftl_write_config_json(struct spdk_bdev *bdev, struct spdk_json_write_ctx *w) 213 { 214 struct ftl_bdev *ftl_bdev = bdev->ctxt; 215 struct spdk_ftl_conf conf; 216 char uuid[SPDK_UUID_STRING_LEN]; 217 218 spdk_ftl_dev_get_conf(ftl_bdev->dev, &conf, sizeof(conf)); 219 220 spdk_json_write_object_begin(w); 221 222 spdk_json_write_named_string(w, "method", "bdev_ftl_create"); 223 224 spdk_json_write_named_object_begin(w, "params"); 225 spdk_json_write_named_string(w, "name", ftl_bdev->bdev.name); 226 227 spdk_json_write_named_uint64(w, "overprovisioning", conf.overprovisioning); 228 spdk_json_write_named_uint64(w, "l2p_dram_limit", conf.l2p_dram_limit); 229 230 if (conf.core_mask) { 231 spdk_json_write_named_string(w, "core_mask", conf.core_mask); 232 } 233 234 spdk_uuid_fmt_lower(uuid, sizeof(uuid), &conf.uuid); 235 spdk_json_write_named_string(w, "uuid", uuid); 236 237 spdk_json_write_named_bool(w, "fast_shutdown", conf.fast_shutdown); 238 239 spdk_json_write_named_string(w, "base_bdev", conf.base_bdev); 240 241 if (conf.cache_bdev) { 242 spdk_json_write_named_string(w, "cache", conf.cache_bdev); 243 } 244 245 spdk_json_write_object_end(w); 246 spdk_json_write_object_end(w); 247 } 248 249 static int 250 bdev_ftl_dump_info_json(void *ctx, struct spdk_json_write_ctx *w) 251 { 252 struct ftl_bdev *ftl_bdev = ctx; 253 struct spdk_ftl_attrs attrs; 254 struct spdk_ftl_conf conf; 255 256 spdk_ftl_dev_get_attrs(ftl_bdev->dev, &attrs, sizeof(attrs)); 257 spdk_ftl_dev_get_conf(ftl_bdev->dev, &conf, sizeof(conf)); 258 259 spdk_json_write_named_object_begin(w, "ftl"); 260 261 spdk_json_write_named_string(w, "base_bdev", conf.base_bdev); 262 263 if (conf.cache_bdev) { 264 spdk_json_write_named_string(w, "cache", conf.cache_bdev); 265 } 266 267 /* ftl */ 268 spdk_json_write_object_end(w); 269 270 return 0; 271 } 272 273 static const struct spdk_bdev_fn_table ftl_fn_table = { 274 .destruct = bdev_ftl_destruct, 275 .submit_request = bdev_ftl_submit_request, 276 .io_type_supported = bdev_ftl_io_type_supported, 277 .get_io_channel = bdev_ftl_get_io_channel, 278 .write_config_json = bdev_ftl_write_config_json, 279 .dump_info_json = bdev_ftl_dump_info_json, 280 }; 281 282 static void 283 bdev_ftl_create_err_complete(struct ftl_bdev *ftl_bdev) 284 { 285 ftl_bdev_init_fn init_cb = ftl_bdev->init_cb; 286 void *init_arg = ftl_bdev->init_arg; 287 int rc = ftl_bdev->rc; 288 289 bdev_ftl_free(ftl_bdev); 290 291 assert(rc); 292 init_cb(NULL, init_arg, rc); 293 } 294 295 static void 296 bdev_ftl_create_err_cleanup_cb(void *ctx, int status) 297 { 298 struct ftl_bdev *ftl_bdev = ctx; 299 300 if (status) { 301 SPDK_ERRLOG("Fatal ERROR of FTL cleanup, name %s\n", ftl_bdev->bdev.name); 302 } 303 304 bdev_ftl_create_err_complete(ftl_bdev); 305 } 306 307 static void 308 bdev_ftl_create_cb(struct spdk_ftl_dev *dev, void *ctx, int status) 309 { 310 struct ftl_bdev *ftl_bdev = ctx; 311 struct ftl_bdev_info info = {}; 312 struct spdk_ftl_attrs attrs; 313 struct spdk_ftl_conf conf; 314 ftl_bdev_init_fn init_cb = ftl_bdev->init_cb; 315 void *init_arg = ftl_bdev->init_arg; 316 317 if (status) { 318 SPDK_ERRLOG("Failed to create FTL device (%d)\n", status); 319 ftl_bdev->rc = status; 320 goto error; 321 } 322 323 spdk_ftl_dev_get_attrs(dev, &attrs, sizeof(attrs)); 324 spdk_ftl_dev_get_conf(dev, &conf, sizeof(conf)); 325 326 ftl_bdev->dev = dev; 327 ftl_bdev->bdev.product_name = "FTL disk"; 328 ftl_bdev->bdev.write_cache = 0; 329 ftl_bdev->bdev.blocklen = attrs.block_size; 330 ftl_bdev->bdev.blockcnt = attrs.num_blocks; 331 ftl_bdev->bdev.uuid = conf.uuid; 332 ftl_bdev->bdev.optimal_io_boundary = attrs.optimum_io_size; 333 ftl_bdev->bdev.split_on_optimal_io_boundary = true; 334 335 SPDK_DEBUGLOG(bdev_ftl, "Creating bdev %s:\n", ftl_bdev->bdev.name); 336 SPDK_DEBUGLOG(bdev_ftl, "\tblock_len:\t%zu\n", attrs.block_size); 337 SPDK_DEBUGLOG(bdev_ftl, "\tnum_blocks:\t%"PRIu64"\n", attrs.num_blocks); 338 339 ftl_bdev->bdev.ctxt = ftl_bdev; 340 ftl_bdev->bdev.fn_table = &ftl_fn_table; 341 ftl_bdev->bdev.module = &g_ftl_if; 342 343 status = spdk_bdev_register(&ftl_bdev->bdev); 344 if (status) { 345 ftl_bdev->rc = status; 346 goto error; 347 } 348 349 info.name = ftl_bdev->bdev.name; 350 info.uuid = ftl_bdev->bdev.uuid; 351 352 init_cb(&info, init_arg, 0); 353 return; 354 355 error: 356 if (ftl_bdev->dev) { 357 /* Cleanup all FTL */ 358 spdk_ftl_dev_set_fast_shutdown(ftl_bdev->dev, false); 359 360 /* FTL was created, but we have got an error, so we need to delete it */ 361 spdk_ftl_dev_free(dev, bdev_ftl_create_err_cleanup_cb, ftl_bdev); 362 } else { 363 bdev_ftl_create_err_complete(ftl_bdev); 364 } 365 } 366 367 static void 368 bdev_ftl_defer_free(struct ftl_deferred_init *init) 369 { 370 spdk_ftl_conf_deinit(&init->conf); 371 free(init); 372 } 373 374 int 375 bdev_ftl_defer_init(const struct spdk_ftl_conf *conf) 376 { 377 struct ftl_deferred_init *init; 378 int rc; 379 380 init = calloc(1, sizeof(*init)); 381 if (!init) { 382 return -ENOMEM; 383 } 384 385 rc = spdk_ftl_conf_copy(&init->conf, conf); 386 if (rc) { 387 free(init); 388 return -ENOMEM; 389 } 390 391 LIST_INSERT_HEAD(&g_deferred_init, init, entry); 392 393 return 0; 394 } 395 396 static void 397 bdev_ftl_create_bdev_event_cb(enum spdk_bdev_event_type type, struct spdk_bdev *bdev, void *ctx) 398 { 399 } 400 401 int 402 bdev_ftl_create_bdev(const struct spdk_ftl_conf *conf, ftl_bdev_init_fn cb, void *cb_arg) 403 { 404 struct ftl_bdev *ftl_bdev; 405 struct spdk_bdev_desc *base_bdev_desc, *cache_bdev_desc; 406 int rc; 407 408 rc = spdk_bdev_open_ext(conf->base_bdev, false, bdev_ftl_create_bdev_event_cb, NULL, 409 &base_bdev_desc); 410 if (rc) { 411 return rc; 412 } 413 rc = spdk_bdev_open_ext(conf->cache_bdev, false, bdev_ftl_create_bdev_event_cb, NULL, 414 &cache_bdev_desc); 415 if (rc) { 416 spdk_bdev_close(base_bdev_desc); 417 return rc; 418 } 419 420 ftl_bdev = calloc(1, sizeof(*ftl_bdev)); 421 if (!ftl_bdev) { 422 SPDK_ERRLOG("Could not allocate ftl_bdev\n"); 423 spdk_bdev_close(base_bdev_desc); 424 spdk_bdev_close(cache_bdev_desc); 425 return -ENOMEM; 426 } 427 428 ftl_bdev->base_bdev_desc = base_bdev_desc; 429 ftl_bdev->cache_bdev_desc = cache_bdev_desc; 430 431 ftl_bdev->bdev.name = strdup(conf->name); 432 if (!ftl_bdev->bdev.name) { 433 rc = -ENOMEM; 434 goto error; 435 } 436 437 ftl_bdev->init_cb = cb; 438 ftl_bdev->init_arg = cb_arg; 439 440 rc = spdk_ftl_dev_init(conf, bdev_ftl_create_cb, ftl_bdev); 441 if (rc) { 442 SPDK_ERRLOG("Could not create FTL device\n"); 443 goto error; 444 } 445 446 return 0; 447 448 error: 449 bdev_ftl_free(ftl_bdev); 450 return rc; 451 } 452 453 static int 454 bdev_ftl_initialize(void) 455 { 456 return spdk_ftl_init(); 457 } 458 459 static void 460 bdev_ftl_event_cb(enum spdk_bdev_event_type type, struct spdk_bdev *bdev, void *ctx) 461 { 462 } 463 464 void 465 bdev_ftl_delete_bdev(const char *name, bool fast_shutdown, spdk_bdev_unregister_cb cb_fn, 466 void *cb_arg) 467 { 468 struct spdk_bdev_desc *ftl_bdev_desc; 469 struct spdk_bdev *bdev; 470 struct ftl_bdev *ftl; 471 int rc; 472 473 rc = spdk_bdev_open_ext(name, false, bdev_ftl_event_cb, NULL, &ftl_bdev_desc); 474 475 if (rc) { 476 goto not_found; 477 } 478 479 bdev = spdk_bdev_desc_get_bdev(ftl_bdev_desc); 480 481 if (bdev->module != &g_ftl_if) { 482 goto bdev_opened; 483 } 484 485 ftl = bdev->ctxt; 486 assert(ftl); 487 spdk_ftl_dev_set_fast_shutdown(ftl->dev, fast_shutdown); 488 spdk_bdev_close(ftl_bdev_desc); 489 490 rc = spdk_bdev_unregister_by_name(name, &g_ftl_if, cb_fn, cb_arg); 491 if (rc) { 492 cb_fn(cb_arg, rc); 493 } 494 495 return; 496 bdev_opened: 497 spdk_bdev_close(ftl_bdev_desc); 498 not_found: 499 cb_fn(cb_arg, -ENODEV); 500 } 501 502 void 503 bdev_ftl_unmap(const char *name, uint64_t lba, uint64_t num_blocks, spdk_ftl_fn cb_fn, 504 void *cb_arg) 505 { 506 struct bdev_ftl_action *action; 507 508 action = bdev_ftl_action_start(name, 0, cb_fn, cb_arg); 509 if (!action) { 510 return; 511 } 512 513 /* It's ok to pass NULL as IO channel - FTL will detect this and use it's internal IO channel for management operations */ 514 action->rc = spdk_ftl_unmap(action->ftl_bdev_dev->dev, NULL, NULL, lba, num_blocks, 515 bdev_ftl_action_finish_cb, action); 516 if (action->rc) { 517 bdev_ftl_action_finish(action); 518 } 519 } 520 521 static void 522 bdev_ftl_get_stats_cb(struct ftl_stats *stats, void *cb_arg) 523 { 524 struct bdev_ftl_action *action = cb_arg; 525 526 bdev_ftl_action_finish(action); 527 } 528 529 void 530 bdev_ftl_get_stats(const char *name, spdk_ftl_fn cb, struct rpc_ftl_stats_ctx *ftl_stats_ctx) 531 { 532 struct bdev_ftl_action *action; 533 534 action = bdev_ftl_action_start(name, 0, cb, ftl_stats_ctx); 535 if (!action) { 536 return; 537 } 538 ftl_stats_ctx->ftl_bdev_desc = action->ftl_bdev_desc; 539 action->rc = spdk_ftl_get_stats(action->ftl_bdev_dev->dev, &ftl_stats_ctx->ftl_stats, 540 bdev_ftl_get_stats_cb, action); 541 if (action->rc) { 542 bdev_ftl_action_finish(action); 543 } 544 } 545 546 void 547 bdev_ftl_get_properties(const char *name, spdk_ftl_fn cb_fn, struct spdk_jsonrpc_request *request) 548 { 549 struct bdev_ftl_action *action; 550 551 action = bdev_ftl_action_start(name, 0, cb_fn, request); 552 if (!action) { 553 return; 554 } 555 556 action->rc = spdk_ftl_get_properties(action->ftl_bdev_dev->dev, request, 557 bdev_ftl_action_finish_cb, action); 558 if (action->rc) { 559 bdev_ftl_action_finish(action); 560 } 561 } 562 563 static void 564 bdev_ftl_finish(void) 565 { 566 spdk_ftl_fini(); 567 } 568 569 static void 570 bdev_ftl_create_deferred_cb(const struct ftl_bdev_info *info, void *ctx, int status) 571 { 572 struct ftl_deferred_init *opts = ctx; 573 574 if (status) { 575 SPDK_ERRLOG("Failed to initialize FTL bdev '%s'\n", opts->conf.name); 576 } 577 578 bdev_ftl_defer_free(opts); 579 580 spdk_bdev_module_examine_done(&g_ftl_if); 581 } 582 583 static void 584 bdev_ftl_examine(struct spdk_bdev *bdev) 585 { 586 struct ftl_deferred_init *opts; 587 int rc; 588 589 LIST_FOREACH(opts, &g_deferred_init, entry) { 590 /* spdk_bdev_module_examine_done will be called by bdev_ftl_create_deferred_cb */ 591 rc = bdev_ftl_create_bdev(&opts->conf, bdev_ftl_create_deferred_cb, opts); 592 if (rc == -ENODEV) { 593 continue; 594 } 595 596 LIST_REMOVE(opts, entry); 597 598 if (rc) { 599 bdev_ftl_create_deferred_cb(NULL, opts, rc); 600 } 601 return; 602 } 603 604 spdk_bdev_module_examine_done(&g_ftl_if); 605 } 606 607 SPDK_LOG_REGISTER_COMPONENT(bdev_ftl) 608 609 /* 610 * Generic function to execute an action on the FTL bdev 611 */ 612 static void 613 bdev_ftl_action_finish(struct bdev_ftl_action *action) 614 { 615 action->cb_fn(action->cb_arg, action->rc); 616 if (action->ftl_bdev_desc) { 617 spdk_bdev_close(action->ftl_bdev_desc); 618 } 619 free(action); 620 } 621 622 static struct bdev_ftl_action * 623 bdev_ftl_action_start(const char *bdev_name, size_t ctx_size, spdk_ftl_fn cb_fn, void *cb_arg) 624 { 625 struct spdk_bdev *bdev; 626 struct bdev_ftl_action *action = calloc(1, sizeof(*action) + ctx_size); 627 628 if (NULL == action) { 629 cb_fn(cb_arg, -ENOMEM); 630 return NULL; 631 } 632 action->cb_arg = cb_arg; 633 action->cb_fn = cb_fn; 634 635 action->rc = spdk_bdev_open_ext(bdev_name, false, bdev_ftl_event_cb, NULL, &action->ftl_bdev_desc); 636 if (action->rc) { 637 goto error; 638 } 639 640 bdev = spdk_bdev_desc_get_bdev(action->ftl_bdev_desc); 641 if (bdev->module != &g_ftl_if) { 642 action->rc = -ENODEV; 643 goto error; 644 } 645 646 action->ftl_bdev_dev = bdev->ctxt; 647 assert(action->ftl_bdev_dev); 648 649 return action; 650 error: 651 bdev_ftl_action_finish(action); 652 return NULL; 653 } 654 655 static void 656 bdev_ftl_action_finish_cb(void *cb_arg, int status) 657 { 658 struct bdev_ftl_action *action = cb_arg; 659 660 action->rc = status; 661 bdev_ftl_action_finish(action); 662 } 663