1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2020 Intel Corporation. 3 * All rights reserved. 4 */ 5 6 #include "spdk/stdinc.h" 7 #include "spdk/bdev.h" 8 #include "spdk/env.h" 9 #include "spdk/thread.h" 10 #include "spdk/json.h" 11 #include "spdk/string.h" 12 #include "spdk/likely.h" 13 #include "spdk/util.h" 14 #include "spdk/string.h" 15 #include "spdk/ftl.h" 16 #include "spdk/log.h" 17 18 #include "bdev_ftl.h" 19 20 struct ftl_bdev { 21 struct spdk_bdev bdev; 22 struct spdk_ftl_dev *dev; 23 ftl_bdev_init_fn init_cb; 24 void *init_arg; 25 int rc; 26 struct spdk_bdev_desc *base_bdev_desc; 27 struct spdk_bdev_desc *cache_bdev_desc; 28 }; 29 30 struct ftl_deferred_init { 31 struct spdk_ftl_conf conf; 32 33 LIST_ENTRY(ftl_deferred_init) entry; 34 }; 35 36 static LIST_HEAD(, ftl_deferred_init) g_deferred_init = LIST_HEAD_INITIALIZER(g_deferred_init); 37 38 static int bdev_ftl_initialize(void); 39 static void bdev_ftl_finish(void); 40 static void bdev_ftl_examine(struct spdk_bdev *bdev); 41 42 static int 43 bdev_ftl_get_ctx_size(void) 44 { 45 return spdk_ftl_io_size(); 46 } 47 48 static struct spdk_bdev_module g_ftl_if = { 49 .name = "ftl", 50 .module_init = bdev_ftl_initialize, 51 .module_fini = bdev_ftl_finish, 52 .examine_disk = bdev_ftl_examine, 53 .get_ctx_size = bdev_ftl_get_ctx_size, 54 }; 55 56 SPDK_BDEV_MODULE_REGISTER(ftl, &g_ftl_if) 57 58 static void 59 bdev_ftl_free(struct ftl_bdev *ftl_bdev) 60 { 61 spdk_bdev_close(ftl_bdev->base_bdev_desc); 62 spdk_bdev_close(ftl_bdev->cache_bdev_desc); 63 free(ftl_bdev->bdev.name); 64 free(ftl_bdev); 65 } 66 67 static void 68 bdev_ftl_dev_free_cb(void *ctx, int status) 69 { 70 struct ftl_bdev *ftl_bdev = ctx; 71 72 spdk_bdev_destruct_done(&ftl_bdev->bdev, status); 73 bdev_ftl_free(ftl_bdev); 74 } 75 76 static int 77 bdev_ftl_destruct(void *ctx) 78 { 79 struct ftl_bdev *ftl_bdev = ctx; 80 81 spdk_ftl_dev_free(ftl_bdev->dev, bdev_ftl_dev_free_cb, ftl_bdev); 82 83 /* return 1 to indicate that the destruction is asynchronous */ 84 return 1; 85 } 86 87 static void 88 bdev_ftl_cb(void *arg, int rc) 89 { 90 struct spdk_bdev_io *bdev_io = arg; 91 enum spdk_bdev_io_status status; 92 93 switch (rc) { 94 case 0: 95 status = SPDK_BDEV_IO_STATUS_SUCCESS; 96 break; 97 case -ENOMEM: 98 status = SPDK_BDEV_IO_STATUS_NOMEM; 99 break; 100 default: 101 status = SPDK_BDEV_IO_STATUS_FAILED; 102 break; 103 } 104 105 spdk_bdev_io_complete(bdev_io, status); 106 } 107 108 static void 109 bdev_ftl_get_buf_cb(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io, 110 bool success) 111 { 112 struct ftl_bdev *ftl_bdev; 113 int rc; 114 115 ftl_bdev = bdev_io->bdev->ctxt; 116 117 if (!success) { 118 spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED); 119 return; 120 } 121 122 rc = spdk_ftl_readv(ftl_bdev->dev, (struct ftl_io *)bdev_io->driver_ctx, 123 ch, 124 bdev_io->u.bdev.offset_blocks, 125 bdev_io->u.bdev.num_blocks, 126 bdev_io->u.bdev.iovs, bdev_io->u.bdev.iovcnt, bdev_ftl_cb, bdev_io); 127 128 if (spdk_unlikely(rc != 0)) { 129 spdk_bdev_io_complete(bdev_io, rc); 130 } 131 } 132 133 static int 134 _bdev_ftl_submit_request(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io) 135 { 136 struct ftl_bdev *ftl_bdev = (struct ftl_bdev *)bdev_io->bdev->ctxt; 137 138 switch (bdev_io->type) { 139 case SPDK_BDEV_IO_TYPE_READ: 140 spdk_bdev_io_get_buf(bdev_io, bdev_ftl_get_buf_cb, 141 bdev_io->u.bdev.num_blocks * bdev_io->bdev->blocklen); 142 return 0; 143 144 case SPDK_BDEV_IO_TYPE_WRITE: 145 return spdk_ftl_writev(ftl_bdev->dev, (struct ftl_io *)bdev_io->driver_ctx, 146 ch, bdev_io->u.bdev.offset_blocks, 147 bdev_io->u.bdev.num_blocks, bdev_io->u.bdev.iovs, 148 bdev_io->u.bdev.iovcnt, bdev_ftl_cb, bdev_io); 149 150 case SPDK_BDEV_IO_TYPE_UNMAP: 151 return spdk_ftl_unmap(ftl_bdev->dev, (struct ftl_io *)bdev_io->driver_ctx, 152 ch, bdev_io->u.bdev.offset_blocks, 153 bdev_io->u.bdev.num_blocks, bdev_ftl_cb, bdev_io); 154 case SPDK_BDEV_IO_TYPE_FLUSH: 155 spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_SUCCESS); 156 return 0; 157 default: 158 return -ENOTSUP; 159 } 160 } 161 162 static void 163 bdev_ftl_submit_request(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io) 164 { 165 int rc = _bdev_ftl_submit_request(ch, bdev_io); 166 167 if (spdk_unlikely(rc != 0)) { 168 spdk_bdev_io_complete(bdev_io, rc); 169 } 170 } 171 172 static bool 173 bdev_ftl_io_type_supported(void *ctx, enum spdk_bdev_io_type io_type) 174 { 175 switch (io_type) { 176 case SPDK_BDEV_IO_TYPE_READ: 177 case SPDK_BDEV_IO_TYPE_WRITE: 178 case SPDK_BDEV_IO_TYPE_FLUSH: 179 case SPDK_BDEV_IO_TYPE_UNMAP: 180 return true; 181 default: 182 return false; 183 } 184 } 185 186 static struct spdk_io_channel * 187 bdev_ftl_get_io_channel(void *ctx) 188 { 189 struct ftl_bdev *ftl_bdev = ctx; 190 191 return spdk_ftl_get_io_channel(ftl_bdev->dev); 192 } 193 194 static void 195 bdev_ftl_write_config_json(struct spdk_bdev *bdev, struct spdk_json_write_ctx *w) 196 { 197 struct ftl_bdev *ftl_bdev = bdev->ctxt; 198 struct spdk_ftl_conf conf; 199 char uuid[SPDK_UUID_STRING_LEN]; 200 201 spdk_ftl_dev_get_conf(ftl_bdev->dev, &conf, sizeof(conf)); 202 203 spdk_json_write_object_begin(w); 204 205 spdk_json_write_named_string(w, "method", "bdev_ftl_create"); 206 207 spdk_json_write_named_object_begin(w, "params"); 208 spdk_json_write_named_string(w, "name", ftl_bdev->bdev.name); 209 210 spdk_json_write_named_uint64(w, "overprovisioning", conf.overprovisioning); 211 spdk_json_write_named_uint64(w, "l2p_dram_limit", conf.l2p_dram_limit); 212 213 if (conf.core_mask) { 214 spdk_json_write_named_string(w, "core_mask", conf.core_mask); 215 } 216 217 spdk_uuid_fmt_lower(uuid, sizeof(uuid), &conf.uuid); 218 spdk_json_write_named_string(w, "uuid", uuid); 219 220 spdk_json_write_named_bool(w, "fast_shutdown", conf.fast_shutdown); 221 222 spdk_json_write_named_string(w, "base_bdev", conf.base_bdev); 223 224 if (conf.cache_bdev) { 225 spdk_json_write_named_string(w, "cache", conf.cache_bdev); 226 } 227 228 spdk_json_write_object_end(w); 229 spdk_json_write_object_end(w); 230 } 231 232 static int 233 bdev_ftl_dump_info_json(void *ctx, struct spdk_json_write_ctx *w) 234 { 235 struct ftl_bdev *ftl_bdev = ctx; 236 struct spdk_ftl_attrs attrs; 237 struct spdk_ftl_conf conf; 238 239 spdk_ftl_dev_get_attrs(ftl_bdev->dev, &attrs, sizeof(attrs)); 240 spdk_ftl_dev_get_conf(ftl_bdev->dev, &conf, sizeof(conf)); 241 242 spdk_json_write_named_object_begin(w, "ftl"); 243 244 spdk_json_write_named_string(w, "base_bdev", conf.base_bdev); 245 246 if (conf.cache_bdev) { 247 spdk_json_write_named_string(w, "cache", conf.cache_bdev); 248 } 249 250 /* ftl */ 251 spdk_json_write_object_end(w); 252 253 return 0; 254 } 255 256 static const struct spdk_bdev_fn_table ftl_fn_table = { 257 .destruct = bdev_ftl_destruct, 258 .submit_request = bdev_ftl_submit_request, 259 .io_type_supported = bdev_ftl_io_type_supported, 260 .get_io_channel = bdev_ftl_get_io_channel, 261 .write_config_json = bdev_ftl_write_config_json, 262 .dump_info_json = bdev_ftl_dump_info_json, 263 }; 264 265 static void 266 bdev_ftl_create_err_complete(struct ftl_bdev *ftl_bdev) 267 { 268 ftl_bdev_init_fn init_cb = ftl_bdev->init_cb; 269 void *init_arg = ftl_bdev->init_arg; 270 int rc = ftl_bdev->rc; 271 272 bdev_ftl_free(ftl_bdev); 273 274 assert(rc); 275 init_cb(NULL, init_arg, rc); 276 } 277 278 static void 279 bdev_ftl_create_err_cleanup_cb(void *ctx, int status) 280 { 281 struct ftl_bdev *ftl_bdev = ctx; 282 283 if (status) { 284 SPDK_ERRLOG("Fatal ERROR of FTL cleanup, name %s\n", ftl_bdev->bdev.name); 285 } 286 287 bdev_ftl_create_err_complete(ftl_bdev); 288 } 289 290 static void 291 bdev_ftl_create_cb(struct spdk_ftl_dev *dev, void *ctx, int status) 292 { 293 struct ftl_bdev *ftl_bdev = ctx; 294 struct ftl_bdev_info info = {}; 295 struct spdk_ftl_attrs attrs; 296 struct spdk_ftl_conf conf; 297 ftl_bdev_init_fn init_cb = ftl_bdev->init_cb; 298 void *init_arg = ftl_bdev->init_arg; 299 300 if (status) { 301 SPDK_ERRLOG("Failed to create FTL device (%d)\n", status); 302 ftl_bdev->rc = status; 303 goto error; 304 } 305 306 spdk_ftl_dev_get_attrs(dev, &attrs, sizeof(attrs)); 307 spdk_ftl_dev_get_conf(dev, &conf, sizeof(conf)); 308 309 ftl_bdev->dev = dev; 310 ftl_bdev->bdev.product_name = "FTL disk"; 311 ftl_bdev->bdev.write_cache = 0; 312 ftl_bdev->bdev.blocklen = attrs.block_size; 313 ftl_bdev->bdev.blockcnt = attrs.num_blocks; 314 ftl_bdev->bdev.uuid = conf.uuid; 315 ftl_bdev->bdev.optimal_io_boundary = attrs.optimum_io_size; 316 ftl_bdev->bdev.split_on_optimal_io_boundary = true; 317 318 SPDK_DEBUGLOG(bdev_ftl, "Creating bdev %s:\n", ftl_bdev->bdev.name); 319 SPDK_DEBUGLOG(bdev_ftl, "\tblock_len:\t%zu\n", attrs.block_size); 320 SPDK_DEBUGLOG(bdev_ftl, "\tnum_blocks:\t%"PRIu64"\n", attrs.num_blocks); 321 322 ftl_bdev->bdev.ctxt = ftl_bdev; 323 ftl_bdev->bdev.fn_table = &ftl_fn_table; 324 ftl_bdev->bdev.module = &g_ftl_if; 325 326 status = spdk_bdev_register(&ftl_bdev->bdev); 327 if (status) { 328 ftl_bdev->rc = status; 329 goto error; 330 } 331 332 info.name = ftl_bdev->bdev.name; 333 info.uuid = ftl_bdev->bdev.uuid; 334 335 init_cb(&info, init_arg, 0); 336 return; 337 338 error: 339 if (ftl_bdev->dev) { 340 /* Cleanup all FTL */ 341 spdk_ftl_dev_set_fast_shutdown(ftl_bdev->dev, false); 342 343 /* FTL was created, but we have got an error, so we need to delete it */ 344 spdk_ftl_dev_free(dev, bdev_ftl_create_err_cleanup_cb, ftl_bdev); 345 } else { 346 bdev_ftl_create_err_complete(ftl_bdev); 347 } 348 } 349 350 static void 351 bdev_ftl_defer_free(struct ftl_deferred_init *init) 352 { 353 spdk_ftl_conf_deinit(&init->conf); 354 free(init); 355 } 356 357 int 358 bdev_ftl_defer_init(const struct spdk_ftl_conf *conf) 359 { 360 struct ftl_deferred_init *init; 361 int rc; 362 363 init = calloc(1, sizeof(*init)); 364 if (!init) { 365 return -ENOMEM; 366 } 367 368 rc = spdk_ftl_conf_copy(&init->conf, conf); 369 if (rc) { 370 free(init); 371 return -ENOMEM; 372 } 373 374 LIST_INSERT_HEAD(&g_deferred_init, init, entry); 375 376 return 0; 377 } 378 379 static void 380 bdev_ftl_create_bdev_event_cb(enum spdk_bdev_event_type type, struct spdk_bdev *bdev, void *ctx) 381 { 382 } 383 384 int 385 bdev_ftl_create_bdev(const struct spdk_ftl_conf *conf, ftl_bdev_init_fn cb, void *cb_arg) 386 { 387 struct ftl_bdev *ftl_bdev; 388 struct spdk_bdev_desc *base_bdev_desc, *cache_bdev_desc; 389 int rc; 390 391 rc = spdk_bdev_open_ext(conf->base_bdev, false, bdev_ftl_create_bdev_event_cb, NULL, 392 &base_bdev_desc); 393 if (rc) { 394 return rc; 395 } 396 rc = spdk_bdev_open_ext(conf->cache_bdev, false, bdev_ftl_create_bdev_event_cb, NULL, 397 &cache_bdev_desc); 398 if (rc) { 399 spdk_bdev_close(base_bdev_desc); 400 return rc; 401 } 402 403 ftl_bdev = calloc(1, sizeof(*ftl_bdev)); 404 if (!ftl_bdev) { 405 SPDK_ERRLOG("Could not allocate ftl_bdev\n"); 406 spdk_bdev_close(base_bdev_desc); 407 spdk_bdev_close(cache_bdev_desc); 408 return -ENOMEM; 409 } 410 411 ftl_bdev->base_bdev_desc = base_bdev_desc; 412 ftl_bdev->cache_bdev_desc = cache_bdev_desc; 413 414 ftl_bdev->bdev.name = strdup(conf->name); 415 if (!ftl_bdev->bdev.name) { 416 rc = -ENOMEM; 417 goto error; 418 } 419 420 ftl_bdev->init_cb = cb; 421 ftl_bdev->init_arg = cb_arg; 422 423 rc = spdk_ftl_dev_init(conf, bdev_ftl_create_cb, ftl_bdev); 424 if (rc) { 425 SPDK_ERRLOG("Could not create FTL device\n"); 426 goto error; 427 } 428 429 return 0; 430 431 error: 432 bdev_ftl_free(ftl_bdev); 433 return rc; 434 } 435 436 static int 437 bdev_ftl_initialize(void) 438 { 439 return spdk_ftl_init(); 440 } 441 442 static void 443 bdev_ftl_event_cb(enum spdk_bdev_event_type type, struct spdk_bdev *bdev, void *ctx) 444 { 445 } 446 447 void 448 bdev_ftl_delete_bdev(const char *name, bool fast_shutdown, spdk_bdev_unregister_cb cb_fn, 449 void *cb_arg) 450 { 451 struct spdk_bdev_desc *ftl_bdev_desc; 452 struct spdk_bdev *bdev; 453 struct ftl_bdev *ftl; 454 int rc; 455 456 rc = spdk_bdev_open_ext(name, false, bdev_ftl_event_cb, NULL, &ftl_bdev_desc); 457 458 if (rc) { 459 goto not_found; 460 } 461 462 bdev = spdk_bdev_desc_get_bdev(ftl_bdev_desc); 463 464 if (bdev->module != &g_ftl_if) { 465 goto bdev_opened; 466 } 467 468 ftl = bdev->ctxt; 469 assert(ftl); 470 spdk_ftl_dev_set_fast_shutdown(ftl->dev, fast_shutdown); 471 spdk_bdev_close(ftl_bdev_desc); 472 473 rc = spdk_bdev_unregister_by_name(name, &g_ftl_if, cb_fn, cb_arg); 474 if (rc) { 475 cb_fn(cb_arg, rc); 476 } 477 478 return; 479 bdev_opened: 480 spdk_bdev_close(ftl_bdev_desc); 481 not_found: 482 cb_fn(cb_arg, -ENODEV); 483 } 484 485 struct ftl_unmap_ctx { 486 struct spdk_bdev_desc *bdev; 487 spdk_ftl_fn cb_fn; 488 void *cb_arg; 489 }; 490 491 static void 492 bdev_ftl_unmap_cb(void *cb_arg, int status) 493 { 494 struct ftl_unmap_ctx *ctx = cb_arg; 495 496 spdk_bdev_close(ctx->bdev); 497 ctx->cb_fn(ctx->cb_arg, status); 498 free(ctx); 499 } 500 501 void 502 bdev_ftl_unmap(const char *name, uint64_t lba, uint64_t num_blocks, spdk_ftl_fn cb_fn, void *cb_arg) 503 { 504 struct spdk_bdev_desc *ftl_bdev_desc; 505 struct spdk_bdev *bdev; 506 struct ftl_bdev *ftl; 507 struct ftl_unmap_ctx *ctx; 508 int rc; 509 510 rc = spdk_bdev_open_ext(name, false, bdev_ftl_event_cb, NULL, &ftl_bdev_desc); 511 512 if (rc) { 513 goto not_found; 514 } 515 516 bdev = spdk_bdev_desc_get_bdev(ftl_bdev_desc); 517 518 if (bdev->module != &g_ftl_if) { 519 rc = -ENODEV; 520 goto bdev_opened; 521 } 522 523 ctx = calloc(1, sizeof(struct ftl_unmap_ctx)); 524 if (!ctx) { 525 rc = -ENOMEM; 526 goto bdev_opened; 527 } 528 529 ctx->bdev = ftl_bdev_desc; 530 ctx->cb_arg = cb_arg; 531 ctx->cb_fn = cb_fn; 532 533 ftl = bdev->ctxt; 534 assert(ftl); 535 /* It's ok to pass NULL as IO channel - FTL will detect this and use it's internal IO channel for management operations */ 536 rc = spdk_ftl_unmap(ftl->dev, NULL, NULL, lba, num_blocks, bdev_ftl_unmap_cb, ctx); 537 538 if (rc) { 539 goto ctx_allocated; 540 } 541 542 return; 543 ctx_allocated: 544 free(ctx); 545 bdev_opened: 546 spdk_bdev_close(ftl_bdev_desc); 547 not_found: 548 cb_fn(cb_arg, rc); 549 } 550 551 static void 552 bdev_ftl_get_stats_cb(struct ftl_stats *stats, void *ctx) 553 { 554 struct rpc_ftl_stats_ctx *ftl_stats_ctx = ctx; 555 556 ftl_stats_ctx->cb(ftl_stats_ctx); 557 558 spdk_bdev_close(ftl_stats_ctx->ftl_bdev_desc); 559 free(ftl_stats_ctx); 560 } 561 562 563 int 564 bdev_ftl_get_stats(const char *name, ftl_bdev_thread_fn cb, struct spdk_jsonrpc_request *request, 565 struct ftl_stats *stats) 566 { 567 struct spdk_bdev_desc *ftl_bdev_desc; 568 struct spdk_bdev *bdev; 569 struct ftl_bdev *ftl; 570 struct rpc_ftl_stats_ctx *ftl_stats_ctx; 571 int rc; 572 573 rc = spdk_bdev_open_ext(name, false, bdev_ftl_event_cb, NULL, &ftl_bdev_desc); 574 if (rc) { 575 goto not_found; 576 } 577 578 bdev = spdk_bdev_desc_get_bdev(ftl_bdev_desc); 579 if (bdev->module != &g_ftl_if) { 580 rc = -ENODEV; 581 goto bdev_opened; 582 } 583 584 ftl_stats_ctx = calloc(1, sizeof(*ftl_stats_ctx)); 585 if (!ftl_stats_ctx) { 586 SPDK_ERRLOG("Could not allocate ftl_stats_ctx\n"); 587 rc = -ENOMEM; 588 goto bdev_opened; 589 } 590 591 ftl = bdev->ctxt; 592 ftl_stats_ctx->request = request; 593 ftl_stats_ctx->ftl_bdev_desc = ftl_bdev_desc; 594 ftl_stats_ctx->cb = cb; 595 ftl_stats_ctx->ftl_stats = stats; 596 597 rc = spdk_ftl_get_stats(ftl->dev, stats, bdev_ftl_get_stats_cb, ftl_stats_ctx); 598 if (rc) { 599 goto stats_allocated; 600 } 601 602 return 0; 603 stats_allocated: 604 free(ftl_stats_ctx); 605 bdev_opened: 606 spdk_bdev_close(ftl_bdev_desc); 607 not_found: 608 return rc; 609 } 610 611 static void 612 bdev_ftl_finish(void) 613 { 614 spdk_ftl_fini(); 615 } 616 617 static void 618 bdev_ftl_create_defered_cb(const struct ftl_bdev_info *info, void *ctx, int status) 619 { 620 struct ftl_deferred_init *opts = ctx; 621 622 if (status) { 623 SPDK_ERRLOG("Failed to initialize FTL bdev '%s'\n", opts->conf.name); 624 } 625 626 bdev_ftl_defer_free(opts); 627 628 spdk_bdev_module_examine_done(&g_ftl_if); 629 } 630 631 static void 632 bdev_ftl_examine(struct spdk_bdev *bdev) 633 { 634 struct ftl_deferred_init *opts; 635 int rc; 636 637 LIST_FOREACH(opts, &g_deferred_init, entry) { 638 /* spdk_bdev_module_examine_done will be called by bdev_ftl_create_defered_cb */ 639 rc = bdev_ftl_create_bdev(&opts->conf, bdev_ftl_create_defered_cb, opts); 640 if (rc == -ENODEV) { 641 continue; 642 } 643 644 LIST_REMOVE(opts, entry); 645 646 if (rc) { 647 bdev_ftl_create_defered_cb(NULL, opts, rc); 648 } 649 return; 650 } 651 652 spdk_bdev_module_examine_done(&g_ftl_if); 653 } 654 655 SPDK_LOG_REGISTER_COMPONENT(bdev_ftl) 656