1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2020 Intel Corporation. 3 * All rights reserved. 4 */ 5 6 #include "spdk/stdinc.h" 7 #include "spdk/bdev.h" 8 #include "spdk/env.h" 9 #include "spdk/thread.h" 10 #include "spdk/json.h" 11 #include "spdk/string.h" 12 #include "spdk/likely.h" 13 #include "spdk/util.h" 14 #include "spdk/string.h" 15 #include "spdk/ftl.h" 16 #include "spdk/log.h" 17 18 #include "bdev_ftl.h" 19 20 struct ftl_bdev { 21 struct spdk_bdev bdev; 22 struct spdk_ftl_dev *dev; 23 ftl_bdev_init_fn init_cb; 24 void *init_arg; 25 int rc; 26 struct spdk_bdev_desc *base_bdev_desc; 27 struct spdk_bdev_desc *cache_bdev_desc; 28 }; 29 30 struct ftl_deferred_init { 31 struct spdk_ftl_conf conf; 32 33 LIST_ENTRY(ftl_deferred_init) entry; 34 }; 35 36 static LIST_HEAD(, ftl_deferred_init) g_deferred_init = LIST_HEAD_INITIALIZER(g_deferred_init); 37 38 static int bdev_ftl_initialize(void); 39 static void bdev_ftl_finish(void); 40 static void bdev_ftl_examine(struct spdk_bdev *bdev); 41 42 static int 43 bdev_ftl_get_ctx_size(void) 44 { 45 return spdk_ftl_io_size(); 46 } 47 48 static struct spdk_bdev_module g_ftl_if = { 49 .name = "ftl", 50 .module_init = bdev_ftl_initialize, 51 .module_fini = bdev_ftl_finish, 52 .examine_disk = bdev_ftl_examine, 53 .get_ctx_size = bdev_ftl_get_ctx_size, 54 }; 55 56 SPDK_BDEV_MODULE_REGISTER(ftl, &g_ftl_if) 57 58 static void 59 bdev_ftl_free(struct ftl_bdev *ftl_bdev) 60 { 61 spdk_bdev_close(ftl_bdev->base_bdev_desc); 62 spdk_bdev_close(ftl_bdev->cache_bdev_desc); 63 free(ftl_bdev->bdev.name); 64 free(ftl_bdev); 65 } 66 67 static void 68 bdev_ftl_dev_free_cb(void *ctx, int status) 69 { 70 struct ftl_bdev *ftl_bdev = ctx; 71 72 spdk_bdev_destruct_done(&ftl_bdev->bdev, status); 73 bdev_ftl_free(ftl_bdev); 74 } 75 76 static int 77 bdev_ftl_destruct(void *ctx) 78 { 79 struct ftl_bdev *ftl_bdev = ctx; 80 81 spdk_ftl_dev_free(ftl_bdev->dev, bdev_ftl_dev_free_cb, ftl_bdev); 82 83 /* return 1 to indicate that the destruction is asynchronous */ 84 return 1; 85 } 86 87 static void 88 bdev_ftl_cb(void *arg, int rc) 89 { 90 struct spdk_bdev_io *bdev_io = arg; 91 enum spdk_bdev_io_status status; 92 93 switch (rc) { 94 case 0: 95 status = SPDK_BDEV_IO_STATUS_SUCCESS; 96 break; 97 case -EAGAIN: 98 case -ENOMEM: 99 status = SPDK_BDEV_IO_STATUS_NOMEM; 100 break; 101 default: 102 status = SPDK_BDEV_IO_STATUS_FAILED; 103 break; 104 } 105 106 spdk_bdev_io_complete(bdev_io, status); 107 } 108 109 static void 110 bdev_ftl_get_buf_cb(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io, 111 bool success) 112 { 113 struct ftl_bdev *ftl_bdev; 114 int rc; 115 116 ftl_bdev = bdev_io->bdev->ctxt; 117 118 if (!success) { 119 spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED); 120 return; 121 } 122 123 rc = spdk_ftl_readv(ftl_bdev->dev, (struct ftl_io *)bdev_io->driver_ctx, 124 ch, 125 bdev_io->u.bdev.offset_blocks, 126 bdev_io->u.bdev.num_blocks, 127 bdev_io->u.bdev.iovs, bdev_io->u.bdev.iovcnt, bdev_ftl_cb, bdev_io); 128 129 if (spdk_unlikely(rc != 0)) { 130 bdev_ftl_cb(bdev_io, rc); 131 } 132 } 133 134 static int 135 _bdev_ftl_submit_request(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io) 136 { 137 struct ftl_bdev *ftl_bdev = (struct ftl_bdev *)bdev_io->bdev->ctxt; 138 139 switch (bdev_io->type) { 140 case SPDK_BDEV_IO_TYPE_READ: 141 spdk_bdev_io_get_buf(bdev_io, bdev_ftl_get_buf_cb, 142 bdev_io->u.bdev.num_blocks * bdev_io->bdev->blocklen); 143 return 0; 144 145 case SPDK_BDEV_IO_TYPE_WRITE: 146 return spdk_ftl_writev(ftl_bdev->dev, (struct ftl_io *)bdev_io->driver_ctx, 147 ch, bdev_io->u.bdev.offset_blocks, 148 bdev_io->u.bdev.num_blocks, bdev_io->u.bdev.iovs, 149 bdev_io->u.bdev.iovcnt, bdev_ftl_cb, bdev_io); 150 151 case SPDK_BDEV_IO_TYPE_UNMAP: 152 return spdk_ftl_unmap(ftl_bdev->dev, (struct ftl_io *)bdev_io->driver_ctx, 153 ch, bdev_io->u.bdev.offset_blocks, 154 bdev_io->u.bdev.num_blocks, bdev_ftl_cb, bdev_io); 155 case SPDK_BDEV_IO_TYPE_FLUSH: 156 spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_SUCCESS); 157 return 0; 158 default: 159 return -ENOTSUP; 160 } 161 } 162 163 static void 164 bdev_ftl_submit_request(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io) 165 { 166 int rc = _bdev_ftl_submit_request(ch, bdev_io); 167 168 if (spdk_unlikely(rc != 0)) { 169 bdev_ftl_cb(bdev_io, rc); 170 } 171 } 172 173 static bool 174 bdev_ftl_io_type_supported(void *ctx, enum spdk_bdev_io_type io_type) 175 { 176 switch (io_type) { 177 case SPDK_BDEV_IO_TYPE_READ: 178 case SPDK_BDEV_IO_TYPE_WRITE: 179 case SPDK_BDEV_IO_TYPE_FLUSH: 180 case SPDK_BDEV_IO_TYPE_UNMAP: 181 return true; 182 default: 183 return false; 184 } 185 } 186 187 static struct spdk_io_channel * 188 bdev_ftl_get_io_channel(void *ctx) 189 { 190 struct ftl_bdev *ftl_bdev = ctx; 191 192 return spdk_ftl_get_io_channel(ftl_bdev->dev); 193 } 194 195 static void 196 bdev_ftl_write_config_json(struct spdk_bdev *bdev, struct spdk_json_write_ctx *w) 197 { 198 struct ftl_bdev *ftl_bdev = bdev->ctxt; 199 struct spdk_ftl_conf conf; 200 char uuid[SPDK_UUID_STRING_LEN]; 201 202 spdk_ftl_dev_get_conf(ftl_bdev->dev, &conf, sizeof(conf)); 203 204 spdk_json_write_object_begin(w); 205 206 spdk_json_write_named_string(w, "method", "bdev_ftl_create"); 207 208 spdk_json_write_named_object_begin(w, "params"); 209 spdk_json_write_named_string(w, "name", ftl_bdev->bdev.name); 210 211 spdk_json_write_named_uint64(w, "overprovisioning", conf.overprovisioning); 212 spdk_json_write_named_uint64(w, "l2p_dram_limit", conf.l2p_dram_limit); 213 214 if (conf.core_mask) { 215 spdk_json_write_named_string(w, "core_mask", conf.core_mask); 216 } 217 218 spdk_uuid_fmt_lower(uuid, sizeof(uuid), &conf.uuid); 219 spdk_json_write_named_string(w, "uuid", uuid); 220 221 spdk_json_write_named_bool(w, "fast_shutdown", conf.fast_shutdown); 222 223 spdk_json_write_named_string(w, "base_bdev", conf.base_bdev); 224 225 if (conf.cache_bdev) { 226 spdk_json_write_named_string(w, "cache", conf.cache_bdev); 227 } 228 229 spdk_json_write_object_end(w); 230 spdk_json_write_object_end(w); 231 } 232 233 static int 234 bdev_ftl_dump_info_json(void *ctx, struct spdk_json_write_ctx *w) 235 { 236 struct ftl_bdev *ftl_bdev = ctx; 237 struct spdk_ftl_attrs attrs; 238 struct spdk_ftl_conf conf; 239 240 spdk_ftl_dev_get_attrs(ftl_bdev->dev, &attrs, sizeof(attrs)); 241 spdk_ftl_dev_get_conf(ftl_bdev->dev, &conf, sizeof(conf)); 242 243 spdk_json_write_named_object_begin(w, "ftl"); 244 245 spdk_json_write_named_string(w, "base_bdev", conf.base_bdev); 246 247 if (conf.cache_bdev) { 248 spdk_json_write_named_string(w, "cache", conf.cache_bdev); 249 } 250 251 /* ftl */ 252 spdk_json_write_object_end(w); 253 254 return 0; 255 } 256 257 static const struct spdk_bdev_fn_table ftl_fn_table = { 258 .destruct = bdev_ftl_destruct, 259 .submit_request = bdev_ftl_submit_request, 260 .io_type_supported = bdev_ftl_io_type_supported, 261 .get_io_channel = bdev_ftl_get_io_channel, 262 .write_config_json = bdev_ftl_write_config_json, 263 .dump_info_json = bdev_ftl_dump_info_json, 264 }; 265 266 static void 267 bdev_ftl_create_err_complete(struct ftl_bdev *ftl_bdev) 268 { 269 ftl_bdev_init_fn init_cb = ftl_bdev->init_cb; 270 void *init_arg = ftl_bdev->init_arg; 271 int rc = ftl_bdev->rc; 272 273 bdev_ftl_free(ftl_bdev); 274 275 assert(rc); 276 init_cb(NULL, init_arg, rc); 277 } 278 279 static void 280 bdev_ftl_create_err_cleanup_cb(void *ctx, int status) 281 { 282 struct ftl_bdev *ftl_bdev = ctx; 283 284 if (status) { 285 SPDK_ERRLOG("Fatal ERROR of FTL cleanup, name %s\n", ftl_bdev->bdev.name); 286 } 287 288 bdev_ftl_create_err_complete(ftl_bdev); 289 } 290 291 static void 292 bdev_ftl_create_cb(struct spdk_ftl_dev *dev, void *ctx, int status) 293 { 294 struct ftl_bdev *ftl_bdev = ctx; 295 struct ftl_bdev_info info = {}; 296 struct spdk_ftl_attrs attrs; 297 struct spdk_ftl_conf conf; 298 ftl_bdev_init_fn init_cb = ftl_bdev->init_cb; 299 void *init_arg = ftl_bdev->init_arg; 300 301 if (status) { 302 SPDK_ERRLOG("Failed to create FTL device (%d)\n", status); 303 ftl_bdev->rc = status; 304 goto error; 305 } 306 307 spdk_ftl_dev_get_attrs(dev, &attrs, sizeof(attrs)); 308 spdk_ftl_dev_get_conf(dev, &conf, sizeof(conf)); 309 310 ftl_bdev->dev = dev; 311 ftl_bdev->bdev.product_name = "FTL disk"; 312 ftl_bdev->bdev.write_cache = 0; 313 ftl_bdev->bdev.blocklen = attrs.block_size; 314 ftl_bdev->bdev.blockcnt = attrs.num_blocks; 315 ftl_bdev->bdev.uuid = conf.uuid; 316 ftl_bdev->bdev.optimal_io_boundary = attrs.optimum_io_size; 317 ftl_bdev->bdev.split_on_optimal_io_boundary = true; 318 319 SPDK_DEBUGLOG(bdev_ftl, "Creating bdev %s:\n", ftl_bdev->bdev.name); 320 SPDK_DEBUGLOG(bdev_ftl, "\tblock_len:\t%zu\n", attrs.block_size); 321 SPDK_DEBUGLOG(bdev_ftl, "\tnum_blocks:\t%"PRIu64"\n", attrs.num_blocks); 322 323 ftl_bdev->bdev.ctxt = ftl_bdev; 324 ftl_bdev->bdev.fn_table = &ftl_fn_table; 325 ftl_bdev->bdev.module = &g_ftl_if; 326 327 status = spdk_bdev_register(&ftl_bdev->bdev); 328 if (status) { 329 ftl_bdev->rc = status; 330 goto error; 331 } 332 333 info.name = ftl_bdev->bdev.name; 334 info.uuid = ftl_bdev->bdev.uuid; 335 336 init_cb(&info, init_arg, 0); 337 return; 338 339 error: 340 if (ftl_bdev->dev) { 341 /* Cleanup all FTL */ 342 spdk_ftl_dev_set_fast_shutdown(ftl_bdev->dev, false); 343 344 /* FTL was created, but we have got an error, so we need to delete it */ 345 spdk_ftl_dev_free(dev, bdev_ftl_create_err_cleanup_cb, ftl_bdev); 346 } else { 347 bdev_ftl_create_err_complete(ftl_bdev); 348 } 349 } 350 351 static void 352 bdev_ftl_defer_free(struct ftl_deferred_init *init) 353 { 354 spdk_ftl_conf_deinit(&init->conf); 355 free(init); 356 } 357 358 int 359 bdev_ftl_defer_init(const struct spdk_ftl_conf *conf) 360 { 361 struct ftl_deferred_init *init; 362 int rc; 363 364 init = calloc(1, sizeof(*init)); 365 if (!init) { 366 return -ENOMEM; 367 } 368 369 rc = spdk_ftl_conf_copy(&init->conf, conf); 370 if (rc) { 371 free(init); 372 return -ENOMEM; 373 } 374 375 LIST_INSERT_HEAD(&g_deferred_init, init, entry); 376 377 return 0; 378 } 379 380 static void 381 bdev_ftl_create_bdev_event_cb(enum spdk_bdev_event_type type, struct spdk_bdev *bdev, void *ctx) 382 { 383 } 384 385 int 386 bdev_ftl_create_bdev(const struct spdk_ftl_conf *conf, ftl_bdev_init_fn cb, void *cb_arg) 387 { 388 struct ftl_bdev *ftl_bdev; 389 struct spdk_bdev_desc *base_bdev_desc, *cache_bdev_desc; 390 int rc; 391 392 rc = spdk_bdev_open_ext(conf->base_bdev, false, bdev_ftl_create_bdev_event_cb, NULL, 393 &base_bdev_desc); 394 if (rc) { 395 return rc; 396 } 397 rc = spdk_bdev_open_ext(conf->cache_bdev, false, bdev_ftl_create_bdev_event_cb, NULL, 398 &cache_bdev_desc); 399 if (rc) { 400 spdk_bdev_close(base_bdev_desc); 401 return rc; 402 } 403 404 ftl_bdev = calloc(1, sizeof(*ftl_bdev)); 405 if (!ftl_bdev) { 406 SPDK_ERRLOG("Could not allocate ftl_bdev\n"); 407 spdk_bdev_close(base_bdev_desc); 408 spdk_bdev_close(cache_bdev_desc); 409 return -ENOMEM; 410 } 411 412 ftl_bdev->base_bdev_desc = base_bdev_desc; 413 ftl_bdev->cache_bdev_desc = cache_bdev_desc; 414 415 ftl_bdev->bdev.name = strdup(conf->name); 416 if (!ftl_bdev->bdev.name) { 417 rc = -ENOMEM; 418 goto error; 419 } 420 421 ftl_bdev->init_cb = cb; 422 ftl_bdev->init_arg = cb_arg; 423 424 rc = spdk_ftl_dev_init(conf, bdev_ftl_create_cb, ftl_bdev); 425 if (rc) { 426 SPDK_ERRLOG("Could not create FTL device\n"); 427 goto error; 428 } 429 430 return 0; 431 432 error: 433 bdev_ftl_free(ftl_bdev); 434 return rc; 435 } 436 437 static int 438 bdev_ftl_initialize(void) 439 { 440 return spdk_ftl_init(); 441 } 442 443 static void 444 bdev_ftl_event_cb(enum spdk_bdev_event_type type, struct spdk_bdev *bdev, void *ctx) 445 { 446 } 447 448 void 449 bdev_ftl_delete_bdev(const char *name, bool fast_shutdown, spdk_bdev_unregister_cb cb_fn, 450 void *cb_arg) 451 { 452 struct spdk_bdev_desc *ftl_bdev_desc; 453 struct spdk_bdev *bdev; 454 struct ftl_bdev *ftl; 455 int rc; 456 457 rc = spdk_bdev_open_ext(name, false, bdev_ftl_event_cb, NULL, &ftl_bdev_desc); 458 459 if (rc) { 460 goto not_found; 461 } 462 463 bdev = spdk_bdev_desc_get_bdev(ftl_bdev_desc); 464 465 if (bdev->module != &g_ftl_if) { 466 goto bdev_opened; 467 } 468 469 ftl = bdev->ctxt; 470 assert(ftl); 471 spdk_ftl_dev_set_fast_shutdown(ftl->dev, fast_shutdown); 472 spdk_bdev_close(ftl_bdev_desc); 473 474 rc = spdk_bdev_unregister_by_name(name, &g_ftl_if, cb_fn, cb_arg); 475 if (rc) { 476 cb_fn(cb_arg, rc); 477 } 478 479 return; 480 bdev_opened: 481 spdk_bdev_close(ftl_bdev_desc); 482 not_found: 483 cb_fn(cb_arg, -ENODEV); 484 } 485 486 struct ftl_unmap_ctx { 487 struct spdk_bdev_desc *bdev; 488 spdk_ftl_fn cb_fn; 489 void *cb_arg; 490 }; 491 492 static void 493 bdev_ftl_unmap_cb(void *cb_arg, int status) 494 { 495 struct ftl_unmap_ctx *ctx = cb_arg; 496 497 spdk_bdev_close(ctx->bdev); 498 ctx->cb_fn(ctx->cb_arg, status); 499 free(ctx); 500 } 501 502 void 503 bdev_ftl_unmap(const char *name, uint64_t lba, uint64_t num_blocks, spdk_ftl_fn cb_fn, void *cb_arg) 504 { 505 struct spdk_bdev_desc *ftl_bdev_desc; 506 struct spdk_bdev *bdev; 507 struct ftl_bdev *ftl; 508 struct ftl_unmap_ctx *ctx; 509 int rc; 510 511 rc = spdk_bdev_open_ext(name, false, bdev_ftl_event_cb, NULL, &ftl_bdev_desc); 512 513 if (rc) { 514 goto not_found; 515 } 516 517 bdev = spdk_bdev_desc_get_bdev(ftl_bdev_desc); 518 519 if (bdev->module != &g_ftl_if) { 520 rc = -ENODEV; 521 goto bdev_opened; 522 } 523 524 ctx = calloc(1, sizeof(struct ftl_unmap_ctx)); 525 if (!ctx) { 526 rc = -ENOMEM; 527 goto bdev_opened; 528 } 529 530 ctx->bdev = ftl_bdev_desc; 531 ctx->cb_arg = cb_arg; 532 ctx->cb_fn = cb_fn; 533 534 ftl = bdev->ctxt; 535 assert(ftl); 536 /* It's ok to pass NULL as IO channel - FTL will detect this and use it's internal IO channel for management operations */ 537 rc = spdk_ftl_unmap(ftl->dev, NULL, NULL, lba, num_blocks, bdev_ftl_unmap_cb, ctx); 538 539 if (rc) { 540 goto ctx_allocated; 541 } 542 543 return; 544 ctx_allocated: 545 free(ctx); 546 bdev_opened: 547 spdk_bdev_close(ftl_bdev_desc); 548 not_found: 549 cb_fn(cb_arg, rc); 550 } 551 552 static void 553 bdev_ftl_get_stats_cb(struct ftl_stats *stats, void *ctx) 554 { 555 struct rpc_ftl_stats_ctx *ftl_stats_ctx = ctx; 556 557 ftl_stats_ctx->cb(ftl_stats_ctx); 558 559 spdk_bdev_close(ftl_stats_ctx->ftl_bdev_desc); 560 free(ftl_stats_ctx); 561 } 562 563 564 int 565 bdev_ftl_get_stats(const char *name, ftl_bdev_thread_fn cb, struct spdk_jsonrpc_request *request, 566 struct ftl_stats *stats) 567 { 568 struct spdk_bdev_desc *ftl_bdev_desc; 569 struct spdk_bdev *bdev; 570 struct ftl_bdev *ftl; 571 struct rpc_ftl_stats_ctx *ftl_stats_ctx; 572 int rc; 573 574 rc = spdk_bdev_open_ext(name, false, bdev_ftl_event_cb, NULL, &ftl_bdev_desc); 575 if (rc) { 576 goto not_found; 577 } 578 579 bdev = spdk_bdev_desc_get_bdev(ftl_bdev_desc); 580 if (bdev->module != &g_ftl_if) { 581 rc = -ENODEV; 582 goto bdev_opened; 583 } 584 585 ftl_stats_ctx = calloc(1, sizeof(*ftl_stats_ctx)); 586 if (!ftl_stats_ctx) { 587 SPDK_ERRLOG("Could not allocate ftl_stats_ctx\n"); 588 rc = -ENOMEM; 589 goto bdev_opened; 590 } 591 592 ftl = bdev->ctxt; 593 ftl_stats_ctx->request = request; 594 ftl_stats_ctx->ftl_bdev_desc = ftl_bdev_desc; 595 ftl_stats_ctx->cb = cb; 596 ftl_stats_ctx->ftl_stats = stats; 597 598 rc = spdk_ftl_get_stats(ftl->dev, stats, bdev_ftl_get_stats_cb, ftl_stats_ctx); 599 if (rc) { 600 goto stats_allocated; 601 } 602 603 return 0; 604 stats_allocated: 605 free(ftl_stats_ctx); 606 bdev_opened: 607 spdk_bdev_close(ftl_bdev_desc); 608 not_found: 609 return rc; 610 } 611 612 static void 613 bdev_ftl_finish(void) 614 { 615 spdk_ftl_fini(); 616 } 617 618 static void 619 bdev_ftl_create_defered_cb(const struct ftl_bdev_info *info, void *ctx, int status) 620 { 621 struct ftl_deferred_init *opts = ctx; 622 623 if (status) { 624 SPDK_ERRLOG("Failed to initialize FTL bdev '%s'\n", opts->conf.name); 625 } 626 627 bdev_ftl_defer_free(opts); 628 629 spdk_bdev_module_examine_done(&g_ftl_if); 630 } 631 632 static void 633 bdev_ftl_examine(struct spdk_bdev *bdev) 634 { 635 struct ftl_deferred_init *opts; 636 int rc; 637 638 LIST_FOREACH(opts, &g_deferred_init, entry) { 639 /* spdk_bdev_module_examine_done will be called by bdev_ftl_create_defered_cb */ 640 rc = bdev_ftl_create_bdev(&opts->conf, bdev_ftl_create_defered_cb, opts); 641 if (rc == -ENODEV) { 642 continue; 643 } 644 645 LIST_REMOVE(opts, entry); 646 647 if (rc) { 648 bdev_ftl_create_defered_cb(NULL, opts, rc); 649 } 650 return; 651 } 652 653 spdk_bdev_module_examine_done(&g_ftl_if); 654 } 655 656 SPDK_LOG_REGISTER_COMPONENT(bdev_ftl) 657