1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2023 Solidigm All Rights Reserved 3 * Copyright (C) 2022 Intel Corporation. 4 * All rights reserved. 5 */ 6 7 #include "spdk/env.h" 8 #include "spdk/bdev_module.h" 9 10 #include "ftl_core.h" 11 #include "ftl_md.h" 12 #include "ftl_nv_cache_io.h" 13 14 struct ftl_md; 15 static void io_submit(struct ftl_md *md); 16 static void io_done(struct ftl_md *md); 17 18 static bool 19 has_mirror(struct ftl_md *md) 20 { 21 if (md->region) { 22 if (md->region->mirror_type != FTL_LAYOUT_REGION_TYPE_INVALID) { 23 return md->mirror_enabled; 24 } 25 } 26 27 return false; 28 } 29 30 static struct ftl_md * 31 ftl_md_get_mirror(struct ftl_md *md) 32 { 33 if (has_mirror(md)) { 34 return md->dev->layout.md[md->region->mirror_type]; 35 } 36 37 return NULL; 38 } 39 40 uint64_t 41 ftl_md_xfer_blocks(struct spdk_ftl_dev *dev) 42 { 43 return 4ULL * dev->xfer_size; 44 } 45 46 static uint64_t 47 xfer_size(struct ftl_md *md) 48 { 49 return ftl_md_xfer_blocks(md->dev) * FTL_BLOCK_SIZE; 50 } 51 52 static void 53 ftl_md_create_heap(struct ftl_md *md, uint64_t vss_blksz) 54 { 55 md->shm_fd = -1; 56 md->vss_data = NULL; 57 md->data = calloc(md->data_blocks, FTL_BLOCK_SIZE + vss_blksz); 58 59 if (md->data && vss_blksz) { 60 md->vss_data = ((char *)md->data) + md->data_blocks * FTL_BLOCK_SIZE; 61 } 62 } 63 64 static void 65 ftl_md_destroy_heap(struct ftl_md *md) 66 { 67 if (md->data) { 68 free(md->data); 69 md->data = NULL; 70 md->vss_data = NULL; 71 } 72 } 73 74 static int 75 ftl_wrapper_open(const char *name, int of, mode_t m) 76 { 77 return open(name, of, m); 78 } 79 80 static void 81 ftl_md_setup_obj(struct ftl_md *md, int flags, 82 const char *name) 83 { 84 char uuid_str[SPDK_UUID_STRING_LEN]; 85 const char *fmt; 86 87 if (!(flags & FTL_MD_CREATE_SHM)) { 88 assert(false); 89 return; 90 } 91 92 /* TODO: temporary, define a proper hugetlbfs mountpoint */ 93 fmt = "/dev/hugepages/ftl_%s_%s"; 94 md->shm_mmap_flags = MAP_SHARED; 95 md->shm_open = ftl_wrapper_open; 96 md->shm_unlink = unlink; 97 98 if (name == NULL || 99 spdk_uuid_fmt_lower(uuid_str, SPDK_UUID_STRING_LEN, &md->dev->conf.uuid) || 100 snprintf(md->name, sizeof(md->name) / sizeof(md->name[0]), 101 fmt, uuid_str, name) <= 0) { 102 md->name[0] = 0; 103 } 104 } 105 106 static void 107 ftl_md_invalidate_shm(struct ftl_md *md) 108 { 109 if (md->dev->sb_shm && md->dev->sb_shm->shm_ready) { 110 md->dev->init_retry = true; 111 md->dev->sb_shm->shm_ready = false; 112 } 113 } 114 115 static void 116 ftl_md_create_shm(struct ftl_md *md, uint64_t vss_blksz, int flags) 117 { 118 struct stat shm_stat; 119 size_t vss_blk_offs; 120 void *shm_ptr; 121 int open_flags = O_RDWR; 122 mode_t open_mode = S_IRUSR | S_IWUSR; 123 124 assert(md->shm_open && md->shm_unlink); 125 md->data = NULL; 126 md->vss_data = NULL; 127 md->shm_sz = 0; 128 129 /* Must have an object name */ 130 if (md->name[0] == 0) { 131 assert(false); 132 return; 133 } 134 135 /* If specified, unlink before create a new SHM object */ 136 if (flags & FTL_MD_CREATE_SHM_NEW) { 137 if (md->shm_unlink(md->name) < 0 && errno != ENOENT) { 138 ftl_md_invalidate_shm(md); 139 return; 140 } 141 open_flags += O_CREAT | O_TRUNC; 142 } 143 144 /* Open existing or create a new SHM object, then query its props */ 145 md->shm_fd = md->shm_open(md->name, open_flags, open_mode); 146 if (md->shm_fd < 0 || fstat(md->shm_fd, &shm_stat) < 0) { 147 goto err_shm; 148 } 149 150 /* Verify open mode hasn't changed */ 151 if ((shm_stat.st_mode & (S_IRWXU | S_IRWXG | S_IRWXO)) != open_mode) { 152 goto err_shm; 153 } 154 155 /* Round up the SHM obj size to the nearest blk size (i.e. page size) */ 156 md->shm_sz = spdk_divide_round_up(md->data_blocks * FTL_BLOCK_SIZE, shm_stat.st_blksize); 157 158 /* Add some blks for VSS metadata */ 159 vss_blk_offs = md->shm_sz; 160 161 if (vss_blksz) { 162 md->shm_sz += spdk_divide_round_up(md->data_blocks * vss_blksz, 163 shm_stat.st_blksize); 164 } 165 166 /* Total SHM obj size */ 167 md->shm_sz *= shm_stat.st_blksize; 168 169 /* Set or check the object size - zero init`d in case of set (FTL_MD_CREATE_SHM_NEW) */ 170 if ((shm_stat.st_size == 0 && (ftruncate(md->shm_fd, md->shm_sz) < 0 || 171 (flags & FTL_MD_CREATE_SHM_NEW) == 0)) 172 || (shm_stat.st_size > 0 && (size_t)shm_stat.st_size != md->shm_sz)) { 173 goto err_shm; 174 } 175 176 /* Create a virtual memory mapping for the object */ 177 shm_ptr = mmap(NULL, md->shm_sz, PROT_READ | PROT_WRITE, md->shm_mmap_flags, 178 md->shm_fd, 0); 179 if (shm_ptr == MAP_FAILED) { 180 goto err_shm; 181 } 182 183 md->data = shm_ptr; 184 if (vss_blksz) { 185 md->vss_data = ((char *)shm_ptr) + vss_blk_offs * shm_stat.st_blksize; 186 } 187 188 /* Lock the pages in memory (i.e. prevent the pages to be paged out) */ 189 if (mlock(md->data, md->shm_sz) < 0) { 190 goto err_map; 191 } 192 193 if (spdk_mem_register(md->data, md->shm_sz)) { 194 goto err_mlock; 195 } 196 md->mem_reg = true; 197 198 return; 199 200 /* Cleanup upon fault */ 201 err_mlock: 202 munlock(md->data, md->shm_sz); 203 204 err_map: 205 munmap(md->data, md->shm_sz); 206 md->data = NULL; 207 md->vss_data = NULL; 208 md->shm_sz = 0; 209 210 err_shm: 211 if (md->shm_fd >= 0) { 212 close(md->shm_fd); 213 md->shm_unlink(md->name); 214 md->shm_fd = -1; 215 } 216 ftl_md_invalidate_shm(md); 217 } 218 219 static void 220 ftl_md_destroy_shm(struct ftl_md *md, int flags) 221 { 222 if (!md->data) { 223 return; 224 } 225 226 assert(md->shm_sz > 0); 227 if (md->mem_reg) { 228 spdk_mem_unregister(md->data, md->shm_sz); 229 md->mem_reg = false; 230 } 231 232 /* Unlock the pages in memory */ 233 munlock(md->data, md->shm_sz); 234 235 /* Remove the virtual memory mapping for the object */ 236 munmap(md->data, md->shm_sz); 237 238 /* Close SHM object fd */ 239 close(md->shm_fd); 240 241 md->data = NULL; 242 md->vss_data = NULL; 243 244 /* If specified, keep the object in SHM */ 245 if (flags & FTL_MD_DESTROY_SHM_KEEP) { 246 return; 247 } 248 249 /* Otherwise destroy/unlink the object */ 250 assert(md->name[0] != 0 && md->shm_unlink != NULL); 251 md->shm_unlink(md->name); 252 } 253 254 struct ftl_md *ftl_md_create(struct spdk_ftl_dev *dev, uint64_t blocks, 255 uint64_t vss_blksz, const char *name, int flags, 256 const struct ftl_layout_region *region) 257 { 258 struct ftl_md *md; 259 260 md = calloc(1, sizeof(*md)); 261 if (!md) { 262 return NULL; 263 } 264 md->dev = dev; 265 md->data_blocks = blocks; 266 md->mirror_enabled = true; 267 268 if (flags != FTL_MD_CREATE_NO_MEM) { 269 if (flags & FTL_MD_CREATE_SHM) { 270 ftl_md_setup_obj(md, flags, name); 271 ftl_md_create_shm(md, vss_blksz, flags); 272 } else { 273 assert((flags & FTL_MD_CREATE_HEAP) == FTL_MD_CREATE_HEAP); 274 ftl_md_create_heap(md, vss_blksz); 275 } 276 277 if (!md->data) { 278 free(md); 279 return NULL; 280 } 281 } 282 283 if (region) { 284 size_t entry_vss_buf_size = vss_blksz * region->entry_size; 285 286 if (entry_vss_buf_size) { 287 md->entry_vss_dma_buf = spdk_malloc(entry_vss_buf_size, FTL_BLOCK_SIZE, 288 NULL, SPDK_ENV_LCORE_ID_ANY, 289 SPDK_MALLOC_DMA); 290 if (!md->entry_vss_dma_buf) { 291 goto err; 292 } 293 } 294 295 ftl_md_set_region(md, region); 296 } 297 298 return md; 299 err: 300 ftl_md_destroy(md, ftl_md_destroy_region_flags(dev, region->type)); 301 return NULL; 302 } 303 304 int 305 ftl_md_unlink(struct spdk_ftl_dev *dev, const char *name, int flags) 306 { 307 struct ftl_md md = { 0 }; 308 309 if (0 == (flags & FTL_MD_CREATE_SHM)) { 310 /* Unlink can be called for shared memory only */ 311 return -EINVAL; 312 } 313 314 md.dev = dev; 315 ftl_md_setup_obj(&md, flags, name); 316 317 return md.shm_unlink(md.name); 318 } 319 320 void 321 ftl_md_destroy(struct ftl_md *md, int flags) 322 { 323 if (!md) { 324 return; 325 } 326 327 if (!md->is_mirror) { 328 ftl_md_free_buf(md, flags); 329 spdk_free(md->entry_vss_dma_buf); 330 } 331 free(md); 332 } 333 334 void 335 ftl_md_free_buf(struct ftl_md *md, int flags) 336 { 337 if (!md) { 338 return; 339 } 340 341 if (md->shm_fd < 0) { 342 assert(flags == 0); 343 ftl_md_destroy_heap(md); 344 } else { 345 ftl_md_destroy_shm(md, flags); 346 } 347 } 348 349 void * 350 ftl_md_get_buffer(struct ftl_md *md) 351 { 352 return md->data; 353 } 354 355 uint64_t 356 ftl_md_get_buffer_size(struct ftl_md *md) 357 { 358 return md->data_blocks * FTL_BLOCK_SIZE; 359 } 360 361 static void 362 ftl_md_vss_buf_init(union ftl_md_vss *buf, uint32_t count, 363 const union ftl_md_vss *vss_pattern) 364 { 365 while (count) { 366 count--; 367 buf[count] = *vss_pattern; 368 } 369 } 370 371 union ftl_md_vss *ftl_md_vss_buf_alloc(struct ftl_layout_region *region, uint32_t count) 372 { 373 union ftl_md_vss *buf = spdk_zmalloc(count * FTL_MD_VSS_SZ, FTL_BLOCK_SIZE, NULL, 374 SPDK_ENV_LCORE_ID_ANY, 375 SPDK_MALLOC_DMA); 376 377 if (!buf) { 378 return NULL; 379 } 380 381 union ftl_md_vss vss_buf = {0}; 382 vss_buf.version.md_version = region->current.version; 383 ftl_md_vss_buf_init(buf, count, &vss_buf); 384 return buf; 385 } 386 387 union ftl_md_vss *ftl_md_get_vss_buffer(struct ftl_md *md) 388 { 389 return md->vss_data; 390 } 391 392 static void 393 io_cleanup(struct ftl_md *md) 394 { 395 spdk_dma_free(md->io.data); 396 md->io.data = NULL; 397 398 spdk_dma_free(md->io.md); 399 md->io.md = NULL; 400 } 401 402 static void 403 exception(void *arg) 404 { 405 struct ftl_md *md = arg; 406 407 md->cb(md->dev, md, -EINVAL); 408 io_cleanup(md); 409 } 410 411 static inline enum ftl_stats_type 412 get_bdev_io_ftl_stats_type(struct spdk_ftl_dev *dev, struct spdk_bdev_io *bdev_io) { 413 struct spdk_bdev *nvc = spdk_bdev_desc_get_bdev(dev->nv_cache.bdev_desc); 414 415 if (bdev_io->bdev == nvc) 416 { 417 return FTL_STATS_TYPE_MD_NV_CACHE; 418 } else 419 { 420 return FTL_STATS_TYPE_MD_BASE; 421 } 422 } 423 424 static void 425 audit_md_vss_version(struct ftl_md *md, uint64_t blocks) 426 { 427 #if defined(DEBUG) 428 union ftl_md_vss *vss = md->io.md; 429 while (blocks) { 430 blocks--; 431 assert(vss[blocks].version.md_version == md->region->current.version); 432 } 433 #endif 434 } 435 436 static void 437 read_write_blocks_cb(struct spdk_bdev_io *bdev_io, bool success, void *arg) 438 { 439 struct ftl_md *md = arg; 440 441 ftl_stats_bdev_io_completed(md->dev, get_bdev_io_ftl_stats_type(md->dev, bdev_io), bdev_io); 442 443 if (spdk_unlikely(!success)) { 444 if (md->io.op == FTL_MD_OP_RESTORE && has_mirror(md)) { 445 md->io.status = -EAGAIN; 446 } else { 447 md->io.status = -EIO; 448 } 449 } else { 450 uint64_t blocks = bdev_io->u.bdev.num_blocks; 451 uint64_t size = blocks * FTL_BLOCK_SIZE; 452 453 if (md->io.op == FTL_MD_OP_RESTORE) { 454 memcpy(md->data + md->io.data_offset, md->io.data, size); 455 456 if (md->vss_data) { 457 uint64_t vss_offset = md->io.data_offset / FTL_BLOCK_SIZE; 458 vss_offset *= FTL_MD_VSS_SZ; 459 audit_md_vss_version(md, blocks); 460 memcpy(md->vss_data + vss_offset, md->io.md, blocks * FTL_MD_VSS_SZ); 461 } 462 } 463 464 md->io.address += blocks; 465 md->io.remaining -= blocks; 466 md->io.data_offset += size; 467 } 468 469 spdk_bdev_free_io(bdev_io); 470 471 io_submit(md); 472 } 473 474 static inline int 475 read_blocks(struct spdk_ftl_dev *dev, struct spdk_bdev_desc *desc, 476 struct spdk_io_channel *ch, 477 void *buf, void *md_buf, 478 uint64_t offset_blocks, uint64_t num_blocks, 479 spdk_bdev_io_completion_cb cb, void *cb_arg) 480 { 481 if (desc == dev->nv_cache.bdev_desc) { 482 return ftl_nv_cache_bdev_read_blocks_with_md(dev, desc, ch, buf, md_buf, 483 offset_blocks, num_blocks, 484 cb, cb_arg); 485 } else if (md_buf) { 486 return spdk_bdev_read_blocks_with_md(desc, ch, buf, md_buf, 487 offset_blocks, num_blocks, 488 cb, cb_arg); 489 } else { 490 return spdk_bdev_read_blocks(desc, ch, buf, 491 offset_blocks, num_blocks, 492 cb, cb_arg); 493 } 494 } 495 496 static inline int 497 write_blocks(struct spdk_ftl_dev *dev, struct spdk_bdev_desc *desc, 498 struct spdk_io_channel *ch, 499 void *buf, void *md_buf, 500 uint64_t offset_blocks, uint64_t num_blocks, 501 spdk_bdev_io_completion_cb cb, void *cb_arg) 502 { 503 if (desc == dev->nv_cache.bdev_desc) { 504 return ftl_nv_cache_bdev_write_blocks_with_md(dev, desc, ch, buf, md_buf, 505 offset_blocks, num_blocks, 506 cb, cb_arg); 507 } else if (md_buf) { 508 return spdk_bdev_write_blocks_with_md(desc, ch, buf, md_buf, offset_blocks, 509 num_blocks, cb, cb_arg); 510 } else { 511 return spdk_bdev_write_blocks(desc, ch, buf, offset_blocks, num_blocks, cb, cb_arg); 512 } 513 } 514 515 static void 516 read_write_blocks(void *_md) 517 { 518 struct ftl_md *md = _md; 519 const struct ftl_layout_region *region = md->region; 520 uint64_t blocks; 521 int rc = 0; 522 523 blocks = spdk_min(md->io.remaining, ftl_md_xfer_blocks(md->dev)); 524 525 switch (md->io.op) { 526 case FTL_MD_OP_RESTORE: 527 rc = read_blocks(md->dev, region->bdev_desc, region->ioch, 528 md->io.data, md->io.md, 529 md->io.address, blocks, 530 read_write_blocks_cb, md); 531 break; 532 case FTL_MD_OP_PERSIST: 533 case FTL_MD_OP_CLEAR: 534 rc = write_blocks(md->dev, region->bdev_desc, region->ioch, 535 md->io.data, md->io.md, 536 md->io.address, blocks, 537 read_write_blocks_cb, md); 538 break; 539 default: 540 ftl_abort(); 541 } 542 543 if (spdk_unlikely(rc)) { 544 if (rc == -ENOMEM) { 545 struct spdk_bdev *bdev = spdk_bdev_desc_get_bdev(region->bdev_desc); 546 md->io.bdev_io_wait.bdev = bdev; 547 md->io.bdev_io_wait.cb_fn = read_write_blocks; 548 md->io.bdev_io_wait.cb_arg = md; 549 spdk_bdev_queue_io_wait(bdev, region->ioch, &md->io.bdev_io_wait); 550 } else { 551 ftl_abort(); 552 } 553 } 554 } 555 556 static void 557 io_submit(struct ftl_md *md) 558 { 559 if (!md->io.remaining || md->io.status) { 560 io_done(md); 561 return; 562 } 563 564 if (md->io.op == FTL_MD_OP_PERSIST) { 565 uint64_t blocks = spdk_min(md->io.remaining, ftl_md_xfer_blocks(md->dev)); 566 567 memcpy(md->io.data, md->data + md->io.data_offset, FTL_BLOCK_SIZE * blocks); 568 569 if (md->vss_data) { 570 uint64_t vss_offset = md->io.data_offset / FTL_BLOCK_SIZE; 571 vss_offset *= FTL_MD_VSS_SZ; 572 assert(md->io.md); 573 memcpy(md->io.md, md->vss_data + vss_offset, FTL_MD_VSS_SZ * blocks); 574 audit_md_vss_version(md, blocks); 575 } 576 } 577 #if defined(DEBUG) 578 if (md->io.md && md->io.op == FTL_MD_OP_CLEAR) { 579 uint64_t blocks = spdk_min(md->io.remaining, ftl_md_xfer_blocks(md->dev)); 580 audit_md_vss_version(md, blocks); 581 } 582 #endif 583 584 read_write_blocks(md); 585 } 586 587 static int 588 io_can_start(struct ftl_md *md) 589 { 590 assert(NULL == md->io.data); 591 if (NULL != md->io.data) { 592 /* Outgoing IO on metadata */ 593 return -EINVAL; 594 } 595 596 if (!md->region) { 597 /* No device region to process data */ 598 return -EINVAL; 599 } 600 601 if (md->region->current.blocks > md->data_blocks) { 602 /* No device region to process data */ 603 FTL_ERRLOG(md->dev, "Blocks number mismatch between metadata object and" 604 "device region\n"); 605 return -EINVAL; 606 } 607 608 return 0; 609 } 610 611 static int 612 io_prepare(struct ftl_md *md, enum ftl_md_ops op) 613 { 614 const struct ftl_layout_region *region = md->region; 615 uint64_t data_size, meta_size = 0; 616 617 /* Allocates buffer for IO */ 618 data_size = xfer_size(md); 619 md->io.data = spdk_zmalloc(data_size, FTL_BLOCK_SIZE, NULL, 620 SPDK_ENV_LCORE_ID_ANY, SPDK_MALLOC_DMA); 621 if (!md->io.data) { 622 return -ENOMEM; 623 } 624 625 if (md->vss_data || md->region->vss_blksz) { 626 meta_size = ftl_md_xfer_blocks(md->dev) * FTL_MD_VSS_SZ; 627 md->io.md = spdk_zmalloc(meta_size, FTL_BLOCK_SIZE, NULL, 628 SPDK_ENV_LCORE_ID_ANY, SPDK_MALLOC_DMA); 629 if (!md->io.md) { 630 spdk_dma_free(md->io.data); 631 md->io.data = NULL; 632 return -ENOMEM; 633 } 634 } 635 636 md->io.address = region->current.offset; 637 md->io.remaining = region->current.blocks; 638 md->io.data_offset = 0; 639 md->io.status = 0; 640 md->io.op = op; 641 642 return 0; 643 } 644 645 static int 646 io_init(struct ftl_md *md, enum ftl_md_ops op) 647 { 648 if (io_can_start(md)) { 649 return -EINVAL; 650 } 651 652 if (io_prepare(md, op)) { 653 return -ENOMEM; 654 } 655 656 return 0; 657 } 658 659 static uint64_t 660 persist_entry_lba(struct ftl_md *md, uint64_t start_entry) 661 { 662 return md->region->current.offset + start_entry * md->region->entry_size; 663 } 664 665 static void 666 persist_entry_cb(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) 667 { 668 struct ftl_md_io_entry_ctx *ctx = cb_arg; 669 struct ftl_md *md = ctx->md; 670 671 ftl_stats_bdev_io_completed(md->dev, get_bdev_io_ftl_stats_type(md->dev, bdev_io), bdev_io); 672 673 spdk_bdev_free_io(bdev_io); 674 675 assert(ctx->remaining > 0); 676 ctx->remaining--; 677 678 if (!success) { 679 ctx->status = -EIO; 680 } 681 682 if (!ctx->remaining) { 683 ctx->cb(ctx->status, ctx->cb_arg); 684 } 685 } 686 687 static int 688 ftl_md_persist_entry_write_blocks(struct ftl_md_io_entry_ctx *ctx, struct ftl_md *md, 689 spdk_bdev_io_wait_cb retry_fn) 690 { 691 int rc; 692 693 rc = write_blocks(md->dev, md->region->bdev_desc, md->region->ioch, 694 ctx->buffer, ctx->vss_buffer, 695 persist_entry_lba(md, ctx->start_entry), md->region->entry_size, 696 persist_entry_cb, ctx); 697 if (spdk_unlikely(rc)) { 698 if (rc == -ENOMEM) { 699 struct spdk_bdev *bdev = spdk_bdev_desc_get_bdev(md->region->bdev_desc); 700 ctx->bdev_io_wait.bdev = bdev; 701 ctx->bdev_io_wait.cb_fn = retry_fn; 702 ctx->bdev_io_wait.cb_arg = ctx; 703 spdk_bdev_queue_io_wait(bdev, md->region->ioch, &ctx->bdev_io_wait); 704 } else { 705 ftl_abort(); 706 } 707 } 708 709 return rc; 710 } 711 712 static void 713 ftl_md_persist_entry_mirror(void *_ctx) 714 { 715 struct ftl_md_io_entry_ctx *ctx = _ctx; 716 struct ftl_md *md_mirror = ftl_md_get_mirror(ctx->md); 717 718 ftl_md_persist_entry_write_blocks(ctx, md_mirror, ftl_md_persist_entry_mirror); 719 } 720 721 static void 722 ftl_md_persist_entry_primary(void *_ctx) 723 { 724 struct ftl_md_io_entry_ctx *ctx = _ctx; 725 struct ftl_md *md = ctx->md; 726 int rc; 727 728 rc = ftl_md_persist_entry_write_blocks(ctx, md, ftl_md_persist_entry_primary); 729 730 if (!rc && has_mirror(md)) { 731 assert(md->region->entry_size == (ftl_md_get_mirror(md))->region->entry_size); 732 733 /* The MD object has mirror so execute persist on it too */ 734 ftl_md_persist_entry_mirror(ctx); 735 ctx->remaining++; 736 } 737 } 738 739 static void 740 _ftl_md_persist_entry(struct ftl_md_io_entry_ctx *ctx) 741 { 742 ctx->status = 0; 743 ctx->remaining = 1; 744 745 /* First execute an IO to the primary region */ 746 ftl_md_persist_entry_primary(ctx); 747 } 748 749 void 750 ftl_md_persist_entry(struct ftl_md *md, uint64_t start_entry, void *buffer, void *vss_buffer, 751 ftl_md_io_entry_cb cb, void *cb_arg, 752 struct ftl_md_io_entry_ctx *ctx) 753 { 754 if (spdk_unlikely(0 == md->region->entry_size)) { 755 /* This MD has not been configured to support persist entry call */ 756 ftl_abort(); 757 } 758 759 /* Initialize persist entry context */ 760 ctx->cb = cb; 761 ctx->cb_arg = cb_arg; 762 ctx->md = md; 763 ctx->start_entry = start_entry; 764 ctx->buffer = buffer; 765 ctx->vss_buffer = vss_buffer ? : md->entry_vss_dma_buf; 766 767 _ftl_md_persist_entry(ctx); 768 } 769 770 static void 771 read_entry_cb(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) 772 { 773 struct ftl_md_io_entry_ctx *ctx = cb_arg; 774 struct ftl_md *md = ctx->md; 775 776 ftl_stats_bdev_io_completed(md->dev, get_bdev_io_ftl_stats_type(md->dev, bdev_io), bdev_io); 777 778 spdk_bdev_free_io(bdev_io); 779 780 if (!success) { 781 if (has_mirror(md)) { 782 md->mirror_enabled = true; 783 784 /* First read from the mirror */ 785 ftl_md_read_entry(ftl_md_get_mirror(md), ctx->start_entry, ctx->buffer, 786 ctx->vss_buffer, 787 ctx->cb, ctx->cb_arg, 788 ctx); 789 return; 790 } else { 791 ctx->status = -EIO; 792 goto finish_io; 793 } 794 } 795 796 finish_io: 797 ctx->cb(ctx->status, ctx->cb_arg); 798 } 799 800 static void 801 ftl_md_read_entry_read_blocks(struct ftl_md_io_entry_ctx *ctx, struct ftl_md *md, 802 spdk_bdev_io_wait_cb retry_fn) 803 { 804 int rc; 805 806 rc = read_blocks(md->dev, md->region->bdev_desc, md->region->ioch, 807 ctx->buffer, ctx->vss_buffer, 808 persist_entry_lba(md, ctx->start_entry), md->region->entry_size, 809 read_entry_cb, ctx); 810 811 if (spdk_unlikely(rc)) { 812 if (rc == -ENOMEM) { 813 struct spdk_bdev *bdev = spdk_bdev_desc_get_bdev(md->region->bdev_desc); 814 ctx->bdev_io_wait.bdev = bdev; 815 ctx->bdev_io_wait.cb_fn = retry_fn; 816 ctx->bdev_io_wait.cb_arg = ctx; 817 spdk_bdev_queue_io_wait(bdev, md->region->ioch, &ctx->bdev_io_wait); 818 } else { 819 ftl_abort(); 820 } 821 } 822 } 823 824 static void 825 _ftl_md_read_entry(void *_ctx) 826 { 827 struct ftl_md_io_entry_ctx *ctx = _ctx; 828 829 ftl_md_read_entry_read_blocks(ctx, ctx->md, _ftl_md_read_entry); 830 } 831 832 void 833 ftl_md_read_entry(struct ftl_md *md, uint64_t start_entry, void *buffer, void *vss_buffer, 834 ftl_md_io_entry_cb cb, void *cb_arg, 835 struct ftl_md_io_entry_ctx *ctx) 836 { 837 if (spdk_unlikely(0 == md->region->entry_size)) { 838 /* This MD has not been configured to support read entry call */ 839 ftl_abort(); 840 } 841 842 ctx->cb = cb; 843 ctx->cb_arg = cb_arg; 844 ctx->md = md; 845 ctx->start_entry = start_entry; 846 ctx->buffer = buffer; 847 ctx->vss_buffer = vss_buffer; 848 849 _ftl_md_read_entry(ctx); 850 } 851 852 void 853 ftl_md_persist_entry_retry(struct ftl_md_io_entry_ctx *ctx) 854 { 855 _ftl_md_persist_entry(ctx); 856 } 857 858 static void 859 persist_mirror_cb(struct spdk_ftl_dev *dev, struct ftl_md *md, int status) 860 { 861 struct ftl_md *primary = md->owner.private; 862 863 if (status) { 864 /* We got an error, stop persist procedure immediately */ 865 primary->io.status = status; 866 io_done(primary); 867 } else { 868 /* Now continue the persist procedure on the primary MD object */ 869 if (0 == io_init(primary, FTL_MD_OP_PERSIST)) { 870 io_submit(primary); 871 } else { 872 spdk_thread_send_msg(spdk_get_thread(), exception, primary); 873 } 874 } 875 } 876 877 void 878 ftl_md_persist(struct ftl_md *md) 879 { 880 if (has_mirror(md)) { 881 struct ftl_md *md_mirror = ftl_md_get_mirror(md); 882 883 md->mirror_enabled = true; 884 885 /* Set callback and context in mirror */ 886 md_mirror->cb = persist_mirror_cb; 887 md_mirror->owner.private = md; 888 889 /* First persist the mirror */ 890 ftl_md_persist(md_mirror); 891 return; 892 } 893 894 if (0 == io_init(md, FTL_MD_OP_PERSIST)) { 895 io_submit(md); 896 } else { 897 spdk_thread_send_msg(spdk_get_thread(), exception, md); 898 } 899 } 900 901 static void 902 restore_mirror_cb(struct spdk_ftl_dev *dev, struct ftl_md *md, int status) 903 { 904 struct ftl_md *primary = md->owner.private; 905 906 if (status) { 907 /* Cannot restore the object from the mirror too, mark error and fail */ 908 primary->io.status = -EIO; 909 io_done(primary); 910 } else { 911 /* 912 * Restoring from the mirror successful. Synchronize mirror to the primary. 913 * Because we read MD content from the mirror, we can disable it, only the primary 914 * requires persisting. 915 */ 916 primary->io.status = 0; 917 primary->mirror_enabled = false; 918 io_cleanup(primary); 919 ftl_md_persist(primary); 920 primary->mirror_enabled = true; 921 } 922 } 923 924 static void 925 restore_sync_cb(struct spdk_ftl_dev *dev, struct ftl_md *md, int status) 926 { 927 struct ftl_md *primary = md->owner.private; 928 929 if (status) { 930 /* Cannot sync the object from the primary to the mirror, mark error and fail */ 931 primary->io.status = -EIO; 932 io_done(primary); 933 } else { 934 primary->cb(dev, primary, primary->io.status); 935 io_cleanup(primary); 936 } 937 } 938 939 static int 940 restore_done(struct ftl_md *md) 941 { 942 if (-EAGAIN == md->io.status) { 943 /* Failed to read MD from primary region, try it from mirror. 944 * At the moment read the mirror entirely, (TODO) in the 945 * feature we can restore from primary and mirror region 946 * with finer granularity. 947 */ 948 949 if (has_mirror(md)) { 950 struct ftl_md *md_mirror = ftl_md_get_mirror(md); 951 952 md->mirror_enabled = true; 953 954 /* Set callback and context in mirror */ 955 md_mirror->cb = restore_mirror_cb; 956 md_mirror->owner.private = md; 957 958 /* First persist the mirror */ 959 ftl_md_restore(md_mirror); 960 return -EAGAIN; 961 } else { 962 return -EIO; 963 } 964 } else if (0 == md->io.status && false == md->dev->sb->clean) { 965 if (has_mirror(md)) { 966 struct ftl_md *md_mirror = ftl_md_get_mirror(md); 967 /* There was a dirty shutdown, synchronize primary to mirror */ 968 969 /* Set callback and context in the mirror */ 970 md_mirror->cb = restore_sync_cb; 971 md_mirror->owner.private = md; 972 973 /* First persist the mirror */ 974 ftl_md_persist(md_mirror); 975 return -EAGAIN; 976 } 977 } 978 979 return md->io.status; 980 } 981 982 static void 983 io_done(struct ftl_md *md) 984 { 985 int status; 986 987 if (md->io.op == FTL_MD_OP_RESTORE) { 988 status = restore_done(md); 989 } else { 990 status = md->io.status; 991 } 992 993 if (status != -EAGAIN) { 994 md->cb(md->dev, md, status); 995 io_cleanup(md); 996 } 997 } 998 999 void 1000 ftl_md_restore(struct ftl_md *md) 1001 { 1002 if (0 == io_init(md, FTL_MD_OP_RESTORE)) { 1003 io_submit(md); 1004 } else { 1005 spdk_thread_send_msg(spdk_get_thread(), exception, md); 1006 } 1007 } 1008 1009 static int 1010 pattern_prepare(struct ftl_md *md, 1011 int data_pattern, union ftl_md_vss *vss_pattern) 1012 { 1013 void *data = md->io.data; 1014 uint64_t data_size = xfer_size(md); 1015 1016 memset(data, data_pattern, data_size); 1017 1018 if (md->io.md) { 1019 if (vss_pattern) { 1020 /* store the VSS pattern... */ 1021 ftl_md_vss_buf_init(md->io.md, ftl_md_xfer_blocks(md->dev), vss_pattern); 1022 } else { 1023 /* ...or default init VSS to 0 */ 1024 union ftl_md_vss vss = {0}; 1025 1026 vss.version.md_version = md->region->current.version; 1027 ftl_md_vss_buf_init(md->io.md, ftl_md_xfer_blocks(md->dev), &vss); 1028 } 1029 } 1030 1031 return 0; 1032 } 1033 1034 static void 1035 clear_mirror_cb(struct spdk_ftl_dev *dev, struct ftl_md *secondary, int status) 1036 { 1037 struct ftl_md *primary = secondary->owner.private; 1038 1039 if (status) { 1040 /* We got an error, stop persist procedure immediately */ 1041 primary->io.status = status; 1042 io_done(primary); 1043 } else { 1044 /* Now continue the persist procedure on the primary MD object */ 1045 if (0 == io_init(primary, FTL_MD_OP_CLEAR) && 1046 0 == pattern_prepare(primary, *(int *)secondary->io.data, 1047 secondary->io.md)) { 1048 io_submit(primary); 1049 } else { 1050 spdk_thread_send_msg(spdk_get_thread(), exception, primary); 1051 } 1052 } 1053 } 1054 1055 void 1056 ftl_md_clear(struct ftl_md *md, int data_pattern, union ftl_md_vss *vss_pattern) 1057 { 1058 if (has_mirror(md)) { 1059 struct ftl_md *md_mirror = ftl_md_get_mirror(md); 1060 1061 md->mirror_enabled = true; 1062 1063 /* Set callback and context in mirror */ 1064 md_mirror->cb = clear_mirror_cb; 1065 md_mirror->owner.private = md; 1066 1067 /* First persist the mirror */ 1068 ftl_md_clear(md_mirror, data_pattern, vss_pattern); 1069 return; 1070 } 1071 1072 if (0 == io_init(md, FTL_MD_OP_CLEAR) && 0 == pattern_prepare(md, data_pattern, vss_pattern)) { 1073 io_submit(md); 1074 } else { 1075 spdk_thread_send_msg(spdk_get_thread(), exception, md); 1076 } 1077 } 1078 1079 const struct ftl_layout_region * 1080 ftl_md_get_region(struct ftl_md *md) 1081 { 1082 return md->region; 1083 } 1084 1085 void 1086 ftl_md_set_region(struct ftl_md *md, 1087 const struct ftl_layout_region *region) 1088 { 1089 assert(region->current.blocks <= md->data_blocks); 1090 md->region = region; 1091 1092 if (md->vss_data) { 1093 union ftl_md_vss vss = {0}; 1094 vss.version.md_version = region->current.version; 1095 ftl_md_vss_buf_init(md->vss_data, md->data_blocks, &vss); 1096 if (region->entry_size) { 1097 assert(md->entry_vss_dma_buf); 1098 ftl_md_vss_buf_init(md->entry_vss_dma_buf, region->entry_size, &vss); 1099 } 1100 } 1101 1102 if (has_mirror(md)) { 1103 md->mirror_enabled = true; 1104 } 1105 } 1106 1107 int 1108 ftl_md_create_region_flags(struct spdk_ftl_dev *dev, int region_type) 1109 { 1110 int flags = FTL_MD_CREATE_SHM; 1111 1112 switch (region_type) { 1113 case FTL_LAYOUT_REGION_TYPE_SB: 1114 if (dev->conf.mode & SPDK_FTL_MODE_CREATE) { 1115 flags |= FTL_MD_CREATE_SHM_NEW; 1116 } 1117 break; 1118 1119 case FTL_LAYOUT_REGION_TYPE_BAND_MD: 1120 case FTL_LAYOUT_REGION_TYPE_NVC_MD: 1121 if (!ftl_fast_startup(dev)) { 1122 flags |= FTL_MD_CREATE_SHM_NEW; 1123 } 1124 break; 1125 case FTL_LAYOUT_REGION_TYPE_VALID_MAP: 1126 case FTL_LAYOUT_REGION_TYPE_TRIM_MD: 1127 if (!ftl_fast_startup(dev) && !ftl_fast_recovery(dev)) { 1128 flags |= FTL_MD_CREATE_SHM_NEW; 1129 } 1130 break; 1131 default: 1132 return FTL_MD_CREATE_HEAP; 1133 } 1134 1135 return flags; 1136 } 1137 1138 int 1139 ftl_md_destroy_region_flags(struct spdk_ftl_dev *dev, int region_type) 1140 { 1141 switch (region_type) { 1142 case FTL_LAYOUT_REGION_TYPE_SB: 1143 case FTL_LAYOUT_REGION_TYPE_BAND_MD: 1144 case FTL_LAYOUT_REGION_TYPE_VALID_MAP: 1145 case FTL_LAYOUT_REGION_TYPE_NVC_MD: 1146 case FTL_LAYOUT_REGION_TYPE_TRIM_MD: 1147 if (dev->conf.fast_shutdown) { 1148 return FTL_MD_DESTROY_SHM_KEEP; 1149 } 1150 break; 1151 1152 default: 1153 break; 1154 } 1155 return 0; 1156 } 1157 1158 int 1159 ftl_md_create_shm_flags(struct spdk_ftl_dev *dev) 1160 { 1161 int flags = FTL_MD_CREATE_SHM; 1162 1163 if (!ftl_fast_startup(dev) && !ftl_fast_recovery(dev)) { 1164 flags |= FTL_MD_CREATE_SHM_NEW; 1165 } 1166 return flags; 1167 } 1168 1169 int 1170 ftl_md_destroy_shm_flags(struct spdk_ftl_dev *dev) 1171 { 1172 return (dev->conf.fast_shutdown) ? FTL_MD_DESTROY_SHM_KEEP : 0; 1173 } 1174