1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (c) Intel Corporation. 3 * All rights reserved. 4 */ 5 6 #include "spdk/env.h" 7 #include "spdk/bdev_module.h" 8 9 #include "ftl_core.h" 10 #include "ftl_md.h" 11 #include "ftl_nv_cache_io.h" 12 13 struct ftl_md; 14 static void io_submit(struct ftl_md *md); 15 static void io_done(struct ftl_md *md); 16 17 static bool 18 has_mirror(struct ftl_md *md) 19 { 20 if (md->region) { 21 if (md->region->mirror_type != FTL_LAYOUT_REGION_TYPE_INVALID) { 22 return md->mirror_enabled; 23 } 24 } 25 26 return false; 27 } 28 29 static int 30 setup_mirror(struct ftl_md *md) 31 { 32 if (!md->mirror) { 33 md->mirror = calloc(1, sizeof(*md->mirror)); 34 if (!md->mirror) { 35 return -ENOMEM; 36 } 37 md->mirror_enabled = true; 38 } 39 40 md->mirror->dev = md->dev; 41 md->mirror->data_blocks = md->data_blocks; 42 md->mirror->data = md->data; 43 md->mirror->vss_data = md->vss_data; 44 45 /* Set proper region in secondary object */ 46 assert(md->region->mirror_type != FTL_LAYOUT_REGION_TYPE_INVALID); 47 md->mirror->region = &md->dev->layout.region[md->region->mirror_type]; 48 49 return 0; 50 } 51 52 uint64_t 53 ftl_md_xfer_blocks(struct spdk_ftl_dev *dev) 54 { 55 return 4ULL * dev->xfer_size; 56 } 57 58 static uint64_t 59 xfer_size(struct ftl_md *md) 60 { 61 return ftl_md_xfer_blocks(md->dev) * FTL_BLOCK_SIZE; 62 } 63 64 static void 65 ftl_md_create_heap(struct ftl_md *md, uint64_t vss_blksz) 66 { 67 md->shm_fd = -1; 68 md->vss_data = NULL; 69 md->data = calloc(md->data_blocks, FTL_BLOCK_SIZE + vss_blksz); 70 71 if (md->data && vss_blksz) { 72 md->vss_data = ((char *)md->data) + md->data_blocks * FTL_BLOCK_SIZE; 73 } 74 } 75 76 static void 77 ftl_md_destroy_heap(struct ftl_md *md) 78 { 79 if (md->data) { 80 free(md->data); 81 md->data = NULL; 82 md->vss_data = NULL; 83 } 84 } 85 86 static int 87 ftl_wrapper_open(const char *name, int of, mode_t m) 88 { 89 return open(name, of, m); 90 } 91 92 static void 93 ftl_md_setup_obj(struct ftl_md *md, int flags, 94 const char *name) 95 { 96 char uuid_str[SPDK_UUID_STRING_LEN]; 97 const char *fmt; 98 99 if (!(flags & FTL_MD_CREATE_SHM)) { 100 assert(false); 101 return; 102 } 103 104 /* TODO: temporary, define a proper hugetlbfs mountpoint */ 105 fmt = "/dev/hugepages/ftl_%s_%s"; 106 md->shm_mmap_flags = MAP_SHARED; 107 md->shm_open = ftl_wrapper_open; 108 md->shm_unlink = unlink; 109 110 if (name == NULL || 111 spdk_uuid_fmt_lower(uuid_str, SPDK_UUID_STRING_LEN, &md->dev->conf.uuid) || 112 snprintf(md->name, sizeof(md->name) / sizeof(md->name[0]), 113 fmt, uuid_str, name) <= 0) { 114 md->name[0] = 0; 115 } 116 } 117 118 static void 119 ftl_md_invalidate_shm(struct ftl_md *md) 120 { 121 if (md->dev->sb_shm && md->dev->sb_shm->shm_ready) { 122 md->dev->init_retry = true; 123 md->dev->sb_shm->shm_ready = false; 124 } 125 } 126 127 static void 128 ftl_md_create_shm(struct ftl_md *md, uint64_t vss_blksz, int flags) 129 { 130 struct stat shm_stat; 131 size_t vss_blk_offs; 132 void *shm_ptr; 133 int open_flags = O_RDWR; 134 mode_t open_mode = S_IRUSR | S_IWUSR; 135 136 assert(md->shm_open && md->shm_unlink); 137 md->data = NULL; 138 md->vss_data = NULL; 139 md->shm_sz = 0; 140 141 /* Must have an object name */ 142 if (md->name[0] == 0) { 143 assert(false); 144 return; 145 } 146 147 /* If specified, unlink before create a new SHM object */ 148 if (flags & FTL_MD_CREATE_SHM_NEW) { 149 if (md->shm_unlink(md->name) < 0 && errno != ENOENT) { 150 ftl_md_invalidate_shm(md); 151 return; 152 } 153 open_flags += O_CREAT | O_TRUNC; 154 } 155 156 /* Open existing or create a new SHM object, then query its props */ 157 md->shm_fd = md->shm_open(md->name, open_flags, open_mode); 158 if (md->shm_fd < 0 || fstat(md->shm_fd, &shm_stat) < 0) { 159 goto err_shm; 160 } 161 162 /* Verify open mode hasn't changed */ 163 if ((shm_stat.st_mode & (S_IRWXU | S_IRWXG | S_IRWXO)) != open_mode) { 164 goto err_shm; 165 } 166 167 /* Round up the SHM obj size to the nearest blk size (i.e. page size) */ 168 md->shm_sz = spdk_divide_round_up(md->data_blocks * FTL_BLOCK_SIZE, shm_stat.st_blksize); 169 170 /* Add some blks for VSS metadata */ 171 vss_blk_offs = md->shm_sz; 172 173 if (vss_blksz) { 174 md->shm_sz += spdk_divide_round_up(md->data_blocks * vss_blksz, 175 shm_stat.st_blksize); 176 } 177 178 /* Total SHM obj size */ 179 md->shm_sz *= shm_stat.st_blksize; 180 181 /* Set or check the object size - zero init`d in case of set (FTL_MD_CREATE_SHM_NEW) */ 182 if ((shm_stat.st_size == 0 && (ftruncate(md->shm_fd, md->shm_sz) < 0 || 183 (flags & FTL_MD_CREATE_SHM_NEW) == 0)) 184 || (shm_stat.st_size > 0 && (size_t)shm_stat.st_size != md->shm_sz)) { 185 goto err_shm; 186 } 187 188 /* Create a virtual memory mapping for the object */ 189 shm_ptr = mmap(NULL, md->shm_sz, PROT_READ | PROT_WRITE, md->shm_mmap_flags, 190 md->shm_fd, 0); 191 if (shm_ptr == MAP_FAILED) { 192 goto err_shm; 193 } 194 195 md->data = shm_ptr; 196 if (vss_blksz) { 197 md->vss_data = ((char *)shm_ptr) + vss_blk_offs * shm_stat.st_blksize; 198 } 199 200 /* Lock the pages in memory (i.e. prevent the pages to be paged out) */ 201 if (mlock(md->data, md->shm_sz) < 0) { 202 goto err_map; 203 } 204 205 if (spdk_mem_register(md->data, md->shm_sz)) { 206 goto err_mlock; 207 } 208 md->mem_reg = true; 209 210 return; 211 212 /* Cleanup upon fault */ 213 err_mlock: 214 munlock(md->data, md->shm_sz); 215 216 err_map: 217 munmap(md->data, md->shm_sz); 218 md->data = NULL; 219 md->vss_data = NULL; 220 md->shm_sz = 0; 221 222 err_shm: 223 if (md->shm_fd >= 0) { 224 close(md->shm_fd); 225 md->shm_unlink(md->name); 226 md->shm_fd = -1; 227 } 228 ftl_md_invalidate_shm(md); 229 } 230 231 static void 232 ftl_md_destroy_shm(struct ftl_md *md, int flags) 233 { 234 if (!md->data) { 235 return; 236 } 237 238 assert(md->shm_sz > 0); 239 if (md->mem_reg) { 240 spdk_mem_unregister(md->data, md->shm_sz); 241 md->mem_reg = false; 242 } 243 244 /* Unlock the pages in memory */ 245 munlock(md->data, md->shm_sz); 246 247 /* Remove the virtual memory mapping for the object */ 248 munmap(md->data, md->shm_sz); 249 250 /* Close SHM object fd */ 251 close(md->shm_fd); 252 253 md->data = NULL; 254 md->vss_data = NULL; 255 256 /* If specified, keep the object in SHM */ 257 if (flags & FTL_MD_DESTROY_SHM_KEEP) { 258 return; 259 } 260 261 /* Otherwise destroy/unlink the object */ 262 assert(md->name[0] != 0 && md->shm_unlink != NULL); 263 md->shm_unlink(md->name); 264 } 265 266 struct ftl_md *ftl_md_create(struct spdk_ftl_dev *dev, uint64_t blocks, 267 uint64_t vss_blksz, const char *name, int flags, 268 const struct ftl_layout_region *region) 269 { 270 struct ftl_md *md; 271 272 md = calloc(1, sizeof(*md)); 273 if (!md) { 274 return NULL; 275 } 276 md->dev = dev; 277 md->data_blocks = blocks; 278 md->mirror_enabled = true; 279 280 if (flags != FTL_MD_CREATE_NO_MEM) { 281 if (flags & FTL_MD_CREATE_SHM) { 282 ftl_md_setup_obj(md, flags, name); 283 ftl_md_create_shm(md, vss_blksz, flags); 284 } else { 285 assert((flags & FTL_MD_CREATE_HEAP) == FTL_MD_CREATE_HEAP); 286 ftl_md_create_heap(md, vss_blksz); 287 } 288 289 if (!md->data) { 290 free(md); 291 return NULL; 292 } 293 } 294 295 if (region) { 296 size_t entry_vss_buf_size = vss_blksz * region->entry_size; 297 298 if (entry_vss_buf_size) { 299 md->entry_vss_dma_buf = spdk_malloc(entry_vss_buf_size, FTL_BLOCK_SIZE, 300 NULL, SPDK_ENV_LCORE_ID_ANY, 301 SPDK_MALLOC_DMA); 302 if (!md->entry_vss_dma_buf) { 303 goto err; 304 } 305 } 306 307 if (ftl_md_set_region(md, region)) { 308 goto err; 309 } 310 } 311 312 return md; 313 err: 314 ftl_md_destroy(md, ftl_md_destroy_region_flags(dev, region->type)); 315 return NULL; 316 } 317 318 int 319 ftl_md_unlink(struct spdk_ftl_dev *dev, const char *name, int flags) 320 { 321 struct ftl_md md = { 0 }; 322 323 if (0 == (flags & FTL_MD_CREATE_SHM)) { 324 /* Unlink can be called for shared memory only */ 325 return -EINVAL; 326 } 327 328 md.dev = dev; 329 ftl_md_setup_obj(&md, flags, name); 330 331 return md.shm_unlink(md.name); 332 } 333 334 void 335 ftl_md_destroy(struct ftl_md *md, int flags) 336 { 337 if (!md) { 338 return; 339 } 340 341 ftl_md_free_buf(md, flags); 342 343 spdk_free(md->entry_vss_dma_buf); 344 345 free(md->mirror); 346 free(md); 347 } 348 349 void 350 ftl_md_free_buf(struct ftl_md *md, int flags) 351 { 352 if (!md) { 353 return; 354 } 355 356 if (md->shm_fd < 0) { 357 assert(flags == 0); 358 ftl_md_destroy_heap(md); 359 } else { 360 ftl_md_destroy_shm(md, flags); 361 } 362 } 363 364 void * 365 ftl_md_get_buffer(struct ftl_md *md) 366 { 367 return md->data; 368 } 369 370 uint64_t 371 ftl_md_get_buffer_size(struct ftl_md *md) 372 { 373 return md->data_blocks * FTL_BLOCK_SIZE; 374 } 375 376 static void 377 ftl_md_vss_buf_init(union ftl_md_vss *buf, uint32_t count, 378 const union ftl_md_vss *vss_pattern) 379 { 380 while (count) { 381 count--; 382 buf[count] = *vss_pattern; 383 } 384 } 385 386 union ftl_md_vss *ftl_md_vss_buf_alloc(struct ftl_layout_region *region, uint32_t count) 387 { 388 union ftl_md_vss *buf = spdk_zmalloc(count * FTL_MD_VSS_SZ, FTL_BLOCK_SIZE, NULL, 389 SPDK_ENV_LCORE_ID_ANY, 390 SPDK_MALLOC_DMA); 391 392 if (!buf) { 393 return NULL; 394 } 395 396 union ftl_md_vss vss_buf = {0}; 397 vss_buf.version.md_version = region->current.version; 398 ftl_md_vss_buf_init(buf, count, &vss_buf); 399 return buf; 400 } 401 402 union ftl_md_vss *ftl_md_get_vss_buffer(struct ftl_md *md) 403 { 404 return md->vss_data; 405 } 406 407 static void 408 io_cleanup(struct ftl_md *md) 409 { 410 spdk_dma_free(md->io.data); 411 md->io.data = NULL; 412 413 spdk_dma_free(md->io.md); 414 md->io.md = NULL; 415 } 416 417 static void 418 exception(void *arg) 419 { 420 struct ftl_md *md = arg; 421 422 md->cb(md->dev, md, -EINVAL); 423 io_cleanup(md); 424 } 425 426 static void 427 audit_md_vss_version(struct ftl_md *md, uint64_t blocks) 428 { 429 #if defined(DEBUG) 430 union ftl_md_vss *vss = md->io.md; 431 while (blocks) { 432 blocks--; 433 assert(vss[blocks].version.md_version == md->region->current.version); 434 } 435 #endif 436 } 437 438 static void 439 read_write_blocks_cb(struct spdk_bdev_io *bdev_io, bool success, void *arg) 440 { 441 struct ftl_md *md = arg; 442 443 if (spdk_unlikely(!success)) { 444 if (md->io.op == FTL_MD_OP_RESTORE && has_mirror(md)) { 445 md->io.status = -EAGAIN; 446 } else { 447 md->io.status = -EIO; 448 } 449 } else { 450 uint64_t blocks = bdev_io->u.bdev.num_blocks; 451 uint64_t size = blocks * FTL_BLOCK_SIZE; 452 453 if (md->io.op == FTL_MD_OP_RESTORE) { 454 memcpy(md->data + md->io.data_offset, md->io.data, size); 455 456 if (md->vss_data) { 457 uint64_t vss_offset = md->io.data_offset / FTL_BLOCK_SIZE; 458 vss_offset *= FTL_MD_VSS_SZ; 459 audit_md_vss_version(md, blocks); 460 memcpy(md->vss_data + vss_offset, md->io.md, blocks * FTL_MD_VSS_SZ); 461 } 462 } 463 464 md->io.address += blocks; 465 md->io.remaining -= blocks; 466 md->io.data_offset += size; 467 } 468 469 spdk_bdev_free_io(bdev_io); 470 471 io_submit(md); 472 } 473 474 static inline int 475 read_blocks(struct spdk_ftl_dev *dev, struct spdk_bdev_desc *desc, 476 struct spdk_io_channel *ch, 477 void *buf, void *md_buf, 478 uint64_t offset_blocks, uint64_t num_blocks, 479 spdk_bdev_io_completion_cb cb, void *cb_arg) 480 { 481 if (desc == dev->nv_cache.bdev_desc) { 482 return ftl_nv_cache_bdev_read_blocks_with_md(dev, desc, ch, buf, md_buf, 483 offset_blocks, num_blocks, 484 cb, cb_arg); 485 } else if (md_buf) { 486 return spdk_bdev_read_blocks_with_md(desc, ch, buf, md_buf, 487 offset_blocks, num_blocks, 488 cb, cb_arg); 489 } else { 490 return spdk_bdev_read_blocks(desc, ch, buf, 491 offset_blocks, num_blocks, 492 cb, cb_arg); 493 } 494 } 495 496 static inline int 497 write_blocks(struct spdk_ftl_dev *dev, struct spdk_bdev_desc *desc, 498 struct spdk_io_channel *ch, 499 void *buf, void *md_buf, 500 uint64_t offset_blocks, uint64_t num_blocks, 501 spdk_bdev_io_completion_cb cb, void *cb_arg) 502 { 503 if (desc == dev->nv_cache.bdev_desc) { 504 return ftl_nv_cache_bdev_write_blocks_with_md(dev, desc, ch, buf, md_buf, 505 offset_blocks, num_blocks, 506 cb, cb_arg); 507 } else if (md_buf) { 508 return spdk_bdev_write_blocks_with_md(desc, ch, buf, md_buf, offset_blocks, 509 num_blocks, cb, cb_arg); 510 } else { 511 return spdk_bdev_write_blocks(desc, ch, buf, offset_blocks, num_blocks, cb, cb_arg); 512 } 513 } 514 515 static void 516 read_write_blocks(void *_md) 517 { 518 struct ftl_md *md = _md; 519 const struct ftl_layout_region *region = md->region; 520 uint64_t blocks; 521 int rc = 0; 522 523 blocks = spdk_min(md->io.remaining, ftl_md_xfer_blocks(md->dev)); 524 525 switch (md->io.op) { 526 case FTL_MD_OP_RESTORE: 527 rc = read_blocks(md->dev, region->bdev_desc, region->ioch, 528 md->io.data, md->io.md, 529 md->io.address, blocks, 530 read_write_blocks_cb, md); 531 break; 532 case FTL_MD_OP_PERSIST: 533 case FTL_MD_OP_CLEAR: 534 rc = write_blocks(md->dev, region->bdev_desc, region->ioch, 535 md->io.data, md->io.md, 536 md->io.address, blocks, 537 read_write_blocks_cb, md); 538 break; 539 default: 540 ftl_abort(); 541 } 542 543 if (spdk_unlikely(rc)) { 544 if (rc == -ENOMEM) { 545 struct spdk_bdev *bdev = spdk_bdev_desc_get_bdev(region->bdev_desc); 546 md->io.bdev_io_wait.bdev = bdev; 547 md->io.bdev_io_wait.cb_fn = read_write_blocks; 548 md->io.bdev_io_wait.cb_arg = md; 549 spdk_bdev_queue_io_wait(bdev, region->ioch, &md->io.bdev_io_wait); 550 } else { 551 ftl_abort(); 552 } 553 } 554 } 555 556 static void 557 io_submit(struct ftl_md *md) 558 { 559 if (!md->io.remaining || md->io.status) { 560 io_done(md); 561 return; 562 } 563 564 if (md->io.op == FTL_MD_OP_PERSIST) { 565 uint64_t blocks = spdk_min(md->io.remaining, ftl_md_xfer_blocks(md->dev)); 566 567 memcpy(md->io.data, md->data + md->io.data_offset, FTL_BLOCK_SIZE * blocks); 568 569 if (md->vss_data) { 570 uint64_t vss_offset = md->io.data_offset / FTL_BLOCK_SIZE; 571 vss_offset *= FTL_MD_VSS_SZ; 572 assert(md->io.md); 573 memcpy(md->io.md, md->vss_data + vss_offset, FTL_MD_VSS_SZ * blocks); 574 audit_md_vss_version(md, blocks); 575 } 576 } 577 #if defined(DEBUG) 578 if (md->io.md && md->io.op == FTL_MD_OP_CLEAR) { 579 uint64_t blocks = spdk_min(md->io.remaining, ftl_md_xfer_blocks(md->dev)); 580 audit_md_vss_version(md, blocks); 581 } 582 #endif 583 584 read_write_blocks(md); 585 } 586 587 static int 588 io_can_start(struct ftl_md *md) 589 { 590 assert(NULL == md->io.data); 591 if (NULL != md->io.data) { 592 /* Outgoing IO on metadata */ 593 return -EINVAL; 594 } 595 596 if (!md->region) { 597 /* No device region to process data */ 598 return -EINVAL; 599 } 600 601 if (md->region->current.blocks > md->data_blocks) { 602 /* No device region to process data */ 603 FTL_ERRLOG(md->dev, "Blocks number mismatch between metadata object and" 604 "device region\n"); 605 return -EINVAL; 606 } 607 608 return 0; 609 } 610 611 static int 612 io_prepare(struct ftl_md *md, enum ftl_md_ops op) 613 { 614 const struct ftl_layout_region *region = md->region; 615 uint64_t data_size, meta_size = 0; 616 617 /* Allocates buffer for IO */ 618 data_size = xfer_size(md); 619 md->io.data = spdk_zmalloc(data_size, FTL_BLOCK_SIZE, NULL, 620 SPDK_ENV_LCORE_ID_ANY, SPDK_MALLOC_DMA); 621 if (!md->io.data) { 622 return -ENOMEM; 623 } 624 625 if (md->vss_data || md->region->vss_blksz) { 626 meta_size = ftl_md_xfer_blocks(md->dev) * FTL_MD_VSS_SZ; 627 md->io.md = spdk_zmalloc(meta_size, FTL_BLOCK_SIZE, NULL, 628 SPDK_ENV_LCORE_ID_ANY, SPDK_MALLOC_DMA); 629 if (!md->io.md) { 630 spdk_dma_free(md->io.data); 631 md->io.data = NULL; 632 return -ENOMEM; 633 } 634 } 635 636 md->io.address = region->current.offset; 637 md->io.remaining = region->current.blocks; 638 md->io.data_offset = 0; 639 md->io.status = 0; 640 md->io.op = op; 641 642 return 0; 643 } 644 645 static int 646 io_init(struct ftl_md *md, enum ftl_md_ops op) 647 { 648 if (io_can_start(md)) { 649 return -EINVAL; 650 } 651 652 if (io_prepare(md, op)) { 653 return -ENOMEM; 654 } 655 656 return 0; 657 } 658 659 static uint64_t 660 persist_entry_lba(struct ftl_md *md, uint64_t start_entry) 661 { 662 return md->region->current.offset + start_entry * md->region->entry_size; 663 } 664 665 static void 666 persist_entry_cb(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) 667 { 668 struct ftl_md_io_entry_ctx *ctx = cb_arg; 669 670 spdk_bdev_free_io(bdev_io); 671 672 assert(ctx->remaining > 0); 673 ctx->remaining--; 674 675 if (!success) { 676 ctx->status = -EIO; 677 } 678 679 if (!ctx->remaining) { 680 ctx->cb(ctx->status, ctx->cb_arg); 681 } 682 } 683 684 static int 685 ftl_md_persist_entry_write_blocks(struct ftl_md_io_entry_ctx *ctx, struct ftl_md *md, 686 spdk_bdev_io_wait_cb retry_fn) 687 { 688 int rc; 689 690 rc = write_blocks(md->dev, md->region->bdev_desc, md->region->ioch, 691 ctx->buffer, ctx->vss_buffer, 692 persist_entry_lba(md, ctx->start_entry), md->region->entry_size, 693 persist_entry_cb, ctx); 694 if (spdk_unlikely(rc)) { 695 if (rc == -ENOMEM) { 696 struct spdk_bdev *bdev = spdk_bdev_desc_get_bdev(md->region->bdev_desc); 697 ctx->bdev_io_wait.bdev = bdev; 698 ctx->bdev_io_wait.cb_fn = retry_fn; 699 ctx->bdev_io_wait.cb_arg = ctx; 700 spdk_bdev_queue_io_wait(bdev, md->region->ioch, &ctx->bdev_io_wait); 701 } else { 702 ftl_abort(); 703 } 704 } 705 706 return rc; 707 } 708 709 static void 710 ftl_md_persist_entry_mirror(void *_ctx) 711 { 712 struct ftl_md_io_entry_ctx *ctx = _ctx; 713 714 ftl_md_persist_entry_write_blocks(ctx, ctx->md->mirror, ftl_md_persist_entry_mirror); 715 } 716 717 static void 718 ftl_md_persist_entry_primary(void *_ctx) 719 { 720 struct ftl_md_io_entry_ctx *ctx = _ctx; 721 struct ftl_md *md = ctx->md; 722 int rc; 723 724 rc = ftl_md_persist_entry_write_blocks(ctx, md, ftl_md_persist_entry_primary); 725 726 if (!rc && has_mirror(md)) { 727 assert(md->region->entry_size == md->mirror->region->entry_size); 728 729 /* The MD object has mirror so execute persist on it too */ 730 ftl_md_persist_entry_mirror(ctx); 731 ctx->remaining++; 732 } 733 } 734 735 static void 736 _ftl_md_persist_entry(struct ftl_md_io_entry_ctx *ctx) 737 { 738 ctx->status = 0; 739 ctx->remaining = 1; 740 741 /* First execute an IO to the primary region */ 742 ftl_md_persist_entry_primary(ctx); 743 } 744 745 void 746 ftl_md_persist_entry(struct ftl_md *md, uint64_t start_entry, void *buffer, void *vss_buffer, 747 ftl_md_io_entry_cb cb, void *cb_arg, 748 struct ftl_md_io_entry_ctx *ctx) 749 { 750 if (spdk_unlikely(0 == md->region->entry_size)) { 751 /* This MD has not been configured to support persist entry call */ 752 ftl_abort(); 753 } 754 755 /* Initialize persist entry context */ 756 ctx->cb = cb; 757 ctx->cb_arg = cb_arg; 758 ctx->md = md; 759 ctx->start_entry = start_entry; 760 ctx->buffer = buffer; 761 ctx->vss_buffer = vss_buffer ? : md->entry_vss_dma_buf; 762 763 _ftl_md_persist_entry(ctx); 764 } 765 766 static void 767 read_entry_cb(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) 768 { 769 struct ftl_md_io_entry_ctx *ctx = cb_arg; 770 struct ftl_md *md = ctx->md; 771 772 spdk_bdev_free_io(bdev_io); 773 774 if (!success) { 775 if (has_mirror(md)) { 776 if (setup_mirror(md)) { 777 /* An error when setup the mirror */ 778 ctx->status = -EIO; 779 goto finish_io; 780 } 781 782 /* First read from the mirror */ 783 ftl_md_read_entry(md->mirror, ctx->start_entry, ctx->buffer, ctx->vss_buffer, 784 ctx->cb, ctx->cb_arg, 785 ctx); 786 return; 787 } else { 788 ctx->status = -EIO; 789 goto finish_io; 790 } 791 } 792 793 finish_io: 794 ctx->cb(ctx->status, ctx->cb_arg); 795 } 796 797 static void 798 ftl_md_read_entry_read_blocks(struct ftl_md_io_entry_ctx *ctx, struct ftl_md *md, 799 spdk_bdev_io_wait_cb retry_fn) 800 { 801 int rc; 802 803 rc = read_blocks(md->dev, md->region->bdev_desc, md->region->ioch, 804 ctx->buffer, ctx->vss_buffer, 805 persist_entry_lba(md, ctx->start_entry), md->region->entry_size, 806 read_entry_cb, ctx); 807 808 if (spdk_unlikely(rc)) { 809 if (rc == -ENOMEM) { 810 struct spdk_bdev *bdev = spdk_bdev_desc_get_bdev(md->region->bdev_desc); 811 ctx->bdev_io_wait.bdev = bdev; 812 ctx->bdev_io_wait.cb_fn = retry_fn; 813 ctx->bdev_io_wait.cb_arg = ctx; 814 spdk_bdev_queue_io_wait(bdev, md->region->ioch, &ctx->bdev_io_wait); 815 } else { 816 ftl_abort(); 817 } 818 } 819 } 820 821 static void 822 _ftl_md_read_entry(void *_ctx) 823 { 824 struct ftl_md_io_entry_ctx *ctx = _ctx; 825 826 ftl_md_read_entry_read_blocks(ctx, ctx->md, _ftl_md_read_entry); 827 } 828 829 void 830 ftl_md_read_entry(struct ftl_md *md, uint64_t start_entry, void *buffer, void *vss_buffer, 831 ftl_md_io_entry_cb cb, void *cb_arg, 832 struct ftl_md_io_entry_ctx *ctx) 833 { 834 if (spdk_unlikely(0 == md->region->entry_size)) { 835 /* This MD has not been configured to support read entry call */ 836 ftl_abort(); 837 } 838 839 ctx->cb = cb; 840 ctx->cb_arg = cb_arg; 841 ctx->md = md; 842 ctx->start_entry = start_entry; 843 ctx->buffer = buffer; 844 ctx->vss_buffer = vss_buffer; 845 846 _ftl_md_read_entry(ctx); 847 } 848 849 void 850 ftl_md_persist_entry_retry(struct ftl_md_io_entry_ctx *ctx) 851 { 852 _ftl_md_persist_entry(ctx); 853 } 854 855 static void 856 persist_mirror_cb(struct spdk_ftl_dev *dev, struct ftl_md *md, int status) 857 { 858 struct ftl_md *primary = md->owner.private; 859 860 if (status) { 861 /* We got an error, stop persist procedure immediately */ 862 primary->io.status = status; 863 io_done(primary); 864 } else { 865 /* Now continue the persist procedure on the primary MD object */ 866 if (0 == io_init(primary, FTL_MD_OP_PERSIST)) { 867 io_submit(primary); 868 } else { 869 spdk_thread_send_msg(spdk_get_thread(), exception, primary); 870 } 871 } 872 } 873 874 void 875 ftl_md_persist(struct ftl_md *md) 876 { 877 if (has_mirror(md)) { 878 if (setup_mirror(md)) { 879 /* An error when setup the mirror */ 880 spdk_thread_send_msg(spdk_get_thread(), exception, md); 881 return; 882 } 883 884 /* Set callback and context in mirror */ 885 md->mirror->cb = persist_mirror_cb; 886 md->mirror->owner.private = md; 887 888 /* First persist the mirror */ 889 ftl_md_persist(md->mirror); 890 return; 891 } 892 893 if (0 == io_init(md, FTL_MD_OP_PERSIST)) { 894 io_submit(md); 895 } else { 896 spdk_thread_send_msg(spdk_get_thread(), exception, md); 897 } 898 } 899 900 static void 901 restore_mirror_cb(struct spdk_ftl_dev *dev, struct ftl_md *md, int status) 902 { 903 struct ftl_md *primary = md->owner.private; 904 905 if (status) { 906 /* Cannot restore the object from the mirror too, mark error and fail */ 907 primary->io.status = -EIO; 908 io_done(primary); 909 } else { 910 /* 911 * Restoring from the mirror successful. Synchronize mirror to the primary. 912 * Because we read MD content from the mirror, we can disable it, only the primary 913 * requires persisting. 914 */ 915 primary->io.status = 0; 916 primary->mirror_enabled = false; 917 io_cleanup(primary); 918 ftl_md_persist(primary); 919 primary->mirror_enabled = true; 920 } 921 } 922 923 static void 924 restore_sync_cb(struct spdk_ftl_dev *dev, struct ftl_md *md, int status) 925 { 926 struct ftl_md *primary = md->owner.private; 927 928 if (status) { 929 /* Cannot sync the object from the primary to the mirror, mark error and fail */ 930 primary->io.status = -EIO; 931 io_done(primary); 932 } else { 933 primary->cb(dev, primary, primary->io.status); 934 io_cleanup(primary); 935 } 936 } 937 938 static int 939 restore_done(struct ftl_md *md) 940 { 941 if (-EAGAIN == md->io.status) { 942 /* Failed to read MD from primary region, try it from mirror. 943 * At the moment read the mirror entirely, (TODO) in the 944 * feature we can restore from primary and mirror region 945 * with finer granularity. 946 */ 947 948 if (has_mirror(md)) { 949 if (setup_mirror(md)) { 950 /* An error when setup the mirror */ 951 return -EIO; 952 } 953 954 /* Set callback and context in mirror */ 955 md->mirror->cb = restore_mirror_cb; 956 md->mirror->owner.private = md; 957 958 /* First persist the mirror */ 959 ftl_md_restore(md->mirror); 960 return -EAGAIN; 961 } else { 962 return -EIO; 963 } 964 } else if (0 == md->io.status && false == md->dev->sb->clean) { 965 if (has_mirror(md)) { 966 /* There was a dirty shutdown, synchronize primary to mirror */ 967 968 /* Set callback and context in the mirror */ 969 md->mirror->cb = restore_sync_cb; 970 md->mirror->owner.private = md; 971 972 /* First persist the mirror */ 973 ftl_md_persist(md->mirror); 974 return -EAGAIN; 975 } 976 } 977 978 return md->io.status; 979 } 980 981 static void 982 io_done(struct ftl_md *md) 983 { 984 int status; 985 986 if (md->io.op == FTL_MD_OP_RESTORE) { 987 status = restore_done(md); 988 } else { 989 status = md->io.status; 990 } 991 992 if (status != -EAGAIN) { 993 md->cb(md->dev, md, status); 994 io_cleanup(md); 995 } 996 } 997 998 void 999 ftl_md_restore(struct ftl_md *md) 1000 { 1001 if (0 == io_init(md, FTL_MD_OP_RESTORE)) { 1002 io_submit(md); 1003 } else { 1004 spdk_thread_send_msg(spdk_get_thread(), exception, md); 1005 } 1006 } 1007 1008 static int 1009 pattern_prepare(struct ftl_md *md, 1010 int data_pattern, union ftl_md_vss *vss_pattern) 1011 { 1012 void *data = md->io.data; 1013 uint64_t data_size = xfer_size(md); 1014 1015 memset(data, data_pattern, data_size); 1016 1017 if (md->io.md) { 1018 if (vss_pattern) { 1019 /* store the VSS pattern... */ 1020 ftl_md_vss_buf_init(md->io.md, ftl_md_xfer_blocks(md->dev), vss_pattern); 1021 } else { 1022 /* ...or default init VSS to 0 */ 1023 union ftl_md_vss vss = {0}; 1024 1025 vss.version.md_version = md->region->current.version; 1026 ftl_md_vss_buf_init(md->io.md, ftl_md_xfer_blocks(md->dev), &vss); 1027 } 1028 } 1029 1030 return 0; 1031 } 1032 1033 static void 1034 clear_mirror_cb(struct spdk_ftl_dev *dev, struct ftl_md *secondary, int status) 1035 { 1036 struct ftl_md *primary = secondary->owner.private; 1037 1038 if (status) { 1039 /* We got an error, stop persist procedure immediately */ 1040 primary->io.status = status; 1041 io_done(primary); 1042 } else { 1043 /* Now continue the persist procedure on the primary MD object */ 1044 if (0 == io_init(primary, FTL_MD_OP_CLEAR) && 1045 0 == pattern_prepare(primary, *(int *)secondary->io.data, 1046 secondary->io.md)) { 1047 io_submit(primary); 1048 } else { 1049 spdk_thread_send_msg(spdk_get_thread(), exception, primary); 1050 } 1051 } 1052 } 1053 1054 void 1055 ftl_md_clear(struct ftl_md *md, int data_pattern, union ftl_md_vss *vss_pattern) 1056 { 1057 if (has_mirror(md)) { 1058 if (setup_mirror(md)) { 1059 /* An error when setup the mirror */ 1060 spdk_thread_send_msg(spdk_get_thread(), exception, md); 1061 return; 1062 } 1063 1064 /* Set callback and context in mirror */ 1065 md->mirror->cb = clear_mirror_cb; 1066 md->mirror->owner.private = md; 1067 1068 /* First persist the mirror */ 1069 ftl_md_clear(md->mirror, data_pattern, vss_pattern); 1070 return; 1071 } 1072 1073 if (0 == io_init(md, FTL_MD_OP_CLEAR) && 0 == pattern_prepare(md, data_pattern, vss_pattern)) { 1074 io_submit(md); 1075 } else { 1076 spdk_thread_send_msg(spdk_get_thread(), exception, md); 1077 } 1078 } 1079 1080 const struct ftl_layout_region * 1081 ftl_md_get_region(struct ftl_md *md) 1082 { 1083 return md->region; 1084 } 1085 1086 int 1087 ftl_md_set_region(struct ftl_md *md, 1088 const struct ftl_layout_region *region) 1089 { 1090 assert(region->current.blocks <= md->data_blocks); 1091 md->region = region; 1092 1093 if (md->vss_data) { 1094 union ftl_md_vss vss = {0}; 1095 vss.version.md_version = region->current.version; 1096 ftl_md_vss_buf_init(md->vss_data, md->data_blocks, &vss); 1097 if (region->entry_size) { 1098 assert(md->entry_vss_dma_buf); 1099 ftl_md_vss_buf_init(md->entry_vss_dma_buf, region->entry_size, &vss); 1100 } 1101 } 1102 1103 if (has_mirror(md)) { 1104 return setup_mirror(md); 1105 } 1106 1107 return 0; 1108 } 1109 1110 int 1111 ftl_md_create_region_flags(struct spdk_ftl_dev *dev, int region_type) 1112 { 1113 int flags = FTL_MD_CREATE_SHM; 1114 1115 switch (region_type) { 1116 case FTL_LAYOUT_REGION_TYPE_SB: 1117 if (dev->conf.mode & SPDK_FTL_MODE_CREATE) { 1118 flags |= FTL_MD_CREATE_SHM_NEW; 1119 } 1120 break; 1121 1122 case FTL_LAYOUT_REGION_TYPE_BAND_MD: 1123 case FTL_LAYOUT_REGION_TYPE_NVC_MD: 1124 if (!ftl_fast_startup(dev)) { 1125 flags |= FTL_MD_CREATE_SHM_NEW; 1126 } 1127 break; 1128 case FTL_LAYOUT_REGION_TYPE_VALID_MAP: 1129 if (!ftl_fast_startup(dev) && !ftl_fast_recovery(dev)) { 1130 flags |= FTL_MD_CREATE_SHM_NEW; 1131 } 1132 break; 1133 default: 1134 return FTL_MD_CREATE_HEAP; 1135 } 1136 1137 return flags; 1138 } 1139 1140 int 1141 ftl_md_destroy_region_flags(struct spdk_ftl_dev *dev, int region_type) 1142 { 1143 switch (region_type) { 1144 case FTL_LAYOUT_REGION_TYPE_SB: 1145 case FTL_LAYOUT_REGION_TYPE_BAND_MD: 1146 case FTL_LAYOUT_REGION_TYPE_VALID_MAP: 1147 case FTL_LAYOUT_REGION_TYPE_NVC_MD: 1148 if (dev->conf.fast_shutdown) { 1149 return FTL_MD_DESTROY_SHM_KEEP; 1150 } 1151 break; 1152 1153 default: 1154 break; 1155 } 1156 return 0; 1157 } 1158 1159 int 1160 ftl_md_create_shm_flags(struct spdk_ftl_dev *dev) 1161 { 1162 int flags = FTL_MD_CREATE_SHM; 1163 1164 if (!ftl_fast_startup(dev) && !ftl_fast_recovery(dev)) { 1165 flags |= FTL_MD_CREATE_SHM_NEW; 1166 } 1167 return flags; 1168 } 1169 1170 int 1171 ftl_md_destroy_shm_flags(struct spdk_ftl_dev *dev) 1172 { 1173 return (dev->conf.fast_shutdown) ? FTL_MD_DESTROY_SHM_KEEP : 0; 1174 } 1175