1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (c) Intel Corporation. 3 * All rights reserved. 4 */ 5 6 #include "spdk/crc32.h" 7 #include "spdk/likely.h" 8 #include "spdk/util.h" 9 #include "spdk/ftl.h" 10 11 #include "ftl_band.h" 12 #include "ftl_io.h" 13 #include "ftl_core.h" 14 #include "ftl_reloc.h" 15 #include "ftl_debug.h" 16 17 /* TODO: define some signature for meta version */ 18 #define FTL_MD_VER 1 19 20 struct __attribute__((packed)) ftl_md_hdr { 21 /* Device instance */ 22 struct spdk_uuid uuid; 23 24 /* Meta version */ 25 uint8_t ver; 26 27 /* Sequence number */ 28 uint64_t seq; 29 30 /* CRC32 checksum */ 31 uint32_t checksum; 32 }; 33 34 /* End metadata layout stored on media (with all three being aligned to block size): */ 35 /* - header */ 36 /* - valid bitmap */ 37 /* - LBA map */ 38 struct __attribute__((packed)) ftl_tail_md { 39 struct ftl_md_hdr hdr; 40 41 /* Max number of blocks */ 42 uint64_t num_blocks; 43 44 uint8_t reserved[4059]; 45 }; 46 SPDK_STATIC_ASSERT(sizeof(struct ftl_tail_md) == FTL_BLOCK_SIZE, "Incorrect metadata size"); 47 48 struct __attribute__((packed)) ftl_head_md { 49 struct ftl_md_hdr hdr; 50 51 /* Number of defrag cycles */ 52 uint64_t wr_cnt; 53 54 /* Number of surfaced LBAs */ 55 uint64_t lba_cnt; 56 57 /* Transfer size */ 58 uint32_t xfer_size; 59 }; 60 61 size_t 62 ftl_tail_md_hdr_num_blocks(void) 63 { 64 return spdk_divide_round_up(sizeof(struct ftl_tail_md), FTL_BLOCK_SIZE); 65 } 66 67 size_t 68 ftl_vld_map_num_blocks(const struct spdk_ftl_dev *dev) 69 { 70 return spdk_divide_round_up(ftl_vld_map_size(dev), FTL_BLOCK_SIZE); 71 } 72 73 size_t 74 ftl_lba_map_num_blocks(const struct spdk_ftl_dev *dev) 75 { 76 return spdk_divide_round_up(ftl_get_num_blocks_in_band(dev) * sizeof(uint64_t), FTL_BLOCK_SIZE); 77 } 78 79 size_t 80 ftl_head_md_num_blocks(const struct spdk_ftl_dev *dev) 81 { 82 return dev->xfer_size; 83 } 84 85 size_t 86 ftl_tail_md_num_blocks(const struct spdk_ftl_dev *dev) 87 { 88 return spdk_divide_round_up(ftl_tail_md_hdr_num_blocks() + 89 ftl_vld_map_num_blocks(dev) + 90 ftl_lba_map_num_blocks(dev), 91 dev->xfer_size) * dev->xfer_size; 92 } 93 94 static uint64_t 95 ftl_band_tail_md_offset(const struct ftl_band *band) 96 { 97 return ftl_band_num_usable_blocks(band) - 98 ftl_tail_md_num_blocks(band->dev); 99 } 100 101 int 102 ftl_band_full(struct ftl_band *band, size_t offset) 103 { 104 return offset == ftl_band_tail_md_offset(band); 105 } 106 107 void 108 ftl_band_write_failed(struct ftl_band *band) 109 { 110 struct spdk_ftl_dev *dev = band->dev; 111 112 band->high_prio = 1; 113 114 ftl_reloc_add(dev->reloc, band, 0, ftl_get_num_blocks_in_band(dev), 1, true); 115 ftl_band_set_state(band, FTL_BAND_STATE_CLOSED); 116 } 117 118 static void 119 ftl_band_free_lba_map(struct ftl_band *band) 120 { 121 struct spdk_ftl_dev *dev = band->dev; 122 struct ftl_lba_map *lba_map = &band->lba_map; 123 124 assert(band->state == FTL_BAND_STATE_CLOSED || 125 band->state == FTL_BAND_STATE_FREE); 126 assert(lba_map->ref_cnt == 0); 127 assert(lba_map->map != NULL); 128 assert(!band->high_prio); 129 130 /* Verify that band's metadata is consistent with l2p */ 131 if (band->num_zones) { 132 assert(ftl_band_validate_md(band) == true); 133 } 134 135 spdk_mempool_put(dev->lba_pool, lba_map->dma_buf); 136 lba_map->map = NULL; 137 lba_map->dma_buf = NULL; 138 } 139 140 static void 141 _ftl_band_set_free(struct ftl_band *band) 142 { 143 struct spdk_ftl_dev *dev = band->dev; 144 struct ftl_band *lband, *prev; 145 146 /* Remove the band from the closed band list */ 147 LIST_REMOVE(band, list_entry); 148 149 /* Keep the list sorted by band's write count */ 150 LIST_FOREACH(lband, &dev->free_bands, list_entry) { 151 if (lband->wr_cnt > band->wr_cnt) { 152 LIST_INSERT_BEFORE(lband, band, list_entry); 153 break; 154 } 155 prev = lband; 156 } 157 158 if (!lband) { 159 if (LIST_EMPTY(&dev->free_bands)) { 160 LIST_INSERT_HEAD(&dev->free_bands, band, list_entry); 161 } else { 162 LIST_INSERT_AFTER(prev, band, list_entry); 163 } 164 } 165 166 #if defined(DEBUG) 167 prev = NULL; 168 LIST_FOREACH(lband, &dev->free_bands, list_entry) { 169 if (!prev) { 170 continue; 171 } 172 assert(prev->wr_cnt <= lband->wr_cnt); 173 } 174 #endif 175 dev->num_free++; 176 ftl_apply_limits(dev); 177 } 178 179 static void 180 _ftl_band_set_preparing(struct ftl_band *band) 181 { 182 struct spdk_ftl_dev *dev = band->dev; 183 184 /* Remove band from free list */ 185 LIST_REMOVE(band, list_entry); 186 187 band->wr_cnt++; 188 189 assert(dev->num_free > 0); 190 dev->num_free--; 191 192 ftl_apply_limits(dev); 193 } 194 195 static void 196 _ftl_band_set_closed(struct ftl_band *band) 197 { 198 struct spdk_ftl_dev *dev = band->dev; 199 200 /* Set the state as free_md() checks for that */ 201 band->state = FTL_BAND_STATE_CLOSED; 202 203 /* Free the lba map if there are no outstanding IOs */ 204 ftl_band_release_lba_map(band); 205 206 if (spdk_likely(band->num_zones)) { 207 LIST_INSERT_HEAD(&dev->shut_bands, band, list_entry); 208 } else { 209 LIST_REMOVE(band, list_entry); 210 } 211 } 212 213 static uint32_t 214 ftl_md_calc_crc(const struct ftl_md_hdr *hdr, size_t size) 215 { 216 size_t checkoff = offsetof(struct ftl_md_hdr, checksum); 217 size_t mdoff = checkoff + sizeof(hdr->checksum); 218 uint32_t crc; 219 220 crc = spdk_crc32c_update(hdr, checkoff, 0); 221 return spdk_crc32c_update((const char *)hdr + mdoff, size - mdoff, crc); 222 } 223 224 static void 225 ftl_set_md_hdr(struct ftl_band *band, struct ftl_md_hdr *hdr, size_t size) 226 { 227 hdr->seq = band->seq; 228 hdr->ver = FTL_MD_VER; 229 hdr->uuid = band->dev->uuid; 230 hdr->checksum = ftl_md_calc_crc(hdr, size); 231 } 232 233 static int 234 ftl_pack_head_md(struct ftl_band *band) 235 { 236 struct spdk_ftl_dev *dev = band->dev; 237 struct ftl_head_md *head = band->lba_map.dma_buf; 238 239 head->wr_cnt = band->wr_cnt; 240 head->lba_cnt = dev->num_lbas; 241 head->xfer_size = dev->xfer_size; 242 ftl_set_md_hdr(band, &head->hdr, sizeof(struct ftl_head_md)); 243 244 return FTL_MD_SUCCESS; 245 } 246 247 static int 248 ftl_pack_tail_md(struct ftl_band *band) 249 { 250 struct spdk_ftl_dev *dev = band->dev; 251 struct ftl_lba_map *lba_map = &band->lba_map; 252 struct ftl_tail_md *tail = lba_map->dma_buf; 253 void *vld_offset; 254 255 vld_offset = (char *)tail + ftl_tail_md_hdr_num_blocks() * FTL_BLOCK_SIZE; 256 257 /* Clear out the buffer */ 258 memset(tail, 0, ftl_tail_md_hdr_num_blocks() * FTL_BLOCK_SIZE); 259 tail->num_blocks = ftl_get_num_blocks_in_band(dev); 260 261 pthread_spin_lock(&lba_map->lock); 262 spdk_bit_array_store_mask(lba_map->vld, vld_offset); 263 pthread_spin_unlock(&lba_map->lock); 264 265 ftl_set_md_hdr(band, &tail->hdr, ftl_tail_md_num_blocks(dev) * FTL_BLOCK_SIZE); 266 267 return FTL_MD_SUCCESS; 268 } 269 270 static int 271 ftl_md_hdr_vld(struct spdk_ftl_dev *dev, const struct ftl_md_hdr *hdr, size_t size) 272 { 273 if (spdk_uuid_compare(&dev->uuid, &hdr->uuid) != 0) { 274 return FTL_MD_NO_MD; 275 } 276 277 if (hdr->ver != FTL_MD_VER) { 278 return FTL_MD_INVALID_VER; 279 } 280 281 if (ftl_md_calc_crc(hdr, size) != hdr->checksum) { 282 return FTL_MD_INVALID_CRC; 283 } 284 285 return FTL_MD_SUCCESS; 286 } 287 288 static int 289 ftl_unpack_tail_md(struct ftl_band *band) 290 { 291 struct spdk_ftl_dev *dev = band->dev; 292 void *vld_offset; 293 struct ftl_lba_map *lba_map = &band->lba_map; 294 struct ftl_tail_md *tail = lba_map->dma_buf; 295 int rc; 296 297 vld_offset = (char *)tail + ftl_tail_md_hdr_num_blocks() * FTL_BLOCK_SIZE; 298 299 rc = ftl_md_hdr_vld(dev, &tail->hdr, ftl_tail_md_num_blocks(dev) * FTL_BLOCK_SIZE); 300 if (rc) { 301 return rc; 302 } 303 304 /* 305 * When restoring from a dirty shutdown it's possible old tail meta wasn't yet cleared - 306 * band had saved head meta, but didn't manage to send erase to all zones. 307 * The already found tail md header is valid, but inconsistent with the head meta. Treat 308 * such a band as open/without valid tail md. 309 */ 310 if (band->seq != tail->hdr.seq) { 311 return FTL_MD_NO_MD; 312 } 313 314 if (tail->num_blocks != ftl_get_num_blocks_in_band(dev)) { 315 return FTL_MD_INVALID_SIZE; 316 } 317 318 spdk_bit_array_load_mask(lba_map->vld, vld_offset); 319 320 return FTL_MD_SUCCESS; 321 } 322 323 static int 324 ftl_unpack_head_md(struct ftl_band *band) 325 { 326 struct spdk_ftl_dev *dev = band->dev; 327 struct ftl_head_md *head = band->lba_map.dma_buf; 328 int rc; 329 330 rc = ftl_md_hdr_vld(dev, &head->hdr, sizeof(struct ftl_head_md)); 331 if (rc) { 332 return rc; 333 } 334 335 band->seq = head->hdr.seq; 336 band->wr_cnt = head->wr_cnt; 337 338 if (dev->global_md.num_lbas == 0) { 339 dev->global_md.num_lbas = head->lba_cnt; 340 } 341 342 if (dev->global_md.num_lbas != head->lba_cnt) { 343 return FTL_MD_INVALID_SIZE; 344 } 345 346 if (dev->xfer_size != head->xfer_size) { 347 return FTL_MD_INVALID_SIZE; 348 } 349 350 return FTL_MD_SUCCESS; 351 } 352 353 struct ftl_addr 354 ftl_band_tail_md_addr(struct ftl_band *band) 355 { 356 struct ftl_addr addr = {}; 357 struct ftl_zone *zone; 358 struct spdk_ftl_dev *dev = band->dev; 359 size_t xfer_size = dev->xfer_size; 360 size_t num_req = ftl_band_tail_md_offset(band) / xfer_size; 361 size_t i; 362 363 if (spdk_unlikely(!band->num_zones)) { 364 return ftl_to_addr(FTL_ADDR_INVALID); 365 } 366 367 /* Metadata should be aligned to xfer size */ 368 assert(ftl_band_tail_md_offset(band) % xfer_size == 0); 369 370 zone = CIRCLEQ_FIRST(&band->zones); 371 for (i = 0; i < num_req % band->num_zones; ++i) { 372 zone = ftl_band_next_zone(band, zone); 373 } 374 375 addr.offset = (num_req / band->num_zones) * xfer_size; 376 addr.offset += zone->info.zone_id; 377 378 return addr; 379 } 380 381 struct ftl_addr 382 ftl_band_head_md_addr(struct ftl_band *band) 383 { 384 if (spdk_unlikely(!band->num_zones)) { 385 return ftl_to_addr(FTL_ADDR_INVALID); 386 } 387 388 return ftl_to_addr(CIRCLEQ_FIRST(&band->zones)->info.zone_id); 389 } 390 391 void 392 ftl_band_set_state(struct ftl_band *band, enum ftl_band_state state) 393 { 394 switch (state) { 395 case FTL_BAND_STATE_FREE: 396 assert(band->state == FTL_BAND_STATE_CLOSED); 397 _ftl_band_set_free(band); 398 break; 399 400 case FTL_BAND_STATE_PREP: 401 assert(band->state == FTL_BAND_STATE_FREE); 402 _ftl_band_set_preparing(band); 403 break; 404 405 case FTL_BAND_STATE_CLOSED: 406 if (band->state != FTL_BAND_STATE_CLOSED) { 407 assert(band->state == FTL_BAND_STATE_CLOSING || band->high_prio); 408 _ftl_band_set_closed(band); 409 } 410 break; 411 412 default: 413 break; 414 } 415 416 band->state = state; 417 } 418 419 void 420 ftl_band_set_addr(struct ftl_band *band, uint64_t lba, struct ftl_addr addr) 421 { 422 struct ftl_lba_map *lba_map = &band->lba_map; 423 uint64_t offset; 424 425 assert(lba != FTL_LBA_INVALID); 426 427 offset = ftl_band_block_offset_from_addr(band, addr); 428 pthread_spin_lock(&lba_map->lock); 429 430 lba_map->num_vld++; 431 lba_map->map[offset] = lba; 432 spdk_bit_array_set(lba_map->vld, offset); 433 434 pthread_spin_unlock(&lba_map->lock); 435 } 436 437 size_t 438 ftl_band_age(const struct ftl_band *band) 439 { 440 return (size_t)(band->dev->seq - band->seq); 441 } 442 443 size_t 444 ftl_band_num_usable_blocks(const struct ftl_band *band) 445 { 446 return band->num_zones * ftl_get_num_blocks_in_zone(band->dev); 447 } 448 449 size_t 450 ftl_band_user_blocks_left(const struct ftl_band *band, size_t offset) 451 { 452 size_t tail_md_offset = ftl_band_tail_md_offset(band); 453 454 if (spdk_unlikely(offset <= ftl_head_md_num_blocks(band->dev))) { 455 return ftl_band_user_blocks(band); 456 } 457 458 if (spdk_unlikely(offset > tail_md_offset)) { 459 return 0; 460 } 461 462 return tail_md_offset - offset; 463 } 464 465 size_t 466 ftl_band_user_blocks(const struct ftl_band *band) 467 { 468 return ftl_band_num_usable_blocks(band) - 469 ftl_head_md_num_blocks(band->dev) - 470 ftl_tail_md_num_blocks(band->dev); 471 } 472 473 struct ftl_band * 474 ftl_band_from_addr(struct spdk_ftl_dev *dev, struct ftl_addr addr) 475 { 476 size_t band_id = ftl_addr_get_band(dev, addr); 477 478 assert(band_id < ftl_get_num_bands(dev)); 479 return &dev->bands[band_id]; 480 } 481 482 struct ftl_zone * 483 ftl_band_zone_from_addr(struct ftl_band *band, struct ftl_addr addr) 484 { 485 size_t pu_id = ftl_addr_get_punit(band->dev, addr); 486 487 assert(pu_id < ftl_get_num_punits(band->dev)); 488 return &band->zone_buf[pu_id]; 489 } 490 491 uint64_t 492 ftl_band_block_offset_from_addr(struct ftl_band *band, struct ftl_addr addr) 493 { 494 assert(ftl_addr_get_band(band->dev, addr) == band->id); 495 assert(ftl_addr_get_punit(band->dev, addr) < ftl_get_num_punits(band->dev)); 496 return addr.offset % ftl_get_num_blocks_in_band(band->dev); 497 } 498 499 struct ftl_addr 500 ftl_band_next_xfer_addr(struct ftl_band *band, struct ftl_addr addr, size_t num_blocks) 501 { 502 struct spdk_ftl_dev *dev = band->dev; 503 struct ftl_zone *zone; 504 size_t num_xfers, num_stripes; 505 uint64_t offset; 506 507 assert(ftl_addr_get_band(dev, addr) == band->id); 508 509 offset = ftl_addr_get_zone_offset(dev, addr); 510 zone = ftl_band_zone_from_addr(band, addr); 511 512 num_blocks += (offset % dev->xfer_size); 513 offset -= (offset % dev->xfer_size); 514 515 #if defined(DEBUG) 516 /* Check that the number of zones has not been changed */ 517 struct ftl_zone *_zone; 518 size_t _num_zones = 0; 519 CIRCLEQ_FOREACH(_zone, &band->zones, circleq) { 520 if (spdk_likely(_zone->info.state != SPDK_BDEV_ZONE_STATE_OFFLINE)) { 521 _num_zones++; 522 } 523 } 524 assert(band->num_zones == _num_zones); 525 #endif 526 assert(band->num_zones != 0); 527 num_stripes = (num_blocks / dev->xfer_size) / band->num_zones; 528 offset += num_stripes * dev->xfer_size; 529 num_blocks -= num_stripes * dev->xfer_size * band->num_zones; 530 531 if (offset > ftl_get_num_blocks_in_zone(dev)) { 532 return ftl_to_addr(FTL_ADDR_INVALID); 533 } 534 535 num_xfers = num_blocks / dev->xfer_size; 536 for (size_t i = 0; i < num_xfers; ++i) { 537 /* When the last zone is reached the block part of the address */ 538 /* needs to be increased by xfer_size */ 539 if (ftl_band_zone_is_last(band, zone)) { 540 offset += dev->xfer_size; 541 if (offset > ftl_get_num_blocks_in_zone(dev)) { 542 return ftl_to_addr(FTL_ADDR_INVALID); 543 } 544 } 545 546 zone = ftl_band_next_operational_zone(band, zone); 547 assert(zone); 548 549 num_blocks -= dev->xfer_size; 550 } 551 552 if (num_blocks) { 553 offset += num_blocks; 554 if (offset > ftl_get_num_blocks_in_zone(dev)) { 555 return ftl_to_addr(FTL_ADDR_INVALID); 556 } 557 } 558 559 addr.offset = zone->info.zone_id + offset; 560 return addr; 561 } 562 563 static size_t 564 ftl_xfer_offset_from_addr(struct ftl_band *band, struct ftl_addr addr) 565 { 566 struct ftl_zone *zone, *current_zone; 567 unsigned int punit_offset = 0; 568 size_t num_stripes, xfer_size = band->dev->xfer_size; 569 uint64_t offset; 570 571 assert(ftl_addr_get_band(band->dev, addr) == band->id); 572 573 offset = ftl_addr_get_zone_offset(band->dev, addr); 574 num_stripes = (offset / xfer_size) * band->num_zones; 575 576 current_zone = ftl_band_zone_from_addr(band, addr); 577 CIRCLEQ_FOREACH(zone, &band->zones, circleq) { 578 if (current_zone == zone) { 579 break; 580 } 581 punit_offset++; 582 } 583 584 return xfer_size * (num_stripes + punit_offset) + offset % xfer_size; 585 } 586 587 struct ftl_addr 588 ftl_band_addr_from_block_offset(struct ftl_band *band, uint64_t block_off) 589 { 590 struct ftl_addr addr = { .offset = 0 }; 591 592 addr.offset = block_off + band->id * ftl_get_num_blocks_in_band(band->dev); 593 return addr; 594 } 595 596 struct ftl_addr 597 ftl_band_next_addr(struct ftl_band *band, struct ftl_addr addr, size_t offset) 598 { 599 uint64_t block_off = ftl_band_block_offset_from_addr(band, addr); 600 return ftl_band_addr_from_block_offset(band, block_off + offset); 601 } 602 603 void 604 ftl_band_acquire_lba_map(struct ftl_band *band) 605 { 606 assert(band->lba_map.map != NULL); 607 band->lba_map.ref_cnt++; 608 } 609 610 int 611 ftl_band_alloc_lba_map(struct ftl_band *band) 612 { 613 struct spdk_ftl_dev *dev = band->dev; 614 struct ftl_lba_map *lba_map = &band->lba_map; 615 616 assert(lba_map->ref_cnt == 0); 617 assert(lba_map->map == NULL); 618 619 lba_map->dma_buf = spdk_mempool_get(dev->lba_pool); 620 621 if (!lba_map->dma_buf) { 622 return -1; 623 } 624 625 memset(lba_map->dma_buf, 0, ftl_lba_map_pool_elem_size(band->dev)); 626 627 lba_map->map = (uint64_t *)((char *)lba_map->dma_buf + FTL_BLOCK_SIZE * 628 (ftl_tail_md_hdr_num_blocks() + ftl_vld_map_num_blocks(dev))); 629 630 lba_map->segments = (char *)lba_map->dma_buf + ftl_tail_md_num_blocks(dev) * FTL_BLOCK_SIZE; 631 632 ftl_band_acquire_lba_map(band); 633 return 0; 634 } 635 636 void 637 ftl_band_release_lba_map(struct ftl_band *band) 638 { 639 struct ftl_lba_map *lba_map = &band->lba_map; 640 641 assert(lba_map->map != NULL); 642 assert(lba_map->ref_cnt > 0); 643 lba_map->ref_cnt--; 644 645 if (lba_map->ref_cnt == 0) { 646 ftl_band_free_lba_map(band); 647 } 648 } 649 650 static void 651 ftl_read_md_cb(struct ftl_io *io, void *arg, int status) 652 { 653 struct ftl_md_io *md_io = (struct ftl_md_io *)io; 654 655 if (!status) { 656 status = md_io->pack_fn(md_io->io.band); 657 } else { 658 status = FTL_MD_IO_FAILURE; 659 } 660 661 md_io->cb_fn(io, md_io->cb_ctx, status); 662 } 663 664 static struct ftl_md_io * 665 ftl_io_init_md_read(struct spdk_ftl_dev *dev, struct ftl_addr addr, 666 struct ftl_band *band, size_t num_blocks, void *buf, 667 ftl_io_fn fn, ftl_md_pack_fn pack_fn, ftl_io_fn cb_fn, void *cb_ctx) 668 { 669 struct ftl_md_io *io; 670 struct ftl_io_init_opts opts = { 671 .dev = dev, 672 .io = NULL, 673 .band = band, 674 .size = sizeof(*io), 675 .flags = FTL_IO_MD | FTL_IO_PHYSICAL_MODE, 676 .type = FTL_IO_READ, 677 .num_blocks = num_blocks, 678 .cb_fn = fn, 679 .iovs = { 680 { 681 .iov_base = buf, 682 .iov_len = num_blocks * FTL_BLOCK_SIZE, 683 } 684 }, 685 .iovcnt = 1, 686 }; 687 688 io = (struct ftl_md_io *)ftl_io_init_internal(&opts); 689 if (!io) { 690 return NULL; 691 } 692 693 io->io.addr = addr; 694 io->pack_fn = pack_fn; 695 io->cb_fn = cb_fn; 696 io->cb_ctx = cb_ctx; 697 698 return io; 699 } 700 701 static struct ftl_io * 702 ftl_io_init_md_write(struct spdk_ftl_dev *dev, struct ftl_band *band, 703 void *data, size_t num_blocks, ftl_io_fn cb) 704 { 705 struct ftl_io_init_opts opts = { 706 .dev = dev, 707 .io = NULL, 708 .band = band, 709 .size = sizeof(struct ftl_io), 710 .flags = FTL_IO_MD | FTL_IO_PHYSICAL_MODE, 711 .type = FTL_IO_WRITE, 712 .num_blocks = num_blocks, 713 .cb_fn = cb, 714 .iovs = { 715 { 716 .iov_base = data, 717 .iov_len = num_blocks * FTL_BLOCK_SIZE, 718 } 719 }, 720 .iovcnt = 1, 721 .md = NULL, 722 }; 723 724 return ftl_io_init_internal(&opts); 725 } 726 727 static int 728 ftl_band_write_md(struct ftl_band *band, size_t num_blocks, 729 ftl_md_pack_fn md_fn, ftl_io_fn cb) 730 { 731 struct spdk_ftl_dev *dev = band->dev; 732 struct ftl_io *io; 733 734 io = ftl_io_init_md_write(dev, band, band->lba_map.dma_buf, num_blocks, cb); 735 if (!io) { 736 return -ENOMEM; 737 } 738 739 md_fn(band); 740 741 ftl_io_write(io); 742 return 0; 743 } 744 745 void 746 ftl_band_md_clear(struct ftl_band *band) 747 { 748 band->seq = 0; 749 band->wr_cnt = 0; 750 band->lba_map.num_vld = 0; 751 band->lba_map.map = NULL; 752 } 753 754 int 755 ftl_band_write_head_md(struct ftl_band *band, ftl_io_fn cb) 756 { 757 return ftl_band_write_md(band, ftl_head_md_num_blocks(band->dev), 758 ftl_pack_head_md, cb); 759 } 760 761 int 762 ftl_band_write_tail_md(struct ftl_band *band, ftl_io_fn cb) 763 { 764 return ftl_band_write_md(band, ftl_tail_md_num_blocks(band->dev), 765 ftl_pack_tail_md, cb); 766 } 767 768 static struct ftl_addr 769 ftl_band_lba_map_addr(struct ftl_band *band, size_t offset) 770 { 771 return ftl_band_next_xfer_addr(band, band->tail_md_addr, 772 ftl_tail_md_hdr_num_blocks() + 773 ftl_vld_map_num_blocks(band->dev) + 774 offset); 775 } 776 777 static int 778 ftl_band_read_md(struct ftl_band *band, size_t num_blocks, struct ftl_addr start_addr, 779 void *buf, ftl_io_fn fn, ftl_md_pack_fn pack_fn, ftl_io_fn cb_fn, void *cb_ctx) 780 { 781 struct spdk_ftl_dev *dev = band->dev; 782 struct ftl_md_io *io; 783 784 if (spdk_unlikely(!band->num_zones)) { 785 return -ENOENT; 786 } 787 788 io = ftl_io_init_md_read(dev, start_addr, band, num_blocks, buf, fn, pack_fn, cb_fn, cb_ctx); 789 if (!io) { 790 return -ENOMEM; 791 } 792 793 ftl_io_read((struct ftl_io *)io); 794 return 0; 795 } 796 797 int 798 ftl_band_read_tail_md(struct ftl_band *band, struct ftl_addr addr, ftl_io_fn cb_fn, void *cb_ctx) 799 { 800 return ftl_band_read_md(band, ftl_tail_md_num_blocks(band->dev), addr, band->lba_map.dma_buf, 801 ftl_read_md_cb, ftl_unpack_tail_md, cb_fn, cb_ctx); 802 } 803 804 static size_t 805 ftl_lba_map_request_segment_done(struct ftl_lba_map_request *request, size_t offset, 806 size_t num_segments) 807 { 808 size_t i, num_done = 0; 809 810 for (i = offset; i < offset + num_segments; ++i) { 811 if (spdk_bit_array_get(request->segments, i)) { 812 spdk_bit_array_clear(request->segments, offset); 813 num_done++; 814 } 815 } 816 817 assert(request->num_pending >= num_done); 818 request->num_pending -= num_done; 819 820 return num_done; 821 } 822 823 static void 824 ftl_lba_map_set_segment_state(struct ftl_lba_map *lba_map, size_t offset, size_t num_segments, 825 enum ftl_lba_map_seg_state state) 826 { 827 size_t i; 828 829 for (i = offset; i < offset + num_segments; ++i) { 830 lba_map->segments[i] = state; 831 } 832 } 833 834 static void 835 ftl_lba_map_request_free(struct spdk_ftl_dev *dev, struct ftl_lba_map_request *request) 836 { 837 spdk_bit_array_clear_mask(request->segments); 838 spdk_mempool_put(dev->lba_request_pool, request); 839 } 840 841 static void 842 ftl_process_lba_map_requests(struct spdk_ftl_dev *dev, struct ftl_lba_map *lba_map, size_t offset, 843 size_t num_segments, int status) 844 { 845 struct ftl_lba_map_request *request, *trequest; 846 size_t num_done; 847 848 LIST_FOREACH_SAFE(request, &lba_map->request_list, list_entry, trequest) { 849 num_done = ftl_lba_map_request_segment_done(request, offset, num_segments); 850 if (request->num_pending == 0 || (status && num_done)) { 851 request->cb(NULL, request->cb_ctx, status); 852 LIST_REMOVE(request, list_entry); 853 ftl_lba_map_request_free(dev, request); 854 } 855 } 856 } 857 858 static size_t 859 ftl_lba_map_offset_from_addr(struct ftl_band *band, struct ftl_addr addr) 860 { 861 size_t offset; 862 struct ftl_addr start_addr = ftl_band_lba_map_addr(band, 0); 863 864 offset = ftl_xfer_offset_from_addr(band, addr) - ftl_xfer_offset_from_addr(band, start_addr); 865 assert(offset < ftl_lba_map_num_blocks(band->dev)); 866 867 return offset; 868 } 869 870 static void 871 ftl_read_lba_map_cb(struct ftl_io *io, void *arg, int status) 872 { 873 struct ftl_lba_map *lba_map = &io->band->lba_map; 874 uint64_t block_off; 875 876 block_off = ftl_lba_map_offset_from_addr(io->band, io->addr); 877 assert(block_off + io->num_blocks <= ftl_lba_map_num_blocks(io->dev)); 878 879 if (!status) { 880 ftl_lba_map_set_segment_state(lba_map, block_off, io->num_blocks, 881 FTL_LBA_MAP_SEG_CACHED); 882 } 883 884 ftl_process_lba_map_requests(io->dev, lba_map, block_off, io->num_blocks, status); 885 } 886 887 static struct ftl_lba_map_request * 888 ftl_lba_map_alloc_request(struct ftl_band *band, size_t offset, size_t num_segments, 889 ftl_io_fn cb, void *cb_ctx) 890 { 891 struct ftl_lba_map_request *request; 892 struct spdk_ftl_dev *dev = band->dev; 893 size_t i; 894 895 request = spdk_mempool_get(dev->lba_request_pool); 896 if (!request) { 897 return NULL; 898 } 899 900 request->cb = cb; 901 request->cb_ctx = cb_ctx; 902 request->num_pending = num_segments; 903 904 for (i = offset; i < offset + num_segments; ++i) { 905 spdk_bit_array_set(request->segments, i); 906 } 907 908 return request; 909 } 910 911 static size_t 912 ftl_lba_map_num_clear_segments(struct ftl_lba_map *lba_map, 913 size_t offset, size_t num_segments) 914 { 915 size_t i, cnt = 0; 916 917 for (i = offset; i < offset + num_segments; ++i) { 918 if (lba_map->segments[i] != FTL_LBA_MAP_SEG_CLEAR) { 919 break; 920 } 921 cnt++; 922 } 923 924 return cnt; 925 } 926 927 int 928 ftl_band_read_lba_map(struct ftl_band *band, size_t offset, size_t lba_cnt, 929 ftl_io_fn cb_fn, void *cb_ctx) 930 { 931 size_t num_blocks, block_off, num_read, num_segments; 932 struct ftl_lba_map *lba_map = &band->lba_map; 933 struct ftl_lba_map_request *request; 934 int rc = 0; 935 936 block_off = offset / FTL_NUM_LBA_IN_BLOCK; 937 num_segments = spdk_divide_round_up(offset + lba_cnt, FTL_NUM_LBA_IN_BLOCK); 938 num_blocks = num_segments - block_off; 939 assert(block_off + num_blocks <= ftl_lba_map_num_blocks(band->dev)); 940 941 request = ftl_lba_map_alloc_request(band, block_off, num_blocks, cb_fn, cb_ctx); 942 if (!request) { 943 return -ENOMEM; 944 } 945 946 while (num_blocks) { 947 if (lba_map->segments[block_off] != FTL_LBA_MAP_SEG_CLEAR) { 948 if (lba_map->segments[block_off] == FTL_LBA_MAP_SEG_CACHED) { 949 ftl_lba_map_request_segment_done(request, block_off, 1); 950 } 951 num_blocks--; 952 block_off++; 953 continue; 954 } 955 956 num_read = ftl_lba_map_num_clear_segments(lba_map, block_off, num_blocks); 957 ftl_lba_map_set_segment_state(lba_map, block_off, num_read, 958 FTL_LBA_MAP_SEG_PENDING); 959 960 rc = ftl_band_read_md(band, num_read, ftl_band_lba_map_addr(band, block_off), 961 (char *)band->lba_map.map + block_off * FTL_BLOCK_SIZE, 962 ftl_read_lba_map_cb, NULL, cb_fn, cb_ctx); 963 if (rc) { 964 ftl_lba_map_request_free(band->dev, request); 965 return rc; 966 } 967 968 assert(num_blocks >= num_read); 969 num_blocks -= num_read; 970 block_off += num_read; 971 } 972 973 if (request->num_pending) { 974 LIST_INSERT_HEAD(&lba_map->request_list, request, list_entry); 975 } else { 976 cb_fn(NULL, cb_ctx, 0); 977 ftl_lba_map_request_free(band->dev, request); 978 } 979 980 return rc; 981 } 982 983 int 984 ftl_band_read_head_md(struct ftl_band *band, ftl_io_fn cb_fn, void *cb_ctx) 985 { 986 return ftl_band_read_md(band, 987 ftl_head_md_num_blocks(band->dev), 988 ftl_band_head_md_addr(band), 989 band->lba_map.dma_buf, 990 ftl_read_md_cb, 991 ftl_unpack_head_md, 992 cb_fn, 993 cb_ctx); 994 } 995 996 void 997 ftl_band_remove_zone(struct ftl_band *band, struct ftl_zone *zone) 998 { 999 CIRCLEQ_REMOVE(&band->zones, zone, circleq); 1000 band->num_zones--; 1001 } 1002 1003 int 1004 ftl_band_write_prep(struct ftl_band *band) 1005 { 1006 struct spdk_ftl_dev *dev = band->dev; 1007 1008 if (ftl_band_alloc_lba_map(band)) { 1009 return -1; 1010 } 1011 1012 band->seq = ++dev->seq; 1013 return 0; 1014 } 1015 1016 struct ftl_zone * 1017 ftl_band_next_operational_zone(struct ftl_band *band, struct ftl_zone *zone) 1018 { 1019 struct ftl_zone *result = NULL; 1020 struct ftl_zone *entry; 1021 1022 if (spdk_unlikely(!band->num_zones)) { 1023 return NULL; 1024 } 1025 1026 /* Erasing band may fail after it was assigned to wptr. */ 1027 /* In such a case zone is no longer in band->zones queue. */ 1028 if (spdk_likely(zone->info.state != SPDK_BDEV_ZONE_STATE_OFFLINE)) { 1029 result = ftl_band_next_zone(band, zone); 1030 } else { 1031 CIRCLEQ_FOREACH_REVERSE(entry, &band->zones, circleq) { 1032 if (entry->info.zone_id > zone->info.zone_id) { 1033 result = entry; 1034 } else { 1035 if (!result) { 1036 result = CIRCLEQ_FIRST(&band->zones); 1037 } 1038 break; 1039 } 1040 } 1041 } 1042 1043 return result; 1044 } 1045 1046 void 1047 ftl_band_clear_lba_map(struct ftl_band *band) 1048 { 1049 struct ftl_lba_map *lba_map = &band->lba_map; 1050 size_t num_segments; 1051 1052 spdk_bit_array_clear_mask(lba_map->vld); 1053 memset(lba_map->map, 0, ftl_lba_map_num_blocks(band->dev) * FTL_BLOCK_SIZE); 1054 1055 /* For open band all lba map segments are already cached */ 1056 assert(band->state == FTL_BAND_STATE_PREP); 1057 num_segments = spdk_divide_round_up(ftl_get_num_blocks_in_band(band->dev), FTL_NUM_LBA_IN_BLOCK); 1058 ftl_lba_map_set_segment_state(&band->lba_map, 0, num_segments, FTL_LBA_MAP_SEG_CACHED); 1059 1060 lba_map->num_vld = 0; 1061 } 1062 1063 size_t 1064 ftl_lba_map_pool_elem_size(struct spdk_ftl_dev *dev) 1065 { 1066 /* Map pool element holds the whole tail md + segments map */ 1067 return ftl_tail_md_num_blocks(dev) * FTL_BLOCK_SIZE + 1068 spdk_divide_round_up(ftl_get_num_blocks_in_band(dev), FTL_NUM_LBA_IN_BLOCK); 1069 } 1070