1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) Intel Corporation. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include "spdk/crc32.h" 35 #include "spdk/likely.h" 36 #include "spdk/util.h" 37 #include "spdk/ftl.h" 38 39 #include "ftl_band.h" 40 #include "ftl_io.h" 41 #include "ftl_core.h" 42 #include "ftl_reloc.h" 43 #include "ftl_debug.h" 44 45 /* TODO: define some signature for meta version */ 46 #define FTL_MD_VER 1 47 48 struct __attribute__((packed)) ftl_md_hdr { 49 /* Device instance */ 50 struct spdk_uuid uuid; 51 52 /* Meta version */ 53 uint8_t ver; 54 55 /* Sequence number */ 56 uint64_t seq; 57 58 /* CRC32 checksum */ 59 uint32_t checksum; 60 }; 61 62 /* End metadata layout stored on media (with all three being aligned to block size): */ 63 /* - header */ 64 /* - valid bitmap */ 65 /* - LBA map */ 66 struct __attribute__((packed)) ftl_tail_md { 67 struct ftl_md_hdr hdr; 68 69 /* Max number of lbks */ 70 uint64_t num_lbks; 71 72 uint8_t reserved[4059]; 73 }; 74 SPDK_STATIC_ASSERT(sizeof(struct ftl_tail_md) == FTL_BLOCK_SIZE, "Incorrect metadata size"); 75 76 struct __attribute__((packed)) ftl_head_md { 77 struct ftl_md_hdr hdr; 78 79 /* Number of defrag cycles */ 80 uint64_t wr_cnt; 81 82 /* Number of surfaced LBAs */ 83 uint64_t lba_cnt; 84 85 /* Transfer size */ 86 uint32_t xfer_size; 87 }; 88 89 size_t 90 ftl_tail_md_hdr_num_lbks(void) 91 { 92 return spdk_divide_round_up(sizeof(struct ftl_tail_md), FTL_BLOCK_SIZE); 93 } 94 95 size_t 96 ftl_vld_map_num_lbks(const struct spdk_ftl_dev *dev) 97 { 98 return spdk_divide_round_up(ftl_vld_map_size(dev), FTL_BLOCK_SIZE); 99 } 100 101 size_t 102 ftl_lba_map_num_lbks(const struct spdk_ftl_dev *dev) 103 { 104 return spdk_divide_round_up(ftl_num_band_lbks(dev) * sizeof(uint64_t), FTL_BLOCK_SIZE); 105 } 106 107 size_t 108 ftl_head_md_num_lbks(const struct spdk_ftl_dev *dev) 109 { 110 return dev->xfer_size; 111 } 112 113 size_t 114 ftl_tail_md_num_lbks(const struct spdk_ftl_dev *dev) 115 { 116 return spdk_divide_round_up(ftl_tail_md_hdr_num_lbks() + 117 ftl_vld_map_num_lbks(dev) + 118 ftl_lba_map_num_lbks(dev), 119 dev->xfer_size) * dev->xfer_size; 120 } 121 122 static uint64_t 123 ftl_band_tail_md_offset(const struct ftl_band *band) 124 { 125 return ftl_band_num_usable_lbks(band) - 126 ftl_tail_md_num_lbks(band->dev); 127 } 128 129 int 130 ftl_band_full(struct ftl_band *band, size_t offset) 131 { 132 return offset == ftl_band_tail_md_offset(band); 133 } 134 135 void 136 ftl_band_write_failed(struct ftl_band *band) 137 { 138 struct spdk_ftl_dev *dev = band->dev; 139 140 band->high_prio = 1; 141 142 if (!dev->df_band) { 143 dev->df_band = band; 144 } 145 146 ftl_reloc_add(dev->reloc, band, 0, ftl_num_band_lbks(dev), 1); 147 ftl_band_set_state(band, FTL_BAND_STATE_CLOSED); 148 } 149 150 static void 151 ftl_band_free_lba_map(struct ftl_band *band) 152 { 153 struct spdk_ftl_dev *dev = band->dev; 154 struct ftl_lba_map *lba_map = &band->lba_map; 155 156 assert(band->state == FTL_BAND_STATE_CLOSED || 157 band->state == FTL_BAND_STATE_FREE); 158 assert(lba_map->ref_cnt == 0); 159 assert(lba_map->map != NULL); 160 assert(!band->high_prio); 161 162 /* Verify that band's metadata is consistent with l2p */ 163 if (band->num_chunks) { 164 assert(ftl_band_validate_md(band) == true); 165 } 166 167 spdk_mempool_put(dev->lba_pool, lba_map->dma_buf); 168 lba_map->map = NULL; 169 lba_map->dma_buf = NULL; 170 } 171 172 static void 173 _ftl_band_set_free(struct ftl_band *band) 174 { 175 struct spdk_ftl_dev *dev = band->dev; 176 struct ftl_band *lband, *prev; 177 178 if (band == dev->df_band) { 179 dev->df_band = NULL; 180 } 181 182 /* Remove the band from the closed band list */ 183 LIST_REMOVE(band, list_entry); 184 185 /* Keep the list sorted by band's write count */ 186 LIST_FOREACH(lband, &dev->free_bands, list_entry) { 187 if (lband->wr_cnt > band->wr_cnt) { 188 LIST_INSERT_BEFORE(lband, band, list_entry); 189 break; 190 } 191 prev = lband; 192 } 193 194 if (!lband) { 195 if (LIST_EMPTY(&dev->free_bands)) { 196 LIST_INSERT_HEAD(&dev->free_bands, band, list_entry); 197 } else { 198 LIST_INSERT_AFTER(prev, band, list_entry); 199 } 200 } 201 202 #if defined(DEBUG) 203 prev = NULL; 204 LIST_FOREACH(lband, &dev->free_bands, list_entry) { 205 if (!prev) { 206 continue; 207 } 208 assert(prev->wr_cnt <= lband->wr_cnt); 209 } 210 #endif 211 dev->num_free++; 212 ftl_apply_limits(dev); 213 } 214 215 static void 216 _ftl_band_set_preparing(struct ftl_band *band) 217 { 218 struct spdk_ftl_dev *dev = band->dev; 219 220 /* Remove band from free list */ 221 LIST_REMOVE(band, list_entry); 222 223 band->wr_cnt++; 224 225 assert(dev->num_free > 0); 226 dev->num_free--; 227 228 ftl_apply_limits(dev); 229 } 230 231 static void 232 _ftl_band_set_closed(struct ftl_band *band) 233 { 234 struct spdk_ftl_dev *dev = band->dev; 235 struct ftl_chunk *chunk; 236 237 /* Set the state as free_md() checks for that */ 238 band->state = FTL_BAND_STATE_CLOSED; 239 240 /* Free the lba map if there are no outstanding IOs */ 241 ftl_band_release_lba_map(band); 242 243 if (spdk_likely(band->num_chunks)) { 244 LIST_INSERT_HEAD(&dev->shut_bands, band, list_entry); 245 CIRCLEQ_FOREACH(chunk, &band->chunks, circleq) { 246 chunk->state = FTL_CHUNK_STATE_CLOSED; 247 } 248 } else { 249 LIST_REMOVE(band, list_entry); 250 } 251 } 252 253 static uint32_t 254 ftl_md_calc_crc(const struct ftl_md_hdr *hdr, size_t size) 255 { 256 size_t checkoff = offsetof(struct ftl_md_hdr, checksum); 257 size_t mdoff = checkoff + sizeof(hdr->checksum); 258 uint32_t crc; 259 260 crc = spdk_crc32c_update(hdr, checkoff, 0); 261 return spdk_crc32c_update((const char *)hdr + mdoff, size - mdoff, crc); 262 } 263 264 static void 265 ftl_set_md_hdr(struct ftl_band *band, struct ftl_md_hdr *hdr, size_t size) 266 { 267 hdr->seq = band->seq; 268 hdr->ver = FTL_MD_VER; 269 hdr->uuid = band->dev->uuid; 270 hdr->checksum = ftl_md_calc_crc(hdr, size); 271 } 272 273 static int 274 ftl_pack_head_md(struct ftl_band *band) 275 { 276 struct spdk_ftl_dev *dev = band->dev; 277 struct ftl_head_md *head = band->lba_map.dma_buf; 278 279 head->wr_cnt = band->wr_cnt; 280 head->lba_cnt = dev->num_lbas; 281 head->xfer_size = dev->xfer_size; 282 ftl_set_md_hdr(band, &head->hdr, sizeof(struct ftl_head_md)); 283 284 return FTL_MD_SUCCESS; 285 } 286 287 static int 288 ftl_pack_tail_md(struct ftl_band *band) 289 { 290 struct spdk_ftl_dev *dev = band->dev; 291 struct ftl_lba_map *lba_map = &band->lba_map; 292 struct ftl_tail_md *tail = lba_map->dma_buf; 293 void *vld_offset; 294 295 vld_offset = (char *)tail + ftl_tail_md_hdr_num_lbks() * FTL_BLOCK_SIZE; 296 297 /* Clear out the buffer */ 298 memset(tail, 0, ftl_tail_md_hdr_num_lbks() * FTL_BLOCK_SIZE); 299 tail->num_lbks = ftl_num_band_lbks(dev); 300 301 pthread_spin_lock(&lba_map->lock); 302 spdk_bit_array_store_mask(lba_map->vld, vld_offset); 303 pthread_spin_unlock(&lba_map->lock); 304 305 ftl_set_md_hdr(band, &tail->hdr, ftl_tail_md_num_lbks(dev) * FTL_BLOCK_SIZE); 306 307 return FTL_MD_SUCCESS; 308 } 309 310 static int 311 ftl_md_hdr_vld(struct spdk_ftl_dev *dev, const struct ftl_md_hdr *hdr, size_t size) 312 { 313 if (spdk_uuid_compare(&dev->uuid, &hdr->uuid) != 0) { 314 return FTL_MD_NO_MD; 315 } 316 317 if (hdr->ver != FTL_MD_VER) { 318 return FTL_MD_INVALID_VER; 319 } 320 321 if (ftl_md_calc_crc(hdr, size) != hdr->checksum) { 322 return FTL_MD_INVALID_CRC; 323 } 324 325 return FTL_MD_SUCCESS; 326 } 327 328 static int 329 ftl_unpack_tail_md(struct ftl_band *band) 330 { 331 struct spdk_ftl_dev *dev = band->dev; 332 void *vld_offset; 333 struct ftl_lba_map *lba_map = &band->lba_map; 334 struct ftl_tail_md *tail = lba_map->dma_buf; 335 int rc; 336 337 vld_offset = (char *)tail + ftl_tail_md_hdr_num_lbks() * FTL_BLOCK_SIZE; 338 339 rc = ftl_md_hdr_vld(dev, &tail->hdr, ftl_tail_md_num_lbks(dev) * FTL_BLOCK_SIZE); 340 if (rc) { 341 return rc; 342 } 343 344 /* 345 * When restoring from a dirty shutdown it's possible old tail meta wasn't yet cleared - 346 * band had saved head meta, but didn't manage to send erase to all chunks. 347 * The already found tail md header is valid, but inconsistent with the head meta. Treat 348 * such a band as open/without valid tail md. 349 */ 350 if (band->seq != tail->hdr.seq) { 351 return FTL_MD_NO_MD; 352 } 353 354 if (tail->num_lbks != ftl_num_band_lbks(dev)) { 355 return FTL_MD_INVALID_SIZE; 356 } 357 358 spdk_bit_array_load_mask(lba_map->vld, vld_offset); 359 360 return FTL_MD_SUCCESS; 361 } 362 363 static int 364 ftl_unpack_head_md(struct ftl_band *band) 365 { 366 struct spdk_ftl_dev *dev = band->dev; 367 struct ftl_head_md *head = band->lba_map.dma_buf; 368 int rc; 369 370 rc = ftl_md_hdr_vld(dev, &head->hdr, sizeof(struct ftl_head_md)); 371 if (rc) { 372 return rc; 373 } 374 375 band->seq = head->hdr.seq; 376 band->wr_cnt = head->wr_cnt; 377 378 if (dev->global_md.num_lbas == 0) { 379 dev->global_md.num_lbas = head->lba_cnt; 380 } 381 382 if (dev->global_md.num_lbas != head->lba_cnt) { 383 return FTL_MD_INVALID_SIZE; 384 } 385 386 if (dev->xfer_size != head->xfer_size) { 387 return FTL_MD_INVALID_SIZE; 388 } 389 390 return FTL_MD_SUCCESS; 391 } 392 393 struct ftl_ppa 394 ftl_band_tail_md_ppa(struct ftl_band *band) 395 { 396 struct ftl_ppa ppa = {}; 397 struct ftl_chunk *chunk; 398 struct spdk_ftl_dev *dev = band->dev; 399 size_t xfer_size = dev->xfer_size; 400 size_t num_req = ftl_band_tail_md_offset(band) / xfer_size; 401 size_t i; 402 403 if (spdk_unlikely(!band->num_chunks)) { 404 return ftl_to_ppa(FTL_PPA_INVALID); 405 } 406 407 /* Metadata should be aligned to xfer size */ 408 assert(ftl_band_tail_md_offset(band) % xfer_size == 0); 409 410 chunk = CIRCLEQ_FIRST(&band->chunks); 411 for (i = 0; i < num_req % band->num_chunks; ++i) { 412 chunk = ftl_band_next_chunk(band, chunk); 413 } 414 415 ppa.lbk = (num_req / band->num_chunks) * xfer_size; 416 ppa.chk = band->id; 417 ppa.pu = chunk->punit->start_ppa.pu; 418 ppa.grp = chunk->punit->start_ppa.grp; 419 420 return ppa; 421 } 422 423 struct ftl_ppa 424 ftl_band_head_md_ppa(struct ftl_band *band) 425 { 426 struct ftl_ppa ppa; 427 428 if (spdk_unlikely(!band->num_chunks)) { 429 return ftl_to_ppa(FTL_PPA_INVALID); 430 } 431 432 ppa = CIRCLEQ_FIRST(&band->chunks)->punit->start_ppa; 433 ppa.chk = band->id; 434 435 return ppa; 436 } 437 438 void 439 ftl_band_set_state(struct ftl_band *band, enum ftl_band_state state) 440 { 441 switch (state) { 442 case FTL_BAND_STATE_FREE: 443 assert(band->state == FTL_BAND_STATE_CLOSED); 444 _ftl_band_set_free(band); 445 break; 446 447 case FTL_BAND_STATE_PREP: 448 assert(band->state == FTL_BAND_STATE_FREE); 449 _ftl_band_set_preparing(band); 450 break; 451 452 case FTL_BAND_STATE_CLOSED: 453 if (band->state != FTL_BAND_STATE_CLOSED) { 454 assert(band->state == FTL_BAND_STATE_CLOSING || band->high_prio); 455 _ftl_band_set_closed(band); 456 } 457 break; 458 459 default: 460 break; 461 } 462 463 band->state = state; 464 } 465 466 void 467 ftl_band_set_addr(struct ftl_band *band, uint64_t lba, struct ftl_ppa ppa) 468 { 469 struct ftl_lba_map *lba_map = &band->lba_map; 470 uint64_t offset; 471 472 assert(lba != FTL_LBA_INVALID); 473 474 offset = ftl_band_lbkoff_from_ppa(band, ppa); 475 pthread_spin_lock(&lba_map->lock); 476 477 lba_map->num_vld++; 478 lba_map->map[offset] = lba; 479 spdk_bit_array_set(lba_map->vld, offset); 480 481 pthread_spin_unlock(&lba_map->lock); 482 } 483 484 size_t 485 ftl_band_age(const struct ftl_band *band) 486 { 487 return (size_t)(band->dev->seq - band->seq); 488 } 489 490 size_t 491 ftl_band_num_usable_lbks(const struct ftl_band *band) 492 { 493 return band->num_chunks * ftl_dev_lbks_in_chunk(band->dev); 494 } 495 496 size_t 497 ftl_band_user_lbks_left(const struct ftl_band *band, size_t offset) 498 { 499 size_t tail_md_offset = ftl_band_tail_md_offset(band); 500 501 if (spdk_unlikely(offset <= ftl_head_md_num_lbks(band->dev))) { 502 return ftl_band_user_lbks(band); 503 } 504 505 if (spdk_unlikely(offset > tail_md_offset)) { 506 return 0; 507 } 508 509 return tail_md_offset - offset; 510 } 511 512 size_t 513 ftl_band_user_lbks(const struct ftl_band *band) 514 { 515 return ftl_band_num_usable_lbks(band) - 516 ftl_head_md_num_lbks(band->dev) - 517 ftl_tail_md_num_lbks(band->dev); 518 } 519 520 struct ftl_band * 521 ftl_band_from_ppa(struct spdk_ftl_dev *dev, struct ftl_ppa ppa) 522 { 523 assert(ppa.chk < ftl_dev_num_bands(dev)); 524 return &dev->bands[ppa.chk]; 525 } 526 527 struct ftl_chunk * 528 ftl_band_chunk_from_ppa(struct ftl_band *band, struct ftl_ppa ppa) 529 { 530 struct spdk_ftl_dev *dev = band->dev; 531 unsigned int punit; 532 533 punit = ftl_ppa_flatten_punit(dev, ppa); 534 assert(punit < ftl_dev_num_punits(dev)); 535 536 return &band->chunk_buf[punit]; 537 } 538 539 uint64_t 540 ftl_band_lbkoff_from_ppa(struct ftl_band *band, struct ftl_ppa ppa) 541 { 542 struct spdk_ftl_dev *dev = band->dev; 543 unsigned int punit; 544 545 punit = ftl_ppa_flatten_punit(dev, ppa); 546 assert(ppa.chk == band->id); 547 548 return punit * ftl_dev_lbks_in_chunk(dev) + ppa.lbk; 549 } 550 551 struct ftl_ppa 552 ftl_band_next_xfer_ppa(struct ftl_band *band, struct ftl_ppa ppa, size_t num_lbks) 553 { 554 struct spdk_ftl_dev *dev = band->dev; 555 struct ftl_chunk *chunk; 556 unsigned int punit_num; 557 size_t num_xfers, num_stripes; 558 559 assert(ppa.chk == band->id); 560 561 punit_num = ftl_ppa_flatten_punit(dev, ppa); 562 chunk = &band->chunk_buf[punit_num]; 563 564 num_lbks += (ppa.lbk % dev->xfer_size); 565 ppa.lbk -= (ppa.lbk % dev->xfer_size); 566 567 #if defined(DEBUG) 568 /* Check that the number of chunks has not been changed */ 569 struct ftl_chunk *_chunk; 570 size_t _num_chunks = 0; 571 CIRCLEQ_FOREACH(_chunk, &band->chunks, circleq) { 572 if (spdk_likely(_chunk->state != FTL_CHUNK_STATE_BAD)) { 573 _num_chunks++; 574 } 575 } 576 assert(band->num_chunks == _num_chunks); 577 #endif 578 assert(band->num_chunks != 0); 579 num_stripes = (num_lbks / dev->xfer_size) / band->num_chunks; 580 ppa.lbk += num_stripes * dev->xfer_size; 581 num_lbks -= num_stripes * dev->xfer_size * band->num_chunks; 582 583 if (ppa.lbk > ftl_dev_lbks_in_chunk(dev)) { 584 return ftl_to_ppa(FTL_PPA_INVALID); 585 } 586 587 num_xfers = num_lbks / dev->xfer_size; 588 for (size_t i = 0; i < num_xfers; ++i) { 589 /* When the last chunk is reached the lbk part of the address */ 590 /* needs to be increased by xfer_size */ 591 if (ftl_band_chunk_is_last(band, chunk)) { 592 ppa.lbk += dev->xfer_size; 593 if (ppa.lbk > ftl_dev_lbks_in_chunk(dev)) { 594 return ftl_to_ppa(FTL_PPA_INVALID); 595 } 596 } 597 598 chunk = ftl_band_next_operational_chunk(band, chunk); 599 assert(chunk); 600 ppa.grp = chunk->start_ppa.grp; 601 ppa.pu = chunk->start_ppa.pu; 602 603 num_lbks -= dev->xfer_size; 604 } 605 606 if (num_lbks) { 607 ppa.lbk += num_lbks; 608 if (ppa.lbk > ftl_dev_lbks_in_chunk(dev)) { 609 return ftl_to_ppa(FTL_PPA_INVALID); 610 } 611 } 612 613 return ppa; 614 } 615 616 static size_t 617 ftl_xfer_offset_from_ppa(struct ftl_band *band, struct ftl_ppa ppa) 618 { 619 struct ftl_chunk *chunk, *current_chunk; 620 unsigned int punit_offset = 0; 621 size_t off, num_stripes, xfer_size = band->dev->xfer_size; 622 623 assert(ppa.chk == band->id); 624 625 num_stripes = (ppa.lbk / xfer_size) * band->num_chunks; 626 off = ppa.lbk % xfer_size; 627 628 current_chunk = ftl_band_chunk_from_ppa(band, ppa); 629 CIRCLEQ_FOREACH(chunk, &band->chunks, circleq) { 630 if (current_chunk == chunk) { 631 break; 632 } 633 punit_offset++; 634 } 635 636 return xfer_size * (num_stripes + punit_offset) + off; 637 } 638 639 struct ftl_ppa 640 ftl_band_ppa_from_lbkoff(struct ftl_band *band, uint64_t lbkoff) 641 { 642 struct ftl_ppa ppa = { .ppa = 0 }; 643 struct spdk_ftl_dev *dev = band->dev; 644 uint64_t punit; 645 646 punit = lbkoff / ftl_dev_lbks_in_chunk(dev) + dev->range.begin; 647 648 ppa.lbk = lbkoff % ftl_dev_lbks_in_chunk(dev); 649 ppa.chk = band->id; 650 ppa.pu = punit / dev->geo.num_grp; 651 ppa.grp = punit % dev->geo.num_grp; 652 653 return ppa; 654 } 655 656 struct ftl_ppa 657 ftl_band_next_ppa(struct ftl_band *band, struct ftl_ppa ppa, size_t offset) 658 { 659 uint64_t lbkoff = ftl_band_lbkoff_from_ppa(band, ppa); 660 return ftl_band_ppa_from_lbkoff(band, lbkoff + offset); 661 } 662 663 void 664 ftl_band_acquire_lba_map(struct ftl_band *band) 665 { 666 assert(band->lba_map.map != NULL); 667 band->lba_map.ref_cnt++; 668 } 669 670 int 671 ftl_band_alloc_lba_map(struct ftl_band *band) 672 { 673 struct spdk_ftl_dev *dev = band->dev; 674 struct ftl_lba_map *lba_map = &band->lba_map; 675 676 assert(lba_map->ref_cnt == 0); 677 assert(lba_map->map == NULL); 678 679 lba_map->dma_buf = spdk_mempool_get(dev->lba_pool); 680 681 if (!lba_map->dma_buf) { 682 return -1; 683 } 684 685 memset(lba_map->dma_buf, 0, ftl_lba_map_pool_elem_size(band->dev)); 686 687 lba_map->map = (uint64_t *)((char *)lba_map->dma_buf + FTL_BLOCK_SIZE * 688 (ftl_tail_md_hdr_num_lbks() + ftl_vld_map_num_lbks(dev))); 689 690 lba_map->segments = (char *)lba_map->dma_buf + ftl_tail_md_num_lbks(dev) * FTL_BLOCK_SIZE; 691 692 ftl_band_acquire_lba_map(band); 693 return 0; 694 } 695 696 void 697 ftl_band_release_lba_map(struct ftl_band *band) 698 { 699 struct ftl_lba_map *lba_map = &band->lba_map; 700 701 assert(lba_map->map != NULL); 702 assert(lba_map->ref_cnt > 0); 703 lba_map->ref_cnt--; 704 705 if (lba_map->ref_cnt == 0) { 706 ftl_band_free_lba_map(band); 707 } 708 } 709 710 static void 711 ftl_read_md_cb(struct ftl_io *io, void *arg, int status) 712 { 713 struct ftl_md_io *md_io = (struct ftl_md_io *)io; 714 715 if (!status) { 716 status = md_io->pack_fn(md_io->io.band); 717 } else { 718 status = FTL_MD_IO_FAILURE; 719 } 720 721 md_io->cb_fn(io, md_io->cb_ctx, status); 722 } 723 724 static struct ftl_md_io * 725 ftl_io_init_md_read(struct spdk_ftl_dev *dev, struct ftl_ppa ppa, 726 struct ftl_band *band, size_t lbk_cnt, void *buf, 727 ftl_io_fn fn, ftl_md_pack_fn pack_fn, ftl_io_fn cb_fn, void *cb_ctx) 728 { 729 struct ftl_md_io *io; 730 struct ftl_io_init_opts opts = { 731 .dev = dev, 732 .io = NULL, 733 .rwb_batch = NULL, 734 .band = band, 735 .size = sizeof(*io), 736 .flags = FTL_IO_MD | FTL_IO_PPA_MODE, 737 .type = FTL_IO_READ, 738 .lbk_cnt = lbk_cnt, 739 .cb_fn = fn, 740 .data = buf, 741 }; 742 743 io = (struct ftl_md_io *)ftl_io_init_internal(&opts); 744 if (!io) { 745 return NULL; 746 } 747 748 io->io.ppa = ppa; 749 io->pack_fn = pack_fn; 750 io->cb_fn = cb_fn; 751 io->cb_ctx = cb_ctx; 752 753 return io; 754 } 755 756 static struct ftl_io * 757 ftl_io_init_md_write(struct spdk_ftl_dev *dev, struct ftl_band *band, 758 void *data, size_t lbk_cnt, ftl_io_fn cb) 759 { 760 struct ftl_io_init_opts opts = { 761 .dev = dev, 762 .io = NULL, 763 .rwb_batch = NULL, 764 .band = band, 765 .size = sizeof(struct ftl_io), 766 .flags = FTL_IO_MD | FTL_IO_PPA_MODE, 767 .type = FTL_IO_WRITE, 768 .lbk_cnt = lbk_cnt, 769 .cb_fn = cb, 770 .data = data, 771 .md = NULL, 772 }; 773 774 return ftl_io_init_internal(&opts); 775 } 776 777 static int 778 ftl_band_write_md(struct ftl_band *band, size_t lbk_cnt, 779 ftl_md_pack_fn md_fn, ftl_io_fn cb) 780 { 781 struct spdk_ftl_dev *dev = band->dev; 782 struct ftl_io *io; 783 784 io = ftl_io_init_md_write(dev, band, band->lba_map.dma_buf, lbk_cnt, cb); 785 if (!io) { 786 return -ENOMEM; 787 } 788 789 md_fn(band); 790 791 ftl_io_write(io); 792 return 0; 793 } 794 795 void 796 ftl_band_md_clear(struct ftl_band *band) 797 { 798 band->seq = 0; 799 band->wr_cnt = 0; 800 band->lba_map.num_vld = 0; 801 band->lba_map.map = NULL; 802 } 803 804 int 805 ftl_band_write_head_md(struct ftl_band *band, ftl_io_fn cb) 806 { 807 return ftl_band_write_md(band, ftl_head_md_num_lbks(band->dev), 808 ftl_pack_head_md, cb); 809 } 810 811 int 812 ftl_band_write_tail_md(struct ftl_band *band, ftl_io_fn cb) 813 { 814 return ftl_band_write_md(band, ftl_tail_md_num_lbks(band->dev), 815 ftl_pack_tail_md, cb); 816 } 817 818 static struct ftl_ppa 819 ftl_band_lba_map_ppa(struct ftl_band *band, size_t offset) 820 { 821 return ftl_band_next_xfer_ppa(band, band->tail_md_ppa, 822 ftl_tail_md_hdr_num_lbks() + 823 ftl_vld_map_num_lbks(band->dev) + 824 offset); 825 } 826 827 static int 828 ftl_band_read_md(struct ftl_band *band, size_t lbk_cnt, struct ftl_ppa start_ppa, 829 void *buf, ftl_io_fn fn, ftl_md_pack_fn pack_fn, ftl_io_fn cb_fn, void *cb_ctx) 830 { 831 struct spdk_ftl_dev *dev = band->dev; 832 struct ftl_md_io *io; 833 834 if (spdk_unlikely(!band->num_chunks)) { 835 return -ENOENT; 836 } 837 838 io = ftl_io_init_md_read(dev, start_ppa, band, lbk_cnt, buf, fn, pack_fn, cb_fn, cb_ctx); 839 if (!io) { 840 return -ENOMEM; 841 } 842 843 ftl_io_read((struct ftl_io *)io); 844 return 0; 845 } 846 847 int 848 ftl_band_read_tail_md(struct ftl_band *band, struct ftl_ppa ppa, ftl_io_fn cb_fn, void *cb_ctx) 849 { 850 return ftl_band_read_md(band, ftl_tail_md_num_lbks(band->dev), ppa, band->lba_map.dma_buf, 851 ftl_read_md_cb, ftl_unpack_tail_md, cb_fn, cb_ctx); 852 } 853 854 static size_t 855 ftl_lba_map_request_segment_done(struct ftl_lba_map_request *request, size_t offset, 856 size_t num_segments) 857 { 858 size_t i, num_done = 0; 859 860 for (i = offset; i < offset + num_segments; ++i) { 861 if (spdk_bit_array_get(request->segments, i)) { 862 spdk_bit_array_clear(request->segments, offset); 863 num_done++; 864 } 865 } 866 867 assert(request->num_pending >= num_done); 868 request->num_pending -= num_done; 869 870 return num_done; 871 } 872 873 static void 874 ftl_lba_map_set_segment_state(struct ftl_lba_map *lba_map, size_t offset, size_t num_segments, 875 enum ftl_lba_map_seg_state state) 876 { 877 size_t i; 878 879 for (i = offset; i < offset + num_segments; ++i) { 880 lba_map->segments[i] = state; 881 } 882 } 883 884 static void 885 ftl_lba_map_request_free(struct spdk_ftl_dev *dev, struct ftl_lba_map_request *request) 886 { 887 spdk_bit_array_clear_mask(request->segments); 888 spdk_mempool_put(dev->lba_request_pool, request); 889 } 890 891 static void 892 ftl_process_lba_map_requests(struct spdk_ftl_dev *dev, struct ftl_lba_map *lba_map, size_t offset, 893 size_t num_segments, int status) 894 { 895 struct ftl_lba_map_request *request, *trequest; 896 size_t num_done; 897 898 LIST_FOREACH_SAFE(request, &lba_map->request_list, list_entry, trequest) { 899 num_done = ftl_lba_map_request_segment_done(request, offset, num_segments); 900 if (request->num_pending == 0 || (status && num_done)) { 901 request->cb(NULL, request->cb_ctx, status); 902 LIST_REMOVE(request, list_entry); 903 ftl_lba_map_request_free(dev, request); 904 } 905 } 906 } 907 908 static size_t 909 ftl_lba_map_offset_from_ppa(struct ftl_band *band, struct ftl_ppa ppa) 910 { 911 size_t offset; 912 struct ftl_ppa start_ppa = ftl_band_lba_map_ppa(band, 0); 913 914 offset = ftl_xfer_offset_from_ppa(band, ppa) - ftl_xfer_offset_from_ppa(band, start_ppa); 915 assert(offset < ftl_lba_map_num_lbks(band->dev)); 916 917 return offset; 918 } 919 920 static void 921 ftl_read_lba_map_cb(struct ftl_io *io, void *arg, int status) 922 { 923 struct ftl_lba_map *lba_map = &io->band->lba_map; 924 uint64_t lbk_off; 925 926 lbk_off = ftl_lba_map_offset_from_ppa(io->band, io->ppa); 927 assert(lbk_off + io->lbk_cnt <= ftl_lba_map_num_lbks(io->dev)); 928 929 if (!status) { 930 ftl_lba_map_set_segment_state(lba_map, lbk_off, io->lbk_cnt, 931 FTL_LBA_MAP_SEG_CACHED); 932 } 933 934 ftl_process_lba_map_requests(io->dev, lba_map, lbk_off, io->lbk_cnt, status); 935 } 936 937 static struct ftl_lba_map_request * 938 ftl_lba_map_alloc_request(struct ftl_band *band, size_t offset, size_t num_segments, 939 ftl_io_fn cb, void *cb_ctx) 940 { 941 struct ftl_lba_map_request *request; 942 struct spdk_ftl_dev *dev = band->dev; 943 size_t i; 944 945 request = spdk_mempool_get(dev->lba_request_pool); 946 if (!request) { 947 return NULL; 948 } 949 950 request->cb = cb; 951 request->cb_ctx = cb_ctx; 952 request->num_pending = num_segments; 953 954 for (i = offset; i < offset + num_segments; ++i) { 955 spdk_bit_array_set(request->segments, i); 956 } 957 958 return request; 959 } 960 961 static size_t 962 ftl_lba_map_num_clear_segments(struct ftl_lba_map *lba_map, 963 size_t offset, size_t num_segments) 964 { 965 size_t i, cnt = 0; 966 967 for (i = offset; i < offset + num_segments; ++i) { 968 if (lba_map->segments[i] != FTL_LBA_MAP_SEG_CLEAR) { 969 break; 970 } 971 cnt++; 972 } 973 974 return cnt; 975 } 976 977 int 978 ftl_band_read_lba_map(struct ftl_band *band, size_t offset, size_t lba_cnt, 979 ftl_io_fn cb_fn, void *cb_ctx) 980 { 981 size_t lbk_cnt, lbk_off, num_read, num_segments; 982 struct ftl_lba_map *lba_map = &band->lba_map; 983 struct ftl_lba_map_request *request; 984 int rc = 0; 985 986 lbk_off = offset / FTL_NUM_LBA_IN_BLOCK; 987 num_segments = spdk_divide_round_up(offset + lba_cnt, FTL_NUM_LBA_IN_BLOCK); 988 lbk_cnt = num_segments - lbk_off; 989 assert(lbk_off + lbk_cnt <= ftl_lba_map_num_lbks(band->dev)); 990 991 request = ftl_lba_map_alloc_request(band, lbk_off, lbk_cnt, cb_fn, cb_ctx); 992 if (!request) { 993 return -ENOMEM; 994 } 995 996 while (lbk_cnt) { 997 if (lba_map->segments[lbk_off] != FTL_LBA_MAP_SEG_CLEAR) { 998 if (lba_map->segments[lbk_off] == FTL_LBA_MAP_SEG_CACHED) { 999 ftl_lba_map_request_segment_done(request, lbk_off, 1); 1000 } 1001 lbk_cnt--; 1002 lbk_off++; 1003 continue; 1004 } 1005 1006 num_read = ftl_lba_map_num_clear_segments(lba_map, lbk_off, lbk_cnt); 1007 ftl_lba_map_set_segment_state(lba_map, lbk_off, num_read, 1008 FTL_LBA_MAP_SEG_PENDING); 1009 1010 rc = ftl_band_read_md(band, num_read, ftl_band_lba_map_ppa(band, lbk_off), 1011 (char *)band->lba_map.map + lbk_off * FTL_BLOCK_SIZE, 1012 ftl_read_lba_map_cb, NULL, cb_fn, cb_ctx); 1013 if (rc) { 1014 ftl_lba_map_request_free(band->dev, request); 1015 return rc; 1016 } 1017 1018 assert(lbk_cnt >= num_read); 1019 lbk_cnt -= num_read; 1020 lbk_off += num_read; 1021 } 1022 1023 if (request->num_pending) { 1024 LIST_INSERT_HEAD(&lba_map->request_list, request, list_entry); 1025 } else { 1026 cb_fn(NULL, cb_ctx, 0); 1027 ftl_lba_map_request_free(band->dev, request); 1028 } 1029 1030 return rc; 1031 } 1032 1033 int 1034 ftl_band_read_head_md(struct ftl_band *band, ftl_io_fn cb_fn, void *cb_ctx) 1035 { 1036 return ftl_band_read_md(band, 1037 ftl_head_md_num_lbks(band->dev), 1038 ftl_band_head_md_ppa(band), 1039 band->lba_map.dma_buf, 1040 ftl_read_md_cb, 1041 ftl_unpack_head_md, 1042 cb_fn, 1043 cb_ctx); 1044 } 1045 1046 static void 1047 ftl_band_remove_chunk(struct ftl_band *band, struct ftl_chunk *chunk) 1048 { 1049 CIRCLEQ_REMOVE(&band->chunks, chunk, circleq); 1050 band->num_chunks--; 1051 } 1052 1053 static void 1054 ftl_erase_fail(struct ftl_io *io, int status) 1055 { 1056 struct ftl_chunk *chunk; 1057 struct ftl_band *band = io->band; 1058 char buf[128]; 1059 1060 SPDK_ERRLOG("Erase failed @ppa: %s, status: %d\n", 1061 ftl_ppa2str(io->ppa, buf, sizeof(buf)), status); 1062 1063 chunk = ftl_band_chunk_from_ppa(band, io->ppa); 1064 chunk->state = FTL_CHUNK_STATE_BAD; 1065 ftl_band_remove_chunk(band, chunk); 1066 band->tail_md_ppa = ftl_band_tail_md_ppa(band); 1067 } 1068 1069 static void 1070 ftl_band_erase_cb(struct ftl_io *io, void *ctx, int status) 1071 { 1072 struct ftl_chunk *chunk; 1073 1074 if (spdk_unlikely(status)) { 1075 ftl_erase_fail(io, status); 1076 return; 1077 } 1078 chunk = ftl_band_chunk_from_ppa(io->band, io->ppa); 1079 chunk->state = FTL_CHUNK_STATE_FREE; 1080 chunk->write_offset = 0; 1081 } 1082 1083 int 1084 ftl_band_erase(struct ftl_band *band) 1085 { 1086 struct ftl_chunk *chunk; 1087 struct ftl_io *io; 1088 int rc = 0; 1089 1090 assert(band->state == FTL_BAND_STATE_CLOSED || 1091 band->state == FTL_BAND_STATE_FREE); 1092 1093 ftl_band_set_state(band, FTL_BAND_STATE_PREP); 1094 1095 CIRCLEQ_FOREACH(chunk, &band->chunks, circleq) { 1096 if (chunk->state == FTL_CHUNK_STATE_FREE) { 1097 continue; 1098 } 1099 1100 io = ftl_io_erase_init(band, 1, ftl_band_erase_cb); 1101 if (!io) { 1102 rc = -ENOMEM; 1103 break; 1104 } 1105 1106 io->ppa = chunk->start_ppa; 1107 rc = ftl_io_erase(io); 1108 if (rc) { 1109 assert(0); 1110 /* TODO: change band's state back to close? */ 1111 break; 1112 } 1113 } 1114 1115 return rc; 1116 } 1117 1118 int 1119 ftl_band_write_prep(struct ftl_band *band) 1120 { 1121 struct spdk_ftl_dev *dev = band->dev; 1122 1123 if (ftl_band_alloc_lba_map(band)) { 1124 return -1; 1125 } 1126 1127 band->seq = ++dev->seq; 1128 return 0; 1129 } 1130 1131 struct ftl_chunk * 1132 ftl_band_next_operational_chunk(struct ftl_band *band, struct ftl_chunk *chunk) 1133 { 1134 struct ftl_chunk *result = NULL; 1135 struct ftl_chunk *entry; 1136 1137 if (spdk_unlikely(!band->num_chunks)) { 1138 return NULL; 1139 } 1140 1141 /* Erasing band may fail after it was assigned to wptr. */ 1142 /* In such a case chunk is no longer in band->chunks queue. */ 1143 if (spdk_likely(chunk->state != FTL_CHUNK_STATE_BAD)) { 1144 result = ftl_band_next_chunk(band, chunk); 1145 } else { 1146 CIRCLEQ_FOREACH_REVERSE(entry, &band->chunks, circleq) { 1147 if (entry->pos > chunk->pos) { 1148 result = entry; 1149 } else { 1150 if (!result) { 1151 result = CIRCLEQ_FIRST(&band->chunks); 1152 } 1153 break; 1154 } 1155 } 1156 } 1157 1158 return result; 1159 } 1160 1161 void 1162 ftl_band_clear_lba_map(struct ftl_band *band) 1163 { 1164 struct ftl_lba_map *lba_map = &band->lba_map; 1165 size_t num_segments; 1166 1167 spdk_bit_array_clear_mask(lba_map->vld); 1168 memset(lba_map->map, 0, ftl_lba_map_num_lbks(band->dev) * FTL_BLOCK_SIZE); 1169 1170 /* For open band all lba map segments are already cached */ 1171 assert(band->state == FTL_BAND_STATE_PREP); 1172 num_segments = spdk_divide_round_up(ftl_num_band_lbks(band->dev), FTL_NUM_LBA_IN_BLOCK); 1173 ftl_lba_map_set_segment_state(&band->lba_map, 0, num_segments, FTL_LBA_MAP_SEG_CACHED); 1174 1175 lba_map->num_vld = 0; 1176 } 1177 1178 size_t 1179 ftl_lba_map_pool_elem_size(struct spdk_ftl_dev *dev) 1180 { 1181 /* Map pool element holds the whole tail md + segments map */ 1182 return ftl_tail_md_num_lbks(dev) * FTL_BLOCK_SIZE + 1183 spdk_divide_round_up(ftl_num_band_lbks(dev), FTL_NUM_LBA_IN_BLOCK); 1184 } 1185