1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) Intel Corporation. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include "spdk/crc32.h" 35 #include "spdk/likely.h" 36 #include "spdk/util.h" 37 #include "spdk/ftl.h" 38 39 #include "ftl_band.h" 40 #include "ftl_io.h" 41 #include "ftl_core.h" 42 #include "ftl_reloc.h" 43 #include "ftl_debug.h" 44 45 /* TODO: define some signature for meta version */ 46 #define FTL_MD_VER 1 47 48 struct __attribute__((packed)) ftl_md_hdr { 49 /* Device instance */ 50 struct spdk_uuid uuid; 51 52 /* Meta version */ 53 uint8_t ver; 54 55 /* Sequence number */ 56 uint64_t seq; 57 58 /* CRC32 checksum */ 59 uint32_t checksum; 60 }; 61 62 /* End metadata layout stored on media (with all three being aligned to block size): */ 63 /* - header */ 64 /* - valid bitmap */ 65 /* - LBA map */ 66 struct __attribute__((packed)) ftl_tail_md { 67 struct ftl_md_hdr hdr; 68 69 /* Max number of lbks */ 70 uint64_t num_lbks; 71 72 uint8_t reserved[4059]; 73 }; 74 SPDK_STATIC_ASSERT(sizeof(struct ftl_tail_md) == FTL_BLOCK_SIZE, "Incorrect metadata size"); 75 76 struct __attribute__((packed)) ftl_head_md { 77 struct ftl_md_hdr hdr; 78 79 /* Number of defrag cycles */ 80 uint64_t wr_cnt; 81 82 /* Number of surfaced LBAs */ 83 uint64_t lba_cnt; 84 85 /* Transfer size */ 86 uint32_t xfer_size; 87 }; 88 89 size_t 90 ftl_tail_md_hdr_num_lbks(void) 91 { 92 return spdk_divide_round_up(sizeof(struct ftl_tail_md), FTL_BLOCK_SIZE); 93 } 94 95 size_t 96 ftl_vld_map_num_lbks(const struct spdk_ftl_dev *dev) 97 { 98 return spdk_divide_round_up(ftl_vld_map_size(dev), FTL_BLOCK_SIZE); 99 } 100 101 size_t 102 ftl_lba_map_num_lbks(const struct spdk_ftl_dev *dev) 103 { 104 return spdk_divide_round_up(ftl_num_band_lbks(dev) * sizeof(uint64_t), FTL_BLOCK_SIZE); 105 } 106 107 size_t 108 ftl_head_md_num_lbks(const struct spdk_ftl_dev *dev) 109 { 110 return dev->xfer_size; 111 } 112 113 size_t 114 ftl_tail_md_num_lbks(const struct spdk_ftl_dev *dev) 115 { 116 return spdk_divide_round_up(ftl_tail_md_hdr_num_lbks() + 117 ftl_vld_map_num_lbks(dev) + 118 ftl_lba_map_num_lbks(dev), 119 dev->xfer_size) * dev->xfer_size; 120 } 121 122 static uint64_t 123 ftl_band_tail_md_offset(const struct ftl_band *band) 124 { 125 return ftl_band_num_usable_lbks(band) - 126 ftl_tail_md_num_lbks(band->dev); 127 } 128 129 int 130 ftl_band_full(struct ftl_band *band, size_t offset) 131 { 132 return offset == ftl_band_tail_md_offset(band); 133 } 134 135 void 136 ftl_band_write_failed(struct ftl_band *band) 137 { 138 struct spdk_ftl_dev *dev = band->dev; 139 140 band->high_prio = 1; 141 band->tail_md_ppa = ftl_to_ppa(FTL_PPA_INVALID); 142 143 if (!dev->df_band) { 144 dev->df_band = band; 145 } 146 147 ftl_reloc_add(dev->reloc, band, 0, ftl_num_band_lbks(dev), 1); 148 ftl_band_set_state(band, FTL_BAND_STATE_CLOSED); 149 } 150 151 void 152 ftl_band_clear_lba_map(struct ftl_band *band) 153 { 154 struct ftl_lba_map *lba_map = &band->lba_map; 155 156 spdk_bit_array_clear_mask(lba_map->vld); 157 memset(lba_map->map, 0, ftl_lba_map_pool_elem_size(band->dev)); 158 159 lba_map->num_vld = 0; 160 } 161 162 static void 163 ftl_band_free_lba_map(struct ftl_band *band) 164 { 165 struct spdk_ftl_dev *dev = band->dev; 166 struct ftl_lba_map *lba_map = &band->lba_map; 167 168 assert(band->state == FTL_BAND_STATE_CLOSED || 169 band->state == FTL_BAND_STATE_FREE); 170 assert(lba_map->ref_cnt == 0); 171 assert(lba_map->map != NULL); 172 assert(!band->high_prio); 173 174 /* Verify that band's metadata is consistent with l2p */ 175 if (band->num_chunks) { 176 assert(ftl_band_validate_md(band) == true); 177 } 178 179 memset(lba_map->map, 0, ftl_lba_map_pool_elem_size(band->dev)); 180 spdk_mempool_put(dev->lba_pool, lba_map->map); 181 spdk_dma_free(lba_map->dma_buf); 182 lba_map->map = NULL; 183 lba_map->dma_buf = NULL; 184 } 185 186 static void 187 _ftl_band_set_free(struct ftl_band *band) 188 { 189 struct spdk_ftl_dev *dev = band->dev; 190 struct ftl_band *lband, *prev; 191 192 if (band == dev->df_band) { 193 dev->df_band = NULL; 194 } 195 196 /* Remove the band from the closed band list */ 197 LIST_REMOVE(band, list_entry); 198 199 /* Keep the list sorted by band's write count */ 200 LIST_FOREACH(lband, &dev->free_bands, list_entry) { 201 if (lband->wr_cnt > band->wr_cnt) { 202 LIST_INSERT_BEFORE(lband, band, list_entry); 203 break; 204 } 205 prev = lband; 206 } 207 208 if (!lband) { 209 if (LIST_EMPTY(&dev->free_bands)) { 210 LIST_INSERT_HEAD(&dev->free_bands, band, list_entry); 211 } else { 212 LIST_INSERT_AFTER(prev, band, list_entry); 213 } 214 } 215 216 #if defined(DEBUG) 217 prev = NULL; 218 LIST_FOREACH(lband, &dev->free_bands, list_entry) { 219 if (!prev) { 220 continue; 221 } 222 assert(prev->wr_cnt <= lband->wr_cnt); 223 } 224 #endif 225 dev->num_free++; 226 ftl_apply_limits(dev); 227 } 228 229 static void 230 _ftl_band_set_preparing(struct ftl_band *band) 231 { 232 struct spdk_ftl_dev *dev = band->dev; 233 234 /* Remove band from free list */ 235 LIST_REMOVE(band, list_entry); 236 237 band->wr_cnt++; 238 239 assert(dev->num_free > 0); 240 dev->num_free--; 241 242 ftl_apply_limits(dev); 243 } 244 245 static void 246 _ftl_band_set_closed(struct ftl_band *band) 247 { 248 struct spdk_ftl_dev *dev = band->dev; 249 struct ftl_chunk *chunk; 250 251 /* Set the state as free_md() checks for that */ 252 band->state = FTL_BAND_STATE_CLOSED; 253 254 /* Free the lba map if there are no outstanding IOs */ 255 ftl_band_release_lba_map(band); 256 257 if (spdk_likely(band->num_chunks)) { 258 LIST_INSERT_HEAD(&dev->shut_bands, band, list_entry); 259 CIRCLEQ_FOREACH(chunk, &band->chunks, circleq) { 260 chunk->state = FTL_CHUNK_STATE_CLOSED; 261 } 262 } else { 263 LIST_REMOVE(band, list_entry); 264 } 265 } 266 267 static uint32_t 268 ftl_md_calc_crc(const struct ftl_md_hdr *hdr, size_t size) 269 { 270 size_t checkoff = offsetof(struct ftl_md_hdr, checksum); 271 size_t mdoff = checkoff + sizeof(hdr->checksum); 272 uint32_t crc; 273 274 crc = spdk_crc32c_update(hdr, checkoff, 0); 275 return spdk_crc32c_update((const char *)hdr + mdoff, size - mdoff, crc); 276 } 277 278 static void 279 ftl_set_md_hdr(struct ftl_band *band, struct ftl_md_hdr *hdr, size_t size) 280 { 281 hdr->seq = band->seq; 282 hdr->ver = FTL_MD_VER; 283 hdr->uuid = band->dev->uuid; 284 hdr->checksum = ftl_md_calc_crc(hdr, size); 285 } 286 287 static int 288 ftl_pack_head_md(struct ftl_band *band) 289 { 290 struct spdk_ftl_dev *dev = band->dev; 291 struct ftl_head_md *head = band->lba_map.dma_buf; 292 293 head->wr_cnt = band->wr_cnt; 294 head->lba_cnt = dev->num_lbas; 295 head->xfer_size = dev->xfer_size; 296 ftl_set_md_hdr(band, &head->hdr, sizeof(struct ftl_head_md)); 297 298 return FTL_MD_SUCCESS; 299 } 300 301 static int 302 ftl_pack_tail_md(struct ftl_band *band) 303 { 304 struct spdk_ftl_dev *dev = band->dev; 305 struct ftl_lba_map *lba_map = &band->lba_map; 306 struct ftl_tail_md *tail = lba_map->dma_buf; 307 size_t map_size; 308 void *vld_offset, *map_offset; 309 310 map_size = ftl_num_band_lbks(dev) * sizeof(uint64_t); 311 vld_offset = (char *)tail + ftl_tail_md_hdr_num_lbks() * FTL_BLOCK_SIZE; 312 map_offset = (char *)vld_offset + ftl_vld_map_num_lbks(dev) * FTL_BLOCK_SIZE; 313 314 /* Clear out the buffer */ 315 memset(tail, 0, ftl_tail_md_num_lbks(dev) * FTL_BLOCK_SIZE); 316 tail->num_lbks = ftl_num_band_lbks(dev); 317 318 pthread_spin_lock(&lba_map->lock); 319 spdk_bit_array_store_mask(lba_map->vld, vld_offset); 320 pthread_spin_unlock(&lba_map->lock); 321 322 memcpy(map_offset, lba_map->map, map_size); 323 ftl_set_md_hdr(band, &tail->hdr, ftl_tail_md_num_lbks(dev) * FTL_BLOCK_SIZE); 324 325 return FTL_MD_SUCCESS; 326 } 327 328 static int 329 ftl_md_hdr_vld(struct spdk_ftl_dev *dev, const struct ftl_md_hdr *hdr, size_t size) 330 { 331 if (spdk_uuid_compare(&dev->uuid, &hdr->uuid) != 0) { 332 return FTL_MD_NO_MD; 333 } 334 335 if (hdr->ver != FTL_MD_VER) { 336 return FTL_MD_INVALID_VER; 337 } 338 339 if (ftl_md_calc_crc(hdr, size) != hdr->checksum) { 340 return FTL_MD_INVALID_CRC; 341 } 342 343 return FTL_MD_SUCCESS; 344 } 345 346 static int 347 ftl_unpack_tail_md(struct ftl_band *band) 348 { 349 struct spdk_ftl_dev *dev = band->dev; 350 size_t map_size; 351 void *vld_offset, *map_offset; 352 struct ftl_lba_map *lba_map = &band->lba_map; 353 struct ftl_tail_md *tail = lba_map->dma_buf; 354 int rc; 355 356 map_size = ftl_num_band_lbks(dev) * sizeof(uint64_t); 357 vld_offset = (char *)tail + ftl_tail_md_hdr_num_lbks() * FTL_BLOCK_SIZE; 358 map_offset = (char *)vld_offset + ftl_vld_map_num_lbks(dev) * FTL_BLOCK_SIZE; 359 360 rc = ftl_md_hdr_vld(dev, &tail->hdr, ftl_tail_md_num_lbks(dev) * FTL_BLOCK_SIZE); 361 if (rc) { 362 return rc; 363 } 364 365 /* 366 * When restoring from a dirty shutdown it's possible old tail meta wasn't yet cleared - 367 * band had saved head meta, but didn't manage to send erase to all chunks. 368 * The already found tail md header is valid, but inconsistent with the head meta. Treat 369 * such a band as open/without valid tail md. 370 */ 371 if (band->seq != tail->hdr.seq) { 372 return FTL_MD_NO_MD; 373 } 374 375 if (tail->num_lbks != ftl_num_band_lbks(dev)) { 376 return FTL_MD_INVALID_SIZE; 377 } 378 379 if (lba_map->vld) { 380 spdk_bit_array_load_mask(lba_map->vld, vld_offset); 381 } 382 383 if (lba_map->map) { 384 memcpy(lba_map->map, map_offset, map_size); 385 } 386 387 return FTL_MD_SUCCESS; 388 } 389 390 static int 391 ftl_unpack_head_md(struct ftl_band *band) 392 { 393 struct spdk_ftl_dev *dev = band->dev; 394 struct ftl_head_md *head = band->lba_map.dma_buf; 395 int rc; 396 397 rc = ftl_md_hdr_vld(dev, &head->hdr, sizeof(struct ftl_head_md)); 398 if (rc) { 399 return rc; 400 } 401 402 band->seq = head->hdr.seq; 403 band->wr_cnt = head->wr_cnt; 404 405 if (dev->global_md.num_lbas == 0) { 406 dev->global_md.num_lbas = head->lba_cnt; 407 } 408 409 if (dev->global_md.num_lbas != head->lba_cnt) { 410 return FTL_MD_INVALID_SIZE; 411 } 412 413 if (dev->xfer_size != head->xfer_size) { 414 return FTL_MD_INVALID_SIZE; 415 } 416 417 return FTL_MD_SUCCESS; 418 } 419 420 struct ftl_ppa 421 ftl_band_tail_md_ppa(struct ftl_band *band) 422 { 423 struct ftl_ppa ppa = {}; 424 struct ftl_chunk *chunk; 425 struct spdk_ftl_dev *dev = band->dev; 426 size_t xfer_size = dev->xfer_size; 427 size_t num_req = ftl_band_tail_md_offset(band) / xfer_size; 428 size_t i; 429 430 if (spdk_unlikely(!band->num_chunks)) { 431 return ftl_to_ppa(FTL_PPA_INVALID); 432 } 433 434 /* Metadata should be aligned to xfer size */ 435 assert(ftl_band_tail_md_offset(band) % xfer_size == 0); 436 437 chunk = CIRCLEQ_FIRST(&band->chunks); 438 for (i = 0; i < num_req % band->num_chunks; ++i) { 439 chunk = ftl_band_next_chunk(band, chunk); 440 } 441 442 ppa.lbk = (num_req / band->num_chunks) * xfer_size; 443 ppa.chk = band->id; 444 ppa.pu = chunk->punit->start_ppa.pu; 445 ppa.grp = chunk->punit->start_ppa.grp; 446 447 return ppa; 448 } 449 450 struct ftl_ppa 451 ftl_band_head_md_ppa(struct ftl_band *band) 452 { 453 struct ftl_ppa ppa; 454 455 if (spdk_unlikely(!band->num_chunks)) { 456 return ftl_to_ppa(FTL_PPA_INVALID); 457 } 458 459 ppa = CIRCLEQ_FIRST(&band->chunks)->punit->start_ppa; 460 ppa.chk = band->id; 461 462 return ppa; 463 } 464 465 void 466 ftl_band_set_state(struct ftl_band *band, enum ftl_band_state state) 467 { 468 switch (state) { 469 case FTL_BAND_STATE_FREE: 470 assert(band->state == FTL_BAND_STATE_CLOSED); 471 _ftl_band_set_free(band); 472 break; 473 474 case FTL_BAND_STATE_PREP: 475 assert(band->state == FTL_BAND_STATE_FREE); 476 _ftl_band_set_preparing(band); 477 break; 478 479 case FTL_BAND_STATE_CLOSED: 480 if (band->state != FTL_BAND_STATE_CLOSED) { 481 assert(band->state == FTL_BAND_STATE_CLOSING); 482 _ftl_band_set_closed(band); 483 } 484 break; 485 486 default: 487 break; 488 } 489 490 band->state = state; 491 } 492 493 void 494 ftl_band_set_addr(struct ftl_band *band, uint64_t lba, struct ftl_ppa ppa) 495 { 496 struct ftl_lba_map *lba_map = &band->lba_map; 497 uint64_t offset; 498 499 assert(lba != FTL_LBA_INVALID); 500 501 offset = ftl_band_lbkoff_from_ppa(band, ppa); 502 pthread_spin_lock(&lba_map->lock); 503 504 lba_map->num_vld++; 505 lba_map->map[offset] = lba; 506 spdk_bit_array_set(lba_map->vld, offset); 507 508 pthread_spin_unlock(&lba_map->lock); 509 } 510 511 size_t 512 ftl_band_age(const struct ftl_band *band) 513 { 514 return (size_t)(band->dev->seq - band->seq); 515 } 516 517 size_t 518 ftl_band_num_usable_lbks(const struct ftl_band *band) 519 { 520 return band->num_chunks * ftl_dev_lbks_in_chunk(band->dev); 521 } 522 523 size_t 524 ftl_band_user_lbks_left(const struct ftl_band *band, size_t offset) 525 { 526 size_t tail_md_offset = ftl_band_tail_md_offset(band); 527 528 if (spdk_unlikely(offset <= ftl_head_md_num_lbks(band->dev))) { 529 return ftl_band_user_lbks(band); 530 } 531 532 if (spdk_unlikely(offset > tail_md_offset)) { 533 return 0; 534 } 535 536 return tail_md_offset - offset; 537 } 538 539 size_t 540 ftl_band_user_lbks(const struct ftl_band *band) 541 { 542 return ftl_band_num_usable_lbks(band) - 543 ftl_head_md_num_lbks(band->dev) - 544 ftl_tail_md_num_lbks(band->dev); 545 } 546 547 struct ftl_band * 548 ftl_band_from_ppa(struct spdk_ftl_dev *dev, struct ftl_ppa ppa) 549 { 550 assert(ppa.chk < ftl_dev_num_bands(dev)); 551 return &dev->bands[ppa.chk]; 552 } 553 554 struct ftl_chunk * 555 ftl_band_chunk_from_ppa(struct ftl_band *band, struct ftl_ppa ppa) 556 { 557 struct spdk_ftl_dev *dev = band->dev; 558 unsigned int punit; 559 560 punit = ftl_ppa_flatten_punit(dev, ppa); 561 assert(punit < ftl_dev_num_punits(dev)); 562 563 return &band->chunk_buf[punit]; 564 } 565 566 uint64_t 567 ftl_band_lbkoff_from_ppa(struct ftl_band *band, struct ftl_ppa ppa) 568 { 569 struct spdk_ftl_dev *dev = band->dev; 570 unsigned int punit; 571 572 punit = ftl_ppa_flatten_punit(dev, ppa); 573 assert(ppa.chk == band->id); 574 575 return punit * ftl_dev_lbks_in_chunk(dev) + ppa.lbk; 576 } 577 578 struct ftl_ppa 579 ftl_band_next_xfer_ppa(struct ftl_band *band, struct ftl_ppa ppa, size_t num_lbks) 580 { 581 struct spdk_ftl_dev *dev = band->dev; 582 struct ftl_chunk *chunk; 583 unsigned int punit_num; 584 size_t num_xfers, num_stripes; 585 586 assert(ppa.chk == band->id); 587 588 punit_num = ftl_ppa_flatten_punit(dev, ppa); 589 chunk = &band->chunk_buf[punit_num]; 590 591 num_lbks += (ppa.lbk % dev->xfer_size); 592 ppa.lbk -= (ppa.lbk % dev->xfer_size); 593 594 #if defined(DEBUG) 595 /* Check that the number of chunks has not been changed */ 596 struct ftl_chunk *_chunk; 597 size_t _num_chunks = 0; 598 CIRCLEQ_FOREACH(_chunk, &band->chunks, circleq) { 599 if (spdk_likely(_chunk->state != FTL_CHUNK_STATE_BAD)) { 600 _num_chunks++; 601 } 602 } 603 assert(band->num_chunks == _num_chunks); 604 #endif 605 assert(band->num_chunks != 0); 606 num_stripes = (num_lbks / dev->xfer_size) / band->num_chunks; 607 ppa.lbk += num_stripes * dev->xfer_size; 608 num_lbks -= num_stripes * dev->xfer_size * band->num_chunks; 609 610 if (ppa.lbk > ftl_dev_lbks_in_chunk(dev)) { 611 return ftl_to_ppa(FTL_PPA_INVALID); 612 } 613 614 num_xfers = num_lbks / dev->xfer_size; 615 for (size_t i = 0; i < num_xfers; ++i) { 616 /* When the last chunk is reached the lbk part of the address */ 617 /* needs to be increased by xfer_size */ 618 if (ftl_band_chunk_is_last(band, chunk)) { 619 ppa.lbk += dev->xfer_size; 620 if (ppa.lbk > ftl_dev_lbks_in_chunk(dev)) { 621 return ftl_to_ppa(FTL_PPA_INVALID); 622 } 623 } 624 625 chunk = ftl_band_next_operational_chunk(band, chunk); 626 ppa.grp = chunk->start_ppa.grp; 627 ppa.pu = chunk->start_ppa.pu; 628 629 num_lbks -= dev->xfer_size; 630 } 631 632 if (num_lbks) { 633 ppa.lbk += num_lbks; 634 if (ppa.lbk > ftl_dev_lbks_in_chunk(dev)) { 635 return ftl_to_ppa(FTL_PPA_INVALID); 636 } 637 } 638 639 return ppa; 640 } 641 642 static size_t 643 ftl_xfer_offset_from_ppa(struct ftl_band *band, struct ftl_ppa ppa) 644 { 645 struct ftl_chunk *chunk, *current_chunk; 646 unsigned int punit_offset = 0; 647 size_t off, num_stripes, xfer_size = band->dev->xfer_size; 648 649 assert(ppa.chk == band->id); 650 651 num_stripes = (ppa.lbk / xfer_size) * band->num_chunks; 652 off = ppa.lbk % xfer_size; 653 654 current_chunk = ftl_band_chunk_from_ppa(band, ppa); 655 CIRCLEQ_FOREACH(chunk, &band->chunks, circleq) { 656 if (current_chunk == chunk) { 657 break; 658 } 659 punit_offset++; 660 } 661 662 return xfer_size * (num_stripes + punit_offset) + off; 663 } 664 665 struct ftl_ppa 666 ftl_band_ppa_from_lbkoff(struct ftl_band *band, uint64_t lbkoff) 667 { 668 struct ftl_ppa ppa = { .ppa = 0 }; 669 struct spdk_ftl_dev *dev = band->dev; 670 uint64_t punit; 671 672 punit = lbkoff / ftl_dev_lbks_in_chunk(dev) + dev->range.begin; 673 674 ppa.lbk = lbkoff % ftl_dev_lbks_in_chunk(dev); 675 ppa.chk = band->id; 676 ppa.pu = punit / dev->geo.num_grp; 677 ppa.grp = punit % dev->geo.num_grp; 678 679 return ppa; 680 } 681 682 struct ftl_ppa 683 ftl_band_next_ppa(struct ftl_band *band, struct ftl_ppa ppa, size_t offset) 684 { 685 uint64_t lbkoff = ftl_band_lbkoff_from_ppa(band, ppa); 686 return ftl_band_ppa_from_lbkoff(band, lbkoff + offset); 687 } 688 689 void 690 ftl_band_acquire_lba_map(struct ftl_band *band) 691 { 692 assert(band->lba_map.map != NULL); 693 band->lba_map.ref_cnt++; 694 } 695 696 int 697 ftl_band_alloc_lba_map(struct ftl_band *band) 698 { 699 struct spdk_ftl_dev *dev = band->dev; 700 struct ftl_lba_map *lba_map = &band->lba_map; 701 702 assert(lba_map->ref_cnt == 0); 703 assert(lba_map->map == NULL); 704 705 lba_map->map = spdk_mempool_get(dev->lba_pool); 706 if (!lba_map->map) { 707 return -1; 708 } 709 710 lba_map->segments = (char *)lba_map->map + ftl_lba_map_num_lbks(dev) * FTL_BLOCK_SIZE; 711 712 lba_map->dma_buf = spdk_dma_zmalloc(ftl_tail_md_num_lbks(dev) * FTL_BLOCK_SIZE, 713 FTL_BLOCK_SIZE, NULL); 714 if (!lba_map->dma_buf) { 715 spdk_mempool_put(dev->lba_pool, lba_map->map); 716 return -1; 717 } 718 719 ftl_band_acquire_lba_map(band); 720 return 0; 721 } 722 723 void 724 ftl_band_release_lba_map(struct ftl_band *band) 725 { 726 struct ftl_lba_map *lba_map = &band->lba_map; 727 728 assert(lba_map->map != NULL); 729 assert(lba_map->ref_cnt > 0); 730 lba_map->ref_cnt--; 731 732 if (lba_map->ref_cnt == 0) { 733 ftl_band_free_lba_map(band); 734 } 735 } 736 737 static void 738 ftl_read_md_cb(struct ftl_io *io, void *arg, int status) 739 { 740 struct ftl_md_io *md_io = (struct ftl_md_io *)io; 741 742 if (!status) { 743 status = md_io->pack_fn(md_io->io.band); 744 } else { 745 status = FTL_MD_IO_FAILURE; 746 } 747 748 md_io->cb_fn(io, md_io->cb_ctx, status); 749 } 750 751 static struct ftl_md_io * 752 ftl_io_init_md_read(struct spdk_ftl_dev *dev, struct ftl_ppa ppa, 753 struct ftl_band *band, size_t lbk_cnt, void *buf, 754 ftl_io_fn fn, ftl_md_pack_fn pack_fn, ftl_io_fn cb_fn, void *cb_ctx) 755 { 756 struct ftl_md_io *io; 757 struct ftl_io_init_opts opts = { 758 .dev = dev, 759 .io = NULL, 760 .rwb_batch = NULL, 761 .band = band, 762 .size = sizeof(*io), 763 .flags = FTL_IO_MD | FTL_IO_PPA_MODE, 764 .type = FTL_IO_READ, 765 .lbk_cnt = lbk_cnt, 766 .cb_fn = fn, 767 .data = buf, 768 }; 769 770 io = (struct ftl_md_io *)ftl_io_init_internal(&opts); 771 if (!io) { 772 return NULL; 773 } 774 775 io->io.ppa = ppa; 776 io->pack_fn = pack_fn; 777 io->cb_fn = cb_fn; 778 io->cb_ctx = cb_ctx; 779 780 return io; 781 } 782 783 static struct ftl_io * 784 ftl_io_init_md_write(struct spdk_ftl_dev *dev, struct ftl_band *band, 785 void *data, size_t lbk_cnt, ftl_io_fn cb) 786 { 787 struct ftl_io_init_opts opts = { 788 .dev = dev, 789 .io = NULL, 790 .rwb_batch = NULL, 791 .band = band, 792 .size = sizeof(struct ftl_io), 793 .flags = FTL_IO_MD | FTL_IO_PPA_MODE, 794 .type = FTL_IO_WRITE, 795 .lbk_cnt = lbk_cnt, 796 .cb_fn = cb, 797 .data = data, 798 .md = NULL, 799 }; 800 801 return ftl_io_init_internal(&opts); 802 } 803 804 static int 805 ftl_band_write_md(struct ftl_band *band, size_t lbk_cnt, 806 ftl_md_pack_fn md_fn, ftl_io_fn cb) 807 { 808 struct spdk_ftl_dev *dev = band->dev; 809 struct ftl_io *io; 810 811 io = ftl_io_init_md_write(dev, band, band->lba_map.dma_buf, lbk_cnt, cb); 812 if (!io) { 813 return -ENOMEM; 814 } 815 816 md_fn(band); 817 818 ftl_io_write(io); 819 return 0; 820 } 821 822 void 823 ftl_band_md_clear(struct ftl_band *band) 824 { 825 band->seq = 0; 826 band->wr_cnt = 0; 827 band->lba_map.num_vld = 0; 828 band->lba_map.map = NULL; 829 } 830 831 int 832 ftl_band_write_head_md(struct ftl_band *band, ftl_io_fn cb) 833 { 834 return ftl_band_write_md(band, ftl_head_md_num_lbks(band->dev), 835 ftl_pack_head_md, cb); 836 } 837 838 int 839 ftl_band_write_tail_md(struct ftl_band *band, ftl_io_fn cb) 840 { 841 return ftl_band_write_md(band, ftl_tail_md_num_lbks(band->dev), 842 ftl_pack_tail_md, cb); 843 } 844 845 static struct ftl_ppa 846 ftl_band_lba_map_ppa(struct ftl_band *band, size_t offset) 847 { 848 return ftl_band_next_xfer_ppa(band, band->tail_md_ppa, 849 ftl_tail_md_hdr_num_lbks() + 850 ftl_vld_map_num_lbks(band->dev) + 851 offset); 852 } 853 854 static int 855 ftl_band_read_md(struct ftl_band *band, size_t lbk_cnt, struct ftl_ppa start_ppa, 856 void *buf, ftl_io_fn fn, ftl_md_pack_fn pack_fn, ftl_io_fn cb_fn, void *cb_ctx) 857 { 858 struct spdk_ftl_dev *dev = band->dev; 859 struct ftl_md_io *io; 860 861 if (spdk_unlikely(!band->num_chunks)) { 862 return -ENOENT; 863 } 864 865 io = ftl_io_init_md_read(dev, start_ppa, band, lbk_cnt, buf, fn, pack_fn, cb_fn, cb_ctx); 866 if (!io) { 867 return -ENOMEM; 868 } 869 870 ftl_io_read((struct ftl_io *)io); 871 return 0; 872 } 873 874 int 875 ftl_band_read_tail_md(struct ftl_band *band, struct ftl_ppa ppa, ftl_io_fn cb_fn, void *cb_ctx) 876 { 877 return ftl_band_read_md(band, ftl_tail_md_num_lbks(band->dev), ppa, band->lba_map.dma_buf, 878 ftl_read_md_cb, ftl_unpack_tail_md, cb_fn, cb_ctx); 879 } 880 881 static size_t 882 ftl_lba_map_request_segment_done(struct ftl_lba_map_request *request, size_t offset, 883 size_t num_segments) 884 { 885 size_t i, num_done = 0; 886 887 for (i = offset; i < offset + num_segments; ++i) { 888 if (spdk_bit_array_get(request->segments, i)) { 889 spdk_bit_array_clear(request->segments, offset); 890 num_done++; 891 } 892 } 893 894 assert(request->num_pending >= num_done); 895 request->num_pending -= num_done; 896 897 return num_done; 898 } 899 900 static void 901 ftl_lba_map_set_segment_state(struct ftl_lba_map *lba_map, size_t offset, size_t num_segments, 902 enum ftl_lba_map_seg_state state) 903 { 904 size_t i; 905 906 for (i = offset; i < offset + num_segments; ++i) { 907 lba_map->segments[i] = state; 908 } 909 } 910 911 static void 912 ftl_lba_map_request_free(struct spdk_ftl_dev *dev, struct ftl_lba_map_request *request) 913 { 914 spdk_bit_array_clear_mask(request->segments); 915 spdk_mempool_put(dev->lba_request_pool, request); 916 } 917 918 static void 919 ftl_process_lba_map_requests(struct spdk_ftl_dev *dev, struct ftl_lba_map *lba_map, size_t offset, 920 size_t num_segments, int status) 921 { 922 struct ftl_lba_map_request *request, *trequest; 923 size_t num_done; 924 925 LIST_FOREACH_SAFE(request, &lba_map->request_list, list_entry, trequest) { 926 num_done = ftl_lba_map_request_segment_done(request, offset, num_segments); 927 if (request->num_pending == 0 || (status && num_done)) { 928 request->cb(NULL, request->cb_ctx, status); 929 LIST_REMOVE(request, list_entry); 930 ftl_lba_map_request_free(dev, request); 931 } 932 } 933 } 934 935 static size_t 936 ftl_lba_map_offset_from_ppa(struct ftl_band *band, struct ftl_ppa ppa) 937 { 938 size_t offset; 939 struct ftl_ppa start_ppa = ftl_band_lba_map_ppa(band, 0); 940 941 offset = ftl_xfer_offset_from_ppa(band, ppa) - ftl_xfer_offset_from_ppa(band, start_ppa); 942 assert(offset < ftl_lba_map_num_lbks(band->dev)); 943 944 return offset; 945 } 946 947 static void 948 ftl_read_lba_map_cb(struct ftl_io *io, void *arg, int status) 949 { 950 struct ftl_lba_map *lba_map = &io->band->lba_map; 951 uint64_t lbk_off; 952 953 lbk_off = ftl_lba_map_offset_from_ppa(io->band, io->ppa); 954 assert(lbk_off + io->lbk_cnt <= ftl_lba_map_num_lbks(io->dev)); 955 956 if (!status) { 957 memcpy((char *)lba_map->map + lbk_off * FTL_BLOCK_SIZE, 958 io->iov[0].iov_base, 959 io->lbk_cnt * FTL_BLOCK_SIZE); 960 961 ftl_lba_map_set_segment_state(lba_map, lbk_off, io->lbk_cnt, 962 FTL_LBA_MAP_SEG_CACHED); 963 } 964 965 ftl_process_lba_map_requests(io->dev, lba_map, lbk_off, io->lbk_cnt, status); 966 } 967 968 static struct ftl_lba_map_request * 969 ftl_lba_map_alloc_request(struct ftl_band *band, size_t offset, size_t num_segments, 970 ftl_io_fn cb, void *cb_ctx) 971 { 972 struct ftl_lba_map_request *request; 973 struct spdk_ftl_dev *dev = band->dev; 974 size_t i; 975 976 request = spdk_mempool_get(dev->lba_request_pool); 977 if (!request) { 978 return NULL; 979 } 980 981 request->cb = cb; 982 request->cb_ctx = cb_ctx; 983 request->num_pending = num_segments; 984 985 for (i = offset; i < offset + num_segments; ++i) { 986 spdk_bit_array_set(request->segments, i); 987 } 988 989 return request; 990 } 991 992 static size_t 993 ftl_lba_map_num_clear_segments(struct ftl_lba_map *lba_map, 994 size_t offset, size_t num_segments) 995 { 996 size_t i, cnt = 0; 997 998 for (i = offset; i < offset + num_segments; ++i) { 999 if (lba_map->segments[i] != FTL_LBA_MAP_SEG_CLEAR) { 1000 break; 1001 } 1002 cnt++; 1003 } 1004 1005 return cnt; 1006 } 1007 1008 int 1009 ftl_band_read_lba_map(struct ftl_band *band, size_t offset, size_t lba_cnt, 1010 ftl_io_fn cb_fn, void *cb_ctx) 1011 { 1012 size_t lbk_cnt, lbk_off, num_read, num_segments; 1013 struct ftl_lba_map *lba_map = &band->lba_map; 1014 struct ftl_lba_map_request *request; 1015 int rc = 0; 1016 1017 lbk_off = offset / FTL_NUM_LBA_IN_BLOCK; 1018 num_segments = spdk_divide_round_up(offset + lba_cnt, FTL_NUM_LBA_IN_BLOCK); 1019 lbk_cnt = num_segments - lbk_off; 1020 assert(lbk_off + lbk_cnt <= ftl_lba_map_num_lbks(band->dev)); 1021 1022 request = ftl_lba_map_alloc_request(band, lbk_off, lbk_cnt, cb_fn, cb_ctx); 1023 if (!request) { 1024 return -ENOMEM; 1025 } 1026 1027 while (lbk_cnt) { 1028 if (lba_map->segments[lbk_off] != FTL_LBA_MAP_SEG_CLEAR) { 1029 if (lba_map->segments[lbk_off] == FTL_LBA_MAP_SEG_CACHED) { 1030 ftl_lba_map_request_segment_done(request, lbk_off, 1); 1031 } 1032 lbk_cnt--; 1033 lbk_off++; 1034 continue; 1035 } 1036 1037 num_read = ftl_lba_map_num_clear_segments(lba_map, lbk_off, lbk_cnt); 1038 ftl_lba_map_set_segment_state(lba_map, lbk_off, num_read, 1039 FTL_LBA_MAP_SEG_PENDING); 1040 1041 rc = ftl_band_read_md(band, num_read, ftl_band_lba_map_ppa(band, lbk_off), 1042 (char *)band->lba_map.dma_buf + lbk_off * FTL_BLOCK_SIZE, 1043 ftl_read_lba_map_cb, NULL, cb_fn, cb_ctx); 1044 if (rc) { 1045 ftl_lba_map_request_free(band->dev, request); 1046 return rc; 1047 } 1048 1049 assert(lbk_cnt >= num_read); 1050 lbk_cnt -= num_read; 1051 lbk_off += num_read; 1052 } 1053 1054 if (request->num_pending) { 1055 LIST_INSERT_HEAD(&lba_map->request_list, request, list_entry); 1056 } else { 1057 cb_fn(NULL, cb_ctx, 0); 1058 ftl_lba_map_request_free(band->dev, request); 1059 } 1060 1061 return rc; 1062 } 1063 1064 int 1065 ftl_band_read_head_md(struct ftl_band *band, ftl_io_fn cb_fn, void *cb_ctx) 1066 { 1067 return ftl_band_read_md(band, 1068 ftl_head_md_num_lbks(band->dev), 1069 ftl_band_head_md_ppa(band), 1070 band->lba_map.dma_buf, 1071 ftl_read_md_cb, 1072 ftl_unpack_head_md, 1073 cb_fn, 1074 cb_ctx); 1075 } 1076 1077 static void 1078 ftl_band_remove_chunk(struct ftl_band *band, struct ftl_chunk *chunk) 1079 { 1080 CIRCLEQ_REMOVE(&band->chunks, chunk, circleq); 1081 band->num_chunks--; 1082 } 1083 1084 static void 1085 ftl_erase_fail(struct ftl_io *io, int status) 1086 { 1087 struct ftl_chunk *chunk; 1088 char buf[128]; 1089 1090 SPDK_ERRLOG("Erase failed @ppa: %s, status: %d\n", 1091 ftl_ppa2str(io->ppa, buf, sizeof(buf)), status); 1092 1093 chunk = ftl_band_chunk_from_ppa(io->band, io->ppa); 1094 chunk->state = FTL_CHUNK_STATE_BAD; 1095 ftl_band_remove_chunk(io->band, chunk); 1096 } 1097 1098 static void 1099 ftl_band_erase_cb(struct ftl_io *io, void *ctx, int status) 1100 { 1101 struct ftl_chunk *chunk; 1102 1103 if (spdk_unlikely(status)) { 1104 ftl_erase_fail(io, status); 1105 return; 1106 } 1107 chunk = ftl_band_chunk_from_ppa(io->band, io->ppa); 1108 chunk->state = FTL_CHUNK_STATE_FREE; 1109 } 1110 1111 int 1112 ftl_band_erase(struct ftl_band *band) 1113 { 1114 struct ftl_chunk *chunk; 1115 struct ftl_io *io; 1116 int rc = 0; 1117 1118 assert(band->state == FTL_BAND_STATE_CLOSED || 1119 band->state == FTL_BAND_STATE_FREE); 1120 1121 ftl_band_set_state(band, FTL_BAND_STATE_PREP); 1122 1123 CIRCLEQ_FOREACH(chunk, &band->chunks, circleq) { 1124 if (chunk->state == FTL_CHUNK_STATE_FREE) { 1125 continue; 1126 } 1127 1128 io = ftl_io_erase_init(band, 1, ftl_band_erase_cb); 1129 if (!io) { 1130 rc = -ENOMEM; 1131 break; 1132 } 1133 1134 io->ppa = chunk->start_ppa; 1135 rc = ftl_io_erase(io); 1136 if (rc) { 1137 assert(0); 1138 /* TODO: change band's state back to close? */ 1139 break; 1140 } 1141 } 1142 1143 return rc; 1144 } 1145 1146 int 1147 ftl_band_write_prep(struct ftl_band *band) 1148 { 1149 struct spdk_ftl_dev *dev = band->dev; 1150 1151 if (ftl_band_alloc_lba_map(band)) { 1152 return -1; 1153 } 1154 1155 band->seq = ++dev->seq; 1156 return 0; 1157 } 1158 1159 struct ftl_chunk * 1160 ftl_band_next_operational_chunk(struct ftl_band *band, struct ftl_chunk *chunk) 1161 { 1162 struct ftl_chunk *result = NULL; 1163 struct ftl_chunk *entry; 1164 1165 if (spdk_unlikely(!band->num_chunks)) { 1166 return NULL; 1167 } 1168 1169 /* Erasing band may fail after it was assigned to wptr. */ 1170 /* In such a case chunk is no longer in band->chunks queue. */ 1171 if (spdk_likely(chunk->state != FTL_CHUNK_STATE_BAD)) { 1172 result = ftl_band_next_chunk(band, chunk); 1173 } else { 1174 CIRCLEQ_FOREACH_REVERSE(entry, &band->chunks, circleq) { 1175 if (entry->pos > chunk->pos) { 1176 result = entry; 1177 } else { 1178 if (!result) { 1179 result = CIRCLEQ_FIRST(&band->chunks); 1180 } 1181 break; 1182 } 1183 } 1184 } 1185 1186 return result; 1187 } 1188 1189 size_t 1190 ftl_lba_map_pool_elem_size(struct spdk_ftl_dev *dev) 1191 { 1192 /* lba map pool element has size capable to store lba map + segments map */ 1193 return ftl_lba_map_num_lbks(dev) * FTL_BLOCK_SIZE + 1194 spdk_divide_round_up(ftl_num_band_lbks(dev), FTL_NUM_LBA_IN_BLOCK); 1195 } 1196