1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (c) Intel Corporation. 3 * All rights reserved. 4 */ 5 6 #include "spdk/likely.h" 7 #include "spdk/log.h" 8 #include "spdk/ftl.h" 9 10 #include "ftl_reloc.h" 11 #include "ftl_core.h" 12 #include "ftl_io.h" 13 #include "ftl_band.h" 14 #include "ftl_debug.h" 15 16 /* Maximum active reloc moves */ 17 #define FTL_RELOC_MAX_MOVES 256 18 19 struct ftl_reloc; 20 struct ftl_band_reloc; 21 22 enum ftl_reloc_move_state { 23 FTL_RELOC_STATE_READ_LBA_MAP, 24 FTL_RELOC_STATE_READ, 25 FTL_RELOC_STATE_WRITE, 26 }; 27 28 enum ftl_band_reloc_state { 29 FTL_BAND_RELOC_STATE_INACTIVE, 30 FTL_BAND_RELOC_STATE_PENDING, 31 FTL_BAND_RELOC_STATE_ACTIVE, 32 FTL_BAND_RELOC_STATE_HIGH_PRIO 33 }; 34 35 struct ftl_reloc_move { 36 struct ftl_band_reloc *breloc; 37 38 /* Start addr */ 39 struct ftl_addr addr; 40 41 /* Number of logical blocks */ 42 size_t num_blocks; 43 44 /* Data buffer */ 45 void *data; 46 47 /* Move state (read lba_map, read, write) */ 48 enum ftl_reloc_move_state state; 49 50 /* IO associated with move */ 51 struct ftl_io *io; 52 53 STAILQ_ENTRY(ftl_reloc_move) entry; 54 }; 55 56 struct ftl_band_reloc { 57 struct ftl_reloc *parent; 58 59 /* Band being relocated */ 60 struct ftl_band *band; 61 62 /* Number of logical blocks to be relocated */ 63 size_t num_blocks; 64 65 /* Bitmap of logical blocks to be relocated */ 66 struct spdk_bit_array *reloc_map; 67 68 /* State of the band reloc */ 69 enum ftl_band_reloc_state state; 70 71 /* The band is being defragged */ 72 bool defrag; 73 74 /* Reloc map iterator */ 75 struct { 76 /* Array of zone offsets */ 77 size_t *zone_offset; 78 79 /* Current zone */ 80 size_t zone_current; 81 } iter; 82 83 /* Number of outstanding moves */ 84 size_t num_outstanding; 85 86 /* Pool of move objects */ 87 struct ftl_reloc_move *moves; 88 89 /* Move queue */ 90 STAILQ_HEAD(, ftl_reloc_move) move_queue; 91 92 TAILQ_ENTRY(ftl_band_reloc) entry; 93 }; 94 95 struct ftl_reloc { 96 /* Device associated with relocate */ 97 struct spdk_ftl_dev *dev; 98 99 /* Indicates relocate is about to halt */ 100 bool halt; 101 102 /* Maximum number of IOs per band */ 103 size_t max_qdepth; 104 105 /* Maximum number of active band relocates */ 106 size_t max_active; 107 108 /* Maximum transfer size (in logical blocks) per single IO */ 109 size_t xfer_size; 110 /* Number of bands being defragged */ 111 size_t num_defrag_bands; 112 113 /* Array of band relocates */ 114 struct ftl_band_reloc *brelocs; 115 116 /* Number of active/priority band relocates */ 117 size_t num_active; 118 119 /* Priority band relocates queue */ 120 TAILQ_HEAD(, ftl_band_reloc) prio_queue; 121 122 /* Active band relocates queue */ 123 TAILQ_HEAD(, ftl_band_reloc) active_queue; 124 125 /* Pending band relocates queue */ 126 TAILQ_HEAD(, ftl_band_reloc) pending_queue; 127 }; 128 129 bool 130 ftl_reloc_is_defrag_active(const struct ftl_reloc *reloc) 131 { 132 return reloc->num_defrag_bands > 0; 133 } 134 135 static size_t 136 ftl_reloc_iter_zone_offset(struct ftl_band_reloc *breloc) 137 { 138 size_t zone = breloc->iter.zone_current; 139 140 return breloc->iter.zone_offset[zone]; 141 } 142 143 static size_t 144 ftl_reloc_iter_zone_done(struct ftl_band_reloc *breloc) 145 { 146 size_t num_blocks = ftl_get_num_blocks_in_zone(breloc->parent->dev); 147 148 return ftl_reloc_iter_zone_offset(breloc) == num_blocks; 149 } 150 151 static void 152 ftl_reloc_clr_block(struct ftl_band_reloc *breloc, size_t block_off) 153 { 154 if (!spdk_bit_array_get(breloc->reloc_map, block_off)) { 155 return; 156 } 157 158 spdk_bit_array_clear(breloc->reloc_map, block_off); 159 assert(breloc->num_blocks); 160 breloc->num_blocks--; 161 } 162 163 static void 164 ftl_reloc_read_lba_map_cb(struct ftl_io *io, void *arg, int status) 165 { 166 struct ftl_reloc_move *move = arg; 167 struct ftl_band_reloc *breloc = move->breloc; 168 169 breloc->num_outstanding--; 170 assert(status == 0); 171 move->state = FTL_RELOC_STATE_WRITE; 172 STAILQ_INSERT_TAIL(&breloc->move_queue, move, entry); 173 } 174 175 static int 176 ftl_reloc_read_lba_map(struct ftl_band_reloc *breloc, struct ftl_reloc_move *move) 177 { 178 struct ftl_band *band = breloc->band; 179 180 breloc->num_outstanding++; 181 return ftl_band_read_lba_map(band, ftl_band_block_offset_from_addr(band, move->addr), 182 move->num_blocks, ftl_reloc_read_lba_map_cb, move); 183 } 184 185 static void 186 ftl_reloc_prep(struct ftl_band_reloc *breloc) 187 { 188 struct ftl_band *band = breloc->band; 189 struct ftl_reloc *reloc = breloc->parent; 190 struct ftl_reloc_move *move; 191 size_t i; 192 193 reloc->num_active++; 194 195 if (!band->high_prio) { 196 if (ftl_band_alloc_lba_map(band)) { 197 SPDK_ERRLOG("Failed to allocate lba map\n"); 198 assert(false); 199 } 200 } else { 201 ftl_band_acquire_lba_map(band); 202 } 203 204 for (i = 0; i < reloc->max_qdepth; ++i) { 205 move = &breloc->moves[i]; 206 move->state = FTL_RELOC_STATE_READ; 207 STAILQ_INSERT_TAIL(&breloc->move_queue, move, entry); 208 } 209 } 210 211 static void 212 ftl_reloc_free_move(struct ftl_band_reloc *breloc, struct ftl_reloc_move *move) 213 { 214 assert(move); 215 spdk_dma_free(move->data); 216 memset(move, 0, sizeof(*move)); 217 move->state = FTL_RELOC_STATE_READ; 218 } 219 220 static void 221 ftl_reloc_write_cb(struct ftl_io *io, void *arg, int status) 222 { 223 struct ftl_reloc_move *move = arg; 224 struct ftl_addr addr = move->addr; 225 struct ftl_band_reloc *breloc = move->breloc; 226 size_t i; 227 228 breloc->num_outstanding--; 229 230 if (status) { 231 SPDK_ERRLOG("Reloc write failed with status: %d\n", status); 232 assert(false); 233 return; 234 } 235 236 for (i = 0; i < move->num_blocks; ++i) { 237 addr.offset = move->addr.offset + i; 238 size_t block_off = ftl_band_block_offset_from_addr(breloc->band, addr); 239 ftl_reloc_clr_block(breloc, block_off); 240 } 241 242 ftl_reloc_free_move(breloc, move); 243 STAILQ_INSERT_TAIL(&breloc->move_queue, move, entry); 244 } 245 246 static void 247 ftl_reloc_read_cb(struct ftl_io *io, void *arg, int status) 248 { 249 struct ftl_reloc_move *move = arg; 250 struct ftl_band_reloc *breloc = move->breloc; 251 252 breloc->num_outstanding--; 253 254 /* TODO: We should handle fail on relocation read. We need to inform */ 255 /* user that this group of blocks is bad (update l2p with bad block address and */ 256 /* put it to lba_map/sector_lba). Maybe we could also retry read with smaller granularity? */ 257 if (status) { 258 SPDK_ERRLOG("Reloc read failed with status: %d\n", status); 259 assert(false); 260 return; 261 } 262 263 move->state = FTL_RELOC_STATE_READ_LBA_MAP; 264 move->io = NULL; 265 STAILQ_INSERT_TAIL(&breloc->move_queue, move, entry); 266 } 267 268 static void 269 ftl_reloc_iter_reset(struct ftl_band_reloc *breloc) 270 { 271 memset(breloc->iter.zone_offset, 0, ftl_get_num_punits(breloc->band->dev) * 272 sizeof(*breloc->iter.zone_offset)); 273 breloc->iter.zone_current = 0; 274 } 275 276 static size_t 277 ftl_reloc_iter_block_offset(struct ftl_band_reloc *breloc) 278 { 279 size_t zone_offset = breloc->iter.zone_current * ftl_get_num_blocks_in_zone(breloc->parent->dev); 280 281 return breloc->iter.zone_offset[breloc->iter.zone_current] + zone_offset; 282 } 283 284 static void 285 ftl_reloc_iter_next_zone(struct ftl_band_reloc *breloc) 286 { 287 size_t num_zones = ftl_get_num_punits(breloc->band->dev); 288 289 breloc->iter.zone_current = (breloc->iter.zone_current + 1) % num_zones; 290 } 291 292 static int 293 ftl_reloc_block_valid(struct ftl_band_reloc *breloc, size_t block_off) 294 { 295 struct ftl_addr addr = ftl_band_addr_from_block_offset(breloc->band, block_off); 296 297 return ftl_addr_is_written(breloc->band, addr) && 298 spdk_bit_array_get(breloc->reloc_map, block_off) && 299 ftl_band_block_offset_valid(breloc->band, block_off); 300 } 301 302 static int 303 ftl_reloc_iter_next(struct ftl_band_reloc *breloc, size_t *block_off) 304 { 305 size_t zone = breloc->iter.zone_current; 306 307 *block_off = ftl_reloc_iter_block_offset(breloc); 308 309 if (ftl_reloc_iter_zone_done(breloc)) { 310 return 0; 311 } 312 313 breloc->iter.zone_offset[zone]++; 314 315 if (!ftl_reloc_block_valid(breloc, *block_off)) { 316 ftl_reloc_clr_block(breloc, *block_off); 317 return 0; 318 } 319 320 return 1; 321 } 322 323 static int 324 ftl_reloc_first_valid_block(struct ftl_band_reloc *breloc, size_t *block_off) 325 { 326 size_t i, num_blocks = ftl_get_num_blocks_in_zone(breloc->parent->dev); 327 328 for (i = ftl_reloc_iter_zone_offset(breloc); i < num_blocks; ++i) { 329 if (ftl_reloc_iter_next(breloc, block_off)) { 330 return 1; 331 } 332 } 333 334 return 0; 335 } 336 337 static int 338 ftl_reloc_iter_done(struct ftl_band_reloc *breloc) 339 { 340 size_t i; 341 size_t num_zones = ftl_get_num_punits(breloc->band->dev); 342 size_t num_blocks = ftl_get_num_blocks_in_zone(breloc->parent->dev); 343 344 for (i = 0; i < num_zones; ++i) { 345 if (breloc->iter.zone_offset[i] != num_blocks) { 346 return 0; 347 } 348 } 349 350 return 1; 351 } 352 353 static size_t 354 ftl_reloc_find_valid_blocks(struct ftl_band_reloc *breloc, 355 size_t _num_blocks, struct ftl_addr *addr) 356 { 357 size_t block_off, num_blocks = 0; 358 359 if (!ftl_reloc_first_valid_block(breloc, &block_off)) { 360 return 0; 361 } 362 363 *addr = ftl_band_addr_from_block_offset(breloc->band, block_off); 364 365 for (num_blocks = 1; num_blocks < _num_blocks; num_blocks++) { 366 if (!ftl_reloc_iter_next(breloc, &block_off)) { 367 break; 368 } 369 } 370 371 return num_blocks; 372 } 373 374 static size_t 375 ftl_reloc_next_blocks(struct ftl_band_reloc *breloc, struct ftl_addr *addr) 376 { 377 size_t i, num_blocks = 0; 378 struct spdk_ftl_dev *dev = breloc->parent->dev; 379 380 for (i = 0; i < ftl_get_num_punits(dev); ++i) { 381 num_blocks = ftl_reloc_find_valid_blocks(breloc, breloc->parent->xfer_size, addr); 382 ftl_reloc_iter_next_zone(breloc); 383 384 if (num_blocks || ftl_reloc_iter_done(breloc)) { 385 break; 386 } 387 } 388 389 return num_blocks; 390 } 391 392 static struct ftl_io * 393 ftl_reloc_io_init(struct ftl_band_reloc *breloc, struct ftl_reloc_move *move, 394 ftl_io_fn fn, enum ftl_io_type io_type, int flags) 395 { 396 size_t block_off, i; 397 struct ftl_addr addr = move->addr; 398 struct ftl_io *io = NULL; 399 struct ftl_io_init_opts opts = { 400 .dev = breloc->parent->dev, 401 .band = breloc->band, 402 .size = sizeof(*io), 403 .flags = flags | FTL_IO_INTERNAL | FTL_IO_PHYSICAL_MODE, 404 .type = io_type, 405 .num_blocks = move->num_blocks, 406 .iovs = { 407 { 408 .iov_base = move->data, 409 .iov_len = move->num_blocks * FTL_BLOCK_SIZE, 410 } 411 }, 412 .iovcnt = 1, 413 .cb_fn = fn, 414 }; 415 416 io = ftl_io_init_internal(&opts); 417 if (!io) { 418 return NULL; 419 } 420 421 io->cb_ctx = move; 422 io->addr = move->addr; 423 424 if (flags & FTL_IO_VECTOR_LBA) { 425 for (i = 0; i < io->num_blocks; ++i, ++addr.offset) { 426 block_off = ftl_band_block_offset_from_addr(breloc->band, addr); 427 428 if (!ftl_band_block_offset_valid(breloc->band, block_off)) { 429 io->lba.vector[i] = FTL_LBA_INVALID; 430 continue; 431 } 432 433 io->lba.vector[i] = breloc->band->lba_map.map[block_off]; 434 } 435 } 436 437 ftl_trace_lba_io_init(io->dev, io); 438 439 return io; 440 } 441 442 static int 443 ftl_reloc_write(struct ftl_band_reloc *breloc, struct ftl_reloc_move *move) 444 { 445 int io_flags = FTL_IO_WEAK | FTL_IO_VECTOR_LBA | FTL_IO_BYPASS_CACHE; 446 447 if (spdk_likely(!move->io)) { 448 move->io = ftl_reloc_io_init(breloc, move, ftl_reloc_write_cb, 449 FTL_IO_WRITE, io_flags); 450 if (!move->io) { 451 ftl_reloc_free_move(breloc, move); 452 STAILQ_INSERT_TAIL(&breloc->move_queue, move, entry); 453 return -ENOMEM; 454 } 455 } 456 457 breloc->num_outstanding++; 458 ftl_io_write(move->io); 459 return 0; 460 } 461 462 static int 463 ftl_reloc_read(struct ftl_band_reloc *breloc, struct ftl_reloc_move *move) 464 { 465 struct ftl_addr addr = {}; 466 467 move->num_blocks = ftl_reloc_next_blocks(breloc, &addr); 468 move->breloc = breloc; 469 move->addr = addr; 470 471 if (!move->num_blocks) { 472 return 0; 473 } 474 475 move->data = spdk_dma_malloc(FTL_BLOCK_SIZE * move->num_blocks, 4096, NULL); 476 if (!move->data) { 477 return -1; 478 } 479 480 move->io = ftl_reloc_io_init(breloc, move, ftl_reloc_read_cb, FTL_IO_READ, 0); 481 if (!move->io) { 482 ftl_reloc_free_move(breloc, move); 483 STAILQ_INSERT_TAIL(&breloc->move_queue, move, entry); 484 SPDK_ERRLOG("Failed to initialize io for relocation."); 485 return -1; 486 } 487 488 breloc->num_outstanding++; 489 ftl_io_read(move->io); 490 return 0; 491 } 492 493 static void 494 ftl_reloc_process_moves(struct ftl_band_reloc *breloc) 495 { 496 struct ftl_reloc_move *move; 497 STAILQ_HEAD(, ftl_reloc_move) move_queue; 498 int rc = 0; 499 500 /* 501 * When IO allocation fails, we do not want to retry immediately so keep moves on 502 * temporary queue 503 */ 504 STAILQ_INIT(&move_queue); 505 STAILQ_SWAP(&breloc->move_queue, &move_queue, ftl_reloc_move); 506 507 while (!STAILQ_EMPTY(&move_queue)) { 508 move = STAILQ_FIRST(&move_queue); 509 STAILQ_REMOVE_HEAD(&move_queue, entry); 510 511 switch (move->state) { 512 case FTL_RELOC_STATE_READ_LBA_MAP: 513 rc = ftl_reloc_read_lba_map(breloc, move); 514 break; 515 case FTL_RELOC_STATE_READ: 516 rc = ftl_reloc_read(breloc, move); 517 break; 518 case FTL_RELOC_STATE_WRITE: 519 rc = ftl_reloc_write(breloc, move); 520 break; 521 default: 522 assert(false); 523 break; 524 } 525 526 if (rc) { 527 SPDK_ERRLOG("Move queue processing failed\n"); 528 assert(false); 529 } 530 } 531 } 532 533 static bool 534 ftl_reloc_done(struct ftl_band_reloc *breloc) 535 { 536 return !breloc->num_outstanding && STAILQ_EMPTY(&breloc->move_queue); 537 } 538 539 static void 540 ftl_reloc_release(struct ftl_band_reloc *breloc) 541 { 542 struct ftl_reloc *reloc = breloc->parent; 543 struct ftl_band *band = breloc->band; 544 545 ftl_reloc_iter_reset(breloc); 546 ftl_band_release_lba_map(band); 547 reloc->num_active--; 548 549 if (breloc->state == FTL_BAND_RELOC_STATE_HIGH_PRIO) { 550 /* High prio band must be relocated as a whole and ANM events will be ignored */ 551 assert(breloc->num_blocks == 0 && ftl_band_empty(band)); 552 TAILQ_REMOVE(&reloc->prio_queue, breloc, entry); 553 band->high_prio = 0; 554 breloc->state = FTL_BAND_RELOC_STATE_INACTIVE; 555 } else { 556 assert(breloc->state == FTL_BAND_RELOC_STATE_ACTIVE); 557 TAILQ_REMOVE(&reloc->active_queue, breloc, entry); 558 breloc->state = FTL_BAND_RELOC_STATE_INACTIVE; 559 560 /* If we got ANM event during relocation put such band back to pending queue */ 561 if (breloc->num_blocks != 0) { 562 breloc->state = FTL_BAND_RELOC_STATE_PENDING; 563 TAILQ_INSERT_TAIL(&reloc->pending_queue, breloc, entry); 564 return; 565 } 566 } 567 568 if (ftl_band_empty(band) && band->state == FTL_BAND_STATE_CLOSED) { 569 ftl_band_set_state(breloc->band, FTL_BAND_STATE_FREE); 570 571 if (breloc->defrag) { 572 breloc->defrag = false; 573 assert(reloc->num_defrag_bands > 0); 574 reloc->num_defrag_bands--; 575 } 576 } 577 } 578 579 static void 580 ftl_process_reloc(struct ftl_band_reloc *breloc) 581 { 582 ftl_reloc_process_moves(breloc); 583 584 if (ftl_reloc_done(breloc)) { 585 ftl_reloc_release(breloc); 586 } 587 } 588 589 static int 590 ftl_band_reloc_init(struct ftl_reloc *reloc, struct ftl_band_reloc *breloc, 591 struct ftl_band *band) 592 { 593 breloc->band = band; 594 breloc->parent = reloc; 595 596 breloc->reloc_map = spdk_bit_array_create(ftl_get_num_blocks_in_band(reloc->dev)); 597 if (!breloc->reloc_map) { 598 SPDK_ERRLOG("Failed to initialize reloc map"); 599 return -1; 600 } 601 602 breloc->iter.zone_offset = calloc(ftl_get_num_punits(band->dev), 603 sizeof(*breloc->iter.zone_offset)); 604 if (!breloc->iter.zone_offset) { 605 SPDK_ERRLOG("Failed to initialize reloc iterator"); 606 return -1; 607 } 608 609 STAILQ_INIT(&breloc->move_queue); 610 611 breloc->moves = calloc(reloc->max_qdepth, sizeof(*breloc->moves)); 612 if (!breloc->moves) { 613 return -1; 614 } 615 616 return 0; 617 } 618 619 static void 620 ftl_band_reloc_free(struct ftl_band_reloc *breloc) 621 { 622 struct ftl_reloc_move *move; 623 624 if (!breloc) { 625 return; 626 } 627 628 assert(breloc->num_outstanding == 0); 629 630 /* Drain write queue if there is active band relocation during shutdown */ 631 if (breloc->state == FTL_BAND_RELOC_STATE_ACTIVE || 632 breloc->state == FTL_BAND_RELOC_STATE_HIGH_PRIO) { 633 assert(breloc->parent->halt); 634 STAILQ_FOREACH(move, &breloc->move_queue, entry) { 635 ftl_reloc_free_move(breloc, move); 636 } 637 } 638 639 spdk_bit_array_free(&breloc->reloc_map); 640 free(breloc->iter.zone_offset); 641 free(breloc->moves); 642 } 643 644 struct ftl_reloc * 645 ftl_reloc_init(struct spdk_ftl_dev *dev) 646 { 647 struct ftl_reloc *reloc; 648 size_t i; 649 650 reloc = calloc(1, sizeof(*reloc)); 651 if (!reloc) { 652 return NULL; 653 } 654 655 reloc->dev = dev; 656 reloc->halt = true; 657 reloc->max_qdepth = dev->conf.max_reloc_qdepth; 658 reloc->max_active = dev->conf.max_active_relocs; 659 reloc->xfer_size = dev->xfer_size; 660 reloc->num_defrag_bands = 0; 661 662 if (reloc->max_qdepth > FTL_RELOC_MAX_MOVES) { 663 goto error; 664 } 665 666 reloc->brelocs = calloc(ftl_get_num_bands(dev), sizeof(*reloc->brelocs)); 667 if (!reloc->brelocs) { 668 goto error; 669 } 670 671 for (i = 0; i < ftl_get_num_bands(reloc->dev); ++i) { 672 if (ftl_band_reloc_init(reloc, &reloc->brelocs[i], &dev->bands[i])) { 673 goto error; 674 } 675 } 676 677 TAILQ_INIT(&reloc->pending_queue); 678 TAILQ_INIT(&reloc->active_queue); 679 TAILQ_INIT(&reloc->prio_queue); 680 681 return reloc; 682 error: 683 ftl_reloc_free(reloc); 684 return NULL; 685 } 686 687 void 688 ftl_reloc_free(struct ftl_reloc *reloc) 689 { 690 size_t i; 691 692 if (!reloc) { 693 return; 694 } 695 696 for (i = 0; i < ftl_get_num_bands(reloc->dev); ++i) { 697 ftl_band_reloc_free(&reloc->brelocs[i]); 698 } 699 700 free(reloc->brelocs); 701 free(reloc); 702 } 703 704 bool 705 ftl_reloc_is_halted(const struct ftl_reloc *reloc) 706 { 707 return reloc->halt; 708 } 709 710 void 711 ftl_reloc_halt(struct ftl_reloc *reloc) 712 { 713 reloc->halt = true; 714 } 715 716 void 717 ftl_reloc_resume(struct ftl_reloc *reloc) 718 { 719 reloc->halt = false; 720 } 721 722 bool 723 ftl_reloc(struct ftl_reloc *reloc) 724 { 725 struct ftl_band_reloc *breloc, *tbreloc; 726 727 if (ftl_reloc_is_halted(reloc)) { 728 return false; 729 } 730 731 /* Process first band from priority queue and return */ 732 breloc = TAILQ_FIRST(&reloc->prio_queue); 733 if (breloc) { 734 ftl_process_reloc(breloc); 735 return true; 736 } 737 738 TAILQ_FOREACH_SAFE(breloc, &reloc->pending_queue, entry, tbreloc) { 739 if (reloc->num_active == reloc->max_active) { 740 break; 741 } 742 743 /* Wait for band to close before relocating */ 744 if (breloc->band->state != FTL_BAND_STATE_CLOSED) { 745 continue; 746 } 747 748 ftl_reloc_prep(breloc); 749 assert(breloc->state == FTL_BAND_RELOC_STATE_PENDING); 750 TAILQ_REMOVE(&reloc->pending_queue, breloc, entry); 751 breloc->state = FTL_BAND_RELOC_STATE_ACTIVE; 752 TAILQ_INSERT_HEAD(&reloc->active_queue, breloc, entry); 753 } 754 755 TAILQ_FOREACH_SAFE(breloc, &reloc->active_queue, entry, tbreloc) { 756 assert(breloc->state == FTL_BAND_RELOC_STATE_ACTIVE); 757 ftl_process_reloc(breloc); 758 } 759 760 return reloc->num_active != 0; 761 } 762 763 void 764 ftl_reloc_add(struct ftl_reloc *reloc, struct ftl_band *band, size_t offset, 765 size_t num_blocks, int prio, bool is_defrag) 766 { 767 struct ftl_band_reloc *breloc = &reloc->brelocs[band->id]; 768 size_t i; 769 770 /* No need to add anything if already at high prio - whole band should be relocated */ 771 if (!prio && band->high_prio) { 772 return; 773 } 774 775 pthread_spin_lock(&band->lba_map.lock); 776 if (band->lba_map.num_vld == 0) { 777 pthread_spin_unlock(&band->lba_map.lock); 778 779 /* If the band is closed and has no valid blocks, free it */ 780 if (band->state == FTL_BAND_STATE_CLOSED) { 781 ftl_band_set_state(band, FTL_BAND_STATE_FREE); 782 } 783 784 return; 785 } 786 pthread_spin_unlock(&band->lba_map.lock); 787 788 for (i = offset; i < offset + num_blocks; ++i) { 789 if (spdk_bit_array_get(breloc->reloc_map, i)) { 790 continue; 791 } 792 spdk_bit_array_set(breloc->reloc_map, i); 793 breloc->num_blocks++; 794 } 795 796 /* If the band is coming from the defrag process, mark it appropriately */ 797 if (is_defrag) { 798 assert(offset == 0 && num_blocks == ftl_get_num_blocks_in_band(band->dev)); 799 reloc->num_defrag_bands++; 800 breloc->defrag = true; 801 } 802 803 if (!prio) { 804 if (breloc->state == FTL_BAND_RELOC_STATE_INACTIVE) { 805 breloc->state = FTL_BAND_RELOC_STATE_PENDING; 806 TAILQ_INSERT_HEAD(&reloc->pending_queue, breloc, entry); 807 } 808 } else { 809 bool active = false; 810 /* If priority band is already on pending or active queue, remove it from it */ 811 switch (breloc->state) { 812 case FTL_BAND_RELOC_STATE_PENDING: 813 TAILQ_REMOVE(&reloc->pending_queue, breloc, entry); 814 break; 815 case FTL_BAND_RELOC_STATE_ACTIVE: 816 active = true; 817 TAILQ_REMOVE(&reloc->active_queue, breloc, entry); 818 break; 819 default: 820 break; 821 } 822 823 breloc->state = FTL_BAND_RELOC_STATE_HIGH_PRIO; 824 TAILQ_INSERT_TAIL(&reloc->prio_queue, breloc, entry); 825 826 /* 827 * If band has been already on active queue it doesn't need any additional 828 * resources 829 */ 830 if (!active) { 831 ftl_reloc_prep(breloc); 832 } 833 } 834 } 835