1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) Intel Corporation. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include "spdk/likely.h" 35 #include "spdk_internal/log.h" 36 #include "spdk/ftl.h" 37 #include "ftl_reloc.h" 38 #include "ftl_core.h" 39 #include "ftl_io.h" 40 #include "ftl_rwb.h" 41 #include "ftl_band.h" 42 #include "ftl_debug.h" 43 44 struct ftl_reloc; 45 struct ftl_band_reloc; 46 47 typedef int (*ftl_reloc_fn)(struct ftl_band_reloc *, struct ftl_io *); 48 49 struct ftl_band_reloc { 50 struct ftl_reloc *parent; 51 52 /* Band being relocated */ 53 struct ftl_band *band; 54 55 /* Number of logical blocks to be relocated */ 56 size_t num_lbks; 57 58 /* Bitmap of logical blocks to be relocated */ 59 struct spdk_bit_array *reloc_map; 60 61 /* Indicates band being acitvely processed */ 62 int active; 63 64 /* Reloc map iterator */ 65 struct { 66 /* Array of chunk offsets */ 67 size_t *chk_offset; 68 69 /* Currently chunk */ 70 size_t chk_current; 71 } iter; 72 73 /* Free IO queue */ 74 struct spdk_ring *free_queue; 75 76 /* Queue of IO ready to be written */ 77 struct spdk_ring *write_queue; 78 79 TAILQ_ENTRY(ftl_band_reloc) entry; 80 81 /* TODO: get rid of md_buf */ 82 void *md_buf; 83 }; 84 85 struct ftl_reloc { 86 /* Device associated with relocate */ 87 struct spdk_ftl_dev *dev; 88 89 /* Indicates relocate is about to halt */ 90 bool halt; 91 92 /* Maximum number of IOs per band */ 93 size_t max_qdepth; 94 95 /* IO buffer */ 96 struct ftl_io **io; 97 98 /* Maximum number of active band relocates */ 99 size_t max_active; 100 101 /* Maximum transfer size (in logical blocks) per single IO */ 102 size_t xfer_size; 103 104 /* Array of band relocates */ 105 struct ftl_band_reloc *brelocs; 106 107 /* Number of active/priority band relocates */ 108 size_t num_active; 109 110 /* Priority band relocates queue */ 111 TAILQ_HEAD(, ftl_band_reloc) prio_queue; 112 113 /* Active band relocates queue */ 114 TAILQ_HEAD(, ftl_band_reloc) active_queue; 115 116 /* Pending band relocates queue */ 117 TAILQ_HEAD(, ftl_band_reloc) pending_queue; 118 }; 119 120 static struct ftl_band_reloc * 121 ftl_io_get_band_reloc(struct ftl_io *io) 122 { 123 return &io->dev->reloc->brelocs[io->band->id]; 124 } 125 126 static size_t 127 ftl_reloc_iter_chk_offset(struct ftl_band_reloc *breloc) 128 { 129 size_t chunk = breloc->iter.chk_current; 130 131 return breloc->iter.chk_offset[chunk]; 132 } 133 134 static size_t 135 ftl_reloc_iter_chk_done(struct ftl_band_reloc *breloc) 136 { 137 size_t num_lbks = ftl_dev_lbks_in_chunk(breloc->parent->dev); 138 139 return ftl_reloc_iter_chk_offset(breloc) == num_lbks; 140 } 141 142 static void 143 ftl_reloc_clr_lbk(struct ftl_band_reloc *breloc, size_t lbkoff) 144 { 145 if (!spdk_bit_array_get(breloc->reloc_map, lbkoff)) { 146 return; 147 } 148 149 spdk_bit_array_clear(breloc->reloc_map, lbkoff); 150 assert(breloc->num_lbks); 151 breloc->num_lbks--; 152 } 153 154 static void 155 _ftl_reloc_prep(struct ftl_band_reloc *breloc) 156 { 157 struct ftl_io *io; 158 struct ftl_reloc *reloc = breloc->parent; 159 struct spdk_ftl_dev *dev = reloc->dev; 160 size_t i; 161 162 for (i = 0; i < reloc->max_qdepth; ++i) { 163 io = ftl_io_alloc(dev->ioch); 164 spdk_ring_enqueue(breloc->free_queue, (void **)&io, 1); 165 } 166 } 167 168 static void 169 ftl_reloc_read_lba_map_cb(void *arg, int status) 170 { 171 struct ftl_io *io = arg; 172 struct ftl_band_reloc *breloc = ftl_io_get_band_reloc(io); 173 174 assert(status == 0); 175 spdk_dma_free(breloc->md_buf); 176 ftl_io_free(io); 177 _ftl_reloc_prep(breloc); 178 } 179 180 static int 181 ftl_reloc_read_lba_map(struct ftl_band_reloc *breloc) 182 { 183 struct ftl_band *band = breloc->band; 184 struct spdk_ftl_dev *dev = band->dev; 185 struct ftl_io *io = ftl_io_alloc(dev->ioch); 186 187 io->dev = dev; 188 io->band = band; 189 io->cb.ctx = io; 190 io->cb.fn = ftl_reloc_read_lba_map_cb; 191 192 breloc->md_buf = spdk_dma_zmalloc(ftl_lba_map_num_lbks(dev) * FTL_BLOCK_SIZE, 193 FTL_BLOCK_SIZE, NULL); 194 if (!breloc->md_buf) { 195 return -1; 196 } 197 198 if (ftl_band_alloc_md(band)) { 199 assert(false); 200 } 201 202 return ftl_band_read_lba_map(band, &band->md, breloc->md_buf, &io->cb); 203 } 204 205 static void 206 ftl_reloc_prep(struct ftl_band_reloc *breloc) 207 { 208 struct ftl_band *band = breloc->band; 209 struct ftl_reloc *reloc = breloc->parent; 210 211 breloc->active = 1; 212 reloc->num_active++; 213 214 if (!band->high_prio) { 215 assert(band->md.lba_map == NULL); 216 ftl_reloc_read_lba_map(breloc); 217 return; 218 } 219 220 _ftl_reloc_prep(breloc); 221 } 222 223 static void 224 ftl_reloc_free_io(struct ftl_band_reloc *breloc, struct ftl_io *io) 225 { 226 spdk_dma_free(io->iov.iov_base); 227 free(io->lbas); 228 spdk_ring_enqueue(breloc->free_queue, (void **)&io, 1); 229 } 230 231 static void 232 ftl_reloc_write_cb(void *arg, int status) 233 { 234 struct ftl_io *io = arg; 235 struct ftl_ppa ppa = io->ppa; 236 struct ftl_band_reloc *breloc = ftl_io_get_band_reloc(io); 237 size_t i; 238 239 if (status) { 240 SPDK_ERRLOG("Reloc write failed with status: %d\n", status); 241 assert(false); 242 return; 243 } 244 245 for (i = 0; i < io->lbk_cnt; ++i) { 246 ppa.lbk = io->ppa.lbk + i; 247 size_t lbkoff = ftl_band_lbkoff_from_ppa(breloc->band, ppa); 248 ftl_reloc_clr_lbk(breloc, lbkoff); 249 } 250 251 ftl_reloc_free_io(breloc, io); 252 } 253 254 static void 255 ftl_reloc_read_cb(void *arg, int status) 256 { 257 struct ftl_io *io = arg; 258 struct ftl_band_reloc *breloc = ftl_io_get_band_reloc(io); 259 260 /* TODO: We should handle fail on relocation read. We need to inform */ 261 /* user that this group of blocks is bad (update l2p with bad block address and */ 262 /* put it to lba_map/sector_lba). Maybe we could also retry read with smaller granularity? */ 263 if (status) { 264 SPDK_ERRLOG("Reloc read failed with status: %d\n", status); 265 assert(false); 266 return; 267 } 268 269 io->flags &= ~FTL_IO_INITIALIZED; 270 spdk_ring_enqueue(breloc->write_queue, (void **)&io, 1); 271 } 272 273 static void 274 ftl_reloc_iter_reset(struct ftl_band_reloc *breloc) 275 { 276 memset(breloc->iter.chk_offset, 0, ftl_dev_num_punits(breloc->band->dev) * 277 sizeof(*breloc->iter.chk_offset)); 278 breloc->iter.chk_current = 0; 279 } 280 281 static size_t 282 ftl_reloc_iter_lbkoff(struct ftl_band_reloc *breloc) 283 { 284 size_t chk_offset = breloc->iter.chk_current * ftl_dev_lbks_in_chunk(breloc->parent->dev); 285 286 return breloc->iter.chk_offset[breloc->iter.chk_current] + chk_offset; 287 } 288 289 static void 290 ftl_reloc_iter_next_chk(struct ftl_band_reloc *breloc) 291 { 292 size_t num_chk = ftl_dev_num_punits(breloc->band->dev); 293 294 breloc->iter.chk_current = (breloc->iter.chk_current + 1) % num_chk; 295 } 296 297 static int 298 ftl_reloc_lbk_valid(struct ftl_band_reloc *breloc, size_t lbkoff) 299 { 300 return spdk_bit_array_get(breloc->reloc_map, lbkoff) && 301 ftl_band_lbkoff_valid(breloc->band, lbkoff); 302 } 303 304 static int 305 ftl_reloc_iter_next(struct ftl_band_reloc *breloc, size_t *lbkoff) 306 { 307 size_t chunk = breloc->iter.chk_current; 308 309 *lbkoff = ftl_reloc_iter_lbkoff(breloc); 310 311 if (ftl_reloc_iter_chk_done(breloc)) { 312 return 0; 313 } 314 315 breloc->iter.chk_offset[chunk]++; 316 317 if (!ftl_reloc_lbk_valid(breloc, *lbkoff)) { 318 ftl_reloc_clr_lbk(breloc, *lbkoff); 319 return 0; 320 } 321 322 return 1; 323 } 324 325 static int 326 ftl_reloc_first_valid_lbk(struct ftl_band_reloc *breloc, size_t *lbkoff) 327 { 328 size_t i, num_lbks = ftl_dev_lbks_in_chunk(breloc->parent->dev); 329 330 for (i = ftl_reloc_iter_chk_offset(breloc); i < num_lbks; ++i) { 331 if (ftl_reloc_iter_next(breloc, lbkoff)) { 332 return 1; 333 } 334 } 335 336 return 0; 337 } 338 339 static int 340 ftl_reloc_iter_done(struct ftl_band_reloc *breloc) 341 { 342 size_t i; 343 size_t num_chks = ftl_dev_num_punits(breloc->band->dev); 344 size_t num_lbks = ftl_dev_lbks_in_chunk(breloc->parent->dev); 345 346 for (i = 0; i < num_chks; ++i) { 347 if (breloc->iter.chk_offset[i] != num_lbks) { 348 return 0; 349 } 350 } 351 352 return 1; 353 } 354 355 static size_t 356 ftl_reloc_find_valid_lbks(struct ftl_band_reloc *breloc, 357 size_t num_lbk, struct ftl_ppa *ppa) 358 { 359 size_t lbkoff, lbk_cnt = 0; 360 361 if (!ftl_reloc_first_valid_lbk(breloc, &lbkoff)) { 362 return 0; 363 } 364 365 *ppa = ftl_band_ppa_from_lbkoff(breloc->band, lbkoff); 366 367 for (lbk_cnt = 1; lbk_cnt < num_lbk; lbk_cnt++) { 368 if (!ftl_reloc_iter_next(breloc, &lbkoff)) { 369 break; 370 } 371 } 372 373 return lbk_cnt; 374 } 375 376 static size_t 377 ftl_reloc_next_lbks(struct ftl_band_reloc *breloc, struct ftl_ppa *ppa) 378 { 379 size_t i, lbk_cnt = 0; 380 struct spdk_ftl_dev *dev = breloc->parent->dev; 381 382 for (i = 0; i < ftl_dev_num_punits(dev); ++i) { 383 lbk_cnt = ftl_reloc_find_valid_lbks(breloc, 384 breloc->parent->xfer_size, ppa); 385 ftl_reloc_iter_next_chk(breloc); 386 387 if (lbk_cnt || ftl_reloc_iter_done(breloc)) { 388 break; 389 } 390 } 391 392 return lbk_cnt; 393 } 394 395 static void 396 ftl_reloc_io_reinit(struct ftl_io *io, struct ftl_band_reloc *breloc, 397 spdk_ftl_fn fn, enum ftl_io_type io_type, int flags) 398 { 399 size_t i; 400 uint64_t lbkoff; 401 struct ftl_ppa ppa = io->ppa; 402 403 ftl_io_reinit(io, fn, io, flags | FTL_IO_INTERNAL, io_type); 404 405 io->ppa = ppa; 406 io->band = breloc->band; 407 io->lbas = calloc(io->lbk_cnt, sizeof(uint64_t)); 408 409 for (i = 0; i < io->lbk_cnt; ++i) { 410 ppa.lbk = io->ppa.lbk + i; 411 lbkoff = ftl_band_lbkoff_from_ppa(breloc->band, ppa); 412 413 if (!ftl_band_lbkoff_valid(breloc->band, lbkoff)) { 414 io->lbas[i] = FTL_LBA_INVALID; 415 continue; 416 } 417 418 io->lbas[i] = breloc->band->md.lba_map[lbkoff]; 419 } 420 421 ftl_trace_lba_io_init(io->dev, io); 422 } 423 424 static int 425 ftl_reloc_write(struct ftl_band_reloc *breloc, struct ftl_io *io) 426 { 427 int rc; 428 429 if (!(io->flags & FTL_IO_INITIALIZED)) { 430 ftl_reloc_io_reinit(io, breloc, ftl_reloc_write_cb, 431 FTL_IO_WRITE, 432 FTL_IO_KEEP_ALIVE | FTL_IO_WEAK | FTL_IO_VECTOR_LBA); 433 } 434 435 rc = ftl_io_write(io); 436 if (rc == -EAGAIN) { 437 spdk_ring_enqueue(breloc->write_queue, (void **)&io, 1); 438 return 0; 439 } 440 441 return rc; 442 } 443 444 static int 445 ftl_reloc_io_init(struct ftl_band_reloc *breloc, struct ftl_io *io, 446 struct ftl_ppa ppa, size_t num_lbks) 447 { 448 struct ftl_io_init_opts opts = { 449 .dev = breloc->parent->dev, 450 .io = io, 451 .rwb_batch = NULL, 452 .band = breloc->band, 453 .size = sizeof(*io), 454 .flags = FTL_IO_KEEP_ALIVE | FTL_IO_INTERNAL | FTL_IO_PPA_MODE, 455 .type = FTL_IO_READ, 456 .iov_cnt = 1, 457 .req_size = num_lbks, 458 .fn = ftl_reloc_read_cb, 459 }; 460 461 opts.data = spdk_dma_malloc(PAGE_SIZE * num_lbks, PAGE_SIZE, NULL); 462 if (!opts.data) { 463 return -1; 464 } 465 466 io = ftl_io_init_internal(&opts); 467 io->ppa = ppa; 468 return 0; 469 } 470 471 static int 472 ftl_reloc_read(struct ftl_band_reloc *breloc, struct ftl_io *io) 473 { 474 struct ftl_ppa ppa; 475 size_t num_lbks; 476 477 num_lbks = ftl_reloc_next_lbks(breloc, &ppa); 478 479 if (!num_lbks) { 480 spdk_ring_enqueue(breloc->free_queue, (void **)&io, 1); 481 return 0; 482 } 483 484 if (ftl_reloc_io_init(breloc, io, ppa, num_lbks)) { 485 SPDK_ERRLOG("Failed to initialize io for relocation."); 486 return -1; 487 } 488 489 ftl_io_read(io); 490 return 0; 491 } 492 493 static void 494 ftl_reloc_process_queue(struct ftl_band_reloc *breloc, struct spdk_ring *queue, 495 ftl_reloc_fn fn) 496 { 497 size_t i, num_ios; 498 struct ftl_reloc *reloc = breloc->parent; 499 500 num_ios = spdk_ring_dequeue(queue, (void **)reloc->io, reloc->max_qdepth); 501 502 for (i = 0; i < num_ios; ++i) { 503 if (fn(breloc, reloc->io[i])) { 504 SPDK_ERRLOG("Reloc queue processing failed\n"); 505 assert(false); 506 } 507 } 508 } 509 510 static void 511 ftl_reloc_process_write_queue(struct ftl_band_reloc *breloc) 512 { 513 ftl_reloc_process_queue(breloc, breloc->write_queue, ftl_reloc_write); 514 } 515 516 static void 517 ftl_reloc_process_free_queue(struct ftl_band_reloc *breloc) 518 { 519 ftl_reloc_process_queue(breloc, breloc->free_queue, ftl_reloc_read); 520 } 521 522 static int 523 ftl_reloc_done(struct ftl_band_reloc *breloc) 524 { 525 struct ftl_reloc *reloc = breloc->parent; 526 527 return spdk_ring_count(breloc->free_queue) == reloc->max_qdepth; 528 } 529 530 static void 531 ftl_reloc_release_io(struct ftl_band_reloc *breloc) 532 { 533 struct ftl_reloc *reloc = breloc->parent; 534 size_t i, num_ios; 535 536 num_ios = spdk_ring_dequeue(breloc->free_queue, (void **)reloc->io, reloc->max_qdepth); 537 538 for (i = 0; i < num_ios; ++i) { 539 ftl_io_free(reloc->io[i]); 540 } 541 } 542 543 static void 544 ftl_reloc_release(struct ftl_band_reloc *breloc) 545 { 546 struct ftl_reloc *reloc = breloc->parent; 547 struct ftl_band *band = breloc->band; 548 549 if (band->high_prio) { 550 band->high_prio = 0; 551 TAILQ_REMOVE(&reloc->prio_queue, breloc, entry); 552 } else { 553 TAILQ_REMOVE(&reloc->active_queue, breloc, entry); 554 } 555 556 ftl_reloc_release_io(breloc); 557 ftl_reloc_iter_reset(breloc); 558 ftl_band_release_md(band); 559 560 breloc->active = 0; 561 reloc->num_active--; 562 563 if (breloc->num_lbks) { 564 TAILQ_INSERT_TAIL(&reloc->pending_queue, breloc, entry); 565 return; 566 } 567 568 if (ftl_band_empty(band)) { 569 ftl_band_set_state(breloc->band, FTL_BAND_STATE_FREE); 570 } 571 } 572 573 static void 574 ftl_process_reloc(struct ftl_band_reloc *breloc) 575 { 576 ftl_reloc_process_free_queue(breloc); 577 578 ftl_reloc_process_write_queue(breloc); 579 580 if (ftl_reloc_done(breloc)) { 581 ftl_reloc_release(breloc); 582 } 583 } 584 585 static int 586 ftl_band_reloc_init(struct ftl_reloc *reloc, struct ftl_band_reloc *breloc, 587 struct ftl_band *band) 588 { 589 breloc->band = band; 590 breloc->parent = reloc; 591 592 breloc->reloc_map = spdk_bit_array_create(ftl_num_band_lbks(reloc->dev)); 593 if (!breloc->reloc_map) { 594 SPDK_ERRLOG("Failed to initialize reloc map"); 595 return -1; 596 } 597 598 breloc->iter.chk_offset = calloc(ftl_dev_num_punits(band->dev), 599 sizeof(*breloc->iter.chk_offset)); 600 if (!breloc->iter.chk_offset) { 601 SPDK_ERRLOG("Failed to initialize reloc iterator"); 602 return -1; 603 } 604 605 breloc->free_queue = spdk_ring_create(SPDK_RING_TYPE_MP_SC, 606 reloc->max_qdepth * 2, 607 SPDK_ENV_SOCKET_ID_ANY); 608 if (!breloc->free_queue) { 609 SPDK_ERRLOG("Failed to initialize reloc free queue"); 610 return -1; 611 } 612 613 breloc->write_queue = spdk_ring_create(SPDK_RING_TYPE_MP_SC, 614 reloc->max_qdepth * 2, 615 SPDK_ENV_SOCKET_ID_ANY); 616 if (!breloc->write_queue) { 617 SPDK_ERRLOG("Failed to initialize reloc write queue"); 618 return -1; 619 } 620 621 return 0; 622 } 623 624 static void 625 ftl_band_reloc_free(struct ftl_band_reloc *breloc) 626 { 627 if (!breloc) { 628 return; 629 } 630 631 spdk_ring_free(breloc->free_queue); 632 spdk_ring_free(breloc->write_queue); 633 spdk_bit_array_free(&breloc->reloc_map); 634 free(breloc->iter.chk_offset); 635 } 636 637 static void 638 ftl_reloc_add_active_queue(struct ftl_band_reloc *breloc) 639 { 640 struct ftl_reloc *reloc = breloc->parent; 641 642 TAILQ_REMOVE(&reloc->pending_queue, breloc, entry); 643 TAILQ_INSERT_HEAD(&reloc->active_queue, breloc, entry); 644 ftl_reloc_prep(breloc); 645 } 646 647 struct ftl_reloc * 648 ftl_reloc_init(struct spdk_ftl_dev *dev) 649 { 650 #define POOL_NAME_LEN 128 651 struct ftl_reloc *reloc; 652 char pool_name[POOL_NAME_LEN]; 653 int rc; 654 size_t i; 655 656 reloc = calloc(1, sizeof(*reloc)); 657 if (!reloc) { 658 return NULL; 659 } 660 661 reloc->dev = dev; 662 reloc->halt = true; 663 reloc->max_qdepth = dev->conf.max_reloc_qdepth; 664 reloc->max_active = dev->conf.max_active_relocs; 665 reloc->xfer_size = dev->xfer_size; 666 667 reloc->brelocs = calloc(ftl_dev_num_bands(dev), sizeof(*reloc->brelocs)); 668 if (!reloc->brelocs) { 669 goto error; 670 } 671 672 reloc->io = calloc(reloc->max_qdepth, sizeof(*reloc->io)); 673 if (!reloc->io) { 674 goto error; 675 } 676 677 for (i = 0; i < ftl_dev_num_bands(reloc->dev); ++i) { 678 if (ftl_band_reloc_init(reloc, &reloc->brelocs[i], &dev->bands[i])) { 679 goto error; 680 } 681 } 682 683 rc = snprintf(pool_name, sizeof(pool_name), "%s-%s", dev->name, "reloc-io-pool"); 684 if (rc < 0 || rc >= POOL_NAME_LEN) { 685 return NULL; 686 } 687 688 TAILQ_INIT(&reloc->pending_queue); 689 TAILQ_INIT(&reloc->active_queue); 690 TAILQ_INIT(&reloc->prio_queue); 691 692 return reloc; 693 error: 694 ftl_reloc_free(reloc); 695 return NULL; 696 } 697 698 void 699 ftl_reloc_free(struct ftl_reloc *reloc) 700 { 701 size_t i; 702 703 if (!reloc) { 704 return; 705 } 706 707 for (i = 0; i < ftl_dev_num_bands(reloc->dev); ++i) { 708 ftl_band_reloc_free(&reloc->brelocs[i]); 709 } 710 711 free(reloc->brelocs); 712 free(reloc->io); 713 free(reloc); 714 } 715 716 bool 717 ftl_reloc_is_halted(const struct ftl_reloc *reloc) 718 { 719 return reloc->halt; 720 } 721 722 void 723 ftl_reloc_halt(struct ftl_reloc *reloc) 724 { 725 reloc->halt = true; 726 } 727 728 void 729 ftl_reloc_resume(struct ftl_reloc *reloc) 730 { 731 reloc->halt = false; 732 } 733 734 void 735 ftl_reloc(struct ftl_reloc *reloc) 736 { 737 struct ftl_band_reloc *breloc, *tbreloc; 738 739 if (ftl_reloc_is_halted(reloc)) { 740 return; 741 } 742 743 /* Process first band from priority queue and return */ 744 breloc = TAILQ_FIRST(&reloc->prio_queue); 745 if (breloc) { 746 if (!breloc->active) { 747 ftl_reloc_prep(breloc); 748 } 749 ftl_process_reloc(breloc); 750 return; 751 } 752 753 TAILQ_FOREACH_SAFE(breloc, &reloc->pending_queue, entry, tbreloc) { 754 if (reloc->num_active == reloc->max_active) { 755 break; 756 } 757 ftl_reloc_add_active_queue(breloc); 758 } 759 760 TAILQ_FOREACH_SAFE(breloc, &reloc->active_queue, entry, tbreloc) { 761 ftl_process_reloc(breloc); 762 } 763 } 764 765 void 766 ftl_reloc_add(struct ftl_reloc *reloc, struct ftl_band *band, size_t offset, 767 size_t num_lbks, int prio) 768 { 769 struct ftl_band_reloc *breloc = &reloc->brelocs[band->id]; 770 size_t i, prev_lbks = breloc->num_lbks; 771 772 for (i = offset; i < offset + num_lbks; ++i) { 773 if (spdk_bit_array_get(breloc->reloc_map, i)) { 774 continue; 775 } 776 spdk_bit_array_set(breloc->reloc_map, i); 777 breloc->num_lbks++; 778 } 779 780 if (!prev_lbks && !prio) { 781 TAILQ_INSERT_HEAD(&reloc->pending_queue, breloc, entry); 782 } 783 784 if (prio) { 785 TAILQ_INSERT_TAIL(&reloc->prio_queue, breloc, entry); 786 ftl_band_acquire_md(breloc->band); 787 } 788 } 789