1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) Intel Corporation. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include "spdk/likely.h" 35 #include "spdk_internal/log.h" 36 #include "spdk/ftl.h" 37 #include "ftl_reloc.h" 38 #include "ftl_core.h" 39 #include "ftl_io.h" 40 #include "ftl_rwb.h" 41 #include "ftl_band.h" 42 #include "ftl_debug.h" 43 44 struct ftl_reloc; 45 struct ftl_band_reloc; 46 47 typedef int (*ftl_reloc_fn)(struct ftl_band_reloc *, struct ftl_io *); 48 49 struct ftl_band_reloc { 50 struct ftl_reloc *parent; 51 52 /* Band being relocated */ 53 struct ftl_band *band; 54 55 /* Number of logical blocks to be relocated */ 56 size_t num_lbks; 57 58 /* Bitmap of logical blocks to be relocated */ 59 struct spdk_bit_array *reloc_map; 60 61 /* Indicates band being acitvely processed */ 62 int active; 63 64 /* Reloc map iterator */ 65 struct { 66 /* Array of chunk offsets */ 67 size_t *chk_offset; 68 69 /* Currently chunk */ 70 size_t chk_current; 71 } iter; 72 73 /* Free IO queue */ 74 struct spdk_ring *free_queue; 75 76 /* Queue of IO ready to be written */ 77 struct spdk_ring *write_queue; 78 79 TAILQ_ENTRY(ftl_band_reloc) entry; 80 81 /* TODO: get rid of md_buf */ 82 void *md_buf; 83 }; 84 85 struct ftl_reloc { 86 /* Device associated with relocate */ 87 struct spdk_ftl_dev *dev; 88 89 /* Indicates relocate is about to halt */ 90 bool halt; 91 92 /* Maximum number of IOs per band */ 93 size_t max_qdepth; 94 95 /* IO buffer */ 96 struct ftl_io **io; 97 98 /* Maximum number of active band relocates */ 99 size_t max_active; 100 101 /* Maximum transfer size (in logical blocks) per single IO */ 102 size_t xfer_size; 103 104 /* Array of band relocates */ 105 struct ftl_band_reloc *brelocs; 106 107 /* Number of active/priority band relocates */ 108 size_t num_active; 109 110 /* Priority band relocates queue */ 111 TAILQ_HEAD(, ftl_band_reloc) prio_queue; 112 113 /* Active band relocates queue */ 114 TAILQ_HEAD(, ftl_band_reloc) active_queue; 115 116 /* Pending band relocates queue */ 117 TAILQ_HEAD(, ftl_band_reloc) pending_queue; 118 }; 119 120 static struct ftl_band_reloc * 121 ftl_io_get_band_reloc(struct ftl_io *io) 122 { 123 return &io->dev->reloc->brelocs[io->band->id]; 124 } 125 126 static size_t 127 ftl_reloc_iter_chk_offset(struct ftl_band_reloc *breloc) 128 { 129 size_t chunk = breloc->iter.chk_current; 130 131 return breloc->iter.chk_offset[chunk]; 132 } 133 134 static size_t 135 ftl_reloc_iter_chk_done(struct ftl_band_reloc *breloc) 136 { 137 size_t num_lbks = ftl_dev_lbks_in_chunk(breloc->parent->dev); 138 139 return ftl_reloc_iter_chk_offset(breloc) == num_lbks; 140 } 141 142 static void 143 ftl_reloc_clr_lbk(struct ftl_band_reloc *breloc, size_t lbkoff) 144 { 145 if (!spdk_bit_array_get(breloc->reloc_map, lbkoff)) { 146 return; 147 } 148 149 spdk_bit_array_clear(breloc->reloc_map, lbkoff); 150 assert(breloc->num_lbks); 151 breloc->num_lbks--; 152 } 153 154 static void 155 _ftl_reloc_prep(struct ftl_band_reloc *breloc) 156 { 157 struct ftl_io *io; 158 struct ftl_reloc *reloc = breloc->parent; 159 struct spdk_ftl_dev *dev = reloc->dev; 160 size_t i; 161 162 for (i = 0; i < reloc->max_qdepth; ++i) { 163 io = ftl_io_alloc(dev->ioch); 164 spdk_ring_enqueue(breloc->free_queue, (void **)&io, 1); 165 } 166 } 167 168 static void 169 ftl_reloc_read_lba_map_cb(void *arg, int status) 170 { 171 struct ftl_io *io = arg; 172 struct ftl_band_reloc *breloc = ftl_io_get_band_reloc(io); 173 174 assert(status == 0); 175 spdk_dma_free(breloc->md_buf); 176 ftl_io_free(io); 177 _ftl_reloc_prep(breloc); 178 } 179 180 static int 181 ftl_reloc_read_lba_map(struct ftl_band_reloc *breloc) 182 { 183 struct ftl_band *band = breloc->band; 184 struct spdk_ftl_dev *dev = band->dev; 185 struct ftl_io *io = ftl_io_alloc(dev->ioch); 186 187 io->dev = dev; 188 io->band = band; 189 io->cb.ctx = io; 190 io->cb.fn = ftl_reloc_read_lba_map_cb; 191 192 breloc->md_buf = spdk_dma_zmalloc(ftl_lba_map_num_lbks(dev) * FTL_BLOCK_SIZE, 193 FTL_BLOCK_SIZE, NULL); 194 if (!breloc->md_buf) { 195 return -1; 196 } 197 198 if (ftl_band_alloc_md(band)) { 199 assert(false); 200 } 201 202 return ftl_band_read_lba_map(band, &band->md, breloc->md_buf, &io->cb); 203 } 204 205 static void 206 ftl_reloc_prep(struct ftl_band_reloc *breloc) 207 { 208 struct ftl_band *band = breloc->band; 209 struct ftl_reloc *reloc = breloc->parent; 210 211 breloc->active = 1; 212 reloc->num_active++; 213 214 if (!band->high_prio) { 215 assert(band->md.lba_map == NULL); 216 ftl_reloc_read_lba_map(breloc); 217 return; 218 } 219 220 _ftl_reloc_prep(breloc); 221 } 222 223 static void 224 ftl_reloc_free_io(struct ftl_band_reloc *breloc, struct ftl_io *io) 225 { 226 spdk_dma_free(io->iov.iov_base); 227 free(io->lbas); 228 spdk_ring_enqueue(breloc->free_queue, (void **)&io, 1); 229 } 230 231 static void 232 ftl_reloc_write_cb(void *arg, int status) 233 { 234 struct ftl_io *io = arg; 235 struct ftl_ppa ppa = io->ppa; 236 struct ftl_band_reloc *breloc = ftl_io_get_band_reloc(io); 237 size_t i; 238 239 if (status) { 240 SPDK_ERRLOG("Reloc write failed with status: %d\n", status); 241 assert(false); 242 return; 243 } 244 245 for (i = 0; i < io->lbk_cnt; ++i) { 246 ppa.lbk = io->ppa.lbk + i; 247 size_t lbkoff = ftl_band_lbkoff_from_ppa(breloc->band, ppa); 248 ftl_reloc_clr_lbk(breloc, lbkoff); 249 } 250 251 ftl_reloc_free_io(breloc, io); 252 } 253 254 static void 255 ftl_reloc_read_cb(void *arg, int status) 256 { 257 struct ftl_io *io = arg; 258 struct ftl_band_reloc *breloc = ftl_io_get_band_reloc(io); 259 260 /* TODO: We should handle fail on relocation read. We need to inform */ 261 /* user that this group of blocks is bad (update l2p with bad block address and */ 262 /* put it to lba_map/sector_lba). Maybe we could also retry read with smaller granularity? */ 263 if (status) { 264 SPDK_ERRLOG("Reloc read failed with status: %d\n", status); 265 assert(false); 266 return; 267 } 268 269 io->flags &= ~FTL_IO_INITIALIZED; 270 spdk_ring_enqueue(breloc->write_queue, (void **)&io, 1); 271 } 272 273 static void 274 ftl_reloc_iter_reset(struct ftl_band_reloc *breloc) 275 { 276 memset(breloc->iter.chk_offset, 0, ftl_dev_num_punits(breloc->band->dev) * 277 sizeof(*breloc->iter.chk_offset)); 278 breloc->iter.chk_current = 0; 279 } 280 281 static size_t 282 ftl_reloc_iter_lbkoff(struct ftl_band_reloc *breloc) 283 { 284 size_t chk_offset = breloc->iter.chk_current * ftl_dev_lbks_in_chunk(breloc->parent->dev); 285 286 return breloc->iter.chk_offset[breloc->iter.chk_current] + chk_offset; 287 } 288 289 static void 290 ftl_reloc_iter_next_chk(struct ftl_band_reloc *breloc) 291 { 292 size_t num_chk = ftl_dev_num_punits(breloc->band->dev); 293 294 breloc->iter.chk_current = (breloc->iter.chk_current + 1) % num_chk; 295 } 296 297 static int 298 ftl_reloc_lbk_valid(struct ftl_band_reloc *breloc, size_t lbkoff) 299 { 300 return spdk_bit_array_get(breloc->reloc_map, lbkoff) && 301 ftl_band_lbkoff_valid(breloc->band, lbkoff); 302 } 303 304 static int 305 ftl_reloc_iter_next(struct ftl_band_reloc *breloc, size_t *lbkoff) 306 { 307 size_t chunk = breloc->iter.chk_current; 308 309 *lbkoff = ftl_reloc_iter_lbkoff(breloc); 310 311 if (ftl_reloc_iter_chk_done(breloc)) { 312 return 0; 313 } 314 315 breloc->iter.chk_offset[chunk]++; 316 317 if (!ftl_reloc_lbk_valid(breloc, *lbkoff)) { 318 ftl_reloc_clr_lbk(breloc, *lbkoff); 319 return 0; 320 } 321 322 return 1; 323 } 324 325 static int 326 ftl_reloc_first_valid_lbk(struct ftl_band_reloc *breloc, size_t *lbkoff) 327 { 328 size_t i, num_lbks = ftl_dev_lbks_in_chunk(breloc->parent->dev); 329 330 for (i = ftl_reloc_iter_chk_offset(breloc); i < num_lbks; ++i) { 331 if (ftl_reloc_iter_next(breloc, lbkoff)) { 332 return 1; 333 } 334 } 335 336 return 0; 337 } 338 339 static int 340 ftl_reloc_iter_done(struct ftl_band_reloc *breloc) 341 { 342 size_t i; 343 size_t num_chks = ftl_dev_num_punits(breloc->band->dev); 344 size_t num_lbks = ftl_dev_lbks_in_chunk(breloc->parent->dev); 345 346 for (i = 0; i < num_chks; ++i) { 347 if (breloc->iter.chk_offset[i] != num_lbks) { 348 return 0; 349 } 350 } 351 352 return 1; 353 } 354 355 static size_t 356 ftl_reloc_find_valid_lbks(struct ftl_band_reloc *breloc, 357 size_t num_lbk, struct ftl_ppa *ppa) 358 { 359 size_t lbkoff, lbk_cnt = 0; 360 361 if (!ftl_reloc_first_valid_lbk(breloc, &lbkoff)) { 362 return 0; 363 } 364 365 *ppa = ftl_band_ppa_from_lbkoff(breloc->band, lbkoff); 366 367 for (lbk_cnt = 1; lbk_cnt < num_lbk; lbk_cnt++) { 368 if (!ftl_reloc_iter_next(breloc, &lbkoff)) { 369 break; 370 } 371 } 372 373 return lbk_cnt; 374 } 375 376 static size_t 377 ftl_reloc_next_lbks(struct ftl_band_reloc *breloc, struct ftl_ppa *ppa) 378 { 379 size_t i, lbk_cnt = 0; 380 struct spdk_ftl_dev *dev = breloc->parent->dev; 381 382 for (i = 0; i < ftl_dev_num_punits(dev); ++i) { 383 lbk_cnt = ftl_reloc_find_valid_lbks(breloc, 384 breloc->parent->xfer_size, ppa); 385 ftl_reloc_iter_next_chk(breloc); 386 387 if (lbk_cnt || ftl_reloc_iter_done(breloc)) { 388 break; 389 } 390 } 391 392 return lbk_cnt; 393 } 394 395 static void 396 ftl_reloc_io_reinit(struct ftl_io *io, struct ftl_band_reloc *breloc, 397 spdk_ftl_fn fn, enum ftl_io_type io_type, int flags) 398 { 399 size_t i; 400 uint64_t lbkoff; 401 struct ftl_ppa ppa = io->ppa; 402 403 ftl_io_reinit(io, fn, io, flags | FTL_IO_INTERNAL, io_type); 404 405 io->ppa = ppa; 406 io->band = breloc->band; 407 io->lbas = calloc(io->lbk_cnt, sizeof(uint64_t)); 408 409 for (i = 0; i < io->lbk_cnt; ++i) { 410 ppa.lbk = io->ppa.lbk + i; 411 lbkoff = ftl_band_lbkoff_from_ppa(breloc->band, ppa); 412 413 if (!ftl_band_lbkoff_valid(breloc->band, lbkoff)) { 414 io->lbas[i] = FTL_LBA_INVALID; 415 continue; 416 } 417 418 io->lbas[i] = breloc->band->md.lba_map[lbkoff]; 419 } 420 421 ftl_trace_lba_io_init(io->dev, io); 422 } 423 424 static int 425 ftl_reloc_write(struct ftl_band_reloc *breloc, struct ftl_io *io) 426 { 427 int rc; 428 429 if (!(io->flags & FTL_IO_INITIALIZED)) { 430 ftl_reloc_io_reinit(io, breloc, ftl_reloc_write_cb, 431 FTL_IO_WRITE, 432 FTL_IO_KEEP_ALIVE | FTL_IO_WEAK | FTL_IO_VECTOR_LBA); 433 } 434 435 rc = ftl_io_write(io); 436 if (rc == -EAGAIN) { 437 spdk_ring_enqueue(breloc->write_queue, (void **)&io, 1); 438 return 0; 439 } 440 441 return rc; 442 } 443 444 static int 445 ftl_reloc_io_init(struct ftl_band_reloc *breloc, struct ftl_io *io, 446 struct ftl_ppa ppa, size_t num_lbks) 447 { 448 struct ftl_io_init_opts opts = { 449 .dev = breloc->parent->dev, 450 .io = io, 451 .rwb_batch = NULL, 452 .band = breloc->band, 453 .size = sizeof(*io), 454 .flags = FTL_IO_KEEP_ALIVE | FTL_IO_INTERNAL | FTL_IO_PPA_MODE, 455 .type = FTL_IO_READ, 456 .iov_cnt = 1, 457 .req_size = num_lbks, 458 .fn = ftl_reloc_read_cb, 459 }; 460 461 opts.data = spdk_dma_malloc(PAGE_SIZE * num_lbks, PAGE_SIZE, NULL); 462 if (!opts.data) { 463 return -1; 464 } 465 466 io = ftl_io_init_internal(&opts); 467 io->ppa = ppa; 468 return 0; 469 } 470 471 static int 472 ftl_reloc_read(struct ftl_band_reloc *breloc, struct ftl_io *io) 473 { 474 struct ftl_ppa ppa; 475 size_t num_lbks; 476 477 num_lbks = ftl_reloc_next_lbks(breloc, &ppa); 478 479 if (!num_lbks) { 480 spdk_ring_enqueue(breloc->free_queue, (void **)&io, 1); 481 return 0; 482 } 483 484 if (ftl_reloc_io_init(breloc, io, ppa, num_lbks)) { 485 SPDK_ERRLOG("Failed to initialize io for relocation."); 486 return -1; 487 } 488 489 return ftl_io_read(io); 490 } 491 492 static void 493 ftl_reloc_process_queue(struct ftl_band_reloc *breloc, struct spdk_ring *queue, 494 ftl_reloc_fn fn) 495 { 496 size_t i, num_ios; 497 struct ftl_reloc *reloc = breloc->parent; 498 499 num_ios = spdk_ring_dequeue(queue, (void **)reloc->io, reloc->max_qdepth); 500 501 for (i = 0; i < num_ios; ++i) { 502 if (fn(breloc, reloc->io[i])) { 503 SPDK_ERRLOG("Reloc queue processing failed\n"); 504 assert(false); 505 } 506 } 507 } 508 509 static void 510 ftl_reloc_process_write_queue(struct ftl_band_reloc *breloc) 511 { 512 ftl_reloc_process_queue(breloc, breloc->write_queue, ftl_reloc_write); 513 } 514 515 static void 516 ftl_reloc_process_free_queue(struct ftl_band_reloc *breloc) 517 { 518 ftl_reloc_process_queue(breloc, breloc->free_queue, ftl_reloc_read); 519 } 520 521 static int 522 ftl_reloc_done(struct ftl_band_reloc *breloc) 523 { 524 struct ftl_reloc *reloc = breloc->parent; 525 526 return spdk_ring_count(breloc->free_queue) == reloc->max_qdepth; 527 } 528 529 static void 530 ftl_reloc_release_io(struct ftl_band_reloc *breloc) 531 { 532 struct ftl_reloc *reloc = breloc->parent; 533 size_t i, num_ios; 534 535 num_ios = spdk_ring_dequeue(breloc->free_queue, (void **)reloc->io, reloc->max_qdepth); 536 537 for (i = 0; i < num_ios; ++i) { 538 ftl_io_free(reloc->io[i]); 539 } 540 } 541 542 static void 543 ftl_reloc_release(struct ftl_band_reloc *breloc) 544 { 545 struct ftl_reloc *reloc = breloc->parent; 546 struct ftl_band *band = breloc->band; 547 548 if (band->high_prio) { 549 band->high_prio = 0; 550 TAILQ_REMOVE(&reloc->prio_queue, breloc, entry); 551 } else { 552 TAILQ_REMOVE(&reloc->active_queue, breloc, entry); 553 } 554 555 ftl_reloc_release_io(breloc); 556 ftl_reloc_iter_reset(breloc); 557 ftl_band_release_md(band); 558 559 breloc->active = 0; 560 reloc->num_active--; 561 562 if (breloc->num_lbks) { 563 TAILQ_INSERT_TAIL(&reloc->pending_queue, breloc, entry); 564 return; 565 } 566 567 if (ftl_band_empty(band)) { 568 ftl_band_set_state(breloc->band, FTL_BAND_STATE_FREE); 569 } 570 } 571 572 static void 573 ftl_process_reloc(struct ftl_band_reloc *breloc) 574 { 575 ftl_reloc_process_free_queue(breloc); 576 577 ftl_reloc_process_write_queue(breloc); 578 579 if (ftl_reloc_done(breloc)) { 580 ftl_reloc_release(breloc); 581 } 582 } 583 584 static int 585 ftl_band_reloc_init(struct ftl_reloc *reloc, struct ftl_band_reloc *breloc, 586 struct ftl_band *band) 587 { 588 breloc->band = band; 589 breloc->parent = reloc; 590 591 breloc->reloc_map = spdk_bit_array_create(ftl_num_band_lbks(reloc->dev)); 592 if (!breloc->reloc_map) { 593 SPDK_ERRLOG("Failed to initialize reloc map"); 594 return -1; 595 } 596 597 breloc->iter.chk_offset = calloc(ftl_dev_num_punits(band->dev), 598 sizeof(*breloc->iter.chk_offset)); 599 if (!breloc->iter.chk_offset) { 600 SPDK_ERRLOG("Failed to initialize reloc iterator"); 601 return -1; 602 } 603 604 breloc->free_queue = spdk_ring_create(SPDK_RING_TYPE_MP_SC, 605 reloc->max_qdepth * 2, 606 SPDK_ENV_SOCKET_ID_ANY); 607 if (!breloc->free_queue) { 608 SPDK_ERRLOG("Failed to initialize reloc free queue"); 609 return -1; 610 } 611 612 breloc->write_queue = spdk_ring_create(SPDK_RING_TYPE_MP_SC, 613 reloc->max_qdepth * 2, 614 SPDK_ENV_SOCKET_ID_ANY); 615 if (!breloc->write_queue) { 616 SPDK_ERRLOG("Failed to initialize reloc write queue"); 617 return -1; 618 } 619 620 return 0; 621 } 622 623 static void 624 ftl_band_reloc_free(struct ftl_band_reloc *breloc) 625 { 626 if (!breloc) { 627 return; 628 } 629 630 spdk_ring_free(breloc->free_queue); 631 spdk_ring_free(breloc->write_queue); 632 spdk_bit_array_free(&breloc->reloc_map); 633 free(breloc->iter.chk_offset); 634 } 635 636 static void 637 ftl_reloc_add_active_queue(struct ftl_band_reloc *breloc) 638 { 639 struct ftl_reloc *reloc = breloc->parent; 640 641 TAILQ_REMOVE(&reloc->pending_queue, breloc, entry); 642 TAILQ_INSERT_HEAD(&reloc->active_queue, breloc, entry); 643 ftl_reloc_prep(breloc); 644 } 645 646 struct ftl_reloc * 647 ftl_reloc_init(struct spdk_ftl_dev *dev) 648 { 649 #define POOL_NAME_LEN 128 650 struct ftl_reloc *reloc; 651 char pool_name[POOL_NAME_LEN]; 652 int rc; 653 size_t i; 654 655 reloc = calloc(1, sizeof(*reloc)); 656 if (!reloc) { 657 return NULL; 658 } 659 660 reloc->dev = dev; 661 reloc->halt = true; 662 reloc->max_qdepth = dev->conf.max_reloc_qdepth; 663 reloc->max_active = dev->conf.max_active_relocs; 664 reloc->xfer_size = dev->xfer_size; 665 666 reloc->brelocs = calloc(ftl_dev_num_bands(dev), sizeof(*reloc->brelocs)); 667 if (!reloc->brelocs) { 668 goto error; 669 } 670 671 reloc->io = calloc(reloc->max_qdepth, sizeof(*reloc->io)); 672 if (!reloc->io) { 673 goto error; 674 } 675 676 for (i = 0; i < ftl_dev_num_bands(reloc->dev); ++i) { 677 if (ftl_band_reloc_init(reloc, &reloc->brelocs[i], &dev->bands[i])) { 678 goto error; 679 } 680 } 681 682 rc = snprintf(pool_name, sizeof(pool_name), "%s-%s", dev->name, "reloc-io-pool"); 683 if (rc < 0 || rc >= POOL_NAME_LEN) { 684 return NULL; 685 } 686 687 TAILQ_INIT(&reloc->pending_queue); 688 TAILQ_INIT(&reloc->active_queue); 689 TAILQ_INIT(&reloc->prio_queue); 690 691 return reloc; 692 error: 693 ftl_reloc_free(reloc); 694 return NULL; 695 } 696 697 void 698 ftl_reloc_free(struct ftl_reloc *reloc) 699 { 700 size_t i; 701 702 if (!reloc) { 703 return; 704 } 705 706 for (i = 0; i < ftl_dev_num_bands(reloc->dev); ++i) { 707 ftl_band_reloc_free(&reloc->brelocs[i]); 708 } 709 710 free(reloc->brelocs); 711 free(reloc->io); 712 free(reloc); 713 } 714 715 bool 716 ftl_reloc_is_halted(const struct ftl_reloc *reloc) 717 { 718 return reloc->halt; 719 } 720 721 void 722 ftl_reloc_halt(struct ftl_reloc *reloc) 723 { 724 reloc->halt = true; 725 } 726 727 void 728 ftl_reloc_resume(struct ftl_reloc *reloc) 729 { 730 reloc->halt = false; 731 } 732 733 void 734 ftl_reloc(struct ftl_reloc *reloc) 735 { 736 struct ftl_band_reloc *breloc, *tbreloc; 737 738 if (ftl_reloc_is_halted(reloc)) { 739 return; 740 } 741 742 /* Process first band from priority queue and return */ 743 breloc = TAILQ_FIRST(&reloc->prio_queue); 744 if (breloc) { 745 if (!breloc->active) { 746 ftl_reloc_prep(breloc); 747 } 748 ftl_process_reloc(breloc); 749 return; 750 } 751 752 TAILQ_FOREACH_SAFE(breloc, &reloc->pending_queue, entry, tbreloc) { 753 if (reloc->num_active == reloc->max_active) { 754 break; 755 } 756 ftl_reloc_add_active_queue(breloc); 757 } 758 759 TAILQ_FOREACH_SAFE(breloc, &reloc->active_queue, entry, tbreloc) { 760 ftl_process_reloc(breloc); 761 } 762 } 763 764 void 765 ftl_reloc_add(struct ftl_reloc *reloc, struct ftl_band *band, size_t offset, 766 size_t num_lbks, int prio) 767 { 768 struct ftl_band_reloc *breloc = &reloc->brelocs[band->id]; 769 size_t i, prev_lbks = breloc->num_lbks; 770 771 for (i = offset; i < offset + num_lbks; ++i) { 772 if (spdk_bit_array_get(breloc->reloc_map, i)) { 773 continue; 774 } 775 spdk_bit_array_set(breloc->reloc_map, i); 776 breloc->num_lbks++; 777 } 778 779 if (!prev_lbks && !prio) { 780 TAILQ_INSERT_HEAD(&reloc->pending_queue, breloc, entry); 781 } 782 783 if (prio) { 784 TAILQ_INSERT_TAIL(&reloc->prio_queue, breloc, entry); 785 ftl_band_acquire_md(breloc->band); 786 } 787 } 788