1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) Intel Corporation. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include "spdk/likely.h" 35 #include "spdk_internal/log.h" 36 #include "spdk/ftl.h" 37 #include "ftl_reloc.h" 38 #include "ftl_core.h" 39 #include "ftl_io.h" 40 #include "ftl_rwb.h" 41 #include "ftl_band.h" 42 #include "ftl_debug.h" 43 44 struct ftl_reloc; 45 struct ftl_band_reloc; 46 47 typedef int (*ftl_reloc_fn)(struct ftl_band_reloc *, struct ftl_io *); 48 49 struct ftl_band_reloc { 50 struct ftl_reloc *parent; 51 52 /* Band being relocated */ 53 struct ftl_band *band; 54 55 /* Number of logical blocks to be relocated */ 56 size_t num_lbks; 57 58 /* Bitmap of logical blocks to be relocated */ 59 struct spdk_bit_array *reloc_map; 60 61 /* Indicates band being acitvely processed */ 62 int active; 63 64 /* Reloc map iterator */ 65 struct { 66 /* Array of chunk offsets */ 67 size_t *chk_offset; 68 69 /* Currently chunk */ 70 size_t chk_current; 71 } iter; 72 73 /* Free IO queue */ 74 struct spdk_ring *free_queue; 75 76 /* Queue of IO ready to be written */ 77 struct spdk_ring *write_queue; 78 79 TAILQ_ENTRY(ftl_band_reloc) entry; 80 81 /* TODO: get rid of md_buf */ 82 void *md_buf; 83 }; 84 85 struct ftl_reloc { 86 /* Device associated with relocate */ 87 struct spdk_ftl_dev *dev; 88 89 /* Indicates relocate is about to halt */ 90 bool halt; 91 92 /* Maximum number of IOs per band */ 93 size_t max_qdepth; 94 95 /* IO buffer */ 96 struct ftl_io **io; 97 98 /* Maximum number of active band relocates */ 99 size_t max_active; 100 101 /* Maximum transfer size (in logical blocks) per single IO */ 102 size_t xfer_size; 103 104 /* Array of band relocates */ 105 struct ftl_band_reloc *brelocs; 106 107 /* Number of active/priority band relocates */ 108 size_t num_active; 109 110 /* Priority band relocates queue */ 111 TAILQ_HEAD(, ftl_band_reloc) prio_queue; 112 113 /* Active band relocates queue */ 114 TAILQ_HEAD(, ftl_band_reloc) active_queue; 115 116 /* Pending band relocates queue */ 117 TAILQ_HEAD(, ftl_band_reloc) pending_queue; 118 }; 119 120 static struct ftl_band_reloc * 121 ftl_io_get_band_reloc(struct ftl_io *io) 122 { 123 return &io->dev->reloc->brelocs[io->band->id]; 124 } 125 126 static size_t 127 ftl_reloc_iter_chk_offset(struct ftl_band_reloc *breloc) 128 { 129 size_t chunk = breloc->iter.chk_current; 130 131 return breloc->iter.chk_offset[chunk]; 132 } 133 134 static size_t 135 ftl_reloc_iter_chk_done(struct ftl_band_reloc *breloc) 136 { 137 size_t num_lbks = ftl_dev_lbks_in_chunk(breloc->parent->dev); 138 139 return ftl_reloc_iter_chk_offset(breloc) == num_lbks; 140 } 141 142 static void 143 ftl_reloc_clr_lbk(struct ftl_band_reloc *breloc, size_t lbkoff) 144 { 145 if (!spdk_bit_array_get(breloc->reloc_map, lbkoff)) { 146 return; 147 } 148 149 spdk_bit_array_clear(breloc->reloc_map, lbkoff); 150 assert(breloc->num_lbks); 151 breloc->num_lbks--; 152 } 153 154 static void 155 _ftl_reloc_prep(struct ftl_band_reloc *breloc) 156 { 157 struct ftl_io *io; 158 struct ftl_reloc *reloc = breloc->parent; 159 struct spdk_ftl_dev *dev = reloc->dev; 160 size_t i; 161 162 for (i = 0; i < reloc->max_qdepth; ++i) { 163 io = ftl_io_alloc(dev->ioch); 164 spdk_ring_enqueue(breloc->free_queue, (void **)&io, 1); 165 } 166 } 167 168 static void 169 ftl_reloc_read_lba_map_cb(void *arg, int status) 170 { 171 struct ftl_io *io = arg; 172 struct ftl_band_reloc *breloc = ftl_io_get_band_reloc(io); 173 174 assert(status == 0); 175 spdk_dma_free(breloc->md_buf); 176 ftl_io_free(io); 177 _ftl_reloc_prep(breloc); 178 } 179 180 static int 181 ftl_reloc_read_lba_map(struct ftl_band_reloc *breloc) 182 { 183 struct ftl_band *band = breloc->band; 184 struct spdk_ftl_dev *dev = band->dev; 185 struct ftl_io *io = ftl_io_alloc(dev->ioch); 186 187 io->dev = dev; 188 io->band = band; 189 io->cb.ctx = io; 190 io->cb.fn = ftl_reloc_read_lba_map_cb; 191 192 breloc->md_buf = spdk_dma_zmalloc(ftl_lba_map_num_lbks(dev) * FTL_BLOCK_SIZE, 193 FTL_BLOCK_SIZE, NULL); 194 if (!breloc->md_buf) { 195 return -1; 196 } 197 198 if (ftl_band_alloc_md(band)) { 199 assert(false); 200 } 201 202 return ftl_band_read_lba_map(band, &band->md, breloc->md_buf, &io->cb); 203 } 204 205 static void 206 ftl_reloc_prep(struct ftl_band_reloc *breloc) 207 { 208 struct ftl_band *band = breloc->band; 209 struct ftl_reloc *reloc = breloc->parent; 210 211 breloc->active = 1; 212 reloc->num_active++; 213 214 if (!band->high_prio) { 215 assert(band->md.lba_map == NULL); 216 ftl_reloc_read_lba_map(breloc); 217 return; 218 } 219 220 _ftl_reloc_prep(breloc); 221 } 222 223 static void 224 ftl_reloc_free_io(struct ftl_band_reloc *breloc, struct ftl_io *io) 225 { 226 spdk_dma_free(io->iov.iov_base); 227 free(io->lbas); 228 spdk_ring_enqueue(breloc->free_queue, (void **)&io, 1); 229 } 230 231 static void 232 ftl_reloc_write_cb(void *arg, int status) 233 { 234 struct ftl_io *io = arg; 235 struct ftl_ppa ppa = io->ppa; 236 struct ftl_band_reloc *breloc = ftl_io_get_band_reloc(io); 237 size_t i; 238 239 if (status) { 240 SPDK_ERRLOG("Reloc write failed with status: %d\n", status); 241 assert(false); 242 return; 243 } 244 245 for (i = 0; i < io->lbk_cnt; ++i) { 246 ppa.lbk = io->ppa.lbk + i; 247 size_t lbkoff = ftl_band_lbkoff_from_ppa(breloc->band, ppa); 248 ftl_reloc_clr_lbk(breloc, lbkoff); 249 } 250 251 ftl_reloc_free_io(breloc, io); 252 } 253 254 static void 255 ftl_reloc_read_cb(void *arg, int status) 256 { 257 struct ftl_io *io = arg; 258 struct ftl_band_reloc *breloc = ftl_io_get_band_reloc(io); 259 260 /* TODO: We should handle fail on relocation read. We need to inform */ 261 /* user that this group of blocks is bad (update l2p with bad block address and */ 262 /* put it to lba_map/sector_lba). Maybe we could also retry read with smaller granularity? */ 263 if (status) { 264 SPDK_ERRLOG("Reloc read failed with status: %d\n", status); 265 assert(false); 266 return; 267 } 268 269 io->flags &= ~FTL_IO_INITIALIZED; 270 spdk_ring_enqueue(breloc->write_queue, (void **)&io, 1); 271 } 272 273 static void 274 ftl_reloc_iter_reset(struct ftl_band_reloc *breloc) 275 { 276 memset(breloc->iter.chk_offset, 0, ftl_dev_num_punits(breloc->band->dev) * 277 sizeof(*breloc->iter.chk_offset)); 278 breloc->iter.chk_current = 0; 279 } 280 281 static size_t 282 ftl_reloc_iter_lbkoff(struct ftl_band_reloc *breloc) 283 { 284 size_t chk_offset = breloc->iter.chk_current * ftl_dev_lbks_in_chunk(breloc->parent->dev); 285 286 return breloc->iter.chk_offset[breloc->iter.chk_current] + chk_offset; 287 } 288 289 static void 290 ftl_reloc_iter_next_chk(struct ftl_band_reloc *breloc) 291 { 292 size_t num_chk = ftl_dev_num_punits(breloc->band->dev); 293 294 breloc->iter.chk_current = (breloc->iter.chk_current + 1) % num_chk; 295 } 296 297 static int 298 ftl_reloc_lbk_valid(struct ftl_band_reloc *breloc, size_t lbkoff) 299 { 300 return spdk_bit_array_get(breloc->reloc_map, lbkoff) && 301 ftl_band_lbkoff_valid(breloc->band, lbkoff); 302 } 303 304 static int 305 ftl_reloc_iter_next(struct ftl_band_reloc *breloc, size_t *lbkoff) 306 { 307 size_t chunk = breloc->iter.chk_current; 308 309 *lbkoff = ftl_reloc_iter_lbkoff(breloc); 310 311 if (ftl_reloc_iter_chk_done(breloc)) { 312 return 0; 313 } 314 315 breloc->iter.chk_offset[chunk]++; 316 317 if (!ftl_reloc_lbk_valid(breloc, *lbkoff)) { 318 ftl_reloc_clr_lbk(breloc, *lbkoff); 319 return 0; 320 } 321 322 return 1; 323 } 324 325 static int 326 ftl_reloc_first_valid_lbk(struct ftl_band_reloc *breloc, size_t *lbkoff) 327 { 328 size_t i, num_lbks = ftl_dev_lbks_in_chunk(breloc->parent->dev); 329 330 for (i = ftl_reloc_iter_chk_offset(breloc); i < num_lbks; ++i) { 331 if (ftl_reloc_iter_next(breloc, lbkoff)) { 332 return 1; 333 } 334 } 335 336 return 0; 337 } 338 339 static int 340 ftl_reloc_iter_done(struct ftl_band_reloc *breloc) 341 { 342 size_t i; 343 size_t num_chks = ftl_dev_num_punits(breloc->band->dev); 344 size_t num_lbks = ftl_dev_lbks_in_chunk(breloc->parent->dev); 345 346 for (i = 0; i < num_chks; ++i) { 347 if (breloc->iter.chk_offset[i] != num_lbks) { 348 return 0; 349 } 350 } 351 352 return 1; 353 } 354 355 static size_t 356 ftl_reloc_find_valid_lbks(struct ftl_band_reloc *breloc, 357 size_t num_lbk, struct ftl_ppa *ppa) 358 { 359 size_t lbkoff, lbk_cnt = 0; 360 361 if (!ftl_reloc_first_valid_lbk(breloc, &lbkoff)) { 362 return 0; 363 } 364 365 *ppa = ftl_band_ppa_from_lbkoff(breloc->band, lbkoff); 366 367 for (lbk_cnt = 1; lbk_cnt < num_lbk; lbk_cnt++) { 368 if (!ftl_reloc_iter_next(breloc, &lbkoff)) { 369 break; 370 } 371 } 372 373 return lbk_cnt; 374 } 375 376 static size_t 377 ftl_reloc_next_lbks(struct ftl_band_reloc *breloc, struct ftl_ppa *ppa) 378 { 379 size_t i, lbk_cnt = 0; 380 struct spdk_ftl_dev *dev = breloc->parent->dev; 381 382 for (i = 0; i < ftl_dev_num_punits(dev); ++i) { 383 lbk_cnt = ftl_reloc_find_valid_lbks(breloc, 384 breloc->parent->xfer_size, ppa); 385 ftl_reloc_iter_next_chk(breloc); 386 387 if (lbk_cnt || ftl_reloc_iter_done(breloc)) { 388 break; 389 } 390 } 391 392 return lbk_cnt; 393 } 394 395 static void 396 ftl_reloc_io_reinit(struct ftl_io *io, struct ftl_band_reloc *breloc, 397 spdk_ftl_fn fn, enum ftl_io_type io_type, int flags) 398 { 399 size_t i; 400 uint64_t lbkoff; 401 struct ftl_ppa ppa = io->ppa; 402 403 ftl_io_reinit(io, fn, io, flags | FTL_IO_INTERNAL, io_type); 404 405 io->ppa = ppa; 406 io->band = breloc->band; 407 io->lbas = calloc(io->lbk_cnt, sizeof(uint64_t)); 408 409 for (i = 0; i < io->lbk_cnt; ++i) { 410 ppa.lbk = io->ppa.lbk + i; 411 lbkoff = ftl_band_lbkoff_from_ppa(breloc->band, ppa); 412 413 if (!ftl_band_lbkoff_valid(breloc->band, lbkoff)) { 414 io->lbas[i] = FTL_LBA_INVALID; 415 continue; 416 } 417 418 io->lbas[i] = breloc->band->md.lba_map[lbkoff]; 419 } 420 421 ftl_trace_lba_io_init(io->dev, io); 422 } 423 424 static int 425 ftl_reloc_write(struct ftl_band_reloc *breloc, struct ftl_io *io) 426 { 427 int rc; 428 429 if (!(io->flags & FTL_IO_INITIALIZED)) { 430 ftl_reloc_io_reinit(io, breloc, ftl_reloc_write_cb, 431 FTL_IO_WRITE, 432 FTL_IO_KEEP_ALIVE | FTL_IO_WEAK | FTL_IO_VECTOR_LBA); 433 } 434 435 rc = ftl_io_write(io); 436 if (rc == -EAGAIN) { 437 spdk_ring_enqueue(breloc->write_queue, (void **)&io, 1); 438 return 0; 439 } 440 441 return rc; 442 } 443 444 static int 445 ftl_reloc_io_init(struct ftl_band_reloc *breloc, struct ftl_io *io, 446 struct ftl_ppa ppa, size_t num_lbks) 447 { 448 struct ftl_io_init_opts opts = { 449 .dev = breloc->parent->dev, 450 .io = io, 451 .rwb_batch = NULL, 452 .band = breloc->band, 453 .size = sizeof(*io), 454 .flags = FTL_IO_KEEP_ALIVE | FTL_IO_INTERNAL | FTL_IO_PPA_MODE, 455 .type = FTL_IO_READ, 456 .iov_cnt = 1, 457 .req_size = num_lbks, 458 .fn = ftl_reloc_read_cb, 459 }; 460 461 opts.data = spdk_dma_malloc(PAGE_SIZE * num_lbks, PAGE_SIZE, NULL); 462 if (!opts.data) { 463 return -1; 464 } 465 466 io = ftl_io_init_internal(&opts); 467 io->ppa = ppa; 468 return 0; 469 } 470 471 static int 472 ftl_reloc_read(struct ftl_band_reloc *breloc, struct ftl_io *io) 473 { 474 struct ftl_ppa ppa; 475 size_t num_lbks; 476 int rc; 477 478 num_lbks = ftl_reloc_next_lbks(breloc, &ppa); 479 480 if (!num_lbks) { 481 spdk_ring_enqueue(breloc->free_queue, (void **)&io, 1); 482 return 0; 483 } 484 485 if (ftl_reloc_io_init(breloc, io, ppa, num_lbks)) { 486 SPDK_ERRLOG("Failed to initialize io for relocation."); 487 return -1; 488 } 489 490 rc = ftl_io_read(io); 491 if (rc == -ENOMEM) { 492 rc = 0; 493 } 494 495 return rc; 496 } 497 498 static void 499 ftl_reloc_process_queue(struct ftl_band_reloc *breloc, struct spdk_ring *queue, 500 ftl_reloc_fn fn) 501 { 502 size_t i, num_ios; 503 struct ftl_reloc *reloc = breloc->parent; 504 505 num_ios = spdk_ring_dequeue(queue, (void **)reloc->io, reloc->max_qdepth); 506 507 for (i = 0; i < num_ios; ++i) { 508 if (fn(breloc, reloc->io[i])) { 509 SPDK_ERRLOG("Reloc queue processing failed\n"); 510 assert(false); 511 } 512 } 513 } 514 515 static void 516 ftl_reloc_process_write_queue(struct ftl_band_reloc *breloc) 517 { 518 ftl_reloc_process_queue(breloc, breloc->write_queue, ftl_reloc_write); 519 } 520 521 static void 522 ftl_reloc_process_free_queue(struct ftl_band_reloc *breloc) 523 { 524 ftl_reloc_process_queue(breloc, breloc->free_queue, ftl_reloc_read); 525 } 526 527 static int 528 ftl_reloc_done(struct ftl_band_reloc *breloc) 529 { 530 struct ftl_reloc *reloc = breloc->parent; 531 532 return spdk_ring_count(breloc->free_queue) == reloc->max_qdepth; 533 } 534 535 static void 536 ftl_reloc_release_io(struct ftl_band_reloc *breloc) 537 { 538 struct ftl_reloc *reloc = breloc->parent; 539 size_t i, num_ios; 540 541 num_ios = spdk_ring_dequeue(breloc->free_queue, (void **)reloc->io, reloc->max_qdepth); 542 543 for (i = 0; i < num_ios; ++i) { 544 ftl_io_free(reloc->io[i]); 545 } 546 } 547 548 static void 549 ftl_reloc_release(struct ftl_band_reloc *breloc) 550 { 551 struct ftl_reloc *reloc = breloc->parent; 552 struct ftl_band *band = breloc->band; 553 554 if (band->high_prio) { 555 band->high_prio = 0; 556 TAILQ_REMOVE(&reloc->prio_queue, breloc, entry); 557 } else { 558 TAILQ_REMOVE(&reloc->active_queue, breloc, entry); 559 } 560 561 ftl_reloc_release_io(breloc); 562 ftl_reloc_iter_reset(breloc); 563 ftl_band_release_md(band); 564 565 breloc->active = 0; 566 reloc->num_active--; 567 568 if (breloc->num_lbks) { 569 TAILQ_INSERT_TAIL(&reloc->pending_queue, breloc, entry); 570 return; 571 } 572 573 if (ftl_band_empty(band)) { 574 ftl_band_set_state(breloc->band, FTL_BAND_STATE_FREE); 575 } 576 } 577 578 static void 579 ftl_process_reloc(struct ftl_band_reloc *breloc) 580 { 581 ftl_reloc_process_free_queue(breloc); 582 583 ftl_reloc_process_write_queue(breloc); 584 585 if (ftl_reloc_done(breloc)) { 586 ftl_reloc_release(breloc); 587 } 588 } 589 590 static int 591 ftl_band_reloc_init(struct ftl_reloc *reloc, struct ftl_band_reloc *breloc, 592 struct ftl_band *band) 593 { 594 breloc->band = band; 595 breloc->parent = reloc; 596 597 breloc->reloc_map = spdk_bit_array_create(ftl_num_band_lbks(reloc->dev)); 598 if (!breloc->reloc_map) { 599 SPDK_ERRLOG("Failed to initialize reloc map"); 600 return -1; 601 } 602 603 breloc->iter.chk_offset = calloc(ftl_dev_num_punits(band->dev), 604 sizeof(*breloc->iter.chk_offset)); 605 if (!breloc->iter.chk_offset) { 606 SPDK_ERRLOG("Failed to initialize reloc iterator"); 607 return -1; 608 } 609 610 breloc->free_queue = spdk_ring_create(SPDK_RING_TYPE_MP_SC, 611 reloc->max_qdepth * 2, 612 SPDK_ENV_SOCKET_ID_ANY); 613 if (!breloc->free_queue) { 614 SPDK_ERRLOG("Failed to initialize reloc free queue"); 615 return -1; 616 } 617 618 breloc->write_queue = spdk_ring_create(SPDK_RING_TYPE_MP_SC, 619 reloc->max_qdepth * 2, 620 SPDK_ENV_SOCKET_ID_ANY); 621 if (!breloc->write_queue) { 622 SPDK_ERRLOG("Failed to initialize reloc write queue"); 623 return -1; 624 } 625 626 return 0; 627 } 628 629 static void 630 ftl_band_reloc_free(struct ftl_band_reloc *breloc) 631 { 632 struct ftl_reloc *reloc = breloc->parent; 633 struct ftl_io *io; 634 size_t i, num_ios; 635 636 if (!breloc) { 637 return; 638 } 639 640 if (breloc->active) { 641 num_ios = spdk_ring_dequeue(breloc->write_queue, (void **)reloc->io, reloc->max_qdepth); 642 for (i = 0; i < num_ios; ++i) { 643 io = reloc->io[i]; 644 if (io->flags & FTL_IO_INITIALIZED) { 645 ftl_reloc_free_io(breloc, io); 646 } 647 } 648 649 ftl_reloc_release_io(breloc); 650 } 651 652 spdk_ring_free(breloc->free_queue); 653 spdk_ring_free(breloc->write_queue); 654 spdk_bit_array_free(&breloc->reloc_map); 655 free(breloc->iter.chk_offset); 656 } 657 658 static void 659 ftl_reloc_add_active_queue(struct ftl_band_reloc *breloc) 660 { 661 struct ftl_reloc *reloc = breloc->parent; 662 663 TAILQ_REMOVE(&reloc->pending_queue, breloc, entry); 664 TAILQ_INSERT_HEAD(&reloc->active_queue, breloc, entry); 665 ftl_reloc_prep(breloc); 666 } 667 668 struct ftl_reloc * 669 ftl_reloc_init(struct spdk_ftl_dev *dev) 670 { 671 #define POOL_NAME_LEN 128 672 struct ftl_reloc *reloc; 673 char pool_name[POOL_NAME_LEN]; 674 int rc; 675 size_t i; 676 677 reloc = calloc(1, sizeof(*reloc)); 678 if (!reloc) { 679 return NULL; 680 } 681 682 reloc->dev = dev; 683 reloc->halt = true; 684 reloc->max_qdepth = dev->conf.max_reloc_qdepth; 685 reloc->max_active = dev->conf.max_active_relocs; 686 reloc->xfer_size = dev->xfer_size; 687 688 reloc->brelocs = calloc(ftl_dev_num_bands(dev), sizeof(*reloc->brelocs)); 689 if (!reloc->brelocs) { 690 goto error; 691 } 692 693 reloc->io = calloc(reloc->max_qdepth, sizeof(*reloc->io)); 694 if (!reloc->io) { 695 goto error; 696 } 697 698 for (i = 0; i < ftl_dev_num_bands(reloc->dev); ++i) { 699 if (ftl_band_reloc_init(reloc, &reloc->brelocs[i], &dev->bands[i])) { 700 goto error; 701 } 702 } 703 704 rc = snprintf(pool_name, sizeof(pool_name), "%s-%s", dev->name, "reloc-io-pool"); 705 if (rc < 0 || rc >= POOL_NAME_LEN) { 706 return NULL; 707 } 708 709 TAILQ_INIT(&reloc->pending_queue); 710 TAILQ_INIT(&reloc->active_queue); 711 TAILQ_INIT(&reloc->prio_queue); 712 713 return reloc; 714 error: 715 ftl_reloc_free(reloc); 716 return NULL; 717 } 718 719 void 720 ftl_reloc_free(struct ftl_reloc *reloc) 721 { 722 size_t i; 723 724 if (!reloc) { 725 return; 726 } 727 728 for (i = 0; i < ftl_dev_num_bands(reloc->dev); ++i) { 729 ftl_band_reloc_free(&reloc->brelocs[i]); 730 } 731 732 free(reloc->brelocs); 733 free(reloc->io); 734 free(reloc); 735 } 736 737 bool 738 ftl_reloc_is_halted(const struct ftl_reloc *reloc) 739 { 740 return reloc->halt; 741 } 742 743 void 744 ftl_reloc_halt(struct ftl_reloc *reloc) 745 { 746 reloc->halt = true; 747 } 748 749 void 750 ftl_reloc_resume(struct ftl_reloc *reloc) 751 { 752 reloc->halt = false; 753 } 754 755 void 756 ftl_reloc(struct ftl_reloc *reloc) 757 { 758 struct ftl_band_reloc *breloc, *tbreloc; 759 760 if (ftl_reloc_is_halted(reloc)) { 761 return; 762 } 763 764 /* Process first band from priority queue and return */ 765 breloc = TAILQ_FIRST(&reloc->prio_queue); 766 if (breloc) { 767 if (!breloc->active) { 768 ftl_reloc_prep(breloc); 769 } 770 ftl_process_reloc(breloc); 771 return; 772 } 773 774 TAILQ_FOREACH_SAFE(breloc, &reloc->pending_queue, entry, tbreloc) { 775 if (reloc->num_active == reloc->max_active) { 776 break; 777 } 778 ftl_reloc_add_active_queue(breloc); 779 } 780 781 TAILQ_FOREACH_SAFE(breloc, &reloc->active_queue, entry, tbreloc) { 782 ftl_process_reloc(breloc); 783 } 784 } 785 786 void 787 ftl_reloc_add(struct ftl_reloc *reloc, struct ftl_band *band, size_t offset, 788 size_t num_lbks, int prio) 789 { 790 struct ftl_band_reloc *breloc = &reloc->brelocs[band->id]; 791 size_t i, prev_lbks = breloc->num_lbks; 792 793 for (i = offset; i < offset + num_lbks; ++i) { 794 if (spdk_bit_array_get(breloc->reloc_map, i)) { 795 continue; 796 } 797 spdk_bit_array_set(breloc->reloc_map, i); 798 breloc->num_lbks++; 799 } 800 801 if (!prev_lbks && !prio) { 802 TAILQ_INSERT_HEAD(&reloc->pending_queue, breloc, entry); 803 } 804 805 if (prio) { 806 TAILQ_INSERT_TAIL(&reloc->prio_queue, breloc, entry); 807 ftl_band_acquire_md(breloc->band); 808 } 809 } 810