1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2022 Intel Corporation. 3 * All rights reserved. 4 */ 5 6 #include "spdk/stdinc.h" 7 #include "spdk/queue.h" 8 #include "spdk/bdev_module.h" 9 10 #include "ftl_core.h" 11 #include "ftl_band.h" 12 #include "ftl_internal.h" 13 14 static void 15 write_rq_end(struct spdk_bdev_io *bdev_io, bool success, void *arg) 16 { 17 struct ftl_rq *rq = arg; 18 struct spdk_ftl_dev *dev = rq->dev; 19 20 ftl_stats_bdev_io_completed(dev, rq->owner.compaction ? FTL_STATS_TYPE_CMP : FTL_STATS_TYPE_GC, 21 bdev_io); 22 23 rq->success = success; 24 25 ftl_p2l_ckpt_issue(rq); 26 27 spdk_bdev_free_io(bdev_io); 28 } 29 30 static void 31 ftl_band_rq_bdev_write(void *_rq) 32 { 33 struct ftl_rq *rq = _rq; 34 struct ftl_band *band = rq->io.band; 35 struct spdk_ftl_dev *dev = band->dev; 36 int rc; 37 38 rc = spdk_bdev_writev_blocks(dev->base_bdev_desc, dev->base_ioch, 39 rq->io_vec, rq->io_vec_size, 40 rq->io.addr, rq->num_blocks, 41 write_rq_end, rq); 42 43 if (spdk_unlikely(rc)) { 44 if (rc == -ENOMEM) { 45 struct spdk_bdev *bdev = spdk_bdev_desc_get_bdev(dev->base_bdev_desc); 46 rq->io.bdev_io_wait.bdev = bdev; 47 rq->io.bdev_io_wait.cb_fn = ftl_band_rq_bdev_write; 48 rq->io.bdev_io_wait.cb_arg = rq; 49 spdk_bdev_queue_io_wait(bdev, dev->base_ioch, &rq->io.bdev_io_wait); 50 } else { 51 ftl_abort(); 52 } 53 } 54 } 55 56 void 57 ftl_band_rq_write(struct ftl_band *band, struct ftl_rq *rq) 58 { 59 struct spdk_ftl_dev *dev = band->dev; 60 61 rq->success = false; 62 rq->io.band = band; 63 rq->io.addr = band->md->iter.addr; 64 65 ftl_band_rq_bdev_write(rq); 66 67 band->queue_depth++; 68 dev->stats.io_activity_total += rq->num_blocks; 69 70 ftl_band_iter_advance(band, rq->num_blocks); 71 if (ftl_band_filled(band, band->md->iter.offset)) { 72 ftl_band_set_state(band, FTL_BAND_STATE_FULL); 73 band->owner.state_change_fn(band); 74 } 75 } 76 77 static void ftl_band_rq_bdev_read(void *_entry); 78 79 static void 80 read_rq_end(struct spdk_bdev_io *bdev_io, bool success, void *arg) 81 { 82 struct ftl_rq_entry *entry = arg; 83 struct ftl_band *band = entry->io.band; 84 struct ftl_rq *rq = ftl_rq_from_entry(entry); 85 86 ftl_stats_bdev_io_completed(band->dev, FTL_STATS_TYPE_GC, bdev_io); 87 88 rq->success = success; 89 if (spdk_unlikely(!success)) { 90 ftl_band_rq_bdev_read(entry); 91 spdk_bdev_free_io(bdev_io); 92 return; 93 } 94 95 assert(band->queue_depth > 0); 96 band->queue_depth--; 97 98 rq->owner.cb(rq); 99 spdk_bdev_free_io(bdev_io); 100 } 101 102 static void 103 ftl_band_rq_bdev_read(void *_entry) 104 { 105 struct ftl_rq_entry *entry = _entry; 106 struct ftl_rq *rq = ftl_rq_from_entry(entry); 107 struct spdk_ftl_dev *dev = rq->dev; 108 int rc; 109 110 rc = spdk_bdev_read_blocks(dev->base_bdev_desc, dev->base_ioch, entry->io_payload, 111 entry->bdev_io.offset_blocks, entry->bdev_io.num_blocks, 112 read_rq_end, entry); 113 if (spdk_unlikely(rc)) { 114 if (rc == -ENOMEM) { 115 struct spdk_bdev *bdev = spdk_bdev_desc_get_bdev(dev->base_bdev_desc); 116 entry->bdev_io.wait_entry.bdev = bdev; 117 entry->bdev_io.wait_entry.cb_fn = ftl_band_rq_bdev_read; 118 entry->bdev_io.wait_entry.cb_arg = entry; 119 spdk_bdev_queue_io_wait(bdev, dev->base_ioch, &entry->bdev_io.wait_entry); 120 } else { 121 ftl_abort(); 122 } 123 } 124 } 125 126 void 127 ftl_band_rq_read(struct ftl_band *band, struct ftl_rq *rq) 128 { 129 struct spdk_ftl_dev *dev = band->dev; 130 struct ftl_rq_entry *entry = &rq->entries[rq->iter.idx]; 131 132 assert(rq->iter.idx + rq->iter.count <= rq->num_blocks); 133 134 rq->success = false; 135 rq->io.band = band; 136 rq->io.addr = band->md->iter.addr; 137 entry->io.band = band; 138 entry->bdev_io.offset_blocks = rq->io.addr; 139 entry->bdev_io.num_blocks = rq->iter.count; 140 141 ftl_band_rq_bdev_read(entry); 142 143 dev->stats.io_activity_total += rq->num_blocks; 144 band->queue_depth++; 145 } 146 147 static void 148 write_brq_end(struct spdk_bdev_io *bdev_io, bool success, void *arg) 149 { 150 struct ftl_basic_rq *brq = arg; 151 struct ftl_band *band = brq->io.band; 152 153 ftl_stats_bdev_io_completed(band->dev, FTL_STATS_TYPE_MD_BASE, bdev_io); 154 155 brq->success = success; 156 157 assert(band->queue_depth > 0); 158 band->queue_depth--; 159 160 brq->owner.cb(brq); 161 spdk_bdev_free_io(bdev_io); 162 } 163 164 static void 165 ftl_band_brq_bdev_write(void *_brq) 166 { 167 struct ftl_basic_rq *brq = _brq; 168 struct spdk_ftl_dev *dev = brq->dev; 169 int rc; 170 171 rc = spdk_bdev_write_blocks(dev->base_bdev_desc, dev->base_ioch, 172 brq->io_payload, brq->io.addr, 173 brq->num_blocks, write_brq_end, brq); 174 175 if (spdk_unlikely(rc)) { 176 if (rc == -ENOMEM) { 177 struct spdk_bdev *bdev = spdk_bdev_desc_get_bdev(dev->base_bdev_desc); 178 brq->io.bdev_io_wait.bdev = bdev; 179 brq->io.bdev_io_wait.cb_fn = ftl_band_brq_bdev_write; 180 brq->io.bdev_io_wait.cb_arg = brq; 181 spdk_bdev_queue_io_wait(bdev, dev->base_ioch, &brq->io.bdev_io_wait); 182 } else { 183 ftl_abort(); 184 } 185 } 186 } 187 188 void 189 ftl_band_basic_rq_write(struct ftl_band *band, struct ftl_basic_rq *brq) 190 { 191 struct spdk_ftl_dev *dev = band->dev; 192 193 brq->io.addr = band->md->iter.addr; 194 brq->io.band = band; 195 brq->success = false; 196 197 ftl_band_brq_bdev_write(brq); 198 199 dev->stats.io_activity_total += brq->num_blocks; 200 band->queue_depth++; 201 ftl_band_iter_advance(band, brq->num_blocks); 202 if (ftl_band_filled(band, band->md->iter.offset)) { 203 ftl_band_set_state(band, FTL_BAND_STATE_FULL); 204 band->owner.state_change_fn(band); 205 } 206 } 207 208 static void 209 read_brq_end(struct spdk_bdev_io *bdev_io, bool success, void *arg) 210 { 211 struct ftl_basic_rq *brq = arg; 212 struct ftl_band *band = brq->io.band; 213 214 ftl_stats_bdev_io_completed(band->dev, FTL_STATS_TYPE_MD_BASE, bdev_io); 215 216 brq->success = success; 217 218 assert(band->queue_depth > 0); 219 band->queue_depth--; 220 221 brq->owner.cb(brq); 222 spdk_bdev_free_io(bdev_io); 223 } 224 225 static void 226 ftl_band_brq_bdev_read(void *_brq) 227 { 228 struct ftl_basic_rq *brq = _brq; 229 struct spdk_ftl_dev *dev = brq->dev; 230 int rc; 231 232 rc = spdk_bdev_read_blocks(dev->base_bdev_desc, dev->base_ioch, 233 brq->io_payload, brq->io.addr, 234 brq->num_blocks, read_brq_end, brq); 235 if (spdk_unlikely(rc)) { 236 if (rc == -ENOMEM) { 237 struct spdk_bdev *bdev = spdk_bdev_desc_get_bdev(dev->base_bdev_desc); 238 brq->io.bdev_io_wait.bdev = bdev; 239 brq->io.bdev_io_wait.cb_fn = ftl_band_brq_bdev_read; 240 brq->io.bdev_io_wait.cb_arg = brq; 241 spdk_bdev_queue_io_wait(bdev, dev->base_ioch, &brq->io.bdev_io_wait); 242 } else { 243 ftl_abort(); 244 } 245 } 246 } 247 248 void 249 ftl_band_basic_rq_read(struct ftl_band *band, struct ftl_basic_rq *brq) 250 { 251 struct spdk_ftl_dev *dev = brq->dev; 252 253 brq->io.band = band; 254 255 ftl_band_brq_bdev_read(brq); 256 257 brq->io.band->queue_depth++; 258 dev->stats.io_activity_total += brq->num_blocks; 259 } 260 261 static void 262 band_open_cb(int status, void *cb_arg) 263 { 264 struct ftl_band *band = cb_arg; 265 266 if (spdk_unlikely(status)) { 267 #ifdef SPDK_FTL_RETRY_ON_ERROR 268 ftl_md_persist_entry_retry(&band->md_persist_entry_ctx); 269 return; 270 #else 271 ftl_abort(); 272 #endif 273 } 274 275 ftl_band_set_state(band, FTL_BAND_STATE_OPEN); 276 } 277 278 void 279 ftl_band_open(struct ftl_band *band, enum ftl_band_type type) 280 { 281 struct spdk_ftl_dev *dev = band->dev; 282 struct ftl_md *md = dev->layout.md[FTL_LAYOUT_REGION_TYPE_BAND_MD]; 283 struct ftl_layout_region *region = &dev->layout.region[FTL_LAYOUT_REGION_TYPE_BAND_MD]; 284 struct ftl_p2l_map *p2l_map = &band->p2l_map; 285 286 ftl_band_set_type(band, type); 287 ftl_band_set_state(band, FTL_BAND_STATE_OPENING); 288 289 memcpy(p2l_map->band_dma_md, band->md, region->entry_size * FTL_BLOCK_SIZE); 290 p2l_map->band_dma_md->state = FTL_BAND_STATE_OPEN; 291 p2l_map->band_dma_md->p2l_map_checksum = 0; 292 293 if (spdk_unlikely(0 != band->p2l_map.num_valid)) { 294 /* 295 * This is inconsistent state, a band with valid block, 296 * it could be moved on the free list 297 */ 298 assert(false && 0 == band->p2l_map.num_valid); 299 ftl_abort(); 300 } 301 302 ftl_md_persist_entry(md, band->id, p2l_map->band_dma_md, NULL, 303 band_open_cb, band, &band->md_persist_entry_ctx); 304 } 305 306 static void 307 band_close_cb(int status, void *cb_arg) 308 { 309 struct ftl_band *band = cb_arg; 310 311 if (spdk_unlikely(status)) { 312 #ifdef SPDK_FTL_RETRY_ON_ERROR 313 ftl_md_persist_entry_retry(&band->md_persist_entry_ctx); 314 return; 315 #else 316 ftl_abort(); 317 #endif 318 } 319 320 band->md->p2l_map_checksum = band->p2l_map.band_dma_md->p2l_map_checksum; 321 ftl_band_set_state(band, FTL_BAND_STATE_CLOSED); 322 } 323 324 static void 325 band_map_write_cb(struct ftl_basic_rq *brq) 326 { 327 struct ftl_band *band = brq->io.band; 328 struct ftl_p2l_map *p2l_map = &band->p2l_map; 329 struct spdk_ftl_dev *dev = band->dev; 330 struct ftl_layout_region *region = &dev->layout.region[FTL_LAYOUT_REGION_TYPE_BAND_MD]; 331 struct ftl_md *md = dev->layout.md[FTL_LAYOUT_REGION_TYPE_BAND_MD]; 332 uint32_t band_map_crc; 333 334 if (spdk_likely(brq->success)) { 335 336 band_map_crc = spdk_crc32c_update(p2l_map->band_map, 337 ftl_tail_md_num_blocks(dev) * FTL_BLOCK_SIZE, 0); 338 memcpy(p2l_map->band_dma_md, band->md, region->entry_size * FTL_BLOCK_SIZE); 339 p2l_map->band_dma_md->state = FTL_BAND_STATE_CLOSED; 340 p2l_map->band_dma_md->p2l_map_checksum = band_map_crc; 341 342 ftl_md_persist_entry(md, band->id, p2l_map->band_dma_md, NULL, 343 band_close_cb, band, &band->md_persist_entry_ctx); 344 } else { 345 #ifdef SPDK_FTL_RETRY_ON_ERROR 346 /* Try to retry in case of failure */ 347 ftl_band_brq_bdev_write(brq); 348 band->queue_depth++; 349 #else 350 ftl_abort(); 351 #endif 352 } 353 } 354 355 void 356 ftl_band_close(struct ftl_band *band) 357 { 358 struct spdk_ftl_dev *dev = band->dev; 359 void *metadata = band->p2l_map.band_map; 360 uint64_t num_blocks = ftl_tail_md_num_blocks(dev); 361 362 /* Write P2L map first, after completion, set the state to close on nvcache, then internally */ 363 band->md->close_seq_id = ftl_get_next_seq_id(dev); 364 ftl_band_set_state(band, FTL_BAND_STATE_CLOSING); 365 ftl_basic_rq_init(dev, &band->metadata_rq, metadata, num_blocks); 366 ftl_basic_rq_set_owner(&band->metadata_rq, band_map_write_cb, band); 367 368 ftl_band_basic_rq_write(band, &band->metadata_rq); 369 } 370 371 static void 372 band_free_cb(int status, void *ctx) 373 { 374 struct ftl_band *band = (struct ftl_band *)ctx; 375 376 if (spdk_unlikely(status)) { 377 #ifdef SPDK_FTL_RETRY_ON_ERROR 378 ftl_md_persist_entry_retry(&band->md_persist_entry_ctx); 379 return; 380 #else 381 ftl_abort(); 382 #endif 383 } 384 385 ftl_band_release_p2l_map(band); 386 FTL_DEBUGLOG(band->dev, "Band is going to free state. Band id: %u\n", band->id); 387 ftl_band_set_state(band, FTL_BAND_STATE_FREE); 388 assert(0 == band->p2l_map.ref_cnt); 389 } 390 391 void 392 ftl_band_free(struct ftl_band *band) 393 { 394 struct spdk_ftl_dev *dev = band->dev; 395 struct ftl_p2l_map *p2l_map = &band->p2l_map; 396 struct ftl_md *md = dev->layout.md[FTL_LAYOUT_REGION_TYPE_BAND_MD]; 397 struct ftl_layout_region *region = &dev->layout.region[FTL_LAYOUT_REGION_TYPE_BAND_MD]; 398 399 memcpy(p2l_map->band_dma_md, band->md, region->entry_size * FTL_BLOCK_SIZE); 400 p2l_map->band_dma_md->state = FTL_BAND_STATE_FREE; 401 p2l_map->band_dma_md->close_seq_id = 0; 402 p2l_map->band_dma_md->p2l_map_checksum = 0; 403 404 ftl_md_persist_entry(md, band->id, p2l_map->band_dma_md, NULL, 405 band_free_cb, band, &band->md_persist_entry_ctx); 406 407 /* TODO: The whole band erase code should probably be done here instead */ 408 } 409 410 static void 411 read_md_cb(struct ftl_basic_rq *brq) 412 { 413 struct ftl_band *band = brq->owner.priv; 414 struct spdk_ftl_dev *dev = band->dev; 415 ftl_band_ops_cb cb; 416 uint32_t band_map_crc; 417 bool success = true; 418 void *priv; 419 420 cb = band->owner.ops_fn; 421 priv = band->owner.priv; 422 423 if (!brq->success) { 424 ftl_band_basic_rq_read(band, &band->metadata_rq); 425 return; 426 } 427 428 band_map_crc = spdk_crc32c_update(band->p2l_map.band_map, 429 ftl_tail_md_num_blocks(band->dev) * FTL_BLOCK_SIZE, 0); 430 if (band->md->p2l_map_checksum && band->md->p2l_map_checksum != band_map_crc) { 431 FTL_ERRLOG(dev, "GC error, inconsistent P2L map CRC\n"); 432 success = false; 433 434 ftl_stats_crc_error(band->dev, FTL_STATS_TYPE_GC); 435 } 436 band->owner.ops_fn = NULL; 437 band->owner.priv = NULL; 438 cb(band, priv, success); 439 } 440 441 static int 442 _read_md(struct ftl_band *band) 443 { 444 struct spdk_ftl_dev *dev = band->dev; 445 struct ftl_basic_rq *rq = &band->metadata_rq; 446 447 if (ftl_band_alloc_p2l_map(band)) { 448 return -ENOMEM; 449 } 450 451 /* Read P2L map */ 452 ftl_basic_rq_init(dev, rq, band->p2l_map.band_map, ftl_p2l_map_num_blocks(dev)); 453 ftl_basic_rq_set_owner(rq, read_md_cb, band); 454 455 rq->io.band = band; 456 rq->io.addr = ftl_band_p2l_map_addr(band); 457 458 ftl_band_basic_rq_read(band, &band->metadata_rq); 459 460 return 0; 461 } 462 463 static void 464 read_md(void *band) 465 { 466 int rc; 467 468 rc = _read_md(band); 469 if (spdk_unlikely(rc)) { 470 spdk_thread_send_msg(spdk_get_thread(), read_md, band); 471 } 472 } 473 474 static void 475 read_tail_md_cb(struct ftl_basic_rq *brq) 476 { 477 struct ftl_band *band = brq->owner.priv; 478 enum ftl_md_status status = FTL_MD_IO_FAILURE; 479 ftl_band_md_cb cb; 480 void *priv; 481 482 if (spdk_unlikely(!brq->success)) { 483 /* Retries the read in case of error */ 484 ftl_band_basic_rq_read(band, &band->metadata_rq); 485 return; 486 } 487 488 cb = band->owner.md_fn; 489 band->owner.md_fn = NULL; 490 491 priv = band->owner.priv; 492 band->owner.priv = NULL; 493 494 status = FTL_MD_SUCCESS; 495 496 cb(band, priv, status); 497 } 498 499 void 500 ftl_band_read_tail_brq_md(struct ftl_band *band, ftl_band_md_cb cb, void *cntx) 501 { 502 struct spdk_ftl_dev *dev = band->dev; 503 struct ftl_basic_rq *rq = &band->metadata_rq; 504 505 ftl_basic_rq_init(dev, rq, band->p2l_map.band_map, ftl_tail_md_num_blocks(dev)); 506 ftl_basic_rq_set_owner(rq, read_tail_md_cb, band); 507 508 assert(!band->owner.md_fn); 509 assert(!band->owner.priv); 510 band->owner.md_fn = cb; 511 band->owner.priv = cntx; 512 513 rq->io.band = band; 514 rq->io.addr = band->tail_md_addr; 515 516 ftl_band_basic_rq_read(band, &band->metadata_rq); 517 } 518 519 void 520 ftl_band_get_next_gc(struct spdk_ftl_dev *dev, ftl_band_ops_cb cb, void *cntx) 521 { 522 struct ftl_band *band = ftl_band_search_next_to_reloc(dev); 523 524 /* if disk is very small, GC start very early that no band is ready for it */ 525 if (spdk_unlikely(!band)) { 526 cb(NULL, cntx, false); 527 return; 528 } 529 530 /* Only one owner is allowed */ 531 assert(!band->queue_depth); 532 assert(!band->owner.ops_fn); 533 assert(!band->owner.priv); 534 band->owner.ops_fn = cb; 535 band->owner.priv = cntx; 536 537 read_md(band); 538 } 539