1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (c) Intel Corporation. 3 * All rights reserved. 4 */ 5 6 #include "spdk/stdinc.h" 7 #include "spdk/queue.h" 8 #include "spdk/bdev_module.h" 9 10 #include "ftl_core.h" 11 #include "ftl_band.h" 12 #include "ftl_internal.h" 13 14 static void 15 write_rq_end(struct spdk_bdev_io *bdev_io, bool success, void *arg) 16 { 17 struct ftl_rq *rq = arg; 18 struct spdk_ftl_dev *dev = rq->dev; 19 20 ftl_stats_bdev_io_completed(dev, rq->owner.compaction ? FTL_STATS_TYPE_CMP : FTL_STATS_TYPE_GC, 21 bdev_io); 22 23 rq->success = success; 24 25 ftl_p2l_ckpt_issue(rq); 26 27 spdk_bdev_free_io(bdev_io); 28 } 29 30 static void 31 ftl_band_rq_bdev_write(void *_rq) 32 { 33 struct ftl_rq *rq = _rq; 34 struct ftl_band *band = rq->io.band; 35 struct spdk_ftl_dev *dev = band->dev; 36 int rc; 37 38 rc = spdk_bdev_writev_blocks(dev->base_bdev_desc, dev->base_ioch, 39 rq->io_vec, rq->io_vec_size, 40 rq->io.addr, rq->num_blocks, 41 write_rq_end, rq); 42 43 if (spdk_unlikely(rc)) { 44 if (rc == -ENOMEM) { 45 struct spdk_bdev *bdev = spdk_bdev_desc_get_bdev(dev->base_bdev_desc); 46 rq->io.bdev_io_wait.bdev = bdev; 47 rq->io.bdev_io_wait.cb_fn = ftl_band_rq_bdev_write; 48 rq->io.bdev_io_wait.cb_arg = rq; 49 spdk_bdev_queue_io_wait(bdev, dev->base_ioch, &rq->io.bdev_io_wait); 50 } else { 51 ftl_abort(); 52 } 53 } 54 } 55 56 void 57 ftl_band_rq_write(struct ftl_band *band, struct ftl_rq *rq) 58 { 59 struct spdk_ftl_dev *dev = band->dev; 60 61 rq->success = false; 62 rq->io.band = band; 63 rq->io.addr = band->md->iter.addr; 64 65 ftl_band_rq_bdev_write(rq); 66 67 band->queue_depth++; 68 dev->stats.io_activity_total += rq->num_blocks; 69 70 ftl_band_iter_advance(band, rq->num_blocks); 71 if (ftl_band_filled(band, band->md->iter.offset)) { 72 ftl_band_set_state(band, FTL_BAND_STATE_FULL); 73 band->owner.state_change_fn(band); 74 } 75 } 76 77 static void ftl_band_rq_bdev_read(void *_entry); 78 79 static void 80 read_rq_end(struct spdk_bdev_io *bdev_io, bool success, void *arg) 81 { 82 struct ftl_rq_entry *entry = arg; 83 struct ftl_band *band = entry->io.band; 84 struct ftl_rq *rq = ftl_rq_from_entry(entry); 85 86 ftl_stats_bdev_io_completed(band->dev, FTL_STATS_TYPE_GC, bdev_io); 87 88 rq->success = success; 89 if (spdk_unlikely(!success)) { 90 ftl_band_rq_bdev_read(entry); 91 spdk_bdev_free_io(bdev_io); 92 return; 93 } 94 95 assert(band->queue_depth > 0); 96 band->queue_depth--; 97 98 rq->owner.cb(rq); 99 spdk_bdev_free_io(bdev_io); 100 } 101 102 static void 103 ftl_band_rq_bdev_read(void *_entry) 104 { 105 struct ftl_rq_entry *entry = _entry; 106 struct ftl_rq *rq = ftl_rq_from_entry(entry); 107 struct spdk_ftl_dev *dev = rq->dev; 108 int rc; 109 110 rc = spdk_bdev_read_blocks(dev->base_bdev_desc, dev->base_ioch, entry->io_payload, 111 entry->bdev_io.offset_blocks, entry->bdev_io.num_blocks, 112 read_rq_end, entry); 113 if (spdk_unlikely(rc)) { 114 if (rc == -ENOMEM) { 115 struct spdk_bdev *bdev = spdk_bdev_desc_get_bdev(dev->base_bdev_desc); 116 entry->bdev_io.wait_entry.bdev = bdev; 117 entry->bdev_io.wait_entry.cb_fn = ftl_band_rq_bdev_read; 118 entry->bdev_io.wait_entry.cb_arg = entry; 119 spdk_bdev_queue_io_wait(bdev, dev->base_ioch, &entry->bdev_io.wait_entry); 120 } else { 121 ftl_abort(); 122 } 123 } 124 } 125 126 void 127 ftl_band_rq_read(struct ftl_band *band, struct ftl_rq *rq) 128 { 129 struct spdk_ftl_dev *dev = band->dev; 130 struct ftl_rq_entry *entry = &rq->entries[rq->iter.idx]; 131 132 assert(rq->iter.idx + rq->iter.count <= rq->num_blocks); 133 134 rq->success = false; 135 rq->io.band = band; 136 rq->io.addr = band->md->iter.addr; 137 entry->io.band = band; 138 entry->bdev_io.offset_blocks = rq->io.addr; 139 entry->bdev_io.num_blocks = rq->iter.count; 140 141 ftl_band_rq_bdev_read(entry); 142 143 dev->stats.io_activity_total += rq->num_blocks; 144 band->queue_depth++; 145 } 146 147 static void 148 write_brq_end(struct spdk_bdev_io *bdev_io, bool success, void *arg) 149 { 150 struct ftl_basic_rq *brq = arg; 151 struct ftl_band *band = brq->io.band; 152 153 ftl_stats_bdev_io_completed(band->dev, FTL_STATS_TYPE_MD_BASE, bdev_io); 154 155 brq->success = success; 156 157 assert(band->queue_depth > 0); 158 band->queue_depth--; 159 160 brq->owner.cb(brq); 161 spdk_bdev_free_io(bdev_io); 162 } 163 164 static void 165 ftl_band_brq_bdev_write(void *_brq) 166 { 167 struct ftl_basic_rq *brq = _brq; 168 struct spdk_ftl_dev *dev = brq->dev; 169 int rc; 170 171 rc = spdk_bdev_write_blocks(dev->base_bdev_desc, dev->base_ioch, 172 brq->io_payload, brq->io.addr, 173 brq->num_blocks, write_brq_end, brq); 174 175 if (spdk_unlikely(rc)) { 176 if (rc == -ENOMEM) { 177 struct spdk_bdev *bdev = spdk_bdev_desc_get_bdev(dev->base_bdev_desc); 178 brq->io.bdev_io_wait.bdev = bdev; 179 brq->io.bdev_io_wait.cb_fn = ftl_band_brq_bdev_write; 180 brq->io.bdev_io_wait.cb_arg = brq; 181 spdk_bdev_queue_io_wait(bdev, dev->base_ioch, &brq->io.bdev_io_wait); 182 } else { 183 ftl_abort(); 184 } 185 } 186 } 187 188 void 189 ftl_band_basic_rq_write(struct ftl_band *band, struct ftl_basic_rq *brq) 190 { 191 struct spdk_ftl_dev *dev = band->dev; 192 193 brq->io.addr = band->md->iter.addr; 194 brq->io.band = band; 195 brq->success = false; 196 197 ftl_band_brq_bdev_write(brq); 198 199 dev->stats.io_activity_total += brq->num_blocks; 200 band->queue_depth++; 201 ftl_band_iter_advance(band, brq->num_blocks); 202 if (ftl_band_filled(band, band->md->iter.offset)) { 203 ftl_band_set_state(band, FTL_BAND_STATE_FULL); 204 band->owner.state_change_fn(band); 205 } 206 } 207 208 static void 209 read_brq_end(struct spdk_bdev_io *bdev_io, bool success, void *arg) 210 { 211 struct ftl_basic_rq *brq = arg; 212 struct ftl_band *band = brq->io.band; 213 214 ftl_stats_bdev_io_completed(band->dev, FTL_STATS_TYPE_MD_BASE, bdev_io); 215 216 brq->success = success; 217 218 assert(band->queue_depth > 0); 219 band->queue_depth--; 220 221 brq->owner.cb(brq); 222 spdk_bdev_free_io(bdev_io); 223 } 224 225 static void 226 ftl_band_brq_bdev_read(void *_brq) 227 { 228 struct ftl_basic_rq *brq = _brq; 229 struct spdk_ftl_dev *dev = brq->dev; 230 int rc; 231 232 rc = spdk_bdev_read_blocks(dev->base_bdev_desc, dev->base_ioch, 233 brq->io_payload, brq->io.addr, 234 brq->num_blocks, read_brq_end, brq); 235 if (spdk_unlikely(rc)) { 236 if (rc == -ENOMEM) { 237 struct spdk_bdev *bdev = spdk_bdev_desc_get_bdev(dev->base_bdev_desc); 238 brq->io.bdev_io_wait.bdev = bdev; 239 brq->io.bdev_io_wait.cb_fn = ftl_band_brq_bdev_read; 240 brq->io.bdev_io_wait.cb_arg = brq; 241 spdk_bdev_queue_io_wait(bdev, dev->base_ioch, &brq->io.bdev_io_wait); 242 } else { 243 ftl_abort(); 244 } 245 } 246 } 247 248 void 249 ftl_band_basic_rq_read(struct ftl_band *band, struct ftl_basic_rq *brq) 250 { 251 struct spdk_ftl_dev *dev = brq->dev; 252 253 brq->io.band = band; 254 255 ftl_band_brq_bdev_read(brq); 256 257 brq->io.band->queue_depth++; 258 dev->stats.io_activity_total += brq->num_blocks; 259 } 260 261 static void 262 band_open_cb(int status, void *cb_arg) 263 { 264 struct ftl_band *band = cb_arg; 265 266 if (spdk_unlikely(status)) { 267 ftl_md_persist_entry_retry(&band->md_persist_entry_ctx); 268 return; 269 } 270 271 ftl_band_set_state(band, FTL_BAND_STATE_OPEN); 272 } 273 274 void 275 ftl_band_open(struct ftl_band *band, enum ftl_band_type type) 276 { 277 struct spdk_ftl_dev *dev = band->dev; 278 struct ftl_md *md = dev->layout.md[FTL_LAYOUT_REGION_TYPE_BAND_MD]; 279 struct ftl_layout_region *region = &dev->layout.region[FTL_LAYOUT_REGION_TYPE_BAND_MD]; 280 struct ftl_p2l_map *p2l_map = &band->p2l_map; 281 282 ftl_band_set_type(band, type); 283 ftl_band_set_state(band, FTL_BAND_STATE_OPENING); 284 285 memcpy(p2l_map->band_dma_md, band->md, region->entry_size * FTL_BLOCK_SIZE); 286 p2l_map->band_dma_md->state = FTL_BAND_STATE_OPEN; 287 p2l_map->band_dma_md->p2l_map_checksum = 0; 288 289 if (spdk_unlikely(0 != band->p2l_map.num_valid)) { 290 /* 291 * This is inconsistent state, a band with valid block, 292 * it could be moved on the free list 293 */ 294 assert(false && 0 == band->p2l_map.num_valid); 295 ftl_abort(); 296 } 297 298 ftl_md_persist_entry(md, band->id, p2l_map->band_dma_md, NULL, 299 band_open_cb, band, &band->md_persist_entry_ctx); 300 } 301 302 static void 303 band_close_cb(int status, void *cb_arg) 304 { 305 struct ftl_band *band = cb_arg; 306 307 if (spdk_unlikely(status)) { 308 ftl_md_persist_entry_retry(&band->md_persist_entry_ctx); 309 return; 310 } 311 312 band->md->p2l_map_checksum = band->p2l_map.band_dma_md->p2l_map_checksum; 313 ftl_band_set_state(band, FTL_BAND_STATE_CLOSED); 314 } 315 316 static void 317 band_map_write_cb(struct ftl_basic_rq *brq) 318 { 319 struct ftl_band *band = brq->io.band; 320 struct ftl_p2l_map *p2l_map = &band->p2l_map; 321 struct spdk_ftl_dev *dev = band->dev; 322 struct ftl_layout_region *region = &dev->layout.region[FTL_LAYOUT_REGION_TYPE_BAND_MD]; 323 struct ftl_md *md = dev->layout.md[FTL_LAYOUT_REGION_TYPE_BAND_MD]; 324 uint32_t band_map_crc; 325 326 if (spdk_likely(brq->success)) { 327 328 band_map_crc = spdk_crc32c_update(p2l_map->band_map, 329 ftl_tail_md_num_blocks(dev) * FTL_BLOCK_SIZE, 0); 330 memcpy(p2l_map->band_dma_md, band->md, region->entry_size * FTL_BLOCK_SIZE); 331 p2l_map->band_dma_md->state = FTL_BAND_STATE_CLOSED; 332 p2l_map->band_dma_md->p2l_map_checksum = band_map_crc; 333 334 ftl_md_persist_entry(md, band->id, p2l_map->band_dma_md, NULL, 335 band_close_cb, band, &band->md_persist_entry_ctx); 336 } else { 337 /* Try to retry in case of failure */ 338 ftl_band_brq_bdev_write(brq); 339 band->queue_depth++; 340 } 341 } 342 343 void 344 ftl_band_close(struct ftl_band *band) 345 { 346 struct spdk_ftl_dev *dev = band->dev; 347 void *metadata = band->p2l_map.band_map; 348 uint64_t num_blocks = ftl_tail_md_num_blocks(dev); 349 350 /* Write P2L map first, after completion, set the state to close on nvcache, then internally */ 351 band->md->close_seq_id = ftl_get_next_seq_id(dev); 352 ftl_band_set_state(band, FTL_BAND_STATE_CLOSING); 353 ftl_basic_rq_init(dev, &band->metadata_rq, metadata, num_blocks); 354 ftl_basic_rq_set_owner(&band->metadata_rq, band_map_write_cb, band); 355 356 ftl_band_basic_rq_write(band, &band->metadata_rq); 357 } 358 359 static void 360 band_free_cb(int status, void *ctx) 361 { 362 struct ftl_band *band = (struct ftl_band *)ctx; 363 364 if (spdk_unlikely(status)) { 365 ftl_md_persist_entry_retry(&band->md_persist_entry_ctx); 366 return; 367 } 368 369 ftl_band_release_p2l_map(band); 370 FTL_DEBUGLOG(band->dev, "Band is going to free state. Band id: %u\n", band->id); 371 ftl_band_set_state(band, FTL_BAND_STATE_FREE); 372 assert(0 == band->p2l_map.ref_cnt); 373 } 374 375 void 376 ftl_band_free(struct ftl_band *band) 377 { 378 struct spdk_ftl_dev *dev = band->dev; 379 struct ftl_p2l_map *p2l_map = &band->p2l_map; 380 struct ftl_md *md = dev->layout.md[FTL_LAYOUT_REGION_TYPE_BAND_MD]; 381 struct ftl_layout_region *region = &dev->layout.region[FTL_LAYOUT_REGION_TYPE_BAND_MD]; 382 383 memcpy(p2l_map->band_dma_md, band->md, region->entry_size * FTL_BLOCK_SIZE); 384 p2l_map->band_dma_md->state = FTL_BAND_STATE_FREE; 385 p2l_map->band_dma_md->close_seq_id = 0; 386 p2l_map->band_dma_md->p2l_map_checksum = 0; 387 388 ftl_md_persist_entry(md, band->id, p2l_map->band_dma_md, NULL, 389 band_free_cb, band, &band->md_persist_entry_ctx); 390 391 /* TODO: The whole band erase code should probably be done here instead */ 392 } 393 394 static void 395 read_md_cb(struct ftl_basic_rq *brq) 396 { 397 struct ftl_band *band = brq->owner.priv; 398 struct spdk_ftl_dev *dev = band->dev; 399 ftl_band_ops_cb cb; 400 uint32_t band_map_crc; 401 bool success = true; 402 void *priv; 403 404 cb = band->owner.ops_fn; 405 priv = band->owner.priv; 406 407 if (!brq->success) { 408 ftl_band_basic_rq_read(band, &band->metadata_rq); 409 return; 410 } 411 412 band_map_crc = spdk_crc32c_update(band->p2l_map.band_map, 413 ftl_tail_md_num_blocks(band->dev) * FTL_BLOCK_SIZE, 0); 414 if (band->md->p2l_map_checksum && band->md->p2l_map_checksum != band_map_crc) { 415 FTL_ERRLOG(dev, "GC error, inconsistent P2L map CRC\n"); 416 success = false; 417 418 ftl_stats_crc_error(band->dev, FTL_STATS_TYPE_GC); 419 } 420 band->owner.ops_fn = NULL; 421 band->owner.priv = NULL; 422 cb(band, priv, success); 423 } 424 425 static int 426 _read_md(struct ftl_band *band) 427 { 428 struct spdk_ftl_dev *dev = band->dev; 429 struct ftl_basic_rq *rq = &band->metadata_rq; 430 431 if (ftl_band_alloc_p2l_map(band)) { 432 return -ENOMEM; 433 } 434 435 /* Read P2L map */ 436 ftl_basic_rq_init(dev, rq, band->p2l_map.band_map, ftl_p2l_map_num_blocks(dev)); 437 ftl_basic_rq_set_owner(rq, read_md_cb, band); 438 439 rq->io.band = band; 440 rq->io.addr = ftl_band_p2l_map_addr(band); 441 442 ftl_band_basic_rq_read(band, &band->metadata_rq); 443 444 return 0; 445 } 446 447 static void 448 read_md(void *band) 449 { 450 int rc; 451 452 rc = _read_md(band); 453 if (spdk_unlikely(rc)) { 454 spdk_thread_send_msg(spdk_get_thread(), read_md, band); 455 } 456 } 457 458 static void 459 read_tail_md_cb(struct ftl_basic_rq *brq) 460 { 461 struct ftl_band *band = brq->owner.priv; 462 enum ftl_md_status status = FTL_MD_IO_FAILURE; 463 ftl_band_md_cb cb; 464 void *priv; 465 466 if (spdk_unlikely(!brq->success)) { 467 /* Retries the read in case of error */ 468 ftl_band_basic_rq_read(band, &band->metadata_rq); 469 return; 470 } 471 472 cb = band->owner.md_fn; 473 band->owner.md_fn = NULL; 474 475 priv = band->owner.priv; 476 band->owner.priv = NULL; 477 478 status = FTL_MD_SUCCESS; 479 480 cb(band, priv, status); 481 } 482 483 void 484 ftl_band_read_tail_brq_md(struct ftl_band *band, ftl_band_md_cb cb, void *cntx) 485 { 486 struct spdk_ftl_dev *dev = band->dev; 487 struct ftl_basic_rq *rq = &band->metadata_rq; 488 489 ftl_basic_rq_init(dev, rq, band->p2l_map.band_map, ftl_tail_md_num_blocks(dev)); 490 ftl_basic_rq_set_owner(rq, read_tail_md_cb, band); 491 492 assert(!band->owner.md_fn); 493 assert(!band->owner.priv); 494 band->owner.md_fn = cb; 495 band->owner.priv = cntx; 496 497 rq->io.band = band; 498 rq->io.addr = band->tail_md_addr; 499 500 ftl_band_basic_rq_read(band, &band->metadata_rq); 501 } 502 503 void 504 ftl_band_get_next_gc(struct spdk_ftl_dev *dev, ftl_band_ops_cb cb, void *cntx) 505 { 506 struct ftl_band *band = ftl_band_search_next_to_reloc(dev); 507 508 /* if disk is very small, GC start very early that no band is ready for it */ 509 if (spdk_unlikely(!band)) { 510 cb(NULL, cntx, false); 511 return; 512 } 513 514 /* Only one owner is allowed */ 515 assert(!band->queue_depth); 516 assert(!band->owner.ops_fn); 517 assert(!band->owner.priv); 518 band->owner.ops_fn = cb; 519 band->owner.priv = cntx; 520 521 read_md(band); 522 } 523