1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2017 Intel Corporation. 3 * All rights reserved. 4 * Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 5 */ 6 7 #include "spdk/stdinc.h" 8 9 #include "spdk/blob.h" 10 #include "spdk/crc32.h" 11 #include "spdk/env.h" 12 #include "spdk/queue.h" 13 #include "spdk/thread.h" 14 #include "spdk/bit_array.h" 15 #include "spdk/bit_pool.h" 16 #include "spdk/likely.h" 17 #include "spdk/util.h" 18 #include "spdk/string.h" 19 20 #include "spdk_internal/assert.h" 21 #include "spdk/log.h" 22 23 #include "blobstore.h" 24 25 #define BLOB_CRC32C_INITIAL 0xffffffffUL 26 27 static int bs_register_md_thread(struct spdk_blob_store *bs); 28 static int bs_unregister_md_thread(struct spdk_blob_store *bs); 29 static void blob_close_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno); 30 static void blob_insert_cluster_on_md_thread(struct spdk_blob *blob, uint32_t cluster_num, 31 uint64_t cluster, uint32_t extent, struct spdk_blob_md_page *page, 32 spdk_blob_op_complete cb_fn, void *cb_arg); 33 34 static int blob_set_xattr(struct spdk_blob *blob, const char *name, const void *value, 35 uint16_t value_len, bool internal); 36 static int blob_get_xattr_value(struct spdk_blob *blob, const char *name, 37 const void **value, size_t *value_len, bool internal); 38 static int blob_remove_xattr(struct spdk_blob *blob, const char *name, bool internal); 39 40 static void blob_write_extent_page(struct spdk_blob *blob, uint32_t extent, uint64_t cluster_num, 41 struct spdk_blob_md_page *page, spdk_blob_op_complete cb_fn, void *cb_arg); 42 43 static inline bool 44 blob_is_esnap_clone(const struct spdk_blob *blob) 45 { 46 assert(blob != NULL); 47 return !!(blob->invalid_flags & SPDK_BLOB_EXTERNAL_SNAPSHOT); 48 } 49 50 static int 51 blob_id_cmp(struct spdk_blob *blob1, struct spdk_blob *blob2) 52 { 53 return (blob1->id < blob2->id ? -1 : blob1->id > blob2->id); 54 } 55 56 RB_GENERATE_STATIC(spdk_blob_tree, spdk_blob, link, blob_id_cmp); 57 58 static void 59 blob_verify_md_op(struct spdk_blob *blob) 60 { 61 assert(blob != NULL); 62 assert(spdk_get_thread() == blob->bs->md_thread); 63 assert(blob->state != SPDK_BLOB_STATE_LOADING); 64 } 65 66 static struct spdk_blob_list * 67 bs_get_snapshot_entry(struct spdk_blob_store *bs, spdk_blob_id blobid) 68 { 69 struct spdk_blob_list *snapshot_entry = NULL; 70 71 TAILQ_FOREACH(snapshot_entry, &bs->snapshots, link) { 72 if (snapshot_entry->id == blobid) { 73 break; 74 } 75 } 76 77 return snapshot_entry; 78 } 79 80 static void 81 bs_claim_md_page(struct spdk_blob_store *bs, uint32_t page) 82 { 83 assert(spdk_spin_held(&bs->used_lock)); 84 assert(page < spdk_bit_array_capacity(bs->used_md_pages)); 85 assert(spdk_bit_array_get(bs->used_md_pages, page) == false); 86 87 spdk_bit_array_set(bs->used_md_pages, page); 88 } 89 90 static void 91 bs_release_md_page(struct spdk_blob_store *bs, uint32_t page) 92 { 93 assert(spdk_spin_held(&bs->used_lock)); 94 assert(page < spdk_bit_array_capacity(bs->used_md_pages)); 95 assert(spdk_bit_array_get(bs->used_md_pages, page) == true); 96 97 spdk_bit_array_clear(bs->used_md_pages, page); 98 } 99 100 static uint32_t 101 bs_claim_cluster(struct spdk_blob_store *bs) 102 { 103 uint32_t cluster_num; 104 105 assert(spdk_spin_held(&bs->used_lock)); 106 107 cluster_num = spdk_bit_pool_allocate_bit(bs->used_clusters); 108 if (cluster_num == UINT32_MAX) { 109 return UINT32_MAX; 110 } 111 112 SPDK_DEBUGLOG(blob, "Claiming cluster %u\n", cluster_num); 113 bs->num_free_clusters--; 114 115 return cluster_num; 116 } 117 118 static void 119 bs_release_cluster(struct spdk_blob_store *bs, uint32_t cluster_num) 120 { 121 assert(spdk_spin_held(&bs->used_lock)); 122 assert(cluster_num < spdk_bit_pool_capacity(bs->used_clusters)); 123 assert(spdk_bit_pool_is_allocated(bs->used_clusters, cluster_num) == true); 124 assert(bs->num_free_clusters < bs->total_clusters); 125 126 SPDK_DEBUGLOG(blob, "Releasing cluster %u\n", cluster_num); 127 128 spdk_bit_pool_free_bit(bs->used_clusters, cluster_num); 129 bs->num_free_clusters++; 130 } 131 132 static int 133 blob_insert_cluster(struct spdk_blob *blob, uint32_t cluster_num, uint64_t cluster) 134 { 135 uint64_t *cluster_lba = &blob->active.clusters[cluster_num]; 136 137 blob_verify_md_op(blob); 138 139 if (*cluster_lba != 0) { 140 return -EEXIST; 141 } 142 143 *cluster_lba = bs_cluster_to_lba(blob->bs, cluster); 144 return 0; 145 } 146 147 static int 148 bs_allocate_cluster(struct spdk_blob *blob, uint32_t cluster_num, 149 uint64_t *cluster, uint32_t *lowest_free_md_page, bool update_map) 150 { 151 uint32_t *extent_page = 0; 152 153 assert(spdk_spin_held(&blob->bs->used_lock)); 154 155 *cluster = bs_claim_cluster(blob->bs); 156 if (*cluster == UINT32_MAX) { 157 /* No more free clusters. Cannot satisfy the request */ 158 return -ENOSPC; 159 } 160 161 if (blob->use_extent_table) { 162 extent_page = bs_cluster_to_extent_page(blob, cluster_num); 163 if (*extent_page == 0) { 164 /* Extent page shall never occupy md_page so start the search from 1 */ 165 if (*lowest_free_md_page == 0) { 166 *lowest_free_md_page = 1; 167 } 168 /* No extent_page is allocated for the cluster */ 169 *lowest_free_md_page = spdk_bit_array_find_first_clear(blob->bs->used_md_pages, 170 *lowest_free_md_page); 171 if (*lowest_free_md_page == UINT32_MAX) { 172 /* No more free md pages. Cannot satisfy the request */ 173 bs_release_cluster(blob->bs, *cluster); 174 return -ENOSPC; 175 } 176 bs_claim_md_page(blob->bs, *lowest_free_md_page); 177 } 178 } 179 180 SPDK_DEBUGLOG(blob, "Claiming cluster %" PRIu64 " for blob 0x%" PRIx64 "\n", *cluster, 181 blob->id); 182 183 if (update_map) { 184 blob_insert_cluster(blob, cluster_num, *cluster); 185 if (blob->use_extent_table && *extent_page == 0) { 186 *extent_page = *lowest_free_md_page; 187 } 188 } 189 190 return 0; 191 } 192 193 static void 194 blob_xattrs_init(struct spdk_blob_xattr_opts *xattrs) 195 { 196 xattrs->count = 0; 197 xattrs->names = NULL; 198 xattrs->ctx = NULL; 199 xattrs->get_value = NULL; 200 } 201 202 void 203 spdk_blob_opts_init(struct spdk_blob_opts *opts, size_t opts_size) 204 { 205 if (!opts) { 206 SPDK_ERRLOG("opts should not be NULL\n"); 207 return; 208 } 209 210 if (!opts_size) { 211 SPDK_ERRLOG("opts_size should not be zero value\n"); 212 return; 213 } 214 215 memset(opts, 0, opts_size); 216 opts->opts_size = opts_size; 217 218 #define FIELD_OK(field) \ 219 offsetof(struct spdk_blob_opts, field) + sizeof(opts->field) <= opts_size 220 221 #define SET_FIELD(field, value) \ 222 if (FIELD_OK(field)) { \ 223 opts->field = value; \ 224 } \ 225 226 SET_FIELD(num_clusters, 0); 227 SET_FIELD(thin_provision, false); 228 SET_FIELD(clear_method, BLOB_CLEAR_WITH_DEFAULT); 229 230 if (FIELD_OK(xattrs)) { 231 blob_xattrs_init(&opts->xattrs); 232 } 233 234 SET_FIELD(use_extent_table, true); 235 236 #undef FIELD_OK 237 #undef SET_FIELD 238 } 239 240 void 241 spdk_blob_open_opts_init(struct spdk_blob_open_opts *opts, size_t opts_size) 242 { 243 if (!opts) { 244 SPDK_ERRLOG("opts should not be NULL\n"); 245 return; 246 } 247 248 if (!opts_size) { 249 SPDK_ERRLOG("opts_size should not be zero value\n"); 250 return; 251 } 252 253 memset(opts, 0, opts_size); 254 opts->opts_size = opts_size; 255 256 #define FIELD_OK(field) \ 257 offsetof(struct spdk_blob_open_opts, field) + sizeof(opts->field) <= opts_size 258 259 #define SET_FIELD(field, value) \ 260 if (FIELD_OK(field)) { \ 261 opts->field = value; \ 262 } \ 263 264 SET_FIELD(clear_method, BLOB_CLEAR_WITH_DEFAULT); 265 266 #undef FIELD_OK 267 #undef SET_FILED 268 } 269 270 static struct spdk_blob * 271 blob_alloc(struct spdk_blob_store *bs, spdk_blob_id id) 272 { 273 struct spdk_blob *blob; 274 275 blob = calloc(1, sizeof(*blob)); 276 if (!blob) { 277 return NULL; 278 } 279 280 blob->id = id; 281 blob->bs = bs; 282 283 blob->parent_id = SPDK_BLOBID_INVALID; 284 285 blob->state = SPDK_BLOB_STATE_DIRTY; 286 blob->extent_rle_found = false; 287 blob->extent_table_found = false; 288 blob->active.num_pages = 1; 289 blob->active.pages = calloc(1, sizeof(*blob->active.pages)); 290 if (!blob->active.pages) { 291 free(blob); 292 return NULL; 293 } 294 295 blob->active.pages[0] = bs_blobid_to_page(id); 296 297 TAILQ_INIT(&blob->xattrs); 298 TAILQ_INIT(&blob->xattrs_internal); 299 TAILQ_INIT(&blob->pending_persists); 300 TAILQ_INIT(&blob->persists_to_complete); 301 302 return blob; 303 } 304 305 static void 306 xattrs_free(struct spdk_xattr_tailq *xattrs) 307 { 308 struct spdk_xattr *xattr, *xattr_tmp; 309 310 TAILQ_FOREACH_SAFE(xattr, xattrs, link, xattr_tmp) { 311 TAILQ_REMOVE(xattrs, xattr, link); 312 free(xattr->name); 313 free(xattr->value); 314 free(xattr); 315 } 316 } 317 318 static void 319 blob_free(struct spdk_blob *blob) 320 { 321 assert(blob != NULL); 322 assert(TAILQ_EMPTY(&blob->pending_persists)); 323 assert(TAILQ_EMPTY(&blob->persists_to_complete)); 324 325 free(blob->active.extent_pages); 326 free(blob->clean.extent_pages); 327 free(blob->active.clusters); 328 free(blob->clean.clusters); 329 free(blob->active.pages); 330 free(blob->clean.pages); 331 332 xattrs_free(&blob->xattrs); 333 xattrs_free(&blob->xattrs_internal); 334 335 if (blob->back_bs_dev) { 336 blob->back_bs_dev->destroy(blob->back_bs_dev); 337 } 338 339 free(blob); 340 } 341 342 struct freeze_io_ctx { 343 struct spdk_bs_cpl cpl; 344 struct spdk_blob *blob; 345 }; 346 347 static void 348 blob_io_sync(struct spdk_io_channel_iter *i) 349 { 350 spdk_for_each_channel_continue(i, 0); 351 } 352 353 static void 354 blob_execute_queued_io(struct spdk_io_channel_iter *i) 355 { 356 struct spdk_io_channel *_ch = spdk_io_channel_iter_get_channel(i); 357 struct spdk_bs_channel *ch = spdk_io_channel_get_ctx(_ch); 358 struct freeze_io_ctx *ctx = spdk_io_channel_iter_get_ctx(i); 359 struct spdk_bs_request_set *set; 360 struct spdk_bs_user_op_args *args; 361 spdk_bs_user_op_t *op, *tmp; 362 363 TAILQ_FOREACH_SAFE(op, &ch->queued_io, link, tmp) { 364 set = (struct spdk_bs_request_set *)op; 365 args = &set->u.user_op; 366 367 if (args->blob == ctx->blob) { 368 TAILQ_REMOVE(&ch->queued_io, op, link); 369 bs_user_op_execute(op); 370 } 371 } 372 373 spdk_for_each_channel_continue(i, 0); 374 } 375 376 static void 377 blob_io_cpl(struct spdk_io_channel_iter *i, int status) 378 { 379 struct freeze_io_ctx *ctx = spdk_io_channel_iter_get_ctx(i); 380 381 ctx->cpl.u.blob_basic.cb_fn(ctx->cpl.u.blob_basic.cb_arg, 0); 382 383 free(ctx); 384 } 385 386 static void 387 blob_freeze_io(struct spdk_blob *blob, spdk_blob_op_complete cb_fn, void *cb_arg) 388 { 389 struct freeze_io_ctx *ctx; 390 391 ctx = calloc(1, sizeof(*ctx)); 392 if (!ctx) { 393 cb_fn(cb_arg, -ENOMEM); 394 return; 395 } 396 397 ctx->cpl.type = SPDK_BS_CPL_TYPE_BS_BASIC; 398 ctx->cpl.u.blob_basic.cb_fn = cb_fn; 399 ctx->cpl.u.blob_basic.cb_arg = cb_arg; 400 ctx->blob = blob; 401 402 /* Freeze I/O on blob */ 403 blob->frozen_refcnt++; 404 405 if (blob->frozen_refcnt == 1) { 406 spdk_for_each_channel(blob->bs, blob_io_sync, ctx, blob_io_cpl); 407 } else { 408 cb_fn(cb_arg, 0); 409 free(ctx); 410 } 411 } 412 413 static void 414 blob_unfreeze_io(struct spdk_blob *blob, spdk_blob_op_complete cb_fn, void *cb_arg) 415 { 416 struct freeze_io_ctx *ctx; 417 418 ctx = calloc(1, sizeof(*ctx)); 419 if (!ctx) { 420 cb_fn(cb_arg, -ENOMEM); 421 return; 422 } 423 424 ctx->cpl.type = SPDK_BS_CPL_TYPE_BS_BASIC; 425 ctx->cpl.u.blob_basic.cb_fn = cb_fn; 426 ctx->cpl.u.blob_basic.cb_arg = cb_arg; 427 ctx->blob = blob; 428 429 assert(blob->frozen_refcnt > 0); 430 431 blob->frozen_refcnt--; 432 433 if (blob->frozen_refcnt == 0) { 434 spdk_for_each_channel(blob->bs, blob_execute_queued_io, ctx, blob_io_cpl); 435 } else { 436 cb_fn(cb_arg, 0); 437 free(ctx); 438 } 439 } 440 441 static int 442 blob_mark_clean(struct spdk_blob *blob) 443 { 444 uint32_t *extent_pages = NULL; 445 uint64_t *clusters = NULL; 446 uint32_t *pages = NULL; 447 448 assert(blob != NULL); 449 450 if (blob->active.num_extent_pages) { 451 assert(blob->active.extent_pages); 452 extent_pages = calloc(blob->active.num_extent_pages, sizeof(*blob->active.extent_pages)); 453 if (!extent_pages) { 454 return -ENOMEM; 455 } 456 memcpy(extent_pages, blob->active.extent_pages, 457 blob->active.num_extent_pages * sizeof(*extent_pages)); 458 } 459 460 if (blob->active.num_clusters) { 461 assert(blob->active.clusters); 462 clusters = calloc(blob->active.num_clusters, sizeof(*blob->active.clusters)); 463 if (!clusters) { 464 free(extent_pages); 465 return -ENOMEM; 466 } 467 memcpy(clusters, blob->active.clusters, blob->active.num_clusters * sizeof(*blob->active.clusters)); 468 } 469 470 if (blob->active.num_pages) { 471 assert(blob->active.pages); 472 pages = calloc(blob->active.num_pages, sizeof(*blob->active.pages)); 473 if (!pages) { 474 free(extent_pages); 475 free(clusters); 476 return -ENOMEM; 477 } 478 memcpy(pages, blob->active.pages, blob->active.num_pages * sizeof(*blob->active.pages)); 479 } 480 481 free(blob->clean.extent_pages); 482 free(blob->clean.clusters); 483 free(blob->clean.pages); 484 485 blob->clean.num_extent_pages = blob->active.num_extent_pages; 486 blob->clean.extent_pages = blob->active.extent_pages; 487 blob->clean.num_clusters = blob->active.num_clusters; 488 blob->clean.clusters = blob->active.clusters; 489 blob->clean.num_pages = blob->active.num_pages; 490 blob->clean.pages = blob->active.pages; 491 492 blob->active.extent_pages = extent_pages; 493 blob->active.clusters = clusters; 494 blob->active.pages = pages; 495 496 /* If the metadata was dirtied again while the metadata was being written to disk, 497 * we do not want to revert the DIRTY state back to CLEAN here. 498 */ 499 if (blob->state == SPDK_BLOB_STATE_LOADING) { 500 blob->state = SPDK_BLOB_STATE_CLEAN; 501 } 502 503 return 0; 504 } 505 506 static int 507 blob_deserialize_xattr(struct spdk_blob *blob, 508 struct spdk_blob_md_descriptor_xattr *desc_xattr, bool internal) 509 { 510 struct spdk_xattr *xattr; 511 512 if (desc_xattr->length != sizeof(desc_xattr->name_length) + 513 sizeof(desc_xattr->value_length) + 514 desc_xattr->name_length + desc_xattr->value_length) { 515 return -EINVAL; 516 } 517 518 xattr = calloc(1, sizeof(*xattr)); 519 if (xattr == NULL) { 520 return -ENOMEM; 521 } 522 523 xattr->name = malloc(desc_xattr->name_length + 1); 524 if (xattr->name == NULL) { 525 free(xattr); 526 return -ENOMEM; 527 } 528 529 xattr->value = malloc(desc_xattr->value_length); 530 if (xattr->value == NULL) { 531 free(xattr->name); 532 free(xattr); 533 return -ENOMEM; 534 } 535 536 memcpy(xattr->name, desc_xattr->name, desc_xattr->name_length); 537 xattr->name[desc_xattr->name_length] = '\0'; 538 xattr->value_len = desc_xattr->value_length; 539 memcpy(xattr->value, 540 (void *)((uintptr_t)desc_xattr->name + desc_xattr->name_length), 541 desc_xattr->value_length); 542 543 TAILQ_INSERT_TAIL(internal ? &blob->xattrs_internal : &blob->xattrs, xattr, link); 544 545 return 0; 546 } 547 548 549 static int 550 blob_parse_page(const struct spdk_blob_md_page *page, struct spdk_blob *blob) 551 { 552 struct spdk_blob_md_descriptor *desc; 553 size_t cur_desc = 0; 554 void *tmp; 555 556 desc = (struct spdk_blob_md_descriptor *)page->descriptors; 557 while (cur_desc < sizeof(page->descriptors)) { 558 if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_PADDING) { 559 if (desc->length == 0) { 560 /* If padding and length are 0, this terminates the page */ 561 break; 562 } 563 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_FLAGS) { 564 struct spdk_blob_md_descriptor_flags *desc_flags; 565 566 desc_flags = (struct spdk_blob_md_descriptor_flags *)desc; 567 568 if (desc_flags->length != sizeof(*desc_flags) - sizeof(*desc)) { 569 return -EINVAL; 570 } 571 572 if ((desc_flags->invalid_flags | SPDK_BLOB_INVALID_FLAGS_MASK) != 573 SPDK_BLOB_INVALID_FLAGS_MASK) { 574 return -EINVAL; 575 } 576 577 if ((desc_flags->data_ro_flags | SPDK_BLOB_DATA_RO_FLAGS_MASK) != 578 SPDK_BLOB_DATA_RO_FLAGS_MASK) { 579 blob->data_ro = true; 580 blob->md_ro = true; 581 } 582 583 if ((desc_flags->md_ro_flags | SPDK_BLOB_MD_RO_FLAGS_MASK) != 584 SPDK_BLOB_MD_RO_FLAGS_MASK) { 585 blob->md_ro = true; 586 } 587 588 if ((desc_flags->data_ro_flags & SPDK_BLOB_READ_ONLY)) { 589 blob->data_ro = true; 590 blob->md_ro = true; 591 } 592 593 blob->invalid_flags = desc_flags->invalid_flags; 594 blob->data_ro_flags = desc_flags->data_ro_flags; 595 blob->md_ro_flags = desc_flags->md_ro_flags; 596 597 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_EXTENT_RLE) { 598 struct spdk_blob_md_descriptor_extent_rle *desc_extent_rle; 599 unsigned int i, j; 600 unsigned int cluster_count = blob->active.num_clusters; 601 602 if (blob->extent_table_found) { 603 /* Extent Table already present in the md, 604 * both descriptors should never be at the same time. */ 605 return -EINVAL; 606 } 607 blob->extent_rle_found = true; 608 609 desc_extent_rle = (struct spdk_blob_md_descriptor_extent_rle *)desc; 610 611 if (desc_extent_rle->length == 0 || 612 (desc_extent_rle->length % sizeof(desc_extent_rle->extents[0]) != 0)) { 613 return -EINVAL; 614 } 615 616 for (i = 0; i < desc_extent_rle->length / sizeof(desc_extent_rle->extents[0]); i++) { 617 for (j = 0; j < desc_extent_rle->extents[i].length; j++) { 618 if (desc_extent_rle->extents[i].cluster_idx != 0) { 619 if (!spdk_bit_pool_is_allocated(blob->bs->used_clusters, 620 desc_extent_rle->extents[i].cluster_idx + j)) { 621 return -EINVAL; 622 } 623 } 624 cluster_count++; 625 } 626 } 627 628 if (cluster_count == 0) { 629 return -EINVAL; 630 } 631 tmp = realloc(blob->active.clusters, cluster_count * sizeof(*blob->active.clusters)); 632 if (tmp == NULL) { 633 return -ENOMEM; 634 } 635 blob->active.clusters = tmp; 636 blob->active.cluster_array_size = cluster_count; 637 638 for (i = 0; i < desc_extent_rle->length / sizeof(desc_extent_rle->extents[0]); i++) { 639 for (j = 0; j < desc_extent_rle->extents[i].length; j++) { 640 if (desc_extent_rle->extents[i].cluster_idx != 0) { 641 blob->active.clusters[blob->active.num_clusters++] = bs_cluster_to_lba(blob->bs, 642 desc_extent_rle->extents[i].cluster_idx + j); 643 } else if (spdk_blob_is_thin_provisioned(blob)) { 644 blob->active.clusters[blob->active.num_clusters++] = 0; 645 } else { 646 return -EINVAL; 647 } 648 } 649 } 650 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_EXTENT_TABLE) { 651 struct spdk_blob_md_descriptor_extent_table *desc_extent_table; 652 uint32_t num_extent_pages = blob->active.num_extent_pages; 653 uint32_t i, j; 654 size_t extent_pages_length; 655 656 desc_extent_table = (struct spdk_blob_md_descriptor_extent_table *)desc; 657 extent_pages_length = desc_extent_table->length - sizeof(desc_extent_table->num_clusters); 658 659 if (blob->extent_rle_found) { 660 /* This means that Extent RLE is present in MD, 661 * both should never be at the same time. */ 662 return -EINVAL; 663 } else if (blob->extent_table_found && 664 desc_extent_table->num_clusters != blob->remaining_clusters_in_et) { 665 /* Number of clusters in this ET does not match number 666 * from previously read EXTENT_TABLE. */ 667 return -EINVAL; 668 } 669 670 if (desc_extent_table->length == 0 || 671 (extent_pages_length % sizeof(desc_extent_table->extent_page[0]) != 0)) { 672 return -EINVAL; 673 } 674 675 blob->extent_table_found = true; 676 677 for (i = 0; i < extent_pages_length / sizeof(desc_extent_table->extent_page[0]); i++) { 678 num_extent_pages += desc_extent_table->extent_page[i].num_pages; 679 } 680 681 if (num_extent_pages > 0) { 682 tmp = realloc(blob->active.extent_pages, num_extent_pages * sizeof(uint32_t)); 683 if (tmp == NULL) { 684 return -ENOMEM; 685 } 686 blob->active.extent_pages = tmp; 687 } 688 blob->active.extent_pages_array_size = num_extent_pages; 689 690 blob->remaining_clusters_in_et = desc_extent_table->num_clusters; 691 692 /* Extent table entries contain md page numbers for extent pages. 693 * Zeroes represent unallocated extent pages, those are run-length-encoded. 694 */ 695 for (i = 0; i < extent_pages_length / sizeof(desc_extent_table->extent_page[0]); i++) { 696 if (desc_extent_table->extent_page[i].page_idx != 0) { 697 assert(desc_extent_table->extent_page[i].num_pages == 1); 698 blob->active.extent_pages[blob->active.num_extent_pages++] = 699 desc_extent_table->extent_page[i].page_idx; 700 } else if (spdk_blob_is_thin_provisioned(blob)) { 701 for (j = 0; j < desc_extent_table->extent_page[i].num_pages; j++) { 702 blob->active.extent_pages[blob->active.num_extent_pages++] = 0; 703 } 704 } else { 705 return -EINVAL; 706 } 707 } 708 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_EXTENT_PAGE) { 709 struct spdk_blob_md_descriptor_extent_page *desc_extent; 710 unsigned int i; 711 unsigned int cluster_count = 0; 712 size_t cluster_idx_length; 713 714 if (blob->extent_rle_found) { 715 /* This means that Extent RLE is present in MD, 716 * both should never be at the same time. */ 717 return -EINVAL; 718 } 719 720 desc_extent = (struct spdk_blob_md_descriptor_extent_page *)desc; 721 cluster_idx_length = desc_extent->length - sizeof(desc_extent->start_cluster_idx); 722 723 if (desc_extent->length <= sizeof(desc_extent->start_cluster_idx) || 724 (cluster_idx_length % sizeof(desc_extent->cluster_idx[0]) != 0)) { 725 return -EINVAL; 726 } 727 728 for (i = 0; i < cluster_idx_length / sizeof(desc_extent->cluster_idx[0]); i++) { 729 if (desc_extent->cluster_idx[i] != 0) { 730 if (!spdk_bit_pool_is_allocated(blob->bs->used_clusters, desc_extent->cluster_idx[i])) { 731 return -EINVAL; 732 } 733 } 734 cluster_count++; 735 } 736 737 if (cluster_count == 0) { 738 return -EINVAL; 739 } 740 741 /* When reading extent pages sequentially starting cluster idx should match 742 * current size of a blob. 743 * If changed to batch reading, this check shall be removed. */ 744 if (desc_extent->start_cluster_idx != blob->active.num_clusters) { 745 return -EINVAL; 746 } 747 748 tmp = realloc(blob->active.clusters, 749 (cluster_count + blob->active.num_clusters) * sizeof(*blob->active.clusters)); 750 if (tmp == NULL) { 751 return -ENOMEM; 752 } 753 blob->active.clusters = tmp; 754 blob->active.cluster_array_size = (cluster_count + blob->active.num_clusters); 755 756 for (i = 0; i < cluster_idx_length / sizeof(desc_extent->cluster_idx[0]); i++) { 757 if (desc_extent->cluster_idx[i] != 0) { 758 blob->active.clusters[blob->active.num_clusters++] = bs_cluster_to_lba(blob->bs, 759 desc_extent->cluster_idx[i]); 760 } else if (spdk_blob_is_thin_provisioned(blob)) { 761 blob->active.clusters[blob->active.num_clusters++] = 0; 762 } else { 763 return -EINVAL; 764 } 765 } 766 assert(desc_extent->start_cluster_idx + cluster_count == blob->active.num_clusters); 767 assert(blob->remaining_clusters_in_et >= cluster_count); 768 blob->remaining_clusters_in_et -= cluster_count; 769 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_XATTR) { 770 int rc; 771 772 rc = blob_deserialize_xattr(blob, 773 (struct spdk_blob_md_descriptor_xattr *) desc, false); 774 if (rc != 0) { 775 return rc; 776 } 777 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_XATTR_INTERNAL) { 778 int rc; 779 780 rc = blob_deserialize_xattr(blob, 781 (struct spdk_blob_md_descriptor_xattr *) desc, true); 782 if (rc != 0) { 783 return rc; 784 } 785 } else { 786 /* Unrecognized descriptor type. Do not fail - just continue to the 787 * next descriptor. If this descriptor is associated with some feature 788 * defined in a newer version of blobstore, that version of blobstore 789 * should create and set an associated feature flag to specify if this 790 * blob can be loaded or not. 791 */ 792 } 793 794 /* Advance to the next descriptor */ 795 cur_desc += sizeof(*desc) + desc->length; 796 if (cur_desc + sizeof(*desc) > sizeof(page->descriptors)) { 797 break; 798 } 799 desc = (struct spdk_blob_md_descriptor *)((uintptr_t)page->descriptors + cur_desc); 800 } 801 802 return 0; 803 } 804 805 static bool bs_load_cur_extent_page_valid(struct spdk_blob_md_page *page); 806 807 static int 808 blob_parse_extent_page(struct spdk_blob_md_page *extent_page, struct spdk_blob *blob) 809 { 810 assert(blob != NULL); 811 assert(blob->state == SPDK_BLOB_STATE_LOADING); 812 813 if (bs_load_cur_extent_page_valid(extent_page) == false) { 814 return -ENOENT; 815 } 816 817 return blob_parse_page(extent_page, blob); 818 } 819 820 static int 821 blob_parse(const struct spdk_blob_md_page *pages, uint32_t page_count, 822 struct spdk_blob *blob) 823 { 824 const struct spdk_blob_md_page *page; 825 uint32_t i; 826 int rc; 827 void *tmp; 828 829 assert(page_count > 0); 830 assert(pages[0].sequence_num == 0); 831 assert(blob != NULL); 832 assert(blob->state == SPDK_BLOB_STATE_LOADING); 833 assert(blob->active.clusters == NULL); 834 835 /* The blobid provided doesn't match what's in the MD, this can 836 * happen for example if a bogus blobid is passed in through open. 837 */ 838 if (blob->id != pages[0].id) { 839 SPDK_ERRLOG("Blobid (%" PRIu64 ") doesn't match what's in metadata (%" PRIu64 ")\n", 840 blob->id, pages[0].id); 841 return -ENOENT; 842 } 843 844 tmp = realloc(blob->active.pages, page_count * sizeof(*blob->active.pages)); 845 if (!tmp) { 846 return -ENOMEM; 847 } 848 blob->active.pages = tmp; 849 850 blob->active.pages[0] = pages[0].id; 851 852 for (i = 1; i < page_count; i++) { 853 assert(spdk_bit_array_get(blob->bs->used_md_pages, pages[i - 1].next)); 854 blob->active.pages[i] = pages[i - 1].next; 855 } 856 blob->active.num_pages = page_count; 857 858 for (i = 0; i < page_count; i++) { 859 page = &pages[i]; 860 861 assert(page->id == blob->id); 862 assert(page->sequence_num == i); 863 864 rc = blob_parse_page(page, blob); 865 if (rc != 0) { 866 return rc; 867 } 868 } 869 870 return 0; 871 } 872 873 static int 874 blob_serialize_add_page(const struct spdk_blob *blob, 875 struct spdk_blob_md_page **pages, 876 uint32_t *page_count, 877 struct spdk_blob_md_page **last_page) 878 { 879 struct spdk_blob_md_page *page, *tmp_pages; 880 881 assert(pages != NULL); 882 assert(page_count != NULL); 883 884 *last_page = NULL; 885 if (*page_count == 0) { 886 assert(*pages == NULL); 887 *pages = spdk_malloc(SPDK_BS_PAGE_SIZE, 0, 888 NULL, SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA); 889 if (*pages == NULL) { 890 return -ENOMEM; 891 } 892 *page_count = 1; 893 } else { 894 assert(*pages != NULL); 895 tmp_pages = spdk_realloc(*pages, SPDK_BS_PAGE_SIZE * (*page_count + 1), 0); 896 if (tmp_pages == NULL) { 897 return -ENOMEM; 898 } 899 (*page_count)++; 900 *pages = tmp_pages; 901 } 902 903 page = &(*pages)[*page_count - 1]; 904 memset(page, 0, sizeof(*page)); 905 page->id = blob->id; 906 page->sequence_num = *page_count - 1; 907 page->next = SPDK_INVALID_MD_PAGE; 908 *last_page = page; 909 910 return 0; 911 } 912 913 /* Transform the in-memory representation 'xattr' into an on-disk xattr descriptor. 914 * Update required_sz on both success and failure. 915 * 916 */ 917 static int 918 blob_serialize_xattr(const struct spdk_xattr *xattr, 919 uint8_t *buf, size_t buf_sz, 920 size_t *required_sz, bool internal) 921 { 922 struct spdk_blob_md_descriptor_xattr *desc; 923 924 *required_sz = sizeof(struct spdk_blob_md_descriptor_xattr) + 925 strlen(xattr->name) + 926 xattr->value_len; 927 928 if (buf_sz < *required_sz) { 929 return -1; 930 } 931 932 desc = (struct spdk_blob_md_descriptor_xattr *)buf; 933 934 desc->type = internal ? SPDK_MD_DESCRIPTOR_TYPE_XATTR_INTERNAL : SPDK_MD_DESCRIPTOR_TYPE_XATTR; 935 desc->length = sizeof(desc->name_length) + 936 sizeof(desc->value_length) + 937 strlen(xattr->name) + 938 xattr->value_len; 939 desc->name_length = strlen(xattr->name); 940 desc->value_length = xattr->value_len; 941 942 memcpy(desc->name, xattr->name, desc->name_length); 943 memcpy((void *)((uintptr_t)desc->name + desc->name_length), 944 xattr->value, 945 desc->value_length); 946 947 return 0; 948 } 949 950 static void 951 blob_serialize_extent_table_entry(const struct spdk_blob *blob, 952 uint64_t start_ep, uint64_t *next_ep, 953 uint8_t **buf, size_t *remaining_sz) 954 { 955 struct spdk_blob_md_descriptor_extent_table *desc; 956 size_t cur_sz; 957 uint64_t i, et_idx; 958 uint32_t extent_page, ep_len; 959 960 /* The buffer must have room for at least num_clusters entry */ 961 cur_sz = sizeof(struct spdk_blob_md_descriptor) + sizeof(desc->num_clusters); 962 if (*remaining_sz < cur_sz) { 963 *next_ep = start_ep; 964 return; 965 } 966 967 desc = (struct spdk_blob_md_descriptor_extent_table *)*buf; 968 desc->type = SPDK_MD_DESCRIPTOR_TYPE_EXTENT_TABLE; 969 970 desc->num_clusters = blob->active.num_clusters; 971 972 ep_len = 1; 973 et_idx = 0; 974 for (i = start_ep; i < blob->active.num_extent_pages; i++) { 975 if (*remaining_sz < cur_sz + sizeof(desc->extent_page[0])) { 976 /* If we ran out of buffer space, return */ 977 break; 978 } 979 980 extent_page = blob->active.extent_pages[i]; 981 /* Verify that next extent_page is unallocated */ 982 if (extent_page == 0 && 983 (i + 1 < blob->active.num_extent_pages && blob->active.extent_pages[i + 1] == 0)) { 984 ep_len++; 985 continue; 986 } 987 desc->extent_page[et_idx].page_idx = extent_page; 988 desc->extent_page[et_idx].num_pages = ep_len; 989 et_idx++; 990 991 ep_len = 1; 992 cur_sz += sizeof(desc->extent_page[et_idx]); 993 } 994 *next_ep = i; 995 996 desc->length = sizeof(desc->num_clusters) + sizeof(desc->extent_page[0]) * et_idx; 997 *remaining_sz -= sizeof(struct spdk_blob_md_descriptor) + desc->length; 998 *buf += sizeof(struct spdk_blob_md_descriptor) + desc->length; 999 } 1000 1001 static int 1002 blob_serialize_extent_table(const struct spdk_blob *blob, 1003 struct spdk_blob_md_page **pages, 1004 struct spdk_blob_md_page *cur_page, 1005 uint32_t *page_count, uint8_t **buf, 1006 size_t *remaining_sz) 1007 { 1008 uint64_t last_extent_page; 1009 int rc; 1010 1011 last_extent_page = 0; 1012 /* At least single extent table entry has to be always persisted. 1013 * Such case occurs with num_extent_pages == 0. */ 1014 while (last_extent_page <= blob->active.num_extent_pages) { 1015 blob_serialize_extent_table_entry(blob, last_extent_page, &last_extent_page, buf, 1016 remaining_sz); 1017 1018 if (last_extent_page == blob->active.num_extent_pages) { 1019 break; 1020 } 1021 1022 rc = blob_serialize_add_page(blob, pages, page_count, &cur_page); 1023 if (rc < 0) { 1024 return rc; 1025 } 1026 1027 *buf = (uint8_t *)cur_page->descriptors; 1028 *remaining_sz = sizeof(cur_page->descriptors); 1029 } 1030 1031 return 0; 1032 } 1033 1034 static void 1035 blob_serialize_extent_rle(const struct spdk_blob *blob, 1036 uint64_t start_cluster, uint64_t *next_cluster, 1037 uint8_t **buf, size_t *buf_sz) 1038 { 1039 struct spdk_blob_md_descriptor_extent_rle *desc_extent_rle; 1040 size_t cur_sz; 1041 uint64_t i, extent_idx; 1042 uint64_t lba, lba_per_cluster, lba_count; 1043 1044 /* The buffer must have room for at least one extent */ 1045 cur_sz = sizeof(struct spdk_blob_md_descriptor) + sizeof(desc_extent_rle->extents[0]); 1046 if (*buf_sz < cur_sz) { 1047 *next_cluster = start_cluster; 1048 return; 1049 } 1050 1051 desc_extent_rle = (struct spdk_blob_md_descriptor_extent_rle *)*buf; 1052 desc_extent_rle->type = SPDK_MD_DESCRIPTOR_TYPE_EXTENT_RLE; 1053 1054 lba_per_cluster = bs_cluster_to_lba(blob->bs, 1); 1055 /* Assert for scan-build false positive */ 1056 assert(lba_per_cluster > 0); 1057 1058 lba = blob->active.clusters[start_cluster]; 1059 lba_count = lba_per_cluster; 1060 extent_idx = 0; 1061 for (i = start_cluster + 1; i < blob->active.num_clusters; i++) { 1062 if ((lba + lba_count) == blob->active.clusters[i] && lba != 0) { 1063 /* Run-length encode sequential non-zero LBA */ 1064 lba_count += lba_per_cluster; 1065 continue; 1066 } else if (lba == 0 && blob->active.clusters[i] == 0) { 1067 /* Run-length encode unallocated clusters */ 1068 lba_count += lba_per_cluster; 1069 continue; 1070 } 1071 desc_extent_rle->extents[extent_idx].cluster_idx = lba / lba_per_cluster; 1072 desc_extent_rle->extents[extent_idx].length = lba_count / lba_per_cluster; 1073 extent_idx++; 1074 1075 cur_sz += sizeof(desc_extent_rle->extents[extent_idx]); 1076 1077 if (*buf_sz < cur_sz) { 1078 /* If we ran out of buffer space, return */ 1079 *next_cluster = i; 1080 break; 1081 } 1082 1083 lba = blob->active.clusters[i]; 1084 lba_count = lba_per_cluster; 1085 } 1086 1087 if (*buf_sz >= cur_sz) { 1088 desc_extent_rle->extents[extent_idx].cluster_idx = lba / lba_per_cluster; 1089 desc_extent_rle->extents[extent_idx].length = lba_count / lba_per_cluster; 1090 extent_idx++; 1091 1092 *next_cluster = blob->active.num_clusters; 1093 } 1094 1095 desc_extent_rle->length = sizeof(desc_extent_rle->extents[0]) * extent_idx; 1096 *buf_sz -= sizeof(struct spdk_blob_md_descriptor) + desc_extent_rle->length; 1097 *buf += sizeof(struct spdk_blob_md_descriptor) + desc_extent_rle->length; 1098 } 1099 1100 static int 1101 blob_serialize_extents_rle(const struct spdk_blob *blob, 1102 struct spdk_blob_md_page **pages, 1103 struct spdk_blob_md_page *cur_page, 1104 uint32_t *page_count, uint8_t **buf, 1105 size_t *remaining_sz) 1106 { 1107 uint64_t last_cluster; 1108 int rc; 1109 1110 last_cluster = 0; 1111 while (last_cluster < blob->active.num_clusters) { 1112 blob_serialize_extent_rle(blob, last_cluster, &last_cluster, buf, remaining_sz); 1113 1114 if (last_cluster == blob->active.num_clusters) { 1115 break; 1116 } 1117 1118 rc = blob_serialize_add_page(blob, pages, page_count, &cur_page); 1119 if (rc < 0) { 1120 return rc; 1121 } 1122 1123 *buf = (uint8_t *)cur_page->descriptors; 1124 *remaining_sz = sizeof(cur_page->descriptors); 1125 } 1126 1127 return 0; 1128 } 1129 1130 static void 1131 blob_serialize_extent_page(const struct spdk_blob *blob, 1132 uint64_t cluster, struct spdk_blob_md_page *page) 1133 { 1134 struct spdk_blob_md_descriptor_extent_page *desc_extent; 1135 uint64_t i, extent_idx; 1136 uint64_t lba, lba_per_cluster; 1137 uint64_t start_cluster_idx = (cluster / SPDK_EXTENTS_PER_EP) * SPDK_EXTENTS_PER_EP; 1138 1139 desc_extent = (struct spdk_blob_md_descriptor_extent_page *) page->descriptors; 1140 desc_extent->type = SPDK_MD_DESCRIPTOR_TYPE_EXTENT_PAGE; 1141 1142 lba_per_cluster = bs_cluster_to_lba(blob->bs, 1); 1143 1144 desc_extent->start_cluster_idx = start_cluster_idx; 1145 extent_idx = 0; 1146 for (i = start_cluster_idx; i < blob->active.num_clusters; i++) { 1147 lba = blob->active.clusters[i]; 1148 desc_extent->cluster_idx[extent_idx++] = lba / lba_per_cluster; 1149 if (extent_idx >= SPDK_EXTENTS_PER_EP) { 1150 break; 1151 } 1152 } 1153 desc_extent->length = sizeof(desc_extent->start_cluster_idx) + 1154 sizeof(desc_extent->cluster_idx[0]) * extent_idx; 1155 } 1156 1157 static void 1158 blob_serialize_flags(const struct spdk_blob *blob, 1159 uint8_t *buf, size_t *buf_sz) 1160 { 1161 struct spdk_blob_md_descriptor_flags *desc; 1162 1163 /* 1164 * Flags get serialized first, so we should always have room for the flags 1165 * descriptor. 1166 */ 1167 assert(*buf_sz >= sizeof(*desc)); 1168 1169 desc = (struct spdk_blob_md_descriptor_flags *)buf; 1170 desc->type = SPDK_MD_DESCRIPTOR_TYPE_FLAGS; 1171 desc->length = sizeof(*desc) - sizeof(struct spdk_blob_md_descriptor); 1172 desc->invalid_flags = blob->invalid_flags; 1173 desc->data_ro_flags = blob->data_ro_flags; 1174 desc->md_ro_flags = blob->md_ro_flags; 1175 1176 *buf_sz -= sizeof(*desc); 1177 } 1178 1179 static int 1180 blob_serialize_xattrs(const struct spdk_blob *blob, 1181 const struct spdk_xattr_tailq *xattrs, bool internal, 1182 struct spdk_blob_md_page **pages, 1183 struct spdk_blob_md_page *cur_page, 1184 uint32_t *page_count, uint8_t **buf, 1185 size_t *remaining_sz) 1186 { 1187 const struct spdk_xattr *xattr; 1188 int rc; 1189 1190 TAILQ_FOREACH(xattr, xattrs, link) { 1191 size_t required_sz = 0; 1192 1193 rc = blob_serialize_xattr(xattr, 1194 *buf, *remaining_sz, 1195 &required_sz, internal); 1196 if (rc < 0) { 1197 /* Need to add a new page to the chain */ 1198 rc = blob_serialize_add_page(blob, pages, page_count, 1199 &cur_page); 1200 if (rc < 0) { 1201 spdk_free(*pages); 1202 *pages = NULL; 1203 *page_count = 0; 1204 return rc; 1205 } 1206 1207 *buf = (uint8_t *)cur_page->descriptors; 1208 *remaining_sz = sizeof(cur_page->descriptors); 1209 1210 /* Try again */ 1211 required_sz = 0; 1212 rc = blob_serialize_xattr(xattr, 1213 *buf, *remaining_sz, 1214 &required_sz, internal); 1215 1216 if (rc < 0) { 1217 spdk_free(*pages); 1218 *pages = NULL; 1219 *page_count = 0; 1220 return rc; 1221 } 1222 } 1223 1224 *remaining_sz -= required_sz; 1225 *buf += required_sz; 1226 } 1227 1228 return 0; 1229 } 1230 1231 static int 1232 blob_serialize(const struct spdk_blob *blob, struct spdk_blob_md_page **pages, 1233 uint32_t *page_count) 1234 { 1235 struct spdk_blob_md_page *cur_page; 1236 int rc; 1237 uint8_t *buf; 1238 size_t remaining_sz; 1239 1240 assert(pages != NULL); 1241 assert(page_count != NULL); 1242 assert(blob != NULL); 1243 assert(blob->state == SPDK_BLOB_STATE_DIRTY); 1244 1245 *pages = NULL; 1246 *page_count = 0; 1247 1248 /* A blob always has at least 1 page, even if it has no descriptors */ 1249 rc = blob_serialize_add_page(blob, pages, page_count, &cur_page); 1250 if (rc < 0) { 1251 return rc; 1252 } 1253 1254 buf = (uint8_t *)cur_page->descriptors; 1255 remaining_sz = sizeof(cur_page->descriptors); 1256 1257 /* Serialize flags */ 1258 blob_serialize_flags(blob, buf, &remaining_sz); 1259 buf += sizeof(struct spdk_blob_md_descriptor_flags); 1260 1261 /* Serialize xattrs */ 1262 rc = blob_serialize_xattrs(blob, &blob->xattrs, false, 1263 pages, cur_page, page_count, &buf, &remaining_sz); 1264 if (rc < 0) { 1265 return rc; 1266 } 1267 1268 /* Serialize internal xattrs */ 1269 rc = blob_serialize_xattrs(blob, &blob->xattrs_internal, true, 1270 pages, cur_page, page_count, &buf, &remaining_sz); 1271 if (rc < 0) { 1272 return rc; 1273 } 1274 1275 if (blob->use_extent_table) { 1276 /* Serialize extent table */ 1277 rc = blob_serialize_extent_table(blob, pages, cur_page, page_count, &buf, &remaining_sz); 1278 } else { 1279 /* Serialize extents */ 1280 rc = blob_serialize_extents_rle(blob, pages, cur_page, page_count, &buf, &remaining_sz); 1281 } 1282 1283 return rc; 1284 } 1285 1286 struct spdk_blob_load_ctx { 1287 struct spdk_blob *blob; 1288 1289 struct spdk_blob_md_page *pages; 1290 uint32_t num_pages; 1291 uint32_t next_extent_page; 1292 spdk_bs_sequence_t *seq; 1293 1294 spdk_bs_sequence_cpl cb_fn; 1295 void *cb_arg; 1296 }; 1297 1298 static uint32_t 1299 blob_md_page_calc_crc(void *page) 1300 { 1301 uint32_t crc; 1302 1303 crc = BLOB_CRC32C_INITIAL; 1304 crc = spdk_crc32c_update(page, SPDK_BS_PAGE_SIZE - 4, crc); 1305 crc ^= BLOB_CRC32C_INITIAL; 1306 1307 return crc; 1308 1309 } 1310 1311 static void 1312 blob_load_final(struct spdk_blob_load_ctx *ctx, int bserrno) 1313 { 1314 struct spdk_blob *blob = ctx->blob; 1315 1316 if (bserrno == 0) { 1317 blob_mark_clean(blob); 1318 } 1319 1320 ctx->cb_fn(ctx->seq, ctx->cb_arg, bserrno); 1321 1322 /* Free the memory */ 1323 spdk_free(ctx->pages); 1324 free(ctx); 1325 } 1326 1327 static void 1328 blob_load_snapshot_cpl(void *cb_arg, struct spdk_blob *snapshot, int bserrno) 1329 { 1330 struct spdk_blob_load_ctx *ctx = cb_arg; 1331 struct spdk_blob *blob = ctx->blob; 1332 1333 if (bserrno == 0) { 1334 blob->back_bs_dev = bs_create_blob_bs_dev(snapshot); 1335 if (blob->back_bs_dev == NULL) { 1336 bserrno = -ENOMEM; 1337 } 1338 } 1339 if (bserrno != 0) { 1340 SPDK_ERRLOG("Snapshot fail\n"); 1341 } 1342 1343 blob_load_final(ctx, bserrno); 1344 } 1345 1346 static void blob_update_clear_method(struct spdk_blob *blob); 1347 1348 static int 1349 blob_load_esnap(struct spdk_blob *blob) 1350 { 1351 struct spdk_blob_store *bs = blob->bs; 1352 struct spdk_bs_dev *bs_dev = NULL; 1353 const void *esnap_id = NULL; 1354 size_t id_len = 0; 1355 int rc; 1356 1357 if (bs->esnap_bs_dev_create == NULL) { 1358 SPDK_NOTICELOG("blob 0x%" PRIx64 " is an esnap clone but the blobstore was opened " 1359 "without support for esnap clones\n", blob->id); 1360 return -ENOTSUP; 1361 } 1362 assert(blob->back_bs_dev == NULL); 1363 1364 rc = blob_get_xattr_value(blob, BLOB_EXTERNAL_SNAPSHOT_ID, &esnap_id, &id_len, true); 1365 if (rc != 0) { 1366 SPDK_ERRLOG("blob 0x%" PRIx64 " is an esnap clone but has no esnap ID\n", blob->id); 1367 return -EINVAL; 1368 } 1369 assert(id_len > 0 && id_len < UINT32_MAX); 1370 1371 SPDK_INFOLOG(blob, "Creating external snapshot device\n"); 1372 1373 rc = bs->esnap_bs_dev_create(blob, esnap_id, (uint32_t)id_len, &bs_dev); 1374 if (rc != 0) { 1375 SPDK_DEBUGLOG(blob_esnap, "blob 0x%" PRIx64 ": failed to load back_bs_dev " 1376 "with error %d\n", blob->id, rc); 1377 return rc; 1378 } 1379 1380 assert(bs_dev != NULL); 1381 SPDK_DEBUGLOG(blob_esnap, "blob 0x%" PRIx64 ": loaded back_bs_dev\n", blob->id); 1382 if ((bs->io_unit_size % bs_dev->blocklen) != 0) { 1383 SPDK_NOTICELOG("blob 0x%" PRIx64 " external snapshot device block size %u is not " 1384 "compatible with blobstore block size %u\n", 1385 blob->id, bs_dev->blocklen, bs->io_unit_size); 1386 bs_dev->destroy(bs_dev); 1387 return -EINVAL; 1388 } 1389 1390 blob->back_bs_dev = bs_dev; 1391 blob->parent_id = SPDK_BLOBID_EXTERNAL_SNAPSHOT; 1392 1393 return 0; 1394 } 1395 1396 static void 1397 blob_load_backing_dev(void *cb_arg) 1398 { 1399 struct spdk_blob_load_ctx *ctx = cb_arg; 1400 struct spdk_blob *blob = ctx->blob; 1401 const void *value; 1402 size_t len; 1403 int rc; 1404 1405 if (blob_is_esnap_clone(blob)) { 1406 rc = blob_load_esnap(blob); 1407 assert((rc == 0) ^ (blob->back_bs_dev == NULL)); 1408 blob_load_final(ctx, rc); 1409 return; 1410 } 1411 1412 if (spdk_blob_is_thin_provisioned(blob)) { 1413 rc = blob_get_xattr_value(blob, BLOB_SNAPSHOT, &value, &len, true); 1414 if (rc == 0) { 1415 if (len != sizeof(spdk_blob_id)) { 1416 blob_load_final(ctx, -EINVAL); 1417 return; 1418 } 1419 /* open snapshot blob and continue in the callback function */ 1420 blob->parent_id = *(spdk_blob_id *)value; 1421 spdk_bs_open_blob(blob->bs, blob->parent_id, 1422 blob_load_snapshot_cpl, ctx); 1423 return; 1424 } else { 1425 /* add zeroes_dev for thin provisioned blob */ 1426 blob->back_bs_dev = bs_create_zeroes_dev(); 1427 } 1428 } else { 1429 /* standard blob */ 1430 blob->back_bs_dev = NULL; 1431 } 1432 blob_load_final(ctx, 0); 1433 } 1434 1435 static void 1436 blob_load_cpl_extents_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 1437 { 1438 struct spdk_blob_load_ctx *ctx = cb_arg; 1439 struct spdk_blob *blob = ctx->blob; 1440 struct spdk_blob_md_page *page; 1441 uint64_t i; 1442 uint32_t crc; 1443 uint64_t lba; 1444 void *tmp; 1445 uint64_t sz; 1446 1447 if (bserrno) { 1448 SPDK_ERRLOG("Extent page read failed: %d\n", bserrno); 1449 blob_load_final(ctx, bserrno); 1450 return; 1451 } 1452 1453 if (ctx->pages == NULL) { 1454 /* First iteration of this function, allocate buffer for single EXTENT_PAGE */ 1455 ctx->pages = spdk_zmalloc(SPDK_BS_PAGE_SIZE, 0, 1456 NULL, SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA); 1457 if (!ctx->pages) { 1458 blob_load_final(ctx, -ENOMEM); 1459 return; 1460 } 1461 ctx->num_pages = 1; 1462 ctx->next_extent_page = 0; 1463 } else { 1464 page = &ctx->pages[0]; 1465 crc = blob_md_page_calc_crc(page); 1466 if (crc != page->crc) { 1467 blob_load_final(ctx, -EINVAL); 1468 return; 1469 } 1470 1471 if (page->next != SPDK_INVALID_MD_PAGE) { 1472 blob_load_final(ctx, -EINVAL); 1473 return; 1474 } 1475 1476 bserrno = blob_parse_extent_page(page, blob); 1477 if (bserrno) { 1478 blob_load_final(ctx, bserrno); 1479 return; 1480 } 1481 } 1482 1483 for (i = ctx->next_extent_page; i < blob->active.num_extent_pages; i++) { 1484 if (blob->active.extent_pages[i] != 0) { 1485 /* Extent page was allocated, read and parse it. */ 1486 lba = bs_md_page_to_lba(blob->bs, blob->active.extent_pages[i]); 1487 ctx->next_extent_page = i + 1; 1488 1489 bs_sequence_read_dev(seq, &ctx->pages[0], lba, 1490 bs_byte_to_lba(blob->bs, SPDK_BS_PAGE_SIZE), 1491 blob_load_cpl_extents_cpl, ctx); 1492 return; 1493 } else { 1494 /* Thin provisioned blobs can point to unallocated extent pages. 1495 * In this case blob size should be increased by up to the amount left in remaining_clusters_in_et. */ 1496 1497 sz = spdk_min(blob->remaining_clusters_in_et, SPDK_EXTENTS_PER_EP); 1498 blob->active.num_clusters += sz; 1499 blob->remaining_clusters_in_et -= sz; 1500 1501 assert(spdk_blob_is_thin_provisioned(blob)); 1502 assert(i + 1 < blob->active.num_extent_pages || blob->remaining_clusters_in_et == 0); 1503 1504 tmp = realloc(blob->active.clusters, blob->active.num_clusters * sizeof(*blob->active.clusters)); 1505 if (tmp == NULL) { 1506 blob_load_final(ctx, -ENOMEM); 1507 return; 1508 } 1509 memset(tmp + sizeof(*blob->active.clusters) * blob->active.cluster_array_size, 0, 1510 sizeof(*blob->active.clusters) * (blob->active.num_clusters - blob->active.cluster_array_size)); 1511 blob->active.clusters = tmp; 1512 blob->active.cluster_array_size = blob->active.num_clusters; 1513 } 1514 } 1515 1516 blob_load_backing_dev(ctx); 1517 } 1518 1519 static void 1520 blob_load_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 1521 { 1522 struct spdk_blob_load_ctx *ctx = cb_arg; 1523 struct spdk_blob *blob = ctx->blob; 1524 struct spdk_blob_md_page *page; 1525 int rc; 1526 uint32_t crc; 1527 uint32_t current_page; 1528 1529 if (ctx->num_pages == 1) { 1530 current_page = bs_blobid_to_page(blob->id); 1531 } else { 1532 assert(ctx->num_pages != 0); 1533 page = &ctx->pages[ctx->num_pages - 2]; 1534 current_page = page->next; 1535 } 1536 1537 if (bserrno) { 1538 SPDK_ERRLOG("Metadata page %d read failed for blobid %" PRIu64 ": %d\n", 1539 current_page, blob->id, bserrno); 1540 blob_load_final(ctx, bserrno); 1541 return; 1542 } 1543 1544 page = &ctx->pages[ctx->num_pages - 1]; 1545 crc = blob_md_page_calc_crc(page); 1546 if (crc != page->crc) { 1547 SPDK_ERRLOG("Metadata page %d crc mismatch for blobid %" PRIu64 "\n", 1548 current_page, blob->id); 1549 blob_load_final(ctx, -EINVAL); 1550 return; 1551 } 1552 1553 if (page->next != SPDK_INVALID_MD_PAGE) { 1554 struct spdk_blob_md_page *tmp_pages; 1555 uint32_t next_page = page->next; 1556 uint64_t next_lba = bs_md_page_to_lba(blob->bs, next_page); 1557 1558 /* Read the next page */ 1559 tmp_pages = spdk_realloc(ctx->pages, (sizeof(*page) * (ctx->num_pages + 1)), 0); 1560 if (tmp_pages == NULL) { 1561 blob_load_final(ctx, -ENOMEM); 1562 return; 1563 } 1564 ctx->num_pages++; 1565 ctx->pages = tmp_pages; 1566 1567 bs_sequence_read_dev(seq, &ctx->pages[ctx->num_pages - 1], 1568 next_lba, 1569 bs_byte_to_lba(blob->bs, sizeof(*page)), 1570 blob_load_cpl, ctx); 1571 return; 1572 } 1573 1574 /* Parse the pages */ 1575 rc = blob_parse(ctx->pages, ctx->num_pages, blob); 1576 if (rc) { 1577 blob_load_final(ctx, rc); 1578 return; 1579 } 1580 1581 if (blob->extent_table_found == true) { 1582 /* If EXTENT_TABLE was found, that means support for it should be enabled. */ 1583 assert(blob->extent_rle_found == false); 1584 blob->use_extent_table = true; 1585 } else { 1586 /* If EXTENT_RLE or no extent_* descriptor was found disable support 1587 * for extent table. No extent_* descriptors means that blob has length of 0 1588 * and no extent_rle descriptors were persisted for it. 1589 * EXTENT_TABLE if used, is always present in metadata regardless of length. */ 1590 blob->use_extent_table = false; 1591 } 1592 1593 /* Check the clear_method stored in metadata vs what may have been passed 1594 * via spdk_bs_open_blob_ext() and update accordingly. 1595 */ 1596 blob_update_clear_method(blob); 1597 1598 spdk_free(ctx->pages); 1599 ctx->pages = NULL; 1600 1601 if (blob->extent_table_found) { 1602 blob_load_cpl_extents_cpl(seq, ctx, 0); 1603 } else { 1604 blob_load_backing_dev(ctx); 1605 } 1606 } 1607 1608 /* Load a blob from disk given a blobid */ 1609 static void 1610 blob_load(spdk_bs_sequence_t *seq, struct spdk_blob *blob, 1611 spdk_bs_sequence_cpl cb_fn, void *cb_arg) 1612 { 1613 struct spdk_blob_load_ctx *ctx; 1614 struct spdk_blob_store *bs; 1615 uint32_t page_num; 1616 uint64_t lba; 1617 1618 blob_verify_md_op(blob); 1619 1620 bs = blob->bs; 1621 1622 ctx = calloc(1, sizeof(*ctx)); 1623 if (!ctx) { 1624 cb_fn(seq, cb_arg, -ENOMEM); 1625 return; 1626 } 1627 1628 ctx->blob = blob; 1629 ctx->pages = spdk_realloc(ctx->pages, SPDK_BS_PAGE_SIZE, 0); 1630 if (!ctx->pages) { 1631 free(ctx); 1632 cb_fn(seq, cb_arg, -ENOMEM); 1633 return; 1634 } 1635 ctx->num_pages = 1; 1636 ctx->cb_fn = cb_fn; 1637 ctx->cb_arg = cb_arg; 1638 ctx->seq = seq; 1639 1640 page_num = bs_blobid_to_page(blob->id); 1641 lba = bs_md_page_to_lba(blob->bs, page_num); 1642 1643 blob->state = SPDK_BLOB_STATE_LOADING; 1644 1645 bs_sequence_read_dev(seq, &ctx->pages[0], lba, 1646 bs_byte_to_lba(bs, SPDK_BS_PAGE_SIZE), 1647 blob_load_cpl, ctx); 1648 } 1649 1650 struct spdk_blob_persist_ctx { 1651 struct spdk_blob *blob; 1652 1653 struct spdk_blob_md_page *pages; 1654 uint32_t next_extent_page; 1655 struct spdk_blob_md_page *extent_page; 1656 1657 spdk_bs_sequence_t *seq; 1658 spdk_bs_sequence_cpl cb_fn; 1659 void *cb_arg; 1660 TAILQ_ENTRY(spdk_blob_persist_ctx) link; 1661 }; 1662 1663 static void 1664 bs_batch_clear_dev(struct spdk_blob_persist_ctx *ctx, spdk_bs_batch_t *batch, uint64_t lba, 1665 uint64_t lba_count) 1666 { 1667 switch (ctx->blob->clear_method) { 1668 case BLOB_CLEAR_WITH_DEFAULT: 1669 case BLOB_CLEAR_WITH_UNMAP: 1670 bs_batch_unmap_dev(batch, lba, lba_count); 1671 break; 1672 case BLOB_CLEAR_WITH_WRITE_ZEROES: 1673 bs_batch_write_zeroes_dev(batch, lba, lba_count); 1674 break; 1675 case BLOB_CLEAR_WITH_NONE: 1676 default: 1677 break; 1678 } 1679 } 1680 1681 static void bs_mark_dirty(spdk_bs_sequence_t *seq, struct spdk_blob_store *bs, 1682 spdk_bs_sequence_cpl cb_fn, void *cb_arg); 1683 1684 static void 1685 blob_persist_complete_cb(void *arg) 1686 { 1687 struct spdk_blob_persist_ctx *ctx = arg; 1688 1689 /* Call user callback */ 1690 ctx->cb_fn(ctx->seq, ctx->cb_arg, 0); 1691 1692 /* Free the memory */ 1693 spdk_free(ctx->pages); 1694 free(ctx); 1695 } 1696 1697 static void blob_persist_start(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno); 1698 1699 static void 1700 blob_persist_complete(spdk_bs_sequence_t *seq, struct spdk_blob_persist_ctx *ctx, int bserrno) 1701 { 1702 struct spdk_blob_persist_ctx *next_persist, *tmp; 1703 struct spdk_blob *blob = ctx->blob; 1704 1705 if (bserrno == 0) { 1706 blob_mark_clean(blob); 1707 } 1708 1709 assert(ctx == TAILQ_FIRST(&blob->persists_to_complete)); 1710 1711 /* Complete all persists that were pending when the current persist started */ 1712 TAILQ_FOREACH_SAFE(next_persist, &blob->persists_to_complete, link, tmp) { 1713 TAILQ_REMOVE(&blob->persists_to_complete, next_persist, link); 1714 spdk_thread_send_msg(spdk_get_thread(), blob_persist_complete_cb, next_persist); 1715 } 1716 1717 if (TAILQ_EMPTY(&blob->pending_persists)) { 1718 return; 1719 } 1720 1721 /* Queue up all pending persists for completion and start blob persist with first one */ 1722 TAILQ_SWAP(&blob->persists_to_complete, &blob->pending_persists, spdk_blob_persist_ctx, link); 1723 next_persist = TAILQ_FIRST(&blob->persists_to_complete); 1724 1725 blob->state = SPDK_BLOB_STATE_DIRTY; 1726 bs_mark_dirty(seq, blob->bs, blob_persist_start, next_persist); 1727 } 1728 1729 static void 1730 blob_persist_clear_extents_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 1731 { 1732 struct spdk_blob_persist_ctx *ctx = cb_arg; 1733 struct spdk_blob *blob = ctx->blob; 1734 struct spdk_blob_store *bs = blob->bs; 1735 size_t i; 1736 1737 if (bserrno != 0) { 1738 blob_persist_complete(seq, ctx, bserrno); 1739 return; 1740 } 1741 1742 spdk_spin_lock(&bs->used_lock); 1743 1744 /* Release all extent_pages that were truncated */ 1745 for (i = blob->active.num_extent_pages; i < blob->active.extent_pages_array_size; i++) { 1746 /* Nothing to release if it was not allocated */ 1747 if (blob->active.extent_pages[i] != 0) { 1748 bs_release_md_page(bs, blob->active.extent_pages[i]); 1749 } 1750 } 1751 1752 spdk_spin_unlock(&bs->used_lock); 1753 1754 if (blob->active.num_extent_pages == 0) { 1755 free(blob->active.extent_pages); 1756 blob->active.extent_pages = NULL; 1757 blob->active.extent_pages_array_size = 0; 1758 } else if (blob->active.num_extent_pages != blob->active.extent_pages_array_size) { 1759 #ifndef __clang_analyzer__ 1760 void *tmp; 1761 1762 /* scan-build really can't figure reallocs, workaround it */ 1763 tmp = realloc(blob->active.extent_pages, sizeof(uint32_t) * blob->active.num_extent_pages); 1764 assert(tmp != NULL); 1765 blob->active.extent_pages = tmp; 1766 #endif 1767 blob->active.extent_pages_array_size = blob->active.num_extent_pages; 1768 } 1769 1770 blob_persist_complete(seq, ctx, bserrno); 1771 } 1772 1773 static void 1774 blob_persist_clear_extents(spdk_bs_sequence_t *seq, struct spdk_blob_persist_ctx *ctx) 1775 { 1776 struct spdk_blob *blob = ctx->blob; 1777 struct spdk_blob_store *bs = blob->bs; 1778 size_t i; 1779 uint64_t lba; 1780 uint64_t lba_count; 1781 spdk_bs_batch_t *batch; 1782 1783 batch = bs_sequence_to_batch(seq, blob_persist_clear_extents_cpl, ctx); 1784 lba_count = bs_byte_to_lba(bs, SPDK_BS_PAGE_SIZE); 1785 1786 /* Clear all extent_pages that were truncated */ 1787 for (i = blob->active.num_extent_pages; i < blob->active.extent_pages_array_size; i++) { 1788 /* Nothing to clear if it was not allocated */ 1789 if (blob->active.extent_pages[i] != 0) { 1790 lba = bs_md_page_to_lba(bs, blob->active.extent_pages[i]); 1791 bs_batch_write_zeroes_dev(batch, lba, lba_count); 1792 } 1793 } 1794 1795 bs_batch_close(batch); 1796 } 1797 1798 static void 1799 blob_persist_clear_clusters_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 1800 { 1801 struct spdk_blob_persist_ctx *ctx = cb_arg; 1802 struct spdk_blob *blob = ctx->blob; 1803 struct spdk_blob_store *bs = blob->bs; 1804 size_t i; 1805 1806 if (bserrno != 0) { 1807 blob_persist_complete(seq, ctx, bserrno); 1808 return; 1809 } 1810 1811 spdk_spin_lock(&bs->used_lock); 1812 /* Release all clusters that were truncated */ 1813 for (i = blob->active.num_clusters; i < blob->active.cluster_array_size; i++) { 1814 uint32_t cluster_num = bs_lba_to_cluster(bs, blob->active.clusters[i]); 1815 1816 /* Nothing to release if it was not allocated */ 1817 if (blob->active.clusters[i] != 0) { 1818 bs_release_cluster(bs, cluster_num); 1819 } 1820 } 1821 spdk_spin_unlock(&bs->used_lock); 1822 1823 if (blob->active.num_clusters == 0) { 1824 free(blob->active.clusters); 1825 blob->active.clusters = NULL; 1826 blob->active.cluster_array_size = 0; 1827 } else if (blob->active.num_clusters != blob->active.cluster_array_size) { 1828 #ifndef __clang_analyzer__ 1829 void *tmp; 1830 1831 /* scan-build really can't figure reallocs, workaround it */ 1832 tmp = realloc(blob->active.clusters, sizeof(*blob->active.clusters) * blob->active.num_clusters); 1833 assert(tmp != NULL); 1834 blob->active.clusters = tmp; 1835 1836 #endif 1837 blob->active.cluster_array_size = blob->active.num_clusters; 1838 } 1839 1840 /* Move on to clearing extent pages */ 1841 blob_persist_clear_extents(seq, ctx); 1842 } 1843 1844 static void 1845 blob_persist_clear_clusters(spdk_bs_sequence_t *seq, struct spdk_blob_persist_ctx *ctx) 1846 { 1847 struct spdk_blob *blob = ctx->blob; 1848 struct spdk_blob_store *bs = blob->bs; 1849 spdk_bs_batch_t *batch; 1850 size_t i; 1851 uint64_t lba; 1852 uint64_t lba_count; 1853 1854 /* Clusters don't move around in blobs. The list shrinks or grows 1855 * at the end, but no changes ever occur in the middle of the list. 1856 */ 1857 1858 batch = bs_sequence_to_batch(seq, blob_persist_clear_clusters_cpl, ctx); 1859 1860 /* Clear all clusters that were truncated */ 1861 lba = 0; 1862 lba_count = 0; 1863 for (i = blob->active.num_clusters; i < blob->active.cluster_array_size; i++) { 1864 uint64_t next_lba = blob->active.clusters[i]; 1865 uint64_t next_lba_count = bs_cluster_to_lba(bs, 1); 1866 1867 if (next_lba > 0 && (lba + lba_count) == next_lba) { 1868 /* This cluster is contiguous with the previous one. */ 1869 lba_count += next_lba_count; 1870 continue; 1871 } else if (next_lba == 0) { 1872 continue; 1873 } 1874 1875 /* This cluster is not contiguous with the previous one. */ 1876 1877 /* If a run of LBAs previously existing, clear them now */ 1878 if (lba_count > 0) { 1879 bs_batch_clear_dev(ctx, batch, lba, lba_count); 1880 } 1881 1882 /* Start building the next batch */ 1883 lba = next_lba; 1884 if (next_lba > 0) { 1885 lba_count = next_lba_count; 1886 } else { 1887 lba_count = 0; 1888 } 1889 } 1890 1891 /* If we ended with a contiguous set of LBAs, clear them now */ 1892 if (lba_count > 0) { 1893 bs_batch_clear_dev(ctx, batch, lba, lba_count); 1894 } 1895 1896 bs_batch_close(batch); 1897 } 1898 1899 static void 1900 blob_persist_zero_pages_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 1901 { 1902 struct spdk_blob_persist_ctx *ctx = cb_arg; 1903 struct spdk_blob *blob = ctx->blob; 1904 struct spdk_blob_store *bs = blob->bs; 1905 size_t i; 1906 1907 if (bserrno != 0) { 1908 blob_persist_complete(seq, ctx, bserrno); 1909 return; 1910 } 1911 1912 spdk_spin_lock(&bs->used_lock); 1913 1914 /* This loop starts at 1 because the first page is special and handled 1915 * below. The pages (except the first) are never written in place, 1916 * so any pages in the clean list must be zeroed. 1917 */ 1918 for (i = 1; i < blob->clean.num_pages; i++) { 1919 bs_release_md_page(bs, blob->clean.pages[i]); 1920 } 1921 1922 if (blob->active.num_pages == 0) { 1923 uint32_t page_num; 1924 1925 page_num = bs_blobid_to_page(blob->id); 1926 bs_release_md_page(bs, page_num); 1927 } 1928 1929 spdk_spin_unlock(&bs->used_lock); 1930 1931 /* Move on to clearing clusters */ 1932 blob_persist_clear_clusters(seq, ctx); 1933 } 1934 1935 static void 1936 blob_persist_zero_pages(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 1937 { 1938 struct spdk_blob_persist_ctx *ctx = cb_arg; 1939 struct spdk_blob *blob = ctx->blob; 1940 struct spdk_blob_store *bs = blob->bs; 1941 uint64_t lba; 1942 uint64_t lba_count; 1943 spdk_bs_batch_t *batch; 1944 size_t i; 1945 1946 if (bserrno != 0) { 1947 blob_persist_complete(seq, ctx, bserrno); 1948 return; 1949 } 1950 1951 batch = bs_sequence_to_batch(seq, blob_persist_zero_pages_cpl, ctx); 1952 1953 lba_count = bs_byte_to_lba(bs, SPDK_BS_PAGE_SIZE); 1954 1955 /* This loop starts at 1 because the first page is special and handled 1956 * below. The pages (except the first) are never written in place, 1957 * so any pages in the clean list must be zeroed. 1958 */ 1959 for (i = 1; i < blob->clean.num_pages; i++) { 1960 lba = bs_md_page_to_lba(bs, blob->clean.pages[i]); 1961 1962 bs_batch_write_zeroes_dev(batch, lba, lba_count); 1963 } 1964 1965 /* The first page will only be zeroed if this is a delete. */ 1966 if (blob->active.num_pages == 0) { 1967 uint32_t page_num; 1968 1969 /* The first page in the metadata goes where the blobid indicates */ 1970 page_num = bs_blobid_to_page(blob->id); 1971 lba = bs_md_page_to_lba(bs, page_num); 1972 1973 bs_batch_write_zeroes_dev(batch, lba, lba_count); 1974 } 1975 1976 bs_batch_close(batch); 1977 } 1978 1979 static void 1980 blob_persist_write_page_root(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 1981 { 1982 struct spdk_blob_persist_ctx *ctx = cb_arg; 1983 struct spdk_blob *blob = ctx->blob; 1984 struct spdk_blob_store *bs = blob->bs; 1985 uint64_t lba; 1986 uint32_t lba_count; 1987 struct spdk_blob_md_page *page; 1988 1989 if (bserrno != 0) { 1990 blob_persist_complete(seq, ctx, bserrno); 1991 return; 1992 } 1993 1994 if (blob->active.num_pages == 0) { 1995 /* Move on to the next step */ 1996 blob_persist_zero_pages(seq, ctx, 0); 1997 return; 1998 } 1999 2000 lba_count = bs_byte_to_lba(bs, sizeof(*page)); 2001 2002 page = &ctx->pages[0]; 2003 /* The first page in the metadata goes where the blobid indicates */ 2004 lba = bs_md_page_to_lba(bs, bs_blobid_to_page(blob->id)); 2005 2006 bs_sequence_write_dev(seq, page, lba, lba_count, 2007 blob_persist_zero_pages, ctx); 2008 } 2009 2010 static void 2011 blob_persist_write_page_chain(spdk_bs_sequence_t *seq, struct spdk_blob_persist_ctx *ctx) 2012 { 2013 struct spdk_blob *blob = ctx->blob; 2014 struct spdk_blob_store *bs = blob->bs; 2015 uint64_t lba; 2016 uint32_t lba_count; 2017 struct spdk_blob_md_page *page; 2018 spdk_bs_batch_t *batch; 2019 size_t i; 2020 2021 /* Clusters don't move around in blobs. The list shrinks or grows 2022 * at the end, but no changes ever occur in the middle of the list. 2023 */ 2024 2025 lba_count = bs_byte_to_lba(bs, sizeof(*page)); 2026 2027 batch = bs_sequence_to_batch(seq, blob_persist_write_page_root, ctx); 2028 2029 /* This starts at 1. The root page is not written until 2030 * all of the others are finished 2031 */ 2032 for (i = 1; i < blob->active.num_pages; i++) { 2033 page = &ctx->pages[i]; 2034 assert(page->sequence_num == i); 2035 2036 lba = bs_md_page_to_lba(bs, blob->active.pages[i]); 2037 2038 bs_batch_write_dev(batch, page, lba, lba_count); 2039 } 2040 2041 bs_batch_close(batch); 2042 } 2043 2044 static int 2045 blob_resize(struct spdk_blob *blob, uint64_t sz) 2046 { 2047 uint64_t i; 2048 uint64_t *tmp; 2049 uint64_t cluster; 2050 uint32_t lfmd; /* lowest free md page */ 2051 uint64_t num_clusters; 2052 uint32_t *ep_tmp; 2053 uint64_t new_num_ep = 0, current_num_ep = 0; 2054 struct spdk_blob_store *bs; 2055 int rc; 2056 2057 bs = blob->bs; 2058 2059 blob_verify_md_op(blob); 2060 2061 if (blob->active.num_clusters == sz) { 2062 return 0; 2063 } 2064 2065 if (blob->active.num_clusters < blob->active.cluster_array_size) { 2066 /* If this blob was resized to be larger, then smaller, then 2067 * larger without syncing, then the cluster array already 2068 * contains spare assigned clusters we can use. 2069 */ 2070 num_clusters = spdk_min(blob->active.cluster_array_size, 2071 sz); 2072 } else { 2073 num_clusters = blob->active.num_clusters; 2074 } 2075 2076 if (blob->use_extent_table) { 2077 /* Round up since every cluster beyond current Extent Table size, 2078 * requires new extent page. */ 2079 new_num_ep = spdk_divide_round_up(sz, SPDK_EXTENTS_PER_EP); 2080 current_num_ep = spdk_divide_round_up(num_clusters, SPDK_EXTENTS_PER_EP); 2081 } 2082 2083 assert(!spdk_spin_held(&bs->used_lock)); 2084 2085 /* Check first that we have enough clusters and md pages before we start claiming them. 2086 * bs->used_lock is held to ensure that clusters we think are free are still free when we go 2087 * to claim them later in this function. 2088 */ 2089 if (sz > num_clusters && spdk_blob_is_thin_provisioned(blob) == false) { 2090 spdk_spin_lock(&bs->used_lock); 2091 if ((sz - num_clusters) > bs->num_free_clusters) { 2092 rc = -ENOSPC; 2093 goto out; 2094 } 2095 lfmd = 0; 2096 for (i = current_num_ep; i < new_num_ep ; i++) { 2097 lfmd = spdk_bit_array_find_first_clear(blob->bs->used_md_pages, lfmd); 2098 if (lfmd == UINT32_MAX) { 2099 /* No more free md pages. Cannot satisfy the request */ 2100 rc = -ENOSPC; 2101 goto out; 2102 } 2103 } 2104 } 2105 2106 if (sz > num_clusters) { 2107 /* Expand the cluster array if necessary. 2108 * We only shrink the array when persisting. 2109 */ 2110 tmp = realloc(blob->active.clusters, sizeof(*blob->active.clusters) * sz); 2111 if (sz > 0 && tmp == NULL) { 2112 rc = -ENOMEM; 2113 goto out; 2114 } 2115 memset(tmp + blob->active.cluster_array_size, 0, 2116 sizeof(*blob->active.clusters) * (sz - blob->active.cluster_array_size)); 2117 blob->active.clusters = tmp; 2118 blob->active.cluster_array_size = sz; 2119 2120 /* Expand the extents table, only if enough clusters were added */ 2121 if (new_num_ep > current_num_ep && blob->use_extent_table) { 2122 ep_tmp = realloc(blob->active.extent_pages, sizeof(*blob->active.extent_pages) * new_num_ep); 2123 if (new_num_ep > 0 && ep_tmp == NULL) { 2124 rc = -ENOMEM; 2125 goto out; 2126 } 2127 memset(ep_tmp + blob->active.extent_pages_array_size, 0, 2128 sizeof(*blob->active.extent_pages) * (new_num_ep - blob->active.extent_pages_array_size)); 2129 blob->active.extent_pages = ep_tmp; 2130 blob->active.extent_pages_array_size = new_num_ep; 2131 } 2132 } 2133 2134 blob->state = SPDK_BLOB_STATE_DIRTY; 2135 2136 if (spdk_blob_is_thin_provisioned(blob) == false) { 2137 cluster = 0; 2138 lfmd = 0; 2139 for (i = num_clusters; i < sz; i++) { 2140 bs_allocate_cluster(blob, i, &cluster, &lfmd, true); 2141 lfmd++; 2142 } 2143 } 2144 2145 blob->active.num_clusters = sz; 2146 blob->active.num_extent_pages = new_num_ep; 2147 2148 rc = 0; 2149 out: 2150 if (spdk_spin_held(&bs->used_lock)) { 2151 spdk_spin_unlock(&bs->used_lock); 2152 } 2153 2154 return rc; 2155 } 2156 2157 static void 2158 blob_persist_generate_new_md(struct spdk_blob_persist_ctx *ctx) 2159 { 2160 spdk_bs_sequence_t *seq = ctx->seq; 2161 struct spdk_blob *blob = ctx->blob; 2162 struct spdk_blob_store *bs = blob->bs; 2163 uint64_t i; 2164 uint32_t page_num; 2165 void *tmp; 2166 int rc; 2167 2168 /* Generate the new metadata */ 2169 rc = blob_serialize(blob, &ctx->pages, &blob->active.num_pages); 2170 if (rc < 0) { 2171 blob_persist_complete(seq, ctx, rc); 2172 return; 2173 } 2174 2175 assert(blob->active.num_pages >= 1); 2176 2177 /* Resize the cache of page indices */ 2178 tmp = realloc(blob->active.pages, blob->active.num_pages * sizeof(*blob->active.pages)); 2179 if (!tmp) { 2180 blob_persist_complete(seq, ctx, -ENOMEM); 2181 return; 2182 } 2183 blob->active.pages = tmp; 2184 2185 /* Assign this metadata to pages. This requires two passes - one to verify that there are 2186 * enough pages and a second to actually claim them. The used_lock is held across 2187 * both passes to ensure things don't change in the middle. 2188 */ 2189 spdk_spin_lock(&bs->used_lock); 2190 page_num = 0; 2191 /* Note that this loop starts at one. The first page location is fixed by the blobid. */ 2192 for (i = 1; i < blob->active.num_pages; i++) { 2193 page_num = spdk_bit_array_find_first_clear(bs->used_md_pages, page_num); 2194 if (page_num == UINT32_MAX) { 2195 spdk_spin_unlock(&bs->used_lock); 2196 blob_persist_complete(seq, ctx, -ENOMEM); 2197 return; 2198 } 2199 page_num++; 2200 } 2201 2202 page_num = 0; 2203 blob->active.pages[0] = bs_blobid_to_page(blob->id); 2204 for (i = 1; i < blob->active.num_pages; i++) { 2205 page_num = spdk_bit_array_find_first_clear(bs->used_md_pages, page_num); 2206 ctx->pages[i - 1].next = page_num; 2207 /* Now that previous metadata page is complete, calculate the crc for it. */ 2208 ctx->pages[i - 1].crc = blob_md_page_calc_crc(&ctx->pages[i - 1]); 2209 blob->active.pages[i] = page_num; 2210 bs_claim_md_page(bs, page_num); 2211 SPDK_DEBUGLOG(blob, "Claiming page %u for blob 0x%" PRIx64 "\n", page_num, 2212 blob->id); 2213 page_num++; 2214 } 2215 spdk_spin_unlock(&bs->used_lock); 2216 ctx->pages[i - 1].crc = blob_md_page_calc_crc(&ctx->pages[i - 1]); 2217 /* Start writing the metadata from last page to first */ 2218 blob->state = SPDK_BLOB_STATE_CLEAN; 2219 blob_persist_write_page_chain(seq, ctx); 2220 } 2221 2222 static void 2223 blob_persist_write_extent_pages(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 2224 { 2225 struct spdk_blob_persist_ctx *ctx = cb_arg; 2226 struct spdk_blob *blob = ctx->blob; 2227 size_t i; 2228 uint32_t extent_page_id; 2229 uint32_t page_count = 0; 2230 int rc; 2231 2232 if (ctx->extent_page != NULL) { 2233 spdk_free(ctx->extent_page); 2234 ctx->extent_page = NULL; 2235 } 2236 2237 if (bserrno != 0) { 2238 blob_persist_complete(seq, ctx, bserrno); 2239 return; 2240 } 2241 2242 /* Only write out Extent Pages when blob was resized. */ 2243 for (i = ctx->next_extent_page; i < blob->active.extent_pages_array_size; i++) { 2244 extent_page_id = blob->active.extent_pages[i]; 2245 if (extent_page_id == 0) { 2246 /* No Extent Page to persist */ 2247 assert(spdk_blob_is_thin_provisioned(blob)); 2248 continue; 2249 } 2250 assert(spdk_bit_array_get(blob->bs->used_md_pages, extent_page_id)); 2251 ctx->next_extent_page = i + 1; 2252 rc = blob_serialize_add_page(ctx->blob, &ctx->extent_page, &page_count, &ctx->extent_page); 2253 if (rc < 0) { 2254 blob_persist_complete(seq, ctx, rc); 2255 return; 2256 } 2257 2258 blob->state = SPDK_BLOB_STATE_DIRTY; 2259 blob_serialize_extent_page(blob, i * SPDK_EXTENTS_PER_EP, ctx->extent_page); 2260 2261 ctx->extent_page->crc = blob_md_page_calc_crc(ctx->extent_page); 2262 2263 bs_sequence_write_dev(seq, ctx->extent_page, bs_md_page_to_lba(blob->bs, extent_page_id), 2264 bs_byte_to_lba(blob->bs, SPDK_BS_PAGE_SIZE), 2265 blob_persist_write_extent_pages, ctx); 2266 return; 2267 } 2268 2269 blob_persist_generate_new_md(ctx); 2270 } 2271 2272 static void 2273 blob_persist_start(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 2274 { 2275 struct spdk_blob_persist_ctx *ctx = cb_arg; 2276 struct spdk_blob *blob = ctx->blob; 2277 2278 if (bserrno != 0) { 2279 blob_persist_complete(seq, ctx, bserrno); 2280 return; 2281 } 2282 2283 if (blob->active.num_pages == 0) { 2284 /* This is the signal that the blob should be deleted. 2285 * Immediately jump to the clean up routine. */ 2286 assert(blob->clean.num_pages > 0); 2287 blob->state = SPDK_BLOB_STATE_CLEAN; 2288 blob_persist_zero_pages(seq, ctx, 0); 2289 return; 2290 2291 } 2292 2293 if (blob->clean.num_clusters < blob->active.num_clusters) { 2294 /* Blob was resized up */ 2295 assert(blob->clean.num_extent_pages <= blob->active.num_extent_pages); 2296 ctx->next_extent_page = spdk_max(1, blob->clean.num_extent_pages) - 1; 2297 } else if (blob->active.num_clusters < blob->active.cluster_array_size) { 2298 /* Blob was resized down */ 2299 assert(blob->clean.num_extent_pages >= blob->active.num_extent_pages); 2300 ctx->next_extent_page = spdk_max(1, blob->active.num_extent_pages) - 1; 2301 } else { 2302 /* No change in size occurred */ 2303 blob_persist_generate_new_md(ctx); 2304 return; 2305 } 2306 2307 blob_persist_write_extent_pages(seq, ctx, 0); 2308 } 2309 2310 struct spdk_bs_mark_dirty { 2311 struct spdk_blob_store *bs; 2312 struct spdk_bs_super_block *super; 2313 spdk_bs_sequence_cpl cb_fn; 2314 void *cb_arg; 2315 }; 2316 2317 static void 2318 bs_mark_dirty_write_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 2319 { 2320 struct spdk_bs_mark_dirty *ctx = cb_arg; 2321 2322 if (bserrno == 0) { 2323 ctx->bs->clean = 0; 2324 } 2325 2326 ctx->cb_fn(seq, ctx->cb_arg, bserrno); 2327 2328 spdk_free(ctx->super); 2329 free(ctx); 2330 } 2331 2332 static void bs_write_super(spdk_bs_sequence_t *seq, struct spdk_blob_store *bs, 2333 struct spdk_bs_super_block *super, spdk_bs_sequence_cpl cb_fn, void *cb_arg); 2334 2335 2336 static void 2337 bs_mark_dirty_write(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 2338 { 2339 struct spdk_bs_mark_dirty *ctx = cb_arg; 2340 2341 if (bserrno != 0) { 2342 bs_mark_dirty_write_cpl(seq, ctx, bserrno); 2343 return; 2344 } 2345 2346 ctx->super->clean = 0; 2347 if (ctx->super->size == 0) { 2348 ctx->super->size = ctx->bs->dev->blockcnt * ctx->bs->dev->blocklen; 2349 } 2350 2351 bs_write_super(seq, ctx->bs, ctx->super, bs_mark_dirty_write_cpl, ctx); 2352 } 2353 2354 static void 2355 bs_mark_dirty(spdk_bs_sequence_t *seq, struct spdk_blob_store *bs, 2356 spdk_bs_sequence_cpl cb_fn, void *cb_arg) 2357 { 2358 struct spdk_bs_mark_dirty *ctx; 2359 2360 /* Blobstore is already marked dirty */ 2361 if (bs->clean == 0) { 2362 cb_fn(seq, cb_arg, 0); 2363 return; 2364 } 2365 2366 ctx = calloc(1, sizeof(*ctx)); 2367 if (!ctx) { 2368 cb_fn(seq, cb_arg, -ENOMEM); 2369 return; 2370 } 2371 ctx->bs = bs; 2372 ctx->cb_fn = cb_fn; 2373 ctx->cb_arg = cb_arg; 2374 2375 ctx->super = spdk_zmalloc(sizeof(*ctx->super), 0x1000, NULL, 2376 SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA); 2377 if (!ctx->super) { 2378 free(ctx); 2379 cb_fn(seq, cb_arg, -ENOMEM); 2380 return; 2381 } 2382 2383 bs_sequence_read_dev(seq, ctx->super, bs_page_to_lba(bs, 0), 2384 bs_byte_to_lba(bs, sizeof(*ctx->super)), 2385 bs_mark_dirty_write, ctx); 2386 } 2387 2388 /* Write a blob to disk */ 2389 static void 2390 blob_persist(spdk_bs_sequence_t *seq, struct spdk_blob *blob, 2391 spdk_bs_sequence_cpl cb_fn, void *cb_arg) 2392 { 2393 struct spdk_blob_persist_ctx *ctx; 2394 2395 blob_verify_md_op(blob); 2396 2397 if (blob->state == SPDK_BLOB_STATE_CLEAN && TAILQ_EMPTY(&blob->persists_to_complete)) { 2398 cb_fn(seq, cb_arg, 0); 2399 return; 2400 } 2401 2402 ctx = calloc(1, sizeof(*ctx)); 2403 if (!ctx) { 2404 cb_fn(seq, cb_arg, -ENOMEM); 2405 return; 2406 } 2407 ctx->blob = blob; 2408 ctx->seq = seq; 2409 ctx->cb_fn = cb_fn; 2410 ctx->cb_arg = cb_arg; 2411 2412 /* Multiple blob persists can affect one another, via blob->state or 2413 * blob mutable data changes. To prevent it, queue up the persists. */ 2414 if (!TAILQ_EMPTY(&blob->persists_to_complete)) { 2415 TAILQ_INSERT_TAIL(&blob->pending_persists, ctx, link); 2416 return; 2417 } 2418 TAILQ_INSERT_HEAD(&blob->persists_to_complete, ctx, link); 2419 2420 bs_mark_dirty(seq, blob->bs, blob_persist_start, ctx); 2421 } 2422 2423 struct spdk_blob_copy_cluster_ctx { 2424 struct spdk_blob *blob; 2425 uint8_t *buf; 2426 uint64_t page; 2427 uint64_t new_cluster; 2428 uint32_t new_extent_page; 2429 spdk_bs_sequence_t *seq; 2430 struct spdk_blob_md_page *new_cluster_page; 2431 }; 2432 2433 static void 2434 blob_allocate_and_copy_cluster_cpl(void *cb_arg, int bserrno) 2435 { 2436 struct spdk_blob_copy_cluster_ctx *ctx = cb_arg; 2437 struct spdk_bs_request_set *set = (struct spdk_bs_request_set *)ctx->seq; 2438 TAILQ_HEAD(, spdk_bs_request_set) requests; 2439 spdk_bs_user_op_t *op; 2440 2441 TAILQ_INIT(&requests); 2442 TAILQ_SWAP(&set->channel->need_cluster_alloc, &requests, spdk_bs_request_set, link); 2443 2444 while (!TAILQ_EMPTY(&requests)) { 2445 op = TAILQ_FIRST(&requests); 2446 TAILQ_REMOVE(&requests, op, link); 2447 if (bserrno == 0) { 2448 bs_user_op_execute(op); 2449 } else { 2450 bs_user_op_abort(op, bserrno); 2451 } 2452 } 2453 2454 spdk_free(ctx->buf); 2455 free(ctx); 2456 } 2457 2458 static void 2459 blob_insert_cluster_cpl(void *cb_arg, int bserrno) 2460 { 2461 struct spdk_blob_copy_cluster_ctx *ctx = cb_arg; 2462 2463 if (bserrno) { 2464 if (bserrno == -EEXIST) { 2465 /* The metadata insert failed because another thread 2466 * allocated the cluster first. Free our cluster 2467 * but continue without error. */ 2468 bserrno = 0; 2469 } 2470 spdk_spin_lock(&ctx->blob->bs->used_lock); 2471 bs_release_cluster(ctx->blob->bs, ctx->new_cluster); 2472 if (ctx->new_extent_page != 0) { 2473 bs_release_md_page(ctx->blob->bs, ctx->new_extent_page); 2474 } 2475 spdk_spin_unlock(&ctx->blob->bs->used_lock); 2476 } 2477 2478 bs_sequence_finish(ctx->seq, bserrno); 2479 } 2480 2481 static void 2482 blob_write_copy_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 2483 { 2484 struct spdk_blob_copy_cluster_ctx *ctx = cb_arg; 2485 uint32_t cluster_number; 2486 2487 if (bserrno) { 2488 /* The write failed, so jump to the final completion handler */ 2489 bs_sequence_finish(seq, bserrno); 2490 return; 2491 } 2492 2493 cluster_number = bs_page_to_cluster(ctx->blob->bs, ctx->page); 2494 2495 blob_insert_cluster_on_md_thread(ctx->blob, cluster_number, ctx->new_cluster, 2496 ctx->new_extent_page, ctx->new_cluster_page, blob_insert_cluster_cpl, ctx); 2497 } 2498 2499 static void 2500 blob_write_copy(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 2501 { 2502 struct spdk_blob_copy_cluster_ctx *ctx = cb_arg; 2503 2504 if (bserrno != 0) { 2505 /* The read failed, so jump to the final completion handler */ 2506 bs_sequence_finish(seq, bserrno); 2507 return; 2508 } 2509 2510 /* Write whole cluster */ 2511 bs_sequence_write_dev(seq, ctx->buf, 2512 bs_cluster_to_lba(ctx->blob->bs, ctx->new_cluster), 2513 bs_cluster_to_lba(ctx->blob->bs, 1), 2514 blob_write_copy_cpl, ctx); 2515 } 2516 2517 static bool 2518 blob_can_copy(struct spdk_blob *blob, uint32_t cluster_start_page, uint64_t *base_lba) 2519 { 2520 uint64_t lba = bs_dev_page_to_lba(blob->back_bs_dev, cluster_start_page); 2521 2522 return (blob->bs->dev->copy != NULL) && 2523 blob->back_bs_dev->translate_lba(blob->back_bs_dev, lba, base_lba); 2524 } 2525 2526 static void 2527 blob_copy(struct spdk_blob_copy_cluster_ctx *ctx, spdk_bs_user_op_t *op, uint64_t src_lba) 2528 { 2529 struct spdk_blob *blob = ctx->blob; 2530 uint64_t lba_count = bs_dev_byte_to_lba(blob->back_bs_dev, blob->bs->cluster_sz); 2531 2532 bs_sequence_copy_dev(ctx->seq, 2533 bs_cluster_to_lba(blob->bs, ctx->new_cluster), 2534 src_lba, 2535 lba_count, 2536 blob_write_copy_cpl, ctx); 2537 } 2538 2539 static void 2540 bs_allocate_and_copy_cluster(struct spdk_blob *blob, 2541 struct spdk_io_channel *_ch, 2542 uint64_t io_unit, spdk_bs_user_op_t *op) 2543 { 2544 struct spdk_bs_cpl cpl; 2545 struct spdk_bs_channel *ch; 2546 struct spdk_blob_copy_cluster_ctx *ctx; 2547 uint32_t cluster_start_page; 2548 uint32_t cluster_number; 2549 bool is_zeroes; 2550 bool can_copy; 2551 uint64_t copy_src_lba; 2552 int rc; 2553 2554 ch = spdk_io_channel_get_ctx(_ch); 2555 2556 if (!TAILQ_EMPTY(&ch->need_cluster_alloc)) { 2557 /* There are already operations pending. Queue this user op 2558 * and return because it will be re-executed when the outstanding 2559 * cluster allocation completes. */ 2560 TAILQ_INSERT_TAIL(&ch->need_cluster_alloc, op, link); 2561 return; 2562 } 2563 2564 /* Round the io_unit offset down to the first page in the cluster */ 2565 cluster_start_page = bs_io_unit_to_cluster_start(blob, io_unit); 2566 2567 /* Calculate which index in the metadata cluster array the corresponding 2568 * cluster is supposed to be at. */ 2569 cluster_number = bs_io_unit_to_cluster_number(blob, io_unit); 2570 2571 ctx = calloc(1, sizeof(*ctx)); 2572 if (!ctx) { 2573 bs_user_op_abort(op, -ENOMEM); 2574 return; 2575 } 2576 2577 assert(blob->bs->cluster_sz % blob->back_bs_dev->blocklen == 0); 2578 2579 ctx->blob = blob; 2580 ctx->page = cluster_start_page; 2581 ctx->new_cluster_page = ch->new_cluster_page; 2582 memset(ctx->new_cluster_page, 0, SPDK_BS_PAGE_SIZE); 2583 can_copy = blob_can_copy(blob, cluster_start_page, ©_src_lba); 2584 2585 is_zeroes = blob->back_bs_dev->is_zeroes(blob->back_bs_dev, 2586 bs_dev_page_to_lba(blob->back_bs_dev, cluster_start_page), 2587 bs_dev_byte_to_lba(blob->back_bs_dev, blob->bs->cluster_sz)); 2588 if (blob->parent_id != SPDK_BLOBID_INVALID && !is_zeroes && !can_copy) { 2589 ctx->buf = spdk_malloc(blob->bs->cluster_sz, blob->back_bs_dev->blocklen, 2590 NULL, SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA); 2591 if (!ctx->buf) { 2592 SPDK_ERRLOG("DMA allocation for cluster of size = %" PRIu32 " failed.\n", 2593 blob->bs->cluster_sz); 2594 free(ctx); 2595 bs_user_op_abort(op, -ENOMEM); 2596 return; 2597 } 2598 } 2599 2600 spdk_spin_lock(&blob->bs->used_lock); 2601 rc = bs_allocate_cluster(blob, cluster_number, &ctx->new_cluster, &ctx->new_extent_page, 2602 false); 2603 spdk_spin_unlock(&blob->bs->used_lock); 2604 if (rc != 0) { 2605 spdk_free(ctx->buf); 2606 free(ctx); 2607 bs_user_op_abort(op, rc); 2608 return; 2609 } 2610 2611 cpl.type = SPDK_BS_CPL_TYPE_BLOB_BASIC; 2612 cpl.u.blob_basic.cb_fn = blob_allocate_and_copy_cluster_cpl; 2613 cpl.u.blob_basic.cb_arg = ctx; 2614 2615 ctx->seq = bs_sequence_start(_ch, &cpl); 2616 if (!ctx->seq) { 2617 spdk_spin_lock(&blob->bs->used_lock); 2618 bs_release_cluster(blob->bs, ctx->new_cluster); 2619 spdk_spin_unlock(&blob->bs->used_lock); 2620 spdk_free(ctx->buf); 2621 free(ctx); 2622 bs_user_op_abort(op, -ENOMEM); 2623 return; 2624 } 2625 2626 /* Queue the user op to block other incoming operations */ 2627 TAILQ_INSERT_TAIL(&ch->need_cluster_alloc, op, link); 2628 2629 if (blob->parent_id != SPDK_BLOBID_INVALID && !is_zeroes) { 2630 if (can_copy) { 2631 blob_copy(ctx, op, copy_src_lba); 2632 } else { 2633 /* Read cluster from backing device */ 2634 bs_sequence_read_bs_dev(ctx->seq, blob->back_bs_dev, ctx->buf, 2635 bs_dev_page_to_lba(blob->back_bs_dev, cluster_start_page), 2636 bs_dev_byte_to_lba(blob->back_bs_dev, blob->bs->cluster_sz), 2637 blob_write_copy, ctx); 2638 } 2639 2640 } else { 2641 blob_insert_cluster_on_md_thread(ctx->blob, cluster_number, ctx->new_cluster, 2642 ctx->new_extent_page, ctx->new_cluster_page, blob_insert_cluster_cpl, ctx); 2643 } 2644 } 2645 2646 static inline bool 2647 blob_calculate_lba_and_lba_count(struct spdk_blob *blob, uint64_t io_unit, uint64_t length, 2648 uint64_t *lba, uint64_t *lba_count) 2649 { 2650 *lba_count = length; 2651 2652 if (!bs_io_unit_is_allocated(blob, io_unit)) { 2653 assert(blob->back_bs_dev != NULL); 2654 *lba = bs_io_unit_to_back_dev_lba(blob, io_unit); 2655 *lba_count = bs_io_unit_to_back_dev_lba(blob, *lba_count); 2656 return false; 2657 } else { 2658 *lba = bs_blob_io_unit_to_lba(blob, io_unit); 2659 return true; 2660 } 2661 } 2662 2663 struct op_split_ctx { 2664 struct spdk_blob *blob; 2665 struct spdk_io_channel *channel; 2666 uint64_t io_unit_offset; 2667 uint64_t io_units_remaining; 2668 void *curr_payload; 2669 enum spdk_blob_op_type op_type; 2670 spdk_bs_sequence_t *seq; 2671 bool in_submit_ctx; 2672 bool completed_in_submit_ctx; 2673 bool done; 2674 }; 2675 2676 static void 2677 blob_request_submit_op_split_next(void *cb_arg, int bserrno) 2678 { 2679 struct op_split_ctx *ctx = cb_arg; 2680 struct spdk_blob *blob = ctx->blob; 2681 struct spdk_io_channel *ch = ctx->channel; 2682 enum spdk_blob_op_type op_type = ctx->op_type; 2683 uint8_t *buf; 2684 uint64_t offset; 2685 uint64_t length; 2686 uint64_t op_length; 2687 2688 if (bserrno != 0 || ctx->io_units_remaining == 0) { 2689 bs_sequence_finish(ctx->seq, bserrno); 2690 if (ctx->in_submit_ctx) { 2691 /* Defer freeing of the ctx object, since it will be 2692 * accessed when this unwinds back to the submisison 2693 * context. 2694 */ 2695 ctx->done = true; 2696 } else { 2697 free(ctx); 2698 } 2699 return; 2700 } 2701 2702 if (ctx->in_submit_ctx) { 2703 /* If this split operation completed in the context 2704 * of its submission, mark the flag and return immediately 2705 * to avoid recursion. 2706 */ 2707 ctx->completed_in_submit_ctx = true; 2708 return; 2709 } 2710 2711 while (true) { 2712 ctx->completed_in_submit_ctx = false; 2713 2714 offset = ctx->io_unit_offset; 2715 length = ctx->io_units_remaining; 2716 buf = ctx->curr_payload; 2717 op_length = spdk_min(length, bs_num_io_units_to_cluster_boundary(blob, 2718 offset)); 2719 2720 /* Update length and payload for next operation */ 2721 ctx->io_units_remaining -= op_length; 2722 ctx->io_unit_offset += op_length; 2723 if (op_type == SPDK_BLOB_WRITE || op_type == SPDK_BLOB_READ) { 2724 ctx->curr_payload += op_length * blob->bs->io_unit_size; 2725 } 2726 2727 assert(!ctx->in_submit_ctx); 2728 ctx->in_submit_ctx = true; 2729 2730 switch (op_type) { 2731 case SPDK_BLOB_READ: 2732 spdk_blob_io_read(blob, ch, buf, offset, op_length, 2733 blob_request_submit_op_split_next, ctx); 2734 break; 2735 case SPDK_BLOB_WRITE: 2736 spdk_blob_io_write(blob, ch, buf, offset, op_length, 2737 blob_request_submit_op_split_next, ctx); 2738 break; 2739 case SPDK_BLOB_UNMAP: 2740 spdk_blob_io_unmap(blob, ch, offset, op_length, 2741 blob_request_submit_op_split_next, ctx); 2742 break; 2743 case SPDK_BLOB_WRITE_ZEROES: 2744 spdk_blob_io_write_zeroes(blob, ch, offset, op_length, 2745 blob_request_submit_op_split_next, ctx); 2746 break; 2747 case SPDK_BLOB_READV: 2748 case SPDK_BLOB_WRITEV: 2749 SPDK_ERRLOG("readv/write not valid\n"); 2750 bs_sequence_finish(ctx->seq, -EINVAL); 2751 free(ctx); 2752 return; 2753 } 2754 2755 #ifndef __clang_analyzer__ 2756 /* scan-build reports a false positive around accessing the ctx here. It 2757 * forms a path that recursively calls this function, but then says 2758 * "assuming ctx->in_submit_ctx is false", when that isn't possible. 2759 * This path does free(ctx), returns to here, and reports a use-after-free 2760 * bug. Wrapping this bit of code so that scan-build doesn't see it 2761 * works around the scan-build bug. 2762 */ 2763 assert(ctx->in_submit_ctx); 2764 ctx->in_submit_ctx = false; 2765 2766 /* If the operation completed immediately, loop back and submit the 2767 * next operation. Otherwise we can return and the next split 2768 * operation will get submitted when this current operation is 2769 * later completed asynchronously. 2770 */ 2771 if (ctx->completed_in_submit_ctx) { 2772 continue; 2773 } else if (ctx->done) { 2774 free(ctx); 2775 } 2776 #endif 2777 break; 2778 } 2779 } 2780 2781 static void 2782 blob_request_submit_op_split(struct spdk_io_channel *ch, struct spdk_blob *blob, 2783 void *payload, uint64_t offset, uint64_t length, 2784 spdk_blob_op_complete cb_fn, void *cb_arg, enum spdk_blob_op_type op_type) 2785 { 2786 struct op_split_ctx *ctx; 2787 spdk_bs_sequence_t *seq; 2788 struct spdk_bs_cpl cpl; 2789 2790 assert(blob != NULL); 2791 2792 ctx = calloc(1, sizeof(struct op_split_ctx)); 2793 if (ctx == NULL) { 2794 cb_fn(cb_arg, -ENOMEM); 2795 return; 2796 } 2797 2798 cpl.type = SPDK_BS_CPL_TYPE_BLOB_BASIC; 2799 cpl.u.blob_basic.cb_fn = cb_fn; 2800 cpl.u.blob_basic.cb_arg = cb_arg; 2801 2802 seq = bs_sequence_start(ch, &cpl); 2803 if (!seq) { 2804 free(ctx); 2805 cb_fn(cb_arg, -ENOMEM); 2806 return; 2807 } 2808 2809 ctx->blob = blob; 2810 ctx->channel = ch; 2811 ctx->curr_payload = payload; 2812 ctx->io_unit_offset = offset; 2813 ctx->io_units_remaining = length; 2814 ctx->op_type = op_type; 2815 ctx->seq = seq; 2816 2817 blob_request_submit_op_split_next(ctx, 0); 2818 } 2819 2820 static void 2821 blob_request_submit_op_single(struct spdk_io_channel *_ch, struct spdk_blob *blob, 2822 void *payload, uint64_t offset, uint64_t length, 2823 spdk_blob_op_complete cb_fn, void *cb_arg, enum spdk_blob_op_type op_type) 2824 { 2825 struct spdk_bs_cpl cpl; 2826 uint64_t lba; 2827 uint64_t lba_count; 2828 bool is_allocated; 2829 2830 assert(blob != NULL); 2831 2832 cpl.type = SPDK_BS_CPL_TYPE_BLOB_BASIC; 2833 cpl.u.blob_basic.cb_fn = cb_fn; 2834 cpl.u.blob_basic.cb_arg = cb_arg; 2835 2836 if (blob->frozen_refcnt) { 2837 /* This blob I/O is frozen */ 2838 spdk_bs_user_op_t *op; 2839 struct spdk_bs_channel *bs_channel = spdk_io_channel_get_ctx(_ch); 2840 2841 op = bs_user_op_alloc(_ch, &cpl, op_type, blob, payload, 0, offset, length); 2842 if (!op) { 2843 cb_fn(cb_arg, -ENOMEM); 2844 return; 2845 } 2846 2847 TAILQ_INSERT_TAIL(&bs_channel->queued_io, op, link); 2848 2849 return; 2850 } 2851 2852 is_allocated = blob_calculate_lba_and_lba_count(blob, offset, length, &lba, &lba_count); 2853 2854 switch (op_type) { 2855 case SPDK_BLOB_READ: { 2856 spdk_bs_batch_t *batch; 2857 2858 batch = bs_batch_open(_ch, &cpl); 2859 if (!batch) { 2860 cb_fn(cb_arg, -ENOMEM); 2861 return; 2862 } 2863 2864 if (is_allocated) { 2865 /* Read from the blob */ 2866 bs_batch_read_dev(batch, payload, lba, lba_count); 2867 } else { 2868 /* Read from the backing block device */ 2869 bs_batch_read_bs_dev(batch, blob->back_bs_dev, payload, lba, lba_count); 2870 } 2871 2872 bs_batch_close(batch); 2873 break; 2874 } 2875 case SPDK_BLOB_WRITE: 2876 case SPDK_BLOB_WRITE_ZEROES: { 2877 if (is_allocated) { 2878 /* Write to the blob */ 2879 spdk_bs_batch_t *batch; 2880 2881 if (lba_count == 0) { 2882 cb_fn(cb_arg, 0); 2883 return; 2884 } 2885 2886 batch = bs_batch_open(_ch, &cpl); 2887 if (!batch) { 2888 cb_fn(cb_arg, -ENOMEM); 2889 return; 2890 } 2891 2892 if (op_type == SPDK_BLOB_WRITE) { 2893 bs_batch_write_dev(batch, payload, lba, lba_count); 2894 } else { 2895 bs_batch_write_zeroes_dev(batch, lba, lba_count); 2896 } 2897 2898 bs_batch_close(batch); 2899 } else { 2900 /* Queue this operation and allocate the cluster */ 2901 spdk_bs_user_op_t *op; 2902 2903 op = bs_user_op_alloc(_ch, &cpl, op_type, blob, payload, 0, offset, length); 2904 if (!op) { 2905 cb_fn(cb_arg, -ENOMEM); 2906 return; 2907 } 2908 2909 bs_allocate_and_copy_cluster(blob, _ch, offset, op); 2910 } 2911 break; 2912 } 2913 case SPDK_BLOB_UNMAP: { 2914 spdk_bs_batch_t *batch; 2915 2916 batch = bs_batch_open(_ch, &cpl); 2917 if (!batch) { 2918 cb_fn(cb_arg, -ENOMEM); 2919 return; 2920 } 2921 2922 if (is_allocated) { 2923 bs_batch_unmap_dev(batch, lba, lba_count); 2924 } 2925 2926 bs_batch_close(batch); 2927 break; 2928 } 2929 case SPDK_BLOB_READV: 2930 case SPDK_BLOB_WRITEV: 2931 SPDK_ERRLOG("readv/write not valid\n"); 2932 cb_fn(cb_arg, -EINVAL); 2933 break; 2934 } 2935 } 2936 2937 static void 2938 blob_request_submit_op(struct spdk_blob *blob, struct spdk_io_channel *_channel, 2939 void *payload, uint64_t offset, uint64_t length, 2940 spdk_blob_op_complete cb_fn, void *cb_arg, enum spdk_blob_op_type op_type) 2941 { 2942 assert(blob != NULL); 2943 2944 if (blob->data_ro && op_type != SPDK_BLOB_READ) { 2945 cb_fn(cb_arg, -EPERM); 2946 return; 2947 } 2948 2949 if (length == 0) { 2950 cb_fn(cb_arg, 0); 2951 return; 2952 } 2953 2954 if (offset + length > bs_cluster_to_lba(blob->bs, blob->active.num_clusters)) { 2955 cb_fn(cb_arg, -EINVAL); 2956 return; 2957 } 2958 if (length <= bs_num_io_units_to_cluster_boundary(blob, offset)) { 2959 blob_request_submit_op_single(_channel, blob, payload, offset, length, 2960 cb_fn, cb_arg, op_type); 2961 } else { 2962 blob_request_submit_op_split(_channel, blob, payload, offset, length, 2963 cb_fn, cb_arg, op_type); 2964 } 2965 } 2966 2967 struct rw_iov_ctx { 2968 struct spdk_blob *blob; 2969 struct spdk_io_channel *channel; 2970 spdk_blob_op_complete cb_fn; 2971 void *cb_arg; 2972 bool read; 2973 int iovcnt; 2974 struct iovec *orig_iov; 2975 uint64_t io_unit_offset; 2976 uint64_t io_units_remaining; 2977 uint64_t io_units_done; 2978 struct spdk_blob_ext_io_opts *ext_io_opts; 2979 struct iovec iov[0]; 2980 }; 2981 2982 static void 2983 rw_iov_done(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 2984 { 2985 assert(cb_arg == NULL); 2986 bs_sequence_finish(seq, bserrno); 2987 } 2988 2989 static void 2990 rw_iov_split_next(void *cb_arg, int bserrno) 2991 { 2992 struct rw_iov_ctx *ctx = cb_arg; 2993 struct spdk_blob *blob = ctx->blob; 2994 struct iovec *iov, *orig_iov; 2995 int iovcnt; 2996 size_t orig_iovoff; 2997 uint64_t io_units_count, io_units_to_boundary, io_unit_offset; 2998 uint64_t byte_count; 2999 3000 if (bserrno != 0 || ctx->io_units_remaining == 0) { 3001 ctx->cb_fn(ctx->cb_arg, bserrno); 3002 free(ctx); 3003 return; 3004 } 3005 3006 io_unit_offset = ctx->io_unit_offset; 3007 io_units_to_boundary = bs_num_io_units_to_cluster_boundary(blob, io_unit_offset); 3008 io_units_count = spdk_min(ctx->io_units_remaining, io_units_to_boundary); 3009 /* 3010 * Get index and offset into the original iov array for our current position in the I/O sequence. 3011 * byte_count will keep track of how many bytes remaining until orig_iov and orig_iovoff will 3012 * point to the current position in the I/O sequence. 3013 */ 3014 byte_count = ctx->io_units_done * blob->bs->io_unit_size; 3015 orig_iov = &ctx->orig_iov[0]; 3016 orig_iovoff = 0; 3017 while (byte_count > 0) { 3018 if (byte_count >= orig_iov->iov_len) { 3019 byte_count -= orig_iov->iov_len; 3020 orig_iov++; 3021 } else { 3022 orig_iovoff = byte_count; 3023 byte_count = 0; 3024 } 3025 } 3026 3027 /* 3028 * Build an iov array for the next I/O in the sequence. byte_count will keep track of how many 3029 * bytes of this next I/O remain to be accounted for in the new iov array. 3030 */ 3031 byte_count = io_units_count * blob->bs->io_unit_size; 3032 iov = &ctx->iov[0]; 3033 iovcnt = 0; 3034 while (byte_count > 0) { 3035 assert(iovcnt < ctx->iovcnt); 3036 iov->iov_len = spdk_min(byte_count, orig_iov->iov_len - orig_iovoff); 3037 iov->iov_base = orig_iov->iov_base + orig_iovoff; 3038 byte_count -= iov->iov_len; 3039 orig_iovoff = 0; 3040 orig_iov++; 3041 iov++; 3042 iovcnt++; 3043 } 3044 3045 ctx->io_unit_offset += io_units_count; 3046 ctx->io_units_remaining -= io_units_count; 3047 ctx->io_units_done += io_units_count; 3048 iov = &ctx->iov[0]; 3049 3050 if (ctx->read) { 3051 spdk_blob_io_readv_ext(ctx->blob, ctx->channel, iov, iovcnt, io_unit_offset, 3052 io_units_count, rw_iov_split_next, ctx, ctx->ext_io_opts); 3053 } else { 3054 spdk_blob_io_writev_ext(ctx->blob, ctx->channel, iov, iovcnt, io_unit_offset, 3055 io_units_count, rw_iov_split_next, ctx, ctx->ext_io_opts); 3056 } 3057 } 3058 3059 static void 3060 blob_request_submit_rw_iov(struct spdk_blob *blob, struct spdk_io_channel *_channel, 3061 struct iovec *iov, int iovcnt, 3062 uint64_t offset, uint64_t length, spdk_blob_op_complete cb_fn, void *cb_arg, bool read, 3063 struct spdk_blob_ext_io_opts *ext_io_opts) 3064 { 3065 struct spdk_bs_cpl cpl; 3066 3067 assert(blob != NULL); 3068 3069 if (!read && blob->data_ro) { 3070 cb_fn(cb_arg, -EPERM); 3071 return; 3072 } 3073 3074 if (length == 0) { 3075 cb_fn(cb_arg, 0); 3076 return; 3077 } 3078 3079 if (offset + length > bs_cluster_to_lba(blob->bs, blob->active.num_clusters)) { 3080 cb_fn(cb_arg, -EINVAL); 3081 return; 3082 } 3083 3084 /* 3085 * For now, we implement readv/writev using a sequence (instead of a batch) to account for having 3086 * to split a request that spans a cluster boundary. For I/O that do not span a cluster boundary, 3087 * there will be no noticeable difference compared to using a batch. For I/O that do span a cluster 3088 * boundary, the target LBAs (after blob offset to LBA translation) may not be contiguous, so we need 3089 * to allocate a separate iov array and split the I/O such that none of the resulting 3090 * smaller I/O cross a cluster boundary. These smaller I/O will be issued in sequence (not in parallel) 3091 * but since this case happens very infrequently, any performance impact will be negligible. 3092 * 3093 * This could be optimized in the future to allocate a big enough iov array to account for all of the iovs 3094 * for all of the smaller I/Os, pre-build all of the iov arrays for the smaller I/Os, then issue them 3095 * in a batch. That would also require creating an intermediate spdk_bs_cpl that would get called 3096 * when the batch was completed, to allow for freeing the memory for the iov arrays. 3097 */ 3098 if (spdk_likely(length <= bs_num_io_units_to_cluster_boundary(blob, offset))) { 3099 uint64_t lba_count; 3100 uint64_t lba; 3101 bool is_allocated; 3102 3103 cpl.type = SPDK_BS_CPL_TYPE_BLOB_BASIC; 3104 cpl.u.blob_basic.cb_fn = cb_fn; 3105 cpl.u.blob_basic.cb_arg = cb_arg; 3106 3107 if (blob->frozen_refcnt) { 3108 /* This blob I/O is frozen */ 3109 enum spdk_blob_op_type op_type; 3110 spdk_bs_user_op_t *op; 3111 struct spdk_bs_channel *bs_channel = spdk_io_channel_get_ctx(_channel); 3112 3113 op_type = read ? SPDK_BLOB_READV : SPDK_BLOB_WRITEV; 3114 op = bs_user_op_alloc(_channel, &cpl, op_type, blob, iov, iovcnt, offset, length); 3115 if (!op) { 3116 cb_fn(cb_arg, -ENOMEM); 3117 return; 3118 } 3119 3120 TAILQ_INSERT_TAIL(&bs_channel->queued_io, op, link); 3121 3122 return; 3123 } 3124 3125 is_allocated = blob_calculate_lba_and_lba_count(blob, offset, length, &lba, &lba_count); 3126 3127 if (read) { 3128 spdk_bs_sequence_t *seq; 3129 3130 seq = bs_sequence_start(_channel, &cpl); 3131 if (!seq) { 3132 cb_fn(cb_arg, -ENOMEM); 3133 return; 3134 } 3135 3136 seq->ext_io_opts = ext_io_opts; 3137 3138 if (is_allocated) { 3139 bs_sequence_readv_dev(seq, iov, iovcnt, lba, lba_count, rw_iov_done, NULL); 3140 } else { 3141 bs_sequence_readv_bs_dev(seq, blob->back_bs_dev, iov, iovcnt, lba, lba_count, 3142 rw_iov_done, NULL); 3143 } 3144 } else { 3145 if (is_allocated) { 3146 spdk_bs_sequence_t *seq; 3147 3148 seq = bs_sequence_start(_channel, &cpl); 3149 if (!seq) { 3150 cb_fn(cb_arg, -ENOMEM); 3151 return; 3152 } 3153 3154 seq->ext_io_opts = ext_io_opts; 3155 3156 bs_sequence_writev_dev(seq, iov, iovcnt, lba, lba_count, rw_iov_done, NULL); 3157 } else { 3158 /* Queue this operation and allocate the cluster */ 3159 spdk_bs_user_op_t *op; 3160 3161 op = bs_user_op_alloc(_channel, &cpl, SPDK_BLOB_WRITEV, blob, iov, iovcnt, offset, 3162 length); 3163 if (!op) { 3164 cb_fn(cb_arg, -ENOMEM); 3165 return; 3166 } 3167 3168 op->ext_io_opts = ext_io_opts; 3169 3170 bs_allocate_and_copy_cluster(blob, _channel, offset, op); 3171 } 3172 } 3173 } else { 3174 struct rw_iov_ctx *ctx; 3175 3176 ctx = calloc(1, sizeof(struct rw_iov_ctx) + iovcnt * sizeof(struct iovec)); 3177 if (ctx == NULL) { 3178 cb_fn(cb_arg, -ENOMEM); 3179 return; 3180 } 3181 3182 ctx->blob = blob; 3183 ctx->channel = _channel; 3184 ctx->cb_fn = cb_fn; 3185 ctx->cb_arg = cb_arg; 3186 ctx->read = read; 3187 ctx->orig_iov = iov; 3188 ctx->iovcnt = iovcnt; 3189 ctx->io_unit_offset = offset; 3190 ctx->io_units_remaining = length; 3191 ctx->io_units_done = 0; 3192 ctx->ext_io_opts = ext_io_opts; 3193 3194 rw_iov_split_next(ctx, 0); 3195 } 3196 } 3197 3198 static struct spdk_blob * 3199 blob_lookup(struct spdk_blob_store *bs, spdk_blob_id blobid) 3200 { 3201 struct spdk_blob find; 3202 3203 if (spdk_bit_array_get(bs->open_blobids, blobid) == 0) { 3204 return NULL; 3205 } 3206 3207 find.id = blobid; 3208 return RB_FIND(spdk_blob_tree, &bs->open_blobs, &find); 3209 } 3210 3211 static void 3212 blob_get_snapshot_and_clone_entries(struct spdk_blob *blob, 3213 struct spdk_blob_list **snapshot_entry, struct spdk_blob_list **clone_entry) 3214 { 3215 assert(blob != NULL); 3216 *snapshot_entry = NULL; 3217 *clone_entry = NULL; 3218 3219 if (blob->parent_id == SPDK_BLOBID_INVALID) { 3220 return; 3221 } 3222 3223 TAILQ_FOREACH(*snapshot_entry, &blob->bs->snapshots, link) { 3224 if ((*snapshot_entry)->id == blob->parent_id) { 3225 break; 3226 } 3227 } 3228 3229 if (*snapshot_entry != NULL) { 3230 TAILQ_FOREACH(*clone_entry, &(*snapshot_entry)->clones, link) { 3231 if ((*clone_entry)->id == blob->id) { 3232 break; 3233 } 3234 } 3235 3236 assert(*clone_entry != NULL); 3237 } 3238 } 3239 3240 static int 3241 bs_channel_create(void *io_device, void *ctx_buf) 3242 { 3243 struct spdk_blob_store *bs = io_device; 3244 struct spdk_bs_channel *channel = ctx_buf; 3245 struct spdk_bs_dev *dev; 3246 uint32_t max_ops = bs->max_channel_ops; 3247 uint32_t i; 3248 3249 dev = bs->dev; 3250 3251 channel->req_mem = calloc(max_ops, sizeof(struct spdk_bs_request_set)); 3252 if (!channel->req_mem) { 3253 return -1; 3254 } 3255 3256 TAILQ_INIT(&channel->reqs); 3257 3258 for (i = 0; i < max_ops; i++) { 3259 TAILQ_INSERT_TAIL(&channel->reqs, &channel->req_mem[i], link); 3260 } 3261 3262 channel->bs = bs; 3263 channel->dev = dev; 3264 channel->dev_channel = dev->create_channel(dev); 3265 3266 if (!channel->dev_channel) { 3267 SPDK_ERRLOG("Failed to create device channel.\n"); 3268 free(channel->req_mem); 3269 return -1; 3270 } 3271 3272 channel->new_cluster_page = spdk_zmalloc(SPDK_BS_PAGE_SIZE, 0, NULL, SPDK_ENV_SOCKET_ID_ANY, 3273 SPDK_MALLOC_DMA); 3274 if (!channel->new_cluster_page) { 3275 SPDK_ERRLOG("Failed to allocate new cluster page\n"); 3276 free(channel->req_mem); 3277 channel->dev->destroy_channel(channel->dev, channel->dev_channel); 3278 return -1; 3279 } 3280 3281 TAILQ_INIT(&channel->need_cluster_alloc); 3282 TAILQ_INIT(&channel->queued_io); 3283 3284 return 0; 3285 } 3286 3287 static void 3288 bs_channel_destroy(void *io_device, void *ctx_buf) 3289 { 3290 struct spdk_bs_channel *channel = ctx_buf; 3291 spdk_bs_user_op_t *op; 3292 3293 while (!TAILQ_EMPTY(&channel->need_cluster_alloc)) { 3294 op = TAILQ_FIRST(&channel->need_cluster_alloc); 3295 TAILQ_REMOVE(&channel->need_cluster_alloc, op, link); 3296 bs_user_op_abort(op, -EIO); 3297 } 3298 3299 while (!TAILQ_EMPTY(&channel->queued_io)) { 3300 op = TAILQ_FIRST(&channel->queued_io); 3301 TAILQ_REMOVE(&channel->queued_io, op, link); 3302 bs_user_op_abort(op, -EIO); 3303 } 3304 3305 free(channel->req_mem); 3306 spdk_free(channel->new_cluster_page); 3307 channel->dev->destroy_channel(channel->dev, channel->dev_channel); 3308 } 3309 3310 static void 3311 bs_dev_destroy(void *io_device) 3312 { 3313 struct spdk_blob_store *bs = io_device; 3314 struct spdk_blob *blob, *blob_tmp; 3315 3316 bs->dev->destroy(bs->dev); 3317 3318 RB_FOREACH_SAFE(blob, spdk_blob_tree, &bs->open_blobs, blob_tmp) { 3319 RB_REMOVE(spdk_blob_tree, &bs->open_blobs, blob); 3320 spdk_bit_array_clear(bs->open_blobids, blob->id); 3321 blob_free(blob); 3322 } 3323 3324 spdk_spin_destroy(&bs->used_lock); 3325 3326 spdk_bit_array_free(&bs->open_blobids); 3327 spdk_bit_array_free(&bs->used_blobids); 3328 spdk_bit_array_free(&bs->used_md_pages); 3329 spdk_bit_pool_free(&bs->used_clusters); 3330 /* 3331 * If this function is called for any reason except a successful unload, 3332 * the unload_cpl type will be NONE and this will be a nop. 3333 */ 3334 bs_call_cpl(&bs->unload_cpl, bs->unload_err); 3335 3336 free(bs); 3337 } 3338 3339 static int 3340 bs_blob_list_add(struct spdk_blob *blob) 3341 { 3342 spdk_blob_id snapshot_id; 3343 struct spdk_blob_list *snapshot_entry = NULL; 3344 struct spdk_blob_list *clone_entry = NULL; 3345 3346 assert(blob != NULL); 3347 3348 snapshot_id = blob->parent_id; 3349 if (snapshot_id == SPDK_BLOBID_INVALID || 3350 snapshot_id == SPDK_BLOBID_EXTERNAL_SNAPSHOT) { 3351 return 0; 3352 } 3353 3354 snapshot_entry = bs_get_snapshot_entry(blob->bs, snapshot_id); 3355 if (snapshot_entry == NULL) { 3356 /* Snapshot not found */ 3357 snapshot_entry = calloc(1, sizeof(struct spdk_blob_list)); 3358 if (snapshot_entry == NULL) { 3359 return -ENOMEM; 3360 } 3361 snapshot_entry->id = snapshot_id; 3362 TAILQ_INIT(&snapshot_entry->clones); 3363 TAILQ_INSERT_TAIL(&blob->bs->snapshots, snapshot_entry, link); 3364 } else { 3365 TAILQ_FOREACH(clone_entry, &snapshot_entry->clones, link) { 3366 if (clone_entry->id == blob->id) { 3367 break; 3368 } 3369 } 3370 } 3371 3372 if (clone_entry == NULL) { 3373 /* Clone not found */ 3374 clone_entry = calloc(1, sizeof(struct spdk_blob_list)); 3375 if (clone_entry == NULL) { 3376 return -ENOMEM; 3377 } 3378 clone_entry->id = blob->id; 3379 TAILQ_INIT(&clone_entry->clones); 3380 TAILQ_INSERT_TAIL(&snapshot_entry->clones, clone_entry, link); 3381 snapshot_entry->clone_count++; 3382 } 3383 3384 return 0; 3385 } 3386 3387 static void 3388 bs_blob_list_remove(struct spdk_blob *blob) 3389 { 3390 struct spdk_blob_list *snapshot_entry = NULL; 3391 struct spdk_blob_list *clone_entry = NULL; 3392 3393 blob_get_snapshot_and_clone_entries(blob, &snapshot_entry, &clone_entry); 3394 3395 if (snapshot_entry == NULL) { 3396 return; 3397 } 3398 3399 blob->parent_id = SPDK_BLOBID_INVALID; 3400 TAILQ_REMOVE(&snapshot_entry->clones, clone_entry, link); 3401 free(clone_entry); 3402 3403 snapshot_entry->clone_count--; 3404 } 3405 3406 static int 3407 bs_blob_list_free(struct spdk_blob_store *bs) 3408 { 3409 struct spdk_blob_list *snapshot_entry; 3410 struct spdk_blob_list *snapshot_entry_tmp; 3411 struct spdk_blob_list *clone_entry; 3412 struct spdk_blob_list *clone_entry_tmp; 3413 3414 TAILQ_FOREACH_SAFE(snapshot_entry, &bs->snapshots, link, snapshot_entry_tmp) { 3415 TAILQ_FOREACH_SAFE(clone_entry, &snapshot_entry->clones, link, clone_entry_tmp) { 3416 TAILQ_REMOVE(&snapshot_entry->clones, clone_entry, link); 3417 free(clone_entry); 3418 } 3419 TAILQ_REMOVE(&bs->snapshots, snapshot_entry, link); 3420 free(snapshot_entry); 3421 } 3422 3423 return 0; 3424 } 3425 3426 static void 3427 bs_free(struct spdk_blob_store *bs) 3428 { 3429 bs_blob_list_free(bs); 3430 3431 bs_unregister_md_thread(bs); 3432 spdk_io_device_unregister(bs, bs_dev_destroy); 3433 } 3434 3435 void 3436 spdk_bs_opts_init(struct spdk_bs_opts *opts, size_t opts_size) 3437 { 3438 3439 if (!opts) { 3440 SPDK_ERRLOG("opts should not be NULL\n"); 3441 return; 3442 } 3443 3444 if (!opts_size) { 3445 SPDK_ERRLOG("opts_size should not be zero value\n"); 3446 return; 3447 } 3448 3449 memset(opts, 0, opts_size); 3450 opts->opts_size = opts_size; 3451 3452 #define FIELD_OK(field) \ 3453 offsetof(struct spdk_bs_opts, field) + sizeof(opts->field) <= opts_size 3454 3455 #define SET_FIELD(field, value) \ 3456 if (FIELD_OK(field)) { \ 3457 opts->field = value; \ 3458 } \ 3459 3460 SET_FIELD(cluster_sz, SPDK_BLOB_OPTS_CLUSTER_SZ); 3461 SET_FIELD(num_md_pages, SPDK_BLOB_OPTS_NUM_MD_PAGES); 3462 SET_FIELD(max_md_ops, SPDK_BLOB_OPTS_NUM_MD_PAGES); 3463 SET_FIELD(max_channel_ops, SPDK_BLOB_OPTS_DEFAULT_CHANNEL_OPS); 3464 SET_FIELD(clear_method, BS_CLEAR_WITH_UNMAP); 3465 3466 if (FIELD_OK(bstype)) { 3467 memset(&opts->bstype, 0, sizeof(opts->bstype)); 3468 } 3469 3470 SET_FIELD(iter_cb_fn, NULL); 3471 SET_FIELD(iter_cb_arg, NULL); 3472 SET_FIELD(force_recover, false); 3473 SET_FIELD(esnap_bs_dev_create, NULL); 3474 3475 #undef FIELD_OK 3476 #undef SET_FIELD 3477 } 3478 3479 static int 3480 bs_opts_verify(struct spdk_bs_opts *opts) 3481 { 3482 if (opts->cluster_sz == 0 || opts->num_md_pages == 0 || opts->max_md_ops == 0 || 3483 opts->max_channel_ops == 0) { 3484 SPDK_ERRLOG("Blobstore options cannot be set to 0\n"); 3485 return -1; 3486 } 3487 3488 return 0; 3489 } 3490 3491 /* START spdk_bs_load */ 3492 3493 /* spdk_bs_load_ctx is used for init, load, unload and dump code paths. */ 3494 3495 struct spdk_bs_load_ctx { 3496 struct spdk_blob_store *bs; 3497 struct spdk_bs_super_block *super; 3498 3499 struct spdk_bs_md_mask *mask; 3500 bool in_page_chain; 3501 uint32_t page_index; 3502 uint32_t cur_page; 3503 struct spdk_blob_md_page *page; 3504 3505 uint64_t num_extent_pages; 3506 uint32_t *extent_page_num; 3507 struct spdk_blob_md_page *extent_pages; 3508 struct spdk_bit_array *used_clusters; 3509 3510 spdk_bs_sequence_t *seq; 3511 spdk_blob_op_with_handle_complete iter_cb_fn; 3512 void *iter_cb_arg; 3513 struct spdk_blob *blob; 3514 spdk_blob_id blobid; 3515 3516 bool force_recover; 3517 3518 /* These fields are used in the spdk_bs_dump path. */ 3519 bool dumping; 3520 FILE *fp; 3521 spdk_bs_dump_print_xattr print_xattr_fn; 3522 char xattr_name[4096]; 3523 }; 3524 3525 static int 3526 bs_alloc(struct spdk_bs_dev *dev, struct spdk_bs_opts *opts, struct spdk_blob_store **_bs, 3527 struct spdk_bs_load_ctx **_ctx) 3528 { 3529 struct spdk_blob_store *bs; 3530 struct spdk_bs_load_ctx *ctx; 3531 uint64_t dev_size; 3532 int rc; 3533 3534 dev_size = dev->blocklen * dev->blockcnt; 3535 if (dev_size < opts->cluster_sz) { 3536 /* Device size cannot be smaller than cluster size of blobstore */ 3537 SPDK_INFOLOG(blob, "Device size %" PRIu64 " is smaller than cluster size %" PRIu32 "\n", 3538 dev_size, opts->cluster_sz); 3539 return -ENOSPC; 3540 } 3541 if (opts->cluster_sz < SPDK_BS_PAGE_SIZE) { 3542 /* Cluster size cannot be smaller than page size */ 3543 SPDK_ERRLOG("Cluster size %" PRIu32 " is smaller than page size %d\n", 3544 opts->cluster_sz, SPDK_BS_PAGE_SIZE); 3545 return -EINVAL; 3546 } 3547 bs = calloc(1, sizeof(struct spdk_blob_store)); 3548 if (!bs) { 3549 return -ENOMEM; 3550 } 3551 3552 ctx = calloc(1, sizeof(struct spdk_bs_load_ctx)); 3553 if (!ctx) { 3554 free(bs); 3555 return -ENOMEM; 3556 } 3557 3558 ctx->bs = bs; 3559 ctx->iter_cb_fn = opts->iter_cb_fn; 3560 ctx->iter_cb_arg = opts->iter_cb_arg; 3561 ctx->force_recover = opts->force_recover; 3562 3563 ctx->super = spdk_zmalloc(sizeof(*ctx->super), 0x1000, NULL, 3564 SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA); 3565 if (!ctx->super) { 3566 free(ctx); 3567 free(bs); 3568 return -ENOMEM; 3569 } 3570 3571 RB_INIT(&bs->open_blobs); 3572 TAILQ_INIT(&bs->snapshots); 3573 bs->dev = dev; 3574 bs->md_thread = spdk_get_thread(); 3575 assert(bs->md_thread != NULL); 3576 3577 /* 3578 * Do not use bs_lba_to_cluster() here since blockcnt may not be an 3579 * even multiple of the cluster size. 3580 */ 3581 bs->cluster_sz = opts->cluster_sz; 3582 bs->total_clusters = dev->blockcnt / (bs->cluster_sz / dev->blocklen); 3583 ctx->used_clusters = spdk_bit_array_create(bs->total_clusters); 3584 if (!ctx->used_clusters) { 3585 spdk_free(ctx->super); 3586 free(ctx); 3587 free(bs); 3588 return -ENOMEM; 3589 } 3590 3591 bs->pages_per_cluster = bs->cluster_sz / SPDK_BS_PAGE_SIZE; 3592 if (spdk_u32_is_pow2(bs->pages_per_cluster)) { 3593 bs->pages_per_cluster_shift = spdk_u32log2(bs->pages_per_cluster); 3594 } 3595 bs->num_free_clusters = bs->total_clusters; 3596 bs->io_unit_size = dev->blocklen; 3597 3598 bs->max_channel_ops = opts->max_channel_ops; 3599 bs->super_blob = SPDK_BLOBID_INVALID; 3600 memcpy(&bs->bstype, &opts->bstype, sizeof(opts->bstype)); 3601 bs->esnap_bs_dev_create = opts->esnap_bs_dev_create; 3602 3603 /* The metadata is assumed to be at least 1 page */ 3604 bs->used_md_pages = spdk_bit_array_create(1); 3605 bs->used_blobids = spdk_bit_array_create(0); 3606 bs->open_blobids = spdk_bit_array_create(0); 3607 3608 spdk_spin_init(&bs->used_lock); 3609 3610 spdk_io_device_register(bs, bs_channel_create, bs_channel_destroy, 3611 sizeof(struct spdk_bs_channel), "blobstore"); 3612 rc = bs_register_md_thread(bs); 3613 if (rc == -1) { 3614 spdk_io_device_unregister(bs, NULL); 3615 spdk_spin_destroy(&bs->used_lock); 3616 spdk_bit_array_free(&bs->open_blobids); 3617 spdk_bit_array_free(&bs->used_blobids); 3618 spdk_bit_array_free(&bs->used_md_pages); 3619 spdk_bit_array_free(&ctx->used_clusters); 3620 spdk_free(ctx->super); 3621 free(ctx); 3622 free(bs); 3623 /* FIXME: this is a lie but don't know how to get a proper error code here */ 3624 return -ENOMEM; 3625 } 3626 3627 *_ctx = ctx; 3628 *_bs = bs; 3629 return 0; 3630 } 3631 3632 static void 3633 bs_load_ctx_fail(struct spdk_bs_load_ctx *ctx, int bserrno) 3634 { 3635 assert(bserrno != 0); 3636 3637 spdk_free(ctx->super); 3638 bs_sequence_finish(ctx->seq, bserrno); 3639 bs_free(ctx->bs); 3640 spdk_bit_array_free(&ctx->used_clusters); 3641 free(ctx); 3642 } 3643 3644 static void 3645 bs_write_super(spdk_bs_sequence_t *seq, struct spdk_blob_store *bs, 3646 struct spdk_bs_super_block *super, spdk_bs_sequence_cpl cb_fn, void *cb_arg) 3647 { 3648 /* Update the values in the super block */ 3649 super->super_blob = bs->super_blob; 3650 memcpy(&super->bstype, &bs->bstype, sizeof(bs->bstype)); 3651 super->crc = blob_md_page_calc_crc(super); 3652 bs_sequence_write_dev(seq, super, bs_page_to_lba(bs, 0), 3653 bs_byte_to_lba(bs, sizeof(*super)), 3654 cb_fn, cb_arg); 3655 } 3656 3657 static void 3658 bs_write_used_clusters(spdk_bs_sequence_t *seq, void *arg, spdk_bs_sequence_cpl cb_fn) 3659 { 3660 struct spdk_bs_load_ctx *ctx = arg; 3661 uint64_t mask_size, lba, lba_count; 3662 3663 /* Write out the used clusters mask */ 3664 mask_size = ctx->super->used_cluster_mask_len * SPDK_BS_PAGE_SIZE; 3665 ctx->mask = spdk_zmalloc(mask_size, 0x1000, NULL, 3666 SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA); 3667 if (!ctx->mask) { 3668 bs_load_ctx_fail(ctx, -ENOMEM); 3669 return; 3670 } 3671 3672 ctx->mask->type = SPDK_MD_MASK_TYPE_USED_CLUSTERS; 3673 ctx->mask->length = ctx->bs->total_clusters; 3674 /* We could get here through the normal unload path, or through dirty 3675 * shutdown recovery. For the normal unload path, we use the mask from 3676 * the bit pool. For dirty shutdown recovery, we don't have a bit pool yet - 3677 * only the bit array from the load ctx. 3678 */ 3679 if (ctx->bs->used_clusters) { 3680 assert(ctx->mask->length == spdk_bit_pool_capacity(ctx->bs->used_clusters)); 3681 spdk_bit_pool_store_mask(ctx->bs->used_clusters, ctx->mask->mask); 3682 } else { 3683 assert(ctx->mask->length == spdk_bit_array_capacity(ctx->used_clusters)); 3684 spdk_bit_array_store_mask(ctx->used_clusters, ctx->mask->mask); 3685 } 3686 lba = bs_page_to_lba(ctx->bs, ctx->super->used_cluster_mask_start); 3687 lba_count = bs_page_to_lba(ctx->bs, ctx->super->used_cluster_mask_len); 3688 bs_sequence_write_dev(seq, ctx->mask, lba, lba_count, cb_fn, arg); 3689 } 3690 3691 static void 3692 bs_write_used_md(spdk_bs_sequence_t *seq, void *arg, spdk_bs_sequence_cpl cb_fn) 3693 { 3694 struct spdk_bs_load_ctx *ctx = arg; 3695 uint64_t mask_size, lba, lba_count; 3696 3697 mask_size = ctx->super->used_page_mask_len * SPDK_BS_PAGE_SIZE; 3698 ctx->mask = spdk_zmalloc(mask_size, 0x1000, NULL, 3699 SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA); 3700 if (!ctx->mask) { 3701 bs_load_ctx_fail(ctx, -ENOMEM); 3702 return; 3703 } 3704 3705 ctx->mask->type = SPDK_MD_MASK_TYPE_USED_PAGES; 3706 ctx->mask->length = ctx->super->md_len; 3707 assert(ctx->mask->length == spdk_bit_array_capacity(ctx->bs->used_md_pages)); 3708 3709 spdk_bit_array_store_mask(ctx->bs->used_md_pages, ctx->mask->mask); 3710 lba = bs_page_to_lba(ctx->bs, ctx->super->used_page_mask_start); 3711 lba_count = bs_page_to_lba(ctx->bs, ctx->super->used_page_mask_len); 3712 bs_sequence_write_dev(seq, ctx->mask, lba, lba_count, cb_fn, arg); 3713 } 3714 3715 static void 3716 bs_write_used_blobids(spdk_bs_sequence_t *seq, void *arg, spdk_bs_sequence_cpl cb_fn) 3717 { 3718 struct spdk_bs_load_ctx *ctx = arg; 3719 uint64_t mask_size, lba, lba_count; 3720 3721 if (ctx->super->used_blobid_mask_len == 0) { 3722 /* 3723 * This is a pre-v3 on-disk format where the blobid mask does not get 3724 * written to disk. 3725 */ 3726 cb_fn(seq, arg, 0); 3727 return; 3728 } 3729 3730 mask_size = ctx->super->used_blobid_mask_len * SPDK_BS_PAGE_SIZE; 3731 ctx->mask = spdk_zmalloc(mask_size, 0x1000, NULL, SPDK_ENV_SOCKET_ID_ANY, 3732 SPDK_MALLOC_DMA); 3733 if (!ctx->mask) { 3734 bs_load_ctx_fail(ctx, -ENOMEM); 3735 return; 3736 } 3737 3738 ctx->mask->type = SPDK_MD_MASK_TYPE_USED_BLOBIDS; 3739 ctx->mask->length = ctx->super->md_len; 3740 assert(ctx->mask->length == spdk_bit_array_capacity(ctx->bs->used_blobids)); 3741 3742 spdk_bit_array_store_mask(ctx->bs->used_blobids, ctx->mask->mask); 3743 lba = bs_page_to_lba(ctx->bs, ctx->super->used_blobid_mask_start); 3744 lba_count = bs_page_to_lba(ctx->bs, ctx->super->used_blobid_mask_len); 3745 bs_sequence_write_dev(seq, ctx->mask, lba, lba_count, cb_fn, arg); 3746 } 3747 3748 static void 3749 blob_set_thin_provision(struct spdk_blob *blob) 3750 { 3751 blob_verify_md_op(blob); 3752 blob->invalid_flags |= SPDK_BLOB_THIN_PROV; 3753 blob->state = SPDK_BLOB_STATE_DIRTY; 3754 } 3755 3756 static void 3757 blob_set_clear_method(struct spdk_blob *blob, enum blob_clear_method clear_method) 3758 { 3759 blob_verify_md_op(blob); 3760 blob->clear_method = clear_method; 3761 blob->md_ro_flags |= (clear_method << SPDK_BLOB_CLEAR_METHOD_SHIFT); 3762 blob->state = SPDK_BLOB_STATE_DIRTY; 3763 } 3764 3765 static void bs_load_iter(void *arg, struct spdk_blob *blob, int bserrno); 3766 3767 static void 3768 bs_delete_corrupted_blob_cpl(void *cb_arg, int bserrno) 3769 { 3770 struct spdk_bs_load_ctx *ctx = cb_arg; 3771 spdk_blob_id id; 3772 int64_t page_num; 3773 3774 /* Iterate to next blob (we can't use spdk_bs_iter_next function as our 3775 * last blob has been removed */ 3776 page_num = bs_blobid_to_page(ctx->blobid); 3777 page_num++; 3778 page_num = spdk_bit_array_find_first_set(ctx->bs->used_blobids, page_num); 3779 if (page_num >= spdk_bit_array_capacity(ctx->bs->used_blobids)) { 3780 bs_load_iter(ctx, NULL, -ENOENT); 3781 return; 3782 } 3783 3784 id = bs_page_to_blobid(page_num); 3785 3786 spdk_bs_open_blob(ctx->bs, id, bs_load_iter, ctx); 3787 } 3788 3789 static void 3790 bs_delete_corrupted_close_cb(void *cb_arg, int bserrno) 3791 { 3792 struct spdk_bs_load_ctx *ctx = cb_arg; 3793 3794 if (bserrno != 0) { 3795 SPDK_ERRLOG("Failed to close corrupted blob\n"); 3796 spdk_bs_iter_next(ctx->bs, ctx->blob, bs_load_iter, ctx); 3797 return; 3798 } 3799 3800 spdk_bs_delete_blob(ctx->bs, ctx->blobid, bs_delete_corrupted_blob_cpl, ctx); 3801 } 3802 3803 static void 3804 bs_delete_corrupted_blob(void *cb_arg, int bserrno) 3805 { 3806 struct spdk_bs_load_ctx *ctx = cb_arg; 3807 uint64_t i; 3808 3809 if (bserrno != 0) { 3810 SPDK_ERRLOG("Failed to close clone of a corrupted blob\n"); 3811 spdk_bs_iter_next(ctx->bs, ctx->blob, bs_load_iter, ctx); 3812 return; 3813 } 3814 3815 /* Snapshot and clone have the same copy of cluster map and extent pages 3816 * at this point. Let's clear both for snapshot now, 3817 * so that it won't be cleared for clone later when we remove snapshot. 3818 * Also set thin provision to pass data corruption check */ 3819 for (i = 0; i < ctx->blob->active.num_clusters; i++) { 3820 ctx->blob->active.clusters[i] = 0; 3821 } 3822 for (i = 0; i < ctx->blob->active.num_extent_pages; i++) { 3823 ctx->blob->active.extent_pages[i] = 0; 3824 } 3825 3826 ctx->blob->md_ro = false; 3827 3828 blob_set_thin_provision(ctx->blob); 3829 3830 ctx->blobid = ctx->blob->id; 3831 3832 spdk_blob_close(ctx->blob, bs_delete_corrupted_close_cb, ctx); 3833 } 3834 3835 static void 3836 bs_update_corrupted_blob(void *cb_arg, int bserrno) 3837 { 3838 struct spdk_bs_load_ctx *ctx = cb_arg; 3839 3840 if (bserrno != 0) { 3841 SPDK_ERRLOG("Failed to close clone of a corrupted blob\n"); 3842 spdk_bs_iter_next(ctx->bs, ctx->blob, bs_load_iter, ctx); 3843 return; 3844 } 3845 3846 ctx->blob->md_ro = false; 3847 blob_remove_xattr(ctx->blob, SNAPSHOT_PENDING_REMOVAL, true); 3848 blob_remove_xattr(ctx->blob, SNAPSHOT_IN_PROGRESS, true); 3849 spdk_blob_set_read_only(ctx->blob); 3850 3851 if (ctx->iter_cb_fn) { 3852 ctx->iter_cb_fn(ctx->iter_cb_arg, ctx->blob, 0); 3853 } 3854 bs_blob_list_add(ctx->blob); 3855 3856 spdk_bs_iter_next(ctx->bs, ctx->blob, bs_load_iter, ctx); 3857 } 3858 3859 static void 3860 bs_examine_clone(void *cb_arg, struct spdk_blob *blob, int bserrno) 3861 { 3862 struct spdk_bs_load_ctx *ctx = cb_arg; 3863 3864 if (bserrno != 0) { 3865 SPDK_ERRLOG("Failed to open clone of a corrupted blob\n"); 3866 spdk_bs_iter_next(ctx->bs, ctx->blob, bs_load_iter, ctx); 3867 return; 3868 } 3869 3870 if (blob->parent_id == ctx->blob->id) { 3871 /* Power failure occurred before updating clone (snapshot delete case) 3872 * or after updating clone (creating snapshot case) - keep snapshot */ 3873 spdk_blob_close(blob, bs_update_corrupted_blob, ctx); 3874 } else { 3875 /* Power failure occurred after updating clone (snapshot delete case) 3876 * or before updating clone (creating snapshot case) - remove snapshot */ 3877 spdk_blob_close(blob, bs_delete_corrupted_blob, ctx); 3878 } 3879 } 3880 3881 static void 3882 bs_load_iter(void *arg, struct spdk_blob *blob, int bserrno) 3883 { 3884 struct spdk_bs_load_ctx *ctx = arg; 3885 const void *value; 3886 size_t len; 3887 int rc = 0; 3888 3889 if (bserrno == 0) { 3890 /* Examine blob if it is corrupted after power failure. Fix 3891 * the ones that can be fixed and remove any other corrupted 3892 * ones. If it is not corrupted just process it */ 3893 rc = blob_get_xattr_value(blob, SNAPSHOT_PENDING_REMOVAL, &value, &len, true); 3894 if (rc != 0) { 3895 rc = blob_get_xattr_value(blob, SNAPSHOT_IN_PROGRESS, &value, &len, true); 3896 if (rc != 0) { 3897 /* Not corrupted - process it and continue with iterating through blobs */ 3898 if (ctx->iter_cb_fn) { 3899 ctx->iter_cb_fn(ctx->iter_cb_arg, blob, 0); 3900 } 3901 bs_blob_list_add(blob); 3902 spdk_bs_iter_next(ctx->bs, blob, bs_load_iter, ctx); 3903 return; 3904 } 3905 3906 } 3907 3908 assert(len == sizeof(spdk_blob_id)); 3909 3910 ctx->blob = blob; 3911 3912 /* Open clone to check if we are able to fix this blob or should we remove it */ 3913 spdk_bs_open_blob(ctx->bs, *(spdk_blob_id *)value, bs_examine_clone, ctx); 3914 return; 3915 } else if (bserrno == -ENOENT) { 3916 bserrno = 0; 3917 } else { 3918 /* 3919 * This case needs to be looked at further. Same problem 3920 * exists with applications that rely on explicit blob 3921 * iteration. We should just skip the blob that failed 3922 * to load and continue on to the next one. 3923 */ 3924 SPDK_ERRLOG("Error in iterating blobs\n"); 3925 } 3926 3927 ctx->iter_cb_fn = NULL; 3928 3929 spdk_free(ctx->super); 3930 spdk_free(ctx->mask); 3931 bs_sequence_finish(ctx->seq, bserrno); 3932 free(ctx); 3933 } 3934 3935 static void bs_dump_read_md_page(spdk_bs_sequence_t *seq, void *cb_arg); 3936 3937 static void 3938 bs_load_complete(struct spdk_bs_load_ctx *ctx) 3939 { 3940 ctx->bs->used_clusters = spdk_bit_pool_create_from_array(ctx->used_clusters); 3941 if (ctx->dumping) { 3942 bs_dump_read_md_page(ctx->seq, ctx); 3943 return; 3944 } 3945 spdk_bs_iter_first(ctx->bs, bs_load_iter, ctx); 3946 } 3947 3948 static void 3949 bs_load_used_blobids_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 3950 { 3951 struct spdk_bs_load_ctx *ctx = cb_arg; 3952 int rc; 3953 3954 /* The type must be correct */ 3955 assert(ctx->mask->type == SPDK_MD_MASK_TYPE_USED_BLOBIDS); 3956 3957 /* The length of the mask (in bits) must not be greater than 3958 * the length of the buffer (converted to bits) */ 3959 assert(ctx->mask->length <= (ctx->super->used_blobid_mask_len * SPDK_BS_PAGE_SIZE * 8)); 3960 3961 /* The length of the mask must be exactly equal to the size 3962 * (in pages) of the metadata region */ 3963 assert(ctx->mask->length == ctx->super->md_len); 3964 3965 rc = spdk_bit_array_resize(&ctx->bs->used_blobids, ctx->mask->length); 3966 if (rc < 0) { 3967 spdk_free(ctx->mask); 3968 bs_load_ctx_fail(ctx, rc); 3969 return; 3970 } 3971 3972 spdk_bit_array_load_mask(ctx->bs->used_blobids, ctx->mask->mask); 3973 bs_load_complete(ctx); 3974 } 3975 3976 static void 3977 bs_load_used_clusters_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 3978 { 3979 struct spdk_bs_load_ctx *ctx = cb_arg; 3980 uint64_t lba, lba_count, mask_size; 3981 int rc; 3982 3983 if (bserrno != 0) { 3984 bs_load_ctx_fail(ctx, bserrno); 3985 return; 3986 } 3987 3988 /* The type must be correct */ 3989 assert(ctx->mask->type == SPDK_MD_MASK_TYPE_USED_CLUSTERS); 3990 /* The length of the mask (in bits) must not be greater than the length of the buffer (converted to bits) */ 3991 assert(ctx->mask->length <= (ctx->super->used_cluster_mask_len * sizeof( 3992 struct spdk_blob_md_page) * 8)); 3993 /* 3994 * The length of the mask must be equal to or larger than the total number of clusters. It may be 3995 * larger than the total number of clusters due to a failure spdk_bs_grow. 3996 */ 3997 assert(ctx->mask->length >= ctx->bs->total_clusters); 3998 if (ctx->mask->length > ctx->bs->total_clusters) { 3999 SPDK_WARNLOG("Shrink the used_custers mask length to total_clusters"); 4000 ctx->mask->length = ctx->bs->total_clusters; 4001 } 4002 4003 rc = spdk_bit_array_resize(&ctx->used_clusters, ctx->mask->length); 4004 if (rc < 0) { 4005 spdk_free(ctx->mask); 4006 bs_load_ctx_fail(ctx, rc); 4007 return; 4008 } 4009 4010 spdk_bit_array_load_mask(ctx->used_clusters, ctx->mask->mask); 4011 ctx->bs->num_free_clusters = spdk_bit_array_count_clear(ctx->used_clusters); 4012 assert(ctx->bs->num_free_clusters <= ctx->bs->total_clusters); 4013 4014 spdk_free(ctx->mask); 4015 4016 /* Read the used blobids mask */ 4017 mask_size = ctx->super->used_blobid_mask_len * SPDK_BS_PAGE_SIZE; 4018 ctx->mask = spdk_zmalloc(mask_size, 0x1000, NULL, SPDK_ENV_SOCKET_ID_ANY, 4019 SPDK_MALLOC_DMA); 4020 if (!ctx->mask) { 4021 bs_load_ctx_fail(ctx, -ENOMEM); 4022 return; 4023 } 4024 lba = bs_page_to_lba(ctx->bs, ctx->super->used_blobid_mask_start); 4025 lba_count = bs_page_to_lba(ctx->bs, ctx->super->used_blobid_mask_len); 4026 bs_sequence_read_dev(seq, ctx->mask, lba, lba_count, 4027 bs_load_used_blobids_cpl, ctx); 4028 } 4029 4030 static void 4031 bs_load_used_pages_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 4032 { 4033 struct spdk_bs_load_ctx *ctx = cb_arg; 4034 uint64_t lba, lba_count, mask_size; 4035 int rc; 4036 4037 if (bserrno != 0) { 4038 bs_load_ctx_fail(ctx, bserrno); 4039 return; 4040 } 4041 4042 /* The type must be correct */ 4043 assert(ctx->mask->type == SPDK_MD_MASK_TYPE_USED_PAGES); 4044 /* The length of the mask (in bits) must not be greater than the length of the buffer (converted to bits) */ 4045 assert(ctx->mask->length <= (ctx->super->used_page_mask_len * SPDK_BS_PAGE_SIZE * 4046 8)); 4047 /* The length of the mask must be exactly equal to the size (in pages) of the metadata region */ 4048 if (ctx->mask->length != ctx->super->md_len) { 4049 SPDK_ERRLOG("mismatched md_len in used_pages mask: " 4050 "mask->length=%" PRIu32 " super->md_len=%" PRIu32 "\n", 4051 ctx->mask->length, ctx->super->md_len); 4052 assert(false); 4053 } 4054 4055 rc = spdk_bit_array_resize(&ctx->bs->used_md_pages, ctx->mask->length); 4056 if (rc < 0) { 4057 spdk_free(ctx->mask); 4058 bs_load_ctx_fail(ctx, rc); 4059 return; 4060 } 4061 4062 spdk_bit_array_load_mask(ctx->bs->used_md_pages, ctx->mask->mask); 4063 spdk_free(ctx->mask); 4064 4065 /* Read the used clusters mask */ 4066 mask_size = ctx->super->used_cluster_mask_len * SPDK_BS_PAGE_SIZE; 4067 ctx->mask = spdk_zmalloc(mask_size, 0x1000, NULL, SPDK_ENV_SOCKET_ID_ANY, 4068 SPDK_MALLOC_DMA); 4069 if (!ctx->mask) { 4070 bs_load_ctx_fail(ctx, -ENOMEM); 4071 return; 4072 } 4073 lba = bs_page_to_lba(ctx->bs, ctx->super->used_cluster_mask_start); 4074 lba_count = bs_page_to_lba(ctx->bs, ctx->super->used_cluster_mask_len); 4075 bs_sequence_read_dev(seq, ctx->mask, lba, lba_count, 4076 bs_load_used_clusters_cpl, ctx); 4077 } 4078 4079 static void 4080 bs_load_read_used_pages(struct spdk_bs_load_ctx *ctx) 4081 { 4082 uint64_t lba, lba_count, mask_size; 4083 4084 /* Read the used pages mask */ 4085 mask_size = ctx->super->used_page_mask_len * SPDK_BS_PAGE_SIZE; 4086 ctx->mask = spdk_zmalloc(mask_size, 0x1000, NULL, 4087 SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA); 4088 if (!ctx->mask) { 4089 bs_load_ctx_fail(ctx, -ENOMEM); 4090 return; 4091 } 4092 4093 lba = bs_page_to_lba(ctx->bs, ctx->super->used_page_mask_start); 4094 lba_count = bs_page_to_lba(ctx->bs, ctx->super->used_page_mask_len); 4095 bs_sequence_read_dev(ctx->seq, ctx->mask, lba, lba_count, 4096 bs_load_used_pages_cpl, ctx); 4097 } 4098 4099 static int 4100 bs_load_replay_md_parse_page(struct spdk_bs_load_ctx *ctx, struct spdk_blob_md_page *page) 4101 { 4102 struct spdk_blob_store *bs = ctx->bs; 4103 struct spdk_blob_md_descriptor *desc; 4104 size_t cur_desc = 0; 4105 4106 desc = (struct spdk_blob_md_descriptor *)page->descriptors; 4107 while (cur_desc < sizeof(page->descriptors)) { 4108 if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_PADDING) { 4109 if (desc->length == 0) { 4110 /* If padding and length are 0, this terminates the page */ 4111 break; 4112 } 4113 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_EXTENT_RLE) { 4114 struct spdk_blob_md_descriptor_extent_rle *desc_extent_rle; 4115 unsigned int i, j; 4116 unsigned int cluster_count = 0; 4117 uint32_t cluster_idx; 4118 4119 desc_extent_rle = (struct spdk_blob_md_descriptor_extent_rle *)desc; 4120 4121 for (i = 0; i < desc_extent_rle->length / sizeof(desc_extent_rle->extents[0]); i++) { 4122 for (j = 0; j < desc_extent_rle->extents[i].length; j++) { 4123 cluster_idx = desc_extent_rle->extents[i].cluster_idx; 4124 /* 4125 * cluster_idx = 0 means an unallocated cluster - don't mark that 4126 * in the used cluster map. 4127 */ 4128 if (cluster_idx != 0) { 4129 SPDK_NOTICELOG("Recover: cluster %" PRIu32 "\n", cluster_idx + j); 4130 spdk_bit_array_set(ctx->used_clusters, cluster_idx + j); 4131 if (bs->num_free_clusters == 0) { 4132 return -ENOSPC; 4133 } 4134 bs->num_free_clusters--; 4135 } 4136 cluster_count++; 4137 } 4138 } 4139 if (cluster_count == 0) { 4140 return -EINVAL; 4141 } 4142 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_EXTENT_PAGE) { 4143 struct spdk_blob_md_descriptor_extent_page *desc_extent; 4144 uint32_t i; 4145 uint32_t cluster_count = 0; 4146 uint32_t cluster_idx; 4147 size_t cluster_idx_length; 4148 4149 desc_extent = (struct spdk_blob_md_descriptor_extent_page *)desc; 4150 cluster_idx_length = desc_extent->length - sizeof(desc_extent->start_cluster_idx); 4151 4152 if (desc_extent->length <= sizeof(desc_extent->start_cluster_idx) || 4153 (cluster_idx_length % sizeof(desc_extent->cluster_idx[0]) != 0)) { 4154 return -EINVAL; 4155 } 4156 4157 for (i = 0; i < cluster_idx_length / sizeof(desc_extent->cluster_idx[0]); i++) { 4158 cluster_idx = desc_extent->cluster_idx[i]; 4159 /* 4160 * cluster_idx = 0 means an unallocated cluster - don't mark that 4161 * in the used cluster map. 4162 */ 4163 if (cluster_idx != 0) { 4164 if (cluster_idx < desc_extent->start_cluster_idx && 4165 cluster_idx >= desc_extent->start_cluster_idx + cluster_count) { 4166 return -EINVAL; 4167 } 4168 spdk_bit_array_set(ctx->used_clusters, cluster_idx); 4169 if (bs->num_free_clusters == 0) { 4170 return -ENOSPC; 4171 } 4172 bs->num_free_clusters--; 4173 } 4174 cluster_count++; 4175 } 4176 4177 if (cluster_count == 0) { 4178 return -EINVAL; 4179 } 4180 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_XATTR) { 4181 /* Skip this item */ 4182 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_XATTR_INTERNAL) { 4183 /* Skip this item */ 4184 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_FLAGS) { 4185 /* Skip this item */ 4186 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_EXTENT_TABLE) { 4187 struct spdk_blob_md_descriptor_extent_table *desc_extent_table; 4188 uint32_t num_extent_pages = ctx->num_extent_pages; 4189 uint32_t i; 4190 size_t extent_pages_length; 4191 void *tmp; 4192 4193 desc_extent_table = (struct spdk_blob_md_descriptor_extent_table *)desc; 4194 extent_pages_length = desc_extent_table->length - sizeof(desc_extent_table->num_clusters); 4195 4196 if (desc_extent_table->length == 0 || 4197 (extent_pages_length % sizeof(desc_extent_table->extent_page[0]) != 0)) { 4198 return -EINVAL; 4199 } 4200 4201 for (i = 0; i < extent_pages_length / sizeof(desc_extent_table->extent_page[0]); i++) { 4202 if (desc_extent_table->extent_page[i].page_idx != 0) { 4203 if (desc_extent_table->extent_page[i].num_pages != 1) { 4204 return -EINVAL; 4205 } 4206 num_extent_pages += 1; 4207 } 4208 } 4209 4210 if (num_extent_pages > 0) { 4211 tmp = realloc(ctx->extent_page_num, num_extent_pages * sizeof(uint32_t)); 4212 if (tmp == NULL) { 4213 return -ENOMEM; 4214 } 4215 ctx->extent_page_num = tmp; 4216 4217 /* Extent table entries contain md page numbers for extent pages. 4218 * Zeroes represent unallocated extent pages, those are run-length-encoded. 4219 */ 4220 for (i = 0; i < extent_pages_length / sizeof(desc_extent_table->extent_page[0]); i++) { 4221 if (desc_extent_table->extent_page[i].page_idx != 0) { 4222 ctx->extent_page_num[ctx->num_extent_pages] = desc_extent_table->extent_page[i].page_idx; 4223 ctx->num_extent_pages += 1; 4224 } 4225 } 4226 } 4227 } else { 4228 /* Error */ 4229 return -EINVAL; 4230 } 4231 /* Advance to the next descriptor */ 4232 cur_desc += sizeof(*desc) + desc->length; 4233 if (cur_desc + sizeof(*desc) > sizeof(page->descriptors)) { 4234 break; 4235 } 4236 desc = (struct spdk_blob_md_descriptor *)((uintptr_t)page->descriptors + cur_desc); 4237 } 4238 return 0; 4239 } 4240 4241 static bool 4242 bs_load_cur_extent_page_valid(struct spdk_blob_md_page *page) 4243 { 4244 uint32_t crc; 4245 struct spdk_blob_md_descriptor *desc = (struct spdk_blob_md_descriptor *)page->descriptors; 4246 size_t desc_len; 4247 4248 crc = blob_md_page_calc_crc(page); 4249 if (crc != page->crc) { 4250 return false; 4251 } 4252 4253 /* Extent page should always be of sequence num 0. */ 4254 if (page->sequence_num != 0) { 4255 return false; 4256 } 4257 4258 /* Descriptor type must be EXTENT_PAGE. */ 4259 if (desc->type != SPDK_MD_DESCRIPTOR_TYPE_EXTENT_PAGE) { 4260 return false; 4261 } 4262 4263 /* Descriptor length cannot exceed the page. */ 4264 desc_len = sizeof(*desc) + desc->length; 4265 if (desc_len > sizeof(page->descriptors)) { 4266 return false; 4267 } 4268 4269 /* It has to be the only descriptor in the page. */ 4270 if (desc_len + sizeof(*desc) <= sizeof(page->descriptors)) { 4271 desc = (struct spdk_blob_md_descriptor *)((uintptr_t)page->descriptors + desc_len); 4272 if (desc->length != 0) { 4273 return false; 4274 } 4275 } 4276 4277 return true; 4278 } 4279 4280 static bool 4281 bs_load_cur_md_page_valid(struct spdk_bs_load_ctx *ctx) 4282 { 4283 uint32_t crc; 4284 struct spdk_blob_md_page *page = ctx->page; 4285 4286 crc = blob_md_page_calc_crc(page); 4287 if (crc != page->crc) { 4288 return false; 4289 } 4290 4291 /* First page of a sequence should match the blobid. */ 4292 if (page->sequence_num == 0 && 4293 bs_page_to_blobid(ctx->cur_page) != page->id) { 4294 return false; 4295 } 4296 assert(bs_load_cur_extent_page_valid(page) == false); 4297 4298 return true; 4299 } 4300 4301 static void bs_load_replay_cur_md_page(struct spdk_bs_load_ctx *ctx); 4302 4303 static void 4304 bs_load_write_used_clusters_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 4305 { 4306 struct spdk_bs_load_ctx *ctx = cb_arg; 4307 4308 if (bserrno != 0) { 4309 bs_load_ctx_fail(ctx, bserrno); 4310 return; 4311 } 4312 4313 bs_load_complete(ctx); 4314 } 4315 4316 static void 4317 bs_load_write_used_blobids_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 4318 { 4319 struct spdk_bs_load_ctx *ctx = cb_arg; 4320 4321 spdk_free(ctx->mask); 4322 ctx->mask = NULL; 4323 4324 if (bserrno != 0) { 4325 bs_load_ctx_fail(ctx, bserrno); 4326 return; 4327 } 4328 4329 bs_write_used_clusters(seq, ctx, bs_load_write_used_clusters_cpl); 4330 } 4331 4332 static void 4333 bs_load_write_used_pages_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 4334 { 4335 struct spdk_bs_load_ctx *ctx = cb_arg; 4336 4337 spdk_free(ctx->mask); 4338 ctx->mask = NULL; 4339 4340 if (bserrno != 0) { 4341 bs_load_ctx_fail(ctx, bserrno); 4342 return; 4343 } 4344 4345 bs_write_used_blobids(seq, ctx, bs_load_write_used_blobids_cpl); 4346 } 4347 4348 static void 4349 bs_load_write_used_md(struct spdk_bs_load_ctx *ctx) 4350 { 4351 bs_write_used_md(ctx->seq, ctx, bs_load_write_used_pages_cpl); 4352 } 4353 4354 static void 4355 bs_load_replay_md_chain_cpl(struct spdk_bs_load_ctx *ctx) 4356 { 4357 uint64_t num_md_clusters; 4358 uint64_t i; 4359 4360 ctx->in_page_chain = false; 4361 4362 do { 4363 ctx->page_index++; 4364 } while (spdk_bit_array_get(ctx->bs->used_md_pages, ctx->page_index) == true); 4365 4366 if (ctx->page_index < ctx->super->md_len) { 4367 ctx->cur_page = ctx->page_index; 4368 bs_load_replay_cur_md_page(ctx); 4369 } else { 4370 /* Claim all of the clusters used by the metadata */ 4371 num_md_clusters = spdk_divide_round_up( 4372 ctx->super->md_start + ctx->super->md_len, ctx->bs->pages_per_cluster); 4373 for (i = 0; i < num_md_clusters; i++) { 4374 spdk_bit_array_set(ctx->used_clusters, i); 4375 } 4376 ctx->bs->num_free_clusters -= num_md_clusters; 4377 spdk_free(ctx->page); 4378 bs_load_write_used_md(ctx); 4379 } 4380 } 4381 4382 static void 4383 bs_load_replay_extent_page_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 4384 { 4385 struct spdk_bs_load_ctx *ctx = cb_arg; 4386 uint32_t page_num; 4387 uint64_t i; 4388 4389 if (bserrno != 0) { 4390 spdk_free(ctx->extent_pages); 4391 bs_load_ctx_fail(ctx, bserrno); 4392 return; 4393 } 4394 4395 for (i = 0; i < ctx->num_extent_pages; i++) { 4396 /* Extent pages are only read when present within in chain md. 4397 * Integrity of md is not right if that page was not a valid extent page. */ 4398 if (bs_load_cur_extent_page_valid(&ctx->extent_pages[i]) != true) { 4399 spdk_free(ctx->extent_pages); 4400 bs_load_ctx_fail(ctx, -EILSEQ); 4401 return; 4402 } 4403 4404 page_num = ctx->extent_page_num[i]; 4405 spdk_bit_array_set(ctx->bs->used_md_pages, page_num); 4406 if (bs_load_replay_md_parse_page(ctx, &ctx->extent_pages[i])) { 4407 spdk_free(ctx->extent_pages); 4408 bs_load_ctx_fail(ctx, -EILSEQ); 4409 return; 4410 } 4411 } 4412 4413 spdk_free(ctx->extent_pages); 4414 free(ctx->extent_page_num); 4415 ctx->extent_page_num = NULL; 4416 ctx->num_extent_pages = 0; 4417 4418 bs_load_replay_md_chain_cpl(ctx); 4419 } 4420 4421 static void 4422 bs_load_replay_extent_pages(struct spdk_bs_load_ctx *ctx) 4423 { 4424 spdk_bs_batch_t *batch; 4425 uint32_t page; 4426 uint64_t lba; 4427 uint64_t i; 4428 4429 ctx->extent_pages = spdk_zmalloc(SPDK_BS_PAGE_SIZE * ctx->num_extent_pages, 0, 4430 NULL, SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA); 4431 if (!ctx->extent_pages) { 4432 bs_load_ctx_fail(ctx, -ENOMEM); 4433 return; 4434 } 4435 4436 batch = bs_sequence_to_batch(ctx->seq, bs_load_replay_extent_page_cpl, ctx); 4437 4438 for (i = 0; i < ctx->num_extent_pages; i++) { 4439 page = ctx->extent_page_num[i]; 4440 assert(page < ctx->super->md_len); 4441 lba = bs_md_page_to_lba(ctx->bs, page); 4442 bs_batch_read_dev(batch, &ctx->extent_pages[i], lba, 4443 bs_byte_to_lba(ctx->bs, SPDK_BS_PAGE_SIZE)); 4444 } 4445 4446 bs_batch_close(batch); 4447 } 4448 4449 static void 4450 bs_load_replay_md_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 4451 { 4452 struct spdk_bs_load_ctx *ctx = cb_arg; 4453 uint32_t page_num; 4454 struct spdk_blob_md_page *page; 4455 4456 if (bserrno != 0) { 4457 bs_load_ctx_fail(ctx, bserrno); 4458 return; 4459 } 4460 4461 page_num = ctx->cur_page; 4462 page = ctx->page; 4463 if (bs_load_cur_md_page_valid(ctx) == true) { 4464 if (page->sequence_num == 0 || ctx->in_page_chain == true) { 4465 spdk_spin_lock(&ctx->bs->used_lock); 4466 bs_claim_md_page(ctx->bs, page_num); 4467 spdk_spin_unlock(&ctx->bs->used_lock); 4468 if (page->sequence_num == 0) { 4469 SPDK_NOTICELOG("Recover: blob 0x%" PRIx32 "\n", page_num); 4470 spdk_bit_array_set(ctx->bs->used_blobids, page_num); 4471 } 4472 if (bs_load_replay_md_parse_page(ctx, page)) { 4473 bs_load_ctx_fail(ctx, -EILSEQ); 4474 return; 4475 } 4476 if (page->next != SPDK_INVALID_MD_PAGE) { 4477 ctx->in_page_chain = true; 4478 ctx->cur_page = page->next; 4479 bs_load_replay_cur_md_page(ctx); 4480 return; 4481 } 4482 if (ctx->num_extent_pages != 0) { 4483 bs_load_replay_extent_pages(ctx); 4484 return; 4485 } 4486 } 4487 } 4488 bs_load_replay_md_chain_cpl(ctx); 4489 } 4490 4491 static void 4492 bs_load_replay_cur_md_page(struct spdk_bs_load_ctx *ctx) 4493 { 4494 uint64_t lba; 4495 4496 assert(ctx->cur_page < ctx->super->md_len); 4497 lba = bs_md_page_to_lba(ctx->bs, ctx->cur_page); 4498 bs_sequence_read_dev(ctx->seq, ctx->page, lba, 4499 bs_byte_to_lba(ctx->bs, SPDK_BS_PAGE_SIZE), 4500 bs_load_replay_md_cpl, ctx); 4501 } 4502 4503 static void 4504 bs_load_replay_md(struct spdk_bs_load_ctx *ctx) 4505 { 4506 ctx->page_index = 0; 4507 ctx->cur_page = 0; 4508 ctx->page = spdk_zmalloc(SPDK_BS_PAGE_SIZE, 0, 4509 NULL, SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA); 4510 if (!ctx->page) { 4511 bs_load_ctx_fail(ctx, -ENOMEM); 4512 return; 4513 } 4514 bs_load_replay_cur_md_page(ctx); 4515 } 4516 4517 static void 4518 bs_recover(struct spdk_bs_load_ctx *ctx) 4519 { 4520 int rc; 4521 4522 SPDK_NOTICELOG("Performing recovery on blobstore\n"); 4523 rc = spdk_bit_array_resize(&ctx->bs->used_md_pages, ctx->super->md_len); 4524 if (rc < 0) { 4525 bs_load_ctx_fail(ctx, -ENOMEM); 4526 return; 4527 } 4528 4529 rc = spdk_bit_array_resize(&ctx->bs->used_blobids, ctx->super->md_len); 4530 if (rc < 0) { 4531 bs_load_ctx_fail(ctx, -ENOMEM); 4532 return; 4533 } 4534 4535 rc = spdk_bit_array_resize(&ctx->used_clusters, ctx->bs->total_clusters); 4536 if (rc < 0) { 4537 bs_load_ctx_fail(ctx, -ENOMEM); 4538 return; 4539 } 4540 4541 rc = spdk_bit_array_resize(&ctx->bs->open_blobids, ctx->super->md_len); 4542 if (rc < 0) { 4543 bs_load_ctx_fail(ctx, -ENOMEM); 4544 return; 4545 } 4546 4547 ctx->bs->num_free_clusters = ctx->bs->total_clusters; 4548 bs_load_replay_md(ctx); 4549 } 4550 4551 static int 4552 bs_parse_super(struct spdk_bs_load_ctx *ctx) 4553 { 4554 int rc; 4555 4556 if (ctx->super->size == 0) { 4557 ctx->super->size = ctx->bs->dev->blockcnt * ctx->bs->dev->blocklen; 4558 } 4559 4560 if (ctx->super->io_unit_size == 0) { 4561 ctx->super->io_unit_size = SPDK_BS_PAGE_SIZE; 4562 } 4563 4564 ctx->bs->clean = 1; 4565 ctx->bs->cluster_sz = ctx->super->cluster_size; 4566 ctx->bs->total_clusters = ctx->super->size / ctx->super->cluster_size; 4567 ctx->bs->pages_per_cluster = ctx->bs->cluster_sz / SPDK_BS_PAGE_SIZE; 4568 if (spdk_u32_is_pow2(ctx->bs->pages_per_cluster)) { 4569 ctx->bs->pages_per_cluster_shift = spdk_u32log2(ctx->bs->pages_per_cluster); 4570 } 4571 ctx->bs->io_unit_size = ctx->super->io_unit_size; 4572 rc = spdk_bit_array_resize(&ctx->used_clusters, ctx->bs->total_clusters); 4573 if (rc < 0) { 4574 return -ENOMEM; 4575 } 4576 ctx->bs->md_start = ctx->super->md_start; 4577 ctx->bs->md_len = ctx->super->md_len; 4578 rc = spdk_bit_array_resize(&ctx->bs->open_blobids, ctx->bs->md_len); 4579 if (rc < 0) { 4580 return -ENOMEM; 4581 } 4582 4583 ctx->bs->total_data_clusters = ctx->bs->total_clusters - spdk_divide_round_up( 4584 ctx->bs->md_start + ctx->bs->md_len, ctx->bs->pages_per_cluster); 4585 ctx->bs->super_blob = ctx->super->super_blob; 4586 memcpy(&ctx->bs->bstype, &ctx->super->bstype, sizeof(ctx->super->bstype)); 4587 4588 return 0; 4589 } 4590 4591 static void 4592 bs_load_super_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 4593 { 4594 struct spdk_bs_load_ctx *ctx = cb_arg; 4595 uint32_t crc; 4596 int rc; 4597 static const char zeros[SPDK_BLOBSTORE_TYPE_LENGTH]; 4598 4599 if (ctx->super->version > SPDK_BS_VERSION || 4600 ctx->super->version < SPDK_BS_INITIAL_VERSION) { 4601 bs_load_ctx_fail(ctx, -EILSEQ); 4602 return; 4603 } 4604 4605 if (memcmp(ctx->super->signature, SPDK_BS_SUPER_BLOCK_SIG, 4606 sizeof(ctx->super->signature)) != 0) { 4607 bs_load_ctx_fail(ctx, -EILSEQ); 4608 return; 4609 } 4610 4611 crc = blob_md_page_calc_crc(ctx->super); 4612 if (crc != ctx->super->crc) { 4613 bs_load_ctx_fail(ctx, -EILSEQ); 4614 return; 4615 } 4616 4617 if (memcmp(&ctx->bs->bstype, &ctx->super->bstype, SPDK_BLOBSTORE_TYPE_LENGTH) == 0) { 4618 SPDK_DEBUGLOG(blob, "Bstype matched - loading blobstore\n"); 4619 } else if (memcmp(&ctx->bs->bstype, zeros, SPDK_BLOBSTORE_TYPE_LENGTH) == 0) { 4620 SPDK_DEBUGLOG(blob, "Bstype wildcard used - loading blobstore regardless bstype\n"); 4621 } else { 4622 SPDK_DEBUGLOG(blob, "Unexpected bstype\n"); 4623 SPDK_LOGDUMP(blob, "Expected:", ctx->bs->bstype.bstype, SPDK_BLOBSTORE_TYPE_LENGTH); 4624 SPDK_LOGDUMP(blob, "Found:", ctx->super->bstype.bstype, SPDK_BLOBSTORE_TYPE_LENGTH); 4625 bs_load_ctx_fail(ctx, -ENXIO); 4626 return; 4627 } 4628 4629 if (ctx->super->size > ctx->bs->dev->blockcnt * ctx->bs->dev->blocklen) { 4630 SPDK_NOTICELOG("Size mismatch, dev size: %" PRIu64 ", blobstore size: %" PRIu64 "\n", 4631 ctx->bs->dev->blockcnt * ctx->bs->dev->blocklen, ctx->super->size); 4632 bs_load_ctx_fail(ctx, -EILSEQ); 4633 return; 4634 } 4635 4636 rc = bs_parse_super(ctx); 4637 if (rc < 0) { 4638 bs_load_ctx_fail(ctx, rc); 4639 return; 4640 } 4641 4642 if (ctx->super->used_blobid_mask_len == 0 || ctx->super->clean == 0 || ctx->force_recover) { 4643 bs_recover(ctx); 4644 } else { 4645 bs_load_read_used_pages(ctx); 4646 } 4647 } 4648 4649 static inline int 4650 bs_opts_copy(struct spdk_bs_opts *src, struct spdk_bs_opts *dst) 4651 { 4652 4653 if (!src->opts_size) { 4654 SPDK_ERRLOG("opts_size should not be zero value\n"); 4655 return -1; 4656 } 4657 4658 #define FIELD_OK(field) \ 4659 offsetof(struct spdk_bs_opts, field) + sizeof(src->field) <= src->opts_size 4660 4661 #define SET_FIELD(field) \ 4662 if (FIELD_OK(field)) { \ 4663 dst->field = src->field; \ 4664 } \ 4665 4666 SET_FIELD(cluster_sz); 4667 SET_FIELD(num_md_pages); 4668 SET_FIELD(max_md_ops); 4669 SET_FIELD(max_channel_ops); 4670 SET_FIELD(clear_method); 4671 4672 if (FIELD_OK(bstype)) { 4673 memcpy(&dst->bstype, &src->bstype, sizeof(dst->bstype)); 4674 } 4675 SET_FIELD(iter_cb_fn); 4676 SET_FIELD(iter_cb_arg); 4677 SET_FIELD(force_recover); 4678 SET_FIELD(esnap_bs_dev_create); 4679 4680 dst->opts_size = src->opts_size; 4681 4682 /* You should not remove this statement, but need to update the assert statement 4683 * if you add a new field, and also add a corresponding SET_FIELD statement */ 4684 SPDK_STATIC_ASSERT(sizeof(struct spdk_bs_opts) == 80, "Incorrect size"); 4685 4686 #undef FIELD_OK 4687 #undef SET_FIELD 4688 4689 return 0; 4690 } 4691 4692 void 4693 spdk_bs_load(struct spdk_bs_dev *dev, struct spdk_bs_opts *o, 4694 spdk_bs_op_with_handle_complete cb_fn, void *cb_arg) 4695 { 4696 struct spdk_blob_store *bs; 4697 struct spdk_bs_cpl cpl; 4698 struct spdk_bs_load_ctx *ctx; 4699 struct spdk_bs_opts opts = {}; 4700 int err; 4701 4702 SPDK_DEBUGLOG(blob, "Loading blobstore from dev %p\n", dev); 4703 4704 if ((SPDK_BS_PAGE_SIZE % dev->blocklen) != 0) { 4705 SPDK_DEBUGLOG(blob, "unsupported dev block length of %d\n", dev->blocklen); 4706 dev->destroy(dev); 4707 cb_fn(cb_arg, NULL, -EINVAL); 4708 return; 4709 } 4710 4711 spdk_bs_opts_init(&opts, sizeof(opts)); 4712 if (o) { 4713 if (bs_opts_copy(o, &opts)) { 4714 return; 4715 } 4716 } 4717 4718 if (opts.max_md_ops == 0 || opts.max_channel_ops == 0) { 4719 dev->destroy(dev); 4720 cb_fn(cb_arg, NULL, -EINVAL); 4721 return; 4722 } 4723 4724 err = bs_alloc(dev, &opts, &bs, &ctx); 4725 if (err) { 4726 dev->destroy(dev); 4727 cb_fn(cb_arg, NULL, err); 4728 return; 4729 } 4730 4731 cpl.type = SPDK_BS_CPL_TYPE_BS_HANDLE; 4732 cpl.u.bs_handle.cb_fn = cb_fn; 4733 cpl.u.bs_handle.cb_arg = cb_arg; 4734 cpl.u.bs_handle.bs = bs; 4735 4736 ctx->seq = bs_sequence_start(bs->md_channel, &cpl); 4737 if (!ctx->seq) { 4738 spdk_free(ctx->super); 4739 free(ctx); 4740 bs_free(bs); 4741 cb_fn(cb_arg, NULL, -ENOMEM); 4742 return; 4743 } 4744 4745 /* Read the super block */ 4746 bs_sequence_read_dev(ctx->seq, ctx->super, bs_page_to_lba(bs, 0), 4747 bs_byte_to_lba(bs, sizeof(*ctx->super)), 4748 bs_load_super_cpl, ctx); 4749 } 4750 4751 /* END spdk_bs_load */ 4752 4753 /* START spdk_bs_dump */ 4754 4755 static void 4756 bs_dump_finish(spdk_bs_sequence_t *seq, struct spdk_bs_load_ctx *ctx, int bserrno) 4757 { 4758 spdk_free(ctx->super); 4759 4760 /* 4761 * We need to defer calling bs_call_cpl() until after 4762 * dev destruction, so tuck these away for later use. 4763 */ 4764 ctx->bs->unload_err = bserrno; 4765 memcpy(&ctx->bs->unload_cpl, &seq->cpl, sizeof(struct spdk_bs_cpl)); 4766 seq->cpl.type = SPDK_BS_CPL_TYPE_NONE; 4767 4768 bs_sequence_finish(seq, 0); 4769 bs_free(ctx->bs); 4770 free(ctx); 4771 } 4772 4773 static void 4774 bs_dump_print_xattr(struct spdk_bs_load_ctx *ctx, struct spdk_blob_md_descriptor *desc) 4775 { 4776 struct spdk_blob_md_descriptor_xattr *desc_xattr; 4777 uint32_t i; 4778 const char *type; 4779 4780 desc_xattr = (struct spdk_blob_md_descriptor_xattr *)desc; 4781 4782 if (desc_xattr->length != 4783 sizeof(desc_xattr->name_length) + sizeof(desc_xattr->value_length) + 4784 desc_xattr->name_length + desc_xattr->value_length) { 4785 } 4786 4787 memcpy(ctx->xattr_name, desc_xattr->name, desc_xattr->name_length); 4788 ctx->xattr_name[desc_xattr->name_length] = '\0'; 4789 if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_XATTR) { 4790 type = "XATTR"; 4791 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_XATTR_INTERNAL) { 4792 type = "XATTR_INTERNAL"; 4793 } else { 4794 assert(false); 4795 type = "XATTR_?"; 4796 } 4797 fprintf(ctx->fp, "%s: name = \"%s\"\n", type, ctx->xattr_name); 4798 fprintf(ctx->fp, " value = \""); 4799 ctx->print_xattr_fn(ctx->fp, ctx->super->bstype.bstype, ctx->xattr_name, 4800 (void *)((uintptr_t)desc_xattr->name + desc_xattr->name_length), 4801 desc_xattr->value_length); 4802 fprintf(ctx->fp, "\"\n"); 4803 for (i = 0; i < desc_xattr->value_length; i++) { 4804 if (i % 16 == 0) { 4805 fprintf(ctx->fp, " "); 4806 } 4807 fprintf(ctx->fp, "%02" PRIx8 " ", *((uint8_t *)desc_xattr->name + desc_xattr->name_length + i)); 4808 if ((i + 1) % 16 == 0) { 4809 fprintf(ctx->fp, "\n"); 4810 } 4811 } 4812 if (i % 16 != 0) { 4813 fprintf(ctx->fp, "\n"); 4814 } 4815 } 4816 4817 struct type_flag_desc { 4818 uint64_t mask; 4819 uint64_t val; 4820 const char *name; 4821 }; 4822 4823 static void 4824 bs_dump_print_type_bits(struct spdk_bs_load_ctx *ctx, uint64_t flags, 4825 struct type_flag_desc *desc, size_t numflags) 4826 { 4827 uint64_t covered = 0; 4828 size_t i; 4829 4830 for (i = 0; i < numflags; i++) { 4831 if ((desc[i].mask & flags) != desc[i].val) { 4832 continue; 4833 } 4834 fprintf(ctx->fp, "\t\t 0x%016" PRIx64 " %s", desc[i].val, desc[i].name); 4835 if (desc[i].mask != desc[i].val) { 4836 fprintf(ctx->fp, " (mask 0x%" PRIx64 " value 0x%" PRIx64 ")", 4837 desc[i].mask, desc[i].val); 4838 } 4839 fprintf(ctx->fp, "\n"); 4840 covered |= desc[i].mask; 4841 } 4842 if ((flags & ~covered) != 0) { 4843 fprintf(ctx->fp, "\t\t 0x%016" PRIx64 " Unknown\n", flags & ~covered); 4844 } 4845 } 4846 4847 static void 4848 bs_dump_print_type_flags(struct spdk_bs_load_ctx *ctx, struct spdk_blob_md_descriptor *desc) 4849 { 4850 struct spdk_blob_md_descriptor_flags *type_desc; 4851 #define ADD_FLAG(f) { f, f, #f } 4852 #define ADD_MASK_VAL(m, v) { m, v, #v } 4853 static struct type_flag_desc invalid[] = { 4854 ADD_FLAG(SPDK_BLOB_THIN_PROV), 4855 ADD_FLAG(SPDK_BLOB_INTERNAL_XATTR), 4856 ADD_FLAG(SPDK_BLOB_EXTENT_TABLE), 4857 }; 4858 static struct type_flag_desc data_ro[] = { 4859 ADD_FLAG(SPDK_BLOB_READ_ONLY), 4860 }; 4861 static struct type_flag_desc md_ro[] = { 4862 ADD_MASK_VAL(SPDK_BLOB_MD_RO_FLAGS_MASK, BLOB_CLEAR_WITH_DEFAULT), 4863 ADD_MASK_VAL(SPDK_BLOB_MD_RO_FLAGS_MASK, BLOB_CLEAR_WITH_NONE), 4864 ADD_MASK_VAL(SPDK_BLOB_MD_RO_FLAGS_MASK, BLOB_CLEAR_WITH_UNMAP), 4865 ADD_MASK_VAL(SPDK_BLOB_MD_RO_FLAGS_MASK, BLOB_CLEAR_WITH_WRITE_ZEROES), 4866 }; 4867 #undef ADD_FLAG 4868 #undef ADD_MASK_VAL 4869 4870 type_desc = (struct spdk_blob_md_descriptor_flags *)desc; 4871 fprintf(ctx->fp, "Flags:\n"); 4872 fprintf(ctx->fp, "\tinvalid: 0x%016" PRIx64 "\n", type_desc->invalid_flags); 4873 bs_dump_print_type_bits(ctx, type_desc->invalid_flags, invalid, 4874 SPDK_COUNTOF(invalid)); 4875 fprintf(ctx->fp, "\tdata_ro: 0x%016" PRIx64 "\n", type_desc->data_ro_flags); 4876 bs_dump_print_type_bits(ctx, type_desc->data_ro_flags, data_ro, 4877 SPDK_COUNTOF(data_ro)); 4878 fprintf(ctx->fp, "\t md_ro: 0x%016" PRIx64 "\n", type_desc->md_ro_flags); 4879 bs_dump_print_type_bits(ctx, type_desc->md_ro_flags, md_ro, 4880 SPDK_COUNTOF(md_ro)); 4881 } 4882 4883 static void 4884 bs_dump_print_extent_table(struct spdk_bs_load_ctx *ctx, struct spdk_blob_md_descriptor *desc) 4885 { 4886 struct spdk_blob_md_descriptor_extent_table *et_desc; 4887 uint64_t num_extent_pages; 4888 uint32_t et_idx; 4889 4890 et_desc = (struct spdk_blob_md_descriptor_extent_table *)desc; 4891 num_extent_pages = (et_desc->length - sizeof(et_desc->num_clusters)) / 4892 sizeof(et_desc->extent_page[0]); 4893 4894 fprintf(ctx->fp, "Extent table:\n"); 4895 for (et_idx = 0; et_idx < num_extent_pages; et_idx++) { 4896 if (et_desc->extent_page[et_idx].page_idx == 0) { 4897 /* Zeroes represent unallocated extent pages. */ 4898 continue; 4899 } 4900 fprintf(ctx->fp, "\tExtent page: %5" PRIu32 " length %3" PRIu32 4901 " at LBA %" PRIu64 "\n", et_desc->extent_page[et_idx].page_idx, 4902 et_desc->extent_page[et_idx].num_pages, 4903 bs_md_page_to_lba(ctx->bs, et_desc->extent_page[et_idx].page_idx)); 4904 } 4905 } 4906 4907 static void 4908 bs_dump_print_md_page(struct spdk_bs_load_ctx *ctx) 4909 { 4910 uint32_t page_idx = ctx->cur_page; 4911 struct spdk_blob_md_page *page = ctx->page; 4912 struct spdk_blob_md_descriptor *desc; 4913 size_t cur_desc = 0; 4914 uint32_t crc; 4915 4916 fprintf(ctx->fp, "=========\n"); 4917 fprintf(ctx->fp, "Metadata Page Index: %" PRIu32 " (0x%" PRIx32 ")\n", page_idx, page_idx); 4918 fprintf(ctx->fp, "Start LBA: %" PRIu64 "\n", bs_md_page_to_lba(ctx->bs, page_idx)); 4919 fprintf(ctx->fp, "Blob ID: 0x%" PRIx64 "\n", page->id); 4920 fprintf(ctx->fp, "Sequence: %" PRIu32 "\n", page->sequence_num); 4921 if (page->next == SPDK_INVALID_MD_PAGE) { 4922 fprintf(ctx->fp, "Next: None\n"); 4923 } else { 4924 fprintf(ctx->fp, "Next: %" PRIu32 "\n", page->next); 4925 } 4926 fprintf(ctx->fp, "In used bit array%s:", ctx->super->clean ? "" : " (not clean: dubious)"); 4927 if (spdk_bit_array_get(ctx->bs->used_md_pages, page_idx)) { 4928 fprintf(ctx->fp, " md"); 4929 } 4930 if (spdk_bit_array_get(ctx->bs->used_blobids, page_idx)) { 4931 fprintf(ctx->fp, " blob"); 4932 } 4933 fprintf(ctx->fp, "\n"); 4934 4935 crc = blob_md_page_calc_crc(page); 4936 fprintf(ctx->fp, "CRC: 0x%" PRIx32 " (%s)\n", page->crc, crc == page->crc ? "OK" : "Mismatch"); 4937 4938 desc = (struct spdk_blob_md_descriptor *)page->descriptors; 4939 while (cur_desc < sizeof(page->descriptors)) { 4940 if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_PADDING) { 4941 if (desc->length == 0) { 4942 /* If padding and length are 0, this terminates the page */ 4943 break; 4944 } 4945 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_EXTENT_RLE) { 4946 struct spdk_blob_md_descriptor_extent_rle *desc_extent_rle; 4947 unsigned int i; 4948 4949 desc_extent_rle = (struct spdk_blob_md_descriptor_extent_rle *)desc; 4950 4951 for (i = 0; i < desc_extent_rle->length / sizeof(desc_extent_rle->extents[0]); i++) { 4952 if (desc_extent_rle->extents[i].cluster_idx != 0) { 4953 fprintf(ctx->fp, "Allocated Extent - Start: %" PRIu32, 4954 desc_extent_rle->extents[i].cluster_idx); 4955 } else { 4956 fprintf(ctx->fp, "Unallocated Extent - "); 4957 } 4958 fprintf(ctx->fp, " Length: %" PRIu32, desc_extent_rle->extents[i].length); 4959 fprintf(ctx->fp, "\n"); 4960 } 4961 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_EXTENT_PAGE) { 4962 struct spdk_blob_md_descriptor_extent_page *desc_extent; 4963 unsigned int i; 4964 4965 desc_extent = (struct spdk_blob_md_descriptor_extent_page *)desc; 4966 4967 for (i = 0; i < desc_extent->length / sizeof(desc_extent->cluster_idx[0]); i++) { 4968 if (desc_extent->cluster_idx[i] != 0) { 4969 fprintf(ctx->fp, "Allocated Extent - Start: %" PRIu32, 4970 desc_extent->cluster_idx[i]); 4971 } else { 4972 fprintf(ctx->fp, "Unallocated Extent"); 4973 } 4974 fprintf(ctx->fp, "\n"); 4975 } 4976 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_XATTR) { 4977 bs_dump_print_xattr(ctx, desc); 4978 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_XATTR_INTERNAL) { 4979 bs_dump_print_xattr(ctx, desc); 4980 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_FLAGS) { 4981 bs_dump_print_type_flags(ctx, desc); 4982 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_EXTENT_TABLE) { 4983 bs_dump_print_extent_table(ctx, desc); 4984 } else { 4985 /* Error */ 4986 fprintf(ctx->fp, "Unknown descriptor type %" PRIu8 "\n", desc->type); 4987 } 4988 /* Advance to the next descriptor */ 4989 cur_desc += sizeof(*desc) + desc->length; 4990 if (cur_desc + sizeof(*desc) > sizeof(page->descriptors)) { 4991 break; 4992 } 4993 desc = (struct spdk_blob_md_descriptor *)((uintptr_t)page->descriptors + cur_desc); 4994 } 4995 } 4996 4997 static void 4998 bs_dump_read_md_page_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 4999 { 5000 struct spdk_bs_load_ctx *ctx = cb_arg; 5001 5002 if (bserrno != 0) { 5003 bs_dump_finish(seq, ctx, bserrno); 5004 return; 5005 } 5006 5007 if (ctx->page->id != 0) { 5008 bs_dump_print_md_page(ctx); 5009 } 5010 5011 ctx->cur_page++; 5012 5013 if (ctx->cur_page < ctx->super->md_len) { 5014 bs_dump_read_md_page(seq, ctx); 5015 } else { 5016 spdk_free(ctx->page); 5017 bs_dump_finish(seq, ctx, 0); 5018 } 5019 } 5020 5021 static void 5022 bs_dump_read_md_page(spdk_bs_sequence_t *seq, void *cb_arg) 5023 { 5024 struct spdk_bs_load_ctx *ctx = cb_arg; 5025 uint64_t lba; 5026 5027 assert(ctx->cur_page < ctx->super->md_len); 5028 lba = bs_page_to_lba(ctx->bs, ctx->super->md_start + ctx->cur_page); 5029 bs_sequence_read_dev(seq, ctx->page, lba, 5030 bs_byte_to_lba(ctx->bs, SPDK_BS_PAGE_SIZE), 5031 bs_dump_read_md_page_cpl, ctx); 5032 } 5033 5034 static void 5035 bs_dump_super_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 5036 { 5037 struct spdk_bs_load_ctx *ctx = cb_arg; 5038 int rc; 5039 5040 fprintf(ctx->fp, "Signature: \"%.8s\" ", ctx->super->signature); 5041 if (memcmp(ctx->super->signature, SPDK_BS_SUPER_BLOCK_SIG, 5042 sizeof(ctx->super->signature)) != 0) { 5043 fprintf(ctx->fp, "(Mismatch)\n"); 5044 bs_dump_finish(seq, ctx, bserrno); 5045 return; 5046 } else { 5047 fprintf(ctx->fp, "(OK)\n"); 5048 } 5049 fprintf(ctx->fp, "Version: %" PRIu32 "\n", ctx->super->version); 5050 fprintf(ctx->fp, "CRC: 0x%x (%s)\n", ctx->super->crc, 5051 (ctx->super->crc == blob_md_page_calc_crc(ctx->super)) ? "OK" : "Mismatch"); 5052 fprintf(ctx->fp, "Blobstore Type: %.*s\n", SPDK_BLOBSTORE_TYPE_LENGTH, ctx->super->bstype.bstype); 5053 fprintf(ctx->fp, "Cluster Size: %" PRIu32 "\n", ctx->super->cluster_size); 5054 fprintf(ctx->fp, "Super Blob ID: "); 5055 if (ctx->super->super_blob == SPDK_BLOBID_INVALID) { 5056 fprintf(ctx->fp, "(None)\n"); 5057 } else { 5058 fprintf(ctx->fp, "0x%" PRIx64 "\n", ctx->super->super_blob); 5059 } 5060 fprintf(ctx->fp, "Clean: %" PRIu32 "\n", ctx->super->clean); 5061 fprintf(ctx->fp, "Used Metadata Page Mask Start: %" PRIu32 "\n", ctx->super->used_page_mask_start); 5062 fprintf(ctx->fp, "Used Metadata Page Mask Length: %" PRIu32 "\n", ctx->super->used_page_mask_len); 5063 fprintf(ctx->fp, "Used Cluster Mask Start: %" PRIu32 "\n", ctx->super->used_cluster_mask_start); 5064 fprintf(ctx->fp, "Used Cluster Mask Length: %" PRIu32 "\n", ctx->super->used_cluster_mask_len); 5065 fprintf(ctx->fp, "Used Blob ID Mask Start: %" PRIu32 "\n", ctx->super->used_blobid_mask_start); 5066 fprintf(ctx->fp, "Used Blob ID Mask Length: %" PRIu32 "\n", ctx->super->used_blobid_mask_len); 5067 fprintf(ctx->fp, "Metadata Start: %" PRIu32 "\n", ctx->super->md_start); 5068 fprintf(ctx->fp, "Metadata Length: %" PRIu32 "\n", ctx->super->md_len); 5069 5070 ctx->cur_page = 0; 5071 ctx->page = spdk_zmalloc(SPDK_BS_PAGE_SIZE, 0, 5072 NULL, SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA); 5073 if (!ctx->page) { 5074 bs_dump_finish(seq, ctx, -ENOMEM); 5075 return; 5076 } 5077 5078 rc = bs_parse_super(ctx); 5079 if (rc < 0) { 5080 bs_load_ctx_fail(ctx, rc); 5081 return; 5082 } 5083 5084 bs_load_read_used_pages(ctx); 5085 } 5086 5087 void 5088 spdk_bs_dump(struct spdk_bs_dev *dev, FILE *fp, spdk_bs_dump_print_xattr print_xattr_fn, 5089 spdk_bs_op_complete cb_fn, void *cb_arg) 5090 { 5091 struct spdk_blob_store *bs; 5092 struct spdk_bs_cpl cpl; 5093 struct spdk_bs_load_ctx *ctx; 5094 struct spdk_bs_opts opts = {}; 5095 int err; 5096 5097 SPDK_DEBUGLOG(blob, "Dumping blobstore from dev %p\n", dev); 5098 5099 spdk_bs_opts_init(&opts, sizeof(opts)); 5100 5101 err = bs_alloc(dev, &opts, &bs, &ctx); 5102 if (err) { 5103 dev->destroy(dev); 5104 cb_fn(cb_arg, err); 5105 return; 5106 } 5107 5108 ctx->dumping = true; 5109 ctx->fp = fp; 5110 ctx->print_xattr_fn = print_xattr_fn; 5111 5112 cpl.type = SPDK_BS_CPL_TYPE_BS_BASIC; 5113 cpl.u.bs_basic.cb_fn = cb_fn; 5114 cpl.u.bs_basic.cb_arg = cb_arg; 5115 5116 ctx->seq = bs_sequence_start(bs->md_channel, &cpl); 5117 if (!ctx->seq) { 5118 spdk_free(ctx->super); 5119 free(ctx); 5120 bs_free(bs); 5121 cb_fn(cb_arg, -ENOMEM); 5122 return; 5123 } 5124 5125 /* Read the super block */ 5126 bs_sequence_read_dev(ctx->seq, ctx->super, bs_page_to_lba(bs, 0), 5127 bs_byte_to_lba(bs, sizeof(*ctx->super)), 5128 bs_dump_super_cpl, ctx); 5129 } 5130 5131 /* END spdk_bs_dump */ 5132 5133 /* START spdk_bs_init */ 5134 5135 static void 5136 bs_init_persist_super_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 5137 { 5138 struct spdk_bs_load_ctx *ctx = cb_arg; 5139 5140 ctx->bs->used_clusters = spdk_bit_pool_create_from_array(ctx->used_clusters); 5141 spdk_free(ctx->super); 5142 free(ctx); 5143 5144 bs_sequence_finish(seq, bserrno); 5145 } 5146 5147 static void 5148 bs_init_trim_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 5149 { 5150 struct spdk_bs_load_ctx *ctx = cb_arg; 5151 5152 /* Write super block */ 5153 bs_sequence_write_dev(seq, ctx->super, bs_page_to_lba(ctx->bs, 0), 5154 bs_byte_to_lba(ctx->bs, sizeof(*ctx->super)), 5155 bs_init_persist_super_cpl, ctx); 5156 } 5157 5158 void 5159 spdk_bs_init(struct spdk_bs_dev *dev, struct spdk_bs_opts *o, 5160 spdk_bs_op_with_handle_complete cb_fn, void *cb_arg) 5161 { 5162 struct spdk_bs_load_ctx *ctx; 5163 struct spdk_blob_store *bs; 5164 struct spdk_bs_cpl cpl; 5165 spdk_bs_sequence_t *seq; 5166 spdk_bs_batch_t *batch; 5167 uint64_t num_md_lba; 5168 uint64_t num_md_pages; 5169 uint64_t num_md_clusters; 5170 uint64_t max_used_cluster_mask_len; 5171 uint32_t i; 5172 struct spdk_bs_opts opts = {}; 5173 int rc; 5174 uint64_t lba, lba_count; 5175 5176 SPDK_DEBUGLOG(blob, "Initializing blobstore on dev %p\n", dev); 5177 5178 if ((SPDK_BS_PAGE_SIZE % dev->blocklen) != 0) { 5179 SPDK_ERRLOG("unsupported dev block length of %d\n", 5180 dev->blocklen); 5181 dev->destroy(dev); 5182 cb_fn(cb_arg, NULL, -EINVAL); 5183 return; 5184 } 5185 5186 spdk_bs_opts_init(&opts, sizeof(opts)); 5187 if (o) { 5188 if (bs_opts_copy(o, &opts)) { 5189 return; 5190 } 5191 } 5192 5193 if (bs_opts_verify(&opts) != 0) { 5194 dev->destroy(dev); 5195 cb_fn(cb_arg, NULL, -EINVAL); 5196 return; 5197 } 5198 5199 rc = bs_alloc(dev, &opts, &bs, &ctx); 5200 if (rc) { 5201 dev->destroy(dev); 5202 cb_fn(cb_arg, NULL, rc); 5203 return; 5204 } 5205 5206 if (opts.num_md_pages == SPDK_BLOB_OPTS_NUM_MD_PAGES) { 5207 /* By default, allocate 1 page per cluster. 5208 * Technically, this over-allocates metadata 5209 * because more metadata will reduce the number 5210 * of usable clusters. This can be addressed with 5211 * more complex math in the future. 5212 */ 5213 bs->md_len = bs->total_clusters; 5214 } else { 5215 bs->md_len = opts.num_md_pages; 5216 } 5217 rc = spdk_bit_array_resize(&bs->used_md_pages, bs->md_len); 5218 if (rc < 0) { 5219 spdk_free(ctx->super); 5220 free(ctx); 5221 bs_free(bs); 5222 cb_fn(cb_arg, NULL, -ENOMEM); 5223 return; 5224 } 5225 5226 rc = spdk_bit_array_resize(&bs->used_blobids, bs->md_len); 5227 if (rc < 0) { 5228 spdk_free(ctx->super); 5229 free(ctx); 5230 bs_free(bs); 5231 cb_fn(cb_arg, NULL, -ENOMEM); 5232 return; 5233 } 5234 5235 rc = spdk_bit_array_resize(&bs->open_blobids, bs->md_len); 5236 if (rc < 0) { 5237 spdk_free(ctx->super); 5238 free(ctx); 5239 bs_free(bs); 5240 cb_fn(cb_arg, NULL, -ENOMEM); 5241 return; 5242 } 5243 5244 memcpy(ctx->super->signature, SPDK_BS_SUPER_BLOCK_SIG, 5245 sizeof(ctx->super->signature)); 5246 ctx->super->version = SPDK_BS_VERSION; 5247 ctx->super->length = sizeof(*ctx->super); 5248 ctx->super->super_blob = bs->super_blob; 5249 ctx->super->clean = 0; 5250 ctx->super->cluster_size = bs->cluster_sz; 5251 ctx->super->io_unit_size = bs->io_unit_size; 5252 memcpy(&ctx->super->bstype, &bs->bstype, sizeof(bs->bstype)); 5253 5254 /* Calculate how many pages the metadata consumes at the front 5255 * of the disk. 5256 */ 5257 5258 /* The super block uses 1 page */ 5259 num_md_pages = 1; 5260 5261 /* The used_md_pages mask requires 1 bit per metadata page, rounded 5262 * up to the nearest page, plus a header. 5263 */ 5264 ctx->super->used_page_mask_start = num_md_pages; 5265 ctx->super->used_page_mask_len = spdk_divide_round_up(sizeof(struct spdk_bs_md_mask) + 5266 spdk_divide_round_up(bs->md_len, 8), 5267 SPDK_BS_PAGE_SIZE); 5268 num_md_pages += ctx->super->used_page_mask_len; 5269 5270 /* The used_clusters mask requires 1 bit per cluster, rounded 5271 * up to the nearest page, plus a header. 5272 */ 5273 ctx->super->used_cluster_mask_start = num_md_pages; 5274 ctx->super->used_cluster_mask_len = spdk_divide_round_up(sizeof(struct spdk_bs_md_mask) + 5275 spdk_divide_round_up(bs->total_clusters, 8), 5276 SPDK_BS_PAGE_SIZE); 5277 /* The blobstore might be extended, then the used_cluster bitmap will need more space. 5278 * Here we calculate the max clusters we can support according to the 5279 * num_md_pages (bs->md_len). 5280 */ 5281 max_used_cluster_mask_len = spdk_divide_round_up(sizeof(struct spdk_bs_md_mask) + 5282 spdk_divide_round_up(bs->md_len, 8), 5283 SPDK_BS_PAGE_SIZE); 5284 max_used_cluster_mask_len = spdk_max(max_used_cluster_mask_len, 5285 ctx->super->used_cluster_mask_len); 5286 num_md_pages += max_used_cluster_mask_len; 5287 5288 /* The used_blobids mask requires 1 bit per metadata page, rounded 5289 * up to the nearest page, plus a header. 5290 */ 5291 ctx->super->used_blobid_mask_start = num_md_pages; 5292 ctx->super->used_blobid_mask_len = spdk_divide_round_up(sizeof(struct spdk_bs_md_mask) + 5293 spdk_divide_round_up(bs->md_len, 8), 5294 SPDK_BS_PAGE_SIZE); 5295 num_md_pages += ctx->super->used_blobid_mask_len; 5296 5297 /* The metadata region size was chosen above */ 5298 ctx->super->md_start = bs->md_start = num_md_pages; 5299 ctx->super->md_len = bs->md_len; 5300 num_md_pages += bs->md_len; 5301 5302 num_md_lba = bs_page_to_lba(bs, num_md_pages); 5303 5304 ctx->super->size = dev->blockcnt * dev->blocklen; 5305 5306 ctx->super->crc = blob_md_page_calc_crc(ctx->super); 5307 5308 num_md_clusters = spdk_divide_round_up(num_md_pages, bs->pages_per_cluster); 5309 if (num_md_clusters > bs->total_clusters) { 5310 SPDK_ERRLOG("Blobstore metadata cannot use more clusters than is available, " 5311 "please decrease number of pages reserved for metadata " 5312 "or increase cluster size.\n"); 5313 spdk_free(ctx->super); 5314 spdk_bit_array_free(&ctx->used_clusters); 5315 free(ctx); 5316 bs_free(bs); 5317 cb_fn(cb_arg, NULL, -ENOMEM); 5318 return; 5319 } 5320 /* Claim all of the clusters used by the metadata */ 5321 for (i = 0; i < num_md_clusters; i++) { 5322 spdk_bit_array_set(ctx->used_clusters, i); 5323 } 5324 5325 bs->num_free_clusters -= num_md_clusters; 5326 bs->total_data_clusters = bs->num_free_clusters; 5327 5328 cpl.type = SPDK_BS_CPL_TYPE_BS_HANDLE; 5329 cpl.u.bs_handle.cb_fn = cb_fn; 5330 cpl.u.bs_handle.cb_arg = cb_arg; 5331 cpl.u.bs_handle.bs = bs; 5332 5333 seq = bs_sequence_start(bs->md_channel, &cpl); 5334 if (!seq) { 5335 spdk_free(ctx->super); 5336 free(ctx); 5337 bs_free(bs); 5338 cb_fn(cb_arg, NULL, -ENOMEM); 5339 return; 5340 } 5341 5342 batch = bs_sequence_to_batch(seq, bs_init_trim_cpl, ctx); 5343 5344 /* Clear metadata space */ 5345 bs_batch_write_zeroes_dev(batch, 0, num_md_lba); 5346 5347 lba = num_md_lba; 5348 lba_count = ctx->bs->dev->blockcnt - lba; 5349 switch (opts.clear_method) { 5350 case BS_CLEAR_WITH_UNMAP: 5351 /* Trim data clusters */ 5352 bs_batch_unmap_dev(batch, lba, lba_count); 5353 break; 5354 case BS_CLEAR_WITH_WRITE_ZEROES: 5355 /* Write_zeroes to data clusters */ 5356 bs_batch_write_zeroes_dev(batch, lba, lba_count); 5357 break; 5358 case BS_CLEAR_WITH_NONE: 5359 default: 5360 break; 5361 } 5362 5363 bs_batch_close(batch); 5364 } 5365 5366 /* END spdk_bs_init */ 5367 5368 /* START spdk_bs_destroy */ 5369 5370 static void 5371 bs_destroy_trim_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 5372 { 5373 struct spdk_bs_load_ctx *ctx = cb_arg; 5374 struct spdk_blob_store *bs = ctx->bs; 5375 5376 /* 5377 * We need to defer calling bs_call_cpl() until after 5378 * dev destruction, so tuck these away for later use. 5379 */ 5380 bs->unload_err = bserrno; 5381 memcpy(&bs->unload_cpl, &seq->cpl, sizeof(struct spdk_bs_cpl)); 5382 seq->cpl.type = SPDK_BS_CPL_TYPE_NONE; 5383 5384 bs_sequence_finish(seq, bserrno); 5385 5386 bs_free(bs); 5387 free(ctx); 5388 } 5389 5390 void 5391 spdk_bs_destroy(struct spdk_blob_store *bs, spdk_bs_op_complete cb_fn, 5392 void *cb_arg) 5393 { 5394 struct spdk_bs_cpl cpl; 5395 spdk_bs_sequence_t *seq; 5396 struct spdk_bs_load_ctx *ctx; 5397 5398 SPDK_DEBUGLOG(blob, "Destroying blobstore\n"); 5399 5400 if (!RB_EMPTY(&bs->open_blobs)) { 5401 SPDK_ERRLOG("Blobstore still has open blobs\n"); 5402 cb_fn(cb_arg, -EBUSY); 5403 return; 5404 } 5405 5406 cpl.type = SPDK_BS_CPL_TYPE_BS_BASIC; 5407 cpl.u.bs_basic.cb_fn = cb_fn; 5408 cpl.u.bs_basic.cb_arg = cb_arg; 5409 5410 ctx = calloc(1, sizeof(*ctx)); 5411 if (!ctx) { 5412 cb_fn(cb_arg, -ENOMEM); 5413 return; 5414 } 5415 5416 ctx->bs = bs; 5417 5418 seq = bs_sequence_start(bs->md_channel, &cpl); 5419 if (!seq) { 5420 free(ctx); 5421 cb_fn(cb_arg, -ENOMEM); 5422 return; 5423 } 5424 5425 /* Write zeroes to the super block */ 5426 bs_sequence_write_zeroes_dev(seq, 5427 bs_page_to_lba(bs, 0), 5428 bs_byte_to_lba(bs, sizeof(struct spdk_bs_super_block)), 5429 bs_destroy_trim_cpl, ctx); 5430 } 5431 5432 /* END spdk_bs_destroy */ 5433 5434 /* START spdk_bs_unload */ 5435 5436 static void 5437 bs_unload_finish(struct spdk_bs_load_ctx *ctx, int bserrno) 5438 { 5439 spdk_bs_sequence_t *seq = ctx->seq; 5440 5441 spdk_free(ctx->super); 5442 5443 /* 5444 * We need to defer calling bs_call_cpl() until after 5445 * dev destruction, so tuck these away for later use. 5446 */ 5447 ctx->bs->unload_err = bserrno; 5448 memcpy(&ctx->bs->unload_cpl, &seq->cpl, sizeof(struct spdk_bs_cpl)); 5449 seq->cpl.type = SPDK_BS_CPL_TYPE_NONE; 5450 5451 bs_sequence_finish(seq, bserrno); 5452 5453 bs_free(ctx->bs); 5454 free(ctx); 5455 } 5456 5457 static void 5458 bs_unload_write_super_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 5459 { 5460 struct spdk_bs_load_ctx *ctx = cb_arg; 5461 5462 bs_unload_finish(ctx, bserrno); 5463 } 5464 5465 static void 5466 bs_unload_write_used_clusters_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 5467 { 5468 struct spdk_bs_load_ctx *ctx = cb_arg; 5469 5470 spdk_free(ctx->mask); 5471 5472 if (bserrno != 0) { 5473 bs_unload_finish(ctx, bserrno); 5474 return; 5475 } 5476 5477 ctx->super->clean = 1; 5478 5479 bs_write_super(seq, ctx->bs, ctx->super, bs_unload_write_super_cpl, ctx); 5480 } 5481 5482 static void 5483 bs_unload_write_used_blobids_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 5484 { 5485 struct spdk_bs_load_ctx *ctx = cb_arg; 5486 5487 spdk_free(ctx->mask); 5488 ctx->mask = NULL; 5489 5490 if (bserrno != 0) { 5491 bs_unload_finish(ctx, bserrno); 5492 return; 5493 } 5494 5495 bs_write_used_clusters(seq, ctx, bs_unload_write_used_clusters_cpl); 5496 } 5497 5498 static void 5499 bs_unload_write_used_pages_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 5500 { 5501 struct spdk_bs_load_ctx *ctx = cb_arg; 5502 5503 spdk_free(ctx->mask); 5504 ctx->mask = NULL; 5505 5506 if (bserrno != 0) { 5507 bs_unload_finish(ctx, bserrno); 5508 return; 5509 } 5510 5511 bs_write_used_blobids(seq, ctx, bs_unload_write_used_blobids_cpl); 5512 } 5513 5514 static void 5515 bs_unload_read_super_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 5516 { 5517 struct spdk_bs_load_ctx *ctx = cb_arg; 5518 5519 if (bserrno != 0) { 5520 bs_unload_finish(ctx, bserrno); 5521 return; 5522 } 5523 5524 bs_write_used_md(seq, cb_arg, bs_unload_write_used_pages_cpl); 5525 } 5526 5527 void 5528 spdk_bs_unload(struct spdk_blob_store *bs, spdk_bs_op_complete cb_fn, void *cb_arg) 5529 { 5530 struct spdk_bs_cpl cpl; 5531 struct spdk_bs_load_ctx *ctx; 5532 5533 SPDK_DEBUGLOG(blob, "Syncing blobstore\n"); 5534 5535 if (!RB_EMPTY(&bs->open_blobs)) { 5536 SPDK_ERRLOG("Blobstore still has open blobs\n"); 5537 cb_fn(cb_arg, -EBUSY); 5538 return; 5539 } 5540 5541 ctx = calloc(1, sizeof(*ctx)); 5542 if (!ctx) { 5543 cb_fn(cb_arg, -ENOMEM); 5544 return; 5545 } 5546 5547 ctx->bs = bs; 5548 5549 ctx->super = spdk_zmalloc(sizeof(*ctx->super), 0x1000, NULL, 5550 SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA); 5551 if (!ctx->super) { 5552 free(ctx); 5553 cb_fn(cb_arg, -ENOMEM); 5554 return; 5555 } 5556 5557 cpl.type = SPDK_BS_CPL_TYPE_BS_BASIC; 5558 cpl.u.bs_basic.cb_fn = cb_fn; 5559 cpl.u.bs_basic.cb_arg = cb_arg; 5560 5561 ctx->seq = bs_sequence_start(bs->md_channel, &cpl); 5562 if (!ctx->seq) { 5563 spdk_free(ctx->super); 5564 free(ctx); 5565 cb_fn(cb_arg, -ENOMEM); 5566 return; 5567 } 5568 5569 /* Read super block */ 5570 bs_sequence_read_dev(ctx->seq, ctx->super, bs_page_to_lba(bs, 0), 5571 bs_byte_to_lba(bs, sizeof(*ctx->super)), 5572 bs_unload_read_super_cpl, ctx); 5573 } 5574 5575 /* END spdk_bs_unload */ 5576 5577 /* START spdk_bs_set_super */ 5578 5579 struct spdk_bs_set_super_ctx { 5580 struct spdk_blob_store *bs; 5581 struct spdk_bs_super_block *super; 5582 }; 5583 5584 static void 5585 bs_set_super_write_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 5586 { 5587 struct spdk_bs_set_super_ctx *ctx = cb_arg; 5588 5589 if (bserrno != 0) { 5590 SPDK_ERRLOG("Unable to write to super block of blobstore\n"); 5591 } 5592 5593 spdk_free(ctx->super); 5594 5595 bs_sequence_finish(seq, bserrno); 5596 5597 free(ctx); 5598 } 5599 5600 static void 5601 bs_set_super_read_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 5602 { 5603 struct spdk_bs_set_super_ctx *ctx = cb_arg; 5604 5605 if (bserrno != 0) { 5606 SPDK_ERRLOG("Unable to read super block of blobstore\n"); 5607 spdk_free(ctx->super); 5608 bs_sequence_finish(seq, bserrno); 5609 free(ctx); 5610 return; 5611 } 5612 5613 bs_write_super(seq, ctx->bs, ctx->super, bs_set_super_write_cpl, ctx); 5614 } 5615 5616 void 5617 spdk_bs_set_super(struct spdk_blob_store *bs, spdk_blob_id blobid, 5618 spdk_bs_op_complete cb_fn, void *cb_arg) 5619 { 5620 struct spdk_bs_cpl cpl; 5621 spdk_bs_sequence_t *seq; 5622 struct spdk_bs_set_super_ctx *ctx; 5623 5624 SPDK_DEBUGLOG(blob, "Setting super blob id on blobstore\n"); 5625 5626 ctx = calloc(1, sizeof(*ctx)); 5627 if (!ctx) { 5628 cb_fn(cb_arg, -ENOMEM); 5629 return; 5630 } 5631 5632 ctx->bs = bs; 5633 5634 ctx->super = spdk_zmalloc(sizeof(*ctx->super), 0x1000, NULL, 5635 SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA); 5636 if (!ctx->super) { 5637 free(ctx); 5638 cb_fn(cb_arg, -ENOMEM); 5639 return; 5640 } 5641 5642 cpl.type = SPDK_BS_CPL_TYPE_BS_BASIC; 5643 cpl.u.bs_basic.cb_fn = cb_fn; 5644 cpl.u.bs_basic.cb_arg = cb_arg; 5645 5646 seq = bs_sequence_start(bs->md_channel, &cpl); 5647 if (!seq) { 5648 spdk_free(ctx->super); 5649 free(ctx); 5650 cb_fn(cb_arg, -ENOMEM); 5651 return; 5652 } 5653 5654 bs->super_blob = blobid; 5655 5656 /* Read super block */ 5657 bs_sequence_read_dev(seq, ctx->super, bs_page_to_lba(bs, 0), 5658 bs_byte_to_lba(bs, sizeof(*ctx->super)), 5659 bs_set_super_read_cpl, ctx); 5660 } 5661 5662 /* END spdk_bs_set_super */ 5663 5664 void 5665 spdk_bs_get_super(struct spdk_blob_store *bs, 5666 spdk_blob_op_with_id_complete cb_fn, void *cb_arg) 5667 { 5668 if (bs->super_blob == SPDK_BLOBID_INVALID) { 5669 cb_fn(cb_arg, SPDK_BLOBID_INVALID, -ENOENT); 5670 } else { 5671 cb_fn(cb_arg, bs->super_blob, 0); 5672 } 5673 } 5674 5675 uint64_t 5676 spdk_bs_get_cluster_size(struct spdk_blob_store *bs) 5677 { 5678 return bs->cluster_sz; 5679 } 5680 5681 uint64_t 5682 spdk_bs_get_page_size(struct spdk_blob_store *bs) 5683 { 5684 return SPDK_BS_PAGE_SIZE; 5685 } 5686 5687 uint64_t 5688 spdk_bs_get_io_unit_size(struct spdk_blob_store *bs) 5689 { 5690 return bs->io_unit_size; 5691 } 5692 5693 uint64_t 5694 spdk_bs_free_cluster_count(struct spdk_blob_store *bs) 5695 { 5696 return bs->num_free_clusters; 5697 } 5698 5699 uint64_t 5700 spdk_bs_total_data_cluster_count(struct spdk_blob_store *bs) 5701 { 5702 return bs->total_data_clusters; 5703 } 5704 5705 static int 5706 bs_register_md_thread(struct spdk_blob_store *bs) 5707 { 5708 bs->md_channel = spdk_get_io_channel(bs); 5709 if (!bs->md_channel) { 5710 SPDK_ERRLOG("Failed to get IO channel.\n"); 5711 return -1; 5712 } 5713 5714 return 0; 5715 } 5716 5717 static int 5718 bs_unregister_md_thread(struct spdk_blob_store *bs) 5719 { 5720 spdk_put_io_channel(bs->md_channel); 5721 5722 return 0; 5723 } 5724 5725 spdk_blob_id 5726 spdk_blob_get_id(struct spdk_blob *blob) 5727 { 5728 assert(blob != NULL); 5729 5730 return blob->id; 5731 } 5732 5733 uint64_t 5734 spdk_blob_get_num_pages(struct spdk_blob *blob) 5735 { 5736 assert(blob != NULL); 5737 5738 return bs_cluster_to_page(blob->bs, blob->active.num_clusters); 5739 } 5740 5741 uint64_t 5742 spdk_blob_get_num_io_units(struct spdk_blob *blob) 5743 { 5744 assert(blob != NULL); 5745 5746 return spdk_blob_get_num_pages(blob) * bs_io_unit_per_page(blob->bs); 5747 } 5748 5749 uint64_t 5750 spdk_blob_get_num_clusters(struct spdk_blob *blob) 5751 { 5752 assert(blob != NULL); 5753 5754 return blob->active.num_clusters; 5755 } 5756 5757 static uint64_t 5758 blob_find_io_unit(struct spdk_blob *blob, uint64_t offset, bool is_allocated) 5759 { 5760 uint64_t blob_io_unit_num = spdk_blob_get_num_io_units(blob); 5761 5762 while (offset < blob_io_unit_num) { 5763 if (bs_io_unit_is_allocated(blob, offset) == is_allocated) { 5764 return offset; 5765 } 5766 5767 offset += bs_num_io_units_to_cluster_boundary(blob, offset); 5768 } 5769 5770 return UINT64_MAX; 5771 } 5772 5773 uint64_t 5774 spdk_blob_get_next_allocated_io_unit(struct spdk_blob *blob, uint64_t offset) 5775 { 5776 return blob_find_io_unit(blob, offset, true); 5777 } 5778 5779 uint64_t 5780 spdk_blob_get_next_unallocated_io_unit(struct spdk_blob *blob, uint64_t offset) 5781 { 5782 return blob_find_io_unit(blob, offset, false); 5783 } 5784 5785 /* START spdk_bs_create_blob */ 5786 5787 static void 5788 bs_create_blob_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 5789 { 5790 struct spdk_blob *blob = cb_arg; 5791 uint32_t page_idx = bs_blobid_to_page(blob->id); 5792 5793 if (bserrno != 0) { 5794 spdk_spin_lock(&blob->bs->used_lock); 5795 spdk_bit_array_clear(blob->bs->used_blobids, page_idx); 5796 bs_release_md_page(blob->bs, page_idx); 5797 spdk_spin_unlock(&blob->bs->used_lock); 5798 } 5799 5800 blob_free(blob); 5801 5802 bs_sequence_finish(seq, bserrno); 5803 } 5804 5805 static int 5806 blob_set_xattrs(struct spdk_blob *blob, const struct spdk_blob_xattr_opts *xattrs, 5807 bool internal) 5808 { 5809 uint64_t i; 5810 size_t value_len = 0; 5811 int rc; 5812 const void *value = NULL; 5813 if (xattrs->count > 0 && xattrs->get_value == NULL) { 5814 return -EINVAL; 5815 } 5816 for (i = 0; i < xattrs->count; i++) { 5817 xattrs->get_value(xattrs->ctx, xattrs->names[i], &value, &value_len); 5818 if (value == NULL || value_len == 0) { 5819 return -EINVAL; 5820 } 5821 rc = blob_set_xattr(blob, xattrs->names[i], value, value_len, internal); 5822 if (rc < 0) { 5823 return rc; 5824 } 5825 } 5826 return 0; 5827 } 5828 5829 static void 5830 blob_opts_copy(const struct spdk_blob_opts *src, struct spdk_blob_opts *dst) 5831 { 5832 #define FIELD_OK(field) \ 5833 offsetof(struct spdk_blob_opts, field) + sizeof(src->field) <= src->opts_size 5834 5835 #define SET_FIELD(field) \ 5836 if (FIELD_OK(field)) { \ 5837 dst->field = src->field; \ 5838 } \ 5839 5840 SET_FIELD(num_clusters); 5841 SET_FIELD(thin_provision); 5842 SET_FIELD(clear_method); 5843 5844 if (FIELD_OK(xattrs)) { 5845 memcpy(&dst->xattrs, &src->xattrs, sizeof(src->xattrs)); 5846 } 5847 5848 SET_FIELD(use_extent_table); 5849 SET_FIELD(esnap_id); 5850 SET_FIELD(esnap_id_len); 5851 5852 dst->opts_size = src->opts_size; 5853 5854 /* You should not remove this statement, but need to update the assert statement 5855 * if you add a new field, and also add a corresponding SET_FIELD statement */ 5856 SPDK_STATIC_ASSERT(sizeof(struct spdk_blob_opts) == 80, "Incorrect size"); 5857 5858 #undef FIELD_OK 5859 #undef SET_FIELD 5860 } 5861 5862 static void 5863 bs_create_blob(struct spdk_blob_store *bs, 5864 const struct spdk_blob_opts *opts, 5865 const struct spdk_blob_xattr_opts *internal_xattrs, 5866 spdk_blob_op_with_id_complete cb_fn, void *cb_arg) 5867 { 5868 struct spdk_blob *blob; 5869 uint32_t page_idx; 5870 struct spdk_bs_cpl cpl; 5871 struct spdk_blob_opts opts_local; 5872 struct spdk_blob_xattr_opts internal_xattrs_default; 5873 spdk_bs_sequence_t *seq; 5874 spdk_blob_id id; 5875 int rc; 5876 5877 assert(spdk_get_thread() == bs->md_thread); 5878 5879 spdk_spin_lock(&bs->used_lock); 5880 page_idx = spdk_bit_array_find_first_clear(bs->used_md_pages, 0); 5881 if (page_idx == UINT32_MAX) { 5882 spdk_spin_unlock(&bs->used_lock); 5883 cb_fn(cb_arg, 0, -ENOMEM); 5884 return; 5885 } 5886 spdk_bit_array_set(bs->used_blobids, page_idx); 5887 bs_claim_md_page(bs, page_idx); 5888 spdk_spin_unlock(&bs->used_lock); 5889 5890 id = bs_page_to_blobid(page_idx); 5891 5892 SPDK_DEBUGLOG(blob, "Creating blob with id %" PRIu64 " at page %u\n", id, page_idx); 5893 5894 spdk_blob_opts_init(&opts_local, sizeof(opts_local)); 5895 if (opts) { 5896 blob_opts_copy(opts, &opts_local); 5897 } 5898 5899 blob = blob_alloc(bs, id); 5900 if (!blob) { 5901 rc = -ENOMEM; 5902 goto error; 5903 } 5904 5905 blob->use_extent_table = opts_local.use_extent_table; 5906 if (blob->use_extent_table) { 5907 blob->invalid_flags |= SPDK_BLOB_EXTENT_TABLE; 5908 } 5909 5910 if (!internal_xattrs) { 5911 blob_xattrs_init(&internal_xattrs_default); 5912 internal_xattrs = &internal_xattrs_default; 5913 } 5914 5915 rc = blob_set_xattrs(blob, &opts_local.xattrs, false); 5916 if (rc < 0) { 5917 goto error; 5918 } 5919 5920 rc = blob_set_xattrs(blob, internal_xattrs, true); 5921 if (rc < 0) { 5922 goto error; 5923 } 5924 5925 if (opts_local.thin_provision) { 5926 blob_set_thin_provision(blob); 5927 } 5928 5929 blob_set_clear_method(blob, opts_local.clear_method); 5930 5931 if (opts_local.esnap_id != NULL) { 5932 if (opts_local.esnap_id_len > UINT16_MAX) { 5933 SPDK_ERRLOG("esnap id length %" PRIu64 "is too long\n", 5934 opts_local.esnap_id_len); 5935 goto error; 5936 5937 } 5938 blob_set_thin_provision(blob); 5939 blob->invalid_flags |= SPDK_BLOB_EXTERNAL_SNAPSHOT; 5940 rc = blob_set_xattr(blob, BLOB_EXTERNAL_SNAPSHOT_ID, 5941 opts_local.esnap_id, opts_local.esnap_id_len, true); 5942 if (rc != 0) { 5943 goto error; 5944 } 5945 } 5946 5947 rc = blob_resize(blob, opts_local.num_clusters); 5948 if (rc < 0) { 5949 goto error; 5950 } 5951 cpl.type = SPDK_BS_CPL_TYPE_BLOBID; 5952 cpl.u.blobid.cb_fn = cb_fn; 5953 cpl.u.blobid.cb_arg = cb_arg; 5954 cpl.u.blobid.blobid = blob->id; 5955 5956 seq = bs_sequence_start(bs->md_channel, &cpl); 5957 if (!seq) { 5958 rc = -ENOMEM; 5959 goto error; 5960 } 5961 5962 blob_persist(seq, blob, bs_create_blob_cpl, blob); 5963 return; 5964 5965 error: 5966 SPDK_ERRLOG("Failed to create blob: %s, size in clusters/size: %lu (clusters)\n", 5967 spdk_strerror(rc), opts_local.num_clusters); 5968 if (blob != NULL) { 5969 blob_free(blob); 5970 } 5971 spdk_spin_lock(&bs->used_lock); 5972 spdk_bit_array_clear(bs->used_blobids, page_idx); 5973 bs_release_md_page(bs, page_idx); 5974 spdk_spin_unlock(&bs->used_lock); 5975 cb_fn(cb_arg, 0, rc); 5976 } 5977 5978 void 5979 spdk_bs_create_blob(struct spdk_blob_store *bs, 5980 spdk_blob_op_with_id_complete cb_fn, void *cb_arg) 5981 { 5982 bs_create_blob(bs, NULL, NULL, cb_fn, cb_arg); 5983 } 5984 5985 void 5986 spdk_bs_create_blob_ext(struct spdk_blob_store *bs, const struct spdk_blob_opts *opts, 5987 spdk_blob_op_with_id_complete cb_fn, void *cb_arg) 5988 { 5989 bs_create_blob(bs, opts, NULL, cb_fn, cb_arg); 5990 } 5991 5992 /* END spdk_bs_create_blob */ 5993 5994 /* START blob_cleanup */ 5995 5996 struct spdk_clone_snapshot_ctx { 5997 struct spdk_bs_cpl cpl; 5998 int bserrno; 5999 bool frozen; 6000 6001 struct spdk_io_channel *channel; 6002 6003 /* Current cluster for inflate operation */ 6004 uint64_t cluster; 6005 6006 /* For inflation force allocation of all unallocated clusters and remove 6007 * thin-provisioning. Otherwise only decouple parent and keep clone thin. */ 6008 bool allocate_all; 6009 6010 struct { 6011 spdk_blob_id id; 6012 struct spdk_blob *blob; 6013 bool md_ro; 6014 } original; 6015 struct { 6016 spdk_blob_id id; 6017 struct spdk_blob *blob; 6018 } new; 6019 6020 /* xattrs specified for snapshot/clones only. They have no impact on 6021 * the original blobs xattrs. */ 6022 const struct spdk_blob_xattr_opts *xattrs; 6023 }; 6024 6025 static void 6026 bs_clone_snapshot_cleanup_finish(void *cb_arg, int bserrno) 6027 { 6028 struct spdk_clone_snapshot_ctx *ctx = cb_arg; 6029 struct spdk_bs_cpl *cpl = &ctx->cpl; 6030 6031 if (bserrno != 0) { 6032 if (ctx->bserrno != 0) { 6033 SPDK_ERRLOG("Cleanup error %d\n", bserrno); 6034 } else { 6035 ctx->bserrno = bserrno; 6036 } 6037 } 6038 6039 switch (cpl->type) { 6040 case SPDK_BS_CPL_TYPE_BLOBID: 6041 cpl->u.blobid.cb_fn(cpl->u.blobid.cb_arg, cpl->u.blobid.blobid, ctx->bserrno); 6042 break; 6043 case SPDK_BS_CPL_TYPE_BLOB_BASIC: 6044 cpl->u.blob_basic.cb_fn(cpl->u.blob_basic.cb_arg, ctx->bserrno); 6045 break; 6046 default: 6047 SPDK_UNREACHABLE(); 6048 break; 6049 } 6050 6051 free(ctx); 6052 } 6053 6054 static void 6055 bs_snapshot_unfreeze_cpl(void *cb_arg, int bserrno) 6056 { 6057 struct spdk_clone_snapshot_ctx *ctx = (struct spdk_clone_snapshot_ctx *)cb_arg; 6058 struct spdk_blob *origblob = ctx->original.blob; 6059 6060 if (bserrno != 0) { 6061 if (ctx->bserrno != 0) { 6062 SPDK_ERRLOG("Unfreeze error %d\n", bserrno); 6063 } else { 6064 ctx->bserrno = bserrno; 6065 } 6066 } 6067 6068 ctx->original.id = origblob->id; 6069 origblob->locked_operation_in_progress = false; 6070 6071 /* Revert md_ro to original state */ 6072 origblob->md_ro = ctx->original.md_ro; 6073 6074 spdk_blob_close(origblob, bs_clone_snapshot_cleanup_finish, ctx); 6075 } 6076 6077 static void 6078 bs_clone_snapshot_origblob_cleanup(void *cb_arg, int bserrno) 6079 { 6080 struct spdk_clone_snapshot_ctx *ctx = (struct spdk_clone_snapshot_ctx *)cb_arg; 6081 struct spdk_blob *origblob = ctx->original.blob; 6082 6083 if (bserrno != 0) { 6084 if (ctx->bserrno != 0) { 6085 SPDK_ERRLOG("Cleanup error %d\n", bserrno); 6086 } else { 6087 ctx->bserrno = bserrno; 6088 } 6089 } 6090 6091 if (ctx->frozen) { 6092 /* Unfreeze any outstanding I/O */ 6093 blob_unfreeze_io(origblob, bs_snapshot_unfreeze_cpl, ctx); 6094 } else { 6095 bs_snapshot_unfreeze_cpl(ctx, 0); 6096 } 6097 6098 } 6099 6100 static void 6101 bs_clone_snapshot_newblob_cleanup(struct spdk_clone_snapshot_ctx *ctx, int bserrno) 6102 { 6103 struct spdk_blob *newblob = ctx->new.blob; 6104 6105 if (bserrno != 0) { 6106 if (ctx->bserrno != 0) { 6107 SPDK_ERRLOG("Cleanup error %d\n", bserrno); 6108 } else { 6109 ctx->bserrno = bserrno; 6110 } 6111 } 6112 6113 ctx->new.id = newblob->id; 6114 spdk_blob_close(newblob, bs_clone_snapshot_origblob_cleanup, ctx); 6115 } 6116 6117 /* END blob_cleanup */ 6118 6119 /* START spdk_bs_create_snapshot */ 6120 6121 static void 6122 bs_snapshot_swap_cluster_maps(struct spdk_blob *blob1, struct spdk_blob *blob2) 6123 { 6124 uint64_t *cluster_temp; 6125 uint32_t *extent_page_temp; 6126 6127 cluster_temp = blob1->active.clusters; 6128 blob1->active.clusters = blob2->active.clusters; 6129 blob2->active.clusters = cluster_temp; 6130 6131 extent_page_temp = blob1->active.extent_pages; 6132 blob1->active.extent_pages = blob2->active.extent_pages; 6133 blob2->active.extent_pages = extent_page_temp; 6134 } 6135 6136 static void 6137 bs_snapshot_origblob_sync_cpl(void *cb_arg, int bserrno) 6138 { 6139 struct spdk_clone_snapshot_ctx *ctx = (struct spdk_clone_snapshot_ctx *)cb_arg; 6140 struct spdk_blob *origblob = ctx->original.blob; 6141 struct spdk_blob *newblob = ctx->new.blob; 6142 6143 if (bserrno != 0) { 6144 bs_snapshot_swap_cluster_maps(newblob, origblob); 6145 bs_clone_snapshot_origblob_cleanup(ctx, bserrno); 6146 return; 6147 } 6148 6149 /* Remove metadata descriptor SNAPSHOT_IN_PROGRESS */ 6150 bserrno = blob_remove_xattr(newblob, SNAPSHOT_IN_PROGRESS, true); 6151 if (bserrno != 0) { 6152 bs_clone_snapshot_origblob_cleanup(ctx, bserrno); 6153 return; 6154 } 6155 6156 bs_blob_list_add(ctx->original.blob); 6157 6158 spdk_blob_set_read_only(newblob); 6159 6160 /* sync snapshot metadata */ 6161 spdk_blob_sync_md(newblob, bs_clone_snapshot_origblob_cleanup, ctx); 6162 } 6163 6164 static void 6165 bs_snapshot_newblob_sync_cpl(void *cb_arg, int bserrno) 6166 { 6167 struct spdk_clone_snapshot_ctx *ctx = (struct spdk_clone_snapshot_ctx *)cb_arg; 6168 struct spdk_blob *origblob = ctx->original.blob; 6169 struct spdk_blob *newblob = ctx->new.blob; 6170 6171 if (bserrno != 0) { 6172 /* return cluster map back to original */ 6173 bs_snapshot_swap_cluster_maps(newblob, origblob); 6174 6175 /* Newblob md sync failed. Valid clusters are only present in origblob. 6176 * Since I/O is frozen on origblob, not changes to zeroed out cluster map should have occurred. 6177 * Newblob needs to be reverted to thin_provisioned state at creation to properly close. */ 6178 blob_set_thin_provision(newblob); 6179 assert(spdk_mem_all_zero(newblob->active.clusters, 6180 newblob->active.num_clusters * sizeof(*newblob->active.clusters))); 6181 assert(spdk_mem_all_zero(newblob->active.extent_pages, 6182 newblob->active.num_extent_pages * sizeof(*newblob->active.extent_pages))); 6183 6184 bs_clone_snapshot_newblob_cleanup(ctx, bserrno); 6185 return; 6186 } 6187 6188 /* Set internal xattr for snapshot id */ 6189 bserrno = blob_set_xattr(origblob, BLOB_SNAPSHOT, &newblob->id, sizeof(spdk_blob_id), true); 6190 if (bserrno != 0) { 6191 /* return cluster map back to original */ 6192 bs_snapshot_swap_cluster_maps(newblob, origblob); 6193 blob_set_thin_provision(newblob); 6194 bs_clone_snapshot_newblob_cleanup(ctx, bserrno); 6195 return; 6196 } 6197 6198 /* Create new back_bs_dev for snapshot */ 6199 origblob->back_bs_dev = bs_create_blob_bs_dev(newblob); 6200 if (origblob->back_bs_dev == NULL) { 6201 /* return cluster map back to original */ 6202 bs_snapshot_swap_cluster_maps(newblob, origblob); 6203 blob_set_thin_provision(newblob); 6204 bs_clone_snapshot_newblob_cleanup(ctx, -EINVAL); 6205 return; 6206 } 6207 6208 bs_blob_list_remove(origblob); 6209 origblob->parent_id = newblob->id; 6210 /* set clone blob as thin provisioned */ 6211 blob_set_thin_provision(origblob); 6212 6213 bs_blob_list_add(newblob); 6214 6215 /* sync clone metadata */ 6216 spdk_blob_sync_md(origblob, bs_snapshot_origblob_sync_cpl, ctx); 6217 } 6218 6219 static void 6220 bs_snapshot_freeze_cpl(void *cb_arg, int rc) 6221 { 6222 struct spdk_clone_snapshot_ctx *ctx = (struct spdk_clone_snapshot_ctx *)cb_arg; 6223 struct spdk_blob *origblob = ctx->original.blob; 6224 struct spdk_blob *newblob = ctx->new.blob; 6225 int bserrno; 6226 6227 if (rc != 0) { 6228 bs_clone_snapshot_newblob_cleanup(ctx, rc); 6229 return; 6230 } 6231 6232 ctx->frozen = true; 6233 6234 if (newblob->back_bs_dev) { 6235 newblob->back_bs_dev->destroy(newblob->back_bs_dev); 6236 } 6237 /* set new back_bs_dev for snapshot */ 6238 newblob->back_bs_dev = origblob->back_bs_dev; 6239 /* Set invalid flags from origblob */ 6240 newblob->invalid_flags = origblob->invalid_flags; 6241 6242 /* inherit parent from original blob if set */ 6243 newblob->parent_id = origblob->parent_id; 6244 if (origblob->parent_id != SPDK_BLOBID_INVALID) { 6245 /* Set internal xattr for snapshot id */ 6246 bserrno = blob_set_xattr(newblob, BLOB_SNAPSHOT, 6247 &origblob->parent_id, sizeof(spdk_blob_id), true); 6248 if (bserrno != 0) { 6249 bs_clone_snapshot_newblob_cleanup(ctx, bserrno); 6250 return; 6251 } 6252 } 6253 6254 /* swap cluster maps */ 6255 bs_snapshot_swap_cluster_maps(newblob, origblob); 6256 6257 /* Set the clear method on the new blob to match the original. */ 6258 blob_set_clear_method(newblob, origblob->clear_method); 6259 6260 /* sync snapshot metadata */ 6261 spdk_blob_sync_md(newblob, bs_snapshot_newblob_sync_cpl, ctx); 6262 } 6263 6264 static void 6265 bs_snapshot_newblob_open_cpl(void *cb_arg, struct spdk_blob *_blob, int bserrno) 6266 { 6267 struct spdk_clone_snapshot_ctx *ctx = (struct spdk_clone_snapshot_ctx *)cb_arg; 6268 struct spdk_blob *origblob = ctx->original.blob; 6269 struct spdk_blob *newblob = _blob; 6270 6271 if (bserrno != 0) { 6272 bs_clone_snapshot_origblob_cleanup(ctx, bserrno); 6273 return; 6274 } 6275 6276 ctx->new.blob = newblob; 6277 assert(spdk_blob_is_thin_provisioned(newblob)); 6278 assert(spdk_mem_all_zero(newblob->active.clusters, 6279 newblob->active.num_clusters * sizeof(*newblob->active.clusters))); 6280 assert(spdk_mem_all_zero(newblob->active.extent_pages, 6281 newblob->active.num_extent_pages * sizeof(*newblob->active.extent_pages))); 6282 6283 blob_freeze_io(origblob, bs_snapshot_freeze_cpl, ctx); 6284 } 6285 6286 static void 6287 bs_snapshot_newblob_create_cpl(void *cb_arg, spdk_blob_id blobid, int bserrno) 6288 { 6289 struct spdk_clone_snapshot_ctx *ctx = (struct spdk_clone_snapshot_ctx *)cb_arg; 6290 struct spdk_blob *origblob = ctx->original.blob; 6291 6292 if (bserrno != 0) { 6293 bs_clone_snapshot_origblob_cleanup(ctx, bserrno); 6294 return; 6295 } 6296 6297 ctx->new.id = blobid; 6298 ctx->cpl.u.blobid.blobid = blobid; 6299 6300 spdk_bs_open_blob(origblob->bs, ctx->new.id, bs_snapshot_newblob_open_cpl, ctx); 6301 } 6302 6303 6304 static void 6305 bs_xattr_snapshot(void *arg, const char *name, 6306 const void **value, size_t *value_len) 6307 { 6308 assert(strncmp(name, SNAPSHOT_IN_PROGRESS, sizeof(SNAPSHOT_IN_PROGRESS)) == 0); 6309 6310 struct spdk_blob *blob = (struct spdk_blob *)arg; 6311 *value = &blob->id; 6312 *value_len = sizeof(blob->id); 6313 } 6314 6315 static void 6316 bs_snapshot_origblob_open_cpl(void *cb_arg, struct spdk_blob *_blob, int bserrno) 6317 { 6318 struct spdk_clone_snapshot_ctx *ctx = (struct spdk_clone_snapshot_ctx *)cb_arg; 6319 struct spdk_blob_opts opts; 6320 struct spdk_blob_xattr_opts internal_xattrs; 6321 char *xattrs_names[] = { SNAPSHOT_IN_PROGRESS }; 6322 6323 if (bserrno != 0) { 6324 bs_clone_snapshot_cleanup_finish(ctx, bserrno); 6325 return; 6326 } 6327 6328 ctx->original.blob = _blob; 6329 6330 if (_blob->data_ro || _blob->md_ro) { 6331 SPDK_DEBUGLOG(blob, "Cannot create snapshot from read only blob with id %" PRIu64 "\n", 6332 _blob->id); 6333 ctx->bserrno = -EINVAL; 6334 spdk_blob_close(_blob, bs_clone_snapshot_cleanup_finish, ctx); 6335 return; 6336 } 6337 6338 if (_blob->locked_operation_in_progress) { 6339 SPDK_DEBUGLOG(blob, "Cannot create snapshot - another operation in progress\n"); 6340 ctx->bserrno = -EBUSY; 6341 spdk_blob_close(_blob, bs_clone_snapshot_cleanup_finish, ctx); 6342 return; 6343 } 6344 6345 _blob->locked_operation_in_progress = true; 6346 6347 spdk_blob_opts_init(&opts, sizeof(opts)); 6348 blob_xattrs_init(&internal_xattrs); 6349 6350 /* Change the size of new blob to the same as in original blob, 6351 * but do not allocate clusters */ 6352 opts.thin_provision = true; 6353 opts.num_clusters = spdk_blob_get_num_clusters(_blob); 6354 opts.use_extent_table = _blob->use_extent_table; 6355 6356 /* If there are any xattrs specified for snapshot, set them now */ 6357 if (ctx->xattrs) { 6358 memcpy(&opts.xattrs, ctx->xattrs, sizeof(*ctx->xattrs)); 6359 } 6360 /* Set internal xattr SNAPSHOT_IN_PROGRESS */ 6361 internal_xattrs.count = 1; 6362 internal_xattrs.ctx = _blob; 6363 internal_xattrs.names = xattrs_names; 6364 internal_xattrs.get_value = bs_xattr_snapshot; 6365 6366 bs_create_blob(_blob->bs, &opts, &internal_xattrs, 6367 bs_snapshot_newblob_create_cpl, ctx); 6368 } 6369 6370 void 6371 spdk_bs_create_snapshot(struct spdk_blob_store *bs, spdk_blob_id blobid, 6372 const struct spdk_blob_xattr_opts *snapshot_xattrs, 6373 spdk_blob_op_with_id_complete cb_fn, void *cb_arg) 6374 { 6375 struct spdk_clone_snapshot_ctx *ctx = calloc(1, sizeof(*ctx)); 6376 6377 if (!ctx) { 6378 cb_fn(cb_arg, SPDK_BLOBID_INVALID, -ENOMEM); 6379 return; 6380 } 6381 ctx->cpl.type = SPDK_BS_CPL_TYPE_BLOBID; 6382 ctx->cpl.u.blobid.cb_fn = cb_fn; 6383 ctx->cpl.u.blobid.cb_arg = cb_arg; 6384 ctx->cpl.u.blobid.blobid = SPDK_BLOBID_INVALID; 6385 ctx->bserrno = 0; 6386 ctx->frozen = false; 6387 ctx->original.id = blobid; 6388 ctx->xattrs = snapshot_xattrs; 6389 6390 spdk_bs_open_blob(bs, ctx->original.id, bs_snapshot_origblob_open_cpl, ctx); 6391 } 6392 /* END spdk_bs_create_snapshot */ 6393 6394 /* START spdk_bs_create_clone */ 6395 6396 static void 6397 bs_xattr_clone(void *arg, const char *name, 6398 const void **value, size_t *value_len) 6399 { 6400 assert(strncmp(name, BLOB_SNAPSHOT, sizeof(BLOB_SNAPSHOT)) == 0); 6401 6402 struct spdk_blob *blob = (struct spdk_blob *)arg; 6403 *value = &blob->id; 6404 *value_len = sizeof(blob->id); 6405 } 6406 6407 static void 6408 bs_clone_newblob_open_cpl(void *cb_arg, struct spdk_blob *_blob, int bserrno) 6409 { 6410 struct spdk_clone_snapshot_ctx *ctx = (struct spdk_clone_snapshot_ctx *)cb_arg; 6411 struct spdk_blob *clone = _blob; 6412 6413 ctx->new.blob = clone; 6414 bs_blob_list_add(clone); 6415 6416 spdk_blob_close(clone, bs_clone_snapshot_origblob_cleanup, ctx); 6417 } 6418 6419 static void 6420 bs_clone_newblob_create_cpl(void *cb_arg, spdk_blob_id blobid, int bserrno) 6421 { 6422 struct spdk_clone_snapshot_ctx *ctx = (struct spdk_clone_snapshot_ctx *)cb_arg; 6423 6424 ctx->cpl.u.blobid.blobid = blobid; 6425 spdk_bs_open_blob(ctx->original.blob->bs, blobid, bs_clone_newblob_open_cpl, ctx); 6426 } 6427 6428 static void 6429 bs_clone_origblob_open_cpl(void *cb_arg, struct spdk_blob *_blob, int bserrno) 6430 { 6431 struct spdk_clone_snapshot_ctx *ctx = (struct spdk_clone_snapshot_ctx *)cb_arg; 6432 struct spdk_blob_opts opts; 6433 struct spdk_blob_xattr_opts internal_xattrs; 6434 char *xattr_names[] = { BLOB_SNAPSHOT }; 6435 6436 if (bserrno != 0) { 6437 bs_clone_snapshot_cleanup_finish(ctx, bserrno); 6438 return; 6439 } 6440 6441 ctx->original.blob = _blob; 6442 ctx->original.md_ro = _blob->md_ro; 6443 6444 if (!_blob->data_ro || !_blob->md_ro) { 6445 SPDK_DEBUGLOG(blob, "Clone not from read-only blob\n"); 6446 ctx->bserrno = -EINVAL; 6447 spdk_blob_close(_blob, bs_clone_snapshot_cleanup_finish, ctx); 6448 return; 6449 } 6450 6451 if (_blob->locked_operation_in_progress) { 6452 SPDK_DEBUGLOG(blob, "Cannot create clone - another operation in progress\n"); 6453 ctx->bserrno = -EBUSY; 6454 spdk_blob_close(_blob, bs_clone_snapshot_cleanup_finish, ctx); 6455 return; 6456 } 6457 6458 _blob->locked_operation_in_progress = true; 6459 6460 spdk_blob_opts_init(&opts, sizeof(opts)); 6461 blob_xattrs_init(&internal_xattrs); 6462 6463 opts.thin_provision = true; 6464 opts.num_clusters = spdk_blob_get_num_clusters(_blob); 6465 opts.use_extent_table = _blob->use_extent_table; 6466 if (ctx->xattrs) { 6467 memcpy(&opts.xattrs, ctx->xattrs, sizeof(*ctx->xattrs)); 6468 } 6469 6470 /* Set internal xattr BLOB_SNAPSHOT */ 6471 internal_xattrs.count = 1; 6472 internal_xattrs.ctx = _blob; 6473 internal_xattrs.names = xattr_names; 6474 internal_xattrs.get_value = bs_xattr_clone; 6475 6476 bs_create_blob(_blob->bs, &opts, &internal_xattrs, 6477 bs_clone_newblob_create_cpl, ctx); 6478 } 6479 6480 void 6481 spdk_bs_create_clone(struct spdk_blob_store *bs, spdk_blob_id blobid, 6482 const struct spdk_blob_xattr_opts *clone_xattrs, 6483 spdk_blob_op_with_id_complete cb_fn, void *cb_arg) 6484 { 6485 struct spdk_clone_snapshot_ctx *ctx = calloc(1, sizeof(*ctx)); 6486 6487 if (!ctx) { 6488 cb_fn(cb_arg, SPDK_BLOBID_INVALID, -ENOMEM); 6489 return; 6490 } 6491 6492 ctx->cpl.type = SPDK_BS_CPL_TYPE_BLOBID; 6493 ctx->cpl.u.blobid.cb_fn = cb_fn; 6494 ctx->cpl.u.blobid.cb_arg = cb_arg; 6495 ctx->cpl.u.blobid.blobid = SPDK_BLOBID_INVALID; 6496 ctx->bserrno = 0; 6497 ctx->xattrs = clone_xattrs; 6498 ctx->original.id = blobid; 6499 6500 spdk_bs_open_blob(bs, ctx->original.id, bs_clone_origblob_open_cpl, ctx); 6501 } 6502 6503 /* END spdk_bs_create_clone */ 6504 6505 /* START spdk_bs_inflate_blob */ 6506 6507 static void 6508 bs_inflate_blob_set_parent_cpl(void *cb_arg, struct spdk_blob *_parent, int bserrno) 6509 { 6510 struct spdk_clone_snapshot_ctx *ctx = (struct spdk_clone_snapshot_ctx *)cb_arg; 6511 struct spdk_blob *_blob = ctx->original.blob; 6512 6513 if (bserrno != 0) { 6514 bs_clone_snapshot_origblob_cleanup(ctx, bserrno); 6515 return; 6516 } 6517 6518 /* Temporarily override md_ro flag for MD modification */ 6519 _blob->md_ro = false; 6520 6521 bserrno = blob_set_xattr(_blob, BLOB_SNAPSHOT, &_parent->id, sizeof(spdk_blob_id), true); 6522 if (bserrno != 0) { 6523 bs_clone_snapshot_origblob_cleanup(ctx, bserrno); 6524 return; 6525 } 6526 6527 assert(_parent != NULL); 6528 6529 bs_blob_list_remove(_blob); 6530 _blob->parent_id = _parent->id; 6531 6532 _blob->back_bs_dev->destroy(_blob->back_bs_dev); 6533 _blob->back_bs_dev = bs_create_blob_bs_dev(_parent); 6534 bs_blob_list_add(_blob); 6535 6536 spdk_blob_sync_md(_blob, bs_clone_snapshot_origblob_cleanup, ctx); 6537 } 6538 6539 static void 6540 bs_inflate_blob_done(struct spdk_clone_snapshot_ctx *ctx) 6541 { 6542 struct spdk_blob *_blob = ctx->original.blob; 6543 struct spdk_blob *_parent; 6544 6545 if (ctx->allocate_all) { 6546 /* remove thin provisioning */ 6547 bs_blob_list_remove(_blob); 6548 _blob->invalid_flags = _blob->invalid_flags & ~SPDK_BLOB_THIN_PROV; 6549 _blob->back_bs_dev->destroy(_blob->back_bs_dev); 6550 _blob->back_bs_dev = NULL; 6551 _blob->parent_id = SPDK_BLOBID_INVALID; 6552 } else { 6553 _parent = ((struct spdk_blob_bs_dev *)(_blob->back_bs_dev))->blob; 6554 if (_parent->parent_id != SPDK_BLOBID_INVALID) { 6555 /* We must change the parent of the inflated blob */ 6556 spdk_bs_open_blob(_blob->bs, _parent->parent_id, 6557 bs_inflate_blob_set_parent_cpl, ctx); 6558 return; 6559 } 6560 6561 bs_blob_list_remove(_blob); 6562 _blob->parent_id = SPDK_BLOBID_INVALID; 6563 _blob->back_bs_dev->destroy(_blob->back_bs_dev); 6564 _blob->back_bs_dev = bs_create_zeroes_dev(); 6565 } 6566 6567 /* Temporarily override md_ro flag for MD modification */ 6568 _blob->md_ro = false; 6569 blob_remove_xattr(_blob, BLOB_SNAPSHOT, true); 6570 _blob->state = SPDK_BLOB_STATE_DIRTY; 6571 6572 spdk_blob_sync_md(_blob, bs_clone_snapshot_origblob_cleanup, ctx); 6573 } 6574 6575 /* Check if cluster needs allocation */ 6576 static inline bool 6577 bs_cluster_needs_allocation(struct spdk_blob *blob, uint64_t cluster, bool allocate_all) 6578 { 6579 struct spdk_blob_bs_dev *b; 6580 6581 assert(blob != NULL); 6582 6583 if (blob->active.clusters[cluster] != 0) { 6584 /* Cluster is already allocated */ 6585 return false; 6586 } 6587 6588 if (blob->parent_id == SPDK_BLOBID_INVALID) { 6589 /* Blob have no parent blob */ 6590 return allocate_all; 6591 } 6592 6593 b = (struct spdk_blob_bs_dev *)blob->back_bs_dev; 6594 return (allocate_all || b->blob->active.clusters[cluster] != 0); 6595 } 6596 6597 static void 6598 bs_inflate_blob_touch_next(void *cb_arg, int bserrno) 6599 { 6600 struct spdk_clone_snapshot_ctx *ctx = (struct spdk_clone_snapshot_ctx *)cb_arg; 6601 struct spdk_blob *_blob = ctx->original.blob; 6602 struct spdk_bs_cpl cpl; 6603 spdk_bs_user_op_t *op; 6604 uint64_t offset; 6605 6606 if (bserrno != 0) { 6607 bs_clone_snapshot_origblob_cleanup(ctx, bserrno); 6608 return; 6609 } 6610 6611 for (; ctx->cluster < _blob->active.num_clusters; ctx->cluster++) { 6612 if (bs_cluster_needs_allocation(_blob, ctx->cluster, ctx->allocate_all)) { 6613 break; 6614 } 6615 } 6616 6617 if (ctx->cluster < _blob->active.num_clusters) { 6618 offset = bs_cluster_to_lba(_blob->bs, ctx->cluster); 6619 6620 /* We may safely increment a cluster before copying */ 6621 ctx->cluster++; 6622 6623 /* Use a dummy 0B read as a context for cluster copy */ 6624 cpl.type = SPDK_BS_CPL_TYPE_BLOB_BASIC; 6625 cpl.u.blob_basic.cb_fn = bs_inflate_blob_touch_next; 6626 cpl.u.blob_basic.cb_arg = ctx; 6627 6628 op = bs_user_op_alloc(ctx->channel, &cpl, SPDK_BLOB_READ, _blob, 6629 NULL, 0, offset, 0); 6630 if (!op) { 6631 bs_clone_snapshot_origblob_cleanup(ctx, -ENOMEM); 6632 return; 6633 } 6634 6635 bs_allocate_and_copy_cluster(_blob, ctx->channel, offset, op); 6636 } else { 6637 bs_inflate_blob_done(ctx); 6638 } 6639 } 6640 6641 static void 6642 bs_inflate_blob_open_cpl(void *cb_arg, struct spdk_blob *_blob, int bserrno) 6643 { 6644 struct spdk_clone_snapshot_ctx *ctx = (struct spdk_clone_snapshot_ctx *)cb_arg; 6645 uint64_t clusters_needed; 6646 uint64_t i; 6647 6648 if (bserrno != 0) { 6649 bs_clone_snapshot_cleanup_finish(ctx, bserrno); 6650 return; 6651 } 6652 6653 ctx->original.blob = _blob; 6654 ctx->original.md_ro = _blob->md_ro; 6655 6656 if (_blob->locked_operation_in_progress) { 6657 SPDK_DEBUGLOG(blob, "Cannot inflate blob - another operation in progress\n"); 6658 ctx->bserrno = -EBUSY; 6659 spdk_blob_close(_blob, bs_clone_snapshot_cleanup_finish, ctx); 6660 return; 6661 } 6662 6663 _blob->locked_operation_in_progress = true; 6664 6665 if (!ctx->allocate_all && _blob->parent_id == SPDK_BLOBID_INVALID) { 6666 /* This blob have no parent, so we cannot decouple it. */ 6667 SPDK_ERRLOG("Cannot decouple parent of blob with no parent.\n"); 6668 bs_clone_snapshot_origblob_cleanup(ctx, -EINVAL); 6669 return; 6670 } 6671 6672 if (spdk_blob_is_thin_provisioned(_blob) == false) { 6673 /* This is not thin provisioned blob. No need to inflate. */ 6674 bs_clone_snapshot_origblob_cleanup(ctx, 0); 6675 return; 6676 } 6677 6678 /* Do two passes - one to verify that we can obtain enough clusters 6679 * and another to actually claim them. 6680 */ 6681 clusters_needed = 0; 6682 for (i = 0; i < _blob->active.num_clusters; i++) { 6683 if (bs_cluster_needs_allocation(_blob, i, ctx->allocate_all)) { 6684 clusters_needed++; 6685 } 6686 } 6687 6688 if (clusters_needed > _blob->bs->num_free_clusters) { 6689 /* Not enough free clusters. Cannot satisfy the request. */ 6690 bs_clone_snapshot_origblob_cleanup(ctx, -ENOSPC); 6691 return; 6692 } 6693 6694 ctx->cluster = 0; 6695 bs_inflate_blob_touch_next(ctx, 0); 6696 } 6697 6698 static void 6699 bs_inflate_blob(struct spdk_blob_store *bs, struct spdk_io_channel *channel, 6700 spdk_blob_id blobid, bool allocate_all, spdk_blob_op_complete cb_fn, void *cb_arg) 6701 { 6702 struct spdk_clone_snapshot_ctx *ctx = calloc(1, sizeof(*ctx)); 6703 6704 if (!ctx) { 6705 cb_fn(cb_arg, -ENOMEM); 6706 return; 6707 } 6708 ctx->cpl.type = SPDK_BS_CPL_TYPE_BLOB_BASIC; 6709 ctx->cpl.u.bs_basic.cb_fn = cb_fn; 6710 ctx->cpl.u.bs_basic.cb_arg = cb_arg; 6711 ctx->bserrno = 0; 6712 ctx->original.id = blobid; 6713 ctx->channel = channel; 6714 ctx->allocate_all = allocate_all; 6715 6716 spdk_bs_open_blob(bs, ctx->original.id, bs_inflate_blob_open_cpl, ctx); 6717 } 6718 6719 void 6720 spdk_bs_inflate_blob(struct spdk_blob_store *bs, struct spdk_io_channel *channel, 6721 spdk_blob_id blobid, spdk_blob_op_complete cb_fn, void *cb_arg) 6722 { 6723 bs_inflate_blob(bs, channel, blobid, true, cb_fn, cb_arg); 6724 } 6725 6726 void 6727 spdk_bs_blob_decouple_parent(struct spdk_blob_store *bs, struct spdk_io_channel *channel, 6728 spdk_blob_id blobid, spdk_blob_op_complete cb_fn, void *cb_arg) 6729 { 6730 bs_inflate_blob(bs, channel, blobid, false, cb_fn, cb_arg); 6731 } 6732 /* END spdk_bs_inflate_blob */ 6733 6734 /* START spdk_blob_resize */ 6735 struct spdk_bs_resize_ctx { 6736 spdk_blob_op_complete cb_fn; 6737 void *cb_arg; 6738 struct spdk_blob *blob; 6739 uint64_t sz; 6740 int rc; 6741 }; 6742 6743 static void 6744 bs_resize_unfreeze_cpl(void *cb_arg, int rc) 6745 { 6746 struct spdk_bs_resize_ctx *ctx = (struct spdk_bs_resize_ctx *)cb_arg; 6747 6748 if (rc != 0) { 6749 SPDK_ERRLOG("Unfreeze failed, rc=%d\n", rc); 6750 } 6751 6752 if (ctx->rc != 0) { 6753 SPDK_ERRLOG("Unfreeze failed, ctx->rc=%d\n", ctx->rc); 6754 rc = ctx->rc; 6755 } 6756 6757 ctx->blob->locked_operation_in_progress = false; 6758 6759 ctx->cb_fn(ctx->cb_arg, rc); 6760 free(ctx); 6761 } 6762 6763 static void 6764 bs_resize_freeze_cpl(void *cb_arg, int rc) 6765 { 6766 struct spdk_bs_resize_ctx *ctx = (struct spdk_bs_resize_ctx *)cb_arg; 6767 6768 if (rc != 0) { 6769 ctx->blob->locked_operation_in_progress = false; 6770 ctx->cb_fn(ctx->cb_arg, rc); 6771 free(ctx); 6772 return; 6773 } 6774 6775 ctx->rc = blob_resize(ctx->blob, ctx->sz); 6776 6777 blob_unfreeze_io(ctx->blob, bs_resize_unfreeze_cpl, ctx); 6778 } 6779 6780 void 6781 spdk_blob_resize(struct spdk_blob *blob, uint64_t sz, spdk_blob_op_complete cb_fn, void *cb_arg) 6782 { 6783 struct spdk_bs_resize_ctx *ctx; 6784 6785 blob_verify_md_op(blob); 6786 6787 SPDK_DEBUGLOG(blob, "Resizing blob 0x%" PRIx64 " to %" PRIu64 " clusters\n", blob->id, sz); 6788 6789 if (blob->md_ro) { 6790 cb_fn(cb_arg, -EPERM); 6791 return; 6792 } 6793 6794 if (sz == blob->active.num_clusters) { 6795 cb_fn(cb_arg, 0); 6796 return; 6797 } 6798 6799 if (blob->locked_operation_in_progress) { 6800 cb_fn(cb_arg, -EBUSY); 6801 return; 6802 } 6803 6804 ctx = calloc(1, sizeof(*ctx)); 6805 if (!ctx) { 6806 cb_fn(cb_arg, -ENOMEM); 6807 return; 6808 } 6809 6810 blob->locked_operation_in_progress = true; 6811 ctx->cb_fn = cb_fn; 6812 ctx->cb_arg = cb_arg; 6813 ctx->blob = blob; 6814 ctx->sz = sz; 6815 blob_freeze_io(blob, bs_resize_freeze_cpl, ctx); 6816 } 6817 6818 /* END spdk_blob_resize */ 6819 6820 6821 /* START spdk_bs_delete_blob */ 6822 6823 static void 6824 bs_delete_close_cpl(void *cb_arg, int bserrno) 6825 { 6826 spdk_bs_sequence_t *seq = cb_arg; 6827 6828 bs_sequence_finish(seq, bserrno); 6829 } 6830 6831 static void 6832 bs_delete_persist_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 6833 { 6834 struct spdk_blob *blob = cb_arg; 6835 6836 if (bserrno != 0) { 6837 /* 6838 * We already removed this blob from the blobstore tailq, so 6839 * we need to free it here since this is the last reference 6840 * to it. 6841 */ 6842 blob_free(blob); 6843 bs_delete_close_cpl(seq, bserrno); 6844 return; 6845 } 6846 6847 /* 6848 * This will immediately decrement the ref_count and call 6849 * the completion routine since the metadata state is clean. 6850 * By calling spdk_blob_close, we reduce the number of call 6851 * points into code that touches the blob->open_ref count 6852 * and the blobstore's blob list. 6853 */ 6854 spdk_blob_close(blob, bs_delete_close_cpl, seq); 6855 } 6856 6857 struct delete_snapshot_ctx { 6858 struct spdk_blob_list *parent_snapshot_entry; 6859 struct spdk_blob *snapshot; 6860 struct spdk_blob_md_page *page; 6861 bool snapshot_md_ro; 6862 struct spdk_blob *clone; 6863 bool clone_md_ro; 6864 spdk_blob_op_with_handle_complete cb_fn; 6865 void *cb_arg; 6866 int bserrno; 6867 uint32_t next_extent_page; 6868 }; 6869 6870 static void 6871 delete_blob_cleanup_finish(void *cb_arg, int bserrno) 6872 { 6873 struct delete_snapshot_ctx *ctx = cb_arg; 6874 6875 if (bserrno != 0) { 6876 SPDK_ERRLOG("Snapshot cleanup error %d\n", bserrno); 6877 } 6878 6879 assert(ctx != NULL); 6880 6881 if (bserrno != 0 && ctx->bserrno == 0) { 6882 ctx->bserrno = bserrno; 6883 } 6884 6885 ctx->cb_fn(ctx->cb_arg, ctx->snapshot, ctx->bserrno); 6886 spdk_free(ctx->page); 6887 free(ctx); 6888 } 6889 6890 static void 6891 delete_snapshot_cleanup_snapshot(void *cb_arg, int bserrno) 6892 { 6893 struct delete_snapshot_ctx *ctx = cb_arg; 6894 6895 if (bserrno != 0) { 6896 ctx->bserrno = bserrno; 6897 SPDK_ERRLOG("Clone cleanup error %d\n", bserrno); 6898 } 6899 6900 if (ctx->bserrno != 0) { 6901 assert(blob_lookup(ctx->snapshot->bs, ctx->snapshot->id) == NULL); 6902 RB_INSERT(spdk_blob_tree, &ctx->snapshot->bs->open_blobs, ctx->snapshot); 6903 spdk_bit_array_set(ctx->snapshot->bs->open_blobids, ctx->snapshot->id); 6904 } 6905 6906 ctx->snapshot->locked_operation_in_progress = false; 6907 ctx->snapshot->md_ro = ctx->snapshot_md_ro; 6908 6909 spdk_blob_close(ctx->snapshot, delete_blob_cleanup_finish, ctx); 6910 } 6911 6912 static void 6913 delete_snapshot_cleanup_clone(void *cb_arg, int bserrno) 6914 { 6915 struct delete_snapshot_ctx *ctx = cb_arg; 6916 6917 ctx->clone->locked_operation_in_progress = false; 6918 ctx->clone->md_ro = ctx->clone_md_ro; 6919 6920 spdk_blob_close(ctx->clone, delete_snapshot_cleanup_snapshot, ctx); 6921 } 6922 6923 static void 6924 delete_snapshot_unfreeze_cpl(void *cb_arg, int bserrno) 6925 { 6926 struct delete_snapshot_ctx *ctx = cb_arg; 6927 6928 if (bserrno) { 6929 ctx->bserrno = bserrno; 6930 delete_snapshot_cleanup_clone(ctx, 0); 6931 return; 6932 } 6933 6934 ctx->clone->locked_operation_in_progress = false; 6935 spdk_blob_close(ctx->clone, delete_blob_cleanup_finish, ctx); 6936 } 6937 6938 static void 6939 delete_snapshot_sync_snapshot_cpl(void *cb_arg, int bserrno) 6940 { 6941 struct delete_snapshot_ctx *ctx = cb_arg; 6942 struct spdk_blob_list *parent_snapshot_entry = NULL; 6943 struct spdk_blob_list *snapshot_entry = NULL; 6944 struct spdk_blob_list *clone_entry = NULL; 6945 struct spdk_blob_list *snapshot_clone_entry = NULL; 6946 6947 if (bserrno) { 6948 SPDK_ERRLOG("Failed to sync MD on blob\n"); 6949 ctx->bserrno = bserrno; 6950 delete_snapshot_cleanup_clone(ctx, 0); 6951 return; 6952 } 6953 6954 /* Get snapshot entry for the snapshot we want to remove */ 6955 snapshot_entry = bs_get_snapshot_entry(ctx->snapshot->bs, ctx->snapshot->id); 6956 6957 assert(snapshot_entry != NULL); 6958 6959 /* Remove clone entry in this snapshot (at this point there can be only one clone) */ 6960 clone_entry = TAILQ_FIRST(&snapshot_entry->clones); 6961 assert(clone_entry != NULL); 6962 TAILQ_REMOVE(&snapshot_entry->clones, clone_entry, link); 6963 snapshot_entry->clone_count--; 6964 assert(TAILQ_EMPTY(&snapshot_entry->clones)); 6965 6966 if (ctx->snapshot->parent_id != SPDK_BLOBID_INVALID) { 6967 /* This snapshot is at the same time a clone of another snapshot - we need to 6968 * update parent snapshot (remove current clone, add new one inherited from 6969 * the snapshot that is being removed) */ 6970 6971 /* Get snapshot entry for parent snapshot and clone entry within that snapshot for 6972 * snapshot that we are removing */ 6973 blob_get_snapshot_and_clone_entries(ctx->snapshot, &parent_snapshot_entry, 6974 &snapshot_clone_entry); 6975 6976 /* Switch clone entry in parent snapshot */ 6977 TAILQ_INSERT_TAIL(&parent_snapshot_entry->clones, clone_entry, link); 6978 TAILQ_REMOVE(&parent_snapshot_entry->clones, snapshot_clone_entry, link); 6979 free(snapshot_clone_entry); 6980 } else { 6981 /* No parent snapshot - just remove clone entry */ 6982 free(clone_entry); 6983 } 6984 6985 /* Restore md_ro flags */ 6986 ctx->clone->md_ro = ctx->clone_md_ro; 6987 ctx->snapshot->md_ro = ctx->snapshot_md_ro; 6988 6989 blob_unfreeze_io(ctx->clone, delete_snapshot_unfreeze_cpl, ctx); 6990 } 6991 6992 static void 6993 delete_snapshot_sync_clone_cpl(void *cb_arg, int bserrno) 6994 { 6995 struct delete_snapshot_ctx *ctx = cb_arg; 6996 uint64_t i; 6997 6998 ctx->snapshot->md_ro = false; 6999 7000 if (bserrno) { 7001 SPDK_ERRLOG("Failed to sync MD on clone\n"); 7002 ctx->bserrno = bserrno; 7003 7004 /* Restore snapshot to previous state */ 7005 bserrno = blob_remove_xattr(ctx->snapshot, SNAPSHOT_PENDING_REMOVAL, true); 7006 if (bserrno != 0) { 7007 delete_snapshot_cleanup_clone(ctx, bserrno); 7008 return; 7009 } 7010 7011 spdk_blob_sync_md(ctx->snapshot, delete_snapshot_cleanup_clone, ctx); 7012 return; 7013 } 7014 7015 /* Clear cluster map entries for snapshot */ 7016 for (i = 0; i < ctx->snapshot->active.num_clusters && i < ctx->clone->active.num_clusters; i++) { 7017 if (ctx->clone->active.clusters[i] == ctx->snapshot->active.clusters[i]) { 7018 ctx->snapshot->active.clusters[i] = 0; 7019 } 7020 } 7021 for (i = 0; i < ctx->snapshot->active.num_extent_pages && 7022 i < ctx->clone->active.num_extent_pages; i++) { 7023 if (ctx->clone->active.extent_pages[i] == ctx->snapshot->active.extent_pages[i]) { 7024 ctx->snapshot->active.extent_pages[i] = 0; 7025 } 7026 } 7027 7028 blob_set_thin_provision(ctx->snapshot); 7029 ctx->snapshot->state = SPDK_BLOB_STATE_DIRTY; 7030 7031 if (ctx->parent_snapshot_entry != NULL) { 7032 ctx->snapshot->back_bs_dev = NULL; 7033 } 7034 7035 spdk_blob_sync_md(ctx->snapshot, delete_snapshot_sync_snapshot_cpl, ctx); 7036 } 7037 7038 static void 7039 delete_snapshot_update_extent_pages_cpl(struct delete_snapshot_ctx *ctx) 7040 { 7041 /* Delete old backing bs_dev from clone (related to snapshot that will be removed) */ 7042 ctx->clone->back_bs_dev->destroy(ctx->clone->back_bs_dev); 7043 7044 /* Set/remove snapshot xattr and switch parent ID and backing bs_dev on clone... */ 7045 if (ctx->parent_snapshot_entry != NULL) { 7046 /* ...to parent snapshot */ 7047 ctx->clone->parent_id = ctx->parent_snapshot_entry->id; 7048 ctx->clone->back_bs_dev = ctx->snapshot->back_bs_dev; 7049 blob_set_xattr(ctx->clone, BLOB_SNAPSHOT, &ctx->parent_snapshot_entry->id, 7050 sizeof(spdk_blob_id), 7051 true); 7052 } else { 7053 /* ...to blobid invalid and zeroes dev */ 7054 ctx->clone->parent_id = SPDK_BLOBID_INVALID; 7055 ctx->clone->back_bs_dev = bs_create_zeroes_dev(); 7056 blob_remove_xattr(ctx->clone, BLOB_SNAPSHOT, true); 7057 } 7058 7059 spdk_blob_sync_md(ctx->clone, delete_snapshot_sync_clone_cpl, ctx); 7060 } 7061 7062 static void 7063 delete_snapshot_update_extent_pages(void *cb_arg, int bserrno) 7064 { 7065 struct delete_snapshot_ctx *ctx = cb_arg; 7066 uint32_t *extent_page; 7067 uint64_t i; 7068 7069 for (i = ctx->next_extent_page; i < ctx->snapshot->active.num_extent_pages && 7070 i < ctx->clone->active.num_extent_pages; i++) { 7071 if (ctx->snapshot->active.extent_pages[i] == 0) { 7072 /* No extent page to use from snapshot */ 7073 continue; 7074 } 7075 7076 extent_page = &ctx->clone->active.extent_pages[i]; 7077 if (*extent_page == 0) { 7078 /* Copy extent page from snapshot when clone did not have a matching one */ 7079 *extent_page = ctx->snapshot->active.extent_pages[i]; 7080 continue; 7081 } 7082 7083 /* Clone and snapshot both contain partially filled matching extent pages. 7084 * Update the clone extent page in place with cluster map containing the mix of both. */ 7085 ctx->next_extent_page = i + 1; 7086 memset(ctx->page, 0, SPDK_BS_PAGE_SIZE); 7087 7088 blob_write_extent_page(ctx->clone, *extent_page, i * SPDK_EXTENTS_PER_EP, ctx->page, 7089 delete_snapshot_update_extent_pages, ctx); 7090 return; 7091 } 7092 delete_snapshot_update_extent_pages_cpl(ctx); 7093 } 7094 7095 static void 7096 delete_snapshot_sync_snapshot_xattr_cpl(void *cb_arg, int bserrno) 7097 { 7098 struct delete_snapshot_ctx *ctx = cb_arg; 7099 uint64_t i; 7100 7101 /* Temporarily override md_ro flag for clone for MD modification */ 7102 ctx->clone_md_ro = ctx->clone->md_ro; 7103 ctx->clone->md_ro = false; 7104 7105 if (bserrno) { 7106 SPDK_ERRLOG("Failed to sync MD with xattr on blob\n"); 7107 ctx->bserrno = bserrno; 7108 delete_snapshot_cleanup_clone(ctx, 0); 7109 return; 7110 } 7111 7112 /* Copy snapshot map to clone map (only unallocated clusters in clone) */ 7113 for (i = 0; i < ctx->snapshot->active.num_clusters && i < ctx->clone->active.num_clusters; i++) { 7114 if (ctx->clone->active.clusters[i] == 0) { 7115 ctx->clone->active.clusters[i] = ctx->snapshot->active.clusters[i]; 7116 } 7117 } 7118 ctx->next_extent_page = 0; 7119 delete_snapshot_update_extent_pages(ctx, 0); 7120 } 7121 7122 static void 7123 delete_snapshot_freeze_io_cb(void *cb_arg, int bserrno) 7124 { 7125 struct delete_snapshot_ctx *ctx = cb_arg; 7126 7127 if (bserrno) { 7128 SPDK_ERRLOG("Failed to freeze I/O on clone\n"); 7129 ctx->bserrno = bserrno; 7130 delete_snapshot_cleanup_clone(ctx, 0); 7131 return; 7132 } 7133 7134 /* Temporarily override md_ro flag for snapshot for MD modification */ 7135 ctx->snapshot_md_ro = ctx->snapshot->md_ro; 7136 ctx->snapshot->md_ro = false; 7137 7138 /* Mark blob as pending for removal for power failure safety, use clone id for recovery */ 7139 ctx->bserrno = blob_set_xattr(ctx->snapshot, SNAPSHOT_PENDING_REMOVAL, &ctx->clone->id, 7140 sizeof(spdk_blob_id), true); 7141 if (ctx->bserrno != 0) { 7142 delete_snapshot_cleanup_clone(ctx, 0); 7143 return; 7144 } 7145 7146 spdk_blob_sync_md(ctx->snapshot, delete_snapshot_sync_snapshot_xattr_cpl, ctx); 7147 } 7148 7149 static void 7150 delete_snapshot_open_clone_cb(void *cb_arg, struct spdk_blob *clone, int bserrno) 7151 { 7152 struct delete_snapshot_ctx *ctx = cb_arg; 7153 7154 if (bserrno) { 7155 SPDK_ERRLOG("Failed to open clone\n"); 7156 ctx->bserrno = bserrno; 7157 delete_snapshot_cleanup_snapshot(ctx, 0); 7158 return; 7159 } 7160 7161 ctx->clone = clone; 7162 7163 if (clone->locked_operation_in_progress) { 7164 SPDK_DEBUGLOG(blob, "Cannot remove blob - another operation in progress on its clone\n"); 7165 ctx->bserrno = -EBUSY; 7166 spdk_blob_close(ctx->clone, delete_snapshot_cleanup_snapshot, ctx); 7167 return; 7168 } 7169 7170 clone->locked_operation_in_progress = true; 7171 7172 blob_freeze_io(clone, delete_snapshot_freeze_io_cb, ctx); 7173 } 7174 7175 static void 7176 update_clone_on_snapshot_deletion(struct spdk_blob *snapshot, struct delete_snapshot_ctx *ctx) 7177 { 7178 struct spdk_blob_list *snapshot_entry = NULL; 7179 struct spdk_blob_list *clone_entry = NULL; 7180 struct spdk_blob_list *snapshot_clone_entry = NULL; 7181 7182 /* Get snapshot entry for the snapshot we want to remove */ 7183 snapshot_entry = bs_get_snapshot_entry(snapshot->bs, snapshot->id); 7184 7185 assert(snapshot_entry != NULL); 7186 7187 /* Get clone of the snapshot (at this point there can be only one clone) */ 7188 clone_entry = TAILQ_FIRST(&snapshot_entry->clones); 7189 assert(snapshot_entry->clone_count == 1); 7190 assert(clone_entry != NULL); 7191 7192 /* Get snapshot entry for parent snapshot and clone entry within that snapshot for 7193 * snapshot that we are removing */ 7194 blob_get_snapshot_and_clone_entries(snapshot, &ctx->parent_snapshot_entry, 7195 &snapshot_clone_entry); 7196 7197 spdk_bs_open_blob(snapshot->bs, clone_entry->id, delete_snapshot_open_clone_cb, ctx); 7198 } 7199 7200 static void 7201 bs_delete_blob_finish(void *cb_arg, struct spdk_blob *blob, int bserrno) 7202 { 7203 spdk_bs_sequence_t *seq = cb_arg; 7204 struct spdk_blob_list *snapshot_entry = NULL; 7205 uint32_t page_num; 7206 7207 if (bserrno) { 7208 SPDK_ERRLOG("Failed to remove blob\n"); 7209 bs_sequence_finish(seq, bserrno); 7210 return; 7211 } 7212 7213 /* Remove snapshot from the list */ 7214 snapshot_entry = bs_get_snapshot_entry(blob->bs, blob->id); 7215 if (snapshot_entry != NULL) { 7216 TAILQ_REMOVE(&blob->bs->snapshots, snapshot_entry, link); 7217 free(snapshot_entry); 7218 } 7219 7220 page_num = bs_blobid_to_page(blob->id); 7221 spdk_bit_array_clear(blob->bs->used_blobids, page_num); 7222 blob->state = SPDK_BLOB_STATE_DIRTY; 7223 blob->active.num_pages = 0; 7224 blob_resize(blob, 0); 7225 7226 blob_persist(seq, blob, bs_delete_persist_cpl, blob); 7227 } 7228 7229 static int 7230 bs_is_blob_deletable(struct spdk_blob *blob, bool *update_clone) 7231 { 7232 struct spdk_blob_list *snapshot_entry = NULL; 7233 struct spdk_blob_list *clone_entry = NULL; 7234 struct spdk_blob *clone = NULL; 7235 bool has_one_clone = false; 7236 7237 /* Check if this is a snapshot with clones */ 7238 snapshot_entry = bs_get_snapshot_entry(blob->bs, blob->id); 7239 if (snapshot_entry != NULL) { 7240 if (snapshot_entry->clone_count > 1) { 7241 SPDK_ERRLOG("Cannot remove snapshot with more than one clone\n"); 7242 return -EBUSY; 7243 } else if (snapshot_entry->clone_count == 1) { 7244 has_one_clone = true; 7245 } 7246 } 7247 7248 /* Check if someone has this blob open (besides this delete context): 7249 * - open_ref = 1 - only this context opened blob, so it is ok to remove it 7250 * - open_ref <= 2 && has_one_clone = true - clone is holding snapshot 7251 * and that is ok, because we will update it accordingly */ 7252 if (blob->open_ref <= 2 && has_one_clone) { 7253 clone_entry = TAILQ_FIRST(&snapshot_entry->clones); 7254 assert(clone_entry != NULL); 7255 clone = blob_lookup(blob->bs, clone_entry->id); 7256 7257 if (blob->open_ref == 2 && clone == NULL) { 7258 /* Clone is closed and someone else opened this blob */ 7259 SPDK_ERRLOG("Cannot remove snapshot because it is open\n"); 7260 return -EBUSY; 7261 } 7262 7263 *update_clone = true; 7264 return 0; 7265 } 7266 7267 if (blob->open_ref > 1) { 7268 SPDK_ERRLOG("Cannot remove snapshot because it is open\n"); 7269 return -EBUSY; 7270 } 7271 7272 assert(has_one_clone == false); 7273 *update_clone = false; 7274 return 0; 7275 } 7276 7277 static void 7278 bs_delete_enomem_close_cpl(void *cb_arg, int bserrno) 7279 { 7280 spdk_bs_sequence_t *seq = cb_arg; 7281 7282 bs_sequence_finish(seq, -ENOMEM); 7283 } 7284 7285 static void 7286 bs_delete_open_cpl(void *cb_arg, struct spdk_blob *blob, int bserrno) 7287 { 7288 spdk_bs_sequence_t *seq = cb_arg; 7289 struct delete_snapshot_ctx *ctx; 7290 bool update_clone = false; 7291 7292 if (bserrno != 0) { 7293 bs_sequence_finish(seq, bserrno); 7294 return; 7295 } 7296 7297 blob_verify_md_op(blob); 7298 7299 ctx = calloc(1, sizeof(*ctx)); 7300 if (ctx == NULL) { 7301 spdk_blob_close(blob, bs_delete_enomem_close_cpl, seq); 7302 return; 7303 } 7304 7305 ctx->snapshot = blob; 7306 ctx->cb_fn = bs_delete_blob_finish; 7307 ctx->cb_arg = seq; 7308 7309 /* Check if blob can be removed and if it is a snapshot with clone on top of it */ 7310 ctx->bserrno = bs_is_blob_deletable(blob, &update_clone); 7311 if (ctx->bserrno) { 7312 spdk_blob_close(blob, delete_blob_cleanup_finish, ctx); 7313 return; 7314 } 7315 7316 if (blob->locked_operation_in_progress) { 7317 SPDK_DEBUGLOG(blob, "Cannot remove blob - another operation in progress\n"); 7318 ctx->bserrno = -EBUSY; 7319 spdk_blob_close(blob, delete_blob_cleanup_finish, ctx); 7320 return; 7321 } 7322 7323 blob->locked_operation_in_progress = true; 7324 7325 /* 7326 * Remove the blob from the blob_store list now, to ensure it does not 7327 * get returned after this point by blob_lookup(). 7328 */ 7329 spdk_bit_array_clear(blob->bs->open_blobids, blob->id); 7330 RB_REMOVE(spdk_blob_tree, &blob->bs->open_blobs, blob); 7331 7332 if (update_clone) { 7333 ctx->page = spdk_zmalloc(SPDK_BS_PAGE_SIZE, 0, NULL, SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA); 7334 if (!ctx->page) { 7335 ctx->bserrno = -ENOMEM; 7336 spdk_blob_close(blob, delete_blob_cleanup_finish, ctx); 7337 return; 7338 } 7339 /* This blob is a snapshot with active clone - update clone first */ 7340 update_clone_on_snapshot_deletion(blob, ctx); 7341 } else { 7342 /* This blob does not have any clones - just remove it */ 7343 bs_blob_list_remove(blob); 7344 bs_delete_blob_finish(seq, blob, 0); 7345 free(ctx); 7346 } 7347 } 7348 7349 void 7350 spdk_bs_delete_blob(struct spdk_blob_store *bs, spdk_blob_id blobid, 7351 spdk_blob_op_complete cb_fn, void *cb_arg) 7352 { 7353 struct spdk_bs_cpl cpl; 7354 spdk_bs_sequence_t *seq; 7355 7356 SPDK_DEBUGLOG(blob, "Deleting blob 0x%" PRIx64 "\n", blobid); 7357 7358 assert(spdk_get_thread() == bs->md_thread); 7359 7360 cpl.type = SPDK_BS_CPL_TYPE_BLOB_BASIC; 7361 cpl.u.blob_basic.cb_fn = cb_fn; 7362 cpl.u.blob_basic.cb_arg = cb_arg; 7363 7364 seq = bs_sequence_start(bs->md_channel, &cpl); 7365 if (!seq) { 7366 cb_fn(cb_arg, -ENOMEM); 7367 return; 7368 } 7369 7370 spdk_bs_open_blob(bs, blobid, bs_delete_open_cpl, seq); 7371 } 7372 7373 /* END spdk_bs_delete_blob */ 7374 7375 /* START spdk_bs_open_blob */ 7376 7377 static void 7378 bs_open_blob_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 7379 { 7380 struct spdk_blob *blob = cb_arg; 7381 struct spdk_blob *existing; 7382 7383 if (bserrno != 0) { 7384 blob_free(blob); 7385 seq->cpl.u.blob_handle.blob = NULL; 7386 bs_sequence_finish(seq, bserrno); 7387 return; 7388 } 7389 7390 existing = blob_lookup(blob->bs, blob->id); 7391 if (existing) { 7392 blob_free(blob); 7393 existing->open_ref++; 7394 seq->cpl.u.blob_handle.blob = existing; 7395 bs_sequence_finish(seq, 0); 7396 return; 7397 } 7398 7399 blob->open_ref++; 7400 7401 spdk_bit_array_set(blob->bs->open_blobids, blob->id); 7402 RB_INSERT(spdk_blob_tree, &blob->bs->open_blobs, blob); 7403 7404 bs_sequence_finish(seq, bserrno); 7405 } 7406 7407 static inline void 7408 blob_open_opts_copy(const struct spdk_blob_open_opts *src, struct spdk_blob_open_opts *dst) 7409 { 7410 #define FIELD_OK(field) \ 7411 offsetof(struct spdk_blob_open_opts, field) + sizeof(src->field) <= src->opts_size 7412 7413 #define SET_FIELD(field) \ 7414 if (FIELD_OK(field)) { \ 7415 dst->field = src->field; \ 7416 } \ 7417 7418 SET_FIELD(clear_method); 7419 7420 dst->opts_size = src->opts_size; 7421 7422 /* You should not remove this statement, but need to update the assert statement 7423 * if you add a new field, and also add a corresponding SET_FIELD statement */ 7424 SPDK_STATIC_ASSERT(sizeof(struct spdk_blob_open_opts) == 16, "Incorrect size"); 7425 7426 #undef FIELD_OK 7427 #undef SET_FIELD 7428 } 7429 7430 static void 7431 bs_open_blob(struct spdk_blob_store *bs, 7432 spdk_blob_id blobid, 7433 struct spdk_blob_open_opts *opts, 7434 spdk_blob_op_with_handle_complete cb_fn, 7435 void *cb_arg) 7436 { 7437 struct spdk_blob *blob; 7438 struct spdk_bs_cpl cpl; 7439 struct spdk_blob_open_opts opts_local; 7440 spdk_bs_sequence_t *seq; 7441 uint32_t page_num; 7442 7443 SPDK_DEBUGLOG(blob, "Opening blob 0x%" PRIx64 "\n", blobid); 7444 assert(spdk_get_thread() == bs->md_thread); 7445 7446 page_num = bs_blobid_to_page(blobid); 7447 if (spdk_bit_array_get(bs->used_blobids, page_num) == false) { 7448 /* Invalid blobid */ 7449 cb_fn(cb_arg, NULL, -ENOENT); 7450 return; 7451 } 7452 7453 blob = blob_lookup(bs, blobid); 7454 if (blob) { 7455 blob->open_ref++; 7456 cb_fn(cb_arg, blob, 0); 7457 return; 7458 } 7459 7460 blob = blob_alloc(bs, blobid); 7461 if (!blob) { 7462 cb_fn(cb_arg, NULL, -ENOMEM); 7463 return; 7464 } 7465 7466 spdk_blob_open_opts_init(&opts_local, sizeof(opts_local)); 7467 if (opts) { 7468 blob_open_opts_copy(opts, &opts_local); 7469 } 7470 7471 blob->clear_method = opts_local.clear_method; 7472 7473 cpl.type = SPDK_BS_CPL_TYPE_BLOB_HANDLE; 7474 cpl.u.blob_handle.cb_fn = cb_fn; 7475 cpl.u.blob_handle.cb_arg = cb_arg; 7476 cpl.u.blob_handle.blob = blob; 7477 7478 seq = bs_sequence_start(bs->md_channel, &cpl); 7479 if (!seq) { 7480 blob_free(blob); 7481 cb_fn(cb_arg, NULL, -ENOMEM); 7482 return; 7483 } 7484 7485 blob_load(seq, blob, bs_open_blob_cpl, blob); 7486 } 7487 7488 void 7489 spdk_bs_open_blob(struct spdk_blob_store *bs, spdk_blob_id blobid, 7490 spdk_blob_op_with_handle_complete cb_fn, void *cb_arg) 7491 { 7492 bs_open_blob(bs, blobid, NULL, cb_fn, cb_arg); 7493 } 7494 7495 void 7496 spdk_bs_open_blob_ext(struct spdk_blob_store *bs, spdk_blob_id blobid, 7497 struct spdk_blob_open_opts *opts, spdk_blob_op_with_handle_complete cb_fn, void *cb_arg) 7498 { 7499 bs_open_blob(bs, blobid, opts, cb_fn, cb_arg); 7500 } 7501 7502 /* END spdk_bs_open_blob */ 7503 7504 /* START spdk_blob_set_read_only */ 7505 int 7506 spdk_blob_set_read_only(struct spdk_blob *blob) 7507 { 7508 blob_verify_md_op(blob); 7509 7510 blob->data_ro_flags |= SPDK_BLOB_READ_ONLY; 7511 7512 blob->state = SPDK_BLOB_STATE_DIRTY; 7513 return 0; 7514 } 7515 /* END spdk_blob_set_read_only */ 7516 7517 /* START spdk_blob_sync_md */ 7518 7519 static void 7520 blob_sync_md_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 7521 { 7522 struct spdk_blob *blob = cb_arg; 7523 7524 if (bserrno == 0 && (blob->data_ro_flags & SPDK_BLOB_READ_ONLY)) { 7525 blob->data_ro = true; 7526 blob->md_ro = true; 7527 } 7528 7529 bs_sequence_finish(seq, bserrno); 7530 } 7531 7532 static void 7533 blob_sync_md(struct spdk_blob *blob, spdk_blob_op_complete cb_fn, void *cb_arg) 7534 { 7535 struct spdk_bs_cpl cpl; 7536 spdk_bs_sequence_t *seq; 7537 7538 cpl.type = SPDK_BS_CPL_TYPE_BLOB_BASIC; 7539 cpl.u.blob_basic.cb_fn = cb_fn; 7540 cpl.u.blob_basic.cb_arg = cb_arg; 7541 7542 seq = bs_sequence_start(blob->bs->md_channel, &cpl); 7543 if (!seq) { 7544 cb_fn(cb_arg, -ENOMEM); 7545 return; 7546 } 7547 7548 blob_persist(seq, blob, blob_sync_md_cpl, blob); 7549 } 7550 7551 void 7552 spdk_blob_sync_md(struct spdk_blob *blob, spdk_blob_op_complete cb_fn, void *cb_arg) 7553 { 7554 blob_verify_md_op(blob); 7555 7556 SPDK_DEBUGLOG(blob, "Syncing blob 0x%" PRIx64 "\n", blob->id); 7557 7558 if (blob->md_ro) { 7559 assert(blob->state == SPDK_BLOB_STATE_CLEAN); 7560 cb_fn(cb_arg, 0); 7561 return; 7562 } 7563 7564 blob_sync_md(blob, cb_fn, cb_arg); 7565 } 7566 7567 /* END spdk_blob_sync_md */ 7568 7569 struct spdk_blob_insert_cluster_ctx { 7570 struct spdk_thread *thread; 7571 struct spdk_blob *blob; 7572 uint32_t cluster_num; /* cluster index in blob */ 7573 uint32_t cluster; /* cluster on disk */ 7574 uint32_t extent_page; /* extent page on disk */ 7575 struct spdk_blob_md_page *page; /* preallocated extent page */ 7576 int rc; 7577 spdk_blob_op_complete cb_fn; 7578 void *cb_arg; 7579 }; 7580 7581 static void 7582 blob_insert_cluster_msg_cpl(void *arg) 7583 { 7584 struct spdk_blob_insert_cluster_ctx *ctx = arg; 7585 7586 ctx->cb_fn(ctx->cb_arg, ctx->rc); 7587 free(ctx); 7588 } 7589 7590 static void 7591 blob_insert_cluster_msg_cb(void *arg, int bserrno) 7592 { 7593 struct spdk_blob_insert_cluster_ctx *ctx = arg; 7594 7595 ctx->rc = bserrno; 7596 spdk_thread_send_msg(ctx->thread, blob_insert_cluster_msg_cpl, ctx); 7597 } 7598 7599 static void 7600 blob_insert_new_ep_cb(void *arg, int bserrno) 7601 { 7602 struct spdk_blob_insert_cluster_ctx *ctx = arg; 7603 uint32_t *extent_page; 7604 7605 extent_page = bs_cluster_to_extent_page(ctx->blob, ctx->cluster_num); 7606 *extent_page = ctx->extent_page; 7607 ctx->blob->state = SPDK_BLOB_STATE_DIRTY; 7608 blob_sync_md(ctx->blob, blob_insert_cluster_msg_cb, ctx); 7609 } 7610 7611 struct spdk_blob_write_extent_page_ctx { 7612 struct spdk_blob_store *bs; 7613 7614 uint32_t extent; 7615 struct spdk_blob_md_page *page; 7616 }; 7617 7618 static void 7619 blob_persist_extent_page_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 7620 { 7621 struct spdk_blob_write_extent_page_ctx *ctx = cb_arg; 7622 7623 free(ctx); 7624 bs_sequence_finish(seq, bserrno); 7625 } 7626 7627 static void 7628 blob_write_extent_page_ready(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 7629 { 7630 struct spdk_blob_write_extent_page_ctx *ctx = cb_arg; 7631 7632 if (bserrno != 0) { 7633 blob_persist_extent_page_cpl(seq, ctx, bserrno); 7634 return; 7635 } 7636 bs_sequence_write_dev(seq, ctx->page, bs_md_page_to_lba(ctx->bs, ctx->extent), 7637 bs_byte_to_lba(ctx->bs, SPDK_BS_PAGE_SIZE), 7638 blob_persist_extent_page_cpl, ctx); 7639 } 7640 7641 static void 7642 blob_write_extent_page(struct spdk_blob *blob, uint32_t extent, uint64_t cluster_num, 7643 struct spdk_blob_md_page *page, spdk_blob_op_complete cb_fn, void *cb_arg) 7644 { 7645 struct spdk_blob_write_extent_page_ctx *ctx; 7646 spdk_bs_sequence_t *seq; 7647 struct spdk_bs_cpl cpl; 7648 7649 ctx = calloc(1, sizeof(*ctx)); 7650 if (!ctx) { 7651 cb_fn(cb_arg, -ENOMEM); 7652 return; 7653 } 7654 ctx->bs = blob->bs; 7655 ctx->extent = extent; 7656 ctx->page = page; 7657 7658 cpl.type = SPDK_BS_CPL_TYPE_BLOB_BASIC; 7659 cpl.u.blob_basic.cb_fn = cb_fn; 7660 cpl.u.blob_basic.cb_arg = cb_arg; 7661 7662 seq = bs_sequence_start(blob->bs->md_channel, &cpl); 7663 if (!seq) { 7664 free(ctx); 7665 cb_fn(cb_arg, -ENOMEM); 7666 return; 7667 } 7668 7669 assert(page); 7670 page->next = SPDK_INVALID_MD_PAGE; 7671 page->id = blob->id; 7672 page->sequence_num = 0; 7673 7674 blob_serialize_extent_page(blob, cluster_num, page); 7675 7676 page->crc = blob_md_page_calc_crc(page); 7677 7678 assert(spdk_bit_array_get(blob->bs->used_md_pages, extent) == true); 7679 7680 bs_mark_dirty(seq, blob->bs, blob_write_extent_page_ready, ctx); 7681 } 7682 7683 static void 7684 blob_insert_cluster_msg(void *arg) 7685 { 7686 struct spdk_blob_insert_cluster_ctx *ctx = arg; 7687 uint32_t *extent_page; 7688 7689 ctx->rc = blob_insert_cluster(ctx->blob, ctx->cluster_num, ctx->cluster); 7690 if (ctx->rc != 0) { 7691 spdk_thread_send_msg(ctx->thread, blob_insert_cluster_msg_cpl, ctx); 7692 return; 7693 } 7694 7695 if (ctx->blob->use_extent_table == false) { 7696 /* Extent table is not used, proceed with sync of md that will only use extents_rle. */ 7697 ctx->blob->state = SPDK_BLOB_STATE_DIRTY; 7698 blob_sync_md(ctx->blob, blob_insert_cluster_msg_cb, ctx); 7699 return; 7700 } 7701 7702 extent_page = bs_cluster_to_extent_page(ctx->blob, ctx->cluster_num); 7703 if (*extent_page == 0) { 7704 /* Extent page requires allocation. 7705 * It was already claimed in the used_md_pages map and placed in ctx. */ 7706 assert(ctx->extent_page != 0); 7707 assert(spdk_bit_array_get(ctx->blob->bs->used_md_pages, ctx->extent_page) == true); 7708 blob_write_extent_page(ctx->blob, ctx->extent_page, ctx->cluster_num, ctx->page, 7709 blob_insert_new_ep_cb, ctx); 7710 } else { 7711 /* It is possible for original thread to allocate extent page for 7712 * different cluster in the same extent page. In such case proceed with 7713 * updating the existing extent page, but release the additional one. */ 7714 if (ctx->extent_page != 0) { 7715 spdk_spin_lock(&ctx->blob->bs->used_lock); 7716 assert(spdk_bit_array_get(ctx->blob->bs->used_md_pages, ctx->extent_page) == true); 7717 bs_release_md_page(ctx->blob->bs, ctx->extent_page); 7718 spdk_spin_unlock(&ctx->blob->bs->used_lock); 7719 ctx->extent_page = 0; 7720 } 7721 /* Extent page already allocated. 7722 * Every cluster allocation, requires just an update of single extent page. */ 7723 blob_write_extent_page(ctx->blob, *extent_page, ctx->cluster_num, ctx->page, 7724 blob_insert_cluster_msg_cb, ctx); 7725 } 7726 } 7727 7728 static void 7729 blob_insert_cluster_on_md_thread(struct spdk_blob *blob, uint32_t cluster_num, 7730 uint64_t cluster, uint32_t extent_page, struct spdk_blob_md_page *page, 7731 spdk_blob_op_complete cb_fn, void *cb_arg) 7732 { 7733 struct spdk_blob_insert_cluster_ctx *ctx; 7734 7735 ctx = calloc(1, sizeof(*ctx)); 7736 if (ctx == NULL) { 7737 cb_fn(cb_arg, -ENOMEM); 7738 return; 7739 } 7740 7741 ctx->thread = spdk_get_thread(); 7742 ctx->blob = blob; 7743 ctx->cluster_num = cluster_num; 7744 ctx->cluster = cluster; 7745 ctx->extent_page = extent_page; 7746 ctx->page = page; 7747 ctx->cb_fn = cb_fn; 7748 ctx->cb_arg = cb_arg; 7749 7750 spdk_thread_send_msg(blob->bs->md_thread, blob_insert_cluster_msg, ctx); 7751 } 7752 7753 /* START spdk_blob_close */ 7754 7755 static void 7756 blob_close_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 7757 { 7758 struct spdk_blob *blob = cb_arg; 7759 7760 if (bserrno == 0) { 7761 blob->open_ref--; 7762 if (blob->open_ref == 0) { 7763 /* 7764 * Blobs with active.num_pages == 0 are deleted blobs. 7765 * these blobs are removed from the blob_store list 7766 * when the deletion process starts - so don't try to 7767 * remove them again. 7768 */ 7769 if (blob->active.num_pages > 0) { 7770 spdk_bit_array_clear(blob->bs->open_blobids, blob->id); 7771 RB_REMOVE(spdk_blob_tree, &blob->bs->open_blobs, blob); 7772 } 7773 blob_free(blob); 7774 } 7775 } 7776 7777 bs_sequence_finish(seq, bserrno); 7778 } 7779 7780 void 7781 spdk_blob_close(struct spdk_blob *blob, spdk_blob_op_complete cb_fn, void *cb_arg) 7782 { 7783 struct spdk_bs_cpl cpl; 7784 spdk_bs_sequence_t *seq; 7785 7786 blob_verify_md_op(blob); 7787 7788 SPDK_DEBUGLOG(blob, "Closing blob 0x%" PRIx64 "\n", blob->id); 7789 7790 if (blob->open_ref == 0) { 7791 cb_fn(cb_arg, -EBADF); 7792 return; 7793 } 7794 7795 cpl.type = SPDK_BS_CPL_TYPE_BLOB_BASIC; 7796 cpl.u.blob_basic.cb_fn = cb_fn; 7797 cpl.u.blob_basic.cb_arg = cb_arg; 7798 7799 seq = bs_sequence_start(blob->bs->md_channel, &cpl); 7800 if (!seq) { 7801 cb_fn(cb_arg, -ENOMEM); 7802 return; 7803 } 7804 7805 /* Sync metadata */ 7806 blob_persist(seq, blob, blob_close_cpl, blob); 7807 } 7808 7809 /* END spdk_blob_close */ 7810 7811 struct spdk_io_channel *spdk_bs_alloc_io_channel(struct spdk_blob_store *bs) 7812 { 7813 return spdk_get_io_channel(bs); 7814 } 7815 7816 void 7817 spdk_bs_free_io_channel(struct spdk_io_channel *channel) 7818 { 7819 spdk_put_io_channel(channel); 7820 } 7821 7822 void 7823 spdk_blob_io_unmap(struct spdk_blob *blob, struct spdk_io_channel *channel, 7824 uint64_t offset, uint64_t length, spdk_blob_op_complete cb_fn, void *cb_arg) 7825 { 7826 blob_request_submit_op(blob, channel, NULL, offset, length, cb_fn, cb_arg, 7827 SPDK_BLOB_UNMAP); 7828 } 7829 7830 void 7831 spdk_blob_io_write_zeroes(struct spdk_blob *blob, struct spdk_io_channel *channel, 7832 uint64_t offset, uint64_t length, spdk_blob_op_complete cb_fn, void *cb_arg) 7833 { 7834 blob_request_submit_op(blob, channel, NULL, offset, length, cb_fn, cb_arg, 7835 SPDK_BLOB_WRITE_ZEROES); 7836 } 7837 7838 void 7839 spdk_blob_io_write(struct spdk_blob *blob, struct spdk_io_channel *channel, 7840 void *payload, uint64_t offset, uint64_t length, 7841 spdk_blob_op_complete cb_fn, void *cb_arg) 7842 { 7843 blob_request_submit_op(blob, channel, payload, offset, length, cb_fn, cb_arg, 7844 SPDK_BLOB_WRITE); 7845 } 7846 7847 void 7848 spdk_blob_io_read(struct spdk_blob *blob, struct spdk_io_channel *channel, 7849 void *payload, uint64_t offset, uint64_t length, 7850 spdk_blob_op_complete cb_fn, void *cb_arg) 7851 { 7852 blob_request_submit_op(blob, channel, payload, offset, length, cb_fn, cb_arg, 7853 SPDK_BLOB_READ); 7854 } 7855 7856 void 7857 spdk_blob_io_writev(struct spdk_blob *blob, struct spdk_io_channel *channel, 7858 struct iovec *iov, int iovcnt, uint64_t offset, uint64_t length, 7859 spdk_blob_op_complete cb_fn, void *cb_arg) 7860 { 7861 blob_request_submit_rw_iov(blob, channel, iov, iovcnt, offset, length, cb_fn, cb_arg, false, NULL); 7862 } 7863 7864 void 7865 spdk_blob_io_readv(struct spdk_blob *blob, struct spdk_io_channel *channel, 7866 struct iovec *iov, int iovcnt, uint64_t offset, uint64_t length, 7867 spdk_blob_op_complete cb_fn, void *cb_arg) 7868 { 7869 blob_request_submit_rw_iov(blob, channel, iov, iovcnt, offset, length, cb_fn, cb_arg, true, NULL); 7870 } 7871 7872 void 7873 spdk_blob_io_writev_ext(struct spdk_blob *blob, struct spdk_io_channel *channel, 7874 struct iovec *iov, int iovcnt, uint64_t offset, uint64_t length, 7875 spdk_blob_op_complete cb_fn, void *cb_arg, struct spdk_blob_ext_io_opts *io_opts) 7876 { 7877 blob_request_submit_rw_iov(blob, channel, iov, iovcnt, offset, length, cb_fn, cb_arg, false, 7878 io_opts); 7879 } 7880 7881 void 7882 spdk_blob_io_readv_ext(struct spdk_blob *blob, struct spdk_io_channel *channel, 7883 struct iovec *iov, int iovcnt, uint64_t offset, uint64_t length, 7884 spdk_blob_op_complete cb_fn, void *cb_arg, struct spdk_blob_ext_io_opts *io_opts) 7885 { 7886 blob_request_submit_rw_iov(blob, channel, iov, iovcnt, offset, length, cb_fn, cb_arg, true, 7887 io_opts); 7888 } 7889 7890 struct spdk_bs_iter_ctx { 7891 int64_t page_num; 7892 struct spdk_blob_store *bs; 7893 7894 spdk_blob_op_with_handle_complete cb_fn; 7895 void *cb_arg; 7896 }; 7897 7898 static void 7899 bs_iter_cpl(void *cb_arg, struct spdk_blob *_blob, int bserrno) 7900 { 7901 struct spdk_bs_iter_ctx *ctx = cb_arg; 7902 struct spdk_blob_store *bs = ctx->bs; 7903 spdk_blob_id id; 7904 7905 if (bserrno == 0) { 7906 ctx->cb_fn(ctx->cb_arg, _blob, bserrno); 7907 free(ctx); 7908 return; 7909 } 7910 7911 ctx->page_num++; 7912 ctx->page_num = spdk_bit_array_find_first_set(bs->used_blobids, ctx->page_num); 7913 if (ctx->page_num >= spdk_bit_array_capacity(bs->used_blobids)) { 7914 ctx->cb_fn(ctx->cb_arg, NULL, -ENOENT); 7915 free(ctx); 7916 return; 7917 } 7918 7919 id = bs_page_to_blobid(ctx->page_num); 7920 7921 spdk_bs_open_blob(bs, id, bs_iter_cpl, ctx); 7922 } 7923 7924 void 7925 spdk_bs_iter_first(struct spdk_blob_store *bs, 7926 spdk_blob_op_with_handle_complete cb_fn, void *cb_arg) 7927 { 7928 struct spdk_bs_iter_ctx *ctx; 7929 7930 ctx = calloc(1, sizeof(*ctx)); 7931 if (!ctx) { 7932 cb_fn(cb_arg, NULL, -ENOMEM); 7933 return; 7934 } 7935 7936 ctx->page_num = -1; 7937 ctx->bs = bs; 7938 ctx->cb_fn = cb_fn; 7939 ctx->cb_arg = cb_arg; 7940 7941 bs_iter_cpl(ctx, NULL, -1); 7942 } 7943 7944 static void 7945 bs_iter_close_cpl(void *cb_arg, int bserrno) 7946 { 7947 struct spdk_bs_iter_ctx *ctx = cb_arg; 7948 7949 bs_iter_cpl(ctx, NULL, -1); 7950 } 7951 7952 void 7953 spdk_bs_iter_next(struct spdk_blob_store *bs, struct spdk_blob *blob, 7954 spdk_blob_op_with_handle_complete cb_fn, void *cb_arg) 7955 { 7956 struct spdk_bs_iter_ctx *ctx; 7957 7958 assert(blob != NULL); 7959 7960 ctx = calloc(1, sizeof(*ctx)); 7961 if (!ctx) { 7962 cb_fn(cb_arg, NULL, -ENOMEM); 7963 return; 7964 } 7965 7966 ctx->page_num = bs_blobid_to_page(blob->id); 7967 ctx->bs = bs; 7968 ctx->cb_fn = cb_fn; 7969 ctx->cb_arg = cb_arg; 7970 7971 /* Close the existing blob */ 7972 spdk_blob_close(blob, bs_iter_close_cpl, ctx); 7973 } 7974 7975 static int 7976 blob_set_xattr(struct spdk_blob *blob, const char *name, const void *value, 7977 uint16_t value_len, bool internal) 7978 { 7979 struct spdk_xattr_tailq *xattrs; 7980 struct spdk_xattr *xattr; 7981 size_t desc_size; 7982 void *tmp; 7983 7984 blob_verify_md_op(blob); 7985 7986 if (blob->md_ro) { 7987 return -EPERM; 7988 } 7989 7990 desc_size = sizeof(struct spdk_blob_md_descriptor_xattr) + strlen(name) + value_len; 7991 if (desc_size > SPDK_BS_MAX_DESC_SIZE) { 7992 SPDK_DEBUGLOG(blob, "Xattr '%s' of size %zu does not fix into single page %zu\n", name, 7993 desc_size, SPDK_BS_MAX_DESC_SIZE); 7994 return -ENOMEM; 7995 } 7996 7997 if (internal) { 7998 xattrs = &blob->xattrs_internal; 7999 blob->invalid_flags |= SPDK_BLOB_INTERNAL_XATTR; 8000 } else { 8001 xattrs = &blob->xattrs; 8002 } 8003 8004 TAILQ_FOREACH(xattr, xattrs, link) { 8005 if (!strcmp(name, xattr->name)) { 8006 tmp = malloc(value_len); 8007 if (!tmp) { 8008 return -ENOMEM; 8009 } 8010 8011 free(xattr->value); 8012 xattr->value_len = value_len; 8013 xattr->value = tmp; 8014 memcpy(xattr->value, value, value_len); 8015 8016 blob->state = SPDK_BLOB_STATE_DIRTY; 8017 8018 return 0; 8019 } 8020 } 8021 8022 xattr = calloc(1, sizeof(*xattr)); 8023 if (!xattr) { 8024 return -ENOMEM; 8025 } 8026 8027 xattr->name = strdup(name); 8028 if (!xattr->name) { 8029 free(xattr); 8030 return -ENOMEM; 8031 } 8032 8033 xattr->value_len = value_len; 8034 xattr->value = malloc(value_len); 8035 if (!xattr->value) { 8036 free(xattr->name); 8037 free(xattr); 8038 return -ENOMEM; 8039 } 8040 memcpy(xattr->value, value, value_len); 8041 TAILQ_INSERT_TAIL(xattrs, xattr, link); 8042 8043 blob->state = SPDK_BLOB_STATE_DIRTY; 8044 8045 return 0; 8046 } 8047 8048 int 8049 spdk_blob_set_xattr(struct spdk_blob *blob, const char *name, const void *value, 8050 uint16_t value_len) 8051 { 8052 return blob_set_xattr(blob, name, value, value_len, false); 8053 } 8054 8055 static int 8056 blob_remove_xattr(struct spdk_blob *blob, const char *name, bool internal) 8057 { 8058 struct spdk_xattr_tailq *xattrs; 8059 struct spdk_xattr *xattr; 8060 8061 blob_verify_md_op(blob); 8062 8063 if (blob->md_ro) { 8064 return -EPERM; 8065 } 8066 xattrs = internal ? &blob->xattrs_internal : &blob->xattrs; 8067 8068 TAILQ_FOREACH(xattr, xattrs, link) { 8069 if (!strcmp(name, xattr->name)) { 8070 TAILQ_REMOVE(xattrs, xattr, link); 8071 free(xattr->value); 8072 free(xattr->name); 8073 free(xattr); 8074 8075 if (internal && TAILQ_EMPTY(&blob->xattrs_internal)) { 8076 blob->invalid_flags &= ~SPDK_BLOB_INTERNAL_XATTR; 8077 } 8078 blob->state = SPDK_BLOB_STATE_DIRTY; 8079 8080 return 0; 8081 } 8082 } 8083 8084 return -ENOENT; 8085 } 8086 8087 int 8088 spdk_blob_remove_xattr(struct spdk_blob *blob, const char *name) 8089 { 8090 return blob_remove_xattr(blob, name, false); 8091 } 8092 8093 static int 8094 blob_get_xattr_value(struct spdk_blob *blob, const char *name, 8095 const void **value, size_t *value_len, bool internal) 8096 { 8097 struct spdk_xattr *xattr; 8098 struct spdk_xattr_tailq *xattrs; 8099 8100 xattrs = internal ? &blob->xattrs_internal : &blob->xattrs; 8101 8102 TAILQ_FOREACH(xattr, xattrs, link) { 8103 if (!strcmp(name, xattr->name)) { 8104 *value = xattr->value; 8105 *value_len = xattr->value_len; 8106 return 0; 8107 } 8108 } 8109 return -ENOENT; 8110 } 8111 8112 int 8113 spdk_blob_get_xattr_value(struct spdk_blob *blob, const char *name, 8114 const void **value, size_t *value_len) 8115 { 8116 blob_verify_md_op(blob); 8117 8118 return blob_get_xattr_value(blob, name, value, value_len, false); 8119 } 8120 8121 struct spdk_xattr_names { 8122 uint32_t count; 8123 const char *names[0]; 8124 }; 8125 8126 static int 8127 blob_get_xattr_names(struct spdk_xattr_tailq *xattrs, struct spdk_xattr_names **names) 8128 { 8129 struct spdk_xattr *xattr; 8130 int count = 0; 8131 8132 TAILQ_FOREACH(xattr, xattrs, link) { 8133 count++; 8134 } 8135 8136 *names = calloc(1, sizeof(struct spdk_xattr_names) + count * sizeof(char *)); 8137 if (*names == NULL) { 8138 return -ENOMEM; 8139 } 8140 8141 TAILQ_FOREACH(xattr, xattrs, link) { 8142 (*names)->names[(*names)->count++] = xattr->name; 8143 } 8144 8145 return 0; 8146 } 8147 8148 int 8149 spdk_blob_get_xattr_names(struct spdk_blob *blob, struct spdk_xattr_names **names) 8150 { 8151 blob_verify_md_op(blob); 8152 8153 return blob_get_xattr_names(&blob->xattrs, names); 8154 } 8155 8156 uint32_t 8157 spdk_xattr_names_get_count(struct spdk_xattr_names *names) 8158 { 8159 assert(names != NULL); 8160 8161 return names->count; 8162 } 8163 8164 const char * 8165 spdk_xattr_names_get_name(struct spdk_xattr_names *names, uint32_t index) 8166 { 8167 if (index >= names->count) { 8168 return NULL; 8169 } 8170 8171 return names->names[index]; 8172 } 8173 8174 void 8175 spdk_xattr_names_free(struct spdk_xattr_names *names) 8176 { 8177 free(names); 8178 } 8179 8180 struct spdk_bs_type 8181 spdk_bs_get_bstype(struct spdk_blob_store *bs) 8182 { 8183 return bs->bstype; 8184 } 8185 8186 void 8187 spdk_bs_set_bstype(struct spdk_blob_store *bs, struct spdk_bs_type bstype) 8188 { 8189 memcpy(&bs->bstype, &bstype, sizeof(bstype)); 8190 } 8191 8192 bool 8193 spdk_blob_is_read_only(struct spdk_blob *blob) 8194 { 8195 assert(blob != NULL); 8196 return (blob->data_ro || blob->md_ro); 8197 } 8198 8199 bool 8200 spdk_blob_is_snapshot(struct spdk_blob *blob) 8201 { 8202 struct spdk_blob_list *snapshot_entry; 8203 8204 assert(blob != NULL); 8205 8206 snapshot_entry = bs_get_snapshot_entry(blob->bs, blob->id); 8207 if (snapshot_entry == NULL) { 8208 return false; 8209 } 8210 8211 return true; 8212 } 8213 8214 bool 8215 spdk_blob_is_clone(struct spdk_blob *blob) 8216 { 8217 assert(blob != NULL); 8218 8219 if (blob->parent_id != SPDK_BLOBID_INVALID) { 8220 assert(spdk_blob_is_thin_provisioned(blob)); 8221 return true; 8222 } 8223 8224 return false; 8225 } 8226 8227 bool 8228 spdk_blob_is_thin_provisioned(struct spdk_blob *blob) 8229 { 8230 assert(blob != NULL); 8231 return !!(blob->invalid_flags & SPDK_BLOB_THIN_PROV); 8232 } 8233 8234 static void 8235 blob_update_clear_method(struct spdk_blob *blob) 8236 { 8237 enum blob_clear_method stored_cm; 8238 8239 assert(blob != NULL); 8240 8241 /* If BLOB_CLEAR_WITH_DEFAULT was passed in, use the setting stored 8242 * in metadata previously. If something other than the default was 8243 * specified, ignore stored value and used what was passed in. 8244 */ 8245 stored_cm = ((blob->md_ro_flags & SPDK_BLOB_CLEAR_METHOD) >> SPDK_BLOB_CLEAR_METHOD_SHIFT); 8246 8247 if (blob->clear_method == BLOB_CLEAR_WITH_DEFAULT) { 8248 blob->clear_method = stored_cm; 8249 } else if (blob->clear_method != stored_cm) { 8250 SPDK_WARNLOG("Using passed in clear method 0x%x instead of stored value of 0x%x\n", 8251 blob->clear_method, stored_cm); 8252 } 8253 } 8254 8255 spdk_blob_id 8256 spdk_blob_get_parent_snapshot(struct spdk_blob_store *bs, spdk_blob_id blob_id) 8257 { 8258 struct spdk_blob_list *snapshot_entry = NULL; 8259 struct spdk_blob_list *clone_entry = NULL; 8260 8261 TAILQ_FOREACH(snapshot_entry, &bs->snapshots, link) { 8262 TAILQ_FOREACH(clone_entry, &snapshot_entry->clones, link) { 8263 if (clone_entry->id == blob_id) { 8264 return snapshot_entry->id; 8265 } 8266 } 8267 } 8268 8269 return SPDK_BLOBID_INVALID; 8270 } 8271 8272 int 8273 spdk_blob_get_clones(struct spdk_blob_store *bs, spdk_blob_id blobid, spdk_blob_id *ids, 8274 size_t *count) 8275 { 8276 struct spdk_blob_list *snapshot_entry, *clone_entry; 8277 size_t n; 8278 8279 snapshot_entry = bs_get_snapshot_entry(bs, blobid); 8280 if (snapshot_entry == NULL) { 8281 *count = 0; 8282 return 0; 8283 } 8284 8285 if (ids == NULL || *count < snapshot_entry->clone_count) { 8286 *count = snapshot_entry->clone_count; 8287 return -ENOMEM; 8288 } 8289 *count = snapshot_entry->clone_count; 8290 8291 n = 0; 8292 TAILQ_FOREACH(clone_entry, &snapshot_entry->clones, link) { 8293 ids[n++] = clone_entry->id; 8294 } 8295 8296 return 0; 8297 } 8298 8299 static void 8300 bs_load_grow_continue(struct spdk_bs_load_ctx *ctx) 8301 { 8302 int rc; 8303 8304 if (ctx->super->size == 0) { 8305 ctx->super->size = ctx->bs->dev->blockcnt * ctx->bs->dev->blocklen; 8306 } 8307 8308 if (ctx->super->io_unit_size == 0) { 8309 ctx->super->io_unit_size = SPDK_BS_PAGE_SIZE; 8310 } 8311 8312 /* Parse the super block */ 8313 ctx->bs->clean = 1; 8314 ctx->bs->cluster_sz = ctx->super->cluster_size; 8315 ctx->bs->total_clusters = ctx->super->size / ctx->super->cluster_size; 8316 ctx->bs->pages_per_cluster = ctx->bs->cluster_sz / SPDK_BS_PAGE_SIZE; 8317 if (spdk_u32_is_pow2(ctx->bs->pages_per_cluster)) { 8318 ctx->bs->pages_per_cluster_shift = spdk_u32log2(ctx->bs->pages_per_cluster); 8319 } 8320 ctx->bs->io_unit_size = ctx->super->io_unit_size; 8321 rc = spdk_bit_array_resize(&ctx->used_clusters, ctx->bs->total_clusters); 8322 if (rc < 0) { 8323 bs_load_ctx_fail(ctx, -ENOMEM); 8324 return; 8325 } 8326 ctx->bs->md_start = ctx->super->md_start; 8327 ctx->bs->md_len = ctx->super->md_len; 8328 rc = spdk_bit_array_resize(&ctx->bs->open_blobids, ctx->bs->md_len); 8329 if (rc < 0) { 8330 bs_load_ctx_fail(ctx, -ENOMEM); 8331 return; 8332 } 8333 8334 ctx->bs->total_data_clusters = ctx->bs->total_clusters - spdk_divide_round_up( 8335 ctx->bs->md_start + ctx->bs->md_len, ctx->bs->pages_per_cluster); 8336 ctx->bs->super_blob = ctx->super->super_blob; 8337 memcpy(&ctx->bs->bstype, &ctx->super->bstype, sizeof(ctx->super->bstype)); 8338 8339 if (ctx->super->used_blobid_mask_len == 0 || ctx->super->clean == 0) { 8340 SPDK_ERRLOG("Can not grow an unclean blobstore, please load it normally to clean it.\n"); 8341 bs_load_ctx_fail(ctx, -EIO); 8342 return; 8343 } else { 8344 bs_load_read_used_pages(ctx); 8345 } 8346 } 8347 8348 static void 8349 bs_load_grow_super_write_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 8350 { 8351 struct spdk_bs_load_ctx *ctx = cb_arg; 8352 8353 if (bserrno != 0) { 8354 bs_load_ctx_fail(ctx, bserrno); 8355 return; 8356 } 8357 bs_load_grow_continue(ctx); 8358 } 8359 8360 static void 8361 bs_load_grow_used_clusters_write_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 8362 { 8363 struct spdk_bs_load_ctx *ctx = cb_arg; 8364 8365 if (bserrno != 0) { 8366 bs_load_ctx_fail(ctx, bserrno); 8367 return; 8368 } 8369 8370 spdk_free(ctx->mask); 8371 8372 bs_sequence_write_dev(ctx->seq, ctx->super, bs_page_to_lba(ctx->bs, 0), 8373 bs_byte_to_lba(ctx->bs, sizeof(*ctx->super)), 8374 bs_load_grow_super_write_cpl, ctx); 8375 } 8376 8377 static void 8378 bs_load_grow_used_clusters_read_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 8379 { 8380 struct spdk_bs_load_ctx *ctx = cb_arg; 8381 uint64_t lba, lba_count; 8382 uint64_t dev_size; 8383 uint64_t total_clusters; 8384 8385 if (bserrno != 0) { 8386 bs_load_ctx_fail(ctx, bserrno); 8387 return; 8388 } 8389 8390 /* The type must be correct */ 8391 assert(ctx->mask->type == SPDK_MD_MASK_TYPE_USED_CLUSTERS); 8392 /* The length of the mask (in bits) must not be greater than the length of the buffer (converted to bits) */ 8393 assert(ctx->mask->length <= (ctx->super->used_cluster_mask_len * sizeof( 8394 struct spdk_blob_md_page) * 8)); 8395 dev_size = ctx->bs->dev->blockcnt * ctx->bs->dev->blocklen; 8396 total_clusters = dev_size / ctx->super->cluster_size; 8397 ctx->mask->length = total_clusters; 8398 8399 lba = bs_page_to_lba(ctx->bs, ctx->super->used_cluster_mask_start); 8400 lba_count = bs_page_to_lba(ctx->bs, ctx->super->used_cluster_mask_len); 8401 bs_sequence_write_dev(ctx->seq, ctx->mask, lba, lba_count, 8402 bs_load_grow_used_clusters_write_cpl, ctx); 8403 } 8404 8405 static void 8406 bs_load_try_to_grow(struct spdk_bs_load_ctx *ctx) 8407 { 8408 uint64_t dev_size, total_clusters, used_cluster_mask_len, max_used_cluster_mask; 8409 uint64_t lba, lba_count, mask_size; 8410 8411 dev_size = ctx->bs->dev->blockcnt * ctx->bs->dev->blocklen; 8412 total_clusters = dev_size / ctx->super->cluster_size; 8413 used_cluster_mask_len = spdk_divide_round_up(sizeof(struct spdk_bs_md_mask) + 8414 spdk_divide_round_up(total_clusters, 8), 8415 SPDK_BS_PAGE_SIZE); 8416 max_used_cluster_mask = ctx->super->used_blobid_mask_start - ctx->super->used_cluster_mask_start; 8417 /* No necessary to grow or no space to grow */ 8418 if (ctx->super->size >= dev_size || used_cluster_mask_len > max_used_cluster_mask) { 8419 SPDK_DEBUGLOG(blob, "No grow\n"); 8420 bs_load_grow_continue(ctx); 8421 return; 8422 } 8423 8424 SPDK_DEBUGLOG(blob, "Resize blobstore\n"); 8425 8426 ctx->super->size = dev_size; 8427 ctx->super->used_cluster_mask_len = used_cluster_mask_len; 8428 ctx->super->crc = blob_md_page_calc_crc(ctx->super); 8429 8430 mask_size = used_cluster_mask_len * SPDK_BS_PAGE_SIZE; 8431 ctx->mask = spdk_zmalloc(mask_size, 0x1000, NULL, SPDK_ENV_SOCKET_ID_ANY, 8432 SPDK_MALLOC_DMA); 8433 if (!ctx->mask) { 8434 bs_load_ctx_fail(ctx, -ENOMEM); 8435 return; 8436 } 8437 lba = bs_page_to_lba(ctx->bs, ctx->super->used_cluster_mask_start); 8438 lba_count = bs_page_to_lba(ctx->bs, ctx->super->used_cluster_mask_len); 8439 bs_sequence_read_dev(ctx->seq, ctx->mask, lba, lba_count, 8440 bs_load_grow_used_clusters_read_cpl, ctx); 8441 } 8442 8443 static void 8444 bs_grow_load_super_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 8445 { 8446 struct spdk_bs_load_ctx *ctx = cb_arg; 8447 uint32_t crc; 8448 static const char zeros[SPDK_BLOBSTORE_TYPE_LENGTH]; 8449 8450 if (ctx->super->version > SPDK_BS_VERSION || 8451 ctx->super->version < SPDK_BS_INITIAL_VERSION) { 8452 bs_load_ctx_fail(ctx, -EILSEQ); 8453 return; 8454 } 8455 8456 if (memcmp(ctx->super->signature, SPDK_BS_SUPER_BLOCK_SIG, 8457 sizeof(ctx->super->signature)) != 0) { 8458 bs_load_ctx_fail(ctx, -EILSEQ); 8459 return; 8460 } 8461 8462 crc = blob_md_page_calc_crc(ctx->super); 8463 if (crc != ctx->super->crc) { 8464 bs_load_ctx_fail(ctx, -EILSEQ); 8465 return; 8466 } 8467 8468 if (memcmp(&ctx->bs->bstype, &ctx->super->bstype, SPDK_BLOBSTORE_TYPE_LENGTH) == 0) { 8469 SPDK_DEBUGLOG(blob, "Bstype matched - loading blobstore\n"); 8470 } else if (memcmp(&ctx->bs->bstype, zeros, SPDK_BLOBSTORE_TYPE_LENGTH) == 0) { 8471 SPDK_DEBUGLOG(blob, "Bstype wildcard used - loading blobstore regardless bstype\n"); 8472 } else { 8473 SPDK_DEBUGLOG(blob, "Unexpected bstype\n"); 8474 SPDK_LOGDUMP(blob, "Expected:", ctx->bs->bstype.bstype, SPDK_BLOBSTORE_TYPE_LENGTH); 8475 SPDK_LOGDUMP(blob, "Found:", ctx->super->bstype.bstype, SPDK_BLOBSTORE_TYPE_LENGTH); 8476 bs_load_ctx_fail(ctx, -ENXIO); 8477 return; 8478 } 8479 8480 if (ctx->super->size > ctx->bs->dev->blockcnt * ctx->bs->dev->blocklen) { 8481 SPDK_NOTICELOG("Size mismatch, dev size: %" PRIu64 ", blobstore size: %" PRIu64 "\n", 8482 ctx->bs->dev->blockcnt * ctx->bs->dev->blocklen, ctx->super->size); 8483 bs_load_ctx_fail(ctx, -EILSEQ); 8484 return; 8485 } 8486 8487 bs_load_try_to_grow(ctx); 8488 8489 } 8490 8491 void 8492 spdk_bs_grow(struct spdk_bs_dev *dev, struct spdk_bs_opts *o, 8493 spdk_bs_op_with_handle_complete cb_fn, void *cb_arg) 8494 { 8495 struct spdk_blob_store *bs; 8496 struct spdk_bs_cpl cpl; 8497 struct spdk_bs_load_ctx *ctx; 8498 struct spdk_bs_opts opts = {}; 8499 int err; 8500 8501 SPDK_DEBUGLOG(blob, "Loading blobstore from dev %p\n", dev); 8502 8503 if ((SPDK_BS_PAGE_SIZE % dev->blocklen) != 0) { 8504 SPDK_DEBUGLOG(blob, "unsupported dev block length of %d\n", dev->blocklen); 8505 dev->destroy(dev); 8506 cb_fn(cb_arg, NULL, -EINVAL); 8507 return; 8508 } 8509 8510 spdk_bs_opts_init(&opts, sizeof(opts)); 8511 if (o) { 8512 if (bs_opts_copy(o, &opts)) { 8513 return; 8514 } 8515 } 8516 8517 if (opts.max_md_ops == 0 || opts.max_channel_ops == 0) { 8518 dev->destroy(dev); 8519 cb_fn(cb_arg, NULL, -EINVAL); 8520 return; 8521 } 8522 8523 err = bs_alloc(dev, &opts, &bs, &ctx); 8524 if (err) { 8525 dev->destroy(dev); 8526 cb_fn(cb_arg, NULL, err); 8527 return; 8528 } 8529 8530 cpl.type = SPDK_BS_CPL_TYPE_BS_HANDLE; 8531 cpl.u.bs_handle.cb_fn = cb_fn; 8532 cpl.u.bs_handle.cb_arg = cb_arg; 8533 cpl.u.bs_handle.bs = bs; 8534 8535 ctx->seq = bs_sequence_start(bs->md_channel, &cpl); 8536 if (!ctx->seq) { 8537 spdk_free(ctx->super); 8538 free(ctx); 8539 bs_free(bs); 8540 cb_fn(cb_arg, NULL, -ENOMEM); 8541 return; 8542 } 8543 8544 /* Read the super block */ 8545 bs_sequence_read_dev(ctx->seq, ctx->super, bs_page_to_lba(bs, 0), 8546 bs_byte_to_lba(bs, sizeof(*ctx->super)), 8547 bs_grow_load_super_cpl, ctx); 8548 } 8549 8550 int 8551 spdk_blob_get_esnap_id(struct spdk_blob *blob, const void **id, size_t *len) 8552 { 8553 if (!blob_is_esnap_clone(blob)) { 8554 return -EINVAL; 8555 } 8556 8557 return blob_get_xattr_value(blob, BLOB_EXTERNAL_SNAPSHOT_ID, id, len, true); 8558 } 8559 8560 SPDK_LOG_REGISTER_COMPONENT(blob) 8561 SPDK_LOG_REGISTER_COMPONENT(blob_esnap) 8562