1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2017 Intel Corporation. 3 * All rights reserved. 4 * Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 5 */ 6 7 #include "spdk/stdinc.h" 8 9 #include "spdk/blob.h" 10 #include "spdk/crc32.h" 11 #include "spdk/env.h" 12 #include "spdk/queue.h" 13 #include "spdk/thread.h" 14 #include "spdk/bit_array.h" 15 #include "spdk/bit_pool.h" 16 #include "spdk/likely.h" 17 #include "spdk/util.h" 18 #include "spdk/string.h" 19 20 #include "spdk_internal/assert.h" 21 #include "spdk/log.h" 22 23 #include "blobstore.h" 24 25 #define BLOB_CRC32C_INITIAL 0xffffffffUL 26 27 static int bs_register_md_thread(struct spdk_blob_store *bs); 28 static int bs_unregister_md_thread(struct spdk_blob_store *bs); 29 static void blob_close_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno); 30 static void blob_insert_cluster_on_md_thread(struct spdk_blob *blob, uint32_t cluster_num, 31 uint64_t cluster, uint32_t extent, struct spdk_blob_md_page *page, 32 spdk_blob_op_complete cb_fn, void *cb_arg); 33 34 static int blob_set_xattr(struct spdk_blob *blob, const char *name, const void *value, 35 uint16_t value_len, bool internal); 36 static int blob_get_xattr_value(struct spdk_blob *blob, const char *name, 37 const void **value, size_t *value_len, bool internal); 38 static int blob_remove_xattr(struct spdk_blob *blob, const char *name, bool internal); 39 40 static void blob_write_extent_page(struct spdk_blob *blob, uint32_t extent, uint64_t cluster_num, 41 struct spdk_blob_md_page *page, spdk_blob_op_complete cb_fn, void *cb_arg); 42 43 /* 44 * External snapshots require a channel per thread per esnap bdev. The tree 45 * is populated lazily as blob IOs are handled by the back_bs_dev. When this 46 * channel is destroyed, all the channels in the tree are destroyed. 47 */ 48 49 struct blob_esnap_channel { 50 RB_ENTRY(blob_esnap_channel) node; 51 spdk_blob_id blob_id; 52 struct spdk_io_channel *channel; 53 }; 54 55 static int blob_esnap_channel_compare(struct blob_esnap_channel *c1, struct blob_esnap_channel *c2); 56 static void blob_esnap_destroy_bs_dev_channels(struct spdk_blob *blob, bool abort_io, 57 spdk_blob_op_with_handle_complete cb_fn, void *cb_arg); 58 static void blob_esnap_destroy_bs_channel(struct spdk_bs_channel *ch); 59 RB_GENERATE_STATIC(blob_esnap_channel_tree, blob_esnap_channel, node, blob_esnap_channel_compare) 60 61 static inline bool 62 blob_is_esnap_clone(const struct spdk_blob *blob) 63 { 64 assert(blob != NULL); 65 return !!(blob->invalid_flags & SPDK_BLOB_EXTERNAL_SNAPSHOT); 66 } 67 68 static int 69 blob_id_cmp(struct spdk_blob *blob1, struct spdk_blob *blob2) 70 { 71 return (blob1->id < blob2->id ? -1 : blob1->id > blob2->id); 72 } 73 74 RB_GENERATE_STATIC(spdk_blob_tree, spdk_blob, link, blob_id_cmp); 75 76 static void 77 blob_verify_md_op(struct spdk_blob *blob) 78 { 79 assert(blob != NULL); 80 assert(spdk_get_thread() == blob->bs->md_thread); 81 assert(blob->state != SPDK_BLOB_STATE_LOADING); 82 } 83 84 static struct spdk_blob_list * 85 bs_get_snapshot_entry(struct spdk_blob_store *bs, spdk_blob_id blobid) 86 { 87 struct spdk_blob_list *snapshot_entry = NULL; 88 89 TAILQ_FOREACH(snapshot_entry, &bs->snapshots, link) { 90 if (snapshot_entry->id == blobid) { 91 break; 92 } 93 } 94 95 return snapshot_entry; 96 } 97 98 static void 99 bs_claim_md_page(struct spdk_blob_store *bs, uint32_t page) 100 { 101 assert(spdk_spin_held(&bs->used_lock)); 102 assert(page < spdk_bit_array_capacity(bs->used_md_pages)); 103 assert(spdk_bit_array_get(bs->used_md_pages, page) == false); 104 105 spdk_bit_array_set(bs->used_md_pages, page); 106 } 107 108 static void 109 bs_release_md_page(struct spdk_blob_store *bs, uint32_t page) 110 { 111 assert(spdk_spin_held(&bs->used_lock)); 112 assert(page < spdk_bit_array_capacity(bs->used_md_pages)); 113 assert(spdk_bit_array_get(bs->used_md_pages, page) == true); 114 115 spdk_bit_array_clear(bs->used_md_pages, page); 116 } 117 118 static uint32_t 119 bs_claim_cluster(struct spdk_blob_store *bs) 120 { 121 uint32_t cluster_num; 122 123 assert(spdk_spin_held(&bs->used_lock)); 124 125 cluster_num = spdk_bit_pool_allocate_bit(bs->used_clusters); 126 if (cluster_num == UINT32_MAX) { 127 return UINT32_MAX; 128 } 129 130 SPDK_DEBUGLOG(blob, "Claiming cluster %u\n", cluster_num); 131 bs->num_free_clusters--; 132 133 return cluster_num; 134 } 135 136 static void 137 bs_release_cluster(struct spdk_blob_store *bs, uint32_t cluster_num) 138 { 139 assert(spdk_spin_held(&bs->used_lock)); 140 assert(cluster_num < spdk_bit_pool_capacity(bs->used_clusters)); 141 assert(spdk_bit_pool_is_allocated(bs->used_clusters, cluster_num) == true); 142 assert(bs->num_free_clusters < bs->total_clusters); 143 144 SPDK_DEBUGLOG(blob, "Releasing cluster %u\n", cluster_num); 145 146 spdk_bit_pool_free_bit(bs->used_clusters, cluster_num); 147 bs->num_free_clusters++; 148 } 149 150 static int 151 blob_insert_cluster(struct spdk_blob *blob, uint32_t cluster_num, uint64_t cluster) 152 { 153 uint64_t *cluster_lba = &blob->active.clusters[cluster_num]; 154 155 blob_verify_md_op(blob); 156 157 if (*cluster_lba != 0) { 158 return -EEXIST; 159 } 160 161 *cluster_lba = bs_cluster_to_lba(blob->bs, cluster); 162 return 0; 163 } 164 165 static int 166 bs_allocate_cluster(struct spdk_blob *blob, uint32_t cluster_num, 167 uint64_t *cluster, uint32_t *lowest_free_md_page, bool update_map) 168 { 169 uint32_t *extent_page = 0; 170 171 assert(spdk_spin_held(&blob->bs->used_lock)); 172 173 *cluster = bs_claim_cluster(blob->bs); 174 if (*cluster == UINT32_MAX) { 175 /* No more free clusters. Cannot satisfy the request */ 176 return -ENOSPC; 177 } 178 179 if (blob->use_extent_table) { 180 extent_page = bs_cluster_to_extent_page(blob, cluster_num); 181 if (*extent_page == 0) { 182 /* Extent page shall never occupy md_page so start the search from 1 */ 183 if (*lowest_free_md_page == 0) { 184 *lowest_free_md_page = 1; 185 } 186 /* No extent_page is allocated for the cluster */ 187 *lowest_free_md_page = spdk_bit_array_find_first_clear(blob->bs->used_md_pages, 188 *lowest_free_md_page); 189 if (*lowest_free_md_page == UINT32_MAX) { 190 /* No more free md pages. Cannot satisfy the request */ 191 bs_release_cluster(blob->bs, *cluster); 192 return -ENOSPC; 193 } 194 bs_claim_md_page(blob->bs, *lowest_free_md_page); 195 } 196 } 197 198 SPDK_DEBUGLOG(blob, "Claiming cluster %" PRIu64 " for blob 0x%" PRIx64 "\n", *cluster, 199 blob->id); 200 201 if (update_map) { 202 blob_insert_cluster(blob, cluster_num, *cluster); 203 if (blob->use_extent_table && *extent_page == 0) { 204 *extent_page = *lowest_free_md_page; 205 } 206 } 207 208 return 0; 209 } 210 211 static void 212 blob_xattrs_init(struct spdk_blob_xattr_opts *xattrs) 213 { 214 xattrs->count = 0; 215 xattrs->names = NULL; 216 xattrs->ctx = NULL; 217 xattrs->get_value = NULL; 218 } 219 220 void 221 spdk_blob_opts_init(struct spdk_blob_opts *opts, size_t opts_size) 222 { 223 if (!opts) { 224 SPDK_ERRLOG("opts should not be NULL\n"); 225 return; 226 } 227 228 if (!opts_size) { 229 SPDK_ERRLOG("opts_size should not be zero value\n"); 230 return; 231 } 232 233 memset(opts, 0, opts_size); 234 opts->opts_size = opts_size; 235 236 #define FIELD_OK(field) \ 237 offsetof(struct spdk_blob_opts, field) + sizeof(opts->field) <= opts_size 238 239 #define SET_FIELD(field, value) \ 240 if (FIELD_OK(field)) { \ 241 opts->field = value; \ 242 } \ 243 244 SET_FIELD(num_clusters, 0); 245 SET_FIELD(thin_provision, false); 246 SET_FIELD(clear_method, BLOB_CLEAR_WITH_DEFAULT); 247 248 if (FIELD_OK(xattrs)) { 249 blob_xattrs_init(&opts->xattrs); 250 } 251 252 SET_FIELD(use_extent_table, true); 253 254 #undef FIELD_OK 255 #undef SET_FIELD 256 } 257 258 void 259 spdk_blob_open_opts_init(struct spdk_blob_open_opts *opts, size_t opts_size) 260 { 261 if (!opts) { 262 SPDK_ERRLOG("opts should not be NULL\n"); 263 return; 264 } 265 266 if (!opts_size) { 267 SPDK_ERRLOG("opts_size should not be zero value\n"); 268 return; 269 } 270 271 memset(opts, 0, opts_size); 272 opts->opts_size = opts_size; 273 274 #define FIELD_OK(field) \ 275 offsetof(struct spdk_blob_open_opts, field) + sizeof(opts->field) <= opts_size 276 277 #define SET_FIELD(field, value) \ 278 if (FIELD_OK(field)) { \ 279 opts->field = value; \ 280 } \ 281 282 SET_FIELD(clear_method, BLOB_CLEAR_WITH_DEFAULT); 283 284 #undef FIELD_OK 285 #undef SET_FILED 286 } 287 288 static struct spdk_blob * 289 blob_alloc(struct spdk_blob_store *bs, spdk_blob_id id) 290 { 291 struct spdk_blob *blob; 292 293 blob = calloc(1, sizeof(*blob)); 294 if (!blob) { 295 return NULL; 296 } 297 298 blob->id = id; 299 blob->bs = bs; 300 301 blob->parent_id = SPDK_BLOBID_INVALID; 302 303 blob->state = SPDK_BLOB_STATE_DIRTY; 304 blob->extent_rle_found = false; 305 blob->extent_table_found = false; 306 blob->active.num_pages = 1; 307 blob->active.pages = calloc(1, sizeof(*blob->active.pages)); 308 if (!blob->active.pages) { 309 free(blob); 310 return NULL; 311 } 312 313 blob->active.pages[0] = bs_blobid_to_page(id); 314 315 TAILQ_INIT(&blob->xattrs); 316 TAILQ_INIT(&blob->xattrs_internal); 317 TAILQ_INIT(&blob->pending_persists); 318 TAILQ_INIT(&blob->persists_to_complete); 319 320 return blob; 321 } 322 323 static void 324 xattrs_free(struct spdk_xattr_tailq *xattrs) 325 { 326 struct spdk_xattr *xattr, *xattr_tmp; 327 328 TAILQ_FOREACH_SAFE(xattr, xattrs, link, xattr_tmp) { 329 TAILQ_REMOVE(xattrs, xattr, link); 330 free(xattr->name); 331 free(xattr->value); 332 free(xattr); 333 } 334 } 335 336 static void 337 blob_free(struct spdk_blob *blob) 338 { 339 assert(blob != NULL); 340 assert(TAILQ_EMPTY(&blob->pending_persists)); 341 assert(TAILQ_EMPTY(&blob->persists_to_complete)); 342 343 free(blob->active.extent_pages); 344 free(blob->clean.extent_pages); 345 free(blob->active.clusters); 346 free(blob->clean.clusters); 347 free(blob->active.pages); 348 free(blob->clean.pages); 349 350 xattrs_free(&blob->xattrs); 351 xattrs_free(&blob->xattrs_internal); 352 353 if (blob->back_bs_dev) { 354 blob->back_bs_dev->destroy(blob->back_bs_dev); 355 } 356 357 free(blob); 358 } 359 360 static void 361 blob_back_bs_destroy_esnap_done(void *ctx, struct spdk_blob *blob, int bserrno) 362 { 363 struct spdk_bs_dev *bs_dev = ctx; 364 365 if (bserrno != 0) { 366 /* 367 * This is probably due to a memory allocation failure when creating the 368 * blob_esnap_destroy_ctx before iterating threads. 369 */ 370 SPDK_ERRLOG("blob 0x%" PRIx64 ": Unable to destroy bs dev channels: error %d\n", 371 blob->id, bserrno); 372 assert(false); 373 } 374 375 SPDK_DEBUGLOG(blob_esnap, "blob 0x%" PRIx64 ": calling destroy on back_bs_dev\n", blob->id); 376 bs_dev->destroy(bs_dev); 377 } 378 379 static void 380 blob_back_bs_destroy(struct spdk_blob *blob) 381 { 382 SPDK_DEBUGLOG(blob_esnap, "blob 0x%" PRIx64 ": preparing to destroy back_bs_dev\n", 383 blob->id); 384 385 blob_esnap_destroy_bs_dev_channels(blob, false, blob_back_bs_destroy_esnap_done, 386 blob->back_bs_dev); 387 blob->back_bs_dev = NULL; 388 } 389 390 struct freeze_io_ctx { 391 struct spdk_bs_cpl cpl; 392 struct spdk_blob *blob; 393 }; 394 395 static void 396 blob_io_sync(struct spdk_io_channel_iter *i) 397 { 398 spdk_for_each_channel_continue(i, 0); 399 } 400 401 static void 402 blob_execute_queued_io(struct spdk_io_channel_iter *i) 403 { 404 struct spdk_io_channel *_ch = spdk_io_channel_iter_get_channel(i); 405 struct spdk_bs_channel *ch = spdk_io_channel_get_ctx(_ch); 406 struct freeze_io_ctx *ctx = spdk_io_channel_iter_get_ctx(i); 407 struct spdk_bs_request_set *set; 408 struct spdk_bs_user_op_args *args; 409 spdk_bs_user_op_t *op, *tmp; 410 411 TAILQ_FOREACH_SAFE(op, &ch->queued_io, link, tmp) { 412 set = (struct spdk_bs_request_set *)op; 413 args = &set->u.user_op; 414 415 if (args->blob == ctx->blob) { 416 TAILQ_REMOVE(&ch->queued_io, op, link); 417 bs_user_op_execute(op); 418 } 419 } 420 421 spdk_for_each_channel_continue(i, 0); 422 } 423 424 static void 425 blob_io_cpl(struct spdk_io_channel_iter *i, int status) 426 { 427 struct freeze_io_ctx *ctx = spdk_io_channel_iter_get_ctx(i); 428 429 ctx->cpl.u.blob_basic.cb_fn(ctx->cpl.u.blob_basic.cb_arg, 0); 430 431 free(ctx); 432 } 433 434 static void 435 blob_freeze_io(struct spdk_blob *blob, spdk_blob_op_complete cb_fn, void *cb_arg) 436 { 437 struct freeze_io_ctx *ctx; 438 439 blob_verify_md_op(blob); 440 441 ctx = calloc(1, sizeof(*ctx)); 442 if (!ctx) { 443 cb_fn(cb_arg, -ENOMEM); 444 return; 445 } 446 447 ctx->cpl.type = SPDK_BS_CPL_TYPE_BS_BASIC; 448 ctx->cpl.u.blob_basic.cb_fn = cb_fn; 449 ctx->cpl.u.blob_basic.cb_arg = cb_arg; 450 ctx->blob = blob; 451 452 /* Freeze I/O on blob */ 453 blob->frozen_refcnt++; 454 455 spdk_for_each_channel(blob->bs, blob_io_sync, ctx, blob_io_cpl); 456 } 457 458 static void 459 blob_unfreeze_io(struct spdk_blob *blob, spdk_blob_op_complete cb_fn, void *cb_arg) 460 { 461 struct freeze_io_ctx *ctx; 462 463 blob_verify_md_op(blob); 464 465 ctx = calloc(1, sizeof(*ctx)); 466 if (!ctx) { 467 cb_fn(cb_arg, -ENOMEM); 468 return; 469 } 470 471 ctx->cpl.type = SPDK_BS_CPL_TYPE_BS_BASIC; 472 ctx->cpl.u.blob_basic.cb_fn = cb_fn; 473 ctx->cpl.u.blob_basic.cb_arg = cb_arg; 474 ctx->blob = blob; 475 476 assert(blob->frozen_refcnt > 0); 477 478 blob->frozen_refcnt--; 479 480 spdk_for_each_channel(blob->bs, blob_execute_queued_io, ctx, blob_io_cpl); 481 } 482 483 static int 484 blob_mark_clean(struct spdk_blob *blob) 485 { 486 uint32_t *extent_pages = NULL; 487 uint64_t *clusters = NULL; 488 uint32_t *pages = NULL; 489 490 assert(blob != NULL); 491 492 if (blob->active.num_extent_pages) { 493 assert(blob->active.extent_pages); 494 extent_pages = calloc(blob->active.num_extent_pages, sizeof(*blob->active.extent_pages)); 495 if (!extent_pages) { 496 return -ENOMEM; 497 } 498 memcpy(extent_pages, blob->active.extent_pages, 499 blob->active.num_extent_pages * sizeof(*extent_pages)); 500 } 501 502 if (blob->active.num_clusters) { 503 assert(blob->active.clusters); 504 clusters = calloc(blob->active.num_clusters, sizeof(*blob->active.clusters)); 505 if (!clusters) { 506 free(extent_pages); 507 return -ENOMEM; 508 } 509 memcpy(clusters, blob->active.clusters, blob->active.num_clusters * sizeof(*blob->active.clusters)); 510 } 511 512 if (blob->active.num_pages) { 513 assert(blob->active.pages); 514 pages = calloc(blob->active.num_pages, sizeof(*blob->active.pages)); 515 if (!pages) { 516 free(extent_pages); 517 free(clusters); 518 return -ENOMEM; 519 } 520 memcpy(pages, blob->active.pages, blob->active.num_pages * sizeof(*blob->active.pages)); 521 } 522 523 free(blob->clean.extent_pages); 524 free(blob->clean.clusters); 525 free(blob->clean.pages); 526 527 blob->clean.num_extent_pages = blob->active.num_extent_pages; 528 blob->clean.extent_pages = blob->active.extent_pages; 529 blob->clean.num_clusters = blob->active.num_clusters; 530 blob->clean.clusters = blob->active.clusters; 531 blob->clean.num_pages = blob->active.num_pages; 532 blob->clean.pages = blob->active.pages; 533 534 blob->active.extent_pages = extent_pages; 535 blob->active.clusters = clusters; 536 blob->active.pages = pages; 537 538 /* If the metadata was dirtied again while the metadata was being written to disk, 539 * we do not want to revert the DIRTY state back to CLEAN here. 540 */ 541 if (blob->state == SPDK_BLOB_STATE_LOADING) { 542 blob->state = SPDK_BLOB_STATE_CLEAN; 543 } 544 545 return 0; 546 } 547 548 static int 549 blob_deserialize_xattr(struct spdk_blob *blob, 550 struct spdk_blob_md_descriptor_xattr *desc_xattr, bool internal) 551 { 552 struct spdk_xattr *xattr; 553 554 if (desc_xattr->length != sizeof(desc_xattr->name_length) + 555 sizeof(desc_xattr->value_length) + 556 desc_xattr->name_length + desc_xattr->value_length) { 557 return -EINVAL; 558 } 559 560 xattr = calloc(1, sizeof(*xattr)); 561 if (xattr == NULL) { 562 return -ENOMEM; 563 } 564 565 xattr->name = malloc(desc_xattr->name_length + 1); 566 if (xattr->name == NULL) { 567 free(xattr); 568 return -ENOMEM; 569 } 570 571 xattr->value = malloc(desc_xattr->value_length); 572 if (xattr->value == NULL) { 573 free(xattr->name); 574 free(xattr); 575 return -ENOMEM; 576 } 577 578 memcpy(xattr->name, desc_xattr->name, desc_xattr->name_length); 579 xattr->name[desc_xattr->name_length] = '\0'; 580 xattr->value_len = desc_xattr->value_length; 581 memcpy(xattr->value, 582 (void *)((uintptr_t)desc_xattr->name + desc_xattr->name_length), 583 desc_xattr->value_length); 584 585 TAILQ_INSERT_TAIL(internal ? &blob->xattrs_internal : &blob->xattrs, xattr, link); 586 587 return 0; 588 } 589 590 591 static int 592 blob_parse_page(const struct spdk_blob_md_page *page, struct spdk_blob *blob) 593 { 594 struct spdk_blob_md_descriptor *desc; 595 size_t cur_desc = 0; 596 void *tmp; 597 598 desc = (struct spdk_blob_md_descriptor *)page->descriptors; 599 while (cur_desc < sizeof(page->descriptors)) { 600 if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_PADDING) { 601 if (desc->length == 0) { 602 /* If padding and length are 0, this terminates the page */ 603 break; 604 } 605 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_FLAGS) { 606 struct spdk_blob_md_descriptor_flags *desc_flags; 607 608 desc_flags = (struct spdk_blob_md_descriptor_flags *)desc; 609 610 if (desc_flags->length != sizeof(*desc_flags) - sizeof(*desc)) { 611 return -EINVAL; 612 } 613 614 if ((desc_flags->invalid_flags | SPDK_BLOB_INVALID_FLAGS_MASK) != 615 SPDK_BLOB_INVALID_FLAGS_MASK) { 616 return -EINVAL; 617 } 618 619 if ((desc_flags->data_ro_flags | SPDK_BLOB_DATA_RO_FLAGS_MASK) != 620 SPDK_BLOB_DATA_RO_FLAGS_MASK) { 621 blob->data_ro = true; 622 blob->md_ro = true; 623 } 624 625 if ((desc_flags->md_ro_flags | SPDK_BLOB_MD_RO_FLAGS_MASK) != 626 SPDK_BLOB_MD_RO_FLAGS_MASK) { 627 blob->md_ro = true; 628 } 629 630 if ((desc_flags->data_ro_flags & SPDK_BLOB_READ_ONLY)) { 631 blob->data_ro = true; 632 blob->md_ro = true; 633 } 634 635 blob->invalid_flags = desc_flags->invalid_flags; 636 blob->data_ro_flags = desc_flags->data_ro_flags; 637 blob->md_ro_flags = desc_flags->md_ro_flags; 638 639 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_EXTENT_RLE) { 640 struct spdk_blob_md_descriptor_extent_rle *desc_extent_rle; 641 unsigned int i, j; 642 unsigned int cluster_count = blob->active.num_clusters; 643 644 if (blob->extent_table_found) { 645 /* Extent Table already present in the md, 646 * both descriptors should never be at the same time. */ 647 return -EINVAL; 648 } 649 blob->extent_rle_found = true; 650 651 desc_extent_rle = (struct spdk_blob_md_descriptor_extent_rle *)desc; 652 653 if (desc_extent_rle->length == 0 || 654 (desc_extent_rle->length % sizeof(desc_extent_rle->extents[0]) != 0)) { 655 return -EINVAL; 656 } 657 658 for (i = 0; i < desc_extent_rle->length / sizeof(desc_extent_rle->extents[0]); i++) { 659 for (j = 0; j < desc_extent_rle->extents[i].length; j++) { 660 if (desc_extent_rle->extents[i].cluster_idx != 0) { 661 if (!spdk_bit_pool_is_allocated(blob->bs->used_clusters, 662 desc_extent_rle->extents[i].cluster_idx + j)) { 663 return -EINVAL; 664 } 665 } 666 cluster_count++; 667 } 668 } 669 670 if (cluster_count == 0) { 671 return -EINVAL; 672 } 673 tmp = realloc(blob->active.clusters, cluster_count * sizeof(*blob->active.clusters)); 674 if (tmp == NULL) { 675 return -ENOMEM; 676 } 677 blob->active.clusters = tmp; 678 blob->active.cluster_array_size = cluster_count; 679 680 for (i = 0; i < desc_extent_rle->length / sizeof(desc_extent_rle->extents[0]); i++) { 681 for (j = 0; j < desc_extent_rle->extents[i].length; j++) { 682 if (desc_extent_rle->extents[i].cluster_idx != 0) { 683 blob->active.clusters[blob->active.num_clusters++] = bs_cluster_to_lba(blob->bs, 684 desc_extent_rle->extents[i].cluster_idx + j); 685 } else if (spdk_blob_is_thin_provisioned(blob)) { 686 blob->active.clusters[blob->active.num_clusters++] = 0; 687 } else { 688 return -EINVAL; 689 } 690 } 691 } 692 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_EXTENT_TABLE) { 693 struct spdk_blob_md_descriptor_extent_table *desc_extent_table; 694 uint32_t num_extent_pages = blob->active.num_extent_pages; 695 uint32_t i, j; 696 size_t extent_pages_length; 697 698 desc_extent_table = (struct spdk_blob_md_descriptor_extent_table *)desc; 699 extent_pages_length = desc_extent_table->length - sizeof(desc_extent_table->num_clusters); 700 701 if (blob->extent_rle_found) { 702 /* This means that Extent RLE is present in MD, 703 * both should never be at the same time. */ 704 return -EINVAL; 705 } else if (blob->extent_table_found && 706 desc_extent_table->num_clusters != blob->remaining_clusters_in_et) { 707 /* Number of clusters in this ET does not match number 708 * from previously read EXTENT_TABLE. */ 709 return -EINVAL; 710 } 711 712 if (desc_extent_table->length == 0 || 713 (extent_pages_length % sizeof(desc_extent_table->extent_page[0]) != 0)) { 714 return -EINVAL; 715 } 716 717 blob->extent_table_found = true; 718 719 for (i = 0; i < extent_pages_length / sizeof(desc_extent_table->extent_page[0]); i++) { 720 num_extent_pages += desc_extent_table->extent_page[i].num_pages; 721 } 722 723 if (num_extent_pages > 0) { 724 tmp = realloc(blob->active.extent_pages, num_extent_pages * sizeof(uint32_t)); 725 if (tmp == NULL) { 726 return -ENOMEM; 727 } 728 blob->active.extent_pages = tmp; 729 } 730 blob->active.extent_pages_array_size = num_extent_pages; 731 732 blob->remaining_clusters_in_et = desc_extent_table->num_clusters; 733 734 /* Extent table entries contain md page numbers for extent pages. 735 * Zeroes represent unallocated extent pages, those are run-length-encoded. 736 */ 737 for (i = 0; i < extent_pages_length / sizeof(desc_extent_table->extent_page[0]); i++) { 738 if (desc_extent_table->extent_page[i].page_idx != 0) { 739 assert(desc_extent_table->extent_page[i].num_pages == 1); 740 blob->active.extent_pages[blob->active.num_extent_pages++] = 741 desc_extent_table->extent_page[i].page_idx; 742 } else if (spdk_blob_is_thin_provisioned(blob)) { 743 for (j = 0; j < desc_extent_table->extent_page[i].num_pages; j++) { 744 blob->active.extent_pages[blob->active.num_extent_pages++] = 0; 745 } 746 } else { 747 return -EINVAL; 748 } 749 } 750 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_EXTENT_PAGE) { 751 struct spdk_blob_md_descriptor_extent_page *desc_extent; 752 unsigned int i; 753 unsigned int cluster_count = 0; 754 size_t cluster_idx_length; 755 756 if (blob->extent_rle_found) { 757 /* This means that Extent RLE is present in MD, 758 * both should never be at the same time. */ 759 return -EINVAL; 760 } 761 762 desc_extent = (struct spdk_blob_md_descriptor_extent_page *)desc; 763 cluster_idx_length = desc_extent->length - sizeof(desc_extent->start_cluster_idx); 764 765 if (desc_extent->length <= sizeof(desc_extent->start_cluster_idx) || 766 (cluster_idx_length % sizeof(desc_extent->cluster_idx[0]) != 0)) { 767 return -EINVAL; 768 } 769 770 for (i = 0; i < cluster_idx_length / sizeof(desc_extent->cluster_idx[0]); i++) { 771 if (desc_extent->cluster_idx[i] != 0) { 772 if (!spdk_bit_pool_is_allocated(blob->bs->used_clusters, desc_extent->cluster_idx[i])) { 773 return -EINVAL; 774 } 775 } 776 cluster_count++; 777 } 778 779 if (cluster_count == 0) { 780 return -EINVAL; 781 } 782 783 /* When reading extent pages sequentially starting cluster idx should match 784 * current size of a blob. 785 * If changed to batch reading, this check shall be removed. */ 786 if (desc_extent->start_cluster_idx != blob->active.num_clusters) { 787 return -EINVAL; 788 } 789 790 tmp = realloc(blob->active.clusters, 791 (cluster_count + blob->active.num_clusters) * sizeof(*blob->active.clusters)); 792 if (tmp == NULL) { 793 return -ENOMEM; 794 } 795 blob->active.clusters = tmp; 796 blob->active.cluster_array_size = (cluster_count + blob->active.num_clusters); 797 798 for (i = 0; i < cluster_idx_length / sizeof(desc_extent->cluster_idx[0]); i++) { 799 if (desc_extent->cluster_idx[i] != 0) { 800 blob->active.clusters[blob->active.num_clusters++] = bs_cluster_to_lba(blob->bs, 801 desc_extent->cluster_idx[i]); 802 } else if (spdk_blob_is_thin_provisioned(blob)) { 803 blob->active.clusters[blob->active.num_clusters++] = 0; 804 } else { 805 return -EINVAL; 806 } 807 } 808 assert(desc_extent->start_cluster_idx + cluster_count == blob->active.num_clusters); 809 assert(blob->remaining_clusters_in_et >= cluster_count); 810 blob->remaining_clusters_in_et -= cluster_count; 811 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_XATTR) { 812 int rc; 813 814 rc = blob_deserialize_xattr(blob, 815 (struct spdk_blob_md_descriptor_xattr *) desc, false); 816 if (rc != 0) { 817 return rc; 818 } 819 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_XATTR_INTERNAL) { 820 int rc; 821 822 rc = blob_deserialize_xattr(blob, 823 (struct spdk_blob_md_descriptor_xattr *) desc, true); 824 if (rc != 0) { 825 return rc; 826 } 827 } else { 828 /* Unrecognized descriptor type. Do not fail - just continue to the 829 * next descriptor. If this descriptor is associated with some feature 830 * defined in a newer version of blobstore, that version of blobstore 831 * should create and set an associated feature flag to specify if this 832 * blob can be loaded or not. 833 */ 834 } 835 836 /* Advance to the next descriptor */ 837 cur_desc += sizeof(*desc) + desc->length; 838 if (cur_desc + sizeof(*desc) > sizeof(page->descriptors)) { 839 break; 840 } 841 desc = (struct spdk_blob_md_descriptor *)((uintptr_t)page->descriptors + cur_desc); 842 } 843 844 return 0; 845 } 846 847 static bool bs_load_cur_extent_page_valid(struct spdk_blob_md_page *page); 848 849 static int 850 blob_parse_extent_page(struct spdk_blob_md_page *extent_page, struct spdk_blob *blob) 851 { 852 assert(blob != NULL); 853 assert(blob->state == SPDK_BLOB_STATE_LOADING); 854 855 if (bs_load_cur_extent_page_valid(extent_page) == false) { 856 return -ENOENT; 857 } 858 859 return blob_parse_page(extent_page, blob); 860 } 861 862 static int 863 blob_parse(const struct spdk_blob_md_page *pages, uint32_t page_count, 864 struct spdk_blob *blob) 865 { 866 const struct spdk_blob_md_page *page; 867 uint32_t i; 868 int rc; 869 void *tmp; 870 871 assert(page_count > 0); 872 assert(pages[0].sequence_num == 0); 873 assert(blob != NULL); 874 assert(blob->state == SPDK_BLOB_STATE_LOADING); 875 assert(blob->active.clusters == NULL); 876 877 /* The blobid provided doesn't match what's in the MD, this can 878 * happen for example if a bogus blobid is passed in through open. 879 */ 880 if (blob->id != pages[0].id) { 881 SPDK_ERRLOG("Blobid (%" PRIu64 ") doesn't match what's in metadata (%" PRIu64 ")\n", 882 blob->id, pages[0].id); 883 return -ENOENT; 884 } 885 886 tmp = realloc(blob->active.pages, page_count * sizeof(*blob->active.pages)); 887 if (!tmp) { 888 return -ENOMEM; 889 } 890 blob->active.pages = tmp; 891 892 blob->active.pages[0] = pages[0].id; 893 894 for (i = 1; i < page_count; i++) { 895 assert(spdk_bit_array_get(blob->bs->used_md_pages, pages[i - 1].next)); 896 blob->active.pages[i] = pages[i - 1].next; 897 } 898 blob->active.num_pages = page_count; 899 900 for (i = 0; i < page_count; i++) { 901 page = &pages[i]; 902 903 assert(page->id == blob->id); 904 assert(page->sequence_num == i); 905 906 rc = blob_parse_page(page, blob); 907 if (rc != 0) { 908 return rc; 909 } 910 } 911 912 return 0; 913 } 914 915 static int 916 blob_serialize_add_page(const struct spdk_blob *blob, 917 struct spdk_blob_md_page **pages, 918 uint32_t *page_count, 919 struct spdk_blob_md_page **last_page) 920 { 921 struct spdk_blob_md_page *page, *tmp_pages; 922 923 assert(pages != NULL); 924 assert(page_count != NULL); 925 926 *last_page = NULL; 927 if (*page_count == 0) { 928 assert(*pages == NULL); 929 *pages = spdk_malloc(SPDK_BS_PAGE_SIZE, 0, 930 NULL, SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA); 931 if (*pages == NULL) { 932 return -ENOMEM; 933 } 934 *page_count = 1; 935 } else { 936 assert(*pages != NULL); 937 tmp_pages = spdk_realloc(*pages, SPDK_BS_PAGE_SIZE * (*page_count + 1), 0); 938 if (tmp_pages == NULL) { 939 return -ENOMEM; 940 } 941 (*page_count)++; 942 *pages = tmp_pages; 943 } 944 945 page = &(*pages)[*page_count - 1]; 946 memset(page, 0, sizeof(*page)); 947 page->id = blob->id; 948 page->sequence_num = *page_count - 1; 949 page->next = SPDK_INVALID_MD_PAGE; 950 *last_page = page; 951 952 return 0; 953 } 954 955 /* Transform the in-memory representation 'xattr' into an on-disk xattr descriptor. 956 * Update required_sz on both success and failure. 957 * 958 */ 959 static int 960 blob_serialize_xattr(const struct spdk_xattr *xattr, 961 uint8_t *buf, size_t buf_sz, 962 size_t *required_sz, bool internal) 963 { 964 struct spdk_blob_md_descriptor_xattr *desc; 965 966 *required_sz = sizeof(struct spdk_blob_md_descriptor_xattr) + 967 strlen(xattr->name) + 968 xattr->value_len; 969 970 if (buf_sz < *required_sz) { 971 return -1; 972 } 973 974 desc = (struct spdk_blob_md_descriptor_xattr *)buf; 975 976 desc->type = internal ? SPDK_MD_DESCRIPTOR_TYPE_XATTR_INTERNAL : SPDK_MD_DESCRIPTOR_TYPE_XATTR; 977 desc->length = sizeof(desc->name_length) + 978 sizeof(desc->value_length) + 979 strlen(xattr->name) + 980 xattr->value_len; 981 desc->name_length = strlen(xattr->name); 982 desc->value_length = xattr->value_len; 983 984 memcpy(desc->name, xattr->name, desc->name_length); 985 memcpy((void *)((uintptr_t)desc->name + desc->name_length), 986 xattr->value, 987 desc->value_length); 988 989 return 0; 990 } 991 992 static void 993 blob_serialize_extent_table_entry(const struct spdk_blob *blob, 994 uint64_t start_ep, uint64_t *next_ep, 995 uint8_t **buf, size_t *remaining_sz) 996 { 997 struct spdk_blob_md_descriptor_extent_table *desc; 998 size_t cur_sz; 999 uint64_t i, et_idx; 1000 uint32_t extent_page, ep_len; 1001 1002 /* The buffer must have room for at least num_clusters entry */ 1003 cur_sz = sizeof(struct spdk_blob_md_descriptor) + sizeof(desc->num_clusters); 1004 if (*remaining_sz < cur_sz) { 1005 *next_ep = start_ep; 1006 return; 1007 } 1008 1009 desc = (struct spdk_blob_md_descriptor_extent_table *)*buf; 1010 desc->type = SPDK_MD_DESCRIPTOR_TYPE_EXTENT_TABLE; 1011 1012 desc->num_clusters = blob->active.num_clusters; 1013 1014 ep_len = 1; 1015 et_idx = 0; 1016 for (i = start_ep; i < blob->active.num_extent_pages; i++) { 1017 if (*remaining_sz < cur_sz + sizeof(desc->extent_page[0])) { 1018 /* If we ran out of buffer space, return */ 1019 break; 1020 } 1021 1022 extent_page = blob->active.extent_pages[i]; 1023 /* Verify that next extent_page is unallocated */ 1024 if (extent_page == 0 && 1025 (i + 1 < blob->active.num_extent_pages && blob->active.extent_pages[i + 1] == 0)) { 1026 ep_len++; 1027 continue; 1028 } 1029 desc->extent_page[et_idx].page_idx = extent_page; 1030 desc->extent_page[et_idx].num_pages = ep_len; 1031 et_idx++; 1032 1033 ep_len = 1; 1034 cur_sz += sizeof(desc->extent_page[et_idx]); 1035 } 1036 *next_ep = i; 1037 1038 desc->length = sizeof(desc->num_clusters) + sizeof(desc->extent_page[0]) * et_idx; 1039 *remaining_sz -= sizeof(struct spdk_blob_md_descriptor) + desc->length; 1040 *buf += sizeof(struct spdk_blob_md_descriptor) + desc->length; 1041 } 1042 1043 static int 1044 blob_serialize_extent_table(const struct spdk_blob *blob, 1045 struct spdk_blob_md_page **pages, 1046 struct spdk_blob_md_page *cur_page, 1047 uint32_t *page_count, uint8_t **buf, 1048 size_t *remaining_sz) 1049 { 1050 uint64_t last_extent_page; 1051 int rc; 1052 1053 last_extent_page = 0; 1054 /* At least single extent table entry has to be always persisted. 1055 * Such case occurs with num_extent_pages == 0. */ 1056 while (last_extent_page <= blob->active.num_extent_pages) { 1057 blob_serialize_extent_table_entry(blob, last_extent_page, &last_extent_page, buf, 1058 remaining_sz); 1059 1060 if (last_extent_page == blob->active.num_extent_pages) { 1061 break; 1062 } 1063 1064 rc = blob_serialize_add_page(blob, pages, page_count, &cur_page); 1065 if (rc < 0) { 1066 return rc; 1067 } 1068 1069 *buf = (uint8_t *)cur_page->descriptors; 1070 *remaining_sz = sizeof(cur_page->descriptors); 1071 } 1072 1073 return 0; 1074 } 1075 1076 static void 1077 blob_serialize_extent_rle(const struct spdk_blob *blob, 1078 uint64_t start_cluster, uint64_t *next_cluster, 1079 uint8_t **buf, size_t *buf_sz) 1080 { 1081 struct spdk_blob_md_descriptor_extent_rle *desc_extent_rle; 1082 size_t cur_sz; 1083 uint64_t i, extent_idx; 1084 uint64_t lba, lba_per_cluster, lba_count; 1085 1086 /* The buffer must have room for at least one extent */ 1087 cur_sz = sizeof(struct spdk_blob_md_descriptor) + sizeof(desc_extent_rle->extents[0]); 1088 if (*buf_sz < cur_sz) { 1089 *next_cluster = start_cluster; 1090 return; 1091 } 1092 1093 desc_extent_rle = (struct spdk_blob_md_descriptor_extent_rle *)*buf; 1094 desc_extent_rle->type = SPDK_MD_DESCRIPTOR_TYPE_EXTENT_RLE; 1095 1096 lba_per_cluster = bs_cluster_to_lba(blob->bs, 1); 1097 /* Assert for scan-build false positive */ 1098 assert(lba_per_cluster > 0); 1099 1100 lba = blob->active.clusters[start_cluster]; 1101 lba_count = lba_per_cluster; 1102 extent_idx = 0; 1103 for (i = start_cluster + 1; i < blob->active.num_clusters; i++) { 1104 if ((lba + lba_count) == blob->active.clusters[i] && lba != 0) { 1105 /* Run-length encode sequential non-zero LBA */ 1106 lba_count += lba_per_cluster; 1107 continue; 1108 } else if (lba == 0 && blob->active.clusters[i] == 0) { 1109 /* Run-length encode unallocated clusters */ 1110 lba_count += lba_per_cluster; 1111 continue; 1112 } 1113 desc_extent_rle->extents[extent_idx].cluster_idx = lba / lba_per_cluster; 1114 desc_extent_rle->extents[extent_idx].length = lba_count / lba_per_cluster; 1115 extent_idx++; 1116 1117 cur_sz += sizeof(desc_extent_rle->extents[extent_idx]); 1118 1119 if (*buf_sz < cur_sz) { 1120 /* If we ran out of buffer space, return */ 1121 *next_cluster = i; 1122 break; 1123 } 1124 1125 lba = blob->active.clusters[i]; 1126 lba_count = lba_per_cluster; 1127 } 1128 1129 if (*buf_sz >= cur_sz) { 1130 desc_extent_rle->extents[extent_idx].cluster_idx = lba / lba_per_cluster; 1131 desc_extent_rle->extents[extent_idx].length = lba_count / lba_per_cluster; 1132 extent_idx++; 1133 1134 *next_cluster = blob->active.num_clusters; 1135 } 1136 1137 desc_extent_rle->length = sizeof(desc_extent_rle->extents[0]) * extent_idx; 1138 *buf_sz -= sizeof(struct spdk_blob_md_descriptor) + desc_extent_rle->length; 1139 *buf += sizeof(struct spdk_blob_md_descriptor) + desc_extent_rle->length; 1140 } 1141 1142 static int 1143 blob_serialize_extents_rle(const struct spdk_blob *blob, 1144 struct spdk_blob_md_page **pages, 1145 struct spdk_blob_md_page *cur_page, 1146 uint32_t *page_count, uint8_t **buf, 1147 size_t *remaining_sz) 1148 { 1149 uint64_t last_cluster; 1150 int rc; 1151 1152 last_cluster = 0; 1153 while (last_cluster < blob->active.num_clusters) { 1154 blob_serialize_extent_rle(blob, last_cluster, &last_cluster, buf, remaining_sz); 1155 1156 if (last_cluster == blob->active.num_clusters) { 1157 break; 1158 } 1159 1160 rc = blob_serialize_add_page(blob, pages, page_count, &cur_page); 1161 if (rc < 0) { 1162 return rc; 1163 } 1164 1165 *buf = (uint8_t *)cur_page->descriptors; 1166 *remaining_sz = sizeof(cur_page->descriptors); 1167 } 1168 1169 return 0; 1170 } 1171 1172 static void 1173 blob_serialize_extent_page(const struct spdk_blob *blob, 1174 uint64_t cluster, struct spdk_blob_md_page *page) 1175 { 1176 struct spdk_blob_md_descriptor_extent_page *desc_extent; 1177 uint64_t i, extent_idx; 1178 uint64_t lba, lba_per_cluster; 1179 uint64_t start_cluster_idx = (cluster / SPDK_EXTENTS_PER_EP) * SPDK_EXTENTS_PER_EP; 1180 1181 desc_extent = (struct spdk_blob_md_descriptor_extent_page *) page->descriptors; 1182 desc_extent->type = SPDK_MD_DESCRIPTOR_TYPE_EXTENT_PAGE; 1183 1184 lba_per_cluster = bs_cluster_to_lba(blob->bs, 1); 1185 1186 desc_extent->start_cluster_idx = start_cluster_idx; 1187 extent_idx = 0; 1188 for (i = start_cluster_idx; i < blob->active.num_clusters; i++) { 1189 lba = blob->active.clusters[i]; 1190 desc_extent->cluster_idx[extent_idx++] = lba / lba_per_cluster; 1191 if (extent_idx >= SPDK_EXTENTS_PER_EP) { 1192 break; 1193 } 1194 } 1195 desc_extent->length = sizeof(desc_extent->start_cluster_idx) + 1196 sizeof(desc_extent->cluster_idx[0]) * extent_idx; 1197 } 1198 1199 static void 1200 blob_serialize_flags(const struct spdk_blob *blob, 1201 uint8_t *buf, size_t *buf_sz) 1202 { 1203 struct spdk_blob_md_descriptor_flags *desc; 1204 1205 /* 1206 * Flags get serialized first, so we should always have room for the flags 1207 * descriptor. 1208 */ 1209 assert(*buf_sz >= sizeof(*desc)); 1210 1211 desc = (struct spdk_blob_md_descriptor_flags *)buf; 1212 desc->type = SPDK_MD_DESCRIPTOR_TYPE_FLAGS; 1213 desc->length = sizeof(*desc) - sizeof(struct spdk_blob_md_descriptor); 1214 desc->invalid_flags = blob->invalid_flags; 1215 desc->data_ro_flags = blob->data_ro_flags; 1216 desc->md_ro_flags = blob->md_ro_flags; 1217 1218 *buf_sz -= sizeof(*desc); 1219 } 1220 1221 static int 1222 blob_serialize_xattrs(const struct spdk_blob *blob, 1223 const struct spdk_xattr_tailq *xattrs, bool internal, 1224 struct spdk_blob_md_page **pages, 1225 struct spdk_blob_md_page *cur_page, 1226 uint32_t *page_count, uint8_t **buf, 1227 size_t *remaining_sz) 1228 { 1229 const struct spdk_xattr *xattr; 1230 int rc; 1231 1232 TAILQ_FOREACH(xattr, xattrs, link) { 1233 size_t required_sz = 0; 1234 1235 rc = blob_serialize_xattr(xattr, 1236 *buf, *remaining_sz, 1237 &required_sz, internal); 1238 if (rc < 0) { 1239 /* Need to add a new page to the chain */ 1240 rc = blob_serialize_add_page(blob, pages, page_count, 1241 &cur_page); 1242 if (rc < 0) { 1243 spdk_free(*pages); 1244 *pages = NULL; 1245 *page_count = 0; 1246 return rc; 1247 } 1248 1249 *buf = (uint8_t *)cur_page->descriptors; 1250 *remaining_sz = sizeof(cur_page->descriptors); 1251 1252 /* Try again */ 1253 required_sz = 0; 1254 rc = blob_serialize_xattr(xattr, 1255 *buf, *remaining_sz, 1256 &required_sz, internal); 1257 1258 if (rc < 0) { 1259 spdk_free(*pages); 1260 *pages = NULL; 1261 *page_count = 0; 1262 return rc; 1263 } 1264 } 1265 1266 *remaining_sz -= required_sz; 1267 *buf += required_sz; 1268 } 1269 1270 return 0; 1271 } 1272 1273 static int 1274 blob_serialize(const struct spdk_blob *blob, struct spdk_blob_md_page **pages, 1275 uint32_t *page_count) 1276 { 1277 struct spdk_blob_md_page *cur_page; 1278 int rc; 1279 uint8_t *buf; 1280 size_t remaining_sz; 1281 1282 assert(pages != NULL); 1283 assert(page_count != NULL); 1284 assert(blob != NULL); 1285 assert(blob->state == SPDK_BLOB_STATE_DIRTY); 1286 1287 *pages = NULL; 1288 *page_count = 0; 1289 1290 /* A blob always has at least 1 page, even if it has no descriptors */ 1291 rc = blob_serialize_add_page(blob, pages, page_count, &cur_page); 1292 if (rc < 0) { 1293 return rc; 1294 } 1295 1296 buf = (uint8_t *)cur_page->descriptors; 1297 remaining_sz = sizeof(cur_page->descriptors); 1298 1299 /* Serialize flags */ 1300 blob_serialize_flags(blob, buf, &remaining_sz); 1301 buf += sizeof(struct spdk_blob_md_descriptor_flags); 1302 1303 /* Serialize xattrs */ 1304 rc = blob_serialize_xattrs(blob, &blob->xattrs, false, 1305 pages, cur_page, page_count, &buf, &remaining_sz); 1306 if (rc < 0) { 1307 return rc; 1308 } 1309 1310 /* Serialize internal xattrs */ 1311 rc = blob_serialize_xattrs(blob, &blob->xattrs_internal, true, 1312 pages, cur_page, page_count, &buf, &remaining_sz); 1313 if (rc < 0) { 1314 return rc; 1315 } 1316 1317 if (blob->use_extent_table) { 1318 /* Serialize extent table */ 1319 rc = blob_serialize_extent_table(blob, pages, cur_page, page_count, &buf, &remaining_sz); 1320 } else { 1321 /* Serialize extents */ 1322 rc = blob_serialize_extents_rle(blob, pages, cur_page, page_count, &buf, &remaining_sz); 1323 } 1324 1325 return rc; 1326 } 1327 1328 struct spdk_blob_load_ctx { 1329 struct spdk_blob *blob; 1330 1331 struct spdk_blob_md_page *pages; 1332 uint32_t num_pages; 1333 uint32_t next_extent_page; 1334 spdk_bs_sequence_t *seq; 1335 1336 spdk_bs_sequence_cpl cb_fn; 1337 void *cb_arg; 1338 }; 1339 1340 static uint32_t 1341 blob_md_page_calc_crc(void *page) 1342 { 1343 uint32_t crc; 1344 1345 crc = BLOB_CRC32C_INITIAL; 1346 crc = spdk_crc32c_update(page, SPDK_BS_PAGE_SIZE - 4, crc); 1347 crc ^= BLOB_CRC32C_INITIAL; 1348 1349 return crc; 1350 1351 } 1352 1353 static void 1354 blob_load_final(struct spdk_blob_load_ctx *ctx, int bserrno) 1355 { 1356 struct spdk_blob *blob = ctx->blob; 1357 1358 if (bserrno == 0) { 1359 blob_mark_clean(blob); 1360 } 1361 1362 ctx->cb_fn(ctx->seq, ctx->cb_arg, bserrno); 1363 1364 /* Free the memory */ 1365 spdk_free(ctx->pages); 1366 free(ctx); 1367 } 1368 1369 static void 1370 blob_load_snapshot_cpl(void *cb_arg, struct spdk_blob *snapshot, int bserrno) 1371 { 1372 struct spdk_blob_load_ctx *ctx = cb_arg; 1373 struct spdk_blob *blob = ctx->blob; 1374 1375 if (bserrno == 0) { 1376 blob->back_bs_dev = bs_create_blob_bs_dev(snapshot); 1377 if (blob->back_bs_dev == NULL) { 1378 bserrno = -ENOMEM; 1379 } 1380 } 1381 if (bserrno != 0) { 1382 SPDK_ERRLOG("Snapshot fail\n"); 1383 } 1384 1385 blob_load_final(ctx, bserrno); 1386 } 1387 1388 static void blob_update_clear_method(struct spdk_blob *blob); 1389 1390 static int 1391 blob_load_esnap(struct spdk_blob *blob, void *blob_ctx) 1392 { 1393 struct spdk_blob_store *bs = blob->bs; 1394 struct spdk_bs_dev *bs_dev = NULL; 1395 const void *esnap_id = NULL; 1396 size_t id_len = 0; 1397 int rc; 1398 1399 if (bs->esnap_bs_dev_create == NULL) { 1400 SPDK_NOTICELOG("blob 0x%" PRIx64 " is an esnap clone but the blobstore was opened " 1401 "without support for esnap clones\n", blob->id); 1402 return -ENOTSUP; 1403 } 1404 assert(blob->back_bs_dev == NULL); 1405 1406 rc = blob_get_xattr_value(blob, BLOB_EXTERNAL_SNAPSHOT_ID, &esnap_id, &id_len, true); 1407 if (rc != 0) { 1408 SPDK_ERRLOG("blob 0x%" PRIx64 " is an esnap clone but has no esnap ID\n", blob->id); 1409 return -EINVAL; 1410 } 1411 assert(id_len > 0 && id_len < UINT32_MAX); 1412 1413 SPDK_INFOLOG(blob, "Creating external snapshot device\n"); 1414 1415 rc = bs->esnap_bs_dev_create(bs->esnap_ctx, blob_ctx, blob, esnap_id, (uint32_t)id_len, 1416 &bs_dev); 1417 if (rc != 0) { 1418 SPDK_DEBUGLOG(blob_esnap, "blob 0x%" PRIx64 ": failed to load back_bs_dev " 1419 "with error %d\n", blob->id, rc); 1420 return rc; 1421 } 1422 1423 assert(bs_dev != NULL); 1424 SPDK_DEBUGLOG(blob_esnap, "blob 0x%" PRIx64 ": loaded back_bs_dev\n", blob->id); 1425 if ((bs->io_unit_size % bs_dev->blocklen) != 0) { 1426 SPDK_NOTICELOG("blob 0x%" PRIx64 " external snapshot device block size %u is not " 1427 "compatible with blobstore block size %u\n", 1428 blob->id, bs_dev->blocklen, bs->io_unit_size); 1429 bs_dev->destroy(bs_dev); 1430 return -EINVAL; 1431 } 1432 1433 blob->back_bs_dev = bs_dev; 1434 blob->parent_id = SPDK_BLOBID_EXTERNAL_SNAPSHOT; 1435 1436 return 0; 1437 } 1438 1439 static void 1440 blob_load_backing_dev(spdk_bs_sequence_t *seq, void *cb_arg) 1441 { 1442 struct spdk_blob_load_ctx *ctx = cb_arg; 1443 struct spdk_blob *blob = ctx->blob; 1444 const void *value; 1445 size_t len; 1446 int rc; 1447 1448 if (blob_is_esnap_clone(blob)) { 1449 rc = blob_load_esnap(blob, seq->cpl.u.blob_handle.esnap_ctx); 1450 assert((rc == 0) ^ (blob->back_bs_dev == NULL)); 1451 blob_load_final(ctx, rc); 1452 return; 1453 } 1454 1455 if (spdk_blob_is_thin_provisioned(blob)) { 1456 rc = blob_get_xattr_value(blob, BLOB_SNAPSHOT, &value, &len, true); 1457 if (rc == 0) { 1458 if (len != sizeof(spdk_blob_id)) { 1459 blob_load_final(ctx, -EINVAL); 1460 return; 1461 } 1462 /* open snapshot blob and continue in the callback function */ 1463 blob->parent_id = *(spdk_blob_id *)value; 1464 spdk_bs_open_blob(blob->bs, blob->parent_id, 1465 blob_load_snapshot_cpl, ctx); 1466 return; 1467 } else { 1468 /* add zeroes_dev for thin provisioned blob */ 1469 blob->back_bs_dev = bs_create_zeroes_dev(); 1470 } 1471 } else { 1472 /* standard blob */ 1473 blob->back_bs_dev = NULL; 1474 } 1475 blob_load_final(ctx, 0); 1476 } 1477 1478 static void 1479 blob_load_cpl_extents_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 1480 { 1481 struct spdk_blob_load_ctx *ctx = cb_arg; 1482 struct spdk_blob *blob = ctx->blob; 1483 struct spdk_blob_md_page *page; 1484 uint64_t i; 1485 uint32_t crc; 1486 uint64_t lba; 1487 void *tmp; 1488 uint64_t sz; 1489 1490 if (bserrno) { 1491 SPDK_ERRLOG("Extent page read failed: %d\n", bserrno); 1492 blob_load_final(ctx, bserrno); 1493 return; 1494 } 1495 1496 if (ctx->pages == NULL) { 1497 /* First iteration of this function, allocate buffer for single EXTENT_PAGE */ 1498 ctx->pages = spdk_zmalloc(SPDK_BS_PAGE_SIZE, 0, 1499 NULL, SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA); 1500 if (!ctx->pages) { 1501 blob_load_final(ctx, -ENOMEM); 1502 return; 1503 } 1504 ctx->num_pages = 1; 1505 ctx->next_extent_page = 0; 1506 } else { 1507 page = &ctx->pages[0]; 1508 crc = blob_md_page_calc_crc(page); 1509 if (crc != page->crc) { 1510 blob_load_final(ctx, -EINVAL); 1511 return; 1512 } 1513 1514 if (page->next != SPDK_INVALID_MD_PAGE) { 1515 blob_load_final(ctx, -EINVAL); 1516 return; 1517 } 1518 1519 bserrno = blob_parse_extent_page(page, blob); 1520 if (bserrno) { 1521 blob_load_final(ctx, bserrno); 1522 return; 1523 } 1524 } 1525 1526 for (i = ctx->next_extent_page; i < blob->active.num_extent_pages; i++) { 1527 if (blob->active.extent_pages[i] != 0) { 1528 /* Extent page was allocated, read and parse it. */ 1529 lba = bs_md_page_to_lba(blob->bs, blob->active.extent_pages[i]); 1530 ctx->next_extent_page = i + 1; 1531 1532 bs_sequence_read_dev(seq, &ctx->pages[0], lba, 1533 bs_byte_to_lba(blob->bs, SPDK_BS_PAGE_SIZE), 1534 blob_load_cpl_extents_cpl, ctx); 1535 return; 1536 } else { 1537 /* Thin provisioned blobs can point to unallocated extent pages. 1538 * In this case blob size should be increased by up to the amount left in remaining_clusters_in_et. */ 1539 1540 sz = spdk_min(blob->remaining_clusters_in_et, SPDK_EXTENTS_PER_EP); 1541 blob->active.num_clusters += sz; 1542 blob->remaining_clusters_in_et -= sz; 1543 1544 assert(spdk_blob_is_thin_provisioned(blob)); 1545 assert(i + 1 < blob->active.num_extent_pages || blob->remaining_clusters_in_et == 0); 1546 1547 tmp = realloc(blob->active.clusters, blob->active.num_clusters * sizeof(*blob->active.clusters)); 1548 if (tmp == NULL) { 1549 blob_load_final(ctx, -ENOMEM); 1550 return; 1551 } 1552 memset(tmp + sizeof(*blob->active.clusters) * blob->active.cluster_array_size, 0, 1553 sizeof(*blob->active.clusters) * (blob->active.num_clusters - blob->active.cluster_array_size)); 1554 blob->active.clusters = tmp; 1555 blob->active.cluster_array_size = blob->active.num_clusters; 1556 } 1557 } 1558 1559 blob_load_backing_dev(seq, ctx); 1560 } 1561 1562 static void 1563 blob_load_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 1564 { 1565 struct spdk_blob_load_ctx *ctx = cb_arg; 1566 struct spdk_blob *blob = ctx->blob; 1567 struct spdk_blob_md_page *page; 1568 int rc; 1569 uint32_t crc; 1570 uint32_t current_page; 1571 1572 if (ctx->num_pages == 1) { 1573 current_page = bs_blobid_to_page(blob->id); 1574 } else { 1575 assert(ctx->num_pages != 0); 1576 page = &ctx->pages[ctx->num_pages - 2]; 1577 current_page = page->next; 1578 } 1579 1580 if (bserrno) { 1581 SPDK_ERRLOG("Metadata page %d read failed for blobid %" PRIu64 ": %d\n", 1582 current_page, blob->id, bserrno); 1583 blob_load_final(ctx, bserrno); 1584 return; 1585 } 1586 1587 page = &ctx->pages[ctx->num_pages - 1]; 1588 crc = blob_md_page_calc_crc(page); 1589 if (crc != page->crc) { 1590 SPDK_ERRLOG("Metadata page %d crc mismatch for blobid %" PRIu64 "\n", 1591 current_page, blob->id); 1592 blob_load_final(ctx, -EINVAL); 1593 return; 1594 } 1595 1596 if (page->next != SPDK_INVALID_MD_PAGE) { 1597 struct spdk_blob_md_page *tmp_pages; 1598 uint32_t next_page = page->next; 1599 uint64_t next_lba = bs_md_page_to_lba(blob->bs, next_page); 1600 1601 /* Read the next page */ 1602 tmp_pages = spdk_realloc(ctx->pages, (sizeof(*page) * (ctx->num_pages + 1)), 0); 1603 if (tmp_pages == NULL) { 1604 blob_load_final(ctx, -ENOMEM); 1605 return; 1606 } 1607 ctx->num_pages++; 1608 ctx->pages = tmp_pages; 1609 1610 bs_sequence_read_dev(seq, &ctx->pages[ctx->num_pages - 1], 1611 next_lba, 1612 bs_byte_to_lba(blob->bs, sizeof(*page)), 1613 blob_load_cpl, ctx); 1614 return; 1615 } 1616 1617 /* Parse the pages */ 1618 rc = blob_parse(ctx->pages, ctx->num_pages, blob); 1619 if (rc) { 1620 blob_load_final(ctx, rc); 1621 return; 1622 } 1623 1624 if (blob->extent_table_found == true) { 1625 /* If EXTENT_TABLE was found, that means support for it should be enabled. */ 1626 assert(blob->extent_rle_found == false); 1627 blob->use_extent_table = true; 1628 } else { 1629 /* If EXTENT_RLE or no extent_* descriptor was found disable support 1630 * for extent table. No extent_* descriptors means that blob has length of 0 1631 * and no extent_rle descriptors were persisted for it. 1632 * EXTENT_TABLE if used, is always present in metadata regardless of length. */ 1633 blob->use_extent_table = false; 1634 } 1635 1636 /* Check the clear_method stored in metadata vs what may have been passed 1637 * via spdk_bs_open_blob_ext() and update accordingly. 1638 */ 1639 blob_update_clear_method(blob); 1640 1641 spdk_free(ctx->pages); 1642 ctx->pages = NULL; 1643 1644 if (blob->extent_table_found) { 1645 blob_load_cpl_extents_cpl(seq, ctx, 0); 1646 } else { 1647 blob_load_backing_dev(seq, ctx); 1648 } 1649 } 1650 1651 /* Load a blob from disk given a blobid */ 1652 static void 1653 blob_load(spdk_bs_sequence_t *seq, struct spdk_blob *blob, 1654 spdk_bs_sequence_cpl cb_fn, void *cb_arg) 1655 { 1656 struct spdk_blob_load_ctx *ctx; 1657 struct spdk_blob_store *bs; 1658 uint32_t page_num; 1659 uint64_t lba; 1660 1661 blob_verify_md_op(blob); 1662 1663 bs = blob->bs; 1664 1665 ctx = calloc(1, sizeof(*ctx)); 1666 if (!ctx) { 1667 cb_fn(seq, cb_arg, -ENOMEM); 1668 return; 1669 } 1670 1671 ctx->blob = blob; 1672 ctx->pages = spdk_realloc(ctx->pages, SPDK_BS_PAGE_SIZE, 0); 1673 if (!ctx->pages) { 1674 free(ctx); 1675 cb_fn(seq, cb_arg, -ENOMEM); 1676 return; 1677 } 1678 ctx->num_pages = 1; 1679 ctx->cb_fn = cb_fn; 1680 ctx->cb_arg = cb_arg; 1681 ctx->seq = seq; 1682 1683 page_num = bs_blobid_to_page(blob->id); 1684 lba = bs_md_page_to_lba(blob->bs, page_num); 1685 1686 blob->state = SPDK_BLOB_STATE_LOADING; 1687 1688 bs_sequence_read_dev(seq, &ctx->pages[0], lba, 1689 bs_byte_to_lba(bs, SPDK_BS_PAGE_SIZE), 1690 blob_load_cpl, ctx); 1691 } 1692 1693 struct spdk_blob_persist_ctx { 1694 struct spdk_blob *blob; 1695 1696 struct spdk_blob_md_page *pages; 1697 uint32_t next_extent_page; 1698 struct spdk_blob_md_page *extent_page; 1699 1700 spdk_bs_sequence_t *seq; 1701 spdk_bs_sequence_cpl cb_fn; 1702 void *cb_arg; 1703 TAILQ_ENTRY(spdk_blob_persist_ctx) link; 1704 }; 1705 1706 static void 1707 bs_batch_clear_dev(struct spdk_blob_persist_ctx *ctx, spdk_bs_batch_t *batch, uint64_t lba, 1708 uint64_t lba_count) 1709 { 1710 switch (ctx->blob->clear_method) { 1711 case BLOB_CLEAR_WITH_DEFAULT: 1712 case BLOB_CLEAR_WITH_UNMAP: 1713 bs_batch_unmap_dev(batch, lba, lba_count); 1714 break; 1715 case BLOB_CLEAR_WITH_WRITE_ZEROES: 1716 bs_batch_write_zeroes_dev(batch, lba, lba_count); 1717 break; 1718 case BLOB_CLEAR_WITH_NONE: 1719 default: 1720 break; 1721 } 1722 } 1723 1724 static void bs_mark_dirty(spdk_bs_sequence_t *seq, struct spdk_blob_store *bs, 1725 spdk_bs_sequence_cpl cb_fn, void *cb_arg); 1726 1727 static void 1728 blob_persist_complete_cb(void *arg) 1729 { 1730 struct spdk_blob_persist_ctx *ctx = arg; 1731 1732 /* Call user callback */ 1733 ctx->cb_fn(ctx->seq, ctx->cb_arg, 0); 1734 1735 /* Free the memory */ 1736 spdk_free(ctx->pages); 1737 free(ctx); 1738 } 1739 1740 static void blob_persist_start(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno); 1741 1742 static void 1743 blob_persist_complete(spdk_bs_sequence_t *seq, struct spdk_blob_persist_ctx *ctx, int bserrno) 1744 { 1745 struct spdk_blob_persist_ctx *next_persist, *tmp; 1746 struct spdk_blob *blob = ctx->blob; 1747 1748 if (bserrno == 0) { 1749 blob_mark_clean(blob); 1750 } 1751 1752 assert(ctx == TAILQ_FIRST(&blob->persists_to_complete)); 1753 1754 /* Complete all persists that were pending when the current persist started */ 1755 TAILQ_FOREACH_SAFE(next_persist, &blob->persists_to_complete, link, tmp) { 1756 TAILQ_REMOVE(&blob->persists_to_complete, next_persist, link); 1757 spdk_thread_send_msg(spdk_get_thread(), blob_persist_complete_cb, next_persist); 1758 } 1759 1760 if (TAILQ_EMPTY(&blob->pending_persists)) { 1761 return; 1762 } 1763 1764 /* Queue up all pending persists for completion and start blob persist with first one */ 1765 TAILQ_SWAP(&blob->persists_to_complete, &blob->pending_persists, spdk_blob_persist_ctx, link); 1766 next_persist = TAILQ_FIRST(&blob->persists_to_complete); 1767 1768 blob->state = SPDK_BLOB_STATE_DIRTY; 1769 bs_mark_dirty(seq, blob->bs, blob_persist_start, next_persist); 1770 } 1771 1772 static void 1773 blob_persist_clear_extents_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 1774 { 1775 struct spdk_blob_persist_ctx *ctx = cb_arg; 1776 struct spdk_blob *blob = ctx->blob; 1777 struct spdk_blob_store *bs = blob->bs; 1778 size_t i; 1779 1780 if (bserrno != 0) { 1781 blob_persist_complete(seq, ctx, bserrno); 1782 return; 1783 } 1784 1785 spdk_spin_lock(&bs->used_lock); 1786 1787 /* Release all extent_pages that were truncated */ 1788 for (i = blob->active.num_extent_pages; i < blob->active.extent_pages_array_size; i++) { 1789 /* Nothing to release if it was not allocated */ 1790 if (blob->active.extent_pages[i] != 0) { 1791 bs_release_md_page(bs, blob->active.extent_pages[i]); 1792 } 1793 } 1794 1795 spdk_spin_unlock(&bs->used_lock); 1796 1797 if (blob->active.num_extent_pages == 0) { 1798 free(blob->active.extent_pages); 1799 blob->active.extent_pages = NULL; 1800 blob->active.extent_pages_array_size = 0; 1801 } else if (blob->active.num_extent_pages != blob->active.extent_pages_array_size) { 1802 #ifndef __clang_analyzer__ 1803 void *tmp; 1804 1805 /* scan-build really can't figure reallocs, workaround it */ 1806 tmp = realloc(blob->active.extent_pages, sizeof(uint32_t) * blob->active.num_extent_pages); 1807 assert(tmp != NULL); 1808 blob->active.extent_pages = tmp; 1809 #endif 1810 blob->active.extent_pages_array_size = blob->active.num_extent_pages; 1811 } 1812 1813 blob_persist_complete(seq, ctx, bserrno); 1814 } 1815 1816 static void 1817 blob_persist_clear_extents(spdk_bs_sequence_t *seq, struct spdk_blob_persist_ctx *ctx) 1818 { 1819 struct spdk_blob *blob = ctx->blob; 1820 struct spdk_blob_store *bs = blob->bs; 1821 size_t i; 1822 uint64_t lba; 1823 uint64_t lba_count; 1824 spdk_bs_batch_t *batch; 1825 1826 batch = bs_sequence_to_batch(seq, blob_persist_clear_extents_cpl, ctx); 1827 lba_count = bs_byte_to_lba(bs, SPDK_BS_PAGE_SIZE); 1828 1829 /* Clear all extent_pages that were truncated */ 1830 for (i = blob->active.num_extent_pages; i < blob->active.extent_pages_array_size; i++) { 1831 /* Nothing to clear if it was not allocated */ 1832 if (blob->active.extent_pages[i] != 0) { 1833 lba = bs_md_page_to_lba(bs, blob->active.extent_pages[i]); 1834 bs_batch_write_zeroes_dev(batch, lba, lba_count); 1835 } 1836 } 1837 1838 bs_batch_close(batch); 1839 } 1840 1841 static void 1842 blob_persist_clear_clusters_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 1843 { 1844 struct spdk_blob_persist_ctx *ctx = cb_arg; 1845 struct spdk_blob *blob = ctx->blob; 1846 struct spdk_blob_store *bs = blob->bs; 1847 size_t i; 1848 1849 if (bserrno != 0) { 1850 blob_persist_complete(seq, ctx, bserrno); 1851 return; 1852 } 1853 1854 spdk_spin_lock(&bs->used_lock); 1855 /* Release all clusters that were truncated */ 1856 for (i = blob->active.num_clusters; i < blob->active.cluster_array_size; i++) { 1857 uint32_t cluster_num = bs_lba_to_cluster(bs, blob->active.clusters[i]); 1858 1859 /* Nothing to release if it was not allocated */ 1860 if (blob->active.clusters[i] != 0) { 1861 bs_release_cluster(bs, cluster_num); 1862 } 1863 } 1864 spdk_spin_unlock(&bs->used_lock); 1865 1866 if (blob->active.num_clusters == 0) { 1867 free(blob->active.clusters); 1868 blob->active.clusters = NULL; 1869 blob->active.cluster_array_size = 0; 1870 } else if (blob->active.num_clusters != blob->active.cluster_array_size) { 1871 #ifndef __clang_analyzer__ 1872 void *tmp; 1873 1874 /* scan-build really can't figure reallocs, workaround it */ 1875 tmp = realloc(blob->active.clusters, sizeof(*blob->active.clusters) * blob->active.num_clusters); 1876 assert(tmp != NULL); 1877 blob->active.clusters = tmp; 1878 1879 #endif 1880 blob->active.cluster_array_size = blob->active.num_clusters; 1881 } 1882 1883 /* Move on to clearing extent pages */ 1884 blob_persist_clear_extents(seq, ctx); 1885 } 1886 1887 static void 1888 blob_persist_clear_clusters(spdk_bs_sequence_t *seq, struct spdk_blob_persist_ctx *ctx) 1889 { 1890 struct spdk_blob *blob = ctx->blob; 1891 struct spdk_blob_store *bs = blob->bs; 1892 spdk_bs_batch_t *batch; 1893 size_t i; 1894 uint64_t lba; 1895 uint64_t lba_count; 1896 1897 /* Clusters don't move around in blobs. The list shrinks or grows 1898 * at the end, but no changes ever occur in the middle of the list. 1899 */ 1900 1901 batch = bs_sequence_to_batch(seq, blob_persist_clear_clusters_cpl, ctx); 1902 1903 /* Clear all clusters that were truncated */ 1904 lba = 0; 1905 lba_count = 0; 1906 for (i = blob->active.num_clusters; i < blob->active.cluster_array_size; i++) { 1907 uint64_t next_lba = blob->active.clusters[i]; 1908 uint64_t next_lba_count = bs_cluster_to_lba(bs, 1); 1909 1910 if (next_lba > 0 && (lba + lba_count) == next_lba) { 1911 /* This cluster is contiguous with the previous one. */ 1912 lba_count += next_lba_count; 1913 continue; 1914 } else if (next_lba == 0) { 1915 continue; 1916 } 1917 1918 /* This cluster is not contiguous with the previous one. */ 1919 1920 /* If a run of LBAs previously existing, clear them now */ 1921 if (lba_count > 0) { 1922 bs_batch_clear_dev(ctx, batch, lba, lba_count); 1923 } 1924 1925 /* Start building the next batch */ 1926 lba = next_lba; 1927 if (next_lba > 0) { 1928 lba_count = next_lba_count; 1929 } else { 1930 lba_count = 0; 1931 } 1932 } 1933 1934 /* If we ended with a contiguous set of LBAs, clear them now */ 1935 if (lba_count > 0) { 1936 bs_batch_clear_dev(ctx, batch, lba, lba_count); 1937 } 1938 1939 bs_batch_close(batch); 1940 } 1941 1942 static void 1943 blob_persist_zero_pages_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 1944 { 1945 struct spdk_blob_persist_ctx *ctx = cb_arg; 1946 struct spdk_blob *blob = ctx->blob; 1947 struct spdk_blob_store *bs = blob->bs; 1948 size_t i; 1949 1950 if (bserrno != 0) { 1951 blob_persist_complete(seq, ctx, bserrno); 1952 return; 1953 } 1954 1955 spdk_spin_lock(&bs->used_lock); 1956 1957 /* This loop starts at 1 because the first page is special and handled 1958 * below. The pages (except the first) are never written in place, 1959 * so any pages in the clean list must be zeroed. 1960 */ 1961 for (i = 1; i < blob->clean.num_pages; i++) { 1962 bs_release_md_page(bs, blob->clean.pages[i]); 1963 } 1964 1965 if (blob->active.num_pages == 0) { 1966 uint32_t page_num; 1967 1968 page_num = bs_blobid_to_page(blob->id); 1969 bs_release_md_page(bs, page_num); 1970 } 1971 1972 spdk_spin_unlock(&bs->used_lock); 1973 1974 /* Move on to clearing clusters */ 1975 blob_persist_clear_clusters(seq, ctx); 1976 } 1977 1978 static void 1979 blob_persist_zero_pages(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 1980 { 1981 struct spdk_blob_persist_ctx *ctx = cb_arg; 1982 struct spdk_blob *blob = ctx->blob; 1983 struct spdk_blob_store *bs = blob->bs; 1984 uint64_t lba; 1985 uint64_t lba_count; 1986 spdk_bs_batch_t *batch; 1987 size_t i; 1988 1989 if (bserrno != 0) { 1990 blob_persist_complete(seq, ctx, bserrno); 1991 return; 1992 } 1993 1994 batch = bs_sequence_to_batch(seq, blob_persist_zero_pages_cpl, ctx); 1995 1996 lba_count = bs_byte_to_lba(bs, SPDK_BS_PAGE_SIZE); 1997 1998 /* This loop starts at 1 because the first page is special and handled 1999 * below. The pages (except the first) are never written in place, 2000 * so any pages in the clean list must be zeroed. 2001 */ 2002 for (i = 1; i < blob->clean.num_pages; i++) { 2003 lba = bs_md_page_to_lba(bs, blob->clean.pages[i]); 2004 2005 bs_batch_write_zeroes_dev(batch, lba, lba_count); 2006 } 2007 2008 /* The first page will only be zeroed if this is a delete. */ 2009 if (blob->active.num_pages == 0) { 2010 uint32_t page_num; 2011 2012 /* The first page in the metadata goes where the blobid indicates */ 2013 page_num = bs_blobid_to_page(blob->id); 2014 lba = bs_md_page_to_lba(bs, page_num); 2015 2016 bs_batch_write_zeroes_dev(batch, lba, lba_count); 2017 } 2018 2019 bs_batch_close(batch); 2020 } 2021 2022 static void 2023 blob_persist_write_page_root(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 2024 { 2025 struct spdk_blob_persist_ctx *ctx = cb_arg; 2026 struct spdk_blob *blob = ctx->blob; 2027 struct spdk_blob_store *bs = blob->bs; 2028 uint64_t lba; 2029 uint32_t lba_count; 2030 struct spdk_blob_md_page *page; 2031 2032 if (bserrno != 0) { 2033 blob_persist_complete(seq, ctx, bserrno); 2034 return; 2035 } 2036 2037 if (blob->active.num_pages == 0) { 2038 /* Move on to the next step */ 2039 blob_persist_zero_pages(seq, ctx, 0); 2040 return; 2041 } 2042 2043 lba_count = bs_byte_to_lba(bs, sizeof(*page)); 2044 2045 page = &ctx->pages[0]; 2046 /* The first page in the metadata goes where the blobid indicates */ 2047 lba = bs_md_page_to_lba(bs, bs_blobid_to_page(blob->id)); 2048 2049 bs_sequence_write_dev(seq, page, lba, lba_count, 2050 blob_persist_zero_pages, ctx); 2051 } 2052 2053 static void 2054 blob_persist_write_page_chain(spdk_bs_sequence_t *seq, struct spdk_blob_persist_ctx *ctx) 2055 { 2056 struct spdk_blob *blob = ctx->blob; 2057 struct spdk_blob_store *bs = blob->bs; 2058 uint64_t lba; 2059 uint32_t lba_count; 2060 struct spdk_blob_md_page *page; 2061 spdk_bs_batch_t *batch; 2062 size_t i; 2063 2064 /* Clusters don't move around in blobs. The list shrinks or grows 2065 * at the end, but no changes ever occur in the middle of the list. 2066 */ 2067 2068 lba_count = bs_byte_to_lba(bs, sizeof(*page)); 2069 2070 batch = bs_sequence_to_batch(seq, blob_persist_write_page_root, ctx); 2071 2072 /* This starts at 1. The root page is not written until 2073 * all of the others are finished 2074 */ 2075 for (i = 1; i < blob->active.num_pages; i++) { 2076 page = &ctx->pages[i]; 2077 assert(page->sequence_num == i); 2078 2079 lba = bs_md_page_to_lba(bs, blob->active.pages[i]); 2080 2081 bs_batch_write_dev(batch, page, lba, lba_count); 2082 } 2083 2084 bs_batch_close(batch); 2085 } 2086 2087 static int 2088 blob_resize(struct spdk_blob *blob, uint64_t sz) 2089 { 2090 uint64_t i; 2091 uint64_t *tmp; 2092 uint64_t cluster; 2093 uint32_t lfmd; /* lowest free md page */ 2094 uint64_t num_clusters; 2095 uint32_t *ep_tmp; 2096 uint64_t new_num_ep = 0, current_num_ep = 0; 2097 struct spdk_blob_store *bs; 2098 int rc; 2099 2100 bs = blob->bs; 2101 2102 blob_verify_md_op(blob); 2103 2104 if (blob->active.num_clusters == sz) { 2105 return 0; 2106 } 2107 2108 if (blob->active.num_clusters < blob->active.cluster_array_size) { 2109 /* If this blob was resized to be larger, then smaller, then 2110 * larger without syncing, then the cluster array already 2111 * contains spare assigned clusters we can use. 2112 */ 2113 num_clusters = spdk_min(blob->active.cluster_array_size, 2114 sz); 2115 } else { 2116 num_clusters = blob->active.num_clusters; 2117 } 2118 2119 if (blob->use_extent_table) { 2120 /* Round up since every cluster beyond current Extent Table size, 2121 * requires new extent page. */ 2122 new_num_ep = spdk_divide_round_up(sz, SPDK_EXTENTS_PER_EP); 2123 current_num_ep = spdk_divide_round_up(num_clusters, SPDK_EXTENTS_PER_EP); 2124 } 2125 2126 assert(!spdk_spin_held(&bs->used_lock)); 2127 2128 /* Check first that we have enough clusters and md pages before we start claiming them. 2129 * bs->used_lock is held to ensure that clusters we think are free are still free when we go 2130 * to claim them later in this function. 2131 */ 2132 if (sz > num_clusters && spdk_blob_is_thin_provisioned(blob) == false) { 2133 spdk_spin_lock(&bs->used_lock); 2134 if ((sz - num_clusters) > bs->num_free_clusters) { 2135 rc = -ENOSPC; 2136 goto out; 2137 } 2138 lfmd = 0; 2139 for (i = current_num_ep; i < new_num_ep ; i++) { 2140 lfmd = spdk_bit_array_find_first_clear(blob->bs->used_md_pages, lfmd); 2141 if (lfmd == UINT32_MAX) { 2142 /* No more free md pages. Cannot satisfy the request */ 2143 rc = -ENOSPC; 2144 goto out; 2145 } 2146 } 2147 } 2148 2149 if (sz > num_clusters) { 2150 /* Expand the cluster array if necessary. 2151 * We only shrink the array when persisting. 2152 */ 2153 tmp = realloc(blob->active.clusters, sizeof(*blob->active.clusters) * sz); 2154 if (sz > 0 && tmp == NULL) { 2155 rc = -ENOMEM; 2156 goto out; 2157 } 2158 memset(tmp + blob->active.cluster_array_size, 0, 2159 sizeof(*blob->active.clusters) * (sz - blob->active.cluster_array_size)); 2160 blob->active.clusters = tmp; 2161 blob->active.cluster_array_size = sz; 2162 2163 /* Expand the extents table, only if enough clusters were added */ 2164 if (new_num_ep > current_num_ep && blob->use_extent_table) { 2165 ep_tmp = realloc(blob->active.extent_pages, sizeof(*blob->active.extent_pages) * new_num_ep); 2166 if (new_num_ep > 0 && ep_tmp == NULL) { 2167 rc = -ENOMEM; 2168 goto out; 2169 } 2170 memset(ep_tmp + blob->active.extent_pages_array_size, 0, 2171 sizeof(*blob->active.extent_pages) * (new_num_ep - blob->active.extent_pages_array_size)); 2172 blob->active.extent_pages = ep_tmp; 2173 blob->active.extent_pages_array_size = new_num_ep; 2174 } 2175 } 2176 2177 blob->state = SPDK_BLOB_STATE_DIRTY; 2178 2179 if (spdk_blob_is_thin_provisioned(blob) == false) { 2180 cluster = 0; 2181 lfmd = 0; 2182 for (i = num_clusters; i < sz; i++) { 2183 bs_allocate_cluster(blob, i, &cluster, &lfmd, true); 2184 /* Do not increment lfmd here. lfmd will get updated 2185 * to the md_page allocated (if any) when a new extent 2186 * page is needed. Just pass that value again, 2187 * bs_allocate_cluster will just start at that index 2188 * to find the next free md_page when needed. 2189 */ 2190 } 2191 } 2192 2193 blob->active.num_clusters = sz; 2194 blob->active.num_extent_pages = new_num_ep; 2195 2196 rc = 0; 2197 out: 2198 if (spdk_spin_held(&bs->used_lock)) { 2199 spdk_spin_unlock(&bs->used_lock); 2200 } 2201 2202 return rc; 2203 } 2204 2205 static void 2206 blob_persist_generate_new_md(struct spdk_blob_persist_ctx *ctx) 2207 { 2208 spdk_bs_sequence_t *seq = ctx->seq; 2209 struct spdk_blob *blob = ctx->blob; 2210 struct spdk_blob_store *bs = blob->bs; 2211 uint64_t i; 2212 uint32_t page_num; 2213 void *tmp; 2214 int rc; 2215 2216 /* Generate the new metadata */ 2217 rc = blob_serialize(blob, &ctx->pages, &blob->active.num_pages); 2218 if (rc < 0) { 2219 blob_persist_complete(seq, ctx, rc); 2220 return; 2221 } 2222 2223 assert(blob->active.num_pages >= 1); 2224 2225 /* Resize the cache of page indices */ 2226 tmp = realloc(blob->active.pages, blob->active.num_pages * sizeof(*blob->active.pages)); 2227 if (!tmp) { 2228 blob_persist_complete(seq, ctx, -ENOMEM); 2229 return; 2230 } 2231 blob->active.pages = tmp; 2232 2233 /* Assign this metadata to pages. This requires two passes - one to verify that there are 2234 * enough pages and a second to actually claim them. The used_lock is held across 2235 * both passes to ensure things don't change in the middle. 2236 */ 2237 spdk_spin_lock(&bs->used_lock); 2238 page_num = 0; 2239 /* Note that this loop starts at one. The first page location is fixed by the blobid. */ 2240 for (i = 1; i < blob->active.num_pages; i++) { 2241 page_num = spdk_bit_array_find_first_clear(bs->used_md_pages, page_num); 2242 if (page_num == UINT32_MAX) { 2243 spdk_spin_unlock(&bs->used_lock); 2244 blob_persist_complete(seq, ctx, -ENOMEM); 2245 return; 2246 } 2247 page_num++; 2248 } 2249 2250 page_num = 0; 2251 blob->active.pages[0] = bs_blobid_to_page(blob->id); 2252 for (i = 1; i < blob->active.num_pages; i++) { 2253 page_num = spdk_bit_array_find_first_clear(bs->used_md_pages, page_num); 2254 ctx->pages[i - 1].next = page_num; 2255 /* Now that previous metadata page is complete, calculate the crc for it. */ 2256 ctx->pages[i - 1].crc = blob_md_page_calc_crc(&ctx->pages[i - 1]); 2257 blob->active.pages[i] = page_num; 2258 bs_claim_md_page(bs, page_num); 2259 SPDK_DEBUGLOG(blob, "Claiming page %u for blob 0x%" PRIx64 "\n", page_num, 2260 blob->id); 2261 page_num++; 2262 } 2263 spdk_spin_unlock(&bs->used_lock); 2264 ctx->pages[i - 1].crc = blob_md_page_calc_crc(&ctx->pages[i - 1]); 2265 /* Start writing the metadata from last page to first */ 2266 blob->state = SPDK_BLOB_STATE_CLEAN; 2267 blob_persist_write_page_chain(seq, ctx); 2268 } 2269 2270 static void 2271 blob_persist_write_extent_pages(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 2272 { 2273 struct spdk_blob_persist_ctx *ctx = cb_arg; 2274 struct spdk_blob *blob = ctx->blob; 2275 size_t i; 2276 uint32_t extent_page_id; 2277 uint32_t page_count = 0; 2278 int rc; 2279 2280 if (ctx->extent_page != NULL) { 2281 spdk_free(ctx->extent_page); 2282 ctx->extent_page = NULL; 2283 } 2284 2285 if (bserrno != 0) { 2286 blob_persist_complete(seq, ctx, bserrno); 2287 return; 2288 } 2289 2290 /* Only write out Extent Pages when blob was resized. */ 2291 for (i = ctx->next_extent_page; i < blob->active.extent_pages_array_size; i++) { 2292 extent_page_id = blob->active.extent_pages[i]; 2293 if (extent_page_id == 0) { 2294 /* No Extent Page to persist */ 2295 assert(spdk_blob_is_thin_provisioned(blob)); 2296 continue; 2297 } 2298 assert(spdk_bit_array_get(blob->bs->used_md_pages, extent_page_id)); 2299 ctx->next_extent_page = i + 1; 2300 rc = blob_serialize_add_page(ctx->blob, &ctx->extent_page, &page_count, &ctx->extent_page); 2301 if (rc < 0) { 2302 blob_persist_complete(seq, ctx, rc); 2303 return; 2304 } 2305 2306 blob->state = SPDK_BLOB_STATE_DIRTY; 2307 blob_serialize_extent_page(blob, i * SPDK_EXTENTS_PER_EP, ctx->extent_page); 2308 2309 ctx->extent_page->crc = blob_md_page_calc_crc(ctx->extent_page); 2310 2311 bs_sequence_write_dev(seq, ctx->extent_page, bs_md_page_to_lba(blob->bs, extent_page_id), 2312 bs_byte_to_lba(blob->bs, SPDK_BS_PAGE_SIZE), 2313 blob_persist_write_extent_pages, ctx); 2314 return; 2315 } 2316 2317 blob_persist_generate_new_md(ctx); 2318 } 2319 2320 static void 2321 blob_persist_start(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 2322 { 2323 struct spdk_blob_persist_ctx *ctx = cb_arg; 2324 struct spdk_blob *blob = ctx->blob; 2325 2326 if (bserrno != 0) { 2327 blob_persist_complete(seq, ctx, bserrno); 2328 return; 2329 } 2330 2331 if (blob->active.num_pages == 0) { 2332 /* This is the signal that the blob should be deleted. 2333 * Immediately jump to the clean up routine. */ 2334 assert(blob->clean.num_pages > 0); 2335 blob->state = SPDK_BLOB_STATE_CLEAN; 2336 blob_persist_zero_pages(seq, ctx, 0); 2337 return; 2338 2339 } 2340 2341 if (blob->clean.num_clusters < blob->active.num_clusters) { 2342 /* Blob was resized up */ 2343 assert(blob->clean.num_extent_pages <= blob->active.num_extent_pages); 2344 ctx->next_extent_page = spdk_max(1, blob->clean.num_extent_pages) - 1; 2345 } else if (blob->active.num_clusters < blob->active.cluster_array_size) { 2346 /* Blob was resized down */ 2347 assert(blob->clean.num_extent_pages >= blob->active.num_extent_pages); 2348 ctx->next_extent_page = spdk_max(1, blob->active.num_extent_pages) - 1; 2349 } else { 2350 /* No change in size occurred */ 2351 blob_persist_generate_new_md(ctx); 2352 return; 2353 } 2354 2355 blob_persist_write_extent_pages(seq, ctx, 0); 2356 } 2357 2358 struct spdk_bs_mark_dirty { 2359 struct spdk_blob_store *bs; 2360 struct spdk_bs_super_block *super; 2361 spdk_bs_sequence_cpl cb_fn; 2362 void *cb_arg; 2363 }; 2364 2365 static void 2366 bs_mark_dirty_write_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 2367 { 2368 struct spdk_bs_mark_dirty *ctx = cb_arg; 2369 2370 if (bserrno == 0) { 2371 ctx->bs->clean = 0; 2372 } 2373 2374 ctx->cb_fn(seq, ctx->cb_arg, bserrno); 2375 2376 spdk_free(ctx->super); 2377 free(ctx); 2378 } 2379 2380 static void bs_write_super(spdk_bs_sequence_t *seq, struct spdk_blob_store *bs, 2381 struct spdk_bs_super_block *super, spdk_bs_sequence_cpl cb_fn, void *cb_arg); 2382 2383 2384 static void 2385 bs_mark_dirty_write(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 2386 { 2387 struct spdk_bs_mark_dirty *ctx = cb_arg; 2388 2389 if (bserrno != 0) { 2390 bs_mark_dirty_write_cpl(seq, ctx, bserrno); 2391 return; 2392 } 2393 2394 ctx->super->clean = 0; 2395 if (ctx->super->size == 0) { 2396 ctx->super->size = ctx->bs->dev->blockcnt * ctx->bs->dev->blocklen; 2397 } 2398 2399 bs_write_super(seq, ctx->bs, ctx->super, bs_mark_dirty_write_cpl, ctx); 2400 } 2401 2402 static void 2403 bs_mark_dirty(spdk_bs_sequence_t *seq, struct spdk_blob_store *bs, 2404 spdk_bs_sequence_cpl cb_fn, void *cb_arg) 2405 { 2406 struct spdk_bs_mark_dirty *ctx; 2407 2408 /* Blobstore is already marked dirty */ 2409 if (bs->clean == 0) { 2410 cb_fn(seq, cb_arg, 0); 2411 return; 2412 } 2413 2414 ctx = calloc(1, sizeof(*ctx)); 2415 if (!ctx) { 2416 cb_fn(seq, cb_arg, -ENOMEM); 2417 return; 2418 } 2419 ctx->bs = bs; 2420 ctx->cb_fn = cb_fn; 2421 ctx->cb_arg = cb_arg; 2422 2423 ctx->super = spdk_zmalloc(sizeof(*ctx->super), 0x1000, NULL, 2424 SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA); 2425 if (!ctx->super) { 2426 free(ctx); 2427 cb_fn(seq, cb_arg, -ENOMEM); 2428 return; 2429 } 2430 2431 bs_sequence_read_dev(seq, ctx->super, bs_page_to_lba(bs, 0), 2432 bs_byte_to_lba(bs, sizeof(*ctx->super)), 2433 bs_mark_dirty_write, ctx); 2434 } 2435 2436 /* Write a blob to disk */ 2437 static void 2438 blob_persist(spdk_bs_sequence_t *seq, struct spdk_blob *blob, 2439 spdk_bs_sequence_cpl cb_fn, void *cb_arg) 2440 { 2441 struct spdk_blob_persist_ctx *ctx; 2442 2443 blob_verify_md_op(blob); 2444 2445 if (blob->state == SPDK_BLOB_STATE_CLEAN && TAILQ_EMPTY(&blob->persists_to_complete)) { 2446 cb_fn(seq, cb_arg, 0); 2447 return; 2448 } 2449 2450 ctx = calloc(1, sizeof(*ctx)); 2451 if (!ctx) { 2452 cb_fn(seq, cb_arg, -ENOMEM); 2453 return; 2454 } 2455 ctx->blob = blob; 2456 ctx->seq = seq; 2457 ctx->cb_fn = cb_fn; 2458 ctx->cb_arg = cb_arg; 2459 2460 /* Multiple blob persists can affect one another, via blob->state or 2461 * blob mutable data changes. To prevent it, queue up the persists. */ 2462 if (!TAILQ_EMPTY(&blob->persists_to_complete)) { 2463 TAILQ_INSERT_TAIL(&blob->pending_persists, ctx, link); 2464 return; 2465 } 2466 TAILQ_INSERT_HEAD(&blob->persists_to_complete, ctx, link); 2467 2468 bs_mark_dirty(seq, blob->bs, blob_persist_start, ctx); 2469 } 2470 2471 struct spdk_blob_copy_cluster_ctx { 2472 struct spdk_blob *blob; 2473 uint8_t *buf; 2474 uint64_t page; 2475 uint64_t new_cluster; 2476 uint32_t new_extent_page; 2477 spdk_bs_sequence_t *seq; 2478 struct spdk_blob_md_page *new_cluster_page; 2479 }; 2480 2481 static void 2482 blob_allocate_and_copy_cluster_cpl(void *cb_arg, int bserrno) 2483 { 2484 struct spdk_blob_copy_cluster_ctx *ctx = cb_arg; 2485 struct spdk_bs_request_set *set = (struct spdk_bs_request_set *)ctx->seq; 2486 TAILQ_HEAD(, spdk_bs_request_set) requests; 2487 spdk_bs_user_op_t *op; 2488 2489 TAILQ_INIT(&requests); 2490 TAILQ_SWAP(&set->channel->need_cluster_alloc, &requests, spdk_bs_request_set, link); 2491 2492 while (!TAILQ_EMPTY(&requests)) { 2493 op = TAILQ_FIRST(&requests); 2494 TAILQ_REMOVE(&requests, op, link); 2495 if (bserrno == 0) { 2496 bs_user_op_execute(op); 2497 } else { 2498 bs_user_op_abort(op, bserrno); 2499 } 2500 } 2501 2502 spdk_free(ctx->buf); 2503 free(ctx); 2504 } 2505 2506 static void 2507 blob_insert_cluster_cpl(void *cb_arg, int bserrno) 2508 { 2509 struct spdk_blob_copy_cluster_ctx *ctx = cb_arg; 2510 2511 if (bserrno) { 2512 if (bserrno == -EEXIST) { 2513 /* The metadata insert failed because another thread 2514 * allocated the cluster first. Free our cluster 2515 * but continue without error. */ 2516 bserrno = 0; 2517 } 2518 spdk_spin_lock(&ctx->blob->bs->used_lock); 2519 bs_release_cluster(ctx->blob->bs, ctx->new_cluster); 2520 if (ctx->new_extent_page != 0) { 2521 bs_release_md_page(ctx->blob->bs, ctx->new_extent_page); 2522 } 2523 spdk_spin_unlock(&ctx->blob->bs->used_lock); 2524 } 2525 2526 bs_sequence_finish(ctx->seq, bserrno); 2527 } 2528 2529 static void 2530 blob_write_copy_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 2531 { 2532 struct spdk_blob_copy_cluster_ctx *ctx = cb_arg; 2533 uint32_t cluster_number; 2534 2535 if (bserrno) { 2536 /* The write failed, so jump to the final completion handler */ 2537 bs_sequence_finish(seq, bserrno); 2538 return; 2539 } 2540 2541 cluster_number = bs_page_to_cluster(ctx->blob->bs, ctx->page); 2542 2543 blob_insert_cluster_on_md_thread(ctx->blob, cluster_number, ctx->new_cluster, 2544 ctx->new_extent_page, ctx->new_cluster_page, blob_insert_cluster_cpl, ctx); 2545 } 2546 2547 static void 2548 blob_write_copy(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 2549 { 2550 struct spdk_blob_copy_cluster_ctx *ctx = cb_arg; 2551 2552 if (bserrno != 0) { 2553 /* The read failed, so jump to the final completion handler */ 2554 bs_sequence_finish(seq, bserrno); 2555 return; 2556 } 2557 2558 /* Write whole cluster */ 2559 bs_sequence_write_dev(seq, ctx->buf, 2560 bs_cluster_to_lba(ctx->blob->bs, ctx->new_cluster), 2561 bs_cluster_to_lba(ctx->blob->bs, 1), 2562 blob_write_copy_cpl, ctx); 2563 } 2564 2565 static bool 2566 blob_can_copy(struct spdk_blob *blob, uint32_t cluster_start_page, uint64_t *base_lba) 2567 { 2568 uint64_t lba = bs_dev_page_to_lba(blob->back_bs_dev, cluster_start_page); 2569 2570 return (!blob_is_esnap_clone(blob) && blob->bs->dev->copy != NULL) && 2571 blob->back_bs_dev->translate_lba(blob->back_bs_dev, lba, base_lba); 2572 } 2573 2574 static void 2575 blob_copy(struct spdk_blob_copy_cluster_ctx *ctx, spdk_bs_user_op_t *op, uint64_t src_lba) 2576 { 2577 struct spdk_blob *blob = ctx->blob; 2578 uint64_t lba_count = bs_dev_byte_to_lba(blob->back_bs_dev, blob->bs->cluster_sz); 2579 2580 bs_sequence_copy_dev(ctx->seq, 2581 bs_cluster_to_lba(blob->bs, ctx->new_cluster), 2582 src_lba, 2583 lba_count, 2584 blob_write_copy_cpl, ctx); 2585 } 2586 2587 static void 2588 bs_allocate_and_copy_cluster(struct spdk_blob *blob, 2589 struct spdk_io_channel *_ch, 2590 uint64_t io_unit, spdk_bs_user_op_t *op) 2591 { 2592 struct spdk_bs_cpl cpl; 2593 struct spdk_bs_channel *ch; 2594 struct spdk_blob_copy_cluster_ctx *ctx; 2595 uint32_t cluster_start_page; 2596 uint32_t cluster_number; 2597 bool is_zeroes; 2598 bool can_copy; 2599 uint64_t copy_src_lba; 2600 int rc; 2601 2602 ch = spdk_io_channel_get_ctx(_ch); 2603 2604 if (!TAILQ_EMPTY(&ch->need_cluster_alloc)) { 2605 /* There are already operations pending. Queue this user op 2606 * and return because it will be re-executed when the outstanding 2607 * cluster allocation completes. */ 2608 TAILQ_INSERT_TAIL(&ch->need_cluster_alloc, op, link); 2609 return; 2610 } 2611 2612 /* Round the io_unit offset down to the first page in the cluster */ 2613 cluster_start_page = bs_io_unit_to_cluster_start(blob, io_unit); 2614 2615 /* Calculate which index in the metadata cluster array the corresponding 2616 * cluster is supposed to be at. */ 2617 cluster_number = bs_io_unit_to_cluster_number(blob, io_unit); 2618 2619 ctx = calloc(1, sizeof(*ctx)); 2620 if (!ctx) { 2621 bs_user_op_abort(op, -ENOMEM); 2622 return; 2623 } 2624 2625 assert(blob->bs->cluster_sz % blob->back_bs_dev->blocklen == 0); 2626 2627 ctx->blob = blob; 2628 ctx->page = cluster_start_page; 2629 ctx->new_cluster_page = ch->new_cluster_page; 2630 memset(ctx->new_cluster_page, 0, SPDK_BS_PAGE_SIZE); 2631 can_copy = blob_can_copy(blob, cluster_start_page, ©_src_lba); 2632 2633 is_zeroes = blob->back_bs_dev->is_zeroes(blob->back_bs_dev, 2634 bs_dev_page_to_lba(blob->back_bs_dev, cluster_start_page), 2635 bs_dev_byte_to_lba(blob->back_bs_dev, blob->bs->cluster_sz)); 2636 if (blob->parent_id != SPDK_BLOBID_INVALID && !is_zeroes && !can_copy) { 2637 ctx->buf = spdk_malloc(blob->bs->cluster_sz, blob->back_bs_dev->blocklen, 2638 NULL, SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA); 2639 if (!ctx->buf) { 2640 SPDK_ERRLOG("DMA allocation for cluster of size = %" PRIu32 " failed.\n", 2641 blob->bs->cluster_sz); 2642 free(ctx); 2643 bs_user_op_abort(op, -ENOMEM); 2644 return; 2645 } 2646 } 2647 2648 spdk_spin_lock(&blob->bs->used_lock); 2649 rc = bs_allocate_cluster(blob, cluster_number, &ctx->new_cluster, &ctx->new_extent_page, 2650 false); 2651 spdk_spin_unlock(&blob->bs->used_lock); 2652 if (rc != 0) { 2653 spdk_free(ctx->buf); 2654 free(ctx); 2655 bs_user_op_abort(op, rc); 2656 return; 2657 } 2658 2659 cpl.type = SPDK_BS_CPL_TYPE_BLOB_BASIC; 2660 cpl.u.blob_basic.cb_fn = blob_allocate_and_copy_cluster_cpl; 2661 cpl.u.blob_basic.cb_arg = ctx; 2662 2663 ctx->seq = bs_sequence_start_blob(_ch, &cpl, blob); 2664 if (!ctx->seq) { 2665 spdk_spin_lock(&blob->bs->used_lock); 2666 bs_release_cluster(blob->bs, ctx->new_cluster); 2667 spdk_spin_unlock(&blob->bs->used_lock); 2668 spdk_free(ctx->buf); 2669 free(ctx); 2670 bs_user_op_abort(op, -ENOMEM); 2671 return; 2672 } 2673 2674 /* Queue the user op to block other incoming operations */ 2675 TAILQ_INSERT_TAIL(&ch->need_cluster_alloc, op, link); 2676 2677 if (blob->parent_id != SPDK_BLOBID_INVALID && !is_zeroes) { 2678 if (can_copy) { 2679 blob_copy(ctx, op, copy_src_lba); 2680 } else { 2681 /* Read cluster from backing device */ 2682 bs_sequence_read_bs_dev(ctx->seq, blob->back_bs_dev, ctx->buf, 2683 bs_dev_page_to_lba(blob->back_bs_dev, cluster_start_page), 2684 bs_dev_byte_to_lba(blob->back_bs_dev, blob->bs->cluster_sz), 2685 blob_write_copy, ctx); 2686 } 2687 2688 } else { 2689 blob_insert_cluster_on_md_thread(ctx->blob, cluster_number, ctx->new_cluster, 2690 ctx->new_extent_page, ctx->new_cluster_page, blob_insert_cluster_cpl, ctx); 2691 } 2692 } 2693 2694 static inline bool 2695 blob_calculate_lba_and_lba_count(struct spdk_blob *blob, uint64_t io_unit, uint64_t length, 2696 uint64_t *lba, uint64_t *lba_count) 2697 { 2698 *lba_count = length; 2699 2700 if (!bs_io_unit_is_allocated(blob, io_unit)) { 2701 assert(blob->back_bs_dev != NULL); 2702 *lba = bs_io_unit_to_back_dev_lba(blob, io_unit); 2703 *lba_count = bs_io_unit_to_back_dev_lba(blob, *lba_count); 2704 return false; 2705 } else { 2706 *lba = bs_blob_io_unit_to_lba(blob, io_unit); 2707 return true; 2708 } 2709 } 2710 2711 struct op_split_ctx { 2712 struct spdk_blob *blob; 2713 struct spdk_io_channel *channel; 2714 uint64_t io_unit_offset; 2715 uint64_t io_units_remaining; 2716 void *curr_payload; 2717 enum spdk_blob_op_type op_type; 2718 spdk_bs_sequence_t *seq; 2719 bool in_submit_ctx; 2720 bool completed_in_submit_ctx; 2721 bool done; 2722 }; 2723 2724 static void 2725 blob_request_submit_op_split_next(void *cb_arg, int bserrno) 2726 { 2727 struct op_split_ctx *ctx = cb_arg; 2728 struct spdk_blob *blob = ctx->blob; 2729 struct spdk_io_channel *ch = ctx->channel; 2730 enum spdk_blob_op_type op_type = ctx->op_type; 2731 uint8_t *buf; 2732 uint64_t offset; 2733 uint64_t length; 2734 uint64_t op_length; 2735 2736 if (bserrno != 0 || ctx->io_units_remaining == 0) { 2737 bs_sequence_finish(ctx->seq, bserrno); 2738 if (ctx->in_submit_ctx) { 2739 /* Defer freeing of the ctx object, since it will be 2740 * accessed when this unwinds back to the submisison 2741 * context. 2742 */ 2743 ctx->done = true; 2744 } else { 2745 free(ctx); 2746 } 2747 return; 2748 } 2749 2750 if (ctx->in_submit_ctx) { 2751 /* If this split operation completed in the context 2752 * of its submission, mark the flag and return immediately 2753 * to avoid recursion. 2754 */ 2755 ctx->completed_in_submit_ctx = true; 2756 return; 2757 } 2758 2759 while (true) { 2760 ctx->completed_in_submit_ctx = false; 2761 2762 offset = ctx->io_unit_offset; 2763 length = ctx->io_units_remaining; 2764 buf = ctx->curr_payload; 2765 op_length = spdk_min(length, bs_num_io_units_to_cluster_boundary(blob, 2766 offset)); 2767 2768 /* Update length and payload for next operation */ 2769 ctx->io_units_remaining -= op_length; 2770 ctx->io_unit_offset += op_length; 2771 if (op_type == SPDK_BLOB_WRITE || op_type == SPDK_BLOB_READ) { 2772 ctx->curr_payload += op_length * blob->bs->io_unit_size; 2773 } 2774 2775 assert(!ctx->in_submit_ctx); 2776 ctx->in_submit_ctx = true; 2777 2778 switch (op_type) { 2779 case SPDK_BLOB_READ: 2780 spdk_blob_io_read(blob, ch, buf, offset, op_length, 2781 blob_request_submit_op_split_next, ctx); 2782 break; 2783 case SPDK_BLOB_WRITE: 2784 spdk_blob_io_write(blob, ch, buf, offset, op_length, 2785 blob_request_submit_op_split_next, ctx); 2786 break; 2787 case SPDK_BLOB_UNMAP: 2788 spdk_blob_io_unmap(blob, ch, offset, op_length, 2789 blob_request_submit_op_split_next, ctx); 2790 break; 2791 case SPDK_BLOB_WRITE_ZEROES: 2792 spdk_blob_io_write_zeroes(blob, ch, offset, op_length, 2793 blob_request_submit_op_split_next, ctx); 2794 break; 2795 case SPDK_BLOB_READV: 2796 case SPDK_BLOB_WRITEV: 2797 SPDK_ERRLOG("readv/write not valid\n"); 2798 bs_sequence_finish(ctx->seq, -EINVAL); 2799 free(ctx); 2800 return; 2801 } 2802 2803 #ifndef __clang_analyzer__ 2804 /* scan-build reports a false positive around accessing the ctx here. It 2805 * forms a path that recursively calls this function, but then says 2806 * "assuming ctx->in_submit_ctx is false", when that isn't possible. 2807 * This path does free(ctx), returns to here, and reports a use-after-free 2808 * bug. Wrapping this bit of code so that scan-build doesn't see it 2809 * works around the scan-build bug. 2810 */ 2811 assert(ctx->in_submit_ctx); 2812 ctx->in_submit_ctx = false; 2813 2814 /* If the operation completed immediately, loop back and submit the 2815 * next operation. Otherwise we can return and the next split 2816 * operation will get submitted when this current operation is 2817 * later completed asynchronously. 2818 */ 2819 if (ctx->completed_in_submit_ctx) { 2820 continue; 2821 } else if (ctx->done) { 2822 free(ctx); 2823 } 2824 #endif 2825 break; 2826 } 2827 } 2828 2829 static void 2830 blob_request_submit_op_split(struct spdk_io_channel *ch, struct spdk_blob *blob, 2831 void *payload, uint64_t offset, uint64_t length, 2832 spdk_blob_op_complete cb_fn, void *cb_arg, enum spdk_blob_op_type op_type) 2833 { 2834 struct op_split_ctx *ctx; 2835 spdk_bs_sequence_t *seq; 2836 struct spdk_bs_cpl cpl; 2837 2838 assert(blob != NULL); 2839 2840 ctx = calloc(1, sizeof(struct op_split_ctx)); 2841 if (ctx == NULL) { 2842 cb_fn(cb_arg, -ENOMEM); 2843 return; 2844 } 2845 2846 cpl.type = SPDK_BS_CPL_TYPE_BLOB_BASIC; 2847 cpl.u.blob_basic.cb_fn = cb_fn; 2848 cpl.u.blob_basic.cb_arg = cb_arg; 2849 2850 seq = bs_sequence_start_blob(ch, &cpl, blob); 2851 if (!seq) { 2852 free(ctx); 2853 cb_fn(cb_arg, -ENOMEM); 2854 return; 2855 } 2856 2857 ctx->blob = blob; 2858 ctx->channel = ch; 2859 ctx->curr_payload = payload; 2860 ctx->io_unit_offset = offset; 2861 ctx->io_units_remaining = length; 2862 ctx->op_type = op_type; 2863 ctx->seq = seq; 2864 2865 blob_request_submit_op_split_next(ctx, 0); 2866 } 2867 2868 static void 2869 blob_request_submit_op_single(struct spdk_io_channel *_ch, struct spdk_blob *blob, 2870 void *payload, uint64_t offset, uint64_t length, 2871 spdk_blob_op_complete cb_fn, void *cb_arg, enum spdk_blob_op_type op_type) 2872 { 2873 struct spdk_bs_cpl cpl; 2874 uint64_t lba; 2875 uint64_t lba_count; 2876 bool is_allocated; 2877 2878 assert(blob != NULL); 2879 2880 cpl.type = SPDK_BS_CPL_TYPE_BLOB_BASIC; 2881 cpl.u.blob_basic.cb_fn = cb_fn; 2882 cpl.u.blob_basic.cb_arg = cb_arg; 2883 2884 if (blob->frozen_refcnt) { 2885 /* This blob I/O is frozen */ 2886 spdk_bs_user_op_t *op; 2887 struct spdk_bs_channel *bs_channel = spdk_io_channel_get_ctx(_ch); 2888 2889 op = bs_user_op_alloc(_ch, &cpl, op_type, blob, payload, 0, offset, length); 2890 if (!op) { 2891 cb_fn(cb_arg, -ENOMEM); 2892 return; 2893 } 2894 2895 TAILQ_INSERT_TAIL(&bs_channel->queued_io, op, link); 2896 2897 return; 2898 } 2899 2900 is_allocated = blob_calculate_lba_and_lba_count(blob, offset, length, &lba, &lba_count); 2901 2902 switch (op_type) { 2903 case SPDK_BLOB_READ: { 2904 spdk_bs_batch_t *batch; 2905 2906 batch = bs_batch_open(_ch, &cpl, blob); 2907 if (!batch) { 2908 cb_fn(cb_arg, -ENOMEM); 2909 return; 2910 } 2911 2912 if (is_allocated) { 2913 /* Read from the blob */ 2914 bs_batch_read_dev(batch, payload, lba, lba_count); 2915 } else { 2916 /* Read from the backing block device */ 2917 bs_batch_read_bs_dev(batch, blob->back_bs_dev, payload, lba, lba_count); 2918 } 2919 2920 bs_batch_close(batch); 2921 break; 2922 } 2923 case SPDK_BLOB_WRITE: 2924 case SPDK_BLOB_WRITE_ZEROES: { 2925 if (is_allocated) { 2926 /* Write to the blob */ 2927 spdk_bs_batch_t *batch; 2928 2929 if (lba_count == 0) { 2930 cb_fn(cb_arg, 0); 2931 return; 2932 } 2933 2934 batch = bs_batch_open(_ch, &cpl, blob); 2935 if (!batch) { 2936 cb_fn(cb_arg, -ENOMEM); 2937 return; 2938 } 2939 2940 if (op_type == SPDK_BLOB_WRITE) { 2941 bs_batch_write_dev(batch, payload, lba, lba_count); 2942 } else { 2943 bs_batch_write_zeroes_dev(batch, lba, lba_count); 2944 } 2945 2946 bs_batch_close(batch); 2947 } else { 2948 /* Queue this operation and allocate the cluster */ 2949 spdk_bs_user_op_t *op; 2950 2951 op = bs_user_op_alloc(_ch, &cpl, op_type, blob, payload, 0, offset, length); 2952 if (!op) { 2953 cb_fn(cb_arg, -ENOMEM); 2954 return; 2955 } 2956 2957 bs_allocate_and_copy_cluster(blob, _ch, offset, op); 2958 } 2959 break; 2960 } 2961 case SPDK_BLOB_UNMAP: { 2962 spdk_bs_batch_t *batch; 2963 2964 batch = bs_batch_open(_ch, &cpl, blob); 2965 if (!batch) { 2966 cb_fn(cb_arg, -ENOMEM); 2967 return; 2968 } 2969 2970 if (is_allocated) { 2971 bs_batch_unmap_dev(batch, lba, lba_count); 2972 } 2973 2974 bs_batch_close(batch); 2975 break; 2976 } 2977 case SPDK_BLOB_READV: 2978 case SPDK_BLOB_WRITEV: 2979 SPDK_ERRLOG("readv/write not valid\n"); 2980 cb_fn(cb_arg, -EINVAL); 2981 break; 2982 } 2983 } 2984 2985 static void 2986 blob_request_submit_op(struct spdk_blob *blob, struct spdk_io_channel *_channel, 2987 void *payload, uint64_t offset, uint64_t length, 2988 spdk_blob_op_complete cb_fn, void *cb_arg, enum spdk_blob_op_type op_type) 2989 { 2990 assert(blob != NULL); 2991 2992 if (blob->data_ro && op_type != SPDK_BLOB_READ) { 2993 cb_fn(cb_arg, -EPERM); 2994 return; 2995 } 2996 2997 if (length == 0) { 2998 cb_fn(cb_arg, 0); 2999 return; 3000 } 3001 3002 if (offset + length > bs_cluster_to_lba(blob->bs, blob->active.num_clusters)) { 3003 cb_fn(cb_arg, -EINVAL); 3004 return; 3005 } 3006 if (length <= bs_num_io_units_to_cluster_boundary(blob, offset)) { 3007 blob_request_submit_op_single(_channel, blob, payload, offset, length, 3008 cb_fn, cb_arg, op_type); 3009 } else { 3010 blob_request_submit_op_split(_channel, blob, payload, offset, length, 3011 cb_fn, cb_arg, op_type); 3012 } 3013 } 3014 3015 struct rw_iov_ctx { 3016 struct spdk_blob *blob; 3017 struct spdk_io_channel *channel; 3018 spdk_blob_op_complete cb_fn; 3019 void *cb_arg; 3020 bool read; 3021 int iovcnt; 3022 struct iovec *orig_iov; 3023 uint64_t io_unit_offset; 3024 uint64_t io_units_remaining; 3025 uint64_t io_units_done; 3026 struct spdk_blob_ext_io_opts *ext_io_opts; 3027 struct iovec iov[0]; 3028 }; 3029 3030 static void 3031 rw_iov_done(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 3032 { 3033 assert(cb_arg == NULL); 3034 bs_sequence_finish(seq, bserrno); 3035 } 3036 3037 static void 3038 rw_iov_split_next(void *cb_arg, int bserrno) 3039 { 3040 struct rw_iov_ctx *ctx = cb_arg; 3041 struct spdk_blob *blob = ctx->blob; 3042 struct iovec *iov, *orig_iov; 3043 int iovcnt; 3044 size_t orig_iovoff; 3045 uint64_t io_units_count, io_units_to_boundary, io_unit_offset; 3046 uint64_t byte_count; 3047 3048 if (bserrno != 0 || ctx->io_units_remaining == 0) { 3049 ctx->cb_fn(ctx->cb_arg, bserrno); 3050 free(ctx); 3051 return; 3052 } 3053 3054 io_unit_offset = ctx->io_unit_offset; 3055 io_units_to_boundary = bs_num_io_units_to_cluster_boundary(blob, io_unit_offset); 3056 io_units_count = spdk_min(ctx->io_units_remaining, io_units_to_boundary); 3057 /* 3058 * Get index and offset into the original iov array for our current position in the I/O sequence. 3059 * byte_count will keep track of how many bytes remaining until orig_iov and orig_iovoff will 3060 * point to the current position in the I/O sequence. 3061 */ 3062 byte_count = ctx->io_units_done * blob->bs->io_unit_size; 3063 orig_iov = &ctx->orig_iov[0]; 3064 orig_iovoff = 0; 3065 while (byte_count > 0) { 3066 if (byte_count >= orig_iov->iov_len) { 3067 byte_count -= orig_iov->iov_len; 3068 orig_iov++; 3069 } else { 3070 orig_iovoff = byte_count; 3071 byte_count = 0; 3072 } 3073 } 3074 3075 /* 3076 * Build an iov array for the next I/O in the sequence. byte_count will keep track of how many 3077 * bytes of this next I/O remain to be accounted for in the new iov array. 3078 */ 3079 byte_count = io_units_count * blob->bs->io_unit_size; 3080 iov = &ctx->iov[0]; 3081 iovcnt = 0; 3082 while (byte_count > 0) { 3083 assert(iovcnt < ctx->iovcnt); 3084 iov->iov_len = spdk_min(byte_count, orig_iov->iov_len - orig_iovoff); 3085 iov->iov_base = orig_iov->iov_base + orig_iovoff; 3086 byte_count -= iov->iov_len; 3087 orig_iovoff = 0; 3088 orig_iov++; 3089 iov++; 3090 iovcnt++; 3091 } 3092 3093 ctx->io_unit_offset += io_units_count; 3094 ctx->io_units_remaining -= io_units_count; 3095 ctx->io_units_done += io_units_count; 3096 iov = &ctx->iov[0]; 3097 3098 if (ctx->read) { 3099 spdk_blob_io_readv_ext(ctx->blob, ctx->channel, iov, iovcnt, io_unit_offset, 3100 io_units_count, rw_iov_split_next, ctx, ctx->ext_io_opts); 3101 } else { 3102 spdk_blob_io_writev_ext(ctx->blob, ctx->channel, iov, iovcnt, io_unit_offset, 3103 io_units_count, rw_iov_split_next, ctx, ctx->ext_io_opts); 3104 } 3105 } 3106 3107 static void 3108 blob_request_submit_rw_iov(struct spdk_blob *blob, struct spdk_io_channel *_channel, 3109 struct iovec *iov, int iovcnt, 3110 uint64_t offset, uint64_t length, spdk_blob_op_complete cb_fn, void *cb_arg, bool read, 3111 struct spdk_blob_ext_io_opts *ext_io_opts) 3112 { 3113 struct spdk_bs_cpl cpl; 3114 3115 assert(blob != NULL); 3116 3117 if (!read && blob->data_ro) { 3118 cb_fn(cb_arg, -EPERM); 3119 return; 3120 } 3121 3122 if (length == 0) { 3123 cb_fn(cb_arg, 0); 3124 return; 3125 } 3126 3127 if (offset + length > bs_cluster_to_lba(blob->bs, blob->active.num_clusters)) { 3128 cb_fn(cb_arg, -EINVAL); 3129 return; 3130 } 3131 3132 /* 3133 * For now, we implement readv/writev using a sequence (instead of a batch) to account for having 3134 * to split a request that spans a cluster boundary. For I/O that do not span a cluster boundary, 3135 * there will be no noticeable difference compared to using a batch. For I/O that do span a cluster 3136 * boundary, the target LBAs (after blob offset to LBA translation) may not be contiguous, so we need 3137 * to allocate a separate iov array and split the I/O such that none of the resulting 3138 * smaller I/O cross a cluster boundary. These smaller I/O will be issued in sequence (not in parallel) 3139 * but since this case happens very infrequently, any performance impact will be negligible. 3140 * 3141 * This could be optimized in the future to allocate a big enough iov array to account for all of the iovs 3142 * for all of the smaller I/Os, pre-build all of the iov arrays for the smaller I/Os, then issue them 3143 * in a batch. That would also require creating an intermediate spdk_bs_cpl that would get called 3144 * when the batch was completed, to allow for freeing the memory for the iov arrays. 3145 */ 3146 if (spdk_likely(length <= bs_num_io_units_to_cluster_boundary(blob, offset))) { 3147 uint64_t lba_count; 3148 uint64_t lba; 3149 bool is_allocated; 3150 3151 cpl.type = SPDK_BS_CPL_TYPE_BLOB_BASIC; 3152 cpl.u.blob_basic.cb_fn = cb_fn; 3153 cpl.u.blob_basic.cb_arg = cb_arg; 3154 3155 if (blob->frozen_refcnt) { 3156 /* This blob I/O is frozen */ 3157 enum spdk_blob_op_type op_type; 3158 spdk_bs_user_op_t *op; 3159 struct spdk_bs_channel *bs_channel = spdk_io_channel_get_ctx(_channel); 3160 3161 op_type = read ? SPDK_BLOB_READV : SPDK_BLOB_WRITEV; 3162 op = bs_user_op_alloc(_channel, &cpl, op_type, blob, iov, iovcnt, offset, length); 3163 if (!op) { 3164 cb_fn(cb_arg, -ENOMEM); 3165 return; 3166 } 3167 3168 TAILQ_INSERT_TAIL(&bs_channel->queued_io, op, link); 3169 3170 return; 3171 } 3172 3173 is_allocated = blob_calculate_lba_and_lba_count(blob, offset, length, &lba, &lba_count); 3174 3175 if (read) { 3176 spdk_bs_sequence_t *seq; 3177 3178 seq = bs_sequence_start_blob(_channel, &cpl, blob); 3179 if (!seq) { 3180 cb_fn(cb_arg, -ENOMEM); 3181 return; 3182 } 3183 3184 seq->ext_io_opts = ext_io_opts; 3185 3186 if (is_allocated) { 3187 bs_sequence_readv_dev(seq, iov, iovcnt, lba, lba_count, rw_iov_done, NULL); 3188 } else { 3189 bs_sequence_readv_bs_dev(seq, blob->back_bs_dev, iov, iovcnt, lba, lba_count, 3190 rw_iov_done, NULL); 3191 } 3192 } else { 3193 if (is_allocated) { 3194 spdk_bs_sequence_t *seq; 3195 3196 seq = bs_sequence_start_blob(_channel, &cpl, blob); 3197 if (!seq) { 3198 cb_fn(cb_arg, -ENOMEM); 3199 return; 3200 } 3201 3202 seq->ext_io_opts = ext_io_opts; 3203 3204 bs_sequence_writev_dev(seq, iov, iovcnt, lba, lba_count, rw_iov_done, NULL); 3205 } else { 3206 /* Queue this operation and allocate the cluster */ 3207 spdk_bs_user_op_t *op; 3208 3209 op = bs_user_op_alloc(_channel, &cpl, SPDK_BLOB_WRITEV, blob, iov, iovcnt, offset, 3210 length); 3211 if (!op) { 3212 cb_fn(cb_arg, -ENOMEM); 3213 return; 3214 } 3215 3216 op->ext_io_opts = ext_io_opts; 3217 3218 bs_allocate_and_copy_cluster(blob, _channel, offset, op); 3219 } 3220 } 3221 } else { 3222 struct rw_iov_ctx *ctx; 3223 3224 ctx = calloc(1, sizeof(struct rw_iov_ctx) + iovcnt * sizeof(struct iovec)); 3225 if (ctx == NULL) { 3226 cb_fn(cb_arg, -ENOMEM); 3227 return; 3228 } 3229 3230 ctx->blob = blob; 3231 ctx->channel = _channel; 3232 ctx->cb_fn = cb_fn; 3233 ctx->cb_arg = cb_arg; 3234 ctx->read = read; 3235 ctx->orig_iov = iov; 3236 ctx->iovcnt = iovcnt; 3237 ctx->io_unit_offset = offset; 3238 ctx->io_units_remaining = length; 3239 ctx->io_units_done = 0; 3240 ctx->ext_io_opts = ext_io_opts; 3241 3242 rw_iov_split_next(ctx, 0); 3243 } 3244 } 3245 3246 static struct spdk_blob * 3247 blob_lookup(struct spdk_blob_store *bs, spdk_blob_id blobid) 3248 { 3249 struct spdk_blob find; 3250 3251 if (spdk_bit_array_get(bs->open_blobids, blobid) == 0) { 3252 return NULL; 3253 } 3254 3255 find.id = blobid; 3256 return RB_FIND(spdk_blob_tree, &bs->open_blobs, &find); 3257 } 3258 3259 static void 3260 blob_get_snapshot_and_clone_entries(struct spdk_blob *blob, 3261 struct spdk_blob_list **snapshot_entry, struct spdk_blob_list **clone_entry) 3262 { 3263 assert(blob != NULL); 3264 *snapshot_entry = NULL; 3265 *clone_entry = NULL; 3266 3267 if (blob->parent_id == SPDK_BLOBID_INVALID) { 3268 return; 3269 } 3270 3271 TAILQ_FOREACH(*snapshot_entry, &blob->bs->snapshots, link) { 3272 if ((*snapshot_entry)->id == blob->parent_id) { 3273 break; 3274 } 3275 } 3276 3277 if (*snapshot_entry != NULL) { 3278 TAILQ_FOREACH(*clone_entry, &(*snapshot_entry)->clones, link) { 3279 if ((*clone_entry)->id == blob->id) { 3280 break; 3281 } 3282 } 3283 3284 assert(*clone_entry != NULL); 3285 } 3286 } 3287 3288 static int 3289 bs_channel_create(void *io_device, void *ctx_buf) 3290 { 3291 struct spdk_blob_store *bs = io_device; 3292 struct spdk_bs_channel *channel = ctx_buf; 3293 struct spdk_bs_dev *dev; 3294 uint32_t max_ops = bs->max_channel_ops; 3295 uint32_t i; 3296 3297 dev = bs->dev; 3298 3299 channel->req_mem = calloc(max_ops, sizeof(struct spdk_bs_request_set)); 3300 if (!channel->req_mem) { 3301 return -1; 3302 } 3303 3304 TAILQ_INIT(&channel->reqs); 3305 3306 for (i = 0; i < max_ops; i++) { 3307 TAILQ_INSERT_TAIL(&channel->reqs, &channel->req_mem[i], link); 3308 } 3309 3310 channel->bs = bs; 3311 channel->dev = dev; 3312 channel->dev_channel = dev->create_channel(dev); 3313 3314 if (!channel->dev_channel) { 3315 SPDK_ERRLOG("Failed to create device channel.\n"); 3316 free(channel->req_mem); 3317 return -1; 3318 } 3319 3320 channel->new_cluster_page = spdk_zmalloc(SPDK_BS_PAGE_SIZE, 0, NULL, SPDK_ENV_SOCKET_ID_ANY, 3321 SPDK_MALLOC_DMA); 3322 if (!channel->new_cluster_page) { 3323 SPDK_ERRLOG("Failed to allocate new cluster page\n"); 3324 free(channel->req_mem); 3325 channel->dev->destroy_channel(channel->dev, channel->dev_channel); 3326 return -1; 3327 } 3328 3329 TAILQ_INIT(&channel->need_cluster_alloc); 3330 TAILQ_INIT(&channel->queued_io); 3331 RB_INIT(&channel->esnap_channels); 3332 3333 return 0; 3334 } 3335 3336 static void 3337 bs_channel_destroy(void *io_device, void *ctx_buf) 3338 { 3339 struct spdk_bs_channel *channel = ctx_buf; 3340 spdk_bs_user_op_t *op; 3341 3342 while (!TAILQ_EMPTY(&channel->need_cluster_alloc)) { 3343 op = TAILQ_FIRST(&channel->need_cluster_alloc); 3344 TAILQ_REMOVE(&channel->need_cluster_alloc, op, link); 3345 bs_user_op_abort(op, -EIO); 3346 } 3347 3348 while (!TAILQ_EMPTY(&channel->queued_io)) { 3349 op = TAILQ_FIRST(&channel->queued_io); 3350 TAILQ_REMOVE(&channel->queued_io, op, link); 3351 bs_user_op_abort(op, -EIO); 3352 } 3353 3354 blob_esnap_destroy_bs_channel(channel); 3355 3356 free(channel->req_mem); 3357 spdk_free(channel->new_cluster_page); 3358 channel->dev->destroy_channel(channel->dev, channel->dev_channel); 3359 } 3360 3361 static void 3362 bs_dev_destroy(void *io_device) 3363 { 3364 struct spdk_blob_store *bs = io_device; 3365 struct spdk_blob *blob, *blob_tmp; 3366 3367 bs->dev->destroy(bs->dev); 3368 3369 RB_FOREACH_SAFE(blob, spdk_blob_tree, &bs->open_blobs, blob_tmp) { 3370 RB_REMOVE(spdk_blob_tree, &bs->open_blobs, blob); 3371 spdk_bit_array_clear(bs->open_blobids, blob->id); 3372 blob_free(blob); 3373 } 3374 3375 spdk_spin_destroy(&bs->used_lock); 3376 3377 spdk_bit_array_free(&bs->open_blobids); 3378 spdk_bit_array_free(&bs->used_blobids); 3379 spdk_bit_array_free(&bs->used_md_pages); 3380 spdk_bit_pool_free(&bs->used_clusters); 3381 /* 3382 * If this function is called for any reason except a successful unload, 3383 * the unload_cpl type will be NONE and this will be a nop. 3384 */ 3385 bs_call_cpl(&bs->unload_cpl, bs->unload_err); 3386 3387 free(bs); 3388 } 3389 3390 static int 3391 bs_blob_list_add(struct spdk_blob *blob) 3392 { 3393 spdk_blob_id snapshot_id; 3394 struct spdk_blob_list *snapshot_entry = NULL; 3395 struct spdk_blob_list *clone_entry = NULL; 3396 3397 assert(blob != NULL); 3398 3399 snapshot_id = blob->parent_id; 3400 if (snapshot_id == SPDK_BLOBID_INVALID || 3401 snapshot_id == SPDK_BLOBID_EXTERNAL_SNAPSHOT) { 3402 return 0; 3403 } 3404 3405 snapshot_entry = bs_get_snapshot_entry(blob->bs, snapshot_id); 3406 if (snapshot_entry == NULL) { 3407 /* Snapshot not found */ 3408 snapshot_entry = calloc(1, sizeof(struct spdk_blob_list)); 3409 if (snapshot_entry == NULL) { 3410 return -ENOMEM; 3411 } 3412 snapshot_entry->id = snapshot_id; 3413 TAILQ_INIT(&snapshot_entry->clones); 3414 TAILQ_INSERT_TAIL(&blob->bs->snapshots, snapshot_entry, link); 3415 } else { 3416 TAILQ_FOREACH(clone_entry, &snapshot_entry->clones, link) { 3417 if (clone_entry->id == blob->id) { 3418 break; 3419 } 3420 } 3421 } 3422 3423 if (clone_entry == NULL) { 3424 /* Clone not found */ 3425 clone_entry = calloc(1, sizeof(struct spdk_blob_list)); 3426 if (clone_entry == NULL) { 3427 return -ENOMEM; 3428 } 3429 clone_entry->id = blob->id; 3430 TAILQ_INIT(&clone_entry->clones); 3431 TAILQ_INSERT_TAIL(&snapshot_entry->clones, clone_entry, link); 3432 snapshot_entry->clone_count++; 3433 } 3434 3435 return 0; 3436 } 3437 3438 static void 3439 bs_blob_list_remove(struct spdk_blob *blob) 3440 { 3441 struct spdk_blob_list *snapshot_entry = NULL; 3442 struct spdk_blob_list *clone_entry = NULL; 3443 3444 blob_get_snapshot_and_clone_entries(blob, &snapshot_entry, &clone_entry); 3445 3446 if (snapshot_entry == NULL) { 3447 return; 3448 } 3449 3450 blob->parent_id = SPDK_BLOBID_INVALID; 3451 TAILQ_REMOVE(&snapshot_entry->clones, clone_entry, link); 3452 free(clone_entry); 3453 3454 snapshot_entry->clone_count--; 3455 } 3456 3457 static int 3458 bs_blob_list_free(struct spdk_blob_store *bs) 3459 { 3460 struct spdk_blob_list *snapshot_entry; 3461 struct spdk_blob_list *snapshot_entry_tmp; 3462 struct spdk_blob_list *clone_entry; 3463 struct spdk_blob_list *clone_entry_tmp; 3464 3465 TAILQ_FOREACH_SAFE(snapshot_entry, &bs->snapshots, link, snapshot_entry_tmp) { 3466 TAILQ_FOREACH_SAFE(clone_entry, &snapshot_entry->clones, link, clone_entry_tmp) { 3467 TAILQ_REMOVE(&snapshot_entry->clones, clone_entry, link); 3468 free(clone_entry); 3469 } 3470 TAILQ_REMOVE(&bs->snapshots, snapshot_entry, link); 3471 free(snapshot_entry); 3472 } 3473 3474 return 0; 3475 } 3476 3477 static void 3478 bs_free(struct spdk_blob_store *bs) 3479 { 3480 bs_blob_list_free(bs); 3481 3482 bs_unregister_md_thread(bs); 3483 spdk_io_device_unregister(bs, bs_dev_destroy); 3484 } 3485 3486 void 3487 spdk_bs_opts_init(struct spdk_bs_opts *opts, size_t opts_size) 3488 { 3489 3490 if (!opts) { 3491 SPDK_ERRLOG("opts should not be NULL\n"); 3492 return; 3493 } 3494 3495 if (!opts_size) { 3496 SPDK_ERRLOG("opts_size should not be zero value\n"); 3497 return; 3498 } 3499 3500 memset(opts, 0, opts_size); 3501 opts->opts_size = opts_size; 3502 3503 #define FIELD_OK(field) \ 3504 offsetof(struct spdk_bs_opts, field) + sizeof(opts->field) <= opts_size 3505 3506 #define SET_FIELD(field, value) \ 3507 if (FIELD_OK(field)) { \ 3508 opts->field = value; \ 3509 } \ 3510 3511 SET_FIELD(cluster_sz, SPDK_BLOB_OPTS_CLUSTER_SZ); 3512 SET_FIELD(num_md_pages, SPDK_BLOB_OPTS_NUM_MD_PAGES); 3513 SET_FIELD(max_md_ops, SPDK_BLOB_OPTS_NUM_MD_PAGES); 3514 SET_FIELD(max_channel_ops, SPDK_BLOB_OPTS_DEFAULT_CHANNEL_OPS); 3515 SET_FIELD(clear_method, BS_CLEAR_WITH_UNMAP); 3516 3517 if (FIELD_OK(bstype)) { 3518 memset(&opts->bstype, 0, sizeof(opts->bstype)); 3519 } 3520 3521 SET_FIELD(iter_cb_fn, NULL); 3522 SET_FIELD(iter_cb_arg, NULL); 3523 SET_FIELD(force_recover, false); 3524 SET_FIELD(esnap_bs_dev_create, NULL); 3525 SET_FIELD(esnap_ctx, NULL); 3526 3527 #undef FIELD_OK 3528 #undef SET_FIELD 3529 } 3530 3531 static int 3532 bs_opts_verify(struct spdk_bs_opts *opts) 3533 { 3534 if (opts->cluster_sz == 0 || opts->num_md_pages == 0 || opts->max_md_ops == 0 || 3535 opts->max_channel_ops == 0) { 3536 SPDK_ERRLOG("Blobstore options cannot be set to 0\n"); 3537 return -1; 3538 } 3539 3540 return 0; 3541 } 3542 3543 /* START spdk_bs_load */ 3544 3545 /* spdk_bs_load_ctx is used for init, load, unload and dump code paths. */ 3546 3547 struct spdk_bs_load_ctx { 3548 struct spdk_blob_store *bs; 3549 struct spdk_bs_super_block *super; 3550 3551 struct spdk_bs_md_mask *mask; 3552 bool in_page_chain; 3553 uint32_t page_index; 3554 uint32_t cur_page; 3555 struct spdk_blob_md_page *page; 3556 3557 uint64_t num_extent_pages; 3558 uint32_t *extent_page_num; 3559 struct spdk_blob_md_page *extent_pages; 3560 struct spdk_bit_array *used_clusters; 3561 3562 spdk_bs_sequence_t *seq; 3563 spdk_blob_op_with_handle_complete iter_cb_fn; 3564 void *iter_cb_arg; 3565 struct spdk_blob *blob; 3566 spdk_blob_id blobid; 3567 3568 bool force_recover; 3569 3570 /* These fields are used in the spdk_bs_dump path. */ 3571 bool dumping; 3572 FILE *fp; 3573 spdk_bs_dump_print_xattr print_xattr_fn; 3574 char xattr_name[4096]; 3575 }; 3576 3577 static int 3578 bs_alloc(struct spdk_bs_dev *dev, struct spdk_bs_opts *opts, struct spdk_blob_store **_bs, 3579 struct spdk_bs_load_ctx **_ctx) 3580 { 3581 struct spdk_blob_store *bs; 3582 struct spdk_bs_load_ctx *ctx; 3583 uint64_t dev_size; 3584 int rc; 3585 3586 dev_size = dev->blocklen * dev->blockcnt; 3587 if (dev_size < opts->cluster_sz) { 3588 /* Device size cannot be smaller than cluster size of blobstore */ 3589 SPDK_INFOLOG(blob, "Device size %" PRIu64 " is smaller than cluster size %" PRIu32 "\n", 3590 dev_size, opts->cluster_sz); 3591 return -ENOSPC; 3592 } 3593 if (opts->cluster_sz < SPDK_BS_PAGE_SIZE) { 3594 /* Cluster size cannot be smaller than page size */ 3595 SPDK_ERRLOG("Cluster size %" PRIu32 " is smaller than page size %d\n", 3596 opts->cluster_sz, SPDK_BS_PAGE_SIZE); 3597 return -EINVAL; 3598 } 3599 bs = calloc(1, sizeof(struct spdk_blob_store)); 3600 if (!bs) { 3601 return -ENOMEM; 3602 } 3603 3604 ctx = calloc(1, sizeof(struct spdk_bs_load_ctx)); 3605 if (!ctx) { 3606 free(bs); 3607 return -ENOMEM; 3608 } 3609 3610 ctx->bs = bs; 3611 ctx->iter_cb_fn = opts->iter_cb_fn; 3612 ctx->iter_cb_arg = opts->iter_cb_arg; 3613 ctx->force_recover = opts->force_recover; 3614 3615 ctx->super = spdk_zmalloc(sizeof(*ctx->super), 0x1000, NULL, 3616 SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA); 3617 if (!ctx->super) { 3618 free(ctx); 3619 free(bs); 3620 return -ENOMEM; 3621 } 3622 3623 RB_INIT(&bs->open_blobs); 3624 TAILQ_INIT(&bs->snapshots); 3625 bs->dev = dev; 3626 bs->md_thread = spdk_get_thread(); 3627 assert(bs->md_thread != NULL); 3628 3629 /* 3630 * Do not use bs_lba_to_cluster() here since blockcnt may not be an 3631 * even multiple of the cluster size. 3632 */ 3633 bs->cluster_sz = opts->cluster_sz; 3634 bs->total_clusters = dev->blockcnt / (bs->cluster_sz / dev->blocklen); 3635 ctx->used_clusters = spdk_bit_array_create(bs->total_clusters); 3636 if (!ctx->used_clusters) { 3637 spdk_free(ctx->super); 3638 free(ctx); 3639 free(bs); 3640 return -ENOMEM; 3641 } 3642 3643 bs->pages_per_cluster = bs->cluster_sz / SPDK_BS_PAGE_SIZE; 3644 if (spdk_u32_is_pow2(bs->pages_per_cluster)) { 3645 bs->pages_per_cluster_shift = spdk_u32log2(bs->pages_per_cluster); 3646 } 3647 bs->num_free_clusters = bs->total_clusters; 3648 bs->io_unit_size = dev->blocklen; 3649 3650 bs->max_channel_ops = opts->max_channel_ops; 3651 bs->super_blob = SPDK_BLOBID_INVALID; 3652 memcpy(&bs->bstype, &opts->bstype, sizeof(opts->bstype)); 3653 bs->esnap_bs_dev_create = opts->esnap_bs_dev_create; 3654 bs->esnap_ctx = opts->esnap_ctx; 3655 3656 /* The metadata is assumed to be at least 1 page */ 3657 bs->used_md_pages = spdk_bit_array_create(1); 3658 bs->used_blobids = spdk_bit_array_create(0); 3659 bs->open_blobids = spdk_bit_array_create(0); 3660 3661 spdk_spin_init(&bs->used_lock); 3662 3663 spdk_io_device_register(bs, bs_channel_create, bs_channel_destroy, 3664 sizeof(struct spdk_bs_channel), "blobstore"); 3665 rc = bs_register_md_thread(bs); 3666 if (rc == -1) { 3667 spdk_io_device_unregister(bs, NULL); 3668 spdk_spin_destroy(&bs->used_lock); 3669 spdk_bit_array_free(&bs->open_blobids); 3670 spdk_bit_array_free(&bs->used_blobids); 3671 spdk_bit_array_free(&bs->used_md_pages); 3672 spdk_bit_array_free(&ctx->used_clusters); 3673 spdk_free(ctx->super); 3674 free(ctx); 3675 free(bs); 3676 /* FIXME: this is a lie but don't know how to get a proper error code here */ 3677 return -ENOMEM; 3678 } 3679 3680 *_ctx = ctx; 3681 *_bs = bs; 3682 return 0; 3683 } 3684 3685 static void 3686 bs_load_ctx_fail(struct spdk_bs_load_ctx *ctx, int bserrno) 3687 { 3688 assert(bserrno != 0); 3689 3690 spdk_free(ctx->super); 3691 bs_sequence_finish(ctx->seq, bserrno); 3692 bs_free(ctx->bs); 3693 spdk_bit_array_free(&ctx->used_clusters); 3694 free(ctx); 3695 } 3696 3697 static void 3698 bs_write_super(spdk_bs_sequence_t *seq, struct spdk_blob_store *bs, 3699 struct spdk_bs_super_block *super, spdk_bs_sequence_cpl cb_fn, void *cb_arg) 3700 { 3701 /* Update the values in the super block */ 3702 super->super_blob = bs->super_blob; 3703 memcpy(&super->bstype, &bs->bstype, sizeof(bs->bstype)); 3704 super->crc = blob_md_page_calc_crc(super); 3705 bs_sequence_write_dev(seq, super, bs_page_to_lba(bs, 0), 3706 bs_byte_to_lba(bs, sizeof(*super)), 3707 cb_fn, cb_arg); 3708 } 3709 3710 static void 3711 bs_write_used_clusters(spdk_bs_sequence_t *seq, void *arg, spdk_bs_sequence_cpl cb_fn) 3712 { 3713 struct spdk_bs_load_ctx *ctx = arg; 3714 uint64_t mask_size, lba, lba_count; 3715 3716 /* Write out the used clusters mask */ 3717 mask_size = ctx->super->used_cluster_mask_len * SPDK_BS_PAGE_SIZE; 3718 ctx->mask = spdk_zmalloc(mask_size, 0x1000, NULL, 3719 SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA); 3720 if (!ctx->mask) { 3721 bs_load_ctx_fail(ctx, -ENOMEM); 3722 return; 3723 } 3724 3725 ctx->mask->type = SPDK_MD_MASK_TYPE_USED_CLUSTERS; 3726 ctx->mask->length = ctx->bs->total_clusters; 3727 /* We could get here through the normal unload path, or through dirty 3728 * shutdown recovery. For the normal unload path, we use the mask from 3729 * the bit pool. For dirty shutdown recovery, we don't have a bit pool yet - 3730 * only the bit array from the load ctx. 3731 */ 3732 if (ctx->bs->used_clusters) { 3733 assert(ctx->mask->length == spdk_bit_pool_capacity(ctx->bs->used_clusters)); 3734 spdk_bit_pool_store_mask(ctx->bs->used_clusters, ctx->mask->mask); 3735 } else { 3736 assert(ctx->mask->length == spdk_bit_array_capacity(ctx->used_clusters)); 3737 spdk_bit_array_store_mask(ctx->used_clusters, ctx->mask->mask); 3738 } 3739 lba = bs_page_to_lba(ctx->bs, ctx->super->used_cluster_mask_start); 3740 lba_count = bs_page_to_lba(ctx->bs, ctx->super->used_cluster_mask_len); 3741 bs_sequence_write_dev(seq, ctx->mask, lba, lba_count, cb_fn, arg); 3742 } 3743 3744 static void 3745 bs_write_used_md(spdk_bs_sequence_t *seq, void *arg, spdk_bs_sequence_cpl cb_fn) 3746 { 3747 struct spdk_bs_load_ctx *ctx = arg; 3748 uint64_t mask_size, lba, lba_count; 3749 3750 mask_size = ctx->super->used_page_mask_len * SPDK_BS_PAGE_SIZE; 3751 ctx->mask = spdk_zmalloc(mask_size, 0x1000, NULL, 3752 SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA); 3753 if (!ctx->mask) { 3754 bs_load_ctx_fail(ctx, -ENOMEM); 3755 return; 3756 } 3757 3758 ctx->mask->type = SPDK_MD_MASK_TYPE_USED_PAGES; 3759 ctx->mask->length = ctx->super->md_len; 3760 assert(ctx->mask->length == spdk_bit_array_capacity(ctx->bs->used_md_pages)); 3761 3762 spdk_bit_array_store_mask(ctx->bs->used_md_pages, ctx->mask->mask); 3763 lba = bs_page_to_lba(ctx->bs, ctx->super->used_page_mask_start); 3764 lba_count = bs_page_to_lba(ctx->bs, ctx->super->used_page_mask_len); 3765 bs_sequence_write_dev(seq, ctx->mask, lba, lba_count, cb_fn, arg); 3766 } 3767 3768 static void 3769 bs_write_used_blobids(spdk_bs_sequence_t *seq, void *arg, spdk_bs_sequence_cpl cb_fn) 3770 { 3771 struct spdk_bs_load_ctx *ctx = arg; 3772 uint64_t mask_size, lba, lba_count; 3773 3774 if (ctx->super->used_blobid_mask_len == 0) { 3775 /* 3776 * This is a pre-v3 on-disk format where the blobid mask does not get 3777 * written to disk. 3778 */ 3779 cb_fn(seq, arg, 0); 3780 return; 3781 } 3782 3783 mask_size = ctx->super->used_blobid_mask_len * SPDK_BS_PAGE_SIZE; 3784 ctx->mask = spdk_zmalloc(mask_size, 0x1000, NULL, SPDK_ENV_SOCKET_ID_ANY, 3785 SPDK_MALLOC_DMA); 3786 if (!ctx->mask) { 3787 bs_load_ctx_fail(ctx, -ENOMEM); 3788 return; 3789 } 3790 3791 ctx->mask->type = SPDK_MD_MASK_TYPE_USED_BLOBIDS; 3792 ctx->mask->length = ctx->super->md_len; 3793 assert(ctx->mask->length == spdk_bit_array_capacity(ctx->bs->used_blobids)); 3794 3795 spdk_bit_array_store_mask(ctx->bs->used_blobids, ctx->mask->mask); 3796 lba = bs_page_to_lba(ctx->bs, ctx->super->used_blobid_mask_start); 3797 lba_count = bs_page_to_lba(ctx->bs, ctx->super->used_blobid_mask_len); 3798 bs_sequence_write_dev(seq, ctx->mask, lba, lba_count, cb_fn, arg); 3799 } 3800 3801 static void 3802 blob_set_thin_provision(struct spdk_blob *blob) 3803 { 3804 blob_verify_md_op(blob); 3805 blob->invalid_flags |= SPDK_BLOB_THIN_PROV; 3806 blob->state = SPDK_BLOB_STATE_DIRTY; 3807 } 3808 3809 static void 3810 blob_set_clear_method(struct spdk_blob *blob, enum blob_clear_method clear_method) 3811 { 3812 blob_verify_md_op(blob); 3813 blob->clear_method = clear_method; 3814 blob->md_ro_flags |= (clear_method << SPDK_BLOB_CLEAR_METHOD_SHIFT); 3815 blob->state = SPDK_BLOB_STATE_DIRTY; 3816 } 3817 3818 static void bs_load_iter(void *arg, struct spdk_blob *blob, int bserrno); 3819 3820 static void 3821 bs_delete_corrupted_blob_cpl(void *cb_arg, int bserrno) 3822 { 3823 struct spdk_bs_load_ctx *ctx = cb_arg; 3824 spdk_blob_id id; 3825 int64_t page_num; 3826 3827 /* Iterate to next blob (we can't use spdk_bs_iter_next function as our 3828 * last blob has been removed */ 3829 page_num = bs_blobid_to_page(ctx->blobid); 3830 page_num++; 3831 page_num = spdk_bit_array_find_first_set(ctx->bs->used_blobids, page_num); 3832 if (page_num >= spdk_bit_array_capacity(ctx->bs->used_blobids)) { 3833 bs_load_iter(ctx, NULL, -ENOENT); 3834 return; 3835 } 3836 3837 id = bs_page_to_blobid(page_num); 3838 3839 spdk_bs_open_blob(ctx->bs, id, bs_load_iter, ctx); 3840 } 3841 3842 static void 3843 bs_delete_corrupted_close_cb(void *cb_arg, int bserrno) 3844 { 3845 struct spdk_bs_load_ctx *ctx = cb_arg; 3846 3847 if (bserrno != 0) { 3848 SPDK_ERRLOG("Failed to close corrupted blob\n"); 3849 spdk_bs_iter_next(ctx->bs, ctx->blob, bs_load_iter, ctx); 3850 return; 3851 } 3852 3853 spdk_bs_delete_blob(ctx->bs, ctx->blobid, bs_delete_corrupted_blob_cpl, ctx); 3854 } 3855 3856 static void 3857 bs_delete_corrupted_blob(void *cb_arg, int bserrno) 3858 { 3859 struct spdk_bs_load_ctx *ctx = cb_arg; 3860 uint64_t i; 3861 3862 if (bserrno != 0) { 3863 SPDK_ERRLOG("Failed to close clone of a corrupted blob\n"); 3864 spdk_bs_iter_next(ctx->bs, ctx->blob, bs_load_iter, ctx); 3865 return; 3866 } 3867 3868 /* Snapshot and clone have the same copy of cluster map and extent pages 3869 * at this point. Let's clear both for snapshot now, 3870 * so that it won't be cleared for clone later when we remove snapshot. 3871 * Also set thin provision to pass data corruption check */ 3872 for (i = 0; i < ctx->blob->active.num_clusters; i++) { 3873 ctx->blob->active.clusters[i] = 0; 3874 } 3875 for (i = 0; i < ctx->blob->active.num_extent_pages; i++) { 3876 ctx->blob->active.extent_pages[i] = 0; 3877 } 3878 3879 ctx->blob->md_ro = false; 3880 3881 blob_set_thin_provision(ctx->blob); 3882 3883 ctx->blobid = ctx->blob->id; 3884 3885 spdk_blob_close(ctx->blob, bs_delete_corrupted_close_cb, ctx); 3886 } 3887 3888 static void 3889 bs_update_corrupted_blob(void *cb_arg, int bserrno) 3890 { 3891 struct spdk_bs_load_ctx *ctx = cb_arg; 3892 3893 if (bserrno != 0) { 3894 SPDK_ERRLOG("Failed to close clone of a corrupted blob\n"); 3895 spdk_bs_iter_next(ctx->bs, ctx->blob, bs_load_iter, ctx); 3896 return; 3897 } 3898 3899 ctx->blob->md_ro = false; 3900 blob_remove_xattr(ctx->blob, SNAPSHOT_PENDING_REMOVAL, true); 3901 blob_remove_xattr(ctx->blob, SNAPSHOT_IN_PROGRESS, true); 3902 spdk_blob_set_read_only(ctx->blob); 3903 3904 if (ctx->iter_cb_fn) { 3905 ctx->iter_cb_fn(ctx->iter_cb_arg, ctx->blob, 0); 3906 } 3907 bs_blob_list_add(ctx->blob); 3908 3909 spdk_bs_iter_next(ctx->bs, ctx->blob, bs_load_iter, ctx); 3910 } 3911 3912 static void 3913 bs_examine_clone(void *cb_arg, struct spdk_blob *blob, int bserrno) 3914 { 3915 struct spdk_bs_load_ctx *ctx = cb_arg; 3916 3917 if (bserrno != 0) { 3918 SPDK_ERRLOG("Failed to open clone of a corrupted blob\n"); 3919 spdk_bs_iter_next(ctx->bs, ctx->blob, bs_load_iter, ctx); 3920 return; 3921 } 3922 3923 if (blob->parent_id == ctx->blob->id) { 3924 /* Power failure occurred before updating clone (snapshot delete case) 3925 * or after updating clone (creating snapshot case) - keep snapshot */ 3926 spdk_blob_close(blob, bs_update_corrupted_blob, ctx); 3927 } else { 3928 /* Power failure occurred after updating clone (snapshot delete case) 3929 * or before updating clone (creating snapshot case) - remove snapshot */ 3930 spdk_blob_close(blob, bs_delete_corrupted_blob, ctx); 3931 } 3932 } 3933 3934 static void 3935 bs_load_iter(void *arg, struct spdk_blob *blob, int bserrno) 3936 { 3937 struct spdk_bs_load_ctx *ctx = arg; 3938 const void *value; 3939 size_t len; 3940 int rc = 0; 3941 3942 if (bserrno == 0) { 3943 /* Examine blob if it is corrupted after power failure. Fix 3944 * the ones that can be fixed and remove any other corrupted 3945 * ones. If it is not corrupted just process it */ 3946 rc = blob_get_xattr_value(blob, SNAPSHOT_PENDING_REMOVAL, &value, &len, true); 3947 if (rc != 0) { 3948 rc = blob_get_xattr_value(blob, SNAPSHOT_IN_PROGRESS, &value, &len, true); 3949 if (rc != 0) { 3950 /* Not corrupted - process it and continue with iterating through blobs */ 3951 if (ctx->iter_cb_fn) { 3952 ctx->iter_cb_fn(ctx->iter_cb_arg, blob, 0); 3953 } 3954 bs_blob_list_add(blob); 3955 spdk_bs_iter_next(ctx->bs, blob, bs_load_iter, ctx); 3956 return; 3957 } 3958 3959 } 3960 3961 assert(len == sizeof(spdk_blob_id)); 3962 3963 ctx->blob = blob; 3964 3965 /* Open clone to check if we are able to fix this blob or should we remove it */ 3966 spdk_bs_open_blob(ctx->bs, *(spdk_blob_id *)value, bs_examine_clone, ctx); 3967 return; 3968 } else if (bserrno == -ENOENT) { 3969 bserrno = 0; 3970 } else { 3971 /* 3972 * This case needs to be looked at further. Same problem 3973 * exists with applications that rely on explicit blob 3974 * iteration. We should just skip the blob that failed 3975 * to load and continue on to the next one. 3976 */ 3977 SPDK_ERRLOG("Error in iterating blobs\n"); 3978 } 3979 3980 ctx->iter_cb_fn = NULL; 3981 3982 spdk_free(ctx->super); 3983 spdk_free(ctx->mask); 3984 bs_sequence_finish(ctx->seq, bserrno); 3985 free(ctx); 3986 } 3987 3988 static void bs_dump_read_md_page(spdk_bs_sequence_t *seq, void *cb_arg); 3989 3990 static void 3991 bs_load_complete(struct spdk_bs_load_ctx *ctx) 3992 { 3993 ctx->bs->used_clusters = spdk_bit_pool_create_from_array(ctx->used_clusters); 3994 if (ctx->dumping) { 3995 bs_dump_read_md_page(ctx->seq, ctx); 3996 return; 3997 } 3998 spdk_bs_iter_first(ctx->bs, bs_load_iter, ctx); 3999 } 4000 4001 static void 4002 bs_load_used_blobids_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 4003 { 4004 struct spdk_bs_load_ctx *ctx = cb_arg; 4005 int rc; 4006 4007 /* The type must be correct */ 4008 assert(ctx->mask->type == SPDK_MD_MASK_TYPE_USED_BLOBIDS); 4009 4010 /* The length of the mask (in bits) must not be greater than 4011 * the length of the buffer (converted to bits) */ 4012 assert(ctx->mask->length <= (ctx->super->used_blobid_mask_len * SPDK_BS_PAGE_SIZE * 8)); 4013 4014 /* The length of the mask must be exactly equal to the size 4015 * (in pages) of the metadata region */ 4016 assert(ctx->mask->length == ctx->super->md_len); 4017 4018 rc = spdk_bit_array_resize(&ctx->bs->used_blobids, ctx->mask->length); 4019 if (rc < 0) { 4020 spdk_free(ctx->mask); 4021 bs_load_ctx_fail(ctx, rc); 4022 return; 4023 } 4024 4025 spdk_bit_array_load_mask(ctx->bs->used_blobids, ctx->mask->mask); 4026 bs_load_complete(ctx); 4027 } 4028 4029 static void 4030 bs_load_used_clusters_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 4031 { 4032 struct spdk_bs_load_ctx *ctx = cb_arg; 4033 uint64_t lba, lba_count, mask_size; 4034 int rc; 4035 4036 if (bserrno != 0) { 4037 bs_load_ctx_fail(ctx, bserrno); 4038 return; 4039 } 4040 4041 /* The type must be correct */ 4042 assert(ctx->mask->type == SPDK_MD_MASK_TYPE_USED_CLUSTERS); 4043 /* The length of the mask (in bits) must not be greater than the length of the buffer (converted to bits) */ 4044 assert(ctx->mask->length <= (ctx->super->used_cluster_mask_len * sizeof( 4045 struct spdk_blob_md_page) * 8)); 4046 /* 4047 * The length of the mask must be equal to or larger than the total number of clusters. It may be 4048 * larger than the total number of clusters due to a failure spdk_bs_grow. 4049 */ 4050 assert(ctx->mask->length >= ctx->bs->total_clusters); 4051 if (ctx->mask->length > ctx->bs->total_clusters) { 4052 SPDK_WARNLOG("Shrink the used_custers mask length to total_clusters"); 4053 ctx->mask->length = ctx->bs->total_clusters; 4054 } 4055 4056 rc = spdk_bit_array_resize(&ctx->used_clusters, ctx->mask->length); 4057 if (rc < 0) { 4058 spdk_free(ctx->mask); 4059 bs_load_ctx_fail(ctx, rc); 4060 return; 4061 } 4062 4063 spdk_bit_array_load_mask(ctx->used_clusters, ctx->mask->mask); 4064 ctx->bs->num_free_clusters = spdk_bit_array_count_clear(ctx->used_clusters); 4065 assert(ctx->bs->num_free_clusters <= ctx->bs->total_clusters); 4066 4067 spdk_free(ctx->mask); 4068 4069 /* Read the used blobids mask */ 4070 mask_size = ctx->super->used_blobid_mask_len * SPDK_BS_PAGE_SIZE; 4071 ctx->mask = spdk_zmalloc(mask_size, 0x1000, NULL, SPDK_ENV_SOCKET_ID_ANY, 4072 SPDK_MALLOC_DMA); 4073 if (!ctx->mask) { 4074 bs_load_ctx_fail(ctx, -ENOMEM); 4075 return; 4076 } 4077 lba = bs_page_to_lba(ctx->bs, ctx->super->used_blobid_mask_start); 4078 lba_count = bs_page_to_lba(ctx->bs, ctx->super->used_blobid_mask_len); 4079 bs_sequence_read_dev(seq, ctx->mask, lba, lba_count, 4080 bs_load_used_blobids_cpl, ctx); 4081 } 4082 4083 static void 4084 bs_load_used_pages_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 4085 { 4086 struct spdk_bs_load_ctx *ctx = cb_arg; 4087 uint64_t lba, lba_count, mask_size; 4088 int rc; 4089 4090 if (bserrno != 0) { 4091 bs_load_ctx_fail(ctx, bserrno); 4092 return; 4093 } 4094 4095 /* The type must be correct */ 4096 assert(ctx->mask->type == SPDK_MD_MASK_TYPE_USED_PAGES); 4097 /* The length of the mask (in bits) must not be greater than the length of the buffer (converted to bits) */ 4098 assert(ctx->mask->length <= (ctx->super->used_page_mask_len * SPDK_BS_PAGE_SIZE * 4099 8)); 4100 /* The length of the mask must be exactly equal to the size (in pages) of the metadata region */ 4101 if (ctx->mask->length != ctx->super->md_len) { 4102 SPDK_ERRLOG("mismatched md_len in used_pages mask: " 4103 "mask->length=%" PRIu32 " super->md_len=%" PRIu32 "\n", 4104 ctx->mask->length, ctx->super->md_len); 4105 assert(false); 4106 } 4107 4108 rc = spdk_bit_array_resize(&ctx->bs->used_md_pages, ctx->mask->length); 4109 if (rc < 0) { 4110 spdk_free(ctx->mask); 4111 bs_load_ctx_fail(ctx, rc); 4112 return; 4113 } 4114 4115 spdk_bit_array_load_mask(ctx->bs->used_md_pages, ctx->mask->mask); 4116 spdk_free(ctx->mask); 4117 4118 /* Read the used clusters mask */ 4119 mask_size = ctx->super->used_cluster_mask_len * SPDK_BS_PAGE_SIZE; 4120 ctx->mask = spdk_zmalloc(mask_size, 0x1000, NULL, SPDK_ENV_SOCKET_ID_ANY, 4121 SPDK_MALLOC_DMA); 4122 if (!ctx->mask) { 4123 bs_load_ctx_fail(ctx, -ENOMEM); 4124 return; 4125 } 4126 lba = bs_page_to_lba(ctx->bs, ctx->super->used_cluster_mask_start); 4127 lba_count = bs_page_to_lba(ctx->bs, ctx->super->used_cluster_mask_len); 4128 bs_sequence_read_dev(seq, ctx->mask, lba, lba_count, 4129 bs_load_used_clusters_cpl, ctx); 4130 } 4131 4132 static void 4133 bs_load_read_used_pages(struct spdk_bs_load_ctx *ctx) 4134 { 4135 uint64_t lba, lba_count, mask_size; 4136 4137 /* Read the used pages mask */ 4138 mask_size = ctx->super->used_page_mask_len * SPDK_BS_PAGE_SIZE; 4139 ctx->mask = spdk_zmalloc(mask_size, 0x1000, NULL, 4140 SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA); 4141 if (!ctx->mask) { 4142 bs_load_ctx_fail(ctx, -ENOMEM); 4143 return; 4144 } 4145 4146 lba = bs_page_to_lba(ctx->bs, ctx->super->used_page_mask_start); 4147 lba_count = bs_page_to_lba(ctx->bs, ctx->super->used_page_mask_len); 4148 bs_sequence_read_dev(ctx->seq, ctx->mask, lba, lba_count, 4149 bs_load_used_pages_cpl, ctx); 4150 } 4151 4152 static int 4153 bs_load_replay_md_parse_page(struct spdk_bs_load_ctx *ctx, struct spdk_blob_md_page *page) 4154 { 4155 struct spdk_blob_store *bs = ctx->bs; 4156 struct spdk_blob_md_descriptor *desc; 4157 size_t cur_desc = 0; 4158 4159 desc = (struct spdk_blob_md_descriptor *)page->descriptors; 4160 while (cur_desc < sizeof(page->descriptors)) { 4161 if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_PADDING) { 4162 if (desc->length == 0) { 4163 /* If padding and length are 0, this terminates the page */ 4164 break; 4165 } 4166 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_EXTENT_RLE) { 4167 struct spdk_blob_md_descriptor_extent_rle *desc_extent_rle; 4168 unsigned int i, j; 4169 unsigned int cluster_count = 0; 4170 uint32_t cluster_idx; 4171 4172 desc_extent_rle = (struct spdk_blob_md_descriptor_extent_rle *)desc; 4173 4174 for (i = 0; i < desc_extent_rle->length / sizeof(desc_extent_rle->extents[0]); i++) { 4175 for (j = 0; j < desc_extent_rle->extents[i].length; j++) { 4176 cluster_idx = desc_extent_rle->extents[i].cluster_idx; 4177 /* 4178 * cluster_idx = 0 means an unallocated cluster - don't mark that 4179 * in the used cluster map. 4180 */ 4181 if (cluster_idx != 0) { 4182 SPDK_NOTICELOG("Recover: cluster %" PRIu32 "\n", cluster_idx + j); 4183 spdk_bit_array_set(ctx->used_clusters, cluster_idx + j); 4184 if (bs->num_free_clusters == 0) { 4185 return -ENOSPC; 4186 } 4187 bs->num_free_clusters--; 4188 } 4189 cluster_count++; 4190 } 4191 } 4192 if (cluster_count == 0) { 4193 return -EINVAL; 4194 } 4195 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_EXTENT_PAGE) { 4196 struct spdk_blob_md_descriptor_extent_page *desc_extent; 4197 uint32_t i; 4198 uint32_t cluster_count = 0; 4199 uint32_t cluster_idx; 4200 size_t cluster_idx_length; 4201 4202 desc_extent = (struct spdk_blob_md_descriptor_extent_page *)desc; 4203 cluster_idx_length = desc_extent->length - sizeof(desc_extent->start_cluster_idx); 4204 4205 if (desc_extent->length <= sizeof(desc_extent->start_cluster_idx) || 4206 (cluster_idx_length % sizeof(desc_extent->cluster_idx[0]) != 0)) { 4207 return -EINVAL; 4208 } 4209 4210 for (i = 0; i < cluster_idx_length / sizeof(desc_extent->cluster_idx[0]); i++) { 4211 cluster_idx = desc_extent->cluster_idx[i]; 4212 /* 4213 * cluster_idx = 0 means an unallocated cluster - don't mark that 4214 * in the used cluster map. 4215 */ 4216 if (cluster_idx != 0) { 4217 if (cluster_idx < desc_extent->start_cluster_idx && 4218 cluster_idx >= desc_extent->start_cluster_idx + cluster_count) { 4219 return -EINVAL; 4220 } 4221 spdk_bit_array_set(ctx->used_clusters, cluster_idx); 4222 if (bs->num_free_clusters == 0) { 4223 return -ENOSPC; 4224 } 4225 bs->num_free_clusters--; 4226 } 4227 cluster_count++; 4228 } 4229 4230 if (cluster_count == 0) { 4231 return -EINVAL; 4232 } 4233 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_XATTR) { 4234 /* Skip this item */ 4235 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_XATTR_INTERNAL) { 4236 /* Skip this item */ 4237 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_FLAGS) { 4238 /* Skip this item */ 4239 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_EXTENT_TABLE) { 4240 struct spdk_blob_md_descriptor_extent_table *desc_extent_table; 4241 uint32_t num_extent_pages = ctx->num_extent_pages; 4242 uint32_t i; 4243 size_t extent_pages_length; 4244 void *tmp; 4245 4246 desc_extent_table = (struct spdk_blob_md_descriptor_extent_table *)desc; 4247 extent_pages_length = desc_extent_table->length - sizeof(desc_extent_table->num_clusters); 4248 4249 if (desc_extent_table->length == 0 || 4250 (extent_pages_length % sizeof(desc_extent_table->extent_page[0]) != 0)) { 4251 return -EINVAL; 4252 } 4253 4254 for (i = 0; i < extent_pages_length / sizeof(desc_extent_table->extent_page[0]); i++) { 4255 if (desc_extent_table->extent_page[i].page_idx != 0) { 4256 if (desc_extent_table->extent_page[i].num_pages != 1) { 4257 return -EINVAL; 4258 } 4259 num_extent_pages += 1; 4260 } 4261 } 4262 4263 if (num_extent_pages > 0) { 4264 tmp = realloc(ctx->extent_page_num, num_extent_pages * sizeof(uint32_t)); 4265 if (tmp == NULL) { 4266 return -ENOMEM; 4267 } 4268 ctx->extent_page_num = tmp; 4269 4270 /* Extent table entries contain md page numbers for extent pages. 4271 * Zeroes represent unallocated extent pages, those are run-length-encoded. 4272 */ 4273 for (i = 0; i < extent_pages_length / sizeof(desc_extent_table->extent_page[0]); i++) { 4274 if (desc_extent_table->extent_page[i].page_idx != 0) { 4275 ctx->extent_page_num[ctx->num_extent_pages] = desc_extent_table->extent_page[i].page_idx; 4276 ctx->num_extent_pages += 1; 4277 } 4278 } 4279 } 4280 } else { 4281 /* Error */ 4282 return -EINVAL; 4283 } 4284 /* Advance to the next descriptor */ 4285 cur_desc += sizeof(*desc) + desc->length; 4286 if (cur_desc + sizeof(*desc) > sizeof(page->descriptors)) { 4287 break; 4288 } 4289 desc = (struct spdk_blob_md_descriptor *)((uintptr_t)page->descriptors + cur_desc); 4290 } 4291 return 0; 4292 } 4293 4294 static bool 4295 bs_load_cur_extent_page_valid(struct spdk_blob_md_page *page) 4296 { 4297 uint32_t crc; 4298 struct spdk_blob_md_descriptor *desc = (struct spdk_blob_md_descriptor *)page->descriptors; 4299 size_t desc_len; 4300 4301 crc = blob_md_page_calc_crc(page); 4302 if (crc != page->crc) { 4303 return false; 4304 } 4305 4306 /* Extent page should always be of sequence num 0. */ 4307 if (page->sequence_num != 0) { 4308 return false; 4309 } 4310 4311 /* Descriptor type must be EXTENT_PAGE. */ 4312 if (desc->type != SPDK_MD_DESCRIPTOR_TYPE_EXTENT_PAGE) { 4313 return false; 4314 } 4315 4316 /* Descriptor length cannot exceed the page. */ 4317 desc_len = sizeof(*desc) + desc->length; 4318 if (desc_len > sizeof(page->descriptors)) { 4319 return false; 4320 } 4321 4322 /* It has to be the only descriptor in the page. */ 4323 if (desc_len + sizeof(*desc) <= sizeof(page->descriptors)) { 4324 desc = (struct spdk_blob_md_descriptor *)((uintptr_t)page->descriptors + desc_len); 4325 if (desc->length != 0) { 4326 return false; 4327 } 4328 } 4329 4330 return true; 4331 } 4332 4333 static bool 4334 bs_load_cur_md_page_valid(struct spdk_bs_load_ctx *ctx) 4335 { 4336 uint32_t crc; 4337 struct spdk_blob_md_page *page = ctx->page; 4338 4339 crc = blob_md_page_calc_crc(page); 4340 if (crc != page->crc) { 4341 return false; 4342 } 4343 4344 /* First page of a sequence should match the blobid. */ 4345 if (page->sequence_num == 0 && 4346 bs_page_to_blobid(ctx->cur_page) != page->id) { 4347 return false; 4348 } 4349 assert(bs_load_cur_extent_page_valid(page) == false); 4350 4351 return true; 4352 } 4353 4354 static void bs_load_replay_cur_md_page(struct spdk_bs_load_ctx *ctx); 4355 4356 static void 4357 bs_load_write_used_clusters_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 4358 { 4359 struct spdk_bs_load_ctx *ctx = cb_arg; 4360 4361 if (bserrno != 0) { 4362 bs_load_ctx_fail(ctx, bserrno); 4363 return; 4364 } 4365 4366 bs_load_complete(ctx); 4367 } 4368 4369 static void 4370 bs_load_write_used_blobids_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 4371 { 4372 struct spdk_bs_load_ctx *ctx = cb_arg; 4373 4374 spdk_free(ctx->mask); 4375 ctx->mask = NULL; 4376 4377 if (bserrno != 0) { 4378 bs_load_ctx_fail(ctx, bserrno); 4379 return; 4380 } 4381 4382 bs_write_used_clusters(seq, ctx, bs_load_write_used_clusters_cpl); 4383 } 4384 4385 static void 4386 bs_load_write_used_pages_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 4387 { 4388 struct spdk_bs_load_ctx *ctx = cb_arg; 4389 4390 spdk_free(ctx->mask); 4391 ctx->mask = NULL; 4392 4393 if (bserrno != 0) { 4394 bs_load_ctx_fail(ctx, bserrno); 4395 return; 4396 } 4397 4398 bs_write_used_blobids(seq, ctx, bs_load_write_used_blobids_cpl); 4399 } 4400 4401 static void 4402 bs_load_write_used_md(struct spdk_bs_load_ctx *ctx) 4403 { 4404 bs_write_used_md(ctx->seq, ctx, bs_load_write_used_pages_cpl); 4405 } 4406 4407 static void 4408 bs_load_replay_md_chain_cpl(struct spdk_bs_load_ctx *ctx) 4409 { 4410 uint64_t num_md_clusters; 4411 uint64_t i; 4412 4413 ctx->in_page_chain = false; 4414 4415 do { 4416 ctx->page_index++; 4417 } while (spdk_bit_array_get(ctx->bs->used_md_pages, ctx->page_index) == true); 4418 4419 if (ctx->page_index < ctx->super->md_len) { 4420 ctx->cur_page = ctx->page_index; 4421 bs_load_replay_cur_md_page(ctx); 4422 } else { 4423 /* Claim all of the clusters used by the metadata */ 4424 num_md_clusters = spdk_divide_round_up( 4425 ctx->super->md_start + ctx->super->md_len, ctx->bs->pages_per_cluster); 4426 for (i = 0; i < num_md_clusters; i++) { 4427 spdk_bit_array_set(ctx->used_clusters, i); 4428 } 4429 ctx->bs->num_free_clusters -= num_md_clusters; 4430 spdk_free(ctx->page); 4431 bs_load_write_used_md(ctx); 4432 } 4433 } 4434 4435 static void 4436 bs_load_replay_extent_page_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 4437 { 4438 struct spdk_bs_load_ctx *ctx = cb_arg; 4439 uint32_t page_num; 4440 uint64_t i; 4441 4442 if (bserrno != 0) { 4443 spdk_free(ctx->extent_pages); 4444 bs_load_ctx_fail(ctx, bserrno); 4445 return; 4446 } 4447 4448 for (i = 0; i < ctx->num_extent_pages; i++) { 4449 /* Extent pages are only read when present within in chain md. 4450 * Integrity of md is not right if that page was not a valid extent page. */ 4451 if (bs_load_cur_extent_page_valid(&ctx->extent_pages[i]) != true) { 4452 spdk_free(ctx->extent_pages); 4453 bs_load_ctx_fail(ctx, -EILSEQ); 4454 return; 4455 } 4456 4457 page_num = ctx->extent_page_num[i]; 4458 spdk_bit_array_set(ctx->bs->used_md_pages, page_num); 4459 if (bs_load_replay_md_parse_page(ctx, &ctx->extent_pages[i])) { 4460 spdk_free(ctx->extent_pages); 4461 bs_load_ctx_fail(ctx, -EILSEQ); 4462 return; 4463 } 4464 } 4465 4466 spdk_free(ctx->extent_pages); 4467 free(ctx->extent_page_num); 4468 ctx->extent_page_num = NULL; 4469 ctx->num_extent_pages = 0; 4470 4471 bs_load_replay_md_chain_cpl(ctx); 4472 } 4473 4474 static void 4475 bs_load_replay_extent_pages(struct spdk_bs_load_ctx *ctx) 4476 { 4477 spdk_bs_batch_t *batch; 4478 uint32_t page; 4479 uint64_t lba; 4480 uint64_t i; 4481 4482 ctx->extent_pages = spdk_zmalloc(SPDK_BS_PAGE_SIZE * ctx->num_extent_pages, 0, 4483 NULL, SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA); 4484 if (!ctx->extent_pages) { 4485 bs_load_ctx_fail(ctx, -ENOMEM); 4486 return; 4487 } 4488 4489 batch = bs_sequence_to_batch(ctx->seq, bs_load_replay_extent_page_cpl, ctx); 4490 4491 for (i = 0; i < ctx->num_extent_pages; i++) { 4492 page = ctx->extent_page_num[i]; 4493 assert(page < ctx->super->md_len); 4494 lba = bs_md_page_to_lba(ctx->bs, page); 4495 bs_batch_read_dev(batch, &ctx->extent_pages[i], lba, 4496 bs_byte_to_lba(ctx->bs, SPDK_BS_PAGE_SIZE)); 4497 } 4498 4499 bs_batch_close(batch); 4500 } 4501 4502 static void 4503 bs_load_replay_md_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 4504 { 4505 struct spdk_bs_load_ctx *ctx = cb_arg; 4506 uint32_t page_num; 4507 struct spdk_blob_md_page *page; 4508 4509 if (bserrno != 0) { 4510 bs_load_ctx_fail(ctx, bserrno); 4511 return; 4512 } 4513 4514 page_num = ctx->cur_page; 4515 page = ctx->page; 4516 if (bs_load_cur_md_page_valid(ctx) == true) { 4517 if (page->sequence_num == 0 || ctx->in_page_chain == true) { 4518 spdk_spin_lock(&ctx->bs->used_lock); 4519 bs_claim_md_page(ctx->bs, page_num); 4520 spdk_spin_unlock(&ctx->bs->used_lock); 4521 if (page->sequence_num == 0) { 4522 SPDK_NOTICELOG("Recover: blob 0x%" PRIx32 "\n", page_num); 4523 spdk_bit_array_set(ctx->bs->used_blobids, page_num); 4524 } 4525 if (bs_load_replay_md_parse_page(ctx, page)) { 4526 bs_load_ctx_fail(ctx, -EILSEQ); 4527 return; 4528 } 4529 if (page->next != SPDK_INVALID_MD_PAGE) { 4530 ctx->in_page_chain = true; 4531 ctx->cur_page = page->next; 4532 bs_load_replay_cur_md_page(ctx); 4533 return; 4534 } 4535 if (ctx->num_extent_pages != 0) { 4536 bs_load_replay_extent_pages(ctx); 4537 return; 4538 } 4539 } 4540 } 4541 bs_load_replay_md_chain_cpl(ctx); 4542 } 4543 4544 static void 4545 bs_load_replay_cur_md_page(struct spdk_bs_load_ctx *ctx) 4546 { 4547 uint64_t lba; 4548 4549 assert(ctx->cur_page < ctx->super->md_len); 4550 lba = bs_md_page_to_lba(ctx->bs, ctx->cur_page); 4551 bs_sequence_read_dev(ctx->seq, ctx->page, lba, 4552 bs_byte_to_lba(ctx->bs, SPDK_BS_PAGE_SIZE), 4553 bs_load_replay_md_cpl, ctx); 4554 } 4555 4556 static void 4557 bs_load_replay_md(struct spdk_bs_load_ctx *ctx) 4558 { 4559 ctx->page_index = 0; 4560 ctx->cur_page = 0; 4561 ctx->page = spdk_zmalloc(SPDK_BS_PAGE_SIZE, 0, 4562 NULL, SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA); 4563 if (!ctx->page) { 4564 bs_load_ctx_fail(ctx, -ENOMEM); 4565 return; 4566 } 4567 bs_load_replay_cur_md_page(ctx); 4568 } 4569 4570 static void 4571 bs_recover(struct spdk_bs_load_ctx *ctx) 4572 { 4573 int rc; 4574 4575 SPDK_NOTICELOG("Performing recovery on blobstore\n"); 4576 rc = spdk_bit_array_resize(&ctx->bs->used_md_pages, ctx->super->md_len); 4577 if (rc < 0) { 4578 bs_load_ctx_fail(ctx, -ENOMEM); 4579 return; 4580 } 4581 4582 rc = spdk_bit_array_resize(&ctx->bs->used_blobids, ctx->super->md_len); 4583 if (rc < 0) { 4584 bs_load_ctx_fail(ctx, -ENOMEM); 4585 return; 4586 } 4587 4588 rc = spdk_bit_array_resize(&ctx->used_clusters, ctx->bs->total_clusters); 4589 if (rc < 0) { 4590 bs_load_ctx_fail(ctx, -ENOMEM); 4591 return; 4592 } 4593 4594 rc = spdk_bit_array_resize(&ctx->bs->open_blobids, ctx->super->md_len); 4595 if (rc < 0) { 4596 bs_load_ctx_fail(ctx, -ENOMEM); 4597 return; 4598 } 4599 4600 ctx->bs->num_free_clusters = ctx->bs->total_clusters; 4601 bs_load_replay_md(ctx); 4602 } 4603 4604 static int 4605 bs_parse_super(struct spdk_bs_load_ctx *ctx) 4606 { 4607 int rc; 4608 4609 if (ctx->super->size == 0) { 4610 ctx->super->size = ctx->bs->dev->blockcnt * ctx->bs->dev->blocklen; 4611 } 4612 4613 if (ctx->super->io_unit_size == 0) { 4614 ctx->super->io_unit_size = SPDK_BS_PAGE_SIZE; 4615 } 4616 4617 ctx->bs->clean = 1; 4618 ctx->bs->cluster_sz = ctx->super->cluster_size; 4619 ctx->bs->total_clusters = ctx->super->size / ctx->super->cluster_size; 4620 ctx->bs->pages_per_cluster = ctx->bs->cluster_sz / SPDK_BS_PAGE_SIZE; 4621 if (spdk_u32_is_pow2(ctx->bs->pages_per_cluster)) { 4622 ctx->bs->pages_per_cluster_shift = spdk_u32log2(ctx->bs->pages_per_cluster); 4623 } 4624 ctx->bs->io_unit_size = ctx->super->io_unit_size; 4625 rc = spdk_bit_array_resize(&ctx->used_clusters, ctx->bs->total_clusters); 4626 if (rc < 0) { 4627 return -ENOMEM; 4628 } 4629 ctx->bs->md_start = ctx->super->md_start; 4630 ctx->bs->md_len = ctx->super->md_len; 4631 rc = spdk_bit_array_resize(&ctx->bs->open_blobids, ctx->bs->md_len); 4632 if (rc < 0) { 4633 return -ENOMEM; 4634 } 4635 4636 ctx->bs->total_data_clusters = ctx->bs->total_clusters - spdk_divide_round_up( 4637 ctx->bs->md_start + ctx->bs->md_len, ctx->bs->pages_per_cluster); 4638 ctx->bs->super_blob = ctx->super->super_blob; 4639 memcpy(&ctx->bs->bstype, &ctx->super->bstype, sizeof(ctx->super->bstype)); 4640 4641 return 0; 4642 } 4643 4644 static void 4645 bs_load_super_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 4646 { 4647 struct spdk_bs_load_ctx *ctx = cb_arg; 4648 uint32_t crc; 4649 int rc; 4650 static const char zeros[SPDK_BLOBSTORE_TYPE_LENGTH]; 4651 4652 if (ctx->super->version > SPDK_BS_VERSION || 4653 ctx->super->version < SPDK_BS_INITIAL_VERSION) { 4654 bs_load_ctx_fail(ctx, -EILSEQ); 4655 return; 4656 } 4657 4658 if (memcmp(ctx->super->signature, SPDK_BS_SUPER_BLOCK_SIG, 4659 sizeof(ctx->super->signature)) != 0) { 4660 bs_load_ctx_fail(ctx, -EILSEQ); 4661 return; 4662 } 4663 4664 crc = blob_md_page_calc_crc(ctx->super); 4665 if (crc != ctx->super->crc) { 4666 bs_load_ctx_fail(ctx, -EILSEQ); 4667 return; 4668 } 4669 4670 if (memcmp(&ctx->bs->bstype, &ctx->super->bstype, SPDK_BLOBSTORE_TYPE_LENGTH) == 0) { 4671 SPDK_DEBUGLOG(blob, "Bstype matched - loading blobstore\n"); 4672 } else if (memcmp(&ctx->bs->bstype, zeros, SPDK_BLOBSTORE_TYPE_LENGTH) == 0) { 4673 SPDK_DEBUGLOG(blob, "Bstype wildcard used - loading blobstore regardless bstype\n"); 4674 } else { 4675 SPDK_DEBUGLOG(blob, "Unexpected bstype\n"); 4676 SPDK_LOGDUMP(blob, "Expected:", ctx->bs->bstype.bstype, SPDK_BLOBSTORE_TYPE_LENGTH); 4677 SPDK_LOGDUMP(blob, "Found:", ctx->super->bstype.bstype, SPDK_BLOBSTORE_TYPE_LENGTH); 4678 bs_load_ctx_fail(ctx, -ENXIO); 4679 return; 4680 } 4681 4682 if (ctx->super->size > ctx->bs->dev->blockcnt * ctx->bs->dev->blocklen) { 4683 SPDK_NOTICELOG("Size mismatch, dev size: %" PRIu64 ", blobstore size: %" PRIu64 "\n", 4684 ctx->bs->dev->blockcnt * ctx->bs->dev->blocklen, ctx->super->size); 4685 bs_load_ctx_fail(ctx, -EILSEQ); 4686 return; 4687 } 4688 4689 rc = bs_parse_super(ctx); 4690 if (rc < 0) { 4691 bs_load_ctx_fail(ctx, rc); 4692 return; 4693 } 4694 4695 if (ctx->super->used_blobid_mask_len == 0 || ctx->super->clean == 0 || ctx->force_recover) { 4696 bs_recover(ctx); 4697 } else { 4698 bs_load_read_used_pages(ctx); 4699 } 4700 } 4701 4702 static inline int 4703 bs_opts_copy(struct spdk_bs_opts *src, struct spdk_bs_opts *dst) 4704 { 4705 4706 if (!src->opts_size) { 4707 SPDK_ERRLOG("opts_size should not be zero value\n"); 4708 return -1; 4709 } 4710 4711 #define FIELD_OK(field) \ 4712 offsetof(struct spdk_bs_opts, field) + sizeof(src->field) <= src->opts_size 4713 4714 #define SET_FIELD(field) \ 4715 if (FIELD_OK(field)) { \ 4716 dst->field = src->field; \ 4717 } \ 4718 4719 SET_FIELD(cluster_sz); 4720 SET_FIELD(num_md_pages); 4721 SET_FIELD(max_md_ops); 4722 SET_FIELD(max_channel_ops); 4723 SET_FIELD(clear_method); 4724 4725 if (FIELD_OK(bstype)) { 4726 memcpy(&dst->bstype, &src->bstype, sizeof(dst->bstype)); 4727 } 4728 SET_FIELD(iter_cb_fn); 4729 SET_FIELD(iter_cb_arg); 4730 SET_FIELD(force_recover); 4731 SET_FIELD(esnap_bs_dev_create); 4732 SET_FIELD(esnap_ctx); 4733 4734 dst->opts_size = src->opts_size; 4735 4736 /* You should not remove this statement, but need to update the assert statement 4737 * if you add a new field, and also add a corresponding SET_FIELD statement */ 4738 SPDK_STATIC_ASSERT(sizeof(struct spdk_bs_opts) == 88, "Incorrect size"); 4739 4740 #undef FIELD_OK 4741 #undef SET_FIELD 4742 4743 return 0; 4744 } 4745 4746 void 4747 spdk_bs_load(struct spdk_bs_dev *dev, struct spdk_bs_opts *o, 4748 spdk_bs_op_with_handle_complete cb_fn, void *cb_arg) 4749 { 4750 struct spdk_blob_store *bs; 4751 struct spdk_bs_cpl cpl; 4752 struct spdk_bs_load_ctx *ctx; 4753 struct spdk_bs_opts opts = {}; 4754 int err; 4755 4756 SPDK_DEBUGLOG(blob, "Loading blobstore from dev %p\n", dev); 4757 4758 if ((SPDK_BS_PAGE_SIZE % dev->blocklen) != 0) { 4759 SPDK_DEBUGLOG(blob, "unsupported dev block length of %d\n", dev->blocklen); 4760 dev->destroy(dev); 4761 cb_fn(cb_arg, NULL, -EINVAL); 4762 return; 4763 } 4764 4765 spdk_bs_opts_init(&opts, sizeof(opts)); 4766 if (o) { 4767 if (bs_opts_copy(o, &opts)) { 4768 return; 4769 } 4770 } 4771 4772 if (opts.max_md_ops == 0 || opts.max_channel_ops == 0) { 4773 dev->destroy(dev); 4774 cb_fn(cb_arg, NULL, -EINVAL); 4775 return; 4776 } 4777 4778 err = bs_alloc(dev, &opts, &bs, &ctx); 4779 if (err) { 4780 dev->destroy(dev); 4781 cb_fn(cb_arg, NULL, err); 4782 return; 4783 } 4784 4785 cpl.type = SPDK_BS_CPL_TYPE_BS_HANDLE; 4786 cpl.u.bs_handle.cb_fn = cb_fn; 4787 cpl.u.bs_handle.cb_arg = cb_arg; 4788 cpl.u.bs_handle.bs = bs; 4789 4790 ctx->seq = bs_sequence_start_bs(bs->md_channel, &cpl); 4791 if (!ctx->seq) { 4792 spdk_free(ctx->super); 4793 free(ctx); 4794 bs_free(bs); 4795 cb_fn(cb_arg, NULL, -ENOMEM); 4796 return; 4797 } 4798 4799 /* Read the super block */ 4800 bs_sequence_read_dev(ctx->seq, ctx->super, bs_page_to_lba(bs, 0), 4801 bs_byte_to_lba(bs, sizeof(*ctx->super)), 4802 bs_load_super_cpl, ctx); 4803 } 4804 4805 /* END spdk_bs_load */ 4806 4807 /* START spdk_bs_dump */ 4808 4809 static void 4810 bs_dump_finish(spdk_bs_sequence_t *seq, struct spdk_bs_load_ctx *ctx, int bserrno) 4811 { 4812 spdk_free(ctx->super); 4813 4814 /* 4815 * We need to defer calling bs_call_cpl() until after 4816 * dev destruction, so tuck these away for later use. 4817 */ 4818 ctx->bs->unload_err = bserrno; 4819 memcpy(&ctx->bs->unload_cpl, &seq->cpl, sizeof(struct spdk_bs_cpl)); 4820 seq->cpl.type = SPDK_BS_CPL_TYPE_NONE; 4821 4822 bs_sequence_finish(seq, 0); 4823 bs_free(ctx->bs); 4824 free(ctx); 4825 } 4826 4827 static void 4828 bs_dump_print_xattr(struct spdk_bs_load_ctx *ctx, struct spdk_blob_md_descriptor *desc) 4829 { 4830 struct spdk_blob_md_descriptor_xattr *desc_xattr; 4831 uint32_t i; 4832 const char *type; 4833 4834 desc_xattr = (struct spdk_blob_md_descriptor_xattr *)desc; 4835 4836 if (desc_xattr->length != 4837 sizeof(desc_xattr->name_length) + sizeof(desc_xattr->value_length) + 4838 desc_xattr->name_length + desc_xattr->value_length) { 4839 } 4840 4841 memcpy(ctx->xattr_name, desc_xattr->name, desc_xattr->name_length); 4842 ctx->xattr_name[desc_xattr->name_length] = '\0'; 4843 if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_XATTR) { 4844 type = "XATTR"; 4845 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_XATTR_INTERNAL) { 4846 type = "XATTR_INTERNAL"; 4847 } else { 4848 assert(false); 4849 type = "XATTR_?"; 4850 } 4851 fprintf(ctx->fp, "%s: name = \"%s\"\n", type, ctx->xattr_name); 4852 fprintf(ctx->fp, " value = \""); 4853 ctx->print_xattr_fn(ctx->fp, ctx->super->bstype.bstype, ctx->xattr_name, 4854 (void *)((uintptr_t)desc_xattr->name + desc_xattr->name_length), 4855 desc_xattr->value_length); 4856 fprintf(ctx->fp, "\"\n"); 4857 for (i = 0; i < desc_xattr->value_length; i++) { 4858 if (i % 16 == 0) { 4859 fprintf(ctx->fp, " "); 4860 } 4861 fprintf(ctx->fp, "%02" PRIx8 " ", *((uint8_t *)desc_xattr->name + desc_xattr->name_length + i)); 4862 if ((i + 1) % 16 == 0) { 4863 fprintf(ctx->fp, "\n"); 4864 } 4865 } 4866 if (i % 16 != 0) { 4867 fprintf(ctx->fp, "\n"); 4868 } 4869 } 4870 4871 struct type_flag_desc { 4872 uint64_t mask; 4873 uint64_t val; 4874 const char *name; 4875 }; 4876 4877 static void 4878 bs_dump_print_type_bits(struct spdk_bs_load_ctx *ctx, uint64_t flags, 4879 struct type_flag_desc *desc, size_t numflags) 4880 { 4881 uint64_t covered = 0; 4882 size_t i; 4883 4884 for (i = 0; i < numflags; i++) { 4885 if ((desc[i].mask & flags) != desc[i].val) { 4886 continue; 4887 } 4888 fprintf(ctx->fp, "\t\t 0x%016" PRIx64 " %s", desc[i].val, desc[i].name); 4889 if (desc[i].mask != desc[i].val) { 4890 fprintf(ctx->fp, " (mask 0x%" PRIx64 " value 0x%" PRIx64 ")", 4891 desc[i].mask, desc[i].val); 4892 } 4893 fprintf(ctx->fp, "\n"); 4894 covered |= desc[i].mask; 4895 } 4896 if ((flags & ~covered) != 0) { 4897 fprintf(ctx->fp, "\t\t 0x%016" PRIx64 " Unknown\n", flags & ~covered); 4898 } 4899 } 4900 4901 static void 4902 bs_dump_print_type_flags(struct spdk_bs_load_ctx *ctx, struct spdk_blob_md_descriptor *desc) 4903 { 4904 struct spdk_blob_md_descriptor_flags *type_desc; 4905 #define ADD_FLAG(f) { f, f, #f } 4906 #define ADD_MASK_VAL(m, v) { m, v, #v } 4907 static struct type_flag_desc invalid[] = { 4908 ADD_FLAG(SPDK_BLOB_THIN_PROV), 4909 ADD_FLAG(SPDK_BLOB_INTERNAL_XATTR), 4910 ADD_FLAG(SPDK_BLOB_EXTENT_TABLE), 4911 }; 4912 static struct type_flag_desc data_ro[] = { 4913 ADD_FLAG(SPDK_BLOB_READ_ONLY), 4914 }; 4915 static struct type_flag_desc md_ro[] = { 4916 ADD_MASK_VAL(SPDK_BLOB_MD_RO_FLAGS_MASK, BLOB_CLEAR_WITH_DEFAULT), 4917 ADD_MASK_VAL(SPDK_BLOB_MD_RO_FLAGS_MASK, BLOB_CLEAR_WITH_NONE), 4918 ADD_MASK_VAL(SPDK_BLOB_MD_RO_FLAGS_MASK, BLOB_CLEAR_WITH_UNMAP), 4919 ADD_MASK_VAL(SPDK_BLOB_MD_RO_FLAGS_MASK, BLOB_CLEAR_WITH_WRITE_ZEROES), 4920 }; 4921 #undef ADD_FLAG 4922 #undef ADD_MASK_VAL 4923 4924 type_desc = (struct spdk_blob_md_descriptor_flags *)desc; 4925 fprintf(ctx->fp, "Flags:\n"); 4926 fprintf(ctx->fp, "\tinvalid: 0x%016" PRIx64 "\n", type_desc->invalid_flags); 4927 bs_dump_print_type_bits(ctx, type_desc->invalid_flags, invalid, 4928 SPDK_COUNTOF(invalid)); 4929 fprintf(ctx->fp, "\tdata_ro: 0x%016" PRIx64 "\n", type_desc->data_ro_flags); 4930 bs_dump_print_type_bits(ctx, type_desc->data_ro_flags, data_ro, 4931 SPDK_COUNTOF(data_ro)); 4932 fprintf(ctx->fp, "\t md_ro: 0x%016" PRIx64 "\n", type_desc->md_ro_flags); 4933 bs_dump_print_type_bits(ctx, type_desc->md_ro_flags, md_ro, 4934 SPDK_COUNTOF(md_ro)); 4935 } 4936 4937 static void 4938 bs_dump_print_extent_table(struct spdk_bs_load_ctx *ctx, struct spdk_blob_md_descriptor *desc) 4939 { 4940 struct spdk_blob_md_descriptor_extent_table *et_desc; 4941 uint64_t num_extent_pages; 4942 uint32_t et_idx; 4943 4944 et_desc = (struct spdk_blob_md_descriptor_extent_table *)desc; 4945 num_extent_pages = (et_desc->length - sizeof(et_desc->num_clusters)) / 4946 sizeof(et_desc->extent_page[0]); 4947 4948 fprintf(ctx->fp, "Extent table:\n"); 4949 for (et_idx = 0; et_idx < num_extent_pages; et_idx++) { 4950 if (et_desc->extent_page[et_idx].page_idx == 0) { 4951 /* Zeroes represent unallocated extent pages. */ 4952 continue; 4953 } 4954 fprintf(ctx->fp, "\tExtent page: %5" PRIu32 " length %3" PRIu32 4955 " at LBA %" PRIu64 "\n", et_desc->extent_page[et_idx].page_idx, 4956 et_desc->extent_page[et_idx].num_pages, 4957 bs_md_page_to_lba(ctx->bs, et_desc->extent_page[et_idx].page_idx)); 4958 } 4959 } 4960 4961 static void 4962 bs_dump_print_md_page(struct spdk_bs_load_ctx *ctx) 4963 { 4964 uint32_t page_idx = ctx->cur_page; 4965 struct spdk_blob_md_page *page = ctx->page; 4966 struct spdk_blob_md_descriptor *desc; 4967 size_t cur_desc = 0; 4968 uint32_t crc; 4969 4970 fprintf(ctx->fp, "=========\n"); 4971 fprintf(ctx->fp, "Metadata Page Index: %" PRIu32 " (0x%" PRIx32 ")\n", page_idx, page_idx); 4972 fprintf(ctx->fp, "Start LBA: %" PRIu64 "\n", bs_md_page_to_lba(ctx->bs, page_idx)); 4973 fprintf(ctx->fp, "Blob ID: 0x%" PRIx64 "\n", page->id); 4974 fprintf(ctx->fp, "Sequence: %" PRIu32 "\n", page->sequence_num); 4975 if (page->next == SPDK_INVALID_MD_PAGE) { 4976 fprintf(ctx->fp, "Next: None\n"); 4977 } else { 4978 fprintf(ctx->fp, "Next: %" PRIu32 "\n", page->next); 4979 } 4980 fprintf(ctx->fp, "In used bit array%s:", ctx->super->clean ? "" : " (not clean: dubious)"); 4981 if (spdk_bit_array_get(ctx->bs->used_md_pages, page_idx)) { 4982 fprintf(ctx->fp, " md"); 4983 } 4984 if (spdk_bit_array_get(ctx->bs->used_blobids, page_idx)) { 4985 fprintf(ctx->fp, " blob"); 4986 } 4987 fprintf(ctx->fp, "\n"); 4988 4989 crc = blob_md_page_calc_crc(page); 4990 fprintf(ctx->fp, "CRC: 0x%" PRIx32 " (%s)\n", page->crc, crc == page->crc ? "OK" : "Mismatch"); 4991 4992 desc = (struct spdk_blob_md_descriptor *)page->descriptors; 4993 while (cur_desc < sizeof(page->descriptors)) { 4994 if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_PADDING) { 4995 if (desc->length == 0) { 4996 /* If padding and length are 0, this terminates the page */ 4997 break; 4998 } 4999 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_EXTENT_RLE) { 5000 struct spdk_blob_md_descriptor_extent_rle *desc_extent_rle; 5001 unsigned int i; 5002 5003 desc_extent_rle = (struct spdk_blob_md_descriptor_extent_rle *)desc; 5004 5005 for (i = 0; i < desc_extent_rle->length / sizeof(desc_extent_rle->extents[0]); i++) { 5006 if (desc_extent_rle->extents[i].cluster_idx != 0) { 5007 fprintf(ctx->fp, "Allocated Extent - Start: %" PRIu32, 5008 desc_extent_rle->extents[i].cluster_idx); 5009 } else { 5010 fprintf(ctx->fp, "Unallocated Extent - "); 5011 } 5012 fprintf(ctx->fp, " Length: %" PRIu32, desc_extent_rle->extents[i].length); 5013 fprintf(ctx->fp, "\n"); 5014 } 5015 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_EXTENT_PAGE) { 5016 struct spdk_blob_md_descriptor_extent_page *desc_extent; 5017 unsigned int i; 5018 5019 desc_extent = (struct spdk_blob_md_descriptor_extent_page *)desc; 5020 5021 for (i = 0; i < desc_extent->length / sizeof(desc_extent->cluster_idx[0]); i++) { 5022 if (desc_extent->cluster_idx[i] != 0) { 5023 fprintf(ctx->fp, "Allocated Extent - Start: %" PRIu32, 5024 desc_extent->cluster_idx[i]); 5025 } else { 5026 fprintf(ctx->fp, "Unallocated Extent"); 5027 } 5028 fprintf(ctx->fp, "\n"); 5029 } 5030 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_XATTR) { 5031 bs_dump_print_xattr(ctx, desc); 5032 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_XATTR_INTERNAL) { 5033 bs_dump_print_xattr(ctx, desc); 5034 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_FLAGS) { 5035 bs_dump_print_type_flags(ctx, desc); 5036 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_EXTENT_TABLE) { 5037 bs_dump_print_extent_table(ctx, desc); 5038 } else { 5039 /* Error */ 5040 fprintf(ctx->fp, "Unknown descriptor type %" PRIu8 "\n", desc->type); 5041 } 5042 /* Advance to the next descriptor */ 5043 cur_desc += sizeof(*desc) + desc->length; 5044 if (cur_desc + sizeof(*desc) > sizeof(page->descriptors)) { 5045 break; 5046 } 5047 desc = (struct spdk_blob_md_descriptor *)((uintptr_t)page->descriptors + cur_desc); 5048 } 5049 } 5050 5051 static void 5052 bs_dump_read_md_page_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 5053 { 5054 struct spdk_bs_load_ctx *ctx = cb_arg; 5055 5056 if (bserrno != 0) { 5057 bs_dump_finish(seq, ctx, bserrno); 5058 return; 5059 } 5060 5061 if (ctx->page->id != 0) { 5062 bs_dump_print_md_page(ctx); 5063 } 5064 5065 ctx->cur_page++; 5066 5067 if (ctx->cur_page < ctx->super->md_len) { 5068 bs_dump_read_md_page(seq, ctx); 5069 } else { 5070 spdk_free(ctx->page); 5071 bs_dump_finish(seq, ctx, 0); 5072 } 5073 } 5074 5075 static void 5076 bs_dump_read_md_page(spdk_bs_sequence_t *seq, void *cb_arg) 5077 { 5078 struct spdk_bs_load_ctx *ctx = cb_arg; 5079 uint64_t lba; 5080 5081 assert(ctx->cur_page < ctx->super->md_len); 5082 lba = bs_page_to_lba(ctx->bs, ctx->super->md_start + ctx->cur_page); 5083 bs_sequence_read_dev(seq, ctx->page, lba, 5084 bs_byte_to_lba(ctx->bs, SPDK_BS_PAGE_SIZE), 5085 bs_dump_read_md_page_cpl, ctx); 5086 } 5087 5088 static void 5089 bs_dump_super_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 5090 { 5091 struct spdk_bs_load_ctx *ctx = cb_arg; 5092 int rc; 5093 5094 fprintf(ctx->fp, "Signature: \"%.8s\" ", ctx->super->signature); 5095 if (memcmp(ctx->super->signature, SPDK_BS_SUPER_BLOCK_SIG, 5096 sizeof(ctx->super->signature)) != 0) { 5097 fprintf(ctx->fp, "(Mismatch)\n"); 5098 bs_dump_finish(seq, ctx, bserrno); 5099 return; 5100 } else { 5101 fprintf(ctx->fp, "(OK)\n"); 5102 } 5103 fprintf(ctx->fp, "Version: %" PRIu32 "\n", ctx->super->version); 5104 fprintf(ctx->fp, "CRC: 0x%x (%s)\n", ctx->super->crc, 5105 (ctx->super->crc == blob_md_page_calc_crc(ctx->super)) ? "OK" : "Mismatch"); 5106 fprintf(ctx->fp, "Blobstore Type: %.*s\n", SPDK_BLOBSTORE_TYPE_LENGTH, ctx->super->bstype.bstype); 5107 fprintf(ctx->fp, "Cluster Size: %" PRIu32 "\n", ctx->super->cluster_size); 5108 fprintf(ctx->fp, "Super Blob ID: "); 5109 if (ctx->super->super_blob == SPDK_BLOBID_INVALID) { 5110 fprintf(ctx->fp, "(None)\n"); 5111 } else { 5112 fprintf(ctx->fp, "0x%" PRIx64 "\n", ctx->super->super_blob); 5113 } 5114 fprintf(ctx->fp, "Clean: %" PRIu32 "\n", ctx->super->clean); 5115 fprintf(ctx->fp, "Used Metadata Page Mask Start: %" PRIu32 "\n", ctx->super->used_page_mask_start); 5116 fprintf(ctx->fp, "Used Metadata Page Mask Length: %" PRIu32 "\n", ctx->super->used_page_mask_len); 5117 fprintf(ctx->fp, "Used Cluster Mask Start: %" PRIu32 "\n", ctx->super->used_cluster_mask_start); 5118 fprintf(ctx->fp, "Used Cluster Mask Length: %" PRIu32 "\n", ctx->super->used_cluster_mask_len); 5119 fprintf(ctx->fp, "Used Blob ID Mask Start: %" PRIu32 "\n", ctx->super->used_blobid_mask_start); 5120 fprintf(ctx->fp, "Used Blob ID Mask Length: %" PRIu32 "\n", ctx->super->used_blobid_mask_len); 5121 fprintf(ctx->fp, "Metadata Start: %" PRIu32 "\n", ctx->super->md_start); 5122 fprintf(ctx->fp, "Metadata Length: %" PRIu32 "\n", ctx->super->md_len); 5123 5124 ctx->cur_page = 0; 5125 ctx->page = spdk_zmalloc(SPDK_BS_PAGE_SIZE, 0, 5126 NULL, SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA); 5127 if (!ctx->page) { 5128 bs_dump_finish(seq, ctx, -ENOMEM); 5129 return; 5130 } 5131 5132 rc = bs_parse_super(ctx); 5133 if (rc < 0) { 5134 bs_load_ctx_fail(ctx, rc); 5135 return; 5136 } 5137 5138 bs_load_read_used_pages(ctx); 5139 } 5140 5141 void 5142 spdk_bs_dump(struct spdk_bs_dev *dev, FILE *fp, spdk_bs_dump_print_xattr print_xattr_fn, 5143 spdk_bs_op_complete cb_fn, void *cb_arg) 5144 { 5145 struct spdk_blob_store *bs; 5146 struct spdk_bs_cpl cpl; 5147 struct spdk_bs_load_ctx *ctx; 5148 struct spdk_bs_opts opts = {}; 5149 int err; 5150 5151 SPDK_DEBUGLOG(blob, "Dumping blobstore from dev %p\n", dev); 5152 5153 spdk_bs_opts_init(&opts, sizeof(opts)); 5154 5155 err = bs_alloc(dev, &opts, &bs, &ctx); 5156 if (err) { 5157 dev->destroy(dev); 5158 cb_fn(cb_arg, err); 5159 return; 5160 } 5161 5162 ctx->dumping = true; 5163 ctx->fp = fp; 5164 ctx->print_xattr_fn = print_xattr_fn; 5165 5166 cpl.type = SPDK_BS_CPL_TYPE_BS_BASIC; 5167 cpl.u.bs_basic.cb_fn = cb_fn; 5168 cpl.u.bs_basic.cb_arg = cb_arg; 5169 5170 ctx->seq = bs_sequence_start_bs(bs->md_channel, &cpl); 5171 if (!ctx->seq) { 5172 spdk_free(ctx->super); 5173 free(ctx); 5174 bs_free(bs); 5175 cb_fn(cb_arg, -ENOMEM); 5176 return; 5177 } 5178 5179 /* Read the super block */ 5180 bs_sequence_read_dev(ctx->seq, ctx->super, bs_page_to_lba(bs, 0), 5181 bs_byte_to_lba(bs, sizeof(*ctx->super)), 5182 bs_dump_super_cpl, ctx); 5183 } 5184 5185 /* END spdk_bs_dump */ 5186 5187 /* START spdk_bs_init */ 5188 5189 static void 5190 bs_init_persist_super_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 5191 { 5192 struct spdk_bs_load_ctx *ctx = cb_arg; 5193 5194 ctx->bs->used_clusters = spdk_bit_pool_create_from_array(ctx->used_clusters); 5195 spdk_free(ctx->super); 5196 free(ctx); 5197 5198 bs_sequence_finish(seq, bserrno); 5199 } 5200 5201 static void 5202 bs_init_trim_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 5203 { 5204 struct spdk_bs_load_ctx *ctx = cb_arg; 5205 5206 /* Write super block */ 5207 bs_sequence_write_dev(seq, ctx->super, bs_page_to_lba(ctx->bs, 0), 5208 bs_byte_to_lba(ctx->bs, sizeof(*ctx->super)), 5209 bs_init_persist_super_cpl, ctx); 5210 } 5211 5212 void 5213 spdk_bs_init(struct spdk_bs_dev *dev, struct spdk_bs_opts *o, 5214 spdk_bs_op_with_handle_complete cb_fn, void *cb_arg) 5215 { 5216 struct spdk_bs_load_ctx *ctx; 5217 struct spdk_blob_store *bs; 5218 struct spdk_bs_cpl cpl; 5219 spdk_bs_sequence_t *seq; 5220 spdk_bs_batch_t *batch; 5221 uint64_t num_md_lba; 5222 uint64_t num_md_pages; 5223 uint64_t num_md_clusters; 5224 uint64_t max_used_cluster_mask_len; 5225 uint32_t i; 5226 struct spdk_bs_opts opts = {}; 5227 int rc; 5228 uint64_t lba, lba_count; 5229 5230 SPDK_DEBUGLOG(blob, "Initializing blobstore on dev %p\n", dev); 5231 5232 if ((SPDK_BS_PAGE_SIZE % dev->blocklen) != 0) { 5233 SPDK_ERRLOG("unsupported dev block length of %d\n", 5234 dev->blocklen); 5235 dev->destroy(dev); 5236 cb_fn(cb_arg, NULL, -EINVAL); 5237 return; 5238 } 5239 5240 spdk_bs_opts_init(&opts, sizeof(opts)); 5241 if (o) { 5242 if (bs_opts_copy(o, &opts)) { 5243 return; 5244 } 5245 } 5246 5247 if (bs_opts_verify(&opts) != 0) { 5248 dev->destroy(dev); 5249 cb_fn(cb_arg, NULL, -EINVAL); 5250 return; 5251 } 5252 5253 rc = bs_alloc(dev, &opts, &bs, &ctx); 5254 if (rc) { 5255 dev->destroy(dev); 5256 cb_fn(cb_arg, NULL, rc); 5257 return; 5258 } 5259 5260 if (opts.num_md_pages == SPDK_BLOB_OPTS_NUM_MD_PAGES) { 5261 /* By default, allocate 1 page per cluster. 5262 * Technically, this over-allocates metadata 5263 * because more metadata will reduce the number 5264 * of usable clusters. This can be addressed with 5265 * more complex math in the future. 5266 */ 5267 bs->md_len = bs->total_clusters; 5268 } else { 5269 bs->md_len = opts.num_md_pages; 5270 } 5271 rc = spdk_bit_array_resize(&bs->used_md_pages, bs->md_len); 5272 if (rc < 0) { 5273 spdk_free(ctx->super); 5274 free(ctx); 5275 bs_free(bs); 5276 cb_fn(cb_arg, NULL, -ENOMEM); 5277 return; 5278 } 5279 5280 rc = spdk_bit_array_resize(&bs->used_blobids, bs->md_len); 5281 if (rc < 0) { 5282 spdk_free(ctx->super); 5283 free(ctx); 5284 bs_free(bs); 5285 cb_fn(cb_arg, NULL, -ENOMEM); 5286 return; 5287 } 5288 5289 rc = spdk_bit_array_resize(&bs->open_blobids, bs->md_len); 5290 if (rc < 0) { 5291 spdk_free(ctx->super); 5292 free(ctx); 5293 bs_free(bs); 5294 cb_fn(cb_arg, NULL, -ENOMEM); 5295 return; 5296 } 5297 5298 memcpy(ctx->super->signature, SPDK_BS_SUPER_BLOCK_SIG, 5299 sizeof(ctx->super->signature)); 5300 ctx->super->version = SPDK_BS_VERSION; 5301 ctx->super->length = sizeof(*ctx->super); 5302 ctx->super->super_blob = bs->super_blob; 5303 ctx->super->clean = 0; 5304 ctx->super->cluster_size = bs->cluster_sz; 5305 ctx->super->io_unit_size = bs->io_unit_size; 5306 memcpy(&ctx->super->bstype, &bs->bstype, sizeof(bs->bstype)); 5307 5308 /* Calculate how many pages the metadata consumes at the front 5309 * of the disk. 5310 */ 5311 5312 /* The super block uses 1 page */ 5313 num_md_pages = 1; 5314 5315 /* The used_md_pages mask requires 1 bit per metadata page, rounded 5316 * up to the nearest page, plus a header. 5317 */ 5318 ctx->super->used_page_mask_start = num_md_pages; 5319 ctx->super->used_page_mask_len = spdk_divide_round_up(sizeof(struct spdk_bs_md_mask) + 5320 spdk_divide_round_up(bs->md_len, 8), 5321 SPDK_BS_PAGE_SIZE); 5322 num_md_pages += ctx->super->used_page_mask_len; 5323 5324 /* The used_clusters mask requires 1 bit per cluster, rounded 5325 * up to the nearest page, plus a header. 5326 */ 5327 ctx->super->used_cluster_mask_start = num_md_pages; 5328 ctx->super->used_cluster_mask_len = spdk_divide_round_up(sizeof(struct spdk_bs_md_mask) + 5329 spdk_divide_round_up(bs->total_clusters, 8), 5330 SPDK_BS_PAGE_SIZE); 5331 /* The blobstore might be extended, then the used_cluster bitmap will need more space. 5332 * Here we calculate the max clusters we can support according to the 5333 * num_md_pages (bs->md_len). 5334 */ 5335 max_used_cluster_mask_len = spdk_divide_round_up(sizeof(struct spdk_bs_md_mask) + 5336 spdk_divide_round_up(bs->md_len, 8), 5337 SPDK_BS_PAGE_SIZE); 5338 max_used_cluster_mask_len = spdk_max(max_used_cluster_mask_len, 5339 ctx->super->used_cluster_mask_len); 5340 num_md_pages += max_used_cluster_mask_len; 5341 5342 /* The used_blobids mask requires 1 bit per metadata page, rounded 5343 * up to the nearest page, plus a header. 5344 */ 5345 ctx->super->used_blobid_mask_start = num_md_pages; 5346 ctx->super->used_blobid_mask_len = spdk_divide_round_up(sizeof(struct spdk_bs_md_mask) + 5347 spdk_divide_round_up(bs->md_len, 8), 5348 SPDK_BS_PAGE_SIZE); 5349 num_md_pages += ctx->super->used_blobid_mask_len; 5350 5351 /* The metadata region size was chosen above */ 5352 ctx->super->md_start = bs->md_start = num_md_pages; 5353 ctx->super->md_len = bs->md_len; 5354 num_md_pages += bs->md_len; 5355 5356 num_md_lba = bs_page_to_lba(bs, num_md_pages); 5357 5358 ctx->super->size = dev->blockcnt * dev->blocklen; 5359 5360 ctx->super->crc = blob_md_page_calc_crc(ctx->super); 5361 5362 num_md_clusters = spdk_divide_round_up(num_md_pages, bs->pages_per_cluster); 5363 if (num_md_clusters > bs->total_clusters) { 5364 SPDK_ERRLOG("Blobstore metadata cannot use more clusters than is available, " 5365 "please decrease number of pages reserved for metadata " 5366 "or increase cluster size.\n"); 5367 spdk_free(ctx->super); 5368 spdk_bit_array_free(&ctx->used_clusters); 5369 free(ctx); 5370 bs_free(bs); 5371 cb_fn(cb_arg, NULL, -ENOMEM); 5372 return; 5373 } 5374 /* Claim all of the clusters used by the metadata */ 5375 for (i = 0; i < num_md_clusters; i++) { 5376 spdk_bit_array_set(ctx->used_clusters, i); 5377 } 5378 5379 bs->num_free_clusters -= num_md_clusters; 5380 bs->total_data_clusters = bs->num_free_clusters; 5381 5382 cpl.type = SPDK_BS_CPL_TYPE_BS_HANDLE; 5383 cpl.u.bs_handle.cb_fn = cb_fn; 5384 cpl.u.bs_handle.cb_arg = cb_arg; 5385 cpl.u.bs_handle.bs = bs; 5386 5387 seq = bs_sequence_start_bs(bs->md_channel, &cpl); 5388 if (!seq) { 5389 spdk_free(ctx->super); 5390 free(ctx); 5391 bs_free(bs); 5392 cb_fn(cb_arg, NULL, -ENOMEM); 5393 return; 5394 } 5395 5396 batch = bs_sequence_to_batch(seq, bs_init_trim_cpl, ctx); 5397 5398 /* Clear metadata space */ 5399 bs_batch_write_zeroes_dev(batch, 0, num_md_lba); 5400 5401 lba = num_md_lba; 5402 lba_count = ctx->bs->dev->blockcnt - lba; 5403 switch (opts.clear_method) { 5404 case BS_CLEAR_WITH_UNMAP: 5405 /* Trim data clusters */ 5406 bs_batch_unmap_dev(batch, lba, lba_count); 5407 break; 5408 case BS_CLEAR_WITH_WRITE_ZEROES: 5409 /* Write_zeroes to data clusters */ 5410 bs_batch_write_zeroes_dev(batch, lba, lba_count); 5411 break; 5412 case BS_CLEAR_WITH_NONE: 5413 default: 5414 break; 5415 } 5416 5417 bs_batch_close(batch); 5418 } 5419 5420 /* END spdk_bs_init */ 5421 5422 /* START spdk_bs_destroy */ 5423 5424 static void 5425 bs_destroy_trim_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 5426 { 5427 struct spdk_bs_load_ctx *ctx = cb_arg; 5428 struct spdk_blob_store *bs = ctx->bs; 5429 5430 /* 5431 * We need to defer calling bs_call_cpl() until after 5432 * dev destruction, so tuck these away for later use. 5433 */ 5434 bs->unload_err = bserrno; 5435 memcpy(&bs->unload_cpl, &seq->cpl, sizeof(struct spdk_bs_cpl)); 5436 seq->cpl.type = SPDK_BS_CPL_TYPE_NONE; 5437 5438 bs_sequence_finish(seq, bserrno); 5439 5440 bs_free(bs); 5441 free(ctx); 5442 } 5443 5444 void 5445 spdk_bs_destroy(struct spdk_blob_store *bs, spdk_bs_op_complete cb_fn, 5446 void *cb_arg) 5447 { 5448 struct spdk_bs_cpl cpl; 5449 spdk_bs_sequence_t *seq; 5450 struct spdk_bs_load_ctx *ctx; 5451 5452 SPDK_DEBUGLOG(blob, "Destroying blobstore\n"); 5453 5454 if (!RB_EMPTY(&bs->open_blobs)) { 5455 SPDK_ERRLOG("Blobstore still has open blobs\n"); 5456 cb_fn(cb_arg, -EBUSY); 5457 return; 5458 } 5459 5460 cpl.type = SPDK_BS_CPL_TYPE_BS_BASIC; 5461 cpl.u.bs_basic.cb_fn = cb_fn; 5462 cpl.u.bs_basic.cb_arg = cb_arg; 5463 5464 ctx = calloc(1, sizeof(*ctx)); 5465 if (!ctx) { 5466 cb_fn(cb_arg, -ENOMEM); 5467 return; 5468 } 5469 5470 ctx->bs = bs; 5471 5472 seq = bs_sequence_start_bs(bs->md_channel, &cpl); 5473 if (!seq) { 5474 free(ctx); 5475 cb_fn(cb_arg, -ENOMEM); 5476 return; 5477 } 5478 5479 /* Write zeroes to the super block */ 5480 bs_sequence_write_zeroes_dev(seq, 5481 bs_page_to_lba(bs, 0), 5482 bs_byte_to_lba(bs, sizeof(struct spdk_bs_super_block)), 5483 bs_destroy_trim_cpl, ctx); 5484 } 5485 5486 /* END spdk_bs_destroy */ 5487 5488 /* START spdk_bs_unload */ 5489 5490 static void 5491 bs_unload_finish(struct spdk_bs_load_ctx *ctx, int bserrno) 5492 { 5493 spdk_bs_sequence_t *seq = ctx->seq; 5494 5495 spdk_free(ctx->super); 5496 5497 /* 5498 * We need to defer calling bs_call_cpl() until after 5499 * dev destruction, so tuck these away for later use. 5500 */ 5501 ctx->bs->unload_err = bserrno; 5502 memcpy(&ctx->bs->unload_cpl, &seq->cpl, sizeof(struct spdk_bs_cpl)); 5503 seq->cpl.type = SPDK_BS_CPL_TYPE_NONE; 5504 5505 bs_sequence_finish(seq, bserrno); 5506 5507 bs_free(ctx->bs); 5508 free(ctx); 5509 } 5510 5511 static void 5512 bs_unload_write_super_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 5513 { 5514 struct spdk_bs_load_ctx *ctx = cb_arg; 5515 5516 bs_unload_finish(ctx, bserrno); 5517 } 5518 5519 static void 5520 bs_unload_write_used_clusters_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 5521 { 5522 struct spdk_bs_load_ctx *ctx = cb_arg; 5523 5524 spdk_free(ctx->mask); 5525 5526 if (bserrno != 0) { 5527 bs_unload_finish(ctx, bserrno); 5528 return; 5529 } 5530 5531 ctx->super->clean = 1; 5532 5533 bs_write_super(seq, ctx->bs, ctx->super, bs_unload_write_super_cpl, ctx); 5534 } 5535 5536 static void 5537 bs_unload_write_used_blobids_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 5538 { 5539 struct spdk_bs_load_ctx *ctx = cb_arg; 5540 5541 spdk_free(ctx->mask); 5542 ctx->mask = NULL; 5543 5544 if (bserrno != 0) { 5545 bs_unload_finish(ctx, bserrno); 5546 return; 5547 } 5548 5549 bs_write_used_clusters(seq, ctx, bs_unload_write_used_clusters_cpl); 5550 } 5551 5552 static void 5553 bs_unload_write_used_pages_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 5554 { 5555 struct spdk_bs_load_ctx *ctx = cb_arg; 5556 5557 spdk_free(ctx->mask); 5558 ctx->mask = NULL; 5559 5560 if (bserrno != 0) { 5561 bs_unload_finish(ctx, bserrno); 5562 return; 5563 } 5564 5565 bs_write_used_blobids(seq, ctx, bs_unload_write_used_blobids_cpl); 5566 } 5567 5568 static void 5569 bs_unload_read_super_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 5570 { 5571 struct spdk_bs_load_ctx *ctx = cb_arg; 5572 5573 if (bserrno != 0) { 5574 bs_unload_finish(ctx, bserrno); 5575 return; 5576 } 5577 5578 bs_write_used_md(seq, cb_arg, bs_unload_write_used_pages_cpl); 5579 } 5580 5581 void 5582 spdk_bs_unload(struct spdk_blob_store *bs, spdk_bs_op_complete cb_fn, void *cb_arg) 5583 { 5584 struct spdk_bs_cpl cpl; 5585 struct spdk_bs_load_ctx *ctx; 5586 5587 SPDK_DEBUGLOG(blob, "Syncing blobstore\n"); 5588 5589 /* 5590 * If external snapshot channels are being destroyed while the blobstore is unloaded, the 5591 * unload is deferred until after the channel destruction completes. 5592 */ 5593 if (bs->esnap_channels_unloading != 0) { 5594 if (bs->esnap_unload_cb_fn != NULL) { 5595 SPDK_ERRLOG("Blobstore unload in progress\n"); 5596 cb_fn(cb_arg, -EBUSY); 5597 return; 5598 } 5599 SPDK_DEBUGLOG(blob_esnap, "Blobstore unload deferred: %" PRIu32 5600 " esnap clones are unloading\n", bs->esnap_channels_unloading); 5601 bs->esnap_unload_cb_fn = cb_fn; 5602 bs->esnap_unload_cb_arg = cb_arg; 5603 return; 5604 } 5605 if (bs->esnap_unload_cb_fn != NULL) { 5606 SPDK_DEBUGLOG(blob_esnap, "Blobstore deferred unload progressing\n"); 5607 assert(bs->esnap_unload_cb_fn == cb_fn); 5608 assert(bs->esnap_unload_cb_arg == cb_arg); 5609 bs->esnap_unload_cb_fn = NULL; 5610 bs->esnap_unload_cb_arg = NULL; 5611 } 5612 5613 if (!RB_EMPTY(&bs->open_blobs)) { 5614 SPDK_ERRLOG("Blobstore still has open blobs\n"); 5615 cb_fn(cb_arg, -EBUSY); 5616 return; 5617 } 5618 5619 ctx = calloc(1, sizeof(*ctx)); 5620 if (!ctx) { 5621 cb_fn(cb_arg, -ENOMEM); 5622 return; 5623 } 5624 5625 ctx->bs = bs; 5626 5627 ctx->super = spdk_zmalloc(sizeof(*ctx->super), 0x1000, NULL, 5628 SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA); 5629 if (!ctx->super) { 5630 free(ctx); 5631 cb_fn(cb_arg, -ENOMEM); 5632 return; 5633 } 5634 5635 cpl.type = SPDK_BS_CPL_TYPE_BS_BASIC; 5636 cpl.u.bs_basic.cb_fn = cb_fn; 5637 cpl.u.bs_basic.cb_arg = cb_arg; 5638 5639 ctx->seq = bs_sequence_start_bs(bs->md_channel, &cpl); 5640 if (!ctx->seq) { 5641 spdk_free(ctx->super); 5642 free(ctx); 5643 cb_fn(cb_arg, -ENOMEM); 5644 return; 5645 } 5646 5647 /* Read super block */ 5648 bs_sequence_read_dev(ctx->seq, ctx->super, bs_page_to_lba(bs, 0), 5649 bs_byte_to_lba(bs, sizeof(*ctx->super)), 5650 bs_unload_read_super_cpl, ctx); 5651 } 5652 5653 /* END spdk_bs_unload */ 5654 5655 /* START spdk_bs_set_super */ 5656 5657 struct spdk_bs_set_super_ctx { 5658 struct spdk_blob_store *bs; 5659 struct spdk_bs_super_block *super; 5660 }; 5661 5662 static void 5663 bs_set_super_write_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 5664 { 5665 struct spdk_bs_set_super_ctx *ctx = cb_arg; 5666 5667 if (bserrno != 0) { 5668 SPDK_ERRLOG("Unable to write to super block of blobstore\n"); 5669 } 5670 5671 spdk_free(ctx->super); 5672 5673 bs_sequence_finish(seq, bserrno); 5674 5675 free(ctx); 5676 } 5677 5678 static void 5679 bs_set_super_read_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 5680 { 5681 struct spdk_bs_set_super_ctx *ctx = cb_arg; 5682 5683 if (bserrno != 0) { 5684 SPDK_ERRLOG("Unable to read super block of blobstore\n"); 5685 spdk_free(ctx->super); 5686 bs_sequence_finish(seq, bserrno); 5687 free(ctx); 5688 return; 5689 } 5690 5691 bs_write_super(seq, ctx->bs, ctx->super, bs_set_super_write_cpl, ctx); 5692 } 5693 5694 void 5695 spdk_bs_set_super(struct spdk_blob_store *bs, spdk_blob_id blobid, 5696 spdk_bs_op_complete cb_fn, void *cb_arg) 5697 { 5698 struct spdk_bs_cpl cpl; 5699 spdk_bs_sequence_t *seq; 5700 struct spdk_bs_set_super_ctx *ctx; 5701 5702 SPDK_DEBUGLOG(blob, "Setting super blob id on blobstore\n"); 5703 5704 ctx = calloc(1, sizeof(*ctx)); 5705 if (!ctx) { 5706 cb_fn(cb_arg, -ENOMEM); 5707 return; 5708 } 5709 5710 ctx->bs = bs; 5711 5712 ctx->super = spdk_zmalloc(sizeof(*ctx->super), 0x1000, NULL, 5713 SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA); 5714 if (!ctx->super) { 5715 free(ctx); 5716 cb_fn(cb_arg, -ENOMEM); 5717 return; 5718 } 5719 5720 cpl.type = SPDK_BS_CPL_TYPE_BS_BASIC; 5721 cpl.u.bs_basic.cb_fn = cb_fn; 5722 cpl.u.bs_basic.cb_arg = cb_arg; 5723 5724 seq = bs_sequence_start_bs(bs->md_channel, &cpl); 5725 if (!seq) { 5726 spdk_free(ctx->super); 5727 free(ctx); 5728 cb_fn(cb_arg, -ENOMEM); 5729 return; 5730 } 5731 5732 bs->super_blob = blobid; 5733 5734 /* Read super block */ 5735 bs_sequence_read_dev(seq, ctx->super, bs_page_to_lba(bs, 0), 5736 bs_byte_to_lba(bs, sizeof(*ctx->super)), 5737 bs_set_super_read_cpl, ctx); 5738 } 5739 5740 /* END spdk_bs_set_super */ 5741 5742 void 5743 spdk_bs_get_super(struct spdk_blob_store *bs, 5744 spdk_blob_op_with_id_complete cb_fn, void *cb_arg) 5745 { 5746 if (bs->super_blob == SPDK_BLOBID_INVALID) { 5747 cb_fn(cb_arg, SPDK_BLOBID_INVALID, -ENOENT); 5748 } else { 5749 cb_fn(cb_arg, bs->super_blob, 0); 5750 } 5751 } 5752 5753 uint64_t 5754 spdk_bs_get_cluster_size(struct spdk_blob_store *bs) 5755 { 5756 return bs->cluster_sz; 5757 } 5758 5759 uint64_t 5760 spdk_bs_get_page_size(struct spdk_blob_store *bs) 5761 { 5762 return SPDK_BS_PAGE_SIZE; 5763 } 5764 5765 uint64_t 5766 spdk_bs_get_io_unit_size(struct spdk_blob_store *bs) 5767 { 5768 return bs->io_unit_size; 5769 } 5770 5771 uint64_t 5772 spdk_bs_free_cluster_count(struct spdk_blob_store *bs) 5773 { 5774 return bs->num_free_clusters; 5775 } 5776 5777 uint64_t 5778 spdk_bs_total_data_cluster_count(struct spdk_blob_store *bs) 5779 { 5780 return bs->total_data_clusters; 5781 } 5782 5783 static int 5784 bs_register_md_thread(struct spdk_blob_store *bs) 5785 { 5786 bs->md_channel = spdk_get_io_channel(bs); 5787 if (!bs->md_channel) { 5788 SPDK_ERRLOG("Failed to get IO channel.\n"); 5789 return -1; 5790 } 5791 5792 return 0; 5793 } 5794 5795 static int 5796 bs_unregister_md_thread(struct spdk_blob_store *bs) 5797 { 5798 spdk_put_io_channel(bs->md_channel); 5799 5800 return 0; 5801 } 5802 5803 spdk_blob_id 5804 spdk_blob_get_id(struct spdk_blob *blob) 5805 { 5806 assert(blob != NULL); 5807 5808 return blob->id; 5809 } 5810 5811 uint64_t 5812 spdk_blob_get_num_pages(struct spdk_blob *blob) 5813 { 5814 assert(blob != NULL); 5815 5816 return bs_cluster_to_page(blob->bs, blob->active.num_clusters); 5817 } 5818 5819 uint64_t 5820 spdk_blob_get_num_io_units(struct spdk_blob *blob) 5821 { 5822 assert(blob != NULL); 5823 5824 return spdk_blob_get_num_pages(blob) * bs_io_unit_per_page(blob->bs); 5825 } 5826 5827 uint64_t 5828 spdk_blob_get_num_clusters(struct spdk_blob *blob) 5829 { 5830 assert(blob != NULL); 5831 5832 return blob->active.num_clusters; 5833 } 5834 5835 static uint64_t 5836 blob_find_io_unit(struct spdk_blob *blob, uint64_t offset, bool is_allocated) 5837 { 5838 uint64_t blob_io_unit_num = spdk_blob_get_num_io_units(blob); 5839 5840 while (offset < blob_io_unit_num) { 5841 if (bs_io_unit_is_allocated(blob, offset) == is_allocated) { 5842 return offset; 5843 } 5844 5845 offset += bs_num_io_units_to_cluster_boundary(blob, offset); 5846 } 5847 5848 return UINT64_MAX; 5849 } 5850 5851 uint64_t 5852 spdk_blob_get_next_allocated_io_unit(struct spdk_blob *blob, uint64_t offset) 5853 { 5854 return blob_find_io_unit(blob, offset, true); 5855 } 5856 5857 uint64_t 5858 spdk_blob_get_next_unallocated_io_unit(struct spdk_blob *blob, uint64_t offset) 5859 { 5860 return blob_find_io_unit(blob, offset, false); 5861 } 5862 5863 /* START spdk_bs_create_blob */ 5864 5865 static void 5866 bs_create_blob_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 5867 { 5868 struct spdk_blob *blob = cb_arg; 5869 uint32_t page_idx = bs_blobid_to_page(blob->id); 5870 5871 if (bserrno != 0) { 5872 spdk_spin_lock(&blob->bs->used_lock); 5873 spdk_bit_array_clear(blob->bs->used_blobids, page_idx); 5874 bs_release_md_page(blob->bs, page_idx); 5875 spdk_spin_unlock(&blob->bs->used_lock); 5876 } 5877 5878 blob_free(blob); 5879 5880 bs_sequence_finish(seq, bserrno); 5881 } 5882 5883 static int 5884 blob_set_xattrs(struct spdk_blob *blob, const struct spdk_blob_xattr_opts *xattrs, 5885 bool internal) 5886 { 5887 uint64_t i; 5888 size_t value_len = 0; 5889 int rc; 5890 const void *value = NULL; 5891 if (xattrs->count > 0 && xattrs->get_value == NULL) { 5892 return -EINVAL; 5893 } 5894 for (i = 0; i < xattrs->count; i++) { 5895 xattrs->get_value(xattrs->ctx, xattrs->names[i], &value, &value_len); 5896 if (value == NULL || value_len == 0) { 5897 return -EINVAL; 5898 } 5899 rc = blob_set_xattr(blob, xattrs->names[i], value, value_len, internal); 5900 if (rc < 0) { 5901 return rc; 5902 } 5903 } 5904 return 0; 5905 } 5906 5907 static void 5908 blob_opts_copy(const struct spdk_blob_opts *src, struct spdk_blob_opts *dst) 5909 { 5910 #define FIELD_OK(field) \ 5911 offsetof(struct spdk_blob_opts, field) + sizeof(src->field) <= src->opts_size 5912 5913 #define SET_FIELD(field) \ 5914 if (FIELD_OK(field)) { \ 5915 dst->field = src->field; \ 5916 } \ 5917 5918 SET_FIELD(num_clusters); 5919 SET_FIELD(thin_provision); 5920 SET_FIELD(clear_method); 5921 5922 if (FIELD_OK(xattrs)) { 5923 memcpy(&dst->xattrs, &src->xattrs, sizeof(src->xattrs)); 5924 } 5925 5926 SET_FIELD(use_extent_table); 5927 SET_FIELD(esnap_id); 5928 SET_FIELD(esnap_id_len); 5929 5930 dst->opts_size = src->opts_size; 5931 5932 /* You should not remove this statement, but need to update the assert statement 5933 * if you add a new field, and also add a corresponding SET_FIELD statement */ 5934 SPDK_STATIC_ASSERT(sizeof(struct spdk_blob_opts) == 80, "Incorrect size"); 5935 5936 #undef FIELD_OK 5937 #undef SET_FIELD 5938 } 5939 5940 static void 5941 bs_create_blob(struct spdk_blob_store *bs, 5942 const struct spdk_blob_opts *opts, 5943 const struct spdk_blob_xattr_opts *internal_xattrs, 5944 spdk_blob_op_with_id_complete cb_fn, void *cb_arg) 5945 { 5946 struct spdk_blob *blob; 5947 uint32_t page_idx; 5948 struct spdk_bs_cpl cpl; 5949 struct spdk_blob_opts opts_local; 5950 struct spdk_blob_xattr_opts internal_xattrs_default; 5951 spdk_bs_sequence_t *seq; 5952 spdk_blob_id id; 5953 int rc; 5954 5955 assert(spdk_get_thread() == bs->md_thread); 5956 5957 spdk_spin_lock(&bs->used_lock); 5958 page_idx = spdk_bit_array_find_first_clear(bs->used_md_pages, 0); 5959 if (page_idx == UINT32_MAX) { 5960 spdk_spin_unlock(&bs->used_lock); 5961 cb_fn(cb_arg, 0, -ENOMEM); 5962 return; 5963 } 5964 spdk_bit_array_set(bs->used_blobids, page_idx); 5965 bs_claim_md_page(bs, page_idx); 5966 spdk_spin_unlock(&bs->used_lock); 5967 5968 id = bs_page_to_blobid(page_idx); 5969 5970 SPDK_DEBUGLOG(blob, "Creating blob with id %" PRIu64 " at page %u\n", id, page_idx); 5971 5972 spdk_blob_opts_init(&opts_local, sizeof(opts_local)); 5973 if (opts) { 5974 blob_opts_copy(opts, &opts_local); 5975 } 5976 5977 blob = blob_alloc(bs, id); 5978 if (!blob) { 5979 rc = -ENOMEM; 5980 goto error; 5981 } 5982 5983 blob->use_extent_table = opts_local.use_extent_table; 5984 if (blob->use_extent_table) { 5985 blob->invalid_flags |= SPDK_BLOB_EXTENT_TABLE; 5986 } 5987 5988 if (!internal_xattrs) { 5989 blob_xattrs_init(&internal_xattrs_default); 5990 internal_xattrs = &internal_xattrs_default; 5991 } 5992 5993 rc = blob_set_xattrs(blob, &opts_local.xattrs, false); 5994 if (rc < 0) { 5995 goto error; 5996 } 5997 5998 rc = blob_set_xattrs(blob, internal_xattrs, true); 5999 if (rc < 0) { 6000 goto error; 6001 } 6002 6003 if (opts_local.thin_provision) { 6004 blob_set_thin_provision(blob); 6005 } 6006 6007 blob_set_clear_method(blob, opts_local.clear_method); 6008 6009 if (opts_local.esnap_id != NULL) { 6010 if (opts_local.esnap_id_len > UINT16_MAX) { 6011 SPDK_ERRLOG("esnap id length %" PRIu64 "is too long\n", 6012 opts_local.esnap_id_len); 6013 goto error; 6014 6015 } 6016 blob_set_thin_provision(blob); 6017 blob->invalid_flags |= SPDK_BLOB_EXTERNAL_SNAPSHOT; 6018 rc = blob_set_xattr(blob, BLOB_EXTERNAL_SNAPSHOT_ID, 6019 opts_local.esnap_id, opts_local.esnap_id_len, true); 6020 if (rc != 0) { 6021 goto error; 6022 } 6023 } 6024 6025 rc = blob_resize(blob, opts_local.num_clusters); 6026 if (rc < 0) { 6027 goto error; 6028 } 6029 cpl.type = SPDK_BS_CPL_TYPE_BLOBID; 6030 cpl.u.blobid.cb_fn = cb_fn; 6031 cpl.u.blobid.cb_arg = cb_arg; 6032 cpl.u.blobid.blobid = blob->id; 6033 6034 seq = bs_sequence_start_bs(bs->md_channel, &cpl); 6035 if (!seq) { 6036 rc = -ENOMEM; 6037 goto error; 6038 } 6039 6040 blob_persist(seq, blob, bs_create_blob_cpl, blob); 6041 return; 6042 6043 error: 6044 SPDK_ERRLOG("Failed to create blob: %s, size in clusters/size: %lu (clusters)\n", 6045 spdk_strerror(rc), opts_local.num_clusters); 6046 if (blob != NULL) { 6047 blob_free(blob); 6048 } 6049 spdk_spin_lock(&bs->used_lock); 6050 spdk_bit_array_clear(bs->used_blobids, page_idx); 6051 bs_release_md_page(bs, page_idx); 6052 spdk_spin_unlock(&bs->used_lock); 6053 cb_fn(cb_arg, 0, rc); 6054 } 6055 6056 void 6057 spdk_bs_create_blob(struct spdk_blob_store *bs, 6058 spdk_blob_op_with_id_complete cb_fn, void *cb_arg) 6059 { 6060 bs_create_blob(bs, NULL, NULL, cb_fn, cb_arg); 6061 } 6062 6063 void 6064 spdk_bs_create_blob_ext(struct spdk_blob_store *bs, const struct spdk_blob_opts *opts, 6065 spdk_blob_op_with_id_complete cb_fn, void *cb_arg) 6066 { 6067 bs_create_blob(bs, opts, NULL, cb_fn, cb_arg); 6068 } 6069 6070 /* END spdk_bs_create_blob */ 6071 6072 /* START blob_cleanup */ 6073 6074 struct spdk_clone_snapshot_ctx { 6075 struct spdk_bs_cpl cpl; 6076 int bserrno; 6077 bool frozen; 6078 6079 struct spdk_io_channel *channel; 6080 6081 /* Current cluster for inflate operation */ 6082 uint64_t cluster; 6083 6084 /* For inflation force allocation of all unallocated clusters and remove 6085 * thin-provisioning. Otherwise only decouple parent and keep clone thin. */ 6086 bool allocate_all; 6087 6088 struct { 6089 spdk_blob_id id; 6090 struct spdk_blob *blob; 6091 bool md_ro; 6092 } original; 6093 struct { 6094 spdk_blob_id id; 6095 struct spdk_blob *blob; 6096 } new; 6097 6098 /* xattrs specified for snapshot/clones only. They have no impact on 6099 * the original blobs xattrs. */ 6100 const struct spdk_blob_xattr_opts *xattrs; 6101 }; 6102 6103 static void 6104 bs_clone_snapshot_cleanup_finish(void *cb_arg, int bserrno) 6105 { 6106 struct spdk_clone_snapshot_ctx *ctx = cb_arg; 6107 struct spdk_bs_cpl *cpl = &ctx->cpl; 6108 6109 if (bserrno != 0) { 6110 if (ctx->bserrno != 0) { 6111 SPDK_ERRLOG("Cleanup error %d\n", bserrno); 6112 } else { 6113 ctx->bserrno = bserrno; 6114 } 6115 } 6116 6117 switch (cpl->type) { 6118 case SPDK_BS_CPL_TYPE_BLOBID: 6119 cpl->u.blobid.cb_fn(cpl->u.blobid.cb_arg, cpl->u.blobid.blobid, ctx->bserrno); 6120 break; 6121 case SPDK_BS_CPL_TYPE_BLOB_BASIC: 6122 cpl->u.blob_basic.cb_fn(cpl->u.blob_basic.cb_arg, ctx->bserrno); 6123 break; 6124 default: 6125 SPDK_UNREACHABLE(); 6126 break; 6127 } 6128 6129 free(ctx); 6130 } 6131 6132 static void 6133 bs_snapshot_unfreeze_cpl(void *cb_arg, int bserrno) 6134 { 6135 struct spdk_clone_snapshot_ctx *ctx = (struct spdk_clone_snapshot_ctx *)cb_arg; 6136 struct spdk_blob *origblob = ctx->original.blob; 6137 6138 if (bserrno != 0) { 6139 if (ctx->bserrno != 0) { 6140 SPDK_ERRLOG("Unfreeze error %d\n", bserrno); 6141 } else { 6142 ctx->bserrno = bserrno; 6143 } 6144 } 6145 6146 ctx->original.id = origblob->id; 6147 origblob->locked_operation_in_progress = false; 6148 6149 /* Revert md_ro to original state */ 6150 origblob->md_ro = ctx->original.md_ro; 6151 6152 spdk_blob_close(origblob, bs_clone_snapshot_cleanup_finish, ctx); 6153 } 6154 6155 static void 6156 bs_clone_snapshot_origblob_cleanup(void *cb_arg, int bserrno) 6157 { 6158 struct spdk_clone_snapshot_ctx *ctx = (struct spdk_clone_snapshot_ctx *)cb_arg; 6159 struct spdk_blob *origblob = ctx->original.blob; 6160 6161 if (bserrno != 0) { 6162 if (ctx->bserrno != 0) { 6163 SPDK_ERRLOG("Cleanup error %d\n", bserrno); 6164 } else { 6165 ctx->bserrno = bserrno; 6166 } 6167 } 6168 6169 if (ctx->frozen) { 6170 /* Unfreeze any outstanding I/O */ 6171 blob_unfreeze_io(origblob, bs_snapshot_unfreeze_cpl, ctx); 6172 } else { 6173 bs_snapshot_unfreeze_cpl(ctx, 0); 6174 } 6175 6176 } 6177 6178 static void 6179 bs_clone_snapshot_newblob_cleanup(struct spdk_clone_snapshot_ctx *ctx, int bserrno) 6180 { 6181 struct spdk_blob *newblob = ctx->new.blob; 6182 6183 if (bserrno != 0) { 6184 if (ctx->bserrno != 0) { 6185 SPDK_ERRLOG("Cleanup error %d\n", bserrno); 6186 } else { 6187 ctx->bserrno = bserrno; 6188 } 6189 } 6190 6191 ctx->new.id = newblob->id; 6192 spdk_blob_close(newblob, bs_clone_snapshot_origblob_cleanup, ctx); 6193 } 6194 6195 /* END blob_cleanup */ 6196 6197 /* START spdk_bs_create_snapshot */ 6198 6199 static void 6200 bs_snapshot_swap_cluster_maps(struct spdk_blob *blob1, struct spdk_blob *blob2) 6201 { 6202 uint64_t *cluster_temp; 6203 uint32_t *extent_page_temp; 6204 6205 cluster_temp = blob1->active.clusters; 6206 blob1->active.clusters = blob2->active.clusters; 6207 blob2->active.clusters = cluster_temp; 6208 6209 extent_page_temp = blob1->active.extent_pages; 6210 blob1->active.extent_pages = blob2->active.extent_pages; 6211 blob2->active.extent_pages = extent_page_temp; 6212 } 6213 6214 /* Copies an internal xattr */ 6215 static int 6216 bs_snapshot_copy_xattr(struct spdk_blob *toblob, struct spdk_blob *fromblob, const char *name) 6217 { 6218 const void *val = NULL; 6219 size_t len; 6220 int bserrno; 6221 6222 bserrno = blob_get_xattr_value(fromblob, name, &val, &len, true); 6223 if (bserrno != 0) { 6224 SPDK_ERRLOG("blob 0x%" PRIx64 " missing %s xattr" 6225 BLOB_EXTERNAL_SNAPSHOT_ID " XATTR\n", fromblob->id, name); 6226 return bserrno; 6227 } 6228 6229 bserrno = blob_set_xattr(toblob, name, val, len, true); 6230 if (bserrno != 0) { 6231 SPDK_ERRLOG("could not set %s XATTR on blob 0x%" PRIx64 "\n", 6232 name, toblob->id); 6233 return bserrno; 6234 } 6235 return 0; 6236 } 6237 6238 static void 6239 bs_snapshot_origblob_sync_cpl(void *cb_arg, int bserrno) 6240 { 6241 struct spdk_clone_snapshot_ctx *ctx = (struct spdk_clone_snapshot_ctx *)cb_arg; 6242 struct spdk_blob *origblob = ctx->original.blob; 6243 struct spdk_blob *newblob = ctx->new.blob; 6244 6245 if (bserrno != 0) { 6246 bs_snapshot_swap_cluster_maps(newblob, origblob); 6247 if (blob_is_esnap_clone(newblob)) { 6248 bs_snapshot_copy_xattr(origblob, newblob, BLOB_EXTERNAL_SNAPSHOT_ID); 6249 origblob->invalid_flags |= SPDK_BLOB_EXTERNAL_SNAPSHOT; 6250 } 6251 bs_clone_snapshot_origblob_cleanup(ctx, bserrno); 6252 return; 6253 } 6254 6255 /* Remove metadata descriptor SNAPSHOT_IN_PROGRESS */ 6256 bserrno = blob_remove_xattr(newblob, SNAPSHOT_IN_PROGRESS, true); 6257 if (bserrno != 0) { 6258 bs_clone_snapshot_origblob_cleanup(ctx, bserrno); 6259 return; 6260 } 6261 6262 bs_blob_list_add(ctx->original.blob); 6263 6264 spdk_blob_set_read_only(newblob); 6265 6266 /* sync snapshot metadata */ 6267 spdk_blob_sync_md(newblob, bs_clone_snapshot_origblob_cleanup, ctx); 6268 } 6269 6270 static void 6271 bs_snapshot_newblob_sync_cpl(void *cb_arg, int bserrno) 6272 { 6273 struct spdk_clone_snapshot_ctx *ctx = (struct spdk_clone_snapshot_ctx *)cb_arg; 6274 struct spdk_blob *origblob = ctx->original.blob; 6275 struct spdk_blob *newblob = ctx->new.blob; 6276 6277 if (bserrno != 0) { 6278 /* return cluster map back to original */ 6279 bs_snapshot_swap_cluster_maps(newblob, origblob); 6280 6281 /* Newblob md sync failed. Valid clusters are only present in origblob. 6282 * Since I/O is frozen on origblob, not changes to zeroed out cluster map should have occurred. 6283 * Newblob needs to be reverted to thin_provisioned state at creation to properly close. */ 6284 blob_set_thin_provision(newblob); 6285 assert(spdk_mem_all_zero(newblob->active.clusters, 6286 newblob->active.num_clusters * sizeof(*newblob->active.clusters))); 6287 assert(spdk_mem_all_zero(newblob->active.extent_pages, 6288 newblob->active.num_extent_pages * sizeof(*newblob->active.extent_pages))); 6289 6290 bs_clone_snapshot_newblob_cleanup(ctx, bserrno); 6291 return; 6292 } 6293 6294 /* Set internal xattr for snapshot id */ 6295 bserrno = blob_set_xattr(origblob, BLOB_SNAPSHOT, &newblob->id, sizeof(spdk_blob_id), true); 6296 if (bserrno != 0) { 6297 /* return cluster map back to original */ 6298 bs_snapshot_swap_cluster_maps(newblob, origblob); 6299 blob_set_thin_provision(newblob); 6300 bs_clone_snapshot_newblob_cleanup(ctx, bserrno); 6301 return; 6302 } 6303 6304 /* Create new back_bs_dev for snapshot */ 6305 origblob->back_bs_dev = bs_create_blob_bs_dev(newblob); 6306 if (origblob->back_bs_dev == NULL) { 6307 /* return cluster map back to original */ 6308 bs_snapshot_swap_cluster_maps(newblob, origblob); 6309 blob_set_thin_provision(newblob); 6310 bs_clone_snapshot_newblob_cleanup(ctx, -EINVAL); 6311 return; 6312 } 6313 6314 /* Remove the xattr that references an external snapshot */ 6315 if (blob_is_esnap_clone(origblob)) { 6316 origblob->invalid_flags &= ~SPDK_BLOB_EXTERNAL_SNAPSHOT; 6317 bserrno = blob_remove_xattr(origblob, BLOB_EXTERNAL_SNAPSHOT_ID, true); 6318 if (bserrno != 0) { 6319 if (bserrno == -ENOENT) { 6320 SPDK_ERRLOG("blob 0x%" PRIx64 " has no " BLOB_EXTERNAL_SNAPSHOT_ID 6321 " xattr to remove\n", origblob->id); 6322 assert(false); 6323 } else { 6324 /* return cluster map back to original */ 6325 bs_snapshot_swap_cluster_maps(newblob, origblob); 6326 blob_set_thin_provision(newblob); 6327 bs_clone_snapshot_newblob_cleanup(ctx, bserrno); 6328 return; 6329 } 6330 } 6331 } 6332 6333 bs_blob_list_remove(origblob); 6334 origblob->parent_id = newblob->id; 6335 /* set clone blob as thin provisioned */ 6336 blob_set_thin_provision(origblob); 6337 6338 bs_blob_list_add(newblob); 6339 6340 /* sync clone metadata */ 6341 spdk_blob_sync_md(origblob, bs_snapshot_origblob_sync_cpl, ctx); 6342 } 6343 6344 static void 6345 bs_snapshot_freeze_cpl(void *cb_arg, int rc) 6346 { 6347 struct spdk_clone_snapshot_ctx *ctx = (struct spdk_clone_snapshot_ctx *)cb_arg; 6348 struct spdk_blob *origblob = ctx->original.blob; 6349 struct spdk_blob *newblob = ctx->new.blob; 6350 int bserrno; 6351 6352 if (rc != 0) { 6353 bs_clone_snapshot_newblob_cleanup(ctx, rc); 6354 return; 6355 } 6356 6357 ctx->frozen = true; 6358 6359 if (blob_is_esnap_clone(origblob)) { 6360 /* Clean up any channels associated with the original blob id because future IO will 6361 * perform IO using the snapshot blob_id. 6362 */ 6363 blob_esnap_destroy_bs_dev_channels(origblob, false, NULL, NULL); 6364 } 6365 if (newblob->back_bs_dev) { 6366 blob_back_bs_destroy(newblob); 6367 } 6368 /* set new back_bs_dev for snapshot */ 6369 newblob->back_bs_dev = origblob->back_bs_dev; 6370 /* Set invalid flags from origblob */ 6371 newblob->invalid_flags = origblob->invalid_flags; 6372 6373 /* inherit parent from original blob if set */ 6374 newblob->parent_id = origblob->parent_id; 6375 switch (origblob->parent_id) { 6376 case SPDK_BLOBID_EXTERNAL_SNAPSHOT: 6377 bserrno = bs_snapshot_copy_xattr(newblob, origblob, BLOB_EXTERNAL_SNAPSHOT_ID); 6378 if (bserrno != 0) { 6379 bs_clone_snapshot_newblob_cleanup(ctx, bserrno); 6380 return; 6381 } 6382 break; 6383 case SPDK_BLOBID_INVALID: 6384 break; 6385 default: 6386 /* Set internal xattr for snapshot id */ 6387 bserrno = blob_set_xattr(newblob, BLOB_SNAPSHOT, 6388 &origblob->parent_id, sizeof(spdk_blob_id), true); 6389 if (bserrno != 0) { 6390 bs_clone_snapshot_newblob_cleanup(ctx, bserrno); 6391 return; 6392 } 6393 } 6394 6395 /* swap cluster maps */ 6396 bs_snapshot_swap_cluster_maps(newblob, origblob); 6397 6398 /* Set the clear method on the new blob to match the original. */ 6399 blob_set_clear_method(newblob, origblob->clear_method); 6400 6401 /* sync snapshot metadata */ 6402 spdk_blob_sync_md(newblob, bs_snapshot_newblob_sync_cpl, ctx); 6403 } 6404 6405 static void 6406 bs_snapshot_newblob_open_cpl(void *cb_arg, struct spdk_blob *_blob, int bserrno) 6407 { 6408 struct spdk_clone_snapshot_ctx *ctx = (struct spdk_clone_snapshot_ctx *)cb_arg; 6409 struct spdk_blob *origblob = ctx->original.blob; 6410 struct spdk_blob *newblob = _blob; 6411 6412 if (bserrno != 0) { 6413 bs_clone_snapshot_origblob_cleanup(ctx, bserrno); 6414 return; 6415 } 6416 6417 ctx->new.blob = newblob; 6418 assert(spdk_blob_is_thin_provisioned(newblob)); 6419 assert(spdk_mem_all_zero(newblob->active.clusters, 6420 newblob->active.num_clusters * sizeof(*newblob->active.clusters))); 6421 assert(spdk_mem_all_zero(newblob->active.extent_pages, 6422 newblob->active.num_extent_pages * sizeof(*newblob->active.extent_pages))); 6423 6424 blob_freeze_io(origblob, bs_snapshot_freeze_cpl, ctx); 6425 } 6426 6427 static void 6428 bs_snapshot_newblob_create_cpl(void *cb_arg, spdk_blob_id blobid, int bserrno) 6429 { 6430 struct spdk_clone_snapshot_ctx *ctx = (struct spdk_clone_snapshot_ctx *)cb_arg; 6431 struct spdk_blob *origblob = ctx->original.blob; 6432 6433 if (bserrno != 0) { 6434 bs_clone_snapshot_origblob_cleanup(ctx, bserrno); 6435 return; 6436 } 6437 6438 ctx->new.id = blobid; 6439 ctx->cpl.u.blobid.blobid = blobid; 6440 6441 spdk_bs_open_blob(origblob->bs, ctx->new.id, bs_snapshot_newblob_open_cpl, ctx); 6442 } 6443 6444 6445 static void 6446 bs_xattr_snapshot(void *arg, const char *name, 6447 const void **value, size_t *value_len) 6448 { 6449 assert(strncmp(name, SNAPSHOT_IN_PROGRESS, sizeof(SNAPSHOT_IN_PROGRESS)) == 0); 6450 6451 struct spdk_blob *blob = (struct spdk_blob *)arg; 6452 *value = &blob->id; 6453 *value_len = sizeof(blob->id); 6454 } 6455 6456 static void 6457 bs_snapshot_origblob_open_cpl(void *cb_arg, struct spdk_blob *_blob, int bserrno) 6458 { 6459 struct spdk_clone_snapshot_ctx *ctx = (struct spdk_clone_snapshot_ctx *)cb_arg; 6460 struct spdk_blob_opts opts; 6461 struct spdk_blob_xattr_opts internal_xattrs; 6462 char *xattrs_names[] = { SNAPSHOT_IN_PROGRESS }; 6463 6464 if (bserrno != 0) { 6465 bs_clone_snapshot_cleanup_finish(ctx, bserrno); 6466 return; 6467 } 6468 6469 ctx->original.blob = _blob; 6470 6471 if (_blob->data_ro || _blob->md_ro) { 6472 SPDK_DEBUGLOG(blob, "Cannot create snapshot from read only blob with id %" PRIu64 "\n", 6473 _blob->id); 6474 ctx->bserrno = -EINVAL; 6475 spdk_blob_close(_blob, bs_clone_snapshot_cleanup_finish, ctx); 6476 return; 6477 } 6478 6479 if (_blob->locked_operation_in_progress) { 6480 SPDK_DEBUGLOG(blob, "Cannot create snapshot - another operation in progress\n"); 6481 ctx->bserrno = -EBUSY; 6482 spdk_blob_close(_blob, bs_clone_snapshot_cleanup_finish, ctx); 6483 return; 6484 } 6485 6486 _blob->locked_operation_in_progress = true; 6487 6488 spdk_blob_opts_init(&opts, sizeof(opts)); 6489 blob_xattrs_init(&internal_xattrs); 6490 6491 /* Change the size of new blob to the same as in original blob, 6492 * but do not allocate clusters */ 6493 opts.thin_provision = true; 6494 opts.num_clusters = spdk_blob_get_num_clusters(_blob); 6495 opts.use_extent_table = _blob->use_extent_table; 6496 6497 /* If there are any xattrs specified for snapshot, set them now */ 6498 if (ctx->xattrs) { 6499 memcpy(&opts.xattrs, ctx->xattrs, sizeof(*ctx->xattrs)); 6500 } 6501 /* Set internal xattr SNAPSHOT_IN_PROGRESS */ 6502 internal_xattrs.count = 1; 6503 internal_xattrs.ctx = _blob; 6504 internal_xattrs.names = xattrs_names; 6505 internal_xattrs.get_value = bs_xattr_snapshot; 6506 6507 bs_create_blob(_blob->bs, &opts, &internal_xattrs, 6508 bs_snapshot_newblob_create_cpl, ctx); 6509 } 6510 6511 void 6512 spdk_bs_create_snapshot(struct spdk_blob_store *bs, spdk_blob_id blobid, 6513 const struct spdk_blob_xattr_opts *snapshot_xattrs, 6514 spdk_blob_op_with_id_complete cb_fn, void *cb_arg) 6515 { 6516 struct spdk_clone_snapshot_ctx *ctx = calloc(1, sizeof(*ctx)); 6517 6518 if (!ctx) { 6519 cb_fn(cb_arg, SPDK_BLOBID_INVALID, -ENOMEM); 6520 return; 6521 } 6522 ctx->cpl.type = SPDK_BS_CPL_TYPE_BLOBID; 6523 ctx->cpl.u.blobid.cb_fn = cb_fn; 6524 ctx->cpl.u.blobid.cb_arg = cb_arg; 6525 ctx->cpl.u.blobid.blobid = SPDK_BLOBID_INVALID; 6526 ctx->bserrno = 0; 6527 ctx->frozen = false; 6528 ctx->original.id = blobid; 6529 ctx->xattrs = snapshot_xattrs; 6530 6531 spdk_bs_open_blob(bs, ctx->original.id, bs_snapshot_origblob_open_cpl, ctx); 6532 } 6533 /* END spdk_bs_create_snapshot */ 6534 6535 /* START spdk_bs_create_clone */ 6536 6537 static void 6538 bs_xattr_clone(void *arg, const char *name, 6539 const void **value, size_t *value_len) 6540 { 6541 assert(strncmp(name, BLOB_SNAPSHOT, sizeof(BLOB_SNAPSHOT)) == 0); 6542 6543 struct spdk_blob *blob = (struct spdk_blob *)arg; 6544 *value = &blob->id; 6545 *value_len = sizeof(blob->id); 6546 } 6547 6548 static void 6549 bs_clone_newblob_open_cpl(void *cb_arg, struct spdk_blob *_blob, int bserrno) 6550 { 6551 struct spdk_clone_snapshot_ctx *ctx = (struct spdk_clone_snapshot_ctx *)cb_arg; 6552 struct spdk_blob *clone = _blob; 6553 6554 ctx->new.blob = clone; 6555 bs_blob_list_add(clone); 6556 6557 spdk_blob_close(clone, bs_clone_snapshot_origblob_cleanup, ctx); 6558 } 6559 6560 static void 6561 bs_clone_newblob_create_cpl(void *cb_arg, spdk_blob_id blobid, int bserrno) 6562 { 6563 struct spdk_clone_snapshot_ctx *ctx = (struct spdk_clone_snapshot_ctx *)cb_arg; 6564 6565 ctx->cpl.u.blobid.blobid = blobid; 6566 spdk_bs_open_blob(ctx->original.blob->bs, blobid, bs_clone_newblob_open_cpl, ctx); 6567 } 6568 6569 static void 6570 bs_clone_origblob_open_cpl(void *cb_arg, struct spdk_blob *_blob, int bserrno) 6571 { 6572 struct spdk_clone_snapshot_ctx *ctx = (struct spdk_clone_snapshot_ctx *)cb_arg; 6573 struct spdk_blob_opts opts; 6574 struct spdk_blob_xattr_opts internal_xattrs; 6575 char *xattr_names[] = { BLOB_SNAPSHOT }; 6576 6577 if (bserrno != 0) { 6578 bs_clone_snapshot_cleanup_finish(ctx, bserrno); 6579 return; 6580 } 6581 6582 ctx->original.blob = _blob; 6583 ctx->original.md_ro = _blob->md_ro; 6584 6585 if (!_blob->data_ro || !_blob->md_ro) { 6586 SPDK_DEBUGLOG(blob, "Clone not from read-only blob\n"); 6587 ctx->bserrno = -EINVAL; 6588 spdk_blob_close(_blob, bs_clone_snapshot_cleanup_finish, ctx); 6589 return; 6590 } 6591 6592 if (_blob->locked_operation_in_progress) { 6593 SPDK_DEBUGLOG(blob, "Cannot create clone - another operation in progress\n"); 6594 ctx->bserrno = -EBUSY; 6595 spdk_blob_close(_blob, bs_clone_snapshot_cleanup_finish, ctx); 6596 return; 6597 } 6598 6599 _blob->locked_operation_in_progress = true; 6600 6601 spdk_blob_opts_init(&opts, sizeof(opts)); 6602 blob_xattrs_init(&internal_xattrs); 6603 6604 opts.thin_provision = true; 6605 opts.num_clusters = spdk_blob_get_num_clusters(_blob); 6606 opts.use_extent_table = _blob->use_extent_table; 6607 if (ctx->xattrs) { 6608 memcpy(&opts.xattrs, ctx->xattrs, sizeof(*ctx->xattrs)); 6609 } 6610 6611 /* Set internal xattr BLOB_SNAPSHOT */ 6612 internal_xattrs.count = 1; 6613 internal_xattrs.ctx = _blob; 6614 internal_xattrs.names = xattr_names; 6615 internal_xattrs.get_value = bs_xattr_clone; 6616 6617 bs_create_blob(_blob->bs, &opts, &internal_xattrs, 6618 bs_clone_newblob_create_cpl, ctx); 6619 } 6620 6621 void 6622 spdk_bs_create_clone(struct spdk_blob_store *bs, spdk_blob_id blobid, 6623 const struct spdk_blob_xattr_opts *clone_xattrs, 6624 spdk_blob_op_with_id_complete cb_fn, void *cb_arg) 6625 { 6626 struct spdk_clone_snapshot_ctx *ctx = calloc(1, sizeof(*ctx)); 6627 6628 if (!ctx) { 6629 cb_fn(cb_arg, SPDK_BLOBID_INVALID, -ENOMEM); 6630 return; 6631 } 6632 6633 ctx->cpl.type = SPDK_BS_CPL_TYPE_BLOBID; 6634 ctx->cpl.u.blobid.cb_fn = cb_fn; 6635 ctx->cpl.u.blobid.cb_arg = cb_arg; 6636 ctx->cpl.u.blobid.blobid = SPDK_BLOBID_INVALID; 6637 ctx->bserrno = 0; 6638 ctx->xattrs = clone_xattrs; 6639 ctx->original.id = blobid; 6640 6641 spdk_bs_open_blob(bs, ctx->original.id, bs_clone_origblob_open_cpl, ctx); 6642 } 6643 6644 /* END spdk_bs_create_clone */ 6645 6646 /* START spdk_bs_inflate_blob */ 6647 6648 static void 6649 bs_inflate_blob_set_parent_cpl(void *cb_arg, struct spdk_blob *_parent, int bserrno) 6650 { 6651 struct spdk_clone_snapshot_ctx *ctx = (struct spdk_clone_snapshot_ctx *)cb_arg; 6652 struct spdk_blob *_blob = ctx->original.blob; 6653 6654 if (bserrno != 0) { 6655 bs_clone_snapshot_origblob_cleanup(ctx, bserrno); 6656 return; 6657 } 6658 6659 /* Temporarily override md_ro flag for MD modification */ 6660 _blob->md_ro = false; 6661 6662 bserrno = blob_set_xattr(_blob, BLOB_SNAPSHOT, &_parent->id, sizeof(spdk_blob_id), true); 6663 if (bserrno != 0) { 6664 bs_clone_snapshot_origblob_cleanup(ctx, bserrno); 6665 return; 6666 } 6667 6668 assert(_parent != NULL); 6669 6670 bs_blob_list_remove(_blob); 6671 _blob->parent_id = _parent->id; 6672 6673 blob_back_bs_destroy(_blob); 6674 _blob->back_bs_dev = bs_create_blob_bs_dev(_parent); 6675 bs_blob_list_add(_blob); 6676 6677 spdk_blob_sync_md(_blob, bs_clone_snapshot_origblob_cleanup, ctx); 6678 } 6679 6680 static void 6681 bs_inflate_blob_done(struct spdk_clone_snapshot_ctx *ctx) 6682 { 6683 struct spdk_blob *_blob = ctx->original.blob; 6684 struct spdk_blob *_parent; 6685 6686 if (ctx->allocate_all) { 6687 /* remove thin provisioning */ 6688 bs_blob_list_remove(_blob); 6689 if (_blob->parent_id == SPDK_BLOBID_EXTERNAL_SNAPSHOT) { 6690 blob_remove_xattr(_blob, BLOB_EXTERNAL_SNAPSHOT_ID, true); 6691 _blob->invalid_flags &= ~SPDK_BLOB_EXTERNAL_SNAPSHOT; 6692 } else { 6693 blob_remove_xattr(_blob, BLOB_SNAPSHOT, true); 6694 } 6695 _blob->invalid_flags = _blob->invalid_flags & ~SPDK_BLOB_THIN_PROV; 6696 blob_back_bs_destroy(_blob); 6697 _blob->parent_id = SPDK_BLOBID_INVALID; 6698 } else { 6699 /* For now, esnap clones always have allocate_all set. */ 6700 assert(!blob_is_esnap_clone(_blob)); 6701 6702 _parent = ((struct spdk_blob_bs_dev *)(_blob->back_bs_dev))->blob; 6703 if (_parent->parent_id != SPDK_BLOBID_INVALID) { 6704 /* We must change the parent of the inflated blob */ 6705 spdk_bs_open_blob(_blob->bs, _parent->parent_id, 6706 bs_inflate_blob_set_parent_cpl, ctx); 6707 return; 6708 } 6709 6710 bs_blob_list_remove(_blob); 6711 _blob->parent_id = SPDK_BLOBID_INVALID; 6712 blob_back_bs_destroy(_blob); 6713 _blob->back_bs_dev = bs_create_zeroes_dev(); 6714 } 6715 6716 /* Temporarily override md_ro flag for MD modification */ 6717 _blob->md_ro = false; 6718 blob_remove_xattr(_blob, BLOB_SNAPSHOT, true); 6719 _blob->state = SPDK_BLOB_STATE_DIRTY; 6720 6721 spdk_blob_sync_md(_blob, bs_clone_snapshot_origblob_cleanup, ctx); 6722 } 6723 6724 /* Check if cluster needs allocation */ 6725 static inline bool 6726 bs_cluster_needs_allocation(struct spdk_blob *blob, uint64_t cluster, bool allocate_all) 6727 { 6728 struct spdk_blob_bs_dev *b; 6729 6730 assert(blob != NULL); 6731 6732 if (blob->active.clusters[cluster] != 0) { 6733 /* Cluster is already allocated */ 6734 return false; 6735 } 6736 6737 if (blob->parent_id == SPDK_BLOBID_INVALID) { 6738 /* Blob have no parent blob */ 6739 return allocate_all; 6740 } 6741 6742 if (blob->parent_id == SPDK_BLOBID_EXTERNAL_SNAPSHOT) { 6743 return true; 6744 } 6745 6746 b = (struct spdk_blob_bs_dev *)blob->back_bs_dev; 6747 return (allocate_all || b->blob->active.clusters[cluster] != 0); 6748 } 6749 6750 static void 6751 bs_inflate_blob_touch_next(void *cb_arg, int bserrno) 6752 { 6753 struct spdk_clone_snapshot_ctx *ctx = (struct spdk_clone_snapshot_ctx *)cb_arg; 6754 struct spdk_blob *_blob = ctx->original.blob; 6755 struct spdk_bs_cpl cpl; 6756 spdk_bs_user_op_t *op; 6757 uint64_t offset; 6758 6759 if (bserrno != 0) { 6760 bs_clone_snapshot_origblob_cleanup(ctx, bserrno); 6761 return; 6762 } 6763 6764 for (; ctx->cluster < _blob->active.num_clusters; ctx->cluster++) { 6765 if (bs_cluster_needs_allocation(_blob, ctx->cluster, ctx->allocate_all)) { 6766 break; 6767 } 6768 } 6769 6770 if (ctx->cluster < _blob->active.num_clusters) { 6771 offset = bs_cluster_to_lba(_blob->bs, ctx->cluster); 6772 6773 /* We may safely increment a cluster before copying */ 6774 ctx->cluster++; 6775 6776 /* Use a dummy 0B read as a context for cluster copy */ 6777 cpl.type = SPDK_BS_CPL_TYPE_BLOB_BASIC; 6778 cpl.u.blob_basic.cb_fn = bs_inflate_blob_touch_next; 6779 cpl.u.blob_basic.cb_arg = ctx; 6780 6781 op = bs_user_op_alloc(ctx->channel, &cpl, SPDK_BLOB_READ, _blob, 6782 NULL, 0, offset, 0); 6783 if (!op) { 6784 bs_clone_snapshot_origblob_cleanup(ctx, -ENOMEM); 6785 return; 6786 } 6787 6788 bs_allocate_and_copy_cluster(_blob, ctx->channel, offset, op); 6789 } else { 6790 bs_inflate_blob_done(ctx); 6791 } 6792 } 6793 6794 static void 6795 bs_inflate_blob_open_cpl(void *cb_arg, struct spdk_blob *_blob, int bserrno) 6796 { 6797 struct spdk_clone_snapshot_ctx *ctx = (struct spdk_clone_snapshot_ctx *)cb_arg; 6798 uint64_t clusters_needed; 6799 uint64_t i; 6800 6801 if (bserrno != 0) { 6802 bs_clone_snapshot_cleanup_finish(ctx, bserrno); 6803 return; 6804 } 6805 6806 ctx->original.blob = _blob; 6807 ctx->original.md_ro = _blob->md_ro; 6808 6809 if (_blob->locked_operation_in_progress) { 6810 SPDK_DEBUGLOG(blob, "Cannot inflate blob - another operation in progress\n"); 6811 ctx->bserrno = -EBUSY; 6812 spdk_blob_close(_blob, bs_clone_snapshot_cleanup_finish, ctx); 6813 return; 6814 } 6815 6816 _blob->locked_operation_in_progress = true; 6817 6818 switch (_blob->parent_id) { 6819 case SPDK_BLOBID_INVALID: 6820 if (!ctx->allocate_all) { 6821 /* This blob has no parent, so we cannot decouple it. */ 6822 SPDK_ERRLOG("Cannot decouple parent of blob with no parent.\n"); 6823 bs_clone_snapshot_origblob_cleanup(ctx, -EINVAL); 6824 return; 6825 } 6826 break; 6827 case SPDK_BLOBID_EXTERNAL_SNAPSHOT: 6828 /* 6829 * It would be better to rely on back_bs_dev->is_zeroes(), to determine which 6830 * clusters require allocation. Until there is a blobstore consumer that 6831 * uses esnaps with an spdk_bs_dev that implements a useful is_zeroes() it is not 6832 * worth the effort. 6833 */ 6834 ctx->allocate_all = true; 6835 break; 6836 default: 6837 break; 6838 } 6839 6840 if (spdk_blob_is_thin_provisioned(_blob) == false) { 6841 /* This is not thin provisioned blob. No need to inflate. */ 6842 bs_clone_snapshot_origblob_cleanup(ctx, 0); 6843 return; 6844 } 6845 6846 /* Do two passes - one to verify that we can obtain enough clusters 6847 * and another to actually claim them. 6848 */ 6849 clusters_needed = 0; 6850 for (i = 0; i < _blob->active.num_clusters; i++) { 6851 if (bs_cluster_needs_allocation(_blob, i, ctx->allocate_all)) { 6852 clusters_needed++; 6853 } 6854 } 6855 6856 if (clusters_needed > _blob->bs->num_free_clusters) { 6857 /* Not enough free clusters. Cannot satisfy the request. */ 6858 bs_clone_snapshot_origblob_cleanup(ctx, -ENOSPC); 6859 return; 6860 } 6861 6862 ctx->cluster = 0; 6863 bs_inflate_blob_touch_next(ctx, 0); 6864 } 6865 6866 static void 6867 bs_inflate_blob(struct spdk_blob_store *bs, struct spdk_io_channel *channel, 6868 spdk_blob_id blobid, bool allocate_all, spdk_blob_op_complete cb_fn, void *cb_arg) 6869 { 6870 struct spdk_clone_snapshot_ctx *ctx = calloc(1, sizeof(*ctx)); 6871 6872 if (!ctx) { 6873 cb_fn(cb_arg, -ENOMEM); 6874 return; 6875 } 6876 ctx->cpl.type = SPDK_BS_CPL_TYPE_BLOB_BASIC; 6877 ctx->cpl.u.bs_basic.cb_fn = cb_fn; 6878 ctx->cpl.u.bs_basic.cb_arg = cb_arg; 6879 ctx->bserrno = 0; 6880 ctx->original.id = blobid; 6881 ctx->channel = channel; 6882 ctx->allocate_all = allocate_all; 6883 6884 spdk_bs_open_blob(bs, ctx->original.id, bs_inflate_blob_open_cpl, ctx); 6885 } 6886 6887 void 6888 spdk_bs_inflate_blob(struct spdk_blob_store *bs, struct spdk_io_channel *channel, 6889 spdk_blob_id blobid, spdk_blob_op_complete cb_fn, void *cb_arg) 6890 { 6891 bs_inflate_blob(bs, channel, blobid, true, cb_fn, cb_arg); 6892 } 6893 6894 void 6895 spdk_bs_blob_decouple_parent(struct spdk_blob_store *bs, struct spdk_io_channel *channel, 6896 spdk_blob_id blobid, spdk_blob_op_complete cb_fn, void *cb_arg) 6897 { 6898 bs_inflate_blob(bs, channel, blobid, false, cb_fn, cb_arg); 6899 } 6900 /* END spdk_bs_inflate_blob */ 6901 6902 /* START spdk_blob_resize */ 6903 struct spdk_bs_resize_ctx { 6904 spdk_blob_op_complete cb_fn; 6905 void *cb_arg; 6906 struct spdk_blob *blob; 6907 uint64_t sz; 6908 int rc; 6909 }; 6910 6911 static void 6912 bs_resize_unfreeze_cpl(void *cb_arg, int rc) 6913 { 6914 struct spdk_bs_resize_ctx *ctx = (struct spdk_bs_resize_ctx *)cb_arg; 6915 6916 if (rc != 0) { 6917 SPDK_ERRLOG("Unfreeze failed, rc=%d\n", rc); 6918 } 6919 6920 if (ctx->rc != 0) { 6921 SPDK_ERRLOG("Unfreeze failed, ctx->rc=%d\n", ctx->rc); 6922 rc = ctx->rc; 6923 } 6924 6925 ctx->blob->locked_operation_in_progress = false; 6926 6927 ctx->cb_fn(ctx->cb_arg, rc); 6928 free(ctx); 6929 } 6930 6931 static void 6932 bs_resize_freeze_cpl(void *cb_arg, int rc) 6933 { 6934 struct spdk_bs_resize_ctx *ctx = (struct spdk_bs_resize_ctx *)cb_arg; 6935 6936 if (rc != 0) { 6937 ctx->blob->locked_operation_in_progress = false; 6938 ctx->cb_fn(ctx->cb_arg, rc); 6939 free(ctx); 6940 return; 6941 } 6942 6943 ctx->rc = blob_resize(ctx->blob, ctx->sz); 6944 6945 blob_unfreeze_io(ctx->blob, bs_resize_unfreeze_cpl, ctx); 6946 } 6947 6948 void 6949 spdk_blob_resize(struct spdk_blob *blob, uint64_t sz, spdk_blob_op_complete cb_fn, void *cb_arg) 6950 { 6951 struct spdk_bs_resize_ctx *ctx; 6952 6953 blob_verify_md_op(blob); 6954 6955 SPDK_DEBUGLOG(blob, "Resizing blob 0x%" PRIx64 " to %" PRIu64 " clusters\n", blob->id, sz); 6956 6957 if (blob->md_ro) { 6958 cb_fn(cb_arg, -EPERM); 6959 return; 6960 } 6961 6962 if (sz == blob->active.num_clusters) { 6963 cb_fn(cb_arg, 0); 6964 return; 6965 } 6966 6967 if (blob->locked_operation_in_progress) { 6968 cb_fn(cb_arg, -EBUSY); 6969 return; 6970 } 6971 6972 ctx = calloc(1, sizeof(*ctx)); 6973 if (!ctx) { 6974 cb_fn(cb_arg, -ENOMEM); 6975 return; 6976 } 6977 6978 blob->locked_operation_in_progress = true; 6979 ctx->cb_fn = cb_fn; 6980 ctx->cb_arg = cb_arg; 6981 ctx->blob = blob; 6982 ctx->sz = sz; 6983 blob_freeze_io(blob, bs_resize_freeze_cpl, ctx); 6984 } 6985 6986 /* END spdk_blob_resize */ 6987 6988 6989 /* START spdk_bs_delete_blob */ 6990 6991 static void 6992 bs_delete_close_cpl(void *cb_arg, int bserrno) 6993 { 6994 spdk_bs_sequence_t *seq = cb_arg; 6995 6996 bs_sequence_finish(seq, bserrno); 6997 } 6998 6999 static void 7000 bs_delete_persist_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 7001 { 7002 struct spdk_blob *blob = cb_arg; 7003 7004 if (bserrno != 0) { 7005 /* 7006 * We already removed this blob from the blobstore tailq, so 7007 * we need to free it here since this is the last reference 7008 * to it. 7009 */ 7010 blob_free(blob); 7011 bs_delete_close_cpl(seq, bserrno); 7012 return; 7013 } 7014 7015 /* 7016 * This will immediately decrement the ref_count and call 7017 * the completion routine since the metadata state is clean. 7018 * By calling spdk_blob_close, we reduce the number of call 7019 * points into code that touches the blob->open_ref count 7020 * and the blobstore's blob list. 7021 */ 7022 spdk_blob_close(blob, bs_delete_close_cpl, seq); 7023 } 7024 7025 struct delete_snapshot_ctx { 7026 struct spdk_blob_list *parent_snapshot_entry; 7027 struct spdk_blob *snapshot; 7028 struct spdk_blob_md_page *page; 7029 bool snapshot_md_ro; 7030 struct spdk_blob *clone; 7031 bool clone_md_ro; 7032 spdk_blob_op_with_handle_complete cb_fn; 7033 void *cb_arg; 7034 int bserrno; 7035 uint32_t next_extent_page; 7036 }; 7037 7038 static void 7039 delete_blob_cleanup_finish(void *cb_arg, int bserrno) 7040 { 7041 struct delete_snapshot_ctx *ctx = cb_arg; 7042 7043 if (bserrno != 0) { 7044 SPDK_ERRLOG("Snapshot cleanup error %d\n", bserrno); 7045 } 7046 7047 assert(ctx != NULL); 7048 7049 if (bserrno != 0 && ctx->bserrno == 0) { 7050 ctx->bserrno = bserrno; 7051 } 7052 7053 ctx->cb_fn(ctx->cb_arg, ctx->snapshot, ctx->bserrno); 7054 spdk_free(ctx->page); 7055 free(ctx); 7056 } 7057 7058 static void 7059 delete_snapshot_cleanup_snapshot(void *cb_arg, int bserrno) 7060 { 7061 struct delete_snapshot_ctx *ctx = cb_arg; 7062 7063 if (bserrno != 0) { 7064 ctx->bserrno = bserrno; 7065 SPDK_ERRLOG("Clone cleanup error %d\n", bserrno); 7066 } 7067 7068 if (ctx->bserrno != 0) { 7069 assert(blob_lookup(ctx->snapshot->bs, ctx->snapshot->id) == NULL); 7070 RB_INSERT(spdk_blob_tree, &ctx->snapshot->bs->open_blobs, ctx->snapshot); 7071 spdk_bit_array_set(ctx->snapshot->bs->open_blobids, ctx->snapshot->id); 7072 } 7073 7074 ctx->snapshot->locked_operation_in_progress = false; 7075 ctx->snapshot->md_ro = ctx->snapshot_md_ro; 7076 7077 spdk_blob_close(ctx->snapshot, delete_blob_cleanup_finish, ctx); 7078 } 7079 7080 static void 7081 delete_snapshot_cleanup_clone(void *cb_arg, int bserrno) 7082 { 7083 struct delete_snapshot_ctx *ctx = cb_arg; 7084 7085 ctx->clone->locked_operation_in_progress = false; 7086 ctx->clone->md_ro = ctx->clone_md_ro; 7087 7088 spdk_blob_close(ctx->clone, delete_snapshot_cleanup_snapshot, ctx); 7089 } 7090 7091 static void 7092 delete_snapshot_unfreeze_cpl(void *cb_arg, int bserrno) 7093 { 7094 struct delete_snapshot_ctx *ctx = cb_arg; 7095 7096 if (bserrno) { 7097 ctx->bserrno = bserrno; 7098 delete_snapshot_cleanup_clone(ctx, 0); 7099 return; 7100 } 7101 7102 ctx->clone->locked_operation_in_progress = false; 7103 spdk_blob_close(ctx->clone, delete_blob_cleanup_finish, ctx); 7104 } 7105 7106 static void 7107 delete_snapshot_sync_snapshot_cpl(void *cb_arg, int bserrno) 7108 { 7109 struct delete_snapshot_ctx *ctx = cb_arg; 7110 struct spdk_blob_list *parent_snapshot_entry = NULL; 7111 struct spdk_blob_list *snapshot_entry = NULL; 7112 struct spdk_blob_list *clone_entry = NULL; 7113 struct spdk_blob_list *snapshot_clone_entry = NULL; 7114 7115 if (bserrno) { 7116 SPDK_ERRLOG("Failed to sync MD on blob\n"); 7117 ctx->bserrno = bserrno; 7118 delete_snapshot_cleanup_clone(ctx, 0); 7119 return; 7120 } 7121 7122 /* Get snapshot entry for the snapshot we want to remove */ 7123 snapshot_entry = bs_get_snapshot_entry(ctx->snapshot->bs, ctx->snapshot->id); 7124 7125 assert(snapshot_entry != NULL); 7126 7127 /* Remove clone entry in this snapshot (at this point there can be only one clone) */ 7128 clone_entry = TAILQ_FIRST(&snapshot_entry->clones); 7129 assert(clone_entry != NULL); 7130 TAILQ_REMOVE(&snapshot_entry->clones, clone_entry, link); 7131 snapshot_entry->clone_count--; 7132 assert(TAILQ_EMPTY(&snapshot_entry->clones)); 7133 7134 switch (ctx->snapshot->parent_id) { 7135 case SPDK_BLOBID_INVALID: 7136 case SPDK_BLOBID_EXTERNAL_SNAPSHOT: 7137 /* No parent snapshot - just remove clone entry */ 7138 free(clone_entry); 7139 break; 7140 default: 7141 /* This snapshot is at the same time a clone of another snapshot - we need to 7142 * update parent snapshot (remove current clone, add new one inherited from 7143 * the snapshot that is being removed) */ 7144 7145 /* Get snapshot entry for parent snapshot and clone entry within that snapshot for 7146 * snapshot that we are removing */ 7147 blob_get_snapshot_and_clone_entries(ctx->snapshot, &parent_snapshot_entry, 7148 &snapshot_clone_entry); 7149 7150 /* Switch clone entry in parent snapshot */ 7151 TAILQ_INSERT_TAIL(&parent_snapshot_entry->clones, clone_entry, link); 7152 TAILQ_REMOVE(&parent_snapshot_entry->clones, snapshot_clone_entry, link); 7153 free(snapshot_clone_entry); 7154 } 7155 7156 /* Restore md_ro flags */ 7157 ctx->clone->md_ro = ctx->clone_md_ro; 7158 ctx->snapshot->md_ro = ctx->snapshot_md_ro; 7159 7160 blob_unfreeze_io(ctx->clone, delete_snapshot_unfreeze_cpl, ctx); 7161 } 7162 7163 static void 7164 delete_snapshot_sync_clone_cpl(void *cb_arg, int bserrno) 7165 { 7166 struct delete_snapshot_ctx *ctx = cb_arg; 7167 uint64_t i; 7168 7169 ctx->snapshot->md_ro = false; 7170 7171 if (bserrno) { 7172 SPDK_ERRLOG("Failed to sync MD on clone\n"); 7173 ctx->bserrno = bserrno; 7174 7175 /* Restore snapshot to previous state */ 7176 bserrno = blob_remove_xattr(ctx->snapshot, SNAPSHOT_PENDING_REMOVAL, true); 7177 if (bserrno != 0) { 7178 delete_snapshot_cleanup_clone(ctx, bserrno); 7179 return; 7180 } 7181 7182 spdk_blob_sync_md(ctx->snapshot, delete_snapshot_cleanup_clone, ctx); 7183 return; 7184 } 7185 7186 /* Clear cluster map entries for snapshot */ 7187 for (i = 0; i < ctx->snapshot->active.num_clusters && i < ctx->clone->active.num_clusters; i++) { 7188 if (ctx->clone->active.clusters[i] == ctx->snapshot->active.clusters[i]) { 7189 ctx->snapshot->active.clusters[i] = 0; 7190 } 7191 } 7192 for (i = 0; i < ctx->snapshot->active.num_extent_pages && 7193 i < ctx->clone->active.num_extent_pages; i++) { 7194 if (ctx->clone->active.extent_pages[i] == ctx->snapshot->active.extent_pages[i]) { 7195 ctx->snapshot->active.extent_pages[i] = 0; 7196 } 7197 } 7198 7199 blob_set_thin_provision(ctx->snapshot); 7200 ctx->snapshot->state = SPDK_BLOB_STATE_DIRTY; 7201 7202 if (ctx->parent_snapshot_entry != NULL) { 7203 ctx->snapshot->back_bs_dev = NULL; 7204 } 7205 7206 spdk_blob_sync_md(ctx->snapshot, delete_snapshot_sync_snapshot_cpl, ctx); 7207 } 7208 7209 static void 7210 delete_snapshot_update_extent_pages_cpl(struct delete_snapshot_ctx *ctx) 7211 { 7212 int bserrno; 7213 7214 /* Delete old backing bs_dev from clone (related to snapshot that will be removed) */ 7215 blob_back_bs_destroy(ctx->clone); 7216 7217 /* Set/remove snapshot xattr and switch parent ID and backing bs_dev on clone... */ 7218 if (ctx->snapshot->parent_id == SPDK_BLOBID_EXTERNAL_SNAPSHOT) { 7219 bserrno = bs_snapshot_copy_xattr(ctx->clone, ctx->snapshot, 7220 BLOB_EXTERNAL_SNAPSHOT_ID); 7221 if (bserrno != 0) { 7222 ctx->bserrno = bserrno; 7223 7224 /* Restore snapshot to previous state */ 7225 bserrno = blob_remove_xattr(ctx->snapshot, SNAPSHOT_PENDING_REMOVAL, true); 7226 if (bserrno != 0) { 7227 delete_snapshot_cleanup_clone(ctx, bserrno); 7228 return; 7229 } 7230 7231 spdk_blob_sync_md(ctx->snapshot, delete_snapshot_cleanup_clone, ctx); 7232 return; 7233 } 7234 ctx->clone->parent_id = SPDK_BLOBID_EXTERNAL_SNAPSHOT; 7235 ctx->clone->back_bs_dev = ctx->snapshot->back_bs_dev; 7236 /* Do not delete the external snapshot along with this snapshot */ 7237 ctx->snapshot->back_bs_dev = NULL; 7238 ctx->clone->invalid_flags |= SPDK_BLOB_EXTERNAL_SNAPSHOT; 7239 } else if (ctx->parent_snapshot_entry != NULL) { 7240 /* ...to parent snapshot */ 7241 ctx->clone->parent_id = ctx->parent_snapshot_entry->id; 7242 ctx->clone->back_bs_dev = ctx->snapshot->back_bs_dev; 7243 blob_set_xattr(ctx->clone, BLOB_SNAPSHOT, &ctx->parent_snapshot_entry->id, 7244 sizeof(spdk_blob_id), 7245 true); 7246 } else { 7247 /* ...to blobid invalid and zeroes dev */ 7248 ctx->clone->parent_id = SPDK_BLOBID_INVALID; 7249 ctx->clone->back_bs_dev = bs_create_zeroes_dev(); 7250 blob_remove_xattr(ctx->clone, BLOB_SNAPSHOT, true); 7251 } 7252 7253 spdk_blob_sync_md(ctx->clone, delete_snapshot_sync_clone_cpl, ctx); 7254 } 7255 7256 static void 7257 delete_snapshot_update_extent_pages(void *cb_arg, int bserrno) 7258 { 7259 struct delete_snapshot_ctx *ctx = cb_arg; 7260 uint32_t *extent_page; 7261 uint64_t i; 7262 7263 for (i = ctx->next_extent_page; i < ctx->snapshot->active.num_extent_pages && 7264 i < ctx->clone->active.num_extent_pages; i++) { 7265 if (ctx->snapshot->active.extent_pages[i] == 0) { 7266 /* No extent page to use from snapshot */ 7267 continue; 7268 } 7269 7270 extent_page = &ctx->clone->active.extent_pages[i]; 7271 if (*extent_page == 0) { 7272 /* Copy extent page from snapshot when clone did not have a matching one */ 7273 *extent_page = ctx->snapshot->active.extent_pages[i]; 7274 continue; 7275 } 7276 7277 /* Clone and snapshot both contain partially filled matching extent pages. 7278 * Update the clone extent page in place with cluster map containing the mix of both. */ 7279 ctx->next_extent_page = i + 1; 7280 memset(ctx->page, 0, SPDK_BS_PAGE_SIZE); 7281 7282 blob_write_extent_page(ctx->clone, *extent_page, i * SPDK_EXTENTS_PER_EP, ctx->page, 7283 delete_snapshot_update_extent_pages, ctx); 7284 return; 7285 } 7286 delete_snapshot_update_extent_pages_cpl(ctx); 7287 } 7288 7289 static void 7290 delete_snapshot_sync_snapshot_xattr_cpl(void *cb_arg, int bserrno) 7291 { 7292 struct delete_snapshot_ctx *ctx = cb_arg; 7293 uint64_t i; 7294 7295 /* Temporarily override md_ro flag for clone for MD modification */ 7296 ctx->clone_md_ro = ctx->clone->md_ro; 7297 ctx->clone->md_ro = false; 7298 7299 if (bserrno) { 7300 SPDK_ERRLOG("Failed to sync MD with xattr on blob\n"); 7301 ctx->bserrno = bserrno; 7302 delete_snapshot_cleanup_clone(ctx, 0); 7303 return; 7304 } 7305 7306 /* Copy snapshot map to clone map (only unallocated clusters in clone) */ 7307 for (i = 0; i < ctx->snapshot->active.num_clusters && i < ctx->clone->active.num_clusters; i++) { 7308 if (ctx->clone->active.clusters[i] == 0) { 7309 ctx->clone->active.clusters[i] = ctx->snapshot->active.clusters[i]; 7310 } 7311 } 7312 ctx->next_extent_page = 0; 7313 delete_snapshot_update_extent_pages(ctx, 0); 7314 } 7315 7316 static void 7317 delete_snapshot_esnap_channels_destroyed_cb(void *cb_arg, struct spdk_blob *blob, int bserrno) 7318 { 7319 struct delete_snapshot_ctx *ctx = cb_arg; 7320 7321 if (bserrno != 0) { 7322 SPDK_ERRLOG("blob 0x%" PRIx64 ": failed to destroy esnap channels: %d\n", 7323 blob->id, bserrno); 7324 /* That error should not stop us from syncing metadata. */ 7325 } 7326 7327 spdk_blob_sync_md(ctx->snapshot, delete_snapshot_sync_snapshot_xattr_cpl, ctx); 7328 } 7329 7330 static void 7331 delete_snapshot_freeze_io_cb(void *cb_arg, int bserrno) 7332 { 7333 struct delete_snapshot_ctx *ctx = cb_arg; 7334 7335 if (bserrno) { 7336 SPDK_ERRLOG("Failed to freeze I/O on clone\n"); 7337 ctx->bserrno = bserrno; 7338 delete_snapshot_cleanup_clone(ctx, 0); 7339 return; 7340 } 7341 7342 /* Temporarily override md_ro flag for snapshot for MD modification */ 7343 ctx->snapshot_md_ro = ctx->snapshot->md_ro; 7344 ctx->snapshot->md_ro = false; 7345 7346 /* Mark blob as pending for removal for power failure safety, use clone id for recovery */ 7347 ctx->bserrno = blob_set_xattr(ctx->snapshot, SNAPSHOT_PENDING_REMOVAL, &ctx->clone->id, 7348 sizeof(spdk_blob_id), true); 7349 if (ctx->bserrno != 0) { 7350 delete_snapshot_cleanup_clone(ctx, 0); 7351 return; 7352 } 7353 7354 if (blob_is_esnap_clone(ctx->snapshot)) { 7355 blob_esnap_destroy_bs_dev_channels(ctx->snapshot, false, 7356 delete_snapshot_esnap_channels_destroyed_cb, 7357 ctx); 7358 return; 7359 } 7360 7361 spdk_blob_sync_md(ctx->snapshot, delete_snapshot_sync_snapshot_xattr_cpl, ctx); 7362 } 7363 7364 static void 7365 delete_snapshot_open_clone_cb(void *cb_arg, struct spdk_blob *clone, int bserrno) 7366 { 7367 struct delete_snapshot_ctx *ctx = cb_arg; 7368 7369 if (bserrno) { 7370 SPDK_ERRLOG("Failed to open clone\n"); 7371 ctx->bserrno = bserrno; 7372 delete_snapshot_cleanup_snapshot(ctx, 0); 7373 return; 7374 } 7375 7376 ctx->clone = clone; 7377 7378 if (clone->locked_operation_in_progress) { 7379 SPDK_DEBUGLOG(blob, "Cannot remove blob - another operation in progress on its clone\n"); 7380 ctx->bserrno = -EBUSY; 7381 spdk_blob_close(ctx->clone, delete_snapshot_cleanup_snapshot, ctx); 7382 return; 7383 } 7384 7385 clone->locked_operation_in_progress = true; 7386 7387 blob_freeze_io(clone, delete_snapshot_freeze_io_cb, ctx); 7388 } 7389 7390 static void 7391 update_clone_on_snapshot_deletion(struct spdk_blob *snapshot, struct delete_snapshot_ctx *ctx) 7392 { 7393 struct spdk_blob_list *snapshot_entry = NULL; 7394 struct spdk_blob_list *clone_entry = NULL; 7395 struct spdk_blob_list *snapshot_clone_entry = NULL; 7396 7397 /* Get snapshot entry for the snapshot we want to remove */ 7398 snapshot_entry = bs_get_snapshot_entry(snapshot->bs, snapshot->id); 7399 7400 assert(snapshot_entry != NULL); 7401 7402 /* Get clone of the snapshot (at this point there can be only one clone) */ 7403 clone_entry = TAILQ_FIRST(&snapshot_entry->clones); 7404 assert(snapshot_entry->clone_count == 1); 7405 assert(clone_entry != NULL); 7406 7407 /* Get snapshot entry for parent snapshot and clone entry within that snapshot for 7408 * snapshot that we are removing */ 7409 blob_get_snapshot_and_clone_entries(snapshot, &ctx->parent_snapshot_entry, 7410 &snapshot_clone_entry); 7411 7412 spdk_bs_open_blob(snapshot->bs, clone_entry->id, delete_snapshot_open_clone_cb, ctx); 7413 } 7414 7415 static void 7416 bs_delete_blob_finish(void *cb_arg, struct spdk_blob *blob, int bserrno) 7417 { 7418 spdk_bs_sequence_t *seq = cb_arg; 7419 struct spdk_blob_list *snapshot_entry = NULL; 7420 uint32_t page_num; 7421 7422 if (bserrno) { 7423 SPDK_ERRLOG("Failed to remove blob\n"); 7424 bs_sequence_finish(seq, bserrno); 7425 return; 7426 } 7427 7428 /* Remove snapshot from the list */ 7429 snapshot_entry = bs_get_snapshot_entry(blob->bs, blob->id); 7430 if (snapshot_entry != NULL) { 7431 TAILQ_REMOVE(&blob->bs->snapshots, snapshot_entry, link); 7432 free(snapshot_entry); 7433 } 7434 7435 page_num = bs_blobid_to_page(blob->id); 7436 spdk_bit_array_clear(blob->bs->used_blobids, page_num); 7437 blob->state = SPDK_BLOB_STATE_DIRTY; 7438 blob->active.num_pages = 0; 7439 blob_resize(blob, 0); 7440 7441 blob_persist(seq, blob, bs_delete_persist_cpl, blob); 7442 } 7443 7444 static int 7445 bs_is_blob_deletable(struct spdk_blob *blob, bool *update_clone) 7446 { 7447 struct spdk_blob_list *snapshot_entry = NULL; 7448 struct spdk_blob_list *clone_entry = NULL; 7449 struct spdk_blob *clone = NULL; 7450 bool has_one_clone = false; 7451 7452 /* Check if this is a snapshot with clones */ 7453 snapshot_entry = bs_get_snapshot_entry(blob->bs, blob->id); 7454 if (snapshot_entry != NULL) { 7455 if (snapshot_entry->clone_count > 1) { 7456 SPDK_ERRLOG("Cannot remove snapshot with more than one clone\n"); 7457 return -EBUSY; 7458 } else if (snapshot_entry->clone_count == 1) { 7459 has_one_clone = true; 7460 } 7461 } 7462 7463 /* Check if someone has this blob open (besides this delete context): 7464 * - open_ref = 1 - only this context opened blob, so it is ok to remove it 7465 * - open_ref <= 2 && has_one_clone = true - clone is holding snapshot 7466 * and that is ok, because we will update it accordingly */ 7467 if (blob->open_ref <= 2 && has_one_clone) { 7468 clone_entry = TAILQ_FIRST(&snapshot_entry->clones); 7469 assert(clone_entry != NULL); 7470 clone = blob_lookup(blob->bs, clone_entry->id); 7471 7472 if (blob->open_ref == 2 && clone == NULL) { 7473 /* Clone is closed and someone else opened this blob */ 7474 SPDK_ERRLOG("Cannot remove snapshot because it is open\n"); 7475 return -EBUSY; 7476 } 7477 7478 *update_clone = true; 7479 return 0; 7480 } 7481 7482 if (blob->open_ref > 1) { 7483 SPDK_ERRLOG("Cannot remove snapshot because it is open\n"); 7484 return -EBUSY; 7485 } 7486 7487 assert(has_one_clone == false); 7488 *update_clone = false; 7489 return 0; 7490 } 7491 7492 static void 7493 bs_delete_enomem_close_cpl(void *cb_arg, int bserrno) 7494 { 7495 spdk_bs_sequence_t *seq = cb_arg; 7496 7497 bs_sequence_finish(seq, -ENOMEM); 7498 } 7499 7500 static void 7501 bs_delete_open_cpl(void *cb_arg, struct spdk_blob *blob, int bserrno) 7502 { 7503 spdk_bs_sequence_t *seq = cb_arg; 7504 struct delete_snapshot_ctx *ctx; 7505 bool update_clone = false; 7506 7507 if (bserrno != 0) { 7508 bs_sequence_finish(seq, bserrno); 7509 return; 7510 } 7511 7512 blob_verify_md_op(blob); 7513 7514 ctx = calloc(1, sizeof(*ctx)); 7515 if (ctx == NULL) { 7516 spdk_blob_close(blob, bs_delete_enomem_close_cpl, seq); 7517 return; 7518 } 7519 7520 ctx->snapshot = blob; 7521 ctx->cb_fn = bs_delete_blob_finish; 7522 ctx->cb_arg = seq; 7523 7524 /* Check if blob can be removed and if it is a snapshot with clone on top of it */ 7525 ctx->bserrno = bs_is_blob_deletable(blob, &update_clone); 7526 if (ctx->bserrno) { 7527 spdk_blob_close(blob, delete_blob_cleanup_finish, ctx); 7528 return; 7529 } 7530 7531 if (blob->locked_operation_in_progress) { 7532 SPDK_DEBUGLOG(blob, "Cannot remove blob - another operation in progress\n"); 7533 ctx->bserrno = -EBUSY; 7534 spdk_blob_close(blob, delete_blob_cleanup_finish, ctx); 7535 return; 7536 } 7537 7538 blob->locked_operation_in_progress = true; 7539 7540 /* 7541 * Remove the blob from the blob_store list now, to ensure it does not 7542 * get returned after this point by blob_lookup(). 7543 */ 7544 spdk_bit_array_clear(blob->bs->open_blobids, blob->id); 7545 RB_REMOVE(spdk_blob_tree, &blob->bs->open_blobs, blob); 7546 7547 if (update_clone) { 7548 ctx->page = spdk_zmalloc(SPDK_BS_PAGE_SIZE, 0, NULL, SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA); 7549 if (!ctx->page) { 7550 ctx->bserrno = -ENOMEM; 7551 spdk_blob_close(blob, delete_blob_cleanup_finish, ctx); 7552 return; 7553 } 7554 /* This blob is a snapshot with active clone - update clone first */ 7555 update_clone_on_snapshot_deletion(blob, ctx); 7556 } else { 7557 /* This blob does not have any clones - just remove it */ 7558 bs_blob_list_remove(blob); 7559 bs_delete_blob_finish(seq, blob, 0); 7560 free(ctx); 7561 } 7562 } 7563 7564 void 7565 spdk_bs_delete_blob(struct spdk_blob_store *bs, spdk_blob_id blobid, 7566 spdk_blob_op_complete cb_fn, void *cb_arg) 7567 { 7568 struct spdk_bs_cpl cpl; 7569 spdk_bs_sequence_t *seq; 7570 7571 SPDK_DEBUGLOG(blob, "Deleting blob 0x%" PRIx64 "\n", blobid); 7572 7573 assert(spdk_get_thread() == bs->md_thread); 7574 7575 cpl.type = SPDK_BS_CPL_TYPE_BLOB_BASIC; 7576 cpl.u.blob_basic.cb_fn = cb_fn; 7577 cpl.u.blob_basic.cb_arg = cb_arg; 7578 7579 seq = bs_sequence_start_bs(bs->md_channel, &cpl); 7580 if (!seq) { 7581 cb_fn(cb_arg, -ENOMEM); 7582 return; 7583 } 7584 7585 spdk_bs_open_blob(bs, blobid, bs_delete_open_cpl, seq); 7586 } 7587 7588 /* END spdk_bs_delete_blob */ 7589 7590 /* START spdk_bs_open_blob */ 7591 7592 static void 7593 bs_open_blob_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 7594 { 7595 struct spdk_blob *blob = cb_arg; 7596 struct spdk_blob *existing; 7597 7598 if (bserrno != 0) { 7599 blob_free(blob); 7600 seq->cpl.u.blob_handle.blob = NULL; 7601 bs_sequence_finish(seq, bserrno); 7602 return; 7603 } 7604 7605 existing = blob_lookup(blob->bs, blob->id); 7606 if (existing) { 7607 blob_free(blob); 7608 existing->open_ref++; 7609 seq->cpl.u.blob_handle.blob = existing; 7610 bs_sequence_finish(seq, 0); 7611 return; 7612 } 7613 7614 blob->open_ref++; 7615 7616 spdk_bit_array_set(blob->bs->open_blobids, blob->id); 7617 RB_INSERT(spdk_blob_tree, &blob->bs->open_blobs, blob); 7618 7619 bs_sequence_finish(seq, bserrno); 7620 } 7621 7622 static inline void 7623 blob_open_opts_copy(const struct spdk_blob_open_opts *src, struct spdk_blob_open_opts *dst) 7624 { 7625 #define FIELD_OK(field) \ 7626 offsetof(struct spdk_blob_open_opts, field) + sizeof(src->field) <= src->opts_size 7627 7628 #define SET_FIELD(field) \ 7629 if (FIELD_OK(field)) { \ 7630 dst->field = src->field; \ 7631 } \ 7632 7633 SET_FIELD(clear_method); 7634 SET_FIELD(esnap_ctx); 7635 7636 dst->opts_size = src->opts_size; 7637 7638 /* You should not remove this statement, but need to update the assert statement 7639 * if you add a new field, and also add a corresponding SET_FIELD statement */ 7640 SPDK_STATIC_ASSERT(sizeof(struct spdk_blob_open_opts) == 24, "Incorrect size"); 7641 7642 #undef FIELD_OK 7643 #undef SET_FIELD 7644 } 7645 7646 static void 7647 bs_open_blob(struct spdk_blob_store *bs, 7648 spdk_blob_id blobid, 7649 struct spdk_blob_open_opts *opts, 7650 spdk_blob_op_with_handle_complete cb_fn, 7651 void *cb_arg) 7652 { 7653 struct spdk_blob *blob; 7654 struct spdk_bs_cpl cpl; 7655 struct spdk_blob_open_opts opts_local; 7656 spdk_bs_sequence_t *seq; 7657 uint32_t page_num; 7658 7659 SPDK_DEBUGLOG(blob, "Opening blob 0x%" PRIx64 "\n", blobid); 7660 assert(spdk_get_thread() == bs->md_thread); 7661 7662 page_num = bs_blobid_to_page(blobid); 7663 if (spdk_bit_array_get(bs->used_blobids, page_num) == false) { 7664 /* Invalid blobid */ 7665 cb_fn(cb_arg, NULL, -ENOENT); 7666 return; 7667 } 7668 7669 blob = blob_lookup(bs, blobid); 7670 if (blob) { 7671 blob->open_ref++; 7672 cb_fn(cb_arg, blob, 0); 7673 return; 7674 } 7675 7676 blob = blob_alloc(bs, blobid); 7677 if (!blob) { 7678 cb_fn(cb_arg, NULL, -ENOMEM); 7679 return; 7680 } 7681 7682 spdk_blob_open_opts_init(&opts_local, sizeof(opts_local)); 7683 if (opts) { 7684 blob_open_opts_copy(opts, &opts_local); 7685 } 7686 7687 blob->clear_method = opts_local.clear_method; 7688 7689 cpl.type = SPDK_BS_CPL_TYPE_BLOB_HANDLE; 7690 cpl.u.blob_handle.cb_fn = cb_fn; 7691 cpl.u.blob_handle.cb_arg = cb_arg; 7692 cpl.u.blob_handle.blob = blob; 7693 cpl.u.blob_handle.esnap_ctx = opts_local.esnap_ctx; 7694 7695 seq = bs_sequence_start_bs(bs->md_channel, &cpl); 7696 if (!seq) { 7697 blob_free(blob); 7698 cb_fn(cb_arg, NULL, -ENOMEM); 7699 return; 7700 } 7701 7702 blob_load(seq, blob, bs_open_blob_cpl, blob); 7703 } 7704 7705 void 7706 spdk_bs_open_blob(struct spdk_blob_store *bs, spdk_blob_id blobid, 7707 spdk_blob_op_with_handle_complete cb_fn, void *cb_arg) 7708 { 7709 bs_open_blob(bs, blobid, NULL, cb_fn, cb_arg); 7710 } 7711 7712 void 7713 spdk_bs_open_blob_ext(struct spdk_blob_store *bs, spdk_blob_id blobid, 7714 struct spdk_blob_open_opts *opts, spdk_blob_op_with_handle_complete cb_fn, void *cb_arg) 7715 { 7716 bs_open_blob(bs, blobid, opts, cb_fn, cb_arg); 7717 } 7718 7719 /* END spdk_bs_open_blob */ 7720 7721 /* START spdk_blob_set_read_only */ 7722 int 7723 spdk_blob_set_read_only(struct spdk_blob *blob) 7724 { 7725 blob_verify_md_op(blob); 7726 7727 blob->data_ro_flags |= SPDK_BLOB_READ_ONLY; 7728 7729 blob->state = SPDK_BLOB_STATE_DIRTY; 7730 return 0; 7731 } 7732 /* END spdk_blob_set_read_only */ 7733 7734 /* START spdk_blob_sync_md */ 7735 7736 static void 7737 blob_sync_md_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 7738 { 7739 struct spdk_blob *blob = cb_arg; 7740 7741 if (bserrno == 0 && (blob->data_ro_flags & SPDK_BLOB_READ_ONLY)) { 7742 blob->data_ro = true; 7743 blob->md_ro = true; 7744 } 7745 7746 bs_sequence_finish(seq, bserrno); 7747 } 7748 7749 static void 7750 blob_sync_md(struct spdk_blob *blob, spdk_blob_op_complete cb_fn, void *cb_arg) 7751 { 7752 struct spdk_bs_cpl cpl; 7753 spdk_bs_sequence_t *seq; 7754 7755 cpl.type = SPDK_BS_CPL_TYPE_BLOB_BASIC; 7756 cpl.u.blob_basic.cb_fn = cb_fn; 7757 cpl.u.blob_basic.cb_arg = cb_arg; 7758 7759 seq = bs_sequence_start_bs(blob->bs->md_channel, &cpl); 7760 if (!seq) { 7761 cb_fn(cb_arg, -ENOMEM); 7762 return; 7763 } 7764 7765 blob_persist(seq, blob, blob_sync_md_cpl, blob); 7766 } 7767 7768 void 7769 spdk_blob_sync_md(struct spdk_blob *blob, spdk_blob_op_complete cb_fn, void *cb_arg) 7770 { 7771 blob_verify_md_op(blob); 7772 7773 SPDK_DEBUGLOG(blob, "Syncing blob 0x%" PRIx64 "\n", blob->id); 7774 7775 if (blob->md_ro) { 7776 assert(blob->state == SPDK_BLOB_STATE_CLEAN); 7777 cb_fn(cb_arg, 0); 7778 return; 7779 } 7780 7781 blob_sync_md(blob, cb_fn, cb_arg); 7782 } 7783 7784 /* END spdk_blob_sync_md */ 7785 7786 struct spdk_blob_insert_cluster_ctx { 7787 struct spdk_thread *thread; 7788 struct spdk_blob *blob; 7789 uint32_t cluster_num; /* cluster index in blob */ 7790 uint32_t cluster; /* cluster on disk */ 7791 uint32_t extent_page; /* extent page on disk */ 7792 struct spdk_blob_md_page *page; /* preallocated extent page */ 7793 int rc; 7794 spdk_blob_op_complete cb_fn; 7795 void *cb_arg; 7796 }; 7797 7798 static void 7799 blob_insert_cluster_msg_cpl(void *arg) 7800 { 7801 struct spdk_blob_insert_cluster_ctx *ctx = arg; 7802 7803 ctx->cb_fn(ctx->cb_arg, ctx->rc); 7804 free(ctx); 7805 } 7806 7807 static void 7808 blob_insert_cluster_msg_cb(void *arg, int bserrno) 7809 { 7810 struct spdk_blob_insert_cluster_ctx *ctx = arg; 7811 7812 ctx->rc = bserrno; 7813 spdk_thread_send_msg(ctx->thread, blob_insert_cluster_msg_cpl, ctx); 7814 } 7815 7816 static void 7817 blob_insert_new_ep_cb(void *arg, int bserrno) 7818 { 7819 struct spdk_blob_insert_cluster_ctx *ctx = arg; 7820 uint32_t *extent_page; 7821 7822 extent_page = bs_cluster_to_extent_page(ctx->blob, ctx->cluster_num); 7823 *extent_page = ctx->extent_page; 7824 ctx->blob->state = SPDK_BLOB_STATE_DIRTY; 7825 blob_sync_md(ctx->blob, blob_insert_cluster_msg_cb, ctx); 7826 } 7827 7828 struct spdk_blob_write_extent_page_ctx { 7829 struct spdk_blob_store *bs; 7830 7831 uint32_t extent; 7832 struct spdk_blob_md_page *page; 7833 }; 7834 7835 static void 7836 blob_persist_extent_page_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 7837 { 7838 struct spdk_blob_write_extent_page_ctx *ctx = cb_arg; 7839 7840 free(ctx); 7841 bs_sequence_finish(seq, bserrno); 7842 } 7843 7844 static void 7845 blob_write_extent_page_ready(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 7846 { 7847 struct spdk_blob_write_extent_page_ctx *ctx = cb_arg; 7848 7849 if (bserrno != 0) { 7850 blob_persist_extent_page_cpl(seq, ctx, bserrno); 7851 return; 7852 } 7853 bs_sequence_write_dev(seq, ctx->page, bs_md_page_to_lba(ctx->bs, ctx->extent), 7854 bs_byte_to_lba(ctx->bs, SPDK_BS_PAGE_SIZE), 7855 blob_persist_extent_page_cpl, ctx); 7856 } 7857 7858 static void 7859 blob_write_extent_page(struct spdk_blob *blob, uint32_t extent, uint64_t cluster_num, 7860 struct spdk_blob_md_page *page, spdk_blob_op_complete cb_fn, void *cb_arg) 7861 { 7862 struct spdk_blob_write_extent_page_ctx *ctx; 7863 spdk_bs_sequence_t *seq; 7864 struct spdk_bs_cpl cpl; 7865 7866 ctx = calloc(1, sizeof(*ctx)); 7867 if (!ctx) { 7868 cb_fn(cb_arg, -ENOMEM); 7869 return; 7870 } 7871 ctx->bs = blob->bs; 7872 ctx->extent = extent; 7873 ctx->page = page; 7874 7875 cpl.type = SPDK_BS_CPL_TYPE_BLOB_BASIC; 7876 cpl.u.blob_basic.cb_fn = cb_fn; 7877 cpl.u.blob_basic.cb_arg = cb_arg; 7878 7879 seq = bs_sequence_start_bs(blob->bs->md_channel, &cpl); 7880 if (!seq) { 7881 free(ctx); 7882 cb_fn(cb_arg, -ENOMEM); 7883 return; 7884 } 7885 7886 assert(page); 7887 page->next = SPDK_INVALID_MD_PAGE; 7888 page->id = blob->id; 7889 page->sequence_num = 0; 7890 7891 blob_serialize_extent_page(blob, cluster_num, page); 7892 7893 page->crc = blob_md_page_calc_crc(page); 7894 7895 assert(spdk_bit_array_get(blob->bs->used_md_pages, extent) == true); 7896 7897 bs_mark_dirty(seq, blob->bs, blob_write_extent_page_ready, ctx); 7898 } 7899 7900 static void 7901 blob_insert_cluster_msg(void *arg) 7902 { 7903 struct spdk_blob_insert_cluster_ctx *ctx = arg; 7904 uint32_t *extent_page; 7905 7906 ctx->rc = blob_insert_cluster(ctx->blob, ctx->cluster_num, ctx->cluster); 7907 if (ctx->rc != 0) { 7908 spdk_thread_send_msg(ctx->thread, blob_insert_cluster_msg_cpl, ctx); 7909 return; 7910 } 7911 7912 if (ctx->blob->use_extent_table == false) { 7913 /* Extent table is not used, proceed with sync of md that will only use extents_rle. */ 7914 ctx->blob->state = SPDK_BLOB_STATE_DIRTY; 7915 blob_sync_md(ctx->blob, blob_insert_cluster_msg_cb, ctx); 7916 return; 7917 } 7918 7919 extent_page = bs_cluster_to_extent_page(ctx->blob, ctx->cluster_num); 7920 if (*extent_page == 0) { 7921 /* Extent page requires allocation. 7922 * It was already claimed in the used_md_pages map and placed in ctx. */ 7923 assert(ctx->extent_page != 0); 7924 assert(spdk_bit_array_get(ctx->blob->bs->used_md_pages, ctx->extent_page) == true); 7925 blob_write_extent_page(ctx->blob, ctx->extent_page, ctx->cluster_num, ctx->page, 7926 blob_insert_new_ep_cb, ctx); 7927 } else { 7928 /* It is possible for original thread to allocate extent page for 7929 * different cluster in the same extent page. In such case proceed with 7930 * updating the existing extent page, but release the additional one. */ 7931 if (ctx->extent_page != 0) { 7932 spdk_spin_lock(&ctx->blob->bs->used_lock); 7933 assert(spdk_bit_array_get(ctx->blob->bs->used_md_pages, ctx->extent_page) == true); 7934 bs_release_md_page(ctx->blob->bs, ctx->extent_page); 7935 spdk_spin_unlock(&ctx->blob->bs->used_lock); 7936 ctx->extent_page = 0; 7937 } 7938 /* Extent page already allocated. 7939 * Every cluster allocation, requires just an update of single extent page. */ 7940 blob_write_extent_page(ctx->blob, *extent_page, ctx->cluster_num, ctx->page, 7941 blob_insert_cluster_msg_cb, ctx); 7942 } 7943 } 7944 7945 static void 7946 blob_insert_cluster_on_md_thread(struct spdk_blob *blob, uint32_t cluster_num, 7947 uint64_t cluster, uint32_t extent_page, struct spdk_blob_md_page *page, 7948 spdk_blob_op_complete cb_fn, void *cb_arg) 7949 { 7950 struct spdk_blob_insert_cluster_ctx *ctx; 7951 7952 ctx = calloc(1, sizeof(*ctx)); 7953 if (ctx == NULL) { 7954 cb_fn(cb_arg, -ENOMEM); 7955 return; 7956 } 7957 7958 ctx->thread = spdk_get_thread(); 7959 ctx->blob = blob; 7960 ctx->cluster_num = cluster_num; 7961 ctx->cluster = cluster; 7962 ctx->extent_page = extent_page; 7963 ctx->page = page; 7964 ctx->cb_fn = cb_fn; 7965 ctx->cb_arg = cb_arg; 7966 7967 spdk_thread_send_msg(blob->bs->md_thread, blob_insert_cluster_msg, ctx); 7968 } 7969 7970 /* START spdk_blob_close */ 7971 7972 static void 7973 blob_close_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 7974 { 7975 struct spdk_blob *blob = cb_arg; 7976 7977 if (bserrno == 0) { 7978 blob->open_ref--; 7979 if (blob->open_ref == 0) { 7980 /* 7981 * Blobs with active.num_pages == 0 are deleted blobs. 7982 * these blobs are removed from the blob_store list 7983 * when the deletion process starts - so don't try to 7984 * remove them again. 7985 */ 7986 if (blob->active.num_pages > 0) { 7987 spdk_bit_array_clear(blob->bs->open_blobids, blob->id); 7988 RB_REMOVE(spdk_blob_tree, &blob->bs->open_blobs, blob); 7989 } 7990 blob_free(blob); 7991 } 7992 } 7993 7994 bs_sequence_finish(seq, bserrno); 7995 } 7996 7997 static void 7998 blob_close_esnap_done(void *cb_arg, struct spdk_blob *blob, int bserrno) 7999 { 8000 spdk_bs_sequence_t *seq = cb_arg; 8001 8002 if (bserrno != 0) { 8003 SPDK_DEBUGLOG(blob_esnap, "blob 0x%" PRIx64 ": close failed with error %d\n", 8004 blob->id, bserrno); 8005 bs_sequence_finish(seq, bserrno); 8006 return; 8007 } 8008 8009 SPDK_DEBUGLOG(blob_esnap, "blob 0x%" PRIx64 ": closed, syncing metadata on thread %s\n", 8010 blob->id, spdk_thread_get_name(spdk_get_thread())); 8011 8012 /* Sync metadata */ 8013 blob_persist(seq, blob, blob_close_cpl, blob); 8014 } 8015 8016 void 8017 spdk_blob_close(struct spdk_blob *blob, spdk_blob_op_complete cb_fn, void *cb_arg) 8018 { 8019 struct spdk_bs_cpl cpl; 8020 spdk_bs_sequence_t *seq; 8021 8022 blob_verify_md_op(blob); 8023 8024 SPDK_DEBUGLOG(blob, "Closing blob 0x%" PRIx64 "\n", blob->id); 8025 8026 if (blob->open_ref == 0) { 8027 cb_fn(cb_arg, -EBADF); 8028 return; 8029 } 8030 8031 cpl.type = SPDK_BS_CPL_TYPE_BLOB_BASIC; 8032 cpl.u.blob_basic.cb_fn = cb_fn; 8033 cpl.u.blob_basic.cb_arg = cb_arg; 8034 8035 seq = bs_sequence_start_bs(blob->bs->md_channel, &cpl); 8036 if (!seq) { 8037 cb_fn(cb_arg, -ENOMEM); 8038 return; 8039 } 8040 8041 if (blob->open_ref == 1 && blob_is_esnap_clone(blob)) { 8042 blob_esnap_destroy_bs_dev_channels(blob, false, blob_close_esnap_done, seq); 8043 return; 8044 } 8045 8046 /* Sync metadata */ 8047 blob_persist(seq, blob, blob_close_cpl, blob); 8048 } 8049 8050 /* END spdk_blob_close */ 8051 8052 struct spdk_io_channel *spdk_bs_alloc_io_channel(struct spdk_blob_store *bs) 8053 { 8054 return spdk_get_io_channel(bs); 8055 } 8056 8057 void 8058 spdk_bs_free_io_channel(struct spdk_io_channel *channel) 8059 { 8060 blob_esnap_destroy_bs_channel(spdk_io_channel_get_ctx(channel)); 8061 spdk_put_io_channel(channel); 8062 } 8063 8064 void 8065 spdk_blob_io_unmap(struct spdk_blob *blob, struct spdk_io_channel *channel, 8066 uint64_t offset, uint64_t length, spdk_blob_op_complete cb_fn, void *cb_arg) 8067 { 8068 blob_request_submit_op(blob, channel, NULL, offset, length, cb_fn, cb_arg, 8069 SPDK_BLOB_UNMAP); 8070 } 8071 8072 void 8073 spdk_blob_io_write_zeroes(struct spdk_blob *blob, struct spdk_io_channel *channel, 8074 uint64_t offset, uint64_t length, spdk_blob_op_complete cb_fn, void *cb_arg) 8075 { 8076 blob_request_submit_op(blob, channel, NULL, offset, length, cb_fn, cb_arg, 8077 SPDK_BLOB_WRITE_ZEROES); 8078 } 8079 8080 void 8081 spdk_blob_io_write(struct spdk_blob *blob, struct spdk_io_channel *channel, 8082 void *payload, uint64_t offset, uint64_t length, 8083 spdk_blob_op_complete cb_fn, void *cb_arg) 8084 { 8085 blob_request_submit_op(blob, channel, payload, offset, length, cb_fn, cb_arg, 8086 SPDK_BLOB_WRITE); 8087 } 8088 8089 void 8090 spdk_blob_io_read(struct spdk_blob *blob, struct spdk_io_channel *channel, 8091 void *payload, uint64_t offset, uint64_t length, 8092 spdk_blob_op_complete cb_fn, void *cb_arg) 8093 { 8094 blob_request_submit_op(blob, channel, payload, offset, length, cb_fn, cb_arg, 8095 SPDK_BLOB_READ); 8096 } 8097 8098 void 8099 spdk_blob_io_writev(struct spdk_blob *blob, struct spdk_io_channel *channel, 8100 struct iovec *iov, int iovcnt, uint64_t offset, uint64_t length, 8101 spdk_blob_op_complete cb_fn, void *cb_arg) 8102 { 8103 blob_request_submit_rw_iov(blob, channel, iov, iovcnt, offset, length, cb_fn, cb_arg, false, NULL); 8104 } 8105 8106 void 8107 spdk_blob_io_readv(struct spdk_blob *blob, struct spdk_io_channel *channel, 8108 struct iovec *iov, int iovcnt, uint64_t offset, uint64_t length, 8109 spdk_blob_op_complete cb_fn, void *cb_arg) 8110 { 8111 blob_request_submit_rw_iov(blob, channel, iov, iovcnt, offset, length, cb_fn, cb_arg, true, NULL); 8112 } 8113 8114 void 8115 spdk_blob_io_writev_ext(struct spdk_blob *blob, struct spdk_io_channel *channel, 8116 struct iovec *iov, int iovcnt, uint64_t offset, uint64_t length, 8117 spdk_blob_op_complete cb_fn, void *cb_arg, struct spdk_blob_ext_io_opts *io_opts) 8118 { 8119 blob_request_submit_rw_iov(blob, channel, iov, iovcnt, offset, length, cb_fn, cb_arg, false, 8120 io_opts); 8121 } 8122 8123 void 8124 spdk_blob_io_readv_ext(struct spdk_blob *blob, struct spdk_io_channel *channel, 8125 struct iovec *iov, int iovcnt, uint64_t offset, uint64_t length, 8126 spdk_blob_op_complete cb_fn, void *cb_arg, struct spdk_blob_ext_io_opts *io_opts) 8127 { 8128 blob_request_submit_rw_iov(blob, channel, iov, iovcnt, offset, length, cb_fn, cb_arg, true, 8129 io_opts); 8130 } 8131 8132 struct spdk_bs_iter_ctx { 8133 int64_t page_num; 8134 struct spdk_blob_store *bs; 8135 8136 spdk_blob_op_with_handle_complete cb_fn; 8137 void *cb_arg; 8138 }; 8139 8140 static void 8141 bs_iter_cpl(void *cb_arg, struct spdk_blob *_blob, int bserrno) 8142 { 8143 struct spdk_bs_iter_ctx *ctx = cb_arg; 8144 struct spdk_blob_store *bs = ctx->bs; 8145 spdk_blob_id id; 8146 8147 if (bserrno == 0) { 8148 ctx->cb_fn(ctx->cb_arg, _blob, bserrno); 8149 free(ctx); 8150 return; 8151 } 8152 8153 ctx->page_num++; 8154 ctx->page_num = spdk_bit_array_find_first_set(bs->used_blobids, ctx->page_num); 8155 if (ctx->page_num >= spdk_bit_array_capacity(bs->used_blobids)) { 8156 ctx->cb_fn(ctx->cb_arg, NULL, -ENOENT); 8157 free(ctx); 8158 return; 8159 } 8160 8161 id = bs_page_to_blobid(ctx->page_num); 8162 8163 spdk_bs_open_blob(bs, id, bs_iter_cpl, ctx); 8164 } 8165 8166 void 8167 spdk_bs_iter_first(struct spdk_blob_store *bs, 8168 spdk_blob_op_with_handle_complete cb_fn, void *cb_arg) 8169 { 8170 struct spdk_bs_iter_ctx *ctx; 8171 8172 ctx = calloc(1, sizeof(*ctx)); 8173 if (!ctx) { 8174 cb_fn(cb_arg, NULL, -ENOMEM); 8175 return; 8176 } 8177 8178 ctx->page_num = -1; 8179 ctx->bs = bs; 8180 ctx->cb_fn = cb_fn; 8181 ctx->cb_arg = cb_arg; 8182 8183 bs_iter_cpl(ctx, NULL, -1); 8184 } 8185 8186 static void 8187 bs_iter_close_cpl(void *cb_arg, int bserrno) 8188 { 8189 struct spdk_bs_iter_ctx *ctx = cb_arg; 8190 8191 bs_iter_cpl(ctx, NULL, -1); 8192 } 8193 8194 void 8195 spdk_bs_iter_next(struct spdk_blob_store *bs, struct spdk_blob *blob, 8196 spdk_blob_op_with_handle_complete cb_fn, void *cb_arg) 8197 { 8198 struct spdk_bs_iter_ctx *ctx; 8199 8200 assert(blob != NULL); 8201 8202 ctx = calloc(1, sizeof(*ctx)); 8203 if (!ctx) { 8204 cb_fn(cb_arg, NULL, -ENOMEM); 8205 return; 8206 } 8207 8208 ctx->page_num = bs_blobid_to_page(blob->id); 8209 ctx->bs = bs; 8210 ctx->cb_fn = cb_fn; 8211 ctx->cb_arg = cb_arg; 8212 8213 /* Close the existing blob */ 8214 spdk_blob_close(blob, bs_iter_close_cpl, ctx); 8215 } 8216 8217 static int 8218 blob_set_xattr(struct spdk_blob *blob, const char *name, const void *value, 8219 uint16_t value_len, bool internal) 8220 { 8221 struct spdk_xattr_tailq *xattrs; 8222 struct spdk_xattr *xattr; 8223 size_t desc_size; 8224 void *tmp; 8225 8226 blob_verify_md_op(blob); 8227 8228 if (blob->md_ro) { 8229 return -EPERM; 8230 } 8231 8232 desc_size = sizeof(struct spdk_blob_md_descriptor_xattr) + strlen(name) + value_len; 8233 if (desc_size > SPDK_BS_MAX_DESC_SIZE) { 8234 SPDK_DEBUGLOG(blob, "Xattr '%s' of size %zu does not fix into single page %zu\n", name, 8235 desc_size, SPDK_BS_MAX_DESC_SIZE); 8236 return -ENOMEM; 8237 } 8238 8239 if (internal) { 8240 xattrs = &blob->xattrs_internal; 8241 blob->invalid_flags |= SPDK_BLOB_INTERNAL_XATTR; 8242 } else { 8243 xattrs = &blob->xattrs; 8244 } 8245 8246 TAILQ_FOREACH(xattr, xattrs, link) { 8247 if (!strcmp(name, xattr->name)) { 8248 tmp = malloc(value_len); 8249 if (!tmp) { 8250 return -ENOMEM; 8251 } 8252 8253 free(xattr->value); 8254 xattr->value_len = value_len; 8255 xattr->value = tmp; 8256 memcpy(xattr->value, value, value_len); 8257 8258 blob->state = SPDK_BLOB_STATE_DIRTY; 8259 8260 return 0; 8261 } 8262 } 8263 8264 xattr = calloc(1, sizeof(*xattr)); 8265 if (!xattr) { 8266 return -ENOMEM; 8267 } 8268 8269 xattr->name = strdup(name); 8270 if (!xattr->name) { 8271 free(xattr); 8272 return -ENOMEM; 8273 } 8274 8275 xattr->value_len = value_len; 8276 xattr->value = malloc(value_len); 8277 if (!xattr->value) { 8278 free(xattr->name); 8279 free(xattr); 8280 return -ENOMEM; 8281 } 8282 memcpy(xattr->value, value, value_len); 8283 TAILQ_INSERT_TAIL(xattrs, xattr, link); 8284 8285 blob->state = SPDK_BLOB_STATE_DIRTY; 8286 8287 return 0; 8288 } 8289 8290 int 8291 spdk_blob_set_xattr(struct spdk_blob *blob, const char *name, const void *value, 8292 uint16_t value_len) 8293 { 8294 return blob_set_xattr(blob, name, value, value_len, false); 8295 } 8296 8297 static int 8298 blob_remove_xattr(struct spdk_blob *blob, const char *name, bool internal) 8299 { 8300 struct spdk_xattr_tailq *xattrs; 8301 struct spdk_xattr *xattr; 8302 8303 blob_verify_md_op(blob); 8304 8305 if (blob->md_ro) { 8306 return -EPERM; 8307 } 8308 xattrs = internal ? &blob->xattrs_internal : &blob->xattrs; 8309 8310 TAILQ_FOREACH(xattr, xattrs, link) { 8311 if (!strcmp(name, xattr->name)) { 8312 TAILQ_REMOVE(xattrs, xattr, link); 8313 free(xattr->value); 8314 free(xattr->name); 8315 free(xattr); 8316 8317 if (internal && TAILQ_EMPTY(&blob->xattrs_internal)) { 8318 blob->invalid_flags &= ~SPDK_BLOB_INTERNAL_XATTR; 8319 } 8320 blob->state = SPDK_BLOB_STATE_DIRTY; 8321 8322 return 0; 8323 } 8324 } 8325 8326 return -ENOENT; 8327 } 8328 8329 int 8330 spdk_blob_remove_xattr(struct spdk_blob *blob, const char *name) 8331 { 8332 return blob_remove_xattr(blob, name, false); 8333 } 8334 8335 static int 8336 blob_get_xattr_value(struct spdk_blob *blob, const char *name, 8337 const void **value, size_t *value_len, bool internal) 8338 { 8339 struct spdk_xattr *xattr; 8340 struct spdk_xattr_tailq *xattrs; 8341 8342 xattrs = internal ? &blob->xattrs_internal : &blob->xattrs; 8343 8344 TAILQ_FOREACH(xattr, xattrs, link) { 8345 if (!strcmp(name, xattr->name)) { 8346 *value = xattr->value; 8347 *value_len = xattr->value_len; 8348 return 0; 8349 } 8350 } 8351 return -ENOENT; 8352 } 8353 8354 int 8355 spdk_blob_get_xattr_value(struct spdk_blob *blob, const char *name, 8356 const void **value, size_t *value_len) 8357 { 8358 blob_verify_md_op(blob); 8359 8360 return blob_get_xattr_value(blob, name, value, value_len, false); 8361 } 8362 8363 struct spdk_xattr_names { 8364 uint32_t count; 8365 const char *names[0]; 8366 }; 8367 8368 static int 8369 blob_get_xattr_names(struct spdk_xattr_tailq *xattrs, struct spdk_xattr_names **names) 8370 { 8371 struct spdk_xattr *xattr; 8372 int count = 0; 8373 8374 TAILQ_FOREACH(xattr, xattrs, link) { 8375 count++; 8376 } 8377 8378 *names = calloc(1, sizeof(struct spdk_xattr_names) + count * sizeof(char *)); 8379 if (*names == NULL) { 8380 return -ENOMEM; 8381 } 8382 8383 TAILQ_FOREACH(xattr, xattrs, link) { 8384 (*names)->names[(*names)->count++] = xattr->name; 8385 } 8386 8387 return 0; 8388 } 8389 8390 int 8391 spdk_blob_get_xattr_names(struct spdk_blob *blob, struct spdk_xattr_names **names) 8392 { 8393 blob_verify_md_op(blob); 8394 8395 return blob_get_xattr_names(&blob->xattrs, names); 8396 } 8397 8398 uint32_t 8399 spdk_xattr_names_get_count(struct spdk_xattr_names *names) 8400 { 8401 assert(names != NULL); 8402 8403 return names->count; 8404 } 8405 8406 const char * 8407 spdk_xattr_names_get_name(struct spdk_xattr_names *names, uint32_t index) 8408 { 8409 if (index >= names->count) { 8410 return NULL; 8411 } 8412 8413 return names->names[index]; 8414 } 8415 8416 void 8417 spdk_xattr_names_free(struct spdk_xattr_names *names) 8418 { 8419 free(names); 8420 } 8421 8422 struct spdk_bs_type 8423 spdk_bs_get_bstype(struct spdk_blob_store *bs) 8424 { 8425 return bs->bstype; 8426 } 8427 8428 void 8429 spdk_bs_set_bstype(struct spdk_blob_store *bs, struct spdk_bs_type bstype) 8430 { 8431 memcpy(&bs->bstype, &bstype, sizeof(bstype)); 8432 } 8433 8434 bool 8435 spdk_blob_is_read_only(struct spdk_blob *blob) 8436 { 8437 assert(blob != NULL); 8438 return (blob->data_ro || blob->md_ro); 8439 } 8440 8441 bool 8442 spdk_blob_is_snapshot(struct spdk_blob *blob) 8443 { 8444 struct spdk_blob_list *snapshot_entry; 8445 8446 assert(blob != NULL); 8447 8448 snapshot_entry = bs_get_snapshot_entry(blob->bs, blob->id); 8449 if (snapshot_entry == NULL) { 8450 return false; 8451 } 8452 8453 return true; 8454 } 8455 8456 bool 8457 spdk_blob_is_clone(struct spdk_blob *blob) 8458 { 8459 assert(blob != NULL); 8460 8461 if (blob->parent_id != SPDK_BLOBID_INVALID) { 8462 assert(spdk_blob_is_thin_provisioned(blob)); 8463 return true; 8464 } 8465 8466 return false; 8467 } 8468 8469 bool 8470 spdk_blob_is_thin_provisioned(struct spdk_blob *blob) 8471 { 8472 assert(blob != NULL); 8473 return !!(blob->invalid_flags & SPDK_BLOB_THIN_PROV); 8474 } 8475 8476 bool 8477 spdk_blob_is_esnap_clone(const struct spdk_blob *blob) 8478 { 8479 return blob_is_esnap_clone(blob); 8480 } 8481 8482 static void 8483 blob_update_clear_method(struct spdk_blob *blob) 8484 { 8485 enum blob_clear_method stored_cm; 8486 8487 assert(blob != NULL); 8488 8489 /* If BLOB_CLEAR_WITH_DEFAULT was passed in, use the setting stored 8490 * in metadata previously. If something other than the default was 8491 * specified, ignore stored value and used what was passed in. 8492 */ 8493 stored_cm = ((blob->md_ro_flags & SPDK_BLOB_CLEAR_METHOD) >> SPDK_BLOB_CLEAR_METHOD_SHIFT); 8494 8495 if (blob->clear_method == BLOB_CLEAR_WITH_DEFAULT) { 8496 blob->clear_method = stored_cm; 8497 } else if (blob->clear_method != stored_cm) { 8498 SPDK_WARNLOG("Using passed in clear method 0x%x instead of stored value of 0x%x\n", 8499 blob->clear_method, stored_cm); 8500 } 8501 } 8502 8503 spdk_blob_id 8504 spdk_blob_get_parent_snapshot(struct spdk_blob_store *bs, spdk_blob_id blob_id) 8505 { 8506 struct spdk_blob_list *snapshot_entry = NULL; 8507 struct spdk_blob_list *clone_entry = NULL; 8508 8509 TAILQ_FOREACH(snapshot_entry, &bs->snapshots, link) { 8510 TAILQ_FOREACH(clone_entry, &snapshot_entry->clones, link) { 8511 if (clone_entry->id == blob_id) { 8512 return snapshot_entry->id; 8513 } 8514 } 8515 } 8516 8517 return SPDK_BLOBID_INVALID; 8518 } 8519 8520 int 8521 spdk_blob_get_clones(struct spdk_blob_store *bs, spdk_blob_id blobid, spdk_blob_id *ids, 8522 size_t *count) 8523 { 8524 struct spdk_blob_list *snapshot_entry, *clone_entry; 8525 size_t n; 8526 8527 snapshot_entry = bs_get_snapshot_entry(bs, blobid); 8528 if (snapshot_entry == NULL) { 8529 *count = 0; 8530 return 0; 8531 } 8532 8533 if (ids == NULL || *count < snapshot_entry->clone_count) { 8534 *count = snapshot_entry->clone_count; 8535 return -ENOMEM; 8536 } 8537 *count = snapshot_entry->clone_count; 8538 8539 n = 0; 8540 TAILQ_FOREACH(clone_entry, &snapshot_entry->clones, link) { 8541 ids[n++] = clone_entry->id; 8542 } 8543 8544 return 0; 8545 } 8546 8547 static void 8548 bs_load_grow_continue(struct spdk_bs_load_ctx *ctx) 8549 { 8550 int rc; 8551 8552 if (ctx->super->size == 0) { 8553 ctx->super->size = ctx->bs->dev->blockcnt * ctx->bs->dev->blocklen; 8554 } 8555 8556 if (ctx->super->io_unit_size == 0) { 8557 ctx->super->io_unit_size = SPDK_BS_PAGE_SIZE; 8558 } 8559 8560 /* Parse the super block */ 8561 ctx->bs->clean = 1; 8562 ctx->bs->cluster_sz = ctx->super->cluster_size; 8563 ctx->bs->total_clusters = ctx->super->size / ctx->super->cluster_size; 8564 ctx->bs->pages_per_cluster = ctx->bs->cluster_sz / SPDK_BS_PAGE_SIZE; 8565 if (spdk_u32_is_pow2(ctx->bs->pages_per_cluster)) { 8566 ctx->bs->pages_per_cluster_shift = spdk_u32log2(ctx->bs->pages_per_cluster); 8567 } 8568 ctx->bs->io_unit_size = ctx->super->io_unit_size; 8569 rc = spdk_bit_array_resize(&ctx->used_clusters, ctx->bs->total_clusters); 8570 if (rc < 0) { 8571 bs_load_ctx_fail(ctx, -ENOMEM); 8572 return; 8573 } 8574 ctx->bs->md_start = ctx->super->md_start; 8575 ctx->bs->md_len = ctx->super->md_len; 8576 rc = spdk_bit_array_resize(&ctx->bs->open_blobids, ctx->bs->md_len); 8577 if (rc < 0) { 8578 bs_load_ctx_fail(ctx, -ENOMEM); 8579 return; 8580 } 8581 8582 ctx->bs->total_data_clusters = ctx->bs->total_clusters - spdk_divide_round_up( 8583 ctx->bs->md_start + ctx->bs->md_len, ctx->bs->pages_per_cluster); 8584 ctx->bs->super_blob = ctx->super->super_blob; 8585 memcpy(&ctx->bs->bstype, &ctx->super->bstype, sizeof(ctx->super->bstype)); 8586 8587 if (ctx->super->used_blobid_mask_len == 0 || ctx->super->clean == 0) { 8588 SPDK_ERRLOG("Can not grow an unclean blobstore, please load it normally to clean it.\n"); 8589 bs_load_ctx_fail(ctx, -EIO); 8590 return; 8591 } else { 8592 bs_load_read_used_pages(ctx); 8593 } 8594 } 8595 8596 static void 8597 bs_load_grow_super_write_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 8598 { 8599 struct spdk_bs_load_ctx *ctx = cb_arg; 8600 8601 if (bserrno != 0) { 8602 bs_load_ctx_fail(ctx, bserrno); 8603 return; 8604 } 8605 bs_load_grow_continue(ctx); 8606 } 8607 8608 static void 8609 bs_load_grow_used_clusters_write_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 8610 { 8611 struct spdk_bs_load_ctx *ctx = cb_arg; 8612 8613 if (bserrno != 0) { 8614 bs_load_ctx_fail(ctx, bserrno); 8615 return; 8616 } 8617 8618 spdk_free(ctx->mask); 8619 8620 bs_sequence_write_dev(ctx->seq, ctx->super, bs_page_to_lba(ctx->bs, 0), 8621 bs_byte_to_lba(ctx->bs, sizeof(*ctx->super)), 8622 bs_load_grow_super_write_cpl, ctx); 8623 } 8624 8625 static void 8626 bs_load_grow_used_clusters_read_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 8627 { 8628 struct spdk_bs_load_ctx *ctx = cb_arg; 8629 uint64_t lba, lba_count; 8630 uint64_t dev_size; 8631 uint64_t total_clusters; 8632 8633 if (bserrno != 0) { 8634 bs_load_ctx_fail(ctx, bserrno); 8635 return; 8636 } 8637 8638 /* The type must be correct */ 8639 assert(ctx->mask->type == SPDK_MD_MASK_TYPE_USED_CLUSTERS); 8640 /* The length of the mask (in bits) must not be greater than the length of the buffer (converted to bits) */ 8641 assert(ctx->mask->length <= (ctx->super->used_cluster_mask_len * sizeof( 8642 struct spdk_blob_md_page) * 8)); 8643 dev_size = ctx->bs->dev->blockcnt * ctx->bs->dev->blocklen; 8644 total_clusters = dev_size / ctx->super->cluster_size; 8645 ctx->mask->length = total_clusters; 8646 8647 lba = bs_page_to_lba(ctx->bs, ctx->super->used_cluster_mask_start); 8648 lba_count = bs_page_to_lba(ctx->bs, ctx->super->used_cluster_mask_len); 8649 bs_sequence_write_dev(ctx->seq, ctx->mask, lba, lba_count, 8650 bs_load_grow_used_clusters_write_cpl, ctx); 8651 } 8652 8653 static void 8654 bs_load_try_to_grow(struct spdk_bs_load_ctx *ctx) 8655 { 8656 uint64_t dev_size, total_clusters, used_cluster_mask_len, max_used_cluster_mask; 8657 uint64_t lba, lba_count, mask_size; 8658 8659 dev_size = ctx->bs->dev->blockcnt * ctx->bs->dev->blocklen; 8660 total_clusters = dev_size / ctx->super->cluster_size; 8661 used_cluster_mask_len = spdk_divide_round_up(sizeof(struct spdk_bs_md_mask) + 8662 spdk_divide_round_up(total_clusters, 8), 8663 SPDK_BS_PAGE_SIZE); 8664 max_used_cluster_mask = ctx->super->used_blobid_mask_start - ctx->super->used_cluster_mask_start; 8665 /* No necessary to grow or no space to grow */ 8666 if (ctx->super->size >= dev_size || used_cluster_mask_len > max_used_cluster_mask) { 8667 SPDK_DEBUGLOG(blob, "No grow\n"); 8668 bs_load_grow_continue(ctx); 8669 return; 8670 } 8671 8672 SPDK_DEBUGLOG(blob, "Resize blobstore\n"); 8673 8674 ctx->super->size = dev_size; 8675 ctx->super->used_cluster_mask_len = used_cluster_mask_len; 8676 ctx->super->crc = blob_md_page_calc_crc(ctx->super); 8677 8678 mask_size = used_cluster_mask_len * SPDK_BS_PAGE_SIZE; 8679 ctx->mask = spdk_zmalloc(mask_size, 0x1000, NULL, SPDK_ENV_SOCKET_ID_ANY, 8680 SPDK_MALLOC_DMA); 8681 if (!ctx->mask) { 8682 bs_load_ctx_fail(ctx, -ENOMEM); 8683 return; 8684 } 8685 lba = bs_page_to_lba(ctx->bs, ctx->super->used_cluster_mask_start); 8686 lba_count = bs_page_to_lba(ctx->bs, ctx->super->used_cluster_mask_len); 8687 bs_sequence_read_dev(ctx->seq, ctx->mask, lba, lba_count, 8688 bs_load_grow_used_clusters_read_cpl, ctx); 8689 } 8690 8691 static void 8692 bs_grow_load_super_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 8693 { 8694 struct spdk_bs_load_ctx *ctx = cb_arg; 8695 uint32_t crc; 8696 static const char zeros[SPDK_BLOBSTORE_TYPE_LENGTH]; 8697 8698 if (ctx->super->version > SPDK_BS_VERSION || 8699 ctx->super->version < SPDK_BS_INITIAL_VERSION) { 8700 bs_load_ctx_fail(ctx, -EILSEQ); 8701 return; 8702 } 8703 8704 if (memcmp(ctx->super->signature, SPDK_BS_SUPER_BLOCK_SIG, 8705 sizeof(ctx->super->signature)) != 0) { 8706 bs_load_ctx_fail(ctx, -EILSEQ); 8707 return; 8708 } 8709 8710 crc = blob_md_page_calc_crc(ctx->super); 8711 if (crc != ctx->super->crc) { 8712 bs_load_ctx_fail(ctx, -EILSEQ); 8713 return; 8714 } 8715 8716 if (memcmp(&ctx->bs->bstype, &ctx->super->bstype, SPDK_BLOBSTORE_TYPE_LENGTH) == 0) { 8717 SPDK_DEBUGLOG(blob, "Bstype matched - loading blobstore\n"); 8718 } else if (memcmp(&ctx->bs->bstype, zeros, SPDK_BLOBSTORE_TYPE_LENGTH) == 0) { 8719 SPDK_DEBUGLOG(blob, "Bstype wildcard used - loading blobstore regardless bstype\n"); 8720 } else { 8721 SPDK_DEBUGLOG(blob, "Unexpected bstype\n"); 8722 SPDK_LOGDUMP(blob, "Expected:", ctx->bs->bstype.bstype, SPDK_BLOBSTORE_TYPE_LENGTH); 8723 SPDK_LOGDUMP(blob, "Found:", ctx->super->bstype.bstype, SPDK_BLOBSTORE_TYPE_LENGTH); 8724 bs_load_ctx_fail(ctx, -ENXIO); 8725 return; 8726 } 8727 8728 if (ctx->super->size > ctx->bs->dev->blockcnt * ctx->bs->dev->blocklen) { 8729 SPDK_NOTICELOG("Size mismatch, dev size: %" PRIu64 ", blobstore size: %" PRIu64 "\n", 8730 ctx->bs->dev->blockcnt * ctx->bs->dev->blocklen, ctx->super->size); 8731 bs_load_ctx_fail(ctx, -EILSEQ); 8732 return; 8733 } 8734 8735 bs_load_try_to_grow(ctx); 8736 8737 } 8738 8739 void 8740 spdk_bs_grow(struct spdk_bs_dev *dev, struct spdk_bs_opts *o, 8741 spdk_bs_op_with_handle_complete cb_fn, void *cb_arg) 8742 { 8743 struct spdk_blob_store *bs; 8744 struct spdk_bs_cpl cpl; 8745 struct spdk_bs_load_ctx *ctx; 8746 struct spdk_bs_opts opts = {}; 8747 int err; 8748 8749 SPDK_DEBUGLOG(blob, "Loading blobstore from dev %p\n", dev); 8750 8751 if ((SPDK_BS_PAGE_SIZE % dev->blocklen) != 0) { 8752 SPDK_DEBUGLOG(blob, "unsupported dev block length of %d\n", dev->blocklen); 8753 dev->destroy(dev); 8754 cb_fn(cb_arg, NULL, -EINVAL); 8755 return; 8756 } 8757 8758 spdk_bs_opts_init(&opts, sizeof(opts)); 8759 if (o) { 8760 if (bs_opts_copy(o, &opts)) { 8761 return; 8762 } 8763 } 8764 8765 if (opts.max_md_ops == 0 || opts.max_channel_ops == 0) { 8766 dev->destroy(dev); 8767 cb_fn(cb_arg, NULL, -EINVAL); 8768 return; 8769 } 8770 8771 err = bs_alloc(dev, &opts, &bs, &ctx); 8772 if (err) { 8773 dev->destroy(dev); 8774 cb_fn(cb_arg, NULL, err); 8775 return; 8776 } 8777 8778 cpl.type = SPDK_BS_CPL_TYPE_BS_HANDLE; 8779 cpl.u.bs_handle.cb_fn = cb_fn; 8780 cpl.u.bs_handle.cb_arg = cb_arg; 8781 cpl.u.bs_handle.bs = bs; 8782 8783 ctx->seq = bs_sequence_start_bs(bs->md_channel, &cpl); 8784 if (!ctx->seq) { 8785 spdk_free(ctx->super); 8786 free(ctx); 8787 bs_free(bs); 8788 cb_fn(cb_arg, NULL, -ENOMEM); 8789 return; 8790 } 8791 8792 /* Read the super block */ 8793 bs_sequence_read_dev(ctx->seq, ctx->super, bs_page_to_lba(bs, 0), 8794 bs_byte_to_lba(bs, sizeof(*ctx->super)), 8795 bs_grow_load_super_cpl, ctx); 8796 } 8797 8798 int 8799 spdk_blob_get_esnap_id(struct spdk_blob *blob, const void **id, size_t *len) 8800 { 8801 if (!blob_is_esnap_clone(blob)) { 8802 return -EINVAL; 8803 } 8804 8805 return blob_get_xattr_value(blob, BLOB_EXTERNAL_SNAPSHOT_ID, id, len, true); 8806 } 8807 8808 struct spdk_io_channel * 8809 blob_esnap_get_io_channel(struct spdk_io_channel *ch, struct spdk_blob *blob) 8810 { 8811 struct spdk_bs_channel *bs_channel = spdk_io_channel_get_ctx(ch); 8812 struct spdk_bs_dev *bs_dev = blob->back_bs_dev; 8813 struct blob_esnap_channel find = {}; 8814 struct blob_esnap_channel *esnap_channel, *existing; 8815 8816 find.blob_id = blob->id; 8817 esnap_channel = RB_FIND(blob_esnap_channel_tree, &bs_channel->esnap_channels, &find); 8818 if (spdk_likely(esnap_channel != NULL)) { 8819 SPDK_DEBUGLOG(blob_esnap, "blob 0x%" PRIx64 ": using cached channel on thread %s\n", 8820 blob->id, spdk_thread_get_name(spdk_get_thread())); 8821 return esnap_channel->channel; 8822 } 8823 8824 SPDK_DEBUGLOG(blob_esnap, "blob 0x%" PRIx64 ": allocating channel on thread %s\n", 8825 blob->id, spdk_thread_get_name(spdk_get_thread())); 8826 8827 esnap_channel = calloc(1, sizeof(*esnap_channel)); 8828 if (esnap_channel == NULL) { 8829 SPDK_NOTICELOG("blob 0x%" PRIx64 " channel allocation failed: no memory\n", 8830 find.blob_id); 8831 return NULL; 8832 } 8833 esnap_channel->channel = bs_dev->create_channel(bs_dev); 8834 if (esnap_channel->channel == NULL) { 8835 SPDK_NOTICELOG("blob 0x%" PRIx64 " back channel allocation failed\n", blob->id); 8836 free(esnap_channel); 8837 return NULL; 8838 } 8839 esnap_channel->blob_id = find.blob_id; 8840 existing = RB_INSERT(blob_esnap_channel_tree, &bs_channel->esnap_channels, esnap_channel); 8841 if (spdk_unlikely(existing != NULL)) { 8842 /* 8843 * This should be unreachable: all modifications to this tree happen on this thread. 8844 */ 8845 SPDK_ERRLOG("blob 0x%" PRIx64 "lost race to allocate a channel\n", find.blob_id); 8846 assert(false); 8847 8848 bs_dev->destroy_channel(bs_dev, esnap_channel->channel); 8849 free(esnap_channel); 8850 8851 return existing->channel; 8852 } 8853 8854 return esnap_channel->channel; 8855 } 8856 8857 static int 8858 blob_esnap_channel_compare(struct blob_esnap_channel *c1, struct blob_esnap_channel *c2) 8859 { 8860 return (c1->blob_id < c2->blob_id ? -1 : c1->blob_id > c2->blob_id); 8861 } 8862 8863 struct blob_esnap_destroy_ctx { 8864 spdk_blob_op_with_handle_complete cb_fn; 8865 void *cb_arg; 8866 struct spdk_blob *blob; 8867 struct spdk_bs_dev *back_bs_dev; 8868 bool abort_io; 8869 }; 8870 8871 static void 8872 blob_esnap_destroy_channels_done(struct spdk_io_channel_iter *i, int status) 8873 { 8874 struct blob_esnap_destroy_ctx *ctx = spdk_io_channel_iter_get_ctx(i); 8875 struct spdk_blob *blob = ctx->blob; 8876 struct spdk_blob_store *bs = blob->bs; 8877 8878 SPDK_DEBUGLOG(blob_esnap, "blob 0x%" PRIx64 ": done destroying channels for this blob\n", 8879 blob->id); 8880 8881 if (ctx->cb_fn != NULL) { 8882 ctx->cb_fn(ctx->cb_arg, blob, status); 8883 } 8884 free(ctx); 8885 8886 bs->esnap_channels_unloading--; 8887 if (bs->esnap_channels_unloading == 0 && bs->esnap_unload_cb_fn != NULL) { 8888 spdk_bs_unload(bs, bs->esnap_unload_cb_fn, bs->esnap_unload_cb_arg); 8889 } 8890 } 8891 8892 static void 8893 blob_esnap_destroy_one_channel(struct spdk_io_channel_iter *i) 8894 { 8895 struct blob_esnap_destroy_ctx *ctx = spdk_io_channel_iter_get_ctx(i); 8896 struct spdk_blob *blob = ctx->blob; 8897 struct spdk_bs_dev *bs_dev = ctx->back_bs_dev; 8898 struct spdk_io_channel *channel = spdk_io_channel_iter_get_channel(i); 8899 struct spdk_bs_channel *bs_channel = spdk_io_channel_get_ctx(channel); 8900 struct blob_esnap_channel *esnap_channel; 8901 struct blob_esnap_channel find = {}; 8902 8903 assert(spdk_get_thread() == spdk_io_channel_get_thread(channel)); 8904 8905 find.blob_id = blob->id; 8906 esnap_channel = RB_FIND(blob_esnap_channel_tree, &bs_channel->esnap_channels, &find); 8907 if (esnap_channel != NULL) { 8908 SPDK_DEBUGLOG(blob_esnap, "blob 0x%" PRIx64 ": destroying channel on thread %s\n", 8909 blob->id, spdk_thread_get_name(spdk_get_thread())); 8910 RB_REMOVE(blob_esnap_channel_tree, &bs_channel->esnap_channels, esnap_channel); 8911 8912 if (ctx->abort_io) { 8913 spdk_bs_user_op_t *op, *tmp; 8914 8915 TAILQ_FOREACH_SAFE(op, &bs_channel->queued_io, link, tmp) { 8916 if (op->back_channel == esnap_channel->channel) { 8917 TAILQ_REMOVE(&bs_channel->queued_io, op, link); 8918 bs_user_op_abort(op, -EIO); 8919 } 8920 } 8921 } 8922 8923 bs_dev->destroy_channel(bs_dev, esnap_channel->channel); 8924 free(esnap_channel); 8925 } 8926 8927 spdk_for_each_channel_continue(i, 0); 8928 } 8929 8930 /* 8931 * Destroy the channels for a specific blob on each thread with a blobstore channel. This should be 8932 * used when closing an esnap clone blob and after decoupling from the parent. 8933 */ 8934 static void 8935 blob_esnap_destroy_bs_dev_channels(struct spdk_blob *blob, bool abort_io, 8936 spdk_blob_op_with_handle_complete cb_fn, void *cb_arg) 8937 { 8938 struct blob_esnap_destroy_ctx *ctx; 8939 8940 if (!blob_is_esnap_clone(blob)) { 8941 if (cb_fn != NULL) { 8942 cb_fn(cb_arg, blob, 0); 8943 } 8944 return; 8945 } 8946 8947 ctx = calloc(1, sizeof(*ctx)); 8948 if (ctx == NULL) { 8949 if (cb_fn != NULL) { 8950 cb_fn(cb_arg, blob, -ENOMEM); 8951 } 8952 return; 8953 } 8954 ctx->cb_fn = cb_fn; 8955 ctx->cb_arg = cb_arg; 8956 ctx->blob = blob; 8957 ctx->back_bs_dev = blob->back_bs_dev; 8958 ctx->abort_io = abort_io; 8959 8960 SPDK_DEBUGLOG(blob_esnap, "blob 0x%" PRIx64 ": destroying channels for this blob\n", 8961 blob->id); 8962 8963 blob->bs->esnap_channels_unloading++; 8964 spdk_for_each_channel(blob->bs, blob_esnap_destroy_one_channel, ctx, 8965 blob_esnap_destroy_channels_done); 8966 } 8967 8968 /* 8969 * Destroy all bs_dev channels on a specific blobstore channel. This should be used when a 8970 * bs_channel is destroyed. 8971 */ 8972 static void 8973 blob_esnap_destroy_bs_channel(struct spdk_bs_channel *ch) 8974 { 8975 struct blob_esnap_channel *esnap_channel, *esnap_channel_tmp; 8976 8977 assert(spdk_get_thread() == spdk_io_channel_get_thread(spdk_io_channel_from_ctx(ch))); 8978 8979 SPDK_DEBUGLOG(blob_esnap, "destroying channels on thread %s\n", 8980 spdk_thread_get_name(spdk_get_thread())); 8981 RB_FOREACH_SAFE(esnap_channel, blob_esnap_channel_tree, &ch->esnap_channels, 8982 esnap_channel_tmp) { 8983 SPDK_DEBUGLOG(blob_esnap, "blob 0x%" PRIx64 8984 ": destroying one channel in thread %s\n", 8985 esnap_channel->blob_id, spdk_thread_get_name(spdk_get_thread())); 8986 RB_REMOVE(blob_esnap_channel_tree, &ch->esnap_channels, esnap_channel); 8987 spdk_put_io_channel(esnap_channel->channel); 8988 free(esnap_channel); 8989 } 8990 SPDK_DEBUGLOG(blob_esnap, "done destroying channels on thread %s\n", 8991 spdk_thread_get_name(spdk_get_thread())); 8992 } 8993 8994 struct set_bs_dev_ctx { 8995 struct spdk_blob *blob; 8996 struct spdk_bs_dev *back_bs_dev; 8997 spdk_blob_op_complete cb_fn; 8998 void *cb_arg; 8999 int bserrno; 9000 }; 9001 9002 static void 9003 blob_set_back_bs_dev_done(void *_ctx, int bserrno) 9004 { 9005 struct set_bs_dev_ctx *ctx = _ctx; 9006 9007 if (bserrno != 0) { 9008 /* Even though the unfreeze failed, the update may have succeed. */ 9009 SPDK_ERRLOG("blob 0x%" PRIx64 ": unfreeze failed with error %d\n", ctx->blob->id, 9010 bserrno); 9011 } 9012 ctx->cb_fn(ctx->cb_arg, ctx->bserrno); 9013 free(ctx); 9014 } 9015 9016 static void 9017 blob_frozen_set_back_bs_dev(void *_ctx, struct spdk_blob *blob, int bserrno) 9018 { 9019 struct set_bs_dev_ctx *ctx = _ctx; 9020 9021 if (bserrno != 0) { 9022 SPDK_ERRLOG("blob 0x%" PRIx64 ": failed to release old back_bs_dev with error %d\n", 9023 blob->id, bserrno); 9024 ctx->bserrno = bserrno; 9025 blob_unfreeze_io(blob, blob_set_back_bs_dev_done, ctx); 9026 return; 9027 } 9028 9029 if (blob->back_bs_dev != NULL) { 9030 blob->back_bs_dev->destroy(blob->back_bs_dev); 9031 } 9032 9033 SPDK_NOTICELOG("blob 0x%" PRIx64 ": hotplugged back_bs_dev\n", blob->id); 9034 blob->back_bs_dev = ctx->back_bs_dev; 9035 ctx->bserrno = 0; 9036 9037 blob_unfreeze_io(blob, blob_set_back_bs_dev_done, ctx); 9038 } 9039 9040 static void 9041 blob_frozen_destroy_esnap_channels(void *_ctx, int bserrno) 9042 { 9043 struct set_bs_dev_ctx *ctx = _ctx; 9044 struct spdk_blob *blob = ctx->blob; 9045 9046 if (bserrno != 0) { 9047 SPDK_ERRLOG("blob 0x%" PRIx64 ": failed to freeze with error %d\n", blob->id, 9048 bserrno); 9049 ctx->cb_fn(ctx->cb_arg, bserrno); 9050 free(ctx); 9051 return; 9052 } 9053 9054 /* 9055 * This does not prevent future reads from the esnap device because any future IO will 9056 * lazily create a new esnap IO channel. 9057 */ 9058 blob_esnap_destroy_bs_dev_channels(blob, true, blob_frozen_set_back_bs_dev, ctx); 9059 } 9060 9061 void 9062 spdk_blob_set_esnap_bs_dev(struct spdk_blob *blob, struct spdk_bs_dev *back_bs_dev, 9063 spdk_blob_op_complete cb_fn, void *cb_arg) 9064 { 9065 struct set_bs_dev_ctx *ctx; 9066 9067 if (!blob_is_esnap_clone(blob)) { 9068 SPDK_ERRLOG("blob 0x%" PRIx64 ": not an esnap clone\n", blob->id); 9069 cb_fn(cb_arg, -EINVAL); 9070 return; 9071 } 9072 9073 ctx = calloc(1, sizeof(*ctx)); 9074 if (ctx == NULL) { 9075 SPDK_ERRLOG("blob 0x%" PRIx64 ": out of memory while setting back_bs_dev\n", 9076 blob->id); 9077 cb_fn(cb_arg, -ENOMEM); 9078 return; 9079 } 9080 ctx->cb_fn = cb_fn; 9081 ctx->cb_arg = cb_arg; 9082 ctx->back_bs_dev = back_bs_dev; 9083 ctx->blob = blob; 9084 blob_freeze_io(blob, blob_frozen_destroy_esnap_channels, ctx); 9085 } 9086 9087 SPDK_LOG_REGISTER_COMPONENT(blob) 9088 SPDK_LOG_REGISTER_COMPONENT(blob_esnap) 9089