1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) Intel Corporation. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include "spdk/stdinc.h" 35 36 #include "spdk/blob.h" 37 #include "spdk/crc32.h" 38 #include "spdk/env.h" 39 #include "spdk/queue.h" 40 #include "spdk/thread.h" 41 #include "spdk/bit_array.h" 42 #include "spdk/bit_pool.h" 43 #include "spdk/likely.h" 44 #include "spdk/util.h" 45 #include "spdk/string.h" 46 47 #include "spdk_internal/assert.h" 48 #include "spdk/log.h" 49 50 #include "blobstore.h" 51 52 #define BLOB_CRC32C_INITIAL 0xffffffffUL 53 54 static int bs_register_md_thread(struct spdk_blob_store *bs); 55 static int bs_unregister_md_thread(struct spdk_blob_store *bs); 56 static void blob_close_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno); 57 static void blob_insert_cluster_on_md_thread(struct spdk_blob *blob, uint32_t cluster_num, 58 uint64_t cluster, uint32_t extent, spdk_blob_op_complete cb_fn, void *cb_arg); 59 60 static int blob_set_xattr(struct spdk_blob *blob, const char *name, const void *value, 61 uint16_t value_len, bool internal); 62 static int blob_get_xattr_value(struct spdk_blob *blob, const char *name, 63 const void **value, size_t *value_len, bool internal); 64 static int blob_remove_xattr(struct spdk_blob *blob, const char *name, bool internal); 65 66 static void blob_insert_extent(struct spdk_blob *blob, uint32_t extent, uint64_t cluster_num, 67 spdk_blob_op_complete cb_fn, void *cb_arg); 68 69 static void 70 blob_verify_md_op(struct spdk_blob *blob) 71 { 72 assert(blob != NULL); 73 assert(spdk_get_thread() == blob->bs->md_thread); 74 assert(blob->state != SPDK_BLOB_STATE_LOADING); 75 } 76 77 static struct spdk_blob_list * 78 bs_get_snapshot_entry(struct spdk_blob_store *bs, spdk_blob_id blobid) 79 { 80 struct spdk_blob_list *snapshot_entry = NULL; 81 82 TAILQ_FOREACH(snapshot_entry, &bs->snapshots, link) { 83 if (snapshot_entry->id == blobid) { 84 break; 85 } 86 } 87 88 return snapshot_entry; 89 } 90 91 static void 92 bs_claim_md_page(struct spdk_blob_store *bs, uint32_t page) 93 { 94 assert(page < spdk_bit_array_capacity(bs->used_md_pages)); 95 assert(spdk_bit_array_get(bs->used_md_pages, page) == false); 96 97 spdk_bit_array_set(bs->used_md_pages, page); 98 } 99 100 static void 101 bs_release_md_page(struct spdk_blob_store *bs, uint32_t page) 102 { 103 assert(page < spdk_bit_array_capacity(bs->used_md_pages)); 104 assert(spdk_bit_array_get(bs->used_md_pages, page) == true); 105 106 spdk_bit_array_clear(bs->used_md_pages, page); 107 } 108 109 static uint32_t 110 bs_claim_cluster(struct spdk_blob_store *bs) 111 { 112 uint32_t cluster_num; 113 114 cluster_num = spdk_bit_pool_allocate_bit(bs->used_clusters); 115 if (cluster_num == UINT32_MAX) { 116 return UINT32_MAX; 117 } 118 119 SPDK_DEBUGLOG(blob, "Claiming cluster %u\n", cluster_num); 120 bs->num_free_clusters--; 121 122 return cluster_num; 123 } 124 125 static void 126 bs_release_cluster(struct spdk_blob_store *bs, uint32_t cluster_num) 127 { 128 assert(cluster_num < spdk_bit_pool_capacity(bs->used_clusters)); 129 assert(spdk_bit_pool_is_allocated(bs->used_clusters, cluster_num) == true); 130 assert(bs->num_free_clusters < bs->total_clusters); 131 132 SPDK_DEBUGLOG(blob, "Releasing cluster %u\n", cluster_num); 133 134 spdk_bit_pool_free_bit(bs->used_clusters, cluster_num); 135 bs->num_free_clusters++; 136 } 137 138 static int 139 blob_insert_cluster(struct spdk_blob *blob, uint32_t cluster_num, uint64_t cluster) 140 { 141 uint64_t *cluster_lba = &blob->active.clusters[cluster_num]; 142 143 blob_verify_md_op(blob); 144 145 if (*cluster_lba != 0) { 146 return -EEXIST; 147 } 148 149 *cluster_lba = bs_cluster_to_lba(blob->bs, cluster); 150 return 0; 151 } 152 153 static int 154 bs_allocate_cluster(struct spdk_blob *blob, uint32_t cluster_num, 155 uint64_t *cluster, uint32_t *lowest_free_md_page, bool update_map) 156 { 157 uint32_t *extent_page = 0; 158 159 *cluster = bs_claim_cluster(blob->bs); 160 if (*cluster == UINT32_MAX) { 161 /* No more free clusters. Cannot satisfy the request */ 162 return -ENOSPC; 163 } 164 165 if (blob->use_extent_table) { 166 extent_page = bs_cluster_to_extent_page(blob, cluster_num); 167 if (*extent_page == 0) { 168 /* Extent page shall never occupy md_page so start the search from 1 */ 169 if (*lowest_free_md_page == 0) { 170 *lowest_free_md_page = 1; 171 } 172 /* No extent_page is allocated for the cluster */ 173 *lowest_free_md_page = spdk_bit_array_find_first_clear(blob->bs->used_md_pages, 174 *lowest_free_md_page); 175 if (*lowest_free_md_page == UINT32_MAX) { 176 /* No more free md pages. Cannot satisfy the request */ 177 bs_release_cluster(blob->bs, *cluster); 178 return -ENOSPC; 179 } 180 bs_claim_md_page(blob->bs, *lowest_free_md_page); 181 } 182 } 183 184 SPDK_DEBUGLOG(blob, "Claiming cluster %" PRIu64 " for blob %" PRIu64 "\n", *cluster, blob->id); 185 186 if (update_map) { 187 blob_insert_cluster(blob, cluster_num, *cluster); 188 if (blob->use_extent_table && *extent_page == 0) { 189 *extent_page = *lowest_free_md_page; 190 } 191 } 192 193 return 0; 194 } 195 196 static void 197 blob_xattrs_init(struct spdk_blob_xattr_opts *xattrs) 198 { 199 xattrs->count = 0; 200 xattrs->names = NULL; 201 xattrs->ctx = NULL; 202 xattrs->get_value = NULL; 203 } 204 205 void 206 spdk_blob_opts_init(struct spdk_blob_opts *opts, size_t opts_size) 207 { 208 if (!opts) { 209 SPDK_ERRLOG("opts should not be NULL\n"); 210 return; 211 } 212 213 if (!opts_size) { 214 SPDK_ERRLOG("opts_size should not be zero value\n"); 215 return; 216 } 217 218 memset(opts, 0, opts_size); 219 opts->opts_size = opts_size; 220 221 #define FIELD_OK(field) \ 222 offsetof(struct spdk_blob_opts, field) + sizeof(opts->field) <= opts_size 223 224 #define SET_FIELD(field, value) \ 225 if (FIELD_OK(field)) { \ 226 opts->field = value; \ 227 } \ 228 229 SET_FIELD(num_clusters, 0); 230 SET_FIELD(thin_provision, false); 231 SET_FIELD(clear_method, BLOB_CLEAR_WITH_DEFAULT); 232 233 if (FIELD_OK(xattrs)) { 234 blob_xattrs_init(&opts->xattrs); 235 } 236 237 SET_FIELD(use_extent_table, true); 238 239 #undef FIELD_OK 240 #undef SET_FIELD 241 } 242 243 void 244 spdk_blob_open_opts_init(struct spdk_blob_open_opts *opts, size_t opts_size) 245 { 246 if (!opts) { 247 SPDK_ERRLOG("opts should not be NULL\n"); 248 return; 249 } 250 251 if (!opts_size) { 252 SPDK_ERRLOG("opts_size should not be zero value\n"); 253 return; 254 } 255 256 memset(opts, 0, opts_size); 257 opts->opts_size = opts_size; 258 259 #define FIELD_OK(field) \ 260 offsetof(struct spdk_blob_open_opts, field) + sizeof(opts->field) <= opts_size 261 262 #define SET_FIELD(field, value) \ 263 if (FIELD_OK(field)) { \ 264 opts->field = value; \ 265 } \ 266 267 SET_FIELD(clear_method, BLOB_CLEAR_WITH_DEFAULT); 268 269 #undef FIELD_OK 270 #undef SET_FILED 271 } 272 273 static struct spdk_blob * 274 blob_alloc(struct spdk_blob_store *bs, spdk_blob_id id) 275 { 276 struct spdk_blob *blob; 277 278 blob = calloc(1, sizeof(*blob)); 279 if (!blob) { 280 return NULL; 281 } 282 283 blob->id = id; 284 blob->bs = bs; 285 286 blob->parent_id = SPDK_BLOBID_INVALID; 287 288 blob->state = SPDK_BLOB_STATE_DIRTY; 289 blob->extent_rle_found = false; 290 blob->extent_table_found = false; 291 blob->active.num_pages = 1; 292 blob->active.pages = calloc(1, sizeof(*blob->active.pages)); 293 if (!blob->active.pages) { 294 free(blob); 295 return NULL; 296 } 297 298 blob->active.pages[0] = bs_blobid_to_page(id); 299 300 TAILQ_INIT(&blob->xattrs); 301 TAILQ_INIT(&blob->xattrs_internal); 302 TAILQ_INIT(&blob->pending_persists); 303 304 return blob; 305 } 306 307 static void 308 xattrs_free(struct spdk_xattr_tailq *xattrs) 309 { 310 struct spdk_xattr *xattr, *xattr_tmp; 311 312 TAILQ_FOREACH_SAFE(xattr, xattrs, link, xattr_tmp) { 313 TAILQ_REMOVE(xattrs, xattr, link); 314 free(xattr->name); 315 free(xattr->value); 316 free(xattr); 317 } 318 } 319 320 static void 321 blob_free(struct spdk_blob *blob) 322 { 323 assert(blob != NULL); 324 assert(TAILQ_EMPTY(&blob->pending_persists)); 325 326 free(blob->active.extent_pages); 327 free(blob->clean.extent_pages); 328 free(blob->active.clusters); 329 free(blob->clean.clusters); 330 free(blob->active.pages); 331 free(blob->clean.pages); 332 333 xattrs_free(&blob->xattrs); 334 xattrs_free(&blob->xattrs_internal); 335 336 if (blob->back_bs_dev) { 337 blob->back_bs_dev->destroy(blob->back_bs_dev); 338 } 339 340 free(blob); 341 } 342 343 struct freeze_io_ctx { 344 struct spdk_bs_cpl cpl; 345 struct spdk_blob *blob; 346 }; 347 348 static void 349 blob_io_sync(struct spdk_io_channel_iter *i) 350 { 351 spdk_for_each_channel_continue(i, 0); 352 } 353 354 static void 355 blob_execute_queued_io(struct spdk_io_channel_iter *i) 356 { 357 struct spdk_io_channel *_ch = spdk_io_channel_iter_get_channel(i); 358 struct spdk_bs_channel *ch = spdk_io_channel_get_ctx(_ch); 359 struct freeze_io_ctx *ctx = spdk_io_channel_iter_get_ctx(i); 360 struct spdk_bs_request_set *set; 361 struct spdk_bs_user_op_args *args; 362 spdk_bs_user_op_t *op, *tmp; 363 364 TAILQ_FOREACH_SAFE(op, &ch->queued_io, link, tmp) { 365 set = (struct spdk_bs_request_set *)op; 366 args = &set->u.user_op; 367 368 if (args->blob == ctx->blob) { 369 TAILQ_REMOVE(&ch->queued_io, op, link); 370 bs_user_op_execute(op); 371 } 372 } 373 374 spdk_for_each_channel_continue(i, 0); 375 } 376 377 static void 378 blob_io_cpl(struct spdk_io_channel_iter *i, int status) 379 { 380 struct freeze_io_ctx *ctx = spdk_io_channel_iter_get_ctx(i); 381 382 ctx->cpl.u.blob_basic.cb_fn(ctx->cpl.u.blob_basic.cb_arg, 0); 383 384 free(ctx); 385 } 386 387 static void 388 blob_freeze_io(struct spdk_blob *blob, spdk_blob_op_complete cb_fn, void *cb_arg) 389 { 390 struct freeze_io_ctx *ctx; 391 392 ctx = calloc(1, sizeof(*ctx)); 393 if (!ctx) { 394 cb_fn(cb_arg, -ENOMEM); 395 return; 396 } 397 398 ctx->cpl.type = SPDK_BS_CPL_TYPE_BS_BASIC; 399 ctx->cpl.u.blob_basic.cb_fn = cb_fn; 400 ctx->cpl.u.blob_basic.cb_arg = cb_arg; 401 ctx->blob = blob; 402 403 /* Freeze I/O on blob */ 404 blob->frozen_refcnt++; 405 406 if (blob->frozen_refcnt == 1) { 407 spdk_for_each_channel(blob->bs, blob_io_sync, ctx, blob_io_cpl); 408 } else { 409 cb_fn(cb_arg, 0); 410 free(ctx); 411 } 412 } 413 414 static void 415 blob_unfreeze_io(struct spdk_blob *blob, spdk_blob_op_complete cb_fn, void *cb_arg) 416 { 417 struct freeze_io_ctx *ctx; 418 419 ctx = calloc(1, sizeof(*ctx)); 420 if (!ctx) { 421 cb_fn(cb_arg, -ENOMEM); 422 return; 423 } 424 425 ctx->cpl.type = SPDK_BS_CPL_TYPE_BS_BASIC; 426 ctx->cpl.u.blob_basic.cb_fn = cb_fn; 427 ctx->cpl.u.blob_basic.cb_arg = cb_arg; 428 ctx->blob = blob; 429 430 assert(blob->frozen_refcnt > 0); 431 432 blob->frozen_refcnt--; 433 434 if (blob->frozen_refcnt == 0) { 435 spdk_for_each_channel(blob->bs, blob_execute_queued_io, ctx, blob_io_cpl); 436 } else { 437 cb_fn(cb_arg, 0); 438 free(ctx); 439 } 440 } 441 442 static int 443 blob_mark_clean(struct spdk_blob *blob) 444 { 445 uint32_t *extent_pages = NULL; 446 uint64_t *clusters = NULL; 447 uint32_t *pages = NULL; 448 449 assert(blob != NULL); 450 451 if (blob->active.num_extent_pages) { 452 assert(blob->active.extent_pages); 453 extent_pages = calloc(blob->active.num_extent_pages, sizeof(*blob->active.extent_pages)); 454 if (!extent_pages) { 455 return -ENOMEM; 456 } 457 memcpy(extent_pages, blob->active.extent_pages, 458 blob->active.num_extent_pages * sizeof(*extent_pages)); 459 } 460 461 if (blob->active.num_clusters) { 462 assert(blob->active.clusters); 463 clusters = calloc(blob->active.num_clusters, sizeof(*blob->active.clusters)); 464 if (!clusters) { 465 free(extent_pages); 466 return -ENOMEM; 467 } 468 memcpy(clusters, blob->active.clusters, blob->active.num_clusters * sizeof(*blob->active.clusters)); 469 } 470 471 if (blob->active.num_pages) { 472 assert(blob->active.pages); 473 pages = calloc(blob->active.num_pages, sizeof(*blob->active.pages)); 474 if (!pages) { 475 free(extent_pages); 476 free(clusters); 477 return -ENOMEM; 478 } 479 memcpy(pages, blob->active.pages, blob->active.num_pages * sizeof(*blob->active.pages)); 480 } 481 482 free(blob->clean.extent_pages); 483 free(blob->clean.clusters); 484 free(blob->clean.pages); 485 486 blob->clean.num_extent_pages = blob->active.num_extent_pages; 487 blob->clean.extent_pages = blob->active.extent_pages; 488 blob->clean.num_clusters = blob->active.num_clusters; 489 blob->clean.clusters = blob->active.clusters; 490 blob->clean.num_pages = blob->active.num_pages; 491 blob->clean.pages = blob->active.pages; 492 493 blob->active.extent_pages = extent_pages; 494 blob->active.clusters = clusters; 495 blob->active.pages = pages; 496 497 /* If the metadata was dirtied again while the metadata was being written to disk, 498 * we do not want to revert the DIRTY state back to CLEAN here. 499 */ 500 if (blob->state == SPDK_BLOB_STATE_LOADING) { 501 blob->state = SPDK_BLOB_STATE_CLEAN; 502 } 503 504 return 0; 505 } 506 507 static int 508 blob_deserialize_xattr(struct spdk_blob *blob, 509 struct spdk_blob_md_descriptor_xattr *desc_xattr, bool internal) 510 { 511 struct spdk_xattr *xattr; 512 513 if (desc_xattr->length != sizeof(desc_xattr->name_length) + 514 sizeof(desc_xattr->value_length) + 515 desc_xattr->name_length + desc_xattr->value_length) { 516 return -EINVAL; 517 } 518 519 xattr = calloc(1, sizeof(*xattr)); 520 if (xattr == NULL) { 521 return -ENOMEM; 522 } 523 524 xattr->name = malloc(desc_xattr->name_length + 1); 525 if (xattr->name == NULL) { 526 free(xattr); 527 return -ENOMEM; 528 } 529 memcpy(xattr->name, desc_xattr->name, desc_xattr->name_length); 530 xattr->name[desc_xattr->name_length] = '\0'; 531 532 xattr->value = malloc(desc_xattr->value_length); 533 if (xattr->value == NULL) { 534 free(xattr->name); 535 free(xattr); 536 return -ENOMEM; 537 } 538 xattr->value_len = desc_xattr->value_length; 539 memcpy(xattr->value, 540 (void *)((uintptr_t)desc_xattr->name + desc_xattr->name_length), 541 desc_xattr->value_length); 542 543 TAILQ_INSERT_TAIL(internal ? &blob->xattrs_internal : &blob->xattrs, xattr, link); 544 545 return 0; 546 } 547 548 549 static int 550 blob_parse_page(const struct spdk_blob_md_page *page, struct spdk_blob *blob) 551 { 552 struct spdk_blob_md_descriptor *desc; 553 size_t cur_desc = 0; 554 void *tmp; 555 556 desc = (struct spdk_blob_md_descriptor *)page->descriptors; 557 while (cur_desc < sizeof(page->descriptors)) { 558 if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_PADDING) { 559 if (desc->length == 0) { 560 /* If padding and length are 0, this terminates the page */ 561 break; 562 } 563 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_FLAGS) { 564 struct spdk_blob_md_descriptor_flags *desc_flags; 565 566 desc_flags = (struct spdk_blob_md_descriptor_flags *)desc; 567 568 if (desc_flags->length != sizeof(*desc_flags) - sizeof(*desc)) { 569 return -EINVAL; 570 } 571 572 if ((desc_flags->invalid_flags | SPDK_BLOB_INVALID_FLAGS_MASK) != 573 SPDK_BLOB_INVALID_FLAGS_MASK) { 574 return -EINVAL; 575 } 576 577 if ((desc_flags->data_ro_flags | SPDK_BLOB_DATA_RO_FLAGS_MASK) != 578 SPDK_BLOB_DATA_RO_FLAGS_MASK) { 579 blob->data_ro = true; 580 blob->md_ro = true; 581 } 582 583 if ((desc_flags->md_ro_flags | SPDK_BLOB_MD_RO_FLAGS_MASK) != 584 SPDK_BLOB_MD_RO_FLAGS_MASK) { 585 blob->md_ro = true; 586 } 587 588 if ((desc_flags->data_ro_flags & SPDK_BLOB_READ_ONLY)) { 589 blob->data_ro = true; 590 blob->md_ro = true; 591 } 592 593 blob->invalid_flags = desc_flags->invalid_flags; 594 blob->data_ro_flags = desc_flags->data_ro_flags; 595 blob->md_ro_flags = desc_flags->md_ro_flags; 596 597 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_EXTENT_RLE) { 598 struct spdk_blob_md_descriptor_extent_rle *desc_extent_rle; 599 unsigned int i, j; 600 unsigned int cluster_count = blob->active.num_clusters; 601 602 if (blob->extent_table_found) { 603 /* Extent Table already present in the md, 604 * both descriptors should never be at the same time. */ 605 return -EINVAL; 606 } 607 blob->extent_rle_found = true; 608 609 desc_extent_rle = (struct spdk_blob_md_descriptor_extent_rle *)desc; 610 611 if (desc_extent_rle->length == 0 || 612 (desc_extent_rle->length % sizeof(desc_extent_rle->extents[0]) != 0)) { 613 return -EINVAL; 614 } 615 616 for (i = 0; i < desc_extent_rle->length / sizeof(desc_extent_rle->extents[0]); i++) { 617 for (j = 0; j < desc_extent_rle->extents[i].length; j++) { 618 if (desc_extent_rle->extents[i].cluster_idx != 0) { 619 if (!spdk_bit_pool_is_allocated(blob->bs->used_clusters, 620 desc_extent_rle->extents[i].cluster_idx + j)) { 621 return -EINVAL; 622 } 623 } 624 cluster_count++; 625 } 626 } 627 628 if (cluster_count == 0) { 629 return -EINVAL; 630 } 631 tmp = realloc(blob->active.clusters, cluster_count * sizeof(*blob->active.clusters)); 632 if (tmp == NULL) { 633 return -ENOMEM; 634 } 635 blob->active.clusters = tmp; 636 blob->active.cluster_array_size = cluster_count; 637 638 for (i = 0; i < desc_extent_rle->length / sizeof(desc_extent_rle->extents[0]); i++) { 639 for (j = 0; j < desc_extent_rle->extents[i].length; j++) { 640 if (desc_extent_rle->extents[i].cluster_idx != 0) { 641 blob->active.clusters[blob->active.num_clusters++] = bs_cluster_to_lba(blob->bs, 642 desc_extent_rle->extents[i].cluster_idx + j); 643 } else if (spdk_blob_is_thin_provisioned(blob)) { 644 blob->active.clusters[blob->active.num_clusters++] = 0; 645 } else { 646 return -EINVAL; 647 } 648 } 649 } 650 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_EXTENT_TABLE) { 651 struct spdk_blob_md_descriptor_extent_table *desc_extent_table; 652 uint32_t num_extent_pages = blob->active.num_extent_pages; 653 uint32_t i, j; 654 size_t extent_pages_length; 655 656 desc_extent_table = (struct spdk_blob_md_descriptor_extent_table *)desc; 657 extent_pages_length = desc_extent_table->length - sizeof(desc_extent_table->num_clusters); 658 659 if (blob->extent_rle_found) { 660 /* This means that Extent RLE is present in MD, 661 * both should never be at the same time. */ 662 return -EINVAL; 663 } else if (blob->extent_table_found && 664 desc_extent_table->num_clusters != blob->remaining_clusters_in_et) { 665 /* Number of clusters in this ET does not match number 666 * from previously read EXTENT_TABLE. */ 667 return -EINVAL; 668 } 669 670 blob->extent_table_found = true; 671 672 if (desc_extent_table->length == 0 || 673 (extent_pages_length % sizeof(desc_extent_table->extent_page[0]) != 0)) { 674 return -EINVAL; 675 } 676 677 for (i = 0; i < extent_pages_length / sizeof(desc_extent_table->extent_page[0]); i++) { 678 num_extent_pages += desc_extent_table->extent_page[i].num_pages; 679 } 680 681 tmp = realloc(blob->active.extent_pages, num_extent_pages * sizeof(uint32_t)); 682 if (tmp == NULL) { 683 return -ENOMEM; 684 } 685 blob->active.extent_pages = tmp; 686 blob->active.extent_pages_array_size = num_extent_pages; 687 688 blob->remaining_clusters_in_et = desc_extent_table->num_clusters; 689 690 /* Extent table entries contain md page numbers for extent pages. 691 * Zeroes represent unallocated extent pages, those are run-length-encoded. 692 */ 693 for (i = 0; i < extent_pages_length / sizeof(desc_extent_table->extent_page[0]); i++) { 694 if (desc_extent_table->extent_page[i].page_idx != 0) { 695 assert(desc_extent_table->extent_page[i].num_pages == 1); 696 blob->active.extent_pages[blob->active.num_extent_pages++] = 697 desc_extent_table->extent_page[i].page_idx; 698 } else if (spdk_blob_is_thin_provisioned(blob)) { 699 for (j = 0; j < desc_extent_table->extent_page[i].num_pages; j++) { 700 blob->active.extent_pages[blob->active.num_extent_pages++] = 0; 701 } 702 } else { 703 return -EINVAL; 704 } 705 } 706 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_EXTENT_PAGE) { 707 struct spdk_blob_md_descriptor_extent_page *desc_extent; 708 unsigned int i; 709 unsigned int cluster_count = 0; 710 size_t cluster_idx_length; 711 712 if (blob->extent_rle_found) { 713 /* This means that Extent RLE is present in MD, 714 * both should never be at the same time. */ 715 return -EINVAL; 716 } 717 718 desc_extent = (struct spdk_blob_md_descriptor_extent_page *)desc; 719 cluster_idx_length = desc_extent->length - sizeof(desc_extent->start_cluster_idx); 720 721 if (desc_extent->length <= sizeof(desc_extent->start_cluster_idx) || 722 (cluster_idx_length % sizeof(desc_extent->cluster_idx[0]) != 0)) { 723 return -EINVAL; 724 } 725 726 for (i = 0; i < cluster_idx_length / sizeof(desc_extent->cluster_idx[0]); i++) { 727 if (desc_extent->cluster_idx[i] != 0) { 728 if (!spdk_bit_pool_is_allocated(blob->bs->used_clusters, desc_extent->cluster_idx[i])) { 729 return -EINVAL; 730 } 731 } 732 cluster_count++; 733 } 734 735 if (cluster_count == 0) { 736 return -EINVAL; 737 } 738 739 /* When reading extent pages sequentially starting cluster idx should match 740 * current size of a blob. 741 * If changed to batch reading, this check shall be removed. */ 742 if (desc_extent->start_cluster_idx != blob->active.num_clusters) { 743 return -EINVAL; 744 } 745 746 tmp = realloc(blob->active.clusters, 747 (cluster_count + blob->active.num_clusters) * sizeof(*blob->active.clusters)); 748 if (tmp == NULL) { 749 return -ENOMEM; 750 } 751 blob->active.clusters = tmp; 752 blob->active.cluster_array_size = (cluster_count + blob->active.num_clusters); 753 754 for (i = 0; i < cluster_idx_length / sizeof(desc_extent->cluster_idx[0]); i++) { 755 if (desc_extent->cluster_idx[i] != 0) { 756 blob->active.clusters[blob->active.num_clusters++] = bs_cluster_to_lba(blob->bs, 757 desc_extent->cluster_idx[i]); 758 } else if (spdk_blob_is_thin_provisioned(blob)) { 759 blob->active.clusters[blob->active.num_clusters++] = 0; 760 } else { 761 return -EINVAL; 762 } 763 } 764 assert(desc_extent->start_cluster_idx + cluster_count == blob->active.num_clusters); 765 assert(blob->remaining_clusters_in_et >= cluster_count); 766 blob->remaining_clusters_in_et -= cluster_count; 767 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_XATTR) { 768 int rc; 769 770 rc = blob_deserialize_xattr(blob, 771 (struct spdk_blob_md_descriptor_xattr *) desc, false); 772 if (rc != 0) { 773 return rc; 774 } 775 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_XATTR_INTERNAL) { 776 int rc; 777 778 rc = blob_deserialize_xattr(blob, 779 (struct spdk_blob_md_descriptor_xattr *) desc, true); 780 if (rc != 0) { 781 return rc; 782 } 783 } else { 784 /* Unrecognized descriptor type. Do not fail - just continue to the 785 * next descriptor. If this descriptor is associated with some feature 786 * defined in a newer version of blobstore, that version of blobstore 787 * should create and set an associated feature flag to specify if this 788 * blob can be loaded or not. 789 */ 790 } 791 792 /* Advance to the next descriptor */ 793 cur_desc += sizeof(*desc) + desc->length; 794 if (cur_desc + sizeof(*desc) > sizeof(page->descriptors)) { 795 break; 796 } 797 desc = (struct spdk_blob_md_descriptor *)((uintptr_t)page->descriptors + cur_desc); 798 } 799 800 return 0; 801 } 802 803 static bool bs_load_cur_extent_page_valid(struct spdk_blob_md_page *page); 804 805 static int 806 blob_parse_extent_page(struct spdk_blob_md_page *extent_page, struct spdk_blob *blob) 807 { 808 assert(blob != NULL); 809 assert(blob->state == SPDK_BLOB_STATE_LOADING); 810 811 if (bs_load_cur_extent_page_valid(extent_page) == false) { 812 return -ENOENT; 813 } 814 815 return blob_parse_page(extent_page, blob); 816 } 817 818 static int 819 blob_parse(const struct spdk_blob_md_page *pages, uint32_t page_count, 820 struct spdk_blob *blob) 821 { 822 const struct spdk_blob_md_page *page; 823 uint32_t i; 824 int rc; 825 void *tmp; 826 827 assert(page_count > 0); 828 assert(pages[0].sequence_num == 0); 829 assert(blob != NULL); 830 assert(blob->state == SPDK_BLOB_STATE_LOADING); 831 assert(blob->active.clusters == NULL); 832 833 /* The blobid provided doesn't match what's in the MD, this can 834 * happen for example if a bogus blobid is passed in through open. 835 */ 836 if (blob->id != pages[0].id) { 837 SPDK_ERRLOG("Blobid (%" PRIu64 ") doesn't match what's in metadata (%" PRIu64 ")\n", 838 blob->id, pages[0].id); 839 return -ENOENT; 840 } 841 842 tmp = realloc(blob->active.pages, page_count * sizeof(*blob->active.pages)); 843 if (!tmp) { 844 return -ENOMEM; 845 } 846 blob->active.pages = tmp; 847 848 blob->active.pages[0] = pages[0].id; 849 850 for (i = 1; i < page_count; i++) { 851 assert(spdk_bit_array_get(blob->bs->used_md_pages, pages[i - 1].next)); 852 blob->active.pages[i] = pages[i - 1].next; 853 } 854 blob->active.num_pages = page_count; 855 856 for (i = 0; i < page_count; i++) { 857 page = &pages[i]; 858 859 assert(page->id == blob->id); 860 assert(page->sequence_num == i); 861 862 rc = blob_parse_page(page, blob); 863 if (rc != 0) { 864 return rc; 865 } 866 } 867 868 return 0; 869 } 870 871 static int 872 blob_serialize_add_page(const struct spdk_blob *blob, 873 struct spdk_blob_md_page **pages, 874 uint32_t *page_count, 875 struct spdk_blob_md_page **last_page) 876 { 877 struct spdk_blob_md_page *page; 878 879 assert(pages != NULL); 880 assert(page_count != NULL); 881 882 if (*page_count == 0) { 883 assert(*pages == NULL); 884 *page_count = 1; 885 *pages = spdk_malloc(SPDK_BS_PAGE_SIZE, 0, 886 NULL, SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA); 887 } else { 888 assert(*pages != NULL); 889 (*page_count)++; 890 *pages = spdk_realloc(*pages, SPDK_BS_PAGE_SIZE * (*page_count), 0); 891 } 892 893 if (*pages == NULL) { 894 *page_count = 0; 895 *last_page = NULL; 896 return -ENOMEM; 897 } 898 899 page = &(*pages)[*page_count - 1]; 900 memset(page, 0, sizeof(*page)); 901 page->id = blob->id; 902 page->sequence_num = *page_count - 1; 903 page->next = SPDK_INVALID_MD_PAGE; 904 *last_page = page; 905 906 return 0; 907 } 908 909 /* Transform the in-memory representation 'xattr' into an on-disk xattr descriptor. 910 * Update required_sz on both success and failure. 911 * 912 */ 913 static int 914 blob_serialize_xattr(const struct spdk_xattr *xattr, 915 uint8_t *buf, size_t buf_sz, 916 size_t *required_sz, bool internal) 917 { 918 struct spdk_blob_md_descriptor_xattr *desc; 919 920 *required_sz = sizeof(struct spdk_blob_md_descriptor_xattr) + 921 strlen(xattr->name) + 922 xattr->value_len; 923 924 if (buf_sz < *required_sz) { 925 return -1; 926 } 927 928 desc = (struct spdk_blob_md_descriptor_xattr *)buf; 929 930 desc->type = internal ? SPDK_MD_DESCRIPTOR_TYPE_XATTR_INTERNAL : SPDK_MD_DESCRIPTOR_TYPE_XATTR; 931 desc->length = sizeof(desc->name_length) + 932 sizeof(desc->value_length) + 933 strlen(xattr->name) + 934 xattr->value_len; 935 desc->name_length = strlen(xattr->name); 936 desc->value_length = xattr->value_len; 937 938 memcpy(desc->name, xattr->name, desc->name_length); 939 memcpy((void *)((uintptr_t)desc->name + desc->name_length), 940 xattr->value, 941 desc->value_length); 942 943 return 0; 944 } 945 946 static void 947 blob_serialize_extent_table_entry(const struct spdk_blob *blob, 948 uint64_t start_ep, uint64_t *next_ep, 949 uint8_t **buf, size_t *remaining_sz) 950 { 951 struct spdk_blob_md_descriptor_extent_table *desc; 952 size_t cur_sz; 953 uint64_t i, et_idx; 954 uint32_t extent_page, ep_len; 955 956 /* The buffer must have room for at least num_clusters entry */ 957 cur_sz = sizeof(struct spdk_blob_md_descriptor) + sizeof(desc->num_clusters); 958 if (*remaining_sz < cur_sz) { 959 *next_ep = start_ep; 960 return; 961 } 962 963 desc = (struct spdk_blob_md_descriptor_extent_table *)*buf; 964 desc->type = SPDK_MD_DESCRIPTOR_TYPE_EXTENT_TABLE; 965 966 desc->num_clusters = blob->active.num_clusters; 967 968 ep_len = 1; 969 et_idx = 0; 970 for (i = start_ep; i < blob->active.num_extent_pages; i++) { 971 if (*remaining_sz < cur_sz + sizeof(desc->extent_page[0])) { 972 /* If we ran out of buffer space, return */ 973 break; 974 } 975 976 extent_page = blob->active.extent_pages[i]; 977 /* Verify that next extent_page is unallocated */ 978 if (extent_page == 0 && 979 (i + 1 < blob->active.num_extent_pages && blob->active.extent_pages[i + 1] == 0)) { 980 ep_len++; 981 continue; 982 } 983 desc->extent_page[et_idx].page_idx = extent_page; 984 desc->extent_page[et_idx].num_pages = ep_len; 985 et_idx++; 986 987 ep_len = 1; 988 cur_sz += sizeof(desc->extent_page[et_idx]); 989 } 990 *next_ep = i; 991 992 desc->length = sizeof(desc->num_clusters) + sizeof(desc->extent_page[0]) * et_idx; 993 *remaining_sz -= sizeof(struct spdk_blob_md_descriptor) + desc->length; 994 *buf += sizeof(struct spdk_blob_md_descriptor) + desc->length; 995 } 996 997 static int 998 blob_serialize_extent_table(const struct spdk_blob *blob, 999 struct spdk_blob_md_page **pages, 1000 struct spdk_blob_md_page *cur_page, 1001 uint32_t *page_count, uint8_t **buf, 1002 size_t *remaining_sz) 1003 { 1004 uint64_t last_extent_page; 1005 int rc; 1006 1007 last_extent_page = 0; 1008 /* At least single extent table entry has to be always persisted. 1009 * Such case occurs with num_extent_pages == 0. */ 1010 while (last_extent_page <= blob->active.num_extent_pages) { 1011 blob_serialize_extent_table_entry(blob, last_extent_page, &last_extent_page, buf, 1012 remaining_sz); 1013 1014 if (last_extent_page == blob->active.num_extent_pages) { 1015 break; 1016 } 1017 1018 rc = blob_serialize_add_page(blob, pages, page_count, &cur_page); 1019 if (rc < 0) { 1020 return rc; 1021 } 1022 1023 *buf = (uint8_t *)cur_page->descriptors; 1024 *remaining_sz = sizeof(cur_page->descriptors); 1025 } 1026 1027 return 0; 1028 } 1029 1030 static void 1031 blob_serialize_extent_rle(const struct spdk_blob *blob, 1032 uint64_t start_cluster, uint64_t *next_cluster, 1033 uint8_t **buf, size_t *buf_sz) 1034 { 1035 struct spdk_blob_md_descriptor_extent_rle *desc_extent_rle; 1036 size_t cur_sz; 1037 uint64_t i, extent_idx; 1038 uint64_t lba, lba_per_cluster, lba_count; 1039 1040 /* The buffer must have room for at least one extent */ 1041 cur_sz = sizeof(struct spdk_blob_md_descriptor) + sizeof(desc_extent_rle->extents[0]); 1042 if (*buf_sz < cur_sz) { 1043 *next_cluster = start_cluster; 1044 return; 1045 } 1046 1047 desc_extent_rle = (struct spdk_blob_md_descriptor_extent_rle *)*buf; 1048 desc_extent_rle->type = SPDK_MD_DESCRIPTOR_TYPE_EXTENT_RLE; 1049 1050 lba_per_cluster = bs_cluster_to_lba(blob->bs, 1); 1051 1052 lba = blob->active.clusters[start_cluster]; 1053 lba_count = lba_per_cluster; 1054 extent_idx = 0; 1055 for (i = start_cluster + 1; i < blob->active.num_clusters; i++) { 1056 if ((lba + lba_count) == blob->active.clusters[i] && lba != 0) { 1057 /* Run-length encode sequential non-zero LBA */ 1058 lba_count += lba_per_cluster; 1059 continue; 1060 } else if (lba == 0 && blob->active.clusters[i] == 0) { 1061 /* Run-length encode unallocated clusters */ 1062 lba_count += lba_per_cluster; 1063 continue; 1064 } 1065 desc_extent_rle->extents[extent_idx].cluster_idx = lba / lba_per_cluster; 1066 desc_extent_rle->extents[extent_idx].length = lba_count / lba_per_cluster; 1067 extent_idx++; 1068 1069 cur_sz += sizeof(desc_extent_rle->extents[extent_idx]); 1070 1071 if (*buf_sz < cur_sz) { 1072 /* If we ran out of buffer space, return */ 1073 *next_cluster = i; 1074 break; 1075 } 1076 1077 lba = blob->active.clusters[i]; 1078 lba_count = lba_per_cluster; 1079 } 1080 1081 if (*buf_sz >= cur_sz) { 1082 desc_extent_rle->extents[extent_idx].cluster_idx = lba / lba_per_cluster; 1083 desc_extent_rle->extents[extent_idx].length = lba_count / lba_per_cluster; 1084 extent_idx++; 1085 1086 *next_cluster = blob->active.num_clusters; 1087 } 1088 1089 desc_extent_rle->length = sizeof(desc_extent_rle->extents[0]) * extent_idx; 1090 *buf_sz -= sizeof(struct spdk_blob_md_descriptor) + desc_extent_rle->length; 1091 *buf += sizeof(struct spdk_blob_md_descriptor) + desc_extent_rle->length; 1092 } 1093 1094 static int 1095 blob_serialize_extents_rle(const struct spdk_blob *blob, 1096 struct spdk_blob_md_page **pages, 1097 struct spdk_blob_md_page *cur_page, 1098 uint32_t *page_count, uint8_t **buf, 1099 size_t *remaining_sz) 1100 { 1101 uint64_t last_cluster; 1102 int rc; 1103 1104 last_cluster = 0; 1105 while (last_cluster < blob->active.num_clusters) { 1106 blob_serialize_extent_rle(blob, last_cluster, &last_cluster, buf, remaining_sz); 1107 1108 if (last_cluster == blob->active.num_clusters) { 1109 break; 1110 } 1111 1112 rc = blob_serialize_add_page(blob, pages, page_count, &cur_page); 1113 if (rc < 0) { 1114 return rc; 1115 } 1116 1117 *buf = (uint8_t *)cur_page->descriptors; 1118 *remaining_sz = sizeof(cur_page->descriptors); 1119 } 1120 1121 return 0; 1122 } 1123 1124 static void 1125 blob_serialize_extent_page(const struct spdk_blob *blob, 1126 uint64_t cluster, struct spdk_blob_md_page *page) 1127 { 1128 struct spdk_blob_md_descriptor_extent_page *desc_extent; 1129 uint64_t i, extent_idx; 1130 uint64_t lba, lba_per_cluster; 1131 uint64_t start_cluster_idx = (cluster / SPDK_EXTENTS_PER_EP) * SPDK_EXTENTS_PER_EP; 1132 1133 desc_extent = (struct spdk_blob_md_descriptor_extent_page *) page->descriptors; 1134 desc_extent->type = SPDK_MD_DESCRIPTOR_TYPE_EXTENT_PAGE; 1135 1136 lba_per_cluster = bs_cluster_to_lba(blob->bs, 1); 1137 1138 desc_extent->start_cluster_idx = start_cluster_idx; 1139 extent_idx = 0; 1140 for (i = start_cluster_idx; i < blob->active.num_clusters; i++) { 1141 lba = blob->active.clusters[i]; 1142 desc_extent->cluster_idx[extent_idx++] = lba / lba_per_cluster; 1143 if (extent_idx >= SPDK_EXTENTS_PER_EP) { 1144 break; 1145 } 1146 } 1147 desc_extent->length = sizeof(desc_extent->start_cluster_idx) + 1148 sizeof(desc_extent->cluster_idx[0]) * extent_idx; 1149 } 1150 1151 static void 1152 blob_serialize_flags(const struct spdk_blob *blob, 1153 uint8_t *buf, size_t *buf_sz) 1154 { 1155 struct spdk_blob_md_descriptor_flags *desc; 1156 1157 /* 1158 * Flags get serialized first, so we should always have room for the flags 1159 * descriptor. 1160 */ 1161 assert(*buf_sz >= sizeof(*desc)); 1162 1163 desc = (struct spdk_blob_md_descriptor_flags *)buf; 1164 desc->type = SPDK_MD_DESCRIPTOR_TYPE_FLAGS; 1165 desc->length = sizeof(*desc) - sizeof(struct spdk_blob_md_descriptor); 1166 desc->invalid_flags = blob->invalid_flags; 1167 desc->data_ro_flags = blob->data_ro_flags; 1168 desc->md_ro_flags = blob->md_ro_flags; 1169 1170 *buf_sz -= sizeof(*desc); 1171 } 1172 1173 static int 1174 blob_serialize_xattrs(const struct spdk_blob *blob, 1175 const struct spdk_xattr_tailq *xattrs, bool internal, 1176 struct spdk_blob_md_page **pages, 1177 struct spdk_blob_md_page *cur_page, 1178 uint32_t *page_count, uint8_t **buf, 1179 size_t *remaining_sz) 1180 { 1181 const struct spdk_xattr *xattr; 1182 int rc; 1183 1184 TAILQ_FOREACH(xattr, xattrs, link) { 1185 size_t required_sz = 0; 1186 1187 rc = blob_serialize_xattr(xattr, 1188 *buf, *remaining_sz, 1189 &required_sz, internal); 1190 if (rc < 0) { 1191 /* Need to add a new page to the chain */ 1192 rc = blob_serialize_add_page(blob, pages, page_count, 1193 &cur_page); 1194 if (rc < 0) { 1195 spdk_free(*pages); 1196 *pages = NULL; 1197 *page_count = 0; 1198 return rc; 1199 } 1200 1201 *buf = (uint8_t *)cur_page->descriptors; 1202 *remaining_sz = sizeof(cur_page->descriptors); 1203 1204 /* Try again */ 1205 required_sz = 0; 1206 rc = blob_serialize_xattr(xattr, 1207 *buf, *remaining_sz, 1208 &required_sz, internal); 1209 1210 if (rc < 0) { 1211 spdk_free(*pages); 1212 *pages = NULL; 1213 *page_count = 0; 1214 return rc; 1215 } 1216 } 1217 1218 *remaining_sz -= required_sz; 1219 *buf += required_sz; 1220 } 1221 1222 return 0; 1223 } 1224 1225 static int 1226 blob_serialize(const struct spdk_blob *blob, struct spdk_blob_md_page **pages, 1227 uint32_t *page_count) 1228 { 1229 struct spdk_blob_md_page *cur_page; 1230 int rc; 1231 uint8_t *buf; 1232 size_t remaining_sz; 1233 1234 assert(pages != NULL); 1235 assert(page_count != NULL); 1236 assert(blob != NULL); 1237 assert(blob->state == SPDK_BLOB_STATE_DIRTY); 1238 1239 *pages = NULL; 1240 *page_count = 0; 1241 1242 /* A blob always has at least 1 page, even if it has no descriptors */ 1243 rc = blob_serialize_add_page(blob, pages, page_count, &cur_page); 1244 if (rc < 0) { 1245 return rc; 1246 } 1247 1248 buf = (uint8_t *)cur_page->descriptors; 1249 remaining_sz = sizeof(cur_page->descriptors); 1250 1251 /* Serialize flags */ 1252 blob_serialize_flags(blob, buf, &remaining_sz); 1253 buf += sizeof(struct spdk_blob_md_descriptor_flags); 1254 1255 /* Serialize xattrs */ 1256 rc = blob_serialize_xattrs(blob, &blob->xattrs, false, 1257 pages, cur_page, page_count, &buf, &remaining_sz); 1258 if (rc < 0) { 1259 return rc; 1260 } 1261 1262 /* Serialize internal xattrs */ 1263 rc = blob_serialize_xattrs(blob, &blob->xattrs_internal, true, 1264 pages, cur_page, page_count, &buf, &remaining_sz); 1265 if (rc < 0) { 1266 return rc; 1267 } 1268 1269 if (blob->use_extent_table) { 1270 /* Serialize extent table */ 1271 rc = blob_serialize_extent_table(blob, pages, cur_page, page_count, &buf, &remaining_sz); 1272 } else { 1273 /* Serialize extents */ 1274 rc = blob_serialize_extents_rle(blob, pages, cur_page, page_count, &buf, &remaining_sz); 1275 } 1276 1277 return rc; 1278 } 1279 1280 struct spdk_blob_load_ctx { 1281 struct spdk_blob *blob; 1282 1283 struct spdk_blob_md_page *pages; 1284 uint32_t num_pages; 1285 uint32_t next_extent_page; 1286 spdk_bs_sequence_t *seq; 1287 1288 spdk_bs_sequence_cpl cb_fn; 1289 void *cb_arg; 1290 }; 1291 1292 static uint32_t 1293 blob_md_page_calc_crc(void *page) 1294 { 1295 uint32_t crc; 1296 1297 crc = BLOB_CRC32C_INITIAL; 1298 crc = spdk_crc32c_update(page, SPDK_BS_PAGE_SIZE - 4, crc); 1299 crc ^= BLOB_CRC32C_INITIAL; 1300 1301 return crc; 1302 1303 } 1304 1305 static void 1306 blob_load_final(struct spdk_blob_load_ctx *ctx, int bserrno) 1307 { 1308 struct spdk_blob *blob = ctx->blob; 1309 1310 if (bserrno == 0) { 1311 blob_mark_clean(blob); 1312 } 1313 1314 ctx->cb_fn(ctx->seq, ctx->cb_arg, bserrno); 1315 1316 /* Free the memory */ 1317 spdk_free(ctx->pages); 1318 free(ctx); 1319 } 1320 1321 static void 1322 blob_load_snapshot_cpl(void *cb_arg, struct spdk_blob *snapshot, int bserrno) 1323 { 1324 struct spdk_blob_load_ctx *ctx = cb_arg; 1325 struct spdk_blob *blob = ctx->blob; 1326 1327 if (bserrno == 0) { 1328 blob->back_bs_dev = bs_create_blob_bs_dev(snapshot); 1329 if (blob->back_bs_dev == NULL) { 1330 bserrno = -ENOMEM; 1331 } 1332 } 1333 if (bserrno != 0) { 1334 SPDK_ERRLOG("Snapshot fail\n"); 1335 } 1336 1337 blob_load_final(ctx, bserrno); 1338 } 1339 1340 static void blob_update_clear_method(struct spdk_blob *blob); 1341 1342 static void 1343 blob_load_backing_dev(void *cb_arg) 1344 { 1345 struct spdk_blob_load_ctx *ctx = cb_arg; 1346 struct spdk_blob *blob = ctx->blob; 1347 const void *value; 1348 size_t len; 1349 int rc; 1350 1351 if (spdk_blob_is_thin_provisioned(blob)) { 1352 rc = blob_get_xattr_value(blob, BLOB_SNAPSHOT, &value, &len, true); 1353 if (rc == 0) { 1354 if (len != sizeof(spdk_blob_id)) { 1355 blob_load_final(ctx, -EINVAL); 1356 return; 1357 } 1358 /* open snapshot blob and continue in the callback function */ 1359 blob->parent_id = *(spdk_blob_id *)value; 1360 spdk_bs_open_blob(blob->bs, blob->parent_id, 1361 blob_load_snapshot_cpl, ctx); 1362 return; 1363 } else { 1364 /* add zeroes_dev for thin provisioned blob */ 1365 blob->back_bs_dev = bs_create_zeroes_dev(); 1366 } 1367 } else { 1368 /* standard blob */ 1369 blob->back_bs_dev = NULL; 1370 } 1371 blob_load_final(ctx, 0); 1372 } 1373 1374 static void 1375 blob_load_cpl_extents_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 1376 { 1377 struct spdk_blob_load_ctx *ctx = cb_arg; 1378 struct spdk_blob *blob = ctx->blob; 1379 struct spdk_blob_md_page *page; 1380 uint64_t i; 1381 uint32_t crc; 1382 uint64_t lba; 1383 void *tmp; 1384 uint64_t sz; 1385 1386 if (bserrno) { 1387 SPDK_ERRLOG("Extent page read failed: %d\n", bserrno); 1388 blob_load_final(ctx, bserrno); 1389 return; 1390 } 1391 1392 if (ctx->pages == NULL) { 1393 /* First iteration of this function, allocate buffer for single EXTENT_PAGE */ 1394 ctx->pages = spdk_zmalloc(SPDK_BS_PAGE_SIZE, 0, 1395 NULL, SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA); 1396 if (!ctx->pages) { 1397 blob_load_final(ctx, -ENOMEM); 1398 return; 1399 } 1400 ctx->num_pages = 1; 1401 ctx->next_extent_page = 0; 1402 } else { 1403 page = &ctx->pages[0]; 1404 crc = blob_md_page_calc_crc(page); 1405 if (crc != page->crc) { 1406 blob_load_final(ctx, -EINVAL); 1407 return; 1408 } 1409 1410 if (page->next != SPDK_INVALID_MD_PAGE) { 1411 blob_load_final(ctx, -EINVAL); 1412 return; 1413 } 1414 1415 bserrno = blob_parse_extent_page(page, blob); 1416 if (bserrno) { 1417 blob_load_final(ctx, bserrno); 1418 return; 1419 } 1420 } 1421 1422 for (i = ctx->next_extent_page; i < blob->active.num_extent_pages; i++) { 1423 if (blob->active.extent_pages[i] != 0) { 1424 /* Extent page was allocated, read and parse it. */ 1425 lba = bs_md_page_to_lba(blob->bs, blob->active.extent_pages[i]); 1426 ctx->next_extent_page = i + 1; 1427 1428 bs_sequence_read_dev(seq, &ctx->pages[0], lba, 1429 bs_byte_to_lba(blob->bs, SPDK_BS_PAGE_SIZE), 1430 blob_load_cpl_extents_cpl, ctx); 1431 return; 1432 } else { 1433 /* Thin provisioned blobs can point to unallocated extent pages. 1434 * In this case blob size should be increased by up to the amount left in remaining_clusters_in_et. */ 1435 1436 sz = spdk_min(blob->remaining_clusters_in_et, SPDK_EXTENTS_PER_EP); 1437 blob->active.num_clusters += sz; 1438 blob->remaining_clusters_in_et -= sz; 1439 1440 assert(spdk_blob_is_thin_provisioned(blob)); 1441 assert(i + 1 < blob->active.num_extent_pages || blob->remaining_clusters_in_et == 0); 1442 1443 tmp = realloc(blob->active.clusters, blob->active.num_clusters * sizeof(*blob->active.clusters)); 1444 if (tmp == NULL) { 1445 blob_load_final(ctx, -ENOMEM); 1446 return; 1447 } 1448 memset(tmp + sizeof(*blob->active.clusters) * blob->active.cluster_array_size, 0, 1449 sizeof(*blob->active.clusters) * (blob->active.num_clusters - blob->active.cluster_array_size)); 1450 blob->active.clusters = tmp; 1451 blob->active.cluster_array_size = blob->active.num_clusters; 1452 } 1453 } 1454 1455 blob_load_backing_dev(ctx); 1456 } 1457 1458 static void 1459 blob_load_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 1460 { 1461 struct spdk_blob_load_ctx *ctx = cb_arg; 1462 struct spdk_blob *blob = ctx->blob; 1463 struct spdk_blob_md_page *page; 1464 int rc; 1465 uint32_t crc; 1466 uint32_t current_page; 1467 1468 if (ctx->num_pages == 1) { 1469 current_page = bs_blobid_to_page(blob->id); 1470 } else { 1471 assert(ctx->num_pages != 0); 1472 page = &ctx->pages[ctx->num_pages - 2]; 1473 current_page = page->next; 1474 } 1475 1476 if (bserrno) { 1477 SPDK_ERRLOG("Metadata page %d read failed for blobid %" PRIu64 ": %d\n", 1478 current_page, blob->id, bserrno); 1479 blob_load_final(ctx, bserrno); 1480 return; 1481 } 1482 1483 page = &ctx->pages[ctx->num_pages - 1]; 1484 crc = blob_md_page_calc_crc(page); 1485 if (crc != page->crc) { 1486 SPDK_ERRLOG("Metadata page %d crc mismatch for blobid %" PRIu64 "\n", 1487 current_page, blob->id); 1488 blob_load_final(ctx, -EINVAL); 1489 return; 1490 } 1491 1492 if (page->next != SPDK_INVALID_MD_PAGE) { 1493 uint32_t next_page = page->next; 1494 uint64_t next_lba = bs_md_page_to_lba(blob->bs, next_page); 1495 1496 /* Read the next page */ 1497 ctx->num_pages++; 1498 ctx->pages = spdk_realloc(ctx->pages, (sizeof(*page) * ctx->num_pages), 0); 1499 if (ctx->pages == NULL) { 1500 blob_load_final(ctx, -ENOMEM); 1501 return; 1502 } 1503 1504 bs_sequence_read_dev(seq, &ctx->pages[ctx->num_pages - 1], 1505 next_lba, 1506 bs_byte_to_lba(blob->bs, sizeof(*page)), 1507 blob_load_cpl, ctx); 1508 return; 1509 } 1510 1511 /* Parse the pages */ 1512 rc = blob_parse(ctx->pages, ctx->num_pages, blob); 1513 if (rc) { 1514 blob_load_final(ctx, rc); 1515 return; 1516 } 1517 1518 if (blob->extent_table_found == true) { 1519 /* If EXTENT_TABLE was found, that means support for it should be enabled. */ 1520 assert(blob->extent_rle_found == false); 1521 blob->use_extent_table = true; 1522 } else { 1523 /* If EXTENT_RLE or no extent_* descriptor was found disable support 1524 * for extent table. No extent_* descriptors means that blob has length of 0 1525 * and no extent_rle descriptors were persisted for it. 1526 * EXTENT_TABLE if used, is always present in metadata regardless of length. */ 1527 blob->use_extent_table = false; 1528 } 1529 1530 /* Check the clear_method stored in metadata vs what may have been passed 1531 * via spdk_bs_open_blob_ext() and update accordingly. 1532 */ 1533 blob_update_clear_method(blob); 1534 1535 spdk_free(ctx->pages); 1536 ctx->pages = NULL; 1537 1538 if (blob->extent_table_found) { 1539 blob_load_cpl_extents_cpl(seq, ctx, 0); 1540 } else { 1541 blob_load_backing_dev(ctx); 1542 } 1543 } 1544 1545 /* Load a blob from disk given a blobid */ 1546 static void 1547 blob_load(spdk_bs_sequence_t *seq, struct spdk_blob *blob, 1548 spdk_bs_sequence_cpl cb_fn, void *cb_arg) 1549 { 1550 struct spdk_blob_load_ctx *ctx; 1551 struct spdk_blob_store *bs; 1552 uint32_t page_num; 1553 uint64_t lba; 1554 1555 blob_verify_md_op(blob); 1556 1557 bs = blob->bs; 1558 1559 ctx = calloc(1, sizeof(*ctx)); 1560 if (!ctx) { 1561 cb_fn(seq, cb_arg, -ENOMEM); 1562 return; 1563 } 1564 1565 ctx->blob = blob; 1566 ctx->pages = spdk_realloc(ctx->pages, SPDK_BS_PAGE_SIZE, 0); 1567 if (!ctx->pages) { 1568 free(ctx); 1569 cb_fn(seq, cb_arg, -ENOMEM); 1570 return; 1571 } 1572 ctx->num_pages = 1; 1573 ctx->cb_fn = cb_fn; 1574 ctx->cb_arg = cb_arg; 1575 ctx->seq = seq; 1576 1577 page_num = bs_blobid_to_page(blob->id); 1578 lba = bs_md_page_to_lba(blob->bs, page_num); 1579 1580 blob->state = SPDK_BLOB_STATE_LOADING; 1581 1582 bs_sequence_read_dev(seq, &ctx->pages[0], lba, 1583 bs_byte_to_lba(bs, SPDK_BS_PAGE_SIZE), 1584 blob_load_cpl, ctx); 1585 } 1586 1587 struct spdk_blob_persist_ctx { 1588 struct spdk_blob *blob; 1589 1590 struct spdk_bs_super_block *super; 1591 1592 struct spdk_blob_md_page *pages; 1593 uint32_t next_extent_page; 1594 struct spdk_blob_md_page *extent_page; 1595 1596 spdk_bs_sequence_t *seq; 1597 spdk_bs_sequence_cpl cb_fn; 1598 void *cb_arg; 1599 TAILQ_ENTRY(spdk_blob_persist_ctx) link; 1600 }; 1601 1602 static void 1603 bs_batch_clear_dev(struct spdk_blob_persist_ctx *ctx, spdk_bs_batch_t *batch, uint64_t lba, 1604 uint32_t lba_count) 1605 { 1606 switch (ctx->blob->clear_method) { 1607 case BLOB_CLEAR_WITH_DEFAULT: 1608 case BLOB_CLEAR_WITH_UNMAP: 1609 bs_batch_unmap_dev(batch, lba, lba_count); 1610 break; 1611 case BLOB_CLEAR_WITH_WRITE_ZEROES: 1612 bs_batch_write_zeroes_dev(batch, lba, lba_count); 1613 break; 1614 case BLOB_CLEAR_WITH_NONE: 1615 default: 1616 break; 1617 } 1618 } 1619 1620 static void blob_persist_check_dirty(struct spdk_blob_persist_ctx *ctx); 1621 1622 static void 1623 blob_persist_complete(spdk_bs_sequence_t *seq, struct spdk_blob_persist_ctx *ctx, int bserrno) 1624 { 1625 struct spdk_blob_persist_ctx *next_persist; 1626 struct spdk_blob *blob = ctx->blob; 1627 1628 if (bserrno == 0) { 1629 blob_mark_clean(blob); 1630 } 1631 1632 assert(ctx == TAILQ_FIRST(&blob->pending_persists)); 1633 TAILQ_REMOVE(&blob->pending_persists, ctx, link); 1634 1635 next_persist = TAILQ_FIRST(&blob->pending_persists); 1636 1637 /* Call user callback */ 1638 ctx->cb_fn(seq, ctx->cb_arg, bserrno); 1639 1640 /* Free the memory */ 1641 spdk_free(ctx->pages); 1642 free(ctx); 1643 1644 if (next_persist != NULL) { 1645 blob_persist_check_dirty(next_persist); 1646 } 1647 } 1648 1649 static void 1650 blob_persist_clear_extents_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 1651 { 1652 struct spdk_blob_persist_ctx *ctx = cb_arg; 1653 struct spdk_blob *blob = ctx->blob; 1654 struct spdk_blob_store *bs = blob->bs; 1655 size_t i; 1656 1657 if (bserrno != 0) { 1658 blob_persist_complete(seq, ctx, bserrno); 1659 return; 1660 } 1661 1662 /* Release all extent_pages that were truncated */ 1663 for (i = blob->active.num_extent_pages; i < blob->active.extent_pages_array_size; i++) { 1664 /* Nothing to release if it was not allocated */ 1665 if (blob->active.extent_pages[i] != 0) { 1666 bs_release_md_page(bs, blob->active.extent_pages[i]); 1667 } 1668 } 1669 1670 if (blob->active.num_extent_pages == 0) { 1671 free(blob->active.extent_pages); 1672 blob->active.extent_pages = NULL; 1673 blob->active.extent_pages_array_size = 0; 1674 } else if (blob->active.num_extent_pages != blob->active.extent_pages_array_size) { 1675 #ifndef __clang_analyzer__ 1676 void *tmp; 1677 1678 /* scan-build really can't figure reallocs, workaround it */ 1679 tmp = realloc(blob->active.extent_pages, sizeof(uint32_t) * blob->active.num_extent_pages); 1680 assert(tmp != NULL); 1681 blob->active.extent_pages = tmp; 1682 #endif 1683 blob->active.extent_pages_array_size = blob->active.num_extent_pages; 1684 } 1685 1686 blob_persist_complete(seq, ctx, bserrno); 1687 } 1688 1689 static void 1690 blob_persist_clear_extents(spdk_bs_sequence_t *seq, struct spdk_blob_persist_ctx *ctx) 1691 { 1692 struct spdk_blob *blob = ctx->blob; 1693 struct spdk_blob_store *bs = blob->bs; 1694 size_t i; 1695 uint64_t lba; 1696 uint32_t lba_count; 1697 spdk_bs_batch_t *batch; 1698 1699 batch = bs_sequence_to_batch(seq, blob_persist_clear_extents_cpl, ctx); 1700 lba_count = bs_byte_to_lba(bs, SPDK_BS_PAGE_SIZE); 1701 1702 /* Clear all extent_pages that were truncated */ 1703 for (i = blob->active.num_extent_pages; i < blob->active.extent_pages_array_size; i++) { 1704 /* Nothing to clear if it was not allocated */ 1705 if (blob->active.extent_pages[i] != 0) { 1706 lba = bs_md_page_to_lba(bs, blob->active.extent_pages[i]); 1707 bs_batch_write_zeroes_dev(batch, lba, lba_count); 1708 } 1709 } 1710 1711 bs_batch_close(batch); 1712 } 1713 1714 static void 1715 blob_persist_clear_clusters_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 1716 { 1717 struct spdk_blob_persist_ctx *ctx = cb_arg; 1718 struct spdk_blob *blob = ctx->blob; 1719 struct spdk_blob_store *bs = blob->bs; 1720 size_t i; 1721 1722 if (bserrno != 0) { 1723 blob_persist_complete(seq, ctx, bserrno); 1724 return; 1725 } 1726 1727 pthread_mutex_lock(&bs->used_clusters_mutex); 1728 /* Release all clusters that were truncated */ 1729 for (i = blob->active.num_clusters; i < blob->active.cluster_array_size; i++) { 1730 uint32_t cluster_num = bs_lba_to_cluster(bs, blob->active.clusters[i]); 1731 1732 /* Nothing to release if it was not allocated */ 1733 if (blob->active.clusters[i] != 0) { 1734 bs_release_cluster(bs, cluster_num); 1735 } 1736 } 1737 pthread_mutex_unlock(&bs->used_clusters_mutex); 1738 1739 if (blob->active.num_clusters == 0) { 1740 free(blob->active.clusters); 1741 blob->active.clusters = NULL; 1742 blob->active.cluster_array_size = 0; 1743 } else if (blob->active.num_clusters != blob->active.cluster_array_size) { 1744 #ifndef __clang_analyzer__ 1745 void *tmp; 1746 1747 /* scan-build really can't figure reallocs, workaround it */ 1748 tmp = realloc(blob->active.clusters, sizeof(*blob->active.clusters) * blob->active.num_clusters); 1749 assert(tmp != NULL); 1750 blob->active.clusters = tmp; 1751 1752 #endif 1753 blob->active.cluster_array_size = blob->active.num_clusters; 1754 } 1755 1756 /* Move on to clearing extent pages */ 1757 blob_persist_clear_extents(seq, ctx); 1758 } 1759 1760 static void 1761 blob_persist_clear_clusters(spdk_bs_sequence_t *seq, struct spdk_blob_persist_ctx *ctx) 1762 { 1763 struct spdk_blob *blob = ctx->blob; 1764 struct spdk_blob_store *bs = blob->bs; 1765 spdk_bs_batch_t *batch; 1766 size_t i; 1767 uint64_t lba; 1768 uint32_t lba_count; 1769 1770 /* Clusters don't move around in blobs. The list shrinks or grows 1771 * at the end, but no changes ever occur in the middle of the list. 1772 */ 1773 1774 batch = bs_sequence_to_batch(seq, blob_persist_clear_clusters_cpl, ctx); 1775 1776 /* Clear all clusters that were truncated */ 1777 lba = 0; 1778 lba_count = 0; 1779 for (i = blob->active.num_clusters; i < blob->active.cluster_array_size; i++) { 1780 uint64_t next_lba = blob->active.clusters[i]; 1781 uint32_t next_lba_count = bs_cluster_to_lba(bs, 1); 1782 1783 if (next_lba > 0 && (lba + lba_count) == next_lba) { 1784 /* This cluster is contiguous with the previous one. */ 1785 lba_count += next_lba_count; 1786 continue; 1787 } else if (next_lba == 0) { 1788 continue; 1789 } 1790 1791 /* This cluster is not contiguous with the previous one. */ 1792 1793 /* If a run of LBAs previously existing, clear them now */ 1794 if (lba_count > 0) { 1795 bs_batch_clear_dev(ctx, batch, lba, lba_count); 1796 } 1797 1798 /* Start building the next batch */ 1799 lba = next_lba; 1800 if (next_lba > 0) { 1801 lba_count = next_lba_count; 1802 } else { 1803 lba_count = 0; 1804 } 1805 } 1806 1807 /* If we ended with a contiguous set of LBAs, clear them now */ 1808 if (lba_count > 0) { 1809 bs_batch_clear_dev(ctx, batch, lba, lba_count); 1810 } 1811 1812 bs_batch_close(batch); 1813 } 1814 1815 static void 1816 blob_persist_zero_pages_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 1817 { 1818 struct spdk_blob_persist_ctx *ctx = cb_arg; 1819 struct spdk_blob *blob = ctx->blob; 1820 struct spdk_blob_store *bs = blob->bs; 1821 size_t i; 1822 1823 if (bserrno != 0) { 1824 blob_persist_complete(seq, ctx, bserrno); 1825 return; 1826 } 1827 1828 /* This loop starts at 1 because the first page is special and handled 1829 * below. The pages (except the first) are never written in place, 1830 * so any pages in the clean list must be zeroed. 1831 */ 1832 for (i = 1; i < blob->clean.num_pages; i++) { 1833 bs_release_md_page(bs, blob->clean.pages[i]); 1834 } 1835 1836 if (blob->active.num_pages == 0) { 1837 uint32_t page_num; 1838 1839 page_num = bs_blobid_to_page(blob->id); 1840 bs_release_md_page(bs, page_num); 1841 } 1842 1843 /* Move on to clearing clusters */ 1844 blob_persist_clear_clusters(seq, ctx); 1845 } 1846 1847 static void 1848 blob_persist_zero_pages(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 1849 { 1850 struct spdk_blob_persist_ctx *ctx = cb_arg; 1851 struct spdk_blob *blob = ctx->blob; 1852 struct spdk_blob_store *bs = blob->bs; 1853 uint64_t lba; 1854 uint32_t lba_count; 1855 spdk_bs_batch_t *batch; 1856 size_t i; 1857 1858 if (bserrno != 0) { 1859 blob_persist_complete(seq, ctx, bserrno); 1860 return; 1861 } 1862 1863 batch = bs_sequence_to_batch(seq, blob_persist_zero_pages_cpl, ctx); 1864 1865 lba_count = bs_byte_to_lba(bs, SPDK_BS_PAGE_SIZE); 1866 1867 /* This loop starts at 1 because the first page is special and handled 1868 * below. The pages (except the first) are never written in place, 1869 * so any pages in the clean list must be zeroed. 1870 */ 1871 for (i = 1; i < blob->clean.num_pages; i++) { 1872 lba = bs_md_page_to_lba(bs, blob->clean.pages[i]); 1873 1874 bs_batch_write_zeroes_dev(batch, lba, lba_count); 1875 } 1876 1877 /* The first page will only be zeroed if this is a delete. */ 1878 if (blob->active.num_pages == 0) { 1879 uint32_t page_num; 1880 1881 /* The first page in the metadata goes where the blobid indicates */ 1882 page_num = bs_blobid_to_page(blob->id); 1883 lba = bs_md_page_to_lba(bs, page_num); 1884 1885 bs_batch_write_zeroes_dev(batch, lba, lba_count); 1886 } 1887 1888 bs_batch_close(batch); 1889 } 1890 1891 static void 1892 blob_persist_write_page_root(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 1893 { 1894 struct spdk_blob_persist_ctx *ctx = cb_arg; 1895 struct spdk_blob *blob = ctx->blob; 1896 struct spdk_blob_store *bs = blob->bs; 1897 uint64_t lba; 1898 uint32_t lba_count; 1899 struct spdk_blob_md_page *page; 1900 1901 if (bserrno != 0) { 1902 blob_persist_complete(seq, ctx, bserrno); 1903 return; 1904 } 1905 1906 if (blob->active.num_pages == 0) { 1907 /* Move on to the next step */ 1908 blob_persist_zero_pages(seq, ctx, 0); 1909 return; 1910 } 1911 1912 lba_count = bs_byte_to_lba(bs, sizeof(*page)); 1913 1914 page = &ctx->pages[0]; 1915 /* The first page in the metadata goes where the blobid indicates */ 1916 lba = bs_md_page_to_lba(bs, bs_blobid_to_page(blob->id)); 1917 1918 bs_sequence_write_dev(seq, page, lba, lba_count, 1919 blob_persist_zero_pages, ctx); 1920 } 1921 1922 static void 1923 blob_persist_write_page_chain(spdk_bs_sequence_t *seq, struct spdk_blob_persist_ctx *ctx) 1924 { 1925 struct spdk_blob *blob = ctx->blob; 1926 struct spdk_blob_store *bs = blob->bs; 1927 uint64_t lba; 1928 uint32_t lba_count; 1929 struct spdk_blob_md_page *page; 1930 spdk_bs_batch_t *batch; 1931 size_t i; 1932 1933 /* Clusters don't move around in blobs. The list shrinks or grows 1934 * at the end, but no changes ever occur in the middle of the list. 1935 */ 1936 1937 lba_count = bs_byte_to_lba(bs, sizeof(*page)); 1938 1939 batch = bs_sequence_to_batch(seq, blob_persist_write_page_root, ctx); 1940 1941 /* This starts at 1. The root page is not written until 1942 * all of the others are finished 1943 */ 1944 for (i = 1; i < blob->active.num_pages; i++) { 1945 page = &ctx->pages[i]; 1946 assert(page->sequence_num == i); 1947 1948 lba = bs_md_page_to_lba(bs, blob->active.pages[i]); 1949 1950 bs_batch_write_dev(batch, page, lba, lba_count); 1951 } 1952 1953 bs_batch_close(batch); 1954 } 1955 1956 static int 1957 blob_resize(struct spdk_blob *blob, uint64_t sz) 1958 { 1959 uint64_t i; 1960 uint64_t *tmp; 1961 uint64_t cluster; 1962 uint32_t lfmd; /* lowest free md page */ 1963 uint64_t num_clusters; 1964 uint32_t *ep_tmp; 1965 uint64_t new_num_ep = 0, current_num_ep = 0; 1966 struct spdk_blob_store *bs; 1967 1968 bs = blob->bs; 1969 1970 blob_verify_md_op(blob); 1971 1972 if (blob->active.num_clusters == sz) { 1973 return 0; 1974 } 1975 1976 if (blob->active.num_clusters < blob->active.cluster_array_size) { 1977 /* If this blob was resized to be larger, then smaller, then 1978 * larger without syncing, then the cluster array already 1979 * contains spare assigned clusters we can use. 1980 */ 1981 num_clusters = spdk_min(blob->active.cluster_array_size, 1982 sz); 1983 } else { 1984 num_clusters = blob->active.num_clusters; 1985 } 1986 1987 if (blob->use_extent_table) { 1988 /* Round up since every cluster beyond current Extent Table size, 1989 * requires new extent page. */ 1990 new_num_ep = spdk_divide_round_up(sz, SPDK_EXTENTS_PER_EP); 1991 current_num_ep = spdk_divide_round_up(num_clusters, SPDK_EXTENTS_PER_EP); 1992 } 1993 1994 /* Check first that we have enough clusters and md pages before we start claiming them. */ 1995 if (sz > num_clusters && spdk_blob_is_thin_provisioned(blob) == false) { 1996 if ((sz - num_clusters) > bs->num_free_clusters) { 1997 return -ENOSPC; 1998 } 1999 lfmd = 0; 2000 for (i = current_num_ep; i < new_num_ep ; i++) { 2001 lfmd = spdk_bit_array_find_first_clear(blob->bs->used_md_pages, lfmd); 2002 if (lfmd == UINT32_MAX) { 2003 /* No more free md pages. Cannot satisfy the request */ 2004 return -ENOSPC; 2005 } 2006 } 2007 } 2008 2009 if (sz > num_clusters) { 2010 /* Expand the cluster array if necessary. 2011 * We only shrink the array when persisting. 2012 */ 2013 tmp = realloc(blob->active.clusters, sizeof(*blob->active.clusters) * sz); 2014 if (sz > 0 && tmp == NULL) { 2015 return -ENOMEM; 2016 } 2017 memset(tmp + blob->active.cluster_array_size, 0, 2018 sizeof(*blob->active.clusters) * (sz - blob->active.cluster_array_size)); 2019 blob->active.clusters = tmp; 2020 blob->active.cluster_array_size = sz; 2021 2022 /* Expand the extents table, only if enough clusters were added */ 2023 if (new_num_ep > current_num_ep && blob->use_extent_table) { 2024 ep_tmp = realloc(blob->active.extent_pages, sizeof(*blob->active.extent_pages) * new_num_ep); 2025 if (new_num_ep > 0 && ep_tmp == NULL) { 2026 return -ENOMEM; 2027 } 2028 memset(ep_tmp + blob->active.extent_pages_array_size, 0, 2029 sizeof(*blob->active.extent_pages) * (new_num_ep - blob->active.extent_pages_array_size)); 2030 blob->active.extent_pages = ep_tmp; 2031 blob->active.extent_pages_array_size = new_num_ep; 2032 } 2033 } 2034 2035 blob->state = SPDK_BLOB_STATE_DIRTY; 2036 2037 if (spdk_blob_is_thin_provisioned(blob) == false) { 2038 cluster = 0; 2039 lfmd = 0; 2040 pthread_mutex_lock(&blob->bs->used_clusters_mutex); 2041 for (i = num_clusters; i < sz; i++) { 2042 bs_allocate_cluster(blob, i, &cluster, &lfmd, true); 2043 lfmd++; 2044 } 2045 pthread_mutex_unlock(&blob->bs->used_clusters_mutex); 2046 } 2047 2048 blob->active.num_clusters = sz; 2049 blob->active.num_extent_pages = new_num_ep; 2050 2051 return 0; 2052 } 2053 2054 static void 2055 blob_persist_generate_new_md(struct spdk_blob_persist_ctx *ctx) 2056 { 2057 spdk_bs_sequence_t *seq = ctx->seq; 2058 struct spdk_blob *blob = ctx->blob; 2059 struct spdk_blob_store *bs = blob->bs; 2060 uint64_t i; 2061 uint32_t page_num; 2062 void *tmp; 2063 int rc; 2064 2065 /* Generate the new metadata */ 2066 rc = blob_serialize(blob, &ctx->pages, &blob->active.num_pages); 2067 if (rc < 0) { 2068 blob_persist_complete(seq, ctx, rc); 2069 return; 2070 } 2071 2072 assert(blob->active.num_pages >= 1); 2073 2074 /* Resize the cache of page indices */ 2075 tmp = realloc(blob->active.pages, blob->active.num_pages * sizeof(*blob->active.pages)); 2076 if (!tmp) { 2077 blob_persist_complete(seq, ctx, -ENOMEM); 2078 return; 2079 } 2080 blob->active.pages = tmp; 2081 2082 /* Assign this metadata to pages. This requires two passes - 2083 * one to verify that there are enough pages and a second 2084 * to actually claim them. */ 2085 page_num = 0; 2086 /* Note that this loop starts at one. The first page location is fixed by the blobid. */ 2087 for (i = 1; i < blob->active.num_pages; i++) { 2088 page_num = spdk_bit_array_find_first_clear(bs->used_md_pages, page_num); 2089 if (page_num == UINT32_MAX) { 2090 blob_persist_complete(seq, ctx, -ENOMEM); 2091 return; 2092 } 2093 page_num++; 2094 } 2095 2096 page_num = 0; 2097 blob->active.pages[0] = bs_blobid_to_page(blob->id); 2098 for (i = 1; i < blob->active.num_pages; i++) { 2099 page_num = spdk_bit_array_find_first_clear(bs->used_md_pages, page_num); 2100 ctx->pages[i - 1].next = page_num; 2101 /* Now that previous metadata page is complete, calculate the crc for it. */ 2102 ctx->pages[i - 1].crc = blob_md_page_calc_crc(&ctx->pages[i - 1]); 2103 blob->active.pages[i] = page_num; 2104 bs_claim_md_page(bs, page_num); 2105 SPDK_DEBUGLOG(blob, "Claiming page %u for blob %" PRIu64 "\n", page_num, blob->id); 2106 page_num++; 2107 } 2108 ctx->pages[i - 1].crc = blob_md_page_calc_crc(&ctx->pages[i - 1]); 2109 /* Start writing the metadata from last page to first */ 2110 blob->state = SPDK_BLOB_STATE_CLEAN; 2111 blob_persist_write_page_chain(seq, ctx); 2112 } 2113 2114 static void 2115 blob_persist_write_extent_pages(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 2116 { 2117 struct spdk_blob_persist_ctx *ctx = cb_arg; 2118 struct spdk_blob *blob = ctx->blob; 2119 size_t i; 2120 uint32_t extent_page_id; 2121 uint32_t page_count = 0; 2122 int rc; 2123 2124 if (ctx->extent_page != NULL) { 2125 spdk_free(ctx->extent_page); 2126 ctx->extent_page = NULL; 2127 } 2128 2129 if (bserrno != 0) { 2130 blob_persist_complete(seq, ctx, bserrno); 2131 return; 2132 } 2133 2134 /* Only write out changed extent pages */ 2135 for (i = ctx->next_extent_page; i < blob->active.num_extent_pages; i++) { 2136 extent_page_id = blob->active.extent_pages[i]; 2137 if (extent_page_id == 0) { 2138 /* No Extent Page to persist */ 2139 assert(spdk_blob_is_thin_provisioned(blob)); 2140 continue; 2141 } 2142 /* Writing out new extent page for the first time. Either active extent pages is larger 2143 * than clean extent pages or there was no extent page assigned due to thin provisioning. */ 2144 if (i >= blob->clean.extent_pages_array_size || blob->clean.extent_pages[i] == 0) { 2145 blob->state = SPDK_BLOB_STATE_DIRTY; 2146 assert(spdk_bit_array_get(blob->bs->used_md_pages, extent_page_id)); 2147 ctx->next_extent_page = i + 1; 2148 rc = blob_serialize_add_page(ctx->blob, &ctx->extent_page, &page_count, &ctx->extent_page); 2149 if (rc < 0) { 2150 blob_persist_complete(seq, ctx, rc); 2151 return; 2152 } 2153 2154 blob_serialize_extent_page(blob, i * SPDK_EXTENTS_PER_EP, ctx->extent_page); 2155 2156 ctx->extent_page->crc = blob_md_page_calc_crc(ctx->extent_page); 2157 2158 bs_sequence_write_dev(seq, ctx->extent_page, bs_md_page_to_lba(blob->bs, extent_page_id), 2159 bs_byte_to_lba(blob->bs, SPDK_BS_PAGE_SIZE), 2160 blob_persist_write_extent_pages, ctx); 2161 return; 2162 } 2163 assert(blob->clean.extent_pages[i] != 0); 2164 } 2165 2166 blob_persist_generate_new_md(ctx); 2167 } 2168 2169 static void 2170 blob_persist_start(struct spdk_blob_persist_ctx *ctx) 2171 { 2172 spdk_bs_sequence_t *seq = ctx->seq; 2173 struct spdk_blob *blob = ctx->blob; 2174 2175 if (blob->active.num_pages == 0) { 2176 /* This is the signal that the blob should be deleted. 2177 * Immediately jump to the clean up routine. */ 2178 assert(blob->clean.num_pages > 0); 2179 blob->state = SPDK_BLOB_STATE_CLEAN; 2180 blob_persist_zero_pages(seq, ctx, 0); 2181 return; 2182 2183 } 2184 2185 blob_persist_write_extent_pages(seq, ctx, 0); 2186 } 2187 2188 static void 2189 blob_persist_dirty_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 2190 { 2191 struct spdk_blob_persist_ctx *ctx = cb_arg; 2192 2193 spdk_free(ctx->super); 2194 2195 if (bserrno != 0) { 2196 blob_persist_complete(seq, ctx, bserrno); 2197 return; 2198 } 2199 2200 ctx->blob->bs->clean = 0; 2201 2202 blob_persist_start(ctx); 2203 } 2204 2205 static void 2206 bs_write_super(spdk_bs_sequence_t *seq, struct spdk_blob_store *bs, 2207 struct spdk_bs_super_block *super, spdk_bs_sequence_cpl cb_fn, void *cb_arg); 2208 2209 2210 static void 2211 blob_persist_dirty(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 2212 { 2213 struct spdk_blob_persist_ctx *ctx = cb_arg; 2214 2215 if (bserrno != 0) { 2216 spdk_free(ctx->super); 2217 blob_persist_complete(seq, ctx, bserrno); 2218 return; 2219 } 2220 2221 ctx->super->clean = 0; 2222 if (ctx->super->size == 0) { 2223 ctx->super->size = ctx->blob->bs->dev->blockcnt * ctx->blob->bs->dev->blocklen; 2224 } 2225 2226 bs_write_super(seq, ctx->blob->bs, ctx->super, blob_persist_dirty_cpl, ctx); 2227 } 2228 2229 static void 2230 blob_persist_check_dirty(struct spdk_blob_persist_ctx *ctx) 2231 { 2232 if (ctx->blob->bs->clean) { 2233 ctx->super = spdk_zmalloc(sizeof(*ctx->super), 0x1000, NULL, 2234 SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA); 2235 if (!ctx->super) { 2236 blob_persist_complete(ctx->seq, ctx, -ENOMEM); 2237 return; 2238 } 2239 2240 bs_sequence_read_dev(ctx->seq, ctx->super, bs_page_to_lba(ctx->blob->bs, 0), 2241 bs_byte_to_lba(ctx->blob->bs, sizeof(*ctx->super)), 2242 blob_persist_dirty, ctx); 2243 } else { 2244 blob_persist_start(ctx); 2245 } 2246 } 2247 2248 /* Write a blob to disk */ 2249 static void 2250 blob_persist(spdk_bs_sequence_t *seq, struct spdk_blob *blob, 2251 spdk_bs_sequence_cpl cb_fn, void *cb_arg) 2252 { 2253 struct spdk_blob_persist_ctx *ctx; 2254 2255 blob_verify_md_op(blob); 2256 2257 if (blob->state == SPDK_BLOB_STATE_CLEAN && TAILQ_EMPTY(&blob->pending_persists)) { 2258 cb_fn(seq, cb_arg, 0); 2259 return; 2260 } 2261 2262 ctx = calloc(1, sizeof(*ctx)); 2263 if (!ctx) { 2264 cb_fn(seq, cb_arg, -ENOMEM); 2265 return; 2266 } 2267 ctx->blob = blob; 2268 ctx->seq = seq; 2269 ctx->cb_fn = cb_fn; 2270 ctx->cb_arg = cb_arg; 2271 ctx->next_extent_page = 0; 2272 2273 /* Multiple blob persists can affect one another, via blob->state or 2274 * blob mutable data changes. To prevent it, queue up the persists. */ 2275 if (!TAILQ_EMPTY(&blob->pending_persists)) { 2276 TAILQ_INSERT_TAIL(&blob->pending_persists, ctx, link); 2277 return; 2278 } 2279 TAILQ_INSERT_HEAD(&blob->pending_persists, ctx, link); 2280 2281 blob_persist_check_dirty(ctx); 2282 } 2283 2284 struct spdk_blob_copy_cluster_ctx { 2285 struct spdk_blob *blob; 2286 uint8_t *buf; 2287 uint64_t page; 2288 uint64_t new_cluster; 2289 uint32_t new_extent_page; 2290 spdk_bs_sequence_t *seq; 2291 }; 2292 2293 static void 2294 blob_allocate_and_copy_cluster_cpl(void *cb_arg, int bserrno) 2295 { 2296 struct spdk_blob_copy_cluster_ctx *ctx = cb_arg; 2297 struct spdk_bs_request_set *set = (struct spdk_bs_request_set *)ctx->seq; 2298 TAILQ_HEAD(, spdk_bs_request_set) requests; 2299 spdk_bs_user_op_t *op; 2300 2301 TAILQ_INIT(&requests); 2302 TAILQ_SWAP(&set->channel->need_cluster_alloc, &requests, spdk_bs_request_set, link); 2303 2304 while (!TAILQ_EMPTY(&requests)) { 2305 op = TAILQ_FIRST(&requests); 2306 TAILQ_REMOVE(&requests, op, link); 2307 if (bserrno == 0) { 2308 bs_user_op_execute(op); 2309 } else { 2310 bs_user_op_abort(op); 2311 } 2312 } 2313 2314 spdk_free(ctx->buf); 2315 free(ctx); 2316 } 2317 2318 static void 2319 blob_insert_cluster_cpl(void *cb_arg, int bserrno) 2320 { 2321 struct spdk_blob_copy_cluster_ctx *ctx = cb_arg; 2322 2323 if (bserrno) { 2324 if (bserrno == -EEXIST) { 2325 /* The metadata insert failed because another thread 2326 * allocated the cluster first. Free our cluster 2327 * but continue without error. */ 2328 bserrno = 0; 2329 } 2330 pthread_mutex_lock(&ctx->blob->bs->used_clusters_mutex); 2331 bs_release_cluster(ctx->blob->bs, ctx->new_cluster); 2332 pthread_mutex_unlock(&ctx->blob->bs->used_clusters_mutex); 2333 if (ctx->new_extent_page != 0) { 2334 bs_release_md_page(ctx->blob->bs, ctx->new_extent_page); 2335 } 2336 } 2337 2338 bs_sequence_finish(ctx->seq, bserrno); 2339 } 2340 2341 static void 2342 blob_write_copy_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 2343 { 2344 struct spdk_blob_copy_cluster_ctx *ctx = cb_arg; 2345 uint32_t cluster_number; 2346 2347 if (bserrno) { 2348 /* The write failed, so jump to the final completion handler */ 2349 bs_sequence_finish(seq, bserrno); 2350 return; 2351 } 2352 2353 cluster_number = bs_page_to_cluster(ctx->blob->bs, ctx->page); 2354 2355 blob_insert_cluster_on_md_thread(ctx->blob, cluster_number, ctx->new_cluster, 2356 ctx->new_extent_page, blob_insert_cluster_cpl, ctx); 2357 } 2358 2359 static void 2360 blob_write_copy(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 2361 { 2362 struct spdk_blob_copy_cluster_ctx *ctx = cb_arg; 2363 2364 if (bserrno != 0) { 2365 /* The read failed, so jump to the final completion handler */ 2366 bs_sequence_finish(seq, bserrno); 2367 return; 2368 } 2369 2370 /* Write whole cluster */ 2371 bs_sequence_write_dev(seq, ctx->buf, 2372 bs_cluster_to_lba(ctx->blob->bs, ctx->new_cluster), 2373 bs_cluster_to_lba(ctx->blob->bs, 1), 2374 blob_write_copy_cpl, ctx); 2375 } 2376 2377 static void 2378 bs_allocate_and_copy_cluster(struct spdk_blob *blob, 2379 struct spdk_io_channel *_ch, 2380 uint64_t io_unit, spdk_bs_user_op_t *op) 2381 { 2382 struct spdk_bs_cpl cpl; 2383 struct spdk_bs_channel *ch; 2384 struct spdk_blob_copy_cluster_ctx *ctx; 2385 uint32_t cluster_start_page; 2386 uint32_t cluster_number; 2387 int rc; 2388 2389 ch = spdk_io_channel_get_ctx(_ch); 2390 2391 if (!TAILQ_EMPTY(&ch->need_cluster_alloc)) { 2392 /* There are already operations pending. Queue this user op 2393 * and return because it will be re-executed when the outstanding 2394 * cluster allocation completes. */ 2395 TAILQ_INSERT_TAIL(&ch->need_cluster_alloc, op, link); 2396 return; 2397 } 2398 2399 /* Round the io_unit offset down to the first page in the cluster */ 2400 cluster_start_page = bs_io_unit_to_cluster_start(blob, io_unit); 2401 2402 /* Calculate which index in the metadata cluster array the corresponding 2403 * cluster is supposed to be at. */ 2404 cluster_number = bs_io_unit_to_cluster_number(blob, io_unit); 2405 2406 ctx = calloc(1, sizeof(*ctx)); 2407 if (!ctx) { 2408 bs_user_op_abort(op); 2409 return; 2410 } 2411 2412 assert(blob->bs->cluster_sz % blob->back_bs_dev->blocklen == 0); 2413 2414 ctx->blob = blob; 2415 ctx->page = cluster_start_page; 2416 2417 if (blob->parent_id != SPDK_BLOBID_INVALID) { 2418 ctx->buf = spdk_malloc(blob->bs->cluster_sz, blob->back_bs_dev->blocklen, 2419 NULL, SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA); 2420 if (!ctx->buf) { 2421 SPDK_ERRLOG("DMA allocation for cluster of size = %" PRIu32 " failed.\n", 2422 blob->bs->cluster_sz); 2423 free(ctx); 2424 bs_user_op_abort(op); 2425 return; 2426 } 2427 } 2428 2429 pthread_mutex_lock(&blob->bs->used_clusters_mutex); 2430 rc = bs_allocate_cluster(blob, cluster_number, &ctx->new_cluster, &ctx->new_extent_page, 2431 false); 2432 pthread_mutex_unlock(&blob->bs->used_clusters_mutex); 2433 if (rc != 0) { 2434 spdk_free(ctx->buf); 2435 free(ctx); 2436 bs_user_op_abort(op); 2437 return; 2438 } 2439 2440 cpl.type = SPDK_BS_CPL_TYPE_BLOB_BASIC; 2441 cpl.u.blob_basic.cb_fn = blob_allocate_and_copy_cluster_cpl; 2442 cpl.u.blob_basic.cb_arg = ctx; 2443 2444 ctx->seq = bs_sequence_start(_ch, &cpl); 2445 if (!ctx->seq) { 2446 pthread_mutex_lock(&blob->bs->used_clusters_mutex); 2447 bs_release_cluster(blob->bs, ctx->new_cluster); 2448 pthread_mutex_unlock(&blob->bs->used_clusters_mutex); 2449 spdk_free(ctx->buf); 2450 free(ctx); 2451 bs_user_op_abort(op); 2452 return; 2453 } 2454 2455 /* Queue the user op to block other incoming operations */ 2456 TAILQ_INSERT_TAIL(&ch->need_cluster_alloc, op, link); 2457 2458 if (blob->parent_id != SPDK_BLOBID_INVALID) { 2459 /* Read cluster from backing device */ 2460 bs_sequence_read_bs_dev(ctx->seq, blob->back_bs_dev, ctx->buf, 2461 bs_dev_page_to_lba(blob->back_bs_dev, cluster_start_page), 2462 bs_dev_byte_to_lba(blob->back_bs_dev, blob->bs->cluster_sz), 2463 blob_write_copy, ctx); 2464 } else { 2465 blob_insert_cluster_on_md_thread(ctx->blob, cluster_number, ctx->new_cluster, 2466 ctx->new_extent_page, blob_insert_cluster_cpl, ctx); 2467 } 2468 } 2469 2470 static inline bool 2471 blob_calculate_lba_and_lba_count(struct spdk_blob *blob, uint64_t io_unit, uint64_t length, 2472 uint64_t *lba, uint32_t *lba_count) 2473 { 2474 *lba_count = length; 2475 2476 if (!bs_io_unit_is_allocated(blob, io_unit)) { 2477 assert(blob->back_bs_dev != NULL); 2478 *lba = bs_io_unit_to_back_dev_lba(blob, io_unit); 2479 *lba_count = bs_io_unit_to_back_dev_lba(blob, *lba_count); 2480 return false; 2481 } else { 2482 *lba = bs_blob_io_unit_to_lba(blob, io_unit); 2483 return true; 2484 } 2485 } 2486 2487 struct op_split_ctx { 2488 struct spdk_blob *blob; 2489 struct spdk_io_channel *channel; 2490 uint64_t io_unit_offset; 2491 uint64_t io_units_remaining; 2492 void *curr_payload; 2493 enum spdk_blob_op_type op_type; 2494 spdk_bs_sequence_t *seq; 2495 }; 2496 2497 static void 2498 blob_request_submit_op_split_next(void *cb_arg, int bserrno) 2499 { 2500 struct op_split_ctx *ctx = cb_arg; 2501 struct spdk_blob *blob = ctx->blob; 2502 struct spdk_io_channel *ch = ctx->channel; 2503 enum spdk_blob_op_type op_type = ctx->op_type; 2504 uint8_t *buf = ctx->curr_payload; 2505 uint64_t offset = ctx->io_unit_offset; 2506 uint64_t length = ctx->io_units_remaining; 2507 uint64_t op_length; 2508 2509 if (bserrno != 0 || ctx->io_units_remaining == 0) { 2510 bs_sequence_finish(ctx->seq, bserrno); 2511 free(ctx); 2512 return; 2513 } 2514 2515 op_length = spdk_min(length, bs_num_io_units_to_cluster_boundary(blob, 2516 offset)); 2517 2518 /* Update length and payload for next operation */ 2519 ctx->io_units_remaining -= op_length; 2520 ctx->io_unit_offset += op_length; 2521 if (op_type == SPDK_BLOB_WRITE || op_type == SPDK_BLOB_READ) { 2522 ctx->curr_payload += op_length * blob->bs->io_unit_size; 2523 } 2524 2525 switch (op_type) { 2526 case SPDK_BLOB_READ: 2527 spdk_blob_io_read(blob, ch, buf, offset, op_length, 2528 blob_request_submit_op_split_next, ctx); 2529 break; 2530 case SPDK_BLOB_WRITE: 2531 spdk_blob_io_write(blob, ch, buf, offset, op_length, 2532 blob_request_submit_op_split_next, ctx); 2533 break; 2534 case SPDK_BLOB_UNMAP: 2535 spdk_blob_io_unmap(blob, ch, offset, op_length, 2536 blob_request_submit_op_split_next, ctx); 2537 break; 2538 case SPDK_BLOB_WRITE_ZEROES: 2539 spdk_blob_io_write_zeroes(blob, ch, offset, op_length, 2540 blob_request_submit_op_split_next, ctx); 2541 break; 2542 case SPDK_BLOB_READV: 2543 case SPDK_BLOB_WRITEV: 2544 SPDK_ERRLOG("readv/write not valid\n"); 2545 bs_sequence_finish(ctx->seq, -EINVAL); 2546 free(ctx); 2547 break; 2548 } 2549 } 2550 2551 static void 2552 blob_request_submit_op_split(struct spdk_io_channel *ch, struct spdk_blob *blob, 2553 void *payload, uint64_t offset, uint64_t length, 2554 spdk_blob_op_complete cb_fn, void *cb_arg, enum spdk_blob_op_type op_type) 2555 { 2556 struct op_split_ctx *ctx; 2557 spdk_bs_sequence_t *seq; 2558 struct spdk_bs_cpl cpl; 2559 2560 assert(blob != NULL); 2561 2562 ctx = calloc(1, sizeof(struct op_split_ctx)); 2563 if (ctx == NULL) { 2564 cb_fn(cb_arg, -ENOMEM); 2565 return; 2566 } 2567 2568 cpl.type = SPDK_BS_CPL_TYPE_BLOB_BASIC; 2569 cpl.u.blob_basic.cb_fn = cb_fn; 2570 cpl.u.blob_basic.cb_arg = cb_arg; 2571 2572 seq = bs_sequence_start(ch, &cpl); 2573 if (!seq) { 2574 free(ctx); 2575 cb_fn(cb_arg, -ENOMEM); 2576 return; 2577 } 2578 2579 ctx->blob = blob; 2580 ctx->channel = ch; 2581 ctx->curr_payload = payload; 2582 ctx->io_unit_offset = offset; 2583 ctx->io_units_remaining = length; 2584 ctx->op_type = op_type; 2585 ctx->seq = seq; 2586 2587 blob_request_submit_op_split_next(ctx, 0); 2588 } 2589 2590 static void 2591 blob_request_submit_op_single(struct spdk_io_channel *_ch, struct spdk_blob *blob, 2592 void *payload, uint64_t offset, uint64_t length, 2593 spdk_blob_op_complete cb_fn, void *cb_arg, enum spdk_blob_op_type op_type) 2594 { 2595 struct spdk_bs_cpl cpl; 2596 uint64_t lba; 2597 uint32_t lba_count; 2598 bool is_allocated; 2599 2600 assert(blob != NULL); 2601 2602 cpl.type = SPDK_BS_CPL_TYPE_BLOB_BASIC; 2603 cpl.u.blob_basic.cb_fn = cb_fn; 2604 cpl.u.blob_basic.cb_arg = cb_arg; 2605 2606 is_allocated = blob_calculate_lba_and_lba_count(blob, offset, length, &lba, &lba_count); 2607 2608 if (blob->frozen_refcnt) { 2609 /* This blob I/O is frozen */ 2610 spdk_bs_user_op_t *op; 2611 struct spdk_bs_channel *bs_channel = spdk_io_channel_get_ctx(_ch); 2612 2613 op = bs_user_op_alloc(_ch, &cpl, op_type, blob, payload, 0, offset, length); 2614 if (!op) { 2615 cb_fn(cb_arg, -ENOMEM); 2616 return; 2617 } 2618 2619 TAILQ_INSERT_TAIL(&bs_channel->queued_io, op, link); 2620 2621 return; 2622 } 2623 2624 switch (op_type) { 2625 case SPDK_BLOB_READ: { 2626 spdk_bs_batch_t *batch; 2627 2628 batch = bs_batch_open(_ch, &cpl); 2629 if (!batch) { 2630 cb_fn(cb_arg, -ENOMEM); 2631 return; 2632 } 2633 2634 if (is_allocated) { 2635 /* Read from the blob */ 2636 bs_batch_read_dev(batch, payload, lba, lba_count); 2637 } else { 2638 /* Read from the backing block device */ 2639 bs_batch_read_bs_dev(batch, blob->back_bs_dev, payload, lba, lba_count); 2640 } 2641 2642 bs_batch_close(batch); 2643 break; 2644 } 2645 case SPDK_BLOB_WRITE: 2646 case SPDK_BLOB_WRITE_ZEROES: { 2647 if (is_allocated) { 2648 /* Write to the blob */ 2649 spdk_bs_batch_t *batch; 2650 2651 if (lba_count == 0) { 2652 cb_fn(cb_arg, 0); 2653 return; 2654 } 2655 2656 batch = bs_batch_open(_ch, &cpl); 2657 if (!batch) { 2658 cb_fn(cb_arg, -ENOMEM); 2659 return; 2660 } 2661 2662 if (op_type == SPDK_BLOB_WRITE) { 2663 bs_batch_write_dev(batch, payload, lba, lba_count); 2664 } else { 2665 bs_batch_write_zeroes_dev(batch, lba, lba_count); 2666 } 2667 2668 bs_batch_close(batch); 2669 } else { 2670 /* Queue this operation and allocate the cluster */ 2671 spdk_bs_user_op_t *op; 2672 2673 op = bs_user_op_alloc(_ch, &cpl, op_type, blob, payload, 0, offset, length); 2674 if (!op) { 2675 cb_fn(cb_arg, -ENOMEM); 2676 return; 2677 } 2678 2679 bs_allocate_and_copy_cluster(blob, _ch, offset, op); 2680 } 2681 break; 2682 } 2683 case SPDK_BLOB_UNMAP: { 2684 spdk_bs_batch_t *batch; 2685 2686 batch = bs_batch_open(_ch, &cpl); 2687 if (!batch) { 2688 cb_fn(cb_arg, -ENOMEM); 2689 return; 2690 } 2691 2692 if (is_allocated) { 2693 bs_batch_unmap_dev(batch, lba, lba_count); 2694 } 2695 2696 bs_batch_close(batch); 2697 break; 2698 } 2699 case SPDK_BLOB_READV: 2700 case SPDK_BLOB_WRITEV: 2701 SPDK_ERRLOG("readv/write not valid\n"); 2702 cb_fn(cb_arg, -EINVAL); 2703 break; 2704 } 2705 } 2706 2707 static void 2708 blob_request_submit_op(struct spdk_blob *blob, struct spdk_io_channel *_channel, 2709 void *payload, uint64_t offset, uint64_t length, 2710 spdk_blob_op_complete cb_fn, void *cb_arg, enum spdk_blob_op_type op_type) 2711 { 2712 assert(blob != NULL); 2713 2714 if (blob->data_ro && op_type != SPDK_BLOB_READ) { 2715 cb_fn(cb_arg, -EPERM); 2716 return; 2717 } 2718 2719 if (offset + length > bs_cluster_to_lba(blob->bs, blob->active.num_clusters)) { 2720 cb_fn(cb_arg, -EINVAL); 2721 return; 2722 } 2723 if (length <= bs_num_io_units_to_cluster_boundary(blob, offset)) { 2724 blob_request_submit_op_single(_channel, blob, payload, offset, length, 2725 cb_fn, cb_arg, op_type); 2726 } else { 2727 blob_request_submit_op_split(_channel, blob, payload, offset, length, 2728 cb_fn, cb_arg, op_type); 2729 } 2730 } 2731 2732 struct rw_iov_ctx { 2733 struct spdk_blob *blob; 2734 struct spdk_io_channel *channel; 2735 spdk_blob_op_complete cb_fn; 2736 void *cb_arg; 2737 bool read; 2738 int iovcnt; 2739 struct iovec *orig_iov; 2740 uint64_t io_unit_offset; 2741 uint64_t io_units_remaining; 2742 uint64_t io_units_done; 2743 struct iovec iov[0]; 2744 }; 2745 2746 static void 2747 rw_iov_done(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 2748 { 2749 assert(cb_arg == NULL); 2750 bs_sequence_finish(seq, bserrno); 2751 } 2752 2753 static void 2754 rw_iov_split_next(void *cb_arg, int bserrno) 2755 { 2756 struct rw_iov_ctx *ctx = cb_arg; 2757 struct spdk_blob *blob = ctx->blob; 2758 struct iovec *iov, *orig_iov; 2759 int iovcnt; 2760 size_t orig_iovoff; 2761 uint64_t io_units_count, io_units_to_boundary, io_unit_offset; 2762 uint64_t byte_count; 2763 2764 if (bserrno != 0 || ctx->io_units_remaining == 0) { 2765 ctx->cb_fn(ctx->cb_arg, bserrno); 2766 free(ctx); 2767 return; 2768 } 2769 2770 io_unit_offset = ctx->io_unit_offset; 2771 io_units_to_boundary = bs_num_io_units_to_cluster_boundary(blob, io_unit_offset); 2772 io_units_count = spdk_min(ctx->io_units_remaining, io_units_to_boundary); 2773 /* 2774 * Get index and offset into the original iov array for our current position in the I/O sequence. 2775 * byte_count will keep track of how many bytes remaining until orig_iov and orig_iovoff will 2776 * point to the current position in the I/O sequence. 2777 */ 2778 byte_count = ctx->io_units_done * blob->bs->io_unit_size; 2779 orig_iov = &ctx->orig_iov[0]; 2780 orig_iovoff = 0; 2781 while (byte_count > 0) { 2782 if (byte_count >= orig_iov->iov_len) { 2783 byte_count -= orig_iov->iov_len; 2784 orig_iov++; 2785 } else { 2786 orig_iovoff = byte_count; 2787 byte_count = 0; 2788 } 2789 } 2790 2791 /* 2792 * Build an iov array for the next I/O in the sequence. byte_count will keep track of how many 2793 * bytes of this next I/O remain to be accounted for in the new iov array. 2794 */ 2795 byte_count = io_units_count * blob->bs->io_unit_size; 2796 iov = &ctx->iov[0]; 2797 iovcnt = 0; 2798 while (byte_count > 0) { 2799 assert(iovcnt < ctx->iovcnt); 2800 iov->iov_len = spdk_min(byte_count, orig_iov->iov_len - orig_iovoff); 2801 iov->iov_base = orig_iov->iov_base + orig_iovoff; 2802 byte_count -= iov->iov_len; 2803 orig_iovoff = 0; 2804 orig_iov++; 2805 iov++; 2806 iovcnt++; 2807 } 2808 2809 ctx->io_unit_offset += io_units_count; 2810 ctx->io_units_remaining -= io_units_count; 2811 ctx->io_units_done += io_units_count; 2812 iov = &ctx->iov[0]; 2813 2814 if (ctx->read) { 2815 spdk_blob_io_readv(ctx->blob, ctx->channel, iov, iovcnt, io_unit_offset, 2816 io_units_count, rw_iov_split_next, ctx); 2817 } else { 2818 spdk_blob_io_writev(ctx->blob, ctx->channel, iov, iovcnt, io_unit_offset, 2819 io_units_count, rw_iov_split_next, ctx); 2820 } 2821 } 2822 2823 static void 2824 blob_request_submit_rw_iov(struct spdk_blob *blob, struct spdk_io_channel *_channel, 2825 struct iovec *iov, int iovcnt, uint64_t offset, uint64_t length, 2826 spdk_blob_op_complete cb_fn, void *cb_arg, bool read) 2827 { 2828 struct spdk_bs_cpl cpl; 2829 2830 assert(blob != NULL); 2831 2832 if (!read && blob->data_ro) { 2833 cb_fn(cb_arg, -EPERM); 2834 return; 2835 } 2836 2837 if (length == 0) { 2838 cb_fn(cb_arg, 0); 2839 return; 2840 } 2841 2842 if (offset + length > bs_cluster_to_lba(blob->bs, blob->active.num_clusters)) { 2843 cb_fn(cb_arg, -EINVAL); 2844 return; 2845 } 2846 2847 /* 2848 * For now, we implement readv/writev using a sequence (instead of a batch) to account for having 2849 * to split a request that spans a cluster boundary. For I/O that do not span a cluster boundary, 2850 * there will be no noticeable difference compared to using a batch. For I/O that do span a cluster 2851 * boundary, the target LBAs (after blob offset to LBA translation) may not be contiguous, so we need 2852 * to allocate a separate iov array and split the I/O such that none of the resulting 2853 * smaller I/O cross a cluster boundary. These smaller I/O will be issued in sequence (not in parallel) 2854 * but since this case happens very infrequently, any performance impact will be negligible. 2855 * 2856 * This could be optimized in the future to allocate a big enough iov array to account for all of the iovs 2857 * for all of the smaller I/Os, pre-build all of the iov arrays for the smaller I/Os, then issue them 2858 * in a batch. That would also require creating an intermediate spdk_bs_cpl that would get called 2859 * when the batch was completed, to allow for freeing the memory for the iov arrays. 2860 */ 2861 if (spdk_likely(length <= bs_num_io_units_to_cluster_boundary(blob, offset))) { 2862 uint32_t lba_count; 2863 uint64_t lba; 2864 bool is_allocated; 2865 2866 cpl.type = SPDK_BS_CPL_TYPE_BLOB_BASIC; 2867 cpl.u.blob_basic.cb_fn = cb_fn; 2868 cpl.u.blob_basic.cb_arg = cb_arg; 2869 2870 if (blob->frozen_refcnt) { 2871 /* This blob I/O is frozen */ 2872 enum spdk_blob_op_type op_type; 2873 spdk_bs_user_op_t *op; 2874 struct spdk_bs_channel *bs_channel = spdk_io_channel_get_ctx(_channel); 2875 2876 op_type = read ? SPDK_BLOB_READV : SPDK_BLOB_WRITEV; 2877 op = bs_user_op_alloc(_channel, &cpl, op_type, blob, iov, iovcnt, offset, length); 2878 if (!op) { 2879 cb_fn(cb_arg, -ENOMEM); 2880 return; 2881 } 2882 2883 TAILQ_INSERT_TAIL(&bs_channel->queued_io, op, link); 2884 2885 return; 2886 } 2887 2888 is_allocated = blob_calculate_lba_and_lba_count(blob, offset, length, &lba, &lba_count); 2889 2890 if (read) { 2891 spdk_bs_sequence_t *seq; 2892 2893 seq = bs_sequence_start(_channel, &cpl); 2894 if (!seq) { 2895 cb_fn(cb_arg, -ENOMEM); 2896 return; 2897 } 2898 2899 if (is_allocated) { 2900 bs_sequence_readv_dev(seq, iov, iovcnt, lba, lba_count, rw_iov_done, NULL); 2901 } else { 2902 bs_sequence_readv_bs_dev(seq, blob->back_bs_dev, iov, iovcnt, lba, lba_count, 2903 rw_iov_done, NULL); 2904 } 2905 } else { 2906 if (is_allocated) { 2907 spdk_bs_sequence_t *seq; 2908 2909 seq = bs_sequence_start(_channel, &cpl); 2910 if (!seq) { 2911 cb_fn(cb_arg, -ENOMEM); 2912 return; 2913 } 2914 2915 bs_sequence_writev_dev(seq, iov, iovcnt, lba, lba_count, rw_iov_done, NULL); 2916 } else { 2917 /* Queue this operation and allocate the cluster */ 2918 spdk_bs_user_op_t *op; 2919 2920 op = bs_user_op_alloc(_channel, &cpl, SPDK_BLOB_WRITEV, blob, iov, iovcnt, offset, 2921 length); 2922 if (!op) { 2923 cb_fn(cb_arg, -ENOMEM); 2924 return; 2925 } 2926 2927 bs_allocate_and_copy_cluster(blob, _channel, offset, op); 2928 } 2929 } 2930 } else { 2931 struct rw_iov_ctx *ctx; 2932 2933 ctx = calloc(1, sizeof(struct rw_iov_ctx) + iovcnt * sizeof(struct iovec)); 2934 if (ctx == NULL) { 2935 cb_fn(cb_arg, -ENOMEM); 2936 return; 2937 } 2938 2939 ctx->blob = blob; 2940 ctx->channel = _channel; 2941 ctx->cb_fn = cb_fn; 2942 ctx->cb_arg = cb_arg; 2943 ctx->read = read; 2944 ctx->orig_iov = iov; 2945 ctx->iovcnt = iovcnt; 2946 ctx->io_unit_offset = offset; 2947 ctx->io_units_remaining = length; 2948 ctx->io_units_done = 0; 2949 2950 rw_iov_split_next(ctx, 0); 2951 } 2952 } 2953 2954 static struct spdk_blob * 2955 blob_lookup(struct spdk_blob_store *bs, spdk_blob_id blobid) 2956 { 2957 struct spdk_blob *blob; 2958 2959 if (spdk_bit_array_get(bs->open_blobids, blobid) == 0) { 2960 return NULL; 2961 } 2962 2963 TAILQ_FOREACH(blob, &bs->blobs, link) { 2964 if (blob->id == blobid) { 2965 return blob; 2966 } 2967 } 2968 2969 return NULL; 2970 } 2971 2972 static void 2973 blob_get_snapshot_and_clone_entries(struct spdk_blob *blob, 2974 struct spdk_blob_list **snapshot_entry, struct spdk_blob_list **clone_entry) 2975 { 2976 assert(blob != NULL); 2977 *snapshot_entry = NULL; 2978 *clone_entry = NULL; 2979 2980 if (blob->parent_id == SPDK_BLOBID_INVALID) { 2981 return; 2982 } 2983 2984 TAILQ_FOREACH(*snapshot_entry, &blob->bs->snapshots, link) { 2985 if ((*snapshot_entry)->id == blob->parent_id) { 2986 break; 2987 } 2988 } 2989 2990 if (*snapshot_entry != NULL) { 2991 TAILQ_FOREACH(*clone_entry, &(*snapshot_entry)->clones, link) { 2992 if ((*clone_entry)->id == blob->id) { 2993 break; 2994 } 2995 } 2996 2997 assert(clone_entry != NULL); 2998 } 2999 } 3000 3001 static int 3002 bs_channel_create(void *io_device, void *ctx_buf) 3003 { 3004 struct spdk_blob_store *bs = io_device; 3005 struct spdk_bs_channel *channel = ctx_buf; 3006 struct spdk_bs_dev *dev; 3007 uint32_t max_ops = bs->max_channel_ops; 3008 uint32_t i; 3009 3010 dev = bs->dev; 3011 3012 channel->req_mem = calloc(max_ops, sizeof(struct spdk_bs_request_set)); 3013 if (!channel->req_mem) { 3014 return -1; 3015 } 3016 3017 TAILQ_INIT(&channel->reqs); 3018 3019 for (i = 0; i < max_ops; i++) { 3020 TAILQ_INSERT_TAIL(&channel->reqs, &channel->req_mem[i], link); 3021 } 3022 3023 channel->bs = bs; 3024 channel->dev = dev; 3025 channel->dev_channel = dev->create_channel(dev); 3026 3027 if (!channel->dev_channel) { 3028 SPDK_ERRLOG("Failed to create device channel.\n"); 3029 free(channel->req_mem); 3030 return -1; 3031 } 3032 3033 TAILQ_INIT(&channel->need_cluster_alloc); 3034 TAILQ_INIT(&channel->queued_io); 3035 3036 return 0; 3037 } 3038 3039 static void 3040 bs_channel_destroy(void *io_device, void *ctx_buf) 3041 { 3042 struct spdk_bs_channel *channel = ctx_buf; 3043 spdk_bs_user_op_t *op; 3044 3045 while (!TAILQ_EMPTY(&channel->need_cluster_alloc)) { 3046 op = TAILQ_FIRST(&channel->need_cluster_alloc); 3047 TAILQ_REMOVE(&channel->need_cluster_alloc, op, link); 3048 bs_user_op_abort(op); 3049 } 3050 3051 while (!TAILQ_EMPTY(&channel->queued_io)) { 3052 op = TAILQ_FIRST(&channel->queued_io); 3053 TAILQ_REMOVE(&channel->queued_io, op, link); 3054 bs_user_op_abort(op); 3055 } 3056 3057 free(channel->req_mem); 3058 channel->dev->destroy_channel(channel->dev, channel->dev_channel); 3059 } 3060 3061 static void 3062 bs_dev_destroy(void *io_device) 3063 { 3064 struct spdk_blob_store *bs = io_device; 3065 struct spdk_blob *blob, *blob_tmp; 3066 3067 bs->dev->destroy(bs->dev); 3068 3069 TAILQ_FOREACH_SAFE(blob, &bs->blobs, link, blob_tmp) { 3070 TAILQ_REMOVE(&bs->blobs, blob, link); 3071 spdk_bit_array_clear(bs->open_blobids, blob->id); 3072 blob_free(blob); 3073 } 3074 3075 pthread_mutex_destroy(&bs->used_clusters_mutex); 3076 3077 spdk_bit_array_free(&bs->open_blobids); 3078 spdk_bit_array_free(&bs->used_blobids); 3079 spdk_bit_array_free(&bs->used_md_pages); 3080 spdk_bit_pool_free(&bs->used_clusters); 3081 /* 3082 * If this function is called for any reason except a successful unload, 3083 * the unload_cpl type will be NONE and this will be a nop. 3084 */ 3085 bs_call_cpl(&bs->unload_cpl, bs->unload_err); 3086 3087 free(bs); 3088 } 3089 3090 static int 3091 bs_blob_list_add(struct spdk_blob *blob) 3092 { 3093 spdk_blob_id snapshot_id; 3094 struct spdk_blob_list *snapshot_entry = NULL; 3095 struct spdk_blob_list *clone_entry = NULL; 3096 3097 assert(blob != NULL); 3098 3099 snapshot_id = blob->parent_id; 3100 if (snapshot_id == SPDK_BLOBID_INVALID) { 3101 return 0; 3102 } 3103 3104 snapshot_entry = bs_get_snapshot_entry(blob->bs, snapshot_id); 3105 if (snapshot_entry == NULL) { 3106 /* Snapshot not found */ 3107 snapshot_entry = calloc(1, sizeof(struct spdk_blob_list)); 3108 if (snapshot_entry == NULL) { 3109 return -ENOMEM; 3110 } 3111 snapshot_entry->id = snapshot_id; 3112 TAILQ_INIT(&snapshot_entry->clones); 3113 TAILQ_INSERT_TAIL(&blob->bs->snapshots, snapshot_entry, link); 3114 } else { 3115 TAILQ_FOREACH(clone_entry, &snapshot_entry->clones, link) { 3116 if (clone_entry->id == blob->id) { 3117 break; 3118 } 3119 } 3120 } 3121 3122 if (clone_entry == NULL) { 3123 /* Clone not found */ 3124 clone_entry = calloc(1, sizeof(struct spdk_blob_list)); 3125 if (clone_entry == NULL) { 3126 return -ENOMEM; 3127 } 3128 clone_entry->id = blob->id; 3129 TAILQ_INIT(&clone_entry->clones); 3130 TAILQ_INSERT_TAIL(&snapshot_entry->clones, clone_entry, link); 3131 snapshot_entry->clone_count++; 3132 } 3133 3134 return 0; 3135 } 3136 3137 static void 3138 bs_blob_list_remove(struct spdk_blob *blob) 3139 { 3140 struct spdk_blob_list *snapshot_entry = NULL; 3141 struct spdk_blob_list *clone_entry = NULL; 3142 3143 blob_get_snapshot_and_clone_entries(blob, &snapshot_entry, &clone_entry); 3144 3145 if (snapshot_entry == NULL) { 3146 return; 3147 } 3148 3149 blob->parent_id = SPDK_BLOBID_INVALID; 3150 TAILQ_REMOVE(&snapshot_entry->clones, clone_entry, link); 3151 free(clone_entry); 3152 3153 snapshot_entry->clone_count--; 3154 } 3155 3156 static int 3157 bs_blob_list_free(struct spdk_blob_store *bs) 3158 { 3159 struct spdk_blob_list *snapshot_entry; 3160 struct spdk_blob_list *snapshot_entry_tmp; 3161 struct spdk_blob_list *clone_entry; 3162 struct spdk_blob_list *clone_entry_tmp; 3163 3164 TAILQ_FOREACH_SAFE(snapshot_entry, &bs->snapshots, link, snapshot_entry_tmp) { 3165 TAILQ_FOREACH_SAFE(clone_entry, &snapshot_entry->clones, link, clone_entry_tmp) { 3166 TAILQ_REMOVE(&snapshot_entry->clones, clone_entry, link); 3167 free(clone_entry); 3168 } 3169 TAILQ_REMOVE(&bs->snapshots, snapshot_entry, link); 3170 free(snapshot_entry); 3171 } 3172 3173 return 0; 3174 } 3175 3176 static void 3177 bs_free(struct spdk_blob_store *bs) 3178 { 3179 bs_blob_list_free(bs); 3180 3181 bs_unregister_md_thread(bs); 3182 spdk_io_device_unregister(bs, bs_dev_destroy); 3183 } 3184 3185 void 3186 spdk_bs_opts_init(struct spdk_bs_opts *opts, size_t opts_size) 3187 { 3188 3189 if (!opts) { 3190 SPDK_ERRLOG("opts should not be NULL\n"); 3191 return; 3192 } 3193 3194 if (!opts_size) { 3195 SPDK_ERRLOG("opts_size should not be zero value\n"); 3196 return; 3197 } 3198 3199 memset(opts, 0, opts_size); 3200 opts->opts_size = opts_size; 3201 3202 #define FIELD_OK(field) \ 3203 offsetof(struct spdk_bs_opts, field) + sizeof(opts->field) <= opts_size 3204 3205 #define SET_FIELD(field, value) \ 3206 if (FIELD_OK(field)) { \ 3207 opts->field = value; \ 3208 } \ 3209 3210 SET_FIELD(cluster_sz, SPDK_BLOB_OPTS_CLUSTER_SZ); 3211 SET_FIELD(num_md_pages, SPDK_BLOB_OPTS_NUM_MD_PAGES); 3212 SET_FIELD(max_md_ops, SPDK_BLOB_OPTS_NUM_MD_PAGES); 3213 SET_FIELD(max_channel_ops, SPDK_BLOB_OPTS_DEFAULT_CHANNEL_OPS); 3214 SET_FIELD(clear_method, BS_CLEAR_WITH_UNMAP); 3215 3216 if (FIELD_OK(bstype)) { 3217 memset(&opts->bstype, 0, sizeof(opts->bstype)); 3218 } 3219 3220 SET_FIELD(iter_cb_fn, NULL); 3221 SET_FIELD(iter_cb_arg, NULL); 3222 3223 #undef FIELD_OK 3224 #undef SET_FIELD 3225 } 3226 3227 static int 3228 bs_opts_verify(struct spdk_bs_opts *opts) 3229 { 3230 if (opts->cluster_sz == 0 || opts->num_md_pages == 0 || opts->max_md_ops == 0 || 3231 opts->max_channel_ops == 0) { 3232 SPDK_ERRLOG("Blobstore options cannot be set to 0\n"); 3233 return -1; 3234 } 3235 3236 return 0; 3237 } 3238 3239 /* START spdk_bs_load */ 3240 3241 /* spdk_bs_load_ctx is used for init, load, unload and dump code paths. */ 3242 3243 struct spdk_bs_load_ctx { 3244 struct spdk_blob_store *bs; 3245 struct spdk_bs_super_block *super; 3246 3247 struct spdk_bs_md_mask *mask; 3248 bool in_page_chain; 3249 uint32_t page_index; 3250 uint32_t cur_page; 3251 struct spdk_blob_md_page *page; 3252 3253 uint64_t num_extent_pages; 3254 uint32_t *extent_page_num; 3255 struct spdk_blob_md_page *extent_pages; 3256 struct spdk_bit_array *used_clusters; 3257 3258 spdk_bs_sequence_t *seq; 3259 spdk_blob_op_with_handle_complete iter_cb_fn; 3260 void *iter_cb_arg; 3261 struct spdk_blob *blob; 3262 spdk_blob_id blobid; 3263 3264 /* These fields are used in the spdk_bs_dump path. */ 3265 FILE *fp; 3266 spdk_bs_dump_print_xattr print_xattr_fn; 3267 char xattr_name[4096]; 3268 }; 3269 3270 static int 3271 bs_alloc(struct spdk_bs_dev *dev, struct spdk_bs_opts *opts, struct spdk_blob_store **_bs, 3272 struct spdk_bs_load_ctx **_ctx) 3273 { 3274 struct spdk_blob_store *bs; 3275 struct spdk_bs_load_ctx *ctx; 3276 uint64_t dev_size; 3277 int rc; 3278 3279 dev_size = dev->blocklen * dev->blockcnt; 3280 if (dev_size < opts->cluster_sz) { 3281 /* Device size cannot be smaller than cluster size of blobstore */ 3282 SPDK_INFOLOG(blob, "Device size %" PRIu64 " is smaller than cluster size %" PRIu32 "\n", 3283 dev_size, opts->cluster_sz); 3284 return -ENOSPC; 3285 } 3286 if (opts->cluster_sz < SPDK_BS_PAGE_SIZE) { 3287 /* Cluster size cannot be smaller than page size */ 3288 SPDK_ERRLOG("Cluster size %" PRIu32 " is smaller than page size %d\n", 3289 opts->cluster_sz, SPDK_BS_PAGE_SIZE); 3290 return -EINVAL; 3291 } 3292 bs = calloc(1, sizeof(struct spdk_blob_store)); 3293 if (!bs) { 3294 return -ENOMEM; 3295 } 3296 3297 ctx = calloc(1, sizeof(struct spdk_bs_load_ctx)); 3298 if (!ctx) { 3299 free(bs); 3300 return -ENOMEM; 3301 } 3302 3303 ctx->bs = bs; 3304 ctx->iter_cb_fn = opts->iter_cb_fn; 3305 ctx->iter_cb_arg = opts->iter_cb_arg; 3306 3307 ctx->super = spdk_zmalloc(sizeof(*ctx->super), 0x1000, NULL, 3308 SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA); 3309 if (!ctx->super) { 3310 free(ctx); 3311 free(bs); 3312 return -ENOMEM; 3313 } 3314 3315 TAILQ_INIT(&bs->blobs); 3316 TAILQ_INIT(&bs->snapshots); 3317 bs->dev = dev; 3318 bs->md_thread = spdk_get_thread(); 3319 assert(bs->md_thread != NULL); 3320 3321 /* 3322 * Do not use bs_lba_to_cluster() here since blockcnt may not be an 3323 * even multiple of the cluster size. 3324 */ 3325 bs->cluster_sz = opts->cluster_sz; 3326 bs->total_clusters = dev->blockcnt / (bs->cluster_sz / dev->blocklen); 3327 ctx->used_clusters = spdk_bit_array_create(bs->total_clusters); 3328 if (!ctx->used_clusters) { 3329 spdk_free(ctx->super); 3330 free(ctx); 3331 free(bs); 3332 return -ENOMEM; 3333 } 3334 3335 bs->pages_per_cluster = bs->cluster_sz / SPDK_BS_PAGE_SIZE; 3336 if (spdk_u32_is_pow2(bs->pages_per_cluster)) { 3337 bs->pages_per_cluster_shift = spdk_u32log2(bs->pages_per_cluster); 3338 } 3339 bs->num_free_clusters = bs->total_clusters; 3340 bs->io_unit_size = dev->blocklen; 3341 3342 bs->max_channel_ops = opts->max_channel_ops; 3343 bs->super_blob = SPDK_BLOBID_INVALID; 3344 memcpy(&bs->bstype, &opts->bstype, sizeof(opts->bstype)); 3345 3346 /* The metadata is assumed to be at least 1 page */ 3347 bs->used_md_pages = spdk_bit_array_create(1); 3348 bs->used_blobids = spdk_bit_array_create(0); 3349 bs->open_blobids = spdk_bit_array_create(0); 3350 3351 pthread_mutex_init(&bs->used_clusters_mutex, NULL); 3352 3353 spdk_io_device_register(bs, bs_channel_create, bs_channel_destroy, 3354 sizeof(struct spdk_bs_channel), "blobstore"); 3355 rc = bs_register_md_thread(bs); 3356 if (rc == -1) { 3357 spdk_io_device_unregister(bs, NULL); 3358 pthread_mutex_destroy(&bs->used_clusters_mutex); 3359 spdk_bit_array_free(&bs->open_blobids); 3360 spdk_bit_array_free(&bs->used_blobids); 3361 spdk_bit_array_free(&bs->used_md_pages); 3362 spdk_bit_array_free(&ctx->used_clusters); 3363 spdk_free(ctx->super); 3364 free(ctx); 3365 free(bs); 3366 /* FIXME: this is a lie but don't know how to get a proper error code here */ 3367 return -ENOMEM; 3368 } 3369 3370 *_ctx = ctx; 3371 *_bs = bs; 3372 return 0; 3373 } 3374 3375 static void 3376 bs_load_ctx_fail(struct spdk_bs_load_ctx *ctx, int bserrno) 3377 { 3378 assert(bserrno != 0); 3379 3380 spdk_free(ctx->super); 3381 bs_sequence_finish(ctx->seq, bserrno); 3382 bs_free(ctx->bs); 3383 spdk_bit_array_free(&ctx->used_clusters); 3384 free(ctx); 3385 } 3386 3387 static void 3388 bs_write_super(spdk_bs_sequence_t *seq, struct spdk_blob_store *bs, 3389 struct spdk_bs_super_block *super, spdk_bs_sequence_cpl cb_fn, void *cb_arg) 3390 { 3391 /* Update the values in the super block */ 3392 super->super_blob = bs->super_blob; 3393 memcpy(&super->bstype, &bs->bstype, sizeof(bs->bstype)); 3394 super->crc = blob_md_page_calc_crc(super); 3395 bs_sequence_write_dev(seq, super, bs_page_to_lba(bs, 0), 3396 bs_byte_to_lba(bs, sizeof(*super)), 3397 cb_fn, cb_arg); 3398 } 3399 3400 static void 3401 bs_write_used_clusters(spdk_bs_sequence_t *seq, void *arg, spdk_bs_sequence_cpl cb_fn) 3402 { 3403 struct spdk_bs_load_ctx *ctx = arg; 3404 uint64_t mask_size, lba, lba_count; 3405 3406 /* Write out the used clusters mask */ 3407 mask_size = ctx->super->used_cluster_mask_len * SPDK_BS_PAGE_SIZE; 3408 ctx->mask = spdk_zmalloc(mask_size, 0x1000, NULL, 3409 SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA); 3410 if (!ctx->mask) { 3411 bs_load_ctx_fail(ctx, -ENOMEM); 3412 return; 3413 } 3414 3415 ctx->mask->type = SPDK_MD_MASK_TYPE_USED_CLUSTERS; 3416 ctx->mask->length = ctx->bs->total_clusters; 3417 /* We could get here through the normal unload path, or through dirty 3418 * shutdown recovery. For the normal unload path, we use the mask from 3419 * the bit pool. For dirty shutdown recovery, we don't have a bit pool yet - 3420 * only the bit array from the load ctx. 3421 */ 3422 if (ctx->bs->used_clusters) { 3423 assert(ctx->mask->length == spdk_bit_pool_capacity(ctx->bs->used_clusters)); 3424 spdk_bit_pool_store_mask(ctx->bs->used_clusters, ctx->mask->mask); 3425 } else { 3426 assert(ctx->mask->length == spdk_bit_array_capacity(ctx->used_clusters)); 3427 spdk_bit_array_store_mask(ctx->used_clusters, ctx->mask->mask); 3428 } 3429 lba = bs_page_to_lba(ctx->bs, ctx->super->used_cluster_mask_start); 3430 lba_count = bs_page_to_lba(ctx->bs, ctx->super->used_cluster_mask_len); 3431 bs_sequence_write_dev(seq, ctx->mask, lba, lba_count, cb_fn, arg); 3432 } 3433 3434 static void 3435 bs_write_used_md(spdk_bs_sequence_t *seq, void *arg, spdk_bs_sequence_cpl cb_fn) 3436 { 3437 struct spdk_bs_load_ctx *ctx = arg; 3438 uint64_t mask_size, lba, lba_count; 3439 3440 mask_size = ctx->super->used_page_mask_len * SPDK_BS_PAGE_SIZE; 3441 ctx->mask = spdk_zmalloc(mask_size, 0x1000, NULL, 3442 SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA); 3443 if (!ctx->mask) { 3444 bs_load_ctx_fail(ctx, -ENOMEM); 3445 return; 3446 } 3447 3448 ctx->mask->type = SPDK_MD_MASK_TYPE_USED_PAGES; 3449 ctx->mask->length = ctx->super->md_len; 3450 assert(ctx->mask->length == spdk_bit_array_capacity(ctx->bs->used_md_pages)); 3451 3452 spdk_bit_array_store_mask(ctx->bs->used_md_pages, ctx->mask->mask); 3453 lba = bs_page_to_lba(ctx->bs, ctx->super->used_page_mask_start); 3454 lba_count = bs_page_to_lba(ctx->bs, ctx->super->used_page_mask_len); 3455 bs_sequence_write_dev(seq, ctx->mask, lba, lba_count, cb_fn, arg); 3456 } 3457 3458 static void 3459 bs_write_used_blobids(spdk_bs_sequence_t *seq, void *arg, spdk_bs_sequence_cpl cb_fn) 3460 { 3461 struct spdk_bs_load_ctx *ctx = arg; 3462 uint64_t mask_size, lba, lba_count; 3463 3464 if (ctx->super->used_blobid_mask_len == 0) { 3465 /* 3466 * This is a pre-v3 on-disk format where the blobid mask does not get 3467 * written to disk. 3468 */ 3469 cb_fn(seq, arg, 0); 3470 return; 3471 } 3472 3473 mask_size = ctx->super->used_blobid_mask_len * SPDK_BS_PAGE_SIZE; 3474 ctx->mask = spdk_zmalloc(mask_size, 0x1000, NULL, SPDK_ENV_SOCKET_ID_ANY, 3475 SPDK_MALLOC_DMA); 3476 if (!ctx->mask) { 3477 bs_load_ctx_fail(ctx, -ENOMEM); 3478 return; 3479 } 3480 3481 ctx->mask->type = SPDK_MD_MASK_TYPE_USED_BLOBIDS; 3482 ctx->mask->length = ctx->super->md_len; 3483 assert(ctx->mask->length == spdk_bit_array_capacity(ctx->bs->used_blobids)); 3484 3485 spdk_bit_array_store_mask(ctx->bs->used_blobids, ctx->mask->mask); 3486 lba = bs_page_to_lba(ctx->bs, ctx->super->used_blobid_mask_start); 3487 lba_count = bs_page_to_lba(ctx->bs, ctx->super->used_blobid_mask_len); 3488 bs_sequence_write_dev(seq, ctx->mask, lba, lba_count, cb_fn, arg); 3489 } 3490 3491 static void 3492 blob_set_thin_provision(struct spdk_blob *blob) 3493 { 3494 blob_verify_md_op(blob); 3495 blob->invalid_flags |= SPDK_BLOB_THIN_PROV; 3496 blob->state = SPDK_BLOB_STATE_DIRTY; 3497 } 3498 3499 static void 3500 blob_set_clear_method(struct spdk_blob *blob, enum blob_clear_method clear_method) 3501 { 3502 blob_verify_md_op(blob); 3503 blob->clear_method = clear_method; 3504 blob->md_ro_flags |= (clear_method << SPDK_BLOB_CLEAR_METHOD_SHIFT); 3505 blob->state = SPDK_BLOB_STATE_DIRTY; 3506 } 3507 3508 static void bs_load_iter(void *arg, struct spdk_blob *blob, int bserrno); 3509 3510 static void 3511 bs_delete_corrupted_blob_cpl(void *cb_arg, int bserrno) 3512 { 3513 struct spdk_bs_load_ctx *ctx = cb_arg; 3514 spdk_blob_id id; 3515 int64_t page_num; 3516 3517 /* Iterate to next blob (we can't use spdk_bs_iter_next function as our 3518 * last blob has been removed */ 3519 page_num = bs_blobid_to_page(ctx->blobid); 3520 page_num++; 3521 page_num = spdk_bit_array_find_first_set(ctx->bs->used_blobids, page_num); 3522 if (page_num >= spdk_bit_array_capacity(ctx->bs->used_blobids)) { 3523 bs_load_iter(ctx, NULL, -ENOENT); 3524 return; 3525 } 3526 3527 id = bs_page_to_blobid(page_num); 3528 3529 spdk_bs_open_blob(ctx->bs, id, bs_load_iter, ctx); 3530 } 3531 3532 static void 3533 bs_delete_corrupted_close_cb(void *cb_arg, int bserrno) 3534 { 3535 struct spdk_bs_load_ctx *ctx = cb_arg; 3536 3537 if (bserrno != 0) { 3538 SPDK_ERRLOG("Failed to close corrupted blob\n"); 3539 spdk_bs_iter_next(ctx->bs, ctx->blob, bs_load_iter, ctx); 3540 return; 3541 } 3542 3543 spdk_bs_delete_blob(ctx->bs, ctx->blobid, bs_delete_corrupted_blob_cpl, ctx); 3544 } 3545 3546 static void 3547 bs_delete_corrupted_blob(void *cb_arg, int bserrno) 3548 { 3549 struct spdk_bs_load_ctx *ctx = cb_arg; 3550 uint64_t i; 3551 3552 if (bserrno != 0) { 3553 SPDK_ERRLOG("Failed to close clone of a corrupted blob\n"); 3554 spdk_bs_iter_next(ctx->bs, ctx->blob, bs_load_iter, ctx); 3555 return; 3556 } 3557 3558 /* Snapshot and clone have the same copy of cluster map and extent pages 3559 * at this point. Let's clear both for snpashot now, 3560 * so that it won't be cleared for clone later when we remove snapshot. 3561 * Also set thin provision to pass data corruption check */ 3562 for (i = 0; i < ctx->blob->active.num_clusters; i++) { 3563 ctx->blob->active.clusters[i] = 0; 3564 } 3565 for (i = 0; i < ctx->blob->active.num_extent_pages; i++) { 3566 ctx->blob->active.extent_pages[i] = 0; 3567 } 3568 3569 ctx->blob->md_ro = false; 3570 3571 blob_set_thin_provision(ctx->blob); 3572 3573 ctx->blobid = ctx->blob->id; 3574 3575 spdk_blob_close(ctx->blob, bs_delete_corrupted_close_cb, ctx); 3576 } 3577 3578 static void 3579 bs_update_corrupted_blob(void *cb_arg, int bserrno) 3580 { 3581 struct spdk_bs_load_ctx *ctx = cb_arg; 3582 3583 if (bserrno != 0) { 3584 SPDK_ERRLOG("Failed to close clone of a corrupted blob\n"); 3585 spdk_bs_iter_next(ctx->bs, ctx->blob, bs_load_iter, ctx); 3586 return; 3587 } 3588 3589 ctx->blob->md_ro = false; 3590 blob_remove_xattr(ctx->blob, SNAPSHOT_PENDING_REMOVAL, true); 3591 blob_remove_xattr(ctx->blob, SNAPSHOT_IN_PROGRESS, true); 3592 spdk_blob_set_read_only(ctx->blob); 3593 3594 if (ctx->iter_cb_fn) { 3595 ctx->iter_cb_fn(ctx->iter_cb_arg, ctx->blob, 0); 3596 } 3597 bs_blob_list_add(ctx->blob); 3598 3599 spdk_bs_iter_next(ctx->bs, ctx->blob, bs_load_iter, ctx); 3600 } 3601 3602 static void 3603 bs_examine_clone(void *cb_arg, struct spdk_blob *blob, int bserrno) 3604 { 3605 struct spdk_bs_load_ctx *ctx = cb_arg; 3606 3607 if (bserrno != 0) { 3608 SPDK_ERRLOG("Failed to open clone of a corrupted blob\n"); 3609 spdk_bs_iter_next(ctx->bs, ctx->blob, bs_load_iter, ctx); 3610 return; 3611 } 3612 3613 if (blob->parent_id == ctx->blob->id) { 3614 /* Power failure occured before updating clone (snapshot delete case) 3615 * or after updating clone (creating snapshot case) - keep snapshot */ 3616 spdk_blob_close(blob, bs_update_corrupted_blob, ctx); 3617 } else { 3618 /* Power failure occured after updating clone (snapshot delete case) 3619 * or before updating clone (creating snapshot case) - remove snapshot */ 3620 spdk_blob_close(blob, bs_delete_corrupted_blob, ctx); 3621 } 3622 } 3623 3624 static void 3625 bs_load_iter(void *arg, struct spdk_blob *blob, int bserrno) 3626 { 3627 struct spdk_bs_load_ctx *ctx = arg; 3628 const void *value; 3629 size_t len; 3630 int rc = 0; 3631 3632 if (bserrno == 0) { 3633 /* Examine blob if it is corrupted after power failure. Fix 3634 * the ones that can be fixed and remove any other corrupted 3635 * ones. If it is not corrupted just process it */ 3636 rc = blob_get_xattr_value(blob, SNAPSHOT_PENDING_REMOVAL, &value, &len, true); 3637 if (rc != 0) { 3638 rc = blob_get_xattr_value(blob, SNAPSHOT_IN_PROGRESS, &value, &len, true); 3639 if (rc != 0) { 3640 /* Not corrupted - process it and continue with iterating through blobs */ 3641 if (ctx->iter_cb_fn) { 3642 ctx->iter_cb_fn(ctx->iter_cb_arg, blob, 0); 3643 } 3644 bs_blob_list_add(blob); 3645 spdk_bs_iter_next(ctx->bs, blob, bs_load_iter, ctx); 3646 return; 3647 } 3648 3649 } 3650 3651 assert(len == sizeof(spdk_blob_id)); 3652 3653 ctx->blob = blob; 3654 3655 /* Open clone to check if we are able to fix this blob or should we remove it */ 3656 spdk_bs_open_blob(ctx->bs, *(spdk_blob_id *)value, bs_examine_clone, ctx); 3657 return; 3658 } else if (bserrno == -ENOENT) { 3659 bserrno = 0; 3660 } else { 3661 /* 3662 * This case needs to be looked at further. Same problem 3663 * exists with applications that rely on explicit blob 3664 * iteration. We should just skip the blob that failed 3665 * to load and continue on to the next one. 3666 */ 3667 SPDK_ERRLOG("Error in iterating blobs\n"); 3668 } 3669 3670 ctx->iter_cb_fn = NULL; 3671 3672 spdk_free(ctx->super); 3673 spdk_free(ctx->mask); 3674 bs_sequence_finish(ctx->seq, bserrno); 3675 free(ctx); 3676 } 3677 3678 static void 3679 bs_load_complete(struct spdk_bs_load_ctx *ctx) 3680 { 3681 ctx->bs->used_clusters = spdk_bit_pool_create_from_array(ctx->used_clusters); 3682 spdk_bs_iter_first(ctx->bs, bs_load_iter, ctx); 3683 } 3684 3685 static void 3686 bs_load_used_blobids_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 3687 { 3688 struct spdk_bs_load_ctx *ctx = cb_arg; 3689 int rc; 3690 3691 /* The type must be correct */ 3692 assert(ctx->mask->type == SPDK_MD_MASK_TYPE_USED_BLOBIDS); 3693 3694 /* The length of the mask (in bits) must not be greater than 3695 * the length of the buffer (converted to bits) */ 3696 assert(ctx->mask->length <= (ctx->super->used_blobid_mask_len * SPDK_BS_PAGE_SIZE * 8)); 3697 3698 /* The length of the mask must be exactly equal to the size 3699 * (in pages) of the metadata region */ 3700 assert(ctx->mask->length == ctx->super->md_len); 3701 3702 rc = spdk_bit_array_resize(&ctx->bs->used_blobids, ctx->mask->length); 3703 if (rc < 0) { 3704 spdk_free(ctx->mask); 3705 bs_load_ctx_fail(ctx, rc); 3706 return; 3707 } 3708 3709 spdk_bit_array_load_mask(ctx->bs->used_blobids, ctx->mask->mask); 3710 bs_load_complete(ctx); 3711 } 3712 3713 static void 3714 bs_load_used_clusters_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 3715 { 3716 struct spdk_bs_load_ctx *ctx = cb_arg; 3717 uint64_t lba, lba_count, mask_size; 3718 int rc; 3719 3720 if (bserrno != 0) { 3721 bs_load_ctx_fail(ctx, bserrno); 3722 return; 3723 } 3724 3725 /* The type must be correct */ 3726 assert(ctx->mask->type == SPDK_MD_MASK_TYPE_USED_CLUSTERS); 3727 /* The length of the mask (in bits) must not be greater than the length of the buffer (converted to bits) */ 3728 assert(ctx->mask->length <= (ctx->super->used_cluster_mask_len * sizeof( 3729 struct spdk_blob_md_page) * 8)); 3730 /* The length of the mask must be exactly equal to the total number of clusters */ 3731 assert(ctx->mask->length == ctx->bs->total_clusters); 3732 3733 rc = spdk_bit_array_resize(&ctx->used_clusters, ctx->mask->length); 3734 if (rc < 0) { 3735 spdk_free(ctx->mask); 3736 bs_load_ctx_fail(ctx, rc); 3737 return; 3738 } 3739 3740 spdk_bit_array_load_mask(ctx->used_clusters, ctx->mask->mask); 3741 ctx->bs->num_free_clusters = spdk_bit_array_count_clear(ctx->used_clusters); 3742 assert(ctx->bs->num_free_clusters <= ctx->bs->total_clusters); 3743 3744 spdk_free(ctx->mask); 3745 3746 /* Read the used blobids mask */ 3747 mask_size = ctx->super->used_blobid_mask_len * SPDK_BS_PAGE_SIZE; 3748 ctx->mask = spdk_zmalloc(mask_size, 0x1000, NULL, SPDK_ENV_SOCKET_ID_ANY, 3749 SPDK_MALLOC_DMA); 3750 if (!ctx->mask) { 3751 bs_load_ctx_fail(ctx, -ENOMEM); 3752 return; 3753 } 3754 lba = bs_page_to_lba(ctx->bs, ctx->super->used_blobid_mask_start); 3755 lba_count = bs_page_to_lba(ctx->bs, ctx->super->used_blobid_mask_len); 3756 bs_sequence_read_dev(seq, ctx->mask, lba, lba_count, 3757 bs_load_used_blobids_cpl, ctx); 3758 } 3759 3760 static void 3761 bs_load_used_pages_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 3762 { 3763 struct spdk_bs_load_ctx *ctx = cb_arg; 3764 uint64_t lba, lba_count, mask_size; 3765 int rc; 3766 3767 if (bserrno != 0) { 3768 bs_load_ctx_fail(ctx, bserrno); 3769 return; 3770 } 3771 3772 /* The type must be correct */ 3773 assert(ctx->mask->type == SPDK_MD_MASK_TYPE_USED_PAGES); 3774 /* The length of the mask (in bits) must not be greater than the length of the buffer (converted to bits) */ 3775 assert(ctx->mask->length <= (ctx->super->used_page_mask_len * SPDK_BS_PAGE_SIZE * 3776 8)); 3777 /* The length of the mask must be exactly equal to the size (in pages) of the metadata region */ 3778 if (ctx->mask->length != ctx->super->md_len) { 3779 SPDK_ERRLOG("mismatched md_len in used_pages mask: " 3780 "mask->length=%" PRIu32 " super->md_len=%" PRIu32 "\n", 3781 ctx->mask->length, ctx->super->md_len); 3782 assert(false); 3783 } 3784 3785 rc = spdk_bit_array_resize(&ctx->bs->used_md_pages, ctx->mask->length); 3786 if (rc < 0) { 3787 spdk_free(ctx->mask); 3788 bs_load_ctx_fail(ctx, rc); 3789 return; 3790 } 3791 3792 spdk_bit_array_load_mask(ctx->bs->used_md_pages, ctx->mask->mask); 3793 spdk_free(ctx->mask); 3794 3795 /* Read the used clusters mask */ 3796 mask_size = ctx->super->used_cluster_mask_len * SPDK_BS_PAGE_SIZE; 3797 ctx->mask = spdk_zmalloc(mask_size, 0x1000, NULL, SPDK_ENV_SOCKET_ID_ANY, 3798 SPDK_MALLOC_DMA); 3799 if (!ctx->mask) { 3800 bs_load_ctx_fail(ctx, -ENOMEM); 3801 return; 3802 } 3803 lba = bs_page_to_lba(ctx->bs, ctx->super->used_cluster_mask_start); 3804 lba_count = bs_page_to_lba(ctx->bs, ctx->super->used_cluster_mask_len); 3805 bs_sequence_read_dev(seq, ctx->mask, lba, lba_count, 3806 bs_load_used_clusters_cpl, ctx); 3807 } 3808 3809 static void 3810 bs_load_read_used_pages(struct spdk_bs_load_ctx *ctx) 3811 { 3812 uint64_t lba, lba_count, mask_size; 3813 3814 /* Read the used pages mask */ 3815 mask_size = ctx->super->used_page_mask_len * SPDK_BS_PAGE_SIZE; 3816 ctx->mask = spdk_zmalloc(mask_size, 0x1000, NULL, 3817 SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA); 3818 if (!ctx->mask) { 3819 bs_load_ctx_fail(ctx, -ENOMEM); 3820 return; 3821 } 3822 3823 lba = bs_page_to_lba(ctx->bs, ctx->super->used_page_mask_start); 3824 lba_count = bs_page_to_lba(ctx->bs, ctx->super->used_page_mask_len); 3825 bs_sequence_read_dev(ctx->seq, ctx->mask, lba, lba_count, 3826 bs_load_used_pages_cpl, ctx); 3827 } 3828 3829 static int 3830 bs_load_replay_md_parse_page(struct spdk_bs_load_ctx *ctx, struct spdk_blob_md_page *page) 3831 { 3832 struct spdk_blob_store *bs = ctx->bs; 3833 struct spdk_blob_md_descriptor *desc; 3834 size_t cur_desc = 0; 3835 3836 desc = (struct spdk_blob_md_descriptor *)page->descriptors; 3837 while (cur_desc < sizeof(page->descriptors)) { 3838 if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_PADDING) { 3839 if (desc->length == 0) { 3840 /* If padding and length are 0, this terminates the page */ 3841 break; 3842 } 3843 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_EXTENT_RLE) { 3844 struct spdk_blob_md_descriptor_extent_rle *desc_extent_rle; 3845 unsigned int i, j; 3846 unsigned int cluster_count = 0; 3847 uint32_t cluster_idx; 3848 3849 desc_extent_rle = (struct spdk_blob_md_descriptor_extent_rle *)desc; 3850 3851 for (i = 0; i < desc_extent_rle->length / sizeof(desc_extent_rle->extents[0]); i++) { 3852 for (j = 0; j < desc_extent_rle->extents[i].length; j++) { 3853 cluster_idx = desc_extent_rle->extents[i].cluster_idx; 3854 /* 3855 * cluster_idx = 0 means an unallocated cluster - don't mark that 3856 * in the used cluster map. 3857 */ 3858 if (cluster_idx != 0) { 3859 spdk_bit_array_set(ctx->used_clusters, cluster_idx + j); 3860 if (bs->num_free_clusters == 0) { 3861 return -ENOSPC; 3862 } 3863 bs->num_free_clusters--; 3864 } 3865 cluster_count++; 3866 } 3867 } 3868 if (cluster_count == 0) { 3869 return -EINVAL; 3870 } 3871 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_EXTENT_PAGE) { 3872 struct spdk_blob_md_descriptor_extent_page *desc_extent; 3873 uint32_t i; 3874 uint32_t cluster_count = 0; 3875 uint32_t cluster_idx; 3876 size_t cluster_idx_length; 3877 3878 desc_extent = (struct spdk_blob_md_descriptor_extent_page *)desc; 3879 cluster_idx_length = desc_extent->length - sizeof(desc_extent->start_cluster_idx); 3880 3881 if (desc_extent->length <= sizeof(desc_extent->start_cluster_idx) || 3882 (cluster_idx_length % sizeof(desc_extent->cluster_idx[0]) != 0)) { 3883 return -EINVAL; 3884 } 3885 3886 for (i = 0; i < cluster_idx_length / sizeof(desc_extent->cluster_idx[0]); i++) { 3887 cluster_idx = desc_extent->cluster_idx[i]; 3888 /* 3889 * cluster_idx = 0 means an unallocated cluster - don't mark that 3890 * in the used cluster map. 3891 */ 3892 if (cluster_idx != 0) { 3893 if (cluster_idx < desc_extent->start_cluster_idx && 3894 cluster_idx >= desc_extent->start_cluster_idx + cluster_count) { 3895 return -EINVAL; 3896 } 3897 spdk_bit_array_set(ctx->used_clusters, cluster_idx); 3898 if (bs->num_free_clusters == 0) { 3899 return -ENOSPC; 3900 } 3901 bs->num_free_clusters--; 3902 } 3903 cluster_count++; 3904 } 3905 3906 if (cluster_count == 0) { 3907 return -EINVAL; 3908 } 3909 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_XATTR) { 3910 /* Skip this item */ 3911 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_XATTR_INTERNAL) { 3912 /* Skip this item */ 3913 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_FLAGS) { 3914 /* Skip this item */ 3915 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_EXTENT_TABLE) { 3916 struct spdk_blob_md_descriptor_extent_table *desc_extent_table; 3917 uint32_t num_extent_pages = ctx->num_extent_pages; 3918 uint32_t i; 3919 size_t extent_pages_length; 3920 void *tmp; 3921 3922 desc_extent_table = (struct spdk_blob_md_descriptor_extent_table *)desc; 3923 extent_pages_length = desc_extent_table->length - sizeof(desc_extent_table->num_clusters); 3924 3925 if (desc_extent_table->length == 0 || 3926 (extent_pages_length % sizeof(desc_extent_table->extent_page[0]) != 0)) { 3927 return -EINVAL; 3928 } 3929 3930 for (i = 0; i < extent_pages_length / sizeof(desc_extent_table->extent_page[0]); i++) { 3931 if (desc_extent_table->extent_page[i].page_idx != 0) { 3932 if (desc_extent_table->extent_page[i].num_pages != 1) { 3933 return -EINVAL; 3934 } 3935 num_extent_pages += 1; 3936 } 3937 } 3938 3939 if (num_extent_pages > 0) { 3940 tmp = realloc(ctx->extent_page_num, num_extent_pages * sizeof(uint32_t)); 3941 if (tmp == NULL) { 3942 return -ENOMEM; 3943 } 3944 ctx->extent_page_num = tmp; 3945 3946 /* Extent table entries contain md page numbers for extent pages. 3947 * Zeroes represent unallocated extent pages, those are run-length-encoded. 3948 */ 3949 for (i = 0; i < extent_pages_length / sizeof(desc_extent_table->extent_page[0]); i++) { 3950 if (desc_extent_table->extent_page[i].page_idx != 0) { 3951 ctx->extent_page_num[ctx->num_extent_pages] = desc_extent_table->extent_page[i].page_idx; 3952 ctx->num_extent_pages += 1; 3953 } 3954 } 3955 } 3956 } else { 3957 /* Error */ 3958 return -EINVAL; 3959 } 3960 /* Advance to the next descriptor */ 3961 cur_desc += sizeof(*desc) + desc->length; 3962 if (cur_desc + sizeof(*desc) > sizeof(page->descriptors)) { 3963 break; 3964 } 3965 desc = (struct spdk_blob_md_descriptor *)((uintptr_t)page->descriptors + cur_desc); 3966 } 3967 return 0; 3968 } 3969 3970 static bool bs_load_cur_extent_page_valid(struct spdk_blob_md_page *page) 3971 { 3972 uint32_t crc; 3973 struct spdk_blob_md_descriptor *desc = (struct spdk_blob_md_descriptor *)page->descriptors; 3974 size_t desc_len; 3975 3976 crc = blob_md_page_calc_crc(page); 3977 if (crc != page->crc) { 3978 return false; 3979 } 3980 3981 /* Extent page should always be of sequence num 0. */ 3982 if (page->sequence_num != 0) { 3983 return false; 3984 } 3985 3986 /* Descriptor type must be EXTENT_PAGE. */ 3987 if (desc->type != SPDK_MD_DESCRIPTOR_TYPE_EXTENT_PAGE) { 3988 return false; 3989 } 3990 3991 /* Descriptor length cannot exceed the page. */ 3992 desc_len = sizeof(*desc) + desc->length; 3993 if (desc_len > sizeof(page->descriptors)) { 3994 return false; 3995 } 3996 3997 /* It has to be the only descriptor in the page. */ 3998 if (desc_len + sizeof(*desc) <= sizeof(page->descriptors)) { 3999 desc = (struct spdk_blob_md_descriptor *)((uintptr_t)page->descriptors + desc_len); 4000 if (desc->length != 0) { 4001 return false; 4002 } 4003 } 4004 4005 return true; 4006 } 4007 4008 static bool bs_load_cur_md_page_valid(struct spdk_bs_load_ctx *ctx) 4009 { 4010 uint32_t crc; 4011 struct spdk_blob_md_page *page = ctx->page; 4012 4013 crc = blob_md_page_calc_crc(page); 4014 if (crc != page->crc) { 4015 return false; 4016 } 4017 4018 /* First page of a sequence should match the blobid. */ 4019 if (page->sequence_num == 0 && 4020 bs_page_to_blobid(ctx->cur_page) != page->id) { 4021 return false; 4022 } 4023 assert(bs_load_cur_extent_page_valid(page) == false); 4024 4025 return true; 4026 } 4027 4028 static void 4029 bs_load_replay_cur_md_page(struct spdk_bs_load_ctx *ctx); 4030 4031 static void 4032 bs_load_write_used_clusters_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 4033 { 4034 struct spdk_bs_load_ctx *ctx = cb_arg; 4035 4036 if (bserrno != 0) { 4037 bs_load_ctx_fail(ctx, bserrno); 4038 return; 4039 } 4040 4041 bs_load_complete(ctx); 4042 } 4043 4044 static void 4045 bs_load_write_used_blobids_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 4046 { 4047 struct spdk_bs_load_ctx *ctx = cb_arg; 4048 4049 spdk_free(ctx->mask); 4050 ctx->mask = NULL; 4051 4052 if (bserrno != 0) { 4053 bs_load_ctx_fail(ctx, bserrno); 4054 return; 4055 } 4056 4057 bs_write_used_clusters(seq, ctx, bs_load_write_used_clusters_cpl); 4058 } 4059 4060 static void 4061 bs_load_write_used_pages_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 4062 { 4063 struct spdk_bs_load_ctx *ctx = cb_arg; 4064 4065 spdk_free(ctx->mask); 4066 ctx->mask = NULL; 4067 4068 if (bserrno != 0) { 4069 bs_load_ctx_fail(ctx, bserrno); 4070 return; 4071 } 4072 4073 bs_write_used_blobids(seq, ctx, bs_load_write_used_blobids_cpl); 4074 } 4075 4076 static void 4077 bs_load_write_used_md(struct spdk_bs_load_ctx *ctx) 4078 { 4079 bs_write_used_md(ctx->seq, ctx, bs_load_write_used_pages_cpl); 4080 } 4081 4082 static void 4083 bs_load_replay_md_chain_cpl(struct spdk_bs_load_ctx *ctx) 4084 { 4085 uint64_t num_md_clusters; 4086 uint64_t i; 4087 4088 ctx->in_page_chain = false; 4089 4090 do { 4091 ctx->page_index++; 4092 } while (spdk_bit_array_get(ctx->bs->used_md_pages, ctx->page_index) == true); 4093 4094 if (ctx->page_index < ctx->super->md_len) { 4095 ctx->cur_page = ctx->page_index; 4096 bs_load_replay_cur_md_page(ctx); 4097 } else { 4098 /* Claim all of the clusters used by the metadata */ 4099 num_md_clusters = spdk_divide_round_up(ctx->super->md_len, ctx->bs->pages_per_cluster); 4100 for (i = 0; i < num_md_clusters; i++) { 4101 spdk_bit_array_set(ctx->used_clusters, i); 4102 } 4103 ctx->bs->num_free_clusters -= num_md_clusters; 4104 spdk_free(ctx->page); 4105 bs_load_write_used_md(ctx); 4106 } 4107 } 4108 4109 static void 4110 bs_load_replay_extent_page_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 4111 { 4112 struct spdk_bs_load_ctx *ctx = cb_arg; 4113 uint32_t page_num; 4114 uint64_t i; 4115 4116 if (bserrno != 0) { 4117 spdk_free(ctx->extent_pages); 4118 bs_load_ctx_fail(ctx, bserrno); 4119 return; 4120 } 4121 4122 for (i = 0; i < ctx->num_extent_pages; i++) { 4123 /* Extent pages are only read when present within in chain md. 4124 * Integrity of md is not right if that page was not a valid extent page. */ 4125 if (bs_load_cur_extent_page_valid(&ctx->extent_pages[i]) != true) { 4126 spdk_free(ctx->extent_pages); 4127 bs_load_ctx_fail(ctx, -EILSEQ); 4128 return; 4129 } 4130 4131 page_num = ctx->extent_page_num[i]; 4132 spdk_bit_array_set(ctx->bs->used_md_pages, page_num); 4133 if (bs_load_replay_md_parse_page(ctx, &ctx->extent_pages[i])) { 4134 spdk_free(ctx->extent_pages); 4135 bs_load_ctx_fail(ctx, -EILSEQ); 4136 return; 4137 } 4138 } 4139 4140 spdk_free(ctx->extent_pages); 4141 free(ctx->extent_page_num); 4142 ctx->extent_page_num = NULL; 4143 ctx->num_extent_pages = 0; 4144 4145 bs_load_replay_md_chain_cpl(ctx); 4146 } 4147 4148 static void 4149 bs_load_replay_extent_pages(struct spdk_bs_load_ctx *ctx) 4150 { 4151 spdk_bs_batch_t *batch; 4152 uint32_t page; 4153 uint64_t lba; 4154 uint64_t i; 4155 4156 ctx->extent_pages = spdk_zmalloc(SPDK_BS_PAGE_SIZE * ctx->num_extent_pages, 0, 4157 NULL, SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA); 4158 if (!ctx->extent_pages) { 4159 bs_load_ctx_fail(ctx, -ENOMEM); 4160 return; 4161 } 4162 4163 batch = bs_sequence_to_batch(ctx->seq, bs_load_replay_extent_page_cpl, ctx); 4164 4165 for (i = 0; i < ctx->num_extent_pages; i++) { 4166 page = ctx->extent_page_num[i]; 4167 assert(page < ctx->super->md_len); 4168 lba = bs_md_page_to_lba(ctx->bs, page); 4169 bs_batch_read_dev(batch, &ctx->extent_pages[i], lba, 4170 bs_byte_to_lba(ctx->bs, SPDK_BS_PAGE_SIZE)); 4171 } 4172 4173 bs_batch_close(batch); 4174 } 4175 4176 static void 4177 bs_load_replay_md_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 4178 { 4179 struct spdk_bs_load_ctx *ctx = cb_arg; 4180 uint32_t page_num; 4181 struct spdk_blob_md_page *page; 4182 4183 if (bserrno != 0) { 4184 bs_load_ctx_fail(ctx, bserrno); 4185 return; 4186 } 4187 4188 page_num = ctx->cur_page; 4189 page = ctx->page; 4190 if (bs_load_cur_md_page_valid(ctx) == true) { 4191 if (page->sequence_num == 0 || ctx->in_page_chain == true) { 4192 bs_claim_md_page(ctx->bs, page_num); 4193 if (page->sequence_num == 0) { 4194 spdk_bit_array_set(ctx->bs->used_blobids, page_num); 4195 } 4196 if (bs_load_replay_md_parse_page(ctx, page)) { 4197 bs_load_ctx_fail(ctx, -EILSEQ); 4198 return; 4199 } 4200 if (page->next != SPDK_INVALID_MD_PAGE) { 4201 ctx->in_page_chain = true; 4202 ctx->cur_page = page->next; 4203 bs_load_replay_cur_md_page(ctx); 4204 return; 4205 } 4206 if (ctx->num_extent_pages != 0) { 4207 bs_load_replay_extent_pages(ctx); 4208 return; 4209 } 4210 } 4211 } 4212 bs_load_replay_md_chain_cpl(ctx); 4213 } 4214 4215 static void 4216 bs_load_replay_cur_md_page(struct spdk_bs_load_ctx *ctx) 4217 { 4218 uint64_t lba; 4219 4220 assert(ctx->cur_page < ctx->super->md_len); 4221 lba = bs_md_page_to_lba(ctx->bs, ctx->cur_page); 4222 bs_sequence_read_dev(ctx->seq, ctx->page, lba, 4223 bs_byte_to_lba(ctx->bs, SPDK_BS_PAGE_SIZE), 4224 bs_load_replay_md_cpl, ctx); 4225 } 4226 4227 static void 4228 bs_load_replay_md(struct spdk_bs_load_ctx *ctx) 4229 { 4230 ctx->page_index = 0; 4231 ctx->cur_page = 0; 4232 ctx->page = spdk_zmalloc(SPDK_BS_PAGE_SIZE, 0, 4233 NULL, SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA); 4234 if (!ctx->page) { 4235 bs_load_ctx_fail(ctx, -ENOMEM); 4236 return; 4237 } 4238 bs_load_replay_cur_md_page(ctx); 4239 } 4240 4241 static void 4242 bs_recover(struct spdk_bs_load_ctx *ctx) 4243 { 4244 int rc; 4245 4246 rc = spdk_bit_array_resize(&ctx->bs->used_md_pages, ctx->super->md_len); 4247 if (rc < 0) { 4248 bs_load_ctx_fail(ctx, -ENOMEM); 4249 return; 4250 } 4251 4252 rc = spdk_bit_array_resize(&ctx->bs->used_blobids, ctx->super->md_len); 4253 if (rc < 0) { 4254 bs_load_ctx_fail(ctx, -ENOMEM); 4255 return; 4256 } 4257 4258 rc = spdk_bit_array_resize(&ctx->used_clusters, ctx->bs->total_clusters); 4259 if (rc < 0) { 4260 bs_load_ctx_fail(ctx, -ENOMEM); 4261 return; 4262 } 4263 4264 rc = spdk_bit_array_resize(&ctx->bs->open_blobids, ctx->super->md_len); 4265 if (rc < 0) { 4266 bs_load_ctx_fail(ctx, -ENOMEM); 4267 return; 4268 } 4269 4270 ctx->bs->num_free_clusters = ctx->bs->total_clusters; 4271 bs_load_replay_md(ctx); 4272 } 4273 4274 static void 4275 bs_load_super_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 4276 { 4277 struct spdk_bs_load_ctx *ctx = cb_arg; 4278 uint32_t crc; 4279 int rc; 4280 static const char zeros[SPDK_BLOBSTORE_TYPE_LENGTH]; 4281 4282 if (ctx->super->version > SPDK_BS_VERSION || 4283 ctx->super->version < SPDK_BS_INITIAL_VERSION) { 4284 bs_load_ctx_fail(ctx, -EILSEQ); 4285 return; 4286 } 4287 4288 if (memcmp(ctx->super->signature, SPDK_BS_SUPER_BLOCK_SIG, 4289 sizeof(ctx->super->signature)) != 0) { 4290 bs_load_ctx_fail(ctx, -EILSEQ); 4291 return; 4292 } 4293 4294 crc = blob_md_page_calc_crc(ctx->super); 4295 if (crc != ctx->super->crc) { 4296 bs_load_ctx_fail(ctx, -EILSEQ); 4297 return; 4298 } 4299 4300 if (memcmp(&ctx->bs->bstype, &ctx->super->bstype, SPDK_BLOBSTORE_TYPE_LENGTH) == 0) { 4301 SPDK_DEBUGLOG(blob, "Bstype matched - loading blobstore\n"); 4302 } else if (memcmp(&ctx->bs->bstype, zeros, SPDK_BLOBSTORE_TYPE_LENGTH) == 0) { 4303 SPDK_DEBUGLOG(blob, "Bstype wildcard used - loading blobstore regardless bstype\n"); 4304 } else { 4305 SPDK_DEBUGLOG(blob, "Unexpected bstype\n"); 4306 SPDK_LOGDUMP(blob, "Expected:", ctx->bs->bstype.bstype, SPDK_BLOBSTORE_TYPE_LENGTH); 4307 SPDK_LOGDUMP(blob, "Found:", ctx->super->bstype.bstype, SPDK_BLOBSTORE_TYPE_LENGTH); 4308 bs_load_ctx_fail(ctx, -ENXIO); 4309 return; 4310 } 4311 4312 if (ctx->super->size > ctx->bs->dev->blockcnt * ctx->bs->dev->blocklen) { 4313 SPDK_NOTICELOG("Size mismatch, dev size: %" PRIu64 ", blobstore size: %" PRIu64 "\n", 4314 ctx->bs->dev->blockcnt * ctx->bs->dev->blocklen, ctx->super->size); 4315 bs_load_ctx_fail(ctx, -EILSEQ); 4316 return; 4317 } 4318 4319 if (ctx->super->size == 0) { 4320 ctx->super->size = ctx->bs->dev->blockcnt * ctx->bs->dev->blocklen; 4321 } 4322 4323 if (ctx->super->io_unit_size == 0) { 4324 ctx->super->io_unit_size = SPDK_BS_PAGE_SIZE; 4325 } 4326 4327 /* Parse the super block */ 4328 ctx->bs->clean = 1; 4329 ctx->bs->cluster_sz = ctx->super->cluster_size; 4330 ctx->bs->total_clusters = ctx->super->size / ctx->super->cluster_size; 4331 ctx->bs->pages_per_cluster = ctx->bs->cluster_sz / SPDK_BS_PAGE_SIZE; 4332 if (spdk_u32_is_pow2(ctx->bs->pages_per_cluster)) { 4333 ctx->bs->pages_per_cluster_shift = spdk_u32log2(ctx->bs->pages_per_cluster); 4334 } 4335 ctx->bs->io_unit_size = ctx->super->io_unit_size; 4336 rc = spdk_bit_array_resize(&ctx->used_clusters, ctx->bs->total_clusters); 4337 if (rc < 0) { 4338 bs_load_ctx_fail(ctx, -ENOMEM); 4339 return; 4340 } 4341 ctx->bs->md_start = ctx->super->md_start; 4342 ctx->bs->md_len = ctx->super->md_len; 4343 ctx->bs->total_data_clusters = ctx->bs->total_clusters - spdk_divide_round_up( 4344 ctx->bs->md_start + ctx->bs->md_len, ctx->bs->pages_per_cluster); 4345 ctx->bs->super_blob = ctx->super->super_blob; 4346 memcpy(&ctx->bs->bstype, &ctx->super->bstype, sizeof(ctx->super->bstype)); 4347 4348 if (ctx->super->used_blobid_mask_len == 0 || ctx->super->clean == 0) { 4349 bs_recover(ctx); 4350 } else { 4351 bs_load_read_used_pages(ctx); 4352 } 4353 } 4354 4355 static inline int 4356 bs_opts_copy(struct spdk_bs_opts *src, struct spdk_bs_opts *dst) 4357 { 4358 4359 if (!src->opts_size) { 4360 SPDK_ERRLOG("opts_size should not be zero value\n"); 4361 return -1; 4362 } 4363 4364 #define FIELD_OK(field) \ 4365 offsetof(struct spdk_bs_opts, field) + sizeof(src->field) <= src->opts_size 4366 4367 #define SET_FIELD(field) \ 4368 if (FIELD_OK(field)) { \ 4369 dst->field = src->field; \ 4370 } \ 4371 4372 SET_FIELD(cluster_sz); 4373 SET_FIELD(num_md_pages); 4374 SET_FIELD(max_md_ops); 4375 SET_FIELD(max_channel_ops); 4376 SET_FIELD(clear_method); 4377 4378 if (FIELD_OK(bstype)) { 4379 memcpy(&dst->bstype, &src->bstype, sizeof(dst->bstype)); 4380 } 4381 SET_FIELD(iter_cb_fn); 4382 SET_FIELD(iter_cb_arg); 4383 4384 dst->opts_size = src->opts_size; 4385 4386 /* You should not remove this statement, but need to update the assert statement 4387 * if you add a new field, and also add a corresponding SET_FIELD statement */ 4388 SPDK_STATIC_ASSERT(sizeof(struct spdk_bs_opts) == 64, "Incorrect size"); 4389 4390 #undef FIELD_OK 4391 #undef SET_FIELD 4392 4393 return 0; 4394 } 4395 4396 void 4397 spdk_bs_load(struct spdk_bs_dev *dev, struct spdk_bs_opts *o, 4398 spdk_bs_op_with_handle_complete cb_fn, void *cb_arg) 4399 { 4400 struct spdk_blob_store *bs; 4401 struct spdk_bs_cpl cpl; 4402 struct spdk_bs_load_ctx *ctx; 4403 struct spdk_bs_opts opts = {}; 4404 int err; 4405 4406 SPDK_DEBUGLOG(blob, "Loading blobstore from dev %p\n", dev); 4407 4408 if ((SPDK_BS_PAGE_SIZE % dev->blocklen) != 0) { 4409 SPDK_DEBUGLOG(blob, "unsupported dev block length of %d\n", dev->blocklen); 4410 dev->destroy(dev); 4411 cb_fn(cb_arg, NULL, -EINVAL); 4412 return; 4413 } 4414 4415 spdk_bs_opts_init(&opts, sizeof(opts)); 4416 if (o) { 4417 if (bs_opts_copy(o, &opts)) { 4418 return; 4419 } 4420 } 4421 4422 if (opts.max_md_ops == 0 || opts.max_channel_ops == 0) { 4423 dev->destroy(dev); 4424 cb_fn(cb_arg, NULL, -EINVAL); 4425 return; 4426 } 4427 4428 err = bs_alloc(dev, &opts, &bs, &ctx); 4429 if (err) { 4430 dev->destroy(dev); 4431 cb_fn(cb_arg, NULL, err); 4432 return; 4433 } 4434 4435 cpl.type = SPDK_BS_CPL_TYPE_BS_HANDLE; 4436 cpl.u.bs_handle.cb_fn = cb_fn; 4437 cpl.u.bs_handle.cb_arg = cb_arg; 4438 cpl.u.bs_handle.bs = bs; 4439 4440 ctx->seq = bs_sequence_start(bs->md_channel, &cpl); 4441 if (!ctx->seq) { 4442 spdk_free(ctx->super); 4443 free(ctx); 4444 bs_free(bs); 4445 cb_fn(cb_arg, NULL, -ENOMEM); 4446 return; 4447 } 4448 4449 /* Read the super block */ 4450 bs_sequence_read_dev(ctx->seq, ctx->super, bs_page_to_lba(bs, 0), 4451 bs_byte_to_lba(bs, sizeof(*ctx->super)), 4452 bs_load_super_cpl, ctx); 4453 } 4454 4455 /* END spdk_bs_load */ 4456 4457 /* START spdk_bs_dump */ 4458 4459 static void 4460 bs_dump_finish(spdk_bs_sequence_t *seq, struct spdk_bs_load_ctx *ctx, int bserrno) 4461 { 4462 spdk_free(ctx->super); 4463 4464 /* 4465 * We need to defer calling bs_call_cpl() until after 4466 * dev destruction, so tuck these away for later use. 4467 */ 4468 ctx->bs->unload_err = bserrno; 4469 memcpy(&ctx->bs->unload_cpl, &seq->cpl, sizeof(struct spdk_bs_cpl)); 4470 seq->cpl.type = SPDK_BS_CPL_TYPE_NONE; 4471 4472 bs_sequence_finish(seq, 0); 4473 bs_free(ctx->bs); 4474 free(ctx); 4475 } 4476 4477 static void bs_dump_read_md_page(spdk_bs_sequence_t *seq, void *cb_arg); 4478 4479 static void 4480 bs_dump_print_md_page(struct spdk_bs_load_ctx *ctx) 4481 { 4482 uint32_t page_idx = ctx->cur_page; 4483 struct spdk_blob_md_page *page = ctx->page; 4484 struct spdk_blob_md_descriptor *desc; 4485 size_t cur_desc = 0; 4486 uint32_t crc; 4487 4488 fprintf(ctx->fp, "=========\n"); 4489 fprintf(ctx->fp, "Metadata Page Index: %" PRIu32 " (0x%" PRIx32 ")\n", page_idx, page_idx); 4490 fprintf(ctx->fp, "Blob ID: 0x%" PRIx64 "\n", page->id); 4491 4492 crc = blob_md_page_calc_crc(page); 4493 fprintf(ctx->fp, "CRC: 0x%" PRIx32 " (%s)\n", page->crc, crc == page->crc ? "OK" : "Mismatch"); 4494 4495 desc = (struct spdk_blob_md_descriptor *)page->descriptors; 4496 while (cur_desc < sizeof(page->descriptors)) { 4497 if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_PADDING) { 4498 if (desc->length == 0) { 4499 /* If padding and length are 0, this terminates the page */ 4500 break; 4501 } 4502 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_EXTENT_RLE) { 4503 struct spdk_blob_md_descriptor_extent_rle *desc_extent_rle; 4504 unsigned int i; 4505 4506 desc_extent_rle = (struct spdk_blob_md_descriptor_extent_rle *)desc; 4507 4508 for (i = 0; i < desc_extent_rle->length / sizeof(desc_extent_rle->extents[0]); i++) { 4509 if (desc_extent_rle->extents[i].cluster_idx != 0) { 4510 fprintf(ctx->fp, "Allocated Extent - Start: %" PRIu32, 4511 desc_extent_rle->extents[i].cluster_idx); 4512 } else { 4513 fprintf(ctx->fp, "Unallocated Extent - "); 4514 } 4515 fprintf(ctx->fp, " Length: %" PRIu32, desc_extent_rle->extents[i].length); 4516 fprintf(ctx->fp, "\n"); 4517 } 4518 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_EXTENT_PAGE) { 4519 struct spdk_blob_md_descriptor_extent_page *desc_extent; 4520 unsigned int i; 4521 4522 desc_extent = (struct spdk_blob_md_descriptor_extent_page *)desc; 4523 4524 for (i = 0; i < desc_extent->length / sizeof(desc_extent->cluster_idx[0]); i++) { 4525 if (desc_extent->cluster_idx[i] != 0) { 4526 fprintf(ctx->fp, "Allocated Extent - Start: %" PRIu32, 4527 desc_extent->cluster_idx[i]); 4528 } else { 4529 fprintf(ctx->fp, "Unallocated Extent"); 4530 } 4531 fprintf(ctx->fp, "\n"); 4532 } 4533 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_XATTR) { 4534 struct spdk_blob_md_descriptor_xattr *desc_xattr; 4535 uint32_t i; 4536 4537 desc_xattr = (struct spdk_blob_md_descriptor_xattr *)desc; 4538 4539 if (desc_xattr->length != 4540 sizeof(desc_xattr->name_length) + sizeof(desc_xattr->value_length) + 4541 desc_xattr->name_length + desc_xattr->value_length) { 4542 } 4543 4544 memcpy(ctx->xattr_name, desc_xattr->name, desc_xattr->name_length); 4545 ctx->xattr_name[desc_xattr->name_length] = '\0'; 4546 fprintf(ctx->fp, "XATTR: name = \"%s\"\n", ctx->xattr_name); 4547 fprintf(ctx->fp, " value = \""); 4548 ctx->print_xattr_fn(ctx->fp, ctx->super->bstype.bstype, ctx->xattr_name, 4549 (void *)((uintptr_t)desc_xattr->name + desc_xattr->name_length), 4550 desc_xattr->value_length); 4551 fprintf(ctx->fp, "\"\n"); 4552 for (i = 0; i < desc_xattr->value_length; i++) { 4553 if (i % 16 == 0) { 4554 fprintf(ctx->fp, " "); 4555 } 4556 fprintf(ctx->fp, "%02" PRIx8 " ", *((uint8_t *)desc_xattr->name + desc_xattr->name_length + i)); 4557 if ((i + 1) % 16 == 0) { 4558 fprintf(ctx->fp, "\n"); 4559 } 4560 } 4561 if (i % 16 != 0) { 4562 fprintf(ctx->fp, "\n"); 4563 } 4564 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_XATTR_INTERNAL) { 4565 /* TODO */ 4566 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_FLAGS) { 4567 /* TODO */ 4568 } else { 4569 /* Error */ 4570 } 4571 /* Advance to the next descriptor */ 4572 cur_desc += sizeof(*desc) + desc->length; 4573 if (cur_desc + sizeof(*desc) > sizeof(page->descriptors)) { 4574 break; 4575 } 4576 desc = (struct spdk_blob_md_descriptor *)((uintptr_t)page->descriptors + cur_desc); 4577 } 4578 } 4579 4580 static void 4581 bs_dump_read_md_page_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 4582 { 4583 struct spdk_bs_load_ctx *ctx = cb_arg; 4584 4585 if (bserrno != 0) { 4586 bs_dump_finish(seq, ctx, bserrno); 4587 return; 4588 } 4589 4590 if (ctx->page->id != 0) { 4591 bs_dump_print_md_page(ctx); 4592 } 4593 4594 ctx->cur_page++; 4595 4596 if (ctx->cur_page < ctx->super->md_len) { 4597 bs_dump_read_md_page(seq, ctx); 4598 } else { 4599 spdk_free(ctx->page); 4600 bs_dump_finish(seq, ctx, 0); 4601 } 4602 } 4603 4604 static void 4605 bs_dump_read_md_page(spdk_bs_sequence_t *seq, void *cb_arg) 4606 { 4607 struct spdk_bs_load_ctx *ctx = cb_arg; 4608 uint64_t lba; 4609 4610 assert(ctx->cur_page < ctx->super->md_len); 4611 lba = bs_page_to_lba(ctx->bs, ctx->super->md_start + ctx->cur_page); 4612 bs_sequence_read_dev(seq, ctx->page, lba, 4613 bs_byte_to_lba(ctx->bs, SPDK_BS_PAGE_SIZE), 4614 bs_dump_read_md_page_cpl, ctx); 4615 } 4616 4617 static void 4618 bs_dump_super_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 4619 { 4620 struct spdk_bs_load_ctx *ctx = cb_arg; 4621 4622 fprintf(ctx->fp, "Signature: \"%.8s\" ", ctx->super->signature); 4623 if (memcmp(ctx->super->signature, SPDK_BS_SUPER_BLOCK_SIG, 4624 sizeof(ctx->super->signature)) != 0) { 4625 fprintf(ctx->fp, "(Mismatch)\n"); 4626 bs_dump_finish(seq, ctx, bserrno); 4627 return; 4628 } else { 4629 fprintf(ctx->fp, "(OK)\n"); 4630 } 4631 fprintf(ctx->fp, "Version: %" PRIu32 "\n", ctx->super->version); 4632 fprintf(ctx->fp, "CRC: 0x%x (%s)\n", ctx->super->crc, 4633 (ctx->super->crc == blob_md_page_calc_crc(ctx->super)) ? "OK" : "Mismatch"); 4634 fprintf(ctx->fp, "Blobstore Type: %.*s\n", SPDK_BLOBSTORE_TYPE_LENGTH, ctx->super->bstype.bstype); 4635 fprintf(ctx->fp, "Cluster Size: %" PRIu32 "\n", ctx->super->cluster_size); 4636 fprintf(ctx->fp, "Super Blob ID: "); 4637 if (ctx->super->super_blob == SPDK_BLOBID_INVALID) { 4638 fprintf(ctx->fp, "(None)\n"); 4639 } else { 4640 fprintf(ctx->fp, "%" PRIu64 "\n", ctx->super->super_blob); 4641 } 4642 fprintf(ctx->fp, "Clean: %" PRIu32 "\n", ctx->super->clean); 4643 fprintf(ctx->fp, "Used Metadata Page Mask Start: %" PRIu32 "\n", ctx->super->used_page_mask_start); 4644 fprintf(ctx->fp, "Used Metadata Page Mask Length: %" PRIu32 "\n", ctx->super->used_page_mask_len); 4645 fprintf(ctx->fp, "Used Cluster Mask Start: %" PRIu32 "\n", ctx->super->used_cluster_mask_start); 4646 fprintf(ctx->fp, "Used Cluster Mask Length: %" PRIu32 "\n", ctx->super->used_cluster_mask_len); 4647 fprintf(ctx->fp, "Used Blob ID Mask Start: %" PRIu32 "\n", ctx->super->used_blobid_mask_start); 4648 fprintf(ctx->fp, "Used Blob ID Mask Length: %" PRIu32 "\n", ctx->super->used_blobid_mask_len); 4649 fprintf(ctx->fp, "Metadata Start: %" PRIu32 "\n", ctx->super->md_start); 4650 fprintf(ctx->fp, "Metadata Length: %" PRIu32 "\n", ctx->super->md_len); 4651 4652 ctx->cur_page = 0; 4653 ctx->page = spdk_zmalloc(SPDK_BS_PAGE_SIZE, 0, 4654 NULL, SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA); 4655 if (!ctx->page) { 4656 bs_dump_finish(seq, ctx, -ENOMEM); 4657 return; 4658 } 4659 bs_dump_read_md_page(seq, ctx); 4660 } 4661 4662 void 4663 spdk_bs_dump(struct spdk_bs_dev *dev, FILE *fp, spdk_bs_dump_print_xattr print_xattr_fn, 4664 spdk_bs_op_complete cb_fn, void *cb_arg) 4665 { 4666 struct spdk_blob_store *bs; 4667 struct spdk_bs_cpl cpl; 4668 spdk_bs_sequence_t *seq; 4669 struct spdk_bs_load_ctx *ctx; 4670 struct spdk_bs_opts opts = {}; 4671 int err; 4672 4673 SPDK_DEBUGLOG(blob, "Dumping blobstore from dev %p\n", dev); 4674 4675 spdk_bs_opts_init(&opts, sizeof(opts)); 4676 4677 err = bs_alloc(dev, &opts, &bs, &ctx); 4678 if (err) { 4679 dev->destroy(dev); 4680 cb_fn(cb_arg, err); 4681 return; 4682 } 4683 4684 ctx->fp = fp; 4685 ctx->print_xattr_fn = print_xattr_fn; 4686 4687 cpl.type = SPDK_BS_CPL_TYPE_BS_BASIC; 4688 cpl.u.bs_basic.cb_fn = cb_fn; 4689 cpl.u.bs_basic.cb_arg = cb_arg; 4690 4691 seq = bs_sequence_start(bs->md_channel, &cpl); 4692 if (!seq) { 4693 spdk_free(ctx->super); 4694 free(ctx); 4695 bs_free(bs); 4696 cb_fn(cb_arg, -ENOMEM); 4697 return; 4698 } 4699 4700 /* Read the super block */ 4701 bs_sequence_read_dev(seq, ctx->super, bs_page_to_lba(bs, 0), 4702 bs_byte_to_lba(bs, sizeof(*ctx->super)), 4703 bs_dump_super_cpl, ctx); 4704 } 4705 4706 /* END spdk_bs_dump */ 4707 4708 /* START spdk_bs_init */ 4709 4710 static void 4711 bs_init_persist_super_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 4712 { 4713 struct spdk_bs_load_ctx *ctx = cb_arg; 4714 4715 ctx->bs->used_clusters = spdk_bit_pool_create_from_array(ctx->used_clusters); 4716 spdk_free(ctx->super); 4717 free(ctx); 4718 4719 bs_sequence_finish(seq, bserrno); 4720 } 4721 4722 static void 4723 bs_init_trim_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 4724 { 4725 struct spdk_bs_load_ctx *ctx = cb_arg; 4726 4727 /* Write super block */ 4728 bs_sequence_write_dev(seq, ctx->super, bs_page_to_lba(ctx->bs, 0), 4729 bs_byte_to_lba(ctx->bs, sizeof(*ctx->super)), 4730 bs_init_persist_super_cpl, ctx); 4731 } 4732 4733 void 4734 spdk_bs_init(struct spdk_bs_dev *dev, struct spdk_bs_opts *o, 4735 spdk_bs_op_with_handle_complete cb_fn, void *cb_arg) 4736 { 4737 struct spdk_bs_load_ctx *ctx; 4738 struct spdk_blob_store *bs; 4739 struct spdk_bs_cpl cpl; 4740 spdk_bs_sequence_t *seq; 4741 spdk_bs_batch_t *batch; 4742 uint64_t num_md_lba; 4743 uint64_t num_md_pages; 4744 uint64_t num_md_clusters; 4745 uint32_t i; 4746 struct spdk_bs_opts opts = {}; 4747 int rc; 4748 uint64_t lba, lba_count; 4749 4750 SPDK_DEBUGLOG(blob, "Initializing blobstore on dev %p\n", dev); 4751 4752 if ((SPDK_BS_PAGE_SIZE % dev->blocklen) != 0) { 4753 SPDK_ERRLOG("unsupported dev block length of %d\n", 4754 dev->blocklen); 4755 dev->destroy(dev); 4756 cb_fn(cb_arg, NULL, -EINVAL); 4757 return; 4758 } 4759 4760 spdk_bs_opts_init(&opts, sizeof(opts)); 4761 if (o) { 4762 if (bs_opts_copy(o, &opts)) { 4763 return; 4764 } 4765 } 4766 4767 if (bs_opts_verify(&opts) != 0) { 4768 dev->destroy(dev); 4769 cb_fn(cb_arg, NULL, -EINVAL); 4770 return; 4771 } 4772 4773 rc = bs_alloc(dev, &opts, &bs, &ctx); 4774 if (rc) { 4775 dev->destroy(dev); 4776 cb_fn(cb_arg, NULL, rc); 4777 return; 4778 } 4779 4780 if (opts.num_md_pages == SPDK_BLOB_OPTS_NUM_MD_PAGES) { 4781 /* By default, allocate 1 page per cluster. 4782 * Technically, this over-allocates metadata 4783 * because more metadata will reduce the number 4784 * of usable clusters. This can be addressed with 4785 * more complex math in the future. 4786 */ 4787 bs->md_len = bs->total_clusters; 4788 } else { 4789 bs->md_len = opts.num_md_pages; 4790 } 4791 rc = spdk_bit_array_resize(&bs->used_md_pages, bs->md_len); 4792 if (rc < 0) { 4793 spdk_free(ctx->super); 4794 free(ctx); 4795 bs_free(bs); 4796 cb_fn(cb_arg, NULL, -ENOMEM); 4797 return; 4798 } 4799 4800 rc = spdk_bit_array_resize(&bs->used_blobids, bs->md_len); 4801 if (rc < 0) { 4802 spdk_free(ctx->super); 4803 free(ctx); 4804 bs_free(bs); 4805 cb_fn(cb_arg, NULL, -ENOMEM); 4806 return; 4807 } 4808 4809 rc = spdk_bit_array_resize(&bs->open_blobids, bs->md_len); 4810 if (rc < 0) { 4811 spdk_free(ctx->super); 4812 free(ctx); 4813 bs_free(bs); 4814 cb_fn(cb_arg, NULL, -ENOMEM); 4815 return; 4816 } 4817 4818 memcpy(ctx->super->signature, SPDK_BS_SUPER_BLOCK_SIG, 4819 sizeof(ctx->super->signature)); 4820 ctx->super->version = SPDK_BS_VERSION; 4821 ctx->super->length = sizeof(*ctx->super); 4822 ctx->super->super_blob = bs->super_blob; 4823 ctx->super->clean = 0; 4824 ctx->super->cluster_size = bs->cluster_sz; 4825 ctx->super->io_unit_size = bs->io_unit_size; 4826 memcpy(&ctx->super->bstype, &bs->bstype, sizeof(bs->bstype)); 4827 4828 /* Calculate how many pages the metadata consumes at the front 4829 * of the disk. 4830 */ 4831 4832 /* The super block uses 1 page */ 4833 num_md_pages = 1; 4834 4835 /* The used_md_pages mask requires 1 bit per metadata page, rounded 4836 * up to the nearest page, plus a header. 4837 */ 4838 ctx->super->used_page_mask_start = num_md_pages; 4839 ctx->super->used_page_mask_len = spdk_divide_round_up(sizeof(struct spdk_bs_md_mask) + 4840 spdk_divide_round_up(bs->md_len, 8), 4841 SPDK_BS_PAGE_SIZE); 4842 num_md_pages += ctx->super->used_page_mask_len; 4843 4844 /* The used_clusters mask requires 1 bit per cluster, rounded 4845 * up to the nearest page, plus a header. 4846 */ 4847 ctx->super->used_cluster_mask_start = num_md_pages; 4848 ctx->super->used_cluster_mask_len = spdk_divide_round_up(sizeof(struct spdk_bs_md_mask) + 4849 spdk_divide_round_up(bs->total_clusters, 8), 4850 SPDK_BS_PAGE_SIZE); 4851 num_md_pages += ctx->super->used_cluster_mask_len; 4852 4853 /* The used_blobids mask requires 1 bit per metadata page, rounded 4854 * up to the nearest page, plus a header. 4855 */ 4856 ctx->super->used_blobid_mask_start = num_md_pages; 4857 ctx->super->used_blobid_mask_len = spdk_divide_round_up(sizeof(struct spdk_bs_md_mask) + 4858 spdk_divide_round_up(bs->md_len, 8), 4859 SPDK_BS_PAGE_SIZE); 4860 num_md_pages += ctx->super->used_blobid_mask_len; 4861 4862 /* The metadata region size was chosen above */ 4863 ctx->super->md_start = bs->md_start = num_md_pages; 4864 ctx->super->md_len = bs->md_len; 4865 num_md_pages += bs->md_len; 4866 4867 num_md_lba = bs_page_to_lba(bs, num_md_pages); 4868 4869 ctx->super->size = dev->blockcnt * dev->blocklen; 4870 4871 ctx->super->crc = blob_md_page_calc_crc(ctx->super); 4872 4873 num_md_clusters = spdk_divide_round_up(num_md_pages, bs->pages_per_cluster); 4874 if (num_md_clusters > bs->total_clusters) { 4875 SPDK_ERRLOG("Blobstore metadata cannot use more clusters than is available, " 4876 "please decrease number of pages reserved for metadata " 4877 "or increase cluster size.\n"); 4878 spdk_free(ctx->super); 4879 spdk_bit_array_free(&ctx->used_clusters); 4880 free(ctx); 4881 bs_free(bs); 4882 cb_fn(cb_arg, NULL, -ENOMEM); 4883 return; 4884 } 4885 /* Claim all of the clusters used by the metadata */ 4886 for (i = 0; i < num_md_clusters; i++) { 4887 spdk_bit_array_set(ctx->used_clusters, i); 4888 } 4889 4890 bs->num_free_clusters -= num_md_clusters; 4891 bs->total_data_clusters = bs->num_free_clusters; 4892 4893 cpl.type = SPDK_BS_CPL_TYPE_BS_HANDLE; 4894 cpl.u.bs_handle.cb_fn = cb_fn; 4895 cpl.u.bs_handle.cb_arg = cb_arg; 4896 cpl.u.bs_handle.bs = bs; 4897 4898 seq = bs_sequence_start(bs->md_channel, &cpl); 4899 if (!seq) { 4900 spdk_free(ctx->super); 4901 free(ctx); 4902 bs_free(bs); 4903 cb_fn(cb_arg, NULL, -ENOMEM); 4904 return; 4905 } 4906 4907 batch = bs_sequence_to_batch(seq, bs_init_trim_cpl, ctx); 4908 4909 /* Clear metadata space */ 4910 bs_batch_write_zeroes_dev(batch, 0, num_md_lba); 4911 4912 lba = num_md_lba; 4913 while (lba < ctx->bs->dev->blockcnt) { 4914 lba_count = spdk_min(UINT32_MAX, ctx->bs->dev->blockcnt - lba); 4915 switch (opts.clear_method) { 4916 case BS_CLEAR_WITH_UNMAP: 4917 /* Trim data clusters */ 4918 bs_batch_unmap_dev(batch, lba, lba_count); 4919 break; 4920 case BS_CLEAR_WITH_WRITE_ZEROES: 4921 /* Write_zeroes to data clusters */ 4922 bs_batch_write_zeroes_dev(batch, lba, lba_count); 4923 break; 4924 case BS_CLEAR_WITH_NONE: 4925 default: 4926 break; 4927 } 4928 lba += lba_count; 4929 } 4930 4931 bs_batch_close(batch); 4932 } 4933 4934 /* END spdk_bs_init */ 4935 4936 /* START spdk_bs_destroy */ 4937 4938 static void 4939 bs_destroy_trim_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 4940 { 4941 struct spdk_bs_load_ctx *ctx = cb_arg; 4942 struct spdk_blob_store *bs = ctx->bs; 4943 4944 /* 4945 * We need to defer calling bs_call_cpl() until after 4946 * dev destruction, so tuck these away for later use. 4947 */ 4948 bs->unload_err = bserrno; 4949 memcpy(&bs->unload_cpl, &seq->cpl, sizeof(struct spdk_bs_cpl)); 4950 seq->cpl.type = SPDK_BS_CPL_TYPE_NONE; 4951 4952 bs_sequence_finish(seq, bserrno); 4953 4954 bs_free(bs); 4955 free(ctx); 4956 } 4957 4958 void 4959 spdk_bs_destroy(struct spdk_blob_store *bs, spdk_bs_op_complete cb_fn, 4960 void *cb_arg) 4961 { 4962 struct spdk_bs_cpl cpl; 4963 spdk_bs_sequence_t *seq; 4964 struct spdk_bs_load_ctx *ctx; 4965 4966 SPDK_DEBUGLOG(blob, "Destroying blobstore\n"); 4967 4968 if (!TAILQ_EMPTY(&bs->blobs)) { 4969 SPDK_ERRLOG("Blobstore still has open blobs\n"); 4970 cb_fn(cb_arg, -EBUSY); 4971 return; 4972 } 4973 4974 cpl.type = SPDK_BS_CPL_TYPE_BS_BASIC; 4975 cpl.u.bs_basic.cb_fn = cb_fn; 4976 cpl.u.bs_basic.cb_arg = cb_arg; 4977 4978 ctx = calloc(1, sizeof(*ctx)); 4979 if (!ctx) { 4980 cb_fn(cb_arg, -ENOMEM); 4981 return; 4982 } 4983 4984 ctx->bs = bs; 4985 4986 seq = bs_sequence_start(bs->md_channel, &cpl); 4987 if (!seq) { 4988 free(ctx); 4989 cb_fn(cb_arg, -ENOMEM); 4990 return; 4991 } 4992 4993 /* Write zeroes to the super block */ 4994 bs_sequence_write_zeroes_dev(seq, 4995 bs_page_to_lba(bs, 0), 4996 bs_byte_to_lba(bs, sizeof(struct spdk_bs_super_block)), 4997 bs_destroy_trim_cpl, ctx); 4998 } 4999 5000 /* END spdk_bs_destroy */ 5001 5002 /* START spdk_bs_unload */ 5003 5004 static void 5005 bs_unload_finish(struct spdk_bs_load_ctx *ctx, int bserrno) 5006 { 5007 spdk_bs_sequence_t *seq = ctx->seq; 5008 5009 spdk_free(ctx->super); 5010 5011 /* 5012 * We need to defer calling bs_call_cpl() until after 5013 * dev destruction, so tuck these away for later use. 5014 */ 5015 ctx->bs->unload_err = bserrno; 5016 memcpy(&ctx->bs->unload_cpl, &seq->cpl, sizeof(struct spdk_bs_cpl)); 5017 seq->cpl.type = SPDK_BS_CPL_TYPE_NONE; 5018 5019 bs_sequence_finish(seq, bserrno); 5020 5021 bs_free(ctx->bs); 5022 free(ctx); 5023 } 5024 5025 static void 5026 bs_unload_write_super_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 5027 { 5028 struct spdk_bs_load_ctx *ctx = cb_arg; 5029 5030 bs_unload_finish(ctx, bserrno); 5031 } 5032 5033 static void 5034 bs_unload_write_used_clusters_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 5035 { 5036 struct spdk_bs_load_ctx *ctx = cb_arg; 5037 5038 spdk_free(ctx->mask); 5039 5040 if (bserrno != 0) { 5041 bs_unload_finish(ctx, bserrno); 5042 return; 5043 } 5044 5045 ctx->super->clean = 1; 5046 5047 bs_write_super(seq, ctx->bs, ctx->super, bs_unload_write_super_cpl, ctx); 5048 } 5049 5050 static void 5051 bs_unload_write_used_blobids_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 5052 { 5053 struct spdk_bs_load_ctx *ctx = cb_arg; 5054 5055 spdk_free(ctx->mask); 5056 ctx->mask = NULL; 5057 5058 if (bserrno != 0) { 5059 bs_unload_finish(ctx, bserrno); 5060 return; 5061 } 5062 5063 bs_write_used_clusters(seq, ctx, bs_unload_write_used_clusters_cpl); 5064 } 5065 5066 static void 5067 bs_unload_write_used_pages_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 5068 { 5069 struct spdk_bs_load_ctx *ctx = cb_arg; 5070 5071 spdk_free(ctx->mask); 5072 ctx->mask = NULL; 5073 5074 if (bserrno != 0) { 5075 bs_unload_finish(ctx, bserrno); 5076 return; 5077 } 5078 5079 bs_write_used_blobids(seq, ctx, bs_unload_write_used_blobids_cpl); 5080 } 5081 5082 static void 5083 bs_unload_read_super_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 5084 { 5085 struct spdk_bs_load_ctx *ctx = cb_arg; 5086 5087 if (bserrno != 0) { 5088 bs_unload_finish(ctx, bserrno); 5089 return; 5090 } 5091 5092 bs_write_used_md(seq, cb_arg, bs_unload_write_used_pages_cpl); 5093 } 5094 5095 void 5096 spdk_bs_unload(struct spdk_blob_store *bs, spdk_bs_op_complete cb_fn, void *cb_arg) 5097 { 5098 struct spdk_bs_cpl cpl; 5099 struct spdk_bs_load_ctx *ctx; 5100 5101 SPDK_DEBUGLOG(blob, "Syncing blobstore\n"); 5102 5103 if (!TAILQ_EMPTY(&bs->blobs)) { 5104 SPDK_ERRLOG("Blobstore still has open blobs\n"); 5105 cb_fn(cb_arg, -EBUSY); 5106 return; 5107 } 5108 5109 ctx = calloc(1, sizeof(*ctx)); 5110 if (!ctx) { 5111 cb_fn(cb_arg, -ENOMEM); 5112 return; 5113 } 5114 5115 ctx->bs = bs; 5116 5117 ctx->super = spdk_zmalloc(sizeof(*ctx->super), 0x1000, NULL, 5118 SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA); 5119 if (!ctx->super) { 5120 free(ctx); 5121 cb_fn(cb_arg, -ENOMEM); 5122 return; 5123 } 5124 5125 cpl.type = SPDK_BS_CPL_TYPE_BS_BASIC; 5126 cpl.u.bs_basic.cb_fn = cb_fn; 5127 cpl.u.bs_basic.cb_arg = cb_arg; 5128 5129 ctx->seq = bs_sequence_start(bs->md_channel, &cpl); 5130 if (!ctx->seq) { 5131 spdk_free(ctx->super); 5132 free(ctx); 5133 cb_fn(cb_arg, -ENOMEM); 5134 return; 5135 } 5136 5137 /* Read super block */ 5138 bs_sequence_read_dev(ctx->seq, ctx->super, bs_page_to_lba(bs, 0), 5139 bs_byte_to_lba(bs, sizeof(*ctx->super)), 5140 bs_unload_read_super_cpl, ctx); 5141 } 5142 5143 /* END spdk_bs_unload */ 5144 5145 /* START spdk_bs_set_super */ 5146 5147 struct spdk_bs_set_super_ctx { 5148 struct spdk_blob_store *bs; 5149 struct spdk_bs_super_block *super; 5150 }; 5151 5152 static void 5153 bs_set_super_write_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 5154 { 5155 struct spdk_bs_set_super_ctx *ctx = cb_arg; 5156 5157 if (bserrno != 0) { 5158 SPDK_ERRLOG("Unable to write to super block of blobstore\n"); 5159 } 5160 5161 spdk_free(ctx->super); 5162 5163 bs_sequence_finish(seq, bserrno); 5164 5165 free(ctx); 5166 } 5167 5168 static void 5169 bs_set_super_read_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 5170 { 5171 struct spdk_bs_set_super_ctx *ctx = cb_arg; 5172 5173 if (bserrno != 0) { 5174 SPDK_ERRLOG("Unable to read super block of blobstore\n"); 5175 spdk_free(ctx->super); 5176 bs_sequence_finish(seq, bserrno); 5177 free(ctx); 5178 return; 5179 } 5180 5181 bs_write_super(seq, ctx->bs, ctx->super, bs_set_super_write_cpl, ctx); 5182 } 5183 5184 void 5185 spdk_bs_set_super(struct spdk_blob_store *bs, spdk_blob_id blobid, 5186 spdk_bs_op_complete cb_fn, void *cb_arg) 5187 { 5188 struct spdk_bs_cpl cpl; 5189 spdk_bs_sequence_t *seq; 5190 struct spdk_bs_set_super_ctx *ctx; 5191 5192 SPDK_DEBUGLOG(blob, "Setting super blob id on blobstore\n"); 5193 5194 ctx = calloc(1, sizeof(*ctx)); 5195 if (!ctx) { 5196 cb_fn(cb_arg, -ENOMEM); 5197 return; 5198 } 5199 5200 ctx->bs = bs; 5201 5202 ctx->super = spdk_zmalloc(sizeof(*ctx->super), 0x1000, NULL, 5203 SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA); 5204 if (!ctx->super) { 5205 free(ctx); 5206 cb_fn(cb_arg, -ENOMEM); 5207 return; 5208 } 5209 5210 cpl.type = SPDK_BS_CPL_TYPE_BS_BASIC; 5211 cpl.u.bs_basic.cb_fn = cb_fn; 5212 cpl.u.bs_basic.cb_arg = cb_arg; 5213 5214 seq = bs_sequence_start(bs->md_channel, &cpl); 5215 if (!seq) { 5216 spdk_free(ctx->super); 5217 free(ctx); 5218 cb_fn(cb_arg, -ENOMEM); 5219 return; 5220 } 5221 5222 bs->super_blob = blobid; 5223 5224 /* Read super block */ 5225 bs_sequence_read_dev(seq, ctx->super, bs_page_to_lba(bs, 0), 5226 bs_byte_to_lba(bs, sizeof(*ctx->super)), 5227 bs_set_super_read_cpl, ctx); 5228 } 5229 5230 /* END spdk_bs_set_super */ 5231 5232 void 5233 spdk_bs_get_super(struct spdk_blob_store *bs, 5234 spdk_blob_op_with_id_complete cb_fn, void *cb_arg) 5235 { 5236 if (bs->super_blob == SPDK_BLOBID_INVALID) { 5237 cb_fn(cb_arg, SPDK_BLOBID_INVALID, -ENOENT); 5238 } else { 5239 cb_fn(cb_arg, bs->super_blob, 0); 5240 } 5241 } 5242 5243 uint64_t 5244 spdk_bs_get_cluster_size(struct spdk_blob_store *bs) 5245 { 5246 return bs->cluster_sz; 5247 } 5248 5249 uint64_t 5250 spdk_bs_get_page_size(struct spdk_blob_store *bs) 5251 { 5252 return SPDK_BS_PAGE_SIZE; 5253 } 5254 5255 uint64_t 5256 spdk_bs_get_io_unit_size(struct spdk_blob_store *bs) 5257 { 5258 return bs->io_unit_size; 5259 } 5260 5261 uint64_t 5262 spdk_bs_free_cluster_count(struct spdk_blob_store *bs) 5263 { 5264 return bs->num_free_clusters; 5265 } 5266 5267 uint64_t 5268 spdk_bs_total_data_cluster_count(struct spdk_blob_store *bs) 5269 { 5270 return bs->total_data_clusters; 5271 } 5272 5273 static int 5274 bs_register_md_thread(struct spdk_blob_store *bs) 5275 { 5276 bs->md_channel = spdk_get_io_channel(bs); 5277 if (!bs->md_channel) { 5278 SPDK_ERRLOG("Failed to get IO channel.\n"); 5279 return -1; 5280 } 5281 5282 return 0; 5283 } 5284 5285 static int 5286 bs_unregister_md_thread(struct spdk_blob_store *bs) 5287 { 5288 spdk_put_io_channel(bs->md_channel); 5289 5290 return 0; 5291 } 5292 5293 spdk_blob_id spdk_blob_get_id(struct spdk_blob *blob) 5294 { 5295 assert(blob != NULL); 5296 5297 return blob->id; 5298 } 5299 5300 uint64_t spdk_blob_get_num_pages(struct spdk_blob *blob) 5301 { 5302 assert(blob != NULL); 5303 5304 return bs_cluster_to_page(blob->bs, blob->active.num_clusters); 5305 } 5306 5307 uint64_t spdk_blob_get_num_io_units(struct spdk_blob *blob) 5308 { 5309 assert(blob != NULL); 5310 5311 return spdk_blob_get_num_pages(blob) * bs_io_unit_per_page(blob->bs); 5312 } 5313 5314 uint64_t spdk_blob_get_num_clusters(struct spdk_blob *blob) 5315 { 5316 assert(blob != NULL); 5317 5318 return blob->active.num_clusters; 5319 } 5320 5321 /* START spdk_bs_create_blob */ 5322 5323 static void 5324 bs_create_blob_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 5325 { 5326 struct spdk_blob *blob = cb_arg; 5327 uint32_t page_idx = bs_blobid_to_page(blob->id); 5328 5329 if (bserrno != 0) { 5330 spdk_bit_array_clear(blob->bs->used_blobids, page_idx); 5331 bs_release_md_page(blob->bs, page_idx); 5332 } 5333 5334 blob_free(blob); 5335 5336 bs_sequence_finish(seq, bserrno); 5337 } 5338 5339 static int 5340 blob_set_xattrs(struct spdk_blob *blob, const struct spdk_blob_xattr_opts *xattrs, 5341 bool internal) 5342 { 5343 uint64_t i; 5344 size_t value_len = 0; 5345 int rc; 5346 const void *value = NULL; 5347 if (xattrs->count > 0 && xattrs->get_value == NULL) { 5348 return -EINVAL; 5349 } 5350 for (i = 0; i < xattrs->count; i++) { 5351 xattrs->get_value(xattrs->ctx, xattrs->names[i], &value, &value_len); 5352 if (value == NULL || value_len == 0) { 5353 return -EINVAL; 5354 } 5355 rc = blob_set_xattr(blob, xattrs->names[i], value, value_len, internal); 5356 if (rc < 0) { 5357 return rc; 5358 } 5359 } 5360 return 0; 5361 } 5362 5363 static void 5364 blob_opts_copy(const struct spdk_blob_opts *src, struct spdk_blob_opts *dst) 5365 { 5366 #define FIELD_OK(field) \ 5367 offsetof(struct spdk_blob_opts, field) + sizeof(src->field) <= src->opts_size 5368 5369 #define SET_FIELD(field) \ 5370 if (FIELD_OK(field)) { \ 5371 dst->field = src->field; \ 5372 } \ 5373 5374 SET_FIELD(num_clusters); 5375 SET_FIELD(thin_provision); 5376 SET_FIELD(clear_method); 5377 5378 if (FIELD_OK(xattrs)) { 5379 memcpy(&dst->xattrs, &src->xattrs, sizeof(src->xattrs)); 5380 } 5381 5382 SET_FIELD(use_extent_table); 5383 5384 dst->opts_size = src->opts_size; 5385 5386 /* You should not remove this statement, but need to update the assert statement 5387 * if you add a new field, and also add a corresponding SET_FIELD statement */ 5388 SPDK_STATIC_ASSERT(sizeof(struct spdk_blob_opts) == 64, "Incorrect size"); 5389 5390 #undef FIELD_OK 5391 #undef SET_FIELD 5392 } 5393 5394 static void 5395 bs_create_blob(struct spdk_blob_store *bs, 5396 const struct spdk_blob_opts *opts, 5397 const struct spdk_blob_xattr_opts *internal_xattrs, 5398 spdk_blob_op_with_id_complete cb_fn, void *cb_arg) 5399 { 5400 struct spdk_blob *blob; 5401 uint32_t page_idx; 5402 struct spdk_bs_cpl cpl; 5403 struct spdk_blob_opts opts_local; 5404 struct spdk_blob_xattr_opts internal_xattrs_default; 5405 spdk_bs_sequence_t *seq; 5406 spdk_blob_id id; 5407 int rc; 5408 5409 assert(spdk_get_thread() == bs->md_thread); 5410 5411 page_idx = spdk_bit_array_find_first_clear(bs->used_md_pages, 0); 5412 if (page_idx == UINT32_MAX) { 5413 cb_fn(cb_arg, 0, -ENOMEM); 5414 return; 5415 } 5416 spdk_bit_array_set(bs->used_blobids, page_idx); 5417 bs_claim_md_page(bs, page_idx); 5418 5419 id = bs_page_to_blobid(page_idx); 5420 5421 SPDK_DEBUGLOG(blob, "Creating blob with id %" PRIu64 " at page %u\n", id, page_idx); 5422 5423 blob = blob_alloc(bs, id); 5424 if (!blob) { 5425 spdk_bit_array_clear(bs->used_blobids, page_idx); 5426 bs_release_md_page(bs, page_idx); 5427 cb_fn(cb_arg, 0, -ENOMEM); 5428 return; 5429 } 5430 5431 spdk_blob_opts_init(&opts_local, sizeof(opts_local)); 5432 if (opts) { 5433 blob_opts_copy(opts, &opts_local); 5434 } 5435 5436 blob->use_extent_table = opts_local.use_extent_table; 5437 if (blob->use_extent_table) { 5438 blob->invalid_flags |= SPDK_BLOB_EXTENT_TABLE; 5439 } 5440 5441 if (!internal_xattrs) { 5442 blob_xattrs_init(&internal_xattrs_default); 5443 internal_xattrs = &internal_xattrs_default; 5444 } 5445 5446 rc = blob_set_xattrs(blob, &opts_local.xattrs, false); 5447 if (rc < 0) { 5448 blob_free(blob); 5449 spdk_bit_array_clear(bs->used_blobids, page_idx); 5450 bs_release_md_page(bs, page_idx); 5451 cb_fn(cb_arg, 0, rc); 5452 return; 5453 } 5454 5455 rc = blob_set_xattrs(blob, internal_xattrs, true); 5456 if (rc < 0) { 5457 blob_free(blob); 5458 spdk_bit_array_clear(bs->used_blobids, page_idx); 5459 bs_release_md_page(bs, page_idx); 5460 cb_fn(cb_arg, 0, rc); 5461 return; 5462 } 5463 5464 if (opts_local.thin_provision) { 5465 blob_set_thin_provision(blob); 5466 } 5467 5468 blob_set_clear_method(blob, opts_local.clear_method); 5469 5470 rc = blob_resize(blob, opts_local.num_clusters); 5471 if (rc < 0) { 5472 blob_free(blob); 5473 spdk_bit_array_clear(bs->used_blobids, page_idx); 5474 bs_release_md_page(bs, page_idx); 5475 cb_fn(cb_arg, 0, rc); 5476 return; 5477 } 5478 cpl.type = SPDK_BS_CPL_TYPE_BLOBID; 5479 cpl.u.blobid.cb_fn = cb_fn; 5480 cpl.u.blobid.cb_arg = cb_arg; 5481 cpl.u.blobid.blobid = blob->id; 5482 5483 seq = bs_sequence_start(bs->md_channel, &cpl); 5484 if (!seq) { 5485 blob_free(blob); 5486 spdk_bit_array_clear(bs->used_blobids, page_idx); 5487 bs_release_md_page(bs, page_idx); 5488 cb_fn(cb_arg, 0, -ENOMEM); 5489 return; 5490 } 5491 5492 blob_persist(seq, blob, bs_create_blob_cpl, blob); 5493 } 5494 5495 void spdk_bs_create_blob(struct spdk_blob_store *bs, 5496 spdk_blob_op_with_id_complete cb_fn, void *cb_arg) 5497 { 5498 bs_create_blob(bs, NULL, NULL, cb_fn, cb_arg); 5499 } 5500 5501 void spdk_bs_create_blob_ext(struct spdk_blob_store *bs, const struct spdk_blob_opts *opts, 5502 spdk_blob_op_with_id_complete cb_fn, void *cb_arg) 5503 { 5504 bs_create_blob(bs, opts, NULL, cb_fn, cb_arg); 5505 } 5506 5507 /* END spdk_bs_create_blob */ 5508 5509 /* START blob_cleanup */ 5510 5511 struct spdk_clone_snapshot_ctx { 5512 struct spdk_bs_cpl cpl; 5513 int bserrno; 5514 bool frozen; 5515 5516 struct spdk_io_channel *channel; 5517 5518 /* Current cluster for inflate operation */ 5519 uint64_t cluster; 5520 5521 /* For inflation force allocation of all unallocated clusters and remove 5522 * thin-provisioning. Otherwise only decouple parent and keep clone thin. */ 5523 bool allocate_all; 5524 5525 struct { 5526 spdk_blob_id id; 5527 struct spdk_blob *blob; 5528 } original; 5529 struct { 5530 spdk_blob_id id; 5531 struct spdk_blob *blob; 5532 } new; 5533 5534 /* xattrs specified for snapshot/clones only. They have no impact on 5535 * the original blobs xattrs. */ 5536 const struct spdk_blob_xattr_opts *xattrs; 5537 }; 5538 5539 static void 5540 bs_clone_snapshot_cleanup_finish(void *cb_arg, int bserrno) 5541 { 5542 struct spdk_clone_snapshot_ctx *ctx = cb_arg; 5543 struct spdk_bs_cpl *cpl = &ctx->cpl; 5544 5545 if (bserrno != 0) { 5546 if (ctx->bserrno != 0) { 5547 SPDK_ERRLOG("Cleanup error %d\n", bserrno); 5548 } else { 5549 ctx->bserrno = bserrno; 5550 } 5551 } 5552 5553 switch (cpl->type) { 5554 case SPDK_BS_CPL_TYPE_BLOBID: 5555 cpl->u.blobid.cb_fn(cpl->u.blobid.cb_arg, cpl->u.blobid.blobid, ctx->bserrno); 5556 break; 5557 case SPDK_BS_CPL_TYPE_BLOB_BASIC: 5558 cpl->u.blob_basic.cb_fn(cpl->u.blob_basic.cb_arg, ctx->bserrno); 5559 break; 5560 default: 5561 SPDK_UNREACHABLE(); 5562 break; 5563 } 5564 5565 free(ctx); 5566 } 5567 5568 static void 5569 bs_snapshot_unfreeze_cpl(void *cb_arg, int bserrno) 5570 { 5571 struct spdk_clone_snapshot_ctx *ctx = (struct spdk_clone_snapshot_ctx *)cb_arg; 5572 struct spdk_blob *origblob = ctx->original.blob; 5573 5574 if (bserrno != 0) { 5575 if (ctx->bserrno != 0) { 5576 SPDK_ERRLOG("Unfreeze error %d\n", bserrno); 5577 } else { 5578 ctx->bserrno = bserrno; 5579 } 5580 } 5581 5582 ctx->original.id = origblob->id; 5583 origblob->locked_operation_in_progress = false; 5584 5585 spdk_blob_close(origblob, bs_clone_snapshot_cleanup_finish, ctx); 5586 } 5587 5588 static void 5589 bs_clone_snapshot_origblob_cleanup(void *cb_arg, int bserrno) 5590 { 5591 struct spdk_clone_snapshot_ctx *ctx = (struct spdk_clone_snapshot_ctx *)cb_arg; 5592 struct spdk_blob *origblob = ctx->original.blob; 5593 5594 if (bserrno != 0) { 5595 if (ctx->bserrno != 0) { 5596 SPDK_ERRLOG("Cleanup error %d\n", bserrno); 5597 } else { 5598 ctx->bserrno = bserrno; 5599 } 5600 } 5601 5602 if (ctx->frozen) { 5603 /* Unfreeze any outstanding I/O */ 5604 blob_unfreeze_io(origblob, bs_snapshot_unfreeze_cpl, ctx); 5605 } else { 5606 bs_snapshot_unfreeze_cpl(ctx, 0); 5607 } 5608 5609 } 5610 5611 static void 5612 bs_clone_snapshot_newblob_cleanup(struct spdk_clone_snapshot_ctx *ctx, int bserrno) 5613 { 5614 struct spdk_blob *newblob = ctx->new.blob; 5615 5616 if (bserrno != 0) { 5617 if (ctx->bserrno != 0) { 5618 SPDK_ERRLOG("Cleanup error %d\n", bserrno); 5619 } else { 5620 ctx->bserrno = bserrno; 5621 } 5622 } 5623 5624 ctx->new.id = newblob->id; 5625 spdk_blob_close(newblob, bs_clone_snapshot_origblob_cleanup, ctx); 5626 } 5627 5628 /* END blob_cleanup */ 5629 5630 /* START spdk_bs_create_snapshot */ 5631 5632 static void 5633 bs_snapshot_swap_cluster_maps(struct spdk_blob *blob1, struct spdk_blob *blob2) 5634 { 5635 uint64_t *cluster_temp; 5636 uint32_t *extent_page_temp; 5637 5638 cluster_temp = blob1->active.clusters; 5639 blob1->active.clusters = blob2->active.clusters; 5640 blob2->active.clusters = cluster_temp; 5641 5642 extent_page_temp = blob1->active.extent_pages; 5643 blob1->active.extent_pages = blob2->active.extent_pages; 5644 blob2->active.extent_pages = extent_page_temp; 5645 } 5646 5647 static void 5648 bs_snapshot_origblob_sync_cpl(void *cb_arg, int bserrno) 5649 { 5650 struct spdk_clone_snapshot_ctx *ctx = (struct spdk_clone_snapshot_ctx *)cb_arg; 5651 struct spdk_blob *origblob = ctx->original.blob; 5652 struct spdk_blob *newblob = ctx->new.blob; 5653 5654 if (bserrno != 0) { 5655 bs_snapshot_swap_cluster_maps(newblob, origblob); 5656 bs_clone_snapshot_origblob_cleanup(ctx, bserrno); 5657 return; 5658 } 5659 5660 /* Remove metadata descriptor SNAPSHOT_IN_PROGRESS */ 5661 bserrno = blob_remove_xattr(newblob, SNAPSHOT_IN_PROGRESS, true); 5662 if (bserrno != 0) { 5663 bs_clone_snapshot_origblob_cleanup(ctx, bserrno); 5664 return; 5665 } 5666 5667 bs_blob_list_add(ctx->original.blob); 5668 5669 spdk_blob_set_read_only(newblob); 5670 5671 /* sync snapshot metadata */ 5672 spdk_blob_sync_md(newblob, bs_clone_snapshot_origblob_cleanup, ctx); 5673 } 5674 5675 static void 5676 bs_snapshot_newblob_sync_cpl(void *cb_arg, int bserrno) 5677 { 5678 struct spdk_clone_snapshot_ctx *ctx = (struct spdk_clone_snapshot_ctx *)cb_arg; 5679 struct spdk_blob *origblob = ctx->original.blob; 5680 struct spdk_blob *newblob = ctx->new.blob; 5681 5682 if (bserrno != 0) { 5683 /* return cluster map back to original */ 5684 bs_snapshot_swap_cluster_maps(newblob, origblob); 5685 5686 /* Newblob md sync failed. Valid clusters are only present in origblob. 5687 * Since I/O is frozen on origblob, not changes to zeroed out cluster map should have occured. 5688 * Newblob needs to be reverted to thin_provisioned state at creation to properly close. */ 5689 blob_set_thin_provision(newblob); 5690 assert(spdk_mem_all_zero(newblob->active.clusters, 5691 newblob->active.num_clusters * sizeof(*newblob->active.clusters))); 5692 assert(spdk_mem_all_zero(newblob->active.extent_pages, 5693 newblob->active.num_extent_pages * sizeof(*newblob->active.extent_pages))); 5694 5695 bs_clone_snapshot_newblob_cleanup(ctx, bserrno); 5696 return; 5697 } 5698 5699 /* Set internal xattr for snapshot id */ 5700 bserrno = blob_set_xattr(origblob, BLOB_SNAPSHOT, &newblob->id, sizeof(spdk_blob_id), true); 5701 if (bserrno != 0) { 5702 /* return cluster map back to original */ 5703 bs_snapshot_swap_cluster_maps(newblob, origblob); 5704 blob_set_thin_provision(newblob); 5705 bs_clone_snapshot_newblob_cleanup(ctx, bserrno); 5706 return; 5707 } 5708 5709 /* Create new back_bs_dev for snapshot */ 5710 origblob->back_bs_dev = bs_create_blob_bs_dev(newblob); 5711 if (origblob->back_bs_dev == NULL) { 5712 /* return cluster map back to original */ 5713 bs_snapshot_swap_cluster_maps(newblob, origblob); 5714 blob_set_thin_provision(newblob); 5715 bs_clone_snapshot_newblob_cleanup(ctx, -EINVAL); 5716 return; 5717 } 5718 5719 bs_blob_list_remove(origblob); 5720 origblob->parent_id = newblob->id; 5721 /* set clone blob as thin provisioned */ 5722 blob_set_thin_provision(origblob); 5723 5724 bs_blob_list_add(newblob); 5725 5726 /* sync clone metadata */ 5727 spdk_blob_sync_md(origblob, bs_snapshot_origblob_sync_cpl, ctx); 5728 } 5729 5730 static void 5731 bs_snapshot_freeze_cpl(void *cb_arg, int rc) 5732 { 5733 struct spdk_clone_snapshot_ctx *ctx = (struct spdk_clone_snapshot_ctx *)cb_arg; 5734 struct spdk_blob *origblob = ctx->original.blob; 5735 struct spdk_blob *newblob = ctx->new.blob; 5736 int bserrno; 5737 5738 if (rc != 0) { 5739 bs_clone_snapshot_newblob_cleanup(ctx, rc); 5740 return; 5741 } 5742 5743 ctx->frozen = true; 5744 5745 /* set new back_bs_dev for snapshot */ 5746 newblob->back_bs_dev = origblob->back_bs_dev; 5747 /* Set invalid flags from origblob */ 5748 newblob->invalid_flags = origblob->invalid_flags; 5749 5750 /* inherit parent from original blob if set */ 5751 newblob->parent_id = origblob->parent_id; 5752 if (origblob->parent_id != SPDK_BLOBID_INVALID) { 5753 /* Set internal xattr for snapshot id */ 5754 bserrno = blob_set_xattr(newblob, BLOB_SNAPSHOT, 5755 &origblob->parent_id, sizeof(spdk_blob_id), true); 5756 if (bserrno != 0) { 5757 bs_clone_snapshot_newblob_cleanup(ctx, bserrno); 5758 return; 5759 } 5760 } 5761 5762 /* swap cluster maps */ 5763 bs_snapshot_swap_cluster_maps(newblob, origblob); 5764 5765 /* Set the clear method on the new blob to match the original. */ 5766 blob_set_clear_method(newblob, origblob->clear_method); 5767 5768 /* sync snapshot metadata */ 5769 spdk_blob_sync_md(newblob, bs_snapshot_newblob_sync_cpl, ctx); 5770 } 5771 5772 static void 5773 bs_snapshot_newblob_open_cpl(void *cb_arg, struct spdk_blob *_blob, int bserrno) 5774 { 5775 struct spdk_clone_snapshot_ctx *ctx = (struct spdk_clone_snapshot_ctx *)cb_arg; 5776 struct spdk_blob *origblob = ctx->original.blob; 5777 struct spdk_blob *newblob = _blob; 5778 5779 if (bserrno != 0) { 5780 bs_clone_snapshot_origblob_cleanup(ctx, bserrno); 5781 return; 5782 } 5783 5784 ctx->new.blob = newblob; 5785 assert(spdk_blob_is_thin_provisioned(newblob)); 5786 assert(spdk_mem_all_zero(newblob->active.clusters, 5787 newblob->active.num_clusters * sizeof(*newblob->active.clusters))); 5788 assert(spdk_mem_all_zero(newblob->active.extent_pages, 5789 newblob->active.num_extent_pages * sizeof(*newblob->active.extent_pages))); 5790 5791 blob_freeze_io(origblob, bs_snapshot_freeze_cpl, ctx); 5792 } 5793 5794 static void 5795 bs_snapshot_newblob_create_cpl(void *cb_arg, spdk_blob_id blobid, int bserrno) 5796 { 5797 struct spdk_clone_snapshot_ctx *ctx = (struct spdk_clone_snapshot_ctx *)cb_arg; 5798 struct spdk_blob *origblob = ctx->original.blob; 5799 5800 if (bserrno != 0) { 5801 bs_clone_snapshot_origblob_cleanup(ctx, bserrno); 5802 return; 5803 } 5804 5805 ctx->new.id = blobid; 5806 ctx->cpl.u.blobid.blobid = blobid; 5807 5808 spdk_bs_open_blob(origblob->bs, ctx->new.id, bs_snapshot_newblob_open_cpl, ctx); 5809 } 5810 5811 5812 static void 5813 bs_xattr_snapshot(void *arg, const char *name, 5814 const void **value, size_t *value_len) 5815 { 5816 assert(strncmp(name, SNAPSHOT_IN_PROGRESS, sizeof(SNAPSHOT_IN_PROGRESS)) == 0); 5817 5818 struct spdk_blob *blob = (struct spdk_blob *)arg; 5819 *value = &blob->id; 5820 *value_len = sizeof(blob->id); 5821 } 5822 5823 static void 5824 bs_snapshot_origblob_open_cpl(void *cb_arg, struct spdk_blob *_blob, int bserrno) 5825 { 5826 struct spdk_clone_snapshot_ctx *ctx = (struct spdk_clone_snapshot_ctx *)cb_arg; 5827 struct spdk_blob_opts opts; 5828 struct spdk_blob_xattr_opts internal_xattrs; 5829 char *xattrs_names[] = { SNAPSHOT_IN_PROGRESS }; 5830 5831 if (bserrno != 0) { 5832 bs_clone_snapshot_cleanup_finish(ctx, bserrno); 5833 return; 5834 } 5835 5836 ctx->original.blob = _blob; 5837 5838 if (_blob->data_ro || _blob->md_ro) { 5839 SPDK_DEBUGLOG(blob, "Cannot create snapshot from read only blob with id %" PRIu64 "\n", 5840 _blob->id); 5841 ctx->bserrno = -EINVAL; 5842 spdk_blob_close(_blob, bs_clone_snapshot_cleanup_finish, ctx); 5843 return; 5844 } 5845 5846 if (_blob->locked_operation_in_progress) { 5847 SPDK_DEBUGLOG(blob, "Cannot create snapshot - another operation in progress\n"); 5848 ctx->bserrno = -EBUSY; 5849 spdk_blob_close(_blob, bs_clone_snapshot_cleanup_finish, ctx); 5850 return; 5851 } 5852 5853 _blob->locked_operation_in_progress = true; 5854 5855 spdk_blob_opts_init(&opts, sizeof(opts)); 5856 blob_xattrs_init(&internal_xattrs); 5857 5858 /* Change the size of new blob to the same as in original blob, 5859 * but do not allocate clusters */ 5860 opts.thin_provision = true; 5861 opts.num_clusters = spdk_blob_get_num_clusters(_blob); 5862 opts.use_extent_table = _blob->use_extent_table; 5863 5864 /* If there are any xattrs specified for snapshot, set them now */ 5865 if (ctx->xattrs) { 5866 memcpy(&opts.xattrs, ctx->xattrs, sizeof(*ctx->xattrs)); 5867 } 5868 /* Set internal xattr SNAPSHOT_IN_PROGRESS */ 5869 internal_xattrs.count = 1; 5870 internal_xattrs.ctx = _blob; 5871 internal_xattrs.names = xattrs_names; 5872 internal_xattrs.get_value = bs_xattr_snapshot; 5873 5874 bs_create_blob(_blob->bs, &opts, &internal_xattrs, 5875 bs_snapshot_newblob_create_cpl, ctx); 5876 } 5877 5878 void spdk_bs_create_snapshot(struct spdk_blob_store *bs, spdk_blob_id blobid, 5879 const struct spdk_blob_xattr_opts *snapshot_xattrs, 5880 spdk_blob_op_with_id_complete cb_fn, void *cb_arg) 5881 { 5882 struct spdk_clone_snapshot_ctx *ctx = calloc(1, sizeof(*ctx)); 5883 5884 if (!ctx) { 5885 cb_fn(cb_arg, SPDK_BLOBID_INVALID, -ENOMEM); 5886 return; 5887 } 5888 ctx->cpl.type = SPDK_BS_CPL_TYPE_BLOBID; 5889 ctx->cpl.u.blobid.cb_fn = cb_fn; 5890 ctx->cpl.u.blobid.cb_arg = cb_arg; 5891 ctx->cpl.u.blobid.blobid = SPDK_BLOBID_INVALID; 5892 ctx->bserrno = 0; 5893 ctx->frozen = false; 5894 ctx->original.id = blobid; 5895 ctx->xattrs = snapshot_xattrs; 5896 5897 spdk_bs_open_blob(bs, ctx->original.id, bs_snapshot_origblob_open_cpl, ctx); 5898 } 5899 /* END spdk_bs_create_snapshot */ 5900 5901 /* START spdk_bs_create_clone */ 5902 5903 static void 5904 bs_xattr_clone(void *arg, const char *name, 5905 const void **value, size_t *value_len) 5906 { 5907 assert(strncmp(name, BLOB_SNAPSHOT, sizeof(BLOB_SNAPSHOT)) == 0); 5908 5909 struct spdk_blob *blob = (struct spdk_blob *)arg; 5910 *value = &blob->id; 5911 *value_len = sizeof(blob->id); 5912 } 5913 5914 static void 5915 bs_clone_newblob_open_cpl(void *cb_arg, struct spdk_blob *_blob, int bserrno) 5916 { 5917 struct spdk_clone_snapshot_ctx *ctx = (struct spdk_clone_snapshot_ctx *)cb_arg; 5918 struct spdk_blob *clone = _blob; 5919 5920 ctx->new.blob = clone; 5921 bs_blob_list_add(clone); 5922 5923 spdk_blob_close(clone, bs_clone_snapshot_origblob_cleanup, ctx); 5924 } 5925 5926 static void 5927 bs_clone_newblob_create_cpl(void *cb_arg, spdk_blob_id blobid, int bserrno) 5928 { 5929 struct spdk_clone_snapshot_ctx *ctx = (struct spdk_clone_snapshot_ctx *)cb_arg; 5930 5931 ctx->cpl.u.blobid.blobid = blobid; 5932 spdk_bs_open_blob(ctx->original.blob->bs, blobid, bs_clone_newblob_open_cpl, ctx); 5933 } 5934 5935 static void 5936 bs_clone_origblob_open_cpl(void *cb_arg, struct spdk_blob *_blob, int bserrno) 5937 { 5938 struct spdk_clone_snapshot_ctx *ctx = (struct spdk_clone_snapshot_ctx *)cb_arg; 5939 struct spdk_blob_opts opts; 5940 struct spdk_blob_xattr_opts internal_xattrs; 5941 char *xattr_names[] = { BLOB_SNAPSHOT }; 5942 5943 if (bserrno != 0) { 5944 bs_clone_snapshot_cleanup_finish(ctx, bserrno); 5945 return; 5946 } 5947 5948 ctx->original.blob = _blob; 5949 5950 if (!_blob->data_ro || !_blob->md_ro) { 5951 SPDK_DEBUGLOG(blob, "Clone not from read-only blob\n"); 5952 ctx->bserrno = -EINVAL; 5953 spdk_blob_close(_blob, bs_clone_snapshot_cleanup_finish, ctx); 5954 return; 5955 } 5956 5957 if (_blob->locked_operation_in_progress) { 5958 SPDK_DEBUGLOG(blob, "Cannot create clone - another operation in progress\n"); 5959 ctx->bserrno = -EBUSY; 5960 spdk_blob_close(_blob, bs_clone_snapshot_cleanup_finish, ctx); 5961 return; 5962 } 5963 5964 _blob->locked_operation_in_progress = true; 5965 5966 spdk_blob_opts_init(&opts, sizeof(opts)); 5967 blob_xattrs_init(&internal_xattrs); 5968 5969 opts.thin_provision = true; 5970 opts.num_clusters = spdk_blob_get_num_clusters(_blob); 5971 opts.use_extent_table = _blob->use_extent_table; 5972 if (ctx->xattrs) { 5973 memcpy(&opts.xattrs, ctx->xattrs, sizeof(*ctx->xattrs)); 5974 } 5975 5976 /* Set internal xattr BLOB_SNAPSHOT */ 5977 internal_xattrs.count = 1; 5978 internal_xattrs.ctx = _blob; 5979 internal_xattrs.names = xattr_names; 5980 internal_xattrs.get_value = bs_xattr_clone; 5981 5982 bs_create_blob(_blob->bs, &opts, &internal_xattrs, 5983 bs_clone_newblob_create_cpl, ctx); 5984 } 5985 5986 void spdk_bs_create_clone(struct spdk_blob_store *bs, spdk_blob_id blobid, 5987 const struct spdk_blob_xattr_opts *clone_xattrs, 5988 spdk_blob_op_with_id_complete cb_fn, void *cb_arg) 5989 { 5990 struct spdk_clone_snapshot_ctx *ctx = calloc(1, sizeof(*ctx)); 5991 5992 if (!ctx) { 5993 cb_fn(cb_arg, SPDK_BLOBID_INVALID, -ENOMEM); 5994 return; 5995 } 5996 5997 ctx->cpl.type = SPDK_BS_CPL_TYPE_BLOBID; 5998 ctx->cpl.u.blobid.cb_fn = cb_fn; 5999 ctx->cpl.u.blobid.cb_arg = cb_arg; 6000 ctx->cpl.u.blobid.blobid = SPDK_BLOBID_INVALID; 6001 ctx->bserrno = 0; 6002 ctx->xattrs = clone_xattrs; 6003 ctx->original.id = blobid; 6004 6005 spdk_bs_open_blob(bs, ctx->original.id, bs_clone_origblob_open_cpl, ctx); 6006 } 6007 6008 /* END spdk_bs_create_clone */ 6009 6010 /* START spdk_bs_inflate_blob */ 6011 6012 static void 6013 bs_inflate_blob_set_parent_cpl(void *cb_arg, struct spdk_blob *_parent, int bserrno) 6014 { 6015 struct spdk_clone_snapshot_ctx *ctx = (struct spdk_clone_snapshot_ctx *)cb_arg; 6016 struct spdk_blob *_blob = ctx->original.blob; 6017 6018 if (bserrno != 0) { 6019 bs_clone_snapshot_origblob_cleanup(ctx, bserrno); 6020 return; 6021 } 6022 6023 assert(_parent != NULL); 6024 6025 bs_blob_list_remove(_blob); 6026 _blob->parent_id = _parent->id; 6027 blob_set_xattr(_blob, BLOB_SNAPSHOT, &_blob->parent_id, 6028 sizeof(spdk_blob_id), true); 6029 6030 _blob->back_bs_dev->destroy(_blob->back_bs_dev); 6031 _blob->back_bs_dev = bs_create_blob_bs_dev(_parent); 6032 bs_blob_list_add(_blob); 6033 6034 spdk_blob_sync_md(_blob, bs_clone_snapshot_origblob_cleanup, ctx); 6035 } 6036 6037 static void 6038 bs_inflate_blob_done(struct spdk_clone_snapshot_ctx *ctx) 6039 { 6040 struct spdk_blob *_blob = ctx->original.blob; 6041 struct spdk_blob *_parent; 6042 6043 if (ctx->allocate_all) { 6044 /* remove thin provisioning */ 6045 bs_blob_list_remove(_blob); 6046 blob_remove_xattr(_blob, BLOB_SNAPSHOT, true); 6047 _blob->invalid_flags = _blob->invalid_flags & ~SPDK_BLOB_THIN_PROV; 6048 _blob->back_bs_dev->destroy(_blob->back_bs_dev); 6049 _blob->back_bs_dev = NULL; 6050 _blob->parent_id = SPDK_BLOBID_INVALID; 6051 } else { 6052 _parent = ((struct spdk_blob_bs_dev *)(_blob->back_bs_dev))->blob; 6053 if (_parent->parent_id != SPDK_BLOBID_INVALID) { 6054 /* We must change the parent of the inflated blob */ 6055 spdk_bs_open_blob(_blob->bs, _parent->parent_id, 6056 bs_inflate_blob_set_parent_cpl, ctx); 6057 return; 6058 } 6059 6060 bs_blob_list_remove(_blob); 6061 blob_remove_xattr(_blob, BLOB_SNAPSHOT, true); 6062 _blob->parent_id = SPDK_BLOBID_INVALID; 6063 _blob->back_bs_dev->destroy(_blob->back_bs_dev); 6064 _blob->back_bs_dev = bs_create_zeroes_dev(); 6065 } 6066 6067 _blob->state = SPDK_BLOB_STATE_DIRTY; 6068 spdk_blob_sync_md(_blob, bs_clone_snapshot_origblob_cleanup, ctx); 6069 } 6070 6071 /* Check if cluster needs allocation */ 6072 static inline bool 6073 bs_cluster_needs_allocation(struct spdk_blob *blob, uint64_t cluster, bool allocate_all) 6074 { 6075 struct spdk_blob_bs_dev *b; 6076 6077 assert(blob != NULL); 6078 6079 if (blob->active.clusters[cluster] != 0) { 6080 /* Cluster is already allocated */ 6081 return false; 6082 } 6083 6084 if (blob->parent_id == SPDK_BLOBID_INVALID) { 6085 /* Blob have no parent blob */ 6086 return allocate_all; 6087 } 6088 6089 b = (struct spdk_blob_bs_dev *)blob->back_bs_dev; 6090 return (allocate_all || b->blob->active.clusters[cluster] != 0); 6091 } 6092 6093 static void 6094 bs_inflate_blob_touch_next(void *cb_arg, int bserrno) 6095 { 6096 struct spdk_clone_snapshot_ctx *ctx = (struct spdk_clone_snapshot_ctx *)cb_arg; 6097 struct spdk_blob *_blob = ctx->original.blob; 6098 uint64_t offset; 6099 6100 if (bserrno != 0) { 6101 bs_clone_snapshot_origblob_cleanup(ctx, bserrno); 6102 return; 6103 } 6104 6105 for (; ctx->cluster < _blob->active.num_clusters; ctx->cluster++) { 6106 if (bs_cluster_needs_allocation(_blob, ctx->cluster, ctx->allocate_all)) { 6107 break; 6108 } 6109 } 6110 6111 if (ctx->cluster < _blob->active.num_clusters) { 6112 offset = bs_cluster_to_lba(_blob->bs, ctx->cluster); 6113 6114 /* We may safely increment a cluster before write */ 6115 ctx->cluster++; 6116 6117 /* Use zero length write to touch a cluster */ 6118 spdk_blob_io_write(_blob, ctx->channel, NULL, offset, 0, 6119 bs_inflate_blob_touch_next, ctx); 6120 } else { 6121 bs_inflate_blob_done(ctx); 6122 } 6123 } 6124 6125 static void 6126 bs_inflate_blob_open_cpl(void *cb_arg, struct spdk_blob *_blob, int bserrno) 6127 { 6128 struct spdk_clone_snapshot_ctx *ctx = (struct spdk_clone_snapshot_ctx *)cb_arg; 6129 uint64_t clusters_needed; 6130 uint64_t i; 6131 6132 if (bserrno != 0) { 6133 bs_clone_snapshot_cleanup_finish(ctx, bserrno); 6134 return; 6135 } 6136 6137 ctx->original.blob = _blob; 6138 6139 if (_blob->locked_operation_in_progress) { 6140 SPDK_DEBUGLOG(blob, "Cannot inflate blob - another operation in progress\n"); 6141 ctx->bserrno = -EBUSY; 6142 spdk_blob_close(_blob, bs_clone_snapshot_cleanup_finish, ctx); 6143 return; 6144 } 6145 6146 _blob->locked_operation_in_progress = true; 6147 6148 if (!ctx->allocate_all && _blob->parent_id == SPDK_BLOBID_INVALID) { 6149 /* This blob have no parent, so we cannot decouple it. */ 6150 SPDK_ERRLOG("Cannot decouple parent of blob with no parent.\n"); 6151 bs_clone_snapshot_origblob_cleanup(ctx, -EINVAL); 6152 return; 6153 } 6154 6155 if (spdk_blob_is_thin_provisioned(_blob) == false) { 6156 /* This is not thin provisioned blob. No need to inflate. */ 6157 bs_clone_snapshot_origblob_cleanup(ctx, 0); 6158 return; 6159 } 6160 6161 /* Do two passes - one to verify that we can obtain enough clusters 6162 * and another to actually claim them. 6163 */ 6164 clusters_needed = 0; 6165 for (i = 0; i < _blob->active.num_clusters; i++) { 6166 if (bs_cluster_needs_allocation(_blob, i, ctx->allocate_all)) { 6167 clusters_needed++; 6168 } 6169 } 6170 6171 if (clusters_needed > _blob->bs->num_free_clusters) { 6172 /* Not enough free clusters. Cannot satisfy the request. */ 6173 bs_clone_snapshot_origblob_cleanup(ctx, -ENOSPC); 6174 return; 6175 } 6176 6177 ctx->cluster = 0; 6178 bs_inflate_blob_touch_next(ctx, 0); 6179 } 6180 6181 static void 6182 bs_inflate_blob(struct spdk_blob_store *bs, struct spdk_io_channel *channel, 6183 spdk_blob_id blobid, bool allocate_all, spdk_blob_op_complete cb_fn, void *cb_arg) 6184 { 6185 struct spdk_clone_snapshot_ctx *ctx = calloc(1, sizeof(*ctx)); 6186 6187 if (!ctx) { 6188 cb_fn(cb_arg, -ENOMEM); 6189 return; 6190 } 6191 ctx->cpl.type = SPDK_BS_CPL_TYPE_BLOB_BASIC; 6192 ctx->cpl.u.bs_basic.cb_fn = cb_fn; 6193 ctx->cpl.u.bs_basic.cb_arg = cb_arg; 6194 ctx->bserrno = 0; 6195 ctx->original.id = blobid; 6196 ctx->channel = channel; 6197 ctx->allocate_all = allocate_all; 6198 6199 spdk_bs_open_blob(bs, ctx->original.id, bs_inflate_blob_open_cpl, ctx); 6200 } 6201 6202 void 6203 spdk_bs_inflate_blob(struct spdk_blob_store *bs, struct spdk_io_channel *channel, 6204 spdk_blob_id blobid, spdk_blob_op_complete cb_fn, void *cb_arg) 6205 { 6206 bs_inflate_blob(bs, channel, blobid, true, cb_fn, cb_arg); 6207 } 6208 6209 void 6210 spdk_bs_blob_decouple_parent(struct spdk_blob_store *bs, struct spdk_io_channel *channel, 6211 spdk_blob_id blobid, spdk_blob_op_complete cb_fn, void *cb_arg) 6212 { 6213 bs_inflate_blob(bs, channel, blobid, false, cb_fn, cb_arg); 6214 } 6215 /* END spdk_bs_inflate_blob */ 6216 6217 /* START spdk_blob_resize */ 6218 struct spdk_bs_resize_ctx { 6219 spdk_blob_op_complete cb_fn; 6220 void *cb_arg; 6221 struct spdk_blob *blob; 6222 uint64_t sz; 6223 int rc; 6224 }; 6225 6226 static void 6227 bs_resize_unfreeze_cpl(void *cb_arg, int rc) 6228 { 6229 struct spdk_bs_resize_ctx *ctx = (struct spdk_bs_resize_ctx *)cb_arg; 6230 6231 if (rc != 0) { 6232 SPDK_ERRLOG("Unfreeze failed, rc=%d\n", rc); 6233 } 6234 6235 if (ctx->rc != 0) { 6236 SPDK_ERRLOG("Unfreeze failed, ctx->rc=%d\n", ctx->rc); 6237 rc = ctx->rc; 6238 } 6239 6240 ctx->blob->locked_operation_in_progress = false; 6241 6242 ctx->cb_fn(ctx->cb_arg, rc); 6243 free(ctx); 6244 } 6245 6246 static void 6247 bs_resize_freeze_cpl(void *cb_arg, int rc) 6248 { 6249 struct spdk_bs_resize_ctx *ctx = (struct spdk_bs_resize_ctx *)cb_arg; 6250 6251 if (rc != 0) { 6252 ctx->blob->locked_operation_in_progress = false; 6253 ctx->cb_fn(ctx->cb_arg, rc); 6254 free(ctx); 6255 return; 6256 } 6257 6258 ctx->rc = blob_resize(ctx->blob, ctx->sz); 6259 6260 blob_unfreeze_io(ctx->blob, bs_resize_unfreeze_cpl, ctx); 6261 } 6262 6263 void 6264 spdk_blob_resize(struct spdk_blob *blob, uint64_t sz, spdk_blob_op_complete cb_fn, void *cb_arg) 6265 { 6266 struct spdk_bs_resize_ctx *ctx; 6267 6268 blob_verify_md_op(blob); 6269 6270 SPDK_DEBUGLOG(blob, "Resizing blob %" PRIu64 " to %" PRIu64 " clusters\n", blob->id, sz); 6271 6272 if (blob->md_ro) { 6273 cb_fn(cb_arg, -EPERM); 6274 return; 6275 } 6276 6277 if (sz == blob->active.num_clusters) { 6278 cb_fn(cb_arg, 0); 6279 return; 6280 } 6281 6282 if (blob->locked_operation_in_progress) { 6283 cb_fn(cb_arg, -EBUSY); 6284 return; 6285 } 6286 6287 ctx = calloc(1, sizeof(*ctx)); 6288 if (!ctx) { 6289 cb_fn(cb_arg, -ENOMEM); 6290 return; 6291 } 6292 6293 blob->locked_operation_in_progress = true; 6294 ctx->cb_fn = cb_fn; 6295 ctx->cb_arg = cb_arg; 6296 ctx->blob = blob; 6297 ctx->sz = sz; 6298 blob_freeze_io(blob, bs_resize_freeze_cpl, ctx); 6299 } 6300 6301 /* END spdk_blob_resize */ 6302 6303 6304 /* START spdk_bs_delete_blob */ 6305 6306 static void 6307 bs_delete_close_cpl(void *cb_arg, int bserrno) 6308 { 6309 spdk_bs_sequence_t *seq = cb_arg; 6310 6311 bs_sequence_finish(seq, bserrno); 6312 } 6313 6314 static void 6315 bs_delete_persist_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 6316 { 6317 struct spdk_blob *blob = cb_arg; 6318 6319 if (bserrno != 0) { 6320 /* 6321 * We already removed this blob from the blobstore tailq, so 6322 * we need to free it here since this is the last reference 6323 * to it. 6324 */ 6325 blob_free(blob); 6326 bs_delete_close_cpl(seq, bserrno); 6327 return; 6328 } 6329 6330 /* 6331 * This will immediately decrement the ref_count and call 6332 * the completion routine since the metadata state is clean. 6333 * By calling spdk_blob_close, we reduce the number of call 6334 * points into code that touches the blob->open_ref count 6335 * and the blobstore's blob list. 6336 */ 6337 spdk_blob_close(blob, bs_delete_close_cpl, seq); 6338 } 6339 6340 struct delete_snapshot_ctx { 6341 struct spdk_blob_list *parent_snapshot_entry; 6342 struct spdk_blob *snapshot; 6343 bool snapshot_md_ro; 6344 struct spdk_blob *clone; 6345 bool clone_md_ro; 6346 spdk_blob_op_with_handle_complete cb_fn; 6347 void *cb_arg; 6348 int bserrno; 6349 }; 6350 6351 static void 6352 delete_blob_cleanup_finish(void *cb_arg, int bserrno) 6353 { 6354 struct delete_snapshot_ctx *ctx = cb_arg; 6355 6356 if (bserrno != 0) { 6357 SPDK_ERRLOG("Snapshot cleanup error %d\n", bserrno); 6358 } 6359 6360 assert(ctx != NULL); 6361 6362 if (bserrno != 0 && ctx->bserrno == 0) { 6363 ctx->bserrno = bserrno; 6364 } 6365 6366 ctx->cb_fn(ctx->cb_arg, ctx->snapshot, ctx->bserrno); 6367 free(ctx); 6368 } 6369 6370 static void 6371 delete_snapshot_cleanup_snapshot(void *cb_arg, int bserrno) 6372 { 6373 struct delete_snapshot_ctx *ctx = cb_arg; 6374 6375 if (bserrno != 0) { 6376 ctx->bserrno = bserrno; 6377 SPDK_ERRLOG("Clone cleanup error %d\n", bserrno); 6378 } 6379 6380 if (ctx->bserrno != 0) { 6381 assert(blob_lookup(ctx->snapshot->bs, ctx->snapshot->id) == NULL); 6382 TAILQ_INSERT_HEAD(&ctx->snapshot->bs->blobs, ctx->snapshot, link); 6383 spdk_bit_array_set(ctx->snapshot->bs->open_blobids, ctx->snapshot->id); 6384 } 6385 6386 ctx->snapshot->locked_operation_in_progress = false; 6387 ctx->snapshot->md_ro = ctx->snapshot_md_ro; 6388 6389 spdk_blob_close(ctx->snapshot, delete_blob_cleanup_finish, ctx); 6390 } 6391 6392 static void 6393 delete_snapshot_cleanup_clone(void *cb_arg, int bserrno) 6394 { 6395 struct delete_snapshot_ctx *ctx = cb_arg; 6396 6397 ctx->clone->locked_operation_in_progress = false; 6398 ctx->clone->md_ro = ctx->clone_md_ro; 6399 6400 spdk_blob_close(ctx->clone, delete_snapshot_cleanup_snapshot, ctx); 6401 } 6402 6403 static void 6404 delete_snapshot_unfreeze_cpl(void *cb_arg, int bserrno) 6405 { 6406 struct delete_snapshot_ctx *ctx = cb_arg; 6407 6408 if (bserrno) { 6409 ctx->bserrno = bserrno; 6410 delete_snapshot_cleanup_clone(ctx, 0); 6411 return; 6412 } 6413 6414 ctx->clone->locked_operation_in_progress = false; 6415 spdk_blob_close(ctx->clone, delete_blob_cleanup_finish, ctx); 6416 } 6417 6418 static void 6419 delete_snapshot_sync_snapshot_cpl(void *cb_arg, int bserrno) 6420 { 6421 struct delete_snapshot_ctx *ctx = cb_arg; 6422 struct spdk_blob_list *parent_snapshot_entry = NULL; 6423 struct spdk_blob_list *snapshot_entry = NULL; 6424 struct spdk_blob_list *clone_entry = NULL; 6425 struct spdk_blob_list *snapshot_clone_entry = NULL; 6426 6427 if (bserrno) { 6428 SPDK_ERRLOG("Failed to sync MD on blob\n"); 6429 ctx->bserrno = bserrno; 6430 delete_snapshot_cleanup_clone(ctx, 0); 6431 return; 6432 } 6433 6434 /* Get snapshot entry for the snapshot we want to remove */ 6435 snapshot_entry = bs_get_snapshot_entry(ctx->snapshot->bs, ctx->snapshot->id); 6436 6437 assert(snapshot_entry != NULL); 6438 6439 /* Remove clone entry in this snapshot (at this point there can be only one clone) */ 6440 clone_entry = TAILQ_FIRST(&snapshot_entry->clones); 6441 assert(clone_entry != NULL); 6442 TAILQ_REMOVE(&snapshot_entry->clones, clone_entry, link); 6443 snapshot_entry->clone_count--; 6444 assert(TAILQ_EMPTY(&snapshot_entry->clones)); 6445 6446 if (ctx->snapshot->parent_id != SPDK_BLOBID_INVALID) { 6447 /* This snapshot is at the same time a clone of another snapshot - we need to 6448 * update parent snapshot (remove current clone, add new one inherited from 6449 * the snapshot that is being removed) */ 6450 6451 /* Get snapshot entry for parent snapshot and clone entry within that snapshot for 6452 * snapshot that we are removing */ 6453 blob_get_snapshot_and_clone_entries(ctx->snapshot, &parent_snapshot_entry, 6454 &snapshot_clone_entry); 6455 6456 /* Switch clone entry in parent snapshot */ 6457 TAILQ_INSERT_TAIL(&parent_snapshot_entry->clones, clone_entry, link); 6458 TAILQ_REMOVE(&parent_snapshot_entry->clones, snapshot_clone_entry, link); 6459 free(snapshot_clone_entry); 6460 } else { 6461 /* No parent snapshot - just remove clone entry */ 6462 free(clone_entry); 6463 } 6464 6465 /* Restore md_ro flags */ 6466 ctx->clone->md_ro = ctx->clone_md_ro; 6467 ctx->snapshot->md_ro = ctx->snapshot_md_ro; 6468 6469 blob_unfreeze_io(ctx->clone, delete_snapshot_unfreeze_cpl, ctx); 6470 } 6471 6472 static void 6473 delete_snapshot_sync_clone_cpl(void *cb_arg, int bserrno) 6474 { 6475 struct delete_snapshot_ctx *ctx = cb_arg; 6476 uint64_t i; 6477 6478 ctx->snapshot->md_ro = false; 6479 6480 if (bserrno) { 6481 SPDK_ERRLOG("Failed to sync MD on clone\n"); 6482 ctx->bserrno = bserrno; 6483 6484 /* Restore snapshot to previous state */ 6485 bserrno = blob_remove_xattr(ctx->snapshot, SNAPSHOT_PENDING_REMOVAL, true); 6486 if (bserrno != 0) { 6487 delete_snapshot_cleanup_clone(ctx, bserrno); 6488 return; 6489 } 6490 6491 spdk_blob_sync_md(ctx->snapshot, delete_snapshot_cleanup_clone, ctx); 6492 return; 6493 } 6494 6495 /* Clear cluster map entries for snapshot */ 6496 for (i = 0; i < ctx->snapshot->active.num_clusters && i < ctx->clone->active.num_clusters; i++) { 6497 if (ctx->clone->active.clusters[i] == ctx->snapshot->active.clusters[i]) { 6498 ctx->snapshot->active.clusters[i] = 0; 6499 } 6500 } 6501 for (i = 0; i < ctx->snapshot->active.num_extent_pages && 6502 i < ctx->clone->active.num_extent_pages; i++) { 6503 if (ctx->clone->active.extent_pages[i] == ctx->snapshot->active.extent_pages[i]) { 6504 ctx->snapshot->active.extent_pages[i] = 0; 6505 } 6506 } 6507 6508 blob_set_thin_provision(ctx->snapshot); 6509 ctx->snapshot->state = SPDK_BLOB_STATE_DIRTY; 6510 6511 if (ctx->parent_snapshot_entry != NULL) { 6512 ctx->snapshot->back_bs_dev = NULL; 6513 } 6514 6515 spdk_blob_sync_md(ctx->snapshot, delete_snapshot_sync_snapshot_cpl, ctx); 6516 } 6517 6518 static void 6519 delete_snapshot_sync_snapshot_xattr_cpl(void *cb_arg, int bserrno) 6520 { 6521 struct delete_snapshot_ctx *ctx = cb_arg; 6522 uint64_t i; 6523 6524 /* Temporarily override md_ro flag for clone for MD modification */ 6525 ctx->clone_md_ro = ctx->clone->md_ro; 6526 ctx->clone->md_ro = false; 6527 6528 if (bserrno) { 6529 SPDK_ERRLOG("Failed to sync MD with xattr on blob\n"); 6530 ctx->bserrno = bserrno; 6531 delete_snapshot_cleanup_clone(ctx, 0); 6532 return; 6533 } 6534 6535 /* Copy snapshot map to clone map (only unallocated clusters in clone) */ 6536 for (i = 0; i < ctx->snapshot->active.num_clusters && i < ctx->clone->active.num_clusters; i++) { 6537 if (ctx->clone->active.clusters[i] == 0) { 6538 ctx->clone->active.clusters[i] = ctx->snapshot->active.clusters[i]; 6539 } 6540 } 6541 for (i = 0; i < ctx->snapshot->active.num_extent_pages && 6542 i < ctx->clone->active.num_extent_pages; i++) { 6543 if (ctx->clone->active.extent_pages[i] == 0) { 6544 ctx->clone->active.extent_pages[i] = ctx->snapshot->active.extent_pages[i]; 6545 } 6546 } 6547 6548 /* Delete old backing bs_dev from clone (related to snapshot that will be removed) */ 6549 ctx->clone->back_bs_dev->destroy(ctx->clone->back_bs_dev); 6550 6551 /* Set/remove snapshot xattr and switch parent ID and backing bs_dev on clone... */ 6552 if (ctx->parent_snapshot_entry != NULL) { 6553 /* ...to parent snapshot */ 6554 ctx->clone->parent_id = ctx->parent_snapshot_entry->id; 6555 ctx->clone->back_bs_dev = ctx->snapshot->back_bs_dev; 6556 blob_set_xattr(ctx->clone, BLOB_SNAPSHOT, &ctx->parent_snapshot_entry->id, 6557 sizeof(spdk_blob_id), 6558 true); 6559 } else { 6560 /* ...to blobid invalid and zeroes dev */ 6561 ctx->clone->parent_id = SPDK_BLOBID_INVALID; 6562 ctx->clone->back_bs_dev = bs_create_zeroes_dev(); 6563 blob_remove_xattr(ctx->clone, BLOB_SNAPSHOT, true); 6564 } 6565 6566 spdk_blob_sync_md(ctx->clone, delete_snapshot_sync_clone_cpl, ctx); 6567 } 6568 6569 static void 6570 delete_snapshot_freeze_io_cb(void *cb_arg, int bserrno) 6571 { 6572 struct delete_snapshot_ctx *ctx = cb_arg; 6573 6574 if (bserrno) { 6575 SPDK_ERRLOG("Failed to freeze I/O on clone\n"); 6576 ctx->bserrno = bserrno; 6577 delete_snapshot_cleanup_clone(ctx, 0); 6578 return; 6579 } 6580 6581 /* Temporarily override md_ro flag for snapshot for MD modification */ 6582 ctx->snapshot_md_ro = ctx->snapshot->md_ro; 6583 ctx->snapshot->md_ro = false; 6584 6585 /* Mark blob as pending for removal for power failure safety, use clone id for recovery */ 6586 ctx->bserrno = blob_set_xattr(ctx->snapshot, SNAPSHOT_PENDING_REMOVAL, &ctx->clone->id, 6587 sizeof(spdk_blob_id), true); 6588 if (ctx->bserrno != 0) { 6589 delete_snapshot_cleanup_clone(ctx, 0); 6590 return; 6591 } 6592 6593 spdk_blob_sync_md(ctx->snapshot, delete_snapshot_sync_snapshot_xattr_cpl, ctx); 6594 } 6595 6596 static void 6597 delete_snapshot_open_clone_cb(void *cb_arg, struct spdk_blob *clone, int bserrno) 6598 { 6599 struct delete_snapshot_ctx *ctx = cb_arg; 6600 6601 if (bserrno) { 6602 SPDK_ERRLOG("Failed to open clone\n"); 6603 ctx->bserrno = bserrno; 6604 delete_snapshot_cleanup_snapshot(ctx, 0); 6605 return; 6606 } 6607 6608 ctx->clone = clone; 6609 6610 if (clone->locked_operation_in_progress) { 6611 SPDK_DEBUGLOG(blob, "Cannot remove blob - another operation in progress on its clone\n"); 6612 ctx->bserrno = -EBUSY; 6613 spdk_blob_close(ctx->clone, delete_snapshot_cleanup_snapshot, ctx); 6614 return; 6615 } 6616 6617 clone->locked_operation_in_progress = true; 6618 6619 blob_freeze_io(clone, delete_snapshot_freeze_io_cb, ctx); 6620 } 6621 6622 static void 6623 update_clone_on_snapshot_deletion(struct spdk_blob *snapshot, struct delete_snapshot_ctx *ctx) 6624 { 6625 struct spdk_blob_list *snapshot_entry = NULL; 6626 struct spdk_blob_list *clone_entry = NULL; 6627 struct spdk_blob_list *snapshot_clone_entry = NULL; 6628 6629 /* Get snapshot entry for the snapshot we want to remove */ 6630 snapshot_entry = bs_get_snapshot_entry(snapshot->bs, snapshot->id); 6631 6632 assert(snapshot_entry != NULL); 6633 6634 /* Get clone of the snapshot (at this point there can be only one clone) */ 6635 clone_entry = TAILQ_FIRST(&snapshot_entry->clones); 6636 assert(snapshot_entry->clone_count == 1); 6637 assert(clone_entry != NULL); 6638 6639 /* Get snapshot entry for parent snapshot and clone entry within that snapshot for 6640 * snapshot that we are removing */ 6641 blob_get_snapshot_and_clone_entries(snapshot, &ctx->parent_snapshot_entry, 6642 &snapshot_clone_entry); 6643 6644 spdk_bs_open_blob(snapshot->bs, clone_entry->id, delete_snapshot_open_clone_cb, ctx); 6645 } 6646 6647 static void 6648 bs_delete_blob_finish(void *cb_arg, struct spdk_blob *blob, int bserrno) 6649 { 6650 spdk_bs_sequence_t *seq = cb_arg; 6651 struct spdk_blob_list *snapshot_entry = NULL; 6652 uint32_t page_num; 6653 6654 if (bserrno) { 6655 SPDK_ERRLOG("Failed to remove blob\n"); 6656 bs_sequence_finish(seq, bserrno); 6657 return; 6658 } 6659 6660 /* Remove snapshot from the list */ 6661 snapshot_entry = bs_get_snapshot_entry(blob->bs, blob->id); 6662 if (snapshot_entry != NULL) { 6663 TAILQ_REMOVE(&blob->bs->snapshots, snapshot_entry, link); 6664 free(snapshot_entry); 6665 } 6666 6667 page_num = bs_blobid_to_page(blob->id); 6668 spdk_bit_array_clear(blob->bs->used_blobids, page_num); 6669 blob->state = SPDK_BLOB_STATE_DIRTY; 6670 blob->active.num_pages = 0; 6671 blob_resize(blob, 0); 6672 6673 blob_persist(seq, blob, bs_delete_persist_cpl, blob); 6674 } 6675 6676 static int 6677 bs_is_blob_deletable(struct spdk_blob *blob, bool *update_clone) 6678 { 6679 struct spdk_blob_list *snapshot_entry = NULL; 6680 struct spdk_blob_list *clone_entry = NULL; 6681 struct spdk_blob *clone = NULL; 6682 bool has_one_clone = false; 6683 6684 /* Check if this is a snapshot with clones */ 6685 snapshot_entry = bs_get_snapshot_entry(blob->bs, blob->id); 6686 if (snapshot_entry != NULL) { 6687 if (snapshot_entry->clone_count > 1) { 6688 SPDK_ERRLOG("Cannot remove snapshot with more than one clone\n"); 6689 return -EBUSY; 6690 } else if (snapshot_entry->clone_count == 1) { 6691 has_one_clone = true; 6692 } 6693 } 6694 6695 /* Check if someone has this blob open (besides this delete context): 6696 * - open_ref = 1 - only this context opened blob, so it is ok to remove it 6697 * - open_ref <= 2 && has_one_clone = true - clone is holding snapshot 6698 * and that is ok, because we will update it accordingly */ 6699 if (blob->open_ref <= 2 && has_one_clone) { 6700 clone_entry = TAILQ_FIRST(&snapshot_entry->clones); 6701 assert(clone_entry != NULL); 6702 clone = blob_lookup(blob->bs, clone_entry->id); 6703 6704 if (blob->open_ref == 2 && clone == NULL) { 6705 /* Clone is closed and someone else opened this blob */ 6706 SPDK_ERRLOG("Cannot remove snapshot because it is open\n"); 6707 return -EBUSY; 6708 } 6709 6710 *update_clone = true; 6711 return 0; 6712 } 6713 6714 if (blob->open_ref > 1) { 6715 SPDK_ERRLOG("Cannot remove snapshot because it is open\n"); 6716 return -EBUSY; 6717 } 6718 6719 assert(has_one_clone == false); 6720 *update_clone = false; 6721 return 0; 6722 } 6723 6724 static void 6725 bs_delete_enomem_close_cpl(void *cb_arg, int bserrno) 6726 { 6727 spdk_bs_sequence_t *seq = cb_arg; 6728 6729 bs_sequence_finish(seq, -ENOMEM); 6730 } 6731 6732 static void 6733 bs_delete_open_cpl(void *cb_arg, struct spdk_blob *blob, int bserrno) 6734 { 6735 spdk_bs_sequence_t *seq = cb_arg; 6736 struct delete_snapshot_ctx *ctx; 6737 bool update_clone = false; 6738 6739 if (bserrno != 0) { 6740 bs_sequence_finish(seq, bserrno); 6741 return; 6742 } 6743 6744 blob_verify_md_op(blob); 6745 6746 ctx = calloc(1, sizeof(*ctx)); 6747 if (ctx == NULL) { 6748 spdk_blob_close(blob, bs_delete_enomem_close_cpl, seq); 6749 return; 6750 } 6751 6752 ctx->snapshot = blob; 6753 ctx->cb_fn = bs_delete_blob_finish; 6754 ctx->cb_arg = seq; 6755 6756 /* Check if blob can be removed and if it is a snapshot with clone on top of it */ 6757 ctx->bserrno = bs_is_blob_deletable(blob, &update_clone); 6758 if (ctx->bserrno) { 6759 spdk_blob_close(blob, delete_blob_cleanup_finish, ctx); 6760 return; 6761 } 6762 6763 if (blob->locked_operation_in_progress) { 6764 SPDK_DEBUGLOG(blob, "Cannot remove blob - another operation in progress\n"); 6765 ctx->bserrno = -EBUSY; 6766 spdk_blob_close(blob, delete_blob_cleanup_finish, ctx); 6767 return; 6768 } 6769 6770 blob->locked_operation_in_progress = true; 6771 6772 /* 6773 * Remove the blob from the blob_store list now, to ensure it does not 6774 * get returned after this point by blob_lookup(). 6775 */ 6776 spdk_bit_array_clear(blob->bs->open_blobids, blob->id); 6777 TAILQ_REMOVE(&blob->bs->blobs, blob, link); 6778 6779 if (update_clone) { 6780 /* This blob is a snapshot with active clone - update clone first */ 6781 update_clone_on_snapshot_deletion(blob, ctx); 6782 } else { 6783 /* This blob does not have any clones - just remove it */ 6784 bs_blob_list_remove(blob); 6785 bs_delete_blob_finish(seq, blob, 0); 6786 free(ctx); 6787 } 6788 } 6789 6790 void 6791 spdk_bs_delete_blob(struct spdk_blob_store *bs, spdk_blob_id blobid, 6792 spdk_blob_op_complete cb_fn, void *cb_arg) 6793 { 6794 struct spdk_bs_cpl cpl; 6795 spdk_bs_sequence_t *seq; 6796 6797 SPDK_DEBUGLOG(blob, "Deleting blob %" PRIu64 "\n", blobid); 6798 6799 assert(spdk_get_thread() == bs->md_thread); 6800 6801 cpl.type = SPDK_BS_CPL_TYPE_BLOB_BASIC; 6802 cpl.u.blob_basic.cb_fn = cb_fn; 6803 cpl.u.blob_basic.cb_arg = cb_arg; 6804 6805 seq = bs_sequence_start(bs->md_channel, &cpl); 6806 if (!seq) { 6807 cb_fn(cb_arg, -ENOMEM); 6808 return; 6809 } 6810 6811 spdk_bs_open_blob(bs, blobid, bs_delete_open_cpl, seq); 6812 } 6813 6814 /* END spdk_bs_delete_blob */ 6815 6816 /* START spdk_bs_open_blob */ 6817 6818 static void 6819 bs_open_blob_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 6820 { 6821 struct spdk_blob *blob = cb_arg; 6822 struct spdk_blob *existing; 6823 6824 if (bserrno != 0) { 6825 blob_free(blob); 6826 seq->cpl.u.blob_handle.blob = NULL; 6827 bs_sequence_finish(seq, bserrno); 6828 return; 6829 } 6830 6831 existing = blob_lookup(blob->bs, blob->id); 6832 if (existing) { 6833 blob_free(blob); 6834 existing->open_ref++; 6835 seq->cpl.u.blob_handle.blob = existing; 6836 bs_sequence_finish(seq, 0); 6837 return; 6838 } 6839 6840 blob->open_ref++; 6841 6842 spdk_bit_array_set(blob->bs->open_blobids, blob->id); 6843 TAILQ_INSERT_HEAD(&blob->bs->blobs, blob, link); 6844 6845 bs_sequence_finish(seq, bserrno); 6846 } 6847 6848 static inline void 6849 blob_open_opts_copy(const struct spdk_blob_open_opts *src, struct spdk_blob_open_opts *dst) 6850 { 6851 #define FIELD_OK(field) \ 6852 offsetof(struct spdk_blob_opts, field) + sizeof(src->field) <= src->opts_size 6853 6854 #define SET_FIELD(field) \ 6855 if (FIELD_OK(field)) { \ 6856 dst->field = src->field; \ 6857 } \ 6858 6859 SET_FIELD(clear_method); 6860 6861 dst->opts_size = src->opts_size; 6862 6863 /* You should not remove this statement, but need to update the assert statement 6864 * if you add a new field, and also add a corresponding SET_FIELD statement */ 6865 SPDK_STATIC_ASSERT(sizeof(struct spdk_blob_open_opts) == 16, "Incorrect size"); 6866 6867 #undef FIELD_OK 6868 #undef SET_FIELD 6869 } 6870 6871 static void 6872 bs_open_blob(struct spdk_blob_store *bs, 6873 spdk_blob_id blobid, 6874 struct spdk_blob_open_opts *opts, 6875 spdk_blob_op_with_handle_complete cb_fn, 6876 void *cb_arg) 6877 { 6878 struct spdk_blob *blob; 6879 struct spdk_bs_cpl cpl; 6880 struct spdk_blob_open_opts opts_local; 6881 spdk_bs_sequence_t *seq; 6882 uint32_t page_num; 6883 6884 SPDK_DEBUGLOG(blob, "Opening blob %" PRIu64 "\n", blobid); 6885 assert(spdk_get_thread() == bs->md_thread); 6886 6887 page_num = bs_blobid_to_page(blobid); 6888 if (spdk_bit_array_get(bs->used_blobids, page_num) == false) { 6889 /* Invalid blobid */ 6890 cb_fn(cb_arg, NULL, -ENOENT); 6891 return; 6892 } 6893 6894 blob = blob_lookup(bs, blobid); 6895 if (blob) { 6896 blob->open_ref++; 6897 cb_fn(cb_arg, blob, 0); 6898 return; 6899 } 6900 6901 blob = blob_alloc(bs, blobid); 6902 if (!blob) { 6903 cb_fn(cb_arg, NULL, -ENOMEM); 6904 return; 6905 } 6906 6907 spdk_blob_open_opts_init(&opts_local, sizeof(opts_local)); 6908 if (opts) { 6909 blob_open_opts_copy(opts, &opts_local); 6910 } 6911 6912 blob->clear_method = opts_local.clear_method; 6913 6914 cpl.type = SPDK_BS_CPL_TYPE_BLOB_HANDLE; 6915 cpl.u.blob_handle.cb_fn = cb_fn; 6916 cpl.u.blob_handle.cb_arg = cb_arg; 6917 cpl.u.blob_handle.blob = blob; 6918 6919 seq = bs_sequence_start(bs->md_channel, &cpl); 6920 if (!seq) { 6921 blob_free(blob); 6922 cb_fn(cb_arg, NULL, -ENOMEM); 6923 return; 6924 } 6925 6926 blob_load(seq, blob, bs_open_blob_cpl, blob); 6927 } 6928 6929 void spdk_bs_open_blob(struct spdk_blob_store *bs, spdk_blob_id blobid, 6930 spdk_blob_op_with_handle_complete cb_fn, void *cb_arg) 6931 { 6932 bs_open_blob(bs, blobid, NULL, cb_fn, cb_arg); 6933 } 6934 6935 void spdk_bs_open_blob_ext(struct spdk_blob_store *bs, spdk_blob_id blobid, 6936 struct spdk_blob_open_opts *opts, spdk_blob_op_with_handle_complete cb_fn, void *cb_arg) 6937 { 6938 bs_open_blob(bs, blobid, opts, cb_fn, cb_arg); 6939 } 6940 6941 /* END spdk_bs_open_blob */ 6942 6943 /* START spdk_blob_set_read_only */ 6944 int spdk_blob_set_read_only(struct spdk_blob *blob) 6945 { 6946 blob_verify_md_op(blob); 6947 6948 blob->data_ro_flags |= SPDK_BLOB_READ_ONLY; 6949 6950 blob->state = SPDK_BLOB_STATE_DIRTY; 6951 return 0; 6952 } 6953 /* END spdk_blob_set_read_only */ 6954 6955 /* START spdk_blob_sync_md */ 6956 6957 static void 6958 blob_sync_md_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 6959 { 6960 struct spdk_blob *blob = cb_arg; 6961 6962 if (bserrno == 0 && (blob->data_ro_flags & SPDK_BLOB_READ_ONLY)) { 6963 blob->data_ro = true; 6964 blob->md_ro = true; 6965 } 6966 6967 bs_sequence_finish(seq, bserrno); 6968 } 6969 6970 static void 6971 blob_sync_md(struct spdk_blob *blob, spdk_blob_op_complete cb_fn, void *cb_arg) 6972 { 6973 struct spdk_bs_cpl cpl; 6974 spdk_bs_sequence_t *seq; 6975 6976 cpl.type = SPDK_BS_CPL_TYPE_BLOB_BASIC; 6977 cpl.u.blob_basic.cb_fn = cb_fn; 6978 cpl.u.blob_basic.cb_arg = cb_arg; 6979 6980 seq = bs_sequence_start(blob->bs->md_channel, &cpl); 6981 if (!seq) { 6982 cb_fn(cb_arg, -ENOMEM); 6983 return; 6984 } 6985 6986 blob_persist(seq, blob, blob_sync_md_cpl, blob); 6987 } 6988 6989 void 6990 spdk_blob_sync_md(struct spdk_blob *blob, spdk_blob_op_complete cb_fn, void *cb_arg) 6991 { 6992 blob_verify_md_op(blob); 6993 6994 SPDK_DEBUGLOG(blob, "Syncing blob %" PRIu64 "\n", blob->id); 6995 6996 if (blob->md_ro) { 6997 assert(blob->state == SPDK_BLOB_STATE_CLEAN); 6998 cb_fn(cb_arg, 0); 6999 return; 7000 } 7001 7002 blob_sync_md(blob, cb_fn, cb_arg); 7003 } 7004 7005 /* END spdk_blob_sync_md */ 7006 7007 struct spdk_blob_insert_cluster_ctx { 7008 struct spdk_thread *thread; 7009 struct spdk_blob *blob; 7010 uint32_t cluster_num; /* cluster index in blob */ 7011 uint32_t cluster; /* cluster on disk */ 7012 uint32_t extent_page; /* extent page on disk */ 7013 int rc; 7014 spdk_blob_op_complete cb_fn; 7015 void *cb_arg; 7016 }; 7017 7018 static void 7019 blob_insert_cluster_msg_cpl(void *arg) 7020 { 7021 struct spdk_blob_insert_cluster_ctx *ctx = arg; 7022 7023 ctx->cb_fn(ctx->cb_arg, ctx->rc); 7024 free(ctx); 7025 } 7026 7027 static void 7028 blob_insert_cluster_msg_cb(void *arg, int bserrno) 7029 { 7030 struct spdk_blob_insert_cluster_ctx *ctx = arg; 7031 7032 ctx->rc = bserrno; 7033 spdk_thread_send_msg(ctx->thread, blob_insert_cluster_msg_cpl, ctx); 7034 } 7035 7036 static void 7037 blob_persist_extent_page_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 7038 { 7039 struct spdk_blob_md_page *page = cb_arg; 7040 7041 bs_sequence_finish(seq, bserrno); 7042 spdk_free(page); 7043 } 7044 7045 static void 7046 blob_insert_extent(struct spdk_blob *blob, uint32_t extent, uint64_t cluster_num, 7047 spdk_blob_op_complete cb_fn, void *cb_arg) 7048 { 7049 spdk_bs_sequence_t *seq; 7050 struct spdk_bs_cpl cpl; 7051 struct spdk_blob_md_page *page = NULL; 7052 uint32_t page_count = 0; 7053 int rc; 7054 7055 cpl.type = SPDK_BS_CPL_TYPE_BLOB_BASIC; 7056 cpl.u.blob_basic.cb_fn = cb_fn; 7057 cpl.u.blob_basic.cb_arg = cb_arg; 7058 7059 seq = bs_sequence_start(blob->bs->md_channel, &cpl); 7060 if (!seq) { 7061 cb_fn(cb_arg, -ENOMEM); 7062 return; 7063 } 7064 rc = blob_serialize_add_page(blob, &page, &page_count, &page); 7065 if (rc < 0) { 7066 bs_sequence_finish(seq, rc); 7067 return; 7068 } 7069 7070 blob_serialize_extent_page(blob, cluster_num, page); 7071 7072 page->crc = blob_md_page_calc_crc(page); 7073 7074 assert(spdk_bit_array_get(blob->bs->used_md_pages, extent) == true); 7075 7076 bs_sequence_write_dev(seq, page, bs_md_page_to_lba(blob->bs, extent), 7077 bs_byte_to_lba(blob->bs, SPDK_BS_PAGE_SIZE), 7078 blob_persist_extent_page_cpl, page); 7079 } 7080 7081 static void 7082 blob_insert_cluster_msg(void *arg) 7083 { 7084 struct spdk_blob_insert_cluster_ctx *ctx = arg; 7085 uint32_t *extent_page; 7086 7087 ctx->rc = blob_insert_cluster(ctx->blob, ctx->cluster_num, ctx->cluster); 7088 if (ctx->rc != 0) { 7089 spdk_thread_send_msg(ctx->thread, blob_insert_cluster_msg_cpl, ctx); 7090 return; 7091 } 7092 7093 if (ctx->blob->use_extent_table == false) { 7094 /* Extent table is not used, proceed with sync of md that will only use extents_rle. */ 7095 ctx->blob->state = SPDK_BLOB_STATE_DIRTY; 7096 blob_sync_md(ctx->blob, blob_insert_cluster_msg_cb, ctx); 7097 return; 7098 } 7099 7100 extent_page = bs_cluster_to_extent_page(ctx->blob, ctx->cluster_num); 7101 if (*extent_page == 0) { 7102 /* Extent page requires allocation. 7103 * It was already claimed in the used_md_pages map and placed in ctx. 7104 * Blob persist will take care of writing out new extent page on disk. */ 7105 assert(ctx->extent_page != 0); 7106 assert(spdk_bit_array_get(ctx->blob->bs->used_md_pages, ctx->extent_page) == true); 7107 *extent_page = ctx->extent_page; 7108 ctx->blob->state = SPDK_BLOB_STATE_DIRTY; 7109 blob_sync_md(ctx->blob, blob_insert_cluster_msg_cb, ctx); 7110 } else { 7111 /* It is possible for original thread to allocate extent page for 7112 * different cluster in the same extent page. In such case proceed with 7113 * updating the existing extent page, but release the additional one. */ 7114 if (ctx->extent_page != 0) { 7115 assert(spdk_bit_array_get(ctx->blob->bs->used_md_pages, ctx->extent_page) == true); 7116 bs_release_md_page(ctx->blob->bs, ctx->extent_page); 7117 ctx->extent_page = 0; 7118 } 7119 /* Extent page already allocated. 7120 * Every cluster allocation, requires just an update of single extent page. */ 7121 blob_insert_extent(ctx->blob, *extent_page, ctx->cluster_num, 7122 blob_insert_cluster_msg_cb, ctx); 7123 } 7124 } 7125 7126 static void 7127 blob_insert_cluster_on_md_thread(struct spdk_blob *blob, uint32_t cluster_num, 7128 uint64_t cluster, uint32_t extent_page, spdk_blob_op_complete cb_fn, void *cb_arg) 7129 { 7130 struct spdk_blob_insert_cluster_ctx *ctx; 7131 7132 ctx = calloc(1, sizeof(*ctx)); 7133 if (ctx == NULL) { 7134 cb_fn(cb_arg, -ENOMEM); 7135 return; 7136 } 7137 7138 ctx->thread = spdk_get_thread(); 7139 ctx->blob = blob; 7140 ctx->cluster_num = cluster_num; 7141 ctx->cluster = cluster; 7142 ctx->extent_page = extent_page; 7143 ctx->cb_fn = cb_fn; 7144 ctx->cb_arg = cb_arg; 7145 7146 spdk_thread_send_msg(blob->bs->md_thread, blob_insert_cluster_msg, ctx); 7147 } 7148 7149 /* START spdk_blob_close */ 7150 7151 static void 7152 blob_close_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 7153 { 7154 struct spdk_blob *blob = cb_arg; 7155 7156 if (bserrno == 0) { 7157 blob->open_ref--; 7158 if (blob->open_ref == 0) { 7159 /* 7160 * Blobs with active.num_pages == 0 are deleted blobs. 7161 * these blobs are removed from the blob_store list 7162 * when the deletion process starts - so don't try to 7163 * remove them again. 7164 */ 7165 if (blob->active.num_pages > 0) { 7166 spdk_bit_array_clear(blob->bs->open_blobids, blob->id); 7167 TAILQ_REMOVE(&blob->bs->blobs, blob, link); 7168 } 7169 blob_free(blob); 7170 } 7171 } 7172 7173 bs_sequence_finish(seq, bserrno); 7174 } 7175 7176 void spdk_blob_close(struct spdk_blob *blob, spdk_blob_op_complete cb_fn, void *cb_arg) 7177 { 7178 struct spdk_bs_cpl cpl; 7179 spdk_bs_sequence_t *seq; 7180 7181 blob_verify_md_op(blob); 7182 7183 SPDK_DEBUGLOG(blob, "Closing blob %" PRIu64 "\n", blob->id); 7184 7185 if (blob->open_ref == 0) { 7186 cb_fn(cb_arg, -EBADF); 7187 return; 7188 } 7189 7190 cpl.type = SPDK_BS_CPL_TYPE_BLOB_BASIC; 7191 cpl.u.blob_basic.cb_fn = cb_fn; 7192 cpl.u.blob_basic.cb_arg = cb_arg; 7193 7194 seq = bs_sequence_start(blob->bs->md_channel, &cpl); 7195 if (!seq) { 7196 cb_fn(cb_arg, -ENOMEM); 7197 return; 7198 } 7199 7200 /* Sync metadata */ 7201 blob_persist(seq, blob, blob_close_cpl, blob); 7202 } 7203 7204 /* END spdk_blob_close */ 7205 7206 struct spdk_io_channel *spdk_bs_alloc_io_channel(struct spdk_blob_store *bs) 7207 { 7208 return spdk_get_io_channel(bs); 7209 } 7210 7211 void spdk_bs_free_io_channel(struct spdk_io_channel *channel) 7212 { 7213 spdk_put_io_channel(channel); 7214 } 7215 7216 void spdk_blob_io_unmap(struct spdk_blob *blob, struct spdk_io_channel *channel, 7217 uint64_t offset, uint64_t length, spdk_blob_op_complete cb_fn, void *cb_arg) 7218 { 7219 blob_request_submit_op(blob, channel, NULL, offset, length, cb_fn, cb_arg, 7220 SPDK_BLOB_UNMAP); 7221 } 7222 7223 void spdk_blob_io_write_zeroes(struct spdk_blob *blob, struct spdk_io_channel *channel, 7224 uint64_t offset, uint64_t length, spdk_blob_op_complete cb_fn, void *cb_arg) 7225 { 7226 blob_request_submit_op(blob, channel, NULL, offset, length, cb_fn, cb_arg, 7227 SPDK_BLOB_WRITE_ZEROES); 7228 } 7229 7230 void spdk_blob_io_write(struct spdk_blob *blob, struct spdk_io_channel *channel, 7231 void *payload, uint64_t offset, uint64_t length, 7232 spdk_blob_op_complete cb_fn, void *cb_arg) 7233 { 7234 blob_request_submit_op(blob, channel, payload, offset, length, cb_fn, cb_arg, 7235 SPDK_BLOB_WRITE); 7236 } 7237 7238 void spdk_blob_io_read(struct spdk_blob *blob, struct spdk_io_channel *channel, 7239 void *payload, uint64_t offset, uint64_t length, 7240 spdk_blob_op_complete cb_fn, void *cb_arg) 7241 { 7242 blob_request_submit_op(blob, channel, payload, offset, length, cb_fn, cb_arg, 7243 SPDK_BLOB_READ); 7244 } 7245 7246 void spdk_blob_io_writev(struct spdk_blob *blob, struct spdk_io_channel *channel, 7247 struct iovec *iov, int iovcnt, uint64_t offset, uint64_t length, 7248 spdk_blob_op_complete cb_fn, void *cb_arg) 7249 { 7250 blob_request_submit_rw_iov(blob, channel, iov, iovcnt, offset, length, cb_fn, cb_arg, false); 7251 } 7252 7253 void spdk_blob_io_readv(struct spdk_blob *blob, struct spdk_io_channel *channel, 7254 struct iovec *iov, int iovcnt, uint64_t offset, uint64_t length, 7255 spdk_blob_op_complete cb_fn, void *cb_arg) 7256 { 7257 blob_request_submit_rw_iov(blob, channel, iov, iovcnt, offset, length, cb_fn, cb_arg, true); 7258 } 7259 7260 struct spdk_bs_iter_ctx { 7261 int64_t page_num; 7262 struct spdk_blob_store *bs; 7263 7264 spdk_blob_op_with_handle_complete cb_fn; 7265 void *cb_arg; 7266 }; 7267 7268 static void 7269 bs_iter_cpl(void *cb_arg, struct spdk_blob *_blob, int bserrno) 7270 { 7271 struct spdk_bs_iter_ctx *ctx = cb_arg; 7272 struct spdk_blob_store *bs = ctx->bs; 7273 spdk_blob_id id; 7274 7275 if (bserrno == 0) { 7276 ctx->cb_fn(ctx->cb_arg, _blob, bserrno); 7277 free(ctx); 7278 return; 7279 } 7280 7281 ctx->page_num++; 7282 ctx->page_num = spdk_bit_array_find_first_set(bs->used_blobids, ctx->page_num); 7283 if (ctx->page_num >= spdk_bit_array_capacity(bs->used_blobids)) { 7284 ctx->cb_fn(ctx->cb_arg, NULL, -ENOENT); 7285 free(ctx); 7286 return; 7287 } 7288 7289 id = bs_page_to_blobid(ctx->page_num); 7290 7291 spdk_bs_open_blob(bs, id, bs_iter_cpl, ctx); 7292 } 7293 7294 void 7295 spdk_bs_iter_first(struct spdk_blob_store *bs, 7296 spdk_blob_op_with_handle_complete cb_fn, void *cb_arg) 7297 { 7298 struct spdk_bs_iter_ctx *ctx; 7299 7300 ctx = calloc(1, sizeof(*ctx)); 7301 if (!ctx) { 7302 cb_fn(cb_arg, NULL, -ENOMEM); 7303 return; 7304 } 7305 7306 ctx->page_num = -1; 7307 ctx->bs = bs; 7308 ctx->cb_fn = cb_fn; 7309 ctx->cb_arg = cb_arg; 7310 7311 bs_iter_cpl(ctx, NULL, -1); 7312 } 7313 7314 static void 7315 bs_iter_close_cpl(void *cb_arg, int bserrno) 7316 { 7317 struct spdk_bs_iter_ctx *ctx = cb_arg; 7318 7319 bs_iter_cpl(ctx, NULL, -1); 7320 } 7321 7322 void 7323 spdk_bs_iter_next(struct spdk_blob_store *bs, struct spdk_blob *blob, 7324 spdk_blob_op_with_handle_complete cb_fn, void *cb_arg) 7325 { 7326 struct spdk_bs_iter_ctx *ctx; 7327 7328 assert(blob != NULL); 7329 7330 ctx = calloc(1, sizeof(*ctx)); 7331 if (!ctx) { 7332 cb_fn(cb_arg, NULL, -ENOMEM); 7333 return; 7334 } 7335 7336 ctx->page_num = bs_blobid_to_page(blob->id); 7337 ctx->bs = bs; 7338 ctx->cb_fn = cb_fn; 7339 ctx->cb_arg = cb_arg; 7340 7341 /* Close the existing blob */ 7342 spdk_blob_close(blob, bs_iter_close_cpl, ctx); 7343 } 7344 7345 static int 7346 blob_set_xattr(struct spdk_blob *blob, const char *name, const void *value, 7347 uint16_t value_len, bool internal) 7348 { 7349 struct spdk_xattr_tailq *xattrs; 7350 struct spdk_xattr *xattr; 7351 size_t desc_size; 7352 void *tmp; 7353 7354 blob_verify_md_op(blob); 7355 7356 if (blob->md_ro) { 7357 return -EPERM; 7358 } 7359 7360 desc_size = sizeof(struct spdk_blob_md_descriptor_xattr) + strlen(name) + value_len; 7361 if (desc_size > SPDK_BS_MAX_DESC_SIZE) { 7362 SPDK_DEBUGLOG(blob, "Xattr '%s' of size %zu does not fix into single page %zu\n", name, 7363 desc_size, SPDK_BS_MAX_DESC_SIZE); 7364 return -ENOMEM; 7365 } 7366 7367 if (internal) { 7368 xattrs = &blob->xattrs_internal; 7369 blob->invalid_flags |= SPDK_BLOB_INTERNAL_XATTR; 7370 } else { 7371 xattrs = &blob->xattrs; 7372 } 7373 7374 TAILQ_FOREACH(xattr, xattrs, link) { 7375 if (!strcmp(name, xattr->name)) { 7376 tmp = malloc(value_len); 7377 if (!tmp) { 7378 return -ENOMEM; 7379 } 7380 7381 free(xattr->value); 7382 xattr->value_len = value_len; 7383 xattr->value = tmp; 7384 memcpy(xattr->value, value, value_len); 7385 7386 blob->state = SPDK_BLOB_STATE_DIRTY; 7387 7388 return 0; 7389 } 7390 } 7391 7392 xattr = calloc(1, sizeof(*xattr)); 7393 if (!xattr) { 7394 return -ENOMEM; 7395 } 7396 7397 xattr->name = strdup(name); 7398 if (!xattr->name) { 7399 free(xattr); 7400 return -ENOMEM; 7401 } 7402 7403 xattr->value_len = value_len; 7404 xattr->value = malloc(value_len); 7405 if (!xattr->value) { 7406 free(xattr->name); 7407 free(xattr); 7408 return -ENOMEM; 7409 } 7410 memcpy(xattr->value, value, value_len); 7411 TAILQ_INSERT_TAIL(xattrs, xattr, link); 7412 7413 blob->state = SPDK_BLOB_STATE_DIRTY; 7414 7415 return 0; 7416 } 7417 7418 int 7419 spdk_blob_set_xattr(struct spdk_blob *blob, const char *name, const void *value, 7420 uint16_t value_len) 7421 { 7422 return blob_set_xattr(blob, name, value, value_len, false); 7423 } 7424 7425 static int 7426 blob_remove_xattr(struct spdk_blob *blob, const char *name, bool internal) 7427 { 7428 struct spdk_xattr_tailq *xattrs; 7429 struct spdk_xattr *xattr; 7430 7431 blob_verify_md_op(blob); 7432 7433 if (blob->md_ro) { 7434 return -EPERM; 7435 } 7436 xattrs = internal ? &blob->xattrs_internal : &blob->xattrs; 7437 7438 TAILQ_FOREACH(xattr, xattrs, link) { 7439 if (!strcmp(name, xattr->name)) { 7440 TAILQ_REMOVE(xattrs, xattr, link); 7441 free(xattr->value); 7442 free(xattr->name); 7443 free(xattr); 7444 7445 if (internal && TAILQ_EMPTY(&blob->xattrs_internal)) { 7446 blob->invalid_flags &= ~SPDK_BLOB_INTERNAL_XATTR; 7447 } 7448 blob->state = SPDK_BLOB_STATE_DIRTY; 7449 7450 return 0; 7451 } 7452 } 7453 7454 return -ENOENT; 7455 } 7456 7457 int 7458 spdk_blob_remove_xattr(struct spdk_blob *blob, const char *name) 7459 { 7460 return blob_remove_xattr(blob, name, false); 7461 } 7462 7463 static int 7464 blob_get_xattr_value(struct spdk_blob *blob, const char *name, 7465 const void **value, size_t *value_len, bool internal) 7466 { 7467 struct spdk_xattr *xattr; 7468 struct spdk_xattr_tailq *xattrs; 7469 7470 xattrs = internal ? &blob->xattrs_internal : &blob->xattrs; 7471 7472 TAILQ_FOREACH(xattr, xattrs, link) { 7473 if (!strcmp(name, xattr->name)) { 7474 *value = xattr->value; 7475 *value_len = xattr->value_len; 7476 return 0; 7477 } 7478 } 7479 return -ENOENT; 7480 } 7481 7482 int 7483 spdk_blob_get_xattr_value(struct spdk_blob *blob, const char *name, 7484 const void **value, size_t *value_len) 7485 { 7486 blob_verify_md_op(blob); 7487 7488 return blob_get_xattr_value(blob, name, value, value_len, false); 7489 } 7490 7491 struct spdk_xattr_names { 7492 uint32_t count; 7493 const char *names[0]; 7494 }; 7495 7496 static int 7497 blob_get_xattr_names(struct spdk_xattr_tailq *xattrs, struct spdk_xattr_names **names) 7498 { 7499 struct spdk_xattr *xattr; 7500 int count = 0; 7501 7502 TAILQ_FOREACH(xattr, xattrs, link) { 7503 count++; 7504 } 7505 7506 *names = calloc(1, sizeof(struct spdk_xattr_names) + count * sizeof(char *)); 7507 if (*names == NULL) { 7508 return -ENOMEM; 7509 } 7510 7511 TAILQ_FOREACH(xattr, xattrs, link) { 7512 (*names)->names[(*names)->count++] = xattr->name; 7513 } 7514 7515 return 0; 7516 } 7517 7518 int 7519 spdk_blob_get_xattr_names(struct spdk_blob *blob, struct spdk_xattr_names **names) 7520 { 7521 blob_verify_md_op(blob); 7522 7523 return blob_get_xattr_names(&blob->xattrs, names); 7524 } 7525 7526 uint32_t 7527 spdk_xattr_names_get_count(struct spdk_xattr_names *names) 7528 { 7529 assert(names != NULL); 7530 7531 return names->count; 7532 } 7533 7534 const char * 7535 spdk_xattr_names_get_name(struct spdk_xattr_names *names, uint32_t index) 7536 { 7537 if (index >= names->count) { 7538 return NULL; 7539 } 7540 7541 return names->names[index]; 7542 } 7543 7544 void 7545 spdk_xattr_names_free(struct spdk_xattr_names *names) 7546 { 7547 free(names); 7548 } 7549 7550 struct spdk_bs_type 7551 spdk_bs_get_bstype(struct spdk_blob_store *bs) 7552 { 7553 return bs->bstype; 7554 } 7555 7556 void 7557 spdk_bs_set_bstype(struct spdk_blob_store *bs, struct spdk_bs_type bstype) 7558 { 7559 memcpy(&bs->bstype, &bstype, sizeof(bstype)); 7560 } 7561 7562 bool 7563 spdk_blob_is_read_only(struct spdk_blob *blob) 7564 { 7565 assert(blob != NULL); 7566 return (blob->data_ro || blob->md_ro); 7567 } 7568 7569 bool 7570 spdk_blob_is_snapshot(struct spdk_blob *blob) 7571 { 7572 struct spdk_blob_list *snapshot_entry; 7573 7574 assert(blob != NULL); 7575 7576 snapshot_entry = bs_get_snapshot_entry(blob->bs, blob->id); 7577 if (snapshot_entry == NULL) { 7578 return false; 7579 } 7580 7581 return true; 7582 } 7583 7584 bool 7585 spdk_blob_is_clone(struct spdk_blob *blob) 7586 { 7587 assert(blob != NULL); 7588 7589 if (blob->parent_id != SPDK_BLOBID_INVALID) { 7590 assert(spdk_blob_is_thin_provisioned(blob)); 7591 return true; 7592 } 7593 7594 return false; 7595 } 7596 7597 bool 7598 spdk_blob_is_thin_provisioned(struct spdk_blob *blob) 7599 { 7600 assert(blob != NULL); 7601 return !!(blob->invalid_flags & SPDK_BLOB_THIN_PROV); 7602 } 7603 7604 static void 7605 blob_update_clear_method(struct spdk_blob *blob) 7606 { 7607 enum blob_clear_method stored_cm; 7608 7609 assert(blob != NULL); 7610 7611 /* If BLOB_CLEAR_WITH_DEFAULT was passed in, use the setting stored 7612 * in metadata previously. If something other than the default was 7613 * specified, ignore stored value and used what was passed in. 7614 */ 7615 stored_cm = ((blob->md_ro_flags & SPDK_BLOB_CLEAR_METHOD) >> SPDK_BLOB_CLEAR_METHOD_SHIFT); 7616 7617 if (blob->clear_method == BLOB_CLEAR_WITH_DEFAULT) { 7618 blob->clear_method = stored_cm; 7619 } else if (blob->clear_method != stored_cm) { 7620 SPDK_WARNLOG("Using passed in clear method 0x%x instead of stored value of 0x%x\n", 7621 blob->clear_method, stored_cm); 7622 } 7623 } 7624 7625 spdk_blob_id 7626 spdk_blob_get_parent_snapshot(struct spdk_blob_store *bs, spdk_blob_id blob_id) 7627 { 7628 struct spdk_blob_list *snapshot_entry = NULL; 7629 struct spdk_blob_list *clone_entry = NULL; 7630 7631 TAILQ_FOREACH(snapshot_entry, &bs->snapshots, link) { 7632 TAILQ_FOREACH(clone_entry, &snapshot_entry->clones, link) { 7633 if (clone_entry->id == blob_id) { 7634 return snapshot_entry->id; 7635 } 7636 } 7637 } 7638 7639 return SPDK_BLOBID_INVALID; 7640 } 7641 7642 int 7643 spdk_blob_get_clones(struct spdk_blob_store *bs, spdk_blob_id blobid, spdk_blob_id *ids, 7644 size_t *count) 7645 { 7646 struct spdk_blob_list *snapshot_entry, *clone_entry; 7647 size_t n; 7648 7649 snapshot_entry = bs_get_snapshot_entry(bs, blobid); 7650 if (snapshot_entry == NULL) { 7651 *count = 0; 7652 return 0; 7653 } 7654 7655 if (ids == NULL || *count < snapshot_entry->clone_count) { 7656 *count = snapshot_entry->clone_count; 7657 return -ENOMEM; 7658 } 7659 *count = snapshot_entry->clone_count; 7660 7661 n = 0; 7662 TAILQ_FOREACH(clone_entry, &snapshot_entry->clones, link) { 7663 ids[n++] = clone_entry->id; 7664 } 7665 7666 return 0; 7667 } 7668 7669 SPDK_LOG_REGISTER_COMPONENT(blob) 7670