1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) Intel Corporation. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include "spdk/stdinc.h" 35 36 #include "spdk/blob.h" 37 #include "spdk/crc32.h" 38 #include "spdk/env.h" 39 #include "spdk/queue.h" 40 #include "spdk/thread.h" 41 #include "spdk/bit_array.h" 42 #include "spdk/bit_pool.h" 43 #include "spdk/likely.h" 44 #include "spdk/util.h" 45 #include "spdk/string.h" 46 47 #include "spdk_internal/assert.h" 48 #include "spdk/log.h" 49 50 #include "blobstore.h" 51 52 #define BLOB_CRC32C_INITIAL 0xffffffffUL 53 54 static int bs_register_md_thread(struct spdk_blob_store *bs); 55 static int bs_unregister_md_thread(struct spdk_blob_store *bs); 56 static void blob_close_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno); 57 static void blob_insert_cluster_on_md_thread(struct spdk_blob *blob, uint32_t cluster_num, 58 uint64_t cluster, uint32_t extent, spdk_blob_op_complete cb_fn, void *cb_arg); 59 60 static int blob_set_xattr(struct spdk_blob *blob, const char *name, const void *value, 61 uint16_t value_len, bool internal); 62 static int blob_get_xattr_value(struct spdk_blob *blob, const char *name, 63 const void **value, size_t *value_len, bool internal); 64 static int blob_remove_xattr(struct spdk_blob *blob, const char *name, bool internal); 65 66 static void blob_write_extent_page(struct spdk_blob *blob, uint32_t extent, uint64_t cluster_num, 67 spdk_blob_op_complete cb_fn, void *cb_arg); 68 69 static void 70 blob_verify_md_op(struct spdk_blob *blob) 71 { 72 assert(blob != NULL); 73 assert(spdk_get_thread() == blob->bs->md_thread); 74 assert(blob->state != SPDK_BLOB_STATE_LOADING); 75 } 76 77 static struct spdk_blob_list * 78 bs_get_snapshot_entry(struct spdk_blob_store *bs, spdk_blob_id blobid) 79 { 80 struct spdk_blob_list *snapshot_entry = NULL; 81 82 TAILQ_FOREACH(snapshot_entry, &bs->snapshots, link) { 83 if (snapshot_entry->id == blobid) { 84 break; 85 } 86 } 87 88 return snapshot_entry; 89 } 90 91 static void 92 bs_claim_md_page(struct spdk_blob_store *bs, uint32_t page) 93 { 94 assert(page < spdk_bit_array_capacity(bs->used_md_pages)); 95 assert(spdk_bit_array_get(bs->used_md_pages, page) == false); 96 97 spdk_bit_array_set(bs->used_md_pages, page); 98 } 99 100 static void 101 bs_release_md_page(struct spdk_blob_store *bs, uint32_t page) 102 { 103 assert(page < spdk_bit_array_capacity(bs->used_md_pages)); 104 assert(spdk_bit_array_get(bs->used_md_pages, page) == true); 105 106 spdk_bit_array_clear(bs->used_md_pages, page); 107 } 108 109 static uint32_t 110 bs_claim_cluster(struct spdk_blob_store *bs) 111 { 112 uint32_t cluster_num; 113 114 cluster_num = spdk_bit_pool_allocate_bit(bs->used_clusters); 115 if (cluster_num == UINT32_MAX) { 116 return UINT32_MAX; 117 } 118 119 SPDK_DEBUGLOG(blob, "Claiming cluster %u\n", cluster_num); 120 bs->num_free_clusters--; 121 122 return cluster_num; 123 } 124 125 static void 126 bs_release_cluster(struct spdk_blob_store *bs, uint32_t cluster_num) 127 { 128 assert(cluster_num < spdk_bit_pool_capacity(bs->used_clusters)); 129 assert(spdk_bit_pool_is_allocated(bs->used_clusters, cluster_num) == true); 130 assert(bs->num_free_clusters < bs->total_clusters); 131 132 SPDK_DEBUGLOG(blob, "Releasing cluster %u\n", cluster_num); 133 134 spdk_bit_pool_free_bit(bs->used_clusters, cluster_num); 135 bs->num_free_clusters++; 136 } 137 138 static int 139 blob_insert_cluster(struct spdk_blob *blob, uint32_t cluster_num, uint64_t cluster) 140 { 141 uint64_t *cluster_lba = &blob->active.clusters[cluster_num]; 142 143 blob_verify_md_op(blob); 144 145 if (*cluster_lba != 0) { 146 return -EEXIST; 147 } 148 149 *cluster_lba = bs_cluster_to_lba(blob->bs, cluster); 150 return 0; 151 } 152 153 static int 154 bs_allocate_cluster(struct spdk_blob *blob, uint32_t cluster_num, 155 uint64_t *cluster, uint32_t *lowest_free_md_page, bool update_map) 156 { 157 uint32_t *extent_page = 0; 158 159 *cluster = bs_claim_cluster(blob->bs); 160 if (*cluster == UINT32_MAX) { 161 /* No more free clusters. Cannot satisfy the request */ 162 return -ENOSPC; 163 } 164 165 if (blob->use_extent_table) { 166 extent_page = bs_cluster_to_extent_page(blob, cluster_num); 167 if (*extent_page == 0) { 168 /* Extent page shall never occupy md_page so start the search from 1 */ 169 if (*lowest_free_md_page == 0) { 170 *lowest_free_md_page = 1; 171 } 172 /* No extent_page is allocated for the cluster */ 173 *lowest_free_md_page = spdk_bit_array_find_first_clear(blob->bs->used_md_pages, 174 *lowest_free_md_page); 175 if (*lowest_free_md_page == UINT32_MAX) { 176 /* No more free md pages. Cannot satisfy the request */ 177 bs_release_cluster(blob->bs, *cluster); 178 return -ENOSPC; 179 } 180 bs_claim_md_page(blob->bs, *lowest_free_md_page); 181 } 182 } 183 184 SPDK_DEBUGLOG(blob, "Claiming cluster %" PRIu64 " for blob %" PRIu64 "\n", *cluster, blob->id); 185 186 if (update_map) { 187 blob_insert_cluster(blob, cluster_num, *cluster); 188 if (blob->use_extent_table && *extent_page == 0) { 189 *extent_page = *lowest_free_md_page; 190 } 191 } 192 193 return 0; 194 } 195 196 static void 197 blob_xattrs_init(struct spdk_blob_xattr_opts *xattrs) 198 { 199 xattrs->count = 0; 200 xattrs->names = NULL; 201 xattrs->ctx = NULL; 202 xattrs->get_value = NULL; 203 } 204 205 void 206 spdk_blob_opts_init(struct spdk_blob_opts *opts, size_t opts_size) 207 { 208 if (!opts) { 209 SPDK_ERRLOG("opts should not be NULL\n"); 210 return; 211 } 212 213 if (!opts_size) { 214 SPDK_ERRLOG("opts_size should not be zero value\n"); 215 return; 216 } 217 218 memset(opts, 0, opts_size); 219 opts->opts_size = opts_size; 220 221 #define FIELD_OK(field) \ 222 offsetof(struct spdk_blob_opts, field) + sizeof(opts->field) <= opts_size 223 224 #define SET_FIELD(field, value) \ 225 if (FIELD_OK(field)) { \ 226 opts->field = value; \ 227 } \ 228 229 SET_FIELD(num_clusters, 0); 230 SET_FIELD(thin_provision, false); 231 SET_FIELD(clear_method, BLOB_CLEAR_WITH_DEFAULT); 232 233 if (FIELD_OK(xattrs)) { 234 blob_xattrs_init(&opts->xattrs); 235 } 236 237 SET_FIELD(use_extent_table, true); 238 239 #undef FIELD_OK 240 #undef SET_FIELD 241 } 242 243 void 244 spdk_blob_open_opts_init(struct spdk_blob_open_opts *opts, size_t opts_size) 245 { 246 if (!opts) { 247 SPDK_ERRLOG("opts should not be NULL\n"); 248 return; 249 } 250 251 if (!opts_size) { 252 SPDK_ERRLOG("opts_size should not be zero value\n"); 253 return; 254 } 255 256 memset(opts, 0, opts_size); 257 opts->opts_size = opts_size; 258 259 #define FIELD_OK(field) \ 260 offsetof(struct spdk_blob_open_opts, field) + sizeof(opts->field) <= opts_size 261 262 #define SET_FIELD(field, value) \ 263 if (FIELD_OK(field)) { \ 264 opts->field = value; \ 265 } \ 266 267 SET_FIELD(clear_method, BLOB_CLEAR_WITH_DEFAULT); 268 269 #undef FIELD_OK 270 #undef SET_FILED 271 } 272 273 static struct spdk_blob * 274 blob_alloc(struct spdk_blob_store *bs, spdk_blob_id id) 275 { 276 struct spdk_blob *blob; 277 278 blob = calloc(1, sizeof(*blob)); 279 if (!blob) { 280 return NULL; 281 } 282 283 blob->id = id; 284 blob->bs = bs; 285 286 blob->parent_id = SPDK_BLOBID_INVALID; 287 288 blob->state = SPDK_BLOB_STATE_DIRTY; 289 blob->extent_rle_found = false; 290 blob->extent_table_found = false; 291 blob->active.num_pages = 1; 292 blob->active.pages = calloc(1, sizeof(*blob->active.pages)); 293 if (!blob->active.pages) { 294 free(blob); 295 return NULL; 296 } 297 298 blob->active.pages[0] = bs_blobid_to_page(id); 299 300 TAILQ_INIT(&blob->xattrs); 301 TAILQ_INIT(&blob->xattrs_internal); 302 TAILQ_INIT(&blob->pending_persists); 303 304 return blob; 305 } 306 307 static void 308 xattrs_free(struct spdk_xattr_tailq *xattrs) 309 { 310 struct spdk_xattr *xattr, *xattr_tmp; 311 312 TAILQ_FOREACH_SAFE(xattr, xattrs, link, xattr_tmp) { 313 TAILQ_REMOVE(xattrs, xattr, link); 314 free(xattr->name); 315 free(xattr->value); 316 free(xattr); 317 } 318 } 319 320 static void 321 blob_free(struct spdk_blob *blob) 322 { 323 assert(blob != NULL); 324 assert(TAILQ_EMPTY(&blob->pending_persists)); 325 326 free(blob->active.extent_pages); 327 free(blob->clean.extent_pages); 328 free(blob->active.clusters); 329 free(blob->clean.clusters); 330 free(blob->active.pages); 331 free(blob->clean.pages); 332 333 xattrs_free(&blob->xattrs); 334 xattrs_free(&blob->xattrs_internal); 335 336 if (blob->back_bs_dev) { 337 blob->back_bs_dev->destroy(blob->back_bs_dev); 338 } 339 340 free(blob); 341 } 342 343 struct freeze_io_ctx { 344 struct spdk_bs_cpl cpl; 345 struct spdk_blob *blob; 346 }; 347 348 static void 349 blob_io_sync(struct spdk_io_channel_iter *i) 350 { 351 spdk_for_each_channel_continue(i, 0); 352 } 353 354 static void 355 blob_execute_queued_io(struct spdk_io_channel_iter *i) 356 { 357 struct spdk_io_channel *_ch = spdk_io_channel_iter_get_channel(i); 358 struct spdk_bs_channel *ch = spdk_io_channel_get_ctx(_ch); 359 struct freeze_io_ctx *ctx = spdk_io_channel_iter_get_ctx(i); 360 struct spdk_bs_request_set *set; 361 struct spdk_bs_user_op_args *args; 362 spdk_bs_user_op_t *op, *tmp; 363 364 TAILQ_FOREACH_SAFE(op, &ch->queued_io, link, tmp) { 365 set = (struct spdk_bs_request_set *)op; 366 args = &set->u.user_op; 367 368 if (args->blob == ctx->blob) { 369 TAILQ_REMOVE(&ch->queued_io, op, link); 370 bs_user_op_execute(op); 371 } 372 } 373 374 spdk_for_each_channel_continue(i, 0); 375 } 376 377 static void 378 blob_io_cpl(struct spdk_io_channel_iter *i, int status) 379 { 380 struct freeze_io_ctx *ctx = spdk_io_channel_iter_get_ctx(i); 381 382 ctx->cpl.u.blob_basic.cb_fn(ctx->cpl.u.blob_basic.cb_arg, 0); 383 384 free(ctx); 385 } 386 387 static void 388 blob_freeze_io(struct spdk_blob *blob, spdk_blob_op_complete cb_fn, void *cb_arg) 389 { 390 struct freeze_io_ctx *ctx; 391 392 ctx = calloc(1, sizeof(*ctx)); 393 if (!ctx) { 394 cb_fn(cb_arg, -ENOMEM); 395 return; 396 } 397 398 ctx->cpl.type = SPDK_BS_CPL_TYPE_BS_BASIC; 399 ctx->cpl.u.blob_basic.cb_fn = cb_fn; 400 ctx->cpl.u.blob_basic.cb_arg = cb_arg; 401 ctx->blob = blob; 402 403 /* Freeze I/O on blob */ 404 blob->frozen_refcnt++; 405 406 if (blob->frozen_refcnt == 1) { 407 spdk_for_each_channel(blob->bs, blob_io_sync, ctx, blob_io_cpl); 408 } else { 409 cb_fn(cb_arg, 0); 410 free(ctx); 411 } 412 } 413 414 static void 415 blob_unfreeze_io(struct spdk_blob *blob, spdk_blob_op_complete cb_fn, void *cb_arg) 416 { 417 struct freeze_io_ctx *ctx; 418 419 ctx = calloc(1, sizeof(*ctx)); 420 if (!ctx) { 421 cb_fn(cb_arg, -ENOMEM); 422 return; 423 } 424 425 ctx->cpl.type = SPDK_BS_CPL_TYPE_BS_BASIC; 426 ctx->cpl.u.blob_basic.cb_fn = cb_fn; 427 ctx->cpl.u.blob_basic.cb_arg = cb_arg; 428 ctx->blob = blob; 429 430 assert(blob->frozen_refcnt > 0); 431 432 blob->frozen_refcnt--; 433 434 if (blob->frozen_refcnt == 0) { 435 spdk_for_each_channel(blob->bs, blob_execute_queued_io, ctx, blob_io_cpl); 436 } else { 437 cb_fn(cb_arg, 0); 438 free(ctx); 439 } 440 } 441 442 static int 443 blob_mark_clean(struct spdk_blob *blob) 444 { 445 uint32_t *extent_pages = NULL; 446 uint64_t *clusters = NULL; 447 uint32_t *pages = NULL; 448 449 assert(blob != NULL); 450 451 if (blob->active.num_extent_pages) { 452 assert(blob->active.extent_pages); 453 extent_pages = calloc(blob->active.num_extent_pages, sizeof(*blob->active.extent_pages)); 454 if (!extent_pages) { 455 return -ENOMEM; 456 } 457 memcpy(extent_pages, blob->active.extent_pages, 458 blob->active.num_extent_pages * sizeof(*extent_pages)); 459 } 460 461 if (blob->active.num_clusters) { 462 assert(blob->active.clusters); 463 clusters = calloc(blob->active.num_clusters, sizeof(*blob->active.clusters)); 464 if (!clusters) { 465 free(extent_pages); 466 return -ENOMEM; 467 } 468 memcpy(clusters, blob->active.clusters, blob->active.num_clusters * sizeof(*blob->active.clusters)); 469 } 470 471 if (blob->active.num_pages) { 472 assert(blob->active.pages); 473 pages = calloc(blob->active.num_pages, sizeof(*blob->active.pages)); 474 if (!pages) { 475 free(extent_pages); 476 free(clusters); 477 return -ENOMEM; 478 } 479 memcpy(pages, blob->active.pages, blob->active.num_pages * sizeof(*blob->active.pages)); 480 } 481 482 free(blob->clean.extent_pages); 483 free(blob->clean.clusters); 484 free(blob->clean.pages); 485 486 blob->clean.num_extent_pages = blob->active.num_extent_pages; 487 blob->clean.extent_pages = blob->active.extent_pages; 488 blob->clean.num_clusters = blob->active.num_clusters; 489 blob->clean.clusters = blob->active.clusters; 490 blob->clean.num_pages = blob->active.num_pages; 491 blob->clean.pages = blob->active.pages; 492 493 blob->active.extent_pages = extent_pages; 494 blob->active.clusters = clusters; 495 blob->active.pages = pages; 496 497 /* If the metadata was dirtied again while the metadata was being written to disk, 498 * we do not want to revert the DIRTY state back to CLEAN here. 499 */ 500 if (blob->state == SPDK_BLOB_STATE_LOADING) { 501 blob->state = SPDK_BLOB_STATE_CLEAN; 502 } 503 504 return 0; 505 } 506 507 static int 508 blob_deserialize_xattr(struct spdk_blob *blob, 509 struct spdk_blob_md_descriptor_xattr *desc_xattr, bool internal) 510 { 511 struct spdk_xattr *xattr; 512 513 if (desc_xattr->length != sizeof(desc_xattr->name_length) + 514 sizeof(desc_xattr->value_length) + 515 desc_xattr->name_length + desc_xattr->value_length) { 516 return -EINVAL; 517 } 518 519 xattr = calloc(1, sizeof(*xattr)); 520 if (xattr == NULL) { 521 return -ENOMEM; 522 } 523 524 xattr->name = malloc(desc_xattr->name_length + 1); 525 if (xattr->name == NULL) { 526 free(xattr); 527 return -ENOMEM; 528 } 529 memcpy(xattr->name, desc_xattr->name, desc_xattr->name_length); 530 xattr->name[desc_xattr->name_length] = '\0'; 531 532 xattr->value = malloc(desc_xattr->value_length); 533 if (xattr->value == NULL) { 534 free(xattr->name); 535 free(xattr); 536 return -ENOMEM; 537 } 538 xattr->value_len = desc_xattr->value_length; 539 memcpy(xattr->value, 540 (void *)((uintptr_t)desc_xattr->name + desc_xattr->name_length), 541 desc_xattr->value_length); 542 543 TAILQ_INSERT_TAIL(internal ? &blob->xattrs_internal : &blob->xattrs, xattr, link); 544 545 return 0; 546 } 547 548 549 static int 550 blob_parse_page(const struct spdk_blob_md_page *page, struct spdk_blob *blob) 551 { 552 struct spdk_blob_md_descriptor *desc; 553 size_t cur_desc = 0; 554 void *tmp; 555 556 desc = (struct spdk_blob_md_descriptor *)page->descriptors; 557 while (cur_desc < sizeof(page->descriptors)) { 558 if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_PADDING) { 559 if (desc->length == 0) { 560 /* If padding and length are 0, this terminates the page */ 561 break; 562 } 563 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_FLAGS) { 564 struct spdk_blob_md_descriptor_flags *desc_flags; 565 566 desc_flags = (struct spdk_blob_md_descriptor_flags *)desc; 567 568 if (desc_flags->length != sizeof(*desc_flags) - sizeof(*desc)) { 569 return -EINVAL; 570 } 571 572 if ((desc_flags->invalid_flags | SPDK_BLOB_INVALID_FLAGS_MASK) != 573 SPDK_BLOB_INVALID_FLAGS_MASK) { 574 return -EINVAL; 575 } 576 577 if ((desc_flags->data_ro_flags | SPDK_BLOB_DATA_RO_FLAGS_MASK) != 578 SPDK_BLOB_DATA_RO_FLAGS_MASK) { 579 blob->data_ro = true; 580 blob->md_ro = true; 581 } 582 583 if ((desc_flags->md_ro_flags | SPDK_BLOB_MD_RO_FLAGS_MASK) != 584 SPDK_BLOB_MD_RO_FLAGS_MASK) { 585 blob->md_ro = true; 586 } 587 588 if ((desc_flags->data_ro_flags & SPDK_BLOB_READ_ONLY)) { 589 blob->data_ro = true; 590 blob->md_ro = true; 591 } 592 593 blob->invalid_flags = desc_flags->invalid_flags; 594 blob->data_ro_flags = desc_flags->data_ro_flags; 595 blob->md_ro_flags = desc_flags->md_ro_flags; 596 597 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_EXTENT_RLE) { 598 struct spdk_blob_md_descriptor_extent_rle *desc_extent_rle; 599 unsigned int i, j; 600 unsigned int cluster_count = blob->active.num_clusters; 601 602 if (blob->extent_table_found) { 603 /* Extent Table already present in the md, 604 * both descriptors should never be at the same time. */ 605 return -EINVAL; 606 } 607 blob->extent_rle_found = true; 608 609 desc_extent_rle = (struct spdk_blob_md_descriptor_extent_rle *)desc; 610 611 if (desc_extent_rle->length == 0 || 612 (desc_extent_rle->length % sizeof(desc_extent_rle->extents[0]) != 0)) { 613 return -EINVAL; 614 } 615 616 for (i = 0; i < desc_extent_rle->length / sizeof(desc_extent_rle->extents[0]); i++) { 617 for (j = 0; j < desc_extent_rle->extents[i].length; j++) { 618 if (desc_extent_rle->extents[i].cluster_idx != 0) { 619 if (!spdk_bit_pool_is_allocated(blob->bs->used_clusters, 620 desc_extent_rle->extents[i].cluster_idx + j)) { 621 return -EINVAL; 622 } 623 } 624 cluster_count++; 625 } 626 } 627 628 if (cluster_count == 0) { 629 return -EINVAL; 630 } 631 tmp = realloc(blob->active.clusters, cluster_count * sizeof(*blob->active.clusters)); 632 if (tmp == NULL) { 633 return -ENOMEM; 634 } 635 blob->active.clusters = tmp; 636 blob->active.cluster_array_size = cluster_count; 637 638 for (i = 0; i < desc_extent_rle->length / sizeof(desc_extent_rle->extents[0]); i++) { 639 for (j = 0; j < desc_extent_rle->extents[i].length; j++) { 640 if (desc_extent_rle->extents[i].cluster_idx != 0) { 641 blob->active.clusters[blob->active.num_clusters++] = bs_cluster_to_lba(blob->bs, 642 desc_extent_rle->extents[i].cluster_idx + j); 643 } else if (spdk_blob_is_thin_provisioned(blob)) { 644 blob->active.clusters[blob->active.num_clusters++] = 0; 645 } else { 646 return -EINVAL; 647 } 648 } 649 } 650 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_EXTENT_TABLE) { 651 struct spdk_blob_md_descriptor_extent_table *desc_extent_table; 652 uint32_t num_extent_pages = blob->active.num_extent_pages; 653 uint32_t i, j; 654 size_t extent_pages_length; 655 656 desc_extent_table = (struct spdk_blob_md_descriptor_extent_table *)desc; 657 extent_pages_length = desc_extent_table->length - sizeof(desc_extent_table->num_clusters); 658 659 if (blob->extent_rle_found) { 660 /* This means that Extent RLE is present in MD, 661 * both should never be at the same time. */ 662 return -EINVAL; 663 } else if (blob->extent_table_found && 664 desc_extent_table->num_clusters != blob->remaining_clusters_in_et) { 665 /* Number of clusters in this ET does not match number 666 * from previously read EXTENT_TABLE. */ 667 return -EINVAL; 668 } 669 670 blob->extent_table_found = true; 671 672 if (desc_extent_table->length == 0 || 673 (extent_pages_length % sizeof(desc_extent_table->extent_page[0]) != 0)) { 674 return -EINVAL; 675 } 676 677 for (i = 0; i < extent_pages_length / sizeof(desc_extent_table->extent_page[0]); i++) { 678 num_extent_pages += desc_extent_table->extent_page[i].num_pages; 679 } 680 681 tmp = realloc(blob->active.extent_pages, num_extent_pages * sizeof(uint32_t)); 682 if (tmp == NULL) { 683 return -ENOMEM; 684 } 685 blob->active.extent_pages = tmp; 686 blob->active.extent_pages_array_size = num_extent_pages; 687 688 blob->remaining_clusters_in_et = desc_extent_table->num_clusters; 689 690 /* Extent table entries contain md page numbers for extent pages. 691 * Zeroes represent unallocated extent pages, those are run-length-encoded. 692 */ 693 for (i = 0; i < extent_pages_length / sizeof(desc_extent_table->extent_page[0]); i++) { 694 if (desc_extent_table->extent_page[i].page_idx != 0) { 695 assert(desc_extent_table->extent_page[i].num_pages == 1); 696 blob->active.extent_pages[blob->active.num_extent_pages++] = 697 desc_extent_table->extent_page[i].page_idx; 698 } else if (spdk_blob_is_thin_provisioned(blob)) { 699 for (j = 0; j < desc_extent_table->extent_page[i].num_pages; j++) { 700 blob->active.extent_pages[blob->active.num_extent_pages++] = 0; 701 } 702 } else { 703 return -EINVAL; 704 } 705 } 706 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_EXTENT_PAGE) { 707 struct spdk_blob_md_descriptor_extent_page *desc_extent; 708 unsigned int i; 709 unsigned int cluster_count = 0; 710 size_t cluster_idx_length; 711 712 if (blob->extent_rle_found) { 713 /* This means that Extent RLE is present in MD, 714 * both should never be at the same time. */ 715 return -EINVAL; 716 } 717 718 desc_extent = (struct spdk_blob_md_descriptor_extent_page *)desc; 719 cluster_idx_length = desc_extent->length - sizeof(desc_extent->start_cluster_idx); 720 721 if (desc_extent->length <= sizeof(desc_extent->start_cluster_idx) || 722 (cluster_idx_length % sizeof(desc_extent->cluster_idx[0]) != 0)) { 723 return -EINVAL; 724 } 725 726 for (i = 0; i < cluster_idx_length / sizeof(desc_extent->cluster_idx[0]); i++) { 727 if (desc_extent->cluster_idx[i] != 0) { 728 if (!spdk_bit_pool_is_allocated(blob->bs->used_clusters, desc_extent->cluster_idx[i])) { 729 return -EINVAL; 730 } 731 } 732 cluster_count++; 733 } 734 735 if (cluster_count == 0) { 736 return -EINVAL; 737 } 738 739 /* When reading extent pages sequentially starting cluster idx should match 740 * current size of a blob. 741 * If changed to batch reading, this check shall be removed. */ 742 if (desc_extent->start_cluster_idx != blob->active.num_clusters) { 743 return -EINVAL; 744 } 745 746 tmp = realloc(blob->active.clusters, 747 (cluster_count + blob->active.num_clusters) * sizeof(*blob->active.clusters)); 748 if (tmp == NULL) { 749 return -ENOMEM; 750 } 751 blob->active.clusters = tmp; 752 blob->active.cluster_array_size = (cluster_count + blob->active.num_clusters); 753 754 for (i = 0; i < cluster_idx_length / sizeof(desc_extent->cluster_idx[0]); i++) { 755 if (desc_extent->cluster_idx[i] != 0) { 756 blob->active.clusters[blob->active.num_clusters++] = bs_cluster_to_lba(blob->bs, 757 desc_extent->cluster_idx[i]); 758 } else if (spdk_blob_is_thin_provisioned(blob)) { 759 blob->active.clusters[blob->active.num_clusters++] = 0; 760 } else { 761 return -EINVAL; 762 } 763 } 764 assert(desc_extent->start_cluster_idx + cluster_count == blob->active.num_clusters); 765 assert(blob->remaining_clusters_in_et >= cluster_count); 766 blob->remaining_clusters_in_et -= cluster_count; 767 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_XATTR) { 768 int rc; 769 770 rc = blob_deserialize_xattr(blob, 771 (struct spdk_blob_md_descriptor_xattr *) desc, false); 772 if (rc != 0) { 773 return rc; 774 } 775 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_XATTR_INTERNAL) { 776 int rc; 777 778 rc = blob_deserialize_xattr(blob, 779 (struct spdk_blob_md_descriptor_xattr *) desc, true); 780 if (rc != 0) { 781 return rc; 782 } 783 } else { 784 /* Unrecognized descriptor type. Do not fail - just continue to the 785 * next descriptor. If this descriptor is associated with some feature 786 * defined in a newer version of blobstore, that version of blobstore 787 * should create and set an associated feature flag to specify if this 788 * blob can be loaded or not. 789 */ 790 } 791 792 /* Advance to the next descriptor */ 793 cur_desc += sizeof(*desc) + desc->length; 794 if (cur_desc + sizeof(*desc) > sizeof(page->descriptors)) { 795 break; 796 } 797 desc = (struct spdk_blob_md_descriptor *)((uintptr_t)page->descriptors + cur_desc); 798 } 799 800 return 0; 801 } 802 803 static bool bs_load_cur_extent_page_valid(struct spdk_blob_md_page *page); 804 805 static int 806 blob_parse_extent_page(struct spdk_blob_md_page *extent_page, struct spdk_blob *blob) 807 { 808 assert(blob != NULL); 809 assert(blob->state == SPDK_BLOB_STATE_LOADING); 810 811 if (bs_load_cur_extent_page_valid(extent_page) == false) { 812 return -ENOENT; 813 } 814 815 return blob_parse_page(extent_page, blob); 816 } 817 818 static int 819 blob_parse(const struct spdk_blob_md_page *pages, uint32_t page_count, 820 struct spdk_blob *blob) 821 { 822 const struct spdk_blob_md_page *page; 823 uint32_t i; 824 int rc; 825 void *tmp; 826 827 assert(page_count > 0); 828 assert(pages[0].sequence_num == 0); 829 assert(blob != NULL); 830 assert(blob->state == SPDK_BLOB_STATE_LOADING); 831 assert(blob->active.clusters == NULL); 832 833 /* The blobid provided doesn't match what's in the MD, this can 834 * happen for example if a bogus blobid is passed in through open. 835 */ 836 if (blob->id != pages[0].id) { 837 SPDK_ERRLOG("Blobid (%" PRIu64 ") doesn't match what's in metadata (%" PRIu64 ")\n", 838 blob->id, pages[0].id); 839 return -ENOENT; 840 } 841 842 tmp = realloc(blob->active.pages, page_count * sizeof(*blob->active.pages)); 843 if (!tmp) { 844 return -ENOMEM; 845 } 846 blob->active.pages = tmp; 847 848 blob->active.pages[0] = pages[0].id; 849 850 for (i = 1; i < page_count; i++) { 851 assert(spdk_bit_array_get(blob->bs->used_md_pages, pages[i - 1].next)); 852 blob->active.pages[i] = pages[i - 1].next; 853 } 854 blob->active.num_pages = page_count; 855 856 for (i = 0; i < page_count; i++) { 857 page = &pages[i]; 858 859 assert(page->id == blob->id); 860 assert(page->sequence_num == i); 861 862 rc = blob_parse_page(page, blob); 863 if (rc != 0) { 864 return rc; 865 } 866 } 867 868 return 0; 869 } 870 871 static int 872 blob_serialize_add_page(const struct spdk_blob *blob, 873 struct spdk_blob_md_page **pages, 874 uint32_t *page_count, 875 struct spdk_blob_md_page **last_page) 876 { 877 struct spdk_blob_md_page *page; 878 879 assert(pages != NULL); 880 assert(page_count != NULL); 881 882 if (*page_count == 0) { 883 assert(*pages == NULL); 884 *page_count = 1; 885 *pages = spdk_malloc(SPDK_BS_PAGE_SIZE, 0, 886 NULL, SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA); 887 } else { 888 assert(*pages != NULL); 889 (*page_count)++; 890 *pages = spdk_realloc(*pages, SPDK_BS_PAGE_SIZE * (*page_count), 0); 891 } 892 893 if (*pages == NULL) { 894 *page_count = 0; 895 *last_page = NULL; 896 return -ENOMEM; 897 } 898 899 page = &(*pages)[*page_count - 1]; 900 memset(page, 0, sizeof(*page)); 901 page->id = blob->id; 902 page->sequence_num = *page_count - 1; 903 page->next = SPDK_INVALID_MD_PAGE; 904 *last_page = page; 905 906 return 0; 907 } 908 909 /* Transform the in-memory representation 'xattr' into an on-disk xattr descriptor. 910 * Update required_sz on both success and failure. 911 * 912 */ 913 static int 914 blob_serialize_xattr(const struct spdk_xattr *xattr, 915 uint8_t *buf, size_t buf_sz, 916 size_t *required_sz, bool internal) 917 { 918 struct spdk_blob_md_descriptor_xattr *desc; 919 920 *required_sz = sizeof(struct spdk_blob_md_descriptor_xattr) + 921 strlen(xattr->name) + 922 xattr->value_len; 923 924 if (buf_sz < *required_sz) { 925 return -1; 926 } 927 928 desc = (struct spdk_blob_md_descriptor_xattr *)buf; 929 930 desc->type = internal ? SPDK_MD_DESCRIPTOR_TYPE_XATTR_INTERNAL : SPDK_MD_DESCRIPTOR_TYPE_XATTR; 931 desc->length = sizeof(desc->name_length) + 932 sizeof(desc->value_length) + 933 strlen(xattr->name) + 934 xattr->value_len; 935 desc->name_length = strlen(xattr->name); 936 desc->value_length = xattr->value_len; 937 938 memcpy(desc->name, xattr->name, desc->name_length); 939 memcpy((void *)((uintptr_t)desc->name + desc->name_length), 940 xattr->value, 941 desc->value_length); 942 943 return 0; 944 } 945 946 static void 947 blob_serialize_extent_table_entry(const struct spdk_blob *blob, 948 uint64_t start_ep, uint64_t *next_ep, 949 uint8_t **buf, size_t *remaining_sz) 950 { 951 struct spdk_blob_md_descriptor_extent_table *desc; 952 size_t cur_sz; 953 uint64_t i, et_idx; 954 uint32_t extent_page, ep_len; 955 956 /* The buffer must have room for at least num_clusters entry */ 957 cur_sz = sizeof(struct spdk_blob_md_descriptor) + sizeof(desc->num_clusters); 958 if (*remaining_sz < cur_sz) { 959 *next_ep = start_ep; 960 return; 961 } 962 963 desc = (struct spdk_blob_md_descriptor_extent_table *)*buf; 964 desc->type = SPDK_MD_DESCRIPTOR_TYPE_EXTENT_TABLE; 965 966 desc->num_clusters = blob->active.num_clusters; 967 968 ep_len = 1; 969 et_idx = 0; 970 for (i = start_ep; i < blob->active.num_extent_pages; i++) { 971 if (*remaining_sz < cur_sz + sizeof(desc->extent_page[0])) { 972 /* If we ran out of buffer space, return */ 973 break; 974 } 975 976 extent_page = blob->active.extent_pages[i]; 977 /* Verify that next extent_page is unallocated */ 978 if (extent_page == 0 && 979 (i + 1 < blob->active.num_extent_pages && blob->active.extent_pages[i + 1] == 0)) { 980 ep_len++; 981 continue; 982 } 983 desc->extent_page[et_idx].page_idx = extent_page; 984 desc->extent_page[et_idx].num_pages = ep_len; 985 et_idx++; 986 987 ep_len = 1; 988 cur_sz += sizeof(desc->extent_page[et_idx]); 989 } 990 *next_ep = i; 991 992 desc->length = sizeof(desc->num_clusters) + sizeof(desc->extent_page[0]) * et_idx; 993 *remaining_sz -= sizeof(struct spdk_blob_md_descriptor) + desc->length; 994 *buf += sizeof(struct spdk_blob_md_descriptor) + desc->length; 995 } 996 997 static int 998 blob_serialize_extent_table(const struct spdk_blob *blob, 999 struct spdk_blob_md_page **pages, 1000 struct spdk_blob_md_page *cur_page, 1001 uint32_t *page_count, uint8_t **buf, 1002 size_t *remaining_sz) 1003 { 1004 uint64_t last_extent_page; 1005 int rc; 1006 1007 last_extent_page = 0; 1008 /* At least single extent table entry has to be always persisted. 1009 * Such case occurs with num_extent_pages == 0. */ 1010 while (last_extent_page <= blob->active.num_extent_pages) { 1011 blob_serialize_extent_table_entry(blob, last_extent_page, &last_extent_page, buf, 1012 remaining_sz); 1013 1014 if (last_extent_page == blob->active.num_extent_pages) { 1015 break; 1016 } 1017 1018 rc = blob_serialize_add_page(blob, pages, page_count, &cur_page); 1019 if (rc < 0) { 1020 return rc; 1021 } 1022 1023 *buf = (uint8_t *)cur_page->descriptors; 1024 *remaining_sz = sizeof(cur_page->descriptors); 1025 } 1026 1027 return 0; 1028 } 1029 1030 static void 1031 blob_serialize_extent_rle(const struct spdk_blob *blob, 1032 uint64_t start_cluster, uint64_t *next_cluster, 1033 uint8_t **buf, size_t *buf_sz) 1034 { 1035 struct spdk_blob_md_descriptor_extent_rle *desc_extent_rle; 1036 size_t cur_sz; 1037 uint64_t i, extent_idx; 1038 uint64_t lba, lba_per_cluster, lba_count; 1039 1040 /* The buffer must have room for at least one extent */ 1041 cur_sz = sizeof(struct spdk_blob_md_descriptor) + sizeof(desc_extent_rle->extents[0]); 1042 if (*buf_sz < cur_sz) { 1043 *next_cluster = start_cluster; 1044 return; 1045 } 1046 1047 desc_extent_rle = (struct spdk_blob_md_descriptor_extent_rle *)*buf; 1048 desc_extent_rle->type = SPDK_MD_DESCRIPTOR_TYPE_EXTENT_RLE; 1049 1050 lba_per_cluster = bs_cluster_to_lba(blob->bs, 1); 1051 1052 lba = blob->active.clusters[start_cluster]; 1053 lba_count = lba_per_cluster; 1054 extent_idx = 0; 1055 for (i = start_cluster + 1; i < blob->active.num_clusters; i++) { 1056 if ((lba + lba_count) == blob->active.clusters[i] && lba != 0) { 1057 /* Run-length encode sequential non-zero LBA */ 1058 lba_count += lba_per_cluster; 1059 continue; 1060 } else if (lba == 0 && blob->active.clusters[i] == 0) { 1061 /* Run-length encode unallocated clusters */ 1062 lba_count += lba_per_cluster; 1063 continue; 1064 } 1065 desc_extent_rle->extents[extent_idx].cluster_idx = lba / lba_per_cluster; 1066 desc_extent_rle->extents[extent_idx].length = lba_count / lba_per_cluster; 1067 extent_idx++; 1068 1069 cur_sz += sizeof(desc_extent_rle->extents[extent_idx]); 1070 1071 if (*buf_sz < cur_sz) { 1072 /* If we ran out of buffer space, return */ 1073 *next_cluster = i; 1074 break; 1075 } 1076 1077 lba = blob->active.clusters[i]; 1078 lba_count = lba_per_cluster; 1079 } 1080 1081 if (*buf_sz >= cur_sz) { 1082 desc_extent_rle->extents[extent_idx].cluster_idx = lba / lba_per_cluster; 1083 desc_extent_rle->extents[extent_idx].length = lba_count / lba_per_cluster; 1084 extent_idx++; 1085 1086 *next_cluster = blob->active.num_clusters; 1087 } 1088 1089 desc_extent_rle->length = sizeof(desc_extent_rle->extents[0]) * extent_idx; 1090 *buf_sz -= sizeof(struct spdk_blob_md_descriptor) + desc_extent_rle->length; 1091 *buf += sizeof(struct spdk_blob_md_descriptor) + desc_extent_rle->length; 1092 } 1093 1094 static int 1095 blob_serialize_extents_rle(const struct spdk_blob *blob, 1096 struct spdk_blob_md_page **pages, 1097 struct spdk_blob_md_page *cur_page, 1098 uint32_t *page_count, uint8_t **buf, 1099 size_t *remaining_sz) 1100 { 1101 uint64_t last_cluster; 1102 int rc; 1103 1104 last_cluster = 0; 1105 while (last_cluster < blob->active.num_clusters) { 1106 blob_serialize_extent_rle(blob, last_cluster, &last_cluster, buf, remaining_sz); 1107 1108 if (last_cluster == blob->active.num_clusters) { 1109 break; 1110 } 1111 1112 rc = blob_serialize_add_page(blob, pages, page_count, &cur_page); 1113 if (rc < 0) { 1114 return rc; 1115 } 1116 1117 *buf = (uint8_t *)cur_page->descriptors; 1118 *remaining_sz = sizeof(cur_page->descriptors); 1119 } 1120 1121 return 0; 1122 } 1123 1124 static void 1125 blob_serialize_extent_page(const struct spdk_blob *blob, 1126 uint64_t cluster, struct spdk_blob_md_page *page) 1127 { 1128 struct spdk_blob_md_descriptor_extent_page *desc_extent; 1129 uint64_t i, extent_idx; 1130 uint64_t lba, lba_per_cluster; 1131 uint64_t start_cluster_idx = (cluster / SPDK_EXTENTS_PER_EP) * SPDK_EXTENTS_PER_EP; 1132 1133 desc_extent = (struct spdk_blob_md_descriptor_extent_page *) page->descriptors; 1134 desc_extent->type = SPDK_MD_DESCRIPTOR_TYPE_EXTENT_PAGE; 1135 1136 lba_per_cluster = bs_cluster_to_lba(blob->bs, 1); 1137 1138 desc_extent->start_cluster_idx = start_cluster_idx; 1139 extent_idx = 0; 1140 for (i = start_cluster_idx; i < blob->active.num_clusters; i++) { 1141 lba = blob->active.clusters[i]; 1142 desc_extent->cluster_idx[extent_idx++] = lba / lba_per_cluster; 1143 if (extent_idx >= SPDK_EXTENTS_PER_EP) { 1144 break; 1145 } 1146 } 1147 desc_extent->length = sizeof(desc_extent->start_cluster_idx) + 1148 sizeof(desc_extent->cluster_idx[0]) * extent_idx; 1149 } 1150 1151 static void 1152 blob_serialize_flags(const struct spdk_blob *blob, 1153 uint8_t *buf, size_t *buf_sz) 1154 { 1155 struct spdk_blob_md_descriptor_flags *desc; 1156 1157 /* 1158 * Flags get serialized first, so we should always have room for the flags 1159 * descriptor. 1160 */ 1161 assert(*buf_sz >= sizeof(*desc)); 1162 1163 desc = (struct spdk_blob_md_descriptor_flags *)buf; 1164 desc->type = SPDK_MD_DESCRIPTOR_TYPE_FLAGS; 1165 desc->length = sizeof(*desc) - sizeof(struct spdk_blob_md_descriptor); 1166 desc->invalid_flags = blob->invalid_flags; 1167 desc->data_ro_flags = blob->data_ro_flags; 1168 desc->md_ro_flags = blob->md_ro_flags; 1169 1170 *buf_sz -= sizeof(*desc); 1171 } 1172 1173 static int 1174 blob_serialize_xattrs(const struct spdk_blob *blob, 1175 const struct spdk_xattr_tailq *xattrs, bool internal, 1176 struct spdk_blob_md_page **pages, 1177 struct spdk_blob_md_page *cur_page, 1178 uint32_t *page_count, uint8_t **buf, 1179 size_t *remaining_sz) 1180 { 1181 const struct spdk_xattr *xattr; 1182 int rc; 1183 1184 TAILQ_FOREACH(xattr, xattrs, link) { 1185 size_t required_sz = 0; 1186 1187 rc = blob_serialize_xattr(xattr, 1188 *buf, *remaining_sz, 1189 &required_sz, internal); 1190 if (rc < 0) { 1191 /* Need to add a new page to the chain */ 1192 rc = blob_serialize_add_page(blob, pages, page_count, 1193 &cur_page); 1194 if (rc < 0) { 1195 spdk_free(*pages); 1196 *pages = NULL; 1197 *page_count = 0; 1198 return rc; 1199 } 1200 1201 *buf = (uint8_t *)cur_page->descriptors; 1202 *remaining_sz = sizeof(cur_page->descriptors); 1203 1204 /* Try again */ 1205 required_sz = 0; 1206 rc = blob_serialize_xattr(xattr, 1207 *buf, *remaining_sz, 1208 &required_sz, internal); 1209 1210 if (rc < 0) { 1211 spdk_free(*pages); 1212 *pages = NULL; 1213 *page_count = 0; 1214 return rc; 1215 } 1216 } 1217 1218 *remaining_sz -= required_sz; 1219 *buf += required_sz; 1220 } 1221 1222 return 0; 1223 } 1224 1225 static int 1226 blob_serialize(const struct spdk_blob *blob, struct spdk_blob_md_page **pages, 1227 uint32_t *page_count) 1228 { 1229 struct spdk_blob_md_page *cur_page; 1230 int rc; 1231 uint8_t *buf; 1232 size_t remaining_sz; 1233 1234 assert(pages != NULL); 1235 assert(page_count != NULL); 1236 assert(blob != NULL); 1237 assert(blob->state == SPDK_BLOB_STATE_DIRTY); 1238 1239 *pages = NULL; 1240 *page_count = 0; 1241 1242 /* A blob always has at least 1 page, even if it has no descriptors */ 1243 rc = blob_serialize_add_page(blob, pages, page_count, &cur_page); 1244 if (rc < 0) { 1245 return rc; 1246 } 1247 1248 buf = (uint8_t *)cur_page->descriptors; 1249 remaining_sz = sizeof(cur_page->descriptors); 1250 1251 /* Serialize flags */ 1252 blob_serialize_flags(blob, buf, &remaining_sz); 1253 buf += sizeof(struct spdk_blob_md_descriptor_flags); 1254 1255 /* Serialize xattrs */ 1256 rc = blob_serialize_xattrs(blob, &blob->xattrs, false, 1257 pages, cur_page, page_count, &buf, &remaining_sz); 1258 if (rc < 0) { 1259 return rc; 1260 } 1261 1262 /* Serialize internal xattrs */ 1263 rc = blob_serialize_xattrs(blob, &blob->xattrs_internal, true, 1264 pages, cur_page, page_count, &buf, &remaining_sz); 1265 if (rc < 0) { 1266 return rc; 1267 } 1268 1269 if (blob->use_extent_table) { 1270 /* Serialize extent table */ 1271 rc = blob_serialize_extent_table(blob, pages, cur_page, page_count, &buf, &remaining_sz); 1272 } else { 1273 /* Serialize extents */ 1274 rc = blob_serialize_extents_rle(blob, pages, cur_page, page_count, &buf, &remaining_sz); 1275 } 1276 1277 return rc; 1278 } 1279 1280 struct spdk_blob_load_ctx { 1281 struct spdk_blob *blob; 1282 1283 struct spdk_blob_md_page *pages; 1284 uint32_t num_pages; 1285 uint32_t next_extent_page; 1286 spdk_bs_sequence_t *seq; 1287 1288 spdk_bs_sequence_cpl cb_fn; 1289 void *cb_arg; 1290 }; 1291 1292 static uint32_t 1293 blob_md_page_calc_crc(void *page) 1294 { 1295 uint32_t crc; 1296 1297 crc = BLOB_CRC32C_INITIAL; 1298 crc = spdk_crc32c_update(page, SPDK_BS_PAGE_SIZE - 4, crc); 1299 crc ^= BLOB_CRC32C_INITIAL; 1300 1301 return crc; 1302 1303 } 1304 1305 static void 1306 blob_load_final(struct spdk_blob_load_ctx *ctx, int bserrno) 1307 { 1308 struct spdk_blob *blob = ctx->blob; 1309 1310 if (bserrno == 0) { 1311 blob_mark_clean(blob); 1312 } 1313 1314 ctx->cb_fn(ctx->seq, ctx->cb_arg, bserrno); 1315 1316 /* Free the memory */ 1317 spdk_free(ctx->pages); 1318 free(ctx); 1319 } 1320 1321 static void 1322 blob_load_snapshot_cpl(void *cb_arg, struct spdk_blob *snapshot, int bserrno) 1323 { 1324 struct spdk_blob_load_ctx *ctx = cb_arg; 1325 struct spdk_blob *blob = ctx->blob; 1326 1327 if (bserrno == 0) { 1328 blob->back_bs_dev = bs_create_blob_bs_dev(snapshot); 1329 if (blob->back_bs_dev == NULL) { 1330 bserrno = -ENOMEM; 1331 } 1332 } 1333 if (bserrno != 0) { 1334 SPDK_ERRLOG("Snapshot fail\n"); 1335 } 1336 1337 blob_load_final(ctx, bserrno); 1338 } 1339 1340 static void blob_update_clear_method(struct spdk_blob *blob); 1341 1342 static void 1343 blob_load_backing_dev(void *cb_arg) 1344 { 1345 struct spdk_blob_load_ctx *ctx = cb_arg; 1346 struct spdk_blob *blob = ctx->blob; 1347 const void *value; 1348 size_t len; 1349 int rc; 1350 1351 if (spdk_blob_is_thin_provisioned(blob)) { 1352 rc = blob_get_xattr_value(blob, BLOB_SNAPSHOT, &value, &len, true); 1353 if (rc == 0) { 1354 if (len != sizeof(spdk_blob_id)) { 1355 blob_load_final(ctx, -EINVAL); 1356 return; 1357 } 1358 /* open snapshot blob and continue in the callback function */ 1359 blob->parent_id = *(spdk_blob_id *)value; 1360 spdk_bs_open_blob(blob->bs, blob->parent_id, 1361 blob_load_snapshot_cpl, ctx); 1362 return; 1363 } else { 1364 /* add zeroes_dev for thin provisioned blob */ 1365 blob->back_bs_dev = bs_create_zeroes_dev(); 1366 } 1367 } else { 1368 /* standard blob */ 1369 blob->back_bs_dev = NULL; 1370 } 1371 blob_load_final(ctx, 0); 1372 } 1373 1374 static void 1375 blob_load_cpl_extents_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 1376 { 1377 struct spdk_blob_load_ctx *ctx = cb_arg; 1378 struct spdk_blob *blob = ctx->blob; 1379 struct spdk_blob_md_page *page; 1380 uint64_t i; 1381 uint32_t crc; 1382 uint64_t lba; 1383 void *tmp; 1384 uint64_t sz; 1385 1386 if (bserrno) { 1387 SPDK_ERRLOG("Extent page read failed: %d\n", bserrno); 1388 blob_load_final(ctx, bserrno); 1389 return; 1390 } 1391 1392 if (ctx->pages == NULL) { 1393 /* First iteration of this function, allocate buffer for single EXTENT_PAGE */ 1394 ctx->pages = spdk_zmalloc(SPDK_BS_PAGE_SIZE, 0, 1395 NULL, SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA); 1396 if (!ctx->pages) { 1397 blob_load_final(ctx, -ENOMEM); 1398 return; 1399 } 1400 ctx->num_pages = 1; 1401 ctx->next_extent_page = 0; 1402 } else { 1403 page = &ctx->pages[0]; 1404 crc = blob_md_page_calc_crc(page); 1405 if (crc != page->crc) { 1406 blob_load_final(ctx, -EINVAL); 1407 return; 1408 } 1409 1410 if (page->next != SPDK_INVALID_MD_PAGE) { 1411 blob_load_final(ctx, -EINVAL); 1412 return; 1413 } 1414 1415 bserrno = blob_parse_extent_page(page, blob); 1416 if (bserrno) { 1417 blob_load_final(ctx, bserrno); 1418 return; 1419 } 1420 } 1421 1422 for (i = ctx->next_extent_page; i < blob->active.num_extent_pages; i++) { 1423 if (blob->active.extent_pages[i] != 0) { 1424 /* Extent page was allocated, read and parse it. */ 1425 lba = bs_md_page_to_lba(blob->bs, blob->active.extent_pages[i]); 1426 ctx->next_extent_page = i + 1; 1427 1428 bs_sequence_read_dev(seq, &ctx->pages[0], lba, 1429 bs_byte_to_lba(blob->bs, SPDK_BS_PAGE_SIZE), 1430 blob_load_cpl_extents_cpl, ctx); 1431 return; 1432 } else { 1433 /* Thin provisioned blobs can point to unallocated extent pages. 1434 * In this case blob size should be increased by up to the amount left in remaining_clusters_in_et. */ 1435 1436 sz = spdk_min(blob->remaining_clusters_in_et, SPDK_EXTENTS_PER_EP); 1437 blob->active.num_clusters += sz; 1438 blob->remaining_clusters_in_et -= sz; 1439 1440 assert(spdk_blob_is_thin_provisioned(blob)); 1441 assert(i + 1 < blob->active.num_extent_pages || blob->remaining_clusters_in_et == 0); 1442 1443 tmp = realloc(blob->active.clusters, blob->active.num_clusters * sizeof(*blob->active.clusters)); 1444 if (tmp == NULL) { 1445 blob_load_final(ctx, -ENOMEM); 1446 return; 1447 } 1448 memset(tmp + sizeof(*blob->active.clusters) * blob->active.cluster_array_size, 0, 1449 sizeof(*blob->active.clusters) * (blob->active.num_clusters - blob->active.cluster_array_size)); 1450 blob->active.clusters = tmp; 1451 blob->active.cluster_array_size = blob->active.num_clusters; 1452 } 1453 } 1454 1455 blob_load_backing_dev(ctx); 1456 } 1457 1458 static void 1459 blob_load_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 1460 { 1461 struct spdk_blob_load_ctx *ctx = cb_arg; 1462 struct spdk_blob *blob = ctx->blob; 1463 struct spdk_blob_md_page *page; 1464 int rc; 1465 uint32_t crc; 1466 uint32_t current_page; 1467 1468 if (ctx->num_pages == 1) { 1469 current_page = bs_blobid_to_page(blob->id); 1470 } else { 1471 assert(ctx->num_pages != 0); 1472 page = &ctx->pages[ctx->num_pages - 2]; 1473 current_page = page->next; 1474 } 1475 1476 if (bserrno) { 1477 SPDK_ERRLOG("Metadata page %d read failed for blobid %" PRIu64 ": %d\n", 1478 current_page, blob->id, bserrno); 1479 blob_load_final(ctx, bserrno); 1480 return; 1481 } 1482 1483 page = &ctx->pages[ctx->num_pages - 1]; 1484 crc = blob_md_page_calc_crc(page); 1485 if (crc != page->crc) { 1486 SPDK_ERRLOG("Metadata page %d crc mismatch for blobid %" PRIu64 "\n", 1487 current_page, blob->id); 1488 blob_load_final(ctx, -EINVAL); 1489 return; 1490 } 1491 1492 if (page->next != SPDK_INVALID_MD_PAGE) { 1493 uint32_t next_page = page->next; 1494 uint64_t next_lba = bs_md_page_to_lba(blob->bs, next_page); 1495 1496 /* Read the next page */ 1497 ctx->num_pages++; 1498 ctx->pages = spdk_realloc(ctx->pages, (sizeof(*page) * ctx->num_pages), 0); 1499 if (ctx->pages == NULL) { 1500 blob_load_final(ctx, -ENOMEM); 1501 return; 1502 } 1503 1504 bs_sequence_read_dev(seq, &ctx->pages[ctx->num_pages - 1], 1505 next_lba, 1506 bs_byte_to_lba(blob->bs, sizeof(*page)), 1507 blob_load_cpl, ctx); 1508 return; 1509 } 1510 1511 /* Parse the pages */ 1512 rc = blob_parse(ctx->pages, ctx->num_pages, blob); 1513 if (rc) { 1514 blob_load_final(ctx, rc); 1515 return; 1516 } 1517 1518 if (blob->extent_table_found == true) { 1519 /* If EXTENT_TABLE was found, that means support for it should be enabled. */ 1520 assert(blob->extent_rle_found == false); 1521 blob->use_extent_table = true; 1522 } else { 1523 /* If EXTENT_RLE or no extent_* descriptor was found disable support 1524 * for extent table. No extent_* descriptors means that blob has length of 0 1525 * and no extent_rle descriptors were persisted for it. 1526 * EXTENT_TABLE if used, is always present in metadata regardless of length. */ 1527 blob->use_extent_table = false; 1528 } 1529 1530 /* Check the clear_method stored in metadata vs what may have been passed 1531 * via spdk_bs_open_blob_ext() and update accordingly. 1532 */ 1533 blob_update_clear_method(blob); 1534 1535 spdk_free(ctx->pages); 1536 ctx->pages = NULL; 1537 1538 if (blob->extent_table_found) { 1539 blob_load_cpl_extents_cpl(seq, ctx, 0); 1540 } else { 1541 blob_load_backing_dev(ctx); 1542 } 1543 } 1544 1545 /* Load a blob from disk given a blobid */ 1546 static void 1547 blob_load(spdk_bs_sequence_t *seq, struct spdk_blob *blob, 1548 spdk_bs_sequence_cpl cb_fn, void *cb_arg) 1549 { 1550 struct spdk_blob_load_ctx *ctx; 1551 struct spdk_blob_store *bs; 1552 uint32_t page_num; 1553 uint64_t lba; 1554 1555 blob_verify_md_op(blob); 1556 1557 bs = blob->bs; 1558 1559 ctx = calloc(1, sizeof(*ctx)); 1560 if (!ctx) { 1561 cb_fn(seq, cb_arg, -ENOMEM); 1562 return; 1563 } 1564 1565 ctx->blob = blob; 1566 ctx->pages = spdk_realloc(ctx->pages, SPDK_BS_PAGE_SIZE, 0); 1567 if (!ctx->pages) { 1568 free(ctx); 1569 cb_fn(seq, cb_arg, -ENOMEM); 1570 return; 1571 } 1572 ctx->num_pages = 1; 1573 ctx->cb_fn = cb_fn; 1574 ctx->cb_arg = cb_arg; 1575 ctx->seq = seq; 1576 1577 page_num = bs_blobid_to_page(blob->id); 1578 lba = bs_md_page_to_lba(blob->bs, page_num); 1579 1580 blob->state = SPDK_BLOB_STATE_LOADING; 1581 1582 bs_sequence_read_dev(seq, &ctx->pages[0], lba, 1583 bs_byte_to_lba(bs, SPDK_BS_PAGE_SIZE), 1584 blob_load_cpl, ctx); 1585 } 1586 1587 struct spdk_blob_persist_ctx { 1588 struct spdk_blob *blob; 1589 1590 struct spdk_bs_super_block *super; 1591 1592 struct spdk_blob_md_page *pages; 1593 uint32_t next_extent_page; 1594 struct spdk_blob_md_page *extent_page; 1595 1596 spdk_bs_sequence_t *seq; 1597 spdk_bs_sequence_cpl cb_fn; 1598 void *cb_arg; 1599 TAILQ_ENTRY(spdk_blob_persist_ctx) link; 1600 }; 1601 1602 static void 1603 bs_batch_clear_dev(struct spdk_blob_persist_ctx *ctx, spdk_bs_batch_t *batch, uint64_t lba, 1604 uint32_t lba_count) 1605 { 1606 switch (ctx->blob->clear_method) { 1607 case BLOB_CLEAR_WITH_DEFAULT: 1608 case BLOB_CLEAR_WITH_UNMAP: 1609 bs_batch_unmap_dev(batch, lba, lba_count); 1610 break; 1611 case BLOB_CLEAR_WITH_WRITE_ZEROES: 1612 bs_batch_write_zeroes_dev(batch, lba, lba_count); 1613 break; 1614 case BLOB_CLEAR_WITH_NONE: 1615 default: 1616 break; 1617 } 1618 } 1619 1620 static void blob_persist_check_dirty(struct spdk_blob_persist_ctx *ctx); 1621 1622 static void 1623 blob_persist_complete(spdk_bs_sequence_t *seq, struct spdk_blob_persist_ctx *ctx, int bserrno) 1624 { 1625 struct spdk_blob_persist_ctx *next_persist; 1626 struct spdk_blob *blob = ctx->blob; 1627 1628 if (bserrno == 0) { 1629 blob_mark_clean(blob); 1630 } 1631 1632 assert(ctx == TAILQ_FIRST(&blob->pending_persists)); 1633 TAILQ_REMOVE(&blob->pending_persists, ctx, link); 1634 1635 next_persist = TAILQ_FIRST(&blob->pending_persists); 1636 1637 /* Call user callback */ 1638 ctx->cb_fn(seq, ctx->cb_arg, bserrno); 1639 1640 /* Free the memory */ 1641 spdk_free(ctx->pages); 1642 free(ctx); 1643 1644 if (next_persist != NULL) { 1645 blob->state = SPDK_BLOB_STATE_DIRTY; 1646 blob_persist_check_dirty(next_persist); 1647 } 1648 } 1649 1650 static void 1651 blob_persist_clear_extents_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 1652 { 1653 struct spdk_blob_persist_ctx *ctx = cb_arg; 1654 struct spdk_blob *blob = ctx->blob; 1655 struct spdk_blob_store *bs = blob->bs; 1656 size_t i; 1657 1658 if (bserrno != 0) { 1659 blob_persist_complete(seq, ctx, bserrno); 1660 return; 1661 } 1662 1663 /* Release all extent_pages that were truncated */ 1664 for (i = blob->active.num_extent_pages; i < blob->active.extent_pages_array_size; i++) { 1665 /* Nothing to release if it was not allocated */ 1666 if (blob->active.extent_pages[i] != 0) { 1667 bs_release_md_page(bs, blob->active.extent_pages[i]); 1668 } 1669 } 1670 1671 if (blob->active.num_extent_pages == 0) { 1672 free(blob->active.extent_pages); 1673 blob->active.extent_pages = NULL; 1674 blob->active.extent_pages_array_size = 0; 1675 } else if (blob->active.num_extent_pages != blob->active.extent_pages_array_size) { 1676 #ifndef __clang_analyzer__ 1677 void *tmp; 1678 1679 /* scan-build really can't figure reallocs, workaround it */ 1680 tmp = realloc(blob->active.extent_pages, sizeof(uint32_t) * blob->active.num_extent_pages); 1681 assert(tmp != NULL); 1682 blob->active.extent_pages = tmp; 1683 #endif 1684 blob->active.extent_pages_array_size = blob->active.num_extent_pages; 1685 } 1686 1687 blob_persist_complete(seq, ctx, bserrno); 1688 } 1689 1690 static void 1691 blob_persist_clear_extents(spdk_bs_sequence_t *seq, struct spdk_blob_persist_ctx *ctx) 1692 { 1693 struct spdk_blob *blob = ctx->blob; 1694 struct spdk_blob_store *bs = blob->bs; 1695 size_t i; 1696 uint64_t lba; 1697 uint32_t lba_count; 1698 spdk_bs_batch_t *batch; 1699 1700 batch = bs_sequence_to_batch(seq, blob_persist_clear_extents_cpl, ctx); 1701 lba_count = bs_byte_to_lba(bs, SPDK_BS_PAGE_SIZE); 1702 1703 /* Clear all extent_pages that were truncated */ 1704 for (i = blob->active.num_extent_pages; i < blob->active.extent_pages_array_size; i++) { 1705 /* Nothing to clear if it was not allocated */ 1706 if (blob->active.extent_pages[i] != 0) { 1707 lba = bs_md_page_to_lba(bs, blob->active.extent_pages[i]); 1708 bs_batch_write_zeroes_dev(batch, lba, lba_count); 1709 } 1710 } 1711 1712 bs_batch_close(batch); 1713 } 1714 1715 static void 1716 blob_persist_clear_clusters_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 1717 { 1718 struct spdk_blob_persist_ctx *ctx = cb_arg; 1719 struct spdk_blob *blob = ctx->blob; 1720 struct spdk_blob_store *bs = blob->bs; 1721 size_t i; 1722 1723 if (bserrno != 0) { 1724 blob_persist_complete(seq, ctx, bserrno); 1725 return; 1726 } 1727 1728 pthread_mutex_lock(&bs->used_clusters_mutex); 1729 /* Release all clusters that were truncated */ 1730 for (i = blob->active.num_clusters; i < blob->active.cluster_array_size; i++) { 1731 uint32_t cluster_num = bs_lba_to_cluster(bs, blob->active.clusters[i]); 1732 1733 /* Nothing to release if it was not allocated */ 1734 if (blob->active.clusters[i] != 0) { 1735 bs_release_cluster(bs, cluster_num); 1736 } 1737 } 1738 pthread_mutex_unlock(&bs->used_clusters_mutex); 1739 1740 if (blob->active.num_clusters == 0) { 1741 free(blob->active.clusters); 1742 blob->active.clusters = NULL; 1743 blob->active.cluster_array_size = 0; 1744 } else if (blob->active.num_clusters != blob->active.cluster_array_size) { 1745 #ifndef __clang_analyzer__ 1746 void *tmp; 1747 1748 /* scan-build really can't figure reallocs, workaround it */ 1749 tmp = realloc(blob->active.clusters, sizeof(*blob->active.clusters) * blob->active.num_clusters); 1750 assert(tmp != NULL); 1751 blob->active.clusters = tmp; 1752 1753 #endif 1754 blob->active.cluster_array_size = blob->active.num_clusters; 1755 } 1756 1757 /* Move on to clearing extent pages */ 1758 blob_persist_clear_extents(seq, ctx); 1759 } 1760 1761 static void 1762 blob_persist_clear_clusters(spdk_bs_sequence_t *seq, struct spdk_blob_persist_ctx *ctx) 1763 { 1764 struct spdk_blob *blob = ctx->blob; 1765 struct spdk_blob_store *bs = blob->bs; 1766 spdk_bs_batch_t *batch; 1767 size_t i; 1768 uint64_t lba; 1769 uint32_t lba_count; 1770 1771 /* Clusters don't move around in blobs. The list shrinks or grows 1772 * at the end, but no changes ever occur in the middle of the list. 1773 */ 1774 1775 batch = bs_sequence_to_batch(seq, blob_persist_clear_clusters_cpl, ctx); 1776 1777 /* Clear all clusters that were truncated */ 1778 lba = 0; 1779 lba_count = 0; 1780 for (i = blob->active.num_clusters; i < blob->active.cluster_array_size; i++) { 1781 uint64_t next_lba = blob->active.clusters[i]; 1782 uint32_t next_lba_count = bs_cluster_to_lba(bs, 1); 1783 1784 if (next_lba > 0 && (lba + lba_count) == next_lba) { 1785 /* This cluster is contiguous with the previous one. */ 1786 lba_count += next_lba_count; 1787 continue; 1788 } else if (next_lba == 0) { 1789 continue; 1790 } 1791 1792 /* This cluster is not contiguous with the previous one. */ 1793 1794 /* If a run of LBAs previously existing, clear them now */ 1795 if (lba_count > 0) { 1796 bs_batch_clear_dev(ctx, batch, lba, lba_count); 1797 } 1798 1799 /* Start building the next batch */ 1800 lba = next_lba; 1801 if (next_lba > 0) { 1802 lba_count = next_lba_count; 1803 } else { 1804 lba_count = 0; 1805 } 1806 } 1807 1808 /* If we ended with a contiguous set of LBAs, clear them now */ 1809 if (lba_count > 0) { 1810 bs_batch_clear_dev(ctx, batch, lba, lba_count); 1811 } 1812 1813 bs_batch_close(batch); 1814 } 1815 1816 static void 1817 blob_persist_zero_pages_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 1818 { 1819 struct spdk_blob_persist_ctx *ctx = cb_arg; 1820 struct spdk_blob *blob = ctx->blob; 1821 struct spdk_blob_store *bs = blob->bs; 1822 size_t i; 1823 1824 if (bserrno != 0) { 1825 blob_persist_complete(seq, ctx, bserrno); 1826 return; 1827 } 1828 1829 /* This loop starts at 1 because the first page is special and handled 1830 * below. The pages (except the first) are never written in place, 1831 * so any pages in the clean list must be zeroed. 1832 */ 1833 for (i = 1; i < blob->clean.num_pages; i++) { 1834 bs_release_md_page(bs, blob->clean.pages[i]); 1835 } 1836 1837 if (blob->active.num_pages == 0) { 1838 uint32_t page_num; 1839 1840 page_num = bs_blobid_to_page(blob->id); 1841 bs_release_md_page(bs, page_num); 1842 } 1843 1844 /* Move on to clearing clusters */ 1845 blob_persist_clear_clusters(seq, ctx); 1846 } 1847 1848 static void 1849 blob_persist_zero_pages(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 1850 { 1851 struct spdk_blob_persist_ctx *ctx = cb_arg; 1852 struct spdk_blob *blob = ctx->blob; 1853 struct spdk_blob_store *bs = blob->bs; 1854 uint64_t lba; 1855 uint32_t lba_count; 1856 spdk_bs_batch_t *batch; 1857 size_t i; 1858 1859 if (bserrno != 0) { 1860 blob_persist_complete(seq, ctx, bserrno); 1861 return; 1862 } 1863 1864 batch = bs_sequence_to_batch(seq, blob_persist_zero_pages_cpl, ctx); 1865 1866 lba_count = bs_byte_to_lba(bs, SPDK_BS_PAGE_SIZE); 1867 1868 /* This loop starts at 1 because the first page is special and handled 1869 * below. The pages (except the first) are never written in place, 1870 * so any pages in the clean list must be zeroed. 1871 */ 1872 for (i = 1; i < blob->clean.num_pages; i++) { 1873 lba = bs_md_page_to_lba(bs, blob->clean.pages[i]); 1874 1875 bs_batch_write_zeroes_dev(batch, lba, lba_count); 1876 } 1877 1878 /* The first page will only be zeroed if this is a delete. */ 1879 if (blob->active.num_pages == 0) { 1880 uint32_t page_num; 1881 1882 /* The first page in the metadata goes where the blobid indicates */ 1883 page_num = bs_blobid_to_page(blob->id); 1884 lba = bs_md_page_to_lba(bs, page_num); 1885 1886 bs_batch_write_zeroes_dev(batch, lba, lba_count); 1887 } 1888 1889 bs_batch_close(batch); 1890 } 1891 1892 static void 1893 blob_persist_write_page_root(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 1894 { 1895 struct spdk_blob_persist_ctx *ctx = cb_arg; 1896 struct spdk_blob *blob = ctx->blob; 1897 struct spdk_blob_store *bs = blob->bs; 1898 uint64_t lba; 1899 uint32_t lba_count; 1900 struct spdk_blob_md_page *page; 1901 1902 if (bserrno != 0) { 1903 blob_persist_complete(seq, ctx, bserrno); 1904 return; 1905 } 1906 1907 if (blob->active.num_pages == 0) { 1908 /* Move on to the next step */ 1909 blob_persist_zero_pages(seq, ctx, 0); 1910 return; 1911 } 1912 1913 lba_count = bs_byte_to_lba(bs, sizeof(*page)); 1914 1915 page = &ctx->pages[0]; 1916 /* The first page in the metadata goes where the blobid indicates */ 1917 lba = bs_md_page_to_lba(bs, bs_blobid_to_page(blob->id)); 1918 1919 bs_sequence_write_dev(seq, page, lba, lba_count, 1920 blob_persist_zero_pages, ctx); 1921 } 1922 1923 static void 1924 blob_persist_write_page_chain(spdk_bs_sequence_t *seq, struct spdk_blob_persist_ctx *ctx) 1925 { 1926 struct spdk_blob *blob = ctx->blob; 1927 struct spdk_blob_store *bs = blob->bs; 1928 uint64_t lba; 1929 uint32_t lba_count; 1930 struct spdk_blob_md_page *page; 1931 spdk_bs_batch_t *batch; 1932 size_t i; 1933 1934 /* Clusters don't move around in blobs. The list shrinks or grows 1935 * at the end, but no changes ever occur in the middle of the list. 1936 */ 1937 1938 lba_count = bs_byte_to_lba(bs, sizeof(*page)); 1939 1940 batch = bs_sequence_to_batch(seq, blob_persist_write_page_root, ctx); 1941 1942 /* This starts at 1. The root page is not written until 1943 * all of the others are finished 1944 */ 1945 for (i = 1; i < blob->active.num_pages; i++) { 1946 page = &ctx->pages[i]; 1947 assert(page->sequence_num == i); 1948 1949 lba = bs_md_page_to_lba(bs, blob->active.pages[i]); 1950 1951 bs_batch_write_dev(batch, page, lba, lba_count); 1952 } 1953 1954 bs_batch_close(batch); 1955 } 1956 1957 static int 1958 blob_resize(struct spdk_blob *blob, uint64_t sz) 1959 { 1960 uint64_t i; 1961 uint64_t *tmp; 1962 uint64_t cluster; 1963 uint32_t lfmd; /* lowest free md page */ 1964 uint64_t num_clusters; 1965 uint32_t *ep_tmp; 1966 uint64_t new_num_ep = 0, current_num_ep = 0; 1967 struct spdk_blob_store *bs; 1968 1969 bs = blob->bs; 1970 1971 blob_verify_md_op(blob); 1972 1973 if (blob->active.num_clusters == sz) { 1974 return 0; 1975 } 1976 1977 if (blob->active.num_clusters < blob->active.cluster_array_size) { 1978 /* If this blob was resized to be larger, then smaller, then 1979 * larger without syncing, then the cluster array already 1980 * contains spare assigned clusters we can use. 1981 */ 1982 num_clusters = spdk_min(blob->active.cluster_array_size, 1983 sz); 1984 } else { 1985 num_clusters = blob->active.num_clusters; 1986 } 1987 1988 if (blob->use_extent_table) { 1989 /* Round up since every cluster beyond current Extent Table size, 1990 * requires new extent page. */ 1991 new_num_ep = spdk_divide_round_up(sz, SPDK_EXTENTS_PER_EP); 1992 current_num_ep = spdk_divide_round_up(num_clusters, SPDK_EXTENTS_PER_EP); 1993 } 1994 1995 /* Check first that we have enough clusters and md pages before we start claiming them. */ 1996 if (sz > num_clusters && spdk_blob_is_thin_provisioned(blob) == false) { 1997 if ((sz - num_clusters) > bs->num_free_clusters) { 1998 return -ENOSPC; 1999 } 2000 lfmd = 0; 2001 for (i = current_num_ep; i < new_num_ep ; i++) { 2002 lfmd = spdk_bit_array_find_first_clear(blob->bs->used_md_pages, lfmd); 2003 if (lfmd == UINT32_MAX) { 2004 /* No more free md pages. Cannot satisfy the request */ 2005 return -ENOSPC; 2006 } 2007 } 2008 } 2009 2010 if (sz > num_clusters) { 2011 /* Expand the cluster array if necessary. 2012 * We only shrink the array when persisting. 2013 */ 2014 tmp = realloc(blob->active.clusters, sizeof(*blob->active.clusters) * sz); 2015 if (sz > 0 && tmp == NULL) { 2016 return -ENOMEM; 2017 } 2018 memset(tmp + blob->active.cluster_array_size, 0, 2019 sizeof(*blob->active.clusters) * (sz - blob->active.cluster_array_size)); 2020 blob->active.clusters = tmp; 2021 blob->active.cluster_array_size = sz; 2022 2023 /* Expand the extents table, only if enough clusters were added */ 2024 if (new_num_ep > current_num_ep && blob->use_extent_table) { 2025 ep_tmp = realloc(blob->active.extent_pages, sizeof(*blob->active.extent_pages) * new_num_ep); 2026 if (new_num_ep > 0 && ep_tmp == NULL) { 2027 return -ENOMEM; 2028 } 2029 memset(ep_tmp + blob->active.extent_pages_array_size, 0, 2030 sizeof(*blob->active.extent_pages) * (new_num_ep - blob->active.extent_pages_array_size)); 2031 blob->active.extent_pages = ep_tmp; 2032 blob->active.extent_pages_array_size = new_num_ep; 2033 } 2034 } 2035 2036 blob->state = SPDK_BLOB_STATE_DIRTY; 2037 2038 if (spdk_blob_is_thin_provisioned(blob) == false) { 2039 cluster = 0; 2040 lfmd = 0; 2041 pthread_mutex_lock(&blob->bs->used_clusters_mutex); 2042 for (i = num_clusters; i < sz; i++) { 2043 bs_allocate_cluster(blob, i, &cluster, &lfmd, true); 2044 lfmd++; 2045 } 2046 pthread_mutex_unlock(&blob->bs->used_clusters_mutex); 2047 } 2048 2049 blob->active.num_clusters = sz; 2050 blob->active.num_extent_pages = new_num_ep; 2051 2052 return 0; 2053 } 2054 2055 static void 2056 blob_persist_generate_new_md(struct spdk_blob_persist_ctx *ctx) 2057 { 2058 spdk_bs_sequence_t *seq = ctx->seq; 2059 struct spdk_blob *blob = ctx->blob; 2060 struct spdk_blob_store *bs = blob->bs; 2061 uint64_t i; 2062 uint32_t page_num; 2063 void *tmp; 2064 int rc; 2065 2066 /* Generate the new metadata */ 2067 rc = blob_serialize(blob, &ctx->pages, &blob->active.num_pages); 2068 if (rc < 0) { 2069 blob_persist_complete(seq, ctx, rc); 2070 return; 2071 } 2072 2073 assert(blob->active.num_pages >= 1); 2074 2075 /* Resize the cache of page indices */ 2076 tmp = realloc(blob->active.pages, blob->active.num_pages * sizeof(*blob->active.pages)); 2077 if (!tmp) { 2078 blob_persist_complete(seq, ctx, -ENOMEM); 2079 return; 2080 } 2081 blob->active.pages = tmp; 2082 2083 /* Assign this metadata to pages. This requires two passes - 2084 * one to verify that there are enough pages and a second 2085 * to actually claim them. */ 2086 page_num = 0; 2087 /* Note that this loop starts at one. The first page location is fixed by the blobid. */ 2088 for (i = 1; i < blob->active.num_pages; i++) { 2089 page_num = spdk_bit_array_find_first_clear(bs->used_md_pages, page_num); 2090 if (page_num == UINT32_MAX) { 2091 blob_persist_complete(seq, ctx, -ENOMEM); 2092 return; 2093 } 2094 page_num++; 2095 } 2096 2097 page_num = 0; 2098 blob->active.pages[0] = bs_blobid_to_page(blob->id); 2099 for (i = 1; i < blob->active.num_pages; i++) { 2100 page_num = spdk_bit_array_find_first_clear(bs->used_md_pages, page_num); 2101 ctx->pages[i - 1].next = page_num; 2102 /* Now that previous metadata page is complete, calculate the crc for it. */ 2103 ctx->pages[i - 1].crc = blob_md_page_calc_crc(&ctx->pages[i - 1]); 2104 blob->active.pages[i] = page_num; 2105 bs_claim_md_page(bs, page_num); 2106 SPDK_DEBUGLOG(blob, "Claiming page %u for blob %" PRIu64 "\n", page_num, blob->id); 2107 page_num++; 2108 } 2109 ctx->pages[i - 1].crc = blob_md_page_calc_crc(&ctx->pages[i - 1]); 2110 /* Start writing the metadata from last page to first */ 2111 blob->state = SPDK_BLOB_STATE_CLEAN; 2112 blob_persist_write_page_chain(seq, ctx); 2113 } 2114 2115 static void 2116 blob_persist_write_extent_pages(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 2117 { 2118 struct spdk_blob_persist_ctx *ctx = cb_arg; 2119 struct spdk_blob *blob = ctx->blob; 2120 size_t i; 2121 uint32_t extent_page_id; 2122 uint32_t page_count = 0; 2123 int rc; 2124 2125 if (ctx->extent_page != NULL) { 2126 spdk_free(ctx->extent_page); 2127 ctx->extent_page = NULL; 2128 } 2129 2130 if (bserrno != 0) { 2131 blob_persist_complete(seq, ctx, bserrno); 2132 return; 2133 } 2134 2135 /* Only write out Extent Pages when blob was resized. */ 2136 for (i = ctx->next_extent_page; i < blob->active.extent_pages_array_size; i++) { 2137 extent_page_id = blob->active.extent_pages[i]; 2138 if (extent_page_id == 0) { 2139 /* No Extent Page to persist */ 2140 assert(spdk_blob_is_thin_provisioned(blob)); 2141 continue; 2142 } 2143 assert(spdk_bit_array_get(blob->bs->used_md_pages, extent_page_id)); 2144 ctx->next_extent_page = i + 1; 2145 rc = blob_serialize_add_page(ctx->blob, &ctx->extent_page, &page_count, &ctx->extent_page); 2146 if (rc < 0) { 2147 blob_persist_complete(seq, ctx, rc); 2148 return; 2149 } 2150 2151 blob->state = SPDK_BLOB_STATE_DIRTY; 2152 blob_serialize_extent_page(blob, i * SPDK_EXTENTS_PER_EP, ctx->extent_page); 2153 2154 ctx->extent_page->crc = blob_md_page_calc_crc(ctx->extent_page); 2155 2156 bs_sequence_write_dev(seq, ctx->extent_page, bs_md_page_to_lba(blob->bs, extent_page_id), 2157 bs_byte_to_lba(blob->bs, SPDK_BS_PAGE_SIZE), 2158 blob_persist_write_extent_pages, ctx); 2159 return; 2160 } 2161 2162 blob_persist_generate_new_md(ctx); 2163 } 2164 2165 static void 2166 blob_persist_start(struct spdk_blob_persist_ctx *ctx) 2167 { 2168 spdk_bs_sequence_t *seq = ctx->seq; 2169 struct spdk_blob *blob = ctx->blob; 2170 2171 if (blob->active.num_pages == 0) { 2172 /* This is the signal that the blob should be deleted. 2173 * Immediately jump to the clean up routine. */ 2174 assert(blob->clean.num_pages > 0); 2175 blob->state = SPDK_BLOB_STATE_CLEAN; 2176 blob_persist_zero_pages(seq, ctx, 0); 2177 return; 2178 2179 } 2180 2181 if (blob->clean.num_clusters < blob->active.num_clusters) { 2182 /* Blob was resized up */ 2183 assert(blob->clean.num_extent_pages <= blob->active.num_extent_pages); 2184 ctx->next_extent_page = spdk_max(1, blob->clean.num_extent_pages) - 1; 2185 } else if (blob->active.num_clusters < blob->active.cluster_array_size) { 2186 /* Blob was resized down */ 2187 assert(blob->clean.num_extent_pages >= blob->active.num_extent_pages); 2188 ctx->next_extent_page = spdk_max(1, blob->active.num_extent_pages) - 1; 2189 } else { 2190 /* No change in size occured */ 2191 blob_persist_generate_new_md(ctx); 2192 return; 2193 } 2194 2195 blob_persist_write_extent_pages(seq, ctx, 0); 2196 } 2197 2198 static void 2199 blob_persist_dirty_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 2200 { 2201 struct spdk_blob_persist_ctx *ctx = cb_arg; 2202 2203 spdk_free(ctx->super); 2204 2205 if (bserrno != 0) { 2206 blob_persist_complete(seq, ctx, bserrno); 2207 return; 2208 } 2209 2210 ctx->blob->bs->clean = 0; 2211 2212 blob_persist_start(ctx); 2213 } 2214 2215 static void 2216 bs_write_super(spdk_bs_sequence_t *seq, struct spdk_blob_store *bs, 2217 struct spdk_bs_super_block *super, spdk_bs_sequence_cpl cb_fn, void *cb_arg); 2218 2219 2220 static void 2221 blob_persist_dirty(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 2222 { 2223 struct spdk_blob_persist_ctx *ctx = cb_arg; 2224 2225 if (bserrno != 0) { 2226 spdk_free(ctx->super); 2227 blob_persist_complete(seq, ctx, bserrno); 2228 return; 2229 } 2230 2231 ctx->super->clean = 0; 2232 if (ctx->super->size == 0) { 2233 ctx->super->size = ctx->blob->bs->dev->blockcnt * ctx->blob->bs->dev->blocklen; 2234 } 2235 2236 bs_write_super(seq, ctx->blob->bs, ctx->super, blob_persist_dirty_cpl, ctx); 2237 } 2238 2239 static void 2240 blob_persist_check_dirty(struct spdk_blob_persist_ctx *ctx) 2241 { 2242 if (ctx->blob->bs->clean) { 2243 ctx->super = spdk_zmalloc(sizeof(*ctx->super), 0x1000, NULL, 2244 SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA); 2245 if (!ctx->super) { 2246 blob_persist_complete(ctx->seq, ctx, -ENOMEM); 2247 return; 2248 } 2249 2250 bs_sequence_read_dev(ctx->seq, ctx->super, bs_page_to_lba(ctx->blob->bs, 0), 2251 bs_byte_to_lba(ctx->blob->bs, sizeof(*ctx->super)), 2252 blob_persist_dirty, ctx); 2253 } else { 2254 blob_persist_start(ctx); 2255 } 2256 } 2257 2258 /* Write a blob to disk */ 2259 static void 2260 blob_persist(spdk_bs_sequence_t *seq, struct spdk_blob *blob, 2261 spdk_bs_sequence_cpl cb_fn, void *cb_arg) 2262 { 2263 struct spdk_blob_persist_ctx *ctx; 2264 2265 blob_verify_md_op(blob); 2266 2267 if (blob->state == SPDK_BLOB_STATE_CLEAN && TAILQ_EMPTY(&blob->pending_persists)) { 2268 cb_fn(seq, cb_arg, 0); 2269 return; 2270 } 2271 2272 ctx = calloc(1, sizeof(*ctx)); 2273 if (!ctx) { 2274 cb_fn(seq, cb_arg, -ENOMEM); 2275 return; 2276 } 2277 ctx->blob = blob; 2278 ctx->seq = seq; 2279 ctx->cb_fn = cb_fn; 2280 ctx->cb_arg = cb_arg; 2281 2282 /* Multiple blob persists can affect one another, via blob->state or 2283 * blob mutable data changes. To prevent it, queue up the persists. */ 2284 if (!TAILQ_EMPTY(&blob->pending_persists)) { 2285 TAILQ_INSERT_TAIL(&blob->pending_persists, ctx, link); 2286 return; 2287 } 2288 TAILQ_INSERT_HEAD(&blob->pending_persists, ctx, link); 2289 2290 blob_persist_check_dirty(ctx); 2291 } 2292 2293 struct spdk_blob_copy_cluster_ctx { 2294 struct spdk_blob *blob; 2295 uint8_t *buf; 2296 uint64_t page; 2297 uint64_t new_cluster; 2298 uint32_t new_extent_page; 2299 spdk_bs_sequence_t *seq; 2300 }; 2301 2302 static void 2303 blob_allocate_and_copy_cluster_cpl(void *cb_arg, int bserrno) 2304 { 2305 struct spdk_blob_copy_cluster_ctx *ctx = cb_arg; 2306 struct spdk_bs_request_set *set = (struct spdk_bs_request_set *)ctx->seq; 2307 TAILQ_HEAD(, spdk_bs_request_set) requests; 2308 spdk_bs_user_op_t *op; 2309 2310 TAILQ_INIT(&requests); 2311 TAILQ_SWAP(&set->channel->need_cluster_alloc, &requests, spdk_bs_request_set, link); 2312 2313 while (!TAILQ_EMPTY(&requests)) { 2314 op = TAILQ_FIRST(&requests); 2315 TAILQ_REMOVE(&requests, op, link); 2316 if (bserrno == 0) { 2317 bs_user_op_execute(op); 2318 } else { 2319 bs_user_op_abort(op); 2320 } 2321 } 2322 2323 spdk_free(ctx->buf); 2324 free(ctx); 2325 } 2326 2327 static void 2328 blob_insert_cluster_cpl(void *cb_arg, int bserrno) 2329 { 2330 struct spdk_blob_copy_cluster_ctx *ctx = cb_arg; 2331 2332 if (bserrno) { 2333 if (bserrno == -EEXIST) { 2334 /* The metadata insert failed because another thread 2335 * allocated the cluster first. Free our cluster 2336 * but continue without error. */ 2337 bserrno = 0; 2338 } 2339 pthread_mutex_lock(&ctx->blob->bs->used_clusters_mutex); 2340 bs_release_cluster(ctx->blob->bs, ctx->new_cluster); 2341 pthread_mutex_unlock(&ctx->blob->bs->used_clusters_mutex); 2342 if (ctx->new_extent_page != 0) { 2343 bs_release_md_page(ctx->blob->bs, ctx->new_extent_page); 2344 } 2345 } 2346 2347 bs_sequence_finish(ctx->seq, bserrno); 2348 } 2349 2350 static void 2351 blob_write_copy_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 2352 { 2353 struct spdk_blob_copy_cluster_ctx *ctx = cb_arg; 2354 uint32_t cluster_number; 2355 2356 if (bserrno) { 2357 /* The write failed, so jump to the final completion handler */ 2358 bs_sequence_finish(seq, bserrno); 2359 return; 2360 } 2361 2362 cluster_number = bs_page_to_cluster(ctx->blob->bs, ctx->page); 2363 2364 blob_insert_cluster_on_md_thread(ctx->blob, cluster_number, ctx->new_cluster, 2365 ctx->new_extent_page, blob_insert_cluster_cpl, ctx); 2366 } 2367 2368 static void 2369 blob_write_copy(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 2370 { 2371 struct spdk_blob_copy_cluster_ctx *ctx = cb_arg; 2372 2373 if (bserrno != 0) { 2374 /* The read failed, so jump to the final completion handler */ 2375 bs_sequence_finish(seq, bserrno); 2376 return; 2377 } 2378 2379 /* Write whole cluster */ 2380 bs_sequence_write_dev(seq, ctx->buf, 2381 bs_cluster_to_lba(ctx->blob->bs, ctx->new_cluster), 2382 bs_cluster_to_lba(ctx->blob->bs, 1), 2383 blob_write_copy_cpl, ctx); 2384 } 2385 2386 static void 2387 bs_allocate_and_copy_cluster(struct spdk_blob *blob, 2388 struct spdk_io_channel *_ch, 2389 uint64_t io_unit, spdk_bs_user_op_t *op) 2390 { 2391 struct spdk_bs_cpl cpl; 2392 struct spdk_bs_channel *ch; 2393 struct spdk_blob_copy_cluster_ctx *ctx; 2394 uint32_t cluster_start_page; 2395 uint32_t cluster_number; 2396 int rc; 2397 2398 ch = spdk_io_channel_get_ctx(_ch); 2399 2400 if (!TAILQ_EMPTY(&ch->need_cluster_alloc)) { 2401 /* There are already operations pending. Queue this user op 2402 * and return because it will be re-executed when the outstanding 2403 * cluster allocation completes. */ 2404 TAILQ_INSERT_TAIL(&ch->need_cluster_alloc, op, link); 2405 return; 2406 } 2407 2408 /* Round the io_unit offset down to the first page in the cluster */ 2409 cluster_start_page = bs_io_unit_to_cluster_start(blob, io_unit); 2410 2411 /* Calculate which index in the metadata cluster array the corresponding 2412 * cluster is supposed to be at. */ 2413 cluster_number = bs_io_unit_to_cluster_number(blob, io_unit); 2414 2415 ctx = calloc(1, sizeof(*ctx)); 2416 if (!ctx) { 2417 bs_user_op_abort(op); 2418 return; 2419 } 2420 2421 assert(blob->bs->cluster_sz % blob->back_bs_dev->blocklen == 0); 2422 2423 ctx->blob = blob; 2424 ctx->page = cluster_start_page; 2425 2426 if (blob->parent_id != SPDK_BLOBID_INVALID) { 2427 ctx->buf = spdk_malloc(blob->bs->cluster_sz, blob->back_bs_dev->blocklen, 2428 NULL, SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA); 2429 if (!ctx->buf) { 2430 SPDK_ERRLOG("DMA allocation for cluster of size = %" PRIu32 " failed.\n", 2431 blob->bs->cluster_sz); 2432 free(ctx); 2433 bs_user_op_abort(op); 2434 return; 2435 } 2436 } 2437 2438 pthread_mutex_lock(&blob->bs->used_clusters_mutex); 2439 rc = bs_allocate_cluster(blob, cluster_number, &ctx->new_cluster, &ctx->new_extent_page, 2440 false); 2441 pthread_mutex_unlock(&blob->bs->used_clusters_mutex); 2442 if (rc != 0) { 2443 spdk_free(ctx->buf); 2444 free(ctx); 2445 bs_user_op_abort(op); 2446 return; 2447 } 2448 2449 cpl.type = SPDK_BS_CPL_TYPE_BLOB_BASIC; 2450 cpl.u.blob_basic.cb_fn = blob_allocate_and_copy_cluster_cpl; 2451 cpl.u.blob_basic.cb_arg = ctx; 2452 2453 ctx->seq = bs_sequence_start(_ch, &cpl); 2454 if (!ctx->seq) { 2455 pthread_mutex_lock(&blob->bs->used_clusters_mutex); 2456 bs_release_cluster(blob->bs, ctx->new_cluster); 2457 pthread_mutex_unlock(&blob->bs->used_clusters_mutex); 2458 spdk_free(ctx->buf); 2459 free(ctx); 2460 bs_user_op_abort(op); 2461 return; 2462 } 2463 2464 /* Queue the user op to block other incoming operations */ 2465 TAILQ_INSERT_TAIL(&ch->need_cluster_alloc, op, link); 2466 2467 if (blob->parent_id != SPDK_BLOBID_INVALID) { 2468 /* Read cluster from backing device */ 2469 bs_sequence_read_bs_dev(ctx->seq, blob->back_bs_dev, ctx->buf, 2470 bs_dev_page_to_lba(blob->back_bs_dev, cluster_start_page), 2471 bs_dev_byte_to_lba(blob->back_bs_dev, blob->bs->cluster_sz), 2472 blob_write_copy, ctx); 2473 } else { 2474 blob_insert_cluster_on_md_thread(ctx->blob, cluster_number, ctx->new_cluster, 2475 ctx->new_extent_page, blob_insert_cluster_cpl, ctx); 2476 } 2477 } 2478 2479 static inline bool 2480 blob_calculate_lba_and_lba_count(struct spdk_blob *blob, uint64_t io_unit, uint64_t length, 2481 uint64_t *lba, uint32_t *lba_count) 2482 { 2483 *lba_count = length; 2484 2485 if (!bs_io_unit_is_allocated(blob, io_unit)) { 2486 assert(blob->back_bs_dev != NULL); 2487 *lba = bs_io_unit_to_back_dev_lba(blob, io_unit); 2488 *lba_count = bs_io_unit_to_back_dev_lba(blob, *lba_count); 2489 return false; 2490 } else { 2491 *lba = bs_blob_io_unit_to_lba(blob, io_unit); 2492 return true; 2493 } 2494 } 2495 2496 struct op_split_ctx { 2497 struct spdk_blob *blob; 2498 struct spdk_io_channel *channel; 2499 uint64_t io_unit_offset; 2500 uint64_t io_units_remaining; 2501 void *curr_payload; 2502 enum spdk_blob_op_type op_type; 2503 spdk_bs_sequence_t *seq; 2504 }; 2505 2506 static void 2507 blob_request_submit_op_split_next(void *cb_arg, int bserrno) 2508 { 2509 struct op_split_ctx *ctx = cb_arg; 2510 struct spdk_blob *blob = ctx->blob; 2511 struct spdk_io_channel *ch = ctx->channel; 2512 enum spdk_blob_op_type op_type = ctx->op_type; 2513 uint8_t *buf = ctx->curr_payload; 2514 uint64_t offset = ctx->io_unit_offset; 2515 uint64_t length = ctx->io_units_remaining; 2516 uint64_t op_length; 2517 2518 if (bserrno != 0 || ctx->io_units_remaining == 0) { 2519 bs_sequence_finish(ctx->seq, bserrno); 2520 free(ctx); 2521 return; 2522 } 2523 2524 op_length = spdk_min(length, bs_num_io_units_to_cluster_boundary(blob, 2525 offset)); 2526 2527 /* Update length and payload for next operation */ 2528 ctx->io_units_remaining -= op_length; 2529 ctx->io_unit_offset += op_length; 2530 if (op_type == SPDK_BLOB_WRITE || op_type == SPDK_BLOB_READ) { 2531 ctx->curr_payload += op_length * blob->bs->io_unit_size; 2532 } 2533 2534 switch (op_type) { 2535 case SPDK_BLOB_READ: 2536 spdk_blob_io_read(blob, ch, buf, offset, op_length, 2537 blob_request_submit_op_split_next, ctx); 2538 break; 2539 case SPDK_BLOB_WRITE: 2540 spdk_blob_io_write(blob, ch, buf, offset, op_length, 2541 blob_request_submit_op_split_next, ctx); 2542 break; 2543 case SPDK_BLOB_UNMAP: 2544 spdk_blob_io_unmap(blob, ch, offset, op_length, 2545 blob_request_submit_op_split_next, ctx); 2546 break; 2547 case SPDK_BLOB_WRITE_ZEROES: 2548 spdk_blob_io_write_zeroes(blob, ch, offset, op_length, 2549 blob_request_submit_op_split_next, ctx); 2550 break; 2551 case SPDK_BLOB_READV: 2552 case SPDK_BLOB_WRITEV: 2553 SPDK_ERRLOG("readv/write not valid\n"); 2554 bs_sequence_finish(ctx->seq, -EINVAL); 2555 free(ctx); 2556 break; 2557 } 2558 } 2559 2560 static void 2561 blob_request_submit_op_split(struct spdk_io_channel *ch, struct spdk_blob *blob, 2562 void *payload, uint64_t offset, uint64_t length, 2563 spdk_blob_op_complete cb_fn, void *cb_arg, enum spdk_blob_op_type op_type) 2564 { 2565 struct op_split_ctx *ctx; 2566 spdk_bs_sequence_t *seq; 2567 struct spdk_bs_cpl cpl; 2568 2569 assert(blob != NULL); 2570 2571 ctx = calloc(1, sizeof(struct op_split_ctx)); 2572 if (ctx == NULL) { 2573 cb_fn(cb_arg, -ENOMEM); 2574 return; 2575 } 2576 2577 cpl.type = SPDK_BS_CPL_TYPE_BLOB_BASIC; 2578 cpl.u.blob_basic.cb_fn = cb_fn; 2579 cpl.u.blob_basic.cb_arg = cb_arg; 2580 2581 seq = bs_sequence_start(ch, &cpl); 2582 if (!seq) { 2583 free(ctx); 2584 cb_fn(cb_arg, -ENOMEM); 2585 return; 2586 } 2587 2588 ctx->blob = blob; 2589 ctx->channel = ch; 2590 ctx->curr_payload = payload; 2591 ctx->io_unit_offset = offset; 2592 ctx->io_units_remaining = length; 2593 ctx->op_type = op_type; 2594 ctx->seq = seq; 2595 2596 blob_request_submit_op_split_next(ctx, 0); 2597 } 2598 2599 static void 2600 blob_request_submit_op_single(struct spdk_io_channel *_ch, struct spdk_blob *blob, 2601 void *payload, uint64_t offset, uint64_t length, 2602 spdk_blob_op_complete cb_fn, void *cb_arg, enum spdk_blob_op_type op_type) 2603 { 2604 struct spdk_bs_cpl cpl; 2605 uint64_t lba; 2606 uint32_t lba_count; 2607 bool is_allocated; 2608 2609 assert(blob != NULL); 2610 2611 cpl.type = SPDK_BS_CPL_TYPE_BLOB_BASIC; 2612 cpl.u.blob_basic.cb_fn = cb_fn; 2613 cpl.u.blob_basic.cb_arg = cb_arg; 2614 2615 is_allocated = blob_calculate_lba_and_lba_count(blob, offset, length, &lba, &lba_count); 2616 2617 if (blob->frozen_refcnt) { 2618 /* This blob I/O is frozen */ 2619 spdk_bs_user_op_t *op; 2620 struct spdk_bs_channel *bs_channel = spdk_io_channel_get_ctx(_ch); 2621 2622 op = bs_user_op_alloc(_ch, &cpl, op_type, blob, payload, 0, offset, length); 2623 if (!op) { 2624 cb_fn(cb_arg, -ENOMEM); 2625 return; 2626 } 2627 2628 TAILQ_INSERT_TAIL(&bs_channel->queued_io, op, link); 2629 2630 return; 2631 } 2632 2633 switch (op_type) { 2634 case SPDK_BLOB_READ: { 2635 spdk_bs_batch_t *batch; 2636 2637 batch = bs_batch_open(_ch, &cpl); 2638 if (!batch) { 2639 cb_fn(cb_arg, -ENOMEM); 2640 return; 2641 } 2642 2643 if (is_allocated) { 2644 /* Read from the blob */ 2645 bs_batch_read_dev(batch, payload, lba, lba_count); 2646 } else { 2647 /* Read from the backing block device */ 2648 bs_batch_read_bs_dev(batch, blob->back_bs_dev, payload, lba, lba_count); 2649 } 2650 2651 bs_batch_close(batch); 2652 break; 2653 } 2654 case SPDK_BLOB_WRITE: 2655 case SPDK_BLOB_WRITE_ZEROES: { 2656 if (is_allocated) { 2657 /* Write to the blob */ 2658 spdk_bs_batch_t *batch; 2659 2660 if (lba_count == 0) { 2661 cb_fn(cb_arg, 0); 2662 return; 2663 } 2664 2665 batch = bs_batch_open(_ch, &cpl); 2666 if (!batch) { 2667 cb_fn(cb_arg, -ENOMEM); 2668 return; 2669 } 2670 2671 if (op_type == SPDK_BLOB_WRITE) { 2672 bs_batch_write_dev(batch, payload, lba, lba_count); 2673 } else { 2674 bs_batch_write_zeroes_dev(batch, lba, lba_count); 2675 } 2676 2677 bs_batch_close(batch); 2678 } else { 2679 /* Queue this operation and allocate the cluster */ 2680 spdk_bs_user_op_t *op; 2681 2682 op = bs_user_op_alloc(_ch, &cpl, op_type, blob, payload, 0, offset, length); 2683 if (!op) { 2684 cb_fn(cb_arg, -ENOMEM); 2685 return; 2686 } 2687 2688 bs_allocate_and_copy_cluster(blob, _ch, offset, op); 2689 } 2690 break; 2691 } 2692 case SPDK_BLOB_UNMAP: { 2693 spdk_bs_batch_t *batch; 2694 2695 batch = bs_batch_open(_ch, &cpl); 2696 if (!batch) { 2697 cb_fn(cb_arg, -ENOMEM); 2698 return; 2699 } 2700 2701 if (is_allocated) { 2702 bs_batch_unmap_dev(batch, lba, lba_count); 2703 } 2704 2705 bs_batch_close(batch); 2706 break; 2707 } 2708 case SPDK_BLOB_READV: 2709 case SPDK_BLOB_WRITEV: 2710 SPDK_ERRLOG("readv/write not valid\n"); 2711 cb_fn(cb_arg, -EINVAL); 2712 break; 2713 } 2714 } 2715 2716 static void 2717 blob_request_submit_op(struct spdk_blob *blob, struct spdk_io_channel *_channel, 2718 void *payload, uint64_t offset, uint64_t length, 2719 spdk_blob_op_complete cb_fn, void *cb_arg, enum spdk_blob_op_type op_type) 2720 { 2721 assert(blob != NULL); 2722 2723 if (blob->data_ro && op_type != SPDK_BLOB_READ) { 2724 cb_fn(cb_arg, -EPERM); 2725 return; 2726 } 2727 2728 if (offset + length > bs_cluster_to_lba(blob->bs, blob->active.num_clusters)) { 2729 cb_fn(cb_arg, -EINVAL); 2730 return; 2731 } 2732 if (length <= bs_num_io_units_to_cluster_boundary(blob, offset)) { 2733 blob_request_submit_op_single(_channel, blob, payload, offset, length, 2734 cb_fn, cb_arg, op_type); 2735 } else { 2736 blob_request_submit_op_split(_channel, blob, payload, offset, length, 2737 cb_fn, cb_arg, op_type); 2738 } 2739 } 2740 2741 struct rw_iov_ctx { 2742 struct spdk_blob *blob; 2743 struct spdk_io_channel *channel; 2744 spdk_blob_op_complete cb_fn; 2745 void *cb_arg; 2746 bool read; 2747 int iovcnt; 2748 struct iovec *orig_iov; 2749 uint64_t io_unit_offset; 2750 uint64_t io_units_remaining; 2751 uint64_t io_units_done; 2752 struct iovec iov[0]; 2753 }; 2754 2755 static void 2756 rw_iov_done(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 2757 { 2758 assert(cb_arg == NULL); 2759 bs_sequence_finish(seq, bserrno); 2760 } 2761 2762 static void 2763 rw_iov_split_next(void *cb_arg, int bserrno) 2764 { 2765 struct rw_iov_ctx *ctx = cb_arg; 2766 struct spdk_blob *blob = ctx->blob; 2767 struct iovec *iov, *orig_iov; 2768 int iovcnt; 2769 size_t orig_iovoff; 2770 uint64_t io_units_count, io_units_to_boundary, io_unit_offset; 2771 uint64_t byte_count; 2772 2773 if (bserrno != 0 || ctx->io_units_remaining == 0) { 2774 ctx->cb_fn(ctx->cb_arg, bserrno); 2775 free(ctx); 2776 return; 2777 } 2778 2779 io_unit_offset = ctx->io_unit_offset; 2780 io_units_to_boundary = bs_num_io_units_to_cluster_boundary(blob, io_unit_offset); 2781 io_units_count = spdk_min(ctx->io_units_remaining, io_units_to_boundary); 2782 /* 2783 * Get index and offset into the original iov array for our current position in the I/O sequence. 2784 * byte_count will keep track of how many bytes remaining until orig_iov and orig_iovoff will 2785 * point to the current position in the I/O sequence. 2786 */ 2787 byte_count = ctx->io_units_done * blob->bs->io_unit_size; 2788 orig_iov = &ctx->orig_iov[0]; 2789 orig_iovoff = 0; 2790 while (byte_count > 0) { 2791 if (byte_count >= orig_iov->iov_len) { 2792 byte_count -= orig_iov->iov_len; 2793 orig_iov++; 2794 } else { 2795 orig_iovoff = byte_count; 2796 byte_count = 0; 2797 } 2798 } 2799 2800 /* 2801 * Build an iov array for the next I/O in the sequence. byte_count will keep track of how many 2802 * bytes of this next I/O remain to be accounted for in the new iov array. 2803 */ 2804 byte_count = io_units_count * blob->bs->io_unit_size; 2805 iov = &ctx->iov[0]; 2806 iovcnt = 0; 2807 while (byte_count > 0) { 2808 assert(iovcnt < ctx->iovcnt); 2809 iov->iov_len = spdk_min(byte_count, orig_iov->iov_len - orig_iovoff); 2810 iov->iov_base = orig_iov->iov_base + orig_iovoff; 2811 byte_count -= iov->iov_len; 2812 orig_iovoff = 0; 2813 orig_iov++; 2814 iov++; 2815 iovcnt++; 2816 } 2817 2818 ctx->io_unit_offset += io_units_count; 2819 ctx->io_units_remaining -= io_units_count; 2820 ctx->io_units_done += io_units_count; 2821 iov = &ctx->iov[0]; 2822 2823 if (ctx->read) { 2824 spdk_blob_io_readv(ctx->blob, ctx->channel, iov, iovcnt, io_unit_offset, 2825 io_units_count, rw_iov_split_next, ctx); 2826 } else { 2827 spdk_blob_io_writev(ctx->blob, ctx->channel, iov, iovcnt, io_unit_offset, 2828 io_units_count, rw_iov_split_next, ctx); 2829 } 2830 } 2831 2832 static void 2833 blob_request_submit_rw_iov(struct spdk_blob *blob, struct spdk_io_channel *_channel, 2834 struct iovec *iov, int iovcnt, uint64_t offset, uint64_t length, 2835 spdk_blob_op_complete cb_fn, void *cb_arg, bool read) 2836 { 2837 struct spdk_bs_cpl cpl; 2838 2839 assert(blob != NULL); 2840 2841 if (!read && blob->data_ro) { 2842 cb_fn(cb_arg, -EPERM); 2843 return; 2844 } 2845 2846 if (length == 0) { 2847 cb_fn(cb_arg, 0); 2848 return; 2849 } 2850 2851 if (offset + length > bs_cluster_to_lba(blob->bs, blob->active.num_clusters)) { 2852 cb_fn(cb_arg, -EINVAL); 2853 return; 2854 } 2855 2856 /* 2857 * For now, we implement readv/writev using a sequence (instead of a batch) to account for having 2858 * to split a request that spans a cluster boundary. For I/O that do not span a cluster boundary, 2859 * there will be no noticeable difference compared to using a batch. For I/O that do span a cluster 2860 * boundary, the target LBAs (after blob offset to LBA translation) may not be contiguous, so we need 2861 * to allocate a separate iov array and split the I/O such that none of the resulting 2862 * smaller I/O cross a cluster boundary. These smaller I/O will be issued in sequence (not in parallel) 2863 * but since this case happens very infrequently, any performance impact will be negligible. 2864 * 2865 * This could be optimized in the future to allocate a big enough iov array to account for all of the iovs 2866 * for all of the smaller I/Os, pre-build all of the iov arrays for the smaller I/Os, then issue them 2867 * in a batch. That would also require creating an intermediate spdk_bs_cpl that would get called 2868 * when the batch was completed, to allow for freeing the memory for the iov arrays. 2869 */ 2870 if (spdk_likely(length <= bs_num_io_units_to_cluster_boundary(blob, offset))) { 2871 uint32_t lba_count; 2872 uint64_t lba; 2873 bool is_allocated; 2874 2875 cpl.type = SPDK_BS_CPL_TYPE_BLOB_BASIC; 2876 cpl.u.blob_basic.cb_fn = cb_fn; 2877 cpl.u.blob_basic.cb_arg = cb_arg; 2878 2879 if (blob->frozen_refcnt) { 2880 /* This blob I/O is frozen */ 2881 enum spdk_blob_op_type op_type; 2882 spdk_bs_user_op_t *op; 2883 struct spdk_bs_channel *bs_channel = spdk_io_channel_get_ctx(_channel); 2884 2885 op_type = read ? SPDK_BLOB_READV : SPDK_BLOB_WRITEV; 2886 op = bs_user_op_alloc(_channel, &cpl, op_type, blob, iov, iovcnt, offset, length); 2887 if (!op) { 2888 cb_fn(cb_arg, -ENOMEM); 2889 return; 2890 } 2891 2892 TAILQ_INSERT_TAIL(&bs_channel->queued_io, op, link); 2893 2894 return; 2895 } 2896 2897 is_allocated = blob_calculate_lba_and_lba_count(blob, offset, length, &lba, &lba_count); 2898 2899 if (read) { 2900 spdk_bs_sequence_t *seq; 2901 2902 seq = bs_sequence_start(_channel, &cpl); 2903 if (!seq) { 2904 cb_fn(cb_arg, -ENOMEM); 2905 return; 2906 } 2907 2908 if (is_allocated) { 2909 bs_sequence_readv_dev(seq, iov, iovcnt, lba, lba_count, rw_iov_done, NULL); 2910 } else { 2911 bs_sequence_readv_bs_dev(seq, blob->back_bs_dev, iov, iovcnt, lba, lba_count, 2912 rw_iov_done, NULL); 2913 } 2914 } else { 2915 if (is_allocated) { 2916 spdk_bs_sequence_t *seq; 2917 2918 seq = bs_sequence_start(_channel, &cpl); 2919 if (!seq) { 2920 cb_fn(cb_arg, -ENOMEM); 2921 return; 2922 } 2923 2924 bs_sequence_writev_dev(seq, iov, iovcnt, lba, lba_count, rw_iov_done, NULL); 2925 } else { 2926 /* Queue this operation and allocate the cluster */ 2927 spdk_bs_user_op_t *op; 2928 2929 op = bs_user_op_alloc(_channel, &cpl, SPDK_BLOB_WRITEV, blob, iov, iovcnt, offset, 2930 length); 2931 if (!op) { 2932 cb_fn(cb_arg, -ENOMEM); 2933 return; 2934 } 2935 2936 bs_allocate_and_copy_cluster(blob, _channel, offset, op); 2937 } 2938 } 2939 } else { 2940 struct rw_iov_ctx *ctx; 2941 2942 ctx = calloc(1, sizeof(struct rw_iov_ctx) + iovcnt * sizeof(struct iovec)); 2943 if (ctx == NULL) { 2944 cb_fn(cb_arg, -ENOMEM); 2945 return; 2946 } 2947 2948 ctx->blob = blob; 2949 ctx->channel = _channel; 2950 ctx->cb_fn = cb_fn; 2951 ctx->cb_arg = cb_arg; 2952 ctx->read = read; 2953 ctx->orig_iov = iov; 2954 ctx->iovcnt = iovcnt; 2955 ctx->io_unit_offset = offset; 2956 ctx->io_units_remaining = length; 2957 ctx->io_units_done = 0; 2958 2959 rw_iov_split_next(ctx, 0); 2960 } 2961 } 2962 2963 static struct spdk_blob * 2964 blob_lookup(struct spdk_blob_store *bs, spdk_blob_id blobid) 2965 { 2966 struct spdk_blob *blob; 2967 2968 if (spdk_bit_array_get(bs->open_blobids, blobid) == 0) { 2969 return NULL; 2970 } 2971 2972 TAILQ_FOREACH(blob, &bs->blobs, link) { 2973 if (blob->id == blobid) { 2974 return blob; 2975 } 2976 } 2977 2978 return NULL; 2979 } 2980 2981 static void 2982 blob_get_snapshot_and_clone_entries(struct spdk_blob *blob, 2983 struct spdk_blob_list **snapshot_entry, struct spdk_blob_list **clone_entry) 2984 { 2985 assert(blob != NULL); 2986 *snapshot_entry = NULL; 2987 *clone_entry = NULL; 2988 2989 if (blob->parent_id == SPDK_BLOBID_INVALID) { 2990 return; 2991 } 2992 2993 TAILQ_FOREACH(*snapshot_entry, &blob->bs->snapshots, link) { 2994 if ((*snapshot_entry)->id == blob->parent_id) { 2995 break; 2996 } 2997 } 2998 2999 if (*snapshot_entry != NULL) { 3000 TAILQ_FOREACH(*clone_entry, &(*snapshot_entry)->clones, link) { 3001 if ((*clone_entry)->id == blob->id) { 3002 break; 3003 } 3004 } 3005 3006 assert(clone_entry != NULL); 3007 } 3008 } 3009 3010 static int 3011 bs_channel_create(void *io_device, void *ctx_buf) 3012 { 3013 struct spdk_blob_store *bs = io_device; 3014 struct spdk_bs_channel *channel = ctx_buf; 3015 struct spdk_bs_dev *dev; 3016 uint32_t max_ops = bs->max_channel_ops; 3017 uint32_t i; 3018 3019 dev = bs->dev; 3020 3021 channel->req_mem = calloc(max_ops, sizeof(struct spdk_bs_request_set)); 3022 if (!channel->req_mem) { 3023 return -1; 3024 } 3025 3026 TAILQ_INIT(&channel->reqs); 3027 3028 for (i = 0; i < max_ops; i++) { 3029 TAILQ_INSERT_TAIL(&channel->reqs, &channel->req_mem[i], link); 3030 } 3031 3032 channel->bs = bs; 3033 channel->dev = dev; 3034 channel->dev_channel = dev->create_channel(dev); 3035 3036 if (!channel->dev_channel) { 3037 SPDK_ERRLOG("Failed to create device channel.\n"); 3038 free(channel->req_mem); 3039 return -1; 3040 } 3041 3042 TAILQ_INIT(&channel->need_cluster_alloc); 3043 TAILQ_INIT(&channel->queued_io); 3044 3045 return 0; 3046 } 3047 3048 static void 3049 bs_channel_destroy(void *io_device, void *ctx_buf) 3050 { 3051 struct spdk_bs_channel *channel = ctx_buf; 3052 spdk_bs_user_op_t *op; 3053 3054 while (!TAILQ_EMPTY(&channel->need_cluster_alloc)) { 3055 op = TAILQ_FIRST(&channel->need_cluster_alloc); 3056 TAILQ_REMOVE(&channel->need_cluster_alloc, op, link); 3057 bs_user_op_abort(op); 3058 } 3059 3060 while (!TAILQ_EMPTY(&channel->queued_io)) { 3061 op = TAILQ_FIRST(&channel->queued_io); 3062 TAILQ_REMOVE(&channel->queued_io, op, link); 3063 bs_user_op_abort(op); 3064 } 3065 3066 free(channel->req_mem); 3067 channel->dev->destroy_channel(channel->dev, channel->dev_channel); 3068 } 3069 3070 static void 3071 bs_dev_destroy(void *io_device) 3072 { 3073 struct spdk_blob_store *bs = io_device; 3074 struct spdk_blob *blob, *blob_tmp; 3075 3076 bs->dev->destroy(bs->dev); 3077 3078 TAILQ_FOREACH_SAFE(blob, &bs->blobs, link, blob_tmp) { 3079 TAILQ_REMOVE(&bs->blobs, blob, link); 3080 spdk_bit_array_clear(bs->open_blobids, blob->id); 3081 blob_free(blob); 3082 } 3083 3084 pthread_mutex_destroy(&bs->used_clusters_mutex); 3085 3086 spdk_bit_array_free(&bs->open_blobids); 3087 spdk_bit_array_free(&bs->used_blobids); 3088 spdk_bit_array_free(&bs->used_md_pages); 3089 spdk_bit_pool_free(&bs->used_clusters); 3090 /* 3091 * If this function is called for any reason except a successful unload, 3092 * the unload_cpl type will be NONE and this will be a nop. 3093 */ 3094 bs_call_cpl(&bs->unload_cpl, bs->unload_err); 3095 3096 free(bs); 3097 } 3098 3099 static int 3100 bs_blob_list_add(struct spdk_blob *blob) 3101 { 3102 spdk_blob_id snapshot_id; 3103 struct spdk_blob_list *snapshot_entry = NULL; 3104 struct spdk_blob_list *clone_entry = NULL; 3105 3106 assert(blob != NULL); 3107 3108 snapshot_id = blob->parent_id; 3109 if (snapshot_id == SPDK_BLOBID_INVALID) { 3110 return 0; 3111 } 3112 3113 snapshot_entry = bs_get_snapshot_entry(blob->bs, snapshot_id); 3114 if (snapshot_entry == NULL) { 3115 /* Snapshot not found */ 3116 snapshot_entry = calloc(1, sizeof(struct spdk_blob_list)); 3117 if (snapshot_entry == NULL) { 3118 return -ENOMEM; 3119 } 3120 snapshot_entry->id = snapshot_id; 3121 TAILQ_INIT(&snapshot_entry->clones); 3122 TAILQ_INSERT_TAIL(&blob->bs->snapshots, snapshot_entry, link); 3123 } else { 3124 TAILQ_FOREACH(clone_entry, &snapshot_entry->clones, link) { 3125 if (clone_entry->id == blob->id) { 3126 break; 3127 } 3128 } 3129 } 3130 3131 if (clone_entry == NULL) { 3132 /* Clone not found */ 3133 clone_entry = calloc(1, sizeof(struct spdk_blob_list)); 3134 if (clone_entry == NULL) { 3135 return -ENOMEM; 3136 } 3137 clone_entry->id = blob->id; 3138 TAILQ_INIT(&clone_entry->clones); 3139 TAILQ_INSERT_TAIL(&snapshot_entry->clones, clone_entry, link); 3140 snapshot_entry->clone_count++; 3141 } 3142 3143 return 0; 3144 } 3145 3146 static void 3147 bs_blob_list_remove(struct spdk_blob *blob) 3148 { 3149 struct spdk_blob_list *snapshot_entry = NULL; 3150 struct spdk_blob_list *clone_entry = NULL; 3151 3152 blob_get_snapshot_and_clone_entries(blob, &snapshot_entry, &clone_entry); 3153 3154 if (snapshot_entry == NULL) { 3155 return; 3156 } 3157 3158 blob->parent_id = SPDK_BLOBID_INVALID; 3159 TAILQ_REMOVE(&snapshot_entry->clones, clone_entry, link); 3160 free(clone_entry); 3161 3162 snapshot_entry->clone_count--; 3163 } 3164 3165 static int 3166 bs_blob_list_free(struct spdk_blob_store *bs) 3167 { 3168 struct spdk_blob_list *snapshot_entry; 3169 struct spdk_blob_list *snapshot_entry_tmp; 3170 struct spdk_blob_list *clone_entry; 3171 struct spdk_blob_list *clone_entry_tmp; 3172 3173 TAILQ_FOREACH_SAFE(snapshot_entry, &bs->snapshots, link, snapshot_entry_tmp) { 3174 TAILQ_FOREACH_SAFE(clone_entry, &snapshot_entry->clones, link, clone_entry_tmp) { 3175 TAILQ_REMOVE(&snapshot_entry->clones, clone_entry, link); 3176 free(clone_entry); 3177 } 3178 TAILQ_REMOVE(&bs->snapshots, snapshot_entry, link); 3179 free(snapshot_entry); 3180 } 3181 3182 return 0; 3183 } 3184 3185 static void 3186 bs_free(struct spdk_blob_store *bs) 3187 { 3188 bs_blob_list_free(bs); 3189 3190 bs_unregister_md_thread(bs); 3191 spdk_io_device_unregister(bs, bs_dev_destroy); 3192 } 3193 3194 void 3195 spdk_bs_opts_init(struct spdk_bs_opts *opts, size_t opts_size) 3196 { 3197 3198 if (!opts) { 3199 SPDK_ERRLOG("opts should not be NULL\n"); 3200 return; 3201 } 3202 3203 if (!opts_size) { 3204 SPDK_ERRLOG("opts_size should not be zero value\n"); 3205 return; 3206 } 3207 3208 memset(opts, 0, opts_size); 3209 opts->opts_size = opts_size; 3210 3211 #define FIELD_OK(field) \ 3212 offsetof(struct spdk_bs_opts, field) + sizeof(opts->field) <= opts_size 3213 3214 #define SET_FIELD(field, value) \ 3215 if (FIELD_OK(field)) { \ 3216 opts->field = value; \ 3217 } \ 3218 3219 SET_FIELD(cluster_sz, SPDK_BLOB_OPTS_CLUSTER_SZ); 3220 SET_FIELD(num_md_pages, SPDK_BLOB_OPTS_NUM_MD_PAGES); 3221 SET_FIELD(max_md_ops, SPDK_BLOB_OPTS_NUM_MD_PAGES); 3222 SET_FIELD(max_channel_ops, SPDK_BLOB_OPTS_DEFAULT_CHANNEL_OPS); 3223 SET_FIELD(clear_method, BS_CLEAR_WITH_UNMAP); 3224 3225 if (FIELD_OK(bstype)) { 3226 memset(&opts->bstype, 0, sizeof(opts->bstype)); 3227 } 3228 3229 SET_FIELD(iter_cb_fn, NULL); 3230 SET_FIELD(iter_cb_arg, NULL); 3231 3232 #undef FIELD_OK 3233 #undef SET_FIELD 3234 } 3235 3236 static int 3237 bs_opts_verify(struct spdk_bs_opts *opts) 3238 { 3239 if (opts->cluster_sz == 0 || opts->num_md_pages == 0 || opts->max_md_ops == 0 || 3240 opts->max_channel_ops == 0) { 3241 SPDK_ERRLOG("Blobstore options cannot be set to 0\n"); 3242 return -1; 3243 } 3244 3245 return 0; 3246 } 3247 3248 /* START spdk_bs_load */ 3249 3250 /* spdk_bs_load_ctx is used for init, load, unload and dump code paths. */ 3251 3252 struct spdk_bs_load_ctx { 3253 struct spdk_blob_store *bs; 3254 struct spdk_bs_super_block *super; 3255 3256 struct spdk_bs_md_mask *mask; 3257 bool in_page_chain; 3258 uint32_t page_index; 3259 uint32_t cur_page; 3260 struct spdk_blob_md_page *page; 3261 3262 uint64_t num_extent_pages; 3263 uint32_t *extent_page_num; 3264 struct spdk_blob_md_page *extent_pages; 3265 struct spdk_bit_array *used_clusters; 3266 3267 spdk_bs_sequence_t *seq; 3268 spdk_blob_op_with_handle_complete iter_cb_fn; 3269 void *iter_cb_arg; 3270 struct spdk_blob *blob; 3271 spdk_blob_id blobid; 3272 3273 /* These fields are used in the spdk_bs_dump path. */ 3274 FILE *fp; 3275 spdk_bs_dump_print_xattr print_xattr_fn; 3276 char xattr_name[4096]; 3277 }; 3278 3279 static int 3280 bs_alloc(struct spdk_bs_dev *dev, struct spdk_bs_opts *opts, struct spdk_blob_store **_bs, 3281 struct spdk_bs_load_ctx **_ctx) 3282 { 3283 struct spdk_blob_store *bs; 3284 struct spdk_bs_load_ctx *ctx; 3285 uint64_t dev_size; 3286 int rc; 3287 3288 dev_size = dev->blocklen * dev->blockcnt; 3289 if (dev_size < opts->cluster_sz) { 3290 /* Device size cannot be smaller than cluster size of blobstore */ 3291 SPDK_INFOLOG(blob, "Device size %" PRIu64 " is smaller than cluster size %" PRIu32 "\n", 3292 dev_size, opts->cluster_sz); 3293 return -ENOSPC; 3294 } 3295 if (opts->cluster_sz < SPDK_BS_PAGE_SIZE) { 3296 /* Cluster size cannot be smaller than page size */ 3297 SPDK_ERRLOG("Cluster size %" PRIu32 " is smaller than page size %d\n", 3298 opts->cluster_sz, SPDK_BS_PAGE_SIZE); 3299 return -EINVAL; 3300 } 3301 bs = calloc(1, sizeof(struct spdk_blob_store)); 3302 if (!bs) { 3303 return -ENOMEM; 3304 } 3305 3306 ctx = calloc(1, sizeof(struct spdk_bs_load_ctx)); 3307 if (!ctx) { 3308 free(bs); 3309 return -ENOMEM; 3310 } 3311 3312 ctx->bs = bs; 3313 ctx->iter_cb_fn = opts->iter_cb_fn; 3314 ctx->iter_cb_arg = opts->iter_cb_arg; 3315 3316 ctx->super = spdk_zmalloc(sizeof(*ctx->super), 0x1000, NULL, 3317 SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA); 3318 if (!ctx->super) { 3319 free(ctx); 3320 free(bs); 3321 return -ENOMEM; 3322 } 3323 3324 TAILQ_INIT(&bs->blobs); 3325 TAILQ_INIT(&bs->snapshots); 3326 bs->dev = dev; 3327 bs->md_thread = spdk_get_thread(); 3328 assert(bs->md_thread != NULL); 3329 3330 /* 3331 * Do not use bs_lba_to_cluster() here since blockcnt may not be an 3332 * even multiple of the cluster size. 3333 */ 3334 bs->cluster_sz = opts->cluster_sz; 3335 bs->total_clusters = dev->blockcnt / (bs->cluster_sz / dev->blocklen); 3336 ctx->used_clusters = spdk_bit_array_create(bs->total_clusters); 3337 if (!ctx->used_clusters) { 3338 spdk_free(ctx->super); 3339 free(ctx); 3340 free(bs); 3341 return -ENOMEM; 3342 } 3343 3344 bs->pages_per_cluster = bs->cluster_sz / SPDK_BS_PAGE_SIZE; 3345 if (spdk_u32_is_pow2(bs->pages_per_cluster)) { 3346 bs->pages_per_cluster_shift = spdk_u32log2(bs->pages_per_cluster); 3347 } 3348 bs->num_free_clusters = bs->total_clusters; 3349 bs->io_unit_size = dev->blocklen; 3350 3351 bs->max_channel_ops = opts->max_channel_ops; 3352 bs->super_blob = SPDK_BLOBID_INVALID; 3353 memcpy(&bs->bstype, &opts->bstype, sizeof(opts->bstype)); 3354 3355 /* The metadata is assumed to be at least 1 page */ 3356 bs->used_md_pages = spdk_bit_array_create(1); 3357 bs->used_blobids = spdk_bit_array_create(0); 3358 bs->open_blobids = spdk_bit_array_create(0); 3359 3360 pthread_mutex_init(&bs->used_clusters_mutex, NULL); 3361 3362 spdk_io_device_register(bs, bs_channel_create, bs_channel_destroy, 3363 sizeof(struct spdk_bs_channel), "blobstore"); 3364 rc = bs_register_md_thread(bs); 3365 if (rc == -1) { 3366 spdk_io_device_unregister(bs, NULL); 3367 pthread_mutex_destroy(&bs->used_clusters_mutex); 3368 spdk_bit_array_free(&bs->open_blobids); 3369 spdk_bit_array_free(&bs->used_blobids); 3370 spdk_bit_array_free(&bs->used_md_pages); 3371 spdk_bit_array_free(&ctx->used_clusters); 3372 spdk_free(ctx->super); 3373 free(ctx); 3374 free(bs); 3375 /* FIXME: this is a lie but don't know how to get a proper error code here */ 3376 return -ENOMEM; 3377 } 3378 3379 *_ctx = ctx; 3380 *_bs = bs; 3381 return 0; 3382 } 3383 3384 static void 3385 bs_load_ctx_fail(struct spdk_bs_load_ctx *ctx, int bserrno) 3386 { 3387 assert(bserrno != 0); 3388 3389 spdk_free(ctx->super); 3390 bs_sequence_finish(ctx->seq, bserrno); 3391 bs_free(ctx->bs); 3392 spdk_bit_array_free(&ctx->used_clusters); 3393 free(ctx); 3394 } 3395 3396 static void 3397 bs_write_super(spdk_bs_sequence_t *seq, struct spdk_blob_store *bs, 3398 struct spdk_bs_super_block *super, spdk_bs_sequence_cpl cb_fn, void *cb_arg) 3399 { 3400 /* Update the values in the super block */ 3401 super->super_blob = bs->super_blob; 3402 memcpy(&super->bstype, &bs->bstype, sizeof(bs->bstype)); 3403 super->crc = blob_md_page_calc_crc(super); 3404 bs_sequence_write_dev(seq, super, bs_page_to_lba(bs, 0), 3405 bs_byte_to_lba(bs, sizeof(*super)), 3406 cb_fn, cb_arg); 3407 } 3408 3409 static void 3410 bs_write_used_clusters(spdk_bs_sequence_t *seq, void *arg, spdk_bs_sequence_cpl cb_fn) 3411 { 3412 struct spdk_bs_load_ctx *ctx = arg; 3413 uint64_t mask_size, lba, lba_count; 3414 3415 /* Write out the used clusters mask */ 3416 mask_size = ctx->super->used_cluster_mask_len * SPDK_BS_PAGE_SIZE; 3417 ctx->mask = spdk_zmalloc(mask_size, 0x1000, NULL, 3418 SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA); 3419 if (!ctx->mask) { 3420 bs_load_ctx_fail(ctx, -ENOMEM); 3421 return; 3422 } 3423 3424 ctx->mask->type = SPDK_MD_MASK_TYPE_USED_CLUSTERS; 3425 ctx->mask->length = ctx->bs->total_clusters; 3426 /* We could get here through the normal unload path, or through dirty 3427 * shutdown recovery. For the normal unload path, we use the mask from 3428 * the bit pool. For dirty shutdown recovery, we don't have a bit pool yet - 3429 * only the bit array from the load ctx. 3430 */ 3431 if (ctx->bs->used_clusters) { 3432 assert(ctx->mask->length == spdk_bit_pool_capacity(ctx->bs->used_clusters)); 3433 spdk_bit_pool_store_mask(ctx->bs->used_clusters, ctx->mask->mask); 3434 } else { 3435 assert(ctx->mask->length == spdk_bit_array_capacity(ctx->used_clusters)); 3436 spdk_bit_array_store_mask(ctx->used_clusters, ctx->mask->mask); 3437 } 3438 lba = bs_page_to_lba(ctx->bs, ctx->super->used_cluster_mask_start); 3439 lba_count = bs_page_to_lba(ctx->bs, ctx->super->used_cluster_mask_len); 3440 bs_sequence_write_dev(seq, ctx->mask, lba, lba_count, cb_fn, arg); 3441 } 3442 3443 static void 3444 bs_write_used_md(spdk_bs_sequence_t *seq, void *arg, spdk_bs_sequence_cpl cb_fn) 3445 { 3446 struct spdk_bs_load_ctx *ctx = arg; 3447 uint64_t mask_size, lba, lba_count; 3448 3449 mask_size = ctx->super->used_page_mask_len * SPDK_BS_PAGE_SIZE; 3450 ctx->mask = spdk_zmalloc(mask_size, 0x1000, NULL, 3451 SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA); 3452 if (!ctx->mask) { 3453 bs_load_ctx_fail(ctx, -ENOMEM); 3454 return; 3455 } 3456 3457 ctx->mask->type = SPDK_MD_MASK_TYPE_USED_PAGES; 3458 ctx->mask->length = ctx->super->md_len; 3459 assert(ctx->mask->length == spdk_bit_array_capacity(ctx->bs->used_md_pages)); 3460 3461 spdk_bit_array_store_mask(ctx->bs->used_md_pages, ctx->mask->mask); 3462 lba = bs_page_to_lba(ctx->bs, ctx->super->used_page_mask_start); 3463 lba_count = bs_page_to_lba(ctx->bs, ctx->super->used_page_mask_len); 3464 bs_sequence_write_dev(seq, ctx->mask, lba, lba_count, cb_fn, arg); 3465 } 3466 3467 static void 3468 bs_write_used_blobids(spdk_bs_sequence_t *seq, void *arg, spdk_bs_sequence_cpl cb_fn) 3469 { 3470 struct spdk_bs_load_ctx *ctx = arg; 3471 uint64_t mask_size, lba, lba_count; 3472 3473 if (ctx->super->used_blobid_mask_len == 0) { 3474 /* 3475 * This is a pre-v3 on-disk format where the blobid mask does not get 3476 * written to disk. 3477 */ 3478 cb_fn(seq, arg, 0); 3479 return; 3480 } 3481 3482 mask_size = ctx->super->used_blobid_mask_len * SPDK_BS_PAGE_SIZE; 3483 ctx->mask = spdk_zmalloc(mask_size, 0x1000, NULL, SPDK_ENV_SOCKET_ID_ANY, 3484 SPDK_MALLOC_DMA); 3485 if (!ctx->mask) { 3486 bs_load_ctx_fail(ctx, -ENOMEM); 3487 return; 3488 } 3489 3490 ctx->mask->type = SPDK_MD_MASK_TYPE_USED_BLOBIDS; 3491 ctx->mask->length = ctx->super->md_len; 3492 assert(ctx->mask->length == spdk_bit_array_capacity(ctx->bs->used_blobids)); 3493 3494 spdk_bit_array_store_mask(ctx->bs->used_blobids, ctx->mask->mask); 3495 lba = bs_page_to_lba(ctx->bs, ctx->super->used_blobid_mask_start); 3496 lba_count = bs_page_to_lba(ctx->bs, ctx->super->used_blobid_mask_len); 3497 bs_sequence_write_dev(seq, ctx->mask, lba, lba_count, cb_fn, arg); 3498 } 3499 3500 static void 3501 blob_set_thin_provision(struct spdk_blob *blob) 3502 { 3503 blob_verify_md_op(blob); 3504 blob->invalid_flags |= SPDK_BLOB_THIN_PROV; 3505 blob->state = SPDK_BLOB_STATE_DIRTY; 3506 } 3507 3508 static void 3509 blob_set_clear_method(struct spdk_blob *blob, enum blob_clear_method clear_method) 3510 { 3511 blob_verify_md_op(blob); 3512 blob->clear_method = clear_method; 3513 blob->md_ro_flags |= (clear_method << SPDK_BLOB_CLEAR_METHOD_SHIFT); 3514 blob->state = SPDK_BLOB_STATE_DIRTY; 3515 } 3516 3517 static void bs_load_iter(void *arg, struct spdk_blob *blob, int bserrno); 3518 3519 static void 3520 bs_delete_corrupted_blob_cpl(void *cb_arg, int bserrno) 3521 { 3522 struct spdk_bs_load_ctx *ctx = cb_arg; 3523 spdk_blob_id id; 3524 int64_t page_num; 3525 3526 /* Iterate to next blob (we can't use spdk_bs_iter_next function as our 3527 * last blob has been removed */ 3528 page_num = bs_blobid_to_page(ctx->blobid); 3529 page_num++; 3530 page_num = spdk_bit_array_find_first_set(ctx->bs->used_blobids, page_num); 3531 if (page_num >= spdk_bit_array_capacity(ctx->bs->used_blobids)) { 3532 bs_load_iter(ctx, NULL, -ENOENT); 3533 return; 3534 } 3535 3536 id = bs_page_to_blobid(page_num); 3537 3538 spdk_bs_open_blob(ctx->bs, id, bs_load_iter, ctx); 3539 } 3540 3541 static void 3542 bs_delete_corrupted_close_cb(void *cb_arg, int bserrno) 3543 { 3544 struct spdk_bs_load_ctx *ctx = cb_arg; 3545 3546 if (bserrno != 0) { 3547 SPDK_ERRLOG("Failed to close corrupted blob\n"); 3548 spdk_bs_iter_next(ctx->bs, ctx->blob, bs_load_iter, ctx); 3549 return; 3550 } 3551 3552 spdk_bs_delete_blob(ctx->bs, ctx->blobid, bs_delete_corrupted_blob_cpl, ctx); 3553 } 3554 3555 static void 3556 bs_delete_corrupted_blob(void *cb_arg, int bserrno) 3557 { 3558 struct spdk_bs_load_ctx *ctx = cb_arg; 3559 uint64_t i; 3560 3561 if (bserrno != 0) { 3562 SPDK_ERRLOG("Failed to close clone of a corrupted blob\n"); 3563 spdk_bs_iter_next(ctx->bs, ctx->blob, bs_load_iter, ctx); 3564 return; 3565 } 3566 3567 /* Snapshot and clone have the same copy of cluster map and extent pages 3568 * at this point. Let's clear both for snpashot now, 3569 * so that it won't be cleared for clone later when we remove snapshot. 3570 * Also set thin provision to pass data corruption check */ 3571 for (i = 0; i < ctx->blob->active.num_clusters; i++) { 3572 ctx->blob->active.clusters[i] = 0; 3573 } 3574 for (i = 0; i < ctx->blob->active.num_extent_pages; i++) { 3575 ctx->blob->active.extent_pages[i] = 0; 3576 } 3577 3578 ctx->blob->md_ro = false; 3579 3580 blob_set_thin_provision(ctx->blob); 3581 3582 ctx->blobid = ctx->blob->id; 3583 3584 spdk_blob_close(ctx->blob, bs_delete_corrupted_close_cb, ctx); 3585 } 3586 3587 static void 3588 bs_update_corrupted_blob(void *cb_arg, int bserrno) 3589 { 3590 struct spdk_bs_load_ctx *ctx = cb_arg; 3591 3592 if (bserrno != 0) { 3593 SPDK_ERRLOG("Failed to close clone of a corrupted blob\n"); 3594 spdk_bs_iter_next(ctx->bs, ctx->blob, bs_load_iter, ctx); 3595 return; 3596 } 3597 3598 ctx->blob->md_ro = false; 3599 blob_remove_xattr(ctx->blob, SNAPSHOT_PENDING_REMOVAL, true); 3600 blob_remove_xattr(ctx->blob, SNAPSHOT_IN_PROGRESS, true); 3601 spdk_blob_set_read_only(ctx->blob); 3602 3603 if (ctx->iter_cb_fn) { 3604 ctx->iter_cb_fn(ctx->iter_cb_arg, ctx->blob, 0); 3605 } 3606 bs_blob_list_add(ctx->blob); 3607 3608 spdk_bs_iter_next(ctx->bs, ctx->blob, bs_load_iter, ctx); 3609 } 3610 3611 static void 3612 bs_examine_clone(void *cb_arg, struct spdk_blob *blob, int bserrno) 3613 { 3614 struct spdk_bs_load_ctx *ctx = cb_arg; 3615 3616 if (bserrno != 0) { 3617 SPDK_ERRLOG("Failed to open clone of a corrupted blob\n"); 3618 spdk_bs_iter_next(ctx->bs, ctx->blob, bs_load_iter, ctx); 3619 return; 3620 } 3621 3622 if (blob->parent_id == ctx->blob->id) { 3623 /* Power failure occured before updating clone (snapshot delete case) 3624 * or after updating clone (creating snapshot case) - keep snapshot */ 3625 spdk_blob_close(blob, bs_update_corrupted_blob, ctx); 3626 } else { 3627 /* Power failure occured after updating clone (snapshot delete case) 3628 * or before updating clone (creating snapshot case) - remove snapshot */ 3629 spdk_blob_close(blob, bs_delete_corrupted_blob, ctx); 3630 } 3631 } 3632 3633 static void 3634 bs_load_iter(void *arg, struct spdk_blob *blob, int bserrno) 3635 { 3636 struct spdk_bs_load_ctx *ctx = arg; 3637 const void *value; 3638 size_t len; 3639 int rc = 0; 3640 3641 if (bserrno == 0) { 3642 /* Examine blob if it is corrupted after power failure. Fix 3643 * the ones that can be fixed and remove any other corrupted 3644 * ones. If it is not corrupted just process it */ 3645 rc = blob_get_xattr_value(blob, SNAPSHOT_PENDING_REMOVAL, &value, &len, true); 3646 if (rc != 0) { 3647 rc = blob_get_xattr_value(blob, SNAPSHOT_IN_PROGRESS, &value, &len, true); 3648 if (rc != 0) { 3649 /* Not corrupted - process it and continue with iterating through blobs */ 3650 if (ctx->iter_cb_fn) { 3651 ctx->iter_cb_fn(ctx->iter_cb_arg, blob, 0); 3652 } 3653 bs_blob_list_add(blob); 3654 spdk_bs_iter_next(ctx->bs, blob, bs_load_iter, ctx); 3655 return; 3656 } 3657 3658 } 3659 3660 assert(len == sizeof(spdk_blob_id)); 3661 3662 ctx->blob = blob; 3663 3664 /* Open clone to check if we are able to fix this blob or should we remove it */ 3665 spdk_bs_open_blob(ctx->bs, *(spdk_blob_id *)value, bs_examine_clone, ctx); 3666 return; 3667 } else if (bserrno == -ENOENT) { 3668 bserrno = 0; 3669 } else { 3670 /* 3671 * This case needs to be looked at further. Same problem 3672 * exists with applications that rely on explicit blob 3673 * iteration. We should just skip the blob that failed 3674 * to load and continue on to the next one. 3675 */ 3676 SPDK_ERRLOG("Error in iterating blobs\n"); 3677 } 3678 3679 ctx->iter_cb_fn = NULL; 3680 3681 spdk_free(ctx->super); 3682 spdk_free(ctx->mask); 3683 bs_sequence_finish(ctx->seq, bserrno); 3684 free(ctx); 3685 } 3686 3687 static void 3688 bs_load_complete(struct spdk_bs_load_ctx *ctx) 3689 { 3690 ctx->bs->used_clusters = spdk_bit_pool_create_from_array(ctx->used_clusters); 3691 spdk_bs_iter_first(ctx->bs, bs_load_iter, ctx); 3692 } 3693 3694 static void 3695 bs_load_used_blobids_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 3696 { 3697 struct spdk_bs_load_ctx *ctx = cb_arg; 3698 int rc; 3699 3700 /* The type must be correct */ 3701 assert(ctx->mask->type == SPDK_MD_MASK_TYPE_USED_BLOBIDS); 3702 3703 /* The length of the mask (in bits) must not be greater than 3704 * the length of the buffer (converted to bits) */ 3705 assert(ctx->mask->length <= (ctx->super->used_blobid_mask_len * SPDK_BS_PAGE_SIZE * 8)); 3706 3707 /* The length of the mask must be exactly equal to the size 3708 * (in pages) of the metadata region */ 3709 assert(ctx->mask->length == ctx->super->md_len); 3710 3711 rc = spdk_bit_array_resize(&ctx->bs->used_blobids, ctx->mask->length); 3712 if (rc < 0) { 3713 spdk_free(ctx->mask); 3714 bs_load_ctx_fail(ctx, rc); 3715 return; 3716 } 3717 3718 spdk_bit_array_load_mask(ctx->bs->used_blobids, ctx->mask->mask); 3719 bs_load_complete(ctx); 3720 } 3721 3722 static void 3723 bs_load_used_clusters_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 3724 { 3725 struct spdk_bs_load_ctx *ctx = cb_arg; 3726 uint64_t lba, lba_count, mask_size; 3727 int rc; 3728 3729 if (bserrno != 0) { 3730 bs_load_ctx_fail(ctx, bserrno); 3731 return; 3732 } 3733 3734 /* The type must be correct */ 3735 assert(ctx->mask->type == SPDK_MD_MASK_TYPE_USED_CLUSTERS); 3736 /* The length of the mask (in bits) must not be greater than the length of the buffer (converted to bits) */ 3737 assert(ctx->mask->length <= (ctx->super->used_cluster_mask_len * sizeof( 3738 struct spdk_blob_md_page) * 8)); 3739 /* The length of the mask must be exactly equal to the total number of clusters */ 3740 assert(ctx->mask->length == ctx->bs->total_clusters); 3741 3742 rc = spdk_bit_array_resize(&ctx->used_clusters, ctx->mask->length); 3743 if (rc < 0) { 3744 spdk_free(ctx->mask); 3745 bs_load_ctx_fail(ctx, rc); 3746 return; 3747 } 3748 3749 spdk_bit_array_load_mask(ctx->used_clusters, ctx->mask->mask); 3750 ctx->bs->num_free_clusters = spdk_bit_array_count_clear(ctx->used_clusters); 3751 assert(ctx->bs->num_free_clusters <= ctx->bs->total_clusters); 3752 3753 spdk_free(ctx->mask); 3754 3755 /* Read the used blobids mask */ 3756 mask_size = ctx->super->used_blobid_mask_len * SPDK_BS_PAGE_SIZE; 3757 ctx->mask = spdk_zmalloc(mask_size, 0x1000, NULL, SPDK_ENV_SOCKET_ID_ANY, 3758 SPDK_MALLOC_DMA); 3759 if (!ctx->mask) { 3760 bs_load_ctx_fail(ctx, -ENOMEM); 3761 return; 3762 } 3763 lba = bs_page_to_lba(ctx->bs, ctx->super->used_blobid_mask_start); 3764 lba_count = bs_page_to_lba(ctx->bs, ctx->super->used_blobid_mask_len); 3765 bs_sequence_read_dev(seq, ctx->mask, lba, lba_count, 3766 bs_load_used_blobids_cpl, ctx); 3767 } 3768 3769 static void 3770 bs_load_used_pages_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 3771 { 3772 struct spdk_bs_load_ctx *ctx = cb_arg; 3773 uint64_t lba, lba_count, mask_size; 3774 int rc; 3775 3776 if (bserrno != 0) { 3777 bs_load_ctx_fail(ctx, bserrno); 3778 return; 3779 } 3780 3781 /* The type must be correct */ 3782 assert(ctx->mask->type == SPDK_MD_MASK_TYPE_USED_PAGES); 3783 /* The length of the mask (in bits) must not be greater than the length of the buffer (converted to bits) */ 3784 assert(ctx->mask->length <= (ctx->super->used_page_mask_len * SPDK_BS_PAGE_SIZE * 3785 8)); 3786 /* The length of the mask must be exactly equal to the size (in pages) of the metadata region */ 3787 if (ctx->mask->length != ctx->super->md_len) { 3788 SPDK_ERRLOG("mismatched md_len in used_pages mask: " 3789 "mask->length=%" PRIu32 " super->md_len=%" PRIu32 "\n", 3790 ctx->mask->length, ctx->super->md_len); 3791 assert(false); 3792 } 3793 3794 rc = spdk_bit_array_resize(&ctx->bs->used_md_pages, ctx->mask->length); 3795 if (rc < 0) { 3796 spdk_free(ctx->mask); 3797 bs_load_ctx_fail(ctx, rc); 3798 return; 3799 } 3800 3801 spdk_bit_array_load_mask(ctx->bs->used_md_pages, ctx->mask->mask); 3802 spdk_free(ctx->mask); 3803 3804 /* Read the used clusters mask */ 3805 mask_size = ctx->super->used_cluster_mask_len * SPDK_BS_PAGE_SIZE; 3806 ctx->mask = spdk_zmalloc(mask_size, 0x1000, NULL, SPDK_ENV_SOCKET_ID_ANY, 3807 SPDK_MALLOC_DMA); 3808 if (!ctx->mask) { 3809 bs_load_ctx_fail(ctx, -ENOMEM); 3810 return; 3811 } 3812 lba = bs_page_to_lba(ctx->bs, ctx->super->used_cluster_mask_start); 3813 lba_count = bs_page_to_lba(ctx->bs, ctx->super->used_cluster_mask_len); 3814 bs_sequence_read_dev(seq, ctx->mask, lba, lba_count, 3815 bs_load_used_clusters_cpl, ctx); 3816 } 3817 3818 static void 3819 bs_load_read_used_pages(struct spdk_bs_load_ctx *ctx) 3820 { 3821 uint64_t lba, lba_count, mask_size; 3822 3823 /* Read the used pages mask */ 3824 mask_size = ctx->super->used_page_mask_len * SPDK_BS_PAGE_SIZE; 3825 ctx->mask = spdk_zmalloc(mask_size, 0x1000, NULL, 3826 SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA); 3827 if (!ctx->mask) { 3828 bs_load_ctx_fail(ctx, -ENOMEM); 3829 return; 3830 } 3831 3832 lba = bs_page_to_lba(ctx->bs, ctx->super->used_page_mask_start); 3833 lba_count = bs_page_to_lba(ctx->bs, ctx->super->used_page_mask_len); 3834 bs_sequence_read_dev(ctx->seq, ctx->mask, lba, lba_count, 3835 bs_load_used_pages_cpl, ctx); 3836 } 3837 3838 static int 3839 bs_load_replay_md_parse_page(struct spdk_bs_load_ctx *ctx, struct spdk_blob_md_page *page) 3840 { 3841 struct spdk_blob_store *bs = ctx->bs; 3842 struct spdk_blob_md_descriptor *desc; 3843 size_t cur_desc = 0; 3844 3845 desc = (struct spdk_blob_md_descriptor *)page->descriptors; 3846 while (cur_desc < sizeof(page->descriptors)) { 3847 if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_PADDING) { 3848 if (desc->length == 0) { 3849 /* If padding and length are 0, this terminates the page */ 3850 break; 3851 } 3852 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_EXTENT_RLE) { 3853 struct spdk_blob_md_descriptor_extent_rle *desc_extent_rle; 3854 unsigned int i, j; 3855 unsigned int cluster_count = 0; 3856 uint32_t cluster_idx; 3857 3858 desc_extent_rle = (struct spdk_blob_md_descriptor_extent_rle *)desc; 3859 3860 for (i = 0; i < desc_extent_rle->length / sizeof(desc_extent_rle->extents[0]); i++) { 3861 for (j = 0; j < desc_extent_rle->extents[i].length; j++) { 3862 cluster_idx = desc_extent_rle->extents[i].cluster_idx; 3863 /* 3864 * cluster_idx = 0 means an unallocated cluster - don't mark that 3865 * in the used cluster map. 3866 */ 3867 if (cluster_idx != 0) { 3868 spdk_bit_array_set(ctx->used_clusters, cluster_idx + j); 3869 if (bs->num_free_clusters == 0) { 3870 return -ENOSPC; 3871 } 3872 bs->num_free_clusters--; 3873 } 3874 cluster_count++; 3875 } 3876 } 3877 if (cluster_count == 0) { 3878 return -EINVAL; 3879 } 3880 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_EXTENT_PAGE) { 3881 struct spdk_blob_md_descriptor_extent_page *desc_extent; 3882 uint32_t i; 3883 uint32_t cluster_count = 0; 3884 uint32_t cluster_idx; 3885 size_t cluster_idx_length; 3886 3887 desc_extent = (struct spdk_blob_md_descriptor_extent_page *)desc; 3888 cluster_idx_length = desc_extent->length - sizeof(desc_extent->start_cluster_idx); 3889 3890 if (desc_extent->length <= sizeof(desc_extent->start_cluster_idx) || 3891 (cluster_idx_length % sizeof(desc_extent->cluster_idx[0]) != 0)) { 3892 return -EINVAL; 3893 } 3894 3895 for (i = 0; i < cluster_idx_length / sizeof(desc_extent->cluster_idx[0]); i++) { 3896 cluster_idx = desc_extent->cluster_idx[i]; 3897 /* 3898 * cluster_idx = 0 means an unallocated cluster - don't mark that 3899 * in the used cluster map. 3900 */ 3901 if (cluster_idx != 0) { 3902 if (cluster_idx < desc_extent->start_cluster_idx && 3903 cluster_idx >= desc_extent->start_cluster_idx + cluster_count) { 3904 return -EINVAL; 3905 } 3906 spdk_bit_array_set(ctx->used_clusters, cluster_idx); 3907 if (bs->num_free_clusters == 0) { 3908 return -ENOSPC; 3909 } 3910 bs->num_free_clusters--; 3911 } 3912 cluster_count++; 3913 } 3914 3915 if (cluster_count == 0) { 3916 return -EINVAL; 3917 } 3918 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_XATTR) { 3919 /* Skip this item */ 3920 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_XATTR_INTERNAL) { 3921 /* Skip this item */ 3922 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_FLAGS) { 3923 /* Skip this item */ 3924 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_EXTENT_TABLE) { 3925 struct spdk_blob_md_descriptor_extent_table *desc_extent_table; 3926 uint32_t num_extent_pages = ctx->num_extent_pages; 3927 uint32_t i; 3928 size_t extent_pages_length; 3929 void *tmp; 3930 3931 desc_extent_table = (struct spdk_blob_md_descriptor_extent_table *)desc; 3932 extent_pages_length = desc_extent_table->length - sizeof(desc_extent_table->num_clusters); 3933 3934 if (desc_extent_table->length == 0 || 3935 (extent_pages_length % sizeof(desc_extent_table->extent_page[0]) != 0)) { 3936 return -EINVAL; 3937 } 3938 3939 for (i = 0; i < extent_pages_length / sizeof(desc_extent_table->extent_page[0]); i++) { 3940 if (desc_extent_table->extent_page[i].page_idx != 0) { 3941 if (desc_extent_table->extent_page[i].num_pages != 1) { 3942 return -EINVAL; 3943 } 3944 num_extent_pages += 1; 3945 } 3946 } 3947 3948 if (num_extent_pages > 0) { 3949 tmp = realloc(ctx->extent_page_num, num_extent_pages * sizeof(uint32_t)); 3950 if (tmp == NULL) { 3951 return -ENOMEM; 3952 } 3953 ctx->extent_page_num = tmp; 3954 3955 /* Extent table entries contain md page numbers for extent pages. 3956 * Zeroes represent unallocated extent pages, those are run-length-encoded. 3957 */ 3958 for (i = 0; i < extent_pages_length / sizeof(desc_extent_table->extent_page[0]); i++) { 3959 if (desc_extent_table->extent_page[i].page_idx != 0) { 3960 ctx->extent_page_num[ctx->num_extent_pages] = desc_extent_table->extent_page[i].page_idx; 3961 ctx->num_extent_pages += 1; 3962 } 3963 } 3964 } 3965 } else { 3966 /* Error */ 3967 return -EINVAL; 3968 } 3969 /* Advance to the next descriptor */ 3970 cur_desc += sizeof(*desc) + desc->length; 3971 if (cur_desc + sizeof(*desc) > sizeof(page->descriptors)) { 3972 break; 3973 } 3974 desc = (struct spdk_blob_md_descriptor *)((uintptr_t)page->descriptors + cur_desc); 3975 } 3976 return 0; 3977 } 3978 3979 static bool bs_load_cur_extent_page_valid(struct spdk_blob_md_page *page) 3980 { 3981 uint32_t crc; 3982 struct spdk_blob_md_descriptor *desc = (struct spdk_blob_md_descriptor *)page->descriptors; 3983 size_t desc_len; 3984 3985 crc = blob_md_page_calc_crc(page); 3986 if (crc != page->crc) { 3987 return false; 3988 } 3989 3990 /* Extent page should always be of sequence num 0. */ 3991 if (page->sequence_num != 0) { 3992 return false; 3993 } 3994 3995 /* Descriptor type must be EXTENT_PAGE. */ 3996 if (desc->type != SPDK_MD_DESCRIPTOR_TYPE_EXTENT_PAGE) { 3997 return false; 3998 } 3999 4000 /* Descriptor length cannot exceed the page. */ 4001 desc_len = sizeof(*desc) + desc->length; 4002 if (desc_len > sizeof(page->descriptors)) { 4003 return false; 4004 } 4005 4006 /* It has to be the only descriptor in the page. */ 4007 if (desc_len + sizeof(*desc) <= sizeof(page->descriptors)) { 4008 desc = (struct spdk_blob_md_descriptor *)((uintptr_t)page->descriptors + desc_len); 4009 if (desc->length != 0) { 4010 return false; 4011 } 4012 } 4013 4014 return true; 4015 } 4016 4017 static bool bs_load_cur_md_page_valid(struct spdk_bs_load_ctx *ctx) 4018 { 4019 uint32_t crc; 4020 struct spdk_blob_md_page *page = ctx->page; 4021 4022 crc = blob_md_page_calc_crc(page); 4023 if (crc != page->crc) { 4024 return false; 4025 } 4026 4027 /* First page of a sequence should match the blobid. */ 4028 if (page->sequence_num == 0 && 4029 bs_page_to_blobid(ctx->cur_page) != page->id) { 4030 return false; 4031 } 4032 assert(bs_load_cur_extent_page_valid(page) == false); 4033 4034 return true; 4035 } 4036 4037 static void 4038 bs_load_replay_cur_md_page(struct spdk_bs_load_ctx *ctx); 4039 4040 static void 4041 bs_load_write_used_clusters_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 4042 { 4043 struct spdk_bs_load_ctx *ctx = cb_arg; 4044 4045 if (bserrno != 0) { 4046 bs_load_ctx_fail(ctx, bserrno); 4047 return; 4048 } 4049 4050 bs_load_complete(ctx); 4051 } 4052 4053 static void 4054 bs_load_write_used_blobids_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 4055 { 4056 struct spdk_bs_load_ctx *ctx = cb_arg; 4057 4058 spdk_free(ctx->mask); 4059 ctx->mask = NULL; 4060 4061 if (bserrno != 0) { 4062 bs_load_ctx_fail(ctx, bserrno); 4063 return; 4064 } 4065 4066 bs_write_used_clusters(seq, ctx, bs_load_write_used_clusters_cpl); 4067 } 4068 4069 static void 4070 bs_load_write_used_pages_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 4071 { 4072 struct spdk_bs_load_ctx *ctx = cb_arg; 4073 4074 spdk_free(ctx->mask); 4075 ctx->mask = NULL; 4076 4077 if (bserrno != 0) { 4078 bs_load_ctx_fail(ctx, bserrno); 4079 return; 4080 } 4081 4082 bs_write_used_blobids(seq, ctx, bs_load_write_used_blobids_cpl); 4083 } 4084 4085 static void 4086 bs_load_write_used_md(struct spdk_bs_load_ctx *ctx) 4087 { 4088 bs_write_used_md(ctx->seq, ctx, bs_load_write_used_pages_cpl); 4089 } 4090 4091 static void 4092 bs_load_replay_md_chain_cpl(struct spdk_bs_load_ctx *ctx) 4093 { 4094 uint64_t num_md_clusters; 4095 uint64_t i; 4096 4097 ctx->in_page_chain = false; 4098 4099 do { 4100 ctx->page_index++; 4101 } while (spdk_bit_array_get(ctx->bs->used_md_pages, ctx->page_index) == true); 4102 4103 if (ctx->page_index < ctx->super->md_len) { 4104 ctx->cur_page = ctx->page_index; 4105 bs_load_replay_cur_md_page(ctx); 4106 } else { 4107 /* Claim all of the clusters used by the metadata */ 4108 num_md_clusters = spdk_divide_round_up(ctx->super->md_len, ctx->bs->pages_per_cluster); 4109 for (i = 0; i < num_md_clusters; i++) { 4110 spdk_bit_array_set(ctx->used_clusters, i); 4111 } 4112 ctx->bs->num_free_clusters -= num_md_clusters; 4113 spdk_free(ctx->page); 4114 bs_load_write_used_md(ctx); 4115 } 4116 } 4117 4118 static void 4119 bs_load_replay_extent_page_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 4120 { 4121 struct spdk_bs_load_ctx *ctx = cb_arg; 4122 uint32_t page_num; 4123 uint64_t i; 4124 4125 if (bserrno != 0) { 4126 spdk_free(ctx->extent_pages); 4127 bs_load_ctx_fail(ctx, bserrno); 4128 return; 4129 } 4130 4131 for (i = 0; i < ctx->num_extent_pages; i++) { 4132 /* Extent pages are only read when present within in chain md. 4133 * Integrity of md is not right if that page was not a valid extent page. */ 4134 if (bs_load_cur_extent_page_valid(&ctx->extent_pages[i]) != true) { 4135 spdk_free(ctx->extent_pages); 4136 bs_load_ctx_fail(ctx, -EILSEQ); 4137 return; 4138 } 4139 4140 page_num = ctx->extent_page_num[i]; 4141 spdk_bit_array_set(ctx->bs->used_md_pages, page_num); 4142 if (bs_load_replay_md_parse_page(ctx, &ctx->extent_pages[i])) { 4143 spdk_free(ctx->extent_pages); 4144 bs_load_ctx_fail(ctx, -EILSEQ); 4145 return; 4146 } 4147 } 4148 4149 spdk_free(ctx->extent_pages); 4150 free(ctx->extent_page_num); 4151 ctx->extent_page_num = NULL; 4152 ctx->num_extent_pages = 0; 4153 4154 bs_load_replay_md_chain_cpl(ctx); 4155 } 4156 4157 static void 4158 bs_load_replay_extent_pages(struct spdk_bs_load_ctx *ctx) 4159 { 4160 spdk_bs_batch_t *batch; 4161 uint32_t page; 4162 uint64_t lba; 4163 uint64_t i; 4164 4165 ctx->extent_pages = spdk_zmalloc(SPDK_BS_PAGE_SIZE * ctx->num_extent_pages, 0, 4166 NULL, SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA); 4167 if (!ctx->extent_pages) { 4168 bs_load_ctx_fail(ctx, -ENOMEM); 4169 return; 4170 } 4171 4172 batch = bs_sequence_to_batch(ctx->seq, bs_load_replay_extent_page_cpl, ctx); 4173 4174 for (i = 0; i < ctx->num_extent_pages; i++) { 4175 page = ctx->extent_page_num[i]; 4176 assert(page < ctx->super->md_len); 4177 lba = bs_md_page_to_lba(ctx->bs, page); 4178 bs_batch_read_dev(batch, &ctx->extent_pages[i], lba, 4179 bs_byte_to_lba(ctx->bs, SPDK_BS_PAGE_SIZE)); 4180 } 4181 4182 bs_batch_close(batch); 4183 } 4184 4185 static void 4186 bs_load_replay_md_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 4187 { 4188 struct spdk_bs_load_ctx *ctx = cb_arg; 4189 uint32_t page_num; 4190 struct spdk_blob_md_page *page; 4191 4192 if (bserrno != 0) { 4193 bs_load_ctx_fail(ctx, bserrno); 4194 return; 4195 } 4196 4197 page_num = ctx->cur_page; 4198 page = ctx->page; 4199 if (bs_load_cur_md_page_valid(ctx) == true) { 4200 if (page->sequence_num == 0 || ctx->in_page_chain == true) { 4201 bs_claim_md_page(ctx->bs, page_num); 4202 if (page->sequence_num == 0) { 4203 spdk_bit_array_set(ctx->bs->used_blobids, page_num); 4204 } 4205 if (bs_load_replay_md_parse_page(ctx, page)) { 4206 bs_load_ctx_fail(ctx, -EILSEQ); 4207 return; 4208 } 4209 if (page->next != SPDK_INVALID_MD_PAGE) { 4210 ctx->in_page_chain = true; 4211 ctx->cur_page = page->next; 4212 bs_load_replay_cur_md_page(ctx); 4213 return; 4214 } 4215 if (ctx->num_extent_pages != 0) { 4216 bs_load_replay_extent_pages(ctx); 4217 return; 4218 } 4219 } 4220 } 4221 bs_load_replay_md_chain_cpl(ctx); 4222 } 4223 4224 static void 4225 bs_load_replay_cur_md_page(struct spdk_bs_load_ctx *ctx) 4226 { 4227 uint64_t lba; 4228 4229 assert(ctx->cur_page < ctx->super->md_len); 4230 lba = bs_md_page_to_lba(ctx->bs, ctx->cur_page); 4231 bs_sequence_read_dev(ctx->seq, ctx->page, lba, 4232 bs_byte_to_lba(ctx->bs, SPDK_BS_PAGE_SIZE), 4233 bs_load_replay_md_cpl, ctx); 4234 } 4235 4236 static void 4237 bs_load_replay_md(struct spdk_bs_load_ctx *ctx) 4238 { 4239 ctx->page_index = 0; 4240 ctx->cur_page = 0; 4241 ctx->page = spdk_zmalloc(SPDK_BS_PAGE_SIZE, 0, 4242 NULL, SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA); 4243 if (!ctx->page) { 4244 bs_load_ctx_fail(ctx, -ENOMEM); 4245 return; 4246 } 4247 bs_load_replay_cur_md_page(ctx); 4248 } 4249 4250 static void 4251 bs_recover(struct spdk_bs_load_ctx *ctx) 4252 { 4253 int rc; 4254 4255 rc = spdk_bit_array_resize(&ctx->bs->used_md_pages, ctx->super->md_len); 4256 if (rc < 0) { 4257 bs_load_ctx_fail(ctx, -ENOMEM); 4258 return; 4259 } 4260 4261 rc = spdk_bit_array_resize(&ctx->bs->used_blobids, ctx->super->md_len); 4262 if (rc < 0) { 4263 bs_load_ctx_fail(ctx, -ENOMEM); 4264 return; 4265 } 4266 4267 rc = spdk_bit_array_resize(&ctx->used_clusters, ctx->bs->total_clusters); 4268 if (rc < 0) { 4269 bs_load_ctx_fail(ctx, -ENOMEM); 4270 return; 4271 } 4272 4273 rc = spdk_bit_array_resize(&ctx->bs->open_blobids, ctx->super->md_len); 4274 if (rc < 0) { 4275 bs_load_ctx_fail(ctx, -ENOMEM); 4276 return; 4277 } 4278 4279 ctx->bs->num_free_clusters = ctx->bs->total_clusters; 4280 bs_load_replay_md(ctx); 4281 } 4282 4283 static void 4284 bs_load_super_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 4285 { 4286 struct spdk_bs_load_ctx *ctx = cb_arg; 4287 uint32_t crc; 4288 int rc; 4289 static const char zeros[SPDK_BLOBSTORE_TYPE_LENGTH]; 4290 4291 if (ctx->super->version > SPDK_BS_VERSION || 4292 ctx->super->version < SPDK_BS_INITIAL_VERSION) { 4293 bs_load_ctx_fail(ctx, -EILSEQ); 4294 return; 4295 } 4296 4297 if (memcmp(ctx->super->signature, SPDK_BS_SUPER_BLOCK_SIG, 4298 sizeof(ctx->super->signature)) != 0) { 4299 bs_load_ctx_fail(ctx, -EILSEQ); 4300 return; 4301 } 4302 4303 crc = blob_md_page_calc_crc(ctx->super); 4304 if (crc != ctx->super->crc) { 4305 bs_load_ctx_fail(ctx, -EILSEQ); 4306 return; 4307 } 4308 4309 if (memcmp(&ctx->bs->bstype, &ctx->super->bstype, SPDK_BLOBSTORE_TYPE_LENGTH) == 0) { 4310 SPDK_DEBUGLOG(blob, "Bstype matched - loading blobstore\n"); 4311 } else if (memcmp(&ctx->bs->bstype, zeros, SPDK_BLOBSTORE_TYPE_LENGTH) == 0) { 4312 SPDK_DEBUGLOG(blob, "Bstype wildcard used - loading blobstore regardless bstype\n"); 4313 } else { 4314 SPDK_DEBUGLOG(blob, "Unexpected bstype\n"); 4315 SPDK_LOGDUMP(blob, "Expected:", ctx->bs->bstype.bstype, SPDK_BLOBSTORE_TYPE_LENGTH); 4316 SPDK_LOGDUMP(blob, "Found:", ctx->super->bstype.bstype, SPDK_BLOBSTORE_TYPE_LENGTH); 4317 bs_load_ctx_fail(ctx, -ENXIO); 4318 return; 4319 } 4320 4321 if (ctx->super->size > ctx->bs->dev->blockcnt * ctx->bs->dev->blocklen) { 4322 SPDK_NOTICELOG("Size mismatch, dev size: %" PRIu64 ", blobstore size: %" PRIu64 "\n", 4323 ctx->bs->dev->blockcnt * ctx->bs->dev->blocklen, ctx->super->size); 4324 bs_load_ctx_fail(ctx, -EILSEQ); 4325 return; 4326 } 4327 4328 if (ctx->super->size == 0) { 4329 ctx->super->size = ctx->bs->dev->blockcnt * ctx->bs->dev->blocklen; 4330 } 4331 4332 if (ctx->super->io_unit_size == 0) { 4333 ctx->super->io_unit_size = SPDK_BS_PAGE_SIZE; 4334 } 4335 4336 /* Parse the super block */ 4337 ctx->bs->clean = 1; 4338 ctx->bs->cluster_sz = ctx->super->cluster_size; 4339 ctx->bs->total_clusters = ctx->super->size / ctx->super->cluster_size; 4340 ctx->bs->pages_per_cluster = ctx->bs->cluster_sz / SPDK_BS_PAGE_SIZE; 4341 if (spdk_u32_is_pow2(ctx->bs->pages_per_cluster)) { 4342 ctx->bs->pages_per_cluster_shift = spdk_u32log2(ctx->bs->pages_per_cluster); 4343 } 4344 ctx->bs->io_unit_size = ctx->super->io_unit_size; 4345 rc = spdk_bit_array_resize(&ctx->used_clusters, ctx->bs->total_clusters); 4346 if (rc < 0) { 4347 bs_load_ctx_fail(ctx, -ENOMEM); 4348 return; 4349 } 4350 ctx->bs->md_start = ctx->super->md_start; 4351 ctx->bs->md_len = ctx->super->md_len; 4352 ctx->bs->total_data_clusters = ctx->bs->total_clusters - spdk_divide_round_up( 4353 ctx->bs->md_start + ctx->bs->md_len, ctx->bs->pages_per_cluster); 4354 ctx->bs->super_blob = ctx->super->super_blob; 4355 memcpy(&ctx->bs->bstype, &ctx->super->bstype, sizeof(ctx->super->bstype)); 4356 4357 if (ctx->super->used_blobid_mask_len == 0 || ctx->super->clean == 0) { 4358 bs_recover(ctx); 4359 } else { 4360 bs_load_read_used_pages(ctx); 4361 } 4362 } 4363 4364 static inline int 4365 bs_opts_copy(struct spdk_bs_opts *src, struct spdk_bs_opts *dst) 4366 { 4367 4368 if (!src->opts_size) { 4369 SPDK_ERRLOG("opts_size should not be zero value\n"); 4370 return -1; 4371 } 4372 4373 #define FIELD_OK(field) \ 4374 offsetof(struct spdk_bs_opts, field) + sizeof(src->field) <= src->opts_size 4375 4376 #define SET_FIELD(field) \ 4377 if (FIELD_OK(field)) { \ 4378 dst->field = src->field; \ 4379 } \ 4380 4381 SET_FIELD(cluster_sz); 4382 SET_FIELD(num_md_pages); 4383 SET_FIELD(max_md_ops); 4384 SET_FIELD(max_channel_ops); 4385 SET_FIELD(clear_method); 4386 4387 if (FIELD_OK(bstype)) { 4388 memcpy(&dst->bstype, &src->bstype, sizeof(dst->bstype)); 4389 } 4390 SET_FIELD(iter_cb_fn); 4391 SET_FIELD(iter_cb_arg); 4392 4393 dst->opts_size = src->opts_size; 4394 4395 /* You should not remove this statement, but need to update the assert statement 4396 * if you add a new field, and also add a corresponding SET_FIELD statement */ 4397 SPDK_STATIC_ASSERT(sizeof(struct spdk_bs_opts) == 64, "Incorrect size"); 4398 4399 #undef FIELD_OK 4400 #undef SET_FIELD 4401 4402 return 0; 4403 } 4404 4405 void 4406 spdk_bs_load(struct spdk_bs_dev *dev, struct spdk_bs_opts *o, 4407 spdk_bs_op_with_handle_complete cb_fn, void *cb_arg) 4408 { 4409 struct spdk_blob_store *bs; 4410 struct spdk_bs_cpl cpl; 4411 struct spdk_bs_load_ctx *ctx; 4412 struct spdk_bs_opts opts = {}; 4413 int err; 4414 4415 SPDK_DEBUGLOG(blob, "Loading blobstore from dev %p\n", dev); 4416 4417 if ((SPDK_BS_PAGE_SIZE % dev->blocklen) != 0) { 4418 SPDK_DEBUGLOG(blob, "unsupported dev block length of %d\n", dev->blocklen); 4419 dev->destroy(dev); 4420 cb_fn(cb_arg, NULL, -EINVAL); 4421 return; 4422 } 4423 4424 spdk_bs_opts_init(&opts, sizeof(opts)); 4425 if (o) { 4426 if (bs_opts_copy(o, &opts)) { 4427 return; 4428 } 4429 } 4430 4431 if (opts.max_md_ops == 0 || opts.max_channel_ops == 0) { 4432 dev->destroy(dev); 4433 cb_fn(cb_arg, NULL, -EINVAL); 4434 return; 4435 } 4436 4437 err = bs_alloc(dev, &opts, &bs, &ctx); 4438 if (err) { 4439 dev->destroy(dev); 4440 cb_fn(cb_arg, NULL, err); 4441 return; 4442 } 4443 4444 cpl.type = SPDK_BS_CPL_TYPE_BS_HANDLE; 4445 cpl.u.bs_handle.cb_fn = cb_fn; 4446 cpl.u.bs_handle.cb_arg = cb_arg; 4447 cpl.u.bs_handle.bs = bs; 4448 4449 ctx->seq = bs_sequence_start(bs->md_channel, &cpl); 4450 if (!ctx->seq) { 4451 spdk_free(ctx->super); 4452 free(ctx); 4453 bs_free(bs); 4454 cb_fn(cb_arg, NULL, -ENOMEM); 4455 return; 4456 } 4457 4458 /* Read the super block */ 4459 bs_sequence_read_dev(ctx->seq, ctx->super, bs_page_to_lba(bs, 0), 4460 bs_byte_to_lba(bs, sizeof(*ctx->super)), 4461 bs_load_super_cpl, ctx); 4462 } 4463 4464 /* END spdk_bs_load */ 4465 4466 /* START spdk_bs_dump */ 4467 4468 static void 4469 bs_dump_finish(spdk_bs_sequence_t *seq, struct spdk_bs_load_ctx *ctx, int bserrno) 4470 { 4471 spdk_free(ctx->super); 4472 4473 /* 4474 * We need to defer calling bs_call_cpl() until after 4475 * dev destruction, so tuck these away for later use. 4476 */ 4477 ctx->bs->unload_err = bserrno; 4478 memcpy(&ctx->bs->unload_cpl, &seq->cpl, sizeof(struct spdk_bs_cpl)); 4479 seq->cpl.type = SPDK_BS_CPL_TYPE_NONE; 4480 4481 bs_sequence_finish(seq, 0); 4482 bs_free(ctx->bs); 4483 free(ctx); 4484 } 4485 4486 static void bs_dump_read_md_page(spdk_bs_sequence_t *seq, void *cb_arg); 4487 4488 static void 4489 bs_dump_print_md_page(struct spdk_bs_load_ctx *ctx) 4490 { 4491 uint32_t page_idx = ctx->cur_page; 4492 struct spdk_blob_md_page *page = ctx->page; 4493 struct spdk_blob_md_descriptor *desc; 4494 size_t cur_desc = 0; 4495 uint32_t crc; 4496 4497 fprintf(ctx->fp, "=========\n"); 4498 fprintf(ctx->fp, "Metadata Page Index: %" PRIu32 " (0x%" PRIx32 ")\n", page_idx, page_idx); 4499 fprintf(ctx->fp, "Blob ID: 0x%" PRIx64 "\n", page->id); 4500 4501 crc = blob_md_page_calc_crc(page); 4502 fprintf(ctx->fp, "CRC: 0x%" PRIx32 " (%s)\n", page->crc, crc == page->crc ? "OK" : "Mismatch"); 4503 4504 desc = (struct spdk_blob_md_descriptor *)page->descriptors; 4505 while (cur_desc < sizeof(page->descriptors)) { 4506 if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_PADDING) { 4507 if (desc->length == 0) { 4508 /* If padding and length are 0, this terminates the page */ 4509 break; 4510 } 4511 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_EXTENT_RLE) { 4512 struct spdk_blob_md_descriptor_extent_rle *desc_extent_rle; 4513 unsigned int i; 4514 4515 desc_extent_rle = (struct spdk_blob_md_descriptor_extent_rle *)desc; 4516 4517 for (i = 0; i < desc_extent_rle->length / sizeof(desc_extent_rle->extents[0]); i++) { 4518 if (desc_extent_rle->extents[i].cluster_idx != 0) { 4519 fprintf(ctx->fp, "Allocated Extent - Start: %" PRIu32, 4520 desc_extent_rle->extents[i].cluster_idx); 4521 } else { 4522 fprintf(ctx->fp, "Unallocated Extent - "); 4523 } 4524 fprintf(ctx->fp, " Length: %" PRIu32, desc_extent_rle->extents[i].length); 4525 fprintf(ctx->fp, "\n"); 4526 } 4527 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_EXTENT_PAGE) { 4528 struct spdk_blob_md_descriptor_extent_page *desc_extent; 4529 unsigned int i; 4530 4531 desc_extent = (struct spdk_blob_md_descriptor_extent_page *)desc; 4532 4533 for (i = 0; i < desc_extent->length / sizeof(desc_extent->cluster_idx[0]); i++) { 4534 if (desc_extent->cluster_idx[i] != 0) { 4535 fprintf(ctx->fp, "Allocated Extent - Start: %" PRIu32, 4536 desc_extent->cluster_idx[i]); 4537 } else { 4538 fprintf(ctx->fp, "Unallocated Extent"); 4539 } 4540 fprintf(ctx->fp, "\n"); 4541 } 4542 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_XATTR) { 4543 struct spdk_blob_md_descriptor_xattr *desc_xattr; 4544 uint32_t i; 4545 4546 desc_xattr = (struct spdk_blob_md_descriptor_xattr *)desc; 4547 4548 if (desc_xattr->length != 4549 sizeof(desc_xattr->name_length) + sizeof(desc_xattr->value_length) + 4550 desc_xattr->name_length + desc_xattr->value_length) { 4551 } 4552 4553 memcpy(ctx->xattr_name, desc_xattr->name, desc_xattr->name_length); 4554 ctx->xattr_name[desc_xattr->name_length] = '\0'; 4555 fprintf(ctx->fp, "XATTR: name = \"%s\"\n", ctx->xattr_name); 4556 fprintf(ctx->fp, " value = \""); 4557 ctx->print_xattr_fn(ctx->fp, ctx->super->bstype.bstype, ctx->xattr_name, 4558 (void *)((uintptr_t)desc_xattr->name + desc_xattr->name_length), 4559 desc_xattr->value_length); 4560 fprintf(ctx->fp, "\"\n"); 4561 for (i = 0; i < desc_xattr->value_length; i++) { 4562 if (i % 16 == 0) { 4563 fprintf(ctx->fp, " "); 4564 } 4565 fprintf(ctx->fp, "%02" PRIx8 " ", *((uint8_t *)desc_xattr->name + desc_xattr->name_length + i)); 4566 if ((i + 1) % 16 == 0) { 4567 fprintf(ctx->fp, "\n"); 4568 } 4569 } 4570 if (i % 16 != 0) { 4571 fprintf(ctx->fp, "\n"); 4572 } 4573 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_XATTR_INTERNAL) { 4574 /* TODO */ 4575 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_FLAGS) { 4576 /* TODO */ 4577 } else { 4578 /* Error */ 4579 } 4580 /* Advance to the next descriptor */ 4581 cur_desc += sizeof(*desc) + desc->length; 4582 if (cur_desc + sizeof(*desc) > sizeof(page->descriptors)) { 4583 break; 4584 } 4585 desc = (struct spdk_blob_md_descriptor *)((uintptr_t)page->descriptors + cur_desc); 4586 } 4587 } 4588 4589 static void 4590 bs_dump_read_md_page_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 4591 { 4592 struct spdk_bs_load_ctx *ctx = cb_arg; 4593 4594 if (bserrno != 0) { 4595 bs_dump_finish(seq, ctx, bserrno); 4596 return; 4597 } 4598 4599 if (ctx->page->id != 0) { 4600 bs_dump_print_md_page(ctx); 4601 } 4602 4603 ctx->cur_page++; 4604 4605 if (ctx->cur_page < ctx->super->md_len) { 4606 bs_dump_read_md_page(seq, ctx); 4607 } else { 4608 spdk_free(ctx->page); 4609 bs_dump_finish(seq, ctx, 0); 4610 } 4611 } 4612 4613 static void 4614 bs_dump_read_md_page(spdk_bs_sequence_t *seq, void *cb_arg) 4615 { 4616 struct spdk_bs_load_ctx *ctx = cb_arg; 4617 uint64_t lba; 4618 4619 assert(ctx->cur_page < ctx->super->md_len); 4620 lba = bs_page_to_lba(ctx->bs, ctx->super->md_start + ctx->cur_page); 4621 bs_sequence_read_dev(seq, ctx->page, lba, 4622 bs_byte_to_lba(ctx->bs, SPDK_BS_PAGE_SIZE), 4623 bs_dump_read_md_page_cpl, ctx); 4624 } 4625 4626 static void 4627 bs_dump_super_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 4628 { 4629 struct spdk_bs_load_ctx *ctx = cb_arg; 4630 4631 fprintf(ctx->fp, "Signature: \"%.8s\" ", ctx->super->signature); 4632 if (memcmp(ctx->super->signature, SPDK_BS_SUPER_BLOCK_SIG, 4633 sizeof(ctx->super->signature)) != 0) { 4634 fprintf(ctx->fp, "(Mismatch)\n"); 4635 bs_dump_finish(seq, ctx, bserrno); 4636 return; 4637 } else { 4638 fprintf(ctx->fp, "(OK)\n"); 4639 } 4640 fprintf(ctx->fp, "Version: %" PRIu32 "\n", ctx->super->version); 4641 fprintf(ctx->fp, "CRC: 0x%x (%s)\n", ctx->super->crc, 4642 (ctx->super->crc == blob_md_page_calc_crc(ctx->super)) ? "OK" : "Mismatch"); 4643 fprintf(ctx->fp, "Blobstore Type: %.*s\n", SPDK_BLOBSTORE_TYPE_LENGTH, ctx->super->bstype.bstype); 4644 fprintf(ctx->fp, "Cluster Size: %" PRIu32 "\n", ctx->super->cluster_size); 4645 fprintf(ctx->fp, "Super Blob ID: "); 4646 if (ctx->super->super_blob == SPDK_BLOBID_INVALID) { 4647 fprintf(ctx->fp, "(None)\n"); 4648 } else { 4649 fprintf(ctx->fp, "%" PRIu64 "\n", ctx->super->super_blob); 4650 } 4651 fprintf(ctx->fp, "Clean: %" PRIu32 "\n", ctx->super->clean); 4652 fprintf(ctx->fp, "Used Metadata Page Mask Start: %" PRIu32 "\n", ctx->super->used_page_mask_start); 4653 fprintf(ctx->fp, "Used Metadata Page Mask Length: %" PRIu32 "\n", ctx->super->used_page_mask_len); 4654 fprintf(ctx->fp, "Used Cluster Mask Start: %" PRIu32 "\n", ctx->super->used_cluster_mask_start); 4655 fprintf(ctx->fp, "Used Cluster Mask Length: %" PRIu32 "\n", ctx->super->used_cluster_mask_len); 4656 fprintf(ctx->fp, "Used Blob ID Mask Start: %" PRIu32 "\n", ctx->super->used_blobid_mask_start); 4657 fprintf(ctx->fp, "Used Blob ID Mask Length: %" PRIu32 "\n", ctx->super->used_blobid_mask_len); 4658 fprintf(ctx->fp, "Metadata Start: %" PRIu32 "\n", ctx->super->md_start); 4659 fprintf(ctx->fp, "Metadata Length: %" PRIu32 "\n", ctx->super->md_len); 4660 4661 ctx->cur_page = 0; 4662 ctx->page = spdk_zmalloc(SPDK_BS_PAGE_SIZE, 0, 4663 NULL, SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA); 4664 if (!ctx->page) { 4665 bs_dump_finish(seq, ctx, -ENOMEM); 4666 return; 4667 } 4668 bs_dump_read_md_page(seq, ctx); 4669 } 4670 4671 void 4672 spdk_bs_dump(struct spdk_bs_dev *dev, FILE *fp, spdk_bs_dump_print_xattr print_xattr_fn, 4673 spdk_bs_op_complete cb_fn, void *cb_arg) 4674 { 4675 struct spdk_blob_store *bs; 4676 struct spdk_bs_cpl cpl; 4677 spdk_bs_sequence_t *seq; 4678 struct spdk_bs_load_ctx *ctx; 4679 struct spdk_bs_opts opts = {}; 4680 int err; 4681 4682 SPDK_DEBUGLOG(blob, "Dumping blobstore from dev %p\n", dev); 4683 4684 spdk_bs_opts_init(&opts, sizeof(opts)); 4685 4686 err = bs_alloc(dev, &opts, &bs, &ctx); 4687 if (err) { 4688 dev->destroy(dev); 4689 cb_fn(cb_arg, err); 4690 return; 4691 } 4692 4693 ctx->fp = fp; 4694 ctx->print_xattr_fn = print_xattr_fn; 4695 4696 cpl.type = SPDK_BS_CPL_TYPE_BS_BASIC; 4697 cpl.u.bs_basic.cb_fn = cb_fn; 4698 cpl.u.bs_basic.cb_arg = cb_arg; 4699 4700 seq = bs_sequence_start(bs->md_channel, &cpl); 4701 if (!seq) { 4702 spdk_free(ctx->super); 4703 free(ctx); 4704 bs_free(bs); 4705 cb_fn(cb_arg, -ENOMEM); 4706 return; 4707 } 4708 4709 /* Read the super block */ 4710 bs_sequence_read_dev(seq, ctx->super, bs_page_to_lba(bs, 0), 4711 bs_byte_to_lba(bs, sizeof(*ctx->super)), 4712 bs_dump_super_cpl, ctx); 4713 } 4714 4715 /* END spdk_bs_dump */ 4716 4717 /* START spdk_bs_init */ 4718 4719 static void 4720 bs_init_persist_super_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 4721 { 4722 struct spdk_bs_load_ctx *ctx = cb_arg; 4723 4724 ctx->bs->used_clusters = spdk_bit_pool_create_from_array(ctx->used_clusters); 4725 spdk_free(ctx->super); 4726 free(ctx); 4727 4728 bs_sequence_finish(seq, bserrno); 4729 } 4730 4731 static void 4732 bs_init_trim_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 4733 { 4734 struct spdk_bs_load_ctx *ctx = cb_arg; 4735 4736 /* Write super block */ 4737 bs_sequence_write_dev(seq, ctx->super, bs_page_to_lba(ctx->bs, 0), 4738 bs_byte_to_lba(ctx->bs, sizeof(*ctx->super)), 4739 bs_init_persist_super_cpl, ctx); 4740 } 4741 4742 void 4743 spdk_bs_init(struct spdk_bs_dev *dev, struct spdk_bs_opts *o, 4744 spdk_bs_op_with_handle_complete cb_fn, void *cb_arg) 4745 { 4746 struct spdk_bs_load_ctx *ctx; 4747 struct spdk_blob_store *bs; 4748 struct spdk_bs_cpl cpl; 4749 spdk_bs_sequence_t *seq; 4750 spdk_bs_batch_t *batch; 4751 uint64_t num_md_lba; 4752 uint64_t num_md_pages; 4753 uint64_t num_md_clusters; 4754 uint32_t i; 4755 struct spdk_bs_opts opts = {}; 4756 int rc; 4757 uint64_t lba, lba_count; 4758 4759 SPDK_DEBUGLOG(blob, "Initializing blobstore on dev %p\n", dev); 4760 4761 if ((SPDK_BS_PAGE_SIZE % dev->blocklen) != 0) { 4762 SPDK_ERRLOG("unsupported dev block length of %d\n", 4763 dev->blocklen); 4764 dev->destroy(dev); 4765 cb_fn(cb_arg, NULL, -EINVAL); 4766 return; 4767 } 4768 4769 spdk_bs_opts_init(&opts, sizeof(opts)); 4770 if (o) { 4771 if (bs_opts_copy(o, &opts)) { 4772 return; 4773 } 4774 } 4775 4776 if (bs_opts_verify(&opts) != 0) { 4777 dev->destroy(dev); 4778 cb_fn(cb_arg, NULL, -EINVAL); 4779 return; 4780 } 4781 4782 rc = bs_alloc(dev, &opts, &bs, &ctx); 4783 if (rc) { 4784 dev->destroy(dev); 4785 cb_fn(cb_arg, NULL, rc); 4786 return; 4787 } 4788 4789 if (opts.num_md_pages == SPDK_BLOB_OPTS_NUM_MD_PAGES) { 4790 /* By default, allocate 1 page per cluster. 4791 * Technically, this over-allocates metadata 4792 * because more metadata will reduce the number 4793 * of usable clusters. This can be addressed with 4794 * more complex math in the future. 4795 */ 4796 bs->md_len = bs->total_clusters; 4797 } else { 4798 bs->md_len = opts.num_md_pages; 4799 } 4800 rc = spdk_bit_array_resize(&bs->used_md_pages, bs->md_len); 4801 if (rc < 0) { 4802 spdk_free(ctx->super); 4803 free(ctx); 4804 bs_free(bs); 4805 cb_fn(cb_arg, NULL, -ENOMEM); 4806 return; 4807 } 4808 4809 rc = spdk_bit_array_resize(&bs->used_blobids, bs->md_len); 4810 if (rc < 0) { 4811 spdk_free(ctx->super); 4812 free(ctx); 4813 bs_free(bs); 4814 cb_fn(cb_arg, NULL, -ENOMEM); 4815 return; 4816 } 4817 4818 rc = spdk_bit_array_resize(&bs->open_blobids, bs->md_len); 4819 if (rc < 0) { 4820 spdk_free(ctx->super); 4821 free(ctx); 4822 bs_free(bs); 4823 cb_fn(cb_arg, NULL, -ENOMEM); 4824 return; 4825 } 4826 4827 memcpy(ctx->super->signature, SPDK_BS_SUPER_BLOCK_SIG, 4828 sizeof(ctx->super->signature)); 4829 ctx->super->version = SPDK_BS_VERSION; 4830 ctx->super->length = sizeof(*ctx->super); 4831 ctx->super->super_blob = bs->super_blob; 4832 ctx->super->clean = 0; 4833 ctx->super->cluster_size = bs->cluster_sz; 4834 ctx->super->io_unit_size = bs->io_unit_size; 4835 memcpy(&ctx->super->bstype, &bs->bstype, sizeof(bs->bstype)); 4836 4837 /* Calculate how many pages the metadata consumes at the front 4838 * of the disk. 4839 */ 4840 4841 /* The super block uses 1 page */ 4842 num_md_pages = 1; 4843 4844 /* The used_md_pages mask requires 1 bit per metadata page, rounded 4845 * up to the nearest page, plus a header. 4846 */ 4847 ctx->super->used_page_mask_start = num_md_pages; 4848 ctx->super->used_page_mask_len = spdk_divide_round_up(sizeof(struct spdk_bs_md_mask) + 4849 spdk_divide_round_up(bs->md_len, 8), 4850 SPDK_BS_PAGE_SIZE); 4851 num_md_pages += ctx->super->used_page_mask_len; 4852 4853 /* The used_clusters mask requires 1 bit per cluster, rounded 4854 * up to the nearest page, plus a header. 4855 */ 4856 ctx->super->used_cluster_mask_start = num_md_pages; 4857 ctx->super->used_cluster_mask_len = spdk_divide_round_up(sizeof(struct spdk_bs_md_mask) + 4858 spdk_divide_round_up(bs->total_clusters, 8), 4859 SPDK_BS_PAGE_SIZE); 4860 num_md_pages += ctx->super->used_cluster_mask_len; 4861 4862 /* The used_blobids mask requires 1 bit per metadata page, rounded 4863 * up to the nearest page, plus a header. 4864 */ 4865 ctx->super->used_blobid_mask_start = num_md_pages; 4866 ctx->super->used_blobid_mask_len = spdk_divide_round_up(sizeof(struct spdk_bs_md_mask) + 4867 spdk_divide_round_up(bs->md_len, 8), 4868 SPDK_BS_PAGE_SIZE); 4869 num_md_pages += ctx->super->used_blobid_mask_len; 4870 4871 /* The metadata region size was chosen above */ 4872 ctx->super->md_start = bs->md_start = num_md_pages; 4873 ctx->super->md_len = bs->md_len; 4874 num_md_pages += bs->md_len; 4875 4876 num_md_lba = bs_page_to_lba(bs, num_md_pages); 4877 4878 ctx->super->size = dev->blockcnt * dev->blocklen; 4879 4880 ctx->super->crc = blob_md_page_calc_crc(ctx->super); 4881 4882 num_md_clusters = spdk_divide_round_up(num_md_pages, bs->pages_per_cluster); 4883 if (num_md_clusters > bs->total_clusters) { 4884 SPDK_ERRLOG("Blobstore metadata cannot use more clusters than is available, " 4885 "please decrease number of pages reserved for metadata " 4886 "or increase cluster size.\n"); 4887 spdk_free(ctx->super); 4888 spdk_bit_array_free(&ctx->used_clusters); 4889 free(ctx); 4890 bs_free(bs); 4891 cb_fn(cb_arg, NULL, -ENOMEM); 4892 return; 4893 } 4894 /* Claim all of the clusters used by the metadata */ 4895 for (i = 0; i < num_md_clusters; i++) { 4896 spdk_bit_array_set(ctx->used_clusters, i); 4897 } 4898 4899 bs->num_free_clusters -= num_md_clusters; 4900 bs->total_data_clusters = bs->num_free_clusters; 4901 4902 cpl.type = SPDK_BS_CPL_TYPE_BS_HANDLE; 4903 cpl.u.bs_handle.cb_fn = cb_fn; 4904 cpl.u.bs_handle.cb_arg = cb_arg; 4905 cpl.u.bs_handle.bs = bs; 4906 4907 seq = bs_sequence_start(bs->md_channel, &cpl); 4908 if (!seq) { 4909 spdk_free(ctx->super); 4910 free(ctx); 4911 bs_free(bs); 4912 cb_fn(cb_arg, NULL, -ENOMEM); 4913 return; 4914 } 4915 4916 batch = bs_sequence_to_batch(seq, bs_init_trim_cpl, ctx); 4917 4918 /* Clear metadata space */ 4919 bs_batch_write_zeroes_dev(batch, 0, num_md_lba); 4920 4921 lba = num_md_lba; 4922 while (lba < ctx->bs->dev->blockcnt) { 4923 lba_count = spdk_min(UINT32_MAX, ctx->bs->dev->blockcnt - lba); 4924 switch (opts.clear_method) { 4925 case BS_CLEAR_WITH_UNMAP: 4926 /* Trim data clusters */ 4927 bs_batch_unmap_dev(batch, lba, lba_count); 4928 break; 4929 case BS_CLEAR_WITH_WRITE_ZEROES: 4930 /* Write_zeroes to data clusters */ 4931 bs_batch_write_zeroes_dev(batch, lba, lba_count); 4932 break; 4933 case BS_CLEAR_WITH_NONE: 4934 default: 4935 break; 4936 } 4937 lba += lba_count; 4938 } 4939 4940 bs_batch_close(batch); 4941 } 4942 4943 /* END spdk_bs_init */ 4944 4945 /* START spdk_bs_destroy */ 4946 4947 static void 4948 bs_destroy_trim_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 4949 { 4950 struct spdk_bs_load_ctx *ctx = cb_arg; 4951 struct spdk_blob_store *bs = ctx->bs; 4952 4953 /* 4954 * We need to defer calling bs_call_cpl() until after 4955 * dev destruction, so tuck these away for later use. 4956 */ 4957 bs->unload_err = bserrno; 4958 memcpy(&bs->unload_cpl, &seq->cpl, sizeof(struct spdk_bs_cpl)); 4959 seq->cpl.type = SPDK_BS_CPL_TYPE_NONE; 4960 4961 bs_sequence_finish(seq, bserrno); 4962 4963 bs_free(bs); 4964 free(ctx); 4965 } 4966 4967 void 4968 spdk_bs_destroy(struct spdk_blob_store *bs, spdk_bs_op_complete cb_fn, 4969 void *cb_arg) 4970 { 4971 struct spdk_bs_cpl cpl; 4972 spdk_bs_sequence_t *seq; 4973 struct spdk_bs_load_ctx *ctx; 4974 4975 SPDK_DEBUGLOG(blob, "Destroying blobstore\n"); 4976 4977 if (!TAILQ_EMPTY(&bs->blobs)) { 4978 SPDK_ERRLOG("Blobstore still has open blobs\n"); 4979 cb_fn(cb_arg, -EBUSY); 4980 return; 4981 } 4982 4983 cpl.type = SPDK_BS_CPL_TYPE_BS_BASIC; 4984 cpl.u.bs_basic.cb_fn = cb_fn; 4985 cpl.u.bs_basic.cb_arg = cb_arg; 4986 4987 ctx = calloc(1, sizeof(*ctx)); 4988 if (!ctx) { 4989 cb_fn(cb_arg, -ENOMEM); 4990 return; 4991 } 4992 4993 ctx->bs = bs; 4994 4995 seq = bs_sequence_start(bs->md_channel, &cpl); 4996 if (!seq) { 4997 free(ctx); 4998 cb_fn(cb_arg, -ENOMEM); 4999 return; 5000 } 5001 5002 /* Write zeroes to the super block */ 5003 bs_sequence_write_zeroes_dev(seq, 5004 bs_page_to_lba(bs, 0), 5005 bs_byte_to_lba(bs, sizeof(struct spdk_bs_super_block)), 5006 bs_destroy_trim_cpl, ctx); 5007 } 5008 5009 /* END spdk_bs_destroy */ 5010 5011 /* START spdk_bs_unload */ 5012 5013 static void 5014 bs_unload_finish(struct spdk_bs_load_ctx *ctx, int bserrno) 5015 { 5016 spdk_bs_sequence_t *seq = ctx->seq; 5017 5018 spdk_free(ctx->super); 5019 5020 /* 5021 * We need to defer calling bs_call_cpl() until after 5022 * dev destruction, so tuck these away for later use. 5023 */ 5024 ctx->bs->unload_err = bserrno; 5025 memcpy(&ctx->bs->unload_cpl, &seq->cpl, sizeof(struct spdk_bs_cpl)); 5026 seq->cpl.type = SPDK_BS_CPL_TYPE_NONE; 5027 5028 bs_sequence_finish(seq, bserrno); 5029 5030 bs_free(ctx->bs); 5031 free(ctx); 5032 } 5033 5034 static void 5035 bs_unload_write_super_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 5036 { 5037 struct spdk_bs_load_ctx *ctx = cb_arg; 5038 5039 bs_unload_finish(ctx, bserrno); 5040 } 5041 5042 static void 5043 bs_unload_write_used_clusters_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 5044 { 5045 struct spdk_bs_load_ctx *ctx = cb_arg; 5046 5047 spdk_free(ctx->mask); 5048 5049 if (bserrno != 0) { 5050 bs_unload_finish(ctx, bserrno); 5051 return; 5052 } 5053 5054 ctx->super->clean = 1; 5055 5056 bs_write_super(seq, ctx->bs, ctx->super, bs_unload_write_super_cpl, ctx); 5057 } 5058 5059 static void 5060 bs_unload_write_used_blobids_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 5061 { 5062 struct spdk_bs_load_ctx *ctx = cb_arg; 5063 5064 spdk_free(ctx->mask); 5065 ctx->mask = NULL; 5066 5067 if (bserrno != 0) { 5068 bs_unload_finish(ctx, bserrno); 5069 return; 5070 } 5071 5072 bs_write_used_clusters(seq, ctx, bs_unload_write_used_clusters_cpl); 5073 } 5074 5075 static void 5076 bs_unload_write_used_pages_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 5077 { 5078 struct spdk_bs_load_ctx *ctx = cb_arg; 5079 5080 spdk_free(ctx->mask); 5081 ctx->mask = NULL; 5082 5083 if (bserrno != 0) { 5084 bs_unload_finish(ctx, bserrno); 5085 return; 5086 } 5087 5088 bs_write_used_blobids(seq, ctx, bs_unload_write_used_blobids_cpl); 5089 } 5090 5091 static void 5092 bs_unload_read_super_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 5093 { 5094 struct spdk_bs_load_ctx *ctx = cb_arg; 5095 5096 if (bserrno != 0) { 5097 bs_unload_finish(ctx, bserrno); 5098 return; 5099 } 5100 5101 bs_write_used_md(seq, cb_arg, bs_unload_write_used_pages_cpl); 5102 } 5103 5104 void 5105 spdk_bs_unload(struct spdk_blob_store *bs, spdk_bs_op_complete cb_fn, void *cb_arg) 5106 { 5107 struct spdk_bs_cpl cpl; 5108 struct spdk_bs_load_ctx *ctx; 5109 5110 SPDK_DEBUGLOG(blob, "Syncing blobstore\n"); 5111 5112 if (!TAILQ_EMPTY(&bs->blobs)) { 5113 SPDK_ERRLOG("Blobstore still has open blobs\n"); 5114 cb_fn(cb_arg, -EBUSY); 5115 return; 5116 } 5117 5118 ctx = calloc(1, sizeof(*ctx)); 5119 if (!ctx) { 5120 cb_fn(cb_arg, -ENOMEM); 5121 return; 5122 } 5123 5124 ctx->bs = bs; 5125 5126 ctx->super = spdk_zmalloc(sizeof(*ctx->super), 0x1000, NULL, 5127 SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA); 5128 if (!ctx->super) { 5129 free(ctx); 5130 cb_fn(cb_arg, -ENOMEM); 5131 return; 5132 } 5133 5134 cpl.type = SPDK_BS_CPL_TYPE_BS_BASIC; 5135 cpl.u.bs_basic.cb_fn = cb_fn; 5136 cpl.u.bs_basic.cb_arg = cb_arg; 5137 5138 ctx->seq = bs_sequence_start(bs->md_channel, &cpl); 5139 if (!ctx->seq) { 5140 spdk_free(ctx->super); 5141 free(ctx); 5142 cb_fn(cb_arg, -ENOMEM); 5143 return; 5144 } 5145 5146 /* Read super block */ 5147 bs_sequence_read_dev(ctx->seq, ctx->super, bs_page_to_lba(bs, 0), 5148 bs_byte_to_lba(bs, sizeof(*ctx->super)), 5149 bs_unload_read_super_cpl, ctx); 5150 } 5151 5152 /* END spdk_bs_unload */ 5153 5154 /* START spdk_bs_set_super */ 5155 5156 struct spdk_bs_set_super_ctx { 5157 struct spdk_blob_store *bs; 5158 struct spdk_bs_super_block *super; 5159 }; 5160 5161 static void 5162 bs_set_super_write_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 5163 { 5164 struct spdk_bs_set_super_ctx *ctx = cb_arg; 5165 5166 if (bserrno != 0) { 5167 SPDK_ERRLOG("Unable to write to super block of blobstore\n"); 5168 } 5169 5170 spdk_free(ctx->super); 5171 5172 bs_sequence_finish(seq, bserrno); 5173 5174 free(ctx); 5175 } 5176 5177 static void 5178 bs_set_super_read_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 5179 { 5180 struct spdk_bs_set_super_ctx *ctx = cb_arg; 5181 5182 if (bserrno != 0) { 5183 SPDK_ERRLOG("Unable to read super block of blobstore\n"); 5184 spdk_free(ctx->super); 5185 bs_sequence_finish(seq, bserrno); 5186 free(ctx); 5187 return; 5188 } 5189 5190 bs_write_super(seq, ctx->bs, ctx->super, bs_set_super_write_cpl, ctx); 5191 } 5192 5193 void 5194 spdk_bs_set_super(struct spdk_blob_store *bs, spdk_blob_id blobid, 5195 spdk_bs_op_complete cb_fn, void *cb_arg) 5196 { 5197 struct spdk_bs_cpl cpl; 5198 spdk_bs_sequence_t *seq; 5199 struct spdk_bs_set_super_ctx *ctx; 5200 5201 SPDK_DEBUGLOG(blob, "Setting super blob id on blobstore\n"); 5202 5203 ctx = calloc(1, sizeof(*ctx)); 5204 if (!ctx) { 5205 cb_fn(cb_arg, -ENOMEM); 5206 return; 5207 } 5208 5209 ctx->bs = bs; 5210 5211 ctx->super = spdk_zmalloc(sizeof(*ctx->super), 0x1000, NULL, 5212 SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA); 5213 if (!ctx->super) { 5214 free(ctx); 5215 cb_fn(cb_arg, -ENOMEM); 5216 return; 5217 } 5218 5219 cpl.type = SPDK_BS_CPL_TYPE_BS_BASIC; 5220 cpl.u.bs_basic.cb_fn = cb_fn; 5221 cpl.u.bs_basic.cb_arg = cb_arg; 5222 5223 seq = bs_sequence_start(bs->md_channel, &cpl); 5224 if (!seq) { 5225 spdk_free(ctx->super); 5226 free(ctx); 5227 cb_fn(cb_arg, -ENOMEM); 5228 return; 5229 } 5230 5231 bs->super_blob = blobid; 5232 5233 /* Read super block */ 5234 bs_sequence_read_dev(seq, ctx->super, bs_page_to_lba(bs, 0), 5235 bs_byte_to_lba(bs, sizeof(*ctx->super)), 5236 bs_set_super_read_cpl, ctx); 5237 } 5238 5239 /* END spdk_bs_set_super */ 5240 5241 void 5242 spdk_bs_get_super(struct spdk_blob_store *bs, 5243 spdk_blob_op_with_id_complete cb_fn, void *cb_arg) 5244 { 5245 if (bs->super_blob == SPDK_BLOBID_INVALID) { 5246 cb_fn(cb_arg, SPDK_BLOBID_INVALID, -ENOENT); 5247 } else { 5248 cb_fn(cb_arg, bs->super_blob, 0); 5249 } 5250 } 5251 5252 uint64_t 5253 spdk_bs_get_cluster_size(struct spdk_blob_store *bs) 5254 { 5255 return bs->cluster_sz; 5256 } 5257 5258 uint64_t 5259 spdk_bs_get_page_size(struct spdk_blob_store *bs) 5260 { 5261 return SPDK_BS_PAGE_SIZE; 5262 } 5263 5264 uint64_t 5265 spdk_bs_get_io_unit_size(struct spdk_blob_store *bs) 5266 { 5267 return bs->io_unit_size; 5268 } 5269 5270 uint64_t 5271 spdk_bs_free_cluster_count(struct spdk_blob_store *bs) 5272 { 5273 return bs->num_free_clusters; 5274 } 5275 5276 uint64_t 5277 spdk_bs_total_data_cluster_count(struct spdk_blob_store *bs) 5278 { 5279 return bs->total_data_clusters; 5280 } 5281 5282 static int 5283 bs_register_md_thread(struct spdk_blob_store *bs) 5284 { 5285 bs->md_channel = spdk_get_io_channel(bs); 5286 if (!bs->md_channel) { 5287 SPDK_ERRLOG("Failed to get IO channel.\n"); 5288 return -1; 5289 } 5290 5291 return 0; 5292 } 5293 5294 static int 5295 bs_unregister_md_thread(struct spdk_blob_store *bs) 5296 { 5297 spdk_put_io_channel(bs->md_channel); 5298 5299 return 0; 5300 } 5301 5302 spdk_blob_id spdk_blob_get_id(struct spdk_blob *blob) 5303 { 5304 assert(blob != NULL); 5305 5306 return blob->id; 5307 } 5308 5309 uint64_t spdk_blob_get_num_pages(struct spdk_blob *blob) 5310 { 5311 assert(blob != NULL); 5312 5313 return bs_cluster_to_page(blob->bs, blob->active.num_clusters); 5314 } 5315 5316 uint64_t spdk_blob_get_num_io_units(struct spdk_blob *blob) 5317 { 5318 assert(blob != NULL); 5319 5320 return spdk_blob_get_num_pages(blob) * bs_io_unit_per_page(blob->bs); 5321 } 5322 5323 uint64_t spdk_blob_get_num_clusters(struct spdk_blob *blob) 5324 { 5325 assert(blob != NULL); 5326 5327 return blob->active.num_clusters; 5328 } 5329 5330 /* START spdk_bs_create_blob */ 5331 5332 static void 5333 bs_create_blob_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 5334 { 5335 struct spdk_blob *blob = cb_arg; 5336 uint32_t page_idx = bs_blobid_to_page(blob->id); 5337 5338 if (bserrno != 0) { 5339 spdk_bit_array_clear(blob->bs->used_blobids, page_idx); 5340 bs_release_md_page(blob->bs, page_idx); 5341 } 5342 5343 blob_free(blob); 5344 5345 bs_sequence_finish(seq, bserrno); 5346 } 5347 5348 static int 5349 blob_set_xattrs(struct spdk_blob *blob, const struct spdk_blob_xattr_opts *xattrs, 5350 bool internal) 5351 { 5352 uint64_t i; 5353 size_t value_len = 0; 5354 int rc; 5355 const void *value = NULL; 5356 if (xattrs->count > 0 && xattrs->get_value == NULL) { 5357 return -EINVAL; 5358 } 5359 for (i = 0; i < xattrs->count; i++) { 5360 xattrs->get_value(xattrs->ctx, xattrs->names[i], &value, &value_len); 5361 if (value == NULL || value_len == 0) { 5362 return -EINVAL; 5363 } 5364 rc = blob_set_xattr(blob, xattrs->names[i], value, value_len, internal); 5365 if (rc < 0) { 5366 return rc; 5367 } 5368 } 5369 return 0; 5370 } 5371 5372 static void 5373 blob_opts_copy(const struct spdk_blob_opts *src, struct spdk_blob_opts *dst) 5374 { 5375 #define FIELD_OK(field) \ 5376 offsetof(struct spdk_blob_opts, field) + sizeof(src->field) <= src->opts_size 5377 5378 #define SET_FIELD(field) \ 5379 if (FIELD_OK(field)) { \ 5380 dst->field = src->field; \ 5381 } \ 5382 5383 SET_FIELD(num_clusters); 5384 SET_FIELD(thin_provision); 5385 SET_FIELD(clear_method); 5386 5387 if (FIELD_OK(xattrs)) { 5388 memcpy(&dst->xattrs, &src->xattrs, sizeof(src->xattrs)); 5389 } 5390 5391 SET_FIELD(use_extent_table); 5392 5393 dst->opts_size = src->opts_size; 5394 5395 /* You should not remove this statement, but need to update the assert statement 5396 * if you add a new field, and also add a corresponding SET_FIELD statement */ 5397 SPDK_STATIC_ASSERT(sizeof(struct spdk_blob_opts) == 64, "Incorrect size"); 5398 5399 #undef FIELD_OK 5400 #undef SET_FIELD 5401 } 5402 5403 static void 5404 bs_create_blob(struct spdk_blob_store *bs, 5405 const struct spdk_blob_opts *opts, 5406 const struct spdk_blob_xattr_opts *internal_xattrs, 5407 spdk_blob_op_with_id_complete cb_fn, void *cb_arg) 5408 { 5409 struct spdk_blob *blob; 5410 uint32_t page_idx; 5411 struct spdk_bs_cpl cpl; 5412 struct spdk_blob_opts opts_local; 5413 struct spdk_blob_xattr_opts internal_xattrs_default; 5414 spdk_bs_sequence_t *seq; 5415 spdk_blob_id id; 5416 int rc; 5417 5418 assert(spdk_get_thread() == bs->md_thread); 5419 5420 page_idx = spdk_bit_array_find_first_clear(bs->used_md_pages, 0); 5421 if (page_idx == UINT32_MAX) { 5422 cb_fn(cb_arg, 0, -ENOMEM); 5423 return; 5424 } 5425 spdk_bit_array_set(bs->used_blobids, page_idx); 5426 bs_claim_md_page(bs, page_idx); 5427 5428 id = bs_page_to_blobid(page_idx); 5429 5430 SPDK_DEBUGLOG(blob, "Creating blob with id %" PRIu64 " at page %u\n", id, page_idx); 5431 5432 blob = blob_alloc(bs, id); 5433 if (!blob) { 5434 spdk_bit_array_clear(bs->used_blobids, page_idx); 5435 bs_release_md_page(bs, page_idx); 5436 cb_fn(cb_arg, 0, -ENOMEM); 5437 return; 5438 } 5439 5440 spdk_blob_opts_init(&opts_local, sizeof(opts_local)); 5441 if (opts) { 5442 blob_opts_copy(opts, &opts_local); 5443 } 5444 5445 blob->use_extent_table = opts_local.use_extent_table; 5446 if (blob->use_extent_table) { 5447 blob->invalid_flags |= SPDK_BLOB_EXTENT_TABLE; 5448 } 5449 5450 if (!internal_xattrs) { 5451 blob_xattrs_init(&internal_xattrs_default); 5452 internal_xattrs = &internal_xattrs_default; 5453 } 5454 5455 rc = blob_set_xattrs(blob, &opts_local.xattrs, false); 5456 if (rc < 0) { 5457 blob_free(blob); 5458 spdk_bit_array_clear(bs->used_blobids, page_idx); 5459 bs_release_md_page(bs, page_idx); 5460 cb_fn(cb_arg, 0, rc); 5461 return; 5462 } 5463 5464 rc = blob_set_xattrs(blob, internal_xattrs, true); 5465 if (rc < 0) { 5466 blob_free(blob); 5467 spdk_bit_array_clear(bs->used_blobids, page_idx); 5468 bs_release_md_page(bs, page_idx); 5469 cb_fn(cb_arg, 0, rc); 5470 return; 5471 } 5472 5473 if (opts_local.thin_provision) { 5474 blob_set_thin_provision(blob); 5475 } 5476 5477 blob_set_clear_method(blob, opts_local.clear_method); 5478 5479 rc = blob_resize(blob, opts_local.num_clusters); 5480 if (rc < 0) { 5481 blob_free(blob); 5482 spdk_bit_array_clear(bs->used_blobids, page_idx); 5483 bs_release_md_page(bs, page_idx); 5484 cb_fn(cb_arg, 0, rc); 5485 return; 5486 } 5487 cpl.type = SPDK_BS_CPL_TYPE_BLOBID; 5488 cpl.u.blobid.cb_fn = cb_fn; 5489 cpl.u.blobid.cb_arg = cb_arg; 5490 cpl.u.blobid.blobid = blob->id; 5491 5492 seq = bs_sequence_start(bs->md_channel, &cpl); 5493 if (!seq) { 5494 blob_free(blob); 5495 spdk_bit_array_clear(bs->used_blobids, page_idx); 5496 bs_release_md_page(bs, page_idx); 5497 cb_fn(cb_arg, 0, -ENOMEM); 5498 return; 5499 } 5500 5501 blob_persist(seq, blob, bs_create_blob_cpl, blob); 5502 } 5503 5504 void spdk_bs_create_blob(struct spdk_blob_store *bs, 5505 spdk_blob_op_with_id_complete cb_fn, void *cb_arg) 5506 { 5507 bs_create_blob(bs, NULL, NULL, cb_fn, cb_arg); 5508 } 5509 5510 void spdk_bs_create_blob_ext(struct spdk_blob_store *bs, const struct spdk_blob_opts *opts, 5511 spdk_blob_op_with_id_complete cb_fn, void *cb_arg) 5512 { 5513 bs_create_blob(bs, opts, NULL, cb_fn, cb_arg); 5514 } 5515 5516 /* END spdk_bs_create_blob */ 5517 5518 /* START blob_cleanup */ 5519 5520 struct spdk_clone_snapshot_ctx { 5521 struct spdk_bs_cpl cpl; 5522 int bserrno; 5523 bool frozen; 5524 5525 struct spdk_io_channel *channel; 5526 5527 /* Current cluster for inflate operation */ 5528 uint64_t cluster; 5529 5530 /* For inflation force allocation of all unallocated clusters and remove 5531 * thin-provisioning. Otherwise only decouple parent and keep clone thin. */ 5532 bool allocate_all; 5533 5534 struct { 5535 spdk_blob_id id; 5536 struct spdk_blob *blob; 5537 } original; 5538 struct { 5539 spdk_blob_id id; 5540 struct spdk_blob *blob; 5541 } new; 5542 5543 /* xattrs specified for snapshot/clones only. They have no impact on 5544 * the original blobs xattrs. */ 5545 const struct spdk_blob_xattr_opts *xattrs; 5546 }; 5547 5548 static void 5549 bs_clone_snapshot_cleanup_finish(void *cb_arg, int bserrno) 5550 { 5551 struct spdk_clone_snapshot_ctx *ctx = cb_arg; 5552 struct spdk_bs_cpl *cpl = &ctx->cpl; 5553 5554 if (bserrno != 0) { 5555 if (ctx->bserrno != 0) { 5556 SPDK_ERRLOG("Cleanup error %d\n", bserrno); 5557 } else { 5558 ctx->bserrno = bserrno; 5559 } 5560 } 5561 5562 switch (cpl->type) { 5563 case SPDK_BS_CPL_TYPE_BLOBID: 5564 cpl->u.blobid.cb_fn(cpl->u.blobid.cb_arg, cpl->u.blobid.blobid, ctx->bserrno); 5565 break; 5566 case SPDK_BS_CPL_TYPE_BLOB_BASIC: 5567 cpl->u.blob_basic.cb_fn(cpl->u.blob_basic.cb_arg, ctx->bserrno); 5568 break; 5569 default: 5570 SPDK_UNREACHABLE(); 5571 break; 5572 } 5573 5574 free(ctx); 5575 } 5576 5577 static void 5578 bs_snapshot_unfreeze_cpl(void *cb_arg, int bserrno) 5579 { 5580 struct spdk_clone_snapshot_ctx *ctx = (struct spdk_clone_snapshot_ctx *)cb_arg; 5581 struct spdk_blob *origblob = ctx->original.blob; 5582 5583 if (bserrno != 0) { 5584 if (ctx->bserrno != 0) { 5585 SPDK_ERRLOG("Unfreeze error %d\n", bserrno); 5586 } else { 5587 ctx->bserrno = bserrno; 5588 } 5589 } 5590 5591 ctx->original.id = origblob->id; 5592 origblob->locked_operation_in_progress = false; 5593 5594 spdk_blob_close(origblob, bs_clone_snapshot_cleanup_finish, ctx); 5595 } 5596 5597 static void 5598 bs_clone_snapshot_origblob_cleanup(void *cb_arg, int bserrno) 5599 { 5600 struct spdk_clone_snapshot_ctx *ctx = (struct spdk_clone_snapshot_ctx *)cb_arg; 5601 struct spdk_blob *origblob = ctx->original.blob; 5602 5603 if (bserrno != 0) { 5604 if (ctx->bserrno != 0) { 5605 SPDK_ERRLOG("Cleanup error %d\n", bserrno); 5606 } else { 5607 ctx->bserrno = bserrno; 5608 } 5609 } 5610 5611 if (ctx->frozen) { 5612 /* Unfreeze any outstanding I/O */ 5613 blob_unfreeze_io(origblob, bs_snapshot_unfreeze_cpl, ctx); 5614 } else { 5615 bs_snapshot_unfreeze_cpl(ctx, 0); 5616 } 5617 5618 } 5619 5620 static void 5621 bs_clone_snapshot_newblob_cleanup(struct spdk_clone_snapshot_ctx *ctx, int bserrno) 5622 { 5623 struct spdk_blob *newblob = ctx->new.blob; 5624 5625 if (bserrno != 0) { 5626 if (ctx->bserrno != 0) { 5627 SPDK_ERRLOG("Cleanup error %d\n", bserrno); 5628 } else { 5629 ctx->bserrno = bserrno; 5630 } 5631 } 5632 5633 ctx->new.id = newblob->id; 5634 spdk_blob_close(newblob, bs_clone_snapshot_origblob_cleanup, ctx); 5635 } 5636 5637 /* END blob_cleanup */ 5638 5639 /* START spdk_bs_create_snapshot */ 5640 5641 static void 5642 bs_snapshot_swap_cluster_maps(struct spdk_blob *blob1, struct spdk_blob *blob2) 5643 { 5644 uint64_t *cluster_temp; 5645 uint32_t *extent_page_temp; 5646 5647 cluster_temp = blob1->active.clusters; 5648 blob1->active.clusters = blob2->active.clusters; 5649 blob2->active.clusters = cluster_temp; 5650 5651 extent_page_temp = blob1->active.extent_pages; 5652 blob1->active.extent_pages = blob2->active.extent_pages; 5653 blob2->active.extent_pages = extent_page_temp; 5654 } 5655 5656 static void 5657 bs_snapshot_origblob_sync_cpl(void *cb_arg, int bserrno) 5658 { 5659 struct spdk_clone_snapshot_ctx *ctx = (struct spdk_clone_snapshot_ctx *)cb_arg; 5660 struct spdk_blob *origblob = ctx->original.blob; 5661 struct spdk_blob *newblob = ctx->new.blob; 5662 5663 if (bserrno != 0) { 5664 bs_snapshot_swap_cluster_maps(newblob, origblob); 5665 bs_clone_snapshot_origblob_cleanup(ctx, bserrno); 5666 return; 5667 } 5668 5669 /* Remove metadata descriptor SNAPSHOT_IN_PROGRESS */ 5670 bserrno = blob_remove_xattr(newblob, SNAPSHOT_IN_PROGRESS, true); 5671 if (bserrno != 0) { 5672 bs_clone_snapshot_origblob_cleanup(ctx, bserrno); 5673 return; 5674 } 5675 5676 bs_blob_list_add(ctx->original.blob); 5677 5678 spdk_blob_set_read_only(newblob); 5679 5680 /* sync snapshot metadata */ 5681 spdk_blob_sync_md(newblob, bs_clone_snapshot_origblob_cleanup, ctx); 5682 } 5683 5684 static void 5685 bs_snapshot_newblob_sync_cpl(void *cb_arg, int bserrno) 5686 { 5687 struct spdk_clone_snapshot_ctx *ctx = (struct spdk_clone_snapshot_ctx *)cb_arg; 5688 struct spdk_blob *origblob = ctx->original.blob; 5689 struct spdk_blob *newblob = ctx->new.blob; 5690 5691 if (bserrno != 0) { 5692 /* return cluster map back to original */ 5693 bs_snapshot_swap_cluster_maps(newblob, origblob); 5694 5695 /* Newblob md sync failed. Valid clusters are only present in origblob. 5696 * Since I/O is frozen on origblob, not changes to zeroed out cluster map should have occured. 5697 * Newblob needs to be reverted to thin_provisioned state at creation to properly close. */ 5698 blob_set_thin_provision(newblob); 5699 assert(spdk_mem_all_zero(newblob->active.clusters, 5700 newblob->active.num_clusters * sizeof(*newblob->active.clusters))); 5701 assert(spdk_mem_all_zero(newblob->active.extent_pages, 5702 newblob->active.num_extent_pages * sizeof(*newblob->active.extent_pages))); 5703 5704 bs_clone_snapshot_newblob_cleanup(ctx, bserrno); 5705 return; 5706 } 5707 5708 /* Set internal xattr for snapshot id */ 5709 bserrno = blob_set_xattr(origblob, BLOB_SNAPSHOT, &newblob->id, sizeof(spdk_blob_id), true); 5710 if (bserrno != 0) { 5711 /* return cluster map back to original */ 5712 bs_snapshot_swap_cluster_maps(newblob, origblob); 5713 blob_set_thin_provision(newblob); 5714 bs_clone_snapshot_newblob_cleanup(ctx, bserrno); 5715 return; 5716 } 5717 5718 /* Create new back_bs_dev for snapshot */ 5719 origblob->back_bs_dev = bs_create_blob_bs_dev(newblob); 5720 if (origblob->back_bs_dev == NULL) { 5721 /* return cluster map back to original */ 5722 bs_snapshot_swap_cluster_maps(newblob, origblob); 5723 blob_set_thin_provision(newblob); 5724 bs_clone_snapshot_newblob_cleanup(ctx, -EINVAL); 5725 return; 5726 } 5727 5728 bs_blob_list_remove(origblob); 5729 origblob->parent_id = newblob->id; 5730 /* set clone blob as thin provisioned */ 5731 blob_set_thin_provision(origblob); 5732 5733 bs_blob_list_add(newblob); 5734 5735 /* sync clone metadata */ 5736 spdk_blob_sync_md(origblob, bs_snapshot_origblob_sync_cpl, ctx); 5737 } 5738 5739 static void 5740 bs_snapshot_freeze_cpl(void *cb_arg, int rc) 5741 { 5742 struct spdk_clone_snapshot_ctx *ctx = (struct spdk_clone_snapshot_ctx *)cb_arg; 5743 struct spdk_blob *origblob = ctx->original.blob; 5744 struct spdk_blob *newblob = ctx->new.blob; 5745 int bserrno; 5746 5747 if (rc != 0) { 5748 bs_clone_snapshot_newblob_cleanup(ctx, rc); 5749 return; 5750 } 5751 5752 ctx->frozen = true; 5753 5754 /* set new back_bs_dev for snapshot */ 5755 newblob->back_bs_dev = origblob->back_bs_dev; 5756 /* Set invalid flags from origblob */ 5757 newblob->invalid_flags = origblob->invalid_flags; 5758 5759 /* inherit parent from original blob if set */ 5760 newblob->parent_id = origblob->parent_id; 5761 if (origblob->parent_id != SPDK_BLOBID_INVALID) { 5762 /* Set internal xattr for snapshot id */ 5763 bserrno = blob_set_xattr(newblob, BLOB_SNAPSHOT, 5764 &origblob->parent_id, sizeof(spdk_blob_id), true); 5765 if (bserrno != 0) { 5766 bs_clone_snapshot_newblob_cleanup(ctx, bserrno); 5767 return; 5768 } 5769 } 5770 5771 /* swap cluster maps */ 5772 bs_snapshot_swap_cluster_maps(newblob, origblob); 5773 5774 /* Set the clear method on the new blob to match the original. */ 5775 blob_set_clear_method(newblob, origblob->clear_method); 5776 5777 /* sync snapshot metadata */ 5778 spdk_blob_sync_md(newblob, bs_snapshot_newblob_sync_cpl, ctx); 5779 } 5780 5781 static void 5782 bs_snapshot_newblob_open_cpl(void *cb_arg, struct spdk_blob *_blob, int bserrno) 5783 { 5784 struct spdk_clone_snapshot_ctx *ctx = (struct spdk_clone_snapshot_ctx *)cb_arg; 5785 struct spdk_blob *origblob = ctx->original.blob; 5786 struct spdk_blob *newblob = _blob; 5787 5788 if (bserrno != 0) { 5789 bs_clone_snapshot_origblob_cleanup(ctx, bserrno); 5790 return; 5791 } 5792 5793 ctx->new.blob = newblob; 5794 assert(spdk_blob_is_thin_provisioned(newblob)); 5795 assert(spdk_mem_all_zero(newblob->active.clusters, 5796 newblob->active.num_clusters * sizeof(*newblob->active.clusters))); 5797 assert(spdk_mem_all_zero(newblob->active.extent_pages, 5798 newblob->active.num_extent_pages * sizeof(*newblob->active.extent_pages))); 5799 5800 blob_freeze_io(origblob, bs_snapshot_freeze_cpl, ctx); 5801 } 5802 5803 static void 5804 bs_snapshot_newblob_create_cpl(void *cb_arg, spdk_blob_id blobid, int bserrno) 5805 { 5806 struct spdk_clone_snapshot_ctx *ctx = (struct spdk_clone_snapshot_ctx *)cb_arg; 5807 struct spdk_blob *origblob = ctx->original.blob; 5808 5809 if (bserrno != 0) { 5810 bs_clone_snapshot_origblob_cleanup(ctx, bserrno); 5811 return; 5812 } 5813 5814 ctx->new.id = blobid; 5815 ctx->cpl.u.blobid.blobid = blobid; 5816 5817 spdk_bs_open_blob(origblob->bs, ctx->new.id, bs_snapshot_newblob_open_cpl, ctx); 5818 } 5819 5820 5821 static void 5822 bs_xattr_snapshot(void *arg, const char *name, 5823 const void **value, size_t *value_len) 5824 { 5825 assert(strncmp(name, SNAPSHOT_IN_PROGRESS, sizeof(SNAPSHOT_IN_PROGRESS)) == 0); 5826 5827 struct spdk_blob *blob = (struct spdk_blob *)arg; 5828 *value = &blob->id; 5829 *value_len = sizeof(blob->id); 5830 } 5831 5832 static void 5833 bs_snapshot_origblob_open_cpl(void *cb_arg, struct spdk_blob *_blob, int bserrno) 5834 { 5835 struct spdk_clone_snapshot_ctx *ctx = (struct spdk_clone_snapshot_ctx *)cb_arg; 5836 struct spdk_blob_opts opts; 5837 struct spdk_blob_xattr_opts internal_xattrs; 5838 char *xattrs_names[] = { SNAPSHOT_IN_PROGRESS }; 5839 5840 if (bserrno != 0) { 5841 bs_clone_snapshot_cleanup_finish(ctx, bserrno); 5842 return; 5843 } 5844 5845 ctx->original.blob = _blob; 5846 5847 if (_blob->data_ro || _blob->md_ro) { 5848 SPDK_DEBUGLOG(blob, "Cannot create snapshot from read only blob with id %" PRIu64 "\n", 5849 _blob->id); 5850 ctx->bserrno = -EINVAL; 5851 spdk_blob_close(_blob, bs_clone_snapshot_cleanup_finish, ctx); 5852 return; 5853 } 5854 5855 if (_blob->locked_operation_in_progress) { 5856 SPDK_DEBUGLOG(blob, "Cannot create snapshot - another operation in progress\n"); 5857 ctx->bserrno = -EBUSY; 5858 spdk_blob_close(_blob, bs_clone_snapshot_cleanup_finish, ctx); 5859 return; 5860 } 5861 5862 _blob->locked_operation_in_progress = true; 5863 5864 spdk_blob_opts_init(&opts, sizeof(opts)); 5865 blob_xattrs_init(&internal_xattrs); 5866 5867 /* Change the size of new blob to the same as in original blob, 5868 * but do not allocate clusters */ 5869 opts.thin_provision = true; 5870 opts.num_clusters = spdk_blob_get_num_clusters(_blob); 5871 opts.use_extent_table = _blob->use_extent_table; 5872 5873 /* If there are any xattrs specified for snapshot, set them now */ 5874 if (ctx->xattrs) { 5875 memcpy(&opts.xattrs, ctx->xattrs, sizeof(*ctx->xattrs)); 5876 } 5877 /* Set internal xattr SNAPSHOT_IN_PROGRESS */ 5878 internal_xattrs.count = 1; 5879 internal_xattrs.ctx = _blob; 5880 internal_xattrs.names = xattrs_names; 5881 internal_xattrs.get_value = bs_xattr_snapshot; 5882 5883 bs_create_blob(_blob->bs, &opts, &internal_xattrs, 5884 bs_snapshot_newblob_create_cpl, ctx); 5885 } 5886 5887 void spdk_bs_create_snapshot(struct spdk_blob_store *bs, spdk_blob_id blobid, 5888 const struct spdk_blob_xattr_opts *snapshot_xattrs, 5889 spdk_blob_op_with_id_complete cb_fn, void *cb_arg) 5890 { 5891 struct spdk_clone_snapshot_ctx *ctx = calloc(1, sizeof(*ctx)); 5892 5893 if (!ctx) { 5894 cb_fn(cb_arg, SPDK_BLOBID_INVALID, -ENOMEM); 5895 return; 5896 } 5897 ctx->cpl.type = SPDK_BS_CPL_TYPE_BLOBID; 5898 ctx->cpl.u.blobid.cb_fn = cb_fn; 5899 ctx->cpl.u.blobid.cb_arg = cb_arg; 5900 ctx->cpl.u.blobid.blobid = SPDK_BLOBID_INVALID; 5901 ctx->bserrno = 0; 5902 ctx->frozen = false; 5903 ctx->original.id = blobid; 5904 ctx->xattrs = snapshot_xattrs; 5905 5906 spdk_bs_open_blob(bs, ctx->original.id, bs_snapshot_origblob_open_cpl, ctx); 5907 } 5908 /* END spdk_bs_create_snapshot */ 5909 5910 /* START spdk_bs_create_clone */ 5911 5912 static void 5913 bs_xattr_clone(void *arg, const char *name, 5914 const void **value, size_t *value_len) 5915 { 5916 assert(strncmp(name, BLOB_SNAPSHOT, sizeof(BLOB_SNAPSHOT)) == 0); 5917 5918 struct spdk_blob *blob = (struct spdk_blob *)arg; 5919 *value = &blob->id; 5920 *value_len = sizeof(blob->id); 5921 } 5922 5923 static void 5924 bs_clone_newblob_open_cpl(void *cb_arg, struct spdk_blob *_blob, int bserrno) 5925 { 5926 struct spdk_clone_snapshot_ctx *ctx = (struct spdk_clone_snapshot_ctx *)cb_arg; 5927 struct spdk_blob *clone = _blob; 5928 5929 ctx->new.blob = clone; 5930 bs_blob_list_add(clone); 5931 5932 spdk_blob_close(clone, bs_clone_snapshot_origblob_cleanup, ctx); 5933 } 5934 5935 static void 5936 bs_clone_newblob_create_cpl(void *cb_arg, spdk_blob_id blobid, int bserrno) 5937 { 5938 struct spdk_clone_snapshot_ctx *ctx = (struct spdk_clone_snapshot_ctx *)cb_arg; 5939 5940 ctx->cpl.u.blobid.blobid = blobid; 5941 spdk_bs_open_blob(ctx->original.blob->bs, blobid, bs_clone_newblob_open_cpl, ctx); 5942 } 5943 5944 static void 5945 bs_clone_origblob_open_cpl(void *cb_arg, struct spdk_blob *_blob, int bserrno) 5946 { 5947 struct spdk_clone_snapshot_ctx *ctx = (struct spdk_clone_snapshot_ctx *)cb_arg; 5948 struct spdk_blob_opts opts; 5949 struct spdk_blob_xattr_opts internal_xattrs; 5950 char *xattr_names[] = { BLOB_SNAPSHOT }; 5951 5952 if (bserrno != 0) { 5953 bs_clone_snapshot_cleanup_finish(ctx, bserrno); 5954 return; 5955 } 5956 5957 ctx->original.blob = _blob; 5958 5959 if (!_blob->data_ro || !_blob->md_ro) { 5960 SPDK_DEBUGLOG(blob, "Clone not from read-only blob\n"); 5961 ctx->bserrno = -EINVAL; 5962 spdk_blob_close(_blob, bs_clone_snapshot_cleanup_finish, ctx); 5963 return; 5964 } 5965 5966 if (_blob->locked_operation_in_progress) { 5967 SPDK_DEBUGLOG(blob, "Cannot create clone - another operation in progress\n"); 5968 ctx->bserrno = -EBUSY; 5969 spdk_blob_close(_blob, bs_clone_snapshot_cleanup_finish, ctx); 5970 return; 5971 } 5972 5973 _blob->locked_operation_in_progress = true; 5974 5975 spdk_blob_opts_init(&opts, sizeof(opts)); 5976 blob_xattrs_init(&internal_xattrs); 5977 5978 opts.thin_provision = true; 5979 opts.num_clusters = spdk_blob_get_num_clusters(_blob); 5980 opts.use_extent_table = _blob->use_extent_table; 5981 if (ctx->xattrs) { 5982 memcpy(&opts.xattrs, ctx->xattrs, sizeof(*ctx->xattrs)); 5983 } 5984 5985 /* Set internal xattr BLOB_SNAPSHOT */ 5986 internal_xattrs.count = 1; 5987 internal_xattrs.ctx = _blob; 5988 internal_xattrs.names = xattr_names; 5989 internal_xattrs.get_value = bs_xattr_clone; 5990 5991 bs_create_blob(_blob->bs, &opts, &internal_xattrs, 5992 bs_clone_newblob_create_cpl, ctx); 5993 } 5994 5995 void spdk_bs_create_clone(struct spdk_blob_store *bs, spdk_blob_id blobid, 5996 const struct spdk_blob_xattr_opts *clone_xattrs, 5997 spdk_blob_op_with_id_complete cb_fn, void *cb_arg) 5998 { 5999 struct spdk_clone_snapshot_ctx *ctx = calloc(1, sizeof(*ctx)); 6000 6001 if (!ctx) { 6002 cb_fn(cb_arg, SPDK_BLOBID_INVALID, -ENOMEM); 6003 return; 6004 } 6005 6006 ctx->cpl.type = SPDK_BS_CPL_TYPE_BLOBID; 6007 ctx->cpl.u.blobid.cb_fn = cb_fn; 6008 ctx->cpl.u.blobid.cb_arg = cb_arg; 6009 ctx->cpl.u.blobid.blobid = SPDK_BLOBID_INVALID; 6010 ctx->bserrno = 0; 6011 ctx->xattrs = clone_xattrs; 6012 ctx->original.id = blobid; 6013 6014 spdk_bs_open_blob(bs, ctx->original.id, bs_clone_origblob_open_cpl, ctx); 6015 } 6016 6017 /* END spdk_bs_create_clone */ 6018 6019 /* START spdk_bs_inflate_blob */ 6020 6021 static void 6022 bs_inflate_blob_set_parent_cpl(void *cb_arg, struct spdk_blob *_parent, int bserrno) 6023 { 6024 struct spdk_clone_snapshot_ctx *ctx = (struct spdk_clone_snapshot_ctx *)cb_arg; 6025 struct spdk_blob *_blob = ctx->original.blob; 6026 6027 if (bserrno != 0) { 6028 bs_clone_snapshot_origblob_cleanup(ctx, bserrno); 6029 return; 6030 } 6031 6032 assert(_parent != NULL); 6033 6034 bs_blob_list_remove(_blob); 6035 _blob->parent_id = _parent->id; 6036 blob_set_xattr(_blob, BLOB_SNAPSHOT, &_blob->parent_id, 6037 sizeof(spdk_blob_id), true); 6038 6039 _blob->back_bs_dev->destroy(_blob->back_bs_dev); 6040 _blob->back_bs_dev = bs_create_blob_bs_dev(_parent); 6041 bs_blob_list_add(_blob); 6042 6043 spdk_blob_sync_md(_blob, bs_clone_snapshot_origblob_cleanup, ctx); 6044 } 6045 6046 static void 6047 bs_inflate_blob_done(struct spdk_clone_snapshot_ctx *ctx) 6048 { 6049 struct spdk_blob *_blob = ctx->original.blob; 6050 struct spdk_blob *_parent; 6051 6052 if (ctx->allocate_all) { 6053 /* remove thin provisioning */ 6054 bs_blob_list_remove(_blob); 6055 blob_remove_xattr(_blob, BLOB_SNAPSHOT, true); 6056 _blob->invalid_flags = _blob->invalid_flags & ~SPDK_BLOB_THIN_PROV; 6057 _blob->back_bs_dev->destroy(_blob->back_bs_dev); 6058 _blob->back_bs_dev = NULL; 6059 _blob->parent_id = SPDK_BLOBID_INVALID; 6060 } else { 6061 _parent = ((struct spdk_blob_bs_dev *)(_blob->back_bs_dev))->blob; 6062 if (_parent->parent_id != SPDK_BLOBID_INVALID) { 6063 /* We must change the parent of the inflated blob */ 6064 spdk_bs_open_blob(_blob->bs, _parent->parent_id, 6065 bs_inflate_blob_set_parent_cpl, ctx); 6066 return; 6067 } 6068 6069 bs_blob_list_remove(_blob); 6070 blob_remove_xattr(_blob, BLOB_SNAPSHOT, true); 6071 _blob->parent_id = SPDK_BLOBID_INVALID; 6072 _blob->back_bs_dev->destroy(_blob->back_bs_dev); 6073 _blob->back_bs_dev = bs_create_zeroes_dev(); 6074 } 6075 6076 _blob->state = SPDK_BLOB_STATE_DIRTY; 6077 spdk_blob_sync_md(_blob, bs_clone_snapshot_origblob_cleanup, ctx); 6078 } 6079 6080 /* Check if cluster needs allocation */ 6081 static inline bool 6082 bs_cluster_needs_allocation(struct spdk_blob *blob, uint64_t cluster, bool allocate_all) 6083 { 6084 struct spdk_blob_bs_dev *b; 6085 6086 assert(blob != NULL); 6087 6088 if (blob->active.clusters[cluster] != 0) { 6089 /* Cluster is already allocated */ 6090 return false; 6091 } 6092 6093 if (blob->parent_id == SPDK_BLOBID_INVALID) { 6094 /* Blob have no parent blob */ 6095 return allocate_all; 6096 } 6097 6098 b = (struct spdk_blob_bs_dev *)blob->back_bs_dev; 6099 return (allocate_all || b->blob->active.clusters[cluster] != 0); 6100 } 6101 6102 static void 6103 bs_inflate_blob_touch_next(void *cb_arg, int bserrno) 6104 { 6105 struct spdk_clone_snapshot_ctx *ctx = (struct spdk_clone_snapshot_ctx *)cb_arg; 6106 struct spdk_blob *_blob = ctx->original.blob; 6107 uint64_t offset; 6108 6109 if (bserrno != 0) { 6110 bs_clone_snapshot_origblob_cleanup(ctx, bserrno); 6111 return; 6112 } 6113 6114 for (; ctx->cluster < _blob->active.num_clusters; ctx->cluster++) { 6115 if (bs_cluster_needs_allocation(_blob, ctx->cluster, ctx->allocate_all)) { 6116 break; 6117 } 6118 } 6119 6120 if (ctx->cluster < _blob->active.num_clusters) { 6121 offset = bs_cluster_to_lba(_blob->bs, ctx->cluster); 6122 6123 /* We may safely increment a cluster before write */ 6124 ctx->cluster++; 6125 6126 /* Use zero length write to touch a cluster */ 6127 spdk_blob_io_write(_blob, ctx->channel, NULL, offset, 0, 6128 bs_inflate_blob_touch_next, ctx); 6129 } else { 6130 bs_inflate_blob_done(ctx); 6131 } 6132 } 6133 6134 static void 6135 bs_inflate_blob_open_cpl(void *cb_arg, struct spdk_blob *_blob, int bserrno) 6136 { 6137 struct spdk_clone_snapshot_ctx *ctx = (struct spdk_clone_snapshot_ctx *)cb_arg; 6138 uint64_t clusters_needed; 6139 uint64_t i; 6140 6141 if (bserrno != 0) { 6142 bs_clone_snapshot_cleanup_finish(ctx, bserrno); 6143 return; 6144 } 6145 6146 ctx->original.blob = _blob; 6147 6148 if (_blob->locked_operation_in_progress) { 6149 SPDK_DEBUGLOG(blob, "Cannot inflate blob - another operation in progress\n"); 6150 ctx->bserrno = -EBUSY; 6151 spdk_blob_close(_blob, bs_clone_snapshot_cleanup_finish, ctx); 6152 return; 6153 } 6154 6155 _blob->locked_operation_in_progress = true; 6156 6157 if (!ctx->allocate_all && _blob->parent_id == SPDK_BLOBID_INVALID) { 6158 /* This blob have no parent, so we cannot decouple it. */ 6159 SPDK_ERRLOG("Cannot decouple parent of blob with no parent.\n"); 6160 bs_clone_snapshot_origblob_cleanup(ctx, -EINVAL); 6161 return; 6162 } 6163 6164 if (spdk_blob_is_thin_provisioned(_blob) == false) { 6165 /* This is not thin provisioned blob. No need to inflate. */ 6166 bs_clone_snapshot_origblob_cleanup(ctx, 0); 6167 return; 6168 } 6169 6170 /* Do two passes - one to verify that we can obtain enough clusters 6171 * and another to actually claim them. 6172 */ 6173 clusters_needed = 0; 6174 for (i = 0; i < _blob->active.num_clusters; i++) { 6175 if (bs_cluster_needs_allocation(_blob, i, ctx->allocate_all)) { 6176 clusters_needed++; 6177 } 6178 } 6179 6180 if (clusters_needed > _blob->bs->num_free_clusters) { 6181 /* Not enough free clusters. Cannot satisfy the request. */ 6182 bs_clone_snapshot_origblob_cleanup(ctx, -ENOSPC); 6183 return; 6184 } 6185 6186 ctx->cluster = 0; 6187 bs_inflate_blob_touch_next(ctx, 0); 6188 } 6189 6190 static void 6191 bs_inflate_blob(struct spdk_blob_store *bs, struct spdk_io_channel *channel, 6192 spdk_blob_id blobid, bool allocate_all, spdk_blob_op_complete cb_fn, void *cb_arg) 6193 { 6194 struct spdk_clone_snapshot_ctx *ctx = calloc(1, sizeof(*ctx)); 6195 6196 if (!ctx) { 6197 cb_fn(cb_arg, -ENOMEM); 6198 return; 6199 } 6200 ctx->cpl.type = SPDK_BS_CPL_TYPE_BLOB_BASIC; 6201 ctx->cpl.u.bs_basic.cb_fn = cb_fn; 6202 ctx->cpl.u.bs_basic.cb_arg = cb_arg; 6203 ctx->bserrno = 0; 6204 ctx->original.id = blobid; 6205 ctx->channel = channel; 6206 ctx->allocate_all = allocate_all; 6207 6208 spdk_bs_open_blob(bs, ctx->original.id, bs_inflate_blob_open_cpl, ctx); 6209 } 6210 6211 void 6212 spdk_bs_inflate_blob(struct spdk_blob_store *bs, struct spdk_io_channel *channel, 6213 spdk_blob_id blobid, spdk_blob_op_complete cb_fn, void *cb_arg) 6214 { 6215 bs_inflate_blob(bs, channel, blobid, true, cb_fn, cb_arg); 6216 } 6217 6218 void 6219 spdk_bs_blob_decouple_parent(struct spdk_blob_store *bs, struct spdk_io_channel *channel, 6220 spdk_blob_id blobid, spdk_blob_op_complete cb_fn, void *cb_arg) 6221 { 6222 bs_inflate_blob(bs, channel, blobid, false, cb_fn, cb_arg); 6223 } 6224 /* END spdk_bs_inflate_blob */ 6225 6226 /* START spdk_blob_resize */ 6227 struct spdk_bs_resize_ctx { 6228 spdk_blob_op_complete cb_fn; 6229 void *cb_arg; 6230 struct spdk_blob *blob; 6231 uint64_t sz; 6232 int rc; 6233 }; 6234 6235 static void 6236 bs_resize_unfreeze_cpl(void *cb_arg, int rc) 6237 { 6238 struct spdk_bs_resize_ctx *ctx = (struct spdk_bs_resize_ctx *)cb_arg; 6239 6240 if (rc != 0) { 6241 SPDK_ERRLOG("Unfreeze failed, rc=%d\n", rc); 6242 } 6243 6244 if (ctx->rc != 0) { 6245 SPDK_ERRLOG("Unfreeze failed, ctx->rc=%d\n", ctx->rc); 6246 rc = ctx->rc; 6247 } 6248 6249 ctx->blob->locked_operation_in_progress = false; 6250 6251 ctx->cb_fn(ctx->cb_arg, rc); 6252 free(ctx); 6253 } 6254 6255 static void 6256 bs_resize_freeze_cpl(void *cb_arg, int rc) 6257 { 6258 struct spdk_bs_resize_ctx *ctx = (struct spdk_bs_resize_ctx *)cb_arg; 6259 6260 if (rc != 0) { 6261 ctx->blob->locked_operation_in_progress = false; 6262 ctx->cb_fn(ctx->cb_arg, rc); 6263 free(ctx); 6264 return; 6265 } 6266 6267 ctx->rc = blob_resize(ctx->blob, ctx->sz); 6268 6269 blob_unfreeze_io(ctx->blob, bs_resize_unfreeze_cpl, ctx); 6270 } 6271 6272 void 6273 spdk_blob_resize(struct spdk_blob *blob, uint64_t sz, spdk_blob_op_complete cb_fn, void *cb_arg) 6274 { 6275 struct spdk_bs_resize_ctx *ctx; 6276 6277 blob_verify_md_op(blob); 6278 6279 SPDK_DEBUGLOG(blob, "Resizing blob %" PRIu64 " to %" PRIu64 " clusters\n", blob->id, sz); 6280 6281 if (blob->md_ro) { 6282 cb_fn(cb_arg, -EPERM); 6283 return; 6284 } 6285 6286 if (sz == blob->active.num_clusters) { 6287 cb_fn(cb_arg, 0); 6288 return; 6289 } 6290 6291 if (blob->locked_operation_in_progress) { 6292 cb_fn(cb_arg, -EBUSY); 6293 return; 6294 } 6295 6296 ctx = calloc(1, sizeof(*ctx)); 6297 if (!ctx) { 6298 cb_fn(cb_arg, -ENOMEM); 6299 return; 6300 } 6301 6302 blob->locked_operation_in_progress = true; 6303 ctx->cb_fn = cb_fn; 6304 ctx->cb_arg = cb_arg; 6305 ctx->blob = blob; 6306 ctx->sz = sz; 6307 blob_freeze_io(blob, bs_resize_freeze_cpl, ctx); 6308 } 6309 6310 /* END spdk_blob_resize */ 6311 6312 6313 /* START spdk_bs_delete_blob */ 6314 6315 static void 6316 bs_delete_close_cpl(void *cb_arg, int bserrno) 6317 { 6318 spdk_bs_sequence_t *seq = cb_arg; 6319 6320 bs_sequence_finish(seq, bserrno); 6321 } 6322 6323 static void 6324 bs_delete_persist_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 6325 { 6326 struct spdk_blob *blob = cb_arg; 6327 6328 if (bserrno != 0) { 6329 /* 6330 * We already removed this blob from the blobstore tailq, so 6331 * we need to free it here since this is the last reference 6332 * to it. 6333 */ 6334 blob_free(blob); 6335 bs_delete_close_cpl(seq, bserrno); 6336 return; 6337 } 6338 6339 /* 6340 * This will immediately decrement the ref_count and call 6341 * the completion routine since the metadata state is clean. 6342 * By calling spdk_blob_close, we reduce the number of call 6343 * points into code that touches the blob->open_ref count 6344 * and the blobstore's blob list. 6345 */ 6346 spdk_blob_close(blob, bs_delete_close_cpl, seq); 6347 } 6348 6349 struct delete_snapshot_ctx { 6350 struct spdk_blob_list *parent_snapshot_entry; 6351 struct spdk_blob *snapshot; 6352 bool snapshot_md_ro; 6353 struct spdk_blob *clone; 6354 bool clone_md_ro; 6355 spdk_blob_op_with_handle_complete cb_fn; 6356 void *cb_arg; 6357 int bserrno; 6358 uint32_t next_extent_page; 6359 }; 6360 6361 static void 6362 delete_blob_cleanup_finish(void *cb_arg, int bserrno) 6363 { 6364 struct delete_snapshot_ctx *ctx = cb_arg; 6365 6366 if (bserrno != 0) { 6367 SPDK_ERRLOG("Snapshot cleanup error %d\n", bserrno); 6368 } 6369 6370 assert(ctx != NULL); 6371 6372 if (bserrno != 0 && ctx->bserrno == 0) { 6373 ctx->bserrno = bserrno; 6374 } 6375 6376 ctx->cb_fn(ctx->cb_arg, ctx->snapshot, ctx->bserrno); 6377 free(ctx); 6378 } 6379 6380 static void 6381 delete_snapshot_cleanup_snapshot(void *cb_arg, int bserrno) 6382 { 6383 struct delete_snapshot_ctx *ctx = cb_arg; 6384 6385 if (bserrno != 0) { 6386 ctx->bserrno = bserrno; 6387 SPDK_ERRLOG("Clone cleanup error %d\n", bserrno); 6388 } 6389 6390 if (ctx->bserrno != 0) { 6391 assert(blob_lookup(ctx->snapshot->bs, ctx->snapshot->id) == NULL); 6392 TAILQ_INSERT_HEAD(&ctx->snapshot->bs->blobs, ctx->snapshot, link); 6393 spdk_bit_array_set(ctx->snapshot->bs->open_blobids, ctx->snapshot->id); 6394 } 6395 6396 ctx->snapshot->locked_operation_in_progress = false; 6397 ctx->snapshot->md_ro = ctx->snapshot_md_ro; 6398 6399 spdk_blob_close(ctx->snapshot, delete_blob_cleanup_finish, ctx); 6400 } 6401 6402 static void 6403 delete_snapshot_cleanup_clone(void *cb_arg, int bserrno) 6404 { 6405 struct delete_snapshot_ctx *ctx = cb_arg; 6406 6407 ctx->clone->locked_operation_in_progress = false; 6408 ctx->clone->md_ro = ctx->clone_md_ro; 6409 6410 spdk_blob_close(ctx->clone, delete_snapshot_cleanup_snapshot, ctx); 6411 } 6412 6413 static void 6414 delete_snapshot_unfreeze_cpl(void *cb_arg, int bserrno) 6415 { 6416 struct delete_snapshot_ctx *ctx = cb_arg; 6417 6418 if (bserrno) { 6419 ctx->bserrno = bserrno; 6420 delete_snapshot_cleanup_clone(ctx, 0); 6421 return; 6422 } 6423 6424 ctx->clone->locked_operation_in_progress = false; 6425 spdk_blob_close(ctx->clone, delete_blob_cleanup_finish, ctx); 6426 } 6427 6428 static void 6429 delete_snapshot_sync_snapshot_cpl(void *cb_arg, int bserrno) 6430 { 6431 struct delete_snapshot_ctx *ctx = cb_arg; 6432 struct spdk_blob_list *parent_snapshot_entry = NULL; 6433 struct spdk_blob_list *snapshot_entry = NULL; 6434 struct spdk_blob_list *clone_entry = NULL; 6435 struct spdk_blob_list *snapshot_clone_entry = NULL; 6436 6437 if (bserrno) { 6438 SPDK_ERRLOG("Failed to sync MD on blob\n"); 6439 ctx->bserrno = bserrno; 6440 delete_snapshot_cleanup_clone(ctx, 0); 6441 return; 6442 } 6443 6444 /* Get snapshot entry for the snapshot we want to remove */ 6445 snapshot_entry = bs_get_snapshot_entry(ctx->snapshot->bs, ctx->snapshot->id); 6446 6447 assert(snapshot_entry != NULL); 6448 6449 /* Remove clone entry in this snapshot (at this point there can be only one clone) */ 6450 clone_entry = TAILQ_FIRST(&snapshot_entry->clones); 6451 assert(clone_entry != NULL); 6452 TAILQ_REMOVE(&snapshot_entry->clones, clone_entry, link); 6453 snapshot_entry->clone_count--; 6454 assert(TAILQ_EMPTY(&snapshot_entry->clones)); 6455 6456 if (ctx->snapshot->parent_id != SPDK_BLOBID_INVALID) { 6457 /* This snapshot is at the same time a clone of another snapshot - we need to 6458 * update parent snapshot (remove current clone, add new one inherited from 6459 * the snapshot that is being removed) */ 6460 6461 /* Get snapshot entry for parent snapshot and clone entry within that snapshot for 6462 * snapshot that we are removing */ 6463 blob_get_snapshot_and_clone_entries(ctx->snapshot, &parent_snapshot_entry, 6464 &snapshot_clone_entry); 6465 6466 /* Switch clone entry in parent snapshot */ 6467 TAILQ_INSERT_TAIL(&parent_snapshot_entry->clones, clone_entry, link); 6468 TAILQ_REMOVE(&parent_snapshot_entry->clones, snapshot_clone_entry, link); 6469 free(snapshot_clone_entry); 6470 } else { 6471 /* No parent snapshot - just remove clone entry */ 6472 free(clone_entry); 6473 } 6474 6475 /* Restore md_ro flags */ 6476 ctx->clone->md_ro = ctx->clone_md_ro; 6477 ctx->snapshot->md_ro = ctx->snapshot_md_ro; 6478 6479 blob_unfreeze_io(ctx->clone, delete_snapshot_unfreeze_cpl, ctx); 6480 } 6481 6482 static void 6483 delete_snapshot_sync_clone_cpl(void *cb_arg, int bserrno) 6484 { 6485 struct delete_snapshot_ctx *ctx = cb_arg; 6486 uint64_t i; 6487 6488 ctx->snapshot->md_ro = false; 6489 6490 if (bserrno) { 6491 SPDK_ERRLOG("Failed to sync MD on clone\n"); 6492 ctx->bserrno = bserrno; 6493 6494 /* Restore snapshot to previous state */ 6495 bserrno = blob_remove_xattr(ctx->snapshot, SNAPSHOT_PENDING_REMOVAL, true); 6496 if (bserrno != 0) { 6497 delete_snapshot_cleanup_clone(ctx, bserrno); 6498 return; 6499 } 6500 6501 spdk_blob_sync_md(ctx->snapshot, delete_snapshot_cleanup_clone, ctx); 6502 return; 6503 } 6504 6505 /* Clear cluster map entries for snapshot */ 6506 for (i = 0; i < ctx->snapshot->active.num_clusters && i < ctx->clone->active.num_clusters; i++) { 6507 if (ctx->clone->active.clusters[i] == ctx->snapshot->active.clusters[i]) { 6508 ctx->snapshot->active.clusters[i] = 0; 6509 } 6510 } 6511 for (i = 0; i < ctx->snapshot->active.num_extent_pages && 6512 i < ctx->clone->active.num_extent_pages; i++) { 6513 if (ctx->clone->active.extent_pages[i] == ctx->snapshot->active.extent_pages[i]) { 6514 ctx->snapshot->active.extent_pages[i] = 0; 6515 } 6516 } 6517 6518 blob_set_thin_provision(ctx->snapshot); 6519 ctx->snapshot->state = SPDK_BLOB_STATE_DIRTY; 6520 6521 if (ctx->parent_snapshot_entry != NULL) { 6522 ctx->snapshot->back_bs_dev = NULL; 6523 } 6524 6525 spdk_blob_sync_md(ctx->snapshot, delete_snapshot_sync_snapshot_cpl, ctx); 6526 } 6527 6528 static void 6529 delete_snapshot_update_extent_pages_cpl(struct delete_snapshot_ctx *ctx) 6530 { 6531 /* Delete old backing bs_dev from clone (related to snapshot that will be removed) */ 6532 ctx->clone->back_bs_dev->destroy(ctx->clone->back_bs_dev); 6533 6534 /* Set/remove snapshot xattr and switch parent ID and backing bs_dev on clone... */ 6535 if (ctx->parent_snapshot_entry != NULL) { 6536 /* ...to parent snapshot */ 6537 ctx->clone->parent_id = ctx->parent_snapshot_entry->id; 6538 ctx->clone->back_bs_dev = ctx->snapshot->back_bs_dev; 6539 blob_set_xattr(ctx->clone, BLOB_SNAPSHOT, &ctx->parent_snapshot_entry->id, 6540 sizeof(spdk_blob_id), 6541 true); 6542 } else { 6543 /* ...to blobid invalid and zeroes dev */ 6544 ctx->clone->parent_id = SPDK_BLOBID_INVALID; 6545 ctx->clone->back_bs_dev = bs_create_zeroes_dev(); 6546 blob_remove_xattr(ctx->clone, BLOB_SNAPSHOT, true); 6547 } 6548 6549 spdk_blob_sync_md(ctx->clone, delete_snapshot_sync_clone_cpl, ctx); 6550 } 6551 6552 static void 6553 delete_snapshot_update_extent_pages(void *cb_arg, int bserrno) 6554 { 6555 struct delete_snapshot_ctx *ctx = cb_arg; 6556 uint32_t *extent_page; 6557 uint64_t i; 6558 6559 for (i = ctx->next_extent_page; i < ctx->snapshot->active.num_extent_pages && 6560 i < ctx->clone->active.num_extent_pages; i++) { 6561 if (ctx->snapshot->active.extent_pages[i] == 0) { 6562 /* No extent page to use from snapshot */ 6563 continue; 6564 } 6565 6566 extent_page = &ctx->clone->active.extent_pages[i]; 6567 if (*extent_page == 0) { 6568 /* Copy extent page from snapshot when clone did not have a matching one */ 6569 *extent_page = ctx->snapshot->active.extent_pages[i]; 6570 continue; 6571 } 6572 6573 /* Clone and snapshot both contain partialy filled matching extent pages. 6574 * Update the clone extent page in place with cluster map containing the mix of both. */ 6575 ctx->next_extent_page = i + 1; 6576 6577 blob_write_extent_page(ctx->clone, *extent_page, i * SPDK_EXTENTS_PER_EP, 6578 delete_snapshot_update_extent_pages, ctx); 6579 return; 6580 } 6581 delete_snapshot_update_extent_pages_cpl(ctx); 6582 } 6583 6584 static void 6585 delete_snapshot_sync_snapshot_xattr_cpl(void *cb_arg, int bserrno) 6586 { 6587 struct delete_snapshot_ctx *ctx = cb_arg; 6588 uint64_t i; 6589 6590 /* Temporarily override md_ro flag for clone for MD modification */ 6591 ctx->clone_md_ro = ctx->clone->md_ro; 6592 ctx->clone->md_ro = false; 6593 6594 if (bserrno) { 6595 SPDK_ERRLOG("Failed to sync MD with xattr on blob\n"); 6596 ctx->bserrno = bserrno; 6597 delete_snapshot_cleanup_clone(ctx, 0); 6598 return; 6599 } 6600 6601 /* Copy snapshot map to clone map (only unallocated clusters in clone) */ 6602 for (i = 0; i < ctx->snapshot->active.num_clusters && i < ctx->clone->active.num_clusters; i++) { 6603 if (ctx->clone->active.clusters[i] == 0) { 6604 ctx->clone->active.clusters[i] = ctx->snapshot->active.clusters[i]; 6605 } 6606 } 6607 ctx->next_extent_page = 0; 6608 delete_snapshot_update_extent_pages(ctx, 0); 6609 } 6610 6611 static void 6612 delete_snapshot_freeze_io_cb(void *cb_arg, int bserrno) 6613 { 6614 struct delete_snapshot_ctx *ctx = cb_arg; 6615 6616 if (bserrno) { 6617 SPDK_ERRLOG("Failed to freeze I/O on clone\n"); 6618 ctx->bserrno = bserrno; 6619 delete_snapshot_cleanup_clone(ctx, 0); 6620 return; 6621 } 6622 6623 /* Temporarily override md_ro flag for snapshot for MD modification */ 6624 ctx->snapshot_md_ro = ctx->snapshot->md_ro; 6625 ctx->snapshot->md_ro = false; 6626 6627 /* Mark blob as pending for removal for power failure safety, use clone id for recovery */ 6628 ctx->bserrno = blob_set_xattr(ctx->snapshot, SNAPSHOT_PENDING_REMOVAL, &ctx->clone->id, 6629 sizeof(spdk_blob_id), true); 6630 if (ctx->bserrno != 0) { 6631 delete_snapshot_cleanup_clone(ctx, 0); 6632 return; 6633 } 6634 6635 spdk_blob_sync_md(ctx->snapshot, delete_snapshot_sync_snapshot_xattr_cpl, ctx); 6636 } 6637 6638 static void 6639 delete_snapshot_open_clone_cb(void *cb_arg, struct spdk_blob *clone, int bserrno) 6640 { 6641 struct delete_snapshot_ctx *ctx = cb_arg; 6642 6643 if (bserrno) { 6644 SPDK_ERRLOG("Failed to open clone\n"); 6645 ctx->bserrno = bserrno; 6646 delete_snapshot_cleanup_snapshot(ctx, 0); 6647 return; 6648 } 6649 6650 ctx->clone = clone; 6651 6652 if (clone->locked_operation_in_progress) { 6653 SPDK_DEBUGLOG(blob, "Cannot remove blob - another operation in progress on its clone\n"); 6654 ctx->bserrno = -EBUSY; 6655 spdk_blob_close(ctx->clone, delete_snapshot_cleanup_snapshot, ctx); 6656 return; 6657 } 6658 6659 clone->locked_operation_in_progress = true; 6660 6661 blob_freeze_io(clone, delete_snapshot_freeze_io_cb, ctx); 6662 } 6663 6664 static void 6665 update_clone_on_snapshot_deletion(struct spdk_blob *snapshot, struct delete_snapshot_ctx *ctx) 6666 { 6667 struct spdk_blob_list *snapshot_entry = NULL; 6668 struct spdk_blob_list *clone_entry = NULL; 6669 struct spdk_blob_list *snapshot_clone_entry = NULL; 6670 6671 /* Get snapshot entry for the snapshot we want to remove */ 6672 snapshot_entry = bs_get_snapshot_entry(snapshot->bs, snapshot->id); 6673 6674 assert(snapshot_entry != NULL); 6675 6676 /* Get clone of the snapshot (at this point there can be only one clone) */ 6677 clone_entry = TAILQ_FIRST(&snapshot_entry->clones); 6678 assert(snapshot_entry->clone_count == 1); 6679 assert(clone_entry != NULL); 6680 6681 /* Get snapshot entry for parent snapshot and clone entry within that snapshot for 6682 * snapshot that we are removing */ 6683 blob_get_snapshot_and_clone_entries(snapshot, &ctx->parent_snapshot_entry, 6684 &snapshot_clone_entry); 6685 6686 spdk_bs_open_blob(snapshot->bs, clone_entry->id, delete_snapshot_open_clone_cb, ctx); 6687 } 6688 6689 static void 6690 bs_delete_blob_finish(void *cb_arg, struct spdk_blob *blob, int bserrno) 6691 { 6692 spdk_bs_sequence_t *seq = cb_arg; 6693 struct spdk_blob_list *snapshot_entry = NULL; 6694 uint32_t page_num; 6695 6696 if (bserrno) { 6697 SPDK_ERRLOG("Failed to remove blob\n"); 6698 bs_sequence_finish(seq, bserrno); 6699 return; 6700 } 6701 6702 /* Remove snapshot from the list */ 6703 snapshot_entry = bs_get_snapshot_entry(blob->bs, blob->id); 6704 if (snapshot_entry != NULL) { 6705 TAILQ_REMOVE(&blob->bs->snapshots, snapshot_entry, link); 6706 free(snapshot_entry); 6707 } 6708 6709 page_num = bs_blobid_to_page(blob->id); 6710 spdk_bit_array_clear(blob->bs->used_blobids, page_num); 6711 blob->state = SPDK_BLOB_STATE_DIRTY; 6712 blob->active.num_pages = 0; 6713 blob_resize(blob, 0); 6714 6715 blob_persist(seq, blob, bs_delete_persist_cpl, blob); 6716 } 6717 6718 static int 6719 bs_is_blob_deletable(struct spdk_blob *blob, bool *update_clone) 6720 { 6721 struct spdk_blob_list *snapshot_entry = NULL; 6722 struct spdk_blob_list *clone_entry = NULL; 6723 struct spdk_blob *clone = NULL; 6724 bool has_one_clone = false; 6725 6726 /* Check if this is a snapshot with clones */ 6727 snapshot_entry = bs_get_snapshot_entry(blob->bs, blob->id); 6728 if (snapshot_entry != NULL) { 6729 if (snapshot_entry->clone_count > 1) { 6730 SPDK_ERRLOG("Cannot remove snapshot with more than one clone\n"); 6731 return -EBUSY; 6732 } else if (snapshot_entry->clone_count == 1) { 6733 has_one_clone = true; 6734 } 6735 } 6736 6737 /* Check if someone has this blob open (besides this delete context): 6738 * - open_ref = 1 - only this context opened blob, so it is ok to remove it 6739 * - open_ref <= 2 && has_one_clone = true - clone is holding snapshot 6740 * and that is ok, because we will update it accordingly */ 6741 if (blob->open_ref <= 2 && has_one_clone) { 6742 clone_entry = TAILQ_FIRST(&snapshot_entry->clones); 6743 assert(clone_entry != NULL); 6744 clone = blob_lookup(blob->bs, clone_entry->id); 6745 6746 if (blob->open_ref == 2 && clone == NULL) { 6747 /* Clone is closed and someone else opened this blob */ 6748 SPDK_ERRLOG("Cannot remove snapshot because it is open\n"); 6749 return -EBUSY; 6750 } 6751 6752 *update_clone = true; 6753 return 0; 6754 } 6755 6756 if (blob->open_ref > 1) { 6757 SPDK_ERRLOG("Cannot remove snapshot because it is open\n"); 6758 return -EBUSY; 6759 } 6760 6761 assert(has_one_clone == false); 6762 *update_clone = false; 6763 return 0; 6764 } 6765 6766 static void 6767 bs_delete_enomem_close_cpl(void *cb_arg, int bserrno) 6768 { 6769 spdk_bs_sequence_t *seq = cb_arg; 6770 6771 bs_sequence_finish(seq, -ENOMEM); 6772 } 6773 6774 static void 6775 bs_delete_open_cpl(void *cb_arg, struct spdk_blob *blob, int bserrno) 6776 { 6777 spdk_bs_sequence_t *seq = cb_arg; 6778 struct delete_snapshot_ctx *ctx; 6779 bool update_clone = false; 6780 6781 if (bserrno != 0) { 6782 bs_sequence_finish(seq, bserrno); 6783 return; 6784 } 6785 6786 blob_verify_md_op(blob); 6787 6788 ctx = calloc(1, sizeof(*ctx)); 6789 if (ctx == NULL) { 6790 spdk_blob_close(blob, bs_delete_enomem_close_cpl, seq); 6791 return; 6792 } 6793 6794 ctx->snapshot = blob; 6795 ctx->cb_fn = bs_delete_blob_finish; 6796 ctx->cb_arg = seq; 6797 6798 /* Check if blob can be removed and if it is a snapshot with clone on top of it */ 6799 ctx->bserrno = bs_is_blob_deletable(blob, &update_clone); 6800 if (ctx->bserrno) { 6801 spdk_blob_close(blob, delete_blob_cleanup_finish, ctx); 6802 return; 6803 } 6804 6805 if (blob->locked_operation_in_progress) { 6806 SPDK_DEBUGLOG(blob, "Cannot remove blob - another operation in progress\n"); 6807 ctx->bserrno = -EBUSY; 6808 spdk_blob_close(blob, delete_blob_cleanup_finish, ctx); 6809 return; 6810 } 6811 6812 blob->locked_operation_in_progress = true; 6813 6814 /* 6815 * Remove the blob from the blob_store list now, to ensure it does not 6816 * get returned after this point by blob_lookup(). 6817 */ 6818 spdk_bit_array_clear(blob->bs->open_blobids, blob->id); 6819 TAILQ_REMOVE(&blob->bs->blobs, blob, link); 6820 6821 if (update_clone) { 6822 /* This blob is a snapshot with active clone - update clone first */ 6823 update_clone_on_snapshot_deletion(blob, ctx); 6824 } else { 6825 /* This blob does not have any clones - just remove it */ 6826 bs_blob_list_remove(blob); 6827 bs_delete_blob_finish(seq, blob, 0); 6828 free(ctx); 6829 } 6830 } 6831 6832 void 6833 spdk_bs_delete_blob(struct spdk_blob_store *bs, spdk_blob_id blobid, 6834 spdk_blob_op_complete cb_fn, void *cb_arg) 6835 { 6836 struct spdk_bs_cpl cpl; 6837 spdk_bs_sequence_t *seq; 6838 6839 SPDK_DEBUGLOG(blob, "Deleting blob %" PRIu64 "\n", blobid); 6840 6841 assert(spdk_get_thread() == bs->md_thread); 6842 6843 cpl.type = SPDK_BS_CPL_TYPE_BLOB_BASIC; 6844 cpl.u.blob_basic.cb_fn = cb_fn; 6845 cpl.u.blob_basic.cb_arg = cb_arg; 6846 6847 seq = bs_sequence_start(bs->md_channel, &cpl); 6848 if (!seq) { 6849 cb_fn(cb_arg, -ENOMEM); 6850 return; 6851 } 6852 6853 spdk_bs_open_blob(bs, blobid, bs_delete_open_cpl, seq); 6854 } 6855 6856 /* END spdk_bs_delete_blob */ 6857 6858 /* START spdk_bs_open_blob */ 6859 6860 static void 6861 bs_open_blob_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 6862 { 6863 struct spdk_blob *blob = cb_arg; 6864 struct spdk_blob *existing; 6865 6866 if (bserrno != 0) { 6867 blob_free(blob); 6868 seq->cpl.u.blob_handle.blob = NULL; 6869 bs_sequence_finish(seq, bserrno); 6870 return; 6871 } 6872 6873 existing = blob_lookup(blob->bs, blob->id); 6874 if (existing) { 6875 blob_free(blob); 6876 existing->open_ref++; 6877 seq->cpl.u.blob_handle.blob = existing; 6878 bs_sequence_finish(seq, 0); 6879 return; 6880 } 6881 6882 blob->open_ref++; 6883 6884 spdk_bit_array_set(blob->bs->open_blobids, blob->id); 6885 TAILQ_INSERT_HEAD(&blob->bs->blobs, blob, link); 6886 6887 bs_sequence_finish(seq, bserrno); 6888 } 6889 6890 static inline void 6891 blob_open_opts_copy(const struct spdk_blob_open_opts *src, struct spdk_blob_open_opts *dst) 6892 { 6893 #define FIELD_OK(field) \ 6894 offsetof(struct spdk_blob_opts, field) + sizeof(src->field) <= src->opts_size 6895 6896 #define SET_FIELD(field) \ 6897 if (FIELD_OK(field)) { \ 6898 dst->field = src->field; \ 6899 } \ 6900 6901 SET_FIELD(clear_method); 6902 6903 dst->opts_size = src->opts_size; 6904 6905 /* You should not remove this statement, but need to update the assert statement 6906 * if you add a new field, and also add a corresponding SET_FIELD statement */ 6907 SPDK_STATIC_ASSERT(sizeof(struct spdk_blob_open_opts) == 16, "Incorrect size"); 6908 6909 #undef FIELD_OK 6910 #undef SET_FIELD 6911 } 6912 6913 static void 6914 bs_open_blob(struct spdk_blob_store *bs, 6915 spdk_blob_id blobid, 6916 struct spdk_blob_open_opts *opts, 6917 spdk_blob_op_with_handle_complete cb_fn, 6918 void *cb_arg) 6919 { 6920 struct spdk_blob *blob; 6921 struct spdk_bs_cpl cpl; 6922 struct spdk_blob_open_opts opts_local; 6923 spdk_bs_sequence_t *seq; 6924 uint32_t page_num; 6925 6926 SPDK_DEBUGLOG(blob, "Opening blob %" PRIu64 "\n", blobid); 6927 assert(spdk_get_thread() == bs->md_thread); 6928 6929 page_num = bs_blobid_to_page(blobid); 6930 if (spdk_bit_array_get(bs->used_blobids, page_num) == false) { 6931 /* Invalid blobid */ 6932 cb_fn(cb_arg, NULL, -ENOENT); 6933 return; 6934 } 6935 6936 blob = blob_lookup(bs, blobid); 6937 if (blob) { 6938 blob->open_ref++; 6939 cb_fn(cb_arg, blob, 0); 6940 return; 6941 } 6942 6943 blob = blob_alloc(bs, blobid); 6944 if (!blob) { 6945 cb_fn(cb_arg, NULL, -ENOMEM); 6946 return; 6947 } 6948 6949 spdk_blob_open_opts_init(&opts_local, sizeof(opts_local)); 6950 if (opts) { 6951 blob_open_opts_copy(opts, &opts_local); 6952 } 6953 6954 blob->clear_method = opts_local.clear_method; 6955 6956 cpl.type = SPDK_BS_CPL_TYPE_BLOB_HANDLE; 6957 cpl.u.blob_handle.cb_fn = cb_fn; 6958 cpl.u.blob_handle.cb_arg = cb_arg; 6959 cpl.u.blob_handle.blob = blob; 6960 6961 seq = bs_sequence_start(bs->md_channel, &cpl); 6962 if (!seq) { 6963 blob_free(blob); 6964 cb_fn(cb_arg, NULL, -ENOMEM); 6965 return; 6966 } 6967 6968 blob_load(seq, blob, bs_open_blob_cpl, blob); 6969 } 6970 6971 void spdk_bs_open_blob(struct spdk_blob_store *bs, spdk_blob_id blobid, 6972 spdk_blob_op_with_handle_complete cb_fn, void *cb_arg) 6973 { 6974 bs_open_blob(bs, blobid, NULL, cb_fn, cb_arg); 6975 } 6976 6977 void spdk_bs_open_blob_ext(struct spdk_blob_store *bs, spdk_blob_id blobid, 6978 struct spdk_blob_open_opts *opts, spdk_blob_op_with_handle_complete cb_fn, void *cb_arg) 6979 { 6980 bs_open_blob(bs, blobid, opts, cb_fn, cb_arg); 6981 } 6982 6983 /* END spdk_bs_open_blob */ 6984 6985 /* START spdk_blob_set_read_only */ 6986 int spdk_blob_set_read_only(struct spdk_blob *blob) 6987 { 6988 blob_verify_md_op(blob); 6989 6990 blob->data_ro_flags |= SPDK_BLOB_READ_ONLY; 6991 6992 blob->state = SPDK_BLOB_STATE_DIRTY; 6993 return 0; 6994 } 6995 /* END spdk_blob_set_read_only */ 6996 6997 /* START spdk_blob_sync_md */ 6998 6999 static void 7000 blob_sync_md_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 7001 { 7002 struct spdk_blob *blob = cb_arg; 7003 7004 if (bserrno == 0 && (blob->data_ro_flags & SPDK_BLOB_READ_ONLY)) { 7005 blob->data_ro = true; 7006 blob->md_ro = true; 7007 } 7008 7009 bs_sequence_finish(seq, bserrno); 7010 } 7011 7012 static void 7013 blob_sync_md(struct spdk_blob *blob, spdk_blob_op_complete cb_fn, void *cb_arg) 7014 { 7015 struct spdk_bs_cpl cpl; 7016 spdk_bs_sequence_t *seq; 7017 7018 cpl.type = SPDK_BS_CPL_TYPE_BLOB_BASIC; 7019 cpl.u.blob_basic.cb_fn = cb_fn; 7020 cpl.u.blob_basic.cb_arg = cb_arg; 7021 7022 seq = bs_sequence_start(blob->bs->md_channel, &cpl); 7023 if (!seq) { 7024 cb_fn(cb_arg, -ENOMEM); 7025 return; 7026 } 7027 7028 blob_persist(seq, blob, blob_sync_md_cpl, blob); 7029 } 7030 7031 void 7032 spdk_blob_sync_md(struct spdk_blob *blob, spdk_blob_op_complete cb_fn, void *cb_arg) 7033 { 7034 blob_verify_md_op(blob); 7035 7036 SPDK_DEBUGLOG(blob, "Syncing blob %" PRIu64 "\n", blob->id); 7037 7038 if (blob->md_ro) { 7039 assert(blob->state == SPDK_BLOB_STATE_CLEAN); 7040 cb_fn(cb_arg, 0); 7041 return; 7042 } 7043 7044 blob_sync_md(blob, cb_fn, cb_arg); 7045 } 7046 7047 /* END spdk_blob_sync_md */ 7048 7049 struct spdk_blob_insert_cluster_ctx { 7050 struct spdk_thread *thread; 7051 struct spdk_blob *blob; 7052 uint32_t cluster_num; /* cluster index in blob */ 7053 uint32_t cluster; /* cluster on disk */ 7054 uint32_t extent_page; /* extent page on disk */ 7055 int rc; 7056 spdk_blob_op_complete cb_fn; 7057 void *cb_arg; 7058 }; 7059 7060 static void 7061 blob_insert_cluster_msg_cpl(void *arg) 7062 { 7063 struct spdk_blob_insert_cluster_ctx *ctx = arg; 7064 7065 ctx->cb_fn(ctx->cb_arg, ctx->rc); 7066 free(ctx); 7067 } 7068 7069 static void 7070 blob_insert_cluster_msg_cb(void *arg, int bserrno) 7071 { 7072 struct spdk_blob_insert_cluster_ctx *ctx = arg; 7073 7074 ctx->rc = bserrno; 7075 spdk_thread_send_msg(ctx->thread, blob_insert_cluster_msg_cpl, ctx); 7076 } 7077 7078 static void 7079 blob_insert_new_ep_cb(void *arg, int bserrno) 7080 { 7081 struct spdk_blob_insert_cluster_ctx *ctx = arg; 7082 uint32_t *extent_page; 7083 7084 extent_page = bs_cluster_to_extent_page(ctx->blob, ctx->cluster_num); 7085 *extent_page = ctx->extent_page; 7086 ctx->blob->state = SPDK_BLOB_STATE_DIRTY; 7087 blob_sync_md(ctx->blob, blob_insert_cluster_msg_cb, ctx); 7088 } 7089 7090 static void 7091 blob_persist_extent_page_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 7092 { 7093 struct spdk_blob_md_page *page = cb_arg; 7094 7095 bs_sequence_finish(seq, bserrno); 7096 spdk_free(page); 7097 } 7098 7099 static void 7100 blob_write_extent_page(struct spdk_blob *blob, uint32_t extent, uint64_t cluster_num, 7101 spdk_blob_op_complete cb_fn, void *cb_arg) 7102 { 7103 spdk_bs_sequence_t *seq; 7104 struct spdk_bs_cpl cpl; 7105 struct spdk_blob_md_page *page = NULL; 7106 uint32_t page_count = 0; 7107 int rc; 7108 7109 cpl.type = SPDK_BS_CPL_TYPE_BLOB_BASIC; 7110 cpl.u.blob_basic.cb_fn = cb_fn; 7111 cpl.u.blob_basic.cb_arg = cb_arg; 7112 7113 seq = bs_sequence_start(blob->bs->md_channel, &cpl); 7114 if (!seq) { 7115 cb_fn(cb_arg, -ENOMEM); 7116 return; 7117 } 7118 rc = blob_serialize_add_page(blob, &page, &page_count, &page); 7119 if (rc < 0) { 7120 bs_sequence_finish(seq, rc); 7121 return; 7122 } 7123 7124 blob_serialize_extent_page(blob, cluster_num, page); 7125 7126 page->crc = blob_md_page_calc_crc(page); 7127 7128 assert(spdk_bit_array_get(blob->bs->used_md_pages, extent) == true); 7129 7130 bs_sequence_write_dev(seq, page, bs_md_page_to_lba(blob->bs, extent), 7131 bs_byte_to_lba(blob->bs, SPDK_BS_PAGE_SIZE), 7132 blob_persist_extent_page_cpl, page); 7133 } 7134 7135 static void 7136 blob_insert_cluster_msg(void *arg) 7137 { 7138 struct spdk_blob_insert_cluster_ctx *ctx = arg; 7139 uint32_t *extent_page; 7140 7141 ctx->rc = blob_insert_cluster(ctx->blob, ctx->cluster_num, ctx->cluster); 7142 if (ctx->rc != 0) { 7143 spdk_thread_send_msg(ctx->thread, blob_insert_cluster_msg_cpl, ctx); 7144 return; 7145 } 7146 7147 if (ctx->blob->use_extent_table == false) { 7148 /* Extent table is not used, proceed with sync of md that will only use extents_rle. */ 7149 ctx->blob->state = SPDK_BLOB_STATE_DIRTY; 7150 blob_sync_md(ctx->blob, blob_insert_cluster_msg_cb, ctx); 7151 return; 7152 } 7153 7154 extent_page = bs_cluster_to_extent_page(ctx->blob, ctx->cluster_num); 7155 if (*extent_page == 0) { 7156 /* Extent page requires allocation. 7157 * It was already claimed in the used_md_pages map and placed in ctx. */ 7158 assert(ctx->extent_page != 0); 7159 assert(spdk_bit_array_get(ctx->blob->bs->used_md_pages, ctx->extent_page) == true); 7160 blob_write_extent_page(ctx->blob, ctx->extent_page, ctx->cluster_num, 7161 blob_insert_new_ep_cb, ctx); 7162 } else { 7163 /* It is possible for original thread to allocate extent page for 7164 * different cluster in the same extent page. In such case proceed with 7165 * updating the existing extent page, but release the additional one. */ 7166 if (ctx->extent_page != 0) { 7167 assert(spdk_bit_array_get(ctx->blob->bs->used_md_pages, ctx->extent_page) == true); 7168 bs_release_md_page(ctx->blob->bs, ctx->extent_page); 7169 ctx->extent_page = 0; 7170 } 7171 /* Extent page already allocated. 7172 * Every cluster allocation, requires just an update of single extent page. */ 7173 blob_write_extent_page(ctx->blob, *extent_page, ctx->cluster_num, 7174 blob_insert_cluster_msg_cb, ctx); 7175 } 7176 } 7177 7178 static void 7179 blob_insert_cluster_on_md_thread(struct spdk_blob *blob, uint32_t cluster_num, 7180 uint64_t cluster, uint32_t extent_page, spdk_blob_op_complete cb_fn, void *cb_arg) 7181 { 7182 struct spdk_blob_insert_cluster_ctx *ctx; 7183 7184 ctx = calloc(1, sizeof(*ctx)); 7185 if (ctx == NULL) { 7186 cb_fn(cb_arg, -ENOMEM); 7187 return; 7188 } 7189 7190 ctx->thread = spdk_get_thread(); 7191 ctx->blob = blob; 7192 ctx->cluster_num = cluster_num; 7193 ctx->cluster = cluster; 7194 ctx->extent_page = extent_page; 7195 ctx->cb_fn = cb_fn; 7196 ctx->cb_arg = cb_arg; 7197 7198 spdk_thread_send_msg(blob->bs->md_thread, blob_insert_cluster_msg, ctx); 7199 } 7200 7201 /* START spdk_blob_close */ 7202 7203 static void 7204 blob_close_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 7205 { 7206 struct spdk_blob *blob = cb_arg; 7207 7208 if (bserrno == 0) { 7209 blob->open_ref--; 7210 if (blob->open_ref == 0) { 7211 /* 7212 * Blobs with active.num_pages == 0 are deleted blobs. 7213 * these blobs are removed from the blob_store list 7214 * when the deletion process starts - so don't try to 7215 * remove them again. 7216 */ 7217 if (blob->active.num_pages > 0) { 7218 spdk_bit_array_clear(blob->bs->open_blobids, blob->id); 7219 TAILQ_REMOVE(&blob->bs->blobs, blob, link); 7220 } 7221 blob_free(blob); 7222 } 7223 } 7224 7225 bs_sequence_finish(seq, bserrno); 7226 } 7227 7228 void spdk_blob_close(struct spdk_blob *blob, spdk_blob_op_complete cb_fn, void *cb_arg) 7229 { 7230 struct spdk_bs_cpl cpl; 7231 spdk_bs_sequence_t *seq; 7232 7233 blob_verify_md_op(blob); 7234 7235 SPDK_DEBUGLOG(blob, "Closing blob %" PRIu64 "\n", blob->id); 7236 7237 if (blob->open_ref == 0) { 7238 cb_fn(cb_arg, -EBADF); 7239 return; 7240 } 7241 7242 cpl.type = SPDK_BS_CPL_TYPE_BLOB_BASIC; 7243 cpl.u.blob_basic.cb_fn = cb_fn; 7244 cpl.u.blob_basic.cb_arg = cb_arg; 7245 7246 seq = bs_sequence_start(blob->bs->md_channel, &cpl); 7247 if (!seq) { 7248 cb_fn(cb_arg, -ENOMEM); 7249 return; 7250 } 7251 7252 /* Sync metadata */ 7253 blob_persist(seq, blob, blob_close_cpl, blob); 7254 } 7255 7256 /* END spdk_blob_close */ 7257 7258 struct spdk_io_channel *spdk_bs_alloc_io_channel(struct spdk_blob_store *bs) 7259 { 7260 return spdk_get_io_channel(bs); 7261 } 7262 7263 void spdk_bs_free_io_channel(struct spdk_io_channel *channel) 7264 { 7265 spdk_put_io_channel(channel); 7266 } 7267 7268 void spdk_blob_io_unmap(struct spdk_blob *blob, struct spdk_io_channel *channel, 7269 uint64_t offset, uint64_t length, spdk_blob_op_complete cb_fn, void *cb_arg) 7270 { 7271 blob_request_submit_op(blob, channel, NULL, offset, length, cb_fn, cb_arg, 7272 SPDK_BLOB_UNMAP); 7273 } 7274 7275 void spdk_blob_io_write_zeroes(struct spdk_blob *blob, struct spdk_io_channel *channel, 7276 uint64_t offset, uint64_t length, spdk_blob_op_complete cb_fn, void *cb_arg) 7277 { 7278 blob_request_submit_op(blob, channel, NULL, offset, length, cb_fn, cb_arg, 7279 SPDK_BLOB_WRITE_ZEROES); 7280 } 7281 7282 void spdk_blob_io_write(struct spdk_blob *blob, struct spdk_io_channel *channel, 7283 void *payload, uint64_t offset, uint64_t length, 7284 spdk_blob_op_complete cb_fn, void *cb_arg) 7285 { 7286 blob_request_submit_op(blob, channel, payload, offset, length, cb_fn, cb_arg, 7287 SPDK_BLOB_WRITE); 7288 } 7289 7290 void spdk_blob_io_read(struct spdk_blob *blob, struct spdk_io_channel *channel, 7291 void *payload, uint64_t offset, uint64_t length, 7292 spdk_blob_op_complete cb_fn, void *cb_arg) 7293 { 7294 blob_request_submit_op(blob, channel, payload, offset, length, cb_fn, cb_arg, 7295 SPDK_BLOB_READ); 7296 } 7297 7298 void spdk_blob_io_writev(struct spdk_blob *blob, struct spdk_io_channel *channel, 7299 struct iovec *iov, int iovcnt, uint64_t offset, uint64_t length, 7300 spdk_blob_op_complete cb_fn, void *cb_arg) 7301 { 7302 blob_request_submit_rw_iov(blob, channel, iov, iovcnt, offset, length, cb_fn, cb_arg, false); 7303 } 7304 7305 void spdk_blob_io_readv(struct spdk_blob *blob, struct spdk_io_channel *channel, 7306 struct iovec *iov, int iovcnt, uint64_t offset, uint64_t length, 7307 spdk_blob_op_complete cb_fn, void *cb_arg) 7308 { 7309 blob_request_submit_rw_iov(blob, channel, iov, iovcnt, offset, length, cb_fn, cb_arg, true); 7310 } 7311 7312 struct spdk_bs_iter_ctx { 7313 int64_t page_num; 7314 struct spdk_blob_store *bs; 7315 7316 spdk_blob_op_with_handle_complete cb_fn; 7317 void *cb_arg; 7318 }; 7319 7320 static void 7321 bs_iter_cpl(void *cb_arg, struct spdk_blob *_blob, int bserrno) 7322 { 7323 struct spdk_bs_iter_ctx *ctx = cb_arg; 7324 struct spdk_blob_store *bs = ctx->bs; 7325 spdk_blob_id id; 7326 7327 if (bserrno == 0) { 7328 ctx->cb_fn(ctx->cb_arg, _blob, bserrno); 7329 free(ctx); 7330 return; 7331 } 7332 7333 ctx->page_num++; 7334 ctx->page_num = spdk_bit_array_find_first_set(bs->used_blobids, ctx->page_num); 7335 if (ctx->page_num >= spdk_bit_array_capacity(bs->used_blobids)) { 7336 ctx->cb_fn(ctx->cb_arg, NULL, -ENOENT); 7337 free(ctx); 7338 return; 7339 } 7340 7341 id = bs_page_to_blobid(ctx->page_num); 7342 7343 spdk_bs_open_blob(bs, id, bs_iter_cpl, ctx); 7344 } 7345 7346 void 7347 spdk_bs_iter_first(struct spdk_blob_store *bs, 7348 spdk_blob_op_with_handle_complete cb_fn, void *cb_arg) 7349 { 7350 struct spdk_bs_iter_ctx *ctx; 7351 7352 ctx = calloc(1, sizeof(*ctx)); 7353 if (!ctx) { 7354 cb_fn(cb_arg, NULL, -ENOMEM); 7355 return; 7356 } 7357 7358 ctx->page_num = -1; 7359 ctx->bs = bs; 7360 ctx->cb_fn = cb_fn; 7361 ctx->cb_arg = cb_arg; 7362 7363 bs_iter_cpl(ctx, NULL, -1); 7364 } 7365 7366 static void 7367 bs_iter_close_cpl(void *cb_arg, int bserrno) 7368 { 7369 struct spdk_bs_iter_ctx *ctx = cb_arg; 7370 7371 bs_iter_cpl(ctx, NULL, -1); 7372 } 7373 7374 void 7375 spdk_bs_iter_next(struct spdk_blob_store *bs, struct spdk_blob *blob, 7376 spdk_blob_op_with_handle_complete cb_fn, void *cb_arg) 7377 { 7378 struct spdk_bs_iter_ctx *ctx; 7379 7380 assert(blob != NULL); 7381 7382 ctx = calloc(1, sizeof(*ctx)); 7383 if (!ctx) { 7384 cb_fn(cb_arg, NULL, -ENOMEM); 7385 return; 7386 } 7387 7388 ctx->page_num = bs_blobid_to_page(blob->id); 7389 ctx->bs = bs; 7390 ctx->cb_fn = cb_fn; 7391 ctx->cb_arg = cb_arg; 7392 7393 /* Close the existing blob */ 7394 spdk_blob_close(blob, bs_iter_close_cpl, ctx); 7395 } 7396 7397 static int 7398 blob_set_xattr(struct spdk_blob *blob, const char *name, const void *value, 7399 uint16_t value_len, bool internal) 7400 { 7401 struct spdk_xattr_tailq *xattrs; 7402 struct spdk_xattr *xattr; 7403 size_t desc_size; 7404 void *tmp; 7405 7406 blob_verify_md_op(blob); 7407 7408 if (blob->md_ro) { 7409 return -EPERM; 7410 } 7411 7412 desc_size = sizeof(struct spdk_blob_md_descriptor_xattr) + strlen(name) + value_len; 7413 if (desc_size > SPDK_BS_MAX_DESC_SIZE) { 7414 SPDK_DEBUGLOG(blob, "Xattr '%s' of size %zu does not fix into single page %zu\n", name, 7415 desc_size, SPDK_BS_MAX_DESC_SIZE); 7416 return -ENOMEM; 7417 } 7418 7419 if (internal) { 7420 xattrs = &blob->xattrs_internal; 7421 blob->invalid_flags |= SPDK_BLOB_INTERNAL_XATTR; 7422 } else { 7423 xattrs = &blob->xattrs; 7424 } 7425 7426 TAILQ_FOREACH(xattr, xattrs, link) { 7427 if (!strcmp(name, xattr->name)) { 7428 tmp = malloc(value_len); 7429 if (!tmp) { 7430 return -ENOMEM; 7431 } 7432 7433 free(xattr->value); 7434 xattr->value_len = value_len; 7435 xattr->value = tmp; 7436 memcpy(xattr->value, value, value_len); 7437 7438 blob->state = SPDK_BLOB_STATE_DIRTY; 7439 7440 return 0; 7441 } 7442 } 7443 7444 xattr = calloc(1, sizeof(*xattr)); 7445 if (!xattr) { 7446 return -ENOMEM; 7447 } 7448 7449 xattr->name = strdup(name); 7450 if (!xattr->name) { 7451 free(xattr); 7452 return -ENOMEM; 7453 } 7454 7455 xattr->value_len = value_len; 7456 xattr->value = malloc(value_len); 7457 if (!xattr->value) { 7458 free(xattr->name); 7459 free(xattr); 7460 return -ENOMEM; 7461 } 7462 memcpy(xattr->value, value, value_len); 7463 TAILQ_INSERT_TAIL(xattrs, xattr, link); 7464 7465 blob->state = SPDK_BLOB_STATE_DIRTY; 7466 7467 return 0; 7468 } 7469 7470 int 7471 spdk_blob_set_xattr(struct spdk_blob *blob, const char *name, const void *value, 7472 uint16_t value_len) 7473 { 7474 return blob_set_xattr(blob, name, value, value_len, false); 7475 } 7476 7477 static int 7478 blob_remove_xattr(struct spdk_blob *blob, const char *name, bool internal) 7479 { 7480 struct spdk_xattr_tailq *xattrs; 7481 struct spdk_xattr *xattr; 7482 7483 blob_verify_md_op(blob); 7484 7485 if (blob->md_ro) { 7486 return -EPERM; 7487 } 7488 xattrs = internal ? &blob->xattrs_internal : &blob->xattrs; 7489 7490 TAILQ_FOREACH(xattr, xattrs, link) { 7491 if (!strcmp(name, xattr->name)) { 7492 TAILQ_REMOVE(xattrs, xattr, link); 7493 free(xattr->value); 7494 free(xattr->name); 7495 free(xattr); 7496 7497 if (internal && TAILQ_EMPTY(&blob->xattrs_internal)) { 7498 blob->invalid_flags &= ~SPDK_BLOB_INTERNAL_XATTR; 7499 } 7500 blob->state = SPDK_BLOB_STATE_DIRTY; 7501 7502 return 0; 7503 } 7504 } 7505 7506 return -ENOENT; 7507 } 7508 7509 int 7510 spdk_blob_remove_xattr(struct spdk_blob *blob, const char *name) 7511 { 7512 return blob_remove_xattr(blob, name, false); 7513 } 7514 7515 static int 7516 blob_get_xattr_value(struct spdk_blob *blob, const char *name, 7517 const void **value, size_t *value_len, bool internal) 7518 { 7519 struct spdk_xattr *xattr; 7520 struct spdk_xattr_tailq *xattrs; 7521 7522 xattrs = internal ? &blob->xattrs_internal : &blob->xattrs; 7523 7524 TAILQ_FOREACH(xattr, xattrs, link) { 7525 if (!strcmp(name, xattr->name)) { 7526 *value = xattr->value; 7527 *value_len = xattr->value_len; 7528 return 0; 7529 } 7530 } 7531 return -ENOENT; 7532 } 7533 7534 int 7535 spdk_blob_get_xattr_value(struct spdk_blob *blob, const char *name, 7536 const void **value, size_t *value_len) 7537 { 7538 blob_verify_md_op(blob); 7539 7540 return blob_get_xattr_value(blob, name, value, value_len, false); 7541 } 7542 7543 struct spdk_xattr_names { 7544 uint32_t count; 7545 const char *names[0]; 7546 }; 7547 7548 static int 7549 blob_get_xattr_names(struct spdk_xattr_tailq *xattrs, struct spdk_xattr_names **names) 7550 { 7551 struct spdk_xattr *xattr; 7552 int count = 0; 7553 7554 TAILQ_FOREACH(xattr, xattrs, link) { 7555 count++; 7556 } 7557 7558 *names = calloc(1, sizeof(struct spdk_xattr_names) + count * sizeof(char *)); 7559 if (*names == NULL) { 7560 return -ENOMEM; 7561 } 7562 7563 TAILQ_FOREACH(xattr, xattrs, link) { 7564 (*names)->names[(*names)->count++] = xattr->name; 7565 } 7566 7567 return 0; 7568 } 7569 7570 int 7571 spdk_blob_get_xattr_names(struct spdk_blob *blob, struct spdk_xattr_names **names) 7572 { 7573 blob_verify_md_op(blob); 7574 7575 return blob_get_xattr_names(&blob->xattrs, names); 7576 } 7577 7578 uint32_t 7579 spdk_xattr_names_get_count(struct spdk_xattr_names *names) 7580 { 7581 assert(names != NULL); 7582 7583 return names->count; 7584 } 7585 7586 const char * 7587 spdk_xattr_names_get_name(struct spdk_xattr_names *names, uint32_t index) 7588 { 7589 if (index >= names->count) { 7590 return NULL; 7591 } 7592 7593 return names->names[index]; 7594 } 7595 7596 void 7597 spdk_xattr_names_free(struct spdk_xattr_names *names) 7598 { 7599 free(names); 7600 } 7601 7602 struct spdk_bs_type 7603 spdk_bs_get_bstype(struct spdk_blob_store *bs) 7604 { 7605 return bs->bstype; 7606 } 7607 7608 void 7609 spdk_bs_set_bstype(struct spdk_blob_store *bs, struct spdk_bs_type bstype) 7610 { 7611 memcpy(&bs->bstype, &bstype, sizeof(bstype)); 7612 } 7613 7614 bool 7615 spdk_blob_is_read_only(struct spdk_blob *blob) 7616 { 7617 assert(blob != NULL); 7618 return (blob->data_ro || blob->md_ro); 7619 } 7620 7621 bool 7622 spdk_blob_is_snapshot(struct spdk_blob *blob) 7623 { 7624 struct spdk_blob_list *snapshot_entry; 7625 7626 assert(blob != NULL); 7627 7628 snapshot_entry = bs_get_snapshot_entry(blob->bs, blob->id); 7629 if (snapshot_entry == NULL) { 7630 return false; 7631 } 7632 7633 return true; 7634 } 7635 7636 bool 7637 spdk_blob_is_clone(struct spdk_blob *blob) 7638 { 7639 assert(blob != NULL); 7640 7641 if (blob->parent_id != SPDK_BLOBID_INVALID) { 7642 assert(spdk_blob_is_thin_provisioned(blob)); 7643 return true; 7644 } 7645 7646 return false; 7647 } 7648 7649 bool 7650 spdk_blob_is_thin_provisioned(struct spdk_blob *blob) 7651 { 7652 assert(blob != NULL); 7653 return !!(blob->invalid_flags & SPDK_BLOB_THIN_PROV); 7654 } 7655 7656 static void 7657 blob_update_clear_method(struct spdk_blob *blob) 7658 { 7659 enum blob_clear_method stored_cm; 7660 7661 assert(blob != NULL); 7662 7663 /* If BLOB_CLEAR_WITH_DEFAULT was passed in, use the setting stored 7664 * in metadata previously. If something other than the default was 7665 * specified, ignore stored value and used what was passed in. 7666 */ 7667 stored_cm = ((blob->md_ro_flags & SPDK_BLOB_CLEAR_METHOD) >> SPDK_BLOB_CLEAR_METHOD_SHIFT); 7668 7669 if (blob->clear_method == BLOB_CLEAR_WITH_DEFAULT) { 7670 blob->clear_method = stored_cm; 7671 } else if (blob->clear_method != stored_cm) { 7672 SPDK_WARNLOG("Using passed in clear method 0x%x instead of stored value of 0x%x\n", 7673 blob->clear_method, stored_cm); 7674 } 7675 } 7676 7677 spdk_blob_id 7678 spdk_blob_get_parent_snapshot(struct spdk_blob_store *bs, spdk_blob_id blob_id) 7679 { 7680 struct spdk_blob_list *snapshot_entry = NULL; 7681 struct spdk_blob_list *clone_entry = NULL; 7682 7683 TAILQ_FOREACH(snapshot_entry, &bs->snapshots, link) { 7684 TAILQ_FOREACH(clone_entry, &snapshot_entry->clones, link) { 7685 if (clone_entry->id == blob_id) { 7686 return snapshot_entry->id; 7687 } 7688 } 7689 } 7690 7691 return SPDK_BLOBID_INVALID; 7692 } 7693 7694 int 7695 spdk_blob_get_clones(struct spdk_blob_store *bs, spdk_blob_id blobid, spdk_blob_id *ids, 7696 size_t *count) 7697 { 7698 struct spdk_blob_list *snapshot_entry, *clone_entry; 7699 size_t n; 7700 7701 snapshot_entry = bs_get_snapshot_entry(bs, blobid); 7702 if (snapshot_entry == NULL) { 7703 *count = 0; 7704 return 0; 7705 } 7706 7707 if (ids == NULL || *count < snapshot_entry->clone_count) { 7708 *count = snapshot_entry->clone_count; 7709 return -ENOMEM; 7710 } 7711 *count = snapshot_entry->clone_count; 7712 7713 n = 0; 7714 TAILQ_FOREACH(clone_entry, &snapshot_entry->clones, link) { 7715 ids[n++] = clone_entry->id; 7716 } 7717 7718 return 0; 7719 } 7720 7721 SPDK_LOG_REGISTER_COMPONENT(blob) 7722