1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) Intel Corporation. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include "spdk/stdinc.h" 35 36 #include "spdk/blob.h" 37 #include "spdk/crc32.h" 38 #include "spdk/env.h" 39 #include "spdk/queue.h" 40 #include "spdk/thread.h" 41 #include "spdk/bit_array.h" 42 #include "spdk/bit_pool.h" 43 #include "spdk/likely.h" 44 #include "spdk/util.h" 45 #include "spdk/string.h" 46 47 #include "spdk_internal/assert.h" 48 #include "spdk/log.h" 49 50 #include "blobstore.h" 51 52 #define BLOB_CRC32C_INITIAL 0xffffffffUL 53 54 static int bs_register_md_thread(struct spdk_blob_store *bs); 55 static int bs_unregister_md_thread(struct spdk_blob_store *bs); 56 static void blob_close_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno); 57 static void blob_insert_cluster_on_md_thread(struct spdk_blob *blob, uint32_t cluster_num, 58 uint64_t cluster, uint32_t extent, spdk_blob_op_complete cb_fn, void *cb_arg); 59 60 static int blob_set_xattr(struct spdk_blob *blob, const char *name, const void *value, 61 uint16_t value_len, bool internal); 62 static int blob_get_xattr_value(struct spdk_blob *blob, const char *name, 63 const void **value, size_t *value_len, bool internal); 64 static int blob_remove_xattr(struct spdk_blob *blob, const char *name, bool internal); 65 66 static void blob_write_extent_page(struct spdk_blob *blob, uint32_t extent, uint64_t cluster_num, 67 spdk_blob_op_complete cb_fn, void *cb_arg); 68 69 static void 70 blob_verify_md_op(struct spdk_blob *blob) 71 { 72 assert(blob != NULL); 73 assert(spdk_get_thread() == blob->bs->md_thread); 74 assert(blob->state != SPDK_BLOB_STATE_LOADING); 75 } 76 77 static struct spdk_blob_list * 78 bs_get_snapshot_entry(struct spdk_blob_store *bs, spdk_blob_id blobid) 79 { 80 struct spdk_blob_list *snapshot_entry = NULL; 81 82 TAILQ_FOREACH(snapshot_entry, &bs->snapshots, link) { 83 if (snapshot_entry->id == blobid) { 84 break; 85 } 86 } 87 88 return snapshot_entry; 89 } 90 91 static void 92 bs_claim_md_page(struct spdk_blob_store *bs, uint32_t page) 93 { 94 assert(page < spdk_bit_array_capacity(bs->used_md_pages)); 95 assert(spdk_bit_array_get(bs->used_md_pages, page) == false); 96 97 spdk_bit_array_set(bs->used_md_pages, page); 98 } 99 100 static void 101 bs_release_md_page(struct spdk_blob_store *bs, uint32_t page) 102 { 103 assert(page < spdk_bit_array_capacity(bs->used_md_pages)); 104 assert(spdk_bit_array_get(bs->used_md_pages, page) == true); 105 106 spdk_bit_array_clear(bs->used_md_pages, page); 107 } 108 109 static uint32_t 110 bs_claim_cluster(struct spdk_blob_store *bs) 111 { 112 uint32_t cluster_num; 113 114 cluster_num = spdk_bit_pool_allocate_bit(bs->used_clusters); 115 if (cluster_num == UINT32_MAX) { 116 return UINT32_MAX; 117 } 118 119 SPDK_DEBUGLOG(blob, "Claiming cluster %u\n", cluster_num); 120 bs->num_free_clusters--; 121 122 return cluster_num; 123 } 124 125 static void 126 bs_release_cluster(struct spdk_blob_store *bs, uint32_t cluster_num) 127 { 128 assert(cluster_num < spdk_bit_pool_capacity(bs->used_clusters)); 129 assert(spdk_bit_pool_is_allocated(bs->used_clusters, cluster_num) == true); 130 assert(bs->num_free_clusters < bs->total_clusters); 131 132 SPDK_DEBUGLOG(blob, "Releasing cluster %u\n", cluster_num); 133 134 spdk_bit_pool_free_bit(bs->used_clusters, cluster_num); 135 bs->num_free_clusters++; 136 } 137 138 static int 139 blob_insert_cluster(struct spdk_blob *blob, uint32_t cluster_num, uint64_t cluster) 140 { 141 uint64_t *cluster_lba = &blob->active.clusters[cluster_num]; 142 143 blob_verify_md_op(blob); 144 145 if (*cluster_lba != 0) { 146 return -EEXIST; 147 } 148 149 *cluster_lba = bs_cluster_to_lba(blob->bs, cluster); 150 return 0; 151 } 152 153 static int 154 bs_allocate_cluster(struct spdk_blob *blob, uint32_t cluster_num, 155 uint64_t *cluster, uint32_t *lowest_free_md_page, bool update_map) 156 { 157 uint32_t *extent_page = 0; 158 159 *cluster = bs_claim_cluster(blob->bs); 160 if (*cluster == UINT32_MAX) { 161 /* No more free clusters. Cannot satisfy the request */ 162 return -ENOSPC; 163 } 164 165 if (blob->use_extent_table) { 166 extent_page = bs_cluster_to_extent_page(blob, cluster_num); 167 if (*extent_page == 0) { 168 /* Extent page shall never occupy md_page so start the search from 1 */ 169 if (*lowest_free_md_page == 0) { 170 *lowest_free_md_page = 1; 171 } 172 /* No extent_page is allocated for the cluster */ 173 *lowest_free_md_page = spdk_bit_array_find_first_clear(blob->bs->used_md_pages, 174 *lowest_free_md_page); 175 if (*lowest_free_md_page == UINT32_MAX) { 176 /* No more free md pages. Cannot satisfy the request */ 177 bs_release_cluster(blob->bs, *cluster); 178 return -ENOSPC; 179 } 180 bs_claim_md_page(blob->bs, *lowest_free_md_page); 181 } 182 } 183 184 SPDK_DEBUGLOG(blob, "Claiming cluster %" PRIu64 " for blob %" PRIu64 "\n", *cluster, blob->id); 185 186 if (update_map) { 187 blob_insert_cluster(blob, cluster_num, *cluster); 188 if (blob->use_extent_table && *extent_page == 0) { 189 *extent_page = *lowest_free_md_page; 190 } 191 } 192 193 return 0; 194 } 195 196 static void 197 blob_xattrs_init(struct spdk_blob_xattr_opts *xattrs) 198 { 199 xattrs->count = 0; 200 xattrs->names = NULL; 201 xattrs->ctx = NULL; 202 xattrs->get_value = NULL; 203 } 204 205 void 206 spdk_blob_opts_init(struct spdk_blob_opts *opts, size_t opts_size) 207 { 208 if (!opts) { 209 SPDK_ERRLOG("opts should not be NULL\n"); 210 return; 211 } 212 213 if (!opts_size) { 214 SPDK_ERRLOG("opts_size should not be zero value\n"); 215 return; 216 } 217 218 memset(opts, 0, opts_size); 219 opts->opts_size = opts_size; 220 221 #define FIELD_OK(field) \ 222 offsetof(struct spdk_blob_opts, field) + sizeof(opts->field) <= opts_size 223 224 #define SET_FIELD(field, value) \ 225 if (FIELD_OK(field)) { \ 226 opts->field = value; \ 227 } \ 228 229 SET_FIELD(num_clusters, 0); 230 SET_FIELD(thin_provision, false); 231 SET_FIELD(clear_method, BLOB_CLEAR_WITH_DEFAULT); 232 233 if (FIELD_OK(xattrs)) { 234 blob_xattrs_init(&opts->xattrs); 235 } 236 237 SET_FIELD(use_extent_table, true); 238 239 #undef FIELD_OK 240 #undef SET_FIELD 241 } 242 243 void 244 spdk_blob_open_opts_init(struct spdk_blob_open_opts *opts, size_t opts_size) 245 { 246 if (!opts) { 247 SPDK_ERRLOG("opts should not be NULL\n"); 248 return; 249 } 250 251 if (!opts_size) { 252 SPDK_ERRLOG("opts_size should not be zero value\n"); 253 return; 254 } 255 256 memset(opts, 0, opts_size); 257 opts->opts_size = opts_size; 258 259 #define FIELD_OK(field) \ 260 offsetof(struct spdk_blob_open_opts, field) + sizeof(opts->field) <= opts_size 261 262 #define SET_FIELD(field, value) \ 263 if (FIELD_OK(field)) { \ 264 opts->field = value; \ 265 } \ 266 267 SET_FIELD(clear_method, BLOB_CLEAR_WITH_DEFAULT); 268 269 #undef FIELD_OK 270 #undef SET_FILED 271 } 272 273 static struct spdk_blob * 274 blob_alloc(struct spdk_blob_store *bs, spdk_blob_id id) 275 { 276 struct spdk_blob *blob; 277 278 blob = calloc(1, sizeof(*blob)); 279 if (!blob) { 280 return NULL; 281 } 282 283 blob->id = id; 284 blob->bs = bs; 285 286 blob->parent_id = SPDK_BLOBID_INVALID; 287 288 blob->state = SPDK_BLOB_STATE_DIRTY; 289 blob->extent_rle_found = false; 290 blob->extent_table_found = false; 291 blob->active.num_pages = 1; 292 blob->active.pages = calloc(1, sizeof(*blob->active.pages)); 293 if (!blob->active.pages) { 294 free(blob); 295 return NULL; 296 } 297 298 blob->active.pages[0] = bs_blobid_to_page(id); 299 300 TAILQ_INIT(&blob->xattrs); 301 TAILQ_INIT(&blob->xattrs_internal); 302 TAILQ_INIT(&blob->pending_persists); 303 TAILQ_INIT(&blob->persists_to_complete); 304 305 return blob; 306 } 307 308 static void 309 xattrs_free(struct spdk_xattr_tailq *xattrs) 310 { 311 struct spdk_xattr *xattr, *xattr_tmp; 312 313 TAILQ_FOREACH_SAFE(xattr, xattrs, link, xattr_tmp) { 314 TAILQ_REMOVE(xattrs, xattr, link); 315 free(xattr->name); 316 free(xattr->value); 317 free(xattr); 318 } 319 } 320 321 static void 322 blob_free(struct spdk_blob *blob) 323 { 324 assert(blob != NULL); 325 assert(TAILQ_EMPTY(&blob->pending_persists)); 326 assert(TAILQ_EMPTY(&blob->persists_to_complete)); 327 328 free(blob->active.extent_pages); 329 free(blob->clean.extent_pages); 330 free(blob->active.clusters); 331 free(blob->clean.clusters); 332 free(blob->active.pages); 333 free(blob->clean.pages); 334 335 xattrs_free(&blob->xattrs); 336 xattrs_free(&blob->xattrs_internal); 337 338 if (blob->back_bs_dev) { 339 blob->back_bs_dev->destroy(blob->back_bs_dev); 340 } 341 342 free(blob); 343 } 344 345 struct freeze_io_ctx { 346 struct spdk_bs_cpl cpl; 347 struct spdk_blob *blob; 348 }; 349 350 static void 351 blob_io_sync(struct spdk_io_channel_iter *i) 352 { 353 spdk_for_each_channel_continue(i, 0); 354 } 355 356 static void 357 blob_execute_queued_io(struct spdk_io_channel_iter *i) 358 { 359 struct spdk_io_channel *_ch = spdk_io_channel_iter_get_channel(i); 360 struct spdk_bs_channel *ch = spdk_io_channel_get_ctx(_ch); 361 struct freeze_io_ctx *ctx = spdk_io_channel_iter_get_ctx(i); 362 struct spdk_bs_request_set *set; 363 struct spdk_bs_user_op_args *args; 364 spdk_bs_user_op_t *op, *tmp; 365 366 TAILQ_FOREACH_SAFE(op, &ch->queued_io, link, tmp) { 367 set = (struct spdk_bs_request_set *)op; 368 args = &set->u.user_op; 369 370 if (args->blob == ctx->blob) { 371 TAILQ_REMOVE(&ch->queued_io, op, link); 372 bs_user_op_execute(op); 373 } 374 } 375 376 spdk_for_each_channel_continue(i, 0); 377 } 378 379 static void 380 blob_io_cpl(struct spdk_io_channel_iter *i, int status) 381 { 382 struct freeze_io_ctx *ctx = spdk_io_channel_iter_get_ctx(i); 383 384 ctx->cpl.u.blob_basic.cb_fn(ctx->cpl.u.blob_basic.cb_arg, 0); 385 386 free(ctx); 387 } 388 389 static void 390 blob_freeze_io(struct spdk_blob *blob, spdk_blob_op_complete cb_fn, void *cb_arg) 391 { 392 struct freeze_io_ctx *ctx; 393 394 ctx = calloc(1, sizeof(*ctx)); 395 if (!ctx) { 396 cb_fn(cb_arg, -ENOMEM); 397 return; 398 } 399 400 ctx->cpl.type = SPDK_BS_CPL_TYPE_BS_BASIC; 401 ctx->cpl.u.blob_basic.cb_fn = cb_fn; 402 ctx->cpl.u.blob_basic.cb_arg = cb_arg; 403 ctx->blob = blob; 404 405 /* Freeze I/O on blob */ 406 blob->frozen_refcnt++; 407 408 if (blob->frozen_refcnt == 1) { 409 spdk_for_each_channel(blob->bs, blob_io_sync, ctx, blob_io_cpl); 410 } else { 411 cb_fn(cb_arg, 0); 412 free(ctx); 413 } 414 } 415 416 static void 417 blob_unfreeze_io(struct spdk_blob *blob, spdk_blob_op_complete cb_fn, void *cb_arg) 418 { 419 struct freeze_io_ctx *ctx; 420 421 ctx = calloc(1, sizeof(*ctx)); 422 if (!ctx) { 423 cb_fn(cb_arg, -ENOMEM); 424 return; 425 } 426 427 ctx->cpl.type = SPDK_BS_CPL_TYPE_BS_BASIC; 428 ctx->cpl.u.blob_basic.cb_fn = cb_fn; 429 ctx->cpl.u.blob_basic.cb_arg = cb_arg; 430 ctx->blob = blob; 431 432 assert(blob->frozen_refcnt > 0); 433 434 blob->frozen_refcnt--; 435 436 if (blob->frozen_refcnt == 0) { 437 spdk_for_each_channel(blob->bs, blob_execute_queued_io, ctx, blob_io_cpl); 438 } else { 439 cb_fn(cb_arg, 0); 440 free(ctx); 441 } 442 } 443 444 static int 445 blob_mark_clean(struct spdk_blob *blob) 446 { 447 uint32_t *extent_pages = NULL; 448 uint64_t *clusters = NULL; 449 uint32_t *pages = NULL; 450 451 assert(blob != NULL); 452 453 if (blob->active.num_extent_pages) { 454 assert(blob->active.extent_pages); 455 extent_pages = calloc(blob->active.num_extent_pages, sizeof(*blob->active.extent_pages)); 456 if (!extent_pages) { 457 return -ENOMEM; 458 } 459 memcpy(extent_pages, blob->active.extent_pages, 460 blob->active.num_extent_pages * sizeof(*extent_pages)); 461 } 462 463 if (blob->active.num_clusters) { 464 assert(blob->active.clusters); 465 clusters = calloc(blob->active.num_clusters, sizeof(*blob->active.clusters)); 466 if (!clusters) { 467 free(extent_pages); 468 return -ENOMEM; 469 } 470 memcpy(clusters, blob->active.clusters, blob->active.num_clusters * sizeof(*blob->active.clusters)); 471 } 472 473 if (blob->active.num_pages) { 474 assert(blob->active.pages); 475 pages = calloc(blob->active.num_pages, sizeof(*blob->active.pages)); 476 if (!pages) { 477 free(extent_pages); 478 free(clusters); 479 return -ENOMEM; 480 } 481 memcpy(pages, blob->active.pages, blob->active.num_pages * sizeof(*blob->active.pages)); 482 } 483 484 free(blob->clean.extent_pages); 485 free(blob->clean.clusters); 486 free(blob->clean.pages); 487 488 blob->clean.num_extent_pages = blob->active.num_extent_pages; 489 blob->clean.extent_pages = blob->active.extent_pages; 490 blob->clean.num_clusters = blob->active.num_clusters; 491 blob->clean.clusters = blob->active.clusters; 492 blob->clean.num_pages = blob->active.num_pages; 493 blob->clean.pages = blob->active.pages; 494 495 blob->active.extent_pages = extent_pages; 496 blob->active.clusters = clusters; 497 blob->active.pages = pages; 498 499 /* If the metadata was dirtied again while the metadata was being written to disk, 500 * we do not want to revert the DIRTY state back to CLEAN here. 501 */ 502 if (blob->state == SPDK_BLOB_STATE_LOADING) { 503 blob->state = SPDK_BLOB_STATE_CLEAN; 504 } 505 506 return 0; 507 } 508 509 static int 510 blob_deserialize_xattr(struct spdk_blob *blob, 511 struct spdk_blob_md_descriptor_xattr *desc_xattr, bool internal) 512 { 513 struct spdk_xattr *xattr; 514 515 if (desc_xattr->length != sizeof(desc_xattr->name_length) + 516 sizeof(desc_xattr->value_length) + 517 desc_xattr->name_length + desc_xattr->value_length) { 518 return -EINVAL; 519 } 520 521 xattr = calloc(1, sizeof(*xattr)); 522 if (xattr == NULL) { 523 return -ENOMEM; 524 } 525 526 xattr->name = malloc(desc_xattr->name_length + 1); 527 if (xattr->name == NULL) { 528 free(xattr); 529 return -ENOMEM; 530 } 531 memcpy(xattr->name, desc_xattr->name, desc_xattr->name_length); 532 xattr->name[desc_xattr->name_length] = '\0'; 533 534 xattr->value = malloc(desc_xattr->value_length); 535 if (xattr->value == NULL) { 536 free(xattr->name); 537 free(xattr); 538 return -ENOMEM; 539 } 540 xattr->value_len = desc_xattr->value_length; 541 memcpy(xattr->value, 542 (void *)((uintptr_t)desc_xattr->name + desc_xattr->name_length), 543 desc_xattr->value_length); 544 545 TAILQ_INSERT_TAIL(internal ? &blob->xattrs_internal : &blob->xattrs, xattr, link); 546 547 return 0; 548 } 549 550 551 static int 552 blob_parse_page(const struct spdk_blob_md_page *page, struct spdk_blob *blob) 553 { 554 struct spdk_blob_md_descriptor *desc; 555 size_t cur_desc = 0; 556 void *tmp; 557 558 desc = (struct spdk_blob_md_descriptor *)page->descriptors; 559 while (cur_desc < sizeof(page->descriptors)) { 560 if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_PADDING) { 561 if (desc->length == 0) { 562 /* If padding and length are 0, this terminates the page */ 563 break; 564 } 565 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_FLAGS) { 566 struct spdk_blob_md_descriptor_flags *desc_flags; 567 568 desc_flags = (struct spdk_blob_md_descriptor_flags *)desc; 569 570 if (desc_flags->length != sizeof(*desc_flags) - sizeof(*desc)) { 571 return -EINVAL; 572 } 573 574 if ((desc_flags->invalid_flags | SPDK_BLOB_INVALID_FLAGS_MASK) != 575 SPDK_BLOB_INVALID_FLAGS_MASK) { 576 return -EINVAL; 577 } 578 579 if ((desc_flags->data_ro_flags | SPDK_BLOB_DATA_RO_FLAGS_MASK) != 580 SPDK_BLOB_DATA_RO_FLAGS_MASK) { 581 blob->data_ro = true; 582 blob->md_ro = true; 583 } 584 585 if ((desc_flags->md_ro_flags | SPDK_BLOB_MD_RO_FLAGS_MASK) != 586 SPDK_BLOB_MD_RO_FLAGS_MASK) { 587 blob->md_ro = true; 588 } 589 590 if ((desc_flags->data_ro_flags & SPDK_BLOB_READ_ONLY)) { 591 blob->data_ro = true; 592 blob->md_ro = true; 593 } 594 595 blob->invalid_flags = desc_flags->invalid_flags; 596 blob->data_ro_flags = desc_flags->data_ro_flags; 597 blob->md_ro_flags = desc_flags->md_ro_flags; 598 599 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_EXTENT_RLE) { 600 struct spdk_blob_md_descriptor_extent_rle *desc_extent_rle; 601 unsigned int i, j; 602 unsigned int cluster_count = blob->active.num_clusters; 603 604 if (blob->extent_table_found) { 605 /* Extent Table already present in the md, 606 * both descriptors should never be at the same time. */ 607 return -EINVAL; 608 } 609 blob->extent_rle_found = true; 610 611 desc_extent_rle = (struct spdk_blob_md_descriptor_extent_rle *)desc; 612 613 if (desc_extent_rle->length == 0 || 614 (desc_extent_rle->length % sizeof(desc_extent_rle->extents[0]) != 0)) { 615 return -EINVAL; 616 } 617 618 for (i = 0; i < desc_extent_rle->length / sizeof(desc_extent_rle->extents[0]); i++) { 619 for (j = 0; j < desc_extent_rle->extents[i].length; j++) { 620 if (desc_extent_rle->extents[i].cluster_idx != 0) { 621 if (!spdk_bit_pool_is_allocated(blob->bs->used_clusters, 622 desc_extent_rle->extents[i].cluster_idx + j)) { 623 return -EINVAL; 624 } 625 } 626 cluster_count++; 627 } 628 } 629 630 if (cluster_count == 0) { 631 return -EINVAL; 632 } 633 tmp = realloc(blob->active.clusters, cluster_count * sizeof(*blob->active.clusters)); 634 if (tmp == NULL) { 635 return -ENOMEM; 636 } 637 blob->active.clusters = tmp; 638 blob->active.cluster_array_size = cluster_count; 639 640 for (i = 0; i < desc_extent_rle->length / sizeof(desc_extent_rle->extents[0]); i++) { 641 for (j = 0; j < desc_extent_rle->extents[i].length; j++) { 642 if (desc_extent_rle->extents[i].cluster_idx != 0) { 643 blob->active.clusters[blob->active.num_clusters++] = bs_cluster_to_lba(blob->bs, 644 desc_extent_rle->extents[i].cluster_idx + j); 645 } else if (spdk_blob_is_thin_provisioned(blob)) { 646 blob->active.clusters[blob->active.num_clusters++] = 0; 647 } else { 648 return -EINVAL; 649 } 650 } 651 } 652 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_EXTENT_TABLE) { 653 struct spdk_blob_md_descriptor_extent_table *desc_extent_table; 654 uint32_t num_extent_pages = blob->active.num_extent_pages; 655 uint32_t i, j; 656 size_t extent_pages_length; 657 658 desc_extent_table = (struct spdk_blob_md_descriptor_extent_table *)desc; 659 extent_pages_length = desc_extent_table->length - sizeof(desc_extent_table->num_clusters); 660 661 if (blob->extent_rle_found) { 662 /* This means that Extent RLE is present in MD, 663 * both should never be at the same time. */ 664 return -EINVAL; 665 } else if (blob->extent_table_found && 666 desc_extent_table->num_clusters != blob->remaining_clusters_in_et) { 667 /* Number of clusters in this ET does not match number 668 * from previously read EXTENT_TABLE. */ 669 return -EINVAL; 670 } 671 672 if (desc_extent_table->length == 0 || 673 (extent_pages_length % sizeof(desc_extent_table->extent_page[0]) != 0)) { 674 return -EINVAL; 675 } 676 677 blob->extent_table_found = true; 678 679 for (i = 0; i < extent_pages_length / sizeof(desc_extent_table->extent_page[0]); i++) { 680 num_extent_pages += desc_extent_table->extent_page[i].num_pages; 681 } 682 683 if (num_extent_pages > 0) { 684 tmp = realloc(blob->active.extent_pages, num_extent_pages * sizeof(uint32_t)); 685 if (tmp == NULL) { 686 return -ENOMEM; 687 } 688 blob->active.extent_pages = tmp; 689 } 690 blob->active.extent_pages_array_size = num_extent_pages; 691 692 blob->remaining_clusters_in_et = desc_extent_table->num_clusters; 693 694 /* Extent table entries contain md page numbers for extent pages. 695 * Zeroes represent unallocated extent pages, those are run-length-encoded. 696 */ 697 for (i = 0; i < extent_pages_length / sizeof(desc_extent_table->extent_page[0]); i++) { 698 if (desc_extent_table->extent_page[i].page_idx != 0) { 699 assert(desc_extent_table->extent_page[i].num_pages == 1); 700 blob->active.extent_pages[blob->active.num_extent_pages++] = 701 desc_extent_table->extent_page[i].page_idx; 702 } else if (spdk_blob_is_thin_provisioned(blob)) { 703 for (j = 0; j < desc_extent_table->extent_page[i].num_pages; j++) { 704 blob->active.extent_pages[blob->active.num_extent_pages++] = 0; 705 } 706 } else { 707 return -EINVAL; 708 } 709 } 710 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_EXTENT_PAGE) { 711 struct spdk_blob_md_descriptor_extent_page *desc_extent; 712 unsigned int i; 713 unsigned int cluster_count = 0; 714 size_t cluster_idx_length; 715 716 if (blob->extent_rle_found) { 717 /* This means that Extent RLE is present in MD, 718 * both should never be at the same time. */ 719 return -EINVAL; 720 } 721 722 desc_extent = (struct spdk_blob_md_descriptor_extent_page *)desc; 723 cluster_idx_length = desc_extent->length - sizeof(desc_extent->start_cluster_idx); 724 725 if (desc_extent->length <= sizeof(desc_extent->start_cluster_idx) || 726 (cluster_idx_length % sizeof(desc_extent->cluster_idx[0]) != 0)) { 727 return -EINVAL; 728 } 729 730 for (i = 0; i < cluster_idx_length / sizeof(desc_extent->cluster_idx[0]); i++) { 731 if (desc_extent->cluster_idx[i] != 0) { 732 if (!spdk_bit_pool_is_allocated(blob->bs->used_clusters, desc_extent->cluster_idx[i])) { 733 return -EINVAL; 734 } 735 } 736 cluster_count++; 737 } 738 739 if (cluster_count == 0) { 740 return -EINVAL; 741 } 742 743 /* When reading extent pages sequentially starting cluster idx should match 744 * current size of a blob. 745 * If changed to batch reading, this check shall be removed. */ 746 if (desc_extent->start_cluster_idx != blob->active.num_clusters) { 747 return -EINVAL; 748 } 749 750 tmp = realloc(blob->active.clusters, 751 (cluster_count + blob->active.num_clusters) * sizeof(*blob->active.clusters)); 752 if (tmp == NULL) { 753 return -ENOMEM; 754 } 755 blob->active.clusters = tmp; 756 blob->active.cluster_array_size = (cluster_count + blob->active.num_clusters); 757 758 for (i = 0; i < cluster_idx_length / sizeof(desc_extent->cluster_idx[0]); i++) { 759 if (desc_extent->cluster_idx[i] != 0) { 760 blob->active.clusters[blob->active.num_clusters++] = bs_cluster_to_lba(blob->bs, 761 desc_extent->cluster_idx[i]); 762 } else if (spdk_blob_is_thin_provisioned(blob)) { 763 blob->active.clusters[blob->active.num_clusters++] = 0; 764 } else { 765 return -EINVAL; 766 } 767 } 768 assert(desc_extent->start_cluster_idx + cluster_count == blob->active.num_clusters); 769 assert(blob->remaining_clusters_in_et >= cluster_count); 770 blob->remaining_clusters_in_et -= cluster_count; 771 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_XATTR) { 772 int rc; 773 774 rc = blob_deserialize_xattr(blob, 775 (struct spdk_blob_md_descriptor_xattr *) desc, false); 776 if (rc != 0) { 777 return rc; 778 } 779 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_XATTR_INTERNAL) { 780 int rc; 781 782 rc = blob_deserialize_xattr(blob, 783 (struct spdk_blob_md_descriptor_xattr *) desc, true); 784 if (rc != 0) { 785 return rc; 786 } 787 } else { 788 /* Unrecognized descriptor type. Do not fail - just continue to the 789 * next descriptor. If this descriptor is associated with some feature 790 * defined in a newer version of blobstore, that version of blobstore 791 * should create and set an associated feature flag to specify if this 792 * blob can be loaded or not. 793 */ 794 } 795 796 /* Advance to the next descriptor */ 797 cur_desc += sizeof(*desc) + desc->length; 798 if (cur_desc + sizeof(*desc) > sizeof(page->descriptors)) { 799 break; 800 } 801 desc = (struct spdk_blob_md_descriptor *)((uintptr_t)page->descriptors + cur_desc); 802 } 803 804 return 0; 805 } 806 807 static bool bs_load_cur_extent_page_valid(struct spdk_blob_md_page *page); 808 809 static int 810 blob_parse_extent_page(struct spdk_blob_md_page *extent_page, struct spdk_blob *blob) 811 { 812 assert(blob != NULL); 813 assert(blob->state == SPDK_BLOB_STATE_LOADING); 814 815 if (bs_load_cur_extent_page_valid(extent_page) == false) { 816 return -ENOENT; 817 } 818 819 return blob_parse_page(extent_page, blob); 820 } 821 822 static int 823 blob_parse(const struct spdk_blob_md_page *pages, uint32_t page_count, 824 struct spdk_blob *blob) 825 { 826 const struct spdk_blob_md_page *page; 827 uint32_t i; 828 int rc; 829 void *tmp; 830 831 assert(page_count > 0); 832 assert(pages[0].sequence_num == 0); 833 assert(blob != NULL); 834 assert(blob->state == SPDK_BLOB_STATE_LOADING); 835 assert(blob->active.clusters == NULL); 836 837 /* The blobid provided doesn't match what's in the MD, this can 838 * happen for example if a bogus blobid is passed in through open. 839 */ 840 if (blob->id != pages[0].id) { 841 SPDK_ERRLOG("Blobid (%" PRIu64 ") doesn't match what's in metadata (%" PRIu64 ")\n", 842 blob->id, pages[0].id); 843 return -ENOENT; 844 } 845 846 tmp = realloc(blob->active.pages, page_count * sizeof(*blob->active.pages)); 847 if (!tmp) { 848 return -ENOMEM; 849 } 850 blob->active.pages = tmp; 851 852 blob->active.pages[0] = pages[0].id; 853 854 for (i = 1; i < page_count; i++) { 855 assert(spdk_bit_array_get(blob->bs->used_md_pages, pages[i - 1].next)); 856 blob->active.pages[i] = pages[i - 1].next; 857 } 858 blob->active.num_pages = page_count; 859 860 for (i = 0; i < page_count; i++) { 861 page = &pages[i]; 862 863 assert(page->id == blob->id); 864 assert(page->sequence_num == i); 865 866 rc = blob_parse_page(page, blob); 867 if (rc != 0) { 868 return rc; 869 } 870 } 871 872 return 0; 873 } 874 875 static int 876 blob_serialize_add_page(const struct spdk_blob *blob, 877 struct spdk_blob_md_page **pages, 878 uint32_t *page_count, 879 struct spdk_blob_md_page **last_page) 880 { 881 struct spdk_blob_md_page *page, *tmp_pages; 882 883 assert(pages != NULL); 884 assert(page_count != NULL); 885 886 *last_page = NULL; 887 if (*page_count == 0) { 888 assert(*pages == NULL); 889 *pages = spdk_malloc(SPDK_BS_PAGE_SIZE, 0, 890 NULL, SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA); 891 if (*pages == NULL) { 892 return -ENOMEM; 893 } 894 *page_count = 1; 895 } else { 896 assert(*pages != NULL); 897 tmp_pages = spdk_realloc(*pages, SPDK_BS_PAGE_SIZE * (*page_count + 1), 0); 898 if (tmp_pages == NULL) { 899 return -ENOMEM; 900 } 901 (*page_count)++; 902 *pages = tmp_pages; 903 } 904 905 page = &(*pages)[*page_count - 1]; 906 memset(page, 0, sizeof(*page)); 907 page->id = blob->id; 908 page->sequence_num = *page_count - 1; 909 page->next = SPDK_INVALID_MD_PAGE; 910 *last_page = page; 911 912 return 0; 913 } 914 915 /* Transform the in-memory representation 'xattr' into an on-disk xattr descriptor. 916 * Update required_sz on both success and failure. 917 * 918 */ 919 static int 920 blob_serialize_xattr(const struct spdk_xattr *xattr, 921 uint8_t *buf, size_t buf_sz, 922 size_t *required_sz, bool internal) 923 { 924 struct spdk_blob_md_descriptor_xattr *desc; 925 926 *required_sz = sizeof(struct spdk_blob_md_descriptor_xattr) + 927 strlen(xattr->name) + 928 xattr->value_len; 929 930 if (buf_sz < *required_sz) { 931 return -1; 932 } 933 934 desc = (struct spdk_blob_md_descriptor_xattr *)buf; 935 936 desc->type = internal ? SPDK_MD_DESCRIPTOR_TYPE_XATTR_INTERNAL : SPDK_MD_DESCRIPTOR_TYPE_XATTR; 937 desc->length = sizeof(desc->name_length) + 938 sizeof(desc->value_length) + 939 strlen(xattr->name) + 940 xattr->value_len; 941 desc->name_length = strlen(xattr->name); 942 desc->value_length = xattr->value_len; 943 944 memcpy(desc->name, xattr->name, desc->name_length); 945 memcpy((void *)((uintptr_t)desc->name + desc->name_length), 946 xattr->value, 947 desc->value_length); 948 949 return 0; 950 } 951 952 static void 953 blob_serialize_extent_table_entry(const struct spdk_blob *blob, 954 uint64_t start_ep, uint64_t *next_ep, 955 uint8_t **buf, size_t *remaining_sz) 956 { 957 struct spdk_blob_md_descriptor_extent_table *desc; 958 size_t cur_sz; 959 uint64_t i, et_idx; 960 uint32_t extent_page, ep_len; 961 962 /* The buffer must have room for at least num_clusters entry */ 963 cur_sz = sizeof(struct spdk_blob_md_descriptor) + sizeof(desc->num_clusters); 964 if (*remaining_sz < cur_sz) { 965 *next_ep = start_ep; 966 return; 967 } 968 969 desc = (struct spdk_blob_md_descriptor_extent_table *)*buf; 970 desc->type = SPDK_MD_DESCRIPTOR_TYPE_EXTENT_TABLE; 971 972 desc->num_clusters = blob->active.num_clusters; 973 974 ep_len = 1; 975 et_idx = 0; 976 for (i = start_ep; i < blob->active.num_extent_pages; i++) { 977 if (*remaining_sz < cur_sz + sizeof(desc->extent_page[0])) { 978 /* If we ran out of buffer space, return */ 979 break; 980 } 981 982 extent_page = blob->active.extent_pages[i]; 983 /* Verify that next extent_page is unallocated */ 984 if (extent_page == 0 && 985 (i + 1 < blob->active.num_extent_pages && blob->active.extent_pages[i + 1] == 0)) { 986 ep_len++; 987 continue; 988 } 989 desc->extent_page[et_idx].page_idx = extent_page; 990 desc->extent_page[et_idx].num_pages = ep_len; 991 et_idx++; 992 993 ep_len = 1; 994 cur_sz += sizeof(desc->extent_page[et_idx]); 995 } 996 *next_ep = i; 997 998 desc->length = sizeof(desc->num_clusters) + sizeof(desc->extent_page[0]) * et_idx; 999 *remaining_sz -= sizeof(struct spdk_blob_md_descriptor) + desc->length; 1000 *buf += sizeof(struct spdk_blob_md_descriptor) + desc->length; 1001 } 1002 1003 static int 1004 blob_serialize_extent_table(const struct spdk_blob *blob, 1005 struct spdk_blob_md_page **pages, 1006 struct spdk_blob_md_page *cur_page, 1007 uint32_t *page_count, uint8_t **buf, 1008 size_t *remaining_sz) 1009 { 1010 uint64_t last_extent_page; 1011 int rc; 1012 1013 last_extent_page = 0; 1014 /* At least single extent table entry has to be always persisted. 1015 * Such case occurs with num_extent_pages == 0. */ 1016 while (last_extent_page <= blob->active.num_extent_pages) { 1017 blob_serialize_extent_table_entry(blob, last_extent_page, &last_extent_page, buf, 1018 remaining_sz); 1019 1020 if (last_extent_page == blob->active.num_extent_pages) { 1021 break; 1022 } 1023 1024 rc = blob_serialize_add_page(blob, pages, page_count, &cur_page); 1025 if (rc < 0) { 1026 return rc; 1027 } 1028 1029 *buf = (uint8_t *)cur_page->descriptors; 1030 *remaining_sz = sizeof(cur_page->descriptors); 1031 } 1032 1033 return 0; 1034 } 1035 1036 static void 1037 blob_serialize_extent_rle(const struct spdk_blob *blob, 1038 uint64_t start_cluster, uint64_t *next_cluster, 1039 uint8_t **buf, size_t *buf_sz) 1040 { 1041 struct spdk_blob_md_descriptor_extent_rle *desc_extent_rle; 1042 size_t cur_sz; 1043 uint64_t i, extent_idx; 1044 uint64_t lba, lba_per_cluster, lba_count; 1045 1046 /* The buffer must have room for at least one extent */ 1047 cur_sz = sizeof(struct spdk_blob_md_descriptor) + sizeof(desc_extent_rle->extents[0]); 1048 if (*buf_sz < cur_sz) { 1049 *next_cluster = start_cluster; 1050 return; 1051 } 1052 1053 desc_extent_rle = (struct spdk_blob_md_descriptor_extent_rle *)*buf; 1054 desc_extent_rle->type = SPDK_MD_DESCRIPTOR_TYPE_EXTENT_RLE; 1055 1056 lba_per_cluster = bs_cluster_to_lba(blob->bs, 1); 1057 1058 lba = blob->active.clusters[start_cluster]; 1059 lba_count = lba_per_cluster; 1060 extent_idx = 0; 1061 for (i = start_cluster + 1; i < blob->active.num_clusters; i++) { 1062 if ((lba + lba_count) == blob->active.clusters[i] && lba != 0) { 1063 /* Run-length encode sequential non-zero LBA */ 1064 lba_count += lba_per_cluster; 1065 continue; 1066 } else if (lba == 0 && blob->active.clusters[i] == 0) { 1067 /* Run-length encode unallocated clusters */ 1068 lba_count += lba_per_cluster; 1069 continue; 1070 } 1071 desc_extent_rle->extents[extent_idx].cluster_idx = lba / lba_per_cluster; 1072 desc_extent_rle->extents[extent_idx].length = lba_count / lba_per_cluster; 1073 extent_idx++; 1074 1075 cur_sz += sizeof(desc_extent_rle->extents[extent_idx]); 1076 1077 if (*buf_sz < cur_sz) { 1078 /* If we ran out of buffer space, return */ 1079 *next_cluster = i; 1080 break; 1081 } 1082 1083 lba = blob->active.clusters[i]; 1084 lba_count = lba_per_cluster; 1085 } 1086 1087 if (*buf_sz >= cur_sz) { 1088 desc_extent_rle->extents[extent_idx].cluster_idx = lba / lba_per_cluster; 1089 desc_extent_rle->extents[extent_idx].length = lba_count / lba_per_cluster; 1090 extent_idx++; 1091 1092 *next_cluster = blob->active.num_clusters; 1093 } 1094 1095 desc_extent_rle->length = sizeof(desc_extent_rle->extents[0]) * extent_idx; 1096 *buf_sz -= sizeof(struct spdk_blob_md_descriptor) + desc_extent_rle->length; 1097 *buf += sizeof(struct spdk_blob_md_descriptor) + desc_extent_rle->length; 1098 } 1099 1100 static int 1101 blob_serialize_extents_rle(const struct spdk_blob *blob, 1102 struct spdk_blob_md_page **pages, 1103 struct spdk_blob_md_page *cur_page, 1104 uint32_t *page_count, uint8_t **buf, 1105 size_t *remaining_sz) 1106 { 1107 uint64_t last_cluster; 1108 int rc; 1109 1110 last_cluster = 0; 1111 while (last_cluster < blob->active.num_clusters) { 1112 blob_serialize_extent_rle(blob, last_cluster, &last_cluster, buf, remaining_sz); 1113 1114 if (last_cluster == blob->active.num_clusters) { 1115 break; 1116 } 1117 1118 rc = blob_serialize_add_page(blob, pages, page_count, &cur_page); 1119 if (rc < 0) { 1120 return rc; 1121 } 1122 1123 *buf = (uint8_t *)cur_page->descriptors; 1124 *remaining_sz = sizeof(cur_page->descriptors); 1125 } 1126 1127 return 0; 1128 } 1129 1130 static void 1131 blob_serialize_extent_page(const struct spdk_blob *blob, 1132 uint64_t cluster, struct spdk_blob_md_page *page) 1133 { 1134 struct spdk_blob_md_descriptor_extent_page *desc_extent; 1135 uint64_t i, extent_idx; 1136 uint64_t lba, lba_per_cluster; 1137 uint64_t start_cluster_idx = (cluster / SPDK_EXTENTS_PER_EP) * SPDK_EXTENTS_PER_EP; 1138 1139 desc_extent = (struct spdk_blob_md_descriptor_extent_page *) page->descriptors; 1140 desc_extent->type = SPDK_MD_DESCRIPTOR_TYPE_EXTENT_PAGE; 1141 1142 lba_per_cluster = bs_cluster_to_lba(blob->bs, 1); 1143 1144 desc_extent->start_cluster_idx = start_cluster_idx; 1145 extent_idx = 0; 1146 for (i = start_cluster_idx; i < blob->active.num_clusters; i++) { 1147 lba = blob->active.clusters[i]; 1148 desc_extent->cluster_idx[extent_idx++] = lba / lba_per_cluster; 1149 if (extent_idx >= SPDK_EXTENTS_PER_EP) { 1150 break; 1151 } 1152 } 1153 desc_extent->length = sizeof(desc_extent->start_cluster_idx) + 1154 sizeof(desc_extent->cluster_idx[0]) * extent_idx; 1155 } 1156 1157 static void 1158 blob_serialize_flags(const struct spdk_blob *blob, 1159 uint8_t *buf, size_t *buf_sz) 1160 { 1161 struct spdk_blob_md_descriptor_flags *desc; 1162 1163 /* 1164 * Flags get serialized first, so we should always have room for the flags 1165 * descriptor. 1166 */ 1167 assert(*buf_sz >= sizeof(*desc)); 1168 1169 desc = (struct spdk_blob_md_descriptor_flags *)buf; 1170 desc->type = SPDK_MD_DESCRIPTOR_TYPE_FLAGS; 1171 desc->length = sizeof(*desc) - sizeof(struct spdk_blob_md_descriptor); 1172 desc->invalid_flags = blob->invalid_flags; 1173 desc->data_ro_flags = blob->data_ro_flags; 1174 desc->md_ro_flags = blob->md_ro_flags; 1175 1176 *buf_sz -= sizeof(*desc); 1177 } 1178 1179 static int 1180 blob_serialize_xattrs(const struct spdk_blob *blob, 1181 const struct spdk_xattr_tailq *xattrs, bool internal, 1182 struct spdk_blob_md_page **pages, 1183 struct spdk_blob_md_page *cur_page, 1184 uint32_t *page_count, uint8_t **buf, 1185 size_t *remaining_sz) 1186 { 1187 const struct spdk_xattr *xattr; 1188 int rc; 1189 1190 TAILQ_FOREACH(xattr, xattrs, link) { 1191 size_t required_sz = 0; 1192 1193 rc = blob_serialize_xattr(xattr, 1194 *buf, *remaining_sz, 1195 &required_sz, internal); 1196 if (rc < 0) { 1197 /* Need to add a new page to the chain */ 1198 rc = blob_serialize_add_page(blob, pages, page_count, 1199 &cur_page); 1200 if (rc < 0) { 1201 spdk_free(*pages); 1202 *pages = NULL; 1203 *page_count = 0; 1204 return rc; 1205 } 1206 1207 *buf = (uint8_t *)cur_page->descriptors; 1208 *remaining_sz = sizeof(cur_page->descriptors); 1209 1210 /* Try again */ 1211 required_sz = 0; 1212 rc = blob_serialize_xattr(xattr, 1213 *buf, *remaining_sz, 1214 &required_sz, internal); 1215 1216 if (rc < 0) { 1217 spdk_free(*pages); 1218 *pages = NULL; 1219 *page_count = 0; 1220 return rc; 1221 } 1222 } 1223 1224 *remaining_sz -= required_sz; 1225 *buf += required_sz; 1226 } 1227 1228 return 0; 1229 } 1230 1231 static int 1232 blob_serialize(const struct spdk_blob *blob, struct spdk_blob_md_page **pages, 1233 uint32_t *page_count) 1234 { 1235 struct spdk_blob_md_page *cur_page; 1236 int rc; 1237 uint8_t *buf; 1238 size_t remaining_sz; 1239 1240 assert(pages != NULL); 1241 assert(page_count != NULL); 1242 assert(blob != NULL); 1243 assert(blob->state == SPDK_BLOB_STATE_DIRTY); 1244 1245 *pages = NULL; 1246 *page_count = 0; 1247 1248 /* A blob always has at least 1 page, even if it has no descriptors */ 1249 rc = blob_serialize_add_page(blob, pages, page_count, &cur_page); 1250 if (rc < 0) { 1251 return rc; 1252 } 1253 1254 buf = (uint8_t *)cur_page->descriptors; 1255 remaining_sz = sizeof(cur_page->descriptors); 1256 1257 /* Serialize flags */ 1258 blob_serialize_flags(blob, buf, &remaining_sz); 1259 buf += sizeof(struct spdk_blob_md_descriptor_flags); 1260 1261 /* Serialize xattrs */ 1262 rc = blob_serialize_xattrs(blob, &blob->xattrs, false, 1263 pages, cur_page, page_count, &buf, &remaining_sz); 1264 if (rc < 0) { 1265 return rc; 1266 } 1267 1268 /* Serialize internal xattrs */ 1269 rc = blob_serialize_xattrs(blob, &blob->xattrs_internal, true, 1270 pages, cur_page, page_count, &buf, &remaining_sz); 1271 if (rc < 0) { 1272 return rc; 1273 } 1274 1275 if (blob->use_extent_table) { 1276 /* Serialize extent table */ 1277 rc = blob_serialize_extent_table(blob, pages, cur_page, page_count, &buf, &remaining_sz); 1278 } else { 1279 /* Serialize extents */ 1280 rc = blob_serialize_extents_rle(blob, pages, cur_page, page_count, &buf, &remaining_sz); 1281 } 1282 1283 return rc; 1284 } 1285 1286 struct spdk_blob_load_ctx { 1287 struct spdk_blob *blob; 1288 1289 struct spdk_blob_md_page *pages; 1290 uint32_t num_pages; 1291 uint32_t next_extent_page; 1292 spdk_bs_sequence_t *seq; 1293 1294 spdk_bs_sequence_cpl cb_fn; 1295 void *cb_arg; 1296 }; 1297 1298 static uint32_t 1299 blob_md_page_calc_crc(void *page) 1300 { 1301 uint32_t crc; 1302 1303 crc = BLOB_CRC32C_INITIAL; 1304 crc = spdk_crc32c_update(page, SPDK_BS_PAGE_SIZE - 4, crc); 1305 crc ^= BLOB_CRC32C_INITIAL; 1306 1307 return crc; 1308 1309 } 1310 1311 static void 1312 blob_load_final(struct spdk_blob_load_ctx *ctx, int bserrno) 1313 { 1314 struct spdk_blob *blob = ctx->blob; 1315 1316 if (bserrno == 0) { 1317 blob_mark_clean(blob); 1318 } 1319 1320 ctx->cb_fn(ctx->seq, ctx->cb_arg, bserrno); 1321 1322 /* Free the memory */ 1323 spdk_free(ctx->pages); 1324 free(ctx); 1325 } 1326 1327 static void 1328 blob_load_snapshot_cpl(void *cb_arg, struct spdk_blob *snapshot, int bserrno) 1329 { 1330 struct spdk_blob_load_ctx *ctx = cb_arg; 1331 struct spdk_blob *blob = ctx->blob; 1332 1333 if (bserrno == 0) { 1334 blob->back_bs_dev = bs_create_blob_bs_dev(snapshot); 1335 if (blob->back_bs_dev == NULL) { 1336 bserrno = -ENOMEM; 1337 } 1338 } 1339 if (bserrno != 0) { 1340 SPDK_ERRLOG("Snapshot fail\n"); 1341 } 1342 1343 blob_load_final(ctx, bserrno); 1344 } 1345 1346 static void blob_update_clear_method(struct spdk_blob *blob); 1347 1348 static void 1349 blob_load_backing_dev(void *cb_arg) 1350 { 1351 struct spdk_blob_load_ctx *ctx = cb_arg; 1352 struct spdk_blob *blob = ctx->blob; 1353 const void *value; 1354 size_t len; 1355 int rc; 1356 1357 if (spdk_blob_is_thin_provisioned(blob)) { 1358 rc = blob_get_xattr_value(blob, BLOB_SNAPSHOT, &value, &len, true); 1359 if (rc == 0) { 1360 if (len != sizeof(spdk_blob_id)) { 1361 blob_load_final(ctx, -EINVAL); 1362 return; 1363 } 1364 /* open snapshot blob and continue in the callback function */ 1365 blob->parent_id = *(spdk_blob_id *)value; 1366 spdk_bs_open_blob(blob->bs, blob->parent_id, 1367 blob_load_snapshot_cpl, ctx); 1368 return; 1369 } else { 1370 /* add zeroes_dev for thin provisioned blob */ 1371 blob->back_bs_dev = bs_create_zeroes_dev(); 1372 } 1373 } else { 1374 /* standard blob */ 1375 blob->back_bs_dev = NULL; 1376 } 1377 blob_load_final(ctx, 0); 1378 } 1379 1380 static void 1381 blob_load_cpl_extents_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 1382 { 1383 struct spdk_blob_load_ctx *ctx = cb_arg; 1384 struct spdk_blob *blob = ctx->blob; 1385 struct spdk_blob_md_page *page; 1386 uint64_t i; 1387 uint32_t crc; 1388 uint64_t lba; 1389 void *tmp; 1390 uint64_t sz; 1391 1392 if (bserrno) { 1393 SPDK_ERRLOG("Extent page read failed: %d\n", bserrno); 1394 blob_load_final(ctx, bserrno); 1395 return; 1396 } 1397 1398 if (ctx->pages == NULL) { 1399 /* First iteration of this function, allocate buffer for single EXTENT_PAGE */ 1400 ctx->pages = spdk_zmalloc(SPDK_BS_PAGE_SIZE, 0, 1401 NULL, SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA); 1402 if (!ctx->pages) { 1403 blob_load_final(ctx, -ENOMEM); 1404 return; 1405 } 1406 ctx->num_pages = 1; 1407 ctx->next_extent_page = 0; 1408 } else { 1409 page = &ctx->pages[0]; 1410 crc = blob_md_page_calc_crc(page); 1411 if (crc != page->crc) { 1412 blob_load_final(ctx, -EINVAL); 1413 return; 1414 } 1415 1416 if (page->next != SPDK_INVALID_MD_PAGE) { 1417 blob_load_final(ctx, -EINVAL); 1418 return; 1419 } 1420 1421 bserrno = blob_parse_extent_page(page, blob); 1422 if (bserrno) { 1423 blob_load_final(ctx, bserrno); 1424 return; 1425 } 1426 } 1427 1428 for (i = ctx->next_extent_page; i < blob->active.num_extent_pages; i++) { 1429 if (blob->active.extent_pages[i] != 0) { 1430 /* Extent page was allocated, read and parse it. */ 1431 lba = bs_md_page_to_lba(blob->bs, blob->active.extent_pages[i]); 1432 ctx->next_extent_page = i + 1; 1433 1434 bs_sequence_read_dev(seq, &ctx->pages[0], lba, 1435 bs_byte_to_lba(blob->bs, SPDK_BS_PAGE_SIZE), 1436 blob_load_cpl_extents_cpl, ctx); 1437 return; 1438 } else { 1439 /* Thin provisioned blobs can point to unallocated extent pages. 1440 * In this case blob size should be increased by up to the amount left in remaining_clusters_in_et. */ 1441 1442 sz = spdk_min(blob->remaining_clusters_in_et, SPDK_EXTENTS_PER_EP); 1443 blob->active.num_clusters += sz; 1444 blob->remaining_clusters_in_et -= sz; 1445 1446 assert(spdk_blob_is_thin_provisioned(blob)); 1447 assert(i + 1 < blob->active.num_extent_pages || blob->remaining_clusters_in_et == 0); 1448 1449 tmp = realloc(blob->active.clusters, blob->active.num_clusters * sizeof(*blob->active.clusters)); 1450 if (tmp == NULL) { 1451 blob_load_final(ctx, -ENOMEM); 1452 return; 1453 } 1454 memset(tmp + sizeof(*blob->active.clusters) * blob->active.cluster_array_size, 0, 1455 sizeof(*blob->active.clusters) * (blob->active.num_clusters - blob->active.cluster_array_size)); 1456 blob->active.clusters = tmp; 1457 blob->active.cluster_array_size = blob->active.num_clusters; 1458 } 1459 } 1460 1461 blob_load_backing_dev(ctx); 1462 } 1463 1464 static void 1465 blob_load_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 1466 { 1467 struct spdk_blob_load_ctx *ctx = cb_arg; 1468 struct spdk_blob *blob = ctx->blob; 1469 struct spdk_blob_md_page *page; 1470 int rc; 1471 uint32_t crc; 1472 uint32_t current_page; 1473 1474 if (ctx->num_pages == 1) { 1475 current_page = bs_blobid_to_page(blob->id); 1476 } else { 1477 assert(ctx->num_pages != 0); 1478 page = &ctx->pages[ctx->num_pages - 2]; 1479 current_page = page->next; 1480 } 1481 1482 if (bserrno) { 1483 SPDK_ERRLOG("Metadata page %d read failed for blobid %" PRIu64 ": %d\n", 1484 current_page, blob->id, bserrno); 1485 blob_load_final(ctx, bserrno); 1486 return; 1487 } 1488 1489 page = &ctx->pages[ctx->num_pages - 1]; 1490 crc = blob_md_page_calc_crc(page); 1491 if (crc != page->crc) { 1492 SPDK_ERRLOG("Metadata page %d crc mismatch for blobid %" PRIu64 "\n", 1493 current_page, blob->id); 1494 blob_load_final(ctx, -EINVAL); 1495 return; 1496 } 1497 1498 if (page->next != SPDK_INVALID_MD_PAGE) { 1499 struct spdk_blob_md_page *tmp_pages; 1500 uint32_t next_page = page->next; 1501 uint64_t next_lba = bs_md_page_to_lba(blob->bs, next_page); 1502 1503 /* Read the next page */ 1504 tmp_pages = spdk_realloc(ctx->pages, (sizeof(*page) * (ctx->num_pages + 1)), 0); 1505 if (tmp_pages == NULL) { 1506 blob_load_final(ctx, -ENOMEM); 1507 return; 1508 } 1509 ctx->num_pages++; 1510 ctx->pages = tmp_pages; 1511 1512 bs_sequence_read_dev(seq, &ctx->pages[ctx->num_pages - 1], 1513 next_lba, 1514 bs_byte_to_lba(blob->bs, sizeof(*page)), 1515 blob_load_cpl, ctx); 1516 return; 1517 } 1518 1519 /* Parse the pages */ 1520 rc = blob_parse(ctx->pages, ctx->num_pages, blob); 1521 if (rc) { 1522 blob_load_final(ctx, rc); 1523 return; 1524 } 1525 1526 if (blob->extent_table_found == true) { 1527 /* If EXTENT_TABLE was found, that means support for it should be enabled. */ 1528 assert(blob->extent_rle_found == false); 1529 blob->use_extent_table = true; 1530 } else { 1531 /* If EXTENT_RLE or no extent_* descriptor was found disable support 1532 * for extent table. No extent_* descriptors means that blob has length of 0 1533 * and no extent_rle descriptors were persisted for it. 1534 * EXTENT_TABLE if used, is always present in metadata regardless of length. */ 1535 blob->use_extent_table = false; 1536 } 1537 1538 /* Check the clear_method stored in metadata vs what may have been passed 1539 * via spdk_bs_open_blob_ext() and update accordingly. 1540 */ 1541 blob_update_clear_method(blob); 1542 1543 spdk_free(ctx->pages); 1544 ctx->pages = NULL; 1545 1546 if (blob->extent_table_found) { 1547 blob_load_cpl_extents_cpl(seq, ctx, 0); 1548 } else { 1549 blob_load_backing_dev(ctx); 1550 } 1551 } 1552 1553 /* Load a blob from disk given a blobid */ 1554 static void 1555 blob_load(spdk_bs_sequence_t *seq, struct spdk_blob *blob, 1556 spdk_bs_sequence_cpl cb_fn, void *cb_arg) 1557 { 1558 struct spdk_blob_load_ctx *ctx; 1559 struct spdk_blob_store *bs; 1560 uint32_t page_num; 1561 uint64_t lba; 1562 1563 blob_verify_md_op(blob); 1564 1565 bs = blob->bs; 1566 1567 ctx = calloc(1, sizeof(*ctx)); 1568 if (!ctx) { 1569 cb_fn(seq, cb_arg, -ENOMEM); 1570 return; 1571 } 1572 1573 ctx->blob = blob; 1574 ctx->pages = spdk_realloc(ctx->pages, SPDK_BS_PAGE_SIZE, 0); 1575 if (!ctx->pages) { 1576 free(ctx); 1577 cb_fn(seq, cb_arg, -ENOMEM); 1578 return; 1579 } 1580 ctx->num_pages = 1; 1581 ctx->cb_fn = cb_fn; 1582 ctx->cb_arg = cb_arg; 1583 ctx->seq = seq; 1584 1585 page_num = bs_blobid_to_page(blob->id); 1586 lba = bs_md_page_to_lba(blob->bs, page_num); 1587 1588 blob->state = SPDK_BLOB_STATE_LOADING; 1589 1590 bs_sequence_read_dev(seq, &ctx->pages[0], lba, 1591 bs_byte_to_lba(bs, SPDK_BS_PAGE_SIZE), 1592 blob_load_cpl, ctx); 1593 } 1594 1595 struct spdk_blob_persist_ctx { 1596 struct spdk_blob *blob; 1597 1598 struct spdk_bs_super_block *super; 1599 1600 struct spdk_blob_md_page *pages; 1601 uint32_t next_extent_page; 1602 struct spdk_blob_md_page *extent_page; 1603 1604 spdk_bs_sequence_t *seq; 1605 spdk_bs_sequence_cpl cb_fn; 1606 void *cb_arg; 1607 TAILQ_ENTRY(spdk_blob_persist_ctx) link; 1608 }; 1609 1610 static void 1611 bs_batch_clear_dev(struct spdk_blob_persist_ctx *ctx, spdk_bs_batch_t *batch, uint64_t lba, 1612 uint64_t lba_count) 1613 { 1614 switch (ctx->blob->clear_method) { 1615 case BLOB_CLEAR_WITH_DEFAULT: 1616 case BLOB_CLEAR_WITH_UNMAP: 1617 bs_batch_unmap_dev(batch, lba, lba_count); 1618 break; 1619 case BLOB_CLEAR_WITH_WRITE_ZEROES: 1620 bs_batch_write_zeroes_dev(batch, lba, lba_count); 1621 break; 1622 case BLOB_CLEAR_WITH_NONE: 1623 default: 1624 break; 1625 } 1626 } 1627 1628 static void blob_persist_check_dirty(struct spdk_blob_persist_ctx *ctx); 1629 1630 static void 1631 blob_persist_complete_cb(void *arg) 1632 { 1633 struct spdk_blob_persist_ctx *ctx = arg; 1634 1635 /* Call user callback */ 1636 ctx->cb_fn(ctx->seq, ctx->cb_arg, 0); 1637 1638 /* Free the memory */ 1639 spdk_free(ctx->pages); 1640 free(ctx); 1641 } 1642 1643 static void 1644 blob_persist_complete(spdk_bs_sequence_t *seq, struct spdk_blob_persist_ctx *ctx, int bserrno) 1645 { 1646 struct spdk_blob_persist_ctx *next_persist, *tmp; 1647 struct spdk_blob *blob = ctx->blob; 1648 1649 if (bserrno == 0) { 1650 blob_mark_clean(blob); 1651 } 1652 1653 assert(ctx == TAILQ_FIRST(&blob->persists_to_complete)); 1654 1655 /* Complete all persists that were pending when the current persist started */ 1656 TAILQ_FOREACH_SAFE(next_persist, &blob->persists_to_complete, link, tmp) { 1657 TAILQ_REMOVE(&blob->persists_to_complete, next_persist, link); 1658 spdk_thread_send_msg(spdk_get_thread(), blob_persist_complete_cb, next_persist); 1659 } 1660 1661 if (TAILQ_EMPTY(&blob->pending_persists)) { 1662 return; 1663 } 1664 1665 /* Queue up all pending persists for completion and start blob persist with first one */ 1666 TAILQ_SWAP(&blob->persists_to_complete, &blob->pending_persists, spdk_blob_persist_ctx, link); 1667 next_persist = TAILQ_FIRST(&blob->persists_to_complete); 1668 1669 blob->state = SPDK_BLOB_STATE_DIRTY; 1670 blob_persist_check_dirty(next_persist); 1671 } 1672 1673 static void 1674 blob_persist_clear_extents_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 1675 { 1676 struct spdk_blob_persist_ctx *ctx = cb_arg; 1677 struct spdk_blob *blob = ctx->blob; 1678 struct spdk_blob_store *bs = blob->bs; 1679 size_t i; 1680 1681 if (bserrno != 0) { 1682 blob_persist_complete(seq, ctx, bserrno); 1683 return; 1684 } 1685 1686 /* Release all extent_pages that were truncated */ 1687 for (i = blob->active.num_extent_pages; i < blob->active.extent_pages_array_size; i++) { 1688 /* Nothing to release if it was not allocated */ 1689 if (blob->active.extent_pages[i] != 0) { 1690 bs_release_md_page(bs, blob->active.extent_pages[i]); 1691 } 1692 } 1693 1694 if (blob->active.num_extent_pages == 0) { 1695 free(blob->active.extent_pages); 1696 blob->active.extent_pages = NULL; 1697 blob->active.extent_pages_array_size = 0; 1698 } else if (blob->active.num_extent_pages != blob->active.extent_pages_array_size) { 1699 #ifndef __clang_analyzer__ 1700 void *tmp; 1701 1702 /* scan-build really can't figure reallocs, workaround it */ 1703 tmp = realloc(blob->active.extent_pages, sizeof(uint32_t) * blob->active.num_extent_pages); 1704 assert(tmp != NULL); 1705 blob->active.extent_pages = tmp; 1706 #endif 1707 blob->active.extent_pages_array_size = blob->active.num_extent_pages; 1708 } 1709 1710 blob_persist_complete(seq, ctx, bserrno); 1711 } 1712 1713 static void 1714 blob_persist_clear_extents(spdk_bs_sequence_t *seq, struct spdk_blob_persist_ctx *ctx) 1715 { 1716 struct spdk_blob *blob = ctx->blob; 1717 struct spdk_blob_store *bs = blob->bs; 1718 size_t i; 1719 uint64_t lba; 1720 uint64_t lba_count; 1721 spdk_bs_batch_t *batch; 1722 1723 batch = bs_sequence_to_batch(seq, blob_persist_clear_extents_cpl, ctx); 1724 lba_count = bs_byte_to_lba(bs, SPDK_BS_PAGE_SIZE); 1725 1726 /* Clear all extent_pages that were truncated */ 1727 for (i = blob->active.num_extent_pages; i < blob->active.extent_pages_array_size; i++) { 1728 /* Nothing to clear if it was not allocated */ 1729 if (blob->active.extent_pages[i] != 0) { 1730 lba = bs_md_page_to_lba(bs, blob->active.extent_pages[i]); 1731 bs_batch_write_zeroes_dev(batch, lba, lba_count); 1732 } 1733 } 1734 1735 bs_batch_close(batch); 1736 } 1737 1738 static void 1739 blob_persist_clear_clusters_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 1740 { 1741 struct spdk_blob_persist_ctx *ctx = cb_arg; 1742 struct spdk_blob *blob = ctx->blob; 1743 struct spdk_blob_store *bs = blob->bs; 1744 size_t i; 1745 1746 if (bserrno != 0) { 1747 blob_persist_complete(seq, ctx, bserrno); 1748 return; 1749 } 1750 1751 pthread_mutex_lock(&bs->used_clusters_mutex); 1752 /* Release all clusters that were truncated */ 1753 for (i = blob->active.num_clusters; i < blob->active.cluster_array_size; i++) { 1754 uint32_t cluster_num = bs_lba_to_cluster(bs, blob->active.clusters[i]); 1755 1756 /* Nothing to release if it was not allocated */ 1757 if (blob->active.clusters[i] != 0) { 1758 bs_release_cluster(bs, cluster_num); 1759 } 1760 } 1761 pthread_mutex_unlock(&bs->used_clusters_mutex); 1762 1763 if (blob->active.num_clusters == 0) { 1764 free(blob->active.clusters); 1765 blob->active.clusters = NULL; 1766 blob->active.cluster_array_size = 0; 1767 } else if (blob->active.num_clusters != blob->active.cluster_array_size) { 1768 #ifndef __clang_analyzer__ 1769 void *tmp; 1770 1771 /* scan-build really can't figure reallocs, workaround it */ 1772 tmp = realloc(blob->active.clusters, sizeof(*blob->active.clusters) * blob->active.num_clusters); 1773 assert(tmp != NULL); 1774 blob->active.clusters = tmp; 1775 1776 #endif 1777 blob->active.cluster_array_size = blob->active.num_clusters; 1778 } 1779 1780 /* Move on to clearing extent pages */ 1781 blob_persist_clear_extents(seq, ctx); 1782 } 1783 1784 static void 1785 blob_persist_clear_clusters(spdk_bs_sequence_t *seq, struct spdk_blob_persist_ctx *ctx) 1786 { 1787 struct spdk_blob *blob = ctx->blob; 1788 struct spdk_blob_store *bs = blob->bs; 1789 spdk_bs_batch_t *batch; 1790 size_t i; 1791 uint64_t lba; 1792 uint64_t lba_count; 1793 1794 /* Clusters don't move around in blobs. The list shrinks or grows 1795 * at the end, but no changes ever occur in the middle of the list. 1796 */ 1797 1798 batch = bs_sequence_to_batch(seq, blob_persist_clear_clusters_cpl, ctx); 1799 1800 /* Clear all clusters that were truncated */ 1801 lba = 0; 1802 lba_count = 0; 1803 for (i = blob->active.num_clusters; i < blob->active.cluster_array_size; i++) { 1804 uint64_t next_lba = blob->active.clusters[i]; 1805 uint64_t next_lba_count = bs_cluster_to_lba(bs, 1); 1806 1807 if (next_lba > 0 && (lba + lba_count) == next_lba) { 1808 /* This cluster is contiguous with the previous one. */ 1809 lba_count += next_lba_count; 1810 continue; 1811 } else if (next_lba == 0) { 1812 continue; 1813 } 1814 1815 /* This cluster is not contiguous with the previous one. */ 1816 1817 /* If a run of LBAs previously existing, clear them now */ 1818 if (lba_count > 0) { 1819 bs_batch_clear_dev(ctx, batch, lba, lba_count); 1820 } 1821 1822 /* Start building the next batch */ 1823 lba = next_lba; 1824 if (next_lba > 0) { 1825 lba_count = next_lba_count; 1826 } else { 1827 lba_count = 0; 1828 } 1829 } 1830 1831 /* If we ended with a contiguous set of LBAs, clear them now */ 1832 if (lba_count > 0) { 1833 bs_batch_clear_dev(ctx, batch, lba, lba_count); 1834 } 1835 1836 bs_batch_close(batch); 1837 } 1838 1839 static void 1840 blob_persist_zero_pages_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 1841 { 1842 struct spdk_blob_persist_ctx *ctx = cb_arg; 1843 struct spdk_blob *blob = ctx->blob; 1844 struct spdk_blob_store *bs = blob->bs; 1845 size_t i; 1846 1847 if (bserrno != 0) { 1848 blob_persist_complete(seq, ctx, bserrno); 1849 return; 1850 } 1851 1852 /* This loop starts at 1 because the first page is special and handled 1853 * below. The pages (except the first) are never written in place, 1854 * so any pages in the clean list must be zeroed. 1855 */ 1856 for (i = 1; i < blob->clean.num_pages; i++) { 1857 bs_release_md_page(bs, blob->clean.pages[i]); 1858 } 1859 1860 if (blob->active.num_pages == 0) { 1861 uint32_t page_num; 1862 1863 page_num = bs_blobid_to_page(blob->id); 1864 bs_release_md_page(bs, page_num); 1865 } 1866 1867 /* Move on to clearing clusters */ 1868 blob_persist_clear_clusters(seq, ctx); 1869 } 1870 1871 static void 1872 blob_persist_zero_pages(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 1873 { 1874 struct spdk_blob_persist_ctx *ctx = cb_arg; 1875 struct spdk_blob *blob = ctx->blob; 1876 struct spdk_blob_store *bs = blob->bs; 1877 uint64_t lba; 1878 uint64_t lba_count; 1879 spdk_bs_batch_t *batch; 1880 size_t i; 1881 1882 if (bserrno != 0) { 1883 blob_persist_complete(seq, ctx, bserrno); 1884 return; 1885 } 1886 1887 batch = bs_sequence_to_batch(seq, blob_persist_zero_pages_cpl, ctx); 1888 1889 lba_count = bs_byte_to_lba(bs, SPDK_BS_PAGE_SIZE); 1890 1891 /* This loop starts at 1 because the first page is special and handled 1892 * below. The pages (except the first) are never written in place, 1893 * so any pages in the clean list must be zeroed. 1894 */ 1895 for (i = 1; i < blob->clean.num_pages; i++) { 1896 lba = bs_md_page_to_lba(bs, blob->clean.pages[i]); 1897 1898 bs_batch_write_zeroes_dev(batch, lba, lba_count); 1899 } 1900 1901 /* The first page will only be zeroed if this is a delete. */ 1902 if (blob->active.num_pages == 0) { 1903 uint32_t page_num; 1904 1905 /* The first page in the metadata goes where the blobid indicates */ 1906 page_num = bs_blobid_to_page(blob->id); 1907 lba = bs_md_page_to_lba(bs, page_num); 1908 1909 bs_batch_write_zeroes_dev(batch, lba, lba_count); 1910 } 1911 1912 bs_batch_close(batch); 1913 } 1914 1915 static void 1916 blob_persist_write_page_root(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 1917 { 1918 struct spdk_blob_persist_ctx *ctx = cb_arg; 1919 struct spdk_blob *blob = ctx->blob; 1920 struct spdk_blob_store *bs = blob->bs; 1921 uint64_t lba; 1922 uint32_t lba_count; 1923 struct spdk_blob_md_page *page; 1924 1925 if (bserrno != 0) { 1926 blob_persist_complete(seq, ctx, bserrno); 1927 return; 1928 } 1929 1930 if (blob->active.num_pages == 0) { 1931 /* Move on to the next step */ 1932 blob_persist_zero_pages(seq, ctx, 0); 1933 return; 1934 } 1935 1936 lba_count = bs_byte_to_lba(bs, sizeof(*page)); 1937 1938 page = &ctx->pages[0]; 1939 /* The first page in the metadata goes where the blobid indicates */ 1940 lba = bs_md_page_to_lba(bs, bs_blobid_to_page(blob->id)); 1941 1942 bs_sequence_write_dev(seq, page, lba, lba_count, 1943 blob_persist_zero_pages, ctx); 1944 } 1945 1946 static void 1947 blob_persist_write_page_chain(spdk_bs_sequence_t *seq, struct spdk_blob_persist_ctx *ctx) 1948 { 1949 struct spdk_blob *blob = ctx->blob; 1950 struct spdk_blob_store *bs = blob->bs; 1951 uint64_t lba; 1952 uint32_t lba_count; 1953 struct spdk_blob_md_page *page; 1954 spdk_bs_batch_t *batch; 1955 size_t i; 1956 1957 /* Clusters don't move around in blobs. The list shrinks or grows 1958 * at the end, but no changes ever occur in the middle of the list. 1959 */ 1960 1961 lba_count = bs_byte_to_lba(bs, sizeof(*page)); 1962 1963 batch = bs_sequence_to_batch(seq, blob_persist_write_page_root, ctx); 1964 1965 /* This starts at 1. The root page is not written until 1966 * all of the others are finished 1967 */ 1968 for (i = 1; i < blob->active.num_pages; i++) { 1969 page = &ctx->pages[i]; 1970 assert(page->sequence_num == i); 1971 1972 lba = bs_md_page_to_lba(bs, blob->active.pages[i]); 1973 1974 bs_batch_write_dev(batch, page, lba, lba_count); 1975 } 1976 1977 bs_batch_close(batch); 1978 } 1979 1980 static int 1981 blob_resize(struct spdk_blob *blob, uint64_t sz) 1982 { 1983 uint64_t i; 1984 uint64_t *tmp; 1985 uint64_t cluster; 1986 uint32_t lfmd; /* lowest free md page */ 1987 uint64_t num_clusters; 1988 uint32_t *ep_tmp; 1989 uint64_t new_num_ep = 0, current_num_ep = 0; 1990 struct spdk_blob_store *bs; 1991 1992 bs = blob->bs; 1993 1994 blob_verify_md_op(blob); 1995 1996 if (blob->active.num_clusters == sz) { 1997 return 0; 1998 } 1999 2000 if (blob->active.num_clusters < blob->active.cluster_array_size) { 2001 /* If this blob was resized to be larger, then smaller, then 2002 * larger without syncing, then the cluster array already 2003 * contains spare assigned clusters we can use. 2004 */ 2005 num_clusters = spdk_min(blob->active.cluster_array_size, 2006 sz); 2007 } else { 2008 num_clusters = blob->active.num_clusters; 2009 } 2010 2011 if (blob->use_extent_table) { 2012 /* Round up since every cluster beyond current Extent Table size, 2013 * requires new extent page. */ 2014 new_num_ep = spdk_divide_round_up(sz, SPDK_EXTENTS_PER_EP); 2015 current_num_ep = spdk_divide_round_up(num_clusters, SPDK_EXTENTS_PER_EP); 2016 } 2017 2018 /* Check first that we have enough clusters and md pages before we start claiming them. */ 2019 if (sz > num_clusters && spdk_blob_is_thin_provisioned(blob) == false) { 2020 if ((sz - num_clusters) > bs->num_free_clusters) { 2021 return -ENOSPC; 2022 } 2023 lfmd = 0; 2024 for (i = current_num_ep; i < new_num_ep ; i++) { 2025 lfmd = spdk_bit_array_find_first_clear(blob->bs->used_md_pages, lfmd); 2026 if (lfmd == UINT32_MAX) { 2027 /* No more free md pages. Cannot satisfy the request */ 2028 return -ENOSPC; 2029 } 2030 } 2031 } 2032 2033 if (sz > num_clusters) { 2034 /* Expand the cluster array if necessary. 2035 * We only shrink the array when persisting. 2036 */ 2037 tmp = realloc(blob->active.clusters, sizeof(*blob->active.clusters) * sz); 2038 if (sz > 0 && tmp == NULL) { 2039 return -ENOMEM; 2040 } 2041 memset(tmp + blob->active.cluster_array_size, 0, 2042 sizeof(*blob->active.clusters) * (sz - blob->active.cluster_array_size)); 2043 blob->active.clusters = tmp; 2044 blob->active.cluster_array_size = sz; 2045 2046 /* Expand the extents table, only if enough clusters were added */ 2047 if (new_num_ep > current_num_ep && blob->use_extent_table) { 2048 ep_tmp = realloc(blob->active.extent_pages, sizeof(*blob->active.extent_pages) * new_num_ep); 2049 if (new_num_ep > 0 && ep_tmp == NULL) { 2050 return -ENOMEM; 2051 } 2052 memset(ep_tmp + blob->active.extent_pages_array_size, 0, 2053 sizeof(*blob->active.extent_pages) * (new_num_ep - blob->active.extent_pages_array_size)); 2054 blob->active.extent_pages = ep_tmp; 2055 blob->active.extent_pages_array_size = new_num_ep; 2056 } 2057 } 2058 2059 blob->state = SPDK_BLOB_STATE_DIRTY; 2060 2061 if (spdk_blob_is_thin_provisioned(blob) == false) { 2062 cluster = 0; 2063 lfmd = 0; 2064 pthread_mutex_lock(&blob->bs->used_clusters_mutex); 2065 for (i = num_clusters; i < sz; i++) { 2066 bs_allocate_cluster(blob, i, &cluster, &lfmd, true); 2067 lfmd++; 2068 } 2069 pthread_mutex_unlock(&blob->bs->used_clusters_mutex); 2070 } 2071 2072 blob->active.num_clusters = sz; 2073 blob->active.num_extent_pages = new_num_ep; 2074 2075 return 0; 2076 } 2077 2078 static void 2079 blob_persist_generate_new_md(struct spdk_blob_persist_ctx *ctx) 2080 { 2081 spdk_bs_sequence_t *seq = ctx->seq; 2082 struct spdk_blob *blob = ctx->blob; 2083 struct spdk_blob_store *bs = blob->bs; 2084 uint64_t i; 2085 uint32_t page_num; 2086 void *tmp; 2087 int rc; 2088 2089 /* Generate the new metadata */ 2090 rc = blob_serialize(blob, &ctx->pages, &blob->active.num_pages); 2091 if (rc < 0) { 2092 blob_persist_complete(seq, ctx, rc); 2093 return; 2094 } 2095 2096 assert(blob->active.num_pages >= 1); 2097 2098 /* Resize the cache of page indices */ 2099 tmp = realloc(blob->active.pages, blob->active.num_pages * sizeof(*blob->active.pages)); 2100 if (!tmp) { 2101 blob_persist_complete(seq, ctx, -ENOMEM); 2102 return; 2103 } 2104 blob->active.pages = tmp; 2105 2106 /* Assign this metadata to pages. This requires two passes - 2107 * one to verify that there are enough pages and a second 2108 * to actually claim them. */ 2109 page_num = 0; 2110 /* Note that this loop starts at one. The first page location is fixed by the blobid. */ 2111 for (i = 1; i < blob->active.num_pages; i++) { 2112 page_num = spdk_bit_array_find_first_clear(bs->used_md_pages, page_num); 2113 if (page_num == UINT32_MAX) { 2114 blob_persist_complete(seq, ctx, -ENOMEM); 2115 return; 2116 } 2117 page_num++; 2118 } 2119 2120 page_num = 0; 2121 blob->active.pages[0] = bs_blobid_to_page(blob->id); 2122 for (i = 1; i < blob->active.num_pages; i++) { 2123 page_num = spdk_bit_array_find_first_clear(bs->used_md_pages, page_num); 2124 ctx->pages[i - 1].next = page_num; 2125 /* Now that previous metadata page is complete, calculate the crc for it. */ 2126 ctx->pages[i - 1].crc = blob_md_page_calc_crc(&ctx->pages[i - 1]); 2127 blob->active.pages[i] = page_num; 2128 bs_claim_md_page(bs, page_num); 2129 SPDK_DEBUGLOG(blob, "Claiming page %u for blob %" PRIu64 "\n", page_num, blob->id); 2130 page_num++; 2131 } 2132 ctx->pages[i - 1].crc = blob_md_page_calc_crc(&ctx->pages[i - 1]); 2133 /* Start writing the metadata from last page to first */ 2134 blob->state = SPDK_BLOB_STATE_CLEAN; 2135 blob_persist_write_page_chain(seq, ctx); 2136 } 2137 2138 static void 2139 blob_persist_write_extent_pages(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 2140 { 2141 struct spdk_blob_persist_ctx *ctx = cb_arg; 2142 struct spdk_blob *blob = ctx->blob; 2143 size_t i; 2144 uint32_t extent_page_id; 2145 uint32_t page_count = 0; 2146 int rc; 2147 2148 if (ctx->extent_page != NULL) { 2149 spdk_free(ctx->extent_page); 2150 ctx->extent_page = NULL; 2151 } 2152 2153 if (bserrno != 0) { 2154 blob_persist_complete(seq, ctx, bserrno); 2155 return; 2156 } 2157 2158 /* Only write out Extent Pages when blob was resized. */ 2159 for (i = ctx->next_extent_page; i < blob->active.extent_pages_array_size; i++) { 2160 extent_page_id = blob->active.extent_pages[i]; 2161 if (extent_page_id == 0) { 2162 /* No Extent Page to persist */ 2163 assert(spdk_blob_is_thin_provisioned(blob)); 2164 continue; 2165 } 2166 assert(spdk_bit_array_get(blob->bs->used_md_pages, extent_page_id)); 2167 ctx->next_extent_page = i + 1; 2168 rc = blob_serialize_add_page(ctx->blob, &ctx->extent_page, &page_count, &ctx->extent_page); 2169 if (rc < 0) { 2170 blob_persist_complete(seq, ctx, rc); 2171 return; 2172 } 2173 2174 blob->state = SPDK_BLOB_STATE_DIRTY; 2175 blob_serialize_extent_page(blob, i * SPDK_EXTENTS_PER_EP, ctx->extent_page); 2176 2177 ctx->extent_page->crc = blob_md_page_calc_crc(ctx->extent_page); 2178 2179 bs_sequence_write_dev(seq, ctx->extent_page, bs_md_page_to_lba(blob->bs, extent_page_id), 2180 bs_byte_to_lba(blob->bs, SPDK_BS_PAGE_SIZE), 2181 blob_persist_write_extent_pages, ctx); 2182 return; 2183 } 2184 2185 blob_persist_generate_new_md(ctx); 2186 } 2187 2188 static void 2189 blob_persist_start(struct spdk_blob_persist_ctx *ctx) 2190 { 2191 spdk_bs_sequence_t *seq = ctx->seq; 2192 struct spdk_blob *blob = ctx->blob; 2193 2194 if (blob->active.num_pages == 0) { 2195 /* This is the signal that the blob should be deleted. 2196 * Immediately jump to the clean up routine. */ 2197 assert(blob->clean.num_pages > 0); 2198 blob->state = SPDK_BLOB_STATE_CLEAN; 2199 blob_persist_zero_pages(seq, ctx, 0); 2200 return; 2201 2202 } 2203 2204 if (blob->clean.num_clusters < blob->active.num_clusters) { 2205 /* Blob was resized up */ 2206 assert(blob->clean.num_extent_pages <= blob->active.num_extent_pages); 2207 ctx->next_extent_page = spdk_max(1, blob->clean.num_extent_pages) - 1; 2208 } else if (blob->active.num_clusters < blob->active.cluster_array_size) { 2209 /* Blob was resized down */ 2210 assert(blob->clean.num_extent_pages >= blob->active.num_extent_pages); 2211 ctx->next_extent_page = spdk_max(1, blob->active.num_extent_pages) - 1; 2212 } else { 2213 /* No change in size occurred */ 2214 blob_persist_generate_new_md(ctx); 2215 return; 2216 } 2217 2218 blob_persist_write_extent_pages(seq, ctx, 0); 2219 } 2220 2221 static void 2222 blob_persist_dirty_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 2223 { 2224 struct spdk_blob_persist_ctx *ctx = cb_arg; 2225 2226 spdk_free(ctx->super); 2227 2228 if (bserrno != 0) { 2229 blob_persist_complete(seq, ctx, bserrno); 2230 return; 2231 } 2232 2233 ctx->blob->bs->clean = 0; 2234 2235 blob_persist_start(ctx); 2236 } 2237 2238 static void 2239 bs_write_super(spdk_bs_sequence_t *seq, struct spdk_blob_store *bs, 2240 struct spdk_bs_super_block *super, spdk_bs_sequence_cpl cb_fn, void *cb_arg); 2241 2242 2243 static void 2244 blob_persist_dirty(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 2245 { 2246 struct spdk_blob_persist_ctx *ctx = cb_arg; 2247 2248 if (bserrno != 0) { 2249 spdk_free(ctx->super); 2250 blob_persist_complete(seq, ctx, bserrno); 2251 return; 2252 } 2253 2254 ctx->super->clean = 0; 2255 if (ctx->super->size == 0) { 2256 ctx->super->size = ctx->blob->bs->dev->blockcnt * ctx->blob->bs->dev->blocklen; 2257 } 2258 2259 bs_write_super(seq, ctx->blob->bs, ctx->super, blob_persist_dirty_cpl, ctx); 2260 } 2261 2262 static void 2263 blob_persist_check_dirty(struct spdk_blob_persist_ctx *ctx) 2264 { 2265 if (ctx->blob->bs->clean) { 2266 ctx->super = spdk_zmalloc(sizeof(*ctx->super), 0x1000, NULL, 2267 SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA); 2268 if (!ctx->super) { 2269 blob_persist_complete(ctx->seq, ctx, -ENOMEM); 2270 return; 2271 } 2272 2273 bs_sequence_read_dev(ctx->seq, ctx->super, bs_page_to_lba(ctx->blob->bs, 0), 2274 bs_byte_to_lba(ctx->blob->bs, sizeof(*ctx->super)), 2275 blob_persist_dirty, ctx); 2276 } else { 2277 blob_persist_start(ctx); 2278 } 2279 } 2280 2281 /* Write a blob to disk */ 2282 static void 2283 blob_persist(spdk_bs_sequence_t *seq, struct spdk_blob *blob, 2284 spdk_bs_sequence_cpl cb_fn, void *cb_arg) 2285 { 2286 struct spdk_blob_persist_ctx *ctx; 2287 2288 blob_verify_md_op(blob); 2289 2290 if (blob->state == SPDK_BLOB_STATE_CLEAN && TAILQ_EMPTY(&blob->persists_to_complete)) { 2291 cb_fn(seq, cb_arg, 0); 2292 return; 2293 } 2294 2295 ctx = calloc(1, sizeof(*ctx)); 2296 if (!ctx) { 2297 cb_fn(seq, cb_arg, -ENOMEM); 2298 return; 2299 } 2300 ctx->blob = blob; 2301 ctx->seq = seq; 2302 ctx->cb_fn = cb_fn; 2303 ctx->cb_arg = cb_arg; 2304 2305 /* Multiple blob persists can affect one another, via blob->state or 2306 * blob mutable data changes. To prevent it, queue up the persists. */ 2307 if (!TAILQ_EMPTY(&blob->persists_to_complete)) { 2308 TAILQ_INSERT_TAIL(&blob->pending_persists, ctx, link); 2309 return; 2310 } 2311 TAILQ_INSERT_HEAD(&blob->persists_to_complete, ctx, link); 2312 2313 blob_persist_check_dirty(ctx); 2314 } 2315 2316 struct spdk_blob_copy_cluster_ctx { 2317 struct spdk_blob *blob; 2318 uint8_t *buf; 2319 uint64_t page; 2320 uint64_t new_cluster; 2321 uint32_t new_extent_page; 2322 spdk_bs_sequence_t *seq; 2323 }; 2324 2325 static void 2326 blob_allocate_and_copy_cluster_cpl(void *cb_arg, int bserrno) 2327 { 2328 struct spdk_blob_copy_cluster_ctx *ctx = cb_arg; 2329 struct spdk_bs_request_set *set = (struct spdk_bs_request_set *)ctx->seq; 2330 TAILQ_HEAD(, spdk_bs_request_set) requests; 2331 spdk_bs_user_op_t *op; 2332 2333 TAILQ_INIT(&requests); 2334 TAILQ_SWAP(&set->channel->need_cluster_alloc, &requests, spdk_bs_request_set, link); 2335 2336 while (!TAILQ_EMPTY(&requests)) { 2337 op = TAILQ_FIRST(&requests); 2338 TAILQ_REMOVE(&requests, op, link); 2339 if (bserrno == 0) { 2340 bs_user_op_execute(op); 2341 } else { 2342 bs_user_op_abort(op); 2343 } 2344 } 2345 2346 spdk_free(ctx->buf); 2347 free(ctx); 2348 } 2349 2350 static void 2351 blob_insert_cluster_cpl(void *cb_arg, int bserrno) 2352 { 2353 struct spdk_blob_copy_cluster_ctx *ctx = cb_arg; 2354 2355 if (bserrno) { 2356 if (bserrno == -EEXIST) { 2357 /* The metadata insert failed because another thread 2358 * allocated the cluster first. Free our cluster 2359 * but continue without error. */ 2360 bserrno = 0; 2361 } 2362 pthread_mutex_lock(&ctx->blob->bs->used_clusters_mutex); 2363 bs_release_cluster(ctx->blob->bs, ctx->new_cluster); 2364 pthread_mutex_unlock(&ctx->blob->bs->used_clusters_mutex); 2365 if (ctx->new_extent_page != 0) { 2366 bs_release_md_page(ctx->blob->bs, ctx->new_extent_page); 2367 } 2368 } 2369 2370 bs_sequence_finish(ctx->seq, bserrno); 2371 } 2372 2373 static void 2374 blob_write_copy_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 2375 { 2376 struct spdk_blob_copy_cluster_ctx *ctx = cb_arg; 2377 uint32_t cluster_number; 2378 2379 if (bserrno) { 2380 /* The write failed, so jump to the final completion handler */ 2381 bs_sequence_finish(seq, bserrno); 2382 return; 2383 } 2384 2385 cluster_number = bs_page_to_cluster(ctx->blob->bs, ctx->page); 2386 2387 blob_insert_cluster_on_md_thread(ctx->blob, cluster_number, ctx->new_cluster, 2388 ctx->new_extent_page, blob_insert_cluster_cpl, ctx); 2389 } 2390 2391 static void 2392 blob_write_copy(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 2393 { 2394 struct spdk_blob_copy_cluster_ctx *ctx = cb_arg; 2395 2396 if (bserrno != 0) { 2397 /* The read failed, so jump to the final completion handler */ 2398 bs_sequence_finish(seq, bserrno); 2399 return; 2400 } 2401 2402 /* Write whole cluster */ 2403 bs_sequence_write_dev(seq, ctx->buf, 2404 bs_cluster_to_lba(ctx->blob->bs, ctx->new_cluster), 2405 bs_cluster_to_lba(ctx->blob->bs, 1), 2406 blob_write_copy_cpl, ctx); 2407 } 2408 2409 static void 2410 bs_allocate_and_copy_cluster(struct spdk_blob *blob, 2411 struct spdk_io_channel *_ch, 2412 uint64_t io_unit, spdk_bs_user_op_t *op) 2413 { 2414 struct spdk_bs_cpl cpl; 2415 struct spdk_bs_channel *ch; 2416 struct spdk_blob_copy_cluster_ctx *ctx; 2417 uint32_t cluster_start_page; 2418 uint32_t cluster_number; 2419 int rc; 2420 2421 ch = spdk_io_channel_get_ctx(_ch); 2422 2423 if (!TAILQ_EMPTY(&ch->need_cluster_alloc)) { 2424 /* There are already operations pending. Queue this user op 2425 * and return because it will be re-executed when the outstanding 2426 * cluster allocation completes. */ 2427 TAILQ_INSERT_TAIL(&ch->need_cluster_alloc, op, link); 2428 return; 2429 } 2430 2431 /* Round the io_unit offset down to the first page in the cluster */ 2432 cluster_start_page = bs_io_unit_to_cluster_start(blob, io_unit); 2433 2434 /* Calculate which index in the metadata cluster array the corresponding 2435 * cluster is supposed to be at. */ 2436 cluster_number = bs_io_unit_to_cluster_number(blob, io_unit); 2437 2438 ctx = calloc(1, sizeof(*ctx)); 2439 if (!ctx) { 2440 bs_user_op_abort(op); 2441 return; 2442 } 2443 2444 assert(blob->bs->cluster_sz % blob->back_bs_dev->blocklen == 0); 2445 2446 ctx->blob = blob; 2447 ctx->page = cluster_start_page; 2448 2449 if (blob->parent_id != SPDK_BLOBID_INVALID) { 2450 ctx->buf = spdk_malloc(blob->bs->cluster_sz, blob->back_bs_dev->blocklen, 2451 NULL, SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA); 2452 if (!ctx->buf) { 2453 SPDK_ERRLOG("DMA allocation for cluster of size = %" PRIu32 " failed.\n", 2454 blob->bs->cluster_sz); 2455 free(ctx); 2456 bs_user_op_abort(op); 2457 return; 2458 } 2459 } 2460 2461 pthread_mutex_lock(&blob->bs->used_clusters_mutex); 2462 rc = bs_allocate_cluster(blob, cluster_number, &ctx->new_cluster, &ctx->new_extent_page, 2463 false); 2464 pthread_mutex_unlock(&blob->bs->used_clusters_mutex); 2465 if (rc != 0) { 2466 spdk_free(ctx->buf); 2467 free(ctx); 2468 bs_user_op_abort(op); 2469 return; 2470 } 2471 2472 cpl.type = SPDK_BS_CPL_TYPE_BLOB_BASIC; 2473 cpl.u.blob_basic.cb_fn = blob_allocate_and_copy_cluster_cpl; 2474 cpl.u.blob_basic.cb_arg = ctx; 2475 2476 ctx->seq = bs_sequence_start(_ch, &cpl); 2477 if (!ctx->seq) { 2478 pthread_mutex_lock(&blob->bs->used_clusters_mutex); 2479 bs_release_cluster(blob->bs, ctx->new_cluster); 2480 pthread_mutex_unlock(&blob->bs->used_clusters_mutex); 2481 spdk_free(ctx->buf); 2482 free(ctx); 2483 bs_user_op_abort(op); 2484 return; 2485 } 2486 2487 /* Queue the user op to block other incoming operations */ 2488 TAILQ_INSERT_TAIL(&ch->need_cluster_alloc, op, link); 2489 2490 if (blob->parent_id != SPDK_BLOBID_INVALID) { 2491 /* Read cluster from backing device */ 2492 bs_sequence_read_bs_dev(ctx->seq, blob->back_bs_dev, ctx->buf, 2493 bs_dev_page_to_lba(blob->back_bs_dev, cluster_start_page), 2494 bs_dev_byte_to_lba(blob->back_bs_dev, blob->bs->cluster_sz), 2495 blob_write_copy, ctx); 2496 } else { 2497 blob_insert_cluster_on_md_thread(ctx->blob, cluster_number, ctx->new_cluster, 2498 ctx->new_extent_page, blob_insert_cluster_cpl, ctx); 2499 } 2500 } 2501 2502 static inline bool 2503 blob_calculate_lba_and_lba_count(struct spdk_blob *blob, uint64_t io_unit, uint64_t length, 2504 uint64_t *lba, uint64_t *lba_count) 2505 { 2506 *lba_count = length; 2507 2508 if (!bs_io_unit_is_allocated(blob, io_unit)) { 2509 assert(blob->back_bs_dev != NULL); 2510 *lba = bs_io_unit_to_back_dev_lba(blob, io_unit); 2511 *lba_count = bs_io_unit_to_back_dev_lba(blob, *lba_count); 2512 return false; 2513 } else { 2514 *lba = bs_blob_io_unit_to_lba(blob, io_unit); 2515 return true; 2516 } 2517 } 2518 2519 struct op_split_ctx { 2520 struct spdk_blob *blob; 2521 struct spdk_io_channel *channel; 2522 uint64_t io_unit_offset; 2523 uint64_t io_units_remaining; 2524 void *curr_payload; 2525 enum spdk_blob_op_type op_type; 2526 spdk_bs_sequence_t *seq; 2527 }; 2528 2529 static void 2530 blob_request_submit_op_split_next(void *cb_arg, int bserrno) 2531 { 2532 struct op_split_ctx *ctx = cb_arg; 2533 struct spdk_blob *blob = ctx->blob; 2534 struct spdk_io_channel *ch = ctx->channel; 2535 enum spdk_blob_op_type op_type = ctx->op_type; 2536 uint8_t *buf = ctx->curr_payload; 2537 uint64_t offset = ctx->io_unit_offset; 2538 uint64_t length = ctx->io_units_remaining; 2539 uint64_t op_length; 2540 2541 if (bserrno != 0 || ctx->io_units_remaining == 0) { 2542 bs_sequence_finish(ctx->seq, bserrno); 2543 free(ctx); 2544 return; 2545 } 2546 2547 op_length = spdk_min(length, bs_num_io_units_to_cluster_boundary(blob, 2548 offset)); 2549 2550 /* Update length and payload for next operation */ 2551 ctx->io_units_remaining -= op_length; 2552 ctx->io_unit_offset += op_length; 2553 if (op_type == SPDK_BLOB_WRITE || op_type == SPDK_BLOB_READ) { 2554 ctx->curr_payload += op_length * blob->bs->io_unit_size; 2555 } 2556 2557 switch (op_type) { 2558 case SPDK_BLOB_READ: 2559 spdk_blob_io_read(blob, ch, buf, offset, op_length, 2560 blob_request_submit_op_split_next, ctx); 2561 break; 2562 case SPDK_BLOB_WRITE: 2563 spdk_blob_io_write(blob, ch, buf, offset, op_length, 2564 blob_request_submit_op_split_next, ctx); 2565 break; 2566 case SPDK_BLOB_UNMAP: 2567 spdk_blob_io_unmap(blob, ch, offset, op_length, 2568 blob_request_submit_op_split_next, ctx); 2569 break; 2570 case SPDK_BLOB_WRITE_ZEROES: 2571 spdk_blob_io_write_zeroes(blob, ch, offset, op_length, 2572 blob_request_submit_op_split_next, ctx); 2573 break; 2574 case SPDK_BLOB_READV: 2575 case SPDK_BLOB_WRITEV: 2576 SPDK_ERRLOG("readv/write not valid\n"); 2577 bs_sequence_finish(ctx->seq, -EINVAL); 2578 free(ctx); 2579 break; 2580 } 2581 } 2582 2583 static void 2584 blob_request_submit_op_split(struct spdk_io_channel *ch, struct spdk_blob *blob, 2585 void *payload, uint64_t offset, uint64_t length, 2586 spdk_blob_op_complete cb_fn, void *cb_arg, enum spdk_blob_op_type op_type) 2587 { 2588 struct op_split_ctx *ctx; 2589 spdk_bs_sequence_t *seq; 2590 struct spdk_bs_cpl cpl; 2591 2592 assert(blob != NULL); 2593 2594 ctx = calloc(1, sizeof(struct op_split_ctx)); 2595 if (ctx == NULL) { 2596 cb_fn(cb_arg, -ENOMEM); 2597 return; 2598 } 2599 2600 cpl.type = SPDK_BS_CPL_TYPE_BLOB_BASIC; 2601 cpl.u.blob_basic.cb_fn = cb_fn; 2602 cpl.u.blob_basic.cb_arg = cb_arg; 2603 2604 seq = bs_sequence_start(ch, &cpl); 2605 if (!seq) { 2606 free(ctx); 2607 cb_fn(cb_arg, -ENOMEM); 2608 return; 2609 } 2610 2611 ctx->blob = blob; 2612 ctx->channel = ch; 2613 ctx->curr_payload = payload; 2614 ctx->io_unit_offset = offset; 2615 ctx->io_units_remaining = length; 2616 ctx->op_type = op_type; 2617 ctx->seq = seq; 2618 2619 blob_request_submit_op_split_next(ctx, 0); 2620 } 2621 2622 static void 2623 blob_request_submit_op_single(struct spdk_io_channel *_ch, struct spdk_blob *blob, 2624 void *payload, uint64_t offset, uint64_t length, 2625 spdk_blob_op_complete cb_fn, void *cb_arg, enum spdk_blob_op_type op_type) 2626 { 2627 struct spdk_bs_cpl cpl; 2628 uint64_t lba; 2629 uint64_t lba_count; 2630 bool is_allocated; 2631 2632 assert(blob != NULL); 2633 2634 cpl.type = SPDK_BS_CPL_TYPE_BLOB_BASIC; 2635 cpl.u.blob_basic.cb_fn = cb_fn; 2636 cpl.u.blob_basic.cb_arg = cb_arg; 2637 2638 if (blob->frozen_refcnt) { 2639 /* This blob I/O is frozen */ 2640 spdk_bs_user_op_t *op; 2641 struct spdk_bs_channel *bs_channel = spdk_io_channel_get_ctx(_ch); 2642 2643 op = bs_user_op_alloc(_ch, &cpl, op_type, blob, payload, 0, offset, length); 2644 if (!op) { 2645 cb_fn(cb_arg, -ENOMEM); 2646 return; 2647 } 2648 2649 TAILQ_INSERT_TAIL(&bs_channel->queued_io, op, link); 2650 2651 return; 2652 } 2653 2654 is_allocated = blob_calculate_lba_and_lba_count(blob, offset, length, &lba, &lba_count); 2655 2656 switch (op_type) { 2657 case SPDK_BLOB_READ: { 2658 spdk_bs_batch_t *batch; 2659 2660 batch = bs_batch_open(_ch, &cpl); 2661 if (!batch) { 2662 cb_fn(cb_arg, -ENOMEM); 2663 return; 2664 } 2665 2666 if (is_allocated) { 2667 /* Read from the blob */ 2668 bs_batch_read_dev(batch, payload, lba, lba_count); 2669 } else { 2670 /* Read from the backing block device */ 2671 bs_batch_read_bs_dev(batch, blob->back_bs_dev, payload, lba, lba_count); 2672 } 2673 2674 bs_batch_close(batch); 2675 break; 2676 } 2677 case SPDK_BLOB_WRITE: 2678 case SPDK_BLOB_WRITE_ZEROES: { 2679 if (is_allocated) { 2680 /* Write to the blob */ 2681 spdk_bs_batch_t *batch; 2682 2683 if (lba_count == 0) { 2684 cb_fn(cb_arg, 0); 2685 return; 2686 } 2687 2688 batch = bs_batch_open(_ch, &cpl); 2689 if (!batch) { 2690 cb_fn(cb_arg, -ENOMEM); 2691 return; 2692 } 2693 2694 if (op_type == SPDK_BLOB_WRITE) { 2695 bs_batch_write_dev(batch, payload, lba, lba_count); 2696 } else { 2697 bs_batch_write_zeroes_dev(batch, lba, lba_count); 2698 } 2699 2700 bs_batch_close(batch); 2701 } else { 2702 /* Queue this operation and allocate the cluster */ 2703 spdk_bs_user_op_t *op; 2704 2705 op = bs_user_op_alloc(_ch, &cpl, op_type, blob, payload, 0, offset, length); 2706 if (!op) { 2707 cb_fn(cb_arg, -ENOMEM); 2708 return; 2709 } 2710 2711 bs_allocate_and_copy_cluster(blob, _ch, offset, op); 2712 } 2713 break; 2714 } 2715 case SPDK_BLOB_UNMAP: { 2716 spdk_bs_batch_t *batch; 2717 2718 batch = bs_batch_open(_ch, &cpl); 2719 if (!batch) { 2720 cb_fn(cb_arg, -ENOMEM); 2721 return; 2722 } 2723 2724 if (is_allocated) { 2725 bs_batch_unmap_dev(batch, lba, lba_count); 2726 } 2727 2728 bs_batch_close(batch); 2729 break; 2730 } 2731 case SPDK_BLOB_READV: 2732 case SPDK_BLOB_WRITEV: 2733 SPDK_ERRLOG("readv/write not valid\n"); 2734 cb_fn(cb_arg, -EINVAL); 2735 break; 2736 } 2737 } 2738 2739 static void 2740 blob_request_submit_op(struct spdk_blob *blob, struct spdk_io_channel *_channel, 2741 void *payload, uint64_t offset, uint64_t length, 2742 spdk_blob_op_complete cb_fn, void *cb_arg, enum spdk_blob_op_type op_type) 2743 { 2744 assert(blob != NULL); 2745 2746 if (blob->data_ro && op_type != SPDK_BLOB_READ) { 2747 cb_fn(cb_arg, -EPERM); 2748 return; 2749 } 2750 2751 if (length == 0) { 2752 cb_fn(cb_arg, 0); 2753 return; 2754 } 2755 2756 if (offset + length > bs_cluster_to_lba(blob->bs, blob->active.num_clusters)) { 2757 cb_fn(cb_arg, -EINVAL); 2758 return; 2759 } 2760 if (length <= bs_num_io_units_to_cluster_boundary(blob, offset)) { 2761 blob_request_submit_op_single(_channel, blob, payload, offset, length, 2762 cb_fn, cb_arg, op_type); 2763 } else { 2764 blob_request_submit_op_split(_channel, blob, payload, offset, length, 2765 cb_fn, cb_arg, op_type); 2766 } 2767 } 2768 2769 struct rw_iov_ctx { 2770 struct spdk_blob *blob; 2771 struct spdk_io_channel *channel; 2772 spdk_blob_op_complete cb_fn; 2773 void *cb_arg; 2774 bool read; 2775 int iovcnt; 2776 struct iovec *orig_iov; 2777 uint64_t io_unit_offset; 2778 uint64_t io_units_remaining; 2779 uint64_t io_units_done; 2780 struct iovec iov[0]; 2781 }; 2782 2783 static void 2784 rw_iov_done(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 2785 { 2786 assert(cb_arg == NULL); 2787 bs_sequence_finish(seq, bserrno); 2788 } 2789 2790 static void 2791 rw_iov_split_next(void *cb_arg, int bserrno) 2792 { 2793 struct rw_iov_ctx *ctx = cb_arg; 2794 struct spdk_blob *blob = ctx->blob; 2795 struct iovec *iov, *orig_iov; 2796 int iovcnt; 2797 size_t orig_iovoff; 2798 uint64_t io_units_count, io_units_to_boundary, io_unit_offset; 2799 uint64_t byte_count; 2800 2801 if (bserrno != 0 || ctx->io_units_remaining == 0) { 2802 ctx->cb_fn(ctx->cb_arg, bserrno); 2803 free(ctx); 2804 return; 2805 } 2806 2807 io_unit_offset = ctx->io_unit_offset; 2808 io_units_to_boundary = bs_num_io_units_to_cluster_boundary(blob, io_unit_offset); 2809 io_units_count = spdk_min(ctx->io_units_remaining, io_units_to_boundary); 2810 /* 2811 * Get index and offset into the original iov array for our current position in the I/O sequence. 2812 * byte_count will keep track of how many bytes remaining until orig_iov and orig_iovoff will 2813 * point to the current position in the I/O sequence. 2814 */ 2815 byte_count = ctx->io_units_done * blob->bs->io_unit_size; 2816 orig_iov = &ctx->orig_iov[0]; 2817 orig_iovoff = 0; 2818 while (byte_count > 0) { 2819 if (byte_count >= orig_iov->iov_len) { 2820 byte_count -= orig_iov->iov_len; 2821 orig_iov++; 2822 } else { 2823 orig_iovoff = byte_count; 2824 byte_count = 0; 2825 } 2826 } 2827 2828 /* 2829 * Build an iov array for the next I/O in the sequence. byte_count will keep track of how many 2830 * bytes of this next I/O remain to be accounted for in the new iov array. 2831 */ 2832 byte_count = io_units_count * blob->bs->io_unit_size; 2833 iov = &ctx->iov[0]; 2834 iovcnt = 0; 2835 while (byte_count > 0) { 2836 assert(iovcnt < ctx->iovcnt); 2837 iov->iov_len = spdk_min(byte_count, orig_iov->iov_len - orig_iovoff); 2838 iov->iov_base = orig_iov->iov_base + orig_iovoff; 2839 byte_count -= iov->iov_len; 2840 orig_iovoff = 0; 2841 orig_iov++; 2842 iov++; 2843 iovcnt++; 2844 } 2845 2846 ctx->io_unit_offset += io_units_count; 2847 ctx->io_units_remaining -= io_units_count; 2848 ctx->io_units_done += io_units_count; 2849 iov = &ctx->iov[0]; 2850 2851 if (ctx->read) { 2852 spdk_blob_io_readv(ctx->blob, ctx->channel, iov, iovcnt, io_unit_offset, 2853 io_units_count, rw_iov_split_next, ctx); 2854 } else { 2855 spdk_blob_io_writev(ctx->blob, ctx->channel, iov, iovcnt, io_unit_offset, 2856 io_units_count, rw_iov_split_next, ctx); 2857 } 2858 } 2859 2860 static void 2861 blob_request_submit_rw_iov(struct spdk_blob *blob, struct spdk_io_channel *_channel, 2862 struct iovec *iov, int iovcnt, uint64_t offset, uint64_t length, 2863 spdk_blob_op_complete cb_fn, void *cb_arg, bool read) 2864 { 2865 struct spdk_bs_cpl cpl; 2866 2867 assert(blob != NULL); 2868 2869 if (!read && blob->data_ro) { 2870 cb_fn(cb_arg, -EPERM); 2871 return; 2872 } 2873 2874 if (length == 0) { 2875 cb_fn(cb_arg, 0); 2876 return; 2877 } 2878 2879 if (offset + length > bs_cluster_to_lba(blob->bs, blob->active.num_clusters)) { 2880 cb_fn(cb_arg, -EINVAL); 2881 return; 2882 } 2883 2884 /* 2885 * For now, we implement readv/writev using a sequence (instead of a batch) to account for having 2886 * to split a request that spans a cluster boundary. For I/O that do not span a cluster boundary, 2887 * there will be no noticeable difference compared to using a batch. For I/O that do span a cluster 2888 * boundary, the target LBAs (after blob offset to LBA translation) may not be contiguous, so we need 2889 * to allocate a separate iov array and split the I/O such that none of the resulting 2890 * smaller I/O cross a cluster boundary. These smaller I/O will be issued in sequence (not in parallel) 2891 * but since this case happens very infrequently, any performance impact will be negligible. 2892 * 2893 * This could be optimized in the future to allocate a big enough iov array to account for all of the iovs 2894 * for all of the smaller I/Os, pre-build all of the iov arrays for the smaller I/Os, then issue them 2895 * in a batch. That would also require creating an intermediate spdk_bs_cpl that would get called 2896 * when the batch was completed, to allow for freeing the memory for the iov arrays. 2897 */ 2898 if (spdk_likely(length <= bs_num_io_units_to_cluster_boundary(blob, offset))) { 2899 uint64_t lba_count; 2900 uint64_t lba; 2901 bool is_allocated; 2902 2903 cpl.type = SPDK_BS_CPL_TYPE_BLOB_BASIC; 2904 cpl.u.blob_basic.cb_fn = cb_fn; 2905 cpl.u.blob_basic.cb_arg = cb_arg; 2906 2907 if (blob->frozen_refcnt) { 2908 /* This blob I/O is frozen */ 2909 enum spdk_blob_op_type op_type; 2910 spdk_bs_user_op_t *op; 2911 struct spdk_bs_channel *bs_channel = spdk_io_channel_get_ctx(_channel); 2912 2913 op_type = read ? SPDK_BLOB_READV : SPDK_BLOB_WRITEV; 2914 op = bs_user_op_alloc(_channel, &cpl, op_type, blob, iov, iovcnt, offset, length); 2915 if (!op) { 2916 cb_fn(cb_arg, -ENOMEM); 2917 return; 2918 } 2919 2920 TAILQ_INSERT_TAIL(&bs_channel->queued_io, op, link); 2921 2922 return; 2923 } 2924 2925 is_allocated = blob_calculate_lba_and_lba_count(blob, offset, length, &lba, &lba_count); 2926 2927 if (read) { 2928 spdk_bs_sequence_t *seq; 2929 2930 seq = bs_sequence_start(_channel, &cpl); 2931 if (!seq) { 2932 cb_fn(cb_arg, -ENOMEM); 2933 return; 2934 } 2935 2936 if (is_allocated) { 2937 bs_sequence_readv_dev(seq, iov, iovcnt, lba, lba_count, rw_iov_done, NULL); 2938 } else { 2939 bs_sequence_readv_bs_dev(seq, blob->back_bs_dev, iov, iovcnt, lba, lba_count, 2940 rw_iov_done, NULL); 2941 } 2942 } else { 2943 if (is_allocated) { 2944 spdk_bs_sequence_t *seq; 2945 2946 seq = bs_sequence_start(_channel, &cpl); 2947 if (!seq) { 2948 cb_fn(cb_arg, -ENOMEM); 2949 return; 2950 } 2951 2952 bs_sequence_writev_dev(seq, iov, iovcnt, lba, lba_count, rw_iov_done, NULL); 2953 } else { 2954 /* Queue this operation and allocate the cluster */ 2955 spdk_bs_user_op_t *op; 2956 2957 op = bs_user_op_alloc(_channel, &cpl, SPDK_BLOB_WRITEV, blob, iov, iovcnt, offset, 2958 length); 2959 if (!op) { 2960 cb_fn(cb_arg, -ENOMEM); 2961 return; 2962 } 2963 2964 bs_allocate_and_copy_cluster(blob, _channel, offset, op); 2965 } 2966 } 2967 } else { 2968 struct rw_iov_ctx *ctx; 2969 2970 ctx = calloc(1, sizeof(struct rw_iov_ctx) + iovcnt * sizeof(struct iovec)); 2971 if (ctx == NULL) { 2972 cb_fn(cb_arg, -ENOMEM); 2973 return; 2974 } 2975 2976 ctx->blob = blob; 2977 ctx->channel = _channel; 2978 ctx->cb_fn = cb_fn; 2979 ctx->cb_arg = cb_arg; 2980 ctx->read = read; 2981 ctx->orig_iov = iov; 2982 ctx->iovcnt = iovcnt; 2983 ctx->io_unit_offset = offset; 2984 ctx->io_units_remaining = length; 2985 ctx->io_units_done = 0; 2986 2987 rw_iov_split_next(ctx, 0); 2988 } 2989 } 2990 2991 static struct spdk_blob * 2992 blob_lookup(struct spdk_blob_store *bs, spdk_blob_id blobid) 2993 { 2994 struct spdk_blob *blob; 2995 2996 if (spdk_bit_array_get(bs->open_blobids, blobid) == 0) { 2997 return NULL; 2998 } 2999 3000 TAILQ_FOREACH(blob, &bs->blobs, link) { 3001 if (blob->id == blobid) { 3002 return blob; 3003 } 3004 } 3005 3006 return NULL; 3007 } 3008 3009 static void 3010 blob_get_snapshot_and_clone_entries(struct spdk_blob *blob, 3011 struct spdk_blob_list **snapshot_entry, struct spdk_blob_list **clone_entry) 3012 { 3013 assert(blob != NULL); 3014 *snapshot_entry = NULL; 3015 *clone_entry = NULL; 3016 3017 if (blob->parent_id == SPDK_BLOBID_INVALID) { 3018 return; 3019 } 3020 3021 TAILQ_FOREACH(*snapshot_entry, &blob->bs->snapshots, link) { 3022 if ((*snapshot_entry)->id == blob->parent_id) { 3023 break; 3024 } 3025 } 3026 3027 if (*snapshot_entry != NULL) { 3028 TAILQ_FOREACH(*clone_entry, &(*snapshot_entry)->clones, link) { 3029 if ((*clone_entry)->id == blob->id) { 3030 break; 3031 } 3032 } 3033 3034 assert(*clone_entry != NULL); 3035 } 3036 } 3037 3038 static int 3039 bs_channel_create(void *io_device, void *ctx_buf) 3040 { 3041 struct spdk_blob_store *bs = io_device; 3042 struct spdk_bs_channel *channel = ctx_buf; 3043 struct spdk_bs_dev *dev; 3044 uint32_t max_ops = bs->max_channel_ops; 3045 uint32_t i; 3046 3047 dev = bs->dev; 3048 3049 channel->req_mem = calloc(max_ops, sizeof(struct spdk_bs_request_set)); 3050 if (!channel->req_mem) { 3051 return -1; 3052 } 3053 3054 TAILQ_INIT(&channel->reqs); 3055 3056 for (i = 0; i < max_ops; i++) { 3057 TAILQ_INSERT_TAIL(&channel->reqs, &channel->req_mem[i], link); 3058 } 3059 3060 channel->bs = bs; 3061 channel->dev = dev; 3062 channel->dev_channel = dev->create_channel(dev); 3063 3064 if (!channel->dev_channel) { 3065 SPDK_ERRLOG("Failed to create device channel.\n"); 3066 free(channel->req_mem); 3067 return -1; 3068 } 3069 3070 TAILQ_INIT(&channel->need_cluster_alloc); 3071 TAILQ_INIT(&channel->queued_io); 3072 3073 return 0; 3074 } 3075 3076 static void 3077 bs_channel_destroy(void *io_device, void *ctx_buf) 3078 { 3079 struct spdk_bs_channel *channel = ctx_buf; 3080 spdk_bs_user_op_t *op; 3081 3082 while (!TAILQ_EMPTY(&channel->need_cluster_alloc)) { 3083 op = TAILQ_FIRST(&channel->need_cluster_alloc); 3084 TAILQ_REMOVE(&channel->need_cluster_alloc, op, link); 3085 bs_user_op_abort(op); 3086 } 3087 3088 while (!TAILQ_EMPTY(&channel->queued_io)) { 3089 op = TAILQ_FIRST(&channel->queued_io); 3090 TAILQ_REMOVE(&channel->queued_io, op, link); 3091 bs_user_op_abort(op); 3092 } 3093 3094 free(channel->req_mem); 3095 channel->dev->destroy_channel(channel->dev, channel->dev_channel); 3096 } 3097 3098 static void 3099 bs_dev_destroy(void *io_device) 3100 { 3101 struct spdk_blob_store *bs = io_device; 3102 struct spdk_blob *blob, *blob_tmp; 3103 3104 bs->dev->destroy(bs->dev); 3105 3106 TAILQ_FOREACH_SAFE(blob, &bs->blobs, link, blob_tmp) { 3107 TAILQ_REMOVE(&bs->blobs, blob, link); 3108 spdk_bit_array_clear(bs->open_blobids, blob->id); 3109 blob_free(blob); 3110 } 3111 3112 pthread_mutex_destroy(&bs->used_clusters_mutex); 3113 3114 spdk_bit_array_free(&bs->open_blobids); 3115 spdk_bit_array_free(&bs->used_blobids); 3116 spdk_bit_array_free(&bs->used_md_pages); 3117 spdk_bit_pool_free(&bs->used_clusters); 3118 /* 3119 * If this function is called for any reason except a successful unload, 3120 * the unload_cpl type will be NONE and this will be a nop. 3121 */ 3122 bs_call_cpl(&bs->unload_cpl, bs->unload_err); 3123 3124 free(bs); 3125 } 3126 3127 static int 3128 bs_blob_list_add(struct spdk_blob *blob) 3129 { 3130 spdk_blob_id snapshot_id; 3131 struct spdk_blob_list *snapshot_entry = NULL; 3132 struct spdk_blob_list *clone_entry = NULL; 3133 3134 assert(blob != NULL); 3135 3136 snapshot_id = blob->parent_id; 3137 if (snapshot_id == SPDK_BLOBID_INVALID) { 3138 return 0; 3139 } 3140 3141 snapshot_entry = bs_get_snapshot_entry(blob->bs, snapshot_id); 3142 if (snapshot_entry == NULL) { 3143 /* Snapshot not found */ 3144 snapshot_entry = calloc(1, sizeof(struct spdk_blob_list)); 3145 if (snapshot_entry == NULL) { 3146 return -ENOMEM; 3147 } 3148 snapshot_entry->id = snapshot_id; 3149 TAILQ_INIT(&snapshot_entry->clones); 3150 TAILQ_INSERT_TAIL(&blob->bs->snapshots, snapshot_entry, link); 3151 } else { 3152 TAILQ_FOREACH(clone_entry, &snapshot_entry->clones, link) { 3153 if (clone_entry->id == blob->id) { 3154 break; 3155 } 3156 } 3157 } 3158 3159 if (clone_entry == NULL) { 3160 /* Clone not found */ 3161 clone_entry = calloc(1, sizeof(struct spdk_blob_list)); 3162 if (clone_entry == NULL) { 3163 return -ENOMEM; 3164 } 3165 clone_entry->id = blob->id; 3166 TAILQ_INIT(&clone_entry->clones); 3167 TAILQ_INSERT_TAIL(&snapshot_entry->clones, clone_entry, link); 3168 snapshot_entry->clone_count++; 3169 } 3170 3171 return 0; 3172 } 3173 3174 static void 3175 bs_blob_list_remove(struct spdk_blob *blob) 3176 { 3177 struct spdk_blob_list *snapshot_entry = NULL; 3178 struct spdk_blob_list *clone_entry = NULL; 3179 3180 blob_get_snapshot_and_clone_entries(blob, &snapshot_entry, &clone_entry); 3181 3182 if (snapshot_entry == NULL) { 3183 return; 3184 } 3185 3186 blob->parent_id = SPDK_BLOBID_INVALID; 3187 TAILQ_REMOVE(&snapshot_entry->clones, clone_entry, link); 3188 free(clone_entry); 3189 3190 snapshot_entry->clone_count--; 3191 } 3192 3193 static int 3194 bs_blob_list_free(struct spdk_blob_store *bs) 3195 { 3196 struct spdk_blob_list *snapshot_entry; 3197 struct spdk_blob_list *snapshot_entry_tmp; 3198 struct spdk_blob_list *clone_entry; 3199 struct spdk_blob_list *clone_entry_tmp; 3200 3201 TAILQ_FOREACH_SAFE(snapshot_entry, &bs->snapshots, link, snapshot_entry_tmp) { 3202 TAILQ_FOREACH_SAFE(clone_entry, &snapshot_entry->clones, link, clone_entry_tmp) { 3203 TAILQ_REMOVE(&snapshot_entry->clones, clone_entry, link); 3204 free(clone_entry); 3205 } 3206 TAILQ_REMOVE(&bs->snapshots, snapshot_entry, link); 3207 free(snapshot_entry); 3208 } 3209 3210 return 0; 3211 } 3212 3213 static void 3214 bs_free(struct spdk_blob_store *bs) 3215 { 3216 bs_blob_list_free(bs); 3217 3218 bs_unregister_md_thread(bs); 3219 spdk_io_device_unregister(bs, bs_dev_destroy); 3220 } 3221 3222 void 3223 spdk_bs_opts_init(struct spdk_bs_opts *opts, size_t opts_size) 3224 { 3225 3226 if (!opts) { 3227 SPDK_ERRLOG("opts should not be NULL\n"); 3228 return; 3229 } 3230 3231 if (!opts_size) { 3232 SPDK_ERRLOG("opts_size should not be zero value\n"); 3233 return; 3234 } 3235 3236 memset(opts, 0, opts_size); 3237 opts->opts_size = opts_size; 3238 3239 #define FIELD_OK(field) \ 3240 offsetof(struct spdk_bs_opts, field) + sizeof(opts->field) <= opts_size 3241 3242 #define SET_FIELD(field, value) \ 3243 if (FIELD_OK(field)) { \ 3244 opts->field = value; \ 3245 } \ 3246 3247 SET_FIELD(cluster_sz, SPDK_BLOB_OPTS_CLUSTER_SZ); 3248 SET_FIELD(num_md_pages, SPDK_BLOB_OPTS_NUM_MD_PAGES); 3249 SET_FIELD(max_md_ops, SPDK_BLOB_OPTS_NUM_MD_PAGES); 3250 SET_FIELD(max_channel_ops, SPDK_BLOB_OPTS_DEFAULT_CHANNEL_OPS); 3251 SET_FIELD(clear_method, BS_CLEAR_WITH_UNMAP); 3252 3253 if (FIELD_OK(bstype)) { 3254 memset(&opts->bstype, 0, sizeof(opts->bstype)); 3255 } 3256 3257 SET_FIELD(iter_cb_fn, NULL); 3258 SET_FIELD(iter_cb_arg, NULL); 3259 3260 #undef FIELD_OK 3261 #undef SET_FIELD 3262 } 3263 3264 static int 3265 bs_opts_verify(struct spdk_bs_opts *opts) 3266 { 3267 if (opts->cluster_sz == 0 || opts->num_md_pages == 0 || opts->max_md_ops == 0 || 3268 opts->max_channel_ops == 0) { 3269 SPDK_ERRLOG("Blobstore options cannot be set to 0\n"); 3270 return -1; 3271 } 3272 3273 return 0; 3274 } 3275 3276 /* START spdk_bs_load */ 3277 3278 /* spdk_bs_load_ctx is used for init, load, unload and dump code paths. */ 3279 3280 struct spdk_bs_load_ctx { 3281 struct spdk_blob_store *bs; 3282 struct spdk_bs_super_block *super; 3283 3284 struct spdk_bs_md_mask *mask; 3285 bool in_page_chain; 3286 uint32_t page_index; 3287 uint32_t cur_page; 3288 struct spdk_blob_md_page *page; 3289 3290 uint64_t num_extent_pages; 3291 uint32_t *extent_page_num; 3292 struct spdk_blob_md_page *extent_pages; 3293 struct spdk_bit_array *used_clusters; 3294 3295 spdk_bs_sequence_t *seq; 3296 spdk_blob_op_with_handle_complete iter_cb_fn; 3297 void *iter_cb_arg; 3298 struct spdk_blob *blob; 3299 spdk_blob_id blobid; 3300 3301 /* These fields are used in the spdk_bs_dump path. */ 3302 FILE *fp; 3303 spdk_bs_dump_print_xattr print_xattr_fn; 3304 char xattr_name[4096]; 3305 }; 3306 3307 static int 3308 bs_alloc(struct spdk_bs_dev *dev, struct spdk_bs_opts *opts, struct spdk_blob_store **_bs, 3309 struct spdk_bs_load_ctx **_ctx) 3310 { 3311 struct spdk_blob_store *bs; 3312 struct spdk_bs_load_ctx *ctx; 3313 uint64_t dev_size; 3314 int rc; 3315 3316 dev_size = dev->blocklen * dev->blockcnt; 3317 if (dev_size < opts->cluster_sz) { 3318 /* Device size cannot be smaller than cluster size of blobstore */ 3319 SPDK_INFOLOG(blob, "Device size %" PRIu64 " is smaller than cluster size %" PRIu32 "\n", 3320 dev_size, opts->cluster_sz); 3321 return -ENOSPC; 3322 } 3323 if (opts->cluster_sz < SPDK_BS_PAGE_SIZE) { 3324 /* Cluster size cannot be smaller than page size */ 3325 SPDK_ERRLOG("Cluster size %" PRIu32 " is smaller than page size %d\n", 3326 opts->cluster_sz, SPDK_BS_PAGE_SIZE); 3327 return -EINVAL; 3328 } 3329 bs = calloc(1, sizeof(struct spdk_blob_store)); 3330 if (!bs) { 3331 return -ENOMEM; 3332 } 3333 3334 ctx = calloc(1, sizeof(struct spdk_bs_load_ctx)); 3335 if (!ctx) { 3336 free(bs); 3337 return -ENOMEM; 3338 } 3339 3340 ctx->bs = bs; 3341 ctx->iter_cb_fn = opts->iter_cb_fn; 3342 ctx->iter_cb_arg = opts->iter_cb_arg; 3343 3344 ctx->super = spdk_zmalloc(sizeof(*ctx->super), 0x1000, NULL, 3345 SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA); 3346 if (!ctx->super) { 3347 free(ctx); 3348 free(bs); 3349 return -ENOMEM; 3350 } 3351 3352 TAILQ_INIT(&bs->blobs); 3353 TAILQ_INIT(&bs->snapshots); 3354 bs->dev = dev; 3355 bs->md_thread = spdk_get_thread(); 3356 assert(bs->md_thread != NULL); 3357 3358 /* 3359 * Do not use bs_lba_to_cluster() here since blockcnt may not be an 3360 * even multiple of the cluster size. 3361 */ 3362 bs->cluster_sz = opts->cluster_sz; 3363 bs->total_clusters = dev->blockcnt / (bs->cluster_sz / dev->blocklen); 3364 ctx->used_clusters = spdk_bit_array_create(bs->total_clusters); 3365 if (!ctx->used_clusters) { 3366 spdk_free(ctx->super); 3367 free(ctx); 3368 free(bs); 3369 return -ENOMEM; 3370 } 3371 3372 bs->pages_per_cluster = bs->cluster_sz / SPDK_BS_PAGE_SIZE; 3373 if (spdk_u32_is_pow2(bs->pages_per_cluster)) { 3374 bs->pages_per_cluster_shift = spdk_u32log2(bs->pages_per_cluster); 3375 } 3376 bs->num_free_clusters = bs->total_clusters; 3377 bs->io_unit_size = dev->blocklen; 3378 3379 bs->max_channel_ops = opts->max_channel_ops; 3380 bs->super_blob = SPDK_BLOBID_INVALID; 3381 memcpy(&bs->bstype, &opts->bstype, sizeof(opts->bstype)); 3382 3383 /* The metadata is assumed to be at least 1 page */ 3384 bs->used_md_pages = spdk_bit_array_create(1); 3385 bs->used_blobids = spdk_bit_array_create(0); 3386 bs->open_blobids = spdk_bit_array_create(0); 3387 3388 pthread_mutex_init(&bs->used_clusters_mutex, NULL); 3389 3390 spdk_io_device_register(bs, bs_channel_create, bs_channel_destroy, 3391 sizeof(struct spdk_bs_channel), "blobstore"); 3392 rc = bs_register_md_thread(bs); 3393 if (rc == -1) { 3394 spdk_io_device_unregister(bs, NULL); 3395 pthread_mutex_destroy(&bs->used_clusters_mutex); 3396 spdk_bit_array_free(&bs->open_blobids); 3397 spdk_bit_array_free(&bs->used_blobids); 3398 spdk_bit_array_free(&bs->used_md_pages); 3399 spdk_bit_array_free(&ctx->used_clusters); 3400 spdk_free(ctx->super); 3401 free(ctx); 3402 free(bs); 3403 /* FIXME: this is a lie but don't know how to get a proper error code here */ 3404 return -ENOMEM; 3405 } 3406 3407 *_ctx = ctx; 3408 *_bs = bs; 3409 return 0; 3410 } 3411 3412 static void 3413 bs_load_ctx_fail(struct spdk_bs_load_ctx *ctx, int bserrno) 3414 { 3415 assert(bserrno != 0); 3416 3417 spdk_free(ctx->super); 3418 bs_sequence_finish(ctx->seq, bserrno); 3419 bs_free(ctx->bs); 3420 spdk_bit_array_free(&ctx->used_clusters); 3421 free(ctx); 3422 } 3423 3424 static void 3425 bs_write_super(spdk_bs_sequence_t *seq, struct spdk_blob_store *bs, 3426 struct spdk_bs_super_block *super, spdk_bs_sequence_cpl cb_fn, void *cb_arg) 3427 { 3428 /* Update the values in the super block */ 3429 super->super_blob = bs->super_blob; 3430 memcpy(&super->bstype, &bs->bstype, sizeof(bs->bstype)); 3431 super->crc = blob_md_page_calc_crc(super); 3432 bs_sequence_write_dev(seq, super, bs_page_to_lba(bs, 0), 3433 bs_byte_to_lba(bs, sizeof(*super)), 3434 cb_fn, cb_arg); 3435 } 3436 3437 static void 3438 bs_write_used_clusters(spdk_bs_sequence_t *seq, void *arg, spdk_bs_sequence_cpl cb_fn) 3439 { 3440 struct spdk_bs_load_ctx *ctx = arg; 3441 uint64_t mask_size, lba, lba_count; 3442 3443 /* Write out the used clusters mask */ 3444 mask_size = ctx->super->used_cluster_mask_len * SPDK_BS_PAGE_SIZE; 3445 ctx->mask = spdk_zmalloc(mask_size, 0x1000, NULL, 3446 SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA); 3447 if (!ctx->mask) { 3448 bs_load_ctx_fail(ctx, -ENOMEM); 3449 return; 3450 } 3451 3452 ctx->mask->type = SPDK_MD_MASK_TYPE_USED_CLUSTERS; 3453 ctx->mask->length = ctx->bs->total_clusters; 3454 /* We could get here through the normal unload path, or through dirty 3455 * shutdown recovery. For the normal unload path, we use the mask from 3456 * the bit pool. For dirty shutdown recovery, we don't have a bit pool yet - 3457 * only the bit array from the load ctx. 3458 */ 3459 if (ctx->bs->used_clusters) { 3460 assert(ctx->mask->length == spdk_bit_pool_capacity(ctx->bs->used_clusters)); 3461 spdk_bit_pool_store_mask(ctx->bs->used_clusters, ctx->mask->mask); 3462 } else { 3463 assert(ctx->mask->length == spdk_bit_array_capacity(ctx->used_clusters)); 3464 spdk_bit_array_store_mask(ctx->used_clusters, ctx->mask->mask); 3465 } 3466 lba = bs_page_to_lba(ctx->bs, ctx->super->used_cluster_mask_start); 3467 lba_count = bs_page_to_lba(ctx->bs, ctx->super->used_cluster_mask_len); 3468 bs_sequence_write_dev(seq, ctx->mask, lba, lba_count, cb_fn, arg); 3469 } 3470 3471 static void 3472 bs_write_used_md(spdk_bs_sequence_t *seq, void *arg, spdk_bs_sequence_cpl cb_fn) 3473 { 3474 struct spdk_bs_load_ctx *ctx = arg; 3475 uint64_t mask_size, lba, lba_count; 3476 3477 mask_size = ctx->super->used_page_mask_len * SPDK_BS_PAGE_SIZE; 3478 ctx->mask = spdk_zmalloc(mask_size, 0x1000, NULL, 3479 SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA); 3480 if (!ctx->mask) { 3481 bs_load_ctx_fail(ctx, -ENOMEM); 3482 return; 3483 } 3484 3485 ctx->mask->type = SPDK_MD_MASK_TYPE_USED_PAGES; 3486 ctx->mask->length = ctx->super->md_len; 3487 assert(ctx->mask->length == spdk_bit_array_capacity(ctx->bs->used_md_pages)); 3488 3489 spdk_bit_array_store_mask(ctx->bs->used_md_pages, ctx->mask->mask); 3490 lba = bs_page_to_lba(ctx->bs, ctx->super->used_page_mask_start); 3491 lba_count = bs_page_to_lba(ctx->bs, ctx->super->used_page_mask_len); 3492 bs_sequence_write_dev(seq, ctx->mask, lba, lba_count, cb_fn, arg); 3493 } 3494 3495 static void 3496 bs_write_used_blobids(spdk_bs_sequence_t *seq, void *arg, spdk_bs_sequence_cpl cb_fn) 3497 { 3498 struct spdk_bs_load_ctx *ctx = arg; 3499 uint64_t mask_size, lba, lba_count; 3500 3501 if (ctx->super->used_blobid_mask_len == 0) { 3502 /* 3503 * This is a pre-v3 on-disk format where the blobid mask does not get 3504 * written to disk. 3505 */ 3506 cb_fn(seq, arg, 0); 3507 return; 3508 } 3509 3510 mask_size = ctx->super->used_blobid_mask_len * SPDK_BS_PAGE_SIZE; 3511 ctx->mask = spdk_zmalloc(mask_size, 0x1000, NULL, SPDK_ENV_SOCKET_ID_ANY, 3512 SPDK_MALLOC_DMA); 3513 if (!ctx->mask) { 3514 bs_load_ctx_fail(ctx, -ENOMEM); 3515 return; 3516 } 3517 3518 ctx->mask->type = SPDK_MD_MASK_TYPE_USED_BLOBIDS; 3519 ctx->mask->length = ctx->super->md_len; 3520 assert(ctx->mask->length == spdk_bit_array_capacity(ctx->bs->used_blobids)); 3521 3522 spdk_bit_array_store_mask(ctx->bs->used_blobids, ctx->mask->mask); 3523 lba = bs_page_to_lba(ctx->bs, ctx->super->used_blobid_mask_start); 3524 lba_count = bs_page_to_lba(ctx->bs, ctx->super->used_blobid_mask_len); 3525 bs_sequence_write_dev(seq, ctx->mask, lba, lba_count, cb_fn, arg); 3526 } 3527 3528 static void 3529 blob_set_thin_provision(struct spdk_blob *blob) 3530 { 3531 blob_verify_md_op(blob); 3532 blob->invalid_flags |= SPDK_BLOB_THIN_PROV; 3533 blob->state = SPDK_BLOB_STATE_DIRTY; 3534 } 3535 3536 static void 3537 blob_set_clear_method(struct spdk_blob *blob, enum blob_clear_method clear_method) 3538 { 3539 blob_verify_md_op(blob); 3540 blob->clear_method = clear_method; 3541 blob->md_ro_flags |= (clear_method << SPDK_BLOB_CLEAR_METHOD_SHIFT); 3542 blob->state = SPDK_BLOB_STATE_DIRTY; 3543 } 3544 3545 static void bs_load_iter(void *arg, struct spdk_blob *blob, int bserrno); 3546 3547 static void 3548 bs_delete_corrupted_blob_cpl(void *cb_arg, int bserrno) 3549 { 3550 struct spdk_bs_load_ctx *ctx = cb_arg; 3551 spdk_blob_id id; 3552 int64_t page_num; 3553 3554 /* Iterate to next blob (we can't use spdk_bs_iter_next function as our 3555 * last blob has been removed */ 3556 page_num = bs_blobid_to_page(ctx->blobid); 3557 page_num++; 3558 page_num = spdk_bit_array_find_first_set(ctx->bs->used_blobids, page_num); 3559 if (page_num >= spdk_bit_array_capacity(ctx->bs->used_blobids)) { 3560 bs_load_iter(ctx, NULL, -ENOENT); 3561 return; 3562 } 3563 3564 id = bs_page_to_blobid(page_num); 3565 3566 spdk_bs_open_blob(ctx->bs, id, bs_load_iter, ctx); 3567 } 3568 3569 static void 3570 bs_delete_corrupted_close_cb(void *cb_arg, int bserrno) 3571 { 3572 struct spdk_bs_load_ctx *ctx = cb_arg; 3573 3574 if (bserrno != 0) { 3575 SPDK_ERRLOG("Failed to close corrupted blob\n"); 3576 spdk_bs_iter_next(ctx->bs, ctx->blob, bs_load_iter, ctx); 3577 return; 3578 } 3579 3580 spdk_bs_delete_blob(ctx->bs, ctx->blobid, bs_delete_corrupted_blob_cpl, ctx); 3581 } 3582 3583 static void 3584 bs_delete_corrupted_blob(void *cb_arg, int bserrno) 3585 { 3586 struct spdk_bs_load_ctx *ctx = cb_arg; 3587 uint64_t i; 3588 3589 if (bserrno != 0) { 3590 SPDK_ERRLOG("Failed to close clone of a corrupted blob\n"); 3591 spdk_bs_iter_next(ctx->bs, ctx->blob, bs_load_iter, ctx); 3592 return; 3593 } 3594 3595 /* Snapshot and clone have the same copy of cluster map and extent pages 3596 * at this point. Let's clear both for snapshot now, 3597 * so that it won't be cleared for clone later when we remove snapshot. 3598 * Also set thin provision to pass data corruption check */ 3599 for (i = 0; i < ctx->blob->active.num_clusters; i++) { 3600 ctx->blob->active.clusters[i] = 0; 3601 } 3602 for (i = 0; i < ctx->blob->active.num_extent_pages; i++) { 3603 ctx->blob->active.extent_pages[i] = 0; 3604 } 3605 3606 ctx->blob->md_ro = false; 3607 3608 blob_set_thin_provision(ctx->blob); 3609 3610 ctx->blobid = ctx->blob->id; 3611 3612 spdk_blob_close(ctx->blob, bs_delete_corrupted_close_cb, ctx); 3613 } 3614 3615 static void 3616 bs_update_corrupted_blob(void *cb_arg, int bserrno) 3617 { 3618 struct spdk_bs_load_ctx *ctx = cb_arg; 3619 3620 if (bserrno != 0) { 3621 SPDK_ERRLOG("Failed to close clone of a corrupted blob\n"); 3622 spdk_bs_iter_next(ctx->bs, ctx->blob, bs_load_iter, ctx); 3623 return; 3624 } 3625 3626 ctx->blob->md_ro = false; 3627 blob_remove_xattr(ctx->blob, SNAPSHOT_PENDING_REMOVAL, true); 3628 blob_remove_xattr(ctx->blob, SNAPSHOT_IN_PROGRESS, true); 3629 spdk_blob_set_read_only(ctx->blob); 3630 3631 if (ctx->iter_cb_fn) { 3632 ctx->iter_cb_fn(ctx->iter_cb_arg, ctx->blob, 0); 3633 } 3634 bs_blob_list_add(ctx->blob); 3635 3636 spdk_bs_iter_next(ctx->bs, ctx->blob, bs_load_iter, ctx); 3637 } 3638 3639 static void 3640 bs_examine_clone(void *cb_arg, struct spdk_blob *blob, int bserrno) 3641 { 3642 struct spdk_bs_load_ctx *ctx = cb_arg; 3643 3644 if (bserrno != 0) { 3645 SPDK_ERRLOG("Failed to open clone of a corrupted blob\n"); 3646 spdk_bs_iter_next(ctx->bs, ctx->blob, bs_load_iter, ctx); 3647 return; 3648 } 3649 3650 if (blob->parent_id == ctx->blob->id) { 3651 /* Power failure occurred before updating clone (snapshot delete case) 3652 * or after updating clone (creating snapshot case) - keep snapshot */ 3653 spdk_blob_close(blob, bs_update_corrupted_blob, ctx); 3654 } else { 3655 /* Power failure occurred after updating clone (snapshot delete case) 3656 * or before updating clone (creating snapshot case) - remove snapshot */ 3657 spdk_blob_close(blob, bs_delete_corrupted_blob, ctx); 3658 } 3659 } 3660 3661 static void 3662 bs_load_iter(void *arg, struct spdk_blob *blob, int bserrno) 3663 { 3664 struct spdk_bs_load_ctx *ctx = arg; 3665 const void *value; 3666 size_t len; 3667 int rc = 0; 3668 3669 if (bserrno == 0) { 3670 /* Examine blob if it is corrupted after power failure. Fix 3671 * the ones that can be fixed and remove any other corrupted 3672 * ones. If it is not corrupted just process it */ 3673 rc = blob_get_xattr_value(blob, SNAPSHOT_PENDING_REMOVAL, &value, &len, true); 3674 if (rc != 0) { 3675 rc = blob_get_xattr_value(blob, SNAPSHOT_IN_PROGRESS, &value, &len, true); 3676 if (rc != 0) { 3677 /* Not corrupted - process it and continue with iterating through blobs */ 3678 if (ctx->iter_cb_fn) { 3679 ctx->iter_cb_fn(ctx->iter_cb_arg, blob, 0); 3680 } 3681 bs_blob_list_add(blob); 3682 spdk_bs_iter_next(ctx->bs, blob, bs_load_iter, ctx); 3683 return; 3684 } 3685 3686 } 3687 3688 assert(len == sizeof(spdk_blob_id)); 3689 3690 ctx->blob = blob; 3691 3692 /* Open clone to check if we are able to fix this blob or should we remove it */ 3693 spdk_bs_open_blob(ctx->bs, *(spdk_blob_id *)value, bs_examine_clone, ctx); 3694 return; 3695 } else if (bserrno == -ENOENT) { 3696 bserrno = 0; 3697 } else { 3698 /* 3699 * This case needs to be looked at further. Same problem 3700 * exists with applications that rely on explicit blob 3701 * iteration. We should just skip the blob that failed 3702 * to load and continue on to the next one. 3703 */ 3704 SPDK_ERRLOG("Error in iterating blobs\n"); 3705 } 3706 3707 ctx->iter_cb_fn = NULL; 3708 3709 spdk_free(ctx->super); 3710 spdk_free(ctx->mask); 3711 bs_sequence_finish(ctx->seq, bserrno); 3712 free(ctx); 3713 } 3714 3715 static void 3716 bs_load_complete(struct spdk_bs_load_ctx *ctx) 3717 { 3718 ctx->bs->used_clusters = spdk_bit_pool_create_from_array(ctx->used_clusters); 3719 spdk_bs_iter_first(ctx->bs, bs_load_iter, ctx); 3720 } 3721 3722 static void 3723 bs_load_used_blobids_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 3724 { 3725 struct spdk_bs_load_ctx *ctx = cb_arg; 3726 int rc; 3727 3728 /* The type must be correct */ 3729 assert(ctx->mask->type == SPDK_MD_MASK_TYPE_USED_BLOBIDS); 3730 3731 /* The length of the mask (in bits) must not be greater than 3732 * the length of the buffer (converted to bits) */ 3733 assert(ctx->mask->length <= (ctx->super->used_blobid_mask_len * SPDK_BS_PAGE_SIZE * 8)); 3734 3735 /* The length of the mask must be exactly equal to the size 3736 * (in pages) of the metadata region */ 3737 assert(ctx->mask->length == ctx->super->md_len); 3738 3739 rc = spdk_bit_array_resize(&ctx->bs->used_blobids, ctx->mask->length); 3740 if (rc < 0) { 3741 spdk_free(ctx->mask); 3742 bs_load_ctx_fail(ctx, rc); 3743 return; 3744 } 3745 3746 spdk_bit_array_load_mask(ctx->bs->used_blobids, ctx->mask->mask); 3747 bs_load_complete(ctx); 3748 } 3749 3750 static void 3751 bs_load_used_clusters_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 3752 { 3753 struct spdk_bs_load_ctx *ctx = cb_arg; 3754 uint64_t lba, lba_count, mask_size; 3755 int rc; 3756 3757 if (bserrno != 0) { 3758 bs_load_ctx_fail(ctx, bserrno); 3759 return; 3760 } 3761 3762 /* The type must be correct */ 3763 assert(ctx->mask->type == SPDK_MD_MASK_TYPE_USED_CLUSTERS); 3764 /* The length of the mask (in bits) must not be greater than the length of the buffer (converted to bits) */ 3765 assert(ctx->mask->length <= (ctx->super->used_cluster_mask_len * sizeof( 3766 struct spdk_blob_md_page) * 8)); 3767 /* The length of the mask must be exactly equal to the total number of clusters */ 3768 assert(ctx->mask->length == ctx->bs->total_clusters); 3769 3770 rc = spdk_bit_array_resize(&ctx->used_clusters, ctx->mask->length); 3771 if (rc < 0) { 3772 spdk_free(ctx->mask); 3773 bs_load_ctx_fail(ctx, rc); 3774 return; 3775 } 3776 3777 spdk_bit_array_load_mask(ctx->used_clusters, ctx->mask->mask); 3778 ctx->bs->num_free_clusters = spdk_bit_array_count_clear(ctx->used_clusters); 3779 assert(ctx->bs->num_free_clusters <= ctx->bs->total_clusters); 3780 3781 spdk_free(ctx->mask); 3782 3783 /* Read the used blobids mask */ 3784 mask_size = ctx->super->used_blobid_mask_len * SPDK_BS_PAGE_SIZE; 3785 ctx->mask = spdk_zmalloc(mask_size, 0x1000, NULL, SPDK_ENV_SOCKET_ID_ANY, 3786 SPDK_MALLOC_DMA); 3787 if (!ctx->mask) { 3788 bs_load_ctx_fail(ctx, -ENOMEM); 3789 return; 3790 } 3791 lba = bs_page_to_lba(ctx->bs, ctx->super->used_blobid_mask_start); 3792 lba_count = bs_page_to_lba(ctx->bs, ctx->super->used_blobid_mask_len); 3793 bs_sequence_read_dev(seq, ctx->mask, lba, lba_count, 3794 bs_load_used_blobids_cpl, ctx); 3795 } 3796 3797 static void 3798 bs_load_used_pages_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 3799 { 3800 struct spdk_bs_load_ctx *ctx = cb_arg; 3801 uint64_t lba, lba_count, mask_size; 3802 int rc; 3803 3804 if (bserrno != 0) { 3805 bs_load_ctx_fail(ctx, bserrno); 3806 return; 3807 } 3808 3809 /* The type must be correct */ 3810 assert(ctx->mask->type == SPDK_MD_MASK_TYPE_USED_PAGES); 3811 /* The length of the mask (in bits) must not be greater than the length of the buffer (converted to bits) */ 3812 assert(ctx->mask->length <= (ctx->super->used_page_mask_len * SPDK_BS_PAGE_SIZE * 3813 8)); 3814 /* The length of the mask must be exactly equal to the size (in pages) of the metadata region */ 3815 if (ctx->mask->length != ctx->super->md_len) { 3816 SPDK_ERRLOG("mismatched md_len in used_pages mask: " 3817 "mask->length=%" PRIu32 " super->md_len=%" PRIu32 "\n", 3818 ctx->mask->length, ctx->super->md_len); 3819 assert(false); 3820 } 3821 3822 rc = spdk_bit_array_resize(&ctx->bs->used_md_pages, ctx->mask->length); 3823 if (rc < 0) { 3824 spdk_free(ctx->mask); 3825 bs_load_ctx_fail(ctx, rc); 3826 return; 3827 } 3828 3829 spdk_bit_array_load_mask(ctx->bs->used_md_pages, ctx->mask->mask); 3830 spdk_free(ctx->mask); 3831 3832 /* Read the used clusters mask */ 3833 mask_size = ctx->super->used_cluster_mask_len * SPDK_BS_PAGE_SIZE; 3834 ctx->mask = spdk_zmalloc(mask_size, 0x1000, NULL, SPDK_ENV_SOCKET_ID_ANY, 3835 SPDK_MALLOC_DMA); 3836 if (!ctx->mask) { 3837 bs_load_ctx_fail(ctx, -ENOMEM); 3838 return; 3839 } 3840 lba = bs_page_to_lba(ctx->bs, ctx->super->used_cluster_mask_start); 3841 lba_count = bs_page_to_lba(ctx->bs, ctx->super->used_cluster_mask_len); 3842 bs_sequence_read_dev(seq, ctx->mask, lba, lba_count, 3843 bs_load_used_clusters_cpl, ctx); 3844 } 3845 3846 static void 3847 bs_load_read_used_pages(struct spdk_bs_load_ctx *ctx) 3848 { 3849 uint64_t lba, lba_count, mask_size; 3850 3851 /* Read the used pages mask */ 3852 mask_size = ctx->super->used_page_mask_len * SPDK_BS_PAGE_SIZE; 3853 ctx->mask = spdk_zmalloc(mask_size, 0x1000, NULL, 3854 SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA); 3855 if (!ctx->mask) { 3856 bs_load_ctx_fail(ctx, -ENOMEM); 3857 return; 3858 } 3859 3860 lba = bs_page_to_lba(ctx->bs, ctx->super->used_page_mask_start); 3861 lba_count = bs_page_to_lba(ctx->bs, ctx->super->used_page_mask_len); 3862 bs_sequence_read_dev(ctx->seq, ctx->mask, lba, lba_count, 3863 bs_load_used_pages_cpl, ctx); 3864 } 3865 3866 static int 3867 bs_load_replay_md_parse_page(struct spdk_bs_load_ctx *ctx, struct spdk_blob_md_page *page) 3868 { 3869 struct spdk_blob_store *bs = ctx->bs; 3870 struct spdk_blob_md_descriptor *desc; 3871 size_t cur_desc = 0; 3872 3873 desc = (struct spdk_blob_md_descriptor *)page->descriptors; 3874 while (cur_desc < sizeof(page->descriptors)) { 3875 if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_PADDING) { 3876 if (desc->length == 0) { 3877 /* If padding and length are 0, this terminates the page */ 3878 break; 3879 } 3880 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_EXTENT_RLE) { 3881 struct spdk_blob_md_descriptor_extent_rle *desc_extent_rle; 3882 unsigned int i, j; 3883 unsigned int cluster_count = 0; 3884 uint32_t cluster_idx; 3885 3886 desc_extent_rle = (struct spdk_blob_md_descriptor_extent_rle *)desc; 3887 3888 for (i = 0; i < desc_extent_rle->length / sizeof(desc_extent_rle->extents[0]); i++) { 3889 for (j = 0; j < desc_extent_rle->extents[i].length; j++) { 3890 cluster_idx = desc_extent_rle->extents[i].cluster_idx; 3891 /* 3892 * cluster_idx = 0 means an unallocated cluster - don't mark that 3893 * in the used cluster map. 3894 */ 3895 if (cluster_idx != 0) { 3896 spdk_bit_array_set(ctx->used_clusters, cluster_idx + j); 3897 if (bs->num_free_clusters == 0) { 3898 return -ENOSPC; 3899 } 3900 bs->num_free_clusters--; 3901 } 3902 cluster_count++; 3903 } 3904 } 3905 if (cluster_count == 0) { 3906 return -EINVAL; 3907 } 3908 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_EXTENT_PAGE) { 3909 struct spdk_blob_md_descriptor_extent_page *desc_extent; 3910 uint32_t i; 3911 uint32_t cluster_count = 0; 3912 uint32_t cluster_idx; 3913 size_t cluster_idx_length; 3914 3915 desc_extent = (struct spdk_blob_md_descriptor_extent_page *)desc; 3916 cluster_idx_length = desc_extent->length - sizeof(desc_extent->start_cluster_idx); 3917 3918 if (desc_extent->length <= sizeof(desc_extent->start_cluster_idx) || 3919 (cluster_idx_length % sizeof(desc_extent->cluster_idx[0]) != 0)) { 3920 return -EINVAL; 3921 } 3922 3923 for (i = 0; i < cluster_idx_length / sizeof(desc_extent->cluster_idx[0]); i++) { 3924 cluster_idx = desc_extent->cluster_idx[i]; 3925 /* 3926 * cluster_idx = 0 means an unallocated cluster - don't mark that 3927 * in the used cluster map. 3928 */ 3929 if (cluster_idx != 0) { 3930 if (cluster_idx < desc_extent->start_cluster_idx && 3931 cluster_idx >= desc_extent->start_cluster_idx + cluster_count) { 3932 return -EINVAL; 3933 } 3934 spdk_bit_array_set(ctx->used_clusters, cluster_idx); 3935 if (bs->num_free_clusters == 0) { 3936 return -ENOSPC; 3937 } 3938 bs->num_free_clusters--; 3939 } 3940 cluster_count++; 3941 } 3942 3943 if (cluster_count == 0) { 3944 return -EINVAL; 3945 } 3946 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_XATTR) { 3947 /* Skip this item */ 3948 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_XATTR_INTERNAL) { 3949 /* Skip this item */ 3950 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_FLAGS) { 3951 /* Skip this item */ 3952 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_EXTENT_TABLE) { 3953 struct spdk_blob_md_descriptor_extent_table *desc_extent_table; 3954 uint32_t num_extent_pages = ctx->num_extent_pages; 3955 uint32_t i; 3956 size_t extent_pages_length; 3957 void *tmp; 3958 3959 desc_extent_table = (struct spdk_blob_md_descriptor_extent_table *)desc; 3960 extent_pages_length = desc_extent_table->length - sizeof(desc_extent_table->num_clusters); 3961 3962 if (desc_extent_table->length == 0 || 3963 (extent_pages_length % sizeof(desc_extent_table->extent_page[0]) != 0)) { 3964 return -EINVAL; 3965 } 3966 3967 for (i = 0; i < extent_pages_length / sizeof(desc_extent_table->extent_page[0]); i++) { 3968 if (desc_extent_table->extent_page[i].page_idx != 0) { 3969 if (desc_extent_table->extent_page[i].num_pages != 1) { 3970 return -EINVAL; 3971 } 3972 num_extent_pages += 1; 3973 } 3974 } 3975 3976 if (num_extent_pages > 0) { 3977 tmp = realloc(ctx->extent_page_num, num_extent_pages * sizeof(uint32_t)); 3978 if (tmp == NULL) { 3979 return -ENOMEM; 3980 } 3981 ctx->extent_page_num = tmp; 3982 3983 /* Extent table entries contain md page numbers for extent pages. 3984 * Zeroes represent unallocated extent pages, those are run-length-encoded. 3985 */ 3986 for (i = 0; i < extent_pages_length / sizeof(desc_extent_table->extent_page[0]); i++) { 3987 if (desc_extent_table->extent_page[i].page_idx != 0) { 3988 ctx->extent_page_num[ctx->num_extent_pages] = desc_extent_table->extent_page[i].page_idx; 3989 ctx->num_extent_pages += 1; 3990 } 3991 } 3992 } 3993 } else { 3994 /* Error */ 3995 return -EINVAL; 3996 } 3997 /* Advance to the next descriptor */ 3998 cur_desc += sizeof(*desc) + desc->length; 3999 if (cur_desc + sizeof(*desc) > sizeof(page->descriptors)) { 4000 break; 4001 } 4002 desc = (struct spdk_blob_md_descriptor *)((uintptr_t)page->descriptors + cur_desc); 4003 } 4004 return 0; 4005 } 4006 4007 static bool bs_load_cur_extent_page_valid(struct spdk_blob_md_page *page) 4008 { 4009 uint32_t crc; 4010 struct spdk_blob_md_descriptor *desc = (struct spdk_blob_md_descriptor *)page->descriptors; 4011 size_t desc_len; 4012 4013 crc = blob_md_page_calc_crc(page); 4014 if (crc != page->crc) { 4015 return false; 4016 } 4017 4018 /* Extent page should always be of sequence num 0. */ 4019 if (page->sequence_num != 0) { 4020 return false; 4021 } 4022 4023 /* Descriptor type must be EXTENT_PAGE. */ 4024 if (desc->type != SPDK_MD_DESCRIPTOR_TYPE_EXTENT_PAGE) { 4025 return false; 4026 } 4027 4028 /* Descriptor length cannot exceed the page. */ 4029 desc_len = sizeof(*desc) + desc->length; 4030 if (desc_len > sizeof(page->descriptors)) { 4031 return false; 4032 } 4033 4034 /* It has to be the only descriptor in the page. */ 4035 if (desc_len + sizeof(*desc) <= sizeof(page->descriptors)) { 4036 desc = (struct spdk_blob_md_descriptor *)((uintptr_t)page->descriptors + desc_len); 4037 if (desc->length != 0) { 4038 return false; 4039 } 4040 } 4041 4042 return true; 4043 } 4044 4045 static bool bs_load_cur_md_page_valid(struct spdk_bs_load_ctx *ctx) 4046 { 4047 uint32_t crc; 4048 struct spdk_blob_md_page *page = ctx->page; 4049 4050 crc = blob_md_page_calc_crc(page); 4051 if (crc != page->crc) { 4052 return false; 4053 } 4054 4055 /* First page of a sequence should match the blobid. */ 4056 if (page->sequence_num == 0 && 4057 bs_page_to_blobid(ctx->cur_page) != page->id) { 4058 return false; 4059 } 4060 assert(bs_load_cur_extent_page_valid(page) == false); 4061 4062 return true; 4063 } 4064 4065 static void 4066 bs_load_replay_cur_md_page(struct spdk_bs_load_ctx *ctx); 4067 4068 static void 4069 bs_load_write_used_clusters_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 4070 { 4071 struct spdk_bs_load_ctx *ctx = cb_arg; 4072 4073 if (bserrno != 0) { 4074 bs_load_ctx_fail(ctx, bserrno); 4075 return; 4076 } 4077 4078 bs_load_complete(ctx); 4079 } 4080 4081 static void 4082 bs_load_write_used_blobids_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 4083 { 4084 struct spdk_bs_load_ctx *ctx = cb_arg; 4085 4086 spdk_free(ctx->mask); 4087 ctx->mask = NULL; 4088 4089 if (bserrno != 0) { 4090 bs_load_ctx_fail(ctx, bserrno); 4091 return; 4092 } 4093 4094 bs_write_used_clusters(seq, ctx, bs_load_write_used_clusters_cpl); 4095 } 4096 4097 static void 4098 bs_load_write_used_pages_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 4099 { 4100 struct spdk_bs_load_ctx *ctx = cb_arg; 4101 4102 spdk_free(ctx->mask); 4103 ctx->mask = NULL; 4104 4105 if (bserrno != 0) { 4106 bs_load_ctx_fail(ctx, bserrno); 4107 return; 4108 } 4109 4110 bs_write_used_blobids(seq, ctx, bs_load_write_used_blobids_cpl); 4111 } 4112 4113 static void 4114 bs_load_write_used_md(struct spdk_bs_load_ctx *ctx) 4115 { 4116 bs_write_used_md(ctx->seq, ctx, bs_load_write_used_pages_cpl); 4117 } 4118 4119 static void 4120 bs_load_replay_md_chain_cpl(struct spdk_bs_load_ctx *ctx) 4121 { 4122 uint64_t num_md_clusters; 4123 uint64_t i; 4124 4125 ctx->in_page_chain = false; 4126 4127 do { 4128 ctx->page_index++; 4129 } while (spdk_bit_array_get(ctx->bs->used_md_pages, ctx->page_index) == true); 4130 4131 if (ctx->page_index < ctx->super->md_len) { 4132 ctx->cur_page = ctx->page_index; 4133 bs_load_replay_cur_md_page(ctx); 4134 } else { 4135 /* Claim all of the clusters used by the metadata */ 4136 num_md_clusters = spdk_divide_round_up( 4137 ctx->super->md_start + ctx->super->md_len, ctx->bs->pages_per_cluster); 4138 for (i = 0; i < num_md_clusters; i++) { 4139 spdk_bit_array_set(ctx->used_clusters, i); 4140 } 4141 ctx->bs->num_free_clusters -= num_md_clusters; 4142 spdk_free(ctx->page); 4143 bs_load_write_used_md(ctx); 4144 } 4145 } 4146 4147 static void 4148 bs_load_replay_extent_page_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 4149 { 4150 struct spdk_bs_load_ctx *ctx = cb_arg; 4151 uint32_t page_num; 4152 uint64_t i; 4153 4154 if (bserrno != 0) { 4155 spdk_free(ctx->extent_pages); 4156 bs_load_ctx_fail(ctx, bserrno); 4157 return; 4158 } 4159 4160 for (i = 0; i < ctx->num_extent_pages; i++) { 4161 /* Extent pages are only read when present within in chain md. 4162 * Integrity of md is not right if that page was not a valid extent page. */ 4163 if (bs_load_cur_extent_page_valid(&ctx->extent_pages[i]) != true) { 4164 spdk_free(ctx->extent_pages); 4165 bs_load_ctx_fail(ctx, -EILSEQ); 4166 return; 4167 } 4168 4169 page_num = ctx->extent_page_num[i]; 4170 spdk_bit_array_set(ctx->bs->used_md_pages, page_num); 4171 if (bs_load_replay_md_parse_page(ctx, &ctx->extent_pages[i])) { 4172 spdk_free(ctx->extent_pages); 4173 bs_load_ctx_fail(ctx, -EILSEQ); 4174 return; 4175 } 4176 } 4177 4178 spdk_free(ctx->extent_pages); 4179 free(ctx->extent_page_num); 4180 ctx->extent_page_num = NULL; 4181 ctx->num_extent_pages = 0; 4182 4183 bs_load_replay_md_chain_cpl(ctx); 4184 } 4185 4186 static void 4187 bs_load_replay_extent_pages(struct spdk_bs_load_ctx *ctx) 4188 { 4189 spdk_bs_batch_t *batch; 4190 uint32_t page; 4191 uint64_t lba; 4192 uint64_t i; 4193 4194 ctx->extent_pages = spdk_zmalloc(SPDK_BS_PAGE_SIZE * ctx->num_extent_pages, 0, 4195 NULL, SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA); 4196 if (!ctx->extent_pages) { 4197 bs_load_ctx_fail(ctx, -ENOMEM); 4198 return; 4199 } 4200 4201 batch = bs_sequence_to_batch(ctx->seq, bs_load_replay_extent_page_cpl, ctx); 4202 4203 for (i = 0; i < ctx->num_extent_pages; i++) { 4204 page = ctx->extent_page_num[i]; 4205 assert(page < ctx->super->md_len); 4206 lba = bs_md_page_to_lba(ctx->bs, page); 4207 bs_batch_read_dev(batch, &ctx->extent_pages[i], lba, 4208 bs_byte_to_lba(ctx->bs, SPDK_BS_PAGE_SIZE)); 4209 } 4210 4211 bs_batch_close(batch); 4212 } 4213 4214 static void 4215 bs_load_replay_md_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 4216 { 4217 struct spdk_bs_load_ctx *ctx = cb_arg; 4218 uint32_t page_num; 4219 struct spdk_blob_md_page *page; 4220 4221 if (bserrno != 0) { 4222 bs_load_ctx_fail(ctx, bserrno); 4223 return; 4224 } 4225 4226 page_num = ctx->cur_page; 4227 page = ctx->page; 4228 if (bs_load_cur_md_page_valid(ctx) == true) { 4229 if (page->sequence_num == 0 || ctx->in_page_chain == true) { 4230 bs_claim_md_page(ctx->bs, page_num); 4231 if (page->sequence_num == 0) { 4232 spdk_bit_array_set(ctx->bs->used_blobids, page_num); 4233 } 4234 if (bs_load_replay_md_parse_page(ctx, page)) { 4235 bs_load_ctx_fail(ctx, -EILSEQ); 4236 return; 4237 } 4238 if (page->next != SPDK_INVALID_MD_PAGE) { 4239 ctx->in_page_chain = true; 4240 ctx->cur_page = page->next; 4241 bs_load_replay_cur_md_page(ctx); 4242 return; 4243 } 4244 if (ctx->num_extent_pages != 0) { 4245 bs_load_replay_extent_pages(ctx); 4246 return; 4247 } 4248 } 4249 } 4250 bs_load_replay_md_chain_cpl(ctx); 4251 } 4252 4253 static void 4254 bs_load_replay_cur_md_page(struct spdk_bs_load_ctx *ctx) 4255 { 4256 uint64_t lba; 4257 4258 assert(ctx->cur_page < ctx->super->md_len); 4259 lba = bs_md_page_to_lba(ctx->bs, ctx->cur_page); 4260 bs_sequence_read_dev(ctx->seq, ctx->page, lba, 4261 bs_byte_to_lba(ctx->bs, SPDK_BS_PAGE_SIZE), 4262 bs_load_replay_md_cpl, ctx); 4263 } 4264 4265 static void 4266 bs_load_replay_md(struct spdk_bs_load_ctx *ctx) 4267 { 4268 ctx->page_index = 0; 4269 ctx->cur_page = 0; 4270 ctx->page = spdk_zmalloc(SPDK_BS_PAGE_SIZE, 0, 4271 NULL, SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA); 4272 if (!ctx->page) { 4273 bs_load_ctx_fail(ctx, -ENOMEM); 4274 return; 4275 } 4276 bs_load_replay_cur_md_page(ctx); 4277 } 4278 4279 static void 4280 bs_recover(struct spdk_bs_load_ctx *ctx) 4281 { 4282 int rc; 4283 4284 rc = spdk_bit_array_resize(&ctx->bs->used_md_pages, ctx->super->md_len); 4285 if (rc < 0) { 4286 bs_load_ctx_fail(ctx, -ENOMEM); 4287 return; 4288 } 4289 4290 rc = spdk_bit_array_resize(&ctx->bs->used_blobids, ctx->super->md_len); 4291 if (rc < 0) { 4292 bs_load_ctx_fail(ctx, -ENOMEM); 4293 return; 4294 } 4295 4296 rc = spdk_bit_array_resize(&ctx->used_clusters, ctx->bs->total_clusters); 4297 if (rc < 0) { 4298 bs_load_ctx_fail(ctx, -ENOMEM); 4299 return; 4300 } 4301 4302 rc = spdk_bit_array_resize(&ctx->bs->open_blobids, ctx->super->md_len); 4303 if (rc < 0) { 4304 bs_load_ctx_fail(ctx, -ENOMEM); 4305 return; 4306 } 4307 4308 ctx->bs->num_free_clusters = ctx->bs->total_clusters; 4309 bs_load_replay_md(ctx); 4310 } 4311 4312 static void 4313 bs_load_super_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 4314 { 4315 struct spdk_bs_load_ctx *ctx = cb_arg; 4316 uint32_t crc; 4317 int rc; 4318 static const char zeros[SPDK_BLOBSTORE_TYPE_LENGTH]; 4319 4320 if (ctx->super->version > SPDK_BS_VERSION || 4321 ctx->super->version < SPDK_BS_INITIAL_VERSION) { 4322 bs_load_ctx_fail(ctx, -EILSEQ); 4323 return; 4324 } 4325 4326 if (memcmp(ctx->super->signature, SPDK_BS_SUPER_BLOCK_SIG, 4327 sizeof(ctx->super->signature)) != 0) { 4328 bs_load_ctx_fail(ctx, -EILSEQ); 4329 return; 4330 } 4331 4332 crc = blob_md_page_calc_crc(ctx->super); 4333 if (crc != ctx->super->crc) { 4334 bs_load_ctx_fail(ctx, -EILSEQ); 4335 return; 4336 } 4337 4338 if (memcmp(&ctx->bs->bstype, &ctx->super->bstype, SPDK_BLOBSTORE_TYPE_LENGTH) == 0) { 4339 SPDK_DEBUGLOG(blob, "Bstype matched - loading blobstore\n"); 4340 } else if (memcmp(&ctx->bs->bstype, zeros, SPDK_BLOBSTORE_TYPE_LENGTH) == 0) { 4341 SPDK_DEBUGLOG(blob, "Bstype wildcard used - loading blobstore regardless bstype\n"); 4342 } else { 4343 SPDK_DEBUGLOG(blob, "Unexpected bstype\n"); 4344 SPDK_LOGDUMP(blob, "Expected:", ctx->bs->bstype.bstype, SPDK_BLOBSTORE_TYPE_LENGTH); 4345 SPDK_LOGDUMP(blob, "Found:", ctx->super->bstype.bstype, SPDK_BLOBSTORE_TYPE_LENGTH); 4346 bs_load_ctx_fail(ctx, -ENXIO); 4347 return; 4348 } 4349 4350 if (ctx->super->size > ctx->bs->dev->blockcnt * ctx->bs->dev->blocklen) { 4351 SPDK_NOTICELOG("Size mismatch, dev size: %" PRIu64 ", blobstore size: %" PRIu64 "\n", 4352 ctx->bs->dev->blockcnt * ctx->bs->dev->blocklen, ctx->super->size); 4353 bs_load_ctx_fail(ctx, -EILSEQ); 4354 return; 4355 } 4356 4357 if (ctx->super->size == 0) { 4358 ctx->super->size = ctx->bs->dev->blockcnt * ctx->bs->dev->blocklen; 4359 } 4360 4361 if (ctx->super->io_unit_size == 0) { 4362 ctx->super->io_unit_size = SPDK_BS_PAGE_SIZE; 4363 } 4364 4365 /* Parse the super block */ 4366 ctx->bs->clean = 1; 4367 ctx->bs->cluster_sz = ctx->super->cluster_size; 4368 ctx->bs->total_clusters = ctx->super->size / ctx->super->cluster_size; 4369 ctx->bs->pages_per_cluster = ctx->bs->cluster_sz / SPDK_BS_PAGE_SIZE; 4370 if (spdk_u32_is_pow2(ctx->bs->pages_per_cluster)) { 4371 ctx->bs->pages_per_cluster_shift = spdk_u32log2(ctx->bs->pages_per_cluster); 4372 } 4373 ctx->bs->io_unit_size = ctx->super->io_unit_size; 4374 rc = spdk_bit_array_resize(&ctx->used_clusters, ctx->bs->total_clusters); 4375 if (rc < 0) { 4376 bs_load_ctx_fail(ctx, -ENOMEM); 4377 return; 4378 } 4379 ctx->bs->md_start = ctx->super->md_start; 4380 ctx->bs->md_len = ctx->super->md_len; 4381 rc = spdk_bit_array_resize(&ctx->bs->open_blobids, ctx->bs->md_len); 4382 if (rc < 0) { 4383 bs_load_ctx_fail(ctx, -ENOMEM); 4384 return; 4385 } 4386 4387 ctx->bs->total_data_clusters = ctx->bs->total_clusters - spdk_divide_round_up( 4388 ctx->bs->md_start + ctx->bs->md_len, ctx->bs->pages_per_cluster); 4389 ctx->bs->super_blob = ctx->super->super_blob; 4390 memcpy(&ctx->bs->bstype, &ctx->super->bstype, sizeof(ctx->super->bstype)); 4391 4392 if (ctx->super->used_blobid_mask_len == 0 || ctx->super->clean == 0) { 4393 bs_recover(ctx); 4394 } else { 4395 bs_load_read_used_pages(ctx); 4396 } 4397 } 4398 4399 static inline int 4400 bs_opts_copy(struct spdk_bs_opts *src, struct spdk_bs_opts *dst) 4401 { 4402 4403 if (!src->opts_size) { 4404 SPDK_ERRLOG("opts_size should not be zero value\n"); 4405 return -1; 4406 } 4407 4408 #define FIELD_OK(field) \ 4409 offsetof(struct spdk_bs_opts, field) + sizeof(src->field) <= src->opts_size 4410 4411 #define SET_FIELD(field) \ 4412 if (FIELD_OK(field)) { \ 4413 dst->field = src->field; \ 4414 } \ 4415 4416 SET_FIELD(cluster_sz); 4417 SET_FIELD(num_md_pages); 4418 SET_FIELD(max_md_ops); 4419 SET_FIELD(max_channel_ops); 4420 SET_FIELD(clear_method); 4421 4422 if (FIELD_OK(bstype)) { 4423 memcpy(&dst->bstype, &src->bstype, sizeof(dst->bstype)); 4424 } 4425 SET_FIELD(iter_cb_fn); 4426 SET_FIELD(iter_cb_arg); 4427 4428 dst->opts_size = src->opts_size; 4429 4430 /* You should not remove this statement, but need to update the assert statement 4431 * if you add a new field, and also add a corresponding SET_FIELD statement */ 4432 SPDK_STATIC_ASSERT(sizeof(struct spdk_bs_opts) == 64, "Incorrect size"); 4433 4434 #undef FIELD_OK 4435 #undef SET_FIELD 4436 4437 return 0; 4438 } 4439 4440 void 4441 spdk_bs_load(struct spdk_bs_dev *dev, struct spdk_bs_opts *o, 4442 spdk_bs_op_with_handle_complete cb_fn, void *cb_arg) 4443 { 4444 struct spdk_blob_store *bs; 4445 struct spdk_bs_cpl cpl; 4446 struct spdk_bs_load_ctx *ctx; 4447 struct spdk_bs_opts opts = {}; 4448 int err; 4449 4450 SPDK_DEBUGLOG(blob, "Loading blobstore from dev %p\n", dev); 4451 4452 if ((SPDK_BS_PAGE_SIZE % dev->blocklen) != 0) { 4453 SPDK_DEBUGLOG(blob, "unsupported dev block length of %d\n", dev->blocklen); 4454 dev->destroy(dev); 4455 cb_fn(cb_arg, NULL, -EINVAL); 4456 return; 4457 } 4458 4459 spdk_bs_opts_init(&opts, sizeof(opts)); 4460 if (o) { 4461 if (bs_opts_copy(o, &opts)) { 4462 return; 4463 } 4464 } 4465 4466 if (opts.max_md_ops == 0 || opts.max_channel_ops == 0) { 4467 dev->destroy(dev); 4468 cb_fn(cb_arg, NULL, -EINVAL); 4469 return; 4470 } 4471 4472 err = bs_alloc(dev, &opts, &bs, &ctx); 4473 if (err) { 4474 dev->destroy(dev); 4475 cb_fn(cb_arg, NULL, err); 4476 return; 4477 } 4478 4479 cpl.type = SPDK_BS_CPL_TYPE_BS_HANDLE; 4480 cpl.u.bs_handle.cb_fn = cb_fn; 4481 cpl.u.bs_handle.cb_arg = cb_arg; 4482 cpl.u.bs_handle.bs = bs; 4483 4484 ctx->seq = bs_sequence_start(bs->md_channel, &cpl); 4485 if (!ctx->seq) { 4486 spdk_free(ctx->super); 4487 free(ctx); 4488 bs_free(bs); 4489 cb_fn(cb_arg, NULL, -ENOMEM); 4490 return; 4491 } 4492 4493 /* Read the super block */ 4494 bs_sequence_read_dev(ctx->seq, ctx->super, bs_page_to_lba(bs, 0), 4495 bs_byte_to_lba(bs, sizeof(*ctx->super)), 4496 bs_load_super_cpl, ctx); 4497 } 4498 4499 /* END spdk_bs_load */ 4500 4501 /* START spdk_bs_dump */ 4502 4503 static void 4504 bs_dump_finish(spdk_bs_sequence_t *seq, struct spdk_bs_load_ctx *ctx, int bserrno) 4505 { 4506 spdk_free(ctx->super); 4507 4508 /* 4509 * We need to defer calling bs_call_cpl() until after 4510 * dev destruction, so tuck these away for later use. 4511 */ 4512 ctx->bs->unload_err = bserrno; 4513 memcpy(&ctx->bs->unload_cpl, &seq->cpl, sizeof(struct spdk_bs_cpl)); 4514 seq->cpl.type = SPDK_BS_CPL_TYPE_NONE; 4515 4516 bs_sequence_finish(seq, 0); 4517 bs_free(ctx->bs); 4518 free(ctx); 4519 } 4520 4521 static void bs_dump_read_md_page(spdk_bs_sequence_t *seq, void *cb_arg); 4522 4523 static void 4524 bs_dump_print_md_page(struct spdk_bs_load_ctx *ctx) 4525 { 4526 uint32_t page_idx = ctx->cur_page; 4527 struct spdk_blob_md_page *page = ctx->page; 4528 struct spdk_blob_md_descriptor *desc; 4529 size_t cur_desc = 0; 4530 uint32_t crc; 4531 4532 fprintf(ctx->fp, "=========\n"); 4533 fprintf(ctx->fp, "Metadata Page Index: %" PRIu32 " (0x%" PRIx32 ")\n", page_idx, page_idx); 4534 fprintf(ctx->fp, "Blob ID: 0x%" PRIx64 "\n", page->id); 4535 4536 crc = blob_md_page_calc_crc(page); 4537 fprintf(ctx->fp, "CRC: 0x%" PRIx32 " (%s)\n", page->crc, crc == page->crc ? "OK" : "Mismatch"); 4538 4539 desc = (struct spdk_blob_md_descriptor *)page->descriptors; 4540 while (cur_desc < sizeof(page->descriptors)) { 4541 if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_PADDING) { 4542 if (desc->length == 0) { 4543 /* If padding and length are 0, this terminates the page */ 4544 break; 4545 } 4546 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_EXTENT_RLE) { 4547 struct spdk_blob_md_descriptor_extent_rle *desc_extent_rle; 4548 unsigned int i; 4549 4550 desc_extent_rle = (struct spdk_blob_md_descriptor_extent_rle *)desc; 4551 4552 for (i = 0; i < desc_extent_rle->length / sizeof(desc_extent_rle->extents[0]); i++) { 4553 if (desc_extent_rle->extents[i].cluster_idx != 0) { 4554 fprintf(ctx->fp, "Allocated Extent - Start: %" PRIu32, 4555 desc_extent_rle->extents[i].cluster_idx); 4556 } else { 4557 fprintf(ctx->fp, "Unallocated Extent - "); 4558 } 4559 fprintf(ctx->fp, " Length: %" PRIu32, desc_extent_rle->extents[i].length); 4560 fprintf(ctx->fp, "\n"); 4561 } 4562 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_EXTENT_PAGE) { 4563 struct spdk_blob_md_descriptor_extent_page *desc_extent; 4564 unsigned int i; 4565 4566 desc_extent = (struct spdk_blob_md_descriptor_extent_page *)desc; 4567 4568 for (i = 0; i < desc_extent->length / sizeof(desc_extent->cluster_idx[0]); i++) { 4569 if (desc_extent->cluster_idx[i] != 0) { 4570 fprintf(ctx->fp, "Allocated Extent - Start: %" PRIu32, 4571 desc_extent->cluster_idx[i]); 4572 } else { 4573 fprintf(ctx->fp, "Unallocated Extent"); 4574 } 4575 fprintf(ctx->fp, "\n"); 4576 } 4577 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_XATTR) { 4578 struct spdk_blob_md_descriptor_xattr *desc_xattr; 4579 uint32_t i; 4580 4581 desc_xattr = (struct spdk_blob_md_descriptor_xattr *)desc; 4582 4583 if (desc_xattr->length != 4584 sizeof(desc_xattr->name_length) + sizeof(desc_xattr->value_length) + 4585 desc_xattr->name_length + desc_xattr->value_length) { 4586 } 4587 4588 memcpy(ctx->xattr_name, desc_xattr->name, desc_xattr->name_length); 4589 ctx->xattr_name[desc_xattr->name_length] = '\0'; 4590 fprintf(ctx->fp, "XATTR: name = \"%s\"\n", ctx->xattr_name); 4591 fprintf(ctx->fp, " value = \""); 4592 ctx->print_xattr_fn(ctx->fp, ctx->super->bstype.bstype, ctx->xattr_name, 4593 (void *)((uintptr_t)desc_xattr->name + desc_xattr->name_length), 4594 desc_xattr->value_length); 4595 fprintf(ctx->fp, "\"\n"); 4596 for (i = 0; i < desc_xattr->value_length; i++) { 4597 if (i % 16 == 0) { 4598 fprintf(ctx->fp, " "); 4599 } 4600 fprintf(ctx->fp, "%02" PRIx8 " ", *((uint8_t *)desc_xattr->name + desc_xattr->name_length + i)); 4601 if ((i + 1) % 16 == 0) { 4602 fprintf(ctx->fp, "\n"); 4603 } 4604 } 4605 if (i % 16 != 0) { 4606 fprintf(ctx->fp, "\n"); 4607 } 4608 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_XATTR_INTERNAL) { 4609 /* TODO */ 4610 } else if (desc->type == SPDK_MD_DESCRIPTOR_TYPE_FLAGS) { 4611 /* TODO */ 4612 } else { 4613 /* Error */ 4614 } 4615 /* Advance to the next descriptor */ 4616 cur_desc += sizeof(*desc) + desc->length; 4617 if (cur_desc + sizeof(*desc) > sizeof(page->descriptors)) { 4618 break; 4619 } 4620 desc = (struct spdk_blob_md_descriptor *)((uintptr_t)page->descriptors + cur_desc); 4621 } 4622 } 4623 4624 static void 4625 bs_dump_read_md_page_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 4626 { 4627 struct spdk_bs_load_ctx *ctx = cb_arg; 4628 4629 if (bserrno != 0) { 4630 bs_dump_finish(seq, ctx, bserrno); 4631 return; 4632 } 4633 4634 if (ctx->page->id != 0) { 4635 bs_dump_print_md_page(ctx); 4636 } 4637 4638 ctx->cur_page++; 4639 4640 if (ctx->cur_page < ctx->super->md_len) { 4641 bs_dump_read_md_page(seq, ctx); 4642 } else { 4643 spdk_free(ctx->page); 4644 bs_dump_finish(seq, ctx, 0); 4645 } 4646 } 4647 4648 static void 4649 bs_dump_read_md_page(spdk_bs_sequence_t *seq, void *cb_arg) 4650 { 4651 struct spdk_bs_load_ctx *ctx = cb_arg; 4652 uint64_t lba; 4653 4654 assert(ctx->cur_page < ctx->super->md_len); 4655 lba = bs_page_to_lba(ctx->bs, ctx->super->md_start + ctx->cur_page); 4656 bs_sequence_read_dev(seq, ctx->page, lba, 4657 bs_byte_to_lba(ctx->bs, SPDK_BS_PAGE_SIZE), 4658 bs_dump_read_md_page_cpl, ctx); 4659 } 4660 4661 static void 4662 bs_dump_super_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 4663 { 4664 struct spdk_bs_load_ctx *ctx = cb_arg; 4665 4666 fprintf(ctx->fp, "Signature: \"%.8s\" ", ctx->super->signature); 4667 if (memcmp(ctx->super->signature, SPDK_BS_SUPER_BLOCK_SIG, 4668 sizeof(ctx->super->signature)) != 0) { 4669 fprintf(ctx->fp, "(Mismatch)\n"); 4670 bs_dump_finish(seq, ctx, bserrno); 4671 return; 4672 } else { 4673 fprintf(ctx->fp, "(OK)\n"); 4674 } 4675 fprintf(ctx->fp, "Version: %" PRIu32 "\n", ctx->super->version); 4676 fprintf(ctx->fp, "CRC: 0x%x (%s)\n", ctx->super->crc, 4677 (ctx->super->crc == blob_md_page_calc_crc(ctx->super)) ? "OK" : "Mismatch"); 4678 fprintf(ctx->fp, "Blobstore Type: %.*s\n", SPDK_BLOBSTORE_TYPE_LENGTH, ctx->super->bstype.bstype); 4679 fprintf(ctx->fp, "Cluster Size: %" PRIu32 "\n", ctx->super->cluster_size); 4680 fprintf(ctx->fp, "Super Blob ID: "); 4681 if (ctx->super->super_blob == SPDK_BLOBID_INVALID) { 4682 fprintf(ctx->fp, "(None)\n"); 4683 } else { 4684 fprintf(ctx->fp, "%" PRIu64 "\n", ctx->super->super_blob); 4685 } 4686 fprintf(ctx->fp, "Clean: %" PRIu32 "\n", ctx->super->clean); 4687 fprintf(ctx->fp, "Used Metadata Page Mask Start: %" PRIu32 "\n", ctx->super->used_page_mask_start); 4688 fprintf(ctx->fp, "Used Metadata Page Mask Length: %" PRIu32 "\n", ctx->super->used_page_mask_len); 4689 fprintf(ctx->fp, "Used Cluster Mask Start: %" PRIu32 "\n", ctx->super->used_cluster_mask_start); 4690 fprintf(ctx->fp, "Used Cluster Mask Length: %" PRIu32 "\n", ctx->super->used_cluster_mask_len); 4691 fprintf(ctx->fp, "Used Blob ID Mask Start: %" PRIu32 "\n", ctx->super->used_blobid_mask_start); 4692 fprintf(ctx->fp, "Used Blob ID Mask Length: %" PRIu32 "\n", ctx->super->used_blobid_mask_len); 4693 fprintf(ctx->fp, "Metadata Start: %" PRIu32 "\n", ctx->super->md_start); 4694 fprintf(ctx->fp, "Metadata Length: %" PRIu32 "\n", ctx->super->md_len); 4695 4696 ctx->cur_page = 0; 4697 ctx->page = spdk_zmalloc(SPDK_BS_PAGE_SIZE, 0, 4698 NULL, SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA); 4699 if (!ctx->page) { 4700 bs_dump_finish(seq, ctx, -ENOMEM); 4701 return; 4702 } 4703 bs_dump_read_md_page(seq, ctx); 4704 } 4705 4706 void 4707 spdk_bs_dump(struct spdk_bs_dev *dev, FILE *fp, spdk_bs_dump_print_xattr print_xattr_fn, 4708 spdk_bs_op_complete cb_fn, void *cb_arg) 4709 { 4710 struct spdk_blob_store *bs; 4711 struct spdk_bs_cpl cpl; 4712 spdk_bs_sequence_t *seq; 4713 struct spdk_bs_load_ctx *ctx; 4714 struct spdk_bs_opts opts = {}; 4715 int err; 4716 4717 SPDK_DEBUGLOG(blob, "Dumping blobstore from dev %p\n", dev); 4718 4719 spdk_bs_opts_init(&opts, sizeof(opts)); 4720 4721 err = bs_alloc(dev, &opts, &bs, &ctx); 4722 if (err) { 4723 dev->destroy(dev); 4724 cb_fn(cb_arg, err); 4725 return; 4726 } 4727 4728 ctx->fp = fp; 4729 ctx->print_xattr_fn = print_xattr_fn; 4730 4731 cpl.type = SPDK_BS_CPL_TYPE_BS_BASIC; 4732 cpl.u.bs_basic.cb_fn = cb_fn; 4733 cpl.u.bs_basic.cb_arg = cb_arg; 4734 4735 seq = bs_sequence_start(bs->md_channel, &cpl); 4736 if (!seq) { 4737 spdk_free(ctx->super); 4738 free(ctx); 4739 bs_free(bs); 4740 cb_fn(cb_arg, -ENOMEM); 4741 return; 4742 } 4743 4744 /* Read the super block */ 4745 bs_sequence_read_dev(seq, ctx->super, bs_page_to_lba(bs, 0), 4746 bs_byte_to_lba(bs, sizeof(*ctx->super)), 4747 bs_dump_super_cpl, ctx); 4748 } 4749 4750 /* END spdk_bs_dump */ 4751 4752 /* START spdk_bs_init */ 4753 4754 static void 4755 bs_init_persist_super_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 4756 { 4757 struct spdk_bs_load_ctx *ctx = cb_arg; 4758 4759 ctx->bs->used_clusters = spdk_bit_pool_create_from_array(ctx->used_clusters); 4760 spdk_free(ctx->super); 4761 free(ctx); 4762 4763 bs_sequence_finish(seq, bserrno); 4764 } 4765 4766 static void 4767 bs_init_trim_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 4768 { 4769 struct spdk_bs_load_ctx *ctx = cb_arg; 4770 4771 /* Write super block */ 4772 bs_sequence_write_dev(seq, ctx->super, bs_page_to_lba(ctx->bs, 0), 4773 bs_byte_to_lba(ctx->bs, sizeof(*ctx->super)), 4774 bs_init_persist_super_cpl, ctx); 4775 } 4776 4777 void 4778 spdk_bs_init(struct spdk_bs_dev *dev, struct spdk_bs_opts *o, 4779 spdk_bs_op_with_handle_complete cb_fn, void *cb_arg) 4780 { 4781 struct spdk_bs_load_ctx *ctx; 4782 struct spdk_blob_store *bs; 4783 struct spdk_bs_cpl cpl; 4784 spdk_bs_sequence_t *seq; 4785 spdk_bs_batch_t *batch; 4786 uint64_t num_md_lba; 4787 uint64_t num_md_pages; 4788 uint64_t num_md_clusters; 4789 uint32_t i; 4790 struct spdk_bs_opts opts = {}; 4791 int rc; 4792 uint64_t lba, lba_count; 4793 4794 SPDK_DEBUGLOG(blob, "Initializing blobstore on dev %p\n", dev); 4795 4796 if ((SPDK_BS_PAGE_SIZE % dev->blocklen) != 0) { 4797 SPDK_ERRLOG("unsupported dev block length of %d\n", 4798 dev->blocklen); 4799 dev->destroy(dev); 4800 cb_fn(cb_arg, NULL, -EINVAL); 4801 return; 4802 } 4803 4804 spdk_bs_opts_init(&opts, sizeof(opts)); 4805 if (o) { 4806 if (bs_opts_copy(o, &opts)) { 4807 return; 4808 } 4809 } 4810 4811 if (bs_opts_verify(&opts) != 0) { 4812 dev->destroy(dev); 4813 cb_fn(cb_arg, NULL, -EINVAL); 4814 return; 4815 } 4816 4817 rc = bs_alloc(dev, &opts, &bs, &ctx); 4818 if (rc) { 4819 dev->destroy(dev); 4820 cb_fn(cb_arg, NULL, rc); 4821 return; 4822 } 4823 4824 if (opts.num_md_pages == SPDK_BLOB_OPTS_NUM_MD_PAGES) { 4825 /* By default, allocate 1 page per cluster. 4826 * Technically, this over-allocates metadata 4827 * because more metadata will reduce the number 4828 * of usable clusters. This can be addressed with 4829 * more complex math in the future. 4830 */ 4831 bs->md_len = bs->total_clusters; 4832 } else { 4833 bs->md_len = opts.num_md_pages; 4834 } 4835 rc = spdk_bit_array_resize(&bs->used_md_pages, bs->md_len); 4836 if (rc < 0) { 4837 spdk_free(ctx->super); 4838 free(ctx); 4839 bs_free(bs); 4840 cb_fn(cb_arg, NULL, -ENOMEM); 4841 return; 4842 } 4843 4844 rc = spdk_bit_array_resize(&bs->used_blobids, bs->md_len); 4845 if (rc < 0) { 4846 spdk_free(ctx->super); 4847 free(ctx); 4848 bs_free(bs); 4849 cb_fn(cb_arg, NULL, -ENOMEM); 4850 return; 4851 } 4852 4853 rc = spdk_bit_array_resize(&bs->open_blobids, bs->md_len); 4854 if (rc < 0) { 4855 spdk_free(ctx->super); 4856 free(ctx); 4857 bs_free(bs); 4858 cb_fn(cb_arg, NULL, -ENOMEM); 4859 return; 4860 } 4861 4862 memcpy(ctx->super->signature, SPDK_BS_SUPER_BLOCK_SIG, 4863 sizeof(ctx->super->signature)); 4864 ctx->super->version = SPDK_BS_VERSION; 4865 ctx->super->length = sizeof(*ctx->super); 4866 ctx->super->super_blob = bs->super_blob; 4867 ctx->super->clean = 0; 4868 ctx->super->cluster_size = bs->cluster_sz; 4869 ctx->super->io_unit_size = bs->io_unit_size; 4870 memcpy(&ctx->super->bstype, &bs->bstype, sizeof(bs->bstype)); 4871 4872 /* Calculate how many pages the metadata consumes at the front 4873 * of the disk. 4874 */ 4875 4876 /* The super block uses 1 page */ 4877 num_md_pages = 1; 4878 4879 /* The used_md_pages mask requires 1 bit per metadata page, rounded 4880 * up to the nearest page, plus a header. 4881 */ 4882 ctx->super->used_page_mask_start = num_md_pages; 4883 ctx->super->used_page_mask_len = spdk_divide_round_up(sizeof(struct spdk_bs_md_mask) + 4884 spdk_divide_round_up(bs->md_len, 8), 4885 SPDK_BS_PAGE_SIZE); 4886 num_md_pages += ctx->super->used_page_mask_len; 4887 4888 /* The used_clusters mask requires 1 bit per cluster, rounded 4889 * up to the nearest page, plus a header. 4890 */ 4891 ctx->super->used_cluster_mask_start = num_md_pages; 4892 ctx->super->used_cluster_mask_len = spdk_divide_round_up(sizeof(struct spdk_bs_md_mask) + 4893 spdk_divide_round_up(bs->total_clusters, 8), 4894 SPDK_BS_PAGE_SIZE); 4895 num_md_pages += ctx->super->used_cluster_mask_len; 4896 4897 /* The used_blobids mask requires 1 bit per metadata page, rounded 4898 * up to the nearest page, plus a header. 4899 */ 4900 ctx->super->used_blobid_mask_start = num_md_pages; 4901 ctx->super->used_blobid_mask_len = spdk_divide_round_up(sizeof(struct spdk_bs_md_mask) + 4902 spdk_divide_round_up(bs->md_len, 8), 4903 SPDK_BS_PAGE_SIZE); 4904 num_md_pages += ctx->super->used_blobid_mask_len; 4905 4906 /* The metadata region size was chosen above */ 4907 ctx->super->md_start = bs->md_start = num_md_pages; 4908 ctx->super->md_len = bs->md_len; 4909 num_md_pages += bs->md_len; 4910 4911 num_md_lba = bs_page_to_lba(bs, num_md_pages); 4912 4913 ctx->super->size = dev->blockcnt * dev->blocklen; 4914 4915 ctx->super->crc = blob_md_page_calc_crc(ctx->super); 4916 4917 num_md_clusters = spdk_divide_round_up(num_md_pages, bs->pages_per_cluster); 4918 if (num_md_clusters > bs->total_clusters) { 4919 SPDK_ERRLOG("Blobstore metadata cannot use more clusters than is available, " 4920 "please decrease number of pages reserved for metadata " 4921 "or increase cluster size.\n"); 4922 spdk_free(ctx->super); 4923 spdk_bit_array_free(&ctx->used_clusters); 4924 free(ctx); 4925 bs_free(bs); 4926 cb_fn(cb_arg, NULL, -ENOMEM); 4927 return; 4928 } 4929 /* Claim all of the clusters used by the metadata */ 4930 for (i = 0; i < num_md_clusters; i++) { 4931 spdk_bit_array_set(ctx->used_clusters, i); 4932 } 4933 4934 bs->num_free_clusters -= num_md_clusters; 4935 bs->total_data_clusters = bs->num_free_clusters; 4936 4937 cpl.type = SPDK_BS_CPL_TYPE_BS_HANDLE; 4938 cpl.u.bs_handle.cb_fn = cb_fn; 4939 cpl.u.bs_handle.cb_arg = cb_arg; 4940 cpl.u.bs_handle.bs = bs; 4941 4942 seq = bs_sequence_start(bs->md_channel, &cpl); 4943 if (!seq) { 4944 spdk_free(ctx->super); 4945 free(ctx); 4946 bs_free(bs); 4947 cb_fn(cb_arg, NULL, -ENOMEM); 4948 return; 4949 } 4950 4951 batch = bs_sequence_to_batch(seq, bs_init_trim_cpl, ctx); 4952 4953 /* Clear metadata space */ 4954 bs_batch_write_zeroes_dev(batch, 0, num_md_lba); 4955 4956 lba = num_md_lba; 4957 lba_count = ctx->bs->dev->blockcnt - lba; 4958 switch (opts.clear_method) { 4959 case BS_CLEAR_WITH_UNMAP: 4960 /* Trim data clusters */ 4961 bs_batch_unmap_dev(batch, lba, lba_count); 4962 break; 4963 case BS_CLEAR_WITH_WRITE_ZEROES: 4964 /* Write_zeroes to data clusters */ 4965 bs_batch_write_zeroes_dev(batch, lba, lba_count); 4966 break; 4967 case BS_CLEAR_WITH_NONE: 4968 default: 4969 break; 4970 } 4971 4972 bs_batch_close(batch); 4973 } 4974 4975 /* END spdk_bs_init */ 4976 4977 /* START spdk_bs_destroy */ 4978 4979 static void 4980 bs_destroy_trim_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 4981 { 4982 struct spdk_bs_load_ctx *ctx = cb_arg; 4983 struct spdk_blob_store *bs = ctx->bs; 4984 4985 /* 4986 * We need to defer calling bs_call_cpl() until after 4987 * dev destruction, so tuck these away for later use. 4988 */ 4989 bs->unload_err = bserrno; 4990 memcpy(&bs->unload_cpl, &seq->cpl, sizeof(struct spdk_bs_cpl)); 4991 seq->cpl.type = SPDK_BS_CPL_TYPE_NONE; 4992 4993 bs_sequence_finish(seq, bserrno); 4994 4995 bs_free(bs); 4996 free(ctx); 4997 } 4998 4999 void 5000 spdk_bs_destroy(struct spdk_blob_store *bs, spdk_bs_op_complete cb_fn, 5001 void *cb_arg) 5002 { 5003 struct spdk_bs_cpl cpl; 5004 spdk_bs_sequence_t *seq; 5005 struct spdk_bs_load_ctx *ctx; 5006 5007 SPDK_DEBUGLOG(blob, "Destroying blobstore\n"); 5008 5009 if (!TAILQ_EMPTY(&bs->blobs)) { 5010 SPDK_ERRLOG("Blobstore still has open blobs\n"); 5011 cb_fn(cb_arg, -EBUSY); 5012 return; 5013 } 5014 5015 cpl.type = SPDK_BS_CPL_TYPE_BS_BASIC; 5016 cpl.u.bs_basic.cb_fn = cb_fn; 5017 cpl.u.bs_basic.cb_arg = cb_arg; 5018 5019 ctx = calloc(1, sizeof(*ctx)); 5020 if (!ctx) { 5021 cb_fn(cb_arg, -ENOMEM); 5022 return; 5023 } 5024 5025 ctx->bs = bs; 5026 5027 seq = bs_sequence_start(bs->md_channel, &cpl); 5028 if (!seq) { 5029 free(ctx); 5030 cb_fn(cb_arg, -ENOMEM); 5031 return; 5032 } 5033 5034 /* Write zeroes to the super block */ 5035 bs_sequence_write_zeroes_dev(seq, 5036 bs_page_to_lba(bs, 0), 5037 bs_byte_to_lba(bs, sizeof(struct spdk_bs_super_block)), 5038 bs_destroy_trim_cpl, ctx); 5039 } 5040 5041 /* END spdk_bs_destroy */ 5042 5043 /* START spdk_bs_unload */ 5044 5045 static void 5046 bs_unload_finish(struct spdk_bs_load_ctx *ctx, int bserrno) 5047 { 5048 spdk_bs_sequence_t *seq = ctx->seq; 5049 5050 spdk_free(ctx->super); 5051 5052 /* 5053 * We need to defer calling bs_call_cpl() until after 5054 * dev destruction, so tuck these away for later use. 5055 */ 5056 ctx->bs->unload_err = bserrno; 5057 memcpy(&ctx->bs->unload_cpl, &seq->cpl, sizeof(struct spdk_bs_cpl)); 5058 seq->cpl.type = SPDK_BS_CPL_TYPE_NONE; 5059 5060 bs_sequence_finish(seq, bserrno); 5061 5062 bs_free(ctx->bs); 5063 free(ctx); 5064 } 5065 5066 static void 5067 bs_unload_write_super_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 5068 { 5069 struct spdk_bs_load_ctx *ctx = cb_arg; 5070 5071 bs_unload_finish(ctx, bserrno); 5072 } 5073 5074 static void 5075 bs_unload_write_used_clusters_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 5076 { 5077 struct spdk_bs_load_ctx *ctx = cb_arg; 5078 5079 spdk_free(ctx->mask); 5080 5081 if (bserrno != 0) { 5082 bs_unload_finish(ctx, bserrno); 5083 return; 5084 } 5085 5086 ctx->super->clean = 1; 5087 5088 bs_write_super(seq, ctx->bs, ctx->super, bs_unload_write_super_cpl, ctx); 5089 } 5090 5091 static void 5092 bs_unload_write_used_blobids_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 5093 { 5094 struct spdk_bs_load_ctx *ctx = cb_arg; 5095 5096 spdk_free(ctx->mask); 5097 ctx->mask = NULL; 5098 5099 if (bserrno != 0) { 5100 bs_unload_finish(ctx, bserrno); 5101 return; 5102 } 5103 5104 bs_write_used_clusters(seq, ctx, bs_unload_write_used_clusters_cpl); 5105 } 5106 5107 static void 5108 bs_unload_write_used_pages_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 5109 { 5110 struct spdk_bs_load_ctx *ctx = cb_arg; 5111 5112 spdk_free(ctx->mask); 5113 ctx->mask = NULL; 5114 5115 if (bserrno != 0) { 5116 bs_unload_finish(ctx, bserrno); 5117 return; 5118 } 5119 5120 bs_write_used_blobids(seq, ctx, bs_unload_write_used_blobids_cpl); 5121 } 5122 5123 static void 5124 bs_unload_read_super_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 5125 { 5126 struct spdk_bs_load_ctx *ctx = cb_arg; 5127 5128 if (bserrno != 0) { 5129 bs_unload_finish(ctx, bserrno); 5130 return; 5131 } 5132 5133 bs_write_used_md(seq, cb_arg, bs_unload_write_used_pages_cpl); 5134 } 5135 5136 void 5137 spdk_bs_unload(struct spdk_blob_store *bs, spdk_bs_op_complete cb_fn, void *cb_arg) 5138 { 5139 struct spdk_bs_cpl cpl; 5140 struct spdk_bs_load_ctx *ctx; 5141 5142 SPDK_DEBUGLOG(blob, "Syncing blobstore\n"); 5143 5144 if (!TAILQ_EMPTY(&bs->blobs)) { 5145 SPDK_ERRLOG("Blobstore still has open blobs\n"); 5146 cb_fn(cb_arg, -EBUSY); 5147 return; 5148 } 5149 5150 ctx = calloc(1, sizeof(*ctx)); 5151 if (!ctx) { 5152 cb_fn(cb_arg, -ENOMEM); 5153 return; 5154 } 5155 5156 ctx->bs = bs; 5157 5158 ctx->super = spdk_zmalloc(sizeof(*ctx->super), 0x1000, NULL, 5159 SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA); 5160 if (!ctx->super) { 5161 free(ctx); 5162 cb_fn(cb_arg, -ENOMEM); 5163 return; 5164 } 5165 5166 cpl.type = SPDK_BS_CPL_TYPE_BS_BASIC; 5167 cpl.u.bs_basic.cb_fn = cb_fn; 5168 cpl.u.bs_basic.cb_arg = cb_arg; 5169 5170 ctx->seq = bs_sequence_start(bs->md_channel, &cpl); 5171 if (!ctx->seq) { 5172 spdk_free(ctx->super); 5173 free(ctx); 5174 cb_fn(cb_arg, -ENOMEM); 5175 return; 5176 } 5177 5178 /* Read super block */ 5179 bs_sequence_read_dev(ctx->seq, ctx->super, bs_page_to_lba(bs, 0), 5180 bs_byte_to_lba(bs, sizeof(*ctx->super)), 5181 bs_unload_read_super_cpl, ctx); 5182 } 5183 5184 /* END spdk_bs_unload */ 5185 5186 /* START spdk_bs_set_super */ 5187 5188 struct spdk_bs_set_super_ctx { 5189 struct spdk_blob_store *bs; 5190 struct spdk_bs_super_block *super; 5191 }; 5192 5193 static void 5194 bs_set_super_write_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 5195 { 5196 struct spdk_bs_set_super_ctx *ctx = cb_arg; 5197 5198 if (bserrno != 0) { 5199 SPDK_ERRLOG("Unable to write to super block of blobstore\n"); 5200 } 5201 5202 spdk_free(ctx->super); 5203 5204 bs_sequence_finish(seq, bserrno); 5205 5206 free(ctx); 5207 } 5208 5209 static void 5210 bs_set_super_read_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 5211 { 5212 struct spdk_bs_set_super_ctx *ctx = cb_arg; 5213 5214 if (bserrno != 0) { 5215 SPDK_ERRLOG("Unable to read super block of blobstore\n"); 5216 spdk_free(ctx->super); 5217 bs_sequence_finish(seq, bserrno); 5218 free(ctx); 5219 return; 5220 } 5221 5222 bs_write_super(seq, ctx->bs, ctx->super, bs_set_super_write_cpl, ctx); 5223 } 5224 5225 void 5226 spdk_bs_set_super(struct spdk_blob_store *bs, spdk_blob_id blobid, 5227 spdk_bs_op_complete cb_fn, void *cb_arg) 5228 { 5229 struct spdk_bs_cpl cpl; 5230 spdk_bs_sequence_t *seq; 5231 struct spdk_bs_set_super_ctx *ctx; 5232 5233 SPDK_DEBUGLOG(blob, "Setting super blob id on blobstore\n"); 5234 5235 ctx = calloc(1, sizeof(*ctx)); 5236 if (!ctx) { 5237 cb_fn(cb_arg, -ENOMEM); 5238 return; 5239 } 5240 5241 ctx->bs = bs; 5242 5243 ctx->super = spdk_zmalloc(sizeof(*ctx->super), 0x1000, NULL, 5244 SPDK_ENV_SOCKET_ID_ANY, SPDK_MALLOC_DMA); 5245 if (!ctx->super) { 5246 free(ctx); 5247 cb_fn(cb_arg, -ENOMEM); 5248 return; 5249 } 5250 5251 cpl.type = SPDK_BS_CPL_TYPE_BS_BASIC; 5252 cpl.u.bs_basic.cb_fn = cb_fn; 5253 cpl.u.bs_basic.cb_arg = cb_arg; 5254 5255 seq = bs_sequence_start(bs->md_channel, &cpl); 5256 if (!seq) { 5257 spdk_free(ctx->super); 5258 free(ctx); 5259 cb_fn(cb_arg, -ENOMEM); 5260 return; 5261 } 5262 5263 bs->super_blob = blobid; 5264 5265 /* Read super block */ 5266 bs_sequence_read_dev(seq, ctx->super, bs_page_to_lba(bs, 0), 5267 bs_byte_to_lba(bs, sizeof(*ctx->super)), 5268 bs_set_super_read_cpl, ctx); 5269 } 5270 5271 /* END spdk_bs_set_super */ 5272 5273 void 5274 spdk_bs_get_super(struct spdk_blob_store *bs, 5275 spdk_blob_op_with_id_complete cb_fn, void *cb_arg) 5276 { 5277 if (bs->super_blob == SPDK_BLOBID_INVALID) { 5278 cb_fn(cb_arg, SPDK_BLOBID_INVALID, -ENOENT); 5279 } else { 5280 cb_fn(cb_arg, bs->super_blob, 0); 5281 } 5282 } 5283 5284 uint64_t 5285 spdk_bs_get_cluster_size(struct spdk_blob_store *bs) 5286 { 5287 return bs->cluster_sz; 5288 } 5289 5290 uint64_t 5291 spdk_bs_get_page_size(struct spdk_blob_store *bs) 5292 { 5293 return SPDK_BS_PAGE_SIZE; 5294 } 5295 5296 uint64_t 5297 spdk_bs_get_io_unit_size(struct spdk_blob_store *bs) 5298 { 5299 return bs->io_unit_size; 5300 } 5301 5302 uint64_t 5303 spdk_bs_free_cluster_count(struct spdk_blob_store *bs) 5304 { 5305 return bs->num_free_clusters; 5306 } 5307 5308 uint64_t 5309 spdk_bs_total_data_cluster_count(struct spdk_blob_store *bs) 5310 { 5311 return bs->total_data_clusters; 5312 } 5313 5314 static int 5315 bs_register_md_thread(struct spdk_blob_store *bs) 5316 { 5317 bs->md_channel = spdk_get_io_channel(bs); 5318 if (!bs->md_channel) { 5319 SPDK_ERRLOG("Failed to get IO channel.\n"); 5320 return -1; 5321 } 5322 5323 return 0; 5324 } 5325 5326 static int 5327 bs_unregister_md_thread(struct spdk_blob_store *bs) 5328 { 5329 spdk_put_io_channel(bs->md_channel); 5330 5331 return 0; 5332 } 5333 5334 spdk_blob_id spdk_blob_get_id(struct spdk_blob *blob) 5335 { 5336 assert(blob != NULL); 5337 5338 return blob->id; 5339 } 5340 5341 uint64_t spdk_blob_get_num_pages(struct spdk_blob *blob) 5342 { 5343 assert(blob != NULL); 5344 5345 return bs_cluster_to_page(blob->bs, blob->active.num_clusters); 5346 } 5347 5348 uint64_t spdk_blob_get_num_io_units(struct spdk_blob *blob) 5349 { 5350 assert(blob != NULL); 5351 5352 return spdk_blob_get_num_pages(blob) * bs_io_unit_per_page(blob->bs); 5353 } 5354 5355 uint64_t spdk_blob_get_num_clusters(struct spdk_blob *blob) 5356 { 5357 assert(blob != NULL); 5358 5359 return blob->active.num_clusters; 5360 } 5361 5362 /* START spdk_bs_create_blob */ 5363 5364 static void 5365 bs_create_blob_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 5366 { 5367 struct spdk_blob *blob = cb_arg; 5368 uint32_t page_idx = bs_blobid_to_page(blob->id); 5369 5370 if (bserrno != 0) { 5371 spdk_bit_array_clear(blob->bs->used_blobids, page_idx); 5372 bs_release_md_page(blob->bs, page_idx); 5373 } 5374 5375 blob_free(blob); 5376 5377 bs_sequence_finish(seq, bserrno); 5378 } 5379 5380 static int 5381 blob_set_xattrs(struct spdk_blob *blob, const struct spdk_blob_xattr_opts *xattrs, 5382 bool internal) 5383 { 5384 uint64_t i; 5385 size_t value_len = 0; 5386 int rc; 5387 const void *value = NULL; 5388 if (xattrs->count > 0 && xattrs->get_value == NULL) { 5389 return -EINVAL; 5390 } 5391 for (i = 0; i < xattrs->count; i++) { 5392 xattrs->get_value(xattrs->ctx, xattrs->names[i], &value, &value_len); 5393 if (value == NULL || value_len == 0) { 5394 return -EINVAL; 5395 } 5396 rc = blob_set_xattr(blob, xattrs->names[i], value, value_len, internal); 5397 if (rc < 0) { 5398 return rc; 5399 } 5400 } 5401 return 0; 5402 } 5403 5404 static void 5405 blob_opts_copy(const struct spdk_blob_opts *src, struct spdk_blob_opts *dst) 5406 { 5407 #define FIELD_OK(field) \ 5408 offsetof(struct spdk_blob_opts, field) + sizeof(src->field) <= src->opts_size 5409 5410 #define SET_FIELD(field) \ 5411 if (FIELD_OK(field)) { \ 5412 dst->field = src->field; \ 5413 } \ 5414 5415 SET_FIELD(num_clusters); 5416 SET_FIELD(thin_provision); 5417 SET_FIELD(clear_method); 5418 5419 if (FIELD_OK(xattrs)) { 5420 memcpy(&dst->xattrs, &src->xattrs, sizeof(src->xattrs)); 5421 } 5422 5423 SET_FIELD(use_extent_table); 5424 5425 dst->opts_size = src->opts_size; 5426 5427 /* You should not remove this statement, but need to update the assert statement 5428 * if you add a new field, and also add a corresponding SET_FIELD statement */ 5429 SPDK_STATIC_ASSERT(sizeof(struct spdk_blob_opts) == 64, "Incorrect size"); 5430 5431 #undef FIELD_OK 5432 #undef SET_FIELD 5433 } 5434 5435 static void 5436 bs_create_blob(struct spdk_blob_store *bs, 5437 const struct spdk_blob_opts *opts, 5438 const struct spdk_blob_xattr_opts *internal_xattrs, 5439 spdk_blob_op_with_id_complete cb_fn, void *cb_arg) 5440 { 5441 struct spdk_blob *blob; 5442 uint32_t page_idx; 5443 struct spdk_bs_cpl cpl; 5444 struct spdk_blob_opts opts_local; 5445 struct spdk_blob_xattr_opts internal_xattrs_default; 5446 spdk_bs_sequence_t *seq; 5447 spdk_blob_id id; 5448 int rc; 5449 5450 assert(spdk_get_thread() == bs->md_thread); 5451 5452 page_idx = spdk_bit_array_find_first_clear(bs->used_md_pages, 0); 5453 if (page_idx == UINT32_MAX) { 5454 cb_fn(cb_arg, 0, -ENOMEM); 5455 return; 5456 } 5457 spdk_bit_array_set(bs->used_blobids, page_idx); 5458 bs_claim_md_page(bs, page_idx); 5459 5460 id = bs_page_to_blobid(page_idx); 5461 5462 SPDK_DEBUGLOG(blob, "Creating blob with id %" PRIu64 " at page %u\n", id, page_idx); 5463 5464 blob = blob_alloc(bs, id); 5465 if (!blob) { 5466 spdk_bit_array_clear(bs->used_blobids, page_idx); 5467 bs_release_md_page(bs, page_idx); 5468 cb_fn(cb_arg, 0, -ENOMEM); 5469 return; 5470 } 5471 5472 spdk_blob_opts_init(&opts_local, sizeof(opts_local)); 5473 if (opts) { 5474 blob_opts_copy(opts, &opts_local); 5475 } 5476 5477 blob->use_extent_table = opts_local.use_extent_table; 5478 if (blob->use_extent_table) { 5479 blob->invalid_flags |= SPDK_BLOB_EXTENT_TABLE; 5480 } 5481 5482 if (!internal_xattrs) { 5483 blob_xattrs_init(&internal_xattrs_default); 5484 internal_xattrs = &internal_xattrs_default; 5485 } 5486 5487 rc = blob_set_xattrs(blob, &opts_local.xattrs, false); 5488 if (rc < 0) { 5489 blob_free(blob); 5490 spdk_bit_array_clear(bs->used_blobids, page_idx); 5491 bs_release_md_page(bs, page_idx); 5492 cb_fn(cb_arg, 0, rc); 5493 return; 5494 } 5495 5496 rc = blob_set_xattrs(blob, internal_xattrs, true); 5497 if (rc < 0) { 5498 blob_free(blob); 5499 spdk_bit_array_clear(bs->used_blobids, page_idx); 5500 bs_release_md_page(bs, page_idx); 5501 cb_fn(cb_arg, 0, rc); 5502 return; 5503 } 5504 5505 if (opts_local.thin_provision) { 5506 blob_set_thin_provision(blob); 5507 } 5508 5509 blob_set_clear_method(blob, opts_local.clear_method); 5510 5511 rc = blob_resize(blob, opts_local.num_clusters); 5512 if (rc < 0) { 5513 blob_free(blob); 5514 spdk_bit_array_clear(bs->used_blobids, page_idx); 5515 bs_release_md_page(bs, page_idx); 5516 cb_fn(cb_arg, 0, rc); 5517 return; 5518 } 5519 cpl.type = SPDK_BS_CPL_TYPE_BLOBID; 5520 cpl.u.blobid.cb_fn = cb_fn; 5521 cpl.u.blobid.cb_arg = cb_arg; 5522 cpl.u.blobid.blobid = blob->id; 5523 5524 seq = bs_sequence_start(bs->md_channel, &cpl); 5525 if (!seq) { 5526 blob_free(blob); 5527 spdk_bit_array_clear(bs->used_blobids, page_idx); 5528 bs_release_md_page(bs, page_idx); 5529 cb_fn(cb_arg, 0, -ENOMEM); 5530 return; 5531 } 5532 5533 blob_persist(seq, blob, bs_create_blob_cpl, blob); 5534 } 5535 5536 void spdk_bs_create_blob(struct spdk_blob_store *bs, 5537 spdk_blob_op_with_id_complete cb_fn, void *cb_arg) 5538 { 5539 bs_create_blob(bs, NULL, NULL, cb_fn, cb_arg); 5540 } 5541 5542 void spdk_bs_create_blob_ext(struct spdk_blob_store *bs, const struct spdk_blob_opts *opts, 5543 spdk_blob_op_with_id_complete cb_fn, void *cb_arg) 5544 { 5545 bs_create_blob(bs, opts, NULL, cb_fn, cb_arg); 5546 } 5547 5548 /* END spdk_bs_create_blob */ 5549 5550 /* START blob_cleanup */ 5551 5552 struct spdk_clone_snapshot_ctx { 5553 struct spdk_bs_cpl cpl; 5554 int bserrno; 5555 bool frozen; 5556 5557 struct spdk_io_channel *channel; 5558 5559 /* Current cluster for inflate operation */ 5560 uint64_t cluster; 5561 5562 /* For inflation force allocation of all unallocated clusters and remove 5563 * thin-provisioning. Otherwise only decouple parent and keep clone thin. */ 5564 bool allocate_all; 5565 5566 struct { 5567 spdk_blob_id id; 5568 struct spdk_blob *blob; 5569 bool md_ro; 5570 } original; 5571 struct { 5572 spdk_blob_id id; 5573 struct spdk_blob *blob; 5574 } new; 5575 5576 /* xattrs specified for snapshot/clones only. They have no impact on 5577 * the original blobs xattrs. */ 5578 const struct spdk_blob_xattr_opts *xattrs; 5579 }; 5580 5581 static void 5582 bs_clone_snapshot_cleanup_finish(void *cb_arg, int bserrno) 5583 { 5584 struct spdk_clone_snapshot_ctx *ctx = cb_arg; 5585 struct spdk_bs_cpl *cpl = &ctx->cpl; 5586 5587 if (bserrno != 0) { 5588 if (ctx->bserrno != 0) { 5589 SPDK_ERRLOG("Cleanup error %d\n", bserrno); 5590 } else { 5591 ctx->bserrno = bserrno; 5592 } 5593 } 5594 5595 switch (cpl->type) { 5596 case SPDK_BS_CPL_TYPE_BLOBID: 5597 cpl->u.blobid.cb_fn(cpl->u.blobid.cb_arg, cpl->u.blobid.blobid, ctx->bserrno); 5598 break; 5599 case SPDK_BS_CPL_TYPE_BLOB_BASIC: 5600 cpl->u.blob_basic.cb_fn(cpl->u.blob_basic.cb_arg, ctx->bserrno); 5601 break; 5602 default: 5603 SPDK_UNREACHABLE(); 5604 break; 5605 } 5606 5607 free(ctx); 5608 } 5609 5610 static void 5611 bs_snapshot_unfreeze_cpl(void *cb_arg, int bserrno) 5612 { 5613 struct spdk_clone_snapshot_ctx *ctx = (struct spdk_clone_snapshot_ctx *)cb_arg; 5614 struct spdk_blob *origblob = ctx->original.blob; 5615 5616 if (bserrno != 0) { 5617 if (ctx->bserrno != 0) { 5618 SPDK_ERRLOG("Unfreeze error %d\n", bserrno); 5619 } else { 5620 ctx->bserrno = bserrno; 5621 } 5622 } 5623 5624 ctx->original.id = origblob->id; 5625 origblob->locked_operation_in_progress = false; 5626 5627 /* Revert md_ro to original state */ 5628 origblob->md_ro = ctx->original.md_ro; 5629 5630 spdk_blob_close(origblob, bs_clone_snapshot_cleanup_finish, ctx); 5631 } 5632 5633 static void 5634 bs_clone_snapshot_origblob_cleanup(void *cb_arg, int bserrno) 5635 { 5636 struct spdk_clone_snapshot_ctx *ctx = (struct spdk_clone_snapshot_ctx *)cb_arg; 5637 struct spdk_blob *origblob = ctx->original.blob; 5638 5639 if (bserrno != 0) { 5640 if (ctx->bserrno != 0) { 5641 SPDK_ERRLOG("Cleanup error %d\n", bserrno); 5642 } else { 5643 ctx->bserrno = bserrno; 5644 } 5645 } 5646 5647 if (ctx->frozen) { 5648 /* Unfreeze any outstanding I/O */ 5649 blob_unfreeze_io(origblob, bs_snapshot_unfreeze_cpl, ctx); 5650 } else { 5651 bs_snapshot_unfreeze_cpl(ctx, 0); 5652 } 5653 5654 } 5655 5656 static void 5657 bs_clone_snapshot_newblob_cleanup(struct spdk_clone_snapshot_ctx *ctx, int bserrno) 5658 { 5659 struct spdk_blob *newblob = ctx->new.blob; 5660 5661 if (bserrno != 0) { 5662 if (ctx->bserrno != 0) { 5663 SPDK_ERRLOG("Cleanup error %d\n", bserrno); 5664 } else { 5665 ctx->bserrno = bserrno; 5666 } 5667 } 5668 5669 ctx->new.id = newblob->id; 5670 spdk_blob_close(newblob, bs_clone_snapshot_origblob_cleanup, ctx); 5671 } 5672 5673 /* END blob_cleanup */ 5674 5675 /* START spdk_bs_create_snapshot */ 5676 5677 static void 5678 bs_snapshot_swap_cluster_maps(struct spdk_blob *blob1, struct spdk_blob *blob2) 5679 { 5680 uint64_t *cluster_temp; 5681 uint32_t *extent_page_temp; 5682 5683 cluster_temp = blob1->active.clusters; 5684 blob1->active.clusters = blob2->active.clusters; 5685 blob2->active.clusters = cluster_temp; 5686 5687 extent_page_temp = blob1->active.extent_pages; 5688 blob1->active.extent_pages = blob2->active.extent_pages; 5689 blob2->active.extent_pages = extent_page_temp; 5690 } 5691 5692 static void 5693 bs_snapshot_origblob_sync_cpl(void *cb_arg, int bserrno) 5694 { 5695 struct spdk_clone_snapshot_ctx *ctx = (struct spdk_clone_snapshot_ctx *)cb_arg; 5696 struct spdk_blob *origblob = ctx->original.blob; 5697 struct spdk_blob *newblob = ctx->new.blob; 5698 5699 if (bserrno != 0) { 5700 bs_snapshot_swap_cluster_maps(newblob, origblob); 5701 bs_clone_snapshot_origblob_cleanup(ctx, bserrno); 5702 return; 5703 } 5704 5705 /* Remove metadata descriptor SNAPSHOT_IN_PROGRESS */ 5706 bserrno = blob_remove_xattr(newblob, SNAPSHOT_IN_PROGRESS, true); 5707 if (bserrno != 0) { 5708 bs_clone_snapshot_origblob_cleanup(ctx, bserrno); 5709 return; 5710 } 5711 5712 bs_blob_list_add(ctx->original.blob); 5713 5714 spdk_blob_set_read_only(newblob); 5715 5716 /* sync snapshot metadata */ 5717 spdk_blob_sync_md(newblob, bs_clone_snapshot_origblob_cleanup, ctx); 5718 } 5719 5720 static void 5721 bs_snapshot_newblob_sync_cpl(void *cb_arg, int bserrno) 5722 { 5723 struct spdk_clone_snapshot_ctx *ctx = (struct spdk_clone_snapshot_ctx *)cb_arg; 5724 struct spdk_blob *origblob = ctx->original.blob; 5725 struct spdk_blob *newblob = ctx->new.blob; 5726 5727 if (bserrno != 0) { 5728 /* return cluster map back to original */ 5729 bs_snapshot_swap_cluster_maps(newblob, origblob); 5730 5731 /* Newblob md sync failed. Valid clusters are only present in origblob. 5732 * Since I/O is frozen on origblob, not changes to zeroed out cluster map should have occurred. 5733 * Newblob needs to be reverted to thin_provisioned state at creation to properly close. */ 5734 blob_set_thin_provision(newblob); 5735 assert(spdk_mem_all_zero(newblob->active.clusters, 5736 newblob->active.num_clusters * sizeof(*newblob->active.clusters))); 5737 assert(spdk_mem_all_zero(newblob->active.extent_pages, 5738 newblob->active.num_extent_pages * sizeof(*newblob->active.extent_pages))); 5739 5740 bs_clone_snapshot_newblob_cleanup(ctx, bserrno); 5741 return; 5742 } 5743 5744 /* Set internal xattr for snapshot id */ 5745 bserrno = blob_set_xattr(origblob, BLOB_SNAPSHOT, &newblob->id, sizeof(spdk_blob_id), true); 5746 if (bserrno != 0) { 5747 /* return cluster map back to original */ 5748 bs_snapshot_swap_cluster_maps(newblob, origblob); 5749 blob_set_thin_provision(newblob); 5750 bs_clone_snapshot_newblob_cleanup(ctx, bserrno); 5751 return; 5752 } 5753 5754 /* Create new back_bs_dev for snapshot */ 5755 origblob->back_bs_dev = bs_create_blob_bs_dev(newblob); 5756 if (origblob->back_bs_dev == NULL) { 5757 /* return cluster map back to original */ 5758 bs_snapshot_swap_cluster_maps(newblob, origblob); 5759 blob_set_thin_provision(newblob); 5760 bs_clone_snapshot_newblob_cleanup(ctx, -EINVAL); 5761 return; 5762 } 5763 5764 bs_blob_list_remove(origblob); 5765 origblob->parent_id = newblob->id; 5766 /* set clone blob as thin provisioned */ 5767 blob_set_thin_provision(origblob); 5768 5769 bs_blob_list_add(newblob); 5770 5771 /* sync clone metadata */ 5772 spdk_blob_sync_md(origblob, bs_snapshot_origblob_sync_cpl, ctx); 5773 } 5774 5775 static void 5776 bs_snapshot_freeze_cpl(void *cb_arg, int rc) 5777 { 5778 struct spdk_clone_snapshot_ctx *ctx = (struct spdk_clone_snapshot_ctx *)cb_arg; 5779 struct spdk_blob *origblob = ctx->original.blob; 5780 struct spdk_blob *newblob = ctx->new.blob; 5781 int bserrno; 5782 5783 if (rc != 0) { 5784 bs_clone_snapshot_newblob_cleanup(ctx, rc); 5785 return; 5786 } 5787 5788 ctx->frozen = true; 5789 5790 /* set new back_bs_dev for snapshot */ 5791 newblob->back_bs_dev = origblob->back_bs_dev; 5792 /* Set invalid flags from origblob */ 5793 newblob->invalid_flags = origblob->invalid_flags; 5794 5795 /* inherit parent from original blob if set */ 5796 newblob->parent_id = origblob->parent_id; 5797 if (origblob->parent_id != SPDK_BLOBID_INVALID) { 5798 /* Set internal xattr for snapshot id */ 5799 bserrno = blob_set_xattr(newblob, BLOB_SNAPSHOT, 5800 &origblob->parent_id, sizeof(spdk_blob_id), true); 5801 if (bserrno != 0) { 5802 bs_clone_snapshot_newblob_cleanup(ctx, bserrno); 5803 return; 5804 } 5805 } 5806 5807 /* swap cluster maps */ 5808 bs_snapshot_swap_cluster_maps(newblob, origblob); 5809 5810 /* Set the clear method on the new blob to match the original. */ 5811 blob_set_clear_method(newblob, origblob->clear_method); 5812 5813 /* sync snapshot metadata */ 5814 spdk_blob_sync_md(newblob, bs_snapshot_newblob_sync_cpl, ctx); 5815 } 5816 5817 static void 5818 bs_snapshot_newblob_open_cpl(void *cb_arg, struct spdk_blob *_blob, int bserrno) 5819 { 5820 struct spdk_clone_snapshot_ctx *ctx = (struct spdk_clone_snapshot_ctx *)cb_arg; 5821 struct spdk_blob *origblob = ctx->original.blob; 5822 struct spdk_blob *newblob = _blob; 5823 5824 if (bserrno != 0) { 5825 bs_clone_snapshot_origblob_cleanup(ctx, bserrno); 5826 return; 5827 } 5828 5829 ctx->new.blob = newblob; 5830 assert(spdk_blob_is_thin_provisioned(newblob)); 5831 assert(spdk_mem_all_zero(newblob->active.clusters, 5832 newblob->active.num_clusters * sizeof(*newblob->active.clusters))); 5833 assert(spdk_mem_all_zero(newblob->active.extent_pages, 5834 newblob->active.num_extent_pages * sizeof(*newblob->active.extent_pages))); 5835 5836 blob_freeze_io(origblob, bs_snapshot_freeze_cpl, ctx); 5837 } 5838 5839 static void 5840 bs_snapshot_newblob_create_cpl(void *cb_arg, spdk_blob_id blobid, int bserrno) 5841 { 5842 struct spdk_clone_snapshot_ctx *ctx = (struct spdk_clone_snapshot_ctx *)cb_arg; 5843 struct spdk_blob *origblob = ctx->original.blob; 5844 5845 if (bserrno != 0) { 5846 bs_clone_snapshot_origblob_cleanup(ctx, bserrno); 5847 return; 5848 } 5849 5850 ctx->new.id = blobid; 5851 ctx->cpl.u.blobid.blobid = blobid; 5852 5853 spdk_bs_open_blob(origblob->bs, ctx->new.id, bs_snapshot_newblob_open_cpl, ctx); 5854 } 5855 5856 5857 static void 5858 bs_xattr_snapshot(void *arg, const char *name, 5859 const void **value, size_t *value_len) 5860 { 5861 assert(strncmp(name, SNAPSHOT_IN_PROGRESS, sizeof(SNAPSHOT_IN_PROGRESS)) == 0); 5862 5863 struct spdk_blob *blob = (struct spdk_blob *)arg; 5864 *value = &blob->id; 5865 *value_len = sizeof(blob->id); 5866 } 5867 5868 static void 5869 bs_snapshot_origblob_open_cpl(void *cb_arg, struct spdk_blob *_blob, int bserrno) 5870 { 5871 struct spdk_clone_snapshot_ctx *ctx = (struct spdk_clone_snapshot_ctx *)cb_arg; 5872 struct spdk_blob_opts opts; 5873 struct spdk_blob_xattr_opts internal_xattrs; 5874 char *xattrs_names[] = { SNAPSHOT_IN_PROGRESS }; 5875 5876 if (bserrno != 0) { 5877 bs_clone_snapshot_cleanup_finish(ctx, bserrno); 5878 return; 5879 } 5880 5881 ctx->original.blob = _blob; 5882 5883 if (_blob->data_ro || _blob->md_ro) { 5884 SPDK_DEBUGLOG(blob, "Cannot create snapshot from read only blob with id %" PRIu64 "\n", 5885 _blob->id); 5886 ctx->bserrno = -EINVAL; 5887 spdk_blob_close(_blob, bs_clone_snapshot_cleanup_finish, ctx); 5888 return; 5889 } 5890 5891 if (_blob->locked_operation_in_progress) { 5892 SPDK_DEBUGLOG(blob, "Cannot create snapshot - another operation in progress\n"); 5893 ctx->bserrno = -EBUSY; 5894 spdk_blob_close(_blob, bs_clone_snapshot_cleanup_finish, ctx); 5895 return; 5896 } 5897 5898 _blob->locked_operation_in_progress = true; 5899 5900 spdk_blob_opts_init(&opts, sizeof(opts)); 5901 blob_xattrs_init(&internal_xattrs); 5902 5903 /* Change the size of new blob to the same as in original blob, 5904 * but do not allocate clusters */ 5905 opts.thin_provision = true; 5906 opts.num_clusters = spdk_blob_get_num_clusters(_blob); 5907 opts.use_extent_table = _blob->use_extent_table; 5908 5909 /* If there are any xattrs specified for snapshot, set them now */ 5910 if (ctx->xattrs) { 5911 memcpy(&opts.xattrs, ctx->xattrs, sizeof(*ctx->xattrs)); 5912 } 5913 /* Set internal xattr SNAPSHOT_IN_PROGRESS */ 5914 internal_xattrs.count = 1; 5915 internal_xattrs.ctx = _blob; 5916 internal_xattrs.names = xattrs_names; 5917 internal_xattrs.get_value = bs_xattr_snapshot; 5918 5919 bs_create_blob(_blob->bs, &opts, &internal_xattrs, 5920 bs_snapshot_newblob_create_cpl, ctx); 5921 } 5922 5923 void spdk_bs_create_snapshot(struct spdk_blob_store *bs, spdk_blob_id blobid, 5924 const struct spdk_blob_xattr_opts *snapshot_xattrs, 5925 spdk_blob_op_with_id_complete cb_fn, void *cb_arg) 5926 { 5927 struct spdk_clone_snapshot_ctx *ctx = calloc(1, sizeof(*ctx)); 5928 5929 if (!ctx) { 5930 cb_fn(cb_arg, SPDK_BLOBID_INVALID, -ENOMEM); 5931 return; 5932 } 5933 ctx->cpl.type = SPDK_BS_CPL_TYPE_BLOBID; 5934 ctx->cpl.u.blobid.cb_fn = cb_fn; 5935 ctx->cpl.u.blobid.cb_arg = cb_arg; 5936 ctx->cpl.u.blobid.blobid = SPDK_BLOBID_INVALID; 5937 ctx->bserrno = 0; 5938 ctx->frozen = false; 5939 ctx->original.id = blobid; 5940 ctx->xattrs = snapshot_xattrs; 5941 5942 spdk_bs_open_blob(bs, ctx->original.id, bs_snapshot_origblob_open_cpl, ctx); 5943 } 5944 /* END spdk_bs_create_snapshot */ 5945 5946 /* START spdk_bs_create_clone */ 5947 5948 static void 5949 bs_xattr_clone(void *arg, const char *name, 5950 const void **value, size_t *value_len) 5951 { 5952 assert(strncmp(name, BLOB_SNAPSHOT, sizeof(BLOB_SNAPSHOT)) == 0); 5953 5954 struct spdk_blob *blob = (struct spdk_blob *)arg; 5955 *value = &blob->id; 5956 *value_len = sizeof(blob->id); 5957 } 5958 5959 static void 5960 bs_clone_newblob_open_cpl(void *cb_arg, struct spdk_blob *_blob, int bserrno) 5961 { 5962 struct spdk_clone_snapshot_ctx *ctx = (struct spdk_clone_snapshot_ctx *)cb_arg; 5963 struct spdk_blob *clone = _blob; 5964 5965 ctx->new.blob = clone; 5966 bs_blob_list_add(clone); 5967 5968 spdk_blob_close(clone, bs_clone_snapshot_origblob_cleanup, ctx); 5969 } 5970 5971 static void 5972 bs_clone_newblob_create_cpl(void *cb_arg, spdk_blob_id blobid, int bserrno) 5973 { 5974 struct spdk_clone_snapshot_ctx *ctx = (struct spdk_clone_snapshot_ctx *)cb_arg; 5975 5976 ctx->cpl.u.blobid.blobid = blobid; 5977 spdk_bs_open_blob(ctx->original.blob->bs, blobid, bs_clone_newblob_open_cpl, ctx); 5978 } 5979 5980 static void 5981 bs_clone_origblob_open_cpl(void *cb_arg, struct spdk_blob *_blob, int bserrno) 5982 { 5983 struct spdk_clone_snapshot_ctx *ctx = (struct spdk_clone_snapshot_ctx *)cb_arg; 5984 struct spdk_blob_opts opts; 5985 struct spdk_blob_xattr_opts internal_xattrs; 5986 char *xattr_names[] = { BLOB_SNAPSHOT }; 5987 5988 if (bserrno != 0) { 5989 bs_clone_snapshot_cleanup_finish(ctx, bserrno); 5990 return; 5991 } 5992 5993 ctx->original.blob = _blob; 5994 ctx->original.md_ro = _blob->md_ro; 5995 5996 if (!_blob->data_ro || !_blob->md_ro) { 5997 SPDK_DEBUGLOG(blob, "Clone not from read-only blob\n"); 5998 ctx->bserrno = -EINVAL; 5999 spdk_blob_close(_blob, bs_clone_snapshot_cleanup_finish, ctx); 6000 return; 6001 } 6002 6003 if (_blob->locked_operation_in_progress) { 6004 SPDK_DEBUGLOG(blob, "Cannot create clone - another operation in progress\n"); 6005 ctx->bserrno = -EBUSY; 6006 spdk_blob_close(_blob, bs_clone_snapshot_cleanup_finish, ctx); 6007 return; 6008 } 6009 6010 _blob->locked_operation_in_progress = true; 6011 6012 spdk_blob_opts_init(&opts, sizeof(opts)); 6013 blob_xattrs_init(&internal_xattrs); 6014 6015 opts.thin_provision = true; 6016 opts.num_clusters = spdk_blob_get_num_clusters(_blob); 6017 opts.use_extent_table = _blob->use_extent_table; 6018 if (ctx->xattrs) { 6019 memcpy(&opts.xattrs, ctx->xattrs, sizeof(*ctx->xattrs)); 6020 } 6021 6022 /* Set internal xattr BLOB_SNAPSHOT */ 6023 internal_xattrs.count = 1; 6024 internal_xattrs.ctx = _blob; 6025 internal_xattrs.names = xattr_names; 6026 internal_xattrs.get_value = bs_xattr_clone; 6027 6028 bs_create_blob(_blob->bs, &opts, &internal_xattrs, 6029 bs_clone_newblob_create_cpl, ctx); 6030 } 6031 6032 void spdk_bs_create_clone(struct spdk_blob_store *bs, spdk_blob_id blobid, 6033 const struct spdk_blob_xattr_opts *clone_xattrs, 6034 spdk_blob_op_with_id_complete cb_fn, void *cb_arg) 6035 { 6036 struct spdk_clone_snapshot_ctx *ctx = calloc(1, sizeof(*ctx)); 6037 6038 if (!ctx) { 6039 cb_fn(cb_arg, SPDK_BLOBID_INVALID, -ENOMEM); 6040 return; 6041 } 6042 6043 ctx->cpl.type = SPDK_BS_CPL_TYPE_BLOBID; 6044 ctx->cpl.u.blobid.cb_fn = cb_fn; 6045 ctx->cpl.u.blobid.cb_arg = cb_arg; 6046 ctx->cpl.u.blobid.blobid = SPDK_BLOBID_INVALID; 6047 ctx->bserrno = 0; 6048 ctx->xattrs = clone_xattrs; 6049 ctx->original.id = blobid; 6050 6051 spdk_bs_open_blob(bs, ctx->original.id, bs_clone_origblob_open_cpl, ctx); 6052 } 6053 6054 /* END spdk_bs_create_clone */ 6055 6056 /* START spdk_bs_inflate_blob */ 6057 6058 static void 6059 bs_inflate_blob_set_parent_cpl(void *cb_arg, struct spdk_blob *_parent, int bserrno) 6060 { 6061 struct spdk_clone_snapshot_ctx *ctx = (struct spdk_clone_snapshot_ctx *)cb_arg; 6062 struct spdk_blob *_blob = ctx->original.blob; 6063 6064 if (bserrno != 0) { 6065 bs_clone_snapshot_origblob_cleanup(ctx, bserrno); 6066 return; 6067 } 6068 6069 /* Temporarily override md_ro flag for MD modification */ 6070 _blob->md_ro = false; 6071 6072 bserrno = blob_set_xattr(_blob, BLOB_SNAPSHOT, &_parent->id, sizeof(spdk_blob_id), true); 6073 if (bserrno != 0) { 6074 bs_clone_snapshot_origblob_cleanup(ctx, bserrno); 6075 return; 6076 } 6077 6078 assert(_parent != NULL); 6079 6080 bs_blob_list_remove(_blob); 6081 _blob->parent_id = _parent->id; 6082 6083 _blob->back_bs_dev->destroy(_blob->back_bs_dev); 6084 _blob->back_bs_dev = bs_create_blob_bs_dev(_parent); 6085 bs_blob_list_add(_blob); 6086 6087 spdk_blob_sync_md(_blob, bs_clone_snapshot_origblob_cleanup, ctx); 6088 } 6089 6090 static void 6091 bs_inflate_blob_done(struct spdk_clone_snapshot_ctx *ctx) 6092 { 6093 struct spdk_blob *_blob = ctx->original.blob; 6094 struct spdk_blob *_parent; 6095 6096 if (ctx->allocate_all) { 6097 /* remove thin provisioning */ 6098 bs_blob_list_remove(_blob); 6099 blob_remove_xattr(_blob, BLOB_SNAPSHOT, true); 6100 _blob->invalid_flags = _blob->invalid_flags & ~SPDK_BLOB_THIN_PROV; 6101 _blob->back_bs_dev->destroy(_blob->back_bs_dev); 6102 _blob->back_bs_dev = NULL; 6103 _blob->parent_id = SPDK_BLOBID_INVALID; 6104 } else { 6105 _parent = ((struct spdk_blob_bs_dev *)(_blob->back_bs_dev))->blob; 6106 if (_parent->parent_id != SPDK_BLOBID_INVALID) { 6107 /* We must change the parent of the inflated blob */ 6108 spdk_bs_open_blob(_blob->bs, _parent->parent_id, 6109 bs_inflate_blob_set_parent_cpl, ctx); 6110 return; 6111 } 6112 6113 bs_blob_list_remove(_blob); 6114 blob_remove_xattr(_blob, BLOB_SNAPSHOT, true); 6115 _blob->parent_id = SPDK_BLOBID_INVALID; 6116 _blob->back_bs_dev->destroy(_blob->back_bs_dev); 6117 _blob->back_bs_dev = bs_create_zeroes_dev(); 6118 } 6119 6120 /* Temporarily override md_ro flag for MD modification */ 6121 _blob->md_ro = false; 6122 _blob->state = SPDK_BLOB_STATE_DIRTY; 6123 6124 spdk_blob_sync_md(_blob, bs_clone_snapshot_origblob_cleanup, ctx); 6125 } 6126 6127 /* Check if cluster needs allocation */ 6128 static inline bool 6129 bs_cluster_needs_allocation(struct spdk_blob *blob, uint64_t cluster, bool allocate_all) 6130 { 6131 struct spdk_blob_bs_dev *b; 6132 6133 assert(blob != NULL); 6134 6135 if (blob->active.clusters[cluster] != 0) { 6136 /* Cluster is already allocated */ 6137 return false; 6138 } 6139 6140 if (blob->parent_id == SPDK_BLOBID_INVALID) { 6141 /* Blob have no parent blob */ 6142 return allocate_all; 6143 } 6144 6145 b = (struct spdk_blob_bs_dev *)blob->back_bs_dev; 6146 return (allocate_all || b->blob->active.clusters[cluster] != 0); 6147 } 6148 6149 static void 6150 bs_inflate_blob_touch_next(void *cb_arg, int bserrno) 6151 { 6152 struct spdk_clone_snapshot_ctx *ctx = (struct spdk_clone_snapshot_ctx *)cb_arg; 6153 struct spdk_blob *_blob = ctx->original.blob; 6154 struct spdk_bs_cpl cpl; 6155 spdk_bs_user_op_t *op; 6156 uint64_t offset; 6157 6158 if (bserrno != 0) { 6159 bs_clone_snapshot_origblob_cleanup(ctx, bserrno); 6160 return; 6161 } 6162 6163 for (; ctx->cluster < _blob->active.num_clusters; ctx->cluster++) { 6164 if (bs_cluster_needs_allocation(_blob, ctx->cluster, ctx->allocate_all)) { 6165 break; 6166 } 6167 } 6168 6169 if (ctx->cluster < _blob->active.num_clusters) { 6170 offset = bs_cluster_to_lba(_blob->bs, ctx->cluster); 6171 6172 /* We may safely increment a cluster before copying */ 6173 ctx->cluster++; 6174 6175 /* Use a dummy 0B read as a context for cluster copy */ 6176 cpl.type = SPDK_BS_CPL_TYPE_BLOB_BASIC; 6177 cpl.u.blob_basic.cb_fn = bs_inflate_blob_touch_next; 6178 cpl.u.blob_basic.cb_arg = ctx; 6179 6180 op = bs_user_op_alloc(ctx->channel, &cpl, SPDK_BLOB_READ, _blob, 6181 NULL, 0, offset, 0); 6182 if (!op) { 6183 bs_clone_snapshot_origblob_cleanup(ctx, -ENOMEM); 6184 return; 6185 } 6186 6187 bs_allocate_and_copy_cluster(_blob, ctx->channel, offset, op); 6188 } else { 6189 bs_inflate_blob_done(ctx); 6190 } 6191 } 6192 6193 static void 6194 bs_inflate_blob_open_cpl(void *cb_arg, struct spdk_blob *_blob, int bserrno) 6195 { 6196 struct spdk_clone_snapshot_ctx *ctx = (struct spdk_clone_snapshot_ctx *)cb_arg; 6197 uint64_t clusters_needed; 6198 uint64_t i; 6199 6200 if (bserrno != 0) { 6201 bs_clone_snapshot_cleanup_finish(ctx, bserrno); 6202 return; 6203 } 6204 6205 ctx->original.blob = _blob; 6206 ctx->original.md_ro = _blob->md_ro; 6207 6208 if (_blob->locked_operation_in_progress) { 6209 SPDK_DEBUGLOG(blob, "Cannot inflate blob - another operation in progress\n"); 6210 ctx->bserrno = -EBUSY; 6211 spdk_blob_close(_blob, bs_clone_snapshot_cleanup_finish, ctx); 6212 return; 6213 } 6214 6215 _blob->locked_operation_in_progress = true; 6216 6217 if (!ctx->allocate_all && _blob->parent_id == SPDK_BLOBID_INVALID) { 6218 /* This blob have no parent, so we cannot decouple it. */ 6219 SPDK_ERRLOG("Cannot decouple parent of blob with no parent.\n"); 6220 bs_clone_snapshot_origblob_cleanup(ctx, -EINVAL); 6221 return; 6222 } 6223 6224 if (spdk_blob_is_thin_provisioned(_blob) == false) { 6225 /* This is not thin provisioned blob. No need to inflate. */ 6226 bs_clone_snapshot_origblob_cleanup(ctx, 0); 6227 return; 6228 } 6229 6230 /* Do two passes - one to verify that we can obtain enough clusters 6231 * and another to actually claim them. 6232 */ 6233 clusters_needed = 0; 6234 for (i = 0; i < _blob->active.num_clusters; i++) { 6235 if (bs_cluster_needs_allocation(_blob, i, ctx->allocate_all)) { 6236 clusters_needed++; 6237 } 6238 } 6239 6240 if (clusters_needed > _blob->bs->num_free_clusters) { 6241 /* Not enough free clusters. Cannot satisfy the request. */ 6242 bs_clone_snapshot_origblob_cleanup(ctx, -ENOSPC); 6243 return; 6244 } 6245 6246 ctx->cluster = 0; 6247 bs_inflate_blob_touch_next(ctx, 0); 6248 } 6249 6250 static void 6251 bs_inflate_blob(struct spdk_blob_store *bs, struct spdk_io_channel *channel, 6252 spdk_blob_id blobid, bool allocate_all, spdk_blob_op_complete cb_fn, void *cb_arg) 6253 { 6254 struct spdk_clone_snapshot_ctx *ctx = calloc(1, sizeof(*ctx)); 6255 6256 if (!ctx) { 6257 cb_fn(cb_arg, -ENOMEM); 6258 return; 6259 } 6260 ctx->cpl.type = SPDK_BS_CPL_TYPE_BLOB_BASIC; 6261 ctx->cpl.u.bs_basic.cb_fn = cb_fn; 6262 ctx->cpl.u.bs_basic.cb_arg = cb_arg; 6263 ctx->bserrno = 0; 6264 ctx->original.id = blobid; 6265 ctx->channel = channel; 6266 ctx->allocate_all = allocate_all; 6267 6268 spdk_bs_open_blob(bs, ctx->original.id, bs_inflate_blob_open_cpl, ctx); 6269 } 6270 6271 void 6272 spdk_bs_inflate_blob(struct spdk_blob_store *bs, struct spdk_io_channel *channel, 6273 spdk_blob_id blobid, spdk_blob_op_complete cb_fn, void *cb_arg) 6274 { 6275 bs_inflate_blob(bs, channel, blobid, true, cb_fn, cb_arg); 6276 } 6277 6278 void 6279 spdk_bs_blob_decouple_parent(struct spdk_blob_store *bs, struct spdk_io_channel *channel, 6280 spdk_blob_id blobid, spdk_blob_op_complete cb_fn, void *cb_arg) 6281 { 6282 bs_inflate_blob(bs, channel, blobid, false, cb_fn, cb_arg); 6283 } 6284 /* END spdk_bs_inflate_blob */ 6285 6286 /* START spdk_blob_resize */ 6287 struct spdk_bs_resize_ctx { 6288 spdk_blob_op_complete cb_fn; 6289 void *cb_arg; 6290 struct spdk_blob *blob; 6291 uint64_t sz; 6292 int rc; 6293 }; 6294 6295 static void 6296 bs_resize_unfreeze_cpl(void *cb_arg, int rc) 6297 { 6298 struct spdk_bs_resize_ctx *ctx = (struct spdk_bs_resize_ctx *)cb_arg; 6299 6300 if (rc != 0) { 6301 SPDK_ERRLOG("Unfreeze failed, rc=%d\n", rc); 6302 } 6303 6304 if (ctx->rc != 0) { 6305 SPDK_ERRLOG("Unfreeze failed, ctx->rc=%d\n", ctx->rc); 6306 rc = ctx->rc; 6307 } 6308 6309 ctx->blob->locked_operation_in_progress = false; 6310 6311 ctx->cb_fn(ctx->cb_arg, rc); 6312 free(ctx); 6313 } 6314 6315 static void 6316 bs_resize_freeze_cpl(void *cb_arg, int rc) 6317 { 6318 struct spdk_bs_resize_ctx *ctx = (struct spdk_bs_resize_ctx *)cb_arg; 6319 6320 if (rc != 0) { 6321 ctx->blob->locked_operation_in_progress = false; 6322 ctx->cb_fn(ctx->cb_arg, rc); 6323 free(ctx); 6324 return; 6325 } 6326 6327 ctx->rc = blob_resize(ctx->blob, ctx->sz); 6328 6329 blob_unfreeze_io(ctx->blob, bs_resize_unfreeze_cpl, ctx); 6330 } 6331 6332 void 6333 spdk_blob_resize(struct spdk_blob *blob, uint64_t sz, spdk_blob_op_complete cb_fn, void *cb_arg) 6334 { 6335 struct spdk_bs_resize_ctx *ctx; 6336 6337 blob_verify_md_op(blob); 6338 6339 SPDK_DEBUGLOG(blob, "Resizing blob %" PRIu64 " to %" PRIu64 " clusters\n", blob->id, sz); 6340 6341 if (blob->md_ro) { 6342 cb_fn(cb_arg, -EPERM); 6343 return; 6344 } 6345 6346 if (sz == blob->active.num_clusters) { 6347 cb_fn(cb_arg, 0); 6348 return; 6349 } 6350 6351 if (blob->locked_operation_in_progress) { 6352 cb_fn(cb_arg, -EBUSY); 6353 return; 6354 } 6355 6356 ctx = calloc(1, sizeof(*ctx)); 6357 if (!ctx) { 6358 cb_fn(cb_arg, -ENOMEM); 6359 return; 6360 } 6361 6362 blob->locked_operation_in_progress = true; 6363 ctx->cb_fn = cb_fn; 6364 ctx->cb_arg = cb_arg; 6365 ctx->blob = blob; 6366 ctx->sz = sz; 6367 blob_freeze_io(blob, bs_resize_freeze_cpl, ctx); 6368 } 6369 6370 /* END spdk_blob_resize */ 6371 6372 6373 /* START spdk_bs_delete_blob */ 6374 6375 static void 6376 bs_delete_close_cpl(void *cb_arg, int bserrno) 6377 { 6378 spdk_bs_sequence_t *seq = cb_arg; 6379 6380 bs_sequence_finish(seq, bserrno); 6381 } 6382 6383 static void 6384 bs_delete_persist_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 6385 { 6386 struct spdk_blob *blob = cb_arg; 6387 6388 if (bserrno != 0) { 6389 /* 6390 * We already removed this blob from the blobstore tailq, so 6391 * we need to free it here since this is the last reference 6392 * to it. 6393 */ 6394 blob_free(blob); 6395 bs_delete_close_cpl(seq, bserrno); 6396 return; 6397 } 6398 6399 /* 6400 * This will immediately decrement the ref_count and call 6401 * the completion routine since the metadata state is clean. 6402 * By calling spdk_blob_close, we reduce the number of call 6403 * points into code that touches the blob->open_ref count 6404 * and the blobstore's blob list. 6405 */ 6406 spdk_blob_close(blob, bs_delete_close_cpl, seq); 6407 } 6408 6409 struct delete_snapshot_ctx { 6410 struct spdk_blob_list *parent_snapshot_entry; 6411 struct spdk_blob *snapshot; 6412 bool snapshot_md_ro; 6413 struct spdk_blob *clone; 6414 bool clone_md_ro; 6415 spdk_blob_op_with_handle_complete cb_fn; 6416 void *cb_arg; 6417 int bserrno; 6418 uint32_t next_extent_page; 6419 }; 6420 6421 static void 6422 delete_blob_cleanup_finish(void *cb_arg, int bserrno) 6423 { 6424 struct delete_snapshot_ctx *ctx = cb_arg; 6425 6426 if (bserrno != 0) { 6427 SPDK_ERRLOG("Snapshot cleanup error %d\n", bserrno); 6428 } 6429 6430 assert(ctx != NULL); 6431 6432 if (bserrno != 0 && ctx->bserrno == 0) { 6433 ctx->bserrno = bserrno; 6434 } 6435 6436 ctx->cb_fn(ctx->cb_arg, ctx->snapshot, ctx->bserrno); 6437 free(ctx); 6438 } 6439 6440 static void 6441 delete_snapshot_cleanup_snapshot(void *cb_arg, int bserrno) 6442 { 6443 struct delete_snapshot_ctx *ctx = cb_arg; 6444 6445 if (bserrno != 0) { 6446 ctx->bserrno = bserrno; 6447 SPDK_ERRLOG("Clone cleanup error %d\n", bserrno); 6448 } 6449 6450 if (ctx->bserrno != 0) { 6451 assert(blob_lookup(ctx->snapshot->bs, ctx->snapshot->id) == NULL); 6452 TAILQ_INSERT_HEAD(&ctx->snapshot->bs->blobs, ctx->snapshot, link); 6453 spdk_bit_array_set(ctx->snapshot->bs->open_blobids, ctx->snapshot->id); 6454 } 6455 6456 ctx->snapshot->locked_operation_in_progress = false; 6457 ctx->snapshot->md_ro = ctx->snapshot_md_ro; 6458 6459 spdk_blob_close(ctx->snapshot, delete_blob_cleanup_finish, ctx); 6460 } 6461 6462 static void 6463 delete_snapshot_cleanup_clone(void *cb_arg, int bserrno) 6464 { 6465 struct delete_snapshot_ctx *ctx = cb_arg; 6466 6467 ctx->clone->locked_operation_in_progress = false; 6468 ctx->clone->md_ro = ctx->clone_md_ro; 6469 6470 spdk_blob_close(ctx->clone, delete_snapshot_cleanup_snapshot, ctx); 6471 } 6472 6473 static void 6474 delete_snapshot_unfreeze_cpl(void *cb_arg, int bserrno) 6475 { 6476 struct delete_snapshot_ctx *ctx = cb_arg; 6477 6478 if (bserrno) { 6479 ctx->bserrno = bserrno; 6480 delete_snapshot_cleanup_clone(ctx, 0); 6481 return; 6482 } 6483 6484 ctx->clone->locked_operation_in_progress = false; 6485 spdk_blob_close(ctx->clone, delete_blob_cleanup_finish, ctx); 6486 } 6487 6488 static void 6489 delete_snapshot_sync_snapshot_cpl(void *cb_arg, int bserrno) 6490 { 6491 struct delete_snapshot_ctx *ctx = cb_arg; 6492 struct spdk_blob_list *parent_snapshot_entry = NULL; 6493 struct spdk_blob_list *snapshot_entry = NULL; 6494 struct spdk_blob_list *clone_entry = NULL; 6495 struct spdk_blob_list *snapshot_clone_entry = NULL; 6496 6497 if (bserrno) { 6498 SPDK_ERRLOG("Failed to sync MD on blob\n"); 6499 ctx->bserrno = bserrno; 6500 delete_snapshot_cleanup_clone(ctx, 0); 6501 return; 6502 } 6503 6504 /* Get snapshot entry for the snapshot we want to remove */ 6505 snapshot_entry = bs_get_snapshot_entry(ctx->snapshot->bs, ctx->snapshot->id); 6506 6507 assert(snapshot_entry != NULL); 6508 6509 /* Remove clone entry in this snapshot (at this point there can be only one clone) */ 6510 clone_entry = TAILQ_FIRST(&snapshot_entry->clones); 6511 assert(clone_entry != NULL); 6512 TAILQ_REMOVE(&snapshot_entry->clones, clone_entry, link); 6513 snapshot_entry->clone_count--; 6514 assert(TAILQ_EMPTY(&snapshot_entry->clones)); 6515 6516 if (ctx->snapshot->parent_id != SPDK_BLOBID_INVALID) { 6517 /* This snapshot is at the same time a clone of another snapshot - we need to 6518 * update parent snapshot (remove current clone, add new one inherited from 6519 * the snapshot that is being removed) */ 6520 6521 /* Get snapshot entry for parent snapshot and clone entry within that snapshot for 6522 * snapshot that we are removing */ 6523 blob_get_snapshot_and_clone_entries(ctx->snapshot, &parent_snapshot_entry, 6524 &snapshot_clone_entry); 6525 6526 /* Switch clone entry in parent snapshot */ 6527 TAILQ_INSERT_TAIL(&parent_snapshot_entry->clones, clone_entry, link); 6528 TAILQ_REMOVE(&parent_snapshot_entry->clones, snapshot_clone_entry, link); 6529 free(snapshot_clone_entry); 6530 } else { 6531 /* No parent snapshot - just remove clone entry */ 6532 free(clone_entry); 6533 } 6534 6535 /* Restore md_ro flags */ 6536 ctx->clone->md_ro = ctx->clone_md_ro; 6537 ctx->snapshot->md_ro = ctx->snapshot_md_ro; 6538 6539 blob_unfreeze_io(ctx->clone, delete_snapshot_unfreeze_cpl, ctx); 6540 } 6541 6542 static void 6543 delete_snapshot_sync_clone_cpl(void *cb_arg, int bserrno) 6544 { 6545 struct delete_snapshot_ctx *ctx = cb_arg; 6546 uint64_t i; 6547 6548 ctx->snapshot->md_ro = false; 6549 6550 if (bserrno) { 6551 SPDK_ERRLOG("Failed to sync MD on clone\n"); 6552 ctx->bserrno = bserrno; 6553 6554 /* Restore snapshot to previous state */ 6555 bserrno = blob_remove_xattr(ctx->snapshot, SNAPSHOT_PENDING_REMOVAL, true); 6556 if (bserrno != 0) { 6557 delete_snapshot_cleanup_clone(ctx, bserrno); 6558 return; 6559 } 6560 6561 spdk_blob_sync_md(ctx->snapshot, delete_snapshot_cleanup_clone, ctx); 6562 return; 6563 } 6564 6565 /* Clear cluster map entries for snapshot */ 6566 for (i = 0; i < ctx->snapshot->active.num_clusters && i < ctx->clone->active.num_clusters; i++) { 6567 if (ctx->clone->active.clusters[i] == ctx->snapshot->active.clusters[i]) { 6568 ctx->snapshot->active.clusters[i] = 0; 6569 } 6570 } 6571 for (i = 0; i < ctx->snapshot->active.num_extent_pages && 6572 i < ctx->clone->active.num_extent_pages; i++) { 6573 if (ctx->clone->active.extent_pages[i] == ctx->snapshot->active.extent_pages[i]) { 6574 ctx->snapshot->active.extent_pages[i] = 0; 6575 } 6576 } 6577 6578 blob_set_thin_provision(ctx->snapshot); 6579 ctx->snapshot->state = SPDK_BLOB_STATE_DIRTY; 6580 6581 if (ctx->parent_snapshot_entry != NULL) { 6582 ctx->snapshot->back_bs_dev = NULL; 6583 } 6584 6585 spdk_blob_sync_md(ctx->snapshot, delete_snapshot_sync_snapshot_cpl, ctx); 6586 } 6587 6588 static void 6589 delete_snapshot_update_extent_pages_cpl(struct delete_snapshot_ctx *ctx) 6590 { 6591 /* Delete old backing bs_dev from clone (related to snapshot that will be removed) */ 6592 ctx->clone->back_bs_dev->destroy(ctx->clone->back_bs_dev); 6593 6594 /* Set/remove snapshot xattr and switch parent ID and backing bs_dev on clone... */ 6595 if (ctx->parent_snapshot_entry != NULL) { 6596 /* ...to parent snapshot */ 6597 ctx->clone->parent_id = ctx->parent_snapshot_entry->id; 6598 ctx->clone->back_bs_dev = ctx->snapshot->back_bs_dev; 6599 blob_set_xattr(ctx->clone, BLOB_SNAPSHOT, &ctx->parent_snapshot_entry->id, 6600 sizeof(spdk_blob_id), 6601 true); 6602 } else { 6603 /* ...to blobid invalid and zeroes dev */ 6604 ctx->clone->parent_id = SPDK_BLOBID_INVALID; 6605 ctx->clone->back_bs_dev = bs_create_zeroes_dev(); 6606 blob_remove_xattr(ctx->clone, BLOB_SNAPSHOT, true); 6607 } 6608 6609 spdk_blob_sync_md(ctx->clone, delete_snapshot_sync_clone_cpl, ctx); 6610 } 6611 6612 static void 6613 delete_snapshot_update_extent_pages(void *cb_arg, int bserrno) 6614 { 6615 struct delete_snapshot_ctx *ctx = cb_arg; 6616 uint32_t *extent_page; 6617 uint64_t i; 6618 6619 for (i = ctx->next_extent_page; i < ctx->snapshot->active.num_extent_pages && 6620 i < ctx->clone->active.num_extent_pages; i++) { 6621 if (ctx->snapshot->active.extent_pages[i] == 0) { 6622 /* No extent page to use from snapshot */ 6623 continue; 6624 } 6625 6626 extent_page = &ctx->clone->active.extent_pages[i]; 6627 if (*extent_page == 0) { 6628 /* Copy extent page from snapshot when clone did not have a matching one */ 6629 *extent_page = ctx->snapshot->active.extent_pages[i]; 6630 continue; 6631 } 6632 6633 /* Clone and snapshot both contain partially filled matching extent pages. 6634 * Update the clone extent page in place with cluster map containing the mix of both. */ 6635 ctx->next_extent_page = i + 1; 6636 6637 blob_write_extent_page(ctx->clone, *extent_page, i * SPDK_EXTENTS_PER_EP, 6638 delete_snapshot_update_extent_pages, ctx); 6639 return; 6640 } 6641 delete_snapshot_update_extent_pages_cpl(ctx); 6642 } 6643 6644 static void 6645 delete_snapshot_sync_snapshot_xattr_cpl(void *cb_arg, int bserrno) 6646 { 6647 struct delete_snapshot_ctx *ctx = cb_arg; 6648 uint64_t i; 6649 6650 /* Temporarily override md_ro flag for clone for MD modification */ 6651 ctx->clone_md_ro = ctx->clone->md_ro; 6652 ctx->clone->md_ro = false; 6653 6654 if (bserrno) { 6655 SPDK_ERRLOG("Failed to sync MD with xattr on blob\n"); 6656 ctx->bserrno = bserrno; 6657 delete_snapshot_cleanup_clone(ctx, 0); 6658 return; 6659 } 6660 6661 /* Copy snapshot map to clone map (only unallocated clusters in clone) */ 6662 for (i = 0; i < ctx->snapshot->active.num_clusters && i < ctx->clone->active.num_clusters; i++) { 6663 if (ctx->clone->active.clusters[i] == 0) { 6664 ctx->clone->active.clusters[i] = ctx->snapshot->active.clusters[i]; 6665 } 6666 } 6667 ctx->next_extent_page = 0; 6668 delete_snapshot_update_extent_pages(ctx, 0); 6669 } 6670 6671 static void 6672 delete_snapshot_freeze_io_cb(void *cb_arg, int bserrno) 6673 { 6674 struct delete_snapshot_ctx *ctx = cb_arg; 6675 6676 if (bserrno) { 6677 SPDK_ERRLOG("Failed to freeze I/O on clone\n"); 6678 ctx->bserrno = bserrno; 6679 delete_snapshot_cleanup_clone(ctx, 0); 6680 return; 6681 } 6682 6683 /* Temporarily override md_ro flag for snapshot for MD modification */ 6684 ctx->snapshot_md_ro = ctx->snapshot->md_ro; 6685 ctx->snapshot->md_ro = false; 6686 6687 /* Mark blob as pending for removal for power failure safety, use clone id for recovery */ 6688 ctx->bserrno = blob_set_xattr(ctx->snapshot, SNAPSHOT_PENDING_REMOVAL, &ctx->clone->id, 6689 sizeof(spdk_blob_id), true); 6690 if (ctx->bserrno != 0) { 6691 delete_snapshot_cleanup_clone(ctx, 0); 6692 return; 6693 } 6694 6695 spdk_blob_sync_md(ctx->snapshot, delete_snapshot_sync_snapshot_xattr_cpl, ctx); 6696 } 6697 6698 static void 6699 delete_snapshot_open_clone_cb(void *cb_arg, struct spdk_blob *clone, int bserrno) 6700 { 6701 struct delete_snapshot_ctx *ctx = cb_arg; 6702 6703 if (bserrno) { 6704 SPDK_ERRLOG("Failed to open clone\n"); 6705 ctx->bserrno = bserrno; 6706 delete_snapshot_cleanup_snapshot(ctx, 0); 6707 return; 6708 } 6709 6710 ctx->clone = clone; 6711 6712 if (clone->locked_operation_in_progress) { 6713 SPDK_DEBUGLOG(blob, "Cannot remove blob - another operation in progress on its clone\n"); 6714 ctx->bserrno = -EBUSY; 6715 spdk_blob_close(ctx->clone, delete_snapshot_cleanup_snapshot, ctx); 6716 return; 6717 } 6718 6719 clone->locked_operation_in_progress = true; 6720 6721 blob_freeze_io(clone, delete_snapshot_freeze_io_cb, ctx); 6722 } 6723 6724 static void 6725 update_clone_on_snapshot_deletion(struct spdk_blob *snapshot, struct delete_snapshot_ctx *ctx) 6726 { 6727 struct spdk_blob_list *snapshot_entry = NULL; 6728 struct spdk_blob_list *clone_entry = NULL; 6729 struct spdk_blob_list *snapshot_clone_entry = NULL; 6730 6731 /* Get snapshot entry for the snapshot we want to remove */ 6732 snapshot_entry = bs_get_snapshot_entry(snapshot->bs, snapshot->id); 6733 6734 assert(snapshot_entry != NULL); 6735 6736 /* Get clone of the snapshot (at this point there can be only one clone) */ 6737 clone_entry = TAILQ_FIRST(&snapshot_entry->clones); 6738 assert(snapshot_entry->clone_count == 1); 6739 assert(clone_entry != NULL); 6740 6741 /* Get snapshot entry for parent snapshot and clone entry within that snapshot for 6742 * snapshot that we are removing */ 6743 blob_get_snapshot_and_clone_entries(snapshot, &ctx->parent_snapshot_entry, 6744 &snapshot_clone_entry); 6745 6746 spdk_bs_open_blob(snapshot->bs, clone_entry->id, delete_snapshot_open_clone_cb, ctx); 6747 } 6748 6749 static void 6750 bs_delete_blob_finish(void *cb_arg, struct spdk_blob *blob, int bserrno) 6751 { 6752 spdk_bs_sequence_t *seq = cb_arg; 6753 struct spdk_blob_list *snapshot_entry = NULL; 6754 uint32_t page_num; 6755 6756 if (bserrno) { 6757 SPDK_ERRLOG("Failed to remove blob\n"); 6758 bs_sequence_finish(seq, bserrno); 6759 return; 6760 } 6761 6762 /* Remove snapshot from the list */ 6763 snapshot_entry = bs_get_snapshot_entry(blob->bs, blob->id); 6764 if (snapshot_entry != NULL) { 6765 TAILQ_REMOVE(&blob->bs->snapshots, snapshot_entry, link); 6766 free(snapshot_entry); 6767 } 6768 6769 page_num = bs_blobid_to_page(blob->id); 6770 spdk_bit_array_clear(blob->bs->used_blobids, page_num); 6771 blob->state = SPDK_BLOB_STATE_DIRTY; 6772 blob->active.num_pages = 0; 6773 blob_resize(blob, 0); 6774 6775 blob_persist(seq, blob, bs_delete_persist_cpl, blob); 6776 } 6777 6778 static int 6779 bs_is_blob_deletable(struct spdk_blob *blob, bool *update_clone) 6780 { 6781 struct spdk_blob_list *snapshot_entry = NULL; 6782 struct spdk_blob_list *clone_entry = NULL; 6783 struct spdk_blob *clone = NULL; 6784 bool has_one_clone = false; 6785 6786 /* Check if this is a snapshot with clones */ 6787 snapshot_entry = bs_get_snapshot_entry(blob->bs, blob->id); 6788 if (snapshot_entry != NULL) { 6789 if (snapshot_entry->clone_count > 1) { 6790 SPDK_ERRLOG("Cannot remove snapshot with more than one clone\n"); 6791 return -EBUSY; 6792 } else if (snapshot_entry->clone_count == 1) { 6793 has_one_clone = true; 6794 } 6795 } 6796 6797 /* Check if someone has this blob open (besides this delete context): 6798 * - open_ref = 1 - only this context opened blob, so it is ok to remove it 6799 * - open_ref <= 2 && has_one_clone = true - clone is holding snapshot 6800 * and that is ok, because we will update it accordingly */ 6801 if (blob->open_ref <= 2 && has_one_clone) { 6802 clone_entry = TAILQ_FIRST(&snapshot_entry->clones); 6803 assert(clone_entry != NULL); 6804 clone = blob_lookup(blob->bs, clone_entry->id); 6805 6806 if (blob->open_ref == 2 && clone == NULL) { 6807 /* Clone is closed and someone else opened this blob */ 6808 SPDK_ERRLOG("Cannot remove snapshot because it is open\n"); 6809 return -EBUSY; 6810 } 6811 6812 *update_clone = true; 6813 return 0; 6814 } 6815 6816 if (blob->open_ref > 1) { 6817 SPDK_ERRLOG("Cannot remove snapshot because it is open\n"); 6818 return -EBUSY; 6819 } 6820 6821 assert(has_one_clone == false); 6822 *update_clone = false; 6823 return 0; 6824 } 6825 6826 static void 6827 bs_delete_enomem_close_cpl(void *cb_arg, int bserrno) 6828 { 6829 spdk_bs_sequence_t *seq = cb_arg; 6830 6831 bs_sequence_finish(seq, -ENOMEM); 6832 } 6833 6834 static void 6835 bs_delete_open_cpl(void *cb_arg, struct spdk_blob *blob, int bserrno) 6836 { 6837 spdk_bs_sequence_t *seq = cb_arg; 6838 struct delete_snapshot_ctx *ctx; 6839 bool update_clone = false; 6840 6841 if (bserrno != 0) { 6842 bs_sequence_finish(seq, bserrno); 6843 return; 6844 } 6845 6846 blob_verify_md_op(blob); 6847 6848 ctx = calloc(1, sizeof(*ctx)); 6849 if (ctx == NULL) { 6850 spdk_blob_close(blob, bs_delete_enomem_close_cpl, seq); 6851 return; 6852 } 6853 6854 ctx->snapshot = blob; 6855 ctx->cb_fn = bs_delete_blob_finish; 6856 ctx->cb_arg = seq; 6857 6858 /* Check if blob can be removed and if it is a snapshot with clone on top of it */ 6859 ctx->bserrno = bs_is_blob_deletable(blob, &update_clone); 6860 if (ctx->bserrno) { 6861 spdk_blob_close(blob, delete_blob_cleanup_finish, ctx); 6862 return; 6863 } 6864 6865 if (blob->locked_operation_in_progress) { 6866 SPDK_DEBUGLOG(blob, "Cannot remove blob - another operation in progress\n"); 6867 ctx->bserrno = -EBUSY; 6868 spdk_blob_close(blob, delete_blob_cleanup_finish, ctx); 6869 return; 6870 } 6871 6872 blob->locked_operation_in_progress = true; 6873 6874 /* 6875 * Remove the blob from the blob_store list now, to ensure it does not 6876 * get returned after this point by blob_lookup(). 6877 */ 6878 spdk_bit_array_clear(blob->bs->open_blobids, blob->id); 6879 TAILQ_REMOVE(&blob->bs->blobs, blob, link); 6880 6881 if (update_clone) { 6882 /* This blob is a snapshot with active clone - update clone first */ 6883 update_clone_on_snapshot_deletion(blob, ctx); 6884 } else { 6885 /* This blob does not have any clones - just remove it */ 6886 bs_blob_list_remove(blob); 6887 bs_delete_blob_finish(seq, blob, 0); 6888 free(ctx); 6889 } 6890 } 6891 6892 void 6893 spdk_bs_delete_blob(struct spdk_blob_store *bs, spdk_blob_id blobid, 6894 spdk_blob_op_complete cb_fn, void *cb_arg) 6895 { 6896 struct spdk_bs_cpl cpl; 6897 spdk_bs_sequence_t *seq; 6898 6899 SPDK_DEBUGLOG(blob, "Deleting blob %" PRIu64 "\n", blobid); 6900 6901 assert(spdk_get_thread() == bs->md_thread); 6902 6903 cpl.type = SPDK_BS_CPL_TYPE_BLOB_BASIC; 6904 cpl.u.blob_basic.cb_fn = cb_fn; 6905 cpl.u.blob_basic.cb_arg = cb_arg; 6906 6907 seq = bs_sequence_start(bs->md_channel, &cpl); 6908 if (!seq) { 6909 cb_fn(cb_arg, -ENOMEM); 6910 return; 6911 } 6912 6913 spdk_bs_open_blob(bs, blobid, bs_delete_open_cpl, seq); 6914 } 6915 6916 /* END spdk_bs_delete_blob */ 6917 6918 /* START spdk_bs_open_blob */ 6919 6920 static void 6921 bs_open_blob_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 6922 { 6923 struct spdk_blob *blob = cb_arg; 6924 struct spdk_blob *existing; 6925 6926 if (bserrno != 0) { 6927 blob_free(blob); 6928 seq->cpl.u.blob_handle.blob = NULL; 6929 bs_sequence_finish(seq, bserrno); 6930 return; 6931 } 6932 6933 existing = blob_lookup(blob->bs, blob->id); 6934 if (existing) { 6935 blob_free(blob); 6936 existing->open_ref++; 6937 seq->cpl.u.blob_handle.blob = existing; 6938 bs_sequence_finish(seq, 0); 6939 return; 6940 } 6941 6942 blob->open_ref++; 6943 6944 spdk_bit_array_set(blob->bs->open_blobids, blob->id); 6945 TAILQ_INSERT_HEAD(&blob->bs->blobs, blob, link); 6946 6947 bs_sequence_finish(seq, bserrno); 6948 } 6949 6950 static inline void 6951 blob_open_opts_copy(const struct spdk_blob_open_opts *src, struct spdk_blob_open_opts *dst) 6952 { 6953 #define FIELD_OK(field) \ 6954 offsetof(struct spdk_blob_opts, field) + sizeof(src->field) <= src->opts_size 6955 6956 #define SET_FIELD(field) \ 6957 if (FIELD_OK(field)) { \ 6958 dst->field = src->field; \ 6959 } \ 6960 6961 SET_FIELD(clear_method); 6962 6963 dst->opts_size = src->opts_size; 6964 6965 /* You should not remove this statement, but need to update the assert statement 6966 * if you add a new field, and also add a corresponding SET_FIELD statement */ 6967 SPDK_STATIC_ASSERT(sizeof(struct spdk_blob_open_opts) == 16, "Incorrect size"); 6968 6969 #undef FIELD_OK 6970 #undef SET_FIELD 6971 } 6972 6973 static void 6974 bs_open_blob(struct spdk_blob_store *bs, 6975 spdk_blob_id blobid, 6976 struct spdk_blob_open_opts *opts, 6977 spdk_blob_op_with_handle_complete cb_fn, 6978 void *cb_arg) 6979 { 6980 struct spdk_blob *blob; 6981 struct spdk_bs_cpl cpl; 6982 struct spdk_blob_open_opts opts_local; 6983 spdk_bs_sequence_t *seq; 6984 uint32_t page_num; 6985 6986 SPDK_DEBUGLOG(blob, "Opening blob %" PRIu64 "\n", blobid); 6987 assert(spdk_get_thread() == bs->md_thread); 6988 6989 page_num = bs_blobid_to_page(blobid); 6990 if (spdk_bit_array_get(bs->used_blobids, page_num) == false) { 6991 /* Invalid blobid */ 6992 cb_fn(cb_arg, NULL, -ENOENT); 6993 return; 6994 } 6995 6996 blob = blob_lookup(bs, blobid); 6997 if (blob) { 6998 blob->open_ref++; 6999 cb_fn(cb_arg, blob, 0); 7000 return; 7001 } 7002 7003 blob = blob_alloc(bs, blobid); 7004 if (!blob) { 7005 cb_fn(cb_arg, NULL, -ENOMEM); 7006 return; 7007 } 7008 7009 spdk_blob_open_opts_init(&opts_local, sizeof(opts_local)); 7010 if (opts) { 7011 blob_open_opts_copy(opts, &opts_local); 7012 } 7013 7014 blob->clear_method = opts_local.clear_method; 7015 7016 cpl.type = SPDK_BS_CPL_TYPE_BLOB_HANDLE; 7017 cpl.u.blob_handle.cb_fn = cb_fn; 7018 cpl.u.blob_handle.cb_arg = cb_arg; 7019 cpl.u.blob_handle.blob = blob; 7020 7021 seq = bs_sequence_start(bs->md_channel, &cpl); 7022 if (!seq) { 7023 blob_free(blob); 7024 cb_fn(cb_arg, NULL, -ENOMEM); 7025 return; 7026 } 7027 7028 blob_load(seq, blob, bs_open_blob_cpl, blob); 7029 } 7030 7031 void spdk_bs_open_blob(struct spdk_blob_store *bs, spdk_blob_id blobid, 7032 spdk_blob_op_with_handle_complete cb_fn, void *cb_arg) 7033 { 7034 bs_open_blob(bs, blobid, NULL, cb_fn, cb_arg); 7035 } 7036 7037 void spdk_bs_open_blob_ext(struct spdk_blob_store *bs, spdk_blob_id blobid, 7038 struct spdk_blob_open_opts *opts, spdk_blob_op_with_handle_complete cb_fn, void *cb_arg) 7039 { 7040 bs_open_blob(bs, blobid, opts, cb_fn, cb_arg); 7041 } 7042 7043 /* END spdk_bs_open_blob */ 7044 7045 /* START spdk_blob_set_read_only */ 7046 int spdk_blob_set_read_only(struct spdk_blob *blob) 7047 { 7048 blob_verify_md_op(blob); 7049 7050 blob->data_ro_flags |= SPDK_BLOB_READ_ONLY; 7051 7052 blob->state = SPDK_BLOB_STATE_DIRTY; 7053 return 0; 7054 } 7055 /* END spdk_blob_set_read_only */ 7056 7057 /* START spdk_blob_sync_md */ 7058 7059 static void 7060 blob_sync_md_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 7061 { 7062 struct spdk_blob *blob = cb_arg; 7063 7064 if (bserrno == 0 && (blob->data_ro_flags & SPDK_BLOB_READ_ONLY)) { 7065 blob->data_ro = true; 7066 blob->md_ro = true; 7067 } 7068 7069 bs_sequence_finish(seq, bserrno); 7070 } 7071 7072 static void 7073 blob_sync_md(struct spdk_blob *blob, spdk_blob_op_complete cb_fn, void *cb_arg) 7074 { 7075 struct spdk_bs_cpl cpl; 7076 spdk_bs_sequence_t *seq; 7077 7078 cpl.type = SPDK_BS_CPL_TYPE_BLOB_BASIC; 7079 cpl.u.blob_basic.cb_fn = cb_fn; 7080 cpl.u.blob_basic.cb_arg = cb_arg; 7081 7082 seq = bs_sequence_start(blob->bs->md_channel, &cpl); 7083 if (!seq) { 7084 cb_fn(cb_arg, -ENOMEM); 7085 return; 7086 } 7087 7088 blob_persist(seq, blob, blob_sync_md_cpl, blob); 7089 } 7090 7091 void 7092 spdk_blob_sync_md(struct spdk_blob *blob, spdk_blob_op_complete cb_fn, void *cb_arg) 7093 { 7094 blob_verify_md_op(blob); 7095 7096 SPDK_DEBUGLOG(blob, "Syncing blob %" PRIu64 "\n", blob->id); 7097 7098 if (blob->md_ro) { 7099 assert(blob->state == SPDK_BLOB_STATE_CLEAN); 7100 cb_fn(cb_arg, 0); 7101 return; 7102 } 7103 7104 blob_sync_md(blob, cb_fn, cb_arg); 7105 } 7106 7107 /* END spdk_blob_sync_md */ 7108 7109 struct spdk_blob_insert_cluster_ctx { 7110 struct spdk_thread *thread; 7111 struct spdk_blob *blob; 7112 uint32_t cluster_num; /* cluster index in blob */ 7113 uint32_t cluster; /* cluster on disk */ 7114 uint32_t extent_page; /* extent page on disk */ 7115 int rc; 7116 spdk_blob_op_complete cb_fn; 7117 void *cb_arg; 7118 }; 7119 7120 static void 7121 blob_insert_cluster_msg_cpl(void *arg) 7122 { 7123 struct spdk_blob_insert_cluster_ctx *ctx = arg; 7124 7125 ctx->cb_fn(ctx->cb_arg, ctx->rc); 7126 free(ctx); 7127 } 7128 7129 static void 7130 blob_insert_cluster_msg_cb(void *arg, int bserrno) 7131 { 7132 struct spdk_blob_insert_cluster_ctx *ctx = arg; 7133 7134 ctx->rc = bserrno; 7135 spdk_thread_send_msg(ctx->thread, blob_insert_cluster_msg_cpl, ctx); 7136 } 7137 7138 static void 7139 blob_insert_new_ep_cb(void *arg, int bserrno) 7140 { 7141 struct spdk_blob_insert_cluster_ctx *ctx = arg; 7142 uint32_t *extent_page; 7143 7144 extent_page = bs_cluster_to_extent_page(ctx->blob, ctx->cluster_num); 7145 *extent_page = ctx->extent_page; 7146 ctx->blob->state = SPDK_BLOB_STATE_DIRTY; 7147 blob_sync_md(ctx->blob, blob_insert_cluster_msg_cb, ctx); 7148 } 7149 7150 static void 7151 blob_persist_extent_page_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 7152 { 7153 struct spdk_blob_md_page *page = cb_arg; 7154 7155 bs_sequence_finish(seq, bserrno); 7156 spdk_free(page); 7157 } 7158 7159 static void 7160 blob_write_extent_page(struct spdk_blob *blob, uint32_t extent, uint64_t cluster_num, 7161 spdk_blob_op_complete cb_fn, void *cb_arg) 7162 { 7163 spdk_bs_sequence_t *seq; 7164 struct spdk_bs_cpl cpl; 7165 struct spdk_blob_md_page *page = NULL; 7166 uint32_t page_count = 0; 7167 int rc; 7168 7169 cpl.type = SPDK_BS_CPL_TYPE_BLOB_BASIC; 7170 cpl.u.blob_basic.cb_fn = cb_fn; 7171 cpl.u.blob_basic.cb_arg = cb_arg; 7172 7173 seq = bs_sequence_start(blob->bs->md_channel, &cpl); 7174 if (!seq) { 7175 cb_fn(cb_arg, -ENOMEM); 7176 return; 7177 } 7178 rc = blob_serialize_add_page(blob, &page, &page_count, &page); 7179 if (rc < 0) { 7180 bs_sequence_finish(seq, rc); 7181 return; 7182 } 7183 7184 blob_serialize_extent_page(blob, cluster_num, page); 7185 7186 page->crc = blob_md_page_calc_crc(page); 7187 7188 assert(spdk_bit_array_get(blob->bs->used_md_pages, extent) == true); 7189 7190 bs_sequence_write_dev(seq, page, bs_md_page_to_lba(blob->bs, extent), 7191 bs_byte_to_lba(blob->bs, SPDK_BS_PAGE_SIZE), 7192 blob_persist_extent_page_cpl, page); 7193 } 7194 7195 static void 7196 blob_insert_cluster_msg(void *arg) 7197 { 7198 struct spdk_blob_insert_cluster_ctx *ctx = arg; 7199 uint32_t *extent_page; 7200 7201 ctx->rc = blob_insert_cluster(ctx->blob, ctx->cluster_num, ctx->cluster); 7202 if (ctx->rc != 0) { 7203 spdk_thread_send_msg(ctx->thread, blob_insert_cluster_msg_cpl, ctx); 7204 return; 7205 } 7206 7207 if (ctx->blob->use_extent_table == false) { 7208 /* Extent table is not used, proceed with sync of md that will only use extents_rle. */ 7209 ctx->blob->state = SPDK_BLOB_STATE_DIRTY; 7210 blob_sync_md(ctx->blob, blob_insert_cluster_msg_cb, ctx); 7211 return; 7212 } 7213 7214 extent_page = bs_cluster_to_extent_page(ctx->blob, ctx->cluster_num); 7215 if (*extent_page == 0) { 7216 /* Extent page requires allocation. 7217 * It was already claimed in the used_md_pages map and placed in ctx. */ 7218 assert(ctx->extent_page != 0); 7219 assert(spdk_bit_array_get(ctx->blob->bs->used_md_pages, ctx->extent_page) == true); 7220 blob_write_extent_page(ctx->blob, ctx->extent_page, ctx->cluster_num, 7221 blob_insert_new_ep_cb, ctx); 7222 } else { 7223 /* It is possible for original thread to allocate extent page for 7224 * different cluster in the same extent page. In such case proceed with 7225 * updating the existing extent page, but release the additional one. */ 7226 if (ctx->extent_page != 0) { 7227 assert(spdk_bit_array_get(ctx->blob->bs->used_md_pages, ctx->extent_page) == true); 7228 bs_release_md_page(ctx->blob->bs, ctx->extent_page); 7229 ctx->extent_page = 0; 7230 } 7231 /* Extent page already allocated. 7232 * Every cluster allocation, requires just an update of single extent page. */ 7233 blob_write_extent_page(ctx->blob, *extent_page, ctx->cluster_num, 7234 blob_insert_cluster_msg_cb, ctx); 7235 } 7236 } 7237 7238 static void 7239 blob_insert_cluster_on_md_thread(struct spdk_blob *blob, uint32_t cluster_num, 7240 uint64_t cluster, uint32_t extent_page, spdk_blob_op_complete cb_fn, void *cb_arg) 7241 { 7242 struct spdk_blob_insert_cluster_ctx *ctx; 7243 7244 ctx = calloc(1, sizeof(*ctx)); 7245 if (ctx == NULL) { 7246 cb_fn(cb_arg, -ENOMEM); 7247 return; 7248 } 7249 7250 ctx->thread = spdk_get_thread(); 7251 ctx->blob = blob; 7252 ctx->cluster_num = cluster_num; 7253 ctx->cluster = cluster; 7254 ctx->extent_page = extent_page; 7255 ctx->cb_fn = cb_fn; 7256 ctx->cb_arg = cb_arg; 7257 7258 spdk_thread_send_msg(blob->bs->md_thread, blob_insert_cluster_msg, ctx); 7259 } 7260 7261 /* START spdk_blob_close */ 7262 7263 static void 7264 blob_close_cpl(spdk_bs_sequence_t *seq, void *cb_arg, int bserrno) 7265 { 7266 struct spdk_blob *blob = cb_arg; 7267 7268 if (bserrno == 0) { 7269 blob->open_ref--; 7270 if (blob->open_ref == 0) { 7271 /* 7272 * Blobs with active.num_pages == 0 are deleted blobs. 7273 * these blobs are removed from the blob_store list 7274 * when the deletion process starts - so don't try to 7275 * remove them again. 7276 */ 7277 if (blob->active.num_pages > 0) { 7278 spdk_bit_array_clear(blob->bs->open_blobids, blob->id); 7279 TAILQ_REMOVE(&blob->bs->blobs, blob, link); 7280 } 7281 blob_free(blob); 7282 } 7283 } 7284 7285 bs_sequence_finish(seq, bserrno); 7286 } 7287 7288 void spdk_blob_close(struct spdk_blob *blob, spdk_blob_op_complete cb_fn, void *cb_arg) 7289 { 7290 struct spdk_bs_cpl cpl; 7291 spdk_bs_sequence_t *seq; 7292 7293 blob_verify_md_op(blob); 7294 7295 SPDK_DEBUGLOG(blob, "Closing blob %" PRIu64 "\n", blob->id); 7296 7297 if (blob->open_ref == 0) { 7298 cb_fn(cb_arg, -EBADF); 7299 return; 7300 } 7301 7302 cpl.type = SPDK_BS_CPL_TYPE_BLOB_BASIC; 7303 cpl.u.blob_basic.cb_fn = cb_fn; 7304 cpl.u.blob_basic.cb_arg = cb_arg; 7305 7306 seq = bs_sequence_start(blob->bs->md_channel, &cpl); 7307 if (!seq) { 7308 cb_fn(cb_arg, -ENOMEM); 7309 return; 7310 } 7311 7312 /* Sync metadata */ 7313 blob_persist(seq, blob, blob_close_cpl, blob); 7314 } 7315 7316 /* END spdk_blob_close */ 7317 7318 struct spdk_io_channel *spdk_bs_alloc_io_channel(struct spdk_blob_store *bs) 7319 { 7320 return spdk_get_io_channel(bs); 7321 } 7322 7323 void spdk_bs_free_io_channel(struct spdk_io_channel *channel) 7324 { 7325 spdk_put_io_channel(channel); 7326 } 7327 7328 void spdk_blob_io_unmap(struct spdk_blob *blob, struct spdk_io_channel *channel, 7329 uint64_t offset, uint64_t length, spdk_blob_op_complete cb_fn, void *cb_arg) 7330 { 7331 blob_request_submit_op(blob, channel, NULL, offset, length, cb_fn, cb_arg, 7332 SPDK_BLOB_UNMAP); 7333 } 7334 7335 void spdk_blob_io_write_zeroes(struct spdk_blob *blob, struct spdk_io_channel *channel, 7336 uint64_t offset, uint64_t length, spdk_blob_op_complete cb_fn, void *cb_arg) 7337 { 7338 blob_request_submit_op(blob, channel, NULL, offset, length, cb_fn, cb_arg, 7339 SPDK_BLOB_WRITE_ZEROES); 7340 } 7341 7342 void spdk_blob_io_write(struct spdk_blob *blob, struct spdk_io_channel *channel, 7343 void *payload, uint64_t offset, uint64_t length, 7344 spdk_blob_op_complete cb_fn, void *cb_arg) 7345 { 7346 blob_request_submit_op(blob, channel, payload, offset, length, cb_fn, cb_arg, 7347 SPDK_BLOB_WRITE); 7348 } 7349 7350 void spdk_blob_io_read(struct spdk_blob *blob, struct spdk_io_channel *channel, 7351 void *payload, uint64_t offset, uint64_t length, 7352 spdk_blob_op_complete cb_fn, void *cb_arg) 7353 { 7354 blob_request_submit_op(blob, channel, payload, offset, length, cb_fn, cb_arg, 7355 SPDK_BLOB_READ); 7356 } 7357 7358 void spdk_blob_io_writev(struct spdk_blob *blob, struct spdk_io_channel *channel, 7359 struct iovec *iov, int iovcnt, uint64_t offset, uint64_t length, 7360 spdk_blob_op_complete cb_fn, void *cb_arg) 7361 { 7362 blob_request_submit_rw_iov(blob, channel, iov, iovcnt, offset, length, cb_fn, cb_arg, false); 7363 } 7364 7365 void spdk_blob_io_readv(struct spdk_blob *blob, struct spdk_io_channel *channel, 7366 struct iovec *iov, int iovcnt, uint64_t offset, uint64_t length, 7367 spdk_blob_op_complete cb_fn, void *cb_arg) 7368 { 7369 blob_request_submit_rw_iov(blob, channel, iov, iovcnt, offset, length, cb_fn, cb_arg, true); 7370 } 7371 7372 struct spdk_bs_iter_ctx { 7373 int64_t page_num; 7374 struct spdk_blob_store *bs; 7375 7376 spdk_blob_op_with_handle_complete cb_fn; 7377 void *cb_arg; 7378 }; 7379 7380 static void 7381 bs_iter_cpl(void *cb_arg, struct spdk_blob *_blob, int bserrno) 7382 { 7383 struct spdk_bs_iter_ctx *ctx = cb_arg; 7384 struct spdk_blob_store *bs = ctx->bs; 7385 spdk_blob_id id; 7386 7387 if (bserrno == 0) { 7388 ctx->cb_fn(ctx->cb_arg, _blob, bserrno); 7389 free(ctx); 7390 return; 7391 } 7392 7393 ctx->page_num++; 7394 ctx->page_num = spdk_bit_array_find_first_set(bs->used_blobids, ctx->page_num); 7395 if (ctx->page_num >= spdk_bit_array_capacity(bs->used_blobids)) { 7396 ctx->cb_fn(ctx->cb_arg, NULL, -ENOENT); 7397 free(ctx); 7398 return; 7399 } 7400 7401 id = bs_page_to_blobid(ctx->page_num); 7402 7403 spdk_bs_open_blob(bs, id, bs_iter_cpl, ctx); 7404 } 7405 7406 void 7407 spdk_bs_iter_first(struct spdk_blob_store *bs, 7408 spdk_blob_op_with_handle_complete cb_fn, void *cb_arg) 7409 { 7410 struct spdk_bs_iter_ctx *ctx; 7411 7412 ctx = calloc(1, sizeof(*ctx)); 7413 if (!ctx) { 7414 cb_fn(cb_arg, NULL, -ENOMEM); 7415 return; 7416 } 7417 7418 ctx->page_num = -1; 7419 ctx->bs = bs; 7420 ctx->cb_fn = cb_fn; 7421 ctx->cb_arg = cb_arg; 7422 7423 bs_iter_cpl(ctx, NULL, -1); 7424 } 7425 7426 static void 7427 bs_iter_close_cpl(void *cb_arg, int bserrno) 7428 { 7429 struct spdk_bs_iter_ctx *ctx = cb_arg; 7430 7431 bs_iter_cpl(ctx, NULL, -1); 7432 } 7433 7434 void 7435 spdk_bs_iter_next(struct spdk_blob_store *bs, struct spdk_blob *blob, 7436 spdk_blob_op_with_handle_complete cb_fn, void *cb_arg) 7437 { 7438 struct spdk_bs_iter_ctx *ctx; 7439 7440 assert(blob != NULL); 7441 7442 ctx = calloc(1, sizeof(*ctx)); 7443 if (!ctx) { 7444 cb_fn(cb_arg, NULL, -ENOMEM); 7445 return; 7446 } 7447 7448 ctx->page_num = bs_blobid_to_page(blob->id); 7449 ctx->bs = bs; 7450 ctx->cb_fn = cb_fn; 7451 ctx->cb_arg = cb_arg; 7452 7453 /* Close the existing blob */ 7454 spdk_blob_close(blob, bs_iter_close_cpl, ctx); 7455 } 7456 7457 static int 7458 blob_set_xattr(struct spdk_blob *blob, const char *name, const void *value, 7459 uint16_t value_len, bool internal) 7460 { 7461 struct spdk_xattr_tailq *xattrs; 7462 struct spdk_xattr *xattr; 7463 size_t desc_size; 7464 void *tmp; 7465 7466 blob_verify_md_op(blob); 7467 7468 if (blob->md_ro) { 7469 return -EPERM; 7470 } 7471 7472 desc_size = sizeof(struct spdk_blob_md_descriptor_xattr) + strlen(name) + value_len; 7473 if (desc_size > SPDK_BS_MAX_DESC_SIZE) { 7474 SPDK_DEBUGLOG(blob, "Xattr '%s' of size %zu does not fix into single page %zu\n", name, 7475 desc_size, SPDK_BS_MAX_DESC_SIZE); 7476 return -ENOMEM; 7477 } 7478 7479 if (internal) { 7480 xattrs = &blob->xattrs_internal; 7481 blob->invalid_flags |= SPDK_BLOB_INTERNAL_XATTR; 7482 } else { 7483 xattrs = &blob->xattrs; 7484 } 7485 7486 TAILQ_FOREACH(xattr, xattrs, link) { 7487 if (!strcmp(name, xattr->name)) { 7488 tmp = malloc(value_len); 7489 if (!tmp) { 7490 return -ENOMEM; 7491 } 7492 7493 free(xattr->value); 7494 xattr->value_len = value_len; 7495 xattr->value = tmp; 7496 memcpy(xattr->value, value, value_len); 7497 7498 blob->state = SPDK_BLOB_STATE_DIRTY; 7499 7500 return 0; 7501 } 7502 } 7503 7504 xattr = calloc(1, sizeof(*xattr)); 7505 if (!xattr) { 7506 return -ENOMEM; 7507 } 7508 7509 xattr->name = strdup(name); 7510 if (!xattr->name) { 7511 free(xattr); 7512 return -ENOMEM; 7513 } 7514 7515 xattr->value_len = value_len; 7516 xattr->value = malloc(value_len); 7517 if (!xattr->value) { 7518 free(xattr->name); 7519 free(xattr); 7520 return -ENOMEM; 7521 } 7522 memcpy(xattr->value, value, value_len); 7523 TAILQ_INSERT_TAIL(xattrs, xattr, link); 7524 7525 blob->state = SPDK_BLOB_STATE_DIRTY; 7526 7527 return 0; 7528 } 7529 7530 int 7531 spdk_blob_set_xattr(struct spdk_blob *blob, const char *name, const void *value, 7532 uint16_t value_len) 7533 { 7534 return blob_set_xattr(blob, name, value, value_len, false); 7535 } 7536 7537 static int 7538 blob_remove_xattr(struct spdk_blob *blob, const char *name, bool internal) 7539 { 7540 struct spdk_xattr_tailq *xattrs; 7541 struct spdk_xattr *xattr; 7542 7543 blob_verify_md_op(blob); 7544 7545 if (blob->md_ro) { 7546 return -EPERM; 7547 } 7548 xattrs = internal ? &blob->xattrs_internal : &blob->xattrs; 7549 7550 TAILQ_FOREACH(xattr, xattrs, link) { 7551 if (!strcmp(name, xattr->name)) { 7552 TAILQ_REMOVE(xattrs, xattr, link); 7553 free(xattr->value); 7554 free(xattr->name); 7555 free(xattr); 7556 7557 if (internal && TAILQ_EMPTY(&blob->xattrs_internal)) { 7558 blob->invalid_flags &= ~SPDK_BLOB_INTERNAL_XATTR; 7559 } 7560 blob->state = SPDK_BLOB_STATE_DIRTY; 7561 7562 return 0; 7563 } 7564 } 7565 7566 return -ENOENT; 7567 } 7568 7569 int 7570 spdk_blob_remove_xattr(struct spdk_blob *blob, const char *name) 7571 { 7572 return blob_remove_xattr(blob, name, false); 7573 } 7574 7575 static int 7576 blob_get_xattr_value(struct spdk_blob *blob, const char *name, 7577 const void **value, size_t *value_len, bool internal) 7578 { 7579 struct spdk_xattr *xattr; 7580 struct spdk_xattr_tailq *xattrs; 7581 7582 xattrs = internal ? &blob->xattrs_internal : &blob->xattrs; 7583 7584 TAILQ_FOREACH(xattr, xattrs, link) { 7585 if (!strcmp(name, xattr->name)) { 7586 *value = xattr->value; 7587 *value_len = xattr->value_len; 7588 return 0; 7589 } 7590 } 7591 return -ENOENT; 7592 } 7593 7594 int 7595 spdk_blob_get_xattr_value(struct spdk_blob *blob, const char *name, 7596 const void **value, size_t *value_len) 7597 { 7598 blob_verify_md_op(blob); 7599 7600 return blob_get_xattr_value(blob, name, value, value_len, false); 7601 } 7602 7603 struct spdk_xattr_names { 7604 uint32_t count; 7605 const char *names[0]; 7606 }; 7607 7608 static int 7609 blob_get_xattr_names(struct spdk_xattr_tailq *xattrs, struct spdk_xattr_names **names) 7610 { 7611 struct spdk_xattr *xattr; 7612 int count = 0; 7613 7614 TAILQ_FOREACH(xattr, xattrs, link) { 7615 count++; 7616 } 7617 7618 *names = calloc(1, sizeof(struct spdk_xattr_names) + count * sizeof(char *)); 7619 if (*names == NULL) { 7620 return -ENOMEM; 7621 } 7622 7623 TAILQ_FOREACH(xattr, xattrs, link) { 7624 (*names)->names[(*names)->count++] = xattr->name; 7625 } 7626 7627 return 0; 7628 } 7629 7630 int 7631 spdk_blob_get_xattr_names(struct spdk_blob *blob, struct spdk_xattr_names **names) 7632 { 7633 blob_verify_md_op(blob); 7634 7635 return blob_get_xattr_names(&blob->xattrs, names); 7636 } 7637 7638 uint32_t 7639 spdk_xattr_names_get_count(struct spdk_xattr_names *names) 7640 { 7641 assert(names != NULL); 7642 7643 return names->count; 7644 } 7645 7646 const char * 7647 spdk_xattr_names_get_name(struct spdk_xattr_names *names, uint32_t index) 7648 { 7649 if (index >= names->count) { 7650 return NULL; 7651 } 7652 7653 return names->names[index]; 7654 } 7655 7656 void 7657 spdk_xattr_names_free(struct spdk_xattr_names *names) 7658 { 7659 free(names); 7660 } 7661 7662 struct spdk_bs_type 7663 spdk_bs_get_bstype(struct spdk_blob_store *bs) 7664 { 7665 return bs->bstype; 7666 } 7667 7668 void 7669 spdk_bs_set_bstype(struct spdk_blob_store *bs, struct spdk_bs_type bstype) 7670 { 7671 memcpy(&bs->bstype, &bstype, sizeof(bstype)); 7672 } 7673 7674 bool 7675 spdk_blob_is_read_only(struct spdk_blob *blob) 7676 { 7677 assert(blob != NULL); 7678 return (blob->data_ro || blob->md_ro); 7679 } 7680 7681 bool 7682 spdk_blob_is_snapshot(struct spdk_blob *blob) 7683 { 7684 struct spdk_blob_list *snapshot_entry; 7685 7686 assert(blob != NULL); 7687 7688 snapshot_entry = bs_get_snapshot_entry(blob->bs, blob->id); 7689 if (snapshot_entry == NULL) { 7690 return false; 7691 } 7692 7693 return true; 7694 } 7695 7696 bool 7697 spdk_blob_is_clone(struct spdk_blob *blob) 7698 { 7699 assert(blob != NULL); 7700 7701 if (blob->parent_id != SPDK_BLOBID_INVALID) { 7702 assert(spdk_blob_is_thin_provisioned(blob)); 7703 return true; 7704 } 7705 7706 return false; 7707 } 7708 7709 bool 7710 spdk_blob_is_thin_provisioned(struct spdk_blob *blob) 7711 { 7712 assert(blob != NULL); 7713 return !!(blob->invalid_flags & SPDK_BLOB_THIN_PROV); 7714 } 7715 7716 static void 7717 blob_update_clear_method(struct spdk_blob *blob) 7718 { 7719 enum blob_clear_method stored_cm; 7720 7721 assert(blob != NULL); 7722 7723 /* If BLOB_CLEAR_WITH_DEFAULT was passed in, use the setting stored 7724 * in metadata previously. If something other than the default was 7725 * specified, ignore stored value and used what was passed in. 7726 */ 7727 stored_cm = ((blob->md_ro_flags & SPDK_BLOB_CLEAR_METHOD) >> SPDK_BLOB_CLEAR_METHOD_SHIFT); 7728 7729 if (blob->clear_method == BLOB_CLEAR_WITH_DEFAULT) { 7730 blob->clear_method = stored_cm; 7731 } else if (blob->clear_method != stored_cm) { 7732 SPDK_WARNLOG("Using passed in clear method 0x%x instead of stored value of 0x%x\n", 7733 blob->clear_method, stored_cm); 7734 } 7735 } 7736 7737 spdk_blob_id 7738 spdk_blob_get_parent_snapshot(struct spdk_blob_store *bs, spdk_blob_id blob_id) 7739 { 7740 struct spdk_blob_list *snapshot_entry = NULL; 7741 struct spdk_blob_list *clone_entry = NULL; 7742 7743 TAILQ_FOREACH(snapshot_entry, &bs->snapshots, link) { 7744 TAILQ_FOREACH(clone_entry, &snapshot_entry->clones, link) { 7745 if (clone_entry->id == blob_id) { 7746 return snapshot_entry->id; 7747 } 7748 } 7749 } 7750 7751 return SPDK_BLOBID_INVALID; 7752 } 7753 7754 int 7755 spdk_blob_get_clones(struct spdk_blob_store *bs, spdk_blob_id blobid, spdk_blob_id *ids, 7756 size_t *count) 7757 { 7758 struct spdk_blob_list *snapshot_entry, *clone_entry; 7759 size_t n; 7760 7761 snapshot_entry = bs_get_snapshot_entry(bs, blobid); 7762 if (snapshot_entry == NULL) { 7763 *count = 0; 7764 return 0; 7765 } 7766 7767 if (ids == NULL || *count < snapshot_entry->clone_count) { 7768 *count = snapshot_entry->clone_count; 7769 return -ENOMEM; 7770 } 7771 *count = snapshot_entry->clone_count; 7772 7773 n = 0; 7774 TAILQ_FOREACH(clone_entry, &snapshot_entry->clones, link) { 7775 ids[n++] = clone_entry->id; 7776 } 7777 7778 return 0; 7779 } 7780 7781 SPDK_LOG_REGISTER_COMPONENT(blob) 7782