1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or https://opensource.org/licenses/CDDL-1.0. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. 23 * Copyright (c) 2011, 2022 by Delphix. All rights reserved. 24 * Copyright (c) 2011 Nexenta Systems, Inc. All rights reserved. 25 * Copyright (c) 2017, Intel Corporation. 26 * Copyright (c) 2019, Klara Inc. 27 * Copyright (c) 2019, Allan Jude 28 * Copyright (c) 2021, Datto, Inc. 29 */ 30 31 #include <sys/sysmacros.h> 32 #include <sys/zfs_context.h> 33 #include <sys/fm/fs/zfs.h> 34 #include <sys/spa.h> 35 #include <sys/txg.h> 36 #include <sys/spa_impl.h> 37 #include <sys/vdev_impl.h> 38 #include <sys/vdev_trim.h> 39 #include <sys/zio_impl.h> 40 #include <sys/zio_compress.h> 41 #include <sys/zio_checksum.h> 42 #include <sys/dmu_objset.h> 43 #include <sys/arc.h> 44 #include <sys/brt.h> 45 #include <sys/ddt.h> 46 #include <sys/blkptr.h> 47 #include <sys/zfeature.h> 48 #include <sys/dsl_scan.h> 49 #include <sys/metaslab_impl.h> 50 #include <sys/time.h> 51 #include <sys/trace_zfs.h> 52 #include <sys/abd.h> 53 #include <sys/dsl_crypt.h> 54 #include <cityhash.h> 55 56 /* 57 * ========================================================================== 58 * I/O type descriptions 59 * ========================================================================== 60 */ 61 const char *const zio_type_name[ZIO_TYPES] = { 62 /* 63 * Note: Linux kernel thread name length is limited 64 * so these names will differ from upstream open zfs. 65 */ 66 "z_null", "z_rd", "z_wr", "z_fr", "z_cl", "z_ioctl", "z_trim" 67 }; 68 69 int zio_dva_throttle_enabled = B_TRUE; 70 static int zio_deadman_log_all = B_FALSE; 71 72 /* 73 * ========================================================================== 74 * I/O kmem caches 75 * ========================================================================== 76 */ 77 static kmem_cache_t *zio_cache; 78 static kmem_cache_t *zio_link_cache; 79 kmem_cache_t *zio_buf_cache[SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT]; 80 kmem_cache_t *zio_data_buf_cache[SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT]; 81 #if defined(ZFS_DEBUG) && !defined(_KERNEL) 82 static uint64_t zio_buf_cache_allocs[SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT]; 83 static uint64_t zio_buf_cache_frees[SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT]; 84 #endif 85 86 /* Mark IOs as "slow" if they take longer than 30 seconds */ 87 static uint_t zio_slow_io_ms = (30 * MILLISEC); 88 89 #define BP_SPANB(indblkshift, level) \ 90 (((uint64_t)1) << ((level) * ((indblkshift) - SPA_BLKPTRSHIFT))) 91 #define COMPARE_META_LEVEL 0x80000000ul 92 /* 93 * The following actions directly effect the spa's sync-to-convergence logic. 94 * The values below define the sync pass when we start performing the action. 95 * Care should be taken when changing these values as they directly impact 96 * spa_sync() performance. Tuning these values may introduce subtle performance 97 * pathologies and should only be done in the context of performance analysis. 98 * These tunables will eventually be removed and replaced with #defines once 99 * enough analysis has been done to determine optimal values. 100 * 101 * The 'zfs_sync_pass_deferred_free' pass must be greater than 1 to ensure that 102 * regular blocks are not deferred. 103 * 104 * Starting in sync pass 8 (zfs_sync_pass_dont_compress), we disable 105 * compression (including of metadata). In practice, we don't have this 106 * many sync passes, so this has no effect. 107 * 108 * The original intent was that disabling compression would help the sync 109 * passes to converge. However, in practice disabling compression increases 110 * the average number of sync passes, because when we turn compression off, a 111 * lot of block's size will change and thus we have to re-allocate (not 112 * overwrite) them. It also increases the number of 128KB allocations (e.g. 113 * for indirect blocks and spacemaps) because these will not be compressed. 114 * The 128K allocations are especially detrimental to performance on highly 115 * fragmented systems, which may have very few free segments of this size, 116 * and may need to load new metaslabs to satisfy 128K allocations. 117 */ 118 119 /* defer frees starting in this pass */ 120 uint_t zfs_sync_pass_deferred_free = 2; 121 122 /* don't compress starting in this pass */ 123 static uint_t zfs_sync_pass_dont_compress = 8; 124 125 /* rewrite new bps starting in this pass */ 126 static uint_t zfs_sync_pass_rewrite = 2; 127 128 /* 129 * An allocating zio is one that either currently has the DVA allocate 130 * stage set or will have it later in its lifetime. 131 */ 132 #define IO_IS_ALLOCATING(zio) ((zio)->io_orig_pipeline & ZIO_STAGE_DVA_ALLOCATE) 133 134 /* 135 * Enable smaller cores by excluding metadata 136 * allocations as well. 137 */ 138 int zio_exclude_metadata = 0; 139 static int zio_requeue_io_start_cut_in_line = 1; 140 141 #ifdef ZFS_DEBUG 142 static const int zio_buf_debug_limit = 16384; 143 #else 144 static const int zio_buf_debug_limit = 0; 145 #endif 146 147 static inline void __zio_execute(zio_t *zio); 148 149 static void zio_taskq_dispatch(zio_t *, zio_taskq_type_t, boolean_t); 150 151 void 152 zio_init(void) 153 { 154 size_t c; 155 156 zio_cache = kmem_cache_create("zio_cache", 157 sizeof (zio_t), 0, NULL, NULL, NULL, NULL, NULL, 0); 158 zio_link_cache = kmem_cache_create("zio_link_cache", 159 sizeof (zio_link_t), 0, NULL, NULL, NULL, NULL, NULL, 0); 160 161 for (c = 0; c < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; c++) { 162 size_t size = (c + 1) << SPA_MINBLOCKSHIFT; 163 size_t align, cflags, data_cflags; 164 char name[32]; 165 166 /* 167 * Create cache for each half-power of 2 size, starting from 168 * SPA_MINBLOCKSIZE. It should give us memory space efficiency 169 * of ~7/8, sufficient for transient allocations mostly using 170 * these caches. 171 */ 172 size_t p2 = size; 173 while (!ISP2(p2)) 174 p2 &= p2 - 1; 175 if (!IS_P2ALIGNED(size, p2 / 2)) 176 continue; 177 178 #ifndef _KERNEL 179 /* 180 * If we are using watchpoints, put each buffer on its own page, 181 * to eliminate the performance overhead of trapping to the 182 * kernel when modifying a non-watched buffer that shares the 183 * page with a watched buffer. 184 */ 185 if (arc_watch && !IS_P2ALIGNED(size, PAGESIZE)) 186 continue; 187 #endif 188 189 if (IS_P2ALIGNED(size, PAGESIZE)) 190 align = PAGESIZE; 191 else 192 align = 1 << (highbit64(size ^ (size - 1)) - 1); 193 194 cflags = (zio_exclude_metadata || size > zio_buf_debug_limit) ? 195 KMC_NODEBUG : 0; 196 data_cflags = KMC_NODEBUG; 197 if (cflags == data_cflags) { 198 /* 199 * Resulting kmem caches would be identical. 200 * Save memory by creating only one. 201 */ 202 (void) snprintf(name, sizeof (name), 203 "zio_buf_comb_%lu", (ulong_t)size); 204 zio_buf_cache[c] = kmem_cache_create(name, size, align, 205 NULL, NULL, NULL, NULL, NULL, cflags); 206 zio_data_buf_cache[c] = zio_buf_cache[c]; 207 continue; 208 } 209 (void) snprintf(name, sizeof (name), "zio_buf_%lu", 210 (ulong_t)size); 211 zio_buf_cache[c] = kmem_cache_create(name, size, align, 212 NULL, NULL, NULL, NULL, NULL, cflags); 213 214 (void) snprintf(name, sizeof (name), "zio_data_buf_%lu", 215 (ulong_t)size); 216 zio_data_buf_cache[c] = kmem_cache_create(name, size, align, 217 NULL, NULL, NULL, NULL, NULL, data_cflags); 218 } 219 220 while (--c != 0) { 221 ASSERT(zio_buf_cache[c] != NULL); 222 if (zio_buf_cache[c - 1] == NULL) 223 zio_buf_cache[c - 1] = zio_buf_cache[c]; 224 225 ASSERT(zio_data_buf_cache[c] != NULL); 226 if (zio_data_buf_cache[c - 1] == NULL) 227 zio_data_buf_cache[c - 1] = zio_data_buf_cache[c]; 228 } 229 230 zio_inject_init(); 231 232 lz4_init(); 233 } 234 235 void 236 zio_fini(void) 237 { 238 size_t n = SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; 239 240 #if defined(ZFS_DEBUG) && !defined(_KERNEL) 241 for (size_t i = 0; i < n; i++) { 242 if (zio_buf_cache_allocs[i] != zio_buf_cache_frees[i]) 243 (void) printf("zio_fini: [%d] %llu != %llu\n", 244 (int)((i + 1) << SPA_MINBLOCKSHIFT), 245 (long long unsigned)zio_buf_cache_allocs[i], 246 (long long unsigned)zio_buf_cache_frees[i]); 247 } 248 #endif 249 250 /* 251 * The same kmem cache can show up multiple times in both zio_buf_cache 252 * and zio_data_buf_cache. Do a wasteful but trivially correct scan to 253 * sort it out. 254 */ 255 for (size_t i = 0; i < n; i++) { 256 kmem_cache_t *cache = zio_buf_cache[i]; 257 if (cache == NULL) 258 continue; 259 for (size_t j = i; j < n; j++) { 260 if (cache == zio_buf_cache[j]) 261 zio_buf_cache[j] = NULL; 262 if (cache == zio_data_buf_cache[j]) 263 zio_data_buf_cache[j] = NULL; 264 } 265 kmem_cache_destroy(cache); 266 } 267 268 for (size_t i = 0; i < n; i++) { 269 kmem_cache_t *cache = zio_data_buf_cache[i]; 270 if (cache == NULL) 271 continue; 272 for (size_t j = i; j < n; j++) { 273 if (cache == zio_data_buf_cache[j]) 274 zio_data_buf_cache[j] = NULL; 275 } 276 kmem_cache_destroy(cache); 277 } 278 279 for (size_t i = 0; i < n; i++) { 280 VERIFY3P(zio_buf_cache[i], ==, NULL); 281 VERIFY3P(zio_data_buf_cache[i], ==, NULL); 282 } 283 284 kmem_cache_destroy(zio_link_cache); 285 kmem_cache_destroy(zio_cache); 286 287 zio_inject_fini(); 288 289 lz4_fini(); 290 } 291 292 /* 293 * ========================================================================== 294 * Allocate and free I/O buffers 295 * ========================================================================== 296 */ 297 298 /* 299 * Use zio_buf_alloc to allocate ZFS metadata. This data will appear in a 300 * crashdump if the kernel panics, so use it judiciously. Obviously, it's 301 * useful to inspect ZFS metadata, but if possible, we should avoid keeping 302 * excess / transient data in-core during a crashdump. 303 */ 304 void * 305 zio_buf_alloc(size_t size) 306 { 307 size_t c = (size - 1) >> SPA_MINBLOCKSHIFT; 308 309 VERIFY3U(c, <, SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT); 310 #if defined(ZFS_DEBUG) && !defined(_KERNEL) 311 atomic_add_64(&zio_buf_cache_allocs[c], 1); 312 #endif 313 314 return (kmem_cache_alloc(zio_buf_cache[c], KM_PUSHPAGE)); 315 } 316 317 /* 318 * Use zio_data_buf_alloc to allocate data. The data will not appear in a 319 * crashdump if the kernel panics. This exists so that we will limit the amount 320 * of ZFS data that shows up in a kernel crashdump. (Thus reducing the amount 321 * of kernel heap dumped to disk when the kernel panics) 322 */ 323 void * 324 zio_data_buf_alloc(size_t size) 325 { 326 size_t c = (size - 1) >> SPA_MINBLOCKSHIFT; 327 328 VERIFY3U(c, <, SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT); 329 330 return (kmem_cache_alloc(zio_data_buf_cache[c], KM_PUSHPAGE)); 331 } 332 333 void 334 zio_buf_free(void *buf, size_t size) 335 { 336 size_t c = (size - 1) >> SPA_MINBLOCKSHIFT; 337 338 VERIFY3U(c, <, SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT); 339 #if defined(ZFS_DEBUG) && !defined(_KERNEL) 340 atomic_add_64(&zio_buf_cache_frees[c], 1); 341 #endif 342 343 kmem_cache_free(zio_buf_cache[c], buf); 344 } 345 346 void 347 zio_data_buf_free(void *buf, size_t size) 348 { 349 size_t c = (size - 1) >> SPA_MINBLOCKSHIFT; 350 351 VERIFY3U(c, <, SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT); 352 353 kmem_cache_free(zio_data_buf_cache[c], buf); 354 } 355 356 static void 357 zio_abd_free(void *abd, size_t size) 358 { 359 (void) size; 360 abd_free((abd_t *)abd); 361 } 362 363 /* 364 * ========================================================================== 365 * Push and pop I/O transform buffers 366 * ========================================================================== 367 */ 368 void 369 zio_push_transform(zio_t *zio, abd_t *data, uint64_t size, uint64_t bufsize, 370 zio_transform_func_t *transform) 371 { 372 zio_transform_t *zt = kmem_alloc(sizeof (zio_transform_t), KM_SLEEP); 373 374 zt->zt_orig_abd = zio->io_abd; 375 zt->zt_orig_size = zio->io_size; 376 zt->zt_bufsize = bufsize; 377 zt->zt_transform = transform; 378 379 zt->zt_next = zio->io_transform_stack; 380 zio->io_transform_stack = zt; 381 382 zio->io_abd = data; 383 zio->io_size = size; 384 } 385 386 void 387 zio_pop_transforms(zio_t *zio) 388 { 389 zio_transform_t *zt; 390 391 while ((zt = zio->io_transform_stack) != NULL) { 392 if (zt->zt_transform != NULL) 393 zt->zt_transform(zio, 394 zt->zt_orig_abd, zt->zt_orig_size); 395 396 if (zt->zt_bufsize != 0) 397 abd_free(zio->io_abd); 398 399 zio->io_abd = zt->zt_orig_abd; 400 zio->io_size = zt->zt_orig_size; 401 zio->io_transform_stack = zt->zt_next; 402 403 kmem_free(zt, sizeof (zio_transform_t)); 404 } 405 } 406 407 /* 408 * ========================================================================== 409 * I/O transform callbacks for subblocks, decompression, and decryption 410 * ========================================================================== 411 */ 412 static void 413 zio_subblock(zio_t *zio, abd_t *data, uint64_t size) 414 { 415 ASSERT(zio->io_size > size); 416 417 if (zio->io_type == ZIO_TYPE_READ) 418 abd_copy(data, zio->io_abd, size); 419 } 420 421 static void 422 zio_decompress(zio_t *zio, abd_t *data, uint64_t size) 423 { 424 if (zio->io_error == 0) { 425 void *tmp = abd_borrow_buf(data, size); 426 int ret = zio_decompress_data(BP_GET_COMPRESS(zio->io_bp), 427 zio->io_abd, tmp, zio->io_size, size, 428 &zio->io_prop.zp_complevel); 429 abd_return_buf_copy(data, tmp, size); 430 431 if (zio_injection_enabled && ret == 0) 432 ret = zio_handle_fault_injection(zio, EINVAL); 433 434 if (ret != 0) 435 zio->io_error = SET_ERROR(EIO); 436 } 437 } 438 439 static void 440 zio_decrypt(zio_t *zio, abd_t *data, uint64_t size) 441 { 442 int ret; 443 void *tmp; 444 blkptr_t *bp = zio->io_bp; 445 spa_t *spa = zio->io_spa; 446 uint64_t dsobj = zio->io_bookmark.zb_objset; 447 uint64_t lsize = BP_GET_LSIZE(bp); 448 dmu_object_type_t ot = BP_GET_TYPE(bp); 449 uint8_t salt[ZIO_DATA_SALT_LEN]; 450 uint8_t iv[ZIO_DATA_IV_LEN]; 451 uint8_t mac[ZIO_DATA_MAC_LEN]; 452 boolean_t no_crypt = B_FALSE; 453 454 ASSERT(BP_USES_CRYPT(bp)); 455 ASSERT3U(size, !=, 0); 456 457 if (zio->io_error != 0) 458 return; 459 460 /* 461 * Verify the cksum of MACs stored in an indirect bp. It will always 462 * be possible to verify this since it does not require an encryption 463 * key. 464 */ 465 if (BP_HAS_INDIRECT_MAC_CKSUM(bp)) { 466 zio_crypt_decode_mac_bp(bp, mac); 467 468 if (BP_GET_COMPRESS(bp) != ZIO_COMPRESS_OFF) { 469 /* 470 * We haven't decompressed the data yet, but 471 * zio_crypt_do_indirect_mac_checksum() requires 472 * decompressed data to be able to parse out the MACs 473 * from the indirect block. We decompress it now and 474 * throw away the result after we are finished. 475 */ 476 tmp = zio_buf_alloc(lsize); 477 ret = zio_decompress_data(BP_GET_COMPRESS(bp), 478 zio->io_abd, tmp, zio->io_size, lsize, 479 &zio->io_prop.zp_complevel); 480 if (ret != 0) { 481 ret = SET_ERROR(EIO); 482 goto error; 483 } 484 ret = zio_crypt_do_indirect_mac_checksum(B_FALSE, 485 tmp, lsize, BP_SHOULD_BYTESWAP(bp), mac); 486 zio_buf_free(tmp, lsize); 487 } else { 488 ret = zio_crypt_do_indirect_mac_checksum_abd(B_FALSE, 489 zio->io_abd, size, BP_SHOULD_BYTESWAP(bp), mac); 490 } 491 abd_copy(data, zio->io_abd, size); 492 493 if (zio_injection_enabled && ot != DMU_OT_DNODE && ret == 0) { 494 ret = zio_handle_decrypt_injection(spa, 495 &zio->io_bookmark, ot, ECKSUM); 496 } 497 if (ret != 0) 498 goto error; 499 500 return; 501 } 502 503 /* 504 * If this is an authenticated block, just check the MAC. It would be 505 * nice to separate this out into its own flag, but when this was done, 506 * we had run out of bits in what is now zio_flag_t. Future cleanup 507 * could make this a flag bit. 508 */ 509 if (BP_IS_AUTHENTICATED(bp)) { 510 if (ot == DMU_OT_OBJSET) { 511 ret = spa_do_crypt_objset_mac_abd(B_FALSE, spa, 512 dsobj, zio->io_abd, size, BP_SHOULD_BYTESWAP(bp)); 513 } else { 514 zio_crypt_decode_mac_bp(bp, mac); 515 ret = spa_do_crypt_mac_abd(B_FALSE, spa, dsobj, 516 zio->io_abd, size, mac); 517 if (zio_injection_enabled && ret == 0) { 518 ret = zio_handle_decrypt_injection(spa, 519 &zio->io_bookmark, ot, ECKSUM); 520 } 521 } 522 abd_copy(data, zio->io_abd, size); 523 524 if (ret != 0) 525 goto error; 526 527 return; 528 } 529 530 zio_crypt_decode_params_bp(bp, salt, iv); 531 532 if (ot == DMU_OT_INTENT_LOG) { 533 tmp = abd_borrow_buf_copy(zio->io_abd, sizeof (zil_chain_t)); 534 zio_crypt_decode_mac_zil(tmp, mac); 535 abd_return_buf(zio->io_abd, tmp, sizeof (zil_chain_t)); 536 } else { 537 zio_crypt_decode_mac_bp(bp, mac); 538 } 539 540 ret = spa_do_crypt_abd(B_FALSE, spa, &zio->io_bookmark, BP_GET_TYPE(bp), 541 BP_GET_DEDUP(bp), BP_SHOULD_BYTESWAP(bp), salt, iv, mac, size, data, 542 zio->io_abd, &no_crypt); 543 if (no_crypt) 544 abd_copy(data, zio->io_abd, size); 545 546 if (ret != 0) 547 goto error; 548 549 return; 550 551 error: 552 /* assert that the key was found unless this was speculative */ 553 ASSERT(ret != EACCES || (zio->io_flags & ZIO_FLAG_SPECULATIVE)); 554 555 /* 556 * If there was a decryption / authentication error return EIO as 557 * the io_error. If this was not a speculative zio, create an ereport. 558 */ 559 if (ret == ECKSUM) { 560 zio->io_error = SET_ERROR(EIO); 561 if ((zio->io_flags & ZIO_FLAG_SPECULATIVE) == 0) { 562 spa_log_error(spa, &zio->io_bookmark, 563 &zio->io_bp->blk_birth); 564 (void) zfs_ereport_post(FM_EREPORT_ZFS_AUTHENTICATION, 565 spa, NULL, &zio->io_bookmark, zio, 0); 566 } 567 } else { 568 zio->io_error = ret; 569 } 570 } 571 572 /* 573 * ========================================================================== 574 * I/O parent/child relationships and pipeline interlocks 575 * ========================================================================== 576 */ 577 zio_t * 578 zio_walk_parents(zio_t *cio, zio_link_t **zl) 579 { 580 list_t *pl = &cio->io_parent_list; 581 582 *zl = (*zl == NULL) ? list_head(pl) : list_next(pl, *zl); 583 if (*zl == NULL) 584 return (NULL); 585 586 ASSERT((*zl)->zl_child == cio); 587 return ((*zl)->zl_parent); 588 } 589 590 zio_t * 591 zio_walk_children(zio_t *pio, zio_link_t **zl) 592 { 593 list_t *cl = &pio->io_child_list; 594 595 ASSERT(MUTEX_HELD(&pio->io_lock)); 596 597 *zl = (*zl == NULL) ? list_head(cl) : list_next(cl, *zl); 598 if (*zl == NULL) 599 return (NULL); 600 601 ASSERT((*zl)->zl_parent == pio); 602 return ((*zl)->zl_child); 603 } 604 605 zio_t * 606 zio_unique_parent(zio_t *cio) 607 { 608 zio_link_t *zl = NULL; 609 zio_t *pio = zio_walk_parents(cio, &zl); 610 611 VERIFY3P(zio_walk_parents(cio, &zl), ==, NULL); 612 return (pio); 613 } 614 615 void 616 zio_add_child(zio_t *pio, zio_t *cio) 617 { 618 /* 619 * Logical I/Os can have logical, gang, or vdev children. 620 * Gang I/Os can have gang or vdev children. 621 * Vdev I/Os can only have vdev children. 622 * The following ASSERT captures all of these constraints. 623 */ 624 ASSERT3S(cio->io_child_type, <=, pio->io_child_type); 625 626 /* Parent should not have READY stage if child doesn't have it. */ 627 IMPLY((cio->io_pipeline & ZIO_STAGE_READY) == 0 && 628 (cio->io_child_type != ZIO_CHILD_VDEV), 629 (pio->io_pipeline & ZIO_STAGE_READY) == 0); 630 631 zio_link_t *zl = kmem_cache_alloc(zio_link_cache, KM_SLEEP); 632 zl->zl_parent = pio; 633 zl->zl_child = cio; 634 635 mutex_enter(&pio->io_lock); 636 mutex_enter(&cio->io_lock); 637 638 ASSERT(pio->io_state[ZIO_WAIT_DONE] == 0); 639 640 uint64_t *countp = pio->io_children[cio->io_child_type]; 641 for (int w = 0; w < ZIO_WAIT_TYPES; w++) 642 countp[w] += !cio->io_state[w]; 643 644 list_insert_head(&pio->io_child_list, zl); 645 list_insert_head(&cio->io_parent_list, zl); 646 647 mutex_exit(&cio->io_lock); 648 mutex_exit(&pio->io_lock); 649 } 650 651 void 652 zio_add_child_first(zio_t *pio, zio_t *cio) 653 { 654 /* 655 * Logical I/Os can have logical, gang, or vdev children. 656 * Gang I/Os can have gang or vdev children. 657 * Vdev I/Os can only have vdev children. 658 * The following ASSERT captures all of these constraints. 659 */ 660 ASSERT3S(cio->io_child_type, <=, pio->io_child_type); 661 662 /* Parent should not have READY stage if child doesn't have it. */ 663 IMPLY((cio->io_pipeline & ZIO_STAGE_READY) == 0 && 664 (cio->io_child_type != ZIO_CHILD_VDEV), 665 (pio->io_pipeline & ZIO_STAGE_READY) == 0); 666 667 zio_link_t *zl = kmem_cache_alloc(zio_link_cache, KM_SLEEP); 668 zl->zl_parent = pio; 669 zl->zl_child = cio; 670 671 ASSERT(list_is_empty(&cio->io_parent_list)); 672 list_insert_head(&cio->io_parent_list, zl); 673 674 mutex_enter(&pio->io_lock); 675 676 ASSERT(pio->io_state[ZIO_WAIT_DONE] == 0); 677 678 uint64_t *countp = pio->io_children[cio->io_child_type]; 679 for (int w = 0; w < ZIO_WAIT_TYPES; w++) 680 countp[w] += !cio->io_state[w]; 681 682 list_insert_head(&pio->io_child_list, zl); 683 684 mutex_exit(&pio->io_lock); 685 } 686 687 static void 688 zio_remove_child(zio_t *pio, zio_t *cio, zio_link_t *zl) 689 { 690 ASSERT(zl->zl_parent == pio); 691 ASSERT(zl->zl_child == cio); 692 693 mutex_enter(&pio->io_lock); 694 mutex_enter(&cio->io_lock); 695 696 list_remove(&pio->io_child_list, zl); 697 list_remove(&cio->io_parent_list, zl); 698 699 mutex_exit(&cio->io_lock); 700 mutex_exit(&pio->io_lock); 701 kmem_cache_free(zio_link_cache, zl); 702 } 703 704 static boolean_t 705 zio_wait_for_children(zio_t *zio, uint8_t childbits, enum zio_wait_type wait) 706 { 707 boolean_t waiting = B_FALSE; 708 709 mutex_enter(&zio->io_lock); 710 ASSERT(zio->io_stall == NULL); 711 for (int c = 0; c < ZIO_CHILD_TYPES; c++) { 712 if (!(ZIO_CHILD_BIT_IS_SET(childbits, c))) 713 continue; 714 715 uint64_t *countp = &zio->io_children[c][wait]; 716 if (*countp != 0) { 717 zio->io_stage >>= 1; 718 ASSERT3U(zio->io_stage, !=, ZIO_STAGE_OPEN); 719 zio->io_stall = countp; 720 waiting = B_TRUE; 721 break; 722 } 723 } 724 mutex_exit(&zio->io_lock); 725 return (waiting); 726 } 727 728 __attribute__((always_inline)) 729 static inline void 730 zio_notify_parent(zio_t *pio, zio_t *zio, enum zio_wait_type wait, 731 zio_t **next_to_executep) 732 { 733 uint64_t *countp = &pio->io_children[zio->io_child_type][wait]; 734 int *errorp = &pio->io_child_error[zio->io_child_type]; 735 736 mutex_enter(&pio->io_lock); 737 if (zio->io_error && !(zio->io_flags & ZIO_FLAG_DONT_PROPAGATE)) 738 *errorp = zio_worst_error(*errorp, zio->io_error); 739 pio->io_reexecute |= zio->io_reexecute; 740 ASSERT3U(*countp, >, 0); 741 742 (*countp)--; 743 744 if (*countp == 0 && pio->io_stall == countp) { 745 zio_taskq_type_t type = 746 pio->io_stage < ZIO_STAGE_VDEV_IO_START ? ZIO_TASKQ_ISSUE : 747 ZIO_TASKQ_INTERRUPT; 748 pio->io_stall = NULL; 749 mutex_exit(&pio->io_lock); 750 751 /* 752 * If we can tell the caller to execute this parent next, do 753 * so. We only do this if the parent's zio type matches the 754 * child's type. Otherwise dispatch the parent zio in its 755 * own taskq. 756 * 757 * Having the caller execute the parent when possible reduces 758 * locking on the zio taskq's, reduces context switch 759 * overhead, and has no recursion penalty. Note that one 760 * read from disk typically causes at least 3 zio's: a 761 * zio_null(), the logical zio_read(), and then a physical 762 * zio. When the physical ZIO completes, we are able to call 763 * zio_done() on all 3 of these zio's from one invocation of 764 * zio_execute() by returning the parent back to 765 * zio_execute(). Since the parent isn't executed until this 766 * thread returns back to zio_execute(), the caller should do 767 * so promptly. 768 * 769 * In other cases, dispatching the parent prevents 770 * overflowing the stack when we have deeply nested 771 * parent-child relationships, as we do with the "mega zio" 772 * of writes for spa_sync(), and the chain of ZIL blocks. 773 */ 774 if (next_to_executep != NULL && *next_to_executep == NULL && 775 pio->io_type == zio->io_type) { 776 *next_to_executep = pio; 777 } else { 778 zio_taskq_dispatch(pio, type, B_FALSE); 779 } 780 } else { 781 mutex_exit(&pio->io_lock); 782 } 783 } 784 785 static void 786 zio_inherit_child_errors(zio_t *zio, enum zio_child c) 787 { 788 if (zio->io_child_error[c] != 0 && zio->io_error == 0) 789 zio->io_error = zio->io_child_error[c]; 790 } 791 792 int 793 zio_bookmark_compare(const void *x1, const void *x2) 794 { 795 const zio_t *z1 = x1; 796 const zio_t *z2 = x2; 797 798 if (z1->io_bookmark.zb_objset < z2->io_bookmark.zb_objset) 799 return (-1); 800 if (z1->io_bookmark.zb_objset > z2->io_bookmark.zb_objset) 801 return (1); 802 803 if (z1->io_bookmark.zb_object < z2->io_bookmark.zb_object) 804 return (-1); 805 if (z1->io_bookmark.zb_object > z2->io_bookmark.zb_object) 806 return (1); 807 808 if (z1->io_bookmark.zb_level < z2->io_bookmark.zb_level) 809 return (-1); 810 if (z1->io_bookmark.zb_level > z2->io_bookmark.zb_level) 811 return (1); 812 813 if (z1->io_bookmark.zb_blkid < z2->io_bookmark.zb_blkid) 814 return (-1); 815 if (z1->io_bookmark.zb_blkid > z2->io_bookmark.zb_blkid) 816 return (1); 817 818 if (z1 < z2) 819 return (-1); 820 if (z1 > z2) 821 return (1); 822 823 return (0); 824 } 825 826 /* 827 * ========================================================================== 828 * Create the various types of I/O (read, write, free, etc) 829 * ========================================================================== 830 */ 831 static zio_t * 832 zio_create(zio_t *pio, spa_t *spa, uint64_t txg, const blkptr_t *bp, 833 abd_t *data, uint64_t lsize, uint64_t psize, zio_done_func_t *done, 834 void *private, zio_type_t type, zio_priority_t priority, 835 zio_flag_t flags, vdev_t *vd, uint64_t offset, 836 const zbookmark_phys_t *zb, enum zio_stage stage, 837 enum zio_stage pipeline) 838 { 839 zio_t *zio; 840 841 IMPLY(type != ZIO_TYPE_TRIM, psize <= SPA_MAXBLOCKSIZE); 842 ASSERT(P2PHASE(psize, SPA_MINBLOCKSIZE) == 0); 843 ASSERT(P2PHASE(offset, SPA_MINBLOCKSIZE) == 0); 844 845 ASSERT(!vd || spa_config_held(spa, SCL_STATE_ALL, RW_READER)); 846 ASSERT(!bp || !(flags & ZIO_FLAG_CONFIG_WRITER)); 847 ASSERT(vd || stage == ZIO_STAGE_OPEN); 848 849 IMPLY(lsize != psize, (flags & ZIO_FLAG_RAW_COMPRESS) != 0); 850 851 zio = kmem_cache_alloc(zio_cache, KM_SLEEP); 852 memset(zio, 0, sizeof (zio_t)); 853 854 mutex_init(&zio->io_lock, NULL, MUTEX_NOLOCKDEP, NULL); 855 cv_init(&zio->io_cv, NULL, CV_DEFAULT, NULL); 856 857 list_create(&zio->io_parent_list, sizeof (zio_link_t), 858 offsetof(zio_link_t, zl_parent_node)); 859 list_create(&zio->io_child_list, sizeof (zio_link_t), 860 offsetof(zio_link_t, zl_child_node)); 861 metaslab_trace_init(&zio->io_alloc_list); 862 863 if (vd != NULL) 864 zio->io_child_type = ZIO_CHILD_VDEV; 865 else if (flags & ZIO_FLAG_GANG_CHILD) 866 zio->io_child_type = ZIO_CHILD_GANG; 867 else if (flags & ZIO_FLAG_DDT_CHILD) 868 zio->io_child_type = ZIO_CHILD_DDT; 869 else 870 zio->io_child_type = ZIO_CHILD_LOGICAL; 871 872 if (bp != NULL) { 873 if (type != ZIO_TYPE_WRITE || 874 zio->io_child_type == ZIO_CHILD_DDT) { 875 zio->io_bp_copy = *bp; 876 zio->io_bp = &zio->io_bp_copy; /* so caller can free */ 877 } else { 878 zio->io_bp = (blkptr_t *)bp; 879 } 880 zio->io_bp_orig = *bp; 881 if (zio->io_child_type == ZIO_CHILD_LOGICAL) 882 zio->io_logical = zio; 883 if (zio->io_child_type > ZIO_CHILD_GANG && BP_IS_GANG(bp)) 884 pipeline |= ZIO_GANG_STAGES; 885 } 886 887 zio->io_spa = spa; 888 zio->io_txg = txg; 889 zio->io_done = done; 890 zio->io_private = private; 891 zio->io_type = type; 892 zio->io_priority = priority; 893 zio->io_vd = vd; 894 zio->io_offset = offset; 895 zio->io_orig_abd = zio->io_abd = data; 896 zio->io_orig_size = zio->io_size = psize; 897 zio->io_lsize = lsize; 898 zio->io_orig_flags = zio->io_flags = flags; 899 zio->io_orig_stage = zio->io_stage = stage; 900 zio->io_orig_pipeline = zio->io_pipeline = pipeline; 901 zio->io_pipeline_trace = ZIO_STAGE_OPEN; 902 zio->io_allocator = ZIO_ALLOCATOR_NONE; 903 904 zio->io_state[ZIO_WAIT_READY] = (stage >= ZIO_STAGE_READY) || 905 (pipeline & ZIO_STAGE_READY) == 0; 906 zio->io_state[ZIO_WAIT_DONE] = (stage >= ZIO_STAGE_DONE); 907 908 if (zb != NULL) 909 zio->io_bookmark = *zb; 910 911 if (pio != NULL) { 912 zio->io_metaslab_class = pio->io_metaslab_class; 913 if (zio->io_logical == NULL) 914 zio->io_logical = pio->io_logical; 915 if (zio->io_child_type == ZIO_CHILD_GANG) 916 zio->io_gang_leader = pio->io_gang_leader; 917 zio_add_child_first(pio, zio); 918 } 919 920 taskq_init_ent(&zio->io_tqent); 921 922 return (zio); 923 } 924 925 void 926 zio_destroy(zio_t *zio) 927 { 928 metaslab_trace_fini(&zio->io_alloc_list); 929 list_destroy(&zio->io_parent_list); 930 list_destroy(&zio->io_child_list); 931 mutex_destroy(&zio->io_lock); 932 cv_destroy(&zio->io_cv); 933 kmem_cache_free(zio_cache, zio); 934 } 935 936 /* 937 * ZIO intended to be between others. Provides synchronization at READY 938 * and DONE pipeline stages and calls the respective callbacks. 939 */ 940 zio_t * 941 zio_null(zio_t *pio, spa_t *spa, vdev_t *vd, zio_done_func_t *done, 942 void *private, zio_flag_t flags) 943 { 944 zio_t *zio; 945 946 zio = zio_create(pio, spa, 0, NULL, NULL, 0, 0, done, private, 947 ZIO_TYPE_NULL, ZIO_PRIORITY_NOW, flags, vd, 0, NULL, 948 ZIO_STAGE_OPEN, ZIO_INTERLOCK_PIPELINE); 949 950 return (zio); 951 } 952 953 /* 954 * ZIO intended to be a root of a tree. Unlike null ZIO does not have a 955 * READY pipeline stage (is ready on creation), so it should not be used 956 * as child of any ZIO that may need waiting for grandchildren READY stage 957 * (any other ZIO type). 958 */ 959 zio_t * 960 zio_root(spa_t *spa, zio_done_func_t *done, void *private, zio_flag_t flags) 961 { 962 zio_t *zio; 963 964 zio = zio_create(NULL, spa, 0, NULL, NULL, 0, 0, done, private, 965 ZIO_TYPE_NULL, ZIO_PRIORITY_NOW, flags, NULL, 0, NULL, 966 ZIO_STAGE_OPEN, ZIO_ROOT_PIPELINE); 967 968 return (zio); 969 } 970 971 static int 972 zfs_blkptr_verify_log(spa_t *spa, const blkptr_t *bp, 973 enum blk_verify_flag blk_verify, const char *fmt, ...) 974 { 975 va_list adx; 976 char buf[256]; 977 978 va_start(adx, fmt); 979 (void) vsnprintf(buf, sizeof (buf), fmt, adx); 980 va_end(adx); 981 982 zfs_dbgmsg("bad blkptr at %px: " 983 "DVA[0]=%#llx/%#llx " 984 "DVA[1]=%#llx/%#llx " 985 "DVA[2]=%#llx/%#llx " 986 "prop=%#llx " 987 "pad=%#llx,%#llx " 988 "phys_birth=%#llx " 989 "birth=%#llx " 990 "fill=%#llx " 991 "cksum=%#llx/%#llx/%#llx/%#llx", 992 bp, 993 (long long)bp->blk_dva[0].dva_word[0], 994 (long long)bp->blk_dva[0].dva_word[1], 995 (long long)bp->blk_dva[1].dva_word[0], 996 (long long)bp->blk_dva[1].dva_word[1], 997 (long long)bp->blk_dva[2].dva_word[0], 998 (long long)bp->blk_dva[2].dva_word[1], 999 (long long)bp->blk_prop, 1000 (long long)bp->blk_pad[0], 1001 (long long)bp->blk_pad[1], 1002 (long long)bp->blk_phys_birth, 1003 (long long)bp->blk_birth, 1004 (long long)bp->blk_fill, 1005 (long long)bp->blk_cksum.zc_word[0], 1006 (long long)bp->blk_cksum.zc_word[1], 1007 (long long)bp->blk_cksum.zc_word[2], 1008 (long long)bp->blk_cksum.zc_word[3]); 1009 switch (blk_verify) { 1010 case BLK_VERIFY_HALT: 1011 zfs_panic_recover("%s: %s", spa_name(spa), buf); 1012 break; 1013 case BLK_VERIFY_LOG: 1014 zfs_dbgmsg("%s: %s", spa_name(spa), buf); 1015 break; 1016 case BLK_VERIFY_ONLY: 1017 break; 1018 } 1019 1020 return (1); 1021 } 1022 1023 /* 1024 * Verify the block pointer fields contain reasonable values. This means 1025 * it only contains known object types, checksum/compression identifiers, 1026 * block sizes within the maximum allowed limits, valid DVAs, etc. 1027 * 1028 * If everything checks out B_TRUE is returned. The zfs_blkptr_verify 1029 * argument controls the behavior when an invalid field is detected. 1030 * 1031 * Values for blk_verify_flag: 1032 * BLK_VERIFY_ONLY: evaluate the block 1033 * BLK_VERIFY_LOG: evaluate the block and log problems 1034 * BLK_VERIFY_HALT: call zfs_panic_recover on error 1035 * 1036 * Values for blk_config_flag: 1037 * BLK_CONFIG_HELD: caller holds SCL_VDEV for writer 1038 * BLK_CONFIG_NEEDED: caller holds no config lock, SCL_VDEV will be 1039 * obtained for reader 1040 * BLK_CONFIG_SKIP: skip checks which require SCL_VDEV, for better 1041 * performance 1042 */ 1043 boolean_t 1044 zfs_blkptr_verify(spa_t *spa, const blkptr_t *bp, 1045 enum blk_config_flag blk_config, enum blk_verify_flag blk_verify) 1046 { 1047 int errors = 0; 1048 1049 if (!DMU_OT_IS_VALID(BP_GET_TYPE(bp))) { 1050 errors += zfs_blkptr_verify_log(spa, bp, blk_verify, 1051 "blkptr at %px has invalid TYPE %llu", 1052 bp, (longlong_t)BP_GET_TYPE(bp)); 1053 } 1054 if (BP_GET_CHECKSUM(bp) >= ZIO_CHECKSUM_FUNCTIONS) { 1055 errors += zfs_blkptr_verify_log(spa, bp, blk_verify, 1056 "blkptr at %px has invalid CHECKSUM %llu", 1057 bp, (longlong_t)BP_GET_CHECKSUM(bp)); 1058 } 1059 if (BP_GET_COMPRESS(bp) >= ZIO_COMPRESS_FUNCTIONS) { 1060 errors += zfs_blkptr_verify_log(spa, bp, blk_verify, 1061 "blkptr at %px has invalid COMPRESS %llu", 1062 bp, (longlong_t)BP_GET_COMPRESS(bp)); 1063 } 1064 if (BP_GET_LSIZE(bp) > SPA_MAXBLOCKSIZE) { 1065 errors += zfs_blkptr_verify_log(spa, bp, blk_verify, 1066 "blkptr at %px has invalid LSIZE %llu", 1067 bp, (longlong_t)BP_GET_LSIZE(bp)); 1068 } 1069 if (BP_GET_PSIZE(bp) > SPA_MAXBLOCKSIZE) { 1070 errors += zfs_blkptr_verify_log(spa, bp, blk_verify, 1071 "blkptr at %px has invalid PSIZE %llu", 1072 bp, (longlong_t)BP_GET_PSIZE(bp)); 1073 } 1074 1075 if (BP_IS_EMBEDDED(bp)) { 1076 if (BPE_GET_ETYPE(bp) >= NUM_BP_EMBEDDED_TYPES) { 1077 errors += zfs_blkptr_verify_log(spa, bp, blk_verify, 1078 "blkptr at %px has invalid ETYPE %llu", 1079 bp, (longlong_t)BPE_GET_ETYPE(bp)); 1080 } 1081 } 1082 1083 /* 1084 * Do not verify individual DVAs if the config is not trusted. This 1085 * will be done once the zio is executed in vdev_mirror_map_alloc. 1086 */ 1087 if (!spa->spa_trust_config) 1088 return (errors == 0); 1089 1090 switch (blk_config) { 1091 case BLK_CONFIG_HELD: 1092 ASSERT(spa_config_held(spa, SCL_VDEV, RW_WRITER)); 1093 break; 1094 case BLK_CONFIG_NEEDED: 1095 spa_config_enter(spa, SCL_VDEV, bp, RW_READER); 1096 break; 1097 case BLK_CONFIG_SKIP: 1098 return (errors == 0); 1099 default: 1100 panic("invalid blk_config %u", blk_config); 1101 } 1102 1103 /* 1104 * Pool-specific checks. 1105 * 1106 * Note: it would be nice to verify that the blk_birth and 1107 * BP_PHYSICAL_BIRTH() are not too large. However, spa_freeze() 1108 * allows the birth time of log blocks (and dmu_sync()-ed blocks 1109 * that are in the log) to be arbitrarily large. 1110 */ 1111 for (int i = 0; i < BP_GET_NDVAS(bp); i++) { 1112 const dva_t *dva = &bp->blk_dva[i]; 1113 uint64_t vdevid = DVA_GET_VDEV(dva); 1114 1115 if (vdevid >= spa->spa_root_vdev->vdev_children) { 1116 errors += zfs_blkptr_verify_log(spa, bp, blk_verify, 1117 "blkptr at %px DVA %u has invalid VDEV %llu", 1118 bp, i, (longlong_t)vdevid); 1119 continue; 1120 } 1121 vdev_t *vd = spa->spa_root_vdev->vdev_child[vdevid]; 1122 if (vd == NULL) { 1123 errors += zfs_blkptr_verify_log(spa, bp, blk_verify, 1124 "blkptr at %px DVA %u has invalid VDEV %llu", 1125 bp, i, (longlong_t)vdevid); 1126 continue; 1127 } 1128 if (vd->vdev_ops == &vdev_hole_ops) { 1129 errors += zfs_blkptr_verify_log(spa, bp, blk_verify, 1130 "blkptr at %px DVA %u has hole VDEV %llu", 1131 bp, i, (longlong_t)vdevid); 1132 continue; 1133 } 1134 if (vd->vdev_ops == &vdev_missing_ops) { 1135 /* 1136 * "missing" vdevs are valid during import, but we 1137 * don't have their detailed info (e.g. asize), so 1138 * we can't perform any more checks on them. 1139 */ 1140 continue; 1141 } 1142 uint64_t offset = DVA_GET_OFFSET(dva); 1143 uint64_t asize = DVA_GET_ASIZE(dva); 1144 if (DVA_GET_GANG(dva)) 1145 asize = vdev_gang_header_asize(vd); 1146 if (offset + asize > vd->vdev_asize) { 1147 errors += zfs_blkptr_verify_log(spa, bp, blk_verify, 1148 "blkptr at %px DVA %u has invalid OFFSET %llu", 1149 bp, i, (longlong_t)offset); 1150 } 1151 } 1152 if (blk_config == BLK_CONFIG_NEEDED) 1153 spa_config_exit(spa, SCL_VDEV, bp); 1154 1155 return (errors == 0); 1156 } 1157 1158 boolean_t 1159 zfs_dva_valid(spa_t *spa, const dva_t *dva, const blkptr_t *bp) 1160 { 1161 (void) bp; 1162 uint64_t vdevid = DVA_GET_VDEV(dva); 1163 1164 if (vdevid >= spa->spa_root_vdev->vdev_children) 1165 return (B_FALSE); 1166 1167 vdev_t *vd = spa->spa_root_vdev->vdev_child[vdevid]; 1168 if (vd == NULL) 1169 return (B_FALSE); 1170 1171 if (vd->vdev_ops == &vdev_hole_ops) 1172 return (B_FALSE); 1173 1174 if (vd->vdev_ops == &vdev_missing_ops) { 1175 return (B_FALSE); 1176 } 1177 1178 uint64_t offset = DVA_GET_OFFSET(dva); 1179 uint64_t asize = DVA_GET_ASIZE(dva); 1180 1181 if (DVA_GET_GANG(dva)) 1182 asize = vdev_gang_header_asize(vd); 1183 if (offset + asize > vd->vdev_asize) 1184 return (B_FALSE); 1185 1186 return (B_TRUE); 1187 } 1188 1189 zio_t * 1190 zio_read(zio_t *pio, spa_t *spa, const blkptr_t *bp, 1191 abd_t *data, uint64_t size, zio_done_func_t *done, void *private, 1192 zio_priority_t priority, zio_flag_t flags, const zbookmark_phys_t *zb) 1193 { 1194 zio_t *zio; 1195 1196 zio = zio_create(pio, spa, BP_PHYSICAL_BIRTH(bp), bp, 1197 data, size, size, done, private, 1198 ZIO_TYPE_READ, priority, flags, NULL, 0, zb, 1199 ZIO_STAGE_OPEN, (flags & ZIO_FLAG_DDT_CHILD) ? 1200 ZIO_DDT_CHILD_READ_PIPELINE : ZIO_READ_PIPELINE); 1201 1202 return (zio); 1203 } 1204 1205 zio_t * 1206 zio_write(zio_t *pio, spa_t *spa, uint64_t txg, blkptr_t *bp, 1207 abd_t *data, uint64_t lsize, uint64_t psize, const zio_prop_t *zp, 1208 zio_done_func_t *ready, zio_done_func_t *children_ready, 1209 zio_done_func_t *done, void *private, zio_priority_t priority, 1210 zio_flag_t flags, const zbookmark_phys_t *zb) 1211 { 1212 zio_t *zio; 1213 1214 ASSERT(zp->zp_checksum >= ZIO_CHECKSUM_OFF && 1215 zp->zp_checksum < ZIO_CHECKSUM_FUNCTIONS && 1216 zp->zp_compress >= ZIO_COMPRESS_OFF && 1217 zp->zp_compress < ZIO_COMPRESS_FUNCTIONS && 1218 DMU_OT_IS_VALID(zp->zp_type) && 1219 zp->zp_level < 32 && 1220 zp->zp_copies > 0 && 1221 zp->zp_copies <= spa_max_replication(spa)); 1222 1223 zio = zio_create(pio, spa, txg, bp, data, lsize, psize, done, private, 1224 ZIO_TYPE_WRITE, priority, flags, NULL, 0, zb, 1225 ZIO_STAGE_OPEN, (flags & ZIO_FLAG_DDT_CHILD) ? 1226 ZIO_DDT_CHILD_WRITE_PIPELINE : ZIO_WRITE_PIPELINE); 1227 1228 zio->io_ready = ready; 1229 zio->io_children_ready = children_ready; 1230 zio->io_prop = *zp; 1231 1232 /* 1233 * Data can be NULL if we are going to call zio_write_override() to 1234 * provide the already-allocated BP. But we may need the data to 1235 * verify a dedup hit (if requested). In this case, don't try to 1236 * dedup (just take the already-allocated BP verbatim). Encrypted 1237 * dedup blocks need data as well so we also disable dedup in this 1238 * case. 1239 */ 1240 if (data == NULL && 1241 (zio->io_prop.zp_dedup_verify || zio->io_prop.zp_encrypt)) { 1242 zio->io_prop.zp_dedup = zio->io_prop.zp_dedup_verify = B_FALSE; 1243 } 1244 1245 return (zio); 1246 } 1247 1248 zio_t * 1249 zio_rewrite(zio_t *pio, spa_t *spa, uint64_t txg, blkptr_t *bp, abd_t *data, 1250 uint64_t size, zio_done_func_t *done, void *private, 1251 zio_priority_t priority, zio_flag_t flags, zbookmark_phys_t *zb) 1252 { 1253 zio_t *zio; 1254 1255 zio = zio_create(pio, spa, txg, bp, data, size, size, done, private, 1256 ZIO_TYPE_WRITE, priority, flags | ZIO_FLAG_IO_REWRITE, NULL, 0, zb, 1257 ZIO_STAGE_OPEN, ZIO_REWRITE_PIPELINE); 1258 1259 return (zio); 1260 } 1261 1262 void 1263 zio_write_override(zio_t *zio, blkptr_t *bp, int copies, boolean_t nopwrite, 1264 boolean_t brtwrite) 1265 { 1266 ASSERT(zio->io_type == ZIO_TYPE_WRITE); 1267 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL); 1268 ASSERT(zio->io_stage == ZIO_STAGE_OPEN); 1269 ASSERT(zio->io_txg == spa_syncing_txg(zio->io_spa)); 1270 ASSERT(!brtwrite || !nopwrite); 1271 1272 /* 1273 * We must reset the io_prop to match the values that existed 1274 * when the bp was first written by dmu_sync() keeping in mind 1275 * that nopwrite and dedup are mutually exclusive. 1276 */ 1277 zio->io_prop.zp_dedup = nopwrite ? B_FALSE : zio->io_prop.zp_dedup; 1278 zio->io_prop.zp_nopwrite = nopwrite; 1279 zio->io_prop.zp_brtwrite = brtwrite; 1280 zio->io_prop.zp_copies = copies; 1281 zio->io_bp_override = bp; 1282 } 1283 1284 void 1285 zio_free(spa_t *spa, uint64_t txg, const blkptr_t *bp) 1286 { 1287 1288 (void) zfs_blkptr_verify(spa, bp, BLK_CONFIG_NEEDED, BLK_VERIFY_HALT); 1289 1290 /* 1291 * The check for EMBEDDED is a performance optimization. We 1292 * process the free here (by ignoring it) rather than 1293 * putting it on the list and then processing it in zio_free_sync(). 1294 */ 1295 if (BP_IS_EMBEDDED(bp)) 1296 return; 1297 1298 /* 1299 * Frees that are for the currently-syncing txg, are not going to be 1300 * deferred, and which will not need to do a read (i.e. not GANG or 1301 * DEDUP), can be processed immediately. Otherwise, put them on the 1302 * in-memory list for later processing. 1303 * 1304 * Note that we only defer frees after zfs_sync_pass_deferred_free 1305 * when the log space map feature is disabled. [see relevant comment 1306 * in spa_sync_iterate_to_convergence()] 1307 */ 1308 if (BP_IS_GANG(bp) || 1309 BP_GET_DEDUP(bp) || 1310 txg != spa->spa_syncing_txg || 1311 (spa_sync_pass(spa) >= zfs_sync_pass_deferred_free && 1312 !spa_feature_is_active(spa, SPA_FEATURE_LOG_SPACEMAP)) || 1313 brt_maybe_exists(spa, bp)) { 1314 metaslab_check_free(spa, bp); 1315 bplist_append(&spa->spa_free_bplist[txg & TXG_MASK], bp); 1316 } else { 1317 VERIFY3P(zio_free_sync(NULL, spa, txg, bp, 0), ==, NULL); 1318 } 1319 } 1320 1321 /* 1322 * To improve performance, this function may return NULL if we were able 1323 * to do the free immediately. This avoids the cost of creating a zio 1324 * (and linking it to the parent, etc). 1325 */ 1326 zio_t * 1327 zio_free_sync(zio_t *pio, spa_t *spa, uint64_t txg, const blkptr_t *bp, 1328 zio_flag_t flags) 1329 { 1330 ASSERT(!BP_IS_HOLE(bp)); 1331 ASSERT(spa_syncing_txg(spa) == txg); 1332 1333 if (BP_IS_EMBEDDED(bp)) 1334 return (NULL); 1335 1336 metaslab_check_free(spa, bp); 1337 arc_freed(spa, bp); 1338 dsl_scan_freed(spa, bp); 1339 1340 if (BP_IS_GANG(bp) || 1341 BP_GET_DEDUP(bp) || 1342 brt_maybe_exists(spa, bp)) { 1343 /* 1344 * GANG, DEDUP and BRT blocks can induce a read (for the gang 1345 * block header, the DDT or the BRT), so issue them 1346 * asynchronously so that this thread is not tied up. 1347 */ 1348 enum zio_stage stage = 1349 ZIO_FREE_PIPELINE | ZIO_STAGE_ISSUE_ASYNC; 1350 1351 return (zio_create(pio, spa, txg, bp, NULL, BP_GET_PSIZE(bp), 1352 BP_GET_PSIZE(bp), NULL, NULL, 1353 ZIO_TYPE_FREE, ZIO_PRIORITY_NOW, 1354 flags, NULL, 0, NULL, ZIO_STAGE_OPEN, stage)); 1355 } else { 1356 metaslab_free(spa, bp, txg, B_FALSE); 1357 return (NULL); 1358 } 1359 } 1360 1361 zio_t * 1362 zio_claim(zio_t *pio, spa_t *spa, uint64_t txg, const blkptr_t *bp, 1363 zio_done_func_t *done, void *private, zio_flag_t flags) 1364 { 1365 zio_t *zio; 1366 1367 (void) zfs_blkptr_verify(spa, bp, (flags & ZIO_FLAG_CONFIG_WRITER) ? 1368 BLK_CONFIG_HELD : BLK_CONFIG_NEEDED, BLK_VERIFY_HALT); 1369 1370 if (BP_IS_EMBEDDED(bp)) 1371 return (zio_null(pio, spa, NULL, NULL, NULL, 0)); 1372 1373 /* 1374 * A claim is an allocation of a specific block. Claims are needed 1375 * to support immediate writes in the intent log. The issue is that 1376 * immediate writes contain committed data, but in a txg that was 1377 * *not* committed. Upon opening the pool after an unclean shutdown, 1378 * the intent log claims all blocks that contain immediate write data 1379 * so that the SPA knows they're in use. 1380 * 1381 * All claims *must* be resolved in the first txg -- before the SPA 1382 * starts allocating blocks -- so that nothing is allocated twice. 1383 * If txg == 0 we just verify that the block is claimable. 1384 */ 1385 ASSERT3U(spa->spa_uberblock.ub_rootbp.blk_birth, <, 1386 spa_min_claim_txg(spa)); 1387 ASSERT(txg == spa_min_claim_txg(spa) || txg == 0); 1388 ASSERT(!BP_GET_DEDUP(bp) || !spa_writeable(spa)); /* zdb(8) */ 1389 1390 zio = zio_create(pio, spa, txg, bp, NULL, BP_GET_PSIZE(bp), 1391 BP_GET_PSIZE(bp), done, private, ZIO_TYPE_CLAIM, ZIO_PRIORITY_NOW, 1392 flags, NULL, 0, NULL, ZIO_STAGE_OPEN, ZIO_CLAIM_PIPELINE); 1393 ASSERT0(zio->io_queued_timestamp); 1394 1395 return (zio); 1396 } 1397 1398 zio_t * 1399 zio_ioctl(zio_t *pio, spa_t *spa, vdev_t *vd, int cmd, 1400 zio_done_func_t *done, void *private, zio_flag_t flags) 1401 { 1402 zio_t *zio; 1403 int c; 1404 1405 if (vd->vdev_children == 0) { 1406 zio = zio_create(pio, spa, 0, NULL, NULL, 0, 0, done, private, 1407 ZIO_TYPE_IOCTL, ZIO_PRIORITY_NOW, flags, vd, 0, NULL, 1408 ZIO_STAGE_OPEN, ZIO_IOCTL_PIPELINE); 1409 1410 zio->io_cmd = cmd; 1411 } else { 1412 zio = zio_null(pio, spa, NULL, NULL, NULL, flags); 1413 1414 for (c = 0; c < vd->vdev_children; c++) 1415 zio_nowait(zio_ioctl(zio, spa, vd->vdev_child[c], cmd, 1416 done, private, flags)); 1417 } 1418 1419 return (zio); 1420 } 1421 1422 zio_t * 1423 zio_trim(zio_t *pio, vdev_t *vd, uint64_t offset, uint64_t size, 1424 zio_done_func_t *done, void *private, zio_priority_t priority, 1425 zio_flag_t flags, enum trim_flag trim_flags) 1426 { 1427 zio_t *zio; 1428 1429 ASSERT0(vd->vdev_children); 1430 ASSERT0(P2PHASE(offset, 1ULL << vd->vdev_ashift)); 1431 ASSERT0(P2PHASE(size, 1ULL << vd->vdev_ashift)); 1432 ASSERT3U(size, !=, 0); 1433 1434 zio = zio_create(pio, vd->vdev_spa, 0, NULL, NULL, size, size, done, 1435 private, ZIO_TYPE_TRIM, priority, flags | ZIO_FLAG_PHYSICAL, 1436 vd, offset, NULL, ZIO_STAGE_OPEN, ZIO_TRIM_PIPELINE); 1437 zio->io_trim_flags = trim_flags; 1438 1439 return (zio); 1440 } 1441 1442 zio_t * 1443 zio_read_phys(zio_t *pio, vdev_t *vd, uint64_t offset, uint64_t size, 1444 abd_t *data, int checksum, zio_done_func_t *done, void *private, 1445 zio_priority_t priority, zio_flag_t flags, boolean_t labels) 1446 { 1447 zio_t *zio; 1448 1449 ASSERT(vd->vdev_children == 0); 1450 ASSERT(!labels || offset + size <= VDEV_LABEL_START_SIZE || 1451 offset >= vd->vdev_psize - VDEV_LABEL_END_SIZE); 1452 ASSERT3U(offset + size, <=, vd->vdev_psize); 1453 1454 zio = zio_create(pio, vd->vdev_spa, 0, NULL, data, size, size, done, 1455 private, ZIO_TYPE_READ, priority, flags | ZIO_FLAG_PHYSICAL, vd, 1456 offset, NULL, ZIO_STAGE_OPEN, ZIO_READ_PHYS_PIPELINE); 1457 1458 zio->io_prop.zp_checksum = checksum; 1459 1460 return (zio); 1461 } 1462 1463 zio_t * 1464 zio_write_phys(zio_t *pio, vdev_t *vd, uint64_t offset, uint64_t size, 1465 abd_t *data, int checksum, zio_done_func_t *done, void *private, 1466 zio_priority_t priority, zio_flag_t flags, boolean_t labels) 1467 { 1468 zio_t *zio; 1469 1470 ASSERT(vd->vdev_children == 0); 1471 ASSERT(!labels || offset + size <= VDEV_LABEL_START_SIZE || 1472 offset >= vd->vdev_psize - VDEV_LABEL_END_SIZE); 1473 ASSERT3U(offset + size, <=, vd->vdev_psize); 1474 1475 zio = zio_create(pio, vd->vdev_spa, 0, NULL, data, size, size, done, 1476 private, ZIO_TYPE_WRITE, priority, flags | ZIO_FLAG_PHYSICAL, vd, 1477 offset, NULL, ZIO_STAGE_OPEN, ZIO_WRITE_PHYS_PIPELINE); 1478 1479 zio->io_prop.zp_checksum = checksum; 1480 1481 if (zio_checksum_table[checksum].ci_flags & ZCHECKSUM_FLAG_EMBEDDED) { 1482 /* 1483 * zec checksums are necessarily destructive -- they modify 1484 * the end of the write buffer to hold the verifier/checksum. 1485 * Therefore, we must make a local copy in case the data is 1486 * being written to multiple places in parallel. 1487 */ 1488 abd_t *wbuf = abd_alloc_sametype(data, size); 1489 abd_copy(wbuf, data, size); 1490 1491 zio_push_transform(zio, wbuf, size, size, NULL); 1492 } 1493 1494 return (zio); 1495 } 1496 1497 /* 1498 * Create a child I/O to do some work for us. 1499 */ 1500 zio_t * 1501 zio_vdev_child_io(zio_t *pio, blkptr_t *bp, vdev_t *vd, uint64_t offset, 1502 abd_t *data, uint64_t size, int type, zio_priority_t priority, 1503 zio_flag_t flags, zio_done_func_t *done, void *private) 1504 { 1505 enum zio_stage pipeline = ZIO_VDEV_CHILD_PIPELINE; 1506 zio_t *zio; 1507 1508 /* 1509 * vdev child I/Os do not propagate their error to the parent. 1510 * Therefore, for correct operation the caller *must* check for 1511 * and handle the error in the child i/o's done callback. 1512 * The only exceptions are i/os that we don't care about 1513 * (OPTIONAL or REPAIR). 1514 */ 1515 ASSERT((flags & ZIO_FLAG_OPTIONAL) || (flags & ZIO_FLAG_IO_REPAIR) || 1516 done != NULL); 1517 1518 if (type == ZIO_TYPE_READ && bp != NULL) { 1519 /* 1520 * If we have the bp, then the child should perform the 1521 * checksum and the parent need not. This pushes error 1522 * detection as close to the leaves as possible and 1523 * eliminates redundant checksums in the interior nodes. 1524 */ 1525 pipeline |= ZIO_STAGE_CHECKSUM_VERIFY; 1526 pio->io_pipeline &= ~ZIO_STAGE_CHECKSUM_VERIFY; 1527 } 1528 1529 if (vd->vdev_ops->vdev_op_leaf) { 1530 ASSERT0(vd->vdev_children); 1531 offset += VDEV_LABEL_START_SIZE; 1532 } 1533 1534 flags |= ZIO_VDEV_CHILD_FLAGS(pio); 1535 1536 /* 1537 * If we've decided to do a repair, the write is not speculative -- 1538 * even if the original read was. 1539 */ 1540 if (flags & ZIO_FLAG_IO_REPAIR) 1541 flags &= ~ZIO_FLAG_SPECULATIVE; 1542 1543 /* 1544 * If we're creating a child I/O that is not associated with a 1545 * top-level vdev, then the child zio is not an allocating I/O. 1546 * If this is a retried I/O then we ignore it since we will 1547 * have already processed the original allocating I/O. 1548 */ 1549 if (flags & ZIO_FLAG_IO_ALLOCATING && 1550 (vd != vd->vdev_top || (flags & ZIO_FLAG_IO_RETRY))) { 1551 ASSERT(pio->io_metaslab_class != NULL); 1552 ASSERT(pio->io_metaslab_class->mc_alloc_throttle_enabled); 1553 ASSERT(type == ZIO_TYPE_WRITE); 1554 ASSERT(priority == ZIO_PRIORITY_ASYNC_WRITE); 1555 ASSERT(!(flags & ZIO_FLAG_IO_REPAIR)); 1556 ASSERT(!(pio->io_flags & ZIO_FLAG_IO_REWRITE) || 1557 pio->io_child_type == ZIO_CHILD_GANG); 1558 1559 flags &= ~ZIO_FLAG_IO_ALLOCATING; 1560 } 1561 1562 zio = zio_create(pio, pio->io_spa, pio->io_txg, bp, data, size, size, 1563 done, private, type, priority, flags, vd, offset, &pio->io_bookmark, 1564 ZIO_STAGE_VDEV_IO_START >> 1, pipeline); 1565 ASSERT3U(zio->io_child_type, ==, ZIO_CHILD_VDEV); 1566 1567 return (zio); 1568 } 1569 1570 zio_t * 1571 zio_vdev_delegated_io(vdev_t *vd, uint64_t offset, abd_t *data, uint64_t size, 1572 zio_type_t type, zio_priority_t priority, zio_flag_t flags, 1573 zio_done_func_t *done, void *private) 1574 { 1575 zio_t *zio; 1576 1577 ASSERT(vd->vdev_ops->vdev_op_leaf); 1578 1579 zio = zio_create(NULL, vd->vdev_spa, 0, NULL, 1580 data, size, size, done, private, type, priority, 1581 flags | ZIO_FLAG_CANFAIL | ZIO_FLAG_DONT_RETRY | ZIO_FLAG_DELEGATED, 1582 vd, offset, NULL, 1583 ZIO_STAGE_VDEV_IO_START >> 1, ZIO_VDEV_CHILD_PIPELINE); 1584 1585 return (zio); 1586 } 1587 1588 void 1589 zio_flush(zio_t *zio, vdev_t *vd) 1590 { 1591 zio_nowait(zio_ioctl(zio, zio->io_spa, vd, DKIOCFLUSHWRITECACHE, 1592 NULL, NULL, 1593 ZIO_FLAG_CANFAIL | ZIO_FLAG_DONT_PROPAGATE | ZIO_FLAG_DONT_RETRY)); 1594 } 1595 1596 void 1597 zio_shrink(zio_t *zio, uint64_t size) 1598 { 1599 ASSERT3P(zio->io_executor, ==, NULL); 1600 ASSERT3U(zio->io_orig_size, ==, zio->io_size); 1601 ASSERT3U(size, <=, zio->io_size); 1602 1603 /* 1604 * We don't shrink for raidz because of problems with the 1605 * reconstruction when reading back less than the block size. 1606 * Note, BP_IS_RAIDZ() assumes no compression. 1607 */ 1608 ASSERT(BP_GET_COMPRESS(zio->io_bp) == ZIO_COMPRESS_OFF); 1609 if (!BP_IS_RAIDZ(zio->io_bp)) { 1610 /* we are not doing a raw write */ 1611 ASSERT3U(zio->io_size, ==, zio->io_lsize); 1612 zio->io_orig_size = zio->io_size = zio->io_lsize = size; 1613 } 1614 } 1615 1616 /* 1617 * Round provided allocation size up to a value that can be allocated 1618 * by at least some vdev(s) in the pool with minimum or no additional 1619 * padding and without extra space usage on others 1620 */ 1621 static uint64_t 1622 zio_roundup_alloc_size(spa_t *spa, uint64_t size) 1623 { 1624 if (size > spa->spa_min_alloc) 1625 return (roundup(size, spa->spa_gcd_alloc)); 1626 return (spa->spa_min_alloc); 1627 } 1628 1629 /* 1630 * ========================================================================== 1631 * Prepare to read and write logical blocks 1632 * ========================================================================== 1633 */ 1634 1635 static zio_t * 1636 zio_read_bp_init(zio_t *zio) 1637 { 1638 blkptr_t *bp = zio->io_bp; 1639 uint64_t psize = 1640 BP_IS_EMBEDDED(bp) ? BPE_GET_PSIZE(bp) : BP_GET_PSIZE(bp); 1641 1642 ASSERT3P(zio->io_bp, ==, &zio->io_bp_copy); 1643 1644 if (BP_GET_COMPRESS(bp) != ZIO_COMPRESS_OFF && 1645 zio->io_child_type == ZIO_CHILD_LOGICAL && 1646 !(zio->io_flags & ZIO_FLAG_RAW_COMPRESS)) { 1647 zio_push_transform(zio, abd_alloc_sametype(zio->io_abd, psize), 1648 psize, psize, zio_decompress); 1649 } 1650 1651 if (((BP_IS_PROTECTED(bp) && !(zio->io_flags & ZIO_FLAG_RAW_ENCRYPT)) || 1652 BP_HAS_INDIRECT_MAC_CKSUM(bp)) && 1653 zio->io_child_type == ZIO_CHILD_LOGICAL) { 1654 zio_push_transform(zio, abd_alloc_sametype(zio->io_abd, psize), 1655 psize, psize, zio_decrypt); 1656 } 1657 1658 if (BP_IS_EMBEDDED(bp) && BPE_GET_ETYPE(bp) == BP_EMBEDDED_TYPE_DATA) { 1659 int psize = BPE_GET_PSIZE(bp); 1660 void *data = abd_borrow_buf(zio->io_abd, psize); 1661 1662 zio->io_pipeline = ZIO_INTERLOCK_PIPELINE; 1663 decode_embedded_bp_compressed(bp, data); 1664 abd_return_buf_copy(zio->io_abd, data, psize); 1665 } else { 1666 ASSERT(!BP_IS_EMBEDDED(bp)); 1667 } 1668 1669 if (BP_GET_DEDUP(bp) && zio->io_child_type == ZIO_CHILD_LOGICAL) 1670 zio->io_pipeline = ZIO_DDT_READ_PIPELINE; 1671 1672 return (zio); 1673 } 1674 1675 static zio_t * 1676 zio_write_bp_init(zio_t *zio) 1677 { 1678 if (!IO_IS_ALLOCATING(zio)) 1679 return (zio); 1680 1681 ASSERT(zio->io_child_type != ZIO_CHILD_DDT); 1682 1683 if (zio->io_bp_override) { 1684 blkptr_t *bp = zio->io_bp; 1685 zio_prop_t *zp = &zio->io_prop; 1686 1687 ASSERT(bp->blk_birth != zio->io_txg); 1688 1689 *bp = *zio->io_bp_override; 1690 zio->io_pipeline = ZIO_INTERLOCK_PIPELINE; 1691 1692 if (zp->zp_brtwrite) 1693 return (zio); 1694 1695 ASSERT(!BP_GET_DEDUP(zio->io_bp_override)); 1696 1697 if (BP_IS_EMBEDDED(bp)) 1698 return (zio); 1699 1700 /* 1701 * If we've been overridden and nopwrite is set then 1702 * set the flag accordingly to indicate that a nopwrite 1703 * has already occurred. 1704 */ 1705 if (!BP_IS_HOLE(bp) && zp->zp_nopwrite) { 1706 ASSERT(!zp->zp_dedup); 1707 ASSERT3U(BP_GET_CHECKSUM(bp), ==, zp->zp_checksum); 1708 zio->io_flags |= ZIO_FLAG_NOPWRITE; 1709 return (zio); 1710 } 1711 1712 ASSERT(!zp->zp_nopwrite); 1713 1714 if (BP_IS_HOLE(bp) || !zp->zp_dedup) 1715 return (zio); 1716 1717 ASSERT((zio_checksum_table[zp->zp_checksum].ci_flags & 1718 ZCHECKSUM_FLAG_DEDUP) || zp->zp_dedup_verify); 1719 1720 if (BP_GET_CHECKSUM(bp) == zp->zp_checksum && 1721 !zp->zp_encrypt) { 1722 BP_SET_DEDUP(bp, 1); 1723 zio->io_pipeline |= ZIO_STAGE_DDT_WRITE; 1724 return (zio); 1725 } 1726 1727 /* 1728 * We were unable to handle this as an override bp, treat 1729 * it as a regular write I/O. 1730 */ 1731 zio->io_bp_override = NULL; 1732 *bp = zio->io_bp_orig; 1733 zio->io_pipeline = zio->io_orig_pipeline; 1734 } 1735 1736 return (zio); 1737 } 1738 1739 static zio_t * 1740 zio_write_compress(zio_t *zio) 1741 { 1742 spa_t *spa = zio->io_spa; 1743 zio_prop_t *zp = &zio->io_prop; 1744 enum zio_compress compress = zp->zp_compress; 1745 blkptr_t *bp = zio->io_bp; 1746 uint64_t lsize = zio->io_lsize; 1747 uint64_t psize = zio->io_size; 1748 uint32_t pass = 1; 1749 1750 /* 1751 * If our children haven't all reached the ready stage, 1752 * wait for them and then repeat this pipeline stage. 1753 */ 1754 if (zio_wait_for_children(zio, ZIO_CHILD_LOGICAL_BIT | 1755 ZIO_CHILD_GANG_BIT, ZIO_WAIT_READY)) { 1756 return (NULL); 1757 } 1758 1759 if (!IO_IS_ALLOCATING(zio)) 1760 return (zio); 1761 1762 if (zio->io_children_ready != NULL) { 1763 /* 1764 * Now that all our children are ready, run the callback 1765 * associated with this zio in case it wants to modify the 1766 * data to be written. 1767 */ 1768 ASSERT3U(zp->zp_level, >, 0); 1769 zio->io_children_ready(zio); 1770 } 1771 1772 ASSERT(zio->io_child_type != ZIO_CHILD_DDT); 1773 ASSERT(zio->io_bp_override == NULL); 1774 1775 if (!BP_IS_HOLE(bp) && bp->blk_birth == zio->io_txg) { 1776 /* 1777 * We're rewriting an existing block, which means we're 1778 * working on behalf of spa_sync(). For spa_sync() to 1779 * converge, it must eventually be the case that we don't 1780 * have to allocate new blocks. But compression changes 1781 * the blocksize, which forces a reallocate, and makes 1782 * convergence take longer. Therefore, after the first 1783 * few passes, stop compressing to ensure convergence. 1784 */ 1785 pass = spa_sync_pass(spa); 1786 1787 ASSERT(zio->io_txg == spa_syncing_txg(spa)); 1788 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL); 1789 ASSERT(!BP_GET_DEDUP(bp)); 1790 1791 if (pass >= zfs_sync_pass_dont_compress) 1792 compress = ZIO_COMPRESS_OFF; 1793 1794 /* Make sure someone doesn't change their mind on overwrites */ 1795 ASSERT(BP_IS_EMBEDDED(bp) || BP_IS_GANG(bp) || 1796 MIN(zp->zp_copies, spa_max_replication(spa)) 1797 == BP_GET_NDVAS(bp)); 1798 } 1799 1800 /* If it's a compressed write that is not raw, compress the buffer. */ 1801 if (compress != ZIO_COMPRESS_OFF && 1802 !(zio->io_flags & ZIO_FLAG_RAW_COMPRESS)) { 1803 void *cbuf = NULL; 1804 psize = zio_compress_data(compress, zio->io_abd, &cbuf, lsize, 1805 zp->zp_complevel); 1806 if (psize == 0) { 1807 compress = ZIO_COMPRESS_OFF; 1808 } else if (psize >= lsize) { 1809 compress = ZIO_COMPRESS_OFF; 1810 if (cbuf != NULL) 1811 zio_buf_free(cbuf, lsize); 1812 } else if (!zp->zp_dedup && !zp->zp_encrypt && 1813 psize <= BPE_PAYLOAD_SIZE && 1814 zp->zp_level == 0 && !DMU_OT_HAS_FILL(zp->zp_type) && 1815 spa_feature_is_enabled(spa, SPA_FEATURE_EMBEDDED_DATA)) { 1816 encode_embedded_bp_compressed(bp, 1817 cbuf, compress, lsize, psize); 1818 BPE_SET_ETYPE(bp, BP_EMBEDDED_TYPE_DATA); 1819 BP_SET_TYPE(bp, zio->io_prop.zp_type); 1820 BP_SET_LEVEL(bp, zio->io_prop.zp_level); 1821 zio_buf_free(cbuf, lsize); 1822 bp->blk_birth = zio->io_txg; 1823 zio->io_pipeline = ZIO_INTERLOCK_PIPELINE; 1824 ASSERT(spa_feature_is_active(spa, 1825 SPA_FEATURE_EMBEDDED_DATA)); 1826 return (zio); 1827 } else { 1828 /* 1829 * Round compressed size up to the minimum allocation 1830 * size of the smallest-ashift device, and zero the 1831 * tail. This ensures that the compressed size of the 1832 * BP (and thus compressratio property) are correct, 1833 * in that we charge for the padding used to fill out 1834 * the last sector. 1835 */ 1836 size_t rounded = (size_t)zio_roundup_alloc_size(spa, 1837 psize); 1838 if (rounded >= lsize) { 1839 compress = ZIO_COMPRESS_OFF; 1840 zio_buf_free(cbuf, lsize); 1841 psize = lsize; 1842 } else { 1843 abd_t *cdata = abd_get_from_buf(cbuf, lsize); 1844 abd_take_ownership_of_buf(cdata, B_TRUE); 1845 abd_zero_off(cdata, psize, rounded - psize); 1846 psize = rounded; 1847 zio_push_transform(zio, cdata, 1848 psize, lsize, NULL); 1849 } 1850 } 1851 1852 /* 1853 * We were unable to handle this as an override bp, treat 1854 * it as a regular write I/O. 1855 */ 1856 zio->io_bp_override = NULL; 1857 *bp = zio->io_bp_orig; 1858 zio->io_pipeline = zio->io_orig_pipeline; 1859 1860 } else if ((zio->io_flags & ZIO_FLAG_RAW_ENCRYPT) != 0 && 1861 zp->zp_type == DMU_OT_DNODE) { 1862 /* 1863 * The DMU actually relies on the zio layer's compression 1864 * to free metadnode blocks that have had all contained 1865 * dnodes freed. As a result, even when doing a raw 1866 * receive, we must check whether the block can be compressed 1867 * to a hole. 1868 */ 1869 psize = zio_compress_data(ZIO_COMPRESS_EMPTY, 1870 zio->io_abd, NULL, lsize, zp->zp_complevel); 1871 if (psize == 0 || psize >= lsize) 1872 compress = ZIO_COMPRESS_OFF; 1873 } else if (zio->io_flags & ZIO_FLAG_RAW_COMPRESS && 1874 !(zio->io_flags & ZIO_FLAG_RAW_ENCRYPT)) { 1875 /* 1876 * If we are raw receiving an encrypted dataset we should not 1877 * take this codepath because it will change the on-disk block 1878 * and decryption will fail. 1879 */ 1880 size_t rounded = MIN((size_t)zio_roundup_alloc_size(spa, psize), 1881 lsize); 1882 1883 if (rounded != psize) { 1884 abd_t *cdata = abd_alloc_linear(rounded, B_TRUE); 1885 abd_zero_off(cdata, psize, rounded - psize); 1886 abd_copy_off(cdata, zio->io_abd, 0, 0, psize); 1887 psize = rounded; 1888 zio_push_transform(zio, cdata, 1889 psize, rounded, NULL); 1890 } 1891 } else { 1892 ASSERT3U(psize, !=, 0); 1893 } 1894 1895 /* 1896 * The final pass of spa_sync() must be all rewrites, but the first 1897 * few passes offer a trade-off: allocating blocks defers convergence, 1898 * but newly allocated blocks are sequential, so they can be written 1899 * to disk faster. Therefore, we allow the first few passes of 1900 * spa_sync() to allocate new blocks, but force rewrites after that. 1901 * There should only be a handful of blocks after pass 1 in any case. 1902 */ 1903 if (!BP_IS_HOLE(bp) && bp->blk_birth == zio->io_txg && 1904 BP_GET_PSIZE(bp) == psize && 1905 pass >= zfs_sync_pass_rewrite) { 1906 VERIFY3U(psize, !=, 0); 1907 enum zio_stage gang_stages = zio->io_pipeline & ZIO_GANG_STAGES; 1908 1909 zio->io_pipeline = ZIO_REWRITE_PIPELINE | gang_stages; 1910 zio->io_flags |= ZIO_FLAG_IO_REWRITE; 1911 } else { 1912 BP_ZERO(bp); 1913 zio->io_pipeline = ZIO_WRITE_PIPELINE; 1914 } 1915 1916 if (psize == 0) { 1917 if (zio->io_bp_orig.blk_birth != 0 && 1918 spa_feature_is_active(spa, SPA_FEATURE_HOLE_BIRTH)) { 1919 BP_SET_LSIZE(bp, lsize); 1920 BP_SET_TYPE(bp, zp->zp_type); 1921 BP_SET_LEVEL(bp, zp->zp_level); 1922 BP_SET_BIRTH(bp, zio->io_txg, 0); 1923 } 1924 zio->io_pipeline = ZIO_INTERLOCK_PIPELINE; 1925 } else { 1926 ASSERT(zp->zp_checksum != ZIO_CHECKSUM_GANG_HEADER); 1927 BP_SET_LSIZE(bp, lsize); 1928 BP_SET_TYPE(bp, zp->zp_type); 1929 BP_SET_LEVEL(bp, zp->zp_level); 1930 BP_SET_PSIZE(bp, psize); 1931 BP_SET_COMPRESS(bp, compress); 1932 BP_SET_CHECKSUM(bp, zp->zp_checksum); 1933 BP_SET_DEDUP(bp, zp->zp_dedup); 1934 BP_SET_BYTEORDER(bp, ZFS_HOST_BYTEORDER); 1935 if (zp->zp_dedup) { 1936 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL); 1937 ASSERT(!(zio->io_flags & ZIO_FLAG_IO_REWRITE)); 1938 ASSERT(!zp->zp_encrypt || 1939 DMU_OT_IS_ENCRYPTED(zp->zp_type)); 1940 zio->io_pipeline = ZIO_DDT_WRITE_PIPELINE; 1941 } 1942 if (zp->zp_nopwrite) { 1943 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL); 1944 ASSERT(!(zio->io_flags & ZIO_FLAG_IO_REWRITE)); 1945 zio->io_pipeline |= ZIO_STAGE_NOP_WRITE; 1946 } 1947 } 1948 return (zio); 1949 } 1950 1951 static zio_t * 1952 zio_free_bp_init(zio_t *zio) 1953 { 1954 blkptr_t *bp = zio->io_bp; 1955 1956 if (zio->io_child_type == ZIO_CHILD_LOGICAL) { 1957 if (BP_GET_DEDUP(bp)) 1958 zio->io_pipeline = ZIO_DDT_FREE_PIPELINE; 1959 } 1960 1961 ASSERT3P(zio->io_bp, ==, &zio->io_bp_copy); 1962 1963 return (zio); 1964 } 1965 1966 /* 1967 * ========================================================================== 1968 * Execute the I/O pipeline 1969 * ========================================================================== 1970 */ 1971 1972 static void 1973 zio_taskq_dispatch(zio_t *zio, zio_taskq_type_t q, boolean_t cutinline) 1974 { 1975 spa_t *spa = zio->io_spa; 1976 zio_type_t t = zio->io_type; 1977 int flags = (cutinline ? TQ_FRONT : 0); 1978 1979 /* 1980 * If we're a config writer or a probe, the normal issue and 1981 * interrupt threads may all be blocked waiting for the config lock. 1982 * In this case, select the otherwise-unused taskq for ZIO_TYPE_NULL. 1983 */ 1984 if (zio->io_flags & (ZIO_FLAG_CONFIG_WRITER | ZIO_FLAG_PROBE)) 1985 t = ZIO_TYPE_NULL; 1986 1987 /* 1988 * A similar issue exists for the L2ARC write thread until L2ARC 2.0. 1989 */ 1990 if (t == ZIO_TYPE_WRITE && zio->io_vd && zio->io_vd->vdev_aux) 1991 t = ZIO_TYPE_NULL; 1992 1993 /* 1994 * If this is a high priority I/O, then use the high priority taskq if 1995 * available. 1996 */ 1997 if ((zio->io_priority == ZIO_PRIORITY_NOW || 1998 zio->io_priority == ZIO_PRIORITY_SYNC_WRITE) && 1999 spa->spa_zio_taskq[t][q + 1].stqs_count != 0) 2000 q++; 2001 2002 ASSERT3U(q, <, ZIO_TASKQ_TYPES); 2003 2004 /* 2005 * NB: We are assuming that the zio can only be dispatched 2006 * to a single taskq at a time. It would be a grievous error 2007 * to dispatch the zio to another taskq at the same time. 2008 */ 2009 ASSERT(taskq_empty_ent(&zio->io_tqent)); 2010 spa_taskq_dispatch_ent(spa, t, q, zio_execute, zio, flags, 2011 &zio->io_tqent, zio); 2012 } 2013 2014 static boolean_t 2015 zio_taskq_member(zio_t *zio, zio_taskq_type_t q) 2016 { 2017 spa_t *spa = zio->io_spa; 2018 2019 taskq_t *tq = taskq_of_curthread(); 2020 2021 for (zio_type_t t = 0; t < ZIO_TYPES; t++) { 2022 spa_taskqs_t *tqs = &spa->spa_zio_taskq[t][q]; 2023 uint_t i; 2024 for (i = 0; i < tqs->stqs_count; i++) { 2025 if (tqs->stqs_taskq[i] == tq) 2026 return (B_TRUE); 2027 } 2028 } 2029 2030 return (B_FALSE); 2031 } 2032 2033 static zio_t * 2034 zio_issue_async(zio_t *zio) 2035 { 2036 ASSERT((zio->io_type != ZIO_TYPE_WRITE) || ZIO_HAS_ALLOCATOR(zio)); 2037 zio_taskq_dispatch(zio, ZIO_TASKQ_ISSUE, B_FALSE); 2038 return (NULL); 2039 } 2040 2041 void 2042 zio_interrupt(void *zio) 2043 { 2044 zio_taskq_dispatch(zio, ZIO_TASKQ_INTERRUPT, B_FALSE); 2045 } 2046 2047 void 2048 zio_delay_interrupt(zio_t *zio) 2049 { 2050 /* 2051 * The timeout_generic() function isn't defined in userspace, so 2052 * rather than trying to implement the function, the zio delay 2053 * functionality has been disabled for userspace builds. 2054 */ 2055 2056 #ifdef _KERNEL 2057 /* 2058 * If io_target_timestamp is zero, then no delay has been registered 2059 * for this IO, thus jump to the end of this function and "skip" the 2060 * delay; issuing it directly to the zio layer. 2061 */ 2062 if (zio->io_target_timestamp != 0) { 2063 hrtime_t now = gethrtime(); 2064 2065 if (now >= zio->io_target_timestamp) { 2066 /* 2067 * This IO has already taken longer than the target 2068 * delay to complete, so we don't want to delay it 2069 * any longer; we "miss" the delay and issue it 2070 * directly to the zio layer. This is likely due to 2071 * the target latency being set to a value less than 2072 * the underlying hardware can satisfy (e.g. delay 2073 * set to 1ms, but the disks take 10ms to complete an 2074 * IO request). 2075 */ 2076 2077 DTRACE_PROBE2(zio__delay__miss, zio_t *, zio, 2078 hrtime_t, now); 2079 2080 zio_interrupt(zio); 2081 } else { 2082 taskqid_t tid; 2083 hrtime_t diff = zio->io_target_timestamp - now; 2084 clock_t expire_at_tick = ddi_get_lbolt() + 2085 NSEC_TO_TICK(diff); 2086 2087 DTRACE_PROBE3(zio__delay__hit, zio_t *, zio, 2088 hrtime_t, now, hrtime_t, diff); 2089 2090 if (NSEC_TO_TICK(diff) == 0) { 2091 /* Our delay is less than a jiffy - just spin */ 2092 zfs_sleep_until(zio->io_target_timestamp); 2093 zio_interrupt(zio); 2094 } else { 2095 /* 2096 * Use taskq_dispatch_delay() in the place of 2097 * OpenZFS's timeout_generic(). 2098 */ 2099 tid = taskq_dispatch_delay(system_taskq, 2100 zio_interrupt, zio, TQ_NOSLEEP, 2101 expire_at_tick); 2102 if (tid == TASKQID_INVALID) { 2103 /* 2104 * Couldn't allocate a task. Just 2105 * finish the zio without a delay. 2106 */ 2107 zio_interrupt(zio); 2108 } 2109 } 2110 } 2111 return; 2112 } 2113 #endif 2114 DTRACE_PROBE1(zio__delay__skip, zio_t *, zio); 2115 zio_interrupt(zio); 2116 } 2117 2118 static void 2119 zio_deadman_impl(zio_t *pio, int ziodepth) 2120 { 2121 zio_t *cio, *cio_next; 2122 zio_link_t *zl = NULL; 2123 vdev_t *vd = pio->io_vd; 2124 2125 if (zio_deadman_log_all || (vd != NULL && vd->vdev_ops->vdev_op_leaf)) { 2126 vdev_queue_t *vq = vd ? &vd->vdev_queue : NULL; 2127 zbookmark_phys_t *zb = &pio->io_bookmark; 2128 uint64_t delta = gethrtime() - pio->io_timestamp; 2129 uint64_t failmode = spa_get_deadman_failmode(pio->io_spa); 2130 2131 zfs_dbgmsg("slow zio[%d]: zio=%px timestamp=%llu " 2132 "delta=%llu queued=%llu io=%llu " 2133 "path=%s " 2134 "last=%llu type=%d " 2135 "priority=%d flags=0x%llx stage=0x%x " 2136 "pipeline=0x%x pipeline-trace=0x%x " 2137 "objset=%llu object=%llu " 2138 "level=%llu blkid=%llu " 2139 "offset=%llu size=%llu " 2140 "error=%d", 2141 ziodepth, pio, pio->io_timestamp, 2142 (u_longlong_t)delta, pio->io_delta, pio->io_delay, 2143 vd ? vd->vdev_path : "NULL", 2144 vq ? vq->vq_io_complete_ts : 0, pio->io_type, 2145 pio->io_priority, (u_longlong_t)pio->io_flags, 2146 pio->io_stage, pio->io_pipeline, pio->io_pipeline_trace, 2147 (u_longlong_t)zb->zb_objset, (u_longlong_t)zb->zb_object, 2148 (u_longlong_t)zb->zb_level, (u_longlong_t)zb->zb_blkid, 2149 (u_longlong_t)pio->io_offset, (u_longlong_t)pio->io_size, 2150 pio->io_error); 2151 (void) zfs_ereport_post(FM_EREPORT_ZFS_DEADMAN, 2152 pio->io_spa, vd, zb, pio, 0); 2153 2154 if (failmode == ZIO_FAILURE_MODE_CONTINUE && 2155 taskq_empty_ent(&pio->io_tqent)) { 2156 zio_interrupt(pio); 2157 } 2158 } 2159 2160 mutex_enter(&pio->io_lock); 2161 for (cio = zio_walk_children(pio, &zl); cio != NULL; cio = cio_next) { 2162 cio_next = zio_walk_children(pio, &zl); 2163 zio_deadman_impl(cio, ziodepth + 1); 2164 } 2165 mutex_exit(&pio->io_lock); 2166 } 2167 2168 /* 2169 * Log the critical information describing this zio and all of its children 2170 * using the zfs_dbgmsg() interface then post deadman event for the ZED. 2171 */ 2172 void 2173 zio_deadman(zio_t *pio, const char *tag) 2174 { 2175 spa_t *spa = pio->io_spa; 2176 char *name = spa_name(spa); 2177 2178 if (!zfs_deadman_enabled || spa_suspended(spa)) 2179 return; 2180 2181 zio_deadman_impl(pio, 0); 2182 2183 switch (spa_get_deadman_failmode(spa)) { 2184 case ZIO_FAILURE_MODE_WAIT: 2185 zfs_dbgmsg("%s waiting for hung I/O to pool '%s'", tag, name); 2186 break; 2187 2188 case ZIO_FAILURE_MODE_CONTINUE: 2189 zfs_dbgmsg("%s restarting hung I/O for pool '%s'", tag, name); 2190 break; 2191 2192 case ZIO_FAILURE_MODE_PANIC: 2193 fm_panic("%s determined I/O to pool '%s' is hung.", tag, name); 2194 break; 2195 } 2196 } 2197 2198 /* 2199 * Execute the I/O pipeline until one of the following occurs: 2200 * (1) the I/O completes; (2) the pipeline stalls waiting for 2201 * dependent child I/Os; (3) the I/O issues, so we're waiting 2202 * for an I/O completion interrupt; (4) the I/O is delegated by 2203 * vdev-level caching or aggregation; (5) the I/O is deferred 2204 * due to vdev-level queueing; (6) the I/O is handed off to 2205 * another thread. In all cases, the pipeline stops whenever 2206 * there's no CPU work; it never burns a thread in cv_wait_io(). 2207 * 2208 * There's no locking on io_stage because there's no legitimate way 2209 * for multiple threads to be attempting to process the same I/O. 2210 */ 2211 static zio_pipe_stage_t *zio_pipeline[]; 2212 2213 /* 2214 * zio_execute() is a wrapper around the static function 2215 * __zio_execute() so that we can force __zio_execute() to be 2216 * inlined. This reduces stack overhead which is important 2217 * because __zio_execute() is called recursively in several zio 2218 * code paths. zio_execute() itself cannot be inlined because 2219 * it is externally visible. 2220 */ 2221 void 2222 zio_execute(void *zio) 2223 { 2224 fstrans_cookie_t cookie; 2225 2226 cookie = spl_fstrans_mark(); 2227 __zio_execute(zio); 2228 spl_fstrans_unmark(cookie); 2229 } 2230 2231 /* 2232 * Used to determine if in the current context the stack is sized large 2233 * enough to allow zio_execute() to be called recursively. A minimum 2234 * stack size of 16K is required to avoid needing to re-dispatch the zio. 2235 */ 2236 static boolean_t 2237 zio_execute_stack_check(zio_t *zio) 2238 { 2239 #if !defined(HAVE_LARGE_STACKS) 2240 dsl_pool_t *dp = spa_get_dsl(zio->io_spa); 2241 2242 /* Executing in txg_sync_thread() context. */ 2243 if (dp && curthread == dp->dp_tx.tx_sync_thread) 2244 return (B_TRUE); 2245 2246 /* Pool initialization outside of zio_taskq context. */ 2247 if (dp && spa_is_initializing(dp->dp_spa) && 2248 !zio_taskq_member(zio, ZIO_TASKQ_ISSUE) && 2249 !zio_taskq_member(zio, ZIO_TASKQ_ISSUE_HIGH)) 2250 return (B_TRUE); 2251 #else 2252 (void) zio; 2253 #endif /* HAVE_LARGE_STACKS */ 2254 2255 return (B_FALSE); 2256 } 2257 2258 __attribute__((always_inline)) 2259 static inline void 2260 __zio_execute(zio_t *zio) 2261 { 2262 ASSERT3U(zio->io_queued_timestamp, >, 0); 2263 2264 while (zio->io_stage < ZIO_STAGE_DONE) { 2265 enum zio_stage pipeline = zio->io_pipeline; 2266 enum zio_stage stage = zio->io_stage; 2267 2268 zio->io_executor = curthread; 2269 2270 ASSERT(!MUTEX_HELD(&zio->io_lock)); 2271 ASSERT(ISP2(stage)); 2272 ASSERT(zio->io_stall == NULL); 2273 2274 do { 2275 stage <<= 1; 2276 } while ((stage & pipeline) == 0); 2277 2278 ASSERT(stage <= ZIO_STAGE_DONE); 2279 2280 /* 2281 * If we are in interrupt context and this pipeline stage 2282 * will grab a config lock that is held across I/O, 2283 * or may wait for an I/O that needs an interrupt thread 2284 * to complete, issue async to avoid deadlock. 2285 * 2286 * For VDEV_IO_START, we cut in line so that the io will 2287 * be sent to disk promptly. 2288 */ 2289 if ((stage & ZIO_BLOCKING_STAGES) && zio->io_vd == NULL && 2290 zio_taskq_member(zio, ZIO_TASKQ_INTERRUPT)) { 2291 boolean_t cut = (stage == ZIO_STAGE_VDEV_IO_START) ? 2292 zio_requeue_io_start_cut_in_line : B_FALSE; 2293 zio_taskq_dispatch(zio, ZIO_TASKQ_ISSUE, cut); 2294 return; 2295 } 2296 2297 /* 2298 * If the current context doesn't have large enough stacks 2299 * the zio must be issued asynchronously to prevent overflow. 2300 */ 2301 if (zio_execute_stack_check(zio)) { 2302 boolean_t cut = (stage == ZIO_STAGE_VDEV_IO_START) ? 2303 zio_requeue_io_start_cut_in_line : B_FALSE; 2304 zio_taskq_dispatch(zio, ZIO_TASKQ_ISSUE, cut); 2305 return; 2306 } 2307 2308 zio->io_stage = stage; 2309 zio->io_pipeline_trace |= zio->io_stage; 2310 2311 /* 2312 * The zio pipeline stage returns the next zio to execute 2313 * (typically the same as this one), or NULL if we should 2314 * stop. 2315 */ 2316 zio = zio_pipeline[highbit64(stage) - 1](zio); 2317 2318 if (zio == NULL) 2319 return; 2320 } 2321 } 2322 2323 2324 /* 2325 * ========================================================================== 2326 * Initiate I/O, either sync or async 2327 * ========================================================================== 2328 */ 2329 int 2330 zio_wait(zio_t *zio) 2331 { 2332 /* 2333 * Some routines, like zio_free_sync(), may return a NULL zio 2334 * to avoid the performance overhead of creating and then destroying 2335 * an unneeded zio. For the callers' simplicity, we accept a NULL 2336 * zio and ignore it. 2337 */ 2338 if (zio == NULL) 2339 return (0); 2340 2341 long timeout = MSEC_TO_TICK(zfs_deadman_ziotime_ms); 2342 int error; 2343 2344 ASSERT3S(zio->io_stage, ==, ZIO_STAGE_OPEN); 2345 ASSERT3P(zio->io_executor, ==, NULL); 2346 2347 zio->io_waiter = curthread; 2348 ASSERT0(zio->io_queued_timestamp); 2349 zio->io_queued_timestamp = gethrtime(); 2350 2351 if (zio->io_type == ZIO_TYPE_WRITE) { 2352 spa_select_allocator(zio); 2353 } 2354 __zio_execute(zio); 2355 2356 mutex_enter(&zio->io_lock); 2357 while (zio->io_executor != NULL) { 2358 error = cv_timedwait_io(&zio->io_cv, &zio->io_lock, 2359 ddi_get_lbolt() + timeout); 2360 2361 if (zfs_deadman_enabled && error == -1 && 2362 gethrtime() - zio->io_queued_timestamp > 2363 spa_deadman_ziotime(zio->io_spa)) { 2364 mutex_exit(&zio->io_lock); 2365 timeout = MSEC_TO_TICK(zfs_deadman_checktime_ms); 2366 zio_deadman(zio, FTAG); 2367 mutex_enter(&zio->io_lock); 2368 } 2369 } 2370 mutex_exit(&zio->io_lock); 2371 2372 error = zio->io_error; 2373 zio_destroy(zio); 2374 2375 return (error); 2376 } 2377 2378 void 2379 zio_nowait(zio_t *zio) 2380 { 2381 /* 2382 * See comment in zio_wait(). 2383 */ 2384 if (zio == NULL) 2385 return; 2386 2387 ASSERT3P(zio->io_executor, ==, NULL); 2388 2389 if (zio->io_child_type == ZIO_CHILD_LOGICAL && 2390 list_is_empty(&zio->io_parent_list)) { 2391 zio_t *pio; 2392 2393 /* 2394 * This is a logical async I/O with no parent to wait for it. 2395 * We add it to the spa_async_root_zio "Godfather" I/O which 2396 * will ensure they complete prior to unloading the pool. 2397 */ 2398 spa_t *spa = zio->io_spa; 2399 pio = spa->spa_async_zio_root[CPU_SEQID_UNSTABLE]; 2400 2401 zio_add_child(pio, zio); 2402 } 2403 2404 ASSERT0(zio->io_queued_timestamp); 2405 zio->io_queued_timestamp = gethrtime(); 2406 if (zio->io_type == ZIO_TYPE_WRITE) { 2407 spa_select_allocator(zio); 2408 } 2409 __zio_execute(zio); 2410 } 2411 2412 /* 2413 * ========================================================================== 2414 * Reexecute, cancel, or suspend/resume failed I/O 2415 * ========================================================================== 2416 */ 2417 2418 static void 2419 zio_reexecute(void *arg) 2420 { 2421 zio_t *pio = arg; 2422 zio_t *cio, *cio_next, *gio; 2423 2424 ASSERT(pio->io_child_type == ZIO_CHILD_LOGICAL); 2425 ASSERT(pio->io_orig_stage == ZIO_STAGE_OPEN); 2426 ASSERT(pio->io_gang_leader == NULL); 2427 ASSERT(pio->io_gang_tree == NULL); 2428 2429 mutex_enter(&pio->io_lock); 2430 pio->io_flags = pio->io_orig_flags; 2431 pio->io_stage = pio->io_orig_stage; 2432 pio->io_pipeline = pio->io_orig_pipeline; 2433 pio->io_reexecute = 0; 2434 pio->io_flags |= ZIO_FLAG_REEXECUTED; 2435 pio->io_pipeline_trace = 0; 2436 pio->io_error = 0; 2437 pio->io_state[ZIO_WAIT_READY] = (pio->io_stage >= ZIO_STAGE_READY) || 2438 (pio->io_pipeline & ZIO_STAGE_READY) == 0; 2439 pio->io_state[ZIO_WAIT_DONE] = (pio->io_stage >= ZIO_STAGE_DONE); 2440 zio_link_t *zl = NULL; 2441 while ((gio = zio_walk_parents(pio, &zl)) != NULL) { 2442 for (int w = 0; w < ZIO_WAIT_TYPES; w++) { 2443 gio->io_children[pio->io_child_type][w] += 2444 !pio->io_state[w]; 2445 } 2446 } 2447 for (int c = 0; c < ZIO_CHILD_TYPES; c++) 2448 pio->io_child_error[c] = 0; 2449 2450 if (IO_IS_ALLOCATING(pio)) 2451 BP_ZERO(pio->io_bp); 2452 2453 /* 2454 * As we reexecute pio's children, new children could be created. 2455 * New children go to the head of pio's io_child_list, however, 2456 * so we will (correctly) not reexecute them. The key is that 2457 * the remainder of pio's io_child_list, from 'cio_next' onward, 2458 * cannot be affected by any side effects of reexecuting 'cio'. 2459 */ 2460 zl = NULL; 2461 for (cio = zio_walk_children(pio, &zl); cio != NULL; cio = cio_next) { 2462 cio_next = zio_walk_children(pio, &zl); 2463 mutex_exit(&pio->io_lock); 2464 zio_reexecute(cio); 2465 mutex_enter(&pio->io_lock); 2466 } 2467 mutex_exit(&pio->io_lock); 2468 2469 /* 2470 * Now that all children have been reexecuted, execute the parent. 2471 * We don't reexecute "The Godfather" I/O here as it's the 2472 * responsibility of the caller to wait on it. 2473 */ 2474 if (!(pio->io_flags & ZIO_FLAG_GODFATHER)) { 2475 pio->io_queued_timestamp = gethrtime(); 2476 __zio_execute(pio); 2477 } 2478 } 2479 2480 void 2481 zio_suspend(spa_t *spa, zio_t *zio, zio_suspend_reason_t reason) 2482 { 2483 if (spa_get_failmode(spa) == ZIO_FAILURE_MODE_PANIC) 2484 fm_panic("Pool '%s' has encountered an uncorrectable I/O " 2485 "failure and the failure mode property for this pool " 2486 "is set to panic.", spa_name(spa)); 2487 2488 cmn_err(CE_WARN, "Pool '%s' has encountered an uncorrectable I/O " 2489 "failure and has been suspended.\n", spa_name(spa)); 2490 2491 (void) zfs_ereport_post(FM_EREPORT_ZFS_IO_FAILURE, spa, NULL, 2492 NULL, NULL, 0); 2493 2494 mutex_enter(&spa->spa_suspend_lock); 2495 2496 if (spa->spa_suspend_zio_root == NULL) 2497 spa->spa_suspend_zio_root = zio_root(spa, NULL, NULL, 2498 ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE | 2499 ZIO_FLAG_GODFATHER); 2500 2501 spa->spa_suspended = reason; 2502 2503 if (zio != NULL) { 2504 ASSERT(!(zio->io_flags & ZIO_FLAG_GODFATHER)); 2505 ASSERT(zio != spa->spa_suspend_zio_root); 2506 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL); 2507 ASSERT(zio_unique_parent(zio) == NULL); 2508 ASSERT(zio->io_stage == ZIO_STAGE_DONE); 2509 zio_add_child(spa->spa_suspend_zio_root, zio); 2510 } 2511 2512 mutex_exit(&spa->spa_suspend_lock); 2513 } 2514 2515 int 2516 zio_resume(spa_t *spa) 2517 { 2518 zio_t *pio; 2519 2520 /* 2521 * Reexecute all previously suspended i/o. 2522 */ 2523 mutex_enter(&spa->spa_suspend_lock); 2524 spa->spa_suspended = ZIO_SUSPEND_NONE; 2525 cv_broadcast(&spa->spa_suspend_cv); 2526 pio = spa->spa_suspend_zio_root; 2527 spa->spa_suspend_zio_root = NULL; 2528 mutex_exit(&spa->spa_suspend_lock); 2529 2530 if (pio == NULL) 2531 return (0); 2532 2533 zio_reexecute(pio); 2534 return (zio_wait(pio)); 2535 } 2536 2537 void 2538 zio_resume_wait(spa_t *spa) 2539 { 2540 mutex_enter(&spa->spa_suspend_lock); 2541 while (spa_suspended(spa)) 2542 cv_wait(&spa->spa_suspend_cv, &spa->spa_suspend_lock); 2543 mutex_exit(&spa->spa_suspend_lock); 2544 } 2545 2546 /* 2547 * ========================================================================== 2548 * Gang blocks. 2549 * 2550 * A gang block is a collection of small blocks that looks to the DMU 2551 * like one large block. When zio_dva_allocate() cannot find a block 2552 * of the requested size, due to either severe fragmentation or the pool 2553 * being nearly full, it calls zio_write_gang_block() to construct the 2554 * block from smaller fragments. 2555 * 2556 * A gang block consists of a gang header (zio_gbh_phys_t) and up to 2557 * three (SPA_GBH_NBLKPTRS) gang members. The gang header is just like 2558 * an indirect block: it's an array of block pointers. It consumes 2559 * only one sector and hence is allocatable regardless of fragmentation. 2560 * The gang header's bps point to its gang members, which hold the data. 2561 * 2562 * Gang blocks are self-checksumming, using the bp's <vdev, offset, txg> 2563 * as the verifier to ensure uniqueness of the SHA256 checksum. 2564 * Critically, the gang block bp's blk_cksum is the checksum of the data, 2565 * not the gang header. This ensures that data block signatures (needed for 2566 * deduplication) are independent of how the block is physically stored. 2567 * 2568 * Gang blocks can be nested: a gang member may itself be a gang block. 2569 * Thus every gang block is a tree in which root and all interior nodes are 2570 * gang headers, and the leaves are normal blocks that contain user data. 2571 * The root of the gang tree is called the gang leader. 2572 * 2573 * To perform any operation (read, rewrite, free, claim) on a gang block, 2574 * zio_gang_assemble() first assembles the gang tree (minus data leaves) 2575 * in the io_gang_tree field of the original logical i/o by recursively 2576 * reading the gang leader and all gang headers below it. This yields 2577 * an in-core tree containing the contents of every gang header and the 2578 * bps for every constituent of the gang block. 2579 * 2580 * With the gang tree now assembled, zio_gang_issue() just walks the gang tree 2581 * and invokes a callback on each bp. To free a gang block, zio_gang_issue() 2582 * calls zio_free_gang() -- a trivial wrapper around zio_free() -- for each bp. 2583 * zio_claim_gang() provides a similarly trivial wrapper for zio_claim(). 2584 * zio_read_gang() is a wrapper around zio_read() that omits reading gang 2585 * headers, since we already have those in io_gang_tree. zio_rewrite_gang() 2586 * performs a zio_rewrite() of the data or, for gang headers, a zio_rewrite() 2587 * of the gang header plus zio_checksum_compute() of the data to update the 2588 * gang header's blk_cksum as described above. 2589 * 2590 * The two-phase assemble/issue model solves the problem of partial failure -- 2591 * what if you'd freed part of a gang block but then couldn't read the 2592 * gang header for another part? Assembling the entire gang tree first 2593 * ensures that all the necessary gang header I/O has succeeded before 2594 * starting the actual work of free, claim, or write. Once the gang tree 2595 * is assembled, free and claim are in-memory operations that cannot fail. 2596 * 2597 * In the event that a gang write fails, zio_dva_unallocate() walks the 2598 * gang tree to immediately free (i.e. insert back into the space map) 2599 * everything we've allocated. This ensures that we don't get ENOSPC 2600 * errors during repeated suspend/resume cycles due to a flaky device. 2601 * 2602 * Gang rewrites only happen during sync-to-convergence. If we can't assemble 2603 * the gang tree, we won't modify the block, so we can safely defer the free 2604 * (knowing that the block is still intact). If we *can* assemble the gang 2605 * tree, then even if some of the rewrites fail, zio_dva_unallocate() will free 2606 * each constituent bp and we can allocate a new block on the next sync pass. 2607 * 2608 * In all cases, the gang tree allows complete recovery from partial failure. 2609 * ========================================================================== 2610 */ 2611 2612 static void 2613 zio_gang_issue_func_done(zio_t *zio) 2614 { 2615 abd_free(zio->io_abd); 2616 } 2617 2618 static zio_t * 2619 zio_read_gang(zio_t *pio, blkptr_t *bp, zio_gang_node_t *gn, abd_t *data, 2620 uint64_t offset) 2621 { 2622 if (gn != NULL) 2623 return (pio); 2624 2625 return (zio_read(pio, pio->io_spa, bp, abd_get_offset(data, offset), 2626 BP_GET_PSIZE(bp), zio_gang_issue_func_done, 2627 NULL, pio->io_priority, ZIO_GANG_CHILD_FLAGS(pio), 2628 &pio->io_bookmark)); 2629 } 2630 2631 static zio_t * 2632 zio_rewrite_gang(zio_t *pio, blkptr_t *bp, zio_gang_node_t *gn, abd_t *data, 2633 uint64_t offset) 2634 { 2635 zio_t *zio; 2636 2637 if (gn != NULL) { 2638 abd_t *gbh_abd = 2639 abd_get_from_buf(gn->gn_gbh, SPA_GANGBLOCKSIZE); 2640 zio = zio_rewrite(pio, pio->io_spa, pio->io_txg, bp, 2641 gbh_abd, SPA_GANGBLOCKSIZE, zio_gang_issue_func_done, NULL, 2642 pio->io_priority, ZIO_GANG_CHILD_FLAGS(pio), 2643 &pio->io_bookmark); 2644 /* 2645 * As we rewrite each gang header, the pipeline will compute 2646 * a new gang block header checksum for it; but no one will 2647 * compute a new data checksum, so we do that here. The one 2648 * exception is the gang leader: the pipeline already computed 2649 * its data checksum because that stage precedes gang assembly. 2650 * (Presently, nothing actually uses interior data checksums; 2651 * this is just good hygiene.) 2652 */ 2653 if (gn != pio->io_gang_leader->io_gang_tree) { 2654 abd_t *buf = abd_get_offset(data, offset); 2655 2656 zio_checksum_compute(zio, BP_GET_CHECKSUM(bp), 2657 buf, BP_GET_PSIZE(bp)); 2658 2659 abd_free(buf); 2660 } 2661 /* 2662 * If we are here to damage data for testing purposes, 2663 * leave the GBH alone so that we can detect the damage. 2664 */ 2665 if (pio->io_gang_leader->io_flags & ZIO_FLAG_INDUCE_DAMAGE) 2666 zio->io_pipeline &= ~ZIO_VDEV_IO_STAGES; 2667 } else { 2668 zio = zio_rewrite(pio, pio->io_spa, pio->io_txg, bp, 2669 abd_get_offset(data, offset), BP_GET_PSIZE(bp), 2670 zio_gang_issue_func_done, NULL, pio->io_priority, 2671 ZIO_GANG_CHILD_FLAGS(pio), &pio->io_bookmark); 2672 } 2673 2674 return (zio); 2675 } 2676 2677 static zio_t * 2678 zio_free_gang(zio_t *pio, blkptr_t *bp, zio_gang_node_t *gn, abd_t *data, 2679 uint64_t offset) 2680 { 2681 (void) gn, (void) data, (void) offset; 2682 2683 zio_t *zio = zio_free_sync(pio, pio->io_spa, pio->io_txg, bp, 2684 ZIO_GANG_CHILD_FLAGS(pio)); 2685 if (zio == NULL) { 2686 zio = zio_null(pio, pio->io_spa, 2687 NULL, NULL, NULL, ZIO_GANG_CHILD_FLAGS(pio)); 2688 } 2689 return (zio); 2690 } 2691 2692 static zio_t * 2693 zio_claim_gang(zio_t *pio, blkptr_t *bp, zio_gang_node_t *gn, abd_t *data, 2694 uint64_t offset) 2695 { 2696 (void) gn, (void) data, (void) offset; 2697 return (zio_claim(pio, pio->io_spa, pio->io_txg, bp, 2698 NULL, NULL, ZIO_GANG_CHILD_FLAGS(pio))); 2699 } 2700 2701 static zio_gang_issue_func_t *zio_gang_issue_func[ZIO_TYPES] = { 2702 NULL, 2703 zio_read_gang, 2704 zio_rewrite_gang, 2705 zio_free_gang, 2706 zio_claim_gang, 2707 NULL 2708 }; 2709 2710 static void zio_gang_tree_assemble_done(zio_t *zio); 2711 2712 static zio_gang_node_t * 2713 zio_gang_node_alloc(zio_gang_node_t **gnpp) 2714 { 2715 zio_gang_node_t *gn; 2716 2717 ASSERT(*gnpp == NULL); 2718 2719 gn = kmem_zalloc(sizeof (*gn), KM_SLEEP); 2720 gn->gn_gbh = zio_buf_alloc(SPA_GANGBLOCKSIZE); 2721 *gnpp = gn; 2722 2723 return (gn); 2724 } 2725 2726 static void 2727 zio_gang_node_free(zio_gang_node_t **gnpp) 2728 { 2729 zio_gang_node_t *gn = *gnpp; 2730 2731 for (int g = 0; g < SPA_GBH_NBLKPTRS; g++) 2732 ASSERT(gn->gn_child[g] == NULL); 2733 2734 zio_buf_free(gn->gn_gbh, SPA_GANGBLOCKSIZE); 2735 kmem_free(gn, sizeof (*gn)); 2736 *gnpp = NULL; 2737 } 2738 2739 static void 2740 zio_gang_tree_free(zio_gang_node_t **gnpp) 2741 { 2742 zio_gang_node_t *gn = *gnpp; 2743 2744 if (gn == NULL) 2745 return; 2746 2747 for (int g = 0; g < SPA_GBH_NBLKPTRS; g++) 2748 zio_gang_tree_free(&gn->gn_child[g]); 2749 2750 zio_gang_node_free(gnpp); 2751 } 2752 2753 static void 2754 zio_gang_tree_assemble(zio_t *gio, blkptr_t *bp, zio_gang_node_t **gnpp) 2755 { 2756 zio_gang_node_t *gn = zio_gang_node_alloc(gnpp); 2757 abd_t *gbh_abd = abd_get_from_buf(gn->gn_gbh, SPA_GANGBLOCKSIZE); 2758 2759 ASSERT(gio->io_gang_leader == gio); 2760 ASSERT(BP_IS_GANG(bp)); 2761 2762 zio_nowait(zio_read(gio, gio->io_spa, bp, gbh_abd, SPA_GANGBLOCKSIZE, 2763 zio_gang_tree_assemble_done, gn, gio->io_priority, 2764 ZIO_GANG_CHILD_FLAGS(gio), &gio->io_bookmark)); 2765 } 2766 2767 static void 2768 zio_gang_tree_assemble_done(zio_t *zio) 2769 { 2770 zio_t *gio = zio->io_gang_leader; 2771 zio_gang_node_t *gn = zio->io_private; 2772 blkptr_t *bp = zio->io_bp; 2773 2774 ASSERT(gio == zio_unique_parent(zio)); 2775 ASSERT(list_is_empty(&zio->io_child_list)); 2776 2777 if (zio->io_error) 2778 return; 2779 2780 /* this ABD was created from a linear buf in zio_gang_tree_assemble */ 2781 if (BP_SHOULD_BYTESWAP(bp)) 2782 byteswap_uint64_array(abd_to_buf(zio->io_abd), zio->io_size); 2783 2784 ASSERT3P(abd_to_buf(zio->io_abd), ==, gn->gn_gbh); 2785 ASSERT(zio->io_size == SPA_GANGBLOCKSIZE); 2786 ASSERT(gn->gn_gbh->zg_tail.zec_magic == ZEC_MAGIC); 2787 2788 abd_free(zio->io_abd); 2789 2790 for (int g = 0; g < SPA_GBH_NBLKPTRS; g++) { 2791 blkptr_t *gbp = &gn->gn_gbh->zg_blkptr[g]; 2792 if (!BP_IS_GANG(gbp)) 2793 continue; 2794 zio_gang_tree_assemble(gio, gbp, &gn->gn_child[g]); 2795 } 2796 } 2797 2798 static void 2799 zio_gang_tree_issue(zio_t *pio, zio_gang_node_t *gn, blkptr_t *bp, abd_t *data, 2800 uint64_t offset) 2801 { 2802 zio_t *gio = pio->io_gang_leader; 2803 zio_t *zio; 2804 2805 ASSERT(BP_IS_GANG(bp) == !!gn); 2806 ASSERT(BP_GET_CHECKSUM(bp) == BP_GET_CHECKSUM(gio->io_bp)); 2807 ASSERT(BP_GET_LSIZE(bp) == BP_GET_PSIZE(bp) || gn == gio->io_gang_tree); 2808 2809 /* 2810 * If you're a gang header, your data is in gn->gn_gbh. 2811 * If you're a gang member, your data is in 'data' and gn == NULL. 2812 */ 2813 zio = zio_gang_issue_func[gio->io_type](pio, bp, gn, data, offset); 2814 2815 if (gn != NULL) { 2816 ASSERT(gn->gn_gbh->zg_tail.zec_magic == ZEC_MAGIC); 2817 2818 for (int g = 0; g < SPA_GBH_NBLKPTRS; g++) { 2819 blkptr_t *gbp = &gn->gn_gbh->zg_blkptr[g]; 2820 if (BP_IS_HOLE(gbp)) 2821 continue; 2822 zio_gang_tree_issue(zio, gn->gn_child[g], gbp, data, 2823 offset); 2824 offset += BP_GET_PSIZE(gbp); 2825 } 2826 } 2827 2828 if (gn == gio->io_gang_tree) 2829 ASSERT3U(gio->io_size, ==, offset); 2830 2831 if (zio != pio) 2832 zio_nowait(zio); 2833 } 2834 2835 static zio_t * 2836 zio_gang_assemble(zio_t *zio) 2837 { 2838 blkptr_t *bp = zio->io_bp; 2839 2840 ASSERT(BP_IS_GANG(bp) && zio->io_gang_leader == NULL); 2841 ASSERT(zio->io_child_type > ZIO_CHILD_GANG); 2842 2843 zio->io_gang_leader = zio; 2844 2845 zio_gang_tree_assemble(zio, bp, &zio->io_gang_tree); 2846 2847 return (zio); 2848 } 2849 2850 static zio_t * 2851 zio_gang_issue(zio_t *zio) 2852 { 2853 blkptr_t *bp = zio->io_bp; 2854 2855 if (zio_wait_for_children(zio, ZIO_CHILD_GANG_BIT, ZIO_WAIT_DONE)) { 2856 return (NULL); 2857 } 2858 2859 ASSERT(BP_IS_GANG(bp) && zio->io_gang_leader == zio); 2860 ASSERT(zio->io_child_type > ZIO_CHILD_GANG); 2861 2862 if (zio->io_child_error[ZIO_CHILD_GANG] == 0) 2863 zio_gang_tree_issue(zio, zio->io_gang_tree, bp, zio->io_abd, 2864 0); 2865 else 2866 zio_gang_tree_free(&zio->io_gang_tree); 2867 2868 zio->io_pipeline = ZIO_INTERLOCK_PIPELINE; 2869 2870 return (zio); 2871 } 2872 2873 static void 2874 zio_gang_inherit_allocator(zio_t *pio, zio_t *cio) 2875 { 2876 cio->io_allocator = pio->io_allocator; 2877 cio->io_wr_iss_tq = pio->io_wr_iss_tq; 2878 } 2879 2880 static void 2881 zio_write_gang_member_ready(zio_t *zio) 2882 { 2883 zio_t *pio = zio_unique_parent(zio); 2884 dva_t *cdva = zio->io_bp->blk_dva; 2885 dva_t *pdva = pio->io_bp->blk_dva; 2886 uint64_t asize; 2887 zio_t *gio __maybe_unused = zio->io_gang_leader; 2888 2889 if (BP_IS_HOLE(zio->io_bp)) 2890 return; 2891 2892 ASSERT(BP_IS_HOLE(&zio->io_bp_orig)); 2893 2894 ASSERT(zio->io_child_type == ZIO_CHILD_GANG); 2895 ASSERT3U(zio->io_prop.zp_copies, ==, gio->io_prop.zp_copies); 2896 ASSERT3U(zio->io_prop.zp_copies, <=, BP_GET_NDVAS(zio->io_bp)); 2897 ASSERT3U(pio->io_prop.zp_copies, <=, BP_GET_NDVAS(pio->io_bp)); 2898 VERIFY3U(BP_GET_NDVAS(zio->io_bp), <=, BP_GET_NDVAS(pio->io_bp)); 2899 2900 mutex_enter(&pio->io_lock); 2901 for (int d = 0; d < BP_GET_NDVAS(zio->io_bp); d++) { 2902 ASSERT(DVA_GET_GANG(&pdva[d])); 2903 asize = DVA_GET_ASIZE(&pdva[d]); 2904 asize += DVA_GET_ASIZE(&cdva[d]); 2905 DVA_SET_ASIZE(&pdva[d], asize); 2906 } 2907 mutex_exit(&pio->io_lock); 2908 } 2909 2910 static void 2911 zio_write_gang_done(zio_t *zio) 2912 { 2913 /* 2914 * The io_abd field will be NULL for a zio with no data. The io_flags 2915 * will initially have the ZIO_FLAG_NODATA bit flag set, but we can't 2916 * check for it here as it is cleared in zio_ready. 2917 */ 2918 if (zio->io_abd != NULL) 2919 abd_free(zio->io_abd); 2920 } 2921 2922 static zio_t * 2923 zio_write_gang_block(zio_t *pio, metaslab_class_t *mc) 2924 { 2925 spa_t *spa = pio->io_spa; 2926 blkptr_t *bp = pio->io_bp; 2927 zio_t *gio = pio->io_gang_leader; 2928 zio_t *zio; 2929 zio_gang_node_t *gn, **gnpp; 2930 zio_gbh_phys_t *gbh; 2931 abd_t *gbh_abd; 2932 uint64_t txg = pio->io_txg; 2933 uint64_t resid = pio->io_size; 2934 uint64_t lsize; 2935 int copies = gio->io_prop.zp_copies; 2936 zio_prop_t zp; 2937 int error; 2938 boolean_t has_data = !(pio->io_flags & ZIO_FLAG_NODATA); 2939 2940 /* 2941 * If one copy was requested, store 2 copies of the GBH, so that we 2942 * can still traverse all the data (e.g. to free or scrub) even if a 2943 * block is damaged. Note that we can't store 3 copies of the GBH in 2944 * all cases, e.g. with encryption, which uses DVA[2] for the IV+salt. 2945 */ 2946 int gbh_copies = copies; 2947 if (gbh_copies == 1) { 2948 gbh_copies = MIN(2, spa_max_replication(spa)); 2949 } 2950 2951 ASSERT(ZIO_HAS_ALLOCATOR(pio)); 2952 int flags = METASLAB_HINTBP_FAVOR | METASLAB_GANG_HEADER; 2953 if (pio->io_flags & ZIO_FLAG_IO_ALLOCATING) { 2954 ASSERT(pio->io_priority == ZIO_PRIORITY_ASYNC_WRITE); 2955 ASSERT(has_data); 2956 2957 flags |= METASLAB_ASYNC_ALLOC; 2958 VERIFY(zfs_refcount_held(&mc->mc_allocator[pio->io_allocator]. 2959 mca_alloc_slots, pio)); 2960 2961 /* 2962 * The logical zio has already placed a reservation for 2963 * 'copies' allocation slots but gang blocks may require 2964 * additional copies. These additional copies 2965 * (i.e. gbh_copies - copies) are guaranteed to succeed 2966 * since metaslab_class_throttle_reserve() always allows 2967 * additional reservations for gang blocks. 2968 */ 2969 VERIFY(metaslab_class_throttle_reserve(mc, gbh_copies - copies, 2970 pio->io_allocator, pio, flags)); 2971 } 2972 2973 error = metaslab_alloc(spa, mc, SPA_GANGBLOCKSIZE, 2974 bp, gbh_copies, txg, pio == gio ? NULL : gio->io_bp, flags, 2975 &pio->io_alloc_list, pio, pio->io_allocator); 2976 if (error) { 2977 if (pio->io_flags & ZIO_FLAG_IO_ALLOCATING) { 2978 ASSERT(pio->io_priority == ZIO_PRIORITY_ASYNC_WRITE); 2979 ASSERT(has_data); 2980 2981 /* 2982 * If we failed to allocate the gang block header then 2983 * we remove any additional allocation reservations that 2984 * we placed here. The original reservation will 2985 * be removed when the logical I/O goes to the ready 2986 * stage. 2987 */ 2988 metaslab_class_throttle_unreserve(mc, 2989 gbh_copies - copies, pio->io_allocator, pio); 2990 } 2991 2992 pio->io_error = error; 2993 return (pio); 2994 } 2995 2996 if (pio == gio) { 2997 gnpp = &gio->io_gang_tree; 2998 } else { 2999 gnpp = pio->io_private; 3000 ASSERT(pio->io_ready == zio_write_gang_member_ready); 3001 } 3002 3003 gn = zio_gang_node_alloc(gnpp); 3004 gbh = gn->gn_gbh; 3005 memset(gbh, 0, SPA_GANGBLOCKSIZE); 3006 gbh_abd = abd_get_from_buf(gbh, SPA_GANGBLOCKSIZE); 3007 3008 /* 3009 * Create the gang header. 3010 */ 3011 zio = zio_rewrite(pio, spa, txg, bp, gbh_abd, SPA_GANGBLOCKSIZE, 3012 zio_write_gang_done, NULL, pio->io_priority, 3013 ZIO_GANG_CHILD_FLAGS(pio), &pio->io_bookmark); 3014 3015 zio_gang_inherit_allocator(pio, zio); 3016 3017 /* 3018 * Create and nowait the gang children. 3019 */ 3020 for (int g = 0; resid != 0; resid -= lsize, g++) { 3021 lsize = P2ROUNDUP(resid / (SPA_GBH_NBLKPTRS - g), 3022 SPA_MINBLOCKSIZE); 3023 ASSERT(lsize >= SPA_MINBLOCKSIZE && lsize <= resid); 3024 3025 zp.zp_checksum = gio->io_prop.zp_checksum; 3026 zp.zp_compress = ZIO_COMPRESS_OFF; 3027 zp.zp_complevel = gio->io_prop.zp_complevel; 3028 zp.zp_type = DMU_OT_NONE; 3029 zp.zp_level = 0; 3030 zp.zp_copies = gio->io_prop.zp_copies; 3031 zp.zp_dedup = B_FALSE; 3032 zp.zp_dedup_verify = B_FALSE; 3033 zp.zp_nopwrite = B_FALSE; 3034 zp.zp_encrypt = gio->io_prop.zp_encrypt; 3035 zp.zp_byteorder = gio->io_prop.zp_byteorder; 3036 memset(zp.zp_salt, 0, ZIO_DATA_SALT_LEN); 3037 memset(zp.zp_iv, 0, ZIO_DATA_IV_LEN); 3038 memset(zp.zp_mac, 0, ZIO_DATA_MAC_LEN); 3039 3040 zio_t *cio = zio_write(zio, spa, txg, &gbh->zg_blkptr[g], 3041 has_data ? abd_get_offset(pio->io_abd, pio->io_size - 3042 resid) : NULL, lsize, lsize, &zp, 3043 zio_write_gang_member_ready, NULL, 3044 zio_write_gang_done, &gn->gn_child[g], pio->io_priority, 3045 ZIO_GANG_CHILD_FLAGS(pio), &pio->io_bookmark); 3046 3047 zio_gang_inherit_allocator(zio, cio); 3048 3049 if (pio->io_flags & ZIO_FLAG_IO_ALLOCATING) { 3050 ASSERT(pio->io_priority == ZIO_PRIORITY_ASYNC_WRITE); 3051 ASSERT(has_data); 3052 3053 /* 3054 * Gang children won't throttle but we should 3055 * account for their work, so reserve an allocation 3056 * slot for them here. 3057 */ 3058 VERIFY(metaslab_class_throttle_reserve(mc, 3059 zp.zp_copies, cio->io_allocator, cio, flags)); 3060 } 3061 zio_nowait(cio); 3062 } 3063 3064 /* 3065 * Set pio's pipeline to just wait for zio to finish. 3066 */ 3067 pio->io_pipeline = ZIO_INTERLOCK_PIPELINE; 3068 3069 zio_nowait(zio); 3070 3071 return (pio); 3072 } 3073 3074 /* 3075 * The zio_nop_write stage in the pipeline determines if allocating a 3076 * new bp is necessary. The nopwrite feature can handle writes in 3077 * either syncing or open context (i.e. zil writes) and as a result is 3078 * mutually exclusive with dedup. 3079 * 3080 * By leveraging a cryptographically secure checksum, such as SHA256, we 3081 * can compare the checksums of the new data and the old to determine if 3082 * allocating a new block is required. Note that our requirements for 3083 * cryptographic strength are fairly weak: there can't be any accidental 3084 * hash collisions, but we don't need to be secure against intentional 3085 * (malicious) collisions. To trigger a nopwrite, you have to be able 3086 * to write the file to begin with, and triggering an incorrect (hash 3087 * collision) nopwrite is no worse than simply writing to the file. 3088 * That said, there are no known attacks against the checksum algorithms 3089 * used for nopwrite, assuming that the salt and the checksums 3090 * themselves remain secret. 3091 */ 3092 static zio_t * 3093 zio_nop_write(zio_t *zio) 3094 { 3095 blkptr_t *bp = zio->io_bp; 3096 blkptr_t *bp_orig = &zio->io_bp_orig; 3097 zio_prop_t *zp = &zio->io_prop; 3098 3099 ASSERT(BP_IS_HOLE(bp)); 3100 ASSERT(BP_GET_LEVEL(bp) == 0); 3101 ASSERT(!(zio->io_flags & ZIO_FLAG_IO_REWRITE)); 3102 ASSERT(zp->zp_nopwrite); 3103 ASSERT(!zp->zp_dedup); 3104 ASSERT(zio->io_bp_override == NULL); 3105 ASSERT(IO_IS_ALLOCATING(zio)); 3106 3107 /* 3108 * Check to see if the original bp and the new bp have matching 3109 * characteristics (i.e. same checksum, compression algorithms, etc). 3110 * If they don't then just continue with the pipeline which will 3111 * allocate a new bp. 3112 */ 3113 if (BP_IS_HOLE(bp_orig) || 3114 !(zio_checksum_table[BP_GET_CHECKSUM(bp)].ci_flags & 3115 ZCHECKSUM_FLAG_NOPWRITE) || 3116 BP_IS_ENCRYPTED(bp) || BP_IS_ENCRYPTED(bp_orig) || 3117 BP_GET_CHECKSUM(bp) != BP_GET_CHECKSUM(bp_orig) || 3118 BP_GET_COMPRESS(bp) != BP_GET_COMPRESS(bp_orig) || 3119 BP_GET_DEDUP(bp) != BP_GET_DEDUP(bp_orig) || 3120 zp->zp_copies != BP_GET_NDVAS(bp_orig)) 3121 return (zio); 3122 3123 /* 3124 * If the checksums match then reset the pipeline so that we 3125 * avoid allocating a new bp and issuing any I/O. 3126 */ 3127 if (ZIO_CHECKSUM_EQUAL(bp->blk_cksum, bp_orig->blk_cksum)) { 3128 ASSERT(zio_checksum_table[zp->zp_checksum].ci_flags & 3129 ZCHECKSUM_FLAG_NOPWRITE); 3130 ASSERT3U(BP_GET_PSIZE(bp), ==, BP_GET_PSIZE(bp_orig)); 3131 ASSERT3U(BP_GET_LSIZE(bp), ==, BP_GET_LSIZE(bp_orig)); 3132 ASSERT(zp->zp_compress != ZIO_COMPRESS_OFF); 3133 ASSERT3U(bp->blk_prop, ==, bp_orig->blk_prop); 3134 3135 /* 3136 * If we're overwriting a block that is currently on an 3137 * indirect vdev, then ignore the nopwrite request and 3138 * allow a new block to be allocated on a concrete vdev. 3139 */ 3140 spa_config_enter(zio->io_spa, SCL_VDEV, FTAG, RW_READER); 3141 for (int d = 0; d < BP_GET_NDVAS(bp_orig); d++) { 3142 vdev_t *tvd = vdev_lookup_top(zio->io_spa, 3143 DVA_GET_VDEV(&bp_orig->blk_dva[d])); 3144 if (tvd->vdev_ops == &vdev_indirect_ops) { 3145 spa_config_exit(zio->io_spa, SCL_VDEV, FTAG); 3146 return (zio); 3147 } 3148 } 3149 spa_config_exit(zio->io_spa, SCL_VDEV, FTAG); 3150 3151 *bp = *bp_orig; 3152 zio->io_pipeline = ZIO_INTERLOCK_PIPELINE; 3153 zio->io_flags |= ZIO_FLAG_NOPWRITE; 3154 } 3155 3156 return (zio); 3157 } 3158 3159 /* 3160 * ========================================================================== 3161 * Block Reference Table 3162 * ========================================================================== 3163 */ 3164 static zio_t * 3165 zio_brt_free(zio_t *zio) 3166 { 3167 blkptr_t *bp; 3168 3169 bp = zio->io_bp; 3170 3171 if (BP_GET_LEVEL(bp) > 0 || 3172 BP_IS_METADATA(bp) || 3173 !brt_maybe_exists(zio->io_spa, bp)) { 3174 return (zio); 3175 } 3176 3177 if (!brt_entry_decref(zio->io_spa, bp)) { 3178 /* 3179 * This isn't the last reference, so we cannot free 3180 * the data yet. 3181 */ 3182 zio->io_pipeline = ZIO_INTERLOCK_PIPELINE; 3183 } 3184 3185 return (zio); 3186 } 3187 3188 /* 3189 * ========================================================================== 3190 * Dedup 3191 * ========================================================================== 3192 */ 3193 static void 3194 zio_ddt_child_read_done(zio_t *zio) 3195 { 3196 blkptr_t *bp = zio->io_bp; 3197 ddt_entry_t *dde = zio->io_private; 3198 ddt_phys_t *ddp; 3199 zio_t *pio = zio_unique_parent(zio); 3200 3201 mutex_enter(&pio->io_lock); 3202 ddp = ddt_phys_select(dde, bp); 3203 if (zio->io_error == 0) 3204 ddt_phys_clear(ddp); /* this ddp doesn't need repair */ 3205 3206 if (zio->io_error == 0 && dde->dde_repair_abd == NULL) 3207 dde->dde_repair_abd = zio->io_abd; 3208 else 3209 abd_free(zio->io_abd); 3210 mutex_exit(&pio->io_lock); 3211 } 3212 3213 static zio_t * 3214 zio_ddt_read_start(zio_t *zio) 3215 { 3216 blkptr_t *bp = zio->io_bp; 3217 3218 ASSERT(BP_GET_DEDUP(bp)); 3219 ASSERT(BP_GET_PSIZE(bp) == zio->io_size); 3220 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL); 3221 3222 if (zio->io_child_error[ZIO_CHILD_DDT]) { 3223 ddt_t *ddt = ddt_select(zio->io_spa, bp); 3224 ddt_entry_t *dde = ddt_repair_start(ddt, bp); 3225 ddt_phys_t *ddp = dde->dde_phys; 3226 ddt_phys_t *ddp_self = ddt_phys_select(dde, bp); 3227 blkptr_t blk; 3228 3229 ASSERT(zio->io_vsd == NULL); 3230 zio->io_vsd = dde; 3231 3232 if (ddp_self == NULL) 3233 return (zio); 3234 3235 for (int p = 0; p < DDT_PHYS_TYPES; p++, ddp++) { 3236 if (ddp->ddp_phys_birth == 0 || ddp == ddp_self) 3237 continue; 3238 ddt_bp_create(ddt->ddt_checksum, &dde->dde_key, ddp, 3239 &blk); 3240 zio_nowait(zio_read(zio, zio->io_spa, &blk, 3241 abd_alloc_for_io(zio->io_size, B_TRUE), 3242 zio->io_size, zio_ddt_child_read_done, dde, 3243 zio->io_priority, ZIO_DDT_CHILD_FLAGS(zio) | 3244 ZIO_FLAG_DONT_PROPAGATE, &zio->io_bookmark)); 3245 } 3246 return (zio); 3247 } 3248 3249 zio_nowait(zio_read(zio, zio->io_spa, bp, 3250 zio->io_abd, zio->io_size, NULL, NULL, zio->io_priority, 3251 ZIO_DDT_CHILD_FLAGS(zio), &zio->io_bookmark)); 3252 3253 return (zio); 3254 } 3255 3256 static zio_t * 3257 zio_ddt_read_done(zio_t *zio) 3258 { 3259 blkptr_t *bp = zio->io_bp; 3260 3261 if (zio_wait_for_children(zio, ZIO_CHILD_DDT_BIT, ZIO_WAIT_DONE)) { 3262 return (NULL); 3263 } 3264 3265 ASSERT(BP_GET_DEDUP(bp)); 3266 ASSERT(BP_GET_PSIZE(bp) == zio->io_size); 3267 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL); 3268 3269 if (zio->io_child_error[ZIO_CHILD_DDT]) { 3270 ddt_t *ddt = ddt_select(zio->io_spa, bp); 3271 ddt_entry_t *dde = zio->io_vsd; 3272 if (ddt == NULL) { 3273 ASSERT(spa_load_state(zio->io_spa) != SPA_LOAD_NONE); 3274 return (zio); 3275 } 3276 if (dde == NULL) { 3277 zio->io_stage = ZIO_STAGE_DDT_READ_START >> 1; 3278 zio_taskq_dispatch(zio, ZIO_TASKQ_ISSUE, B_FALSE); 3279 return (NULL); 3280 } 3281 if (dde->dde_repair_abd != NULL) { 3282 abd_copy(zio->io_abd, dde->dde_repair_abd, 3283 zio->io_size); 3284 zio->io_child_error[ZIO_CHILD_DDT] = 0; 3285 } 3286 ddt_repair_done(ddt, dde); 3287 zio->io_vsd = NULL; 3288 } 3289 3290 ASSERT(zio->io_vsd == NULL); 3291 3292 return (zio); 3293 } 3294 3295 static boolean_t 3296 zio_ddt_collision(zio_t *zio, ddt_t *ddt, ddt_entry_t *dde) 3297 { 3298 spa_t *spa = zio->io_spa; 3299 boolean_t do_raw = !!(zio->io_flags & ZIO_FLAG_RAW); 3300 3301 ASSERT(!(zio->io_bp_override && do_raw)); 3302 3303 /* 3304 * Note: we compare the original data, not the transformed data, 3305 * because when zio->io_bp is an override bp, we will not have 3306 * pushed the I/O transforms. That's an important optimization 3307 * because otherwise we'd compress/encrypt all dmu_sync() data twice. 3308 * However, we should never get a raw, override zio so in these 3309 * cases we can compare the io_abd directly. This is useful because 3310 * it allows us to do dedup verification even if we don't have access 3311 * to the original data (for instance, if the encryption keys aren't 3312 * loaded). 3313 */ 3314 3315 for (int p = DDT_PHYS_SINGLE; p <= DDT_PHYS_TRIPLE; p++) { 3316 zio_t *lio = dde->dde_lead_zio[p]; 3317 3318 if (lio != NULL && do_raw) { 3319 return (lio->io_size != zio->io_size || 3320 abd_cmp(zio->io_abd, lio->io_abd) != 0); 3321 } else if (lio != NULL) { 3322 return (lio->io_orig_size != zio->io_orig_size || 3323 abd_cmp(zio->io_orig_abd, lio->io_orig_abd) != 0); 3324 } 3325 } 3326 3327 for (int p = DDT_PHYS_SINGLE; p <= DDT_PHYS_TRIPLE; p++) { 3328 ddt_phys_t *ddp = &dde->dde_phys[p]; 3329 3330 if (ddp->ddp_phys_birth != 0 && do_raw) { 3331 blkptr_t blk = *zio->io_bp; 3332 uint64_t psize; 3333 abd_t *tmpabd; 3334 int error; 3335 3336 ddt_bp_fill(ddp, &blk, ddp->ddp_phys_birth); 3337 psize = BP_GET_PSIZE(&blk); 3338 3339 if (psize != zio->io_size) 3340 return (B_TRUE); 3341 3342 ddt_exit(ddt); 3343 3344 tmpabd = abd_alloc_for_io(psize, B_TRUE); 3345 3346 error = zio_wait(zio_read(NULL, spa, &blk, tmpabd, 3347 psize, NULL, NULL, ZIO_PRIORITY_SYNC_READ, 3348 ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE | 3349 ZIO_FLAG_RAW, &zio->io_bookmark)); 3350 3351 if (error == 0) { 3352 if (abd_cmp(tmpabd, zio->io_abd) != 0) 3353 error = SET_ERROR(ENOENT); 3354 } 3355 3356 abd_free(tmpabd); 3357 ddt_enter(ddt); 3358 return (error != 0); 3359 } else if (ddp->ddp_phys_birth != 0) { 3360 arc_buf_t *abuf = NULL; 3361 arc_flags_t aflags = ARC_FLAG_WAIT; 3362 blkptr_t blk = *zio->io_bp; 3363 int error; 3364 3365 ddt_bp_fill(ddp, &blk, ddp->ddp_phys_birth); 3366 3367 if (BP_GET_LSIZE(&blk) != zio->io_orig_size) 3368 return (B_TRUE); 3369 3370 ddt_exit(ddt); 3371 3372 error = arc_read(NULL, spa, &blk, 3373 arc_getbuf_func, &abuf, ZIO_PRIORITY_SYNC_READ, 3374 ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE, 3375 &aflags, &zio->io_bookmark); 3376 3377 if (error == 0) { 3378 if (abd_cmp_buf(zio->io_orig_abd, abuf->b_data, 3379 zio->io_orig_size) != 0) 3380 error = SET_ERROR(ENOENT); 3381 arc_buf_destroy(abuf, &abuf); 3382 } 3383 3384 ddt_enter(ddt); 3385 return (error != 0); 3386 } 3387 } 3388 3389 return (B_FALSE); 3390 } 3391 3392 static void 3393 zio_ddt_child_write_ready(zio_t *zio) 3394 { 3395 int p = zio->io_prop.zp_copies; 3396 ddt_t *ddt = ddt_select(zio->io_spa, zio->io_bp); 3397 ddt_entry_t *dde = zio->io_private; 3398 ddt_phys_t *ddp = &dde->dde_phys[p]; 3399 zio_t *pio; 3400 3401 if (zio->io_error) 3402 return; 3403 3404 ddt_enter(ddt); 3405 3406 ASSERT(dde->dde_lead_zio[p] == zio); 3407 3408 ddt_phys_fill(ddp, zio->io_bp); 3409 3410 zio_link_t *zl = NULL; 3411 while ((pio = zio_walk_parents(zio, &zl)) != NULL) 3412 ddt_bp_fill(ddp, pio->io_bp, zio->io_txg); 3413 3414 ddt_exit(ddt); 3415 } 3416 3417 static void 3418 zio_ddt_child_write_done(zio_t *zio) 3419 { 3420 int p = zio->io_prop.zp_copies; 3421 ddt_t *ddt = ddt_select(zio->io_spa, zio->io_bp); 3422 ddt_entry_t *dde = zio->io_private; 3423 ddt_phys_t *ddp = &dde->dde_phys[p]; 3424 3425 ddt_enter(ddt); 3426 3427 ASSERT(ddp->ddp_refcnt == 0); 3428 ASSERT(dde->dde_lead_zio[p] == zio); 3429 dde->dde_lead_zio[p] = NULL; 3430 3431 if (zio->io_error == 0) { 3432 zio_link_t *zl = NULL; 3433 while (zio_walk_parents(zio, &zl) != NULL) 3434 ddt_phys_addref(ddp); 3435 } else { 3436 ddt_phys_clear(ddp); 3437 } 3438 3439 ddt_exit(ddt); 3440 } 3441 3442 static zio_t * 3443 zio_ddt_write(zio_t *zio) 3444 { 3445 spa_t *spa = zio->io_spa; 3446 blkptr_t *bp = zio->io_bp; 3447 uint64_t txg = zio->io_txg; 3448 zio_prop_t *zp = &zio->io_prop; 3449 int p = zp->zp_copies; 3450 zio_t *cio = NULL; 3451 ddt_t *ddt = ddt_select(spa, bp); 3452 ddt_entry_t *dde; 3453 ddt_phys_t *ddp; 3454 3455 ASSERT(BP_GET_DEDUP(bp)); 3456 ASSERT(BP_GET_CHECKSUM(bp) == zp->zp_checksum); 3457 ASSERT(BP_IS_HOLE(bp) || zio->io_bp_override); 3458 ASSERT(!(zio->io_bp_override && (zio->io_flags & ZIO_FLAG_RAW))); 3459 3460 ddt_enter(ddt); 3461 dde = ddt_lookup(ddt, bp, B_TRUE); 3462 ddp = &dde->dde_phys[p]; 3463 3464 if (zp->zp_dedup_verify && zio_ddt_collision(zio, ddt, dde)) { 3465 /* 3466 * If we're using a weak checksum, upgrade to a strong checksum 3467 * and try again. If we're already using a strong checksum, 3468 * we can't resolve it, so just convert to an ordinary write. 3469 * (And automatically e-mail a paper to Nature?) 3470 */ 3471 if (!(zio_checksum_table[zp->zp_checksum].ci_flags & 3472 ZCHECKSUM_FLAG_DEDUP)) { 3473 zp->zp_checksum = spa_dedup_checksum(spa); 3474 zio_pop_transforms(zio); 3475 zio->io_stage = ZIO_STAGE_OPEN; 3476 BP_ZERO(bp); 3477 } else { 3478 zp->zp_dedup = B_FALSE; 3479 BP_SET_DEDUP(bp, B_FALSE); 3480 } 3481 ASSERT(!BP_GET_DEDUP(bp)); 3482 zio->io_pipeline = ZIO_WRITE_PIPELINE; 3483 ddt_exit(ddt); 3484 return (zio); 3485 } 3486 3487 if (ddp->ddp_phys_birth != 0 || dde->dde_lead_zio[p] != NULL) { 3488 if (ddp->ddp_phys_birth != 0) 3489 ddt_bp_fill(ddp, bp, txg); 3490 if (dde->dde_lead_zio[p] != NULL) 3491 zio_add_child(zio, dde->dde_lead_zio[p]); 3492 else 3493 ddt_phys_addref(ddp); 3494 } else if (zio->io_bp_override) { 3495 ASSERT(bp->blk_birth == txg); 3496 ASSERT(BP_EQUAL(bp, zio->io_bp_override)); 3497 ddt_phys_fill(ddp, bp); 3498 ddt_phys_addref(ddp); 3499 } else { 3500 cio = zio_write(zio, spa, txg, bp, zio->io_orig_abd, 3501 zio->io_orig_size, zio->io_orig_size, zp, 3502 zio_ddt_child_write_ready, NULL, 3503 zio_ddt_child_write_done, dde, zio->io_priority, 3504 ZIO_DDT_CHILD_FLAGS(zio), &zio->io_bookmark); 3505 3506 zio_push_transform(cio, zio->io_abd, zio->io_size, 0, NULL); 3507 dde->dde_lead_zio[p] = cio; 3508 } 3509 3510 ddt_exit(ddt); 3511 3512 zio_nowait(cio); 3513 3514 return (zio); 3515 } 3516 3517 static ddt_entry_t *freedde; /* for debugging */ 3518 3519 static zio_t * 3520 zio_ddt_free(zio_t *zio) 3521 { 3522 spa_t *spa = zio->io_spa; 3523 blkptr_t *bp = zio->io_bp; 3524 ddt_t *ddt = ddt_select(spa, bp); 3525 ddt_entry_t *dde; 3526 ddt_phys_t *ddp; 3527 3528 ASSERT(BP_GET_DEDUP(bp)); 3529 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL); 3530 3531 ddt_enter(ddt); 3532 freedde = dde = ddt_lookup(ddt, bp, B_TRUE); 3533 if (dde) { 3534 ddp = ddt_phys_select(dde, bp); 3535 if (ddp) 3536 ddt_phys_decref(ddp); 3537 } 3538 ddt_exit(ddt); 3539 3540 return (zio); 3541 } 3542 3543 /* 3544 * ========================================================================== 3545 * Allocate and free blocks 3546 * ========================================================================== 3547 */ 3548 3549 static zio_t * 3550 zio_io_to_allocate(spa_t *spa, int allocator) 3551 { 3552 zio_t *zio; 3553 3554 ASSERT(MUTEX_HELD(&spa->spa_allocs[allocator].spaa_lock)); 3555 3556 zio = avl_first(&spa->spa_allocs[allocator].spaa_tree); 3557 if (zio == NULL) 3558 return (NULL); 3559 3560 ASSERT(IO_IS_ALLOCATING(zio)); 3561 ASSERT(ZIO_HAS_ALLOCATOR(zio)); 3562 3563 /* 3564 * Try to place a reservation for this zio. If we're unable to 3565 * reserve then we throttle. 3566 */ 3567 ASSERT3U(zio->io_allocator, ==, allocator); 3568 if (!metaslab_class_throttle_reserve(zio->io_metaslab_class, 3569 zio->io_prop.zp_copies, allocator, zio, 0)) { 3570 return (NULL); 3571 } 3572 3573 avl_remove(&spa->spa_allocs[allocator].spaa_tree, zio); 3574 ASSERT3U(zio->io_stage, <, ZIO_STAGE_DVA_ALLOCATE); 3575 3576 return (zio); 3577 } 3578 3579 static zio_t * 3580 zio_dva_throttle(zio_t *zio) 3581 { 3582 spa_t *spa = zio->io_spa; 3583 zio_t *nio; 3584 metaslab_class_t *mc; 3585 3586 /* locate an appropriate allocation class */ 3587 mc = spa_preferred_class(spa, zio->io_size, zio->io_prop.zp_type, 3588 zio->io_prop.zp_level, zio->io_prop.zp_zpl_smallblk); 3589 3590 if (zio->io_priority == ZIO_PRIORITY_SYNC_WRITE || 3591 !mc->mc_alloc_throttle_enabled || 3592 zio->io_child_type == ZIO_CHILD_GANG || 3593 zio->io_flags & ZIO_FLAG_NODATA) { 3594 return (zio); 3595 } 3596 3597 ASSERT(zio->io_type == ZIO_TYPE_WRITE); 3598 ASSERT(ZIO_HAS_ALLOCATOR(zio)); 3599 ASSERT(zio->io_child_type > ZIO_CHILD_GANG); 3600 ASSERT3U(zio->io_queued_timestamp, >, 0); 3601 ASSERT(zio->io_stage == ZIO_STAGE_DVA_THROTTLE); 3602 3603 int allocator = zio->io_allocator; 3604 zio->io_metaslab_class = mc; 3605 mutex_enter(&spa->spa_allocs[allocator].spaa_lock); 3606 avl_add(&spa->spa_allocs[allocator].spaa_tree, zio); 3607 nio = zio_io_to_allocate(spa, allocator); 3608 mutex_exit(&spa->spa_allocs[allocator].spaa_lock); 3609 return (nio); 3610 } 3611 3612 static void 3613 zio_allocate_dispatch(spa_t *spa, int allocator) 3614 { 3615 zio_t *zio; 3616 3617 mutex_enter(&spa->spa_allocs[allocator].spaa_lock); 3618 zio = zio_io_to_allocate(spa, allocator); 3619 mutex_exit(&spa->spa_allocs[allocator].spaa_lock); 3620 if (zio == NULL) 3621 return; 3622 3623 ASSERT3U(zio->io_stage, ==, ZIO_STAGE_DVA_THROTTLE); 3624 ASSERT0(zio->io_error); 3625 zio_taskq_dispatch(zio, ZIO_TASKQ_ISSUE, B_TRUE); 3626 } 3627 3628 static zio_t * 3629 zio_dva_allocate(zio_t *zio) 3630 { 3631 spa_t *spa = zio->io_spa; 3632 metaslab_class_t *mc; 3633 blkptr_t *bp = zio->io_bp; 3634 int error; 3635 int flags = 0; 3636 3637 if (zio->io_gang_leader == NULL) { 3638 ASSERT(zio->io_child_type > ZIO_CHILD_GANG); 3639 zio->io_gang_leader = zio; 3640 } 3641 3642 ASSERT(BP_IS_HOLE(bp)); 3643 ASSERT0(BP_GET_NDVAS(bp)); 3644 ASSERT3U(zio->io_prop.zp_copies, >, 0); 3645 ASSERT3U(zio->io_prop.zp_copies, <=, spa_max_replication(spa)); 3646 ASSERT3U(zio->io_size, ==, BP_GET_PSIZE(bp)); 3647 3648 if (zio->io_flags & ZIO_FLAG_NODATA) 3649 flags |= METASLAB_DONT_THROTTLE; 3650 if (zio->io_flags & ZIO_FLAG_GANG_CHILD) 3651 flags |= METASLAB_GANG_CHILD; 3652 if (zio->io_priority == ZIO_PRIORITY_ASYNC_WRITE) 3653 flags |= METASLAB_ASYNC_ALLOC; 3654 3655 /* 3656 * if not already chosen, locate an appropriate allocation class 3657 */ 3658 mc = zio->io_metaslab_class; 3659 if (mc == NULL) { 3660 mc = spa_preferred_class(spa, zio->io_size, 3661 zio->io_prop.zp_type, zio->io_prop.zp_level, 3662 zio->io_prop.zp_zpl_smallblk); 3663 zio->io_metaslab_class = mc; 3664 } 3665 3666 /* 3667 * Try allocating the block in the usual metaslab class. 3668 * If that's full, allocate it in the normal class. 3669 * If that's full, allocate as a gang block, 3670 * and if all are full, the allocation fails (which shouldn't happen). 3671 * 3672 * Note that we do not fall back on embedded slog (ZIL) space, to 3673 * preserve unfragmented slog space, which is critical for decent 3674 * sync write performance. If a log allocation fails, we will fall 3675 * back to spa_sync() which is abysmal for performance. 3676 */ 3677 ASSERT(ZIO_HAS_ALLOCATOR(zio)); 3678 error = metaslab_alloc(spa, mc, zio->io_size, bp, 3679 zio->io_prop.zp_copies, zio->io_txg, NULL, flags, 3680 &zio->io_alloc_list, zio, zio->io_allocator); 3681 3682 /* 3683 * Fallback to normal class when an alloc class is full 3684 */ 3685 if (error == ENOSPC && mc != spa_normal_class(spa)) { 3686 /* 3687 * If throttling, transfer reservation over to normal class. 3688 * The io_allocator slot can remain the same even though we 3689 * are switching classes. 3690 */ 3691 if (mc->mc_alloc_throttle_enabled && 3692 (zio->io_flags & ZIO_FLAG_IO_ALLOCATING)) { 3693 metaslab_class_throttle_unreserve(mc, 3694 zio->io_prop.zp_copies, zio->io_allocator, zio); 3695 zio->io_flags &= ~ZIO_FLAG_IO_ALLOCATING; 3696 3697 VERIFY(metaslab_class_throttle_reserve( 3698 spa_normal_class(spa), 3699 zio->io_prop.zp_copies, zio->io_allocator, zio, 3700 flags | METASLAB_MUST_RESERVE)); 3701 } 3702 zio->io_metaslab_class = mc = spa_normal_class(spa); 3703 if (zfs_flags & ZFS_DEBUG_METASLAB_ALLOC) { 3704 zfs_dbgmsg("%s: metaslab allocation failure, " 3705 "trying normal class: zio %px, size %llu, error %d", 3706 spa_name(spa), zio, (u_longlong_t)zio->io_size, 3707 error); 3708 } 3709 3710 error = metaslab_alloc(spa, mc, zio->io_size, bp, 3711 zio->io_prop.zp_copies, zio->io_txg, NULL, flags, 3712 &zio->io_alloc_list, zio, zio->io_allocator); 3713 } 3714 3715 if (error == ENOSPC && zio->io_size > SPA_MINBLOCKSIZE) { 3716 if (zfs_flags & ZFS_DEBUG_METASLAB_ALLOC) { 3717 zfs_dbgmsg("%s: metaslab allocation failure, " 3718 "trying ganging: zio %px, size %llu, error %d", 3719 spa_name(spa), zio, (u_longlong_t)zio->io_size, 3720 error); 3721 } 3722 return (zio_write_gang_block(zio, mc)); 3723 } 3724 if (error != 0) { 3725 if (error != ENOSPC || 3726 (zfs_flags & ZFS_DEBUG_METASLAB_ALLOC)) { 3727 zfs_dbgmsg("%s: metaslab allocation failure: zio %px, " 3728 "size %llu, error %d", 3729 spa_name(spa), zio, (u_longlong_t)zio->io_size, 3730 error); 3731 } 3732 zio->io_error = error; 3733 } 3734 3735 return (zio); 3736 } 3737 3738 static zio_t * 3739 zio_dva_free(zio_t *zio) 3740 { 3741 metaslab_free(zio->io_spa, zio->io_bp, zio->io_txg, B_FALSE); 3742 3743 return (zio); 3744 } 3745 3746 static zio_t * 3747 zio_dva_claim(zio_t *zio) 3748 { 3749 int error; 3750 3751 error = metaslab_claim(zio->io_spa, zio->io_bp, zio->io_txg); 3752 if (error) 3753 zio->io_error = error; 3754 3755 return (zio); 3756 } 3757 3758 /* 3759 * Undo an allocation. This is used by zio_done() when an I/O fails 3760 * and we want to give back the block we just allocated. 3761 * This handles both normal blocks and gang blocks. 3762 */ 3763 static void 3764 zio_dva_unallocate(zio_t *zio, zio_gang_node_t *gn, blkptr_t *bp) 3765 { 3766 ASSERT(bp->blk_birth == zio->io_txg || BP_IS_HOLE(bp)); 3767 ASSERT(zio->io_bp_override == NULL); 3768 3769 if (!BP_IS_HOLE(bp)) 3770 metaslab_free(zio->io_spa, bp, bp->blk_birth, B_TRUE); 3771 3772 if (gn != NULL) { 3773 for (int g = 0; g < SPA_GBH_NBLKPTRS; g++) { 3774 zio_dva_unallocate(zio, gn->gn_child[g], 3775 &gn->gn_gbh->zg_blkptr[g]); 3776 } 3777 } 3778 } 3779 3780 /* 3781 * Try to allocate an intent log block. Return 0 on success, errno on failure. 3782 */ 3783 int 3784 zio_alloc_zil(spa_t *spa, objset_t *os, uint64_t txg, blkptr_t *new_bp, 3785 uint64_t size, boolean_t *slog) 3786 { 3787 int error = 1; 3788 zio_alloc_list_t io_alloc_list; 3789 3790 ASSERT(txg > spa_syncing_txg(spa)); 3791 3792 metaslab_trace_init(&io_alloc_list); 3793 3794 /* 3795 * Block pointer fields are useful to metaslabs for stats and debugging. 3796 * Fill in the obvious ones before calling into metaslab_alloc(). 3797 */ 3798 BP_SET_TYPE(new_bp, DMU_OT_INTENT_LOG); 3799 BP_SET_PSIZE(new_bp, size); 3800 BP_SET_LEVEL(new_bp, 0); 3801 3802 /* 3803 * When allocating a zil block, we don't have information about 3804 * the final destination of the block except the objset it's part 3805 * of, so we just hash the objset ID to pick the allocator to get 3806 * some parallelism. 3807 */ 3808 int flags = METASLAB_ZIL; 3809 int allocator = (uint_t)cityhash4(0, 0, 0, 3810 os->os_dsl_dataset->ds_object) % spa->spa_alloc_count; 3811 error = metaslab_alloc(spa, spa_log_class(spa), size, new_bp, 1, 3812 txg, NULL, flags, &io_alloc_list, NULL, allocator); 3813 *slog = (error == 0); 3814 if (error != 0) { 3815 error = metaslab_alloc(spa, spa_embedded_log_class(spa), size, 3816 new_bp, 1, txg, NULL, flags, 3817 &io_alloc_list, NULL, allocator); 3818 } 3819 if (error != 0) { 3820 error = metaslab_alloc(spa, spa_normal_class(spa), size, 3821 new_bp, 1, txg, NULL, flags, 3822 &io_alloc_list, NULL, allocator); 3823 } 3824 metaslab_trace_fini(&io_alloc_list); 3825 3826 if (error == 0) { 3827 BP_SET_LSIZE(new_bp, size); 3828 BP_SET_PSIZE(new_bp, size); 3829 BP_SET_COMPRESS(new_bp, ZIO_COMPRESS_OFF); 3830 BP_SET_CHECKSUM(new_bp, 3831 spa_version(spa) >= SPA_VERSION_SLIM_ZIL 3832 ? ZIO_CHECKSUM_ZILOG2 : ZIO_CHECKSUM_ZILOG); 3833 BP_SET_TYPE(new_bp, DMU_OT_INTENT_LOG); 3834 BP_SET_LEVEL(new_bp, 0); 3835 BP_SET_DEDUP(new_bp, 0); 3836 BP_SET_BYTEORDER(new_bp, ZFS_HOST_BYTEORDER); 3837 3838 /* 3839 * encrypted blocks will require an IV and salt. We generate 3840 * these now since we will not be rewriting the bp at 3841 * rewrite time. 3842 */ 3843 if (os->os_encrypted) { 3844 uint8_t iv[ZIO_DATA_IV_LEN]; 3845 uint8_t salt[ZIO_DATA_SALT_LEN]; 3846 3847 BP_SET_CRYPT(new_bp, B_TRUE); 3848 VERIFY0(spa_crypt_get_salt(spa, 3849 dmu_objset_id(os), salt)); 3850 VERIFY0(zio_crypt_generate_iv(iv)); 3851 3852 zio_crypt_encode_params_bp(new_bp, salt, iv); 3853 } 3854 } else { 3855 zfs_dbgmsg("%s: zil block allocation failure: " 3856 "size %llu, error %d", spa_name(spa), (u_longlong_t)size, 3857 error); 3858 } 3859 3860 return (error); 3861 } 3862 3863 /* 3864 * ========================================================================== 3865 * Read and write to physical devices 3866 * ========================================================================== 3867 */ 3868 3869 /* 3870 * Issue an I/O to the underlying vdev. Typically the issue pipeline 3871 * stops after this stage and will resume upon I/O completion. 3872 * However, there are instances where the vdev layer may need to 3873 * continue the pipeline when an I/O was not issued. Since the I/O 3874 * that was sent to the vdev layer might be different than the one 3875 * currently active in the pipeline (see vdev_queue_io()), we explicitly 3876 * force the underlying vdev layers to call either zio_execute() or 3877 * zio_interrupt() to ensure that the pipeline continues with the correct I/O. 3878 */ 3879 static zio_t * 3880 zio_vdev_io_start(zio_t *zio) 3881 { 3882 vdev_t *vd = zio->io_vd; 3883 uint64_t align; 3884 spa_t *spa = zio->io_spa; 3885 3886 zio->io_delay = 0; 3887 3888 ASSERT(zio->io_error == 0); 3889 ASSERT(zio->io_child_error[ZIO_CHILD_VDEV] == 0); 3890 3891 if (vd == NULL) { 3892 if (!(zio->io_flags & ZIO_FLAG_CONFIG_WRITER)) 3893 spa_config_enter(spa, SCL_ZIO, zio, RW_READER); 3894 3895 /* 3896 * The mirror_ops handle multiple DVAs in a single BP. 3897 */ 3898 vdev_mirror_ops.vdev_op_io_start(zio); 3899 return (NULL); 3900 } 3901 3902 ASSERT3P(zio->io_logical, !=, zio); 3903 if (zio->io_type == ZIO_TYPE_WRITE) { 3904 ASSERT(spa->spa_trust_config); 3905 3906 /* 3907 * Note: the code can handle other kinds of writes, 3908 * but we don't expect them. 3909 */ 3910 if (zio->io_vd->vdev_noalloc) { 3911 ASSERT(zio->io_flags & 3912 (ZIO_FLAG_PHYSICAL | ZIO_FLAG_SELF_HEAL | 3913 ZIO_FLAG_RESILVER | ZIO_FLAG_INDUCE_DAMAGE)); 3914 } 3915 } 3916 3917 align = 1ULL << vd->vdev_top->vdev_ashift; 3918 3919 if (!(zio->io_flags & ZIO_FLAG_PHYSICAL) && 3920 P2PHASE(zio->io_size, align) != 0) { 3921 /* Transform logical writes to be a full physical block size. */ 3922 uint64_t asize = P2ROUNDUP(zio->io_size, align); 3923 abd_t *abuf = abd_alloc_sametype(zio->io_abd, asize); 3924 ASSERT(vd == vd->vdev_top); 3925 if (zio->io_type == ZIO_TYPE_WRITE) { 3926 abd_copy(abuf, zio->io_abd, zio->io_size); 3927 abd_zero_off(abuf, zio->io_size, asize - zio->io_size); 3928 } 3929 zio_push_transform(zio, abuf, asize, asize, zio_subblock); 3930 } 3931 3932 /* 3933 * If this is not a physical io, make sure that it is properly aligned 3934 * before proceeding. 3935 */ 3936 if (!(zio->io_flags & ZIO_FLAG_PHYSICAL)) { 3937 ASSERT0(P2PHASE(zio->io_offset, align)); 3938 ASSERT0(P2PHASE(zio->io_size, align)); 3939 } else { 3940 /* 3941 * For physical writes, we allow 512b aligned writes and assume 3942 * the device will perform a read-modify-write as necessary. 3943 */ 3944 ASSERT0(P2PHASE(zio->io_offset, SPA_MINBLOCKSIZE)); 3945 ASSERT0(P2PHASE(zio->io_size, SPA_MINBLOCKSIZE)); 3946 } 3947 3948 VERIFY(zio->io_type != ZIO_TYPE_WRITE || spa_writeable(spa)); 3949 3950 /* 3951 * If this is a repair I/O, and there's no self-healing involved -- 3952 * that is, we're just resilvering what we expect to resilver -- 3953 * then don't do the I/O unless zio's txg is actually in vd's DTL. 3954 * This prevents spurious resilvering. 3955 * 3956 * There are a few ways that we can end up creating these spurious 3957 * resilver i/os: 3958 * 3959 * 1. A resilver i/o will be issued if any DVA in the BP has a 3960 * dirty DTL. The mirror code will issue resilver writes to 3961 * each DVA, including the one(s) that are not on vdevs with dirty 3962 * DTLs. 3963 * 3964 * 2. With nested replication, which happens when we have a 3965 * "replacing" or "spare" vdev that's a child of a mirror or raidz. 3966 * For example, given mirror(replacing(A+B), C), it's likely that 3967 * only A is out of date (it's the new device). In this case, we'll 3968 * read from C, then use the data to resilver A+B -- but we don't 3969 * actually want to resilver B, just A. The top-level mirror has no 3970 * way to know this, so instead we just discard unnecessary repairs 3971 * as we work our way down the vdev tree. 3972 * 3973 * 3. ZTEST also creates mirrors of mirrors, mirrors of raidz, etc. 3974 * The same logic applies to any form of nested replication: ditto 3975 * + mirror, RAID-Z + replacing, etc. 3976 * 3977 * However, indirect vdevs point off to other vdevs which may have 3978 * DTL's, so we never bypass them. The child i/os on concrete vdevs 3979 * will be properly bypassed instead. 3980 * 3981 * Leaf DTL_PARTIAL can be empty when a legitimate write comes from 3982 * a dRAID spare vdev. For example, when a dRAID spare is first 3983 * used, its spare blocks need to be written to but the leaf vdev's 3984 * of such blocks can have empty DTL_PARTIAL. 3985 * 3986 * There seemed no clean way to allow such writes while bypassing 3987 * spurious ones. At this point, just avoid all bypassing for dRAID 3988 * for correctness. 3989 */ 3990 if ((zio->io_flags & ZIO_FLAG_IO_REPAIR) && 3991 !(zio->io_flags & ZIO_FLAG_SELF_HEAL) && 3992 zio->io_txg != 0 && /* not a delegated i/o */ 3993 vd->vdev_ops != &vdev_indirect_ops && 3994 vd->vdev_top->vdev_ops != &vdev_draid_ops && 3995 !vdev_dtl_contains(vd, DTL_PARTIAL, zio->io_txg, 1)) { 3996 ASSERT(zio->io_type == ZIO_TYPE_WRITE); 3997 zio_vdev_io_bypass(zio); 3998 return (zio); 3999 } 4000 4001 /* 4002 * Select the next best leaf I/O to process. Distributed spares are 4003 * excluded since they dispatch the I/O directly to a leaf vdev after 4004 * applying the dRAID mapping. 4005 */ 4006 if (vd->vdev_ops->vdev_op_leaf && 4007 vd->vdev_ops != &vdev_draid_spare_ops && 4008 (zio->io_type == ZIO_TYPE_READ || 4009 zio->io_type == ZIO_TYPE_WRITE || 4010 zio->io_type == ZIO_TYPE_TRIM)) { 4011 4012 if ((zio = vdev_queue_io(zio)) == NULL) 4013 return (NULL); 4014 4015 if (!vdev_accessible(vd, zio)) { 4016 zio->io_error = SET_ERROR(ENXIO); 4017 zio_interrupt(zio); 4018 return (NULL); 4019 } 4020 zio->io_delay = gethrtime(); 4021 } 4022 4023 vd->vdev_ops->vdev_op_io_start(zio); 4024 return (NULL); 4025 } 4026 4027 static zio_t * 4028 zio_vdev_io_done(zio_t *zio) 4029 { 4030 vdev_t *vd = zio->io_vd; 4031 vdev_ops_t *ops = vd ? vd->vdev_ops : &vdev_mirror_ops; 4032 boolean_t unexpected_error = B_FALSE; 4033 4034 if (zio_wait_for_children(zio, ZIO_CHILD_VDEV_BIT, ZIO_WAIT_DONE)) { 4035 return (NULL); 4036 } 4037 4038 ASSERT(zio->io_type == ZIO_TYPE_READ || 4039 zio->io_type == ZIO_TYPE_WRITE || zio->io_type == ZIO_TYPE_TRIM); 4040 4041 if (zio->io_delay) 4042 zio->io_delay = gethrtime() - zio->io_delay; 4043 4044 if (vd != NULL && vd->vdev_ops->vdev_op_leaf && 4045 vd->vdev_ops != &vdev_draid_spare_ops) { 4046 vdev_queue_io_done(zio); 4047 4048 if (zio_injection_enabled && zio->io_error == 0) 4049 zio->io_error = zio_handle_device_injections(vd, zio, 4050 EIO, EILSEQ); 4051 4052 if (zio_injection_enabled && zio->io_error == 0) 4053 zio->io_error = zio_handle_label_injection(zio, EIO); 4054 4055 if (zio->io_error && zio->io_type != ZIO_TYPE_TRIM) { 4056 if (!vdev_accessible(vd, zio)) { 4057 zio->io_error = SET_ERROR(ENXIO); 4058 } else { 4059 unexpected_error = B_TRUE; 4060 } 4061 } 4062 } 4063 4064 ops->vdev_op_io_done(zio); 4065 4066 if (unexpected_error && vd->vdev_remove_wanted == B_FALSE) 4067 VERIFY(vdev_probe(vd, zio) == NULL); 4068 4069 return (zio); 4070 } 4071 4072 /* 4073 * This function is used to change the priority of an existing zio that is 4074 * currently in-flight. This is used by the arc to upgrade priority in the 4075 * event that a demand read is made for a block that is currently queued 4076 * as a scrub or async read IO. Otherwise, the high priority read request 4077 * would end up having to wait for the lower priority IO. 4078 */ 4079 void 4080 zio_change_priority(zio_t *pio, zio_priority_t priority) 4081 { 4082 zio_t *cio, *cio_next; 4083 zio_link_t *zl = NULL; 4084 4085 ASSERT3U(priority, <, ZIO_PRIORITY_NUM_QUEUEABLE); 4086 4087 if (pio->io_vd != NULL && pio->io_vd->vdev_ops->vdev_op_leaf) { 4088 vdev_queue_change_io_priority(pio, priority); 4089 } else { 4090 pio->io_priority = priority; 4091 } 4092 4093 mutex_enter(&pio->io_lock); 4094 for (cio = zio_walk_children(pio, &zl); cio != NULL; cio = cio_next) { 4095 cio_next = zio_walk_children(pio, &zl); 4096 zio_change_priority(cio, priority); 4097 } 4098 mutex_exit(&pio->io_lock); 4099 } 4100 4101 /* 4102 * For non-raidz ZIOs, we can just copy aside the bad data read from the 4103 * disk, and use that to finish the checksum ereport later. 4104 */ 4105 static void 4106 zio_vsd_default_cksum_finish(zio_cksum_report_t *zcr, 4107 const abd_t *good_buf) 4108 { 4109 /* no processing needed */ 4110 zfs_ereport_finish_checksum(zcr, good_buf, zcr->zcr_cbdata, B_FALSE); 4111 } 4112 4113 void 4114 zio_vsd_default_cksum_report(zio_t *zio, zio_cksum_report_t *zcr) 4115 { 4116 void *abd = abd_alloc_sametype(zio->io_abd, zio->io_size); 4117 4118 abd_copy(abd, zio->io_abd, zio->io_size); 4119 4120 zcr->zcr_cbinfo = zio->io_size; 4121 zcr->zcr_cbdata = abd; 4122 zcr->zcr_finish = zio_vsd_default_cksum_finish; 4123 zcr->zcr_free = zio_abd_free; 4124 } 4125 4126 static zio_t * 4127 zio_vdev_io_assess(zio_t *zio) 4128 { 4129 vdev_t *vd = zio->io_vd; 4130 4131 if (zio_wait_for_children(zio, ZIO_CHILD_VDEV_BIT, ZIO_WAIT_DONE)) { 4132 return (NULL); 4133 } 4134 4135 if (vd == NULL && !(zio->io_flags & ZIO_FLAG_CONFIG_WRITER)) 4136 spa_config_exit(zio->io_spa, SCL_ZIO, zio); 4137 4138 if (zio->io_vsd != NULL) { 4139 zio->io_vsd_ops->vsd_free(zio); 4140 zio->io_vsd = NULL; 4141 } 4142 4143 if (zio_injection_enabled && zio->io_error == 0) 4144 zio->io_error = zio_handle_fault_injection(zio, EIO); 4145 4146 /* 4147 * If the I/O failed, determine whether we should attempt to retry it. 4148 * 4149 * On retry, we cut in line in the issue queue, since we don't want 4150 * compression/checksumming/etc. work to prevent our (cheap) IO reissue. 4151 */ 4152 if (zio->io_error && vd == NULL && 4153 !(zio->io_flags & (ZIO_FLAG_DONT_RETRY | ZIO_FLAG_IO_RETRY))) { 4154 ASSERT(!(zio->io_flags & ZIO_FLAG_DONT_QUEUE)); /* not a leaf */ 4155 ASSERT(!(zio->io_flags & ZIO_FLAG_IO_BYPASS)); /* not a leaf */ 4156 zio->io_error = 0; 4157 zio->io_flags |= ZIO_FLAG_IO_RETRY | ZIO_FLAG_DONT_AGGREGATE; 4158 zio->io_stage = ZIO_STAGE_VDEV_IO_START >> 1; 4159 zio_taskq_dispatch(zio, ZIO_TASKQ_ISSUE, 4160 zio_requeue_io_start_cut_in_line); 4161 return (NULL); 4162 } 4163 4164 /* 4165 * If we got an error on a leaf device, convert it to ENXIO 4166 * if the device is not accessible at all. 4167 */ 4168 if (zio->io_error && vd != NULL && vd->vdev_ops->vdev_op_leaf && 4169 !vdev_accessible(vd, zio)) 4170 zio->io_error = SET_ERROR(ENXIO); 4171 4172 /* 4173 * If we can't write to an interior vdev (mirror or RAID-Z), 4174 * set vdev_cant_write so that we stop trying to allocate from it. 4175 */ 4176 if (zio->io_error == ENXIO && zio->io_type == ZIO_TYPE_WRITE && 4177 vd != NULL && !vd->vdev_ops->vdev_op_leaf) { 4178 vdev_dbgmsg(vd, "zio_vdev_io_assess(zio=%px) setting " 4179 "cant_write=TRUE due to write failure with ENXIO", 4180 zio); 4181 vd->vdev_cant_write = B_TRUE; 4182 } 4183 4184 /* 4185 * If a cache flush returns ENOTSUP or ENOTTY, we know that no future 4186 * attempts will ever succeed. In this case we set a persistent 4187 * boolean flag so that we don't bother with it in the future. 4188 */ 4189 if ((zio->io_error == ENOTSUP || zio->io_error == ENOTTY) && 4190 zio->io_type == ZIO_TYPE_IOCTL && 4191 zio->io_cmd == DKIOCFLUSHWRITECACHE && vd != NULL) 4192 vd->vdev_nowritecache = B_TRUE; 4193 4194 if (zio->io_error) 4195 zio->io_pipeline = ZIO_INTERLOCK_PIPELINE; 4196 4197 return (zio); 4198 } 4199 4200 void 4201 zio_vdev_io_reissue(zio_t *zio) 4202 { 4203 ASSERT(zio->io_stage == ZIO_STAGE_VDEV_IO_START); 4204 ASSERT(zio->io_error == 0); 4205 4206 zio->io_stage >>= 1; 4207 } 4208 4209 void 4210 zio_vdev_io_redone(zio_t *zio) 4211 { 4212 ASSERT(zio->io_stage == ZIO_STAGE_VDEV_IO_DONE); 4213 4214 zio->io_stage >>= 1; 4215 } 4216 4217 void 4218 zio_vdev_io_bypass(zio_t *zio) 4219 { 4220 ASSERT(zio->io_stage == ZIO_STAGE_VDEV_IO_START); 4221 ASSERT(zio->io_error == 0); 4222 4223 zio->io_flags |= ZIO_FLAG_IO_BYPASS; 4224 zio->io_stage = ZIO_STAGE_VDEV_IO_ASSESS >> 1; 4225 } 4226 4227 /* 4228 * ========================================================================== 4229 * Encrypt and store encryption parameters 4230 * ========================================================================== 4231 */ 4232 4233 4234 /* 4235 * This function is used for ZIO_STAGE_ENCRYPT. It is responsible for 4236 * managing the storage of encryption parameters and passing them to the 4237 * lower-level encryption functions. 4238 */ 4239 static zio_t * 4240 zio_encrypt(zio_t *zio) 4241 { 4242 zio_prop_t *zp = &zio->io_prop; 4243 spa_t *spa = zio->io_spa; 4244 blkptr_t *bp = zio->io_bp; 4245 uint64_t psize = BP_GET_PSIZE(bp); 4246 uint64_t dsobj = zio->io_bookmark.zb_objset; 4247 dmu_object_type_t ot = BP_GET_TYPE(bp); 4248 void *enc_buf = NULL; 4249 abd_t *eabd = NULL; 4250 uint8_t salt[ZIO_DATA_SALT_LEN]; 4251 uint8_t iv[ZIO_DATA_IV_LEN]; 4252 uint8_t mac[ZIO_DATA_MAC_LEN]; 4253 boolean_t no_crypt = B_FALSE; 4254 4255 /* the root zio already encrypted the data */ 4256 if (zio->io_child_type == ZIO_CHILD_GANG) 4257 return (zio); 4258 4259 /* only ZIL blocks are re-encrypted on rewrite */ 4260 if (!IO_IS_ALLOCATING(zio) && ot != DMU_OT_INTENT_LOG) 4261 return (zio); 4262 4263 if (!(zp->zp_encrypt || BP_IS_ENCRYPTED(bp))) { 4264 BP_SET_CRYPT(bp, B_FALSE); 4265 return (zio); 4266 } 4267 4268 /* if we are doing raw encryption set the provided encryption params */ 4269 if (zio->io_flags & ZIO_FLAG_RAW_ENCRYPT) { 4270 ASSERT0(BP_GET_LEVEL(bp)); 4271 BP_SET_CRYPT(bp, B_TRUE); 4272 BP_SET_BYTEORDER(bp, zp->zp_byteorder); 4273 if (ot != DMU_OT_OBJSET) 4274 zio_crypt_encode_mac_bp(bp, zp->zp_mac); 4275 4276 /* dnode blocks must be written out in the provided byteorder */ 4277 if (zp->zp_byteorder != ZFS_HOST_BYTEORDER && 4278 ot == DMU_OT_DNODE) { 4279 void *bswap_buf = zio_buf_alloc(psize); 4280 abd_t *babd = abd_get_from_buf(bswap_buf, psize); 4281 4282 ASSERT3U(BP_GET_COMPRESS(bp), ==, ZIO_COMPRESS_OFF); 4283 abd_copy_to_buf(bswap_buf, zio->io_abd, psize); 4284 dmu_ot_byteswap[DMU_OT_BYTESWAP(ot)].ob_func(bswap_buf, 4285 psize); 4286 4287 abd_take_ownership_of_buf(babd, B_TRUE); 4288 zio_push_transform(zio, babd, psize, psize, NULL); 4289 } 4290 4291 if (DMU_OT_IS_ENCRYPTED(ot)) 4292 zio_crypt_encode_params_bp(bp, zp->zp_salt, zp->zp_iv); 4293 return (zio); 4294 } 4295 4296 /* indirect blocks only maintain a cksum of the lower level MACs */ 4297 if (BP_GET_LEVEL(bp) > 0) { 4298 BP_SET_CRYPT(bp, B_TRUE); 4299 VERIFY0(zio_crypt_do_indirect_mac_checksum_abd(B_TRUE, 4300 zio->io_orig_abd, BP_GET_LSIZE(bp), BP_SHOULD_BYTESWAP(bp), 4301 mac)); 4302 zio_crypt_encode_mac_bp(bp, mac); 4303 return (zio); 4304 } 4305 4306 /* 4307 * Objset blocks are a special case since they have 2 256-bit MACs 4308 * embedded within them. 4309 */ 4310 if (ot == DMU_OT_OBJSET) { 4311 ASSERT0(DMU_OT_IS_ENCRYPTED(ot)); 4312 ASSERT3U(BP_GET_COMPRESS(bp), ==, ZIO_COMPRESS_OFF); 4313 BP_SET_CRYPT(bp, B_TRUE); 4314 VERIFY0(spa_do_crypt_objset_mac_abd(B_TRUE, spa, dsobj, 4315 zio->io_abd, psize, BP_SHOULD_BYTESWAP(bp))); 4316 return (zio); 4317 } 4318 4319 /* unencrypted object types are only authenticated with a MAC */ 4320 if (!DMU_OT_IS_ENCRYPTED(ot)) { 4321 BP_SET_CRYPT(bp, B_TRUE); 4322 VERIFY0(spa_do_crypt_mac_abd(B_TRUE, spa, dsobj, 4323 zio->io_abd, psize, mac)); 4324 zio_crypt_encode_mac_bp(bp, mac); 4325 return (zio); 4326 } 4327 4328 /* 4329 * Later passes of sync-to-convergence may decide to rewrite data 4330 * in place to avoid more disk reallocations. This presents a problem 4331 * for encryption because this constitutes rewriting the new data with 4332 * the same encryption key and IV. However, this only applies to blocks 4333 * in the MOS (particularly the spacemaps) and we do not encrypt the 4334 * MOS. We assert that the zio is allocating or an intent log write 4335 * to enforce this. 4336 */ 4337 ASSERT(IO_IS_ALLOCATING(zio) || ot == DMU_OT_INTENT_LOG); 4338 ASSERT(BP_GET_LEVEL(bp) == 0 || ot == DMU_OT_INTENT_LOG); 4339 ASSERT(spa_feature_is_active(spa, SPA_FEATURE_ENCRYPTION)); 4340 ASSERT3U(psize, !=, 0); 4341 4342 enc_buf = zio_buf_alloc(psize); 4343 eabd = abd_get_from_buf(enc_buf, psize); 4344 abd_take_ownership_of_buf(eabd, B_TRUE); 4345 4346 /* 4347 * For an explanation of what encryption parameters are stored 4348 * where, see the block comment in zio_crypt.c. 4349 */ 4350 if (ot == DMU_OT_INTENT_LOG) { 4351 zio_crypt_decode_params_bp(bp, salt, iv); 4352 } else { 4353 BP_SET_CRYPT(bp, B_TRUE); 4354 } 4355 4356 /* Perform the encryption. This should not fail */ 4357 VERIFY0(spa_do_crypt_abd(B_TRUE, spa, &zio->io_bookmark, 4358 BP_GET_TYPE(bp), BP_GET_DEDUP(bp), BP_SHOULD_BYTESWAP(bp), 4359 salt, iv, mac, psize, zio->io_abd, eabd, &no_crypt)); 4360 4361 /* encode encryption metadata into the bp */ 4362 if (ot == DMU_OT_INTENT_LOG) { 4363 /* 4364 * ZIL blocks store the MAC in the embedded checksum, so the 4365 * transform must always be applied. 4366 */ 4367 zio_crypt_encode_mac_zil(enc_buf, mac); 4368 zio_push_transform(zio, eabd, psize, psize, NULL); 4369 } else { 4370 BP_SET_CRYPT(bp, B_TRUE); 4371 zio_crypt_encode_params_bp(bp, salt, iv); 4372 zio_crypt_encode_mac_bp(bp, mac); 4373 4374 if (no_crypt) { 4375 ASSERT3U(ot, ==, DMU_OT_DNODE); 4376 abd_free(eabd); 4377 } else { 4378 zio_push_transform(zio, eabd, psize, psize, NULL); 4379 } 4380 } 4381 4382 return (zio); 4383 } 4384 4385 /* 4386 * ========================================================================== 4387 * Generate and verify checksums 4388 * ========================================================================== 4389 */ 4390 static zio_t * 4391 zio_checksum_generate(zio_t *zio) 4392 { 4393 blkptr_t *bp = zio->io_bp; 4394 enum zio_checksum checksum; 4395 4396 if (bp == NULL) { 4397 /* 4398 * This is zio_write_phys(). 4399 * We're either generating a label checksum, or none at all. 4400 */ 4401 checksum = zio->io_prop.zp_checksum; 4402 4403 if (checksum == ZIO_CHECKSUM_OFF) 4404 return (zio); 4405 4406 ASSERT(checksum == ZIO_CHECKSUM_LABEL); 4407 } else { 4408 if (BP_IS_GANG(bp) && zio->io_child_type == ZIO_CHILD_GANG) { 4409 ASSERT(!IO_IS_ALLOCATING(zio)); 4410 checksum = ZIO_CHECKSUM_GANG_HEADER; 4411 } else { 4412 checksum = BP_GET_CHECKSUM(bp); 4413 } 4414 } 4415 4416 zio_checksum_compute(zio, checksum, zio->io_abd, zio->io_size); 4417 4418 return (zio); 4419 } 4420 4421 static zio_t * 4422 zio_checksum_verify(zio_t *zio) 4423 { 4424 zio_bad_cksum_t info; 4425 blkptr_t *bp = zio->io_bp; 4426 int error; 4427 4428 ASSERT(zio->io_vd != NULL); 4429 4430 if (bp == NULL) { 4431 /* 4432 * This is zio_read_phys(). 4433 * We're either verifying a label checksum, or nothing at all. 4434 */ 4435 if (zio->io_prop.zp_checksum == ZIO_CHECKSUM_OFF) 4436 return (zio); 4437 4438 ASSERT3U(zio->io_prop.zp_checksum, ==, ZIO_CHECKSUM_LABEL); 4439 } 4440 4441 if ((error = zio_checksum_error(zio, &info)) != 0) { 4442 zio->io_error = error; 4443 if (error == ECKSUM && 4444 !(zio->io_flags & ZIO_FLAG_SPECULATIVE)) { 4445 mutex_enter(&zio->io_vd->vdev_stat_lock); 4446 zio->io_vd->vdev_stat.vs_checksum_errors++; 4447 mutex_exit(&zio->io_vd->vdev_stat_lock); 4448 (void) zfs_ereport_start_checksum(zio->io_spa, 4449 zio->io_vd, &zio->io_bookmark, zio, 4450 zio->io_offset, zio->io_size, &info); 4451 } 4452 } 4453 4454 return (zio); 4455 } 4456 4457 /* 4458 * Called by RAID-Z to ensure we don't compute the checksum twice. 4459 */ 4460 void 4461 zio_checksum_verified(zio_t *zio) 4462 { 4463 zio->io_pipeline &= ~ZIO_STAGE_CHECKSUM_VERIFY; 4464 } 4465 4466 /* 4467 * ========================================================================== 4468 * Error rank. Error are ranked in the order 0, ENXIO, ECKSUM, EIO, other. 4469 * An error of 0 indicates success. ENXIO indicates whole-device failure, 4470 * which may be transient (e.g. unplugged) or permanent. ECKSUM and EIO 4471 * indicate errors that are specific to one I/O, and most likely permanent. 4472 * Any other error is presumed to be worse because we weren't expecting it. 4473 * ========================================================================== 4474 */ 4475 int 4476 zio_worst_error(int e1, int e2) 4477 { 4478 static int zio_error_rank[] = { 0, ENXIO, ECKSUM, EIO }; 4479 int r1, r2; 4480 4481 for (r1 = 0; r1 < sizeof (zio_error_rank) / sizeof (int); r1++) 4482 if (e1 == zio_error_rank[r1]) 4483 break; 4484 4485 for (r2 = 0; r2 < sizeof (zio_error_rank) / sizeof (int); r2++) 4486 if (e2 == zio_error_rank[r2]) 4487 break; 4488 4489 return (r1 > r2 ? e1 : e2); 4490 } 4491 4492 /* 4493 * ========================================================================== 4494 * I/O completion 4495 * ========================================================================== 4496 */ 4497 static zio_t * 4498 zio_ready(zio_t *zio) 4499 { 4500 blkptr_t *bp = zio->io_bp; 4501 zio_t *pio, *pio_next; 4502 zio_link_t *zl = NULL; 4503 4504 if (zio_wait_for_children(zio, ZIO_CHILD_LOGICAL_BIT | 4505 ZIO_CHILD_GANG_BIT | ZIO_CHILD_DDT_BIT, ZIO_WAIT_READY)) { 4506 return (NULL); 4507 } 4508 4509 if (zio->io_ready) { 4510 ASSERT(IO_IS_ALLOCATING(zio)); 4511 ASSERT(bp->blk_birth == zio->io_txg || BP_IS_HOLE(bp) || 4512 (zio->io_flags & ZIO_FLAG_NOPWRITE)); 4513 ASSERT(zio->io_children[ZIO_CHILD_GANG][ZIO_WAIT_READY] == 0); 4514 4515 zio->io_ready(zio); 4516 } 4517 4518 #ifdef ZFS_DEBUG 4519 if (bp != NULL && bp != &zio->io_bp_copy) 4520 zio->io_bp_copy = *bp; 4521 #endif 4522 4523 if (zio->io_error != 0) { 4524 zio->io_pipeline = ZIO_INTERLOCK_PIPELINE; 4525 4526 if (zio->io_flags & ZIO_FLAG_IO_ALLOCATING) { 4527 ASSERT(IO_IS_ALLOCATING(zio)); 4528 ASSERT(zio->io_priority == ZIO_PRIORITY_ASYNC_WRITE); 4529 ASSERT(zio->io_metaslab_class != NULL); 4530 ASSERT(ZIO_HAS_ALLOCATOR(zio)); 4531 4532 /* 4533 * We were unable to allocate anything, unreserve and 4534 * issue the next I/O to allocate. 4535 */ 4536 metaslab_class_throttle_unreserve( 4537 zio->io_metaslab_class, zio->io_prop.zp_copies, 4538 zio->io_allocator, zio); 4539 zio_allocate_dispatch(zio->io_spa, zio->io_allocator); 4540 } 4541 } 4542 4543 mutex_enter(&zio->io_lock); 4544 zio->io_state[ZIO_WAIT_READY] = 1; 4545 pio = zio_walk_parents(zio, &zl); 4546 mutex_exit(&zio->io_lock); 4547 4548 /* 4549 * As we notify zio's parents, new parents could be added. 4550 * New parents go to the head of zio's io_parent_list, however, 4551 * so we will (correctly) not notify them. The remainder of zio's 4552 * io_parent_list, from 'pio_next' onward, cannot change because 4553 * all parents must wait for us to be done before they can be done. 4554 */ 4555 for (; pio != NULL; pio = pio_next) { 4556 pio_next = zio_walk_parents(zio, &zl); 4557 zio_notify_parent(pio, zio, ZIO_WAIT_READY, NULL); 4558 } 4559 4560 if (zio->io_flags & ZIO_FLAG_NODATA) { 4561 if (bp != NULL && BP_IS_GANG(bp)) { 4562 zio->io_flags &= ~ZIO_FLAG_NODATA; 4563 } else { 4564 ASSERT((uintptr_t)zio->io_abd < SPA_MAXBLOCKSIZE); 4565 zio->io_pipeline &= ~ZIO_VDEV_IO_STAGES; 4566 } 4567 } 4568 4569 if (zio_injection_enabled && 4570 zio->io_spa->spa_syncing_txg == zio->io_txg) 4571 zio_handle_ignored_writes(zio); 4572 4573 return (zio); 4574 } 4575 4576 /* 4577 * Update the allocation throttle accounting. 4578 */ 4579 static void 4580 zio_dva_throttle_done(zio_t *zio) 4581 { 4582 zio_t *lio __maybe_unused = zio->io_logical; 4583 zio_t *pio = zio_unique_parent(zio); 4584 vdev_t *vd = zio->io_vd; 4585 int flags = METASLAB_ASYNC_ALLOC; 4586 4587 ASSERT3P(zio->io_bp, !=, NULL); 4588 ASSERT3U(zio->io_type, ==, ZIO_TYPE_WRITE); 4589 ASSERT3U(zio->io_priority, ==, ZIO_PRIORITY_ASYNC_WRITE); 4590 ASSERT3U(zio->io_child_type, ==, ZIO_CHILD_VDEV); 4591 ASSERT(vd != NULL); 4592 ASSERT3P(vd, ==, vd->vdev_top); 4593 ASSERT(zio_injection_enabled || !(zio->io_flags & ZIO_FLAG_IO_RETRY)); 4594 ASSERT(!(zio->io_flags & ZIO_FLAG_IO_REPAIR)); 4595 ASSERT(zio->io_flags & ZIO_FLAG_IO_ALLOCATING); 4596 ASSERT(!(lio->io_flags & ZIO_FLAG_IO_REWRITE)); 4597 ASSERT(!(lio->io_orig_flags & ZIO_FLAG_NODATA)); 4598 4599 /* 4600 * Parents of gang children can have two flavors -- ones that 4601 * allocated the gang header (will have ZIO_FLAG_IO_REWRITE set) 4602 * and ones that allocated the constituent blocks. The allocation 4603 * throttle needs to know the allocating parent zio so we must find 4604 * it here. 4605 */ 4606 if (pio->io_child_type == ZIO_CHILD_GANG) { 4607 /* 4608 * If our parent is a rewrite gang child then our grandparent 4609 * would have been the one that performed the allocation. 4610 */ 4611 if (pio->io_flags & ZIO_FLAG_IO_REWRITE) 4612 pio = zio_unique_parent(pio); 4613 flags |= METASLAB_GANG_CHILD; 4614 } 4615 4616 ASSERT(IO_IS_ALLOCATING(pio)); 4617 ASSERT(ZIO_HAS_ALLOCATOR(pio)); 4618 ASSERT3P(zio, !=, zio->io_logical); 4619 ASSERT(zio->io_logical != NULL); 4620 ASSERT(!(zio->io_flags & ZIO_FLAG_IO_REPAIR)); 4621 ASSERT0(zio->io_flags & ZIO_FLAG_NOPWRITE); 4622 ASSERT(zio->io_metaslab_class != NULL); 4623 4624 mutex_enter(&pio->io_lock); 4625 metaslab_group_alloc_decrement(zio->io_spa, vd->vdev_id, pio, flags, 4626 pio->io_allocator, B_TRUE); 4627 mutex_exit(&pio->io_lock); 4628 4629 metaslab_class_throttle_unreserve(zio->io_metaslab_class, 1, 4630 pio->io_allocator, pio); 4631 4632 /* 4633 * Call into the pipeline to see if there is more work that 4634 * needs to be done. If there is work to be done it will be 4635 * dispatched to another taskq thread. 4636 */ 4637 zio_allocate_dispatch(zio->io_spa, pio->io_allocator); 4638 } 4639 4640 static zio_t * 4641 zio_done(zio_t *zio) 4642 { 4643 /* 4644 * Always attempt to keep stack usage minimal here since 4645 * we can be called recursively up to 19 levels deep. 4646 */ 4647 const uint64_t psize = zio->io_size; 4648 zio_t *pio, *pio_next; 4649 zio_link_t *zl = NULL; 4650 4651 /* 4652 * If our children haven't all completed, 4653 * wait for them and then repeat this pipeline stage. 4654 */ 4655 if (zio_wait_for_children(zio, ZIO_CHILD_ALL_BITS, ZIO_WAIT_DONE)) { 4656 return (NULL); 4657 } 4658 4659 /* 4660 * If the allocation throttle is enabled, then update the accounting. 4661 * We only track child I/Os that are part of an allocating async 4662 * write. We must do this since the allocation is performed 4663 * by the logical I/O but the actual write is done by child I/Os. 4664 */ 4665 if (zio->io_flags & ZIO_FLAG_IO_ALLOCATING && 4666 zio->io_child_type == ZIO_CHILD_VDEV) { 4667 ASSERT(zio->io_metaslab_class != NULL); 4668 ASSERT(zio->io_metaslab_class->mc_alloc_throttle_enabled); 4669 zio_dva_throttle_done(zio); 4670 } 4671 4672 /* 4673 * If the allocation throttle is enabled, verify that 4674 * we have decremented the refcounts for every I/O that was throttled. 4675 */ 4676 if (zio->io_flags & ZIO_FLAG_IO_ALLOCATING) { 4677 ASSERT(zio->io_type == ZIO_TYPE_WRITE); 4678 ASSERT(zio->io_priority == ZIO_PRIORITY_ASYNC_WRITE); 4679 ASSERT(zio->io_bp != NULL); 4680 ASSERT(ZIO_HAS_ALLOCATOR(zio)); 4681 4682 metaslab_group_alloc_verify(zio->io_spa, zio->io_bp, zio, 4683 zio->io_allocator); 4684 VERIFY(zfs_refcount_not_held(&zio->io_metaslab_class-> 4685 mc_allocator[zio->io_allocator].mca_alloc_slots, zio)); 4686 } 4687 4688 4689 for (int c = 0; c < ZIO_CHILD_TYPES; c++) 4690 for (int w = 0; w < ZIO_WAIT_TYPES; w++) 4691 ASSERT(zio->io_children[c][w] == 0); 4692 4693 if (zio->io_bp != NULL && !BP_IS_EMBEDDED(zio->io_bp)) { 4694 ASSERT(zio->io_bp->blk_pad[0] == 0); 4695 ASSERT(zio->io_bp->blk_pad[1] == 0); 4696 ASSERT(memcmp(zio->io_bp, &zio->io_bp_copy, 4697 sizeof (blkptr_t)) == 0 || 4698 (zio->io_bp == zio_unique_parent(zio)->io_bp)); 4699 if (zio->io_type == ZIO_TYPE_WRITE && !BP_IS_HOLE(zio->io_bp) && 4700 zio->io_bp_override == NULL && 4701 !(zio->io_flags & ZIO_FLAG_IO_REPAIR)) { 4702 ASSERT3U(zio->io_prop.zp_copies, <=, 4703 BP_GET_NDVAS(zio->io_bp)); 4704 ASSERT(BP_COUNT_GANG(zio->io_bp) == 0 || 4705 (BP_COUNT_GANG(zio->io_bp) == 4706 BP_GET_NDVAS(zio->io_bp))); 4707 } 4708 if (zio->io_flags & ZIO_FLAG_NOPWRITE) 4709 VERIFY(BP_EQUAL(zio->io_bp, &zio->io_bp_orig)); 4710 } 4711 4712 /* 4713 * If there were child vdev/gang/ddt errors, they apply to us now. 4714 */ 4715 zio_inherit_child_errors(zio, ZIO_CHILD_VDEV); 4716 zio_inherit_child_errors(zio, ZIO_CHILD_GANG); 4717 zio_inherit_child_errors(zio, ZIO_CHILD_DDT); 4718 4719 /* 4720 * If the I/O on the transformed data was successful, generate any 4721 * checksum reports now while we still have the transformed data. 4722 */ 4723 if (zio->io_error == 0) { 4724 while (zio->io_cksum_report != NULL) { 4725 zio_cksum_report_t *zcr = zio->io_cksum_report; 4726 uint64_t align = zcr->zcr_align; 4727 uint64_t asize = P2ROUNDUP(psize, align); 4728 abd_t *adata = zio->io_abd; 4729 4730 if (adata != NULL && asize != psize) { 4731 adata = abd_alloc(asize, B_TRUE); 4732 abd_copy(adata, zio->io_abd, psize); 4733 abd_zero_off(adata, psize, asize - psize); 4734 } 4735 4736 zio->io_cksum_report = zcr->zcr_next; 4737 zcr->zcr_next = NULL; 4738 zcr->zcr_finish(zcr, adata); 4739 zfs_ereport_free_checksum(zcr); 4740 4741 if (adata != NULL && asize != psize) 4742 abd_free(adata); 4743 } 4744 } 4745 4746 zio_pop_transforms(zio); /* note: may set zio->io_error */ 4747 4748 vdev_stat_update(zio, psize); 4749 4750 /* 4751 * If this I/O is attached to a particular vdev is slow, exceeding 4752 * 30 seconds to complete, post an error described the I/O delay. 4753 * We ignore these errors if the device is currently unavailable. 4754 */ 4755 if (zio->io_delay >= MSEC2NSEC(zio_slow_io_ms)) { 4756 if (zio->io_vd != NULL && !vdev_is_dead(zio->io_vd)) { 4757 /* 4758 * We want to only increment our slow IO counters if 4759 * the IO is valid (i.e. not if the drive is removed). 4760 * 4761 * zfs_ereport_post() will also do these checks, but 4762 * it can also ratelimit and have other failures, so we 4763 * need to increment the slow_io counters independent 4764 * of it. 4765 */ 4766 if (zfs_ereport_is_valid(FM_EREPORT_ZFS_DELAY, 4767 zio->io_spa, zio->io_vd, zio)) { 4768 mutex_enter(&zio->io_vd->vdev_stat_lock); 4769 zio->io_vd->vdev_stat.vs_slow_ios++; 4770 mutex_exit(&zio->io_vd->vdev_stat_lock); 4771 4772 (void) zfs_ereport_post(FM_EREPORT_ZFS_DELAY, 4773 zio->io_spa, zio->io_vd, &zio->io_bookmark, 4774 zio, 0); 4775 } 4776 } 4777 } 4778 4779 if (zio->io_error) { 4780 /* 4781 * If this I/O is attached to a particular vdev, 4782 * generate an error message describing the I/O failure 4783 * at the block level. We ignore these errors if the 4784 * device is currently unavailable. 4785 */ 4786 if (zio->io_error != ECKSUM && zio->io_vd != NULL && 4787 !vdev_is_dead(zio->io_vd)) { 4788 int ret = zfs_ereport_post(FM_EREPORT_ZFS_IO, 4789 zio->io_spa, zio->io_vd, &zio->io_bookmark, zio, 0); 4790 if (ret != EALREADY) { 4791 mutex_enter(&zio->io_vd->vdev_stat_lock); 4792 if (zio->io_type == ZIO_TYPE_READ) 4793 zio->io_vd->vdev_stat.vs_read_errors++; 4794 else if (zio->io_type == ZIO_TYPE_WRITE) 4795 zio->io_vd->vdev_stat.vs_write_errors++; 4796 mutex_exit(&zio->io_vd->vdev_stat_lock); 4797 } 4798 } 4799 4800 if ((zio->io_error == EIO || !(zio->io_flags & 4801 (ZIO_FLAG_SPECULATIVE | ZIO_FLAG_DONT_PROPAGATE))) && 4802 zio == zio->io_logical) { 4803 /* 4804 * For logical I/O requests, tell the SPA to log the 4805 * error and generate a logical data ereport. 4806 */ 4807 spa_log_error(zio->io_spa, &zio->io_bookmark, 4808 &zio->io_bp->blk_birth); 4809 (void) zfs_ereport_post(FM_EREPORT_ZFS_DATA, 4810 zio->io_spa, NULL, &zio->io_bookmark, zio, 0); 4811 } 4812 } 4813 4814 if (zio->io_error && zio == zio->io_logical) { 4815 /* 4816 * Determine whether zio should be reexecuted. This will 4817 * propagate all the way to the root via zio_notify_parent(). 4818 */ 4819 ASSERT(zio->io_vd == NULL && zio->io_bp != NULL); 4820 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL); 4821 4822 if (IO_IS_ALLOCATING(zio) && 4823 !(zio->io_flags & ZIO_FLAG_CANFAIL)) { 4824 if (zio->io_error != ENOSPC) 4825 zio->io_reexecute |= ZIO_REEXECUTE_NOW; 4826 else 4827 zio->io_reexecute |= ZIO_REEXECUTE_SUSPEND; 4828 } 4829 4830 if ((zio->io_type == ZIO_TYPE_READ || 4831 zio->io_type == ZIO_TYPE_FREE) && 4832 !(zio->io_flags & ZIO_FLAG_SCAN_THREAD) && 4833 zio->io_error == ENXIO && 4834 spa_load_state(zio->io_spa) == SPA_LOAD_NONE && 4835 spa_get_failmode(zio->io_spa) != ZIO_FAILURE_MODE_CONTINUE) 4836 zio->io_reexecute |= ZIO_REEXECUTE_SUSPEND; 4837 4838 if (!(zio->io_flags & ZIO_FLAG_CANFAIL) && !zio->io_reexecute) 4839 zio->io_reexecute |= ZIO_REEXECUTE_SUSPEND; 4840 4841 /* 4842 * Here is a possibly good place to attempt to do 4843 * either combinatorial reconstruction or error correction 4844 * based on checksums. It also might be a good place 4845 * to send out preliminary ereports before we suspend 4846 * processing. 4847 */ 4848 } 4849 4850 /* 4851 * If there were logical child errors, they apply to us now. 4852 * We defer this until now to avoid conflating logical child 4853 * errors with errors that happened to the zio itself when 4854 * updating vdev stats and reporting FMA events above. 4855 */ 4856 zio_inherit_child_errors(zio, ZIO_CHILD_LOGICAL); 4857 4858 if ((zio->io_error || zio->io_reexecute) && 4859 IO_IS_ALLOCATING(zio) && zio->io_gang_leader == zio && 4860 !(zio->io_flags & (ZIO_FLAG_IO_REWRITE | ZIO_FLAG_NOPWRITE))) 4861 zio_dva_unallocate(zio, zio->io_gang_tree, zio->io_bp); 4862 4863 zio_gang_tree_free(&zio->io_gang_tree); 4864 4865 /* 4866 * Godfather I/Os should never suspend. 4867 */ 4868 if ((zio->io_flags & ZIO_FLAG_GODFATHER) && 4869 (zio->io_reexecute & ZIO_REEXECUTE_SUSPEND)) 4870 zio->io_reexecute &= ~ZIO_REEXECUTE_SUSPEND; 4871 4872 if (zio->io_reexecute) { 4873 /* 4874 * This is a logical I/O that wants to reexecute. 4875 * 4876 * Reexecute is top-down. When an i/o fails, if it's not 4877 * the root, it simply notifies its parent and sticks around. 4878 * The parent, seeing that it still has children in zio_done(), 4879 * does the same. This percolates all the way up to the root. 4880 * The root i/o will reexecute or suspend the entire tree. 4881 * 4882 * This approach ensures that zio_reexecute() honors 4883 * all the original i/o dependency relationships, e.g. 4884 * parents not executing until children are ready. 4885 */ 4886 ASSERT(zio->io_child_type == ZIO_CHILD_LOGICAL); 4887 4888 zio->io_gang_leader = NULL; 4889 4890 mutex_enter(&zio->io_lock); 4891 zio->io_state[ZIO_WAIT_DONE] = 1; 4892 mutex_exit(&zio->io_lock); 4893 4894 /* 4895 * "The Godfather" I/O monitors its children but is 4896 * not a true parent to them. It will track them through 4897 * the pipeline but severs its ties whenever they get into 4898 * trouble (e.g. suspended). This allows "The Godfather" 4899 * I/O to return status without blocking. 4900 */ 4901 zl = NULL; 4902 for (pio = zio_walk_parents(zio, &zl); pio != NULL; 4903 pio = pio_next) { 4904 zio_link_t *remove_zl = zl; 4905 pio_next = zio_walk_parents(zio, &zl); 4906 4907 if ((pio->io_flags & ZIO_FLAG_GODFATHER) && 4908 (zio->io_reexecute & ZIO_REEXECUTE_SUSPEND)) { 4909 zio_remove_child(pio, zio, remove_zl); 4910 /* 4911 * This is a rare code path, so we don't 4912 * bother with "next_to_execute". 4913 */ 4914 zio_notify_parent(pio, zio, ZIO_WAIT_DONE, 4915 NULL); 4916 } 4917 } 4918 4919 if ((pio = zio_unique_parent(zio)) != NULL) { 4920 /* 4921 * We're not a root i/o, so there's nothing to do 4922 * but notify our parent. Don't propagate errors 4923 * upward since we haven't permanently failed yet. 4924 */ 4925 ASSERT(!(zio->io_flags & ZIO_FLAG_GODFATHER)); 4926 zio->io_flags |= ZIO_FLAG_DONT_PROPAGATE; 4927 /* 4928 * This is a rare code path, so we don't bother with 4929 * "next_to_execute". 4930 */ 4931 zio_notify_parent(pio, zio, ZIO_WAIT_DONE, NULL); 4932 } else if (zio->io_reexecute & ZIO_REEXECUTE_SUSPEND) { 4933 /* 4934 * We'd fail again if we reexecuted now, so suspend 4935 * until conditions improve (e.g. device comes online). 4936 */ 4937 zio_suspend(zio->io_spa, zio, ZIO_SUSPEND_IOERR); 4938 } else { 4939 /* 4940 * Reexecution is potentially a huge amount of work. 4941 * Hand it off to the otherwise-unused claim taskq. 4942 */ 4943 ASSERT(taskq_empty_ent(&zio->io_tqent)); 4944 spa_taskq_dispatch_ent(zio->io_spa, 4945 ZIO_TYPE_CLAIM, ZIO_TASKQ_ISSUE, 4946 zio_reexecute, zio, 0, &zio->io_tqent, NULL); 4947 } 4948 return (NULL); 4949 } 4950 4951 ASSERT(list_is_empty(&zio->io_child_list)); 4952 ASSERT(zio->io_reexecute == 0); 4953 ASSERT(zio->io_error == 0 || (zio->io_flags & ZIO_FLAG_CANFAIL)); 4954 4955 /* 4956 * Report any checksum errors, since the I/O is complete. 4957 */ 4958 while (zio->io_cksum_report != NULL) { 4959 zio_cksum_report_t *zcr = zio->io_cksum_report; 4960 zio->io_cksum_report = zcr->zcr_next; 4961 zcr->zcr_next = NULL; 4962 zcr->zcr_finish(zcr, NULL); 4963 zfs_ereport_free_checksum(zcr); 4964 } 4965 4966 /* 4967 * It is the responsibility of the done callback to ensure that this 4968 * particular zio is no longer discoverable for adoption, and as 4969 * such, cannot acquire any new parents. 4970 */ 4971 if (zio->io_done) 4972 zio->io_done(zio); 4973 4974 mutex_enter(&zio->io_lock); 4975 zio->io_state[ZIO_WAIT_DONE] = 1; 4976 mutex_exit(&zio->io_lock); 4977 4978 /* 4979 * We are done executing this zio. We may want to execute a parent 4980 * next. See the comment in zio_notify_parent(). 4981 */ 4982 zio_t *next_to_execute = NULL; 4983 zl = NULL; 4984 for (pio = zio_walk_parents(zio, &zl); pio != NULL; pio = pio_next) { 4985 zio_link_t *remove_zl = zl; 4986 pio_next = zio_walk_parents(zio, &zl); 4987 zio_remove_child(pio, zio, remove_zl); 4988 zio_notify_parent(pio, zio, ZIO_WAIT_DONE, &next_to_execute); 4989 } 4990 4991 if (zio->io_waiter != NULL) { 4992 mutex_enter(&zio->io_lock); 4993 zio->io_executor = NULL; 4994 cv_broadcast(&zio->io_cv); 4995 mutex_exit(&zio->io_lock); 4996 } else { 4997 zio_destroy(zio); 4998 } 4999 5000 return (next_to_execute); 5001 } 5002 5003 /* 5004 * ========================================================================== 5005 * I/O pipeline definition 5006 * ========================================================================== 5007 */ 5008 static zio_pipe_stage_t *zio_pipeline[] = { 5009 NULL, 5010 zio_read_bp_init, 5011 zio_write_bp_init, 5012 zio_free_bp_init, 5013 zio_issue_async, 5014 zio_write_compress, 5015 zio_encrypt, 5016 zio_checksum_generate, 5017 zio_nop_write, 5018 zio_brt_free, 5019 zio_ddt_read_start, 5020 zio_ddt_read_done, 5021 zio_ddt_write, 5022 zio_ddt_free, 5023 zio_gang_assemble, 5024 zio_gang_issue, 5025 zio_dva_throttle, 5026 zio_dva_allocate, 5027 zio_dva_free, 5028 zio_dva_claim, 5029 zio_ready, 5030 zio_vdev_io_start, 5031 zio_vdev_io_done, 5032 zio_vdev_io_assess, 5033 zio_checksum_verify, 5034 zio_done 5035 }; 5036 5037 5038 5039 5040 /* 5041 * Compare two zbookmark_phys_t's to see which we would reach first in a 5042 * pre-order traversal of the object tree. 5043 * 5044 * This is simple in every case aside from the meta-dnode object. For all other 5045 * objects, we traverse them in order (object 1 before object 2, and so on). 5046 * However, all of these objects are traversed while traversing object 0, since 5047 * the data it points to is the list of objects. Thus, we need to convert to a 5048 * canonical representation so we can compare meta-dnode bookmarks to 5049 * non-meta-dnode bookmarks. 5050 * 5051 * We do this by calculating "equivalents" for each field of the zbookmark. 5052 * zbookmarks outside of the meta-dnode use their own object and level, and 5053 * calculate the level 0 equivalent (the first L0 blkid that is contained in the 5054 * blocks this bookmark refers to) by multiplying their blkid by their span 5055 * (the number of L0 blocks contained within one block at their level). 5056 * zbookmarks inside the meta-dnode calculate their object equivalent 5057 * (which is L0equiv * dnodes per data block), use 0 for their L0equiv, and use 5058 * level + 1<<31 (any value larger than a level could ever be) for their level. 5059 * This causes them to always compare before a bookmark in their object 5060 * equivalent, compare appropriately to bookmarks in other objects, and to 5061 * compare appropriately to other bookmarks in the meta-dnode. 5062 */ 5063 int 5064 zbookmark_compare(uint16_t dbss1, uint8_t ibs1, uint16_t dbss2, uint8_t ibs2, 5065 const zbookmark_phys_t *zb1, const zbookmark_phys_t *zb2) 5066 { 5067 /* 5068 * These variables represent the "equivalent" values for the zbookmark, 5069 * after converting zbookmarks inside the meta dnode to their 5070 * normal-object equivalents. 5071 */ 5072 uint64_t zb1obj, zb2obj; 5073 uint64_t zb1L0, zb2L0; 5074 uint64_t zb1level, zb2level; 5075 5076 if (zb1->zb_object == zb2->zb_object && 5077 zb1->zb_level == zb2->zb_level && 5078 zb1->zb_blkid == zb2->zb_blkid) 5079 return (0); 5080 5081 IMPLY(zb1->zb_level > 0, ibs1 >= SPA_MINBLOCKSHIFT); 5082 IMPLY(zb2->zb_level > 0, ibs2 >= SPA_MINBLOCKSHIFT); 5083 5084 /* 5085 * BP_SPANB calculates the span in blocks. 5086 */ 5087 zb1L0 = (zb1->zb_blkid) * BP_SPANB(ibs1, zb1->zb_level); 5088 zb2L0 = (zb2->zb_blkid) * BP_SPANB(ibs2, zb2->zb_level); 5089 5090 if (zb1->zb_object == DMU_META_DNODE_OBJECT) { 5091 zb1obj = zb1L0 * (dbss1 << (SPA_MINBLOCKSHIFT - DNODE_SHIFT)); 5092 zb1L0 = 0; 5093 zb1level = zb1->zb_level + COMPARE_META_LEVEL; 5094 } else { 5095 zb1obj = zb1->zb_object; 5096 zb1level = zb1->zb_level; 5097 } 5098 5099 if (zb2->zb_object == DMU_META_DNODE_OBJECT) { 5100 zb2obj = zb2L0 * (dbss2 << (SPA_MINBLOCKSHIFT - DNODE_SHIFT)); 5101 zb2L0 = 0; 5102 zb2level = zb2->zb_level + COMPARE_META_LEVEL; 5103 } else { 5104 zb2obj = zb2->zb_object; 5105 zb2level = zb2->zb_level; 5106 } 5107 5108 /* Now that we have a canonical representation, do the comparison. */ 5109 if (zb1obj != zb2obj) 5110 return (zb1obj < zb2obj ? -1 : 1); 5111 else if (zb1L0 != zb2L0) 5112 return (zb1L0 < zb2L0 ? -1 : 1); 5113 else if (zb1level != zb2level) 5114 return (zb1level > zb2level ? -1 : 1); 5115 /* 5116 * This can (theoretically) happen if the bookmarks have the same object 5117 * and level, but different blkids, if the block sizes are not the same. 5118 * There is presently no way to change the indirect block sizes 5119 */ 5120 return (0); 5121 } 5122 5123 /* 5124 * This function checks the following: given that last_block is the place that 5125 * our traversal stopped last time, does that guarantee that we've visited 5126 * every node under subtree_root? Therefore, we can't just use the raw output 5127 * of zbookmark_compare. We have to pass in a modified version of 5128 * subtree_root; by incrementing the block id, and then checking whether 5129 * last_block is before or equal to that, we can tell whether or not having 5130 * visited last_block implies that all of subtree_root's children have been 5131 * visited. 5132 */ 5133 boolean_t 5134 zbookmark_subtree_completed(const dnode_phys_t *dnp, 5135 const zbookmark_phys_t *subtree_root, const zbookmark_phys_t *last_block) 5136 { 5137 zbookmark_phys_t mod_zb = *subtree_root; 5138 mod_zb.zb_blkid++; 5139 ASSERT0(last_block->zb_level); 5140 5141 /* The objset_phys_t isn't before anything. */ 5142 if (dnp == NULL) 5143 return (B_FALSE); 5144 5145 /* 5146 * We pass in 1ULL << (DNODE_BLOCK_SHIFT - SPA_MINBLOCKSHIFT) for the 5147 * data block size in sectors, because that variable is only used if 5148 * the bookmark refers to a block in the meta-dnode. Since we don't 5149 * know without examining it what object it refers to, and there's no 5150 * harm in passing in this value in other cases, we always pass it in. 5151 * 5152 * We pass in 0 for the indirect block size shift because zb2 must be 5153 * level 0. The indirect block size is only used to calculate the span 5154 * of the bookmark, but since the bookmark must be level 0, the span is 5155 * always 1, so the math works out. 5156 * 5157 * If you make changes to how the zbookmark_compare code works, be sure 5158 * to make sure that this code still works afterwards. 5159 */ 5160 return (zbookmark_compare(dnp->dn_datablkszsec, dnp->dn_indblkshift, 5161 1ULL << (DNODE_BLOCK_SHIFT - SPA_MINBLOCKSHIFT), 0, &mod_zb, 5162 last_block) <= 0); 5163 } 5164 5165 /* 5166 * This function is similar to zbookmark_subtree_completed(), but returns true 5167 * if subtree_root is equal or ahead of last_block, i.e. still to be done. 5168 */ 5169 boolean_t 5170 zbookmark_subtree_tbd(const dnode_phys_t *dnp, 5171 const zbookmark_phys_t *subtree_root, const zbookmark_phys_t *last_block) 5172 { 5173 ASSERT0(last_block->zb_level); 5174 if (dnp == NULL) 5175 return (B_FALSE); 5176 return (zbookmark_compare(dnp->dn_datablkszsec, dnp->dn_indblkshift, 5177 1ULL << (DNODE_BLOCK_SHIFT - SPA_MINBLOCKSHIFT), 0, subtree_root, 5178 last_block) >= 0); 5179 } 5180 5181 EXPORT_SYMBOL(zio_type_name); 5182 EXPORT_SYMBOL(zio_buf_alloc); 5183 EXPORT_SYMBOL(zio_data_buf_alloc); 5184 EXPORT_SYMBOL(zio_buf_free); 5185 EXPORT_SYMBOL(zio_data_buf_free); 5186 5187 ZFS_MODULE_PARAM(zfs_zio, zio_, slow_io_ms, INT, ZMOD_RW, 5188 "Max I/O completion time (milliseconds) before marking it as slow"); 5189 5190 ZFS_MODULE_PARAM(zfs_zio, zio_, requeue_io_start_cut_in_line, INT, ZMOD_RW, 5191 "Prioritize requeued I/O"); 5192 5193 ZFS_MODULE_PARAM(zfs, zfs_, sync_pass_deferred_free, UINT, ZMOD_RW, 5194 "Defer frees starting in this pass"); 5195 5196 ZFS_MODULE_PARAM(zfs, zfs_, sync_pass_dont_compress, UINT, ZMOD_RW, 5197 "Don't compress starting in this pass"); 5198 5199 ZFS_MODULE_PARAM(zfs, zfs_, sync_pass_rewrite, UINT, ZMOD_RW, 5200 "Rewrite new bps starting in this pass"); 5201 5202 ZFS_MODULE_PARAM(zfs_zio, zio_, dva_throttle_enabled, INT, ZMOD_RW, 5203 "Throttle block allocations in the ZIO pipeline"); 5204 5205 ZFS_MODULE_PARAM(zfs_zio, zio_, deadman_log_all, INT, ZMOD_RW, 5206 "Log all slow ZIOs, not just those with vdevs"); 5207