Lines Matching defs:scn
125 static void scan_ds_queue_clear(dsl_scan_t *scn);
126 static void scan_ds_prefetch_queue_clear(dsl_scan_t *scn);
127 static boolean_t scan_ds_queue_contains(dsl_scan_t *scn, uint64_t dsobj,
129 static void scan_ds_queue_insert(dsl_scan_t *scn, uint64_t dsobj, uint64_t txg);
130 static void scan_ds_queue_remove(dsl_scan_t *scn, uint64_t dsobj);
131 static void scan_ds_queue_sync(dsl_scan_t *scn, dmu_tx_t *tx);
133 static void read_by_block_level(dsl_scan_t *scn, zbookmark_phys_t zb);
230 #define DSL_SCAN_IS_SCRUB_RESILVER(scn) \
231 ((scn)->scn_phys.scn_func == POOL_SCAN_SCRUB || \
232 (scn)->scn_phys.scn_func == POOL_SCAN_RESILVER)
234 #define DSL_SCAN_IS_SCRUB(scn) \
235 ((scn)->scn_phys.scn_func == POOL_SCAN_SCRUB)
252 /* In core node for the scn->scn_queue. Represents a dataset to be scanned */
353 avl_node_t spic_avl_node; /* link into scn->scn_prefetch_queue */
365 static void scan_io_queues_destroy(dsl_scan_t *scn);
421 dsl_scan_is_running(const dsl_scan_t *scn)
423 return (scn->scn_phys.scn_state == DSS_SCANNING);
473 dsl_scan_t *scn;
477 scn = dp->dp_scan = kmem_zalloc(sizeof (dsl_scan_t), KM_SLEEP);
478 scn->scn_dp = dp;
485 ASSERT(!scn->scn_async_destroying);
486 scn->scn_async_destroying = spa_feature_is_active(dp->dp_spa,
495 scn->scn_maxinflight_bytes = MIN(arc_c_max / 4, MAX(1ULL << 20,
498 avl_create(&scn->scn_queue, scan_ds_queue_compare, sizeof (scan_ds_t),
500 mutex_init(&scn->scn_queue_lock, NULL, MUTEX_DEFAULT, NULL);
501 avl_create(&scn->scn_prefetch_queue, scan_prefetch_queue_compare,
512 scn->scn_restart_txg = txg;
516 (longlong_t)scn->scn_restart_txg);
524 &scn->scn_phys.scn_queue_obj);
528 ERRORSCRUB_PHYS_NUMINTS, &scn->errorscrub_phys);
535 &scn->scn_phys);
539 * does properly update the scn->scn_phys structure and notify
555 scn->scn_async_destroying) {
561 memcpy(&scn->scn_phys, zaptmp,
563 scn->scn_phys.scn_flags = overflow;
566 if (scn->scn_phys.scn_state == DSS_FINISHED ||
567 scn->scn_phys.scn_state == DSS_CANCELED)
583 scn->scn_issued_before_pass = scn->scn_phys.scn_examined -
584 scn->scn_phys.scn_skipped;
586 if (dsl_scan_is_running(scn) &&
595 scn->scn_restart_txg = txg;
599 (longlong_t)scn->scn_restart_txg);
607 if (scn->scn_phys.scn_errors > 0) {
608 scn->scn_restart_txg = txg;
613 (u_longlong_t)scn->scn_restart_txg);
621 memcpy(&scn->scn_phys_cached, &scn->scn_phys, sizeof (scn->scn_phys));
624 if (scn->scn_phys.scn_queue_obj != 0) {
629 scn->scn_phys.scn_queue_obj);
632 scan_ds_queue_insert(scn,
640 ddt_walk_init(spa, scn->scn_phys.scn_max_txg);
652 dsl_scan_t *scn = dp->dp_scan;
654 if (scn->scn_taskq != NULL)
655 taskq_destroy(scn->scn_taskq);
657 scan_ds_queue_clear(scn);
658 avl_destroy(&scn->scn_queue);
659 mutex_destroy(&scn->scn_queue_lock);
660 scan_ds_prefetch_queue_clear(scn);
661 avl_destroy(&scn->scn_prefetch_queue);
669 dsl_scan_restarting(dsl_scan_t *scn, dmu_tx_t *tx)
671 return (scn->scn_restart_txg != 0 &&
672 scn->scn_restart_txg <= tx->tx_txg);
701 dsl_errorscrub_is_paused(const dsl_scan_t *scn)
703 return (dsl_errorscrubbing(scn->scn_dp) &&
704 scn->errorscrub_phys.dep_paused_flags);
708 dsl_scan_is_paused_scrub(const dsl_scan_t *scn)
710 return (dsl_scan_scrubbing(scn->scn_dp) &&
711 scn->scn_phys.scn_flags & DSF_SCRUB_PAUSED);
715 dsl_errorscrub_sync_state(dsl_scan_t *scn, dmu_tx_t *tx)
717 scn->errorscrub_phys.dep_cursor =
718 zap_cursor_serialize(&scn->errorscrub_cursor);
720 VERIFY0(zap_update(scn->scn_dp->dp_meta_objset,
723 &scn->errorscrub_phys, tx));
729 dsl_scan_t *scn = dmu_tx_pool(tx)->dp_scan;
731 dsl_pool_t *dp = scn->scn_dp;
734 ASSERT(!dsl_scan_is_running(scn));
735 ASSERT(!dsl_errorscrubbing(scn->scn_dp));
738 memset(&scn->errorscrub_phys, 0, sizeof (scn->errorscrub_phys));
739 scn->errorscrub_phys.dep_func = *funcp;
740 scn->errorscrub_phys.dep_state = DSS_ERRORSCRUBBING;
741 scn->errorscrub_phys.dep_start_time = gethrestime_sec();
742 scn->errorscrub_phys.dep_to_examine = spa_get_last_errlog_size(spa);
743 scn->errorscrub_phys.dep_examined = 0;
744 scn->errorscrub_phys.dep_errors = 0;
745 scn->errorscrub_phys.dep_cursor = 0;
746 zap_cursor_init_serialized(&scn->errorscrub_cursor,
748 scn->errorscrub_phys.dep_cursor);
753 dsl_errorscrub_sync_state(scn, tx);
764 dsl_scan_t *scn = dmu_tx_pool(tx)->dp_scan;
766 if (dsl_scan_is_running(scn) || (dsl_errorscrubbing(scn->scn_dp))) {
770 if (spa_get_last_errlog_size(scn->scn_dp->dp_spa) == 0) {
799 dsl_scan_sync_state(dsl_scan_t *scn, dmu_tx_t *tx, state_sync_type_t sync_type)
802 spa_t *spa = scn->scn_dp->dp_spa;
804 ASSERT(sync_type != SYNC_MANDATORY || scn->scn_queues_pending == 0);
805 if (scn->scn_queues_pending == 0) {
821 if (scn->scn_phys.scn_queue_obj != 0)
822 scan_ds_queue_sync(scn, tx);
823 VERIFY0(zap_update(scn->scn_dp->dp_meta_objset,
826 &scn->scn_phys, tx));
827 memcpy(&scn->scn_phys_cached, &scn->scn_phys,
828 sizeof (scn->scn_phys));
830 if (scn->scn_checkpointing)
834 scn->scn_checkpointing = B_FALSE;
835 scn->scn_last_checkpoint = ddi_get_lbolt();
837 VERIFY0(zap_update(scn->scn_dp->dp_meta_objset,
840 &scn->scn_phys_cached, tx));
848 dsl_scan_t *scn = dmu_tx_pool(tx)->dp_scan;
849 vdev_t *rvd = scn->scn_dp->dp_spa->spa_root_vdev;
851 if (dsl_scan_is_running(scn) || vdev_rebuild_active(rvd) ||
852 dsl_errorscrubbing(scn->scn_dp))
862 dsl_scan_t *scn = dmu_tx_pool(tx)->dp_scan;
864 dsl_pool_t *dp = scn->scn_dp;
867 ASSERT(!dsl_scan_is_running(scn));
870 memset(&scn->scn_phys, 0, sizeof (scn->scn_phys));
876 memset(&scn->errorscrub_phys, 0, sizeof (scn->errorscrub_phys));
877 dsl_errorscrub_sync_state(scn, tx);
879 scn->scn_phys.scn_func = setup_sync_arg->func;
880 scn->scn_phys.scn_state = DSS_SCANNING;
881 scn->scn_phys.scn_min_txg = setup_sync_arg->txgstart;
883 scn->scn_phys.scn_max_txg = tx->tx_txg;
885 scn->scn_phys.scn_max_txg = setup_sync_arg->txgend;
887 scn->scn_phys.scn_ddt_class_max = DDT_CLASSES - 1; /* the entire DDT */
888 scn->scn_phys.scn_start_time = gethrestime_sec();
889 scn->scn_phys.scn_errors = 0;
890 scn->scn_phys.scn_to_examine = spa->spa_root_vdev->vdev_stat.vs_alloc;
891 scn->scn_issued_before_pass = 0;
892 scn->scn_restart_txg = 0;
893 scn->scn_done_txg = 0;
894 scn->scn_last_checkpoint = 0;
895 scn->scn_checkpointing = B_FALSE;
899 if (DSL_SCAN_IS_SCRUB_RESILVER(scn)) {
900 scn->scn_phys.scn_ddt_class_max = zfs_scrub_ddt_class_max;
906 &scn->scn_phys.scn_min_txg, &scn->scn_phys.scn_max_txg)) {
923 if (scn->scn_phys.scn_min_txg > TXG_INITIAL)
924 scn->scn_phys.scn_ddt_class_max = DDT_CLASS_DITTO;
933 if (scn->scn_phys.scn_func == POOL_SCAN_RESILVER) {
962 scn->scn_phys.scn_queue_obj = zap_create(dp->dp_meta_objset,
965 memcpy(&scn->scn_phys_cached, &scn->scn_phys, sizeof (scn->scn_phys));
967 ddt_walk_init(spa, scn->scn_phys.scn_max_txg);
969 dsl_scan_sync_state(scn, tx, SYNC_MANDATORY);
973 setup_sync_arg->func, (u_longlong_t)scn->scn_phys.scn_min_txg,
974 (u_longlong_t)scn->scn_phys.scn_max_txg);
987 dsl_scan_t *scn = dp->dp_scan;
1017 int err = dsl_scrub_set_pause_resume(scn->scn_dp,
1032 if (func == POOL_SCAN_SCRUB && dsl_scan_is_paused_scrub(scn)) {
1034 int err = dsl_scrub_set_pause_resume(scn->scn_dp,
1053 dsl_errorscrub_done(dsl_scan_t *scn, boolean_t complete, dmu_tx_t *tx)
1055 dsl_pool_t *dp = scn->scn_dp;
1067 scn->errorscrub_phys.dep_state = complete ? DSS_FINISHED : DSS_CANCELED;
1070 scn->errorscrub_phys.dep_end_time = gethrestime_sec();
1071 zap_cursor_fini(&scn->errorscrub_cursor);
1076 ASSERT(!dsl_errorscrubbing(scn->scn_dp));
1080 dsl_scan_done(dsl_scan_t *scn, boolean_t complete, dmu_tx_t *tx)
1094 dsl_pool_t *dp = scn->scn_dp;
1104 if (scn->scn_phys.scn_queue_obj != 0) {
1106 scn->scn_phys.scn_queue_obj, tx));
1107 scn->scn_phys.scn_queue_obj = 0;
1109 scan_ds_queue_clear(scn);
1110 scan_ds_prefetch_queue_clear(scn);
1112 scn->scn_phys.scn_flags &= ~DSF_SCRUB_PAUSED;
1118 if (!dsl_scan_is_running(scn)) {
1119 ASSERT(!scn->scn_is_sorted);
1123 if (scn->scn_is_sorted) {
1124 scan_io_queues_destroy(scn);
1125 scn->scn_is_sorted = B_FALSE;
1127 if (scn->scn_taskq != NULL) {
1128 taskq_destroy(scn->scn_taskq);
1129 scn->scn_taskq = NULL;
1133 scn->scn_phys.scn_state = complete ? DSS_FINISHED : DSS_CANCELED;
1137 if (dsl_scan_restarting(scn, tx)) {
1146 if (DSL_SCAN_IS_SCRUB(scn)) {
1151 &scn->scn_phys.scn_max_txg, tx));
1152 spa->spa_scrubbed_last_txg = scn->scn_phys.scn_max_txg;
1156 if (DSL_SCAN_IS_SCRUB_RESILVER(scn)) {
1172 scn->scn_phys.scn_max_txg, B_TRUE, B_FALSE);
1174 if (scn->scn_phys.scn_min_txg) {
1226 scn->scn_phys.scn_end_time = gethrestime_sec();
1231 ASSERT(!dsl_scan_is_running(scn));
1239 dsl_scan_t *scn = dp->dp_scan;
1250 if (dsl_errorscrub_is_paused(scn))
1265 dsl_scan_t *scn = dp->dp_scan;
1269 scn->errorscrub_phys.dep_paused_flags = B_TRUE;
1270 dsl_errorscrub_sync_state(scn, tx);
1274 if (dsl_errorscrub_is_paused(scn)) {
1285 scn->errorscrub_phys.dep_paused_flags = B_FALSE;
1288 &scn->errorscrub_cursor,
1290 scn->errorscrub_phys.dep_cursor);
1292 dsl_errorscrub_sync_state(scn, tx);
1301 dsl_scan_t *scn = dmu_tx_pool(tx)->dp_scan;
1303 if (!dsl_errorscrubbing(scn->scn_dp))
1312 dsl_scan_t *scn = dmu_tx_pool(tx)->dp_scan;
1314 dsl_errorscrub_done(scn, B_FALSE, tx);
1315 dsl_errorscrub_sync_state(scn, tx);
1316 spa_event_notify(scn->scn_dp->dp_spa, NULL, NULL,
1324 dsl_scan_t *scn = dmu_tx_pool(tx)->dp_scan;
1326 if (!dsl_scan_is_running(scn))
1335 dsl_scan_t *scn = dmu_tx_pool(tx)->dp_scan;
1337 dsl_scan_done(scn, B_FALSE, tx);
1338 dsl_scan_sync_state(scn, tx, SYNC_MANDATORY);
1339 spa_event_notify(scn->scn_dp->dp_spa, NULL, NULL, ESC_ZFS_SCRUB_ABORT);
1359 dsl_scan_t *scn = dp->dp_scan;
1367 if (dsl_scan_is_paused_scrub(scn))
1382 dsl_scan_t *scn = dp->dp_scan;
1387 scn->scn_phys.scn_flags |= DSF_SCRUB_PAUSED;
1388 scn->scn_phys_cached.scn_flags |= DSF_SCRUB_PAUSED;
1389 dsl_scan_sync_state(scn, tx, SYNC_CACHED);
1394 if (dsl_scan_is_paused_scrub(scn)) {
1403 scn->scn_phys.scn_flags &= ~DSF_SCRUB_PAUSED;
1404 scn->scn_phys_cached.scn_flags &= ~DSF_SCRUB_PAUSED;
1405 dsl_scan_sync_state(scn, tx, SYNC_CACHED);
1473 scan_ds_queue_clear(dsl_scan_t *scn)
1477 while ((sds = avl_destroy_nodes(&scn->scn_queue, &cookie)) != NULL) {
1483 scan_ds_queue_contains(dsl_scan_t *scn, uint64_t dsobj, uint64_t *txg)
1488 sds = avl_find(&scn->scn_queue, &srch, NULL);
1495 scan_ds_queue_insert(dsl_scan_t *scn, uint64_t dsobj, uint64_t txg)
1504 VERIFY3P(avl_find(&scn->scn_queue, sds, &where), ==, NULL);
1505 avl_insert(&scn->scn_queue, sds, where);
1509 scan_ds_queue_remove(dsl_scan_t *scn, uint64_t dsobj)
1515 sds = avl_find(&scn->scn_queue, &srch, NULL);
1517 avl_remove(&scn->scn_queue, sds);
1522 scan_ds_queue_sync(dsl_scan_t *scn, dmu_tx_t *tx)
1524 dsl_pool_t *dp = scn->scn_dp;
1529 ASSERT0(scn->scn_queues_pending);
1530 ASSERT(scn->scn_phys.scn_queue_obj != 0);
1533 scn->scn_phys.scn_queue_obj, tx));
1534 scn->scn_phys.scn_queue_obj = zap_create(dp->dp_meta_objset, ot,
1536 for (scan_ds_t *sds = avl_first(&scn->scn_queue);
1537 sds != NULL; sds = AVL_NEXT(&scn->scn_queue, sds)) {
1539 scn->scn_phys.scn_queue_obj, sds->sds_dsobj,
1577 dsl_scan_should_clear(dsl_scan_t *scn)
1579 spa_t *spa = scn->scn_dp->dp_spa;
1580 vdev_t *rvd = scn->scn_dp->dp_spa->spa_root_vdev;
1614 ASSERT0(scn->scn_queues_pending);
1626 return (scn->scn_clearing);
1630 dsl_scan_check_suspend(dsl_scan_t *scn, const zbookmark_phys_t *zb)
1636 if (scn->scn_suspending)
1639 if (!ZB_IS_ZERO(&scn->scn_phys.scn_bookmark))
1661 uint64_t scan_time_ns = curr_time_ns - scn->scn_sync_start_time;
1663 scn->scn_dp->dp_spa->spa_sync_starttime;
1666 uint_t mintime = (scn->scn_phys.scn_func == POOL_SCAN_RESILVER) ?
1670 (scn->scn_dp->dp_dirty_total >= dirty_min_bytes ||
1671 txg_sync_waiting(scn->scn_dp) ||
1673 spa_shutting_down(scn->scn_dp->dp_spa) ||
1674 (zfs_scan_strict_mem_lim && dsl_scan_should_clear(scn)) ||
1675 !ddt_walk_ready(scn->scn_dp->dp_spa)) {
1683 SET_BOOKMARK(&scn->scn_phys.scn_bookmark,
1691 scn->scn_phys.scn_bookmark = *zb;
1694 dsl_scan_phys_t *scnp = &scn->scn_phys;
1703 scn->scn_suspending = B_TRUE;
1710 dsl_error_scrub_check_suspend(dsl_scan_t *scn, const zbookmark_phys_t *zb)
1723 uint64_t error_scrub_time_ns = curr_time_ns - scn->scn_sync_start_time;
1725 scn->scn_dp->dp_spa->spa_sync_starttime;
1729 (txg_sync_waiting(scn->scn_dp) ||
1731 spa_shutting_down(scn->scn_dp->dp_spa)) {
1757 dsl_scan_t *scn = dp->dp_scan;
1763 BP_GET_LOGICAL_BIRTH(bp) <= scn->scn_phys.scn_cur_min_txg)
1779 VERIFY(0 == scan_funcs[scn->scn_phys.scn_func](dp, bp, &zb));
1791 dsl_scan_t *scn = dp->dp_scan;
1799 BP_GET_LOGICAL_BIRTH(bp) <= scn->scn_phys.scn_cur_min_txg)
1815 VERIFY(0 == scan_funcs[scn->scn_phys.scn_func](dp, bp, &zb));
1870 scan_prefetch_ctx_create(dsl_scan_t *scn, dnode_phys_t *dnp, const void *tag)
1877 spc->spc_scn = scn;
1898 scan_ds_prefetch_queue_clear(dsl_scan_t *scn)
1900 spa_t *spa = scn->scn_dp->dp_spa;
1905 while ((spic = avl_destroy_nodes(&scn->scn_prefetch_queue,
1907 scan_prefetch_ctx_rele(spic->spic_spc, scn);
1939 dsl_scan_t *scn = spc->spc_scn;
1940 spa_t *spa = scn->scn_dp->dp_spa;
1947 BP_GET_LOGICAL_BIRTH(bp) <= scn->scn_phys.scn_cur_min_txg ||
1955 scan_prefetch_ctx_add_ref(spc, scn);
1967 if (avl_find(&scn->scn_prefetch_queue, spic, &idx) != NULL) {
1970 scan_prefetch_ctx_rele(spc, scn);
1975 avl_insert(&scn->scn_prefetch_queue, spic, idx);
1981 dsl_scan_prefetch_dnode(dsl_scan_t *scn, dnode_phys_t *dnp,
1993 spc = scan_prefetch_ctx_create(scn, dnp, FTAG);
2016 dsl_scan_t *scn = spc->spc_scn;
2017 spa_t *spa = scn->scn_dp->dp_spa;
2027 if (buf == NULL || scn->scn_prefetch_stop)
2049 dsl_scan_prefetch_dnode(scn, cdnp,
2055 dsl_scan_prefetch_dnode(scn, &osp->os_meta_dnode,
2060 dsl_scan_prefetch_dnode(scn,
2064 dsl_scan_prefetch_dnode(scn,
2067 dsl_scan_prefetch_dnode(scn,
2076 scan_prefetch_ctx_rele(spc, scn);
2082 dsl_scan_t *scn = arg;
2083 spa_t *spa = scn->scn_dp->dp_spa;
2087 while (!scn->scn_prefetch_stop) {
2098 while (!scn->scn_prefetch_stop &&
2099 (avl_numnodes(&scn->scn_prefetch_queue) == 0 ||
2100 spa->spa_scrub_inflight >= scn->scn_maxinflight_bytes)) {
2105 if (scn->scn_prefetch_stop) {
2111 spic = avl_first(&scn->scn_prefetch_queue);
2113 avl_remove(&scn->scn_prefetch_queue, spic);
2131 (void) arc_read(scn->scn_zio_root, spa, bp,
2138 ASSERT(scn->scn_prefetch_stop);
2142 while ((spic = avl_first(&scn->scn_prefetch_queue)) != NULL) {
2143 avl_remove(&scn->scn_prefetch_queue, spic);
2144 scan_prefetch_ctx_rele(spic->spic_spc, scn);
2147 ASSERT0(avl_numnodes(&scn->scn_prefetch_queue));
2152 dsl_scan_check_resume(dsl_scan_t *scn, const dnode_phys_t *dnp,
2158 if (!ZB_IS_ZERO(&scn->scn_phys.scn_bookmark) &&
2165 &scn->scn_phys.scn_bookmark))
2174 &scn->scn_phys.scn_bookmark)) {
2180 memset(&scn->scn_phys.scn_bookmark, 0, sizeof (*zb));
2187 dnode_phys_t *dnp, dsl_dataset_t *ds, dsl_scan_t *scn,
2198 dsl_scan_recurse(dsl_scan_t *scn, dsl_dataset_t *ds, dmu_objset_type_t ostype,
2202 dsl_pool_t *dp = scn->scn_dp;
2217 scn->scn_phys.scn_errors++;
2232 scn->scn_phys.scn_errors++;
2242 ds, scn, ostype, tx);
2260 scn->scn_phys.scn_errors++;
2266 dsl_scan_visitdnode(scn, ds, ostype,
2279 scn->scn_phys.scn_errors++;
2285 dsl_scan_visitdnode(scn, ds, osp->os_type,
2296 dsl_scan_visitdnode(scn, ds, osp->os_type,
2299 dsl_scan_visitdnode(scn, ds, osp->os_type,
2302 dsl_scan_visitdnode(scn, ds, osp->os_type,
2313 scn->scn_phys.scn_errors++;
2322 dsl_scan_visitdnode(dsl_scan_t *scn, dsl_dataset_t *ds,
2334 &czb, dnp, ds, scn, ostype, tx);
2342 &czb, dnp, ds, scn, ostype, tx);
2352 dnode_phys_t *dnp, dsl_dataset_t *ds, dsl_scan_t *scn,
2355 dsl_pool_t *dp = scn->scn_dp;
2357 if (dsl_scan_check_suspend(scn, zb))
2360 if (dsl_scan_check_resume(scn, dnp, zb))
2363 scn->scn_visited_this_txg++;
2366 scn->scn_holes_this_txg++;
2391 if (BP_GET_LOGICAL_BIRTH(bp) <= scn->scn_phys.scn_cur_min_txg) {
2392 scn->scn_lt_min_this_txg++;
2396 if (dsl_scan_recurse(scn, ds, ostype, dnp, bp, zb, tx) != 0)
2405 scn->scn_phys.scn_ddt_class_max, bp)) {
2406 scn->scn_ddt_contained_this_txg++;
2417 if (BP_GET_BIRTH(bp) > scn->scn_phys.scn_cur_max_txg) {
2418 scn->scn_gt_max_this_txg++;
2422 scan_funcs[scn->scn_phys.scn_func](dp, bp, zb);
2426 dsl_scan_visit_rootbp(dsl_scan_t *scn, dsl_dataset_t *ds, blkptr_t *bp,
2435 if (ZB_IS_ZERO(&scn->scn_phys.scn_bookmark)) {
2436 SET_BOOKMARK(&scn->scn_prefetch_bookmark,
2439 scn->scn_prefetch_bookmark = scn->scn_phys.scn_bookmark;
2442 scn->scn_objsets_visited_this_txg++;
2444 spc = scan_prefetch_ctx_create(scn, NULL, FTAG);
2448 dsl_scan_visitbp(bp, &zb, NULL, ds, scn, DMU_OST_NONE, tx);
2503 dsl_scan_t *scn = dp->dp_scan;
2506 if (!dsl_scan_is_running(scn))
2509 ds_destroyed_scn_phys(ds, &scn->scn_phys);
2510 ds_destroyed_scn_phys(ds, &scn->scn_phys_cached);
2512 if (scan_ds_queue_contains(scn, ds->ds_object, &mintxg)) {
2513 scan_ds_queue_remove(scn, ds->ds_object);
2515 scan_ds_queue_insert(scn,
2519 if (zap_lookup_int_key(dp->dp_meta_objset, scn->scn_phys.scn_queue_obj,
2523 scn->scn_phys.scn_queue_obj, ds->ds_object, tx));
2531 scn->scn_phys.scn_queue_obj,
2552 dsl_scan_sync_state(scn, tx, SYNC_CACHED);
2579 dsl_scan_t *scn = dp->dp_scan;
2582 if (!dsl_scan_is_running(scn))
2587 ds_snapshotted_bookmark(ds, &scn->scn_phys.scn_bookmark);
2588 ds_snapshotted_bookmark(ds, &scn->scn_phys_cached.scn_bookmark);
2590 if (scan_ds_queue_contains(scn, ds->ds_object, &mintxg)) {
2591 scan_ds_queue_remove(scn, ds->ds_object);
2592 scan_ds_queue_insert(scn,
2596 if (zap_lookup_int_key(dp->dp_meta_objset, scn->scn_phys.scn_queue_obj,
2599 scn->scn_phys.scn_queue_obj, ds->ds_object, tx));
2601 scn->scn_phys.scn_queue_obj,
2610 dsl_scan_sync_state(scn, tx, SYNC_CACHED);
2643 dsl_scan_t *scn = dp->dp_scan;
2647 if (!dsl_scan_is_running(scn))
2650 ds_clone_swapped_bookmark(ds1, ds2, &scn->scn_phys.scn_bookmark);
2651 ds_clone_swapped_bookmark(ds1, ds2, &scn->scn_phys_cached.scn_bookmark);
2656 ds1_queued = scan_ds_queue_contains(scn, ds1->ds_object, &mintxg1);
2657 ds2_queued = scan_ds_queue_contains(scn, ds2->ds_object, &mintxg2);
2678 scan_ds_queue_remove(scn, ds1->ds_object);
2679 scan_ds_queue_insert(scn, ds2->ds_object, mintxg1);
2681 scan_ds_queue_remove(scn, ds2->ds_object);
2682 scan_ds_queue_insert(scn, ds1->ds_object, mintxg2);
2693 scn->scn_phys.scn_queue_obj, ds1->ds_object, &mintxg1) == 0;
2695 scn->scn_phys.scn_queue_obj, ds2->ds_object, &mintxg2) == 0;
2716 scn->scn_phys.scn_queue_obj, ds1->ds_object, tx));
2718 scn->scn_phys.scn_queue_obj, ds2->ds_object, mintxg1, tx));
2726 scn->scn_phys.scn_queue_obj, ds2->ds_object, tx));
2728 scn->scn_phys.scn_queue_obj, ds1->ds_object, mintxg2, tx));
2736 dsl_scan_sync_state(scn, tx, SYNC_CACHED);
2745 dsl_scan_t *scn = dp->dp_scan;
2764 mutex_enter(&scn->scn_queue_lock);
2765 scan_ds_queue_insert(scn, ds->ds_object,
2767 mutex_exit(&scn->scn_queue_lock);
2773 dsl_scan_visitds(dsl_scan_t *scn, uint64_t dsobj, dmu_tx_t *tx)
2775 dsl_pool_t *dp = scn->scn_dp;
2780 if (scn->scn_phys.scn_cur_min_txg >=
2781 scn->scn_phys.scn_max_txg) {
2813 (longlong_t)scn->scn_phys.scn_cur_min_txg,
2814 (longlong_t)scn->scn_phys.scn_max_txg);
2844 dsl_scan_visit_rootbp(scn, ds, &dsl_dataset_phys(ds)->ds_bp, tx);
2852 (longlong_t)scn->scn_phys.scn_cur_min_txg,
2853 (longlong_t)scn->scn_phys.scn_cur_max_txg,
2854 (int)scn->scn_suspending);
2857 if (scn->scn_suspending)
2867 if (scn->scn_phys.scn_flags & DSF_VISIT_DS_AGAIN) {
2870 scn->scn_phys.scn_flags &= ~DSF_VISIT_DS_AGAIN;
2871 scan_ds_queue_insert(scn, ds->ds_object,
2872 scn->scn_phys.scn_cur_max_txg);
2880 scan_ds_queue_insert(scn,
2909 scan_ds_queue_insert(scn,
2932 dsl_scan_t *scn = dp->dp_scan;
2959 mutex_enter(&scn->scn_queue_lock);
2960 scan_ds_queue_insert(scn, ds->ds_object,
2962 mutex_exit(&scn->scn_queue_lock);
2968 dsl_scan_ddt_entry(dsl_scan_t *scn, enum zio_checksum checksum,
2976 if (!dsl_scan_is_running(scn))
2990 if (scn->scn_done_txg != 0)
2997 if (phys_birth == 0 || phys_birth > scn->scn_phys.scn_max_txg)
3001 scn->scn_visited_this_txg++;
3002 scan_funcs[scn->scn_phys.scn_func](scn->scn_dp, &bp, &zb);
3040 dsl_scan_ddt(dsl_scan_t *scn, dmu_tx_t *tx)
3042 ddt_bookmark_t *ddb = &scn->scn_phys.scn_ddt_bookmark;
3047 while ((error = ddt_walk(scn->scn_dp->dp_spa, ddb, &ddlwe)) == 0) {
3050 if (ddb->ddb_class > scn->scn_phys.scn_ddt_class_max)
3059 ddt = scn->scn_dp->dp_spa->spa_ddt[ddb->ddb_checksum];
3062 dsl_scan_ddt_entry(scn, ddb->ddb_checksum, ddt, &ddlwe, tx);
3065 if (dsl_scan_check_suspend(scn, NULL))
3070 dsl_scan_check_suspend(scn, NULL);
3075 scn->scn_dp->dp_spa->spa_name,
3076 (int)scn->scn_phys.scn_ddt_class_max,
3077 (int)scn->scn_suspending);
3081 scn->scn_dp->dp_spa->spa_name,
3082 (int)scn->scn_phys.scn_ddt_class_max,
3083 (int)scn->scn_suspending);
3087 ddb->ddb_class > scn->scn_phys.scn_ddt_class_max);
3100 dsl_scan_visit(dsl_scan_t *scn, dmu_tx_t *tx)
3103 dsl_pool_t *dp = scn->scn_dp;
3105 if (scn->scn_phys.scn_ddt_bookmark.ddb_class <=
3106 scn->scn_phys.scn_ddt_class_max) {
3107 scn->scn_phys.scn_cur_min_txg = scn->scn_phys.scn_min_txg;
3108 scn->scn_phys.scn_cur_max_txg = scn->scn_phys.scn_max_txg;
3109 dsl_scan_ddt(scn, tx);
3110 if (scn->scn_suspending)
3114 if (scn->scn_phys.scn_bookmark.zb_objset == DMU_META_OBJSET) {
3117 scn->scn_phys.scn_cur_min_txg = scn->scn_phys.scn_min_txg;
3118 scn->scn_phys.scn_cur_max_txg = scn->scn_phys.scn_max_txg;
3119 dsl_scan_visit_rootbp(scn, NULL,
3121 if (scn->scn_suspending)
3128 dsl_scan_visitds(scn,
3131 ASSERT(!scn->scn_suspending);
3132 } else if (scn->scn_phys.scn_bookmark.zb_objset !=
3134 uint64_t dsobj = scn->scn_phys.scn_bookmark.zb_objset;
3141 dsl_scan_visitds(scn, dsobj, tx);
3142 if (scn->scn_suspending)
3150 memset(&scn->scn_phys.scn_bookmark, 0, sizeof (zbookmark_phys_t));
3156 while ((sds = avl_first(&scn->scn_queue)) != NULL) {
3162 scan_ds_queue_remove(scn, dsobj);
3168 scn->scn_phys.scn_cur_min_txg =
3169 MAX(scn->scn_phys.scn_min_txg, txg);
3171 scn->scn_phys.scn_cur_min_txg =
3172 MAX(scn->scn_phys.scn_min_txg,
3175 scn->scn_phys.scn_cur_max_txg = dsl_scan_ds_maxtxg(ds);
3178 dsl_scan_visitds(scn, dsobj, tx);
3179 if (scn->scn_suspending)
3184 scn->scn_phys.scn_bookmark.zb_objset = ZB_DESTROYED_OBJSET;
3185 ASSERT0(scn->scn_suspending);
3226 scan_io_queue_check_suspend(dsl_scan_t *scn)
3230 uint64_t scan_time_ns = curr_time_ns - scn->scn_sync_start_time;
3232 scn->scn_dp->dp_spa->spa_sync_starttime;
3235 uint_t mintime = (scn->scn_phys.scn_func == POOL_SCAN_RESILVER) ?
3239 (scn->scn_dp->dp_dirty_total >= dirty_min_bytes ||
3240 txg_sync_waiting(scn->scn_dp) ||
3242 spa_shutting_down(scn->scn_dp->dp_spa));
3256 dsl_scan_t *scn = queue->q_scn;
3263 if (scan_io_queue_check_suspend(scn)) {
3269 scan_exec_io(scn->scn_dp, &bp, sio->sio_flags,
3367 dsl_scan_t *scn = queue->q_scn;
3371 ASSERT(scn->scn_is_sorted);
3373 if (!scn->scn_checkpointing && !scn->scn_clearing)
3385 if ((zfs_scan_issue_strategy < 1 && scn->scn_checkpointing) ||
3522 scan_io_queues_run(dsl_scan_t *scn)
3524 spa_t *spa = scn->scn_dp->dp_spa;
3526 ASSERT(scn->scn_is_sorted);
3529 if (scn->scn_queues_pending == 0)
3532 if (scn->scn_taskq == NULL) {
3542 scn->scn_taskq = taskq_create("dsl_scan_iss", nthreads,
3551 VERIFY(taskq_dispatch(scn->scn_taskq,
3563 taskq_wait(scn->scn_taskq);
3567 dsl_scan_async_block_should_pause(dsl_scan_t *scn)
3575 scn->scn_visited_this_txg >= zfs_async_block_max_blocks) {
3580 scn->scn_dedup_frees_this_txg >= zfs_max_async_dedup_frees) {
3584 elapsed_nanosecs = gethrtime() - scn->scn_sync_start_time;
3586 (NSEC2MSEC(elapsed_nanosecs) > scn->scn_async_block_min_time_ms &&
3587 txg_sync_waiting(scn->scn_dp)) ||
3588 spa_shutting_down(scn->scn_dp->dp_spa));
3594 dsl_scan_t *scn = arg;
3596 if (!scn->scn_is_bptree ||
3598 if (dsl_scan_async_block_should_pause(scn))
3602 zio_nowait(zio_free_sync(scn->scn_zio_root, scn->scn_dp->dp_spa,
3605 -bp_get_dsize_sync(scn->scn_dp->dp_spa, bp),
3607 scn->scn_visited_this_txg++;
3609 scn->scn_dedup_frees_this_txg++;
3614 dsl_scan_update_stats(dsl_scan_t *scn)
3616 spa_t *spa = scn->scn_dp->dp_spa;
3635 scn->scn_avg_seg_size_this_txg = 0;
3636 scn->scn_avg_zio_size_this_txg = 0;
3637 scn->scn_segs_this_txg = 0;
3638 scn->scn_zios_this_txg = 0;
3642 scn->scn_avg_seg_size_this_txg = seg_size_total / seg_count_total;
3643 scn->scn_avg_zio_size_this_txg = zio_size_total / zio_count_total;
3644 scn->scn_segs_this_txg = seg_count_total;
3645 scn->scn_zios_this_txg = zio_count_total;
3661 dsl_scan_t *scn = arg;
3664 if (dsl_scan_async_block_should_pause(scn))
3667 spa_vdev_indirect_mark_obsolete(scn->scn_dp->dp_spa,
3670 scn->scn_visited_this_txg++;
3675 dsl_scan_active(dsl_scan_t *scn)
3677 spa_t *spa = scn->scn_dp->dp_spa;
3685 if ((dsl_scan_is_running(scn) && !dsl_scan_is_paused_scrub(scn)) ||
3686 (scn->scn_async_destroying && !scn->scn_async_stalled))
3689 if (spa_version(scn->scn_dp->dp_spa) >= SPA_VERSION_DEADLISTS) {
3690 (void) bpobj_space(&scn->scn_dp->dp_free_bpobj,
3698 dsl_errorscrub_active(dsl_scan_t *scn)
3700 spa_t *spa = scn->scn_dp->dp_spa;
3705 if (dsl_errorscrubbing(scn->scn_dp))
3783 dsl_scan_t *scn = dp->dp_scan;
3792 scn->scn_is_bptree = B_FALSE;
3793 scn->scn_async_block_min_time_ms = zfs_free_min_time_ms;
3794 scn->scn_zio_root = zio_root(spa, NULL,
3797 bpobj_dsl_scan_free_block_cb, scn, tx);
3798 VERIFY0(zio_wait(scn->scn_zio_root));
3799 scn->scn_zio_root = NULL;
3806 ASSERT(scn->scn_async_destroying);
3807 scn->scn_is_bptree = B_TRUE;
3808 scn->scn_zio_root = zio_root(spa, NULL,
3811 dp->dp_bptree_obj, B_TRUE, dsl_scan_free_block_cb, scn, tx);
3812 VERIFY0(zio_wait(scn->scn_zio_root));
3813 scn->scn_zio_root = NULL;
3833 scn->scn_async_destroying = B_FALSE;
3834 scn->scn_async_stalled = B_FALSE;
3844 scn->scn_async_stalled =
3845 (scn->scn_visited_this_txg == 0);
3848 if (scn->scn_visited_this_txg) {
3851 (longlong_t)scn->scn_visited_this_txg,
3853 NSEC2MSEC(gethrtime() - scn->scn_sync_start_time),
3855 scn->scn_visited_this_txg = 0;
3856 scn->scn_dedup_frees_this_txg = 0;
3868 if (dp->dp_free_dir != NULL && !scn->scn_async_destroying &&
3896 if (dp->dp_free_dir != NULL && !scn->scn_async_destroying &&
3913 scn->scn_is_bptree = B_FALSE;
3914 scn->scn_async_block_min_time_ms = zfs_obsolete_min_time_ms;
3916 dsl_scan_obsolete_block_cb, scn, tx);
3947 read_by_block_level(dsl_scan_t *scn, zbookmark_phys_t zb)
3949 dsl_pool_t *dp = scn->scn_dp;
4026 dsl_scan_t *scn = dp->dp_scan;
4048 scn->scn_zio_root = zio_root(spa, NULL, NULL,
4051 read_by_block_level(scn, zb);
4053 (void) zio_wait(scn->scn_zio_root);
4054 scn->scn_zio_root = NULL;
4056 scn->errorscrub_phys.dep_examined++;
4057 scn->errorscrub_phys.dep_to_examine--;
4060 dsl_error_scrub_check_suspend(scn, &zb)) {
4128 scn->scn_zio_root = zio_root(spa, NULL, NULL,
4131 read_by_block_level(scn, zb);
4133 (void) zio_wait(scn->scn_zio_root);
4134 scn->scn_zio_root = NULL;
4136 scn->errorscrub_phys.dep_examined++;
4137 scn->errorscrub_phys.dep_to_examine--;
4140 dsl_error_scrub_check_suspend(scn, &zb)) {
4156 dsl_scan_t *scn = dp->dp_scan;
4173 if (!dsl_errorscrub_active(scn) || dsl_errorscrub_is_paused(scn)) {
4177 if (dsl_scan_resilvering(scn->scn_dp)) {
4179 dsl_scan_cancel(scn->scn_dp);
4184 scn->scn_sync_start_time = gethrtime();
4191 uint64_t scan_time_ns = gethrtime() - scn->scn_sync_start_time;
4195 !txg_sync_waiting(scn->scn_dp) &&
4196 !spa_shutting_down(scn->scn_dp->dp_spa) &&
4199 scan_time_ns = gethrtime() - scn->scn_sync_start_time;
4213 for (; zap_cursor_retrieve(&scn->errorscrub_cursor, za) == 0;
4214 zap_cursor_advance(&scn->errorscrub_cursor)) {
4217 scn->scn_zio_root = zio_root(dp->dp_spa, NULL,
4220 read_by_block_level(scn, *zb);
4223 (void) zio_wait(scn->scn_zio_root);
4224 scn->scn_zio_root = NULL;
4226 scn->errorscrub_phys.dep_examined += 1;
4227 scn->errorscrub_phys.dep_to_examine -= 1;
4230 dsl_error_scrub_check_suspend(scn, zb)) {
4237 dsl_errorscrub_done(scn, B_TRUE, tx);
4239 dsl_errorscrub_sync_state(scn, tx);
4246 for (; zap_cursor_retrieve(&scn->errorscrub_cursor, za) == 0;
4247 zap_cursor_advance(&scn->errorscrub_cursor)) {
4302 dsl_errorscrub_done(scn, B_TRUE, tx);
4304 dsl_errorscrub_sync_state(scn, tx);
4319 dsl_scan_t *scn = dp->dp_scan;
4336 scn->scn_phys.scn_to_examine - scn->scn_phys.scn_skipped;
4338 scn->scn_issued_before_pass + spa->spa_scan_pass_issued;
4359 if (dsl_scan_restarting(scn, tx) || restart_early) {
4365 dsl_scan_done(scn, B_FALSE, tx);
4385 if (!scn->scn_async_stalled && !dsl_scan_active(scn))
4389 scn->scn_visited_this_txg = 0;
4390 scn->scn_dedup_frees_this_txg = 0;
4391 scn->scn_holes_this_txg = 0;
4392 scn->scn_lt_min_this_txg = 0;
4393 scn->scn_gt_max_this_txg = 0;
4394 scn->scn_ddt_contained_this_txg = 0;
4395 scn->scn_objsets_visited_this_txg = 0;
4396 scn->scn_avg_seg_size_this_txg = 0;
4397 scn->scn_segs_this_txg = 0;
4398 scn->scn_avg_zio_size_this_txg = 0;
4399 scn->scn_zios_this_txg = 0;
4400 scn->scn_suspending = B_FALSE;
4401 scn->scn_sync_start_time = gethrtime();
4415 if (!dsl_scan_is_running(scn) || dsl_scan_is_paused_scrub(scn))
4432 uint64_t scan_time_ns = gethrtime() - scn->scn_sync_start_time;
4433 uint_t mintime = (scn->scn_phys.scn_func ==
4438 !txg_sync_waiting(scn->scn_dp) &&
4439 !spa_shutting_down(scn->scn_dp->dp_spa) &&
4442 scan_time_ns = gethrtime() - scn->scn_sync_start_time;
4453 scn->scn_issued_before_pass += spa->spa_scan_pass_issued;
4463 scn->scn_is_sorted = B_TRUE;
4464 if (scn->scn_last_checkpoint == 0)
4465 scn->scn_last_checkpoint = ddi_get_lbolt();
4473 if (scn->scn_is_sorted) {
4484 if (scn->scn_checkpointing ||
4485 ddi_get_lbolt() - scn->scn_last_checkpoint >
4487 if (!scn->scn_checkpointing)
4491 scn->scn_checkpointing = B_TRUE;
4492 scn->scn_clearing = B_TRUE;
4494 boolean_t should_clear = dsl_scan_should_clear(scn);
4495 if (should_clear && !scn->scn_clearing) {
4498 scn->scn_clearing = B_TRUE;
4499 } else if (!should_clear && scn->scn_clearing) {
4502 scn->scn_clearing = B_FALSE;
4506 ASSERT0(scn->scn_checkpointing);
4507 ASSERT0(scn->scn_clearing);
4510 if (!scn->scn_clearing && scn->scn_done_txg == 0) {
4512 dsl_scan_phys_t *scnp = &scn->scn_phys;
4521 scn->scn_maxinflight_bytes = MIN(arc_c_max / 4, MAX(1ULL << 20,
4546 scn->scn_zio_root = zio_root(dp->dp_spa, NULL,
4549 scn->scn_prefetch_stop = B_FALSE;
4551 dsl_scan_prefetch_thread, scn, TQ_SLEEP);
4555 dsl_scan_visit(scn, tx);
4559 scn->scn_prefetch_stop = B_TRUE;
4564 (void) zio_wait(scn->scn_zio_root);
4565 scn->scn_zio_root = NULL;
4570 (longlong_t)scn->scn_visited_this_txg,
4573 scn->scn_sync_start_time),
4574 (longlong_t)scn->scn_objsets_visited_this_txg,
4575 (longlong_t)scn->scn_holes_this_txg,
4576 (longlong_t)scn->scn_lt_min_this_txg,
4577 (longlong_t)scn->scn_ddt_contained_this_txg,
4578 (longlong_t)scn->scn_gt_max_this_txg);
4580 if (!scn->scn_suspending) {
4581 ASSERT0(avl_numnodes(&scn->scn_queue));
4582 scn->scn_done_txg = tx->tx_txg + 1;
4583 if (scn->scn_is_sorted) {
4584 scn->scn_checkpointing = B_TRUE;
4585 scn->scn_clearing = B_TRUE;
4586 scn->scn_issued_before_pass +=
4594 } else if (scn->scn_is_sorted && scn->scn_queues_pending != 0) {
4595 ASSERT(scn->scn_clearing);
4598 scn->scn_zio_root = zio_root(dp->dp_spa, NULL,
4600 scan_io_queues_run(scn);
4601 (void) zio_wait(scn->scn_zio_root);
4602 scn->scn_zio_root = NULL;
4605 (void) dsl_scan_should_clear(scn);
4606 dsl_scan_update_stats(scn);
4610 (longlong_t)scn->scn_zios_this_txg,
4612 (longlong_t)scn->scn_segs_this_txg,
4614 scn->scn_sync_start_time),
4615 (longlong_t)scn->scn_avg_zio_size_this_txg,
4616 (longlong_t)scn->scn_avg_seg_size_this_txg);
4617 } else if (scn->scn_done_txg != 0 && scn->scn_done_txg <= tx->tx_txg) {
4622 ASSERT3U(scn->scn_done_txg, !=, 0);
4624 ASSERT0(scn->scn_queues_pending);
4625 dsl_scan_done(scn, B_TRUE, tx);
4629 dsl_scan_sync_state(scn, tx, sync_type);
4654 count_block_skipped(dsl_scan_t *scn, const blkptr_t *bp, boolean_t all)
4658 atomic_add_64(&scn->scn_phys.scn_skipped,
4713 dsl_scan_t *scn = queue->q_scn;
4718 atomic_add_64(&scn->scn_queues_pending, 1);
4797 dsl_scan_t *scn = dp->dp_scan;
4805 if (phys_birth <= scn->scn_phys.scn_min_txg ||
4806 phys_birth >= scn->scn_phys.scn_max_txg) {
4807 count_block_skipped(scn, bp, B_TRUE);
4814 ASSERT(DSL_SCAN_IS_SCRUB_RESILVER(scn));
4815 if (scn->scn_phys.scn_func == POOL_SCAN_SCRUB) {
4819 ASSERT3U(scn->scn_phys.scn_func, ==, POOL_SCAN_RESILVER);
4836 scn->scn_phys.scn_examined += asize;
4848 count_block_skipped(scn, bp, B_TRUE);
4903 dsl_scan_t *scn = dp->dp_scan;
4909 ASSERT3U(scn->scn_maxinflight_bytes, >, 0);
4911 while (spa->spa_scrub_inflight >= scn->scn_maxinflight_bytes)
4915 pio = scn->scn_zio_root;
5068 dsl_scan_t *scn = vd->vdev_spa->spa_dsl_pool->dp_scan;
5071 q->q_scn = scn;
5092 dsl_scan_t *scn = queue->q_scn;
5099 atomic_add_64(&scn->scn_queues_pending, -1);
5139 scan_io_queues_destroy(dsl_scan_t *scn)
5141 vdev_t *rvd = scn->scn_dp->dp_spa->spa_root_vdev;
5158 dsl_scan_t *scn = dp->dp_scan;
5211 atomic_add_64(&scn->scn_queues_pending, -1);
5219 count_block_skipped(scn, &tmpbp, B_FALSE);
5236 dsl_scan_t *scn = dp->dp_scan;
5239 ASSERT(scn != NULL);
5240 if (!dsl_scan_is_running(scn))