122 	tx_state_t *tx = &dp->dp_tx;  in txg_init()  local
124 	memset(tx, 0, sizeof (tx_state_t));  in txg_init()
126 	tx->tx_cpu = vmem_zalloc(max_ncpus * sizeof (tx_cpu_t), KM_SLEEP);  in txg_init()
131 		mutex_init(&tx->tx_cpu[c].tc_lock, NULL, MUTEX_DEFAULT, NULL);  in txg_init()
132 		mutex_init(&tx->tx_cpu[c].tc_open_lock, NULL, MUTEX_NOLOCKDEP,  in txg_init()
135 			cv_init(&tx->tx_cpu[c].tc_cv[i], NULL, CV_DEFAULT,  in txg_init()
137 			list_create(&tx->tx_cpu[c].tc_callbacks[i],  in txg_init()
143 	mutex_init(&tx->tx_sync_lock, NULL, MUTEX_DEFAULT, NULL);  in txg_init()
145 	cv_init(&tx->tx_sync_more_cv, NULL, CV_DEFAULT, NULL);  in txg_init()
146 	cv_init(&tx->tx_sync_done_cv, NULL, CV_DEFAULT, NULL);  in txg_init()
147 	cv_init(&tx->tx_quiesce_more_cv, NULL, CV_DEFAULT, NULL);  in txg_init()
148 	cv_init(&tx->tx_quiesce_done_cv, NULL, CV_DEFAULT, NULL);  in txg_init()
149 	cv_init(&tx->tx_exit_cv, NULL, CV_DEFAULT, NULL);  in txg_init()
151 	tx->tx_open_txg = txg;  in txg_init()
160 	tx_state_t *tx = &dp->dp_tx;  in txg_fini()  local
163 	ASSERT0(tx->tx_threads);  in txg_fini()
165 	mutex_destroy(&tx->tx_sync_lock);  in txg_fini()
167 	cv_destroy(&tx->tx_sync_more_cv);  in txg_fini()
168 	cv_destroy(&tx->tx_sync_done_cv);  in txg_fini()
169 	cv_destroy(&tx->tx_quiesce_more_cv);  in txg_fini()
170 	cv_destroy(&tx->tx_quiesce_done_cv);  in txg_fini()
171 	cv_destroy(&tx->tx_exit_cv);  in txg_fini()
176 		mutex_destroy(&tx->tx_cpu[c].tc_open_lock);  in txg_fini()
177 		mutex_destroy(&tx->tx_cpu[c].tc_lock);  in txg_fini()
179 			cv_destroy(&tx->tx_cpu[c].tc_cv[i]);  in txg_fini()
180 			list_destroy(&tx->tx_cpu[c].tc_callbacks[i]);  in txg_fini()
184 	if (tx->tx_commit_cb_taskq != NULL)  in txg_fini()
185 		taskq_destroy(tx->tx_commit_cb_taskq);  in txg_fini()
187 	vmem_free(tx->tx_cpu, max_ncpus * sizeof (tx_cpu_t));  in txg_fini()
189 	memset(tx, 0, sizeof (tx_state_t));  in txg_fini()
198 	tx_state_t *tx = &dp->dp_tx;  in txg_sync_start()  local
200 	mutex_enter(&tx->tx_sync_lock);  in txg_sync_start()
204 	ASSERT0(tx->tx_threads);  in txg_sync_start()
206 	tx->tx_threads = 2;  in txg_sync_start()
208 	tx->tx_quiesce_thread = thread_create(NULL, 0, txg_quiesce_thread,  in txg_sync_start()
216 	tx->tx_sync_thread = thread_create(NULL, 0, txg_sync_thread,  in txg_sync_start()
219 	mutex_exit(&tx->tx_sync_lock);  in txg_sync_start()
223 txg_thread_enter(tx_state_t *tx, callb_cpr_t *cpr)  in txg_thread_enter()  argument
225 	CALLB_CPR_INIT(cpr, &tx->tx_sync_lock, callb_generic_cpr, FTAG);  in txg_thread_enter()
226 	mutex_enter(&tx->tx_sync_lock);  in txg_thread_enter()
230 txg_thread_exit(tx_state_t *tx, callb_cpr_t *cpr, kthread_t **tpp)  in txg_thread_exit()  argument
234 	tx->tx_threads--;  in txg_thread_exit()
235 	cv_broadcast(&tx->tx_exit_cv);  in txg_thread_exit()
236 	CALLB_CPR_EXIT(cpr);		/* drops &tx->tx_sync_lock */  in txg_thread_exit()
241 txg_thread_wait(tx_state_t *tx, callb_cpr_t *cpr, kcondvar_t *cv, clock_t time)  in txg_thread_wait()  argument
246 		(void) cv_timedwait_idle(cv, &tx->tx_sync_lock,  in txg_thread_wait()
249 		cv_wait_idle(cv, &tx->tx_sync_lock);  in txg_thread_wait()
252 	CALLB_CPR_SAFE_END(cpr, &tx->tx_sync_lock);  in txg_thread_wait()
261 	tx_state_t *tx = &dp->dp_tx;  in txg_sync_stop()  local
267 	ASSERT3U(tx->tx_threads, ==, 2);  in txg_sync_stop()
272 	txg_wait_synced(dp, tx->tx_open_txg + TXG_DEFER_SIZE);  in txg_sync_stop()
277 	mutex_enter(&tx->tx_sync_lock);  in txg_sync_stop()
279 	ASSERT3U(tx->tx_threads, ==, 2);  in txg_sync_stop()
281 	tx->tx_exiting = 1;  in txg_sync_stop()
283 	cv_broadcast(&tx->tx_quiesce_more_cv);  in txg_sync_stop()
284 	cv_broadcast(&tx->tx_quiesce_done_cv);  in txg_sync_stop()
285 	cv_broadcast(&tx->tx_sync_more_cv);  in txg_sync_stop()
287 	while (tx->tx_threads != 0)  in txg_sync_stop()
288 		cv_wait(&tx->tx_exit_cv, &tx->tx_sync_lock);  in txg_sync_stop()
290 	tx->tx_exiting = 0;  in txg_sync_stop()
292 	mutex_exit(&tx->tx_sync_lock);  in txg_sync_stop()
319 	tx_state_t *tx = &dp->dp_tx;  in txg_hold_open()  local
329 	tc = &tx->tx_cpu[CPU_SEQID_UNSTABLE];  in txg_hold_open()
332 	txg = tx->tx_open_txg;  in txg_hold_open()
388 	tx_state_t *tx = &dp->dp_tx;  in txg_quiesce()  local
397 		mutex_enter(&tx->tx_cpu[c].tc_open_lock);  in txg_quiesce()
399 	ASSERT(txg == tx->tx_open_txg);  in txg_quiesce()
400 	tx->tx_open_txg++;  in txg_quiesce()
401 	tx->tx_open_time = tx_open_time = gethrtime();  in txg_quiesce()
404 	DTRACE_PROBE2(txg__opened, dsl_pool_t *, dp, uint64_t, tx->tx_open_txg);  in txg_quiesce()
411 		mutex_exit(&tx->tx_cpu[c].tc_open_lock);  in txg_quiesce()
421 		tx_cpu_t *tc = &tx->tx_cpu[c];  in txg_quiesce()
451 	tx_state_t *tx = &dp->dp_tx;  in txg_dispatch_callbacks()  local
455 		tx_cpu_t *tc = &tx->tx_cpu[c];  in txg_dispatch_callbacks()
466 		if (tx->tx_commit_cb_taskq == NULL) {  in txg_dispatch_callbacks()
470 			tx->tx_commit_cb_taskq = taskq_create("tx_commit_cb",  in txg_dispatch_callbacks()
482 		(void) taskq_dispatch(tx->tx_commit_cb_taskq,  in txg_dispatch_callbacks()
495 	tx_state_t *tx = &dp->dp_tx;  in txg_wait_callbacks()  local
497 	if (tx->tx_commit_cb_taskq != NULL)  in txg_wait_callbacks()
498 		taskq_wait_outstanding(tx->tx_commit_cb_taskq, 0);  in txg_wait_callbacks()
504 	tx_state_t *tx = &dp->dp_tx;  in txg_is_quiescing()  local
505 	ASSERT(MUTEX_HELD(&tx->tx_sync_lock));  in txg_is_quiescing()
506 	return (tx->tx_quiescing_txg != 0);  in txg_is_quiescing()
512 	tx_state_t *tx = &dp->dp_tx;  in txg_has_quiesced_to_sync()  local
513 	ASSERT(MUTEX_HELD(&tx->tx_sync_lock));  in txg_has_quiesced_to_sync()
514 	return (tx->tx_quiesced_txg != 0);  in txg_has_quiesced_to_sync()
522 	tx_state_t *tx = &dp->dp_tx;  in txg_sync_thread()  local
527 	txg_thread_enter(tx, &cpr);  in txg_sync_thread()
542 		    !tx->tx_exiting && timer > 0 &&  in txg_sync_thread()
543 		    tx->tx_synced_txg >= tx->tx_sync_txg_waiting &&  in txg_sync_thread()
546 			    (u_longlong_t)tx->tx_synced_txg,  in txg_sync_thread()
547 			    (u_longlong_t)tx->tx_sync_txg_waiting, dp);  in txg_sync_thread()
548 			txg_thread_wait(tx, &cpr, &tx->tx_sync_more_cv, timer);  in txg_sync_thread()
566 		while (!tx->tx_exiting && !txg_has_quiesced_to_sync(dp)) {  in txg_sync_thread()
568 				txg_thread_wait(tx, &cpr,  in txg_sync_thread()
569 				    &tx->tx_quiesce_done_cv, 0);  in txg_sync_thread()
572 			if (tx->tx_quiesce_txg_waiting < tx->tx_open_txg+1)  in txg_sync_thread()
573 				tx->tx_quiesce_txg_waiting = tx->tx_open_txg+1;  in txg_sync_thread()
574 			cv_broadcast(&tx->tx_quiesce_more_cv);  in txg_sync_thread()
575 			txg_thread_wait(tx, &cpr, &tx->tx_quiesce_done_cv, 0);  in txg_sync_thread()
578 		if (tx->tx_exiting)  in txg_sync_thread()
579 			txg_thread_exit(tx, &cpr, &tx->tx_sync_thread);  in txg_sync_thread()
586 		ASSERT(tx->tx_quiesced_txg != 0);  in txg_sync_thread()
587 		txg = tx->tx_quiesced_txg;  in txg_sync_thread()
588 		tx->tx_quiesced_txg = 0;  in txg_sync_thread()
589 		tx->tx_syncing_txg = txg;  in txg_sync_thread()
591 		cv_broadcast(&tx->tx_quiesce_more_cv);  in txg_sync_thread()
594 		    (u_longlong_t)txg, (u_longlong_t)tx->tx_quiesce_txg_waiting,  in txg_sync_thread()
595 		    (u_longlong_t)tx->tx_sync_txg_waiting);  in txg_sync_thread()
596 		mutex_exit(&tx->tx_sync_lock);  in txg_sync_thread()
604 		mutex_enter(&tx->tx_sync_lock);  in txg_sync_thread()
605 		tx->tx_synced_txg = txg;  in txg_sync_thread()
606 		tx->tx_syncing_txg = 0;  in txg_sync_thread()
608 		cv_broadcast(&tx->tx_sync_done_cv);  in txg_sync_thread()
621 	tx_state_t *tx = &dp->dp_tx;  in txg_quiesce_thread()  local
624 	txg_thread_enter(tx, &cpr);  in txg_quiesce_thread()
636 		while (!tx->tx_exiting &&  in txg_quiesce_thread()
637 		    (tx->tx_open_txg >= tx->tx_quiesce_txg_waiting ||  in txg_quiesce_thread()
639 			txg_thread_wait(tx, &cpr, &tx->tx_quiesce_more_cv, 0);  in txg_quiesce_thread()
641 		if (tx->tx_exiting)  in txg_quiesce_thread()
642 			txg_thread_exit(tx, &cpr, &tx->tx_quiesce_thread);  in txg_quiesce_thread()
644 		txg = tx->tx_open_txg;  in txg_quiesce_thread()
647 		    (u_longlong_t)tx->tx_quiesce_txg_waiting,  in txg_quiesce_thread()
648 		    (u_longlong_t)tx->tx_sync_txg_waiting);  in txg_quiesce_thread()
649 		tx->tx_quiescing_txg = txg;  in txg_quiesce_thread()
651 		mutex_exit(&tx->tx_sync_lock);  in txg_quiesce_thread()
653 		mutex_enter(&tx->tx_sync_lock);  in txg_quiesce_thread()
660 		tx->tx_quiescing_txg = 0;  in txg_quiesce_thread()
661 		tx->tx_quiesced_txg = txg;  in txg_quiesce_thread()
663 		cv_broadcast(&tx->tx_sync_more_cv);  in txg_quiesce_thread()
664 		cv_broadcast(&tx->tx_quiesce_done_cv);  in txg_quiesce_thread()
676 	tx_state_t *tx = &dp->dp_tx;  in txg_delay()  local
680 	if (tx->tx_open_txg > txg ||  in txg_delay()
681 	    tx->tx_syncing_txg == txg-1 || tx->tx_synced_txg == txg-1)  in txg_delay()
684 	mutex_enter(&tx->tx_sync_lock);  in txg_delay()
685 	if (tx->tx_open_txg > txg || tx->tx_synced_txg == txg-1) {  in txg_delay()
686 		mutex_exit(&tx->tx_sync_lock);  in txg_delay()
691 	    tx->tx_syncing_txg < txg-1 && !txg_stalled(dp)) {  in txg_delay()
692 		(void) cv_timedwait_hires(&tx->tx_quiesce_more_cv,  in txg_delay()
693 		    &tx->tx_sync_lock, delay, resolution, 0);  in txg_delay()
698 	mutex_exit(&tx->tx_sync_lock);  in txg_delay()
704 	tx_state_t *tx = &dp->dp_tx;  in txg_wait_synced_impl()  local
708 	mutex_enter(&tx->tx_sync_lock);  in txg_wait_synced_impl()
709 	ASSERT3U(tx->tx_threads, ==, 2);  in txg_wait_synced_impl()
711 		txg = tx->tx_open_txg + TXG_DEFER_SIZE;  in txg_wait_synced_impl()
712 	if (tx->tx_sync_txg_waiting < txg)  in txg_wait_synced_impl()
713 		tx->tx_sync_txg_waiting = txg;  in txg_wait_synced_impl()
715 	    (u_longlong_t)txg, (u_longlong_t)tx->tx_quiesce_txg_waiting,  in txg_wait_synced_impl()
716 	    (u_longlong_t)tx->tx_sync_txg_waiting);  in txg_wait_synced_impl()
717 	while (tx->tx_synced_txg < txg) {  in txg_wait_synced_impl()
720 		    (u_longlong_t)tx->tx_synced_txg,  in txg_wait_synced_impl()
721 		    (u_longlong_t)tx->tx_sync_txg_waiting, dp);  in txg_wait_synced_impl()
722 		cv_broadcast(&tx->tx_sync_more_cv);  in txg_wait_synced_impl()
729 			if (cv_wait_io_sig(&tx->tx_sync_done_cv,  in txg_wait_synced_impl()
730 			    &tx->tx_sync_lock) == 0) {  in txg_wait_synced_impl()
731 				mutex_exit(&tx->tx_sync_lock);  in txg_wait_synced_impl()
735 			cv_wait_io(&tx->tx_sync_done_cv, &tx->tx_sync_lock);  in txg_wait_synced_impl()
738 	mutex_exit(&tx->tx_sync_lock);  in txg_wait_synced_impl()
765 	tx_state_t *tx = &dp->dp_tx;  in txg_wait_open()  local
769 	mutex_enter(&tx->tx_sync_lock);  in txg_wait_open()
770 	ASSERT3U(tx->tx_threads, ==, 2);  in txg_wait_open()
772 		txg = tx->tx_open_txg + 1;  in txg_wait_open()
773 	if (tx->tx_quiesce_txg_waiting < txg && should_quiesce)  in txg_wait_open()
774 		tx->tx_quiesce_txg_waiting = txg;  in txg_wait_open()
776 	    (u_longlong_t)txg, (u_longlong_t)tx->tx_quiesce_txg_waiting,  in txg_wait_open()
777 	    (u_longlong_t)tx->tx_sync_txg_waiting);  in txg_wait_open()
778 	while (tx->tx_open_txg < txg) {  in txg_wait_open()
779 		cv_broadcast(&tx->tx_quiesce_more_cv);  in txg_wait_open()
787 			cv_wait_io(&tx->tx_quiesce_done_cv, &tx->tx_sync_lock);  in txg_wait_open()
789 			cv_wait_idle(&tx->tx_quiesce_done_cv,  in txg_wait_open()
790 			    &tx->tx_sync_lock);  in txg_wait_open()
793 	mutex_exit(&tx->tx_sync_lock);  in txg_wait_open()
802 	tx_state_t *tx = &dp->dp_tx;  in txg_kick()  local
806 	if (tx->tx_sync_txg_waiting >= txg)  in txg_kick()
809 	mutex_enter(&tx->tx_sync_lock);  in txg_kick()
810 	if (tx->tx_sync_txg_waiting < txg) {  in txg_kick()
811 		tx->tx_sync_txg_waiting = txg;  in txg_kick()
812 		cv_broadcast(&tx->tx_sync_more_cv);  in txg_kick()
814 	mutex_exit(&tx->tx_sync_lock);  in txg_kick()
820 	tx_state_t *tx = &dp->dp_tx;  in txg_stalled()  local
821 	return (tx->tx_quiesce_txg_waiting > tx->tx_open_txg);  in txg_stalled()
827 	tx_state_t *tx = &dp->dp_tx;  in txg_sync_waiting()  local
829 	return (tx->tx_syncing_txg <= tx->tx_sync_txg_waiting ||  in txg_sync_waiting()
830 	    tx->tx_quiesced_txg != 0);  in txg_sync_waiting()