380 threadpool_create(struct threadpool *const pool, struct cpu_info *ci,  in threadpool_create()  argument
392 	mutex_init(&pool->tp_lock, MUTEX_DEFAULT, IPL_VM);  in threadpool_create()
394 	TAILQ_INIT(&pool->tp_jobs);  in threadpool_create()
395 	TAILQ_INIT(&pool->tp_idle_threads);  in threadpool_create()
396 	pool->tp_refcnt = 1;		/* dispatcher's reference */  in threadpool_create()
397 	pool->tp_flags = 0;  in threadpool_create()
398 	pool->tp_cpu = ci;  in threadpool_create()
399 	pool->tp_pri = pri;  in threadpool_create()
401 	pool->tp_dispatcher.tpt_lwp = NULL;  in threadpool_create()
402 	pool->tp_dispatcher.tpt_pool = pool;  in threadpool_create()
403 	pool->tp_dispatcher.tpt_job = NULL;  in threadpool_create()
404 	cv_init(&pool->tp_dispatcher.tpt_cv, "pooldisp");  in threadpool_create()
412 	    &pool->tp_dispatcher, &lwp, "pooldisp%s", suffix);  in threadpool_create()
416 	mutex_spin_enter(&pool->tp_lock);  in threadpool_create()
417 	pool->tp_dispatcher.tpt_lwp = lwp;  in threadpool_create()
418 	cv_broadcast(&pool->tp_dispatcher.tpt_cv);  in threadpool_create()
419 	mutex_spin_exit(&pool->tp_lock);  in threadpool_create()
421 	SDT_PROBE3(sdt, kernel, threadpool, create__success,  ci, pri, pool);  in threadpool_create()
425 	KASSERT(pool->tp_dispatcher.tpt_job == NULL);  in threadpool_create()
426 	KASSERT(pool->tp_dispatcher.tpt_pool == pool);  in threadpool_create()
427 	KASSERT(pool->tp_flags == 0);  in threadpool_create()
428 	KASSERT(pool->tp_refcnt == 0);  in threadpool_create()
429 	KASSERT(TAILQ_EMPTY(&pool->tp_idle_threads));  in threadpool_create()
430 	KASSERT(TAILQ_EMPTY(&pool->tp_jobs));  in threadpool_create()
431 	KASSERT(!cv_has_waiters(&pool->tp_dispatcher.tpt_cv));  in threadpool_create()
432 	cv_destroy(&pool->tp_dispatcher.tpt_cv);  in threadpool_create()
433 	mutex_destroy(&pool->tp_lock);  in threadpool_create()
441 threadpool_destroy(struct threadpool *pool)  in threadpool_destroy()  argument
445 	SDT_PROBE1(sdt, kernel, threadpool, destroy,  pool);  in threadpool_destroy()
448 	mutex_spin_enter(&pool->tp_lock);  in threadpool_destroy()
449 	KASSERT(TAILQ_EMPTY(&pool->tp_jobs));  in threadpool_destroy()
450 	pool->tp_flags |= THREADPOOL_DYING;  in threadpool_destroy()
451 	cv_broadcast(&pool->tp_dispatcher.tpt_cv);  in threadpool_destroy()
452 	TAILQ_FOREACH(thread, &pool->tp_idle_threads, tpt_entry)  in threadpool_destroy()
454 	while (0 < pool->tp_refcnt) {  in threadpool_destroy()
456 		    pool, pool->tp_refcnt);  in threadpool_destroy()
457 		cv_wait(&pool->tp_dispatcher.tpt_cv, &pool->tp_lock);  in threadpool_destroy()
459 	mutex_spin_exit(&pool->tp_lock);  in threadpool_destroy()
461 	KASSERT(pool->tp_dispatcher.tpt_job == NULL);  in threadpool_destroy()
462 	KASSERT(pool->tp_dispatcher.tpt_pool == pool);  in threadpool_destroy()
463 	KASSERT(pool->tp_flags == THREADPOOL_DYING);  in threadpool_destroy()
464 	KASSERT(pool->tp_refcnt == 0);  in threadpool_destroy()
465 	KASSERT(TAILQ_EMPTY(&pool->tp_idle_threads));  in threadpool_destroy()
466 	KASSERT(TAILQ_EMPTY(&pool->tp_jobs));  in threadpool_destroy()
467 	KASSERT(!cv_has_waiters(&pool->tp_dispatcher.tpt_cv));  in threadpool_destroy()
468 	cv_destroy(&pool->tp_dispatcher.tpt_cv);  in threadpool_destroy()
469 	mutex_destroy(&pool->tp_lock);  in threadpool_destroy()
473 threadpool_hold(struct threadpool *pool)  in threadpool_hold()  argument
476 	KASSERT(mutex_owned(&pool->tp_lock));  in threadpool_hold()
477 	pool->tp_refcnt++;  in threadpool_hold()
478 	KASSERT(pool->tp_refcnt != 0);  in threadpool_hold()
482 threadpool_rele(struct threadpool *pool)  in threadpool_rele()  argument
485 	KASSERT(mutex_owned(&pool->tp_lock));  in threadpool_rele()
486 	KASSERT(0 < pool->tp_refcnt);  in threadpool_rele()
487 	if (--pool->tp_refcnt == 0)  in threadpool_rele()
488 		cv_broadcast(&pool->tp_dispatcher.tpt_cv);  in threadpool_rele()
542 threadpool_put(struct threadpool *pool, pri_t pri)  in threadpool_put()  argument
545 	    container_of(pool, struct threadpool_unbound, tpu_pool);  in threadpool_put()
550 	SDT_PROBE2(sdt, kernel, threadpool, put,  pool, pri);  in threadpool_put()
556 		SDT_PROBE2(sdt, kernel, threadpool, put__destroy,  pool, pri);  in threadpool_put()
645 	struct threadpool **poolp, *pool;  in threadpool_percpu_ref()  local
648 	pool = *poolp;  in threadpool_percpu_ref()
651 	return pool;  in threadpool_percpu_ref()
658 	struct threadpool **poolp, *pool;  in threadpool_percpu_ref_remote()  local
668 	pool = *poolp;  in threadpool_percpu_ref_remote()
671 	return pool;  in threadpool_percpu_ref_remote()
851 threadpool_schedule_job(struct threadpool *pool, struct threadpool_job *job)  in threadpool_schedule_job()  argument
856 	SDT_PROBE2(sdt, kernel, threadpool, schedule__job,  pool, job);  in threadpool_schedule_job()
866 		    pool, job);  in threadpool_schedule_job()
873 	mutex_spin_enter(&pool->tp_lock);  in threadpool_schedule_job()
874 	if (__predict_false(TAILQ_EMPTY(&pool->tp_idle_threads))) {  in threadpool_schedule_job()
877 		    pool, job);  in threadpool_schedule_job()
878 		job->job_thread = &pool->tp_dispatcher;  in threadpool_schedule_job()
879 		TAILQ_INSERT_TAIL(&pool->tp_jobs, job, job_entry);  in threadpool_schedule_job()
882 		job->job_thread = TAILQ_FIRST(&pool->tp_idle_threads);  in threadpool_schedule_job()
884 		    pool, job, job->job_thread->tpt_lwp);  in threadpool_schedule_job()
885 		TAILQ_REMOVE(&pool->tp_idle_threads, job->job_thread,  in threadpool_schedule_job()
893 	mutex_spin_exit(&pool->tp_lock);  in threadpool_schedule_job()
897 threadpool_cancel_job_async(struct threadpool *pool, struct threadpool_job *job)  in threadpool_cancel_job_async()  argument
926 	} else if (job->job_thread == &pool->tp_dispatcher) {  in threadpool_cancel_job_async()
929 		mutex_spin_enter(&pool->tp_lock);  in threadpool_cancel_job_async()
930 		TAILQ_REMOVE(&pool->tp_jobs, job, job_entry);  in threadpool_cancel_job_async()
931 		mutex_spin_exit(&pool->tp_lock);  in threadpool_cancel_job_async()
941 threadpool_cancel_job(struct threadpool *pool, struct threadpool_job *job)  in threadpool_cancel_job()  argument
953 	if (threadpool_cancel_job_async(pool, job))  in threadpool_cancel_job()
967 	struct threadpool *const pool = dispatcher->tpt_pool;  in threadpool_dispatcher_thread()  local
973 	KASSERT((pool->tp_cpu == NULL) || (pool->tp_cpu == curcpu()));  in threadpool_dispatcher_thread()
974 	KASSERT((pool->tp_cpu == NULL) || (curlwp->l_pflag & LP_BOUND));  in threadpool_dispatcher_thread()
977 	mutex_spin_enter(&pool->tp_lock);  in threadpool_dispatcher_thread()
979 		cv_wait(&dispatcher->tpt_cv, &pool->tp_lock);  in threadpool_dispatcher_thread()
981 	SDT_PROBE1(sdt, kernel, threadpool, dispatcher__start,  pool);  in threadpool_dispatcher_thread()
985 		while (TAILQ_EMPTY(&pool->tp_jobs)) {  in threadpool_dispatcher_thread()
986 			if (ISSET(pool->tp_flags, THREADPOOL_DYING)) {  in threadpool_dispatcher_thread()
988 				    dispatcher__dying,  pool);  in threadpool_dispatcher_thread()
991 			cv_wait(&dispatcher->tpt_cv, &pool->tp_lock);  in threadpool_dispatcher_thread()
993 		if (__predict_false(TAILQ_EMPTY(&pool->tp_jobs)))  in threadpool_dispatcher_thread()
997 		if (TAILQ_EMPTY(&pool->tp_idle_threads)) {  in threadpool_dispatcher_thread()
999 			    pool);  in threadpool_dispatcher_thread()
1000 			threadpool_hold(pool);  in threadpool_dispatcher_thread()
1001 			mutex_spin_exit(&pool->tp_lock);  in threadpool_dispatcher_thread()
1006 			thread->tpt_pool = pool;  in threadpool_dispatcher_thread()
1012 			if (pool->tp_pri < PRI_KERNEL)  in threadpool_dispatcher_thread()
1014 			threadnamesuffix(suffix, sizeof(suffix), pool->tp_cpu,  in threadpool_dispatcher_thread()
1015 			    pool->tp_pri);  in threadpool_dispatcher_thread()
1016 			error = kthread_create(pool->tp_pri, ktflags,  in threadpool_dispatcher_thread()
1017 			    pool->tp_cpu, &threadpool_thread, thread, &lwp,  in threadpool_dispatcher_thread()
1020 			mutex_spin_enter(&pool->tp_lock);  in threadpool_dispatcher_thread()
1023 				threadpool_rele(pool);  in threadpool_dispatcher_thread()
1026 				    &pool->tp_lock);  in threadpool_dispatcher_thread()
1034 			TAILQ_INSERT_TAIL(&pool->tp_idle_threads, thread,  in threadpool_dispatcher_thread()
1043 		struct threadpool_job *const job = TAILQ_FIRST(&pool->tp_jobs);  in threadpool_dispatcher_thread()
1050 		mutex_spin_exit(&pool->tp_lock);  in threadpool_dispatcher_thread()
1055 			mutex_spin_enter(&pool->tp_lock);  in threadpool_dispatcher_thread()
1056 			TAILQ_REMOVE(&pool->tp_jobs, job, job_entry);  in threadpool_dispatcher_thread()
1058 				    TAILQ_EMPTY(&pool->tp_idle_threads))) {  in threadpool_dispatcher_thread()
1064 				    dispatcher__race,  pool, job);  in threadpool_dispatcher_thread()
1065 				TAILQ_INSERT_HEAD(&pool->tp_jobs, job,  in threadpool_dispatcher_thread()
1073 				    TAILQ_FIRST(&pool->tp_idle_threads);  in threadpool_dispatcher_thread()
1078 				TAILQ_REMOVE(&pool->tp_idle_threads, thread,  in threadpool_dispatcher_thread()
1084 			mutex_spin_exit(&pool->tp_lock);  in threadpool_dispatcher_thread()
1089 		mutex_spin_enter(&pool->tp_lock);  in threadpool_dispatcher_thread()
1091 	threadpool_rele(pool);  in threadpool_dispatcher_thread()
1092 	mutex_spin_exit(&pool->tp_lock);  in threadpool_dispatcher_thread()
1094 	SDT_PROBE1(sdt, kernel, threadpool, dispatcher__exit,  pool);  in threadpool_dispatcher_thread()
1105 	struct threadpool *const pool = thread->tpt_pool;  in threadpool_thread()  local
1107 	KASSERT((pool->tp_cpu == NULL) || (pool->tp_cpu == curcpu()));  in threadpool_thread()
1108 	KASSERT((pool->tp_cpu == NULL) || (curlwp->l_pflag & LP_BOUND));  in threadpool_thread()
1111 	mutex_spin_enter(&pool->tp_lock);  in threadpool_thread()
1113 		cv_wait(&thread->tpt_cv, &pool->tp_lock);  in threadpool_thread()
1115 	SDT_PROBE1(sdt, kernel, threadpool, thread__start,  pool);  in threadpool_thread()
1121 			if (ISSET(pool->tp_flags, THREADPOOL_DYING)) {  in threadpool_thread()
1123 				    thread__dying,  pool);  in threadpool_thread()
1126 			if (cv_timedwait(&thread->tpt_cv, &pool->tp_lock,  in threadpool_thread()
1131 			TAILQ_REMOVE(&pool->tp_idle_threads, thread,  in threadpool_thread()
1146 		mutex_spin_exit(&pool->tp_lock);  in threadpool_thread()
1148 		SDT_PROBE2(sdt, kernel, threadpool, thread__job,  pool, job);  in threadpool_thread()
1163 		mutex_spin_enter(&pool->tp_lock);  in threadpool_thread()
1166 		TAILQ_INSERT_TAIL(&pool->tp_idle_threads, thread, tpt_entry);  in threadpool_thread()
1168 	threadpool_rele(pool);  in threadpool_thread()
1169 	mutex_spin_exit(&pool->tp_lock);  in threadpool_thread()
1171 	SDT_PROBE1(sdt, kernel, threadpool, thread__exit,  pool);  in threadpool_thread()