107 SDT_PROBE_DEFINE1(sdt, kernel, threadpool, get,
109 SDT_PROBE_DEFINE1(sdt, kernel, threadpool, get__create,
111 SDT_PROBE_DEFINE1(sdt, kernel, threadpool, get__race,
113 SDT_PROBE_DEFINE2(sdt, kernel, threadpool, put,
115 SDT_PROBE_DEFINE2(sdt, kernel, threadpool, put__destroy,
118 SDT_PROBE_DEFINE1(sdt, kernel, threadpool, percpu__get,
120 SDT_PROBE_DEFINE1(sdt, kernel, threadpool, percpu__get__create,
122 SDT_PROBE_DEFINE1(sdt, kernel, threadpool, percpu__get__race,
124 SDT_PROBE_DEFINE2(sdt, kernel, threadpool, percpu__put,
126 SDT_PROBE_DEFINE2(sdt, kernel, threadpool, percpu__put__destroy,
129 SDT_PROBE_DEFINE2(sdt, kernel, threadpool, create,
131 SDT_PROBE_DEFINE3(sdt, kernel, threadpool, create__success,
133 SDT_PROBE_DEFINE3(sdt, kernel, threadpool, create__failure,
135 SDT_PROBE_DEFINE1(sdt, kernel, threadpool, destroy,
137 SDT_PROBE_DEFINE2(sdt, kernel, threadpool, destroy__wait,
140 SDT_PROBE_DEFINE2(sdt, kernel, threadpool, schedule__job,
142 SDT_PROBE_DEFINE2(sdt, kernel, threadpool, schedule__job__running,
144 SDT_PROBE_DEFINE2(sdt, kernel, threadpool, schedule__job__dispatcher,
146 SDT_PROBE_DEFINE3(sdt, kernel, threadpool, schedule__job__thread,
151 SDT_PROBE_DEFINE1(sdt, kernel, threadpool, dispatcher__start,
153 SDT_PROBE_DEFINE1(sdt, kernel, threadpool, dispatcher__dying,
155 SDT_PROBE_DEFINE1(sdt, kernel, threadpool, dispatcher__spawn,
157 SDT_PROBE_DEFINE2(sdt, kernel, threadpool, dispatcher__race,
160 SDT_PROBE_DEFINE3(sdt, kernel, threadpool, dispatcher__assign,
164 SDT_PROBE_DEFINE1(sdt, kernel, threadpool, dispatcher__exit,
167 SDT_PROBE_DEFINE1(sdt, kernel, threadpool, thread__start,
169 SDT_PROBE_DEFINE1(sdt, kernel, threadpool, thread__dying,
171 SDT_PROBE_DEFINE2(sdt, kernel, threadpool, thread__job,
173 SDT_PROBE_DEFINE1(sdt, kernel, threadpool, thread__exit,
184 	struct threadpool		*tpt_pool;
190 struct threadpool {  struct
202 static void	threadpool_hold(struct threadpool *);  argument
203 static void	threadpool_rele(struct threadpool *);
227 	struct threadpool		tpu_pool;
380 threadpool_create(struct threadpool *const pool, struct cpu_info *ci,  in threadpool_create()
390 	SDT_PROBE2(sdt, kernel, threadpool, create,  ci, pri);  in threadpool_create()
421 	SDT_PROBE3(sdt, kernel, threadpool, create__success,  ci, pri, pool);  in threadpool_create()
434 	SDT_PROBE3(sdt, kernel, threadpool, create__failure,  ci, pri, error);  in threadpool_create()
441 threadpool_destroy(struct threadpool *pool)  in threadpool_destroy()
445 	SDT_PROBE1(sdt, kernel, threadpool, destroy,  pool);  in threadpool_destroy()
455 		SDT_PROBE2(sdt, kernel, threadpool, destroy__wait,  in threadpool_destroy()
473 threadpool_hold(struct threadpool *pool)  in threadpool_hold()
482 threadpool_rele(struct threadpool *pool)  in threadpool_rele()
494 threadpool_get(struct threadpool **poolp, pri_t pri)  in threadpool_get()
501 	SDT_PROBE1(sdt, kernel, threadpool, get,  pri);  in threadpool_get()
510 		SDT_PROBE1(sdt, kernel, threadpool, get__create,  pri);  in threadpool_get()
524 			SDT_PROBE1(sdt, kernel, threadpool, get__race,  pri);  in threadpool_get()
542 threadpool_put(struct threadpool *pool, pri_t pri)  in threadpool_put()
550 	SDT_PROBE2(sdt, kernel, threadpool, put,  pool, pri);  in threadpool_put()
556 		SDT_PROBE2(sdt, kernel, threadpool, put__destroy,  pool, pri);  in threadpool_put()
579 	SDT_PROBE1(sdt, kernel, threadpool, percpu__get,  pri);  in threadpool_percpu_get()
588 		SDT_PROBE1(sdt, kernel, threadpool, percpu__get__create,  pri);  in threadpool_percpu_get()
600 			SDT_PROBE1(sdt, kernel, threadpool, percpu__get__race,  in threadpool_percpu_get()
624 	SDT_PROBE2(sdt, kernel, threadpool, percpu__put,  pool_percpu, pri);  in threadpool_percpu_put()
630 		SDT_PROBE2(sdt, kernel, threadpool, percpu__put__destroy,  in threadpool_percpu_put()
642 struct threadpool *
645 	struct threadpool **poolp, *pool;  in threadpool_percpu_ref()
654 struct threadpool *
658 	struct threadpool **poolp, *pool;  in threadpool_percpu_ref_remote()
682 	pool_percpu->tpp_percpu = percpu_create(sizeof(struct threadpool *),  in threadpool_percpu_create()
699 fail:	percpu_free(pool_percpu->tpp_percpu, sizeof(struct threadpool *));  in threadpool_percpu_create()
708 	percpu_free(pool_percpu->tpp_percpu, sizeof(struct threadpool *));  in threadpool_percpu_destroy()
715 	struct threadpool **const poolp = vpoolp;  in threadpool_percpu_init()
731 	struct threadpool **const poolp = vpoolp;  in threadpool_percpu_ok()
741 	struct threadpool **const poolp = vpoolp;  in threadpool_percpu_fini()
851 threadpool_schedule_job(struct threadpool *pool, struct threadpool_job *job)  in threadpool_schedule_job()
856 	SDT_PROBE2(sdt, kernel, threadpool, schedule__job,  pool, job);  in threadpool_schedule_job()
865 		SDT_PROBE2(sdt, kernel, threadpool, schedule__job__running,  in threadpool_schedule_job()
876 		SDT_PROBE2(sdt, kernel, threadpool, schedule__job__dispatcher,  in threadpool_schedule_job()
883 		SDT_PROBE3(sdt, kernel, threadpool, schedule__job__thread,  in threadpool_schedule_job()
897 threadpool_cancel_job_async(struct threadpool *pool, struct threadpool_job *job)  in threadpool_cancel_job_async()
941 threadpool_cancel_job(struct threadpool *pool, struct threadpool_job *job)  in threadpool_cancel_job()
967 	struct threadpool *const pool = dispatcher->tpt_pool;  in threadpool_dispatcher_thread()
981 	SDT_PROBE1(sdt, kernel, threadpool, dispatcher__start,  pool);  in threadpool_dispatcher_thread()
987 				SDT_PROBE1(sdt, kernel, threadpool,  in threadpool_dispatcher_thread()
998 			SDT_PROBE1(sdt, kernel, threadpool, dispatcher__spawn,  in threadpool_dispatcher_thread()
1063 				SDT_PROBE2(sdt, kernel, threadpool,  in threadpool_dispatcher_thread()
1075 				SDT_PROBE2(sdt, kernel, threadpool,  in threadpool_dispatcher_thread()
1094 	SDT_PROBE1(sdt, kernel, threadpool, dispatcher__exit,  pool);  in threadpool_dispatcher_thread()
1105 	struct threadpool *const pool = thread->tpt_pool;  in threadpool_thread()
1115 	SDT_PROBE1(sdt, kernel, threadpool, thread__start,  pool);  in threadpool_thread()
1122 				SDT_PROBE1(sdt, kernel, threadpool,  in threadpool_thread()
1148 		SDT_PROBE2(sdt, kernel, threadpool, thread__job,  pool, job);  in threadpool_thread()
1171 	SDT_PROBE1(sdt, kernel, threadpool, thread__exit,  pool);  in threadpool_thread()