1.\" $NetBSD: threadpool.9,v 1.4 2020/09/07 01:07:38 riastradh Exp $ 2.\" 3.\" Copyright (c) 2014 The NetBSD Foundation, Inc. 4.\" All rights reserved. 5.\" 6.\" This code is derived from software contributed to The NetBSD Foundation 7.\" by Taylor R. Campbell. 8.\" 9.\" Redistribution and use in source and binary forms, with or without 10.\" modification, are permitted provided that the following conditions 11.\" are met: 12.\" 1. Redistributions of source code must retain the above copyright 13.\" notice, this list of conditions and the following disclaimer. 14.\" 2. Redistributions in binary form must reproduce the above copyright 15.\" notice, this list of conditions and the following disclaimer in the 16.\" documentation and/or other materials provided with the distribution. 17.\" 18.\" THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 19.\" ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 20.\" TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 21.\" PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 22.\" BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 23.\" CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 24.\" SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 25.\" INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 26.\" CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 27.\" ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 28.\" POSSIBILITY OF SUCH DAMAGE. 29.\" 30.Dd December 26, 2018 31.Dt THREADPOOL 9 32.Os 33.\" 34.Sh NAME 35.Nm threadpool 36.Nd shared pools of kthreads 37.\""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" 38.Sh SYNOPSIS 39.In sys/threadpool.h 40.\"""""""""""""""""""""""""""""""""""" 41.Vt typedef void threadpool_job_fn_t(struct threadpool_job *); 42.\"""""""""""""""""""""""""""""""""""" 43.Ft int 44.Fn threadpool_get "struct threadpool **poolp" "pri_t pri" 45.\" 46.Ft void 47.Fn threadpool_put "struct threadpool *pool" "pri_t pri" 48.\"""""""""""""""""""""""""""""""""""" 49.Ft int 50.Fn threadpool_percpu_get "struct threadpool_percpu **pool_percpup" "pri_t pri" 51.\" 52.Ft void 53.Fn threadpool_percpu_put "struct threadpool_percpu *pool_percpu" "pri_t pri" 54.\" 55.Ft struct threadpool * 56.Fn threadpool_percpu_ref "struct threadpool_percpu *pool" 57.\" 58.Ft struct threadpool * 59.Fn threadpool_percpu_ref_remote "struct threadpool_percpu *pool" "struct cpu_info *ci" 60.\"""""""""""""""""""""""""""""""""""" 61.Ft void 62.Fn threadpool_job_init "struct threadpool_job *job" "threadpool_job_fn_t fn" "kmutex_t *interlock" "const char *fmt" "..." 63.\" 64.Ft void 65.Fn threadpool_job_destroy "struct threadpool_job *job" 66.\" 67.Ft void 68.Fn threadpool_job_done "struct threadpool_job *job" 69.\"""""""""""""""""""""""""""""""""""" 70.Ft void 71.Fn threadpool_schedule_job "struct threadpool *pool" "struct threadpool_job *job" 72.\" 73.Ft void 74.Fn threadpool_cancel_job "struct threadpool *pool" "struct threadpool_job *job" 75.\" 76.Ft bool 77.Fn threadpool_cancel_job_async "struct threadpool *pool" "struct threadpool_job *job" 78.\""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" 79.Sh DESCRIPTION 80The 81.Nm 82abstraction is provided to share a pool of 83.Xr kthread 9 84kernel threads for medium- to long-term actions, called jobs, which can 85be scheduled from contexts that do not allow sleeping. 86.Pp 87For each priority level, there is one unbound thread pool, and one 88collection of per-CPU thread pools. 89Access to the unbound thread pools is provided by 90.Fn threadpool_get 91and 92.Fn threadpool_put . 93Access to the per-CPU thread pools is provided by 94.Fn threadpool_percpu_get 95and 96.Fn threadpool_percpu_put . 97.Pp 98Job state is stored in the 99.Vt threadpool_job 100structure. 101Callers of the 102.Nm 103abstraction 104must allocate memory for 105.Vt threadpool_job 106structures, but should consider them opaque, and should not inspect or 107copy them. 108Each job represented by a 109.Vt threadpool_job 110structure will be run only once at a time, until the action associated 111with it calls 112.Fn threadpool_job_done . 113.Pp 114Jobs are run in thread context and may take arbitrarily long to run or 115sleep arbitrarily long. 116.\" The 117.\" .Nm 118.\" abstraction is intended as a building block for cheaper abstractions, 119.\" namely 120.\" .Xr task 9 121.\" and 122.\" .Xr workqueue 9 . 123.\" It should generally not be used directly. 124.\""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" 125.Sh FUNCTIONS 126.\" 127.Bl -tag -width abcd 128.\"""""""""""""""""""""""""""""""""""" 129.It Fn threadpool_get "poolp" "pri" 130Obtain a reference to the unbound thread pool at priority 131.Fa pri 132and store it in 133.Fa poolp . 134.Pp 135May sleep. 136.\" 137.It Fn threadpool_put "pool" "pri" 138Release the reference to the unbound thread pool 139.Fa pool 140at priority 141.Fa pri , 142which must be the same as the priority that was passed to 143.Fn threadpool_get 144to obtain 145.Fa pool . 146.Pp 147May sleep. 148.Pp 149Do not use 150.Fn threadpool_put 151with thread pools obtained from 152.Fn threadpool_percpu_ref 153or 154.Fn threadpool_percpu_ref_remote . 155.\"""""""""""""""""""""""""""""""""""" 156.It Fn threadpool_percpu_get "pool_percpup" "pri" 157Obtain a reference to the per-CPU thread pool at priority 158.Fa pri 159and store it in 160.Fa pool_percpup . 161.Pp 162Use 163.Fn threadpool_percpu_ref 164or 165.Fn threadpool_percpu_ref_remote 166with it to get at the thread pool for a particular CPU. 167.Pp 168May sleep. 169.\" 170.It Fn threadpool_percpu_put "pool_percpu" "pri" 171Release a reference to the per-CPU thread pool 172.Fa pool_percpu 173at priority 174.Fa pri . 175.Pp 176May sleep. 177.\" 178.It Fn threadpool_percpu_ref "pool_percpu" 179Return the thread pool in 180.Fa pool_percpu 181for the current CPU. 182.Pp 183The resulting thread pool pointer is stable until 184.Fa pool_percpu 185is released with 186.Fn threadpool_percpu_put . 187Using it to schedule or cancel a job does not require being on the same 188CPU. 189.Pp 190Do not use 191.Fn threadpool_put 192with thread pools obtained from 193.Fn threadpool_percpu_ref . 194.\" 195.It Fn threadpool_percpu_ref_remote "pool_percpu" "ci" 196Return the thread pool in 197.Fa pool_percpu 198for the CPU whose 199.Vt struct cpu_info 200is given by 201.Fa ci . 202.Pp 203The resulting thread pool pointer is stable until 204.Fa pool_percpu 205is released with 206.Fn threadpool_percpu_put . 207Using it to schedule or cancel a job does not require being on the same 208CPU, but it is faster and friendlier to the cache to use 209.Fn threadpool_percpu_ref 210and use the resulting thread pool only on the same CPU. 211.Pp 212Do not use 213.Fn threadpool_put 214with thread pools obtained from 215.Fn threadpool_percpu_ref_remote . 216.\"""""""""""""""""""""""""""""""""""" 217.It Fn threadpool_job_init "job" "fn" "interlock" "fmt" "..." 218Initialize the threadpool job 219.Fa job 220to run 221.Fa fn 222when scheduled and to interlock with 223.Fa interlock . 224The argument 225.Fa fmt 226is a 227.Xr printf 9 228format string for the job's name. 229.Pp 230The mutex 231.Fa interlock 232is used to synchronize job scheduling and completion. 233The action 234.Fa fn 235is required to eventually call 236.Fn threadpool_job_done , 237with 238.Fa interlock 239held. 240This is so that while the job is running and may be waiting for work 241to do, scheduling the job has no effect, but as soon as the job is 242done, scheduling the job will cause it to run again. 243.Pp 244To change the action of a job, you must use 245.Fn threadpool_job_destroy 246first and then call 247.Fn threadpool_job_init 248again. 249.\" 250.It Fn threadpool_job_destroy "job" 251Destroy the threadpool job 252.Fa job . 253.Fa job 254must not currently be scheduled to run. 255If it may still be scheduled, you can use 256.Fn threadpool_cancel_job 257to cancel it. 258However, 259.Fn threadpool_cancel_job_async 260is not enough. 261.\" 262.It Fn threadpool_job_done "job" 263Notify that 264.Fa job 265is done, so that subsequent calls to 266.Fn threadpool_schedule_job 267will cause it to re-run its action. 268.Pp 269.Fn threadpool_job_done 270must be called exactly once by a job's action, and may not be called in 271any other context. 272.\"""""""""""""""""""""""""""""""""""" 273.It Fn threadpool_schedule_job "pool" "job" 274Schedule 275.Fa job 276to run in a thread in 277.Fa pool 278as soon as possible, creating a new thread if necessary. 279.Pp 280Caller must hold the interlock of 281.Fa job . 282.Pp 283.Fn threadpool_schedule_job 284may be called in any context, including hard interrupt context, except 285at interrupt priority levels above 286.Vt IPL_VM . 287.\" 288.It Fn threadpool_cancel_job "pool" "job" 289Cancel 290.Fa job 291if it has been scheduled but has not yet been assigned a thread, or 292wait for it to complete if it has. 293.Pp 294Caller must hold the interlock of 295.Fa job , 296which may be released in order to wait for completion. 297.Pp 298If 299.Fa job 300has not been scheduled, 301.Fn threadpool_cancel_job 302returns immediately. 303If 304.Fa job 305has been scheduled, it must have been scheduled in 306.Fa pool , 307not in any other thread pool. 308.Pp 309May sleep. 310.\" 311.It Fn threadpool_cancel_job_async "pool" "job" 312Try to cancel 313.Fa job 314like 315.Fn threadpool_cancel_job , 316but if it is already running, return 317.Vt false 318instead of waiting; 319otherwise, if it was not scheduled, or if it was scheduled and has not 320yet begun to run, return 321.Vt true . 322.Pp 323Caller must hold the interlock of 324.Fa job . 325.Pp 326.Fn threadpool_cancel_job_async 327may be called in any context, including hard interrupt context, except 328at interrupt priority levels above 329.Vt IPL_VM . 330.\"""""""""""""""""""""""""""""""""""" 331.El 332.\""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" 333.Sh CODE REFERENCES 334The 335.Nm 336abstraction is implemented in 337.Pa sys/kern/kern_threadpool.c . 338.\""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" 339.Sh SEE ALSO 340.Xr kthread 9 , 341.\" .Xr softint 9 , 342.\" .Xr task 9 , 343.Xr workqueue 9 344