1.\" $NetBSD: threadpool.9,v 1.3 2018/12/26 21:48:55 thorpej Exp $ 2.\" 3.\" Copyright (c) 2014 The NetBSD Foundation, Inc. 4.\" All rights reserved. 5.\" 6.\" This code is derived from software contributed to The NetBSD Foundation 7.\" by Taylor R. Campbell. 8.\" 9.\" Redistribution and use in source and binary forms, with or without 10.\" modification, are permitted provided that the following conditions 11.\" are met: 12.\" 1. Redistributions of source code must retain the above copyright 13.\" notice, this list of conditions and the following disclaimer. 14.\" 2. Redistributions in binary form must reproduce the above copyright 15.\" notice, this list of conditions and the following disclaimer in the 16.\" documentation and/or other materials provided with the distribution. 17.\" 18.\" THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 19.\" ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 20.\" TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 21.\" PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 22.\" BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 23.\" CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 24.\" SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 25.\" INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 26.\" CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 27.\" ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 28.\" POSSIBILITY OF SUCH DAMAGE. 29.\" 30.Dd December 26, 2018 31.Dt THREADPOOL 9 32.Os 33.\" 34.Sh NAME 35.Nm threadpool 36.Nd shared pools of kthreads 37.\""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" 38.Sh SYNOPSIS 39.In sys/threadpool.h 40.\"""""""""""""""""""""""""""""""""""" 41.Vt typedef void threadpool_job_fn_t(struct threadpool_job *); 42.\"""""""""""""""""""""""""""""""""""" 43.Ft int 44.Fn threadpool_get "struct threadpool **poolp" "pri_t pri" 45.\" 46.Ft void 47.Fn threadpool_put "struct threadpool *pool" "pri_t pri" 48.\"""""""""""""""""""""""""""""""""""" 49.Ft int 50.Fn threadpool_percpu_get "struct threadpool_percpu **pool_percpup" "pri_t pri" 51.\" 52.Ft void 53.Fn threadpool_percpu_put "struct threadpool_percpu *pool_percpu" "pri_t pri" 54.\" 55.Ft struct threadpool * 56.Fn threadpool_percpu_ref "struct threadpool_percpu *pool" 57.\" 58.Ft struct threadpool * 59.Fn threadpool_percpu_ref_remote "struct threadpool_percpu *pool" "struct cpu_info *ci" 60.\"""""""""""""""""""""""""""""""""""" 61.Ft void 62.Fn threadpool_job_init "struct threadpool_job *job" "threadpool_job_fn_t fn" "kmutex_t *interlock" 63.\" 64.Ft void 65.Fn threadpool_job_destroy "struct threadpool_job *job" 66.\" 67.Ft void 68.Fn threadpool_job_done "struct threadpool_job *job" 69.\"""""""""""""""""""""""""""""""""""" 70.Ft void 71.Fn threadpool_schedule_job "struct threadpool *pool" "struct threadpool_job *job" 72.\" 73.Ft void 74.Fn threadpool_cancel_job "struct threadpool *pool" "struct threadpool_job *job" 75.\" 76.Ft bool 77.Fn threadpool_cancel_job_async "struct threadpool *pool" "struct threadpool_job *job" 78.\""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" 79.Sh DESCRIPTION 80The 81.Nm 82abstraction is provided to share a pool of 83.Xr kthread 9 84kernel threads for medium- to long-term actions, called jobs, which can 85be scheduled from contexts that do not allow sleeping. 86.Pp 87For each priority level, there is one unbound thread pool, and one 88collection of per-CPU thread pools. 89Access to the unbound thread pools is provided by 90.Fn threadpool_get 91and 92.Fn threadpool_put . 93Access to the per-CPU thread pools is provided by 94.Fn threadpool_percpu_get 95and 96.Fn threadpool_percpu_put . 97.Pp 98Job state is stored in the 99.Vt threadpool_job 100structure. 101Callers of the 102.Nm 103abstraction 104must allocate memory for 105.Vt threadpool_job 106structures, but should consider them opaque, and should not inspect or 107copy them. 108Each job represented by a 109.Vt threadpool_job 110structure will be run only once at a time, until the action associated 111with it calls 112.Fn threadpool_job_done . 113.Pp 114Jobs are run in thread context and may take arbitrarily long to run or 115sleep arbitrarily long. 116.\" The 117.\" .Nm 118.\" abstraction is intended as a building block for cheaper abstractions, 119.\" namely 120.\" .Xr task 9 121.\" and 122.\" .Xr workqueue 9 . 123.\" It should generally not be used directly. 124.\""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" 125.Sh FUNCTIONS 126.\" 127.Bl -tag -width abcd 128.\"""""""""""""""""""""""""""""""""""" 129.It Fn threadpool_get "poolp" "pri" 130Obtain a reference to the unbound thread pool at priority 131.Fa pri 132and store it in 133.Fa poolp . 134.Pp 135May sleep. 136.\" 137.It Fn threadpool_put "pool" "pri" 138Release the reference to the unbound thread pool 139.Fa pool 140at priority 141.Fa pri , 142which must be the same as the priority that was passed to 143.Fn threadpool_get 144to obtain 145.Fa pool . 146.Pp 147May sleep. 148.Pp 149Do not use 150.Fn threadpool_put 151with thread pools obtained from 152.Fn threadpool_percpu_ref 153or 154.Fn threadpool_percpu_ref_remote . 155.\"""""""""""""""""""""""""""""""""""" 156.It Fn threadpool_percpu_get "pool_percpup" "pri" 157Obtain a reference to the per-CPU thread pool at priority 158.Fa pri 159and store it in 160.Fa pool_percpup . 161.Pp 162Use 163.Fn threadpool_percpu_ref 164or 165.Fn threadpool_percpu_ref_remote 166with it to get at the thread pool for a particular CPU. 167.Pp 168May sleep. 169.\" 170.It Fn threadpool_percpu_put "pool_percpu" "pri" 171Release a reference to the per-CPU thread pool 172.Fa pool_percpu 173at priority 174.Fa pri . 175.Pp 176May sleep. 177.\" 178.It Fn threadpool_percpu_ref "pool_percpu" 179Return the thread pool in 180.Fa pool_percpu 181for the current CPU. 182.Pp 183The resulting thread pool pointer is stable until 184.Fa pool_percpu 185is released with 186.Fn threadpool_percpu_put . 187Using it to schedule or cancel a job does not require being on the same 188CPU. 189.Pp 190Do not use 191.Fn threadpool_put 192with thread pools obtained from 193.Fn threadpool_percpu_ref . 194.\" 195.It Fn threadpool_percpu_ref_remote "pool_percpu" "ci" 196Return the thread pool in 197.Fa pool_percpu 198for the CPU whose 199.Vt struct cpu_info 200is given by 201.Fa ci . 202.Pp 203The resulting thread pool pointer is stable until 204.Fa pool_percpu 205is released with 206.Fn threadpool_percpu_put . 207Using it to schedule or cancel a job does not require being on the same 208CPU, but it is faster and friendlier to the cache to use 209.Fn threadpool_percpu_ref 210and use the resulting thread pool only on the same CPU. 211.Pp 212Do not use 213.Fn threadpool_put 214with thread pools obtained from 215.Fn threadpool_percpu_ref_remote . 216.\"""""""""""""""""""""""""""""""""""" 217.It Fn threadpool_job_init "job" "fn" "interlock" 218Initialize the threadpool job 219.Fa job 220to run 221.Fa fn 222when scheduled and to interlock with 223.Fa interlock . 224.Pp 225The mutex 226.Fa interlock 227is used to synchronize job scheduling and completion. 228The action 229.Fa fn 230is required to eventually call 231.Fn threadpool_job_done , 232with 233.Fa interlock 234held. 235This is so that while the job is running and may be waiting for work 236to do, scheduling the job has no effect, but as soon as the job is 237done, scheduling the job will cause it to run again. 238.Pp 239To change the action of a job, you must use 240.Fn threadpool_job_destroy 241first and then call 242.Fn threadpool_job_init 243again. 244.\" 245.It Fn threadpool_job_destroy "job" 246Destroy the threadpool job 247.Fa job . 248.Fa job 249must not currently be scheduled to run. 250If it may still be scheduled, you can use 251.Fn threadpool_cancel_job 252to cancel it. 253However, 254.Fn threadpool_cancel_job_async 255is not enough. 256.\" 257.It Fn threadpool_job_done "job" 258Notify that 259.Fa job 260is done, so that subsequent calls to 261.Fn threadpool_schedule_job 262will cause it to re-run its action. 263.Pp 264.Fn threadpool_job_done 265must be called exactly once by a job's action, and may not be called in 266any other context. 267.\"""""""""""""""""""""""""""""""""""" 268.It Fn threadpool_schedule_job "pool" "job" 269Schedule 270.Fa job 271to run in a thread in 272.Fa pool 273as soon as possible, creating a new thread if necessary. 274.Pp 275Caller must hold the interlock of 276.Fa job . 277.Pp 278.Fn threadpool_schedule_job 279may be called in any context, including hard interrupt context, except 280at interrupt priority levels above 281.Vt IPL_VM . 282.\" 283.It Fn threadpool_cancel_job "pool" "job" 284Cancel 285.Fa job 286if it has been scheduled but has not yet been assigned a thread, or 287wait for it to complete if it has. 288.Pp 289Caller must hold the interlock of 290.Fa job , 291which may be released in order to wait for completion. 292.Pp 293If 294.Fa job 295has not been scheduled, 296.Fn threadpool_cancel_job 297returns immediately. 298If 299.Fa job 300has been scheduled, it must have been scheduled in 301.Fa pool , 302not in any other thread pool. 303.Pp 304May sleep. 305.\" 306.It Fn threadpool_cancel_job_async "pool" "job" 307Try to cancel 308.Fa job 309like 310.Fn threadpool_cancel_job , 311but if it is already running, return 312.Vt false 313instead of waiting; 314otherwise, if it was not scheduled, or if it was scheduled and has not 315yet begun to run, return 316.Vt true . 317.Pp 318Caller must hold the interlock of 319.Fa job . 320.Pp 321.Fn threadpool_cancel_job_async 322may be called in any context, including hard interrupt context, except 323at interrupt priority levels above 324.Vt IPL_VM . 325.\"""""""""""""""""""""""""""""""""""" 326.El 327.\""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" 328.Sh CODE REFERENCES 329The 330.Nm 331abstraction is implemented in 332.Pa sys/kern/kern_threadpool.c . 333.\""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" 334.Sh SEE ALSO 335.Xr kthread 9 , 336.\" .Xr softint 9 , 337.\" .Xr task 9 , 338.Xr workqueue 9 339