1*0Sstevel@tonic-gate /*
2*0Sstevel@tonic-gate * CDDL HEADER START
3*0Sstevel@tonic-gate *
4*0Sstevel@tonic-gate * The contents of this file are subject to the terms of the
5*0Sstevel@tonic-gate * Common Development and Distribution License, Version 1.0 only
6*0Sstevel@tonic-gate * (the "License"). You may not use this file except in compliance
7*0Sstevel@tonic-gate * with the License.
8*0Sstevel@tonic-gate *
9*0Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10*0Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing.
11*0Sstevel@tonic-gate * See the License for the specific language governing permissions
12*0Sstevel@tonic-gate * and limitations under the License.
13*0Sstevel@tonic-gate *
14*0Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each
15*0Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16*0Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the
17*0Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying
18*0Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner]
19*0Sstevel@tonic-gate *
20*0Sstevel@tonic-gate * CDDL HEADER END
21*0Sstevel@tonic-gate */
22*0Sstevel@tonic-gate /*
23*0Sstevel@tonic-gate * Copyright 2003 Sun Microsystems, Inc. All rights reserved.
24*0Sstevel@tonic-gate * Use is subject to license terms.
25*0Sstevel@tonic-gate */
26*0Sstevel@tonic-gate
27*0Sstevel@tonic-gate #pragma ident "%Z%%M% %I% %E% SMI"
28*0Sstevel@tonic-gate
29*0Sstevel@tonic-gate /*
30*0Sstevel@tonic-gate * crypto_bufcall(9F) group of routines.
31*0Sstevel@tonic-gate */
32*0Sstevel@tonic-gate
33*0Sstevel@tonic-gate #include <sys/types.h>
34*0Sstevel@tonic-gate #include <sys/sunddi.h>
35*0Sstevel@tonic-gate #include <sys/callb.h>
36*0Sstevel@tonic-gate #include <sys/ksynch.h>
37*0Sstevel@tonic-gate #include <sys/systm.h>
38*0Sstevel@tonic-gate #include <sys/taskq_impl.h>
39*0Sstevel@tonic-gate #include <sys/crypto/api.h>
40*0Sstevel@tonic-gate #include <sys/crypto/sched_impl.h>
41*0Sstevel@tonic-gate
42*0Sstevel@tonic-gate /*
43*0Sstevel@tonic-gate * All pending crypto bufcalls are put on a list. cbuf_list_lock
44*0Sstevel@tonic-gate * protects changes to this list.
45*0Sstevel@tonic-gate *
46*0Sstevel@tonic-gate * The following locking order is maintained in the code - The
47*0Sstevel@tonic-gate * global cbuf_list_lock followed by the individual lock
48*0Sstevel@tonic-gate * in a crypto bufcall structure (kc_lock).
49*0Sstevel@tonic-gate */
50*0Sstevel@tonic-gate kmutex_t cbuf_list_lock;
51*0Sstevel@tonic-gate kcondvar_t cbuf_list_cv; /* cv the service thread waits on */
52*0Sstevel@tonic-gate static kcf_cbuf_elem_t *cbuf_list_head;
53*0Sstevel@tonic-gate static kcf_cbuf_elem_t *cbuf_list_tail;
54*0Sstevel@tonic-gate
55*0Sstevel@tonic-gate /*
56*0Sstevel@tonic-gate * Allocate and return a handle to be used for crypto_bufcall().
57*0Sstevel@tonic-gate * Can be called from user context only.
58*0Sstevel@tonic-gate */
59*0Sstevel@tonic-gate crypto_bc_t
crypto_bufcall_alloc(void)60*0Sstevel@tonic-gate crypto_bufcall_alloc(void)
61*0Sstevel@tonic-gate {
62*0Sstevel@tonic-gate kcf_cbuf_elem_t *cbufp;
63*0Sstevel@tonic-gate
64*0Sstevel@tonic-gate cbufp = kmem_zalloc(sizeof (kcf_cbuf_elem_t), KM_SLEEP);
65*0Sstevel@tonic-gate mutex_init(&cbufp->kc_lock, NULL, MUTEX_DEFAULT, NULL);
66*0Sstevel@tonic-gate cv_init(&cbufp->kc_cv, NULL, CV_DEFAULT, NULL);
67*0Sstevel@tonic-gate cbufp->kc_state = CBUF_FREE;
68*0Sstevel@tonic-gate
69*0Sstevel@tonic-gate return (cbufp);
70*0Sstevel@tonic-gate }
71*0Sstevel@tonic-gate
72*0Sstevel@tonic-gate /*
73*0Sstevel@tonic-gate * Free the handle if possible. Returns CRYPTO_SUCCESS if the handle
74*0Sstevel@tonic-gate * is freed. Else it returns CRYPTO_BUSY.
75*0Sstevel@tonic-gate *
76*0Sstevel@tonic-gate * The client should do a crypto_unbufcall() if it receives a
77*0Sstevel@tonic-gate * CRYPTO_BUSY.
78*0Sstevel@tonic-gate *
79*0Sstevel@tonic-gate * Can be called both from user and interrupt context.
80*0Sstevel@tonic-gate */
81*0Sstevel@tonic-gate int
crypto_bufcall_free(crypto_bc_t bc)82*0Sstevel@tonic-gate crypto_bufcall_free(crypto_bc_t bc)
83*0Sstevel@tonic-gate {
84*0Sstevel@tonic-gate kcf_cbuf_elem_t *cbufp = (kcf_cbuf_elem_t *)bc;
85*0Sstevel@tonic-gate
86*0Sstevel@tonic-gate mutex_enter(&cbufp->kc_lock);
87*0Sstevel@tonic-gate if (cbufp->kc_state != CBUF_FREE) {
88*0Sstevel@tonic-gate mutex_exit(&cbufp->kc_lock);
89*0Sstevel@tonic-gate return (CRYPTO_BUSY);
90*0Sstevel@tonic-gate }
91*0Sstevel@tonic-gate mutex_exit(&cbufp->kc_lock);
92*0Sstevel@tonic-gate
93*0Sstevel@tonic-gate mutex_destroy(&cbufp->kc_lock);
94*0Sstevel@tonic-gate cv_destroy(&cbufp->kc_cv);
95*0Sstevel@tonic-gate kmem_free(cbufp, sizeof (kcf_cbuf_elem_t));
96*0Sstevel@tonic-gate
97*0Sstevel@tonic-gate return (CRYPTO_SUCCESS);
98*0Sstevel@tonic-gate }
99*0Sstevel@tonic-gate
100*0Sstevel@tonic-gate /*
101*0Sstevel@tonic-gate * Schedule func() to be called when queue space is available to
102*0Sstevel@tonic-gate * submit a crypto request.
103*0Sstevel@tonic-gate *
104*0Sstevel@tonic-gate * Can be called both from user and interrupt context.
105*0Sstevel@tonic-gate */
106*0Sstevel@tonic-gate int
crypto_bufcall(crypto_bc_t bc,void (* func)(void * arg),void * arg)107*0Sstevel@tonic-gate crypto_bufcall(crypto_bc_t bc, void (*func)(void *arg), void *arg)
108*0Sstevel@tonic-gate {
109*0Sstevel@tonic-gate kcf_cbuf_elem_t *cbufp;
110*0Sstevel@tonic-gate
111*0Sstevel@tonic-gate cbufp = (kcf_cbuf_elem_t *)bc;
112*0Sstevel@tonic-gate if (cbufp == NULL || func == NULL) {
113*0Sstevel@tonic-gate return (CRYPTO_ARGUMENTS_BAD);
114*0Sstevel@tonic-gate }
115*0Sstevel@tonic-gate
116*0Sstevel@tonic-gate mutex_enter(&cbuf_list_lock);
117*0Sstevel@tonic-gate mutex_enter(&cbufp->kc_lock);
118*0Sstevel@tonic-gate if (cbufp->kc_state != CBUF_FREE) {
119*0Sstevel@tonic-gate mutex_exit(&cbufp->kc_lock);
120*0Sstevel@tonic-gate mutex_exit(&cbuf_list_lock);
121*0Sstevel@tonic-gate return (CRYPTO_BUSY);
122*0Sstevel@tonic-gate }
123*0Sstevel@tonic-gate
124*0Sstevel@tonic-gate cbufp->kc_state = CBUF_WAITING;
125*0Sstevel@tonic-gate cbufp->kc_func = func;
126*0Sstevel@tonic-gate cbufp->kc_arg = arg;
127*0Sstevel@tonic-gate cbufp->kc_prev = cbufp->kc_next = NULL;
128*0Sstevel@tonic-gate
129*0Sstevel@tonic-gate if (cbuf_list_head == NULL) {
130*0Sstevel@tonic-gate cbuf_list_head = cbuf_list_tail = cbufp;
131*0Sstevel@tonic-gate } else {
132*0Sstevel@tonic-gate cbuf_list_tail->kc_next = cbufp;
133*0Sstevel@tonic-gate cbufp->kc_prev = cbuf_list_tail;
134*0Sstevel@tonic-gate cbuf_list_tail = cbufp;
135*0Sstevel@tonic-gate }
136*0Sstevel@tonic-gate
137*0Sstevel@tonic-gate /*
138*0Sstevel@tonic-gate * Signal the crypto_bufcall_service thread to start
139*0Sstevel@tonic-gate * working on this crypto bufcall request.
140*0Sstevel@tonic-gate */
141*0Sstevel@tonic-gate cv_signal(&cbuf_list_cv);
142*0Sstevel@tonic-gate mutex_exit(&cbufp->kc_lock);
143*0Sstevel@tonic-gate mutex_exit(&cbuf_list_lock);
144*0Sstevel@tonic-gate
145*0Sstevel@tonic-gate return (CRYPTO_SUCCESS);
146*0Sstevel@tonic-gate }
147*0Sstevel@tonic-gate
148*0Sstevel@tonic-gate /*
149*0Sstevel@tonic-gate * Cancel a pending crypto bufcall request. If the bufcall
150*0Sstevel@tonic-gate * is currently executing, we wait till it is complete.
151*0Sstevel@tonic-gate *
152*0Sstevel@tonic-gate * Can only be called from user context.
153*0Sstevel@tonic-gate */
154*0Sstevel@tonic-gate int
crypto_unbufcall(crypto_bc_t bc)155*0Sstevel@tonic-gate crypto_unbufcall(crypto_bc_t bc)
156*0Sstevel@tonic-gate {
157*0Sstevel@tonic-gate kcf_cbuf_elem_t *cbufp = (kcf_cbuf_elem_t *)bc;
158*0Sstevel@tonic-gate
159*0Sstevel@tonic-gate mutex_enter(&cbuf_list_lock);
160*0Sstevel@tonic-gate mutex_enter(&cbufp->kc_lock);
161*0Sstevel@tonic-gate
162*0Sstevel@tonic-gate if (cbufp->kc_state == CBUF_WAITING) {
163*0Sstevel@tonic-gate kcf_cbuf_elem_t *nextp = cbufp->kc_next;
164*0Sstevel@tonic-gate kcf_cbuf_elem_t *prevp = cbufp->kc_prev;
165*0Sstevel@tonic-gate
166*0Sstevel@tonic-gate if (nextp != NULL)
167*0Sstevel@tonic-gate nextp->kc_prev = prevp;
168*0Sstevel@tonic-gate else
169*0Sstevel@tonic-gate cbuf_list_tail = prevp;
170*0Sstevel@tonic-gate
171*0Sstevel@tonic-gate if (prevp != NULL)
172*0Sstevel@tonic-gate prevp->kc_next = nextp;
173*0Sstevel@tonic-gate else
174*0Sstevel@tonic-gate cbuf_list_head = nextp;
175*0Sstevel@tonic-gate cbufp->kc_state = CBUF_FREE;
176*0Sstevel@tonic-gate } else if (cbufp->kc_state == CBUF_RUNNING) {
177*0Sstevel@tonic-gate mutex_exit(&cbuf_list_lock);
178*0Sstevel@tonic-gate /*
179*0Sstevel@tonic-gate * crypto_bufcall_service thread is working
180*0Sstevel@tonic-gate * on this element. We will wait for that
181*0Sstevel@tonic-gate * thread to signal us when done.
182*0Sstevel@tonic-gate */
183*0Sstevel@tonic-gate while (cbufp->kc_state == CBUF_RUNNING)
184*0Sstevel@tonic-gate cv_wait(&cbufp->kc_cv, &cbufp->kc_lock);
185*0Sstevel@tonic-gate mutex_exit(&cbufp->kc_lock);
186*0Sstevel@tonic-gate
187*0Sstevel@tonic-gate return (CRYPTO_SUCCESS);
188*0Sstevel@tonic-gate }
189*0Sstevel@tonic-gate
190*0Sstevel@tonic-gate mutex_exit(&cbufp->kc_lock);
191*0Sstevel@tonic-gate mutex_exit(&cbuf_list_lock);
192*0Sstevel@tonic-gate
193*0Sstevel@tonic-gate return (CRYPTO_SUCCESS);
194*0Sstevel@tonic-gate }
195*0Sstevel@tonic-gate
196*0Sstevel@tonic-gate /*
197*0Sstevel@tonic-gate * We sample the number of jobs. We do not hold the lock
198*0Sstevel@tonic-gate * as it is not necessary to get the exact count.
199*0Sstevel@tonic-gate */
200*0Sstevel@tonic-gate #define KCF_GSWQ_AVAIL (gswq->gs_maxjobs - gswq->gs_njobs)
201*0Sstevel@tonic-gate
202*0Sstevel@tonic-gate /*
203*0Sstevel@tonic-gate * One queue space each for init, update, and final.
204*0Sstevel@tonic-gate */
205*0Sstevel@tonic-gate #define GSWQ_MINFREE 3
206*0Sstevel@tonic-gate
207*0Sstevel@tonic-gate /*
208*0Sstevel@tonic-gate * Go through the list of crypto bufcalls and do the necessary
209*0Sstevel@tonic-gate * callbacks.
210*0Sstevel@tonic-gate */
211*0Sstevel@tonic-gate static void
kcf_run_cbufcalls(void)212*0Sstevel@tonic-gate kcf_run_cbufcalls(void)
213*0Sstevel@tonic-gate {
214*0Sstevel@tonic-gate kcf_cbuf_elem_t *cbufp;
215*0Sstevel@tonic-gate int count;
216*0Sstevel@tonic-gate
217*0Sstevel@tonic-gate mutex_enter(&cbuf_list_lock);
218*0Sstevel@tonic-gate
219*0Sstevel@tonic-gate /*
220*0Sstevel@tonic-gate * Get estimate of available queue space from KCF_GSWQ_AVAIL.
221*0Sstevel@tonic-gate * We can call 'n' crypto bufcall callback functions where
222*0Sstevel@tonic-gate * n * GSWQ_MINFREE <= available queue space.
223*0Sstevel@tonic-gate *
224*0Sstevel@tonic-gate * TO DO - Extend the check to taskqs of hardware providers.
225*0Sstevel@tonic-gate * For now, we handle only the software providers.
226*0Sstevel@tonic-gate */
227*0Sstevel@tonic-gate count = KCF_GSWQ_AVAIL;
228*0Sstevel@tonic-gate while ((cbufp = cbuf_list_head) != NULL) {
229*0Sstevel@tonic-gate if (GSWQ_MINFREE <= count) {
230*0Sstevel@tonic-gate count -= GSWQ_MINFREE;
231*0Sstevel@tonic-gate mutex_enter(&cbufp->kc_lock);
232*0Sstevel@tonic-gate cbuf_list_head = cbufp->kc_next;
233*0Sstevel@tonic-gate cbufp->kc_state = CBUF_RUNNING;
234*0Sstevel@tonic-gate mutex_exit(&cbufp->kc_lock);
235*0Sstevel@tonic-gate mutex_exit(&cbuf_list_lock);
236*0Sstevel@tonic-gate
237*0Sstevel@tonic-gate (*cbufp->kc_func)(cbufp->kc_arg);
238*0Sstevel@tonic-gate
239*0Sstevel@tonic-gate mutex_enter(&cbufp->kc_lock);
240*0Sstevel@tonic-gate cbufp->kc_state = CBUF_FREE;
241*0Sstevel@tonic-gate cv_broadcast(&cbufp->kc_cv);
242*0Sstevel@tonic-gate mutex_exit(&cbufp->kc_lock);
243*0Sstevel@tonic-gate
244*0Sstevel@tonic-gate mutex_enter(&cbuf_list_lock);
245*0Sstevel@tonic-gate } else {
246*0Sstevel@tonic-gate /*
247*0Sstevel@tonic-gate * There is not enough queue space in this
248*0Sstevel@tonic-gate * round. We bail out and try again
249*0Sstevel@tonic-gate * later.
250*0Sstevel@tonic-gate */
251*0Sstevel@tonic-gate break;
252*0Sstevel@tonic-gate }
253*0Sstevel@tonic-gate }
254*0Sstevel@tonic-gate if (cbuf_list_head == NULL)
255*0Sstevel@tonic-gate cbuf_list_tail = NULL;
256*0Sstevel@tonic-gate
257*0Sstevel@tonic-gate mutex_exit(&cbuf_list_lock);
258*0Sstevel@tonic-gate }
259*0Sstevel@tonic-gate
260*0Sstevel@tonic-gate /*
261*0Sstevel@tonic-gate * Background processing of crypto bufcalls.
262*0Sstevel@tonic-gate */
263*0Sstevel@tonic-gate void
crypto_bufcall_service(void)264*0Sstevel@tonic-gate crypto_bufcall_service(void)
265*0Sstevel@tonic-gate {
266*0Sstevel@tonic-gate callb_cpr_t cprinfo;
267*0Sstevel@tonic-gate
268*0Sstevel@tonic-gate CALLB_CPR_INIT(&cprinfo, &cbuf_list_lock, callb_generic_cpr,
269*0Sstevel@tonic-gate "crypto_bufcall_service");
270*0Sstevel@tonic-gate
271*0Sstevel@tonic-gate mutex_enter(&cbuf_list_lock);
272*0Sstevel@tonic-gate
273*0Sstevel@tonic-gate for (;;) {
274*0Sstevel@tonic-gate if (cbuf_list_head != NULL && KCF_GSWQ_AVAIL >= GSWQ_MINFREE) {
275*0Sstevel@tonic-gate mutex_exit(&cbuf_list_lock);
276*0Sstevel@tonic-gate kcf_run_cbufcalls();
277*0Sstevel@tonic-gate mutex_enter(&cbuf_list_lock);
278*0Sstevel@tonic-gate }
279*0Sstevel@tonic-gate
280*0Sstevel@tonic-gate if (cbuf_list_head != NULL) {
281*0Sstevel@tonic-gate /*
282*0Sstevel@tonic-gate * Wait 30 seconds for queue space to become available.
283*0Sstevel@tonic-gate * This number is reasonable as it does not cause
284*0Sstevel@tonic-gate * much CPU overhead. We could wait on a condition
285*0Sstevel@tonic-gate * variable and the global software dequeue routine can
286*0Sstevel@tonic-gate * signal us. But, it adds overhead to that routine
287*0Sstevel@tonic-gate * which we want to avoid. Also, the client is prepared
288*0Sstevel@tonic-gate * to wait any way.
289*0Sstevel@tonic-gate */
290*0Sstevel@tonic-gate CALLB_CPR_SAFE_BEGIN(&cprinfo);
291*0Sstevel@tonic-gate mutex_exit(&cbuf_list_lock);
292*0Sstevel@tonic-gate delay(30 * drv_usectohz(1000000));
293*0Sstevel@tonic-gate mutex_enter(&cbuf_list_lock);
294*0Sstevel@tonic-gate CALLB_CPR_SAFE_END(&cprinfo, &cbuf_list_lock);
295*0Sstevel@tonic-gate }
296*0Sstevel@tonic-gate
297*0Sstevel@tonic-gate /* Wait for new work to arrive */
298*0Sstevel@tonic-gate if (cbuf_list_head == NULL) {
299*0Sstevel@tonic-gate CALLB_CPR_SAFE_BEGIN(&cprinfo);
300*0Sstevel@tonic-gate cv_wait(&cbuf_list_cv, &cbuf_list_lock);
301*0Sstevel@tonic-gate CALLB_CPR_SAFE_END(&cprinfo, &cbuf_list_lock);
302*0Sstevel@tonic-gate }
303*0Sstevel@tonic-gate }
304*0Sstevel@tonic-gate }
305