xref: /freebsd-src/sys/dev/qat/qat/qat_ocf.c (revision d1bdc2821fcd416ab9b238580386eb605a6128d0)
1 /* SPDX-License-Identifier: BSD-3-Clause */
2 /* Copyright(c) 2007-2022 Intel Corporation */
3 /* System headers */
4 #include <sys/param.h>
5 #include <sys/systm.h>
6 #include <sys/bus.h>
7 #include <sys/cpu.h>
8 #include <sys/kernel.h>
9 #include <sys/mbuf.h>
10 #include <sys/module.h>
11 #include <sys/mutex.h>
12 #include <sys/sysctl.h>
13 
14 /* Cryptodev headers */
15 #include <opencrypto/cryptodev.h>
16 #include "cryptodev_if.h"
17 
18 /* QAT specific headers */
19 #include "cpa.h"
20 #include "cpa_cy_im.h"
21 #include "cpa_cy_sym_dp.h"
22 #include "adf_accel_devices.h"
23 #include "adf_common_drv.h"
24 #include "lac_sym_hash_defs.h"
25 #include "lac_sym_qat_hash_defs_lookup.h"
26 
27 /* To get only IRQ instances */
28 #include "icp_accel_devices.h"
29 #include "icp_adf_accel_mgr.h"
30 #include "lac_sal_types.h"
31 
32 /* To disable AEAD HW MAC verification */
33 #include "icp_sal_user.h"
34 
35 /* QAT OCF specific headers */
36 #include "qat_ocf_mem_pool.h"
37 #include "qat_ocf_utils.h"
38 
39 #define QAT_OCF_MAX_INSTANCES (256)
40 #define QAT_OCF_SESSION_WAIT_TIMEOUT_MS (1000)
41 
42 MALLOC_DEFINE(M_QAT_OCF, "qat_ocf", "qat_ocf(4) memory allocations");
43 
44 /* QAT OCF internal structures */
45 struct qat_ocf_softc {
46 	device_t sc_dev;
47 	struct sysctl_oid *rc;
48 	uint32_t enabled;
49 	int32_t cryptodev_id;
50 	struct qat_ocf_instance cyInstHandles[QAT_OCF_MAX_INSTANCES];
51 	int32_t numCyInstances;
52 };
53 
54 /* Function definitions */
55 static void qat_ocf_freesession(device_t dev, crypto_session_t cses);
56 static int qat_ocf_probesession(device_t dev,
57 				const struct crypto_session_params *csp);
58 static int qat_ocf_newsession(device_t dev,
59 			      crypto_session_t cses,
60 			      const struct crypto_session_params *csp);
61 static int qat_ocf_attach(device_t dev);
62 static int qat_ocf_detach(device_t dev);
63 
64 static void
65 symDpCallback(CpaCySymDpOpData *pOpData,
66 	      CpaStatus result,
67 	      CpaBoolean verifyResult)
68 {
69 	struct qat_ocf_cookie *qat_cookie;
70 	struct cryptop *crp;
71 	struct qat_ocf_dsession *qat_dsession = NULL;
72 	struct qat_ocf_session *qat_session = NULL;
73 	struct qat_ocf_instance *qat_instance = NULL;
74 	CpaStatus status;
75 	int rc = 0;
76 
77 	qat_cookie = (struct qat_ocf_cookie *)pOpData->pCallbackTag;
78 	if (!qat_cookie)
79 		return;
80 
81 	crp = qat_cookie->crp_op;
82 
83 	qat_dsession = crypto_get_driver_session(crp->crp_session);
84 	qat_instance = qat_dsession->qatInstance;
85 
86 	status = qat_ocf_cookie_dma_post_sync(crp, pOpData);
87 	if (CPA_STATUS_SUCCESS != status) {
88 		rc = EIO;
89 		goto exit;
90 	}
91 
92 	status = qat_ocf_cookie_dma_unload(crp, pOpData);
93 	if (CPA_STATUS_SUCCESS != status) {
94 		rc = EIO;
95 		goto exit;
96 	}
97 
98 	/* Verify result */
99 	if (CPA_STATUS_SUCCESS != result) {
100 		rc = EBADMSG;
101 		goto exit;
102 	}
103 
104 	/* Verify digest by FW (GCM and CCM only) */
105 	if (CPA_TRUE != verifyResult) {
106 		rc = EBADMSG;
107 		goto exit;
108 	}
109 
110 	if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
111 		qat_session = &qat_dsession->encSession;
112 	else
113 		qat_session = &qat_dsession->decSession;
114 
115 	/* Copy back digest result if it's stored in separated buffer */
116 	if (pOpData->digestResult && qat_session->authLen > 0) {
117 		if ((crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) != 0) {
118 			char icv[QAT_OCF_MAX_DIGEST] = { 0 };
119 			crypto_copydata(crp,
120 					crp->crp_digest_start,
121 					qat_session->authLen,
122 					icv);
123 			if (timingsafe_bcmp(icv,
124 					    qat_cookie->qat_ocf_digest,
125 					    qat_session->authLen) != 0) {
126 				rc = EBADMSG;
127 				goto exit;
128 			}
129 		} else {
130 			crypto_copyback(crp,
131 					crp->crp_digest_start,
132 					qat_session->authLen,
133 					qat_cookie->qat_ocf_digest);
134 		}
135 	}
136 
137 exit:
138 	qat_ocf_cookie_free(qat_instance, qat_cookie);
139 	crp->crp_etype = rc;
140 	crypto_done(crp);
141 
142 	return;
143 }
144 
145 static inline CpaPhysicalAddr
146 qatVirtToPhys(void *virtAddr)
147 {
148 	return (CpaPhysicalAddr)vtophys(virtAddr);
149 }
150 
151 static int
152 qat_ocf_probesession(device_t dev, const struct crypto_session_params *csp)
153 {
154 	if ((csp->csp_flags & ~(CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD)) !=
155 	    0) {
156 		return EINVAL;
157 	}
158 
159 	switch (csp->csp_mode) {
160 	case CSP_MODE_CIPHER:
161 		switch (csp->csp_cipher_alg) {
162 		case CRYPTO_AES_CBC:
163 		case CRYPTO_AES_ICM:
164 			if (csp->csp_ivlen != AES_BLOCK_LEN)
165 				return EINVAL;
166 			break;
167 		case CRYPTO_AES_XTS:
168 			if (csp->csp_ivlen != AES_XTS_IV_LEN)
169 				return EINVAL;
170 			break;
171 		default:
172 			return EINVAL;
173 		}
174 		break;
175 	case CSP_MODE_DIGEST:
176 		switch (csp->csp_auth_alg) {
177 		case CRYPTO_SHA1:
178 		case CRYPTO_SHA1_HMAC:
179 		case CRYPTO_SHA2_256:
180 		case CRYPTO_SHA2_256_HMAC:
181 		case CRYPTO_SHA2_384:
182 		case CRYPTO_SHA2_384_HMAC:
183 		case CRYPTO_SHA2_512:
184 		case CRYPTO_SHA2_512_HMAC:
185 			break;
186 		case CRYPTO_AES_NIST_GMAC:
187 			if (csp->csp_ivlen != AES_GCM_IV_LEN)
188 				return EINVAL;
189 			break;
190 		default:
191 			return EINVAL;
192 		}
193 		break;
194 	case CSP_MODE_AEAD:
195 		switch (csp->csp_cipher_alg) {
196 		case CRYPTO_AES_NIST_GCM_16:
197 			if (csp->csp_ivlen != AES_GCM_IV_LEN)
198 				return EINVAL;
199 			break;
200 		default:
201 			return EINVAL;
202 		}
203 		break;
204 	case CSP_MODE_ETA:
205 		switch (csp->csp_auth_alg) {
206 		case CRYPTO_SHA1_HMAC:
207 		case CRYPTO_SHA2_256_HMAC:
208 		case CRYPTO_SHA2_384_HMAC:
209 		case CRYPTO_SHA2_512_HMAC:
210 			switch (csp->csp_cipher_alg) {
211 			case CRYPTO_AES_CBC:
212 			case CRYPTO_AES_ICM:
213 				if (csp->csp_ivlen != AES_BLOCK_LEN)
214 					return EINVAL;
215 				break;
216 			case CRYPTO_AES_XTS:
217 				if (csp->csp_ivlen != AES_XTS_IV_LEN)
218 					return EINVAL;
219 				break;
220 			default:
221 				return EINVAL;
222 			}
223 			break;
224 		default:
225 			return EINVAL;
226 		}
227 		break;
228 	default:
229 		return EINVAL;
230 	}
231 
232 	return CRYPTODEV_PROBE_HARDWARE;
233 }
234 
235 static CpaStatus
236 qat_ocf_session_init(device_t dev,
237 		     struct cryptop *crp,
238 		     struct qat_ocf_instance *qat_instance,
239 		     struct qat_ocf_session *qat_ssession)
240 {
241 	CpaStatus status = CPA_STATUS_SUCCESS;
242 	/* Crytpodev structures */
243 	crypto_session_t cses;
244 	const struct crypto_session_params *csp;
245 	/* DP API Session configuration */
246 	CpaCySymSessionSetupData sessionSetupData = { 0 };
247 	CpaCySymSessionCtx sessionCtx = NULL;
248 	Cpa32U sessionCtxSize = 0;
249 
250 	cses = crp->crp_session;
251 	if (NULL == cses) {
252 		device_printf(dev, "no crypto session in cryptodev request\n");
253 		return CPA_STATUS_FAIL;
254 	}
255 
256 	csp = crypto_get_params(cses);
257 	if (NULL == csp) {
258 		device_printf(dev, "no session in cryptodev session\n");
259 		return CPA_STATUS_FAIL;
260 	}
261 
262 	/* Common fields */
263 	sessionSetupData.sessionPriority = CPA_CY_PRIORITY_HIGH;
264 	/* Cipher key */
265 	if (crp->crp_cipher_key)
266 		sessionSetupData.cipherSetupData.pCipherKey =
267 		    crp->crp_cipher_key;
268 	else
269 		sessionSetupData.cipherSetupData.pCipherKey =
270 		    csp->csp_cipher_key;
271 	sessionSetupData.cipherSetupData.cipherKeyLenInBytes =
272 	    csp->csp_cipher_klen;
273 
274 	/* Auth key */
275 	if (crp->crp_auth_key)
276 		sessionSetupData.hashSetupData.authModeSetupData.authKey =
277 		    crp->crp_auth_key;
278 	else
279 		sessionSetupData.hashSetupData.authModeSetupData.authKey =
280 		    csp->csp_auth_key;
281 	sessionSetupData.hashSetupData.authModeSetupData.authKeyLenInBytes =
282 	    csp->csp_auth_klen;
283 
284 	qat_ssession->aadLen = crp->crp_aad_length;
285 	if (CPA_TRUE == is_sep_aad_supported(csp))
286 		sessionSetupData.hashSetupData.authModeSetupData.aadLenInBytes =
287 		    crp->crp_aad_length;
288 	else
289 		sessionSetupData.hashSetupData.authModeSetupData.aadLenInBytes =
290 		    0;
291 
292 	/* Just setup algorithm - regardless of mode */
293 	if (csp->csp_cipher_alg) {
294 		sessionSetupData.symOperation = CPA_CY_SYM_OP_CIPHER;
295 
296 		switch (csp->csp_cipher_alg) {
297 		case CRYPTO_AES_CBC:
298 			sessionSetupData.cipherSetupData.cipherAlgorithm =
299 			    CPA_CY_SYM_CIPHER_AES_CBC;
300 			break;
301 		case CRYPTO_AES_ICM:
302 			sessionSetupData.cipherSetupData.cipherAlgorithm =
303 			    CPA_CY_SYM_CIPHER_AES_CTR;
304 			break;
305 		case CRYPTO_AES_XTS:
306 			sessionSetupData.cipherSetupData.cipherAlgorithm =
307 			    CPA_CY_SYM_CIPHER_AES_XTS;
308 			break;
309 		case CRYPTO_AES_NIST_GCM_16:
310 			sessionSetupData.cipherSetupData.cipherAlgorithm =
311 			    CPA_CY_SYM_CIPHER_AES_GCM;
312 			sessionSetupData.hashSetupData.hashAlgorithm =
313 			    CPA_CY_SYM_HASH_AES_GCM;
314 			sessionSetupData.hashSetupData.hashMode =
315 			    CPA_CY_SYM_HASH_MODE_AUTH;
316 			break;
317 		default:
318 			device_printf(dev,
319 				      "cipher_alg: %d not supported\n",
320 				      csp->csp_cipher_alg);
321 			status = CPA_STATUS_UNSUPPORTED;
322 			goto fail;
323 		}
324 	}
325 
326 	if (csp->csp_auth_alg) {
327 		switch (csp->csp_auth_alg) {
328 		case CRYPTO_SHA1_HMAC:
329 			sessionSetupData.hashSetupData.hashAlgorithm =
330 			    CPA_CY_SYM_HASH_SHA1;
331 			sessionSetupData.hashSetupData.hashMode =
332 			    CPA_CY_SYM_HASH_MODE_AUTH;
333 			break;
334 		case CRYPTO_SHA1:
335 			sessionSetupData.hashSetupData.hashAlgorithm =
336 			    CPA_CY_SYM_HASH_SHA1;
337 			sessionSetupData.hashSetupData.hashMode =
338 			    CPA_CY_SYM_HASH_MODE_PLAIN;
339 			break;
340 
341 		case CRYPTO_SHA2_256_HMAC:
342 			sessionSetupData.hashSetupData.hashAlgorithm =
343 			    CPA_CY_SYM_HASH_SHA256;
344 			sessionSetupData.hashSetupData.hashMode =
345 			    CPA_CY_SYM_HASH_MODE_AUTH;
346 			break;
347 		case CRYPTO_SHA2_256:
348 			sessionSetupData.hashSetupData.hashAlgorithm =
349 			    CPA_CY_SYM_HASH_SHA256;
350 			sessionSetupData.hashSetupData.hashMode =
351 			    CPA_CY_SYM_HASH_MODE_PLAIN;
352 			break;
353 
354 		case CRYPTO_SHA2_224_HMAC:
355 			sessionSetupData.hashSetupData.hashAlgorithm =
356 			    CPA_CY_SYM_HASH_SHA224;
357 			sessionSetupData.hashSetupData.hashMode =
358 			    CPA_CY_SYM_HASH_MODE_AUTH;
359 			break;
360 		case CRYPTO_SHA2_224:
361 			sessionSetupData.hashSetupData.hashAlgorithm =
362 			    CPA_CY_SYM_HASH_SHA224;
363 			sessionSetupData.hashSetupData.hashMode =
364 			    CPA_CY_SYM_HASH_MODE_PLAIN;
365 			break;
366 
367 		case CRYPTO_SHA2_384_HMAC:
368 			sessionSetupData.hashSetupData.hashAlgorithm =
369 			    CPA_CY_SYM_HASH_SHA384;
370 			sessionSetupData.hashSetupData.hashMode =
371 			    CPA_CY_SYM_HASH_MODE_AUTH;
372 			break;
373 		case CRYPTO_SHA2_384:
374 			sessionSetupData.hashSetupData.hashAlgorithm =
375 			    CPA_CY_SYM_HASH_SHA384;
376 			sessionSetupData.hashSetupData.hashMode =
377 			    CPA_CY_SYM_HASH_MODE_PLAIN;
378 			break;
379 
380 		case CRYPTO_SHA2_512_HMAC:
381 			sessionSetupData.hashSetupData.hashAlgorithm =
382 			    CPA_CY_SYM_HASH_SHA512;
383 			sessionSetupData.hashSetupData.hashMode =
384 			    CPA_CY_SYM_HASH_MODE_AUTH;
385 			break;
386 		case CRYPTO_SHA2_512:
387 			sessionSetupData.hashSetupData.hashAlgorithm =
388 			    CPA_CY_SYM_HASH_SHA512;
389 			sessionSetupData.hashSetupData.hashMode =
390 			    CPA_CY_SYM_HASH_MODE_PLAIN;
391 			break;
392 		case CRYPTO_AES_NIST_GMAC:
393 			sessionSetupData.hashSetupData.hashAlgorithm =
394 			    CPA_CY_SYM_HASH_AES_GMAC;
395 			break;
396 		default:
397 			status = CPA_STATUS_UNSUPPORTED;
398 			goto fail;
399 		}
400 	} /* csp->csp_auth_alg */
401 
402 	/* Setting digest-length if no cipher-only mode is set */
403 	if (csp->csp_mode != CSP_MODE_CIPHER) {
404 		lac_sym_qat_hash_defs_t *pHashDefsInfo = NULL;
405 		if (csp->csp_auth_mlen) {
406 			sessionSetupData.hashSetupData.digestResultLenInBytes =
407 			    csp->csp_auth_mlen;
408 			qat_ssession->authLen = csp->csp_auth_mlen;
409 		} else {
410 			LacSymQat_HashDefsLookupGet(
411 			    qat_instance->cyInstHandle,
412 			    sessionSetupData.hashSetupData.hashAlgorithm,
413 			    &pHashDefsInfo);
414 			if (NULL == pHashDefsInfo) {
415 				device_printf(
416 				    dev,
417 				    "unable to find corresponding hash data\n");
418 				status = CPA_STATUS_UNSUPPORTED;
419 				goto fail;
420 			}
421 			sessionSetupData.hashSetupData.digestResultLenInBytes =
422 			    pHashDefsInfo->algInfo->digestLength;
423 			qat_ssession->authLen =
424 			    pHashDefsInfo->algInfo->digestLength;
425 		}
426 		sessionSetupData.verifyDigest = CPA_FALSE;
427 	}
428 
429 	switch (csp->csp_mode) {
430 	case CSP_MODE_AEAD:
431 	case CSP_MODE_ETA:
432 		sessionSetupData.symOperation =
433 		    CPA_CY_SYM_OP_ALGORITHM_CHAINING;
434 		/* Place the digest result in a buffer unrelated to srcBuffer */
435 		sessionSetupData.digestIsAppended = CPA_FALSE;
436 		/* Due to FW limitation to verify only appended MACs */
437 		sessionSetupData.verifyDigest = CPA_FALSE;
438 		if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
439 			sessionSetupData.cipherSetupData.cipherDirection =
440 			    CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT;
441 			sessionSetupData.algChainOrder =
442 			    CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH;
443 		} else {
444 			sessionSetupData.cipherSetupData.cipherDirection =
445 			    CPA_CY_SYM_CIPHER_DIRECTION_DECRYPT;
446 			sessionSetupData.algChainOrder =
447 			    CPA_CY_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER;
448 		}
449 		break;
450 	case CSP_MODE_CIPHER:
451 		if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
452 			sessionSetupData.cipherSetupData.cipherDirection =
453 			    CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT;
454 		} else {
455 			sessionSetupData.cipherSetupData.cipherDirection =
456 			    CPA_CY_SYM_CIPHER_DIRECTION_DECRYPT;
457 		}
458 		sessionSetupData.symOperation = CPA_CY_SYM_OP_CIPHER;
459 		break;
460 	case CSP_MODE_DIGEST:
461 		sessionSetupData.symOperation = CPA_CY_SYM_OP_HASH;
462 		if (csp->csp_auth_alg == CRYPTO_AES_NIST_GMAC) {
463 			sessionSetupData.symOperation =
464 			    CPA_CY_SYM_OP_ALGORITHM_CHAINING;
465 			/* GMAC is always encrypt */
466 			sessionSetupData.cipherSetupData.cipherDirection =
467 			    CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT;
468 			sessionSetupData.algChainOrder =
469 			    CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH;
470 			sessionSetupData.cipherSetupData.cipherAlgorithm =
471 			    CPA_CY_SYM_CIPHER_AES_GCM;
472 			sessionSetupData.hashSetupData.hashAlgorithm =
473 			    CPA_CY_SYM_HASH_AES_GMAC;
474 			sessionSetupData.hashSetupData.hashMode =
475 			    CPA_CY_SYM_HASH_MODE_AUTH;
476 			/* Same key for cipher and auth */
477 			sessionSetupData.cipherSetupData.pCipherKey =
478 			    csp->csp_auth_key;
479 			sessionSetupData.cipherSetupData.cipherKeyLenInBytes =
480 			    csp->csp_auth_klen;
481 			/* Generated GMAC stored in separated buffer */
482 			sessionSetupData.digestIsAppended = CPA_FALSE;
483 			/* Digest verification not allowed in GMAC case */
484 			sessionSetupData.verifyDigest = CPA_FALSE;
485 			/* No AAD allowed */
486 			sessionSetupData.hashSetupData.authModeSetupData
487 			    .aadLenInBytes = 0;
488 		} else {
489 			sessionSetupData.cipherSetupData.cipherDirection =
490 			    CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT;
491 			sessionSetupData.symOperation = CPA_CY_SYM_OP_HASH;
492 			sessionSetupData.digestIsAppended = CPA_FALSE;
493 		}
494 		break;
495 	default:
496 		device_printf(dev,
497 			      "%s: unhandled crypto algorithm %d, %d\n",
498 			      __func__,
499 			      csp->csp_cipher_alg,
500 			      csp->csp_auth_alg);
501 		status = CPA_STATUS_FAIL;
502 		goto fail;
503 	}
504 
505 	/* Extracting session size */
506 	status = cpaCySymSessionCtxGetSize(qat_instance->cyInstHandle,
507 					   &sessionSetupData,
508 					   &sessionCtxSize);
509 	if (CPA_STATUS_SUCCESS != status) {
510 		device_printf(dev, "unable to get session size\n");
511 		goto fail;
512 	}
513 
514 	/* Allocating contiguous memory for session */
515 	sessionCtx = contigmalloc(sessionCtxSize,
516 				  M_QAT_OCF,
517 				  M_NOWAIT,
518 				  0,
519 				  ~1UL,
520 				  1 << (ilog2(sessionCtxSize - 1) + 1),
521 				  0);
522 	if (NULL == sessionCtx) {
523 		device_printf(dev, "unable to allocate memory for session\n");
524 		status = CPA_STATUS_RESOURCE;
525 		goto fail;
526 	}
527 
528 	status = cpaCySymDpInitSession(qat_instance->cyInstHandle,
529 				       &sessionSetupData,
530 				       sessionCtx);
531 	if (CPA_STATUS_SUCCESS != status) {
532 		device_printf(dev, "session initialization failed\n");
533 		goto fail;
534 	}
535 
536 	/* NOTE: lets keep double session (both directions) approach to overcome
537 	 * lack of direction update in FBSD QAT.
538 	 */
539 	qat_ssession->sessionCtx = sessionCtx;
540 	qat_ssession->sessionCtxSize = sessionCtxSize;
541 
542 	return CPA_STATUS_SUCCESS;
543 
544 fail:
545 	/* Release resources if any */
546 	if (sessionCtx)
547 		free(sessionCtx, M_QAT_OCF);
548 
549 	return status;
550 }
551 
552 static int
553 qat_ocf_newsession(device_t dev,
554 		   crypto_session_t cses,
555 		   const struct crypto_session_params *csp)
556 {
557 	/* Cryptodev QAT structures */
558 	struct qat_ocf_softc *qat_softc;
559 	struct qat_ocf_dsession *qat_dsession;
560 	struct qat_ocf_instance *qat_instance;
561 	u_int cpu_id = PCPU_GET(cpuid);
562 
563 	/* Create cryptodev session */
564 	qat_softc = device_get_softc(dev);
565 	if (qat_softc->numCyInstances > 0) {
566 		qat_instance =
567 		    &qat_softc
568 			 ->cyInstHandles[cpu_id % qat_softc->numCyInstances];
569 		qat_dsession = crypto_get_driver_session(cses);
570 		if (NULL == qat_dsession) {
571 			device_printf(dev, "Unable to create new session\n");
572 			return (EINVAL);
573 		}
574 
575 		/* Add only instance at this point remaining operations moved to
576 		 * lazy session init */
577 		qat_dsession->qatInstance = qat_instance;
578 	} else {
579 		return ENXIO;
580 	}
581 
582 	return 0;
583 }
584 
585 static CpaStatus
586 qat_ocf_remove_session(device_t dev,
587 		       CpaInstanceHandle cyInstHandle,
588 		       struct qat_ocf_session *qat_session)
589 {
590 	CpaStatus status = CPA_STATUS_SUCCESS;
591 
592 	if (NULL == qat_session->sessionCtx)
593 		return CPA_STATUS_SUCCESS;
594 
595 	/* User callback is executed right before decrementing pending
596 	 * callback atomic counter. To avoid removing session rejection
597 	 * we have to wait a very short while for counter update
598 	 * after call back execution. */
599 	status = qat_ocf_wait_for_session(qat_session->sessionCtx,
600 					  QAT_OCF_SESSION_WAIT_TIMEOUT_MS);
601 	if (CPA_STATUS_SUCCESS != status) {
602 		device_printf(dev, "waiting for session un-busy failed\n");
603 		return CPA_STATUS_FAIL;
604 	}
605 
606 	status = cpaCySymDpRemoveSession(cyInstHandle, qat_session->sessionCtx);
607 	if (CPA_STATUS_SUCCESS != status) {
608 		device_printf(dev, "error while removing session\n");
609 		return CPA_STATUS_FAIL;
610 	}
611 
612 	explicit_bzero(qat_session->sessionCtx, qat_session->sessionCtxSize);
613 	free(qat_session->sessionCtx, M_QAT_OCF);
614 	qat_session->sessionCtx = NULL;
615 	qat_session->sessionCtxSize = 0;
616 
617 	return CPA_STATUS_SUCCESS;
618 }
619 
620 static void
621 qat_ocf_freesession(device_t dev, crypto_session_t cses)
622 {
623 	CpaStatus status = CPA_STATUS_SUCCESS;
624 	struct qat_ocf_dsession *qat_dsession = NULL;
625 	struct qat_ocf_instance *qat_instance = NULL;
626 
627 	qat_dsession = crypto_get_driver_session(cses);
628 	qat_instance = qat_dsession->qatInstance;
629 	mtx_lock(&qat_instance->cyInstMtx);
630 	status = qat_ocf_remove_session(dev,
631 					qat_dsession->qatInstance->cyInstHandle,
632 					&qat_dsession->encSession);
633 	if (CPA_STATUS_SUCCESS != status)
634 		device_printf(dev, "unable to remove encrypt session\n");
635 	status = qat_ocf_remove_session(dev,
636 					qat_dsession->qatInstance->cyInstHandle,
637 					&qat_dsession->decSession);
638 	if (CPA_STATUS_SUCCESS != status)
639 		device_printf(dev, "unable to remove decrypt session\n");
640 	mtx_unlock(&qat_instance->cyInstMtx);
641 }
642 
643 /* QAT GCM/CCM FW API are only algorithms which support separated AAD. */
644 static CpaStatus
645 qat_ocf_load_aad_gcm(struct cryptop *crp, struct qat_ocf_cookie *qat_cookie)
646 {
647 	CpaCySymDpOpData *pOpData;
648 
649 	pOpData = &qat_cookie->pOpdata;
650 
651 	if (NULL != crp->crp_aad)
652 		memcpy(qat_cookie->qat_ocf_gcm_aad,
653 		       crp->crp_aad,
654 		       crp->crp_aad_length);
655 	else
656 		crypto_copydata(crp,
657 				crp->crp_aad_start,
658 				crp->crp_aad_length,
659 				qat_cookie->qat_ocf_gcm_aad);
660 
661 	pOpData->pAdditionalAuthData = qat_cookie->qat_ocf_gcm_aad;
662 	pOpData->additionalAuthData = qat_cookie->qat_ocf_gcm_aad_paddr;
663 
664 	return CPA_STATUS_SUCCESS;
665 }
666 
667 static CpaStatus
668 qat_ocf_load_aad(struct cryptop *crp, struct qat_ocf_cookie *qat_cookie)
669 {
670 	CpaStatus status = CPA_STATUS_SUCCESS;
671 	const struct crypto_session_params *csp;
672 	CpaCySymDpOpData *pOpData;
673 	struct qat_ocf_load_cb_arg args;
674 
675 	pOpData = &qat_cookie->pOpdata;
676 	pOpData->pAdditionalAuthData = NULL;
677 	pOpData->additionalAuthData = 0UL;
678 
679 	if (crp->crp_aad_length == 0)
680 		return CPA_STATUS_SUCCESS;
681 
682 	if (crp->crp_aad_length > ICP_QAT_FW_CCM_GCM_AAD_SZ_MAX)
683 		return CPA_STATUS_FAIL;
684 
685 	csp = crypto_get_params(crp->crp_session);
686 
687 	/* Handle GCM/CCM case */
688 	if (CPA_TRUE == is_sep_aad_supported(csp))
689 		return qat_ocf_load_aad_gcm(crp, qat_cookie);
690 
691 	if (NULL == crp->crp_aad) {
692 		/* AAD already embedded in source buffer */
693 		pOpData->messageLenToCipherInBytes = crp->crp_payload_length;
694 		pOpData->cryptoStartSrcOffsetInBytes = crp->crp_payload_start;
695 
696 		pOpData->messageLenToHashInBytes =
697 		    crp->crp_aad_length + crp->crp_payload_length;
698 		pOpData->hashStartSrcOffsetInBytes = crp->crp_aad_start;
699 
700 		return CPA_STATUS_SUCCESS;
701 	}
702 
703 	/* Separated AAD not supported by QAT - lets place the content
704 	 * of ADD buffer at the very beginning of source SGL */
705 	args.crp_op = crp;
706 	args.qat_cookie = qat_cookie;
707 	args.pOpData = pOpData;
708 	args.error = 0;
709 	status = bus_dmamap_load(qat_cookie->gcm_aad_dma_mem.dma_tag,
710 				 qat_cookie->gcm_aad_dma_mem.dma_map,
711 				 crp->crp_aad,
712 				 crp->crp_aad_length,
713 				 qat_ocf_crypto_load_aadbuf_cb,
714 				 &args,
715 				 BUS_DMA_NOWAIT);
716 	qat_cookie->is_sep_aad_used = CPA_TRUE;
717 
718 	/* Right after this step we have AAD placed in the first flat buffer
719 	 * in source SGL */
720 	pOpData->messageLenToCipherInBytes = crp->crp_payload_length;
721 	pOpData->cryptoStartSrcOffsetInBytes =
722 	    crp->crp_aad_length + crp->crp_aad_start + crp->crp_payload_start;
723 
724 	pOpData->messageLenToHashInBytes =
725 	    crp->crp_aad_length + crp->crp_payload_length;
726 	pOpData->hashStartSrcOffsetInBytes = crp->crp_aad_start;
727 
728 	return status;
729 }
730 
731 static CpaStatus
732 qat_ocf_load(struct cryptop *crp, struct qat_ocf_cookie *qat_cookie)
733 {
734 	CpaStatus status = CPA_STATUS_SUCCESS;
735 	CpaCySymDpOpData *pOpData;
736 	struct qat_ocf_load_cb_arg args;
737 	/* cryptodev internals */
738 	const struct crypto_session_params *csp;
739 
740 	pOpData = &qat_cookie->pOpdata;
741 
742 	csp = crypto_get_params(crp->crp_session);
743 
744 	/* Load IV buffer if present */
745 	if (csp->csp_ivlen > 0) {
746 		memset(qat_cookie->qat_ocf_iv_buf,
747 		       0,
748 		       sizeof(qat_cookie->qat_ocf_iv_buf));
749 		crypto_read_iv(crp, qat_cookie->qat_ocf_iv_buf);
750 		pOpData->iv = qat_cookie->qat_ocf_iv_buf_paddr;
751 		pOpData->pIv = qat_cookie->qat_ocf_iv_buf;
752 		pOpData->ivLenInBytes = csp->csp_ivlen;
753 	}
754 
755 	/* GCM/CCM - load AAD to separated buffer
756 	 * AES+SHA - load AAD to first flat in SGL */
757 	status = qat_ocf_load_aad(crp, qat_cookie);
758 	if (CPA_STATUS_SUCCESS != status)
759 		goto fail;
760 
761 	/* Load source buffer */
762 	args.crp_op = crp;
763 	args.qat_cookie = qat_cookie;
764 	args.pOpData = pOpData;
765 	args.error = 0;
766 	status = bus_dmamap_load_crp_buffer(qat_cookie->src_dma_mem.dma_tag,
767 					    qat_cookie->src_dma_mem.dma_map,
768 					    &crp->crp_buf,
769 					    qat_ocf_crypto_load_buf_cb,
770 					    &args,
771 					    BUS_DMA_NOWAIT);
772 	if (CPA_STATUS_SUCCESS != status)
773 		goto fail;
774 	pOpData->srcBuffer = qat_cookie->src_buffer_list_paddr;
775 	pOpData->srcBufferLen = CPA_DP_BUFLIST;
776 
777 	/* Load destination buffer */
778 	if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) {
779 		status =
780 		    bus_dmamap_load_crp_buffer(qat_cookie->dst_dma_mem.dma_tag,
781 					       qat_cookie->dst_dma_mem.dma_map,
782 					       &crp->crp_obuf,
783 					       qat_ocf_crypto_load_obuf_cb,
784 					       &args,
785 					       BUS_DMA_NOWAIT);
786 		if (CPA_STATUS_SUCCESS != status)
787 			goto fail;
788 		pOpData->dstBuffer = qat_cookie->dst_buffer_list_paddr;
789 		pOpData->dstBufferLen = CPA_DP_BUFLIST;
790 	} else {
791 		pOpData->dstBuffer = pOpData->srcBuffer;
792 		pOpData->dstBufferLen = pOpData->srcBufferLen;
793 	}
794 
795 	if (CPA_TRUE == is_use_sep_digest(csp))
796 		pOpData->digestResult = qat_cookie->qat_ocf_digest_paddr;
797 	else
798 		pOpData->digestResult = 0UL;
799 
800 	/* GMAC - aka zero length buffer */
801 	if (CPA_TRUE == is_gmac_exception(csp))
802 		pOpData->messageLenToCipherInBytes = 0;
803 
804 fail:
805 	return status;
806 }
807 
808 static int
809 qat_ocf_check_input(device_t dev, struct cryptop *crp)
810 {
811 	const struct crypto_session_params *csp;
812 	csp = crypto_get_params(crp->crp_session);
813 
814 	if (crypto_buffer_len(&crp->crp_buf) > QAT_OCF_MAX_LEN)
815 		return E2BIG;
816 
817 	if (CPA_TRUE == is_sep_aad_supported(csp) &&
818 	    (crp->crp_aad_length > ICP_QAT_FW_CCM_GCM_AAD_SZ_MAX))
819 		return EBADMSG;
820 
821 	return 0;
822 }
823 
824 static int
825 qat_ocf_process(device_t dev, struct cryptop *crp, int hint)
826 {
827 	CpaStatus status = CPA_STATUS_SUCCESS;
828 	int rc = 0;
829 	struct qat_ocf_dsession *qat_dsession = NULL;
830 	struct qat_ocf_session *qat_session = NULL;
831 	struct qat_ocf_instance *qat_instance = NULL;
832 	CpaCySymDpOpData *pOpData = NULL;
833 	struct qat_ocf_cookie *qat_cookie = NULL;
834 	CpaBoolean memLoaded = CPA_FALSE;
835 
836 	rc = qat_ocf_check_input(dev, crp);
837 	if (rc)
838 		goto fail;
839 
840 	qat_dsession = crypto_get_driver_session(crp->crp_session);
841 
842 	if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
843 		qat_session = &qat_dsession->encSession;
844 	else
845 		qat_session = &qat_dsession->decSession;
846 	qat_instance = qat_dsession->qatInstance;
847 
848 	status = qat_ocf_cookie_alloc(qat_instance, &qat_cookie);
849 	if (CPA_STATUS_SUCCESS != status) {
850 		rc = EAGAIN;
851 		goto fail;
852 	}
853 
854 	qat_cookie->crp_op = crp;
855 
856 	/* Common request fields */
857 	pOpData = &qat_cookie->pOpdata;
858 	pOpData->instanceHandle = qat_instance->cyInstHandle;
859 	pOpData->sessionCtx = NULL;
860 
861 	/* Cipher fields */
862 	pOpData->cryptoStartSrcOffsetInBytes = crp->crp_payload_start;
863 	pOpData->messageLenToCipherInBytes = crp->crp_payload_length;
864 	/* Digest fields - any exceptions from this basic rules are covered
865 	 * in qat_ocf_load */
866 	pOpData->hashStartSrcOffsetInBytes = crp->crp_payload_start;
867 	pOpData->messageLenToHashInBytes = crp->crp_payload_length;
868 
869 	status = qat_ocf_load(crp, qat_cookie);
870 	if (CPA_STATUS_SUCCESS != status) {
871 		device_printf(dev,
872 			      "unable to load OCF buffers to QAT DMA "
873 			      "transaction\n");
874 		rc = EIO;
875 		goto fail;
876 	}
877 	memLoaded = CPA_TRUE;
878 
879 	status = qat_ocf_cookie_dma_pre_sync(crp, pOpData);
880 	if (CPA_STATUS_SUCCESS != status) {
881 		device_printf(dev, "unable to sync DMA buffers\n");
882 		rc = EIO;
883 		goto fail;
884 	}
885 
886 	mtx_lock(&qat_instance->cyInstMtx);
887 	/* Session initialization at the first request. It's done
888 	 * in such way to overcome missing QAT specific session data
889 	 * such like AAD length and limited possibility to update
890 	 * QAT session while handling traffic.
891 	 */
892 	if (NULL == qat_session->sessionCtx) {
893 		status =
894 		    qat_ocf_session_init(dev, crp, qat_instance, qat_session);
895 		if (CPA_STATUS_SUCCESS != status) {
896 			mtx_unlock(&qat_instance->cyInstMtx);
897 			device_printf(dev, "unable to init session\n");
898 			rc = EIO;
899 			goto fail;
900 		}
901 	} else {
902 		status = qat_ocf_handle_session_update(qat_dsession, crp);
903 		if (CPA_STATUS_RESOURCE == status) {
904 			mtx_unlock(&qat_instance->cyInstMtx);
905 			rc = EAGAIN;
906 			goto fail;
907 		} else if (CPA_STATUS_SUCCESS != status) {
908 			mtx_unlock(&qat_instance->cyInstMtx);
909 			rc = EIO;
910 			goto fail;
911 		}
912 	}
913 	pOpData->sessionCtx = qat_session->sessionCtx;
914 	status = cpaCySymDpEnqueueOp(pOpData, CPA_TRUE);
915 	mtx_unlock(&qat_instance->cyInstMtx);
916 	if (CPA_STATUS_SUCCESS != status) {
917 		if (CPA_STATUS_RETRY == status) {
918 			rc = EAGAIN;
919 			goto fail;
920 		}
921 		device_printf(dev,
922 			      "unable to send request. Status: %d\n",
923 			      status);
924 		rc = EIO;
925 		goto fail;
926 	}
927 
928 	return 0;
929 fail:
930 	if (qat_cookie) {
931 		if (memLoaded)
932 			qat_ocf_cookie_dma_unload(crp, pOpData);
933 		qat_ocf_cookie_free(qat_instance, qat_cookie);
934 	}
935 	crp->crp_etype = rc;
936 	crypto_done(crp);
937 
938 	return 0;
939 }
940 
941 static void
942 qat_ocf_identify(driver_t *drv, device_t parent)
943 {
944 	if (device_find_child(parent, "qat_ocf", -1) == NULL &&
945 	    BUS_ADD_CHILD(parent, 200, "qat_ocf", -1) == 0)
946 		device_printf(parent, "qat_ocf: could not attach!");
947 }
948 
949 static int
950 qat_ocf_probe(device_t dev)
951 {
952 	device_set_desc(dev, "QAT engine");
953 	return (BUS_PROBE_NOWILDCARD);
954 }
955 
956 static CpaStatus
957 qat_ocf_get_irq_instances(CpaInstanceHandle *cyInstHandles,
958 			  Cpa16U cyInstHandlesSize,
959 			  Cpa16U *foundInstances)
960 {
961 	CpaStatus status = CPA_STATUS_SUCCESS;
962 	icp_accel_dev_t **pAdfInsts = NULL;
963 	icp_accel_dev_t *dev_addr = NULL;
964 	sal_t *baseAddr = NULL;
965 	sal_list_t *listTemp = NULL;
966 	CpaInstanceHandle cyInstHandle;
967 	CpaInstanceInfo2 info;
968 	Cpa16U numDevices;
969 	Cpa32U instCtr = 0;
970 	Cpa32U i;
971 
972 	/* Get the number of devices */
973 	status = icp_amgr_getNumInstances(&numDevices);
974 	if (CPA_STATUS_SUCCESS != status)
975 		return status;
976 
977 	/* Allocate memory to store addr of accel_devs */
978 	pAdfInsts =
979 	    malloc(numDevices * sizeof(icp_accel_dev_t *), M_QAT_OCF, M_WAITOK);
980 
981 	/* Get ADF to return all accel_devs that support either
982 	 * symmetric or asymmetric crypto */
983 	status = icp_amgr_getAllAccelDevByCapabilities(
984 	    (ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC), pAdfInsts, &numDevices);
985 	if (CPA_STATUS_SUCCESS != status) {
986 		free(pAdfInsts, M_QAT_OCF);
987 		return status;
988 	}
989 
990 	for (i = 0; i < numDevices; i++) {
991 		dev_addr = (icp_accel_dev_t *)pAdfInsts[i];
992 		baseAddr = dev_addr->pSalHandle;
993 		if (NULL == baseAddr)
994 			continue;
995 		listTemp = baseAddr->sym_services;
996 		if (NULL == listTemp) {
997 			listTemp = baseAddr->crypto_services;
998 		}
999 
1000 		while (NULL != listTemp) {
1001 			cyInstHandle = SalList_getObject(listTemp);
1002 			status = cpaCyInstanceGetInfo2(cyInstHandle, &info);
1003 			if (CPA_STATUS_SUCCESS != status)
1004 				continue;
1005 			listTemp = SalList_next(listTemp);
1006 			if (CPA_TRUE == info.isPolled)
1007 				continue;
1008 			if (instCtr >= cyInstHandlesSize)
1009 				break;
1010 			cyInstHandles[instCtr++] = cyInstHandle;
1011 		}
1012 	}
1013 	free(pAdfInsts, M_QAT_OCF);
1014 	*foundInstances = instCtr;
1015 
1016 	return CPA_STATUS_SUCCESS;
1017 }
1018 
1019 static CpaStatus
1020 qat_ocf_start_instances(struct qat_ocf_softc *qat_softc, device_t dev)
1021 {
1022 	CpaStatus status = CPA_STATUS_SUCCESS;
1023 	Cpa16U numInstances = 0;
1024 	CpaInstanceHandle cyInstHandles[QAT_OCF_MAX_INSTANCES] = { 0 };
1025 	CpaInstanceHandle cyInstHandle = NULL;
1026 	Cpa32U startedInstances = 0;
1027 	Cpa32U i;
1028 
1029 	qat_softc->numCyInstances = 0;
1030 	status = qat_ocf_get_irq_instances(cyInstHandles,
1031 					   QAT_OCF_MAX_INSTANCES,
1032 					   &numInstances);
1033 	if (CPA_STATUS_SUCCESS != status)
1034 		return status;
1035 
1036 	for (i = 0; i < numInstances; i++) {
1037 		struct qat_ocf_instance *qat_ocf_instance;
1038 
1039 		cyInstHandle = cyInstHandles[i];
1040 		if (!cyInstHandle)
1041 			continue;
1042 
1043 		/* Starting instance */
1044 		status = cpaCyStartInstance(cyInstHandle);
1045 		if (CPA_STATUS_SUCCESS != status) {
1046 			device_printf(qat_softc->sc_dev,
1047 				      "unable to get start instance\n");
1048 			continue;
1049 		}
1050 
1051 		qat_ocf_instance = &qat_softc->cyInstHandles[startedInstances];
1052 		qat_ocf_instance->cyInstHandle = cyInstHandle;
1053 		mtx_init(&qat_ocf_instance->cyInstMtx,
1054 			 "Instance MTX",
1055 			 NULL,
1056 			 MTX_DEF);
1057 
1058 		status =
1059 		    cpaCySetAddressTranslation(cyInstHandle, qatVirtToPhys);
1060 		if (CPA_STATUS_SUCCESS != status) {
1061 			device_printf(qat_softc->sc_dev,
1062 				      "unable to add virt to phys callback\n");
1063 			goto fail;
1064 		}
1065 
1066 		status = cpaCySymDpRegCbFunc(cyInstHandle, symDpCallback);
1067 		if (CPA_STATUS_SUCCESS != status) {
1068 			device_printf(qat_softc->sc_dev,
1069 				      "unable to add user callback\n");
1070 			goto fail;
1071 		}
1072 
1073 		/* Initialize cookie pool */
1074 		status = qat_ocf_cookie_pool_init(qat_ocf_instance, dev);
1075 		if (CPA_STATUS_SUCCESS != status) {
1076 			device_printf(qat_softc->sc_dev,
1077 				      "unable to create cookie pool\n");
1078 			goto fail;
1079 		}
1080 
1081 		/* Disable forcing HW MAC validation for AEAD */
1082 		status = icp_sal_setForceAEADMACVerify(cyInstHandle, CPA_FALSE);
1083 		if (CPA_STATUS_SUCCESS != status) {
1084 			device_printf(
1085 			    qat_softc->sc_dev,
1086 			    "unable to disable AEAD HW MAC verification\n");
1087 			goto fail;
1088 		}
1089 
1090 		qat_ocf_instance->driver_id = qat_softc->cryptodev_id;
1091 
1092 		startedInstances++;
1093 		continue;
1094 	fail:
1095 		mtx_destroy(&qat_ocf_instance->cyInstMtx);
1096 
1097 		/* Stop instance */
1098 		status = cpaCyStopInstance(cyInstHandle);
1099 		if (CPA_STATUS_SUCCESS != status)
1100 			device_printf(qat_softc->sc_dev,
1101 				      "unable to stop the instance\n");
1102 	}
1103 	qat_softc->numCyInstances = startedInstances;
1104 
1105 	return CPA_STATUS_SUCCESS;
1106 }
1107 
1108 static CpaStatus
1109 qat_ocf_stop_instances(struct qat_ocf_softc *qat_softc)
1110 {
1111 	CpaStatus status = CPA_STATUS_SUCCESS;
1112 	int i;
1113 
1114 	for (i = 0; i < qat_softc->numCyInstances; i++) {
1115 		struct qat_ocf_instance *qat_instance;
1116 
1117 		qat_instance = &qat_softc->cyInstHandles[i];
1118 		status = cpaCyStopInstance(qat_instance->cyInstHandle);
1119 		if (CPA_STATUS_SUCCESS != status) {
1120 			pr_err("QAT: stopping instance id: %d failed\n", i);
1121 			continue;
1122 		}
1123 		qat_ocf_cookie_pool_deinit(qat_instance);
1124 		mtx_destroy(&qat_instance->cyInstMtx);
1125 	}
1126 
1127 	qat_softc->numCyInstances = 0;
1128 
1129 	return status;
1130 }
1131 
1132 static int
1133 qat_ocf_deinit(struct qat_ocf_softc *qat_softc)
1134 {
1135 	int status = 0;
1136 	CpaStatus cpaStatus;
1137 
1138 	if (qat_softc->cryptodev_id >= 0) {
1139 		crypto_unregister_all(qat_softc->cryptodev_id);
1140 		qat_softc->cryptodev_id = -1;
1141 	}
1142 
1143 	/* Stop QAT instances */
1144 	cpaStatus = qat_ocf_stop_instances(qat_softc);
1145 	if (CPA_STATUS_SUCCESS != cpaStatus) {
1146 		device_printf(qat_softc->sc_dev, "unable to stop instances\n");
1147 		status = EIO;
1148 	}
1149 
1150 	return status;
1151 }
1152 
1153 static int
1154 qat_ocf_init(struct qat_ocf_softc *qat_softc)
1155 {
1156 	int32_t cryptodev_id;
1157 
1158 	/* Starting instances for OCF */
1159 	if (qat_ocf_start_instances(qat_softc, qat_softc->sc_dev)) {
1160 		device_printf(qat_softc->sc_dev,
1161 			      "unable to get QAT IRQ instances\n");
1162 		goto fail;
1163 	}
1164 
1165 	/* Register only if instances available */
1166 	if (qat_softc->numCyInstances) {
1167 		cryptodev_id =
1168 		    crypto_get_driverid(qat_softc->sc_dev,
1169 					sizeof(struct qat_ocf_dsession),
1170 					CRYPTOCAP_F_HARDWARE);
1171 		if (cryptodev_id < 0) {
1172 			device_printf(qat_softc->sc_dev,
1173 				      "cannot initialize!\n");
1174 			goto fail;
1175 		}
1176 		qat_softc->cryptodev_id = cryptodev_id;
1177 	}
1178 
1179 	return 0;
1180 fail:
1181 	qat_ocf_deinit(qat_softc);
1182 
1183 	return ENXIO;
1184 }
1185 
1186 static int qat_ocf_sysctl_handle(SYSCTL_HANDLER_ARGS)
1187 {
1188 	struct qat_ocf_softc *qat_softc = NULL;
1189 	int ret = 0;
1190 	device_t dev = arg1;
1191 	u_int enabled;
1192 
1193 	qat_softc = device_get_softc(dev);
1194 	enabled = qat_softc->enabled;
1195 
1196 	ret = sysctl_handle_int(oidp, &enabled, 0, req);
1197 	if (ret || !req->newptr)
1198 		return (ret);
1199 
1200 	if (qat_softc->enabled != enabled) {
1201 		if (enabled) {
1202 			ret = qat_ocf_init(qat_softc);
1203 
1204 		} else {
1205 			ret = qat_ocf_deinit(qat_softc);
1206 		}
1207 
1208 		if (!ret)
1209 			qat_softc->enabled = enabled;
1210 	}
1211 
1212 	return ret;
1213 }
1214 
1215 static int
1216 qat_ocf_attach(device_t dev)
1217 {
1218 	int status;
1219 	struct qat_ocf_softc *qat_softc;
1220 
1221 	qat_softc = device_get_softc(dev);
1222 	qat_softc->sc_dev = dev;
1223 	qat_softc->cryptodev_id = -1;
1224 	qat_softc->enabled = 1;
1225 
1226 	qat_softc->rc =
1227 	    SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
1228 			    SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
1229 			    OID_AUTO,
1230 			    "enable",
1231 			    CTLTYPE_INT | CTLFLAG_RWTUN | CTLFLAG_MPSAFE,
1232 			    dev,
1233 			    0,
1234 			    qat_ocf_sysctl_handle,
1235 			    "I",
1236 			    "QAT OCF support enablement");
1237 
1238 	if (!qat_softc->rc)
1239 		return ENOMEM;
1240 	if (qat_softc->enabled) {
1241 		status = qat_ocf_init(qat_softc);
1242 		if (status) {
1243 			device_printf(dev, "qat_ocf init failed\n");
1244 			goto fail;
1245 		}
1246 	}
1247 
1248 	return 0;
1249 fail:
1250 	qat_ocf_deinit(qat_softc);
1251 
1252 	return (ENXIO);
1253 }
1254 
1255 static int
1256 qat_ocf_detach(device_t dev)
1257 {
1258 	struct qat_ocf_softc *qat_softc = device_get_softc(dev);
1259 
1260 	return qat_ocf_deinit(qat_softc);
1261 }
1262 
1263 static device_method_t qat_ocf_methods[] =
1264     { DEVMETHOD(device_identify, qat_ocf_identify),
1265       DEVMETHOD(device_probe, qat_ocf_probe),
1266       DEVMETHOD(device_attach, qat_ocf_attach),
1267       DEVMETHOD(device_detach, qat_ocf_detach),
1268 
1269       /* Cryptodev interface */
1270       DEVMETHOD(cryptodev_probesession, qat_ocf_probesession),
1271       DEVMETHOD(cryptodev_newsession, qat_ocf_newsession),
1272       DEVMETHOD(cryptodev_freesession, qat_ocf_freesession),
1273       DEVMETHOD(cryptodev_process, qat_ocf_process),
1274 
1275       DEVMETHOD_END };
1276 
1277 static driver_t qat_ocf_driver = {
1278 	.name = "qat_ocf",
1279 	.methods = qat_ocf_methods,
1280 	.size = sizeof(struct qat_ocf_softc),
1281 };
1282 
1283 
1284 DRIVER_MODULE_ORDERED(qat,
1285 		      nexus,
1286 		      qat_ocf_driver,
1287 		      NULL,
1288 		      NULL,
1289 		      SI_ORDER_ANY);
1290 MODULE_VERSION(qat, 1);
1291 MODULE_DEPEND(qat, qat_c62x, 1, 1, 1);
1292 MODULE_DEPEND(qat, qat_200xx, 1, 1, 1);
1293 MODULE_DEPEND(qat, qat_c3xxx, 1, 1, 1);
1294 MODULE_DEPEND(qat, qat_c4xxx, 1, 1, 1);
1295 MODULE_DEPEND(qat, qat_dh895xcc, 1, 1, 1);
1296 MODULE_DEPEND(qat, qat_4xxx, 1, 1, 1);
1297 MODULE_DEPEND(qat, crypto, 1, 1, 1);
1298 MODULE_DEPEND(qat, qat_common, 1, 1, 1);
1299 MODULE_DEPEND(qat, qat_api, 1, 1, 1);
1300 MODULE_DEPEND(qat, linuxkpi, 1, 1, 1);
1301