1 /* SPDX-License-Identifier: BSD-3-Clause */ 2 /* Copyright(c) 2007-2022 Intel Corporation */ 3 /* System headers */ 4 #include <sys/param.h> 5 #include <sys/systm.h> 6 #include <sys/bus.h> 7 #include <sys/cpu.h> 8 #include <sys/kernel.h> 9 #include <sys/mbuf.h> 10 #include <sys/module.h> 11 #include <sys/mutex.h> 12 #include <sys/sysctl.h> 13 14 /* Cryptodev headers */ 15 #include <opencrypto/cryptodev.h> 16 #include "cryptodev_if.h" 17 18 /* QAT specific headers */ 19 #include "cpa.h" 20 #include "cpa_cy_im.h" 21 #include "cpa_cy_sym_dp.h" 22 #include "adf_accel_devices.h" 23 #include "adf_common_drv.h" 24 #include "lac_sym_hash_defs.h" 25 #include "lac_sym_qat_hash_defs_lookup.h" 26 27 /* To get only IRQ instances */ 28 #include "icp_accel_devices.h" 29 #include "icp_adf_accel_mgr.h" 30 #include "lac_sal_types.h" 31 32 /* To disable AEAD HW MAC verification */ 33 #include "icp_sal_user.h" 34 35 /* QAT OCF specific headers */ 36 #include "qat_ocf_mem_pool.h" 37 #include "qat_ocf_utils.h" 38 39 #define QAT_OCF_MAX_INSTANCES (256) 40 #define QAT_OCF_SESSION_WAIT_TIMEOUT_MS (1000) 41 42 MALLOC_DEFINE(M_QAT_OCF, "qat_ocf", "qat_ocf(4) memory allocations"); 43 44 /* QAT OCF internal structures */ 45 struct qat_ocf_softc { 46 device_t sc_dev; 47 struct sysctl_oid *rc; 48 uint32_t enabled; 49 int32_t cryptodev_id; 50 struct qat_ocf_instance cyInstHandles[QAT_OCF_MAX_INSTANCES]; 51 int32_t numCyInstances; 52 }; 53 54 /* Function definitions */ 55 static void qat_ocf_freesession(device_t dev, crypto_session_t cses); 56 static int qat_ocf_probesession(device_t dev, 57 const struct crypto_session_params *csp); 58 static int qat_ocf_newsession(device_t dev, 59 crypto_session_t cses, 60 const struct crypto_session_params *csp); 61 static int qat_ocf_attach(device_t dev); 62 static int qat_ocf_detach(device_t dev); 63 64 static void 65 symDpCallback(CpaCySymDpOpData *pOpData, 66 CpaStatus result, 67 CpaBoolean verifyResult) 68 { 69 struct qat_ocf_cookie *qat_cookie; 70 struct cryptop *crp; 71 struct qat_ocf_dsession *qat_dsession = NULL; 72 struct qat_ocf_session *qat_session = NULL; 73 struct qat_ocf_instance *qat_instance = NULL; 74 CpaStatus status; 75 int rc = 0; 76 77 qat_cookie = (struct qat_ocf_cookie *)pOpData->pCallbackTag; 78 if (!qat_cookie) 79 return; 80 81 crp = qat_cookie->crp_op; 82 83 qat_dsession = crypto_get_driver_session(crp->crp_session); 84 qat_instance = qat_dsession->qatInstance; 85 86 status = qat_ocf_cookie_dma_post_sync(crp, pOpData); 87 if (CPA_STATUS_SUCCESS != status) { 88 rc = EIO; 89 goto exit; 90 } 91 92 status = qat_ocf_cookie_dma_unload(crp, pOpData); 93 if (CPA_STATUS_SUCCESS != status) { 94 rc = EIO; 95 goto exit; 96 } 97 98 /* Verify result */ 99 if (CPA_STATUS_SUCCESS != result) { 100 rc = EBADMSG; 101 goto exit; 102 } 103 104 /* Verify digest by FW (GCM and CCM only) */ 105 if (CPA_TRUE != verifyResult) { 106 rc = EBADMSG; 107 goto exit; 108 } 109 110 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) 111 qat_session = &qat_dsession->encSession; 112 else 113 qat_session = &qat_dsession->decSession; 114 115 /* Copy back digest result if it's stored in separated buffer */ 116 if (pOpData->digestResult && qat_session->authLen > 0) { 117 if ((crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) != 0) { 118 char icv[QAT_OCF_MAX_DIGEST] = { 0 }; 119 crypto_copydata(crp, 120 crp->crp_digest_start, 121 qat_session->authLen, 122 icv); 123 if (timingsafe_bcmp(icv, 124 qat_cookie->qat_ocf_digest, 125 qat_session->authLen) != 0) { 126 rc = EBADMSG; 127 goto exit; 128 } 129 } else { 130 crypto_copyback(crp, 131 crp->crp_digest_start, 132 qat_session->authLen, 133 qat_cookie->qat_ocf_digest); 134 } 135 } 136 137 exit: 138 qat_ocf_cookie_free(qat_instance, qat_cookie); 139 crp->crp_etype = rc; 140 crypto_done(crp); 141 142 return; 143 } 144 145 static inline CpaPhysicalAddr 146 qatVirtToPhys(void *virtAddr) 147 { 148 return (CpaPhysicalAddr)vtophys(virtAddr); 149 } 150 151 static int 152 qat_ocf_probesession(device_t dev, const struct crypto_session_params *csp) 153 { 154 if ((csp->csp_flags & ~(CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD)) != 155 0) { 156 return EINVAL; 157 } 158 159 switch (csp->csp_mode) { 160 case CSP_MODE_CIPHER: 161 switch (csp->csp_cipher_alg) { 162 case CRYPTO_AES_CBC: 163 case CRYPTO_AES_ICM: 164 if (csp->csp_ivlen != AES_BLOCK_LEN) 165 return EINVAL; 166 break; 167 case CRYPTO_AES_XTS: 168 if (csp->csp_ivlen != AES_XTS_IV_LEN) 169 return EINVAL; 170 break; 171 default: 172 return EINVAL; 173 } 174 break; 175 case CSP_MODE_DIGEST: 176 switch (csp->csp_auth_alg) { 177 case CRYPTO_SHA1: 178 case CRYPTO_SHA1_HMAC: 179 case CRYPTO_SHA2_256: 180 case CRYPTO_SHA2_256_HMAC: 181 case CRYPTO_SHA2_384: 182 case CRYPTO_SHA2_384_HMAC: 183 case CRYPTO_SHA2_512: 184 case CRYPTO_SHA2_512_HMAC: 185 break; 186 case CRYPTO_AES_NIST_GMAC: 187 if (csp->csp_ivlen != AES_GCM_IV_LEN) 188 return EINVAL; 189 break; 190 default: 191 return EINVAL; 192 } 193 break; 194 case CSP_MODE_AEAD: 195 switch (csp->csp_cipher_alg) { 196 case CRYPTO_AES_NIST_GCM_16: 197 if (csp->csp_ivlen != AES_GCM_IV_LEN) 198 return EINVAL; 199 break; 200 default: 201 return EINVAL; 202 } 203 break; 204 case CSP_MODE_ETA: 205 switch (csp->csp_auth_alg) { 206 case CRYPTO_SHA1_HMAC: 207 case CRYPTO_SHA2_256_HMAC: 208 case CRYPTO_SHA2_384_HMAC: 209 case CRYPTO_SHA2_512_HMAC: 210 switch (csp->csp_cipher_alg) { 211 case CRYPTO_AES_CBC: 212 case CRYPTO_AES_ICM: 213 if (csp->csp_ivlen != AES_BLOCK_LEN) 214 return EINVAL; 215 break; 216 case CRYPTO_AES_XTS: 217 if (csp->csp_ivlen != AES_XTS_IV_LEN) 218 return EINVAL; 219 break; 220 default: 221 return EINVAL; 222 } 223 break; 224 default: 225 return EINVAL; 226 } 227 break; 228 default: 229 return EINVAL; 230 } 231 232 return CRYPTODEV_PROBE_HARDWARE; 233 } 234 235 static CpaStatus 236 qat_ocf_session_init(device_t dev, 237 struct cryptop *crp, 238 struct qat_ocf_instance *qat_instance, 239 struct qat_ocf_session *qat_ssession) 240 { 241 CpaStatus status = CPA_STATUS_SUCCESS; 242 /* Crytpodev structures */ 243 crypto_session_t cses; 244 const struct crypto_session_params *csp; 245 /* DP API Session configuration */ 246 CpaCySymSessionSetupData sessionSetupData = { 0 }; 247 CpaCySymSessionCtx sessionCtx = NULL; 248 Cpa32U sessionCtxSize = 0; 249 250 cses = crp->crp_session; 251 if (NULL == cses) { 252 device_printf(dev, "no crypto session in cryptodev request\n"); 253 return CPA_STATUS_FAIL; 254 } 255 256 csp = crypto_get_params(cses); 257 if (NULL == csp) { 258 device_printf(dev, "no session in cryptodev session\n"); 259 return CPA_STATUS_FAIL; 260 } 261 262 /* Common fields */ 263 sessionSetupData.sessionPriority = CPA_CY_PRIORITY_HIGH; 264 /* Cipher key */ 265 if (crp->crp_cipher_key) 266 sessionSetupData.cipherSetupData.pCipherKey = 267 crp->crp_cipher_key; 268 else 269 sessionSetupData.cipherSetupData.pCipherKey = 270 csp->csp_cipher_key; 271 sessionSetupData.cipherSetupData.cipherKeyLenInBytes = 272 csp->csp_cipher_klen; 273 274 /* Auth key */ 275 if (crp->crp_auth_key) 276 sessionSetupData.hashSetupData.authModeSetupData.authKey = 277 crp->crp_auth_key; 278 else 279 sessionSetupData.hashSetupData.authModeSetupData.authKey = 280 csp->csp_auth_key; 281 sessionSetupData.hashSetupData.authModeSetupData.authKeyLenInBytes = 282 csp->csp_auth_klen; 283 284 qat_ssession->aadLen = crp->crp_aad_length; 285 if (CPA_TRUE == is_sep_aad_supported(csp)) 286 sessionSetupData.hashSetupData.authModeSetupData.aadLenInBytes = 287 crp->crp_aad_length; 288 else 289 sessionSetupData.hashSetupData.authModeSetupData.aadLenInBytes = 290 0; 291 292 /* Just setup algorithm - regardless of mode */ 293 if (csp->csp_cipher_alg) { 294 sessionSetupData.symOperation = CPA_CY_SYM_OP_CIPHER; 295 296 switch (csp->csp_cipher_alg) { 297 case CRYPTO_AES_CBC: 298 sessionSetupData.cipherSetupData.cipherAlgorithm = 299 CPA_CY_SYM_CIPHER_AES_CBC; 300 break; 301 case CRYPTO_AES_ICM: 302 sessionSetupData.cipherSetupData.cipherAlgorithm = 303 CPA_CY_SYM_CIPHER_AES_CTR; 304 break; 305 case CRYPTO_AES_XTS: 306 sessionSetupData.cipherSetupData.cipherAlgorithm = 307 CPA_CY_SYM_CIPHER_AES_XTS; 308 break; 309 case CRYPTO_AES_NIST_GCM_16: 310 sessionSetupData.cipherSetupData.cipherAlgorithm = 311 CPA_CY_SYM_CIPHER_AES_GCM; 312 sessionSetupData.hashSetupData.hashAlgorithm = 313 CPA_CY_SYM_HASH_AES_GCM; 314 sessionSetupData.hashSetupData.hashMode = 315 CPA_CY_SYM_HASH_MODE_AUTH; 316 break; 317 default: 318 device_printf(dev, 319 "cipher_alg: %d not supported\n", 320 csp->csp_cipher_alg); 321 status = CPA_STATUS_UNSUPPORTED; 322 goto fail; 323 } 324 } 325 326 if (csp->csp_auth_alg) { 327 switch (csp->csp_auth_alg) { 328 case CRYPTO_SHA1_HMAC: 329 sessionSetupData.hashSetupData.hashAlgorithm = 330 CPA_CY_SYM_HASH_SHA1; 331 sessionSetupData.hashSetupData.hashMode = 332 CPA_CY_SYM_HASH_MODE_AUTH; 333 break; 334 case CRYPTO_SHA1: 335 sessionSetupData.hashSetupData.hashAlgorithm = 336 CPA_CY_SYM_HASH_SHA1; 337 sessionSetupData.hashSetupData.hashMode = 338 CPA_CY_SYM_HASH_MODE_PLAIN; 339 break; 340 341 case CRYPTO_SHA2_256_HMAC: 342 sessionSetupData.hashSetupData.hashAlgorithm = 343 CPA_CY_SYM_HASH_SHA256; 344 sessionSetupData.hashSetupData.hashMode = 345 CPA_CY_SYM_HASH_MODE_AUTH; 346 break; 347 case CRYPTO_SHA2_256: 348 sessionSetupData.hashSetupData.hashAlgorithm = 349 CPA_CY_SYM_HASH_SHA256; 350 sessionSetupData.hashSetupData.hashMode = 351 CPA_CY_SYM_HASH_MODE_PLAIN; 352 break; 353 354 case CRYPTO_SHA2_224_HMAC: 355 sessionSetupData.hashSetupData.hashAlgorithm = 356 CPA_CY_SYM_HASH_SHA224; 357 sessionSetupData.hashSetupData.hashMode = 358 CPA_CY_SYM_HASH_MODE_AUTH; 359 break; 360 case CRYPTO_SHA2_224: 361 sessionSetupData.hashSetupData.hashAlgorithm = 362 CPA_CY_SYM_HASH_SHA224; 363 sessionSetupData.hashSetupData.hashMode = 364 CPA_CY_SYM_HASH_MODE_PLAIN; 365 break; 366 367 case CRYPTO_SHA2_384_HMAC: 368 sessionSetupData.hashSetupData.hashAlgorithm = 369 CPA_CY_SYM_HASH_SHA384; 370 sessionSetupData.hashSetupData.hashMode = 371 CPA_CY_SYM_HASH_MODE_AUTH; 372 break; 373 case CRYPTO_SHA2_384: 374 sessionSetupData.hashSetupData.hashAlgorithm = 375 CPA_CY_SYM_HASH_SHA384; 376 sessionSetupData.hashSetupData.hashMode = 377 CPA_CY_SYM_HASH_MODE_PLAIN; 378 break; 379 380 case CRYPTO_SHA2_512_HMAC: 381 sessionSetupData.hashSetupData.hashAlgorithm = 382 CPA_CY_SYM_HASH_SHA512; 383 sessionSetupData.hashSetupData.hashMode = 384 CPA_CY_SYM_HASH_MODE_AUTH; 385 break; 386 case CRYPTO_SHA2_512: 387 sessionSetupData.hashSetupData.hashAlgorithm = 388 CPA_CY_SYM_HASH_SHA512; 389 sessionSetupData.hashSetupData.hashMode = 390 CPA_CY_SYM_HASH_MODE_PLAIN; 391 break; 392 case CRYPTO_AES_NIST_GMAC: 393 sessionSetupData.hashSetupData.hashAlgorithm = 394 CPA_CY_SYM_HASH_AES_GMAC; 395 break; 396 default: 397 status = CPA_STATUS_UNSUPPORTED; 398 goto fail; 399 } 400 } /* csp->csp_auth_alg */ 401 402 /* Setting digest-length if no cipher-only mode is set */ 403 if (csp->csp_mode != CSP_MODE_CIPHER) { 404 lac_sym_qat_hash_defs_t *pHashDefsInfo = NULL; 405 if (csp->csp_auth_mlen) { 406 sessionSetupData.hashSetupData.digestResultLenInBytes = 407 csp->csp_auth_mlen; 408 qat_ssession->authLen = csp->csp_auth_mlen; 409 } else { 410 LacSymQat_HashDefsLookupGet( 411 qat_instance->cyInstHandle, 412 sessionSetupData.hashSetupData.hashAlgorithm, 413 &pHashDefsInfo); 414 if (NULL == pHashDefsInfo) { 415 device_printf( 416 dev, 417 "unable to find corresponding hash data\n"); 418 status = CPA_STATUS_UNSUPPORTED; 419 goto fail; 420 } 421 sessionSetupData.hashSetupData.digestResultLenInBytes = 422 pHashDefsInfo->algInfo->digestLength; 423 qat_ssession->authLen = 424 pHashDefsInfo->algInfo->digestLength; 425 } 426 sessionSetupData.verifyDigest = CPA_FALSE; 427 } 428 429 switch (csp->csp_mode) { 430 case CSP_MODE_AEAD: 431 case CSP_MODE_ETA: 432 sessionSetupData.symOperation = 433 CPA_CY_SYM_OP_ALGORITHM_CHAINING; 434 /* Place the digest result in a buffer unrelated to srcBuffer */ 435 sessionSetupData.digestIsAppended = CPA_FALSE; 436 /* Due to FW limitation to verify only appended MACs */ 437 sessionSetupData.verifyDigest = CPA_FALSE; 438 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 439 sessionSetupData.cipherSetupData.cipherDirection = 440 CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT; 441 sessionSetupData.algChainOrder = 442 CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH; 443 } else { 444 sessionSetupData.cipherSetupData.cipherDirection = 445 CPA_CY_SYM_CIPHER_DIRECTION_DECRYPT; 446 sessionSetupData.algChainOrder = 447 CPA_CY_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER; 448 } 449 break; 450 case CSP_MODE_CIPHER: 451 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) { 452 sessionSetupData.cipherSetupData.cipherDirection = 453 CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT; 454 } else { 455 sessionSetupData.cipherSetupData.cipherDirection = 456 CPA_CY_SYM_CIPHER_DIRECTION_DECRYPT; 457 } 458 sessionSetupData.symOperation = CPA_CY_SYM_OP_CIPHER; 459 break; 460 case CSP_MODE_DIGEST: 461 sessionSetupData.symOperation = CPA_CY_SYM_OP_HASH; 462 if (csp->csp_auth_alg == CRYPTO_AES_NIST_GMAC) { 463 sessionSetupData.symOperation = 464 CPA_CY_SYM_OP_ALGORITHM_CHAINING; 465 /* GMAC is always encrypt */ 466 sessionSetupData.cipherSetupData.cipherDirection = 467 CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT; 468 sessionSetupData.algChainOrder = 469 CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH; 470 sessionSetupData.cipherSetupData.cipherAlgorithm = 471 CPA_CY_SYM_CIPHER_AES_GCM; 472 sessionSetupData.hashSetupData.hashAlgorithm = 473 CPA_CY_SYM_HASH_AES_GMAC; 474 sessionSetupData.hashSetupData.hashMode = 475 CPA_CY_SYM_HASH_MODE_AUTH; 476 /* Same key for cipher and auth */ 477 sessionSetupData.cipherSetupData.pCipherKey = 478 csp->csp_auth_key; 479 sessionSetupData.cipherSetupData.cipherKeyLenInBytes = 480 csp->csp_auth_klen; 481 /* Generated GMAC stored in separated buffer */ 482 sessionSetupData.digestIsAppended = CPA_FALSE; 483 /* Digest verification not allowed in GMAC case */ 484 sessionSetupData.verifyDigest = CPA_FALSE; 485 /* No AAD allowed */ 486 sessionSetupData.hashSetupData.authModeSetupData 487 .aadLenInBytes = 0; 488 } else { 489 sessionSetupData.cipherSetupData.cipherDirection = 490 CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT; 491 sessionSetupData.symOperation = CPA_CY_SYM_OP_HASH; 492 sessionSetupData.digestIsAppended = CPA_FALSE; 493 } 494 break; 495 default: 496 device_printf(dev, 497 "%s: unhandled crypto algorithm %d, %d\n", 498 __func__, 499 csp->csp_cipher_alg, 500 csp->csp_auth_alg); 501 status = CPA_STATUS_FAIL; 502 goto fail; 503 } 504 505 /* Extracting session size */ 506 status = cpaCySymSessionCtxGetSize(qat_instance->cyInstHandle, 507 &sessionSetupData, 508 &sessionCtxSize); 509 if (CPA_STATUS_SUCCESS != status) { 510 device_printf(dev, "unable to get session size\n"); 511 goto fail; 512 } 513 514 /* Allocating contiguous memory for session */ 515 sessionCtx = contigmalloc(sessionCtxSize, 516 M_QAT_OCF, 517 M_NOWAIT, 518 0, 519 ~1UL, 520 1 << (ilog2(sessionCtxSize - 1) + 1), 521 0); 522 if (NULL == sessionCtx) { 523 device_printf(dev, "unable to allocate memory for session\n"); 524 status = CPA_STATUS_RESOURCE; 525 goto fail; 526 } 527 528 status = cpaCySymDpInitSession(qat_instance->cyInstHandle, 529 &sessionSetupData, 530 sessionCtx); 531 if (CPA_STATUS_SUCCESS != status) { 532 device_printf(dev, "session initialization failed\n"); 533 goto fail; 534 } 535 536 /* NOTE: lets keep double session (both directions) approach to overcome 537 * lack of direction update in FBSD QAT. 538 */ 539 qat_ssession->sessionCtx = sessionCtx; 540 qat_ssession->sessionCtxSize = sessionCtxSize; 541 542 return CPA_STATUS_SUCCESS; 543 544 fail: 545 /* Release resources if any */ 546 if (sessionCtx) 547 free(sessionCtx, M_QAT_OCF); 548 549 return status; 550 } 551 552 static int 553 qat_ocf_newsession(device_t dev, 554 crypto_session_t cses, 555 const struct crypto_session_params *csp) 556 { 557 /* Cryptodev QAT structures */ 558 struct qat_ocf_softc *qat_softc; 559 struct qat_ocf_dsession *qat_dsession; 560 struct qat_ocf_instance *qat_instance; 561 u_int cpu_id = PCPU_GET(cpuid); 562 563 /* Create cryptodev session */ 564 qat_softc = device_get_softc(dev); 565 if (qat_softc->numCyInstances > 0) { 566 qat_instance = 567 &qat_softc 568 ->cyInstHandles[cpu_id % qat_softc->numCyInstances]; 569 qat_dsession = crypto_get_driver_session(cses); 570 if (NULL == qat_dsession) { 571 device_printf(dev, "Unable to create new session\n"); 572 return (EINVAL); 573 } 574 575 /* Add only instance at this point remaining operations moved to 576 * lazy session init */ 577 qat_dsession->qatInstance = qat_instance; 578 } else { 579 return ENXIO; 580 } 581 582 return 0; 583 } 584 585 static CpaStatus 586 qat_ocf_remove_session(device_t dev, 587 CpaInstanceHandle cyInstHandle, 588 struct qat_ocf_session *qat_session) 589 { 590 CpaStatus status = CPA_STATUS_SUCCESS; 591 592 if (NULL == qat_session->sessionCtx) 593 return CPA_STATUS_SUCCESS; 594 595 /* User callback is executed right before decrementing pending 596 * callback atomic counter. To avoid removing session rejection 597 * we have to wait a very short while for counter update 598 * after call back execution. */ 599 status = qat_ocf_wait_for_session(qat_session->sessionCtx, 600 QAT_OCF_SESSION_WAIT_TIMEOUT_MS); 601 if (CPA_STATUS_SUCCESS != status) { 602 device_printf(dev, "waiting for session un-busy failed\n"); 603 return CPA_STATUS_FAIL; 604 } 605 606 status = cpaCySymDpRemoveSession(cyInstHandle, qat_session->sessionCtx); 607 if (CPA_STATUS_SUCCESS != status) { 608 device_printf(dev, "error while removing session\n"); 609 return CPA_STATUS_FAIL; 610 } 611 612 explicit_bzero(qat_session->sessionCtx, qat_session->sessionCtxSize); 613 free(qat_session->sessionCtx, M_QAT_OCF); 614 qat_session->sessionCtx = NULL; 615 qat_session->sessionCtxSize = 0; 616 617 return CPA_STATUS_SUCCESS; 618 } 619 620 static void 621 qat_ocf_freesession(device_t dev, crypto_session_t cses) 622 { 623 CpaStatus status = CPA_STATUS_SUCCESS; 624 struct qat_ocf_dsession *qat_dsession = NULL; 625 struct qat_ocf_instance *qat_instance = NULL; 626 627 qat_dsession = crypto_get_driver_session(cses); 628 qat_instance = qat_dsession->qatInstance; 629 mtx_lock(&qat_instance->cyInstMtx); 630 status = qat_ocf_remove_session(dev, 631 qat_dsession->qatInstance->cyInstHandle, 632 &qat_dsession->encSession); 633 if (CPA_STATUS_SUCCESS != status) 634 device_printf(dev, "unable to remove encrypt session\n"); 635 status = qat_ocf_remove_session(dev, 636 qat_dsession->qatInstance->cyInstHandle, 637 &qat_dsession->decSession); 638 if (CPA_STATUS_SUCCESS != status) 639 device_printf(dev, "unable to remove decrypt session\n"); 640 mtx_unlock(&qat_instance->cyInstMtx); 641 } 642 643 /* QAT GCM/CCM FW API are only algorithms which support separated AAD. */ 644 static CpaStatus 645 qat_ocf_load_aad_gcm(struct cryptop *crp, struct qat_ocf_cookie *qat_cookie) 646 { 647 CpaCySymDpOpData *pOpData; 648 649 pOpData = &qat_cookie->pOpdata; 650 651 if (NULL != crp->crp_aad) 652 memcpy(qat_cookie->qat_ocf_gcm_aad, 653 crp->crp_aad, 654 crp->crp_aad_length); 655 else 656 crypto_copydata(crp, 657 crp->crp_aad_start, 658 crp->crp_aad_length, 659 qat_cookie->qat_ocf_gcm_aad); 660 661 pOpData->pAdditionalAuthData = qat_cookie->qat_ocf_gcm_aad; 662 pOpData->additionalAuthData = qat_cookie->qat_ocf_gcm_aad_paddr; 663 664 return CPA_STATUS_SUCCESS; 665 } 666 667 static CpaStatus 668 qat_ocf_load_aad(struct cryptop *crp, struct qat_ocf_cookie *qat_cookie) 669 { 670 CpaStatus status = CPA_STATUS_SUCCESS; 671 const struct crypto_session_params *csp; 672 CpaCySymDpOpData *pOpData; 673 struct qat_ocf_load_cb_arg args; 674 675 pOpData = &qat_cookie->pOpdata; 676 pOpData->pAdditionalAuthData = NULL; 677 pOpData->additionalAuthData = 0UL; 678 679 if (crp->crp_aad_length == 0) 680 return CPA_STATUS_SUCCESS; 681 682 if (crp->crp_aad_length > ICP_QAT_FW_CCM_GCM_AAD_SZ_MAX) 683 return CPA_STATUS_FAIL; 684 685 csp = crypto_get_params(crp->crp_session); 686 687 /* Handle GCM/CCM case */ 688 if (CPA_TRUE == is_sep_aad_supported(csp)) 689 return qat_ocf_load_aad_gcm(crp, qat_cookie); 690 691 if (NULL == crp->crp_aad) { 692 /* AAD already embedded in source buffer */ 693 pOpData->messageLenToCipherInBytes = crp->crp_payload_length; 694 pOpData->cryptoStartSrcOffsetInBytes = crp->crp_payload_start; 695 696 pOpData->messageLenToHashInBytes = 697 crp->crp_aad_length + crp->crp_payload_length; 698 pOpData->hashStartSrcOffsetInBytes = crp->crp_aad_start; 699 700 return CPA_STATUS_SUCCESS; 701 } 702 703 /* Separated AAD not supported by QAT - lets place the content 704 * of ADD buffer at the very beginning of source SGL */ 705 args.crp_op = crp; 706 args.qat_cookie = qat_cookie; 707 args.pOpData = pOpData; 708 args.error = 0; 709 status = bus_dmamap_load(qat_cookie->gcm_aad_dma_mem.dma_tag, 710 qat_cookie->gcm_aad_dma_mem.dma_map, 711 crp->crp_aad, 712 crp->crp_aad_length, 713 qat_ocf_crypto_load_aadbuf_cb, 714 &args, 715 BUS_DMA_NOWAIT); 716 qat_cookie->is_sep_aad_used = CPA_TRUE; 717 718 /* Right after this step we have AAD placed in the first flat buffer 719 * in source SGL */ 720 pOpData->messageLenToCipherInBytes = crp->crp_payload_length; 721 pOpData->cryptoStartSrcOffsetInBytes = 722 crp->crp_aad_length + crp->crp_aad_start + crp->crp_payload_start; 723 724 pOpData->messageLenToHashInBytes = 725 crp->crp_aad_length + crp->crp_payload_length; 726 pOpData->hashStartSrcOffsetInBytes = crp->crp_aad_start; 727 728 return status; 729 } 730 731 static CpaStatus 732 qat_ocf_load(struct cryptop *crp, struct qat_ocf_cookie *qat_cookie) 733 { 734 CpaStatus status = CPA_STATUS_SUCCESS; 735 CpaCySymDpOpData *pOpData; 736 struct qat_ocf_load_cb_arg args; 737 /* cryptodev internals */ 738 const struct crypto_session_params *csp; 739 740 pOpData = &qat_cookie->pOpdata; 741 742 csp = crypto_get_params(crp->crp_session); 743 744 /* Load IV buffer if present */ 745 if (csp->csp_ivlen > 0) { 746 memset(qat_cookie->qat_ocf_iv_buf, 747 0, 748 sizeof(qat_cookie->qat_ocf_iv_buf)); 749 crypto_read_iv(crp, qat_cookie->qat_ocf_iv_buf); 750 pOpData->iv = qat_cookie->qat_ocf_iv_buf_paddr; 751 pOpData->pIv = qat_cookie->qat_ocf_iv_buf; 752 pOpData->ivLenInBytes = csp->csp_ivlen; 753 } 754 755 /* GCM/CCM - load AAD to separated buffer 756 * AES+SHA - load AAD to first flat in SGL */ 757 status = qat_ocf_load_aad(crp, qat_cookie); 758 if (CPA_STATUS_SUCCESS != status) 759 goto fail; 760 761 /* Load source buffer */ 762 args.crp_op = crp; 763 args.qat_cookie = qat_cookie; 764 args.pOpData = pOpData; 765 args.error = 0; 766 status = bus_dmamap_load_crp_buffer(qat_cookie->src_dma_mem.dma_tag, 767 qat_cookie->src_dma_mem.dma_map, 768 &crp->crp_buf, 769 qat_ocf_crypto_load_buf_cb, 770 &args, 771 BUS_DMA_NOWAIT); 772 if (CPA_STATUS_SUCCESS != status) 773 goto fail; 774 pOpData->srcBuffer = qat_cookie->src_buffer_list_paddr; 775 pOpData->srcBufferLen = CPA_DP_BUFLIST; 776 777 /* Load destination buffer */ 778 if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { 779 status = 780 bus_dmamap_load_crp_buffer(qat_cookie->dst_dma_mem.dma_tag, 781 qat_cookie->dst_dma_mem.dma_map, 782 &crp->crp_obuf, 783 qat_ocf_crypto_load_obuf_cb, 784 &args, 785 BUS_DMA_NOWAIT); 786 if (CPA_STATUS_SUCCESS != status) 787 goto fail; 788 pOpData->dstBuffer = qat_cookie->dst_buffer_list_paddr; 789 pOpData->dstBufferLen = CPA_DP_BUFLIST; 790 } else { 791 pOpData->dstBuffer = pOpData->srcBuffer; 792 pOpData->dstBufferLen = pOpData->srcBufferLen; 793 } 794 795 if (CPA_TRUE == is_use_sep_digest(csp)) 796 pOpData->digestResult = qat_cookie->qat_ocf_digest_paddr; 797 else 798 pOpData->digestResult = 0UL; 799 800 /* GMAC - aka zero length buffer */ 801 if (CPA_TRUE == is_gmac_exception(csp)) 802 pOpData->messageLenToCipherInBytes = 0; 803 804 fail: 805 return status; 806 } 807 808 static int 809 qat_ocf_check_input(device_t dev, struct cryptop *crp) 810 { 811 const struct crypto_session_params *csp; 812 csp = crypto_get_params(crp->crp_session); 813 814 if (crypto_buffer_len(&crp->crp_buf) > QAT_OCF_MAX_LEN) 815 return E2BIG; 816 817 if (CPA_TRUE == is_sep_aad_supported(csp) && 818 (crp->crp_aad_length > ICP_QAT_FW_CCM_GCM_AAD_SZ_MAX)) 819 return EBADMSG; 820 821 return 0; 822 } 823 824 static int 825 qat_ocf_process(device_t dev, struct cryptop *crp, int hint) 826 { 827 CpaStatus status = CPA_STATUS_SUCCESS; 828 int rc = 0; 829 struct qat_ocf_dsession *qat_dsession = NULL; 830 struct qat_ocf_session *qat_session = NULL; 831 struct qat_ocf_instance *qat_instance = NULL; 832 CpaCySymDpOpData *pOpData = NULL; 833 struct qat_ocf_cookie *qat_cookie = NULL; 834 CpaBoolean memLoaded = CPA_FALSE; 835 836 rc = qat_ocf_check_input(dev, crp); 837 if (rc) 838 goto fail; 839 840 qat_dsession = crypto_get_driver_session(crp->crp_session); 841 842 if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) 843 qat_session = &qat_dsession->encSession; 844 else 845 qat_session = &qat_dsession->decSession; 846 qat_instance = qat_dsession->qatInstance; 847 848 status = qat_ocf_cookie_alloc(qat_instance, &qat_cookie); 849 if (CPA_STATUS_SUCCESS != status) { 850 rc = EAGAIN; 851 goto fail; 852 } 853 854 qat_cookie->crp_op = crp; 855 856 /* Common request fields */ 857 pOpData = &qat_cookie->pOpdata; 858 pOpData->instanceHandle = qat_instance->cyInstHandle; 859 pOpData->sessionCtx = NULL; 860 861 /* Cipher fields */ 862 pOpData->cryptoStartSrcOffsetInBytes = crp->crp_payload_start; 863 pOpData->messageLenToCipherInBytes = crp->crp_payload_length; 864 /* Digest fields - any exceptions from this basic rules are covered 865 * in qat_ocf_load */ 866 pOpData->hashStartSrcOffsetInBytes = crp->crp_payload_start; 867 pOpData->messageLenToHashInBytes = crp->crp_payload_length; 868 869 status = qat_ocf_load(crp, qat_cookie); 870 if (CPA_STATUS_SUCCESS != status) { 871 device_printf(dev, 872 "unable to load OCF buffers to QAT DMA " 873 "transaction\n"); 874 rc = EIO; 875 goto fail; 876 } 877 memLoaded = CPA_TRUE; 878 879 status = qat_ocf_cookie_dma_pre_sync(crp, pOpData); 880 if (CPA_STATUS_SUCCESS != status) { 881 device_printf(dev, "unable to sync DMA buffers\n"); 882 rc = EIO; 883 goto fail; 884 } 885 886 mtx_lock(&qat_instance->cyInstMtx); 887 /* Session initialization at the first request. It's done 888 * in such way to overcome missing QAT specific session data 889 * such like AAD length and limited possibility to update 890 * QAT session while handling traffic. 891 */ 892 if (NULL == qat_session->sessionCtx) { 893 status = 894 qat_ocf_session_init(dev, crp, qat_instance, qat_session); 895 if (CPA_STATUS_SUCCESS != status) { 896 mtx_unlock(&qat_instance->cyInstMtx); 897 device_printf(dev, "unable to init session\n"); 898 rc = EIO; 899 goto fail; 900 } 901 } else { 902 status = qat_ocf_handle_session_update(qat_dsession, crp); 903 if (CPA_STATUS_RESOURCE == status) { 904 mtx_unlock(&qat_instance->cyInstMtx); 905 rc = EAGAIN; 906 goto fail; 907 } else if (CPA_STATUS_SUCCESS != status) { 908 mtx_unlock(&qat_instance->cyInstMtx); 909 rc = EIO; 910 goto fail; 911 } 912 } 913 pOpData->sessionCtx = qat_session->sessionCtx; 914 status = cpaCySymDpEnqueueOp(pOpData, CPA_TRUE); 915 mtx_unlock(&qat_instance->cyInstMtx); 916 if (CPA_STATUS_SUCCESS != status) { 917 if (CPA_STATUS_RETRY == status) { 918 rc = EAGAIN; 919 goto fail; 920 } 921 device_printf(dev, 922 "unable to send request. Status: %d\n", 923 status); 924 rc = EIO; 925 goto fail; 926 } 927 928 return 0; 929 fail: 930 if (qat_cookie) { 931 if (memLoaded) 932 qat_ocf_cookie_dma_unload(crp, pOpData); 933 qat_ocf_cookie_free(qat_instance, qat_cookie); 934 } 935 crp->crp_etype = rc; 936 crypto_done(crp); 937 938 return 0; 939 } 940 941 static void 942 qat_ocf_identify(driver_t *drv, device_t parent) 943 { 944 if (device_find_child(parent, "qat_ocf", -1) == NULL && 945 BUS_ADD_CHILD(parent, 200, "qat_ocf", -1) == 0) 946 device_printf(parent, "qat_ocf: could not attach!"); 947 } 948 949 static int 950 qat_ocf_probe(device_t dev) 951 { 952 device_set_desc(dev, "QAT engine"); 953 return (BUS_PROBE_NOWILDCARD); 954 } 955 956 static CpaStatus 957 qat_ocf_get_irq_instances(CpaInstanceHandle *cyInstHandles, 958 Cpa16U cyInstHandlesSize, 959 Cpa16U *foundInstances) 960 { 961 CpaStatus status = CPA_STATUS_SUCCESS; 962 icp_accel_dev_t **pAdfInsts = NULL; 963 icp_accel_dev_t *dev_addr = NULL; 964 sal_t *baseAddr = NULL; 965 sal_list_t *listTemp = NULL; 966 CpaInstanceHandle cyInstHandle; 967 CpaInstanceInfo2 info; 968 Cpa16U numDevices; 969 Cpa32U instCtr = 0; 970 Cpa32U i; 971 972 /* Get the number of devices */ 973 status = icp_amgr_getNumInstances(&numDevices); 974 if (CPA_STATUS_SUCCESS != status) 975 return status; 976 977 /* Allocate memory to store addr of accel_devs */ 978 pAdfInsts = 979 malloc(numDevices * sizeof(icp_accel_dev_t *), M_QAT_OCF, M_WAITOK); 980 981 /* Get ADF to return all accel_devs that support either 982 * symmetric or asymmetric crypto */ 983 status = icp_amgr_getAllAccelDevByCapabilities( 984 (ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC), pAdfInsts, &numDevices); 985 if (CPA_STATUS_SUCCESS != status) { 986 free(pAdfInsts, M_QAT_OCF); 987 return status; 988 } 989 990 for (i = 0; i < numDevices; i++) { 991 dev_addr = (icp_accel_dev_t *)pAdfInsts[i]; 992 baseAddr = dev_addr->pSalHandle; 993 if (NULL == baseAddr) 994 continue; 995 listTemp = baseAddr->sym_services; 996 if (NULL == listTemp) { 997 listTemp = baseAddr->crypto_services; 998 } 999 1000 while (NULL != listTemp) { 1001 cyInstHandle = SalList_getObject(listTemp); 1002 status = cpaCyInstanceGetInfo2(cyInstHandle, &info); 1003 if (CPA_STATUS_SUCCESS != status) 1004 continue; 1005 listTemp = SalList_next(listTemp); 1006 if (CPA_TRUE == info.isPolled) 1007 continue; 1008 if (instCtr >= cyInstHandlesSize) 1009 break; 1010 cyInstHandles[instCtr++] = cyInstHandle; 1011 } 1012 } 1013 free(pAdfInsts, M_QAT_OCF); 1014 *foundInstances = instCtr; 1015 1016 return CPA_STATUS_SUCCESS; 1017 } 1018 1019 static CpaStatus 1020 qat_ocf_start_instances(struct qat_ocf_softc *qat_softc, device_t dev) 1021 { 1022 CpaStatus status = CPA_STATUS_SUCCESS; 1023 Cpa16U numInstances = 0; 1024 CpaInstanceHandle cyInstHandles[QAT_OCF_MAX_INSTANCES] = { 0 }; 1025 CpaInstanceHandle cyInstHandle = NULL; 1026 Cpa32U startedInstances = 0; 1027 Cpa32U i; 1028 1029 qat_softc->numCyInstances = 0; 1030 status = qat_ocf_get_irq_instances(cyInstHandles, 1031 QAT_OCF_MAX_INSTANCES, 1032 &numInstances); 1033 if (CPA_STATUS_SUCCESS != status) 1034 return status; 1035 1036 for (i = 0; i < numInstances; i++) { 1037 struct qat_ocf_instance *qat_ocf_instance; 1038 1039 cyInstHandle = cyInstHandles[i]; 1040 if (!cyInstHandle) 1041 continue; 1042 1043 /* Starting instance */ 1044 status = cpaCyStartInstance(cyInstHandle); 1045 if (CPA_STATUS_SUCCESS != status) { 1046 device_printf(qat_softc->sc_dev, 1047 "unable to get start instance\n"); 1048 continue; 1049 } 1050 1051 qat_ocf_instance = &qat_softc->cyInstHandles[startedInstances]; 1052 qat_ocf_instance->cyInstHandle = cyInstHandle; 1053 mtx_init(&qat_ocf_instance->cyInstMtx, 1054 "Instance MTX", 1055 NULL, 1056 MTX_DEF); 1057 1058 status = 1059 cpaCySetAddressTranslation(cyInstHandle, qatVirtToPhys); 1060 if (CPA_STATUS_SUCCESS != status) { 1061 device_printf(qat_softc->sc_dev, 1062 "unable to add virt to phys callback\n"); 1063 goto fail; 1064 } 1065 1066 status = cpaCySymDpRegCbFunc(cyInstHandle, symDpCallback); 1067 if (CPA_STATUS_SUCCESS != status) { 1068 device_printf(qat_softc->sc_dev, 1069 "unable to add user callback\n"); 1070 goto fail; 1071 } 1072 1073 /* Initialize cookie pool */ 1074 status = qat_ocf_cookie_pool_init(qat_ocf_instance, dev); 1075 if (CPA_STATUS_SUCCESS != status) { 1076 device_printf(qat_softc->sc_dev, 1077 "unable to create cookie pool\n"); 1078 goto fail; 1079 } 1080 1081 /* Disable forcing HW MAC validation for AEAD */ 1082 status = icp_sal_setForceAEADMACVerify(cyInstHandle, CPA_FALSE); 1083 if (CPA_STATUS_SUCCESS != status) { 1084 device_printf( 1085 qat_softc->sc_dev, 1086 "unable to disable AEAD HW MAC verification\n"); 1087 goto fail; 1088 } 1089 1090 qat_ocf_instance->driver_id = qat_softc->cryptodev_id; 1091 1092 startedInstances++; 1093 continue; 1094 fail: 1095 mtx_destroy(&qat_ocf_instance->cyInstMtx); 1096 1097 /* Stop instance */ 1098 status = cpaCyStopInstance(cyInstHandle); 1099 if (CPA_STATUS_SUCCESS != status) 1100 device_printf(qat_softc->sc_dev, 1101 "unable to stop the instance\n"); 1102 } 1103 qat_softc->numCyInstances = startedInstances; 1104 1105 return CPA_STATUS_SUCCESS; 1106 } 1107 1108 static CpaStatus 1109 qat_ocf_stop_instances(struct qat_ocf_softc *qat_softc) 1110 { 1111 CpaStatus status = CPA_STATUS_SUCCESS; 1112 int i; 1113 1114 for (i = 0; i < qat_softc->numCyInstances; i++) { 1115 struct qat_ocf_instance *qat_instance; 1116 1117 qat_instance = &qat_softc->cyInstHandles[i]; 1118 status = cpaCyStopInstance(qat_instance->cyInstHandle); 1119 if (CPA_STATUS_SUCCESS != status) { 1120 pr_err("QAT: stopping instance id: %d failed\n", i); 1121 continue; 1122 } 1123 qat_ocf_cookie_pool_deinit(qat_instance); 1124 mtx_destroy(&qat_instance->cyInstMtx); 1125 } 1126 1127 qat_softc->numCyInstances = 0; 1128 1129 return status; 1130 } 1131 1132 static int 1133 qat_ocf_deinit(struct qat_ocf_softc *qat_softc) 1134 { 1135 int status = 0; 1136 CpaStatus cpaStatus; 1137 1138 if (qat_softc->cryptodev_id >= 0) { 1139 crypto_unregister_all(qat_softc->cryptodev_id); 1140 qat_softc->cryptodev_id = -1; 1141 } 1142 1143 /* Stop QAT instances */ 1144 cpaStatus = qat_ocf_stop_instances(qat_softc); 1145 if (CPA_STATUS_SUCCESS != cpaStatus) { 1146 device_printf(qat_softc->sc_dev, "unable to stop instances\n"); 1147 status = EIO; 1148 } 1149 1150 return status; 1151 } 1152 1153 static int 1154 qat_ocf_init(struct qat_ocf_softc *qat_softc) 1155 { 1156 int32_t cryptodev_id; 1157 1158 /* Starting instances for OCF */ 1159 if (qat_ocf_start_instances(qat_softc, qat_softc->sc_dev)) { 1160 device_printf(qat_softc->sc_dev, 1161 "unable to get QAT IRQ instances\n"); 1162 goto fail; 1163 } 1164 1165 /* Register only if instances available */ 1166 if (qat_softc->numCyInstances) { 1167 cryptodev_id = 1168 crypto_get_driverid(qat_softc->sc_dev, 1169 sizeof(struct qat_ocf_dsession), 1170 CRYPTOCAP_F_HARDWARE); 1171 if (cryptodev_id < 0) { 1172 device_printf(qat_softc->sc_dev, 1173 "cannot initialize!\n"); 1174 goto fail; 1175 } 1176 qat_softc->cryptodev_id = cryptodev_id; 1177 } 1178 1179 return 0; 1180 fail: 1181 qat_ocf_deinit(qat_softc); 1182 1183 return ENXIO; 1184 } 1185 1186 static int qat_ocf_sysctl_handle(SYSCTL_HANDLER_ARGS) 1187 { 1188 struct qat_ocf_softc *qat_softc = NULL; 1189 int ret = 0; 1190 device_t dev = arg1; 1191 u_int enabled; 1192 1193 qat_softc = device_get_softc(dev); 1194 enabled = qat_softc->enabled; 1195 1196 ret = sysctl_handle_int(oidp, &enabled, 0, req); 1197 if (ret || !req->newptr) 1198 return (ret); 1199 1200 if (qat_softc->enabled != enabled) { 1201 if (enabled) { 1202 ret = qat_ocf_init(qat_softc); 1203 1204 } else { 1205 ret = qat_ocf_deinit(qat_softc); 1206 } 1207 1208 if (!ret) 1209 qat_softc->enabled = enabled; 1210 } 1211 1212 return ret; 1213 } 1214 1215 static int 1216 qat_ocf_attach(device_t dev) 1217 { 1218 int status; 1219 struct qat_ocf_softc *qat_softc; 1220 1221 qat_softc = device_get_softc(dev); 1222 qat_softc->sc_dev = dev; 1223 qat_softc->cryptodev_id = -1; 1224 qat_softc->enabled = 1; 1225 1226 qat_softc->rc = 1227 SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev), 1228 SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), 1229 OID_AUTO, 1230 "enable", 1231 CTLTYPE_INT | CTLFLAG_RWTUN | CTLFLAG_MPSAFE, 1232 dev, 1233 0, 1234 qat_ocf_sysctl_handle, 1235 "I", 1236 "QAT OCF support enablement"); 1237 1238 if (!qat_softc->rc) 1239 return ENOMEM; 1240 if (qat_softc->enabled) { 1241 status = qat_ocf_init(qat_softc); 1242 if (status) { 1243 device_printf(dev, "qat_ocf init failed\n"); 1244 goto fail; 1245 } 1246 } 1247 1248 return 0; 1249 fail: 1250 qat_ocf_deinit(qat_softc); 1251 1252 return (ENXIO); 1253 } 1254 1255 static int 1256 qat_ocf_detach(device_t dev) 1257 { 1258 struct qat_ocf_softc *qat_softc = device_get_softc(dev); 1259 1260 return qat_ocf_deinit(qat_softc); 1261 } 1262 1263 static device_method_t qat_ocf_methods[] = 1264 { DEVMETHOD(device_identify, qat_ocf_identify), 1265 DEVMETHOD(device_probe, qat_ocf_probe), 1266 DEVMETHOD(device_attach, qat_ocf_attach), 1267 DEVMETHOD(device_detach, qat_ocf_detach), 1268 1269 /* Cryptodev interface */ 1270 DEVMETHOD(cryptodev_probesession, qat_ocf_probesession), 1271 DEVMETHOD(cryptodev_newsession, qat_ocf_newsession), 1272 DEVMETHOD(cryptodev_freesession, qat_ocf_freesession), 1273 DEVMETHOD(cryptodev_process, qat_ocf_process), 1274 1275 DEVMETHOD_END }; 1276 1277 static driver_t qat_ocf_driver = { 1278 .name = "qat_ocf", 1279 .methods = qat_ocf_methods, 1280 .size = sizeof(struct qat_ocf_softc), 1281 }; 1282 1283 1284 DRIVER_MODULE_ORDERED(qat, 1285 nexus, 1286 qat_ocf_driver, 1287 NULL, 1288 NULL, 1289 SI_ORDER_ANY); 1290 MODULE_VERSION(qat, 1); 1291 MODULE_DEPEND(qat, qat_c62x, 1, 1, 1); 1292 MODULE_DEPEND(qat, qat_200xx, 1, 1, 1); 1293 MODULE_DEPEND(qat, qat_c3xxx, 1, 1, 1); 1294 MODULE_DEPEND(qat, qat_c4xxx, 1, 1, 1); 1295 MODULE_DEPEND(qat, qat_dh895xcc, 1, 1, 1); 1296 MODULE_DEPEND(qat, qat_4xxx, 1, 1, 1); 1297 MODULE_DEPEND(qat, crypto, 1, 1, 1); 1298 MODULE_DEPEND(qat, qat_common, 1, 1, 1); 1299 MODULE_DEPEND(qat, qat_api, 1, 1, 1); 1300 MODULE_DEPEND(qat, linuxkpi, 1, 1, 1); 1301