1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2015-2019 Intel Corporation 3 */ 4 5 #include <openssl/evp.h> 6 7 #include <rte_mempool.h> 8 #include <rte_mbuf.h> 9 #include <rte_crypto_sym.h> 10 #include <rte_bus_pci.h> 11 #include <rte_byteorder.h> 12 13 #include "qat_sym.h" 14 15 16 /** Decrypt a single partial block 17 * Depends on openssl libcrypto 18 * Uses ECB+XOR to do CFB encryption, same result, more performant 19 */ 20 static inline int 21 bpi_cipher_decrypt(uint8_t *src, uint8_t *dst, 22 uint8_t *iv, int ivlen, int srclen, 23 void *bpi_ctx) 24 { 25 EVP_CIPHER_CTX *ctx = (EVP_CIPHER_CTX *)bpi_ctx; 26 int encrypted_ivlen; 27 uint8_t encrypted_iv[BPI_MAX_ENCR_IV_LEN]; 28 uint8_t *encr = encrypted_iv; 29 30 /* ECB method: encrypt (not decrypt!) the IV, then XOR with plaintext */ 31 if (EVP_EncryptUpdate(ctx, encrypted_iv, &encrypted_ivlen, iv, ivlen) 32 <= 0) 33 goto cipher_decrypt_err; 34 35 for (; srclen != 0; --srclen, ++dst, ++src, ++encr) 36 *dst = *src ^ *encr; 37 38 return 0; 39 40 cipher_decrypt_err: 41 QAT_DP_LOG(ERR, "libcrypto ECB cipher decrypt for BPI IV failed"); 42 return -EINVAL; 43 } 44 45 46 static inline uint32_t 47 qat_bpicipher_preprocess(struct qat_sym_session *ctx, 48 struct rte_crypto_op *op) 49 { 50 int block_len = qat_cipher_get_block_size(ctx->qat_cipher_alg); 51 struct rte_crypto_sym_op *sym_op = op->sym; 52 uint8_t last_block_len = block_len > 0 ? 53 sym_op->cipher.data.length % block_len : 0; 54 55 if (last_block_len && 56 ctx->qat_dir == ICP_QAT_HW_CIPHER_DECRYPT) { 57 58 /* Decrypt last block */ 59 uint8_t *last_block, *dst, *iv; 60 uint32_t last_block_offset = sym_op->cipher.data.offset + 61 sym_op->cipher.data.length - last_block_len; 62 last_block = (uint8_t *) rte_pktmbuf_mtod_offset(sym_op->m_src, 63 uint8_t *, last_block_offset); 64 65 if (unlikely((sym_op->m_dst != NULL) 66 && (sym_op->m_dst != sym_op->m_src))) 67 /* out-of-place operation (OOP) */ 68 dst = (uint8_t *) rte_pktmbuf_mtod_offset(sym_op->m_dst, 69 uint8_t *, last_block_offset); 70 else 71 dst = last_block; 72 73 if (last_block_len < sym_op->cipher.data.length) 74 /* use previous block ciphertext as IV */ 75 iv = last_block - block_len; 76 else 77 /* runt block, i.e. less than one full block */ 78 iv = rte_crypto_op_ctod_offset(op, uint8_t *, 79 ctx->cipher_iv.offset); 80 81 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG 82 QAT_DP_HEXDUMP_LOG(DEBUG, "BPI: src before pre-process:", 83 last_block, last_block_len); 84 if (sym_op->m_dst != NULL) 85 QAT_DP_HEXDUMP_LOG(DEBUG, "BPI:dst before pre-process:", 86 dst, last_block_len); 87 #endif 88 bpi_cipher_decrypt(last_block, dst, iv, block_len, 89 last_block_len, ctx->bpi_ctx); 90 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG 91 QAT_DP_HEXDUMP_LOG(DEBUG, "BPI: src after pre-process:", 92 last_block, last_block_len); 93 if (sym_op->m_dst != NULL) 94 QAT_DP_HEXDUMP_LOG(DEBUG, "BPI: dst after pre-process:", 95 dst, last_block_len); 96 #endif 97 } 98 99 return sym_op->cipher.data.length - last_block_len; 100 } 101 102 static inline void 103 set_cipher_iv(uint16_t iv_length, uint16_t iv_offset, 104 struct icp_qat_fw_la_cipher_req_params *cipher_param, 105 struct rte_crypto_op *op, 106 struct icp_qat_fw_la_bulk_req *qat_req) 107 { 108 /* copy IV into request if it fits */ 109 if (iv_length <= sizeof(cipher_param->u.cipher_IV_array)) { 110 rte_memcpy(cipher_param->u.cipher_IV_array, 111 rte_crypto_op_ctod_offset(op, uint8_t *, 112 iv_offset), 113 iv_length); 114 } else { 115 ICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_SET( 116 qat_req->comn_hdr.serv_specif_flags, 117 ICP_QAT_FW_CIPH_IV_64BIT_PTR); 118 cipher_param->u.s.cipher_IV_ptr = 119 rte_crypto_op_ctophys_offset(op, 120 iv_offset); 121 } 122 } 123 124 /** Set IV for CCM is special case, 0th byte is set to q-1 125 * where q is padding of nonce in 16 byte block 126 */ 127 static inline void 128 set_cipher_iv_ccm(uint16_t iv_length, uint16_t iv_offset, 129 struct icp_qat_fw_la_cipher_req_params *cipher_param, 130 struct rte_crypto_op *op, uint8_t q, uint8_t aad_len_field_sz) 131 { 132 rte_memcpy(((uint8_t *)cipher_param->u.cipher_IV_array) + 133 ICP_QAT_HW_CCM_NONCE_OFFSET, 134 rte_crypto_op_ctod_offset(op, uint8_t *, 135 iv_offset) + ICP_QAT_HW_CCM_NONCE_OFFSET, 136 iv_length); 137 *(uint8_t *)&cipher_param->u.cipher_IV_array[0] = 138 q - ICP_QAT_HW_CCM_NONCE_OFFSET; 139 140 if (aad_len_field_sz) 141 rte_memcpy(&op->sym->aead.aad.data[ICP_QAT_HW_CCM_NONCE_OFFSET], 142 rte_crypto_op_ctod_offset(op, uint8_t *, 143 iv_offset) + ICP_QAT_HW_CCM_NONCE_OFFSET, 144 iv_length); 145 } 146 147 int 148 qat_sym_build_request(void *in_op, uint8_t *out_msg, 149 void *op_cookie, enum qat_device_gen qat_dev_gen) 150 { 151 int ret = 0; 152 struct qat_sym_session *ctx = NULL; 153 struct icp_qat_fw_la_cipher_req_params *cipher_param; 154 struct icp_qat_fw_la_auth_req_params *auth_param; 155 register struct icp_qat_fw_la_bulk_req *qat_req; 156 uint8_t do_auth = 0, do_cipher = 0, do_aead = 0; 157 uint32_t cipher_len = 0, cipher_ofs = 0; 158 uint32_t auth_len = 0, auth_ofs = 0; 159 uint32_t min_ofs = 0; 160 uint64_t src_buf_start = 0, dst_buf_start = 0; 161 uint64_t auth_data_end = 0; 162 uint8_t do_sgl = 0; 163 uint8_t in_place = 1; 164 int alignment_adjustment = 0; 165 struct rte_crypto_op *op = (struct rte_crypto_op *)in_op; 166 struct qat_sym_op_cookie *cookie = 167 (struct qat_sym_op_cookie *)op_cookie; 168 169 if (unlikely(op->type != RTE_CRYPTO_OP_TYPE_SYMMETRIC)) { 170 QAT_DP_LOG(ERR, "QAT PMD only supports symmetric crypto " 171 "operation requests, op (%p) is not a " 172 "symmetric operation.", op); 173 return -EINVAL; 174 } 175 176 if (unlikely(op->sess_type == RTE_CRYPTO_OP_SESSIONLESS)) { 177 QAT_DP_LOG(ERR, "QAT PMD only supports session oriented" 178 " requests, op (%p) is sessionless.", op); 179 return -EINVAL; 180 } else if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) { 181 ctx = (struct qat_sym_session *)get_sym_session_private_data( 182 op->sym->session, qat_sym_driver_id); 183 #ifdef RTE_LIB_SECURITY 184 } else { 185 ctx = (struct qat_sym_session *)get_sec_session_private_data( 186 op->sym->sec_session); 187 if (likely(ctx)) { 188 if (unlikely(ctx->bpi_ctx == NULL)) { 189 QAT_DP_LOG(ERR, "QAT PMD only supports security" 190 " operation requests for" 191 " DOCSIS, op (%p) is not for" 192 " DOCSIS.", op); 193 return -EINVAL; 194 } else if (unlikely(((op->sym->m_dst != NULL) && 195 (op->sym->m_dst != op->sym->m_src)) || 196 op->sym->m_src->nb_segs > 1)) { 197 QAT_DP_LOG(ERR, "OOP and/or multi-segment" 198 " buffers not supported for" 199 " DOCSIS security."); 200 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 201 return -EINVAL; 202 } 203 } 204 #endif 205 } 206 207 if (unlikely(ctx == NULL)) { 208 QAT_DP_LOG(ERR, "Session was not created for this device"); 209 return -EINVAL; 210 } 211 212 if (unlikely(ctx->min_qat_dev_gen > qat_dev_gen)) { 213 QAT_DP_LOG(ERR, "Session alg not supported on this device gen"); 214 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION; 215 return -EINVAL; 216 } 217 218 qat_req = (struct icp_qat_fw_la_bulk_req *)out_msg; 219 rte_mov128((uint8_t *)qat_req, (const uint8_t *)&(ctx->fw_req)); 220 qat_req->comn_mid.opaque_data = (uint64_t)(uintptr_t)op; 221 cipher_param = (void *)&qat_req->serv_specif_rqpars; 222 auth_param = (void *)((uint8_t *)cipher_param + 223 ICP_QAT_FW_HASH_REQUEST_PARAMETERS_OFFSET); 224 225 if (ctx->qat_cmd == ICP_QAT_FW_LA_CMD_HASH_CIPHER || 226 ctx->qat_cmd == ICP_QAT_FW_LA_CMD_CIPHER_HASH) { 227 /* AES-GCM or AES-CCM */ 228 if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_128 || 229 ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_64 || 230 (ctx->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_AES128 231 && ctx->qat_mode == ICP_QAT_HW_CIPHER_CTR_MODE 232 && ctx->qat_hash_alg == 233 ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC)) { 234 do_aead = 1; 235 } else { 236 do_auth = 1; 237 do_cipher = 1; 238 } 239 } else if (ctx->qat_cmd == ICP_QAT_FW_LA_CMD_AUTH) { 240 do_auth = 1; 241 do_cipher = 0; 242 } else if (ctx->qat_cmd == ICP_QAT_FW_LA_CMD_CIPHER) { 243 do_auth = 0; 244 do_cipher = 1; 245 } 246 247 if (do_cipher) { 248 249 if (ctx->qat_cipher_alg == 250 ICP_QAT_HW_CIPHER_ALGO_SNOW_3G_UEA2 || 251 ctx->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_KASUMI || 252 ctx->qat_cipher_alg == 253 ICP_QAT_HW_CIPHER_ALGO_ZUC_3G_128_EEA3) { 254 255 if (unlikely( 256 (op->sym->cipher.data.length % BYTE_LENGTH != 0) || 257 (op->sym->cipher.data.offset % BYTE_LENGTH != 0))) { 258 QAT_DP_LOG(ERR, 259 "SNOW3G/KASUMI/ZUC in QAT PMD only supports byte aligned values"); 260 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 261 return -EINVAL; 262 } 263 cipher_len = op->sym->cipher.data.length >> 3; 264 cipher_ofs = op->sym->cipher.data.offset >> 3; 265 266 } else if (ctx->bpi_ctx) { 267 /* DOCSIS - only send complete blocks to device. 268 * Process any partial block using CFB mode. 269 * Even if 0 complete blocks, still send this to device 270 * to get into rx queue for post-process and dequeuing 271 */ 272 cipher_len = qat_bpicipher_preprocess(ctx, op); 273 cipher_ofs = op->sym->cipher.data.offset; 274 } else { 275 cipher_len = op->sym->cipher.data.length; 276 cipher_ofs = op->sym->cipher.data.offset; 277 } 278 279 set_cipher_iv(ctx->cipher_iv.length, ctx->cipher_iv.offset, 280 cipher_param, op, qat_req); 281 min_ofs = cipher_ofs; 282 } 283 284 if (do_auth) { 285 286 if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2 || 287 ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_KASUMI_F9 || 288 ctx->qat_hash_alg == 289 ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3) { 290 if (unlikely( 291 (op->sym->auth.data.offset % BYTE_LENGTH != 0) || 292 (op->sym->auth.data.length % BYTE_LENGTH != 0))) { 293 QAT_DP_LOG(ERR, 294 "For SNOW3G/KASUMI/ZUC, QAT PMD only supports byte aligned values"); 295 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 296 return -EINVAL; 297 } 298 auth_ofs = op->sym->auth.data.offset >> 3; 299 auth_len = op->sym->auth.data.length >> 3; 300 301 auth_param->u1.aad_adr = 302 rte_crypto_op_ctophys_offset(op, 303 ctx->auth_iv.offset); 304 305 } else if (ctx->qat_hash_alg == 306 ICP_QAT_HW_AUTH_ALGO_GALOIS_128 || 307 ctx->qat_hash_alg == 308 ICP_QAT_HW_AUTH_ALGO_GALOIS_64) { 309 /* AES-GMAC */ 310 set_cipher_iv(ctx->auth_iv.length, 311 ctx->auth_iv.offset, 312 cipher_param, op, qat_req); 313 auth_ofs = op->sym->auth.data.offset; 314 auth_len = op->sym->auth.data.length; 315 316 auth_param->u1.aad_adr = 0; 317 auth_param->u2.aad_sz = 0; 318 319 /* 320 * If len(iv)==12B fw computes J0 321 */ 322 if (ctx->auth_iv.length == 12) { 323 ICP_QAT_FW_LA_GCM_IV_LEN_FLAG_SET( 324 qat_req->comn_hdr.serv_specif_flags, 325 ICP_QAT_FW_LA_GCM_IV_LEN_12_OCTETS); 326 327 } 328 } else { 329 auth_ofs = op->sym->auth.data.offset; 330 auth_len = op->sym->auth.data.length; 331 332 } 333 min_ofs = auth_ofs; 334 335 auth_param->auth_res_addr = 336 op->sym->auth.digest.phys_addr; 337 338 } 339 340 if (do_aead) { 341 /* 342 * This address may used for setting AAD physical pointer 343 * into IV offset from op 344 */ 345 rte_iova_t aad_phys_addr_aead = op->sym->aead.aad.phys_addr; 346 if (ctx->qat_hash_alg == 347 ICP_QAT_HW_AUTH_ALGO_GALOIS_128 || 348 ctx->qat_hash_alg == 349 ICP_QAT_HW_AUTH_ALGO_GALOIS_64) { 350 /* 351 * If len(iv)==12B fw computes J0 352 */ 353 if (ctx->cipher_iv.length == 12) { 354 ICP_QAT_FW_LA_GCM_IV_LEN_FLAG_SET( 355 qat_req->comn_hdr.serv_specif_flags, 356 ICP_QAT_FW_LA_GCM_IV_LEN_12_OCTETS); 357 } 358 set_cipher_iv(ctx->cipher_iv.length, 359 ctx->cipher_iv.offset, 360 cipher_param, op, qat_req); 361 362 } else if (ctx->qat_hash_alg == 363 ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC) { 364 365 /* In case of AES-CCM this may point to user selected 366 * memory or iv offset in cypto_op 367 */ 368 uint8_t *aad_data = op->sym->aead.aad.data; 369 /* This is true AAD length, it not includes 18 bytes of 370 * preceding data 371 */ 372 uint8_t aad_ccm_real_len = 0; 373 uint8_t aad_len_field_sz = 0; 374 uint32_t msg_len_be = 375 rte_bswap32(op->sym->aead.data.length); 376 377 if (ctx->aad_len > ICP_QAT_HW_CCM_AAD_DATA_OFFSET) { 378 aad_len_field_sz = ICP_QAT_HW_CCM_AAD_LEN_INFO; 379 aad_ccm_real_len = ctx->aad_len - 380 ICP_QAT_HW_CCM_AAD_B0_LEN - 381 ICP_QAT_HW_CCM_AAD_LEN_INFO; 382 } else { 383 /* 384 * aad_len not greater than 18, so no actual aad 385 * data, then use IV after op for B0 block 386 */ 387 aad_data = rte_crypto_op_ctod_offset(op, 388 uint8_t *, 389 ctx->cipher_iv.offset); 390 aad_phys_addr_aead = 391 rte_crypto_op_ctophys_offset(op, 392 ctx->cipher_iv.offset); 393 } 394 395 uint8_t q = ICP_QAT_HW_CCM_NQ_CONST - 396 ctx->cipher_iv.length; 397 398 aad_data[0] = ICP_QAT_HW_CCM_BUILD_B0_FLAGS( 399 aad_len_field_sz, 400 ctx->digest_length, q); 401 402 if (q > ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE) { 403 memcpy(aad_data + ctx->cipher_iv.length + 404 ICP_QAT_HW_CCM_NONCE_OFFSET + 405 (q - ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE), 406 (uint8_t *)&msg_len_be, 407 ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE); 408 } else { 409 memcpy(aad_data + ctx->cipher_iv.length + 410 ICP_QAT_HW_CCM_NONCE_OFFSET, 411 (uint8_t *)&msg_len_be 412 + (ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE 413 - q), q); 414 } 415 416 if (aad_len_field_sz > 0) { 417 *(uint16_t *)&aad_data[ICP_QAT_HW_CCM_AAD_B0_LEN] 418 = rte_bswap16(aad_ccm_real_len); 419 420 if ((aad_ccm_real_len + aad_len_field_sz) 421 % ICP_QAT_HW_CCM_AAD_B0_LEN) { 422 uint8_t pad_len = 0; 423 uint8_t pad_idx = 0; 424 425 pad_len = ICP_QAT_HW_CCM_AAD_B0_LEN - 426 ((aad_ccm_real_len + aad_len_field_sz) % 427 ICP_QAT_HW_CCM_AAD_B0_LEN); 428 pad_idx = ICP_QAT_HW_CCM_AAD_B0_LEN + 429 aad_ccm_real_len + aad_len_field_sz; 430 memset(&aad_data[pad_idx], 431 0, pad_len); 432 } 433 434 } 435 436 set_cipher_iv_ccm(ctx->cipher_iv.length, 437 ctx->cipher_iv.offset, 438 cipher_param, op, q, 439 aad_len_field_sz); 440 441 } 442 443 cipher_len = op->sym->aead.data.length; 444 cipher_ofs = op->sym->aead.data.offset; 445 auth_len = op->sym->aead.data.length; 446 auth_ofs = op->sym->aead.data.offset; 447 448 auth_param->u1.aad_adr = aad_phys_addr_aead; 449 auth_param->auth_res_addr = op->sym->aead.digest.phys_addr; 450 min_ofs = op->sym->aead.data.offset; 451 } 452 453 if (op->sym->m_src->nb_segs > 1 || 454 (op->sym->m_dst && op->sym->m_dst->nb_segs > 1)) 455 do_sgl = 1; 456 457 /* adjust for chain case */ 458 if (do_cipher && do_auth) 459 min_ofs = cipher_ofs < auth_ofs ? cipher_ofs : auth_ofs; 460 461 if (unlikely(min_ofs >= rte_pktmbuf_data_len(op->sym->m_src) && do_sgl)) 462 min_ofs = 0; 463 464 if (unlikely((op->sym->m_dst != NULL) && 465 (op->sym->m_dst != op->sym->m_src))) { 466 /* Out-of-place operation (OOP) 467 * Don't align DMA start. DMA the minimum data-set 468 * so as not to overwrite data in dest buffer 469 */ 470 in_place = 0; 471 src_buf_start = 472 rte_pktmbuf_iova_offset(op->sym->m_src, min_ofs); 473 dst_buf_start = 474 rte_pktmbuf_iova_offset(op->sym->m_dst, min_ofs); 475 476 } else { 477 /* In-place operation 478 * Start DMA at nearest aligned address below min_ofs 479 */ 480 src_buf_start = 481 rte_pktmbuf_iova_offset(op->sym->m_src, min_ofs) 482 & QAT_64_BTYE_ALIGN_MASK; 483 484 if (unlikely((rte_pktmbuf_iova(op->sym->m_src) - 485 rte_pktmbuf_headroom(op->sym->m_src)) 486 > src_buf_start)) { 487 /* alignment has pushed addr ahead of start of mbuf 488 * so revert and take the performance hit 489 */ 490 src_buf_start = 491 rte_pktmbuf_iova_offset(op->sym->m_src, 492 min_ofs); 493 } 494 dst_buf_start = src_buf_start; 495 496 /* remember any adjustment for later, note, can be +/- */ 497 alignment_adjustment = src_buf_start - 498 rte_pktmbuf_iova_offset(op->sym->m_src, min_ofs); 499 } 500 501 if (do_cipher || do_aead) { 502 cipher_param->cipher_offset = 503 (uint32_t)rte_pktmbuf_iova_offset( 504 op->sym->m_src, cipher_ofs) - src_buf_start; 505 cipher_param->cipher_length = cipher_len; 506 } else { 507 cipher_param->cipher_offset = 0; 508 cipher_param->cipher_length = 0; 509 } 510 511 if (do_auth || do_aead) { 512 auth_param->auth_off = (uint32_t)rte_pktmbuf_iova_offset( 513 op->sym->m_src, auth_ofs) - src_buf_start; 514 auth_param->auth_len = auth_len; 515 } else { 516 auth_param->auth_off = 0; 517 auth_param->auth_len = 0; 518 } 519 520 qat_req->comn_mid.dst_length = 521 qat_req->comn_mid.src_length = 522 (cipher_param->cipher_offset + cipher_param->cipher_length) 523 > (auth_param->auth_off + auth_param->auth_len) ? 524 (cipher_param->cipher_offset + cipher_param->cipher_length) 525 : (auth_param->auth_off + auth_param->auth_len); 526 527 if (do_auth && do_cipher) { 528 /* Handle digest-encrypted cases, i.e. 529 * auth-gen-then-cipher-encrypt and 530 * cipher-decrypt-then-auth-verify 531 */ 532 /* First find the end of the data */ 533 if (do_sgl) { 534 uint32_t remaining_off = auth_param->auth_off + 535 auth_param->auth_len + alignment_adjustment; 536 struct rte_mbuf *sgl_buf = 537 (in_place ? 538 op->sym->m_src : op->sym->m_dst); 539 540 while (remaining_off >= rte_pktmbuf_data_len(sgl_buf) 541 && sgl_buf->next != NULL) { 542 remaining_off -= rte_pktmbuf_data_len(sgl_buf); 543 sgl_buf = sgl_buf->next; 544 } 545 546 auth_data_end = (uint64_t)rte_pktmbuf_iova_offset( 547 sgl_buf, remaining_off); 548 } else { 549 auth_data_end = (in_place ? 550 src_buf_start : dst_buf_start) + 551 auth_param->auth_off + auth_param->auth_len; 552 } 553 /* Then check if digest-encrypted conditions are met */ 554 if ((auth_param->auth_off + auth_param->auth_len < 555 cipher_param->cipher_offset + 556 cipher_param->cipher_length) && 557 (op->sym->auth.digest.phys_addr == 558 auth_data_end)) { 559 /* Handle partial digest encryption */ 560 if (cipher_param->cipher_offset + 561 cipher_param->cipher_length < 562 auth_param->auth_off + 563 auth_param->auth_len + 564 ctx->digest_length) 565 qat_req->comn_mid.dst_length = 566 qat_req->comn_mid.src_length = 567 auth_param->auth_off + 568 auth_param->auth_len + 569 ctx->digest_length; 570 struct icp_qat_fw_comn_req_hdr *header = 571 &qat_req->comn_hdr; 572 ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET( 573 header->serv_specif_flags, 574 ICP_QAT_FW_LA_DIGEST_IN_BUFFER); 575 } 576 } 577 578 if (do_sgl) { 579 580 ICP_QAT_FW_COMN_PTR_TYPE_SET(qat_req->comn_hdr.comn_req_flags, 581 QAT_COMN_PTR_TYPE_SGL); 582 ret = qat_sgl_fill_array(op->sym->m_src, 583 (int64_t)(src_buf_start - rte_pktmbuf_iova(op->sym->m_src)), 584 &cookie->qat_sgl_src, 585 qat_req->comn_mid.src_length, 586 QAT_SYM_SGL_MAX_NUMBER); 587 588 if (unlikely(ret)) { 589 QAT_DP_LOG(ERR, "QAT PMD Cannot fill sgl array"); 590 return ret; 591 } 592 593 if (in_place) 594 qat_req->comn_mid.dest_data_addr = 595 qat_req->comn_mid.src_data_addr = 596 cookie->qat_sgl_src_phys_addr; 597 else { 598 ret = qat_sgl_fill_array(op->sym->m_dst, 599 (int64_t)(dst_buf_start - 600 rte_pktmbuf_iova(op->sym->m_dst)), 601 &cookie->qat_sgl_dst, 602 qat_req->comn_mid.dst_length, 603 QAT_SYM_SGL_MAX_NUMBER); 604 605 if (unlikely(ret)) { 606 QAT_DP_LOG(ERR, "QAT PMD can't fill sgl array"); 607 return ret; 608 } 609 610 qat_req->comn_mid.src_data_addr = 611 cookie->qat_sgl_src_phys_addr; 612 qat_req->comn_mid.dest_data_addr = 613 cookie->qat_sgl_dst_phys_addr; 614 } 615 qat_req->comn_mid.src_length = 0; 616 qat_req->comn_mid.dst_length = 0; 617 } else { 618 qat_req->comn_mid.src_data_addr = src_buf_start; 619 qat_req->comn_mid.dest_data_addr = dst_buf_start; 620 } 621 622 /* Handle Single-Pass GCM */ 623 if (ctx->is_single_pass) { 624 cipher_param->spc_aad_addr = op->sym->aead.aad.phys_addr; 625 cipher_param->spc_auth_res_addr = 626 op->sym->aead.digest.phys_addr; 627 } 628 629 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG 630 QAT_DP_HEXDUMP_LOG(DEBUG, "qat_req:", qat_req, 631 sizeof(struct icp_qat_fw_la_bulk_req)); 632 QAT_DP_HEXDUMP_LOG(DEBUG, "src_data:", 633 rte_pktmbuf_mtod(op->sym->m_src, uint8_t*), 634 rte_pktmbuf_data_len(op->sym->m_src)); 635 if (do_cipher) { 636 uint8_t *cipher_iv_ptr = rte_crypto_op_ctod_offset(op, 637 uint8_t *, 638 ctx->cipher_iv.offset); 639 QAT_DP_HEXDUMP_LOG(DEBUG, "cipher iv:", cipher_iv_ptr, 640 ctx->cipher_iv.length); 641 } 642 643 if (do_auth) { 644 if (ctx->auth_iv.length) { 645 uint8_t *auth_iv_ptr = rte_crypto_op_ctod_offset(op, 646 uint8_t *, 647 ctx->auth_iv.offset); 648 QAT_DP_HEXDUMP_LOG(DEBUG, "auth iv:", auth_iv_ptr, 649 ctx->auth_iv.length); 650 } 651 QAT_DP_HEXDUMP_LOG(DEBUG, "digest:", op->sym->auth.digest.data, 652 ctx->digest_length); 653 } 654 655 if (do_aead) { 656 QAT_DP_HEXDUMP_LOG(DEBUG, "digest:", op->sym->aead.digest.data, 657 ctx->digest_length); 658 QAT_DP_HEXDUMP_LOG(DEBUG, "aad:", op->sym->aead.aad.data, 659 ctx->aad_len); 660 } 661 #endif 662 return 0; 663 } 664