1 /* SPDX-License-Identifier: BSD-3-Clause 2 * 3 * Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved. 4 * Copyright 2016-2021 NXP 5 * 6 */ 7 8 #include <time.h> 9 #include <net/if.h> 10 #include <unistd.h> 11 12 #include <rte_ip.h> 13 #include <rte_mbuf.h> 14 #include <rte_cryptodev.h> 15 #include <rte_malloc.h> 16 #include <rte_memcpy.h> 17 #include <rte_string_fns.h> 18 #include <rte_cycles.h> 19 #include <rte_kvargs.h> 20 #include <rte_dev.h> 21 #include <cryptodev_pmd.h> 22 #include <rte_common.h> 23 #include <rte_fslmc.h> 24 #include <fslmc_vfio.h> 25 #include <dpaa2_hw_pvt.h> 26 #include <dpaa2_hw_dpio.h> 27 #include <dpaa2_hw_mempool.h> 28 #include <fsl_dpopr.h> 29 #include <fsl_dpseci.h> 30 #include <fsl_mc_sys.h> 31 #include <rte_hexdump.h> 32 33 #include "dpaa2_sec_priv.h" 34 #include "dpaa2_sec_event.h" 35 #include "dpaa2_sec_logs.h" 36 37 /* RTA header files */ 38 #include <desc/ipsec.h> 39 #include <desc/pdcp.h> 40 #include <desc/sdap.h> 41 #include <desc/algo.h> 42 43 /* Minimum job descriptor consists of a oneword job descriptor HEADER and 44 * a pointer to the shared descriptor 45 */ 46 #define MIN_JOB_DESC_SIZE (CAAM_CMD_SZ + CAAM_PTR_SZ) 47 #define FSL_VENDOR_ID 0x1957 48 #define FSL_DEVICE_ID 0x410 49 #define FSL_SUBSYSTEM_SEC 1 50 #define FSL_MC_DPSECI_DEVID 3 51 52 #define NO_PREFETCH 0 53 54 #define DRIVER_DUMP_MODE "drv_dump_mode" 55 #define DRIVER_STRICT_ORDER "drv_strict_order" 56 57 /* DPAA2_SEC_DP_DUMP levels */ 58 enum dpaa2_sec_dump_levels { 59 DPAA2_SEC_DP_NO_DUMP, 60 DPAA2_SEC_DP_ERR_DUMP, 61 DPAA2_SEC_DP_FULL_DUMP 62 }; 63 64 uint8_t cryptodev_driver_id; 65 uint8_t dpaa2_sec_dp_dump = DPAA2_SEC_DP_ERR_DUMP; 66 67 #ifdef RTE_LIB_SECURITY 68 static inline int 69 build_proto_compound_sg_fd(dpaa2_sec_session *sess, 70 struct rte_crypto_op *op, 71 struct qbman_fd *fd, uint16_t bpid) 72 { 73 struct rte_crypto_sym_op *sym_op = op->sym; 74 struct ctxt_priv *priv = sess->ctxt; 75 struct qbman_fle *fle, *sge, *ip_fle, *op_fle; 76 struct sec_flow_context *flc; 77 struct rte_mbuf *mbuf; 78 uint32_t in_len = 0, out_len = 0; 79 80 if (sym_op->m_dst) 81 mbuf = sym_op->m_dst; 82 else 83 mbuf = sym_op->m_src; 84 85 /* first FLE entry used to store mbuf and session ctxt */ 86 fle = (struct qbman_fle *)rte_malloc(NULL, 87 FLE_SG_MEM_SIZE(mbuf->nb_segs + sym_op->m_src->nb_segs), 88 RTE_CACHE_LINE_SIZE); 89 if (unlikely(!fle)) { 90 DPAA2_SEC_DP_ERR("Proto:SG: Memory alloc failed for SGE"); 91 return -ENOMEM; 92 } 93 memset(fle, 0, FLE_SG_MEM_SIZE(mbuf->nb_segs + sym_op->m_src->nb_segs)); 94 DPAA2_SET_FLE_ADDR(fle, (size_t)op); 95 DPAA2_FLE_SAVE_CTXT(fle, (ptrdiff_t)priv); 96 97 /* Save the shared descriptor */ 98 flc = &priv->flc_desc[0].flc; 99 100 op_fle = fle + 1; 101 ip_fle = fle + 2; 102 sge = fle + 3; 103 104 if (likely(bpid < MAX_BPID)) { 105 DPAA2_SET_FD_BPID(fd, bpid); 106 DPAA2_SET_FLE_BPID(op_fle, bpid); 107 DPAA2_SET_FLE_BPID(ip_fle, bpid); 108 } else { 109 DPAA2_SET_FD_IVP(fd); 110 DPAA2_SET_FLE_IVP(op_fle); 111 DPAA2_SET_FLE_IVP(ip_fle); 112 } 113 114 /* Configure FD as a FRAME LIST */ 115 DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(op_fle)); 116 DPAA2_SET_FD_COMPOUND_FMT(fd); 117 DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc)); 118 119 /* Configure Output FLE with Scatter/Gather Entry */ 120 DPAA2_SET_FLE_SG_EXT(op_fle); 121 DPAA2_SET_FLE_ADDR(op_fle, DPAA2_VADDR_TO_IOVA(sge)); 122 123 /* Configure Output SGE for Encap/Decap */ 124 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf)); 125 DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off); 126 /* o/p segs */ 127 while (mbuf->next) { 128 sge->length = mbuf->data_len; 129 out_len += sge->length; 130 sge++; 131 mbuf = mbuf->next; 132 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf)); 133 DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off); 134 } 135 /* using buf_len for last buf - so that extra data can be added */ 136 sge->length = mbuf->buf_len - mbuf->data_off; 137 out_len += sge->length; 138 139 DPAA2_SET_FLE_FIN(sge); 140 op_fle->length = out_len; 141 142 sge++; 143 mbuf = sym_op->m_src; 144 145 /* Configure Input FLE with Scatter/Gather Entry */ 146 DPAA2_SET_FLE_ADDR(ip_fle, DPAA2_VADDR_TO_IOVA(sge)); 147 DPAA2_SET_FLE_SG_EXT(ip_fle); 148 DPAA2_SET_FLE_FIN(ip_fle); 149 150 /* Configure input SGE for Encap/Decap */ 151 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf)); 152 DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off); 153 sge->length = mbuf->data_len; 154 in_len += sge->length; 155 156 mbuf = mbuf->next; 157 /* i/p segs */ 158 while (mbuf) { 159 sge++; 160 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf)); 161 DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off); 162 sge->length = mbuf->data_len; 163 in_len += sge->length; 164 mbuf = mbuf->next; 165 } 166 ip_fle->length = in_len; 167 DPAA2_SET_FLE_FIN(sge); 168 169 /* In case of PDCP, per packet HFN is stored in 170 * mbuf priv after sym_op. 171 */ 172 if (sess->ctxt_type == DPAA2_SEC_PDCP && sess->pdcp.hfn_ovd) { 173 uint32_t hfn_ovd = *(uint32_t *)((uint8_t *)op + 174 sess->pdcp.hfn_ovd_offset); 175 /*enable HFN override override */ 176 DPAA2_SET_FLE_INTERNAL_JD(ip_fle, hfn_ovd); 177 DPAA2_SET_FLE_INTERNAL_JD(op_fle, hfn_ovd); 178 DPAA2_SET_FD_INTERNAL_JD(fd, hfn_ovd); 179 } 180 DPAA2_SET_FD_LEN(fd, ip_fle->length); 181 182 return 0; 183 } 184 185 static inline int 186 build_proto_compound_fd(dpaa2_sec_session *sess, 187 struct rte_crypto_op *op, 188 struct qbman_fd *fd, uint16_t bpid) 189 { 190 struct rte_crypto_sym_op *sym_op = op->sym; 191 struct ctxt_priv *priv = sess->ctxt; 192 struct qbman_fle *fle, *ip_fle, *op_fle; 193 struct sec_flow_context *flc; 194 struct rte_mbuf *src_mbuf = sym_op->m_src; 195 struct rte_mbuf *dst_mbuf = sym_op->m_dst; 196 int retval; 197 198 if (!dst_mbuf) 199 dst_mbuf = src_mbuf; 200 201 /* Save the shared descriptor */ 202 flc = &priv->flc_desc[0].flc; 203 204 /* we are using the first FLE entry to store Mbuf */ 205 retval = rte_mempool_get(priv->fle_pool, (void **)(&fle)); 206 if (retval) { 207 DPAA2_SEC_DP_ERR("Memory alloc failed"); 208 return -ENOMEM; 209 } 210 memset(fle, 0, FLE_POOL_BUF_SIZE); 211 DPAA2_SET_FLE_ADDR(fle, (size_t)op); 212 DPAA2_FLE_SAVE_CTXT(fle, (ptrdiff_t)priv); 213 214 op_fle = fle + 1; 215 ip_fle = fle + 2; 216 217 if (likely(bpid < MAX_BPID)) { 218 DPAA2_SET_FD_BPID(fd, bpid); 219 DPAA2_SET_FLE_BPID(op_fle, bpid); 220 DPAA2_SET_FLE_BPID(ip_fle, bpid); 221 } else { 222 DPAA2_SET_FD_IVP(fd); 223 DPAA2_SET_FLE_IVP(op_fle); 224 DPAA2_SET_FLE_IVP(ip_fle); 225 } 226 227 /* Configure FD as a FRAME LIST */ 228 DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(op_fle)); 229 DPAA2_SET_FD_COMPOUND_FMT(fd); 230 DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc)); 231 232 /* Configure Output FLE with dst mbuf data */ 233 DPAA2_SET_FLE_ADDR(op_fle, DPAA2_MBUF_VADDR_TO_IOVA(dst_mbuf)); 234 DPAA2_SET_FLE_OFFSET(op_fle, dst_mbuf->data_off); 235 DPAA2_SET_FLE_LEN(op_fle, dst_mbuf->buf_len); 236 237 /* Configure Input FLE with src mbuf data */ 238 DPAA2_SET_FLE_ADDR(ip_fle, DPAA2_MBUF_VADDR_TO_IOVA(src_mbuf)); 239 DPAA2_SET_FLE_OFFSET(ip_fle, src_mbuf->data_off); 240 DPAA2_SET_FLE_LEN(ip_fle, src_mbuf->pkt_len); 241 242 DPAA2_SET_FD_LEN(fd, ip_fle->length); 243 DPAA2_SET_FLE_FIN(ip_fle); 244 245 /* In case of PDCP, per packet HFN is stored in 246 * mbuf priv after sym_op. 247 */ 248 if (sess->ctxt_type == DPAA2_SEC_PDCP && sess->pdcp.hfn_ovd) { 249 uint32_t hfn_ovd = *(uint32_t *)((uint8_t *)op + 250 sess->pdcp.hfn_ovd_offset); 251 /*enable HFN override override */ 252 DPAA2_SET_FLE_INTERNAL_JD(ip_fle, hfn_ovd); 253 DPAA2_SET_FLE_INTERNAL_JD(op_fle, hfn_ovd); 254 DPAA2_SET_FD_INTERNAL_JD(fd, hfn_ovd); 255 } 256 257 return 0; 258 259 } 260 261 static inline int 262 build_proto_fd(dpaa2_sec_session *sess, 263 struct rte_crypto_op *op, 264 struct qbman_fd *fd, uint16_t bpid) 265 { 266 struct rte_crypto_sym_op *sym_op = op->sym; 267 if (sym_op->m_dst) 268 return build_proto_compound_fd(sess, op, fd, bpid); 269 270 struct ctxt_priv *priv = sess->ctxt; 271 struct sec_flow_context *flc; 272 struct rte_mbuf *mbuf = sym_op->m_src; 273 274 if (likely(bpid < MAX_BPID)) 275 DPAA2_SET_FD_BPID(fd, bpid); 276 else 277 DPAA2_SET_FD_IVP(fd); 278 279 /* Save the shared descriptor */ 280 flc = &priv->flc_desc[0].flc; 281 282 DPAA2_SET_FD_ADDR(fd, DPAA2_MBUF_VADDR_TO_IOVA(sym_op->m_src)); 283 DPAA2_SET_FD_OFFSET(fd, sym_op->m_src->data_off); 284 DPAA2_SET_FD_LEN(fd, sym_op->m_src->pkt_len); 285 DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc)); 286 287 /* save physical address of mbuf */ 288 op->sym->aead.digest.phys_addr = mbuf->buf_iova; 289 mbuf->buf_iova = (size_t)op; 290 291 return 0; 292 } 293 #endif 294 295 static inline int 296 build_authenc_gcm_sg_fd(dpaa2_sec_session *sess, 297 struct rte_crypto_op *op, 298 struct qbman_fd *fd, __rte_unused uint16_t bpid) 299 { 300 struct rte_crypto_sym_op *sym_op = op->sym; 301 struct ctxt_priv *priv = sess->ctxt; 302 struct qbman_fle *fle, *sge, *ip_fle, *op_fle; 303 struct sec_flow_context *flc; 304 uint32_t auth_only_len = sess->ext_params.aead_ctxt.auth_only_len; 305 int icv_len = sess->digest_length; 306 uint8_t *old_icv; 307 struct rte_mbuf *mbuf; 308 uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, 309 sess->iv.offset); 310 311 if (sym_op->m_dst) 312 mbuf = sym_op->m_dst; 313 else 314 mbuf = sym_op->m_src; 315 316 /* first FLE entry used to store mbuf and session ctxt */ 317 fle = (struct qbman_fle *)rte_malloc(NULL, 318 FLE_SG_MEM_SIZE(mbuf->nb_segs + sym_op->m_src->nb_segs), 319 RTE_CACHE_LINE_SIZE); 320 if (unlikely(!fle)) { 321 DPAA2_SEC_ERR("GCM SG: Memory alloc failed for SGE"); 322 return -ENOMEM; 323 } 324 memset(fle, 0, FLE_SG_MEM_SIZE(mbuf->nb_segs + sym_op->m_src->nb_segs)); 325 DPAA2_SET_FLE_ADDR(fle, (size_t)op); 326 DPAA2_FLE_SAVE_CTXT(fle, (size_t)priv); 327 328 op_fle = fle + 1; 329 ip_fle = fle + 2; 330 sge = fle + 3; 331 332 /* Save the shared descriptor */ 333 flc = &priv->flc_desc[0].flc; 334 335 /* Configure FD as a FRAME LIST */ 336 DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(op_fle)); 337 DPAA2_SET_FD_COMPOUND_FMT(fd); 338 DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc)); 339 340 DPAA2_SEC_DP_DEBUG("GCM SG: auth_off: 0x%x/length %d, digest-len=%d\n" 341 "iv-len=%d data_off: 0x%x\n", 342 sym_op->aead.data.offset, 343 sym_op->aead.data.length, 344 sess->digest_length, 345 sess->iv.length, 346 sym_op->m_src->data_off); 347 348 /* Configure Output FLE with Scatter/Gather Entry */ 349 DPAA2_SET_FLE_SG_EXT(op_fle); 350 DPAA2_SET_FLE_ADDR(op_fle, DPAA2_VADDR_TO_IOVA(sge)); 351 352 if (auth_only_len) 353 DPAA2_SET_FLE_INTERNAL_JD(op_fle, auth_only_len); 354 355 op_fle->length = (sess->dir == DIR_ENC) ? 356 (sym_op->aead.data.length + icv_len) : 357 sym_op->aead.data.length; 358 359 /* Configure Output SGE for Encap/Decap */ 360 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf)); 361 DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off + sym_op->aead.data.offset); 362 sge->length = mbuf->data_len - sym_op->aead.data.offset; 363 364 mbuf = mbuf->next; 365 /* o/p segs */ 366 while (mbuf) { 367 sge++; 368 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf)); 369 DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off); 370 sge->length = mbuf->data_len; 371 mbuf = mbuf->next; 372 } 373 sge->length -= icv_len; 374 375 if (sess->dir == DIR_ENC) { 376 sge++; 377 DPAA2_SET_FLE_ADDR(sge, 378 DPAA2_VADDR_TO_IOVA(sym_op->aead.digest.data)); 379 sge->length = icv_len; 380 } 381 DPAA2_SET_FLE_FIN(sge); 382 383 sge++; 384 mbuf = sym_op->m_src; 385 386 /* Configure Input FLE with Scatter/Gather Entry */ 387 DPAA2_SET_FLE_ADDR(ip_fle, DPAA2_VADDR_TO_IOVA(sge)); 388 DPAA2_SET_FLE_SG_EXT(ip_fle); 389 DPAA2_SET_FLE_FIN(ip_fle); 390 ip_fle->length = (sess->dir == DIR_ENC) ? 391 (sym_op->aead.data.length + sess->iv.length + auth_only_len) : 392 (sym_op->aead.data.length + sess->iv.length + auth_only_len + 393 icv_len); 394 395 /* Configure Input SGE for Encap/Decap */ 396 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(IV_ptr)); 397 sge->length = sess->iv.length; 398 399 sge++; 400 if (auth_only_len) { 401 DPAA2_SET_FLE_ADDR(sge, 402 DPAA2_VADDR_TO_IOVA(sym_op->aead.aad.data)); 403 sge->length = auth_only_len; 404 sge++; 405 } 406 407 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf)); 408 DPAA2_SET_FLE_OFFSET(sge, sym_op->aead.data.offset + 409 mbuf->data_off); 410 sge->length = mbuf->data_len - sym_op->aead.data.offset; 411 412 mbuf = mbuf->next; 413 /* i/p segs */ 414 while (mbuf) { 415 sge++; 416 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf)); 417 DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off); 418 sge->length = mbuf->data_len; 419 mbuf = mbuf->next; 420 } 421 422 if (sess->dir == DIR_DEC) { 423 sge++; 424 old_icv = (uint8_t *)(sge + 1); 425 memcpy(old_icv, sym_op->aead.digest.data, icv_len); 426 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(old_icv)); 427 sge->length = icv_len; 428 } 429 430 DPAA2_SET_FLE_FIN(sge); 431 if (auth_only_len) { 432 DPAA2_SET_FLE_INTERNAL_JD(ip_fle, auth_only_len); 433 DPAA2_SET_FD_INTERNAL_JD(fd, auth_only_len); 434 } 435 DPAA2_SET_FD_LEN(fd, ip_fle->length); 436 437 return 0; 438 } 439 440 static inline int 441 build_authenc_gcm_fd(dpaa2_sec_session *sess, 442 struct rte_crypto_op *op, 443 struct qbman_fd *fd, uint16_t bpid) 444 { 445 struct rte_crypto_sym_op *sym_op = op->sym; 446 struct ctxt_priv *priv = sess->ctxt; 447 struct qbman_fle *fle, *sge; 448 struct sec_flow_context *flc; 449 uint32_t auth_only_len = sess->ext_params.aead_ctxt.auth_only_len; 450 int icv_len = sess->digest_length, retval; 451 uint8_t *old_icv; 452 struct rte_mbuf *dst; 453 uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, 454 sess->iv.offset); 455 456 if (sym_op->m_dst) 457 dst = sym_op->m_dst; 458 else 459 dst = sym_op->m_src; 460 461 /* TODO we are using the first FLE entry to store Mbuf and session ctxt. 462 * Currently we donot know which FLE has the mbuf stored. 463 * So while retreiving we can go back 1 FLE from the FD -ADDR 464 * to get the MBUF Addr from the previous FLE. 465 * We can have a better approach to use the inline Mbuf 466 */ 467 retval = rte_mempool_get(priv->fle_pool, (void **)(&fle)); 468 if (retval) { 469 DPAA2_SEC_ERR("GCM: Memory alloc failed for SGE"); 470 return -ENOMEM; 471 } 472 memset(fle, 0, FLE_POOL_BUF_SIZE); 473 DPAA2_SET_FLE_ADDR(fle, (size_t)op); 474 DPAA2_FLE_SAVE_CTXT(fle, (ptrdiff_t)priv); 475 fle = fle + 1; 476 sge = fle + 2; 477 if (likely(bpid < MAX_BPID)) { 478 DPAA2_SET_FD_BPID(fd, bpid); 479 DPAA2_SET_FLE_BPID(fle, bpid); 480 DPAA2_SET_FLE_BPID(fle + 1, bpid); 481 DPAA2_SET_FLE_BPID(sge, bpid); 482 DPAA2_SET_FLE_BPID(sge + 1, bpid); 483 DPAA2_SET_FLE_BPID(sge + 2, bpid); 484 DPAA2_SET_FLE_BPID(sge + 3, bpid); 485 } else { 486 DPAA2_SET_FD_IVP(fd); 487 DPAA2_SET_FLE_IVP(fle); 488 DPAA2_SET_FLE_IVP((fle + 1)); 489 DPAA2_SET_FLE_IVP(sge); 490 DPAA2_SET_FLE_IVP((sge + 1)); 491 DPAA2_SET_FLE_IVP((sge + 2)); 492 DPAA2_SET_FLE_IVP((sge + 3)); 493 } 494 495 /* Save the shared descriptor */ 496 flc = &priv->flc_desc[0].flc; 497 /* Configure FD as a FRAME LIST */ 498 DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(fle)); 499 DPAA2_SET_FD_COMPOUND_FMT(fd); 500 DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc)); 501 502 DPAA2_SEC_DP_DEBUG("GCM: auth_off: 0x%x/length %d, digest-len=%d\n" 503 "iv-len=%d data_off: 0x%x\n", 504 sym_op->aead.data.offset, 505 sym_op->aead.data.length, 506 sess->digest_length, 507 sess->iv.length, 508 sym_op->m_src->data_off); 509 510 /* Configure Output FLE with Scatter/Gather Entry */ 511 DPAA2_SET_FLE_ADDR(fle, DPAA2_VADDR_TO_IOVA(sge)); 512 if (auth_only_len) 513 DPAA2_SET_FLE_INTERNAL_JD(fle, auth_only_len); 514 fle->length = (sess->dir == DIR_ENC) ? 515 (sym_op->aead.data.length + icv_len) : 516 sym_op->aead.data.length; 517 518 DPAA2_SET_FLE_SG_EXT(fle); 519 520 /* Configure Output SGE for Encap/Decap */ 521 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(dst)); 522 DPAA2_SET_FLE_OFFSET(sge, dst->data_off + sym_op->aead.data.offset); 523 sge->length = sym_op->aead.data.length; 524 525 if (sess->dir == DIR_ENC) { 526 sge++; 527 DPAA2_SET_FLE_ADDR(sge, 528 DPAA2_VADDR_TO_IOVA(sym_op->aead.digest.data)); 529 sge->length = sess->digest_length; 530 } 531 DPAA2_SET_FLE_FIN(sge); 532 533 sge++; 534 fle++; 535 536 /* Configure Input FLE with Scatter/Gather Entry */ 537 DPAA2_SET_FLE_ADDR(fle, DPAA2_VADDR_TO_IOVA(sge)); 538 DPAA2_SET_FLE_SG_EXT(fle); 539 DPAA2_SET_FLE_FIN(fle); 540 fle->length = (sess->dir == DIR_ENC) ? 541 (sym_op->aead.data.length + sess->iv.length + auth_only_len) : 542 (sym_op->aead.data.length + sess->iv.length + auth_only_len + 543 sess->digest_length); 544 545 /* Configure Input SGE for Encap/Decap */ 546 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(IV_ptr)); 547 sge->length = sess->iv.length; 548 sge++; 549 if (auth_only_len) { 550 DPAA2_SET_FLE_ADDR(sge, 551 DPAA2_VADDR_TO_IOVA(sym_op->aead.aad.data)); 552 sge->length = auth_only_len; 553 DPAA2_SET_FLE_BPID(sge, bpid); 554 sge++; 555 } 556 557 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(sym_op->m_src)); 558 DPAA2_SET_FLE_OFFSET(sge, sym_op->aead.data.offset + 559 sym_op->m_src->data_off); 560 sge->length = sym_op->aead.data.length; 561 if (sess->dir == DIR_DEC) { 562 sge++; 563 old_icv = (uint8_t *)(sge + 1); 564 memcpy(old_icv, sym_op->aead.digest.data, 565 sess->digest_length); 566 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(old_icv)); 567 sge->length = sess->digest_length; 568 } 569 DPAA2_SET_FLE_FIN(sge); 570 571 if (auth_only_len) { 572 DPAA2_SET_FLE_INTERNAL_JD(fle, auth_only_len); 573 DPAA2_SET_FD_INTERNAL_JD(fd, auth_only_len); 574 } 575 576 DPAA2_SET_FD_LEN(fd, fle->length); 577 return 0; 578 } 579 580 static inline int 581 build_authenc_sg_fd(dpaa2_sec_session *sess, 582 struct rte_crypto_op *op, 583 struct qbman_fd *fd, __rte_unused uint16_t bpid) 584 { 585 struct rte_crypto_sym_op *sym_op = op->sym; 586 struct ctxt_priv *priv = sess->ctxt; 587 struct qbman_fle *fle, *sge, *ip_fle, *op_fle; 588 struct sec_flow_context *flc; 589 uint16_t auth_hdr_len = sym_op->cipher.data.offset - 590 sym_op->auth.data.offset; 591 uint16_t auth_tail_len = sym_op->auth.data.length - 592 sym_op->cipher.data.length - auth_hdr_len; 593 uint32_t auth_only_len = (auth_tail_len << 16) | auth_hdr_len; 594 int icv_len = sess->digest_length; 595 uint8_t *old_icv; 596 struct rte_mbuf *mbuf; 597 uint8_t *iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, 598 sess->iv.offset); 599 600 if (sym_op->m_dst) 601 mbuf = sym_op->m_dst; 602 else 603 mbuf = sym_op->m_src; 604 605 /* first FLE entry used to store mbuf and session ctxt */ 606 fle = (struct qbman_fle *)rte_malloc(NULL, 607 FLE_SG_MEM_SIZE(mbuf->nb_segs + sym_op->m_src->nb_segs), 608 RTE_CACHE_LINE_SIZE); 609 if (unlikely(!fle)) { 610 DPAA2_SEC_ERR("AUTHENC SG: Memory alloc failed for SGE"); 611 return -ENOMEM; 612 } 613 memset(fle, 0, FLE_SG_MEM_SIZE(mbuf->nb_segs + sym_op->m_src->nb_segs)); 614 DPAA2_SET_FLE_ADDR(fle, (size_t)op); 615 DPAA2_FLE_SAVE_CTXT(fle, (ptrdiff_t)priv); 616 617 op_fle = fle + 1; 618 ip_fle = fle + 2; 619 sge = fle + 3; 620 621 /* Save the shared descriptor */ 622 flc = &priv->flc_desc[0].flc; 623 624 /* Configure FD as a FRAME LIST */ 625 DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(op_fle)); 626 DPAA2_SET_FD_COMPOUND_FMT(fd); 627 DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc)); 628 629 DPAA2_SEC_DP_DEBUG( 630 "AUTHENC SG: auth_off: 0x%x/length %d, digest-len=%d\n" 631 "cipher_off: 0x%x/length %d, iv-len=%d data_off: 0x%x\n", 632 sym_op->auth.data.offset, 633 sym_op->auth.data.length, 634 sess->digest_length, 635 sym_op->cipher.data.offset, 636 sym_op->cipher.data.length, 637 sess->iv.length, 638 sym_op->m_src->data_off); 639 640 /* Configure Output FLE with Scatter/Gather Entry */ 641 DPAA2_SET_FLE_SG_EXT(op_fle); 642 DPAA2_SET_FLE_ADDR(op_fle, DPAA2_VADDR_TO_IOVA(sge)); 643 644 if (auth_only_len) 645 DPAA2_SET_FLE_INTERNAL_JD(op_fle, auth_only_len); 646 647 op_fle->length = (sess->dir == DIR_ENC) ? 648 (sym_op->cipher.data.length + icv_len) : 649 sym_op->cipher.data.length; 650 651 /* Configure Output SGE for Encap/Decap */ 652 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf)); 653 DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off + sym_op->auth.data.offset); 654 sge->length = mbuf->data_len - sym_op->auth.data.offset; 655 656 mbuf = mbuf->next; 657 /* o/p segs */ 658 while (mbuf) { 659 sge++; 660 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf)); 661 DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off); 662 sge->length = mbuf->data_len; 663 mbuf = mbuf->next; 664 } 665 sge->length -= icv_len; 666 667 if (sess->dir == DIR_ENC) { 668 sge++; 669 DPAA2_SET_FLE_ADDR(sge, 670 DPAA2_VADDR_TO_IOVA(sym_op->auth.digest.data)); 671 sge->length = icv_len; 672 } 673 DPAA2_SET_FLE_FIN(sge); 674 675 sge++; 676 mbuf = sym_op->m_src; 677 678 /* Configure Input FLE with Scatter/Gather Entry */ 679 DPAA2_SET_FLE_ADDR(ip_fle, DPAA2_VADDR_TO_IOVA(sge)); 680 DPAA2_SET_FLE_SG_EXT(ip_fle); 681 DPAA2_SET_FLE_FIN(ip_fle); 682 ip_fle->length = (sess->dir == DIR_ENC) ? 683 (sym_op->auth.data.length + sess->iv.length) : 684 (sym_op->auth.data.length + sess->iv.length + 685 icv_len); 686 687 /* Configure Input SGE for Encap/Decap */ 688 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(iv_ptr)); 689 sge->length = sess->iv.length; 690 691 sge++; 692 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf)); 693 DPAA2_SET_FLE_OFFSET(sge, sym_op->auth.data.offset + 694 mbuf->data_off); 695 sge->length = mbuf->data_len - sym_op->auth.data.offset; 696 697 mbuf = mbuf->next; 698 /* i/p segs */ 699 while (mbuf) { 700 sge++; 701 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf)); 702 DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off); 703 sge->length = mbuf->data_len; 704 mbuf = mbuf->next; 705 } 706 sge->length -= icv_len; 707 708 if (sess->dir == DIR_DEC) { 709 sge++; 710 old_icv = (uint8_t *)(sge + 1); 711 memcpy(old_icv, sym_op->auth.digest.data, 712 icv_len); 713 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(old_icv)); 714 sge->length = icv_len; 715 } 716 717 DPAA2_SET_FLE_FIN(sge); 718 if (auth_only_len) { 719 DPAA2_SET_FLE_INTERNAL_JD(ip_fle, auth_only_len); 720 DPAA2_SET_FD_INTERNAL_JD(fd, auth_only_len); 721 } 722 DPAA2_SET_FD_LEN(fd, ip_fle->length); 723 724 return 0; 725 } 726 727 static inline int 728 build_authenc_fd(dpaa2_sec_session *sess, 729 struct rte_crypto_op *op, 730 struct qbman_fd *fd, uint16_t bpid) 731 { 732 struct rte_crypto_sym_op *sym_op = op->sym; 733 struct ctxt_priv *priv = sess->ctxt; 734 struct qbman_fle *fle, *sge; 735 struct sec_flow_context *flc; 736 uint16_t auth_hdr_len = sym_op->cipher.data.offset - 737 sym_op->auth.data.offset; 738 uint16_t auth_tail_len = sym_op->auth.data.length - 739 sym_op->cipher.data.length - auth_hdr_len; 740 uint32_t auth_only_len = (auth_tail_len << 16) | auth_hdr_len; 741 742 int icv_len = sess->digest_length, retval; 743 uint8_t *old_icv; 744 uint8_t *iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, 745 sess->iv.offset); 746 struct rte_mbuf *dst; 747 748 if (sym_op->m_dst) 749 dst = sym_op->m_dst; 750 else 751 dst = sym_op->m_src; 752 753 /* we are using the first FLE entry to store Mbuf. 754 * Currently we donot know which FLE has the mbuf stored. 755 * So while retreiving we can go back 1 FLE from the FD -ADDR 756 * to get the MBUF Addr from the previous FLE. 757 * We can have a better approach to use the inline Mbuf 758 */ 759 retval = rte_mempool_get(priv->fle_pool, (void **)(&fle)); 760 if (retval) { 761 DPAA2_SEC_ERR("Memory alloc failed for SGE"); 762 return -ENOMEM; 763 } 764 memset(fle, 0, FLE_POOL_BUF_SIZE); 765 DPAA2_SET_FLE_ADDR(fle, (size_t)op); 766 DPAA2_FLE_SAVE_CTXT(fle, (ptrdiff_t)priv); 767 fle = fle + 1; 768 sge = fle + 2; 769 if (likely(bpid < MAX_BPID)) { 770 DPAA2_SET_FD_BPID(fd, bpid); 771 DPAA2_SET_FLE_BPID(fle, bpid); 772 DPAA2_SET_FLE_BPID(fle + 1, bpid); 773 DPAA2_SET_FLE_BPID(sge, bpid); 774 DPAA2_SET_FLE_BPID(sge + 1, bpid); 775 DPAA2_SET_FLE_BPID(sge + 2, bpid); 776 DPAA2_SET_FLE_BPID(sge + 3, bpid); 777 } else { 778 DPAA2_SET_FD_IVP(fd); 779 DPAA2_SET_FLE_IVP(fle); 780 DPAA2_SET_FLE_IVP((fle + 1)); 781 DPAA2_SET_FLE_IVP(sge); 782 DPAA2_SET_FLE_IVP((sge + 1)); 783 DPAA2_SET_FLE_IVP((sge + 2)); 784 DPAA2_SET_FLE_IVP((sge + 3)); 785 } 786 787 /* Save the shared descriptor */ 788 flc = &priv->flc_desc[0].flc; 789 /* Configure FD as a FRAME LIST */ 790 DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(fle)); 791 DPAA2_SET_FD_COMPOUND_FMT(fd); 792 DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc)); 793 794 DPAA2_SEC_DP_DEBUG( 795 "AUTHENC: auth_off: 0x%x/length %d, digest-len=%d\n" 796 "cipher_off: 0x%x/length %d, iv-len=%d data_off: 0x%x\n", 797 sym_op->auth.data.offset, 798 sym_op->auth.data.length, 799 sess->digest_length, 800 sym_op->cipher.data.offset, 801 sym_op->cipher.data.length, 802 sess->iv.length, 803 sym_op->m_src->data_off); 804 805 /* Configure Output FLE with Scatter/Gather Entry */ 806 DPAA2_SET_FLE_ADDR(fle, DPAA2_VADDR_TO_IOVA(sge)); 807 if (auth_only_len) 808 DPAA2_SET_FLE_INTERNAL_JD(fle, auth_only_len); 809 fle->length = (sess->dir == DIR_ENC) ? 810 (sym_op->cipher.data.length + icv_len) : 811 sym_op->cipher.data.length; 812 813 DPAA2_SET_FLE_SG_EXT(fle); 814 815 /* Configure Output SGE for Encap/Decap */ 816 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(dst)); 817 DPAA2_SET_FLE_OFFSET(sge, sym_op->cipher.data.offset + 818 dst->data_off); 819 sge->length = sym_op->cipher.data.length; 820 821 if (sess->dir == DIR_ENC) { 822 sge++; 823 DPAA2_SET_FLE_ADDR(sge, 824 DPAA2_VADDR_TO_IOVA(sym_op->auth.digest.data)); 825 sge->length = sess->digest_length; 826 DPAA2_SET_FD_LEN(fd, (sym_op->auth.data.length + 827 sess->iv.length)); 828 } 829 DPAA2_SET_FLE_FIN(sge); 830 831 sge++; 832 fle++; 833 834 /* Configure Input FLE with Scatter/Gather Entry */ 835 DPAA2_SET_FLE_ADDR(fle, DPAA2_VADDR_TO_IOVA(sge)); 836 DPAA2_SET_FLE_SG_EXT(fle); 837 DPAA2_SET_FLE_FIN(fle); 838 fle->length = (sess->dir == DIR_ENC) ? 839 (sym_op->auth.data.length + sess->iv.length) : 840 (sym_op->auth.data.length + sess->iv.length + 841 sess->digest_length); 842 843 /* Configure Input SGE for Encap/Decap */ 844 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(iv_ptr)); 845 sge->length = sess->iv.length; 846 sge++; 847 848 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(sym_op->m_src)); 849 DPAA2_SET_FLE_OFFSET(sge, sym_op->auth.data.offset + 850 sym_op->m_src->data_off); 851 sge->length = sym_op->auth.data.length; 852 if (sess->dir == DIR_DEC) { 853 sge++; 854 old_icv = (uint8_t *)(sge + 1); 855 memcpy(old_icv, sym_op->auth.digest.data, 856 sess->digest_length); 857 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(old_icv)); 858 sge->length = sess->digest_length; 859 DPAA2_SET_FD_LEN(fd, (sym_op->auth.data.length + 860 sess->digest_length + 861 sess->iv.length)); 862 } 863 DPAA2_SET_FLE_FIN(sge); 864 if (auth_only_len) { 865 DPAA2_SET_FLE_INTERNAL_JD(fle, auth_only_len); 866 DPAA2_SET_FD_INTERNAL_JD(fd, auth_only_len); 867 } 868 return 0; 869 } 870 871 static inline int build_auth_sg_fd( 872 dpaa2_sec_session *sess, 873 struct rte_crypto_op *op, 874 struct qbman_fd *fd, 875 __rte_unused uint16_t bpid) 876 { 877 struct rte_crypto_sym_op *sym_op = op->sym; 878 struct qbman_fle *fle, *sge, *ip_fle, *op_fle; 879 struct sec_flow_context *flc; 880 struct ctxt_priv *priv = sess->ctxt; 881 int data_len, data_offset; 882 uint8_t *old_digest; 883 struct rte_mbuf *mbuf; 884 885 data_len = sym_op->auth.data.length; 886 data_offset = sym_op->auth.data.offset; 887 888 if (sess->auth_alg == RTE_CRYPTO_AUTH_SNOW3G_UIA2 || 889 sess->auth_alg == RTE_CRYPTO_AUTH_ZUC_EIA3) { 890 if ((data_len & 7) || (data_offset & 7)) { 891 DPAA2_SEC_ERR("AUTH: len/offset must be full bytes"); 892 return -ENOTSUP; 893 } 894 895 data_len = data_len >> 3; 896 data_offset = data_offset >> 3; 897 } 898 899 mbuf = sym_op->m_src; 900 fle = (struct qbman_fle *)rte_malloc(NULL, 901 FLE_SG_MEM_SIZE(mbuf->nb_segs), 902 RTE_CACHE_LINE_SIZE); 903 if (unlikely(!fle)) { 904 DPAA2_SEC_ERR("AUTH SG: Memory alloc failed for SGE"); 905 return -ENOMEM; 906 } 907 memset(fle, 0, FLE_SG_MEM_SIZE(mbuf->nb_segs)); 908 /* first FLE entry used to store mbuf and session ctxt */ 909 DPAA2_SET_FLE_ADDR(fle, (size_t)op); 910 DPAA2_FLE_SAVE_CTXT(fle, (ptrdiff_t)priv); 911 op_fle = fle + 1; 912 ip_fle = fle + 2; 913 sge = fle + 3; 914 915 flc = &priv->flc_desc[DESC_INITFINAL].flc; 916 /* sg FD */ 917 DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc)); 918 DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(op_fle)); 919 DPAA2_SET_FD_COMPOUND_FMT(fd); 920 921 /* o/p fle */ 922 DPAA2_SET_FLE_ADDR(op_fle, 923 DPAA2_VADDR_TO_IOVA(sym_op->auth.digest.data)); 924 op_fle->length = sess->digest_length; 925 926 /* i/p fle */ 927 DPAA2_SET_FLE_SG_EXT(ip_fle); 928 DPAA2_SET_FLE_ADDR(ip_fle, DPAA2_VADDR_TO_IOVA(sge)); 929 ip_fle->length = data_len; 930 931 if (sess->iv.length) { 932 uint8_t *iv_ptr; 933 934 iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, 935 sess->iv.offset); 936 937 if (sess->auth_alg == RTE_CRYPTO_AUTH_SNOW3G_UIA2) { 938 iv_ptr = conv_to_snow_f9_iv(iv_ptr); 939 sge->length = 12; 940 } else if (sess->auth_alg == RTE_CRYPTO_AUTH_ZUC_EIA3) { 941 iv_ptr = conv_to_zuc_eia_iv(iv_ptr); 942 sge->length = 8; 943 } else { 944 sge->length = sess->iv.length; 945 } 946 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(iv_ptr)); 947 ip_fle->length += sge->length; 948 sge++; 949 } 950 /* i/p 1st seg */ 951 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf)); 952 DPAA2_SET_FLE_OFFSET(sge, data_offset + mbuf->data_off); 953 954 if (data_len <= (mbuf->data_len - data_offset)) { 955 sge->length = data_len; 956 data_len = 0; 957 } else { 958 sge->length = mbuf->data_len - data_offset; 959 960 /* remaining i/p segs */ 961 while ((data_len = data_len - sge->length) && 962 (mbuf = mbuf->next)) { 963 sge++; 964 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf)); 965 DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off); 966 if (data_len > mbuf->data_len) 967 sge->length = mbuf->data_len; 968 else 969 sge->length = data_len; 970 } 971 } 972 973 if (sess->dir == DIR_DEC) { 974 /* Digest verification case */ 975 sge++; 976 old_digest = (uint8_t *)(sge + 1); 977 rte_memcpy(old_digest, sym_op->auth.digest.data, 978 sess->digest_length); 979 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(old_digest)); 980 sge->length = sess->digest_length; 981 ip_fle->length += sess->digest_length; 982 } 983 DPAA2_SET_FLE_FIN(sge); 984 DPAA2_SET_FLE_FIN(ip_fle); 985 DPAA2_SET_FD_LEN(fd, ip_fle->length); 986 987 return 0; 988 } 989 990 static inline int 991 build_auth_fd(dpaa2_sec_session *sess, struct rte_crypto_op *op, 992 struct qbman_fd *fd, uint16_t bpid) 993 { 994 struct rte_crypto_sym_op *sym_op = op->sym; 995 struct qbman_fle *fle, *sge; 996 struct sec_flow_context *flc; 997 struct ctxt_priv *priv = sess->ctxt; 998 int data_len, data_offset; 999 uint8_t *old_digest; 1000 int retval; 1001 1002 data_len = sym_op->auth.data.length; 1003 data_offset = sym_op->auth.data.offset; 1004 1005 if (sess->auth_alg == RTE_CRYPTO_AUTH_SNOW3G_UIA2 || 1006 sess->auth_alg == RTE_CRYPTO_AUTH_ZUC_EIA3) { 1007 if ((data_len & 7) || (data_offset & 7)) { 1008 DPAA2_SEC_ERR("AUTH: len/offset must be full bytes"); 1009 return -ENOTSUP; 1010 } 1011 1012 data_len = data_len >> 3; 1013 data_offset = data_offset >> 3; 1014 } 1015 1016 retval = rte_mempool_get(priv->fle_pool, (void **)(&fle)); 1017 if (retval) { 1018 DPAA2_SEC_ERR("AUTH Memory alloc failed for SGE"); 1019 return -ENOMEM; 1020 } 1021 memset(fle, 0, FLE_POOL_BUF_SIZE); 1022 /* TODO we are using the first FLE entry to store Mbuf. 1023 * Currently we donot know which FLE has the mbuf stored. 1024 * So while retreiving we can go back 1 FLE from the FD -ADDR 1025 * to get the MBUF Addr from the previous FLE. 1026 * We can have a better approach to use the inline Mbuf 1027 */ 1028 DPAA2_SET_FLE_ADDR(fle, (size_t)op); 1029 DPAA2_FLE_SAVE_CTXT(fle, (ptrdiff_t)priv); 1030 fle = fle + 1; 1031 sge = fle + 2; 1032 1033 if (likely(bpid < MAX_BPID)) { 1034 DPAA2_SET_FD_BPID(fd, bpid); 1035 DPAA2_SET_FLE_BPID(fle, bpid); 1036 DPAA2_SET_FLE_BPID(fle + 1, bpid); 1037 DPAA2_SET_FLE_BPID(sge, bpid); 1038 DPAA2_SET_FLE_BPID(sge + 1, bpid); 1039 } else { 1040 DPAA2_SET_FD_IVP(fd); 1041 DPAA2_SET_FLE_IVP(fle); 1042 DPAA2_SET_FLE_IVP((fle + 1)); 1043 DPAA2_SET_FLE_IVP(sge); 1044 DPAA2_SET_FLE_IVP((sge + 1)); 1045 } 1046 1047 flc = &priv->flc_desc[DESC_INITFINAL].flc; 1048 DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc)); 1049 DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(fle)); 1050 DPAA2_SET_FD_COMPOUND_FMT(fd); 1051 1052 DPAA2_SET_FLE_ADDR(fle, DPAA2_VADDR_TO_IOVA(sym_op->auth.digest.data)); 1053 fle->length = sess->digest_length; 1054 fle++; 1055 1056 /* Setting input FLE */ 1057 DPAA2_SET_FLE_ADDR(fle, DPAA2_VADDR_TO_IOVA(sge)); 1058 DPAA2_SET_FLE_SG_EXT(fle); 1059 fle->length = data_len; 1060 1061 if (sess->iv.length) { 1062 uint8_t *iv_ptr; 1063 1064 iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, 1065 sess->iv.offset); 1066 1067 if (sess->auth_alg == RTE_CRYPTO_AUTH_SNOW3G_UIA2) { 1068 iv_ptr = conv_to_snow_f9_iv(iv_ptr); 1069 sge->length = 12; 1070 } else if (sess->auth_alg == RTE_CRYPTO_AUTH_ZUC_EIA3) { 1071 iv_ptr = conv_to_zuc_eia_iv(iv_ptr); 1072 sge->length = 8; 1073 } else { 1074 sge->length = sess->iv.length; 1075 } 1076 1077 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(iv_ptr)); 1078 fle->length = fle->length + sge->length; 1079 sge++; 1080 } 1081 1082 /* Setting data to authenticate */ 1083 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(sym_op->m_src)); 1084 DPAA2_SET_FLE_OFFSET(sge, data_offset + sym_op->m_src->data_off); 1085 sge->length = data_len; 1086 1087 if (sess->dir == DIR_DEC) { 1088 sge++; 1089 old_digest = (uint8_t *)(sge + 1); 1090 rte_memcpy(old_digest, sym_op->auth.digest.data, 1091 sess->digest_length); 1092 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(old_digest)); 1093 sge->length = sess->digest_length; 1094 fle->length = fle->length + sess->digest_length; 1095 } 1096 1097 DPAA2_SET_FLE_FIN(sge); 1098 DPAA2_SET_FLE_FIN(fle); 1099 DPAA2_SET_FD_LEN(fd, fle->length); 1100 1101 return 0; 1102 } 1103 1104 static int 1105 build_cipher_sg_fd(dpaa2_sec_session *sess, struct rte_crypto_op *op, 1106 struct qbman_fd *fd, __rte_unused uint16_t bpid) 1107 { 1108 struct rte_crypto_sym_op *sym_op = op->sym; 1109 struct qbman_fle *ip_fle, *op_fle, *sge, *fle; 1110 int data_len, data_offset; 1111 struct sec_flow_context *flc; 1112 struct ctxt_priv *priv = sess->ctxt; 1113 struct rte_mbuf *mbuf; 1114 uint8_t *iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, 1115 sess->iv.offset); 1116 1117 data_len = sym_op->cipher.data.length; 1118 data_offset = sym_op->cipher.data.offset; 1119 1120 if (sess->cipher_alg == RTE_CRYPTO_CIPHER_SNOW3G_UEA2 || 1121 sess->cipher_alg == RTE_CRYPTO_CIPHER_ZUC_EEA3) { 1122 if ((data_len & 7) || (data_offset & 7)) { 1123 DPAA2_SEC_ERR("CIPHER: len/offset must be full bytes"); 1124 return -ENOTSUP; 1125 } 1126 1127 data_len = data_len >> 3; 1128 data_offset = data_offset >> 3; 1129 } 1130 1131 if (sym_op->m_dst) 1132 mbuf = sym_op->m_dst; 1133 else 1134 mbuf = sym_op->m_src; 1135 1136 /* first FLE entry used to store mbuf and session ctxt */ 1137 fle = (struct qbman_fle *)rte_malloc(NULL, 1138 FLE_SG_MEM_SIZE(mbuf->nb_segs + sym_op->m_src->nb_segs), 1139 RTE_CACHE_LINE_SIZE); 1140 if (!fle) { 1141 DPAA2_SEC_ERR("CIPHER SG: Memory alloc failed for SGE"); 1142 return -ENOMEM; 1143 } 1144 memset(fle, 0, FLE_SG_MEM_SIZE(mbuf->nb_segs + sym_op->m_src->nb_segs)); 1145 /* first FLE entry used to store mbuf and session ctxt */ 1146 DPAA2_SET_FLE_ADDR(fle, (size_t)op); 1147 DPAA2_FLE_SAVE_CTXT(fle, (ptrdiff_t)priv); 1148 1149 op_fle = fle + 1; 1150 ip_fle = fle + 2; 1151 sge = fle + 3; 1152 1153 flc = &priv->flc_desc[0].flc; 1154 1155 DPAA2_SEC_DP_DEBUG( 1156 "CIPHER SG: cipher_off: 0x%x/length %d, ivlen=%d" 1157 " data_off: 0x%x\n", 1158 data_offset, 1159 data_len, 1160 sess->iv.length, 1161 sym_op->m_src->data_off); 1162 1163 /* o/p fle */ 1164 DPAA2_SET_FLE_ADDR(op_fle, DPAA2_VADDR_TO_IOVA(sge)); 1165 op_fle->length = data_len; 1166 DPAA2_SET_FLE_SG_EXT(op_fle); 1167 1168 /* o/p 1st seg */ 1169 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf)); 1170 DPAA2_SET_FLE_OFFSET(sge, data_offset + mbuf->data_off); 1171 sge->length = mbuf->data_len - data_offset; 1172 1173 mbuf = mbuf->next; 1174 /* o/p segs */ 1175 while (mbuf) { 1176 sge++; 1177 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf)); 1178 DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off); 1179 sge->length = mbuf->data_len; 1180 mbuf = mbuf->next; 1181 } 1182 DPAA2_SET_FLE_FIN(sge); 1183 1184 DPAA2_SEC_DP_DEBUG( 1185 "CIPHER SG: 1 - flc = %p, fle = %p FLEaddr = %x-%x, len %d\n", 1186 flc, fle, fle->addr_hi, fle->addr_lo, 1187 fle->length); 1188 1189 /* i/p fle */ 1190 mbuf = sym_op->m_src; 1191 sge++; 1192 DPAA2_SET_FLE_ADDR(ip_fle, DPAA2_VADDR_TO_IOVA(sge)); 1193 ip_fle->length = sess->iv.length + data_len; 1194 DPAA2_SET_FLE_SG_EXT(ip_fle); 1195 1196 /* i/p IV */ 1197 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(iv_ptr)); 1198 DPAA2_SET_FLE_OFFSET(sge, 0); 1199 sge->length = sess->iv.length; 1200 1201 sge++; 1202 1203 /* i/p 1st seg */ 1204 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf)); 1205 DPAA2_SET_FLE_OFFSET(sge, data_offset + mbuf->data_off); 1206 sge->length = mbuf->data_len - data_offset; 1207 1208 mbuf = mbuf->next; 1209 /* i/p segs */ 1210 while (mbuf) { 1211 sge++; 1212 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(mbuf)); 1213 DPAA2_SET_FLE_OFFSET(sge, mbuf->data_off); 1214 sge->length = mbuf->data_len; 1215 mbuf = mbuf->next; 1216 } 1217 DPAA2_SET_FLE_FIN(sge); 1218 DPAA2_SET_FLE_FIN(ip_fle); 1219 1220 /* sg fd */ 1221 DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(op_fle)); 1222 DPAA2_SET_FD_LEN(fd, ip_fle->length); 1223 DPAA2_SET_FD_COMPOUND_FMT(fd); 1224 DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc)); 1225 1226 DPAA2_SEC_DP_DEBUG( 1227 "CIPHER SG: fdaddr =%" PRIx64 " bpid =%d meta =%d" 1228 " off =%d, len =%d\n", 1229 DPAA2_GET_FD_ADDR(fd), 1230 DPAA2_GET_FD_BPID(fd), 1231 rte_dpaa2_bpid_info[bpid].meta_data_size, 1232 DPAA2_GET_FD_OFFSET(fd), 1233 DPAA2_GET_FD_LEN(fd)); 1234 return 0; 1235 } 1236 1237 static int 1238 build_cipher_fd(dpaa2_sec_session *sess, struct rte_crypto_op *op, 1239 struct qbman_fd *fd, uint16_t bpid) 1240 { 1241 struct rte_crypto_sym_op *sym_op = op->sym; 1242 struct qbman_fle *fle, *sge; 1243 int retval, data_len, data_offset; 1244 struct sec_flow_context *flc; 1245 struct ctxt_priv *priv = sess->ctxt; 1246 uint8_t *iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, 1247 sess->iv.offset); 1248 struct rte_mbuf *dst; 1249 1250 data_len = sym_op->cipher.data.length; 1251 data_offset = sym_op->cipher.data.offset; 1252 1253 if (sess->cipher_alg == RTE_CRYPTO_CIPHER_SNOW3G_UEA2 || 1254 sess->cipher_alg == RTE_CRYPTO_CIPHER_ZUC_EEA3) { 1255 if ((data_len & 7) || (data_offset & 7)) { 1256 DPAA2_SEC_ERR("CIPHER: len/offset must be full bytes"); 1257 return -ENOTSUP; 1258 } 1259 1260 data_len = data_len >> 3; 1261 data_offset = data_offset >> 3; 1262 } 1263 1264 if (sym_op->m_dst) 1265 dst = sym_op->m_dst; 1266 else 1267 dst = sym_op->m_src; 1268 1269 retval = rte_mempool_get(priv->fle_pool, (void **)(&fle)); 1270 if (retval) { 1271 DPAA2_SEC_ERR("CIPHER: Memory alloc failed for SGE"); 1272 return -ENOMEM; 1273 } 1274 memset(fle, 0, FLE_POOL_BUF_SIZE); 1275 /* TODO we are using the first FLE entry to store Mbuf. 1276 * Currently we donot know which FLE has the mbuf stored. 1277 * So while retreiving we can go back 1 FLE from the FD -ADDR 1278 * to get the MBUF Addr from the previous FLE. 1279 * We can have a better approach to use the inline Mbuf 1280 */ 1281 DPAA2_SET_FLE_ADDR(fle, (size_t)op); 1282 DPAA2_FLE_SAVE_CTXT(fle, (ptrdiff_t)priv); 1283 fle = fle + 1; 1284 sge = fle + 2; 1285 1286 if (likely(bpid < MAX_BPID)) { 1287 DPAA2_SET_FD_BPID(fd, bpid); 1288 DPAA2_SET_FLE_BPID(fle, bpid); 1289 DPAA2_SET_FLE_BPID(fle + 1, bpid); 1290 DPAA2_SET_FLE_BPID(sge, bpid); 1291 DPAA2_SET_FLE_BPID(sge + 1, bpid); 1292 } else { 1293 DPAA2_SET_FD_IVP(fd); 1294 DPAA2_SET_FLE_IVP(fle); 1295 DPAA2_SET_FLE_IVP((fle + 1)); 1296 DPAA2_SET_FLE_IVP(sge); 1297 DPAA2_SET_FLE_IVP((sge + 1)); 1298 } 1299 1300 flc = &priv->flc_desc[0].flc; 1301 DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(fle)); 1302 DPAA2_SET_FD_LEN(fd, data_len + sess->iv.length); 1303 DPAA2_SET_FD_COMPOUND_FMT(fd); 1304 DPAA2_SET_FD_FLC(fd, DPAA2_VADDR_TO_IOVA(flc)); 1305 1306 DPAA2_SEC_DP_DEBUG( 1307 "CIPHER: cipher_off: 0x%x/length %d, ivlen=%d," 1308 " data_off: 0x%x\n", 1309 data_offset, 1310 data_len, 1311 sess->iv.length, 1312 sym_op->m_src->data_off); 1313 1314 DPAA2_SET_FLE_ADDR(fle, DPAA2_MBUF_VADDR_TO_IOVA(dst)); 1315 DPAA2_SET_FLE_OFFSET(fle, data_offset + dst->data_off); 1316 1317 fle->length = data_len + sess->iv.length; 1318 1319 DPAA2_SEC_DP_DEBUG( 1320 "CIPHER: 1 - flc = %p, fle = %p FLEaddr = %x-%x, length %d\n", 1321 flc, fle, fle->addr_hi, fle->addr_lo, 1322 fle->length); 1323 1324 fle++; 1325 1326 DPAA2_SET_FLE_ADDR(fle, DPAA2_VADDR_TO_IOVA(sge)); 1327 fle->length = data_len + sess->iv.length; 1328 1329 DPAA2_SET_FLE_SG_EXT(fle); 1330 1331 DPAA2_SET_FLE_ADDR(sge, DPAA2_VADDR_TO_IOVA(iv_ptr)); 1332 sge->length = sess->iv.length; 1333 1334 sge++; 1335 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(sym_op->m_src)); 1336 DPAA2_SET_FLE_OFFSET(sge, data_offset + sym_op->m_src->data_off); 1337 1338 sge->length = data_len; 1339 DPAA2_SET_FLE_FIN(sge); 1340 DPAA2_SET_FLE_FIN(fle); 1341 1342 DPAA2_SEC_DP_DEBUG( 1343 "CIPHER: fdaddr =%" PRIx64 " bpid =%d meta =%d" 1344 " off =%d, len =%d\n", 1345 DPAA2_GET_FD_ADDR(fd), 1346 DPAA2_GET_FD_BPID(fd), 1347 rte_dpaa2_bpid_info[bpid].meta_data_size, 1348 DPAA2_GET_FD_OFFSET(fd), 1349 DPAA2_GET_FD_LEN(fd)); 1350 1351 return 0; 1352 } 1353 1354 static inline int 1355 build_sec_fd(struct rte_crypto_op *op, 1356 struct qbman_fd *fd, uint16_t bpid) 1357 { 1358 int ret = -1; 1359 dpaa2_sec_session *sess; 1360 1361 if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) 1362 sess = (dpaa2_sec_session *)get_sym_session_private_data( 1363 op->sym->session, cryptodev_driver_id); 1364 #ifdef RTE_LIB_SECURITY 1365 else if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) 1366 sess = (dpaa2_sec_session *)get_sec_session_private_data( 1367 op->sym->sec_session); 1368 #endif 1369 else 1370 return -ENOTSUP; 1371 1372 if (!sess) 1373 return -EINVAL; 1374 1375 /* Any of the buffer is segmented*/ 1376 if (!rte_pktmbuf_is_contiguous(op->sym->m_src) || 1377 ((op->sym->m_dst != NULL) && 1378 !rte_pktmbuf_is_contiguous(op->sym->m_dst))) { 1379 switch (sess->ctxt_type) { 1380 case DPAA2_SEC_CIPHER: 1381 ret = build_cipher_sg_fd(sess, op, fd, bpid); 1382 break; 1383 case DPAA2_SEC_AUTH: 1384 ret = build_auth_sg_fd(sess, op, fd, bpid); 1385 break; 1386 case DPAA2_SEC_AEAD: 1387 ret = build_authenc_gcm_sg_fd(sess, op, fd, bpid); 1388 break; 1389 case DPAA2_SEC_CIPHER_HASH: 1390 ret = build_authenc_sg_fd(sess, op, fd, bpid); 1391 break; 1392 #ifdef RTE_LIB_SECURITY 1393 case DPAA2_SEC_IPSEC: 1394 case DPAA2_SEC_PDCP: 1395 ret = build_proto_compound_sg_fd(sess, op, fd, bpid); 1396 break; 1397 #endif 1398 case DPAA2_SEC_HASH_CIPHER: 1399 default: 1400 DPAA2_SEC_ERR("error: Unsupported session"); 1401 } 1402 } else { 1403 switch (sess->ctxt_type) { 1404 case DPAA2_SEC_CIPHER: 1405 ret = build_cipher_fd(sess, op, fd, bpid); 1406 break; 1407 case DPAA2_SEC_AUTH: 1408 ret = build_auth_fd(sess, op, fd, bpid); 1409 break; 1410 case DPAA2_SEC_AEAD: 1411 ret = build_authenc_gcm_fd(sess, op, fd, bpid); 1412 break; 1413 case DPAA2_SEC_CIPHER_HASH: 1414 ret = build_authenc_fd(sess, op, fd, bpid); 1415 break; 1416 #ifdef RTE_LIB_SECURITY 1417 case DPAA2_SEC_IPSEC: 1418 ret = build_proto_fd(sess, op, fd, bpid); 1419 break; 1420 case DPAA2_SEC_PDCP: 1421 ret = build_proto_compound_fd(sess, op, fd, bpid); 1422 break; 1423 #endif 1424 case DPAA2_SEC_HASH_CIPHER: 1425 default: 1426 DPAA2_SEC_ERR("error: Unsupported session"); 1427 ret = -ENOTSUP; 1428 } 1429 } 1430 return ret; 1431 } 1432 1433 static uint16_t 1434 dpaa2_sec_enqueue_burst(void *qp, struct rte_crypto_op **ops, 1435 uint16_t nb_ops) 1436 { 1437 /* Function to transmit the frames to given device and VQ*/ 1438 uint32_t loop; 1439 int32_t ret; 1440 struct qbman_fd fd_arr[MAX_TX_RING_SLOTS]; 1441 uint32_t frames_to_send, retry_count; 1442 struct qbman_eq_desc eqdesc; 1443 struct dpaa2_sec_qp *dpaa2_qp = (struct dpaa2_sec_qp *)qp; 1444 struct qbman_swp *swp; 1445 uint16_t num_tx = 0; 1446 uint32_t flags[MAX_TX_RING_SLOTS] = {0}; 1447 /*todo - need to support multiple buffer pools */ 1448 uint16_t bpid; 1449 struct rte_mempool *mb_pool; 1450 1451 if (unlikely(nb_ops == 0)) 1452 return 0; 1453 1454 if (ops[0]->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { 1455 DPAA2_SEC_ERR("sessionless crypto op not supported"); 1456 return 0; 1457 } 1458 /*Prepare enqueue descriptor*/ 1459 qbman_eq_desc_clear(&eqdesc); 1460 qbman_eq_desc_set_no_orp(&eqdesc, DPAA2_EQ_RESP_ERR_FQ); 1461 qbman_eq_desc_set_response(&eqdesc, 0, 0); 1462 qbman_eq_desc_set_fq(&eqdesc, dpaa2_qp->tx_vq.fqid); 1463 1464 if (!DPAA2_PER_LCORE_DPIO) { 1465 ret = dpaa2_affine_qbman_swp(); 1466 if (ret) { 1467 DPAA2_SEC_ERR( 1468 "Failed to allocate IO portal, tid: %d\n", 1469 rte_gettid()); 1470 return 0; 1471 } 1472 } 1473 swp = DPAA2_PER_LCORE_PORTAL; 1474 1475 while (nb_ops) { 1476 frames_to_send = (nb_ops > dpaa2_eqcr_size) ? 1477 dpaa2_eqcr_size : nb_ops; 1478 1479 for (loop = 0; loop < frames_to_send; loop++) { 1480 if (*dpaa2_seqn((*ops)->sym->m_src)) { 1481 if (*dpaa2_seqn((*ops)->sym->m_src) & QBMAN_ENQUEUE_FLAG_DCA) { 1482 DPAA2_PER_LCORE_DQRR_SIZE--; 1483 DPAA2_PER_LCORE_DQRR_HELD &= ~(1 << 1484 *dpaa2_seqn((*ops)->sym->m_src) & 1485 QBMAN_EQCR_DCA_IDXMASK); 1486 } 1487 flags[loop] = *dpaa2_seqn((*ops)->sym->m_src); 1488 *dpaa2_seqn((*ops)->sym->m_src) = DPAA2_INVALID_MBUF_SEQN; 1489 } 1490 1491 /*Clear the unused FD fields before sending*/ 1492 memset(&fd_arr[loop], 0, sizeof(struct qbman_fd)); 1493 mb_pool = (*ops)->sym->m_src->pool; 1494 bpid = mempool_to_bpid(mb_pool); 1495 ret = build_sec_fd(*ops, &fd_arr[loop], bpid); 1496 if (ret) { 1497 DPAA2_SEC_ERR("error: Improper packet contents" 1498 " for crypto operation"); 1499 goto skip_tx; 1500 } 1501 ops++; 1502 } 1503 1504 loop = 0; 1505 retry_count = 0; 1506 while (loop < frames_to_send) { 1507 ret = qbman_swp_enqueue_multiple(swp, &eqdesc, 1508 &fd_arr[loop], 1509 &flags[loop], 1510 frames_to_send - loop); 1511 if (unlikely(ret < 0)) { 1512 retry_count++; 1513 if (retry_count > DPAA2_MAX_TX_RETRY_COUNT) { 1514 num_tx += loop; 1515 nb_ops -= loop; 1516 goto skip_tx; 1517 } 1518 } else { 1519 loop += ret; 1520 retry_count = 0; 1521 } 1522 } 1523 1524 num_tx += loop; 1525 nb_ops -= loop; 1526 } 1527 skip_tx: 1528 dpaa2_qp->tx_vq.tx_pkts += num_tx; 1529 dpaa2_qp->tx_vq.err_pkts += nb_ops; 1530 return num_tx; 1531 } 1532 1533 #ifdef RTE_LIB_SECURITY 1534 static inline struct rte_crypto_op * 1535 sec_simple_fd_to_mbuf(const struct qbman_fd *fd) 1536 { 1537 struct rte_crypto_op *op; 1538 uint16_t len = DPAA2_GET_FD_LEN(fd); 1539 int16_t diff = 0; 1540 dpaa2_sec_session *sess_priv __rte_unused; 1541 1542 struct rte_mbuf *mbuf = DPAA2_INLINE_MBUF_FROM_BUF( 1543 DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd)), 1544 rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size); 1545 1546 diff = len - mbuf->pkt_len; 1547 mbuf->pkt_len += diff; 1548 mbuf->data_len += diff; 1549 op = (struct rte_crypto_op *)(size_t)mbuf->buf_iova; 1550 mbuf->buf_iova = op->sym->aead.digest.phys_addr; 1551 op->sym->aead.digest.phys_addr = 0L; 1552 1553 sess_priv = (dpaa2_sec_session *)get_sec_session_private_data( 1554 op->sym->sec_session); 1555 if (sess_priv->dir == DIR_ENC) 1556 mbuf->data_off += SEC_FLC_DHR_OUTBOUND; 1557 else 1558 mbuf->data_off += SEC_FLC_DHR_INBOUND; 1559 1560 return op; 1561 } 1562 #endif 1563 1564 static inline struct rte_crypto_op * 1565 sec_fd_to_mbuf(const struct qbman_fd *fd) 1566 { 1567 struct qbman_fle *fle; 1568 struct rte_crypto_op *op; 1569 struct ctxt_priv *priv; 1570 struct rte_mbuf *dst, *src; 1571 1572 #ifdef RTE_LIB_SECURITY 1573 if (DPAA2_FD_GET_FORMAT(fd) == qbman_fd_single) 1574 return sec_simple_fd_to_mbuf(fd); 1575 #endif 1576 fle = (struct qbman_fle *)DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd)); 1577 1578 DPAA2_SEC_DP_DEBUG("FLE addr = %x - %x, offset = %x\n", 1579 fle->addr_hi, fle->addr_lo, fle->fin_bpid_offset); 1580 1581 /* we are using the first FLE entry to store Mbuf. 1582 * Currently we donot know which FLE has the mbuf stored. 1583 * So while retreiving we can go back 1 FLE from the FD -ADDR 1584 * to get the MBUF Addr from the previous FLE. 1585 * We can have a better approach to use the inline Mbuf 1586 */ 1587 1588 if (unlikely(DPAA2_GET_FD_IVP(fd))) { 1589 /* TODO complete it. */ 1590 DPAA2_SEC_ERR("error: non inline buffer"); 1591 return NULL; 1592 } 1593 op = (struct rte_crypto_op *)DPAA2_GET_FLE_ADDR((fle - 1)); 1594 1595 /* Prefeth op */ 1596 src = op->sym->m_src; 1597 rte_prefetch0(src); 1598 1599 if (op->sym->m_dst) { 1600 dst = op->sym->m_dst; 1601 rte_prefetch0(dst); 1602 } else 1603 dst = src; 1604 1605 #ifdef RTE_LIB_SECURITY 1606 if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) { 1607 uint16_t len = DPAA2_GET_FD_LEN(fd); 1608 dst->pkt_len = len; 1609 while (dst->next != NULL) { 1610 len -= dst->data_len; 1611 dst = dst->next; 1612 } 1613 dst->data_len = len; 1614 } 1615 #endif 1616 DPAA2_SEC_DP_DEBUG("mbuf %p BMAN buf addr %p," 1617 " fdaddr =%" PRIx64 " bpid =%d meta =%d off =%d, len =%d\n", 1618 (void *)dst, 1619 dst->buf_addr, 1620 DPAA2_GET_FD_ADDR(fd), 1621 DPAA2_GET_FD_BPID(fd), 1622 rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size, 1623 DPAA2_GET_FD_OFFSET(fd), 1624 DPAA2_GET_FD_LEN(fd)); 1625 1626 /* free the fle memory */ 1627 if (likely(rte_pktmbuf_is_contiguous(src))) { 1628 priv = (struct ctxt_priv *)(size_t)DPAA2_GET_FLE_CTXT(fle - 1); 1629 rte_mempool_put(priv->fle_pool, (void *)(fle-1)); 1630 } else 1631 rte_free((void *)(fle-1)); 1632 1633 return op; 1634 } 1635 1636 static void 1637 dpaa2_sec_dump(struct rte_crypto_op *op) 1638 { 1639 int i; 1640 dpaa2_sec_session *sess = NULL; 1641 struct ctxt_priv *priv; 1642 uint8_t bufsize; 1643 struct rte_crypto_sym_op *sym_op; 1644 1645 if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) 1646 sess = (dpaa2_sec_session *)get_sym_session_private_data( 1647 op->sym->session, cryptodev_driver_id); 1648 #ifdef RTE_LIBRTE_SECURITY 1649 else if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) 1650 sess = (dpaa2_sec_session *)get_sec_session_private_data( 1651 op->sym->sec_session); 1652 #endif 1653 1654 if (sess == NULL) 1655 goto mbuf_dump; 1656 1657 priv = (struct ctxt_priv *)sess->ctxt; 1658 printf("\n****************************************\n" 1659 "session params:\n\tContext type:\t%d\n\tDirection:\t%s\n" 1660 "\tCipher alg:\t%d\n\tAuth alg:\t%d\n\tAead alg:\t%d\n" 1661 "\tCipher key len:\t%zd\n", sess->ctxt_type, 1662 (sess->dir == DIR_ENC) ? "DIR_ENC" : "DIR_DEC", 1663 sess->cipher_alg, sess->auth_alg, sess->aead_alg, 1664 sess->cipher_key.length); 1665 rte_hexdump(stdout, "cipher key", sess->cipher_key.data, 1666 sess->cipher_key.length); 1667 rte_hexdump(stdout, "auth key", sess->auth_key.data, 1668 sess->auth_key.length); 1669 printf("\tAuth key len:\t%zd\n\tIV len:\t\t%d\n\tIV offset:\t%d\n" 1670 "\tdigest length:\t%d\n\tstatus:\t\t%d\n\taead auth only" 1671 " len:\t%d\n\taead cipher text:\t%d\n", 1672 sess->auth_key.length, sess->iv.length, sess->iv.offset, 1673 sess->digest_length, sess->status, 1674 sess->ext_params.aead_ctxt.auth_only_len, 1675 sess->ext_params.aead_ctxt.auth_cipher_text); 1676 #ifdef RTE_LIBRTE_SECURITY 1677 printf("PDCP session params:\n" 1678 "\tDomain:\t\t%d\n\tBearer:\t\t%d\n\tpkt_dir:\t%d\n\thfn_ovd:" 1679 "\t%d\n\tsn_size:\t%d\n\thfn_ovd_offset:\t%d\n\thfn:\t\t%d\n" 1680 "\thfn_threshold:\t0x%x\n", sess->pdcp.domain, 1681 sess->pdcp.bearer, sess->pdcp.pkt_dir, sess->pdcp.hfn_ovd, 1682 sess->pdcp.sn_size, sess->pdcp.hfn_ovd_offset, sess->pdcp.hfn, 1683 sess->pdcp.hfn_threshold); 1684 1685 #endif 1686 bufsize = (uint8_t)priv->flc_desc[0].flc.word1_sdl; 1687 printf("Descriptor Dump:\n"); 1688 for (i = 0; i < bufsize; i++) 1689 printf("\tDESC[%d]:0x%x\n", i, priv->flc_desc[0].desc[i]); 1690 1691 printf("\n"); 1692 mbuf_dump: 1693 sym_op = op->sym; 1694 if (sym_op->m_src) { 1695 printf("Source mbuf:\n"); 1696 rte_pktmbuf_dump(stdout, sym_op->m_src, sym_op->m_src->data_len); 1697 } 1698 if (sym_op->m_dst) { 1699 printf("Destination mbuf:\n"); 1700 rte_pktmbuf_dump(stdout, sym_op->m_dst, sym_op->m_dst->data_len); 1701 } 1702 1703 printf("Session address = %p\ncipher offset: %d, length: %d\n" 1704 "auth offset: %d, length: %d\n aead offset: %d, length: %d\n" 1705 , sym_op->session, 1706 sym_op->cipher.data.offset, sym_op->cipher.data.length, 1707 sym_op->auth.data.offset, sym_op->auth.data.length, 1708 sym_op->aead.data.offset, sym_op->aead.data.length); 1709 printf("\n"); 1710 1711 } 1712 1713 static void 1714 dpaa2_sec_free_eqresp_buf(uint16_t eqresp_ci) 1715 { 1716 struct dpaa2_dpio_dev *dpio_dev = DPAA2_PER_LCORE_DPIO; 1717 struct rte_crypto_op *op; 1718 struct qbman_fd *fd; 1719 1720 fd = qbman_result_eqresp_fd(&dpio_dev->eqresp[eqresp_ci]); 1721 op = sec_fd_to_mbuf(fd); 1722 /* Instead of freeing, enqueue it to the sec tx queue (sec->core) 1723 * after setting an error in FD. But this will have performance impact. 1724 */ 1725 rte_pktmbuf_free(op->sym->m_src); 1726 } 1727 1728 static void 1729 dpaa2_sec_set_enqueue_descriptor(struct dpaa2_queue *dpaa2_q, 1730 struct rte_mbuf *m, 1731 struct qbman_eq_desc *eqdesc) 1732 { 1733 struct dpaa2_dpio_dev *dpio_dev = DPAA2_PER_LCORE_DPIO; 1734 struct eqresp_metadata *eqresp_meta; 1735 struct dpaa2_sec_dev_private *priv = dpaa2_q->crypto_data->dev_private; 1736 uint16_t orpid, seqnum; 1737 uint8_t dq_idx; 1738 1739 if (*dpaa2_seqn(m) & DPAA2_ENQUEUE_FLAG_ORP) { 1740 orpid = (*dpaa2_seqn(m) & DPAA2_EQCR_OPRID_MASK) >> 1741 DPAA2_EQCR_OPRID_SHIFT; 1742 seqnum = (*dpaa2_seqn(m) & DPAA2_EQCR_SEQNUM_MASK) >> 1743 DPAA2_EQCR_SEQNUM_SHIFT; 1744 1745 1746 if (!priv->en_loose_ordered) { 1747 qbman_eq_desc_set_orp(eqdesc, 1, orpid, seqnum, 0); 1748 qbman_eq_desc_set_response(eqdesc, (uint64_t) 1749 DPAA2_VADDR_TO_IOVA(&dpio_dev->eqresp[ 1750 dpio_dev->eqresp_pi]), 1); 1751 qbman_eq_desc_set_token(eqdesc, 1); 1752 1753 eqresp_meta = &dpio_dev->eqresp_meta[dpio_dev->eqresp_pi]; 1754 eqresp_meta->dpaa2_q = dpaa2_q; 1755 eqresp_meta->mp = m->pool; 1756 1757 dpio_dev->eqresp_pi + 1 < MAX_EQ_RESP_ENTRIES ? 1758 dpio_dev->eqresp_pi++ : (dpio_dev->eqresp_pi = 0); 1759 } else { 1760 qbman_eq_desc_set_orp(eqdesc, 0, orpid, seqnum, 0); 1761 } 1762 } else { 1763 dq_idx = *dpaa2_seqn(m) - 1; 1764 qbman_eq_desc_set_dca(eqdesc, 1, dq_idx, 0); 1765 DPAA2_PER_LCORE_DQRR_SIZE--; 1766 DPAA2_PER_LCORE_DQRR_HELD &= ~(1 << dq_idx); 1767 } 1768 *dpaa2_seqn(m) = DPAA2_INVALID_MBUF_SEQN; 1769 } 1770 1771 1772 static uint16_t 1773 dpaa2_sec_enqueue_burst_ordered(void *qp, struct rte_crypto_op **ops, 1774 uint16_t nb_ops) 1775 { 1776 /* Function to transmit the frames to given device and VQ*/ 1777 uint32_t loop; 1778 int32_t ret; 1779 struct qbman_fd fd_arr[MAX_TX_RING_SLOTS]; 1780 uint32_t frames_to_send, num_free_eq_desc, retry_count; 1781 struct qbman_eq_desc eqdesc[MAX_TX_RING_SLOTS]; 1782 struct dpaa2_sec_qp *dpaa2_qp = (struct dpaa2_sec_qp *)qp; 1783 struct qbman_swp *swp; 1784 uint16_t num_tx = 0; 1785 uint16_t bpid; 1786 struct rte_mempool *mb_pool; 1787 struct dpaa2_sec_dev_private *priv = 1788 dpaa2_qp->tx_vq.crypto_data->dev_private; 1789 1790 if (unlikely(nb_ops == 0)) 1791 return 0; 1792 1793 if (ops[0]->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { 1794 DPAA2_SEC_ERR("sessionless crypto op not supported"); 1795 return 0; 1796 } 1797 1798 if (!DPAA2_PER_LCORE_DPIO) { 1799 ret = dpaa2_affine_qbman_swp(); 1800 if (ret) { 1801 DPAA2_SEC_ERR("Failure in affining portal"); 1802 return 0; 1803 } 1804 } 1805 swp = DPAA2_PER_LCORE_PORTAL; 1806 1807 while (nb_ops) { 1808 frames_to_send = (nb_ops > dpaa2_eqcr_size) ? 1809 dpaa2_eqcr_size : nb_ops; 1810 1811 if (!priv->en_loose_ordered) { 1812 if (*dpaa2_seqn((*ops)->sym->m_src)) { 1813 num_free_eq_desc = dpaa2_free_eq_descriptors(); 1814 if (num_free_eq_desc < frames_to_send) 1815 frames_to_send = num_free_eq_desc; 1816 } 1817 } 1818 1819 for (loop = 0; loop < frames_to_send; loop++) { 1820 /*Prepare enqueue descriptor*/ 1821 qbman_eq_desc_clear(&eqdesc[loop]); 1822 qbman_eq_desc_set_fq(&eqdesc[loop], dpaa2_qp->tx_vq.fqid); 1823 1824 if (*dpaa2_seqn((*ops)->sym->m_src)) 1825 dpaa2_sec_set_enqueue_descriptor( 1826 &dpaa2_qp->tx_vq, 1827 (*ops)->sym->m_src, 1828 &eqdesc[loop]); 1829 else 1830 qbman_eq_desc_set_no_orp(&eqdesc[loop], 1831 DPAA2_EQ_RESP_ERR_FQ); 1832 1833 /*Clear the unused FD fields before sending*/ 1834 memset(&fd_arr[loop], 0, sizeof(struct qbman_fd)); 1835 mb_pool = (*ops)->sym->m_src->pool; 1836 bpid = mempool_to_bpid(mb_pool); 1837 ret = build_sec_fd(*ops, &fd_arr[loop], bpid); 1838 if (ret) { 1839 DPAA2_SEC_ERR("error: Improper packet contents" 1840 " for crypto operation"); 1841 goto skip_tx; 1842 } 1843 ops++; 1844 } 1845 1846 loop = 0; 1847 retry_count = 0; 1848 while (loop < frames_to_send) { 1849 ret = qbman_swp_enqueue_multiple_desc(swp, 1850 &eqdesc[loop], &fd_arr[loop], 1851 frames_to_send - loop); 1852 if (unlikely(ret < 0)) { 1853 retry_count++; 1854 if (retry_count > DPAA2_MAX_TX_RETRY_COUNT) { 1855 num_tx += loop; 1856 nb_ops -= loop; 1857 goto skip_tx; 1858 } 1859 } else { 1860 loop += ret; 1861 retry_count = 0; 1862 } 1863 } 1864 1865 num_tx += loop; 1866 nb_ops -= loop; 1867 } 1868 1869 skip_tx: 1870 dpaa2_qp->tx_vq.tx_pkts += num_tx; 1871 dpaa2_qp->tx_vq.err_pkts += nb_ops; 1872 return num_tx; 1873 } 1874 1875 static uint16_t 1876 dpaa2_sec_dequeue_burst(void *qp, struct rte_crypto_op **ops, 1877 uint16_t nb_ops) 1878 { 1879 /* Function is responsible to receive frames for a given device and VQ*/ 1880 struct dpaa2_sec_qp *dpaa2_qp = (struct dpaa2_sec_qp *)qp; 1881 struct qbman_result *dq_storage; 1882 uint32_t fqid = dpaa2_qp->rx_vq.fqid; 1883 int ret, num_rx = 0; 1884 uint8_t is_last = 0, status; 1885 struct qbman_swp *swp; 1886 const struct qbman_fd *fd; 1887 struct qbman_pull_desc pulldesc; 1888 1889 if (!DPAA2_PER_LCORE_DPIO) { 1890 ret = dpaa2_affine_qbman_swp(); 1891 if (ret) { 1892 DPAA2_SEC_ERR( 1893 "Failed to allocate IO portal, tid: %d\n", 1894 rte_gettid()); 1895 return 0; 1896 } 1897 } 1898 swp = DPAA2_PER_LCORE_PORTAL; 1899 dq_storage = dpaa2_qp->rx_vq.q_storage->dq_storage[0]; 1900 1901 qbman_pull_desc_clear(&pulldesc); 1902 qbman_pull_desc_set_numframes(&pulldesc, 1903 (nb_ops > dpaa2_dqrr_size) ? 1904 dpaa2_dqrr_size : nb_ops); 1905 qbman_pull_desc_set_fq(&pulldesc, fqid); 1906 qbman_pull_desc_set_storage(&pulldesc, dq_storage, 1907 (dma_addr_t)DPAA2_VADDR_TO_IOVA(dq_storage), 1908 1); 1909 1910 /*Issue a volatile dequeue command. */ 1911 while (1) { 1912 if (qbman_swp_pull(swp, &pulldesc)) { 1913 DPAA2_SEC_WARN( 1914 "SEC VDQ command is not issued : QBMAN busy"); 1915 /* Portal was busy, try again */ 1916 continue; 1917 } 1918 break; 1919 }; 1920 1921 /* Receive the packets till Last Dequeue entry is found with 1922 * respect to the above issues PULL command. 1923 */ 1924 while (!is_last) { 1925 /* Check if the previous issued command is completed. 1926 * Also seems like the SWP is shared between the Ethernet Driver 1927 * and the SEC driver. 1928 */ 1929 while (!qbman_check_command_complete(dq_storage)) 1930 ; 1931 1932 /* Loop until the dq_storage is updated with 1933 * new token by QBMAN 1934 */ 1935 while (!qbman_check_new_result(dq_storage)) 1936 ; 1937 /* Check whether Last Pull command is Expired and 1938 * setting Condition for Loop termination 1939 */ 1940 if (qbman_result_DQ_is_pull_complete(dq_storage)) { 1941 is_last = 1; 1942 /* Check for valid frame. */ 1943 status = (uint8_t)qbman_result_DQ_flags(dq_storage); 1944 if (unlikely( 1945 (status & QBMAN_DQ_STAT_VALIDFRAME) == 0)) { 1946 DPAA2_SEC_DP_DEBUG("No frame is delivered\n"); 1947 continue; 1948 } 1949 } 1950 1951 fd = qbman_result_DQ_fd(dq_storage); 1952 ops[num_rx] = sec_fd_to_mbuf(fd); 1953 1954 if (unlikely(fd->simple.frc)) { 1955 /* TODO Parse SEC errors */ 1956 if (dpaa2_sec_dp_dump > DPAA2_SEC_DP_NO_DUMP) { 1957 DPAA2_SEC_DP_ERR("SEC returned Error - %x\n", 1958 fd->simple.frc); 1959 if (dpaa2_sec_dp_dump > DPAA2_SEC_DP_ERR_DUMP) 1960 dpaa2_sec_dump(ops[num_rx]); 1961 } 1962 1963 dpaa2_qp->rx_vq.err_pkts += 1; 1964 ops[num_rx]->status = RTE_CRYPTO_OP_STATUS_ERROR; 1965 } else { 1966 ops[num_rx]->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 1967 } 1968 1969 num_rx++; 1970 dq_storage++; 1971 } /* End of Packet Rx loop */ 1972 1973 dpaa2_qp->rx_vq.rx_pkts += num_rx; 1974 1975 DPAA2_SEC_DP_DEBUG("SEC RX pkts %d err pkts %" PRIu64 "\n", num_rx, 1976 dpaa2_qp->rx_vq.err_pkts); 1977 /*Return the total number of packets received to DPAA2 app*/ 1978 return num_rx; 1979 } 1980 1981 /** Release queue pair */ 1982 static int 1983 dpaa2_sec_queue_pair_release(struct rte_cryptodev *dev, uint16_t queue_pair_id) 1984 { 1985 struct dpaa2_sec_qp *qp = 1986 (struct dpaa2_sec_qp *)dev->data->queue_pairs[queue_pair_id]; 1987 1988 PMD_INIT_FUNC_TRACE(); 1989 1990 if (qp->rx_vq.q_storage) { 1991 dpaa2_free_dq_storage(qp->rx_vq.q_storage); 1992 rte_free(qp->rx_vq.q_storage); 1993 } 1994 rte_free(qp); 1995 1996 dev->data->queue_pairs[queue_pair_id] = NULL; 1997 1998 return 0; 1999 } 2000 2001 /** Setup a queue pair */ 2002 static int 2003 dpaa2_sec_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id, 2004 __rte_unused const struct rte_cryptodev_qp_conf *qp_conf, 2005 __rte_unused int socket_id) 2006 { 2007 struct dpaa2_sec_dev_private *priv = dev->data->dev_private; 2008 struct dpaa2_sec_qp *qp; 2009 struct fsl_mc_io *dpseci = (struct fsl_mc_io *)priv->hw; 2010 struct dpseci_rx_queue_cfg cfg; 2011 int32_t retcode; 2012 2013 PMD_INIT_FUNC_TRACE(); 2014 2015 /* If qp is already in use free ring memory and qp metadata. */ 2016 if (dev->data->queue_pairs[qp_id] != NULL) { 2017 DPAA2_SEC_INFO("QP already setup"); 2018 return 0; 2019 } 2020 2021 DPAA2_SEC_DEBUG("dev =%p, queue =%d, conf =%p", 2022 dev, qp_id, qp_conf); 2023 2024 memset(&cfg, 0, sizeof(struct dpseci_rx_queue_cfg)); 2025 2026 qp = rte_malloc(NULL, sizeof(struct dpaa2_sec_qp), 2027 RTE_CACHE_LINE_SIZE); 2028 if (!qp) { 2029 DPAA2_SEC_ERR("malloc failed for rx/tx queues"); 2030 return -ENOMEM; 2031 } 2032 2033 qp->rx_vq.crypto_data = dev->data; 2034 qp->tx_vq.crypto_data = dev->data; 2035 qp->rx_vq.q_storage = rte_malloc("sec dq storage", 2036 sizeof(struct queue_storage_info_t), 2037 RTE_CACHE_LINE_SIZE); 2038 if (!qp->rx_vq.q_storage) { 2039 DPAA2_SEC_ERR("malloc failed for q_storage"); 2040 return -ENOMEM; 2041 } 2042 memset(qp->rx_vq.q_storage, 0, sizeof(struct queue_storage_info_t)); 2043 2044 if (dpaa2_alloc_dq_storage(qp->rx_vq.q_storage)) { 2045 DPAA2_SEC_ERR("Unable to allocate dequeue storage"); 2046 return -ENOMEM; 2047 } 2048 2049 dev->data->queue_pairs[qp_id] = qp; 2050 2051 cfg.options = cfg.options | DPSECI_QUEUE_OPT_USER_CTX; 2052 cfg.user_ctx = (size_t)(&qp->rx_vq); 2053 retcode = dpseci_set_rx_queue(dpseci, CMD_PRI_LOW, priv->token, 2054 qp_id, &cfg); 2055 return retcode; 2056 } 2057 2058 /** Returns the size of the aesni gcm session structure */ 2059 static unsigned int 2060 dpaa2_sec_sym_session_get_size(struct rte_cryptodev *dev __rte_unused) 2061 { 2062 PMD_INIT_FUNC_TRACE(); 2063 2064 return sizeof(dpaa2_sec_session); 2065 } 2066 2067 static int 2068 dpaa2_sec_cipher_init(struct rte_cryptodev *dev, 2069 struct rte_crypto_sym_xform *xform, 2070 dpaa2_sec_session *session) 2071 { 2072 struct dpaa2_sec_dev_private *dev_priv = dev->data->dev_private; 2073 struct alginfo cipherdata; 2074 int bufsize, ret = 0; 2075 struct ctxt_priv *priv; 2076 struct sec_flow_context *flc; 2077 2078 PMD_INIT_FUNC_TRACE(); 2079 2080 /* For SEC CIPHER only one descriptor is required. */ 2081 priv = (struct ctxt_priv *)rte_zmalloc(NULL, 2082 sizeof(struct ctxt_priv) + sizeof(struct sec_flc_desc), 2083 RTE_CACHE_LINE_SIZE); 2084 if (priv == NULL) { 2085 DPAA2_SEC_ERR("No Memory for priv CTXT"); 2086 return -ENOMEM; 2087 } 2088 2089 priv->fle_pool = dev_priv->fle_pool; 2090 2091 flc = &priv->flc_desc[0].flc; 2092 2093 session->ctxt_type = DPAA2_SEC_CIPHER; 2094 session->cipher_key.data = rte_zmalloc(NULL, xform->cipher.key.length, 2095 RTE_CACHE_LINE_SIZE); 2096 if (session->cipher_key.data == NULL && xform->cipher.key.length > 0) { 2097 DPAA2_SEC_ERR("No Memory for cipher key"); 2098 rte_free(priv); 2099 return -ENOMEM; 2100 } 2101 session->cipher_key.length = xform->cipher.key.length; 2102 2103 memcpy(session->cipher_key.data, xform->cipher.key.data, 2104 xform->cipher.key.length); 2105 cipherdata.key = (size_t)session->cipher_key.data; 2106 cipherdata.keylen = session->cipher_key.length; 2107 cipherdata.key_enc_flags = 0; 2108 cipherdata.key_type = RTA_DATA_IMM; 2109 2110 /* Set IV parameters */ 2111 session->iv.offset = xform->cipher.iv.offset; 2112 session->iv.length = xform->cipher.iv.length; 2113 session->dir = (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ? 2114 DIR_ENC : DIR_DEC; 2115 2116 switch (xform->cipher.algo) { 2117 case RTE_CRYPTO_CIPHER_AES_CBC: 2118 cipherdata.algtype = OP_ALG_ALGSEL_AES; 2119 cipherdata.algmode = OP_ALG_AAI_CBC; 2120 session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CBC; 2121 bufsize = cnstr_shdsc_blkcipher(priv->flc_desc[0].desc, 1, 0, 2122 SHR_NEVER, &cipherdata, 2123 session->iv.length, 2124 session->dir); 2125 break; 2126 case RTE_CRYPTO_CIPHER_3DES_CBC: 2127 cipherdata.algtype = OP_ALG_ALGSEL_3DES; 2128 cipherdata.algmode = OP_ALG_AAI_CBC; 2129 session->cipher_alg = RTE_CRYPTO_CIPHER_3DES_CBC; 2130 bufsize = cnstr_shdsc_blkcipher(priv->flc_desc[0].desc, 1, 0, 2131 SHR_NEVER, &cipherdata, 2132 session->iv.length, 2133 session->dir); 2134 break; 2135 case RTE_CRYPTO_CIPHER_DES_CBC: 2136 cipherdata.algtype = OP_ALG_ALGSEL_DES; 2137 cipherdata.algmode = OP_ALG_AAI_CBC; 2138 session->cipher_alg = RTE_CRYPTO_CIPHER_DES_CBC; 2139 bufsize = cnstr_shdsc_blkcipher(priv->flc_desc[0].desc, 1, 0, 2140 SHR_NEVER, &cipherdata, 2141 session->iv.length, 2142 session->dir); 2143 break; 2144 case RTE_CRYPTO_CIPHER_AES_CTR: 2145 cipherdata.algtype = OP_ALG_ALGSEL_AES; 2146 cipherdata.algmode = OP_ALG_AAI_CTR; 2147 session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CTR; 2148 bufsize = cnstr_shdsc_blkcipher(priv->flc_desc[0].desc, 1, 0, 2149 SHR_NEVER, &cipherdata, 2150 session->iv.length, 2151 session->dir); 2152 break; 2153 case RTE_CRYPTO_CIPHER_SNOW3G_UEA2: 2154 cipherdata.algtype = OP_ALG_ALGSEL_SNOW_F8; 2155 session->cipher_alg = RTE_CRYPTO_CIPHER_SNOW3G_UEA2; 2156 bufsize = cnstr_shdsc_snow_f8(priv->flc_desc[0].desc, 1, 0, 2157 &cipherdata, 2158 session->dir); 2159 break; 2160 case RTE_CRYPTO_CIPHER_ZUC_EEA3: 2161 cipherdata.algtype = OP_ALG_ALGSEL_ZUCE; 2162 session->cipher_alg = RTE_CRYPTO_CIPHER_ZUC_EEA3; 2163 bufsize = cnstr_shdsc_zuce(priv->flc_desc[0].desc, 1, 0, 2164 &cipherdata, 2165 session->dir); 2166 break; 2167 case RTE_CRYPTO_CIPHER_KASUMI_F8: 2168 case RTE_CRYPTO_CIPHER_AES_F8: 2169 case RTE_CRYPTO_CIPHER_AES_ECB: 2170 case RTE_CRYPTO_CIPHER_3DES_ECB: 2171 case RTE_CRYPTO_CIPHER_3DES_CTR: 2172 case RTE_CRYPTO_CIPHER_AES_XTS: 2173 case RTE_CRYPTO_CIPHER_ARC4: 2174 case RTE_CRYPTO_CIPHER_NULL: 2175 DPAA2_SEC_ERR("Crypto: Unsupported Cipher alg %u", 2176 xform->cipher.algo); 2177 ret = -ENOTSUP; 2178 goto error_out; 2179 default: 2180 DPAA2_SEC_ERR("Crypto: Undefined Cipher specified %u", 2181 xform->cipher.algo); 2182 ret = -ENOTSUP; 2183 goto error_out; 2184 } 2185 2186 if (bufsize < 0) { 2187 DPAA2_SEC_ERR("Crypto: Descriptor build failed"); 2188 ret = -EINVAL; 2189 goto error_out; 2190 } 2191 2192 flc->word1_sdl = (uint8_t)bufsize; 2193 session->ctxt = priv; 2194 2195 #ifdef CAAM_DESC_DEBUG 2196 int i; 2197 for (i = 0; i < bufsize; i++) 2198 DPAA2_SEC_DEBUG("DESC[%d]:0x%x", i, priv->flc_desc[0].desc[i]); 2199 #endif 2200 return ret; 2201 2202 error_out: 2203 rte_free(session->cipher_key.data); 2204 rte_free(priv); 2205 return ret; 2206 } 2207 2208 static int 2209 dpaa2_sec_auth_init(struct rte_cryptodev *dev, 2210 struct rte_crypto_sym_xform *xform, 2211 dpaa2_sec_session *session) 2212 { 2213 struct dpaa2_sec_dev_private *dev_priv = dev->data->dev_private; 2214 struct alginfo authdata; 2215 int bufsize, ret = 0; 2216 struct ctxt_priv *priv; 2217 struct sec_flow_context *flc; 2218 2219 PMD_INIT_FUNC_TRACE(); 2220 2221 /* For SEC AUTH three descriptors are required for various stages */ 2222 priv = (struct ctxt_priv *)rte_zmalloc(NULL, 2223 sizeof(struct ctxt_priv) + 3 * 2224 sizeof(struct sec_flc_desc), 2225 RTE_CACHE_LINE_SIZE); 2226 if (priv == NULL) { 2227 DPAA2_SEC_ERR("No Memory for priv CTXT"); 2228 return -ENOMEM; 2229 } 2230 2231 priv->fle_pool = dev_priv->fle_pool; 2232 flc = &priv->flc_desc[DESC_INITFINAL].flc; 2233 2234 session->ctxt_type = DPAA2_SEC_AUTH; 2235 session->auth_key.length = xform->auth.key.length; 2236 if (xform->auth.key.length) { 2237 session->auth_key.data = rte_zmalloc(NULL, 2238 xform->auth.key.length, 2239 RTE_CACHE_LINE_SIZE); 2240 if (session->auth_key.data == NULL) { 2241 DPAA2_SEC_ERR("Unable to allocate memory for auth key"); 2242 rte_free(priv); 2243 return -ENOMEM; 2244 } 2245 memcpy(session->auth_key.data, xform->auth.key.data, 2246 xform->auth.key.length); 2247 authdata.key = (size_t)session->auth_key.data; 2248 authdata.key_enc_flags = 0; 2249 authdata.key_type = RTA_DATA_IMM; 2250 } 2251 authdata.keylen = session->auth_key.length; 2252 2253 session->digest_length = xform->auth.digest_length; 2254 session->dir = (xform->auth.op == RTE_CRYPTO_AUTH_OP_GENERATE) ? 2255 DIR_ENC : DIR_DEC; 2256 2257 switch (xform->auth.algo) { 2258 case RTE_CRYPTO_AUTH_SHA1_HMAC: 2259 authdata.algtype = OP_ALG_ALGSEL_SHA1; 2260 authdata.algmode = OP_ALG_AAI_HMAC; 2261 session->auth_alg = RTE_CRYPTO_AUTH_SHA1_HMAC; 2262 bufsize = cnstr_shdsc_hmac(priv->flc_desc[DESC_INITFINAL].desc, 2263 1, 0, SHR_NEVER, &authdata, 2264 !session->dir, 2265 session->digest_length); 2266 break; 2267 case RTE_CRYPTO_AUTH_MD5_HMAC: 2268 authdata.algtype = OP_ALG_ALGSEL_MD5; 2269 authdata.algmode = OP_ALG_AAI_HMAC; 2270 session->auth_alg = RTE_CRYPTO_AUTH_MD5_HMAC; 2271 bufsize = cnstr_shdsc_hmac(priv->flc_desc[DESC_INITFINAL].desc, 2272 1, 0, SHR_NEVER, &authdata, 2273 !session->dir, 2274 session->digest_length); 2275 break; 2276 case RTE_CRYPTO_AUTH_SHA256_HMAC: 2277 authdata.algtype = OP_ALG_ALGSEL_SHA256; 2278 authdata.algmode = OP_ALG_AAI_HMAC; 2279 session->auth_alg = RTE_CRYPTO_AUTH_SHA256_HMAC; 2280 bufsize = cnstr_shdsc_hmac(priv->flc_desc[DESC_INITFINAL].desc, 2281 1, 0, SHR_NEVER, &authdata, 2282 !session->dir, 2283 session->digest_length); 2284 break; 2285 case RTE_CRYPTO_AUTH_SHA384_HMAC: 2286 authdata.algtype = OP_ALG_ALGSEL_SHA384; 2287 authdata.algmode = OP_ALG_AAI_HMAC; 2288 session->auth_alg = RTE_CRYPTO_AUTH_SHA384_HMAC; 2289 bufsize = cnstr_shdsc_hmac(priv->flc_desc[DESC_INITFINAL].desc, 2290 1, 0, SHR_NEVER, &authdata, 2291 !session->dir, 2292 session->digest_length); 2293 break; 2294 case RTE_CRYPTO_AUTH_SHA512_HMAC: 2295 authdata.algtype = OP_ALG_ALGSEL_SHA512; 2296 authdata.algmode = OP_ALG_AAI_HMAC; 2297 session->auth_alg = RTE_CRYPTO_AUTH_SHA512_HMAC; 2298 bufsize = cnstr_shdsc_hmac(priv->flc_desc[DESC_INITFINAL].desc, 2299 1, 0, SHR_NEVER, &authdata, 2300 !session->dir, 2301 session->digest_length); 2302 break; 2303 case RTE_CRYPTO_AUTH_SHA224_HMAC: 2304 authdata.algtype = OP_ALG_ALGSEL_SHA224; 2305 authdata.algmode = OP_ALG_AAI_HMAC; 2306 session->auth_alg = RTE_CRYPTO_AUTH_SHA224_HMAC; 2307 bufsize = cnstr_shdsc_hmac(priv->flc_desc[DESC_INITFINAL].desc, 2308 1, 0, SHR_NEVER, &authdata, 2309 !session->dir, 2310 session->digest_length); 2311 break; 2312 case RTE_CRYPTO_AUTH_SNOW3G_UIA2: 2313 authdata.algtype = OP_ALG_ALGSEL_SNOW_F9; 2314 authdata.algmode = OP_ALG_AAI_F9; 2315 session->auth_alg = RTE_CRYPTO_AUTH_SNOW3G_UIA2; 2316 session->iv.offset = xform->auth.iv.offset; 2317 session->iv.length = xform->auth.iv.length; 2318 bufsize = cnstr_shdsc_snow_f9(priv->flc_desc[DESC_INITFINAL].desc, 2319 1, 0, &authdata, 2320 !session->dir, 2321 session->digest_length); 2322 break; 2323 case RTE_CRYPTO_AUTH_ZUC_EIA3: 2324 authdata.algtype = OP_ALG_ALGSEL_ZUCA; 2325 authdata.algmode = OP_ALG_AAI_F9; 2326 session->auth_alg = RTE_CRYPTO_AUTH_ZUC_EIA3; 2327 session->iv.offset = xform->auth.iv.offset; 2328 session->iv.length = xform->auth.iv.length; 2329 bufsize = cnstr_shdsc_zuca(priv->flc_desc[DESC_INITFINAL].desc, 2330 1, 0, &authdata, 2331 !session->dir, 2332 session->digest_length); 2333 break; 2334 case RTE_CRYPTO_AUTH_SHA1: 2335 authdata.algtype = OP_ALG_ALGSEL_SHA1; 2336 authdata.algmode = OP_ALG_AAI_HASH; 2337 session->auth_alg = RTE_CRYPTO_AUTH_SHA1; 2338 bufsize = cnstr_shdsc_hash(priv->flc_desc[DESC_INITFINAL].desc, 2339 1, 0, SHR_NEVER, &authdata, 2340 !session->dir, 2341 session->digest_length); 2342 break; 2343 case RTE_CRYPTO_AUTH_MD5: 2344 authdata.algtype = OP_ALG_ALGSEL_MD5; 2345 authdata.algmode = OP_ALG_AAI_HASH; 2346 session->auth_alg = RTE_CRYPTO_AUTH_MD5; 2347 bufsize = cnstr_shdsc_hash(priv->flc_desc[DESC_INITFINAL].desc, 2348 1, 0, SHR_NEVER, &authdata, 2349 !session->dir, 2350 session->digest_length); 2351 break; 2352 case RTE_CRYPTO_AUTH_SHA256: 2353 authdata.algtype = OP_ALG_ALGSEL_SHA256; 2354 authdata.algmode = OP_ALG_AAI_HASH; 2355 session->auth_alg = RTE_CRYPTO_AUTH_SHA256; 2356 bufsize = cnstr_shdsc_hash(priv->flc_desc[DESC_INITFINAL].desc, 2357 1, 0, SHR_NEVER, &authdata, 2358 !session->dir, 2359 session->digest_length); 2360 break; 2361 case RTE_CRYPTO_AUTH_SHA384: 2362 authdata.algtype = OP_ALG_ALGSEL_SHA384; 2363 authdata.algmode = OP_ALG_AAI_HASH; 2364 session->auth_alg = RTE_CRYPTO_AUTH_SHA384; 2365 bufsize = cnstr_shdsc_hash(priv->flc_desc[DESC_INITFINAL].desc, 2366 1, 0, SHR_NEVER, &authdata, 2367 !session->dir, 2368 session->digest_length); 2369 break; 2370 case RTE_CRYPTO_AUTH_SHA512: 2371 authdata.algtype = OP_ALG_ALGSEL_SHA512; 2372 authdata.algmode = OP_ALG_AAI_HASH; 2373 session->auth_alg = RTE_CRYPTO_AUTH_SHA512; 2374 bufsize = cnstr_shdsc_hash(priv->flc_desc[DESC_INITFINAL].desc, 2375 1, 0, SHR_NEVER, &authdata, 2376 !session->dir, 2377 session->digest_length); 2378 break; 2379 case RTE_CRYPTO_AUTH_SHA224: 2380 authdata.algtype = OP_ALG_ALGSEL_SHA224; 2381 authdata.algmode = OP_ALG_AAI_HASH; 2382 session->auth_alg = RTE_CRYPTO_AUTH_SHA224; 2383 bufsize = cnstr_shdsc_hash(priv->flc_desc[DESC_INITFINAL].desc, 2384 1, 0, SHR_NEVER, &authdata, 2385 !session->dir, 2386 session->digest_length); 2387 break; 2388 case RTE_CRYPTO_AUTH_AES_XCBC_MAC: 2389 authdata.algtype = OP_ALG_ALGSEL_AES; 2390 authdata.algmode = OP_ALG_AAI_XCBC_MAC; 2391 session->auth_alg = RTE_CRYPTO_AUTH_AES_XCBC_MAC; 2392 bufsize = cnstr_shdsc_aes_mac( 2393 priv->flc_desc[DESC_INITFINAL].desc, 2394 1, 0, SHR_NEVER, &authdata, 2395 !session->dir, 2396 session->digest_length); 2397 break; 2398 case RTE_CRYPTO_AUTH_AES_CMAC: 2399 authdata.algtype = OP_ALG_ALGSEL_AES; 2400 authdata.algmode = OP_ALG_AAI_CMAC; 2401 session->auth_alg = RTE_CRYPTO_AUTH_AES_CMAC; 2402 bufsize = cnstr_shdsc_aes_mac( 2403 priv->flc_desc[DESC_INITFINAL].desc, 2404 1, 0, SHR_NEVER, &authdata, 2405 !session->dir, 2406 session->digest_length); 2407 break; 2408 case RTE_CRYPTO_AUTH_AES_CBC_MAC: 2409 case RTE_CRYPTO_AUTH_AES_GMAC: 2410 case RTE_CRYPTO_AUTH_KASUMI_F9: 2411 case RTE_CRYPTO_AUTH_NULL: 2412 DPAA2_SEC_ERR("Crypto: Unsupported auth alg %un", 2413 xform->auth.algo); 2414 ret = -ENOTSUP; 2415 goto error_out; 2416 default: 2417 DPAA2_SEC_ERR("Crypto: Undefined Auth specified %u", 2418 xform->auth.algo); 2419 ret = -ENOTSUP; 2420 goto error_out; 2421 } 2422 2423 if (bufsize < 0) { 2424 DPAA2_SEC_ERR("Crypto: Invalid buffer length"); 2425 ret = -EINVAL; 2426 goto error_out; 2427 } 2428 2429 flc->word1_sdl = (uint8_t)bufsize; 2430 session->ctxt = priv; 2431 #ifdef CAAM_DESC_DEBUG 2432 int i; 2433 for (i = 0; i < bufsize; i++) 2434 DPAA2_SEC_DEBUG("DESC[%d]:0x%x", 2435 i, priv->flc_desc[DESC_INITFINAL].desc[i]); 2436 #endif 2437 2438 return ret; 2439 2440 error_out: 2441 rte_free(session->auth_key.data); 2442 rte_free(priv); 2443 return ret; 2444 } 2445 2446 static int 2447 dpaa2_sec_aead_init(struct rte_cryptodev *dev, 2448 struct rte_crypto_sym_xform *xform, 2449 dpaa2_sec_session *session) 2450 { 2451 struct dpaa2_sec_aead_ctxt *ctxt = &session->ext_params.aead_ctxt; 2452 struct dpaa2_sec_dev_private *dev_priv = dev->data->dev_private; 2453 struct alginfo aeaddata; 2454 int bufsize; 2455 struct ctxt_priv *priv; 2456 struct sec_flow_context *flc; 2457 struct rte_crypto_aead_xform *aead_xform = &xform->aead; 2458 int err, ret = 0; 2459 2460 PMD_INIT_FUNC_TRACE(); 2461 2462 /* Set IV parameters */ 2463 session->iv.offset = aead_xform->iv.offset; 2464 session->iv.length = aead_xform->iv.length; 2465 session->ctxt_type = DPAA2_SEC_AEAD; 2466 2467 /* For SEC AEAD only one descriptor is required */ 2468 priv = (struct ctxt_priv *)rte_zmalloc(NULL, 2469 sizeof(struct ctxt_priv) + sizeof(struct sec_flc_desc), 2470 RTE_CACHE_LINE_SIZE); 2471 if (priv == NULL) { 2472 DPAA2_SEC_ERR("No Memory for priv CTXT"); 2473 return -ENOMEM; 2474 } 2475 2476 priv->fle_pool = dev_priv->fle_pool; 2477 flc = &priv->flc_desc[0].flc; 2478 2479 session->aead_key.data = rte_zmalloc(NULL, aead_xform->key.length, 2480 RTE_CACHE_LINE_SIZE); 2481 if (session->aead_key.data == NULL && aead_xform->key.length > 0) { 2482 DPAA2_SEC_ERR("No Memory for aead key"); 2483 rte_free(priv); 2484 return -ENOMEM; 2485 } 2486 memcpy(session->aead_key.data, aead_xform->key.data, 2487 aead_xform->key.length); 2488 2489 session->digest_length = aead_xform->digest_length; 2490 session->aead_key.length = aead_xform->key.length; 2491 ctxt->auth_only_len = aead_xform->aad_length; 2492 2493 aeaddata.key = (size_t)session->aead_key.data; 2494 aeaddata.keylen = session->aead_key.length; 2495 aeaddata.key_enc_flags = 0; 2496 aeaddata.key_type = RTA_DATA_IMM; 2497 2498 switch (aead_xform->algo) { 2499 case RTE_CRYPTO_AEAD_AES_GCM: 2500 aeaddata.algtype = OP_ALG_ALGSEL_AES; 2501 aeaddata.algmode = OP_ALG_AAI_GCM; 2502 session->aead_alg = RTE_CRYPTO_AEAD_AES_GCM; 2503 break; 2504 case RTE_CRYPTO_AEAD_AES_CCM: 2505 DPAA2_SEC_ERR("Crypto: Unsupported AEAD alg %u", 2506 aead_xform->algo); 2507 ret = -ENOTSUP; 2508 goto error_out; 2509 default: 2510 DPAA2_SEC_ERR("Crypto: Undefined AEAD specified %u", 2511 aead_xform->algo); 2512 ret = -ENOTSUP; 2513 goto error_out; 2514 } 2515 session->dir = (aead_xform->op == RTE_CRYPTO_AEAD_OP_ENCRYPT) ? 2516 DIR_ENC : DIR_DEC; 2517 2518 priv->flc_desc[0].desc[0] = aeaddata.keylen; 2519 err = rta_inline_query(IPSEC_AUTH_VAR_AES_DEC_BASE_DESC_LEN, 2520 DESC_JOB_IO_LEN, 2521 (unsigned int *)priv->flc_desc[0].desc, 2522 &priv->flc_desc[0].desc[1], 1); 2523 2524 if (err < 0) { 2525 DPAA2_SEC_ERR("Crypto: Incorrect key lengths"); 2526 ret = -EINVAL; 2527 goto error_out; 2528 } 2529 if (priv->flc_desc[0].desc[1] & 1) { 2530 aeaddata.key_type = RTA_DATA_IMM; 2531 } else { 2532 aeaddata.key = DPAA2_VADDR_TO_IOVA(aeaddata.key); 2533 aeaddata.key_type = RTA_DATA_PTR; 2534 } 2535 priv->flc_desc[0].desc[0] = 0; 2536 priv->flc_desc[0].desc[1] = 0; 2537 2538 if (session->dir == DIR_ENC) 2539 bufsize = cnstr_shdsc_gcm_encap( 2540 priv->flc_desc[0].desc, 1, 0, SHR_NEVER, 2541 &aeaddata, session->iv.length, 2542 session->digest_length); 2543 else 2544 bufsize = cnstr_shdsc_gcm_decap( 2545 priv->flc_desc[0].desc, 1, 0, SHR_NEVER, 2546 &aeaddata, session->iv.length, 2547 session->digest_length); 2548 if (bufsize < 0) { 2549 DPAA2_SEC_ERR("Crypto: Invalid buffer length"); 2550 ret = -EINVAL; 2551 goto error_out; 2552 } 2553 2554 flc->word1_sdl = (uint8_t)bufsize; 2555 session->ctxt = priv; 2556 #ifdef CAAM_DESC_DEBUG 2557 int i; 2558 for (i = 0; i < bufsize; i++) 2559 DPAA2_SEC_DEBUG("DESC[%d]:0x%x\n", 2560 i, priv->flc_desc[0].desc[i]); 2561 #endif 2562 return ret; 2563 2564 error_out: 2565 rte_free(session->aead_key.data); 2566 rte_free(priv); 2567 return ret; 2568 } 2569 2570 2571 static int 2572 dpaa2_sec_aead_chain_init(struct rte_cryptodev *dev, 2573 struct rte_crypto_sym_xform *xform, 2574 dpaa2_sec_session *session) 2575 { 2576 struct dpaa2_sec_dev_private *dev_priv = dev->data->dev_private; 2577 struct alginfo authdata, cipherdata; 2578 int bufsize; 2579 struct ctxt_priv *priv; 2580 struct sec_flow_context *flc; 2581 struct rte_crypto_cipher_xform *cipher_xform; 2582 struct rte_crypto_auth_xform *auth_xform; 2583 int err, ret = 0; 2584 2585 PMD_INIT_FUNC_TRACE(); 2586 2587 if (session->ext_params.aead_ctxt.auth_cipher_text) { 2588 cipher_xform = &xform->cipher; 2589 auth_xform = &xform->next->auth; 2590 session->ctxt_type = 2591 (cipher_xform->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ? 2592 DPAA2_SEC_CIPHER_HASH : DPAA2_SEC_HASH_CIPHER; 2593 } else { 2594 cipher_xform = &xform->next->cipher; 2595 auth_xform = &xform->auth; 2596 session->ctxt_type = 2597 (cipher_xform->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ? 2598 DPAA2_SEC_HASH_CIPHER : DPAA2_SEC_CIPHER_HASH; 2599 } 2600 2601 /* Set IV parameters */ 2602 session->iv.offset = cipher_xform->iv.offset; 2603 session->iv.length = cipher_xform->iv.length; 2604 2605 /* For SEC AEAD only one descriptor is required */ 2606 priv = (struct ctxt_priv *)rte_zmalloc(NULL, 2607 sizeof(struct ctxt_priv) + sizeof(struct sec_flc_desc), 2608 RTE_CACHE_LINE_SIZE); 2609 if (priv == NULL) { 2610 DPAA2_SEC_ERR("No Memory for priv CTXT"); 2611 return -ENOMEM; 2612 } 2613 2614 priv->fle_pool = dev_priv->fle_pool; 2615 flc = &priv->flc_desc[0].flc; 2616 2617 session->cipher_key.data = rte_zmalloc(NULL, cipher_xform->key.length, 2618 RTE_CACHE_LINE_SIZE); 2619 if (session->cipher_key.data == NULL && cipher_xform->key.length > 0) { 2620 DPAA2_SEC_ERR("No Memory for cipher key"); 2621 rte_free(priv); 2622 return -ENOMEM; 2623 } 2624 session->cipher_key.length = cipher_xform->key.length; 2625 session->auth_key.data = rte_zmalloc(NULL, auth_xform->key.length, 2626 RTE_CACHE_LINE_SIZE); 2627 if (session->auth_key.data == NULL && auth_xform->key.length > 0) { 2628 DPAA2_SEC_ERR("No Memory for auth key"); 2629 rte_free(session->cipher_key.data); 2630 rte_free(priv); 2631 return -ENOMEM; 2632 } 2633 session->auth_key.length = auth_xform->key.length; 2634 memcpy(session->cipher_key.data, cipher_xform->key.data, 2635 cipher_xform->key.length); 2636 memcpy(session->auth_key.data, auth_xform->key.data, 2637 auth_xform->key.length); 2638 2639 authdata.key = (size_t)session->auth_key.data; 2640 authdata.keylen = session->auth_key.length; 2641 authdata.key_enc_flags = 0; 2642 authdata.key_type = RTA_DATA_IMM; 2643 2644 session->digest_length = auth_xform->digest_length; 2645 2646 switch (auth_xform->algo) { 2647 case RTE_CRYPTO_AUTH_SHA1_HMAC: 2648 authdata.algtype = OP_ALG_ALGSEL_SHA1; 2649 authdata.algmode = OP_ALG_AAI_HMAC; 2650 session->auth_alg = RTE_CRYPTO_AUTH_SHA1_HMAC; 2651 break; 2652 case RTE_CRYPTO_AUTH_MD5_HMAC: 2653 authdata.algtype = OP_ALG_ALGSEL_MD5; 2654 authdata.algmode = OP_ALG_AAI_HMAC; 2655 session->auth_alg = RTE_CRYPTO_AUTH_MD5_HMAC; 2656 break; 2657 case RTE_CRYPTO_AUTH_SHA224_HMAC: 2658 authdata.algtype = OP_ALG_ALGSEL_SHA224; 2659 authdata.algmode = OP_ALG_AAI_HMAC; 2660 session->auth_alg = RTE_CRYPTO_AUTH_SHA224_HMAC; 2661 break; 2662 case RTE_CRYPTO_AUTH_SHA256_HMAC: 2663 authdata.algtype = OP_ALG_ALGSEL_SHA256; 2664 authdata.algmode = OP_ALG_AAI_HMAC; 2665 session->auth_alg = RTE_CRYPTO_AUTH_SHA256_HMAC; 2666 break; 2667 case RTE_CRYPTO_AUTH_SHA384_HMAC: 2668 authdata.algtype = OP_ALG_ALGSEL_SHA384; 2669 authdata.algmode = OP_ALG_AAI_HMAC; 2670 session->auth_alg = RTE_CRYPTO_AUTH_SHA384_HMAC; 2671 break; 2672 case RTE_CRYPTO_AUTH_SHA512_HMAC: 2673 authdata.algtype = OP_ALG_ALGSEL_SHA512; 2674 authdata.algmode = OP_ALG_AAI_HMAC; 2675 session->auth_alg = RTE_CRYPTO_AUTH_SHA512_HMAC; 2676 break; 2677 case RTE_CRYPTO_AUTH_AES_XCBC_MAC: 2678 authdata.algtype = OP_ALG_ALGSEL_AES; 2679 authdata.algmode = OP_ALG_AAI_XCBC_MAC; 2680 session->auth_alg = RTE_CRYPTO_AUTH_AES_XCBC_MAC; 2681 break; 2682 case RTE_CRYPTO_AUTH_AES_CMAC: 2683 authdata.algtype = OP_ALG_ALGSEL_AES; 2684 authdata.algmode = OP_ALG_AAI_CMAC; 2685 session->auth_alg = RTE_CRYPTO_AUTH_AES_CMAC; 2686 break; 2687 case RTE_CRYPTO_AUTH_AES_CBC_MAC: 2688 case RTE_CRYPTO_AUTH_AES_GMAC: 2689 case RTE_CRYPTO_AUTH_SNOW3G_UIA2: 2690 case RTE_CRYPTO_AUTH_NULL: 2691 case RTE_CRYPTO_AUTH_SHA1: 2692 case RTE_CRYPTO_AUTH_SHA256: 2693 case RTE_CRYPTO_AUTH_SHA512: 2694 case RTE_CRYPTO_AUTH_SHA224: 2695 case RTE_CRYPTO_AUTH_SHA384: 2696 case RTE_CRYPTO_AUTH_MD5: 2697 case RTE_CRYPTO_AUTH_KASUMI_F9: 2698 case RTE_CRYPTO_AUTH_ZUC_EIA3: 2699 DPAA2_SEC_ERR("Crypto: Unsupported auth alg %u", 2700 auth_xform->algo); 2701 ret = -ENOTSUP; 2702 goto error_out; 2703 default: 2704 DPAA2_SEC_ERR("Crypto: Undefined Auth specified %u", 2705 auth_xform->algo); 2706 ret = -ENOTSUP; 2707 goto error_out; 2708 } 2709 cipherdata.key = (size_t)session->cipher_key.data; 2710 cipherdata.keylen = session->cipher_key.length; 2711 cipherdata.key_enc_flags = 0; 2712 cipherdata.key_type = RTA_DATA_IMM; 2713 2714 switch (cipher_xform->algo) { 2715 case RTE_CRYPTO_CIPHER_AES_CBC: 2716 cipherdata.algtype = OP_ALG_ALGSEL_AES; 2717 cipherdata.algmode = OP_ALG_AAI_CBC; 2718 session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CBC; 2719 break; 2720 case RTE_CRYPTO_CIPHER_3DES_CBC: 2721 cipherdata.algtype = OP_ALG_ALGSEL_3DES; 2722 cipherdata.algmode = OP_ALG_AAI_CBC; 2723 session->cipher_alg = RTE_CRYPTO_CIPHER_3DES_CBC; 2724 break; 2725 case RTE_CRYPTO_CIPHER_DES_CBC: 2726 cipherdata.algtype = OP_ALG_ALGSEL_DES; 2727 cipherdata.algmode = OP_ALG_AAI_CBC; 2728 session->cipher_alg = RTE_CRYPTO_CIPHER_DES_CBC; 2729 break; 2730 case RTE_CRYPTO_CIPHER_AES_CTR: 2731 cipherdata.algtype = OP_ALG_ALGSEL_AES; 2732 cipherdata.algmode = OP_ALG_AAI_CTR; 2733 session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CTR; 2734 break; 2735 case RTE_CRYPTO_CIPHER_SNOW3G_UEA2: 2736 case RTE_CRYPTO_CIPHER_ZUC_EEA3: 2737 case RTE_CRYPTO_CIPHER_NULL: 2738 case RTE_CRYPTO_CIPHER_3DES_ECB: 2739 case RTE_CRYPTO_CIPHER_3DES_CTR: 2740 case RTE_CRYPTO_CIPHER_AES_ECB: 2741 case RTE_CRYPTO_CIPHER_KASUMI_F8: 2742 DPAA2_SEC_ERR("Crypto: Unsupported Cipher alg %u", 2743 cipher_xform->algo); 2744 ret = -ENOTSUP; 2745 goto error_out; 2746 default: 2747 DPAA2_SEC_ERR("Crypto: Undefined Cipher specified %u", 2748 cipher_xform->algo); 2749 ret = -ENOTSUP; 2750 goto error_out; 2751 } 2752 session->dir = (cipher_xform->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ? 2753 DIR_ENC : DIR_DEC; 2754 2755 priv->flc_desc[0].desc[0] = cipherdata.keylen; 2756 priv->flc_desc[0].desc[1] = authdata.keylen; 2757 err = rta_inline_query(IPSEC_AUTH_VAR_AES_DEC_BASE_DESC_LEN, 2758 DESC_JOB_IO_LEN, 2759 (unsigned int *)priv->flc_desc[0].desc, 2760 &priv->flc_desc[0].desc[2], 2); 2761 2762 if (err < 0) { 2763 DPAA2_SEC_ERR("Crypto: Incorrect key lengths"); 2764 ret = -EINVAL; 2765 goto error_out; 2766 } 2767 if (priv->flc_desc[0].desc[2] & 1) { 2768 cipherdata.key_type = RTA_DATA_IMM; 2769 } else { 2770 cipherdata.key = DPAA2_VADDR_TO_IOVA(cipherdata.key); 2771 cipherdata.key_type = RTA_DATA_PTR; 2772 } 2773 if (priv->flc_desc[0].desc[2] & (1 << 1)) { 2774 authdata.key_type = RTA_DATA_IMM; 2775 } else { 2776 authdata.key = DPAA2_VADDR_TO_IOVA(authdata.key); 2777 authdata.key_type = RTA_DATA_PTR; 2778 } 2779 priv->flc_desc[0].desc[0] = 0; 2780 priv->flc_desc[0].desc[1] = 0; 2781 priv->flc_desc[0].desc[2] = 0; 2782 2783 if (session->ctxt_type == DPAA2_SEC_CIPHER_HASH) { 2784 bufsize = cnstr_shdsc_authenc(priv->flc_desc[0].desc, 1, 2785 0, SHR_SERIAL, 2786 &cipherdata, &authdata, 2787 session->iv.length, 2788 session->digest_length, 2789 session->dir); 2790 if (bufsize < 0) { 2791 DPAA2_SEC_ERR("Crypto: Invalid buffer length"); 2792 ret = -EINVAL; 2793 goto error_out; 2794 } 2795 } else { 2796 DPAA2_SEC_ERR("Hash before cipher not supported"); 2797 ret = -ENOTSUP; 2798 goto error_out; 2799 } 2800 2801 flc->word1_sdl = (uint8_t)bufsize; 2802 session->ctxt = priv; 2803 #ifdef CAAM_DESC_DEBUG 2804 int i; 2805 for (i = 0; i < bufsize; i++) 2806 DPAA2_SEC_DEBUG("DESC[%d]:0x%x", 2807 i, priv->flc_desc[0].desc[i]); 2808 #endif 2809 2810 return ret; 2811 2812 error_out: 2813 rte_free(session->cipher_key.data); 2814 rte_free(session->auth_key.data); 2815 rte_free(priv); 2816 return ret; 2817 } 2818 2819 static int 2820 dpaa2_sec_set_session_parameters(struct rte_cryptodev *dev, 2821 struct rte_crypto_sym_xform *xform, void *sess) 2822 { 2823 dpaa2_sec_session *session = sess; 2824 int ret; 2825 2826 PMD_INIT_FUNC_TRACE(); 2827 2828 if (unlikely(sess == NULL)) { 2829 DPAA2_SEC_ERR("Invalid session struct"); 2830 return -EINVAL; 2831 } 2832 2833 memset(session, 0, sizeof(dpaa2_sec_session)); 2834 /* Default IV length = 0 */ 2835 session->iv.length = 0; 2836 2837 /* Cipher Only */ 2838 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER && xform->next == NULL) { 2839 ret = dpaa2_sec_cipher_init(dev, xform, session); 2840 2841 /* Authentication Only */ 2842 } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH && 2843 xform->next == NULL) { 2844 ret = dpaa2_sec_auth_init(dev, xform, session); 2845 2846 /* Cipher then Authenticate */ 2847 } else if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER && 2848 xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) { 2849 session->ext_params.aead_ctxt.auth_cipher_text = true; 2850 if (xform->cipher.algo == RTE_CRYPTO_CIPHER_NULL) 2851 ret = dpaa2_sec_auth_init(dev, xform, session); 2852 else if (xform->next->auth.algo == RTE_CRYPTO_AUTH_NULL) 2853 ret = dpaa2_sec_cipher_init(dev, xform, session); 2854 else 2855 ret = dpaa2_sec_aead_chain_init(dev, xform, session); 2856 /* Authenticate then Cipher */ 2857 } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH && 2858 xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER) { 2859 session->ext_params.aead_ctxt.auth_cipher_text = false; 2860 if (xform->auth.algo == RTE_CRYPTO_AUTH_NULL) 2861 ret = dpaa2_sec_cipher_init(dev, xform, session); 2862 else if (xform->next->cipher.algo == RTE_CRYPTO_CIPHER_NULL) 2863 ret = dpaa2_sec_auth_init(dev, xform, session); 2864 else 2865 ret = dpaa2_sec_aead_chain_init(dev, xform, session); 2866 /* AEAD operation for AES-GCM kind of Algorithms */ 2867 } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD && 2868 xform->next == NULL) { 2869 ret = dpaa2_sec_aead_init(dev, xform, session); 2870 2871 } else { 2872 DPAA2_SEC_ERR("Invalid crypto type"); 2873 return -EINVAL; 2874 } 2875 2876 return ret; 2877 } 2878 2879 #ifdef RTE_LIB_SECURITY 2880 static int 2881 dpaa2_sec_ipsec_aead_init(struct rte_crypto_aead_xform *aead_xform, 2882 dpaa2_sec_session *session, 2883 struct alginfo *aeaddata) 2884 { 2885 PMD_INIT_FUNC_TRACE(); 2886 2887 session->aead_key.data = rte_zmalloc(NULL, aead_xform->key.length, 2888 RTE_CACHE_LINE_SIZE); 2889 if (session->aead_key.data == NULL && aead_xform->key.length > 0) { 2890 DPAA2_SEC_ERR("No Memory for aead key"); 2891 return -ENOMEM; 2892 } 2893 memcpy(session->aead_key.data, aead_xform->key.data, 2894 aead_xform->key.length); 2895 2896 session->digest_length = aead_xform->digest_length; 2897 session->aead_key.length = aead_xform->key.length; 2898 2899 aeaddata->key = (size_t)session->aead_key.data; 2900 aeaddata->keylen = session->aead_key.length; 2901 aeaddata->key_enc_flags = 0; 2902 aeaddata->key_type = RTA_DATA_IMM; 2903 2904 switch (aead_xform->algo) { 2905 case RTE_CRYPTO_AEAD_AES_GCM: 2906 switch (session->digest_length) { 2907 case 8: 2908 aeaddata->algtype = OP_PCL_IPSEC_AES_GCM8; 2909 break; 2910 case 12: 2911 aeaddata->algtype = OP_PCL_IPSEC_AES_GCM12; 2912 break; 2913 case 16: 2914 aeaddata->algtype = OP_PCL_IPSEC_AES_GCM16; 2915 break; 2916 default: 2917 DPAA2_SEC_ERR("Crypto: Undefined GCM digest %d", 2918 session->digest_length); 2919 return -EINVAL; 2920 } 2921 aeaddata->algmode = OP_ALG_AAI_GCM; 2922 session->aead_alg = RTE_CRYPTO_AEAD_AES_GCM; 2923 break; 2924 case RTE_CRYPTO_AEAD_AES_CCM: 2925 switch (session->digest_length) { 2926 case 8: 2927 aeaddata->algtype = OP_PCL_IPSEC_AES_CCM8; 2928 break; 2929 case 12: 2930 aeaddata->algtype = OP_PCL_IPSEC_AES_CCM12; 2931 break; 2932 case 16: 2933 aeaddata->algtype = OP_PCL_IPSEC_AES_CCM16; 2934 break; 2935 default: 2936 DPAA2_SEC_ERR("Crypto: Undefined CCM digest %d", 2937 session->digest_length); 2938 return -EINVAL; 2939 } 2940 aeaddata->algmode = OP_ALG_AAI_CCM; 2941 session->aead_alg = RTE_CRYPTO_AEAD_AES_CCM; 2942 break; 2943 default: 2944 DPAA2_SEC_ERR("Crypto: Undefined AEAD specified %u", 2945 aead_xform->algo); 2946 return -ENOTSUP; 2947 } 2948 session->dir = (aead_xform->op == RTE_CRYPTO_AEAD_OP_ENCRYPT) ? 2949 DIR_ENC : DIR_DEC; 2950 2951 return 0; 2952 } 2953 2954 static int 2955 dpaa2_sec_ipsec_proto_init(struct rte_crypto_cipher_xform *cipher_xform, 2956 struct rte_crypto_auth_xform *auth_xform, 2957 dpaa2_sec_session *session, 2958 struct alginfo *cipherdata, 2959 struct alginfo *authdata) 2960 { 2961 if (cipher_xform) { 2962 session->cipher_key.data = rte_zmalloc(NULL, 2963 cipher_xform->key.length, 2964 RTE_CACHE_LINE_SIZE); 2965 if (session->cipher_key.data == NULL && 2966 cipher_xform->key.length > 0) { 2967 DPAA2_SEC_ERR("No Memory for cipher key"); 2968 return -ENOMEM; 2969 } 2970 2971 session->cipher_key.length = cipher_xform->key.length; 2972 memcpy(session->cipher_key.data, cipher_xform->key.data, 2973 cipher_xform->key.length); 2974 session->cipher_alg = cipher_xform->algo; 2975 } else { 2976 session->cipher_key.data = NULL; 2977 session->cipher_key.length = 0; 2978 session->cipher_alg = RTE_CRYPTO_CIPHER_NULL; 2979 } 2980 2981 if (auth_xform) { 2982 session->auth_key.data = rte_zmalloc(NULL, 2983 auth_xform->key.length, 2984 RTE_CACHE_LINE_SIZE); 2985 if (session->auth_key.data == NULL && 2986 auth_xform->key.length > 0) { 2987 DPAA2_SEC_ERR("No Memory for auth key"); 2988 return -ENOMEM; 2989 } 2990 session->auth_key.length = auth_xform->key.length; 2991 memcpy(session->auth_key.data, auth_xform->key.data, 2992 auth_xform->key.length); 2993 session->auth_alg = auth_xform->algo; 2994 session->digest_length = auth_xform->digest_length; 2995 } else { 2996 session->auth_key.data = NULL; 2997 session->auth_key.length = 0; 2998 session->auth_alg = RTE_CRYPTO_AUTH_NULL; 2999 } 3000 3001 authdata->key = (size_t)session->auth_key.data; 3002 authdata->keylen = session->auth_key.length; 3003 authdata->key_enc_flags = 0; 3004 authdata->key_type = RTA_DATA_IMM; 3005 switch (session->auth_alg) { 3006 case RTE_CRYPTO_AUTH_SHA1_HMAC: 3007 authdata->algtype = OP_PCL_IPSEC_HMAC_SHA1_96; 3008 authdata->algmode = OP_ALG_AAI_HMAC; 3009 break; 3010 case RTE_CRYPTO_AUTH_MD5_HMAC: 3011 authdata->algtype = OP_PCL_IPSEC_HMAC_MD5_96; 3012 authdata->algmode = OP_ALG_AAI_HMAC; 3013 break; 3014 case RTE_CRYPTO_AUTH_SHA256_HMAC: 3015 authdata->algtype = OP_PCL_IPSEC_HMAC_SHA2_256_128; 3016 authdata->algmode = OP_ALG_AAI_HMAC; 3017 if (session->digest_length != 16) 3018 DPAA2_SEC_WARN( 3019 "+++Using sha256-hmac truncated len is non-standard," 3020 "it will not work with lookaside proto"); 3021 break; 3022 case RTE_CRYPTO_AUTH_SHA384_HMAC: 3023 authdata->algtype = OP_PCL_IPSEC_HMAC_SHA2_384_192; 3024 authdata->algmode = OP_ALG_AAI_HMAC; 3025 break; 3026 case RTE_CRYPTO_AUTH_SHA512_HMAC: 3027 authdata->algtype = OP_PCL_IPSEC_HMAC_SHA2_512_256; 3028 authdata->algmode = OP_ALG_AAI_HMAC; 3029 break; 3030 case RTE_CRYPTO_AUTH_AES_XCBC_MAC: 3031 authdata->algtype = OP_PCL_IPSEC_AES_XCBC_MAC_96; 3032 authdata->algmode = OP_ALG_AAI_XCBC_MAC; 3033 break; 3034 case RTE_CRYPTO_AUTH_AES_CMAC: 3035 authdata->algtype = OP_PCL_IPSEC_AES_CMAC_96; 3036 authdata->algmode = OP_ALG_AAI_CMAC; 3037 break; 3038 case RTE_CRYPTO_AUTH_NULL: 3039 authdata->algtype = OP_PCL_IPSEC_HMAC_NULL; 3040 break; 3041 case RTE_CRYPTO_AUTH_SHA224_HMAC: 3042 case RTE_CRYPTO_AUTH_SNOW3G_UIA2: 3043 case RTE_CRYPTO_AUTH_SHA1: 3044 case RTE_CRYPTO_AUTH_SHA256: 3045 case RTE_CRYPTO_AUTH_SHA512: 3046 case RTE_CRYPTO_AUTH_SHA224: 3047 case RTE_CRYPTO_AUTH_SHA384: 3048 case RTE_CRYPTO_AUTH_MD5: 3049 case RTE_CRYPTO_AUTH_AES_GMAC: 3050 case RTE_CRYPTO_AUTH_KASUMI_F9: 3051 case RTE_CRYPTO_AUTH_AES_CBC_MAC: 3052 case RTE_CRYPTO_AUTH_ZUC_EIA3: 3053 DPAA2_SEC_ERR("Crypto: Unsupported auth alg %u", 3054 session->auth_alg); 3055 return -ENOTSUP; 3056 default: 3057 DPAA2_SEC_ERR("Crypto: Undefined Auth specified %u", 3058 session->auth_alg); 3059 return -ENOTSUP; 3060 } 3061 cipherdata->key = (size_t)session->cipher_key.data; 3062 cipherdata->keylen = session->cipher_key.length; 3063 cipherdata->key_enc_flags = 0; 3064 cipherdata->key_type = RTA_DATA_IMM; 3065 3066 switch (session->cipher_alg) { 3067 case RTE_CRYPTO_CIPHER_AES_CBC: 3068 cipherdata->algtype = OP_PCL_IPSEC_AES_CBC; 3069 cipherdata->algmode = OP_ALG_AAI_CBC; 3070 break; 3071 case RTE_CRYPTO_CIPHER_3DES_CBC: 3072 cipherdata->algtype = OP_PCL_IPSEC_3DES; 3073 cipherdata->algmode = OP_ALG_AAI_CBC; 3074 break; 3075 case RTE_CRYPTO_CIPHER_DES_CBC: 3076 cipherdata->algtype = OP_PCL_IPSEC_DES; 3077 cipherdata->algmode = OP_ALG_AAI_CBC; 3078 break; 3079 case RTE_CRYPTO_CIPHER_AES_CTR: 3080 cipherdata->algtype = OP_PCL_IPSEC_AES_CTR; 3081 cipherdata->algmode = OP_ALG_AAI_CTR; 3082 break; 3083 case RTE_CRYPTO_CIPHER_NULL: 3084 cipherdata->algtype = OP_PCL_IPSEC_NULL; 3085 break; 3086 case RTE_CRYPTO_CIPHER_SNOW3G_UEA2: 3087 case RTE_CRYPTO_CIPHER_ZUC_EEA3: 3088 case RTE_CRYPTO_CIPHER_3DES_ECB: 3089 case RTE_CRYPTO_CIPHER_3DES_CTR: 3090 case RTE_CRYPTO_CIPHER_AES_ECB: 3091 case RTE_CRYPTO_CIPHER_KASUMI_F8: 3092 DPAA2_SEC_ERR("Crypto: Unsupported Cipher alg %u", 3093 session->cipher_alg); 3094 return -ENOTSUP; 3095 default: 3096 DPAA2_SEC_ERR("Crypto: Undefined Cipher specified %u", 3097 session->cipher_alg); 3098 return -ENOTSUP; 3099 } 3100 3101 return 0; 3102 } 3103 3104 static int 3105 dpaa2_sec_set_ipsec_session(struct rte_cryptodev *dev, 3106 struct rte_security_session_conf *conf, 3107 void *sess) 3108 { 3109 struct rte_security_ipsec_xform *ipsec_xform = &conf->ipsec; 3110 struct rte_crypto_cipher_xform *cipher_xform = NULL; 3111 struct rte_crypto_auth_xform *auth_xform = NULL; 3112 struct rte_crypto_aead_xform *aead_xform = NULL; 3113 dpaa2_sec_session *session = (dpaa2_sec_session *)sess; 3114 struct ctxt_priv *priv; 3115 struct alginfo authdata, cipherdata; 3116 int bufsize; 3117 struct sec_flow_context *flc; 3118 struct dpaa2_sec_dev_private *dev_priv = dev->data->dev_private; 3119 int ret = -1; 3120 3121 PMD_INIT_FUNC_TRACE(); 3122 3123 priv = (struct ctxt_priv *)rte_zmalloc(NULL, 3124 sizeof(struct ctxt_priv) + 3125 sizeof(struct sec_flc_desc), 3126 RTE_CACHE_LINE_SIZE); 3127 3128 if (priv == NULL) { 3129 DPAA2_SEC_ERR("No memory for priv CTXT"); 3130 return -ENOMEM; 3131 } 3132 3133 priv->fle_pool = dev_priv->fle_pool; 3134 flc = &priv->flc_desc[0].flc; 3135 3136 if (ipsec_xform->life.bytes_hard_limit != 0 || 3137 ipsec_xform->life.bytes_soft_limit != 0 || 3138 ipsec_xform->life.packets_hard_limit != 0 || 3139 ipsec_xform->life.packets_soft_limit != 0) 3140 return -ENOTSUP; 3141 3142 memset(session, 0, sizeof(dpaa2_sec_session)); 3143 3144 if (conf->crypto_xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) { 3145 cipher_xform = &conf->crypto_xform->cipher; 3146 if (conf->crypto_xform->next) 3147 auth_xform = &conf->crypto_xform->next->auth; 3148 ret = dpaa2_sec_ipsec_proto_init(cipher_xform, auth_xform, 3149 session, &cipherdata, &authdata); 3150 } else if (conf->crypto_xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) { 3151 auth_xform = &conf->crypto_xform->auth; 3152 if (conf->crypto_xform->next) 3153 cipher_xform = &conf->crypto_xform->next->cipher; 3154 ret = dpaa2_sec_ipsec_proto_init(cipher_xform, auth_xform, 3155 session, &cipherdata, &authdata); 3156 } else if (conf->crypto_xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) { 3157 aead_xform = &conf->crypto_xform->aead; 3158 ret = dpaa2_sec_ipsec_aead_init(aead_xform, 3159 session, &cipherdata); 3160 authdata.keylen = 0; 3161 authdata.algtype = 0; 3162 } else { 3163 DPAA2_SEC_ERR("XFORM not specified"); 3164 ret = -EINVAL; 3165 goto out; 3166 } 3167 if (ret) { 3168 DPAA2_SEC_ERR("Failed to process xform"); 3169 goto out; 3170 } 3171 3172 session->ctxt_type = DPAA2_SEC_IPSEC; 3173 if (ipsec_xform->direction == RTE_SECURITY_IPSEC_SA_DIR_EGRESS) { 3174 uint8_t *hdr = NULL; 3175 struct ip ip4_hdr; 3176 struct rte_ipv6_hdr ip6_hdr; 3177 struct ipsec_encap_pdb encap_pdb; 3178 3179 flc->dhr = SEC_FLC_DHR_OUTBOUND; 3180 /* For Sec Proto only one descriptor is required. */ 3181 memset(&encap_pdb, 0, sizeof(struct ipsec_encap_pdb)); 3182 3183 /* copy algo specific data to PDB */ 3184 switch (cipherdata.algtype) { 3185 case OP_PCL_IPSEC_AES_CTR: 3186 encap_pdb.ctr.ctr_initial = 0x00000001; 3187 encap_pdb.ctr.ctr_nonce = ipsec_xform->salt; 3188 break; 3189 case OP_PCL_IPSEC_AES_GCM8: 3190 case OP_PCL_IPSEC_AES_GCM12: 3191 case OP_PCL_IPSEC_AES_GCM16: 3192 memcpy(encap_pdb.gcm.salt, 3193 (uint8_t *)&(ipsec_xform->salt), 4); 3194 break; 3195 } 3196 3197 encap_pdb.options = (IPVERSION << PDBNH_ESP_ENCAP_SHIFT) | 3198 PDBOPTS_ESP_OIHI_PDB_INL | 3199 PDBOPTS_ESP_IVSRC | 3200 PDBHMO_ESP_SNR; 3201 if (ipsec_xform->options.dec_ttl) 3202 encap_pdb.options |= PDBHMO_ESP_ENCAP_DTTL; 3203 if (ipsec_xform->options.esn) 3204 encap_pdb.options |= PDBOPTS_ESP_ESN; 3205 encap_pdb.spi = ipsec_xform->spi; 3206 session->dir = DIR_ENC; 3207 if (ipsec_xform->tunnel.type == 3208 RTE_SECURITY_IPSEC_TUNNEL_IPV4) { 3209 encap_pdb.ip_hdr_len = sizeof(struct ip); 3210 ip4_hdr.ip_v = IPVERSION; 3211 ip4_hdr.ip_hl = 5; 3212 ip4_hdr.ip_len = rte_cpu_to_be_16(sizeof(ip4_hdr)); 3213 ip4_hdr.ip_tos = ipsec_xform->tunnel.ipv4.dscp; 3214 ip4_hdr.ip_id = 0; 3215 ip4_hdr.ip_off = 0; 3216 ip4_hdr.ip_ttl = ipsec_xform->tunnel.ipv4.ttl; 3217 ip4_hdr.ip_p = IPPROTO_ESP; 3218 ip4_hdr.ip_sum = 0; 3219 ip4_hdr.ip_src = ipsec_xform->tunnel.ipv4.src_ip; 3220 ip4_hdr.ip_dst = ipsec_xform->tunnel.ipv4.dst_ip; 3221 ip4_hdr.ip_sum = calc_chksum((uint16_t *)(void *) 3222 &ip4_hdr, sizeof(struct ip)); 3223 hdr = (uint8_t *)&ip4_hdr; 3224 } else if (ipsec_xform->tunnel.type == 3225 RTE_SECURITY_IPSEC_TUNNEL_IPV6) { 3226 ip6_hdr.vtc_flow = rte_cpu_to_be_32( 3227 DPAA2_IPv6_DEFAULT_VTC_FLOW | 3228 ((ipsec_xform->tunnel.ipv6.dscp << 3229 RTE_IPV6_HDR_TC_SHIFT) & 3230 RTE_IPV6_HDR_TC_MASK) | 3231 ((ipsec_xform->tunnel.ipv6.flabel << 3232 RTE_IPV6_HDR_FL_SHIFT) & 3233 RTE_IPV6_HDR_FL_MASK)); 3234 /* Payload length will be updated by HW */ 3235 ip6_hdr.payload_len = 0; 3236 ip6_hdr.hop_limits = 3237 ipsec_xform->tunnel.ipv6.hlimit; 3238 ip6_hdr.proto = (ipsec_xform->proto == 3239 RTE_SECURITY_IPSEC_SA_PROTO_ESP) ? 3240 IPPROTO_ESP : IPPROTO_AH; 3241 memcpy(&ip6_hdr.src_addr, 3242 &ipsec_xform->tunnel.ipv6.src_addr, 16); 3243 memcpy(&ip6_hdr.dst_addr, 3244 &ipsec_xform->tunnel.ipv6.dst_addr, 16); 3245 encap_pdb.ip_hdr_len = sizeof(struct rte_ipv6_hdr); 3246 hdr = (uint8_t *)&ip6_hdr; 3247 } 3248 3249 bufsize = cnstr_shdsc_ipsec_new_encap(priv->flc_desc[0].desc, 3250 1, 0, (rta_sec_era >= RTA_SEC_ERA_10) ? 3251 SHR_WAIT : SHR_SERIAL, &encap_pdb, 3252 hdr, &cipherdata, &authdata); 3253 } else if (ipsec_xform->direction == 3254 RTE_SECURITY_IPSEC_SA_DIR_INGRESS) { 3255 struct ipsec_decap_pdb decap_pdb; 3256 3257 flc->dhr = SEC_FLC_DHR_INBOUND; 3258 memset(&decap_pdb, 0, sizeof(struct ipsec_decap_pdb)); 3259 /* copy algo specific data to PDB */ 3260 switch (cipherdata.algtype) { 3261 case OP_PCL_IPSEC_AES_CTR: 3262 decap_pdb.ctr.ctr_initial = 0x00000001; 3263 decap_pdb.ctr.ctr_nonce = ipsec_xform->salt; 3264 break; 3265 case OP_PCL_IPSEC_AES_GCM8: 3266 case OP_PCL_IPSEC_AES_GCM12: 3267 case OP_PCL_IPSEC_AES_GCM16: 3268 memcpy(decap_pdb.gcm.salt, 3269 (uint8_t *)&(ipsec_xform->salt), 4); 3270 break; 3271 } 3272 3273 decap_pdb.options = (ipsec_xform->tunnel.type == 3274 RTE_SECURITY_IPSEC_TUNNEL_IPV4) ? 3275 sizeof(struct ip) << 16 : 3276 sizeof(struct rte_ipv6_hdr) << 16; 3277 if (ipsec_xform->options.esn) 3278 decap_pdb.options |= PDBOPTS_ESP_ESN; 3279 3280 if (ipsec_xform->replay_win_sz) { 3281 uint32_t win_sz; 3282 win_sz = rte_align32pow2(ipsec_xform->replay_win_sz); 3283 3284 if (rta_sec_era < RTA_SEC_ERA_10 && win_sz > 128) { 3285 DPAA2_SEC_INFO("Max Anti replay Win sz = 128"); 3286 win_sz = 128; 3287 } 3288 switch (win_sz) { 3289 case 1: 3290 case 2: 3291 case 4: 3292 case 8: 3293 case 16: 3294 case 32: 3295 decap_pdb.options |= PDBOPTS_ESP_ARS32; 3296 break; 3297 case 64: 3298 decap_pdb.options |= PDBOPTS_ESP_ARS64; 3299 break; 3300 case 256: 3301 decap_pdb.options |= PDBOPTS_ESP_ARS256; 3302 break; 3303 case 512: 3304 decap_pdb.options |= PDBOPTS_ESP_ARS512; 3305 break; 3306 case 1024: 3307 decap_pdb.options |= PDBOPTS_ESP_ARS1024; 3308 break; 3309 case 128: 3310 default: 3311 decap_pdb.options |= PDBOPTS_ESP_ARS128; 3312 } 3313 } 3314 session->dir = DIR_DEC; 3315 bufsize = cnstr_shdsc_ipsec_new_decap(priv->flc_desc[0].desc, 3316 1, 0, (rta_sec_era >= RTA_SEC_ERA_10) ? 3317 SHR_WAIT : SHR_SERIAL, 3318 &decap_pdb, &cipherdata, &authdata); 3319 } else 3320 goto out; 3321 3322 if (bufsize < 0) { 3323 DPAA2_SEC_ERR("Crypto: Invalid buffer length"); 3324 goto out; 3325 } 3326 3327 flc->word1_sdl = (uint8_t)bufsize; 3328 3329 /* Enable the stashing control bit */ 3330 DPAA2_SET_FLC_RSC(flc); 3331 flc->word2_rflc_31_0 = lower_32_bits( 3332 (size_t)&(((struct dpaa2_sec_qp *) 3333 dev->data->queue_pairs[0])->rx_vq) | 0x14); 3334 flc->word3_rflc_63_32 = upper_32_bits( 3335 (size_t)&(((struct dpaa2_sec_qp *) 3336 dev->data->queue_pairs[0])->rx_vq)); 3337 3338 /* Set EWS bit i.e. enable write-safe */ 3339 DPAA2_SET_FLC_EWS(flc); 3340 /* Set BS = 1 i.e reuse input buffers as output buffers */ 3341 DPAA2_SET_FLC_REUSE_BS(flc); 3342 /* Set FF = 10; reuse input buffers if they provide sufficient space */ 3343 DPAA2_SET_FLC_REUSE_FF(flc); 3344 3345 session->ctxt = priv; 3346 3347 return 0; 3348 out: 3349 rte_free(session->auth_key.data); 3350 rte_free(session->cipher_key.data); 3351 rte_free(priv); 3352 return ret; 3353 } 3354 3355 static int 3356 dpaa2_sec_set_pdcp_session(struct rte_cryptodev *dev, 3357 struct rte_security_session_conf *conf, 3358 void *sess) 3359 { 3360 struct rte_security_pdcp_xform *pdcp_xform = &conf->pdcp; 3361 struct rte_crypto_sym_xform *xform = conf->crypto_xform; 3362 struct rte_crypto_auth_xform *auth_xform = NULL; 3363 struct rte_crypto_cipher_xform *cipher_xform = NULL; 3364 dpaa2_sec_session *session = (dpaa2_sec_session *)sess; 3365 struct ctxt_priv *priv; 3366 struct dpaa2_sec_dev_private *dev_priv = dev->data->dev_private; 3367 struct alginfo authdata, cipherdata; 3368 struct alginfo *p_authdata = NULL; 3369 int bufsize = -1; 3370 struct sec_flow_context *flc; 3371 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN 3372 int swap = true; 3373 #else 3374 int swap = false; 3375 #endif 3376 3377 PMD_INIT_FUNC_TRACE(); 3378 3379 memset(session, 0, sizeof(dpaa2_sec_session)); 3380 3381 priv = (struct ctxt_priv *)rte_zmalloc(NULL, 3382 sizeof(struct ctxt_priv) + 3383 sizeof(struct sec_flc_desc), 3384 RTE_CACHE_LINE_SIZE); 3385 3386 if (priv == NULL) { 3387 DPAA2_SEC_ERR("No memory for priv CTXT"); 3388 return -ENOMEM; 3389 } 3390 3391 priv->fle_pool = dev_priv->fle_pool; 3392 flc = &priv->flc_desc[0].flc; 3393 3394 /* find xfrm types */ 3395 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) { 3396 cipher_xform = &xform->cipher; 3397 if (xform->next != NULL && 3398 xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) { 3399 session->ext_params.aead_ctxt.auth_cipher_text = true; 3400 auth_xform = &xform->next->auth; 3401 } 3402 } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) { 3403 auth_xform = &xform->auth; 3404 if (xform->next != NULL && 3405 xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER) { 3406 session->ext_params.aead_ctxt.auth_cipher_text = false; 3407 cipher_xform = &xform->next->cipher; 3408 } 3409 } else { 3410 DPAA2_SEC_ERR("Invalid crypto type"); 3411 return -EINVAL; 3412 } 3413 3414 session->ctxt_type = DPAA2_SEC_PDCP; 3415 if (cipher_xform) { 3416 session->cipher_key.data = rte_zmalloc(NULL, 3417 cipher_xform->key.length, 3418 RTE_CACHE_LINE_SIZE); 3419 if (session->cipher_key.data == NULL && 3420 cipher_xform->key.length > 0) { 3421 DPAA2_SEC_ERR("No Memory for cipher key"); 3422 rte_free(priv); 3423 return -ENOMEM; 3424 } 3425 session->cipher_key.length = cipher_xform->key.length; 3426 memcpy(session->cipher_key.data, cipher_xform->key.data, 3427 cipher_xform->key.length); 3428 session->dir = 3429 (cipher_xform->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ? 3430 DIR_ENC : DIR_DEC; 3431 session->cipher_alg = cipher_xform->algo; 3432 } else { 3433 session->cipher_key.data = NULL; 3434 session->cipher_key.length = 0; 3435 session->cipher_alg = RTE_CRYPTO_CIPHER_NULL; 3436 session->dir = DIR_ENC; 3437 } 3438 3439 session->pdcp.domain = pdcp_xform->domain; 3440 session->pdcp.bearer = pdcp_xform->bearer; 3441 session->pdcp.pkt_dir = pdcp_xform->pkt_dir; 3442 session->pdcp.sn_size = pdcp_xform->sn_size; 3443 session->pdcp.hfn = pdcp_xform->hfn; 3444 session->pdcp.hfn_threshold = pdcp_xform->hfn_threshold; 3445 session->pdcp.hfn_ovd = pdcp_xform->hfn_ovrd; 3446 /* hfv ovd offset location is stored in iv.offset value*/ 3447 if (cipher_xform) 3448 session->pdcp.hfn_ovd_offset = cipher_xform->iv.offset; 3449 3450 cipherdata.key = (size_t)session->cipher_key.data; 3451 cipherdata.keylen = session->cipher_key.length; 3452 cipherdata.key_enc_flags = 0; 3453 cipherdata.key_type = RTA_DATA_IMM; 3454 3455 switch (session->cipher_alg) { 3456 case RTE_CRYPTO_CIPHER_SNOW3G_UEA2: 3457 cipherdata.algtype = PDCP_CIPHER_TYPE_SNOW; 3458 break; 3459 case RTE_CRYPTO_CIPHER_ZUC_EEA3: 3460 cipherdata.algtype = PDCP_CIPHER_TYPE_ZUC; 3461 break; 3462 case RTE_CRYPTO_CIPHER_AES_CTR: 3463 cipherdata.algtype = PDCP_CIPHER_TYPE_AES; 3464 break; 3465 case RTE_CRYPTO_CIPHER_NULL: 3466 cipherdata.algtype = PDCP_CIPHER_TYPE_NULL; 3467 break; 3468 default: 3469 DPAA2_SEC_ERR("Crypto: Undefined Cipher specified %u", 3470 session->cipher_alg); 3471 goto out; 3472 } 3473 3474 if (auth_xform) { 3475 session->auth_key.data = rte_zmalloc(NULL, 3476 auth_xform->key.length, 3477 RTE_CACHE_LINE_SIZE); 3478 if (!session->auth_key.data && 3479 auth_xform->key.length > 0) { 3480 DPAA2_SEC_ERR("No Memory for auth key"); 3481 rte_free(session->cipher_key.data); 3482 rte_free(priv); 3483 return -ENOMEM; 3484 } 3485 session->auth_key.length = auth_xform->key.length; 3486 memcpy(session->auth_key.data, auth_xform->key.data, 3487 auth_xform->key.length); 3488 session->auth_alg = auth_xform->algo; 3489 } else { 3490 session->auth_key.data = NULL; 3491 session->auth_key.length = 0; 3492 session->auth_alg = 0; 3493 } 3494 authdata.key = (size_t)session->auth_key.data; 3495 authdata.keylen = session->auth_key.length; 3496 authdata.key_enc_flags = 0; 3497 authdata.key_type = RTA_DATA_IMM; 3498 3499 if (session->auth_alg) { 3500 switch (session->auth_alg) { 3501 case RTE_CRYPTO_AUTH_SNOW3G_UIA2: 3502 authdata.algtype = PDCP_AUTH_TYPE_SNOW; 3503 break; 3504 case RTE_CRYPTO_AUTH_ZUC_EIA3: 3505 authdata.algtype = PDCP_AUTH_TYPE_ZUC; 3506 break; 3507 case RTE_CRYPTO_AUTH_AES_CMAC: 3508 authdata.algtype = PDCP_AUTH_TYPE_AES; 3509 break; 3510 case RTE_CRYPTO_AUTH_NULL: 3511 authdata.algtype = PDCP_AUTH_TYPE_NULL; 3512 break; 3513 default: 3514 DPAA2_SEC_ERR("Crypto: Unsupported auth alg %u", 3515 session->auth_alg); 3516 goto out; 3517 } 3518 3519 p_authdata = &authdata; 3520 } else if (pdcp_xform->domain == RTE_SECURITY_PDCP_MODE_CONTROL) { 3521 DPAA2_SEC_ERR("Crypto: Integrity must for c-plane"); 3522 goto out; 3523 } 3524 3525 if (pdcp_xform->sdap_enabled) { 3526 int nb_keys_to_inline = 3527 rta_inline_pdcp_sdap_query(authdata.algtype, 3528 cipherdata.algtype, 3529 session->pdcp.sn_size, 3530 session->pdcp.hfn_ovd); 3531 if (nb_keys_to_inline >= 1) { 3532 cipherdata.key = DPAA2_VADDR_TO_IOVA(cipherdata.key); 3533 cipherdata.key_type = RTA_DATA_PTR; 3534 } 3535 if (nb_keys_to_inline >= 2) { 3536 authdata.key = DPAA2_VADDR_TO_IOVA(authdata.key); 3537 authdata.key_type = RTA_DATA_PTR; 3538 } 3539 } else { 3540 if (rta_inline_pdcp_query(authdata.algtype, 3541 cipherdata.algtype, 3542 session->pdcp.sn_size, 3543 session->pdcp.hfn_ovd)) { 3544 cipherdata.key = DPAA2_VADDR_TO_IOVA(cipherdata.key); 3545 cipherdata.key_type = RTA_DATA_PTR; 3546 } 3547 } 3548 3549 if (pdcp_xform->domain == RTE_SECURITY_PDCP_MODE_CONTROL) { 3550 if (session->dir == DIR_ENC) 3551 bufsize = cnstr_shdsc_pdcp_c_plane_encap( 3552 priv->flc_desc[0].desc, 1, swap, 3553 pdcp_xform->hfn, 3554 session->pdcp.sn_size, 3555 pdcp_xform->bearer, 3556 pdcp_xform->pkt_dir, 3557 pdcp_xform->hfn_threshold, 3558 &cipherdata, &authdata); 3559 else if (session->dir == DIR_DEC) 3560 bufsize = cnstr_shdsc_pdcp_c_plane_decap( 3561 priv->flc_desc[0].desc, 1, swap, 3562 pdcp_xform->hfn, 3563 session->pdcp.sn_size, 3564 pdcp_xform->bearer, 3565 pdcp_xform->pkt_dir, 3566 pdcp_xform->hfn_threshold, 3567 &cipherdata, &authdata); 3568 3569 } else if (pdcp_xform->domain == RTE_SECURITY_PDCP_MODE_SHORT_MAC) { 3570 bufsize = cnstr_shdsc_pdcp_short_mac(priv->flc_desc[0].desc, 3571 1, swap, &authdata); 3572 } else { 3573 if (session->dir == DIR_ENC) { 3574 if (pdcp_xform->sdap_enabled) 3575 bufsize = cnstr_shdsc_pdcp_sdap_u_plane_encap( 3576 priv->flc_desc[0].desc, 1, swap, 3577 session->pdcp.sn_size, 3578 pdcp_xform->hfn, 3579 pdcp_xform->bearer, 3580 pdcp_xform->pkt_dir, 3581 pdcp_xform->hfn_threshold, 3582 &cipherdata, p_authdata); 3583 else 3584 bufsize = cnstr_shdsc_pdcp_u_plane_encap( 3585 priv->flc_desc[0].desc, 1, swap, 3586 session->pdcp.sn_size, 3587 pdcp_xform->hfn, 3588 pdcp_xform->bearer, 3589 pdcp_xform->pkt_dir, 3590 pdcp_xform->hfn_threshold, 3591 &cipherdata, p_authdata); 3592 } else if (session->dir == DIR_DEC) { 3593 if (pdcp_xform->sdap_enabled) 3594 bufsize = cnstr_shdsc_pdcp_sdap_u_plane_decap( 3595 priv->flc_desc[0].desc, 1, swap, 3596 session->pdcp.sn_size, 3597 pdcp_xform->hfn, 3598 pdcp_xform->bearer, 3599 pdcp_xform->pkt_dir, 3600 pdcp_xform->hfn_threshold, 3601 &cipherdata, p_authdata); 3602 else 3603 bufsize = cnstr_shdsc_pdcp_u_plane_decap( 3604 priv->flc_desc[0].desc, 1, swap, 3605 session->pdcp.sn_size, 3606 pdcp_xform->hfn, 3607 pdcp_xform->bearer, 3608 pdcp_xform->pkt_dir, 3609 pdcp_xform->hfn_threshold, 3610 &cipherdata, p_authdata); 3611 } 3612 } 3613 3614 if (bufsize < 0) { 3615 DPAA2_SEC_ERR("Crypto: Invalid buffer length"); 3616 goto out; 3617 } 3618 3619 /* Enable the stashing control bit */ 3620 DPAA2_SET_FLC_RSC(flc); 3621 flc->word2_rflc_31_0 = lower_32_bits( 3622 (size_t)&(((struct dpaa2_sec_qp *) 3623 dev->data->queue_pairs[0])->rx_vq) | 0x14); 3624 flc->word3_rflc_63_32 = upper_32_bits( 3625 (size_t)&(((struct dpaa2_sec_qp *) 3626 dev->data->queue_pairs[0])->rx_vq)); 3627 3628 flc->word1_sdl = (uint8_t)bufsize; 3629 3630 /* TODO - check the perf impact or 3631 * align as per descriptor type 3632 * Set EWS bit i.e. enable write-safe 3633 * DPAA2_SET_FLC_EWS(flc); 3634 */ 3635 3636 /* Set BS = 1 i.e reuse input buffers as output buffers */ 3637 DPAA2_SET_FLC_REUSE_BS(flc); 3638 /* Set FF = 10; reuse input buffers if they provide sufficient space */ 3639 DPAA2_SET_FLC_REUSE_FF(flc); 3640 3641 session->ctxt = priv; 3642 3643 return 0; 3644 out: 3645 rte_free(session->auth_key.data); 3646 rte_free(session->cipher_key.data); 3647 rte_free(priv); 3648 return -EINVAL; 3649 } 3650 3651 static int 3652 dpaa2_sec_security_session_create(void *dev, 3653 struct rte_security_session_conf *conf, 3654 struct rte_security_session *sess, 3655 struct rte_mempool *mempool) 3656 { 3657 void *sess_private_data; 3658 struct rte_cryptodev *cdev = (struct rte_cryptodev *)dev; 3659 int ret; 3660 3661 if (rte_mempool_get(mempool, &sess_private_data)) { 3662 DPAA2_SEC_ERR("Couldn't get object from session mempool"); 3663 return -ENOMEM; 3664 } 3665 3666 switch (conf->protocol) { 3667 case RTE_SECURITY_PROTOCOL_IPSEC: 3668 ret = dpaa2_sec_set_ipsec_session(cdev, conf, 3669 sess_private_data); 3670 break; 3671 case RTE_SECURITY_PROTOCOL_MACSEC: 3672 return -ENOTSUP; 3673 case RTE_SECURITY_PROTOCOL_PDCP: 3674 ret = dpaa2_sec_set_pdcp_session(cdev, conf, 3675 sess_private_data); 3676 break; 3677 default: 3678 return -EINVAL; 3679 } 3680 if (ret != 0) { 3681 DPAA2_SEC_ERR("Failed to configure session parameters"); 3682 /* Return session to mempool */ 3683 rte_mempool_put(mempool, sess_private_data); 3684 return ret; 3685 } 3686 3687 set_sec_session_private_data(sess, sess_private_data); 3688 3689 return ret; 3690 } 3691 3692 /** Clear the memory of session so it doesn't leave key material behind */ 3693 static int 3694 dpaa2_sec_security_session_destroy(void *dev __rte_unused, 3695 struct rte_security_session *sess) 3696 { 3697 PMD_INIT_FUNC_TRACE(); 3698 void *sess_priv = get_sec_session_private_data(sess); 3699 3700 dpaa2_sec_session *s = (dpaa2_sec_session *)sess_priv; 3701 3702 if (sess_priv) { 3703 struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv); 3704 3705 rte_free(s->ctxt); 3706 rte_free(s->cipher_key.data); 3707 rte_free(s->auth_key.data); 3708 memset(s, 0, sizeof(dpaa2_sec_session)); 3709 set_sec_session_private_data(sess, NULL); 3710 rte_mempool_put(sess_mp, sess_priv); 3711 } 3712 return 0; 3713 } 3714 #endif 3715 static int 3716 dpaa2_sec_sym_session_configure(struct rte_cryptodev *dev, 3717 struct rte_crypto_sym_xform *xform, 3718 struct rte_cryptodev_sym_session *sess, 3719 struct rte_mempool *mempool) 3720 { 3721 void *sess_private_data; 3722 int ret; 3723 3724 if (rte_mempool_get(mempool, &sess_private_data)) { 3725 DPAA2_SEC_ERR("Couldn't get object from session mempool"); 3726 return -ENOMEM; 3727 } 3728 3729 ret = dpaa2_sec_set_session_parameters(dev, xform, sess_private_data); 3730 if (ret != 0) { 3731 DPAA2_SEC_ERR("Failed to configure session parameters"); 3732 /* Return session to mempool */ 3733 rte_mempool_put(mempool, sess_private_data); 3734 return ret; 3735 } 3736 3737 set_sym_session_private_data(sess, dev->driver_id, 3738 sess_private_data); 3739 3740 return 0; 3741 } 3742 3743 /** Clear the memory of session so it doesn't leave key material behind */ 3744 static void 3745 dpaa2_sec_sym_session_clear(struct rte_cryptodev *dev, 3746 struct rte_cryptodev_sym_session *sess) 3747 { 3748 PMD_INIT_FUNC_TRACE(); 3749 uint8_t index = dev->driver_id; 3750 void *sess_priv = get_sym_session_private_data(sess, index); 3751 dpaa2_sec_session *s = (dpaa2_sec_session *)sess_priv; 3752 3753 if (sess_priv) { 3754 rte_free(s->ctxt); 3755 rte_free(s->cipher_key.data); 3756 rte_free(s->auth_key.data); 3757 memset(s, 0, sizeof(dpaa2_sec_session)); 3758 struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv); 3759 set_sym_session_private_data(sess, index, NULL); 3760 rte_mempool_put(sess_mp, sess_priv); 3761 } 3762 } 3763 3764 static int 3765 dpaa2_sec_dev_configure(struct rte_cryptodev *dev __rte_unused, 3766 struct rte_cryptodev_config *config __rte_unused) 3767 { 3768 PMD_INIT_FUNC_TRACE(); 3769 3770 return 0; 3771 } 3772 3773 static int 3774 dpaa2_sec_dev_start(struct rte_cryptodev *dev) 3775 { 3776 struct dpaa2_sec_dev_private *priv = dev->data->dev_private; 3777 struct fsl_mc_io *dpseci = (struct fsl_mc_io *)priv->hw; 3778 struct dpseci_attr attr; 3779 struct dpaa2_queue *dpaa2_q; 3780 struct dpaa2_sec_qp **qp = (struct dpaa2_sec_qp **) 3781 dev->data->queue_pairs; 3782 struct dpseci_rx_queue_attr rx_attr; 3783 struct dpseci_tx_queue_attr tx_attr; 3784 int ret, i; 3785 3786 PMD_INIT_FUNC_TRACE(); 3787 3788 /* Change the tx burst function if ordered queues are used */ 3789 if (priv->en_ordered) 3790 dev->enqueue_burst = dpaa2_sec_enqueue_burst_ordered; 3791 3792 memset(&attr, 0, sizeof(struct dpseci_attr)); 3793 3794 ret = dpseci_enable(dpseci, CMD_PRI_LOW, priv->token); 3795 if (ret) { 3796 DPAA2_SEC_ERR("DPSECI with HW_ID = %d ENABLE FAILED", 3797 priv->hw_id); 3798 goto get_attr_failure; 3799 } 3800 ret = dpseci_get_attributes(dpseci, CMD_PRI_LOW, priv->token, &attr); 3801 if (ret) { 3802 DPAA2_SEC_ERR("DPSEC ATTRIBUTE READ FAILED, disabling DPSEC"); 3803 goto get_attr_failure; 3804 } 3805 for (i = 0; i < attr.num_rx_queues && qp[i]; i++) { 3806 dpaa2_q = &qp[i]->rx_vq; 3807 dpseci_get_rx_queue(dpseci, CMD_PRI_LOW, priv->token, i, 3808 &rx_attr); 3809 dpaa2_q->fqid = rx_attr.fqid; 3810 DPAA2_SEC_DEBUG("rx_fqid: %d", dpaa2_q->fqid); 3811 } 3812 for (i = 0; i < attr.num_tx_queues && qp[i]; i++) { 3813 dpaa2_q = &qp[i]->tx_vq; 3814 dpseci_get_tx_queue(dpseci, CMD_PRI_LOW, priv->token, i, 3815 &tx_attr); 3816 dpaa2_q->fqid = tx_attr.fqid; 3817 DPAA2_SEC_DEBUG("tx_fqid: %d", dpaa2_q->fqid); 3818 } 3819 3820 return 0; 3821 get_attr_failure: 3822 dpseci_disable(dpseci, CMD_PRI_LOW, priv->token); 3823 return -1; 3824 } 3825 3826 static void 3827 dpaa2_sec_dev_stop(struct rte_cryptodev *dev) 3828 { 3829 struct dpaa2_sec_dev_private *priv = dev->data->dev_private; 3830 struct fsl_mc_io *dpseci = (struct fsl_mc_io *)priv->hw; 3831 int ret; 3832 3833 PMD_INIT_FUNC_TRACE(); 3834 3835 ret = dpseci_disable(dpseci, CMD_PRI_LOW, priv->token); 3836 if (ret) { 3837 DPAA2_SEC_ERR("Failure in disabling dpseci %d device", 3838 priv->hw_id); 3839 return; 3840 } 3841 3842 ret = dpseci_reset(dpseci, CMD_PRI_LOW, priv->token); 3843 if (ret < 0) { 3844 DPAA2_SEC_ERR("SEC Device cannot be reset:Error = %0x", ret); 3845 return; 3846 } 3847 } 3848 3849 static int 3850 dpaa2_sec_dev_close(struct rte_cryptodev *dev __rte_unused) 3851 { 3852 PMD_INIT_FUNC_TRACE(); 3853 3854 return 0; 3855 } 3856 3857 static void 3858 dpaa2_sec_dev_infos_get(struct rte_cryptodev *dev, 3859 struct rte_cryptodev_info *info) 3860 { 3861 struct dpaa2_sec_dev_private *internals = dev->data->dev_private; 3862 3863 PMD_INIT_FUNC_TRACE(); 3864 if (info != NULL) { 3865 info->max_nb_queue_pairs = internals->max_nb_queue_pairs; 3866 info->feature_flags = dev->feature_flags; 3867 info->capabilities = dpaa2_sec_capabilities; 3868 /* No limit of number of sessions */ 3869 info->sym.max_nb_sessions = 0; 3870 info->driver_id = cryptodev_driver_id; 3871 } 3872 } 3873 3874 static 3875 void dpaa2_sec_stats_get(struct rte_cryptodev *dev, 3876 struct rte_cryptodev_stats *stats) 3877 { 3878 struct dpaa2_sec_dev_private *priv = dev->data->dev_private; 3879 struct fsl_mc_io dpseci; 3880 struct dpseci_sec_counters counters = {0}; 3881 struct dpaa2_sec_qp **qp = (struct dpaa2_sec_qp **) 3882 dev->data->queue_pairs; 3883 int ret, i; 3884 3885 PMD_INIT_FUNC_TRACE(); 3886 if (stats == NULL) { 3887 DPAA2_SEC_ERR("Invalid stats ptr NULL"); 3888 return; 3889 } 3890 for (i = 0; i < dev->data->nb_queue_pairs; i++) { 3891 if (qp == NULL || qp[i] == NULL) { 3892 DPAA2_SEC_DEBUG("Uninitialised queue pair"); 3893 continue; 3894 } 3895 3896 stats->enqueued_count += qp[i]->tx_vq.tx_pkts; 3897 stats->dequeued_count += qp[i]->rx_vq.rx_pkts; 3898 stats->enqueue_err_count += qp[i]->tx_vq.err_pkts; 3899 stats->dequeue_err_count += qp[i]->rx_vq.err_pkts; 3900 } 3901 3902 /* In case as secondary process access stats, MCP portal in priv-hw 3903 * may have primary process address. Need the secondary process 3904 * based MCP portal address for this object. 3905 */ 3906 dpseci.regs = dpaa2_get_mcp_ptr(MC_PORTAL_INDEX); 3907 ret = dpseci_get_sec_counters(&dpseci, CMD_PRI_LOW, priv->token, 3908 &counters); 3909 if (ret) { 3910 DPAA2_SEC_ERR("SEC counters failed"); 3911 } else { 3912 DPAA2_SEC_INFO("dpseci hardware stats:" 3913 "\n\tNum of Requests Dequeued = %" PRIu64 3914 "\n\tNum of Outbound Encrypt Requests = %" PRIu64 3915 "\n\tNum of Inbound Decrypt Requests = %" PRIu64 3916 "\n\tNum of Outbound Bytes Encrypted = %" PRIu64 3917 "\n\tNum of Outbound Bytes Protected = %" PRIu64 3918 "\n\tNum of Inbound Bytes Decrypted = %" PRIu64 3919 "\n\tNum of Inbound Bytes Validated = %" PRIu64, 3920 counters.dequeued_requests, 3921 counters.ob_enc_requests, 3922 counters.ib_dec_requests, 3923 counters.ob_enc_bytes, 3924 counters.ob_prot_bytes, 3925 counters.ib_dec_bytes, 3926 counters.ib_valid_bytes); 3927 } 3928 } 3929 3930 static 3931 void dpaa2_sec_stats_reset(struct rte_cryptodev *dev) 3932 { 3933 int i; 3934 struct dpaa2_sec_qp **qp = (struct dpaa2_sec_qp **) 3935 (dev->data->queue_pairs); 3936 3937 PMD_INIT_FUNC_TRACE(); 3938 3939 for (i = 0; i < dev->data->nb_queue_pairs; i++) { 3940 if (qp[i] == NULL) { 3941 DPAA2_SEC_DEBUG("Uninitialised queue pair"); 3942 continue; 3943 } 3944 qp[i]->tx_vq.rx_pkts = 0; 3945 qp[i]->tx_vq.tx_pkts = 0; 3946 qp[i]->tx_vq.err_pkts = 0; 3947 qp[i]->rx_vq.rx_pkts = 0; 3948 qp[i]->rx_vq.tx_pkts = 0; 3949 qp[i]->rx_vq.err_pkts = 0; 3950 } 3951 } 3952 3953 static void __rte_hot 3954 dpaa2_sec_process_parallel_event(struct qbman_swp *swp, 3955 const struct qbman_fd *fd, 3956 const struct qbman_result *dq, 3957 struct dpaa2_queue *rxq, 3958 struct rte_event *ev) 3959 { 3960 /* Prefetching mbuf */ 3961 rte_prefetch0((void *)(size_t)(DPAA2_GET_FD_ADDR(fd)- 3962 rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size)); 3963 3964 /* Prefetching ipsec crypto_op stored in priv data of mbuf */ 3965 rte_prefetch0((void *)(size_t)(DPAA2_GET_FD_ADDR(fd)-64)); 3966 3967 ev->flow_id = rxq->ev.flow_id; 3968 ev->sub_event_type = rxq->ev.sub_event_type; 3969 ev->event_type = RTE_EVENT_TYPE_CRYPTODEV; 3970 ev->op = RTE_EVENT_OP_NEW; 3971 ev->sched_type = rxq->ev.sched_type; 3972 ev->queue_id = rxq->ev.queue_id; 3973 ev->priority = rxq->ev.priority; 3974 ev->event_ptr = sec_fd_to_mbuf(fd); 3975 3976 qbman_swp_dqrr_consume(swp, dq); 3977 } 3978 static void 3979 dpaa2_sec_process_atomic_event(struct qbman_swp *swp __rte_unused, 3980 const struct qbman_fd *fd, 3981 const struct qbman_result *dq, 3982 struct dpaa2_queue *rxq, 3983 struct rte_event *ev) 3984 { 3985 uint8_t dqrr_index; 3986 struct rte_crypto_op *crypto_op = (struct rte_crypto_op *)ev->event_ptr; 3987 /* Prefetching mbuf */ 3988 rte_prefetch0((void *)(size_t)(DPAA2_GET_FD_ADDR(fd)- 3989 rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size)); 3990 3991 /* Prefetching ipsec crypto_op stored in priv data of mbuf */ 3992 rte_prefetch0((void *)(size_t)(DPAA2_GET_FD_ADDR(fd)-64)); 3993 3994 ev->flow_id = rxq->ev.flow_id; 3995 ev->sub_event_type = rxq->ev.sub_event_type; 3996 ev->event_type = RTE_EVENT_TYPE_CRYPTODEV; 3997 ev->op = RTE_EVENT_OP_NEW; 3998 ev->sched_type = rxq->ev.sched_type; 3999 ev->queue_id = rxq->ev.queue_id; 4000 ev->priority = rxq->ev.priority; 4001 4002 ev->event_ptr = sec_fd_to_mbuf(fd); 4003 dqrr_index = qbman_get_dqrr_idx(dq); 4004 *dpaa2_seqn(crypto_op->sym->m_src) = QBMAN_ENQUEUE_FLAG_DCA | dqrr_index; 4005 DPAA2_PER_LCORE_DQRR_SIZE++; 4006 DPAA2_PER_LCORE_DQRR_HELD |= 1 << dqrr_index; 4007 DPAA2_PER_LCORE_DQRR_MBUF(dqrr_index) = crypto_op->sym->m_src; 4008 } 4009 4010 static void __rte_hot 4011 dpaa2_sec_process_ordered_event(struct qbman_swp *swp, 4012 const struct qbman_fd *fd, 4013 const struct qbman_result *dq, 4014 struct dpaa2_queue *rxq, 4015 struct rte_event *ev) 4016 { 4017 struct rte_crypto_op *crypto_op = (struct rte_crypto_op *)ev->event_ptr; 4018 4019 /* Prefetching mbuf */ 4020 rte_prefetch0((void *)(size_t)(DPAA2_GET_FD_ADDR(fd)- 4021 rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size)); 4022 4023 /* Prefetching ipsec crypto_op stored in priv data of mbuf */ 4024 rte_prefetch0((void *)(size_t)(DPAA2_GET_FD_ADDR(fd)-64)); 4025 4026 ev->flow_id = rxq->ev.flow_id; 4027 ev->sub_event_type = rxq->ev.sub_event_type; 4028 ev->event_type = RTE_EVENT_TYPE_CRYPTODEV; 4029 ev->op = RTE_EVENT_OP_NEW; 4030 ev->sched_type = rxq->ev.sched_type; 4031 ev->queue_id = rxq->ev.queue_id; 4032 ev->priority = rxq->ev.priority; 4033 ev->event_ptr = sec_fd_to_mbuf(fd); 4034 4035 *dpaa2_seqn(crypto_op->sym->m_src) = DPAA2_ENQUEUE_FLAG_ORP; 4036 *dpaa2_seqn(crypto_op->sym->m_src) |= qbman_result_DQ_odpid(dq) << 4037 DPAA2_EQCR_OPRID_SHIFT; 4038 *dpaa2_seqn(crypto_op->sym->m_src) |= qbman_result_DQ_seqnum(dq) << 4039 DPAA2_EQCR_SEQNUM_SHIFT; 4040 4041 qbman_swp_dqrr_consume(swp, dq); 4042 } 4043 4044 int 4045 dpaa2_sec_eventq_attach(const struct rte_cryptodev *dev, 4046 int qp_id, 4047 struct dpaa2_dpcon_dev *dpcon, 4048 const struct rte_event *event) 4049 { 4050 struct dpaa2_sec_dev_private *priv = dev->data->dev_private; 4051 struct fsl_mc_io *dpseci = (struct fsl_mc_io *)priv->hw; 4052 struct dpaa2_sec_qp *qp = dev->data->queue_pairs[qp_id]; 4053 struct dpseci_rx_queue_cfg cfg; 4054 uint8_t priority; 4055 int ret; 4056 4057 if (event->sched_type == RTE_SCHED_TYPE_PARALLEL) 4058 qp->rx_vq.cb = dpaa2_sec_process_parallel_event; 4059 else if (event->sched_type == RTE_SCHED_TYPE_ATOMIC) 4060 qp->rx_vq.cb = dpaa2_sec_process_atomic_event; 4061 else if (event->sched_type == RTE_SCHED_TYPE_ORDERED) 4062 qp->rx_vq.cb = dpaa2_sec_process_ordered_event; 4063 else 4064 return -EINVAL; 4065 4066 priority = (RTE_EVENT_DEV_PRIORITY_LOWEST / event->priority) * 4067 (dpcon->num_priorities - 1); 4068 4069 memset(&cfg, 0, sizeof(struct dpseci_rx_queue_cfg)); 4070 cfg.options = DPSECI_QUEUE_OPT_DEST; 4071 cfg.dest_cfg.dest_type = DPSECI_DEST_DPCON; 4072 cfg.dest_cfg.dest_id = dpcon->dpcon_id; 4073 cfg.dest_cfg.priority = priority; 4074 4075 cfg.options |= DPSECI_QUEUE_OPT_USER_CTX; 4076 cfg.user_ctx = (size_t)(qp); 4077 if (event->sched_type == RTE_SCHED_TYPE_ATOMIC) { 4078 cfg.options |= DPSECI_QUEUE_OPT_ORDER_PRESERVATION; 4079 cfg.order_preservation_en = 1; 4080 } 4081 4082 if (event->sched_type == RTE_SCHED_TYPE_ORDERED) { 4083 struct opr_cfg ocfg; 4084 4085 /* Restoration window size = 256 frames */ 4086 ocfg.oprrws = 3; 4087 /* Restoration window size = 512 frames for LX2 */ 4088 if (dpaa2_svr_family == SVR_LX2160A) 4089 ocfg.oprrws = 4; 4090 /* Auto advance NESN window enabled */ 4091 ocfg.oa = 1; 4092 /* Late arrival window size disabled */ 4093 ocfg.olws = 0; 4094 /* ORL resource exhaustaion advance NESN disabled */ 4095 ocfg.oeane = 0; 4096 4097 if (priv->en_loose_ordered) 4098 ocfg.oloe = 1; 4099 else 4100 ocfg.oloe = 0; 4101 4102 ret = dpseci_set_opr(dpseci, CMD_PRI_LOW, priv->token, 4103 qp_id, OPR_OPT_CREATE, &ocfg); 4104 if (ret) { 4105 RTE_LOG(ERR, PMD, "Error setting opr: ret: %d\n", ret); 4106 return ret; 4107 } 4108 qp->tx_vq.cb_eqresp_free = dpaa2_sec_free_eqresp_buf; 4109 priv->en_ordered = 1; 4110 } 4111 4112 ret = dpseci_set_rx_queue(dpseci, CMD_PRI_LOW, priv->token, 4113 qp_id, &cfg); 4114 if (ret) { 4115 RTE_LOG(ERR, PMD, "Error in dpseci_set_queue: ret: %d\n", ret); 4116 return ret; 4117 } 4118 4119 memcpy(&qp->rx_vq.ev, event, sizeof(struct rte_event)); 4120 4121 return 0; 4122 } 4123 4124 int 4125 dpaa2_sec_eventq_detach(const struct rte_cryptodev *dev, 4126 int qp_id) 4127 { 4128 struct dpaa2_sec_dev_private *priv = dev->data->dev_private; 4129 struct fsl_mc_io *dpseci = (struct fsl_mc_io *)priv->hw; 4130 struct dpseci_rx_queue_cfg cfg; 4131 int ret; 4132 4133 memset(&cfg, 0, sizeof(struct dpseci_rx_queue_cfg)); 4134 cfg.options = DPSECI_QUEUE_OPT_DEST; 4135 cfg.dest_cfg.dest_type = DPSECI_DEST_NONE; 4136 4137 ret = dpseci_set_rx_queue(dpseci, CMD_PRI_LOW, priv->token, 4138 qp_id, &cfg); 4139 if (ret) 4140 RTE_LOG(ERR, PMD, "Error in dpseci_set_queue: ret: %d\n", ret); 4141 4142 return ret; 4143 } 4144 4145 static struct rte_cryptodev_ops crypto_ops = { 4146 .dev_configure = dpaa2_sec_dev_configure, 4147 .dev_start = dpaa2_sec_dev_start, 4148 .dev_stop = dpaa2_sec_dev_stop, 4149 .dev_close = dpaa2_sec_dev_close, 4150 .dev_infos_get = dpaa2_sec_dev_infos_get, 4151 .stats_get = dpaa2_sec_stats_get, 4152 .stats_reset = dpaa2_sec_stats_reset, 4153 .queue_pair_setup = dpaa2_sec_queue_pair_setup, 4154 .queue_pair_release = dpaa2_sec_queue_pair_release, 4155 .sym_session_get_size = dpaa2_sec_sym_session_get_size, 4156 .sym_session_configure = dpaa2_sec_sym_session_configure, 4157 .sym_session_clear = dpaa2_sec_sym_session_clear, 4158 /* Raw data-path API related operations */ 4159 .sym_get_raw_dp_ctx_size = dpaa2_sec_get_dp_ctx_size, 4160 .sym_configure_raw_dp_ctx = dpaa2_sec_configure_raw_dp_ctx, 4161 }; 4162 4163 #ifdef RTE_LIB_SECURITY 4164 static const struct rte_security_capability * 4165 dpaa2_sec_capabilities_get(void *device __rte_unused) 4166 { 4167 return dpaa2_sec_security_cap; 4168 } 4169 4170 static const struct rte_security_ops dpaa2_sec_security_ops = { 4171 .session_create = dpaa2_sec_security_session_create, 4172 .session_update = NULL, 4173 .session_stats_get = NULL, 4174 .session_destroy = dpaa2_sec_security_session_destroy, 4175 .set_pkt_metadata = NULL, 4176 .capabilities_get = dpaa2_sec_capabilities_get 4177 }; 4178 #endif 4179 4180 static int 4181 dpaa2_sec_uninit(const struct rte_cryptodev *dev) 4182 { 4183 struct dpaa2_sec_dev_private *priv = dev->data->dev_private; 4184 struct fsl_mc_io *dpseci = (struct fsl_mc_io *)priv->hw; 4185 int ret; 4186 4187 PMD_INIT_FUNC_TRACE(); 4188 4189 /* Function is reverse of dpaa2_sec_dev_init. 4190 * It does the following: 4191 * 1. Detach a DPSECI from attached resources i.e. buffer pools, dpbp_id 4192 * 2. Close the DPSECI device 4193 * 3. Free the allocated resources. 4194 */ 4195 4196 /*Close the device at underlying layer*/ 4197 ret = dpseci_close(dpseci, CMD_PRI_LOW, priv->token); 4198 if (ret) { 4199 DPAA2_SEC_ERR("Failure closing dpseci device: err(%d)", ret); 4200 return -1; 4201 } 4202 4203 /*Free the allocated memory for ethernet private data and dpseci*/ 4204 priv->hw = NULL; 4205 rte_free(dpseci); 4206 rte_free(dev->security_ctx); 4207 rte_mempool_free(priv->fle_pool); 4208 4209 DPAA2_SEC_INFO("Closing DPAA2_SEC device %s on numa socket %u", 4210 dev->data->name, rte_socket_id()); 4211 4212 return 0; 4213 } 4214 4215 static int 4216 check_devargs_handler(const char *key, const char *value, 4217 void *opaque) 4218 { 4219 struct rte_cryptodev *dev = (struct rte_cryptodev *)opaque; 4220 struct dpaa2_sec_dev_private *priv = dev->data->dev_private; 4221 4222 if (!strcmp(key, "drv_strict_order")) { 4223 priv->en_loose_ordered = false; 4224 } else if (!strcmp(key, "drv_dump_mode")) { 4225 dpaa2_sec_dp_dump = atoi(value); 4226 if (dpaa2_sec_dp_dump > DPAA2_SEC_DP_FULL_DUMP) { 4227 DPAA2_SEC_WARN("WARN: DPAA2_SEC_DP_DUMP_LEVEL is not " 4228 "supported, changing to FULL error" 4229 " prints\n"); 4230 dpaa2_sec_dp_dump = DPAA2_SEC_DP_FULL_DUMP; 4231 } 4232 } else 4233 return -1; 4234 4235 return 0; 4236 } 4237 4238 static void 4239 dpaa2_sec_get_devargs(struct rte_cryptodev *cryptodev, const char *key) 4240 { 4241 struct rte_kvargs *kvlist; 4242 struct rte_devargs *devargs; 4243 4244 devargs = cryptodev->device->devargs; 4245 if (!devargs) 4246 return; 4247 4248 kvlist = rte_kvargs_parse(devargs->args, NULL); 4249 if (!kvlist) 4250 return; 4251 4252 if (!rte_kvargs_count(kvlist, key)) { 4253 rte_kvargs_free(kvlist); 4254 return; 4255 } 4256 4257 rte_kvargs_process(kvlist, key, 4258 check_devargs_handler, (void *)cryptodev); 4259 rte_kvargs_free(kvlist); 4260 } 4261 4262 static int 4263 dpaa2_sec_dev_init(struct rte_cryptodev *cryptodev) 4264 { 4265 struct dpaa2_sec_dev_private *internals; 4266 struct rte_device *dev = cryptodev->device; 4267 struct rte_dpaa2_device *dpaa2_dev; 4268 #ifdef RTE_LIB_SECURITY 4269 struct rte_security_ctx *security_instance; 4270 #endif 4271 struct fsl_mc_io *dpseci; 4272 uint16_t token; 4273 struct dpseci_attr attr; 4274 int retcode, hw_id; 4275 char str[30]; 4276 4277 PMD_INIT_FUNC_TRACE(); 4278 dpaa2_dev = container_of(dev, struct rte_dpaa2_device, device); 4279 hw_id = dpaa2_dev->object_id; 4280 4281 cryptodev->driver_id = cryptodev_driver_id; 4282 cryptodev->dev_ops = &crypto_ops; 4283 4284 cryptodev->enqueue_burst = dpaa2_sec_enqueue_burst; 4285 cryptodev->dequeue_burst = dpaa2_sec_dequeue_burst; 4286 cryptodev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | 4287 RTE_CRYPTODEV_FF_HW_ACCELERATED | 4288 RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING | 4289 RTE_CRYPTODEV_FF_SECURITY | 4290 RTE_CRYPTODEV_FF_SYM_RAW_DP | 4291 RTE_CRYPTODEV_FF_IN_PLACE_SGL | 4292 RTE_CRYPTODEV_FF_OOP_SGL_IN_SGL_OUT | 4293 RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT | 4294 RTE_CRYPTODEV_FF_OOP_LB_IN_SGL_OUT | 4295 RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT; 4296 4297 internals = cryptodev->data->dev_private; 4298 4299 /* 4300 * For secondary processes, we don't initialise any further as primary 4301 * has already done this work. Only check we don't need a different 4302 * RX function 4303 */ 4304 if (rte_eal_process_type() != RTE_PROC_PRIMARY) { 4305 DPAA2_SEC_DEBUG("Device already init by primary process"); 4306 return 0; 4307 } 4308 #ifdef RTE_LIB_SECURITY 4309 /* Initialize security_ctx only for primary process*/ 4310 security_instance = rte_malloc("rte_security_instances_ops", 4311 sizeof(struct rte_security_ctx), 0); 4312 if (security_instance == NULL) 4313 return -ENOMEM; 4314 security_instance->device = (void *)cryptodev; 4315 security_instance->ops = &dpaa2_sec_security_ops; 4316 security_instance->sess_cnt = 0; 4317 cryptodev->security_ctx = security_instance; 4318 #endif 4319 /*Open the rte device via MC and save the handle for further use*/ 4320 dpseci = (struct fsl_mc_io *)rte_calloc(NULL, 1, 4321 sizeof(struct fsl_mc_io), 0); 4322 if (!dpseci) { 4323 DPAA2_SEC_ERR( 4324 "Error in allocating the memory for dpsec object"); 4325 return -ENOMEM; 4326 } 4327 dpseci->regs = dpaa2_get_mcp_ptr(MC_PORTAL_INDEX); 4328 4329 retcode = dpseci_open(dpseci, CMD_PRI_LOW, hw_id, &token); 4330 if (retcode != 0) { 4331 DPAA2_SEC_ERR("Cannot open the dpsec device: Error = %x", 4332 retcode); 4333 goto init_error; 4334 } 4335 retcode = dpseci_get_attributes(dpseci, CMD_PRI_LOW, token, &attr); 4336 if (retcode != 0) { 4337 DPAA2_SEC_ERR( 4338 "Cannot get dpsec device attributed: Error = %x", 4339 retcode); 4340 goto init_error; 4341 } 4342 snprintf(cryptodev->data->name, sizeof(cryptodev->data->name), 4343 "dpsec-%u", hw_id); 4344 4345 internals->max_nb_queue_pairs = attr.num_tx_queues; 4346 cryptodev->data->nb_queue_pairs = internals->max_nb_queue_pairs; 4347 internals->hw = dpseci; 4348 internals->token = token; 4349 internals->en_loose_ordered = true; 4350 4351 snprintf(str, sizeof(str), "sec_fle_pool_p%d_%d", 4352 getpid(), cryptodev->data->dev_id); 4353 internals->fle_pool = rte_mempool_create((const char *)str, 4354 FLE_POOL_NUM_BUFS, 4355 FLE_POOL_BUF_SIZE, 4356 FLE_POOL_CACHE_SIZE, 0, 4357 NULL, NULL, NULL, NULL, 4358 SOCKET_ID_ANY, 0); 4359 if (!internals->fle_pool) { 4360 DPAA2_SEC_ERR("Mempool (%s) creation failed", str); 4361 goto init_error; 4362 } 4363 4364 dpaa2_sec_get_devargs(cryptodev, DRIVER_DUMP_MODE); 4365 dpaa2_sec_get_devargs(cryptodev, DRIVER_STRICT_ORDER); 4366 DPAA2_SEC_INFO("driver %s: created", cryptodev->data->name); 4367 return 0; 4368 4369 init_error: 4370 DPAA2_SEC_ERR("driver %s: create failed", cryptodev->data->name); 4371 4372 /* dpaa2_sec_uninit(crypto_dev_name); */ 4373 return -EFAULT; 4374 } 4375 4376 static int 4377 cryptodev_dpaa2_sec_probe(struct rte_dpaa2_driver *dpaa2_drv __rte_unused, 4378 struct rte_dpaa2_device *dpaa2_dev) 4379 { 4380 struct rte_cryptodev *cryptodev; 4381 char cryptodev_name[RTE_CRYPTODEV_NAME_MAX_LEN]; 4382 4383 int retval; 4384 4385 snprintf(cryptodev_name, sizeof(cryptodev_name), "dpsec-%d", 4386 dpaa2_dev->object_id); 4387 4388 cryptodev = rte_cryptodev_pmd_allocate(cryptodev_name, rte_socket_id()); 4389 if (cryptodev == NULL) 4390 return -ENOMEM; 4391 4392 if (rte_eal_process_type() == RTE_PROC_PRIMARY) { 4393 cryptodev->data->dev_private = rte_zmalloc_socket( 4394 "cryptodev private structure", 4395 sizeof(struct dpaa2_sec_dev_private), 4396 RTE_CACHE_LINE_SIZE, 4397 rte_socket_id()); 4398 4399 if (cryptodev->data->dev_private == NULL) 4400 rte_panic("Cannot allocate memzone for private " 4401 "device data"); 4402 } 4403 4404 dpaa2_dev->cryptodev = cryptodev; 4405 cryptodev->device = &dpaa2_dev->device; 4406 4407 /* init user callbacks */ 4408 TAILQ_INIT(&(cryptodev->link_intr_cbs)); 4409 4410 if (dpaa2_svr_family == SVR_LX2160A) 4411 rta_set_sec_era(RTA_SEC_ERA_10); 4412 else 4413 rta_set_sec_era(RTA_SEC_ERA_8); 4414 4415 DPAA2_SEC_INFO("2-SEC ERA is %d", rta_get_sec_era()); 4416 4417 /* Invoke PMD device initialization function */ 4418 retval = dpaa2_sec_dev_init(cryptodev); 4419 if (retval == 0) { 4420 rte_cryptodev_pmd_probing_finish(cryptodev); 4421 return 0; 4422 } 4423 4424 if (rte_eal_process_type() == RTE_PROC_PRIMARY) 4425 rte_free(cryptodev->data->dev_private); 4426 4427 cryptodev->attached = RTE_CRYPTODEV_DETACHED; 4428 4429 return -ENXIO; 4430 } 4431 4432 static int 4433 cryptodev_dpaa2_sec_remove(struct rte_dpaa2_device *dpaa2_dev) 4434 { 4435 struct rte_cryptodev *cryptodev; 4436 int ret; 4437 4438 cryptodev = dpaa2_dev->cryptodev; 4439 if (cryptodev == NULL) 4440 return -ENODEV; 4441 4442 ret = dpaa2_sec_uninit(cryptodev); 4443 if (ret) 4444 return ret; 4445 4446 return rte_cryptodev_pmd_destroy(cryptodev); 4447 } 4448 4449 static struct rte_dpaa2_driver rte_dpaa2_sec_driver = { 4450 .drv_flags = RTE_DPAA2_DRV_IOVA_AS_VA, 4451 .drv_type = DPAA2_CRYPTO, 4452 .driver = { 4453 .name = "DPAA2 SEC PMD" 4454 }, 4455 .probe = cryptodev_dpaa2_sec_probe, 4456 .remove = cryptodev_dpaa2_sec_remove, 4457 }; 4458 4459 static struct cryptodev_driver dpaa2_sec_crypto_drv; 4460 4461 RTE_PMD_REGISTER_DPAA2(CRYPTODEV_NAME_DPAA2_SEC_PMD, rte_dpaa2_sec_driver); 4462 RTE_PMD_REGISTER_CRYPTO_DRIVER(dpaa2_sec_crypto_drv, 4463 rte_dpaa2_sec_driver.driver, cryptodev_driver_id); 4464 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_DPAA2_SEC_PMD, 4465 DRIVER_STRICT_ORDER "=<int>" 4466 DRIVER_DUMP_MODE "=<int>"); 4467 RTE_LOG_REGISTER(dpaa2_logtype_sec, pmd.crypto.dpaa2, NOTICE); 4468