1 /* SPDX-License-Identifier: BSD-3-Clause 2 * 3 * Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved. 4 * Copyright 2017 NXP 5 * 6 */ 7 8 #include <fcntl.h> 9 #include <unistd.h> 10 #include <sched.h> 11 #include <net/if.h> 12 13 #include <rte_byteorder.h> 14 #include <rte_common.h> 15 #include <rte_cryptodev_pmd.h> 16 #include <rte_crypto.h> 17 #include <rte_cryptodev.h> 18 #include <rte_security_driver.h> 19 #include <rte_cycles.h> 20 #include <rte_dev.h> 21 #include <rte_kvargs.h> 22 #include <rte_malloc.h> 23 #include <rte_mbuf.h> 24 #include <rte_memcpy.h> 25 #include <rte_string_fns.h> 26 27 #include <fsl_usd.h> 28 #include <fsl_qman.h> 29 #include <of.h> 30 31 /* RTA header files */ 32 #include <hw/desc/common.h> 33 #include <hw/desc/algo.h> 34 #include <hw/desc/ipsec.h> 35 36 #include <rte_dpaa_bus.h> 37 #include <dpaa_sec.h> 38 #include <dpaa_sec_log.h> 39 40 enum rta_sec_era rta_sec_era; 41 42 static uint8_t cryptodev_driver_id; 43 44 static __thread struct rte_crypto_op **dpaa_sec_ops; 45 static __thread int dpaa_sec_op_nb; 46 47 static int 48 dpaa_sec_attach_sess_q(struct dpaa_sec_qp *qp, dpaa_sec_session *sess); 49 50 static inline void 51 dpaa_sec_op_ending(struct dpaa_sec_op_ctx *ctx) 52 { 53 if (!ctx->fd_status) { 54 ctx->op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 55 } else { 56 PMD_RX_LOG(ERR, "SEC return err: 0x%x", ctx->fd_status); 57 ctx->op->status = RTE_CRYPTO_OP_STATUS_ERROR; 58 } 59 60 /* report op status to sym->op and then free the ctx memeory */ 61 rte_mempool_put(ctx->ctx_pool, (void *)ctx); 62 } 63 64 static inline struct dpaa_sec_op_ctx * 65 dpaa_sec_alloc_ctx(dpaa_sec_session *ses) 66 { 67 struct dpaa_sec_op_ctx *ctx; 68 int retval; 69 70 retval = rte_mempool_get(ses->ctx_pool, (void **)(&ctx)); 71 if (!ctx || retval) { 72 PMD_TX_LOG(ERR, "Alloc sec descriptor failed!"); 73 return NULL; 74 } 75 /* 76 * Clear SG memory. There are 16 SG entries of 16 Bytes each. 77 * one call to dcbz_64() clear 64 bytes, hence calling it 4 times 78 * to clear all the SG entries. dpaa_sec_alloc_ctx() is called for 79 * each packet, memset is costlier than dcbz_64(). 80 */ 81 dcbz_64(&ctx->job.sg[SG_CACHELINE_0]); 82 dcbz_64(&ctx->job.sg[SG_CACHELINE_1]); 83 dcbz_64(&ctx->job.sg[SG_CACHELINE_2]); 84 dcbz_64(&ctx->job.sg[SG_CACHELINE_3]); 85 86 ctx->ctx_pool = ses->ctx_pool; 87 ctx->vtop_offset = (size_t) ctx 88 - rte_mempool_virt2iova(ctx); 89 90 return ctx; 91 } 92 93 static inline rte_iova_t 94 dpaa_mem_vtop(void *vaddr) 95 { 96 const struct rte_memseg *ms; 97 98 ms = rte_mem_virt2memseg(vaddr, NULL); 99 if (ms) 100 return ms->iova + RTE_PTR_DIFF(vaddr, ms->addr); 101 return (size_t)NULL; 102 } 103 104 /* virtual address conversin when mempool support is available for ctx */ 105 static inline phys_addr_t 106 dpaa_mem_vtop_ctx(struct dpaa_sec_op_ctx *ctx, void *vaddr) 107 { 108 return (size_t)vaddr - ctx->vtop_offset; 109 } 110 111 static inline void * 112 dpaa_mem_ptov(rte_iova_t paddr) 113 { 114 return rte_mem_iova2virt(paddr); 115 } 116 117 static void 118 ern_sec_fq_handler(struct qman_portal *qm __rte_unused, 119 struct qman_fq *fq, 120 const struct qm_mr_entry *msg) 121 { 122 RTE_LOG_DP(ERR, PMD, "sec fq %d error, RC = %x, seqnum = %x\n", 123 fq->fqid, msg->ern.rc, msg->ern.seqnum); 124 } 125 126 /* initialize the queue with dest chan as caam chan so that 127 * all the packets in this queue could be dispatched into caam 128 */ 129 static int 130 dpaa_sec_init_rx(struct qman_fq *fq_in, rte_iova_t hwdesc, 131 uint32_t fqid_out) 132 { 133 struct qm_mcc_initfq fq_opts; 134 uint32_t flags; 135 int ret = -1; 136 137 /* Clear FQ options */ 138 memset(&fq_opts, 0x00, sizeof(struct qm_mcc_initfq)); 139 140 flags = QMAN_INITFQ_FLAG_SCHED; 141 fq_opts.we_mask = QM_INITFQ_WE_DESTWQ | QM_INITFQ_WE_CONTEXTA | 142 QM_INITFQ_WE_CONTEXTB; 143 144 qm_fqd_context_a_set64(&fq_opts.fqd, hwdesc); 145 fq_opts.fqd.context_b = fqid_out; 146 fq_opts.fqd.dest.channel = qm_channel_caam; 147 fq_opts.fqd.dest.wq = 0; 148 149 fq_in->cb.ern = ern_sec_fq_handler; 150 151 PMD_INIT_LOG(DEBUG, "in-%x out-%x", fq_in->fqid, fqid_out); 152 153 ret = qman_init_fq(fq_in, flags, &fq_opts); 154 if (unlikely(ret != 0)) 155 PMD_INIT_LOG(ERR, "qman_init_fq failed %d", ret); 156 157 return ret; 158 } 159 160 /* something is put into in_fq and caam put the crypto result into out_fq */ 161 static enum qman_cb_dqrr_result 162 dqrr_out_fq_cb_rx(struct qman_portal *qm __always_unused, 163 struct qman_fq *fq __always_unused, 164 const struct qm_dqrr_entry *dqrr) 165 { 166 const struct qm_fd *fd; 167 struct dpaa_sec_job *job; 168 struct dpaa_sec_op_ctx *ctx; 169 170 if (dpaa_sec_op_nb >= DPAA_SEC_BURST) 171 return qman_cb_dqrr_defer; 172 173 if (!(dqrr->stat & QM_DQRR_STAT_FD_VALID)) 174 return qman_cb_dqrr_consume; 175 176 fd = &dqrr->fd; 177 /* sg is embedded in an op ctx, 178 * sg[0] is for output 179 * sg[1] for input 180 */ 181 job = dpaa_mem_ptov(qm_fd_addr_get64(fd)); 182 183 ctx = container_of(job, struct dpaa_sec_op_ctx, job); 184 ctx->fd_status = fd->status; 185 if (ctx->op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) { 186 struct qm_sg_entry *sg_out; 187 uint32_t len; 188 189 sg_out = &job->sg[0]; 190 hw_sg_to_cpu(sg_out); 191 len = sg_out->length; 192 ctx->op->sym->m_src->pkt_len = len; 193 ctx->op->sym->m_src->data_len = len; 194 } 195 dpaa_sec_ops[dpaa_sec_op_nb++] = ctx->op; 196 dpaa_sec_op_ending(ctx); 197 198 return qman_cb_dqrr_consume; 199 } 200 201 /* caam result is put into this queue */ 202 static int 203 dpaa_sec_init_tx(struct qman_fq *fq) 204 { 205 int ret; 206 struct qm_mcc_initfq opts; 207 uint32_t flags; 208 209 flags = QMAN_FQ_FLAG_NO_ENQUEUE | QMAN_FQ_FLAG_LOCKED | 210 QMAN_FQ_FLAG_DYNAMIC_FQID; 211 212 ret = qman_create_fq(0, flags, fq); 213 if (unlikely(ret)) { 214 PMD_INIT_LOG(ERR, "qman_create_fq failed"); 215 return ret; 216 } 217 218 memset(&opts, 0, sizeof(opts)); 219 opts.we_mask = QM_INITFQ_WE_DESTWQ | QM_INITFQ_WE_FQCTRL | 220 QM_INITFQ_WE_CONTEXTA | QM_INITFQ_WE_CONTEXTB; 221 222 /* opts.fqd.dest.channel = dpaa_sec_pool_chan; */ 223 224 fq->cb.dqrr = dqrr_out_fq_cb_rx; 225 fq->cb.ern = ern_sec_fq_handler; 226 227 ret = qman_init_fq(fq, 0, &opts); 228 if (unlikely(ret)) { 229 PMD_INIT_LOG(ERR, "unable to init caam source fq!"); 230 return ret; 231 } 232 233 return ret; 234 } 235 236 static inline int is_cipher_only(dpaa_sec_session *ses) 237 { 238 return ((ses->cipher_alg != RTE_CRYPTO_CIPHER_NULL) && 239 (ses->auth_alg == RTE_CRYPTO_AUTH_NULL)); 240 } 241 242 static inline int is_auth_only(dpaa_sec_session *ses) 243 { 244 return ((ses->cipher_alg == RTE_CRYPTO_CIPHER_NULL) && 245 (ses->auth_alg != RTE_CRYPTO_AUTH_NULL)); 246 } 247 248 static inline int is_aead(dpaa_sec_session *ses) 249 { 250 return ((ses->cipher_alg == 0) && 251 (ses->auth_alg == 0) && 252 (ses->aead_alg != 0)); 253 } 254 255 static inline int is_auth_cipher(dpaa_sec_session *ses) 256 { 257 return ((ses->cipher_alg != RTE_CRYPTO_CIPHER_NULL) && 258 (ses->auth_alg != RTE_CRYPTO_AUTH_NULL) && 259 (ses->proto_alg != RTE_SECURITY_PROTOCOL_IPSEC)); 260 } 261 262 static inline int is_proto_ipsec(dpaa_sec_session *ses) 263 { 264 return (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC); 265 } 266 267 static inline int is_encode(dpaa_sec_session *ses) 268 { 269 return ses->dir == DIR_ENC; 270 } 271 272 static inline int is_decode(dpaa_sec_session *ses) 273 { 274 return ses->dir == DIR_DEC; 275 } 276 277 static inline void 278 caam_auth_alg(dpaa_sec_session *ses, struct alginfo *alginfo_a) 279 { 280 switch (ses->auth_alg) { 281 case RTE_CRYPTO_AUTH_NULL: 282 ses->digest_length = 0; 283 break; 284 case RTE_CRYPTO_AUTH_MD5_HMAC: 285 alginfo_a->algtype = 286 (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ? 287 OP_PCL_IPSEC_HMAC_MD5_96 : OP_ALG_ALGSEL_MD5; 288 alginfo_a->algmode = OP_ALG_AAI_HMAC; 289 break; 290 case RTE_CRYPTO_AUTH_SHA1_HMAC: 291 alginfo_a->algtype = 292 (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ? 293 OP_PCL_IPSEC_HMAC_SHA1_96 : OP_ALG_ALGSEL_SHA1; 294 alginfo_a->algmode = OP_ALG_AAI_HMAC; 295 break; 296 case RTE_CRYPTO_AUTH_SHA224_HMAC: 297 alginfo_a->algtype = 298 (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ? 299 OP_PCL_IPSEC_HMAC_SHA1_160 : OP_ALG_ALGSEL_SHA224; 300 alginfo_a->algmode = OP_ALG_AAI_HMAC; 301 break; 302 case RTE_CRYPTO_AUTH_SHA256_HMAC: 303 alginfo_a->algtype = 304 (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ? 305 OP_PCL_IPSEC_HMAC_SHA2_256_128 : OP_ALG_ALGSEL_SHA256; 306 alginfo_a->algmode = OP_ALG_AAI_HMAC; 307 break; 308 case RTE_CRYPTO_AUTH_SHA384_HMAC: 309 alginfo_a->algtype = 310 (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ? 311 OP_PCL_IPSEC_HMAC_SHA2_384_192 : OP_ALG_ALGSEL_SHA384; 312 alginfo_a->algmode = OP_ALG_AAI_HMAC; 313 break; 314 case RTE_CRYPTO_AUTH_SHA512_HMAC: 315 alginfo_a->algtype = 316 (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ? 317 OP_PCL_IPSEC_HMAC_SHA2_512_256 : OP_ALG_ALGSEL_SHA512; 318 alginfo_a->algmode = OP_ALG_AAI_HMAC; 319 break; 320 default: 321 PMD_INIT_LOG(ERR, "unsupported auth alg %u", ses->auth_alg); 322 } 323 } 324 325 static inline void 326 caam_cipher_alg(dpaa_sec_session *ses, struct alginfo *alginfo_c) 327 { 328 switch (ses->cipher_alg) { 329 case RTE_CRYPTO_CIPHER_NULL: 330 break; 331 case RTE_CRYPTO_CIPHER_AES_CBC: 332 alginfo_c->algtype = 333 (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ? 334 OP_PCL_IPSEC_AES_CBC : OP_ALG_ALGSEL_AES; 335 alginfo_c->algmode = OP_ALG_AAI_CBC; 336 break; 337 case RTE_CRYPTO_CIPHER_3DES_CBC: 338 alginfo_c->algtype = 339 (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ? 340 OP_PCL_IPSEC_3DES : OP_ALG_ALGSEL_3DES; 341 alginfo_c->algmode = OP_ALG_AAI_CBC; 342 break; 343 case RTE_CRYPTO_CIPHER_AES_CTR: 344 alginfo_c->algtype = 345 (ses->proto_alg == RTE_SECURITY_PROTOCOL_IPSEC) ? 346 OP_PCL_IPSEC_AES_CTR : OP_ALG_ALGSEL_AES; 347 alginfo_c->algmode = OP_ALG_AAI_CTR; 348 break; 349 default: 350 PMD_INIT_LOG(ERR, "unsupported cipher alg %d", ses->cipher_alg); 351 } 352 } 353 354 static inline void 355 caam_aead_alg(dpaa_sec_session *ses, struct alginfo *alginfo) 356 { 357 switch (ses->aead_alg) { 358 case RTE_CRYPTO_AEAD_AES_GCM: 359 alginfo->algtype = OP_ALG_ALGSEL_AES; 360 alginfo->algmode = OP_ALG_AAI_GCM; 361 break; 362 default: 363 PMD_INIT_LOG(ERR, "unsupported AEAD alg %d", ses->aead_alg); 364 } 365 } 366 367 368 /* prepare command block of the session */ 369 static int 370 dpaa_sec_prep_cdb(dpaa_sec_session *ses) 371 { 372 struct alginfo alginfo_c = {0}, alginfo_a = {0}, alginfo = {0}; 373 uint32_t shared_desc_len = 0; 374 struct sec_cdb *cdb = &ses->cdb; 375 int err; 376 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN 377 int swap = false; 378 #else 379 int swap = true; 380 #endif 381 382 memset(cdb, 0, sizeof(struct sec_cdb)); 383 384 if (is_cipher_only(ses)) { 385 caam_cipher_alg(ses, &alginfo_c); 386 if (alginfo_c.algtype == (unsigned int)DPAA_SEC_ALG_UNSUPPORT) { 387 PMD_TX_LOG(ERR, "not supported cipher alg\n"); 388 return -ENOTSUP; 389 } 390 391 alginfo_c.key = (size_t)ses->cipher_key.data; 392 alginfo_c.keylen = ses->cipher_key.length; 393 alginfo_c.key_enc_flags = 0; 394 alginfo_c.key_type = RTA_DATA_IMM; 395 396 shared_desc_len = cnstr_shdsc_blkcipher( 397 cdb->sh_desc, true, 398 swap, &alginfo_c, 399 NULL, 400 ses->iv.length, 401 ses->dir); 402 } else if (is_auth_only(ses)) { 403 caam_auth_alg(ses, &alginfo_a); 404 if (alginfo_a.algtype == (unsigned int)DPAA_SEC_ALG_UNSUPPORT) { 405 PMD_TX_LOG(ERR, "not supported auth alg\n"); 406 return -ENOTSUP; 407 } 408 409 alginfo_a.key = (size_t)ses->auth_key.data; 410 alginfo_a.keylen = ses->auth_key.length; 411 alginfo_a.key_enc_flags = 0; 412 alginfo_a.key_type = RTA_DATA_IMM; 413 414 shared_desc_len = cnstr_shdsc_hmac(cdb->sh_desc, true, 415 swap, &alginfo_a, 416 !ses->dir, 417 ses->digest_length); 418 } else if (is_aead(ses)) { 419 caam_aead_alg(ses, &alginfo); 420 if (alginfo.algtype == (unsigned int)DPAA_SEC_ALG_UNSUPPORT) { 421 PMD_TX_LOG(ERR, "not supported aead alg\n"); 422 return -ENOTSUP; 423 } 424 alginfo.key = (size_t)ses->aead_key.data; 425 alginfo.keylen = ses->aead_key.length; 426 alginfo.key_enc_flags = 0; 427 alginfo.key_type = RTA_DATA_IMM; 428 429 if (ses->dir == DIR_ENC) 430 shared_desc_len = cnstr_shdsc_gcm_encap( 431 cdb->sh_desc, true, swap, 432 &alginfo, 433 ses->iv.length, 434 ses->digest_length); 435 else 436 shared_desc_len = cnstr_shdsc_gcm_decap( 437 cdb->sh_desc, true, swap, 438 &alginfo, 439 ses->iv.length, 440 ses->digest_length); 441 } else { 442 caam_cipher_alg(ses, &alginfo_c); 443 if (alginfo_c.algtype == (unsigned int)DPAA_SEC_ALG_UNSUPPORT) { 444 PMD_TX_LOG(ERR, "not supported cipher alg\n"); 445 return -ENOTSUP; 446 } 447 448 alginfo_c.key = (size_t)ses->cipher_key.data; 449 alginfo_c.keylen = ses->cipher_key.length; 450 alginfo_c.key_enc_flags = 0; 451 alginfo_c.key_type = RTA_DATA_IMM; 452 453 caam_auth_alg(ses, &alginfo_a); 454 if (alginfo_a.algtype == (unsigned int)DPAA_SEC_ALG_UNSUPPORT) { 455 PMD_TX_LOG(ERR, "not supported auth alg\n"); 456 return -ENOTSUP; 457 } 458 459 alginfo_a.key = (size_t)ses->auth_key.data; 460 alginfo_a.keylen = ses->auth_key.length; 461 alginfo_a.key_enc_flags = 0; 462 alginfo_a.key_type = RTA_DATA_IMM; 463 464 cdb->sh_desc[0] = alginfo_c.keylen; 465 cdb->sh_desc[1] = alginfo_a.keylen; 466 err = rta_inline_query(IPSEC_AUTH_VAR_AES_DEC_BASE_DESC_LEN, 467 MIN_JOB_DESC_SIZE, 468 (unsigned int *)cdb->sh_desc, 469 &cdb->sh_desc[2], 2); 470 471 if (err < 0) { 472 PMD_TX_LOG(ERR, "Crypto: Incorrect key lengths"); 473 return err; 474 } 475 if (cdb->sh_desc[2] & 1) 476 alginfo_c.key_type = RTA_DATA_IMM; 477 else { 478 alginfo_c.key = (size_t)dpaa_mem_vtop( 479 (void *)(size_t)alginfo_c.key); 480 alginfo_c.key_type = RTA_DATA_PTR; 481 } 482 if (cdb->sh_desc[2] & (1<<1)) 483 alginfo_a.key_type = RTA_DATA_IMM; 484 else { 485 alginfo_a.key = (size_t)dpaa_mem_vtop( 486 (void *)(size_t)alginfo_a.key); 487 alginfo_a.key_type = RTA_DATA_PTR; 488 } 489 cdb->sh_desc[0] = 0; 490 cdb->sh_desc[1] = 0; 491 cdb->sh_desc[2] = 0; 492 if (is_proto_ipsec(ses)) { 493 if (ses->dir == DIR_ENC) { 494 shared_desc_len = cnstr_shdsc_ipsec_new_encap( 495 cdb->sh_desc, 496 true, swap, &ses->encap_pdb, 497 (uint8_t *)&ses->ip4_hdr, 498 &alginfo_c, &alginfo_a); 499 } else if (ses->dir == DIR_DEC) { 500 shared_desc_len = cnstr_shdsc_ipsec_new_decap( 501 cdb->sh_desc, 502 true, swap, &ses->decap_pdb, 503 &alginfo_c, &alginfo_a); 504 } 505 } else { 506 /* Auth_only_len is set as 0 here and it will be 507 * overwritten in fd for each packet. 508 */ 509 shared_desc_len = cnstr_shdsc_authenc(cdb->sh_desc, 510 true, swap, &alginfo_c, &alginfo_a, 511 ses->iv.length, 0, 512 ses->digest_length, ses->dir); 513 } 514 } 515 cdb->sh_hdr.hi.field.idlen = shared_desc_len; 516 cdb->sh_hdr.hi.word = rte_cpu_to_be_32(cdb->sh_hdr.hi.word); 517 cdb->sh_hdr.lo.word = rte_cpu_to_be_32(cdb->sh_hdr.lo.word); 518 519 return 0; 520 } 521 522 /* qp is lockless, should be accessed by only one thread */ 523 static int 524 dpaa_sec_deq(struct dpaa_sec_qp *qp, struct rte_crypto_op **ops, int nb_ops) 525 { 526 struct qman_fq *fq; 527 unsigned int pkts = 0; 528 int ret; 529 struct qm_dqrr_entry *dq; 530 531 fq = &qp->outq; 532 ret = qman_set_vdq(fq, (nb_ops > DPAA_MAX_DEQUEUE_NUM_FRAMES) ? 533 DPAA_MAX_DEQUEUE_NUM_FRAMES : nb_ops); 534 if (ret) 535 return 0; 536 537 do { 538 const struct qm_fd *fd; 539 struct dpaa_sec_job *job; 540 struct dpaa_sec_op_ctx *ctx; 541 struct rte_crypto_op *op; 542 543 dq = qman_dequeue(fq); 544 if (!dq) 545 continue; 546 547 fd = &dq->fd; 548 /* sg is embedded in an op ctx, 549 * sg[0] is for output 550 * sg[1] for input 551 */ 552 job = dpaa_mem_ptov(qm_fd_addr_get64(fd)); 553 554 ctx = container_of(job, struct dpaa_sec_op_ctx, job); 555 ctx->fd_status = fd->status; 556 op = ctx->op; 557 if (op->sess_type == RTE_CRYPTO_OP_SECURITY_SESSION) { 558 struct qm_sg_entry *sg_out; 559 uint32_t len; 560 561 sg_out = &job->sg[0]; 562 hw_sg_to_cpu(sg_out); 563 len = sg_out->length; 564 op->sym->m_src->pkt_len = len; 565 op->sym->m_src->data_len = len; 566 } 567 if (!ctx->fd_status) { 568 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 569 } else { 570 printf("\nSEC return err: 0x%x", ctx->fd_status); 571 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 572 } 573 ops[pkts++] = op; 574 575 /* report op status to sym->op and then free the ctx memeory */ 576 rte_mempool_put(ctx->ctx_pool, (void *)ctx); 577 578 qman_dqrr_consume(fq, dq); 579 } while (fq->flags & QMAN_FQ_STATE_VDQCR); 580 581 return pkts; 582 } 583 584 static inline struct dpaa_sec_job * 585 build_auth_only_sg(struct rte_crypto_op *op, dpaa_sec_session *ses) 586 { 587 struct rte_crypto_sym_op *sym = op->sym; 588 struct rte_mbuf *mbuf = sym->m_src; 589 struct dpaa_sec_job *cf; 590 struct dpaa_sec_op_ctx *ctx; 591 struct qm_sg_entry *sg, *out_sg, *in_sg; 592 phys_addr_t start_addr; 593 uint8_t *old_digest, extra_segs; 594 595 if (is_decode(ses)) 596 extra_segs = 3; 597 else 598 extra_segs = 2; 599 600 if ((mbuf->nb_segs + extra_segs) > MAX_SG_ENTRIES) { 601 PMD_TX_LOG(ERR, "Auth: Max sec segs supported is %d\n", 602 MAX_SG_ENTRIES); 603 return NULL; 604 } 605 ctx = dpaa_sec_alloc_ctx(ses); 606 if (!ctx) 607 return NULL; 608 609 cf = &ctx->job; 610 ctx->op = op; 611 old_digest = ctx->digest; 612 613 /* output */ 614 out_sg = &cf->sg[0]; 615 qm_sg_entry_set64(out_sg, sym->auth.digest.phys_addr); 616 out_sg->length = ses->digest_length; 617 cpu_to_hw_sg(out_sg); 618 619 /* input */ 620 in_sg = &cf->sg[1]; 621 /* need to extend the input to a compound frame */ 622 in_sg->extension = 1; 623 in_sg->final = 1; 624 in_sg->length = sym->auth.data.length; 625 qm_sg_entry_set64(in_sg, dpaa_mem_vtop_ctx(ctx, &cf->sg[2])); 626 627 /* 1st seg */ 628 sg = in_sg + 1; 629 qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf)); 630 sg->length = mbuf->data_len - sym->auth.data.offset; 631 sg->offset = sym->auth.data.offset; 632 633 /* Successive segs */ 634 mbuf = mbuf->next; 635 while (mbuf) { 636 cpu_to_hw_sg(sg); 637 sg++; 638 qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf)); 639 sg->length = mbuf->data_len; 640 mbuf = mbuf->next; 641 } 642 643 if (is_decode(ses)) { 644 /* Digest verification case */ 645 cpu_to_hw_sg(sg); 646 sg++; 647 rte_memcpy(old_digest, sym->auth.digest.data, 648 ses->digest_length); 649 start_addr = dpaa_mem_vtop_ctx(ctx, old_digest); 650 qm_sg_entry_set64(sg, start_addr); 651 sg->length = ses->digest_length; 652 in_sg->length += ses->digest_length; 653 } else { 654 /* Digest calculation case */ 655 sg->length -= ses->digest_length; 656 } 657 sg->final = 1; 658 cpu_to_hw_sg(sg); 659 cpu_to_hw_sg(in_sg); 660 661 return cf; 662 } 663 664 /** 665 * packet looks like: 666 * |<----data_len------->| 667 * |ip_header|ah_header|icv|payload| 668 * ^ 669 * | 670 * mbuf->pkt.data 671 */ 672 static inline struct dpaa_sec_job * 673 build_auth_only(struct rte_crypto_op *op, dpaa_sec_session *ses) 674 { 675 struct rte_crypto_sym_op *sym = op->sym; 676 struct rte_mbuf *mbuf = sym->m_src; 677 struct dpaa_sec_job *cf; 678 struct dpaa_sec_op_ctx *ctx; 679 struct qm_sg_entry *sg; 680 rte_iova_t start_addr; 681 uint8_t *old_digest; 682 683 ctx = dpaa_sec_alloc_ctx(ses); 684 if (!ctx) 685 return NULL; 686 687 cf = &ctx->job; 688 ctx->op = op; 689 old_digest = ctx->digest; 690 691 start_addr = rte_pktmbuf_iova(mbuf); 692 /* output */ 693 sg = &cf->sg[0]; 694 qm_sg_entry_set64(sg, sym->auth.digest.phys_addr); 695 sg->length = ses->digest_length; 696 cpu_to_hw_sg(sg); 697 698 /* input */ 699 sg = &cf->sg[1]; 700 if (is_decode(ses)) { 701 /* need to extend the input to a compound frame */ 702 sg->extension = 1; 703 qm_sg_entry_set64(sg, dpaa_mem_vtop_ctx(ctx, &cf->sg[2])); 704 sg->length = sym->auth.data.length + ses->digest_length; 705 sg->final = 1; 706 cpu_to_hw_sg(sg); 707 708 sg = &cf->sg[2]; 709 /* hash result or digest, save digest first */ 710 rte_memcpy(old_digest, sym->auth.digest.data, 711 ses->digest_length); 712 qm_sg_entry_set64(sg, start_addr + sym->auth.data.offset); 713 sg->length = sym->auth.data.length; 714 cpu_to_hw_sg(sg); 715 716 /* let's check digest by hw */ 717 start_addr = dpaa_mem_vtop_ctx(ctx, old_digest); 718 sg++; 719 qm_sg_entry_set64(sg, start_addr); 720 sg->length = ses->digest_length; 721 sg->final = 1; 722 cpu_to_hw_sg(sg); 723 } else { 724 qm_sg_entry_set64(sg, start_addr + sym->auth.data.offset); 725 sg->length = sym->auth.data.length; 726 sg->final = 1; 727 cpu_to_hw_sg(sg); 728 } 729 730 return cf; 731 } 732 733 static inline struct dpaa_sec_job * 734 build_cipher_only_sg(struct rte_crypto_op *op, dpaa_sec_session *ses) 735 { 736 struct rte_crypto_sym_op *sym = op->sym; 737 struct dpaa_sec_job *cf; 738 struct dpaa_sec_op_ctx *ctx; 739 struct qm_sg_entry *sg, *out_sg, *in_sg; 740 struct rte_mbuf *mbuf; 741 uint8_t req_segs; 742 uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, 743 ses->iv.offset); 744 745 if (sym->m_dst) { 746 mbuf = sym->m_dst; 747 req_segs = mbuf->nb_segs + sym->m_src->nb_segs + 3; 748 } else { 749 mbuf = sym->m_src; 750 req_segs = mbuf->nb_segs * 2 + 3; 751 } 752 753 if (req_segs > MAX_SG_ENTRIES) { 754 PMD_TX_LOG(ERR, "Cipher: Max sec segs supported is %d\n", 755 MAX_SG_ENTRIES); 756 return NULL; 757 } 758 759 ctx = dpaa_sec_alloc_ctx(ses); 760 if (!ctx) 761 return NULL; 762 763 cf = &ctx->job; 764 ctx->op = op; 765 766 /* output */ 767 out_sg = &cf->sg[0]; 768 out_sg->extension = 1; 769 out_sg->length = sym->cipher.data.length; 770 qm_sg_entry_set64(out_sg, dpaa_mem_vtop_ctx(ctx, &cf->sg[2])); 771 cpu_to_hw_sg(out_sg); 772 773 /* 1st seg */ 774 sg = &cf->sg[2]; 775 qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf)); 776 sg->length = mbuf->data_len - sym->cipher.data.offset; 777 sg->offset = sym->cipher.data.offset; 778 779 /* Successive segs */ 780 mbuf = mbuf->next; 781 while (mbuf) { 782 cpu_to_hw_sg(sg); 783 sg++; 784 qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf)); 785 sg->length = mbuf->data_len; 786 mbuf = mbuf->next; 787 } 788 sg->final = 1; 789 cpu_to_hw_sg(sg); 790 791 /* input */ 792 mbuf = sym->m_src; 793 in_sg = &cf->sg[1]; 794 in_sg->extension = 1; 795 in_sg->final = 1; 796 in_sg->length = sym->cipher.data.length + ses->iv.length; 797 798 sg++; 799 qm_sg_entry_set64(in_sg, dpaa_mem_vtop_ctx(ctx, sg)); 800 cpu_to_hw_sg(in_sg); 801 802 /* IV */ 803 qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr)); 804 sg->length = ses->iv.length; 805 cpu_to_hw_sg(sg); 806 807 /* 1st seg */ 808 sg++; 809 qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf)); 810 sg->length = mbuf->data_len - sym->cipher.data.offset; 811 sg->offset = sym->cipher.data.offset; 812 813 /* Successive segs */ 814 mbuf = mbuf->next; 815 while (mbuf) { 816 cpu_to_hw_sg(sg); 817 sg++; 818 qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf)); 819 sg->length = mbuf->data_len; 820 mbuf = mbuf->next; 821 } 822 sg->final = 1; 823 cpu_to_hw_sg(sg); 824 825 return cf; 826 } 827 828 static inline struct dpaa_sec_job * 829 build_cipher_only(struct rte_crypto_op *op, dpaa_sec_session *ses) 830 { 831 struct rte_crypto_sym_op *sym = op->sym; 832 struct dpaa_sec_job *cf; 833 struct dpaa_sec_op_ctx *ctx; 834 struct qm_sg_entry *sg; 835 rte_iova_t src_start_addr, dst_start_addr; 836 uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, 837 ses->iv.offset); 838 839 ctx = dpaa_sec_alloc_ctx(ses); 840 if (!ctx) 841 return NULL; 842 843 cf = &ctx->job; 844 ctx->op = op; 845 846 src_start_addr = rte_pktmbuf_iova(sym->m_src); 847 848 if (sym->m_dst) 849 dst_start_addr = rte_pktmbuf_iova(sym->m_dst); 850 else 851 dst_start_addr = src_start_addr; 852 853 /* output */ 854 sg = &cf->sg[0]; 855 qm_sg_entry_set64(sg, dst_start_addr + sym->cipher.data.offset); 856 sg->length = sym->cipher.data.length + ses->iv.length; 857 cpu_to_hw_sg(sg); 858 859 /* input */ 860 sg = &cf->sg[1]; 861 862 /* need to extend the input to a compound frame */ 863 sg->extension = 1; 864 sg->final = 1; 865 sg->length = sym->cipher.data.length + ses->iv.length; 866 qm_sg_entry_set64(sg, dpaa_mem_vtop_ctx(ctx, &cf->sg[2])); 867 cpu_to_hw_sg(sg); 868 869 sg = &cf->sg[2]; 870 qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr)); 871 sg->length = ses->iv.length; 872 cpu_to_hw_sg(sg); 873 874 sg++; 875 qm_sg_entry_set64(sg, src_start_addr + sym->cipher.data.offset); 876 sg->length = sym->cipher.data.length; 877 sg->final = 1; 878 cpu_to_hw_sg(sg); 879 880 return cf; 881 } 882 883 static inline struct dpaa_sec_job * 884 build_cipher_auth_gcm_sg(struct rte_crypto_op *op, dpaa_sec_session *ses) 885 { 886 struct rte_crypto_sym_op *sym = op->sym; 887 struct dpaa_sec_job *cf; 888 struct dpaa_sec_op_ctx *ctx; 889 struct qm_sg_entry *sg, *out_sg, *in_sg; 890 struct rte_mbuf *mbuf; 891 uint8_t req_segs; 892 uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, 893 ses->iv.offset); 894 895 if (sym->m_dst) { 896 mbuf = sym->m_dst; 897 req_segs = mbuf->nb_segs + sym->m_src->nb_segs + 4; 898 } else { 899 mbuf = sym->m_src; 900 req_segs = mbuf->nb_segs * 2 + 4; 901 } 902 903 if (ses->auth_only_len) 904 req_segs++; 905 906 if (req_segs > MAX_SG_ENTRIES) { 907 PMD_TX_LOG(ERR, "AEAD: Max sec segs supported is %d\n", 908 MAX_SG_ENTRIES); 909 return NULL; 910 } 911 912 ctx = dpaa_sec_alloc_ctx(ses); 913 if (!ctx) 914 return NULL; 915 916 cf = &ctx->job; 917 ctx->op = op; 918 919 rte_prefetch0(cf->sg); 920 921 /* output */ 922 out_sg = &cf->sg[0]; 923 out_sg->extension = 1; 924 if (is_encode(ses)) 925 out_sg->length = sym->aead.data.length + ses->auth_only_len 926 + ses->digest_length; 927 else 928 out_sg->length = sym->aead.data.length + ses->auth_only_len; 929 930 /* output sg entries */ 931 sg = &cf->sg[2]; 932 qm_sg_entry_set64(out_sg, dpaa_mem_vtop_ctx(ctx, sg)); 933 cpu_to_hw_sg(out_sg); 934 935 /* 1st seg */ 936 qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf)); 937 sg->length = mbuf->data_len - sym->aead.data.offset + 938 ses->auth_only_len; 939 sg->offset = sym->aead.data.offset - ses->auth_only_len; 940 941 /* Successive segs */ 942 mbuf = mbuf->next; 943 while (mbuf) { 944 cpu_to_hw_sg(sg); 945 sg++; 946 qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf)); 947 sg->length = mbuf->data_len; 948 mbuf = mbuf->next; 949 } 950 sg->length -= ses->digest_length; 951 952 if (is_encode(ses)) { 953 cpu_to_hw_sg(sg); 954 /* set auth output */ 955 sg++; 956 qm_sg_entry_set64(sg, sym->aead.digest.phys_addr); 957 sg->length = ses->digest_length; 958 } 959 sg->final = 1; 960 cpu_to_hw_sg(sg); 961 962 /* input */ 963 mbuf = sym->m_src; 964 in_sg = &cf->sg[1]; 965 in_sg->extension = 1; 966 in_sg->final = 1; 967 if (is_encode(ses)) 968 in_sg->length = ses->iv.length + sym->aead.data.length 969 + ses->auth_only_len; 970 else 971 in_sg->length = ses->iv.length + sym->aead.data.length 972 + ses->auth_only_len + ses->digest_length; 973 974 /* input sg entries */ 975 sg++; 976 qm_sg_entry_set64(in_sg, dpaa_mem_vtop_ctx(ctx, sg)); 977 cpu_to_hw_sg(in_sg); 978 979 /* 1st seg IV */ 980 qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr)); 981 sg->length = ses->iv.length; 982 cpu_to_hw_sg(sg); 983 984 /* 2nd seg auth only */ 985 if (ses->auth_only_len) { 986 sg++; 987 qm_sg_entry_set64(sg, dpaa_mem_vtop(sym->aead.aad.data)); 988 sg->length = ses->auth_only_len; 989 cpu_to_hw_sg(sg); 990 } 991 992 /* 3rd seg */ 993 sg++; 994 qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf)); 995 sg->length = mbuf->data_len - sym->aead.data.offset; 996 sg->offset = sym->aead.data.offset; 997 998 /* Successive segs */ 999 mbuf = mbuf->next; 1000 while (mbuf) { 1001 cpu_to_hw_sg(sg); 1002 sg++; 1003 qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf)); 1004 sg->length = mbuf->data_len; 1005 mbuf = mbuf->next; 1006 } 1007 1008 if (is_decode(ses)) { 1009 cpu_to_hw_sg(sg); 1010 sg++; 1011 memcpy(ctx->digest, sym->aead.digest.data, 1012 ses->digest_length); 1013 qm_sg_entry_set64(sg, dpaa_mem_vtop_ctx(ctx, ctx->digest)); 1014 sg->length = ses->digest_length; 1015 } 1016 sg->final = 1; 1017 cpu_to_hw_sg(sg); 1018 1019 return cf; 1020 } 1021 1022 static inline struct dpaa_sec_job * 1023 build_cipher_auth_gcm(struct rte_crypto_op *op, dpaa_sec_session *ses) 1024 { 1025 struct rte_crypto_sym_op *sym = op->sym; 1026 struct dpaa_sec_job *cf; 1027 struct dpaa_sec_op_ctx *ctx; 1028 struct qm_sg_entry *sg; 1029 uint32_t length = 0; 1030 rte_iova_t src_start_addr, dst_start_addr; 1031 uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, 1032 ses->iv.offset); 1033 1034 src_start_addr = sym->m_src->buf_iova + sym->m_src->data_off; 1035 1036 if (sym->m_dst) 1037 dst_start_addr = sym->m_dst->buf_iova + sym->m_dst->data_off; 1038 else 1039 dst_start_addr = src_start_addr; 1040 1041 ctx = dpaa_sec_alloc_ctx(ses); 1042 if (!ctx) 1043 return NULL; 1044 1045 cf = &ctx->job; 1046 ctx->op = op; 1047 1048 /* input */ 1049 rte_prefetch0(cf->sg); 1050 sg = &cf->sg[2]; 1051 qm_sg_entry_set64(&cf->sg[1], dpaa_mem_vtop_ctx(ctx, sg)); 1052 if (is_encode(ses)) { 1053 qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr)); 1054 sg->length = ses->iv.length; 1055 length += sg->length; 1056 cpu_to_hw_sg(sg); 1057 1058 sg++; 1059 if (ses->auth_only_len) { 1060 qm_sg_entry_set64(sg, 1061 dpaa_mem_vtop(sym->aead.aad.data)); 1062 sg->length = ses->auth_only_len; 1063 length += sg->length; 1064 cpu_to_hw_sg(sg); 1065 sg++; 1066 } 1067 qm_sg_entry_set64(sg, src_start_addr + sym->aead.data.offset); 1068 sg->length = sym->aead.data.length; 1069 length += sg->length; 1070 sg->final = 1; 1071 cpu_to_hw_sg(sg); 1072 } else { 1073 qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr)); 1074 sg->length = ses->iv.length; 1075 length += sg->length; 1076 cpu_to_hw_sg(sg); 1077 1078 sg++; 1079 if (ses->auth_only_len) { 1080 qm_sg_entry_set64(sg, 1081 dpaa_mem_vtop(sym->aead.aad.data)); 1082 sg->length = ses->auth_only_len; 1083 length += sg->length; 1084 cpu_to_hw_sg(sg); 1085 sg++; 1086 } 1087 qm_sg_entry_set64(sg, src_start_addr + sym->aead.data.offset); 1088 sg->length = sym->aead.data.length; 1089 length += sg->length; 1090 cpu_to_hw_sg(sg); 1091 1092 memcpy(ctx->digest, sym->aead.digest.data, 1093 ses->digest_length); 1094 sg++; 1095 1096 qm_sg_entry_set64(sg, dpaa_mem_vtop_ctx(ctx, ctx->digest)); 1097 sg->length = ses->digest_length; 1098 length += sg->length; 1099 sg->final = 1; 1100 cpu_to_hw_sg(sg); 1101 } 1102 /* input compound frame */ 1103 cf->sg[1].length = length; 1104 cf->sg[1].extension = 1; 1105 cf->sg[1].final = 1; 1106 cpu_to_hw_sg(&cf->sg[1]); 1107 1108 /* output */ 1109 sg++; 1110 qm_sg_entry_set64(&cf->sg[0], dpaa_mem_vtop_ctx(ctx, sg)); 1111 qm_sg_entry_set64(sg, 1112 dst_start_addr + sym->aead.data.offset - ses->auth_only_len); 1113 sg->length = sym->aead.data.length + ses->auth_only_len; 1114 length = sg->length; 1115 if (is_encode(ses)) { 1116 cpu_to_hw_sg(sg); 1117 /* set auth output */ 1118 sg++; 1119 qm_sg_entry_set64(sg, sym->aead.digest.phys_addr); 1120 sg->length = ses->digest_length; 1121 length += sg->length; 1122 } 1123 sg->final = 1; 1124 cpu_to_hw_sg(sg); 1125 1126 /* output compound frame */ 1127 cf->sg[0].length = length; 1128 cf->sg[0].extension = 1; 1129 cpu_to_hw_sg(&cf->sg[0]); 1130 1131 return cf; 1132 } 1133 1134 static inline struct dpaa_sec_job * 1135 build_cipher_auth_sg(struct rte_crypto_op *op, dpaa_sec_session *ses) 1136 { 1137 struct rte_crypto_sym_op *sym = op->sym; 1138 struct dpaa_sec_job *cf; 1139 struct dpaa_sec_op_ctx *ctx; 1140 struct qm_sg_entry *sg, *out_sg, *in_sg; 1141 struct rte_mbuf *mbuf; 1142 uint8_t req_segs; 1143 uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, 1144 ses->iv.offset); 1145 1146 if (sym->m_dst) { 1147 mbuf = sym->m_dst; 1148 req_segs = mbuf->nb_segs + sym->m_src->nb_segs + 4; 1149 } else { 1150 mbuf = sym->m_src; 1151 req_segs = mbuf->nb_segs * 2 + 4; 1152 } 1153 1154 if (req_segs > MAX_SG_ENTRIES) { 1155 PMD_TX_LOG(ERR, "Cipher-Auth: Max sec segs supported is %d\n", 1156 MAX_SG_ENTRIES); 1157 return NULL; 1158 } 1159 1160 ctx = dpaa_sec_alloc_ctx(ses); 1161 if (!ctx) 1162 return NULL; 1163 1164 cf = &ctx->job; 1165 ctx->op = op; 1166 1167 rte_prefetch0(cf->sg); 1168 1169 /* output */ 1170 out_sg = &cf->sg[0]; 1171 out_sg->extension = 1; 1172 if (is_encode(ses)) 1173 out_sg->length = sym->auth.data.length + ses->digest_length; 1174 else 1175 out_sg->length = sym->auth.data.length; 1176 1177 /* output sg entries */ 1178 sg = &cf->sg[2]; 1179 qm_sg_entry_set64(out_sg, dpaa_mem_vtop_ctx(ctx, sg)); 1180 cpu_to_hw_sg(out_sg); 1181 1182 /* 1st seg */ 1183 qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf)); 1184 sg->length = mbuf->data_len - sym->auth.data.offset; 1185 sg->offset = sym->auth.data.offset; 1186 1187 /* Successive segs */ 1188 mbuf = mbuf->next; 1189 while (mbuf) { 1190 cpu_to_hw_sg(sg); 1191 sg++; 1192 qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf)); 1193 sg->length = mbuf->data_len; 1194 mbuf = mbuf->next; 1195 } 1196 sg->length -= ses->digest_length; 1197 1198 if (is_encode(ses)) { 1199 cpu_to_hw_sg(sg); 1200 /* set auth output */ 1201 sg++; 1202 qm_sg_entry_set64(sg, sym->auth.digest.phys_addr); 1203 sg->length = ses->digest_length; 1204 } 1205 sg->final = 1; 1206 cpu_to_hw_sg(sg); 1207 1208 /* input */ 1209 mbuf = sym->m_src; 1210 in_sg = &cf->sg[1]; 1211 in_sg->extension = 1; 1212 in_sg->final = 1; 1213 if (is_encode(ses)) 1214 in_sg->length = ses->iv.length + sym->auth.data.length; 1215 else 1216 in_sg->length = ses->iv.length + sym->auth.data.length 1217 + ses->digest_length; 1218 1219 /* input sg entries */ 1220 sg++; 1221 qm_sg_entry_set64(in_sg, dpaa_mem_vtop_ctx(ctx, sg)); 1222 cpu_to_hw_sg(in_sg); 1223 1224 /* 1st seg IV */ 1225 qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr)); 1226 sg->length = ses->iv.length; 1227 cpu_to_hw_sg(sg); 1228 1229 /* 2nd seg */ 1230 sg++; 1231 qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf)); 1232 sg->length = mbuf->data_len - sym->auth.data.offset; 1233 sg->offset = sym->auth.data.offset; 1234 1235 /* Successive segs */ 1236 mbuf = mbuf->next; 1237 while (mbuf) { 1238 cpu_to_hw_sg(sg); 1239 sg++; 1240 qm_sg_entry_set64(sg, rte_pktmbuf_mtophys(mbuf)); 1241 sg->length = mbuf->data_len; 1242 mbuf = mbuf->next; 1243 } 1244 1245 sg->length -= ses->digest_length; 1246 if (is_decode(ses)) { 1247 cpu_to_hw_sg(sg); 1248 sg++; 1249 memcpy(ctx->digest, sym->auth.digest.data, 1250 ses->digest_length); 1251 qm_sg_entry_set64(sg, dpaa_mem_vtop_ctx(ctx, ctx->digest)); 1252 sg->length = ses->digest_length; 1253 } 1254 sg->final = 1; 1255 cpu_to_hw_sg(sg); 1256 1257 return cf; 1258 } 1259 1260 static inline struct dpaa_sec_job * 1261 build_cipher_auth(struct rte_crypto_op *op, dpaa_sec_session *ses) 1262 { 1263 struct rte_crypto_sym_op *sym = op->sym; 1264 struct dpaa_sec_job *cf; 1265 struct dpaa_sec_op_ctx *ctx; 1266 struct qm_sg_entry *sg; 1267 rte_iova_t src_start_addr, dst_start_addr; 1268 uint32_t length = 0; 1269 uint8_t *IV_ptr = rte_crypto_op_ctod_offset(op, uint8_t *, 1270 ses->iv.offset); 1271 1272 src_start_addr = sym->m_src->buf_iova + sym->m_src->data_off; 1273 if (sym->m_dst) 1274 dst_start_addr = sym->m_dst->buf_iova + sym->m_dst->data_off; 1275 else 1276 dst_start_addr = src_start_addr; 1277 1278 ctx = dpaa_sec_alloc_ctx(ses); 1279 if (!ctx) 1280 return NULL; 1281 1282 cf = &ctx->job; 1283 ctx->op = op; 1284 1285 /* input */ 1286 rte_prefetch0(cf->sg); 1287 sg = &cf->sg[2]; 1288 qm_sg_entry_set64(&cf->sg[1], dpaa_mem_vtop_ctx(ctx, sg)); 1289 if (is_encode(ses)) { 1290 qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr)); 1291 sg->length = ses->iv.length; 1292 length += sg->length; 1293 cpu_to_hw_sg(sg); 1294 1295 sg++; 1296 qm_sg_entry_set64(sg, src_start_addr + sym->auth.data.offset); 1297 sg->length = sym->auth.data.length; 1298 length += sg->length; 1299 sg->final = 1; 1300 cpu_to_hw_sg(sg); 1301 } else { 1302 qm_sg_entry_set64(sg, dpaa_mem_vtop(IV_ptr)); 1303 sg->length = ses->iv.length; 1304 length += sg->length; 1305 cpu_to_hw_sg(sg); 1306 1307 sg++; 1308 1309 qm_sg_entry_set64(sg, src_start_addr + sym->auth.data.offset); 1310 sg->length = sym->auth.data.length; 1311 length += sg->length; 1312 cpu_to_hw_sg(sg); 1313 1314 memcpy(ctx->digest, sym->auth.digest.data, 1315 ses->digest_length); 1316 sg++; 1317 1318 qm_sg_entry_set64(sg, dpaa_mem_vtop_ctx(ctx, ctx->digest)); 1319 sg->length = ses->digest_length; 1320 length += sg->length; 1321 sg->final = 1; 1322 cpu_to_hw_sg(sg); 1323 } 1324 /* input compound frame */ 1325 cf->sg[1].length = length; 1326 cf->sg[1].extension = 1; 1327 cf->sg[1].final = 1; 1328 cpu_to_hw_sg(&cf->sg[1]); 1329 1330 /* output */ 1331 sg++; 1332 qm_sg_entry_set64(&cf->sg[0], dpaa_mem_vtop_ctx(ctx, sg)); 1333 qm_sg_entry_set64(sg, dst_start_addr + sym->cipher.data.offset); 1334 sg->length = sym->cipher.data.length; 1335 length = sg->length; 1336 if (is_encode(ses)) { 1337 cpu_to_hw_sg(sg); 1338 /* set auth output */ 1339 sg++; 1340 qm_sg_entry_set64(sg, sym->auth.digest.phys_addr); 1341 sg->length = ses->digest_length; 1342 length += sg->length; 1343 } 1344 sg->final = 1; 1345 cpu_to_hw_sg(sg); 1346 1347 /* output compound frame */ 1348 cf->sg[0].length = length; 1349 cf->sg[0].extension = 1; 1350 cpu_to_hw_sg(&cf->sg[0]); 1351 1352 return cf; 1353 } 1354 1355 static inline struct dpaa_sec_job * 1356 build_proto(struct rte_crypto_op *op, dpaa_sec_session *ses) 1357 { 1358 struct rte_crypto_sym_op *sym = op->sym; 1359 struct dpaa_sec_job *cf; 1360 struct dpaa_sec_op_ctx *ctx; 1361 struct qm_sg_entry *sg; 1362 phys_addr_t src_start_addr, dst_start_addr; 1363 1364 ctx = dpaa_sec_alloc_ctx(ses); 1365 if (!ctx) 1366 return NULL; 1367 cf = &ctx->job; 1368 ctx->op = op; 1369 1370 src_start_addr = rte_pktmbuf_mtophys(sym->m_src); 1371 1372 if (sym->m_dst) 1373 dst_start_addr = rte_pktmbuf_mtophys(sym->m_dst); 1374 else 1375 dst_start_addr = src_start_addr; 1376 1377 /* input */ 1378 sg = &cf->sg[1]; 1379 qm_sg_entry_set64(sg, src_start_addr); 1380 sg->length = sym->m_src->pkt_len; 1381 sg->final = 1; 1382 cpu_to_hw_sg(sg); 1383 1384 sym->m_src->packet_type &= ~RTE_PTYPE_L4_MASK; 1385 /* output */ 1386 sg = &cf->sg[0]; 1387 qm_sg_entry_set64(sg, dst_start_addr); 1388 sg->length = sym->m_src->buf_len - sym->m_src->data_off; 1389 cpu_to_hw_sg(sg); 1390 1391 return cf; 1392 } 1393 1394 static uint16_t 1395 dpaa_sec_enqueue_burst(void *qp, struct rte_crypto_op **ops, 1396 uint16_t nb_ops) 1397 { 1398 /* Function to transmit the frames to given device and queuepair */ 1399 uint32_t loop; 1400 struct dpaa_sec_qp *dpaa_qp = (struct dpaa_sec_qp *)qp; 1401 uint16_t num_tx = 0; 1402 struct qm_fd fds[DPAA_SEC_BURST], *fd; 1403 uint32_t frames_to_send; 1404 struct rte_crypto_op *op; 1405 struct dpaa_sec_job *cf; 1406 dpaa_sec_session *ses; 1407 struct dpaa_sec_op_ctx *ctx; 1408 uint32_t auth_only_len; 1409 struct qman_fq *inq[DPAA_SEC_BURST]; 1410 1411 while (nb_ops) { 1412 frames_to_send = (nb_ops > DPAA_SEC_BURST) ? 1413 DPAA_SEC_BURST : nb_ops; 1414 for (loop = 0; loop < frames_to_send; loop++) { 1415 op = *(ops++); 1416 switch (op->sess_type) { 1417 case RTE_CRYPTO_OP_WITH_SESSION: 1418 ses = (dpaa_sec_session *) 1419 get_session_private_data( 1420 op->sym->session, 1421 cryptodev_driver_id); 1422 break; 1423 case RTE_CRYPTO_OP_SECURITY_SESSION: 1424 ses = (dpaa_sec_session *) 1425 get_sec_session_private_data( 1426 op->sym->sec_session); 1427 break; 1428 default: 1429 PMD_TX_LOG(ERR, 1430 "sessionless crypto op not supported"); 1431 frames_to_send = loop; 1432 nb_ops = loop; 1433 goto send_pkts; 1434 } 1435 if (unlikely(!ses->qp || ses->qp != qp)) { 1436 PMD_INIT_LOG(DEBUG, "sess->qp - %p qp %p", 1437 ses->qp, qp); 1438 if (dpaa_sec_attach_sess_q(qp, ses)) { 1439 frames_to_send = loop; 1440 nb_ops = loop; 1441 goto send_pkts; 1442 } 1443 } 1444 1445 auth_only_len = op->sym->auth.data.length - 1446 op->sym->cipher.data.length; 1447 if (rte_pktmbuf_is_contiguous(op->sym->m_src)) { 1448 if (is_auth_only(ses)) { 1449 cf = build_auth_only(op, ses); 1450 } else if (is_cipher_only(ses)) { 1451 cf = build_cipher_only(op, ses); 1452 } else if (is_aead(ses)) { 1453 cf = build_cipher_auth_gcm(op, ses); 1454 auth_only_len = ses->auth_only_len; 1455 } else if (is_auth_cipher(ses)) { 1456 cf = build_cipher_auth(op, ses); 1457 } else if (is_proto_ipsec(ses)) { 1458 cf = build_proto(op, ses); 1459 } else { 1460 PMD_TX_LOG(ERR, "not supported sec op"); 1461 frames_to_send = loop; 1462 nb_ops = loop; 1463 goto send_pkts; 1464 } 1465 } else { 1466 if (is_auth_only(ses)) { 1467 cf = build_auth_only_sg(op, ses); 1468 } else if (is_cipher_only(ses)) { 1469 cf = build_cipher_only_sg(op, ses); 1470 } else if (is_aead(ses)) { 1471 cf = build_cipher_auth_gcm_sg(op, ses); 1472 auth_only_len = ses->auth_only_len; 1473 } else if (is_auth_cipher(ses)) { 1474 cf = build_cipher_auth_sg(op, ses); 1475 } else { 1476 PMD_TX_LOG(ERR, "not supported sec op"); 1477 frames_to_send = loop; 1478 nb_ops = loop; 1479 goto send_pkts; 1480 } 1481 } 1482 if (unlikely(!cf)) { 1483 frames_to_send = loop; 1484 nb_ops = loop; 1485 goto send_pkts; 1486 } 1487 1488 fd = &fds[loop]; 1489 inq[loop] = ses->inq; 1490 fd->opaque_addr = 0; 1491 fd->cmd = 0; 1492 ctx = container_of(cf, struct dpaa_sec_op_ctx, job); 1493 qm_fd_addr_set64(fd, dpaa_mem_vtop_ctx(ctx, cf->sg)); 1494 fd->_format1 = qm_fd_compound; 1495 fd->length29 = 2 * sizeof(struct qm_sg_entry); 1496 /* Auth_only_len is set as 0 in descriptor and it is 1497 * overwritten here in the fd.cmd which will update 1498 * the DPOVRD reg. 1499 */ 1500 if (auth_only_len) 1501 fd->cmd = 0x80000000 | auth_only_len; 1502 1503 } 1504 send_pkts: 1505 loop = 0; 1506 while (loop < frames_to_send) { 1507 loop += qman_enqueue_multi_fq(&inq[loop], &fds[loop], 1508 frames_to_send - loop); 1509 } 1510 nb_ops -= frames_to_send; 1511 num_tx += frames_to_send; 1512 } 1513 1514 dpaa_qp->tx_pkts += num_tx; 1515 dpaa_qp->tx_errs += nb_ops - num_tx; 1516 1517 return num_tx; 1518 } 1519 1520 static uint16_t 1521 dpaa_sec_dequeue_burst(void *qp, struct rte_crypto_op **ops, 1522 uint16_t nb_ops) 1523 { 1524 uint16_t num_rx; 1525 struct dpaa_sec_qp *dpaa_qp = (struct dpaa_sec_qp *)qp; 1526 1527 num_rx = dpaa_sec_deq(dpaa_qp, ops, nb_ops); 1528 1529 dpaa_qp->rx_pkts += num_rx; 1530 dpaa_qp->rx_errs += nb_ops - num_rx; 1531 1532 PMD_RX_LOG(DEBUG, "SEC Received %d Packets\n", num_rx); 1533 1534 return num_rx; 1535 } 1536 1537 /** Release queue pair */ 1538 static int 1539 dpaa_sec_queue_pair_release(struct rte_cryptodev *dev, 1540 uint16_t qp_id) 1541 { 1542 struct dpaa_sec_dev_private *internals; 1543 struct dpaa_sec_qp *qp = NULL; 1544 1545 PMD_INIT_FUNC_TRACE(); 1546 1547 PMD_INIT_LOG(DEBUG, "dev =%p, queue =%d", dev, qp_id); 1548 1549 internals = dev->data->dev_private; 1550 if (qp_id >= internals->max_nb_queue_pairs) { 1551 PMD_INIT_LOG(ERR, "Max supported qpid %d", 1552 internals->max_nb_queue_pairs); 1553 return -EINVAL; 1554 } 1555 1556 qp = &internals->qps[qp_id]; 1557 qp->internals = NULL; 1558 dev->data->queue_pairs[qp_id] = NULL; 1559 1560 return 0; 1561 } 1562 1563 /** Setup a queue pair */ 1564 static int 1565 dpaa_sec_queue_pair_setup(struct rte_cryptodev *dev, uint16_t qp_id, 1566 __rte_unused const struct rte_cryptodev_qp_conf *qp_conf, 1567 __rte_unused int socket_id, 1568 __rte_unused struct rte_mempool *session_pool) 1569 { 1570 struct dpaa_sec_dev_private *internals; 1571 struct dpaa_sec_qp *qp = NULL; 1572 1573 PMD_INIT_LOG(DEBUG, "dev =%p, queue =%d, conf =%p", 1574 dev, qp_id, qp_conf); 1575 1576 internals = dev->data->dev_private; 1577 if (qp_id >= internals->max_nb_queue_pairs) { 1578 PMD_INIT_LOG(ERR, "Max supported qpid %d", 1579 internals->max_nb_queue_pairs); 1580 return -EINVAL; 1581 } 1582 1583 qp = &internals->qps[qp_id]; 1584 qp->internals = internals; 1585 dev->data->queue_pairs[qp_id] = qp; 1586 1587 return 0; 1588 } 1589 1590 /** Start queue pair */ 1591 static int 1592 dpaa_sec_queue_pair_start(__rte_unused struct rte_cryptodev *dev, 1593 __rte_unused uint16_t queue_pair_id) 1594 { 1595 PMD_INIT_FUNC_TRACE(); 1596 1597 return 0; 1598 } 1599 1600 /** Stop queue pair */ 1601 static int 1602 dpaa_sec_queue_pair_stop(__rte_unused struct rte_cryptodev *dev, 1603 __rte_unused uint16_t queue_pair_id) 1604 { 1605 PMD_INIT_FUNC_TRACE(); 1606 1607 return 0; 1608 } 1609 1610 /** Return the number of allocated queue pairs */ 1611 static uint32_t 1612 dpaa_sec_queue_pair_count(struct rte_cryptodev *dev) 1613 { 1614 PMD_INIT_FUNC_TRACE(); 1615 1616 return dev->data->nb_queue_pairs; 1617 } 1618 1619 /** Returns the size of session structure */ 1620 static unsigned int 1621 dpaa_sec_session_get_size(struct rte_cryptodev *dev __rte_unused) 1622 { 1623 PMD_INIT_FUNC_TRACE(); 1624 1625 return sizeof(dpaa_sec_session); 1626 } 1627 1628 static int 1629 dpaa_sec_cipher_init(struct rte_cryptodev *dev __rte_unused, 1630 struct rte_crypto_sym_xform *xform, 1631 dpaa_sec_session *session) 1632 { 1633 session->cipher_alg = xform->cipher.algo; 1634 session->iv.length = xform->cipher.iv.length; 1635 session->iv.offset = xform->cipher.iv.offset; 1636 session->cipher_key.data = rte_zmalloc(NULL, xform->cipher.key.length, 1637 RTE_CACHE_LINE_SIZE); 1638 if (session->cipher_key.data == NULL && xform->cipher.key.length > 0) { 1639 PMD_INIT_LOG(ERR, "No Memory for cipher key\n"); 1640 return -ENOMEM; 1641 } 1642 session->cipher_key.length = xform->cipher.key.length; 1643 1644 memcpy(session->cipher_key.data, xform->cipher.key.data, 1645 xform->cipher.key.length); 1646 session->dir = (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) ? 1647 DIR_ENC : DIR_DEC; 1648 1649 return 0; 1650 } 1651 1652 static int 1653 dpaa_sec_auth_init(struct rte_cryptodev *dev __rte_unused, 1654 struct rte_crypto_sym_xform *xform, 1655 dpaa_sec_session *session) 1656 { 1657 session->auth_alg = xform->auth.algo; 1658 session->auth_key.data = rte_zmalloc(NULL, xform->auth.key.length, 1659 RTE_CACHE_LINE_SIZE); 1660 if (session->auth_key.data == NULL && xform->auth.key.length > 0) { 1661 PMD_INIT_LOG(ERR, "No Memory for auth key\n"); 1662 return -ENOMEM; 1663 } 1664 session->auth_key.length = xform->auth.key.length; 1665 session->digest_length = xform->auth.digest_length; 1666 1667 memcpy(session->auth_key.data, xform->auth.key.data, 1668 xform->auth.key.length); 1669 session->dir = (xform->auth.op == RTE_CRYPTO_AUTH_OP_GENERATE) ? 1670 DIR_ENC : DIR_DEC; 1671 1672 return 0; 1673 } 1674 1675 static int 1676 dpaa_sec_aead_init(struct rte_cryptodev *dev __rte_unused, 1677 struct rte_crypto_sym_xform *xform, 1678 dpaa_sec_session *session) 1679 { 1680 session->aead_alg = xform->aead.algo; 1681 session->iv.length = xform->aead.iv.length; 1682 session->iv.offset = xform->aead.iv.offset; 1683 session->auth_only_len = xform->aead.aad_length; 1684 session->aead_key.data = rte_zmalloc(NULL, xform->aead.key.length, 1685 RTE_CACHE_LINE_SIZE); 1686 if (session->aead_key.data == NULL && xform->aead.key.length > 0) { 1687 PMD_INIT_LOG(ERR, "No Memory for aead key\n"); 1688 return -ENOMEM; 1689 } 1690 session->aead_key.length = xform->aead.key.length; 1691 session->digest_length = xform->aead.digest_length; 1692 1693 memcpy(session->aead_key.data, xform->aead.key.data, 1694 xform->aead.key.length); 1695 session->dir = (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT) ? 1696 DIR_ENC : DIR_DEC; 1697 1698 return 0; 1699 } 1700 1701 static struct qman_fq * 1702 dpaa_sec_attach_rxq(struct dpaa_sec_dev_private *qi) 1703 { 1704 unsigned int i; 1705 1706 for (i = 0; i < qi->max_nb_sessions; i++) { 1707 if (qi->inq_attach[i] == 0) { 1708 qi->inq_attach[i] = 1; 1709 return &qi->inq[i]; 1710 } 1711 } 1712 PMD_DRV_LOG(ERR, "All ses session in use %x", qi->max_nb_sessions); 1713 1714 return NULL; 1715 } 1716 1717 static int 1718 dpaa_sec_detach_rxq(struct dpaa_sec_dev_private *qi, struct qman_fq *fq) 1719 { 1720 unsigned int i; 1721 1722 for (i = 0; i < qi->max_nb_sessions; i++) { 1723 if (&qi->inq[i] == fq) { 1724 qman_retire_fq(fq, NULL); 1725 qman_oos_fq(fq); 1726 qi->inq_attach[i] = 0; 1727 return 0; 1728 } 1729 } 1730 return -1; 1731 } 1732 1733 static int 1734 dpaa_sec_attach_sess_q(struct dpaa_sec_qp *qp, dpaa_sec_session *sess) 1735 { 1736 int ret; 1737 1738 sess->qp = qp; 1739 ret = dpaa_sec_prep_cdb(sess); 1740 if (ret) { 1741 PMD_DRV_LOG(ERR, "Unable to prepare sec cdb"); 1742 return -1; 1743 } 1744 1745 ret = dpaa_sec_init_rx(sess->inq, dpaa_mem_vtop(&sess->cdb), 1746 qman_fq_fqid(&qp->outq)); 1747 if (ret) 1748 PMD_DRV_LOG(ERR, "Unable to init sec queue"); 1749 1750 return ret; 1751 } 1752 1753 static int 1754 dpaa_sec_qp_attach_sess(struct rte_cryptodev *dev __rte_unused, 1755 uint16_t qp_id __rte_unused, 1756 void *ses __rte_unused) 1757 { 1758 PMD_INIT_FUNC_TRACE(); 1759 return 0; 1760 } 1761 1762 static int 1763 dpaa_sec_qp_detach_sess(struct rte_cryptodev *dev, 1764 uint16_t qp_id __rte_unused, 1765 void *ses) 1766 { 1767 dpaa_sec_session *sess = ses; 1768 struct dpaa_sec_dev_private *qi = dev->data->dev_private; 1769 1770 PMD_INIT_FUNC_TRACE(); 1771 1772 if (sess->inq) 1773 dpaa_sec_detach_rxq(qi, sess->inq); 1774 sess->inq = NULL; 1775 1776 sess->qp = NULL; 1777 1778 return 0; 1779 } 1780 1781 static int 1782 dpaa_sec_set_session_parameters(struct rte_cryptodev *dev, 1783 struct rte_crypto_sym_xform *xform, void *sess) 1784 { 1785 struct dpaa_sec_dev_private *internals = dev->data->dev_private; 1786 dpaa_sec_session *session = sess; 1787 1788 PMD_INIT_FUNC_TRACE(); 1789 1790 if (unlikely(sess == NULL)) { 1791 RTE_LOG(ERR, PMD, "invalid session struct\n"); 1792 return -EINVAL; 1793 } 1794 1795 /* Default IV length = 0 */ 1796 session->iv.length = 0; 1797 1798 /* Cipher Only */ 1799 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER && xform->next == NULL) { 1800 session->auth_alg = RTE_CRYPTO_AUTH_NULL; 1801 dpaa_sec_cipher_init(dev, xform, session); 1802 1803 /* Authentication Only */ 1804 } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH && 1805 xform->next == NULL) { 1806 session->cipher_alg = RTE_CRYPTO_CIPHER_NULL; 1807 dpaa_sec_auth_init(dev, xform, session); 1808 1809 /* Cipher then Authenticate */ 1810 } else if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER && 1811 xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) { 1812 if (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 1813 dpaa_sec_cipher_init(dev, xform, session); 1814 dpaa_sec_auth_init(dev, xform->next, session); 1815 } else { 1816 PMD_DRV_LOG(ERR, "Not supported: Auth then Cipher"); 1817 return -EINVAL; 1818 } 1819 1820 /* Authenticate then Cipher */ 1821 } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH && 1822 xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER) { 1823 if (xform->next->cipher.op == RTE_CRYPTO_CIPHER_OP_DECRYPT) { 1824 dpaa_sec_auth_init(dev, xform, session); 1825 dpaa_sec_cipher_init(dev, xform->next, session); 1826 } else { 1827 PMD_DRV_LOG(ERR, "Not supported: Auth then Cipher"); 1828 return -EINVAL; 1829 } 1830 1831 /* AEAD operation for AES-GCM kind of Algorithms */ 1832 } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD && 1833 xform->next == NULL) { 1834 dpaa_sec_aead_init(dev, xform, session); 1835 1836 } else { 1837 PMD_DRV_LOG(ERR, "Invalid crypto type"); 1838 return -EINVAL; 1839 } 1840 session->ctx_pool = internals->ctx_pool; 1841 session->inq = dpaa_sec_attach_rxq(internals); 1842 if (session->inq == NULL) { 1843 PMD_DRV_LOG(ERR, "unable to attach sec queue"); 1844 goto err1; 1845 } 1846 1847 return 0; 1848 1849 err1: 1850 rte_free(session->cipher_key.data); 1851 rte_free(session->auth_key.data); 1852 memset(session, 0, sizeof(dpaa_sec_session)); 1853 1854 return -EINVAL; 1855 } 1856 1857 static int 1858 dpaa_sec_session_configure(struct rte_cryptodev *dev, 1859 struct rte_crypto_sym_xform *xform, 1860 struct rte_cryptodev_sym_session *sess, 1861 struct rte_mempool *mempool) 1862 { 1863 void *sess_private_data; 1864 int ret; 1865 1866 PMD_INIT_FUNC_TRACE(); 1867 1868 if (rte_mempool_get(mempool, &sess_private_data)) { 1869 CDEV_LOG_ERR( 1870 "Couldn't get object from session mempool"); 1871 return -ENOMEM; 1872 } 1873 1874 ret = dpaa_sec_set_session_parameters(dev, xform, sess_private_data); 1875 if (ret != 0) { 1876 PMD_DRV_LOG(ERR, "DPAA PMD: failed to configure " 1877 "session parameters"); 1878 1879 /* Return session to mempool */ 1880 rte_mempool_put(mempool, sess_private_data); 1881 return ret; 1882 } 1883 1884 set_session_private_data(sess, dev->driver_id, 1885 sess_private_data); 1886 1887 1888 return 0; 1889 } 1890 1891 /** Clear the memory of session so it doesn't leave key material behind */ 1892 static void 1893 dpaa_sec_session_clear(struct rte_cryptodev *dev, 1894 struct rte_cryptodev_sym_session *sess) 1895 { 1896 struct dpaa_sec_dev_private *qi = dev->data->dev_private; 1897 uint8_t index = dev->driver_id; 1898 void *sess_priv = get_session_private_data(sess, index); 1899 1900 PMD_INIT_FUNC_TRACE(); 1901 1902 dpaa_sec_session *s = (dpaa_sec_session *)sess_priv; 1903 1904 if (sess_priv) { 1905 struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv); 1906 1907 if (s->inq) 1908 dpaa_sec_detach_rxq(qi, s->inq); 1909 rte_free(s->cipher_key.data); 1910 rte_free(s->auth_key.data); 1911 memset(s, 0, sizeof(dpaa_sec_session)); 1912 set_session_private_data(sess, index, NULL); 1913 rte_mempool_put(sess_mp, sess_priv); 1914 } 1915 } 1916 1917 static int 1918 dpaa_sec_set_ipsec_session(__rte_unused struct rte_cryptodev *dev, 1919 struct rte_security_session_conf *conf, 1920 void *sess) 1921 { 1922 struct dpaa_sec_dev_private *internals = dev->data->dev_private; 1923 struct rte_security_ipsec_xform *ipsec_xform = &conf->ipsec; 1924 struct rte_crypto_auth_xform *auth_xform; 1925 struct rte_crypto_cipher_xform *cipher_xform; 1926 dpaa_sec_session *session = (dpaa_sec_session *)sess; 1927 1928 PMD_INIT_FUNC_TRACE(); 1929 1930 if (ipsec_xform->direction == RTE_SECURITY_IPSEC_SA_DIR_EGRESS) { 1931 cipher_xform = &conf->crypto_xform->cipher; 1932 auth_xform = &conf->crypto_xform->next->auth; 1933 } else { 1934 auth_xform = &conf->crypto_xform->auth; 1935 cipher_xform = &conf->crypto_xform->next->cipher; 1936 } 1937 session->proto_alg = conf->protocol; 1938 session->cipher_key.data = rte_zmalloc(NULL, 1939 cipher_xform->key.length, 1940 RTE_CACHE_LINE_SIZE); 1941 if (session->cipher_key.data == NULL && 1942 cipher_xform->key.length > 0) { 1943 RTE_LOG(ERR, PMD, "No Memory for cipher key\n"); 1944 return -ENOMEM; 1945 } 1946 1947 session->cipher_key.length = cipher_xform->key.length; 1948 session->auth_key.data = rte_zmalloc(NULL, 1949 auth_xform->key.length, 1950 RTE_CACHE_LINE_SIZE); 1951 if (session->auth_key.data == NULL && 1952 auth_xform->key.length > 0) { 1953 RTE_LOG(ERR, PMD, "No Memory for auth key\n"); 1954 rte_free(session->cipher_key.data); 1955 return -ENOMEM; 1956 } 1957 session->auth_key.length = auth_xform->key.length; 1958 memcpy(session->cipher_key.data, cipher_xform->key.data, 1959 cipher_xform->key.length); 1960 memcpy(session->auth_key.data, auth_xform->key.data, 1961 auth_xform->key.length); 1962 1963 switch (auth_xform->algo) { 1964 case RTE_CRYPTO_AUTH_SHA1_HMAC: 1965 session->auth_alg = RTE_CRYPTO_AUTH_SHA1_HMAC; 1966 break; 1967 case RTE_CRYPTO_AUTH_MD5_HMAC: 1968 session->auth_alg = RTE_CRYPTO_AUTH_MD5_HMAC; 1969 break; 1970 case RTE_CRYPTO_AUTH_SHA256_HMAC: 1971 session->auth_alg = RTE_CRYPTO_AUTH_SHA256_HMAC; 1972 break; 1973 case RTE_CRYPTO_AUTH_SHA384_HMAC: 1974 session->auth_alg = RTE_CRYPTO_AUTH_SHA384_HMAC; 1975 break; 1976 case RTE_CRYPTO_AUTH_SHA512_HMAC: 1977 session->auth_alg = RTE_CRYPTO_AUTH_SHA512_HMAC; 1978 break; 1979 case RTE_CRYPTO_AUTH_AES_CMAC: 1980 session->auth_alg = RTE_CRYPTO_AUTH_AES_CMAC; 1981 break; 1982 case RTE_CRYPTO_AUTH_NULL: 1983 session->auth_alg = RTE_CRYPTO_AUTH_NULL; 1984 break; 1985 case RTE_CRYPTO_AUTH_SHA224_HMAC: 1986 case RTE_CRYPTO_AUTH_AES_XCBC_MAC: 1987 case RTE_CRYPTO_AUTH_SNOW3G_UIA2: 1988 case RTE_CRYPTO_AUTH_SHA1: 1989 case RTE_CRYPTO_AUTH_SHA256: 1990 case RTE_CRYPTO_AUTH_SHA512: 1991 case RTE_CRYPTO_AUTH_SHA224: 1992 case RTE_CRYPTO_AUTH_SHA384: 1993 case RTE_CRYPTO_AUTH_MD5: 1994 case RTE_CRYPTO_AUTH_AES_GMAC: 1995 case RTE_CRYPTO_AUTH_KASUMI_F9: 1996 case RTE_CRYPTO_AUTH_AES_CBC_MAC: 1997 case RTE_CRYPTO_AUTH_ZUC_EIA3: 1998 RTE_LOG(ERR, PMD, "Crypto: Unsupported auth alg %u\n", 1999 auth_xform->algo); 2000 goto out; 2001 default: 2002 RTE_LOG(ERR, PMD, "Crypto: Undefined Auth specified %u\n", 2003 auth_xform->algo); 2004 goto out; 2005 } 2006 2007 switch (cipher_xform->algo) { 2008 case RTE_CRYPTO_CIPHER_AES_CBC: 2009 session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CBC; 2010 break; 2011 case RTE_CRYPTO_CIPHER_3DES_CBC: 2012 session->cipher_alg = RTE_CRYPTO_CIPHER_3DES_CBC; 2013 break; 2014 case RTE_CRYPTO_CIPHER_AES_CTR: 2015 session->cipher_alg = RTE_CRYPTO_CIPHER_AES_CTR; 2016 break; 2017 case RTE_CRYPTO_CIPHER_NULL: 2018 case RTE_CRYPTO_CIPHER_SNOW3G_UEA2: 2019 case RTE_CRYPTO_CIPHER_3DES_ECB: 2020 case RTE_CRYPTO_CIPHER_AES_ECB: 2021 case RTE_CRYPTO_CIPHER_KASUMI_F8: 2022 RTE_LOG(ERR, PMD, "Crypto: Unsupported Cipher alg %u\n", 2023 cipher_xform->algo); 2024 goto out; 2025 default: 2026 RTE_LOG(ERR, PMD, "Crypto: Undefined Cipher specified %u\n", 2027 cipher_xform->algo); 2028 goto out; 2029 } 2030 2031 if (ipsec_xform->direction == RTE_SECURITY_IPSEC_SA_DIR_EGRESS) { 2032 memset(&session->encap_pdb, 0, sizeof(struct ipsec_encap_pdb) + 2033 sizeof(session->ip4_hdr)); 2034 session->ip4_hdr.ip_v = IPVERSION; 2035 session->ip4_hdr.ip_hl = 5; 2036 session->ip4_hdr.ip_len = rte_cpu_to_be_16( 2037 sizeof(session->ip4_hdr)); 2038 session->ip4_hdr.ip_tos = ipsec_xform->tunnel.ipv4.dscp; 2039 session->ip4_hdr.ip_id = 0; 2040 session->ip4_hdr.ip_off = 0; 2041 session->ip4_hdr.ip_ttl = ipsec_xform->tunnel.ipv4.ttl; 2042 session->ip4_hdr.ip_p = (ipsec_xform->proto == 2043 RTE_SECURITY_IPSEC_SA_PROTO_ESP) ? IPPROTO_ESP 2044 : IPPROTO_AH; 2045 session->ip4_hdr.ip_sum = 0; 2046 session->ip4_hdr.ip_src = ipsec_xform->tunnel.ipv4.src_ip; 2047 session->ip4_hdr.ip_dst = ipsec_xform->tunnel.ipv4.dst_ip; 2048 session->ip4_hdr.ip_sum = calc_chksum((uint16_t *) 2049 (void *)&session->ip4_hdr, 2050 sizeof(struct ip)); 2051 2052 session->encap_pdb.options = 2053 (IPVERSION << PDBNH_ESP_ENCAP_SHIFT) | 2054 PDBOPTS_ESP_OIHI_PDB_INL | 2055 PDBOPTS_ESP_IVSRC | 2056 PDBHMO_ESP_ENCAP_DTTL; 2057 session->encap_pdb.spi = ipsec_xform->spi; 2058 session->encap_pdb.ip_hdr_len = sizeof(struct ip); 2059 2060 session->dir = DIR_ENC; 2061 } else if (ipsec_xform->direction == 2062 RTE_SECURITY_IPSEC_SA_DIR_INGRESS) { 2063 memset(&session->decap_pdb, 0, sizeof(struct ipsec_decap_pdb)); 2064 session->decap_pdb.options = sizeof(struct ip) << 16; 2065 session->dir = DIR_DEC; 2066 } else 2067 goto out; 2068 session->ctx_pool = internals->ctx_pool; 2069 session->inq = dpaa_sec_attach_rxq(internals); 2070 if (session->inq == NULL) { 2071 PMD_DRV_LOG(ERR, "unable to attach sec queue"); 2072 goto out; 2073 } 2074 2075 2076 return 0; 2077 out: 2078 rte_free(session->auth_key.data); 2079 rte_free(session->cipher_key.data); 2080 memset(session, 0, sizeof(dpaa_sec_session)); 2081 return -1; 2082 } 2083 2084 static int 2085 dpaa_sec_security_session_create(void *dev, 2086 struct rte_security_session_conf *conf, 2087 struct rte_security_session *sess, 2088 struct rte_mempool *mempool) 2089 { 2090 void *sess_private_data; 2091 struct rte_cryptodev *cdev = (struct rte_cryptodev *)dev; 2092 int ret; 2093 2094 if (rte_mempool_get(mempool, &sess_private_data)) { 2095 CDEV_LOG_ERR( 2096 "Couldn't get object from session mempool"); 2097 return -ENOMEM; 2098 } 2099 2100 switch (conf->protocol) { 2101 case RTE_SECURITY_PROTOCOL_IPSEC: 2102 ret = dpaa_sec_set_ipsec_session(cdev, conf, 2103 sess_private_data); 2104 break; 2105 case RTE_SECURITY_PROTOCOL_MACSEC: 2106 return -ENOTSUP; 2107 default: 2108 return -EINVAL; 2109 } 2110 if (ret != 0) { 2111 PMD_DRV_LOG(ERR, 2112 "DPAA2 PMD: failed to configure session parameters"); 2113 2114 /* Return session to mempool */ 2115 rte_mempool_put(mempool, sess_private_data); 2116 return ret; 2117 } 2118 2119 set_sec_session_private_data(sess, sess_private_data); 2120 2121 return ret; 2122 } 2123 2124 /** Clear the memory of session so it doesn't leave key material behind */ 2125 static int 2126 dpaa_sec_security_session_destroy(void *dev __rte_unused, 2127 struct rte_security_session *sess) 2128 { 2129 PMD_INIT_FUNC_TRACE(); 2130 void *sess_priv = get_sec_session_private_data(sess); 2131 2132 dpaa_sec_session *s = (dpaa_sec_session *)sess_priv; 2133 2134 if (sess_priv) { 2135 struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv); 2136 2137 rte_free(s->cipher_key.data); 2138 rte_free(s->auth_key.data); 2139 memset(sess, 0, sizeof(dpaa_sec_session)); 2140 set_sec_session_private_data(sess, NULL); 2141 rte_mempool_put(sess_mp, sess_priv); 2142 } 2143 return 0; 2144 } 2145 2146 2147 static int 2148 dpaa_sec_dev_configure(struct rte_cryptodev *dev __rte_unused, 2149 struct rte_cryptodev_config *config __rte_unused) 2150 { 2151 PMD_INIT_FUNC_TRACE(); 2152 2153 return 0; 2154 } 2155 2156 static int 2157 dpaa_sec_dev_start(struct rte_cryptodev *dev __rte_unused) 2158 { 2159 PMD_INIT_FUNC_TRACE(); 2160 return 0; 2161 } 2162 2163 static void 2164 dpaa_sec_dev_stop(struct rte_cryptodev *dev __rte_unused) 2165 { 2166 PMD_INIT_FUNC_TRACE(); 2167 } 2168 2169 static int 2170 dpaa_sec_dev_close(struct rte_cryptodev *dev __rte_unused) 2171 { 2172 PMD_INIT_FUNC_TRACE(); 2173 return 0; 2174 } 2175 2176 static void 2177 dpaa_sec_dev_infos_get(struct rte_cryptodev *dev, 2178 struct rte_cryptodev_info *info) 2179 { 2180 struct dpaa_sec_dev_private *internals = dev->data->dev_private; 2181 2182 PMD_INIT_FUNC_TRACE(); 2183 if (info != NULL) { 2184 info->max_nb_queue_pairs = internals->max_nb_queue_pairs; 2185 info->feature_flags = dev->feature_flags; 2186 info->capabilities = dpaa_sec_capabilities; 2187 info->sym.max_nb_sessions = internals->max_nb_sessions; 2188 info->sym.max_nb_sessions_per_qp = 2189 RTE_DPAA_SEC_PMD_MAX_NB_SESSIONS / 2190 RTE_DPAA_MAX_NB_SEC_QPS; 2191 info->driver_id = cryptodev_driver_id; 2192 } 2193 } 2194 2195 static struct rte_cryptodev_ops crypto_ops = { 2196 .dev_configure = dpaa_sec_dev_configure, 2197 .dev_start = dpaa_sec_dev_start, 2198 .dev_stop = dpaa_sec_dev_stop, 2199 .dev_close = dpaa_sec_dev_close, 2200 .dev_infos_get = dpaa_sec_dev_infos_get, 2201 .queue_pair_setup = dpaa_sec_queue_pair_setup, 2202 .queue_pair_release = dpaa_sec_queue_pair_release, 2203 .queue_pair_start = dpaa_sec_queue_pair_start, 2204 .queue_pair_stop = dpaa_sec_queue_pair_stop, 2205 .queue_pair_count = dpaa_sec_queue_pair_count, 2206 .session_get_size = dpaa_sec_session_get_size, 2207 .session_configure = dpaa_sec_session_configure, 2208 .session_clear = dpaa_sec_session_clear, 2209 .qp_attach_session = dpaa_sec_qp_attach_sess, 2210 .qp_detach_session = dpaa_sec_qp_detach_sess, 2211 }; 2212 2213 static const struct rte_security_capability * 2214 dpaa_sec_capabilities_get(void *device __rte_unused) 2215 { 2216 return dpaa_sec_security_cap; 2217 } 2218 2219 struct rte_security_ops dpaa_sec_security_ops = { 2220 .session_create = dpaa_sec_security_session_create, 2221 .session_update = NULL, 2222 .session_stats_get = NULL, 2223 .session_destroy = dpaa_sec_security_session_destroy, 2224 .set_pkt_metadata = NULL, 2225 .capabilities_get = dpaa_sec_capabilities_get 2226 }; 2227 2228 static int 2229 dpaa_sec_uninit(struct rte_cryptodev *dev) 2230 { 2231 struct dpaa_sec_dev_private *internals = dev->data->dev_private; 2232 2233 if (dev == NULL) 2234 return -ENODEV; 2235 2236 rte_free(dev->security_ctx); 2237 2238 rte_mempool_free(internals->ctx_pool); 2239 rte_free(internals); 2240 2241 PMD_INIT_LOG(INFO, "Closing DPAA_SEC device %s on numa socket %u\n", 2242 dev->data->name, rte_socket_id()); 2243 2244 return 0; 2245 } 2246 2247 static int 2248 dpaa_sec_dev_init(struct rte_cryptodev *cryptodev) 2249 { 2250 struct dpaa_sec_dev_private *internals; 2251 struct rte_security_ctx *security_instance; 2252 struct dpaa_sec_qp *qp; 2253 uint32_t i, flags; 2254 int ret; 2255 char str[20]; 2256 2257 PMD_INIT_FUNC_TRACE(); 2258 2259 cryptodev->driver_id = cryptodev_driver_id; 2260 cryptodev->dev_ops = &crypto_ops; 2261 2262 cryptodev->enqueue_burst = dpaa_sec_enqueue_burst; 2263 cryptodev->dequeue_burst = dpaa_sec_dequeue_burst; 2264 cryptodev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | 2265 RTE_CRYPTODEV_FF_HW_ACCELERATED | 2266 RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING | 2267 RTE_CRYPTODEV_FF_SECURITY | 2268 RTE_CRYPTODEV_FF_MBUF_SCATTER_GATHER; 2269 2270 internals = cryptodev->data->dev_private; 2271 internals->max_nb_queue_pairs = RTE_DPAA_MAX_NB_SEC_QPS; 2272 internals->max_nb_sessions = RTE_DPAA_SEC_PMD_MAX_NB_SESSIONS; 2273 2274 /* 2275 * For secondary processes, we don't initialise any further as primary 2276 * has already done this work. Only check we don't need a different 2277 * RX function 2278 */ 2279 if (rte_eal_process_type() != RTE_PROC_PRIMARY) { 2280 PMD_INIT_LOG(DEBUG, "Device already init by primary process"); 2281 return 0; 2282 } 2283 2284 /* Initialize security_ctx only for primary process*/ 2285 security_instance = rte_malloc("rte_security_instances_ops", 2286 sizeof(struct rte_security_ctx), 0); 2287 if (security_instance == NULL) 2288 return -ENOMEM; 2289 security_instance->device = (void *)cryptodev; 2290 security_instance->ops = &dpaa_sec_security_ops; 2291 security_instance->sess_cnt = 0; 2292 cryptodev->security_ctx = security_instance; 2293 2294 for (i = 0; i < internals->max_nb_queue_pairs; i++) { 2295 /* init qman fq for queue pair */ 2296 qp = &internals->qps[i]; 2297 ret = dpaa_sec_init_tx(&qp->outq); 2298 if (ret) { 2299 PMD_INIT_LOG(ERR, "config tx of queue pair %d", i); 2300 goto init_error; 2301 } 2302 } 2303 2304 flags = QMAN_FQ_FLAG_LOCKED | QMAN_FQ_FLAG_DYNAMIC_FQID | 2305 QMAN_FQ_FLAG_TO_DCPORTAL; 2306 for (i = 0; i < internals->max_nb_sessions; i++) { 2307 /* create rx qman fq for sessions*/ 2308 ret = qman_create_fq(0, flags, &internals->inq[i]); 2309 if (unlikely(ret != 0)) { 2310 PMD_INIT_LOG(ERR, "sec qman_create_fq failed"); 2311 goto init_error; 2312 } 2313 } 2314 2315 sprintf(str, "ctx_pool_%d", cryptodev->data->dev_id); 2316 internals->ctx_pool = rte_mempool_create((const char *)str, 2317 CTX_POOL_NUM_BUFS, 2318 CTX_POOL_BUF_SIZE, 2319 CTX_POOL_CACHE_SIZE, 0, 2320 NULL, NULL, NULL, NULL, 2321 SOCKET_ID_ANY, 0); 2322 if (!internals->ctx_pool) { 2323 RTE_LOG(ERR, PMD, "%s create failed\n", str); 2324 goto init_error; 2325 } 2326 2327 PMD_INIT_LOG(DEBUG, "driver %s: created\n", cryptodev->data->name); 2328 return 0; 2329 2330 init_error: 2331 PMD_INIT_LOG(ERR, "driver %s: create failed\n", cryptodev->data->name); 2332 2333 dpaa_sec_uninit(cryptodev); 2334 return -EFAULT; 2335 } 2336 2337 static int 2338 cryptodev_dpaa_sec_probe(struct rte_dpaa_driver *dpaa_drv, 2339 struct rte_dpaa_device *dpaa_dev) 2340 { 2341 struct rte_cryptodev *cryptodev; 2342 char cryptodev_name[RTE_CRYPTODEV_NAME_MAX_LEN]; 2343 2344 int retval; 2345 2346 sprintf(cryptodev_name, "dpaa_sec-%d", dpaa_dev->id.dev_id); 2347 2348 cryptodev = rte_cryptodev_pmd_allocate(cryptodev_name, rte_socket_id()); 2349 if (cryptodev == NULL) 2350 return -ENOMEM; 2351 2352 if (rte_eal_process_type() == RTE_PROC_PRIMARY) { 2353 cryptodev->data->dev_private = rte_zmalloc_socket( 2354 "cryptodev private structure", 2355 sizeof(struct dpaa_sec_dev_private), 2356 RTE_CACHE_LINE_SIZE, 2357 rte_socket_id()); 2358 2359 if (cryptodev->data->dev_private == NULL) 2360 rte_panic("Cannot allocate memzone for private " 2361 "device data"); 2362 } 2363 2364 dpaa_dev->crypto_dev = cryptodev; 2365 cryptodev->device = &dpaa_dev->device; 2366 cryptodev->device->driver = &dpaa_drv->driver; 2367 2368 /* init user callbacks */ 2369 TAILQ_INIT(&(cryptodev->link_intr_cbs)); 2370 2371 /* if sec device version is not configured */ 2372 if (!rta_get_sec_era()) { 2373 const struct device_node *caam_node; 2374 2375 for_each_compatible_node(caam_node, NULL, "fsl,sec-v4.0") { 2376 const uint32_t *prop = of_get_property(caam_node, 2377 "fsl,sec-era", 2378 NULL); 2379 if (prop) { 2380 rta_set_sec_era( 2381 INTL_SEC_ERA(rte_cpu_to_be_32(*prop))); 2382 break; 2383 } 2384 } 2385 } 2386 2387 /* Invoke PMD device initialization function */ 2388 retval = dpaa_sec_dev_init(cryptodev); 2389 if (retval == 0) 2390 return 0; 2391 2392 /* In case of error, cleanup is done */ 2393 if (rte_eal_process_type() == RTE_PROC_PRIMARY) 2394 rte_free(cryptodev->data->dev_private); 2395 2396 rte_cryptodev_pmd_release_device(cryptodev); 2397 2398 return -ENXIO; 2399 } 2400 2401 static int 2402 cryptodev_dpaa_sec_remove(struct rte_dpaa_device *dpaa_dev) 2403 { 2404 struct rte_cryptodev *cryptodev; 2405 int ret; 2406 2407 cryptodev = dpaa_dev->crypto_dev; 2408 if (cryptodev == NULL) 2409 return -ENODEV; 2410 2411 ret = dpaa_sec_uninit(cryptodev); 2412 if (ret) 2413 return ret; 2414 2415 return rte_cryptodev_pmd_destroy(cryptodev); 2416 } 2417 2418 static struct rte_dpaa_driver rte_dpaa_sec_driver = { 2419 .drv_type = FSL_DPAA_CRYPTO, 2420 .driver = { 2421 .name = "DPAA SEC PMD" 2422 }, 2423 .probe = cryptodev_dpaa_sec_probe, 2424 .remove = cryptodev_dpaa_sec_remove, 2425 }; 2426 2427 static struct cryptodev_driver dpaa_sec_crypto_drv; 2428 2429 RTE_PMD_REGISTER_DPAA(CRYPTODEV_NAME_DPAA_SEC_PMD, rte_dpaa_sec_driver); 2430 RTE_PMD_REGISTER_CRYPTO_DRIVER(dpaa_sec_crypto_drv, rte_dpaa_sec_driver.driver, 2431 cryptodev_driver_id); 2432